11

I have written some code stream video over a websocket so a sourcebuffer which works in Chrome and Edge.

However, when I run this in Firefox, the video never plays back, just a spinning wheel animation is displayed. When I check the <video> statistics, It reads HAVE_METADATA as the ready state and NETWORK_LOADING as the network state.

The code looks follows:

<!DOCTYPE html>
<html>
  <head>
    <meta charset="utf-8"/>
  </head>
  <body>
    <video controls></video>
    <script>
      var mime = 'video/mp4; codecs="avc1.4D401E,mp4a.40.2"';
      var address = 'ws://localhost:54132'

      /* Media Source */

      var source = new MediaSource();
      var video = document.querySelector('video');
      video.src = URL.createObjectURL(source);
      source.addEventListener('sourceopen', sourceOpen);

      /* Buffer */

      var buffer;
      var socket;
      var queue = [];
      var offset = -1;
      var timescale;

      // When the media source opens:
      function sourceOpen() {
        buffer = source.addSourceBuffer(mime);
        buffer.addEventListener('updateend', processQueue);

        socket = new WebSocket(address);
        socket.binaryType = 'arraybuffer';
        socket.onmessage = onMessage; 
      }

      // When more data is received.
      function onMessage(event) {
        queue.push(event.data);
        processQueue();
      }

      // Process queue if possible.
      function processQueue() {
        if ((queue.length == 0) || (buffer.updating)) {
          return;
        }

        var data = queue.shift();
        if (offset === -1) {
          var parsed = parseMP4(data);
          if (parsed.hasOwnProperty('moov')) {
            timescale = parsed.moov.mvhd.timescale;
          } else if (parsed.hasOwnProperty('moof')) {
            offset = 0 - (parsed.moof.traf[0].tfdt.baseMediaDecodeTime / this.timescale - 0.4);
            buffer.timestampOffset = offset;
          }
        }

        // console.log('appending ' + data.byteLength + ' bytes');
        buffer.appendBuffer(data);
      }

      // Parse out the offset.
      function parseMP4(data) {
        // SNIP for brevity
      }
    </script>
  </body>
</html>
Hans
  • 2,448
  • 2
  • 24
  • 30
  • 1
    do you get error event on video element or source buffer ? what happens when you directly use the video URL as "src" (without media source) – sbr Jun 30 '16 at 02:31
  • No error event. The data is generated on the fly by the server (from an IP stream for example), so there is no URL I can point the video element to. – Hans Jun 30 '16 at 16:19
  • Can you create a plnkr http://plnkr.co to demonstrate? – guest271314 Jul 01 '16 at 15:21
  • No, the video is fed from a custom c# server application. All the code (except for the bit that parses the MP4 container) is up there. – Hans Jul 01 '16 at 16:22
  • 1
    i don't see the part wheere you call `.endOfStream()` or `.play()`, so it won't start playing. – dandavis Jul 01 '16 at 17:43
  • _"No, the video is fed from a custom c# server application"_ You could use an alternative, open source, video source to demonstrate issue , see https://creativecommons.org/about/videos/ – guest271314 Jul 01 '16 at 18:00
  • @dandavis It is a continuous stream of video, so there is no end of stream. I am aware there is no play call. I am triggering the playback manually using the video element controls. – Hans Jul 01 '16 at 18:36
  • @guest271314, I don't think I understand how that will help me create a plunkr? That page lists media files which I know can be viewed in a – Hans Jul 01 '16 at 18:43
  • You can create a mock websocket stream response at plnkr to reproduce issue at firefox – guest271314 Jul 01 '16 at 18:45
  • What does `parseMP4` function return? Not certain how to possibly resolve issue if issue cannot be reproduced – guest271314 Jul 01 '16 at 18:52
  • Check this documentation: https://developer.mozilla.org/es/docs/Web/API/MediaSource – toto Jul 02 '16 at 05:57

2 Answers2

3

Could not reproduce <video> element not playing at firefox 47.

Merged approaches at Mocking Websocket Message Events to create mock WebSocket events; bufferAll.html demo at Let's Make a Netflix An Intro to Streaming Media on the Web for MediaSource usage pattern.

Included <progress> and progress event to notify user of media loading status.

<!DOCTYPE html>
<html>
  <head>
    <meta charset="utf-8"/>
  </head>
  <body>
    <progress min="0" value="0"></progress><br><label></label><br>
    <video controls></video>
    <script>
        // http://nickdesaulniers.github.io/netfix/demo/bufferAll.html
        // http://jsfiddle.net/adamboduch/JVfkt/
        // The global web socket.    
        var sock, sourceBuffer;
        sock = new WebSocket( "ws://mock" );
        sock.onerror = function(e) {
          console.log("sock error", e)
        }
        // This is unchanging production code that doesn"t know
        // we"re mocking the web socket.
        sock.onmessage = function( e ) {
          console.log("socket message", e.data);
          sourceBuffer.appendBuffer(e.data);
        }; 
      var video = document.querySelector("video");
      var progress = document.querySelector("progress");
      var label = document.querySelector("label");
      var assetURL = "http://nickdesaulniers.github.io/netfix/"
                     + "demo/frag_bunny.mp4";
      // Need to be specific for Blink regarding codecs
      // ./mp4info frag_bunny.mp4 | grep Codec
      var mimeCodec = 'video/mp4; codecs="avc1.42E01E, mp4a.40.2"';

      if ("MediaSource" in window 
          && MediaSource.isTypeSupported(mimeCodec)) {
        var mediaSource = new MediaSource;
        //console.log(mediaSource.readyState); // closed
        video.src = URL.createObjectURL(mediaSource);
        mediaSource.addEventListener("sourceopen", sourceOpen);
      } else {
        console.error("Unsupported MIME type or codec: ", mimeCodec);
      }
      video.addEventListener("canplay", function() {
        alert("video canplay")
      })
      function sourceOpen (_) {
        //console.log(this.readyState); // open
        var mediaSource = this;
        sourceBuffer = mediaSource.addSourceBuffer(mimeCodec);
        fetchAB(assetURL, function (buf) {
          sourceBuffer.addEventListener("updateend", function (event) {
          console.log("sourceBuffer updateend event;"
                      + "mediaSource.readyState:"
                     , mediaSource.readyState);
            // mediaSource.endOfStream();
            // video.play();
            // console.log(mediaSource.readyState); // ended
          });
          
        });
      };
      // mock `WebSocket` message
      function fetchAB (url, cb) {
        var xhr = new XMLHttpRequest;
        xhr.open("get", url);
        var file = url.split("/").pop();
        xhr.responseType = "arraybuffer";
        xhr.onload = function () {
          // mock `WebSocket` message
          sock.dispatchEvent( new MessageEvent( "message", {
            data: xhr.response
        }));
        console.log("video sent to sock", sock);
        cb();
        };
        xhr.onprogress = function(e) {
           progress.max = e.total;
           progress.value = e.loaded;
           label.innerHTML = "loading " + file + " ...<br>"
                             + e.loaded + " of " 
                             + e.total + " bytes loaded";
        }
        xhr.send();
      };  
    </script>
  </body>
  </html>

plnkr http://plnkr.co/edit/RCIqDXTB2BL3lec9bhfz

guest271314
  • 1
  • 15
  • 104
  • 177
  • 1
    After a minute on Firefox 47, I get a spinning circle and nothing happens after that. – ManoDestra Jul 07 '16 at 13:15
  • And doesn't work at all for me on IE or Chrome. Odd. – ManoDestra Jul 07 '16 at 13:26
  • 1
    @ManoDestra _"After a minute on Firefox 47, I get a spinning circle and nothing happens after that."_ Did you press play at video controls? Cannot reproduce that effect, here, at firefox 47. `alert()` is called; video plays for full `1:00` length. Are any errors logged at `console`? Do not believe chrome supports the specific codec at Question `var mime = 'video/mp4; codecs="avc1.4D401E,mp4a.40.2"';`; did you check `console` at plnkr at chrome? Not certain about ie. Note, actual Question is specifically addressing issue at firefox _"Unable to stream video over a websocket to Firefox"_ – guest271314 Jul 08 '16 at 02:15
  • 1
    On Firefox, it worked after pressing play for a minute, but got a spinner thereafter. Nothing after the first minute. On IE and Chrome, neither worked at all upon pressing play. So, this may well make sense here. – ManoDestra Jul 08 '16 at 02:28
  • 1
    @ManoDestra Yes, total video duration is one minute `1:00`. See also https://hacks.mozilla.org/2015/07/streaming-media-on-demand-with-media-source-extensions/ , https://w3c.github.io/media-source/ , https://gist.github.com/granoeste/8727308 – guest271314 Jul 08 '16 at 03:00
0
<!DOCTYPE html>
<html>
  <head>
    <meta charset="utf-8"/>
  </head>
  <body>
    <progress min="0" value="0"></progress><br><label></label><br>
    <video controls></video>
    <script>
        // http://nickdesaulniers.github.io/netfix/demo/bufferAll.html
        // http://jsfiddle.net/adamboduch/JVfkt/
        // The global web socket.    
        var sock, sourceBuffer;
        sock = new WebSocket( "ws://mock" );
        sock.onerror = function(e) {
          console.log("sock error", e)
        }
        // This is unchanging production code that doesn"t know
        // we"re mocking the web socket.
        sock.onmessage = function( e ) {
          console.log("socket message", e.data);
          sourceBuffer.appendBuffer(e.data);
        }; 
      var video = document.querySelector("video");
      var progress = document.querySelector("progress");
      var label = document.querySelector("label");
      var assetURL = "http://nickdesaulniers.github.io/netfix/"
                     + "demo/frag_bunny.mp4";
      // Need to be specific for Blink regarding codecs
      // ./mp4info frag_bunny.mp4 | grep Codec
      var mimeCodec = 'video/mp4; codecs="avc1.42E01E, mp4a.40.2"';

      if ("MediaSource" in window 
          && MediaSource.isTypeSupported(mimeCodec)) {
        var mediaSource = new MediaSource;
        //console.log(mediaSource.readyState); // closed
        video.src = URL.createObjectURL(mediaSource);
        mediaSource.addEventListener("sourceopen", sourceOpen);
      } else {
        console.error("Unsupported MIME type or codec: ", mimeCodec);
      }
      video.addEventListener("canplay", function() {
        alert("video canplay")
      })
      function sourceOpen (_) {
        //console.log(this.readyState); // open
        var mediaSource = this;
        sourceBuffer = mediaSource.addSourceBuffer(mimeCodec);
        fetchAB(assetURL, function (buf) {
          sourceBuffer.addEventListener("updateend", function (event) {
          console.log("sourceBuffer updateend event;"
                      + "mediaSource.readyState:"
                     , mediaSource.readyState);
            // mediaSource.endOfStream();
            // video.play();
            // console.log(mediaSource.readyState); // ended
          });

        });
      };
      // mock `WebSocket` message
      function fetchAB (url, cb) {
        var xhr = new XMLHttpRequest;
        xhr.open("get", url);
        var file = url.split("/").pop();
        xhr.responseType = "arraybuffer";
        xhr.onload = function () {
          // mock `WebSocket` message
          sock.dispatchEvent( new MessageEvent( "message", {
            data: xhr.response
        }));
        console.log("video sent to sock", sock);
        cb();
        };
        xhr.onprogress = function(e) {
           progress.max = e.total;
           progress.value = e.loaded;
           label.innerHTML = "loading " + file + " ...<br>"
                             + e.loaded + " of " 
                             + e.total + " bytes loaded";
        }
        xhr.send();
      };  
    </script>
  </body>
  </html>
John
  • 19
  • 2