feat: add fmp4 emsg ID3 support (#1370)

* feat: add fmp4 emsg ID3 support

* fix worker function

* add id3fn to finishLoading

* fix captions tests

* add media segment request test

src/media-segment-request.js

@@ -462,7 +462,7 @@ const handleSegmentBytes = ({
     // Note that the start time returned by the probe reflects the baseMediaDecodeTime, as
     // that is the true start of the segment (where the playback engine should begin
     // decoding).
-    const finishLoading = (captions) => {
+    const finishLoading = (captions, id3Frames) => {
       // if the track still has audio at this point it is only possible
       // for it to be audio only. See `tracks.video && tracks.audio` if statement
       // above.
@@ -471,6 +471,9 @@ const handleSegmentBytes = ({
         data: bytesAsUint8Array,
         type: trackInfo.hasAudio && !trackInfo.isMuxed ? 'audio' : 'video'
       });
+      if (id3Frames && id3Frames.length) {
+        id3Fn(segment, id3Frames);
+      }
       if (captions && captions.length) {
         captionsFn(segment, captions);
       }
@@ -516,7 +519,19 @@ const handleSegmentBytes = ({
             message.logs.forEach(function(log) {
               onTransmuxerLog(merge(log, {stream: 'mp4CaptionParser'}));
             });
-            finishLoading(message.captions);
+
+            workerCallback({
+              action: 'probeEmsgID3',
+              data: bytesAsUint8Array,
+              transmuxer: segment.transmuxer,
+              offset: startTime,
+              callback: ({emsgData, id3Frames}) => {
+                // transfer bytes back to us
+                bytes = emsgData.buffer;
+                segment.bytes = bytesAsUint8Array = emsgData;
+                finishLoading(message.captions, id3Frames);
+              }
+            });
           }
         });
       }

src/transmuxer-worker.js

@@ -227,6 +227,24 @@ class MessageHandlers {
     }, [data.buffer]);
   }
 
+  /**
+   * Probes an mp4 segment for EMSG boxes containing ID3 data.
+   * https://aomediacodec.github.io/id3-emsg/
+   *
+   * @param {Uint8Array} data segment data
+   * @param {number} offset segment start time
+   * @return {Object[]} an array of ID3 frames
+   */
+  probeEmsgID3({data, offset}) {
+    const id3Frames = mp4probe.getEmsgID3(data, offset);
+
+    this.self.postMessage({
+      action: 'probeEmsgID3',
+      id3Frames,
+      emsgData: data
+    }, [data.buffer]);
+  }
+
   /**
    * Probe an mpeg2-ts segment to determine the start time of the segment in it's
    * internal "media time," as well as whether it contains video and/or audio.

test/media-segment-request.test.js

@@ -1357,6 +1357,17 @@ QUnit.test('non-TS segment will get parsed for captions', function(assert) {
         }
       });
     }
+
+    if (event.action === 'probeEmsgID3') {
+      transmuxer.trigger({
+        type: 'message',
+        data: {
+          action: 'probeEmsgID3',
+          emsgData: event.data,
+          id3Frames: []
+        }
+      });
+    }
   };
 
   mediaSegmentRequest({
@@ -1498,6 +1509,17 @@ QUnit.test('non-TS segment will get parsed for captions on next segment request
         }
       });
     }
+
+    if (event.action === 'probeEmsgID3') {
+      transmuxer.trigger({
+        type: 'message',
+        data: {
+          action: 'probeEmsgID3',
+          emsgData: event.data,
+          id3Frames: []
+        }
+      });
+    }
   };
 
   mediaSegmentRequest({
@@ -1553,3 +1575,133 @@ QUnit.test('non-TS segment will get parsed for captions on next segment request
   // Simulate receiving the init segment after the media
   this.standardXHRResponse(initReq, mp4VideoInit());
 });
+
+QUnit.test('can get emsg ID3 frames from fmp4 segment', function(assert) {
+  const done = assert.async();
+  let gotEmsgId3 = 0;
+  let gotData = 0;
+  // expected frame data
+  const id3Frames = [{
+    cueTime: 1,
+    duration: 0,
+    frames: [{
+      id: 'TXXX',
+      description: 'foo bar',
+      data: { key: 'value' }
+    },
+    {
+      id: 'PRIV',
+      owner: 'priv-owner@foo.bar',
+      // 'foo'
+      data: new Uint8Array([0x66, 0x6F, 0x6F])
+    }]
+  },
+  {
+    cueTime: 3,
+    duration: 0,
+    frames: [{
+      id: 'PRIV',
+      owner: 'priv-owner@foo.bar',
+      // 'bar'
+      data: new Uint8Array([0x62, 0x61, 0x72])
+    },
+    {
+      id: 'TXXX',
+      description: 'bar foo',
+      data: { key: 'value' }
+    }]
+  }];
+  const transmuxer = new videojs.EventTarget();
+
+  transmuxer.postMessage = (event) => {
+    if (event.action === 'pushMp4Captions') {
+      transmuxer.trigger({
+        type: 'message',
+        data: {
+          action: 'mp4Captions',
+          data: event.data,
+          captions: 'foo bar',
+          logs: []
+        }
+      });
+    }
+
+    if (event.action === 'probeMp4StartTime') {
+      transmuxer.trigger({
+        type: 'message',
+        data: {
+          action: 'probeMp4StartTime',
+          data: event.data,
+          timingInfo: {}
+        }
+      });
+    }
+
+    if (event.action === 'probeMp4Tracks') {
+      transmuxer.trigger({
+        type: 'message',
+        data: {
+          action: 'probeMp4Tracks',
+          data: event.data,
+          tracks: [{type: 'video', codec: 'avc1.4d400d'}]
+        }
+      });
+    }
+
+    if (event.action === 'probeEmsgID3') {
+      transmuxer.trigger({
+        type: 'message',
+        data: {
+          action: 'probeEmsgID3',
+          emsgData: event.data,
+          id3Frames
+        }
+      });
+    }
+  };
+
+  mediaSegmentRequest({
+    xhr: this.xhr,
+    xhrOptions: this.xhrOptions,
+    decryptionWorker: this.mockDecrypter,
+    segment: {
+      transmuxer,
+      resolvedUri: 'mp4Video.mp4',
+      map: {
+        resolvedUri: 'mp4VideoInit.mp4'
+      }
+    },
+    progressFn: this.noop,
+    trackInfoFn: this.noop,
+    timingInfoFn: this.noop,
+    id3Fn: (segment, _id3Frames) => {
+      gotEmsgId3++;
+      assert.deepEqual(_id3Frames, id3Frames, 'got expected emsg id3 data.');
+    },
+    captionsFn: this.noop,
+    dataFn: (segment, segmentData) => {
+      gotData++;
+      assert.ok(segmentData, 'init segment bytes in map');
+      assert.ok(segment.map.tracks, 'added tracks');
+      assert.ok(segment.map.tracks.video, 'added video track');
+    },
+    doneFn: () => {
+      assert.equal(gotEmsgId3, 1, 'received emsg ID3 event');
+      assert.equal(gotData, 1, 'received data event');
+      transmuxer.off();
+      done();
+    }
+  });
+  assert.equal(this.requests.length, 2, 'there are two requests');
+
+  const initReq = this.requests.shift();
+  const segmentReq = this.requests.shift();
+
+  assert.equal(initReq.uri, 'mp4VideoInit.mp4', 'the first request is for the init segment');
+  assert.equal(segmentReq.uri, 'mp4Video.mp4', 'the second request is for a segment');
+
+  // Simulate receiving the media first
+  this.standardXHRResponse(segmentReq, mp4Video());
+  // Simulate receiving the init segment after the media
+  this.standardXHRResponse(initReq, mp4VideoInit());
+});