Add support for H.264 profiles besides Constrained Baseline Profile (#…

…2830)
aws · Jan 25, 2024 · d6487f4 · d6487f4
1 parent 635e9ec
commit d6487f4
Show file tree

Hide file tree

Showing 31 changed files with 13,211 additions and 6,603 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - Add support for node 20 and drop support for node < 18.
+- Add support for H.264 profiles besides Constrained Baseline Profile.
 
 ### Removed
 

diff --git a/demos/browser/app/meetingV2/meetingV2.html b/demos/browser/app/meetingV2/meetingV2.html
@@ -172,6 +172,9 @@ <h5 class="modal-title" id="additional-options-modal-label">Additional Options</
             <option value="default">Meeting Default</option>
             <option value="vp8">VP8</option>
             <option value="h264ConstrainedBaselineProfile">H.264 Constrained Baseline Profile</option>
+            <option value="h264BaselineProfile">H.264 Baseline Profile</option>
+            <option value="h264MainProfile">H.264 Main Profile</option>
+            <option value="h264HighProfile">H.264 High Profile</option>
             <option value="vp9Profile0" id="vp9-video-codec">VP9 Profile 0</option>
             <option value="av1Main" id="av1Main-video-codec">AV1 Main Profile</option>
           </select>
@@ -180,6 +183,9 @@ <h5 class="modal-title" id="additional-options-modal-label">Additional Options</
             <option value="default">Meeting Default</option>
             <option value="vp8">VP8</option>
             <option value="h264ConstrainedBaselineProfile">H.264 Constrained Baseline Profile</option>
+            <option value="h264BaselineProfile">H.264 Baseline Profile</option>
+            <option value="h264MainProfile">H.264 Main Profile</option>
+            <option value="h264HighProfile">H.264 High Profile</option>
             <option value="vp9Profile0" id="vp9Profile0-content-codec">VP9 Profile 0</option>
             <option value="av1Main"id="av1Main-content-codec">AV1 Main Profile</option>
           </select>

diff --git a/demos/browser/app/meetingV2/meetingV2.ts b/demos/browser/app/meetingV2/meetingV2.ts
@@ -3940,50 +3940,42 @@ export class DemoMeetingApp
         this.requestedContentMaxResolution = VideoQualitySettings.VideoResolutionFHD;
     }
 
-    const chosenVideoSendCodec = (document.getElementById('videoCodecSelect') as HTMLSelectElement).value;
-    switch (chosenVideoSendCodec) {
-      case 'vp8':
-        this.videoCodecPreferences = [VideoCodecCapability.vp8()];
-        break;
-      case 'h264ConstrainedBaselineProfile':
-        // If `h264ConstrainedBaselineProfile` is explicitly selected, include VP8 as fallback
-        this.videoCodecPreferences = [VideoCodecCapability.h264ConstrainedBaselineProfile(), VideoCodecCapability.vp8()];
-        break;
-      case 'av1Main':
-        this.videoCodecPreferences = [VideoCodecCapability.av1Main(), VideoCodecCapability.h264ConstrainedBaselineProfile(), VideoCodecCapability.vp8()];
-        this.enableSimulcast  = false; // simulcast does not work for AV1
-        break;
-      case 'vp9Profile0':
-        this.videoCodecPreferences = [VideoCodecCapability.vp9Profile0(), VideoCodecCapability.h264ConstrainedBaselineProfile(), VideoCodecCapability.vp8()];
-        this.enableSimulcast  = false; // simulcast does not work for VP9
-        break;
-      default:
-        // If left on 'Meeting Default', use the existing behavior when `setVideoCodecSendPreferences` is not called
-        // which should be equivalent to `this.videoCodecPreferences = [VideoCodecCapability.h264ConstrainedBaselineProfile()]`
-        break;
+    const getCodecPreferences = (chosenCodec: string) => {
+      // We will always include H.264 CBP and VP8 and fallbacks when those are not the codecs selected, so that
+      // we always have a widely available codec in use.
+      switch (chosenCodec) {
+        case 'vp8':
+          return [VideoCodecCapability.vp8()];
+        case 'h264ConstrainedBaselineProfile':
+          return [VideoCodecCapability.h264ConstrainedBaselineProfile(), VideoCodecCapability.vp8()];
+        case 'h264BaselineProfile':
+          return [VideoCodecCapability.h264BaselineProfile(), VideoCodecCapability.h264ConstrainedBaselineProfile(), VideoCodecCapability.vp8()];
+        case 'h264MainProfile':
+          return [VideoCodecCapability.h264MainProfile(), VideoCodecCapability.h264ConstrainedBaselineProfile(), VideoCodecCapability.vp8()];
+        case 'h264HighProfile':
+          // Include both Constrained High (typically offered on Safari) and High
+          return [VideoCodecCapability.h264HighProfile(), VideoCodecCapability.h264ConstrainedHighProfile(), VideoCodecCapability.h264ConstrainedBaselineProfile(), VideoCodecCapability.vp8()];
+        case 'av1Main':
+          return [VideoCodecCapability.av1Main(), VideoCodecCapability.h264ConstrainedBaselineProfile(), VideoCodecCapability.vp8()];
+        case 'vp9Profile0':
+          return [VideoCodecCapability.vp9Profile0(), VideoCodecCapability.h264ConstrainedBaselineProfile(), VideoCodecCapability.vp8()];
+        default:
+          // If left on 'Meeting Default', use the existing behavior when `setVideoCodecSendPreferences` is not called
+          // which should be equivalent to `this.videoCodecPreferences = [VideoCodecCapability.h264ConstrainedBaselineProfile()]`
+          return [];
+      }
     }
 
-    const chosenContentSendCodec = (document.getElementById('contentCodecSelect') as HTMLSelectElement).value;
-    switch (chosenContentSendCodec) {
-      case 'vp8':
-        this.contentCodecPreferences = [VideoCodecCapability.vp8()];
-        break;
-      case 'h264ConstrainedBaselineProfile':
-        // If `h264ConstrainedBaselineProfile` is explicitly selected, include VP8 as fallback
-        this.contentCodecPreferences = [VideoCodecCapability.h264ConstrainedBaselineProfile(), VideoCodecCapability.vp8()];
-        break;
-      case 'av1Main':
-        this.contentCodecPreferences = [VideoCodecCapability.av1Main(), VideoCodecCapability.h264ConstrainedBaselineProfile(), VideoCodecCapability.vp8()];
-        break;
-      case 'vp9Profile0':
-        this.contentCodecPreferences = [VideoCodecCapability.vp9Profile0(), VideoCodecCapability.h264ConstrainedBaselineProfile(), VideoCodecCapability.vp8()];
-        break;
-      default:
-        // If left on 'Meeting Default', use the existing behavior when `setVideoCodecSendPreferences` is not called
-        // which should be equivalent to `this.videoCodecPreferences = [VideoCodecCapability.h264ConstrainedBaselineProfile()]`
-        break;
+    const chosenVideoSendCodec = (document.getElementById('videoCodecSelect') as HTMLSelectElement).value;
+    this.videoCodecPreferences = getCodecPreferences(chosenVideoSendCodec);
+    if (!['av1Main', 'vp9Profile0'].includes(chosenVideoSendCodec)) {
+      // Attempting to use simulcast with VP9 or AV1 will lead to unexpected behavior (e.g. SVC instead)
+      this.enableSimulcast = false;
     }
 
+    const chosenContentSendCodec = (document.getElementById('contentCodecSelect') as HTMLSelectElement).value;
+    this.contentCodecPreferences = getCodecPreferences(chosenContentSendCodec);
+
     this.audioCapability = (document.getElementById('audioCapabilitySelect') as HTMLSelectElement).value;
     this.videoCapability = (document.getElementById('videoCapabilitySelect') as HTMLSelectElement).value;
     this.contentCapability = (document.getElementById('contentCapabilitySelect') as HTMLSelectElement).value;

diff --git a/demos/browser/app/meetingV2/video/VideoTileCollection.ts b/demos/browser/app/meetingV2/video/VideoTileCollection.ts
@@ -281,6 +281,11 @@ export default class VideoTileCollection implements AudioVideoObserver {
       const tileId = videoTile.id();
       const tileIndex = this.tileIdToTileIndex[tileId];
       const demoVideoTile = this.tileIndexToDemoVideoTile.get(tileIndex);
+      if (demoVideoTile === undefined) {
+        // Demo tile doesn't exist yet, we probably haven't received any frame
+        continue;
+      }
+
       if (tileState.localTile || tileState.isContent) {
         uplinkCount++;
         demoVideoTile.collectVideoStats(VideoUpstreamMetricsKeyStats, videoMetricReport[tileState.boundAttendeeId], 'Upstream', uplinkStats);