Fix CHANGELOG (#1879)

.github/workflows/audio-video-transform-integration.yml

@@ -76,6 +76,9 @@
         - name: Run Background Blur Integration Test
           if:  steps.test_needed.outputs.integ_test_required == 'true'
           run: npm run test:integration-background-blur
+        - name: Run Background Replacement Integration Test
+          if:  steps.test_needed.outputs.integ_test_required == 'true'
+          run: npm run test:integration-background-replacement
         - name: Run Echo Reduction Integration Test
           if:  steps.test_needed.outputs.integ_test_required == 'true'
           run: npm run test:integration-echo-reduction

.github/workflows/transcription-integration.yml

@@ -76,4 +76,4 @@
           run: npm ci
         - name: Run Transcription Integration Test
           if:  steps.test_needed.outputs.integ_test_required == 'true'
-          run: npm run test:integration-transcription
+          run: npm run test:integration-transcription

CHANGELOG.md

@@ -10,18 +10,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Add `supportDownlinkBandwidthEstimation` API to check whether browsers support downlink bandwidth estimation 
   which requires for priority based downlink policy to work.
 - Add `keepLastFrameWhenPaused` in `DefaultVideoTile` as an option to keep last frame when pausing a video tile.
-
-### Removed
-
-### Fixed
-
-### Changed
-
-## [2.24.0] - 2021-12-07
-### Added
 - Add error name for custom device controller error.
 - Added pagination option to meeting demo when priority downlink policy is used.
 - Add `ApplicationMetadata` to enable builders to send their application name or version to the Amazon Chime backend. This is an opt-in addition.
+- Add `BackgroundReplacementVideoFrameProcessor` API to create a `VideoFrameProcessor` that will replace the background of a video stream with an image.
 
 ### Removed
 

demos/browser/app/meetingV2/meetingV2.ts

@@ -15,6 +15,10 @@ import {
   BackgroundBlurProcessor,
   BackgroundBlurVideoFrameProcessor,
   BackgroundBlurVideoFrameProcessorObserver,
+  BackgroundReplacementProcessor,
+  BackgroundReplacementVideoFrameProcessor,
+  BackgroundReplacementVideoFrameProcessorObserver,
+  BackgroundReplacementOptions,
   ClientMetricReport,
   ClientVideoStreamReceivingReport,
   ConsoleLogger,
@@ -168,7 +172,7 @@ const BACKGROUND_BLUR_ASSET_SPEC = (BACKGROUND_BLUR_ASSET_GROUP || BACKGROUND_BL
   revisionID: BACKGROUND_BLUR_REVISION_ID,
 }
 
-type VideoFilterName = 'Emojify' | 'CircularCut' | 'NoOp' | 'Segmentation' | 'Resize (9/16)' | 'Background Blur 10% CPU' | 'Background Blur 20% CPU' | 'Background Blur 30% CPU' | 'Background Blur 40% CPU' | 'None';
+type VideoFilterName = 'Emojify' | 'CircularCut' | 'NoOp' | 'Segmentation' | 'Resize (9/16)' | 'Background Blur 10% CPU' | 'Background Blur 20% CPU' | 'Background Blur 30% CPU' | 'Background Blur 40% CPU' | 'Background Replacement' | 'None';
 
 const VIDEO_FILTERS: VideoFilterName[] = ['Emojify', 'CircularCut', 'NoOp', 'Resize (9/16)'];
 
@@ -243,6 +247,7 @@ export class DemoMeetingApp
   attendeeIdPresenceHandler: (undefined | ((attendeeId: string, present: boolean, externalUserId: string, dropped: boolean) => void)) = undefined;
   activeSpeakerHandler: (undefined | ((attendeeIds: string[]) => void)) = undefined;
   blurObserver: (undefined | BackgroundBlurVideoFrameProcessorObserver ) = undefined;
+  replacementObserver: (undefined | BackgroundReplacementVideoFrameProcessorObserver ) = undefined;
 
   showActiveSpeakerScores = false;
   meeting: string | null = null;
@@ -294,6 +299,7 @@ export class DemoMeetingApp
   voiceFocusIsActive = false;
 
   supportsBackgroundBlur = false;
+  supportsBackgroundReplacement = false;
 
   enableLiveTranscription = false;
   noWordSeparatorForTranscription = false;
@@ -311,6 +317,8 @@ export class DemoMeetingApp
   joinInfo: any | undefined;
 
   bbprocessor: BackgroundBlurProcessor | undefined;
+  brprocessor: BackgroundReplacementProcessor | undefined;
+  replacementOptions: BackgroundReplacementOptions | undefined;
 
   // This is an extremely minimal reactive programming approach: these elements
   // will be updated when the Amazon Voice Focus display state changes.
@@ -474,6 +482,48 @@ export class DemoMeetingApp
       }
   }
 
+  async createReplacementImageBlob(startColor: string, endColor: string): Promise<Blob> {
+    const canvas = document.createElement("canvas");
+    canvas.width = 500; 
+    canvas.height = 500;
+    var ctx = canvas.getContext("2d");
+    var grd = ctx.createLinearGradient(0, 0, 250, 0);
+    grd.addColorStop(0, startColor);
+    grd.addColorStop(1, endColor);
+    ctx.fillStyle = grd;
+    ctx.fillRect(0, 0, 500, 500);
+    const blob = await new Promise<Blob> (resolve => {
+      canvas.toBlob(resolve);
+    });
+    return blob;
+  }
+
+  /**
+  * The image blob in this demo is created by generating an image
+  * from a canvas, but another common scenario would be to provide 
+  * an image blob from fetching a URL.
+  *   const image = await fetch('https://someimage.jpeg');
+  *   const imageBlob = await image.blob();
+  */
+  async getBackgroundReplacementOptions(): Promise<BackgroundReplacementOptions> {
+    if (!this.replacementOptions) {
+      const imageBlob = await this.createReplacementImageBlob("#000428", "#004e92");
+      this.replacementOptions = { imageBlob };
+    }
+    return this.replacementOptions;
+  }
+
+  async initBackgroundReplacement(): Promise<void> {
+    const logger = new ConsoleLogger('SDK', LogLevel.DEBUG);
+    try {
+      this.supportsBackgroundReplacement = await BackgroundReplacementVideoFrameProcessor.isSupported(this.getBackgroundBlurSpec(), await this.getBackgroundReplacementOptions());
+    }
+    catch (e) {
+      logger.warn(`[DEMO] Does not support background replacement: ${e.message}`);
+      this.supportsBackgroundReplacement = false;
+    }
+  }
+
   private async onVoiceFocusSettingChanged(): Promise<void> {
     this.log('[DEMO] Amazon Voice Focus setting toggled to', this.enableVoiceFocus);
     this.openAudioInputFromSelectionAndPreview();
@@ -602,6 +652,7 @@ export class DemoMeetingApp
 
           await this.initVoiceFocus();
           await this.initBackgroundBlur();
+          await this.initBackgroundReplacement();
           await this.populateAllDeviceLists();
           await this.populateVideoFilterInputList(false);
           await this.populateVideoFilterInputList(true);
@@ -2351,6 +2402,10 @@ export class DemoMeetingApp
         filters.push('Background Blur 30% CPU');
         filters.push('Background Blur 40% CPU');
       }
+
+      if (this.supportsBackgroundReplacement) {
+        filters.push('Background Replacement');
+      }
     }
 
     this.populateFilterList(isPreviewWindow, genericName, filters);
@@ -2924,6 +2979,21 @@ export class DemoMeetingApp
       return this.bbprocessor;
     }
 
+    if (videoFilter.startsWith('Background Replacement')) {
+      console.log("background replacement - create called from videoFilterToProcessor!")
+
+      // In the event that frames start being dropped we should take some action to remove the background replacement.
+      this.replacementObserver = {
+        filterFrameDurationHigh: (event) => {
+          this.log(`background filter duration high: framed dropped - ${event.framesDropped}, avg - ${event.avgFilterDurationMillis} ms, frame rate - ${event.framerate}, period - ${event.periodMillis} ms`);
+        }
+      };
+
+      this.brprocessor = await BackgroundReplacementVideoFrameProcessor.create(this.getBackgroundBlurSpec(), await this.getBackgroundReplacementOptions());
+      this.brprocessor.addObserver(this.replacementObserver);
+      return this.brprocessor;
+    }
+
     return null;
   }
 
@@ -3168,6 +3238,9 @@ export class DemoMeetingApp
       // remove blur event observer
       this.bbprocessor?.removeObserver(this.blurObserver);
 
+      // remove replacement event observer
+      this.brprocessor?.removeObserver(this.replacementObserver);
+
       // Stop any video processor.
       await this.chosenVideoTransformDevice?.stop();
 
@@ -3187,13 +3260,17 @@ export class DemoMeetingApp
         this.eventReporter?.destroy();
       }
 
+      await this.bbprocessor?.destroy();
+      await this.brprocessor?.destroy();
+
       this.audioVideo = undefined;
       this.voiceFocusDevice = undefined;
       this.meetingSession = undefined;
       this.activeSpeakerHandler = undefined;
       this.currentAudioInputDevice = undefined;
       this.eventReporter = undefined;
       this.bbprocessor = undefined;
+      this.brprocessor = undefined;
     };
 
     const onLeftMeeting = async () => {

demos/browser/script/copy-assets.js

@@ -11,7 +11,7 @@ assetDefinitions.forEach(assetDef => {
     const files = fs.readdirSync(assetDef.path);
 
     if (!fs.existsSync(assetDef.outputPath)) {
-        fs.mkdirSync(assetDef.outputPath, {recursive: true});
+        fs.mkdirSync(assetDef.outputPath, { recursive: true });
     }
 
     files.filter(f => assetDef.ext.includes(f.split('.').pop())).forEach(f => {

demos/browser/webpack.config.hot.js

@@ -42,6 +42,7 @@ module.exports = env => {
         '/join': 'http://127.0.0.1:8081',
         '/end': 'http://127.0.0.1:8081',
         '/fetch_credentials': 'http://127.0.0.1:8081',
+        '/replacement_image': 'http://127.0.0.1:8081',
       }
     },
     plugins: [

demos/serverless/src/handlers.js

@@ -248,6 +248,11 @@ exports.audio_file = async (event, context) => {
   return response(200, 'audio/mpeg', fs.readFileSync('./speech.mp3', {encoding: 'base64'}), true);
 };
 
+exports.replacement_image = async (event, context) => {
+  const image = fs.readFileSync('./FallImage.jpeg');
+  return response(200, 'image/jpeg', image.toString('base64') , true);
+};
+
 exports.fetch_credentials = async (event, context) => {
   const awsCredentials = {
     accessKeyId: AWS.config.credentials.accessKeyId,

demos/serverless/template.yaml

@@ -47,6 +47,10 @@ Globals:
         BROWSER_EVENT_INGESTION_LOG_GROUP_NAME: !Ref ChimeBrowserEventIngestionLogs
         CAPTURE_S3_DESTINATION_PREFIX: !Ref ChimeMediaCaptureS3BucketPrefix
         AWS_ACCOUNT_ID: !Sub "${AWS::AccountId}"
+  Api:
+    BinaryMediaTypes:
+      # These is equivalent to image/* when deployed
+      - image~1*
 Resources:
 
   ChimeMeetingsAccessPolicy: