-
Notifications
You must be signed in to change notification settings - Fork 477
/
BackgroundReplacementVideoFrameProcessor.ts
141 lines (127 loc) · 4.95 KB
/
BackgroundReplacementVideoFrameProcessor.ts
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0
import BackgroundFilterSpec from '../backgroundfilter/BackgroundFilterSpec';
import BackgroundFilterVideoFrameProcessor from '../backgroundfilter/BackgroundFilterVideoFrameProcessor';
import BackgroundReplacementProcessor from '../backgroundreplacementprocessor/BackgroundReplacementProcessor';
import ConsoleLogger from '../logger/ConsoleLogger';
import LogLevel from '../logger/LogLevel';
import NoOpVideoFrameProcessor from '../videoframeprocessor/NoOpVideoFrameProcessor';
import BackgroundReplacementFilter from './BackgroundReplacementFilter';
import BackgroundReplacementOptions from './BackgroundReplacementOptions';
/**
* No-op implementation of the background replacement processor. An instance of this class will be returned when a user attempts
* to create a background replacement processor when it is not supported.
*/
/** @internal */
class NoOpBackgroundReplacementProcessor
extends NoOpVideoFrameProcessor
implements BackgroundReplacementProcessor {
/**
* no-op
* @returns
*/
async loadAssets(): Promise<void> {
return;
}
/**
* no-op
*/
addObserver(): void {}
/**
* no-op
*/
removeObserver(): void {}
/**
* no-op
*/
async setImageBlob(): Promise<void> {
return;
}
}
/**
* [[BackgroundReplacementVideoFrameProcessor]]
* Creates a background replacement processor which identifies the foreground person and replaces the background.
*/
export default class BackgroundReplacementVideoFrameProcessor extends BackgroundFilterVideoFrameProcessor {
/**
* A factory method that will call the private constructor to instantiate the processor and asynchronously
* initialize the worker, wasm, and ML models. Upon completion of the initialization the promise will either
* be resolved or rejected.
* @param spec The spec defines the assets that will be used for adding background filter to a frame
* @param imagePath The background replacement image path
*/
static async create(
spec?: BackgroundFilterSpec,
options?: BackgroundReplacementOptions
): Promise<BackgroundReplacementProcessor | undefined> {
spec = this.resolveSpec(spec);
options = this.resolveOptions(options);
await this.resolveOptionsAsync(options);
const { logger } = options;
const supported = await BackgroundReplacementVideoFrameProcessor.isSupported(spec, options);
// if background replacement is not supported do not initialize. The processor will become a no op if not supported.
if (!supported) {
logger.warn('Using no-op processor because background replacement is not supported');
return new NoOpBackgroundReplacementProcessor();
}
logger.info('Using background replacement filter');
const processor = new BackgroundReplacementFilter(spec, options);
await processor.loadAssets();
return processor;
}
/**
* Based on the options that are passed in set defaults for options
* @param options the options that are passed in
* @returns An updated set of options with defaults set
*/
protected static resolveOptions(
options: BackgroundReplacementOptions = {}
): BackgroundReplacementOptions {
const processorOptions: BackgroundReplacementOptions = { ...options };
if (!processorOptions.logger) {
processorOptions.logger = new ConsoleLogger('BackgroundReplacementProcessor', LogLevel.INFO);
}
return super.resolveOptions(processorOptions);
}
private static async resolveOptionsAsync(options: BackgroundReplacementOptions): Promise<void> {
if (!options.imageBlob) {
const canvas = document.createElement('canvas');
canvas.width = 100;
canvas.height = 100;
const ctx = canvas.getContext('2d');
ctx.fillStyle = 'blue';
ctx.fillRect(0, 0, 100, 100);
const blob = await new Promise<Blob>(resolve => {
canvas.toBlob(resolve);
});
options.imageBlob = blob;
}
return;
}
/**
* This method will detect the environment in which it is being used and determine if background
* replacement can be used.
* @param spec The {@link BackgroundFilterSpec} spec that will be used to initialize assets
* @param options options such as logger and imagePath
* @returns a boolean promise that will resolve to true if supported and false if not
*/
static async isSupported(
spec?: BackgroundFilterSpec,
options?: BackgroundReplacementOptions
): Promise<boolean> {
spec = this.resolveSpec(spec);
options = this.resolveOptions(options);
await this.resolveOptionsAsync(options);
const imageBlob = options.imageBlob;
const imageUrl = URL.createObjectURL(imageBlob);
try {
await BackgroundReplacementFilter.loadImage(imageUrl);
} catch (e) {
options.logger.info(`Failed to fetch load replacement image ${e.message}`);
return false;
} finally {
URL.revokeObjectURL(imageUrl);
}
return super.isSupported(spec, options);
}
}