Skip to content
This repository has been archived by the owner on Sep 11, 2024. It is now read-only.

Add autoplay for voice messages #6710

Merged
merged 1 commit into from
Sep 2, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/audio/ManagedPlayback.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ export class ManagedPlayback extends Playback {
}

public async play(): Promise<void> {
this.manager.playOnly(this);
this.manager.pauseAllExcept(this);
return super.play();
}

Expand Down
7 changes: 6 additions & 1 deletion src/audio/Playback.ts
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,8 @@ export class Playback extends EventEmitter implements IDestroyable {
}

public destroy() {
// Dev note: It's critical that we call stop() during cleanup to ensure that downstream callers
// are aware of the final clock position before the user triggered an unload.
// noinspection JSIgnoredPromiseFromCall - not concerned about being called async here
this.stop();
this.removeAllListeners();
Expand Down Expand Up @@ -177,9 +179,12 @@ export class Playback extends EventEmitter implements IDestroyable {

this.waveformObservable.update(this.resampledWaveform);

this.emit(PlaybackState.Stopped); // signal that we're not decoding anymore
this.clock.flagLoadTime(); // must happen first because setting the duration fires a clock update
this.clock.durationSeconds = this.element ? this.element.duration : this.audioBuf.duration;

// Signal that we're not decoding anymore. This is done last to ensure the clock is updated for
// when the downstream callers try to use it.
this.emit(PlaybackState.Stopped); // signal that we're not decoding anymore
}

private onPlaybackEnd = async () => {
Expand Down
6 changes: 3 additions & 3 deletions src/audio/PlaybackClock.ts
Original file line number Diff line number Diff line change
Expand Up @@ -89,9 +89,9 @@ export class PlaybackClock implements IDestroyable {
return this.observable;
}

private checkTime = () => {
private checkTime = (force = false) => {
const now = this.timeSeconds; // calculated dynamically
if (this.lastCheck !== now) {
if (this.lastCheck !== now || force) {
this.observable.update([now, this.durationSeconds]);
this.lastCheck = now;
}
Expand Down Expand Up @@ -141,7 +141,7 @@ export class PlaybackClock implements IDestroyable {
public syncTo(contextTime: number, clipTime: number) {
this.clipStart = contextTime - clipTime;
this.stopped = false; // count as a mid-stream pause (if we were stopped)
this.checkTime();
this.checkTime(true);
}

public destroy() {
Expand Down
12 changes: 7 additions & 5 deletions src/audio/PlaybackManager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

import { DEFAULT_WAVEFORM, Playback } from "./Playback";
import { DEFAULT_WAVEFORM, Playback, PlaybackState } from "./Playback";
import { ManagedPlayback } from "./ManagedPlayback";

/**
Expand All @@ -34,12 +34,14 @@ export class PlaybackManager {
}

/**
* Stops all other playback instances. If no playback is provided, all instances
* are stopped.
* Pauses all other playback instances. If no playback is provided, all playing
* instances are paused.
* @param playback Optional. The playback to leave untouched.
*/
public playOnly(playback?: Playback) {
this.instances.filter(p => p !== playback).forEach(p => p.stop());
public pauseAllExcept(playback?: Playback) {
this.instances
.filter(p => p !== playback && p.currentState === PlaybackState.Playing)
.forEach(p => p.pause());
}

public destroyPlaybackInstance(playback: ManagedPlayback) {
Expand Down
212 changes: 212 additions & 0 deletions src/audio/PlaybackQueue.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,212 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

import { MatrixClient, MatrixEvent, Room } from "matrix-js-sdk";
import { Playback, PlaybackState } from "./Playback";
import { UPDATE_EVENT } from "../stores/AsyncStore";
import { MatrixClientPeg } from "../MatrixClientPeg";
import { arrayFastClone } from "../utils/arrays";
import { PlaybackManager } from "./PlaybackManager";
import { isVoiceMessage } from "../utils/EventUtils";
import RoomViewStore from "../stores/RoomViewStore";

/**
* Audio playback queue management for a given room. This keeps track of where the user
* was at for each playback, what order the playbacks were played in, and triggers subsequent
* playbacks.
*
* Currently this is only intended to be used by voice messages.
*
* The primary mechanics are:
* * Persisted clock state for each playback instance (tied to Event ID).
* * Limited memory of playback order (see code; not persisted).
* * Autoplay of next eligible playback instance.
*/
export class PlaybackQueue {
private static queues = new Map<string, PlaybackQueue>(); // keyed by room ID

private playbacks = new Map<string, Playback>(); // keyed by event ID
private clockStates = new Map<string, number>(); // keyed by event ID
private playbackIdOrder: string[] = []; // event IDs, last == current
private currentPlaybackId: string; // event ID, broken out from above for ease of use
private recentFullPlays = new Set<string>(); // event IDs

constructor(private client: MatrixClient, private room: Room) {
this.loadClocks();

RoomViewStore.addListener(() => {
if (RoomViewStore.getRoomId() === this.room.roomId) {
// Reset the state of the playbacks before they start mounting and enqueuing updates.
// We reset the entirety of the queue, including order, to ensure the user isn't left
// confused with what order the messages are playing in.
this.currentPlaybackId = null; // this in particular stops autoplay when the room is switched to
this.recentFullPlays = new Set<string>();
this.playbackIdOrder = [];
}
});
}

public static forRoom(roomId: string): PlaybackQueue {
const cli = MatrixClientPeg.get();
const room = cli.getRoom(roomId);
if (!room) throw new Error("Unknown room");
if (PlaybackQueue.queues.has(room.roomId)) {
return PlaybackQueue.queues.get(room.roomId);
}
const queue = new PlaybackQueue(cli, room);
PlaybackQueue.queues.set(room.roomId, queue);
return queue;
}

private persistClocks() {
localStorage.setItem(
`mx_voice_message_clocks_${this.room.roomId}`,
JSON.stringify(Array.from(this.clockStates.entries())),
);
}

private loadClocks() {
const val = localStorage.getItem(`mx_voice_message_clocks_${this.room.roomId}`);
if (!!val) {
this.clockStates = new Map<string, number>(JSON.parse(val));
}
}

public unsortedEnqueue(mxEvent: MatrixEvent, playback: Playback) {
// We don't ever detach our listeners: we expect the Playback to clean up for us
this.playbacks.set(mxEvent.getId(), playback);
playback.on(UPDATE_EVENT, (state) => this.onPlaybackStateChange(playback, mxEvent, state));
playback.clockInfo.liveData.onUpdate((clock) => this.onPlaybackClock(playback, mxEvent, clock));
}

private onPlaybackStateChange(playback: Playback, mxEvent: MatrixEvent, newState: PlaybackState) {
// Remember where the user got to in playback
const wasLastPlaying = this.currentPlaybackId === mxEvent.getId();
if (newState === PlaybackState.Stopped && this.clockStates.has(mxEvent.getId()) && !wasLastPlaying) {
// noinspection JSIgnoredPromiseFromCall
playback.skipTo(this.clockStates.get(mxEvent.getId()));
} else if (newState === PlaybackState.Stopped) {
// Remove the now-useless clock for some space savings
this.clockStates.delete(mxEvent.getId());

if (wasLastPlaying) {
this.recentFullPlays.add(this.currentPlaybackId);
const orderClone = arrayFastClone(this.playbackIdOrder);
const last = orderClone.pop();
if (last === this.currentPlaybackId) {
const next = orderClone.pop();
if (next) {
const instance = this.playbacks.get(next);
if (!instance) {
console.warn(
"Voice message queue desync: Missing playback for next message: "
+ `Current=${this.currentPlaybackId} Last=${last} Next=${next}`,
);
} else {
this.playbackIdOrder = orderClone;
PlaybackManager.instance.pauseAllExcept(instance);

// This should cause a Play event, which will re-populate our playback order
// and update our current playback ID.
// noinspection JSIgnoredPromiseFromCall
instance.play();
}
} else {
// else no explicit next event, so find an event we haven't played that comes next. The live
// timeline is already most recent last, so we can iterate down that.
const timeline = arrayFastClone(this.room.getLiveTimeline().getEvents());
let scanForVoiceMessage = false;
let nextEv: MatrixEvent;
for (const event of timeline) {
if (event.getId() === mxEvent.getId()) {
scanForVoiceMessage = true;
continue;
}
if (!scanForVoiceMessage) continue;

// Dev note: This is where we'd break to cause text/non-voice messages to
// interrupt automatic playback.

const isRightType = isVoiceMessage(event);
const havePlayback = this.playbacks.has(event.getId());
const isRecentlyCompleted = this.recentFullPlays.has(event.getId());
if (isRightType && havePlayback && !isRecentlyCompleted) {
nextEv = event;
break;
}
}
if (!nextEv) {
// if we don't have anywhere to go, reset the recent playback queue so the user
// can start a new chain of playbacks.
this.recentFullPlays = new Set<string>();
this.playbackIdOrder = [];
} else {
this.playbackIdOrder = orderClone;

const instance = this.playbacks.get(nextEv.getId());
PlaybackManager.instance.pauseAllExcept(instance);

// This should cause a Play event, which will re-populate our playback order
// and update our current playback ID.
// noinspection JSIgnoredPromiseFromCall
instance.play();
}
}
} else {
console.warn(
"Voice message queue desync: Expected playback stop to be last in order. "
+ `Current=${this.currentPlaybackId} Last=${last} EventID=${mxEvent.getId()}`,
);
}
}
}

if (newState === PlaybackState.Playing) {
const order = this.playbackIdOrder;
if (this.currentPlaybackId !== mxEvent.getId() && !!this.currentPlaybackId) {
if (order.length === 0 || order[order.length - 1] !== this.currentPlaybackId) {
const lastInstance = this.playbacks.get(this.currentPlaybackId);
if (
lastInstance.currentState === PlaybackState.Playing
|| lastInstance.currentState === PlaybackState.Paused
) {
order.push(this.currentPlaybackId);
}
}
}

this.currentPlaybackId = mxEvent.getId();
if (order.length === 0 || order[order.length - 1] !== this.currentPlaybackId) {
order.push(this.currentPlaybackId);
}
}

// Only persist clock information on pause/stop (end) to avoid overwhelming the storage.
// This should get triggered from normal voice message component unmount due to the playback
// stopping itself for cleanup.
if (newState === PlaybackState.Paused || newState === PlaybackState.Stopped) {
this.persistClocks();
}
}

private onPlaybackClock(playback: Playback, mxEvent: MatrixEvent, clocks: number[]) {
if (playback.currentState === PlaybackState.Decoding) return; // ignore pre-ready values

if (playback.currentState !== PlaybackState.Stopped) {
this.clockStates.set(mxEvent.getId(), clocks[0]); // [0] is the current seek position
}
}
}
6 changes: 6 additions & 0 deletions src/components/views/messages/MAudioBody.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ import { IMediaEventContent } from "../../../customisations/models/IMediaEventCo
import MFileBody from "./MFileBody";
import { IBodyProps } from "./IBodyProps";
import { PlaybackManager } from "../../../audio/PlaybackManager";
import { isVoiceMessage } from "../../../utils/EventUtils";
import { PlaybackQueue } from "../../../audio/PlaybackQueue";

interface IState {
error?: Error;
Expand Down Expand Up @@ -67,6 +69,10 @@ export default class MAudioBody extends React.PureComponent<IBodyProps, IState>
playback.clockInfo.populatePlaceholdersFrom(this.props.mxEvent);
this.setState({ playback });

if (isVoiceMessage(this.props.mxEvent)) {
PlaybackQueue.forRoom(this.props.mxEvent.getRoomId()).unsortedEnqueue(this.props.mxEvent, playback);
}

// Note: the components later on will handle preparing the Playback class for us.
}

Expand Down
2 changes: 1 addition & 1 deletion src/components/views/rooms/VoiceRecordComposerTile.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ export default class VoiceRecordComposerTile extends React.PureComponent<IProps,

try {
// stop any noises which might be happening
await PlaybackManager.instance.playOnly(null);
await PlaybackManager.instance.pauseAllExcept(null);

const recorder = VoiceRecordingStore.instance.startRecording();
await recorder.start();
Expand Down