Skip to content

Commit

Permalink
New: Support multiple audio tracks (#299)
Browse files Browse the repository at this point in the history 
This commit adds support for selection of different audio tracks when
available in the manifest. It selects based on a combination of the
language and "role" of the audio track. This combination should be
unique in order to unambiguously select audio tracks - roles are made
unique by the backend, as audio0, audio1, etc.

One thing to be careful of in all this is how this interacts with
selecting HD/SD video. Previous implementation would cache the HD/SD
variant to switch to. Now, there are multiple HD variants and multiple
SD variants, because the HD stream could have different audio. Thus this
required some refactoring of how we pick the HD/SD streams, so that we
maintain the same audio stream while changing video quality. In the
process, I've also fixed a bug in how we size the video for audio-only
representations (before, the size would end up being so small that the
play button icon would leak into the controls).
  • Loading branch information
@bhh1988
bhh1988 authored Aug 30, 2017
1 parent 4ec7b65 commit 70f5b81
Show file tree
Hide file tree
Showing 8 changed files with 405 additions and 64 deletions.
4 changes: 4 additions & 0 deletions src/i18n/en-US.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,10 @@ of=of
subtitles=Subtitles
# Label for turning off subtitles in media player
off=Off
# Label for audio tracks in media player
media_audio=Audio
# Label for alternate audio tracks in media player
track=Track


# 3D Preview
Expand Down
110 changes: 84 additions & 26 deletions src/lib/viewers/media/DashViewer.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ const CSS_CLASS_HD = 'bp-media-controls-is-hd';
const SEGMENT_SIZE = 5;
const MAX_BUFFER = SEGMENT_SIZE * 12; // 60 sec
const MANIFEST = 'manifest.mpd';
const DEFAULT_VIDEO_WIDTH_PX = 854;
const DEFAULT_VIDEO_HEIGHT_PX = 480;

@autobind
class DashViewer extends VideoBaseViewer {
Expand All @@ -27,9 +29,10 @@ class DashViewer extends VideoBaseViewer {
this.switchHistory = [];

// tracks
this.hdRepresentation = {};
this.sdRepresentation = {};
this.hdVideoId = -1;
this.sdVideoId = -1;
this.textTracks = []; // Must be sorted by representation id
this.audioTracks = [];

// dash specific class
this.wrapperEl.classList.add(CSS_CLASS_DASH);
Expand Down Expand Up @@ -191,25 +194,21 @@ class DashViewer extends VideoBaseViewer {
}

/**
* Handler for hd video
* Given a videoId (e.g. hd video id), enables the track with that video ID
* while maintaining the SAME AUDIO as the active track.
*
* @private
* @param {number} videoId - The id of the video used in the variant (provided by Shaka)
* @return {void}
*/
enableHD() {
this.showLoadingIcon(this.hdRepresentation.id);
this.player.selectVariantTrack(this.hdRepresentation, true);
}

/**
* Handler for sd video
*
* @private
* @return {void}
*/
enableSD() {
this.showLoadingIcon(this.sdRepresentation.id);
this.player.selectVariantTrack(this.sdRepresentation, true);
enableVideoId(videoId) {
const tracks = this.player.getVariantTracks();
const activeTrack = this.getActiveTrack();
const newTrack = tracks.find((track) => track.videoId === videoId && track.audioId === activeTrack.audioId);
if (newTrack && newTrack.id !== activeTrack.id) {
this.showLoadingIcon(newTrack.id);
this.player.selectVariantTrack(newTrack, true);
}
}

/**
Expand Down Expand Up @@ -244,6 +243,21 @@ class DashViewer extends VideoBaseViewer {
}
}

/**
* Handler for audio track
*
* @private
* @emits audiochange
* @return {void}
*/
handleAudioTrack() {
const audioIdx = parseInt(this.cache.get('media-audiotracks'), 10);
if (this.audioTracks[audioIdx] !== undefined) {
const track = this.audioTracks[audioIdx];
this.player.selectAudioLanguage(track.language, track.role);
}
}

/**
* Handler for hd/sd/auto video
*
Expand All @@ -257,11 +271,11 @@ class DashViewer extends VideoBaseViewer {
switch (quality) {
case 'hd':
this.enableAdaptation(false);
this.enableHD();
this.enableVideoId(this.hdVideoId);
break;
case 'sd':
this.enableAdaptation(false);
this.enableSD();
this.enableVideoId(this.sdVideoId);
break;
case 'auto':
default:
Expand All @@ -283,7 +297,7 @@ class DashViewer extends VideoBaseViewer {
*/
adaptationHandler() {
const activeTrack = this.getActiveTrack();
if (activeTrack.id === this.hdRepresentation.id) {
if (activeTrack.videoId === this.hdVideoId) {
this.wrapperEl.classList.add(CSS_CLASS_HD);
} else {
this.wrapperEl.classList.remove(CSS_CLASS_HD);
Expand Down Expand Up @@ -329,6 +343,7 @@ class DashViewer extends VideoBaseViewer {
super.addEventListenersForMediaControls();
this.mediaControls.addListener('qualitychange', this.handleQuality);
this.mediaControls.addListener('subtitlechange', this.handleSubtitle);
this.mediaControls.addListener('audiochange', this.handleAudioTrack);
}

/**
Expand All @@ -345,6 +360,38 @@ class DashViewer extends VideoBaseViewer {
);
}
}

/**
* Loads alternate audio streams
*
* @return {void}
*/
loadAlternateAudio() {
const variants = this.player.getVariantTracks().sort((track1, track2) => track1.audioId - track2.audioId);
const audioIds = [];
const uniqueAudioVariants = [];

let i = 0;
for (i = 0; i < variants.length; i++) {
const audioTrack = variants[i];
if (audioIds.indexOf(audioTrack.audioId) < 0) {
audioIds.push(audioTrack.audioId);
uniqueAudioVariants.push(audioTrack);
}
}

this.audioTracks = uniqueAudioVariants.map((track) => ({
language: track.language,
role: track.roles[0]
}));

if (this.audioTracks.length > 1) {
// translate the language first
const languages = this.audioTracks.map((track) => getLanguageName(track.language) || track.language);
this.mediaControls.initAlternateAudio(languages);
}
}

/**
* Handler for meta data load for the media element.
*
Expand All @@ -365,6 +412,7 @@ class DashViewer extends VideoBaseViewer {
this.startBandwidthTracking();
this.handleQuality(); // should come after gettings rep ids
this.loadSubtitles();
this.loadAlternateAudio();
this.showPlayButton();

this.loaded = true;
Expand Down Expand Up @@ -402,14 +450,24 @@ class DashViewer extends VideoBaseViewer {

// Iterate over all available video representations and find the one that
// seems the biggest so that the video player is set to the max size
const hdRepresentation = tracks.reduce((a, b) => (a.width > b.width ? a : b));
const sdRepresentation = tracks.reduce((a, b) => (a.width < b.width ? a : b));
const hdRep = tracks.reduce((a, b) => (a.width > b.width ? a : b));
const sdRep = tracks.reduce((a, b) => (a.width < b.width ? a : b));
if (this.player.isAudioOnly()) {
// There is only audio, no video
this.videoWidth = DEFAULT_VIDEO_WIDTH_PX;
this.videoHeight = DEFAULT_VIDEO_HEIGHT_PX;
} else {
this.videoWidth = hdRep.width;
this.videoHeight = hdRep.height;
this.sdVideoId = sdRep.videoId;

// If there is an HD representation separate from the SD
if (hdRep.videoId !== sdRep.videoId) {
this.hdVideoId = hdRep.videoId;
}
}

this.videoWidth = hdRepresentation.width;
this.videoHeight = hdRepresentation.height;
this.aspect = this.videoWidth / this.videoHeight;
this.hdRepresentation = hdRepresentation;
this.sdRepresentation = sdRepresentation;
}

/**
Expand Down
21 changes: 21 additions & 0 deletions src/lib/viewers/media/MediaControls.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,16 @@ class MediaControls extends EventEmitter {
this.emit('subtitlechange');
}

/**
* Audio-track handler
*
* @private
* @return {void}
*/
handleAudioTrack() {
this.emit('audiochange');
}

/**
* Attaches settings menu
*
Expand Down Expand Up @@ -877,6 +887,17 @@ class MediaControls extends EventEmitter {
this.settings.addListener('subtitles', this.handleSubtitle);
this.settings.loadSubtitles(subtitles, language);
}

/**
* Takes a list of audio tracks and populates the settings menu
*
* @param {Array} audioLanguages - An ordered list of languages of the audio tracks
* @return {void}
*/
initAlternateAudio(audioLanguages) {
this.settings.addListener('audiotracks', this.handleAudioTrack);
this.settings.loadAlternateAudio(audioLanguages);
}
}

export default MediaControls;
68 changes: 58 additions & 10 deletions src/lib/viewers/media/Settings.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ const CLASS_SETTINGS = 'bp-media-settings';
const CLASS_SETTINGS_SELECTED = 'bp-media-settings-selected';
const CLASS_SETTINGS_OPEN = 'bp-media-settings-is-open';
const CLASS_SETTINGS_SUBTITLES_UNAVAILABLE = 'bp-media-settings-subtitles-unavailable';
const CLASS_SETTINGS_AUDIOTRACKS_UNAVAILABLE = 'bp-media-settings-audiotracks-unavailable';
const CLASS_SETTINGS_SUBTITLES_ON = 'bp-media-settings-subtitles-on';
const SELECTOR_SETTINGS_SUB_ITEM = '.bp-media-settings-sub-item';
const SELECTOR_SETTINGS_VALUE = '.bp-media-settings-value';
Expand All @@ -32,6 +33,11 @@ const SETTINGS_TEMPLATE = `<div class="bp-media-settings">
<div class="bp-media-settings-value">${__('off')}</div>
<div class="bp-media-settings-arrow">${ICON_ARROW_RIGHT}</div>
</div>
<div class="bp-media-settings-item bp-media-settings-item-audiotracks bp-media-settings-is-hidden" data-type="audiotracks" tabindex="0" role="menuitem" aria-haspopup="true">
<div class="bp-media-settings-label" aria-label="${__('media_audio')}">${__('media_audio')}</div>
<div class="bp-media-settings-value"></div>
<div class="bp-media-settings-arrow">${ICON_ARROW_RIGHT}</div>
</div>
</div>
<div class="bp-media-settings-menu-speed bp-media-settings-menu" role="menu">
<div class="bp-media-settings-sub-item bp-media-settings-sub-item-speed" data-type="menu" tabindex="0" role="menuitem" aria-haspopup="true">
Expand Down Expand Up @@ -87,9 +93,15 @@ const SETTINGS_TEMPLATE = `<div class="bp-media-settings">
<div class="bp-media-settings-value">${__('off')}</div>
</div>
</div>
<div class="bp-media-settings-menu-audiotracks bp-media-settings-menu bp-media-settings-is-hidden" role="menu">
<div class="bp-media-settings-sub-item bp-media-settings-sub-item-audiotracks" data-type="menu" tabindex="0" role="menuitem" aria-haspopup="true">
<div class="bp-media-settings-arrow">${ICON_ARROW_LEFT}</div>
<div class="bp-media-settings-label" aria-label="${__('media_audio')}">${__('media_audio')}</div>
</div>
</div>
</div>`;

const SUBTITLES_SUBITEM_TEMPLATE = `<div class="bp-media-settings-sub-item" data-type="subtitles" data-value="{{dataValue}}" tabindex="0" role="menuitemradio">
const SUBMENU_SUBITEM_TEMPLATE = `<div class="bp-media-settings-sub-item" data-type="{{dataType}}" data-value="{{dataValue}}" tabindex="0" role="menuitemradio">
<div class="bp-media-settings-icon">${ICON_CHECK_MARK}</div>
<div class="bp-media-settings-value"></div>
</div>`;
Expand Down Expand Up @@ -147,6 +159,7 @@ class Settings extends EventEmitter {

addActivationListener(this.settingsEl, this.menuEventHandler);
this.containerEl.classList.add(CLASS_SETTINGS_SUBTITLES_UNAVAILABLE);
this.containerEl.classList.add(CLASS_SETTINGS_AUDIOTRACKS_UNAVAILABLE);
this.init();
}

Expand Down Expand Up @@ -319,7 +332,8 @@ class Settings extends EventEmitter {
} else if (event.type === 'keydown') {
const key = decodeKeydown(event).toLowerCase();
const menuEl = menuItem.parentElement;
const itemIdx = [].findIndex.call(menuEl.children, (e) => {
const visibleOptions = [].filter.call(menuEl.children, (option) => option.offsetParent !== null);
const itemIdx = [].findIndex.call(visibleOptions, (e) => {
return e.contains(menuItem);
});

Expand All @@ -333,21 +347,21 @@ class Settings extends EventEmitter {
case 'arrowup': {
this.containerEl.classList.add(CLASS_ELEM_KEYBOARD_FOCUS);
if (itemIdx > 0) {
const newNode = menuEl.children[itemIdx - 1];
const newNode = visibleOptions[itemIdx - 1];
newNode.focus();
}
break;
}
case 'arrowdown': {
this.containerEl.classList.add(CLASS_ELEM_KEYBOARD_FOCUS);
if (itemIdx >= 0 && itemIdx < menuEl.children.length - 1) {
const newNode = menuEl.children[itemIdx + 1];
if (itemIdx >= 0 && itemIdx < visibleOptions.length - 1) {
const newNode = visibleOptions[itemIdx + 1];
newNode.focus();
}
break;
}
case 'arrowleft': {
if (itemIdx >= 0 && !menuEl.children[itemIdx].classList.contains('bp-media-settings-item')) {
if (itemIdx >= 0 && !visibleOptions[itemIdx].classList.contains('bp-media-settings-item')) {
// Go back to the main menu
this.reset();
this.firstMenuItem.focus();
Expand All @@ -356,7 +370,7 @@ class Settings extends EventEmitter {
}
case 'arrowright': {
if (itemIdx >= 0) {
const curNode = menuEl.children[itemIdx];
const curNode = visibleOptions[itemIdx];
const dataType = curNode.getAttribute('data-type');
if (curNode.classList.contains('bp-media-settings-item') && dataType !== 'menu') {
this.showSubMenu(dataType);
Expand Down Expand Up @@ -422,8 +436,11 @@ class Settings extends EventEmitter {

// Remove the checkmark from the prior selected option in the sub menu
const prevSelected = this.getSelectedOption(type);
prevSelected.classList.remove(CLASS_SETTINGS_SELECTED);
prevSelected.removeAttribute('aria-checked');
if (prevSelected) {
// this may not exist, for instance, when first initializing audio tracks
prevSelected.classList.remove(CLASS_SETTINGS_SELECTED);
prevSelected.removeAttribute('aria-checked');
}

// Add a checkmark to the new selected option in the sub menu
option.classList.add(CLASS_SETTINGS_SELECTED);
Expand Down Expand Up @@ -588,7 +605,10 @@ class Settings extends EventEmitter {
this.subtitles = subtitles;
this.language = language;
this.subtitles.forEach((subtitle, idx) => {
insertTemplate(subtitlesSubMenu, SUBTITLES_SUBITEM_TEMPLATE.replace(/{{dataValue}}/g, idx));
insertTemplate(
subtitlesSubMenu,
SUBMENU_SUBITEM_TEMPLATE.replace(/{{dataType}}/g, 'subtitles').replace(/{{dataValue}}/g, idx)
);
const languageNode = subtitlesSubMenu.lastChild.querySelector('.bp-media-settings-value');
languageNode.textContent = subtitle;
});
Expand All @@ -602,6 +622,34 @@ class Settings extends EventEmitter {

this.reset();
}

/**
* Takes an ordered list of audio languages and populates the settings menu
*
* @param {Array} audioLanguages - An ordered list of languages of the audio tracks
* @return {void}
*/
loadAlternateAudio(audioLanguages) {
const audioTracksSubMenu = this.settingsEl.querySelector('.bp-media-settings-menu-audiotracks');
audioLanguages.forEach((language, idx) => {
insertTemplate(
audioTracksSubMenu,
SUBMENU_SUBITEM_TEMPLATE.replace(/{{dataType}}/g, 'audiotracks').replace(/{{dataValue}}/g, idx)
);
const trackNode = audioTracksSubMenu.lastChild.querySelector('.bp-media-settings-value');
// It's common for the language to be unknown and show up as "und" language code. Just omit
// the language in this case, otherwise display the language too
let textContent = `${__('track')} ${idx + 1}`;
if (language !== 'und') {
textContent = `${textContent} (${language})`;
}
trackNode.textContent = textContent;
});
this.chooseOption('audiotracks', '0');

this.containerEl.classList.remove(CLASS_SETTINGS_AUDIOTRACKS_UNAVAILABLE);
this.reset();
}
}

export default Settings;
Loading

0 comments on commit 70f5b81

Please sign in to comment.