Wrapping audiocontext elements in userInteraction listener (prevents Chrome interfering)

This commit is contained in:
James Barnsley
2020-11-28 21:57:12 +13:00
parent 082706a1f9
commit e266227921
13 changed files with 906 additions and 574 deletions

BIN
src/assets/silence.mp3 Normal file

Binary file not shown.

View File

@ -3,7 +3,6 @@ import { bindActionCreators } from 'redux';
import { Route, Switch } from 'react-router-dom';
import { connect } from 'react-redux';
import ReactGA from 'react-ga';
import localForage from 'localforage';
import * as Sentry from '@sentry/browser';
import Sidebar from './components/Sidebar';
@ -62,15 +61,15 @@ import * as spotifyActions from './services/spotify/actions';
import * as lastfmActions from './services/lastfm/actions';
import * as geniusActions from './services/genius/actions';
import * as snapcastActions from './services/snapcast/actions';
import MediaSession from './components/MediaSession';
export class App extends React.Component {
constructor(props) {
super(props);
const {
allow_reporting,
test_mode,
} = this.props;
const { allow_reporting } = this.props;
this.state = {
userHasInteracted: false,
};
if (allow_reporting) {
ReactGA.initialize('UA-64701652-3');
@ -127,16 +126,6 @@ export class App extends React.Component {
window.language = props.language;
}
componentWillUnmount() {
window.removeEventListener(
'beforeinstallprompt',
this.handleInstallPrompt,
false,
);
window.removeEventListener('focus', this.handleFocusAndBlur, false);
window.removeEventListener('blur', this.handleFocusAndBlur, false);
}
componentDidMount() {
const {
history,
@ -145,7 +134,6 @@ export class App extends React.Component {
mopidyActions,
pusherActions,
snapcastActions,
coreActions,
} = this.props;
window.addEventListener(
@ -155,6 +143,7 @@ export class App extends React.Component {
);
window.addEventListener('focus', this.handleFocusAndBlur, false);
window.addEventListener('blur', this.handleFocusAndBlur, false);
this.listenForUserInteraction();
// Fire up our services
mopidyActions.connect();
@ -201,11 +190,21 @@ export class App extends React.Component {
}
}
componentWillUnmount() {
window.removeEventListener(
'beforeinstallprompt',
this.handleInstallPrompt,
false,
);
window.removeEventListener('focus', this.handleFocusAndBlur, false);
window.removeEventListener('blur', this.handleFocusAndBlur, false);
}
/**
* Using Visibility API, detect whether the browser is in focus or not
*
* This is used to keep background requests lean, preventing a queue of requests building up
* for when focus is retained. Seems most obvious on mobile devices with Chrome as it has throttled
* This is used to keep background requests lean, preventing a queue of requests building up for
* when focus is retained. Seems most obvious on mobile devices with Chrome as it has throttled
* quota significantly: https://developers.google.com/web/updates/2017/03/background_tabs
*
* @param e Event
@ -222,28 +221,59 @@ export class App extends React.Component {
installPrompt(e);
}
render() {
let className = `${this.props.theme}-theme app-inner`;
listenForUserInteraction = () => {
const userInteracted = () => {
this.setState({ userHasInteracted: true });
window.removeEventListener('mouseover', userInteracted, false);
window.removeEventListener('scroll', userInteracted, false);
window.removeEventListener('keydown', userInteracted, false);
}
window.addEventListener('mouseover', userInteracted, false);
window.addEventListener('scroll', userInteracted, false);
window.addEventListener('keydown', userInteracted, false);
}
render = () => {
const {
uiActions,
location,
history,
debug_info,
theme,
dragging,
touch_dragging,
context_menu,
sidebar_open,
slim_mode,
wide_scrollbar_enabled,
smooth_scrolling_enabled,
hotkeys_enabled,
} = this.props;
const {
userHasInteracted,
} = this.state;
let className = `${theme}-theme app-inner`;
className += ` ${navigator.onLine ? 'online' : 'offline'}`
if (this.props.wide_scrollbar_enabled) {
if (wide_scrollbar_enabled) {
className += ' wide-scrollbar';
}
if (this.props.dragging) {
if (dragging) {
className += ' dragging';
}
if (this.props.sidebar_open) {
if (sidebar_open) {
className += ' sidebar-open';
}
if (this.props.touch_dragging) {
if (touch_dragging) {
className += ' touch-dragging';
}
if (this.props.context_menu) {
if (context_menu) {
className += ' context-menu-open';
}
if (this.props.slim_mode) {
if (slim_mode) {
className += ' slim-mode';
}
if (this.props.smooth_scrolling_enabled) {
if (smooth_scrolling_enabled) {
className += ' smooth-scrolling-enabled';
}
if (isTouchDevice()) {
@ -271,13 +301,13 @@ export class App extends React.Component {
<Route>
<div>
<Sidebar
location={this.props.location}
history={this.props.history}
location={location}
history={history}
tabIndex="3"
/>
<PlaybackControls
history={this.props.history}
slim_mode={this.props.slim_mode}
history={history}
slim_mode={slim_mode}
tabIndex="2"
/>
@ -379,16 +409,16 @@ export class App extends React.Component {
</div>
<ResizeListener
uiActions={this.props.uiActions}
slim_mode={this.props.slim_mode}
uiActions={uiActions}
slim_mode={slim_mode}
/>
{this.props.hotkeys_enabled && <Hotkeys history={this.props.history} />}
{hotkeys_enabled && <Hotkeys history={history} />}
<ContextMenu />
<Dragger />
<Notifications />
<Stream />
{this.props.debug_info ? <DebugInfo /> : null}
{userHasInteracted && <Stream />}
{userHasInteracted && <MediaSession />}
{debug_info && <DebugInfo />}
</div>
);
}
@ -421,7 +451,7 @@ const mapStateToProps = (state) => {
authorization: spotify_authorized,
},
} = state;
return {
language,
theme,

View File

@ -0,0 +1,119 @@
import React from 'react';
import { connect } from 'react-redux';
import { bindActionCreators } from 'redux';
import * as mopidyActions from '../services/mopidy/actions';
class MediaSession extends React.Component {
constructor(props) {
super(props);
this.state = {
current_track: null,
stream_title: null,
};
}
componentDidMount() {
const {
mediaSession,
} = navigator;
mediaSession.setActionHandler('play', () => this.actionHandler('play'));
mediaSession.setActionHandler('pause', () => this.actionHandler('pause'));
mediaSession.setActionHandler('seekbackward', () => this.actionHandler('seekbackward'));
mediaSession.setActionHandler('seekforward', () => this.actionHandler('seekforward'));
mediaSession.setActionHandler('previoustrack', () => this.actionHandler('previous'));
mediaSession.setActionHandler('nexttrack', () => this.actionHandler('next'));
}
static getDerivedStateFromProps({ current_track, stream_title, play_state }, state) {
if (current_track) {
navigator.mediaSession.metadata = new window.MediaMetadata({
title: stream_title || current_track.title,
artist: current_track.artists[0].name,
album: current_track.album.name,
artwork: [
{
src: current_track.images ? current_track.images.small : '',
sizes: '96x96',
type: 'image/png',
},
{
src: current_track.images ? current_track.images.medium : '',
sizes: '256x256',
type: 'image/png',
},
{
src: current_track.images ? current_track.images.huge : '',
sizes: '512x512',
type: 'image/png',
},
],
});
}
navigator.mediaSession.playbackState = play_state;
return {
...state,
current_track,
stream_title,
};
}
actionHandler = (action) => {
const {
mopidyActions: actions,
time_position,
} = this.props;
switch (action) {
case 'seekbackward': {
let newposition = time_position - 30000; // 30 seconds
if (newposition <= 0) newposition = 0;
return actions.setTimePosition(newposition);
}
case 'seekforward': {
return actions.setTimePosition(time_position + 30000); // 30 seconds
}
default: {
return actions[action]();
}
}
}
render = () => {
return (
// eslint-disable-next-line jsx-a11y/media-has-caption
<audio
id="media-session"
src="/iris/assets/silence.mp3"
autoPlay
loop
style={{ display: 'none' }}
/>
);
};
}
const mapStateToProps = (state) => {
const {
items,
current_track,
stream_title,
} = state.core;
return {
current_track: current_track ? items[current_track.uri] || current_track : null,
stream_title,
play_state: state.mopidy.play_state,
time_position: state.mopidy.time_position,
volume: state.mopidy.volume,
mute: state.mopidy.mute,
};
};
const mapDispatchToProps = (dispatch) => ({
mopidyActions: bindActionCreators(mopidyActions, dispatch),
});
export default connect(mapStateToProps, mapDispatchToProps)(MediaSession);

View File

@ -31,58 +31,7 @@ class PlaybackControls extends React.Component {
};
}
componentDidMount() {
this.setupPlayer();
}
// Create audio element
//
setupPlayer = () => {
const {
mopidyActions: {
next,
previous,
pause,
play,
},
} = this.props;
if (!this.audioTag) {
this.audioTag = document.createElement('audio');
document.body.appendChild(this.audioTag);
this.audioTag.id = 'audioTag';
this.audioTag.src = 'https://ia800206.us.archive.org/16/items/SilentRingtone/silence_64kb.mp3';
this.audioTag.loop = true;
setTimeout(() => this.audioTag.play(), 1000);
}
console.debug(navigator.mediaSession)
navigator.mediaSession.setActionHandler('play', () => play());
navigator.mediaSession.setActionHandler('pause', () => pause());
navigator.mediaSession.setActionHandler('seekbackward', () => console.log('seekbackward'));
navigator.mediaSession.setActionHandler('seekforward', () => console.log('seekforward'));
navigator.mediaSession.setActionHandler('previoustrack', () => previous());
navigator.mediaSession.setActionHandler('nexttrack', () => next());
}
static getDerivedStateFromProps({ current_track, stream_title, play_state }, state) {
if (current_track) {
navigator.mediaSession.metadata = new window.MediaMetadata({
title: current_track.title,
artist: current_track.artists[0].name,
artwork: [
{
src: current_track.images ? current_track.images.medium : '',
sizes: '96x96',
type: 'image/png',
},
]
});
};
navigator.mediaSession.playbackState = play_state;
static getDerivedStateFromProps({ current_track, stream_title }, state) {
return {
...state,
current_track,

View File

@ -100,9 +100,9 @@ class CodecMessage extends BaseMessage {
let decoder = new TextDecoder("utf-8");
this.codec = decoder.decode(buffer.slice(30, 30 + codecSize));
let payloadSize = view.getInt32(30 + codecSize, true);
console.log("payload size: " + payloadSize);
//console.log("payload size: " + payloadSize);
this.payload = buffer.slice(34 + codecSize, 34 + codecSize + payloadSize);
console.log("payload: " + this.payload);
//console.log("payload: " + this.payload);
}
}
class TimeMessage extends BaseMessage {
@ -240,7 +240,7 @@ class PcmChunkMessage extends BaseMessage {
this.timestamp = new Tv(view.getInt32(26, true), view.getInt32(30, true));
// this.payloadSize = view.getUint32(34, true);
this.payload = buffer.slice(38); //, this.payloadSize + 38));// , this.payloadSize);
// console.log("ts: " + this.timestamp.sec + " " + this.timestamp.usec + ", payload: " + this.payloadSize + ", len: " + this.payload.byteLength);
//console.log("ts: " + this.timestamp.sec + " " + this.timestamp.usec + ", payload: " + this.payloadSize + ", len: " + this.payload.byteLength);
}
readFrames(frames) {
let frameCnt = frames;
@ -250,7 +250,7 @@ class PcmChunkMessage extends BaseMessage {
let begin = this.idx * frameSize;
this.idx += frameCnt;
let end = begin + frameCnt * frameSize;
// console.log("readFrames: " + frames + ", result: " + frameCnt + ", begin: " + begin + ", end: " + end + ", payload: " + this.payload.byteLength);
//console.log("readFrames: " + frames + ", result: " + frameCnt + ", begin: " + begin + ", end: " + end + ", payload: " + this.payload.byteLength);
return this.payload.slice(begin, end);
}
getFrameCount() {
@ -308,7 +308,7 @@ class AudioStream {
setVolume(percent, muted) {
let base = 10;
this.volume = percent / 100; // (Math.pow(base, percent / 100) - 1) / (base - 1);
console.log("setVolume: " + percent + " => " + this.volume + ", muted: " + this.muted);
//console.log("setVolume: " + percent + " => " + this.volume + ", muted: " + this.muted);
this.muted = muted;
}
addChunk(chunk) {
@ -321,7 +321,7 @@ class AudioStream {
// todo: consider buffer ms
if (age > 5000 + this.bufferMs) {
this.chunks.shift();
console.log("Dropping old chunk: " + age.toFixed(2) + ", left: " + this.chunks.length);
//console.log("Dropping old chunk: " + age.toFixed(2) + ", left: " + this.chunks.length);
}
else
break;
@ -346,18 +346,18 @@ class AudioStream {
let secs = Math.floor(Date.now() / 1000);
if (this.lastLog != secs) {
this.lastLog = secs;
console.log("age: " + age.toFixed(2) + ", req: " + reqChunkDuration);
//console.log("age: " + age.toFixed(2) + ", req: " + reqChunkDuration);
}
if (age < -reqChunkDuration) {
console.log("age: " + age.toFixed(2) + " < req: " + reqChunkDuration * -1 + ", chunk.startMs: " + this.chunk.startMs().toFixed(2) + ", timestamp: " + this.chunk.timestamp.getMilliseconds().toFixed(2));
console.log("Chunk too young, returning silence");
//console.log("age: " + age.toFixed(2) + " < req: " + reqChunkDuration * -1 + ", chunk.startMs: " + this.chunk.startMs().toFixed(2) + ", timestamp: " + this.chunk.timestamp.getMilliseconds().toFixed(2));
//console.log("Chunk too young, returning silence");
}
else {
if (Math.abs(age) > 5) {
// We are 5ms apart, do a hard sync, i.e. don't play faster/slower,
// but seek to the desired position instead
while (this.chunk && age > this.chunk.duration()) {
console.log("Chunk too old, dropping (age: " + age.toFixed(2) + " > " + this.chunk.duration().toFixed(2) + ")");
//console.log("Chunk too old, dropping (age: " + age.toFixed(2) + " > " + this.chunk.duration().toFixed(2) + ")");
this.chunk = this.chunks.shift();
if (!this.chunk)
break;
@ -365,11 +365,11 @@ class AudioStream {
}
if (this.chunk) {
if (age > 0) {
console.log("Fast forwarding " + age.toFixed(2) + "ms");
//console.log("Fast forwarding " + age.toFixed(2) + "ms");
this.chunk.readFrames(Math.floor(age * this.chunk.sampleFormat.msRate()));
}
else if (age < 0) {
console.log("Playing silence " + -age.toFixed(2) + "ms");
//console.log("Playing silence " + -age.toFixed(2) + "ms");
let silentFrames = Math.floor(-age * this.chunk.sampleFormat.msRate());
left.fill(0, 0, silentFrames);
right.fill(0, 0, silentFrames);
@ -430,7 +430,7 @@ class AudioStream {
left[pos + 1] = left[pos];
right[pos + 1] = right[pos];
pos++;
// console.log("Add: " + pos);
//console.log("Add: " + pos);
}
}
pos++;
@ -440,13 +440,13 @@ class AudioStream {
}
}
if (addFrames != 0)
console.debug("Pos: " + pos + ", frames: " + frames + ", add: " + addFrames + ", everyN: " + everyN);
//console.debug("Pos: " + pos + ", frames: " + frames + ", add: " + addFrames + ", everyN: " + everyN);
if (read == readFrames)
read = frames;
}
}
if (read < frames) {
console.log("Failed to get chunk, read: " + read + "/" + frames + ", chunks left: " + this.chunks.length);
//console.log("Failed to get chunk, read: " + read + "/" + frames + ", chunks left: " + this.chunks.length);
left.fill(0, pos);
right.fill(0, pos);
}
@ -484,7 +484,7 @@ class TimeProvider {
this.diff = sorted[Math.floor(sorted.length / 2)];
}
// console.debug("c2s: " + c2s.toFixed(2) + ", s2c: " + s2c.toFixed(2) + ", diff: " + this.diff.toFixed(2) + ", now: " + this.now().toFixed(2) + ", server.now: " + this.serverNow().toFixed(2) + ", win.now: " + window.performance.now().toFixed(2));
// console.log("now: " + this.now() + "\t" + this.now() + "\t" + this.now());
//console.log("now: " + this.now() + "\t" + this.now() + "\t" + this.now());
}
now() {
if (!this.ctx) {
@ -561,7 +561,7 @@ class OpusDecoder extends Decoder {
format.rate = view.getUint32(4, true);
format.bits = view.getUint16(8, true);
format.channels = view.getUint16(10, true);
console.log("Opus samplerate: " + format.toString());
//console.log("Opus samplerate: " + format.toString());
return format;
}
decode(chunk) {
@ -576,7 +576,7 @@ class FlacDecoder extends Decoder {
this.decoder = Flac.create_libflac_decoder(true);
if (this.decoder) {
let init_status = Flac.init_decoder_stream(this.decoder, this.read_callback_fn.bind(this), this.write_callback_fn.bind(this), this.error_callback_fn.bind(this), this.metadata_callback_fn.bind(this), false);
console.log("Flac init: " + init_status);
//console.log("Flac init: " + init_status);
Flac.setOptions(this.decoder, { analyseSubframes: true, analyseResiduals: true });
}
this.sampleFormat = new SampleFormat();
@ -584,37 +584,37 @@ class FlacDecoder extends Decoder {
// this.pcmChunk = new PcmChunkMessage();
// Flac.setOptions(this.decoder, {analyseSubframes: analyse_frames, analyseResiduals: analyse_residuals});
// flac_ok &= init_status == 0;
// console.log("flac init : " + flac_ok);//DEBUG
//console.log("flac init : " + flac_ok);//DEBUG
}
decode(chunk) {
// console.log("Flac decode: " + chunk.payload.byteLength);
//console.log("Flac decode: " + chunk.payload.byteLength);
this.flacChunk = chunk.payload.slice(0);
this.pcmChunk = chunk;
this.pcmChunk.clearPayload();
this.cacheInfo = { cachedBlocks: 0, isCachedChunk: true };
// console.log("Flac len: " + this.flacChunk.byteLength);
//console.log("Flac len: " + this.flacChunk.byteLength);
while (this.flacChunk.byteLength && Flac.FLAC__stream_decoder_process_single(this.decoder)) {
let state = Flac.FLAC__stream_decoder_get_state(this.decoder);
// console.log("State: " + state);
//console.log("State: " + state);
}
// console.log("Pcm payload: " + this.pcmChunk!.payloadSize());
//console.log("Pcm payload: " + this.pcmChunk!.payloadSize());
if (this.cacheInfo.cachedBlocks > 0) {
let diffMs = this.cacheInfo.cachedBlocks / this.sampleFormat.msRate();
// console.log("Cached: " + this.cacheInfo.cachedBlocks + ", " + diffMs + "ms");
//console.log("Cached: " + this.cacheInfo.cachedBlocks + ", " + diffMs + "ms");
this.pcmChunk.timestamp.setMilliseconds(this.pcmChunk.timestamp.getMilliseconds() - diffMs);
}
return this.pcmChunk;
}
read_callback_fn(bufferSize) {
// console.log(' decode read callback, buffer bytes max=', bufferSize);
//console.log(' decode read callback, buffer bytes max=', bufferSize);
if (this.header) {
console.log(" header: " + this.header.byteLength);
//console.log(" header: " + this.header.byteLength);
let data = new Uint8Array(this.header);
this.header = null;
return { buffer: data, readDataLength: data.byteLength, error: false };
}
else if (this.flacChunk) {
// console.log(" flacChunk: " + this.flacChunk.byteLength);
//console.log(" flacChunk: " + this.flacChunk.byteLength);
// a fresh read => next call to write will not be from cached data
this.cacheInfo.isCachedChunk = false;
let data = new Uint8Array(this.flacChunk.slice(0, Math.min(bufferSize, this.flacChunk.byteLength)));
@ -624,7 +624,7 @@ class FlacDecoder extends Decoder {
return { buffer: new Uint8Array(0), readDataLength: 0, error: false };
}
write_callback_fn(data, frameInfo) {
// console.log(" write frame metadata: " + frameInfo + ", len: " + data.length);
//console.log(" write frame metadata: " + frameInfo + ", len: " + data.length);
if (this.cacheInfo.isCachedChunk) {
// there was no call to read, so it's some cached data
this.cacheInfo.cachedBlocks += frameInfo.blocksize;
@ -633,22 +633,22 @@ class FlacDecoder extends Decoder {
let view = new DataView(payload);
for (let channel = 0; channel < frameInfo.channels; ++channel) {
let channelData = new DataView(data[channel].buffer, 0, data[channel].buffer.byteLength);
// console.log("channelData: " + channelData.byteLength + ", blocksize: " + frameInfo.blocksize);
//console.log("channelData: " + channelData.byteLength + ", blocksize: " + frameInfo.blocksize);
for (let i = 0; i < frameInfo.blocksize; ++i) {
view.setInt16(2 * (frameInfo.channels * i + channel), channelData.getInt16(2 * i, true), true);
}
}
this.pcmChunk.addPayload(payload);
// console.log("write: " + payload.byteLength + ", len: " + this.pcmChunk!.payloadSize());
//console.log("write: " + payload.byteLength + ", len: " + this.pcmChunk!.payloadSize());
}
/** @memberOf decode */
metadata_callback_fn(data) {
console.info('meta data: ', data);
//console.info('meta data: ', data);
// let view = new DataView(data);
this.sampleFormat.rate = data.sampleRate;
this.sampleFormat.channels = data.channels;
this.sampleFormat.bits = data.bitsPerSample;
console.log("metadata_callback_fn, sampleformat: " + this.sampleFormat.toString());
//console.log("metadata_callback_fn, sampleformat: " + this.sampleFormat.toString());
}
/** @memberOf decode */
error_callback_fn(err, errMsg) {
@ -712,7 +712,7 @@ class SnapStream {
let type = view.getUint16(0, true);
if (type == 1) {
let codec = new CodecMessage(msg.data);
console.log("Codec: " + codec.codec);
//console.log("Codec: " + codec.codec);
if (codec.codec == "flac") {
this.decoder = new FlacDecoder();
}
@ -744,7 +744,7 @@ class SnapStream {
this.gainNode.gain.value = this.serverSettings.muted ? 0 : this.serverSettings.volumePercent / 100;
// this.timeProvider = new TimeProvider(this.ctx);
this.stream = new AudioStream(this.timeProvider, this.sampleFormat, this.bufferMs);
console.log("Base latency: " + this.ctx.baseLatency + ", output latency: " + this.ctx.outputLatency);
//console.log("Base latency: " + this.ctx.baseLatency + ", output latency: " + this.ctx.outputLatency);
this.play();
}
}
@ -764,14 +764,14 @@ class SnapStream {
this.gainNode.gain.value = this.serverSettings.muted ? 0 : this.serverSettings.volumePercent / 100;
}
this.bufferMs = this.serverSettings.bufferMs - this.serverSettings.latency;
console.log("ServerSettings bufferMs: " + this.serverSettings.bufferMs + ", latency: " + this.serverSettings.latency + ", volume: " + this.serverSettings.volumePercent + ", muted: " + this.serverSettings.muted);
//console.log("ServerSettings bufferMs: " + this.serverSettings.bufferMs + ", latency: " + this.serverSettings.latency + ", volume: " + this.serverSettings.volumePercent + ", muted: " + this.serverSettings.muted);
}
else if (type == 4) {
if (this.timeProvider) {
let time = new TimeMessage(msg.data);
this.timeProvider.setDiff(time.latency.getMilliseconds(), this.timeProvider.now() - time.sent.getMilliseconds());
}
// console.log("Time sec: " + time.latency.sec + ", usec: " + time.latency.usec + ", diff: " + this.timeProvider.diff);
//console.log("Time sec: " + time.latency.sec + ", usec: " + time.latency.usec + ", diff: " + this.timeProvider.diff);
}
else {
console.info("Message not handled, type: " + type);
@ -814,10 +814,10 @@ class SnapStream {
let t = new TimeMessage();
t.latency.setMilliseconds(this.timeProvider.now());
this.sendMessage(t);
// console.log("prepareSource median: " + Math.round(this.median * 10) / 10);
//console.log("prepareSource median: " + Math.round(this.median * 10) / 10);
}
stopAudio() {
if (this.ctx) {
if (this.ctx && this.ctx.state !== 'closed') {
this.ctx.close();
}
while (this.audioBuffers.length > 0) {