Improve Handy integration (#2555)

* Refactor interactive into context
* Stop the interactive device when leaving page
* Show interactive state if not ready
* Handle navigation and looping
This commit is contained in:
WithoutPants 2022-05-10 16:38:34 +10:00 committed by GitHub
parent bc85614ff9
commit ea2fcd9d7f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 575 additions and 65 deletions

View File

@ -35,6 +35,7 @@ import * as GQL from "./core/generated-graphql";
import { LoadingIndicator, TITLE_SUFFIX } from "./components/Shared";
import { ConfigurationProvider } from "./hooks/Config";
import { ManualProvider } from "./components/Help/Manual";
import { InteractiveProvider } from "./hooks/Interactive/context";
initPolyfills();
@ -150,12 +151,14 @@ export const App: React.FC = () => {
<ToastProvider>
<LightboxProvider>
<ManualProvider>
<Helmet
titleTemplate={`%s ${TITLE_SUFFIX}`}
defaultTitle="Stash"
/>
{maybeRenderNavbar()}
<div className="main container-fluid">{renderContent()}</div>
<InteractiveProvider>
<Helmet
titleTemplate={`%s ${TITLE_SUFFIX}`}
defaultTitle="Stash"
/>
{maybeRenderNavbar()}
<div className="main container-fluid">{renderContent()}</div>
</InteractiveProvider>
</ManualProvider>
</LightboxProvider>
</ToastProvider>

View File

@ -1,13 +1,16 @@
### ✨ New Features
* Show Handy status on scene player where applicable. ([#2555](https://github.com/stashapp/stash/pull/2555))
* Added recommendations to home page. ([#2571](https://github.com/stashapp/stash/pull/2571))
* Add support for VTT and SRT captions for scenes. ([#2462](https://github.com/stashapp/stash/pull/2462))
* Added option to require a number of scroll attempts before navigating to next/previous image in Lightbox. ([#2544](https://github.com/stashapp/stash/pull/2544))
### 🎨 Improvements
* Added Handy server sync button to Interface settings page. ([#2555](https://github.com/stashapp/stash/pull/2555))
* Changed playback rate options to be the same as those provided by YouTube. ([#2550](https://github.com/stashapp/stash/pull/2550))
* Display error message on fatal error when running stash with double-click in Windows. ([#2543](https://github.com/stashapp/stash/pull/2543))
### 🐛 Bug fixes
* Fix long Handy initialisation delay. ([#2555](https://github.com/stashapp/stash/pull/2555))
* Fix lightbox autoplaying while offscreen. ([#2563](https://github.com/stashapp/stash/pull/2563))
* Fix playback rate resetting when seeking. ([#2550](https://github.com/stashapp/stash/pull/2550))
* Fix video not starting when clicking scene scrubber. ([#2546](https://github.com/stashapp/stash/pull/2546))

View File

@ -1,5 +1,11 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import React, { useContext, useEffect, useRef, useState } from "react";
import React, {
useCallback,
useContext,
useEffect,
useRef,
useState,
} from "react";
import VideoJS, { VideoJsPlayer, VideoJsPlayerOptions } from "video.js";
import "videojs-vtt-thumbnails-freetube";
import "videojs-seek-buttons";
@ -15,7 +21,11 @@ import cx from "classnames";
import * as GQL from "src/core/generated-graphql";
import { ScenePlayerScrubber } from "./ScenePlayerScrubber";
import { ConfigurationContext } from "src/hooks/Config";
import { Interactive } from "src/utils/interactive";
import {
ConnectionState,
InteractiveContext,
} from "src/hooks/Interactive/context";
import { SceneInteractiveStatus } from "src/hooks/Interactive/status";
import { languageMap } from "src/utils/caption";
export const VIDEO_PLAYER_ID = "VideoJsPlayer";
@ -117,11 +127,18 @@ export const ScenePlayer: React.FC<IScenePlayerProps> = ({
const [time, setTime] = useState(0);
const [interactiveClient] = useState(
new Interactive(config?.handyKey || "", config?.funscriptOffset || 0)
);
const {
interactive: interactiveClient,
uploadScript,
currentScript,
initialised: interactiveInitialised,
state: interactiveState,
} = React.useContext(InteractiveContext);
const [initialTimestamp] = useState(timestamp);
const [ready, setReady] = useState(false);
const started = useRef(false);
const interactiveReady = useRef(false);
const maxLoopDuration = config?.maximumLoopDuration ?? 0;
@ -188,10 +205,18 @@ export const ScenePlayer: React.FC<IScenePlayerProps> = ({
}, []);
useEffect(() => {
if (scene?.interactive) {
interactiveClient.uploadScript(scene.paths.funscript || "");
if (scene?.interactive && interactiveInitialised) {
interactiveReady.current = false;
uploadScript(scene.paths.funscript || "").then(() => {
interactiveReady.current = true;
});
}
}, [interactiveClient, scene?.interactive, scene?.paths.funscript]);
}, [
uploadScript,
interactiveInitialised,
scene?.interactive,
scene?.paths.funscript,
]);
useEffect(() => {
if (skipButtonsRef.current) {
@ -222,6 +247,24 @@ export const ScenePlayer: React.FC<IScenePlayerProps> = ({
};
}, []);
const start = useCallback(() => {
const player = playerRef.current;
if (player && scene) {
started.current = true;
player
.play()
?.then(() => {
if (initialTimestamp > 0) {
player.currentTime(initialTimestamp);
}
})
.catch(() => {
if (scene.paths.screenshot) player.poster(scene.paths.screenshot);
});
}
}, [scene, initialTimestamp]);
useEffect(() => {
let prevCaptionOffset = 0;
@ -374,6 +417,10 @@ export const ScenePlayer: React.FC<IScenePlayerProps> = ({
}
}
// always stop the interactive client on initialisation
interactiveClient.pause();
interactiveReady.current = false;
if (!scene || scene.id === sceneId.current) return;
sceneId.current = scene.id;
@ -420,80 +467,75 @@ export const ScenePlayer: React.FC<IScenePlayerProps> = ({
player.currentTime(0);
player.loop(
const looping =
!!scene.file.duration &&
maxLoopDuration !== 0 &&
scene.file.duration < maxLoopDuration
);
maxLoopDuration !== 0 &&
scene.file.duration < maxLoopDuration;
player.loop(looping);
interactiveClient.setLooping(looping);
player.on("loadstart", function (this: VideoJsPlayer) {
function loadstart(this: VideoJsPlayer) {
// handle offset after loading so that we get the correct current source
handleOffset(this);
});
}
player.on("play", function (this: VideoJsPlayer) {
player.poster("");
if (scene.interactive) {
player.on("loadstart", loadstart);
function onPlay(this: VideoJsPlayer) {
this.poster("");
if (scene?.interactive && interactiveReady.current) {
interactiveClient.play(this.currentTime());
}
});
}
player.on("play", onPlay);
player.on("pause", () => {
if (scene.interactive) {
interactiveClient.pause();
}
});
function pause() {
interactiveClient.pause();
}
player.on("pause", pause);
player.on("timeupdate", function (this: VideoJsPlayer) {
if (scene.interactive) {
function timeupdate(this: VideoJsPlayer) {
if (scene?.interactive && interactiveReady.current) {
interactiveClient.ensurePlaying(this.currentTime());
}
setTime(this.currentTime());
});
}
player.on("timeupdate", timeupdate);
player.on("seeking", function (this: VideoJsPlayer) {
function seeking(this: VideoJsPlayer) {
this.play();
});
}
player.on("seeking", seeking);
player.on("error", () => {
function error() {
handleError(true);
});
}
player.on("error", error);
// changing source (eg when seeking) resets the playback rate
// so set the default in addition to the current rate
player.on("ratechange", function (this: VideoJsPlayer) {
function ratechange(this: VideoJsPlayer) {
this.defaultPlaybackRate(this.playbackRate());
});
}
player.on("ratechange", ratechange);
player.on("loadedmetadata", () => {
if (!player.videoWidth() && !player.videoHeight()) {
function loadedmetadata(this: VideoJsPlayer) {
if (!this.videoWidth() && !this.videoHeight()) {
// Occurs during preload when videos with supported audio/unsupported video are preloaded.
// Treat this as a decoding error and try the next source without playing.
// However on Safari we get an media event when m3u8 is loaded which needs to be ignored.
const currentFile = player.currentSrc();
const currentFile = this.currentSrc();
if (currentFile != null && !currentFile.includes("m3u8")) {
// const play = !player.paused();
// handleError(play);
player.error(MediaError.MEDIA_ERR_SRC_NOT_SUPPORTED);
this.error(MediaError.MEDIA_ERR_SRC_NOT_SUPPORTED);
}
}
});
}
player.on("loadedmetadata", loadedmetadata);
player.load();
if (auto) {
player
.play()
?.then(() => {
if (initialTimestamp > 0) {
player.currentTime(initialTimestamp);
}
})
.catch(() => {
if (scene.paths.screenshot) player.poster(scene.paths.screenshot);
});
}
if ((player as any).vttThumbnails?.src)
(player as any).vttThumbnails?.src(scene?.paths.vtt);
else
@ -501,6 +543,25 @@ export const ScenePlayer: React.FC<IScenePlayerProps> = ({
src: scene?.paths.vtt,
showTimestamp: true,
});
setReady(true);
started.current = false;
return () => {
setReady(false);
// stop the interactive client
interactiveClient.pause();
player.off("loadstart", loadstart);
player.off("play", onPlay);
player.off("pause", pause);
player.off("timeupdate", timeupdate);
player.off("seeking", seeking);
player.off("error", error);
player.off("ratechange", ratechange);
player.off("loadedmetadata", loadedmetadata);
};
}, [
scene,
config?.autostartVideo,
@ -508,6 +569,35 @@ export const ScenePlayer: React.FC<IScenePlayerProps> = ({
initialTimestamp,
autoplay,
interactiveClient,
start,
]);
useEffect(() => {
if (!ready || started.current) {
return;
}
const auto =
autoplay || (config?.autostartVideo ?? false) || initialTimestamp > 0;
// check if we're waiting for the interactive client
const interactiveWaiting =
scene?.interactive &&
interactiveClient.handyKey &&
currentScript !== scene.paths.funscript;
if (scene && auto && !interactiveWaiting) {
start();
}
}, [
config?.autostartVideo,
initialTimestamp,
scene,
ready,
interactiveClient,
currentScript,
autoplay,
start,
]);
useEffect(() => {
@ -550,6 +640,9 @@ export const ScenePlayer: React.FC<IScenePlayerProps> = ({
className="video-js vjs-big-play-centered"
/>
</div>
{scene?.interactive &&
(interactiveState !== ConnectionState.Ready ||
playerRef.current?.paused()) && <SceneInteractiveStatus />}
{scene && (
<ScenePlayerScrubber
scene={scene}

View File

@ -1,6 +1,6 @@
import React from "react";
import { Form } from "react-bootstrap";
import { useIntl } from "react-intl";
import { Button, Form } from "react-bootstrap";
import { FormattedMessage, useIntl } from "react-intl";
import { DurationInput, LoadingIndicator } from "src/components/Shared";
import { CheckboxGroup } from "./CheckboxGroup";
import { SettingSection } from "../SettingSection";
@ -19,6 +19,11 @@ import {
imageLightboxScrollModeIntlMap,
} from "src/core/enums";
import { useInterfaceLocalForage } from "src/hooks";
import {
ConnectionState,
connectionStateLabel,
InteractiveContext,
} from "src/hooks/Interactive/context";
const allMenuItems = [
{ id: "scenes", headingID: "scenes" },
@ -38,6 +43,16 @@ export const SettingsInterfacePanel: React.FC = () => {
SettingStateContext
);
const {
interactive,
state: interactiveState,
error: interactiveError,
serverOffset: interactiveServerOffset,
initialised: interactiveInitialised,
initialise: initialiseInteractive,
sync: interactiveSync,
} = React.useContext(InteractiveContext);
const [, setInterfaceLocalForage] = useInterfaceLocalForage();
function saveLightboxSettings(v: Partial<GQL.ConfigImageLightboxInput>) {
@ -397,6 +412,70 @@ export const SettingsInterfacePanel: React.FC = () => {
value={iface.handyKey ?? undefined}
onChange={(v) => saveInterface({ handyKey: v })}
/>
{interactive.handyKey && (
<>
<div className="setting" id="handy-status">
<div>
<h3>
{intl.formatMessage({
id: "config.ui.handy_connection.status.heading",
})}
</h3>
<div className="value">
<FormattedMessage
id={connectionStateLabel(interactiveState)}
/>
{interactiveError && <span>: {interactiveError}</span>}
</div>
</div>
<div>
{!interactiveInitialised && (
<Button
disabled={
interactiveState === ConnectionState.Connecting ||
interactiveState === ConnectionState.Syncing
}
onClick={() => initialiseInteractive()}
>
{intl.formatMessage({
id: "config.ui.handy_connection.connect",
})}
</Button>
)}
</div>
</div>
<div className="setting" id="handy-server-offset">
<div>
<h3>
{intl.formatMessage({
id: "config.ui.handy_connection.server_offset.heading",
})}
</h3>
<div className="value">
{interactiveServerOffset.toFixed()}ms
</div>
</div>
<div>
{interactiveInitialised && (
<Button
disabled={
!interactiveInitialised ||
interactiveState === ConnectionState.Syncing
}
onClick={() => interactiveSync()}
>
{intl.formatMessage({
id: "config.ui.handy_connection.sync",
})}
</Button>
)}
</div>
</div>
</>
)}
<NumberSetting
headingID="config.ui.funscript_offset.heading"
subHeadingID="config.ui.funscript_offset.description"

View File

@ -0,0 +1,195 @@
import React, { useCallback, useEffect, useState } from "react";
import { ConfigurationContext } from "../Config";
import { useLocalForage } from "../LocalForage";
import { Interactive as InteractiveAPI } from "./interactive";
export enum ConnectionState {
Missing,
Disconnected,
Error,
Connecting,
Syncing,
Uploading,
Ready,
}
export function connectionStateLabel(s: ConnectionState) {
const prefix = "handy_connection_status";
switch (s) {
case ConnectionState.Missing:
return `${prefix}.missing`;
case ConnectionState.Connecting:
return `${prefix}.connecting`;
case ConnectionState.Disconnected:
return `${prefix}.disconnected`;
case ConnectionState.Error:
return `${prefix}.error`;
case ConnectionState.Syncing:
return `${prefix}.syncing`;
case ConnectionState.Uploading:
return `${prefix}.uploading`;
case ConnectionState.Ready:
return `${prefix}.ready`;
}
}
export interface IState {
interactive: InteractiveAPI;
state: ConnectionState;
serverOffset: number;
initialised: boolean;
currentScript?: string;
error?: string;
initialise: () => Promise<void>;
uploadScript: (funscriptPath: string) => Promise<void>;
sync: () => Promise<void>;
}
export const InteractiveContext = React.createContext<IState>({
interactive: new InteractiveAPI("", 0),
state: ConnectionState.Missing,
serverOffset: 0,
initialised: false,
initialise: () => {
return Promise.resolve();
},
uploadScript: () => {
return Promise.resolve();
},
sync: () => {
return Promise.resolve();
},
});
const LOCAL_FORAGE_KEY = "interactive";
interface IInteractiveState {
serverOffset: number;
}
export const InteractiveProvider: React.FC = ({ children }) => {
const [{ data: config }, setConfig] = useLocalForage<IInteractiveState>(
LOCAL_FORAGE_KEY,
{ serverOffset: 0 }
);
const { configuration: stashConfig } = React.useContext(ConfigurationContext);
const [state, setState] = useState<ConnectionState>(ConnectionState.Missing);
const [handyKey, setHandyKey] = useState<string | undefined>(undefined);
const [currentScript, setCurrentScript] = useState<string | undefined>(
undefined
);
const [scriptOffset, setScriptOffset] = useState<number>(0);
const [interactive] = useState<InteractiveAPI>(new InteractiveAPI("", 0));
const [initialised, setInitialised] = useState(false);
const [error, setError] = useState<string | undefined>();
const initialise = useCallback(async () => {
setError(undefined);
if (!config?.serverOffset) {
setState(ConnectionState.Syncing);
const offset = await interactive.sync();
setConfig({ serverOffset: offset });
setState(ConnectionState.Ready);
setInitialised(true);
} else {
interactive.setServerTimeOffset(config.serverOffset);
setState(ConnectionState.Connecting);
try {
await interactive.connect();
setState(ConnectionState.Ready);
setInitialised(true);
} catch (e) {
if (e instanceof Error) {
setError(e.message ?? e.toString());
setState(ConnectionState.Error);
}
}
}
}, [config, interactive, setConfig]);
useEffect(() => {
if (!stashConfig) {
return;
}
setHandyKey(stashConfig.interface.handyKey ?? undefined);
setScriptOffset(stashConfig.interface.funscriptOffset ?? 0);
}, [stashConfig]);
useEffect(() => {
if (!config) {
return;
}
const oldKey = interactive.handyKey;
interactive.handyKey = handyKey ?? "";
interactive.scriptOffset = scriptOffset;
if (oldKey !== interactive.handyKey && interactive.handyKey) {
initialise();
}
}, [handyKey, scriptOffset, config, interactive, initialise]);
const sync = useCallback(async () => {
if (
!interactive.handyKey ||
state === ConnectionState.Syncing ||
!initialised
) {
return;
}
setState(ConnectionState.Syncing);
const offset = await interactive.sync();
setConfig({ serverOffset: offset });
setState(ConnectionState.Ready);
}, [interactive, state, setConfig, initialised]);
const uploadScript = useCallback(
async (funscriptPath: string) => {
interactive.pause();
if (
!interactive.handyKey ||
!funscriptPath ||
funscriptPath === currentScript
) {
return Promise.resolve();
}
setState(ConnectionState.Uploading);
try {
await interactive.uploadScript(funscriptPath);
setCurrentScript(funscriptPath);
setState(ConnectionState.Ready);
} catch (e) {
setState(ConnectionState.Error);
}
},
[interactive, currentScript]
);
return (
<InteractiveContext.Provider
value={{
interactive,
state,
error,
currentScript,
serverOffset: config?.serverOffset ?? 0,
initialised,
initialise,
uploadScript,
sync,
}}
>
{children}
</InteractiveContext.Provider>
);
};
export default InteractiveProvider;

View File

@ -0,0 +1,35 @@
div.scene-interactive-status {
opacity: 0.75;
padding: 0.75rem;
position: absolute;
&.interactive-status-disconnected,
&.interactive-status-error svg {
color: $danger;
}
&.interactive-status-connecting svg,
&.interactive-status-syncing svg,
&.interactive-status-uploading svg {
animation: 1s ease 0s infinite alternate fadepulse;
color: $warning;
}
&.interactive-status-ready svg {
color: $success;
}
.status-text {
margin-left: 0.5rem;
}
}
@keyframes fadepulse {
0% {
opacity: 0.4;
}
100% {
opacity: 1;
}
}

View File

@ -3,6 +3,7 @@ import {
HandyMode,
HsspSetupResult,
CsvUploadResponse,
HandyFirmwareStatus,
} from "thehandy/lib/types";
interface IFunscript {
@ -92,10 +93,10 @@ async function uploadCsv(
// Interactive currently uses the Handy API, but could be expanded to use buttplug.io
// via buttplugio/buttplug-rs-ffi's WASM module.
export class Interactive {
private _connected: boolean;
private _playing: boolean;
private _scriptOffset: number;
private _handy: Handy;
_connected: boolean;
_playing: boolean;
_scriptOffset: number;
_handy: Handy;
constructor(handyKey: string, scriptOffset: number) {
this._handy = new Handy();
@ -105,23 +106,42 @@ export class Interactive {
this._playing = false;
}
async connect() {
const connected = await this._handy.getConnected();
if (!connected) {
throw new Error("Handy not connected");
}
// check the firmware and make sure it's compatible
const info = await this._handy.getInfo();
if (info.fwStatus === HandyFirmwareStatus.updateRequired) {
throw new Error("Handy firmware update required");
}
}
set handyKey(key: string) {
this._handy.connectionKey = key;
}
get handyKey(): string {
return this._handy.connectionKey;
}
set scriptOffset(offset: number) {
this._scriptOffset = offset;
}
async uploadScript(funscriptPath: string) {
if (!(this._handy.connectionKey && funscriptPath)) {
return;
}
// Calibrates the latency between the browser client and the Handy server's
// This is done before a script upload to ensure a synchronized experience
await this._handy.getServerTimeOffset();
const csv = await fetch(funscriptPath)
.then((response) => response.json())
.then((json) => convertFunscriptToCSV(json));
const fileName = `${Math.round(Math.random() * 100000000)}.csv`;
const csvFile = new File([csv], fileName);
const tempURL = await uploadCsv(csvFile).then((response) => response.url);
await this._handy.setMode(HandyMode.hssp);
@ -131,6 +151,14 @@ export class Interactive {
.then((result) => result === HsspSetupResult.downloaded);
}
async sync() {
return this._handy.getServerTimeOffset();
}
setServerTimeOffset(offset: number) {
this._handy.estimatedServerTimeOffset = offset;
}
async play(position: number) {
if (!this._connected) {
return;
@ -157,4 +185,11 @@ export class Interactive {
}
await this.play(position);
}
async setLooping(looping: boolean) {
if (!this._connected) {
return;
}
this._handy.setHsspLoop(looping);
}
}

View File

@ -0,0 +1,47 @@
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import React from "react";
import { FormattedMessage } from "react-intl";
import {
ConnectionState,
connectionStateLabel,
InteractiveContext,
} from "./context";
export const SceneInteractiveStatus: React.FC = ({}) => {
const { state, error } = React.useContext(InteractiveContext);
function getStateClass() {
switch (state) {
case ConnectionState.Connecting:
return "interactive-status-connecting";
case ConnectionState.Disconnected:
return "interactive-status-disconnected";
case ConnectionState.Error:
return "interactive-status-error";
case ConnectionState.Syncing:
return "interactive-status-uploading";
case ConnectionState.Uploading:
return "interactive-status-syncing";
case ConnectionState.Ready:
return "interactive-status-ready";
}
return "";
}
if (state === ConnectionState.Missing) {
return <></>;
}
return (
<div className={`scene-interactive-status ${getStateClass()}`}>
<FontAwesomeIcon pulse icon="circle" size="xs" />
<span className="status-text">
<FormattedMessage id={connectionStateLabel(state)} />
{error && <span>: {error}</span>}
</span>
</div>
);
};
export default SceneInteractiveStatus;

View File

@ -20,6 +20,7 @@
@import "src/components/Wall/styles.scss";
@import "src/components/Tagger/styles.scss";
@import "src/hooks/Lightbox/lightbox.scss";
@import "src/hooks/Interactive/interactive.scss";
@import "src/components/Dialogs/IdentifyDialog/styles.scss";
@import "src/components/Dialogs/styles.scss";
@import "../node_modules/flag-icon-css/css/flag-icon.min.css";

View File

@ -454,6 +454,16 @@
"description": "Handy connection key to use for interactive scenes. Setting this key will allow Stash to share your current scene information with handyfeeling.com",
"heading": "Handy Connection Key"
},
"handy_connection": {
"connect": "Connect",
"server_offset": {
"heading": "Server Offset"
},
"status": {
"heading": "Handy Connection Status"
},
"sync": "Sync"
},
"images": {
"heading": "Images",
"options": {
@ -736,6 +746,15 @@
"NON_BINARY": "Non-Binary"
},
"hair_color": "Hair Colour",
"handy_connection_status": {
"connecting": "Connecting",
"disconnected": "Disconnected",
"error": "Error connecting to Handy",
"missing": "Missing",
"ready": "Ready",
"syncing": "Syncing with server",
"uploading": "Uploading script"
},
"hasMarkers": "Has Markers",
"height": "Height",
"help": "Help",