From ef46451b80709d852776d859aa17e8d7d498f2ff Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sat, 17 Aug 2024 13:16:48 -0500 Subject: [PATCH] Live player fixes (#13143) * Jump to live when exceeding buffer time threshold in MSE player * clean up * Try adjusting playback rate instead of jumping to live * clean up * fallback to webrtc if enabled before jsmpeg * baseline * clean up * remove comments * adaptive playback rate and intelligent switching improvements * increase logging and reset live mode after camera is no longer active on dashboard only * jump to live on safari/iOS * clean up * clean up * refactor camera live mode hook * remove key listener * resolve conflicts --- web/src/components/player/LivePlayer.tsx | 14 +-- web/src/components/player/MsePlayer.tsx | 140 +++++++++++++++++++-- web/src/hooks/use-camera-live-mode.ts | 86 +++++++------ web/src/types/frigateConfig.ts | 7 +- web/src/views/live/DraggableGridLayout.tsx | 37 ++---- web/src/views/live/LiveCameraView.tsx | 29 +++-- web/src/views/live/LiveDashboardView.tsx | 35 +----- 7 files changed, 228 insertions(+), 120 deletions(-) diff --git a/web/src/components/player/LivePlayer.tsx b/web/src/components/player/LivePlayer.tsx index 67057a278..9a0b6f3db 100644 --- a/web/src/components/player/LivePlayer.tsx +++ b/web/src/components/player/LivePlayer.tsx @@ -13,7 +13,6 @@ import { LivePlayerMode, VideoResolutionType, } from "@/types/live"; -import useCameraLiveMode from "@/hooks/use-camera-live-mode"; import { getIconForLabel } from "@/utils/iconUtil"; import Chip from "../indicators/Chip"; import { capitalizeFirstLetter } from "@/utils/stringUtil"; @@ -25,7 +24,7 @@ type LivePlayerProps = { containerRef?: React.MutableRefObject; className?: string; cameraConfig: CameraConfig; - preferredLiveMode?: LivePlayerMode; + preferredLiveMode: LivePlayerMode; showStillWithoutActivity?: boolean; windowVisible?: boolean; playAudio?: boolean; @@ -36,6 +35,7 @@ type LivePlayerProps = { onClick?: () => void; setFullResolution?: React.Dispatch>; onError?: (error: LivePlayerError) => void; + onResetLiveMode?: () => void; }; export default function LivePlayer({ @@ -54,6 +54,7 @@ export default function LivePlayer({ onClick, setFullResolution, onError, + onResetLiveMode, }: LivePlayerProps) { const internalContainerRef = useRef(null); // camera activity @@ -70,8 +71,6 @@ export default function LivePlayer({ // camera live state - const liveMode = useCameraLiveMode(cameraConfig, preferredLiveMode); - const [liveReady, setLiveReady] = useState(false); const liveReadyRef = useRef(liveReady); @@ -91,6 +90,7 @@ export default function LivePlayer({ const timer = setTimeout(() => { if (liveReadyRef.current && !cameraActiveRef.current) { setLiveReady(false); + onResetLiveMode?.(); } }, 500); @@ -152,7 +152,7 @@ export default function LivePlayer({ let player; if (!autoLive) { player = null; - } else if (liveMode == "webrtc") { + } else if (preferredLiveMode == "webrtc") { player = ( ); - } else if (liveMode == "mse") { + } else if (preferredLiveMode == "mse") { if ("MediaSource" in window || "ManagedMediaSource" in window) { player = ( ); } - } else if (liveMode == "jsmpeg") { + } else if (preferredLiveMode == "jsmpeg") { if (cameraActive || !showStillWithoutActivity || liveReady) { player = ( ([]); + const bufferIndex = useRef(0); const [wsState, setWsState] = useState(WebSocket.CLOSED); const [connectTS, setConnectTS] = useState(0); @@ -133,6 +139,13 @@ function MSEPlayer({ } }, [bufferTimeout]); + const handlePause = useCallback(() => { + // don't let the user pause the live stream + if (isPlaying && playbackEnabled) { + videoRef.current?.play(); + } + }, [isPlaying, playbackEnabled]); + const onOpen = () => { setWsState(WebSocket.OPEN); @@ -193,6 +206,7 @@ function MSEPlayer({ const onMse = () => { if ("ManagedMediaSource" in window) { + // safari const MediaSource = window.ManagedMediaSource; msRef.current?.addEventListener( @@ -224,6 +238,7 @@ function MSEPlayer({ videoRef.current.srcObject = msRef.current; } } else { + // non safari msRef.current?.addEventListener( "sourceopen", () => { @@ -247,15 +262,35 @@ function MSEPlayer({ }, { once: true }, ); - videoRef.current!.src = URL.createObjectURL(msRef.current!); - videoRef.current!.srcObject = null; + if (videoRef.current && msRef.current) { + videoRef.current.src = URL.createObjectURL(msRef.current); + videoRef.current.srcObject = null; + } } play(); onmessageRef.current["mse"] = (msg) => { if (msg.type !== "mse") return; - const sb = msRef.current?.addSourceBuffer(msg.value); + let sb: SourceBuffer | undefined; + try { + sb = msRef.current?.addSourceBuffer(msg.value); + if (sb?.mode) { + sb.mode = "segments"; + } + } catch (e) { + // Safari sometimes throws this error + if (e instanceof DOMException && e.name === "InvalidStateError") { + if (wsRef.current) { + onDisconnect(); + } + onError?.("mse-decode"); + return; + } else { + throw e; // Re-throw if it's not the error we're handling + } + } + sb?.addEventListener("updateend", () => { if (sb.updating) return; @@ -302,6 +337,43 @@ function MSEPlayer({ return video.buffered.end(video.buffered.length - 1) - video.currentTime; }; + const jumpToLive = () => { + if (!videoRef.current) return; + + const buffered = videoRef.current.buffered; + if (buffered.length > 0) { + const liveEdge = buffered.end(buffered.length - 1); + // Jump to the live edge + videoRef.current.currentTime = liveEdge - 0.75; + lastJumpTimeRef.current = Date.now(); + } + }; + + const calculateAdaptiveBufferThreshold = () => { + const filledEntries = bufferTimes.current.length; + const sum = bufferTimes.current.reduce((a, b) => a + b, 0); + const averageBufferTime = filledEntries ? sum / filledEntries : 0; + return averageBufferTime * (isSafari || isIOS ? 3 : 1.5); + }; + + const calculateAdaptivePlaybackRate = ( + bufferTime: number, + bufferThreshold: number, + ) => { + const alpha = 0.2; // aggressiveness of playback rate increase + const beta = 0.5; // steepness of exponential growth + + // don't adjust playback rate if we're close enough to live + if ( + (bufferTime <= bufferThreshold && bufferThreshold < 3) || + bufferTime < 3 + ) { + return 1; + } + const rate = 1 + alpha * Math.exp(beta * bufferTime - bufferThreshold); + return Math.min(rate, 2); + }; + useEffect(() => { if (!playbackEnabled) { return; @@ -386,21 +458,71 @@ function MSEPlayer({ handleLoadedMetadata?.(); onPlaying?.(); setIsPlaying(true); + lastJumpTimeRef.current = Date.now(); }} muted={!audioEnabled} - onPause={() => videoRef.current?.play()} + onPause={handlePause} onProgress={() => { + const bufferTime = getBufferedTime(videoRef.current); + + if ( + videoRef.current && + (videoRef.current.playbackRate === 1 || bufferTime < 3) + ) { + if (bufferTimes.current.length < MAX_BUFFER_ENTRIES) { + bufferTimes.current.push(bufferTime); + } else { + bufferTimes.current[bufferIndex.current] = bufferTime; + bufferIndex.current = + (bufferIndex.current + 1) % MAX_BUFFER_ENTRIES; + } + } + + const bufferThreshold = calculateAdaptiveBufferThreshold(); + // if we have > 3 seconds of buffered data and we're still not playing, // something might be wrong - maybe codec issue, no audio, etc // so mark the player as playing so that error handlers will fire - if ( - !isPlaying && - playbackEnabled && - getBufferedTime(videoRef.current) > 3 - ) { + if (!isPlaying && playbackEnabled && bufferTime > 3) { setIsPlaying(true); + lastJumpTimeRef.current = Date.now(); onPlaying?.(); } + + // if we have more than 10 seconds of buffer, something's wrong so error out + if ( + isPlaying && + playbackEnabled && + (bufferThreshold > 10 || bufferTime > 10) + ) { + onDisconnect(); + onError?.("stalled"); + } + + const playbackRate = calculateAdaptivePlaybackRate( + bufferTime, + bufferThreshold, + ); + + // if we're above our rolling average threshold or have > 3 seconds of + // buffered data and we're playing, we may have drifted from actual live + // time, so increase playback rate to compensate - non safari/ios only + if ( + videoRef.current && + isPlaying && + playbackEnabled && + Date.now() - lastJumpTimeRef.current > BUFFERING_COOLDOWN_TIMEOUT + ) { + // Jump to live on Safari/iOS due to a change of playback rate causing re-buffering + if (isSafari || isIOS) { + if (bufferTime > 3) { + jumpToLive(); + } + } else { + videoRef.current.playbackRate = playbackRate; + } + } + if (onError != undefined) { if (videoRef.current?.paused) { return; diff --git a/web/src/hooks/use-camera-live-mode.ts b/web/src/hooks/use-camera-live-mode.ts index 2a1997045..edf165951 100644 --- a/web/src/hooks/use-camera-live-mode.ts +++ b/web/src/hooks/use-camera-live-mode.ts @@ -1,49 +1,65 @@ import { CameraConfig, FrigateConfig } from "@/types/frigateConfig"; -import { useMemo } from "react"; +import { useCallback, useEffect, useState } from "react"; import useSWR from "swr"; -import { usePersistence } from "./use-persistence"; import { LivePlayerMode } from "@/types/live"; export default function useCameraLiveMode( - cameraConfig: CameraConfig, - preferredMode?: LivePlayerMode, -): LivePlayerMode | undefined { + cameras: CameraConfig[], + windowVisible: boolean, +) { const { data: config } = useSWR("config"); + const [preferredLiveModes, setPreferredLiveModes] = useState<{ + [key: string]: LivePlayerMode; + }>({}); - const restreamEnabled = useMemo(() => { - if (!config) { - return false; - } + useEffect(() => { + if (!cameras) return; - return ( - cameraConfig && - Object.keys(config.go2rtc.streams || {}).includes( - cameraConfig.live.stream_name, - ) + const mseSupported = + "MediaSource" in window || "ManagedMediaSource" in window; + + const newPreferredLiveModes = cameras.reduce( + (acc, camera) => { + const isRestreamed = + config && + Object.keys(config.go2rtc.streams || {}).includes( + camera.live.stream_name, + ); + + if (!mseSupported) { + acc[camera.name] = isRestreamed ? "webrtc" : "jsmpeg"; + } else { + acc[camera.name] = isRestreamed ? "mse" : "jsmpeg"; + } + return acc; + }, + {} as { [key: string]: LivePlayerMode }, ); - }, [config, cameraConfig]); - const defaultLiveMode = useMemo(() => { - if (config) { - if (restreamEnabled) { - return preferredMode || "mse"; - } - return "jsmpeg"; - } + setPreferredLiveModes(newPreferredLiveModes); + }, [cameras, config, windowVisible]); - return undefined; - }, [config, preferredMode, restreamEnabled]); - const [viewSource] = usePersistence( - `${cameraConfig.name}-source`, - defaultLiveMode, + const resetPreferredLiveMode = useCallback( + (cameraName: string) => { + const mseSupported = + "MediaSource" in window || "ManagedMediaSource" in window; + const isRestreamed = + config && Object.keys(config.go2rtc.streams || {}).includes(cameraName); + + setPreferredLiveModes((prevModes) => { + const newModes = { ...prevModes }; + + if (!mseSupported) { + newModes[cameraName] = isRestreamed ? "webrtc" : "jsmpeg"; + } else { + newModes[cameraName] = isRestreamed ? "mse" : "jsmpeg"; + } + + return newModes; + }); + }, + [config], ); - if ( - restreamEnabled && - (preferredMode == "mse" || preferredMode == "webrtc") - ) { - return preferredMode; - } else { - return viewSource; - } + return { preferredLiveModes, setPreferredLiveModes, resetPreferredLiveMode }; } diff --git a/web/src/types/frigateConfig.ts b/web/src/types/frigateConfig.ts index 26228dbaa..a38cbd847 100644 --- a/web/src/types/frigateConfig.ts +++ b/web/src/types/frigateConfig.ts @@ -298,7 +298,12 @@ export interface FrigateConfig { retry_interval: number; }; - go2rtc: Record; + go2rtc: { + streams: string[]; + webrtc: { + candidates: string[]; + }; + }; camera_groups: { [groupName: string]: CameraGroupConfig }; diff --git a/web/src/views/live/DraggableGridLayout.tsx b/web/src/views/live/DraggableGridLayout.tsx index 165829719..fc2d9bb52 100644 --- a/web/src/views/live/DraggableGridLayout.tsx +++ b/web/src/views/live/DraggableGridLayout.tsx @@ -41,6 +41,7 @@ import { TooltipContent, } from "@/components/ui/tooltip"; import { Toaster } from "@/components/ui/sonner"; +import useCameraLiveMode from "@/hooks/use-camera-live-mode"; type DraggableGridLayoutProps = { cameras: CameraConfig[]; @@ -75,36 +76,8 @@ export default function DraggableGridLayout({ // preferred live modes per camera - const [preferredLiveModes, setPreferredLiveModes] = useState<{ - [key: string]: LivePlayerMode; - }>({}); - - useEffect(() => { - if (!cameras) return; - - const mseSupported = - "MediaSource" in window || "ManagedMediaSource" in window; - - const newPreferredLiveModes = cameras.reduce( - (acc, camera) => { - const isRestreamed = - config && - Object.keys(config.go2rtc.streams || {}).includes( - camera.live.stream_name, - ); - - if (!mseSupported) { - acc[camera.name] = isRestreamed ? "webrtc" : "jsmpeg"; - } else { - acc[camera.name] = isRestreamed ? "mse" : "jsmpeg"; - } - return acc; - }, - {} as { [key: string]: LivePlayerMode }, - ); - - setPreferredLiveModes(newPreferredLiveModes); - }, [cameras, config, windowVisible]); + const { preferredLiveModes, setPreferredLiveModes, resetPreferredLiveMode } = + useCameraLiveMode(cameras, windowVisible); const ResponsiveGridLayout = useMemo(() => WidthProvider(Responsive), []); @@ -477,6 +450,7 @@ export default function DraggableGridLayout({ return newModes; }); }} + onResetLiveMode={() => resetPreferredLiveMode(camera.name)} > {isEditMode && showCircles && } @@ -635,6 +609,7 @@ type LivePlayerGridItemProps = { preferredLiveMode: LivePlayerMode; onClick: () => void; onError: (e: LivePlayerError) => void; + onResetLiveMode: () => void; }; const LivePlayerGridItem = React.forwardRef< @@ -655,6 +630,7 @@ const LivePlayerGridItem = React.forwardRef< preferredLiveMode, onClick, onError, + onResetLiveMode, ...props }, ref, @@ -676,6 +652,7 @@ const LivePlayerGridItem = React.forwardRef< preferredLiveMode={preferredLiveMode} onClick={onClick} onError={onError} + onResetLiveMode={onResetLiveMode} containerRef={ref as React.RefObject} /> {children} diff --git a/web/src/views/live/LiveCameraView.tsx b/web/src/views/live/LiveCameraView.tsx index db162198c..9dab7d916 100644 --- a/web/src/views/live/LiveCameraView.tsx +++ b/web/src/views/live/LiveCameraView.tsx @@ -227,6 +227,10 @@ export default function LiveCameraView({ return "webrtc"; } + if (!isRestreamed) { + return "jsmpeg"; + } + return "mse"; }, [lowBandwidth, mic, webRTC, isRestreamed]); @@ -286,14 +290,23 @@ export default function LiveCameraView({ } }, [fullscreen, isPortrait, cameraAspectRatio, containerAspectRatio]); - const handleError = useCallback((e: LivePlayerError) => { - if (e == "mse-decode") { - setWebRTC(true); - } else { - setWebRTC(false); - setLowBandwidth(true); - } - }, []); + const handleError = useCallback( + (e: LivePlayerError) => { + if (e) { + if ( + !webRTC && + config && + config.go2rtc?.webrtc?.candidates?.length > 0 + ) { + setWebRTC(true); + } else { + setWebRTC(false); + setLowBandwidth(true); + } + } + }, + [config, webRTC], + ); return ( diff --git a/web/src/views/live/LiveDashboardView.tsx b/web/src/views/live/LiveDashboardView.tsx index 00c046224..cac604e26 100644 --- a/web/src/views/live/LiveDashboardView.tsx +++ b/web/src/views/live/LiveDashboardView.tsx @@ -28,8 +28,9 @@ import DraggableGridLayout from "./DraggableGridLayout"; import { IoClose } from "react-icons/io5"; import { LuLayoutDashboard } from "react-icons/lu"; import { cn } from "@/lib/utils"; -import { LivePlayerError, LivePlayerMode } from "@/types/live"; +import { LivePlayerError } from "@/types/live"; import { FaCompress, FaExpand } from "react-icons/fa"; +import useCameraLiveMode from "@/hooks/use-camera-live-mode"; import { useResizeObserver } from "@/hooks/resize-observer"; type LiveDashboardViewProps = { @@ -129,9 +130,6 @@ export default function LiveDashboardView({ // camera live views const [autoLiveView] = usePersistence("autoLiveView", true); - const [preferredLiveModes, setPreferredLiveModes] = useState<{ - [key: string]: LivePlayerMode; - }>({}); const [{ height: containerHeight }] = useResizeObserver(containerRef); @@ -186,32 +184,8 @@ export default function LiveDashboardView({ }; }, []); - useEffect(() => { - if (!cameras) return; - - const mseSupported = - "MediaSource" in window || "ManagedMediaSource" in window; - - const newPreferredLiveModes = cameras.reduce( - (acc, camera) => { - const isRestreamed = - config && - Object.keys(config.go2rtc.streams || {}).includes( - camera.live.stream_name, - ); - - if (!mseSupported) { - acc[camera.name] = isRestreamed ? "webrtc" : "jsmpeg"; - } else { - acc[camera.name] = isRestreamed ? "mse" : "jsmpeg"; - } - return acc; - }, - {} as { [key: string]: LivePlayerMode }, - ); - - setPreferredLiveModes(newPreferredLiveModes); - }, [cameras, config, windowVisible]); + const { preferredLiveModes, setPreferredLiveModes, resetPreferredLiveMode } = + useCameraLiveMode(cameras, windowVisible); const cameraRef = useCallback( (node: HTMLElement | null) => { @@ -381,6 +355,7 @@ export default function LiveDashboardView({ autoLive={autoLiveView} onClick={() => onSelectCamera(camera.name)} onError={(e) => handleError(camera.name, e)} + onResetLiveMode={() => resetPreferredLiveMode(camera.name)} /> ); })}