From 53a2a865f112f622b56e842503dd101a8489e1de Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sat, 29 Jun 2024 10:02:30 -0500 Subject: [PATCH] Live player fixes and improvements (#12202) * Live player fixes and improvements * remove comment * Simplify wording --- docs/docs/configuration/live.md | 14 +- web/src/components/camera/CameraImage.tsx | 19 ++- .../components/player/BirdseyeLivePlayer.tsx | 4 +- web/src/components/player/JSMpegPlayer.tsx | 85 ++++++++---- web/src/components/player/LivePlayer.tsx | 20 +-- web/src/components/player/MsePlayer.tsx | 123 +++++++++++++++--- web/src/pages/UIPlayground.tsx | 4 +- web/src/views/live/DraggableGridLayout.tsx | 45 ++++++- web/src/views/live/LiveDashboardView.tsx | 38 +++++- 9 files changed, 267 insertions(+), 85 deletions(-) diff --git a/docs/docs/configuration/live.md b/docs/docs/configuration/live.md index 42b7be83b..efd970a1d 100644 --- a/docs/docs/configuration/live.md +++ b/docs/docs/configuration/live.md @@ -7,13 +7,15 @@ Frigate intelligently displays your camera streams on the Live view dashboard. Y ## Live View technologies -Frigate intelligently uses three different streaming technologies to display your camera streams. The highest quality and fluency of the Live view requires the bundled `go2rtc` to be configured as shown in the [step by step guide](/guides/configuring_go2rtc). +Frigate intelligently uses three different streaming technologies to display your camera streams on the dashboard and the single camera view, switching between available modes based on network bandwidth, player errors, or required features like two-way talk. The highest quality and fluency of the Live view requires the bundled `go2rtc` to be configured as shown in the [step by step guide](/guides/configuring_go2rtc). -| Source | Latency | Frame Rate | Resolution | Audio | Requires go2rtc | Other Limitations | -| ------ | ------- | ------------------------------------- | -------------- | ---------------------------- | --------------- | ------------------------------------------------ | -| jsmpeg | low | same as `detect -> fps`, capped at 10 | same as detect | no | no | none | -| mse | low | native | native | yes (depends on audio codec) | yes | iPhone requires iOS 17.1+, Firefox is h.264 only | -| webrtc | lowest | native | native | yes (depends on audio codec) | yes | requires extra config, doesn't support h.265 | +The jsmpeg live view will use more browser and client GPU resources. Using go2rtc is highly recommended and will provide a superior experience. + +| Source | Latency | Frame Rate | Resolution | Audio | Requires go2rtc | Other Limitations | +| ------ | ------- | ------------------------------------- | ---------- | ---------------------------- | --------------- | ------------------------------------------------------------------------------------ | +| jsmpeg | low | same as `detect -> fps`, capped at 10 | 720p | no | no | resolution is configurable, but go2rtc is recommended if you want higher resolutions | +| mse | low | native | native | yes (depends on audio codec) | yes | iPhone requires iOS 17.1+, Firefox is h.264 only | +| webrtc | lowest | native | native | yes (depends on audio codec) | yes | requires extra config, doesn't support h.265 | ### Audio Support diff --git a/web/src/components/camera/CameraImage.tsx b/web/src/components/camera/CameraImage.tsx index 7994c19e1..92d0572c1 100644 --- a/web/src/components/camera/CameraImage.tsx +++ b/web/src/components/camera/CameraImage.tsx @@ -26,13 +26,12 @@ export default function CameraImage({ const { name } = config ? config.cameras[camera] : ""; const enabled = config ? config.cameras[camera].enabled : "True"; - const [isPortraitImage, setIsPortraitImage] = useState(false); const [{ width: containerWidth, height: containerHeight }] = useResizeObserver(containerRef); const requestHeight = useMemo(() => { - if (!config || containerHeight == 0) { + if (!config || containerHeight == 0 || !hasLoaded) { return 360; } @@ -40,7 +39,14 @@ export default function CameraImage({ config.cameras[camera].detect.height, Math.round(containerHeight * (isDesktop ? 1.1 : 1.25)), ); - }, [config, camera, containerHeight]); + }, [config, camera, containerHeight, hasLoaded]); + + const isPortraitImage = useMemo(() => { + if (imgRef.current && containerWidth && containerHeight && hasLoaded) { + const { naturalHeight, naturalWidth } = imgRef.current; + return naturalWidth / naturalHeight < containerWidth / containerHeight; + } + }, [containerWidth, containerHeight, hasLoaded]); useEffect(() => { if (!config || !imgRef.current) { @@ -61,13 +67,6 @@ export default function CameraImage({ onLoad={() => { setHasLoaded(true); - if (imgRef.current) { - const { naturalHeight, naturalWidth } = imgRef.current; - setIsPortraitImage( - naturalWidth / naturalHeight < containerWidth / containerHeight, - ); - } - if (onload) { onload(); } diff --git a/web/src/components/player/BirdseyeLivePlayer.tsx b/web/src/components/player/BirdseyeLivePlayer.tsx index 127933f09..235e14785 100644 --- a/web/src/components/player/BirdseyeLivePlayer.tsx +++ b/web/src/components/player/BirdseyeLivePlayer.tsx @@ -12,7 +12,7 @@ type LivePlayerProps = { birdseyeConfig: BirdseyeConfig; liveMode: LivePlayerMode; onClick?: () => void; - containerRef?: React.MutableRefObject; + containerRef: React.MutableRefObject; }; export default function BirdseyeLivePlayer({ @@ -54,6 +54,7 @@ export default function BirdseyeLivePlayer({ width={birdseyeConfig.width} height={birdseyeConfig.height} containerRef={containerRef} + playbackEnabled={true} /> ); } else { @@ -62,6 +63,7 @@ export default function BirdseyeLivePlayer({ return (
; + containerRef: React.MutableRefObject; + playbackEnabled: boolean; onPlaying?: () => void; }; @@ -20,18 +21,21 @@ export default function JSMpegPlayer({ height, className, containerRef, + playbackEnabled, onPlaying, }: JSMpegPlayerProps) { const url = `${baseUrl.replace(/^http/, "ws")}live/jsmpeg/${camera}`; - const playerRef = useRef(null); - const videoRef = useRef(null); + const videoRef = useRef(null); + const canvasRef = useRef(null); const internalContainerRef = useRef(null); const onPlayingRef = useRef(onPlaying); const [showCanvas, setShowCanvas] = useState(false); const selectedContainerRef = useMemo( - () => containerRef ?? internalContainerRef, - [containerRef, internalContainerRef], + () => (containerRef.current ? containerRef : internalContainerRef), + // we know that these deps are correct + // eslint-disable-next-line react-hooks/exhaustive-deps + [containerRef, containerRef.current, internalContainerRef], ); const [{ width: containerWidth, height: containerHeight }] = @@ -83,39 +87,64 @@ export default function JSMpegPlayer({ } }, [scaledHeight, aspectRatio]); - const uniqueId = useId(); - useEffect(() => { onPlayingRef.current = onPlaying; }, [onPlaying]); useEffect(() => { - if (!playerRef.current || videoRef.current) { + if (!selectedContainerRef?.current || !url) { return; } - videoRef.current = new JSMpeg.VideoElement( - playerRef.current, - url, - { canvas: `#${CSS.escape(uniqueId)}` }, - { - protocols: [], - audio: false, - videoBufferSize: 1024 * 1024 * 4, - onPlay: () => { - setShowCanvas(true); - onPlayingRef.current?.(); - }, - }, - ); - }, [url, uniqueId]); + const videoWrapper = videoRef.current; + const canvas = canvasRef.current; + let hasData = false; + let videoElement: JSMpeg.VideoElement | null = null; + + if (videoWrapper && playbackEnabled) { + // Delayed init to avoid issues with react strict mode + const initPlayer = setTimeout(() => { + videoElement = new JSMpeg.VideoElement( + videoWrapper, + url, + { canvas: canvas }, + { + protocols: [], + audio: false, + videoBufferSize: 1024 * 1024 * 4, + onVideoDecode: () => { + if (!hasData) { + hasData = true; + setShowCanvas(true); + onPlayingRef.current?.(); + } + }, + }, + ); + }, 0); + + return () => { + clearTimeout(initPlayer); + if (videoElement) { + try { + // this causes issues in react strict mode + // https://stackoverflow.com/questions/76822128/issue-with-cycjimmy-jsmpeg-player-in-react-18-cannot-read-properties-of-null-o + videoElement.destroy(); + // eslint-disable-next-line no-empty + } catch (e) {} + } + }; + } + // we know that these deps are correct + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [playbackEnabled, url]); return ( -
-
-
+
+
+
(null); // camera activity const { activeMotion, activeTracking, objects, offline } = @@ -73,20 +74,12 @@ export default function LivePlayer({ const [liveReady, setLiveReady] = useState(false); useEffect(() => { - if (!autoLive) { - return; - } - - if (!liveReady) { - if (cameraActive && liveMode == "jsmpeg") { - setLiveReady(true); - } - + if (!autoLive || !liveReady) { return; } if (!cameraActive) { - setTimeout(() => setLiveReady(false), 500); + setLiveReady(false); } // live mode won't change // eslint-disable-next-line react-hooks/exhaustive-deps @@ -181,7 +174,8 @@ export default function LivePlayer({ camera={cameraConfig.live.stream_name} width={cameraConfig.detect.width} height={cameraConfig.detect.height} - containerRef={containerRef} + playbackEnabled={cameraActive || !showStillWithoutActivity} + containerRef={containerRef ?? internalContainerRef} onPlaying={playerIsPlaying} /> ); @@ -194,7 +188,7 @@ export default function LivePlayer({ return (
(WebSocket.CLOSED); const [connectTS, setConnectTS] = useState(0); const [bufferTimeout, setBufferTimeout] = useState(); + const [errorCount, setErrorCount] = useState(0); const videoRef = useRef(null); const wsRef = useRef(null); @@ -117,12 +119,21 @@ function MSEPlayer({ }, [wsURL]); const onDisconnect = useCallback(() => { - if (wsRef.current && wsState == WebSocket.OPEN) { + if (bufferTimeout) { + clearTimeout(bufferTimeout); + setBufferTimeout(undefined); + } + + if ((isSafari || isIOS) && safariPlaying) { + setSafariPlaying(false); + } + + if (wsRef.current && wsState != WebSocket.CLOSED) { setWsState(WebSocket.CLOSED); wsRef.current.close(); wsRef.current = null; } - }, [wsState]); + }, [wsState, bufferTimeout, safariPlaying]); const onOpen = () => { setWsState(WebSocket.OPEN); @@ -162,6 +173,26 @@ function MSEPlayer({ reconnect(); }; + const sendWithTimeout = (value: object, timeout: number) => { + return new Promise((resolve, reject) => { + const timeoutId = setTimeout(() => { + reject(new Error("Timeout waiting for response")); + }, timeout); + + send(value); + + // Override the onmessageRef handler for mse type to resolve the promise on response + const originalHandler = onmessageRef.current["mse"]; + onmessageRef.current["mse"] = (msg) => { + if (msg.type === "mse") { + clearTimeout(timeoutId); + if (originalHandler) originalHandler(msg); + resolve(); + } + }; + }); + }; + const onMse = () => { if ("ManagedMediaSource" in window) { const MediaSource = window.ManagedMediaSource; @@ -169,10 +200,22 @@ function MSEPlayer({ msRef.current?.addEventListener( "sourceopen", () => { - send({ - type: "mse", - // @ts-expect-error for typing - value: codecs(MediaSource.isTypeSupported), + sendWithTimeout( + { + type: "mse", + // @ts-expect-error for typing + value: codecs(MediaSource.isTypeSupported), + }, + 3000, + ).catch(() => { + if (wsRef.current) { + onDisconnect(); + } + if (isIOS || isSafari) { + onError?.("mse-decode"); + } else { + onError?.("startup"); + } }); }, { once: true }, @@ -187,9 +230,21 @@ function MSEPlayer({ "sourceopen", () => { URL.revokeObjectURL(videoRef.current?.src || ""); - send({ - type: "mse", - value: codecs(MediaSource.isTypeSupported), + sendWithTimeout( + { + type: "mse", + value: codecs(MediaSource.isTypeSupported), + }, + 3000, + ).catch(() => { + if (wsRef.current) { + onDisconnect(); + } + if (isIOS || isSafari) { + onError?.("mse-decode"); + } else { + onError?.("startup"); + } }); }, { once: true }, @@ -260,10 +315,6 @@ function MSEPlayer({ return () => { onDisconnect(); - if (bufferTimeout) { - clearTimeout(bufferTimeout); - setBufferTimeout(undefined); - } }; // we know that these deps are correct // eslint-disable-next-line react-hooks/exhaustive-deps @@ -305,6 +356,23 @@ function MSEPlayer({ videoRef.current.requestPictureInPicture(); }, [pip, videoRef]); + // ensure we disconnect for slower connections + + useEffect(() => { + if (wsState === WebSocket.OPEN && !playbackEnabled) { + if (bufferTimeout) { + clearTimeout(bufferTimeout); + setBufferTimeout(undefined); + } + + setTimeout(() => { + if (!playbackEnabled) onDisconnect(); + }, 10000); + } + // we know that these deps are correct + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [playbackEnabled]); + return (