Live player fixes (#13143)

* Jump to live when exceeding buffer time threshold in MSE player

* clean up

* Try adjusting playback rate instead of jumping to live

* clean up

* fallback to webrtc if enabled before jsmpeg

* baseline

* clean up

* remove comments

* adaptive playback rate and intelligent switching improvements

* increase logging and reset live mode after camera is no longer active on dashboard only

* jump to live on safari/iOS

* clean up

* clean up

* refactor camera live mode hook

* remove key listener

* resolve conflicts

web/src/components/player/LivePlayer.tsx

@@ -13,7 +13,6 @@ import {
   LivePlayerMode,
   VideoResolutionType,
 } from "@/types/live";
-import useCameraLiveMode from "@/hooks/use-camera-live-mode";
 import { getIconForLabel } from "@/utils/iconUtil";
 import Chip from "../indicators/Chip";
 import { capitalizeFirstLetter } from "@/utils/stringUtil";
@@ -25,7 +24,7 @@ type LivePlayerProps = {
   containerRef?: React.MutableRefObject<HTMLDivElement | null>;
   className?: string;
   cameraConfig: CameraConfig;
-  preferredLiveMode?: LivePlayerMode;
+  preferredLiveMode: LivePlayerMode;
   showStillWithoutActivity?: boolean;
   windowVisible?: boolean;
   playAudio?: boolean;
@@ -36,6 +35,7 @@ type LivePlayerProps = {
   onClick?: () => void;
   setFullResolution?: React.Dispatch<React.SetStateAction<VideoResolutionType>>;
   onError?: (error: LivePlayerError) => void;
+  onResetLiveMode?: () => void;
 };
 
 export default function LivePlayer({
@@ -54,6 +54,7 @@ export default function LivePlayer({
   onClick,
   setFullResolution,
   onError,
+  onResetLiveMode,
 }: LivePlayerProps) {
   const internalContainerRef = useRef<HTMLDivElement | null>(null);
   // camera activity
@@ -70,8 +71,6 @@ export default function LivePlayer({
 
   // camera live state
 
-  const liveMode = useCameraLiveMode(cameraConfig, preferredLiveMode);
-
   const [liveReady, setLiveReady] = useState(false);
 
   const liveReadyRef = useRef(liveReady);
@@ -91,6 +90,7 @@ export default function LivePlayer({
       const timer = setTimeout(() => {
         if (liveReadyRef.current && !cameraActiveRef.current) {
           setLiveReady(false);
+          onResetLiveMode?.();
         }
       }, 500);
 
@@ -152,7 +152,7 @@ export default function LivePlayer({
   let player;
   if (!autoLive) {
     player = null;
-  } else if (liveMode == "webrtc") {
+  } else if (preferredLiveMode == "webrtc") {
     player = (
       <WebRtcPlayer
         className={`size-full rounded-lg md:rounded-2xl ${liveReady ? "" : "hidden"}`}
@@ -166,7 +166,7 @@ export default function LivePlayer({
         onError={onError}
       />
     );
-  } else if (liveMode == "mse") {
+  } else if (preferredLiveMode == "mse") {
     if ("MediaSource" in window || "ManagedMediaSource" in window) {
       player = (
         <MSEPlayer
@@ -187,7 +187,7 @@ export default function LivePlayer({
         </div>
       );
     }
-  } else if (liveMode == "jsmpeg") {
+  } else if (preferredLiveMode == "jsmpeg") {
     if (cameraActive || !showStillWithoutActivity || liveReady) {
       player = (
         <JSMpegPlayer

web/src/components/player/MsePlayer.tsx

@@ -32,6 +32,7 @@ function MSEPlayer({
   onError,
 }: MSEPlayerProps) {
   const RECONNECT_TIMEOUT: number = 10000;
+  const BUFFERING_COOLDOWN_TIMEOUT: number = 5000;
 
   const CODECS: string[] = [
     "avc1.640029", // H.264 high 4.1 (Chromecast 1st and 2nd Gen)
@@ -46,6 +47,11 @@ function MSEPlayer({
 
   const visibilityCheck: boolean = !pip;
   const [isPlaying, setIsPlaying] = useState(false);
+  const lastJumpTimeRef = useRef(0);
+
+  const MAX_BUFFER_ENTRIES = 10; // Size of the rolling window  of buffered times
+  const bufferTimes = useRef<number[]>([]);
+  const bufferIndex = useRef(0);
 
   const [wsState, setWsState] = useState<number>(WebSocket.CLOSED);
   const [connectTS, setConnectTS] = useState<number>(0);
@@ -133,6 +139,13 @@ function MSEPlayer({
     }
   }, [bufferTimeout]);
 
+  const handlePause = useCallback(() => {
+    // don't let the user pause the live stream
+    if (isPlaying && playbackEnabled) {
+      videoRef.current?.play();
+    }
+  }, [isPlaying, playbackEnabled]);
+
   const onOpen = () => {
     setWsState(WebSocket.OPEN);
 
@@ -193,6 +206,7 @@ function MSEPlayer({
 
   const onMse = () => {
     if ("ManagedMediaSource" in window) {
+      // safari
       const MediaSource = window.ManagedMediaSource;
 
       msRef.current?.addEventListener(
@@ -224,6 +238,7 @@ function MSEPlayer({
         videoRef.current.srcObject = msRef.current;
       }
     } else {
+      // non safari
       msRef.current?.addEventListener(
         "sourceopen",
         () => {
@@ -247,15 +262,35 @@ function MSEPlayer({
         },
         { once: true },
       );
-      videoRef.current!.src = URL.createObjectURL(msRef.current!);
-      videoRef.current!.srcObject = null;
+      if (videoRef.current && msRef.current) {
+        videoRef.current.src = URL.createObjectURL(msRef.current);
+        videoRef.current.srcObject = null;
+      }
     }
     play();
 
     onmessageRef.current["mse"] = (msg) => {
       if (msg.type !== "mse") return;
 
-      const sb = msRef.current?.addSourceBuffer(msg.value);
+      let sb: SourceBuffer | undefined;
+      try {
+        sb = msRef.current?.addSourceBuffer(msg.value);
+        if (sb?.mode) {
+          sb.mode = "segments";
+        }
+      } catch (e) {
+        // Safari sometimes throws this error
+        if (e instanceof DOMException && e.name === "InvalidStateError") {
+          if (wsRef.current) {
+            onDisconnect();
+          }
+          onError?.("mse-decode");
+          return;
+        } else {
+          throw e; // Re-throw if it's not the error we're handling
+        }
+      }
+
       sb?.addEventListener("updateend", () => {
         if (sb.updating) return;
 
@@ -302,6 +337,43 @@ function MSEPlayer({
     return video.buffered.end(video.buffered.length - 1) - video.currentTime;
   };
 
+  const jumpToLive = () => {
+    if (!videoRef.current) return;
+
+    const buffered = videoRef.current.buffered;
+    if (buffered.length > 0) {
+      const liveEdge = buffered.end(buffered.length - 1);
+      // Jump to the live edge
+      videoRef.current.currentTime = liveEdge - 0.75;
+      lastJumpTimeRef.current = Date.now();
+    }
+  };
+
+  const calculateAdaptiveBufferThreshold = () => {
+    const filledEntries = bufferTimes.current.length;
+    const sum = bufferTimes.current.reduce((a, b) => a + b, 0);
+    const averageBufferTime = filledEntries ? sum / filledEntries : 0;
+    return averageBufferTime * (isSafari || isIOS ? 3 : 1.5);
+  };
+
+  const calculateAdaptivePlaybackRate = (
+    bufferTime: number,
+    bufferThreshold: number,
+  ) => {
+    const alpha = 0.2; // aggressiveness of playback rate increase
+    const beta = 0.5; // steepness of exponential growth
+
+    // don't adjust playback rate if we're close enough to live
+    if (
+      (bufferTime <= bufferThreshold && bufferThreshold < 3) ||
+      bufferTime < 3
+    ) {
+      return 1;
+    }
+    const rate = 1 + alpha * Math.exp(beta * bufferTime - bufferThreshold);
+    return Math.min(rate, 2);
+  };
+
   useEffect(() => {
     if (!playbackEnabled) {
       return;
@@ -386,21 +458,71 @@ function MSEPlayer({
         handleLoadedMetadata?.();
         onPlaying?.();
         setIsPlaying(true);
+        lastJumpTimeRef.current = Date.now();
       }}
       muted={!audioEnabled}
-      onPause={() => videoRef.current?.play()}
+      onPause={handlePause}
       onProgress={() => {
+        const bufferTime = getBufferedTime(videoRef.current);
+
+        if (
+          videoRef.current &&
+          (videoRef.current.playbackRate === 1 || bufferTime < 3)
+        ) {
+          if (bufferTimes.current.length < MAX_BUFFER_ENTRIES) {
+            bufferTimes.current.push(bufferTime);
+          } else {
+            bufferTimes.current[bufferIndex.current] = bufferTime;
+            bufferIndex.current =
+              (bufferIndex.current + 1) % MAX_BUFFER_ENTRIES;
+          }
+        }
+
+        const bufferThreshold = calculateAdaptiveBufferThreshold();
+
         // if we have > 3 seconds of buffered data and we're still not playing,
         // something might be wrong - maybe codec issue, no audio, etc
         // so mark the player as playing so that error handlers will fire
-        if (
-          !isPlaying &&
-          playbackEnabled &&
-          getBufferedTime(videoRef.current) > 3
-        ) {
+        if (!isPlaying && playbackEnabled && bufferTime > 3) {
           setIsPlaying(true);
+          lastJumpTimeRef.current = Date.now();
           onPlaying?.();
         }
+
+        // if we have more than 10 seconds of buffer, something's wrong so error out
+        if (
+          isPlaying &&
+          playbackEnabled &&
+          (bufferThreshold > 10 || bufferTime > 10)
+        ) {
+          onDisconnect();
+          onError?.("stalled");
+        }
+
+        const playbackRate = calculateAdaptivePlaybackRate(
+          bufferTime,
+          bufferThreshold,
+        );
+
+        // if we're above our rolling average threshold or have > 3 seconds of
+        // buffered data and we're playing, we may have drifted from actual live
+        // time, so increase playback rate to compensate - non safari/ios only
+        if (
+          videoRef.current &&
+          isPlaying &&
+          playbackEnabled &&
+          Date.now() - lastJumpTimeRef.current > BUFFERING_COOLDOWN_TIMEOUT
+        ) {
+          // Jump to live on Safari/iOS due to a change of playback rate causing re-buffering
+          if (isSafari || isIOS) {
+            if (bufferTime > 3) {
+              jumpToLive();
+            }
+          } else {
+            videoRef.current.playbackRate = playbackRate;
+          }
+        }
+
         if (onError != undefined) {
           if (videoRef.current?.paused) {
             return;

web/src/hooks/use-camera-live-mode.ts

@@ -1,49 +1,65 @@
 import { CameraConfig, FrigateConfig } from "@/types/frigateConfig";
-import { useMemo } from "react";
+import { useCallback, useEffect, useState } from "react";
 import useSWR from "swr";
-import { usePersistence } from "./use-persistence";
 import { LivePlayerMode } from "@/types/live";
 
 export default function useCameraLiveMode(
-  cameraConfig: CameraConfig,
-  preferredMode?: LivePlayerMode,
-): LivePlayerMode | undefined {
+  cameras: CameraConfig[],
+  windowVisible: boolean,
+) {
   const { data: config } = useSWR<FrigateConfig>("config");
+  const [preferredLiveModes, setPreferredLiveModes] = useState<{
+    [key: string]: LivePlayerMode;
+  }>({});
 
-  const restreamEnabled = useMemo(() => {
-    if (!config) {
-      return false;
-    }
+  useEffect(() => {
+    if (!cameras) return;
 
-    return (
-      cameraConfig &&
-      Object.keys(config.go2rtc.streams || {}).includes(
-        cameraConfig.live.stream_name,
-      )
+    const mseSupported =
+      "MediaSource" in window || "ManagedMediaSource" in window;
+
+    const newPreferredLiveModes = cameras.reduce(
+      (acc, camera) => {
+        const isRestreamed =
+          config &&
+          Object.keys(config.go2rtc.streams || {}).includes(
+            camera.live.stream_name,
+          );
+
+        if (!mseSupported) {
+          acc[camera.name] = isRestreamed ? "webrtc" : "jsmpeg";
+        } else {
+          acc[camera.name] = isRestreamed ? "mse" : "jsmpeg";
+        }
+        return acc;
+      },
+      {} as { [key: string]: LivePlayerMode },
     );
-  }, [config, cameraConfig]);
-  const defaultLiveMode = useMemo<LivePlayerMode | undefined>(() => {
-    if (config) {
-      if (restreamEnabled) {
-        return preferredMode || "mse";
-      }
 
-      return "jsmpeg";
-    }
+    setPreferredLiveModes(newPreferredLiveModes);
+  }, [cameras, config, windowVisible]);
 
-    return undefined;
-  }, [config, preferredMode, restreamEnabled]);
-  const [viewSource] = usePersistence<LivePlayerMode>(
-    `${cameraConfig.name}-source`,
-    defaultLiveMode,
+  const resetPreferredLiveMode = useCallback(
+    (cameraName: string) => {
+      const mseSupported =
+        "MediaSource" in window || "ManagedMediaSource" in window;
+      const isRestreamed =
+        config && Object.keys(config.go2rtc.streams || {}).includes(cameraName);
+
+      setPreferredLiveModes((prevModes) => {
+        const newModes = { ...prevModes };
+
+        if (!mseSupported) {
+          newModes[cameraName] = isRestreamed ? "webrtc" : "jsmpeg";
+        } else {
+          newModes[cameraName] = isRestreamed ? "mse" : "jsmpeg";
+        }
+
+        return newModes;
+      });
+    },
+    [config],
   );
 
-  if (
-    restreamEnabled &&
-    (preferredMode == "mse" || preferredMode == "webrtc")
-  ) {
-    return preferredMode;
-  } else {
-    return viewSource;
-  }
+  return { preferredLiveModes, setPreferredLiveModes, resetPreferredLiveMode };
 }

web/src/types/frigateConfig.ts

@@ -298,7 +298,12 @@ export interface FrigateConfig {
     retry_interval: number;
   };
 
-  go2rtc: Record<string, unknown>;
+  go2rtc: {
+    streams: string[];
+    webrtc: {
+      candidates: string[];
+    };
+  };
 
   camera_groups: { [groupName: string]: CameraGroupConfig };
 

web/src/views/live/DraggableGridLayout.tsx

@@ -41,6 +41,7 @@ import {
   TooltipContent,
 } from "@/components/ui/tooltip";
 import { Toaster } from "@/components/ui/sonner";
+import useCameraLiveMode from "@/hooks/use-camera-live-mode";
 
 type DraggableGridLayoutProps = {
   cameras: CameraConfig[];
@@ -75,36 +76,8 @@ export default function DraggableGridLayout({
 
   // preferred live modes per camera
 
-  const [preferredLiveModes, setPreferredLiveModes] = useState<{
-    [key: string]: LivePlayerMode;
-  }>({});
-
-  useEffect(() => {
-    if (!cameras) return;
-
-    const mseSupported =
-      "MediaSource" in window || "ManagedMediaSource" in window;
-
-    const newPreferredLiveModes = cameras.reduce(
-      (acc, camera) => {
-        const isRestreamed =
-          config &&
-          Object.keys(config.go2rtc.streams || {}).includes(
-            camera.live.stream_name,
-          );
-
-        if (!mseSupported) {
-          acc[camera.name] = isRestreamed ? "webrtc" : "jsmpeg";
-        } else {
-          acc[camera.name] = isRestreamed ? "mse" : "jsmpeg";
-        }
-        return acc;
-      },
-      {} as { [key: string]: LivePlayerMode },
-    );
-
-    setPreferredLiveModes(newPreferredLiveModes);
-  }, [cameras, config, windowVisible]);
+  const { preferredLiveModes, setPreferredLiveModes, resetPreferredLiveMode } =
+    useCameraLiveMode(cameras, windowVisible);
 
   const ResponsiveGridLayout = useMemo(() => WidthProvider(Responsive), []);
 
@@ -477,6 +450,7 @@ export default function DraggableGridLayout({
                       return newModes;
                     });
                   }}
+                  onResetLiveMode={() => resetPreferredLiveMode(camera.name)}
                 >
                   {isEditMode && showCircles && <CornerCircles />}
                 </LivePlayerGridItem>
@@ -635,6 +609,7 @@ type LivePlayerGridItemProps = {
   preferredLiveMode: LivePlayerMode;
   onClick: () => void;
   onError: (e: LivePlayerError) => void;
+  onResetLiveMode: () => void;
 };
 
 const LivePlayerGridItem = React.forwardRef<
@@ -655,6 +630,7 @@ const LivePlayerGridItem = React.forwardRef<
       preferredLiveMode,
       onClick,
       onError,
+      onResetLiveMode,
       ...props
     },
     ref,
@@ -676,6 +652,7 @@ const LivePlayerGridItem = React.forwardRef<
           preferredLiveMode={preferredLiveMode}
           onClick={onClick}
           onError={onError}
+          onResetLiveMode={onResetLiveMode}
           containerRef={ref as React.RefObject<HTMLDivElement>}
         />
         {children}

web/src/views/live/LiveCameraView.tsx

@@ -227,6 +227,10 @@ export default function LiveCameraView({
       return "webrtc";
     }
 
+    if (!isRestreamed) {
+      return "jsmpeg";
+    }
+
     return "mse";
   }, [lowBandwidth, mic, webRTC, isRestreamed]);
 
@@ -286,14 +290,23 @@ export default function LiveCameraView({
     }
   }, [fullscreen, isPortrait, cameraAspectRatio, containerAspectRatio]);
 
-  const handleError = useCallback((e: LivePlayerError) => {
-    if (e == "mse-decode") {
-      setWebRTC(true);
-    } else {
-      setWebRTC(false);
-      setLowBandwidth(true);
-    }
-  }, []);
+  const handleError = useCallback(
+    (e: LivePlayerError) => {
+      if (e) {
+        if (
+          !webRTC &&
+          config &&
+          config.go2rtc?.webrtc?.candidates?.length > 0
+        ) {
+          setWebRTC(true);
+        } else {
+          setWebRTC(false);
+          setLowBandwidth(true);
+        }
+      }
+    },
+    [config, webRTC],
+  );
 
   return (
     <TransformWrapper minScale={1.0} wheel={{ smoothStep: 0.005 }}>

web/src/views/live/LiveDashboardView.tsx

@@ -28,8 +28,9 @@ import DraggableGridLayout from "./DraggableGridLayout";
 import { IoClose } from "react-icons/io5";
 import { LuLayoutDashboard } from "react-icons/lu";
 import { cn } from "@/lib/utils";
-import { LivePlayerError, LivePlayerMode } from "@/types/live";
+import { LivePlayerError } from "@/types/live";
 import { FaCompress, FaExpand } from "react-icons/fa";
+import useCameraLiveMode from "@/hooks/use-camera-live-mode";
 import { useResizeObserver } from "@/hooks/resize-observer";
 
 type LiveDashboardViewProps = {
@@ -129,9 +130,6 @@ export default function LiveDashboardView({
   // camera live views
 
   const [autoLiveView] = usePersistence("autoLiveView", true);
-  const [preferredLiveModes, setPreferredLiveModes] = useState<{
-    [key: string]: LivePlayerMode;
-  }>({});
 
   const [{ height: containerHeight }] = useResizeObserver(containerRef);
 
@@ -186,32 +184,8 @@ export default function LiveDashboardView({
     };
   }, []);
 
-  useEffect(() => {
-    if (!cameras) return;
-
-    const mseSupported =
-      "MediaSource" in window || "ManagedMediaSource" in window;
-
-    const newPreferredLiveModes = cameras.reduce(
-      (acc, camera) => {
-        const isRestreamed =
-          config &&
-          Object.keys(config.go2rtc.streams || {}).includes(
-            camera.live.stream_name,
-          );
-
-        if (!mseSupported) {
-          acc[camera.name] = isRestreamed ? "webrtc" : "jsmpeg";
-        } else {
-          acc[camera.name] = isRestreamed ? "mse" : "jsmpeg";
-        }
-        return acc;
-      },
-      {} as { [key: string]: LivePlayerMode },
-    );
-
-    setPreferredLiveModes(newPreferredLiveModes);
-  }, [cameras, config, windowVisible]);
+  const { preferredLiveModes, setPreferredLiveModes, resetPreferredLiveMode } =
+    useCameraLiveMode(cameras, windowVisible);
 
   const cameraRef = useCallback(
     (node: HTMLElement | null) => {
@@ -381,6 +355,7 @@ export default function LiveDashboardView({
                   autoLive={autoLiveView}
                   onClick={() => onSelectCamera(camera.name)}
                   onError={(e) => handleError(camera.name, e)}
+                  onResetLiveMode={() => resetPreferredLiveMode(camera.name)}
                 />
               );
             })}