Handle error when live view stalls (#11665)

* Handle error when live view stalls

* Manually calculate buffer timeout

* Formatting

web/src/components/player/LivePlayer.tsx

@@ -8,7 +8,11 @@ import JSMpegPlayer from "./JSMpegPlayer";
 import { MdCircle } from "react-icons/md";
 import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip";
 import { useCameraActivity } from "@/hooks/use-camera-activity";
-import { LivePlayerMode, VideoResolutionType } from "@/types/live";
+import {
+  LivePlayerError,
+  LivePlayerMode,
+  VideoResolutionType,
+} from "@/types/live";
 import useCameraLiveMode from "@/hooks/use-camera-live-mode";
 import { getIconForLabel } from "@/utils/iconUtil";
 import Chip from "../indicators/Chip";
@@ -30,6 +34,7 @@ type LivePlayerProps = {
   autoLive?: boolean;
   onClick?: () => void;
   setFullResolution?: React.Dispatch<React.SetStateAction<VideoResolutionType>>;
+  onError?: (error: LivePlayerError) => void;
 };
 
 export default function LivePlayer({
@@ -47,6 +52,7 @@ export default function LivePlayer({
   autoLive = true,
   onClick,
   setFullResolution,
+  onError,
 }: LivePlayerProps) {
   // camera activity
 
@@ -145,6 +151,7 @@ export default function LivePlayer({
           onPlaying={() => setLiveReady(true)}
           pip={pip}
           setFullResolution={setFullResolution}
+          onError={onError}
         />
       );
     } else {

web/src/components/player/MsePlayer.tsx

@@ -1,5 +1,5 @@
 import { baseUrl } from "@/api/baseUrl";
-import { VideoResolutionType } from "@/types/live";
+import { LivePlayerError, VideoResolutionType } from "@/types/live";
 import {
   SetStateAction,
   useCallback,
@@ -17,6 +17,7 @@ type MSEPlayerProps = {
   pip?: boolean;
   onPlaying?: () => void;
   setFullResolution?: React.Dispatch<SetStateAction<VideoResolutionType>>;
+  onError?: (error: LivePlayerError) => void;
 };
 
 function MSEPlayer({
@@ -27,6 +28,7 @@ function MSEPlayer({
   pip = false,
   onPlaying,
   setFullResolution,
+  onError,
 }: MSEPlayerProps) {
   const RECONNECT_TIMEOUT: number = 30000;
 
@@ -45,6 +47,7 @@ function MSEPlayer({
 
   const [wsState, setWsState] = useState<number>(WebSocket.CLOSED);
   const [connectTS, setConnectTS] = useState<number>(0);
+  const [bufferTimeout, setBufferTimeout] = useState<NodeJS.Timeout>();
 
   const videoRef = useRef<HTMLVideoElement>(null);
   const wsRef = useRef<WebSocket | null>(null);
@@ -308,7 +311,34 @@ function MSEPlayer({
         onPlaying?.();
       }}
       muted={!audioEnabled}
-      onError={() => {
+      onProgress={
+        onError != undefined
+          ? () => {
+              if (videoRef.current?.paused) {
+                return;
+              }
+
+              if (bufferTimeout) {
+                clearTimeout(bufferTimeout);
+                setBufferTimeout(undefined);
+              }
+
+              setBufferTimeout(
+                setTimeout(() => {
+                  onError("stalled");
+                }, 3000),
+              );
+            }
+          : undefined
+      }
+      onError={(e) => {
+        if (
+          // @ts-expect-error code does exist
+          e.target.error.code == MediaError.MEDIA_ERR_NETWORK
+        ) {
+          onError?.("startup");
+        }
+
         if (wsRef.current) {
           wsRef.current.close();
           wsRef.current = null;

web/src/components/player/WebRTCPlayer.tsx

@@ -1,4 +1,5 @@
 import { baseUrl } from "@/api/baseUrl";
+import { LivePlayerError } from "@/types/live";
 import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 
 type WebRtcPlayerProps = {
@@ -10,6 +11,7 @@ type WebRtcPlayerProps = {
   iOSCompatFullScreen?: boolean; // ios doesn't support fullscreen divs so we must support the video element
   pip?: boolean;
   onPlaying?: () => void;
+  onError?: (error: LivePlayerError) => void;
 };
 
 export default function WebRtcPlayer({
@@ -21,6 +23,7 @@ export default function WebRtcPlayer({
   iOSCompatFullScreen = false,
   pip = false,
   onPlaying,
+  onError,
 }: WebRtcPlayerProps) {
   // metadata
 
@@ -32,6 +35,7 @@ export default function WebRtcPlayer({
 
   const pcRef = useRef<RTCPeerConnection | undefined>();
   const videoRef = useRef<HTMLVideoElement | null>(null);
+  const [bufferTimeout, setBufferTimeout] = useState<NodeJS.Timeout>();
 
   const PeerConnection = useCallback(
     async (media: string) => {
@@ -198,11 +202,39 @@ export default function WebRtcPlayer({
       playsInline
       muted={!audioEnabled}
       onLoadedData={onPlaying}
+      onProgress={
+        onError != undefined
+          ? () => {
+              if (videoRef.current?.paused) {
+                return;
+              }
+
+              if (bufferTimeout) {
+                clearTimeout(bufferTimeout);
+                setBufferTimeout(undefined);
+              }
+
+              setBufferTimeout(
+                setTimeout(() => {
+                  onError("stalled");
+                }, 3000),
+              );
+            }
+          : undefined
+      }
       onClick={
         iOSCompatFullScreen
           ? () => setiOSCompatControls(!iOSCompatControls)
           : undefined
       }
+      onError={(e) => {
+        if (
+          // @ts-expect-error code does exist
+          e.target.error.code == MediaError.MEDIA_ERR_NETWORK
+        ) {
+          onError?.("startup");
+        }
+      }}
     />
   );
 }

web/src/types/live.ts

@@ -30,3 +30,5 @@ export type LiveStreamMetadata = {
   producers: LiveProducerMetadata[];
   consumers: LiveConsumerMetadata[];
 };
+
+export type LivePlayerError = "stalled" | "startup";

web/src/views/live/LiveCameraView.tsx

@@ -191,6 +191,7 @@ export default function LiveCameraView({
   const [audio, setAudio] = useState(false);
   const [mic, setMic] = useState(false);
   const [pip, setPip] = useState(false);
+  const [lowBandwidth, setLowBandwidth] = useState(false);
 
   const [fullResolution, setFullResolution] = useState<VideoResolutionType>({
     width: 0,
@@ -202,8 +203,14 @@ export default function LiveCameraView({
       return "webrtc";
     }
 
+    if (lowBandwidth) {
+      return "jsmpeg";
+    }
+
     return "mse";
-  }, [mic]);
+  }, [lowBandwidth, mic]);
+
+  // layout state
 
   const windowAspectRatio = useMemo(() => {
     return windowWidth / windowHeight;
@@ -419,6 +426,7 @@ export default function LiveCameraView({
                 pip={pip}
                 setFullResolution={setFullResolution}
                 containerRef={containerRef}
+                onError={() => setLowBandwidth(true)}
               />
             </div>
             {camera.onvif.host != "" && (