Try webrtc when mse fails with decoding error (#11745)

* Try webrtc if enabled and mse fails with decoding error

* default to jsmpeg if webrtc times out

* check for mic first

web/src/components/player/LivePlayer.tsx

@@ -138,6 +138,7 @@ export default function LivePlayer({
         iOSCompatFullScreen={iOSCompatFullScreen}
         onPlaying={() => setLiveReady(true)}
         pip={pip}
+        onError={onError}
       />
     );
   } else if (liveMode == "mse") {

web/src/components/player/MsePlayer.tsx

@@ -8,6 +8,7 @@ import {
   useRef,
   useState,
 } from "react";
+import { isIOS, isSafari } from "react-device-detect";
 
 type MSEPlayerProps = {
   camera: string;
@@ -311,9 +312,11 @@ function MSEPlayer({
         onPlaying?.();
       }}
       muted={!audioEnabled}
-      onProgress={
+      onProgress={() => {
-        onError != undefined
+        if (isSafari || isIOS) {
-          ? () => {
+          onPlaying?.();
+        }
+        if (onError != undefined) {
           if (videoRef.current?.paused) {
             return;
           }
@@ -329,8 +332,7 @@ function MSEPlayer({
             }, 3000),
           );
         }
-          : undefined
+      }}
-      }
       onError={(e) => {
         if (
           // @ts-expect-error code does exist
@@ -339,6 +341,16 @@ function MSEPlayer({
           onError?.("startup");
         }
 
+        if (
+          // @ts-expect-error code does exist
+          e.target.error.code == MediaError.MEDIA_ERR_DECODE &&
+          (isSafari || isIOS)
+        ) {
+          onError?.("mse-decode");
+          clearTimeout(bufferTimeout);
+          setBufferTimeout(undefined);
+        }
+
         if (wsRef.current) {
           wsRef.current.close();
           wsRef.current = null;

web/src/components/player/WebRTCPlayer.tsx

@@ -36,6 +36,7 @@ export default function WebRtcPlayer({
   const pcRef = useRef<RTCPeerConnection | undefined>();
   const videoRef = useRef<HTMLVideoElement | null>(null);
   const [bufferTimeout, setBufferTimeout] = useState<NodeJS.Timeout>();
+  const videoLoadTimeoutRef = useRef<NodeJS.Timeout>();
 
   const PeerConnection = useCallback(
     async (media: string) => {
@@ -193,6 +194,27 @@ export default function WebRtcPlayer({
     videoRef.current.requestPictureInPicture();
   }, [pip, videoRef]);
 
+  useEffect(() => {
+    videoLoadTimeoutRef.current = setTimeout(() => {
+      onError?.("stalled");
+    }, 5000);
+
+    return () => {
+      if (videoLoadTimeoutRef.current) {
+        clearTimeout(videoLoadTimeoutRef.current);
+      }
+    };
+    // we know that these deps are correct
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, []);
+
+  const handleLoadedData = () => {
+    if (videoLoadTimeoutRef.current) {
+      clearTimeout(videoLoadTimeoutRef.current);
+    }
+    onPlaying?.();
+  };
+
   return (
     <video
       ref={videoRef}
@@ -201,7 +223,7 @@ export default function WebRtcPlayer({
       autoPlay
       playsInline
       muted={!audioEnabled}
-      onLoadedData={onPlaying}
+      onLoadedData={handleLoadedData}
       onProgress={
         onError != undefined
           ? () => {

web/src/types/live.ts

@@ -31,4 +31,4 @@ export type LiveStreamMetadata = {
   consumers: LiveConsumerMetadata[];
 };
 
-export type LivePlayerError = "stalled" | "startup";
+export type LivePlayerError = "stalled" | "startup" | "mse-decode";

web/src/views/live/LiveCameraView.tsx

@@ -190,6 +190,7 @@ export default function LiveCameraView({
 
   const [audio, setAudio] = useState(false);
   const [mic, setMic] = useState(false);
+  const [webRTC, setWebRTC] = useState(false);
   const [pip, setPip] = useState(false);
   const [lowBandwidth, setLowBandwidth] = useState(false);
 
@@ -203,12 +204,20 @@ export default function LiveCameraView({
       return "webrtc";
     }
 
+    if (webRTC && isRestreamed) {
+      return "webrtc";
+    }
+
+    if (webRTC && !isRestreamed) {
+      return "jsmpeg";
+    }
+
     if (lowBandwidth) {
       return "jsmpeg";
     }
 
     return "mse";
-  }, [lowBandwidth, mic]);
+  }, [lowBandwidth, mic, webRTC, isRestreamed]);
 
   // layout state
 
@@ -426,7 +435,14 @@ export default function LiveCameraView({
                 pip={pip}
                 setFullResolution={setFullResolution}
                 containerRef={containerRef}
-                onError={() => setLowBandwidth(true)}
+                onError={(e) => {
+                  if (e == "mse-decode") {
+                    setWebRTC(true);
+                  } else {
+                    setWebRTC(false);
+                    setLowBandwidth(true);
+                  }
+                }}
               />
             </div>
             {camera.onvif.host != "" && (