Use full resolution aspect ratio when available (#11173)

* base recordings and live views off of actual video resolution

* don't set for jsmpeg

* reset when changing main cam

* rename

* Only use resolution for main camera

* fix lint

---------

Co-authored-by: Nicolas Mowen <nickmowen213@gmail.com>

web/src/components/player/HlsVideoPlayer.tsx

@@ -1,8 +1,15 @@
-import { MutableRefObject, useEffect, useRef, useState } from "react";
+import {
+  MutableRefObject,
+  useCallback,
+  useEffect,
+  useRef,
+  useState,
+} from "react";
 import Hls from "hls.js";
 import { isAndroid, isDesktop, isMobile } from "react-device-detect";
 import { TransformComponent, TransformWrapper } from "react-zoom-pan-pinch";
 import VideoControls from "./VideoControls";
+import { VideoResolutionType } from "@/types/live";
 
 // Android native hls does not seek correctly
 const USE_NATIVE_HLS = !isAndroid;
@@ -21,6 +28,7 @@ type HlsVideoPlayerProps = {
   onPlayerLoaded?: () => void;
   onTimeUpdate?: (time: number) => void;
   onPlaying?: () => void;
+  setFullResolution?: React.Dispatch<React.SetStateAction<VideoResolutionType>>;
 };
 export default function HlsVideoPlayer({
   videoRef,
@@ -31,6 +39,7 @@ export default function HlsVideoPlayer({
   onPlayerLoaded,
   onTimeUpdate,
   onPlaying,
+  setFullResolution,
 }: HlsVideoPlayerProps) {
   // playback
 
@@ -38,6 +47,18 @@ export default function HlsVideoPlayer({
   const [useHlsCompat, setUseHlsCompat] = useState(false);
   const [loadedMetadata, setLoadedMetadata] = useState(false);
 
+  const handleLoadedMetadata = useCallback(() => {
+    setLoadedMetadata(true);
+    if (videoRef.current) {
+      if (setFullResolution) {
+        setFullResolution({
+          width: videoRef.current.videoWidth,
+          height: videoRef.current.videoHeight,
+        });
+      }
+    }
+  }, [videoRef, setFullResolution]);
+
   useEffect(() => {
     if (!videoRef.current) {
       return;
@@ -193,7 +214,7 @@ export default function HlsVideoPlayer({
               : undefined
           }
           onLoadedData={onPlayerLoaded}
-          onLoadedMetadata={() => setLoadedMetadata(true)}
+          onLoadedMetadata={handleLoadedMetadata}
           onEnded={onClipEnded}
           onError={(e) => {
             if (

web/src/components/player/LivePlayer.tsx

@@ -8,7 +8,7 @@ import JSMpegPlayer from "./JSMpegPlayer";
 import { MdCircle } from "react-icons/md";
 import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip";
 import { useCameraActivity } from "@/hooks/use-camera-activity";
-import { LivePlayerMode } from "@/types/live";
+import { LivePlayerMode, VideoResolutionType } from "@/types/live";
 import useCameraLiveMode from "@/hooks/use-camera-live-mode";
 import { getIconForLabel } from "@/utils/iconUtil";
 import Chip from "../indicators/Chip";
@@ -27,6 +27,7 @@ type LivePlayerProps = {
   iOSCompatFullScreen?: boolean;
   pip?: boolean;
   onClick?: () => void;
+  setFullResolution?: React.Dispatch<React.SetStateAction<VideoResolutionType>>;
 };
 
 export default function LivePlayer({
@@ -41,6 +42,7 @@ export default function LivePlayer({
   iOSCompatFullScreen = false,
   pip,
   onClick,
+  setFullResolution,
 }: LivePlayerProps) {
   const [cameraHovered, setCameraHovered] = useState(false);
 
@@ -123,6 +125,7 @@ export default function LivePlayer({
           audioEnabled={playAudio}
           onPlaying={() => setLiveReady(true)}
           pip={pip}
+          setFullResolution={setFullResolution}
         />
       );
     } else {

web/src/components/player/MsePlayer.tsx

@@ -1,5 +1,13 @@
 import { baseUrl } from "@/api/baseUrl";
-import { useCallback, useEffect, useMemo, useRef, useState } from "react";
+import { VideoResolutionType } from "@/types/live";
+import {
+  SetStateAction,
+  useCallback,
+  useEffect,
+  useMemo,
+  useRef,
+  useState,
+} from "react";
 
 type MSEPlayerProps = {
   camera: string;
@@ -8,6 +16,7 @@ type MSEPlayerProps = {
   audioEnabled?: boolean;
   pip?: boolean;
   onPlaying?: () => void;
+  setFullResolution?: React.Dispatch<SetStateAction<VideoResolutionType>>;
 };
 
 function MSEPlayer({
@@ -17,6 +26,7 @@ function MSEPlayer({
   audioEnabled = false,
   pip = false,
   onPlaying,
+  setFullResolution,
 }: MSEPlayerProps) {
   let connectTS: number = 0;
 
@@ -50,6 +60,15 @@ function MSEPlayer({
     return `${baseUrl.replace(/^http/, "ws")}live/mse/api/ws?src=${camera}`;
   }, [camera]);
 
+  const handleLoadedMetadata = useCallback(() => {
+    if (videoRef.current && setFullResolution) {
+      setFullResolution({
+        width: videoRef.current.videoWidth,
+        height: videoRef.current.videoHeight,
+      });
+    }
+  }, [setFullResolution]);
+
   const play = () => {
     const currentVideo = videoRef.current;
 
@@ -286,6 +305,7 @@ function MSEPlayer({
       playsInline
       preload="auto"
       onLoadedData={onPlaying}
+      onLoadedMetadata={handleLoadedMetadata}
       muted={!audioEnabled}
     />
   );

web/src/components/player/dynamic/DynamicVideoPlayer.tsx

@@ -9,6 +9,7 @@ import { DynamicVideoController } from "./DynamicVideoController";
 import HlsVideoPlayer from "../HlsVideoPlayer";
 import { TimeRange } from "@/types/timeline";
 import ActivityIndicator from "@/components/indicators/activity-indicator";
+import { VideoResolutionType } from "@/types/live";
 
 /**
  * Dynamically switches between video playback and scrubbing preview player.
@@ -24,6 +25,7 @@ type DynamicVideoPlayerProps = {
   onControllerReady: (controller: DynamicVideoController) => void;
   onTimestampUpdate?: (timestamp: number) => void;
   onClipEnded?: () => void;
+  setFullResolution: React.Dispatch<React.SetStateAction<VideoResolutionType>>;
 };
 export default function DynamicVideoPlayer({
   className,
@@ -36,6 +38,7 @@ export default function DynamicVideoPlayer({
   onControllerReady,
   onTimestampUpdate,
   onClipEnded,
+  setFullResolution,
 }: DynamicVideoPlayerProps) {
   const apiHost = useApiHost();
   const { data: config } = useSWR<FrigateConfig>("config");
@@ -182,6 +185,7 @@ export default function DynamicVideoPlayer({
           setIsLoading(false);
           setNoRecording(false);
         }}
+        setFullResolution={setFullResolution}
       />
       <PreviewPlayer
         className={`${isScrubbing || isLoading ? "visible" : "hidden"} ${className}`}

web/src/types/live.ts

@@ -1 +1,5 @@
 export type LivePlayerMode = "webrtc" | "mse" | "jsmpeg" | "debug";
+export type VideoResolutionType = {
+  width: number;
+  height: number;
+};

web/src/types/record.ts

@@ -38,3 +38,6 @@ export type RecordingStartingPoint = {
   startTime: number;
   severity: ReviewSeverity;
 };
+
+export const ASPECT_VERTICAL_LAYOUT = 1.5;
+export const ASPECT_WIDE_LAYOUT = 2;

web/src/views/events/RecordingView.tsx

@@ -41,6 +41,8 @@ import MobileReviewSettingsDrawer from "@/components/overlay/MobileReviewSetting
 import Logo from "@/components/Logo";
 import { Skeleton } from "@/components/ui/skeleton";
 import { FaVideo } from "react-icons/fa";
+import { VideoResolutionType } from "@/types/live";
+import { ASPECT_VERTICAL_LAYOUT, ASPECT_WIDE_LAYOUT } from "@/types/record";
 
 const SEGMENT_DURATION = 30;
 
@@ -189,9 +191,18 @@ export function RecordingView({
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [currentTime, scrubbing]);
 
+  const [fullResolution, setFullResolution] = useState<VideoResolutionType>({
+    width: 0,
+    height: 0,
+  });
+
   const onSelectCamera = useCallback(
     (newCam: string) => {
       setMainCamera(newCam);
+      setFullResolution({
+        width: 0,
+        height: 0,
+      });
       setPlaybackStart(currentTime);
     },
     [currentTime],
@@ -205,6 +216,10 @@ export function RecordingView({
         return undefined;
       }
 
+      if (cam == mainCamera && fullResolution.width && fullResolution.height) {
+        return fullResolution.width / fullResolution.height;
+      }
+
       const camera = config.cameras[cam];
 
       if (!camera) {
@@ -213,7 +228,7 @@ export function RecordingView({
 
       return camera.detect.width / camera.detect.height;
     },
-    [config],
+    [config, fullResolution, mainCamera],
   );
 
   const mainCameraAspect = useMemo(() => {
@@ -221,9 +236,9 @@ export function RecordingView({
 
     if (!aspectRatio) {
       return "normal";
-    } else if (aspectRatio > 2) {
+    } else if (aspectRatio > ASPECT_WIDE_LAYOUT) {
       return "wide";
-    } else if (aspectRatio < 16 / 9) {
+    } else if (aspectRatio < ASPECT_VERTICAL_LAYOUT) {
       return "tall";
     } else {
       return "normal";
@@ -397,6 +412,7 @@ export function RecordingView({
                   mainControllerRef.current = controller;
                 }}
                 isScrubbing={scrubbing || exportMode == "timeline"}
+                setFullResolution={setFullResolution}
               />
             </div>
             {isDesktop && (

web/src/views/live/LiveCameraView.tsx

@@ -20,6 +20,7 @@ import { TooltipProvider } from "@/components/ui/tooltip";
 import { useResizeObserver } from "@/hooks/resize-observer";
 import useKeyboardListener from "@/hooks/use-keyboard-listener";
 import { CameraConfig } from "@/types/frigateConfig";
+import { VideoResolutionType } from "@/types/live";
 import { CameraPtzInfo } from "@/types/ptz";
 import { RecordingStartingPoint } from "@/types/record";
 import React, {
@@ -149,14 +150,24 @@ export default function LiveCameraView({ camera }: LiveCameraViewProps) {
   const [fullscreen, setFullscreen] = useState(false);
   const [pip, setPip] = useState(false);
 
+  const [fullResolution, setFullResolution] = useState<VideoResolutionType>({
+    width: 0,
+    height: 0,
+  });
+
   const growClassName = useMemo(() => {
-    const aspect = camera.detect.width / camera.detect.height;
+    let aspect;
+    if (fullResolution.width && fullResolution.height) {
+      aspect = fullResolution.width / fullResolution.height;
+    } else {
+      aspect = camera.detect.width / camera.detect.height;
+    }
 
     if (isMobile) {
       if (isPortrait) {
         return "absolute left-2 right-2 top-[50%] -translate-y-[50%]";
       } else {
-        if (aspect > 16 / 9) {
+        if (aspect > 1.5) {
           return "p-2 absolute left-0 top-[50%] -translate-y-[50%]";
         } else {
           return "p-2 absolute top-2 bottom-2 left-[50%] -translate-x-[50%]";
@@ -165,7 +176,7 @@ export default function LiveCameraView({ camera }: LiveCameraViewProps) {
     }
 
     if (fullscreen) {
-      if (aspect > 16 / 9) {
+      if (aspect > 1.5) {
         return "absolute inset-x-2 top-[50%] -translate-y-[50%]";
       } else {
         return "absolute inset-y-2 left-[50%] -translate-x-[50%]";
@@ -173,7 +184,7 @@ export default function LiveCameraView({ camera }: LiveCameraViewProps) {
     } else {
       return "absolute top-2 bottom-2 left-[50%] -translate-x-[50%]";
     }
-  }, [camera, fullscreen, isPortrait]);
+  }, [camera, fullscreen, isPortrait, fullResolution]);
 
   const preferredLiveMode = useMemo(() => {
     if (isSafari || mic) {
@@ -188,8 +199,12 @@ export default function LiveCameraView({ camera }: LiveCameraViewProps) {
   }, [windowWidth, windowHeight]);
 
   const cameraAspectRatio = useMemo(() => {
+    if (fullResolution.width && fullResolution.height) {
+      return fullResolution.width / fullResolution.height;
+    } else {
       return camera.detect.width / camera.detect.height;
-  }, [camera]);
+    }
+  }, [camera, fullResolution]);
 
   const aspectRatio = useMemo<number>(() => {
     if (isMobile || fullscreen) {
@@ -347,6 +362,7 @@ export default function LiveCameraView({ camera }: LiveCameraViewProps) {
               iOSCompatFullScreen={isIOS}
               preferredLiveMode={preferredLiveMode}
               pip={pip}
+              setFullResolution={setFullResolution}
             />
           </div>
           {camera.onvif.host != "" && (