Improve thumbnails and live player (#9828)

* Don't show gif until event is over and fix aspect

* Be more efficient about updating events

* ensure previews are sorted

* Don't show live view when window is not visible

* Move debug camera to separate view

* Improve jpg loading

* Ensure still is updated on player live finish

* Don't reload when window not visible

* Only disconnect instead of full remove

* Use start time instead of event over to determine gif

frigate/http.py

@@ -2294,9 +2294,9 @@ def preview_thumbnail(camera_name, frame_time):
     file_check = f"{file_start}-{frame_time}.jpg"
     selected_preview = None
 
-    for file in os.listdir(preview_dir):
+    for file in sorted(os.listdir(preview_dir)):
         if file.startswith(file_start):
-            if file < file_check:
+            if file > file_check:
                 selected_preview = file
                 break
 

web/src/components/camera/AutoUpdatingCameraImage.tsx

@@ -1,4 +1,4 @@
-import { useCallback, useState } from "react";
+import { useCallback, useEffect, useState } from "react";
 import CameraImage from "./CameraImage";
 
 type AutoUpdatingCameraImageProps = {
@@ -20,19 +20,41 @@ export default function AutoUpdatingCameraImage({
 }: AutoUpdatingCameraImageProps) {
   const [key, setKey] = useState(Date.now());
   const [fps, setFps] = useState<string>("0");
+  const [timeoutId, setTimeoutId] = useState<NodeJS.Timeout>();
+
+  useEffect(() => {
+    if (reloadInterval == -1) {
+      return;
+    }
+
+    setKey(Date.now());
+
+    return () => {
+      if (timeoutId) {
+        clearTimeout(timeoutId);
+        setTimeoutId(undefined);
+      }
+    };
+  }, [reloadInterval]);
 
   const handleLoad = useCallback(() => {
+    if (reloadInterval == -1) {
+      return;
+    }
+
     const loadTime = Date.now() - key;
 
     if (showFps) {
       setFps((1000 / Math.max(loadTime, reloadInterval)).toFixed(1));
     }
 
+    setTimeoutId(
       setTimeout(
         () => {
           setKey(Date.now());
         },
         loadTime > reloadInterval ? 1 : reloadInterval
+      )
     );
   }, [key, setFps]);
 

web/src/components/camera/DebugCameraImage.tsx

@@ -0,0 +1,150 @@
+import { Switch } from "../ui/switch";
+import { Label } from "../ui/label";
+import { CameraConfig } from "@/types/frigateConfig";
+import { Button } from "../ui/button";
+import { LuSettings } from "react-icons/lu";
+import { useCallback, useMemo, useState } from "react";
+import { Card, CardContent, CardHeader, CardTitle } from "../ui/card";
+import { usePersistence } from "@/hooks/use-persistence";
+import AutoUpdatingCameraImage from "./AutoUpdatingCameraImage";
+
+type Options = { [key: string]: boolean };
+
+const emptyObject = Object.freeze({});
+
+type DebugCameraImageProps = {
+  className?: string;
+  cameraConfig: CameraConfig;
+};
+
+export default function DebugCameraImage({
+  className,
+  cameraConfig,
+}: DebugCameraImageProps) {
+  const [showSettings, setShowSettings] = useState(false);
+  const [options, setOptions] = usePersistence(
+    `${cameraConfig?.name}-feed`,
+    emptyObject
+  );
+  const handleSetOption = useCallback(
+    (id: string, value: boolean) => {
+      const newOptions = { ...options, [id]: value };
+      setOptions(newOptions);
+    },
+    [options]
+  );
+  const searchParams = useMemo(
+    () =>
+      new URLSearchParams(
+        Object.keys(options).reduce((memo, key) => {
+          //@ts-ignore we know this is correct
+          memo.push([key, options[key] === true ? "1" : "0"]);
+          return memo;
+        }, [])
+      ),
+    [options]
+  );
+  const handleToggleSettings = useCallback(() => {
+    setShowSettings(!showSettings);
+  }, [showSettings]);
+
+  return (
+    <div className={className}>
+      <AutoUpdatingCameraImage
+        camera={cameraConfig.name}
+        searchParams={searchParams}
+      />
+      <Button onClick={handleToggleSettings} variant="link" size="sm">
+        <span className="w-5 h-5">
+          <LuSettings />
+        </span>{" "}
+        <span>{showSettings ? "Hide" : "Show"} Options</span>
+      </Button>
+      {showSettings ? (
+        <Card>
+          <CardHeader>
+            <CardTitle>Options</CardTitle>
+          </CardHeader>
+          <CardContent>
+            <DebugSettings
+              handleSetOption={handleSetOption}
+              options={options}
+            />
+          </CardContent>
+        </Card>
+      ) : null}
+    </div>
+  );
+}
+
+type DebugSettingsProps = {
+  handleSetOption: (id: string, value: boolean) => void;
+  options: Options;
+};
+
+function DebugSettings({ handleSetOption, options }: DebugSettingsProps) {
+  return (
+    <div className="grid grid-cols-2 md:grid-cols-3 lg:grid-cols-4 gap-4">
+      <div className="flex items-center space-x-2">
+        <Switch
+          id="bbox"
+          checked={options["bbox"]}
+          onCheckedChange={(isChecked) => {
+            handleSetOption("bbox", isChecked);
+          }}
+        />
+        <Label htmlFor="bbox">Bounding Box</Label>
+      </div>
+      <div className="flex items-center space-x-2">
+        <Switch
+          id="timestamp"
+          checked={options["timestamp"]}
+          onCheckedChange={(isChecked) => {
+            handleSetOption("timestamp", isChecked);
+          }}
+        />
+        <Label htmlFor="timestamp">Timestamp</Label>
+      </div>
+      <div className="flex items-center space-x-2">
+        <Switch
+          id="zones"
+          checked={options["zones"]}
+          onCheckedChange={(isChecked) => {
+            handleSetOption("zones", isChecked);
+          }}
+        />
+        <Label htmlFor="zones">Zones</Label>
+      </div>
+      <div className="flex items-center space-x-2">
+        <Switch
+          id="mask"
+          checked={options["mask"]}
+          onCheckedChange={(isChecked) => {
+            handleSetOption("mask", isChecked);
+          }}
+        />
+        <Label htmlFor="mask">Mask</Label>
+      </div>
+      <div className="flex items-center space-x-2">
+        <Switch
+          id="motion"
+          checked={options["motion"]}
+          onCheckedChange={(isChecked) => {
+            handleSetOption("motion", isChecked);
+          }}
+        />
+        <Label htmlFor="motion">Motion</Label>
+      </div>
+      <div className="flex items-center space-x-2">
+        <Switch
+          id="regions"
+          checked={options["regions"]}
+          onCheckedChange={(isChecked) => {
+            handleSetOption("regions", isChecked);
+          }}
+        />
+        <Label htmlFor="regions">Regions</Label>
+      </div>
+    </div>
+  );
+}

web/src/components/image/AnimatedEventThumbnail.tsx

@@ -2,18 +2,50 @@ import { baseUrl } from "@/api/baseUrl";
 import { Event as FrigateEvent } from "@/types/event";
 import TimeAgo from "../dynamic/TimeAgo";
 import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip";
+import { useMemo } from "react";
+import { useApiHost } from "@/api";
+import useSWR from "swr";
+import { FrigateConfig } from "@/types/frigateConfig";
 
 type AnimatedEventThumbnailProps = {
   event: FrigateEvent;
 };
 export function AnimatedEventThumbnail({ event }: AnimatedEventThumbnailProps) {
+  const apiHost = useApiHost();
+  const { data: config } = useSWR<FrigateConfig>("config");
+
+  const imageUrl = useMemo(() => {
+    if (Date.now() / 1000 < event.start_time + 20) {
+      return `${apiHost}api/preview/${event.camera}/${event.start_time}/thumbnail.jpg`;
+    }
+
+    return `${baseUrl}api/events/${event.id}/preview.gif`;
+  }, [event]);
+
+  const aspect = useMemo(() => {
+    if (!config) {
+      return "";
+    }
+
+    const detect = config.cameras[event.camera].detect;
+    const aspect = detect.width / detect.height;
+
+    if (aspect > 2) {
+      return "aspect-wide";
+    } else if (aspect < 1) {
+      return "aspect-tall";
+    } else {
+      return "aspect-video";
+    }
+  }, [config, event]);
+
   return (
     <Tooltip>
       <TooltipTrigger asChild>
         <div
-          className="relative rounded bg-cover aspect-video h-24 bg-no-repeat bg-center mr-4"
+          className={`relative rounded bg-cover h-24 bg-no-repeat bg-center mr-4 ${aspect}`}
           style={{
-            backgroundImage: `url(${baseUrl}api/events/${event.id}/preview.gif)`,
+            backgroundImage: `url(${imageUrl})`,
           }}
         >
           <div className="absolute bottom-0 w-full h-6 bg-gradient-to-t from-slate-900/50 to-transparent rounded">

web/src/components/player/LivePlayer.tsx

@@ -2,13 +2,7 @@ import WebRtcPlayer from "./WebRTCPlayer";
 import { CameraConfig } from "@/types/frigateConfig";
 import AutoUpdatingCameraImage from "../camera/AutoUpdatingCameraImage";
 import ActivityIndicator from "../ui/activity-indicator";
-import { Button } from "../ui/button";
+import { useEffect, useMemo, useState } from "react";
-import { LuSettings } from "react-icons/lu";
-import { useCallback, useEffect, useMemo, useState } from "react";
-import { Card, CardContent, CardHeader, CardTitle } from "../ui/card";
-import { Switch } from "../ui/switch";
-import { Label } from "../ui/label";
-import { usePersistence } from "@/hooks/use-persistence";
 import MSEPlayer from "./MsePlayer";
 import JSMpegPlayer from "./JSMpegPlayer";
 import { MdCircle, MdLeakAdd } from "react-icons/md";
@@ -19,31 +13,33 @@ import { useRecordingsState } from "@/api/ws";
 import { LivePlayerMode } from "@/types/live";
 import useCameraLiveMode from "@/hooks/use-camera-live-mode";
 
-const emptyObject = Object.freeze({});
-
 type LivePlayerProps = {
   className?: string;
   cameraConfig: CameraConfig;
   preferredLiveMode?: LivePlayerMode;
   showStillWithoutActivity?: boolean;
+  windowVisible?: boolean;
 };
 
-type Options = { [key: string]: boolean };
-
 export default function LivePlayer({
   className,
   cameraConfig,
   preferredLiveMode,
   showStillWithoutActivity = true,
+  windowVisible = true,
 }: LivePlayerProps) {
   // camera activity
+
   const { activeMotion, activeAudio, activeTracking } =
     useCameraActivity(cameraConfig);
 
   const cameraActive = useMemo(
-    () => activeMotion || activeTracking,
+    () => windowVisible && (activeMotion || activeTracking),
-    [activeMotion, activeTracking]
+    [activeMotion, activeTracking, windowVisible]
   );
+
+  // camera live state
+
   const liveMode = useCameraLiveMode(cameraConfig, preferredLiveMode);
 
   const [liveReady, setLiveReady] = useState(false);
@@ -63,34 +59,23 @@ export default function LivePlayer({
 
   const { payload: recording } = useRecordingsState(cameraConfig.name);
 
-  // debug view settings
+  // camera still state
 
-  const [showSettings, setShowSettings] = useState(false);
+  const stillReloadInterval = useMemo(() => {
-  const [options, setOptions] = usePersistence(
+    if (!windowVisible) {
-    `${cameraConfig?.name}-feed`,
+      return -1; // no reason to update the image when the window is not visible
-    emptyObject
+    }
-  );
+
-  const handleSetOption = useCallback(
+    if (liveReady) {
-    (id: string, value: boolean) => {
+      return 60000;
-      const newOptions = { ...options, [id]: value };
+    }
-      setOptions(newOptions);
+
-    },
+    if (cameraActive) {
-    [options]
+      return 200;
-  );
+    }
-  const searchParams = useMemo(
+
-    () =>
+    return 30000;
-      new URLSearchParams(
+  }, []);
-        Object.keys(options).reduce((memo, key) => {
-          //@ts-ignore we know this is correct
-          memo.push([key, options[key] === true ? "1" : "0"]);
-          return memo;
-        }, [])
-      ),
-    [options]
-  );
-  const handleToggleSettings = useCallback(() => {
-    setShowSettings(!showSettings);
-  }, [showSettings]);
 
   if (!cameraConfig) {
     return <ActivityIndicator />;
@@ -111,6 +96,7 @@ export default function LivePlayer({
         <MSEPlayer
           className={`rounded-2xl h-full ${liveReady ? "" : "hidden"}`}
           camera={cameraConfig.name}
+          playbackEnabled={cameraActive}
           onPlaying={() => setLiveReady(true)}
         />
       );
@@ -131,34 +117,6 @@ export default function LivePlayer({
         height={cameraConfig.detect.height}
       />
     );
-  } else if (liveMode == "debug") {
-    player = (
-      <>
-        <AutoUpdatingCameraImage
-          camera={cameraConfig.name}
-          searchParams={searchParams}
-        />
-        <Button onClick={handleToggleSettings} variant="link" size="sm">
-          <span className="w-5 h-5">
-            <LuSettings />
-          </span>{" "}
-          <span>{showSettings ? "Hide" : "Show"} Options</span>
-        </Button>
-        {showSettings ? (
-          <Card>
-            <CardHeader>
-              <CardTitle>Options</CardTitle>
-            </CardHeader>
-            <CardContent>
-              <DebugSettings
-                handleSetOption={handleSetOption}
-                options={options}
-              />
-            </CardContent>
-          </Card>
-        ) : null}
-      </>
-    );
   } else {
     player = <ActivityIndicator />;
   }
@@ -173,8 +131,7 @@ export default function LivePlayer({
     >
       <div className="absolute top-0 left-0 right-0 rounded-2xl z-10 w-full h-[30%] bg-gradient-to-b from-black/20 to-transparent pointer-events-none"></div>
       <div className="absolute bottom-0 left-0 right-0 rounded-2xl z-10 w-full h-[10%] bg-gradient-to-t from-black/20 to-transparent pointer-events-none"></div>
-
+      {player}
-      {(showStillWithoutActivity == false || cameraActive) && player}
 
       <div
         className={`absolute left-0 top-0 right-0 bottom-0 w-full ${
@@ -185,7 +142,7 @@ export default function LivePlayer({
           className="w-full h-full"
           camera={cameraConfig.name}
           showFps={false}
-          reloadInterval={cameraActive && !liveReady ? 200 : 30000}
+          reloadInterval={stillReloadInterval}
         />
       </div>
 
@@ -220,75 +177,3 @@ export default function LivePlayer({
     </div>
   );
 }
-
-type DebugSettingsProps = {
-  handleSetOption: (id: string, value: boolean) => void;
-  options: Options;
-};
-
-function DebugSettings({ handleSetOption, options }: DebugSettingsProps) {
-  return (
-    <div className="grid grid-cols-2 md:grid-cols-3 lg:grid-cols-4 gap-4">
-      <div className="flex items-center space-x-2">
-        <Switch
-          id="bbox"
-          checked={options["bbox"]}
-          onCheckedChange={(isChecked) => {
-            handleSetOption("bbox", isChecked);
-          }}
-        />
-        <Label htmlFor="bbox">Bounding Box</Label>
-      </div>
-      <div className="flex items-center space-x-2">
-        <Switch
-          id="timestamp"
-          checked={options["timestamp"]}
-          onCheckedChange={(isChecked) => {
-            handleSetOption("timestamp", isChecked);
-          }}
-        />
-        <Label htmlFor="timestamp">Timestamp</Label>
-      </div>
-      <div className="flex items-center space-x-2">
-        <Switch
-          id="zones"
-          checked={options["zones"]}
-          onCheckedChange={(isChecked) => {
-            handleSetOption("zones", isChecked);
-          }}
-        />
-        <Label htmlFor="zones">Zones</Label>
-      </div>
-      <div className="flex items-center space-x-2">
-        <Switch
-          id="mask"
-          checked={options["mask"]}
-          onCheckedChange={(isChecked) => {
-            handleSetOption("mask", isChecked);
-          }}
-        />
-        <Label htmlFor="mask">Mask</Label>
-      </div>
-      <div className="flex items-center space-x-2">
-        <Switch
-          id="motion"
-          checked={options["motion"]}
-          onCheckedChange={(isChecked) => {
-            handleSetOption("motion", isChecked);
-          }}
-        />
-        <Label htmlFor="motion">Motion</Label>
-      </div>
-      <div className="flex items-center space-x-2">
-        <Switch
-          id="regions"
-          checked={options["regions"]}
-          onCheckedChange={(isChecked) => {
-            handleSetOption("regions", isChecked);
-          }}
-        />
-        <Label htmlFor="regions">Regions</Label>
-      </div>
-    </div>
-  );
-}

web/src/components/player/MsePlayer.tsx

@@ -4,10 +4,16 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 type MSEPlayerProps = {
   camera: string;
   className?: string;
+  playbackEnabled?: boolean;
   onPlaying?: () => void;
 };
 
-function MSEPlayer({ camera, className, onPlaying }: MSEPlayerProps) {
+function MSEPlayer({
+  camera,
+  className,
+  playbackEnabled = true,
+  onPlaying,
+}: MSEPlayerProps) {
   let connectTS: number = 0;
 
   const RECONNECT_TIMEOUT: number = 30000;
@@ -203,6 +209,10 @@ function MSEPlayer({ camera, className, onPlaying }: MSEPlayerProps) {
   };
 
   useEffect(() => {
+    if (!playbackEnabled) {
+      return;
+    }
+
     // iOS 17.1+ uses ManagedMediaSource
     const MediaSourceConstructor =
       "ManagedMediaSource" in window ? window.ManagedMediaSource : MediaSource;
@@ -236,18 +246,12 @@ function MSEPlayer({ camera, className, onPlaying }: MSEPlayerProps) {
       observer.observe(videoRef.current!);
     }
 
-    return () => {
-      onDisconnect();
-    };
-  }, [onDisconnect, onConnect]);
-
-  useEffect(() => {
     onConnect();
 
     return () => {
       onDisconnect();
     };
-  }, [wsURL]);
+  }, [playbackEnabled, onDisconnect, onConnect]);
 
   return (
     <video

web/src/pages/Live.tsx

@@ -1,22 +1,46 @@
+import { useFrigateEvents } from "@/api/ws";
 import { AnimatedEventThumbnail } from "@/components/image/AnimatedEventThumbnail";
 import LivePlayer from "@/components/player/LivePlayer";
 import { ScrollArea, ScrollBar } from "@/components/ui/scroll-area";
 import { TooltipProvider } from "@/components/ui/tooltip";
 import { Event as FrigateEvent } from "@/types/event";
 import { FrigateConfig } from "@/types/frigateConfig";
-import { useMemo } from "react";
+import { useCallback, useEffect, useMemo, useState } from "react";
 import useSWR from "swr";
 
 function Live() {
   const { data: config } = useSWR<FrigateConfig>("config");
 
   // recent events
-
+  const { payload: eventUpdate } = useFrigateEvents();
-  const { data: allEvents } = useSWR<FrigateEvent[]>(
+  const { data: allEvents, mutate: updateEvents } = useSWR<FrigateEvent[]>(
     ["events", { limit: 10 }],
-    { refreshInterval: 60000 }
+    { revalidateOnFocus: false }
   );
 
+  useEffect(() => {
+    if (!eventUpdate) {
+      return;
+    }
+
+    // if event is ended and was saved, update events list
+    if (
+      eventUpdate.type == "end" &&
+      (eventUpdate.after.has_clip || eventUpdate.after.has_snapshot)
+    ) {
+      updateEvents();
+      return;
+    }
+
+    // if event is updated and has become a saved event, update events list
+    if (
+      !(eventUpdate.before.has_clip || eventUpdate.before.has_snapshot) &&
+      (eventUpdate.after.has_clip || eventUpdate.after.has_snapshot)
+    ) {
+      updateEvents();
+    }
+  }, [eventUpdate]);
+
   const events = useMemo(() => {
     if (!allEvents) {
       return [];
@@ -40,6 +64,19 @@ function Live() {
       .sort((aConf, bConf) => aConf.ui.order - bConf.ui.order);
   }, [config]);
 
+  const [windowVisible, setWindowVisible] = useState(true);
+  const visibilityListener = useCallback(() => {
+    setWindowVisible(document.visibilityState == "visible");
+  }, []);
+
+  useEffect(() => {
+    addEventListener("visibilitychange", visibilityListener);
+
+    return () => {
+      removeEventListener("visibilitychange", visibilityListener);
+    };
+  }, []);
+
   return (
     <>
       {events && events.length > 0 && (
@@ -62,7 +99,7 @@ function Live() {
           if (aspectRatio > 2) {
             grow = "md:col-span-2 aspect-wide";
           } else if (aspectRatio < 1) {
-            grow = `md:row-span-2 aspect-[8/9] md:h-full`;
+            grow = `md:row-span-2 aspect-tall md:h-full`;
           } else {
             grow = "aspect-video";
           }
@@ -70,6 +107,7 @@ function Live() {
             <LivePlayer
               key={camera.name}
               className={`mb-2 md:mb-0 rounded-2xl bg-black ${grow}`}
+              windowVisible={windowVisible}
               cameraConfig={camera}
               preferredLiveMode="mse"
             />

web/tailwind.config.js

@@ -25,7 +25,7 @@ module.exports = {
       },
       aspectRatio: {
         wide: "32 / 9",
-        tall: "9 / 16",
+        tall: "8 / 9",
       },
       colors: {
         border: "hsl(var(--border))",