Improve thumbnails and live player (#9828)

* Don't show gif until event is over and fix aspect * Be more efficient about updating events * ensure previews are sorted * Don't show live view when window is not visible * Move debug camera to separate view * Improve jpg loading * Ensure still is updated on player live finish * Don't reload when window not visible * Only disconnect instead of full remove * Use start time instead of event over to determine gif
2024-12-19 19:06:16 +01:00 · 2024-02-12 18:28:36 -07:00 · 2024-02-12 18:28:36 -07:00 · 63bc1b1582
commit 63bc1b1582
parent f54cb21bd0
8 changed files with 298 additions and 167 deletions
--- a/frigate/http.py
+++ b/frigate/http.py
@ -2294,9 +2294,9 @@ def preview_thumbnail(camera_name, frame_time):
    file_check = f"{file_start}-{frame_time}.jpg"
    selected_preview = None

-    for file in os.listdir(preview_dir):
+    for file in sorted(os.listdir(preview_dir)):
        if file.startswith(file_start):
-            if file < file_check:
+            if file > file_check:
                selected_preview = file
                break

--- a/web/src/components/camera/AutoUpdatingCameraImage.tsx
+++ b/web/src/components/camera/AutoUpdatingCameraImage.tsx
@ -1,4 +1,4 @@
-import { useCallback, useState } from "react";
+import { useCallback, useEffect, useState } from "react";
 import CameraImage from "./CameraImage";

 type AutoUpdatingCameraImageProps = {
@ -20,19 +20,41 @@ export default function AutoUpdatingCameraImage({
 }: AutoUpdatingCameraImageProps) {
  const [key, setKey] = useState(Date.now());
  const [fps, setFps] = useState<string>("0");
+  const [timeoutId, setTimeoutId] = useState<NodeJS.Timeout>();
+
+  useEffect(() => {
+    if (reloadInterval == -1) {
+      return;
+    }
+
+    setKey(Date.now());
+
+    return () => {
+      if (timeoutId) {
+        clearTimeout(timeoutId);
+        setTimeoutId(undefined);
+      }
+    };
+  }, [reloadInterval]);

  const handleLoad = useCallback(() => {
+    if (reloadInterval == -1) {
+      return;
+    }
+
    const loadTime = Date.now() - key;

    if (showFps) {
      setFps((1000 / Math.max(loadTime, reloadInterval)).toFixed(1));
    }

-    setTimeout(
-      () => {
-        setKey(Date.now());
-      },
-      loadTime > reloadInterval ? 1 : reloadInterval
+    setTimeoutId(
+      setTimeout(
+        () => {
+          setKey(Date.now());
+        },
+        loadTime > reloadInterval ? 1 : reloadInterval
+      )
    );
  }, [key, setFps]);

--- a/web/src/components/camera/DebugCameraImage.tsx
+++ b/web/src/components/camera/DebugCameraImage.tsx
@ -0,0 +1,150 @@
+import { Switch } from "../ui/switch";
+import { Label } from "../ui/label";
+import { CameraConfig } from "@/types/frigateConfig";
+import { Button } from "../ui/button";
+import { LuSettings } from "react-icons/lu";
+import { useCallback, useMemo, useState } from "react";
+import { Card, CardContent, CardHeader, CardTitle } from "../ui/card";
+import { usePersistence } from "@/hooks/use-persistence";
+import AutoUpdatingCameraImage from "./AutoUpdatingCameraImage";
+
+type Options = { [key: string]: boolean };
+
+const emptyObject = Object.freeze({});
+
+type DebugCameraImageProps = {
+  className?: string;
+  cameraConfig: CameraConfig;
+};
+
+export default function DebugCameraImage({
+  className,
+  cameraConfig,
+}: DebugCameraImageProps) {
+  const [showSettings, setShowSettings] = useState(false);
+  const [options, setOptions] = usePersistence(
+    `${cameraConfig?.name}-feed`,
+    emptyObject
+  );
+  const handleSetOption = useCallback(
+    (id: string, value: boolean) => {
+      const newOptions = { ...options, [id]: value };
+      setOptions(newOptions);
+    },
+    [options]
+  );
+  const searchParams = useMemo(
+    () =>
+      new URLSearchParams(
+        Object.keys(options).reduce((memo, key) => {
+          //@ts-ignore we know this is correct
+          memo.push([key, options[key] === true ? "1" : "0"]);
+          return memo;
+        }, [])
+      ),
+    [options]
+  );
+  const handleToggleSettings = useCallback(() => {
+    setShowSettings(!showSettings);
+  }, [showSettings]);
+
+  return (
+    <div className={className}>
+      <AutoUpdatingCameraImage
+        camera={cameraConfig.name}
+        searchParams={searchParams}
+      />
+      <Button onClick={handleToggleSettings} variant="link" size="sm">
+        <span className="w-5 h-5">
+          <LuSettings />
+        </span>{" "}
+        <span>{showSettings ? "Hide" : "Show"} Options</span>
+      </Button>
+      {showSettings ? (
+        <Card>
+          <CardHeader>
+            <CardTitle>Options</CardTitle>
+          </CardHeader>
+          <CardContent>
+            <DebugSettings
+              handleSetOption={handleSetOption}
+              options={options}
+            />
+          </CardContent>
+        </Card>
+      ) : null}
+    </div>
+  );
+}
+
+type DebugSettingsProps = {
+  handleSetOption: (id: string, value: boolean) => void;
+  options: Options;
+};
+
+function DebugSettings({ handleSetOption, options }: DebugSettingsProps) {
+  return (
+    <div className="grid grid-cols-2 md:grid-cols-3 lg:grid-cols-4 gap-4">
+      <div className="flex items-center space-x-2">
+        <Switch
+          id="bbox"
+          checked={options["bbox"]}
+          onCheckedChange={(isChecked) => {
+            handleSetOption("bbox", isChecked);
+          }}
+        />
+        <Label htmlFor="bbox">Bounding Box</Label>
+      </div>
+      <div className="flex items-center space-x-2">
+        <Switch
+          id="timestamp"
+          checked={options["timestamp"]}
+          onCheckedChange={(isChecked) => {
+            handleSetOption("timestamp", isChecked);
+          }}
+        />
+        <Label htmlFor="timestamp">Timestamp</Label>
+      </div>
+      <div className="flex items-center space-x-2">
+        <Switch
+          id="zones"
+          checked={options["zones"]}
+          onCheckedChange={(isChecked) => {
+            handleSetOption("zones", isChecked);
+          }}
+        />
+        <Label htmlFor="zones">Zones</Label>
+      </div>
+      <div className="flex items-center space-x-2">
+        <Switch
+          id="mask"
+          checked={options["mask"]}
+          onCheckedChange={(isChecked) => {
+            handleSetOption("mask", isChecked);
+          }}
+        />
+        <Label htmlFor="mask">Mask</Label>
+      </div>
+      <div className="flex items-center space-x-2">
+        <Switch
+          id="motion"
+          checked={options["motion"]}
+          onCheckedChange={(isChecked) => {
+            handleSetOption("motion", isChecked);
+          }}
+        />
+        <Label htmlFor="motion">Motion</Label>
+      </div>
+      <div className="flex items-center space-x-2">
+        <Switch
+          id="regions"
+          checked={options["regions"]}
+          onCheckedChange={(isChecked) => {
+            handleSetOption("regions", isChecked);
+          }}
+        />
+        <Label htmlFor="regions">Regions</Label>
+      </div>
+    </div>
+  );
+}
--- a/web/src/components/image/AnimatedEventThumbnail.tsx
+++ b/web/src/components/image/AnimatedEventThumbnail.tsx
@ -2,18 +2,50 @@ import { baseUrl } from "@/api/baseUrl";
 import { Event as FrigateEvent } from "@/types/event";
 import TimeAgo from "../dynamic/TimeAgo";
 import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip";
+import { useMemo } from "react";
+import { useApiHost } from "@/api";
+import useSWR from "swr";
+import { FrigateConfig } from "@/types/frigateConfig";

 type AnimatedEventThumbnailProps = {
  event: FrigateEvent;
 };
 export function AnimatedEventThumbnail({ event }: AnimatedEventThumbnailProps) {
+  const apiHost = useApiHost();
+  const { data: config } = useSWR<FrigateConfig>("config");
+
+  const imageUrl = useMemo(() => {
+    if (Date.now() / 1000 < event.start_time + 20) {
+      return `${apiHost}api/preview/${event.camera}/${event.start_time}/thumbnail.jpg`;
+    }
+
+    return `${baseUrl}api/events/${event.id}/preview.gif`;
+  }, [event]);
+
+  const aspect = useMemo(() => {
+    if (!config) {
+      return "";
+    }
+
+    const detect = config.cameras[event.camera].detect;
+    const aspect = detect.width / detect.height;
+
+    if (aspect > 2) {
+      return "aspect-wide";
+    } else if (aspect < 1) {
+      return "aspect-tall";
+    } else {
+      return "aspect-video";
+    }
+  }, [config, event]);
+
  return (
    <Tooltip>
      <TooltipTrigger asChild>
        <div
-          className="relative rounded bg-cover aspect-video h-24 bg-no-repeat bg-center mr-4"
+          className={`relative rounded bg-cover h-24 bg-no-repeat bg-center mr-4 ${aspect}`}
          style={{
-            backgroundImage: `url(${baseUrl}api/events/${event.id}/preview.gif)`,
+            backgroundImage: `url(${imageUrl})`,
          }}
        >
          <div className="absolute bottom-0 w-full h-6 bg-gradient-to-t from-slate-900/50 to-transparent rounded">
--- a/web/src/components/player/LivePlayer.tsx
+++ b/web/src/components/player/LivePlayer.tsx
@ -2,13 +2,7 @@ import WebRtcPlayer from "./WebRTCPlayer";
 import { CameraConfig } from "@/types/frigateConfig";
 import AutoUpdatingCameraImage from "../camera/AutoUpdatingCameraImage";
 import ActivityIndicator from "../ui/activity-indicator";
-import { Button } from "../ui/button";
-import { LuSettings } from "react-icons/lu";
-import { useCallback, useEffect, useMemo, useState } from "react";
-import { Card, CardContent, CardHeader, CardTitle } from "../ui/card";
-import { Switch } from "../ui/switch";
-import { Label } from "../ui/label";
-import { usePersistence } from "@/hooks/use-persistence";
+import { useEffect, useMemo, useState } from "react";
 import MSEPlayer from "./MsePlayer";
 import JSMpegPlayer from "./JSMpegPlayer";
 import { MdCircle, MdLeakAdd } from "react-icons/md";
@ -19,31 +13,33 @@ import { useRecordingsState } from "@/api/ws";
 import { LivePlayerMode } from "@/types/live";
 import useCameraLiveMode from "@/hooks/use-camera-live-mode";

-const emptyObject = Object.freeze({});
-
 type LivePlayerProps = {
  className?: string;
  cameraConfig: CameraConfig;
  preferredLiveMode?: LivePlayerMode;
  showStillWithoutActivity?: boolean;
+  windowVisible?: boolean;
 };

-type Options = { [key: string]: boolean };
-
 export default function LivePlayer({
  className,
  cameraConfig,
  preferredLiveMode,
  showStillWithoutActivity = true,
+  windowVisible = true,
 }: LivePlayerProps) {
  // camera activity
+
  const { activeMotion, activeAudio, activeTracking } =
    useCameraActivity(cameraConfig);

  const cameraActive = useMemo(
-    () => activeMotion || activeTracking,
-    [activeMotion, activeTracking]
+    () => windowVisible && (activeMotion || activeTracking),
+    [activeMotion, activeTracking, windowVisible]
  );
+
+  // camera live state
+
  const liveMode = useCameraLiveMode(cameraConfig, preferredLiveMode);

  const [liveReady, setLiveReady] = useState(false);
@ -63,34 +59,23 @@ export default function LivePlayer({

  const { payload: recording } = useRecordingsState(cameraConfig.name);

-  // debug view settings
+  // camera still state

-  const [showSettings, setShowSettings] = useState(false);
-  const [options, setOptions] = usePersistence(
-    `${cameraConfig?.name}-feed`,
-    emptyObject
-  );
-  const handleSetOption = useCallback(
-    (id: string, value: boolean) => {
-      const newOptions = { ...options, [id]: value };
-      setOptions(newOptions);
-    },
-    [options]
-  );
-  const searchParams = useMemo(
-    () =>
-      new URLSearchParams(
-        Object.keys(options).reduce((memo, key) => {
-          //@ts-ignore we know this is correct
-          memo.push([key, options[key] === true ? "1" : "0"]);
-          return memo;
-        }, [])
-      ),
-    [options]
-  );
-  const handleToggleSettings = useCallback(() => {
-    setShowSettings(!showSettings);
-  }, [showSettings]);
+  const stillReloadInterval = useMemo(() => {
+    if (!windowVisible) {
+      return -1; // no reason to update the image when the window is not visible
+    }
+
+    if (liveReady) {
+      return 60000;
+    }
+
+    if (cameraActive) {
+      return 200;
+    }
+
+    return 30000;
+  }, []);

  if (!cameraConfig) {
    return <ActivityIndicator />;
@ -111,6 +96,7 @@ export default function LivePlayer({
        <MSEPlayer
          className={`rounded-2xl h-full ${liveReady ? "" : "hidden"}`}
          camera={cameraConfig.name}
+          playbackEnabled={cameraActive}
          onPlaying={() => setLiveReady(true)}
        />
      );
@ -131,34 +117,6 @@ export default function LivePlayer({
        height={cameraConfig.detect.height}
      />
    );
-  } else if (liveMode == "debug") {
-    player = (
-      <>
-        <AutoUpdatingCameraImage
-          camera={cameraConfig.name}
-          searchParams={searchParams}
-        />
-        <Button onClick={handleToggleSettings} variant="link" size="sm">
-          <span className="w-5 h-5">
-            <LuSettings />
-          </span>{" "}
-          <span>{showSettings ? "Hide" : "Show"} Options</span>
-        </Button>
-        {showSettings ? (
-          <Card>
-            <CardHeader>
-              <CardTitle>Options</CardTitle>
-            </CardHeader>
-            <CardContent>
-              <DebugSettings
-                handleSetOption={handleSetOption}
-                options={options}
-              />
-            </CardContent>
-          </Card>
-        ) : null}
-      </>
-    );
  } else {
    player = <ActivityIndicator />;
  }
@ -173,8 +131,7 @@ export default function LivePlayer({
    >
      <div className="absolute top-0 left-0 right-0 rounded-2xl z-10 w-full h-[30%] bg-gradient-to-b from-black/20 to-transparent pointer-events-none"></div>
      <div className="absolute bottom-0 left-0 right-0 rounded-2xl z-10 w-full h-[10%] bg-gradient-to-t from-black/20 to-transparent pointer-events-none"></div>
-
-      {(showStillWithoutActivity == false || cameraActive) && player}
+      {player}

      <div
        className={`absolute left-0 top-0 right-0 bottom-0 w-full ${
@ -185,7 +142,7 @@ export default function LivePlayer({
          className="w-full h-full"
          camera={cameraConfig.name}
          showFps={false}
-          reloadInterval={cameraActive && !liveReady ? 200 : 30000}
+          reloadInterval={stillReloadInterval}
        />
      </div>

@ -220,75 +177,3 @@ export default function LivePlayer({
    </div>
  );
 }
-
-type DebugSettingsProps = {
-  handleSetOption: (id: string, value: boolean) => void;
-  options: Options;
-};
-
-function DebugSettings({ handleSetOption, options }: DebugSettingsProps) {
-  return (
-    <div className="grid grid-cols-2 md:grid-cols-3 lg:grid-cols-4 gap-4">
-      <div className="flex items-center space-x-2">
-        <Switch
-          id="bbox"
-          checked={options["bbox"]}
-          onCheckedChange={(isChecked) => {
-            handleSetOption("bbox", isChecked);
-          }}
-        />
-        <Label htmlFor="bbox">Bounding Box</Label>
-      </div>
-      <div className="flex items-center space-x-2">
-        <Switch
-          id="timestamp"
-          checked={options["timestamp"]}
-          onCheckedChange={(isChecked) => {
-            handleSetOption("timestamp", isChecked);
-          }}
-        />
-        <Label htmlFor="timestamp">Timestamp</Label>
-      </div>
-      <div className="flex items-center space-x-2">
-        <Switch
-          id="zones"
-          checked={options["zones"]}
-          onCheckedChange={(isChecked) => {
-            handleSetOption("zones", isChecked);
-          }}
-        />
-        <Label htmlFor="zones">Zones</Label>
-      </div>
-      <div className="flex items-center space-x-2">
-        <Switch
-          id="mask"
-          checked={options["mask"]}
-          onCheckedChange={(isChecked) => {
-            handleSetOption("mask", isChecked);
-          }}
-        />
-        <Label htmlFor="mask">Mask</Label>
-      </div>
-      <div className="flex items-center space-x-2">
-        <Switch
-          id="motion"
-          checked={options["motion"]}
-          onCheckedChange={(isChecked) => {
-            handleSetOption("motion", isChecked);
-          }}
-        />
-        <Label htmlFor="motion">Motion</Label>
-      </div>
-      <div className="flex items-center space-x-2">
-        <Switch
-          id="regions"
-          checked={options["regions"]}
-          onCheckedChange={(isChecked) => {
-            handleSetOption("regions", isChecked);
-          }}
-        />
-        <Label htmlFor="regions">Regions</Label>
-      </div>
-    </div>
-  );
-}
--- a/web/src/components/player/MsePlayer.tsx
+++ b/web/src/components/player/MsePlayer.tsx
@ -4,10 +4,16 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 type MSEPlayerProps = {
  camera: string;
  className?: string;
+  playbackEnabled?: boolean;
  onPlaying?: () => void;
 };

-function MSEPlayer({ camera, className, onPlaying }: MSEPlayerProps) {
+function MSEPlayer({
+  camera,
+  className,
+  playbackEnabled = true,
+  onPlaying,
+}: MSEPlayerProps) {
  let connectTS: number = 0;

  const RECONNECT_TIMEOUT: number = 30000;
@ -203,6 +209,10 @@ function MSEPlayer({ camera, className, onPlaying }: MSEPlayerProps) {
  };

  useEffect(() => {
+    if (!playbackEnabled) {
+      return;
+    }
+
    // iOS 17.1+ uses ManagedMediaSource
    const MediaSourceConstructor =
      "ManagedMediaSource" in window ? window.ManagedMediaSource : MediaSource;
@ -236,18 +246,12 @@ function MSEPlayer({ camera, className, onPlaying }: MSEPlayerProps) {
      observer.observe(videoRef.current!);
    }

-    return () => {
-      onDisconnect();
-    };
-  }, [onDisconnect, onConnect]);
-
-  useEffect(() => {
    onConnect();

    return () => {
      onDisconnect();
    };
-  }, [wsURL]);
+  }, [playbackEnabled, onDisconnect, onConnect]);

  return (
    <video
--- a/web/src/pages/Live.tsx
+++ b/web/src/pages/Live.tsx
@ -1,22 +1,46 @@
+import { useFrigateEvents } from "@/api/ws";
 import { AnimatedEventThumbnail } from "@/components/image/AnimatedEventThumbnail";
 import LivePlayer from "@/components/player/LivePlayer";
 import { ScrollArea, ScrollBar } from "@/components/ui/scroll-area";
 import { TooltipProvider } from "@/components/ui/tooltip";
 import { Event as FrigateEvent } from "@/types/event";
 import { FrigateConfig } from "@/types/frigateConfig";
-import { useMemo } from "react";
+import { useCallback, useEffect, useMemo, useState } from "react";
 import useSWR from "swr";

 function Live() {
  const { data: config } = useSWR<FrigateConfig>("config");

  // recent events
-
-  const { data: allEvents } = useSWR<FrigateEvent[]>(
+  const { payload: eventUpdate } = useFrigateEvents();
+  const { data: allEvents, mutate: updateEvents } = useSWR<FrigateEvent[]>(
    ["events", { limit: 10 }],
-    { refreshInterval: 60000 }
+    { revalidateOnFocus: false }
  );

+  useEffect(() => {
+    if (!eventUpdate) {
+      return;
+    }
+
+    // if event is ended and was saved, update events list
+    if (
+      eventUpdate.type == "end" &&
+      (eventUpdate.after.has_clip || eventUpdate.after.has_snapshot)
+    ) {
+      updateEvents();
+      return;
+    }
+
+    // if event is updated and has become a saved event, update events list
+    if (
+      !(eventUpdate.before.has_clip || eventUpdate.before.has_snapshot) &&
+      (eventUpdate.after.has_clip || eventUpdate.after.has_snapshot)
+    ) {
+      updateEvents();
+    }
+  }, [eventUpdate]);
+
  const events = useMemo(() => {
    if (!allEvents) {
      return [];
@ -40,6 +64,19 @@ function Live() {
      .sort((aConf, bConf) => aConf.ui.order - bConf.ui.order);
  }, [config]);

+  const [windowVisible, setWindowVisible] = useState(true);
+  const visibilityListener = useCallback(() => {
+    setWindowVisible(document.visibilityState == "visible");
+  }, []);
+
+  useEffect(() => {
+    addEventListener("visibilitychange", visibilityListener);
+
+    return () => {
+      removeEventListener("visibilitychange", visibilityListener);
+    };
+  }, []);
+
  return (
    <>
      {events && events.length > 0 && (
@ -62,7 +99,7 @@ function Live() {
          if (aspectRatio > 2) {
            grow = "md:col-span-2 aspect-wide";
          } else if (aspectRatio < 1) {
-            grow = `md:row-span-2 aspect-[8/9] md:h-full`;
+            grow = `md:row-span-2 aspect-tall md:h-full`;
          } else {
            grow = "aspect-video";
          }
@ -70,6 +107,7 @@ function Live() {
            <LivePlayer
              key={camera.name}
              className={`mb-2 md:mb-0 rounded-2xl bg-black ${grow}`}
+              windowVisible={windowVisible}
              cameraConfig={camera}
              preferredLiveMode="mse"
            />
--- a/web/tailwind.config.js
+++ b/web/tailwind.config.js
@ -25,7 +25,7 @@ module.exports = {
      },
      aspectRatio: {
        wide: "32 / 9",
-        tall: "9 / 16",
+        tall: "8 / 9",
      },
      colors: {
        border: "hsl(var(--border))",