Add ability to submit frames from recordings (#11212)

* add ability to parse and upload image from recording to frigate+ * Show dialog with current frame to be uploaded * Implement uploading image in frontend * Cleanup * Update title
2024-12-19 19:06:16 +01:00 · 2024-05-03 08:00:19 -06:00 · 2024-05-03 08:00:19 -06:00 · e7950abec3
commit e7950abec3
parent b69c1828cb
8 changed files with 274 additions and 31 deletions
--- a/frigate/api/media.py
+++ b/frigate/api/media.py
@ -26,6 +26,7 @@ from frigate.const import (
 )
 from frigate.models import Event, Previews, Recordings, Regions, ReviewSegment
 from frigate.util.builtin import get_tz_modifiers
+from frigate.util.image import get_image_from_recording

 logger = logging.getLogger(__name__)

@ -205,30 +206,20 @@ def get_snapshot_from_recording(camera_name: str, frame_time: str):
    try:
        recording: Recordings = recording_query.get()
        time_in_segment = frame_time - recording.start_time
+        image_data = get_image_from_recording(recording.path, time_in_segment)

-        ffmpeg_cmd = [
-            "ffmpeg",
-            "-hide_banner",
-            "-loglevel",
-            "warning",
-            "-ss",
-            f"00:00:{time_in_segment}",
-            "-i",
-            recording.path,
-            "-frames:v",
-            "1",
-            "-c:v",
-            "png",
-            "-f",
-            "image2pipe",
-            "-",
-        ]
+        if not image_data:
+            return make_response(
+                jsonify(
+                    {
+                        "success": False,
+                        "message": f"Unable to parse frame at time {frame_time}",
+                    }
+                ),
+                404,
+            )

-        process = sp.run(
-            ffmpeg_cmd,
-            capture_output=True,
-        )
-        response = make_response(process.stdout)
+        response = make_response(image_data)
        response.headers["Content-Type"] = "image/png"
        return response
    except DoesNotExist:
@ -243,6 +234,71 @@ def get_snapshot_from_recording(camera_name: str, frame_time: str):
        )


+@MediaBp.route("/<camera_name>/plus/<frame_time>", methods=("POST",))
+def submit_recording_snapshot_to_plus(camera_name: str, frame_time: str):
+    if camera_name not in current_app.frigate_config.cameras:
+        return make_response(
+            jsonify({"success": False, "message": "Camera not found"}),
+            404,
+        )
+
+    frame_time = float(frame_time)
+    recording_query = (
+        Recordings.select(
+            Recordings.path,
+            Recordings.start_time,
+        )
+        .where(
+            (
+                (frame_time >= Recordings.start_time)
+                & (frame_time <= Recordings.end_time)
+            )
+        )
+        .where(Recordings.camera == camera_name)
+        .order_by(Recordings.start_time.desc())
+        .limit(1)
+    )
+
+    try:
+        recording: Recordings = recording_query.get()
+        time_in_segment = frame_time - recording.start_time
+        image_data = get_image_from_recording(recording.path, time_in_segment)
+
+        if not image_data:
+            return make_response(
+                jsonify(
+                    {
+                        "success": False,
+                        "message": f"Unable to parse frame at time {frame_time}",
+                    }
+                ),
+                404,
+            )
+
+        nd = cv2.imdecode(np.frombuffer(image_data, dtype=np.int8), cv2.IMREAD_COLOR)
+        current_app.plus_api.upload_image(nd, camera_name)
+
+        return make_response(
+            jsonify(
+                {
+                    "success": True,
+                    "message": "Successfully submitted image.",
+                }
+            ),
+            200,
+        )
+    except DoesNotExist:
+        return make_response(
+            jsonify(
+                {
+                    "success": False,
+                    "message": "Recording not found at {}".format(frame_time),
+                }
+            ),
+            404,
+        )
+
+
@MediaBp.route("/recordings/storage", methods=["GET"])
 def get_recordings_storage_usage():
    recording_stats = current_app.stats_emitter.get_latest_stats()["service"][
--- a/frigate/util/image.py
+++ b/frigate/util/image.py
@ -2,6 +2,7 @@

 import datetime
 import logging
+import subprocess as sp
 from abc import ABC, abstractmethod
 from multiprocessing import shared_memory
 from string import printable
@ -746,3 +747,37 @@ def add_mask(mask: str, mask_img: np.ndarray):
        ]
    )
    cv2.fillPoly(mask_img, pts=[contour], color=(0))
+
+
+def get_image_from_recording(
+    file_path: str, relative_frame_time: float
+) -> Optional[any]:
+    """retrieve a frame from given time in recording file."""
+
+    ffmpeg_cmd = [
+        "ffmpeg",
+        "-hide_banner",
+        "-loglevel",
+        "warning",
+        "-ss",
+        f"00:00:{relative_frame_time}",
+        "-i",
+        file_path,
+        "-frames:v",
+        "1",
+        "-c:v",
+        "png",
+        "-f",
+        "image2pipe",
+        "-",
+    ]
+
+    process = sp.run(
+        ffmpeg_cmd,
+        capture_output=True,
+    )
+
+    if process.returncode == 0:
+        return process.stdout
+    else:
+        return None
--- a/web/src/components/icons/FrigatePlusIcon.tsx
+++ b/web/src/components/icons/FrigatePlusIcon.tsx
@ -0,0 +1,21 @@
+import { LuPlus } from "react-icons/lu";
+import Logo from "../Logo";
+
+type FrigatePlusIconProps = {
+  className?: string;
+  onClick?: () => void;
+};
+export default function FrigatePlusIcon({
+  className,
+  onClick,
+}: FrigatePlusIconProps) {
+  return (
+    <div
+      className={`relative flex items-center ${className ?? ""}`}
+      onClick={onClick}
+    >
+      <Logo className="size-full" />
+      <LuPlus className="absolute size-2 translate-x-3 translate-y-3/4" />
+    </div>
+  );
+}
--- a/web/src/components/player/HlsVideoPlayer.tsx
+++ b/web/src/components/player/HlsVideoPlayer.tsx
@ -10,6 +10,10 @@ import { isAndroid, isDesktop, isMobile } from "react-device-detect";
 import { TransformComponent, TransformWrapper } from "react-zoom-pan-pinch";
 import VideoControls from "./VideoControls";
 import { VideoResolutionType } from "@/types/live";
+import useSWR from "swr";
+import { FrigateConfig } from "@/types/frigateConfig";
+import { AxiosResponse } from "axios";
+import { toast } from "sonner";

 // Android native hls does not seek correctly
 const USE_NATIVE_HLS = !isAndroid;
@ -29,6 +33,7 @@ type HlsVideoPlayerProps = {
  onTimeUpdate?: (time: number) => void;
  onPlaying?: () => void;
  setFullResolution?: React.Dispatch<React.SetStateAction<VideoResolutionType>>;
+  onUploadFrame?: (playTime: number) => Promise<AxiosResponse> | undefined;
 };
 export default function HlsVideoPlayer({
  videoRef,
@ -40,7 +45,10 @@ export default function HlsVideoPlayer({
  onTimeUpdate,
  onPlaying,
  setFullResolution,
+  onUploadFrame,
 }: HlsVideoPlayerProps) {
+  const { data: config } = useSWR<FrigateConfig>("config");
+
  // playback

  const hlsRef = useRef<Hls>();
@ -137,10 +145,15 @@ export default function HlsVideoPlayer({
        className="absolute bottom-5 left-1/2 -translate-x-1/2 z-50"
        video={videoRef.current}
        isPlaying={isPlaying}
-        show={visible && controls}
+        show={visible && (controls || controlsOpen)}
        muted={muted}
        volume={volume}
-        controlsOpen={controlsOpen}
+        features={{
+          volume: true,
+          seek: true,
+          playbackRate: true,
+          plusUpload: config?.plus?.enabled == true,
+        }}
        setControlsOpen={setControlsOpen}
        setMuted={setMuted}
        playbackRate={videoRef.current?.playbackRate ?? 1}
@ -168,6 +181,21 @@ export default function HlsVideoPlayer({
        onSetPlaybackRate={(rate) =>
          videoRef.current ? (videoRef.current.playbackRate = rate) : null
        }
+        onUploadFrame={async () => {
+          if (videoRef.current && onUploadFrame) {
+            const resp = await onUploadFrame(videoRef.current.currentTime);
+
+            if (resp && resp.status == 200) {
+              toast.success("Successfully submitted frame to Frigate Plus", {
+                position: "top-center",
+              });
+            } else {
+              toast.success("Failed to submit frame to Frigate Plus", {
+                position: "top-center",
+              });
+            }
+          }
+        }}
      />
      <TransformComponent
        wrapperStyle={{
--- a/web/src/components/player/VideoControls.tsx
+++ b/web/src/components/player/VideoControls.tsx
@ -1,4 +1,4 @@
-import { useCallback, useMemo } from "react";
+import { useCallback, useMemo, useState } from "react";
 import { isSafari } from "react-device-detect";
 import { LuPause, LuPlay } from "react-icons/lu";
 import {
@ -18,17 +18,30 @@ import {
 } from "react-icons/md";
 import useKeyboardListener from "@/hooks/use-keyboard-listener";
 import { VolumeSlider } from "../ui/slider";
+import FrigatePlusIcon from "../icons/FrigatePlusIcon";
+import {
+  AlertDialog,
+  AlertDialogAction,
+  AlertDialogCancel,
+  AlertDialogContent,
+  AlertDialogFooter,
+  AlertDialogHeader,
+  AlertDialogTitle,
+  AlertDialogTrigger,
+} from "../ui/alert-dialog";

 type VideoControls = {
  volume?: boolean;
  seek?: boolean;
  playbackRate?: boolean;
+  plusUpload?: boolean;
 };

 const CONTROLS_DEFAULT: VideoControls = {
  volume: true,
  seek: true,
  playbackRate: true,
+  plusUpload: false,
 };
 const PLAYBACK_RATE_DEFAULT = isSafari ? [0.5, 1, 2] : [0.5, 1, 2, 4, 8, 16];

@ -40,7 +53,6 @@ type VideoControlsProps = {
  show: boolean;
  muted?: boolean;
  volume?: number;
-  controlsOpen?: boolean;
  playbackRates?: number[];
  playbackRate: number;
  hotKeys?: boolean;
@ -49,6 +61,7 @@ type VideoControlsProps = {
  onPlayPause: (play: boolean) => void;
  onSeek: (diff: number) => void;
  onSetPlaybackRate: (rate: number) => void;
+  onUploadFrame?: () => void;
 };
 export default function VideoControls({
  className,
@ -58,7 +71,6 @@ export default function VideoControls({
  show,
  muted,
  volume,
-  controlsOpen,
  playbackRates = PLAYBACK_RATE_DEFAULT,
  playbackRate,
  hotKeys = true,
@ -67,6 +79,7 @@ export default function VideoControls({
  onPlayPause,
  onSeek,
  onSetPlaybackRate,
+  onUploadFrame,
 }: VideoControlsProps) {
  const onReplay = useCallback(
    (e: React.MouseEvent<SVGElement>) => {
@ -189,7 +202,6 @@ export default function VideoControls({
      )}
      {features.playbackRate && (
        <DropdownMenu
-          open={controlsOpen == true}
          onOpenChange={(open) => {
            if (setControlsOpen) {
              setControlsOpen(open);
@ -214,6 +226,84 @@ export default function VideoControls({
          </DropdownMenuContent>
        </DropdownMenu>
      )}
+      {features.plusUpload && onUploadFrame && (
+        <FrigatePlusUploadButton
+          video={video}
+          onClose={() => {
+            if (setControlsOpen) {
+              setControlsOpen(false);
+            }
+          }}
+          onOpen={() => {
+            onPlayPause(false);
+
+            if (setControlsOpen) {
+              setControlsOpen(true);
+            }
+          }}
+          onUploadFrame={onUploadFrame}
+        />
+      )}
    </div>
  );
 }
+
+type FrigatePlusUploadButtonProps = {
+  video?: HTMLVideoElement | null;
+  onOpen: () => void;
+  onClose: () => void;
+  onUploadFrame: () => void;
+};
+function FrigatePlusUploadButton({
+  video,
+  onOpen,
+  onClose,
+  onUploadFrame,
+}: FrigatePlusUploadButtonProps) {
+  const [videoImg, setVideoImg] = useState<string>();
+
+  return (
+    <AlertDialog
+      onOpenChange={(open) => {
+        if (!open) {
+          onClose();
+        }
+      }}
+    >
+      <AlertDialogTrigger asChild>
+        <FrigatePlusIcon
+          className="size-5 cursor-pointer"
+          onClick={() => {
+            onOpen();
+
+            if (video) {
+              const videoSize = [video.clientWidth, video.clientHeight];
+              const canvas = document.createElement("canvas");
+              canvas.width = videoSize[0];
+              canvas.height = videoSize[1];
+
+              const context = canvas?.getContext("2d");
+
+              if (context) {
+                context.drawImage(video, 0, 0, videoSize[0], videoSize[1]);
+                setVideoImg(canvas.toDataURL("image/webp"));
+              }
+            }
+          }}
+        />
+      </AlertDialogTrigger>
+      <AlertDialogContent className="md:max-w-[80%]">
+        <AlertDialogHeader>
+          <AlertDialogTitle>Submit this frame to Frigate+?</AlertDialogTitle>
+        </AlertDialogHeader>
+        <img className="w-full object-contain" src={videoImg} />
+        <AlertDialogFooter>
+          <AlertDialogAction className="bg-selected" onClick={onUploadFrame}>
+            Submit
+          </AlertDialogAction>
+          <AlertDialogCancel>Cancel</AlertDialogCancel>
+        </AlertDialogFooter>
+      </AlertDialogContent>
+    </AlertDialog>
+  );
+}
--- a/web/src/components/player/dynamic/DynamicVideoController.ts
+++ b/web/src/components/player/dynamic/DynamicVideoController.ts
@ -101,7 +101,7 @@ export class DynamicVideoController {
        this.playerController.pause();
      }
    } else {
-      console.log(`seek time is 0`);
+      // no op
    }
  }

--- a/web/src/components/player/dynamic/DynamicVideoPlayer.tsx
+++ b/web/src/components/player/dynamic/DynamicVideoPlayer.tsx
@ -10,6 +10,7 @@ import HlsVideoPlayer from "../HlsVideoPlayer";
 import { TimeRange } from "@/types/timeline";
 import ActivityIndicator from "@/components/indicators/activity-indicator";
 import { VideoResolutionType } from "@/types/live";
+import axios from "axios";

 /**
 * Dynamically switches between video playback and scrubbing preview player.
@ -127,6 +128,18 @@ export default function DynamicVideoPlayer({
    [controller, onTimestampUpdate, isScrubbing, isLoading],
  );

+  const onUploadFrameToPlus = useCallback(
+    (playTime: number) => {
+      if (!controller) {
+        return;
+      }
+
+      const time = controller.getProgress(playTime);
+      return axios.post(`/${camera}/plus/${time}`);
+    },
+    [camera, controller],
+  );
+
  // state of playback player

  const recordingParams = useMemo(() => {
@ -186,6 +199,7 @@ export default function DynamicVideoPlayer({
          setNoRecording(false);
        }}
        setFullResolution={setFullResolution}
+        onUploadFrame={onUploadFrameToPlus}
      />
      <PreviewPlayer
        className={`${isScrubbing || isLoading ? "visible" : "hidden"} ${className}`}
--- a/web/src/views/events/EventView.tsx
+++ b/web/src/views/events/EventView.tsx
@ -975,10 +975,9 @@ function MotionReview({
          playbackRate: true,
        }}
        isPlaying={playing}
-        show={!scrubbing}
+        show={!scrubbing || controlsOpen}
        playbackRates={[4, 8, 12, 16]}
        playbackRate={playbackRate}
-        controlsOpen={controlsOpen}
        setControlsOpen={setControlsOpen}
        onPlayPause={setPlaying}
        onSeek={(diff) => {