From e7950abec344e15a00511ac409ff651a70184e2f Mon Sep 17 00:00:00 2001
From: Nicolas Mowen <nickmowen213@gmail.com>
Date: Fri, 3 May 2024 08:00:19 -0600
Subject: [PATCH] Add ability to submit frames from recordings (#11212)

* add ability to parse and upload image from recording to frigate+

* Show dialog with current frame to be uploaded

* Implement uploading image in frontend

* Cleanup

* Update title
---
 frigate/api/media.py                          | 100 ++++++++++++++----
 frigate/util/image.py                         |  35 ++++++
 web/src/components/icons/FrigatePlusIcon.tsx  |  21 ++++
 web/src/components/player/HlsVideoPlayer.tsx  |  32 +++++-
 web/src/components/player/VideoControls.tsx   |  98 ++++++++++++++++-
 .../player/dynamic/DynamicVideoController.ts  |   2 +-
 .../player/dynamic/DynamicVideoPlayer.tsx     |  14 +++
 web/src/views/events/EventView.tsx            |   3 +-
 8 files changed, 274 insertions(+), 31 deletions(-)
 create mode 100644 web/src/components/icons/FrigatePlusIcon.tsx

diff --git a/frigate/api/media.py b/frigate/api/media.py
index be4ea08d8..d493b6fa9 100644
--- a/frigate/api/media.py
+++ b/frigate/api/media.py
@@ -26,6 +26,7 @@ from frigate.const import (
 )
 from frigate.models import Event, Previews, Recordings, Regions, ReviewSegment
 from frigate.util.builtin import get_tz_modifiers
+from frigate.util.image import get_image_from_recording
 
 logger = logging.getLogger(__name__)
 
@@ -205,30 +206,20 @@ def get_snapshot_from_recording(camera_name: str, frame_time: str):
     try:
         recording: Recordings = recording_query.get()
         time_in_segment = frame_time - recording.start_time
+        image_data = get_image_from_recording(recording.path, time_in_segment)
 
-        ffmpeg_cmd = [
-            "ffmpeg",
-            "-hide_banner",
-            "-loglevel",
-            "warning",
-            "-ss",
-            f"00:00:{time_in_segment}",
-            "-i",
-            recording.path,
-            "-frames:v",
-            "1",
-            "-c:v",
-            "png",
-            "-f",
-            "image2pipe",
-            "-",
-        ]
+        if not image_data:
+            return make_response(
+                jsonify(
+                    {
+                        "success": False,
+                        "message": f"Unable to parse frame at time {frame_time}",
+                    }
+                ),
+                404,
+            )
 
-        process = sp.run(
-            ffmpeg_cmd,
-            capture_output=True,
-        )
-        response = make_response(process.stdout)
+        response = make_response(image_data)
         response.headers["Content-Type"] = "image/png"
         return response
     except DoesNotExist:
@@ -243,6 +234,71 @@ def get_snapshot_from_recording(camera_name: str, frame_time: str):
         )
 
 
+@MediaBp.route("/<camera_name>/plus/<frame_time>", methods=("POST",))
+def submit_recording_snapshot_to_plus(camera_name: str, frame_time: str):
+    if camera_name not in current_app.frigate_config.cameras:
+        return make_response(
+            jsonify({"success": False, "message": "Camera not found"}),
+            404,
+        )
+
+    frame_time = float(frame_time)
+    recording_query = (
+        Recordings.select(
+            Recordings.path,
+            Recordings.start_time,
+        )
+        .where(
+            (
+                (frame_time >= Recordings.start_time)
+                & (frame_time <= Recordings.end_time)
+            )
+        )
+        .where(Recordings.camera == camera_name)
+        .order_by(Recordings.start_time.desc())
+        .limit(1)
+    )
+
+    try:
+        recording: Recordings = recording_query.get()
+        time_in_segment = frame_time - recording.start_time
+        image_data = get_image_from_recording(recording.path, time_in_segment)
+
+        if not image_data:
+            return make_response(
+                jsonify(
+                    {
+                        "success": False,
+                        "message": f"Unable to parse frame at time {frame_time}",
+                    }
+                ),
+                404,
+            )
+
+        nd = cv2.imdecode(np.frombuffer(image_data, dtype=np.int8), cv2.IMREAD_COLOR)
+        current_app.plus_api.upload_image(nd, camera_name)
+
+        return make_response(
+            jsonify(
+                {
+                    "success": True,
+                    "message": "Successfully submitted image.",
+                }
+            ),
+            200,
+        )
+    except DoesNotExist:
+        return make_response(
+            jsonify(
+                {
+                    "success": False,
+                    "message": "Recording not found at {}".format(frame_time),
+                }
+            ),
+            404,
+        )
+
+
 @MediaBp.route("/recordings/storage", methods=["GET"])
 def get_recordings_storage_usage():
     recording_stats = current_app.stats_emitter.get_latest_stats()["service"][
diff --git a/frigate/util/image.py b/frigate/util/image.py
index 67f8b5c22..3962d9600 100644
--- a/frigate/util/image.py
+++ b/frigate/util/image.py
@@ -2,6 +2,7 @@
 
 import datetime
 import logging
+import subprocess as sp
 from abc import ABC, abstractmethod
 from multiprocessing import shared_memory
 from string import printable
@@ -746,3 +747,37 @@ def add_mask(mask: str, mask_img: np.ndarray):
         ]
     )
     cv2.fillPoly(mask_img, pts=[contour], color=(0))
+
+
+def get_image_from_recording(
+    file_path: str, relative_frame_time: float
+) -> Optional[any]:
+    """retrieve a frame from given time in recording file."""
+
+    ffmpeg_cmd = [
+        "ffmpeg",
+        "-hide_banner",
+        "-loglevel",
+        "warning",
+        "-ss",
+        f"00:00:{relative_frame_time}",
+        "-i",
+        file_path,
+        "-frames:v",
+        "1",
+        "-c:v",
+        "png",
+        "-f",
+        "image2pipe",
+        "-",
+    ]
+
+    process = sp.run(
+        ffmpeg_cmd,
+        capture_output=True,
+    )
+
+    if process.returncode == 0:
+        return process.stdout
+    else:
+        return None
diff --git a/web/src/components/icons/FrigatePlusIcon.tsx b/web/src/components/icons/FrigatePlusIcon.tsx
new file mode 100644
index 000000000..1a9ff2e05
--- /dev/null
+++ b/web/src/components/icons/FrigatePlusIcon.tsx
@@ -0,0 +1,21 @@
+import { LuPlus } from "react-icons/lu";
+import Logo from "../Logo";
+
+type FrigatePlusIconProps = {
+  className?: string;
+  onClick?: () => void;
+};
+export default function FrigatePlusIcon({
+  className,
+  onClick,
+}: FrigatePlusIconProps) {
+  return (
+    <div
+      className={`relative flex items-center ${className ?? ""}`}
+      onClick={onClick}
+    >
+      <Logo className="size-full" />
+      <LuPlus className="absolute size-2 translate-x-3 translate-y-3/4" />
+    </div>
+  );
+}
diff --git a/web/src/components/player/HlsVideoPlayer.tsx b/web/src/components/player/HlsVideoPlayer.tsx
index 5f96a017d..879506936 100644
--- a/web/src/components/player/HlsVideoPlayer.tsx
+++ b/web/src/components/player/HlsVideoPlayer.tsx
@@ -10,6 +10,10 @@ import { isAndroid, isDesktop, isMobile } from "react-device-detect";
 import { TransformComponent, TransformWrapper } from "react-zoom-pan-pinch";
 import VideoControls from "./VideoControls";
 import { VideoResolutionType } from "@/types/live";
+import useSWR from "swr";
+import { FrigateConfig } from "@/types/frigateConfig";
+import { AxiosResponse } from "axios";
+import { toast } from "sonner";
 
 // Android native hls does not seek correctly
 const USE_NATIVE_HLS = !isAndroid;
@@ -29,6 +33,7 @@ type HlsVideoPlayerProps = {
   onTimeUpdate?: (time: number) => void;
   onPlaying?: () => void;
   setFullResolution?: React.Dispatch<React.SetStateAction<VideoResolutionType>>;
+  onUploadFrame?: (playTime: number) => Promise<AxiosResponse> | undefined;
 };
 export default function HlsVideoPlayer({
   videoRef,
@@ -40,7 +45,10 @@ export default function HlsVideoPlayer({
   onTimeUpdate,
   onPlaying,
   setFullResolution,
+  onUploadFrame,
 }: HlsVideoPlayerProps) {
+  const { data: config } = useSWR<FrigateConfig>("config");
+
   // playback
 
   const hlsRef = useRef<Hls>();
@@ -137,10 +145,15 @@ export default function HlsVideoPlayer({
         className="absolute bottom-5 left-1/2 -translate-x-1/2 z-50"
         video={videoRef.current}
         isPlaying={isPlaying}
-        show={visible && controls}
+        show={visible && (controls || controlsOpen)}
         muted={muted}
         volume={volume}
-        controlsOpen={controlsOpen}
+        features={{
+          volume: true,
+          seek: true,
+          playbackRate: true,
+          plusUpload: config?.plus?.enabled == true,
+        }}
         setControlsOpen={setControlsOpen}
         setMuted={setMuted}
         playbackRate={videoRef.current?.playbackRate ?? 1}
@@ -168,6 +181,21 @@ export default function HlsVideoPlayer({
         onSetPlaybackRate={(rate) =>
           videoRef.current ? (videoRef.current.playbackRate = rate) : null
         }
+        onUploadFrame={async () => {
+          if (videoRef.current && onUploadFrame) {
+            const resp = await onUploadFrame(videoRef.current.currentTime);
+
+            if (resp && resp.status == 200) {
+              toast.success("Successfully submitted frame to Frigate Plus", {
+                position: "top-center",
+              });
+            } else {
+              toast.success("Failed to submit frame to Frigate Plus", {
+                position: "top-center",
+              });
+            }
+          }
+        }}
       />
       <TransformComponent
         wrapperStyle={{
diff --git a/web/src/components/player/VideoControls.tsx b/web/src/components/player/VideoControls.tsx
index 872289aa0..135f664ed 100644
--- a/web/src/components/player/VideoControls.tsx
+++ b/web/src/components/player/VideoControls.tsx
@@ -1,4 +1,4 @@
-import { useCallback, useMemo } from "react";
+import { useCallback, useMemo, useState } from "react";
 import { isSafari } from "react-device-detect";
 import { LuPause, LuPlay } from "react-icons/lu";
 import {
@@ -18,17 +18,30 @@ import {
 } from "react-icons/md";
 import useKeyboardListener from "@/hooks/use-keyboard-listener";
 import { VolumeSlider } from "../ui/slider";
+import FrigatePlusIcon from "../icons/FrigatePlusIcon";
+import {
+  AlertDialog,
+  AlertDialogAction,
+  AlertDialogCancel,
+  AlertDialogContent,
+  AlertDialogFooter,
+  AlertDialogHeader,
+  AlertDialogTitle,
+  AlertDialogTrigger,
+} from "../ui/alert-dialog";
 
 type VideoControls = {
   volume?: boolean;
   seek?: boolean;
   playbackRate?: boolean;
+  plusUpload?: boolean;
 };
 
 const CONTROLS_DEFAULT: VideoControls = {
   volume: true,
   seek: true,
   playbackRate: true,
+  plusUpload: false,
 };
 const PLAYBACK_RATE_DEFAULT = isSafari ? [0.5, 1, 2] : [0.5, 1, 2, 4, 8, 16];
 
@@ -40,7 +53,6 @@ type VideoControlsProps = {
   show: boolean;
   muted?: boolean;
   volume?: number;
-  controlsOpen?: boolean;
   playbackRates?: number[];
   playbackRate: number;
   hotKeys?: boolean;
@@ -49,6 +61,7 @@ type VideoControlsProps = {
   onPlayPause: (play: boolean) => void;
   onSeek: (diff: number) => void;
   onSetPlaybackRate: (rate: number) => void;
+  onUploadFrame?: () => void;
 };
 export default function VideoControls({
   className,
@@ -58,7 +71,6 @@ export default function VideoControls({
   show,
   muted,
   volume,
-  controlsOpen,
   playbackRates = PLAYBACK_RATE_DEFAULT,
   playbackRate,
   hotKeys = true,
@@ -67,6 +79,7 @@ export default function VideoControls({
   onPlayPause,
   onSeek,
   onSetPlaybackRate,
+  onUploadFrame,
 }: VideoControlsProps) {
   const onReplay = useCallback(
     (e: React.MouseEvent<SVGElement>) => {
@@ -189,7 +202,6 @@ export default function VideoControls({
       )}
       {features.playbackRate && (
         <DropdownMenu
-          open={controlsOpen == true}
           onOpenChange={(open) => {
             if (setControlsOpen) {
               setControlsOpen(open);
@@ -214,6 +226,84 @@ export default function VideoControls({
           </DropdownMenuContent>
         </DropdownMenu>
       )}
+      {features.plusUpload && onUploadFrame && (
+        <FrigatePlusUploadButton
+          video={video}
+          onClose={() => {
+            if (setControlsOpen) {
+              setControlsOpen(false);
+            }
+          }}
+          onOpen={() => {
+            onPlayPause(false);
+
+            if (setControlsOpen) {
+              setControlsOpen(true);
+            }
+          }}
+          onUploadFrame={onUploadFrame}
+        />
+      )}
     </div>
   );
 }
+
+type FrigatePlusUploadButtonProps = {
+  video?: HTMLVideoElement | null;
+  onOpen: () => void;
+  onClose: () => void;
+  onUploadFrame: () => void;
+};
+function FrigatePlusUploadButton({
+  video,
+  onOpen,
+  onClose,
+  onUploadFrame,
+}: FrigatePlusUploadButtonProps) {
+  const [videoImg, setVideoImg] = useState<string>();
+
+  return (
+    <AlertDialog
+      onOpenChange={(open) => {
+        if (!open) {
+          onClose();
+        }
+      }}
+    >
+      <AlertDialogTrigger asChild>
+        <FrigatePlusIcon
+          className="size-5 cursor-pointer"
+          onClick={() => {
+            onOpen();
+
+            if (video) {
+              const videoSize = [video.clientWidth, video.clientHeight];
+              const canvas = document.createElement("canvas");
+              canvas.width = videoSize[0];
+              canvas.height = videoSize[1];
+
+              const context = canvas?.getContext("2d");
+
+              if (context) {
+                context.drawImage(video, 0, 0, videoSize[0], videoSize[1]);
+                setVideoImg(canvas.toDataURL("image/webp"));
+              }
+            }
+          }}
+        />
+      </AlertDialogTrigger>
+      <AlertDialogContent className="md:max-w-[80%]">
+        <AlertDialogHeader>
+          <AlertDialogTitle>Submit this frame to Frigate+?</AlertDialogTitle>
+        </AlertDialogHeader>
+        <img className="w-full object-contain" src={videoImg} />
+        <AlertDialogFooter>
+          <AlertDialogAction className="bg-selected" onClick={onUploadFrame}>
+            Submit
+          </AlertDialogAction>
+          <AlertDialogCancel>Cancel</AlertDialogCancel>
+        </AlertDialogFooter>
+      </AlertDialogContent>
+    </AlertDialog>
+  );
+}
diff --git a/web/src/components/player/dynamic/DynamicVideoController.ts b/web/src/components/player/dynamic/DynamicVideoController.ts
index e8772c49e..9ccc06c23 100644
--- a/web/src/components/player/dynamic/DynamicVideoController.ts
+++ b/web/src/components/player/dynamic/DynamicVideoController.ts
@@ -101,7 +101,7 @@ export class DynamicVideoController {
         this.playerController.pause();
       }
     } else {
-      console.log(`seek time is 0`);
+      // no op
     }
   }
 
diff --git a/web/src/components/player/dynamic/DynamicVideoPlayer.tsx b/web/src/components/player/dynamic/DynamicVideoPlayer.tsx
index 3f144e04b..0c095de97 100644
--- a/web/src/components/player/dynamic/DynamicVideoPlayer.tsx
+++ b/web/src/components/player/dynamic/DynamicVideoPlayer.tsx
@@ -10,6 +10,7 @@ import HlsVideoPlayer from "../HlsVideoPlayer";
 import { TimeRange } from "@/types/timeline";
 import ActivityIndicator from "@/components/indicators/activity-indicator";
 import { VideoResolutionType } from "@/types/live";
+import axios from "axios";
 
 /**
  * Dynamically switches between video playback and scrubbing preview player.
@@ -127,6 +128,18 @@ export default function DynamicVideoPlayer({
     [controller, onTimestampUpdate, isScrubbing, isLoading],
   );
 
+  const onUploadFrameToPlus = useCallback(
+    (playTime: number) => {
+      if (!controller) {
+        return;
+      }
+
+      const time = controller.getProgress(playTime);
+      return axios.post(`/${camera}/plus/${time}`);
+    },
+    [camera, controller],
+  );
+
   // state of playback player
 
   const recordingParams = useMemo(() => {
@@ -186,6 +199,7 @@ export default function DynamicVideoPlayer({
           setNoRecording(false);
         }}
         setFullResolution={setFullResolution}
+        onUploadFrame={onUploadFrameToPlus}
       />
       <PreviewPlayer
         className={`${isScrubbing || isLoading ? "visible" : "hidden"} ${className}`}
diff --git a/web/src/views/events/EventView.tsx b/web/src/views/events/EventView.tsx
index 18f7731dd..63f05766e 100644
--- a/web/src/views/events/EventView.tsx
+++ b/web/src/views/events/EventView.tsx
@@ -975,10 +975,9 @@ function MotionReview({
           playbackRate: true,
         }}
         isPlaying={playing}
-        show={!scrubbing}
+        show={!scrubbing || controlsOpen}
         playbackRates={[4, 8, 12, 16]}
         playbackRate={playbackRate}
-        controlsOpen={controlsOpen}
         setControlsOpen={setControlsOpen}
         onPlayPause={setPlaying}
         onSeek={(diff) => {