diff --git a/frigate/api/media.py b/frigate/api/media.py index be4ea08d8..d493b6fa9 100644 --- a/frigate/api/media.py +++ b/frigate/api/media.py @@ -26,6 +26,7 @@ from frigate.const import ( ) from frigate.models import Event, Previews, Recordings, Regions, ReviewSegment from frigate.util.builtin import get_tz_modifiers +from frigate.util.image import get_image_from_recording logger = logging.getLogger(__name__) @@ -205,30 +206,20 @@ def get_snapshot_from_recording(camera_name: str, frame_time: str): try: recording: Recordings = recording_query.get() time_in_segment = frame_time - recording.start_time + image_data = get_image_from_recording(recording.path, time_in_segment) - ffmpeg_cmd = [ - "ffmpeg", - "-hide_banner", - "-loglevel", - "warning", - "-ss", - f"00:00:{time_in_segment}", - "-i", - recording.path, - "-frames:v", - "1", - "-c:v", - "png", - "-f", - "image2pipe", - "-", - ] + if not image_data: + return make_response( + jsonify( + { + "success": False, + "message": f"Unable to parse frame at time {frame_time}", + } + ), + 404, + ) - process = sp.run( - ffmpeg_cmd, - capture_output=True, - ) - response = make_response(process.stdout) + response = make_response(image_data) response.headers["Content-Type"] = "image/png" return response except DoesNotExist: @@ -243,6 +234,71 @@ def get_snapshot_from_recording(camera_name: str, frame_time: str): ) +@MediaBp.route("//plus/", methods=("POST",)) +def submit_recording_snapshot_to_plus(camera_name: str, frame_time: str): + if camera_name not in current_app.frigate_config.cameras: + return make_response( + jsonify({"success": False, "message": "Camera not found"}), + 404, + ) + + frame_time = float(frame_time) + recording_query = ( + Recordings.select( + Recordings.path, + Recordings.start_time, + ) + .where( + ( + (frame_time >= Recordings.start_time) + & (frame_time <= Recordings.end_time) + ) + ) + .where(Recordings.camera == camera_name) + .order_by(Recordings.start_time.desc()) + .limit(1) + ) + + try: + recording: Recordings = recording_query.get() + time_in_segment = frame_time - recording.start_time + image_data = get_image_from_recording(recording.path, time_in_segment) + + if not image_data: + return make_response( + jsonify( + { + "success": False, + "message": f"Unable to parse frame at time {frame_time}", + } + ), + 404, + ) + + nd = cv2.imdecode(np.frombuffer(image_data, dtype=np.int8), cv2.IMREAD_COLOR) + current_app.plus_api.upload_image(nd, camera_name) + + return make_response( + jsonify( + { + "success": True, + "message": "Successfully submitted image.", + } + ), + 200, + ) + except DoesNotExist: + return make_response( + jsonify( + { + "success": False, + "message": "Recording not found at {}".format(frame_time), + } + ), + 404, + ) + + @MediaBp.route("/recordings/storage", methods=["GET"]) def get_recordings_storage_usage(): recording_stats = current_app.stats_emitter.get_latest_stats()["service"][ diff --git a/frigate/util/image.py b/frigate/util/image.py index 67f8b5c22..3962d9600 100644 --- a/frigate/util/image.py +++ b/frigate/util/image.py @@ -2,6 +2,7 @@ import datetime import logging +import subprocess as sp from abc import ABC, abstractmethod from multiprocessing import shared_memory from string import printable @@ -746,3 +747,37 @@ def add_mask(mask: str, mask_img: np.ndarray): ] ) cv2.fillPoly(mask_img, pts=[contour], color=(0)) + + +def get_image_from_recording( + file_path: str, relative_frame_time: float +) -> Optional[any]: + """retrieve a frame from given time in recording file.""" + + ffmpeg_cmd = [ + "ffmpeg", + "-hide_banner", + "-loglevel", + "warning", + "-ss", + f"00:00:{relative_frame_time}", + "-i", + file_path, + "-frames:v", + "1", + "-c:v", + "png", + "-f", + "image2pipe", + "-", + ] + + process = sp.run( + ffmpeg_cmd, + capture_output=True, + ) + + if process.returncode == 0: + return process.stdout + else: + return None diff --git a/web/src/components/icons/FrigatePlusIcon.tsx b/web/src/components/icons/FrigatePlusIcon.tsx new file mode 100644 index 000000000..1a9ff2e05 --- /dev/null +++ b/web/src/components/icons/FrigatePlusIcon.tsx @@ -0,0 +1,21 @@ +import { LuPlus } from "react-icons/lu"; +import Logo from "../Logo"; + +type FrigatePlusIconProps = { + className?: string; + onClick?: () => void; +}; +export default function FrigatePlusIcon({ + className, + onClick, +}: FrigatePlusIconProps) { + return ( +
+ + +
+ ); +} diff --git a/web/src/components/player/HlsVideoPlayer.tsx b/web/src/components/player/HlsVideoPlayer.tsx index 5f96a017d..879506936 100644 --- a/web/src/components/player/HlsVideoPlayer.tsx +++ b/web/src/components/player/HlsVideoPlayer.tsx @@ -10,6 +10,10 @@ import { isAndroid, isDesktop, isMobile } from "react-device-detect"; import { TransformComponent, TransformWrapper } from "react-zoom-pan-pinch"; import VideoControls from "./VideoControls"; import { VideoResolutionType } from "@/types/live"; +import useSWR from "swr"; +import { FrigateConfig } from "@/types/frigateConfig"; +import { AxiosResponse } from "axios"; +import { toast } from "sonner"; // Android native hls does not seek correctly const USE_NATIVE_HLS = !isAndroid; @@ -29,6 +33,7 @@ type HlsVideoPlayerProps = { onTimeUpdate?: (time: number) => void; onPlaying?: () => void; setFullResolution?: React.Dispatch>; + onUploadFrame?: (playTime: number) => Promise | undefined; }; export default function HlsVideoPlayer({ videoRef, @@ -40,7 +45,10 @@ export default function HlsVideoPlayer({ onTimeUpdate, onPlaying, setFullResolution, + onUploadFrame, }: HlsVideoPlayerProps) { + const { data: config } = useSWR("config"); + // playback const hlsRef = useRef(); @@ -137,10 +145,15 @@ export default function HlsVideoPlayer({ className="absolute bottom-5 left-1/2 -translate-x-1/2 z-50" video={videoRef.current} isPlaying={isPlaying} - show={visible && controls} + show={visible && (controls || controlsOpen)} muted={muted} volume={volume} - controlsOpen={controlsOpen} + features={{ + volume: true, + seek: true, + playbackRate: true, + plusUpload: config?.plus?.enabled == true, + }} setControlsOpen={setControlsOpen} setMuted={setMuted} playbackRate={videoRef.current?.playbackRate ?? 1} @@ -168,6 +181,21 @@ export default function HlsVideoPlayer({ onSetPlaybackRate={(rate) => videoRef.current ? (videoRef.current.playbackRate = rate) : null } + onUploadFrame={async () => { + if (videoRef.current && onUploadFrame) { + const resp = await onUploadFrame(videoRef.current.currentTime); + + if (resp && resp.status == 200) { + toast.success("Successfully submitted frame to Frigate Plus", { + position: "top-center", + }); + } else { + toast.success("Failed to submit frame to Frigate Plus", { + position: "top-center", + }); + } + } + }} /> void; onSeek: (diff: number) => void; onSetPlaybackRate: (rate: number) => void; + onUploadFrame?: () => void; }; export default function VideoControls({ className, @@ -58,7 +71,6 @@ export default function VideoControls({ show, muted, volume, - controlsOpen, playbackRates = PLAYBACK_RATE_DEFAULT, playbackRate, hotKeys = true, @@ -67,6 +79,7 @@ export default function VideoControls({ onPlayPause, onSeek, onSetPlaybackRate, + onUploadFrame, }: VideoControlsProps) { const onReplay = useCallback( (e: React.MouseEvent) => { @@ -189,7 +202,6 @@ export default function VideoControls({ )} {features.playbackRate && ( { if (setControlsOpen) { setControlsOpen(open); @@ -214,6 +226,84 @@ export default function VideoControls({ )} + {features.plusUpload && onUploadFrame && ( + { + if (setControlsOpen) { + setControlsOpen(false); + } + }} + onOpen={() => { + onPlayPause(false); + + if (setControlsOpen) { + setControlsOpen(true); + } + }} + onUploadFrame={onUploadFrame} + /> + )} ); } + +type FrigatePlusUploadButtonProps = { + video?: HTMLVideoElement | null; + onOpen: () => void; + onClose: () => void; + onUploadFrame: () => void; +}; +function FrigatePlusUploadButton({ + video, + onOpen, + onClose, + onUploadFrame, +}: FrigatePlusUploadButtonProps) { + const [videoImg, setVideoImg] = useState(); + + return ( + { + if (!open) { + onClose(); + } + }} + > + + { + onOpen(); + + if (video) { + const videoSize = [video.clientWidth, video.clientHeight]; + const canvas = document.createElement("canvas"); + canvas.width = videoSize[0]; + canvas.height = videoSize[1]; + + const context = canvas?.getContext("2d"); + + if (context) { + context.drawImage(video, 0, 0, videoSize[0], videoSize[1]); + setVideoImg(canvas.toDataURL("image/webp")); + } + } + }} + /> + + + + Submit this frame to Frigate+? + + + + + Submit + + Cancel + + + + ); +} diff --git a/web/src/components/player/dynamic/DynamicVideoController.ts b/web/src/components/player/dynamic/DynamicVideoController.ts index e8772c49e..9ccc06c23 100644 --- a/web/src/components/player/dynamic/DynamicVideoController.ts +++ b/web/src/components/player/dynamic/DynamicVideoController.ts @@ -101,7 +101,7 @@ export class DynamicVideoController { this.playerController.pause(); } } else { - console.log(`seek time is 0`); + // no op } } diff --git a/web/src/components/player/dynamic/DynamicVideoPlayer.tsx b/web/src/components/player/dynamic/DynamicVideoPlayer.tsx index 3f144e04b..0c095de97 100644 --- a/web/src/components/player/dynamic/DynamicVideoPlayer.tsx +++ b/web/src/components/player/dynamic/DynamicVideoPlayer.tsx @@ -10,6 +10,7 @@ import HlsVideoPlayer from "../HlsVideoPlayer"; import { TimeRange } from "@/types/timeline"; import ActivityIndicator from "@/components/indicators/activity-indicator"; import { VideoResolutionType } from "@/types/live"; +import axios from "axios"; /** * Dynamically switches between video playback and scrubbing preview player. @@ -127,6 +128,18 @@ export default function DynamicVideoPlayer({ [controller, onTimestampUpdate, isScrubbing, isLoading], ); + const onUploadFrameToPlus = useCallback( + (playTime: number) => { + if (!controller) { + return; + } + + const time = controller.getProgress(playTime); + return axios.post(`/${camera}/plus/${time}`); + }, + [camera, controller], + ); + // state of playback player const recordingParams = useMemo(() => { @@ -186,6 +199,7 @@ export default function DynamicVideoPlayer({ setNoRecording(false); }} setFullResolution={setFullResolution} + onUploadFrame={onUploadFrameToPlus} /> {