From a1905f560475cd6d5d0c26beadf1f58c3bb2a399 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Thu, 18 Apr 2024 21:34:57 -0600 Subject: [PATCH] Remove gifs and use existing views (#11027) * Use existing components for preview filmstrip instead of gif * Allow setting format --- frigate/api/media.py | 164 ++++++++++++++++-- web/src/components/card/AnimatedEventCard.tsx | 65 ++++--- .../player/PreviewThumbnailPlayer.tsx | 70 +++++--- 3 files changed, 238 insertions(+), 61 deletions(-) diff --git a/frigate/api/media.py b/frigate/api/media.py index 345636e25..5387b2866 100644 --- a/frigate/api/media.py +++ b/frigate/api/media.py @@ -14,14 +14,7 @@ from urllib.parse import unquote import cv2 import numpy as np import pytz -from flask import ( - Blueprint, - Response, - current_app, - jsonify, - make_response, - request, -) +from flask import Blueprint, Response, current_app, jsonify, make_response, request from peewee import DoesNotExist, fn from tzlocal import get_localzone_name from werkzeug.utils import secure_filename @@ -36,9 +29,7 @@ from frigate.const import ( ) from frigate.models import Event, Previews, Recordings, Regions, ReviewSegment from frigate.record.export import PlaybackFactorEnum, RecordingExporter -from frigate.util.builtin import ( - get_tz_modifiers, -) +from frigate.util.builtin import get_tz_modifiers logger = logging.getLogger(__name__) @@ -1322,8 +1313,151 @@ def preview_gif(camera_name: str, start_ts, end_ts, max_cache_age=2592000): return response -@MediaBp.route("/review//preview.gif") +@MediaBp.route("//start//end//preview.mp4") +@MediaBp.route("//start//end//preview.mp4") +def preview_mp4(camera_name: str, start_ts, end_ts, max_cache_age=2592000): + if datetime.fromtimestamp(start_ts) < datetime.now().replace(minute=0, second=0): + # has preview mp4 + preview: Previews = ( + Previews.select( + Previews.camera, + Previews.path, + Previews.duration, + Previews.start_time, + Previews.end_time, + ) + .where( + Previews.start_time.between(start_ts, end_ts) + | Previews.end_time.between(start_ts, end_ts) + | ((start_ts > Previews.start_time) & (end_ts < Previews.end_time)) + ) + .where(Previews.camera == camera_name) + .limit(1) + .get() + ) + + if not preview: + return make_response( + jsonify({"success": False, "message": "Preview not found"}), 404 + ) + + diff = start_ts - preview.start_time + minutes = int(diff / 60) + seconds = int(diff % 60) + ffmpeg_cmd = [ + "ffmpeg", + "-hide_banner", + "-loglevel", + "warning", + "-ss", + f"00:{minutes}:{seconds}", + "-t", + f"{end_ts - start_ts}", + "-i", + preview.path, + "-r", + "8", + "-vf", + "setpts=0.12*PTS", + "-loop", + "0", + "-c:v", + "copy", + "-f", + "mp4", + "-", + ] + + process = sp.run( + ffmpeg_cmd, + capture_output=True, + ) + + if process.returncode != 0: + logger.error(process.stderr) + return make_response( + jsonify({"success": False, "message": "Unable to create preview gif"}), + 500, + ) + + gif_bytes = process.stdout + else: + # need to generate from existing images + preview_dir = os.path.join(CACHE_DIR, "preview_frames") + file_start = f"preview_{camera_name}" + start_file = f"{file_start}-{start_ts}.{PREVIEW_FRAME_TYPE}" + end_file = f"{file_start}-{end_ts}.{PREVIEW_FRAME_TYPE}" + selected_previews = [] + + for file in sorted(os.listdir(preview_dir)): + if not file.startswith(file_start): + continue + + if file < start_file: + continue + + if file > end_file: + break + + selected_previews.append(f"file '{os.path.join(preview_dir, file)}'") + selected_previews.append("duration 0.12") + + if not selected_previews: + return make_response( + jsonify({"success": False, "message": "Preview not found"}), 404 + ) + + last_file = selected_previews[-2] + selected_previews.append(last_file) + + ffmpeg_cmd = [ + "ffmpeg", + "-hide_banner", + "-loglevel", + "warning", + "-f", + "concat", + "-y", + "-protocol_whitelist", + "pipe,file", + "-safe", + "0", + "-i", + "/dev/stdin", + "-loop", + "0", + "-c:v", + "libx264", + "-f", + "gif", + "-", + ] + + process = sp.run( + ffmpeg_cmd, + input=str.encode("\n".join(selected_previews)), + capture_output=True, + ) + + if process.returncode != 0: + logger.error(process.stderr) + return make_response( + jsonify({"success": False, "message": "Unable to create preview gif"}), + 500, + ) + + gif_bytes = process.stdout + + response = make_response(gif_bytes) + response.headers["Content-Type"] = "image/gif" + response.headers["Cache-Control"] = f"private, max-age={max_cache_age}" + return response + + +@MediaBp.route("/review//preview") def review_preview(id: str): + format = request.args.get("format", default="gif") + try: review: ReviewSegment = ReviewSegment.get(ReviewSegment.id == id) except DoesNotExist: @@ -1336,7 +1470,11 @@ def review_preview(id: str): end_ts = ( review.end_time + padding if review.end_time else datetime.now().timestamp() ) - return preview_gif(review.camera, start_ts, end_ts) + + if format == "gif": + return preview_gif(review.camera, start_ts, end_ts) + else: + return preview_mp4(review.camera, start_ts, end_ts) @MediaBp.route("/preview//thumbnail.jpg") diff --git a/web/src/components/card/AnimatedEventCard.tsx b/web/src/components/card/AnimatedEventCard.tsx index 93ecbe919..08a2670bd 100644 --- a/web/src/components/card/AnimatedEventCard.tsx +++ b/web/src/components/card/AnimatedEventCard.tsx @@ -1,14 +1,17 @@ -import { baseUrl } from "@/api/baseUrl"; import TimeAgo from "../dynamic/TimeAgo"; import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip"; -import { useCallback, useMemo, useState } from "react"; +import { useCallback, useMemo } from "react"; import useSWR from "swr"; import { FrigateConfig } from "@/types/frigateConfig"; import { ReviewSegment } from "@/types/review"; import { useNavigate } from "react-router-dom"; -import { Skeleton } from "../ui/skeleton"; import { RecordingStartingPoint } from "@/types/record"; import axios from "axios"; +import { Preview } from "@/types/preview"; +import { + InProgressPreview, + VideoPreview, +} from "../player/PreviewThumbnailPlayer"; type AnimatedEventCardProps = { event: ReviewSegment; @@ -16,6 +19,12 @@ type AnimatedEventCardProps = { export function AnimatedEventCard({ event }: AnimatedEventCardProps) { const { data: config } = useSWR("config"); + // preview + + const { data: previews } = useSWR( + `/preview/${event.camera}/start/${Math.round(event.start_time)}/end/${Math.round(event.end_time || event.start_time + 20)}`, + ); + // interaction const navigate = useNavigate(); @@ -35,16 +44,6 @@ export function AnimatedEventCard({ event }: AnimatedEventCardProps) { // image behavior - const [loaded, setLoaded] = useState(false); - const [error, setError] = useState(0); - const imageUrl = useMemo(() => { - if (error > 0) { - return `${baseUrl}api/review/${event.id}/preview.gif?key=${error}`; - } - - return `${baseUrl}api/review/${event.id}/preview.gif`; - }, [error, event]); - const aspectRatio = useMemo(() => { if (!config) { return 1; @@ -63,18 +62,36 @@ export function AnimatedEventCard({ event }: AnimatedEventCardProps) { aspectRatio: aspectRatio, }} > - setLoaded(true)} - onError={() => { - if (error < 2) { - setError(error + 1); - } - }} - /> - {!loaded && } + > + {previews ? ( + {}} + setIgnoreClick={() => {}} + isPlayingBack={() => {}} + /> + ) : ( + {}} + setIgnoreClick={() => {}} + isPlayingBack={() => {}} + /> + )} +
diff --git a/web/src/components/player/PreviewThumbnailPlayer.tsx b/web/src/components/player/PreviewThumbnailPlayer.tsx index 5d7cc6a15..a7968075d 100644 --- a/web/src/components/player/PreviewThumbnailPlayer.tsx +++ b/web/src/components/player/PreviewThumbnailPlayer.tsx @@ -342,15 +342,19 @@ type VideoPreviewProps = { relevantPreview: Preview; startTime: number; endTime?: number; + showProgress?: boolean; + loop?: boolean; setReviewed: () => void; setIgnoreClick: (ignore: boolean) => void; isPlayingBack: (ended: boolean) => void; onTimeUpdate?: (time: number | undefined) => void; }; -function VideoPreview({ +export function VideoPreview({ relevantPreview, startTime, endTime, + showProgress = true, + loop = false, setReviewed, setIgnoreClick, isPlayingBack, @@ -425,6 +429,11 @@ function VideoPreview({ if (playerPercent > 100) { setReviewed(); + if (loop && playerRef.current) { + playerRef.current.currentTime = playerStartTime; + return; + } + if (isMobile) { isPlayingBack(false); @@ -553,17 +562,19 @@ function VideoPreview({ > - + {showProgress && ( + + )}
); } @@ -572,14 +583,18 @@ const MIN_LOAD_TIMEOUT_MS = 200; type InProgressPreviewProps = { review: ReviewSegment; timeRange: TimeRange; + showProgress?: boolean; + loop?: boolean; setReviewed: (reviewId: string) => void; setIgnoreClick: (ignore: boolean) => void; isPlayingBack: (ended: boolean) => void; onTimeUpdate?: (time: number | undefined) => void; }; -function InProgressPreview({ +export function InProgressPreview({ review, timeRange, + showProgress = true, + loop = false, setReviewed, setIgnoreClick, isPlayingBack, @@ -615,6 +630,11 @@ function InProgressPreview({ setReviewed(review.id); } + if (loop) { + setKey(0); + return; + } + if (isMobile) { isPlayingBack(false); @@ -717,17 +737,19 @@ function InProgressPreview({ src={`${apiHost}api/preview/${previewFrames[key]}/thumbnail.webp`} onLoad={handleLoad} /> - + {showProgress && ( + + )}
); }