From 049f27d710fb1d4bb0b3b67fafd842b257a076a1 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Thu, 11 Apr 2024 06:42:16 -0600 Subject: [PATCH] Ongoing review segments (#10924) * Update review maintainer to save events when ongoing * Handle previews for in progress review items * Reset DB items in app * Handle in progress review items * Scroll back down to selected event item * Handle undefined end time * Formatting * remove unused * Make export handles have full resolution * reduce preview thumbnail props * fix missing return Co-authored-by: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> --------- Co-authored-by: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> --- frigate/api/media.py | 15 ++++- frigate/api/review.py | 14 ++++- frigate/app.py | 8 +++ frigate/events/maintainer.py | 5 -- frigate/review/maintainer.py | 33 +++++++--- web/src/components/card/ReviewCard.tsx | 4 +- .../player/PreviewThumbnailPlayer.tsx | 62 +++++++++++-------- .../timeline/MotionReviewTimeline.tsx | 6 +- .../components/timeline/ReviewTimeline.tsx | 2 - web/src/hooks/use-camera-activity.ts | 6 +- web/src/pages/Events.tsx | 16 +++-- web/src/types/review.ts | 2 +- web/src/views/events/EventView.tsx | 47 +++++++++++--- web/src/views/live/LiveDashboardView.tsx | 4 +- 14 files changed, 160 insertions(+), 64 deletions(-) diff --git a/frigate/api/media.py b/frigate/api/media.py index c72d9b933..345636e25 100644 --- a/frigate/api/media.py +++ b/frigate/api/media.py @@ -1333,7 +1333,9 @@ def review_preview(id: str): padding = 8 start_ts = review.start_time - padding - end_ts = review.end_time + padding + end_ts = ( + review.end_time + padding if review.end_time else datetime.now().timestamp() + ) return preview_gif(review.camera, start_ts, end_ts) @@ -1344,8 +1346,15 @@ def preview_thumbnail(file_name: str): safe_file_name_current = secure_filename(file_name) preview_dir = os.path.join(CACHE_DIR, "preview_frames") - with open(os.path.join(preview_dir, safe_file_name_current), "rb") as image_file: - jpg_bytes = image_file.read() + try: + with open( + os.path.join(preview_dir, safe_file_name_current), "rb" + ) as image_file: + jpg_bytes = image_file.read() + except FileNotFoundError: + return make_response( + jsonify({"success": False, "message": "Image file not found"}), 404 + ) response = make_response(jpg_bytes) response.headers["Content-Type"] = "image/jpeg" diff --git a/frigate/api/review.py b/frigate/api/review.py index 03d9dad91..5d317c1cf 100644 --- a/frigate/api/review.py +++ b/frigate/api/review.py @@ -27,10 +27,18 @@ def review(): before = request.args.get("before", type=float, default=datetime.now().timestamp()) after = request.args.get( - "after", type=float, default=(datetime.now() - timedelta(hours=18)).timestamp() + "after", type=float, default=(datetime.now() - timedelta(hours=24)).timestamp() ) - clauses = [((ReviewSegment.start_time > after) & (ReviewSegment.end_time < before))] + clauses = [ + ( + (ReviewSegment.start_time > after) + & ( + (ReviewSegment.end_time.is_null(True)) + | (ReviewSegment.end_time < before) + ) + ) + ] if cameras != "all": camera_list = cameras.split(",") @@ -45,6 +53,7 @@ def review(): for label in filtered_labels: label_clauses.append( (ReviewSegment.data["objects"].cast("text") % f'*"{label}"*') + | (ReviewSegment.data["audio"].cast("text") % f'*"{label}"*') ) label_clause = reduce(operator.or_, label_clauses) @@ -94,6 +103,7 @@ def review_summary(): for label in filtered_labels: label_clauses.append( (ReviewSegment.data["objects"].cast("text") % f'*"{label}"*') + | (ReviewSegment.data["audio"].cast("text") % f'*"{label}"*') ) label_clause = reduce(operator.or_, label_clauses) diff --git a/frigate/app.py b/frigate/app.py index 74d3f2f58..596abed72 100644 --- a/frigate/app.py +++ b/frigate/app.py @@ -667,6 +667,14 @@ class FrigateApp: logger.info("Stopping...") self.stop_event.set() + # set an end_time on entries without an end_time before exiting + Event.update(end_time=datetime.datetime.now().timestamp()).where( + Event.end_time == None + ).execute() + ReviewSegment.update(end_time=datetime.datetime.now().timestamp()).where( + ReviewSegment.end_time == None + ).execute() + # Stop Communicators self.inter_process_communicator.stop() self.inter_config_updater.stop() diff --git a/frigate/events/maintainer.py b/frigate/events/maintainer.py index e6f66edc3..0ef3ffaac 100644 --- a/frigate/events/maintainer.py +++ b/frigate/events/maintainer.py @@ -1,4 +1,3 @@ -import datetime import logging import threading from multiprocessing import Queue @@ -112,10 +111,6 @@ class EventProcessor(threading.Thread): self.handle_external_detection(event_type, event_data) - # set an end_time on events without an end_time before exiting - Event.update(end_time=datetime.datetime.now().timestamp()).where( - Event.end_time == None - ).execute() self.event_receiver.stop() self.event_end_publisher.stop() logger.info("Exiting event processor...") diff --git a/frigate/review/maintainer.py b/frigate/review/maintainer.py index 5cbd3a476..06fba8fdd 100644 --- a/frigate/review/maintainer.py +++ b/frigate/review/maintainer.py @@ -66,6 +66,7 @@ class PendingReviewSegment: # thumbnail self.frame = np.zeros((THUMB_HEIGHT * 3 // 2, THUMB_WIDTH), np.uint8) self.frame_active_count = 0 + self.frame_path = os.path.join(CLIPS_DIR, f"thumb-{self.camera}-{self.id}.jpg") def update_frame( self, camera_config: CameraConfig, frame, objects: list[TrackedObject] @@ -98,19 +99,19 @@ class PendingReviewSegment: color_frame, dsize=(width, THUMB_HEIGHT), interpolation=cv2.INTER_AREA ) - def end(self) -> dict: - path = os.path.join(CLIPS_DIR, f"thumb-{self.camera}-{self.id}.jpg") - if self.frame is not None: - cv2.imwrite(path, self.frame, [int(cv2.IMWRITE_WEBP_QUALITY), 60]) + cv2.imwrite( + self.frame_path, self.frame, [int(cv2.IMWRITE_WEBP_QUALITY), 60] + ) + def get_data(self, ended: bool) -> dict: return { ReviewSegment.id: self.id, ReviewSegment.camera: self.camera, ReviewSegment.start_time: self.start_time, - ReviewSegment.end_time: self.last_update, + ReviewSegment.end_time: self.last_update if ended else None, ReviewSegment.severity: self.severity.value, - ReviewSegment.thumb_path: path, + ReviewSegment.thumb_path: self.frame_path, ReviewSegment.data: { "detections": list(set(self.detections.keys())), "objects": list(set(self.detections.values())), @@ -141,9 +142,20 @@ class ReviewSegmentMaintainer(threading.Thread): self.stop_event = stop_event + def update_segment(self, segment: PendingReviewSegment) -> None: + """Update segment.""" + seg_data = segment.get_data(ended=False) + self.requestor.send_data(UPSERT_REVIEW_SEGMENT, seg_data) + self.requestor.send_data( + "reviews", + json.dumps( + {"type": "update", "review": {k.name: v for k, v in seg_data.items()}} + ), + ) + def end_segment(self, segment: PendingReviewSegment) -> None: """End segment.""" - seg_data = segment.end() + seg_data = segment.get_data(ended=True) self.requestor.send_data(UPSERT_REVIEW_SEGMENT, seg_data) self.requestor.send_data( "reviews", @@ -179,6 +191,7 @@ class ReviewSegmentMaintainer(threading.Thread): ) segment.update_frame(camera_config, yuv_frame, active_objects) self.frame_manager.close(frame_id) + self.update_segment(segment) for object in active_objects: if not object["sub_label"]: @@ -263,6 +276,7 @@ class ReviewSegmentMaintainer(threading.Thread): camera_config, yuv_frame, active_objects ) self.frame_manager.close(frame_id) + self.update_segment(self.active_review_segments[camera]) elif len(motion) >= 20: self.active_review_segments[camera] = PendingReviewSegment( camera, @@ -398,6 +412,11 @@ class ReviewSegmentMaintainer(threading.Thread): "end_time" ] + self.config_subscriber.stop() + self.requestor.stop() + self.detection_subscriber.stop() + logger.info("Exiting review maintainer...") + def get_active_objects( frame_time: float, camera_config: CameraConfig, all_objects: list[TrackedObject] diff --git a/web/src/components/card/ReviewCard.tsx b/web/src/components/card/ReviewCard.tsx index 66710d6ff..941beaa2e 100644 --- a/web/src/components/card/ReviewCard.tsx +++ b/web/src/components/card/ReviewCard.tsx @@ -27,7 +27,9 @@ export default function ReviewCard({ config?.ui.time_format == "24hour" ? "%H:%M" : "%I:%M %p", ); const isSelected = useMemo( - () => event.start_time <= currentTime && event.end_time >= currentTime, + () => + event.start_time <= currentTime && + (event.end_time ?? Date.now() / 1000) >= currentTime, [event, currentTime], ); diff --git a/web/src/components/player/PreviewThumbnailPlayer.tsx b/web/src/components/player/PreviewThumbnailPlayer.tsx index a7a54ae60..b4994de60 100644 --- a/web/src/components/player/PreviewThumbnailPlayer.tsx +++ b/web/src/components/player/PreviewThumbnailPlayer.tsx @@ -21,11 +21,14 @@ import { useSwipeable } from "react-swipeable"; import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip"; import ImageLoadingIndicator from "../indicators/ImageLoadingIndicator"; import useContextMenu from "@/hooks/use-contextmenu"; +import ActivityIndicator from "../indicators/activity-indicator"; +import { TimeRange } from "@/types/timeline"; type PreviewPlayerProps = { review: ReviewSegment; allPreviews?: Preview[]; scrollLock?: boolean; + timeRange: TimeRange; onTimeUpdate?: (time: number | undefined) => void; setReviewed: (review: ReviewSegment) => void; onClick: (review: ReviewSegment, ctrl: boolean) => void; @@ -43,6 +46,7 @@ export default function PreviewThumbnailPlayer({ review, allPreviews, scrollLock = false, + timeRange, setReviewed, onClick, onTimeUpdate, @@ -70,8 +74,10 @@ export default function PreviewThumbnailPlayer({ }); const handleSetReviewed = useCallback(() => { - review.has_been_reviewed = true; - setReviewed(review); + if (review.end_time && !review.has_been_reviewed) { + review.has_been_reviewed = true; + setReviewed(review); + } }, [review, setReviewed]); useContextMenu(imgRef, () => { @@ -91,7 +97,7 @@ export default function PreviewThumbnailPlayer({ return false; } - if (review.end_time > preview.end) { + if ((review.end_time ?? timeRange.before) > preview.end) { multiHour = true; } @@ -108,7 +114,8 @@ export default function PreviewThumbnailPlayer({ const firstPrev = allPreviews[firstIndex]; const firstDuration = firstPrev.end - review.start_time; - const secondDuration = review.end_time - firstPrev.end; + const secondDuration = + (review.end_time ?? timeRange.before) - firstPrev.end; if (firstDuration > secondDuration) { // the first preview is longer than the second, return the first @@ -123,7 +130,7 @@ export default function PreviewThumbnailPlayer({ return undefined; } - }, [allPreviews, review]); + }, [allPreviews, review, timeRange]); // Hover Playback @@ -183,6 +190,7 @@ export default function PreviewThumbnailPlayer({
- + {review.end_time ? ( + + ) : ( +
+ +
+ )} {formattedDate}
@@ -270,6 +284,7 @@ export default function PreviewThumbnailPlayer({ type PreviewContentProps = { review: ReviewSegment; relevantPreview: Preview | undefined; + timeRange: TimeRange; setReviewed: () => void; setIgnoreClick: (ignore: boolean) => void; isPlayingBack: (ended: boolean) => void; @@ -278,6 +293,7 @@ type PreviewContentProps = { function PreviewContent({ review, relevantPreview, + timeRange, setReviewed, setIgnoreClick, isPlayingBack, @@ -288,8 +304,9 @@ function PreviewContent({ if (relevantPreview) { return ( void; setIgnoreClick: (ignore: boolean) => void; isPlayingBack: (ended: boolean) => void; onTimeUpdate?: (time: number | undefined) => void; }; function VideoPreview({ - review, relevantPreview, + startTime, + endTime, setReviewed, setIgnoreClick, isPlayingBack, @@ -339,16 +359,13 @@ function VideoPreview({ } // start with a bit of padding - return Math.max( - 0, - review.start_time - relevantPreview.start - PREVIEW_PADDING, - ); + return Math.max(0, startTime - relevantPreview.start - PREVIEW_PADDING); // we know that these deps are correct // eslint-disable-next-line react-hooks/exhaustive-deps }, []); const playerDuration = useMemo( - () => review.end_time - review.start_time + PREVIEW_PADDING, + () => (endTime ?? relevantPreview.end) - startTime + PREVIEW_PADDING, // we know that these deps are correct // eslint-disable-next-line react-hooks/exhaustive-deps [], @@ -389,21 +406,14 @@ function VideoPreview({ // end with a bit of padding const playerPercent = (playerProgress / playerDuration) * 100; - if ( - setReviewed && - !review.has_been_reviewed && - lastPercent < 50 && - playerPercent > 50 - ) { + if (setReviewed && lastPercent < 50 && playerPercent > 50) { setReviewed(); } setLastPercent(playerPercent); if (playerPercent > 100) { - if (!review.has_been_reviewed) { - setReviewed(); - } + setReviewed(); if (isMobile) { isPlayingBack(false); @@ -468,7 +478,7 @@ function VideoPreview({ setIgnoreClick(true); } - if (setReviewed && !review.has_been_reviewed) { + if (setReviewed) { setReviewed(); } @@ -551,6 +561,7 @@ function VideoPreview({ const MIN_LOAD_TIMEOUT_MS = 200; type InProgressPreviewProps = { review: ReviewSegment; + timeRange: TimeRange; setReviewed: (reviewId: string) => void; setIgnoreClick: (ignore: boolean) => void; isPlayingBack: (ended: boolean) => void; @@ -558,6 +569,7 @@ type InProgressPreviewProps = { }; function InProgressPreview({ review, + timeRange, setReviewed, setIgnoreClick, isPlayingBack, @@ -567,7 +579,7 @@ function InProgressPreview({ const sliderRef = useRef(null); const { data: previewFrames } = useSWR( `preview/${review.camera}/start/${Math.floor(review.start_time) - PREVIEW_PADDING}/end/${ - Math.ceil(review.end_time) + PREVIEW_PADDING + Math.ceil(review.end_time ?? timeRange.before) + PREVIEW_PADDING }/frames`, { revalidateOnFocus: false }, ); diff --git a/web/src/components/timeline/MotionReviewTimeline.tsx b/web/src/components/timeline/MotionReviewTimeline.tsx index fef69a75f..2834437cd 100644 --- a/web/src/components/timeline/MotionReviewTimeline.tsx +++ b/web/src/components/timeline/MotionReviewTimeline.tsx @@ -100,8 +100,10 @@ export function MotionReviewTimeline({ const overlappingReviewItems = events.some( (item) => (item.start_time >= motionStart && item.start_time < motionEnd) || - (item.end_time > motionStart && item.end_time <= motionEnd) || - (item.start_time <= motionStart && item.end_time >= motionEnd), + ((item.end_time ?? timelineStart) > motionStart && + (item.end_time ?? timelineStart) <= motionEnd) || + (item.start_time <= motionStart && + (item.end_time ?? timelineStart) >= motionEnd), ); if ((!segmentMotion || overlappingReviewItems) && motionOnly) { diff --git a/web/src/components/timeline/ReviewTimeline.tsx b/web/src/components/timeline/ReviewTimeline.tsx index 16aec8083..2da91414d 100644 --- a/web/src/components/timeline/ReviewTimeline.tsx +++ b/web/src/components/timeline/ReviewTimeline.tsx @@ -132,7 +132,6 @@ export function ReviewTimeline({ draggableElementTime: exportStartTime, draggableElementLatestTime: paddedExportEndTime, setDraggableElementTime: setExportStartTime, - alignSetTimeToSegment: true, timelineDuration, timelineStartAligned, isDragging: isDraggingExportStart, @@ -157,7 +156,6 @@ export function ReviewTimeline({ draggableElementTime: exportEndTime, draggableElementEarliestTime: paddedExportStartTime, setDraggableElementTime: setExportEndTime, - alignSetTimeToSegment: true, timelineDuration, timelineStartAligned, isDragging: isDraggingExportEnd, diff --git a/web/src/hooks/use-camera-activity.ts b/web/src/hooks/use-camera-activity.ts index eb5b50dfe..99b8702b5 100644 --- a/web/src/hooks/use-camera-activity.ts +++ b/web/src/hooks/use-camera-activity.ts @@ -107,8 +107,10 @@ export function useCameraMotionNextTimestamp( const overlappingReviewItems = reviewItems.some( (item) => (item.start_time >= motionStart && item.start_time < motionEnd) || - (item.end_time > motionStart && item.end_time <= motionEnd) || - (item.start_time <= motionStart && item.end_time >= motionEnd), + ((item.end_time ?? Date.now() / 1000) > motionStart && + (item.end_time ?? Date.now() / 1000) <= motionEnd) || + (item.start_time <= motionStart && + (item.end_time ?? Date.now() / 1000) >= motionEnd), ); if (!segmentMotion || overlappingReviewItems) { diff --git a/web/src/pages/Events.tsx b/web/src/pages/Events.tsx index 91b8f1839..c9dac4460 100644 --- a/web/src/pages/Events.tsx +++ b/web/src/pages/Events.tsx @@ -204,7 +204,7 @@ export default function Events() { const newData = [...data]; newData.forEach((seg) => { - if (seg.severity == severity) { + if (seg.end_time && seg.severity == severity) { seg.has_been_reviewed = true; } }); @@ -214,10 +214,16 @@ export default function Events() { { revalidate: false, populateCache: true }, ); - await axios.post(`reviews/viewed`, { - ids: currentItems?.map((seg) => seg.id), - }); - reloadData(); + const itemsToMarkReviewed = currentItems + ?.filter((seg) => seg.end_time) + ?.map((seg) => seg.id); + + if (itemsToMarkReviewed.length > 0) { + await axios.post(`reviews/viewed`, { + ids: itemsToMarkReviewed, + }); + reloadData(); + } }, [reloadData, updateSegments], ); diff --git a/web/src/types/review.ts b/web/src/types/review.ts index e18d3b785..ea5bbb250 100644 --- a/web/src/types/review.ts +++ b/web/src/types/review.ts @@ -3,7 +3,7 @@ export interface ReviewSegment { camera: string; severity: ReviewSeverity; start_time: number; - end_time: number; + end_time?: number; thumb_path: string; has_been_reviewed: boolean; data: ReviewData; diff --git a/web/src/views/events/EventView.tsx b/web/src/views/events/EventView.tsx index def4aa972..048141bbd 100644 --- a/web/src/views/events/EventView.tsx +++ b/web/src/views/events/EventView.tsx @@ -43,6 +43,7 @@ import { TimeRange } from "@/types/timeline"; import { useCameraMotionNextTimestamp } from "@/hooks/use-camera-activity"; import useOptimisticState from "@/hooks/use-optimistic-state"; import { Skeleton } from "@/components/ui/skeleton"; +import scrollIntoView from "scroll-into-view-if-needed"; type EventViewProps = { reviews?: ReviewSegment[]; @@ -293,6 +294,7 @@ export default function EventView({ severity={severity} filter={filter} timeRange={timeRange} + startTime={startTime} markItemAsReviewed={markItemAsReviewed} markAllItemsAsReviewed={markAllItemsAsReviewed} onSelectReview={onSelectReview} @@ -331,6 +333,7 @@ type DetectionReviewProps = { severity: ReviewSeverity; filter?: ReviewFilter; timeRange: { before: number; after: number }; + startTime?: number; markItemAsReviewed: (review: ReviewSegment) => void; markAllItemsAsReviewed: (currentItems: ReviewSegment[]) => void; onSelectReview: (review: ReviewSegment, ctrl: boolean) => void; @@ -345,6 +348,7 @@ function DetectionReview({ severity, filter, timeRange, + startTime, markItemAsReviewed, markAllItemsAsReviewed, onSelectReview, @@ -495,6 +499,26 @@ function DetectionReview({ [minimap], ); + // existing review item + + useEffect(() => { + if (!startTime || !currentItems || currentItems.length == 0) { + return; + } + + const element = contentRef.current?.querySelector( + `[data-start="${startTime}"]`, + ); + if (element) { + scrollIntoView(element, { + scrollMode: "if-needed", + behavior: "smooth", + }); + } + // only run when start time changes + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [startTime]); + return ( <>
+ const matchingItem = reviewItems?.all.find((item) => { + const endTime = item.end_time ?? timeRange.before; + + return ( ((item.start_time >= segmentStartTime && item.start_time < segmentEndTime) || - (item.end_time > segmentStartTime && - item.end_time <= segmentEndTime) || + (endTime > segmentStartTime && endTime <= segmentEndTime) || (item.start_time <= segmentStartTime && - item.end_time >= segmentEndTime)) && - item.camera === cameraName, - ); + endTime >= segmentEndTime)) && + item.camera === cameraName + ); + item.start_time < segmentEndTime) || + (endTime > segmentStartTime && endTime <= segmentEndTime) || + (item.start_time <= segmentStartTime && + endTime >= segmentEndTime)) && + item.camera === cameraName; + }); return matchingItem ? matchingItem.severity : null; } @@ -805,6 +837,7 @@ function MotionReview({ reviewItems, motionData, currentTime, + timeRange, motionOnly, alignStartDateToTimeline, ], diff --git a/web/src/views/live/LiveDashboardView.tsx b/web/src/views/live/LiveDashboardView.tsx index c9f775da7..c8ddd1822 100644 --- a/web/src/views/live/LiveDashboardView.tsx +++ b/web/src/views/live/LiveDashboardView.tsx @@ -47,8 +47,8 @@ export default function LiveDashboardView({ } // if event is ended and was saved, update events list - if (eventUpdate.type == "end" && eventUpdate.review.severity == "alert") { - setTimeout(() => updateEvents(), 1000); + if (eventUpdate.review.severity == "alert") { + setTimeout(() => updateEvents(), eventUpdate.type == "end" ? 1000 : 6000); return; } }, [eventUpdate, updateEvents]);