Ongoing review segments (#10924)

* Update review maintainer to save events when ongoing

* Handle previews for in progress review items

* Reset DB items in app

* Handle in progress review items

* Scroll back down to selected event item

* Handle undefined end time

* Formatting

* remove unused

* Make export handles have full resolution

* reduce preview thumbnail props

* fix missing return

Co-authored-by: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>

---------

Co-authored-by: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>

frigate/api/media.py

@@ -1333,7 +1333,9 @@ def review_preview(id: str):
 
     padding = 8
     start_ts = review.start_time - padding
-    end_ts = review.end_time + padding
+    end_ts = (
+        review.end_time + padding if review.end_time else datetime.now().timestamp()
+    )
     return preview_gif(review.camera, start_ts, end_ts)
 
 
@@ -1344,8 +1346,15 @@ def preview_thumbnail(file_name: str):
     safe_file_name_current = secure_filename(file_name)
     preview_dir = os.path.join(CACHE_DIR, "preview_frames")
 
-    with open(os.path.join(preview_dir, safe_file_name_current), "rb") as image_file:
+    try:
+        with open(
+            os.path.join(preview_dir, safe_file_name_current), "rb"
+        ) as image_file:
             jpg_bytes = image_file.read()
+    except FileNotFoundError:
+        return make_response(
+            jsonify({"success": False, "message": "Image file not found"}), 404
+        )
 
     response = make_response(jpg_bytes)
     response.headers["Content-Type"] = "image/jpeg"

frigate/api/review.py

@@ -27,10 +27,18 @@ def review():
 
     before = request.args.get("before", type=float, default=datetime.now().timestamp())
     after = request.args.get(
-        "after", type=float, default=(datetime.now() - timedelta(hours=18)).timestamp()
+        "after", type=float, default=(datetime.now() - timedelta(hours=24)).timestamp()
     )
 
-    clauses = [((ReviewSegment.start_time > after) & (ReviewSegment.end_time < before))]
+    clauses = [
+        (
+            (ReviewSegment.start_time > after)
+            & (
+                (ReviewSegment.end_time.is_null(True))
+                | (ReviewSegment.end_time < before)
+            )
+        )
+    ]
 
     if cameras != "all":
         camera_list = cameras.split(",")
@@ -45,6 +53,7 @@ def review():
         for label in filtered_labels:
             label_clauses.append(
                 (ReviewSegment.data["objects"].cast("text") % f'*"{label}"*')
+                | (ReviewSegment.data["audio"].cast("text") % f'*"{label}"*')
             )
 
         label_clause = reduce(operator.or_, label_clauses)
@@ -94,6 +103,7 @@ def review_summary():
         for label in filtered_labels:
             label_clauses.append(
                 (ReviewSegment.data["objects"].cast("text") % f'*"{label}"*')
+                | (ReviewSegment.data["audio"].cast("text") % f'*"{label}"*')
             )
 
         label_clause = reduce(operator.or_, label_clauses)

frigate/app.py

@@ -667,6 +667,14 @@ class FrigateApp:
         logger.info("Stopping...")
         self.stop_event.set()
 
+        # set an end_time on entries without an end_time before exiting
+        Event.update(end_time=datetime.datetime.now().timestamp()).where(
+            Event.end_time == None
+        ).execute()
+        ReviewSegment.update(end_time=datetime.datetime.now().timestamp()).where(
+            ReviewSegment.end_time == None
+        ).execute()
+
         # Stop Communicators
         self.inter_process_communicator.stop()
         self.inter_config_updater.stop()

frigate/events/maintainer.py

@@ -1,4 +1,3 @@
-import datetime
 import logging
 import threading
 from multiprocessing import Queue
@@ -112,10 +111,6 @@ class EventProcessor(threading.Thread):
 
                 self.handle_external_detection(event_type, event_data)
 
-        # set an end_time on events without an end_time before exiting
-        Event.update(end_time=datetime.datetime.now().timestamp()).where(
-            Event.end_time == None
-        ).execute()
         self.event_receiver.stop()
         self.event_end_publisher.stop()
         logger.info("Exiting event processor...")

frigate/review/maintainer.py

@@ -66,6 +66,7 @@ class PendingReviewSegment:
         # thumbnail
         self.frame = np.zeros((THUMB_HEIGHT * 3 // 2, THUMB_WIDTH), np.uint8)
         self.frame_active_count = 0
+        self.frame_path = os.path.join(CLIPS_DIR, f"thumb-{self.camera}-{self.id}.jpg")
 
     def update_frame(
         self, camera_config: CameraConfig, frame, objects: list[TrackedObject]
@@ -98,19 +99,19 @@ class PendingReviewSegment:
             color_frame, dsize=(width, THUMB_HEIGHT), interpolation=cv2.INTER_AREA
         )
 
-    def end(self) -> dict:
-        path = os.path.join(CLIPS_DIR, f"thumb-{self.camera}-{self.id}.jpg")
-
         if self.frame is not None:
-            cv2.imwrite(path, self.frame, [int(cv2.IMWRITE_WEBP_QUALITY), 60])
+            cv2.imwrite(
+                self.frame_path, self.frame, [int(cv2.IMWRITE_WEBP_QUALITY), 60]
+            )
 
+    def get_data(self, ended: bool) -> dict:
         return {
             ReviewSegment.id: self.id,
             ReviewSegment.camera: self.camera,
             ReviewSegment.start_time: self.start_time,
-            ReviewSegment.end_time: self.last_update,
+            ReviewSegment.end_time: self.last_update if ended else None,
             ReviewSegment.severity: self.severity.value,
-            ReviewSegment.thumb_path: path,
+            ReviewSegment.thumb_path: self.frame_path,
             ReviewSegment.data: {
                 "detections": list(set(self.detections.keys())),
                 "objects": list(set(self.detections.values())),
@@ -141,9 +142,20 @@ class ReviewSegmentMaintainer(threading.Thread):
 
         self.stop_event = stop_event
 
+    def update_segment(self, segment: PendingReviewSegment) -> None:
+        """Update segment."""
+        seg_data = segment.get_data(ended=False)
+        self.requestor.send_data(UPSERT_REVIEW_SEGMENT, seg_data)
+        self.requestor.send_data(
+            "reviews",
+            json.dumps(
+                {"type": "update", "review": {k.name: v for k, v in seg_data.items()}}
+            ),
+        )
+
     def end_segment(self, segment: PendingReviewSegment) -> None:
         """End segment."""
-        seg_data = segment.end()
+        seg_data = segment.get_data(ended=True)
         self.requestor.send_data(UPSERT_REVIEW_SEGMENT, seg_data)
         self.requestor.send_data(
             "reviews",
@@ -179,6 +191,7 @@ class ReviewSegmentMaintainer(threading.Thread):
                 )
                 segment.update_frame(camera_config, yuv_frame, active_objects)
                 self.frame_manager.close(frame_id)
+                self.update_segment(segment)
 
             for object in active_objects:
                 if not object["sub_label"]:
@@ -263,6 +276,7 @@ class ReviewSegmentMaintainer(threading.Thread):
                 camera_config, yuv_frame, active_objects
             )
             self.frame_manager.close(frame_id)
+            self.update_segment(self.active_review_segments[camera])
         elif len(motion) >= 20:
             self.active_review_segments[camera] = PendingReviewSegment(
                 camera,
@@ -398,6 +412,11 @@ class ReviewSegmentMaintainer(threading.Thread):
                             "end_time"
                         ]
 
+        self.config_subscriber.stop()
+        self.requestor.stop()
+        self.detection_subscriber.stop()
+        logger.info("Exiting review maintainer...")
+
 
 def get_active_objects(
     frame_time: float, camera_config: CameraConfig, all_objects: list[TrackedObject]

web/src/components/card/ReviewCard.tsx

@@ -27,7 +27,9 @@ export default function ReviewCard({
     config?.ui.time_format == "24hour" ? "%H:%M" : "%I:%M %p",
   );
   const isSelected = useMemo(
-    () => event.start_time <= currentTime && event.end_time >= currentTime,
+    () =>
+      event.start_time <= currentTime &&
+      (event.end_time ?? Date.now() / 1000) >= currentTime,
     [event, currentTime],
   );
 

web/src/components/player/PreviewThumbnailPlayer.tsx

@@ -21,11 +21,14 @@ import { useSwipeable } from "react-swipeable";
 import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip";
 import ImageLoadingIndicator from "../indicators/ImageLoadingIndicator";
 import useContextMenu from "@/hooks/use-contextmenu";
+import ActivityIndicator from "../indicators/activity-indicator";
+import { TimeRange } from "@/types/timeline";
 
 type PreviewPlayerProps = {
   review: ReviewSegment;
   allPreviews?: Preview[];
   scrollLock?: boolean;
+  timeRange: TimeRange;
   onTimeUpdate?: (time: number | undefined) => void;
   setReviewed: (review: ReviewSegment) => void;
   onClick: (review: ReviewSegment, ctrl: boolean) => void;
@@ -43,6 +46,7 @@ export default function PreviewThumbnailPlayer({
   review,
   allPreviews,
   scrollLock = false,
+  timeRange,
   setReviewed,
   onClick,
   onTimeUpdate,
@@ -70,8 +74,10 @@ export default function PreviewThumbnailPlayer({
   });
 
   const handleSetReviewed = useCallback(() => {
+    if (review.end_time && !review.has_been_reviewed) {
       review.has_been_reviewed = true;
       setReviewed(review);
+    }
   }, [review, setReviewed]);
 
   useContextMenu(imgRef, () => {
@@ -91,7 +97,7 @@ export default function PreviewThumbnailPlayer({
         return false;
       }
 
-      if (review.end_time > preview.end) {
+      if ((review.end_time ?? timeRange.before) > preview.end) {
         multiHour = true;
       }
 
@@ -108,7 +114,8 @@ export default function PreviewThumbnailPlayer({
 
     const firstPrev = allPreviews[firstIndex];
     const firstDuration = firstPrev.end - review.start_time;
-    const secondDuration = review.end_time - firstPrev.end;
+    const secondDuration =
+      (review.end_time ?? timeRange.before) - firstPrev.end;
 
     if (firstDuration > secondDuration) {
       // the first preview is longer than the second, return the first
@@ -123,7 +130,7 @@ export default function PreviewThumbnailPlayer({
 
       return undefined;
     }
-  }, [allPreviews, review]);
+  }, [allPreviews, review, timeRange]);
 
   // Hover Playback
 
@@ -183,6 +190,7 @@ export default function PreviewThumbnailPlayer({
           <PreviewContent
             review={review}
             relevantPreview={relevantPreview}
+            timeRange={timeRange}
             setReviewed={handleSetReviewed}
             setIgnoreClick={setIgnoreClick}
             isPlayingBack={setPlayback}
@@ -256,7 +264,13 @@ export default function PreviewThumbnailPlayer({
             <div className="absolute top-0 inset-x-0 rounded-t-l z-10 w-full h-[30%] bg-gradient-to-b from-black/60 to-transparent pointer-events-none"></div>
             <div className="absolute bottom-0 inset-x-0 rounded-b-l z-10 w-full h-[20%] bg-gradient-to-t from-black/60 to-transparent pointer-events-none">
               <div className="flex h-full justify-between items-end mx-3 pb-1 text-white text-sm">
+                {review.end_time ? (
                   <TimeAgo time={review.start_time * 1000} dense />
+                ) : (
+                  <div>
+                    <ActivityIndicator size={24} />
+                  </div>
+                )}
                 {formattedDate}
               </div>
             </div>
@@ -270,6 +284,7 @@ export default function PreviewThumbnailPlayer({
 type PreviewContentProps = {
   review: ReviewSegment;
   relevantPreview: Preview | undefined;
+  timeRange: TimeRange;
   setReviewed: () => void;
   setIgnoreClick: (ignore: boolean) => void;
   isPlayingBack: (ended: boolean) => void;
@@ -278,6 +293,7 @@ type PreviewContentProps = {
 function PreviewContent({
   review,
   relevantPreview,
+  timeRange,
   setReviewed,
   setIgnoreClick,
   isPlayingBack,
@@ -288,8 +304,9 @@ function PreviewContent({
   if (relevantPreview) {
     return (
       <VideoPreview
-        review={review}
         relevantPreview={relevantPreview}
+        startTime={review.start_time}
+        endTime={review.end_time}
         setReviewed={setReviewed}
         setIgnoreClick={setIgnoreClick}
         isPlayingBack={isPlayingBack}
@@ -300,6 +317,7 @@ function PreviewContent({
     return (
       <InProgressPreview
         review={review}
+        timeRange={timeRange}
         setReviewed={setReviewed}
         setIgnoreClick={setIgnoreClick}
         isPlayingBack={isPlayingBack}
@@ -311,16 +329,18 @@ function PreviewContent({
 
 const PREVIEW_PADDING = 16;
 type VideoPreviewProps = {
-  review: ReviewSegment;
   relevantPreview: Preview;
+  startTime: number;
+  endTime?: number;
   setReviewed: () => void;
   setIgnoreClick: (ignore: boolean) => void;
   isPlayingBack: (ended: boolean) => void;
   onTimeUpdate?: (time: number | undefined) => void;
 };
 function VideoPreview({
-  review,
   relevantPreview,
+  startTime,
+  endTime,
   setReviewed,
   setIgnoreClick,
   isPlayingBack,
@@ -339,16 +359,13 @@ function VideoPreview({
     }
 
     // start with a bit of padding
-    return Math.max(
+    return Math.max(0, startTime - relevantPreview.start - PREVIEW_PADDING);
-      0,
-      review.start_time - relevantPreview.start - PREVIEW_PADDING,
-    );
 
     // we know that these deps are correct
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, []);
   const playerDuration = useMemo(
-    () => review.end_time - review.start_time + PREVIEW_PADDING,
+    () => (endTime ?? relevantPreview.end) - startTime + PREVIEW_PADDING,
     // we know that these deps are correct
     // eslint-disable-next-line react-hooks/exhaustive-deps
     [],
@@ -389,21 +406,14 @@ function VideoPreview({
     // end with a bit of padding
     const playerPercent = (playerProgress / playerDuration) * 100;
 
-    if (
+    if (setReviewed && lastPercent < 50 && playerPercent > 50) {
-      setReviewed &&
-      !review.has_been_reviewed &&
-      lastPercent < 50 &&
-      playerPercent > 50
-    ) {
       setReviewed();
     }
 
     setLastPercent(playerPercent);
 
     if (playerPercent > 100) {
-      if (!review.has_been_reviewed) {
       setReviewed();
-      }
 
       if (isMobile) {
         isPlayingBack(false);
@@ -468,7 +478,7 @@ function VideoPreview({
         setIgnoreClick(true);
       }
 
-      if (setReviewed && !review.has_been_reviewed) {
+      if (setReviewed) {
         setReviewed();
       }
 
@@ -551,6 +561,7 @@ function VideoPreview({
 const MIN_LOAD_TIMEOUT_MS = 200;
 type InProgressPreviewProps = {
   review: ReviewSegment;
+  timeRange: TimeRange;
   setReviewed: (reviewId: string) => void;
   setIgnoreClick: (ignore: boolean) => void;
   isPlayingBack: (ended: boolean) => void;
@@ -558,6 +569,7 @@ type InProgressPreviewProps = {
 };
 function InProgressPreview({
   review,
+  timeRange,
   setReviewed,
   setIgnoreClick,
   isPlayingBack,
@@ -567,7 +579,7 @@ function InProgressPreview({
   const sliderRef = useRef<HTMLDivElement | null>(null);
   const { data: previewFrames } = useSWR<string[]>(
     `preview/${review.camera}/start/${Math.floor(review.start_time) - PREVIEW_PADDING}/end/${
-      Math.ceil(review.end_time) + PREVIEW_PADDING
+      Math.ceil(review.end_time ?? timeRange.before) + PREVIEW_PADDING
     }/frames`,
     { revalidateOnFocus: false },
   );

web/src/components/timeline/MotionReviewTimeline.tsx

@@ -100,8 +100,10 @@ export function MotionReviewTimeline({
       const overlappingReviewItems = events.some(
         (item) =>
           (item.start_time >= motionStart && item.start_time < motionEnd) ||
-          (item.end_time > motionStart && item.end_time <= motionEnd) ||
+          ((item.end_time ?? timelineStart) > motionStart &&
-          (item.start_time <= motionStart && item.end_time >= motionEnd),
+            (item.end_time ?? timelineStart) <= motionEnd) ||
+          (item.start_time <= motionStart &&
+            (item.end_time ?? timelineStart) >= motionEnd),
       );
 
       if ((!segmentMotion || overlappingReviewItems) && motionOnly) {

web/src/components/timeline/ReviewTimeline.tsx

@@ -132,7 +132,6 @@ export function ReviewTimeline({
     draggableElementTime: exportStartTime,
     draggableElementLatestTime: paddedExportEndTime,
     setDraggableElementTime: setExportStartTime,
-    alignSetTimeToSegment: true,
     timelineDuration,
     timelineStartAligned,
     isDragging: isDraggingExportStart,
@@ -157,7 +156,6 @@ export function ReviewTimeline({
     draggableElementTime: exportEndTime,
     draggableElementEarliestTime: paddedExportStartTime,
     setDraggableElementTime: setExportEndTime,
-    alignSetTimeToSegment: true,
     timelineDuration,
     timelineStartAligned,
     isDragging: isDraggingExportEnd,

web/src/hooks/use-camera-activity.ts

@@ -107,8 +107,10 @@ export function useCameraMotionNextTimestamp(
       const overlappingReviewItems = reviewItems.some(
         (item) =>
           (item.start_time >= motionStart && item.start_time < motionEnd) ||
-          (item.end_time > motionStart && item.end_time <= motionEnd) ||
+          ((item.end_time ?? Date.now() / 1000) > motionStart &&
-          (item.start_time <= motionStart && item.end_time >= motionEnd),
+            (item.end_time ?? Date.now() / 1000) <= motionEnd) ||
+          (item.start_time <= motionStart &&
+            (item.end_time ?? Date.now() / 1000) >= motionEnd),
       );
 
       if (!segmentMotion || overlappingReviewItems) {

web/src/pages/Events.tsx

@@ -204,7 +204,7 @@ export default function Events() {
           const newData = [...data];
 
           newData.forEach((seg) => {
-            if (seg.severity == severity) {
+            if (seg.end_time && seg.severity == severity) {
               seg.has_been_reviewed = true;
             }
           });
@@ -214,10 +214,16 @@ export default function Events() {
         { revalidate: false, populateCache: true },
       );
 
+      const itemsToMarkReviewed = currentItems
+        ?.filter((seg) => seg.end_time)
+        ?.map((seg) => seg.id);
+
+      if (itemsToMarkReviewed.length > 0) {
         await axios.post(`reviews/viewed`, {
-        ids: currentItems?.map((seg) => seg.id),
+          ids: itemsToMarkReviewed,
         });
         reloadData();
+      }
     },
     [reloadData, updateSegments],
   );

web/src/types/review.ts

@@ -3,7 +3,7 @@ export interface ReviewSegment {
   camera: string;
   severity: ReviewSeverity;
   start_time: number;
-  end_time: number;
+  end_time?: number;
   thumb_path: string;
   has_been_reviewed: boolean;
   data: ReviewData;

web/src/views/events/EventView.tsx

@@ -43,6 +43,7 @@ import { TimeRange } from "@/types/timeline";
 import { useCameraMotionNextTimestamp } from "@/hooks/use-camera-activity";
 import useOptimisticState from "@/hooks/use-optimistic-state";
 import { Skeleton } from "@/components/ui/skeleton";
+import scrollIntoView from "scroll-into-view-if-needed";
 
 type EventViewProps = {
   reviews?: ReviewSegment[];
@@ -293,6 +294,7 @@ export default function EventView({
             severity={severity}
             filter={filter}
             timeRange={timeRange}
+            startTime={startTime}
             markItemAsReviewed={markItemAsReviewed}
             markAllItemsAsReviewed={markAllItemsAsReviewed}
             onSelectReview={onSelectReview}
@@ -331,6 +333,7 @@ type DetectionReviewProps = {
   severity: ReviewSeverity;
   filter?: ReviewFilter;
   timeRange: { before: number; after: number };
+  startTime?: number;
   markItemAsReviewed: (review: ReviewSegment) => void;
   markAllItemsAsReviewed: (currentItems: ReviewSegment[]) => void;
   onSelectReview: (review: ReviewSegment, ctrl: boolean) => void;
@@ -345,6 +348,7 @@ function DetectionReview({
   severity,
   filter,
   timeRange,
+  startTime,
   markItemAsReviewed,
   markAllItemsAsReviewed,
   onSelectReview,
@@ -495,6 +499,26 @@ function DetectionReview({
     [minimap],
   );
 
+  // existing review item
+
+  useEffect(() => {
+    if (!startTime || !currentItems || currentItems.length == 0) {
+      return;
+    }
+
+    const element = contentRef.current?.querySelector(
+      `[data-start="${startTime}"]`,
+    );
+    if (element) {
+      scrollIntoView(element, {
+        scrollMode: "if-needed",
+        behavior: "smooth",
+      });
+    }
+    // only run when start time changes
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [startTime]);
+
   return (
     <>
       <div
@@ -546,6 +570,7 @@ function DetectionReview({
                     <PreviewThumbnailPlayer
                       review={value}
                       allPreviews={relevantPreviews}
+                      timeRange={timeRange}
                       setReviewed={markItemAsReviewed}
                       scrollLock={scrollLock}
                       onTimeUpdate={onPreviewTimeUpdate}
@@ -787,16 +812,23 @@ function MotionReview({
       } else {
         const segmentStartTime = alignStartDateToTimeline(currentTime);
         const segmentEndTime = segmentStartTime + segmentDuration;
-        const matchingItem = reviewItems?.all.find(
+        const matchingItem = reviewItems?.all.find((item) => {
-          (item) =>
+          const endTime = item.end_time ?? timeRange.before;
+
+          return (
             ((item.start_time >= segmentStartTime &&
               item.start_time < segmentEndTime) ||
-              (item.end_time > segmentStartTime &&
+              (endTime > segmentStartTime && endTime <= segmentEndTime) ||
-                item.end_time <= segmentEndTime) ||
               (item.start_time <= segmentStartTime &&
-                item.end_time >= segmentEndTime)) &&
+                endTime >= segmentEndTime)) &&
-            item.camera === cameraName,
+            item.camera === cameraName
           );
+            item.start_time < segmentEndTime) ||
+            (endTime > segmentStartTime && endTime <= segmentEndTime) ||
+            (item.start_time <= segmentStartTime &&
+              endTime >= segmentEndTime)) &&
+            item.camera === cameraName;
+        });
 
         return matchingItem ? matchingItem.severity : null;
       }
@@ -805,6 +837,7 @@ function MotionReview({
       reviewItems,
       motionData,
       currentTime,
+      timeRange,
       motionOnly,
       alignStartDateToTimeline,
     ],

web/src/views/live/LiveDashboardView.tsx

@@ -47,8 +47,8 @@ export default function LiveDashboardView({
     }
 
     // if event is ended and was saved, update events list
-    if (eventUpdate.type == "end" && eventUpdate.review.severity == "alert") {
+    if (eventUpdate.review.severity == "alert") {
-      setTimeout(() => updateEvents(), 1000);
+      setTimeout(() => updateEvents(), eventUpdate.type == "end" ? 1000 : 6000);
       return;
     }
   }, [eventUpdate, updateEvents]);