Ongoing review segments (#10924)

* Update review maintainer to save events when ongoing * Handle previews for in progress review items * Reset DB items in app * Handle in progress review items * Scroll back down to selected event item * Handle undefined end time * Formatting * remove unused * Make export handles have full resolution * reduce preview thumbnail props * fix missing return Co-authored-by: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> --------- Co-authored-by: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>
2024-12-19 19:06:16 +01:00 · 2024-04-11 06:42:16 -06:00 · 2024-04-11 06:42:16 -06:00 · 049f27d710
commit 049f27d710
parent cf7698e7e1
14 changed files with 160 additions and 64 deletions
--- a/frigate/api/media.py
+++ b/frigate/api/media.py
@ -1333,7 +1333,9 @@ def review_preview(id: str):

    padding = 8
    start_ts = review.start_time - padding
-    end_ts = review.end_time + padding
+    end_ts = (
+        review.end_time + padding if review.end_time else datetime.now().timestamp()
+    )
    return preview_gif(review.camera, start_ts, end_ts)


@ -1344,8 +1346,15 @@ def preview_thumbnail(file_name: str):
    safe_file_name_current = secure_filename(file_name)
    preview_dir = os.path.join(CACHE_DIR, "preview_frames")

-    with open(os.path.join(preview_dir, safe_file_name_current), "rb") as image_file:
-        jpg_bytes = image_file.read()
+    try:
+        with open(
+            os.path.join(preview_dir, safe_file_name_current), "rb"
+        ) as image_file:
+            jpg_bytes = image_file.read()
+    except FileNotFoundError:
+        return make_response(
+            jsonify({"success": False, "message": "Image file not found"}), 404
+        )

    response = make_response(jpg_bytes)
    response.headers["Content-Type"] = "image/jpeg"
--- a/frigate/api/review.py
+++ b/frigate/api/review.py
@ -27,10 +27,18 @@ def review():

    before = request.args.get("before", type=float, default=datetime.now().timestamp())
    after = request.args.get(
-        "after", type=float, default=(datetime.now() - timedelta(hours=18)).timestamp()
+        "after", type=float, default=(datetime.now() - timedelta(hours=24)).timestamp()
    )

-    clauses = [((ReviewSegment.start_time > after) & (ReviewSegment.end_time < before))]
+    clauses = [
+        (
+            (ReviewSegment.start_time > after)
+            & (
+                (ReviewSegment.end_time.is_null(True))
+                | (ReviewSegment.end_time < before)
+            )
+        )
+    ]

    if cameras != "all":
        camera_list = cameras.split(",")
@ -45,6 +53,7 @@ def review():
        for label in filtered_labels:
            label_clauses.append(
                (ReviewSegment.data["objects"].cast("text") % f'*"{label}"*')
+                | (ReviewSegment.data["audio"].cast("text") % f'*"{label}"*')
            )

        label_clause = reduce(operator.or_, label_clauses)
@ -94,6 +103,7 @@ def review_summary():
        for label in filtered_labels:
            label_clauses.append(
                (ReviewSegment.data["objects"].cast("text") % f'*"{label}"*')
+                | (ReviewSegment.data["audio"].cast("text") % f'*"{label}"*')
            )

        label_clause = reduce(operator.or_, label_clauses)
--- a/frigate/app.py
+++ b/frigate/app.py
@ -667,6 +667,14 @@ class FrigateApp:
        logger.info("Stopping...")
        self.stop_event.set()

+        # set an end_time on entries without an end_time before exiting
+        Event.update(end_time=datetime.datetime.now().timestamp()).where(
+            Event.end_time == None
+        ).execute()
+        ReviewSegment.update(end_time=datetime.datetime.now().timestamp()).where(
+            ReviewSegment.end_time == None
+        ).execute()
+
        # Stop Communicators
        self.inter_process_communicator.stop()
        self.inter_config_updater.stop()
--- a/frigate/events/maintainer.py
+++ b/frigate/events/maintainer.py
@ -1,4 +1,3 @@
-import datetime
 import logging
 import threading
 from multiprocessing import Queue
@ -112,10 +111,6 @@ class EventProcessor(threading.Thread):

                self.handle_external_detection(event_type, event_data)

-        # set an end_time on events without an end_time before exiting
-        Event.update(end_time=datetime.datetime.now().timestamp()).where(
-            Event.end_time == None
-        ).execute()
        self.event_receiver.stop()
        self.event_end_publisher.stop()
        logger.info("Exiting event processor...")
--- a/frigate/review/maintainer.py
+++ b/frigate/review/maintainer.py
@ -66,6 +66,7 @@ class PendingReviewSegment:
        # thumbnail
        self.frame = np.zeros((THUMB_HEIGHT * 3 // 2, THUMB_WIDTH), np.uint8)
        self.frame_active_count = 0
+        self.frame_path = os.path.join(CLIPS_DIR, f"thumb-{self.camera}-{self.id}.jpg")

    def update_frame(
        self, camera_config: CameraConfig, frame, objects: list[TrackedObject]
@ -98,19 +99,19 @@ class PendingReviewSegment:
            color_frame, dsize=(width, THUMB_HEIGHT), interpolation=cv2.INTER_AREA
        )

-    def end(self) -> dict:
-        path = os.path.join(CLIPS_DIR, f"thumb-{self.camera}-{self.id}.jpg")
-
        if self.frame is not None:
-            cv2.imwrite(path, self.frame, [int(cv2.IMWRITE_WEBP_QUALITY), 60])
+            cv2.imwrite(
+                self.frame_path, self.frame, [int(cv2.IMWRITE_WEBP_QUALITY), 60]
+            )

+    def get_data(self, ended: bool) -> dict:
        return {
            ReviewSegment.id: self.id,
            ReviewSegment.camera: self.camera,
            ReviewSegment.start_time: self.start_time,
-            ReviewSegment.end_time: self.last_update,
+            ReviewSegment.end_time: self.last_update if ended else None,
            ReviewSegment.severity: self.severity.value,
-            ReviewSegment.thumb_path: path,
+            ReviewSegment.thumb_path: self.frame_path,
            ReviewSegment.data: {
                "detections": list(set(self.detections.keys())),
                "objects": list(set(self.detections.values())),
@ -141,9 +142,20 @@ class ReviewSegmentMaintainer(threading.Thread):

        self.stop_event = stop_event

+    def update_segment(self, segment: PendingReviewSegment) -> None:
+        """Update segment."""
+        seg_data = segment.get_data(ended=False)
+        self.requestor.send_data(UPSERT_REVIEW_SEGMENT, seg_data)
+        self.requestor.send_data(
+            "reviews",
+            json.dumps(
+                {"type": "update", "review": {k.name: v for k, v in seg_data.items()}}
+            ),
+        )
+
    def end_segment(self, segment: PendingReviewSegment) -> None:
        """End segment."""
-        seg_data = segment.end()
+        seg_data = segment.get_data(ended=True)
        self.requestor.send_data(UPSERT_REVIEW_SEGMENT, seg_data)
        self.requestor.send_data(
            "reviews",
@ -179,6 +191,7 @@ class ReviewSegmentMaintainer(threading.Thread):
                )
                segment.update_frame(camera_config, yuv_frame, active_objects)
                self.frame_manager.close(frame_id)
+                self.update_segment(segment)

            for object in active_objects:
                if not object["sub_label"]:
@ -263,6 +276,7 @@ class ReviewSegmentMaintainer(threading.Thread):
                camera_config, yuv_frame, active_objects
            )
            self.frame_manager.close(frame_id)
+            self.update_segment(self.active_review_segments[camera])
        elif len(motion) >= 20:
            self.active_review_segments[camera] = PendingReviewSegment(
                camera,
@ -398,6 +412,11 @@ class ReviewSegmentMaintainer(threading.Thread):
                            "end_time"
                        ]

+        self.config_subscriber.stop()
+        self.requestor.stop()
+        self.detection_subscriber.stop()
+        logger.info("Exiting review maintainer...")
+

 def get_active_objects(
    frame_time: float, camera_config: CameraConfig, all_objects: list[TrackedObject]
--- a/web/src/components/card/ReviewCard.tsx
+++ b/web/src/components/card/ReviewCard.tsx
@ -27,7 +27,9 @@ export default function ReviewCard({
    config?.ui.time_format == "24hour" ? "%H:%M" : "%I:%M %p",
  );
  const isSelected = useMemo(
-    () => event.start_time <= currentTime && event.end_time >= currentTime,
+    () =>
+      event.start_time <= currentTime &&
+      (event.end_time ?? Date.now() / 1000) >= currentTime,
    [event, currentTime],
  );

--- a/web/src/components/player/PreviewThumbnailPlayer.tsx
+++ b/web/src/components/player/PreviewThumbnailPlayer.tsx
@ -21,11 +21,14 @@ import { useSwipeable } from "react-swipeable";
 import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip";
 import ImageLoadingIndicator from "../indicators/ImageLoadingIndicator";
 import useContextMenu from "@/hooks/use-contextmenu";
+import ActivityIndicator from "../indicators/activity-indicator";
+import { TimeRange } from "@/types/timeline";

 type PreviewPlayerProps = {
  review: ReviewSegment;
  allPreviews?: Preview[];
  scrollLock?: boolean;
+  timeRange: TimeRange;
  onTimeUpdate?: (time: number | undefined) => void;
  setReviewed: (review: ReviewSegment) => void;
  onClick: (review: ReviewSegment, ctrl: boolean) => void;
@ -43,6 +46,7 @@ export default function PreviewThumbnailPlayer({
  review,
  allPreviews,
  scrollLock = false,
+  timeRange,
  setReviewed,
  onClick,
  onTimeUpdate,
@ -70,8 +74,10 @@ export default function PreviewThumbnailPlayer({
  });

  const handleSetReviewed = useCallback(() => {
-    review.has_been_reviewed = true;
-    setReviewed(review);
+    if (review.end_time && !review.has_been_reviewed) {
+      review.has_been_reviewed = true;
+      setReviewed(review);
+    }
  }, [review, setReviewed]);

  useContextMenu(imgRef, () => {
@ -91,7 +97,7 @@ export default function PreviewThumbnailPlayer({
        return false;
      }

-      if (review.end_time > preview.end) {
+      if ((review.end_time ?? timeRange.before) > preview.end) {
        multiHour = true;
      }

@ -108,7 +114,8 @@ export default function PreviewThumbnailPlayer({

    const firstPrev = allPreviews[firstIndex];
    const firstDuration = firstPrev.end - review.start_time;
-    const secondDuration = review.end_time - firstPrev.end;
+    const secondDuration =
+      (review.end_time ?? timeRange.before) - firstPrev.end;

    if (firstDuration > secondDuration) {
      // the first preview is longer than the second, return the first
@ -123,7 +130,7 @@ export default function PreviewThumbnailPlayer({

      return undefined;
    }
-  }, [allPreviews, review]);
+  }, [allPreviews, review, timeRange]);

  // Hover Playback

@ -183,6 +190,7 @@ export default function PreviewThumbnailPlayer({
          <PreviewContent
            review={review}
            relevantPreview={relevantPreview}
+            timeRange={timeRange}
            setReviewed={handleSetReviewed}
            setIgnoreClick={setIgnoreClick}
            isPlayingBack={setPlayback}
@ -256,7 +264,13 @@ export default function PreviewThumbnailPlayer({
            <div className="absolute top-0 inset-x-0 rounded-t-l z-10 w-full h-[30%] bg-gradient-to-b from-black/60 to-transparent pointer-events-none"></div>
            <div className="absolute bottom-0 inset-x-0 rounded-b-l z-10 w-full h-[20%] bg-gradient-to-t from-black/60 to-transparent pointer-events-none">
              <div className="flex h-full justify-between items-end mx-3 pb-1 text-white text-sm">
-                <TimeAgo time={review.start_time * 1000} dense />
+                {review.end_time ? (
+                  <TimeAgo time={review.start_time * 1000} dense />
+                ) : (
+                  <div>
+                    <ActivityIndicator size={24} />
+                  </div>
+                )}
                {formattedDate}
              </div>
            </div>
@ -270,6 +284,7 @@ export default function PreviewThumbnailPlayer({
 type PreviewContentProps = {
  review: ReviewSegment;
  relevantPreview: Preview | undefined;
+  timeRange: TimeRange;
  setReviewed: () => void;
  setIgnoreClick: (ignore: boolean) => void;
  isPlayingBack: (ended: boolean) => void;
@ -278,6 +293,7 @@ type PreviewContentProps = {
 function PreviewContent({
  review,
  relevantPreview,
+  timeRange,
  setReviewed,
  setIgnoreClick,
  isPlayingBack,
@ -288,8 +304,9 @@ function PreviewContent({
  if (relevantPreview) {
    return (
      <VideoPreview
-        review={review}
        relevantPreview={relevantPreview}
+        startTime={review.start_time}
+        endTime={review.end_time}
        setReviewed={setReviewed}
        setIgnoreClick={setIgnoreClick}
        isPlayingBack={isPlayingBack}
@ -300,6 +317,7 @@ function PreviewContent({
    return (
      <InProgressPreview
        review={review}
+        timeRange={timeRange}
        setReviewed={setReviewed}
        setIgnoreClick={setIgnoreClick}
        isPlayingBack={isPlayingBack}
@ -311,16 +329,18 @@ function PreviewContent({

 const PREVIEW_PADDING = 16;
 type VideoPreviewProps = {
-  review: ReviewSegment;
  relevantPreview: Preview;
+  startTime: number;
+  endTime?: number;
  setReviewed: () => void;
  setIgnoreClick: (ignore: boolean) => void;
  isPlayingBack: (ended: boolean) => void;
  onTimeUpdate?: (time: number | undefined) => void;
 };
 function VideoPreview({
-  review,
  relevantPreview,
+  startTime,
+  endTime,
  setReviewed,
  setIgnoreClick,
  isPlayingBack,
@ -339,16 +359,13 @@ function VideoPreview({
    }

    // start with a bit of padding
-    return Math.max(
-      0,
-      review.start_time - relevantPreview.start - PREVIEW_PADDING,
-    );
+    return Math.max(0, startTime - relevantPreview.start - PREVIEW_PADDING);

    // we know that these deps are correct
    // eslint-disable-next-line react-hooks/exhaustive-deps
  }, []);
  const playerDuration = useMemo(
-    () => review.end_time - review.start_time + PREVIEW_PADDING,
+    () => (endTime ?? relevantPreview.end) - startTime + PREVIEW_PADDING,
    // we know that these deps are correct
    // eslint-disable-next-line react-hooks/exhaustive-deps
    [],
@ -389,21 +406,14 @@ function VideoPreview({
    // end with a bit of padding
    const playerPercent = (playerProgress / playerDuration) * 100;

-    if (
-      setReviewed &&
-      !review.has_been_reviewed &&
-      lastPercent < 50 &&
-      playerPercent > 50
-    ) {
+    if (setReviewed && lastPercent < 50 && playerPercent > 50) {
      setReviewed();
    }

    setLastPercent(playerPercent);

    if (playerPercent > 100) {
-      if (!review.has_been_reviewed) {
-        setReviewed();
-      }
+      setReviewed();

      if (isMobile) {
        isPlayingBack(false);
@ -468,7 +478,7 @@ function VideoPreview({
        setIgnoreClick(true);
      }

-      if (setReviewed && !review.has_been_reviewed) {
+      if (setReviewed) {
        setReviewed();
      }

@ -551,6 +561,7 @@ function VideoPreview({
 const MIN_LOAD_TIMEOUT_MS = 200;
 type InProgressPreviewProps = {
  review: ReviewSegment;
+  timeRange: TimeRange;
  setReviewed: (reviewId: string) => void;
  setIgnoreClick: (ignore: boolean) => void;
  isPlayingBack: (ended: boolean) => void;
@ -558,6 +569,7 @@ type InProgressPreviewProps = {
 };
 function InProgressPreview({
  review,
+  timeRange,
  setReviewed,
  setIgnoreClick,
  isPlayingBack,
@ -567,7 +579,7 @@ function InProgressPreview({
  const sliderRef = useRef<HTMLDivElement | null>(null);
  const { data: previewFrames } = useSWR<string[]>(
    `preview/${review.camera}/start/${Math.floor(review.start_time) - PREVIEW_PADDING}/end/${
-      Math.ceil(review.end_time) + PREVIEW_PADDING
+      Math.ceil(review.end_time ?? timeRange.before) + PREVIEW_PADDING
    }/frames`,
    { revalidateOnFocus: false },
  );
--- a/web/src/components/timeline/MotionReviewTimeline.tsx
+++ b/web/src/components/timeline/MotionReviewTimeline.tsx
@ -100,8 +100,10 @@ export function MotionReviewTimeline({
      const overlappingReviewItems = events.some(
        (item) =>
          (item.start_time >= motionStart && item.start_time < motionEnd) ||
-          (item.end_time > motionStart && item.end_time <= motionEnd) ||
-          (item.start_time <= motionStart && item.end_time >= motionEnd),
+          ((item.end_time ?? timelineStart) > motionStart &&
+            (item.end_time ?? timelineStart) <= motionEnd) ||
+          (item.start_time <= motionStart &&
+            (item.end_time ?? timelineStart) >= motionEnd),
      );

      if ((!segmentMotion || overlappingReviewItems) && motionOnly) {
--- a/web/src/components/timeline/ReviewTimeline.tsx
+++ b/web/src/components/timeline/ReviewTimeline.tsx
@ -132,7 +132,6 @@ export function ReviewTimeline({
    draggableElementTime: exportStartTime,
    draggableElementLatestTime: paddedExportEndTime,
    setDraggableElementTime: setExportStartTime,
-    alignSetTimeToSegment: true,
    timelineDuration,
    timelineStartAligned,
    isDragging: isDraggingExportStart,
@ -157,7 +156,6 @@ export function ReviewTimeline({
    draggableElementTime: exportEndTime,
    draggableElementEarliestTime: paddedExportStartTime,
    setDraggableElementTime: setExportEndTime,
-    alignSetTimeToSegment: true,
    timelineDuration,
    timelineStartAligned,
    isDragging: isDraggingExportEnd,
--- a/web/src/hooks/use-camera-activity.ts
+++ b/web/src/hooks/use-camera-activity.ts
@ -107,8 +107,10 @@ export function useCameraMotionNextTimestamp(
      const overlappingReviewItems = reviewItems.some(
        (item) =>
          (item.start_time >= motionStart && item.start_time < motionEnd) ||
-          (item.end_time > motionStart && item.end_time <= motionEnd) ||
-          (item.start_time <= motionStart && item.end_time >= motionEnd),
+          ((item.end_time ?? Date.now() / 1000) > motionStart &&
+            (item.end_time ?? Date.now() / 1000) <= motionEnd) ||
+          (item.start_time <= motionStart &&
+            (item.end_time ?? Date.now() / 1000) >= motionEnd),
      );

      if (!segmentMotion || overlappingReviewItems) {
--- a/web/src/pages/Events.tsx
+++ b/web/src/pages/Events.tsx
@ -204,7 +204,7 @@ export default function Events() {
          const newData = [...data];

          newData.forEach((seg) => {
-            if (seg.severity == severity) {
+            if (seg.end_time && seg.severity == severity) {
              seg.has_been_reviewed = true;
            }
          });
@ -214,10 +214,16 @@ export default function Events() {
        { revalidate: false, populateCache: true },
      );

-      await axios.post(`reviews/viewed`, {
-        ids: currentItems?.map((seg) => seg.id),
-      });
-      reloadData();
+      const itemsToMarkReviewed = currentItems
+        ?.filter((seg) => seg.end_time)
+        ?.map((seg) => seg.id);
+
+      if (itemsToMarkReviewed.length > 0) {
+        await axios.post(`reviews/viewed`, {
+          ids: itemsToMarkReviewed,
+        });
+        reloadData();
+      }
    },
    [reloadData, updateSegments],
  );
--- a/web/src/types/review.ts
+++ b/web/src/types/review.ts
@ -3,7 +3,7 @@ export interface ReviewSegment {
  camera: string;
  severity: ReviewSeverity;
  start_time: number;
-  end_time: number;
+  end_time?: number;
  thumb_path: string;
  has_been_reviewed: boolean;
  data: ReviewData;
--- a/web/src/views/events/EventView.tsx
+++ b/web/src/views/events/EventView.tsx
@ -43,6 +43,7 @@ import { TimeRange } from "@/types/timeline";
 import { useCameraMotionNextTimestamp } from "@/hooks/use-camera-activity";
 import useOptimisticState from "@/hooks/use-optimistic-state";
 import { Skeleton } from "@/components/ui/skeleton";
+import scrollIntoView from "scroll-into-view-if-needed";

 type EventViewProps = {
  reviews?: ReviewSegment[];
@ -293,6 +294,7 @@ export default function EventView({
            severity={severity}
            filter={filter}
            timeRange={timeRange}
+            startTime={startTime}
            markItemAsReviewed={markItemAsReviewed}
            markAllItemsAsReviewed={markAllItemsAsReviewed}
            onSelectReview={onSelectReview}
@ -331,6 +333,7 @@ type DetectionReviewProps = {
  severity: ReviewSeverity;
  filter?: ReviewFilter;
  timeRange: { before: number; after: number };
+  startTime?: number;
  markItemAsReviewed: (review: ReviewSegment) => void;
  markAllItemsAsReviewed: (currentItems: ReviewSegment[]) => void;
  onSelectReview: (review: ReviewSegment, ctrl: boolean) => void;
@ -345,6 +348,7 @@ function DetectionReview({
  severity,
  filter,
  timeRange,
+  startTime,
  markItemAsReviewed,
  markAllItemsAsReviewed,
  onSelectReview,
@ -495,6 +499,26 @@ function DetectionReview({
    [minimap],
  );

+  // existing review item
+
+  useEffect(() => {
+    if (!startTime || !currentItems || currentItems.length == 0) {
+      return;
+    }
+
+    const element = contentRef.current?.querySelector(
+      `[data-start="${startTime}"]`,
+    );
+    if (element) {
+      scrollIntoView(element, {
+        scrollMode: "if-needed",
+        behavior: "smooth",
+      });
+    }
+    // only run when start time changes
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [startTime]);
+
  return (
    <>
      <div
@ -546,6 +570,7 @@ function DetectionReview({
                    <PreviewThumbnailPlayer
                      review={value}
                      allPreviews={relevantPreviews}
+                      timeRange={timeRange}
                      setReviewed={markItemAsReviewed}
                      scrollLock={scrollLock}
                      onTimeUpdate={onPreviewTimeUpdate}
@ -787,16 +812,23 @@ function MotionReview({
      } else {
        const segmentStartTime = alignStartDateToTimeline(currentTime);
        const segmentEndTime = segmentStartTime + segmentDuration;
-        const matchingItem = reviewItems?.all.find(
-          (item) =>
+        const matchingItem = reviewItems?.all.find((item) => {
+          const endTime = item.end_time ?? timeRange.before;
+
+          return (
            ((item.start_time >= segmentStartTime &&
              item.start_time < segmentEndTime) ||
-              (item.end_time > segmentStartTime &&
-                item.end_time <= segmentEndTime) ||
+              (endTime > segmentStartTime && endTime <= segmentEndTime) ||
              (item.start_time <= segmentStartTime &&
-                item.end_time >= segmentEndTime)) &&
-            item.camera === cameraName,
-        );
+                endTime >= segmentEndTime)) &&
+            item.camera === cameraName
+          );
+            item.start_time < segmentEndTime) ||
+            (endTime > segmentStartTime && endTime <= segmentEndTime) ||
+            (item.start_time <= segmentStartTime &&
+              endTime >= segmentEndTime)) &&
+            item.camera === cameraName;
+        });

        return matchingItem ? matchingItem.severity : null;
      }
@ -805,6 +837,7 @@ function MotionReview({
      reviewItems,
      motionData,
      currentTime,
+      timeRange,
      motionOnly,
      alignStartDateToTimeline,
    ],
--- a/web/src/views/live/LiveDashboardView.tsx
+++ b/web/src/views/live/LiveDashboardView.tsx
@ -47,8 +47,8 @@ export default function LiveDashboardView({
    }

    // if event is ended and was saved, update events list
-    if (eventUpdate.type == "end" && eventUpdate.review.severity == "alert") {
-      setTimeout(() => updateEvents(), 1000);
+    if (eventUpdate.review.severity == "alert") {
+      setTimeout(() => updateEvents(), eventUpdate.type == "end" ? 1000 : 6000);
      return;
    }
  }, [eventUpdate, updateEvents]);