Improve review segmentation behavior (#19850)

* Refactor active objects to class

* Keep segment going when detection is newer than end of alert

* Cleanup logic

* Fix

* Cleanup ending

* Adjust timing

* Improve detection saving

* Don't have padding at end for in progress reviews

* Add review config for cutoff times

docs/docs/configuration/reference.md

@@ -390,6 +390,8 @@ review:
     labels:
       - car
       - person
+    # Time to cutoff alerts after no alert-causing activity has occurred (default: shown below)
+    cutoff_time: 40
     # Optional: required zones for an object to be marked as an alert (default: none)
     # NOTE: when settings required zones globally, this zone must exist on all cameras
     #       or the config will be considered invalid. In that case the required_zones
@@ -404,6 +406,8 @@ review:
     labels:
       - car
       - person
+    # Time to cutoff detections after no detection-causing activity has occurred (default: shown below)
+    cutoff_time: 30
     # Optional: required zones for an object to be marked as a detection (default: none)
     # NOTE: when settings required zones globally, this zone must exist on all cameras
     #       or the config will be considered invalid. In that case the required_zones

frigate/config/camera/review.py

@@ -26,6 +26,10 @@ class AlertsConfig(FrigateBaseModel):
     enabled_in_config: Optional[bool] = Field(
         default=None, title="Keep track of original state of alerts."
     )
+    cutoff_time: int = Field(
+        default=40,
+        title="Time to cutoff alerts after no alert-causing activity has occurred.",
+    )
 
     @field_validator("required_zones", mode="before")
     @classmethod
@@ -48,6 +52,10 @@ class DetectionsConfig(FrigateBaseModel):
         default_factory=list,
         title="List of required zones to be entered in order to save the event as a detection.",
     )
+    cutoff_time: int = Field(
+        default=30,
+        title="Time to cutoff detection after no detection-causing activity has occurred.",
+    )
 
     enabled_in_config: Optional[bool] = Field(
         default=None, title="Keep track of original state of detections."

frigate/review/maintainer.py

@@ -40,9 +40,6 @@ logger = logging.getLogger(__name__)
 THUMB_HEIGHT = 180
 THUMB_WIDTH = 320
 
-THRESHOLD_ALERT_ACTIVITY = 120
-THRESHOLD_DETECTION_ACTIVITY = 30
-
 
 class PendingReviewSegment:
     def __init__(
@@ -64,8 +61,12 @@ class PendingReviewSegment:
         self.sub_labels = sub_labels
         self.zones = zones
         self.audio = audio
-        self.last_update = frame_time
         self.thumb_time: float | None = None
+        self.last_alert_time: float | None = None
+        self.last_detection_time: float = frame_time
+
+        if severity == SeverityEnum.alert:
+            self.last_alert_time = frame_time
 
         # thumbnail
         self._frame = np.zeros((THUMB_HEIGHT * 3 // 2, THUMB_WIDTH), np.uint8)
@@ -127,12 +128,20 @@ class PendingReviewSegment:
             )
 
     def get_data(self, ended: bool) -> dict:
+        end_time = None
+
+        if ended:
+            if self.severity == SeverityEnum.alert:
+                end_time = self.last_alert_time
+            else:
+                end_time = self.last_detection_time
+
         return copy.deepcopy(
             {
                 ReviewSegment.id.name: self.id,
                 ReviewSegment.camera.name: self.camera,
                 ReviewSegment.start_time.name: self.start_time,
-                ReviewSegment.end_time.name: self.last_update if ended else None,
+                ReviewSegment.end_time.name: end_time,
                 ReviewSegment.severity.name: self.severity.value,
                 ReviewSegment.thumb_path.name: self.frame_path,
                 ReviewSegment.data.name: {
@@ -148,6 +157,102 @@ class PendingReviewSegment:
         )
 
 
+class ActiveObjects:
+    def __init__(
+        self,
+        frame_time: float,
+        camera_config: CameraConfig,
+        all_objects: list[TrackedObject],
+    ):
+        self.camera_config = camera_config
+
+        # get current categorization of objects to know if
+        # these objects are currently being categorized
+        self.categorized_objects = {
+            "alerts": [],
+            "detections": [],
+        }
+
+        for o in all_objects:
+            if (
+                o["motionless_count"] >= camera_config.detect.stationary.threshold
+                and not o["pending_loitering"]
+            ):
+                # no stationary objects unless loitering
+                continue
+
+            if o["position_changes"] == 0:
+                # object must have moved at least once
+                continue
+
+            if o["frame_time"] != frame_time:
+                # object must be detected in this frame
+                continue
+
+            if o["false_positive"]:
+                # object must not be a false positive
+                continue
+
+            if (
+                o["label"] in camera_config.review.alerts.labels
+                and (
+                    not camera_config.review.alerts.required_zones
+                    or (
+                        len(o["current_zones"]) > 0
+                        and set(o["current_zones"])
+                        & set(camera_config.review.alerts.required_zones)
+                    )
+                )
+                and camera_config.review.alerts.enabled
+            ):
+                self.categorized_objects["alerts"].append(o)
+                continue
+
+            if (
+                (
+                    camera_config.review.detections.labels is None
+                    or o["label"] in camera_config.review.detections.labels
+                )
+                and (
+                    not camera_config.review.detections.required_zones
+                    or (
+                        len(o["current_zones"]) > 0
+                        and set(o["current_zones"])
+                        & set(camera_config.review.detections.required_zones)
+                    )
+                )
+                and camera_config.review.detections.enabled
+            ):
+                self.categorized_objects["detections"].append(o)
+                continue
+
+    def has_active_objects(self) -> bool:
+        return (
+            len(self.categorized_objects["alerts"]) > 0
+            or len(self.categorized_objects["detections"]) > 0
+        )
+
+    def has_activity_category(self, severity: SeverityEnum) -> bool:
+        if (
+            severity == SeverityEnum.alert
+            and len(self.categorized_objects["alerts"]) > 0
+        ):
+            return True
+
+        if (
+            severity == SeverityEnum.detection
+            and len(self.categorized_objects["detections"]) > 0
+        ):
+            return True
+
+        return False
+
+    def get_all_objects(self) -> list[TrackedObject]:
+        return (
+            self.categorized_objects["alerts"] + self.categorized_objects["detections"]
+        )
+
+
 class ReviewSegmentMaintainer(threading.Thread):
     """Maintain review segments."""
 
@@ -238,9 +343,10 @@ class ReviewSegmentMaintainer(threading.Thread):
         self,
         segment: PendingReviewSegment,
         prev_data: dict[str, Any],
-    ) -> None:
+    ) -> float:
         """End segment."""
         final_data = segment.get_data(ended=True)
+        end_time = final_data[ReviewSegment.end_time.name]
         self.requestor.send_data(UPSERT_REVIEW_SEGMENT, final_data)
         review_update = {
             "type": "end",
@@ -254,13 +360,14 @@ class ReviewSegmentMaintainer(threading.Thread):
         self.review_publisher.publish(review_update, segment.camera)
         self.requestor.send_data(f"{segment.camera}/review_status", "NONE")
         self.active_review_segments[segment.camera] = None
+        return end_time
 
-    def end_segment(self, camera: str) -> None:
+    def forcibly_end_segment(self, camera: str) -> float:
-        """End the pending segment for a camera."""
+        """Forcibly end the pending segment for a camera."""
         segment = self.active_review_segments.get(camera)
         if segment:
             prev_data = segment.get_data(False)
-            self._publish_segment_end(segment, prev_data)
+            return self._publish_segment_end(segment, prev_data)
 
     def update_existing_segment(
         self,
@@ -273,21 +380,30 @@ class ReviewSegmentMaintainer(threading.Thread):
         camera_config = self.config.cameras[segment.camera]
 
         # get active objects + objects loitering in loitering zones
-        active_objects = get_active_objects(
+        activity = ActiveObjects(frame_time, camera_config, objects)
-            frame_time, camera_config, objects
-        ) + get_loitering_objects(frame_time, camera_config, objects)
         prev_data = segment.get_data(False)
         has_activity = False
 
-        if len(active_objects) > 0:
+        if activity.has_active_objects():
             has_activity = True
             should_update_image = False
             should_update_state = False
 
-            if frame_time > segment.last_update:
+            if activity.has_activity_category(SeverityEnum.alert):
-                segment.last_update = frame_time
+                # update current time for last alert activity
+                segment.last_alert_time = frame_time
 
-            for object in active_objects:
+                if segment.severity != SeverityEnum.alert:
+                    # if segment is not alert category but current activity is
+                    # update this segment to be an alert
+                    segment.severity = SeverityEnum.alert
+                    should_update_state = True
+                    should_update_image = True
+
+            if activity.has_activity_category(SeverityEnum.detection):
+                segment.last_detection_time = frame_time
+
+            for object in activity.get_all_objects():
                 if not object["sub_label"]:
                     segment.detections[object["id"]] = object["label"]
                 elif object["sub_label"][0] in self.config.model.all_attributes:
@@ -296,33 +412,13 @@ class ReviewSegmentMaintainer(threading.Thread):
                     segment.detections[object["id"]] = f"{object['label']}-verified"
                     segment.sub_labels[object["id"]] = object["sub_label"][0]
 
-                # if object is alert label
-                # and has entered required zones or required zones is not set
-                # mark this review as alert
-                if (
-                    segment.severity != SeverityEnum.alert
-                    and object["label"] in camera_config.review.alerts.labels
-                    and (
-                        not camera_config.review.alerts.required_zones
-                        or (
-                            len(object["current_zones"]) > 0
-                            and set(object["current_zones"])
-                            & set(camera_config.review.alerts.required_zones)
-                        )
-                    )
-                    and camera_config.review.alerts.enabled
-                ):
-                    segment.severity = SeverityEnum.alert
-                    should_update_state = True
-                    should_update_image = True
-
                 # keep zones up to date
                 if len(object["current_zones"]) > 0:
                     for zone in object["current_zones"]:
                         if zone not in segment.zones:
                             segment.zones.append(zone)
 
-            if len(active_objects) > segment.frame_active_count:
+            if len(activity.get_all_objects()) > segment.frame_active_count:
                 should_update_state = True
                 should_update_image = True
 
@@ -343,7 +439,11 @@ class ReviewSegmentMaintainer(threading.Thread):
                         yuv_frame = None
 
                     self._publish_segment_update(
-                        segment, camera_config, yuv_frame, active_objects, prev_data
+                        segment,
+                        camera_config,
+                        yuv_frame,
+                        activity.get_all_objects(),
+                        prev_data,
                     )
                     self.frame_manager.close(frame_name)
                 except FileNotFoundError:
@@ -369,10 +469,50 @@ class ReviewSegmentMaintainer(threading.Thread):
                     return
 
             if segment.severity == SeverityEnum.alert and frame_time > (
-                segment.last_update + THRESHOLD_ALERT_ACTIVITY
+                segment.last_alert_time + camera_config.review.alerts.cutoff_time
+            ):
+                needs_new_detection = (
+                    segment.last_detection_time > segment.last_alert_time
+                    and (
+                        segment.last_detection_time
+                        + camera_config.review.detections.cutoff_time
+                    )
+                    > frame_time
+                )
+                last_detection_time = segment.last_detection_time
+
+                end_time = self._publish_segment_end(segment, prev_data)
+
+                if needs_new_detection:
+                    new_detections: dict[str, str] = {}
+                    new_zones = set()
+
+                    for o in activity.categorized_objects["detections"]:
+                        new_detections[o["id"]] = o["label"]
+                        new_zones.update(o["current_zones"])
+
+                    if new_detections:
+                        self.active_review_segments[activity.camera_config.name] = (
+                            PendingReviewSegment(
+                                activity.camera_config.name,
+                                end_time,
+                                SeverityEnum.detection,
+                                new_detections,
+                                sub_labels={},
+                                audio=set(),
+                                zones=list(new_zones),
+                            )
+                        )
+                        self._publish_segment_start(
+                            self.active_review_segments[activity.camera_config.name]
+                        )
+                        self.active_review_segments[
+                            activity.camera_config.name
+                        ].last_detection_time = last_detection_time
+            elif segment.severity == SeverityEnum.detection and frame_time > (
+                segment.last_detection_time
+                + camera_config.review.detections.cutoff_time
             ):
-                self._publish_segment_end(segment, prev_data)
-            elif frame_time > (segment.last_update + THRESHOLD_DETECTION_ACTIVITY):
                 self._publish_segment_end(segment, prev_data)
 
     def check_if_new_segment(
@@ -384,15 +524,26 @@ class ReviewSegmentMaintainer(threading.Thread):
     ) -> None:
         """Check if a new review segment should be created."""
         camera_config = self.config.cameras[camera]
-        active_objects = get_active_objects(frame_time, camera_config, objects)
+        activity = ActiveObjects(frame_time, camera_config, objects)
 
-        if len(active_objects) > 0:
+        if activity.has_active_objects():
             detections: dict[str, str] = {}
             sub_labels: dict[str, str] = {}
             zones: list[str] = []
-            severity = None
+            severity: SeverityEnum | None = None
 
-            for object in active_objects:
+            # if activity is alert category mark this review as alert
+            if severity != SeverityEnum.alert and activity.has_activity_category(
+                SeverityEnum.alert
+            ):
+                severity = SeverityEnum.alert
+
+            # if object is detection label and not already higher severity
+            # mark this review as detection
+            if not severity and activity.has_activity_category(SeverityEnum.detection):
+                severity = SeverityEnum.detection
+
+            for object in activity.get_all_objects():
                 if not object["sub_label"]:
                     detections[object["id"]] = object["label"]
                 elif object["sub_label"][0] in self.config.model.all_attributes:
@@ -401,46 +552,6 @@ class ReviewSegmentMaintainer(threading.Thread):
                     detections[object["id"]] = f"{object['label']}-verified"
                     sub_labels[object["id"]] = object["sub_label"][0]
 
-                # if object is alert label
-                # and has entered required zones or required zones is not set
-                # mark this review as alert
-                if (
-                    severity != SeverityEnum.alert
-                    and object["label"] in camera_config.review.alerts.labels
-                    and (
-                        not camera_config.review.alerts.required_zones
-                        or (
-                            len(object["current_zones"]) > 0
-                            and set(object["current_zones"])
-                            & set(camera_config.review.alerts.required_zones)
-                        )
-                    )
-                    and camera_config.review.alerts.enabled
-                ):
-                    severity = SeverityEnum.alert
-
-                # if object is detection label
-                # and review is not already a detection or alert
-                # and has entered required zones or required zones is not set
-                # mark this review as detection
-                if (
-                    not severity
-                    and (
-                        camera_config.review.detections.labels is None
-                        or object["label"] in (camera_config.review.detections.labels)
-                    )
-                    and (
-                        not camera_config.review.detections.required_zones
-                        or (
-                            len(object["current_zones"]) > 0
-                            and set(object["current_zones"])
-                            & set(camera_config.review.detections.required_zones)
-                        )
-                    )
-                    and camera_config.review.detections.enabled
-                ):
-                    severity = SeverityEnum.detection
-
                 for zone in object["current_zones"]:
                     if zone not in zones:
                         zones.append(zone)
@@ -466,7 +577,7 @@ class ReviewSegmentMaintainer(threading.Thread):
                         return
 
                     self.active_review_segments[camera].update_frame(
-                        camera_config, yuv_frame, active_objects
+                        camera_config, yuv_frame, activity.get_all_objects()
                     )
                     self.frame_manager.close(frame_name)
                     self._publish_segment_start(self.active_review_segments[camera])
@@ -480,11 +591,11 @@ class ReviewSegmentMaintainer(threading.Thread):
 
             if "record" in updated_topics:
                 for camera in updated_topics["record"]:
-                    self.end_segment(camera)
+                    self.forcibly_end_segment(camera)
 
             if "enabled" in updated_topics:
                 for camera in updated_topics["enabled"]:
-                    self.end_segment(camera)
+                    self.forcibly_end_segment(camera)
 
             (topic, data) = self.detection_subscriber.check_for_update(timeout=1)
 
@@ -534,7 +645,7 @@ class ReviewSegmentMaintainer(threading.Thread):
                     current_segment.severity == SeverityEnum.detection
                     and not self.config.cameras[camera].review.detections.enabled
                 ):
-                    self.end_segment(camera)
+                    self.forcibly_end_segment(camera)
                     continue
 
             # If we reach here, the segment can be processed (if it exists)
@@ -549,9 +660,6 @@ class ReviewSegmentMaintainer(threading.Thread):
                 elif topic == DetectionTypeEnum.audio and len(audio_detections) > 0:
                     camera_config = self.config.cameras[camera]
 
-                    if frame_time > current_segment.last_update:
-                        current_segment.last_update = frame_time
-
                     for audio in audio_detections:
                         if (
                             audio in camera_config.review.alerts.labels
@@ -559,11 +667,13 @@ class ReviewSegmentMaintainer(threading.Thread):
                         ):
                             current_segment.audio.add(audio)
                             current_segment.severity = SeverityEnum.alert
+                            current_segment.last_alert_time = frame_time
                         elif (
                             camera_config.review.detections.labels is None
                             or audio in camera_config.review.detections.labels
                         ) and camera_config.review.detections.enabled:
                             current_segment.audio.add(audio)
+                            current_segment.last_detection_time = frame_time
                 elif topic == DetectionTypeEnum.api or topic == DetectionTypeEnum.lpr:
                     if manual_info["state"] == ManualEventState.complete:
                         current_segment.detections[manual_info["event_id"]] = (
@@ -579,7 +689,7 @@ class ReviewSegmentMaintainer(threading.Thread):
                             and self.config.cameras[camera].review.detections.enabled
                         ):
                             current_segment.severity = SeverityEnum.detection
-                        current_segment.last_update = manual_info["end_time"]
+                        current_segment.last_alert_time = manual_info["end_time"]
                     elif manual_info["state"] == ManualEventState.start:
                         self.indefinite_events[camera][manual_info["event_id"]] = (
                             manual_info["label"]
@@ -599,7 +709,8 @@ class ReviewSegmentMaintainer(threading.Thread):
                             current_segment.severity = SeverityEnum.detection
 
                         # temporarily make it so this event can not end
-                        current_segment.last_update = sys.maxsize
+                        current_segment.last_alert_time = sys.maxsize
+                        current_segment.last_detection_time = sys.maxsize
                     elif manual_info["state"] == ManualEventState.end:
                         event_id = manual_info["event_id"]
 
@@ -607,7 +718,12 @@ class ReviewSegmentMaintainer(threading.Thread):
                             self.indefinite_events[camera].pop(event_id)
 
                             if len(self.indefinite_events[camera]) == 0:
-                                current_segment.last_update = manual_info["end_time"]
+                                current_segment.last_alert_time = manual_info[
+                                    "end_time"
+                                ]
+                                current_segment.last_detection_time = manual_info[
+                                    "end_time"
+                                ]
                         else:
                             logger.error(
                                 f"Event with ID {event_id} has a set duration and can not be ended manually."
@@ -675,11 +791,17 @@ class ReviewSegmentMaintainer(threading.Thread):
                             # temporarily make it so this event can not end
                             self.active_review_segments[
                                 camera
-                            ].last_update = sys.maxsize
+                            ].last_alert_time = sys.maxsize
+                            self.active_review_segments[
+                                camera
+                            ].last_detection_time = sys.maxsize
                         elif manual_info["state"] == ManualEventState.complete:
                             self.active_review_segments[
                                 camera
-                            ].last_update = manual_info["end_time"]
+                            ].last_alert_time = manual_info["end_time"]
+                            self.active_review_segments[
+                                camera
+                            ].last_detection_time = manual_info["end_time"]
                     else:
                         logger.warning(
                             f"Manual event API has been called for {camera}, but alerts are disabled. This manual event will not appear as an alert."
@@ -703,11 +825,17 @@ class ReviewSegmentMaintainer(threading.Thread):
                             # temporarily make it so this event can not end
                             self.active_review_segments[
                                 camera
-                            ].last_update = sys.maxsize
+                            ].last_alert_time = sys.maxsize
+                            self.active_review_segments[
+                                camera
+                            ].last_detection_time = sys.maxsize
                         elif manual_info["state"] == ManualEventState.complete:
                             self.active_review_segments[
                                 camera
-                            ].last_update = manual_info["end_time"]
+                            ].last_alert_time = manual_info["end_time"]
+                            self.active_review_segments[
+                                camera
+                            ].last_detection_time = manual_info["end_time"]
                     else:
                         logger.warning(
                             f"Dedicated LPR camera API has been called for {camera}, but detections are disabled. LPR events will not appear as a detection."
@@ -717,46 +845,3 @@ class ReviewSegmentMaintainer(threading.Thread):
         self.requestor.stop()
         self.detection_subscriber.stop()
         logger.info("Exiting review maintainer...")
-
-
-def get_active_objects(
-    frame_time: float, camera_config: CameraConfig, all_objects: list[TrackedObject]
-) -> list[TrackedObject]:
-    """get active objects for detection."""
-    return [
-        o
-        for o in all_objects
-        if o["motionless_count"]
-        < camera_config.detect.stationary.threshold  # no stationary objects
-        and o["position_changes"] > 0  # object must have moved at least once
-        and o["frame_time"] == frame_time  # object must be detected in this frame
-        and not o["false_positive"]  # object must not be a false positive
-        and (
-            o["label"] in camera_config.review.alerts.labels
-            or (
-                camera_config.review.detections.labels is None
-                or o["label"] in camera_config.review.detections.labels
-            )
-        )  # object must be in the alerts or detections label list
-    ]
-
-
-def get_loitering_objects(
-    frame_time: float, camera_config: CameraConfig, all_objects: list[TrackedObject]
-) -> list[TrackedObject]:
-    """get loitering objects for detection."""
-    return [
-        o
-        for o in all_objects
-        if o["pending_loitering"]  # object must be pending loitering
-        and o["position_changes"] > 0  # object must have moved at least once
-        and o["frame_time"] == frame_time  # object must be detected in this frame
-        and not o["false_positive"]  # object must not be a false positive
-        and (
-            o["label"] in camera_config.review.alerts.labels
-            or (
-                camera_config.review.detections.labels is None
-                or o["label"] in camera_config.review.detections.labels
-            )
-        )  # object must be in the alerts or detections label list
-    ]

web/src/components/preview/ScrubbablePreview.tsx

@@ -315,9 +315,9 @@ export function InProgressPreview({
   const apiHost = useApiHost();
   const sliderRef = useRef<HTMLDivElement | null>(null);
   const { data: previewFrames } = useSWR<string[]>(
-    `preview/${camera}/start/${Math.floor(startTime) - PREVIEW_PADDING}/end/${
+    `preview/${camera}/start/${Math.floor(startTime) - PREVIEW_PADDING}/end/${Math.ceil(
-      Math.ceil(endTime ?? timeRange.before) + PREVIEW_PADDING
+      endTime ?? timeRange.before,
-    }/frames`,
+    )}/frames`,
     { revalidateOnFocus: false },
   );