From 3a1e1d0841100de49177cba23a75472ffd11c9d4 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Sun, 31 Aug 2025 15:36:12 -0600 Subject: [PATCH] Improve review segmentation behavior (#19850) * Refactor active objects to class * Keep segment going when detection is newer than end of alert * Cleanup logic * Fix * Cleanup ending * Adjust timing * Improve detection saving * Don't have padding at end for in progress reviews * Add review config for cutoff times --- docs/docs/configuration/reference.md | 4 + frigate/config/camera/review.py | 8 + frigate/review/maintainer.py | 369 +++++++++++------- .../components/preview/ScrubbablePreview.tsx | 6 +- 4 files changed, 242 insertions(+), 145 deletions(-) diff --git a/docs/docs/configuration/reference.md b/docs/docs/configuration/reference.md index 9b2127b83..068f25d78 100644 --- a/docs/docs/configuration/reference.md +++ b/docs/docs/configuration/reference.md @@ -390,6 +390,8 @@ review: labels: - car - person + # Time to cutoff alerts after no alert-causing activity has occurred (default: shown below) + cutoff_time: 40 # Optional: required zones for an object to be marked as an alert (default: none) # NOTE: when settings required zones globally, this zone must exist on all cameras # or the config will be considered invalid. In that case the required_zones @@ -404,6 +406,8 @@ review: labels: - car - person + # Time to cutoff detections after no detection-causing activity has occurred (default: shown below) + cutoff_time: 30 # Optional: required zones for an object to be marked as a detection (default: none) # NOTE: when settings required zones globally, this zone must exist on all cameras # or the config will be considered invalid. In that case the required_zones diff --git a/frigate/config/camera/review.py b/frigate/config/camera/review.py index 51268339b..17f97efd1 100644 --- a/frigate/config/camera/review.py +++ b/frigate/config/camera/review.py @@ -26,6 +26,10 @@ class AlertsConfig(FrigateBaseModel): enabled_in_config: Optional[bool] = Field( default=None, title="Keep track of original state of alerts." ) + cutoff_time: int = Field( + default=40, + title="Time to cutoff alerts after no alert-causing activity has occurred.", + ) @field_validator("required_zones", mode="before") @classmethod @@ -48,6 +52,10 @@ class DetectionsConfig(FrigateBaseModel): default_factory=list, title="List of required zones to be entered in order to save the event as a detection.", ) + cutoff_time: int = Field( + default=30, + title="Time to cutoff detection after no detection-causing activity has occurred.", + ) enabled_in_config: Optional[bool] = Field( default=None, title="Keep track of original state of detections." diff --git a/frigate/review/maintainer.py b/frigate/review/maintainer.py index b279c9c44..202b9b701 100644 --- a/frigate/review/maintainer.py +++ b/frigate/review/maintainer.py @@ -40,9 +40,6 @@ logger = logging.getLogger(__name__) THUMB_HEIGHT = 180 THUMB_WIDTH = 320 -THRESHOLD_ALERT_ACTIVITY = 120 -THRESHOLD_DETECTION_ACTIVITY = 30 - class PendingReviewSegment: def __init__( @@ -64,8 +61,12 @@ class PendingReviewSegment: self.sub_labels = sub_labels self.zones = zones self.audio = audio - self.last_update = frame_time self.thumb_time: float | None = None + self.last_alert_time: float | None = None + self.last_detection_time: float = frame_time + + if severity == SeverityEnum.alert: + self.last_alert_time = frame_time # thumbnail self._frame = np.zeros((THUMB_HEIGHT * 3 // 2, THUMB_WIDTH), np.uint8) @@ -127,12 +128,20 @@ class PendingReviewSegment: ) def get_data(self, ended: bool) -> dict: + end_time = None + + if ended: + if self.severity == SeverityEnum.alert: + end_time = self.last_alert_time + else: + end_time = self.last_detection_time + return copy.deepcopy( { ReviewSegment.id.name: self.id, ReviewSegment.camera.name: self.camera, ReviewSegment.start_time.name: self.start_time, - ReviewSegment.end_time.name: self.last_update if ended else None, + ReviewSegment.end_time.name: end_time, ReviewSegment.severity.name: self.severity.value, ReviewSegment.thumb_path.name: self.frame_path, ReviewSegment.data.name: { @@ -148,6 +157,102 @@ class PendingReviewSegment: ) +class ActiveObjects: + def __init__( + self, + frame_time: float, + camera_config: CameraConfig, + all_objects: list[TrackedObject], + ): + self.camera_config = camera_config + + # get current categorization of objects to know if + # these objects are currently being categorized + self.categorized_objects = { + "alerts": [], + "detections": [], + } + + for o in all_objects: + if ( + o["motionless_count"] >= camera_config.detect.stationary.threshold + and not o["pending_loitering"] + ): + # no stationary objects unless loitering + continue + + if o["position_changes"] == 0: + # object must have moved at least once + continue + + if o["frame_time"] != frame_time: + # object must be detected in this frame + continue + + if o["false_positive"]: + # object must not be a false positive + continue + + if ( + o["label"] in camera_config.review.alerts.labels + and ( + not camera_config.review.alerts.required_zones + or ( + len(o["current_zones"]) > 0 + and set(o["current_zones"]) + & set(camera_config.review.alerts.required_zones) + ) + ) + and camera_config.review.alerts.enabled + ): + self.categorized_objects["alerts"].append(o) + continue + + if ( + ( + camera_config.review.detections.labels is None + or o["label"] in camera_config.review.detections.labels + ) + and ( + not camera_config.review.detections.required_zones + or ( + len(o["current_zones"]) > 0 + and set(o["current_zones"]) + & set(camera_config.review.detections.required_zones) + ) + ) + and camera_config.review.detections.enabled + ): + self.categorized_objects["detections"].append(o) + continue + + def has_active_objects(self) -> bool: + return ( + len(self.categorized_objects["alerts"]) > 0 + or len(self.categorized_objects["detections"]) > 0 + ) + + def has_activity_category(self, severity: SeverityEnum) -> bool: + if ( + severity == SeverityEnum.alert + and len(self.categorized_objects["alerts"]) > 0 + ): + return True + + if ( + severity == SeverityEnum.detection + and len(self.categorized_objects["detections"]) > 0 + ): + return True + + return False + + def get_all_objects(self) -> list[TrackedObject]: + return ( + self.categorized_objects["alerts"] + self.categorized_objects["detections"] + ) + + class ReviewSegmentMaintainer(threading.Thread): """Maintain review segments.""" @@ -238,9 +343,10 @@ class ReviewSegmentMaintainer(threading.Thread): self, segment: PendingReviewSegment, prev_data: dict[str, Any], - ) -> None: + ) -> float: """End segment.""" final_data = segment.get_data(ended=True) + end_time = final_data[ReviewSegment.end_time.name] self.requestor.send_data(UPSERT_REVIEW_SEGMENT, final_data) review_update = { "type": "end", @@ -254,13 +360,14 @@ class ReviewSegmentMaintainer(threading.Thread): self.review_publisher.publish(review_update, segment.camera) self.requestor.send_data(f"{segment.camera}/review_status", "NONE") self.active_review_segments[segment.camera] = None + return end_time - def end_segment(self, camera: str) -> None: - """End the pending segment for a camera.""" + def forcibly_end_segment(self, camera: str) -> float: + """Forcibly end the pending segment for a camera.""" segment = self.active_review_segments.get(camera) if segment: prev_data = segment.get_data(False) - self._publish_segment_end(segment, prev_data) + return self._publish_segment_end(segment, prev_data) def update_existing_segment( self, @@ -273,21 +380,30 @@ class ReviewSegmentMaintainer(threading.Thread): camera_config = self.config.cameras[segment.camera] # get active objects + objects loitering in loitering zones - active_objects = get_active_objects( - frame_time, camera_config, objects - ) + get_loitering_objects(frame_time, camera_config, objects) + activity = ActiveObjects(frame_time, camera_config, objects) prev_data = segment.get_data(False) has_activity = False - if len(active_objects) > 0: + if activity.has_active_objects(): has_activity = True should_update_image = False should_update_state = False - if frame_time > segment.last_update: - segment.last_update = frame_time + if activity.has_activity_category(SeverityEnum.alert): + # update current time for last alert activity + segment.last_alert_time = frame_time - for object in active_objects: + if segment.severity != SeverityEnum.alert: + # if segment is not alert category but current activity is + # update this segment to be an alert + segment.severity = SeverityEnum.alert + should_update_state = True + should_update_image = True + + if activity.has_activity_category(SeverityEnum.detection): + segment.last_detection_time = frame_time + + for object in activity.get_all_objects(): if not object["sub_label"]: segment.detections[object["id"]] = object["label"] elif object["sub_label"][0] in self.config.model.all_attributes: @@ -296,33 +412,13 @@ class ReviewSegmentMaintainer(threading.Thread): segment.detections[object["id"]] = f"{object['label']}-verified" segment.sub_labels[object["id"]] = object["sub_label"][0] - # if object is alert label - # and has entered required zones or required zones is not set - # mark this review as alert - if ( - segment.severity != SeverityEnum.alert - and object["label"] in camera_config.review.alerts.labels - and ( - not camera_config.review.alerts.required_zones - or ( - len(object["current_zones"]) > 0 - and set(object["current_zones"]) - & set(camera_config.review.alerts.required_zones) - ) - ) - and camera_config.review.alerts.enabled - ): - segment.severity = SeverityEnum.alert - should_update_state = True - should_update_image = True - # keep zones up to date if len(object["current_zones"]) > 0: for zone in object["current_zones"]: if zone not in segment.zones: segment.zones.append(zone) - if len(active_objects) > segment.frame_active_count: + if len(activity.get_all_objects()) > segment.frame_active_count: should_update_state = True should_update_image = True @@ -343,7 +439,11 @@ class ReviewSegmentMaintainer(threading.Thread): yuv_frame = None self._publish_segment_update( - segment, camera_config, yuv_frame, active_objects, prev_data + segment, + camera_config, + yuv_frame, + activity.get_all_objects(), + prev_data, ) self.frame_manager.close(frame_name) except FileNotFoundError: @@ -369,10 +469,50 @@ class ReviewSegmentMaintainer(threading.Thread): return if segment.severity == SeverityEnum.alert and frame_time > ( - segment.last_update + THRESHOLD_ALERT_ACTIVITY + segment.last_alert_time + camera_config.review.alerts.cutoff_time + ): + needs_new_detection = ( + segment.last_detection_time > segment.last_alert_time + and ( + segment.last_detection_time + + camera_config.review.detections.cutoff_time + ) + > frame_time + ) + last_detection_time = segment.last_detection_time + + end_time = self._publish_segment_end(segment, prev_data) + + if needs_new_detection: + new_detections: dict[str, str] = {} + new_zones = set() + + for o in activity.categorized_objects["detections"]: + new_detections[o["id"]] = o["label"] + new_zones.update(o["current_zones"]) + + if new_detections: + self.active_review_segments[activity.camera_config.name] = ( + PendingReviewSegment( + activity.camera_config.name, + end_time, + SeverityEnum.detection, + new_detections, + sub_labels={}, + audio=set(), + zones=list(new_zones), + ) + ) + self._publish_segment_start( + self.active_review_segments[activity.camera_config.name] + ) + self.active_review_segments[ + activity.camera_config.name + ].last_detection_time = last_detection_time + elif segment.severity == SeverityEnum.detection and frame_time > ( + segment.last_detection_time + + camera_config.review.detections.cutoff_time ): - self._publish_segment_end(segment, prev_data) - elif frame_time > (segment.last_update + THRESHOLD_DETECTION_ACTIVITY): self._publish_segment_end(segment, prev_data) def check_if_new_segment( @@ -384,15 +524,26 @@ class ReviewSegmentMaintainer(threading.Thread): ) -> None: """Check if a new review segment should be created.""" camera_config = self.config.cameras[camera] - active_objects = get_active_objects(frame_time, camera_config, objects) + activity = ActiveObjects(frame_time, camera_config, objects) - if len(active_objects) > 0: + if activity.has_active_objects(): detections: dict[str, str] = {} sub_labels: dict[str, str] = {} zones: list[str] = [] - severity = None + severity: SeverityEnum | None = None - for object in active_objects: + # if activity is alert category mark this review as alert + if severity != SeverityEnum.alert and activity.has_activity_category( + SeverityEnum.alert + ): + severity = SeverityEnum.alert + + # if object is detection label and not already higher severity + # mark this review as detection + if not severity and activity.has_activity_category(SeverityEnum.detection): + severity = SeverityEnum.detection + + for object in activity.get_all_objects(): if not object["sub_label"]: detections[object["id"]] = object["label"] elif object["sub_label"][0] in self.config.model.all_attributes: @@ -401,46 +552,6 @@ class ReviewSegmentMaintainer(threading.Thread): detections[object["id"]] = f"{object['label']}-verified" sub_labels[object["id"]] = object["sub_label"][0] - # if object is alert label - # and has entered required zones or required zones is not set - # mark this review as alert - if ( - severity != SeverityEnum.alert - and object["label"] in camera_config.review.alerts.labels - and ( - not camera_config.review.alerts.required_zones - or ( - len(object["current_zones"]) > 0 - and set(object["current_zones"]) - & set(camera_config.review.alerts.required_zones) - ) - ) - and camera_config.review.alerts.enabled - ): - severity = SeverityEnum.alert - - # if object is detection label - # and review is not already a detection or alert - # and has entered required zones or required zones is not set - # mark this review as detection - if ( - not severity - and ( - camera_config.review.detections.labels is None - or object["label"] in (camera_config.review.detections.labels) - ) - and ( - not camera_config.review.detections.required_zones - or ( - len(object["current_zones"]) > 0 - and set(object["current_zones"]) - & set(camera_config.review.detections.required_zones) - ) - ) - and camera_config.review.detections.enabled - ): - severity = SeverityEnum.detection - for zone in object["current_zones"]: if zone not in zones: zones.append(zone) @@ -466,7 +577,7 @@ class ReviewSegmentMaintainer(threading.Thread): return self.active_review_segments[camera].update_frame( - camera_config, yuv_frame, active_objects + camera_config, yuv_frame, activity.get_all_objects() ) self.frame_manager.close(frame_name) self._publish_segment_start(self.active_review_segments[camera]) @@ -480,11 +591,11 @@ class ReviewSegmentMaintainer(threading.Thread): if "record" in updated_topics: for camera in updated_topics["record"]: - self.end_segment(camera) + self.forcibly_end_segment(camera) if "enabled" in updated_topics: for camera in updated_topics["enabled"]: - self.end_segment(camera) + self.forcibly_end_segment(camera) (topic, data) = self.detection_subscriber.check_for_update(timeout=1) @@ -534,7 +645,7 @@ class ReviewSegmentMaintainer(threading.Thread): current_segment.severity == SeverityEnum.detection and not self.config.cameras[camera].review.detections.enabled ): - self.end_segment(camera) + self.forcibly_end_segment(camera) continue # If we reach here, the segment can be processed (if it exists) @@ -549,9 +660,6 @@ class ReviewSegmentMaintainer(threading.Thread): elif topic == DetectionTypeEnum.audio and len(audio_detections) > 0: camera_config = self.config.cameras[camera] - if frame_time > current_segment.last_update: - current_segment.last_update = frame_time - for audio in audio_detections: if ( audio in camera_config.review.alerts.labels @@ -559,11 +667,13 @@ class ReviewSegmentMaintainer(threading.Thread): ): current_segment.audio.add(audio) current_segment.severity = SeverityEnum.alert + current_segment.last_alert_time = frame_time elif ( camera_config.review.detections.labels is None or audio in camera_config.review.detections.labels ) and camera_config.review.detections.enabled: current_segment.audio.add(audio) + current_segment.last_detection_time = frame_time elif topic == DetectionTypeEnum.api or topic == DetectionTypeEnum.lpr: if manual_info["state"] == ManualEventState.complete: current_segment.detections[manual_info["event_id"]] = ( @@ -579,7 +689,7 @@ class ReviewSegmentMaintainer(threading.Thread): and self.config.cameras[camera].review.detections.enabled ): current_segment.severity = SeverityEnum.detection - current_segment.last_update = manual_info["end_time"] + current_segment.last_alert_time = manual_info["end_time"] elif manual_info["state"] == ManualEventState.start: self.indefinite_events[camera][manual_info["event_id"]] = ( manual_info["label"] @@ -599,7 +709,8 @@ class ReviewSegmentMaintainer(threading.Thread): current_segment.severity = SeverityEnum.detection # temporarily make it so this event can not end - current_segment.last_update = sys.maxsize + current_segment.last_alert_time = sys.maxsize + current_segment.last_detection_time = sys.maxsize elif manual_info["state"] == ManualEventState.end: event_id = manual_info["event_id"] @@ -607,7 +718,12 @@ class ReviewSegmentMaintainer(threading.Thread): self.indefinite_events[camera].pop(event_id) if len(self.indefinite_events[camera]) == 0: - current_segment.last_update = manual_info["end_time"] + current_segment.last_alert_time = manual_info[ + "end_time" + ] + current_segment.last_detection_time = manual_info[ + "end_time" + ] else: logger.error( f"Event with ID {event_id} has a set duration and can not be ended manually." @@ -675,11 +791,17 @@ class ReviewSegmentMaintainer(threading.Thread): # temporarily make it so this event can not end self.active_review_segments[ camera - ].last_update = sys.maxsize + ].last_alert_time = sys.maxsize + self.active_review_segments[ + camera + ].last_detection_time = sys.maxsize elif manual_info["state"] == ManualEventState.complete: self.active_review_segments[ camera - ].last_update = manual_info["end_time"] + ].last_alert_time = manual_info["end_time"] + self.active_review_segments[ + camera + ].last_detection_time = manual_info["end_time"] else: logger.warning( f"Manual event API has been called for {camera}, but alerts are disabled. This manual event will not appear as an alert." @@ -703,11 +825,17 @@ class ReviewSegmentMaintainer(threading.Thread): # temporarily make it so this event can not end self.active_review_segments[ camera - ].last_update = sys.maxsize + ].last_alert_time = sys.maxsize + self.active_review_segments[ + camera + ].last_detection_time = sys.maxsize elif manual_info["state"] == ManualEventState.complete: self.active_review_segments[ camera - ].last_update = manual_info["end_time"] + ].last_alert_time = manual_info["end_time"] + self.active_review_segments[ + camera + ].last_detection_time = manual_info["end_time"] else: logger.warning( f"Dedicated LPR camera API has been called for {camera}, but detections are disabled. LPR events will not appear as a detection." @@ -717,46 +845,3 @@ class ReviewSegmentMaintainer(threading.Thread): self.requestor.stop() self.detection_subscriber.stop() logger.info("Exiting review maintainer...") - - -def get_active_objects( - frame_time: float, camera_config: CameraConfig, all_objects: list[TrackedObject] -) -> list[TrackedObject]: - """get active objects for detection.""" - return [ - o - for o in all_objects - if o["motionless_count"] - < camera_config.detect.stationary.threshold # no stationary objects - and o["position_changes"] > 0 # object must have moved at least once - and o["frame_time"] == frame_time # object must be detected in this frame - and not o["false_positive"] # object must not be a false positive - and ( - o["label"] in camera_config.review.alerts.labels - or ( - camera_config.review.detections.labels is None - or o["label"] in camera_config.review.detections.labels - ) - ) # object must be in the alerts or detections label list - ] - - -def get_loitering_objects( - frame_time: float, camera_config: CameraConfig, all_objects: list[TrackedObject] -) -> list[TrackedObject]: - """get loitering objects for detection.""" - return [ - o - for o in all_objects - if o["pending_loitering"] # object must be pending loitering - and o["position_changes"] > 0 # object must have moved at least once - and o["frame_time"] == frame_time # object must be detected in this frame - and not o["false_positive"] # object must not be a false positive - and ( - o["label"] in camera_config.review.alerts.labels - or ( - camera_config.review.detections.labels is None - or o["label"] in camera_config.review.detections.labels - ) - ) # object must be in the alerts or detections label list - ] diff --git a/web/src/components/preview/ScrubbablePreview.tsx b/web/src/components/preview/ScrubbablePreview.tsx index f3aa4f158..53f839ef6 100644 --- a/web/src/components/preview/ScrubbablePreview.tsx +++ b/web/src/components/preview/ScrubbablePreview.tsx @@ -315,9 +315,9 @@ export function InProgressPreview({ const apiHost = useApiHost(); const sliderRef = useRef(null); const { data: previewFrames } = useSWR( - `preview/${camera}/start/${Math.floor(startTime) - PREVIEW_PADDING}/end/${ - Math.ceil(endTime ?? timeRange.before) + PREVIEW_PADDING - }/frames`, + `preview/${camera}/start/${Math.floor(startTime) - PREVIEW_PADDING}/end/${Math.ceil( + endTime ?? timeRange.before, + )}/frames`, { revalidateOnFocus: false }, );