From 0b71cfaf06d50eebffa15ef6bad197324a134c95 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Tue, 8 Oct 2024 08:41:54 -0600 Subject: [PATCH] Handle loitering objects (#14221) --- frigate/object_processing.py | 11 ++++++ frigate/review/maintainer.py | 75 +++++++++++++++++++++++++++++++----- 2 files changed, 77 insertions(+), 9 deletions(-) diff --git a/frigate/object_processing.py b/frigate/object_processing.py index 15874ca4e..a5b06b76f 100644 --- a/frigate/object_processing.py +++ b/frigate/object_processing.py @@ -138,6 +138,7 @@ class TrackedObject: self.last_published = 0 self.frame = None self.active = True + self.pending_loitering = False self.previous = self.to_dict() def _is_false_positive(self): @@ -194,6 +195,8 @@ class TrackedObject: # check zones current_zones = [] bottom_center = (obj_data["centroid"][0], obj_data["box"][3]) + in_loitering_zone = False + # check each zone for name, zone in self.camera_config.zones.items(): # if the zone is not for this object type, skip @@ -207,6 +210,10 @@ class TrackedObject: if name in self.current_zones or not zone_filtered(self, zone.filters): # an object is only considered present in a zone if it has a zone inertia of 3+ if zone_score >= zone.inertia: + # if the zone has loitering time, update loitering status + if zone.loitering_time > 0: + in_loitering_zone = True + loitering_score = self.zone_loitering.get(name, 0) + 1 # loitering time is configured as seconds, convert to count of frames @@ -227,6 +234,9 @@ class TrackedObject: if 0 < zone_score < zone.inertia: self.zone_presence[name] = zone_score - 1 + # update loitering status + self.pending_loitering = in_loitering_zone + # maintain attributes for attr in obj_data["attributes"]: if self.attributes[attr["label"]] < attr["score"]: @@ -305,6 +315,7 @@ class TrackedObject: "has_snapshot": self.has_snapshot, "attributes": self.attributes, "current_attributes": self.obj_data["attributes"], + "pending_loitering": self.pending_loitering, } if include_thumbnail: diff --git a/frigate/review/maintainer.py b/frigate/review/maintainer.py index 3b5980a85..db1cec80b 100644 --- a/frigate/review/maintainer.py +++ b/frigate/review/maintainer.py @@ -167,7 +167,7 @@ class ReviewSegmentMaintainer(threading.Thread): # clear ongoing review segments from last instance self.requestor.send_data(CLEAR_ONGOING_REVIEW_SEGMENTS, "") - def new_segment( + def _publish_segment_start( self, segment: PendingReviewSegment, ) -> None: @@ -186,7 +186,7 @@ class ReviewSegmentMaintainer(threading.Thread): ), ) - def update_segment( + def _publish_segment_update( self, segment: PendingReviewSegment, camera_config: CameraConfig, @@ -211,7 +211,7 @@ class ReviewSegmentMaintainer(threading.Thread): ), ) - def end_segment( + def _publish_segment_end( self, segment: PendingReviewSegment, prev_data: dict[str, any], @@ -241,8 +241,10 @@ class ReviewSegmentMaintainer(threading.Thread): camera_config = self.config.cameras[segment.camera] active_objects = get_active_objects(frame_time, camera_config, objects) prev_data = segment.get_data(False) + has_activity = False if len(active_objects) > 0: + has_activity = True should_update = False if frame_time > segment.last_update: @@ -295,13 +297,45 @@ class ReviewSegmentMaintainer(threading.Thread): logger.debug(f"Failed to get frame {frame_id} from SHM") return - self.update_segment( + self._publish_segment_update( segment, camera_config, yuv_frame, active_objects, prev_data ) self.frame_manager.close(frame_id) except FileNotFoundError: return - else: + + # check if there are any objects pending loitering on this camera + loitering_objects = get_loitering_objects(frame_time, camera_config, objects) + + if loitering_objects: + has_activity = True + + if frame_time > segment.last_update: + segment.last_update = frame_time + + for object in loitering_objects: + # if object is alert label + # and has entered loitering zone + # mark this review as alert + if ( + segment.severity != SeverityEnum.alert + and object["label"] in camera_config.review.alerts.labels + and ( + len(object["current_zones"]) > 0 + and set(object["current_zones"]) + & set(camera_config.review.alerts.required_zones) + ) + ): + segment.severity = SeverityEnum.alert + should_update = True + + # keep zones up to date + if len(object["current_zones"]) > 0: + for zone in object["current_zones"]: + if zone not in segment.zones: + segment.zones.append(zone) + + if not has_activity: if not segment.has_frame: try: frame_id = f"{camera_config.name}{frame_time}" @@ -315,16 +349,18 @@ class ReviewSegmentMaintainer(threading.Thread): segment.save_full_frame(camera_config, yuv_frame) self.frame_manager.close(frame_id) - self.update_segment(segment, camera_config, None, [], prev_data) + self._publish_segment_update( + segment, camera_config, None, [], prev_data + ) except FileNotFoundError: return if segment.severity == SeverityEnum.alert and frame_time > ( segment.last_update + THRESHOLD_ALERT_ACTIVITY ): - self.end_segment(segment, prev_data) + self._publish_segment_end(segment, prev_data) elif frame_time > (segment.last_update + THRESHOLD_DETECTION_ACTIVITY): - self.end_segment(segment, prev_data) + self._publish_segment_end(segment, prev_data) def check_if_new_segment( self, @@ -418,7 +454,7 @@ class ReviewSegmentMaintainer(threading.Thread): camera_config, yuv_frame, active_objects ) self.frame_manager.close(frame_id) - self.new_segment(self.active_review_segments[camera]) + self._publish_segment_start(self.active_review_segments[camera]) except FileNotFoundError: return @@ -609,3 +645,24 @@ def get_active_objects( ) ) # object must be in the alerts or detections label list ] + + +def get_loitering_objects( + frame_time: float, camera_config: CameraConfig, all_objects: list[TrackedObject] +) -> list[TrackedObject]: + """get loitering objects for detection.""" + return [ + o + for o in all_objects + if o["pending_loitering"] # object must be pending loitering + and o["position_changes"] > 0 # object must have moved at least once + and o["frame_time"] == frame_time # object must be detected in this frame + and not o["false_positive"] # object must not be a false positive + and ( + o["label"] in camera_config.review.alerts.labels + or ( + camera_config.review.detections.labels is None + or o["label"] in camera_config.review.detections.labels + ) + ) # object must be in the alerts or detections label list + ]