From da62c41e87cc6216111ed129bfb8838d6a867f83 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sun, 13 Apr 2025 13:10:35 -0500 Subject: [PATCH] Improve object tracking (#17671) * Save initial frame of new objects to frame cache Objects that move quickly through the frame and are only seen briefly may not have the update() method called to save thumbnail_data, and may not have the initial frame saved to the tracked object frame cache. This caused a "Frame missing from frame cache" message that was patched by #7313 but this sometimes caused the wrong frame to be chosen for the thumb/snapshot. * Tracking tweaks - When registering new objects, use the past detections from Norfair to populate self.positions and self.stationary_box_history. This prevents the first call of update_position() from triggering a +1 on the object's stationary count (because the iou would be 1.0). - Add a specific tracker for dedicated LPR cam license_plate objects using a lower R value and higher distance threshold to account for fast moving plates. - Add helpful debug messages and keep them disabled with `if False:` --- frigate/camera/state.py | 21 ++++++++++++++ frigate/track/norfair_tracker.py | 48 ++++++++++++++++++++++++++------ frigate/track/tracked_object.py | 8 +++++- 3 files changed, 68 insertions(+), 9 deletions(-) diff --git a/frigate/camera/state.py b/frigate/camera/state.py index fee6e4e23..c307bea1e 100644 --- a/frigate/camera/state.py +++ b/frigate/camera/state.py @@ -263,6 +263,27 @@ class CameraState: current_detections[id], ) + # add initial frame to frame cache + self.frame_cache[frame_time] = np.copy(current_frame) + + # save initial thumbnail data and best object + thumbnail_data = { + "frame_time": frame_time, + "box": new_obj.obj_data["box"], + "area": new_obj.obj_data["area"], + "region": new_obj.obj_data["region"], + "score": new_obj.obj_data["score"], + "attributes": new_obj.obj_data["attributes"], + "current_estimated_speed": 0, + "velocity_angle": 0, + "path_data": [], + "recognized_license_plate": None, + "recognized_license_plate_score": None, + } + new_obj.thumbnail_data = thumbnail_data + tracked_objects[id].thumbnail_data = thumbnail_data + self.best_objects[new_obj.obj_data["label"]] = new_obj + # call event handlers for c in self.callbacks["start"]: c(self.name, new_obj, frame_name) diff --git a/frigate/track/norfair_tracker.py b/frigate/track/norfair_tracker.py index 3487aa8c0..ec2bdfa7d 100644 --- a/frigate/track/norfair_tracker.py +++ b/frigate/track/norfair_tracker.py @@ -129,6 +129,11 @@ class NorfairTracker(ObjectTracker): "distance_function": frigate_distance, "distance_threshold": 2.5, }, + "license_plate": { + "filter_factory": OptimizedKalmanFilterFactory(R=2.5, Q=0.05), + "distance_function": frigate_distance, + "distance_threshold": 3.75, + }, } # Define autotracking PTZ-specific configurations @@ -273,17 +278,24 @@ class NorfairTracker(ObjectTracker): ) self.tracked_objects[id] = obj self.disappeared[id] = 0 + if obj_match: + boxes = [p.data["box"] for p in obj_match.past_detections] + else: + boxes = [obj["box"]] + + xmins, ymins, xmaxs, ymaxs = zip(*boxes) + self.positions[id] = { - "xmins": [], - "ymins": [], - "xmaxs": [], - "ymaxs": [], + "xmins": list(xmins), + "ymins": list(ymins), + "xmaxs": list(xmaxs), + "ymaxs": list(ymaxs), "xmin": 0, "ymin": 0, "xmax": self.detect_config.width, "ymax": self.detect_config.height, } - self.stationary_box_history[id] = [] + self.stationary_box_history[id] = boxes def deregister(self, id, track_id): obj = self.tracked_objects[id] @@ -369,9 +381,9 @@ class NorfairTracker(ObjectTracker): } return False - # if there are less than 10 entries for the position, add the bounding box + # if there are more than 5 and less than 10 entries for the position, add the bounding box # and recompute the position box - if len(position["xmins"]) < 10: + if 5 <= len(position["xmins"]) < 10: position["xmins"].append(xmin) position["ymins"].append(ymin) position["xmaxs"].append(xmax) @@ -599,7 +611,10 @@ class NorfairTracker(ObjectTracker): # print a table to the console with norfair tracked object info if False: - self.print_objects_as_table(self.trackers["person"]["ptz"].tracked_objects) + if len(self.trackers["license_plate"]["static"].tracked_objects) > 0: + self.print_objects_as_table( + self.trackers["license_plate"]["static"].tracked_objects + ) # Get tracked objects from type-specific trackers for object_trackers in self.trackers.values(): @@ -644,3 +659,20 @@ class NorfairTracker(ObjectTracker): color=(255, 0, 0), thickness=None, ) + + if False: + # draw the current formatted time on the frame + from datetime import datetime + + formatted_time = datetime.fromtimestamp(frame_time).strftime( + "%m/%d/%Y %I:%M:%S %p" + ) + + frame = Drawer.text( + frame, + formatted_time, + position=(10, 50), + size=1.5, + color=(255, 255, 255), + thickness=None, + ) diff --git a/frigate/track/tracked_object.py b/frigate/track/tracked_object.py index b7c3af287..58a4e0b83 100644 --- a/frigate/track/tracked_object.py +++ b/frigate/track/tracked_object.py @@ -144,8 +144,14 @@ class TrackedObject: obj_data, self.camera_config.frame_shape, ): + # use the current frame time if the object's frame time isn't in the frame cache + selected_frame_time = ( + current_frame_time + if obj_data["frame_time"] not in self.frame_cache.keys() + else obj_data["frame_time"] + ) self.thumbnail_data = { - "frame_time": current_frame_time, + "frame_time": selected_frame_time, "box": obj_data["box"], "area": obj_data["area"], "region": obj_data["region"],