Improve object tracking (#17671)

* Save initial frame of new objects to frame cache
Objects that move quickly through the frame and are only seen briefly may not have the update() method called to save thumbnail_data, and may not have the initial frame saved to the tracked object frame cache. This caused a "Frame missing from frame cache" message that was patched by #7313 but this sometimes caused the wrong frame to be chosen for the thumb/snapshot.

* Tracking tweaks
- When registering new objects, use the past detections from Norfair to populate self.positions and self.stationary_box_history. This prevents the first call of update_position() from triggering a +1 on the object's stationary count (because the iou would be 1.0).
- Add a specific tracker for dedicated LPR cam license_plate objects using a lower R value and higher distance threshold to account for fast moving plates.
- Add helpful debug messages and keep them disabled with `if False:`

frigate/camera/state.py

@@ -263,6 +263,27 @@ class CameraState:
                 current_detections[id],
             )
 
+            # add initial frame to frame cache
+            self.frame_cache[frame_time] = np.copy(current_frame)
+
+            # save initial thumbnail data and best object
+            thumbnail_data = {
+                "frame_time": frame_time,
+                "box": new_obj.obj_data["box"],
+                "area": new_obj.obj_data["area"],
+                "region": new_obj.obj_data["region"],
+                "score": new_obj.obj_data["score"],
+                "attributes": new_obj.obj_data["attributes"],
+                "current_estimated_speed": 0,
+                "velocity_angle": 0,
+                "path_data": [],
+                "recognized_license_plate": None,
+                "recognized_license_plate_score": None,
+            }
+            new_obj.thumbnail_data = thumbnail_data
+            tracked_objects[id].thumbnail_data = thumbnail_data
+            self.best_objects[new_obj.obj_data["label"]] = new_obj
+
             # call event handlers
             for c in self.callbacks["start"]:
                 c(self.name, new_obj, frame_name)

frigate/track/norfair_tracker.py

@@ -129,6 +129,11 @@ class NorfairTracker(ObjectTracker):
                 "distance_function": frigate_distance,
                 "distance_threshold": 2.5,
             },
+            "license_plate": {
+                "filter_factory": OptimizedKalmanFilterFactory(R=2.5, Q=0.05),
+                "distance_function": frigate_distance,
+                "distance_threshold": 3.75,
+            },
         }
 
         # Define autotracking PTZ-specific configurations
@@ -273,17 +278,24 @@ class NorfairTracker(ObjectTracker):
         )
         self.tracked_objects[id] = obj
         self.disappeared[id] = 0
+        if obj_match:
+            boxes = [p.data["box"] for p in obj_match.past_detections]
+        else:
+            boxes = [obj["box"]]
+
+        xmins, ymins, xmaxs, ymaxs = zip(*boxes)
+
         self.positions[id] = {
-            "xmins": [],
-            "ymins": [],
-            "xmaxs": [],
-            "ymaxs": [],
+            "xmins": list(xmins),
+            "ymins": list(ymins),
+            "xmaxs": list(xmaxs),
+            "ymaxs": list(ymaxs),
             "xmin": 0,
             "ymin": 0,
             "xmax": self.detect_config.width,
             "ymax": self.detect_config.height,
         }
-        self.stationary_box_history[id] = []
+        self.stationary_box_history[id] = boxes
 
     def deregister(self, id, track_id):
         obj = self.tracked_objects[id]
@@ -369,9 +381,9 @@ class NorfairTracker(ObjectTracker):
                 }
                 return False
 
-        # if there are less than 10 entries for the position, add the bounding box
+        # if there are more than 5 and less than 10 entries for the position, add the bounding box
         # and recompute the position box
-        if len(position["xmins"]) < 10:
+        if 5 <= len(position["xmins"]) < 10:
             position["xmins"].append(xmin)
             position["ymins"].append(ymin)
             position["xmaxs"].append(xmax)
@@ -599,7 +611,10 @@ class NorfairTracker(ObjectTracker):
 
         # print a table to the console with norfair tracked object info
         if False:
-            self.print_objects_as_table(self.trackers["person"]["ptz"].tracked_objects)
+            if len(self.trackers["license_plate"]["static"].tracked_objects) > 0:
+                self.print_objects_as_table(
+                    self.trackers["license_plate"]["static"].tracked_objects
+                )
 
         # Get tracked objects from type-specific trackers
         for object_trackers in self.trackers.values():
@@ -644,3 +659,20 @@ class NorfairTracker(ObjectTracker):
                 color=(255, 0, 0),
                 thickness=None,
             )
+
+        if False:
+            # draw the current formatted time on the frame
+            from datetime import datetime
+
+            formatted_time = datetime.fromtimestamp(frame_time).strftime(
+                "%m/%d/%Y %I:%M:%S %p"
+            )
+
+            frame = Drawer.text(
+                frame,
+                formatted_time,
+                position=(10, 50),
+                size=1.5,
+                color=(255, 255, 255),
+                thickness=None,
+            )

frigate/track/tracked_object.py

@@ -144,8 +144,14 @@ class TrackedObject:
                 obj_data,
                 self.camera_config.frame_shape,
             ):
+                # use the current frame time if the object's frame time isn't in the frame cache
+                selected_frame_time = (
+                    current_frame_time
+                    if obj_data["frame_time"] not in self.frame_cache.keys()
+                    else obj_data["frame_time"]
+                )
                 self.thumbnail_data = {
-                    "frame_time": current_frame_time,
+                    "frame_time": selected_frame_time,
                     "box": obj_data["box"],
                     "area": obj_data["area"],
                     "region": obj_data["region"],