Autotracking tweaks (#8400)

* optimize motion and velocity estimation

* change recommended fps and fix config validate

* remove unneeded var

* process at most 3 objects per second

* fix test

docs/docs/configuration/autotracking.md

@@ -113,7 +113,7 @@ If you initially calibrate with zooming disabled and then enable zooming at a la
 
 Every PTZ camera is different, so autotracking may not perform ideally in every situation. This experimental feature was initially developed using an EmpireTech/Dahua SD1A404XB-GNR.
 
-The object tracker in Frigate estimates the motion of the PTZ so that tracked objects are preserved when the camera moves. In most cases (especially for faster moving objects), the default 5 fps is insufficient for the motion estimator to perform accurately. 10 fps is the current recommendation. Higher frame rates will likely not be more performant and will only slow down Frigate and the motion estimator. Adjust your camera to output at least 10 frames per second and change the `fps` parameter in the [detect configuration](index.md) of your configuration file.
+The object tracker in Frigate estimates the motion of the PTZ so that tracked objects are preserved when the camera moves. In most cases 5 fps is sufficient, but if you plan to track faster moving objects, you may want to increase this slightly. Higher frame rates (> 10fps) will only slow down Frigate and the motion estimator and may lead to dropped frames, especially if you are using experimental zooming.
 
 A fast [detector](object_detectors.md) is recommended. CPU detectors will not perform well or won't work at all. You can watch Frigate's debug viewer for your camera to see a thicker colored box around the object currently being autotracked.
 

frigate/config.py

@@ -171,7 +171,7 @@ class PtzAutotrackConfig(FrigateBaseModel):
     timeout: int = Field(
         default=10, title="Seconds to delay before returning to preset."
     )
-    movement_weights: Optional[Union[float, List[float]]] = Field(
+    movement_weights: Optional[Union[str, List[str]]] = Field(
         default=[],
         title="Internal value used for PTZ movements based on the speed of your camera's motor.",
     )

frigate/const.py

@@ -60,7 +60,7 @@ REQUEST_REGION_GRID = "request_region_grid"
 
 # Autotracking
 
-AUTOTRACKING_MAX_AREA_RATIO = 0.5
+AUTOTRACKING_MAX_AREA_RATIO = 0.6
 AUTOTRACKING_MOTION_MIN_DISTANCE = 20
 AUTOTRACKING_MOTION_MAX_POINTS = 500
 AUTOTRACKING_MAX_MOVE_METRICS = 500

frigate/object_processing.py

@@ -248,10 +248,8 @@ class TrackedObject:
             if self.obj_data["frame_time"] - self.previous["frame_time"] > 60:
                 significant_change = True
 
-            # update autotrack at half fps
+            # update autotrack at most 3 objects per second
-            if self.obj_data["frame_time"] - self.previous["frame_time"] > (
+            if self.obj_data["frame_time"] - self.previous["frame_time"] >= (1 / 3):
-                1 / (self.camera_config.detect.fps / 2)
-            ):
                 autotracker_update = True
 
         self.obj_data.update(obj_data)

frigate/ptz/autotrack.py

@@ -215,8 +215,8 @@ class PtzAutoTracker:
             maxlen=round(camera_config.detect.fps * 1.5)
         )
         self.tracked_object_metrics[camera] = {
-            "max_target_box": 1
+            "max_target_box": AUTOTRACKING_MAX_AREA_RATIO
-            - (AUTOTRACKING_MAX_AREA_RATIO ** self.zoom_factor[camera])
+            ** (1 / self.zoom_factor[camera])
         }
 
         self.calibrating[camera] = False
@@ -268,6 +268,10 @@ class PtzAutoTracker:
 
             if camera_config.onvif.autotracking.movement_weights:
                 if len(camera_config.onvif.autotracking.movement_weights) == 5:
+                    camera_config.onvif.autotracking.movement_weights = [
+                        float(val)
+                        for val in camera_config.onvif.autotracking.movement_weights
+                    ]
                     self.ptz_metrics[camera][
                         "ptz_min_zoom"
                     ].value = camera_config.onvif.autotracking.movement_weights[0]
@@ -521,7 +525,11 @@ class PtzAutoTracker:
         camera_height = camera_config.frame_shape[0]
 
         # Extract areas and calculate weighted average
-        areas = [obj["area"] for obj in self.tracked_object_history[camera]]
+        # grab the largest dimension of the bounding box and create a square from that
+        areas = [
+            max(obj["box"][2] - obj["box"][0], obj["box"][3] - obj["box"][1]) ** 2
+            for obj in self.tracked_object_history[camera]
+        ]
 
         filtered_areas = (
             remove_outliers(areas)
@@ -686,19 +694,20 @@ class PtzAutoTracker:
         camera_height = camera_config.frame_shape[0]
         camera_fps = camera_config.detect.fps
 
+        # estimate_velocity is a numpy array of bbox top,left and bottom,right velocities
         velocities = obj.obj_data["estimate_velocity"]
         logger.debug(f"{camera}: Velocities from norfair: {velocities}")
 
         # if we are close enough to zero, return right away
         if np.all(np.round(velocities) == 0):
-            return True, np.zeros((2, 2))
+            return True, np.zeros((4,))
 
         # Thresholds
         x_mags_thresh = camera_width / camera_fps / 2
         y_mags_thresh = camera_height / camera_fps / 2
         dir_thresh = 0.93
-        delta_thresh = 12
+        delta_thresh = 20
-        var_thresh = 5
+        var_thresh = 10
 
         # Check magnitude
         x_mags = np.abs(velocities[:, 0])
@@ -722,7 +731,6 @@ class PtzAutoTracker:
                 np.linalg.norm(velocities[0]) * np.linalg.norm(velocities[1])
             )
             dir_thresh = 0.6 if np.all(delta < delta_thresh / 2) else dir_thresh
-            print(f"cosine sim: {cosine_sim}")
             invalid_dirs = cosine_sim < dir_thresh
 
         # Combine
@@ -752,10 +760,10 @@ class PtzAutoTracker:
                 )
             )
             # invalid velocity
-            return False, np.zeros((2, 2))
+            return False, np.zeros((4,))
         else:
             logger.debug(f"{camera}: Valid velocity ")
-            return True, np.mean(velocities, axis=0)
+            return True, velocities.flatten()
 
     def _get_distance_threshold(self, camera, obj):
         # Returns true if Euclidean distance from object to center of frame is
@@ -836,7 +844,7 @@ class PtzAutoTracker:
         # ensure object is not moving quickly
         below_velocity_threshold = np.all(
             np.abs(average_velocity)
-            < np.array([velocity_threshold_x, velocity_threshold_y])
+            < np.tile([velocity_threshold_x, velocity_threshold_y], 2)
         ) or np.all(average_velocity == 0)
 
         below_area_threshold = (
@@ -938,7 +946,7 @@ class PtzAutoTracker:
         camera_height = camera_config.frame_shape[0]
         camera_fps = camera_config.detect.fps
 
-        average_velocity = np.zeros((2, 2))
+        average_velocity = np.zeros((4,))
         predicted_box = obj.obj_data["box"]
 
         centroid_x = obj.obj_data["centroid"][0]
@@ -966,7 +974,6 @@ class PtzAutoTracker:
                 # this box could exceed the frame boundaries if velocity is high
                 # but we'll handle that in _enqueue_move() as two separate moves
                 current_box = np.array(obj.obj_data["box"])
-                average_velocity = np.tile(average_velocity, 2)
                 predicted_box = (
                     current_box
                     + camera_fps * predicted_movement_time * average_velocity
@@ -1010,7 +1017,10 @@ class PtzAutoTracker:
         zoom = 0
         result = None
         current_zoom_level = self.ptz_metrics[camera]["ptz_zoom_level"].value
-        target_box = obj.obj_data["area"] / (camera_width * camera_height)
+        target_box = max(
+            obj.obj_data["box"][2] - obj.obj_data["box"][0],
+            obj.obj_data["box"][3] - obj.obj_data["box"][1],
+        ) ** 2 / (camera_width * camera_height)
 
         # absolute zooming separately from pan/tilt
         if camera_config.onvif.autotracking.zooming == ZoomingModeEnum.absolute:
@@ -1061,24 +1071,15 @@ class PtzAutoTracker:
                         < self.tracked_object_metrics[camera]["max_target_box"]
                         else self.tracked_object_metrics[camera]["max_target_box"]
                     )
-                    zoom = (
+                    ratio = limit / self.tracked_object_metrics[camera]["target_box"]
-                        2
+                    zoom = (ratio - 1) / (ratio + 1)
-                        * (
-                            limit
-                            / (
-                                self.tracked_object_metrics[camera]["target_box"]
-                                + limit
-                            )
-                        )
-                        - 1
-                    )
                     logger.debug(f"{camera}: Zoom calculation: {zoom}")
                     if not result:
                         # zoom out with special condition if zooming out because of velocity, edges, etc.
-                        zoom = -(1 - zoom) if zoom > 0 else -(zoom + 1)
+                        zoom = -(1 - zoom) if zoom > 0 else -(zoom * 2 + 1)
                     if result:
                         # zoom in
-                        zoom = 1 - zoom if zoom > 0 else (zoom + 1)
+                        zoom = 1 - zoom if zoom > 0 else (zoom * 2 + 1)
 
         logger.debug(f"{camera}: Zooming: {result} Zoom amount: {zoom}")
 
@@ -1199,8 +1200,8 @@ class PtzAutoTracker:
                 )
                 self.tracked_object[camera] = None
                 self.tracked_object_metrics[camera] = {
-                    "max_target_box": 1
+                    "max_target_box": AUTOTRACKING_MAX_AREA_RATIO
-                    - (AUTOTRACKING_MAX_AREA_RATIO ** self.zoom_factor[camera])
+                    ** (1 / self.zoom_factor[camera])
                 }
 
     def camera_maintenance(self, camera):

frigate/test/test_config.py

@@ -1651,11 +1651,11 @@ class TestConfig(unittest.TestCase):
 
         runtime_config = frigate_config.runtime_config()
         assert runtime_config.cameras["back"].onvif.autotracking.movement_weights == [
-            0,
+            "0.0",
-            1,
+            "1.0",
-            1.23,
+            "1.23",
-            2.34,
+            "2.34",
-            0.50,
+            "0.5",
         ]
 
     def test_fails_invalid_movement_weights(self):