From af24eb7dbfa1fde43475e06045c14eb284e616e4 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Wed, 1 Nov 2023 06:12:43 -0500 Subject: [PATCH] Autotracking tweaks (#8400) * optimize motion and velocity estimation * change recommended fps and fix config validate * remove unneeded var * process at most 3 objects per second * fix test --- docs/docs/configuration/autotracking.md | 2 +- frigate/config.py | 2 +- frigate/const.py | 2 +- frigate/object_processing.py | 6 +-- frigate/ptz/autotrack.py | 57 +++++++++++++------------ frigate/test/test_config.py | 10 ++--- 6 files changed, 39 insertions(+), 40 deletions(-) diff --git a/docs/docs/configuration/autotracking.md b/docs/docs/configuration/autotracking.md index 4191a32c9..6fc6fdcef 100644 --- a/docs/docs/configuration/autotracking.md +++ b/docs/docs/configuration/autotracking.md @@ -113,7 +113,7 @@ If you initially calibrate with zooming disabled and then enable zooming at a la Every PTZ camera is different, so autotracking may not perform ideally in every situation. This experimental feature was initially developed using an EmpireTech/Dahua SD1A404XB-GNR. -The object tracker in Frigate estimates the motion of the PTZ so that tracked objects are preserved when the camera moves. In most cases (especially for faster moving objects), the default 5 fps is insufficient for the motion estimator to perform accurately. 10 fps is the current recommendation. Higher frame rates will likely not be more performant and will only slow down Frigate and the motion estimator. Adjust your camera to output at least 10 frames per second and change the `fps` parameter in the [detect configuration](index.md) of your configuration file. +The object tracker in Frigate estimates the motion of the PTZ so that tracked objects are preserved when the camera moves. In most cases 5 fps is sufficient, but if you plan to track faster moving objects, you may want to increase this slightly. Higher frame rates (> 10fps) will only slow down Frigate and the motion estimator and may lead to dropped frames, especially if you are using experimental zooming. A fast [detector](object_detectors.md) is recommended. CPU detectors will not perform well or won't work at all. You can watch Frigate's debug viewer for your camera to see a thicker colored box around the object currently being autotracked. diff --git a/frigate/config.py b/frigate/config.py index ebec6c843..b413a8cc8 100644 --- a/frigate/config.py +++ b/frigate/config.py @@ -171,7 +171,7 @@ class PtzAutotrackConfig(FrigateBaseModel): timeout: int = Field( default=10, title="Seconds to delay before returning to preset." ) - movement_weights: Optional[Union[float, List[float]]] = Field( + movement_weights: Optional[Union[str, List[str]]] = Field( default=[], title="Internal value used for PTZ movements based on the speed of your camera's motor.", ) diff --git a/frigate/const.py b/frigate/const.py index 40bdbe08f..9fe18a0f2 100644 --- a/frigate/const.py +++ b/frigate/const.py @@ -60,7 +60,7 @@ REQUEST_REGION_GRID = "request_region_grid" # Autotracking -AUTOTRACKING_MAX_AREA_RATIO = 0.5 +AUTOTRACKING_MAX_AREA_RATIO = 0.6 AUTOTRACKING_MOTION_MIN_DISTANCE = 20 AUTOTRACKING_MOTION_MAX_POINTS = 500 AUTOTRACKING_MAX_MOVE_METRICS = 500 diff --git a/frigate/object_processing.py b/frigate/object_processing.py index 64794361f..afd2cb1a4 100644 --- a/frigate/object_processing.py +++ b/frigate/object_processing.py @@ -248,10 +248,8 @@ class TrackedObject: if self.obj_data["frame_time"] - self.previous["frame_time"] > 60: significant_change = True - # update autotrack at half fps - if self.obj_data["frame_time"] - self.previous["frame_time"] > ( - 1 / (self.camera_config.detect.fps / 2) - ): + # update autotrack at most 3 objects per second + if self.obj_data["frame_time"] - self.previous["frame_time"] >= (1 / 3): autotracker_update = True self.obj_data.update(obj_data) diff --git a/frigate/ptz/autotrack.py b/frigate/ptz/autotrack.py index 1bc58d5d7..92cbc5cb3 100644 --- a/frigate/ptz/autotrack.py +++ b/frigate/ptz/autotrack.py @@ -215,8 +215,8 @@ class PtzAutoTracker: maxlen=round(camera_config.detect.fps * 1.5) ) self.tracked_object_metrics[camera] = { - "max_target_box": 1 - - (AUTOTRACKING_MAX_AREA_RATIO ** self.zoom_factor[camera]) + "max_target_box": AUTOTRACKING_MAX_AREA_RATIO + ** (1 / self.zoom_factor[camera]) } self.calibrating[camera] = False @@ -268,6 +268,10 @@ class PtzAutoTracker: if camera_config.onvif.autotracking.movement_weights: if len(camera_config.onvif.autotracking.movement_weights) == 5: + camera_config.onvif.autotracking.movement_weights = [ + float(val) + for val in camera_config.onvif.autotracking.movement_weights + ] self.ptz_metrics[camera][ "ptz_min_zoom" ].value = camera_config.onvif.autotracking.movement_weights[0] @@ -521,7 +525,11 @@ class PtzAutoTracker: camera_height = camera_config.frame_shape[0] # Extract areas and calculate weighted average - areas = [obj["area"] for obj in self.tracked_object_history[camera]] + # grab the largest dimension of the bounding box and create a square from that + areas = [ + max(obj["box"][2] - obj["box"][0], obj["box"][3] - obj["box"][1]) ** 2 + for obj in self.tracked_object_history[camera] + ] filtered_areas = ( remove_outliers(areas) @@ -686,19 +694,20 @@ class PtzAutoTracker: camera_height = camera_config.frame_shape[0] camera_fps = camera_config.detect.fps + # estimate_velocity is a numpy array of bbox top,left and bottom,right velocities velocities = obj.obj_data["estimate_velocity"] logger.debug(f"{camera}: Velocities from norfair: {velocities}") # if we are close enough to zero, return right away if np.all(np.round(velocities) == 0): - return True, np.zeros((2, 2)) + return True, np.zeros((4,)) # Thresholds x_mags_thresh = camera_width / camera_fps / 2 y_mags_thresh = camera_height / camera_fps / 2 dir_thresh = 0.93 - delta_thresh = 12 - var_thresh = 5 + delta_thresh = 20 + var_thresh = 10 # Check magnitude x_mags = np.abs(velocities[:, 0]) @@ -722,7 +731,6 @@ class PtzAutoTracker: np.linalg.norm(velocities[0]) * np.linalg.norm(velocities[1]) ) dir_thresh = 0.6 if np.all(delta < delta_thresh / 2) else dir_thresh - print(f"cosine sim: {cosine_sim}") invalid_dirs = cosine_sim < dir_thresh # Combine @@ -752,10 +760,10 @@ class PtzAutoTracker: ) ) # invalid velocity - return False, np.zeros((2, 2)) + return False, np.zeros((4,)) else: logger.debug(f"{camera}: Valid velocity ") - return True, np.mean(velocities, axis=0) + return True, velocities.flatten() def _get_distance_threshold(self, camera, obj): # Returns true if Euclidean distance from object to center of frame is @@ -836,7 +844,7 @@ class PtzAutoTracker: # ensure object is not moving quickly below_velocity_threshold = np.all( np.abs(average_velocity) - < np.array([velocity_threshold_x, velocity_threshold_y]) + < np.tile([velocity_threshold_x, velocity_threshold_y], 2) ) or np.all(average_velocity == 0) below_area_threshold = ( @@ -938,7 +946,7 @@ class PtzAutoTracker: camera_height = camera_config.frame_shape[0] camera_fps = camera_config.detect.fps - average_velocity = np.zeros((2, 2)) + average_velocity = np.zeros((4,)) predicted_box = obj.obj_data["box"] centroid_x = obj.obj_data["centroid"][0] @@ -966,7 +974,6 @@ class PtzAutoTracker: # this box could exceed the frame boundaries if velocity is high # but we'll handle that in _enqueue_move() as two separate moves current_box = np.array(obj.obj_data["box"]) - average_velocity = np.tile(average_velocity, 2) predicted_box = ( current_box + camera_fps * predicted_movement_time * average_velocity @@ -1010,7 +1017,10 @@ class PtzAutoTracker: zoom = 0 result = None current_zoom_level = self.ptz_metrics[camera]["ptz_zoom_level"].value - target_box = obj.obj_data["area"] / (camera_width * camera_height) + target_box = max( + obj.obj_data["box"][2] - obj.obj_data["box"][0], + obj.obj_data["box"][3] - obj.obj_data["box"][1], + ) ** 2 / (camera_width * camera_height) # absolute zooming separately from pan/tilt if camera_config.onvif.autotracking.zooming == ZoomingModeEnum.absolute: @@ -1061,24 +1071,15 @@ class PtzAutoTracker: < self.tracked_object_metrics[camera]["max_target_box"] else self.tracked_object_metrics[camera]["max_target_box"] ) - zoom = ( - 2 - * ( - limit - / ( - self.tracked_object_metrics[camera]["target_box"] - + limit - ) - ) - - 1 - ) + ratio = limit / self.tracked_object_metrics[camera]["target_box"] + zoom = (ratio - 1) / (ratio + 1) logger.debug(f"{camera}: Zoom calculation: {zoom}") if not result: # zoom out with special condition if zooming out because of velocity, edges, etc. - zoom = -(1 - zoom) if zoom > 0 else -(zoom + 1) + zoom = -(1 - zoom) if zoom > 0 else -(zoom * 2 + 1) if result: # zoom in - zoom = 1 - zoom if zoom > 0 else (zoom + 1) + zoom = 1 - zoom if zoom > 0 else (zoom * 2 + 1) logger.debug(f"{camera}: Zooming: {result} Zoom amount: {zoom}") @@ -1199,8 +1200,8 @@ class PtzAutoTracker: ) self.tracked_object[camera] = None self.tracked_object_metrics[camera] = { - "max_target_box": 1 - - (AUTOTRACKING_MAX_AREA_RATIO ** self.zoom_factor[camera]) + "max_target_box": AUTOTRACKING_MAX_AREA_RATIO + ** (1 / self.zoom_factor[camera]) } def camera_maintenance(self, camera): diff --git a/frigate/test/test_config.py b/frigate/test/test_config.py index 3e649bb5b..8c40b5338 100644 --- a/frigate/test/test_config.py +++ b/frigate/test/test_config.py @@ -1651,11 +1651,11 @@ class TestConfig(unittest.TestCase): runtime_config = frigate_config.runtime_config() assert runtime_config.cameras["back"].onvif.autotracking.movement_weights == [ - 0, - 1, - 1.23, - 2.34, - 0.50, + "0.0", + "1.0", + "1.23", + "2.34", + "0.5", ] def test_fails_invalid_movement_weights(self):