Add object filter ratio (#2952)

* Add object ratio config parameters

Issue: #2948

* Add config test for object filter ratios

Issue: #2948

* Address review comments

- Accept `ratio` default
- Rename `bounds` to `box` for consistency
- Add migration for new field

Issue: #2948

* Fix logical errors

- field migrations require default values
- `clipped` referenced the wrong index for region, since it shifted
- missed an inclusion of `ratio` for detections in `process_frames`
- revert naming `o[2]` as `box` since it is out of scope!

This has now been test-run against a video, so I believe the kinks are
worked out.

Issue: #2948

* Update contributing notes for `make`

Issue: #2948

* Fix migration

- Ensure that defaults match between Event and migration script
- Deconflict migration script number (from rebase)

Issue: #2948

* Filter objects out of ratio bounds

Issue: #2948

* Update migration file to 009

Issue: #2948

docs/docs/configuration/index.md

@@ -194,6 +194,10 @@ objects:
       min_area: 5000
       # Optional: maximum width*height of the bounding box for the detected object (default: 24000000)
       max_area: 100000
+      # Optional: minimum width/height of the bounding box for the detected object (default: 0)
+      min_ratio: 0.5
+      # Optional: maximum width/height of the bounding box for the detected object (default: 24000000)
+      max_ratio: 2.0
       # Optional: minimum score for the object to initiate tracking (default: shown below)
       min_score: 0.5
       # Optional: minimum decimal percentage for tracked object's computed score to be considered a true positive (default: shown below)

docs/docs/contributing.md

@@ -40,9 +40,7 @@ Fork [blakeblackshear/frigate-hass-integration](https://github.com/blakeblackshe
 
 ### Setup
 
-#### 1. Build the docker container locally with the appropriate make command
+#### 1. Build the version information and docker container locally by running `make`
-
-For x86 machines, use `make amd64_frigate`
 
 #### 2. Create a local config file for testing
 

docs/docs/guides/false_positives.md

@@ -3,7 +3,11 @@ id: false_positives
 title: Reducing false positives
 ---
 
-Tune your object filters to adjust false positives: `min_area`, `max_area`, `min_score`, `threshold`.
+Tune your object filters to adjust false positives: `min_area`, `max_area`, `min_ratio`, `max_ratio`, `min_score`, `threshold`.
+
+The `min_area` and `max_area` values are compared against the area (number of pixels) from a given detected object. If the area is outside this range, the object will be ignored as a false positive. This allows objects that must be too small or too large to be ignored.
+
+Similarly, the `min_ratio` and `max_ratio` values are compared against a given detected object's width/height ratio (in pixels). If the ratio is outside this range, the object will be ignored as a false positive. This allows objects that are proportionally too short-and-wide (higher ratio) or too tall-and-narrow (smaller ratio) to be ignored.
 
 For object filters in your configuration, any single detection below `min_score` will be ignored as a false positive. `threshold` is based on the median of the history of scores (padded to 3 values) for a tracked object. Consider the following frames when `min_score` is set to 0.6 and threshold is set to 0.85:
 

docs/docs/integrations/mqtt.md

@@ -52,6 +52,7 @@ Message published for each changed event. The first message is published when th
     "score": 0.7890625,
     "box": [424, 500, 536, 712],
     "area": 23744,
+    "ratio": 2.113207,
     "region": [264, 450, 667, 853],
     "current_zones": ["driveway"],
     "entered_zones": ["yard", "driveway"],
@@ -75,6 +76,7 @@ Message published for each changed event. The first message is published when th
     "score": 0.87890625,
     "box": [432, 496, 544, 854],
     "area": 40096,
+    "ratio": 1.251397,
     "region": [218, 440, 693, 915],
     "current_zones": ["yard", "driveway"],
     "entered_zones": ["yard", "driveway"],

frigate/config.py

@@ -210,6 +210,14 @@ class FilterConfig(FrigateBaseModel):
     max_area: int = Field(
         default=24000000, title="Maximum area of bounding box for object to be counted."
     )
+    min_ratio: float = Field(
+        default=0,
+        title="Minimum ratio of bounding box's width/height for object to be counted.",
+    )
+    max_ratio: float = Field(
+        default=24000000,
+        title="Maximum ratio of bounding box's width/height for object to be counted.",
+    )
     threshold: float = Field(
         default=0.7,
         title="Average detection confidence threshold for object to be counted.",

frigate/events.py

@@ -105,6 +105,7 @@ class EventProcessor(threading.Thread):
                     region=event_data["region"],
                     box=event_data["box"],
                     area=event_data["area"],
+                    ratio=event_data["ratio"],
                     has_clip=event_data["has_clip"],
                     has_snapshot=event_data["has_snapshot"],
                 ).where(Event.id == event_data["id"]).execute()
@@ -124,6 +125,7 @@ class EventProcessor(threading.Thread):
                         region=event_data["region"],
                         box=event_data["box"],
                         area=event_data["area"],
+                        ratio=event_data["ratio"],
                         has_clip=event_data["has_clip"],
                         has_snapshot=event_data["has_snapshot"],
                     ).where(Event.id == event_data["id"]).execute()

frigate/models.py

@@ -20,6 +20,7 @@ class Event(Model):
     box = JSONField()
     area = IntegerField()
     retain_indefinitely = BooleanField(default=False)
+    ratio = FloatField(default=1.0)
 
 
 class Recordings(Model):

frigate/object_processing.py

@@ -192,6 +192,7 @@ class TrackedObject:
             "score": self.obj_data["score"],
             "box": self.obj_data["box"],
             "area": self.obj_data["area"],
+            "ratio": self.obj_data["ratio"],
             "region": self.obj_data["region"],
             "stationary": self.obj_data["motionless_count"]
             > self.camera_config.detect.stationary.threshold,
@@ -341,6 +342,14 @@ def zone_filtered(obj: TrackedObject, object_config):
         if obj_settings.threshold > obj.computed_score:
             return True
 
+        # if the object is not proportionally wide enough
+        if obj_settings.min_ratio > obj.obj_data["ratio"]:
+            return True
+
+        # if the object is proportionally too wide
+        if obj_settings.max_ratio < obj.obj_data["ratio"]:
+            return True
+
     return False
 
 

frigate/objects.py

@@ -150,7 +150,8 @@ class ObjectTracker:
                     "score": obj[1],
                     "box": obj[2],
                     "area": obj[3],
-                    "region": obj[4],
+                    "ratio": obj[4],
+                    "region": obj[5],
                     "frame_time": frame_time,
                 }
             )

frigate/test/test_config.py

@@ -1268,6 +1268,36 @@ class TestConfig(unittest.TestCase):
             ValidationError, lambda: frigate_config.runtime_config.cameras
         )
 
+    def test_object_filter_ratios_work(self):
+        config = {
+            "mqtt": {"host": "mqtt"},
+            "objects": {
+                "track": ["person", "dog"],
+                "filters": {"dog": {"min_ratio": 0.2, "max_ratio": 10.1}},
+            },
+            "cameras": {
+                "back": {
+                    "ffmpeg": {
+                        "inputs": [
+                            {"path": "rtsp://10.0.0.1:554/video", "roles": ["detect"]}
+                        ]
+                    },
+                    "detect": {
+                        "height": 1080,
+                        "width": 1920,
+                        "fps": 5,
+                    },
+                }
+            },
+        }
+        frigate_config = FrigateConfig(**config)
+        assert config == frigate_config.dict(exclude_unset=True)
+
+        runtime_config = frigate_config.runtime_config
+        assert "dog" in runtime_config.cameras["back"].objects.filters
+        assert runtime_config.cameras["back"].objects.filters["dog"].min_ratio == 0.2
+        assert runtime_config.cameras["back"].objects.filters["dog"].max_ratio == 10.1
+
 
 if __name__ == "__main__":
     unittest.main(verbosity=2)

frigate/util.py

@@ -522,7 +522,7 @@ def clipped(obj, frame_shape):
     # if the object is within 5 pixels of the region border, and the region is not on the edge
     # consider the object to be clipped
     box = obj[2]
-    region = obj[4]
+    region = obj[5]
     if (
         (region[0] > 5 and box[0] - region[0] <= 5)
         or (region[1] > 5 and box[1] - region[1] <= 5)

frigate/video.py

@@ -38,6 +38,10 @@ logger = logging.getLogger(__name__)
 
 def filtered(obj, objects_to_track, object_filters):
     object_name = obj[0]
+    object_score = obj[1]
+    object_box = obj[2]
+    object_area = obj[3]
+    object_ratio = obj[4]
 
     if not object_name in objects_to_track:
         return True
@@ -47,24 +51,35 @@ def filtered(obj, objects_to_track, object_filters):
 
         # if the min area is larger than the
         # detected object, don't add it to detected objects
-        if obj_settings.min_area > obj[3]:
+        if obj_settings.min_area > object_area:
             return True
 
         # if the detected object is larger than the
         # max area, don't add it to detected objects
-        if obj_settings.max_area < obj[3]:
+        if obj_settings.max_area < object_area:
             return True
 
         # if the score is lower than the min_score, skip
-        if obj_settings.min_score > obj[1]:
+        if obj_settings.min_score > object_score:
+            return True
+
+        # if the object is not proportionally wide enough
+        if obj_settings.min_ratio > object_ratio:
+            return True
+
+        # if the object is proportionally too wide
+        if obj_settings.max_ratio < object_ratio:
             return True
 
         if not obj_settings.mask is None:
             # compute the coordinates of the object and make sure
-            # the location isnt outside the bounds of the image (can happen from rounding)
+            # the location isn't outside the bounds of the image (can happen from rounding)
-            y_location = min(int(obj[2][3]), len(obj_settings.mask) - 1)
+            object_xmin = object_box[0]
+            object_xmax = object_box[2]
+            object_ymax = object_box[3]
+            y_location = min(int(object_ymax), len(obj_settings.mask) - 1)
             x_location = min(
-                int((obj[2][2] - obj[2][0]) / 2.0) + obj[2][0],
+                int((object_xmax + object_xmin) / 2.0),
                 len(obj_settings.mask[0]) - 1,
             )
 
@@ -429,11 +444,16 @@ def detect(
         y_min = int((box[0] * size) + region[1])
         x_max = int((box[3] * size) + region[0])
         y_max = int((box[2] * size) + region[1])
+        width = x_max - x_min
+        height = y_max - y_min
+        area = width * height
+        ratio = width / height
         det = (
             d[0],
             d[1],
             (x_min, y_min, x_max, y_max),
-            (x_max - x_min) * (y_max - y_min),
+            area,
+            ratio,
             region,
         )
         # apply object filters
@@ -580,6 +600,7 @@ def process_frames(
                     obj["score"],
                     obj["box"],
                     obj["area"],
+                    obj["ratio"],
                     obj["region"],
                 )
                 for obj in object_tracker.tracked_objects.values()
@@ -615,8 +636,14 @@ def process_frames(
                 for group in detected_object_groups.values():
 
                     # apply non-maxima suppression to suppress weak, overlapping bounding boxes
+                    # o[2] is the box of the object: xmin, ymin, xmax, ymax
                     boxes = [
-                        (o[2][0], o[2][1], o[2][2] - o[2][0], o[2][3] - o[2][1])
+                        (
+                            o[2][0],
+                            o[2][1],
+                            o[2][2] - o[2][0],
+                            o[2][3] - o[2][1],
+                        )
                         for o in group
                     ]
                     confidences = [o[1] for o in group]

migrations/009_add_object_filter_ratio.py

@@ -0,0 +1,38 @@
+"""Peewee migrations -- 009_add_object_filter_ratio.py.
+
+Some examples (model - class or model name)::
+
+    > Model = migrator.orm['model_name']            # Return model in current state by name
+
+    > migrator.sql(sql)                             # Run custom SQL
+    > migrator.python(func, *args, **kwargs)        # Run python code
+    > migrator.create_model(Model)                  # Create a model (could be used as decorator)
+    > migrator.remove_model(model, cascade=True)    # Remove a model
+    > migrator.add_fields(model, **fields)          # Add fields to a model
+    > migrator.change_fields(model, **fields)       # Change fields
+    > migrator.remove_fields(model, *field_names, cascade=True)
+    > migrator.rename_field(model, old_field_name, new_field_name)
+    > migrator.rename_table(model, new_table_name)
+    > migrator.add_index(model, *col_names, unique=False)
+    > migrator.drop_index(model, *col_names)
+    > migrator.add_not_null(model, *field_names)
+    > migrator.drop_not_null(model, *field_names)
+    > migrator.add_default(model, field_name, default)
+
+"""
+
+import peewee as pw
+from frigate.models import Event
+
+SQL = pw.SQL
+
+
+def migrate(migrator, database, fake=False, **kwargs):
+    migrator.add_fields(
+        Event,
+        ratio=pw.FloatField(default=1.0),  # Assume that existing detections are square
+    )
+
+
+def rollback(migrator, database, fake=False, **kwargs):
+    migrator.remove_fields(Event, ["ratio"])