From 15e4f5c77144a298b17629f3cc829f3ffe68130d Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Tue, 9 Apr 2024 16:51:38 -0600 Subject: [PATCH] use relative coordinates for masks & zones (#10912) * Handle zones and masks as relative coords * Ensure that zone coords are saved as relative * Get motion mask working with relative coordinates * Rewrite object mask to use relative coordinates as well * Formatting * Fix always trying to convert * fix mask logic --- frigate/api/app.py | 11 ++- frigate/api/review.py | 2 +- frigate/config.py | 114 ++++++++++++++++++++++++- frigate/test/test_config.py | 162 +++++++++++++++++++++++++++--------- frigate/util/image.py | 17 +++- 5 files changed, 258 insertions(+), 48 deletions(-) diff --git a/frigate/api/app.py b/frigate/api/app.py index d307e9384..a324b6a05 100644 --- a/frigate/api/app.py +++ b/frigate/api/app.py @@ -137,7 +137,10 @@ def stats_history(): @bp.route("/config") def config(): - config = current_app.frigate_config.model_dump(mode="json", exclude_none=True) + config_obj: FrigateConfig = current_app.frigate_config + config: dict[str, dict[str, any]] = config_obj.model_dump( + mode="json", exclude_none=True + ) # remove the mqtt password config["mqtt"].pop("password", None) @@ -154,9 +157,13 @@ def config(): for cmd in camera_dict["ffmpeg_cmds"]: cmd["cmd"] = clean_camera_user_pass(" ".join(cmd["cmd"])) + # ensure that zones are relative + for zone_name, zone in config_obj.cameras[camera_name].zones.items(): + camera_dict["zones"][zone_name]["color"] = zone.color + config["plus"] = {"enabled": current_app.plus_api.is_active()} - for detector, detector_config in config["detectors"].items(): + for detector_config in config["detectors"].values(): detector_config["model"]["labelmap"] = ( current_app.frigate_config.model.merged_labelmap ) diff --git a/frigate/api/review.py b/frigate/api/review.py index d3a49de9f..03d9dad91 100644 --- a/frigate/api/review.py +++ b/frigate/api/review.py @@ -434,7 +434,7 @@ def motion_activity(): .fillna(0.0) .to_frame() ) - cameras = df["camera"].resample(f"{scale}S").agg(lambda x: ",".join(set(x))) + cameras = df["camera"].resample(f"{scale}s").agg(lambda x: ",".join(set(x))) df = motion.join(cameras) length = df.shape[0] diff --git a/frigate/config.py b/frigate/config.py index 9317ae54c..5c4b890bc 100644 --- a/frigate/config.py +++ b/frigate/config.py @@ -354,6 +354,34 @@ class RuntimeMotionConfig(MotionConfig): frame_shape = config.get("frame_shape", (1, 1)) mask = config.get("mask", "") + + # masks and zones are saved as relative coordinates + # we know if any points are > 1 then it is using the + # old native resolution coordinates + if mask: + if isinstance(mask, list) and any(x > "1.0" for x in mask[0].split(",")): + relative_masks = [] + for m in mask: + points = m.split(",") + relative_masks.append( + ",".join( + [ + f"{round(int(points[i]) / frame_shape[1], 3)},{round(int(points[i + 1]) / frame_shape[0], 3)}" + for i in range(0, len(points), 2) + ] + ) + ) + + mask = relative_masks + elif isinstance(mask, str) and any(x > "1.0" for x in mask.split(",")): + points = mask.split(",") + mask = ",".join( + [ + f"{round(int(points[i]) / frame_shape[1], 3)},{round(int(points[i + 1]) / frame_shape[0], 3)}" + for i in range(0, len(points), 2) + ] + ) + config["raw_mask"] = mask if mask: @@ -484,11 +512,40 @@ class RuntimeFilterConfig(FilterConfig): raw_mask: Optional[Union[str, List[str]]] = None def __init__(self, **config): + frame_shape = config.get("frame_shape", (1, 1)) mask = config.get("mask") + + # masks and zones are saved as relative coordinates + # we know if any points are > 1 then it is using the + # old native resolution coordinates + if mask: + if isinstance(mask, list) and any(x > "1.0" for x in mask[0].split(",")): + relative_masks = [] + for m in mask: + points = m.split(",") + relative_masks.append( + ",".join( + [ + f"{round(int(points[i]) / frame_shape[1], 3)},{round(int(points[i + 1]) / frame_shape[0], 3)}" + for i in range(0, len(points), 2) + ] + ) + ) + + mask = relative_masks + elif isinstance(mask, str) and any(x > "1.0" for x in mask.split(",")): + points = mask.split(",") + mask = ",".join( + [ + f"{round(int(points[i]) / frame_shape[1], 3)},{round(int(points[i + 1]) / frame_shape[0], 3)}" + for i in range(0, len(points), 2) + ] + ) + config["raw_mask"] = mask if mask is not None: - config["mask"] = create_mask(config.get("frame_shape", (1, 1)), mask) + config["mask"] = create_mask(frame_shape, mask) super().__init__(**config) @@ -539,17 +596,61 @@ class ZoneConfig(BaseModel): super().__init__(**config) self._color = config.get("color", (0, 0, 0)) - coordinates = config["coordinates"] + self._contour = config.get("contour", np.array([])) + def generate_contour(self, frame_shape: tuple[int, int]): + coordinates = self.coordinates + + # masks and zones are saved as relative coordinates + # we know if any points are > 1 then it is using the + # old native resolution coordinates if isinstance(coordinates, list): + explicit = any(p.split(",")[0] > "1.0" for p in coordinates) self._contour = np.array( - [[int(p.split(",")[0]), int(p.split(",")[1])] for p in coordinates] + [ + ( + [int(p.split(",")[0]), int(p.split(",")[1])] + if explicit + else [ + int(float(p.split(",")[0]) * frame_shape[1]), + int(float(p.split(",")[1]) * frame_shape[0]), + ] + ) + for p in coordinates + ] ) + + if explicit: + self.coordinates = ",".join( + [ + f'{round(int(p.split(",")[0]) / frame_shape[1], 3)},{round(int(p.split(",")[1]) / frame_shape[0], 3)}' + for p in coordinates + ] + ) elif isinstance(coordinates, str): points = coordinates.split(",") + explicit = any(p > "1.0" for p in points) self._contour = np.array( - [[int(points[i]), int(points[i + 1])] for i in range(0, len(points), 2)] + [ + ( + [int(points[i]), int(points[i + 1])] + if explicit + else [ + int(float(points[i]) * frame_shape[1]), + int(float(points[i + 1]) * frame_shape[0]), + ] + ) + for i in range(0, len(points), 2) + ] ) + + if explicit: + self.coordinates = ",".join( + [ + f"{round(int(points[i]) / frame_shape[1], 3)},{round(int(points[i + 1]) / frame_shape[0], 3)}" + for i in range(0, len(points), 2) + ] + ) else: self._contour = np.array([]) @@ -1346,6 +1447,11 @@ class FrigateConfig(FrigateBaseModel): ) camera_config.motion.enabled_in_config = camera_config.motion.enabled + # generate zone contours + if len(camera_config.zones) > 0: + for zone in camera_config.zones.values(): + zone.generate_contour(camera_config.frame_shape) + # Set live view stream if none is set if not camera_config.live.stream_name: camera_config.live.stream_name = name diff --git a/frigate/test/test_config.py b/frigate/test/test_config.py index 949438540..be935d431 100644 --- a/frigate/test/test_config.py +++ b/frigate/test/test_config.py @@ -64,7 +64,7 @@ class TestConfig(unittest.TestCase): def test_config_class(self): frigate_config = FrigateConfig(**self.minimal) - assert self.minimal == frigate_config.dict(exclude_unset=True) + assert self.minimal == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert "cpu" in runtime_config.detectors.keys() @@ -157,7 +157,7 @@ class TestConfig(unittest.TestCase): }, } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert "dog" in runtime_config.cameras["back"].objects.track @@ -183,7 +183,7 @@ class TestConfig(unittest.TestCase): }, } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert not runtime_config.cameras["back"].birdseye.enabled @@ -209,7 +209,7 @@ class TestConfig(unittest.TestCase): }, } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert runtime_config.cameras["back"].birdseye.enabled @@ -234,7 +234,7 @@ class TestConfig(unittest.TestCase): }, } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert runtime_config.cameras["back"].birdseye.enabled @@ -263,7 +263,7 @@ class TestConfig(unittest.TestCase): }, } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert "cat" in runtime_config.cameras["back"].objects.track @@ -288,7 +288,7 @@ class TestConfig(unittest.TestCase): }, } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert "dog" in runtime_config.cameras["back"].objects.filters @@ -316,7 +316,7 @@ class TestConfig(unittest.TestCase): }, } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert "dog" in runtime_config.cameras["back"].objects.filters @@ -345,7 +345,7 @@ class TestConfig(unittest.TestCase): }, } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert "dog" in runtime_config.cameras["back"].objects.filters @@ -375,7 +375,7 @@ class TestConfig(unittest.TestCase): }, } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() back_camera = runtime_config.cameras["back"] @@ -383,6 +383,55 @@ class TestConfig(unittest.TestCase): assert len(back_camera.objects.filters["dog"].raw_mask) == 2 assert len(back_camera.objects.filters["person"].raw_mask) == 1 + def test_motion_mask_relative_matches_explicit(self): + config = { + "mqtt": {"host": "mqtt"}, + "record": { + "events": {"retain": {"default": 20, "objects": {"person": 30}}} + }, + "cameras": { + "explicit": { + "ffmpeg": { + "inputs": [ + {"path": "rtsp://10.0.0.1:554/video", "roles": ["detect"]} + ] + }, + "detect": { + "height": 400, + "width": 800, + "fps": 5, + }, + "motion": { + "mask": [ + "0,0,200,100,600,300,800,400", + ] + }, + }, + "relative": { + "ffmpeg": { + "inputs": [ + {"path": "rtsp://10.0.0.1:554/video", "roles": ["detect"]} + ] + }, + "detect": { + "height": 400, + "width": 800, + "fps": 5, + }, + "motion": { + "mask": [ + "0.0,0.0,0.25,0.25,0.75,0.75,1.0,1.0", + ] + }, + }, + }, + } + frigate_config = FrigateConfig(**config).runtime_config() + assert np.array_equal( + frigate_config.cameras["explicit"].motion.mask, + frigate_config.cameras["relative"].motion.mask, + ) + def test_default_input_args(self): config = { "mqtt": {"host": "mqtt"}, @@ -406,7 +455,7 @@ class TestConfig(unittest.TestCase): } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert "-rtsp_transport" in runtime_config.cameras["back"].ffmpeg_cmds[0]["cmd"] @@ -435,7 +484,7 @@ class TestConfig(unittest.TestCase): }, } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert "-re" in runtime_config.cameras["back"].ffmpeg_cmds[0]["cmd"] @@ -465,7 +514,7 @@ class TestConfig(unittest.TestCase): }, } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert "-re" in runtime_config.cameras["back"].ffmpeg_cmds[0]["cmd"] @@ -500,7 +549,7 @@ class TestConfig(unittest.TestCase): }, } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert "-re" in runtime_config.cameras["back"].ffmpeg_cmds[0]["cmd"] @@ -530,7 +579,7 @@ class TestConfig(unittest.TestCase): }, } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert ( @@ -608,7 +657,7 @@ class TestConfig(unittest.TestCase): }, } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert isinstance( @@ -616,6 +665,41 @@ class TestConfig(unittest.TestCase): ) assert runtime_config.cameras["back"].zones["test"].color != (0, 0, 0) + def test_zone_relative_matches_explicit(self): + config = { + "mqtt": {"host": "mqtt"}, + "record": { + "events": {"retain": {"default": 20, "objects": {"person": 30}}} + }, + "cameras": { + "back": { + "ffmpeg": { + "inputs": [ + {"path": "rtsp://10.0.0.1:554/video", "roles": ["detect"]} + ] + }, + "detect": { + "height": 400, + "width": 800, + "fps": 5, + }, + "zones": { + "explicit": { + "coordinates": "0,0,200,100,600,300,800,400", + }, + "relative": { + "coordinates": "0.0,0.0,0.25,0.25,0.75,0.75,1.0,1.0", + }, + }, + } + }, + } + frigate_config = FrigateConfig(**config).runtime_config() + assert np.array_equal( + frigate_config.cameras["back"].zones["explicit"].contour, + frigate_config.cameras["back"].zones["relative"].contour, + ) + def test_clips_should_default_to_global_objects(self): config = { "mqtt": {"host": "mqtt"}, @@ -640,7 +724,7 @@ class TestConfig(unittest.TestCase): }, } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() back_camera = runtime_config.cameras["back"] @@ -671,7 +755,7 @@ class TestConfig(unittest.TestCase): } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() ffmpeg_cmds = runtime_config.cameras["back"].ffmpeg_cmds @@ -702,7 +786,7 @@ class TestConfig(unittest.TestCase): } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert runtime_config.cameras["back"].detect.max_disappeared == 5 * 5 @@ -730,7 +814,7 @@ class TestConfig(unittest.TestCase): } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert runtime_config.cameras["back"].motion.frame_height == 100 @@ -758,7 +842,7 @@ class TestConfig(unittest.TestCase): } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert round(runtime_config.cameras["back"].motion.contour_area) == 10 @@ -787,7 +871,7 @@ class TestConfig(unittest.TestCase): } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert runtime_config.model.merged_labelmap[7] == "truck" @@ -815,7 +899,7 @@ class TestConfig(unittest.TestCase): } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert runtime_config.model.merged_labelmap[0] == "person" @@ -844,7 +928,7 @@ class TestConfig(unittest.TestCase): } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert runtime_config.model.merged_labelmap[0] == "person" @@ -878,7 +962,7 @@ class TestConfig(unittest.TestCase): } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config(PlusApi()) assert runtime_config.model.merged_labelmap[0] == "amazon" @@ -1012,7 +1096,7 @@ class TestConfig(unittest.TestCase): }, } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert runtime_config.cameras["back"].detect.max_disappeared == 1 @@ -1040,7 +1124,7 @@ class TestConfig(unittest.TestCase): }, } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert runtime_config.cameras["back"].detect.max_disappeared == 25 @@ -1069,7 +1153,7 @@ class TestConfig(unittest.TestCase): }, } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert runtime_config.cameras["back"].detect.max_disappeared == 1 @@ -1102,7 +1186,7 @@ class TestConfig(unittest.TestCase): }, } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert runtime_config.cameras["back"].snapshots.enabled @@ -1130,7 +1214,7 @@ class TestConfig(unittest.TestCase): }, } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert runtime_config.cameras["back"].snapshots.bounding_box @@ -1163,7 +1247,7 @@ class TestConfig(unittest.TestCase): }, } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert runtime_config.cameras["back"].snapshots.bounding_box is False @@ -1193,7 +1277,7 @@ class TestConfig(unittest.TestCase): }, } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert runtime_config.cameras["back"].live.quality == 4 @@ -1220,7 +1304,7 @@ class TestConfig(unittest.TestCase): }, } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert runtime_config.cameras["back"].live.quality == 8 @@ -1251,7 +1335,7 @@ class TestConfig(unittest.TestCase): }, } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert runtime_config.cameras["back"].live.quality == 7 @@ -1280,7 +1364,7 @@ class TestConfig(unittest.TestCase): }, } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert runtime_config.cameras["back"].timestamp_style.position == "bl" @@ -1307,7 +1391,7 @@ class TestConfig(unittest.TestCase): }, } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert runtime_config.cameras["back"].timestamp_style.position == "tl" @@ -1336,7 +1420,7 @@ class TestConfig(unittest.TestCase): }, } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert runtime_config.cameras["back"].timestamp_style.position == "bl" @@ -1365,7 +1449,7 @@ class TestConfig(unittest.TestCase): }, } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert runtime_config.cameras["back"].snapshots.retain.default == 1.5 @@ -1505,7 +1589,7 @@ class TestConfig(unittest.TestCase): }, } frigate_config = FrigateConfig(**config) - assert config == frigate_config.dict(exclude_unset=True) + assert config == frigate_config.model_dump(exclude_unset=True) runtime_config = frigate_config.runtime_config() assert "dog" in runtime_config.cameras["back"].objects.filters diff --git a/frigate/util/image.py b/frigate/util/image.py index ef6c75ae4..67f8b5c22 100644 --- a/frigate/util/image.py +++ b/frigate/util/image.py @@ -727,9 +727,22 @@ def create_mask(frame_shape, mask): return mask_img -def add_mask(mask, mask_img): +def add_mask(mask: str, mask_img: np.ndarray): points = mask.split(",") + + # masks and zones are saved as relative coordinates + # we know if any points are > 1 then it is using the + # old native resolution coordinates + if any(x > "1.0" for x in points): + raise Exception("add mask expects relative coordinates only") + contour = np.array( - [[int(points[i]), int(points[i + 1])] for i in range(0, len(points), 2)] + [ + [ + int(float(points[i]) * mask_img.shape[1]), + int(float(points[i + 1]) * mask_img.shape[0]), + ] + for i in range(0, len(points), 2) + ] ) cv2.fillPoly(mask_img, pts=[contour], color=(0))