use relative coordinates for masks & zones (#10912)

* Handle zones and masks as relative coords

* Ensure that zone coords are saved as relative

* Get motion mask working with relative coordinates

* Rewrite object mask to use relative coordinates as well

* Formatting

* Fix always trying to convert

* fix mask logic
This commit is contained in:
Nicolas Mowen 2024-04-09 16:51:38 -06:00 committed by GitHub
parent ef52a1d6f0
commit 15e4f5c771
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 258 additions and 48 deletions

View File

@ -137,7 +137,10 @@ def stats_history():
@bp.route("/config")
def config():
config = current_app.frigate_config.model_dump(mode="json", exclude_none=True)
config_obj: FrigateConfig = current_app.frigate_config
config: dict[str, dict[str, any]] = config_obj.model_dump(
mode="json", exclude_none=True
)
# remove the mqtt password
config["mqtt"].pop("password", None)
@ -154,9 +157,13 @@ def config():
for cmd in camera_dict["ffmpeg_cmds"]:
cmd["cmd"] = clean_camera_user_pass(" ".join(cmd["cmd"]))
# ensure that zones are relative
for zone_name, zone in config_obj.cameras[camera_name].zones.items():
camera_dict["zones"][zone_name]["color"] = zone.color
config["plus"] = {"enabled": current_app.plus_api.is_active()}
for detector, detector_config in config["detectors"].items():
for detector_config in config["detectors"].values():
detector_config["model"]["labelmap"] = (
current_app.frigate_config.model.merged_labelmap
)

View File

@ -434,7 +434,7 @@ def motion_activity():
.fillna(0.0)
.to_frame()
)
cameras = df["camera"].resample(f"{scale}S").agg(lambda x: ",".join(set(x)))
cameras = df["camera"].resample(f"{scale}s").agg(lambda x: ",".join(set(x)))
df = motion.join(cameras)
length = df.shape[0]

View File

@ -354,6 +354,34 @@ class RuntimeMotionConfig(MotionConfig):
frame_shape = config.get("frame_shape", (1, 1))
mask = config.get("mask", "")
# masks and zones are saved as relative coordinates
# we know if any points are > 1 then it is using the
# old native resolution coordinates
if mask:
if isinstance(mask, list) and any(x > "1.0" for x in mask[0].split(",")):
relative_masks = []
for m in mask:
points = m.split(",")
relative_masks.append(
",".join(
[
f"{round(int(points[i]) / frame_shape[1], 3)},{round(int(points[i + 1]) / frame_shape[0], 3)}"
for i in range(0, len(points), 2)
]
)
)
mask = relative_masks
elif isinstance(mask, str) and any(x > "1.0" for x in mask.split(",")):
points = mask.split(",")
mask = ",".join(
[
f"{round(int(points[i]) / frame_shape[1], 3)},{round(int(points[i + 1]) / frame_shape[0], 3)}"
for i in range(0, len(points), 2)
]
)
config["raw_mask"] = mask
if mask:
@ -484,11 +512,40 @@ class RuntimeFilterConfig(FilterConfig):
raw_mask: Optional[Union[str, List[str]]] = None
def __init__(self, **config):
frame_shape = config.get("frame_shape", (1, 1))
mask = config.get("mask")
# masks and zones are saved as relative coordinates
# we know if any points are > 1 then it is using the
# old native resolution coordinates
if mask:
if isinstance(mask, list) and any(x > "1.0" for x in mask[0].split(",")):
relative_masks = []
for m in mask:
points = m.split(",")
relative_masks.append(
",".join(
[
f"{round(int(points[i]) / frame_shape[1], 3)},{round(int(points[i + 1]) / frame_shape[0], 3)}"
for i in range(0, len(points), 2)
]
)
)
mask = relative_masks
elif isinstance(mask, str) and any(x > "1.0" for x in mask.split(",")):
points = mask.split(",")
mask = ",".join(
[
f"{round(int(points[i]) / frame_shape[1], 3)},{round(int(points[i + 1]) / frame_shape[0], 3)}"
for i in range(0, len(points), 2)
]
)
config["raw_mask"] = mask
if mask is not None:
config["mask"] = create_mask(config.get("frame_shape", (1, 1)), mask)
config["mask"] = create_mask(frame_shape, mask)
super().__init__(**config)
@ -539,17 +596,61 @@ class ZoneConfig(BaseModel):
super().__init__(**config)
self._color = config.get("color", (0, 0, 0))
coordinates = config["coordinates"]
self._contour = config.get("contour", np.array([]))
def generate_contour(self, frame_shape: tuple[int, int]):
coordinates = self.coordinates
# masks and zones are saved as relative coordinates
# we know if any points are > 1 then it is using the
# old native resolution coordinates
if isinstance(coordinates, list):
explicit = any(p.split(",")[0] > "1.0" for p in coordinates)
self._contour = np.array(
[[int(p.split(",")[0]), int(p.split(",")[1])] for p in coordinates]
[
(
[int(p.split(",")[0]), int(p.split(",")[1])]
if explicit
else [
int(float(p.split(",")[0]) * frame_shape[1]),
int(float(p.split(",")[1]) * frame_shape[0]),
]
)
for p in coordinates
]
)
if explicit:
self.coordinates = ",".join(
[
f'{round(int(p.split(",")[0]) / frame_shape[1], 3)},{round(int(p.split(",")[1]) / frame_shape[0], 3)}'
for p in coordinates
]
)
elif isinstance(coordinates, str):
points = coordinates.split(",")
explicit = any(p > "1.0" for p in points)
self._contour = np.array(
[[int(points[i]), int(points[i + 1])] for i in range(0, len(points), 2)]
[
(
[int(points[i]), int(points[i + 1])]
if explicit
else [
int(float(points[i]) * frame_shape[1]),
int(float(points[i + 1]) * frame_shape[0]),
]
)
for i in range(0, len(points), 2)
]
)
if explicit:
self.coordinates = ",".join(
[
f"{round(int(points[i]) / frame_shape[1], 3)},{round(int(points[i + 1]) / frame_shape[0], 3)}"
for i in range(0, len(points), 2)
]
)
else:
self._contour = np.array([])
@ -1346,6 +1447,11 @@ class FrigateConfig(FrigateBaseModel):
)
camera_config.motion.enabled_in_config = camera_config.motion.enabled
# generate zone contours
if len(camera_config.zones) > 0:
for zone in camera_config.zones.values():
zone.generate_contour(camera_config.frame_shape)
# Set live view stream if none is set
if not camera_config.live.stream_name:
camera_config.live.stream_name = name

View File

@ -64,7 +64,7 @@ class TestConfig(unittest.TestCase):
def test_config_class(self):
frigate_config = FrigateConfig(**self.minimal)
assert self.minimal == frigate_config.dict(exclude_unset=True)
assert self.minimal == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert "cpu" in runtime_config.detectors.keys()
@ -157,7 +157,7 @@ class TestConfig(unittest.TestCase):
},
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert "dog" in runtime_config.cameras["back"].objects.track
@ -183,7 +183,7 @@ class TestConfig(unittest.TestCase):
},
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert not runtime_config.cameras["back"].birdseye.enabled
@ -209,7 +209,7 @@ class TestConfig(unittest.TestCase):
},
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert runtime_config.cameras["back"].birdseye.enabled
@ -234,7 +234,7 @@ class TestConfig(unittest.TestCase):
},
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert runtime_config.cameras["back"].birdseye.enabled
@ -263,7 +263,7 @@ class TestConfig(unittest.TestCase):
},
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert "cat" in runtime_config.cameras["back"].objects.track
@ -288,7 +288,7 @@ class TestConfig(unittest.TestCase):
},
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert "dog" in runtime_config.cameras["back"].objects.filters
@ -316,7 +316,7 @@ class TestConfig(unittest.TestCase):
},
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert "dog" in runtime_config.cameras["back"].objects.filters
@ -345,7 +345,7 @@ class TestConfig(unittest.TestCase):
},
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert "dog" in runtime_config.cameras["back"].objects.filters
@ -375,7 +375,7 @@ class TestConfig(unittest.TestCase):
},
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
back_camera = runtime_config.cameras["back"]
@ -383,6 +383,55 @@ class TestConfig(unittest.TestCase):
assert len(back_camera.objects.filters["dog"].raw_mask) == 2
assert len(back_camera.objects.filters["person"].raw_mask) == 1
def test_motion_mask_relative_matches_explicit(self):
config = {
"mqtt": {"host": "mqtt"},
"record": {
"events": {"retain": {"default": 20, "objects": {"person": 30}}}
},
"cameras": {
"explicit": {
"ffmpeg": {
"inputs": [
{"path": "rtsp://10.0.0.1:554/video", "roles": ["detect"]}
]
},
"detect": {
"height": 400,
"width": 800,
"fps": 5,
},
"motion": {
"mask": [
"0,0,200,100,600,300,800,400",
]
},
},
"relative": {
"ffmpeg": {
"inputs": [
{"path": "rtsp://10.0.0.1:554/video", "roles": ["detect"]}
]
},
"detect": {
"height": 400,
"width": 800,
"fps": 5,
},
"motion": {
"mask": [
"0.0,0.0,0.25,0.25,0.75,0.75,1.0,1.0",
]
},
},
},
}
frigate_config = FrigateConfig(**config).runtime_config()
assert np.array_equal(
frigate_config.cameras["explicit"].motion.mask,
frigate_config.cameras["relative"].motion.mask,
)
def test_default_input_args(self):
config = {
"mqtt": {"host": "mqtt"},
@ -406,7 +455,7 @@ class TestConfig(unittest.TestCase):
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert "-rtsp_transport" in runtime_config.cameras["back"].ffmpeg_cmds[0]["cmd"]
@ -435,7 +484,7 @@ class TestConfig(unittest.TestCase):
},
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert "-re" in runtime_config.cameras["back"].ffmpeg_cmds[0]["cmd"]
@ -465,7 +514,7 @@ class TestConfig(unittest.TestCase):
},
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert "-re" in runtime_config.cameras["back"].ffmpeg_cmds[0]["cmd"]
@ -500,7 +549,7 @@ class TestConfig(unittest.TestCase):
},
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert "-re" in runtime_config.cameras["back"].ffmpeg_cmds[0]["cmd"]
@ -530,7 +579,7 @@ class TestConfig(unittest.TestCase):
},
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert (
@ -608,7 +657,7 @@ class TestConfig(unittest.TestCase):
},
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert isinstance(
@ -616,6 +665,41 @@ class TestConfig(unittest.TestCase):
)
assert runtime_config.cameras["back"].zones["test"].color != (0, 0, 0)
def test_zone_relative_matches_explicit(self):
config = {
"mqtt": {"host": "mqtt"},
"record": {
"events": {"retain": {"default": 20, "objects": {"person": 30}}}
},
"cameras": {
"back": {
"ffmpeg": {
"inputs": [
{"path": "rtsp://10.0.0.1:554/video", "roles": ["detect"]}
]
},
"detect": {
"height": 400,
"width": 800,
"fps": 5,
},
"zones": {
"explicit": {
"coordinates": "0,0,200,100,600,300,800,400",
},
"relative": {
"coordinates": "0.0,0.0,0.25,0.25,0.75,0.75,1.0,1.0",
},
},
}
},
}
frigate_config = FrigateConfig(**config).runtime_config()
assert np.array_equal(
frigate_config.cameras["back"].zones["explicit"].contour,
frigate_config.cameras["back"].zones["relative"].contour,
)
def test_clips_should_default_to_global_objects(self):
config = {
"mqtt": {"host": "mqtt"},
@ -640,7 +724,7 @@ class TestConfig(unittest.TestCase):
},
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
back_camera = runtime_config.cameras["back"]
@ -671,7 +755,7 @@ class TestConfig(unittest.TestCase):
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
ffmpeg_cmds = runtime_config.cameras["back"].ffmpeg_cmds
@ -702,7 +786,7 @@ class TestConfig(unittest.TestCase):
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert runtime_config.cameras["back"].detect.max_disappeared == 5 * 5
@ -730,7 +814,7 @@ class TestConfig(unittest.TestCase):
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert runtime_config.cameras["back"].motion.frame_height == 100
@ -758,7 +842,7 @@ class TestConfig(unittest.TestCase):
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert round(runtime_config.cameras["back"].motion.contour_area) == 10
@ -787,7 +871,7 @@ class TestConfig(unittest.TestCase):
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert runtime_config.model.merged_labelmap[7] == "truck"
@ -815,7 +899,7 @@ class TestConfig(unittest.TestCase):
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert runtime_config.model.merged_labelmap[0] == "person"
@ -844,7 +928,7 @@ class TestConfig(unittest.TestCase):
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert runtime_config.model.merged_labelmap[0] == "person"
@ -878,7 +962,7 @@ class TestConfig(unittest.TestCase):
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config(PlusApi())
assert runtime_config.model.merged_labelmap[0] == "amazon"
@ -1012,7 +1096,7 @@ class TestConfig(unittest.TestCase):
},
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert runtime_config.cameras["back"].detect.max_disappeared == 1
@ -1040,7 +1124,7 @@ class TestConfig(unittest.TestCase):
},
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert runtime_config.cameras["back"].detect.max_disappeared == 25
@ -1069,7 +1153,7 @@ class TestConfig(unittest.TestCase):
},
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert runtime_config.cameras["back"].detect.max_disappeared == 1
@ -1102,7 +1186,7 @@ class TestConfig(unittest.TestCase):
},
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert runtime_config.cameras["back"].snapshots.enabled
@ -1130,7 +1214,7 @@ class TestConfig(unittest.TestCase):
},
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert runtime_config.cameras["back"].snapshots.bounding_box
@ -1163,7 +1247,7 @@ class TestConfig(unittest.TestCase):
},
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert runtime_config.cameras["back"].snapshots.bounding_box is False
@ -1193,7 +1277,7 @@ class TestConfig(unittest.TestCase):
},
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert runtime_config.cameras["back"].live.quality == 4
@ -1220,7 +1304,7 @@ class TestConfig(unittest.TestCase):
},
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert runtime_config.cameras["back"].live.quality == 8
@ -1251,7 +1335,7 @@ class TestConfig(unittest.TestCase):
},
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert runtime_config.cameras["back"].live.quality == 7
@ -1280,7 +1364,7 @@ class TestConfig(unittest.TestCase):
},
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert runtime_config.cameras["back"].timestamp_style.position == "bl"
@ -1307,7 +1391,7 @@ class TestConfig(unittest.TestCase):
},
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert runtime_config.cameras["back"].timestamp_style.position == "tl"
@ -1336,7 +1420,7 @@ class TestConfig(unittest.TestCase):
},
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert runtime_config.cameras["back"].timestamp_style.position == "bl"
@ -1365,7 +1449,7 @@ class TestConfig(unittest.TestCase):
},
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert runtime_config.cameras["back"].snapshots.retain.default == 1.5
@ -1505,7 +1589,7 @@ class TestConfig(unittest.TestCase):
},
}
frigate_config = FrigateConfig(**config)
assert config == frigate_config.dict(exclude_unset=True)
assert config == frigate_config.model_dump(exclude_unset=True)
runtime_config = frigate_config.runtime_config()
assert "dog" in runtime_config.cameras["back"].objects.filters

View File

@ -727,9 +727,22 @@ def create_mask(frame_shape, mask):
return mask_img
def add_mask(mask, mask_img):
def add_mask(mask: str, mask_img: np.ndarray):
points = mask.split(",")
# masks and zones are saved as relative coordinates
# we know if any points are > 1 then it is using the
# old native resolution coordinates
if any(x > "1.0" for x in points):
raise Exception("add mask expects relative coordinates only")
contour = np.array(
[[int(points[i]), int(points[i + 1])] for i in range(0, len(points), 2)]
[
[
int(float(points[i]) * mask_img.shape[1]),
int(float(points[i + 1]) * mask_img.shape[0]),
]
for i in range(0, len(points), 2)
]
)
cv2.fillPoly(mask_img, pts=[contour], color=(0))