From 0d16bd0144791b3e37192c4d798730a3ce553183 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Mon, 24 Apr 2023 07:24:28 -0500 Subject: [PATCH] False positives (#6217) * add false positive submission * switch timeline events to x,y,w,h * update docs * fix type checks * convert to upsert * fix config test --- .devcontainer/devcontainer.json | 86 +++++---- docker/rootfs/usr/local/nginx/conf/nginx.conf | 1 + docs/docs/configuration/advanced.md | 1 - docs/docs/integrations/api.md | 18 +- frigate/config.py | 13 +- frigate/detectors/detector_config.py | 13 ++ frigate/detectors/plugins/cpu_tfl.py | 2 +- frigate/detectors/plugins/edgetpu_tfl.py | 2 +- frigate/events.py | 165 +++++++++-------- frigate/http.py | 101 ++++++++++- frigate/models.py | 4 + frigate/object_processing.py | 2 +- frigate/plus.py | 63 +++++++ frigate/test/test_config.py | 7 +- frigate/timeline.py | 95 +++------- frigate/util.py | 11 ++ migrations/014_event_updates_for_fp.py | 52 ++++++ web/src/routes/Events.jsx | 170 ++++++++++++++---- 18 files changed, 572 insertions(+), 234 deletions(-) create mode 100644 migrations/014_event_updates_for_fp.py diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 7c91f4f6f..3e654335f 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -37,42 +37,54 @@ "onAutoForward": "silent" } }, - "extensions": [ - "ms-python.vscode-pylance", - "ms-python.python", - "visualstudioexptteam.vscodeintellicode", - "mhutchie.git-graph", - "ms-azuretools.vscode-docker", - "streetsidesoftware.code-spell-checker", - "esbenp.prettier-vscode", - "dbaeumer.vscode-eslint", - "mikestead.dotenv", - "csstools.postcss", - "blanu.vscode-styled-jsx", - "bradlc.vscode-tailwindcss" - ], - "settings": { - "remote.autoForwardPorts": false, - "python.linting.pylintEnabled": true, - "python.linting.enabled": true, - "python.formatting.provider": "black", - "python.languageServer": "Pylance", - "editor.formatOnPaste": false, - "editor.formatOnSave": true, - "editor.formatOnType": true, - "python.testing.pytestEnabled": false, - "python.testing.unittestEnabled": true, - "python.testing.unittestArgs": ["-v", "-s", "./frigate/test"], - "files.trimTrailingWhitespace": true, - "eslint.workingDirectories": ["./web"], - "[json][jsonc]": { - "editor.defaultFormatter": "esbenp.prettier-vscode" - }, - "[jsx][js][tsx][ts]": { - "editor.codeActionsOnSave": ["source.addMissingImports", "source.fixAll"], - "editor.tabSize": 2 - }, - "cSpell.ignoreWords": ["rtmp"], - "cSpell.words": ["preact"] + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance", + "ms-python.black-formatter", + "visualstudioexptteam.vscodeintellicode", + "mhutchie.git-graph", + "ms-azuretools.vscode-docker", + "streetsidesoftware.code-spell-checker", + "esbenp.prettier-vscode", + "dbaeumer.vscode-eslint", + "mikestead.dotenv", + "csstools.postcss", + "blanu.vscode-styled-jsx", + "bradlc.vscode-tailwindcss" + ], + "settings": { + "remote.autoForwardPorts": false, + "python.linting.pylintEnabled": true, + "python.linting.enabled": true, + "python.formatting.provider": "none", + "python.languageServer": "Pylance", + "editor.formatOnPaste": false, + "editor.formatOnSave": true, + "editor.formatOnType": true, + "python.testing.pytestEnabled": false, + "python.testing.unittestEnabled": true, + "python.testing.unittestArgs": ["-v", "-s", "./frigate/test"], + "files.trimTrailingWhitespace": true, + "eslint.workingDirectories": ["./web"], + "[python]": { + "editor.defaultFormatter": "ms-python.black-formatter", + "editor.formatOnSave": true + }, + "[json][jsonc]": { + "editor.defaultFormatter": "esbenp.prettier-vscode" + }, + "[jsx][js][tsx][ts]": { + "editor.codeActionsOnSave": [ + "source.addMissingImports", + "source.fixAll" + ], + "editor.tabSize": 2 + }, + "cSpell.ignoreWords": ["rtmp"], + "cSpell.words": ["preact"] + } + } } } diff --git a/docker/rootfs/usr/local/nginx/conf/nginx.conf b/docker/rootfs/usr/local/nginx/conf/nginx.conf index eff884c14..72725efe4 100644 --- a/docker/rootfs/usr/local/nginx/conf/nginx.conf +++ b/docker/rootfs/usr/local/nginx/conf/nginx.conf @@ -221,6 +221,7 @@ http { add_header 'Access-Control-Allow-Origin' '*'; add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; proxy_pass http://frigate_api/; proxy_pass_request_headers on; proxy_set_header Host $host; diff --git a/docs/docs/configuration/advanced.md b/docs/docs/configuration/advanced.md index 818c4b9cd..40efd67e5 100644 --- a/docs/docs/configuration/advanced.md +++ b/docs/docs/configuration/advanced.md @@ -24,7 +24,6 @@ Examples of available modules are: - `frigate.app` - `frigate.mqtt` - `frigate.object_detection` -- `frigate.zeroconf` - `detector.` - `watchdog.` - `ffmpeg..` NOTE: All FFmpeg logs are sent as `error` level. diff --git a/docs/docs/integrations/api.md b/docs/docs/integrations/api.md index 2b89288db..cc5a6576a 100644 --- a/docs/docs/integrations/api.md +++ b/docs/docs/integrations/api.md @@ -172,11 +172,11 @@ Events from the database. Accepts the following query string parameters: Timeline of key moments of an event(s) from the database. Accepts the following query string parameters: -| param | Type | Description | -| -------------------- | ---- | --------------------------------------------- | -| `camera` | int | Name of camera | -| `source_id` | str | ID of tracked object | -| `limit` | int | Limit the number of events returned | +| param | Type | Description | +| ----------- | ---- | ----------------------------------- | +| `camera` | str | Name of camera | +| `source_id` | str | ID of tracked object | +| `limit` | int | Limit the number of events returned | ### `GET /api/events/summary` @@ -198,6 +198,14 @@ Sets retain to true for the event id. Submits the snapshot of the event to Frigate+ for labeling. +| param | Type | Description | +| -------------------- | ---- | ---------------------------------- | +| `include_annotation` | int | Submit annotation to Frigate+ too. | + +### `PUT /api/events//false_positive` + +Submits the snapshot of the event to Frigate+ for labeling and adds the detection as a false positive. + ### `DELETE /api/events//retain` Sets retain to false for the event id (event may be deleted quickly after removing). diff --git a/frigate/config.py b/frigate/config.py index b83f3cbde..b62fd29fe 100644 --- a/frigate/config.py +++ b/frigate/config.py @@ -18,6 +18,7 @@ from frigate.const import ( REGEX_CAMERA_NAME, YAML_EXT, ) +from frigate.detectors.detector_config import BaseDetectorConfig from frigate.util import ( create_mask, deep_merge, @@ -770,7 +771,7 @@ def verify_config_roles(camera_config: CameraConfig) -> None: def verify_valid_live_stream_name( frigate_config: FrigateConfig, camera_config: CameraConfig -) -> None: +) -> ValueError | None: """Verify that a restream exists to use for live view.""" if ( camera_config.live.stream_name @@ -848,7 +849,7 @@ class FrigateConfig(FrigateBaseModel): model: ModelConfig = Field( default_factory=ModelConfig, title="Detection model configuration." ) - detectors: Dict[str, DetectorConfig] = Field( + detectors: Dict[str, BaseDetectorConfig] = Field( default=DEFAULT_DETECTORS, title="Detector hardware configuration.", ) @@ -1031,7 +1032,15 @@ class FrigateConfig(FrigateBaseModel): detector_config.model.dict(exclude_unset=True), config.model.dict(exclude_unset=True), ) + + if not "path" in merged_model: + if detector_config.type == "cpu": + merged_model["path"] = "/cpu_model.tflite" + elif detector_config.type == "edgetpu": + merged_model["path"] = "/edgetpu_model.tflite" + detector_config.model = ModelConfig.parse_obj(merged_model) + detector_config.model.compute_model_hash() config.detectors[key] = detector_config return config diff --git a/frigate/detectors/detector_config.py b/frigate/detectors/detector_config.py index 25dede107..46f92e6ab 100644 --- a/frigate/detectors/detector_config.py +++ b/frigate/detectors/detector_config.py @@ -1,3 +1,4 @@ +import hashlib import logging from enum import Enum from typing import Dict, List, Optional, Tuple, Union, Literal @@ -49,6 +50,7 @@ class ModelConfig(BaseModel): ) _merged_labelmap: Optional[Dict[int, str]] = PrivateAttr() _colormap: Dict[int, Tuple[int, int, int]] = PrivateAttr() + _model_hash: str = PrivateAttr() @property def merged_labelmap(self) -> Dict[int, str]: @@ -58,6 +60,10 @@ class ModelConfig(BaseModel): def colormap(self) -> Dict[int, Tuple[int, int, int]]: return self._colormap + @property + def model_hash(self) -> str: + return self._model_hash + def __init__(self, **config): super().__init__(**config) @@ -67,6 +73,13 @@ class ModelConfig(BaseModel): } self._colormap = {} + def compute_model_hash(self) -> None: + with open(self.path, "rb") as f: + file_hash = hashlib.md5() + while chunk := f.read(8192): + file_hash.update(chunk) + self._model_hash = file_hash.hexdigest() + def create_colormap(self, enabled_labels: set[str]) -> None: """Get a list of colors for enabled labels.""" cmap = plt.cm.get_cmap("tab10", len(enabled_labels)) diff --git a/frigate/detectors/plugins/cpu_tfl.py b/frigate/detectors/plugins/cpu_tfl.py index fb9cbbfae..b22ac9a54 100644 --- a/frigate/detectors/plugins/cpu_tfl.py +++ b/frigate/detectors/plugins/cpu_tfl.py @@ -27,7 +27,7 @@ class CpuTfl(DetectionApi): def __init__(self, detector_config: CpuDetectorConfig): self.interpreter = Interpreter( - model_path=detector_config.model.path or "/cpu_model.tflite", + model_path=detector_config.model.path, num_threads=detector_config.num_threads or 3, ) diff --git a/frigate/detectors/plugins/edgetpu_tfl.py b/frigate/detectors/plugins/edgetpu_tfl.py index 840d41f66..6837adcb0 100644 --- a/frigate/detectors/plugins/edgetpu_tfl.py +++ b/frigate/detectors/plugins/edgetpu_tfl.py @@ -37,7 +37,7 @@ class EdgeTpuTfl(DetectionApi): edge_tpu_delegate = load_delegate("libedgetpu.so.1.0", device_config) logger.info("TPU found") self.interpreter = Interpreter( - model_path=detector_config.model.path or "/edgetpu_model.tflite", + model_path=detector_config.model.path, experimental_delegates=[edge_tpu_delegate], ) except ValueError: diff --git a/frigate/events.py b/frigate/events.py index 416b28fd7..558cb2139 100644 --- a/frigate/events.py +++ b/frigate/events.py @@ -12,6 +12,7 @@ from frigate.const import CLIPS_DIR from frigate.models import Event from frigate.timeline import TimelineSourceEnum from frigate.types import CameraMetricsTypes +from frigate.util import to_relative_box from multiprocessing.queues import Queue from multiprocessing.synchronize import Event as MpEvent @@ -20,22 +21,18 @@ from typing import Dict logger = logging.getLogger(__name__) -def should_insert_db(prev_event: Event, current_event: Event) -> bool: - """If current event has new clip or snapshot.""" - return (not prev_event["has_clip"] and not prev_event["has_snapshot"]) and ( - current_event["has_clip"] or current_event["has_snapshot"] - ) - - def should_update_db(prev_event: Event, current_event: Event) -> bool: """If current_event has updated fields and (clip or snapshot).""" if current_event["has_clip"] or current_event["has_snapshot"]: + # if this is the first time has_clip or has_snapshot turned true + if not prev_event["has_clip"] and not prev_event["has_snapshot"]: + return True + # or if any of the following values changed if ( prev_event["top_score"] != current_event["top_score"] or prev_event["entered_zones"] != current_event["entered_zones"] or prev_event["thumbnail"] != current_event["thumbnail"] - or prev_event["has_clip"] != current_event["has_clip"] - or prev_event["has_snapshot"] != current_event["has_snapshot"] + or prev_event["end_time"] != current_event["end_time"] ): return True return False @@ -85,81 +82,91 @@ class EventProcessor(threading.Thread): ) ) - event_config: EventsConfig = self.config.cameras[camera].record.events - + # if this is the first message, just store it and continue, its not time to insert it in the db if event_type == "start": self.events_in_process[event_data["id"]] = event_data + continue - elif event_type == "update" and should_insert_db( - self.events_in_process[event_data["id"]], event_data - ): + if should_update_db(self.events_in_process[event_data["id"]], event_data): + camera_config = self.config.cameras[camera] + event_config: EventsConfig = camera_config.record.events + width = camera_config.detect.width + height = camera_config.detect.height + first_detector = list(self.config.detectors.values())[0] + + start_time = event_data["start_time"] - event_config.pre_capture + end_time = ( + None + if event_data["end_time"] is None + else event_data["end_time"] + event_config.post_capture + ) + # score of the snapshot + score = ( + None + if event_data["snapshot"] is None + else event_data["snapshot"]["score"] + ) + # detection region in the snapshot + region = ( + None + if event_data["snapshot"] is None + else to_relative_box( + width, + height, + event_data["snapshot"]["region"], + ) + ) + # bounding box for the snapshot + box = ( + None + if event_data["snapshot"] is None + else to_relative_box( + width, + height, + event_data["snapshot"]["box"], + ) + ) + + # keep these from being set back to false because the event + # may have started while recordings and snapshots were enabled + # this would be an issue for long running events + if self.events_in_process[event_data["id"]]["has_clip"]: + event_data["has_clip"] = True + if self.events_in_process[event_data["id"]]["has_snapshot"]: + event_data["has_snapshot"] = True + + event = { + Event.id: event_data["id"], + Event.label: event_data["label"], + Event.camera: camera, + Event.start_time: start_time, + Event.end_time: end_time, + Event.top_score: event_data["top_score"], + Event.score: score, + Event.zones: list(event_data["entered_zones"]), + Event.thumbnail: event_data["thumbnail"], + Event.region: region, + Event.box: box, + Event.has_clip: event_data["has_clip"], + Event.has_snapshot: event_data["has_snapshot"], + Event.model_hash: first_detector.model.model_hash, + Event.model_type: first_detector.model.model_type, + Event.detector_type: first_detector.type, + } + + ( + Event.insert(event) + .on_conflict( + conflict_target=[Event.id], + update=event, + ) + .execute() + ) + + # update the stored copy for comparison on future update messages self.events_in_process[event_data["id"]] = event_data - # TODO: this will generate a lot of db activity possibly - Event.insert( - id=event_data["id"], - label=event_data["label"], - camera=camera, - start_time=event_data["start_time"] - event_config.pre_capture, - end_time=None, - top_score=event_data["top_score"], - false_positive=event_data["false_positive"], - zones=list(event_data["entered_zones"]), - thumbnail=event_data["thumbnail"], - region=event_data["region"], - box=event_data["box"], - area=event_data["area"], - has_clip=event_data["has_clip"], - has_snapshot=event_data["has_snapshot"], - ).execute() - - elif event_type == "update" and should_update_db( - self.events_in_process[event_data["id"]], event_data - ): - self.events_in_process[event_data["id"]] = event_data - # TODO: this will generate a lot of db activity possibly - Event.update( - label=event_data["label"], - camera=camera, - start_time=event_data["start_time"] - event_config.pre_capture, - end_time=None, - top_score=event_data["top_score"], - false_positive=event_data["false_positive"], - zones=list(event_data["entered_zones"]), - thumbnail=event_data["thumbnail"], - region=event_data["region"], - box=event_data["box"], - area=event_data["area"], - ratio=event_data["ratio"], - has_clip=event_data["has_clip"], - has_snapshot=event_data["has_snapshot"], - ).where(Event.id == event_data["id"]).execute() - - elif event_type == "end": - if event_data["has_clip"] or event_data["has_snapshot"]: - # Full update for valid end of event - Event.update( - label=event_data["label"], - camera=camera, - start_time=event_data["start_time"] - event_config.pre_capture, - end_time=event_data["end_time"] + event_config.post_capture, - top_score=event_data["top_score"], - false_positive=event_data["false_positive"], - zones=list(event_data["entered_zones"]), - thumbnail=event_data["thumbnail"], - region=event_data["region"], - box=event_data["box"], - area=event_data["area"], - ratio=event_data["ratio"], - has_clip=event_data["has_clip"], - has_snapshot=event_data["has_snapshot"], - ).where(Event.id == event_data["id"]).execute() - else: - # Event ended after clip & snapshot disabled, - # only end time should be updated. - Event.update( - end_time=event_data["end_time"] + event_config.post_capture - ).where(Event.id == event_data["id"]).execute() + if event_type == "end": del self.events_in_process[event_data["id"]] self.event_processed_queue.put((event_data["id"], camera)) diff --git a/frigate/http.py b/frigate/http.py index cb8c1cbe8..efe99c182 100644 --- a/frigate/http.py +++ b/frigate/http.py @@ -35,6 +35,7 @@ from frigate.config import FrigateConfig from frigate.const import CLIPS_DIR, MAX_SEGMENT_DURATION, RECORD_DIR from frigate.models import Event, Recordings, Timeline from frigate.object_processing import TrackedObject +from frigate.plus import PlusApi from frigate.stats import stats_snapshot from frigate.util import ( clean_camera_user_pass, @@ -42,6 +43,7 @@ from frigate.util import ( restart_frigate, vainfo_hwaccel, get_tz_modifiers, + to_relative_box, ) from frigate.storage import StorageMaintainer from frigate.version import VERSION @@ -57,7 +59,7 @@ def create_app( stats_tracking, detected_frames_processor, storage_maintainer: StorageMaintainer, - plus_api, + plus_api: PlusApi, ): app = Flask(__name__) @@ -179,6 +181,10 @@ def send_to_plus(id): 400, ) + include_annotation = ( + request.json.get("include_annotation") if request.is_json else None + ) + try: event = Event.get(Event.id == id) except DoesNotExist: @@ -186,6 +192,10 @@ def send_to_plus(id): logger.error(message) return make_response(jsonify({"success": False, "message": message}), 404) + # events from before the conversion to relative dimensions cant include annotations + if any(d > 1 for d in event.box): + include_annotation = None + if event.end_time is None: logger.error(f"Unable to load clean png for in-progress event: {event.id}") return make_response( @@ -238,9 +248,96 @@ def send_to_plus(id): event.plus_id = plus_id event.save() + if not include_annotation is None: + region = event.region + box = event.box + + try: + current_app.plus_api.add_annotation( + event.plus_id, + box, + event.label, + ) + except Exception as ex: + logger.exception(ex) + return make_response( + jsonify({"success": False, "message": str(ex)}), + 400, + ) + return make_response(jsonify({"success": True, "plus_id": plus_id}), 200) +@bp.route("/events//false_positive", methods=("PUT",)) +def false_positive(id): + if not current_app.plus_api.is_active(): + message = "PLUS_API_KEY environment variable is not set" + logger.error(message) + return make_response( + jsonify( + { + "success": False, + "message": message, + } + ), + 400, + ) + + try: + event = Event.get(Event.id == id) + except DoesNotExist: + message = f"Event {id} not found" + logger.error(message) + return make_response(jsonify({"success": False, "message": message}), 404) + + # events from before the conversion to relative dimensions cant include annotations + if any(d > 1 for d in event.box): + message = f"Events prior to 0.13 cannot be submitted as false positives" + logger.error(message) + return make_response(jsonify({"success": False, "message": message}), 400) + + if event.false_positive: + message = f"False positive already submitted to Frigate+" + logger.error(message) + return make_response(jsonify({"success": False, "message": message}), 400) + + if not event.plus_id: + plus_response = send_to_plus(id) + if plus_response.status_code != 200: + return plus_response + # need to refetch the event now that it has a plus_id + event = Event.get(Event.id == id) + + region = event.region + box = event.box + + # provide top score if score is unavailable + score = event.top_score if event.score is None else event.score + + try: + current_app.plus_api.add_false_positive( + event.plus_id, + region, + box, + score, + event.label, + event.model_hash, + event.model_type, + event.detector_type, + ) + except Exception as ex: + logger.exception(ex) + return make_response( + jsonify({"success": False, "message": str(ex)}), + 400, + ) + + event.false_positive = True + event.save() + + return make_response(jsonify({"success": True, "plus_id": event.plus_id}), 200) + + @bp.route("/events//retain", methods=("DELETE",)) def delete_retain(id): try: @@ -654,6 +751,8 @@ def events(): Event.retain_indefinitely, Event.sub_label, Event.top_score, + Event.false_positive, + Event.box, ] if camera != "all": diff --git a/frigate/models.py b/frigate/models.py index 9cfd89988..cc1a29f31 100644 --- a/frigate/models.py +++ b/frigate/models.py @@ -19,6 +19,7 @@ class Event(Model): # type: ignore[misc] start_time = DateTimeField() end_time = DateTimeField() top_score = FloatField() + score = FloatField() false_positive = BooleanField() zones = JSONField() thumbnail = TextField() @@ -30,6 +31,9 @@ class Event(Model): # type: ignore[misc] retain_indefinitely = BooleanField(default=False) ratio = FloatField(default=1.0) plus_id = CharField(max_length=30) + model_hash = CharField(max_length=32) + detector_type = CharField(max_length=32) + model_type = CharField(max_length=32) class Timeline(Model): # type: ignore[misc] diff --git a/frigate/object_processing.py b/frigate/object_processing.py index 97ef6f1ef..db2745309 100644 --- a/frigate/object_processing.py +++ b/frigate/object_processing.py @@ -185,7 +185,7 @@ class TrackedObject: "id": self.obj_data["id"], "camera": self.camera, "frame_time": self.obj_data["frame_time"], - "snapshot_time": snapshot_time, + "snapshot": self.thumbnail_data, "label": self.obj_data["label"], "sub_label": self.obj_data.get("sub_label"), "top_score": self.top_score, diff --git a/frigate/plus.py b/frigate/plus.py index faf70752e..4b59b961e 100644 --- a/frigate/plus.py +++ b/frigate/plus.py @@ -3,6 +3,7 @@ import json import logging import os import re +from typing import List import requests from frigate.const import PLUS_ENV_VAR, PLUS_API_HOST from requests.models import Response @@ -79,6 +80,13 @@ class PlusApi: json=data, ) + def _put(self, path: str, data: dict) -> Response: + return requests.put( + f"{self.host}/v1/{path}", + headers=self._get_authorization_header(), + json=data, + ) + def is_active(self) -> bool: return self._is_active @@ -124,3 +132,58 @@ class PlusApi: # return image id return str(presigned_urls.get("imageId")) + + def add_false_positive( + self, + plus_id: str, + region: List[float], + bbox: List[float], + score: float, + label: str, + model_hash: str, + model_type: str, + detector_type: str, + ) -> None: + r = self._put( + f"image/{plus_id}/false_positive", + { + "label": label, + "x": bbox[0], + "y": bbox[1], + "w": bbox[2], + "h": bbox[3], + "regionX": region[0], + "regionY": region[1], + "regionW": region[2], + "regionH": region[3], + "score": score, + "model_hash": model_hash, + "model_type": model_type, + "detector_type": detector_type, + }, + ) + + if not r.ok: + raise Exception(r.text) + + def add_annotation( + self, + plus_id: str, + bbox: List[float], + label: str, + difficult: bool = False, + ) -> None: + r = self._put( + f"image/{plus_id}/annotation", + { + "label": label, + "x": bbox[0], + "y": bbox[1], + "w": bbox[2], + "h": bbox[3], + "difficult": difficult, + }, + ) + + if not r.ok: + raise Exception(r.text) diff --git a/frigate/test/test_config.py b/frigate/test/test_config.py index be9bdcdfd..b978453c1 100644 --- a/frigate/test/test_config.py +++ b/frigate/test/test_config.py @@ -54,7 +54,8 @@ class TestConfig(unittest.TestCase): "type": "openvino", }, }, - "model": {"path": "/default.tflite", "width": 512}, + # needs to be a file that will exist, doesnt matter what + "model": {"path": "/etc/hosts", "width": 512}, } frigate_config = FrigateConfig(**(deep_merge(config, self.minimal))) @@ -72,10 +73,10 @@ class TestConfig(unittest.TestCase): assert runtime_config.detectors["edgetpu"].device is None assert runtime_config.detectors["openvino"].device is None - assert runtime_config.model.path == "/default.tflite" + assert runtime_config.model.path == "/etc/hosts" assert runtime_config.detectors["cpu"].model.path == "/cpu_model.tflite" assert runtime_config.detectors["edgetpu"].model.path == "/edgetpu_model.tflite" - assert runtime_config.detectors["openvino"].model.path == "/default.tflite" + assert runtime_config.detectors["openvino"].model.path == "/etc/hosts" assert runtime_config.model.width == 512 assert runtime_config.detectors["cpu"].model.width == 512 diff --git a/frigate/timeline.py b/frigate/timeline.py index 5c3c11f3d..c351e3e68 100644 --- a/frigate/timeline.py +++ b/frigate/timeline.py @@ -12,6 +12,8 @@ from frigate.models import Timeline from multiprocessing.queues import Queue from multiprocessing.synchronize import Event as MpEvent +from frigate.util import to_relative_box + logger = logging.getLogger(__name__) @@ -64,77 +66,36 @@ class TimelineProcessor(threading.Thread): """Handle object detection.""" camera_config = self.config.cameras[camera] + timeline_entry = { + Timeline.timestamp: event_data["frame_time"], + Timeline.camera: camera, + Timeline.source: "tracked_object", + Timeline.source_id: event_data["id"], + Timeline.data: { + "box": to_relative_box( + camera_config.detect.width, + camera_config.detect.height, + event_data["box"], + ), + "label": event_data["label"], + "region": to_relative_box( + camera_config.detect.width, + camera_config.detect.height, + event_data["region"], + ), + }, + } if event_type == "start": - Timeline.insert( - timestamp=event_data["frame_time"], - camera=camera, - source="tracked_object", - source_id=event_data["id"], - class_type="visible", - data={ - "box": [ - event_data["box"][0] / camera_config.detect.width, - event_data["box"][1] / camera_config.detect.height, - event_data["box"][2] / camera_config.detect.width, - event_data["box"][3] / camera_config.detect.height, - ], - "label": event_data["label"], - "region": [ - event_data["region"][0] / camera_config.detect.width, - event_data["region"][1] / camera_config.detect.height, - event_data["region"][2] / camera_config.detect.width, - event_data["region"][3] / camera_config.detect.height, - ], - }, - ).execute() + timeline_entry[Timeline.class_type] = "visible" + Timeline.insert(timeline_entry).execute() elif ( event_type == "update" and prev_event_data["current_zones"] != event_data["current_zones"] and len(event_data["current_zones"]) > 0 ): - Timeline.insert( - timestamp=event_data["frame_time"], - camera=camera, - source="tracked_object", - source_id=event_data["id"], - class_type="entered_zone", - data={ - "box": [ - event_data["box"][0] / camera_config.detect.width, - event_data["box"][1] / camera_config.detect.height, - event_data["box"][2] / camera_config.detect.width, - event_data["box"][3] / camera_config.detect.height, - ], - "label": event_data["label"], - "region": [ - event_data["region"][0] / camera_config.detect.width, - event_data["region"][1] / camera_config.detect.height, - event_data["region"][2] / camera_config.detect.width, - event_data["region"][3] / camera_config.detect.height, - ], - "zones": event_data["current_zones"], - }, - ).execute() + timeline_entry[Timeline.class_type] = "entered_zone" + timeline_entry[Timeline.data]["zones"] = event_data["current_zones"] + Timeline.insert(timeline_entry).execute() elif event_type == "end": - Timeline.insert( - timestamp=event_data["frame_time"], - camera=camera, - source="tracked_object", - source_id=event_data["id"], - class_type="gone", - data={ - "box": [ - event_data["box"][0] / camera_config.detect.width, - event_data["box"][1] / camera_config.detect.height, - event_data["box"][2] / camera_config.detect.width, - event_data["box"][3] / camera_config.detect.height, - ], - "label": event_data["label"], - "region": [ - event_data["region"][0] / camera_config.detect.width, - event_data["region"][1] / camera_config.detect.height, - event_data["region"][2] / camera_config.detect.width, - event_data["region"][3] / camera_config.detect.height, - ], - }, - ).execute() + timeline_entry[Timeline.class_type] = "gone" + Timeline.insert(timeline_entry).execute() diff --git a/frigate/util.py b/frigate/util.py index 887baf8de..12139bf99 100755 --- a/frigate/util.py +++ b/frigate/util.py @@ -1065,3 +1065,14 @@ def get_tz_modifiers(tz_name: str) -> Tuple[str, str]: hour_modifier = f"{hours_offset} hour" minute_modifier = f"{minutes_offset} minute" return hour_modifier, minute_modifier + + +def to_relative_box( + width: int, height: int, box: Tuple[int, int, int, int] +) -> Tuple[int, int, int, int]: + return ( + box[0] / width, # x + box[1] / height, # y + (box[2] - box[0]) / width, # w + (box[3] - box[1]) / height, # h + ) diff --git a/migrations/014_event_updates_for_fp.py b/migrations/014_event_updates_for_fp.py new file mode 100644 index 000000000..5d075596b --- /dev/null +++ b/migrations/014_event_updates_for_fp.py @@ -0,0 +1,52 @@ +"""Peewee migrations + +Some examples (model - class or model name):: + + > Model = migrator.orm['model_name'] # Return model in current state by name + + > migrator.sql(sql) # Run custom SQL + > migrator.python(func, *args, **kwargs) # Run python code + > migrator.create_model(Model) # Create a model (could be used as decorator) + > migrator.remove_model(model, cascade=True) # Remove a model + > migrator.add_fields(model, **fields) # Add fields to a model + > migrator.change_fields(model, **fields) # Change fields + > migrator.remove_fields(model, *field_names, cascade=True) + > migrator.rename_field(model, old_field_name, new_field_name) + > migrator.rename_table(model, new_table_name) + > migrator.add_index(model, *col_names, unique=False) + > migrator.drop_index(model, *col_names) + > migrator.add_not_null(model, *field_names) + > migrator.drop_not_null(model, *field_names) + > migrator.add_default(model, field_name, default) + +""" + +import datetime as dt +import peewee as pw +from playhouse.sqlite_ext import * +from decimal import ROUND_HALF_EVEN +from frigate.models import Event + +try: + import playhouse.postgres_ext as pw_pext +except ImportError: + pass + +SQL = pw.SQL + + +def migrate(migrator, database, fake=False, **kwargs): + migrator.add_fields( + Event, + score=pw.FloatField(null=True), + model_hash=pw.CharField(max_length=32, null=True), + detector_type=pw.CharField(max_length=32, null=True), + model_type=pw.CharField(max_length=32, null=True), + ) + + migrator.drop_not_null(Event, "area", "false_positive") + migrator.add_default(Event, "false_positive", 0) + + +def rollback(migrator, database, fake=False, **kwargs): + pass diff --git a/web/src/routes/Events.jsx b/web/src/routes/Events.jsx index 836b99ae6..af0af5b67 100644 --- a/web/src/routes/Events.jsx +++ b/web/src/routes/Events.jsx @@ -3,6 +3,7 @@ import { route } from 'preact-router'; import ActivityIndicator from '../components/ActivityIndicator'; import Heading from '../components/Heading'; import { Tabs, TextTab } from '../components/Tabs'; +import Link from '../components/Link'; import { useApiHost } from '../api'; import useSWR from 'swr'; import useSWRInfinite from 'swr/infinite'; @@ -57,7 +58,12 @@ export default function Events({ path, ...props }) { showDownloadMenu: false, showDatePicker: false, showCalendar: false, - showPlusConfig: false, + showPlusSubmit: false, + }); + const [plusSubmitEvent, setPlusSubmitEvent] = useState({ + id: null, + label: null, + validBox: null, }); const [uploading, setUploading] = useState([]); const [viewEvent, setViewEvent] = useState(); @@ -65,6 +71,8 @@ export default function Events({ path, ...props }) { const [eventDetailType, setEventDetailType] = useState('clip'); const [downloadEvent, setDownloadEvent] = useState({ id: null, + label: null, + box: null, has_clip: false, has_snapshot: false, plus_id: undefined, @@ -198,6 +206,8 @@ export default function Events({ path, ...props }) { e.stopPropagation(); setDownloadEvent((_prev) => ({ id: event.id, + box: event.box, + label: event.label, has_clip: event.has_clip, has_snapshot: event.has_snapshot, plus_id: event.plus_id, @@ -207,6 +217,16 @@ export default function Events({ path, ...props }) { setState({ ...state, showDownloadMenu: true }); }; + const showSubmitToPlus = (event_id, label, box, e) => { + if (e) { + e.stopPropagation(); + } + // if any of the box coordinates are > 1, then the box data is from an older version + // and not valid to submit to plus with the snapshot image + setPlusSubmitEvent({ id: event_id, label, validBox: !box.some((d) => d > 1) }); + setState({ ...state, showDownloadMenu: false, showPlusSubmit: true }); + }; + const handleSelectDateRange = useCallback( (dates) => { setSearchParams({ ...searchParams, before: dates.before, after: dates.after }); @@ -251,23 +271,16 @@ export default function Events({ path, ...props }) { [size, setSize, isValidating, isDone] ); - const onSendToPlus = async (id, e) => { - if (e) { - e.stopPropagation(); - } - + const onSendToPlus = async (id, false_positive, validBox) => { if (uploading.includes(id)) { return; } - if (!config.plus.enabled) { - setState({ ...state, showDownloadMenu: false, showPlusConfig: true }); - return; - } - setUploading((prev) => [...prev, id]); - const response = await axios.post(`events/${id}/plus`); + const response = false_positive + ? await axios.put(`events/${id}/false_positive`) + : await axios.post(`events/${id}/plus`, validBox ? { include_annotation: 1 } : {}); if (response.status === 200) { mutate( @@ -289,6 +302,8 @@ export default function Events({ path, ...props }) { if (state.showDownloadMenu && downloadEvent.id === id) { setState({ ...state, showDownloadMenu: false }); } + + setState({ ...state, showPlusSubmit: false }); }; const handleEventDetailTabChange = (index) => { @@ -375,12 +390,12 @@ export default function Events({ path, ...props }) { download /> )} - {(downloadEvent.end_time && downloadEvent.has_snapshot && !downloadEvent.plus_id) && ( + {downloadEvent.end_time && downloadEvent.has_snapshot && !downloadEvent.plus_id && ( onSendToPlus(downloadEvent.id)} + onSelect={() => showSubmitToPlus(downloadEvent.id, downloadEvent.label, downloadEvent.box)} /> )} {downloadEvent.plus_id && ( @@ -435,25 +450,96 @@ export default function Events({ path, ...props }) { /> )} - {state.showPlusConfig && ( + {state.showPlusSubmit && ( -
- Setup a Frigate+ Account -

In order to submit images to Frigate+, you first need to setup an account.

- - https://plus.frigate.video - -
-
- -
+ {config.plus.enabled ? ( + <> +
+ Submit to Frigate+ + + {`${plusSubmitEvent.label}`} + + {plusSubmitEvent.validBox ? ( +

+ Objects in locations you want to avoid are not false positives. Submitting them as false positives + will confuse the model. +

+ ) : ( +

+ Events prior to version 0.13 can only be submitted to Frigate+ without annotations. +

+ )} +
+ {plusSubmitEvent.validBox ? ( +
+ + + +
+ ) : ( +
+ + +
+ )} + + ) : ( + <> +
+ Setup a Frigate+ Account +

In order to submit images to Frigate+, you first need to setup an account.

+ + https://plus.frigate.video + +
+
+ +
+ + )} )} {deleteFavoriteState.showDeleteFavorite && ( @@ -539,12 +625,20 @@ export default function Events({ path, ...props }) { {event.end_time && event.has_snapshot && ( {event.plus_id ? ( -
Sent to Frigate+
+
+ + Edit in Frigate+ + +
) : ( @@ -617,8 +711,12 @@ export default function Events({ path, ...props }) { style={{ left: `${Math.round(eventOverlay.data.box[0] * 100)}%`, top: `${Math.round(eventOverlay.data.box[1] * 100)}%`, - right: `${Math.round((1 - eventOverlay.data.box[2]) * 100)}%`, - bottom: `${Math.round((1 - eventOverlay.data.box[3]) * 100)}%`, + right: `${Math.round( + (1 - eventOverlay.data.box[2] - eventOverlay.data.box[0]) * 100 + )}%`, + bottom: `${Math.round( + (1 - eventOverlay.data.box[3] - eventOverlay.data.box[1]) * 100 + )}%`, }} > {eventOverlay.class_type == 'entered_zone' ? (