From c5b8d13beb63b9429b76cc450646b6ad99b261ae Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Mon, 17 Jul 2023 05:07:15 -0600 Subject: [PATCH] Save audio scores and add audio filter config (#7185) * Send and save score for external events * Add audio filters config * Fix access * Add docs * Cleanup --- docs/docs/configuration/index.md | 6 ++++++ frigate/config.py | 18 +++++++++++++++++- frigate/const.py | 1 + frigate/events/audio.py | 10 +++++++--- frigate/events/external.py | 2 ++ frigate/events/maintainer.py | 6 +++++- frigate/http.py | 1 + 7 files changed, 39 insertions(+), 5 deletions(-) diff --git a/docs/docs/configuration/index.md b/docs/docs/configuration/index.md index d3573251a..05177798d 100644 --- a/docs/docs/configuration/index.md +++ b/docs/docs/configuration/index.md @@ -154,6 +154,12 @@ audio: - scream - speech - yell + # Optional: Filters to configure detection. + filters: + # Label that matches label in listen config. + speech: + # Minimum score that triggers an audio event (default: shown below) + threshold: 0.8 # Optional: logger verbosity settings logger: diff --git a/frigate/config.py b/frigate/config.py index bca0ba080..59e086989 100644 --- a/frigate/config.py +++ b/frigate/config.py @@ -11,7 +11,13 @@ import numpy as np from pydantic import BaseModel, Extra, Field, parse_obj_as, validator from pydantic.fields import PrivateAttr -from frigate.const import CACHE_DIR, DEFAULT_DB_PATH, REGEX_CAMERA_NAME, YAML_EXT +from frigate.const import ( + AUDIO_MIN_CONFIDENCE, + CACHE_DIR, + DEFAULT_DB_PATH, + REGEX_CAMERA_NAME, + YAML_EXT, +) from frigate.detectors import DetectorConfig, ModelConfig from frigate.detectors.detector_config import BaseDetectorConfig from frigate.ffmpeg_presets import ( @@ -334,6 +340,15 @@ class FilterConfig(FrigateBaseModel): ) +class AudioFilterConfig(FrigateBaseModel): + threshold: float = Field( + default=0.8, + ge=AUDIO_MIN_CONFIDENCE, + lt=1.0, + title="Minimum detection confidence threshold for audio to be counted.", + ) + + class RuntimeFilterConfig(FilterConfig): mask: Optional[np.ndarray] raw_mask: Optional[Union[str, List[str]]] @@ -424,6 +439,7 @@ class AudioConfig(FrigateBaseModel): listen: List[str] = Field( default=DEFAULT_LISTEN_AUDIO, title="Audio to listen for." ) + filters: Optional[Dict[str, AudioFilterConfig]] = Field(title="Audio filters.") enabled_in_config: Optional[bool] = Field( title="Keep track of original state of audio detection." ) diff --git a/frigate/const.py b/frigate/const.py index b6b0e44bd..4dec7f366 100644 --- a/frigate/const.py +++ b/frigate/const.py @@ -28,6 +28,7 @@ AUDIO_DURATION = 0.975 AUDIO_FORMAT = "s16le" AUDIO_MAX_BIT_RANGE = 32768.0 AUDIO_SAMPLE_RATE = 16000 +AUDIO_MIN_CONFIDENCE = 0.5 # Regex Consts diff --git a/frigate/events/audio.py b/frigate/events/audio.py index d2b3d8298..9fb134fc8 100644 --- a/frigate/events/audio.py +++ b/frigate/events/audio.py @@ -19,6 +19,7 @@ from frigate.const import ( AUDIO_DURATION, AUDIO_FORMAT, AUDIO_MAX_BIT_RANGE, + AUDIO_MIN_CONFIDENCE, AUDIO_SAMPLE_RATE, CACHE_DIR, FRIGATE_LOCALHOST, @@ -130,7 +131,7 @@ class AudioTfl: return detections - def detect(self, tensor_input, threshold=0.8): + def detect(self, tensor_input, threshold=AUDIO_MIN_CONFIDENCE): detections = [] if self.stop_event.is_set(): @@ -200,7 +201,10 @@ class AudioEventMaintainer(threading.Thread): if label not in self.config.audio.listen: continue - self.handle_detection(label, score) + if score > (self.config.audio.filters or {}).get(label, {}).get( + "threshold", 0.8 + ): + self.handle_detection(label, score) self.expire_detections() @@ -233,7 +237,7 @@ class AudioEventMaintainer(threading.Thread): resp = requests.post( f"{FRIGATE_LOCALHOST}/api/events/{self.config.name}/{label}/create", - json={"duration": None, "source_type": "audio"}, + json={"duration": None, "score": score, "source_type": "audio"}, ) if resp.status_code == 200: diff --git a/frigate/events/external.py b/frigate/events/external.py index 376f47211..c30f041a8 100644 --- a/frigate/events/external.py +++ b/frigate/events/external.py @@ -31,6 +31,7 @@ class ExternalEventProcessor: label: str, source_type: str, sub_label: Optional[str], + score: int, duration: Optional[int], include_recording: bool, draw: dict[str, any], @@ -56,6 +57,7 @@ class ExternalEventProcessor: "id": event_id, "label": label, "sub_label": sub_label, + "score": score, "camera": camera, "start_time": now - camera_config.record.events.pre_capture, "end_time": now diff --git a/frigate/events/maintainer.py b/frigate/events/maintainer.py index 0c9986693..f2989c065 100644 --- a/frigate/events/maintainer.py +++ b/frigate/events/maintainer.py @@ -230,7 +230,11 @@ class EventProcessor(threading.Thread): Event.has_clip: event_data["has_clip"], Event.has_snapshot: event_data["has_snapshot"], Event.zones: [], - Event.data: {"type": event_data["type"]}, + Event.data: { + "type": event_data["type"], + "score": event_data["score"], + "top_score": event_data["score"], + }, } Event.insert(event).execute() elif event_type == "end": diff --git a/frigate/http.py b/frigate/http.py index 7859605e6..f55fc7862 100644 --- a/frigate/http.py +++ b/frigate/http.py @@ -899,6 +899,7 @@ def create_event(camera_name, label): label, json.get("source_type", "api"), json.get("sub_label", None), + json.get("score", 0), json.get("duration", 30), json.get("include_recording", True), json.get("draw", {}),