From d1082ec305ac9b4120b2890918d374b0dcc93eb8 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Tue, 2 Apr 2024 07:07:50 -0600 Subject: [PATCH] Support manual detections in review items (#10784) * Only update frame time if it is older * Support manual detections as review items * Don't handle api detections in recordings * Store recordings for manual events --- frigate/comms/detections_updater.py | 1 + frigate/events/external.py | 49 ++++++++++++++++++--- frigate/record/maintainer.py | 12 ++++-- frigate/review/maintainer.py | 66 +++++++++++++++++++++++++++-- 4 files changed, 117 insertions(+), 11 deletions(-) diff --git a/frigate/comms/detections_updater.py b/frigate/comms/detections_updater.py index ff544dfbd..fa4f56252 100644 --- a/frigate/comms/detections_updater.py +++ b/frigate/comms/detections_updater.py @@ -13,6 +13,7 @@ SOCKET_SUB = "ipc:///tmp/cache/detect_sun" class DetectionTypeEnum(str, Enum): all = "" + api = "api" video = "video" audio = "audio" diff --git a/frigate/events/external.py b/frigate/events/external.py index 7bae21071..6794ce4eb 100644 --- a/frigate/events/external.py +++ b/frigate/events/external.py @@ -6,10 +6,12 @@ import logging import os import random import string +from enum import Enum from typing import Optional import cv2 +from frigate.comms.detections_updater import DetectionPublisher, DetectionTypeEnum from frigate.comms.events_updater import EventUpdatePublisher from frigate.config import CameraConfig, FrigateConfig from frigate.const import CLIPS_DIR @@ -19,11 +21,19 @@ from frigate.util.image import draw_box_with_label logger = logging.getLogger(__name__) +class ManualEventState(str, Enum): + complete = "complete" + start = "start" + end = "end" + + class ExternalEventProcessor: def __init__(self, config: FrigateConfig) -> None: self.config = config self.default_thumbnail = None self.event_sender = EventUpdatePublisher() + self.detection_updater = DetectionPublisher(DetectionTypeEnum.api) + self.event_camera = {} def create_manual_event( self, @@ -47,6 +57,11 @@ class ExternalEventProcessor: thumbnail = self._write_images( camera_config, label, event_id, draw, snapshot_frame ) + end = ( + now + duration + camera_config.record.events.post_capture + if duration is not None + else None + ) self.event_sender.publish( ( @@ -60,11 +75,7 @@ class ExternalEventProcessor: "score": score, "camera": camera, "start_time": now - camera_config.record.events.pre_capture, - "end_time": now - + duration - + camera_config.record.events.post_capture - if duration is not None - else None, + "end_time": end, "thumbnail": thumbnail, "has_clip": camera_config.record.enabled and include_recording, "has_snapshot": True, @@ -73,6 +84,23 @@ class ExternalEventProcessor: ) ) + if source_type == "api": + self.event_camera[event_id] = camera + self.detection_updater.send_data( + ( + camera, + now, + { + "state": ( + ManualEventState.complete if end else ManualEventState.start + ), + "label": f"{label}: {sub_label}" if sub_label else label, + "event_id": event_id, + "end_time": end, + }, + ) + ) + return event_id def finish_manual_event(self, event_id: str, end_time: float) -> None: @@ -86,6 +114,16 @@ class ExternalEventProcessor: ) ) + if event_id in self.event_camera: + self.detection_updater.send_data( + ( + self.event_camera[event_id], + end_time, + {"state": ManualEventState.end, "event_id": event_id}, + ) + ) + self.event_camera.pop(event_id) + def _write_images( self, camera_config: CameraConfig, @@ -143,3 +181,4 @@ class ExternalEventProcessor: def stop(self): self.event_sender.stop() + self.detection_updater.stop() diff --git a/frigate/record/maintainer.py b/frigate/record/maintainer.py index 5a4fc1e49..072c80327 100644 --- a/frigate/record/maintainer.py +++ b/frigate/record/maintainer.py @@ -165,6 +165,7 @@ class RecordingMaintainer(threading.Thread): Event.select( Event.start_time, Event.end_time, + Event.data, ) .where( Event.camera == camera, @@ -188,7 +189,7 @@ class RecordingMaintainer(threading.Thread): ) async def validate_and_move_segment( - self, camera: str, events: Event, recording: dict[str, any] + self, camera: str, events: list[Event], recording: dict[str, any] ) -> None: cache_path = recording["cache_path"] start_time = recording["start_time"] @@ -256,6 +257,7 @@ class RecordingMaintainer(threading.Thread): duration, cache_path, record_mode, + event.data["type"] == "api", ) # if it doesn't overlap with an event, go ahead and drop the segment # if it ends more than the configured pre_capture for the camera @@ -347,11 +349,12 @@ class RecordingMaintainer(threading.Thread): duration: float, cache_path: str, store_mode: RetainModeEnum, + manual_event: bool = False, # if this segment is being moved due to a manual event ) -> Optional[Recordings]: segment_info = self.segment_stats(camera, start_time, end_time) # check if the segment shouldn't be stored - if segment_info.should_discard_segment(store_mode): + if not manual_event and segment_info.should_discard_segment(store_mode): Path(cache_path).unlink(missing_ok=True) self.end_time_cache.pop(cache_path, None) return @@ -424,7 +427,8 @@ class RecordingMaintainer(threading.Thread): Recordings.duration: duration, Recordings.motion: segment_info.motion_count, # TODO: update this to store list of active objects at some point - Recordings.objects: segment_info.active_object_count, + Recordings.objects: segment_info.active_object_count + + (1 if manual_event else 0), Recordings.regions: segment_info.region_count, Recordings.dBFS: segment_info.average_dBFS, Recordings.segment_size: segment_size, @@ -507,6 +511,8 @@ class RecordingMaintainer(threading.Thread): audio_detections, ) ) + elif topic == DetectionTypeEnum.api: + continue if frame_time < run_start - stale_frame_count_threshold: stale_frame_count += 1 diff --git a/frigate/review/maintainer.py b/frigate/review/maintainer.py index 6633c2015..5cbd3a476 100644 --- a/frigate/review/maintainer.py +++ b/frigate/review/maintainer.py @@ -5,6 +5,7 @@ import logging import os import random import string +import sys import threading from enum import Enum from multiprocessing.synchronize import Event as MpEvent @@ -18,6 +19,7 @@ from frigate.comms.detections_updater import DetectionSubscriber, DetectionTypeE from frigate.comms.inter_process import InterProcessRequestor from frigate.config import CameraConfig, FrigateConfig from frigate.const import ALL_ATTRIBUTE_LABELS, CLIPS_DIR, UPSERT_REVIEW_SEGMENT +from frigate.events.external import ManualEventState from frigate.models import ReviewSegment from frigate.object_processing import TrackedObject from frigate.util.image import SharedMemoryFrameManager, calculate_16_9_crop @@ -134,6 +136,9 @@ class ReviewSegmentMaintainer(threading.Thread): self.config_subscriber = ConfigSubscriber("config/record/") self.detection_subscriber = DetectionSubscriber(DetectionTypeEnum.all) + # manual events + self.indefinite_events: dict[str, dict[str, any]] = {} + self.stop_event = stop_event def end_segment(self, segment: PendingReviewSegment) -> None: @@ -160,7 +165,8 @@ class ReviewSegmentMaintainer(threading.Thread): active_objects = get_active_objects(frame_time, camera_config, objects) if len(active_objects) > 0: - segment.last_update = frame_time + if frame_time > segment.last_update: + segment.last_update = frame_time # update type for this segment now that active objects are detected if segment.severity == SeverityEnum.signification_motion: @@ -198,7 +204,8 @@ class ReviewSegmentMaintainer(threading.Thread): segment.severity == SeverityEnum.signification_motion and len(motion) >= THRESHOLD_MOTION_ACTIVITY ): - segment.last_update = frame_time + if frame_time > segment.last_update: + segment.last_update = frame_time else: if segment.severity == SeverityEnum.alert and frame_time > ( segment.last_update + THRESHOLD_ALERT_ACTIVITY @@ -302,6 +309,15 @@ class ReviewSegmentMaintainer(threading.Thread): dBFS, audio_detections, ) = data + elif topic == DetectionTypeEnum.api: + ( + camera, + frame_time, + manual_info, + ) = data + + if camera not in self.indefinite_events: + self.indefinite_events[camera] = {} if not self.config.cameras[camera].record.enabled: continue @@ -317,8 +333,31 @@ class ReviewSegmentMaintainer(threading.Thread): motion_boxes, ) elif topic == DetectionTypeEnum.audio and len(audio_detections) > 0: - current_segment.last_update = frame_time + if frame_time > current_segment.last_update: + current_segment.last_update = frame_time + current_segment.audio.update(audio_detections) + elif topic == DetectionTypeEnum.api: + if manual_info["state"] == ManualEventState.complete: + current_segment.detections[manual_info["event_id"]] = ( + manual_info["label"] + ) + current_segment.severity = SeverityEnum.alert + current_segment.last_update = manual_info["end_time"] + elif manual_info["state"] == ManualEventState.start: + self.indefinite_events[camera][manual_info["event_id"]] = ( + manual_info["label"] + ) + current_segment.detections[manual_info["event_id"]] = ( + manual_info["label"] + ) + current_segment.severity = SeverityEnum.alert + + # temporarily make it so this event can not end + current_segment.last_update = sys.maxsize + elif manual_info["state"] == ManualEventState.end: + self.indefinite_events[camera].pop(manual_info["event_id"]) + current_segment.last_update = manual_info["end_time"] else: if topic == DetectionTypeEnum.video: self.check_if_new_segment( @@ -337,6 +376,27 @@ class ReviewSegmentMaintainer(threading.Thread): set(audio_detections), [], ) + elif topic == DetectionTypeEnum.api: + self.active_review_segments[camera] = PendingReviewSegment( + camera, + frame_time, + SeverityEnum.alert, + {manual_info["event_id"]: manual_info["label"]}, + set(), + set(), + [], + ) + + if manual_info["state"] == ManualEventState.start: + self.indefinite_events[camera][manual_info["event_id"]] = ( + manual_info["label"] + ) + # temporarily make it so this event can not end + self.active_review_segments[camera] = sys.maxsize + elif manual_info["state"] == ManualEventState.complete: + self.active_review_segments[camera].last_update = manual_info[ + "end_time" + ] def get_active_objects(