Support manual detections in review items (#10784)

* Only update frame time if it is older

* Support manual detections as review items

* Don't handle api detections in recordings

* Store recordings for manual events

frigate/comms/detections_updater.py

@@ -13,6 +13,7 @@ SOCKET_SUB = "ipc:///tmp/cache/detect_sun"
 
 class DetectionTypeEnum(str, Enum):
     all = ""
+    api = "api"
     video = "video"
     audio = "audio"
 

frigate/events/external.py

@@ -6,10 +6,12 @@ import logging
 import os
 import random
 import string
+from enum import Enum
 from typing import Optional
 
 import cv2
 
+from frigate.comms.detections_updater import DetectionPublisher, DetectionTypeEnum
 from frigate.comms.events_updater import EventUpdatePublisher
 from frigate.config import CameraConfig, FrigateConfig
 from frigate.const import CLIPS_DIR
@@ -19,11 +21,19 @@ from frigate.util.image import draw_box_with_label
 logger = logging.getLogger(__name__)
 
 
+class ManualEventState(str, Enum):
+    complete = "complete"
+    start = "start"
+    end = "end"
+
+
 class ExternalEventProcessor:
     def __init__(self, config: FrigateConfig) -> None:
         self.config = config
         self.default_thumbnail = None
         self.event_sender = EventUpdatePublisher()
+        self.detection_updater = DetectionPublisher(DetectionTypeEnum.api)
+        self.event_camera = {}
 
     def create_manual_event(
         self,
@@ -47,6 +57,11 @@ class ExternalEventProcessor:
         thumbnail = self._write_images(
             camera_config, label, event_id, draw, snapshot_frame
         )
+        end = (
+            now + duration + camera_config.record.events.post_capture
+            if duration is not None
+            else None
+        )
 
         self.event_sender.publish(
             (
@@ -60,11 +75,7 @@ class ExternalEventProcessor:
                     "score": score,
                     "camera": camera,
                     "start_time": now - camera_config.record.events.pre_capture,
-                    "end_time": now
+                    "end_time": end,
-                    + duration
-                    + camera_config.record.events.post_capture
-                    if duration is not None
-                    else None,
                     "thumbnail": thumbnail,
                     "has_clip": camera_config.record.enabled and include_recording,
                     "has_snapshot": True,
@@ -73,6 +84,23 @@ class ExternalEventProcessor:
             )
         )
 
+        if source_type == "api":
+            self.event_camera[event_id] = camera
+            self.detection_updater.send_data(
+                (
+                    camera,
+                    now,
+                    {
+                        "state": (
+                            ManualEventState.complete if end else ManualEventState.start
+                        ),
+                        "label": f"{label}: {sub_label}" if sub_label else label,
+                        "event_id": event_id,
+                        "end_time": end,
+                    },
+                )
+            )
+
         return event_id
 
     def finish_manual_event(self, event_id: str, end_time: float) -> None:
@@ -86,6 +114,16 @@ class ExternalEventProcessor:
             )
         )
 
+        if event_id in self.event_camera:
+            self.detection_updater.send_data(
+                (
+                    self.event_camera[event_id],
+                    end_time,
+                    {"state": ManualEventState.end, "event_id": event_id},
+                )
+            )
+            self.event_camera.pop(event_id)
+
     def _write_images(
         self,
         camera_config: CameraConfig,
@@ -143,3 +181,4 @@ class ExternalEventProcessor:
 
     def stop(self):
         self.event_sender.stop()
+        self.detection_updater.stop()

frigate/record/maintainer.py

@@ -165,6 +165,7 @@ class RecordingMaintainer(threading.Thread):
                 Event.select(
                     Event.start_time,
                     Event.end_time,
+                    Event.data,
                 )
                 .where(
                     Event.camera == camera,
@@ -188,7 +189,7 @@ class RecordingMaintainer(threading.Thread):
         )
 
     async def validate_and_move_segment(
-        self, camera: str, events: Event, recording: dict[str, any]
+        self, camera: str, events: list[Event], recording: dict[str, any]
     ) -> None:
         cache_path = recording["cache_path"]
         start_time = recording["start_time"]
@@ -256,6 +257,7 @@ class RecordingMaintainer(threading.Thread):
                     duration,
                     cache_path,
                     record_mode,
+                    event.data["type"] == "api",
                 )
             # if it doesn't overlap with an event, go ahead and drop the segment
             # if it ends more than the configured pre_capture for the camera
@@ -347,11 +349,12 @@ class RecordingMaintainer(threading.Thread):
         duration: float,
         cache_path: str,
         store_mode: RetainModeEnum,
+        manual_event: bool = False,  # if this segment is being moved due to a manual event
     ) -> Optional[Recordings]:
         segment_info = self.segment_stats(camera, start_time, end_time)
 
         # check if the segment shouldn't be stored
-        if segment_info.should_discard_segment(store_mode):
+        if not manual_event and segment_info.should_discard_segment(store_mode):
             Path(cache_path).unlink(missing_ok=True)
             self.end_time_cache.pop(cache_path, None)
             return
@@ -424,7 +427,8 @@ class RecordingMaintainer(threading.Thread):
                     Recordings.duration: duration,
                     Recordings.motion: segment_info.motion_count,
                     # TODO: update this to store list of active objects at some point
-                    Recordings.objects: segment_info.active_object_count,
+                    Recordings.objects: segment_info.active_object_count
+                    + (1 if manual_event else 0),
                     Recordings.regions: segment_info.region_count,
                     Recordings.dBFS: segment_info.average_dBFS,
                     Recordings.segment_size: segment_size,
@@ -507,6 +511,8 @@ class RecordingMaintainer(threading.Thread):
                                 audio_detections,
                             )
                         )
+                elif topic == DetectionTypeEnum.api:
+                    continue
 
                 if frame_time < run_start - stale_frame_count_threshold:
                     stale_frame_count += 1

frigate/review/maintainer.py

@@ -5,6 +5,7 @@ import logging
 import os
 import random
 import string
+import sys
 import threading
 from enum import Enum
 from multiprocessing.synchronize import Event as MpEvent
@@ -18,6 +19,7 @@ from frigate.comms.detections_updater import DetectionSubscriber, DetectionTypeE
 from frigate.comms.inter_process import InterProcessRequestor
 from frigate.config import CameraConfig, FrigateConfig
 from frigate.const import ALL_ATTRIBUTE_LABELS, CLIPS_DIR, UPSERT_REVIEW_SEGMENT
+from frigate.events.external import ManualEventState
 from frigate.models import ReviewSegment
 from frigate.object_processing import TrackedObject
 from frigate.util.image import SharedMemoryFrameManager, calculate_16_9_crop
@@ -134,6 +136,9 @@ class ReviewSegmentMaintainer(threading.Thread):
         self.config_subscriber = ConfigSubscriber("config/record/")
         self.detection_subscriber = DetectionSubscriber(DetectionTypeEnum.all)
 
+        # manual events
+        self.indefinite_events: dict[str, dict[str, any]] = {}
+
         self.stop_event = stop_event
 
     def end_segment(self, segment: PendingReviewSegment) -> None:
@@ -160,6 +165,7 @@ class ReviewSegmentMaintainer(threading.Thread):
         active_objects = get_active_objects(frame_time, camera_config, objects)
 
         if len(active_objects) > 0:
+            if frame_time > segment.last_update:
                 segment.last_update = frame_time
 
             # update type for this segment now that active objects are detected
@@ -198,6 +204,7 @@ class ReviewSegmentMaintainer(threading.Thread):
             segment.severity == SeverityEnum.signification_motion
             and len(motion) >= THRESHOLD_MOTION_ACTIVITY
         ):
+            if frame_time > segment.last_update:
                 segment.last_update = frame_time
         else:
             if segment.severity == SeverityEnum.alert and frame_time > (
@@ -302,6 +309,15 @@ class ReviewSegmentMaintainer(threading.Thread):
                     dBFS,
                     audio_detections,
                 ) = data
+            elif topic == DetectionTypeEnum.api:
+                (
+                    camera,
+                    frame_time,
+                    manual_info,
+                ) = data
+
+                if camera not in self.indefinite_events:
+                    self.indefinite_events[camera] = {}
 
             if not self.config.cameras[camera].record.enabled:
                 continue
@@ -317,8 +333,31 @@ class ReviewSegmentMaintainer(threading.Thread):
                         motion_boxes,
                     )
                 elif topic == DetectionTypeEnum.audio and len(audio_detections) > 0:
+                    if frame_time > current_segment.last_update:
                         current_segment.last_update = frame_time
+
                     current_segment.audio.update(audio_detections)
+                elif topic == DetectionTypeEnum.api:
+                    if manual_info["state"] == ManualEventState.complete:
+                        current_segment.detections[manual_info["event_id"]] = (
+                            manual_info["label"]
+                        )
+                        current_segment.severity = SeverityEnum.alert
+                        current_segment.last_update = manual_info["end_time"]
+                    elif manual_info["state"] == ManualEventState.start:
+                        self.indefinite_events[camera][manual_info["event_id"]] = (
+                            manual_info["label"]
+                        )
+                        current_segment.detections[manual_info["event_id"]] = (
+                            manual_info["label"]
+                        )
+                        current_segment.severity = SeverityEnum.alert
+
+                        # temporarily make it so this event can not end
+                        current_segment.last_update = sys.maxsize
+                    elif manual_info["state"] == ManualEventState.end:
+                        self.indefinite_events[camera].pop(manual_info["event_id"])
+                        current_segment.last_update = manual_info["end_time"]
             else:
                 if topic == DetectionTypeEnum.video:
                     self.check_if_new_segment(
@@ -337,6 +376,27 @@ class ReviewSegmentMaintainer(threading.Thread):
                         set(audio_detections),
                         [],
                     )
+                elif topic == DetectionTypeEnum.api:
+                    self.active_review_segments[camera] = PendingReviewSegment(
+                        camera,
+                        frame_time,
+                        SeverityEnum.alert,
+                        {manual_info["event_id"]: manual_info["label"]},
+                        set(),
+                        set(),
+                        [],
+                    )
+
+                    if manual_info["state"] == ManualEventState.start:
+                        self.indefinite_events[camera][manual_info["event_id"]] = (
+                            manual_info["label"]
+                        )
+                        # temporarily make it so this event can not end
+                        self.active_review_segments[camera] = sys.maxsize
+                    elif manual_info["state"] == ManualEventState.complete:
+                        self.active_review_segments[camera].last_update = manual_info[
+                            "end_time"
+                        ]
 
 
 def get_active_objects(