Save initial camera state to update when websocket connects (#11174)

* Send camera state to dispatcher

* Fix logic

* Cleanup

* Send camera activitiy in on connect

* Support reading initial camera state

* Fix key

* Formatting

* Sorting

frigate/comms/dispatcher.py

@@ -1,6 +1,7 @@
 """Handle communication between Frigate and other applications."""
 
 import datetime
+import json
 import logging
 from abc import ABC, abstractmethod
 from typing import Any, Callable, Optional
@@ -12,6 +13,7 @@ from frigate.const import (
     INSERT_MANY_RECORDINGS,
     INSERT_PREVIEW,
     REQUEST_REGION_GRID,
+    UPDATE_CAMERA_ACTIVITY,
     UPSERT_REVIEW_SEGMENT,
 )
 from frigate.models import Previews, Recordings, ReviewSegment
@@ -76,6 +78,8 @@ class Dispatcher:
         for comm in self.comms:
             comm.subscribe(self._receive)
 
+        self.camera_activity = {}
+
     def _receive(self, topic: str, payload: str) -> Optional[Any]:
         """Handle receiving of payload from communicators."""
         if topic.endswith("set"):
@@ -122,6 +126,10 @@ class Dispatcher:
             ReviewSegment.update(end_time=datetime.datetime.now().timestamp()).where(
                 ReviewSegment.end_time == None
             ).execute()
+        elif topic == UPDATE_CAMERA_ACTIVITY:
+            self.camera_activity = payload
+        elif topic == "onConnect":
+            self.publish("camera_activity", json.dumps(self.camera_activity))
         else:
             self.publish(topic, payload, retain=False)
 

frigate/comms/ws.py

@@ -50,6 +50,10 @@ class WebSocketClient(Communicator):  # type: ignore[misc]
         class _WebSocketHandler(WebSocket):  # type: ignore[misc]
             receiver = self._dispatcher
 
+            def opened(self) -> None:
+                """A new websocket is opened, we need to send an update message"""
+                threading.Timer(1.0, self.receiver, ("onConnect", "")).start()
+
             def received_message(self, message: WebSocket.received_message) -> None:
                 try:
                     json_message = json.loads(message.data.decode("utf-8"))

frigate/const.py

@@ -80,6 +80,7 @@ INSERT_PREVIEW = "insert_preview"
 REQUEST_REGION_GRID = "request_region_grid"
 UPSERT_REVIEW_SEGMENT = "upsert_review_segment"
 CLEAR_ONGOING_REVIEW_SEGMENTS = "clear_ongoing_review_segments"
+UPDATE_CAMERA_ACTIVITY = "update_camera_activity"
 
 # Autotracking
 

frigate/object_processing.py

@@ -16,6 +16,7 @@ import numpy as np
 from frigate.comms.detections_updater import DetectionPublisher, DetectionTypeEnum
 from frigate.comms.dispatcher import Dispatcher
 from frigate.comms.events_updater import EventEndSubscriber, EventUpdatePublisher
+from frigate.comms.inter_process import InterProcessRequestor
 from frigate.config import (
     CameraConfig,
     FrigateConfig,
@@ -24,7 +25,7 @@ from frigate.config import (
     SnapshotsConfig,
     ZoomingModeEnum,
 )
-from frigate.const import CLIPS_DIR
+from frigate.const import ALL_ATTRIBUTE_LABELS, CLIPS_DIR, UPDATE_CAMERA_ACTIVITY
 from frigate.events.types import EventStateEnum, EventTypeEnum
 from frigate.ptz.autotrack import PtzAutoTrackerThread
 from frigate.util.image import (
@@ -724,8 +725,31 @@ class CameraState:
 
         # TODO: can i switch to looking this up and only changing when an event ends?
         # maintain best objects
+        camera_activity: dict[str, list[any]] = {
+            "motion": len(motion_boxes) > 0,
+            "objects": [],
+        }
+
         for obj in tracked_objects.values():
             object_type = obj.obj_data["label"]
+            active = (
+                obj.obj_data["motionless_count"]
+                < self.camera_config.detect.stationary.threshold
+            )
+
+            if not obj.false_positive:
+                label = object_type
+
+                if (
+                    obj.obj_data.get("sub_label")
+                    and obj.obj_data.get("sub_label")[0] in ALL_ATTRIBUTE_LABELS
+                ):
+                    label = obj.obj_data["sub_label"]
+
+                camera_activity["objects"].append(
+                    {"id": obj.obj_data["id"], "label": label, "stationary": not active}
+                )
+
             # if the object's thumbnail is not from the current frame
             if obj.false_positive or obj.thumbnail_data["frame_time"] != frame_time:
                 continue
@@ -752,6 +776,9 @@ class CameraState:
                 for c in self.callbacks["snapshot"]:
                     c(self.name, self.best_objects[object_type], frame_time)
 
+        for c in self.callbacks["camera_activity"]:
+            c(self.name, camera_activity)
+
         # update overall camera state for each object type
         obj_counter = Counter(
             obj.obj_data["label"]
@@ -841,10 +868,14 @@ class TrackedObjectProcessor(threading.Thread):
         self.frame_manager = SharedMemoryFrameManager()
         self.last_motion_detected: dict[str, float] = {}
         self.ptz_autotracker_thread = ptz_autotracker_thread
+
+        self.requestor = InterProcessRequestor()
         self.detection_publisher = DetectionPublisher(DetectionTypeEnum.video)
         self.event_sender = EventUpdatePublisher()
         self.event_end_subscriber = EventEndSubscriber()
 
+        self.camera_activity: dict[str, dict[str, any]] = {}
+
         def start(camera, obj: TrackedObject, current_frame_time):
             self.event_sender.publish(
                 (
@@ -962,6 +993,13 @@ class TrackedObjectProcessor(threading.Thread):
         def object_status(camera, object_name, status):
             self.dispatcher.publish(f"{camera}/{object_name}", status, retain=False)
 
+        def camera_activity(camera, activity):
+            last_activity = self.camera_activity.get(camera)
+
+            if not last_activity or activity != last_activity:
+                self.camera_activity[camera] = activity
+                self.requestor.send_data(UPDATE_CAMERA_ACTIVITY, self.camera_activity)
+
         for camera in self.config.cameras.keys():
             camera_state = CameraState(
                 camera, self.config, self.frame_manager, self.ptz_autotracker_thread
@@ -972,6 +1010,7 @@ class TrackedObjectProcessor(threading.Thread):
             camera_state.on("end", end)
             camera_state.on("snapshot", snapshot)
             camera_state.on("object_status", object_status)
+            camera_state.on("camera_activity", camera_activity)
             self.camera_states[camera] = camera_state
 
         # {
@@ -1228,6 +1267,7 @@ class TrackedObjectProcessor(threading.Thread):
                 event_id, camera = update
                 self.camera_states[camera].finished(event_id)
 
+        self.requestor.stop()
         self.detection_publisher.stop()
         self.event_sender.stop()
         self.event_end_subscriber.stop()

web/src/api/ws.tsx

@@ -2,7 +2,12 @@ import { baseUrl } from "./baseUrl";
 import { useCallback, useEffect, useState } from "react";
 import useWebSocket, { ReadyState } from "react-use-websocket";
 import { FrigateConfig } from "@/types/frigateConfig";
-import { FrigateEvent, FrigateReview, ToggleableSetting } from "@/types/ws";
+import {
+  FrigateCameraState,
+  FrigateEvent,
+  FrigateReview,
+  ToggleableSetting,
+} from "@/types/ws";
 import { FrigateStats } from "@/types/stats";
 import useSWR from "swr";
 import { createContainer } from "react-tracked";
@@ -193,6 +198,16 @@ export function useFrigateStats(): { payload: FrigateStats } {
   return { payload: JSON.parse(payload as string) };
 }
 
+export function useInitialCameraState(camera: string): {
+  payload: FrigateCameraState;
+} {
+  const {
+    value: { payload },
+  } = useWs("camera_activity", "");
+  const data = JSON.parse(payload as string);
+  return { payload: data ? data[camera] : undefined };
+}
+
 export function useMotionActivity(camera: string): { payload: string } {
   const {
     value: { payload },

web/src/components/Statusbar.tsx

@@ -131,7 +131,11 @@ export default function Statusbar() {
                 );
 
                 if (link) {
-                  return <Link to={link}>{message}</Link>;
+                  return (
+                    <Link key={id} to={link}>
+                      {message}
+                    </Link>
+                  );
                 } else {
                   return message;
                 }

web/src/components/player/LivePlayer.tsx

@@ -48,7 +48,7 @@ export default function LivePlayer({
 
   // camera activity
 
-  const { activeMotion, activeTracking, activeObjects } =
+  const { activeMotion, activeTracking, objects } =
     useCameraActivity(cameraConfig);
 
   const cameraActive = useMemo(
@@ -166,7 +166,7 @@ export default function LivePlayer({
       <div className="absolute bottom-0 inset-x-0 rounded-lg md:rounded-2xl z-10 w-full h-[10%] bg-gradient-to-t from-black/20 to-transparent pointer-events-none"></div>
       {player}
 
-      {activeObjects.length > 0 && (
+      {objects.length > 0 && (
         <div className="absolute left-0 top-2 z-40">
           <Tooltip>
             <div className="flex">
@@ -177,7 +177,7 @@ export default function LivePlayer({
                   >
                     {[
                       ...new Set([
-                        ...(activeObjects || []).map(({ label }) => label),
+                        ...(objects || []).map(({ label }) => label),
                       ]),
                     ]
                       .map((label) => {
@@ -189,11 +189,7 @@ export default function LivePlayer({
               </TooltipTrigger>
             </div>
             <TooltipContent className="capitalize">
-              {[
-                ...new Set([
-                  ...(activeObjects || []).map(({ label }) => label),
-                ]),
-              ]
+              {[...new Set([...(objects || []).map(({ label }) => label)])]
                 .filter(
                   (label) =>
                     label !== undefined && !label.includes("-verified"),

web/src/hooks/use-camera-activity.ts

@@ -1,72 +1,97 @@
-import { useFrigateEvents, useMotionActivity } from "@/api/ws";
+import {
+  useFrigateEvents,
+  useInitialCameraState,
+  useMotionActivity,
+} from "@/api/ws";
 import { CameraConfig } from "@/types/frigateConfig";
 import { MotionData, ReviewSegment } from "@/types/review";
 import { useEffect, useMemo, useState } from "react";
 import { useTimelineUtils } from "./use-timeline-utils";
-
-type ActiveObjectType = {
-  id: string;
-  label: string;
-  stationary: boolean;
-};
+import { ObjectType } from "@/types/ws";
+import useDeepMemo from "./use-deep-memo";
 
 type useCameraActivityReturn = {
   activeTracking: boolean;
   activeMotion: boolean;
-  activeObjects: ActiveObjectType[];
+  objects: ObjectType[];
 };
 
 export function useCameraActivity(
   camera: CameraConfig,
 ): useCameraActivityReturn {
-  const [activeObjects, setActiveObjects] = useState<ActiveObjectType[]>([]);
+  const [objects, setObjects] = useState<ObjectType[]>([]);
+
+  // init camera activity
+
+  const { payload: initialCameraState } = useInitialCameraState(camera.name);
+
+  const updatedCameraState = useDeepMemo(initialCameraState);
+
+  useEffect(() => {
+    if (updatedCameraState) {
+      setObjects(updatedCameraState.objects);
+    }
+  }, [updatedCameraState]);
+
+  // handle camera activity
+
   const hasActiveObjects = useMemo(
-    () => activeObjects.filter((obj) => !obj.stationary).length > 0,
-    [activeObjects],
+    () => objects.filter((obj) => !obj.stationary).length > 0,
+    [objects],
   );
 
   const { payload: detectingMotion } = useMotionActivity(camera.name);
   const { payload: event } = useFrigateEvents();
+  const updatedEvent = useDeepMemo(event);
 
   useEffect(() => {
-    if (!event) {
+    if (!updatedEvent) {
       return;
     }
 
-    if (event.after.camera != camera.name) {
+    if (updatedEvent.after.camera != camera.name) {
       return;
     }
 
-    const eventIndex = activeObjects.findIndex(
-      (obj) => obj.id === event.after.id,
+    const updatedEventIndex = objects.findIndex(
+      (obj) => obj.id === updatedEvent.after.id,
     );
 
-    if (event.type == "end") {
-      if (eventIndex != -1) {
-        const newActiveObjects = [...activeObjects];
-        newActiveObjects.splice(eventIndex, 1);
-        setActiveObjects(newActiveObjects);
+    if (updatedEvent.type == "end") {
+      if (updatedEventIndex != -1) {
+        const newActiveObjects = [...objects];
+        newActiveObjects.splice(updatedEventIndex, 1);
+        setObjects(newActiveObjects);
       }
     } else {
-      if (eventIndex == -1) {
-        // add unknown event to list if not stationary
-        if (!event.after.stationary) {
-          const newActiveObject: ActiveObjectType = {
-            id: event.after.id,
-            label: event.after.label,
-            stationary: event.after.stationary,
+      if (updatedEventIndex == -1) {
+        // add unknown updatedEvent to list if not stationary
+        if (!updatedEvent.after.stationary) {
+          const newActiveObject: ObjectType = {
+            id: updatedEvent.after.id,
+            label: updatedEvent.after.label,
+            stationary: updatedEvent.after.stationary,
           };
-          const newActiveObjects = [...activeObjects, newActiveObject];
-          setActiveObjects(newActiveObjects);
+          const newActiveObjects = [...objects, newActiveObject];
+          setObjects(newActiveObjects);
         }
+      } else {
+        const newObjects = [...objects];
+        newObjects[updatedEventIndex].label =
+          updatedEvent.after.sub_label ?? updatedEvent.after.label;
+        newObjects[updatedEventIndex].stationary =
+          updatedEvent.after.stationary;
+        setObjects(newObjects);
       }
     }
-  }, [camera, event, activeObjects]);
+  }, [camera, updatedEvent, objects]);
 
   return {
     activeTracking: hasActiveObjects,
-    activeMotion: detectingMotion == "ON",
-    activeObjects,
+    activeMotion: detectingMotion
+      ? detectingMotion == "ON"
+      : initialCameraState?.motion == true,
+    objects,
   };
 }
 

web/src/types/ws.ts

@@ -41,4 +41,15 @@ export interface FrigateEvent {
   after: FrigateObjectState;
 }
 
+export type ObjectType = {
+  id: string;
+  label: string;
+  stationary: boolean;
+};
+
+export interface FrigateCameraState {
+  motion: boolean;
+  objects: ObjectType[];
+}
+
 export type ToggleableSetting = "ON" | "OFF";