From 9ae839ad723dcb2b3c89544419a5b96779a44062 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Mon, 18 Nov 2024 12:26:44 -0600 Subject: [PATCH] Tracked object metadata changes (#15055) * add enum and change topic name * frontend renaming * docs * only display sublabel score if it it exists * remove debug print --- docs/docs/integrations/mqtt.md | 12 ++++++++++++ frigate/comms/dispatcher.py | 12 +++++++++--- frigate/embeddings/maintainer.py | 7 ++++++- frigate/types.py | 4 ++++ web/src/api/ws.tsx | 4 ++-- .../components/overlay/detail/SearchDetailDialog.tsx | 2 +- web/src/pages/Explore.tsx | 8 ++++---- web/src/views/explore/ExploreView.tsx | 6 +++--- 8 files changed, 41 insertions(+), 14 deletions(-) diff --git a/docs/docs/integrations/mqtt.md b/docs/docs/integrations/mqtt.md index e606d29fc..194821cbd 100644 --- a/docs/docs/integrations/mqtt.md +++ b/docs/docs/integrations/mqtt.md @@ -94,6 +94,18 @@ Message published for each changed tracked object. The first message is publishe } ``` +### `frigate/tracked_object_update` + +Message published for updates to tracked object metadata, for example when GenAI runs and returns a tracked object description. + +```json +{ + "type": "description", + "id": "1607123955.475377-mxklsc", + "description": "The car is a red sedan moving away from the camera." +} +``` + ### `frigate/reviews` Message published for each changed review item. The first message is published when the `detection` or `alert` is initiated. When additional objects are detected or when a zone change occurs, it will publish a, `update` message with the same id. When the review activity has ended a final `end` message is published. diff --git a/frigate/comms/dispatcher.py b/frigate/comms/dispatcher.py index 1f480fa9c..2bddc97a5 100644 --- a/frigate/comms/dispatcher.py +++ b/frigate/comms/dispatcher.py @@ -22,7 +22,7 @@ from frigate.const import ( ) from frigate.models import Event, Previews, Recordings, ReviewSegment from frigate.ptz.onvif import OnvifCommandEnum, OnvifController -from frigate.types import ModelStatusTypesEnum +from frigate.types import ModelStatusTypesEnum, TrackedObjectUpdateTypesEnum from frigate.util.object import get_camera_regions_grid from frigate.util.services import restart_frigate @@ -137,8 +137,14 @@ class Dispatcher: event.data["description"] = payload["description"] event.save() self.publish( - "event_update", - json.dumps({"id": event.id, "description": event.data["description"]}), + "tracked_object_update", + json.dumps( + { + "type": TrackedObjectUpdateTypesEnum.description, + "id": event.id, + "description": event.data["description"], + } + ), ) def handle_update_model_state(): diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py index 12c8bac72..dde8f8df4 100644 --- a/frigate/embeddings/maintainer.py +++ b/frigate/embeddings/maintainer.py @@ -24,6 +24,7 @@ from frigate.const import CLIPS_DIR, UPDATE_EVENT_DESCRIPTION from frigate.events.types import EventTypeEnum from frigate.genai import get_genai_client from frigate.models import Event +from frigate.types import TrackedObjectUpdateTypesEnum from frigate.util.builtin import serialize from frigate.util.image import SharedMemoryFrameManager, calculate_region @@ -287,7 +288,11 @@ class EmbeddingMaintainer(threading.Thread): # fire and forget description update self.requestor.send_data( UPDATE_EVENT_DESCRIPTION, - {"id": event.id, "description": description}, + { + "type": TrackedObjectUpdateTypesEnum.description, + "id": event.id, + "description": description, + }, ) # Embed the description diff --git a/frigate/types.py b/frigate/types.py index 3e6ad46cc..11ab31238 100644 --- a/frigate/types.py +++ b/frigate/types.py @@ -19,3 +19,7 @@ class ModelStatusTypesEnum(str, Enum): downloading = "downloading" downloaded = "downloaded" error = "error" + + +class TrackedObjectUpdateTypesEnum(str, Enum): + description = "description" diff --git a/web/src/api/ws.tsx b/web/src/api/ws.tsx index c7bb74095..9b8924d1b 100644 --- a/web/src/api/ws.tsx +++ b/web/src/api/ws.tsx @@ -407,9 +407,9 @@ export function useImproveContrast(camera: string): { return { payload: payload as ToggleableSetting, send }; } -export function useEventUpdate(): { payload: string } { +export function useTrackedObjectUpdate(): { payload: string } { const { value: { payload }, - } = useWs("event_update", ""); + } = useWs("tracked_object_update", ""); return useDeepMemo(JSON.parse(payload as string)); } diff --git a/web/src/components/overlay/detail/SearchDetailDialog.tsx b/web/src/components/overlay/detail/SearchDetailDialog.tsx index 5eca9a934..f7af31606 100644 --- a/web/src/components/overlay/detail/SearchDetailDialog.tsx +++ b/web/src/components/overlay/detail/SearchDetailDialog.tsx @@ -309,7 +309,7 @@ function ObjectDetailsTab({ return undefined; } - if (search.sub_label) { + if (search.sub_label && search.data?.sub_label_score) { return Math.round((search.data?.sub_label_score ?? 0) * 100); } else { return undefined; diff --git a/web/src/pages/Explore.tsx b/web/src/pages/Explore.tsx index 711666807..2bf2bb022 100644 --- a/web/src/pages/Explore.tsx +++ b/web/src/pages/Explore.tsx @@ -1,6 +1,6 @@ import { useEmbeddingsReindexProgress, - useEventUpdate, + useTrackedObjectUpdate, useModelState, } from "@/api/ws"; import ActivityIndicator from "@/components/indicators/activity-indicator"; @@ -227,15 +227,15 @@ export default function Explore() { // mutation and revalidation - const eventUpdate = useEventUpdate(); + const trackedObjectUpdate = useTrackedObjectUpdate(); useEffect(() => { - if (eventUpdate) { + if (trackedObjectUpdate) { mutate(); } // mutate / revalidate when event description updates come in // eslint-disable-next-line react-hooks/exhaustive-deps - }, [eventUpdate]); + }, [trackedObjectUpdate]); // embeddings reindex progress diff --git a/web/src/views/explore/ExploreView.tsx b/web/src/views/explore/ExploreView.tsx index f37c37453..ea9c3cbef 100644 --- a/web/src/views/explore/ExploreView.tsx +++ b/web/src/views/explore/ExploreView.tsx @@ -15,7 +15,7 @@ import { SearchResult } from "@/types/search"; import ImageLoadingIndicator from "@/components/indicators/ImageLoadingIndicator"; import useImageLoaded from "@/hooks/use-image-loaded"; import ActivityIndicator from "@/components/indicators/activity-indicator"; -import { useEventUpdate } from "@/api/ws"; +import { useTrackedObjectUpdate } from "@/api/ws"; import { isEqual } from "lodash"; import TimeAgo from "@/components/dynamic/TimeAgo"; import SearchResultActions from "@/components/menu/SearchResultActions"; @@ -72,13 +72,13 @@ export default function ExploreView({ }, {}); }, [events]); - const eventUpdate = useEventUpdate(); + const trackedObjectUpdate = useTrackedObjectUpdate(); useEffect(() => { mutate(); // mutate / revalidate when event description updates come in // eslint-disable-next-line react-hooks/exhaustive-deps - }, [eventUpdate]); + }, [trackedObjectUpdate]); // update search detail when results change