From 5b1b6b5be082fb18ae8ac3b66f81c00ebc8a9493 Mon Sep 17 00:00:00 2001
From: Nicolas Mowen <nickmowen213@gmail.com>
Date: Sun, 17 Nov 2024 10:25:49 -0700
Subject: [PATCH 01/10] Fix round robin (#15035)

* Move camera SHM frame creation to main process

* Don't reset frame index

* Don't fail if shm exists

* Set more types
---
 frigate/app.py        |  8 +++++++-
 frigate/util/image.py | 14 +++++++++-----
 frigate/video.py      | 20 ++++++++++----------
 3 files changed, 26 insertions(+), 16 deletions(-)

diff --git a/frigate/app.py b/frigate/app.py
index 96edfbd15..f56ed1a8b 100644
--- a/frigate/app.py
+++ b/frigate/app.py
@@ -68,7 +68,7 @@ from frigate.stats.util import stats_init
 from frigate.storage import StorageMaintainer
 from frigate.timeline import TimelineProcessor
 from frigate.util.builtin import empty_and_close_queue
-from frigate.util.image import UntrackedSharedMemory
+from frigate.util.image import SharedMemoryFrameManager, UntrackedSharedMemory
 from frigate.util.object import get_camera_regions_grid
 from frigate.version import VERSION
 from frigate.video import capture_camera, track_camera
@@ -426,12 +426,18 @@ class FrigateApp:
 
     def start_camera_capture_processes(self) -> None:
         shm_frame_count = self.shm_frame_count()
+        frame_manager = SharedMemoryFrameManager()
 
         for name, config in self.config.cameras.items():
             if not self.config.cameras[name].enabled:
                 logger.info(f"Capture process not started for disabled camera {name}")
                 continue
 
+            # pre-create shms
+            for i in range(shm_frame_count):
+                frame_size = config.frame_shape_yuv[0] * config.frame_shape_yuv[1]
+                frame_manager.create(f"{config.name}{i}", frame_size)
+
             capture_process = util.Process(
                 target=capture_camera,
                 name=f"camera_capture:{name}",
diff --git a/frigate/util/image.py b/frigate/util/image.py
index 4e3161192..7b22c138e 100644
--- a/frigate/util/image.py
+++ b/frigate/util/image.py
@@ -790,11 +790,15 @@ class SharedMemoryFrameManager(FrameManager):
         self.shm_store: dict[str, UntrackedSharedMemory] = {}
 
     def create(self, name: str, size) -> AnyStr:
-        shm = UntrackedSharedMemory(
-            name=name,
-            create=True,
-            size=size,
-        )
+        try:
+            shm = UntrackedSharedMemory(
+                name=name,
+                create=True,
+                size=size,
+            )
+        except FileExistsError:
+            shm = UntrackedSharedMemory(name=name)
+
         self.shm_store[name] = shm
         return shm.buf
 
diff --git a/frigate/video.py b/frigate/video.py
index 4e7fe660d..5af3e13f4 100755
--- a/frigate/video.py
+++ b/frigate/video.py
@@ -94,7 +94,8 @@ def capture_frames(
     ffmpeg_process,
     config: CameraConfig,
     shm_frame_count: int,
-    frame_shape,
+    frame_index: int,
+    frame_shape: tuple[int, int],
     frame_manager: FrameManager,
     frame_queue,
     fps: mp.Value,
@@ -108,12 +109,6 @@ def capture_frames(
     skipped_eps = EventsPerSecond()
     skipped_eps.start()
 
-    # pre-create shms
-    for i in range(shm_frame_count):
-        frame_manager.create(f"{config.name}{i}", frame_size)
-
-    frame_index = 0
-
     while True:
         fps.value = frame_rate.eps()
         skipped_fps.value = skipped_eps.eps()
@@ -159,7 +154,7 @@ class CameraWatchdog(threading.Thread):
         camera_name,
         config: CameraConfig,
         shm_frame_count: int,
-        frame_queue,
+        frame_queue: mp.Queue,
         camera_fps,
         skipped_fps,
         ffmpeg_pid,
@@ -181,6 +176,7 @@ class CameraWatchdog(threading.Thread):
         self.frame_shape = self.config.frame_shape_yuv
         self.frame_size = self.frame_shape[0] * self.frame_shape[1]
         self.fps_overflow_count = 0
+        self.frame_index = 0
         self.stop_event = stop_event
         self.sleeptime = self.config.ffmpeg.retry_interval
 
@@ -302,6 +298,7 @@ class CameraWatchdog(threading.Thread):
         self.capture_thread = CameraCapture(
             self.config,
             self.shm_frame_count,
+            self.frame_index,
             self.ffmpeg_detect_process,
             self.frame_shape,
             self.frame_queue,
@@ -342,9 +339,10 @@ class CameraCapture(threading.Thread):
         self,
         config: CameraConfig,
         shm_frame_count: int,
+        frame_index: int,
         ffmpeg_process,
-        frame_shape,
-        frame_queue,
+        frame_shape: tuple[int, int],
+        frame_queue: mp.Queue,
         fps,
         skipped_fps,
         stop_event,
@@ -353,6 +351,7 @@ class CameraCapture(threading.Thread):
         self.name = f"capture:{config.name}"
         self.config = config
         self.shm_frame_count = shm_frame_count
+        self.frame_index = frame_index
         self.frame_shape = frame_shape
         self.frame_queue = frame_queue
         self.fps = fps
@@ -368,6 +367,7 @@ class CameraCapture(threading.Thread):
             self.ffmpeg_process,
             self.config,
             self.shm_frame_count,
+            self.frame_index,
             self.frame_shape,
             self.frame_manager,
             self.frame_queue,

From 474c248c9d7501ff7e9677578331d4ca4beff476 Mon Sep 17 00:00:00 2001
From: Nicolas Mowen <nickmowen213@gmail.com>
Date: Sun, 17 Nov 2024 15:57:58 -0700
Subject: [PATCH 02/10] Cleanup correctly (#15043)

---
 frigate/app.py   | 5 +++--
 frigate/video.py | 2 --
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/frigate/app.py b/frigate/app.py
index f56ed1a8b..6518c1ddf 100644
--- a/frigate/app.py
+++ b/frigate/app.py
@@ -91,6 +91,7 @@ class FrigateApp:
         self.processes: dict[str, int] = {}
         self.embeddings: Optional[EmbeddingsContext] = None
         self.region_grids: dict[str, list[list[dict[str, int]]]] = {}
+        self.frame_manager = SharedMemoryFrameManager()
         self.config = config
 
     def ensure_dirs(self) -> None:
@@ -426,7 +427,6 @@ class FrigateApp:
 
     def start_camera_capture_processes(self) -> None:
         shm_frame_count = self.shm_frame_count()
-        frame_manager = SharedMemoryFrameManager()
 
         for name, config in self.config.cameras.items():
             if not self.config.cameras[name].enabled:
@@ -436,7 +436,7 @@ class FrigateApp:
             # pre-create shms
             for i in range(shm_frame_count):
                 frame_size = config.frame_shape_yuv[0] * config.frame_shape_yuv[1]
-                frame_manager.create(f"{config.name}{i}", frame_size)
+                self.frame_manager.create(f"{config.name}{i}", frame_size)
 
             capture_process = util.Process(
                 target=capture_camera,
@@ -717,6 +717,7 @@ class FrigateApp:
         self.event_metadata_updater.stop()
         self.inter_zmq_proxy.stop()
 
+        self.frame_manager.cleanup()
         while len(self.detection_shms) > 0:
             shm = self.detection_shms.pop()
             shm.close()
diff --git a/frigate/video.py b/frigate/video.py
index 5af3e13f4..96b562e8c 100755
--- a/frigate/video.py
+++ b/frigate/video.py
@@ -145,8 +145,6 @@ def capture_frames(
 
         frame_index = 0 if frame_index == shm_frame_count - 1 else frame_index + 1
 
-    frame_manager.cleanup()
-
 
 class CameraWatchdog(threading.Thread):
     def __init__(

From 26c3f9f9148ccfc46a461c6baec000b984827108 Mon Sep 17 00:00:00 2001
From: Nicolas Mowen <nickmowen213@gmail.com>
Date: Mon, 18 Nov 2024 08:38:58 -0700
Subject: [PATCH 03/10] Fix birdseye (#15051)

---
 frigate/output/birdseye.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/frigate/output/birdseye.py b/frigate/output/birdseye.py
index cab155b9b..6d6391f14 100644
--- a/frigate/output/birdseye.py
+++ b/frigate/output/birdseye.py
@@ -388,7 +388,7 @@ class BirdsEyeFrameManager:
                 for cam, cam_data in self.cameras.items()
                 if self.config.cameras[cam].birdseye.enabled
                 and cam_data["last_active_frame"] > 0
-                and cam_data["current_frame"] - cam_data["last_active_frame"]
+                and cam_data["current_frame_time"] - cam_data["last_active_frame"]
                 < self.inactivity_threshold
             ]
         )
@@ -405,7 +405,7 @@ class BirdsEyeFrameManager:
                 limited_active_cameras = sorted(
                     active_cameras,
                     key=lambda active_camera: (
-                        self.cameras[active_camera]["current_frame"]
+                        self.cameras[active_camera]["current_frame_time"]
                         - self.cameras[active_camera]["last_active_frame"]
                     ),
                 )
@@ -517,7 +517,7 @@ class BirdsEyeFrameManager:
                 self.copy_to_position(
                     position[1],
                     position[0],
-                    frame,
+                    self.cameras[position[0]]["current_frame"],
                 )
 
         return True
@@ -689,7 +689,8 @@ class BirdsEyeFrameManager:
             return False
 
         # update the last active frame for the camera
-        self.cameras[camera]["current_frame"] = frame_time
+        self.cameras[camera]["current_frame"] = frame.copy()
+        self.cameras[camera]["current_frame_time"] = frame_time
         if self.camera_active(camera_config.mode, object_count, motion_count):
             self.cameras[camera]["last_active_frame"] = frame_time
 
@@ -755,7 +756,7 @@ class Birdseye:
         current_tracked_objects: list[dict[str, any]],
         motion_boxes: list[list[int]],
         frame_time: float,
-        frame,
+        frame: np.ndarray,
     ) -> None:
         # check if there is an updated config
         while True:

From 0b203a3673085da90f4074148b3bc657624f35ec Mon Sep 17 00:00:00 2001
From: Nicolas Mowen <nickmowen213@gmail.com>
Date: Mon, 18 Nov 2024 09:14:49 -0700
Subject: [PATCH 04/10] fix writing to birdseye restream buffer (#15052)

---
 frigate/output/birdseye.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/frigate/output/birdseye.py b/frigate/output/birdseye.py
index 6d6391f14..00f17c8f4 100644
--- a/frigate/output/birdseye.py
+++ b/frigate/output/birdseye.py
@@ -740,9 +740,10 @@ class Birdseye:
         )
         self.birdseye_manager = BirdsEyeFrameManager(config, stop_event)
         self.config_subscriber = ConfigSubscriber("config/birdseye/")
+        self.frame_manager = SharedMemoryFrameManager()
 
         if config.birdseye.restream:
-            self.birdseye_buffer = SharedMemoryFrameManager().create(
+            self.birdseye_buffer = self.frame_manager.create(
                 "birdseye",
                 self.birdseye_manager.yuv_shape[0] * self.birdseye_manager.yuv_shape[1],
             )

From 66f71aecf7755771bc9a3c1dd7cf741a6e1b6955 Mon Sep 17 00:00:00 2001
From: Bazyl Ichabod Horsey <31446064+bazylhorsey@users.noreply.github.com>
Date: Mon, 18 Nov 2024 12:26:36 -0600
Subject: [PATCH 05/10] fix regex for cookie_name to be general snake case
 (#14854)

* fix regex for cookie_name to be general snake case

* Update frigate/config/auth.py

Co-authored-by: Blake Blackshear <blake.blackshear@gmail.com>

---------

Co-authored-by: Blake Blackshear <blake.blackshear@gmail.com>
---
 frigate/config/auth.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frigate/config/auth.py b/frigate/config/auth.py
index 91a692461..a202fb1af 100644
--- a/frigate/config/auth.py
+++ b/frigate/config/auth.py
@@ -13,7 +13,7 @@ class AuthConfig(FrigateBaseModel):
         default=False, title="Reset the admin password on startup"
     )
     cookie_name: str = Field(
-        default="frigate_token", title="Name for jwt token cookie", pattern=r"^[a-z]_*$"
+        default="frigate_token", title="Name for jwt token cookie", pattern=r"^[a-z_]+$"
     )
     cookie_secure: bool = Field(default=False, title="Set secure flag on cookie")
     session_length: int = Field(

From 9ae839ad723dcb2b3c89544419a5b96779a44062 Mon Sep 17 00:00:00 2001
From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>
Date: Mon, 18 Nov 2024 12:26:44 -0600
Subject: [PATCH 06/10] Tracked object metadata changes (#15055)

* add enum and change topic name

* frontend renaming

* docs

* only display sublabel score if it it exists

* remove debug print
---
 docs/docs/integrations/mqtt.md                       | 12 ++++++++++++
 frigate/comms/dispatcher.py                          | 12 +++++++++---
 frigate/embeddings/maintainer.py                     |  7 ++++++-
 frigate/types.py                                     |  4 ++++
 web/src/api/ws.tsx                                   |  4 ++--
 .../components/overlay/detail/SearchDetailDialog.tsx |  2 +-
 web/src/pages/Explore.tsx                            |  8 ++++----
 web/src/views/explore/ExploreView.tsx                |  6 +++---
 8 files changed, 41 insertions(+), 14 deletions(-)

diff --git a/docs/docs/integrations/mqtt.md b/docs/docs/integrations/mqtt.md
index e606d29fc..194821cbd 100644
--- a/docs/docs/integrations/mqtt.md
+++ b/docs/docs/integrations/mqtt.md
@@ -94,6 +94,18 @@ Message published for each changed tracked object. The first message is publishe
 }
 ```
 
+### `frigate/tracked_object_update`
+
+Message published for updates to tracked object metadata, for example when GenAI runs and returns a tracked object description.
+
+```json
+{
+  "type": "description",
+  "id": "1607123955.475377-mxklsc",
+  "description": "The car is a red sedan moving away from the camera."
+}
+```
+
 ### `frigate/reviews`
 
 Message published for each changed review item. The first message is published when the `detection` or `alert` is initiated. When additional objects are detected or when a zone change occurs, it will publish a, `update` message with the same id. When the review activity has ended a final `end` message is published.
diff --git a/frigate/comms/dispatcher.py b/frigate/comms/dispatcher.py
index 1f480fa9c..2bddc97a5 100644
--- a/frigate/comms/dispatcher.py
+++ b/frigate/comms/dispatcher.py
@@ -22,7 +22,7 @@ from frigate.const import (
 )
 from frigate.models import Event, Previews, Recordings, ReviewSegment
 from frigate.ptz.onvif import OnvifCommandEnum, OnvifController
-from frigate.types import ModelStatusTypesEnum
+from frigate.types import ModelStatusTypesEnum, TrackedObjectUpdateTypesEnum
 from frigate.util.object import get_camera_regions_grid
 from frigate.util.services import restart_frigate
 
@@ -137,8 +137,14 @@ class Dispatcher:
             event.data["description"] = payload["description"]
             event.save()
             self.publish(
-                "event_update",
-                json.dumps({"id": event.id, "description": event.data["description"]}),
+                "tracked_object_update",
+                json.dumps(
+                    {
+                        "type": TrackedObjectUpdateTypesEnum.description,
+                        "id": event.id,
+                        "description": event.data["description"],
+                    }
+                ),
             )
 
         def handle_update_model_state():
diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py
index 12c8bac72..dde8f8df4 100644
--- a/frigate/embeddings/maintainer.py
+++ b/frigate/embeddings/maintainer.py
@@ -24,6 +24,7 @@ from frigate.const import CLIPS_DIR, UPDATE_EVENT_DESCRIPTION
 from frigate.events.types import EventTypeEnum
 from frigate.genai import get_genai_client
 from frigate.models import Event
+from frigate.types import TrackedObjectUpdateTypesEnum
 from frigate.util.builtin import serialize
 from frigate.util.image import SharedMemoryFrameManager, calculate_region
 
@@ -287,7 +288,11 @@ class EmbeddingMaintainer(threading.Thread):
         # fire and forget description update
         self.requestor.send_data(
             UPDATE_EVENT_DESCRIPTION,
-            {"id": event.id, "description": description},
+            {
+                "type": TrackedObjectUpdateTypesEnum.description,
+                "id": event.id,
+                "description": description,
+            },
         )
 
         # Embed the description
diff --git a/frigate/types.py b/frigate/types.py
index 3e6ad46cc..11ab31238 100644
--- a/frigate/types.py
+++ b/frigate/types.py
@@ -19,3 +19,7 @@ class ModelStatusTypesEnum(str, Enum):
     downloading = "downloading"
     downloaded = "downloaded"
     error = "error"
+
+
+class TrackedObjectUpdateTypesEnum(str, Enum):
+    description = "description"
diff --git a/web/src/api/ws.tsx b/web/src/api/ws.tsx
index c7bb74095..9b8924d1b 100644
--- a/web/src/api/ws.tsx
+++ b/web/src/api/ws.tsx
@@ -407,9 +407,9 @@ export function useImproveContrast(camera: string): {
   return { payload: payload as ToggleableSetting, send };
 }
 
-export function useEventUpdate(): { payload: string } {
+export function useTrackedObjectUpdate(): { payload: string } {
   const {
     value: { payload },
-  } = useWs("event_update", "");
+  } = useWs("tracked_object_update", "");
   return useDeepMemo(JSON.parse(payload as string));
 }
diff --git a/web/src/components/overlay/detail/SearchDetailDialog.tsx b/web/src/components/overlay/detail/SearchDetailDialog.tsx
index 5eca9a934..f7af31606 100644
--- a/web/src/components/overlay/detail/SearchDetailDialog.tsx
+++ b/web/src/components/overlay/detail/SearchDetailDialog.tsx
@@ -309,7 +309,7 @@ function ObjectDetailsTab({
       return undefined;
     }
 
-    if (search.sub_label) {
+    if (search.sub_label && search.data?.sub_label_score) {
       return Math.round((search.data?.sub_label_score ?? 0) * 100);
     } else {
       return undefined;
diff --git a/web/src/pages/Explore.tsx b/web/src/pages/Explore.tsx
index 711666807..2bf2bb022 100644
--- a/web/src/pages/Explore.tsx
+++ b/web/src/pages/Explore.tsx
@@ -1,6 +1,6 @@
 import {
   useEmbeddingsReindexProgress,
-  useEventUpdate,
+  useTrackedObjectUpdate,
   useModelState,
 } from "@/api/ws";
 import ActivityIndicator from "@/components/indicators/activity-indicator";
@@ -227,15 +227,15 @@ export default function Explore() {
 
   // mutation and revalidation
 
-  const eventUpdate = useEventUpdate();
+  const trackedObjectUpdate = useTrackedObjectUpdate();
 
   useEffect(() => {
-    if (eventUpdate) {
+    if (trackedObjectUpdate) {
       mutate();
     }
     // mutate / revalidate when event description updates come in
     // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [eventUpdate]);
+  }, [trackedObjectUpdate]);
 
   // embeddings reindex progress
 
diff --git a/web/src/views/explore/ExploreView.tsx b/web/src/views/explore/ExploreView.tsx
index f37c37453..ea9c3cbef 100644
--- a/web/src/views/explore/ExploreView.tsx
+++ b/web/src/views/explore/ExploreView.tsx
@@ -15,7 +15,7 @@ import { SearchResult } from "@/types/search";
 import ImageLoadingIndicator from "@/components/indicators/ImageLoadingIndicator";
 import useImageLoaded from "@/hooks/use-image-loaded";
 import ActivityIndicator from "@/components/indicators/activity-indicator";
-import { useEventUpdate } from "@/api/ws";
+import { useTrackedObjectUpdate } from "@/api/ws";
 import { isEqual } from "lodash";
 import TimeAgo from "@/components/dynamic/TimeAgo";
 import SearchResultActions from "@/components/menu/SearchResultActions";
@@ -72,13 +72,13 @@ export default function ExploreView({
     }, {});
   }, [events]);
 
-  const eventUpdate = useEventUpdate();
+  const trackedObjectUpdate = useTrackedObjectUpdate();
 
   useEffect(() => {
     mutate();
     // mutate / revalidate when event description updates come in
     // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [eventUpdate]);
+  }, [trackedObjectUpdate]);
 
   // update search detail when results change
 

From a67ff3843afe5cde45a03bcaa7a374dff67e4e25 Mon Sep 17 00:00:00 2001
From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>
Date: Tue, 19 Nov 2024 09:41:16 -0600
Subject: [PATCH 07/10] Update genai docs (#15070)

---
 docs/docs/configuration/genai.md | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/docs/docs/configuration/genai.md b/docs/docs/configuration/genai.md
index da4b8afd8..67872876b 100644
--- a/docs/docs/configuration/genai.md
+++ b/docs/docs/configuration/genai.md
@@ -142,6 +142,10 @@ Frigate's thumbnail search excels at identifying specific details about tracked
 
 While generating simple descriptions of detected objects is useful, understanding intent provides a deeper layer of insight. Instead of just recognizing "what" is in a scene, Frigate’s default prompts aim to infer "why" it might be there or "what" it could do next. Descriptions tell you what’s happening, but intent gives context. For instance, a person walking toward a door might seem like a visitor, but if they’re moving quickly after hours, you can infer a potential break-in attempt. Detecting a person loitering near a door at night can trigger an alert sooner than simply noting "a person standing by the door," helping you respond based on the situation’s context.
 
+### Using GenAI for notifications
+
+Frigate provides an [MQTT topic](/integrations/mqtt), `frigate/tracked_object_update`, that is updated with a JSON payload containing `event_id` and `description` when your AI provider returns a description for a tracked object. This description could be used directly in notifications, such as sending alerts to your phone or making audio announcements. If additional details from the tracked object are needed, you can query the [HTTP API](/integrations/api) using the `event_id`, eg: `http://frigate_ip:5000/api/events/<event_id>`.
+
 ## Custom Prompts
 
 Frigate sends multiple frames from the tracked object along with a prompt to your Generative AI provider asking it to generate a description. The default prompt is as follows:
@@ -172,7 +176,7 @@ genai:
 
 Prompts can also be overriden at the camera level to provide a more detailed prompt to the model about your specific camera, if you desire. By default, descriptions will be generated for all tracked objects and all zones. But you can also optionally specify `objects` and `required_zones` to only generate descriptions for certain tracked objects or zones.
 
-Optionally, you can generate the description using a snapshot (if enabled) by setting `use_snapshot` to `True`. By default, this is set to `False`, which sends the thumbnails collected over the object's lifetime to the model. Using a snapshot provides the AI with a higher-resolution image (typically downscaled by the AI itself), but the trade-off is that only a single image is used, which might limit the model's ability to determine object movement or direction.
+Optionally, you can generate the description using a snapshot (if enabled) by setting `use_snapshot` to `True`. By default, this is set to `False`, which sends the uncompressed images from the `detect` stream collected over the object's lifetime to the model. Once the object lifecycle ends, only a single compressed and cropped thumbnail is saved with the tracked object. Using a snapshot might be useful when you want to _regenerate_ a tracked object's description as it will provide the AI with a higher-quality image (typically downscaled by the AI itself) than the cropped/compressed thumbnail. Using a snapshot otherwise has a trade-off in that only a single image is sent to your provider, which will limit the model's ability to determine object movement or direction.
 
 ```yaml
 cameras:

From 66277fbb6c3085acbb0f669b0c6fc4e9eee3d5b0 Mon Sep 17 00:00:00 2001
From: Nicolas Mowen <nickmowen213@gmail.com>
Date: Tue, 19 Nov 2024 11:20:04 -0700
Subject: [PATCH 08/10] Fix embeddings (#15072)

* Fix embeddings reading frames

* Fix event update reading

* Formatting

* Pin AIO http to fix build failure

* Pin starlette
---
 docker/main/requirements-wheels.txt |  2 ++
 frigate/comms/events_updater.py     |  2 +-
 frigate/embeddings/maintainer.py    |  9 +++++----
 frigate/events/maintainer.py        |  2 +-
 frigate/object_processing.py        | 27 +++++++++++++++------------
 5 files changed, 24 insertions(+), 18 deletions(-)

diff --git a/docker/main/requirements-wheels.txt b/docker/main/requirements-wheels.txt
index 795456588..4db88ccd2 100644
--- a/docker/main/requirements-wheels.txt
+++ b/docker/main/requirements-wheels.txt
@@ -1,5 +1,7 @@
 click == 8.1.*
 # FastAPI
+aiohttp == 3.11.2
+starlette == 0.41.2
 starlette-context == 0.3.6
 fastapi == 0.115.*
 uvicorn == 0.30.*
diff --git a/frigate/comms/events_updater.py b/frigate/comms/events_updater.py
index 7a5772273..98b6ccb7a 100644
--- a/frigate/comms/events_updater.py
+++ b/frigate/comms/events_updater.py
@@ -14,7 +14,7 @@ class EventUpdatePublisher(Publisher):
         super().__init__("update")
 
     def publish(
-        self, payload: tuple[EventTypeEnum, EventStateEnum, str, dict[str, any]]
+        self, payload: tuple[EventTypeEnum, EventStateEnum, str, str, dict[str, any]]
     ) -> None:
         super().publish(payload)
 
diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py
index dde8f8df4..d58a7f431 100644
--- a/frigate/embeddings/maintainer.py
+++ b/frigate/embeddings/maintainer.py
@@ -114,7 +114,7 @@ class EmbeddingMaintainer(threading.Thread):
         if update is None:
             return
 
-        source_type, _, camera, data = update
+        source_type, _, camera, frame_name, data = update
 
         if not camera or source_type != EventTypeEnum.tracked_object:
             return
@@ -134,8 +134,9 @@ class EmbeddingMaintainer(threading.Thread):
 
         # Create our own thumbnail based on the bounding box and the frame time
         try:
-            frame_id = f"{camera}{data['frame_time']}"
-            yuv_frame = self.frame_manager.get(frame_id, camera_config.frame_shape_yuv)
+            yuv_frame = self.frame_manager.get(
+                frame_name, camera_config.frame_shape_yuv
+            )
 
             if yuv_frame is not None:
                 data["thumbnail"] = self._create_thumbnail(yuv_frame, data["box"])
@@ -147,7 +148,7 @@ class EmbeddingMaintainer(threading.Thread):
 
                 self.tracked_events[data["id"]].append(data)
 
-                self.frame_manager.close(frame_id)
+                self.frame_manager.close(frame_name)
         except FileNotFoundError:
             pass
 
diff --git a/frigate/events/maintainer.py b/frigate/events/maintainer.py
index b17bd5d35..3a4209ec3 100644
--- a/frigate/events/maintainer.py
+++ b/frigate/events/maintainer.py
@@ -75,7 +75,7 @@ class EventProcessor(threading.Thread):
             if update == None:
                 continue
 
-            source_type, event_type, camera, event_data = update
+            source_type, event_type, camera, _, event_data = update
 
             logger.debug(
                 f"Event received: {source_type} {event_type} {camera} {event_data['id']}"
diff --git a/frigate/object_processing.py b/frigate/object_processing.py
index 23c84eedd..937c935ba 100644
--- a/frigate/object_processing.py
+++ b/frigate/object_processing.py
@@ -262,7 +262,7 @@ class CameraState:
 
             # call event handlers
             for c in self.callbacks["start"]:
-                c(self.name, new_obj, frame_time)
+                c(self.name, new_obj, frame_name)
 
         for id in updated_ids:
             updated_obj = tracked_objects[id]
@@ -272,7 +272,7 @@ class CameraState:
 
             if autotracker_update or significant_update:
                 for c in self.callbacks["autotrack"]:
-                    c(self.name, updated_obj, frame_time)
+                    c(self.name, updated_obj, frame_name)
 
             if thumb_update and current_frame is not None:
                 # ensure this frame is stored in the cache
@@ -293,7 +293,7 @@ class CameraState:
             ) or significant_update:
                 # call event handlers
                 for c in self.callbacks["update"]:
-                    c(self.name, updated_obj, frame_time)
+                    c(self.name, updated_obj, frame_name)
                 updated_obj.last_published = frame_time
 
         for id in removed_ids:
@@ -302,7 +302,7 @@ class CameraState:
             if "end_time" not in removed_obj.obj_data:
                 removed_obj.obj_data["end_time"] = frame_time
                 for c in self.callbacks["end"]:
-                    c(self.name, removed_obj, frame_time)
+                    c(self.name, removed_obj, frame_name)
 
         # TODO: can i switch to looking this up and only changing when an event ends?
         # maintain best objects
@@ -368,11 +368,11 @@ class CameraState:
                 ):
                     self.best_objects[object_type] = obj
                     for c in self.callbacks["snapshot"]:
-                        c(self.name, self.best_objects[object_type], frame_time)
+                        c(self.name, self.best_objects[object_type], frame_name)
             else:
                 self.best_objects[object_type] = obj
                 for c in self.callbacks["snapshot"]:
-                    c(self.name, self.best_objects[object_type], frame_time)
+                    c(self.name, self.best_objects[object_type], frame_name)
 
         for c in self.callbacks["camera_activity"]:
             c(self.name, camera_activity)
@@ -447,7 +447,7 @@ class CameraState:
                     c(self.name, obj_name, 0)
                 self.active_object_counts[obj_name] = 0
             for c in self.callbacks["snapshot"]:
-                c(self.name, self.best_objects[obj_name], frame_time)
+                c(self.name, self.best_objects[obj_name], frame_name)
 
         # cleanup thumbnail frame cache
         current_thumb_frames = {
@@ -518,17 +518,18 @@ class TrackedObjectProcessor(threading.Thread):
         self.zone_data = defaultdict(lambda: defaultdict(dict))
         self.active_zone_data = defaultdict(lambda: defaultdict(dict))
 
-        def start(camera, obj: TrackedObject, current_frame_time):
+        def start(camera: str, obj: TrackedObject, frame_name: str):
             self.event_sender.publish(
                 (
                     EventTypeEnum.tracked_object,
                     EventStateEnum.start,
                     camera,
+                    frame_name,
                     obj.to_dict(),
                 )
             )
 
-        def update(camera, obj: TrackedObject, current_frame_time):
+        def update(camera: str, obj: TrackedObject, frame_name: str):
             obj.has_snapshot = self.should_save_snapshot(camera, obj)
             obj.has_clip = self.should_retain_recording(camera, obj)
             after = obj.to_dict()
@@ -544,14 +545,15 @@ class TrackedObjectProcessor(threading.Thread):
                     EventTypeEnum.tracked_object,
                     EventStateEnum.update,
                     camera,
+                    frame_name,
                     obj.to_dict(include_thumbnail=True),
                 )
             )
 
-        def autotrack(camera, obj: TrackedObject, current_frame_time):
+        def autotrack(camera: str, obj: TrackedObject, frame_name: str):
             self.ptz_autotracker_thread.ptz_autotracker.autotrack_object(camera, obj)
 
-        def end(camera, obj: TrackedObject, current_frame_time):
+        def end(camera: str, obj: TrackedObject, frame_name: str):
             # populate has_snapshot
             obj.has_snapshot = self.should_save_snapshot(camera, obj)
             obj.has_clip = self.should_retain_recording(camera, obj)
@@ -606,11 +608,12 @@ class TrackedObjectProcessor(threading.Thread):
                     EventTypeEnum.tracked_object,
                     EventStateEnum.end,
                     camera,
+                    frame_name,
                     obj.to_dict(include_thumbnail=True),
                 )
             )
 
-        def snapshot(camera, obj: TrackedObject, current_frame_time):
+        def snapshot(camera, obj: TrackedObject, frame_name: str):
             mqtt_config: MqttConfig = self.config.cameras[camera].mqtt
             if mqtt_config.enabled and self.should_mqtt_snapshot(camera, obj):
                 jpg_bytes = obj.get_jpg_bytes(

From 0df091f38781cec0cba1ca66e0e42a720151c38a Mon Sep 17 00:00:00 2001
From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>
Date: Tue, 19 Nov 2024 14:33:01 -0600
Subject: [PATCH 09/10] Fix link to api in genai docs (#15075)

---
 docs/docs/configuration/genai.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/docs/configuration/genai.md b/docs/docs/configuration/genai.md
index 67872876b..fac44ed03 100644
--- a/docs/docs/configuration/genai.md
+++ b/docs/docs/configuration/genai.md
@@ -144,7 +144,7 @@ While generating simple descriptions of detected objects is useful, understandin
 
 ### Using GenAI for notifications
 
-Frigate provides an [MQTT topic](/integrations/mqtt), `frigate/tracked_object_update`, that is updated with a JSON payload containing `event_id` and `description` when your AI provider returns a description for a tracked object. This description could be used directly in notifications, such as sending alerts to your phone or making audio announcements. If additional details from the tracked object are needed, you can query the [HTTP API](/integrations/api) using the `event_id`, eg: `http://frigate_ip:5000/api/events/<event_id>`.
+Frigate provides an [MQTT topic](/integrations/mqtt), `frigate/tracked_object_update`, that is updated with a JSON payload containing `event_id` and `description` when your AI provider returns a description for a tracked object. This description could be used directly in notifications, such as sending alerts to your phone or making audio announcements. If additional details from the tracked object are needed, you can query the [HTTP API](/integrations/api/event-events-event-id-get) using the `event_id`, eg: `http://frigate_ip:5000/api/events/<event_id>`.
 
 ## Custom Prompts
 

From e76f4e9bd9e85e81e2f88921b70566e5ed54dca6 Mon Sep 17 00:00:00 2001
From: Rui Alves <alvesrui951@gmail.com>
Date: Tue, 19 Nov 2024 23:35:10 +0000
Subject: [PATCH 10/10] Started unit tests for the review controller (#15077)

* Started unit tests for the review controller

* Revert "Started unit tests for the review controller"

This reverts commit 7746eb146f813a1226544844d42eda31ed60432b.

* Started unit tests for the review controller

* FIrst test

* Added test for review endpoint (time filter - after + before)

* Assert expected event

* Added more tests for review endpoint

* Added test for review endpoint with all filters

* Added test for review endpoint with limit

* Comment

* Renamed tests to increase readability
---
 .cspell/frigate-dictionary.txt            |   3 +
 frigate/test/http_api/__init__.py         |   0
 frigate/test/http_api/base_http_test.py   | 162 ++++++++++++++++++++++
 frigate/test/http_api/test_http_review.py | 110 +++++++++++++++
 4 files changed, 275 insertions(+)
 create mode 100644 frigate/test/http_api/__init__.py
 create mode 100644 frigate/test/http_api/base_http_test.py
 create mode 100644 frigate/test/http_api/test_http_review.py

diff --git a/.cspell/frigate-dictionary.txt b/.cspell/frigate-dictionary.txt
index b019f8492..64fd7ca72 100644
--- a/.cspell/frigate-dictionary.txt
+++ b/.cspell/frigate-dictionary.txt
@@ -12,6 +12,7 @@ argmax
 argmin
 argpartition
 ascontiguousarray
+astype
 authelia
 authentik
 autodetected
@@ -195,6 +196,7 @@ poweroff
 preexec
 probesize
 protobuf
+pstate
 psutil
 pubkey
 putenv
@@ -278,6 +280,7 @@ uvicorn
 vaapi
 vainfo
 variations
+vbios
 vconcat
 vitb
 vstream
diff --git a/frigate/test/http_api/__init__.py b/frigate/test/http_api/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/frigate/test/http_api/base_http_test.py b/frigate/test/http_api/base_http_test.py
new file mode 100644
index 000000000..013785692
--- /dev/null
+++ b/frigate/test/http_api/base_http_test.py
@@ -0,0 +1,162 @@
+import datetime
+import logging
+import os
+import unittest
+
+from peewee_migrate import Router
+from playhouse.sqlite_ext import SqliteExtDatabase
+from playhouse.sqliteq import SqliteQueueDatabase
+
+from frigate.api.fastapi_app import create_fastapi_app
+from frigate.config import FrigateConfig
+from frigate.models import Event, ReviewSegment
+from frigate.review.maintainer import SeverityEnum
+from frigate.test.const import TEST_DB, TEST_DB_CLEANUPS
+
+
+class BaseTestHttp(unittest.TestCase):
+    def setUp(self, models):
+        # setup clean database for each test run
+        migrate_db = SqliteExtDatabase("test.db")
+        del logging.getLogger("peewee_migrate").handlers[:]
+        router = Router(migrate_db)
+        router.run()
+        migrate_db.close()
+        self.db = SqliteQueueDatabase(TEST_DB)
+        self.db.bind(models)
+
+        self.minimal_config = {
+            "mqtt": {"host": "mqtt"},
+            "cameras": {
+                "front_door": {
+                    "ffmpeg": {
+                        "inputs": [
+                            {"path": "rtsp://10.0.0.1:554/video", "roles": ["detect"]}
+                        ]
+                    },
+                    "detect": {
+                        "height": 1080,
+                        "width": 1920,
+                        "fps": 5,
+                    },
+                }
+            },
+        }
+        self.test_stats = {
+            "detection_fps": 13.7,
+            "detectors": {
+                "cpu1": {
+                    "detection_start": 0.0,
+                    "inference_speed": 91.43,
+                    "pid": 42,
+                },
+                "cpu2": {
+                    "detection_start": 0.0,
+                    "inference_speed": 84.99,
+                    "pid": 44,
+                },
+            },
+            "front_door": {
+                "camera_fps": 0.0,
+                "capture_pid": 53,
+                "detection_fps": 0.0,
+                "pid": 52,
+                "process_fps": 0.0,
+                "skipped_fps": 0.0,
+            },
+            "service": {
+                "storage": {
+                    "/dev/shm": {
+                        "free": 50.5,
+                        "mount_type": "tmpfs",
+                        "total": 67.1,
+                        "used": 16.6,
+                    },
+                    "/media/frigate/clips": {
+                        "free": 42429.9,
+                        "mount_type": "ext4",
+                        "total": 244529.7,
+                        "used": 189607.0,
+                    },
+                    "/media/frigate/recordings": {
+                        "free": 0.2,
+                        "mount_type": "ext4",
+                        "total": 8.0,
+                        "used": 7.8,
+                    },
+                    "/tmp/cache": {
+                        "free": 976.8,
+                        "mount_type": "tmpfs",
+                        "total": 1000.0,
+                        "used": 23.2,
+                    },
+                },
+                "uptime": 101113,
+                "version": "0.10.1",
+                "latest_version": "0.11",
+            },
+        }
+
+    def tearDown(self):
+        if not self.db.is_closed():
+            self.db.close()
+
+        try:
+            for file in TEST_DB_CLEANUPS:
+                os.remove(file)
+        except OSError:
+            pass
+
+    def create_app(self, stats=None):
+        return create_fastapi_app(
+            FrigateConfig(**self.minimal_config),
+            self.db,
+            None,
+            None,
+            None,
+            None,
+            None,
+            stats,
+            None,
+        )
+
+    def insert_mock_event(
+        self,
+        id: str,
+        start_time: datetime.datetime = datetime.datetime.now().timestamp(),
+    ) -> Event:
+        """Inserts a basic event model with a given id."""
+        return Event.insert(
+            id=id,
+            label="Mock",
+            camera="front_door",
+            start_time=start_time,
+            end_time=start_time + 20,
+            top_score=100,
+            false_positive=False,
+            zones=list(),
+            thumbnail="",
+            region=[],
+            box=[],
+            area=0,
+            has_clip=True,
+            has_snapshot=True,
+        ).execute()
+
+    def insert_mock_review_segment(
+        self,
+        id: str,
+        start_time: datetime.datetime = datetime.datetime.now().timestamp(),
+        end_time: datetime.datetime = datetime.datetime.now().timestamp() + 20,
+    ) -> Event:
+        """Inserts a basic event model with a given id."""
+        return ReviewSegment.insert(
+            id=id,
+            camera="front_door",
+            start_time=start_time,
+            end_time=end_time,
+            has_been_reviewed=False,
+            severity=SeverityEnum.alert,
+            thumb_path=False,
+            data={},
+        ).execute()
diff --git a/frigate/test/http_api/test_http_review.py b/frigate/test/http_api/test_http_review.py
new file mode 100644
index 000000000..19e1f26f8
--- /dev/null
+++ b/frigate/test/http_api/test_http_review.py
@@ -0,0 +1,110 @@
+import datetime
+
+from fastapi.testclient import TestClient
+
+from frigate.models import Event, ReviewSegment
+from frigate.test.http_api.base_http_test import BaseTestHttp
+
+
+class TestHttpReview(BaseTestHttp):
+    def setUp(self):
+        super().setUp([Event, ReviewSegment])
+
+    # Does not return any data point since the end time (before parameter) is not passed and the review segment end_time is 2 seconds from now
+    def test_get_review_no_filters_no_matches(self):
+        app = super().create_app()
+        now = datetime.datetime.now().timestamp()
+
+        with TestClient(app) as client:
+            super().insert_mock_review_segment("123456.random", now, now + 2)
+            reviews_response = client.get("/review")
+            assert reviews_response.status_code == 200
+            reviews_in_response = reviews_response.json()
+            assert len(reviews_in_response) == 0
+
+    def test_get_review_no_filters(self):
+        app = super().create_app()
+        now = datetime.datetime.now().timestamp()
+
+        with TestClient(app) as client:
+            super().insert_mock_review_segment("123456.random", now - 2, now - 1)
+            reviews_response = client.get("/review")
+            assert reviews_response.status_code == 200
+            reviews_in_response = reviews_response.json()
+            assert len(reviews_in_response) == 1
+
+    def test_get_review_with_time_filter_no_matches(self):
+        app = super().create_app()
+        now = datetime.datetime.now().timestamp()
+
+        with TestClient(app) as client:
+            id = "123456.random"
+            super().insert_mock_review_segment(id, now, now + 2)
+            params = {
+                "after": now,
+                "before": now + 3,
+            }
+            reviews_response = client.get("/review", params=params)
+            assert reviews_response.status_code == 200
+            reviews_in_response = reviews_response.json()
+            assert len(reviews_in_response) == 0
+
+    def test_get_review_with_time_filter(self):
+        app = super().create_app()
+        now = datetime.datetime.now().timestamp()
+
+        with TestClient(app) as client:
+            id = "123456.random"
+            super().insert_mock_review_segment(id, now, now + 2)
+            params = {
+                "after": now - 1,
+                "before": now + 3,
+            }
+            reviews_response = client.get("/review", params=params)
+            assert reviews_response.status_code == 200
+            reviews_in_response = reviews_response.json()
+            assert len(reviews_in_response) == 1
+            assert reviews_in_response[0]["id"] == id
+
+    def test_get_review_with_limit_filter(self):
+        app = super().create_app()
+        now = datetime.datetime.now().timestamp()
+
+        with TestClient(app) as client:
+            id = "123456.random"
+            id2 = "654321.random"
+            super().insert_mock_review_segment(id, now, now + 2)
+            super().insert_mock_review_segment(id2, now + 1, now + 2)
+            params = {
+                "limit": 1,
+                "after": now,
+                "before": now + 3,
+            }
+            reviews_response = client.get("/review", params=params)
+            assert reviews_response.status_code == 200
+            reviews_in_response = reviews_response.json()
+            assert len(reviews_in_response) == 1
+            assert reviews_in_response[0]["id"] == id2
+
+    def test_get_review_with_all_filters(self):
+        app = super().create_app()
+        now = datetime.datetime.now().timestamp()
+
+        with TestClient(app) as client:
+            id = "123456.random"
+            super().insert_mock_review_segment(id, now, now + 2)
+            params = {
+                "cameras": "front_door",
+                "labels": "all",
+                "zones": "all",
+                "reviewed": 0,
+                "limit": 1,
+                "severity": "alert",
+                "after": now - 1,
+                "before": now + 3,
+            }
+            reviews_response = client.get("/review", params=params)
+            assert reviews_response.status_code == 200
+            reviews_in_response = reviews_response.json()
+            assert len(reviews_in_response) == 1
+            assert reviews_in_response[0]["id"] == id