Make Birdseye clickable (#18628)

* keep track of layout changes and publish on change

* websocket hook

* clickable overlay div to navigate to full camera view

frigate/comms/dispatcher.py

@@ -21,6 +21,7 @@ from frigate.const import (
     INSERT_PREVIEW,
     NOTIFICATION_TEST,
     REQUEST_REGION_GRID,
+    UPDATE_BIRDSEYE_LAYOUT,
     UPDATE_CAMERA_ACTIVITY,
     UPDATE_EMBEDDINGS_REINDEX_PROGRESS,
     UPDATE_EVENT_DESCRIPTION,
@@ -55,6 +56,7 @@ class Dispatcher:
         self.camera_activity = CameraActivityManager(config, self.publish)
         self.model_state = {}
         self.embeddings_reindex = {}
+        self.birdseye_layout = {}
 
         self._camera_settings_handlers: dict[str, Callable] = {
             "audio": self._on_audio_command,
@@ -168,6 +170,14 @@ class Dispatcher:
                 json.dumps(self.embeddings_reindex.copy()),
             )
 
+        def handle_update_birdseye_layout():
+            if payload:
+                self.birdseye_layout = payload
+                self.publish("birdseye_layout", json.dumps(self.birdseye_layout))
+
+        def handle_birdseye_layout():
+            self.publish("birdseye_layout", json.dumps(self.birdseye_layout.copy()))
+
         def handle_on_connect():
             camera_status = self.camera_activity.last_camera_activity.copy()
             cameras_with_status = camera_status.keys()
@@ -205,6 +215,7 @@ class Dispatcher:
                 "embeddings_reindex_progress",
                 json.dumps(self.embeddings_reindex.copy()),
             )
+            self.publish("birdseye_layout", json.dumps(self.birdseye_layout.copy()))
 
         def handle_notification_test():
             self.publish("notification_test", "Test notification")
@@ -220,10 +231,12 @@ class Dispatcher:
             UPDATE_EVENT_DESCRIPTION: handle_update_event_description,
             UPDATE_MODEL_STATE: handle_update_model_state,
             UPDATE_EMBEDDINGS_REINDEX_PROGRESS: handle_update_embeddings_reindex_progress,
+            UPDATE_BIRDSEYE_LAYOUT: handle_update_birdseye_layout,
             NOTIFICATION_TEST: handle_notification_test,
             "restart": handle_restart,
             "embeddingsReindexProgress": handle_embeddings_reindex_progress,
             "modelState": handle_model_state,
+            "birdseyeLayout": handle_birdseye_layout,
             "onConnect": handle_on_connect,
         }
 

frigate/const.py

@@ -109,6 +109,7 @@ UPDATE_CAMERA_ACTIVITY = "update_camera_activity"
 UPDATE_EVENT_DESCRIPTION = "update_event_description"
 UPDATE_MODEL_STATE = "update_model_state"
 UPDATE_EMBEDDINGS_REINDEX_PROGRESS = "handle_embeddings_reindex_progress"
+UPDATE_BIRDSEYE_LAYOUT = "update_birdseye_layout"
 NOTIFICATION_TEST = "notification_test"
 
 # Stats Values

frigate/output/birdseye.py

@@ -15,8 +15,9 @@ from typing import Any, Optional
 import cv2
 import numpy as np
 
+from frigate.comms.inter_process import InterProcessRequestor
 from frigate.config import BirdseyeModeEnum, FfmpegConfig, FrigateConfig
-from frigate.const import BASE_DIR, BIRDSEYE_PIPE, INSTALL_DIR
+from frigate.const import BASE_DIR, BIRDSEYE_PIPE, INSTALL_DIR, UPDATE_BIRDSEYE_LAYOUT
 from frigate.util.image import (
     SharedMemoryFrameManager,
     copy_yuv_to_position,
@@ -380,10 +381,24 @@ class BirdsEyeFrameManager:
         if mode == BirdseyeModeEnum.objects and object_box_count > 0:
             return True
 
-    def update_frame(self, frame: Optional[np.ndarray] = None) -> bool:
+    def get_camera_coordinates(self) -> dict[str, dict[str, int]]:
+        """Return the coordinates of each camera in the current layout."""
+        coordinates = {}
+        for row in self.camera_layout:
+            for position in row:
+                camera_name, (x, y, width, height) = position
+                coordinates[camera_name] = {
+                    "x": x,
+                    "y": y,
+                    "width": width,
+                    "height": height,
+                }
+        return coordinates
+
+    def update_frame(self, frame: Optional[np.ndarray] = None) -> tuple[bool, bool]:
         """
         Update birdseye, optionally with a new frame.
-        When no frame is passed, check the layout and update for any disabled cameras.
+        Returns (frame_changed, layout_changed) to indicate if the frame or layout changed.
         """
 
         # determine how many cameras are tracking objects within the last inactivity_threshold seconds
@@ -421,19 +436,21 @@ class BirdsEyeFrameManager:
                 max_camera_refresh = True
                 self.last_refresh_time = now
 
-        # Track if the frame changes
+        # Track if the frame or layout changes
         frame_changed = False
+        layout_changed = False
 
         # If no active cameras and layout is already empty, no update needed
         if len(active_cameras) == 0:
             # if the layout is already cleared
             if len(self.camera_layout) == 0:
-                return False
+                return False, False
             # if the layout needs to be cleared
             self.camera_layout = []
             self.active_cameras = set()
             self.clear_frame()
             frame_changed = True
+            layout_changed = True
         else:
             # Determine if layout needs resetting
             if len(self.active_cameras) - len(active_cameras) == 0:
@@ -453,7 +470,7 @@ class BirdsEyeFrameManager:
                 logger.debug("Resetting Birdseye layout...")
                 self.clear_frame()
                 self.active_cameras = active_cameras
-
+                layout_changed = True  # Layout is changing due to reset
                 # this also converts added_cameras from a set to a list since we need
                 # to pop elements in order
                 active_cameras_to_add = sorted(
@@ -503,7 +520,7 @@ class BirdsEyeFrameManager:
                     # decrease scaling coefficient until height of all cameras can fit into the birdseye canvas
                     while calculating:
                         if self.stop_event.is_set():
-                            return
+                            return frame_changed, layout_changed
 
                         layout_candidate = self.calculate_layout(
                             active_cameras_to_add, coefficient
@@ -517,7 +534,7 @@ class BirdsEyeFrameManager:
                                 logger.error(
                                     "Error finding appropriate birdseye layout"
                                 )
-                                return
+                                return frame_changed, layout_changed
                         calculating = False
                         self.canvas.set_coefficient(len(active_cameras), coefficient)
 
@@ -535,7 +552,7 @@ class BirdsEyeFrameManager:
             if frame is not None:  # Frame presence indicates a potential change
                 frame_changed = True
 
-        return frame_changed
+        return frame_changed, layout_changed
 
     def calculate_layout(
         self,
@@ -687,7 +704,11 @@ class BirdsEyeFrameManager:
         motion_count: int,
         frame_time: float,
         frame: np.ndarray,
-    ) -> bool:
+    ) -> tuple[bool, bool]:
+        """
+        Update birdseye for a specific camera with new frame data.
+        Returns (frame_changed, layout_changed) to indicate if the frame or layout changed.
+        """
         # don't process if birdseye is disabled for this camera
         camera_config = self.config.cameras[camera]
         force_update = False
@@ -700,7 +721,7 @@ class BirdsEyeFrameManager:
                 self.cameras[camera]["last_active_frame"] = 0
                 force_update = True
             else:
-                return False
+                return False, False
 
         # update the last active frame for the camera
         self.cameras[camera]["current_frame"] = frame.copy()
@@ -712,21 +733,22 @@ class BirdsEyeFrameManager:
 
         # limit output to 10 fps
         if not force_update and (now - self.last_output_time) < 1 / 10:
-            return False
+            return False, False
 
         try:
-            updated_frame = self.update_frame(frame)
+            frame_changed, layout_changed = self.update_frame(frame)
         except Exception:
-            updated_frame = False
+            frame_changed, layout_changed = False, False
             self.active_cameras = []
             self.camera_layout = []
             print(traceback.format_exc())
 
         # if the frame was updated or the fps is too low, send frame
-        if force_update or updated_frame or (now - self.last_output_time) > 1:
+        if force_update or frame_changed or (now - self.last_output_time) > 1:
             self.last_output_time = now
-            return True
-        return False
+            return True, layout_changed
+
+        return False, layout_changed
 
 
 class Birdseye:
@@ -755,6 +777,7 @@ class Birdseye:
         self.birdseye_manager = BirdsEyeFrameManager(config, stop_event)
         self.frame_manager = SharedMemoryFrameManager()
         self.stop_event = stop_event
+        self.requestor = InterProcessRequestor()
 
         if config.birdseye.restream:
             self.birdseye_buffer = self.frame_manager.create(
@@ -789,15 +812,20 @@ class Birdseye:
         frame_time: float,
         frame: np.ndarray,
     ) -> None:
-        if self.birdseye_manager.update(
+        frame_changed, frame_layout_changed = self.birdseye_manager.update(
             camera,
             len([o for o in current_tracked_objects if not o["stationary"]]),
             len(motion_boxes),
             frame_time,
             frame,
-        ):
+        )
+        if frame_changed:
             self.__send_new_frame()
 
+            if frame_layout_changed:
+                coordinates = self.birdseye_manager.get_camera_coordinates()
+                self.requestor.send_data(UPDATE_BIRDSEYE_LAYOUT, coordinates)
+
     def stop(self) -> None:
         self.converter.join()
         self.broadcaster.join()

web/src/api/ws.tsx

@@ -426,6 +426,40 @@ export function useEmbeddingsReindexProgress(
   return { payload: data };
 }
 
+export function useBirdseyeLayout(revalidateOnFocus: boolean = true): {
+  payload: string;
+} {
+  const {
+    value: { payload },
+    send: sendCommand,
+  } = useWs("birdseye_layout", "birdseyeLayout");
+
+  const data = useDeepMemo(JSON.parse(payload as string));
+
+  useEffect(() => {
+    let listener = undefined;
+    if (revalidateOnFocus) {
+      sendCommand("birdseyeLayout");
+      listener = () => {
+        if (document.visibilityState == "visible") {
+          sendCommand("birdseyeLayout");
+        }
+      };
+      addEventListener("visibilitychange", listener);
+    }
+
+    return () => {
+      if (listener) {
+        removeEventListener("visibilitychange", listener);
+      }
+    };
+    // we know that these deps are correct
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [revalidateOnFocus]);
+
+  return { payload: data };
+}
+
 export function useMotionActivity(camera: string): { payload: string } {
   const {
     value: { payload },

web/src/components/player/BirdseyeLivePlayer.tsx

@@ -13,6 +13,7 @@ type LivePlayerProps = {
   liveMode: LivePlayerMode;
   pip?: boolean;
   containerRef: React.MutableRefObject<HTMLDivElement | null>;
+  playerRef?: React.MutableRefObject<HTMLDivElement | null>;
   onClick?: () => void;
 };
 
@@ -22,6 +23,7 @@ export default function BirdseyeLivePlayer({
   liveMode,
   pip,
   containerRef,
+  playerRef,
   onClick,
 }: LivePlayerProps) {
   let player;
@@ -76,7 +78,9 @@ export default function BirdseyeLivePlayer({
     >
       <div className="pointer-events-none absolute inset-x-0 top-0 z-10 h-[30%] w-full rounded-lg bg-gradient-to-b from-black/20 to-transparent md:rounded-2xl"></div>
       <div className="pointer-events-none absolute inset-x-0 bottom-0 z-10 h-[10%] w-full rounded-lg bg-gradient-to-t from-black/20 to-transparent md:rounded-2xl"></div>
-      <div className="size-full">{player}</div>
+      <div className="size-full" ref={playerRef}>
+        {player}
+      </div>
     </div>
   );
 }

web/src/views/live/LiveBirdseyeView.tsx

@@ -1,11 +1,13 @@
+import { useBirdseyeLayout } from "@/api/ws";
 import CameraFeatureToggle from "@/components/dynamic/CameraFeatureToggle";
 import ActivityIndicator from "@/components/indicators/activity-indicator";
 import BirdseyeLivePlayer from "@/components/player/BirdseyeLivePlayer";
 import { Button } from "@/components/ui/button";
 import { TooltipProvider } from "@/components/ui/tooltip";
 import { useResizeObserver } from "@/hooks/resize-observer";
+import { cn } from "@/lib/utils";
 import { FrigateConfig } from "@/types/frigateConfig";
-import { useEffect, useMemo, useRef, useState } from "react";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import {
   isDesktop,
   isFirefox,
@@ -122,6 +124,72 @@ export default function LiveBirdseyeView({
     return "mse";
   }, [config]);
 
+  const birdseyeLayout = useBirdseyeLayout();
+
+  // Click overlay handling
+
+  const playerRef = useRef<HTMLDivElement | null>(null);
+  const handleOverlayClick = useCallback(
+    (
+      e: React.MouseEvent<HTMLDivElement> | React.TouchEvent<HTMLDivElement>,
+    ) => {
+      let clientX;
+      let clientY;
+      if ("TouchEvent" in window && e.nativeEvent instanceof TouchEvent) {
+        clientX = e.nativeEvent.touches[0].clientX;
+        clientY = e.nativeEvent.touches[0].clientY;
+      } else if (e.nativeEvent instanceof MouseEvent) {
+        clientX = e.nativeEvent.clientX;
+        clientY = e.nativeEvent.clientY;
+      }
+
+      if (
+        playerRef.current &&
+        clientX &&
+        clientY &&
+        config &&
+        birdseyeLayout?.payload
+      ) {
+        const playerRect = playerRef.current.getBoundingClientRect();
+
+        // Calculate coordinates relative to player div, accounting for offset
+        const rawX = clientX - playerRect.left;
+        const rawY = clientY - playerRect.top;
+
+        // Ensure click is within player bounds
+        if (
+          rawX < 0 ||
+          rawX > playerRect.width ||
+          rawY < 0 ||
+          rawY > playerRect.height
+        ) {
+          return;
+        }
+
+        // Scale click coordinates to birdseye canvas resolution
+        const canvasX = rawX * (config.birdseye.width / playerRect.width);
+        const canvasY = rawY * (config.birdseye.height / playerRect.height);
+
+        for (const [cameraName, coords] of Object.entries(
+          birdseyeLayout.payload,
+        )) {
+          const parsedCoords =
+            typeof coords === "string" ? JSON.parse(coords) : coords;
+          if (
+            canvasX >= parsedCoords.x &&
+            canvasX < parsedCoords.x + parsedCoords.width &&
+            canvasY >= parsedCoords.y &&
+            canvasY < parsedCoords.y + parsedCoords.height
+          ) {
+            navigate(`/#${cameraName}`);
+            break;
+          }
+        }
+      }
+    },
+    [playerRef, config, birdseyeLayout, navigate],
+  );
+
   if (!config) {
     return <ActivityIndicator />;
   }
@@ -215,16 +283,21 @@ export default function LiveBirdseyeView({
             }}
           >
             <div
-              className={growClassName}
+              className={cn(
+                "flex flex-col items-center justify-center",
+                growClassName,
+              )}
               style={{
                 aspectRatio: constrainedAspectRatio,
               }}
+              onClick={handleOverlayClick}
             >
               <BirdseyeLivePlayer
                 className={`${fullscreen ? "*:rounded-none" : ""}`}
                 birdseyeConfig={config.birdseye}
                 liveMode={preferredLiveMode}
                 containerRef={containerRef}
+                playerRef={playerRef}
                 pip={pip}
               />
             </div>