From 9a5162752c066c7b74d15cdd779df1cc9371f5b0 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sun, 8 Jun 2025 13:06:17 -0500 Subject: [PATCH] Make Birdseye clickable (#18628) * keep track of layout changes and publish on change * websocket hook * clickable overlay div to navigate to full camera view --- frigate/comms/dispatcher.py | 13 ++++ frigate/const.py | 1 + frigate/output/birdseye.py | 66 +++++++++++----- web/src/api/ws.tsx | 34 ++++++++ .../components/player/BirdseyeLivePlayer.tsx | 6 +- web/src/views/live/LiveBirdseyeView.tsx | 77 ++++++++++++++++++- 6 files changed, 175 insertions(+), 22 deletions(-) diff --git a/frigate/comms/dispatcher.py b/frigate/comms/dispatcher.py index 6fee166b7..0a9c439f4 100644 --- a/frigate/comms/dispatcher.py +++ b/frigate/comms/dispatcher.py @@ -21,6 +21,7 @@ from frigate.const import ( INSERT_PREVIEW, NOTIFICATION_TEST, REQUEST_REGION_GRID, + UPDATE_BIRDSEYE_LAYOUT, UPDATE_CAMERA_ACTIVITY, UPDATE_EMBEDDINGS_REINDEX_PROGRESS, UPDATE_EVENT_DESCRIPTION, @@ -55,6 +56,7 @@ class Dispatcher: self.camera_activity = CameraActivityManager(config, self.publish) self.model_state = {} self.embeddings_reindex = {} + self.birdseye_layout = {} self._camera_settings_handlers: dict[str, Callable] = { "audio": self._on_audio_command, @@ -168,6 +170,14 @@ class Dispatcher: json.dumps(self.embeddings_reindex.copy()), ) + def handle_update_birdseye_layout(): + if payload: + self.birdseye_layout = payload + self.publish("birdseye_layout", json.dumps(self.birdseye_layout)) + + def handle_birdseye_layout(): + self.publish("birdseye_layout", json.dumps(self.birdseye_layout.copy())) + def handle_on_connect(): camera_status = self.camera_activity.last_camera_activity.copy() cameras_with_status = camera_status.keys() @@ -205,6 +215,7 @@ class Dispatcher: "embeddings_reindex_progress", json.dumps(self.embeddings_reindex.copy()), ) + self.publish("birdseye_layout", json.dumps(self.birdseye_layout.copy())) def handle_notification_test(): self.publish("notification_test", "Test notification") @@ -220,10 +231,12 @@ class Dispatcher: UPDATE_EVENT_DESCRIPTION: handle_update_event_description, UPDATE_MODEL_STATE: handle_update_model_state, UPDATE_EMBEDDINGS_REINDEX_PROGRESS: handle_update_embeddings_reindex_progress, + UPDATE_BIRDSEYE_LAYOUT: handle_update_birdseye_layout, NOTIFICATION_TEST: handle_notification_test, "restart": handle_restart, "embeddingsReindexProgress": handle_embeddings_reindex_progress, "modelState": handle_model_state, + "birdseyeLayout": handle_birdseye_layout, "onConnect": handle_on_connect, } diff --git a/frigate/const.py b/frigate/const.py index 183506a04..69335902e 100644 --- a/frigate/const.py +++ b/frigate/const.py @@ -109,6 +109,7 @@ UPDATE_CAMERA_ACTIVITY = "update_camera_activity" UPDATE_EVENT_DESCRIPTION = "update_event_description" UPDATE_MODEL_STATE = "update_model_state" UPDATE_EMBEDDINGS_REINDEX_PROGRESS = "handle_embeddings_reindex_progress" +UPDATE_BIRDSEYE_LAYOUT = "update_birdseye_layout" NOTIFICATION_TEST = "notification_test" # Stats Values diff --git a/frigate/output/birdseye.py b/frigate/output/birdseye.py index 78686fd63..a19436d5e 100644 --- a/frigate/output/birdseye.py +++ b/frigate/output/birdseye.py @@ -15,8 +15,9 @@ from typing import Any, Optional import cv2 import numpy as np +from frigate.comms.inter_process import InterProcessRequestor from frigate.config import BirdseyeModeEnum, FfmpegConfig, FrigateConfig -from frigate.const import BASE_DIR, BIRDSEYE_PIPE, INSTALL_DIR +from frigate.const import BASE_DIR, BIRDSEYE_PIPE, INSTALL_DIR, UPDATE_BIRDSEYE_LAYOUT from frigate.util.image import ( SharedMemoryFrameManager, copy_yuv_to_position, @@ -380,10 +381,24 @@ class BirdsEyeFrameManager: if mode == BirdseyeModeEnum.objects and object_box_count > 0: return True - def update_frame(self, frame: Optional[np.ndarray] = None) -> bool: + def get_camera_coordinates(self) -> dict[str, dict[str, int]]: + """Return the coordinates of each camera in the current layout.""" + coordinates = {} + for row in self.camera_layout: + for position in row: + camera_name, (x, y, width, height) = position + coordinates[camera_name] = { + "x": x, + "y": y, + "width": width, + "height": height, + } + return coordinates + + def update_frame(self, frame: Optional[np.ndarray] = None) -> tuple[bool, bool]: """ Update birdseye, optionally with a new frame. - When no frame is passed, check the layout and update for any disabled cameras. + Returns (frame_changed, layout_changed) to indicate if the frame or layout changed. """ # determine how many cameras are tracking objects within the last inactivity_threshold seconds @@ -421,19 +436,21 @@ class BirdsEyeFrameManager: max_camera_refresh = True self.last_refresh_time = now - # Track if the frame changes + # Track if the frame or layout changes frame_changed = False + layout_changed = False # If no active cameras and layout is already empty, no update needed if len(active_cameras) == 0: # if the layout is already cleared if len(self.camera_layout) == 0: - return False + return False, False # if the layout needs to be cleared self.camera_layout = [] self.active_cameras = set() self.clear_frame() frame_changed = True + layout_changed = True else: # Determine if layout needs resetting if len(self.active_cameras) - len(active_cameras) == 0: @@ -453,7 +470,7 @@ class BirdsEyeFrameManager: logger.debug("Resetting Birdseye layout...") self.clear_frame() self.active_cameras = active_cameras - + layout_changed = True # Layout is changing due to reset # this also converts added_cameras from a set to a list since we need # to pop elements in order active_cameras_to_add = sorted( @@ -503,7 +520,7 @@ class BirdsEyeFrameManager: # decrease scaling coefficient until height of all cameras can fit into the birdseye canvas while calculating: if self.stop_event.is_set(): - return + return frame_changed, layout_changed layout_candidate = self.calculate_layout( active_cameras_to_add, coefficient @@ -517,7 +534,7 @@ class BirdsEyeFrameManager: logger.error( "Error finding appropriate birdseye layout" ) - return + return frame_changed, layout_changed calculating = False self.canvas.set_coefficient(len(active_cameras), coefficient) @@ -535,7 +552,7 @@ class BirdsEyeFrameManager: if frame is not None: # Frame presence indicates a potential change frame_changed = True - return frame_changed + return frame_changed, layout_changed def calculate_layout( self, @@ -687,7 +704,11 @@ class BirdsEyeFrameManager: motion_count: int, frame_time: float, frame: np.ndarray, - ) -> bool: + ) -> tuple[bool, bool]: + """ + Update birdseye for a specific camera with new frame data. + Returns (frame_changed, layout_changed) to indicate if the frame or layout changed. + """ # don't process if birdseye is disabled for this camera camera_config = self.config.cameras[camera] force_update = False @@ -700,7 +721,7 @@ class BirdsEyeFrameManager: self.cameras[camera]["last_active_frame"] = 0 force_update = True else: - return False + return False, False # update the last active frame for the camera self.cameras[camera]["current_frame"] = frame.copy() @@ -712,21 +733,22 @@ class BirdsEyeFrameManager: # limit output to 10 fps if not force_update and (now - self.last_output_time) < 1 / 10: - return False + return False, False try: - updated_frame = self.update_frame(frame) + frame_changed, layout_changed = self.update_frame(frame) except Exception: - updated_frame = False + frame_changed, layout_changed = False, False self.active_cameras = [] self.camera_layout = [] print(traceback.format_exc()) # if the frame was updated or the fps is too low, send frame - if force_update or updated_frame or (now - self.last_output_time) > 1: + if force_update or frame_changed or (now - self.last_output_time) > 1: self.last_output_time = now - return True - return False + return True, layout_changed + + return False, layout_changed class Birdseye: @@ -755,6 +777,7 @@ class Birdseye: self.birdseye_manager = BirdsEyeFrameManager(config, stop_event) self.frame_manager = SharedMemoryFrameManager() self.stop_event = stop_event + self.requestor = InterProcessRequestor() if config.birdseye.restream: self.birdseye_buffer = self.frame_manager.create( @@ -789,15 +812,20 @@ class Birdseye: frame_time: float, frame: np.ndarray, ) -> None: - if self.birdseye_manager.update( + frame_changed, frame_layout_changed = self.birdseye_manager.update( camera, len([o for o in current_tracked_objects if not o["stationary"]]), len(motion_boxes), frame_time, frame, - ): + ) + if frame_changed: self.__send_new_frame() + if frame_layout_changed: + coordinates = self.birdseye_manager.get_camera_coordinates() + self.requestor.send_data(UPDATE_BIRDSEYE_LAYOUT, coordinates) + def stop(self) -> None: self.converter.join() self.broadcaster.join() diff --git a/web/src/api/ws.tsx b/web/src/api/ws.tsx index 79bf9e79d..78c596e13 100644 --- a/web/src/api/ws.tsx +++ b/web/src/api/ws.tsx @@ -426,6 +426,40 @@ export function useEmbeddingsReindexProgress( return { payload: data }; } +export function useBirdseyeLayout(revalidateOnFocus: boolean = true): { + payload: string; +} { + const { + value: { payload }, + send: sendCommand, + } = useWs("birdseye_layout", "birdseyeLayout"); + + const data = useDeepMemo(JSON.parse(payload as string)); + + useEffect(() => { + let listener = undefined; + if (revalidateOnFocus) { + sendCommand("birdseyeLayout"); + listener = () => { + if (document.visibilityState == "visible") { + sendCommand("birdseyeLayout"); + } + }; + addEventListener("visibilitychange", listener); + } + + return () => { + if (listener) { + removeEventListener("visibilitychange", listener); + } + }; + // we know that these deps are correct + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [revalidateOnFocus]); + + return { payload: data }; +} + export function useMotionActivity(camera: string): { payload: string } { const { value: { payload }, diff --git a/web/src/components/player/BirdseyeLivePlayer.tsx b/web/src/components/player/BirdseyeLivePlayer.tsx index 286f19216..2e9461293 100644 --- a/web/src/components/player/BirdseyeLivePlayer.tsx +++ b/web/src/components/player/BirdseyeLivePlayer.tsx @@ -13,6 +13,7 @@ type LivePlayerProps = { liveMode: LivePlayerMode; pip?: boolean; containerRef: React.MutableRefObject; + playerRef?: React.MutableRefObject; onClick?: () => void; }; @@ -22,6 +23,7 @@ export default function BirdseyeLivePlayer({ liveMode, pip, containerRef, + playerRef, onClick, }: LivePlayerProps) { let player; @@ -76,7 +78,9 @@ export default function BirdseyeLivePlayer({ >
-
{player}
+
+ {player} +
); } diff --git a/web/src/views/live/LiveBirdseyeView.tsx b/web/src/views/live/LiveBirdseyeView.tsx index ca28180bf..efded68f5 100644 --- a/web/src/views/live/LiveBirdseyeView.tsx +++ b/web/src/views/live/LiveBirdseyeView.tsx @@ -1,11 +1,13 @@ +import { useBirdseyeLayout } from "@/api/ws"; import CameraFeatureToggle from "@/components/dynamic/CameraFeatureToggle"; import ActivityIndicator from "@/components/indicators/activity-indicator"; import BirdseyeLivePlayer from "@/components/player/BirdseyeLivePlayer"; import { Button } from "@/components/ui/button"; import { TooltipProvider } from "@/components/ui/tooltip"; import { useResizeObserver } from "@/hooks/resize-observer"; +import { cn } from "@/lib/utils"; import { FrigateConfig } from "@/types/frigateConfig"; -import { useEffect, useMemo, useRef, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { isDesktop, isFirefox, @@ -122,6 +124,72 @@ export default function LiveBirdseyeView({ return "mse"; }, [config]); + const birdseyeLayout = useBirdseyeLayout(); + + // Click overlay handling + + const playerRef = useRef(null); + const handleOverlayClick = useCallback( + ( + e: React.MouseEvent | React.TouchEvent, + ) => { + let clientX; + let clientY; + if ("TouchEvent" in window && e.nativeEvent instanceof TouchEvent) { + clientX = e.nativeEvent.touches[0].clientX; + clientY = e.nativeEvent.touches[0].clientY; + } else if (e.nativeEvent instanceof MouseEvent) { + clientX = e.nativeEvent.clientX; + clientY = e.nativeEvent.clientY; + } + + if ( + playerRef.current && + clientX && + clientY && + config && + birdseyeLayout?.payload + ) { + const playerRect = playerRef.current.getBoundingClientRect(); + + // Calculate coordinates relative to player div, accounting for offset + const rawX = clientX - playerRect.left; + const rawY = clientY - playerRect.top; + + // Ensure click is within player bounds + if ( + rawX < 0 || + rawX > playerRect.width || + rawY < 0 || + rawY > playerRect.height + ) { + return; + } + + // Scale click coordinates to birdseye canvas resolution + const canvasX = rawX * (config.birdseye.width / playerRect.width); + const canvasY = rawY * (config.birdseye.height / playerRect.height); + + for (const [cameraName, coords] of Object.entries( + birdseyeLayout.payload, + )) { + const parsedCoords = + typeof coords === "string" ? JSON.parse(coords) : coords; + if ( + canvasX >= parsedCoords.x && + canvasX < parsedCoords.x + parsedCoords.width && + canvasY >= parsedCoords.y && + canvasY < parsedCoords.y + parsedCoords.height + ) { + navigate(`/#${cameraName}`); + break; + } + } + } + }, + [playerRef, config, birdseyeLayout, navigate], + ); + if (!config) { return ; } @@ -215,16 +283,21 @@ export default function LiveBirdseyeView({ }} >