Make Birdseye clickable (#18628)

* keep track of layout changes and publish on change

* websocket hook

* clickable overlay div to navigate to full camera view
This commit is contained in:
Josh Hawkins 2025-06-08 13:06:17 -05:00 committed by GitHub
parent eb83f2ac47
commit 9a5162752c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 175 additions and 22 deletions

View File

@ -21,6 +21,7 @@ from frigate.const import (
INSERT_PREVIEW,
NOTIFICATION_TEST,
REQUEST_REGION_GRID,
UPDATE_BIRDSEYE_LAYOUT,
UPDATE_CAMERA_ACTIVITY,
UPDATE_EMBEDDINGS_REINDEX_PROGRESS,
UPDATE_EVENT_DESCRIPTION,
@ -55,6 +56,7 @@ class Dispatcher:
self.camera_activity = CameraActivityManager(config, self.publish)
self.model_state = {}
self.embeddings_reindex = {}
self.birdseye_layout = {}
self._camera_settings_handlers: dict[str, Callable] = {
"audio": self._on_audio_command,
@ -168,6 +170,14 @@ class Dispatcher:
json.dumps(self.embeddings_reindex.copy()),
)
def handle_update_birdseye_layout():
if payload:
self.birdseye_layout = payload
self.publish("birdseye_layout", json.dumps(self.birdseye_layout))
def handle_birdseye_layout():
self.publish("birdseye_layout", json.dumps(self.birdseye_layout.copy()))
def handle_on_connect():
camera_status = self.camera_activity.last_camera_activity.copy()
cameras_with_status = camera_status.keys()
@ -205,6 +215,7 @@ class Dispatcher:
"embeddings_reindex_progress",
json.dumps(self.embeddings_reindex.copy()),
)
self.publish("birdseye_layout", json.dumps(self.birdseye_layout.copy()))
def handle_notification_test():
self.publish("notification_test", "Test notification")
@ -220,10 +231,12 @@ class Dispatcher:
UPDATE_EVENT_DESCRIPTION: handle_update_event_description,
UPDATE_MODEL_STATE: handle_update_model_state,
UPDATE_EMBEDDINGS_REINDEX_PROGRESS: handle_update_embeddings_reindex_progress,
UPDATE_BIRDSEYE_LAYOUT: handle_update_birdseye_layout,
NOTIFICATION_TEST: handle_notification_test,
"restart": handle_restart,
"embeddingsReindexProgress": handle_embeddings_reindex_progress,
"modelState": handle_model_state,
"birdseyeLayout": handle_birdseye_layout,
"onConnect": handle_on_connect,
}

View File

@ -109,6 +109,7 @@ UPDATE_CAMERA_ACTIVITY = "update_camera_activity"
UPDATE_EVENT_DESCRIPTION = "update_event_description"
UPDATE_MODEL_STATE = "update_model_state"
UPDATE_EMBEDDINGS_REINDEX_PROGRESS = "handle_embeddings_reindex_progress"
UPDATE_BIRDSEYE_LAYOUT = "update_birdseye_layout"
NOTIFICATION_TEST = "notification_test"
# Stats Values

View File

@ -15,8 +15,9 @@ from typing import Any, Optional
import cv2
import numpy as np
from frigate.comms.inter_process import InterProcessRequestor
from frigate.config import BirdseyeModeEnum, FfmpegConfig, FrigateConfig
from frigate.const import BASE_DIR, BIRDSEYE_PIPE, INSTALL_DIR
from frigate.const import BASE_DIR, BIRDSEYE_PIPE, INSTALL_DIR, UPDATE_BIRDSEYE_LAYOUT
from frigate.util.image import (
SharedMemoryFrameManager,
copy_yuv_to_position,
@ -380,10 +381,24 @@ class BirdsEyeFrameManager:
if mode == BirdseyeModeEnum.objects and object_box_count > 0:
return True
def update_frame(self, frame: Optional[np.ndarray] = None) -> bool:
def get_camera_coordinates(self) -> dict[str, dict[str, int]]:
"""Return the coordinates of each camera in the current layout."""
coordinates = {}
for row in self.camera_layout:
for position in row:
camera_name, (x, y, width, height) = position
coordinates[camera_name] = {
"x": x,
"y": y,
"width": width,
"height": height,
}
return coordinates
def update_frame(self, frame: Optional[np.ndarray] = None) -> tuple[bool, bool]:
"""
Update birdseye, optionally with a new frame.
When no frame is passed, check the layout and update for any disabled cameras.
Returns (frame_changed, layout_changed) to indicate if the frame or layout changed.
"""
# determine how many cameras are tracking objects within the last inactivity_threshold seconds
@ -421,19 +436,21 @@ class BirdsEyeFrameManager:
max_camera_refresh = True
self.last_refresh_time = now
# Track if the frame changes
# Track if the frame or layout changes
frame_changed = False
layout_changed = False
# If no active cameras and layout is already empty, no update needed
if len(active_cameras) == 0:
# if the layout is already cleared
if len(self.camera_layout) == 0:
return False
return False, False
# if the layout needs to be cleared
self.camera_layout = []
self.active_cameras = set()
self.clear_frame()
frame_changed = True
layout_changed = True
else:
# Determine if layout needs resetting
if len(self.active_cameras) - len(active_cameras) == 0:
@ -453,7 +470,7 @@ class BirdsEyeFrameManager:
logger.debug("Resetting Birdseye layout...")
self.clear_frame()
self.active_cameras = active_cameras
layout_changed = True # Layout is changing due to reset
# this also converts added_cameras from a set to a list since we need
# to pop elements in order
active_cameras_to_add = sorted(
@ -503,7 +520,7 @@ class BirdsEyeFrameManager:
# decrease scaling coefficient until height of all cameras can fit into the birdseye canvas
while calculating:
if self.stop_event.is_set():
return
return frame_changed, layout_changed
layout_candidate = self.calculate_layout(
active_cameras_to_add, coefficient
@ -517,7 +534,7 @@ class BirdsEyeFrameManager:
logger.error(
"Error finding appropriate birdseye layout"
)
return
return frame_changed, layout_changed
calculating = False
self.canvas.set_coefficient(len(active_cameras), coefficient)
@ -535,7 +552,7 @@ class BirdsEyeFrameManager:
if frame is not None: # Frame presence indicates a potential change
frame_changed = True
return frame_changed
return frame_changed, layout_changed
def calculate_layout(
self,
@ -687,7 +704,11 @@ class BirdsEyeFrameManager:
motion_count: int,
frame_time: float,
frame: np.ndarray,
) -> bool:
) -> tuple[bool, bool]:
"""
Update birdseye for a specific camera with new frame data.
Returns (frame_changed, layout_changed) to indicate if the frame or layout changed.
"""
# don't process if birdseye is disabled for this camera
camera_config = self.config.cameras[camera]
force_update = False
@ -700,7 +721,7 @@ class BirdsEyeFrameManager:
self.cameras[camera]["last_active_frame"] = 0
force_update = True
else:
return False
return False, False
# update the last active frame for the camera
self.cameras[camera]["current_frame"] = frame.copy()
@ -712,21 +733,22 @@ class BirdsEyeFrameManager:
# limit output to 10 fps
if not force_update and (now - self.last_output_time) < 1 / 10:
return False
return False, False
try:
updated_frame = self.update_frame(frame)
frame_changed, layout_changed = self.update_frame(frame)
except Exception:
updated_frame = False
frame_changed, layout_changed = False, False
self.active_cameras = []
self.camera_layout = []
print(traceback.format_exc())
# if the frame was updated or the fps is too low, send frame
if force_update or updated_frame or (now - self.last_output_time) > 1:
if force_update or frame_changed or (now - self.last_output_time) > 1:
self.last_output_time = now
return True
return False
return True, layout_changed
return False, layout_changed
class Birdseye:
@ -755,6 +777,7 @@ class Birdseye:
self.birdseye_manager = BirdsEyeFrameManager(config, stop_event)
self.frame_manager = SharedMemoryFrameManager()
self.stop_event = stop_event
self.requestor = InterProcessRequestor()
if config.birdseye.restream:
self.birdseye_buffer = self.frame_manager.create(
@ -789,15 +812,20 @@ class Birdseye:
frame_time: float,
frame: np.ndarray,
) -> None:
if self.birdseye_manager.update(
frame_changed, frame_layout_changed = self.birdseye_manager.update(
camera,
len([o for o in current_tracked_objects if not o["stationary"]]),
len(motion_boxes),
frame_time,
frame,
):
)
if frame_changed:
self.__send_new_frame()
if frame_layout_changed:
coordinates = self.birdseye_manager.get_camera_coordinates()
self.requestor.send_data(UPDATE_BIRDSEYE_LAYOUT, coordinates)
def stop(self) -> None:
self.converter.join()
self.broadcaster.join()

View File

@ -426,6 +426,40 @@ export function useEmbeddingsReindexProgress(
return { payload: data };
}
export function useBirdseyeLayout(revalidateOnFocus: boolean = true): {
payload: string;
} {
const {
value: { payload },
send: sendCommand,
} = useWs("birdseye_layout", "birdseyeLayout");
const data = useDeepMemo(JSON.parse(payload as string));
useEffect(() => {
let listener = undefined;
if (revalidateOnFocus) {
sendCommand("birdseyeLayout");
listener = () => {
if (document.visibilityState == "visible") {
sendCommand("birdseyeLayout");
}
};
addEventListener("visibilitychange", listener);
}
return () => {
if (listener) {
removeEventListener("visibilitychange", listener);
}
};
// we know that these deps are correct
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [revalidateOnFocus]);
return { payload: data };
}
export function useMotionActivity(camera: string): { payload: string } {
const {
value: { payload },

View File

@ -13,6 +13,7 @@ type LivePlayerProps = {
liveMode: LivePlayerMode;
pip?: boolean;
containerRef: React.MutableRefObject<HTMLDivElement | null>;
playerRef?: React.MutableRefObject<HTMLDivElement | null>;
onClick?: () => void;
};
@ -22,6 +23,7 @@ export default function BirdseyeLivePlayer({
liveMode,
pip,
containerRef,
playerRef,
onClick,
}: LivePlayerProps) {
let player;
@ -76,7 +78,9 @@ export default function BirdseyeLivePlayer({
>
<div className="pointer-events-none absolute inset-x-0 top-0 z-10 h-[30%] w-full rounded-lg bg-gradient-to-b from-black/20 to-transparent md:rounded-2xl"></div>
<div className="pointer-events-none absolute inset-x-0 bottom-0 z-10 h-[10%] w-full rounded-lg bg-gradient-to-t from-black/20 to-transparent md:rounded-2xl"></div>
<div className="size-full">{player}</div>
<div className="size-full" ref={playerRef}>
{player}
</div>
</div>
);
}

View File

@ -1,11 +1,13 @@
import { useBirdseyeLayout } from "@/api/ws";
import CameraFeatureToggle from "@/components/dynamic/CameraFeatureToggle";
import ActivityIndicator from "@/components/indicators/activity-indicator";
import BirdseyeLivePlayer from "@/components/player/BirdseyeLivePlayer";
import { Button } from "@/components/ui/button";
import { TooltipProvider } from "@/components/ui/tooltip";
import { useResizeObserver } from "@/hooks/resize-observer";
import { cn } from "@/lib/utils";
import { FrigateConfig } from "@/types/frigateConfig";
import { useEffect, useMemo, useRef, useState } from "react";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import {
isDesktop,
isFirefox,
@ -122,6 +124,72 @@ export default function LiveBirdseyeView({
return "mse";
}, [config]);
const birdseyeLayout = useBirdseyeLayout();
// Click overlay handling
const playerRef = useRef<HTMLDivElement | null>(null);
const handleOverlayClick = useCallback(
(
e: React.MouseEvent<HTMLDivElement> | React.TouchEvent<HTMLDivElement>,
) => {
let clientX;
let clientY;
if ("TouchEvent" in window && e.nativeEvent instanceof TouchEvent) {
clientX = e.nativeEvent.touches[0].clientX;
clientY = e.nativeEvent.touches[0].clientY;
} else if (e.nativeEvent instanceof MouseEvent) {
clientX = e.nativeEvent.clientX;
clientY = e.nativeEvent.clientY;
}
if (
playerRef.current &&
clientX &&
clientY &&
config &&
birdseyeLayout?.payload
) {
const playerRect = playerRef.current.getBoundingClientRect();
// Calculate coordinates relative to player div, accounting for offset
const rawX = clientX - playerRect.left;
const rawY = clientY - playerRect.top;
// Ensure click is within player bounds
if (
rawX < 0 ||
rawX > playerRect.width ||
rawY < 0 ||
rawY > playerRect.height
) {
return;
}
// Scale click coordinates to birdseye canvas resolution
const canvasX = rawX * (config.birdseye.width / playerRect.width);
const canvasY = rawY * (config.birdseye.height / playerRect.height);
for (const [cameraName, coords] of Object.entries(
birdseyeLayout.payload,
)) {
const parsedCoords =
typeof coords === "string" ? JSON.parse(coords) : coords;
if (
canvasX >= parsedCoords.x &&
canvasX < parsedCoords.x + parsedCoords.width &&
canvasY >= parsedCoords.y &&
canvasY < parsedCoords.y + parsedCoords.height
) {
navigate(`/#${cameraName}`);
break;
}
}
}
},
[playerRef, config, birdseyeLayout, navigate],
);
if (!config) {
return <ActivityIndicator />;
}
@ -215,16 +283,21 @@ export default function LiveBirdseyeView({
}}
>
<div
className={growClassName}
className={cn(
"flex flex-col items-center justify-center",
growClassName,
)}
style={{
aspectRatio: constrainedAspectRatio,
}}
onClick={handleOverlayClick}
>
<BirdseyeLivePlayer
className={`${fullscreen ? "*:rounded-none" : ""}`}
birdseyeConfig={config.birdseye}
liveMode={preferredLiveMode}
containerRef={containerRef}
playerRef={playerRef}
pip={pip}
/>
</div>