Mirrors/blakeblackshear.frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2025-10-27 10:52:11 +01:00
Code Issues Projects Releases Wiki Activity

update birdseye layout calculations

Browse Source
This commit is contained in:
Blake Blackshear 2021-06-09 07:41:30 -05:00
parent 4eed27e178
commit c70419bd0b
4 changed files with 285 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
docker-compose.yml
View File

@ -4,17 +4,17 @@ services:
container_name: frigate-dev
user: vscode
privileged: true
shm_size: "256mb"
build:
context: .
dockerfile: docker/Dockerfile.dev
devices:
- /dev/bus/usb:/dev/bus/usb
- /dev/dri:/dev/dri # for intel hwaccel, needs to be updated for your hardware
volumes:
- /etc/localtime:/etc/localtime:ro
- .:/lab/frigate:cached
- ./config/config.yml:/config/config.yml:ro
- ./debug:/media/frigate
- /dev/bus/usb:/dev/bus/usb
- /dev/dri:/dev/dri # for intel hwaccel, needs to be updated for your hardware
ports:
- "1935:1935"
- "5000:5000"

206
frigate/output.py
View File

@ -1,3 +1,4 @@
import cv2
import datetime
import math
import multiprocessing as mp
@ -18,7 +19,7 @@ from ws4py.server.wsgirefserver import (
from ws4py.server.wsgiutils import WebSocketWSGIApplication
from ws4py.websocket import WebSocket
from frigate.util import SharedMemoryFrameManager
from frigate.util import SharedMemoryFrameManager, get_yuv_crop, copy_yuv_to_position
class FFMpegConverter:
@ -39,7 +40,10 @@ class FFMpegConverter:
self.process.stdin.write(b)
def read(self, length):
try:
return self.process.stdout.read1(length)
except ValueError:
return False
def exit(self):
self.process.terminate()
@ -69,7 +73,9 @@ class BroadcastThread(threading.Thread):
class BirdsEyeFrameManager:
def __init__(self, height, width):
def __init__(self, config, frame_manager: SharedMemoryFrameManager, height, width):
self.config = config
self.frame_manager = frame_manager
self.frame_shape = (height, width)
self.yuv_shape = (height * 3 // 2, width)
self.frame = np.ndarray(self.yuv_shape, dtype=np.uint8)
@ -81,69 +87,169 @@ class BirdsEyeFrameManager:
self.frame[:] = self.blank_frame
self.last_active_frames = {}
self.cameras = {}
for camera, settings in self.config.cameras.items():
# precalculate the coordinates for all the channels
y, u1, u2, v1, v2 = get_yuv_crop(
settings.frame_shape_yuv,
(
0,
0,
settings.frame_shape[1],
settings.frame_shape[0],
),
)
self.cameras[camera] = {
"last_active_frame": 0.0,
"layout_frame": 0.0,
"channel_dims": {
"y": y,
"u1": u1,
"u2": u2,
"v1": v1,
"v2": v2,
},
}
self.camera_layout = []
self.active_cameras = set()
self.layout_dim = 0
self.last_output_time = 0.0
def clear_frame(self):
self.frame[:] = self.blank_frame
def update(self, camera, object_count, motion_count, frame_time, frame) -> bool:
def copy_to_position(self, position, camera=None, frame_time=None):
if camera is None:
frame = None
channel_dims = None
else:
frame = self.frame_manager.get(
f"{camera}{frame_time}", self.config.cameras[camera].frame_shape_yuv
)
channel_dims = self.cameras[camera]["channel_dims"]
# maintain time of most recent active frame for each camera
if object_count > 0:
self.last_active_frames[camera] = frame_time
# TODO: avoid the remaining work if exceeding 5 fps and return False
copy_yuv_to_position(position, self.frame, self.layout_dim, frame, channel_dims)
def update_frame(self):
# determine how many cameras are tracking objects within the last 30 seconds
now = datetime.datetime.now().timestamp()
active_cameras = [
active_cameras = set(
[
cam
for cam, frame_time in self.last_active_frames.items()
if now - frame_time < 30
for cam, cam_data in self.cameras.items()
if now - cam_data["last_active_frame"] < 30
]
)
if len(active_cameras) == 0 and len(self.camera_layout) == 0:
# if there are no active cameras
if len(active_cameras) == 0:
# if the layout is already cleared
if len(self.camera_layout) == 0:
return False
# if the sqrt of the layout and the active cameras don't round to the same value,
# we need to resize the layout
if round(math.sqrt(len(active_cameras))) != round(
math.sqrt(len(self.camera_layout))
):
# decide on a layout for the birdseye view (try to avoid too much churn)
self.columns = math.ceil(math.sqrt(len(active_cameras)))
self.rows = round(math.sqrt(len(active_cameras)))
self.camera_layout = [None] * (self.columns * self.rows)
# if the layout needs to be cleared
else:
self.camera_layout = []
self.clear_frame()
return True
# remove inactive cameras from the layout
self.camera_layout = [
cam if cam in active_cameras else None for cam in self.camera_layout
]
# place the active cameras in the layout
while len(active_cameras) > 0:
cam = active_cameras.pop()
if cam in self.camera_layout:
continue
# place camera in the first available spot in the layout
for i in range(0, len(self.camera_layout) - 1):
if self.camera_layout[i] is None:
self.camera_layout[i] = cam
break
# calculate layout dimensions
layout_dim = math.ceil(math.sqrt(len(active_cameras)))
# reset the layout if it needs to be different
if layout_dim != self.layout_dim:
self.layout_dim = layout_dim
self.camera_layout = [None] * layout_dim * layout_dim
# calculate resolution of each position in the layout
width = self.frame_shape[1] / self.columns
height = self.frame_shape[0] / self.rows
self.layout_frame_shape = (
self.frame_shape[0] // layout_dim, # height
self.frame_shape[1] // layout_dim, # width
)
# For each camera in the layout:
# - resize the current frame and copy into the birdseye view
self.clear_frame()
self.frame[:] = frame
for cam_data in self.cameras.values():
cam_data["layout_frame"] = 0.0
self.active_cameras = set()
removed_cameras = self.active_cameras.difference(active_cameras)
added_cameras = active_cameras.difference(self.active_cameras)
self.active_cameras = active_cameras
# update each position in the layout
for position, camera in enumerate(self.camera_layout, start=0):
# if this camera was removed, replace it or clear it
if camera in removed_cameras:
# if replacing this camera with a newly added one
if len(added_cameras) > 0:
added_camera = added_cameras.pop()
self.camera_layout[position] = added_camera
self.copy_to_position(
position,
added_camera,
self.cameras[added_camera]["last_active_frame"],
)
self.cameras[added_camera]["layout_frame"] = self.cameras[
added_camera
]["last_active_frame"]
# if removing this camera with no replacement
else:
self.camera_layout[position] = None
self.copy_to_position(position)
removed_cameras.remove(camera)
# if an empty spot and there are cameras to add
elif camera is None and len(added_cameras) > 0:
added_camera = added_cameras.pop()
self.camera_layout[position] = added_camera
self.copy_to_position(
position,
added_camera,
self.cameras[added_camera]["last_active_frame"],
)
self.cameras[added_camera]["layout_frame"] = self.cameras[added_camera][
"last_active_frame"
]
# if not an empty spot and the camera has a newer frame, copy it
elif (
not camera is None
and self.cameras[camera]["last_active_frame"]
!= self.cameras[camera]["layout_frame"]
):
self.copy_to_position(
position, camera, self.cameras[camera]["last_active_frame"]
)
self.cameras[camera]["layout_frame"] = self.cameras[camera][
"last_active_frame"
]
return True
def update(self, camera, object_count, motion_count, frame_time, frame) -> bool:
# update the last active frame for the camera
if object_count > 0:
last_active_frame = self.cameras[camera]["last_active_frame"]
# cleanup the old frame
if last_active_frame != 0.0:
frame_id = f"{camera}{last_active_frame}"
self.frame_manager.delete(frame_id)
self.cameras[camera]["last_active_frame"] = frame_time
now = datetime.datetime.now().timestamp()
# limit output to ~24 fps
if (now - self.last_output_time) < 0.04:
return False
self.last_output_time = now
return self.update_frame()
def output_frames(config, video_output_queue):
threading.current_thread().name = f"output"
@ -183,7 +289,7 @@ def output_frames(config, video_output_queue):
camera, converters[camera], websocket_server
)
converters["birdseye"] = FFMpegConverter(1920, 1080, 640, 320, "1000k")
converters["birdseye"] = FFMpegConverter(1920, 1080, 1280, 720, "2000k")
broadcasters["birdseye"] = BroadcastThread(
"birdseye", converters["birdseye"], websocket_server
)
@ -193,7 +299,7 @@ def output_frames(config, video_output_queue):
for t in broadcasters.values():
t.start()
birdseye_manager = BirdsEyeFrameManager(1080, 1920)
birdseye_manager = BirdsEyeFrameManager(config, frame_manager, 1080, 1920)
while not stop_event.is_set():
try:
@ -233,9 +339,14 @@ def output_frames(config, video_output_queue):
converters["birdseye"].write(birdseye_manager.frame.tobytes())
if camera in previous_frames:
frame_manager.delete(previous_frames[camera])
# if the birdseye manager still needs this frame, don't delete it
if (
birdseye_manager.cameras[camera]["last_active_frame"]
!= previous_frames[camera]
):
frame_manager.delete(f"{camera}{previous_frames[camera]}")
previous_frames[camera] = frame_id
previous_frames[camera] = frame_time
while not video_output_queue.empty():
(
@ -259,4 +370,5 @@ def output_frames(config, video_output_queue):
websocket_server.manager.join()
websocket_server.shutdown()
websocket_thread.join()
# TODO: use actual logger
print("exiting output process...")

27
frigate/test/test_copy_yuv_to_position.py Normal file
View File

@ -0,0 +1,27 @@
import cv2
import numpy as np
from unittest import TestCase, main
from frigate.util import copy_yuv_to_position
class TestCopyYuvToPosition(TestCase):
def setUp(self):
self.source_frame_bgr = np.zeros((400, 800, 3), np.uint8)
self.source_frame_bgr[:] = (0, 0, 255)
self.source_yuv_frame = cv2.cvtColor(
self.source_frame_bgr, cv2.COLOR_BGR2YUV_I420
)
self.dest_frame_bgr = np.zeros((400, 800, 3), np.uint8)
self.dest_frame_bgr[:] = (112, 202, 50)
self.dest_frame_bgr[100:300, 200:600] = (255, 0, 0)
self.dest_yuv_frame = cv2.cvtColor(self.dest_frame_bgr, cv2.COLOR_BGR2YUV_I420)
def test_copy_yuv_to_position(self):
copy_yuv_to_position(1, self.dest_yuv_frame, 3)
# cv2.imwrite(f"source_frame_yuv.jpg", self.source_yuv_frame)
# cv2.imwrite(f"dest_frame_yuv.jpg", self.dest_yuv_frame)
if __name__ == "__main__":
main(verbosity=2)

93
frigate/util.py
View File

@ -3,6 +3,7 @@ import datetime
import hashlib
import json
import logging
import math
import signal
import subprocess as sp
import threading
@ -233,6 +234,98 @@ def yuv_crop_and_resize(frame, region, height=None):
return yuv_cropped_frame
def copy_yuv_to_position(
position,
destination_frame,
destination_dim,
source_frame=None,
source_channel_dim=None,
):
# TODO: consider calculating this on layout reflow instead of all the time
layout_shape = (
(destination_frame.shape[0] // 3 * 2) // destination_dim,
destination_frame.shape[1] // destination_dim,
)
# calculate the x and y offset for the frame in the layout
y_offset = layout_shape[0] * math.floor(position / destination_dim)
x_offset = layout_shape[1] * (position % destination_dim)
# get the coordinates of the channels for this position in the layout
y, u1, u2, v1, v2 = get_yuv_crop(
destination_frame.shape,
(
x_offset,
y_offset,
x_offset + layout_shape[1],
y_offset + layout_shape[0],
),
)
if source_frame is None:
# clear y
destination_frame[
y[1] : y[3],
y[0] : y[2],
] = 16
# clear u1
destination_frame[u1[1] : u1[3], u1[0] : u1[2]] = 128
# clear u2
destination_frame[u2[1] : u2[3], u2[0] : u2[2]] = 128
# clear v1
destination_frame[v1[1] : v1[3], v1[0] : v1[2]] = 128
# clear v2
destination_frame[v2[1] : v2[3], v2[0] : v2[2]] = 128
else:
interpolation = cv2.INTER_AREA
# resize/copy y channel
destination_frame[y[1] : y[3], y[0] : y[2]] = cv2.resize(
source_frame[
source_channel_dim["y"][1] : source_channel_dim["y"][3],
source_channel_dim["y"][0] : source_channel_dim["y"][2],
],
dsize=(y[2] - y[0], y[3] - y[1]),
interpolation=interpolation,
)
# resize/copy u1
destination_frame[u1[1] : u1[3], u1[0] : u1[2]] = cv2.resize(
source_frame[
source_channel_dim["u1"][1] : source_channel_dim["u1"][3],
source_channel_dim["u1"][0] : source_channel_dim["u1"][2],
],
dsize=(u1[2] - u1[0], u1[3] - u1[1]),
interpolation=interpolation,
)
# resize/copy u2
destination_frame[u2[1] : u2[3], u2[0] : u2[2]] = cv2.resize(
source_frame[
source_channel_dim["u2"][1] : source_channel_dim["u2"][3],
source_channel_dim["u2"][0] : source_channel_dim["u2"][2],
],
dsize=(u2[2] - u2[0], u2[3] - u2[1]),
interpolation=interpolation,
)
# resize/copy v1
destination_frame[v1[1] : v1[3], v1[0] : v1[2]] = cv2.resize(
source_frame[
source_channel_dim["v1"][1] : source_channel_dim["v1"][3],
source_channel_dim["v1"][0] : source_channel_dim["v1"][2],
],
dsize=(v1[2] - v1[0], v1[3] - v1[1]),
interpolation=interpolation,
)
# resize/copy v2
destination_frame[v2[1] : v2[3], v2[0] : v2[2]] = cv2.resize(
source_frame[
source_channel_dim["v2"][1] : source_channel_dim["v2"][3],
source_channel_dim["v2"][0] : source_channel_dim["v2"][2],
],
dsize=(v2[2] - v2[0], v2[3] - v2[1]),
interpolation=interpolation,
)
def yuv_region_2_rgb(frame, region):
try:
# TODO: does this copy the numpy array?