diff --git a/docker-compose.yml b/docker-compose.yml index 9a7eb578a..197d9e11e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,17 +4,17 @@ services: container_name: frigate-dev user: vscode privileged: true + shm_size: "256mb" build: context: . dockerfile: docker/Dockerfile.dev - devices: - - /dev/bus/usb:/dev/bus/usb - - /dev/dri:/dev/dri # for intel hwaccel, needs to be updated for your hardware volumes: - /etc/localtime:/etc/localtime:ro - .:/lab/frigate:cached - ./config/config.yml:/config/config.yml:ro - ./debug:/media/frigate + - /dev/bus/usb:/dev/bus/usb + - /dev/dri:/dev/dri # for intel hwaccel, needs to be updated for your hardware ports: - "1935:1935" - "5000:5000" diff --git a/frigate/output.py b/frigate/output.py index 6f7b4eb70..d06044c4a 100644 --- a/frigate/output.py +++ b/frigate/output.py @@ -1,3 +1,4 @@ +import cv2 import datetime import math import multiprocessing as mp @@ -18,7 +19,7 @@ from ws4py.server.wsgirefserver import ( from ws4py.server.wsgiutils import WebSocketWSGIApplication from ws4py.websocket import WebSocket -from frigate.util import SharedMemoryFrameManager +from frigate.util import SharedMemoryFrameManager, get_yuv_crop, copy_yuv_to_position class FFMpegConverter: @@ -39,7 +40,10 @@ class FFMpegConverter: self.process.stdin.write(b) def read(self, length): - return self.process.stdout.read1(length) + try: + return self.process.stdout.read1(length) + except ValueError: + return False def exit(self): self.process.terminate() @@ -69,7 +73,9 @@ class BroadcastThread(threading.Thread): class BirdsEyeFrameManager: - def __init__(self, height, width): + def __init__(self, config, frame_manager: SharedMemoryFrameManager, height, width): + self.config = config + self.frame_manager = frame_manager self.frame_shape = (height, width) self.yuv_shape = (height * 3 // 2, width) self.frame = np.ndarray(self.yuv_shape, dtype=np.uint8) @@ -81,69 +87,169 @@ class BirdsEyeFrameManager: self.frame[:] = self.blank_frame - self.last_active_frames = {} + self.cameras = {} + for camera, settings in self.config.cameras.items(): + # precalculate the coordinates for all the channels + y, u1, u2, v1, v2 = get_yuv_crop( + settings.frame_shape_yuv, + ( + 0, + 0, + settings.frame_shape[1], + settings.frame_shape[0], + ), + ) + self.cameras[camera] = { + "last_active_frame": 0.0, + "layout_frame": 0.0, + "channel_dims": { + "y": y, + "u1": u1, + "u2": u2, + "v1": v1, + "v2": v2, + }, + } + self.camera_layout = [] + self.active_cameras = set() + self.layout_dim = 0 + self.last_output_time = 0.0 def clear_frame(self): self.frame[:] = self.blank_frame - def update(self, camera, object_count, motion_count, frame_time, frame) -> bool: + def copy_to_position(self, position, camera=None, frame_time=None): + if camera is None: + frame = None + channel_dims = None + else: + frame = self.frame_manager.get( + f"{camera}{frame_time}", self.config.cameras[camera].frame_shape_yuv + ) + channel_dims = self.cameras[camera]["channel_dims"] - # maintain time of most recent active frame for each camera - if object_count > 0: - self.last_active_frames[camera] = frame_time - - # TODO: avoid the remaining work if exceeding 5 fps and return False + copy_yuv_to_position(position, self.frame, self.layout_dim, frame, channel_dims) + def update_frame(self): # determine how many cameras are tracking objects within the last 30 seconds now = datetime.datetime.now().timestamp() - active_cameras = [ - cam - for cam, frame_time in self.last_active_frames.items() - if now - frame_time < 30 - ] + active_cameras = set( + [ + cam + for cam, cam_data in self.cameras.items() + if now - cam_data["last_active_frame"] < 30 + ] + ) - if len(active_cameras) == 0 and len(self.camera_layout) == 0: - return False + # if there are no active cameras + if len(active_cameras) == 0: + # if the layout is already cleared + if len(self.camera_layout) == 0: + return False + # if the layout needs to be cleared + else: + self.camera_layout = [] + self.clear_frame() + return True - # if the sqrt of the layout and the active cameras don't round to the same value, - # we need to resize the layout - if round(math.sqrt(len(active_cameras))) != round( - math.sqrt(len(self.camera_layout)) - ): - # decide on a layout for the birdseye view (try to avoid too much churn) - self.columns = math.ceil(math.sqrt(len(active_cameras))) - self.rows = round(math.sqrt(len(active_cameras))) + # calculate layout dimensions + layout_dim = math.ceil(math.sqrt(len(active_cameras))) + + # reset the layout if it needs to be different + if layout_dim != self.layout_dim: + self.layout_dim = layout_dim + + self.camera_layout = [None] * layout_dim * layout_dim + + # calculate resolution of each position in the layout + self.layout_frame_shape = ( + self.frame_shape[0] // layout_dim, # height + self.frame_shape[1] // layout_dim, # width + ) - self.camera_layout = [None] * (self.columns * self.rows) self.clear_frame() - # remove inactive cameras from the layout - self.camera_layout = [ - cam if cam in active_cameras else None for cam in self.camera_layout - ] - # place the active cameras in the layout - while len(active_cameras) > 0: - cam = active_cameras.pop() - if cam in self.camera_layout: - continue - # place camera in the first available spot in the layout - for i in range(0, len(self.camera_layout) - 1): - if self.camera_layout[i] is None: - self.camera_layout[i] = cam - break + for cam_data in self.cameras.values(): + cam_data["layout_frame"] = 0.0 - # calculate resolution of each position in the layout - width = self.frame_shape[1] / self.columns - height = self.frame_shape[0] / self.rows + self.active_cameras = set() - # For each camera in the layout: - # - resize the current frame and copy into the birdseye view + removed_cameras = self.active_cameras.difference(active_cameras) + added_cameras = active_cameras.difference(self.active_cameras) - self.frame[:] = frame + self.active_cameras = active_cameras + + # update each position in the layout + for position, camera in enumerate(self.camera_layout, start=0): + + # if this camera was removed, replace it or clear it + if camera in removed_cameras: + # if replacing this camera with a newly added one + if len(added_cameras) > 0: + added_camera = added_cameras.pop() + self.camera_layout[position] = added_camera + self.copy_to_position( + position, + added_camera, + self.cameras[added_camera]["last_active_frame"], + ) + self.cameras[added_camera]["layout_frame"] = self.cameras[ + added_camera + ]["last_active_frame"] + # if removing this camera with no replacement + else: + self.camera_layout[position] = None + self.copy_to_position(position) + removed_cameras.remove(camera) + # if an empty spot and there are cameras to add + elif camera is None and len(added_cameras) > 0: + added_camera = added_cameras.pop() + self.camera_layout[position] = added_camera + self.copy_to_position( + position, + added_camera, + self.cameras[added_camera]["last_active_frame"], + ) + self.cameras[added_camera]["layout_frame"] = self.cameras[added_camera][ + "last_active_frame" + ] + # if not an empty spot and the camera has a newer frame, copy it + elif ( + not camera is None + and self.cameras[camera]["last_active_frame"] + != self.cameras[camera]["layout_frame"] + ): + self.copy_to_position( + position, camera, self.cameras[camera]["last_active_frame"] + ) + self.cameras[camera]["layout_frame"] = self.cameras[camera][ + "last_active_frame" + ] return True + def update(self, camera, object_count, motion_count, frame_time, frame) -> bool: + + # update the last active frame for the camera + if object_count > 0: + last_active_frame = self.cameras[camera]["last_active_frame"] + # cleanup the old frame + if last_active_frame != 0.0: + frame_id = f"{camera}{last_active_frame}" + self.frame_manager.delete(frame_id) + self.cameras[camera]["last_active_frame"] = frame_time + + now = datetime.datetime.now().timestamp() + + # limit output to ~24 fps + if (now - self.last_output_time) < 0.04: + return False + + self.last_output_time = now + + return self.update_frame() + def output_frames(config, video_output_queue): threading.current_thread().name = f"output" @@ -183,7 +289,7 @@ def output_frames(config, video_output_queue): camera, converters[camera], websocket_server ) - converters["birdseye"] = FFMpegConverter(1920, 1080, 640, 320, "1000k") + converters["birdseye"] = FFMpegConverter(1920, 1080, 1280, 720, "2000k") broadcasters["birdseye"] = BroadcastThread( "birdseye", converters["birdseye"], websocket_server ) @@ -193,7 +299,7 @@ def output_frames(config, video_output_queue): for t in broadcasters.values(): t.start() - birdseye_manager = BirdsEyeFrameManager(1080, 1920) + birdseye_manager = BirdsEyeFrameManager(config, frame_manager, 1080, 1920) while not stop_event.is_set(): try: @@ -233,9 +339,14 @@ def output_frames(config, video_output_queue): converters["birdseye"].write(birdseye_manager.frame.tobytes()) if camera in previous_frames: - frame_manager.delete(previous_frames[camera]) + # if the birdseye manager still needs this frame, don't delete it + if ( + birdseye_manager.cameras[camera]["last_active_frame"] + != previous_frames[camera] + ): + frame_manager.delete(f"{camera}{previous_frames[camera]}") - previous_frames[camera] = frame_id + previous_frames[camera] = frame_time while not video_output_queue.empty(): ( @@ -259,4 +370,5 @@ def output_frames(config, video_output_queue): websocket_server.manager.join() websocket_server.shutdown() websocket_thread.join() + # TODO: use actual logger print("exiting output process...") diff --git a/frigate/test/test_copy_yuv_to_position.py b/frigate/test/test_copy_yuv_to_position.py new file mode 100644 index 000000000..6517853c4 --- /dev/null +++ b/frigate/test/test_copy_yuv_to_position.py @@ -0,0 +1,27 @@ +import cv2 +import numpy as np +from unittest import TestCase, main +from frigate.util import copy_yuv_to_position + + +class TestCopyYuvToPosition(TestCase): + def setUp(self): + self.source_frame_bgr = np.zeros((400, 800, 3), np.uint8) + self.source_frame_bgr[:] = (0, 0, 255) + self.source_yuv_frame = cv2.cvtColor( + self.source_frame_bgr, cv2.COLOR_BGR2YUV_I420 + ) + + self.dest_frame_bgr = np.zeros((400, 800, 3), np.uint8) + self.dest_frame_bgr[:] = (112, 202, 50) + self.dest_frame_bgr[100:300, 200:600] = (255, 0, 0) + self.dest_yuv_frame = cv2.cvtColor(self.dest_frame_bgr, cv2.COLOR_BGR2YUV_I420) + + def test_copy_yuv_to_position(self): + copy_yuv_to_position(1, self.dest_yuv_frame, 3) + # cv2.imwrite(f"source_frame_yuv.jpg", self.source_yuv_frame) + # cv2.imwrite(f"dest_frame_yuv.jpg", self.dest_yuv_frame) + + +if __name__ == "__main__": + main(verbosity=2) diff --git a/frigate/util.py b/frigate/util.py index 3d8650e97..8f2ecc2f5 100755 --- a/frigate/util.py +++ b/frigate/util.py @@ -3,6 +3,7 @@ import datetime import hashlib import json import logging +import math import signal import subprocess as sp import threading @@ -233,6 +234,98 @@ def yuv_crop_and_resize(frame, region, height=None): return yuv_cropped_frame +def copy_yuv_to_position( + position, + destination_frame, + destination_dim, + source_frame=None, + source_channel_dim=None, +): + # TODO: consider calculating this on layout reflow instead of all the time + layout_shape = ( + (destination_frame.shape[0] // 3 * 2) // destination_dim, + destination_frame.shape[1] // destination_dim, + ) + # calculate the x and y offset for the frame in the layout + y_offset = layout_shape[0] * math.floor(position / destination_dim) + x_offset = layout_shape[1] * (position % destination_dim) + + # get the coordinates of the channels for this position in the layout + y, u1, u2, v1, v2 = get_yuv_crop( + destination_frame.shape, + ( + x_offset, + y_offset, + x_offset + layout_shape[1], + y_offset + layout_shape[0], + ), + ) + + if source_frame is None: + # clear y + destination_frame[ + y[1] : y[3], + y[0] : y[2], + ] = 16 + + # clear u1 + destination_frame[u1[1] : u1[3], u1[0] : u1[2]] = 128 + # clear u2 + destination_frame[u2[1] : u2[3], u2[0] : u2[2]] = 128 + # clear v1 + destination_frame[v1[1] : v1[3], v1[0] : v1[2]] = 128 + # clear v2 + destination_frame[v2[1] : v2[3], v2[0] : v2[2]] = 128 + else: + interpolation = cv2.INTER_AREA + # resize/copy y channel + destination_frame[y[1] : y[3], y[0] : y[2]] = cv2.resize( + source_frame[ + source_channel_dim["y"][1] : source_channel_dim["y"][3], + source_channel_dim["y"][0] : source_channel_dim["y"][2], + ], + dsize=(y[2] - y[0], y[3] - y[1]), + interpolation=interpolation, + ) + + # resize/copy u1 + destination_frame[u1[1] : u1[3], u1[0] : u1[2]] = cv2.resize( + source_frame[ + source_channel_dim["u1"][1] : source_channel_dim["u1"][3], + source_channel_dim["u1"][0] : source_channel_dim["u1"][2], + ], + dsize=(u1[2] - u1[0], u1[3] - u1[1]), + interpolation=interpolation, + ) + # resize/copy u2 + destination_frame[u2[1] : u2[3], u2[0] : u2[2]] = cv2.resize( + source_frame[ + source_channel_dim["u2"][1] : source_channel_dim["u2"][3], + source_channel_dim["u2"][0] : source_channel_dim["u2"][2], + ], + dsize=(u2[2] - u2[0], u2[3] - u2[1]), + interpolation=interpolation, + ) + # resize/copy v1 + destination_frame[v1[1] : v1[3], v1[0] : v1[2]] = cv2.resize( + source_frame[ + source_channel_dim["v1"][1] : source_channel_dim["v1"][3], + source_channel_dim["v1"][0] : source_channel_dim["v1"][2], + ], + dsize=(v1[2] - v1[0], v1[3] - v1[1]), + interpolation=interpolation, + ) + # resize/copy v2 + destination_frame[v2[1] : v2[3], v2[0] : v2[2]] = cv2.resize( + source_frame[ + source_channel_dim["v2"][1] : source_channel_dim["v2"][3], + source_channel_dim["v2"][0] : source_channel_dim["v2"][2], + ], + dsize=(v2[2] - v2[0], v2[3] - v2[1]), + interpolation=interpolation, + ) + + def yuv_region_2_rgb(frame, region): try: # TODO: does this copy the numpy array?