update birdseye layout calculations

docker-compose.yml

@@ -4,17 +4,17 @@ services:
     container_name: frigate-dev
     user: vscode
     privileged: true
+    shm_size: "256mb"
     build:
       context: .
       dockerfile: docker/Dockerfile.dev
-    devices:
-      - /dev/bus/usb:/dev/bus/usb
-      - /dev/dri:/dev/dri # for intel hwaccel, needs to be updated for your hardware
     volumes:
       - /etc/localtime:/etc/localtime:ro
       - .:/lab/frigate:cached
       - ./config/config.yml:/config/config.yml:ro
       - ./debug:/media/frigate
+      - /dev/bus/usb:/dev/bus/usb
+      - /dev/dri:/dev/dri # for intel hwaccel, needs to be updated for your hardware
     ports:
       - "1935:1935"
       - "5000:5000"

frigate/output.py

@@ -1,3 +1,4 @@
+import cv2
 import datetime
 import math
 import multiprocessing as mp
@@ -18,7 +19,7 @@ from ws4py.server.wsgirefserver import (
 from ws4py.server.wsgiutils import WebSocketWSGIApplication
 from ws4py.websocket import WebSocket
 
-from frigate.util import SharedMemoryFrameManager
+from frigate.util import SharedMemoryFrameManager, get_yuv_crop, copy_yuv_to_position
 
 
 class FFMpegConverter:
@@ -39,7 +40,10 @@ class FFMpegConverter:
         self.process.stdin.write(b)
 
     def read(self, length):
-        return self.process.stdout.read1(length)
+        try:
+            return self.process.stdout.read1(length)
+        except ValueError:
+            return False
 
     def exit(self):
         self.process.terminate()
@@ -69,7 +73,9 @@ class BroadcastThread(threading.Thread):
 
 
 class BirdsEyeFrameManager:
-    def __init__(self, height, width):
+    def __init__(self, config, frame_manager: SharedMemoryFrameManager, height, width):
+        self.config = config
+        self.frame_manager = frame_manager
         self.frame_shape = (height, width)
         self.yuv_shape = (height * 3 // 2, width)
         self.frame = np.ndarray(self.yuv_shape, dtype=np.uint8)
@@ -81,69 +87,169 @@ class BirdsEyeFrameManager:
 
         self.frame[:] = self.blank_frame
 
-        self.last_active_frames = {}
+        self.cameras = {}
+        for camera, settings in self.config.cameras.items():
+            # precalculate the coordinates for all the channels
+            y, u1, u2, v1, v2 = get_yuv_crop(
+                settings.frame_shape_yuv,
+                (
+                    0,
+                    0,
+                    settings.frame_shape[1],
+                    settings.frame_shape[0],
+                ),
+            )
+            self.cameras[camera] = {
+                "last_active_frame": 0.0,
+                "layout_frame": 0.0,
+                "channel_dims": {
+                    "y": y,
+                    "u1": u1,
+                    "u2": u2,
+                    "v1": v1,
+                    "v2": v2,
+                },
+            }
+
         self.camera_layout = []
+        self.active_cameras = set()
+        self.layout_dim = 0
+        self.last_output_time = 0.0
 
     def clear_frame(self):
         self.frame[:] = self.blank_frame
 
-    def update(self, camera, object_count, motion_count, frame_time, frame) -> bool:
+    def copy_to_position(self, position, camera=None, frame_time=None):
+        if camera is None:
+            frame = None
+            channel_dims = None
+        else:
+            frame = self.frame_manager.get(
+                f"{camera}{frame_time}", self.config.cameras[camera].frame_shape_yuv
+            )
+            channel_dims = self.cameras[camera]["channel_dims"]
 
-        # maintain time of most recent active frame for each camera
+        copy_yuv_to_position(position, self.frame, self.layout_dim, frame, channel_dims)
-        if object_count > 0:
-            self.last_active_frames[camera] = frame_time
-
-        # TODO: avoid the remaining work if exceeding 5 fps and return False
 
+    def update_frame(self):
         # determine how many cameras are tracking objects within the last 30 seconds
         now = datetime.datetime.now().timestamp()
-        active_cameras = [
+        active_cameras = set(
-            cam
+            [
-            for cam, frame_time in self.last_active_frames.items()
+                cam
-            if now - frame_time < 30
+                for cam, cam_data in self.cameras.items()
-        ]
+                if now - cam_data["last_active_frame"] < 30
+            ]
+        )
 
-        if len(active_cameras) == 0 and len(self.camera_layout) == 0:
+        # if there are no active cameras
-            return False
+        if len(active_cameras) == 0:
+            # if the layout is already cleared
+            if len(self.camera_layout) == 0:
+                return False
+            # if the layout needs to be cleared
+            else:
+                self.camera_layout = []
+                self.clear_frame()
+                return True
 
-        # if the sqrt of the layout and the active cameras don't round to the same value,
+        # calculate layout dimensions
-        # we need to resize the layout
+        layout_dim = math.ceil(math.sqrt(len(active_cameras)))
-        if round(math.sqrt(len(active_cameras))) != round(
+
-            math.sqrt(len(self.camera_layout))
+        # reset the layout if it needs to be different
-        ):
+        if layout_dim != self.layout_dim:
-            # decide on a layout for the birdseye view (try to avoid too much churn)
+            self.layout_dim = layout_dim
-            self.columns = math.ceil(math.sqrt(len(active_cameras)))
+
-            self.rows = round(math.sqrt(len(active_cameras)))
+            self.camera_layout = [None] * layout_dim * layout_dim
+
+            # calculate resolution of each position in the layout
+            self.layout_frame_shape = (
+                self.frame_shape[0] // layout_dim,  # height
+                self.frame_shape[1] // layout_dim,  # width
+            )
 
-            self.camera_layout = [None] * (self.columns * self.rows)
             self.clear_frame()
 
-        # remove inactive cameras from the layout
+            for cam_data in self.cameras.values():
-        self.camera_layout = [
+                cam_data["layout_frame"] = 0.0
-            cam if cam in active_cameras else None for cam in self.camera_layout
-        ]
-        # place the active cameras in the layout
-        while len(active_cameras) > 0:
-            cam = active_cameras.pop()
-            if cam in self.camera_layout:
-                continue
-            # place camera in the first available spot in the layout
-            for i in range(0, len(self.camera_layout) - 1):
-                if self.camera_layout[i] is None:
-                    self.camera_layout[i] = cam
-                    break
 
-        # calculate resolution of each position in the layout
+            self.active_cameras = set()
-        width = self.frame_shape[1] / self.columns
-        height = self.frame_shape[0] / self.rows
 
-        # For each camera in the layout:
+        removed_cameras = self.active_cameras.difference(active_cameras)
-        #   - resize the current frame and copy into the birdseye view
+        added_cameras = active_cameras.difference(self.active_cameras)
 
-        self.frame[:] = frame
+        self.active_cameras = active_cameras
+
+        # update each position in the layout
+        for position, camera in enumerate(self.camera_layout, start=0):
+
+            # if this camera was removed, replace it or clear it
+            if camera in removed_cameras:
+                # if replacing this camera with a newly added one
+                if len(added_cameras) > 0:
+                    added_camera = added_cameras.pop()
+                    self.camera_layout[position] = added_camera
+                    self.copy_to_position(
+                        position,
+                        added_camera,
+                        self.cameras[added_camera]["last_active_frame"],
+                    )
+                    self.cameras[added_camera]["layout_frame"] = self.cameras[
+                        added_camera
+                    ]["last_active_frame"]
+                # if removing this camera with no replacement
+                else:
+                    self.camera_layout[position] = None
+                    self.copy_to_position(position)
+                removed_cameras.remove(camera)
+            # if an empty spot and there are cameras to add
+            elif camera is None and len(added_cameras) > 0:
+                added_camera = added_cameras.pop()
+                self.camera_layout[position] = added_camera
+                self.copy_to_position(
+                    position,
+                    added_camera,
+                    self.cameras[added_camera]["last_active_frame"],
+                )
+                self.cameras[added_camera]["layout_frame"] = self.cameras[added_camera][
+                    "last_active_frame"
+                ]
+            # if not an empty spot and the camera has a newer frame, copy it
+            elif (
+                not camera is None
+                and self.cameras[camera]["last_active_frame"]
+                != self.cameras[camera]["layout_frame"]
+            ):
+                self.copy_to_position(
+                    position, camera, self.cameras[camera]["last_active_frame"]
+                )
+                self.cameras[camera]["layout_frame"] = self.cameras[camera][
+                    "last_active_frame"
+                ]
 
         return True
 
+    def update(self, camera, object_count, motion_count, frame_time, frame) -> bool:
+
+        # update the last active frame for the camera
+        if object_count > 0:
+            last_active_frame = self.cameras[camera]["last_active_frame"]
+            # cleanup the old frame
+            if last_active_frame != 0.0:
+                frame_id = f"{camera}{last_active_frame}"
+                self.frame_manager.delete(frame_id)
+            self.cameras[camera]["last_active_frame"] = frame_time
+
+        now = datetime.datetime.now().timestamp()
+
+        # limit output to ~24 fps
+        if (now - self.last_output_time) < 0.04:
+            return False
+
+        self.last_output_time = now
+
+        return self.update_frame()
+
 
 def output_frames(config, video_output_queue):
     threading.current_thread().name = f"output"
@@ -183,7 +289,7 @@ def output_frames(config, video_output_queue):
             camera, converters[camera], websocket_server
         )
 
-    converters["birdseye"] = FFMpegConverter(1920, 1080, 640, 320, "1000k")
+    converters["birdseye"] = FFMpegConverter(1920, 1080, 1280, 720, "2000k")
     broadcasters["birdseye"] = BroadcastThread(
         "birdseye", converters["birdseye"], websocket_server
     )
@@ -193,7 +299,7 @@ def output_frames(config, video_output_queue):
     for t in broadcasters.values():
         t.start()
 
-    birdseye_manager = BirdsEyeFrameManager(1080, 1920)
+    birdseye_manager = BirdsEyeFrameManager(config, frame_manager, 1080, 1920)
 
     while not stop_event.is_set():
         try:
@@ -233,9 +339,14 @@ def output_frames(config, video_output_queue):
                 converters["birdseye"].write(birdseye_manager.frame.tobytes())
 
         if camera in previous_frames:
-            frame_manager.delete(previous_frames[camera])
+            # if the birdseye manager still needs this frame, don't delete it
+            if (
+                birdseye_manager.cameras[camera]["last_active_frame"]
+                != previous_frames[camera]
+            ):
+                frame_manager.delete(f"{camera}{previous_frames[camera]}")
 
-        previous_frames[camera] = frame_id
+        previous_frames[camera] = frame_time
 
     while not video_output_queue.empty():
         (
@@ -259,4 +370,5 @@ def output_frames(config, video_output_queue):
     websocket_server.manager.join()
     websocket_server.shutdown()
     websocket_thread.join()
+    # TODO: use actual logger
     print("exiting output process...")

frigate/test/test_copy_yuv_to_position.py

@@ -0,0 +1,27 @@
+import cv2
+import numpy as np
+from unittest import TestCase, main
+from frigate.util import copy_yuv_to_position
+
+
+class TestCopyYuvToPosition(TestCase):
+    def setUp(self):
+        self.source_frame_bgr = np.zeros((400, 800, 3), np.uint8)
+        self.source_frame_bgr[:] = (0, 0, 255)
+        self.source_yuv_frame = cv2.cvtColor(
+            self.source_frame_bgr, cv2.COLOR_BGR2YUV_I420
+        )
+
+        self.dest_frame_bgr = np.zeros((400, 800, 3), np.uint8)
+        self.dest_frame_bgr[:] = (112, 202, 50)
+        self.dest_frame_bgr[100:300, 200:600] = (255, 0, 0)
+        self.dest_yuv_frame = cv2.cvtColor(self.dest_frame_bgr, cv2.COLOR_BGR2YUV_I420)
+
+    def test_copy_yuv_to_position(self):
+        copy_yuv_to_position(1, self.dest_yuv_frame, 3)
+        # cv2.imwrite(f"source_frame_yuv.jpg", self.source_yuv_frame)
+        # cv2.imwrite(f"dest_frame_yuv.jpg", self.dest_yuv_frame)
+
+
+if __name__ == "__main__":
+    main(verbosity=2)

frigate/util.py

@@ -3,6 +3,7 @@ import datetime
 import hashlib
 import json
 import logging
+import math
 import signal
 import subprocess as sp
 import threading
@@ -233,6 +234,98 @@ def yuv_crop_and_resize(frame, region, height=None):
     return yuv_cropped_frame
 
 
+def copy_yuv_to_position(
+    position,
+    destination_frame,
+    destination_dim,
+    source_frame=None,
+    source_channel_dim=None,
+):
+    # TODO: consider calculating this on layout reflow instead of all the time
+    layout_shape = (
+        (destination_frame.shape[0] // 3 * 2) // destination_dim,
+        destination_frame.shape[1] // destination_dim,
+    )
+    # calculate the x and y offset for the frame in the layout
+    y_offset = layout_shape[0] * math.floor(position / destination_dim)
+    x_offset = layout_shape[1] * (position % destination_dim)
+
+    # get the coordinates of the channels for this position in the layout
+    y, u1, u2, v1, v2 = get_yuv_crop(
+        destination_frame.shape,
+        (
+            x_offset,
+            y_offset,
+            x_offset + layout_shape[1],
+            y_offset + layout_shape[0],
+        ),
+    )
+
+    if source_frame is None:
+        # clear y
+        destination_frame[
+            y[1] : y[3],
+            y[0] : y[2],
+        ] = 16
+
+        # clear u1
+        destination_frame[u1[1] : u1[3], u1[0] : u1[2]] = 128
+        # clear u2
+        destination_frame[u2[1] : u2[3], u2[0] : u2[2]] = 128
+        # clear v1
+        destination_frame[v1[1] : v1[3], v1[0] : v1[2]] = 128
+        # clear v2
+        destination_frame[v2[1] : v2[3], v2[0] : v2[2]] = 128
+    else:
+        interpolation = cv2.INTER_AREA
+        # resize/copy y channel
+        destination_frame[y[1] : y[3], y[0] : y[2]] = cv2.resize(
+            source_frame[
+                source_channel_dim["y"][1] : source_channel_dim["y"][3],
+                source_channel_dim["y"][0] : source_channel_dim["y"][2],
+            ],
+            dsize=(y[2] - y[0], y[3] - y[1]),
+            interpolation=interpolation,
+        )
+
+        # resize/copy u1
+        destination_frame[u1[1] : u1[3], u1[0] : u1[2]] = cv2.resize(
+            source_frame[
+                source_channel_dim["u1"][1] : source_channel_dim["u1"][3],
+                source_channel_dim["u1"][0] : source_channel_dim["u1"][2],
+            ],
+            dsize=(u1[2] - u1[0], u1[3] - u1[1]),
+            interpolation=interpolation,
+        )
+        # resize/copy u2
+        destination_frame[u2[1] : u2[3], u2[0] : u2[2]] = cv2.resize(
+            source_frame[
+                source_channel_dim["u2"][1] : source_channel_dim["u2"][3],
+                source_channel_dim["u2"][0] : source_channel_dim["u2"][2],
+            ],
+            dsize=(u2[2] - u2[0], u2[3] - u2[1]),
+            interpolation=interpolation,
+        )
+        # resize/copy v1
+        destination_frame[v1[1] : v1[3], v1[0] : v1[2]] = cv2.resize(
+            source_frame[
+                source_channel_dim["v1"][1] : source_channel_dim["v1"][3],
+                source_channel_dim["v1"][0] : source_channel_dim["v1"][2],
+            ],
+            dsize=(v1[2] - v1[0], v1[3] - v1[1]),
+            interpolation=interpolation,
+        )
+        # resize/copy v2
+        destination_frame[v2[1] : v2[3], v2[0] : v2[2]] = cv2.resize(
+            source_frame[
+                source_channel_dim["v2"][1] : source_channel_dim["v2"][3],
+                source_channel_dim["v2"][0] : source_channel_dim["v2"][2],
+            ],
+            dsize=(v2[2] - v2[0], v2[3] - v2[1]),
+            interpolation=interpolation,
+        )
+
+
 def yuv_region_2_rgb(frame, region):
     try:
         # TODO: does this copy the numpy array?