basic plumbing for birdseye view

frigate/app.py

@@ -14,6 +14,7 @@ from peewee_migrate import Router
 from playhouse.sqlite_ext import SqliteExtDatabase
 from playhouse.sqliteq import SqliteQueueDatabase
 
+from frigate.birdseye import BirdsEyeFrameOutputter
 from frigate.config import FrigateConfig
 from frigate.const import RECORD_DIR, CLIPS_DIR, CACHE_DIR
 from frigate.edgetpu import EdgeTPUProcess
@@ -250,6 +251,10 @@ class FrigateApp:
             capture_process.start()
             logger.info(f"Capture process started for {name}: {capture_process.pid}")
 
+    def start_birdseye_outputter(self):
+        self.birdseye_outputter = BirdsEyeFrameOutputter(self.stop_event)
+        self.birdseye_outputter.start()
+
     def start_event_processor(self):
         self.event_processor = EventProcessor(
             self.config,
@@ -306,6 +311,7 @@ class FrigateApp:
         self.start_detected_frames_processor()
         self.start_camera_processors()
         self.start_camera_capture_processes()
+        self.start_birdseye_outputter()
         self.init_stats()
         self.init_web_server()
         self.start_event_processor()

frigate/birdseye.py

@@ -0,0 +1,84 @@
+import multiprocessing as mp
+import numpy as np
+import subprocess as sp
+import logging
+import threading
+from frigate.util import SharedMemoryFrameManager
+
+logger = logging.getLogger(__name__)
+
+# methods for maintaining the birdseyeframe in the object processing thread
+# avoids work when no clients are listening
+class BirdsEyeFrameManager:
+    def __init__(self):
+        # self.config = config
+        self.frame_manager = SharedMemoryFrameManager()
+        self._frame_shape = (1080, 1920)
+        self.frame_shape_yuv = (self._frame_shape[0] * 3 // 2, self._frame_shape[1])
+        self.frame_shm = mp.shared_memory.SharedMemory(
+            name=f"birdseye-frame",
+            create=True,
+            size=self.frame_shape_yuv[0] * self.frame_shape_yuv[1],
+        )
+        self.frame = np.ndarray(
+            self.frame_shape_yuv, dtype=np.uint8, buffer=self.frame_shm.buf
+        )
+
+        # initialize the frame as black and with the frigate logo
+        self.blank_frame = np.zeros((1080 * 3 // 2, 1920), np.uint8)
+        self.blank_frame[:] = 128
+        self.blank_frame[0:1080, 0:1920] = 16
+
+        self.frame[:] = self.blank_frame
+
+    def update_frame(self, camera, object_count, motion_count, frame_time, frame):
+        # determine how many cameras are tracking objects (or recently were)
+        # decide on a layout for the birdseye view (try to avoid too much churn)
+        # calculate position of each camera
+        # calculate resolution of each position in the layout
+        # if layout is changing, wipe the frame black again
+        # For each camera currently tracking objects (alphabetical):
+        #   - resize the current frame and copy into the birdseye view
+        # signal to birdseye process that the frame is ready to send
+
+        self.frame[:] = frame
+
+
+# separate process for managing the external ffmpeg process and sending frame
+# bytes to ffmpeg
+class BirdsEyeFrameOutputter(threading.Thread):
+    def __init__(self, stop_event):
+        threading.Thread.__init__(self)
+        self.stop_event = stop_event
+        self.frame_shm = mp.shared_memory.SharedMemory(
+            name=f"birdseye-frame",
+            create=False,
+        )
+
+    def start_ffmpeg(self):
+        ffmpeg_cmd = "ffmpeg -f rawvideo -pix_fmt yuv420p -video_size 1920x1080 -i pipe: -f mpegts -codec:v mpeg1video -b:v 1000k -bf 0 http://localhost:8081/birdseye".split(
+            " "
+        )
+        self.process = sp.Popen(
+            ffmpeg_cmd,
+            stdout=sp.DEVNULL,
+            # TODO: logging
+            stderr=sp.DEVNULL,
+            stdin=sp.PIPE,
+            start_new_session=True,
+        )
+
+    def run(self):
+        self.start_ffmpeg()
+
+        while not self.stop_event.wait(1):
+            if self.process.poll() != None:
+                logger.info(f"ffmpeg process is not running. restarting ...")
+                self.start_ffmpeg()
+
+            self.process.stdin.write(self.frame_shm.buf.tobytes())
+
+
+# separate process for passing jsmpeg packets over websockets
+# signals to the frame manager when a client is listening
+# class JSMpegSocketServer:

frigate/object_processing.py

@@ -17,6 +17,7 @@ import cv2
 import matplotlib.pyplot as plt
 import numpy as np
 
+from frigate.birdseye import BirdsEyeFrameManager
 from frigate.config import FrigateConfig, CameraConfig
 from frigate.const import RECORD_DIR, CLIPS_DIR, CACHE_DIR
 from frigate.edgetpu import load_labels
@@ -563,6 +564,7 @@ class TrackedObjectProcessor(threading.Thread):
         self.stop_event = stop_event
         self.camera_states: Dict[str, CameraState] = {}
         self.frame_manager = SharedMemoryFrameManager()
+        self.birdseye_frame_manager = BirdsEyeFrameManager()
 
         def start(camera, obj: TrackedObject, current_frame_time):
             self.event_queue.put(("start", camera, obj.to_dict()))
@@ -717,6 +719,14 @@ class TrackedObjectProcessor(threading.Thread):
                 frame_time, current_tracked_objects, motion_boxes, regions
             )
 
+            self.birdseye_frame_manager.update_frame(
+                camera,
+                len(current_tracked_objects),
+                len(motion_boxes),
+                camera_state.current_frame_time,
+                camera_state._current_frame,
+            )
+
             # update zone counts for each label
             # for each zone in the current camera
             for zone in self.config.cameras[camera].zones.keys():

web/public/jsmpeg.html

@@ -0,0 +1,22 @@
+<!DOCTYPE html>
+<html>
+<head>
+	<title>JSMpeg Stream Client</title>
+	<style type="text/css">
+		html, body {
+			background-color: #111;
+			text-align: center;
+		}
+	</style>
+
+</head>
+<body>
+	<canvas id="video-canvas"></canvas>
+	<script type="text/javascript" src="jsmpeg.min.js"></script>
+	<script type="text/javascript">
+		var canvas = document.getElementById('video-canvas');
+		var url = 'ws://'+document.location.hostname+':5000/live/birdseye';
+		var player = new JSMpeg.Player(url, {canvas: canvas});
+	</script>
+</body>
+</html>