experimental: running ffmpeg directly and capturing raw frames

Dockerfile

@@ -52,7 +52,8 @@ RUN  pip install -U pip \
  numpy \
  Flask \
  paho-mqtt \
- PyYAML
+ PyYAML \
+ ffmpeg-python
 
 # Download & build OpenCV
 RUN wget -q -P /usr/local/src/ --no-check-certificate https://github.com/opencv/opencv/archive/4.0.1.zip

frigate/object_detection.py

@@ -90,12 +90,12 @@ class FramePrepper(threading.Thread):
                 frame_time = self.frame_time.value
             
             # convert to RGB
-            cropped_frame_rgb = cv2.cvtColor(cropped_frame, cv2.COLOR_BGR2RGB)
+            #cropped_frame_rgb = cv2.cvtColor(cropped_frame, cv2.COLOR_BGR2RGB)
             # Resize to 300x300 if needed
-            if cropped_frame_rgb.shape != (300, 300, 3):
+            if cropped_frame.shape != (300, 300, 3):
-                cropped_frame_rgb = cv2.resize(cropped_frame_rgb, dsize=(300, 300), interpolation=cv2.INTER_LINEAR)
+                cropped_frame = cv2.resize(cropped_frame, dsize=(300, 300), interpolation=cv2.INTER_LINEAR)
             # Expand dimensions since the model expects images to have shape: [1, 300, 300, 3]
-            frame_expanded = np.expand_dims(cropped_frame_rgb, axis=0)
+            frame_expanded = np.expand_dims(cropped_frame, axis=0)
 
             # add the frame to the queue
             if not self.prepped_frame_queue.full():

frigate/video.py

@@ -6,6 +6,7 @@ import threading
 import ctypes
 import multiprocessing as mp
 import numpy as np
+import ffmpeg
 from . util import tonumpyarray
 from . object_detection import FramePrepper
 from . objects import ObjectCleaner, BestPersonFrame
@@ -16,48 +17,41 @@ def fetch_frames(shared_arr, shared_frame_time, frame_lock, frame_ready, frame_s
     # convert shared memory array into numpy and shape into image array
     arr = tonumpyarray(shared_arr).reshape(frame_shape)
 
-    # start the video capture
+    ffmpeg_process = (
-    video = cv2.VideoCapture()
+        ffmpeg
-    video.open(rtsp_url)
+        .input(rtsp_url, 
-    print("Opening the RTSP Url...")
+            rtsp_transport="tcp", 
-    # keep the buffer small so we minimize old data
+            stimeout=5000000, 
-    video.set(cv2.CAP_PROP_BUFFERSIZE,1)
+            use_wallclock_as_timestamps=1,
+            fflags="+genpts",
+            avoid_negative_ts="make_zero")
+        .output('pipe:', format='rawvideo', pix_fmt='rgb24')
+    )
+
+    print(ffmpeg_process.compile())
+
+    ffmpeg_process = ffmpeg_process.run_async(pipe_stdout=True)
 
-    bad_frame_counter = 0
     while True:
-        # check if the video stream is still open, and reopen if needed
+        in_bytes = ffmpeg_process.stdout.read(frame_shape[0] * frame_shape[1] * frame_shape[2])
-        if not video.isOpened():
+        if not in_bytes:
-            success = video.open(rtsp_url)
+            print("No bytes received. Waiting 1 second before trying again.")
-            if not success:
+            time.sleep(1)
-                time.sleep(1)
+            continue
-                continue
+        frame = (
-        # grab the frame, but dont decode it yet
+            np
-        ret = video.grab()
+            .frombuffer(in_bytes, np.uint8)
-        # snapshot the time the frame was grabbed
+            .reshape(frame_shape)
-        frame_time = datetime.datetime.now()
+        )
-        if ret:
+        # Lock access and update frame
-            # go ahead and decode the current frame
+        with frame_lock:
-            ret, frame = video.retrieve()
+            shared_frame_time.value = datetime.datetime.now().timestamp()
-            if ret:
+            arr[:] = frame
-                # Lock access and update frame
+        # Notify with the condition that a new frame is ready
-                with frame_lock:
+        with frame_ready:
-                    arr[:] = frame
+            frame_ready.notify_all()
-                    shared_frame_time.value = frame_time.timestamp()
-                # Notify with the condition that a new frame is ready
-                with frame_ready:
-                    frame_ready.notify_all()
-                bad_frame_counter = 0
-            else:
-                print("Unable to decode frame")
-                bad_frame_counter += 1
-        else:
-            print("Unable to grab a frame")
-            bad_frame_counter += 1
 
-        if bad_frame_counter > 100:
+    ffmpeg_process.wait()
-            video.release()
-    
-    video.release()
 
 # Stores 2 seconds worth of frames when motion is detected so they can be used for other threads
 class FrameTracker(threading.Thread):
@@ -279,7 +273,7 @@ class Camera:
             frame = self.shared_frame_np.copy()
 
         # convert to RGB for drawing
-        frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+        #frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
         # draw the bounding boxes on the screen
         for obj in detected_objects:
             color = (255,0,0)