From cd35481e92f206119874c5e0acabaf9dca12de8d Mon Sep 17 00:00:00 2001
From: winstona <w.wonton@gmail.com>
Date: Wed, 18 Oct 2023 04:52:48 -0700
Subject: [PATCH] Fix recording events intermittently missing (#8162)

* fix queues not emptying fully by changing gets to a blocking call with short timeout

* add extra error/warning messages when there's a possibility of missing recording segments
---
 frigate/record/maintainer.py | 35 ++++++++++++++++++++++++++++++++---
 1 file changed, 32 insertions(+), 3 deletions(-)

diff --git a/frigate/record/maintainer.py b/frigate/record/maintainer.py
index 9aa60585d..c2264d63f 100644
--- a/frigate/record/maintainer.py
+++ b/frigate/record/maintainer.py
@@ -410,7 +410,8 @@ class RecordingMaintainer(threading.Thread):
         wait_time = 0.0
         while not self.stop_event.wait(wait_time):
             run_start = datetime.datetime.now().timestamp()
-
+            stale_frame_count = 0
+            stale_frame_count_threshold = 10
             # empty the object recordings info queue
             while True:
                 try:
@@ -420,7 +421,10 @@ class RecordingMaintainer(threading.Thread):
                         current_tracked_objects,
                         motion_boxes,
                         regions,
-                    ) = self.object_recordings_info_queue.get(False)
+                    ) = self.object_recordings_info_queue.get(True, timeout=0.1)
+
+                    if frame_time < run_start - stale_frame_count_threshold:
+                        stale_frame_count += 1
 
                     if self.process_info[camera]["record_enabled"].value:
                         self.object_recordings_info[camera].append(
@@ -432,17 +436,32 @@ class RecordingMaintainer(threading.Thread):
                             )
                         )
                 except queue.Empty:
+                    q_size = self.object_recordings_info_queue.qsize()
+                    if q_size > 5:
+                        logger.warning(
+                            f"object_recordings_info loop queue not empty ({q_size}) - recording segments may be missing"
+                        )
                     break
 
+            if stale_frame_count > 0:
+                logger.error(
+                    f"Found {stale_frame_count} old frames, segments from recordings may be missing"
+                )
+
             # empty the audio recordings info queue if audio is enabled
             if self.audio_recordings_info_queue:
+                stale_frame_count = 0
+
                 while True:
                     try:
                         (
                             camera,
                             frame_time,
                             dBFS,
-                        ) = self.audio_recordings_info_queue.get(False)
+                        ) = self.audio_recordings_info_queue.get(True, timeout=0.1)
+
+                        if frame_time < run_start - stale_frame_count_threshold:
+                            stale_frame_count += 1
 
                         if self.process_info[camera]["record_enabled"].value:
                             self.audio_recordings_info[camera].append(
@@ -452,8 +471,18 @@ class RecordingMaintainer(threading.Thread):
                                 )
                             )
                     except queue.Empty:
+                        q_size = self.audio_recordings_info_queue.qsize()
+                        if q_size > 5:
+                            logger.warning(
+                                f"object_recordings_info loop audio queue not empty ({q_size}) - recording segments may be missing"
+                            )
                         break
 
+                if stale_frame_count > 0:
+                    logger.error(
+                        f"Found {stale_frame_count} old audio frames, segments from recordings may be missing"
+                    )
+
             try:
                 asyncio.run(self.move_files())
             except Exception as e: