Write recording segments to cache with timezone info (#8468)

* Store recording segments with timezone info

* Don't use _

* Use different separator due to timezone

frigate/config.py

@@ -17,6 +17,7 @@ from frigate.const import (
     ALL_ATTRIBUTE_LABELS,
     AUDIO_MIN_CONFIDENCE,
     CACHE_DIR,
+    CACHE_SEGMENT_FORMAT,
     DEFAULT_DB_PATH,
     REGEX_CAMERA_NAME,
     YAML_EXT,
@@ -865,7 +866,7 @@ class CameraConfig(FrigateBaseModel):
 
             ffmpeg_output_args = (
                 record_args
-                + [f"{os.path.join(CACHE_DIR, self.name)}-%Y%m%d%H%M%S.mp4"]
+                + [f"{os.path.join(CACHE_DIR, self.name)}@{CACHE_SEGMENT_FORMAT}.mp4"]
                 + ffmpeg_output_args
             )
 

frigate/const.py

@@ -50,6 +50,7 @@ DRIVER_INTEL_iHD = "iHD"
 
 # Record Values
 
+CACHE_SEGMENT_FORMAT = "%Y%m%d%H%M%S%z"
 MAX_SEGMENT_DURATION = 600
 MAX_PLAYLIST_SECONDS = 7200  # support 2 hour segments for a single playlist to account for cameras with inconsistent segment times
 

frigate/record/maintainer.py

@@ -20,6 +20,7 @@ import psutil
 from frigate.config import FrigateConfig, RetainModeEnum
 from frigate.const import (
     CACHE_DIR,
+    CACHE_SEGMENT_FORMAT,
     INSERT_MANY_RECORDINGS,
     MAX_SEGMENT_DURATION,
     RECORD_DIR,
@@ -74,15 +75,13 @@ class RecordingMaintainer(threading.Thread):
         self.end_time_cache: dict[str, Tuple[datetime.datetime, float]] = {}
 
     async def move_files(self) -> None:
-        cache_files = sorted(
+        cache_files = [
-            [
+            d
-                d
+            for d in os.listdir(CACHE_DIR)
-                for d in os.listdir(CACHE_DIR)
+            if os.path.isfile(os.path.join(CACHE_DIR, d))
-                if os.path.isfile(os.path.join(CACHE_DIR, d))
+            and d.endswith(".mp4")
-                and d.endswith(".mp4")
+            and not d.startswith("clip_")
-                and not d.startswith("clip_")
+        ]
-            ]
-        )
 
         files_in_use = []
         for process in psutil.process_iter():
@@ -106,8 +105,12 @@ class RecordingMaintainer(threading.Thread):
 
             cache_path = os.path.join(CACHE_DIR, cache)
             basename = os.path.splitext(cache)[0]
-            camera, date = basename.rsplit("-", maxsplit=1)
+            camera, date = basename.rsplit("@", maxsplit=1)
-            start_time = datetime.datetime.strptime(date, "%Y%m%d%H%M%S")
+
+            # important that start_time is utc because recordings are stored and compared in utc
+            start_time = datetime.datetime.strptime(
+                date, CACHE_SEGMENT_FORMAT
+            ).astimezone(datetime.timezone.utc)
 
             grouped_recordings[camera].append(
                 {
@@ -119,6 +122,11 @@ class RecordingMaintainer(threading.Thread):
         # delete all cached files past the most recent 5
         keep_count = 5
         for camera in grouped_recordings.keys():
+            # sort based on start time
+            grouped_recordings[camera] = sorted(
+                grouped_recordings[camera], key=lambda s: s["start_time"]
+            )
+
             segment_count = len(grouped_recordings[camera])
             if segment_count > keep_count:
                 logger.warning(
@@ -218,7 +226,7 @@ class RecordingMaintainer(threading.Thread):
         # if cached file's start_time is earlier than the retain days for the camera
         if start_time <= (
             (
-                datetime.datetime.now()
+                datetime.datetime.now().astimezone(datetime.timezone.utc)
                 - datetime.timedelta(
                     days=self.config.cameras[camera].record.retain.days
                 )
@@ -262,8 +270,8 @@ class RecordingMaintainer(threading.Thread):
                 )
                 retain_cutoff = datetime.datetime.fromtimestamp(
                     most_recently_processed_frame_time - pre_capture
-                ).astimezone(datetime.timezone.utc)
+                )
-                if end_time.astimezone(datetime.timezone.utc) < retain_cutoff:
+                if end_time < retain_cutoff:
                     Path(cache_path).unlink(missing_ok=True)
                     self.end_time_cache.pop(cache_path, None)
         # else retain days includes this segment
@@ -275,10 +283,11 @@ class RecordingMaintainer(threading.Thread):
             )
 
             # ensure delayed segment info does not lead to lost segments
-            if datetime.datetime.fromtimestamp(
+            if (
-                most_recently_processed_frame_time
+                datetime.datetime.fromtimestamp(
-            ).astimezone(datetime.timezone.utc) >= end_time.astimezone(
+                    most_recently_processed_frame_time
-                datetime.timezone.utc
+                ).astimezone(datetime.timezone.utc)
+                >= end_time
             ):
                 record_mode = self.config.cameras[camera].record.retain.mode
                 return await self.move_segment(
@@ -345,18 +354,18 @@ class RecordingMaintainer(threading.Thread):
             self.end_time_cache.pop(cache_path, None)
             return
 
+        # directory will be in utc due to start_time being in utc
         directory = os.path.join(
             RECORD_DIR,
-            start_time.astimezone(tz=datetime.timezone.utc).strftime("%Y-%m-%d/%H"),
+            start_time.strftime("%Y-%m-%d/%H"),
             camera,
         )
 
         if not os.path.exists(directory):
             os.makedirs(directory)
 
-        file_name = (
+        # file will be in utc due to start_time being in utc
-            f"{start_time.replace(tzinfo=datetime.timezone.utc).strftime('%M.%S.mp4')}"
+        file_name = f"{start_time.strftime('%M.%S.mp4')}"
-        )
         file_path = os.path.join(directory, file_name)
 
         try:

frigate/video.py

@@ -16,6 +16,7 @@ from frigate.const import (
     ALL_ATTRIBUTE_LABELS,
     ATTRIBUTE_LABEL_MAP,
     CACHE_DIR,
+    CACHE_SEGMENT_FORMAT,
     REQUEST_REGION_GRID,
 )
 from frigate.log import LogPipe
@@ -300,19 +301,19 @@ class CameraWatchdog(threading.Thread):
                 and not d.startswith("clip_")
             ]
         )
-        newest_segment_timestamp = latest_segment
+        newest_segment_time = latest_segment
 
         for file in cache_files:
             if self.camera_name in file:
                 basename = os.path.splitext(file)[0]
-                _, date = basename.rsplit("-", maxsplit=1)
+                _, date = basename.rsplit("@", maxsplit=1)
-                ts = datetime.datetime.strptime(date, "%Y%m%d%H%M%S").astimezone(
+                segment_time = datetime.datetime.strptime(
-                    datetime.timezone.utc
+                    date, CACHE_SEGMENT_FORMAT
-                )
+                ).astimezone(datetime.timezone.utc)
-                if ts > newest_segment_timestamp:
+                if segment_time > newest_segment_time:
-                    newest_segment_timestamp = ts
+                    newest_segment_time = segment_time
 
-        return newest_segment_timestamp
+        return newest_segment_time
 
 
 class CameraCapture(threading.Thread):