From 89dd114da1174f837ad35d5d51e8f90375348b05 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Sun, 5 Nov 2023 13:30:29 -0700 Subject: [PATCH] Write recording segments to cache with timezone info (#8468) * Store recording segments with timezone info * Don't use _ * Use different separator due to timezone --- frigate/config.py | 3 +- frigate/const.py | 1 + frigate/record/maintainer.py | 53 +++++++++++++++++++++--------------- frigate/video.py | 17 ++++++------ 4 files changed, 43 insertions(+), 31 deletions(-) diff --git a/frigate/config.py b/frigate/config.py index 82e518923..777cdc152 100644 --- a/frigate/config.py +++ b/frigate/config.py @@ -17,6 +17,7 @@ from frigate.const import ( ALL_ATTRIBUTE_LABELS, AUDIO_MIN_CONFIDENCE, CACHE_DIR, + CACHE_SEGMENT_FORMAT, DEFAULT_DB_PATH, REGEX_CAMERA_NAME, YAML_EXT, @@ -865,7 +866,7 @@ class CameraConfig(FrigateBaseModel): ffmpeg_output_args = ( record_args - + [f"{os.path.join(CACHE_DIR, self.name)}-%Y%m%d%H%M%S.mp4"] + + [f"{os.path.join(CACHE_DIR, self.name)}@{CACHE_SEGMENT_FORMAT}.mp4"] + ffmpeg_output_args ) diff --git a/frigate/const.py b/frigate/const.py index 9fe18a0f2..69697a4b1 100644 --- a/frigate/const.py +++ b/frigate/const.py @@ -50,6 +50,7 @@ DRIVER_INTEL_iHD = "iHD" # Record Values +CACHE_SEGMENT_FORMAT = "%Y%m%d%H%M%S%z" MAX_SEGMENT_DURATION = 600 MAX_PLAYLIST_SECONDS = 7200 # support 2 hour segments for a single playlist to account for cameras with inconsistent segment times diff --git a/frigate/record/maintainer.py b/frigate/record/maintainer.py index 0520e086a..8e9f900bb 100644 --- a/frigate/record/maintainer.py +++ b/frigate/record/maintainer.py @@ -20,6 +20,7 @@ import psutil from frigate.config import FrigateConfig, RetainModeEnum from frigate.const import ( CACHE_DIR, + CACHE_SEGMENT_FORMAT, INSERT_MANY_RECORDINGS, MAX_SEGMENT_DURATION, RECORD_DIR, @@ -74,15 +75,13 @@ class RecordingMaintainer(threading.Thread): self.end_time_cache: dict[str, Tuple[datetime.datetime, float]] = {} async def move_files(self) -> None: - cache_files = sorted( - [ - d - for d in os.listdir(CACHE_DIR) - if os.path.isfile(os.path.join(CACHE_DIR, d)) - and d.endswith(".mp4") - and not d.startswith("clip_") - ] - ) + cache_files = [ + d + for d in os.listdir(CACHE_DIR) + if os.path.isfile(os.path.join(CACHE_DIR, d)) + and d.endswith(".mp4") + and not d.startswith("clip_") + ] files_in_use = [] for process in psutil.process_iter(): @@ -106,8 +105,12 @@ class RecordingMaintainer(threading.Thread): cache_path = os.path.join(CACHE_DIR, cache) basename = os.path.splitext(cache)[0] - camera, date = basename.rsplit("-", maxsplit=1) - start_time = datetime.datetime.strptime(date, "%Y%m%d%H%M%S") + camera, date = basename.rsplit("@", maxsplit=1) + + # important that start_time is utc because recordings are stored and compared in utc + start_time = datetime.datetime.strptime( + date, CACHE_SEGMENT_FORMAT + ).astimezone(datetime.timezone.utc) grouped_recordings[camera].append( { @@ -119,6 +122,11 @@ class RecordingMaintainer(threading.Thread): # delete all cached files past the most recent 5 keep_count = 5 for camera in grouped_recordings.keys(): + # sort based on start time + grouped_recordings[camera] = sorted( + grouped_recordings[camera], key=lambda s: s["start_time"] + ) + segment_count = len(grouped_recordings[camera]) if segment_count > keep_count: logger.warning( @@ -218,7 +226,7 @@ class RecordingMaintainer(threading.Thread): # if cached file's start_time is earlier than the retain days for the camera if start_time <= ( ( - datetime.datetime.now() + datetime.datetime.now().astimezone(datetime.timezone.utc) - datetime.timedelta( days=self.config.cameras[camera].record.retain.days ) @@ -262,8 +270,8 @@ class RecordingMaintainer(threading.Thread): ) retain_cutoff = datetime.datetime.fromtimestamp( most_recently_processed_frame_time - pre_capture - ).astimezone(datetime.timezone.utc) - if end_time.astimezone(datetime.timezone.utc) < retain_cutoff: + ) + if end_time < retain_cutoff: Path(cache_path).unlink(missing_ok=True) self.end_time_cache.pop(cache_path, None) # else retain days includes this segment @@ -275,10 +283,11 @@ class RecordingMaintainer(threading.Thread): ) # ensure delayed segment info does not lead to lost segments - if datetime.datetime.fromtimestamp( - most_recently_processed_frame_time - ).astimezone(datetime.timezone.utc) >= end_time.astimezone( - datetime.timezone.utc + if ( + datetime.datetime.fromtimestamp( + most_recently_processed_frame_time + ).astimezone(datetime.timezone.utc) + >= end_time ): record_mode = self.config.cameras[camera].record.retain.mode return await self.move_segment( @@ -345,18 +354,18 @@ class RecordingMaintainer(threading.Thread): self.end_time_cache.pop(cache_path, None) return + # directory will be in utc due to start_time being in utc directory = os.path.join( RECORD_DIR, - start_time.astimezone(tz=datetime.timezone.utc).strftime("%Y-%m-%d/%H"), + start_time.strftime("%Y-%m-%d/%H"), camera, ) if not os.path.exists(directory): os.makedirs(directory) - file_name = ( - f"{start_time.replace(tzinfo=datetime.timezone.utc).strftime('%M.%S.mp4')}" - ) + # file will be in utc due to start_time being in utc + file_name = f"{start_time.strftime('%M.%S.mp4')}" file_path = os.path.join(directory, file_name) try: diff --git a/frigate/video.py b/frigate/video.py index 7ab41a5a6..b5fafec0b 100755 --- a/frigate/video.py +++ b/frigate/video.py @@ -16,6 +16,7 @@ from frigate.const import ( ALL_ATTRIBUTE_LABELS, ATTRIBUTE_LABEL_MAP, CACHE_DIR, + CACHE_SEGMENT_FORMAT, REQUEST_REGION_GRID, ) from frigate.log import LogPipe @@ -300,19 +301,19 @@ class CameraWatchdog(threading.Thread): and not d.startswith("clip_") ] ) - newest_segment_timestamp = latest_segment + newest_segment_time = latest_segment for file in cache_files: if self.camera_name in file: basename = os.path.splitext(file)[0] - _, date = basename.rsplit("-", maxsplit=1) - ts = datetime.datetime.strptime(date, "%Y%m%d%H%M%S").astimezone( - datetime.timezone.utc - ) - if ts > newest_segment_timestamp: - newest_segment_timestamp = ts + _, date = basename.rsplit("@", maxsplit=1) + segment_time = datetime.datetime.strptime( + date, CACHE_SEGMENT_FORMAT + ).astimezone(datetime.timezone.utc) + if segment_time > newest_segment_time: + newest_segment_time = segment_time - return newest_segment_timestamp + return newest_segment_time class CameraCapture(threading.Thread):