From 65ddd9185584acd61c00e35046459d16fa4ebdcf Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Sun, 29 Nov 2020 21:31:02 -0600 Subject: [PATCH] add recording maintenance --- frigate/app.py | 7 +++ frigate/config.py | 10 ++-- frigate/record.py | 113 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 124 insertions(+), 6 deletions(-) create mode 100644 frigate/record.py diff --git a/frigate/app.py b/frigate/app.py index 15e4db119..f922f9906 100644 --- a/frigate/app.py +++ b/frigate/app.py @@ -16,6 +16,7 @@ from frigate.log import log_process, root_configurer from frigate.models import Event from frigate.mqtt import create_mqtt_client from frigate.object_processing import TrackedObjectProcessor +from frigate.record import RecordingMaintainer from frigate.video import capture_camera, track_camera from frigate.watchdog import FrigateWatchdog @@ -120,6 +121,10 @@ class FrigateApp(): self.event_cleanup = EventCleanup(self.config, self.stop_event) self.event_cleanup.start() + def start_recording_maintainer(self): + self.recording_maintainer = RecordingMaintainer(self.config, self.stop_event) + self.recording_maintainer.start() + def start_watchdog(self): self.frigate_watchdog = FrigateWatchdog(self.detectors, self.stop_event) self.frigate_watchdog.start() @@ -138,6 +143,7 @@ class FrigateApp(): self.init_web_server() self.start_event_processor() self.start_event_cleanup() + self.start_recording_maintainer() self.start_watchdog() self.flask_app.run(host='127.0.0.1', port=5001, debug=False) self.stop() @@ -149,6 +155,7 @@ class FrigateApp(): self.detected_frames_processor.join() self.event_processor.join() self.event_cleanup.join() + self.recording_maintainer.join() self.frigate_watchdog.join() for detector in self.detectors.values(): diff --git a/frigate/config.py b/frigate/config.py index 16bafc1f3..265eacde1 100644 --- a/frigate/config.py +++ b/frigate/config.py @@ -104,7 +104,6 @@ def filters_for_all_tracked_objects(object_config): OBJECTS_SCHEMA = vol.Schema(vol.All(filters_for_all_tracked_objects, { vol.Optional('track', default=['person']): [str], - # TODO: this should populate filters for all tracked objects vol.Optional('filters', default = {}): FILTER_SCHEMA.extend({ str: {vol.Optional('min_score', default=0.5): float}}) } )) @@ -617,7 +616,6 @@ class CameraConfig(): def _get_ffmpeg_cmd(self, ffmpeg_input, cache_dir): ffmpeg_output_args = [] - # TODO: ensure output args exist for each role and each role is only used once if 'detect' in ffmpeg_input.roles: ffmpeg_output_args = self.ffmpeg.output_args['detect'] + ffmpeg_output_args + ['pipe:'] if self.fps: @@ -630,10 +628,10 @@ class CameraConfig(): ffmpeg_output_args = self.ffmpeg.output_args['clips'] + [ f"{os.path.join(cache_dir, self.name)}-%Y%m%d%H%M%S.mp4" ] + ffmpeg_output_args - # if 'record' in ffmpeg_input.roles and self.save_clips.enabled: - # ffmpeg_output_args = self.ffmpeg.output_args['record'] + [ - # f"{os.path.join(cache_dir, self.name)}-%Y%m%d%H%M%S.mp4" - # ] + ffmpeg_output_args + if 'record' in ffmpeg_input.roles and self.record.enabled: + ffmpeg_output_args = self.ffmpeg.output_args['record'] + [ + f"{os.path.join(self.record.record_dir, self.name)}-%Y%m%d%H%M%S.mp4" + ] + ffmpeg_output_args return (['ffmpeg'] + ffmpeg_input.global_args + ffmpeg_input.hwaccel_args + diff --git a/frigate/record.py b/frigate/record.py new file mode 100644 index 000000000..505dbd4dc --- /dev/null +++ b/frigate/record.py @@ -0,0 +1,113 @@ +import datetime +import json +import logging +import os +import queue +import subprocess as sp +import threading +import time +from collections import defaultdict +from pathlib import Path + +import psutil + +from frigate.config import FrigateConfig + +logger = logging.getLogger(__name__) + +SECONDS_IN_DAY = 60 * 60 * 24 + +class RecordingMaintainer(threading.Thread): + def __init__(self, config: FrigateConfig, stop_event): + threading.Thread.__init__(self) + self.name = 'recording_maint' + self.config = config + record_dirs = list(set([camera.record.record_dir for camera in self.config.cameras.values()])) + self.record_dir = None if len(record_dirs) == 0 else record_dirs[0] + self.stop_event = stop_event + + def move_files(self): + if self.record_dir is None: + return + + recordings = [d for d in os.listdir(self.record_dir) if os.path.isfile(os.path.join(self.record_dir, d)) and d.endswith(".mp4")] + + files_in_use = [] + for process in psutil.process_iter(): + if process.name() != 'ffmpeg': + continue + try: + flist = process.open_files() + if flist: + for nt in flist: + if nt.path.startswith(self.record_dir): + files_in_use.append(nt.path.split('/')[-1]) + except: + continue + + for f in recordings: + if f in files_in_use: + continue + + camera = '-'.join(f.split('-')[:-1]) + start_time = datetime.datetime.strptime(f.split('-')[-1].split('.')[0], '%Y%m%d%H%M%S') + + ffprobe_cmd = " ".join([ + 'ffprobe', + '-v', + 'error', + '-show_entries', + 'format=duration', + '-of', + 'default=noprint_wrappers=1:nokey=1', + f"{os.path.join(self.record_dir,f)}" + ]) + p = sp.Popen(ffprobe_cmd, stdout=sp.PIPE, shell=True) + (output, err) = p.communicate() + p_status = p.wait() + if p_status == 0: + duration = float(output.decode('utf-8').strip()) + else: + logger.info(f"bad file: {f}") + os.remove(os.path.join(self.record_dir,f)) + continue + + directory = os.path.join(self.record_dir, start_time.strftime('%Y-%m/%d/%H'), camera) + + if not os.path.exists(directory): + os.makedirs(directory) + + file_name = f"{start_time.strftime('%M.%S.mp4')}" + + os.rename(os.path.join(self.record_dir,f), os.path.join(directory,file_name)) + + def expire_files(self): + delete_before = {} + for name, camera in self.config.cameras.items(): + delete_before[name] = datetime.datetime.now().timestamp() - SECONDS_IN_DAY*camera.record.retain_days + + for p in Path('/media/frigate/recordings').rglob("*.mp4"): + if not p.parent in delete_before: + continue + if p.stat().st_mtime < delete_before[p.parent]: + p.unlink(missing_ok=True) + + def run(self): + counter = 0 + self.expire_files() + while(True): + if self.stop_event.is_set(): + logger.info(f"Exiting recording maintenance...") + break + + # only expire events every 10 minutes, but check for new files every 10 seconds + time.sleep(10) + counter = counter + 1 + if counter < 60: + self.expire_files() + counter = 0 + + self.move_files() + + + \ No newline at end of file