From 598d3aeda2b2a63ddb2b931975bcb6c2dc6e0728 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Tue, 1 Dec 2020 07:22:23 -0600 Subject: [PATCH] make directories constants --- frigate/app.py | 9 +++++++- frigate/config.py | 45 ++++++++---------------------------- frigate/const.py | 3 +++ frigate/events.py | 22 ++++++++---------- frigate/object_processing.py | 3 ++- frigate/record.py | 20 +++++++--------- 6 files changed, 40 insertions(+), 62 deletions(-) create mode 100644 frigate/const.py diff --git a/frigate/app.py b/frigate/app.py index a74fa2b3d..a773b9067 100644 --- a/frigate/app.py +++ b/frigate/app.py @@ -9,6 +9,7 @@ import yaml from playhouse.sqlite_ext import SqliteExtDatabase from frigate.config import FrigateConfig +from frigate.const import RECORD_DIR, CLIPS_DIR, CACHE_DIR from frigate.edgetpu import EdgeTPUProcess from frigate.events import EventProcessor, EventCleanup from frigate.http import create_app @@ -32,6 +33,11 @@ class FrigateApp(): self.detection_shms: List[mp.shared_memory.SharedMemory] = [] self.log_queue = mp.Queue() self.camera_metrics = {} + + def ensure_dirs(self): + for d in [RECORD_DIR, CLIPS_DIR, CACHE_DIR]: + if not os.path.exists(d) and not os.path.islink(d): + os.makedirs(d) def init_logger(self): self.log_process = mp.Process(target=log_process, args=(self.log_queue,), name='log_process') @@ -64,7 +70,7 @@ class FrigateApp(): self.detected_frames_queue = mp.Queue(maxsize=len(self.config.cameras.keys())*2) def init_database(self): - self.db = SqliteExtDatabase(f"/{os.path.join(self.config.save_clips.clips_dir, 'frigate.db')}") + self.db = SqliteExtDatabase(f"/{os.path.join(CLIPS_DIR, 'frigate.db')}") models = [Event] self.db.bind(models) self.db.create_tables(models, safe=True) @@ -131,6 +137,7 @@ class FrigateApp(): def start(self): self.init_logger() + self.ensure_dirs() # TODO: exit if config doesnt parse self.init_config() self.init_queues() diff --git a/frigate/config.py b/frigate/config.py index 265eacde1..56cdffa43 100644 --- a/frigate/config.py +++ b/frigate/config.py @@ -9,6 +9,8 @@ import numpy as np import voluptuous as vol import yaml +from frigate.const import RECORD_DIR, CLIPS_DIR, CACHE_DIR + DETECTORS_SCHEMA = vol.Schema( { vol.Required(str): { @@ -48,8 +50,6 @@ SAVE_CLIPS_RETAIN_SCHEMA = vol.Schema( SAVE_CLIPS_SCHEMA = vol.Schema( { vol.Optional('max_seconds', default=300): int, - vol.Optional('clips_dir', default='/media/frigate/clips'): str, - vol.Optional('cache_dir', default='/tmp/cache'): str, vol.Optional('retain', default={}): SAVE_CLIPS_RETAIN_SCHEMA } ) @@ -198,7 +198,6 @@ FRIGATE_CONFIG_SCHEMA = vol.Schema( vol.Optional('record', default={}): { vol.Optional('enabled', default=False): bool, vol.Optional('retain_days', default=30): int, - vol.Optional('record_dir', default='/media/frigate/recordings'): str }, vol.Optional('ffmpeg', default={}): GLOBAL_FFMPEG_SCHEMA, vol.Optional('objects', default={}): OBJECTS_SCHEMA, @@ -331,22 +330,12 @@ class SaveClipsRetainConfig(): class SaveClipsConfig(): def __init__(self, config): self._max_seconds = config['max_seconds'] - self._clips_dir = config['clips_dir'] - self._cache_dir = config['cache_dir'] self._retain = SaveClipsRetainConfig(config['retain'], config['retain']) @property def max_seconds(self): return self._max_seconds - - @property - def clips_dir(self): - return self._clips_dir - - @property - def cache_dir(self): - return self._cache_dir - + @property def retain(self): return self._retain @@ -354,8 +343,6 @@ class SaveClipsConfig(): def to_dict(self): return { 'max_seconds': self.max_seconds, - 'clips_dir': self.clips_dir, - 'cache_dir': self.cache_dir, 'retain': self.retain.to_dict() } @@ -363,7 +350,6 @@ class RecordConfig(): def __init__(self, global_config, config): self._enabled = config.get('enabled', global_config['enabled']) self._retain_days = config.get('retain_days', global_config['retain_days']) - self._record_dir = global_config['record_dir'] @property def enabled(self): @@ -373,15 +359,10 @@ class RecordConfig(): def retain_days(self): return self._retain_days - @property - def record_dir(self): - return self._record_dir - def to_dict(self): return { 'enabled': self.enabled, 'retain_days': self.retain_days, - 'record_dir': self.record_dir, } class FilterConfig(): @@ -564,7 +545,7 @@ class ZoneConfig(): } class CameraConfig(): - def __init__(self, name, config, cache_dir, global_config): + def __init__(self, name, config, global_config): self._name = name self._ffmpeg = CameraFfmpegConfig(global_config['ffmpeg'], config['ffmpeg']) self._height = config.get('height') @@ -585,7 +566,7 @@ class CameraConfig(): for ffmpeg_input in self._ffmpeg.inputs: self._ffmpeg_cmds.append({ 'roles': ffmpeg_input.roles, - 'cmd': self._get_ffmpeg_cmd(ffmpeg_input, cache_dir) + 'cmd': self._get_ffmpeg_cmd(ffmpeg_input) }) @@ -614,7 +595,7 @@ class CameraConfig(): return mask_img - def _get_ffmpeg_cmd(self, ffmpeg_input, cache_dir): + def _get_ffmpeg_cmd(self, ffmpeg_input): ffmpeg_output_args = [] if 'detect' in ffmpeg_input.roles: ffmpeg_output_args = self.ffmpeg.output_args['detect'] + ffmpeg_output_args + ['pipe:'] @@ -626,11 +607,11 @@ class CameraConfig(): ] + ffmpeg_output_args if 'clips' in ffmpeg_input.roles and self.save_clips.enabled: ffmpeg_output_args = self.ffmpeg.output_args['clips'] + [ - f"{os.path.join(cache_dir, self.name)}-%Y%m%d%H%M%S.mp4" + f"{os.path.join(CACHE_DIR, self.name)}-%Y%m%d%H%M%S.mp4" ] + ffmpeg_output_args if 'record' in ffmpeg_input.roles and self.record.enabled: ffmpeg_output_args = self.ffmpeg.output_args['record'] + [ - f"{os.path.join(self.record.record_dir, self.name)}-%Y%m%d%H%M%S.mp4" + f"{os.path.join(RECORD_DIR, self.name)}-%Y%m%d%H%M%S.mp4" ] + ffmpeg_output_args return (['ffmpeg'] + ffmpeg_input.global_args + @@ -746,9 +727,7 @@ class FrigateConfig(): self._detectors = { name: DetectorConfig(d) for name, d in config['detectors'].items() } self._mqtt = MqttConfig(config['mqtt']) self._save_clips = SaveClipsConfig(config['save_clips']) - self._cameras = { name: CameraConfig(name, c, self._save_clips.cache_dir, config) for name, c in config['cameras'].items() } - - self._ensure_dirs() + self._cameras = { name: CameraConfig(name, c, config) for name, c in config['cameras'].items() } def _sub_env_vars(self, config): frigate_env_vars = {k: v for k, v in os.environ.items() if k.startswith('FRIGATE_')} @@ -761,12 +740,6 @@ class FrigateConfig(): i['path'] = i['path'].format(**frigate_env_vars) return config - - def _ensure_dirs(self): - record_dirs = list(set([camera.record.record_dir for camera in self.cameras.values()])) - for d in [self.save_clips.cache_dir, self.save_clips.clips_dir] + record_dirs: - if not os.path.exists(d) and not os.path.islink(d): - os.makedirs(d) def _load_file(self, config_file): with open(config_file) as f: diff --git a/frigate/const.py b/frigate/const.py new file mode 100644 index 000000000..2ea9f9f68 --- /dev/null +++ b/frigate/const.py @@ -0,0 +1,3 @@ +CLIPS_DIR = '/media/frigate/clips' +RECORD_DIR = '/media/frigate/recordings' +CACHE_DIR = '/tmp/cache' \ No newline at end of file diff --git a/frigate/events.py b/frigate/events.py index 44eeded1c..91429c9d4 100644 --- a/frigate/events.py +++ b/frigate/events.py @@ -12,6 +12,7 @@ from pathlib import Path import psutil from frigate.config import FrigateConfig +from frigate.const import RECORD_DIR, CLIPS_DIR, CACHE_DIR from frigate.models import Event from peewee import fn @@ -23,8 +24,6 @@ class EventProcessor(threading.Thread): threading.Thread.__init__(self) self.name = 'event_processor' self.config = config - self.cache_dir = self.config.save_clips.cache_dir - self.clips_dir = self.config.save_clips.clips_dir self.camera_processes = camera_processes self.cached_clips = {} self.event_queue = event_queue @@ -33,7 +32,7 @@ class EventProcessor(threading.Thread): self.stop_event = stop_event def refresh_cache(self): - cached_files = os.listdir(self.cache_dir) + cached_files = os.listdir(CACHE_DIR) files_in_use = [] for process in psutil.process_iter(): @@ -43,7 +42,7 @@ class EventProcessor(threading.Thread): flist = process.open_files() if flist: for nt in flist: - if nt.path.startswith(self.cache_dir): + if nt.path.startswith(CACHE_DIR): files_in_use.append(nt.path.split('/')[-1]) except: continue @@ -63,7 +62,7 @@ class EventProcessor(threading.Thread): 'format=duration', '-of', 'default=noprint_wrappers=1:nokey=1', - f"{os.path.join(self.cache_dir,f)}" + f"{os.path.join(CACHE_DIR,f)}" ]) p = sp.Popen(ffprobe_cmd, stdout=sp.PIPE, shell=True) (output, err) = p.communicate() @@ -72,7 +71,7 @@ class EventProcessor(threading.Thread): duration = float(output.decode('utf-8').strip()) else: logger.info(f"bad file: {f}") - os.remove(os.path.join(self.cache_dir,f)) + os.remove(os.path.join(CACHE_DIR,f)) continue self.cached_clips[f] = { @@ -95,7 +94,7 @@ class EventProcessor(threading.Thread): for f, data in list(self.cached_clips.items()): if earliest_event-90 > data['start_time']+data['duration']: del self.cached_clips[f] - os.remove(os.path.join(self.cache_dir,f)) + os.remove(os.path.join(CACHE_DIR,f)) def create_clip(self, camera, event_data, pre_capture): # get all clips from the camera with the event sorted @@ -117,7 +116,7 @@ class EventProcessor(threading.Thread): # clip starts after playlist ends, finish if clip['start_time'] > playlist_end: break - playlist_lines.append(f"file '{os.path.join(self.cache_dir,clip['path'])}'") + playlist_lines.append(f"file '{os.path.join(CACHE_DIR,clip['path'])}'") # if this is the starting clip, add an inpoint if clip['start_time'] < playlist_start: playlist_lines.append(f"inpoint {int(playlist_start-clip['start_time'])}") @@ -139,7 +138,7 @@ class EventProcessor(threading.Thread): '-', '-c', 'copy', - f"{os.path.join(self.clips_dir, clip_name)}.mp4" + f"{os.path.join(CLIPS_DIR, clip_name)}.mp4" ] p = sp.run(ffmpeg_cmd, input="\n".join(playlist_lines), encoding='ascii', capture_output=True) @@ -203,7 +202,6 @@ class EventCleanup(threading.Thread): threading.Thread.__init__(self) self.name = 'event_cleanup' self.config = config - self.clips_dir = self.config.save_clips.clips_dir self.stop_event = stop_event def run(self): @@ -244,7 +242,7 @@ class EventCleanup(threading.Thread): # delete the grabbed clips from disk for event in expired_events: clip_name = f"{event.camera}-{event.id}" - clip = Path(f"{os.path.join(self.clips_dir, clip_name)}.mp4") + clip = Path(f"{os.path.join(CLIPS_DIR, clip_name)}.mp4") clip.unlink(missing_ok=True) # delete the event for this type from the db delete_query = ( @@ -278,7 +276,7 @@ class EventCleanup(threading.Thread): # delete the grabbed clips from disk for event in expired_events: clip_name = f"{event.camera}-{event.id}" - clip = Path(f"{os.path.join(self.clips_dir, clip_name)}.mp4") + clip = Path(f"{os.path.join(CLIPS_DIR, clip_name)}.mp4") clip.unlink(missing_ok=True) # delete the event for this type from the db delete_query = ( diff --git a/frigate/object_processing.py b/frigate/object_processing.py index 871198bd3..f37109cb1 100644 --- a/frigate/object_processing.py +++ b/frigate/object_processing.py @@ -18,6 +18,7 @@ import matplotlib.pyplot as plt import numpy as np from frigate.config import FrigateConfig, CameraConfig +from frigate.const import RECORD_DIR, CLIPS_DIR, CACHE_DIR from frigate.edgetpu import load_labels from frigate.util import SharedMemoryFrameManager, draw_box_with_label @@ -418,7 +419,7 @@ class TrackedObjectProcessor(threading.Thread): self.client.publish(f"{self.topic_prefix}/events", json.dumps(message), retain=False) if self.config.cameras[camera].save_clips.enabled and not obj.false_positive: thumbnail_file_name = f"{camera}-{obj.obj_data['id']}.jpg" - with open(os.path.join(self.config.save_clips.clips_dir, thumbnail_file_name), 'wb') as f: + with open(os.path.join(CLIPS_DIR, thumbnail_file_name), 'wb') as f: f.write(obj.get_jpg_bytes()) self.event_queue.put(('end', camera, obj.to_dict(include_thumbnail=True))) diff --git a/frigate/record.py b/frigate/record.py index 6dc5db480..f7ecb2032 100644 --- a/frigate/record.py +++ b/frigate/record.py @@ -12,6 +12,7 @@ from pathlib import Path import psutil from frigate.config import FrigateConfig +from frigate.const import RECORD_DIR, CLIPS_DIR, CACHE_DIR logger = logging.getLogger(__name__) @@ -34,15 +35,10 @@ class RecordingMaintainer(threading.Thread): threading.Thread.__init__(self) self.name = 'recording_maint' self.config = config - record_dirs = list(set([camera.record.record_dir for camera in self.config.cameras.values()])) - self.record_dir = None if len(record_dirs) == 0 else record_dirs[0] self.stop_event = stop_event def move_files(self): - if self.record_dir is None: - return - - recordings = [d for d in os.listdir(self.record_dir) if os.path.isfile(os.path.join(self.record_dir, d)) and d.endswith(".mp4")] + recordings = [d for d in os.listdir(RECORD_DIR) if os.path.isfile(os.path.join(RECORD_DIR, d)) and d.endswith(".mp4")] files_in_use = [] for process in psutil.process_iter(): @@ -52,7 +48,7 @@ class RecordingMaintainer(threading.Thread): flist = process.open_files() if flist: for nt in flist: - if nt.path.startswith(self.record_dir): + if nt.path.startswith(RECORD_DIR): files_in_use.append(nt.path.split('/')[-1]) except: continue @@ -72,7 +68,7 @@ class RecordingMaintainer(threading.Thread): 'format=duration', '-of', 'default=noprint_wrappers=1:nokey=1', - f"{os.path.join(self.record_dir,f)}" + f"{os.path.join(RECORD_DIR,f)}" ]) p = sp.Popen(ffprobe_cmd, stdout=sp.PIPE, shell=True) (output, err) = p.communicate() @@ -81,17 +77,17 @@ class RecordingMaintainer(threading.Thread): duration = float(output.decode('utf-8').strip()) else: logger.info(f"bad file: {f}") - os.remove(os.path.join(self.record_dir,f)) + os.remove(os.path.join(RECORD_DIR,f)) continue - directory = os.path.join(self.record_dir, start_time.strftime('%Y-%m/%d/%H'), camera) + directory = os.path.join(RECORD_DIR, start_time.strftime('%Y-%m/%d/%H'), camera) if not os.path.exists(directory): os.makedirs(directory) file_name = f"{start_time.strftime('%M.%S.mp4')}" - os.rename(os.path.join(self.record_dir,f), os.path.join(directory,file_name)) + os.rename(os.path.join(RECORD_DIR,f), os.path.join(directory,file_name)) def expire_files(self): delete_before = {} @@ -117,7 +113,7 @@ class RecordingMaintainer(threading.Thread): counter = counter + 1 if counter > 60: self.expire_files() - remove_empty_directories(self.record_dir) + remove_empty_directories(RECORD_DIR) counter = 0 self.move_files()