From 63e14a98f97693141c6c2f2ce4db881c6e03ac29 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Wed, 13 Jan 2021 06:49:05 -0600 Subject: [PATCH] add retention settings for snapshots --- frigate/config.py | 30 ++++--- frigate/events.py | 155 ++++++++++++++++++++---------------- frigate/test/test_config.py | 2 +- 3 files changed, 109 insertions(+), 78 deletions(-) diff --git a/frigate/config.py b/frigate/config.py index 3d86f1a63..c890e2ff2 100644 --- a/frigate/config.py +++ b/frigate/config.py @@ -43,7 +43,7 @@ MQTT_SCHEMA = vol.Schema( } ) -CLIPS_RETAIN_SCHEMA = vol.Schema( +RETAIN_SCHEMA = vol.Schema( { vol.Required('default',default=10): int, 'objects': { @@ -56,7 +56,7 @@ CLIPS_SCHEMA = vol.Schema( { vol.Optional('max_seconds', default=300): int, 'tmpfs_cache_size': str, - vol.Optional('retain', default={}): CLIPS_RETAIN_SCHEMA + vol.Optional('retain', default={}): RETAIN_SCHEMA } ) @@ -183,7 +183,7 @@ CAMERAS_SCHEMA = vol.Schema(vol.All( vol.Optional('pre_capture', default=5): int, vol.Optional('post_capture', default=5): int, 'objects': [str], - vol.Optional('retain', default={}): CLIPS_RETAIN_SCHEMA, + vol.Optional('retain', default={}): RETAIN_SCHEMA, }, vol.Optional('record', default={}): { 'enabled': bool, @@ -197,7 +197,8 @@ CAMERAS_SCHEMA = vol.Schema(vol.All( vol.Optional('timestamp', default=False): bool, vol.Optional('bounding_box', default=False): bool, vol.Optional('crop', default=False): bool, - 'height': int + 'height': int, + vol.Optional('retain', default={}): RETAIN_SCHEMA, }, vol.Optional('mqtt', default={}): { vol.Optional('enabled', default=True): bool, @@ -228,6 +229,9 @@ FRIGATE_CONFIG_SCHEMA = vol.Schema( vol.Optional('default', default='info'): vol.In(['info', 'debug', 'warning', 'error', 'critical']), vol.Optional('logs', default={}): {str: vol.In(['info', 'debug', 'warning', 'error', 'critical']) } }, + vol.Optional('snapshots', default={}): { + vol.Optional('retain', default={}): RETAIN_SCHEMA + }, vol.Optional('clips', default={}): CLIPS_SCHEMA, vol.Optional('record', default={}): { vol.Optional('enabled', default=False): bool, @@ -406,7 +410,7 @@ class CameraFfmpegConfig(): def output_args(self): return {k: v if isinstance(v, list) else v.split(' ') for k, v in self._output_args.items()} -class ClipsRetainConfig(): +class RetainConfig(): def __init__(self, global_config, config): self._default = config.get('default', global_config.get('default')) self._objects = config.get('objects', global_config.get('objects', {})) @@ -429,7 +433,7 @@ class ClipsConfig(): def __init__(self, config): self._max_seconds = config['max_seconds'] self._tmpfs_cache_size = config.get('tmpfs_cache_size', '').strip() - self._retain = ClipsRetainConfig(config['retain'], config['retain']) + self._retain = RetainConfig(config['retain'], config['retain']) @property def max_seconds(self): @@ -523,12 +527,13 @@ class ObjectConfig(): } class CameraSnapshotsConfig(): - def __init__(self, config): + def __init__(self, global_config, config): self._enabled = config['enabled'] self._timestamp = config['timestamp'] self._bounding_box = config['bounding_box'] self._crop = config['crop'] self._height = config.get('height') + self._retain = RetainConfig(global_config['snapshots']['retain'], config['retain']) @property def enabled(self): @@ -550,13 +555,18 @@ class CameraSnapshotsConfig(): def height(self): return self._height + @property + def retain(self): + return self._retain + def to_dict(self): return { 'enabled': self.enabled, 'timestamp': self.timestamp, 'bounding_box': self.bounding_box, 'crop': self.crop, - 'height': self.height + 'height': self.height, + 'retain': self.retain.to_dict() } class CameraMqttConfig(): @@ -602,7 +612,7 @@ class CameraClipsConfig(): self._pre_capture = config['pre_capture'] self._post_capture = config['post_capture'] self._objects = config.get('objects', global_config['objects']['track']) - self._retain = ClipsRetainConfig(global_config['clips']['retain'], config['retain']) + self._retain = RetainConfig(global_config['clips']['retain'], config['retain']) @property def enabled(self): @@ -758,7 +768,7 @@ class CameraConfig(): self._clips = CameraClipsConfig(global_config, config['clips']) self._record = RecordConfig(global_config['record'], config['record']) self._rtmp = CameraRtmpConfig(global_config, config['rtmp']) - self._snapshots = CameraSnapshotsConfig(config['snapshots']) + self._snapshots = CameraSnapshotsConfig(global_config, config['snapshots']) self._mqtt = CameraMqttConfig(config['mqtt']) self._objects = ObjectConfig(global_config['objects'], config.get('objects', {})) self._motion = MotionConfig(global_config['motion'], config['motion'], self._height) diff --git a/frigate/events.py b/frigate/events.py index 7f6821f48..d96256ff2 100644 --- a/frigate/events.py +++ b/frigate/events.py @@ -200,7 +200,86 @@ class EventCleanup(threading.Thread): self.name = 'event_cleanup' self.config = config self.stop_event = stop_event + self.camera_keys = list(self.config.cameras.keys()) + def expire(self, media): + ## Expire events from unlisted cameras based on the global config + if media == 'clips': + retain_config = self.config.clips.retain + file_extension = 'mp4' + update_params = {'has_clip': False} + else: + retain_config = self.config.snapshots.retain + file_extension = 'jpg' + update_params = {'has_snapshot': False} + + distinct_labels = (Event.select(Event.label) + .where(Event.camera.not_in(self.camera_keys)) + .distinct()) + + # loop over object types in db + for l in distinct_labels: + # get expiration time for this label + expire_days = retain_config.objects.get(l.label, retain_config.default) + expire_after = (datetime.datetime.now() - datetime.timedelta(days=expire_days)).timestamp() + # grab all events after specific time + expired_events = ( + Event.select() + .where(Event.camera.not_in(self.camera_keys), + Event.start_time < expire_after, + Event.label == l.label) + ) + # delete the media from disk + for event in expired_events: + media_name = f"{event.camera}-{event.id}" + media = Path(f"{os.path.join(CLIPS_DIR, media_name)}.{file_extension}") + media.unlink(missing_ok=True) + # update the clips attribute for the db entry + update_query = ( + Event.update(update_params) + .where(Event.camera.not_in(self.camera_keys), + Event.start_time < expire_after, + Event.label == l.label) + ) + update_query.execute() + + ## Expire events from cameras based on the camera config + for name, camera in self.config.cameras.items(): + if media == 'clips': + retain_config = camera.clips.retain + else: + retain_config = camera.snapshots.retain + # get distinct objects in database for this camera + distinct_labels = (Event.select(Event.label) + .where(Event.camera == name) + .distinct()) + + # loop over object types in db + for l in distinct_labels: + # get expiration time for this label + expire_days = retain_config.objects.get(l.label, retain_config.default) + expire_after = (datetime.datetime.now() - datetime.timedelta(days=expire_days)).timestamp() + # grab all events after specific time + expired_events = ( + Event.select() + .where(Event.camera == name, + Event.start_time < expire_after, + Event.label == l.label) + ) + # delete the grabbed clips from disk + for event in expired_events: + media_name = f"{event.camera}-{event.id}" + media = Path(f"{os.path.join(CLIPS_DIR, media_name)}.{file_extension}") + media.unlink(missing_ok=True) + # update the clips attribute for the db entry + update_query = ( + Event.update(update_params) + .where( Event.camera == name, + Event.start_time < expire_after, + Event.label == l.label) + ) + update_query.execute() + def run(self): counter = 0 while(True): @@ -215,71 +294,13 @@ class EventCleanup(threading.Thread): continue counter = 0 - camera_keys = list(self.config.cameras.keys()) + self.expire('clips') + self.expire('snapshots') - # Expire events from unlisted cameras based on the global config - retain_config = self.config.clips.retain - - distinct_labels = (Event.select(Event.label) - .where(Event.camera.not_in(camera_keys)) - .distinct()) - - # loop over object types in db - for l in distinct_labels: - # get expiration time for this label - expire_days = retain_config.objects.get(l.label, retain_config.default) - expire_after = (datetime.datetime.now() - datetime.timedelta(days=expire_days)).timestamp() - # grab all events after specific time - expired_events = ( - Event.select() - .where(Event.camera.not_in(camera_keys), - Event.start_time < expire_after, - Event.label == l.label) - ) - # delete the grabbed clips from disk - for event in expired_events: - clip_name = f"{event.camera}-{event.id}" - clip = Path(f"{os.path.join(CLIPS_DIR, clip_name)}.mp4") - clip.unlink(missing_ok=True) - # delete the event for this type from the db - delete_query = ( - Event.delete() - .where(Event.camera.not_in(camera_keys), - Event.start_time < expire_after, - Event.label == l.label) - ) - delete_query.execute() - - # Expire events from cameras based on the camera config - for name, camera in self.config.cameras.items(): - retain_config = camera.clips.retain - # get distinct objects in database for this camera - distinct_labels = (Event.select(Event.label) - .where(Event.camera == name) - .distinct()) - - # loop over object types in db - for l in distinct_labels: - # get expiration time for this label - expire_days = retain_config.objects.get(l.label, retain_config.default) - expire_after = (datetime.datetime.now() - datetime.timedelta(days=expire_days)).timestamp() - # grab all events after specific time - expired_events = ( - Event.select() - .where(Event.camera == name, - Event.start_time < expire_after, - Event.label == l.label) - ) - # delete the grabbed clips from disk - for event in expired_events: - clip_name = f"{event.camera}-{event.id}" - clip = Path(f"{os.path.join(CLIPS_DIR, clip_name)}.mp4") - clip.unlink(missing_ok=True) - # delete the event for this type from the db - delete_query = ( - Event.delete() - .where( Event.camera == name, - Event.start_time < expire_after, - Event.label == l.label) - ) - delete_query.execute() + # drop events from db where has_clip and has_snapshot are false + delete_query = ( + Event.delete() + .where( Event.has_clip == False, + Event.has_snapshot == False) + ) + delete_query.execute() diff --git a/frigate/test/test_config.py b/frigate/test/test_config.py index c73f9b763..dac583c1c 100644 --- a/frigate/test/test_config.py +++ b/frigate/test/test_config.py @@ -325,7 +325,7 @@ class TestConfig(TestCase): 'ffmpeg': { 'inputs': [ { 'path': 'rtsp://10.0.0.1:554/video', 'roles': ['detect', 'rtmp'] }, - { 'path': 'rtsp://10.0.0.1:554/clips', 'roles': ['clips'] } + { 'path': 'rtsp://10.0.0.1:554/record', 'roles': ['record'] } ] }, 'height': 1080,