From a476bc9885540b7e33fac32df62b6ecf4c9c7339 Mon Sep 17 00:00:00 2001 From: Jason Hunter Date: Fri, 9 Jul 2021 16:14:16 -0400 Subject: [PATCH] initial commit --- frigate/config.py | 101 +++++++-------- frigate/events.py | 250 ++++++++---------------------------- frigate/http.py | 157 ++++++++++++++++++++-- frigate/record.py | 165 ++++++++++++++++++++++-- frigate/test/test_config.py | 1 - web/src/routes/Event.jsx | 6 +- 6 files changed, 402 insertions(+), 278 deletions(-) diff --git a/frigate/config.py b/frigate/config.py index 695ab1059..275208bfe 100644 --- a/frigate/config.py +++ b/frigate/config.py @@ -69,13 +69,32 @@ class RetainConfig(BaseModel): ) +# DEPRECATED: Will eventually be removed class ClipsConfig(BaseModel): + enabled: bool = Field(default=False, title="Save clips.") max_seconds: int = Field(default=300, title="Maximum clip duration.") + pre_capture: int = Field(default=5, title="Seconds to capture before event starts.") + post_capture: int = Field(default=5, title="Seconds to capture after event ends.") + required_zones: List[str] = Field( + default_factory=list, + title="List of required zones to be entered in order to save the clip.", + ) + objects: Optional[List[str]] = Field( + title="List of objects to be detected in order to save the clip.", + ) retain: RetainConfig = Field( default_factory=RetainConfig, title="Clip retention settings." ) +class RecordConfig(BaseModel): + enabled: bool = Field(default=False, title="Enable record on all cameras.") + retain_days: int = Field(default=0, title="Recording retention period in days.") + events: ClipsConfig = Field( + default_factory=ClipsConfig, title="Event specific settings." + ) + + class MotionConfig(BaseModel): threshold: int = Field( default=25, @@ -264,26 +283,11 @@ FFMPEG_INPUT_ARGS_DEFAULT = [ ] DETECT_FFMPEG_OUTPUT_ARGS_DEFAULT = ["-f", "rawvideo", "-pix_fmt", "yuv420p"] RTMP_FFMPEG_OUTPUT_ARGS_DEFAULT = ["-c", "copy", "-f", "flv"] -SAVE_CLIPS_FFMPEG_OUTPUT_ARGS_DEFAULT = [ - "-f", - "segment", - "-segment_time", - "10", - "-segment_format", - "mp4", - "-reset_timestamps", - "1", - "-strftime", - "1", - "-c", - "copy", - "-an", -] RECORD_FFMPEG_OUTPUT_ARGS_DEFAULT = [ "-f", "segment", "-segment_time", - "60", + "10", "-segment_format", "mp4", "-reset_timestamps", @@ -305,10 +309,6 @@ class FfmpegOutputArgsConfig(BaseModel): default=RECORD_FFMPEG_OUTPUT_ARGS_DEFAULT, title="Record role FFmpeg output arguments.", ) - clips: Union[str, List[str]] = Field( - default=SAVE_CLIPS_FFMPEG_OUTPUT_ARGS_DEFAULT, - title="Clips role FFmpeg output arguments.", - ) rtmp: Union[str, List[str]] = Field( default=RTMP_FFMPEG_OUTPUT_ARGS_DEFAULT, title="RTMP role FFmpeg output arguments.", @@ -423,20 +423,6 @@ class CameraMqttConfig(BaseModel): ) -class CameraClipsConfig(BaseModel): - enabled: bool = Field(default=False, title="Save clips.") - pre_capture: int = Field(default=5, title="Seconds to capture before event starts.") - post_capture: int = Field(default=5, title="Seconds to capture after event ends.") - required_zones: List[str] = Field( - default_factory=list, - title="List of required zones to be entered in order to save the clip.", - ) - objects: Optional[List[str]] = Field( - title="List of objects to be detected in order to save the clip.", - ) - retain: RetainConfig = Field(default_factory=RetainConfig, title="Clip retention.") - - class CameraRtmpConfig(BaseModel): enabled: bool = Field(default=True, title="RTMP restreaming enabled.") @@ -446,11 +432,6 @@ class CameraLiveConfig(BaseModel): quality: int = Field(default=8, ge=1, le=31, title="Live camera view quality") -class RecordConfig(BaseModel): - enabled: bool = Field(default=False, title="Enable record on all cameras.") - retain_days: int = Field(default=30, title="Recording retention period in days.") - - class CameraConfig(BaseModel): name: Optional[str] = Field(title="Camera name.") ffmpeg: CameraFfmpegConfig = Field(title="FFmpeg configuration for the camera.") @@ -466,9 +447,7 @@ class CameraConfig(BaseModel): zones: Dict[str, ZoneConfig] = Field( default_factory=dict, title="Zone configuration." ) - clips: CameraClipsConfig = Field( - default_factory=CameraClipsConfig, title="Clip configuration." - ) + clips: ClipsConfig = Field(default_factory=ClipsConfig, title="Clip configuration.") record: RecordConfig = Field( default_factory=RecordConfig, title="Record configuration." ) @@ -541,18 +520,9 @@ class CameraConfig(BaseModel): ffmpeg_output_args = ( rtmp_args + [f"rtmp://127.0.0.1/live/{self.name}"] + ffmpeg_output_args ) - if "clips" in ffmpeg_input.roles: - clips_args = ( - self.ffmpeg.output_args.clips - if isinstance(self.ffmpeg.output_args.clips, list) - else self.ffmpeg.output_args.clips.split(" ") - ) - ffmpeg_output_args = ( - clips_args - + [f"{os.path.join(CACHE_DIR, self.name)}-%Y%m%d%H%M%S.mp4"] - + ffmpeg_output_args - ) - if "record" in ffmpeg_input.roles and self.record.enabled: + if any(role in ["clips", "record"] for role in ffmpeg_input.roles) and ( + self.record.enabled or self.clips.enabled + ): record_args = ( self.ffmpeg.output_args.record if isinstance(self.ffmpeg.output_args.record, list) @@ -560,7 +530,7 @@ class CameraConfig(BaseModel): ) ffmpeg_output_args = ( record_args - + [f"{os.path.join(RECORD_DIR, self.name)}-%Y%m%d%H%M%S.mp4"] + + [f"{os.path.join(CACHE_DIR, self.name)}-%Y%m%d%H%M%S.mp4"] + ffmpeg_output_args ) @@ -700,7 +670,7 @@ class FrigateConfig(BaseModel): # Global config to propegate down to camera level global_config = config.dict( include={ - "clips": {"retain"}, + "clips": ..., "record": ..., "snapshots": ..., "objects": ..., @@ -713,7 +683,9 @@ class FrigateConfig(BaseModel): for name, camera in config.cameras.items(): merged_config = deep_merge(camera.dict(exclude_unset=True), global_config) - camera_config = CameraConfig.parse_obj({"name": name, **merged_config}) + camera_config: CameraConfig = CameraConfig.parse_obj( + {"name": name, **merged_config} + ) # FFMPEG input substitution for input in camera_config.ffmpeg.inputs: @@ -776,6 +748,21 @@ class FrigateConfig(BaseModel): config.cameras[name] = camera_config + # Merge Clips configuration for backward compatibility + if camera_config.clips.enabled: + logger.warn( + "Clips configuration is deprecated. Configure clip settings under record -> events." + ) + if not camera_config.record.enabled: + camera_config.record.enabled = True + camera_config.record.retain_days = 0 + camera_config.record.events = ClipsConfig.parse_obj( + deep_merge( + camera_config.clips.dict(exclude_unset=True), + camera_config.record.events.dict(exclude_unset=True), + ) + ) + return config @validator("cameras") diff --git a/frigate/events.py b/frigate/events.py index b8ff6cfd7..f0c5d13a5 100644 --- a/frigate/events.py +++ b/frigate/events.py @@ -1,20 +1,14 @@ import datetime -import json import logging import os import queue -import subprocess as sp import threading import time -from collections import defaultdict from pathlib import Path -import psutil -import shutil - -from frigate.config import FrigateConfig -from frigate.const import RECORD_DIR, CLIPS_DIR, CACHE_DIR -from frigate.models import Event +from frigate.config import FrigateConfig, RecordConfig +from frigate.const import CLIPS_DIR +from frigate.models import Event, Recordings from peewee import fn @@ -39,8 +33,16 @@ class EventProcessor(threading.Thread): if event_data["false_positive"]: return False - # if there are required zones and there is no overlap - required_zones = self.config.cameras[camera].clips.required_zones + record_config: RecordConfig = self.config.cameras[camera].record + + # Recording clips is disabled + if not record_config.enabled or ( + record_config.retain_days == 0 and not record_config.events.enabled + ): + return False + + # If there are required zones and there is no overlap + required_zones = record_config.events.required_zones if len(required_zones) > 0 and not set(event_data["entered_zones"]) & set( required_zones ): @@ -49,208 +51,65 @@ class EventProcessor(threading.Thread): ) return False - return True - - def refresh_cache(self): - cached_files = os.listdir(CACHE_DIR) - - files_in_use = [] - for process in psutil.process_iter(): - try: - if process.name() != "ffmpeg": - continue - - flist = process.open_files() - if flist: - for nt in flist: - if nt.path.startswith(CACHE_DIR): - files_in_use.append(nt.path.split("/")[-1]) - except: - continue - - for f in cached_files: - if f in files_in_use or f in self.cached_clips: - continue - - basename = os.path.splitext(f)[0] - camera, date = basename.rsplit("-", maxsplit=1) - start_time = datetime.datetime.strptime(date, "%Y%m%d%H%M%S") - - ffprobe_cmd = [ - "ffprobe", - "-v", - "error", - "-show_entries", - "format=duration", - "-of", - "default=noprint_wrappers=1:nokey=1", - f"{os.path.join(CACHE_DIR, f)}", - ] - p = sp.run(ffprobe_cmd, capture_output=True) - if p.returncode == 0: - duration = float(p.stdout.decode().strip()) - else: - logger.info(f"bad file: {f}") - os.remove(os.path.join(CACHE_DIR, f)) - continue - - self.cached_clips[f] = { - "path": f, - "camera": camera, - "start_time": start_time.timestamp(), - "duration": duration, - } - - if len(self.events_in_process) > 0: - earliest_event = min( - self.events_in_process.values(), key=lambda x: x["start_time"] - )["start_time"] - else: - earliest_event = datetime.datetime.now().timestamp() - - # if the earliest event is more tha max seconds ago, cap it - max_seconds = self.config.clips.max_seconds - earliest_event = max( - earliest_event, - datetime.datetime.now().timestamp() - self.config.clips.max_seconds, - ) - - for f, data in list(self.cached_clips.items()): - if earliest_event - 90 > data["start_time"] + data["duration"]: - del self.cached_clips[f] - logger.debug(f"Cleaning up cached file {f}") - os.remove(os.path.join(CACHE_DIR, f)) - - # if we are still using more than 90% of the cache, proactively cleanup - cache_usage = shutil.disk_usage("/tmp/cache") - while ( - cache_usage.used / cache_usage.total > 0.9 - and cache_usage.free < 200000000 - and len(self.cached_clips) > 0 + # If the required objects are not present + if ( + record_config.events.objects is not None + and event_data["label"] not in record_config.events.objects ): - logger.warning("More than 90% of the cache is used.") - logger.warning( - "Consider increasing space available at /tmp/cache or reducing max_seconds in your clips config." + logger.debug( + f"Not creating clip for {event_data['id']} because it did not contain required objects" ) - logger.warning("Proactively cleaning up the cache...") - oldest_clip = min(self.cached_clips.values(), key=lambda x: x["start_time"]) - del self.cached_clips[oldest_clip["path"]] - os.remove(os.path.join(CACHE_DIR, oldest_clip["path"])) - cache_usage = shutil.disk_usage("/tmp/cache") - - def create_clip(self, camera, event_data, pre_capture, post_capture): - # get all clips from the camera with the event sorted - sorted_clips = sorted( - [c for c in self.cached_clips.values() if c["camera"] == camera], - key=lambda i: i["start_time"], - ) - - # if there are no clips in the cache or we are still waiting on a needed file check every 5 seconds - wait_count = 0 - while ( - len(sorted_clips) == 0 - or sorted_clips[-1]["start_time"] + sorted_clips[-1]["duration"] - < event_data["end_time"] + post_capture - ): - if wait_count > 4: - logger.warning( - f"Unable to create clip for {camera} and event {event_data['id']}. There were no cache files for this event." - ) - return False - logger.debug(f"No cache clips for {camera}. Waiting...") - time.sleep(5) - self.refresh_cache() - # get all clips from the camera with the event sorted - sorted_clips = sorted( - [c for c in self.cached_clips.values() if c["camera"] == camera], - key=lambda i: i["start_time"], - ) - wait_count += 1 - - playlist_start = event_data["start_time"] - pre_capture - playlist_end = event_data["end_time"] + post_capture - playlist_lines = [] - for clip in sorted_clips: - # clip ends before playlist start time, skip - if clip["start_time"] + clip["duration"] < playlist_start: - continue - # clip starts after playlist ends, finish - if clip["start_time"] > playlist_end: - break - playlist_lines.append(f"file '{os.path.join(CACHE_DIR,clip['path'])}'") - # if this is the starting clip, add an inpoint - if clip["start_time"] < playlist_start: - playlist_lines.append( - f"inpoint {int(playlist_start-clip['start_time'])}" - ) - # if this is the ending clip, add an outpoint - if clip["start_time"] + clip["duration"] > playlist_end: - playlist_lines.append( - f"outpoint {int(playlist_end-clip['start_time'])}" - ) - - clip_name = f"{camera}-{event_data['id']}" - ffmpeg_cmd = [ - "ffmpeg", - "-y", - "-protocol_whitelist", - "pipe,file", - "-f", - "concat", - "-safe", - "0", - "-i", - "-", - "-c", - "copy", - "-movflags", - "+faststart", - f"{os.path.join(CLIPS_DIR, clip_name)}.mp4", - ] - - p = sp.run( - ffmpeg_cmd, - input="\n".join(playlist_lines), - encoding="ascii", - capture_output=True, - ) - if p.returncode != 0: - logger.error(p.stderr) return False + return True + def verify_clip(self, camera, end_time): + # check every 5 seconds for the last required recording + for _ in range(4): + recordings_count = ( + Recordings.select() + .where(Recordings.camera == camera, Recordings.end_time > end_time) + .limit(1) + .count() + ) + if recordings_count > 0: + return True + logger.debug(f"Missing recording for {camera} clip. Waiting...") + time.sleep(5) + + logger.warning( + f"Unable to verify clip for {camera}. There were no recordings for this camera." + ) + return False + def run(self): while not self.stop_event.is_set(): try: event_type, camera, event_data = self.event_queue.get(timeout=10) except queue.Empty: - if not self.stop_event.is_set(): - self.refresh_cache() + # if not self.stop_event.is_set(): + # self.refresh_cache() continue logger.debug(f"Event received: {event_type} {camera} {event_data['id']}") - self.refresh_cache() + # self.refresh_cache() if event_type == "start": self.events_in_process[event_data["id"]] = event_data if event_type == "end": - clips_config = self.config.cameras[camera].clips + record_config: RecordConfig = self.config.cameras[camera].record - clip_created = False - if self.should_create_clip(camera, event_data): - if clips_config.enabled and ( - clips_config.objects is None - or event_data["label"] in clips_config.objects - ): - clip_created = self.create_clip( - camera, - event_data, - clips_config.pre_capture, - clips_config.post_capture, - ) + has_clip = self.should_create_clip(camera, event_data) - if clip_created or event_data["has_snapshot"]: + # Wait for recordings to be ready + if has_clip: + has_clip = self.verify_clip( + camera, + event_data["end_time"] + record_config.events.post_capture, + ) + + if has_clip or event_data["has_snapshot"]: Event.create( id=event_data["id"], label=event_data["label"], @@ -261,11 +120,12 @@ class EventProcessor(threading.Thread): false_positive=event_data["false_positive"], zones=list(event_data["entered_zones"]), thumbnail=event_data["thumbnail"], - has_clip=clip_created, + has_clip=has_clip, has_snapshot=event_data["has_snapshot"], ) + del self.events_in_process[event_data["id"]] - self.event_processed_queue.put((event_data["id"], camera, clip_created)) + self.event_processed_queue.put((event_data["id"], camera, has_clip)) logger.info(f"Exiting event processor...") diff --git a/frigate/http.py b/frigate/http.py index 55b04ce75..4c7fde091 100644 --- a/frigate/http.py +++ b/frigate/http.py @@ -6,11 +6,13 @@ import glob import logging import os import re +import subprocess as sp import time from functools import reduce from pathlib import Path import cv2 +from flask.helpers import send_file import numpy as np from flask import ( @@ -223,6 +225,32 @@ def event_snapshot(id): return response +@bp.route("/events//clip.mp4") +def event_clip(id): + event: Event = Event.get(Event.id == id) + + if event is None: + return "Event not found.", 404 + + if not event.has_clip: + return "Clip not available", 404 + + event_config = current_app.frigate_config.cameras[event.camera].record.events + start_ts = event.start_time - event_config.pre_capture + end_ts = event.end_time + event_config.post_capture + clip_path = os.path.join(CLIPS_DIR, f"{event.camera}-{id}.mp4") + + if not os.path.isfile(clip_path): + return recording_clip(event.camera, start_ts, end_ts) + + return send_file( + clip_path, + mimetype="video/mp4", + as_attachment=True, + attachment_filename=f"{event.camera}_{start_ts}-{end_ts}.mp4", + ) + + @bp.route("/events") def events(): limit = request.args.get("limit", 100) @@ -517,14 +545,84 @@ def recordings(camera_name): ) -@bp.route("/vod////") -def vod(year_month, day, hour, camera): - start_date = datetime.strptime(f"{year_month}-{day} {hour}", "%Y-%m-%d %H") - end_date = start_date + timedelta(hours=1) - timedelta(milliseconds=1) - start_ts = start_date.timestamp() - end_ts = end_date.timestamp() +@bp.route("//start//end//clip.mp4") +@bp.route("//start//end//clip.mp4") +def recording_clip(camera, start_ts, end_ts): + recordings = ( + Recordings.select() + .where( + (Recordings.start_time.between(start_ts, end_ts)) + | (Recordings.end_time.between(start_ts, end_ts)) + ) + .where(Recordings.camera == camera) + .order_by(Recordings.start_time.asc()) + ) - # Select all recordings where either the start or end dates fall in the requested hour + playlist_lines = [] + clip: Recordings + for clip in recordings: + playlist_lines.append(f"file '{clip.path}'") + # if this is the starting clip, add an inpoint + if clip.start_time < start_ts: + playlist_lines.append(f"inpoint {int(start_ts - clip.start_time)}") + # if this is the ending clip, add an outpoint + if clip.end_time > end_ts: + playlist_lines.append(f"outpoint {int(end_ts - clip.start_time)}") + + path = f"/tmp/cache/tmp_clip_{camera}_{start_ts}-{end_ts}.mp4" + + ffmpeg_cmd = [ + "ffmpeg", + "-y", + "-protocol_whitelist", + "pipe,file", + "-f", + "concat", + "-safe", + "0", + "-i", + "-", + "-c", + "copy", + "-f", + "mp4", + "-movflags", + "+faststart", + path, + ] + + p = sp.run( + ffmpeg_cmd, + input="\n".join(playlist_lines), + encoding="ascii", + capture_output=True, + ) + if p.returncode != 0: + logger.error(p.stderr) + return f"Could not create clip from recordings for {camera}.", 500 + + mp4_bytes = None + try: + # read clip from disk + with open(path, "rb") as mp4_file: + mp4_bytes = mp4_file.read() + + # delete after we have the bytes + os.remove(path) + except DoesNotExist: + return f"Could not create clip from recordings for {camera}.", 500 + + response = make_response(mp4_bytes) + response.mimetype = "video/mp4" + response.headers[ + "Content-Disposition" + ] = f"attachment; filename={camera}_{start_ts}-{end_ts}.mp4" + return response + + +@bp.route("/vod//start//end/") +@bp.route("/vod//start//end/") +def vod_ts(camera, start_ts, end_ts): recordings = ( Recordings.select() .where( @@ -553,9 +651,13 @@ def vod(year_month, day, hour, camera): clips.append(clip) durations.append(duration) + if not clips: + return "No recordings found.", 404 + + hour_ago = datetime.now() - timedelta(hours=1) return jsonify( { - "cache": datetime.now() - timedelta(hours=1) > start_date, + "cache": hour_ago.timestamp() > start_ts, "discontinuity": False, "durations": durations, "sequences": [{"clips": clips}], @@ -563,6 +665,45 @@ def vod(year_month, day, hour, camera): ) +@bp.route("/vod////") +def vod_hour(year_month, day, hour, camera): + start_date = datetime.strptime(f"{year_month}-{day} {hour}", "%Y-%m-%d %H") + end_date = start_date + timedelta(hours=1) - timedelta(milliseconds=1) + start_ts = start_date.timestamp() + end_ts = end_date.timestamp() + + return vod_ts(camera, start_ts, end_ts) + + +@bp.route("/vod/event/") +def vod_event(id): + event: Event = Event.get(Event.id == id) + + if event is None: + return "Event not found.", 404 + + if not event.has_clip: + return "Clip not available", 404 + + event_config = current_app.frigate_config.cameras[event.camera].record.events + start_ts = event.start_time - event_config.pre_capture + end_ts = event.end_time + event_config.post_capture + clip_path = os.path.join(CLIPS_DIR, f"{event.camera}-{id}.mp4") + + if not os.path.isfile(clip_path): + return vod_ts(event.camera, start_ts, end_ts) + + duration = int((end_ts - start_ts) * 1000) + return jsonify( + { + "cache": True, + "discontinuity": False, + "durations": [duration], + "sequences": [{"clips": [{"type": "source", "path": clip_path}]}], + } + ) + + def imagestream(detected_frames_processor, camera_name, fps, height, draw_options): while True: # max out at specified FPS diff --git a/frigate/record.py b/frigate/record.py index fcf493ce5..95cfe8766 100644 --- a/frigate/record.py +++ b/frigate/record.py @@ -3,6 +3,7 @@ import itertools import logging import os import random +import shutil import string import subprocess as sp import threading @@ -10,9 +11,11 @@ from pathlib import Path import psutil +from peewee import JOIN + from frigate.config import FrigateConfig -from frigate.const import RECORD_DIR -from frigate.models import Recordings +from frigate.const import CACHE_DIR, RECORD_DIR +from frigate.models import Event, Recordings logger = logging.getLogger(__name__) @@ -45,8 +48,10 @@ class RecordingMaintainer(threading.Thread): def move_files(self): recordings = [ d - for d in os.listdir(RECORD_DIR) - if os.path.isfile(os.path.join(RECORD_DIR, d)) and d.endswith(".mp4") + for d in os.listdir(CACHE_DIR) + if os.path.isfile(os.path.join(CACHE_DIR, d)) + and d.endswith(".mp4") + and not d.startswith("tmp_clip") ] files_in_use = [] @@ -57,7 +62,7 @@ class RecordingMaintainer(threading.Thread): flist = process.open_files() if flist: for nt in flist: - if nt.path.startswith(RECORD_DIR): + if nt.path.startswith(CACHE_DIR): files_in_use.append(nt.path.split("/")[-1]) except: continue @@ -66,6 +71,7 @@ class RecordingMaintainer(threading.Thread): if f in files_in_use: continue + cache_path = os.path.join(CACHE_DIR, f) basename = os.path.splitext(f)[0] camera, date = basename.rsplit("-", maxsplit=1) start_time = datetime.datetime.strptime(date, "%Y%m%d%H%M%S") @@ -78,7 +84,7 @@ class RecordingMaintainer(threading.Thread): "format=duration", "-of", "default=noprint_wrappers=1:nokey=1", - f"{os.path.join(RECORD_DIR, f)}", + f"{cache_path}", ] p = sp.run(ffprobe_cmd, capture_output=True) if p.returncode == 0: @@ -86,7 +92,7 @@ class RecordingMaintainer(threading.Thread): end_time = start_time + datetime.timedelta(seconds=duration) else: logger.info(f"bad file: {f}") - os.remove(os.path.join(RECORD_DIR, f)) + Path(cache_path).unlink(missing_ok=True) continue directory = os.path.join( @@ -99,7 +105,7 @@ class RecordingMaintainer(threading.Thread): file_name = f"{start_time.strftime('%M.%S.mp4')}" file_path = os.path.join(directory, file_name) - os.rename(os.path.join(RECORD_DIR, f), file_path) + shutil.move(cache_path, file_path) rand_id = "".join( random.choices(string.ascii_lowercase + string.digits, k=6) @@ -113,7 +119,135 @@ class RecordingMaintainer(threading.Thread): duration=duration, ) + def expire_recordings(self): + event_recordings = Recordings.select( + Recordings.id.alias("recording_id"), + Recordings.camera, + Recordings.path, + Recordings.end_time, + Event.id.alias("event_id"), + Event.label, + ).join( + Event, + on=( + (Recordings.camera == Event.camera) + & ( + (Recordings.start_time.between(Event.start_time, Event.end_time)) + | (Recordings.end_time.between(Event.start_time, Event.end_time)) + ), + ), + ) + + retain = {} + for recording in event_recordings: + # Set default to delete + if recording.path not in retain: + retain[recording.path] = False + + # Handle deleted cameras that still have recordings and events + if recording.camera in self.config.cameras: + record_config = self.config.cameras[recording.camera].record + else: + record_config = self.config.record + + # Check event retention and set to True if within window + expire_days_event = ( + 0 + if not record_config.events.enabled + else record_config.events.retain.objects.get( + recording.event.label, record_config.events.retain.default + ) + ) + expire_before_event = ( + datetime.datetime.now() - datetime.timedelta(days=expire_days_event) + ).timestamp() + if recording.end_time >= expire_before_event: + retain[recording.path] = True + + # Check recording retention and set to True if within window + expire_days_record = record_config.retain_days + expire_before_record = ( + datetime.datetime.now() - datetime.timedelta(days=expire_days_record) + ).timestamp() + if recording.end_time > expire_before_record: + retain[recording.path] = True + + # Actually expire recordings + for path, keep in retain.items(): + if not keep: + Path(path).unlink(missing_ok=True) + Recordings.delete_by_id(recording.recording_id) + + # Update Event + event_no_recordings = ( + Event.select() + .join( + Recordings, + JOIN.LEFT_OUTER, + on=( + (Recordings.camera == Event.camera) + & ( + ( + Recordings.start_time.between( + Event.start_time, Event.end_time + ) + ) + | ( + Recordings.end_time.between( + Event.start_time, Event.end_time + ) + ) + ), + ), + ) + .where(Recordings.id.is_null()) + ) + update = Event.update(has_clip=False).where(Event.id << event_no_recordings) + update.execute() + + event_paths = list(retain.keys()) + + # Handle deleted cameras + no_camera_recordings: Recordings = Recordings.select().where( + Recordings.camera.not_in(list(self.config.cameras.keys())), + Recordings.path.not_in(event_paths), + ) + + for recording in no_camera_recordings: + expire_days = self.config.record.retain_days + expire_before = ( + datetime.datetime.now() - datetime.timedelta(days=expire_days) + ).timestamp() + if recording.end_time >= expire_before: + Path(recording.path).unlink(missing_ok=True) + Recordings.delete_by_id(recording.id) + + # When deleting recordings without events, we have to keep at LEAST the configured max clip duration + for camera, config in self.config.cameras.items(): + min_end = ( + datetime.datetime.now() + - datetime.timedelta(seconds=config.record.events.max_seconds) + ).timestamp() + recordings: Recordings = Recordings.select().where( + Recordings.camera == camera, + Recordings.path.not_in(event_paths), + Recordings.end_time < min_end, + ) + + for recording in recordings: + expire_days = config.record.retain_days + expire_before = ( + datetime.datetime.now() - datetime.timedelta(days=expire_days) + ).timestamp() + if recording.end_time >= expire_before: + Path(recording.path).unlink(missing_ok=True) + Recordings.delete_by_id(recording.id) + def expire_files(self): + default_expire = ( + datetime.datetime.now().timestamp() + - SECONDS_IN_DAY * self.config.record.retain_days + ) delete_before = {} for name, camera in self.config.cameras.items(): delete_before[name] = ( @@ -122,19 +256,22 @@ class RecordingMaintainer(threading.Thread): ) for p in Path("/media/frigate/recordings").rglob("*.mp4"): - if not p.parent.name in delete_before: + # Ignore files that have a record in the recordings DB + if Recordings.select().where(Recordings.path == str(p)).count(): continue - if p.stat().st_mtime < delete_before[p.parent.name]: - Recordings.delete().where(Recordings.path == str(p)).execute() + if p.stat().st_mtime < delete_before.get(p.parent.name, default_expire): p.unlink(missing_ok=True) def run(self): - for counter in itertools.cycle(range(60)): - if self.stop_event.wait(10): + # only expire events every 10 minutes, but check for new files every 5 seconds + for counter in itertools.cycle(range(120)): + if self.stop_event.wait(5): logger.info(f"Exiting recording maintenance...") break - # only expire events every 10 minutes, but check for new files every 10 seconds + if counter % 12 == 0: + self.expire_recordings() + if counter == 0: self.expire_files() remove_empty_directories(RECORD_DIR) diff --git a/frigate/test/test_config.py b/frigate/test/test_config.py index 23f3499d3..c3c77a866 100644 --- a/frigate/test/test_config.py +++ b/frigate/test/test_config.py @@ -198,7 +198,6 @@ class TestConfig(unittest.TestCase): assert len(back_camera.objects.filters["person"].raw_mask) == 1 def test_default_input_args(self): - config = { "mqtt": {"host": "mqtt"}, "cameras": { diff --git a/web/src/routes/Event.jsx b/web/src/routes/Event.jsx index ae6cb89b1..f1d4aea87 100644 --- a/web/src/routes/Event.jsx +++ b/web/src/routes/Event.jsx @@ -115,8 +115,8 @@ export default function Event({ eventId }) { options={{ sources: [ { - src: `${apiHost}/clips/${data.camera}-${eventId}.mp4`, - type: 'video/mp4', + src: `${apiHost}/vod/event/${eventId}/index.m3u8`, + type: 'application/vnd.apple.mpegurl', }, ], poster: data.has_snapshot @@ -127,7 +127,7 @@ export default function Event({ eventId }) { onReady={(player) => {}} />
-