From 568e620963d0520811cafb4ef7d71e4a67affc50 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Fri, 30 May 2025 17:01:39 -0600 Subject: [PATCH] Tiered recordings (#18492) * Implement tiered recording * Add migration for record config * Update docs * Update reference docs * Fix preview query * Fix incorrect accesses * Fix * Fix * Fix * Fix --- docs/docs/configuration/record.md | 52 +++++------------------- docs/docs/configuration/reference.md | 16 ++++---- frigate/config/camera/record.py | 28 ++++++++----- frigate/config/config.py | 30 +------------- frigate/record/cleanup.py | 42 +++++++++++++------ frigate/record/maintainer.py | 25 +++++++----- frigate/util/config.py | 60 +++++++++++++++++++++++++++- 7 files changed, 143 insertions(+), 110 deletions(-) diff --git a/docs/docs/configuration/record.md b/docs/docs/configuration/record.md index 52c0f0c88..2745ef27d 100644 --- a/docs/docs/configuration/record.md +++ b/docs/docs/configuration/record.md @@ -13,14 +13,15 @@ H265 recordings can be viewed in Chrome 108+, Edge and Safari only. All other br ### Most conservative: Ensure all video is saved -For users deploying Frigate in environments where it is important to have contiguous video stored even if there was no detectable motion, the following config will store all video for 3 days. After 3 days, only video containing motion and overlapping with alerts or detections will be retained until 30 days have passed. +For users deploying Frigate in environments where it is important to have contiguous video stored even if there was no detectable motion, the following config will store all video for 3 days. After 3 days, only video containing motion will be saved for 7 days. After 7 days, only video containing motion and overlapping with alerts or detections will be retained until 30 days have passed. ```yaml record: enabled: True - retain: + continuous: days: 3 - mode: all + motion: + days: 7 alerts: retain: days: 30 @@ -38,9 +39,8 @@ In order to reduce storage requirements, you can adjust your config to only reta ```yaml record: enabled: True - retain: + motion: days: 3 - mode: motion alerts: retain: days: 30 @@ -58,7 +58,7 @@ If you only want to retain video that occurs during a tracked object, this confi ```yaml record: enabled: True - retain: + continuous: days: 0 alerts: retain: @@ -80,15 +80,17 @@ Retention configs support decimals meaning they can be configured to retain `0.5 ::: -### Continuous Recording +### Continuous and Motion Recording -The number of days to retain continuous recordings can be set via the following config where X is a number, by default continuous recording is disabled. +The number of days to retain continuous and motion recordings can be set via the following config where X is a number, by default continuous recording is disabled. ```yaml record: enabled: True - retain: + continuous: days: 1 # <- number of days to keep continuous recordings + motion: + days: 2 # <- number of days to keep motion recordings ``` Continuous recording supports different retention modes [which are described below](#what-do-the-different-retain-modes-mean) @@ -112,38 +114,6 @@ This configuration will retain recording segments that overlap with alerts and d **WARNING**: Recordings still must be enabled in the config. If a camera has recordings disabled in the config, enabling via the methods listed above will have no effect. -## What do the different retain modes mean? - -Frigate saves from the stream with the `record` role in 10 second segments. These options determine which recording segments are kept for continuous recording (but can also affect tracked objects). - -Let's say you have Frigate configured so that your doorbell camera would retain the last **2** days of continuous recording. - -- With the `all` option all 48 hours of those two days would be kept and viewable. -- With the `motion` option the only parts of those 48 hours would be segments that Frigate detected motion. This is the middle ground option that won't keep all 48 hours, but will likely keep all segments of interest along with the potential for some extra segments. -- With the `active_objects` option the only segments that would be kept are those where there was a true positive object that was not considered stationary. - -The same options are available with alerts and detections, except it will only save the recordings when it overlaps with a review item of that type. - -A configuration example of the above retain modes where all `motion` segments are stored for 7 days and `active objects` are stored for 14 days would be as follows: - -```yaml -record: - enabled: True - retain: - days: 7 - mode: motion - alerts: - retain: - days: 14 - mode: active_objects - detections: - retain: - days: 14 - mode: active_objects -``` - -The above configuration example can be added globally or on a per camera basis. - ## Can I have "continuous" recordings, but only at certain times? Using Frigate UI, Home Assistant, or MQTT, cameras can be automated to only record in certain situations or at certain times. diff --git a/docs/docs/configuration/reference.md b/docs/docs/configuration/reference.md index b8360c902..4da2fe320 100644 --- a/docs/docs/configuration/reference.md +++ b/docs/docs/configuration/reference.md @@ -440,18 +440,18 @@ record: expire_interval: 60 # Optional: Sync recordings with disk on startup and once a day (default: shown below). sync_recordings: False - # Optional: Retention settings for recording - retain: + # Optional: Continuous retention settings + continuous: + # Optional: Number of days to retain recordings regardless of tracked objects or motion (default: shown below) + # NOTE: This should be set to 0 and retention should be defined in alerts and detections section below + # if you only want to retain recordings of alerts and detections. + days: 0 + # Optional: Motion retention settings + motion: # Optional: Number of days to retain recordings regardless of tracked objects (default: shown below) # NOTE: This should be set to 0 and retention should be defined in alerts and detections section below # if you only want to retain recordings of alerts and detections. days: 0 - # Optional: Mode for retention. Available options are: all, motion, and active_objects - # all - save all recording segments regardless of activity - # motion - save all recordings segments with any detected motion - # active_objects - save all recording segments with active/moving objects - # NOTE: this mode only applies when the days setting above is greater than 0 - mode: all # Optional: Recording Export Settings export: # Optional: Timelapse Output Args (default: shown below). diff --git a/frigate/config/camera/record.py b/frigate/config/camera/record.py index 52d11e2a5..09a7a84d5 100644 --- a/frigate/config/camera/record.py +++ b/frigate/config/camera/record.py @@ -22,27 +22,31 @@ __all__ = [ DEFAULT_TIME_LAPSE_FFMPEG_ARGS = "-vf setpts=0.04*PTS -r 30" +class RecordRetainConfig(FrigateBaseModel): + days: float = Field(default=0, ge=0, title="Default retention period.") + + class RetainModeEnum(str, Enum): all = "all" motion = "motion" active_objects = "active_objects" -class RecordRetainConfig(FrigateBaseModel): - days: float = Field(default=0, title="Default retention period.") - mode: RetainModeEnum = Field(default=RetainModeEnum.all, title="Retain mode.") - - class ReviewRetainConfig(FrigateBaseModel): - days: float = Field(default=10, title="Default retention period.") + days: float = Field(default=10, ge=0, title="Default retention period.") mode: RetainModeEnum = Field(default=RetainModeEnum.motion, title="Retain mode.") class EventsConfig(FrigateBaseModel): pre_capture: int = Field( - default=5, title="Seconds to retain before event starts.", le=MAX_PRE_CAPTURE + default=5, + title="Seconds to retain before event starts.", + le=MAX_PRE_CAPTURE, + ge=0, + ) + post_capture: int = Field( + default=5, ge=0, title="Seconds to retain after event ends." ) - post_capture: int = Field(default=5, title="Seconds to retain after event ends.") retain: ReviewRetainConfig = Field( default_factory=ReviewRetainConfig, title="Event retention settings." ) @@ -77,8 +81,12 @@ class RecordConfig(FrigateBaseModel): default=60, title="Number of minutes to wait between cleanup runs.", ) - retain: RecordRetainConfig = Field( - default_factory=RecordRetainConfig, title="Record retention settings." + continuous: RecordRetainConfig = Field( + default_factory=RecordRetainConfig, + title="Continuous recording retention settings.", + ) + motion: RecordRetainConfig = Field( + default_factory=RecordRetainConfig, title="Motion recording retention settings." ) detections: EventsConfig = Field( default_factory=EventsConfig, title="Detection specific retention settings." diff --git a/frigate/config/config.py b/frigate/config/config.py index d912a574d..49e57f3cf 100644 --- a/frigate/config/config.py +++ b/frigate/config/config.py @@ -48,7 +48,7 @@ from .camera.genai import GenAIConfig from .camera.motion import MotionConfig from .camera.notification import NotificationConfig from .camera.objects import FilterConfig, ObjectConfig -from .camera.record import RecordConfig, RetainModeEnum +from .camera.record import RecordConfig from .camera.review import ReviewConfig from .camera.snapshots import SnapshotsConfig from .camera.timestamp import TimestampStyleConfig @@ -204,33 +204,6 @@ def verify_valid_live_stream_names( ) -def verify_recording_retention(camera_config: CameraConfig) -> None: - """Verify that recording retention modes are ranked correctly.""" - rank_map = { - RetainModeEnum.all: 0, - RetainModeEnum.motion: 1, - RetainModeEnum.active_objects: 2, - } - - if ( - camera_config.record.retain.days != 0 - and rank_map[camera_config.record.retain.mode] - > rank_map[camera_config.record.alerts.retain.mode] - ): - logger.warning( - f"{camera_config.name}: Recording retention is configured for {camera_config.record.retain.mode} and alert retention is configured for {camera_config.record.alerts.retain.mode}. The more restrictive retention policy will be applied." - ) - - if ( - camera_config.record.retain.days != 0 - and rank_map[camera_config.record.retain.mode] - > rank_map[camera_config.record.detections.retain.mode] - ): - logger.warning( - f"{camera_config.name}: Recording retention is configured for {camera_config.record.retain.mode} and detection retention is configured for {camera_config.record.detections.retain.mode}. The more restrictive retention policy will be applied." - ) - - def verify_recording_segments_setup_with_reasonable_time( camera_config: CameraConfig, ) -> None: @@ -697,7 +670,6 @@ class FrigateConfig(FrigateBaseModel): verify_config_roles(camera_config) verify_valid_live_stream_names(self, camera_config) - verify_recording_retention(camera_config) verify_recording_segments_setup_with_reasonable_time(camera_config) verify_zone_objects_are_tracked(camera_config) verify_required_zones_exist(camera_config) diff --git a/frigate/record/cleanup.py b/frigate/record/cleanup.py index 1de08a899..9d1e28306 100644 --- a/frigate/record/cleanup.py +++ b/frigate/record/cleanup.py @@ -100,7 +100,11 @@ class RecordingCleanup(threading.Thread): ).execute() def expire_existing_camera_recordings( - self, expire_date: float, config: CameraConfig, reviews: ReviewSegment + self, + continuous_expire_date: float, + motion_expire_date: float, + config: CameraConfig, + reviews: ReviewSegment, ) -> None: """Delete recordings for existing camera based on retention config.""" # Get the timestamp for cutoff of retained days @@ -116,8 +120,14 @@ class RecordingCleanup(threading.Thread): Recordings.motion, ) .where( - Recordings.camera == config.name, - Recordings.end_time < expire_date, + (Recordings.camera == config.name) + & ( + ( + (Recordings.end_time < continuous_expire_date) + & (Recordings.motion == 0) + ) + | (Recordings.end_time < motion_expire_date) + ) ) .order_by(Recordings.start_time) .namedtuples() @@ -188,7 +198,7 @@ class RecordingCleanup(threading.Thread): Recordings.id << deleted_recordings_list[i : i + max_deletes] ).execute() - previews: Previews = ( + previews: list[Previews] = ( Previews.select( Previews.id, Previews.start_time, @@ -196,8 +206,9 @@ class RecordingCleanup(threading.Thread): Previews.path, ) .where( - Previews.camera == config.name, - Previews.end_time < expire_date, + (Previews.camera == config.name) + & (Previews.end_time < continuous_expire_date) + & (Previews.end_time < motion_expire_date) ) .order_by(Previews.start_time) .namedtuples() @@ -253,7 +264,9 @@ class RecordingCleanup(threading.Thread): logger.debug("Start deleted cameras.") # Handle deleted cameras - expire_days = self.config.record.retain.days + expire_days = max( + self.config.record.continuous.days, self.config.record.motion.days + ) expire_before = ( datetime.datetime.now() - datetime.timedelta(days=expire_days) ).timestamp() @@ -291,9 +304,12 @@ class RecordingCleanup(threading.Thread): now = datetime.datetime.now() self.expire_review_segments(config, now) - - expire_days = config.record.retain.days - expire_date = (now - datetime.timedelta(days=expire_days)).timestamp() + continuous_expire_date = ( + now - datetime.timedelta(days=config.record.continuous.days) + ).timestamp() + motion_expire_date = ( + now - datetime.timedelta(days=config.record.motion.days) + ).timestamp() # Get all the reviews to check against reviews: ReviewSegment = ( @@ -306,13 +322,15 @@ class RecordingCleanup(threading.Thread): ReviewSegment.camera == camera, # need to ensure segments for all reviews starting # before the expire date are included - ReviewSegment.start_time < expire_date, + ReviewSegment.start_time < motion_expire_date, ) .order_by(ReviewSegment.start_time) .namedtuples() ) - self.expire_existing_camera_recordings(expire_date, config, reviews) + self.expire_existing_camera_recordings( + continuous_expire_date, motion_expire_date, config, reviews + ) logger.debug(f"End camera: {camera}.") logger.debug("End all cameras.") diff --git a/frigate/record/maintainer.py b/frigate/record/maintainer.py index 7f13451d6..ace9a5d24 100644 --- a/frigate/record/maintainer.py +++ b/frigate/record/maintainer.py @@ -285,12 +285,16 @@ class RecordingMaintainer(threading.Thread): Path(cache_path).unlink(missing_ok=True) return - # if cached file's start_time is earlier than the retain days for the camera - # meaning continuous recording is not enabled - if start_time <= ( - datetime.datetime.now().astimezone(datetime.timezone.utc) - - datetime.timedelta(days=self.config.cameras[camera].record.retain.days) - ): + record_config = self.config.cameras[camera].record + highest = None + + if record_config.continuous.days > 0: + highest = "continuous" + elif record_config.motion.days > 0: + highest = "motion" + + # continuous / motion recording is not enabled + if highest is None: # if the cached segment overlaps with the review items: overlaps = False for review in reviews: @@ -344,8 +348,7 @@ class RecordingMaintainer(threading.Thread): ).astimezone(datetime.timezone.utc) if end_time < retain_cutoff: self.drop_segment(cache_path) - # else retain days includes this segment - # meaning continuous recording is enabled + # continuous / motion is enabled else: # assume that empty means the relevant recording info has not been received yet camera_info = self.object_recordings_info[camera] @@ -360,7 +363,11 @@ class RecordingMaintainer(threading.Thread): ).astimezone(datetime.timezone.utc) >= end_time ): - record_mode = self.config.cameras[camera].record.retain.mode + record_mode = ( + RetainModeEnum.all + if highest == "continuous" + else RetainModeEnum.motion + ) return await self.move_segment( camera, start_time, end_time, duration, cache_path, record_mode ) diff --git a/frigate/util/config.py b/frigate/util/config.py index 70492adbc..98267b9ea 100644 --- a/frigate/util/config.py +++ b/frigate/util/config.py @@ -13,7 +13,7 @@ from frigate.util.services import get_video_properties logger = logging.getLogger(__name__) -CURRENT_CONFIG_VERSION = "0.16-0" +CURRENT_CONFIG_VERSION = "0.17-0" DEFAULT_CONFIG_FILE = os.path.join(CONFIG_DIR, "config.yml") @@ -91,6 +91,13 @@ def migrate_frigate_config(config_file: str): yaml.dump(new_config, f) previous_version = "0.16-0" + if previous_version < "0.17-0": + logger.info(f"Migrating frigate config from {previous_version} to 0.17-0...") + new_config = migrate_017_0(config) + with open(config_file, "w") as f: + yaml.dump(new_config, f) + previous_version = "0.17-0" + logger.info("Finished frigate config migration...") @@ -340,6 +347,57 @@ def migrate_016_0(config: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any] return new_config +def migrate_017_0(config: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]]: + """Handle migrating frigate config to 0.16-0""" + new_config = config.copy() + + # migrate global to new recording configuration + global_record_retain = config.get("record", {}).get("retain") + + if global_record_retain: + continuous = {"days": 0} + motion = {"days": 0} + days = global_record_retain.get("days") + mode = global_record_retain.get("mode", "all") + + if days: + if mode == "all": + continuous["days"] = days + else: + motion["days"] = days + + new_config["record"]["continuous"] = continuous + new_config["record"]["motion"] = motion + + del new_config["record"]["retain"] + + for name, camera in config.get("cameras", {}).items(): + camera_config: dict[str, dict[str, Any]] = camera.copy() + camera_record_retain = camera_config.get("record", {}).get("retain") + + if camera_record_retain: + continuous = {"days": 0} + motion = {"days": 0} + days = camera_record_retain.get("days") + mode = camera_record_retain.get("mode", "all") + + if days: + if mode == "all": + continuous["days"] = days + else: + motion["days"] = days + + camera_config["record"]["continuous"] = continuous + camera_config["record"]["motion"] = motion + + del camera_config["record"]["retain"] + + new_config["cameras"][name] = camera_config + + new_config["version"] = "0.17-0" + return new_config + + def get_relative_coordinates( mask: Optional[Union[str, list]], frame_shape: tuple[int, int] ) -> Union[str, list]: