diff --git a/docs/docs/configuration/record.md b/docs/docs/configuration/record.md index 7aeec9d0c..b879a228e 100644 --- a/docs/docs/configuration/record.md +++ b/docs/docs/configuration/record.md @@ -7,6 +7,10 @@ Recordings can be enabled and are stored at `/media/frigate/recordings`. The fol H265 recordings can be viewed in Edge and Safari only. All other browsers require recordings to be encoded with H264. +## Will Frigate delete old recordings if my storage runs out? + +As of Frigate 0.12 if there is less than an hour left of storage, the oldest 2 hours of recordings will be deleted. + ## What if I don't want 24/7 recordings? If you only used clips in previous versions with recordings disabled, you can use the following config to get the same behavior. This is also the default behavior when recordings are enabled. @@ -25,23 +29,23 @@ When `retain -> days` is set to `0`, segments will be deleted from the cache if ## Can I have "24/7" recordings, but only at certain times? -Using Frigate UI, HomeAssistant, or MQTT, cameras can be automated to only record in certain situations or at certain times. +Using Frigate UI, HomeAssistant, or MQTT, cameras can be automated to only record in certain situations or at certain times. **WARNING**: Recordings still must be enabled in the config. If a camera has recordings disabled in the config, enabling via the methods listed above will have no effect. ## What do the different retain modes mean? -Frigate saves from the stream with the `record` role in 10 second segments. These options determine which recording segments are kept for 24/7 recording (but can also affect events). +Frigate saves from the stream with the `record` role in 10 second segments. These options determine which recording segments are kept for 24/7 recording (but can also affect events). -Let's say you have frigate configured so that your doorbell camera would retain the last **2** days of 24/7 recording. -- With the `all` option all 48 hours of those two days would be kept and viewable. +Let's say you have frigate configured so that your doorbell camera would retain the last **2** days of 24/7 recording. +- With the `all` option all 48 hours of those two days would be kept and viewable. - With the `motion` option the only parts of those 48 hours would be segments that frigate detected motion. This is the middle ground option that won't keep all 48 hours, but will likely keep all segments of interest along with the potential for some extra segments. - With the `active_objects` option the only segments that would be kept are those where there was a true positive object that was not considered stationary. The same options are available with events. Let's consider a scenario where you drive up and park in your driveway, go inside, then come back out 4 hours later. - With the `all` option all segments for the duration of the event would be saved for the event. This event would have 4 hours of footage. -- With the `motion` option all segments for the duration of the event with motion would be saved. This means any segment where a car drove by in the street, person walked by, lighting changed, etc. would be saved. -- With the `active_objects` it would only keep segments where the object was active. In this case the only segments that would be saved would be the ones where the car was driving up, you going inside, you coming outside, and the car driving away. Essentially reducing the 4 hours to a minute or two of event footage. +- With the `motion` option all segments for the duration of the event with motion would be saved. This means any segment where a car drove by in the street, person walked by, lighting changed, etc. would be saved. +- With the `active_objects` it would only keep segments where the object was active. In this case the only segments that would be saved would be the ones where the car was driving up, you going inside, you coming outside, and the car driving away. Essentially reducing the 4 hours to a minute or two of event footage. A configuration example of the above retain modes where all `motion` segments are stored for 7 days and `active objects` are stored for 14 days would be as follows: ```yaml diff --git a/docs/docs/installation.md b/docs/docs/installation.md index 5a6d02eb7..5fe8e59fd 100644 --- a/docs/docs/installation.md +++ b/docs/docs/installation.md @@ -21,12 +21,6 @@ Windows is not officially supported, but some users have had success getting it Frigate uses the following locations for read/write operations in the container. Docker volume mappings can be used to map these to any location on your host machine. -:::caution - -Note that Frigate does not currently support limiting recordings based on available disk space automatically. If using recordings, you must specify retention settings for a number of days that will fit within the available disk space of your drive or Frigate will crash. - -::: - - `/media/frigate/clips`: Used for snapshot storage. In the future, it will likely be renamed from `clips` to `snapshots`. The file structure here cannot be modified and isn't intended to be browsed or managed manually. - `/media/frigate/recordings`: Internal system storage for recording segments. The file structure here cannot be modified and isn't intended to be browsed or managed manually. - `/media/frigate/frigate.db`: Default location for the sqlite database. You will also see several files alongside this file while frigate is running. If moving the database location (often needed when using a network drive at `/media/frigate`), it is recommended to mount a volume with docker at `/db` and change the storage location of the database to `/db/frigate.db` in the config file. diff --git a/frigate/app.py b/frigate/app.py index 81603b4e0..b97f580a0 100644 --- a/frigate/app.py +++ b/frigate/app.py @@ -1,23 +1,17 @@ -import json import logging import multiprocessing as mp from multiprocessing.queues import Queue from multiprocessing.synchronize import Event -from multiprocessing.context import Process import os import signal import sys -import threading -from logging.handlers import QueueHandler from typing import Optional from types import FrameType import traceback -import yaml from peewee_migrate import Router from playhouse.sqlite_ext import SqliteExtDatabase from playhouse.sqliteq import SqliteQueueDatabase -from pydantic import ValidationError from frigate.config import DetectorTypeEnum, FrigateConfig from frigate.const import CACHE_DIR, CLIPS_DIR, RECORD_DIR @@ -32,6 +26,7 @@ from frigate.output import output_frames from frigate.plus import PlusApi from frigate.record import RecordingCleanup, RecordingMaintainer from frigate.stats import StatsEmitter, stats_init +from frigate.storage import StorageMaintainer from frigate.version import VERSION from frigate.video import capture_camera, track_camera from frigate.watchdog import FrigateWatchdog @@ -310,6 +305,10 @@ class FrigateApp: self.recording_cleanup = RecordingCleanup(self.config, self.stop_event) self.recording_cleanup.start() + def start_storage_maintainer(self) -> None: + self.storage_maintainer = StorageMaintainer(self.config, self.stop_event) + self.storage_maintainer.start() + def start_stats_emitter(self) -> None: self.stats_emitter = StatsEmitter( self.config, @@ -369,6 +368,7 @@ class FrigateApp: self.start_event_cleanup() self.start_recording_maintainer() self.start_recording_cleanup() + self.start_storage_maintainer() self.start_stats_emitter() self.start_watchdog() # self.zeroconf = broadcast_zeroconf(self.config.mqtt.client_id) diff --git a/frigate/http.py b/frigate/http.py index 9b3c35dea..b5df480cd 100644 --- a/frigate/http.py +++ b/frigate/http.py @@ -751,6 +751,7 @@ def recordings(camera_name): Recordings.id, Recordings.start_time, Recordings.end_time, + Recordings.segment_size, Recordings.motion, Recordings.objects, ) diff --git a/frigate/models.py b/frigate/models.py index 45d387bec..4facf4bfa 100644 --- a/frigate/models.py +++ b/frigate/models.py @@ -41,3 +41,4 @@ class Recordings(Model): # type: ignore[misc] duration = FloatField() motion = IntegerField(null=True) objects = IntegerField(null=True) + segment_size = FloatField(default=0) # this should be stored as MB diff --git a/frigate/record.py b/frigate/record.py index c602b4a98..21e42dfc0 100644 --- a/frigate/record.py +++ b/frigate/record.py @@ -284,6 +284,15 @@ class RecordingMaintainer(threading.Thread): f"Copied {file_path} in {datetime.datetime.now().timestamp()-start_frame} seconds." ) + try: + segment_size = round( + float(os.path.getsize(cache_path)) / 1000000, 1 + ) + except OSError: + segment_size = 0 + + os.remove(cache_path) + rand_id = "".join( random.choices(string.ascii_lowercase + string.digits, k=6) ) @@ -297,10 +306,8 @@ class RecordingMaintainer(threading.Thread): motion=motion_count, # TODO: update this to store list of active objects at some point objects=active_count, + segment_size=segment_size, ) - else: - logger.warning(f"Ignoring segment because {file_path} already exists.") - os.remove(cache_path) except Exception as e: logger.error(f"Unable to store recording segment {cache_path}") Path(cache_path).unlink(missing_ok=True) diff --git a/frigate/storage.py b/frigate/storage.py new file mode 100644 index 000000000..2720b7aca --- /dev/null +++ b/frigate/storage.py @@ -0,0 +1,172 @@ +"""Handle storage retention and usage.""" + +import logging +from pathlib import Path +import shutil +import threading + +from peewee import fn + +from frigate.config import FrigateConfig +from frigate.const import RECORD_DIR +from frigate.models import Event, Recordings + +logger = logging.getLogger(__name__) +bandwidth_equation = Recordings.segment_size / ( + Recordings.end_time - Recordings.start_time +) + + +class StorageMaintainer(threading.Thread): + """Maintain frigates recording storage.""" + + def __init__(self, config: FrigateConfig, stop_event) -> None: + threading.Thread.__init__(self) + self.name = "storage_maintainer" + self.config = config + self.stop_event = stop_event + self.camera_storage_stats: dict[str, dict] = {} + + def calculate_camera_bandwidth(self) -> None: + """Calculate an average MB/hr for each camera.""" + for camera in self.config.cameras.keys(): + # cameras with < 50 segments should be refreshed to keep size accurate + # when few segments are available + if self.camera_storage_stats.get(camera, {}).get("needs_refresh", True): + self.camera_storage_stats[camera] = { + "needs_refresh": ( + Recordings.select(fn.COUNT(Recordings.id)) + .where( + Recordings.camera == camera, Recordings.segment_size != 0 + ) + .scalar() + < 50 + ) + } + + # calculate MB/hr + try: + bandwidth = round( + Recordings.select(fn.AVG(bandwidth_equation)) + .where(Recordings.camera == camera, Recordings.segment_size != 0) + .limit(100) + .scalar() + * 3600, + 2, + ) + except TypeError: + bandwidth = 0 + + self.camera_storage_stats[camera]["bandwidth"] = bandwidth + logger.debug(f"{camera} has a bandwidth of {bandwidth} MB/hr.") + + def check_storage_needs_cleanup(self) -> bool: + """Return if storage needs cleanup.""" + # currently runs cleanup if less than 1 hour of space is left + # disk_usage should not spin up disks + hourly_bandwidth = sum( + [b["bandwidth"] for b in self.camera_storage_stats.values()] + ) + remaining_storage = round(shutil.disk_usage(RECORD_DIR).free / 1000000, 1) + logger.debug( + f"Storage cleanup check: {hourly_bandwidth} hourly with remaining storage: {remaining_storage}." + ) + return remaining_storage < hourly_bandwidth + + def reduce_storage_consumption(self) -> None: + """Remove oldest hour of recordings.""" + logger.debug("Starting storage cleanup.") + deleted_segments_size = 0 + hourly_bandwidth = sum( + [b["bandwidth"] for b in self.camera_storage_stats.values()] + ) + + recordings: Recordings = Recordings.select().order_by( + Recordings.start_time.asc() + ) + retained_events: Event = ( + Event.select() + .where( + Event.retain_indefinitely == True, + Event.has_clip, + ) + .order_by(Event.start_time.asc()) + .objects() + ) + + event_start = 0 + deleted_recordings = set() + for recording in recordings.objects().iterator(): + # check if 1 hour of storage has been reclaimed + if deleted_segments_size > hourly_bandwidth: + break + + keep = False + + # Now look for a reason to keep this recording segment + for idx in range(event_start, len(retained_events)): + event = retained_events[idx] + + # if the event starts in the future, stop checking events + # and let this recording segment expire + if event.start_time > recording.end_time: + keep = False + break + + # if the event is in progress or ends after the recording starts, keep it + # and stop looking at events + if event.end_time is None or event.end_time >= recording.start_time: + keep = True + break + + # if the event ends before this recording segment starts, skip + # this event and check the next event for an overlap. + # since the events and recordings are sorted, we can skip events + # that end before the previous recording segment started on future segments + if event.end_time < recording.start_time: + event_start = idx + + # Delete recordings not retained indefinitely + if not keep: + deleted_segments_size += recording.segment_size + Path(recording.path).unlink(missing_ok=True) + deleted_recordings.add(recording.id) + + # check if need to delete retained segments + if deleted_segments_size < hourly_bandwidth: + logger.error( + f"Could not clear {hourly_bandwidth} currently {deleted_segments_size}, retained recordings must be deleted." + ) + recordings = Recordings.select().order_by(Recordings.start_time.asc()) + + for recording in recordings.objects().iterator(): + if deleted_segments_size > hourly_bandwidth: + break + + deleted_segments_size += recording.segment_size + Path(recording.path).unlink(missing_ok=True) + deleted_recordings.add(recording.id) + + logger.debug(f"Expiring {len(deleted_recordings)} recordings") + # delete up to 100,000 at a time + max_deletes = 100000 + deleted_recordings_list = list(deleted_recordings) + for i in range(0, len(deleted_recordings_list), max_deletes): + Recordings.delete().where( + Recordings.id << deleted_recordings_list[i : i + max_deletes] + ).execute() + + def run(self): + """Check every 5 minutes if storage needs to be cleaned up.""" + while not self.stop_event.wait(300): + + if not self.camera_storage_stats or True in [ + r["needs_refresh"] for r in self.camera_storage_stats.values() + ]: + self.calculate_camera_bandwidth() + logger.debug(f"Default camera bandwidths: {self.camera_storage_stats}.") + + if self.check_storage_needs_cleanup(): + self.reduce_storage_consumption() + + logger.info(f"Exiting storage maintainer...") diff --git a/frigate/test/test_storage.py b/frigate/test/test_storage.py new file mode 100644 index 000000000..22ea6fc1e --- /dev/null +++ b/frigate/test/test_storage.py @@ -0,0 +1,239 @@ +import datetime +import json +import logging +import os +import unittest +from unittest.mock import MagicMock, patch + +from peewee import DoesNotExist +from peewee_migrate import Router +from playhouse.sqlite_ext import SqliteExtDatabase +from playhouse.sqliteq import SqliteQueueDatabase +from playhouse.shortcuts import model_to_dict + +from frigate.config import FrigateConfig +from frigate.http import create_app +from frigate.models import Event, Recordings +from frigate.storage import StorageMaintainer + +from frigate.test.const import TEST_DB, TEST_DB_CLEANUPS + + +class TestHttp(unittest.TestCase): + def setUp(self): + # setup clean database for each test run + migrate_db = SqliteExtDatabase("test.db") + del logging.getLogger("peewee_migrate").handlers[:] + router = Router(migrate_db) + router.run() + migrate_db.close() + self.db = SqliteQueueDatabase(TEST_DB) + models = [Event, Recordings] + self.db.bind(models) + + self.minimal_config = { + "mqtt": {"host": "mqtt"}, + "cameras": { + "front_door": { + "ffmpeg": { + "inputs": [ + {"path": "rtsp://10.0.0.1:554/video", "roles": ["detect"]} + ] + }, + "detect": { + "height": 1080, + "width": 1920, + "fps": 5, + }, + } + }, + } + self.double_cam_config = { + "mqtt": {"host": "mqtt"}, + "cameras": { + "front_door": { + "ffmpeg": { + "inputs": [ + {"path": "rtsp://10.0.0.1:554/video", "roles": ["detect"]} + ] + }, + "detect": { + "height": 1080, + "width": 1920, + "fps": 5, + }, + }, + "back_door": { + "ffmpeg": { + "inputs": [ + {"path": "rtsp://10.0.0.2:554/video", "roles": ["detect"]} + ] + }, + "detect": { + "height": 1080, + "width": 1920, + "fps": 5, + }, + }, + }, + } + + def tearDown(self): + if not self.db.is_closed(): + self.db.close() + + try: + for file in TEST_DB_CLEANUPS: + os.remove(file) + except OSError: + pass + + def test_segment_calculations(self): + """Test that the segment calculations are correct.""" + config = FrigateConfig(**self.double_cam_config) + storage = StorageMaintainer(config, MagicMock()) + + time_keep = datetime.datetime.now().timestamp() + rec_fd_id = "1234567.frontdoor" + rec_bd_id = "1234568.backdoor" + _insert_mock_recording( + rec_fd_id, + time_keep, + time_keep + 10, + camera="front_door", + seg_size=4, + seg_dur=10, + ) + _insert_mock_recording( + rec_bd_id, + time_keep + 10, + time_keep + 20, + camera="back_door", + seg_size=8, + seg_dur=20, + ) + storage.calculate_camera_bandwidth() + assert storage.camera_storage_stats == { + "front_door": {"bandwidth": 1440, "needs_refresh": True}, + "back_door": {"bandwidth": 2880, "needs_refresh": True}, + } + + def test_segment_calculations_with_zero_segments(self): + """Ensure segment calculation does not fail when migrating from previous version.""" + config = FrigateConfig(**self.minimal_config) + storage = StorageMaintainer(config, MagicMock()) + + time_keep = datetime.datetime.now().timestamp() + rec_fd_id = "1234567.frontdoor" + _insert_mock_recording( + rec_fd_id, + time_keep, + time_keep + 10, + camera="front_door", + seg_size=0, + seg_dur=10, + ) + storage.calculate_camera_bandwidth() + assert storage.camera_storage_stats == { + "front_door": {"bandwidth": 0, "needs_refresh": True}, + } + + def test_storage_cleanup(self): + """Ensure that all recordings are cleaned up when necessary.""" + config = FrigateConfig(**self.minimal_config) + storage = StorageMaintainer(config, MagicMock()) + + id = "123456.keep" + time_keep = datetime.datetime.now().timestamp() + _insert_mock_event(id, time_keep, time_keep + 30, True) + rec_k_id = "1234567.keep" + rec_k2_id = "1234568.keep" + rec_k3_id = "1234569.keep" + _insert_mock_recording(rec_k_id, time_keep, time_keep + 10) + _insert_mock_recording(rec_k2_id, time_keep + 10, time_keep + 20) + _insert_mock_recording(rec_k3_id, time_keep + 20, time_keep + 30) + + id2 = "7890.delete" + time_delete = datetime.datetime.now().timestamp() - 360 + _insert_mock_event(id2, time_delete, time_delete + 30, False) + rec_d_id = "78901.delete" + rec_d2_id = "78902.delete" + rec_d3_id = "78903.delete" + _insert_mock_recording(rec_d_id, time_delete, time_delete + 10) + _insert_mock_recording(rec_d2_id, time_delete + 10, time_delete + 20) + _insert_mock_recording(rec_d3_id, time_delete + 20, time_delete + 30) + + storage.calculate_camera_bandwidth() + storage.reduce_storage_consumption() + with self.assertRaises(DoesNotExist): + assert Recordings.get(Recordings.id == rec_k_id) + assert Recordings.get(Recordings.id == rec_k2_id) + assert Recordings.get(Recordings.id == rec_k3_id) + Recordings.get(Recordings.id == rec_d_id) + Recordings.get(Recordings.id == rec_d2_id) + Recordings.get(Recordings.id == rec_d3_id) + + def test_storage_cleanup_keeps_retained(self): + """Ensure that all recordings are cleaned up when necessary.""" + config = FrigateConfig(**self.minimal_config) + storage = StorageMaintainer(config, MagicMock()) + + id = "123456.keep" + time_keep = datetime.datetime.now().timestamp() + _insert_mock_event(id, time_keep, time_keep + 30, True) + rec_k_id = "1234567.keep" + rec_k2_id = "1234568.keep" + rec_k3_id = "1234569.keep" + _insert_mock_recording(rec_k_id, time_keep, time_keep + 10) + _insert_mock_recording(rec_k2_id, time_keep + 10, time_keep + 20) + _insert_mock_recording(rec_k3_id, time_keep + 20, time_keep + 30) + + time_delete = datetime.datetime.now().timestamp() - 7200 + for i in range(0, 59): + _insert_mock_recording( + f"{123456 + i}.delete", time_delete, time_delete + 600 + ) + + storage.calculate_camera_bandwidth() + storage.reduce_storage_consumption() + assert Recordings.get(Recordings.id == rec_k_id) + assert Recordings.get(Recordings.id == rec_k2_id) + assert Recordings.get(Recordings.id == rec_k3_id) + + +def _insert_mock_event(id: str, start: int, end: int, retain: bool) -> Event: + """Inserts a basic event model with a given id.""" + return Event.insert( + id=id, + label="Mock", + camera="front_door", + start_time=start, + end_time=end, + top_score=100, + false_positive=False, + zones=list(), + thumbnail="", + region=[], + box=[], + area=0, + has_clip=True, + has_snapshot=True, + retain_indefinitely=retain, + ).execute() + + +def _insert_mock_recording( + id: str, start: int, end: int, camera="front_door", seg_size=8, seg_dur=10 +) -> Event: + """Inserts a basic recording model with a given id.""" + return Recordings.insert( + id=id, + camera=camera, + path=f"/recordings/{id}", + start_time=start, + end_time=end, + duration=seg_dur, + motion=True, + objects=True, + segment_size=seg_size, + ).execute() diff --git a/migrations/012_add_segment_size.py b/migrations/012_add_segment_size.py new file mode 100644 index 000000000..c5ac1ed93 --- /dev/null +++ b/migrations/012_add_segment_size.py @@ -0,0 +1,46 @@ +"""Peewee migrations -- 012_add_segment_size.py. + +Some examples (model - class or model name):: + + > Model = migrator.orm['model_name'] # Return model in current state by name + + > migrator.sql(sql) # Run custom SQL + > migrator.python(func, *args, **kwargs) # Run python code + > migrator.create_model(Model) # Create a model (could be used as decorator) + > migrator.remove_model(model, cascade=True) # Remove a model + > migrator.add_fields(model, **fields) # Add fields to a model + > migrator.change_fields(model, **fields) # Change fields + > migrator.remove_fields(model, *field_names, cascade=True) + > migrator.rename_field(model, old_field_name, new_field_name) + > migrator.rename_table(model, new_table_name) + > migrator.add_index(model, *col_names, unique=False) + > migrator.drop_index(model, *col_names) + > migrator.add_not_null(model, *field_names) + > migrator.drop_not_null(model, *field_names) + > migrator.add_default(model, field_name, default) + +""" + +import datetime as dt +import peewee as pw +from playhouse.sqlite_ext import * +from decimal import ROUND_HALF_EVEN +from frigate.models import Recordings + +try: + import playhouse.postgres_ext as pw_pext +except ImportError: + pass + +SQL = pw.SQL + + +def migrate(migrator, database, fake=False, **kwargs): + migrator.add_fields( + Recordings, + segment_size=pw.FloatField(default=0), + ) + + +def rollback(migrator, database, fake=False, **kwargs): + migrator.remove_fields(Recordings, ["segment_size"])