Mirrors/blakeblackshear.frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2025-01-07 00:06:57 +01:00
Code Issues Projects Releases Wiki Activity

Cleanup batching (#14836)

Browse Source 
* Implement batching for event cleanup

* remove import

* add debug logging
This commit is contained in:
Josh Hawkins 2024-11-06 11:05:44 -06:00 committed by GitHub
parent 2eb5fbf112
commit bc371acb3e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 36 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
frigate/events/cleanup.py
View File

@ -21,6 +21,9 @@ class EventCleanupType(str, Enum):
snapshots = "snapshots"
CHUNK_SIZE = 50
class EventCleanup(threading.Thread):
def __init__(
self, config: FrigateConfig, stop_event: MpEvent, db: SqliteVecQueueDatabase
@ -107,6 +110,7 @@ class EventCleanup(threading.Thread):
.namedtuples()
.iterator()
)
logger.debug(f"{len(expired_events)} events can be expired")
# delete the media from disk
for expired in expired_events:
media_name = f"{expired.camera}-{expired.id}"
@ -125,13 +129,34 @@ class EventCleanup(threading.Thread):
logger.warning(f"Unable to delete event images: {e}")
# update the clips attribute for the db entry
update_query = Event.update(update_params).where(
query = Event.select(Event.id).where(
Event.camera.not_in(self.camera_keys),
Event.start_time < expire_after,
Event.label == event.label,
Event.retain_indefinitely == False,
)
update_query.execute()
events_to_update = []
for batch in query.iterator():
events_to_update.extend([event.id for event in batch])
if len(events_to_update) >= CHUNK_SIZE:
logger.debug(
f"Updating {update_params} for {len(events_to_update)} events"
)
Event.update(update_params).where(
Event.id << events_to_update
).execute()
events_to_update = []
# Update any remaining events
if events_to_update:
logger.debug(
f"Updating clips/snapshots attribute for {len(events_to_update)} events"
)
Event.update(update_params).where(
Event.id << events_to_update
).execute()
events_to_update = []
@ -196,7 +221,11 @@ class EventCleanup(threading.Thread):
logger.warning(f"Unable to delete event images: {e}")
# update the clips attribute for the db entry
Event.update(update_params).where(Event.id << events_to_update).execute()
for i in range(0, len(events_to_update), CHUNK_SIZE):
batch = events_to_update[i : i + CHUNK_SIZE]
logger.debug(f"Updating {update_params} for {len(batch)} events")
Event.update(update_params).where(Event.id << batch).execute()
return events_to_update
def run(self) -> None:
@ -222,10 +251,11 @@ class EventCleanup(threading.Thread):
.iterator()
)
events_to_delete = [e.id for e in events]
logger.debug(f"Found {len(events_to_delete)} events that can be expired")
if len(events_to_delete) > 0:
chunk_size = 50
for i in range(0, len(events_to_delete), chunk_size):
chunk = events_to_delete[i : i + chunk_size]
for i in range(0, len(events_to_delete), CHUNK_SIZE):
chunk = events_to_delete[i : i + CHUNK_SIZE]
logger.debug(f"Deleting {len(chunk)} events from the database")
Event.delete().where(Event.id << chunk).execute()
if self.config.semantic_search.enabled: