From 9fc93c72a065301a37df770a82d63cbe0c87bdb9 Mon Sep 17 00:00:00 2001
From: Blake Blackshear <blakeb@blakeshome.com>
Date: Sat, 21 Oct 2023 11:53:33 -0400
Subject: [PATCH] more consistent use of iterators in select queries (#8258)

---
 frigate/events/cleanup.py    | 42 ++++++++++++++++++++++--------------
 frigate/http.py              | 13 +++++++----
 frigate/record/cleanup.py    | 23 +++++++++++++-------
 frigate/record/maintainer.py |  2 ++
 frigate/storage.py           | 41 ++++++++++++++++++++++-------------
 5 files changed, 78 insertions(+), 43 deletions(-)

diff --git a/frigate/events/cleanup.py b/frigate/events/cleanup.py
index 1f1fb5f76..faba73979 100644
--- a/frigate/events/cleanup.py
+++ b/frigate/events/cleanup.py
@@ -83,14 +83,19 @@ class EventCleanup(threading.Thread):
                 datetime.datetime.now() - datetime.timedelta(days=expire_days)
             ).timestamp()
             # grab all events after specific time
-            expired_events = Event.select(
-                Event.id,
-                Event.camera,
-            ).where(
-                Event.camera.not_in(self.camera_keys),
-                Event.start_time < expire_after,
-                Event.label == event.label,
-                Event.retain_indefinitely == False,
+            expired_events = (
+                Event.select(
+                    Event.id,
+                    Event.camera,
+                )
+                .where(
+                    Event.camera.not_in(self.camera_keys),
+                    Event.start_time < expire_after,
+                    Event.label == event.label,
+                    Event.retain_indefinitely == False,
+                )
+                .namedtuples()
+                .iterator()
             )
             # delete the media from disk
             for event in expired_events:
@@ -136,14 +141,19 @@ class EventCleanup(threading.Thread):
                     datetime.datetime.now() - datetime.timedelta(days=expire_days)
                 ).timestamp()
                 # grab all events after specific time
-                expired_events = Event.select(
-                    Event.id,
-                    Event.camera,
-                ).where(
-                    Event.camera == name,
-                    Event.start_time < expire_after,
-                    Event.label == event.label,
-                    Event.retain_indefinitely == False,
+                expired_events = (
+                    Event.select(
+                        Event.id,
+                        Event.camera,
+                    )
+                    .where(
+                        Event.camera == name,
+                        Event.start_time < expire_after,
+                        Event.label == event.label,
+                        Event.retain_indefinitely == False,
+                    )
+                    .namedtuples()
+                    .iterator()
                 )
 
                 # delete the grabbed clips from disk
diff --git a/frigate/http.py b/frigate/http.py
index 42d9c1610..0061d338f 100644
--- a/frigate/http.py
+++ b/frigate/http.py
@@ -958,9 +958,10 @@ def events():
         .order_by(Event.start_time.desc())
         .limit(limit)
         .dicts()
+        .iterator()
     )
 
-    return jsonify([e for e in events])
+    return jsonify(list(events))
 
 
 @bp.route("/events/<camera_name>/<label>/create", methods=["POST"])
@@ -1490,6 +1491,7 @@ def recordings_summary(camera_name):
                 ),
             ).desc()
         )
+        .namedtuples()
     )
 
     event_groups = (
@@ -1511,14 +1513,14 @@ def recordings_summary(camera_name):
                 ),
             ),
         )
-        .objects()
+        .namedtuples()
     )
 
     event_map = {g.hour: g.count for g in event_groups}
 
     days = {}
 
-    for recording_group in recording_groups.objects():
+    for recording_group in recording_groups:
         parts = recording_group.hour.split()
         hour = parts[1]
         day = parts[0]
@@ -1562,9 +1564,11 @@ def recordings(camera_name):
             Recordings.start_time <= before,
         )
         .order_by(Recordings.start_time)
+        .dicts()
+        .iterator()
     )
 
-    return jsonify([e for e in recordings.dicts()])
+    return jsonify(list(recordings))
 
 
 @bp.route("/<camera_name>/start/<int:start_ts>/end/<int:end_ts>/clip.mp4")
@@ -1669,6 +1673,7 @@ def vod_ts(camera_name, start_ts, end_ts):
         )
         .where(Recordings.camera == camera_name)
         .order_by(Recordings.start_time.asc())
+        .iterator()
     )
 
     clips = []
diff --git a/frigate/record/cleanup.py b/frigate/record/cleanup.py
index 1ce61bf40..5a81c91ae 100644
--- a/frigate/record/cleanup.py
+++ b/frigate/record/cleanup.py
@@ -48,12 +48,17 @@ class RecordingCleanup(threading.Thread):
         expire_before = (
             datetime.datetime.now() - datetime.timedelta(days=expire_days)
         ).timestamp()
-        no_camera_recordings: Recordings = Recordings.select(
-            Recordings.id,
-            Recordings.path,
-        ).where(
-            Recordings.camera.not_in(list(self.config.cameras.keys())),
-            Recordings.end_time < expire_before,
+        no_camera_recordings: Recordings = (
+            Recordings.select(
+                Recordings.id,
+                Recordings.path,
+            )
+            .where(
+                Recordings.camera.not_in(list(self.config.cameras.keys())),
+                Recordings.end_time < expire_before,
+            )
+            .namedtuples()
+            .iterator()
         )
 
         deleted_recordings = set()
@@ -95,6 +100,8 @@ class RecordingCleanup(threading.Thread):
                     Recordings.end_time < expire_date,
                 )
                 .order_by(Recordings.start_time)
+                .namedtuples()
+                .iterator()
             )
 
             # Get all the events to check against
@@ -111,14 +118,14 @@ class RecordingCleanup(threading.Thread):
                     Event.has_clip,
                 )
                 .order_by(Event.start_time)
-                .objects()
+                .namedtuples()
             )
 
             # loop over recordings and see if they overlap with any non-expired events
             # TODO: expire segments based on segment stats according to config
             event_start = 0
             deleted_recordings = set()
-            for recording in recordings.objects().iterator():
+            for recording in recordings:
                 keep = False
                 # Now look for a reason to keep this recording segment
                 for idx in range(event_start, len(events)):
diff --git a/frigate/record/maintainer.py b/frigate/record/maintainer.py
index 3cda65c29..0370bbf3e 100644
--- a/frigate/record/maintainer.py
+++ b/frigate/record/maintainer.py
@@ -163,6 +163,8 @@ class RecordingMaintainer(threading.Thread):
                     Event.has_clip,
                 )
                 .order_by(Event.start_time)
+                .namedtuples()
+                .iterator()
             )
 
             tasks.extend(
diff --git a/frigate/storage.py b/frigate/storage.py
index 33a2998ab..257ab8985 100644
--- a/frigate/storage.py
+++ b/frigate/storage.py
@@ -99,13 +99,19 @@ class StorageMaintainer(threading.Thread):
             [b["bandwidth"] for b in self.camera_storage_stats.values()]
         )
 
-        recordings: Recordings = Recordings.select(
-            Recordings.id,
-            Recordings.start_time,
-            Recordings.end_time,
-            Recordings.segment_size,
-            Recordings.path,
-        ).order_by(Recordings.start_time.asc())
+        recordings: Recordings = (
+            Recordings.select(
+                Recordings.id,
+                Recordings.start_time,
+                Recordings.end_time,
+                Recordings.segment_size,
+                Recordings.path,
+            )
+            .order_by(Recordings.start_time.asc())
+            .namedtuples()
+            .iterator()
+        )
+
         retained_events: Event = (
             Event.select(
                 Event.start_time,
@@ -116,12 +122,12 @@ class StorageMaintainer(threading.Thread):
                 Event.has_clip,
             )
             .order_by(Event.start_time.asc())
-            .objects()
+            .namedtuples()
         )
 
         event_start = 0
         deleted_recordings = set()
-        for recording in recordings.objects().iterator():
+        for recording in recordings:
             # check if 1 hour of storage has been reclaimed
             if deleted_segments_size > hourly_bandwidth:
                 break
@@ -162,13 +168,18 @@ class StorageMaintainer(threading.Thread):
             logger.error(
                 f"Could not clear {hourly_bandwidth} MB, currently {deleted_segments_size} MB have been cleared. Retained recordings must be deleted."
             )
-            recordings = Recordings.select(
-                Recordings.id,
-                Recordings.path,
-                Recordings.segment_size,
-            ).order_by(Recordings.start_time.asc())
+            recordings = (
+                Recordings.select(
+                    Recordings.id,
+                    Recordings.path,
+                    Recordings.segment_size,
+                )
+                .order_by(Recordings.start_time.asc())
+                .namedtuples()
+                .iterator()
+            )
 
-            for recording in recordings.objects().iterator():
+            for recording in recordings:
                 if deleted_segments_size > hourly_bandwidth:
                     break