From 5ad391977efc0e7720b533b6c928027675050422 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Tue, 29 Nov 2022 18:59:56 -0700 Subject: [PATCH] FEAT: Storage API & Frontend (#3409) * Get storage output stats for each camera * Add storage route * Add storage route * Add storage page * Cleanup * Add stats and show more storage * Add tests for mb abbrev util fun * Rewrite storage logic to use storage maintainer and segment sizes * Include storage maintainer for http * Use correct format * Remove utils * Fix tests * Remove total from equation * Multiply by 100 to get percent * Add basic storage info * Fix storage stats * Fix endpoint and ui * Fix formatting --- frigate/app.py | 3 +- frigate/http.py | 26 ++++++++- frigate/storage.py | 20 +++++++ frigate/test/test_http.py | 17 +++--- test.db-journal | Bin 29240 -> 0 bytes web/src/Sidebar.jsx | 1 + web/src/app.tsx | 1 + web/src/routes/Storage.jsx | 109 +++++++++++++++++++++++++++++++++++++ web/src/routes/index.js | 5 ++ 9 files changed, 172 insertions(+), 10 deletions(-) delete mode 100644 test.db-journal create mode 100644 web/src/routes/Storage.jsx diff --git a/frigate/app.py b/frigate/app.py index f62d4a78a..e1cbef35a 100644 --- a/frigate/app.py +++ b/frigate/app.py @@ -163,6 +163,7 @@ class FrigateApp: self.db, self.stats_tracking, self.detected_frames_processor, + self.storage_maintainer, self.plus_api, ) @@ -362,13 +363,13 @@ class FrigateApp: self.start_detected_frames_processor() self.start_camera_processors() self.start_camera_capture_processes() + self.start_storage_maintainer() self.init_stats() self.init_web_server() self.start_event_processor() self.start_event_cleanup() self.start_recording_maintainer() self.start_recording_cleanup() - self.start_storage_maintainer() self.start_stats_emitter() self.start_watchdog() # self.zeroconf = broadcast_zeroconf(self.config.mqtt.client_id) diff --git a/frigate/http.py b/frigate/http.py index 57517f168..a1f9aea4e 100644 --- a/frigate/http.py +++ b/frigate/http.py @@ -27,12 +27,12 @@ from flask import ( from peewee import SqliteDatabase, operator, fn, DoesNotExist from playhouse.shortcuts import model_to_dict -from frigate.config import CameraConfig -from frigate.const import CLIPS_DIR +from frigate.const import CLIPS_DIR, RECORD_DIR from frigate.models import Event, Recordings from frigate.object_processing import TrackedObject from frigate.stats import stats_snapshot from frigate.util import clean_camera_user_pass, ffprobe_stream, vainfo_hwaccel +from frigate.storage import StorageMaintainer from frigate.version import VERSION logger = logging.getLogger(__name__) @@ -45,6 +45,7 @@ def create_app( database: SqliteDatabase, stats_tracking, detected_frames_processor, + storage_maintainer: StorageMaintainer, plus_api, ): app = Flask(__name__) @@ -62,6 +63,7 @@ def create_app( app.frigate_config = frigate_config app.stats_tracking = stats_tracking app.detected_frames_processor = detected_frames_processor + app.storage_maintainer = storage_maintainer app.plus_api = plus_api app.camera_error_image = None @@ -690,6 +692,26 @@ def latest_frame(camera_name): return "Camera named {} not found".format(camera_name), 404 +@bp.route("/recordings/storage", methods=["GET"]) +def get_recordings_storage_usage(): + recording_stats = stats_snapshot( + current_app.frigate_config, current_app.stats_tracking + )["service"]["storage"][RECORD_DIR] + total_mb = recording_stats["total"] + + camera_usages: dict[ + str, dict + ] = current_app.storage_maintainer.calculate_camera_usages() + + for camera_name in camera_usages.keys(): + if camera_usages.get(camera_name, {}).get("usage"): + camera_usages[camera_name]["usage_percent"] = ( + camera_usages.get(camera_name, {}).get("usage", 0) / total_mb + ) * 100 + + return jsonify(camera_usages) + + # return hourly summary for recordings of camera @bp.route("//recordings/summary") def recordings_summary(camera_name): diff --git a/frigate/storage.py b/frigate/storage.py index 2720b7aca..1ac67a00c 100644 --- a/frigate/storage.py +++ b/frigate/storage.py @@ -60,6 +60,26 @@ class StorageMaintainer(threading.Thread): self.camera_storage_stats[camera]["bandwidth"] = bandwidth logger.debug(f"{camera} has a bandwidth of {bandwidth} MB/hr.") + def calculate_camera_usages(self) -> dict[str, dict]: + """Calculate the storage usage of each camera.""" + usages: dict[str, dict] = {} + + for camera in self.config.cameras.keys(): + camera_storage = ( + Recordings.select(fn.SUM(Recordings.segment_size)) + .where(Recordings.camera == camera, Recordings.segment_size != 0) + .scalar() + ) + + usages[camera] = { + "usage": camera_storage, + "bandwidth": self.camera_storage_stats.get(camera, {}).get( + "bandwidth", 0 + ), + } + + return usages + def check_storage_needs_cleanup(self) -> bool: """Return if storage needs cleanup.""" # currently runs cleanup if less than 1 hour of space is left diff --git a/frigate/test/test_http.py b/frigate/test/test_http.py index 04acdd681..12105926e 100644 --- a/frigate/test/test_http.py +++ b/frigate/test/test_http.py @@ -114,7 +114,7 @@ class TestHttp(unittest.TestCase): def test_get_event_list(self): app = create_app( - FrigateConfig(**self.minimal_config), self.db, None, None, PlusApi() + FrigateConfig(**self.minimal_config), self.db, None, None, None, PlusApi() ) id = "123456.random" id2 = "7890.random" @@ -143,7 +143,7 @@ class TestHttp(unittest.TestCase): def test_get_good_event(self): app = create_app( - FrigateConfig(**self.minimal_config), self.db, None, None, PlusApi() + FrigateConfig(**self.minimal_config), self.db, None, None, None, PlusApi() ) id = "123456.random" @@ -157,7 +157,7 @@ class TestHttp(unittest.TestCase): def test_get_bad_event(self): app = create_app( - FrigateConfig(**self.minimal_config), self.db, None, None, PlusApi() + FrigateConfig(**self.minimal_config), self.db, None, None, None, PlusApi() ) id = "123456.random" bad_id = "654321.other" @@ -170,7 +170,7 @@ class TestHttp(unittest.TestCase): def test_delete_event(self): app = create_app( - FrigateConfig(**self.minimal_config), self.db, None, None, PlusApi() + FrigateConfig(**self.minimal_config), self.db, None, None, None, PlusApi() ) id = "123456.random" @@ -185,7 +185,7 @@ class TestHttp(unittest.TestCase): def test_event_retention(self): app = create_app( - FrigateConfig(**self.minimal_config), self.db, None, None, PlusApi() + FrigateConfig(**self.minimal_config), self.db, None, None, None, PlusApi() ) id = "123456.random" @@ -204,7 +204,7 @@ class TestHttp(unittest.TestCase): def test_set_delete_sub_label(self): app = create_app( - FrigateConfig(**self.minimal_config), self.db, None, None, PlusApi() + FrigateConfig(**self.minimal_config), self.db, None, None, None, PlusApi() ) id = "123456.random" sub_label = "sub" @@ -232,7 +232,7 @@ class TestHttp(unittest.TestCase): def test_sub_label_list(self): app = create_app( - FrigateConfig(**self.minimal_config), self.db, None, None, PlusApi() + FrigateConfig(**self.minimal_config), self.db, None, None, None, PlusApi() ) id = "123456.random" sub_label = "sub" @@ -254,6 +254,7 @@ class TestHttp(unittest.TestCase): self.db, None, None, + None, PlusApi(), ) @@ -268,6 +269,7 @@ class TestHttp(unittest.TestCase): self.db, None, None, + None, PlusApi(), ) id = "123456.random" @@ -285,6 +287,7 @@ class TestHttp(unittest.TestCase): self.db, None, None, + None, PlusApi(), ) mock_stats.return_value = self.test_stats diff --git a/test.db-journal b/test.db-journal deleted file mode 100644 index 3649988aaf8590e0b99c00b63e90214a145aebbb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 29240 zcmeI5&u`mg7{~3zO`0@m1{YsAxE@2Z*w)0(kCthO&_yM*%UGJida|6vZOsz9i`{A4 z1$9H>I3e*5a7E%z;ItDr1R99zIB?n@up5tm#cASJMdEqsR3t zH_x9h=3Wcm_x@lQCjBb$>rZsfGfaX`7pUNP0|>BNcQy@27j~`DP;&%@R-R>t_EH zY5#DfIfiA50!?FcQqSsssrMXP6q>D0W4+O)u7;N1G^mjlerLqf*Sd8n#b4rC%P{-; zL;ZnnI;uZe?>ZTB4cFB0-#*Z)MbmOpQ9fB80t5{kPKm`(#RH z+7x$bBlT=B_I9gzXS+e*JuH!2^*zBQ6IHr+yn3m3A35Z^lnT2~)v@9F{%?UieS9&^^E~^_mKT#ev%Qa8FrOe2d3FCH2>G)dhpVX=6goy2&8EbW*Pk^A zQ8;m|1J$O8N`Y%S`0Q!}TUQS(+i;8rBo89aWPEyTnL0(MPq)Zd!?@~px_>zAnwl{X zg-+uxh1($Hl4xJE)!x83i0#f5+NO43?^{l6gOR@H#mVUbyVk>5E3}bL@jS-rk?v@Q zsk#oiYnT+i29ILf*@sD17VthV59RN5_9l9CwG~!q6|K zEu6YBnRH`1`OXt`*;h3pHgoHdUL{_1$o1W4eD`pl>F21p&-qCzI6wddKmY_l00ck) z1V8`;KmY_l00hnvf$;s0tUD*X{&9|07E=KM5C8!X009sH0T2KI5C8!X0D)No!9GCF zeeoar0JD5h0|Fob0w4eaAOHd&00JNY0w4ea=ZQeD55VDlfb+Dfm9ny^Auvfmp@LmvLdg@%Jme-tdnZh zQ}wf=s9MiWRiFlGQ(~3n@@l2HGRHA>Qm&{&?Vdi)K%$z51Ny@MfgYz>uGQqyYL(-d z8>CtGhV4?~0o6}Rpt|V>V)aTDxma9H`ZHD39<`&4DN%k3)k`A~r>a!SN?B1798;qq sWp79~dBQkJLab78MXppUEXQ0YrBXPamx{tZc`Uh7tW{TEC4N)>0q|*!i2wiq diff --git a/web/src/Sidebar.jsx b/web/src/Sidebar.jsx index 040ec6f2d..c7d937da9 100644 --- a/web/src/Sidebar.jsx +++ b/web/src/Sidebar.jsx @@ -44,6 +44,7 @@ export default function Sidebar() { {birdseye?.enabled ? : null} +
diff --git a/web/src/app.tsx b/web/src/app.tsx index 09d9bbfb8..536cc82e3 100644 --- a/web/src/app.tsx +++ b/web/src/app.tsx @@ -35,6 +35,7 @@ export default function App() { path="/recording/:camera/:date?/:hour?/:minute?/:second?" getComponent={Routes.getRecording} /> + diff --git a/web/src/routes/Storage.jsx b/web/src/routes/Storage.jsx new file mode 100644 index 000000000..afb9cbe2c --- /dev/null +++ b/web/src/routes/Storage.jsx @@ -0,0 +1,109 @@ +import { h, Fragment } from 'preact'; +import ActivityIndicator from '../components/ActivityIndicator'; +import Heading from '../components/Heading'; +import { useWs } from '../api/ws'; +import useSWR from 'swr'; +import { Table, Tbody, Thead, Tr, Th, Td } from '../components/Table'; + +const emptyObject = Object.freeze({}); + +export default function Storage() { + const { data: storage } = useSWR('recordings/storage'); + + const { + value: { payload: stats }, + } = useWs('stats'); + const { data: initialStats } = useSWR('stats'); + + const { service } = stats || initialStats || emptyObject; + + return ( +
+ Storage + + {(!service || !storage) ? ( +
+ +
+ ) : ( + + Overview +
+
+
Data
+
+ + + + + + + + + + + + + + + +
LocationUsed MBTotal MB
Snapshots & Recordings{service['storage']['/media/frigate/recordings']['used']}{service['storage']['/media/frigate/recordings']['total']}
+
+
+
+
Memory
+
+ + + + + + + + + + + + + + + + + + + + +
LocationUsed MBTotal MB
/dev/shm{service['storage']['/dev/shm']['used']}{service['storage']['/dev/shm']['total']}
/tmp/cache{service['storage']['/tmp/cache']['used']}{service['storage']['/tmp/cache']['total']}
+
+
+
+ + Cameras +
+ {Object.entries(storage).map(([name, camera]) => ( +
+
{name}
+
+ + + + + + + + + + + + + +
UsageStream Bandwidth
{Math.round(camera['usage_percent'] ?? 0)}%{camera['bandwidth']} MB/hr
+
+
+ ))} +
+ + )} +
+ ); +} diff --git a/web/src/routes/index.js b/web/src/routes/index.js index 1a8af6384..39c9ed05c 100644 --- a/web/src/routes/index.js +++ b/web/src/routes/index.js @@ -33,6 +33,11 @@ export async function getSystem(_url, _cb, _props) { return module.default; } +export async function getStorage(_url, _cb, _props) { + const module = await import('./Storage.jsx'); + return module.default; +} + export async function getStyleGuide(_url, _cb, _props) { const module = await import('./StyleGuide.jsx'); return module.default;