diff --git a/frigate/camera/maintainer.py b/frigate/camera/maintainer.py index 5bd97136c..865fe4725 100644 --- a/frigate/camera/maintainer.py +++ b/frigate/camera/maintainer.py @@ -2,8 +2,6 @@ import logging import multiprocessing as mp -import os -import shutil import threading from multiprocessing import Queue from multiprocessing.managers import DictProxy, SyncManager @@ -16,11 +14,11 @@ from frigate.config.camera.updater import ( CameraConfigUpdateEnum, CameraConfigUpdateSubscriber, ) -from frigate.const import SHM_FRAMES_VAR from frigate.models import Regions from frigate.util.builtin import empty_and_close_queue from frigate.util.image import SharedMemoryFrameManager, UntrackedSharedMemory from frigate.util.object import get_camera_regions_grid +from frigate.util.services import calculate_shm_requirements from frigate.video import CameraCapture, CameraTracker logger = logging.getLogger(__name__) @@ -74,53 +72,25 @@ class CameraMaintainer(threading.Thread): ) def __calculate_shm_frame_count(self) -> int: - total_shm = round(shutil.disk_usage("/dev/shm").total / pow(2, 20), 1) + shm_stats = calculate_shm_requirements(self.config) - # required for log files + nginx cache - min_req_shm = 40 + 10 - - if self.config.birdseye.restream: - min_req_shm += 8 - - available_shm = total_shm - min_req_shm - cam_total_frame_size = 0.0 - - for camera in self.config.cameras.values(): - if ( - camera.enabled_in_config - and camera.detect.width - and camera.detect.height - ): - cam_total_frame_size += round( - (camera.detect.width * camera.detect.height * 1.5 + 270480) - / 1048576, - 1, - ) - - # leave room for 2 cameras that are added dynamically, if a user wants to add more cameras they may need to increase the SHM size and restart after adding them. - cam_total_frame_size += 2 * round( - (1280 * 720 * 1.5 + 270480) / 1048576, - 1, - ) - - if cam_total_frame_size == 0.0: + if not shm_stats: + # /dev/shm not available return 0 - shm_frame_count = min( - int(os.environ.get(SHM_FRAMES_VAR, "50")), - int(available_shm / (cam_total_frame_size)), - ) - logger.debug( - f"Calculated total camera size {available_shm} / {cam_total_frame_size} :: {shm_frame_count} frames for each camera in SHM" + f"Calculated total camera size {shm_stats['available']} / " + f"{shm_stats['camera_frame_size']} :: {shm_stats['shm_frame_count']} " + f"frames for each camera in SHM" ) - if shm_frame_count < 20: + if shm_stats["shm_frame_count"] < 20: logger.warning( - f"The current SHM size of {total_shm}MB is too small, recommend increasing it to at least {round(min_req_shm + cam_total_frame_size * 20)}MB." + f"The current SHM size of {shm_stats['total']}MB is too small, " + f"recommend increasing it to at least {shm_stats['min_shm']}MB." ) - return shm_frame_count + return shm_stats["shm_frame_count"] def __start_camera_processor( self, name: str, config: CameraConfig, runtime: bool = False diff --git a/frigate/stats/util.py b/frigate/stats/util.py index ee93bb6e6..6d20f8f9a 100644 --- a/frigate/stats/util.py +++ b/frigate/stats/util.py @@ -8,7 +8,6 @@ from json import JSONDecodeError from multiprocessing.managers import DictProxy from typing import Any, Optional -import psutil import requests from requests.exceptions import RequestException @@ -18,9 +17,11 @@ from frigate.data_processing.types import DataProcessorMetrics from frigate.object_detection.base import ObjectDetectProcess from frigate.types import StatsTrackingTypes from frigate.util.services import ( + calculate_shm_requirements, get_amd_gpu_stats, get_bandwidth_stats, get_cpu_stats, + get_fs_type, get_intel_gpu_stats, get_jetson_stats, get_nvidia_gpu_stats, @@ -70,16 +71,6 @@ def stats_init( return stats_tracking -def get_fs_type(path: str) -> str: - bestMatch = "" - fsType = "" - for part in psutil.disk_partitions(all=True): - if path.startswith(part.mountpoint) and len(bestMatch) < len(part.mountpoint): - fsType = part.fstype - bestMatch = part.mountpoint - return fsType - - def read_temperature(path: str) -> Optional[float]: if os.path.isfile(path): with open(path) as f: @@ -389,7 +380,7 @@ def stats_snapshot( "last_updated": int(time.time()), } - for path in [RECORD_DIR, CLIPS_DIR, CACHE_DIR, "/dev/shm"]: + for path in [RECORD_DIR, CLIPS_DIR, CACHE_DIR]: try: storage_stats = shutil.disk_usage(path) except (FileNotFoundError, OSError): @@ -403,6 +394,8 @@ def stats_snapshot( "mount_type": get_fs_type(path), } + stats["service"]["storage"]["/dev/shm"] = calculate_shm_requirements(config) + stats["processes"] = {} for name, pid in stats_tracking["processes"].items(): stats["processes"][name] = { diff --git a/frigate/util/services.py b/frigate/util/services.py index 185770eb7..50aa2e2b7 100644 --- a/frigate/util/services.py +++ b/frigate/util/services.py @@ -6,6 +6,7 @@ import logging import os import re import resource +import shutil import signal import subprocess as sp import traceback @@ -22,6 +23,7 @@ from frigate.const import ( DRIVER_ENV_VAR, FFMPEG_HWACCEL_NVIDIA, FFMPEG_HWACCEL_VAAPI, + SHM_FRAMES_VAR, ) from frigate.util.builtin import clean_camera_user_pass, escape_special_characters @@ -768,3 +770,65 @@ def set_file_limit() -> None: logger.debug( f"File limit set. New soft limit: {new_soft}, Hard limit remains: {current_hard}" ) + + +def get_fs_type(path: str) -> str: + bestMatch = "" + fsType = "" + for part in psutil.disk_partitions(all=True): + if path.startswith(part.mountpoint) and len(bestMatch) < len(part.mountpoint): + fsType = part.fstype + bestMatch = part.mountpoint + return fsType + + +def calculate_shm_requirements(config) -> dict: + try: + storage_stats = shutil.disk_usage("/dev/shm") + except (FileNotFoundError, OSError): + return {} + + total_mb = round(storage_stats.total / pow(2, 20), 1) + used_mb = round(storage_stats.used / pow(2, 20), 1) + free_mb = round(storage_stats.free / pow(2, 20), 1) + + # required for log files + nginx cache + min_req_shm = 40 + 10 + + if config.birdseye.restream: + min_req_shm += 8 + + available_shm = total_mb - min_req_shm + cam_total_frame_size = 0.0 + + for camera in config.cameras.values(): + if camera.enabled_in_config and camera.detect.width and camera.detect.height: + cam_total_frame_size += round( + (camera.detect.width * camera.detect.height * 1.5 + 270480) / 1048576, + 1, + ) + + # leave room for 2 cameras that are added dynamically, if a user wants to add more cameras they may need to increase the SHM size and restart after adding them. + cam_total_frame_size += 2 * round( + (1280 * 720 * 1.5 + 270480) / 1048576, + 1, + ) + + shm_frame_count = min( + int(os.environ.get(SHM_FRAMES_VAR, "50")), + int(available_shm / cam_total_frame_size), + ) + + # minimum required shm recommendation + min_shm = round(min_req_shm + cam_total_frame_size * 20) + + return { + "total": total_mb, + "used": used_mb, + "free": free_mb, + "mount_type": get_fs_type("/dev/shm"), + "available": round(available_shm, 1), + "camera_frame_size": cam_total_frame_size, + "shm_frame_count": shm_frame_count, + "min_shm": min_shm, + } diff --git a/web/public/locales/en/views/system.json b/web/public/locales/en/views/system.json index 059f05f9f..c03d10652 100644 --- a/web/public/locales/en/views/system.json +++ b/web/public/locales/en/views/system.json @@ -91,6 +91,11 @@ "tips": "This value represents the total storage used by the recordings in Frigate's database. Frigate does not track storage usage for all files on your disk.", "earliestRecording": "Earliest recording available:" }, + "shm": { + "title": "SHM (shared memory) allocation", + "warning": "The current SHM size of {{total}}MB is too small. Increase it to at least {{min_shm}}MB.", + "readTheDocumentation": "Read the documentation" + }, "cameraStorage": { "title": "Camera Storage", "camera": "Camera", diff --git a/web/src/types/stats.ts b/web/src/types/stats.ts index a196dff18..c98ebe80f 100644 --- a/web/src/types/stats.ts +++ b/web/src/types/stats.ts @@ -79,6 +79,7 @@ export type StorageStats = { total: number; used: number; mount_type: string; + min_shm?: number; }; export type PotentialProblem = { diff --git a/web/src/views/system/StorageMetrics.tsx b/web/src/views/system/StorageMetrics.tsx index 6ae40089a..050badce2 100644 --- a/web/src/views/system/StorageMetrics.tsx +++ b/web/src/views/system/StorageMetrics.tsx @@ -14,6 +14,10 @@ import { useFormattedTimestamp, useTimezone } from "@/hooks/use-date-utils"; import { RecordingsSummary } from "@/types/review"; import { useTranslation } from "react-i18next"; import { TZDate } from "react-day-picker"; +import { Link } from "react-router-dom"; +import { useDocDomain } from "@/hooks/use-doc-domain"; +import { LuExternalLink } from "react-icons/lu"; +import { FaExclamationTriangle } from "react-icons/fa"; type CameraStorage = { [key: string]: { @@ -36,6 +40,7 @@ export default function StorageMetrics({ }); const { t } = useTranslation(["views/system"]); const timezone = useTimezone(config); + const { getLocaleDocUrl } = useDocDomain(); const totalStorage = useMemo(() => { if (!cameraStorage || !stats) { @@ -142,7 +147,46 @@ export default function StorageMetrics({ />