From d7a446e0f669d6c3675629003fcf1eae5b89f6b8 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Fri, 13 Jun 2025 09:43:38 -0500 Subject: [PATCH] Ensure logging config is propagated to forked processes (#18704) * Move log level initialization to log * Use logger config * Formatting * Fix config order * Set process names --------- Co-authored-by: Nicolas Mowen --- frigate/app.py | 1 + frigate/config/logger.py | 29 ++++------------------------- frigate/embeddings/__init__.py | 2 +- frigate/events/audio.py | 2 +- frigate/log.py | 26 ++++++++++++++++++++++++++ frigate/object_detection/base.py | 8 +++++++- frigate/output/output.py | 2 +- frigate/record/record.py | 2 +- frigate/review/review.py | 2 +- frigate/util/process.py | 12 +++++++++++- 10 files changed, 54 insertions(+), 32 deletions(-) diff --git a/frigate/app.py b/frigate/app.py index f78bd561c..8f7fa71ef 100644 --- a/frigate/app.py +++ b/frigate/app.py @@ -385,6 +385,7 @@ class FrigateApp: name, self.detection_queue, list(self.config.cameras.keys()), + self.config, detector_config, ) diff --git a/frigate/config/logger.py b/frigate/config/logger.py index a3eed23d0..0ba3e6972 100644 --- a/frigate/config/logger.py +++ b/frigate/config/logger.py @@ -1,20 +1,11 @@ -import logging -from enum import Enum - from pydantic import Field, ValidationInfo, model_validator from typing_extensions import Self +from frigate.log import LogLevel, apply_log_levels + from .base import FrigateBaseModel -__all__ = ["LoggerConfig", "LogLevel"] - - -class LogLevel(str, Enum): - debug = "debug" - info = "info" - warning = "warning" - error = "error" - critical = "critical" +__all__ = ["LoggerConfig"] class LoggerConfig(FrigateBaseModel): @@ -26,18 +17,6 @@ class LoggerConfig(FrigateBaseModel): @model_validator(mode="after") def post_validation(self, info: ValidationInfo) -> Self: if isinstance(info.context, dict) and info.context.get("install", False): - logging.getLogger().setLevel(self.default.value.upper()) - - log_levels = { - "absl": LogLevel.error, - "httpx": LogLevel.error, - "tensorflow": LogLevel.error, - "werkzeug": LogLevel.error, - "ws4py": LogLevel.error, - **self.logs, - } - - for log, level in log_levels.items(): - logging.getLogger(log).setLevel(level.value.upper()) + apply_log_levels(self.default.value.upper(), self.logs) return self diff --git a/frigate/embeddings/__init__.py b/frigate/embeddings/__init__.py index 9870c9460..191acf0fd 100644 --- a/frigate/embeddings/__init__.py +++ b/frigate/embeddings/__init__.py @@ -35,7 +35,7 @@ class EmbeddingProcess(FrigateProcess): self.metrics = metrics def run(self) -> None: - self.pre_run_setup() + self.pre_run_setup(self.config.logger) maintainer = EmbeddingMaintainer( self.config, self.metrics, diff --git a/frigate/events/audio.py b/frigate/events/audio.py index 791ba80e4..d7242cf2b 100644 --- a/frigate/events/audio.py +++ b/frigate/events/audio.py @@ -100,7 +100,7 @@ class AudioProcessor(util.Process): self.transcription_model_runner = None def run(self) -> None: - self.pre_run_setup() + self.pre_run_setup(self.config.logger) audio_threads: list[AudioEventMaintainer] = [] threading.current_thread().name = "process:audio_manager" diff --git a/frigate/log.py b/frigate/log.py index f535a278c..2e9c781f1 100644 --- a/frigate/log.py +++ b/frigate/log.py @@ -5,6 +5,7 @@ import os import sys import threading from collections import deque +from enum import Enum from logging.handlers import QueueHandler, QueueListener from multiprocessing.managers import SyncManager from queue import Queue @@ -33,6 +34,15 @@ LOG_HANDLER.addFilter( not in record.getMessage() ) + +class LogLevel(str, Enum): + debug = "debug" + info = "info" + warning = "warning" + error = "error" + critical = "critical" + + log_listener: Optional[QueueListener] = None log_queue: Optional[Queue] = None @@ -61,6 +71,22 @@ def _stop_logging() -> None: log_listener = None +def apply_log_levels(default: str, log_levels: dict[str, LogLevel]) -> None: + logging.getLogger().setLevel(default) + + log_levels = { + "absl": LogLevel.error, + "httpx": LogLevel.error, + "tensorflow": LogLevel.error, + "werkzeug": LogLevel.error, + "ws4py": LogLevel.error, + **log_levels, + } + + for log, level in log_levels.items(): + logging.getLogger(log).setLevel(level.value.upper()) + + # When a multiprocessing.Process exits, python tries to flush stdout and stderr. However, if the # process is created after a thread (for example a logging thread) is created and the process fork # happens while an internal lock is held, the stdout/err flush can cause a deadlock. diff --git a/frigate/object_detection/base.py b/frigate/object_detection/base.py index d203e8574..e86b1b036 100644 --- a/frigate/object_detection/base.py +++ b/frigate/object_detection/base.py @@ -12,6 +12,7 @@ from frigate.comms.object_detector_signaler import ( ObjectDetectorPublisher, ObjectDetectorSubscriber, ) +from frigate.config import FrigateConfig from frigate.detectors import create_detector from frigate.detectors.detector_config import ( BaseDetectorConfig, @@ -92,6 +93,7 @@ class DetectorRunner(util.Process): cameras: list[str], avg_speed: Value, start_time: Value, + config: FrigateConfig, detector_config: BaseDetectorConfig, ) -> None: super().__init__(name=name, daemon=True) @@ -99,6 +101,7 @@ class DetectorRunner(util.Process): self.cameras = cameras self.avg_speed = avg_speed self.start_time = start_time + self.config = config self.detector_config = detector_config self.outputs: dict = {} @@ -108,7 +111,7 @@ class DetectorRunner(util.Process): self.outputs[name] = {"shm": out_shm, "np": out_np} def run(self) -> None: - self.pre_run_setup() + self.pre_run_setup(self.config.logger) frame_manager = SharedMemoryFrameManager() object_detector = LocalObjectDetector(detector_config=self.detector_config) @@ -161,6 +164,7 @@ class ObjectDetectProcess: name: str, detection_queue: Queue, cameras: list[str], + config: FrigateConfig, detector_config: BaseDetectorConfig, ): self.name = name @@ -169,6 +173,7 @@ class ObjectDetectProcess: self.avg_inference_speed = Value("d", 0.01) self.detection_start = Value("d", 0.0) self.detect_process: util.Process | None = None + self.config = config self.detector_config = detector_config self.start_or_restart() @@ -195,6 +200,7 @@ class ObjectDetectProcess: self.cameras, self.avg_inference_speed, self.detection_start, + self.config, self.detector_config, ) self.detect_process.start() diff --git a/frigate/output/output.py b/frigate/output/output.py index 8c60e51c7..0cb8a649f 100644 --- a/frigate/output/output.py +++ b/frigate/output/output.py @@ -77,7 +77,7 @@ class OutputProcess(util.Process): self.config = config def run(self) -> None: - self.pre_run_setup() + self.pre_run_setup(self.config.logger) frame_manager = SharedMemoryFrameManager() diff --git a/frigate/record/record.py b/frigate/record/record.py index 40a943a43..153560a11 100644 --- a/frigate/record/record.py +++ b/frigate/record/record.py @@ -18,7 +18,7 @@ class RecordProcess(FrigateProcess): self.config = config def run(self) -> None: - self.pre_run_setup() + self.pre_run_setup(self.config.logger) db = SqliteQueueDatabase( self.config.database.path, pragmas={ diff --git a/frigate/review/review.py b/frigate/review/review.py index 00910e439..e687f4f45 100644 --- a/frigate/review/review.py +++ b/frigate/review/review.py @@ -15,7 +15,7 @@ class ReviewProcess(util.Process): self.config = config def run(self) -> None: - self.pre_run_setup() + self.pre_run_setup(self.config.logger) maintainer = ReviewSegmentMaintainer( self.config, self.stop_event, diff --git a/frigate/util/process.py b/frigate/util/process.py index 3501e585e..6e3459c6b 100644 --- a/frigate/util/process.py +++ b/frigate/util/process.py @@ -7,7 +7,10 @@ import threading from logging.handlers import QueueHandler from typing import Callable, Optional +from setproctitle import setproctitle + import frigate.log +from frigate.config.logger import LoggerConfig class BaseProcess(mp.Process): @@ -50,7 +53,9 @@ class Process(BaseProcess): def before_start(self) -> None: self.__log_queue = frigate.log.log_listener.queue - def pre_run_setup(self) -> None: + def pre_run_setup(self, logConfig: LoggerConfig | None = None) -> None: + setproctitle(self.name) + threading.current_thread().name = f"process:{self.name}" faulthandler.enable() def receiveSignal(signalNumber, frame): @@ -68,3 +73,8 @@ class Process(BaseProcess): self.logger = logging.getLogger(self.name) logging.basicConfig(handlers=[], force=True) logging.getLogger().addHandler(QueueHandler(self.__log_queue)) + + if logConfig: + frigate.log.apply_log_levels( + logConfig.default.value.upper(), logConfig.logs + )