From ebdf36e0b93a545151228710789b2298fe16ab74 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Sun, 20 Nov 2022 06:36:01 -0700 Subject: [PATCH] Refactor mqtt to handle reconnects and not cause frigate to stop. (#4440) * Refactor mqtt client * Protect callback method * Use async to handle reconnects * Set types and clenup * Don't set connected until rc code is checked --- frigate/app.py | 4 +- frigate/mqtt.py | 359 +++++++++++++++++++++-------------- frigate/object_processing.py | 13 +- frigate/stats.py | 4 +- 4 files changed, 230 insertions(+), 150 deletions(-) diff --git a/frigate/app.py b/frigate/app.py index f72093546..d3bae31b6 100644 --- a/frigate/app.py +++ b/frigate/app.py @@ -20,7 +20,7 @@ from frigate.events import EventCleanup, EventProcessor from frigate.http import create_app from frigate.log import log_process, root_configurer from frigate.models import Event, Recordings -from frigate.mqtt import MqttSocketRelay, create_mqtt_client +from frigate.mqtt import FrigateMqttClient, MqttSocketRelay from frigate.object_processing import TrackedObjectProcessor from frigate.output import output_frames from frigate.plus import PlusApi @@ -169,7 +169,7 @@ class FrigateApp: self.restream.add_cameras() def init_mqtt(self) -> None: - self.mqtt_client = create_mqtt_client(self.config, self.camera_metrics) + self.mqtt_client = FrigateMqttClient(self.config, self.camera_metrics) def start_mqtt_relay(self) -> None: self.mqtt_relay = MqttSocketRelay( diff --git a/frigate/mqtt.py b/frigate/mqtt.py index 82e1c4f0f..53a073d5e 100644 --- a/frigate/mqtt.py +++ b/frigate/mqtt.py @@ -1,3 +1,4 @@ +import datetime import json import logging import threading @@ -13,21 +14,82 @@ from ws4py.server.wsgiutils import WebSocketWSGIApplication from ws4py.websocket import WebSocket from frigate.config import FrigateConfig +from frigate.types import CameraMetricsTypes from frigate.util import restart_frigate logger = logging.getLogger(__name__) -def create_mqtt_client(config: FrigateConfig, camera_metrics): - mqtt_config = config.mqtt +class FrigateMqttClient: + """Frigate wrapper for mqtt client.""" - def on_recordings_command(client, userdata, message): + def __init__( + self, config: FrigateConfig, camera_metrics: dict[str, CameraMetricsTypes] + ) -> None: + self.config = config + self.mqtt_config = config.mqtt + self.camera_metrics = camera_metrics + self.connected: bool = False + self._start() + + def _set_initial_topics(self) -> None: + """Set initial state topics.""" + for camera_name, camera in self.config.cameras.items(): + self.publish( + f"{self.mqtt_config.topic_prefix}/{camera_name}/recordings/state", + "ON" if camera.record.enabled else "OFF", + retain=True, + ) + self.publish( + f"{self.mqtt_config.topic_prefix}/{camera_name}/snapshots/state", + "ON" if camera.snapshots.enabled else "OFF", + retain=True, + ) + self.publish( + f"{self.mqtt_config.topic_prefix}/{camera_name}/detect/state", + "ON" if camera.detect.enabled else "OFF", + retain=True, + ) + self.publish( + f"{self.mqtt_config.topic_prefix}/{camera_name}/motion/state", + "ON", + retain=True, + ) + self.publish( + f"{self.mqtt_config.topic_prefix}/{camera_name}/improve_contrast/state", + "ON" if camera.motion.improve_contrast else "OFF", + retain=True, + ) + self.publish( + f"{self.mqtt_config.topic_prefix}/{camera_name}/motion_threshold/state", + camera.motion.threshold, + retain=True, + ) + self.publish( + f"{self.mqtt_config.topic_prefix}/{camera_name}/motion_contour_area/state", + camera.motion.contour_area, + retain=True, + ) + self.publish( + f"{self.mqtt_config.topic_prefix}/{camera_name}/motion", + "OFF", + retain=False, + ) + + self.publish( + self.mqtt_config.topic_prefix + "/available", "online", retain=True + ) + + def on_recordings_command( + self, client: mqtt.Client, userdata, message: mqtt.MQTTMessage + ) -> None: + """Callback for recordings topic.""" payload = message.payload.decode() logger.debug(f"on_recordings_toggle: {message.topic} {payload}") camera_name = message.topic.split("/")[-3] - record_settings = config.cameras[camera_name].record + record_settings = self.config.cameras[camera_name].record if payload == "ON": if not record_settings.enabled: @@ -41,15 +103,18 @@ def create_mqtt_client(config: FrigateConfig, camera_metrics): logger.warning(f"Received unsupported value at {message.topic}: {payload}") state_topic = f"{message.topic[:-4]}/state" - client.publish(state_topic, payload, retain=True) + self.publish(state_topic, payload, retain=True) - def on_snapshots_command(client, userdata, message): + def on_snapshots_command( + self, client: mqtt.Client, userdata, message: mqtt.MQTTMessage + ) -> None: + """Callback for snapshots topic.""" payload = message.payload.decode() logger.debug(f"on_snapshots_toggle: {message.topic} {payload}") camera_name = message.topic.split("/")[-3] - snapshots_settings = config.cameras[camera_name].snapshots + snapshots_settings = self.config.cameras[camera_name].snapshots if payload == "ON": if not snapshots_settings.enabled: @@ -63,91 +128,107 @@ def create_mqtt_client(config: FrigateConfig, camera_metrics): logger.warning(f"Received unsupported value at {message.topic}: {payload}") state_topic = f"{message.topic[:-4]}/state" - client.publish(state_topic, payload, retain=True) + self.publish(state_topic, payload, retain=True) - def on_detect_command(client, userdata, message): + def on_detect_command( + self, client: mqtt.Client, userdata, message: mqtt.MQTTMessage + ) -> None: + """Callback for detect topic.""" payload = message.payload.decode() logger.debug(f"on_detect_toggle: {message.topic} {payload}") camera_name = message.topic.split("/")[-3] - detect_settings = config.cameras[camera_name].detect + detect_settings = self.config.cameras[camera_name].detect if payload == "ON": - if not camera_metrics[camera_name]["detection_enabled"].value: + if not self.camera_metrics[camera_name]["detection_enabled"].value: logger.info(f"Turning on detection for {camera_name} via mqtt") - camera_metrics[camera_name]["detection_enabled"].value = True + self.camera_metrics[camera_name]["detection_enabled"].value = True detect_settings.enabled = True - if not camera_metrics[camera_name]["motion_enabled"].value: + if not self.camera_metrics[camera_name]["motion_enabled"].value: logger.info( f"Turning on motion for {camera_name} due to detection being enabled." ) - camera_metrics[camera_name]["motion_enabled"].value = True + self.camera_metrics[camera_name]["motion_enabled"].value = True state_topic = f"{message.topic[:-11]}/motion/state" - client.publish(state_topic, payload, retain=True) + self.publish(state_topic, payload, retain=True) elif payload == "OFF": - if camera_metrics[camera_name]["detection_enabled"].value: + if self.camera_metrics[camera_name]["detection_enabled"].value: logger.info(f"Turning off detection for {camera_name} via mqtt") - camera_metrics[camera_name]["detection_enabled"].value = False + self.camera_metrics[camera_name]["detection_enabled"].value = False detect_settings.enabled = False else: logger.warning(f"Received unsupported value at {message.topic}: {payload}") state_topic = f"{message.topic[:-4]}/state" - client.publish(state_topic, payload, retain=True) + self.publish(state_topic, payload, retain=True) - def on_motion_command(client, userdata, message): + def on_motion_command( + self, client: mqtt.Client, userdata, message: mqtt.MQTTMessage + ) -> None: + """Callback for motion topic.""" payload = message.payload.decode() logger.debug(f"on_motion_toggle: {message.topic} {payload}") camera_name = message.topic.split("/")[-3] if payload == "ON": - if not camera_metrics[camera_name]["motion_enabled"].value: + if not self.camera_metrics[camera_name]["motion_enabled"].value: logger.info(f"Turning on motion for {camera_name} via mqtt") - camera_metrics[camera_name]["motion_enabled"].value = True + self.camera_metrics[camera_name]["motion_enabled"].value = True elif payload == "OFF": - if camera_metrics[camera_name]["detection_enabled"].value: + if self.camera_metrics[camera_name]["detection_enabled"].value: logger.error( f"Turning off motion is not allowed when detection is enabled." ) return - if camera_metrics[camera_name]["motion_enabled"].value: + if self.camera_metrics[camera_name]["motion_enabled"].value: logger.info(f"Turning off motion for {camera_name} via mqtt") - camera_metrics[camera_name]["motion_enabled"].value = False + self.camera_metrics[camera_name]["motion_enabled"].value = False else: logger.warning(f"Received unsupported value at {message.topic}: {payload}") state_topic = f"{message.topic[:-4]}/state" - client.publish(state_topic, payload, retain=True) + self.publish(state_topic, payload, retain=True) - def on_improve_contrast_command(client, userdata, message): + def on_improve_contrast_command( + self, client: mqtt.Client, userdata, message: mqtt.MQTTMessage + ) -> None: + """Callback for improve_contrast topic.""" payload = message.payload.decode() logger.debug(f"on_improve_contrast_toggle: {message.topic} {payload}") camera_name = message.topic.split("/")[-3] - motion_settings = config.cameras[camera_name].motion + motion_settings = self.config.cameras[camera_name].motion if payload == "ON": - if not camera_metrics[camera_name]["improve_contrast_enabled"].value: + if not self.camera_metrics[camera_name]["improve_contrast_enabled"].value: logger.info(f"Turning on improve contrast for {camera_name} via mqtt") - camera_metrics[camera_name]["improve_contrast_enabled"].value = True + self.camera_metrics[camera_name][ + "improve_contrast_enabled" + ].value = True motion_settings.improve_contrast = True elif payload == "OFF": - if camera_metrics[camera_name]["improve_contrast_enabled"].value: + if self.camera_metrics[camera_name]["improve_contrast_enabled"].value: logger.info(f"Turning off improve contrast for {camera_name} via mqtt") - camera_metrics[camera_name]["improve_contrast_enabled"].value = False + self.camera_metrics[camera_name][ + "improve_contrast_enabled" + ].value = False motion_settings.improve_contrast = False else: logger.warning(f"Received unsupported value at {message.topic}: {payload}") state_topic = f"{message.topic[:-4]}/state" - client.publish(state_topic, payload, retain=True) + self.publish(state_topic, payload, retain=True) - def on_motion_threshold_command(client, userdata, message): + def on_motion_threshold_command( + self, client: mqtt.Client, userdata, message: mqtt.MQTTMessage + ) -> None: + """Callback for motion threshold topic.""" try: payload = int(message.payload.decode()) except ValueError: @@ -160,16 +241,19 @@ def create_mqtt_client(config: FrigateConfig, camera_metrics): camera_name = message.topic.split("/")[-3] - motion_settings = config.cameras[camera_name].motion + motion_settings = self.config.cameras[camera_name].motion logger.info(f"Setting motion threshold for {camera_name} via mqtt: {payload}") - camera_metrics[camera_name]["motion_threshold"].value = payload + self.camera_metrics[camera_name]["motion_threshold"].value = payload motion_settings.threshold = payload state_topic = f"{message.topic[:-4]}/state" - client.publish(state_topic, payload, retain=True) + self.publish(state_topic, payload, retain=True) - def on_motion_contour_area_command(client, userdata, message): + def on_motion_contour_area_command( + self, client: mqtt.Client, userdata, message: mqtt.MQTTMessage + ) -> None: + """Callback for motion contour topic.""" try: payload = int(message.payload.decode()) except ValueError: @@ -182,21 +266,25 @@ def create_mqtt_client(config: FrigateConfig, camera_metrics): camera_name = message.topic.split("/")[-3] - motion_settings = config.cameras[camera_name].motion + motion_settings = self.config.cameras[camera_name].motion logger.info( f"Setting motion contour area for {camera_name} via mqtt: {payload}" ) - camera_metrics[camera_name]["motion_contour_area"].value = payload + self.camera_metrics[camera_name]["motion_contour_area"].value = payload motion_settings.contour_area = payload state_topic = f"{message.topic[:-4]}/state" - client.publish(state_topic, payload, retain=True) + self.publish(state_topic, payload, retain=True) - def on_restart_command(client, userdata, message): + def on_restart_command( + client: mqtt.Client, userdata, message: mqtt.MQTTMessage + ) -> None: + """Callback to restart frigate.""" restart_frigate() - def on_connect(client, userdata, flags, rc): + def _on_connect(self, client: mqtt.Client, userdata, flags, rc) -> None: + """Mqtt connection callback.""" threading.current_thread().name = "mqtt" if rc != 0: if rc == 3: @@ -215,118 +303,103 @@ def create_mqtt_client(config: FrigateConfig, camera_metrics): + str(rc) ) + self.connected = True logger.debug("MQTT connected") - client.subscribe(f"{mqtt_config.topic_prefix}/#") - client.publish(mqtt_config.topic_prefix + "/available", "online", retain=True) + client.subscribe(f"{self.mqtt_config.topic_prefix}/#") + self._set_initial_topics() - client = mqtt.Client(client_id=mqtt_config.client_id) - client.on_connect = on_connect - client.will_set( - mqtt_config.topic_prefix + "/available", payload="offline", qos=1, retain=True - ) + def _on_disconnect(self, client: mqtt.Client, userdata, flags, rc) -> None: + """Mqtt disconnection callback.""" + self.connected = False + logger.error("MQTT disconnected") - # register callbacks - for name in config.cameras.keys(): - client.message_callback_add( - f"{mqtt_config.topic_prefix}/{name}/recordings/set", on_recordings_command - ) - client.message_callback_add( - f"{mqtt_config.topic_prefix}/{name}/snapshots/set", on_snapshots_command - ) - client.message_callback_add( - f"{mqtt_config.topic_prefix}/{name}/detect/set", on_detect_command - ) - client.message_callback_add( - f"{mqtt_config.topic_prefix}/{name}/motion/set", on_motion_command - ) - client.message_callback_add( - f"{mqtt_config.topic_prefix}/{name}/improve_contrast/set", - on_improve_contrast_command, - ) - client.message_callback_add( - f"{mqtt_config.topic_prefix}/{name}/motion_threshold/set", - on_motion_threshold_command, - ) - client.message_callback_add( - f"{mqtt_config.topic_prefix}/{name}/motion_contour_area/set", - on_motion_contour_area_command, + def _start(self) -> None: + """Start mqtt client.""" + self.client = mqtt.Client(client_id=self.mqtt_config.client_id) + self.client.on_connect = self._on_connect + self.client.will_set( + self.mqtt_config.topic_prefix + "/available", + payload="offline", + qos=1, + retain=True, ) - client.message_callback_add( - f"{mqtt_config.topic_prefix}/restart", on_restart_command - ) - - if not mqtt_config.tls_ca_certs is None: - if ( - not mqtt_config.tls_client_cert is None - and not mqtt_config.tls_client_key is None - ): - client.tls_set( - mqtt_config.tls_ca_certs, - mqtt_config.tls_client_cert, - mqtt_config.tls_client_key, + # register callbacks + for name in self.config.cameras.keys(): + self.client.message_callback_add( + f"{self.mqtt_config.topic_prefix}/{name}/recordings/set", + self.on_recordings_command, + ) + self.client.message_callback_add( + f"{self.mqtt_config.topic_prefix}/{name}/snapshots/set", + self.on_snapshots_command, + ) + self.client.message_callback_add( + f"{self.mqtt_config.topic_prefix}/{name}/detect/set", + self.on_detect_command, + ) + self.client.message_callback_add( + f"{self.mqtt_config.topic_prefix}/{name}/motion/set", + self.on_motion_command, + ) + self.client.message_callback_add( + f"{self.mqtt_config.topic_prefix}/{name}/improve_contrast/set", + self.on_improve_contrast_command, + ) + self.client.message_callback_add( + f"{self.mqtt_config.topic_prefix}/{name}/motion_threshold/set", + self.on_motion_threshold_command, + ) + self.client.message_callback_add( + f"{self.mqtt_config.topic_prefix}/{name}/motion_contour_area/set", + self.on_motion_contour_area_command, ) - else: - client.tls_set(mqtt_config.tls_ca_certs) - if not mqtt_config.tls_insecure is None: - client.tls_insecure_set(mqtt_config.tls_insecure) - if not mqtt_config.user is None: - client.username_pw_set(mqtt_config.user, password=mqtt_config.password) - try: - client.connect(mqtt_config.host, mqtt_config.port, 60) - except Exception as e: - logger.error(f"Unable to connect to MQTT server: {e}") - raise - client.loop_start() - - for name in config.cameras.keys(): - client.publish( - f"{mqtt_config.topic_prefix}/{name}/recordings/state", - "ON" if config.cameras[name].record.enabled else "OFF", - retain=True, - ) - client.publish( - f"{mqtt_config.topic_prefix}/{name}/snapshots/state", - "ON" if config.cameras[name].snapshots.enabled else "OFF", - retain=True, - ) - client.publish( - f"{mqtt_config.topic_prefix}/{name}/detect/state", - "ON" if config.cameras[name].detect.enabled else "OFF", - retain=True, - ) - client.publish( - f"{mqtt_config.topic_prefix}/{name}/motion/state", - "ON", - retain=True, - ) - client.publish( - f"{mqtt_config.topic_prefix}/{name}/improve_contrast/state", - "ON" if config.cameras[name].motion.improve_contrast else "OFF", - retain=True, - ) - client.publish( - f"{mqtt_config.topic_prefix}/{name}/motion_threshold/state", - config.cameras[name].motion.threshold, - retain=True, - ) - client.publish( - f"{mqtt_config.topic_prefix}/{name}/motion_contour_area/state", - config.cameras[name].motion.contour_area, - retain=True, - ) - client.publish( - f"{mqtt_config.topic_prefix}/{name}/motion", - "OFF", - retain=False, + self.client.message_callback_add( + f"{self.mqtt_config.topic_prefix}/restart", self.on_restart_command ) - return client + if not self.mqtt_config.tls_ca_certs is None: + if ( + not self.mqtt_config.tls_client_cert is None + and not self.mqtt_config.tls_client_key is None + ): + self.client.tls_set( + self.mqtt_config.tls_ca_certs, + self.mqtt_config.tls_client_cert, + self.mqtt_config.tls_client_key, + ) + else: + self.client.tls_set(self.mqtt_config.tls_ca_certs) + if not self.mqtt_config.tls_insecure is None: + self.client.tls_insecure_set(self.mqtt_config.tls_insecure) + if not self.mqtt_config.user is None: + self.client.username_pw_set( + self.mqtt_config.user, password=self.mqtt_config.password + ) + try: + # https://stackoverflow.com/a/55390477 + # with connect_async, retries are handled automatically + self.client.connect_async(self.mqtt_config.host, self.mqtt_config.port, 60) + self.client.loop_start() + except Exception as e: + logger.error(f"Unable to connect to MQTT server: {e}") + return + + def publish(self, topic: str, payload, retain: bool = False) -> None: + """Wrapper for publishing when client is in valid state.""" + if not self.connected: + logger.error(f"Unable to publish to {topic}: client is not connected") + return + + self.client.publish(topic, payload, retain=retain) + + def add_topic_callback(self, topic: str, callback) -> None: + self.client.message_callback_add(topic, callback) class MqttSocketRelay: - def __init__(self, mqtt_client, topic_prefix): + def __init__(self, mqtt_client: FrigateMqttClient, topic_prefix: str): self.mqtt_client = mqtt_client self.topic_prefix = topic_prefix @@ -389,7 +462,7 @@ class MqttSocketRelay: self.websocket_server.manager.broadcast(ws_message) - self.mqtt_client.message_callback_add(f"{self.topic_prefix}/#", send) + self.mqtt_client.add_topic_callback(f"{self.topic_prefix}/#", send) self.websocket_thread.start() diff --git a/frigate/object_processing.py b/frigate/object_processing.py index 56acf7667..575e697e5 100644 --- a/frigate/object_processing.py +++ b/frigate/object_processing.py @@ -12,8 +12,15 @@ from typing import Callable import cv2 import numpy as np -from frigate.config import CameraConfig, SnapshotsConfig, RecordConfig, FrigateConfig +from frigate.config import ( + CameraConfig, + MqttConfig, + SnapshotsConfig, + RecordConfig, + FrigateConfig, +) from frigate.const import CLIPS_DIR +from frigate.mqtt import FrigateMqttClient from frigate.util import ( SharedMemoryFrameManager, calculate_region, @@ -626,7 +633,7 @@ class TrackedObjectProcessor(threading.Thread): def __init__( self, config: FrigateConfig, - client, + client: FrigateMqttClient, topic_prefix, tracked_objects_queue, event_queue, @@ -724,7 +731,7 @@ class TrackedObjectProcessor(threading.Thread): self.event_queue.put(("end", camera, obj.to_dict(include_thumbnail=True))) def snapshot(camera, obj: TrackedObject, current_frame_time): - mqtt_config = self.config.cameras[camera].mqtt + mqtt_config: MqttConfig = self.config.cameras[camera].mqtt if mqtt_config.enabled and self.should_mqtt_snapshot(camera, obj): jpg_bytes = obj.get_jpg_bytes( timestamp=mqtt_config.timestamp, diff --git a/frigate/stats.py b/frigate/stats.py index 62c8eeec6..e8b278e27 100644 --- a/frigate/stats.py +++ b/frigate/stats.py @@ -7,11 +7,11 @@ import shutil import os import requests from typing import Optional, Any -from paho.mqtt.client import Client from multiprocessing.synchronize import Event as MpEvent from frigate.config import FrigateConfig from frigate.const import RECORD_DIR, CLIPS_DIR, CACHE_DIR +from frigate.mqtt import FrigateMqttClient from frigate.types import StatsTrackingTypes, CameraMetricsTypes from frigate.version import VERSION from frigate.util import get_cpu_stats @@ -146,7 +146,7 @@ class StatsEmitter(threading.Thread): self, config: FrigateConfig, stats_tracking: StatsTrackingTypes, - mqtt_client: Client, + mqtt_client: FrigateMqttClient, topic_prefix: str, stop_event: MpEvent, ):