Mirrors/blakeblackshear.frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2026-04-19 23:08:08 +02:00
Code Issues Projects Releases Wiki Activity

More mypy cleanup (#22658)

Browse Source 
* Halfway point for fixing data processing

* Fix mixin types missing

* Cleanup LPR mypy

* Cleanup audio mypy

* Cleanup bird mypy

* Cleanup mypy for custom classification

* remove whisper

* Fix DB typing

* Cleanup events mypy

* Clenaup

* fix type evaluation

* Cleanup

* Fix broken imports
This commit is contained in:
Nicolas Mowen
2026-03-26 12:54:12 -06:00
committed by GitHub
parent 4772e6a2ab
commit 03d0139497
22 changed files with 398 additions and 274 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
frigate/data_processing/post/api.py
View File

@@ -17,7 +17,7 @@ class PostProcessorApi(ABC):
self,
config: FrigateConfig,
metrics: DataProcessorMetrics,
model_runner: DataProcessorModelRunner,
model_runner: DataProcessorModelRunner | None,
) -> None:
self.config = config
self.metrics = metrics
@@ -41,7 +41,7 @@ class PostProcessorApi(ABC):
@abstractmethod
def handle_request(
self, topic: str, request_data: dict[str, Any]
) -> dict[str, Any] | None:
) -> dict[str, Any] | str | None:
"""Handle metadata requests.
Args:
request_data (dict): containing data about requested change to process.

17
frigate/data_processing/post/audio_transcription.py
View File

@@ -4,7 +4,7 @@ import logging
import os
import threading
import time
from typing import Optional
from typing import Any, Optional
from peewee import DoesNotExist
@@ -17,6 +17,7 @@ from frigate.const import (
UPDATE_EVENT_DESCRIPTION,
)
from frigate.data_processing.types import PostProcessDataEnum
from frigate.embeddings.embeddings import Embeddings
from frigate.types import TrackedObjectUpdateTypesEnum
from frigate.util.audio import get_audio_from_recording
@@ -31,7 +32,7 @@ class AudioTranscriptionPostProcessor(PostProcessorApi):
self,
config: FrigateConfig,
requestor: InterProcessRequestor,
embeddings,
embeddings: Embeddings,
metrics: DataProcessorMetrics,
):
super().__init__(config, metrics, None)
@@ -40,7 +41,7 @@ class AudioTranscriptionPostProcessor(PostProcessorApi):
self.embeddings = embeddings
self.recognizer = None
self.transcription_lock = threading.Lock()
self.transcription_thread = None
self.transcription_thread: threading.Thread | None = None
self.transcription_running = False
# faster-whisper handles model downloading automatically
@@ -69,7 +70,7 @@ class AudioTranscriptionPostProcessor(PostProcessorApi):
self.recognizer = None
def process_data(
self, data: dict[str, any], data_type: PostProcessDataEnum
self, data: dict[str, Any], data_type: PostProcessDataEnum
) -> None:
"""Transcribe audio from a recording.
@@ -141,13 +142,13 @@ class AudioTranscriptionPostProcessor(PostProcessorApi):
except Exception as e:
logger.error(f"Error in audio transcription post-processing: {e}")
def __transcribe_audio(self, audio_data: bytes) -> Optional[tuple[str, float]]:
def __transcribe_audio(self, audio_data: bytes) -> Optional[str]:
"""Transcribe WAV audio data using faster-whisper."""
if not self.recognizer:
logger.debug("Recognizer not initialized")
return None
try:
try: # type: ignore[unreachable]
# Save audio data to a temporary wav (faster-whisper expects a file)
temp_wav = os.path.join(CACHE_DIR, f"temp_audio_{int(time.time())}.wav")
with open(temp_wav, "wb") as f:
@@ -176,7 +177,7 @@ class AudioTranscriptionPostProcessor(PostProcessorApi):
logger.error(f"Error transcribing audio: {e}")
return None
def _transcription_wrapper(self, event: dict[str, any]) -> None:
def _transcription_wrapper(self, event: dict[str, Any]) -> None:
"""Wrapper to run transcription and reset running flag when done."""
try:
self.process_data(
@@ -194,7 +195,7 @@ class AudioTranscriptionPostProcessor(PostProcessorApi):
self.requestor.send_data(UPDATE_AUDIO_TRANSCRIPTION_STATE, "idle")
def handle_request(self, topic: str, request_data: dict[str, any]) -> str | None:
def handle_request(self, topic: str, request_data: dict[str, Any]) -> str | None:
if topic == "transcribe_audio":
event = request_data["event"]

8
frigate/data_processing/post/license_plate.py
View File

@@ -29,7 +29,7 @@ from .api import PostProcessorApi
logger = logging.getLogger(__name__)
class LicensePlatePostProcessor(LicensePlateProcessingMixin, PostProcessorApi):
class LicensePlatePostProcessor(LicensePlateProcessingMixin, PostProcessorApi): # type: ignore[misc]
def __init__(
self,
config: FrigateConfig,
@@ -71,7 +71,7 @@ class LicensePlatePostProcessor(LicensePlateProcessingMixin, PostProcessorApi):
# don't run LPR post processing for now
return
event_id = data["event_id"]
event_id = data["event_id"] # type: ignore[unreachable]
camera_name = data["camera"]
if data_type == PostProcessDataEnum.recording:
@@ -225,7 +225,7 @@ class LicensePlatePostProcessor(LicensePlateProcessingMixin, PostProcessorApi):
logger.debug(f"Post processing plate: {event_id}, {frame_time}")
self.lpr_process(keyframe_obj_data, frame)
def handle_request(self, topic, request_data) -> dict[str, Any] | None:
def handle_request(self, topic: str, request_data: dict) -> dict[str, Any] | None:
if topic == EmbeddingsRequestEnum.reprocess_plate.value:
event = request_data["event"]
@@ -242,3 +242,5 @@ class LicensePlatePostProcessor(LicensePlateProcessingMixin, PostProcessorApi):
"message": "Successfully requested reprocessing of license plate.",
"success": True,
}
return None

46
frigate/data_processing/post/object_descriptions.py
View File

@@ -24,7 +24,7 @@ from frigate.util.file import get_event_thumbnail_bytes, load_event_snapshot_ima
from frigate.util.image import create_thumbnail, ensure_jpeg_bytes
if TYPE_CHECKING:
from frigate.embeddings import Embeddings
from frigate.embeddings.embeddings import Embeddings
from ..post.api import PostProcessorApi
from ..types import DataProcessorMetrics
@@ -139,7 +139,7 @@ class ObjectDescriptionProcessor(PostProcessorApi):
):
self._process_genai_description(event, camera_config, thumbnail)
else:
self.cleanup_event(event.id)
self.cleanup_event(str(event.id))
def __regenerate_description(self, event_id: str, source: str, force: bool) -> None:
"""Regenerate the description for an event."""
@@ -149,17 +149,17 @@ class ObjectDescriptionProcessor(PostProcessorApi):
logger.error(f"Event {event_id} not found for description regeneration")
return
if self.genai_client is None:
logger.error("GenAI not enabled")
return
camera_config = self.config.cameras[event.camera]
camera_config = self.config.cameras[str(event.camera)]
if not camera_config.objects.genai.enabled and not force:
logger.error(f"GenAI not enabled for camera {event.camera}")
return
thumbnail = get_event_thumbnail_bytes(event)
if thumbnail is None:
logger.error("No thumbnail available for %s", event.id)
return
# ensure we have a jpeg to pass to the model
thumbnail = ensure_jpeg_bytes(thumbnail)
@@ -187,7 +187,9 @@ class ObjectDescriptionProcessor(PostProcessorApi):
)
)
self._genai_embed_description(event, embed_image)
self._genai_embed_description(
event, [img for img in embed_image if img is not None]
)
def process_data(self, frame_data: dict, data_type: PostProcessDataEnum) -> None:
"""Process a frame update."""
@@ -241,7 +243,7 @@ class ObjectDescriptionProcessor(PostProcessorApi):
# Crop snapshot based on region
# provide full image if region doesn't exist (manual events)
height, width = img.shape[:2]
x1_rel, y1_rel, width_rel, height_rel = event.data.get(
x1_rel, y1_rel, width_rel, height_rel = event.data.get( # type: ignore[attr-defined]
"region", [0, 0, 1, 1]
)
x1, y1 = int(x1_rel * width), int(y1_rel * height)
@@ -258,14 +260,16 @@ class ObjectDescriptionProcessor(PostProcessorApi):
return None
def _process_genai_description(
self, event: Event, camera_config: CameraConfig, thumbnail
self, event: Event, camera_config: CameraConfig, thumbnail: bytes
) -> None:
event_id = str(event.id)
if event.has_snapshot and camera_config.objects.genai.use_snapshot:
snapshot_image = self._read_and_crop_snapshot(event)
if not snapshot_image:
return
num_thumbnails = len(self.tracked_events.get(event.id, []))
num_thumbnails = len(self.tracked_events.get(event_id, []))
# ensure we have a jpeg to pass to the model
thumbnail = ensure_jpeg_bytes(thumbnail)
@@ -277,7 +281,7 @@ class ObjectDescriptionProcessor(PostProcessorApi):
else (
[
data["thumbnail"][:] if data.get("thumbnail") else None
for data in self.tracked_events[event.id]
for data in self.tracked_events[event_id]
if data.get("thumbnail")
]
if num_thumbnails > 0
@@ -286,22 +290,22 @@ class ObjectDescriptionProcessor(PostProcessorApi):
)
if camera_config.objects.genai.debug_save_thumbnails and num_thumbnails > 0:
logger.debug(f"Saving {num_thumbnails} thumbnails for event {event.id}")
logger.debug(f"Saving {num_thumbnails} thumbnails for event {event_id}")
Path(os.path.join(CLIPS_DIR, f"genai-requests/{event.id}")).mkdir(
Path(os.path.join(CLIPS_DIR, f"genai-requests/{event_id}")).mkdir(
parents=True, exist_ok=True
)
for idx, data in enumerate(self.tracked_events[event.id], 1):
for idx, data in enumerate(self.tracked_events[event_id], 1):
jpg_bytes: bytes | None = data["thumbnail"]
if jpg_bytes is None:
logger.warning(f"Unable to save thumbnail {idx} for {event.id}.")
logger.warning(f"Unable to save thumbnail {idx} for {event_id}.")
else:
with open(
os.path.join(
CLIPS_DIR,
f"genai-requests/{event.id}/{idx}.jpg",
f"genai-requests/{event_id}/{idx}.jpg",
),
"wb",
) as j:
@@ -310,7 +314,7 @@ class ObjectDescriptionProcessor(PostProcessorApi):
# Generate the description. Call happens in a thread since it is network bound.
threading.Thread(
target=self._genai_embed_description,
name=f"_genai_embed_description_{event.id}",
name=f"_genai_embed_description_{event_id}",
daemon=True,
args=(
event,
@@ -319,12 +323,12 @@ class ObjectDescriptionProcessor(PostProcessorApi):
).start()
# Clean up tracked events and early request state
self.cleanup_event(event.id)
self.cleanup_event(event_id)
def _genai_embed_description(self, event: Event, thumbnails: list[bytes]) -> None:
"""Embed the description for an event."""
start = datetime.datetime.now().timestamp()
camera_config = self.config.cameras[event.camera]
camera_config = self.config.cameras[str(event.camera)]
description = self.genai_client.generate_object_description(
camera_config, thumbnails, event
)
@@ -346,7 +350,7 @@ class ObjectDescriptionProcessor(PostProcessorApi):
# Embed the description
if self.config.semantic_search.enabled:
self.embeddings.embed_description(event.id, description)
self.embeddings.embed_description(str(event.id), description)
# Check semantic trigger for this description
if self.semantic_trigger_processor is not None:

29
frigate/data_processing/post/review_descriptions.py
View File

@@ -48,8 +48,8 @@ class ReviewDescriptionProcessor(PostProcessorApi):
self.metrics = metrics
self.genai_client = client
self.review_desc_speed = InferenceSpeed(self.metrics.review_desc_speed)
self.review_descs_dps = EventsPerSecond()
self.review_descs_dps.start()
self.review_desc_dps = EventsPerSecond()
self.review_desc_dps.start()
def calculate_frame_count(
self,
@@ -59,7 +59,7 @@ class ReviewDescriptionProcessor(PostProcessorApi):
) -> int:
"""Calculate optimal number of frames based on context size, image source, and resolution.
Token usage varies by resolution: larger images (ultrawide aspect ratios) use more tokens.
Token usage varies by resolution: larger images (ultra-wide aspect ratios) use more tokens.
Estimates ~1 token per 1250 pixels. Targets 98% context utilization with safety margin.
Capped at 20 frames.
"""
@@ -68,7 +68,11 @@ class ReviewDescriptionProcessor(PostProcessorApi):
detect_width = camera_config.detect.width
detect_height = camera_config.detect.height
aspect_ratio = detect_width / detect_height
if not detect_width or not detect_height:
aspect_ratio = 16 / 9
else:
aspect_ratio = detect_width / detect_height
if image_source == ImageSourceEnum.recordings:
if aspect_ratio >= 1:
@@ -99,8 +103,10 @@ class ReviewDescriptionProcessor(PostProcessorApi):
return min(max(max_frames, 3), 20)
def process_data(self, data, data_type):
self.metrics.review_desc_dps.value = self.review_descs_dps.eps()
def process_data(
self, data: dict[str, Any], data_type: PostProcessDataEnum
) -> None:
self.metrics.review_desc_dps.value = self.review_desc_dps.eps()
if data_type != PostProcessDataEnum.review:
return
@@ -186,7 +192,7 @@ class ReviewDescriptionProcessor(PostProcessorApi):
)
# kickoff analysis
self.review_descs_dps.update()
self.review_desc_dps.update()
threading.Thread(
target=run_analysis,
args=(
@@ -202,7 +208,7 @@ class ReviewDescriptionProcessor(PostProcessorApi):
),
).start()
def handle_request(self, topic, request_data):
def handle_request(self, topic: str, request_data: dict[str, Any]) -> str | None:
if topic == EmbeddingsRequestEnum.summarize_review.value:
start_ts = request_data["start_ts"]
end_ts = request_data["end_ts"]
@@ -327,7 +333,7 @@ class ReviewDescriptionProcessor(PostProcessorApi):
file_start = f"preview_{camera}-"
start_file = f"{file_start}{start_time}.webp"
end_file = f"{file_start}{end_time}.webp"
all_frames = []
all_frames: list[str] = []
for file in sorted(os.listdir(preview_dir)):
if not file.startswith(file_start):
@@ -465,7 +471,7 @@ class ReviewDescriptionProcessor(PostProcessorApi):
thumb_data = cv2.imread(thumb_path)
if thumb_data is None:
logger.warning(
logger.warning( # type: ignore[unreachable]
"Could not read preview frame at %s, skipping", thumb_path
)
continue
@@ -488,13 +494,12 @@ class ReviewDescriptionProcessor(PostProcessorApi):
return thumbs
@staticmethod
def run_analysis(
requestor: InterProcessRequestor,
genai_client: GenAIClient,
review_inference_speed: InferenceSpeed,
camera_config: CameraConfig,
final_data: dict[str, str],
final_data: dict[str, Any],
thumbs: list[bytes],
genai_config: GenAIReviewConfig,
labelmap_objects: list[str],

16
frigate/data_processing/post/semantic_trigger.py
View File

@@ -19,6 +19,7 @@ from frigate.config import FrigateConfig
from frigate.const import CONFIG_DIR
from frigate.data_processing.types import PostProcessDataEnum
from frigate.db.sqlitevecq import SqliteVecQueueDatabase
from frigate.embeddings.embeddings import Embeddings
from frigate.embeddings.util import ZScoreNormalization
from frigate.models import Event, Trigger
from frigate.util.builtin import cosine_distance
@@ -40,8 +41,8 @@ class SemanticTriggerProcessor(PostProcessorApi):
requestor: InterProcessRequestor,
sub_label_publisher: EventMetadataPublisher,
metrics: DataProcessorMetrics,
embeddings,
):
embeddings: Embeddings,
) -> None:
super().__init__(config, metrics, None)
self.db = db
self.embeddings = embeddings
@@ -236,11 +237,14 @@ class SemanticTriggerProcessor(PostProcessorApi):
return
# Skip the event if not an object
if event.data.get("type") != "object":
if event.data.get("type") != "object": # type: ignore[attr-defined]
return
thumbnail_bytes = get_event_thumbnail_bytes(event)
if thumbnail_bytes is None:
return
nparr = np.frombuffer(thumbnail_bytes, np.uint8)
thumbnail = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
@@ -262,8 +266,10 @@ class SemanticTriggerProcessor(PostProcessorApi):
thumbnail,
)
def handle_request(self, topic, request_data):
def handle_request(
self, topic: str, request_data: dict[str, Any]
) -> dict[str, Any] | str | None:
return None
def expire_object(self, object_id, camera):
def expire_object(self, object_id: str, camera: str) -> None:
pass