diff --git a/frigate/data_processing/post/review_descriptions.py b/frigate/data_processing/post/review_descriptions.py index 3b7d1d6d9..59f47340d 100644 --- a/frigate/data_processing/post/review_descriptions.py +++ b/frigate/data_processing/post/review_descriptions.py @@ -12,7 +12,7 @@ import cv2 from frigate.comms.inter_process import InterProcessRequestor from frigate.config import FrigateConfig -from frigate.const import CLIPS_DIR, UPDATE_REVIEW_DESCRIPTION +from frigate.const import CACHE_DIR, CLIPS_DIR, UPDATE_REVIEW_DESCRIPTION from frigate.data_processing.types import PostProcessDataEnum from frigate.genai import GenAIClient from frigate.util.builtin import EventsPerSecond, InferenceSpeed @@ -34,7 +34,6 @@ class ReviewDescriptionProcessor(PostProcessorApi): super().__init__(config, metrics, None) self.requestor = requestor self.metrics = metrics - self.tracked_review_items: dict[str, list[tuple[int, bytes]]] = {} self.genai_client = client self.review_desc_speed = InferenceSpeed(self.metrics.review_desc_speed) self.review_descs_dps = EventsPerSecond() @@ -54,27 +53,38 @@ class ReviewDescriptionProcessor(PostProcessorApi): id = data["after"]["id"] if data["type"] == "new" or data["type"] == "update": - if id not in self.tracked_review_items: - self.tracked_review_items[id] = [] + return + else: + final_data = data["after"] - thumb_time = data["after"]["data"]["thumb_time"] - thumb_path = data["after"]["thumb_path"] + if ( + final_data["severity"] == "alert" + and not self.config.cameras[camera].review.genai.alerts + ): + return + elif ( + final_data["severity"] == "detection" + and not self.config.cameras[camera].review.genai.detections + ): + return - if thumb_time and thumb_path: - if ( - len(self.tracked_review_items[id]) > 0 - and self.tracked_review_items[id][0] == thumb_time - ): - # we have already processed this thumbnail - return + frames = self.get_cache_frames( + camera, final_data["start_time"], final_data["end_time"] + ) + if not frames: + frames = [final_data["thumb_path"]] + + thumbs = [] + + for idx, thumb_path in enumerate(frames): thumb_data = cv2.imread(thumb_path) ret, jpg = cv2.imencode( ".jpg", thumb_data, [int(cv2.IMWRITE_JPEG_QUALITY), 100] ) if ret: - self.tracked_review_items[id].append((thumb_time, jpg.tobytes())) + thumbs.append(jpg.tobytes()) if self.config.cameras[ data["after"]["camera"] @@ -87,29 +97,10 @@ class ReviewDescriptionProcessor(PostProcessorApi): thumb_path, os.path.join( CLIPS_DIR, - f"genai-requests/{id}/{thumb_time}.webp", + f"genai-requests/{id}/{idx}.webp", ), ) - else: - if id not in self.tracked_review_items: - return - - final_data = data["after"] - - if ( - final_data["severity"] == "alert" - and not self.config.cameras[camera].review.genai.alerts - ): - self.tracked_review_items.pop(id) - return - elif ( - final_data["severity"] == "detection" - and not self.config.cameras[camera].review.genai.detections - ): - self.tracked_review_items.pop(id) - return - # kickoff analysis self.review_descs_dps.update() threading.Thread( @@ -120,14 +111,47 @@ class ReviewDescriptionProcessor(PostProcessorApi): self.review_desc_speed, camera, final_data, - copy.copy([r[1] for r in self.tracked_review_items[id]]), + thumbs, ), ).start() - self.tracked_review_items.pop(id) def handle_request(self, request_data): pass + def get_cache_frames( + self, camera: str, start_time: float, end_time: float + ) -> list[str]: + preview_dir = os.path.join(CACHE_DIR, "preview_frames") + file_start = f"preview_{camera}" + start_file = f"{file_start}-{start_time}.webp" + end_file = f"{file_start}-{end_time}.webp" + all_frames = [] + + for file in sorted(os.listdir(preview_dir)): + if not file.startswith(file_start): + continue + + if file < start_file: + continue + + if file > end_file: + break + + all_frames.append(os.path.join(preview_dir, file)) + + frame_count = len(all_frames) + if frame_count <= 10: + return all_frames + + selected_frames = [] + step_size = (frame_count - 1) / 9 + + for i in range(10): + index = round(i * step_size) + selected_frames.append(all_frames[index]) + + return selected_frames + @staticmethod def run_analysis( diff --git a/frigate/genai/__init__.py b/frigate/genai/__init__.py index bcb95e756..12007ca43 100644 --- a/frigate/genai/__init__.py +++ b/frigate/genai/__init__.py @@ -70,6 +70,9 @@ class GenAIClient: **IMPORTANT:** - Values must be plain strings, floats, or integers — no nested objects, no extra commentary. """ + logger.debug( + f"Sending {len(thumbnails)} images to create review description on {review_data['camera']}" + ) response = self._send(context_prompt, thumbnails) if response: diff --git a/frigate/stats/util.py b/frigate/stats/util.py index 7d7a27653..a071dae1f 100644 --- a/frigate/stats/util.py +++ b/frigate/stats/util.py @@ -356,7 +356,7 @@ def stats_snapshot( embeddings_metrics.yolov9_lpr_pps.value, 2 ) - if embeddings_metrics.review_desc_dps.value > 0.0: + if embeddings_metrics.review_desc_speed.value > 0.0: stats["embeddings"]["review_description_speed"] = round( embeddings_metrics.review_desc_speed.value * 1000, 2 )