diff --git a/frigate/config/camera/genai.py b/frigate/config/camera/genai.py index 9a4fb71ff..b47839dd5 100644 --- a/frigate/config/camera/genai.py +++ b/frigate/config/camera/genai.py @@ -22,6 +22,4 @@ class GenAIConfig(FrigateBaseModel): api_key: Optional[EnvString] = Field(default=None, title="Provider API key.") base_url: Optional[str] = Field(default=None, title="Provider base url.") model: str = Field(default="gpt-4o", title="GenAI model.") - provider: GenAIProviderEnum = Field( - default=GenAIProviderEnum.openai, title="GenAI provider." - ) + provider: GenAIProviderEnum | None = Field(default=None, title="GenAI provider.") diff --git a/frigate/config/camera/review.py b/frigate/config/camera/review.py index cb01e5107..51268339b 100644 --- a/frigate/config/camera/review.py +++ b/frigate/config/camera/review.py @@ -69,6 +69,10 @@ class GenAIReviewConfig(FrigateBaseModel): ) alerts: bool = Field(default=True, title="Enable GenAI for alerts.") detections: bool = Field(default=False, title="Enable GenAI for detections.") + additional_concerns: list[str] = Field( + default=[], + title="Additional concerns that GenAI should make note of on this camera.", + ) debug_save_thumbnails: bool = Field( default=False, title="Save thumbnails sent to generative AI for debugging purposes.", @@ -76,6 +80,10 @@ class GenAIReviewConfig(FrigateBaseModel): enabled_in_config: Optional[bool] = Field( default=None, title="Keep track of original state of generative AI." ) + preferred_language: str | None = Field( + title="Preferred language for GenAI Response", + default=None, + ) class ReviewConfig(FrigateBaseModel): diff --git a/frigate/data_processing/post/review_descriptions.py b/frigate/data_processing/post/review_descriptions.py index 59f47340d..cc116d291 100644 --- a/frigate/data_processing/post/review_descriptions.py +++ b/frigate/data_processing/post/review_descriptions.py @@ -46,8 +46,9 @@ class ReviewDescriptionProcessor(PostProcessorApi): return camera = data["after"]["camera"] + camera_config = self.config.cameras[camera] - if not self.config.cameras[camera].review.genai.enabled: + if not camera_config.review.genai.enabled: return id = data["after"]["id"] @@ -59,12 +60,12 @@ class ReviewDescriptionProcessor(PostProcessorApi): if ( final_data["severity"] == "alert" - and not self.config.cameras[camera].review.genai.alerts + and not camera_config.review.genai.alerts ): return elif ( final_data["severity"] == "detection" - and not self.config.cameras[camera].review.genai.detections + and not camera_config.review.genai.detections ): return @@ -86,9 +87,7 @@ class ReviewDescriptionProcessor(PostProcessorApi): if ret: thumbs.append(jpg.tobytes()) - if self.config.cameras[ - data["after"]["camera"] - ].review.genai.debug_save_thumbnails: + if camera_config.review.genai.debug_save_thumbnails: id = data["after"]["id"] Path(os.path.join(CLIPS_DIR, f"genai-requests/{id}")).mkdir( parents=True, exist_ok=True @@ -112,6 +111,8 @@ class ReviewDescriptionProcessor(PostProcessorApi): camera, final_data, thumbs, + camera_config.review.genai.additional_concerns, + camera_config.review.genai.preferred_language, ), ).start() @@ -161,6 +162,8 @@ def run_analysis( camera: str, final_data: dict[str, str], thumbs: list[bytes], + concerns: list[str], + preferred_language: str | None, ) -> None: start = datetime.datetime.now().timestamp() metadata = genai_client.generate_review_description( @@ -172,6 +175,8 @@ def run_analysis( "timestamp": datetime.datetime.fromtimestamp(final_data["end_time"]), }, thumbs, + concerns, + preferred_language, ) review_inference_speed.update(datetime.datetime.now().timestamp() - start) diff --git a/frigate/data_processing/post/types.py b/frigate/data_processing/post/types.py index d79a063b5..4e0534a8c 100644 --- a/frigate/data_processing/post/types.py +++ b/frigate/data_processing/post/types.py @@ -13,3 +13,7 @@ class ReviewMetadata(BaseModel): le=3, description="An integer representing the potential threat level (1-3). 1: Minor anomaly. 2: Moderate concern. 3: High threat. Only include this field if a clear security concern is observable; otherwise, omit it.", ) + other_concerns: list[str] | None = Field( + default=None, + description="Other concerns highlighted by the user that are observed.", + ) diff --git a/frigate/genai/__init__.py b/frigate/genai/__init__.py index 12007ca43..6e7071fce 100644 --- a/frigate/genai/__init__.py +++ b/frigate/genai/__init__.py @@ -36,39 +36,60 @@ class GenAIClient: self.provider = self._init_provider() def generate_review_description( - self, review_data: dict[str, Any], thumbnails: list[bytes] + self, + review_data: dict[str, Any], + thumbnails: list[bytes], + concerns: list[str], + preferred_language: str | None, ) -> ReviewMetadata | None: """Generate a description for the review item activity.""" + if concerns: + concern_list = "\n - ".join(concerns) + concern_prompt = f""" +- `other_concerns` (list of strings): Include a list of any of the following concerns that are occurring: + - {concern_list} +""" + + else: + concern_prompt = "" + + if preferred_language: + language_prompt = f"Provide your answer in {preferred_language}" + else: + language_prompt = "" + context_prompt = f""" - Please analyze the image(s), which are in chronological order, strictly from the perspective of the {review_data["camera"].replace("_", " ")} security camera. +Please analyze the image(s), which are in chronological order, strictly from the perspective of the {review_data["camera"].replace("_", " ")} security camera. - Your task is to provide a **neutral, factual, and objective description** of the scene, while also: - - Clearly stating **what is happening** based on observable actions and movements. - - Including **reasonable, evidence-based inferences** about the likely activity or context, but only if directly supported by visible details. +Your task is to provide a **neutral, factual, and objective description** of the scene, while also: +- Clearly stating **what is happening** based on observable actions and movements. +- Including **reasonable, evidence-based inferences** about the likely activity or context, but only if directly supported by visible details. - When forming your description: - - **Facts first**: Describe the physical setting, people, and objects exactly as seen. - - **Then context**: Briefly note plausible purposes or activities (e.g., “appears to be delivering a package” if carrying a box to a door). - - Clearly separate certain facts (“A person is holding a ladder”) from reasonable inferences (“likely performing maintenance”). - - Do not speculate beyond what is visible, and do not imply hostility, criminal intent, or other strong judgments unless there is unambiguous visual evidence. +When forming your description: +- **Facts first**: Describe the time, physical setting, people, and objects exactly as seen. +- **Then context**: Briefly note plausible purposes or activities (e.g., “appears to be delivering a package” if carrying a box to a door). +- Clearly separate certain facts (“A person is holding an object with horizontal rungs”) from reasonable inferences (“likely a ladder”). +- Do not speculate beyond what is visible, and do not imply hostility, criminal intent, or other strong judgments unless there is unambiguous visual evidence. - Here is information already known: - - Activity occurred at {review_data["timestamp"].strftime("%I:%M %p")} - - Detected objects: {review_data["objects"]} - - Recognized objects: {review_data["recognized_objects"]} - - Zones involved: {review_data["zones"]} +Here is information already known: +- Activity occurred at {review_data["timestamp"].strftime("%I:%M %p")} +- Detected objects: {review_data["objects"]} +- Recognized objects: {review_data["recognized_objects"]} +- Zones involved: {review_data["zones"]} - Your response **MUST** be a flat JSON object with: - - `scene` (string): A full description including setting, entities, actions, and any plausible supported inferences. - - `confidence` (float): A number 0–1 for overall confidence in the analysis. - - `potential_threat_level` (integer, optional): Include only if there is a clear, observable security concern: - - 0 = Normal activity is occurring - - 1 = Unusual but not overtly threatening - - 2 = Suspicious or potentially harmful - - 3 = Clear and immediate threat +Your response **MUST** be a flat JSON object with: +- `scene` (string): A full description including setting, entities, actions, and any plausible supported inferences. +- `confidence` (float): A number 0-1 for overall confidence in the analysis. +- `potential_threat_level` (integer, optional): Include only if there is a clear, observable security concern: + - 0 = Normal activity is occurring + - 1 = Unusual but not overtly threatening + - 2 = Suspicious or potentially harmful + - 3 = Clear and immediate threat +{concern_prompt} - **IMPORTANT:** - - Values must be plain strings, floats, or integers — no nested objects, no extra commentary. +**IMPORTANT:** +- Values must be plain strings, floats, or integers — no nested objects, no extra commentary. +{language_prompt} """ logger.debug( f"Sending {len(thumbnails)} images to create review description on {review_data['camera']}" @@ -116,6 +137,9 @@ class GenAIClient: def get_genai_client(config: FrigateConfig) -> Optional[GenAIClient]: """Get the GenAI client.""" + if not config.genai.provider: + return None + load_providers() provider = PROVIDERS.get(config.genai.provider) if provider: diff --git a/web/src/components/overlay/detail/ReviewDetailDialog.tsx b/web/src/components/overlay/detail/ReviewDetailDialog.tsx index 688f2b9dc..70eaa5872 100644 --- a/web/src/components/overlay/detail/ReviewDetailDialog.tsx +++ b/web/src/components/overlay/detail/ReviewDetailDialog.tsx @@ -76,20 +76,31 @@ export default function ReviewDetailDialog({ const aiAnalysis = useMemo(() => review?.data?.metadata, [review]); const aiThreatLevel = useMemo(() => { - if (!aiAnalysis?.potential_threat_level) { + if ( + !aiAnalysis || + (!aiAnalysis.potential_threat_level && !aiAnalysis.other_concerns) + ) { return "None"; } + let concerns = ""; switch (aiAnalysis.potential_threat_level) { case ThreatLevel.UNUSUAL: - return "Unusual Activity"; + concerns = "• Unusual Activity\n"; + break; case ThreatLevel.SUSPICIOUS: - return "Suspicious Activity"; + concerns = "• Suspicious Activity\n"; + break; case ThreatLevel.DANGER: - return "Danger"; + concerns = "• Danger\n"; + break; } - return "Unknown"; + (aiAnalysis.other_concerns ?? []).forEach((c) => { + concerns += `• ${c}\n`; + }); + + return concerns || "None"; }, [aiAnalysis]); const hasMismatch = useMemo(() => { @@ -258,8 +269,8 @@ export default function ReviewDetailDialog({ {aiAnalysis != undefined && (
AI Analysis @@ -267,7 +278,7 @@ export default function ReviewDetailDialog({
{aiAnalysis.scene}
Score
{aiAnalysis.confidence * 100}%
-
Threat Level
+
Concerns
{aiThreatLevel}
)} diff --git a/web/src/types/review.ts b/web/src/types/review.ts index 6f08571fd..bf2af7d1c 100644 --- a/web/src/types/review.ts +++ b/web/src/types/review.ts @@ -22,6 +22,7 @@ export type ReviewData = { scene: string; confidence: number; potential_threat_level?: number; + other_concerns?: string[]; }; };