Add config for users to define additional concerns that GenAI should make note of in review summary (#19463)

* Don't default to openai

* Improve UI

* Allow configuring additional concerns that users may want the AI to note

* Formatting

* Add preferred language config

* Remove unused

frigate/config/camera/genai.py

@@ -22,6 +22,4 @@ class GenAIConfig(FrigateBaseModel):
     api_key: Optional[EnvString] = Field(default=None, title="Provider API key.")
     base_url: Optional[str] = Field(default=None, title="Provider base url.")
     model: str = Field(default="gpt-4o", title="GenAI model.")
-    provider: GenAIProviderEnum = Field(
+    provider: GenAIProviderEnum | None = Field(default=None, title="GenAI provider.")
-        default=GenAIProviderEnum.openai, title="GenAI provider."
-    )

frigate/config/camera/review.py

@@ -69,6 +69,10 @@ class GenAIReviewConfig(FrigateBaseModel):
     )
     alerts: bool = Field(default=True, title="Enable GenAI for alerts.")
     detections: bool = Field(default=False, title="Enable GenAI for detections.")
+    additional_concerns: list[str] = Field(
+        default=[],
+        title="Additional concerns that GenAI should make note of on this camera.",
+    )
     debug_save_thumbnails: bool = Field(
         default=False,
         title="Save thumbnails sent to generative AI for debugging purposes.",
@@ -76,6 +80,10 @@ class GenAIReviewConfig(FrigateBaseModel):
     enabled_in_config: Optional[bool] = Field(
         default=None, title="Keep track of original state of generative AI."
     )
+    preferred_language: str | None = Field(
+        title="Preferred language for GenAI Response",
+        default=None,
+    )
 
 
 class ReviewConfig(FrigateBaseModel):

frigate/data_processing/post/review_descriptions.py

@@ -46,8 +46,9 @@ class ReviewDescriptionProcessor(PostProcessorApi):
             return
 
         camera = data["after"]["camera"]
+        camera_config = self.config.cameras[camera]
 
-        if not self.config.cameras[camera].review.genai.enabled:
+        if not camera_config.review.genai.enabled:
             return
 
         id = data["after"]["id"]
@@ -59,12 +60,12 @@ class ReviewDescriptionProcessor(PostProcessorApi):
 
             if (
                 final_data["severity"] == "alert"
-                and not self.config.cameras[camera].review.genai.alerts
+                and not camera_config.review.genai.alerts
             ):
                 return
             elif (
                 final_data["severity"] == "detection"
-                and not self.config.cameras[camera].review.genai.detections
+                and not camera_config.review.genai.detections
             ):
                 return
 
@@ -86,9 +87,7 @@ class ReviewDescriptionProcessor(PostProcessorApi):
                 if ret:
                     thumbs.append(jpg.tobytes())
 
-                if self.config.cameras[
+                if camera_config.review.genai.debug_save_thumbnails:
-                    data["after"]["camera"]
-                ].review.genai.debug_save_thumbnails:
                     id = data["after"]["id"]
                     Path(os.path.join(CLIPS_DIR, f"genai-requests/{id}")).mkdir(
                         parents=True, exist_ok=True
@@ -112,6 +111,8 @@ class ReviewDescriptionProcessor(PostProcessorApi):
                     camera,
                     final_data,
                     thumbs,
+                    camera_config.review.genai.additional_concerns,
+                    camera_config.review.genai.preferred_language,
                 ),
             ).start()
 
@@ -161,6 +162,8 @@ def run_analysis(
     camera: str,
     final_data: dict[str, str],
     thumbs: list[bytes],
+    concerns: list[str],
+    preferred_language: str | None,
 ) -> None:
     start = datetime.datetime.now().timestamp()
     metadata = genai_client.generate_review_description(
@@ -172,6 +175,8 @@ def run_analysis(
             "timestamp": datetime.datetime.fromtimestamp(final_data["end_time"]),
         },
         thumbs,
+        concerns,
+        preferred_language,
     )
     review_inference_speed.update(datetime.datetime.now().timestamp() - start)
 

frigate/data_processing/post/types.py

@@ -13,3 +13,7 @@ class ReviewMetadata(BaseModel):
         le=3,
         description="An integer representing the potential threat level (1-3). 1: Minor anomaly. 2: Moderate concern. 3: High threat. Only include this field if a clear security concern is observable; otherwise, omit it.",
     )
+    other_concerns: list[str] | None = Field(
+        default=None,
+        description="Other concerns highlighted by the user that are observed.",
+    )

frigate/genai/__init__.py

@@ -36,39 +36,60 @@ class GenAIClient:
         self.provider = self._init_provider()
 
     def generate_review_description(
-        self, review_data: dict[str, Any], thumbnails: list[bytes]
+        self,
+        review_data: dict[str, Any],
+        thumbnails: list[bytes],
+        concerns: list[str],
+        preferred_language: str | None,
     ) -> ReviewMetadata | None:
         """Generate a description for the review item activity."""
+        if concerns:
+            concern_list = "\n    - ".join(concerns)
+            concern_prompt = f"""
+- `other_concerns` (list of strings): Include a list of any of the following concerns that are occurring:
+    - {concern_list}
+"""
+
+        else:
+            concern_prompt = ""
+
+        if preferred_language:
+            language_prompt = f"Provide your answer in {preferred_language}"
+        else:
+            language_prompt = ""
+
         context_prompt = f"""
-        Please analyze the image(s), which are in chronological order, strictly from the perspective of the {review_data["camera"].replace("_", " ")} security camera.
+Please analyze the image(s), which are in chronological order, strictly from the perspective of the {review_data["camera"].replace("_", " ")} security camera.
 
-        Your task is to provide a **neutral, factual, and objective description** of the scene, while also:
+Your task is to provide a **neutral, factual, and objective description** of the scene, while also:
-        - Clearly stating **what is happening** based on observable actions and movements.
+- Clearly stating **what is happening** based on observable actions and movements.
-        - Including **reasonable, evidence-based inferences** about the likely activity or context, but only if directly supported by visible details.
+- Including **reasonable, evidence-based inferences** about the likely activity or context, but only if directly supported by visible details.
 
-        When forming your description:
+When forming your description:
-        - **Facts first**: Describe the physical setting, people, and objects exactly as seen.
+- **Facts first**: Describe the time, physical setting, people, and objects exactly as seen.
-        - **Then context**: Briefly note plausible purposes or activities (e.g., “appears to be delivering a package” if carrying a box to a door).
+- **Then context**: Briefly note plausible purposes or activities (e.g., “appears to be delivering a package” if carrying a box to a door).
-        - Clearly separate certain facts (“A person is holding a ladder”) from reasonable inferences (“likely performing maintenance”).
+- Clearly separate certain facts (“A person is holding an object with horizontal rungs”) from reasonable inferences (“likely a ladder”).
-        - Do not speculate beyond what is visible, and do not imply hostility, criminal intent, or other strong judgments unless there is unambiguous visual evidence.
+- Do not speculate beyond what is visible, and do not imply hostility, criminal intent, or other strong judgments unless there is unambiguous visual evidence.
 
-        Here is information already known:
+Here is information already known:
-        - Activity occurred at {review_data["timestamp"].strftime("%I:%M %p")}
+- Activity occurred at {review_data["timestamp"].strftime("%I:%M %p")}
-        - Detected objects: {review_data["objects"]}
+- Detected objects: {review_data["objects"]}
-        - Recognized objects: {review_data["recognized_objects"]}
+- Recognized objects: {review_data["recognized_objects"]}
-        - Zones involved: {review_data["zones"]}
+- Zones involved: {review_data["zones"]}
 
-        Your response **MUST** be a flat JSON object with:
+Your response **MUST** be a flat JSON object with:
-        - `scene` (string): A full description including setting, entities, actions, and any plausible supported inferences.
+- `scene` (string): A full description including setting, entities, actions, and any plausible supported inferences.
-        - `confidence` (float): A number 0–1 for overall confidence in the analysis.
+- `confidence` (float): A number 0-1 for overall confidence in the analysis.
-        - `potential_threat_level` (integer, optional): Include only if there is a clear, observable security concern:
+- `potential_threat_level` (integer, optional): Include only if there is a clear, observable security concern:
-            - 0 = Normal activity is occurring
+    - 0 = Normal activity is occurring
-            - 1 = Unusual but not overtly threatening
+    - 1 = Unusual but not overtly threatening
-            - 2 = Suspicious or potentially harmful
+    - 2 = Suspicious or potentially harmful
-            - 3 = Clear and immediate threat
+    - 3 = Clear and immediate threat
+{concern_prompt}
 
-        **IMPORTANT:**
+**IMPORTANT:**
-        - Values must be plain strings, floats, or integers — no nested objects, no extra commentary.
+- Values must be plain strings, floats, or integers — no nested objects, no extra commentary.
+{language_prompt}
         """
         logger.debug(
             f"Sending {len(thumbnails)} images to create review description on {review_data['camera']}"
@@ -116,6 +137,9 @@ class GenAIClient:
 
 def get_genai_client(config: FrigateConfig) -> Optional[GenAIClient]:
     """Get the GenAI client."""
+    if not config.genai.provider:
+        return None
+
     load_providers()
     provider = PROVIDERS.get(config.genai.provider)
     if provider:

web/src/components/overlay/detail/ReviewDetailDialog.tsx

@@ -76,20 +76,31 @@ export default function ReviewDetailDialog({
   const aiAnalysis = useMemo(() => review?.data?.metadata, [review]);
 
   const aiThreatLevel = useMemo(() => {
-    if (!aiAnalysis?.potential_threat_level) {
+    if (
+      !aiAnalysis ||
+      (!aiAnalysis.potential_threat_level && !aiAnalysis.other_concerns)
+    ) {
       return "None";
     }
 
+    let concerns = "";
     switch (aiAnalysis.potential_threat_level) {
       case ThreatLevel.UNUSUAL:
-        return "Unusual Activity";
+        concerns = "• Unusual Activity\n";
+        break;
       case ThreatLevel.SUSPICIOUS:
-        return "Suspicious Activity";
+        concerns = "• Suspicious Activity\n";
+        break;
       case ThreatLevel.DANGER:
-        return "Danger";
+        concerns = "• Danger\n";
+        break;
     }
 
-    return "Unknown";
+    (aiAnalysis.other_concerns ?? []).forEach((c) => {
+      concerns += `• ${c}\n`;
+    });
+
+    return concerns || "None";
   }, [aiAnalysis]);
 
   const hasMismatch = useMemo(() => {
@@ -258,8 +269,8 @@ export default function ReviewDetailDialog({
               {aiAnalysis != undefined && (
                 <div
                   className={cn(
-                    "m-2 flex h-full w-full flex-col gap-2 rounded-md bg-card p-2",
+                    "flex h-full w-full flex-col gap-2 rounded-md bg-card p-2",
-                    isDesktop && "w-[90%]",
+                    isDesktop && "m-2 w-[90%]",
                   )}
                 >
                   AI Analysis
@@ -267,7 +278,7 @@ export default function ReviewDetailDialog({
                   <div className="text-sm">{aiAnalysis.scene}</div>
                   <div className="text-sm text-primary/40">Score</div>
                   <div className="text-sm">{aiAnalysis.confidence * 100}%</div>
-                  <div className="text-sm text-primary/40">Threat Level</div>
+                  <div className="text-sm text-primary/40">Concerns</div>
                   <div className="text-sm">{aiThreatLevel}</div>
                 </div>
               )}

web/src/types/review.ts

@@ -22,6 +22,7 @@ export type ReviewData = {
     scene: string;
     confidence: number;
     potential_threat_level?: number;
+    other_concerns?: string[];
   };
 };