Genai review summaries (#19473)

* Generate review item summaries with requests

* Adjust logic to only send important items

* Don't mention ladder

* Adjust prompt to be more specific

* Add more relaxed nature for normal activity

* Cleanup summary

* Update ollama client

* Add more directions to analyze the frames in order

* Remove environment from prompt

docker/main/requirements-wheels.txt

@@ -48,7 +48,7 @@ onnxruntime == 1.22.* ; platform_machine == 'aarch64'
 transformers == 4.45.*
 # Generative AI
 google-generativeai == 0.8.*
-ollama == 0.3.*
+ollama == 0.5.*
 openai == 1.65.*
 # push notifications
 py-vapid == 1.9.*

frigate/api/review.py

@@ -6,7 +6,7 @@ from functools import reduce
 from pathlib import Path
 
 import pandas as pd
-from fastapi import APIRouter
+from fastapi import APIRouter, Request
 from fastapi.params import Depends
 from fastapi.responses import JSONResponse
 from peewee import Case, DoesNotExist, IntegrityError, fn, operator
@@ -26,6 +26,8 @@ from frigate.api.defs.response.review_response import (
     ReviewSummaryResponse,
 )
 from frigate.api.defs.tags import Tags
+from frigate.config import FrigateConfig
+from frigate.embeddings import EmbeddingsContext
 from frigate.models import Recordings, ReviewSegment, UserReviewStatus
 from frigate.review.types import SeverityEnum
 from frigate.util.builtin import get_tz_modifiers
@@ -606,3 +608,35 @@ async def set_not_reviewed(
         content=({"success": True, "message": f"Set Review {review_id} as not viewed"}),
         status_code=200,
     )
+
+
+@router.post(
+    "/review/summarize/start/{start_ts}/end/{end_ts}",
+    description="Use GenAI to summarize review items over a period of time.",
+)
+def generate_review_summary(request: Request, start_ts: float, end_ts: float):
+    config: FrigateConfig = request.app.frigate_config
+
+    if not config.genai.provider:
+        return JSONResponse(
+            content=(
+                {
+                    "success": False,
+                    "message": "GenAI must be configured to use this feature.",
+                }
+            ),
+            status_code=400,
+        )
+
+    context: EmbeddingsContext = request.app.embeddings
+    summary = context.generate_review_summary(start_ts, end_ts)
+
+    if summary:
+        return JSONResponse(
+            content=({"success": True, "summary": summary}), status_code=200
+        )
+    else:
+        return JSONResponse(
+            content=({"success": False, "message": "Failed to create summary."}),
+            status_code=500,
+        )

frigate/comms/embeddings_updater.py

@@ -29,6 +29,8 @@ class EmbeddingsRequestEnum(Enum):
     reindex = "reindex"
     # LPR
     reprocess_plate = "reprocess_plate"
+    # Review Descriptions
+    summarize_review = "summarize_review"
 
 
 class EmbeddingsResponder:

frigate/data_processing/post/api.py

@@ -39,7 +39,9 @@ class PostProcessorApi(ABC):
         pass
 
     @abstractmethod
-    def handle_request(self, request_data: dict[str, Any]) -> dict[str, Any] | None:
+    def handle_request(
+        self, topic: str, request_data: dict[str, Any]
+    ) -> dict[str, Any] | None:
         """Handle metadata requests.
         Args:
             request_data (dict): containing data about requested change to process.

frigate/data_processing/post/review_descriptions.py

@@ -7,14 +7,18 @@ import os
 import shutil
 import threading
 from pathlib import Path
+from typing import Any
 
 import cv2
 
+from frigate.comms.embeddings_updater import EmbeddingsRequestEnum
 from frigate.comms.inter_process import InterProcessRequestor
 from frigate.config import FrigateConfig
+from frigate.config.camera.review import GenAIReviewConfig
 from frigate.const import CACHE_DIR, CLIPS_DIR, UPDATE_REVIEW_DESCRIPTION
 from frigate.data_processing.types import PostProcessDataEnum
 from frigate.genai import GenAIClient
+from frigate.models import ReviewSegment
 from frigate.util.builtin import EventsPerSecond, InferenceSpeed
 
 from ..post.api import PostProcessorApi
@@ -111,13 +115,49 @@ class ReviewDescriptionProcessor(PostProcessorApi):
                     camera,
                     final_data,
                     thumbs,
-                    camera_config.review.genai.additional_concerns,
+                    camera_config.review.genai,
-                    camera_config.review.genai.preferred_language,
                 ),
             ).start()
 
-    def handle_request(self, request_data):
+    def handle_request(self, topic, request_data):
-        pass
+        if topic == EmbeddingsRequestEnum.summarize_review.value:
+            start_ts = request_data["start_ts"]
+            end_ts = request_data["end_ts"]
+            items: list[dict[str, Any]] = [
+                r["data"]["metadata"]
+                for r in (
+                    ReviewSegment.select(ReviewSegment.data)
+                    .where(
+                        (ReviewSegment.data["metadata"].is_null(False))
+                        & (ReviewSegment.start_time < end_ts)
+                        & (ReviewSegment.end_time > start_ts)
+                    )
+                    .order_by(ReviewSegment.start_time.asc())
+                    .dicts()
+                    .iterator()
+                )
+            ]
+
+            if len(items) == 0:
+                logger.debug("No review items with metadata found during time period")
+                return None
+
+            important_items = list(
+                filter(
+                    lambda item: item.get("potential_threat_level", 0) > 0
+                    or item.get("other_concerns"),
+                    items,
+                )
+            )
+
+            if not important_items:
+                return "No concerns were found during this time period."
+
+            return self.genai_client.generate_review_summary(
+                start_ts, end_ts, important_items
+            )
+        else:
+            return None
 
     def get_cache_frames(
         self, camera: str, start_time: float, end_time: float
@@ -162,12 +202,12 @@ def run_analysis(
     camera: str,
     final_data: dict[str, str],
     thumbs: list[bytes],
-    concerns: list[str],
+    genai_config: GenAIReviewConfig,
-    preferred_language: str | None,
 ) -> None:
     start = datetime.datetime.now().timestamp()
     metadata = genai_client.generate_review_description(
         {
+            "id": final_data["id"],
             "camera": camera,
             "objects": final_data["data"]["objects"],
             "recognized_objects": final_data["data"]["sub_labels"],
@@ -175,8 +215,9 @@ def run_analysis(
             "timestamp": datetime.datetime.fromtimestamp(final_data["end_time"]),
         },
         thumbs,
-        concerns,
+        genai_config.additional_concerns,
-        preferred_language,
+        genai_config.preferred_language,
+        genai_config.debug_save_thumbnails,
     )
     review_inference_speed.update(datetime.datetime.now().timestamp() - start)
 

frigate/embeddings/__init__.py

@@ -313,3 +313,9 @@ class EmbeddingsContext:
             EmbeddingsRequestEnum.embed_thumbnail.value,
             {"id": str(event_id), "thumbnail": str(thumbnail), "upsert": False},
         )
+
+    def generate_review_summary(self, start_ts: float, end_ts: float) -> str | None:
+        return self.requestor.send_data(
+            EmbeddingsRequestEnum.summarize_review.value,
+            {"start_ts": start_ts, "end_ts": end_ts},
+        )

frigate/embeddings/maintainer.py

@@ -66,7 +66,7 @@ from frigate.data_processing.types import DataProcessorMetrics, PostProcessDataE
 from frigate.db.sqlitevecq import SqliteVecQueueDatabase
 from frigate.events.types import EventTypeEnum, RegenerateDescriptionEnum
 from frigate.genai import get_genai_client
-from frigate.models import Event, Recordings, Trigger
+from frigate.models import Event, Recordings, ReviewSegment, Trigger
 from frigate.types import TrackedObjectUpdateTypesEnum
 from frigate.util.builtin import serialize
 from frigate.util.image import (
@@ -121,7 +121,7 @@ class EmbeddingMaintainer(threading.Thread):
             ),
             load_vec_extension=True,
         )
-        models = [Event, Recordings, Trigger]
+        models = [Event, Recordings, ReviewSegment, Trigger]
         db.bind(models)
 
         if config.semantic_search.enabled:

frigate/genai/__init__.py

@@ -1,5 +1,6 @@
 """Generative AI module for Frigate."""
 
+import datetime
 import importlib
 import logging
 import os
@@ -9,6 +10,7 @@ from typing import Any, Optional
 from playhouse.shortcuts import model_to_dict
 
 from frigate.config import CameraConfig, FrigateConfig, GenAIConfig, GenAIProviderEnum
+from frigate.const import CLIPS_DIR
 from frigate.data_processing.post.types import ReviewMetadata
 from frigate.models import Event
 
@@ -41,6 +43,7 @@ class GenAIClient:
         thumbnails: list[bytes],
         concerns: list[str],
         preferred_language: str | None,
+        debug_save: bool,
     ) -> ReviewMetadata | None:
         """Generate a description for the review item activity."""
         if concerns:
@@ -59,34 +62,39 @@ class GenAIClient:
             language_prompt = ""
 
         context_prompt = f"""
-Please analyze the image(s), which are in chronological order, strictly from the perspective of the {review_data["camera"].replace("_", " ")} security camera.
+Please analyze the sequence of images ({len(thumbnails)} total) taken in chronological order from the perspective of the {review_data["camera"].replace("_", " ")} security camera.
 
-Your task is to provide a **neutral, factual, and objective description** of the scene, while also:
+Your task is to provide a clear, security-focused description of the scene that:
-- Clearly stating **what is happening** based on observable actions and movements.
+1. States exactly what is happening based on observable actions and movements.
-- Including **reasonable, evidence-based inferences** about the likely activity or context, but only if directly supported by visible details.
+2. Identifies and emphasizes behaviors that match patterns of suspicious activity.
+3. Assigns a potential_threat_level based on the definitions below, applying them consistently.
+
+Facts come first, but identifying security risks is the primary goal.
 
 When forming your description:
-- **Facts first**: Describe the time, physical setting, people, and objects exactly as seen.
+- Describe the time, people, and objects exactly as seen. Include any observable environmental changes (e.g., lighting changes triggered by activity).
-- **Then context**: Briefly note plausible purposes or activities (e.g., “appears to be delivering a package” if carrying a box to a door).
+- Time of day should **increase suspicion only when paired with unusual or security-relevant behaviors**. Do not raise the threat level for common residential activities (e.g., residents walking pets, retrieving mail, gardening, playing with pets, supervising children) even at unusual hours, unless other suspicious indicators are present.
-- Clearly separate certain facts (“A person is holding an object with horizontal rungs”) from reasonable inferences (“likely a ladder”).
+- Focus on behaviors that are uncharacteristic of innocent activity: loitering without clear purpose, avoiding cameras, inspecting vehicles/doors, changing behavior when lights activate, scanning surroundings without an apparent benign reason.
-- Do not speculate beyond what is visible, and do not imply hostility, criminal intent, or other strong judgments unless there is unambiguous visual evidence.
+- **Benign context override**: If scanning or looking around is clearly part of an innocent activity (such as playing with a dog, gardening, supervising children, or watching for a pet), do not treat it as suspicious.
 
-Here is information already known:
+Your response MUST be a flat JSON object with:
-- Activity occurred at {review_data["timestamp"].strftime("%I:%M %p")}
-- Detected objects: {review_data["objects"]}
-- Recognized objects: {review_data["recognized_objects"]}
-- Zones involved: {review_data["zones"]}
-
-Your response **MUST** be a flat JSON object with:
 - `scene` (string): A full description including setting, entities, actions, and any plausible supported inferences.
-- `confidence` (float): A number 0-1 for overall confidence in the analysis.
+- `confidence` (float): 0-1 confidence in the analysis.
-- `potential_threat_level` (integer, optional): Include only if there is a clear, observable security concern:
+- `potential_threat_level` (integer): 0, 1, or 2 as defined below.
-    - 0 = Normal activity is occurring
-    - 1 = Unusual but not overtly threatening
-    - 2 = Suspicious or potentially harmful
-    - 3 = Clear and immediate threat
 {concern_prompt}
 
+Threat-level definitions:
+- 0 — Typical or expected activity for this location/time (includes residents, guests, or known animals engaged in normal activities, even if they glance around or scan surroundings).
+- 1 — Unusual or suspicious activity: At least one security-relevant behavior is present **and not explainable by a normal residential activity**.
+- 2 — Active or immediate threat: Breaking in, vandalism, aggression, weapon display.
+
+Sequence details:
+- Frame 1 = earliest, Frame 10 = latest
+- Activity occurred at {review_data["timestamp"].strftime("%I:%M %p")}
+- Detected objects: {list(set(review_data["objects"]))}
+- Recognized objects: {list(set(review_data["recognized_objects"])) or "None"}
+- Zones involved: {review_data["zones"]}
+
 **IMPORTANT:**
 - Values must be plain strings, floats, or integers — no nested objects, no extra commentary.
 {language_prompt}
@@ -94,6 +102,16 @@ Your response **MUST** be a flat JSON object with:
         logger.debug(
             f"Sending {len(thumbnails)} images to create review description on {review_data['camera']}"
         )
+
+        if debug_save:
+            with open(
+                os.path.join(
+                    CLIPS_DIR, "genai-requests", review_data["id"], "prompt.txt"
+                ),
+                "w",
+            ) as f:
+                f.write(context_prompt)
+
         response = self._send(context_prompt, thumbnails)
 
         if response:
@@ -112,6 +130,36 @@ Your response **MUST** be a flat JSON object with:
         else:
             return None
 
+    def generate_review_summary(
+        self, start_ts: float, end_ts: float, segments: list[dict[str, Any]]
+    ) -> str | None:
+        """Generate a summary of review item descriptions over a period of time."""
+        time_range = f"{datetime.datetime.fromtimestamp(start_ts).strftime('%I:%M %p')} to {datetime.datetime.fromtimestamp(end_ts).strftime('%I:%M %p')}"
+        timeline_summary_prompt = f"""
+You are a security officer. Time range: {time_range}.
+Input: JSON list with "scene", "confidence", "potential_threat_level" (1-2), "other_concerns".
+Write a report:
+
+Security Summary - {time_range}
+[One-sentence overview of activity]
+[Chronological bullet list of events with timestamps if in scene]
+[Final threat assessment]
+
+Rules:
+- List events in order.
+- Highlight potential_threat_level ≥ 1 with exact times.
+- Note any of the additional concerns which are present.
+- Note unusual activity even if not threats.
+- If no threats: "Final assessment: Only normal activity observed during this period."
+- No commentary, questions, or recommendations.
+- Output only the report.
+        """
+
+        for item in segments:
+            timeline_summary_prompt += f"\n{item}"
+
+        return self._send(timeline_summary_prompt, [])
+
     def generate_object_description(
         self,
         camera_config: CameraConfig,

frigate/genai/ollama.py

@@ -47,8 +47,8 @@ class OllamaClient(GenAIClient):
             result = self.provider.generate(
                 self.genai_config.model,
                 prompt,
-                images=images,
+                images=images if images else None,
-                options={"keep_alive": "1h"},
+                keep_alive="1h",
             )
             return result["response"].strip()
         except (TimeoutException, ResponseError) as e:

web/src/components/overlay/detail/ReviewDetailDialog.tsx

@@ -85,9 +85,6 @@ export default function ReviewDetailDialog({
 
     let concerns = "";
     switch (aiAnalysis.potential_threat_level) {
-      case ThreatLevel.UNUSUAL:
-        concerns = "• Unusual Activity\n";
-        break;
       case ThreatLevel.SUSPICIOUS:
         concerns = "• Suspicious Activity\n";
         break;

web/src/types/review.ts

@@ -81,7 +81,6 @@ export type ConsolidatedSegmentData = {
 export type TimelineZoomDirection = "in" | "out" | null;
 
 export enum ThreatLevel {
-  UNUSUAL = 1,
+  SUSPICIOUS = 1,
-  SUSPICIOUS = 2,
+  DANGER = 2,
-  DANGER = 3,
 }