diff --git a/docker/main/requirements-wheels.txt b/docker/main/requirements-wheels.txt index eabb75bef..7a2e2d6df 100644 --- a/docker/main/requirements-wheels.txt +++ b/docker/main/requirements-wheels.txt @@ -48,7 +48,7 @@ onnxruntime == 1.22.* ; platform_machine == 'aarch64' transformers == 4.45.* # Generative AI google-generativeai == 0.8.* -ollama == 0.3.* +ollama == 0.5.* openai == 1.65.* # push notifications py-vapid == 1.9.* diff --git a/frigate/api/review.py b/frigate/api/review.py index e6d010db7..2ff97eeea 100644 --- a/frigate/api/review.py +++ b/frigate/api/review.py @@ -6,7 +6,7 @@ from functools import reduce from pathlib import Path import pandas as pd -from fastapi import APIRouter +from fastapi import APIRouter, Request from fastapi.params import Depends from fastapi.responses import JSONResponse from peewee import Case, DoesNotExist, IntegrityError, fn, operator @@ -26,6 +26,8 @@ from frigate.api.defs.response.review_response import ( ReviewSummaryResponse, ) from frigate.api.defs.tags import Tags +from frigate.config import FrigateConfig +from frigate.embeddings import EmbeddingsContext from frigate.models import Recordings, ReviewSegment, UserReviewStatus from frigate.review.types import SeverityEnum from frigate.util.builtin import get_tz_modifiers @@ -606,3 +608,35 @@ async def set_not_reviewed( content=({"success": True, "message": f"Set Review {review_id} as not viewed"}), status_code=200, ) + + +@router.post( + "/review/summarize/start/{start_ts}/end/{end_ts}", + description="Use GenAI to summarize review items over a period of time.", +) +def generate_review_summary(request: Request, start_ts: float, end_ts: float): + config: FrigateConfig = request.app.frigate_config + + if not config.genai.provider: + return JSONResponse( + content=( + { + "success": False, + "message": "GenAI must be configured to use this feature.", + } + ), + status_code=400, + ) + + context: EmbeddingsContext = request.app.embeddings + summary = context.generate_review_summary(start_ts, end_ts) + + if summary: + return JSONResponse( + content=({"success": True, "summary": summary}), status_code=200 + ) + else: + return JSONResponse( + content=({"success": False, "message": "Failed to create summary."}), + status_code=500, + ) diff --git a/frigate/comms/embeddings_updater.py b/frigate/comms/embeddings_updater.py index 58829733b..f7fd9c2bf 100644 --- a/frigate/comms/embeddings_updater.py +++ b/frigate/comms/embeddings_updater.py @@ -29,6 +29,8 @@ class EmbeddingsRequestEnum(Enum): reindex = "reindex" # LPR reprocess_plate = "reprocess_plate" + # Review Descriptions + summarize_review = "summarize_review" class EmbeddingsResponder: diff --git a/frigate/data_processing/post/api.py b/frigate/data_processing/post/api.py index cd6dda128..c341bd8ef 100644 --- a/frigate/data_processing/post/api.py +++ b/frigate/data_processing/post/api.py @@ -39,7 +39,9 @@ class PostProcessorApi(ABC): pass @abstractmethod - def handle_request(self, request_data: dict[str, Any]) -> dict[str, Any] | None: + def handle_request( + self, topic: str, request_data: dict[str, Any] + ) -> dict[str, Any] | None: """Handle metadata requests. Args: request_data (dict): containing data about requested change to process. diff --git a/frigate/data_processing/post/review_descriptions.py b/frigate/data_processing/post/review_descriptions.py index cc116d291..6a6127e00 100644 --- a/frigate/data_processing/post/review_descriptions.py +++ b/frigate/data_processing/post/review_descriptions.py @@ -7,14 +7,18 @@ import os import shutil import threading from pathlib import Path +from typing import Any import cv2 +from frigate.comms.embeddings_updater import EmbeddingsRequestEnum from frigate.comms.inter_process import InterProcessRequestor from frigate.config import FrigateConfig +from frigate.config.camera.review import GenAIReviewConfig from frigate.const import CACHE_DIR, CLIPS_DIR, UPDATE_REVIEW_DESCRIPTION from frigate.data_processing.types import PostProcessDataEnum from frigate.genai import GenAIClient +from frigate.models import ReviewSegment from frigate.util.builtin import EventsPerSecond, InferenceSpeed from ..post.api import PostProcessorApi @@ -111,13 +115,49 @@ class ReviewDescriptionProcessor(PostProcessorApi): camera, final_data, thumbs, - camera_config.review.genai.additional_concerns, - camera_config.review.genai.preferred_language, + camera_config.review.genai, ), ).start() - def handle_request(self, request_data): - pass + def handle_request(self, topic, request_data): + if topic == EmbeddingsRequestEnum.summarize_review.value: + start_ts = request_data["start_ts"] + end_ts = request_data["end_ts"] + items: list[dict[str, Any]] = [ + r["data"]["metadata"] + for r in ( + ReviewSegment.select(ReviewSegment.data) + .where( + (ReviewSegment.data["metadata"].is_null(False)) + & (ReviewSegment.start_time < end_ts) + & (ReviewSegment.end_time > start_ts) + ) + .order_by(ReviewSegment.start_time.asc()) + .dicts() + .iterator() + ) + ] + + if len(items) == 0: + logger.debug("No review items with metadata found during time period") + return None + + important_items = list( + filter( + lambda item: item.get("potential_threat_level", 0) > 0 + or item.get("other_concerns"), + items, + ) + ) + + if not important_items: + return "No concerns were found during this time period." + + return self.genai_client.generate_review_summary( + start_ts, end_ts, important_items + ) + else: + return None def get_cache_frames( self, camera: str, start_time: float, end_time: float @@ -162,12 +202,12 @@ def run_analysis( camera: str, final_data: dict[str, str], thumbs: list[bytes], - concerns: list[str], - preferred_language: str | None, + genai_config: GenAIReviewConfig, ) -> None: start = datetime.datetime.now().timestamp() metadata = genai_client.generate_review_description( { + "id": final_data["id"], "camera": camera, "objects": final_data["data"]["objects"], "recognized_objects": final_data["data"]["sub_labels"], @@ -175,8 +215,9 @@ def run_analysis( "timestamp": datetime.datetime.fromtimestamp(final_data["end_time"]), }, thumbs, - concerns, - preferred_language, + genai_config.additional_concerns, + genai_config.preferred_language, + genai_config.debug_save_thumbnails, ) review_inference_speed.update(datetime.datetime.now().timestamp() - start) diff --git a/frigate/embeddings/__init__.py b/frigate/embeddings/__init__.py index 03316ca58..0a854fcfa 100644 --- a/frigate/embeddings/__init__.py +++ b/frigate/embeddings/__init__.py @@ -313,3 +313,9 @@ class EmbeddingsContext: EmbeddingsRequestEnum.embed_thumbnail.value, {"id": str(event_id), "thumbnail": str(thumbnail), "upsert": False}, ) + + def generate_review_summary(self, start_ts: float, end_ts: float) -> str | None: + return self.requestor.send_data( + EmbeddingsRequestEnum.summarize_review.value, + {"start_ts": start_ts, "end_ts": end_ts}, + ) diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py index ae613b623..a129b9677 100644 --- a/frigate/embeddings/maintainer.py +++ b/frigate/embeddings/maintainer.py @@ -66,7 +66,7 @@ from frigate.data_processing.types import DataProcessorMetrics, PostProcessDataE from frigate.db.sqlitevecq import SqliteVecQueueDatabase from frigate.events.types import EventTypeEnum, RegenerateDescriptionEnum from frigate.genai import get_genai_client -from frigate.models import Event, Recordings, Trigger +from frigate.models import Event, Recordings, ReviewSegment, Trigger from frigate.types import TrackedObjectUpdateTypesEnum from frigate.util.builtin import serialize from frigate.util.image import ( @@ -121,7 +121,7 @@ class EmbeddingMaintainer(threading.Thread): ), load_vec_extension=True, ) - models = [Event, Recordings, Trigger] + models = [Event, Recordings, ReviewSegment, Trigger] db.bind(models) if config.semantic_search.enabled: diff --git a/frigate/genai/__init__.py b/frigate/genai/__init__.py index 6e7071fce..da73d2ab8 100644 --- a/frigate/genai/__init__.py +++ b/frigate/genai/__init__.py @@ -1,5 +1,6 @@ """Generative AI module for Frigate.""" +import datetime import importlib import logging import os @@ -9,6 +10,7 @@ from typing import Any, Optional from playhouse.shortcuts import model_to_dict from frigate.config import CameraConfig, FrigateConfig, GenAIConfig, GenAIProviderEnum +from frigate.const import CLIPS_DIR from frigate.data_processing.post.types import ReviewMetadata from frigate.models import Event @@ -41,6 +43,7 @@ class GenAIClient: thumbnails: list[bytes], concerns: list[str], preferred_language: str | None, + debug_save: bool, ) -> ReviewMetadata | None: """Generate a description for the review item activity.""" if concerns: @@ -59,34 +62,39 @@ class GenAIClient: language_prompt = "" context_prompt = f""" -Please analyze the image(s), which are in chronological order, strictly from the perspective of the {review_data["camera"].replace("_", " ")} security camera. +Please analyze the sequence of images ({len(thumbnails)} total) taken in chronological order from the perspective of the {review_data["camera"].replace("_", " ")} security camera. -Your task is to provide a **neutral, factual, and objective description** of the scene, while also: -- Clearly stating **what is happening** based on observable actions and movements. -- Including **reasonable, evidence-based inferences** about the likely activity or context, but only if directly supported by visible details. +Your task is to provide a clear, security-focused description of the scene that: +1. States exactly what is happening based on observable actions and movements. +2. Identifies and emphasizes behaviors that match patterns of suspicious activity. +3. Assigns a potential_threat_level based on the definitions below, applying them consistently. + +Facts come first, but identifying security risks is the primary goal. When forming your description: -- **Facts first**: Describe the time, physical setting, people, and objects exactly as seen. -- **Then context**: Briefly note plausible purposes or activities (e.g., “appears to be delivering a package” if carrying a box to a door). -- Clearly separate certain facts (“A person is holding an object with horizontal rungs”) from reasonable inferences (“likely a ladder”). -- Do not speculate beyond what is visible, and do not imply hostility, criminal intent, or other strong judgments unless there is unambiguous visual evidence. +- Describe the time, people, and objects exactly as seen. Include any observable environmental changes (e.g., lighting changes triggered by activity). +- Time of day should **increase suspicion only when paired with unusual or security-relevant behaviors**. Do not raise the threat level for common residential activities (e.g., residents walking pets, retrieving mail, gardening, playing with pets, supervising children) even at unusual hours, unless other suspicious indicators are present. +- Focus on behaviors that are uncharacteristic of innocent activity: loitering without clear purpose, avoiding cameras, inspecting vehicles/doors, changing behavior when lights activate, scanning surroundings without an apparent benign reason. +- **Benign context override**: If scanning or looking around is clearly part of an innocent activity (such as playing with a dog, gardening, supervising children, or watching for a pet), do not treat it as suspicious. -Here is information already known: -- Activity occurred at {review_data["timestamp"].strftime("%I:%M %p")} -- Detected objects: {review_data["objects"]} -- Recognized objects: {review_data["recognized_objects"]} -- Zones involved: {review_data["zones"]} - -Your response **MUST** be a flat JSON object with: +Your response MUST be a flat JSON object with: - `scene` (string): A full description including setting, entities, actions, and any plausible supported inferences. -- `confidence` (float): A number 0-1 for overall confidence in the analysis. -- `potential_threat_level` (integer, optional): Include only if there is a clear, observable security concern: - - 0 = Normal activity is occurring - - 1 = Unusual but not overtly threatening - - 2 = Suspicious or potentially harmful - - 3 = Clear and immediate threat +- `confidence` (float): 0-1 confidence in the analysis. +- `potential_threat_level` (integer): 0, 1, or 2 as defined below. {concern_prompt} +Threat-level definitions: +- 0 — Typical or expected activity for this location/time (includes residents, guests, or known animals engaged in normal activities, even if they glance around or scan surroundings). +- 1 — Unusual or suspicious activity: At least one security-relevant behavior is present **and not explainable by a normal residential activity**. +- 2 — Active or immediate threat: Breaking in, vandalism, aggression, weapon display. + +Sequence details: +- Frame 1 = earliest, Frame 10 = latest +- Activity occurred at {review_data["timestamp"].strftime("%I:%M %p")} +- Detected objects: {list(set(review_data["objects"]))} +- Recognized objects: {list(set(review_data["recognized_objects"])) or "None"} +- Zones involved: {review_data["zones"]} + **IMPORTANT:** - Values must be plain strings, floats, or integers — no nested objects, no extra commentary. {language_prompt} @@ -94,6 +102,16 @@ Your response **MUST** be a flat JSON object with: logger.debug( f"Sending {len(thumbnails)} images to create review description on {review_data['camera']}" ) + + if debug_save: + with open( + os.path.join( + CLIPS_DIR, "genai-requests", review_data["id"], "prompt.txt" + ), + "w", + ) as f: + f.write(context_prompt) + response = self._send(context_prompt, thumbnails) if response: @@ -112,6 +130,36 @@ Your response **MUST** be a flat JSON object with: else: return None + def generate_review_summary( + self, start_ts: float, end_ts: float, segments: list[dict[str, Any]] + ) -> str | None: + """Generate a summary of review item descriptions over a period of time.""" + time_range = f"{datetime.datetime.fromtimestamp(start_ts).strftime('%I:%M %p')} to {datetime.datetime.fromtimestamp(end_ts).strftime('%I:%M %p')}" + timeline_summary_prompt = f""" +You are a security officer. Time range: {time_range}. +Input: JSON list with "scene", "confidence", "potential_threat_level" (1-2), "other_concerns". +Write a report: + +Security Summary - {time_range} +[One-sentence overview of activity] +[Chronological bullet list of events with timestamps if in scene] +[Final threat assessment] + +Rules: +- List events in order. +- Highlight potential_threat_level ≥ 1 with exact times. +- Note any of the additional concerns which are present. +- Note unusual activity even if not threats. +- If no threats: "Final assessment: Only normal activity observed during this period." +- No commentary, questions, or recommendations. +- Output only the report. + """ + + for item in segments: + timeline_summary_prompt += f"\n{item}" + + return self._send(timeline_summary_prompt, []) + def generate_object_description( self, camera_config: CameraConfig, diff --git a/frigate/genai/ollama.py b/frigate/genai/ollama.py index ea88579cb..c392fadb9 100644 --- a/frigate/genai/ollama.py +++ b/frigate/genai/ollama.py @@ -47,8 +47,8 @@ class OllamaClient(GenAIClient): result = self.provider.generate( self.genai_config.model, prompt, - images=images, - options={"keep_alive": "1h"}, + images=images if images else None, + keep_alive="1h", ) return result["response"].strip() except (TimeoutException, ResponseError) as e: diff --git a/web/src/components/overlay/detail/ReviewDetailDialog.tsx b/web/src/components/overlay/detail/ReviewDetailDialog.tsx index 70eaa5872..1f647fb04 100644 --- a/web/src/components/overlay/detail/ReviewDetailDialog.tsx +++ b/web/src/components/overlay/detail/ReviewDetailDialog.tsx @@ -85,9 +85,6 @@ export default function ReviewDetailDialog({ let concerns = ""; switch (aiAnalysis.potential_threat_level) { - case ThreatLevel.UNUSUAL: - concerns = "• Unusual Activity\n"; - break; case ThreatLevel.SUSPICIOUS: concerns = "• Suspicious Activity\n"; break; diff --git a/web/src/types/review.ts b/web/src/types/review.ts index bf2af7d1c..e0385ae83 100644 --- a/web/src/types/review.ts +++ b/web/src/types/review.ts @@ -81,7 +81,6 @@ export type ConsolidatedSegmentData = { export type TimelineZoomDirection = "in" | "out" | null; export enum ThreatLevel { - UNUSUAL = 1, - SUSPICIOUS = 2, - DANGER = 3, + SUSPICIOUS = 1, + DANGER = 2, }