From d4732e1e1b80013d41075db848f662477e006bea Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Wed, 13 Aug 2025 09:28:01 -0600 Subject: [PATCH] Add ability to pass additional args to Ollama (#19484) * Call out recognized objects more specifically * Cleanup * Make keep_alive and options configurable * Generalize * Use for other providers --- frigate/config/camera/genai.py | 5 +++- .../post/review_descriptions.py | 4 ++- frigate/genai/__init__.py | 29 ++++++++++--------- frigate/genai/gemini.py | 4 ++- frigate/genai/ollama.py | 2 +- frigate/genai/openai.py | 4 ++- 6 files changed, 29 insertions(+), 19 deletions(-) diff --git a/frigate/config/camera/genai.py b/frigate/config/camera/genai.py index b47839dd5..3c6baeb15 100644 --- a/frigate/config/camera/genai.py +++ b/frigate/config/camera/genai.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Optional +from typing import Any, Optional from pydantic import Field @@ -23,3 +23,6 @@ class GenAIConfig(FrigateBaseModel): base_url: Optional[str] = Field(default=None, title="Provider base url.") model: str = Field(default="gpt-4o", title="GenAI model.") provider: GenAIProviderEnum | None = Field(default=None, title="GenAI provider.") + provider_options: dict[str, Any] = Field( + default={}, title="GenAI Provider extra options." + ) diff --git a/frigate/data_processing/post/review_descriptions.py b/frigate/data_processing/post/review_descriptions.py index 6a6127e00..68496fc18 100644 --- a/frigate/data_processing/post/review_descriptions.py +++ b/frigate/data_processing/post/review_descriptions.py @@ -209,7 +209,9 @@ def run_analysis( { "id": final_data["id"], "camera": camera, - "objects": final_data["data"]["objects"], + "objects": list( + filter(lambda o: "-verified" not in o, final_data["data"]["objects"]) + ), "recognized_objects": final_data["data"]["sub_labels"], "zones": final_data["data"]["zones"], "timestamp": datetime.datetime.fromtimestamp(final_data["end_time"]), diff --git a/frigate/genai/__init__.py b/frigate/genai/__init__.py index da73d2ab8..ba936db3c 100644 --- a/frigate/genai/__init__.py +++ b/frigate/genai/__init__.py @@ -46,20 +46,21 @@ class GenAIClient: debug_save: bool, ) -> ReviewMetadata | None: """Generate a description for the review item activity.""" - if concerns: - concern_list = "\n - ".join(concerns) - concern_prompt = f""" + + def get_concern_prompt() -> str: + if concerns: + concern_list = "\n - ".join(concerns) + return f""" - `other_concerns` (list of strings): Include a list of any of the following concerns that are occurring: - - {concern_list} -""" + - {concern_list}""" + else: + return "" - else: - concern_prompt = "" - - if preferred_language: - language_prompt = f"Provide your answer in {preferred_language}" - else: - language_prompt = "" + def get_language_prompt() -> str: + if preferred_language: + return f"Provide your answer in {preferred_language}" + else: + return "" context_prompt = f""" Please analyze the sequence of images ({len(thumbnails)} total) taken in chronological order from the perspective of the {review_data["camera"].replace("_", " ")} security camera. @@ -81,7 +82,7 @@ Your response MUST be a flat JSON object with: - `scene` (string): A full description including setting, entities, actions, and any plausible supported inferences. - `confidence` (float): 0-1 confidence in the analysis. - `potential_threat_level` (integer): 0, 1, or 2 as defined below. -{concern_prompt} +{get_concern_prompt()} Threat-level definitions: - 0 — Typical or expected activity for this location/time (includes residents, guests, or known animals engaged in normal activities, even if they glance around or scan surroundings). @@ -97,7 +98,7 @@ Sequence details: **IMPORTANT:** - Values must be plain strings, floats, or integers — no nested objects, no extra commentary. -{language_prompt} +{get_language_prompt()} """ logger.debug( f"Sending {len(thumbnails)} images to create review description on {review_data['camera']}" diff --git a/frigate/genai/gemini.py b/frigate/genai/gemini.py index 750454e25..8c355b37a 100644 --- a/frigate/genai/gemini.py +++ b/frigate/genai/gemini.py @@ -21,7 +21,9 @@ class GeminiClient(GenAIClient): def _init_provider(self): """Initialize the client.""" genai.configure(api_key=self.genai_config.api_key) - return genai.GenerativeModel(self.genai_config.model) + return genai.GenerativeModel( + self.genai_config.model, **self.genai_config.provider_options + ) def _send(self, prompt: str, images: list[bytes]) -> Optional[str]: """Submit a request to Gemini.""" diff --git a/frigate/genai/ollama.py b/frigate/genai/ollama.py index c392fadb9..0fb44d785 100644 --- a/frigate/genai/ollama.py +++ b/frigate/genai/ollama.py @@ -48,7 +48,7 @@ class OllamaClient(GenAIClient): self.genai_config.model, prompt, images=images if images else None, - keep_alive="1h", + **self.genai_config.provider_options, ) return result["response"].strip() except (TimeoutException, ResponseError) as e: diff --git a/frigate/genai/openai.py b/frigate/genai/openai.py index 76ba8cb44..eb3016fad 100644 --- a/frigate/genai/openai.py +++ b/frigate/genai/openai.py @@ -21,7 +21,9 @@ class OpenAIClient(GenAIClient): def _init_provider(self): """Initialize the client.""" - return OpenAI(api_key=self.genai_config.api_key) + return OpenAI( + api_key=self.genai_config.api_key, **self.genai_config.provider_options + ) def _send(self, prompt: str, images: list[bytes]) -> Optional[str]: """Submit a request to OpenAI."""