diff --git a/docs/docs/configuration/genai.md b/docs/docs/configuration/genai.md index 2ec9a6276..da4b8afd8 100644 --- a/docs/docs/configuration/genai.md +++ b/docs/docs/configuration/genai.md @@ -35,7 +35,7 @@ cameras: :::warning -Using Ollama on CPU is not recommended, high inference times and a lack of support for multi-modal parallel requests will make using Generative AI impractical. +Using Ollama on CPU is not recommended, high inference times make using Generative AI impractical. ::: @@ -43,7 +43,7 @@ Using Ollama on CPU is not recommended, high inference times and a lack of suppo Most of the 7b parameter 4-bit vision models will fit inside 8GB of VRAM. There is also a [Docker container](https://hub.docker.com/r/ollama/ollama) available. -Parallel requests also come with some caveats, and multi-modal parallel requests are currently not supported by Ollama. Depending on your hardware and the number of requests made to the Ollama API, these limitations may prevent Ollama from being an optimal solution for many users. See the [Ollama documentation](https://github.com/ollama/ollama/blob/main/docs/faq.md#how-does-ollama-handle-concurrent-requests). +Parallel requests also come with some caveats. You will need to set `OLLAMA_NUM_PARALLEL=1` and choose a `OLLAMA_MAX_QUEUE` and `OLLAMA_MAX_LOADED_MODELS` values that are appropriate for your hardware and preferences. See the [Ollama documentation](https://github.com/ollama/ollama/blob/main/docs/faq.md#how-does-ollama-handle-concurrent-requests). ### Supported Models diff --git a/frigate/genai/__init__.py b/frigate/genai/__init__.py index ebbea7476..2c0aadbd9 100644 --- a/frigate/genai/__init__.py +++ b/frigate/genai/__init__.py @@ -1,6 +1,7 @@ """Generative AI module for Frigate.""" import importlib +import logging import os from typing import Optional @@ -9,6 +10,8 @@ from playhouse.shortcuts import model_to_dict from frigate.config import CameraConfig, FrigateConfig, GenAIConfig, GenAIProviderEnum from frigate.models import Event +logger = logging.getLogger(__name__) + PROVIDERS = {} @@ -41,6 +44,7 @@ class GenAIClient: event.label, camera_config.genai.prompt, ).format(**model_to_dict(event)) + logger.debug(f"Sending images to genai provider with prompt: {prompt}") return self._send(prompt, thumbnails) def _init_provider(self):