Bug fixes (#14263)

* Simplify loitering logic

* Fix divide by zero

* Add device config for semantic search

* Add docs

docs/docs/configuration/reference.md

@@ -518,6 +518,8 @@ semantic_search:
   enabled: False
   # Optional: Re-index embeddings database from historical tracked objects (default: shown below)
   reindex: False
+  # Optional: Set device used to run embeddings, options are AUTO, CPU, GPU. (default: shown below)
+  device: "AUTO"
 
 # Optional: Configuration for AI generated tracked object descriptions
 # NOTE: Semantic Search must be enabled for this to do anything.

frigate/app.py

@@ -276,7 +276,7 @@ class FrigateApp:
     def init_embeddings_client(self) -> None:
         if self.config.semantic_search.enabled:
             # Create a client for other processes to use
-            self.embeddings = EmbeddingsContext(self.db)
+            self.embeddings = EmbeddingsContext(self.config, self.db)
 
     def init_external_event_processor(self) -> None:
         self.external_event_processor = ExternalEventProcessor(self.config)

frigate/config/semantic_search.py

@@ -12,3 +12,4 @@ class SemanticSearchConfig(FrigateBaseModel):
     reindex: Optional[bool] = Field(
         default=False, title="Reindex all detections on startup."
     )
+    device: str = Field(default="AUTO", title="Device Type")

frigate/embeddings/__init__.py

@@ -55,7 +55,7 @@ def manage_embeddings(config: FrigateConfig) -> None:
     models = [Event]
     db.bind(models)
 
-    embeddings = Embeddings(db)
+    embeddings = Embeddings(config.semantic_search, db)
 
     # Check if we need to re-index events
     if config.semantic_search.reindex:
@@ -70,8 +70,8 @@ def manage_embeddings(config: FrigateConfig) -> None:
 
 
 class EmbeddingsContext:
-    def __init__(self, db: SqliteVecQueueDatabase):
-        self.embeddings = Embeddings(db)
+    def __init__(self, config: FrigateConfig, db: SqliteVecQueueDatabase):
+        self.embeddings = Embeddings(config.semantic_search, db)
         self.thumb_stats = ZScoreNormalization()
         self.desc_stats = ZScoreNormalization()
 

frigate/embeddings/embeddings.py

@@ -12,6 +12,7 @@ from PIL import Image
 from playhouse.shortcuts import model_to_dict
 
 from frigate.comms.inter_process import InterProcessRequestor
+from frigate.config.semantic_search import SemanticSearchConfig
 from frigate.const import UPDATE_MODEL_STATE
 from frigate.db.sqlitevecq import SqliteVecQueueDatabase
 from frigate.models import Event
@@ -80,7 +81,10 @@ def deserialize(bytes_data: bytes) -> List[float]:
 class Embeddings:
     """SQLite-vec embeddings database."""
 
-    def __init__(self, db: SqliteVecQueueDatabase) -> None:
+    def __init__(
+        self, config: SemanticSearchConfig, db: SqliteVecQueueDatabase
+    ) -> None:
+        self.config = config
         self.db = db
         self.requestor = InterProcessRequestor()
 
@@ -118,7 +122,7 @@ class Embeddings:
             },
             embedding_function=jina_text_embedding_function,
             model_type="text",
-            force_cpu=True,
+            device="CPU",
         )
 
         self.vision_embedding = GenericONNXEmbedding(
@@ -130,6 +134,7 @@ class Embeddings:
             },
             embedding_function=jina_vision_embedding_function,
             model_type="vision",
+            device=self.config.device,
         )
 
     def _create_tables(self):

frigate/embeddings/functions/onnx.py

@@ -42,7 +42,7 @@ class GenericONNXEmbedding:
         embedding_function: Callable[[List[np.ndarray]], np.ndarray],
         model_type: str,
         tokenizer_file: Optional[str] = None,
-        force_cpu: bool = False,
+        device: str = "AUTO",
     ):
         self.model_name = model_name
         self.model_file = model_file
@@ -51,7 +51,7 @@ class GenericONNXEmbedding:
         self.embedding_function = embedding_function
         self.model_type = model_type  # 'text' or 'vision'
         self.providers, self.provider_options = get_ort_providers(
-            force_cpu=force_cpu, requires_fp16=True
+            force_cpu=device == "CPU", requires_fp16=True, openvino_device=device
         )
 
         self.download_path = os.path.join(MODEL_CACHE_DIR, self.model_name)

frigate/embeddings/maintainer.py

@@ -42,7 +42,7 @@ class EmbeddingMaintainer(threading.Thread):
         threading.Thread.__init__(self)
         self.name = "embeddings_maintainer"
         self.config = config
-        self.embeddings = Embeddings(db)
+        self.embeddings = Embeddings(config.semantic_search, db)
         self.event_subscriber = EventUpdateSubscriber()
         self.event_end_subscriber = EventEndSubscriber()
         self.event_metadata_subscriber = EventMetadataSubscriber(

frigate/events/cleanup.py

@@ -36,7 +36,7 @@ class EventCleanup(threading.Thread):
         self.camera_labels: dict[str, dict[str, any]] = {}
 
         if self.config.semantic_search.enabled:
-            self.embeddings = Embeddings(self.db)
+            self.embeddings = Embeddings(self.config.semantic_search, self.db)
 
     def get_removed_camera_labels(self) -> list[Event]:
         """Get a list of distinct labels for removed cameras."""

frigate/review/maintainer.py

@@ -239,7 +239,11 @@ class ReviewSegmentMaintainer(threading.Thread):
     ) -> None:
         """Validate if existing review segment should continue."""
         camera_config = self.config.cameras[segment.camera]
-        active_objects = get_active_objects(frame_time, camera_config, objects)
+
+        # get active objects + objects loitering in loitering zones
+        active_objects = get_active_objects(
+            frame_time, camera_config, objects
+        ) + get_loitering_objects(frame_time, camera_config, objects)
         prev_data = segment.get_data(False)
         has_activity = False
 
@@ -304,37 +308,6 @@ class ReviewSegmentMaintainer(threading.Thread):
                 except FileNotFoundError:
                     return
 
-        # check if there are any objects pending loitering on this camera
-        loitering_objects = get_loitering_objects(frame_time, camera_config, objects)
-
-        if loitering_objects:
-            has_activity = True
-
-            if frame_time > segment.last_update:
-                segment.last_update = frame_time
-
-            for object in loitering_objects:
-                # if object is alert label
-                # and has entered loitering zone
-                # mark this review as alert
-                if (
-                    segment.severity != SeverityEnum.alert
-                    and object["label"] in camera_config.review.alerts.labels
-                    and (
-                        len(object["current_zones"]) > 0
-                        and set(object["current_zones"])
-                        & set(camera_config.review.alerts.required_zones)
-                    )
-                ):
-                    segment.severity = SeverityEnum.alert
-                    should_update = True
-
-                # keep zones up to date
-                if len(object["current_zones"]) > 0:
-                    for zone in object["current_zones"]:
-                        if zone not in segment.zones:
-                            segment.zones.append(zone)
-
         if not has_activity:
             if not segment.has_frame:
                 try:

frigate/util/services.py

@@ -318,10 +318,11 @@ def get_intel_gpu_stats() -> dict[str, str]:
                     if video_frame is not None:
                         video[key].append(float(video_frame))
 
-        results["gpu"] = (
-            f"{round(((sum(render['global']) / len(render['global'])) + (sum(video['global']) / len(video['global']))) / 2, 2)}%"
-        )
-        results["mem"] = "-%"
+        if render["global"]:
+            results["gpu"] = (
+                f"{round(((sum(render['global']) / len(render['global'])) + (sum(video['global']) / len(video['global']))) / 2, 2)}%"
+            )
+            results["mem"] = "-%"
 
         if len(render.keys()) > 1:
             results["clients"] = {}