From 76c35307b2ac4689ece53016354fad872c465b78 Mon Sep 17 00:00:00 2001
From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>
Date: Tue, 4 Mar 2025 08:34:19 -0600
Subject: [PATCH] Ensure genai thumbnails are always jpegs (#16939)

---
 frigate/embeddings/maintainer.py | 12 +++++++++++-
 frigate/util/image.py            | 19 +++++++++++++++++++
 2 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py
index dfaed532e..b19a626b2 100644
--- a/frigate/embeddings/maintainer.py
+++ b/frigate/embeddings/maintainer.py
@@ -48,7 +48,11 @@ from frigate.genai import get_genai_client
 from frigate.models import Event
 from frigate.types import TrackedObjectUpdateTypesEnum
 from frigate.util.builtin import serialize
-from frigate.util.image import SharedMemoryFrameManager, calculate_region
+from frigate.util.image import (
+    SharedMemoryFrameManager,
+    calculate_region,
+    ensure_jpeg_bytes,
+)
 from frigate.util.path import get_event_thumbnail_bytes
 
 from .embeddings import Embeddings
@@ -374,6 +378,9 @@ class EmbeddingMaintainer(threading.Thread):
 
         num_thumbnails = len(self.tracked_events.get(event.id, []))
 
+        # ensure we have a jpeg to pass to the model
+        thumbnail = ensure_jpeg_bytes(thumbnail)
+
         embed_image = (
             [snapshot_image]
             if event.has_snapshot and camera_config.genai.use_snapshot
@@ -503,6 +510,9 @@ class EmbeddingMaintainer(threading.Thread):
 
         thumbnail = get_event_thumbnail_bytes(event)
 
+        # ensure we have a jpeg to pass to the model
+        thumbnail = ensure_jpeg_bytes(thumbnail)
+
         logger.debug(
             f"Trying {source} regeneration for {event}, has_snapshot: {event.has_snapshot}"
         )
diff --git a/frigate/util/image.py b/frigate/util/image.py
index 20806372c..0b80efe88 100644
--- a/frigate/util/image.py
+++ b/frigate/util/image.py
@@ -975,3 +975,22 @@ def get_histogram(image, x_min, y_min, x_max, y_max):
         [image_bgr], [0, 1, 2], None, [8, 8, 8], [0, 256, 0, 256, 0, 256]
     )
     return cv2.normalize(hist, hist).flatten()
+
+
+def ensure_jpeg_bytes(image_data):
+    """Ensure image data is jpeg bytes for genai"""
+    try:
+        img_array = np.frombuffer(image_data, dtype=np.uint8)
+        img = cv2.imdecode(img_array, cv2.IMREAD_COLOR)
+
+        if img is None:
+            return image_data
+
+        success, encoded_img = cv2.imencode(".jpg", img)
+
+        if success:
+            return encoded_img.tobytes()
+    except Exception as e:
+        logger.warning(f"Error when converting thumbnail to jpeg for genai: {e}")
+
+    return image_data