From 00371546a3c394bf1e4806b67e3d489854aa0b94 Mon Sep 17 00:00:00 2001
From: leccelecce <24962424+leccelecce@users.noreply.github.com>
Date: Mon, 23 Dec 2024 14:05:34 +0000
Subject: [PATCH] GenAI: add ability to save JPGs sent to provider (#15643)

* GenAI: add ability to save JPGs sent to provider

* Remove mention from GenAI docs

* Change config name to debug_save_thumbnails

* Change  folder structure to clips/genai-requests/{event_id}/{1.jpg}
---
 docs/docs/configuration/reference.md |  2 ++
 frigate/config/camera/genai.py       |  4 ++++
 frigate/embeddings/maintainer.py     | 31 +++++++++++++++++++++++++++-
 3 files changed, 36 insertions(+), 1 deletion(-)

diff --git a/docs/docs/configuration/reference.md b/docs/docs/configuration/reference.md
index b13b137d2..626e4f3c8 100644
--- a/docs/docs/configuration/reference.md
+++ b/docs/docs/configuration/reference.md
@@ -760,6 +760,8 @@ cameras:
         - cat
       # Optional: Restrict generation to objects that entered any of the listed zones (default: none, all zones qualify)
       required_zones: []
+      # Optional: Save thumbnails sent to generative AI for review/debugging purposes (default: shown below)
+      debug_save_thumbnails: False
 
 # Optional
 ui:
diff --git a/frigate/config/camera/genai.py b/frigate/config/camera/genai.py
index 35c26eaf8..e6b327836 100644
--- a/frigate/config/camera/genai.py
+++ b/frigate/config/camera/genai.py
@@ -38,6 +38,10 @@ class GenAICameraConfig(BaseModel):
         default_factory=list,
         title="List of required zones to be entered in order to run generative AI.",
     )
+    debug_save_thumbnails: bool = Field(
+        default=False,
+        title="Save thumbnails sent to generative AI for debugging purposes.",
+    )
 
     @field_validator("required_zones", mode="before")
     @classmethod
diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py
index 4f81ec2d6..341a2e25d 100644
--- a/frigate/embeddings/maintainer.py
+++ b/frigate/embeddings/maintainer.py
@@ -5,6 +5,7 @@ import logging
 import os
 import threading
 from multiprocessing.synchronize import Event as MpEvent
+from pathlib import Path
 from typing import Optional
 
 import cv2
@@ -217,6 +218,8 @@ class EmbeddingMaintainer(threading.Thread):
                             _, buffer = cv2.imencode(".jpg", cropped_image)
                             snapshot_image = buffer.tobytes()
 
+                    num_thumbnails = len(self.tracked_events.get(event_id, []))
+
                     embed_image = (
                         [snapshot_image]
                         if event.has_snapshot and camera_config.genai.use_snapshot
@@ -225,11 +228,37 @@ class EmbeddingMaintainer(threading.Thread):
                                 data["thumbnail"]
                                 for data in self.tracked_events[event_id]
                             ]
-                            if len(self.tracked_events.get(event_id, [])) > 0
+                            if num_thumbnails > 0
                             else [thumbnail]
                         )
                     )
 
+                    if camera_config.genai.debug_save_thumbnails and num_thumbnails > 0:
+                        logger.debug(
+                            f"Saving {num_thumbnails} thumbnails for event {event.id}"
+                        )
+
+                        Path(
+                            os.path.join(CLIPS_DIR, f"genai-requests/{event.id}")
+                        ).mkdir(parents=True, exist_ok=True)
+
+                        for idx, data in enumerate(self.tracked_events[event_id], 1):
+                            jpg_bytes: bytes = data["thumbnail"]
+
+                            if jpg_bytes is None:
+                                logger.warning(
+                                    f"Unable to save thumbnail {idx} for {event.id}."
+                                )
+                            else:
+                                with open(
+                                    os.path.join(
+                                        CLIPS_DIR,
+                                        f"genai-requests/{event.id}/{idx}.jpg",
+                                    ),
+                                    "wb",
+                                ) as j:
+                                    j.write(jpg_bytes)
+
                     # Generate the description. Call happens in a thread since it is network bound.
                     threading.Thread(
                         target=self._embed_description,