Add embeddings reindex progress to the UI (#14268)

* refactor dispatcher

* add reindex to dictionary

* add circular progress bar component

* Add progress to UI when embeddings are reindexing

* readd comments to dispatcher for clarity

* Only report progress every 10 events so we don't spam the logs and websocket

* clean up

.cspell/frigate-dictionary.txt

@@ -212,6 +212,7 @@ rcond
 RDONLY
 rebranded
 referer
+reindex
 Reolink
 restream
 restreamed

frigate/comms/dispatcher.py

@@ -15,6 +15,7 @@ from frigate.const import (
     INSERT_PREVIEW,
     REQUEST_REGION_GRID,
     UPDATE_CAMERA_ACTIVITY,
+    UPDATE_EMBEDDINGS_REINDEX_PROGRESS,
     UPDATE_EVENT_DESCRIPTION,
     UPDATE_MODEL_STATE,
     UPSERT_REVIEW_SEGMENT,
@@ -86,35 +87,27 @@ class Dispatcher:
 
         self.camera_activity = {}
         self.model_state = {}
+        self.embeddings_reindex = {}
 
     def _receive(self, topic: str, payload: str) -> Optional[Any]:
         """Handle receiving of payload from communicators."""
-        if topic.endswith("set"):
+
+        def handle_camera_command(command_type, camera_name, payload):
             try:
-                # example /cam_name/detect/set payload=ON|OFF
+                if command_type == "set":
-                if topic.count("/") == 2:
+                    self._camera_settings_handlers[camera_name](camera_name, payload)
-                    camera_name = topic.split("/")[-3]
+                elif command_type == "ptz":
-                    command = topic.split("/")[-2]
+                    self._on_ptz_command(camera_name, payload)
-                    self._camera_settings_handlers[command](camera_name, payload)
+            except KeyError:
-                elif topic.count("/") == 1:
+                logger.error(f"Invalid command type: {command_type}")
-                    command = topic.split("/")[-2]
+
-                    self._global_settings_handlers[command](payload)
+        def handle_restart():
-            except IndexError:
-                logger.error(f"Received invalid set command: {topic}")
-                return
-        elif topic.endswith("ptz"):
-            try:
-                # example /cam_name/ptz payload=MOVE_UP|MOVE_DOWN|STOP...
-                camera_name = topic.split("/")[-2]
-                self._on_ptz_command(camera_name, payload)
-            except IndexError:
-                logger.error(f"Received invalid ptz command: {topic}")
-                return
-        elif topic == "restart":
             restart_frigate()
-        elif topic == INSERT_MANY_RECORDINGS:
+
+        def handle_insert_many_recordings():
             Recordings.insert_many(payload).execute()
-        elif topic == REQUEST_REGION_GRID:
+
+        def handle_request_region_grid():
             camera = payload
             grid = get_camera_regions_grid(
                 camera,
@@ -122,24 +115,25 @@ class Dispatcher:
                 max(self.config.model.width, self.config.model.height),
             )
             return grid
-        elif topic == INSERT_PREVIEW:
+
+        def handle_insert_preview():
             Previews.insert(payload).execute()
-        elif topic == UPSERT_REVIEW_SEGMENT:
+
-            (
+        def handle_upsert_review_segment():
-                ReviewSegment.insert(payload)
+            ReviewSegment.insert(payload).on_conflict(
-                .on_conflict(
+                conflict_target=[ReviewSegment.id],
-                    conflict_target=[ReviewSegment.id],
+                update=payload,
-                    update=payload,
-                )
-                .execute()
-            )
-        elif topic == CLEAR_ONGOING_REVIEW_SEGMENTS:
-            ReviewSegment.update(end_time=datetime.datetime.now().timestamp()).where(
-                ReviewSegment.end_time == None
             ).execute()
-        elif topic == UPDATE_CAMERA_ACTIVITY:
+
+        def handle_clear_ongoing_review_segments():
+            ReviewSegment.update(end_time=datetime.datetime.now().timestamp()).where(
+                ReviewSegment.end_time.is_null(True)
+            ).execute()
+
+        def handle_update_camera_activity():
             self.camera_activity = payload
-        elif topic == UPDATE_EVENT_DESCRIPTION:
+
+        def handle_update_event_description():
             event: Event = Event.get(Event.id == payload["id"])
             event.data["description"] = payload["description"]
             event.save()
@@ -147,15 +141,30 @@ class Dispatcher:
                 "event_update",
                 json.dumps({"id": event.id, "description": event.data["description"]}),
             )
-        elif topic == UPDATE_MODEL_STATE:
+
+        def handle_update_model_state():
             model = payload["model"]
             state = payload["state"]
             self.model_state[model] = ModelStatusTypesEnum[state]
             self.publish("model_state", json.dumps(self.model_state))
-        elif topic == "modelState":
+
-            model_state = self.model_state.copy()
+        def handle_model_state():
-            self.publish("model_state", json.dumps(model_state))
+            self.publish("model_state", json.dumps(self.model_state.copy()))
-        elif topic == "onConnect":
+
+        def handle_update_embeddings_reindex_progress():
+            self.embeddings_reindex = payload
+            self.publish(
+                "embeddings_reindex_progress",
+                json.dumps(payload),
+            )
+
+        def handle_embeddings_reindex_progress():
+            self.publish(
+                "embeddings_reindex_progress",
+                json.dumps(self.embeddings_reindex.copy()),
+            )
+
+        def handle_on_connect():
             camera_status = self.camera_activity.copy()
 
             for camera in camera_status.keys():
@@ -170,6 +179,46 @@ class Dispatcher:
                 }
 
             self.publish("camera_activity", json.dumps(camera_status))
+
+        # Dictionary mapping topic to handlers
+        topic_handlers = {
+            INSERT_MANY_RECORDINGS: handle_insert_many_recordings,
+            REQUEST_REGION_GRID: handle_request_region_grid,
+            INSERT_PREVIEW: handle_insert_preview,
+            UPSERT_REVIEW_SEGMENT: handle_upsert_review_segment,
+            CLEAR_ONGOING_REVIEW_SEGMENTS: handle_clear_ongoing_review_segments,
+            UPDATE_CAMERA_ACTIVITY: handle_update_camera_activity,
+            UPDATE_EVENT_DESCRIPTION: handle_update_event_description,
+            UPDATE_MODEL_STATE: handle_update_model_state,
+            UPDATE_EMBEDDINGS_REINDEX_PROGRESS: handle_update_embeddings_reindex_progress,
+            "restart": handle_restart,
+            "embeddingsReindexProgress": handle_embeddings_reindex_progress,
+            "modelState": handle_model_state,
+            "onConnect": handle_on_connect,
+        }
+
+        if topic.endswith("set") or topic.endswith("ptz"):
+            try:
+                parts = topic.split("/")
+                if len(parts) == 3 and topic.endswith("set"):
+                    # example /cam_name/detect/set payload=ON|OFF
+                    camera_name = parts[-3]
+                    command = parts[-2]
+                    handle_camera_command("set", camera_name, payload)
+                elif len(parts) == 2 and topic.endswith("set"):
+                    command = parts[-2]
+                    self._global_settings_handlers[command](payload)
+                elif len(parts) == 2 and topic.endswith("ptz"):
+                    # example /cam_name/ptz payload=MOVE_UP|MOVE_DOWN|STOP...
+                    camera_name = parts[-2]
+                    handle_camera_command("ptz", camera_name, payload)
+            except IndexError:
+                logger.error(
+                    f"Received invalid {topic.split('/')[-1]} command: {topic}"
+                )
+                return
+        elif topic in topic_handlers:
+            return topic_handlers[topic]()
         else:
             self.publish(topic, payload, retain=False)
 

frigate/const.py

@@ -85,6 +85,7 @@ CLEAR_ONGOING_REVIEW_SEGMENTS = "clear_ongoing_review_segments"
 UPDATE_CAMERA_ACTIVITY = "update_camera_activity"
 UPDATE_EVENT_DESCRIPTION = "update_event_description"
 UPDATE_MODEL_STATE = "update_model_state"
+UPDATE_EMBEDDINGS_REINDEX_PROGRESS = "handle_embeddings_reindex_progress"
 
 # Stats Values
 

frigate/embeddings/embeddings.py

@@ -10,7 +10,7 @@ from playhouse.shortcuts import model_to_dict
 
 from frigate.comms.inter_process import InterProcessRequestor
 from frigate.config.semantic_search import SemanticSearchConfig
-from frigate.const import UPDATE_MODEL_STATE
+from frigate.const import UPDATE_EMBEDDINGS_REINDEX_PROGRESS, UPDATE_MODEL_STATE
 from frigate.db.sqlitevecq import SqliteVecQueueDatabase
 from frigate.models import Event
 from frigate.types import ModelStatusTypesEnum
@@ -165,19 +165,36 @@ class Embeddings:
         return embedding
 
     def reindex(self) -> None:
-        logger.info("Indexing event embeddings...")
+        logger.info("Indexing tracked object embeddings...")
 
         self._drop_tables()
         self._create_tables()
 
         st = time.time()
         totals = {
-            "thumb": 0,
+            "thumbnails": 0,
-            "desc": 0,
+            "descriptions": 0,
+            "processed_objects": 0,
+            "total_objects": 0,
         }
 
+        self.requestor.send_data(UPDATE_EMBEDDINGS_REINDEX_PROGRESS, totals)
+
+        # Get total count of events to process
+        total_events = (
+            Event.select()
+            .where(
+                (Event.has_clip == True | Event.has_snapshot == True)
+                & Event.thumbnail.is_null(False)
+            )
+            .count()
+        )
+        totals["total_objects"] = total_events
+
         batch_size = 100
         current_page = 1
+        processed_events = 0
+
         events = (
             Event.select()
             .where(
@@ -193,11 +210,29 @@ class Embeddings:
             for event in events:
                 thumbnail = base64.b64decode(event.thumbnail)
                 self.upsert_thumbnail(event.id, thumbnail)
-                totals["thumb"] += 1
+                totals["thumbnails"] += 1
+
                 if description := event.data.get("description", "").strip():
-                    totals["desc"] += 1
+                    totals["descriptions"] += 1
                     self.upsert_description(event.id, description)
 
+                totals["processed_objects"] += 1
+
+                # report progress every 10 events so we don't spam the logs
+                if (totals["processed_objects"] % 10) == 0:
+                    progress = (processed_events / total_events) * 100
+                    logger.debug(
+                        "Processed %d/%d events (%.2f%% complete) | Thumbnails: %d, Descriptions: %d",
+                        processed_events,
+                        total_events,
+                        progress,
+                        totals["thumbnails"],
+                        totals["descriptions"],
+                    )
+
+                self.requestor.send_data(UPDATE_EMBEDDINGS_REINDEX_PROGRESS, totals)
+
+            # Move to the next page
             current_page += 1
             events = (
                 Event.select()
@@ -211,7 +246,8 @@ class Embeddings:
 
         logger.info(
             "Embedded %d thumbnails and %d descriptions in %s seconds",
-            totals["thumb"],
+            totals["thumbnails"],
-            totals["desc"],
+            totals["descriptions"],
             time.time() - st,
         )
+        self.requestor.send_data(UPDATE_EMBEDDINGS_REINDEX_PROGRESS, totals)

web/src/api/ws.tsx

@@ -2,6 +2,7 @@ import { baseUrl } from "./baseUrl";
 import { useCallback, useEffect, useState } from "react";
 import useWebSocket, { ReadyState } from "react-use-websocket";
 import {
+  EmbeddingsReindexProgressType,
   FrigateCameraState,
   FrigateEvent,
   FrigateReview,
@@ -302,6 +303,42 @@ export function useModelState(
   return { payload: data ? data[model] : undefined };
 }
 
+export function useEmbeddingsReindexProgress(
+  revalidateOnFocus: boolean = true,
+): {
+  payload: EmbeddingsReindexProgressType;
+} {
+  const {
+    value: { payload },
+    send: sendCommand,
+  } = useWs("embeddings_reindex_progress", "embeddingsReindexProgress");
+
+  const data = useDeepMemo(JSON.parse(payload as string));
+
+  useEffect(() => {
+    let listener = undefined;
+    if (revalidateOnFocus) {
+      sendCommand("embeddingsReindexProgress");
+      listener = () => {
+        if (document.visibilityState == "visible") {
+          sendCommand("embeddingsReindexProgress");
+        }
+      };
+      addEventListener("visibilitychange", listener);
+    }
+
+    return () => {
+      if (listener) {
+        removeEventListener("visibilitychange", listener);
+      }
+    };
+    // we know that these deps are correct
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [revalidateOnFocus]);
+
+  return { payload: data };
+}
+
 export function useMotionActivity(camera: string): { payload: string } {
   const {
     value: { payload },

web/src/components/ui/circular-progress-bar.tsx

@@ -0,0 +1,108 @@
+import { cn } from "@/lib/utils";
+
+interface Props {
+  max: number;
+  value: number;
+  min: number;
+  gaugePrimaryColor: string;
+  gaugeSecondaryColor: string;
+  className?: string;
+}
+
+export default function AnimatedCircularProgressBar({
+  max = 100,
+  min = 0,
+  value = 0,
+  gaugePrimaryColor,
+  gaugeSecondaryColor,
+  className,
+}: Props) {
+  const circumference = 2 * Math.PI * 45;
+  const percentPx = circumference / 100;
+  const currentPercent = Math.floor(((value - min) / (max - min)) * 100);
+
+  return (
+    <div
+      className={cn("relative size-40 text-2xl font-semibold", className)}
+      style={
+        {
+          "--circle-size": "100px",
+          "--circumference": circumference,
+          "--percent-to-px": `${percentPx}px`,
+          "--gap-percent": "5",
+          "--offset-factor": "0",
+          "--transition-length": "1s",
+          "--transition-step": "200ms",
+          "--delay": "0s",
+          "--percent-to-deg": "3.6deg",
+          transform: "translateZ(0)",
+        } as React.CSSProperties
+      }
+    >
+      <svg
+        fill="none"
+        className="size-full"
+        strokeWidth="2"
+        viewBox="0 0 100 100"
+      >
+        {currentPercent <= 90 && currentPercent >= 0 && (
+          <circle
+            cx="50"
+            cy="50"
+            r="45"
+            strokeWidth="10"
+            strokeDashoffset="0"
+            strokeLinecap="round"
+            strokeLinejoin="round"
+            className="opacity-100"
+            style={
+              {
+                stroke: gaugeSecondaryColor,
+                "--stroke-percent": 90 - currentPercent,
+                "--offset-factor-secondary": "calc(1 - var(--offset-factor))",
+                strokeDasharray:
+                  "calc(var(--stroke-percent) * var(--percent-to-px)) var(--circumference)",
+                transform:
+                  "rotate(calc(1turn - 90deg - (var(--gap-percent) * var(--percent-to-deg) * var(--offset-factor-secondary)))) scaleY(-1)",
+                transition: "all var(--transition-length) ease var(--delay)",
+                transformOrigin:
+                  "calc(var(--circle-size) / 2) calc(var(--circle-size) / 2)",
+              } as React.CSSProperties
+            }
+          />
+        )}
+        <circle
+          cx="50"
+          cy="50"
+          r="45"
+          strokeWidth="10"
+          strokeDashoffset="0"
+          strokeLinecap="round"
+          strokeLinejoin="round"
+          className="opacity-100"
+          style={
+            {
+              stroke: gaugePrimaryColor,
+              "--stroke-percent": currentPercent,
+              strokeDasharray:
+                "calc(var(--stroke-percent) * var(--percent-to-px)) var(--circumference)",
+              transition:
+                "var(--transition-length) ease var(--delay),stroke var(--transition-length) ease var(--delay)",
+              transitionProperty: "stroke-dasharray,transform",
+              transform:
+                "rotate(calc(-90deg + var(--gap-percent) * var(--offset-factor) * var(--percent-to-deg)))",
+              transformOrigin:
+                "calc(var(--circle-size) / 2) calc(var(--circle-size) / 2)",
+            } as React.CSSProperties
+          }
+        />
+      </svg>
+      <span
+        data-current-value={currentPercent}
+        className="duration-[var(--transition-length)] delay-[var(--delay)] absolute inset-0 m-auto size-fit ease-linear animate-in fade-in"
+      >
+        {currentPercent}%
+      </span>
+    </div>
+  );
+}

web/src/pages/Explore.tsx

@@ -1,5 +1,10 @@
-import { useEventUpdate, useModelState } from "@/api/ws";
+import {
+  useEmbeddingsReindexProgress,
+  useEventUpdate,
+  useModelState,
+} from "@/api/ws";
 import ActivityIndicator from "@/components/indicators/activity-indicator";
+import AnimatedCircularProgressBar from "@/components/ui/circular-progress-bar";
 import { useApiFilterArgs } from "@/hooks/use-api-filter";
 import { useTimezone } from "@/hooks/use-date-utils";
 import { FrigateConfig } from "@/types/frigateConfig";
@@ -182,6 +187,18 @@ export default function Explore() {
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [eventUpdate]);
 
+  // embeddings reindex progress
+
+  const { payload: reindexProgress } = useEmbeddingsReindexProgress();
+
+  const embeddingsReindexing = useMemo(
+    () =>
+      reindexProgress
+        ? reindexProgress.total_objects - reindexProgress.processed_objects > 0
+        : undefined,
+    [reindexProgress],
+  );
+
   // model states
 
   const { payload: textModelState } = useModelState(
@@ -238,59 +255,101 @@ export default function Explore() {
 
   return (
     <>
-      {config?.semantic_search.enabled && !allModelsLoaded ? (
+      {config?.semantic_search.enabled &&
+      (!allModelsLoaded || embeddingsReindexing) ? (
         <div className="absolute inset-0 left-1/2 top-1/2 flex h-96 w-96 -translate-x-1/2 -translate-y-1/2">
-          <div className="flex flex-col items-center justify-center space-y-3 rounded-lg bg-background/50 p-5">
+          <div className="flex max-w-96 flex-col items-center justify-center space-y-3 rounded-lg bg-background/50 p-5">
             <div className="my-5 flex flex-col items-center gap-2 text-xl">
               <TbExclamationCircle className="mb-3 size-10" />
               <div>Search Unavailable</div>
             </div>
-            <div className="max-w-96 text-center">
+            {embeddingsReindexing && (
-              Frigate is downloading the necessary embeddings models to support
+              <>
-              semantic searching. This may take several minutes depending on the
+                <div className="text-center text-primary-variant">
-              speed of your network connection.
+                  Search can be used after tracked object embeddings have
-            </div>
+                  finished reindexing.
-            <div className="flex w-96 flex-col gap-2 py-5">
+                </div>
-              <div className="flex flex-row items-center justify-center gap-2">
+                <div className="pt-5 text-center">
-                {renderModelStateIcon(visionModelState)}
+                  <AnimatedCircularProgressBar
-                Vision model
+                    min={0}
-              </div>
+                    max={reindexProgress.total_objects}
-              <div className="flex flex-row items-center justify-center gap-2">
+                    value={reindexProgress.processed_objects}
-                {renderModelStateIcon(visionFeatureExtractorState)}
+                    gaugePrimaryColor="hsl(var(--selected))"
-                Vision model feature extractor
+                    gaugeSecondaryColor="hsl(var(--secondary))"
-              </div>
+                  />
-              <div className="flex flex-row items-center justify-center gap-2">
+                </div>
-                {renderModelStateIcon(textModelState)}
+                <div className="flex w-96 flex-col gap-2 py-5">
-                Text model
+                  <div className="flex flex-row items-center justify-center gap-3">
-              </div>
+                    <span className="text-primary-variant">
-              <div className="flex flex-row items-center justify-center gap-2">
+                      Thumbnails embedded:
-                {renderModelStateIcon(textTokenizerState)}
+                    </span>
-                Text tokenizer
+                    {reindexProgress.thumbnails}
-              </div>
+                  </div>
-            </div>
+                  <div className="flex flex-row items-center justify-center gap-3">
-            {(textModelState === "error" ||
+                    <span className="text-primary-variant">
-              textTokenizerState === "error" ||
+                      Descriptions embedded:
-              visionModelState === "error" ||
+                    </span>
-              visionFeatureExtractorState === "error") && (
+                    {reindexProgress.descriptions}
-              <div className="my-3 max-w-96 text-center text-danger">
+                  </div>
-                An error has occurred. Check Frigate logs.
+                  <div className="flex flex-row items-center justify-center gap-3">
-              </div>
+                    <span className="text-primary-variant">
+                      Tracked objects processed:
+                    </span>
+                    {reindexProgress.processed_objects}
+                  </div>
+                </div>
+              </>
+            )}
+            {!allModelsLoaded && (
+              <>
+                <div className="text-center text-primary-variant">
+                  Frigate is downloading the necessary embeddings models to
+                  support semantic searching. This may take several minutes
+                  depending on the speed of your network connection.
+                </div>
+                <div className="flex w-96 flex-col gap-2 py-5">
+                  <div className="flex flex-row items-center justify-center gap-2">
+                    {renderModelStateIcon(visionModelState)}
+                    Vision model
+                  </div>
+                  <div className="flex flex-row items-center justify-center gap-2">
+                    {renderModelStateIcon(visionFeatureExtractorState)}
+                    Vision model feature extractor
+                  </div>
+                  <div className="flex flex-row items-center justify-center gap-2">
+                    {renderModelStateIcon(textModelState)}
+                    Text model
+                  </div>
+                  <div className="flex flex-row items-center justify-center gap-2">
+                    {renderModelStateIcon(textTokenizerState)}
+                    Text tokenizer
+                  </div>
+                </div>
+                {(textModelState === "error" ||
+                  textTokenizerState === "error" ||
+                  visionModelState === "error" ||
+                  visionFeatureExtractorState === "error") && (
+                  <div className="my-3 max-w-96 text-center text-danger">
+                    An error has occurred. Check Frigate logs.
+                  </div>
+                )}
+                <div className="text-center text-primary-variant">
+                  You may want to reindex the embeddings of your tracked objects
+                  once the models are downloaded.
+                </div>
+                <div className="flex items-center text-primary-variant">
+                  <Link
+                    to="https://docs.frigate.video/configuration/semantic_search"
+                    target="_blank"
+                    rel="noopener noreferrer"
+                    className="inline"
+                  >
+                    Read the documentation{" "}
+                    <LuExternalLink className="ml-2 inline-flex size-3" />
+                  </Link>
+                </div>
+              </>
             )}
-            <div className="max-w-96 text-center">
-              You may want to reindex the embeddings of your tracked objects
-              once the models are downloaded.
-            </div>
-            <div className="flex max-w-96 items-center text-primary-variant">
-              <Link
-                to="https://docs.frigate.video/configuration/semantic_search"
-                target="_blank"
-                rel="noopener noreferrer"
-                className="inline"
-              >
-                Read the documentation{" "}
-                <LuExternalLink className="ml-2 inline-flex size-3" />
-              </Link>
-            </div>
           </div>
         </div>
       ) : (

web/src/types/ws.ts

@@ -62,4 +62,11 @@ export type ModelState =
   | "downloaded"
   | "error";
 
+export type EmbeddingsReindexProgressType = {
+  thumbnails: number;
+  descriptions: number;
+  processed_objects: number;
+  total_objects: number;
+};
+
 export type ToggleableSetting = "ON" | "OFF";