Improve Face Library Management (#17213)

* Set maximum number of face images to be kept

* Fix vertical camera scaling

* adjust wording

* Add attributes to search data

* Add button to train face from event

* Handle event id saving in API

frigate/api/classification.py

@@ -6,6 +6,7 @@ import random
 import shutil
 import string
 
+import cv2
 from fastapi import APIRouter, Depends, Request, UploadFile
 from fastapi.responses import JSONResponse
 from pathvalidate import sanitize_filename
@@ -14,9 +15,11 @@ from playhouse.shortcuts import model_to_dict
 
 from frigate.api.auth import require_role
 from frigate.api.defs.tags import Tags
+from frigate.config.camera import DetectConfig
 from frigate.const import FACE_DIR
 from frigate.embeddings import EmbeddingsContext
 from frigate.models import Event
+from frigate.util.path import get_event_snapshot
 
 logger = logging.getLogger(__name__)
 
@@ -87,16 +90,27 @@ def train_face(request: Request, name: str, body: dict = None):
         )
 
     json: dict[str, any] = body or {}
-    training_file = os.path.join(
+    training_file_name = sanitize_filename(json.get("training_file", ""))
-        FACE_DIR, f"train/{sanitize_filename(json.get('training_file', ''))}"
+    training_file = os.path.join(FACE_DIR, f"train/{training_file_name}")
-    )
+    event_id = json.get("event_id")
 
-    if not training_file or not os.path.isfile(training_file):
+    if not training_file_name and not event_id:
         return JSONResponse(
             content=(
                 {
                     "success": False,
-                    "message": f"Invalid filename or no file exists: {training_file}",
+                    "message": "A training file or event_id must be passed.",
+                }
+            ),
+            status_code=400,
+        )
+
+    if training_file_name and not os.path.isfile(training_file):
+        return JSONResponse(
+            content=(
+                {
+                    "success": False,
+                    "message": f"Invalid filename or no file exists: {training_file_name}",
                 }
             ),
             status_code=404,
@@ -106,7 +120,36 @@ def train_face(request: Request, name: str, body: dict = None):
     rand_id = "".join(random.choices(string.ascii_lowercase + string.digits, k=6))
     new_name = f"{sanitized_name}-{rand_id}.webp"
     new_file = os.path.join(FACE_DIR, f"{sanitized_name}/{new_name}")
-    shutil.move(training_file, new_file)
+
+    if training_file_name:
+        shutil.move(training_file, new_file)
+    else:
+        try:
+            event: Event = Event.get(Event.id == event_id)
+        except DoesNotExist:
+            return JSONResponse(
+                content=(
+                    {
+                        "success": False,
+                        "message": f"Invalid event_id or no event exists: {event_id}",
+                    }
+                ),
+                status_code=404,
+            )
+
+        snapshot = get_event_snapshot(event)
+        face_box = event.data["attributes"][0]["box"]
+        detect_config: DetectConfig = request.app.frigate_config.cameras[
+            event.camera
+        ].detect
+
+        # crop onto the face box minus the bounding box itself
+        x1 = int(face_box[0] * detect_config.width) + 2
+        y1 = int(face_box[1] * detect_config.height) + 2
+        x2 = x1 + int(face_box[2] * detect_config.width) - 4
+        y2 = y1 + int(face_box[3] * detect_config.height) - 4
+        face = snapshot[y1:y2, x1:x2]
+        cv2.imwrite(new_file, face)
 
     context: EmbeddingsContext = request.app.embeddings
     context.clear_face_classifier()
@@ -115,7 +158,7 @@ def train_face(request: Request, name: str, body: dict = None):
         content=(
             {
                 "success": True,
-                "message": f"Successfully saved {training_file} as {new_name}.",
+                "message": f"Successfully saved {training_file_name} as {new_name}.",
             }
         ),
         status_code=200,

frigate/api/event.py

@@ -701,6 +701,7 @@ def events_search(request: Request, params: EventsSearchQueryParams = Depends())
             for k, v in event["data"].items()
             if k
             in [
+                "attributes",
                 "type",
                 "score",
                 "top_score",

frigate/data_processing/real_time/face.py

@@ -28,6 +28,7 @@ logger = logging.getLogger(__name__)
 
 
 MAX_DETECTION_HEIGHT = 1080
+MAX_FACE_ATTEMPTS = 100
 MIN_MATCHING_FACES = 2
 
 
@@ -482,6 +483,16 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
                 )
                 shutil.move(current_file, new_file)
 
+                files = sorted(
+                    os.listdir(folder),
+                    key=lambda f: os.path.getctime(os.path.join(folder, f)),
+                    reverse=True,
+                )
+
+                # delete oldest face image if maximum is reached
+                if len(files) > MAX_FACE_ATTEMPTS:
+                    os.unlink(os.path.join(folder, files[-1]))
+
     def expire_object(self, object_id: str):
         if object_id in self.detected_faces:
             self.detected_faces.pop(object_id)

frigate/util/path.py

@@ -4,6 +4,9 @@ import base64
 import os
 from pathlib import Path
 
+import cv2
+from numpy import ndarray
+
 from frigate.const import CLIPS_DIR, THUMB_DIR
 from frigate.models import Event
 
@@ -21,6 +24,11 @@ def get_event_thumbnail_bytes(event: Event) -> bytes | None:
             return None
 
 
+def get_event_snapshot(event: Event) -> ndarray:
+    media_name = f"{event.camera}-{event.id}"
+    return cv2.imread(f"{os.path.join(CLIPS_DIR, media_name)}.jpg")
+
+
 ### Deletion
 
 

web/public/locales/en/views/faceLibrary.json

@@ -25,7 +25,7 @@
   },
   "readTheDocs": "Read the documentation to view more details on refining images for the Face Library",
   "trainFaceAs": "Train Face as:",
-  "trainFaceAsPerson": "Train Face as Person",
+  "trainFace": "Train Face",
   "toast": {
     "success": {
       "uploadedImage": "Successfully uploaded image.",

web/src/components/overlay/detail/SearchDetailDialog.tsx

@@ -57,6 +57,7 @@ import {
   DropdownMenu,
   DropdownMenuContent,
   DropdownMenuItem,
+  DropdownMenuLabel,
   DropdownMenuTrigger,
 } from "@/components/ui/dropdown-menu";
 import { TransformComponent, TransformWrapper } from "react-zoom-pan-pinch";
@@ -69,11 +70,12 @@ import {
   PopoverContent,
   PopoverTrigger,
 } from "@/components/ui/popover";
-import { LuInfo } from "react-icons/lu";
+import { LuInfo, LuSearch } from "react-icons/lu";
 import { TooltipPortal } from "@radix-ui/react-tooltip";
 import { FaPencilAlt } from "react-icons/fa";
 import TextEntryDialog from "@/components/overlay/dialog/TextEntryDialog";
 import { useTranslation } from "react-i18next";
+import { TbFaceId } from "react-icons/tb";
 
 const SEARCH_TABS = [
   "details",
@@ -99,7 +101,7 @@ export default function SearchDetailDialog({
   setSimilarity,
   setInputFocused,
 }: SearchDetailDialogProps) {
-  const { t } = useTranslation(["views/explore"]);
+  const { t } = useTranslation(["views/explore", "views/faceLibrary"]);
   const { data: config } = useSWR<FrigateConfig>("config", {
     revalidateOnFocus: false,
   });
@@ -555,6 +557,48 @@ function ObjectDetailsTab({
     [search, apiHost, mutate, setSearch, t],
   );
 
+  // face training
+
+  const hasFace = useMemo(() => {
+    if (!config?.face_recognition.enabled || !search) {
+      return false;
+    }
+
+    return search.data.attributes?.find((attr) => attr.label == "face");
+  }, [config, search]);
+
+  const { data: faceData } = useSWR(hasFace ? "faces" : null);
+
+  const faceNames = useMemo<string[]>(
+    () =>
+      faceData ? Object.keys(faceData).filter((face) => face != "train") : [],
+    [faceData],
+  );
+
+  const onTrainFace = useCallback(
+    (trainName: string) => {
+      axios
+        .post(`/faces/train/${trainName}/classify`, { event_id: search.id })
+        .then((resp) => {
+          if (resp.status == 200) {
+            toast.success(t("toast.success.trainedFace"), {
+              position: "top-center",
+            });
+          }
+        })
+        .catch((error) => {
+          const errorMessage =
+            error.response?.data?.message ||
+            error.response?.data?.detail ||
+            "Unknown error";
+          toast.error(t("toast.error.trainFailed", { errorMessage }), {
+            position: "top-center",
+          });
+        });
+    },
+    [search, t],
+  );
+
   return (
     <div className="flex flex-col gap-5">
       <div className="flex w-full flex-row">
@@ -673,20 +717,53 @@ function ObjectDetailsTab({
             draggable={false}
             src={`${apiHost}api/events/${search.id}/thumbnail.webp`}
           />
-          {config?.semantic_search.enabled && search.data.type == "object" && (
+          <div className="flex w-full flex-row gap-2">
-            <Button
+            {config?.semantic_search.enabled &&
-              aria-label={t("itemMenu.findSimilar.aria")}
+              search.data.type == "object" && (
-              onClick={() => {
+                <Button
-                setSearch(undefined);
+                  className="w-full"
+                  aria-label={t("itemMenu.findSimilar.aria")}
+                  onClick={() => {
+                    setSearch(undefined);
 
-                if (setSimilarity) {
+                    if (setSimilarity) {
-                  setSimilarity();
+                      setSimilarity();
-                }
+                    }
-              }}
+                  }}
-            >
+                >
-              {t("itemMenu.findSimilar.label")}
+                  <div className="flex gap-1">
-            </Button>
+                    <LuSearch />
-          )}
+                    {t("itemMenu.findSimilar.label")}
+                  </div>
+                </Button>
+              )}
+            {hasFace && (
+              <DropdownMenu>
+                <DropdownMenuTrigger asChild>
+                  <Button className="w-full">
+                    <div className="flex gap-1">
+                      <TbFaceId />
+                      {t("trainFace", { ns: "views/faceLibrary" })}
+                    </div>
+                  </Button>
+                </DropdownMenuTrigger>
+                <DropdownMenuContent>
+                  <DropdownMenuLabel>
+                    {t("trainFaceAs", { ns: "views/faceLibrary" })}
+                  </DropdownMenuLabel>
+                  {faceNames.map((faceName) => (
+                    <DropdownMenuItem
+                      key={faceName}
+                      className="cursor-pointer capitalize"
+                      onClick={() => onTrainFace(faceName)}
+                    >
+                      {faceName}
+                    </DropdownMenuItem>
+                  ))}
+                </DropdownMenuContent>
+              </DropdownMenu>
+            )}
+          </div>
         </div>
       </div>
       <div className="flex flex-col gap-1.5">

web/src/pages/FaceLibrary.tsx

@@ -472,7 +472,7 @@ function FaceAttempt({
                   ))}
                 </DropdownMenuContent>
               </DropdownMenu>
-              <TooltipContent>{t("trainFaceAsPerson")}</TooltipContent>
+              <TooltipContent>{t("trainFace")}</TooltipContent>
             </Tooltip>
             <Tooltip>
               <TooltipTrigger>

web/src/types/search.ts

@@ -50,6 +50,7 @@ export type SearchResult = {
     score: number;
     sub_label_score?: number;
     region: number[];
+    attributes?: [{ box: number[]; label: string; score: number }];
     box: number[];
     area: number;
     ratio: number;

web/src/views/settings/MotionTunerView.tsx

@@ -323,7 +323,7 @@ export default function MotionTunerView({
       </div>
 
       {cameraConfig ? (
-        <div className="flex md:h-dvh md:max-h-full md:w-7/12 md:grow">
+        <div className="flex max-h-[70%] md:h-dvh md:max-h-full md:w-7/12 md:grow">
           <div className="size-full min-h-10">
             <AutoUpdatingCameraImage
               camera={cameraConfig.name}

web/src/views/settings/ObjectSettingsView.tsx

@@ -296,7 +296,7 @@ export default function ObjectSettingsView({
       </div>
 
       {cameraConfig ? (
-        <div className="flex md:h-dvh md:max-h-full md:w-7/12 md:grow">
+        <div className="flex max-h-[70%] md:h-dvh md:max-h-full md:w-7/12 md:grow">
           <div ref={containerRef} className="relative size-full min-h-10">
             <AutoUpdatingCameraImage
               camera={cameraConfig.name}