From ff8e145b906d742d43c66c2f18d98704968e4698 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Mon, 17 Mar 2025 13:50:13 -0600 Subject: [PATCH] Face setup wizard (#17203) * Fix login page * Increase face image size and add time ago * Add component for indicating steps in a wizard * Split out form inputs from dialog * Add wizard for adding new face to library * Simplify dialog * Translations * Fix scaling bug * Fix key missing * Improve multi select * Adjust wording and spacing * Add tip for face training * Fix padding * Remove text for buttons on mobile --- frigate/data_processing/real_time/face.py | 10 +- web/public/locales/en/common.json | 4 +- web/public/locales/en/views/faceLibrary.json | 9 +- .../components/indicators/StepIndicator.tsx | 28 +++ web/src/components/input/ImageEntry.tsx | 58 ++++++ web/src/components/input/TextEntry.tsx | 68 +++++++ .../overlay/detail/FaceCreateWizardDialog.tsx | 168 ++++++++++++++++++ .../overlay/dialog/TextEntryDialog.tsx | 79 ++------ .../overlay/dialog/UploadImageDialog.tsx | 63 ++----- web/src/pages/FaceLibrary.tsx | 160 +++++++---------- web/src/pages/LoginPage.tsx | 18 +- 11 files changed, 442 insertions(+), 223 deletions(-) create mode 100644 web/src/components/indicators/StepIndicator.tsx create mode 100644 web/src/components/input/ImageEntry.tsx create mode 100644 web/src/components/input/TextEntry.tsx create mode 100644 web/src/components/overlay/detail/FaceCreateWizardDialog.tsx diff --git a/frigate/data_processing/real_time/face.py b/frigate/data_processing/real_time/face.py index e70801812..b51b7a20f 100644 --- a/frigate/data_processing/real_time/face.py +++ b/frigate/data_processing/real_time/face.py @@ -227,6 +227,8 @@ class FaceRealTimeProcessor(RealTimeProcessorApi): scale_factor = MAX_DETECTION_HEIGHT / input.shape[0] new_width = int(scale_factor * input.shape[1]) input = cv2.resize(input, (new_width, MAX_DETECTION_HEIGHT)) + else: + scale_factor = 1 self.face_detector.setInputSize((input.shape[1], input.shape[0])) faces = self.face_detector.detect(input) @@ -241,10 +243,10 @@ class FaceRealTimeProcessor(RealTimeProcessorApi): continue raw_bbox = potential_face[0:4].astype(np.uint16) - x: int = max(raw_bbox[0], 0) - y: int = max(raw_bbox[1], 0) - w: int = raw_bbox[2] - h: int = raw_bbox[3] + x: int = int(max(raw_bbox[0], 0) / scale_factor) + y: int = int(max(raw_bbox[1], 0) / scale_factor) + w: int = int(raw_bbox[2] / scale_factor) + h: int = int(raw_bbox[3] / scale_factor) bbox = (x, y, x + w, y + h) if face is None or area(bbox) > area(face): diff --git a/web/public/locales/en/common.json b/web/public/locales/en/common.json index 4ddd9244e..14b88f707 100644 --- a/web/public/locales/en/common.json +++ b/web/public/locales/en/common.json @@ -64,6 +64,7 @@ "button": { "apply": "Apply", "reset": "Reset", + "done": "Done", "enabled": "Enabled", "enable": "Enable", "disabled": "Disabled", @@ -94,7 +95,8 @@ "play": "Play", "unselect": "Unselect", "export": "Export", - "deleteNow": "Delete Now" + "deleteNow": "Delete Now", + "next": "Next" }, "menu": { "system": "System", diff --git a/web/public/locales/en/views/faceLibrary.json b/web/public/locales/en/views/faceLibrary.json index b95f744d7..4028690e3 100644 --- a/web/public/locales/en/views/faceLibrary.json +++ b/web/public/locales/en/views/faceLibrary.json @@ -1,4 +1,7 @@ { + "description": { + "addFace": "Walk through adding a new face to the Face Library." + }, "documentTitle": "Face Library - Frigate", "uploadFaceImage": { "title": "Upload Face Image", @@ -6,7 +9,8 @@ }, "createFaceLibrary": { "title": "Create Face Library", - "desc": "Create a new face library" + "desc": "Create a new face library", + "nextSteps": "It is recommended to use the Train tab to select and train images for each person as they are detected. When building a strong foundation it is strongly recommended to only train on images that are straight-on. Ignore images from cameras that recognize faces from an angle." }, "train": { "title": "Train", @@ -19,12 +23,13 @@ "uploadImage": "Upload Image", "reprocessFace": "Reprocess Face" }, + "readTheDocs": "Read the documentation to view more details on refining images for the Face Library", "trainFaceAs": "Train Face as:", "trainFaceAsPerson": "Train Face as Person", "toast": { "success": { "uploadedImage": "Successfully uploaded image.", - "addFaceLibrary": "Successfully add face library.", + "addFaceLibrary": "{{name}} has successfully been added to the Face Library!", "deletedFace": "Successfully deleted face.", "trainedFace": "Successfully trained face.", "updatedFaceScore": "Successfully updated face score." diff --git a/web/src/components/indicators/StepIndicator.tsx b/web/src/components/indicators/StepIndicator.tsx new file mode 100644 index 000000000..641ae32ca --- /dev/null +++ b/web/src/components/indicators/StepIndicator.tsx @@ -0,0 +1,28 @@ +import { cn } from "@/lib/utils"; + +type StepIndicatorProps = { + steps: string[]; + currentStep: number; +}; +export default function StepIndicator({ + steps, + currentStep, +}: StepIndicatorProps) { + return ( +
+ {steps.map((name, idx) => ( +
+
+ {idx + 1} +
+
{name}
+
+ ))} +
+ ); +} diff --git a/web/src/components/input/ImageEntry.tsx b/web/src/components/input/ImageEntry.tsx new file mode 100644 index 000000000..afb399177 --- /dev/null +++ b/web/src/components/input/ImageEntry.tsx @@ -0,0 +1,58 @@ +import { Form, FormControl, FormField, FormItem } from "@/components/ui/form"; +import { Input } from "@/components/ui/input"; +import { zodResolver } from "@hookform/resolvers/zod"; +import React, { useCallback } from "react"; +import { useForm } from "react-hook-form"; + +import { z } from "zod"; + +type ImageEntryProps = { + onSave: (file: File) => void; + children?: React.ReactNode; +}; +export default function ImageEntry({ onSave, children }: ImageEntryProps) { + const formSchema = z.object({ + file: z.instanceof(FileList, { message: "Please select an image file." }), + }); + + const form = useForm>({ + resolver: zodResolver(formSchema), + }); + const fileRef = form.register("file"); + + // upload handler + + const onSubmit = useCallback( + (data: z.infer) => { + if (!data["file"] || Object.keys(data.file).length == 0) { + return; + } + + onSave(data["file"]["0"]); + }, + [onSave], + ); + + return ( +
+ + ( + + + + + + )} + /> + {children} + + + ); +} diff --git a/web/src/components/input/TextEntry.tsx b/web/src/components/input/TextEntry.tsx new file mode 100644 index 000000000..c9fa8a8a9 --- /dev/null +++ b/web/src/components/input/TextEntry.tsx @@ -0,0 +1,68 @@ +import { Form, FormControl, FormField, FormItem } from "@/components/ui/form"; +import { Input } from "@/components/ui/input"; +import { zodResolver } from "@hookform/resolvers/zod"; +import React, { useCallback } from "react"; +import { useForm } from "react-hook-form"; + +import { z } from "zod"; + +type TextEntryProps = { + defaultValue?: string; + placeholder?: string; + allowEmpty?: boolean; + onSave: (text: string) => void; + children?: React.ReactNode; +}; +export default function TextEntry({ + defaultValue, + placeholder, + allowEmpty, + onSave, + children, +}: TextEntryProps) { + const formSchema = z.object({ + text: z.string(), + }); + + const form = useForm>({ + resolver: zodResolver(formSchema), + defaultValues: { text: defaultValue }, + }); + const fileRef = form.register("text"); + + // upload handler + + const onSubmit = useCallback( + (data: z.infer) => { + if (!allowEmpty && !data["text"]) { + return; + } + onSave(data["text"]); + }, + [onSave, allowEmpty], + ); + + return ( +
+ + ( + + + + + + )} + /> + {children} + + + ); +} diff --git a/web/src/components/overlay/detail/FaceCreateWizardDialog.tsx b/web/src/components/overlay/detail/FaceCreateWizardDialog.tsx new file mode 100644 index 000000000..659ac4c88 --- /dev/null +++ b/web/src/components/overlay/detail/FaceCreateWizardDialog.tsx @@ -0,0 +1,168 @@ +import StepIndicator from "@/components/indicators/StepIndicator"; +import ImageEntry from "@/components/input/ImageEntry"; +import TextEntry from "@/components/input/TextEntry"; +import { + MobilePage, + MobilePageContent, + MobilePageDescription, + MobilePageHeader, + MobilePageTitle, +} from "@/components/mobile/MobilePage"; +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { cn } from "@/lib/utils"; +import axios from "axios"; +import { useCallback, useState } from "react"; +import { isDesktop } from "react-device-detect"; +import { useTranslation } from "react-i18next"; +import { LuExternalLink } from "react-icons/lu"; +import { Link } from "react-router-dom"; +import { toast } from "sonner"; + +const STEPS = ["Enter Face Name", "Upload Face Image", "Next Steps"]; + +type CreateFaceWizardDialogProps = { + open: boolean; + setOpen: (open: boolean) => void; + onFinish: () => void; +}; +export default function CreateFaceWizardDialog({ + open, + setOpen, + onFinish, +}: CreateFaceWizardDialogProps) { + const { t } = useTranslation("views/faceLibrary"); + + // wizard + + const [step, setStep] = useState(0); + const [name, setName] = useState(""); + + const handleReset = useCallback(() => { + setStep(0); + setName(""); + setOpen(false); + }, [setOpen]); + + // data handling + + const onUploadImage = useCallback( + (file: File) => { + const formData = new FormData(); + formData.append("file", file); + axios + .post(`faces/${name}/register`, formData, { + headers: { + "Content-Type": "multipart/form-data", + }, + }) + .then((resp) => { + if (resp.status == 200) { + setStep(2); + toast.success(t("toast.success.uploadedImage"), { + position: "top-center", + }); + } + }) + .catch((error) => { + const errorMessage = + error.response?.data?.message || + error.response?.data?.detail || + "Unknown error"; + toast.error(t("toast.error.uploadingImageFailed", { errorMessage }), { + position: "top-center", + }); + }); + }, + [name, t], + ); + + // layout + + const Overlay = isDesktop ? Dialog : MobilePage; + const Content = isDesktop ? DialogContent : MobilePageContent; + const Header = isDesktop ? DialogHeader : MobilePageHeader; + const Title = isDesktop ? DialogTitle : MobilePageTitle; + const Description = isDesktop ? DialogDescription : MobilePageDescription; + + return ( + { + if (!open) { + handleReset(); + } + }} + > + +
+ {t("button.addFace")} + {isDesktop && {t("description.addFace")}} +
+ + {step == 0 && ( + { + setName(name); + setStep(1); + }} + > +
+ +
+ + )} + {step == 1 && ( + +
+ +
+ + )} + {step == 2 && ( +
+ {t("toast.success.addFaceLibrary", { name })} +

+ {t("createFaceLibrary.nextSteps")} +

+
+ + {t("readTheDocs")} + + +
+
+ +
+
+ )} + + + ); +} diff --git a/web/src/components/overlay/dialog/TextEntryDialog.tsx b/web/src/components/overlay/dialog/TextEntryDialog.tsx index a25c023ea..6fc1f9ad3 100644 --- a/web/src/components/overlay/dialog/TextEntryDialog.tsx +++ b/web/src/components/overlay/dialog/TextEntryDialog.tsx @@ -1,3 +1,4 @@ +import TextEntry from "@/components/input/TextEntry"; import { Button } from "@/components/ui/button"; import { Dialog, @@ -7,15 +8,8 @@ import { DialogHeader, DialogTitle, } from "@/components/ui/dialog"; -import { Form, FormControl, FormField, FormItem } from "@/components/ui/form"; -import { Input } from "@/components/ui/input"; -import { zodResolver } from "@hookform/resolvers/zod"; -import { useCallback, useEffect } from "react"; -import { useForm } from "react-hook-form"; import { useTranslation } from "react-i18next"; -import { z } from "zod"; - type TextEntryDialogProps = { open: boolean; title: string; @@ -35,35 +29,7 @@ export default function TextEntryDialog({ defaultValue = "", allowEmpty = false, }: TextEntryDialogProps) { - const formSchema = z.object({ - text: z.string(), - }); - - const { t } = useTranslation("components/dialog"); - - const form = useForm>({ - resolver: zodResolver(formSchema), - defaultValues: { text: defaultValue }, - }); - const fileRef = form.register("text"); - - // upload handler - - const onSubmit = useCallback( - (data: z.infer) => { - if (!allowEmpty && !data["text"]) { - return; - } - onSave(data["text"]); - }, - [onSave, allowEmpty], - ); - - useEffect(() => { - if (open) { - form.reset({ text: defaultValue }); - } - }, [open, defaultValue, form]); + const { t } = useTranslation("common"); return ( @@ -72,33 +38,20 @@ export default function TextEntryDialog({ {title} {description && {description}} -
- - ( - - - - - - )} - /> - - - - - - + + + + + +
); diff --git a/web/src/components/overlay/dialog/UploadImageDialog.tsx b/web/src/components/overlay/dialog/UploadImageDialog.tsx index 6a01a7fab..7fab82eea 100644 --- a/web/src/components/overlay/dialog/UploadImageDialog.tsx +++ b/web/src/components/overlay/dialog/UploadImageDialog.tsx @@ -1,3 +1,4 @@ +import ImageEntry from "@/components/input/ImageEntry"; import { Button } from "@/components/ui/button"; import { Dialog, @@ -7,12 +8,7 @@ import { DialogHeader, DialogTitle, } from "@/components/ui/dialog"; -import { Form, FormControl, FormField, FormItem } from "@/components/ui/form"; -import { Input } from "@/components/ui/input"; -import { zodResolver } from "@hookform/resolvers/zod"; -import { useCallback } from "react"; -import { useForm } from "react-hook-form"; -import { z } from "zod"; +import { useTranslation } from "react-i18next"; type UploadImageDialogProps = { open: boolean; @@ -28,27 +24,7 @@ export default function UploadImageDialog({ setOpen, onSave, }: UploadImageDialogProps) { - const formSchema = z.object({ - file: z.instanceof(FileList, { message: "Please select an image file." }), - }); - - const form = useForm>({ - resolver: zodResolver(formSchema), - }); - const fileRef = form.register("file"); - - // upload handler - - const onSubmit = useCallback( - (data: z.infer) => { - if (!data["file"] || Object.keys(data.file).length == 0) { - return; - } - - onSave(data["file"]["0"]); - }, - [onSave], - ); + const { t } = useTranslation("common"); return ( @@ -57,31 +33,14 @@ export default function UploadImageDialog({ {title} {description && {description}} -
- - ( - - - - - - )} - /> - - - - - - + + + + + +
); diff --git a/web/src/pages/FaceLibrary.tsx b/web/src/pages/FaceLibrary.tsx index 33fbb69d1..afa196f35 100644 --- a/web/src/pages/FaceLibrary.tsx +++ b/web/src/pages/FaceLibrary.tsx @@ -1,7 +1,8 @@ import { baseUrl } from "@/api/baseUrl"; +import TimeAgo from "@/components/dynamic/TimeAgo"; import AddFaceIcon from "@/components/icons/AddFaceIcon"; import ActivityIndicator from "@/components/indicators/activity-indicator"; -import TextEntryDialog from "@/components/overlay/dialog/TextEntryDialog"; +import CreateFaceWizardDialog from "@/components/overlay/detail/FaceCreateWizardDialog"; import UploadImageDialog from "@/components/overlay/dialog/UploadImageDialog"; import { Button } from "@/components/ui/button"; import { @@ -25,6 +26,7 @@ import { cn } from "@/lib/utils"; import { FrigateConfig } from "@/types/frigateConfig"; import axios from "axios"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { isDesktop } from "react-device-detect"; import { useTranslation } from "react-i18next"; import { LuImagePlus, LuRefreshCw, LuScanFace, LuTrash2 } from "react-icons/lu"; import { toast } from "sonner"; @@ -115,42 +117,16 @@ export default function FaceLibrary() { [pageToggle, refreshFaces, t], ); - const onAddName = useCallback( - (name: string) => { - axios - .post(`faces/${name}/create`, { - headers: { - "Content-Type": "multipart/form-data", - }, - }) - .then((resp) => { - if (resp.status == 200) { - setAddFace(false); - refreshFaces(); - toast.success(t("toast.success.addFaceLibrary"), { - position: "top-center", - }); - } - }) - .catch((error) => { - const errorMessage = - error.response?.data?.message || - error.response?.data?.detail || - "Unknown error"; - toast.error(t("toast.error.addFaceLibraryFailed", { errorMessage }), { - position: "top-center", - }); - }); - }, - [refreshFaces, t], - ); - // face multiselect const [selectedFaces, setSelectedFaces] = useState([]); const onClickFace = useCallback( - (imageId: string) => { + (imageId: string, ctrl: boolean) => { + if (selectedFaces.length == 0 && !ctrl) { + return; + } + const index = selectedFaces.indexOf(imageId); if (index != -1) { @@ -172,33 +148,42 @@ export default function FaceLibrary() { [selectedFaces, setSelectedFaces], ); - const onDelete = useCallback(() => { - axios - .post(`/faces/train/delete`, { ids: selectedFaces }) - .then((resp) => { - setSelectedFaces([]); + const onDelete = useCallback( + (name: string, ids: string[]) => { + axios + .post(`/faces/${name}/delete`, { ids }) + .then((resp) => { + setSelectedFaces([]); - if (resp.status == 200) { - toast.success(t("toast.success.deletedFace"), { + if (resp.status == 200) { + toast.success(t("toast.success.deletedFace"), { + position: "top-center", + }); + + if (faceImages.length == 1) { + // face has been deleted + setPageToggle(""); + } + + refreshFaces(); + } + }) + .catch((error) => { + const errorMessage = + error.response?.data?.message || + error.response?.data?.detail || + "Unknown error"; + toast.error(t("toast.error.deleteFaceFailed", { errorMessage }), { position: "top-center", }); - refreshFaces(); - } - }) - .catch((error) => { - const errorMessage = - error.response?.data?.message || - error.response?.data?.detail || - "Unknown error"; - toast.error(t("toast.error.deleteFaceFailed", { errorMessage }), { - position: "top-center", }); - }); - }, [selectedFaces, refreshFaces, t]); + }, + [faceImages, refreshFaces, setPageToggle, t], + ); // keyboard - useKeyboardListener(["a"], (key, modifiers) => { + useKeyboardListener(["a", "Escape"], (key, modifiers) => { if (modifiers.repeat || !modifiers.down) { return; } @@ -209,6 +194,9 @@ export default function FaceLibrary() { setSelectedFaces([...trainImages]); } break; + case "Escape": + setSelectedFaces([]); + break; } }); @@ -228,12 +216,10 @@ export default function FaceLibrary() { onSave={onUploadImage} /> -
@@ -283,21 +269,24 @@ export default function FaceLibrary() { {selectedFaces?.length > 0 ? (
-
) : (
{pageToggle != "train" && ( )}
@@ -317,7 +306,7 @@ export default function FaceLibrary() { ))}
@@ -329,7 +318,7 @@ type TrainingGridProps = { attemptImages: string[]; faceNames: string[]; selectedFaces: string[]; - onClickFace: (image: string) => void; + onClickFace: (image: string, ctrl: boolean) => void; onRefresh: () => void; }; function TrainingGrid({ @@ -349,7 +338,7 @@ function TrainingGrid({ faceNames={faceNames} threshold={config.face_recognition.recognition_threshold} selected={selectedFaces.includes(image)} - onClick={() => onClickFace(image)} + onClick={(meta) => onClickFace(image, meta)} onRefresh={onRefresh} /> ))} @@ -362,7 +351,7 @@ type FaceAttemptProps = { faceNames: string[]; threshold: number; selected: boolean; - onClick: () => void; + onClick: (meta: boolean) => void; onRefresh: () => void; }; function FaceAttempt({ @@ -378,6 +367,7 @@ function FaceAttempt({ const parts = image.split("-"); return { + timestamp: Number.parseFloat(parts[0]), eventId: `${parts[0]}-${parts[1]}`, name: parts[2], score: parts[3], @@ -439,10 +429,13 @@ function FaceAttempt({ ? "shadow-selected outline-selected" : "outline-transparent duration-500", )} - onClick={onClick} + onClick={(e) => onClick(e.metaKey || e.ctrlKey)} > -
- +
+ +
+ +
@@ -500,9 +493,9 @@ function FaceAttempt({ type FaceGridProps = { faceImages: string[]; pageToggle: string; - onRefresh: () => void; + onDelete: (name: string, ids: string[]) => void; }; -function FaceGrid({ faceImages, pageToggle, onRefresh }: FaceGridProps) { +function FaceGrid({ faceImages, pageToggle, onDelete }: FaceGridProps) { return (
{faceImages.map((image: string) => ( @@ -510,7 +503,7 @@ function FaceGrid({ faceImages, pageToggle, onRefresh }: FaceGridProps) { key={image} name={pageToggle} image={image} - onRefresh={onRefresh} + onDelete={onDelete} /> ))}
@@ -520,31 +513,10 @@ function FaceGrid({ faceImages, pageToggle, onRefresh }: FaceGridProps) { type FaceImageProps = { name: string; image: string; - onRefresh: () => void; + onDelete: (name: string, ids: string[]) => void; }; -function FaceImage({ name, image, onRefresh }: FaceImageProps) { +function FaceImage({ name, image, onDelete }: FaceImageProps) { const { t } = useTranslation(["views/faceLibrary"]); - const onDelete = useCallback(() => { - axios - .post(`/faces/${name}/delete`, { ids: [image] }) - .then((resp) => { - if (resp.status == 200) { - toast.success(t("toast.success.deletedFace"), { - position: "top-center", - }); - onRefresh(); - } - }) - .catch((error) => { - const errorMessage = - error.response?.data?.message || - error.response?.data?.detail || - "Unknown error"; - toast.error(t("toast.error.deleteFaceFailed", { errorMessage }), { - position: "top-center", - }); - }); - }, [name, image, onRefresh, t]); return (
@@ -561,7 +533,7 @@ function FaceImage({ name, image, onRefresh }: FaceImageProps) { onDelete(name, [image])} /> {t("button.deleteFaceAttempts")} diff --git a/web/src/pages/LoginPage.tsx b/web/src/pages/LoginPage.tsx index d79a7c953..8cf87f206 100644 --- a/web/src/pages/LoginPage.tsx +++ b/web/src/pages/LoginPage.tsx @@ -1,20 +1,24 @@ import { UserAuthForm } from "@/components/auth/AuthForm"; import Logo from "@/components/Logo"; import { ThemeProvider } from "@/context/theme-provider"; +import "@/utils/i18n"; +import { LanguageProvider } from "@/context/language-provider"; function LoginPage() { return ( -
-
-
-
- + +
+
+
+
+ +
+
-
-
+ ); }