Semantic Search Triggers (#18969)

* semantic trigger test

* database and model

* config

* embeddings maintainer and trigger post-processor

* api to create, edit, delete triggers

* frontend and i18n keys

* use thumbnail and description for trigger types

* image picker tweaks

* initial sync

* thumbnail file management

* clean up logs and use saved thumbnail on frontend

* publish mqtt messages

* webpush changes to enable trigger notifications

* add enabled switch

* add triggers from explore

* renaming and deletion fixes

* fix typing

* UI updates and add last triggering event time and link

* log exception instead of return in endpoint

* highlight entry in UI when triggered

* save and delete thumbnails directly

* remove alert action for now and add descriptions

* tweaks

* clean up

* fix types

* docs

* docs tweaks

* docs

* reuse enum

web/src/components/card/SearchThumbnailFooter.tsx

@@ -15,6 +15,7 @@ type SearchThumbnailProps = {
   refreshResults: () => void;
   showObjectLifecycle: () => void;
   showSnapshot: () => void;
+  addTrigger: () => void;
 };
 
 export default function SearchThumbnailFooter({
@@ -24,6 +25,7 @@ export default function SearchThumbnailFooter({
   refreshResults,
   showObjectLifecycle,
   showSnapshot,
+  addTrigger,
 }: SearchThumbnailProps) {
   const { t } = useTranslation(["views/search"]);
   const { data: config } = useSWR<FrigateConfig>("config");
@@ -61,6 +63,7 @@ export default function SearchThumbnailFooter({
           refreshResults={refreshResults}
           showObjectLifecycle={showObjectLifecycle}
           showSnapshot={showSnapshot}
+          addTrigger={addTrigger}
         />
       </div>
     </div>

web/src/components/menu/SearchResultActions.tsx

@@ -41,6 +41,7 @@ import {
 import useSWR from "swr";
 
 import { Trans, useTranslation } from "react-i18next";
+import { BsFillLightningFill } from "react-icons/bs";
 
 type SearchResultActionsProps = {
   searchResult: SearchResult;
@@ -48,6 +49,7 @@ type SearchResultActionsProps = {
   refreshResults: () => void;
   showObjectLifecycle: () => void;
   showSnapshot: () => void;
+  addTrigger: () => void;
   isContextMenu?: boolean;
   children?: ReactNode;
 };
@@ -58,6 +60,7 @@ export default function SearchResultActions({
   refreshResults,
   showObjectLifecycle,
   showSnapshot,
+  addTrigger,
   isContextMenu = false,
   children,
 }: SearchResultActionsProps) {
@@ -138,6 +141,16 @@ export default function SearchResultActions({
           <span>{t("itemMenu.findSimilar.label")}</span>
         </MenuItem>
       )}
+      {config?.semantic_search?.enabled &&
+        searchResult.data.type == "object" && (
+          <MenuItem
+            aria-label={t("itemMenu.addTrigger.aria")}
+            onClick={addTrigger}
+          >
+            <BsFillLightningFill className="mr-2 size-4" />
+            <span>{t("itemMenu.addTrigger.label")}</span>
+          </MenuItem>
+        )}
       {isMobileOnly &&
         config?.plus?.enabled &&
         searchResult.has_snapshot &&

web/src/components/overlay/CreateTriggerDialog.tsx

@@ -0,0 +1,416 @@
+import { useEffect, useMemo } from "react";
+import { useTranslation } from "react-i18next";
+import { useForm } from "react-hook-form";
+import { zodResolver } from "@hookform/resolvers/zod";
+import { z } from "zod";
+import useSWR from "swr";
+import {
+  Dialog,
+  DialogContent,
+  DialogDescription,
+  DialogFooter,
+  DialogHeader,
+  DialogTitle,
+} from "@/components/ui/dialog";
+import {
+  Form,
+  FormControl,
+  FormDescription,
+  FormField,
+  FormItem,
+  FormLabel,
+  FormMessage,
+} from "@/components/ui/form";
+import { Input } from "@/components/ui/input";
+import {
+  Select,
+  SelectContent,
+  SelectItem,
+  SelectTrigger,
+  SelectValue,
+} from "@/components/ui/select";
+import { Checkbox } from "@/components/ui/checkbox";
+import { Button } from "@/components/ui/button";
+import ActivityIndicator from "@/components/indicators/activity-indicator";
+import { FrigateConfig } from "@/types/frigateConfig";
+import ImagePicker from "@/components/overlay/ImagePicker";
+import { Trigger, TriggerAction, TriggerType } from "@/types/trigger";
+import { Switch } from "@/components/ui/switch";
+import { Textarea } from "../ui/textarea";
+
+type CreateTriggerDialogProps = {
+  show: boolean;
+  trigger: Trigger | null;
+  selectedCamera: string;
+  isLoading: boolean;
+  onCreate: (
+    enabled: boolean,
+    name: string,
+    type: TriggerType,
+    data: string,
+    threshold: number,
+    actions: TriggerAction[],
+  ) => void;
+  onEdit: (trigger: Trigger) => void;
+  onCancel: () => void;
+};
+
+export default function CreateTriggerDialog({
+  show,
+  trigger,
+  selectedCamera,
+  isLoading,
+  onCreate,
+  onEdit,
+  onCancel,
+}: CreateTriggerDialogProps) {
+  const { t } = useTranslation("views/settings");
+  const { data: config } = useSWR<FrigateConfig>("config");
+
+  const existingTriggerNames = useMemo(() => {
+    if (
+      !config ||
+      !selectedCamera ||
+      !config.cameras[selectedCamera]?.semantic_search?.triggers
+    ) {
+      return [];
+    }
+    return Object.keys(config.cameras[selectedCamera].semantic_search.triggers);
+  }, [config, selectedCamera]);
+
+  const formSchema = z.object({
+    enabled: z.boolean(),
+    name: z
+      .string()
+      .min(2, t("triggers.dialog.form.name.error.minLength"))
+      .regex(
+        /^[a-zA-Z0-9_-]+$/,
+        t("triggers.dialog.form.name.error.invalidCharacters"),
+      )
+      .refine(
+        (value) =>
+          !existingTriggerNames.includes(value) || value === trigger?.name,
+        t("triggers.dialog.form.name.error.alreadyExists"),
+      ),
+    type: z.enum(["thumbnail", "description"]),
+    data: z.string().min(1, t("triggers.dialog.form.content.error.required")),
+    threshold: z
+      .number()
+      .min(0, t("triggers.dialog.form.threshold.error.min"))
+      .max(1, t("triggers.dialog.form.threshold.error.max")),
+    actions: z.array(z.enum(["notification"])),
+  });
+
+  const form = useForm<z.infer<typeof formSchema>>({
+    resolver: zodResolver(formSchema),
+    mode: "onChange",
+    defaultValues: {
+      enabled: trigger?.enabled ?? true,
+      name: trigger?.name ?? "",
+      type: trigger?.type ?? "description",
+      data: trigger?.data ?? "",
+      threshold: trigger?.threshold ?? 0.5,
+      actions: trigger?.actions ?? [],
+    },
+  });
+
+  const onSubmit = async (values: z.infer<typeof formSchema>) => {
+    if (trigger) {
+      onEdit({ ...values });
+    } else {
+      onCreate(
+        values.enabled,
+        values.name,
+        values.type,
+        values.data,
+        values.threshold,
+        values.actions,
+      );
+    }
+  };
+
+  useEffect(() => {
+    if (!show) {
+      form.reset({
+        enabled: true,
+        name: "",
+        type: "description",
+        data: "",
+        threshold: 0.5,
+        actions: [],
+      });
+    } else if (trigger) {
+      form.reset(
+        {
+          enabled: trigger.enabled,
+          name: trigger.name,
+          type: trigger.type,
+          data: trigger.data,
+          threshold: trigger.threshold,
+          actions: trigger.actions,
+        },
+        { keepDirty: false, keepTouched: false }, // Reset validation state
+      );
+      // Trigger validation to ensure isValid updates
+      // form.trigger();
+    }
+  }, [show, trigger, form]);
+
+  const handleCancel = () => {
+    form.reset();
+    onCancel();
+  };
+
+  return (
+    <Dialog open={show} onOpenChange={onCancel}>
+      <DialogContent className="sm:max-w-[425px]">
+        <DialogHeader>
+          <DialogTitle>
+            {t(
+              trigger
+                ? "triggers.dialog.editTrigger.title"
+                : "triggers.dialog.createTrigger.title",
+            )}
+          </DialogTitle>
+          <DialogDescription>
+            {t(
+              trigger
+                ? "triggers.dialog.editTrigger.desc"
+                : "triggers.dialog.createTrigger.desc",
+              { camera: selectedCamera },
+            )}
+          </DialogDescription>
+        </DialogHeader>
+
+        <Form {...form}>
+          <form
+            onSubmit={form.handleSubmit(onSubmit)}
+            className="space-y-5 py-4"
+          >
+            <FormField
+              control={form.control}
+              name="name"
+              render={({ field }) => (
+                <FormItem>
+                  <FormLabel>{t("triggers.dialog.form.name.title")}</FormLabel>
+                  <FormControl>
+                    <Input
+                      placeholder={t("triggers.dialog.form.name.placeholder")}
+                      className="h-10"
+                      {...field}
+                    />
+                  </FormControl>
+                  <FormMessage />
+                </FormItem>
+              )}
+            />
+
+            <FormField
+              control={form.control}
+              name="enabled"
+              render={({ field }) => (
+                <FormItem className="flex flex-row items-center justify-between">
+                  <div className="space-y-0.5">
+                    <FormLabel className="text-base">
+                      {t("enabled", { ns: "common" })}
+                    </FormLabel>
+                    <div className="text-sm text-muted-foreground">
+                      {t("triggers.dialog.form.enabled.description")}
+                    </div>
+                  </div>
+                  <FormControl>
+                    <Switch
+                      checked={field.value}
+                      onCheckedChange={field.onChange}
+                    />
+                  </FormControl>
+                </FormItem>
+              )}
+            />
+
+            <FormField
+              control={form.control}
+              name="type"
+              render={({ field }) => (
+                <FormItem>
+                  <FormLabel>{t("triggers.dialog.form.type.title")}</FormLabel>
+                  <Select
+                    onValueChange={field.onChange}
+                    defaultValue={field.value}
+                  >
+                    <FormControl>
+                      <SelectTrigger className="h-10">
+                        <SelectValue
+                          placeholder={t(
+                            "triggers.dialog.form.type.placeholder",
+                          )}
+                        />
+                      </SelectTrigger>
+                    </FormControl>
+                    <SelectContent>
+                      <SelectItem value="thumbnail">
+                        {t("triggers.type.thumbnail")}
+                      </SelectItem>
+                      <SelectItem value="description">
+                        {t("triggers.type.description")}
+                      </SelectItem>
+                    </SelectContent>
+                  </Select>
+                  <FormMessage />
+                </FormItem>
+              )}
+            />
+
+            <FormField
+              control={form.control}
+              name="data"
+              render={({ field }) => (
+                <FormItem>
+                  <FormLabel>
+                    {t("triggers.dialog.form.content.title")}
+                  </FormLabel>
+                  {form.watch("type") === "thumbnail" ? (
+                    <>
+                      <FormControl>
+                        <ImagePicker
+                          selectedImageId={field.value}
+                          setSelectedImageId={field.onChange}
+                          camera={selectedCamera}
+                        />
+                      </FormControl>
+                      <FormDescription>
+                        {t("triggers.dialog.form.content.imageDesc")}
+                      </FormDescription>
+                    </>
+                  ) : (
+                    <>
+                      <FormControl>
+                        <Textarea
+                          placeholder={t(
+                            "triggers.dialog.form.content.textPlaceholder",
+                          )}
+                          {...field}
+                        />
+                      </FormControl>
+                      <FormDescription>
+                        {t("triggers.dialog.form.content.textDesc")}
+                      </FormDescription>
+                    </>
+                  )}
+
+                  <FormMessage />
+                </FormItem>
+              )}
+            />
+
+            <FormField
+              control={form.control}
+              name="threshold"
+              render={({ field }) => (
+                <FormItem>
+                  <FormLabel>
+                    {t("triggers.dialog.form.threshold.title")}
+                  </FormLabel>
+                  <FormControl>
+                    <Input
+                      type="number"
+                      step="0.01"
+                      min="0"
+                      max="1"
+                      placeholder="0.50"
+                      className="h-10"
+                      {...field}
+                      onChange={(e) => {
+                        const value = parseFloat(e.target.value);
+                        field.onChange(isNaN(value) ? 0 : value);
+                      }}
+                    />
+                  </FormControl>
+                  <FormMessage />
+                </FormItem>
+              )}
+            />
+
+            <FormField
+              control={form.control}
+              name="actions"
+              render={() => (
+                <FormItem>
+                  <FormLabel>
+                    {t("triggers.dialog.form.actions.title")}
+                  </FormLabel>
+                  <div className="space-y-2">
+                    {["notification"].map((action) => (
+                      <div key={action} className="flex items-center space-x-2">
+                        <FormControl>
+                          <Checkbox
+                            checked={form
+                              .watch("actions")
+                              .includes(action as TriggerAction)}
+                            onCheckedChange={(checked) => {
+                              const currentActions = form.getValues("actions");
+                              if (checked) {
+                                form.setValue("actions", [
+                                  ...currentActions,
+                                  action as TriggerAction,
+                                ]);
+                              } else {
+                                form.setValue(
+                                  "actions",
+                                  currentActions.filter((a) => a !== action),
+                                );
+                              }
+                            }}
+                          />
+                        </FormControl>
+                        <FormLabel className="text-sm font-normal">
+                          {t(`triggers.actions.${action}`)}
+                        </FormLabel>
+                      </div>
+                    ))}
+                  </div>
+                  <FormDescription>
+                    {t("triggers.dialog.form.actions.desc")}
+                  </FormDescription>
+                  <FormMessage />
+                </FormItem>
+              )}
+            />
+
+            <DialogFooter className="flex gap-2 pt-2 sm:justify-end">
+              <div className="flex flex-1 flex-col justify-end">
+                <div className="flex flex-row gap-2 pt-5">
+                  <Button
+                    className="flex flex-1"
+                    aria-label={t("button.cancel", { ns: "common" })}
+                    disabled={isLoading}
+                    onClick={handleCancel}
+                    type="button"
+                  >
+                    {t("button.cancel", { ns: "common" })}
+                  </Button>
+                  <Button
+                    variant="select"
+                    aria-label={t("button.save", { ns: "common" })}
+                    disabled={isLoading || !form.formState.isValid}
+                    className="flex flex-1"
+                    type="submit"
+                  >
+                    {isLoading ? (
+                      <div className="flex flex-row items-center gap-2">
+                        <ActivityIndicator />
+                        <span>{t("button.saving", { ns: "common" })}</span>
+                      </div>
+                    ) : (
+                      t("button.save", { ns: "common" })
+                    )}
+                  </Button>
+                </div>
+              </div>
+            </DialogFooter>
+          </form>
+        </Form>
+      </DialogContent>
+    </Dialog>
+  );
+}

web/src/components/overlay/DeleteTriggerDialog.tsx

@@ -0,0 +1,80 @@
+import { useTranslation } from "react-i18next";
+import {
+  Dialog,
+  DialogContent,
+  DialogDescription,
+  DialogFooter,
+  DialogHeader,
+  DialogTitle,
+} from "@/components/ui/dialog";
+import { Button } from "@/components/ui/button";
+import { Trans } from "react-i18next";
+import ActivityIndicator from "@/components/indicators/activity-indicator";
+
+type DeleteTriggerDialogProps = {
+  show: boolean;
+  triggerName: string;
+  isLoading: boolean;
+  onCancel: () => void;
+  onDelete: () => void;
+};
+
+export default function DeleteTriggerDialog({
+  show,
+  triggerName,
+  isLoading,
+  onCancel,
+  onDelete,
+}: DeleteTriggerDialogProps) {
+  const { t } = useTranslation("views/settings");
+
+  return (
+    <Dialog open={show} onOpenChange={onCancel}>
+      <DialogContent className="sm:max-w-[425px]">
+        <DialogHeader>
+          <DialogTitle>{t("triggers.dialog.deleteTrigger.title")}</DialogTitle>
+          <DialogDescription>
+            <Trans
+              ns={"views/settings"}
+              values={{ triggerName }}
+              components={{ strong: <span className="font-medium" /> }}
+            >
+              triggers.dialog.deleteTrigger.desc
+            </Trans>
+          </DialogDescription>
+        </DialogHeader>
+        <DialogFooter className="flex gap-3 sm:justify-end">
+          <div className="flex flex-1 flex-col justify-end">
+            <div className="flex flex-row gap-2 pt-5">
+              <Button
+                className="flex flex-1"
+                aria-label={t("button.cancel", { ns: "common" })}
+                onClick={onCancel}
+                type="button"
+                disabled={isLoading}
+              >
+                {t("button.cancel", { ns: "common" })}
+              </Button>
+              <Button
+                variant="destructive"
+                aria-label={t("button.delete", { ns: "common" })}
+                className="flex flex-1 text-white"
+                onClick={onDelete}
+                disabled={isLoading}
+              >
+                {isLoading ? (
+                  <div className="flex flex-row items-center gap-2">
+                    <ActivityIndicator />
+                    <span>{t("button.delete", { ns: "common" })}</span>
+                  </div>
+                ) : (
+                  t("button.delete", { ns: "common" })
+                )}
+              </Button>
+            </div>
+          </div>
+        </DialogFooter>
+      </DialogContent>
+    </Dialog>
+  );
+}

web/src/components/overlay/DeleteUserDialog.tsx

@@ -60,7 +60,7 @@ export default function DeleteUserDialog({
               <Button
                 variant="destructive"
                 aria-label={t("button.delete", { ns: "common" })}
-                className="flex flex-1"
+                className="flex flex-1 text-white"
                 onClick={onDelete}
               >
                 {t("button.delete", { ns: "common" })}

web/src/components/overlay/ImagePicker.tsx

@@ -0,0 +1,172 @@
+import { useCallback, useMemo, useRef, useState } from "react";
+import { useTranslation } from "react-i18next";
+import useSWR from "swr";
+import {
+  Dialog,
+  DialogContent,
+  DialogTitle,
+  DialogTrigger,
+} from "@/components/ui/dialog";
+import { IoClose } from "react-icons/io5";
+import { Input } from "@/components/ui/input";
+import { Button } from "@/components/ui/button";
+import Heading from "@/components/ui/heading";
+import { cn } from "@/lib/utils";
+import { Event } from "@/types/event";
+import { useApiHost } from "@/api";
+import { isDesktop, isMobile } from "react-device-detect";
+
+type ImagePickerProps = {
+  selectedImageId?: string;
+  setSelectedImageId?: (id: string) => void;
+  camera: string;
+};
+
+export default function ImagePicker({
+  selectedImageId,
+  setSelectedImageId,
+  camera,
+}: ImagePickerProps) {
+  const { t } = useTranslation(["components/dialog"]);
+  const [open, setOpen] = useState(false);
+  const containerRef = useRef<HTMLDivElement>(null);
+  const [searchTerm, setSearchTerm] = useState("");
+
+  const { data: events } = useSWR<Event[]>(
+    `events?camera=${camera}&limit=100`,
+    {
+      revalidateOnFocus: false,
+    },
+  );
+  const apiHost = useApiHost();
+
+  const images = useMemo(() => {
+    if (!events) return [];
+    return events.filter(
+      (event) =>
+        (event.label.toLowerCase().includes(searchTerm.toLowerCase()) ||
+          (event.sub_label &&
+            event.sub_label.toLowerCase().includes(searchTerm.toLowerCase())) ||
+          searchTerm === "") &&
+        event.camera === camera,
+    );
+  }, [events, searchTerm, camera]);
+
+  const selectedImage = useMemo(
+    () => images.find((img) => img.id === selectedImageId),
+    [images, selectedImageId],
+  );
+
+  const handleImageSelect = useCallback(
+    (id: string) => {
+      if (setSelectedImageId) {
+        setSelectedImageId(id);
+      }
+      setSearchTerm("");
+      setOpen(false);
+    },
+    [setSelectedImageId],
+  );
+
+  return (
+    <div ref={containerRef}>
+      <Dialog
+        open={open}
+        onOpenChange={(open) => {
+          setOpen(open);
+        }}
+      >
+        <DialogTrigger asChild>
+          {!selectedImageId ? (
+            <Button
+              className="mt-2 w-full text-muted-foreground"
+              aria-label={t("imagePicker.selectImage")}
+            >
+              {t("imagePicker.selectImage")}
+            </Button>
+          ) : (
+            <div className="hover:cursor-pointer">
+              <div className="my-3 flex w-full flex-row items-center justify-between gap-2">
+                <div className="flex flex-row items-center gap-2">
+                  <img
+                    src={
+                      selectedImage
+                        ? `${apiHost}api/events/${selectedImage.id}/thumbnail.webp`
+                        : `${apiHost}clips/triggers/${camera}/${selectedImageId}.webp`
+                    }
+                    alt={selectedImage?.label || "Selected image"}
+                    className="h-8 w-8 rounded object-cover"
+                  />
+                  <div className="text-sm smart-capitalize">
+                    {selectedImage?.label || selectedImageId}
+                    {selectedImage?.sub_label
+                      ? ` (${selectedImage.sub_label})`
+                      : ""}
+                  </div>
+                </div>
+                <IoClose
+                  className="mx-2 hover:cursor-pointer"
+                  onClick={() => {
+                    if (setSelectedImageId) {
+                      setSelectedImageId("");
+                    }
+                  }}
+                />
+              </div>
+            </div>
+          )}
+        </DialogTrigger>
+        <DialogTitle className="sr-only">
+          {t("imagePicker.selectImage")}
+        </DialogTitle>
+        <DialogContent
+          className={cn(
+            "scrollbar-container overflow-y-auto",
+            isDesktop && "max-h-[75dvh] sm:max-w-xl md:max-w-3xl",
+            isMobile && "px-4",
+          )}
+        >
+          <div className="mb-3 flex flex-row items-center justify-between">
+            <Heading as="h4">{t("imagePicker.selectImage")}</Heading>
+            <span tabIndex={0} className="sr-only" />
+          </div>
+          <Input
+            type="text"
+            placeholder={t("imagePicker.search.placeholder")}
+            className="text-md mb-3 md:text-sm"
+            value={searchTerm}
+            onChange={(e) => setSearchTerm(e.target.value)}
+          />
+          <div className="scrollbar-container flex h-full flex-col overflow-y-auto">
+            <div className="grid grid-cols-3 gap-2 pr-1">
+              {images.length === 0 ? (
+                <div className="col-span-3 text-center text-sm text-muted-foreground">
+                  {t("imagePicker.noImages")}
+                </div>
+              ) : (
+                images.map((image) => (
+                  <div
+                    key={image.id}
+                    className={cn(
+                      "flex flex-row items-center justify-center rounded-lg p-1 hover:cursor-pointer",
+                      selectedImageId === image.id
+                        ? "bg-selected text-white"
+                        : "hover:bg-secondary-foreground",
+                    )}
+                  >
+                    <img
+                      src={`${apiHost}api/events/${image.id}/thumbnail.webp`}
+                      alt={image.label}
+                      className="rounded object-cover"
+                      onClick={() => handleImageSelect(image.id)}
+                    />
+                  </div>
+                ))
+              )}
+            </div>
+          </div>
+        </DialogContent>
+      </Dialog>
+    </div>
+  );
+}