Fix similarity search (#13856)

* add event_id param to api

* exclude query from filtertype

* update review pane link for similarity search

* update filter group for similarity param and fix switch bug

* unneeded prop

* update query and input for similarity search param

* use undefined instead of empty string for query with similarity search

frigate/api/event.py

@@ -353,7 +353,10 @@ def events_search():
     after = request.args.get("after", type=float)
     before = request.args.get("before", type=float)
 
-    if not query:
+    # for similarity search
+    event_id = request.args.get("event_id", type=str)
+
+    if not query and not event_id:
         return make_response(
             jsonify(
                 {
@@ -432,7 +435,7 @@ def events_search():
     if search_type == "similarity":
         # Grab the ids of events that match the thumbnail image embeddings
         try:
-            search_event: Event = Event.get(Event.id == query)
+            search_event: Event = Event.get(Event.id == event_id)
         except DoesNotExist:
             return make_response(
                 jsonify(

web/src/components/filter/SearchFilterGroup.tsx

@@ -43,7 +43,6 @@ type SearchFilterGroupProps = {
   className: string;
   filters?: SearchFilters[];
   filter?: SearchFilter;
-  searchTerm: string;
   filterList?: FilterList;
   onUpdateFilter: (filter: SearchFilter) => void;
 };
@@ -51,7 +50,6 @@ export default function SearchFilterGroup({
   className,
   filters = DEFAULT_REVIEW_FILTERS,
   filter,
-  searchTerm,
   filterList,
   onUpdateFilter,
 }: SearchFilterGroupProps) {
@@ -213,7 +211,7 @@ export default function SearchFilterGroup({
       )}
       {config?.semantic_search?.enabled &&
         filters.includes("source") &&
-        !searchTerm.includes("similarity:") && (
+        !filter?.search_type?.includes("similarity") && (
           <SearchTypeButton
             selectedSearchSources={
               filter?.search_type ?? ["thumbnail", "description"]
@@ -914,7 +912,7 @@ export function SearchTypeContent({
         <div className="my-2.5 flex flex-col gap-2.5">
           <FilterSwitch
             label="Thumbnail Image"
-            isChecked={selectedSearchSources?.includes("thumbnail") ?? false}
+            isChecked={currentSearchSources?.includes("thumbnail") ?? false}
             onCheckedChange={(isChecked) => {
               const updatedSources = currentSearchSources
                 ? [...currentSearchSources]

web/src/components/input/InputWithTags.tsx

@@ -180,17 +180,11 @@ export default function InputWithTags({
 
   const createFilter = useCallback(
     (type: FilterType, value: string) => {
-      if (
-        allSuggestions[type as keyof SearchFilter]?.includes(value) ||
-        type === "before" ||
-        type === "after"
-      ) {
+      if (allSuggestions[type as FilterType]?.includes(value)) {
         const newFilters = { ...filters };
         let timestamp = 0;
 
         switch (type) {
-          case "query":
-            break;
           case "before":
           case "after":
             timestamp = convertLocalDateToTimestamp(value);
@@ -268,9 +262,7 @@ export default function InputWithTags({
     (filterType: FilterType, filterValue: string) => {
       const trimmedValue = filterValue.trim();
       if (
-        allSuggestions[filterType as keyof SearchFilter]?.includes(
-          trimmedValue,
-        ) ||
+        allSuggestions[filterType]?.includes(trimmedValue) ||
         ((filterType === "before" || filterType === "after") &&
           trimmedValue.match(/^\d{8}$/))
       ) {
@@ -429,14 +421,14 @@ export default function InputWithTags({
   }, [currentFilterType, inputValue, updateSuggestions]);
 
   useEffect(() => {
-    if (search?.startsWith("similarity:")) {
+    if (filters?.search_type && filters?.search_type.includes("similarity")) {
       setIsSimilaritySearch(true);
       setInputValue("");
     } else {
       setIsSimilaritySearch(false);
       setInputValue(search || "");
     }
-  }, [search]);
+  }, [filters, search]);
 
   return (
     <>
@@ -585,56 +577,57 @@ export default function InputWithTags({
                   </span>
                 )}
                 {Object.entries(filters).map(([filterType, filterValues]) =>
-                  Array.isArray(filterValues) ? (
-                    filterValues
-                      .filter(() => filterType !== "query")
-                      .map((value, index) => (
+                  Array.isArray(filterValues)
+                    ? filterValues
+                        .filter(() => filterType !== "query")
+                        .filter(() => !filterValues.includes("similarity"))
+                        .map((value, index) => (
+                          <span
+                            key={`${filterType}-${index}`}
+                            className="inline-flex items-center whitespace-nowrap rounded-full bg-green-100 px-2 py-0.5 text-sm capitalize text-green-800"
+                          >
+                            {filterType.replaceAll("_", " ")}:{" "}
+                            {value.replaceAll("_", " ")}
+                            <button
+                              onClick={() =>
+                                removeFilter(filterType as FilterType, value)
+                              }
+                              className="ml-1 focus:outline-none"
+                              aria-label={`Remove ${filterType}:${value.replaceAll("_", " ")} filter`}
+                            >
+                              <LuX className="h-3 w-3" />
+                            </button>
+                          </span>
+                        ))
+                    : filterType !== "event_id" && (
                         <span
-                          key={`${filterType}-${index}`}
+                          key={filterType}
                           className="inline-flex items-center whitespace-nowrap rounded-full bg-green-100 px-2 py-0.5 text-sm capitalize text-green-800"
                         >
-                          {filterType.replaceAll("_", " ")}:{" "}
-                          {value.replaceAll("_", " ")}
+                          {filterType}:
+                          {filterType === "before" || filterType === "after"
+                            ? new Date(
+                                (filterType === "before"
+                                  ? (filterValues as number) + 1
+                                  : (filterValues as number)) * 1000,
+                              ).toLocaleDateString(
+                                window.navigator?.language || "en-US",
+                              )
+                            : filterValues}
                           <button
                             onClick={() =>
-                              removeFilter(filterType as FilterType, value)
+                              removeFilter(
+                                filterType as FilterType,
+                                filterValues as string | number,
+                              )
                             }
                             className="ml-1 focus:outline-none"
-                            aria-label={`Remove ${filterType}:${value.replaceAll("_", " ")} filter`}
+                            aria-label={`Remove ${filterType}:${filterValues} filter`}
                           >
                             <LuX className="h-3 w-3" />
                           </button>
                         </span>
-                      ))
-                  ) : (
-                    <span
-                      key={filterType}
-                      className="inline-flex items-center whitespace-nowrap rounded-full bg-green-100 px-2 py-0.5 text-sm capitalize text-green-800"
-                    >
-                      {filterType}:
-                      {filterType === "before" || filterType === "after"
-                        ? new Date(
-                            (filterType === "before"
-                              ? (filterValues as number) + 1
-                              : (filterValues as number)) * 1000,
-                          ).toLocaleDateString(
-                            window.navigator?.language || "en-US",
-                          )
-                        : filterValues}
-                      <button
-                        onClick={() =>
-                          removeFilter(
-                            filterType as FilterType,
-                            filterValues as string | number,
-                          )
-                        }
-                        className="ml-1 focus:outline-none"
-                        aria-label={`Remove ${filterType}:${filterValues} filter`}
-                      >
-                        <LuX className="h-3 w-3" />
-                      </button>
-                    </span>
-                  ),
+                      ),
                 )}
               </div>
             </CommandGroup>

web/src/components/overlay/detail/ReviewDetailDialog.tsx

@@ -370,7 +370,9 @@ function EventItem({
                     <Chip
                       className="cursor-pointer rounded-md bg-gray-500 bg-gradient-to-br from-gray-400 to-gray-500"
                       onClick={() => {
-                        navigate(`/explore?similarity_search_id=${event.id}`);
+                        navigate(
+                          `/explore?search_type=similarity&event_id=${event.id}`,
+                        );
                       }}
                     >
                       <FaImages className="size-4 text-white" />

web/src/pages/Explore.tsx

@@ -1,5 +1,4 @@
 import { useApiFilterArgs } from "@/hooks/use-api-filter";
-import { useSearchEffect } from "@/hooks/use-overlay-state";
 import { SearchFilter, SearchQuery, SearchResult } from "@/types/search";
 import SearchView from "@/views/search/SearchView";
 import { useCallback, useEffect, useMemo, useState } from "react";
@@ -21,37 +20,22 @@ export default function Explore() {
     [searchSearchParams],
   );
 
-  // search filter
-
-  const similaritySearch = useMemo(() => {
-    if (!searchTerm.includes("similarity:")) {
-      return undefined;
-    }
-
-    return searchTerm.split(":")[1];
-  }, [searchTerm]);
-
-  // search api
-
-  useSearchEffect("query", (query) => {
-    setSearch(query);
-    return false;
-  });
-
-  useSearchEffect("similarity_search_id", (similarityId) => {
-    setSearch(`similarity:${similarityId}`);
-    // @ts-expect-error we want to clear this
-    setSearchFilter({ ...searchFilter, similarity_search_id: undefined });
-    return false;
-  });
+  const similaritySearch = useMemo(
+    () => searchSearchParams["search_type"] == "similarity",
+    [searchSearchParams],
+  );
 
   useEffect(() => {
     if (!searchTerm && !search) {
       return;
     }
 
+    // switch back to normal search when query is entered
     setSearchFilter({
       ...searchFilter,
+      search_type:
+        similaritySearch && search ? undefined : searchFilter?.search_type,
+      event_id: similaritySearch && search ? undefined : searchFilter?.event_id,
       query: search.length > 0 ? search : undefined,
     });
     // only update when search is updated
@@ -59,41 +43,18 @@ export default function Explore() {
   }, [search]);
 
   const searchQuery: SearchQuery = useMemo(() => {
-    if (similaritySearch) {
-      return [
-        "events/search",
-        {
-          query: similaritySearch,
-          cameras: searchSearchParams["cameras"],
-          labels: searchSearchParams["labels"],
-          sub_labels: searchSearchParams["subLabels"],
-          zones: searchSearchParams["zones"],
-          before: searchSearchParams["before"],
-          after: searchSearchParams["after"],
-          include_thumbnails: 0,
-          search_type: "similarity",
-        },
-      ];
+    // no search parameters
+    if (searchSearchParams && Object.keys(searchSearchParams).length === 0) {
+      return null;
     }
 
-    if (searchTerm) {
-      return [
-        "events/search",
-        {
-          query: searchTerm,
-          cameras: searchSearchParams["cameras"],
-          labels: searchSearchParams["labels"],
-          sub_labels: searchSearchParams["subLabels"],
-          zones: searchSearchParams["zones"],
-          before: searchSearchParams["before"],
-          after: searchSearchParams["after"],
-          search_type: searchSearchParams["search_type"],
-          include_thumbnails: 0,
-        },
-      ];
-    }
-
-    if (searchSearchParams && Object.keys(searchSearchParams).length !== 0) {
+    // parameters, but no search term and not similarity
+    if (
+      searchSearchParams &&
+      Object.keys(searchSearchParams).length !== 0 &&
+      !searchTerm &&
+      !similaritySearch
+    ) {
       return [
         "events",
         {
@@ -112,7 +73,26 @@ export default function Explore() {
       ];
     }
 
-    return null;
+    // parameters and search term
+    if (!similaritySearch) {
+      setSearch(searchTerm);
+    }
+
+    return [
+      "events/search",
+      {
+        query: similaritySearch ? undefined : searchTerm,
+        cameras: searchSearchParams["cameras"],
+        labels: searchSearchParams["labels"],
+        sub_labels: searchSearchParams["subLabels"],
+        zones: searchSearchParams["zones"],
+        before: searchSearchParams["before"],
+        after: searchSearchParams["after"],
+        search_type: searchSearchParams["search_type"],
+        event_id: searchSearchParams["event_id"],
+        include_thumbnails: 0,
+      },
+    ];
   }, [searchTerm, searchSearchParams, similaritySearch]);
 
   // paging
@@ -205,7 +185,13 @@ export default function Explore() {
           searchResults={searchResults}
           isLoading={(isLoadingInitialData || isLoadingMore) ?? true}
           setSearch={setSearch}
-          setSimilaritySearch={(search) => setSearch(`similarity:${search.id}`)}
+          setSimilaritySearch={(search) => {
+            setSearchFilter({
+              ...searchFilter,
+              search_type: ["similarity"],
+              event_id: search.id,
+            });
+          }}
           setSearchFilter={setSearchFilter}
           onUpdateFilter={setSearchFilter}
           loadMore={loadMore}

web/src/types/search.ts

@@ -56,7 +56,7 @@ export type SearchQueryParams = {
 };
 
 export type SearchQuery = [string, SearchQueryParams] | null;
-export type FilterType = keyof SearchFilter;
+export type FilterType = Exclude<keyof SearchFilter, "query">;
 
 export type SavedSearchQuery = {
   name: string;

web/src/views/search/SearchView.tsx

@@ -281,7 +281,6 @@ export default function SearchView({
               "w-full justify-between md:justify-start lg:justify-end",
             )}
             filter={searchFilter}
-            searchTerm={searchTerm}
             onUpdateFilter={onUpdateFilter}
           />
         )}