Fix similarity search (#13856)

* add event_id param to api * exclude query from filtertype * update review pane link for similarity search * update filter group for similarity param and fix switch bug * unneeded prop * update query and input for similarity search param * use undefined instead of empty string for query with similarity search
2024-11-21 19:07:46 +01:00 · 2024-09-20 12:05:55 -05:00 · 2024-09-20 12:05:55 -05:00 · 176af55e8c
commit 176af55e8c
parent 1a51ce712c
7 changed files with 100 additions and 119 deletions
--- a/frigate/api/event.py
+++ b/frigate/api/event.py
@ -353,7 +353,10 @@ def events_search():
    after = request.args.get("after", type=float)
    before = request.args.get("before", type=float)

-    if not query:
+    # for similarity search
+    event_id = request.args.get("event_id", type=str)
+
+    if not query and not event_id:
        return make_response(
            jsonify(
                {
@ -432,7 +435,7 @@ def events_search():
    if search_type == "similarity":
        # Grab the ids of events that match the thumbnail image embeddings
        try:
-            search_event: Event = Event.get(Event.id == query)
+            search_event: Event = Event.get(Event.id == event_id)
        except DoesNotExist:
            return make_response(
                jsonify(
--- a/web/src/components/filter/SearchFilterGroup.tsx
+++ b/web/src/components/filter/SearchFilterGroup.tsx
@ -43,7 +43,6 @@ type SearchFilterGroupProps = {
  className: string;
  filters?: SearchFilters[];
  filter?: SearchFilter;
-  searchTerm: string;
  filterList?: FilterList;
  onUpdateFilter: (filter: SearchFilter) => void;
 };
@ -51,7 +50,6 @@ export default function SearchFilterGroup({
  className,
  filters = DEFAULT_REVIEW_FILTERS,
  filter,
-  searchTerm,
  filterList,
  onUpdateFilter,
 }: SearchFilterGroupProps) {
@ -213,7 +211,7 @@ export default function SearchFilterGroup({
      )}
      {config?.semantic_search?.enabled &&
        filters.includes("source") &&
-        !searchTerm.includes("similarity:") && (
+        !filter?.search_type?.includes("similarity") && (
          <SearchTypeButton
            selectedSearchSources={
              filter?.search_type ?? ["thumbnail", "description"]
@ -914,7 +912,7 @@ export function SearchTypeContent({
        <div className="my-2.5 flex flex-col gap-2.5">
          <FilterSwitch
            label="Thumbnail Image"
-            isChecked={selectedSearchSources?.includes("thumbnail") ?? false}
+            isChecked={currentSearchSources?.includes("thumbnail") ?? false}
            onCheckedChange={(isChecked) => {
              const updatedSources = currentSearchSources
                ? [...currentSearchSources]
--- a/web/src/components/input/InputWithTags.tsx
+++ b/web/src/components/input/InputWithTags.tsx
@ -180,17 +180,11 @@ export default function InputWithTags({

  const createFilter = useCallback(
    (type: FilterType, value: string) => {
-      if (
-        allSuggestions[type as keyof SearchFilter]?.includes(value) ||
-        type === "before" ||
-        type === "after"
-      ) {
+      if (allSuggestions[type as FilterType]?.includes(value)) {
        const newFilters = { ...filters };
        let timestamp = 0;

        switch (type) {
-          case "query":
-            break;
          case "before":
          case "after":
            timestamp = convertLocalDateToTimestamp(value);
@ -268,9 +262,7 @@ export default function InputWithTags({
    (filterType: FilterType, filterValue: string) => {
      const trimmedValue = filterValue.trim();
      if (
-        allSuggestions[filterType as keyof SearchFilter]?.includes(
-          trimmedValue,
-        ) ||
+        allSuggestions[filterType]?.includes(trimmedValue) ||
        ((filterType === "before" || filterType === "after") &&
          trimmedValue.match(/^\d{8}$/))
      ) {
@ -429,14 +421,14 @@ export default function InputWithTags({
  }, [currentFilterType, inputValue, updateSuggestions]);

  useEffect(() => {
-    if (search?.startsWith("similarity:")) {
+    if (filters?.search_type && filters?.search_type.includes("similarity")) {
      setIsSimilaritySearch(true);
      setInputValue("");
    } else {
      setIsSimilaritySearch(false);
      setInputValue(search || "");
    }
-  }, [search]);
+  }, [filters, search]);

  return (
    <>
@ -585,9 +577,10 @@ export default function InputWithTags({
                  </span>
                )}
                {Object.entries(filters).map(([filterType, filterValues]) =>
-                  Array.isArray(filterValues) ? (
-                    filterValues
+                  Array.isArray(filterValues)
+                    ? filterValues
                        .filter(() => filterType !== "query")
+                        .filter(() => !filterValues.includes("similarity"))
                        .map((value, index) => (
                          <span
                            key={`${filterType}-${index}`}
@ -606,7 +599,7 @@ export default function InputWithTags({
                            </button>
                          </span>
                        ))
-                  ) : (
+                    : filterType !== "event_id" && (
                        <span
                          key={filterType}
                          className="inline-flex items-center whitespace-nowrap rounded-full bg-green-100 px-2 py-0.5 text-sm capitalize text-green-800"
--- a/web/src/components/overlay/detail/ReviewDetailDialog.tsx
+++ b/web/src/components/overlay/detail/ReviewDetailDialog.tsx
@ -370,7 +370,9 @@ function EventItem({
                    <Chip
                      className="cursor-pointer rounded-md bg-gray-500 bg-gradient-to-br from-gray-400 to-gray-500"
                      onClick={() => {
-                        navigate(`/explore?similarity_search_id=${event.id}`);
+                        navigate(
+                          `/explore?search_type=similarity&event_id=${event.id}`,
+                        );
                      }}
                    >
                      <FaImages className="size-4 text-white" />
--- a/web/src/pages/Explore.tsx
+++ b/web/src/pages/Explore.tsx
@ -1,5 +1,4 @@
 import { useApiFilterArgs } from "@/hooks/use-api-filter";
-import { useSearchEffect } from "@/hooks/use-overlay-state";
 import { SearchFilter, SearchQuery, SearchResult } from "@/types/search";
 import SearchView from "@/views/search/SearchView";
 import { useCallback, useEffect, useMemo, useState } from "react";
@ -21,37 +20,22 @@ export default function Explore() {
    [searchSearchParams],
  );

-  // search filter
-
-  const similaritySearch = useMemo(() => {
-    if (!searchTerm.includes("similarity:")) {
-      return undefined;
-    }
-
-    return searchTerm.split(":")[1];
-  }, [searchTerm]);
-
-  // search api
-
-  useSearchEffect("query", (query) => {
-    setSearch(query);
-    return false;
-  });
-
-  useSearchEffect("similarity_search_id", (similarityId) => {
-    setSearch(`similarity:${similarityId}`);
-    // @ts-expect-error we want to clear this
-    setSearchFilter({ ...searchFilter, similarity_search_id: undefined });
-    return false;
-  });
+  const similaritySearch = useMemo(
+    () => searchSearchParams["search_type"] == "similarity",
+    [searchSearchParams],
+  );

  useEffect(() => {
    if (!searchTerm && !search) {
      return;
    }

+    // switch back to normal search when query is entered
    setSearchFilter({
      ...searchFilter,
+      search_type:
+        similaritySearch && search ? undefined : searchFilter?.search_type,
+      event_id: similaritySearch && search ? undefined : searchFilter?.event_id,
      query: search.length > 0 ? search : undefined,
    });
    // only update when search is updated
@ -59,41 +43,18 @@ export default function Explore() {
  }, [search]);

  const searchQuery: SearchQuery = useMemo(() => {
-    if (similaritySearch) {
-      return [
-        "events/search",
-        {
-          query: similaritySearch,
-          cameras: searchSearchParams["cameras"],
-          labels: searchSearchParams["labels"],
-          sub_labels: searchSearchParams["subLabels"],
-          zones: searchSearchParams["zones"],
-          before: searchSearchParams["before"],
-          after: searchSearchParams["after"],
-          include_thumbnails: 0,
-          search_type: "similarity",
-        },
-      ];
+    // no search parameters
+    if (searchSearchParams && Object.keys(searchSearchParams).length === 0) {
+      return null;
    }

-    if (searchTerm) {
-      return [
-        "events/search",
-        {
-          query: searchTerm,
-          cameras: searchSearchParams["cameras"],
-          labels: searchSearchParams["labels"],
-          sub_labels: searchSearchParams["subLabels"],
-          zones: searchSearchParams["zones"],
-          before: searchSearchParams["before"],
-          after: searchSearchParams["after"],
-          search_type: searchSearchParams["search_type"],
-          include_thumbnails: 0,
-        },
-      ];
-    }
-
-    if (searchSearchParams && Object.keys(searchSearchParams).length !== 0) {
+    // parameters, but no search term and not similarity
+    if (
+      searchSearchParams &&
+      Object.keys(searchSearchParams).length !== 0 &&
+      !searchTerm &&
+      !similaritySearch
+    ) {
      return [
        "events",
        {
@ -112,7 +73,26 @@ export default function Explore() {
      ];
    }

-    return null;
+    // parameters and search term
+    if (!similaritySearch) {
+      setSearch(searchTerm);
+    }
+
+    return [
+      "events/search",
+      {
+        query: similaritySearch ? undefined : searchTerm,
+        cameras: searchSearchParams["cameras"],
+        labels: searchSearchParams["labels"],
+        sub_labels: searchSearchParams["subLabels"],
+        zones: searchSearchParams["zones"],
+        before: searchSearchParams["before"],
+        after: searchSearchParams["after"],
+        search_type: searchSearchParams["search_type"],
+        event_id: searchSearchParams["event_id"],
+        include_thumbnails: 0,
+      },
+    ];
  }, [searchTerm, searchSearchParams, similaritySearch]);

  // paging
@ -205,7 +185,13 @@ export default function Explore() {
          searchResults={searchResults}
          isLoading={(isLoadingInitialData || isLoadingMore) ?? true}
          setSearch={setSearch}
-          setSimilaritySearch={(search) => setSearch(`similarity:${search.id}`)}
+          setSimilaritySearch={(search) => {
+            setSearchFilter({
+              ...searchFilter,
+              search_type: ["similarity"],
+              event_id: search.id,
+            });
+          }}
          setSearchFilter={setSearchFilter}
          onUpdateFilter={setSearchFilter}
          loadMore={loadMore}
--- a/web/src/types/search.ts
+++ b/web/src/types/search.ts
@ -56,7 +56,7 @@ export type SearchQueryParams = {
 };

 export type SearchQuery = [string, SearchQueryParams] | null;
-export type FilterType = keyof SearchFilter;
+export type FilterType = Exclude<keyof SearchFilter, "query">;

 export type SavedSearchQuery = {
  name: string;
--- a/web/src/views/search/SearchView.tsx
+++ b/web/src/views/search/SearchView.tsx
@ -281,7 +281,6 @@ export default function SearchView({
              "w-full justify-between md:justify-start lg:justify-end",
            )}
            filter={searchFilter}
-            searchTerm={searchTerm}
            onUpdateFilter={onUpdateFilter}
          />
        )}