Add ability to filter based on search type (#13641)

2024-11-21 19:07:46 +01:00 · 2024-09-09 14:45:19 -05:00 · 2024-09-09 14:45:19 -05:00 · cae11cbb86
commit cae11cbb86
parent 03ff3e639f
5 changed files with 113 additions and 28 deletions
--- a/frigate/api/event.py
+++ b/frigate/api/event.py
@ -274,7 +274,7 @@ def event_ids():
@EventBp.route("/events/search")
 def events_search():
    query = request.args.get("query", type=str)
-    search_type = request.args.get("search_type", "text", type=str)
+    search_type = request.args.get("search_type", "thumbnail,description", type=str)
    include_thumbnails = request.args.get("include_thumbnails", default=1, type=int)
    limit = request.args.get("limit", 50, type=int)

@ -358,7 +358,7 @@ def events_search():
    thumb_ids = {}
    desc_ids = {}

-    if search_type == "thumbnail":
+    if search_type == "similarity":
        # Grab the ids of events that match the thumbnail image embeddings
        try:
            search_event: Event = Event.get(Event.id == query)
@ -386,29 +386,34 @@ def events_search():
            )
        )
    else:
-        thumb_result = context.embeddings.thumbnail.query(
-            query_texts=[query],
-            n_results=limit,
-            where=where,
-        )
-        # Do a rudimentary normalization of the difference in distances returned by CLIP and MiniLM.
-        thumb_ids = dict(
-            zip(
-                thumb_result["ids"][0],
-                context.thumb_stats.normalize(thumb_result["distances"][0]),
+        search_types = search_type.split(",")
+
+        if "thumbnail" in search_types:
+            thumb_result = context.embeddings.thumbnail.query(
+                query_texts=[query],
+                n_results=limit,
+                where=where,
            )
-        )
-        desc_result = context.embeddings.description.query(
-            query_texts=[query],
-            n_results=limit,
-            where=where,
-        )
-        desc_ids = dict(
-            zip(
-                desc_result["ids"][0],
-                context.desc_stats.normalize(desc_result["distances"][0]),
+            # Do a rudimentary normalization of the difference in distances returned by CLIP and MiniLM.
+            thumb_ids = dict(
+                zip(
+                    thumb_result["ids"][0],
+                    context.thumb_stats.normalize(thumb_result["distances"][0]),
+                )
+            )
+
+        if "description" in search_types:
+            desc_result = context.embeddings.description.query(
+                query_texts=[query],
+                n_results=limit,
+                where=where,
+            )
+            desc_ids = dict(
+                zip(
+                    desc_result["ids"][0],
+                    context.desc_stats.normalize(desc_result["distances"][0]),
+                )
            )
-        )

    results = {}
    for event_id in thumb_ids.keys() | desc_ids:
--- a/web/src/components/filter/SearchFilterGroup.tsx
+++ b/web/src/components/filter/SearchFilterGroup.tsx
@ -17,7 +17,7 @@ import FilterSwitch from "./FilterSwitch";
 import { FilterList } from "@/types/filter";
 import { CalendarRangeFilterButton } from "./CalendarFilterButton";
 import { CamerasFilterButton } from "./CamerasFilterButton";
-import { SearchFilter } from "@/types/search";
+import { SearchFilter, SearchSource } from "@/types/search";
 import { DateRange } from "react-day-picker";

 const SEARCH_FILTERS = ["cameras", "date", "general"] as const;
@ -103,6 +103,7 @@ export default function SearchFilterGroup({
      cameras: Object.keys(config?.cameras || {}),
      labels: Object.values(allLabels || {}),
      zones: Object.values(allZones || {}),
+      search_type: ["thumbnail", "description"] as SearchSource[],
    }),
    [config, allLabels, allZones],
  );
@ -178,12 +179,18 @@ export default function SearchFilterGroup({
          selectedLabels={filter?.labels}
          allZones={filterValues.zones}
          selectedZones={filter?.zones}
+          selectedSearchSources={
+            filter?.search_type ?? ["thumbnail", "description"]
+          }
          updateLabelFilter={(newLabels) => {
            onUpdateFilter({ ...filter, labels: newLabels });
          }}
          updateZoneFilter={(newZones) =>
            onUpdateFilter({ ...filter, zones: newZones })
          }
+          updateSearchSourceFilter={(newSearchSource) =>
+            onUpdateFilter({ ...filter, search_type: newSearchSource })
+          }
        />
      )}
      {isMobile && mobileSettingsFeatures.length > 0 && (
@ -211,16 +218,20 @@ type GeneralFilterButtonProps = {
  selectedLabels: string[] | undefined;
  allZones: string[];
  selectedZones?: string[];
+  selectedSearchSources: SearchSource[];
  updateLabelFilter: (labels: string[] | undefined) => void;
  updateZoneFilter: (zones: string[] | undefined) => void;
+  updateSearchSourceFilter: (sources: SearchSource[]) => void;
 };
 function GeneralFilterButton({
  allLabels,
  selectedLabels,
  allZones,
  selectedZones,
+  selectedSearchSources,
  updateLabelFilter,
  updateZoneFilter,
+  updateSearchSourceFilter,
 }: GeneralFilterButtonProps) {
  const [open, setOpen] = useState(false);
  const [currentLabels, setCurrentLabels] = useState<string[] | undefined>(
@ -229,6 +240,9 @@ function GeneralFilterButton({
  const [currentZones, setCurrentZones] = useState<string[] | undefined>(
    selectedZones,
  );
+  const [currentSearchSources, setCurrentSearchSources] = useState<
+    SearchSource[]
+  >(selectedSearchSources);

  const trigger = (
    <Button
@ -256,10 +270,14 @@ function GeneralFilterButton({
      allZones={allZones}
      selectedZones={selectedZones}
      currentZones={currentZones}
+      selectedSearchSources={selectedSearchSources}
+      currentSearchSources={currentSearchSources}
      setCurrentZones={setCurrentZones}
      updateZoneFilter={updateZoneFilter}
-      updateLabelFilter={updateLabelFilter}
      setCurrentLabels={setCurrentLabels}
+      updateLabelFilter={updateLabelFilter}
+      setCurrentSearchSources={setCurrentSearchSources}
+      updateSearchSourceFilter={updateSearchSourceFilter}
      onClose={() => setOpen(false)}
    />
  );
@ -308,10 +326,14 @@ type GeneralFilterContentProps = {
  allZones?: string[];
  selectedZones?: string[];
  currentZones?: string[];
+  selectedSearchSources: SearchSource[];
+  currentSearchSources: SearchSource[];
  updateLabelFilter: (labels: string[] | undefined) => void;
  setCurrentLabels: (labels: string[] | undefined) => void;
  updateZoneFilter?: (zones: string[] | undefined) => void;
  setCurrentZones?: (zones: string[] | undefined) => void;
+  setCurrentSearchSources: (sources: SearchSource[]) => void;
+  updateSearchSourceFilter: (sources: SearchSource[]) => void;
  onClose: () => void;
 };
 export function GeneralFilterContent({
@ -321,15 +343,62 @@ export function GeneralFilterContent({
  allZones,
  selectedZones,
  currentZones,
+  selectedSearchSources,
+  currentSearchSources,
  updateLabelFilter,
  setCurrentLabels,
  updateZoneFilter,
  setCurrentZones,
+  setCurrentSearchSources,
+  updateSearchSourceFilter,
  onClose,
 }: GeneralFilterContentProps) {
  return (
    <>
      <div className="scrollbar-container h-auto max-h-[80dvh] overflow-y-auto overflow-x-hidden">
+        <div className="my-2.5 flex flex-col gap-2.5">
+          <FilterSwitch
+            label="Thumbnail Image"
+            isChecked={currentSearchSources?.includes("thumbnail") ?? false}
+            onCheckedChange={(isChecked) => {
+              const updatedSources = currentSearchSources
+                ? [...currentSearchSources]
+                : [];
+
+              if (isChecked) {
+                updatedSources.push("thumbnail");
+                setCurrentSearchSources(updatedSources);
+              } else {
+                if (updatedSources.length > 1) {
+                  const index = updatedSources.indexOf("thumbnail");
+                  if (index !== -1) updatedSources.splice(index, 1);
+                  setCurrentSearchSources(updatedSources);
+                }
+              }
+            }}
+          />
+          <FilterSwitch
+            label="Description"
+            isChecked={currentSearchSources?.includes("description") ?? false}
+            onCheckedChange={(isChecked) => {
+              const updatedSources = currentSearchSources
+                ? [...currentSearchSources]
+                : [];
+
+              if (isChecked) {
+                updatedSources.push("description");
+                setCurrentSearchSources(updatedSources);
+              } else {
+                if (updatedSources.length > 1) {
+                  const index = updatedSources.indexOf("description");
+                  if (index !== -1) updatedSources.splice(index, 1);
+                  setCurrentSearchSources(updatedSources);
+                }
+              }
+            }}
+          />
+          <DropdownMenuSeparator />
+        </div>
        <div className="mb-5 mt-2.5 flex items-center justify-between">
          <Label
            className="mx-2 cursor-pointer text-primary"
@ -351,6 +420,7 @@ export function GeneralFilterContent({
        <div className="my-2.5 flex flex-col gap-2.5">
          {allLabels.map((item) => (
            <FilterSwitch
+              key={item}
              label={item.replaceAll("_", " ")}
              isChecked={currentLabels?.includes(item) ?? false}
              onCheckedChange={(isChecked) => {
@ -397,6 +467,7 @@ export function GeneralFilterContent({
            <div className="my-2.5 flex flex-col gap-2.5">
              {allZones.map((item) => (
                <FilterSwitch
+                  key={item}
                  label={item.replaceAll("_", " ")}
                  isChecked={currentZones?.includes(item) ?? false}
                  onCheckedChange={(isChecked) => {
@ -438,6 +509,10 @@ export function GeneralFilterContent({
              updateZoneFilter(currentZones);
            }

+            if (selectedSearchSources != currentSearchSources) {
+              updateSearchSourceFilter(currentSearchSources);
+            }
+
            onClose();
          }}
        >
--- a/web/src/pages/Search.tsx
+++ b/web/src/pages/Search.tsx
@ -75,7 +75,7 @@ export default function Search() {
          before: searchSearchParams["before"],
          after: searchSearchParams["after"],
          include_thumbnails: 0,
-          search_type: "thumbnail",
+          search_type: "similarity",
        },
      ];
    }
@ -89,6 +89,7 @@ export default function Search() {
        zones: searchSearchParams["zones"],
        before: searchSearchParams["before"],
        after: searchSearchParams["after"],
+        search_type: searchSearchParams["search_type"],
        include_thumbnails: 0,
      },
    ];
@ -192,6 +193,7 @@ export default function Search() {
        allPreviews={allPreviews}
        isLoading={isLoading}
        setSearch={setSearch}
+        similaritySearch={similaritySearch}
        setSimilaritySearch={setSimilaritySearch}
        onUpdateFilter={onUpdateFilter}
        onOpenSearch={onOpenSearch}
--- a/web/src/types/search.ts
+++ b/web/src/types/search.ts
@ -1,4 +1,4 @@
-type SearchSource = "thumbnail" | "description";
+export type SearchSource = "similarity" | "thumbnail" | "description";

 export type SearchResult = {
  id: string;
@ -21,4 +21,5 @@ export type SearchFilter = {
  zones?: string[];
  before?: number;
  after?: number;
+  search_type?: SearchSource[];
 };
--- a/web/src/views/search/SearchView.tsx
+++ b/web/src/views/search/SearchView.tsx
@ -32,6 +32,7 @@ type SearchViewProps = {
  searchResults?: SearchResult[];
  allPreviews?: Preview[];
  isLoading: boolean;
+  similaritySearch?: SearchResult;
  setSearch: (search: string) => void;
  setSimilaritySearch: (search: SearchResult) => void;
  onUpdateFilter: (filter: SearchFilter) => void;
@ -44,6 +45,7 @@ export default function SearchView({
  searchResults,
  allPreviews,
  isLoading,
+  similaritySearch,
  setSearch,
  setSimilaritySearch,
  onUpdateFilter,
@ -112,7 +114,7 @@ export default function SearchView({
            placeholder={
              isMobileOnly ? "Search" : "Search for a detected object..."
            }
-            value={search}
+            value={similaritySearch ? "" : search}
            onChange={(e) => setSearch(e.target.value)}
          />
          {search && (