Add ability to filter based on search type (#13641)

frigate/api/event.py

@@ -274,7 +274,7 @@ def event_ids():
 @EventBp.route("/events/search")
 def events_search():
     query = request.args.get("query", type=str)
-    search_type = request.args.get("search_type", "text", type=str)
+    search_type = request.args.get("search_type", "thumbnail,description", type=str)
     include_thumbnails = request.args.get("include_thumbnails", default=1, type=int)
     limit = request.args.get("limit", 50, type=int)
 
@@ -358,7 +358,7 @@ def events_search():
     thumb_ids = {}
     desc_ids = {}
 
-    if search_type == "thumbnail":
+    if search_type == "similarity":
         # Grab the ids of events that match the thumbnail image embeddings
         try:
             search_event: Event = Event.get(Event.id == query)
@@ -386,29 +386,34 @@ def events_search():
             )
         )
     else:
-        thumb_result = context.embeddings.thumbnail.query(
+        search_types = search_type.split(",")
-            query_texts=[query],
+
-            n_results=limit,
+        if "thumbnail" in search_types:
-            where=where,
+            thumb_result = context.embeddings.thumbnail.query(
-        )
+                query_texts=[query],
-        # Do a rudimentary normalization of the difference in distances returned by CLIP and MiniLM.
+                n_results=limit,
-        thumb_ids = dict(
+                where=where,
-            zip(
-                thumb_result["ids"][0],
-                context.thumb_stats.normalize(thumb_result["distances"][0]),
             )
-        )
+            # Do a rudimentary normalization of the difference in distances returned by CLIP and MiniLM.
-        desc_result = context.embeddings.description.query(
+            thumb_ids = dict(
-            query_texts=[query],
+                zip(
-            n_results=limit,
+                    thumb_result["ids"][0],
-            where=where,
+                    context.thumb_stats.normalize(thumb_result["distances"][0]),
-        )
+                )
-        desc_ids = dict(
+            )
-            zip(
+
-                desc_result["ids"][0],
+        if "description" in search_types:
-                context.desc_stats.normalize(desc_result["distances"][0]),
+            desc_result = context.embeddings.description.query(
+                query_texts=[query],
+                n_results=limit,
+                where=where,
+            )
+            desc_ids = dict(
+                zip(
+                    desc_result["ids"][0],
+                    context.desc_stats.normalize(desc_result["distances"][0]),
+                )
             )
-        )
 
     results = {}
     for event_id in thumb_ids.keys() | desc_ids:

web/src/components/filter/SearchFilterGroup.tsx

@@ -17,7 +17,7 @@ import FilterSwitch from "./FilterSwitch";
 import { FilterList } from "@/types/filter";
 import { CalendarRangeFilterButton } from "./CalendarFilterButton";
 import { CamerasFilterButton } from "./CamerasFilterButton";
-import { SearchFilter } from "@/types/search";
+import { SearchFilter, SearchSource } from "@/types/search";
 import { DateRange } from "react-day-picker";
 
 const SEARCH_FILTERS = ["cameras", "date", "general"] as const;
@@ -103,6 +103,7 @@ export default function SearchFilterGroup({
       cameras: Object.keys(config?.cameras || {}),
       labels: Object.values(allLabels || {}),
       zones: Object.values(allZones || {}),
+      search_type: ["thumbnail", "description"] as SearchSource[],
     }),
     [config, allLabels, allZones],
   );
@@ -178,12 +179,18 @@ export default function SearchFilterGroup({
           selectedLabels={filter?.labels}
           allZones={filterValues.zones}
           selectedZones={filter?.zones}
+          selectedSearchSources={
+            filter?.search_type ?? ["thumbnail", "description"]
+          }
           updateLabelFilter={(newLabels) => {
             onUpdateFilter({ ...filter, labels: newLabels });
           }}
           updateZoneFilter={(newZones) =>
             onUpdateFilter({ ...filter, zones: newZones })
           }
+          updateSearchSourceFilter={(newSearchSource) =>
+            onUpdateFilter({ ...filter, search_type: newSearchSource })
+          }
         />
       )}
       {isMobile && mobileSettingsFeatures.length > 0 && (
@@ -211,16 +218,20 @@ type GeneralFilterButtonProps = {
   selectedLabels: string[] | undefined;
   allZones: string[];
   selectedZones?: string[];
+  selectedSearchSources: SearchSource[];
   updateLabelFilter: (labels: string[] | undefined) => void;
   updateZoneFilter: (zones: string[] | undefined) => void;
+  updateSearchSourceFilter: (sources: SearchSource[]) => void;
 };
 function GeneralFilterButton({
   allLabels,
   selectedLabels,
   allZones,
   selectedZones,
+  selectedSearchSources,
   updateLabelFilter,
   updateZoneFilter,
+  updateSearchSourceFilter,
 }: GeneralFilterButtonProps) {
   const [open, setOpen] = useState(false);
   const [currentLabels, setCurrentLabels] = useState<string[] | undefined>(
@@ -229,6 +240,9 @@ function GeneralFilterButton({
   const [currentZones, setCurrentZones] = useState<string[] | undefined>(
     selectedZones,
   );
+  const [currentSearchSources, setCurrentSearchSources] = useState<
+    SearchSource[]
+  >(selectedSearchSources);
 
   const trigger = (
     <Button
@@ -256,10 +270,14 @@ function GeneralFilterButton({
       allZones={allZones}
       selectedZones={selectedZones}
       currentZones={currentZones}
+      selectedSearchSources={selectedSearchSources}
+      currentSearchSources={currentSearchSources}
       setCurrentZones={setCurrentZones}
       updateZoneFilter={updateZoneFilter}
-      updateLabelFilter={updateLabelFilter}
       setCurrentLabels={setCurrentLabels}
+      updateLabelFilter={updateLabelFilter}
+      setCurrentSearchSources={setCurrentSearchSources}
+      updateSearchSourceFilter={updateSearchSourceFilter}
       onClose={() => setOpen(false)}
     />
   );
@@ -308,10 +326,14 @@ type GeneralFilterContentProps = {
   allZones?: string[];
   selectedZones?: string[];
   currentZones?: string[];
+  selectedSearchSources: SearchSource[];
+  currentSearchSources: SearchSource[];
   updateLabelFilter: (labels: string[] | undefined) => void;
   setCurrentLabels: (labels: string[] | undefined) => void;
   updateZoneFilter?: (zones: string[] | undefined) => void;
   setCurrentZones?: (zones: string[] | undefined) => void;
+  setCurrentSearchSources: (sources: SearchSource[]) => void;
+  updateSearchSourceFilter: (sources: SearchSource[]) => void;
   onClose: () => void;
 };
 export function GeneralFilterContent({
@@ -321,15 +343,62 @@ export function GeneralFilterContent({
   allZones,
   selectedZones,
   currentZones,
+  selectedSearchSources,
+  currentSearchSources,
   updateLabelFilter,
   setCurrentLabels,
   updateZoneFilter,
   setCurrentZones,
+  setCurrentSearchSources,
+  updateSearchSourceFilter,
   onClose,
 }: GeneralFilterContentProps) {
   return (
     <>
       <div className="scrollbar-container h-auto max-h-[80dvh] overflow-y-auto overflow-x-hidden">
+        <div className="my-2.5 flex flex-col gap-2.5">
+          <FilterSwitch
+            label="Thumbnail Image"
+            isChecked={currentSearchSources?.includes("thumbnail") ?? false}
+            onCheckedChange={(isChecked) => {
+              const updatedSources = currentSearchSources
+                ? [...currentSearchSources]
+                : [];
+
+              if (isChecked) {
+                updatedSources.push("thumbnail");
+                setCurrentSearchSources(updatedSources);
+              } else {
+                if (updatedSources.length > 1) {
+                  const index = updatedSources.indexOf("thumbnail");
+                  if (index !== -1) updatedSources.splice(index, 1);
+                  setCurrentSearchSources(updatedSources);
+                }
+              }
+            }}
+          />
+          <FilterSwitch
+            label="Description"
+            isChecked={currentSearchSources?.includes("description") ?? false}
+            onCheckedChange={(isChecked) => {
+              const updatedSources = currentSearchSources
+                ? [...currentSearchSources]
+                : [];
+
+              if (isChecked) {
+                updatedSources.push("description");
+                setCurrentSearchSources(updatedSources);
+              } else {
+                if (updatedSources.length > 1) {
+                  const index = updatedSources.indexOf("description");
+                  if (index !== -1) updatedSources.splice(index, 1);
+                  setCurrentSearchSources(updatedSources);
+                }
+              }
+            }}
+          />
+          <DropdownMenuSeparator />
+        </div>
         <div className="mb-5 mt-2.5 flex items-center justify-between">
           <Label
             className="mx-2 cursor-pointer text-primary"
@@ -351,6 +420,7 @@ export function GeneralFilterContent({
         <div className="my-2.5 flex flex-col gap-2.5">
           {allLabels.map((item) => (
             <FilterSwitch
+              key={item}
               label={item.replaceAll("_", " ")}
               isChecked={currentLabels?.includes(item) ?? false}
               onCheckedChange={(isChecked) => {
@@ -397,6 +467,7 @@ export function GeneralFilterContent({
             <div className="my-2.5 flex flex-col gap-2.5">
               {allZones.map((item) => (
                 <FilterSwitch
+                  key={item}
                   label={item.replaceAll("_", " ")}
                   isChecked={currentZones?.includes(item) ?? false}
                   onCheckedChange={(isChecked) => {
@@ -438,6 +509,10 @@ export function GeneralFilterContent({
               updateZoneFilter(currentZones);
             }
 
+            if (selectedSearchSources != currentSearchSources) {
+              updateSearchSourceFilter(currentSearchSources);
+            }
+
             onClose();
           }}
         >

web/src/pages/Search.tsx

@@ -75,7 +75,7 @@ export default function Search() {
           before: searchSearchParams["before"],
           after: searchSearchParams["after"],
           include_thumbnails: 0,
-          search_type: "thumbnail",
+          search_type: "similarity",
         },
       ];
     }
@@ -89,6 +89,7 @@ export default function Search() {
         zones: searchSearchParams["zones"],
         before: searchSearchParams["before"],
         after: searchSearchParams["after"],
+        search_type: searchSearchParams["search_type"],
         include_thumbnails: 0,
       },
     ];
@@ -192,6 +193,7 @@ export default function Search() {
         allPreviews={allPreviews}
         isLoading={isLoading}
         setSearch={setSearch}
+        similaritySearch={similaritySearch}
         setSimilaritySearch={setSimilaritySearch}
         onUpdateFilter={onUpdateFilter}
         onOpenSearch={onOpenSearch}

web/src/types/search.ts

@@ -1,4 +1,4 @@
-type SearchSource = "thumbnail" | "description";
+export type SearchSource = "similarity" | "thumbnail" | "description";
 
 export type SearchResult = {
   id: string;
@@ -21,4 +21,5 @@ export type SearchFilter = {
   zones?: string[];
   before?: number;
   after?: number;
+  search_type?: SearchSource[];
 };

web/src/views/search/SearchView.tsx

@@ -32,6 +32,7 @@ type SearchViewProps = {
   searchResults?: SearchResult[];
   allPreviews?: Preview[];
   isLoading: boolean;
+  similaritySearch?: SearchResult;
   setSearch: (search: string) => void;
   setSimilaritySearch: (search: SearchResult) => void;
   onUpdateFilter: (filter: SearchFilter) => void;
@@ -44,6 +45,7 @@ export default function SearchView({
   searchResults,
   allPreviews,
   isLoading,
+  similaritySearch,
   setSearch,
   setSimilaritySearch,
   onUpdateFilter,
@@ -112,7 +114,7 @@ export default function SearchView({
             placeholder={
               isMobileOnly ? "Search" : "Search for a detected object..."
             }
-            value={search}
+            value={similaritySearch ? "" : search}
             onChange={(e) => setSearch(e.target.value)}
           />
           {search && (