Adjust Explore settings (#14409)

* Re-add search source chip without confidence percentage

* add confidence to tooltip only

* move search type to settings

* padding tweak

* docs update

* docs clarity

docs/docs/configuration/semantic_search.md

@@ -53,7 +53,7 @@ semantic_search:
 ## Usage
 
 1. Semantic search is used in conjunction with the other filters available on the Search page. Use a combination of traditional filtering and semantic search for the best results.
-2. The comparison between text and image embedding distances generally means that results matching `description` will appear first, even if a `thumbnail` embedding may be a better match. Play with the "Search Type" filter to help find what you are looking for.
-3. Make your search language and tone closely match your descriptions. If you are using thumbnail search, phrase your query as an image caption.
+2. Because of how the AI models Frigate uses have been trained, the comparison between text and image embedding distances generally means that results matching `description` will appear first, even if a `thumbnail` embedding may be a better match. Play with the "Search Type" setting to help find what you are looking for. Note that if you are generating descriptions for specific objects or zones only, this may cause search results to prioritize the objects with descriptions even if the the ones without them are more relevant.
+3. Make your search language and tone closely match your descriptions. If you are using thumbnail search, **phrase your query as an image caption**. For example "red car" will not work as well as "red sedan driving down a residential street on a sunny day".
 4. Semantic search on thumbnails tends to return better results when matching large subjects that take up most of the frame. Small things like "cat" tend to not work well.
-5. Experiment! Find a tracked object you want to test and start typing keywords to see what works for you.
+5. Experiment! Find a tracked object you want to test and start typing keywords and phrases to see what works for you.

web/src/components/overlay/dialog/SearchFilterDialog.tsx

@@ -65,9 +65,7 @@ export default function SearchFilterDialog({
         (currentFilter.min_score ?? 0) > 0.5 ||
         (currentFilter.max_score ?? 1) < 1 ||
         (currentFilter.zones?.length ?? 0) > 0 ||
-        (currentFilter.sub_labels?.length ?? 0) > 0 ||
-        (!currentFilter.search_type?.includes("similarity") &&
-          (currentFilter.search_type?.length ?? 2) !== 2)),
+        (currentFilter.sub_labels?.length ?? 0) > 0),
     [currentFilter],
   );
 
@@ -115,20 +113,6 @@ export default function SearchFilterDialog({
           setCurrentFilter({ ...currentFilter, min_score: min, max_score: max })
         }
       />
-      {config?.semantic_search?.enabled &&
-        !currentFilter?.search_type?.includes("similarity") && (
-          <SearchTypeContent
-            searchSources={
-              currentFilter?.search_type ?? ["thumbnail", "description"]
-            }
-            setSearchSources={(newSearchSource) =>
-              setCurrentFilter({
-                ...currentFilter,
-                search_type: newSearchSource,
-              })
-            }
-          />
-        )}
       {isDesktop && <DropdownMenuSeparator />}
       <div className="flex items-center justify-evenly p-2">
         <Button
@@ -491,59 +475,3 @@ export function ScoreFilterContent({
     </div>
   );
 }
-
-type SearchTypeContentProps = {
-  searchSources: SearchSource[] | undefined;
-  setSearchSources: (sources: SearchSource[] | undefined) => void;
-};
-export function SearchTypeContent({
-  searchSources,
-  setSearchSources,
-}: SearchTypeContentProps) {
-  return (
-    <>
-      <div className="overflow-x-hidden">
-        <DropdownMenuSeparator className="mb-3" />
-        <div className="text-lg">Search Sources</div>
-        <div className="mt-2.5 flex flex-col gap-2.5">
-          <FilterSwitch
-            label="Thumbnail Image"
-            isChecked={searchSources?.includes("thumbnail") ?? false}
-            onCheckedChange={(isChecked) => {
-              const updatedSources = searchSources ? [...searchSources] : [];
-
-              if (isChecked) {
-                updatedSources.push("thumbnail");
-                setSearchSources(updatedSources);
-              } else {
-                if (updatedSources.length > 1) {
-                  const index = updatedSources.indexOf("thumbnail");
-                  if (index !== -1) updatedSources.splice(index, 1);
-                  setSearchSources(updatedSources);
-                }
-              }
-            }}
-          />
-          <FilterSwitch
-            label="Description"
-            isChecked={searchSources?.includes("description") ?? false}
-            onCheckedChange={(isChecked) => {
-              const updatedSources = searchSources ? [...searchSources] : [];
-
-              if (isChecked) {
-                updatedSources.push("description");
-                setSearchSources(updatedSources);
-              } else {
-                if (updatedSources.length > 1) {
-                  const index = updatedSources.indexOf("description");
-                  if (index !== -1) updatedSources.splice(index, 1);
-                  setSearchSources(updatedSources);
-                }
-              }
-            }}
-          />
-        </div>
-      </div>
-    </>
-  );
-}

web/src/components/settings/SearchSettings.tsx

@@ -13,23 +13,36 @@ import {
   SelectTrigger,
 } from "@/components/ui/select";
 import { DropdownMenuSeparator } from "../ui/dropdown-menu";
+import FilterSwitch from "../filter/FilterSwitch";
+import { SearchFilter, SearchSource } from "@/types/search";
+import useSWR from "swr";
+import { FrigateConfig } from "@/types/frigateConfig";
 
 type SearchSettingsProps = {
   className?: string;
   columns: number;
   defaultView: string;
+  filter?: SearchFilter;
   setColumns: (columns: number) => void;
   setDefaultView: (view: string) => void;
+  onUpdateFilter: (filter: SearchFilter) => void;
 };
 export default function SearchSettings({
   className,
   columns,
   setColumns,
   defaultView,
+  filter,
   setDefaultView,
+  onUpdateFilter,
 }: SearchSettingsProps) {
+  const { data: config } = useSWR<FrigateConfig>("config");
   const [open, setOpen] = useState(false);
 
+  const [searchSources, setSearchSources] = useState<SearchSource[]>([
+    "thumbnail",
+  ]);
+
   const trigger = (
     <Button className="flex items-center gap-2" size="sm">
       <FaCog className="text-secondary-foreground" />
@@ -94,6 +107,15 @@ export default function SearchSettings({
           </div>
         </>
       )}
+      {config?.semantic_search?.enabled && (
+        <SearchTypeContent
+          searchSources={searchSources}
+          setSearchSources={(sources) => {
+            setSearchSources(sources as SearchSource[]);
+            onUpdateFilter({ ...filter, search_type: sources });
+          }}
+        />
+      )}
     </div>
   );
 
@@ -113,3 +135,65 @@ export default function SearchSettings({
     />
   );
 }
+
+type SearchTypeContentProps = {
+  searchSources: SearchSource[] | undefined;
+  setSearchSources: (sources: SearchSource[] | undefined) => void;
+};
+export function SearchTypeContent({
+  searchSources,
+  setSearchSources,
+}: SearchTypeContentProps) {
+  return (
+    <>
+      <div className="overflow-x-hidden">
+        <DropdownMenuSeparator className="mb-3" />
+        <div className="space-y-0.5">
+          <div className="text-md">Search Source</div>
+          <div className="space-y-1 text-xs text-muted-foreground">
+            Choose whether to search the thumbnails or descriptions of your
+            tracked objects.
+          </div>
+        </div>
+        <div className="mt-2.5 flex flex-col gap-2.5">
+          <FilterSwitch
+            label="Thumbnail Image"
+            isChecked={searchSources?.includes("thumbnail") ?? false}
+            onCheckedChange={(isChecked) => {
+              const updatedSources = searchSources ? [...searchSources] : [];
+
+              if (isChecked) {
+                updatedSources.push("thumbnail");
+                setSearchSources(updatedSources);
+              } else {
+                if (updatedSources.length > 1) {
+                  const index = updatedSources.indexOf("thumbnail");
+                  if (index !== -1) updatedSources.splice(index, 1);
+                  setSearchSources(updatedSources);
+                }
+              }
+            }}
+          />
+          <FilterSwitch
+            label="Description"
+            isChecked={searchSources?.includes("description") ?? false}
+            onCheckedChange={(isChecked) => {
+              const updatedSources = searchSources ? [...searchSources] : [];
+
+              if (isChecked) {
+                updatedSources.push("description");
+                setSearchSources(updatedSources);
+              } else {
+                if (updatedSources.length > 1) {
+                  const index = updatedSources.indexOf("description");
+                  if (index !== -1) updatedSources.splice(index, 1);
+                  setSearchSources(updatedSources);
+                }
+              }
+            }}
+          />
+        </div>
+      </div>
+    </>
+  );
+}

web/src/views/search/SearchView.tsx

@@ -10,7 +10,7 @@ import { FrigateConfig } from "@/types/frigateConfig";
 import { SearchFilter, SearchResult, SearchSource } from "@/types/search";
 import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import { isMobileOnly } from "react-device-detect";
-import { LuSearchX } from "react-icons/lu";
+import { LuImage, LuSearchX, LuText } from "react-icons/lu";
 import useSWR from "swr";
 import ExploreView from "../explore/ExploreView";
 import useKeyboardListener, {
@@ -23,6 +23,13 @@ import { isEqual } from "lodash";
 import { formatDateToLocaleString } from "@/utils/dateUtil";
 import SearchThumbnailFooter from "@/components/card/SearchThumbnailFooter";
 import SearchSettings from "@/components/settings/SearchSettings";
+import {
+  Tooltip,
+  TooltipContent,
+  TooltipTrigger,
+} from "@/components/ui/tooltip";
+import Chip from "@/components/indicators/Chip";
+import { TooltipPortal } from "@radix-ui/react-tooltip";
 
 type SearchViewProps = {
   search: string;
@@ -182,6 +189,21 @@ export default function SearchView({
     setSelectedIndex(0);
   }, [searchTerm, searchFilter]);
 
+  // confidence score
+
+  const zScoreToConfidence = (score: number) => {
+    // Normalizing is not needed for similarity searches
+    // Sigmoid function for normalized: 1 / (1 + e^x)
+    // Cosine for similarity
+    if (searchFilter) {
+      const notNormalized = searchFilter?.search_type?.includes("similarity");
+
+      const confidence = notNormalized ? 1 - score : 1 / (1 + Math.exp(score));
+
+      return Math.round(confidence * 100);
+    }
+  };
+
   // update search detail when results change
 
   useEffect(() => {
@@ -351,6 +373,8 @@ export default function SearchView({
                 setColumns={setColumns}
                 defaultView={defaultView}
                 setDefaultView={setDefaultView}
+                filter={searchFilter}
+                onUpdateFilter={onUpdateFilter}
               />
               <ScrollBar orientation="horizontal" className="h-0" />
             </div>
@@ -398,6 +422,30 @@ export default function SearchView({
                         searchResult={value}
                         onClick={() => onSelectSearch(value, index)}
                       />
+                      {(searchTerm ||
+                        searchFilter?.search_type?.includes("similarity")) && (
+                        <div className={cn("absolute right-2 top-2 z-40")}>
+                          <Tooltip>
+                            <TooltipTrigger>
+                              <Chip
+                                className={`flex select-none items-center justify-between space-x-1 bg-gray-500 bg-gradient-to-br from-gray-400 to-gray-500 text-xs capitalize text-white`}
+                              >
+                                {value.search_source == "thumbnail" ? (
+                                  <LuImage className="size-3" />
+                                ) : (
+                                  <LuText className="size-3" />
+                                )}
+                              </Chip>
+                            </TooltipTrigger>
+                            <TooltipPortal>
+                              <TooltipContent>
+                                Matched {value.search_source} at{" "}
+                                {zScoreToConfidence(value.search_distance)}%
+                              </TooltipContent>
+                            </TooltipPortal>
+                          </Tooltip>
+                        </div>
+                      )}
                     </div>
                     <div
                       className={`review-item-ring pointer-events-none absolute inset-0 z-10 size-full rounded-lg outline outline-[3px] -outline-offset-[2.8px] ${selected ? `shadow-selected outline-selected` : "outline-transparent duration-500"}`}