diff --git a/docs/docs/configuration/semantic_search.md b/docs/docs/configuration/semantic_search.md index a569e8f1a..18093a479 100644 --- a/docs/docs/configuration/semantic_search.md +++ b/docs/docs/configuration/semantic_search.md @@ -53,7 +53,7 @@ semantic_search: ## Usage 1. Semantic search is used in conjunction with the other filters available on the Search page. Use a combination of traditional filtering and semantic search for the best results. -2. The comparison between text and image embedding distances generally means that results matching `description` will appear first, even if a `thumbnail` embedding may be a better match. Play with the "Search Type" filter to help find what you are looking for. -3. Make your search language and tone closely match your descriptions. If you are using thumbnail search, phrase your query as an image caption. +2. Because of how the AI models Frigate uses have been trained, the comparison between text and image embedding distances generally means that results matching `description` will appear first, even if a `thumbnail` embedding may be a better match. Play with the "Search Type" setting to help find what you are looking for. Note that if you are generating descriptions for specific objects or zones only, this may cause search results to prioritize the objects with descriptions even if the the ones without them are more relevant. +3. Make your search language and tone closely match your descriptions. If you are using thumbnail search, **phrase your query as an image caption**. For example "red car" will not work as well as "red sedan driving down a residential street on a sunny day". 4. Semantic search on thumbnails tends to return better results when matching large subjects that take up most of the frame. Small things like "cat" tend to not work well. -5. Experiment! Find a tracked object you want to test and start typing keywords to see what works for you. +5. Experiment! Find a tracked object you want to test and start typing keywords and phrases to see what works for you. diff --git a/web/src/components/overlay/dialog/SearchFilterDialog.tsx b/web/src/components/overlay/dialog/SearchFilterDialog.tsx index ad9fe1c2b..ed091b350 100644 --- a/web/src/components/overlay/dialog/SearchFilterDialog.tsx +++ b/web/src/components/overlay/dialog/SearchFilterDialog.tsx @@ -65,9 +65,7 @@ export default function SearchFilterDialog({ (currentFilter.min_score ?? 0) > 0.5 || (currentFilter.max_score ?? 1) < 1 || (currentFilter.zones?.length ?? 0) > 0 || - (currentFilter.sub_labels?.length ?? 0) > 0 || - (!currentFilter.search_type?.includes("similarity") && - (currentFilter.search_type?.length ?? 2) !== 2)), + (currentFilter.sub_labels?.length ?? 0) > 0), [currentFilter], ); @@ -115,20 +113,6 @@ export default function SearchFilterDialog({ setCurrentFilter({ ...currentFilter, min_score: min, max_score: max }) } /> - {config?.semantic_search?.enabled && - !currentFilter?.search_type?.includes("similarity") && ( - - setCurrentFilter({ - ...currentFilter, - search_type: newSearchSource, - }) - } - /> - )} {isDesktop && }
)} + {config?.semantic_search?.enabled && ( + { + setSearchSources(sources as SearchSource[]); + onUpdateFilter({ ...filter, search_type: sources }); + }} + /> + )} ); @@ -113,3 +135,65 @@ export default function SearchSettings({ /> ); } + +type SearchTypeContentProps = { + searchSources: SearchSource[] | undefined; + setSearchSources: (sources: SearchSource[] | undefined) => void; +}; +export function SearchTypeContent({ + searchSources, + setSearchSources, +}: SearchTypeContentProps) { + return ( + <> +
+ +
+
Search Source
+
+ Choose whether to search the thumbnails or descriptions of your + tracked objects. +
+
+
+ { + const updatedSources = searchSources ? [...searchSources] : []; + + if (isChecked) { + updatedSources.push("thumbnail"); + setSearchSources(updatedSources); + } else { + if (updatedSources.length > 1) { + const index = updatedSources.indexOf("thumbnail"); + if (index !== -1) updatedSources.splice(index, 1); + setSearchSources(updatedSources); + } + } + }} + /> + { + const updatedSources = searchSources ? [...searchSources] : []; + + if (isChecked) { + updatedSources.push("description"); + setSearchSources(updatedSources); + } else { + if (updatedSources.length > 1) { + const index = updatedSources.indexOf("description"); + if (index !== -1) updatedSources.splice(index, 1); + setSearchSources(updatedSources); + } + } + }} + /> +
+
+ + ); +} diff --git a/web/src/views/search/SearchView.tsx b/web/src/views/search/SearchView.tsx index 665f7a4fd..07842fed6 100644 --- a/web/src/views/search/SearchView.tsx +++ b/web/src/views/search/SearchView.tsx @@ -10,7 +10,7 @@ import { FrigateConfig } from "@/types/frigateConfig"; import { SearchFilter, SearchResult, SearchSource } from "@/types/search"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { isMobileOnly } from "react-device-detect"; -import { LuSearchX } from "react-icons/lu"; +import { LuImage, LuSearchX, LuText } from "react-icons/lu"; import useSWR from "swr"; import ExploreView from "../explore/ExploreView"; import useKeyboardListener, { @@ -23,6 +23,13 @@ import { isEqual } from "lodash"; import { formatDateToLocaleString } from "@/utils/dateUtil"; import SearchThumbnailFooter from "@/components/card/SearchThumbnailFooter"; import SearchSettings from "@/components/settings/SearchSettings"; +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from "@/components/ui/tooltip"; +import Chip from "@/components/indicators/Chip"; +import { TooltipPortal } from "@radix-ui/react-tooltip"; type SearchViewProps = { search: string; @@ -182,6 +189,21 @@ export default function SearchView({ setSelectedIndex(0); }, [searchTerm, searchFilter]); + // confidence score + + const zScoreToConfidence = (score: number) => { + // Normalizing is not needed for similarity searches + // Sigmoid function for normalized: 1 / (1 + e^x) + // Cosine for similarity + if (searchFilter) { + const notNormalized = searchFilter?.search_type?.includes("similarity"); + + const confidence = notNormalized ? 1 - score : 1 / (1 + Math.exp(score)); + + return Math.round(confidence * 100); + } + }; + // update search detail when results change useEffect(() => { @@ -351,6 +373,8 @@ export default function SearchView({ setColumns={setColumns} defaultView={defaultView} setDefaultView={setDefaultView} + filter={searchFilter} + onUpdateFilter={onUpdateFilter} /> @@ -398,6 +422,30 @@ export default function SearchView({ searchResult={value} onClick={() => onSelectSearch(value, index)} /> + {(searchTerm || + searchFilter?.search_type?.includes("similarity")) && ( +
+ + + + {value.search_source == "thumbnail" ? ( + + ) : ( + + )} + + + + + Matched {value.search_source} at{" "} + {zScoreToConfidence(value.search_distance)}% + + + +
+ )}