import { useEventUpdate, useModelState } from "@/api/ws"; import ActivityIndicator from "@/components/indicators/activity-indicator"; import { useApiFilterArgs } from "@/hooks/use-api-filter"; import { useTimezone } from "@/hooks/use-date-utils"; import { FrigateConfig } from "@/types/frigateConfig"; import { SearchFilter, SearchQuery, SearchResult } from "@/types/search"; import { ModelState } from "@/types/ws"; import SearchView from "@/views/search/SearchView"; import { useCallback, useEffect, useMemo, useState } from "react"; import { LuCheck, LuExternalLink, LuX } from "react-icons/lu"; import { TbExclamationCircle } from "react-icons/tb"; import { Link } from "react-router-dom"; import useSWR from "swr"; import useSWRInfinite from "swr/infinite"; const API_LIMIT = 25; export default function Explore() { // search field handler const { data: config } = useSWR("config", { revalidateOnFocus: false, }); const timezone = useTimezone(config); const [search, setSearch] = useState(""); const [searchFilter, setSearchFilter, searchSearchParams] = useApiFilterArgs(); const searchTerm = useMemo( () => searchSearchParams?.["query"] || "", [searchSearchParams], ); const similaritySearch = useMemo( () => searchSearchParams["search_type"] == "similarity", [searchSearchParams], ); useEffect(() => { if (!searchTerm && !search) { return; } // switch back to normal search when query is entered setSearchFilter({ ...searchFilter, search_type: similaritySearch && search ? undefined : searchFilter?.search_type, event_id: similaritySearch && search ? undefined : searchFilter?.event_id, query: search.length > 0 ? search : undefined, }); // only update when search is updated // eslint-disable-next-line react-hooks/exhaustive-deps }, [search]); const searchQuery: SearchQuery = useMemo(() => { // no search parameters if (searchSearchParams && Object.keys(searchSearchParams).length === 0) { return null; } // parameters, but no search term and not similarity if ( searchSearchParams && Object.keys(searchSearchParams).length !== 0 && !searchTerm && !similaritySearch ) { return [ "events", { cameras: searchSearchParams["cameras"], labels: searchSearchParams["labels"], sub_labels: searchSearchParams["sub_labels"], zones: searchSearchParams["zones"], before: searchSearchParams["before"], after: searchSearchParams["after"], time_range: searchSearchParams["time_range"], search_type: searchSearchParams["search_type"], limit: Object.keys(searchSearchParams).length == 0 ? API_LIMIT : undefined, timezone, in_progress: 0, include_thumbnails: 0, }, ]; } // parameters and search term if (!similaritySearch) { setSearch(searchTerm); } return [ "events/search", { query: similaritySearch ? undefined : searchTerm, cameras: searchSearchParams["cameras"], labels: searchSearchParams["labels"], sub_labels: searchSearchParams["sub_labels"], zones: searchSearchParams["zones"], before: searchSearchParams["before"], after: searchSearchParams["after"], time_range: searchSearchParams["time_range"], search_type: searchSearchParams["search_type"], event_id: searchSearchParams["event_id"], timezone, include_thumbnails: 0, }, ]; }, [searchTerm, searchSearchParams, similaritySearch, timezone]); // paging const getKey = ( pageIndex: number, previousPageData: SearchResult[] | null, ): SearchQuery => { if (previousPageData && !previousPageData.length) return null; // reached the end if (!searchQuery) return null; const [url, params] = searchQuery; // If it's not the first page, use the last item's start_time as the 'before' parameter if (pageIndex > 0 && previousPageData) { const lastDate = previousPageData[previousPageData.length - 1].start_time; return [ url, { ...params, before: lastDate.toString(), limit: API_LIMIT }, ]; } // For the first page, use the original params return [url, { ...params, limit: API_LIMIT }]; }; const { data, size, setSize, isValidating, mutate } = useSWRInfinite< SearchResult[] >(getKey, { revalidateFirstPage: true, revalidateOnFocus: true, revalidateAll: false, }); const searchResults = useMemo( () => (data ? ([] as SearchResult[]).concat(...data) : []), [data], ); const isLoadingInitialData = !data && !isValidating; const isLoadingMore = isLoadingInitialData || (size > 0 && data && typeof data[size - 1] === "undefined"); const isEmpty = data?.[0]?.length === 0; const isReachingEnd = isEmpty || (data && data[data.length - 1]?.length < API_LIMIT); const loadMore = useCallback(() => { if (!isReachingEnd && !isLoadingMore) { if (searchQuery) { const [url] = searchQuery; // for embeddings, only load 100 results for description and similarity if (url === "events/search" && searchResults.length >= 100) { return; } } setSize(size + 1); } }, [isReachingEnd, isLoadingMore, setSize, size, searchResults, searchQuery]); // mutation and revalidation const eventUpdate = useEventUpdate(); useEffect(() => { mutate(); // mutate / revalidate when event description updates come in // eslint-disable-next-line react-hooks/exhaustive-deps }, [eventUpdate]); // model states const { payload: minilmModelState } = useModelState( "sentence-transformers/all-MiniLM-L6-v2-model.onnx", ); const { payload: minilmTokenizerState } = useModelState( "sentence-transformers/all-MiniLM-L6-v2-tokenizer", ); const { payload: clipImageModelState } = useModelState( "clip-clip_image_model_vitb32.onnx", ); const { payload: clipTextModelState } = useModelState( "clip-clip_text_model_vitb32.onnx", ); const allModelsLoaded = useMemo(() => { return ( minilmModelState === "downloaded" && minilmTokenizerState === "downloaded" && clipImageModelState === "downloaded" && clipTextModelState === "downloaded" ); }, [ minilmModelState, minilmTokenizerState, clipImageModelState, clipTextModelState, ]); const renderModelStateIcon = (modelState: ModelState) => { if (modelState === "downloading") { return ; } if (modelState === "downloaded") { return ; } if (modelState === "not_downloaded" || modelState === "error") { return ; } return null; }; if ( !minilmModelState || !minilmTokenizerState || !clipImageModelState || !clipTextModelState ) { return ( ); } return ( <> {!allModelsLoaded ? (
Search Unavailable
Frigate is downloading the necessary embeddings models to support semantic searching. This may take several minutes depending on the speed of your network connection.
{renderModelStateIcon(clipImageModelState)} CLIP image model
{renderModelStateIcon(clipTextModelState)} CLIP text model
{renderModelStateIcon(minilmModelState)} MiniLM sentence model
{renderModelStateIcon(minilmTokenizerState)} MiniLM tokenizer
{(minilmModelState === "error" || clipImageModelState === "error" || clipTextModelState === "error") && (
An error has occurred. Check Frigate logs.
)}
You may want to reindex the embeddings of your tracked objects once the models are downloaded.
Read the documentation{" "}
) : ( { setSearchFilter({ ...searchFilter, search_type: ["similarity"], event_id: search.id, }); }} setSearchFilter={setSearchFilter} onUpdateFilter={setSearchFilter} loadMore={loadMore} hasMore={!isReachingEnd} /> )} ); }