import { useEmbeddingsReindexProgress, useTrackedObjectUpdate, useModelState, } from "@/api/ws"; import ActivityIndicator from "@/components/indicators/activity-indicator"; import AnimatedCircularProgressBar from "@/components/ui/circular-progress-bar"; import { useApiFilterArgs } from "@/hooks/use-api-filter"; import { useTimezone } from "@/hooks/use-date-utils"; import { usePersistence } from "@/hooks/use-persistence"; import { FrigateConfig } from "@/types/frigateConfig"; import { SearchFilter, SearchQuery, SearchResult } from "@/types/search"; import { ModelState } from "@/types/ws"; import { formatSecondsToDuration } from "@/utils/dateUtil"; import SearchView from "@/views/search/SearchView"; import { useCallback, useEffect, useMemo, useState } from "react"; import { isMobileOnly } from "react-device-detect"; import { LuCheck, LuExternalLink, LuX } from "react-icons/lu"; import { TbExclamationCircle } from "react-icons/tb"; import { Link } from "react-router-dom"; import { toast } from "sonner"; import useSWR from "swr"; import useSWRInfinite from "swr/infinite"; const API_LIMIT = 25; export default function Explore() { // search field handler const { data: config } = useSWR("config", { revalidateOnFocus: false, }); // grid const [columnCount, setColumnCount] = usePersistence("exploreGridColumns", 4); const gridColumns = useMemo(() => { if (isMobileOnly) { return 2; } return columnCount ?? 4; }, [columnCount]); // default layout const [defaultView, setDefaultView, defaultViewLoaded] = usePersistence( "exploreDefaultView", "summary", ); const timezone = useTimezone(config); const [search, setSearch] = useState(""); const [searchFilter, setSearchFilter, searchSearchParams] = useApiFilterArgs(); const searchTerm = useMemo( () => searchSearchParams?.["query"] || "", [searchSearchParams], ); const similaritySearch = useMemo( () => searchSearchParams["search_type"] == "similarity", [searchSearchParams], ); useEffect(() => { if (!searchTerm && !search) { return; } // switch back to normal search when query is entered setSearchFilter({ ...searchFilter, search_type: similaritySearch && search ? undefined : searchFilter?.search_type, event_id: similaritySearch && search ? undefined : searchFilter?.event_id, query: search.length > 0 ? search : undefined, }); // only update when search is updated // eslint-disable-next-line react-hooks/exhaustive-deps }, [search]); const searchQuery: SearchQuery = useMemo(() => { // no search parameters if (searchSearchParams && Object.keys(searchSearchParams).length === 0) { if (defaultView == "grid") { return ["events", {}]; } else { return null; } } // parameters, but no search term and not similarity if ( searchSearchParams && Object.keys(searchSearchParams).length !== 0 && !searchTerm && !similaritySearch ) { return [ "events", { cameras: searchSearchParams["cameras"], labels: searchSearchParams["labels"], sub_labels: searchSearchParams["sub_labels"], zones: searchSearchParams["zones"], before: searchSearchParams["before"], after: searchSearchParams["after"], time_range: searchSearchParams["time_range"], search_type: searchSearchParams["search_type"], min_score: searchSearchParams["min_score"], max_score: searchSearchParams["max_score"], has_snapshot: searchSearchParams["has_snapshot"], is_submitted: searchSearchParams["is_submitted"], has_clip: searchSearchParams["has_clip"], event_id: searchSearchParams["event_id"], sort: searchSearchParams["sort"], limit: Object.keys(searchSearchParams).length == 0 ? API_LIMIT : undefined, timezone, in_progress: 0, include_thumbnails: 0, }, ]; } // parameters and search term if (!similaritySearch) { setSearch(searchTerm); } return [ "events/search", { query: similaritySearch ? undefined : searchTerm, cameras: searchSearchParams["cameras"], labels: searchSearchParams["labels"], sub_labels: searchSearchParams["sub_labels"], zones: searchSearchParams["zones"], before: searchSearchParams["before"], after: searchSearchParams["after"], time_range: searchSearchParams["time_range"], search_type: searchSearchParams["search_type"], min_score: searchSearchParams["min_score"], max_score: searchSearchParams["max_score"], has_snapshot: searchSearchParams["has_snapshot"], is_submitted: searchSearchParams["is_submitted"], has_clip: searchSearchParams["has_clip"], event_id: searchSearchParams["event_id"], sort: searchSearchParams["sort"], timezone, include_thumbnails: 0, }, ]; }, [searchTerm, searchSearchParams, similaritySearch, timezone, defaultView]); // paging const getKey = ( pageIndex: number, previousPageData: SearchResult[] | null, ): SearchQuery => { if (previousPageData && !previousPageData.length) return null; // reached the end if (!searchQuery) return null; const [url, params] = searchQuery; const isAscending = params.sort?.includes("date_asc"); if (pageIndex > 0 && previousPageData) { const lastDate = previousPageData[previousPageData.length - 1].start_time; return [ url, { ...params, [isAscending ? "after" : "before"]: lastDate.toString(), limit: API_LIMIT, }, ]; } // For the first page, use the original params return [url, { ...params, limit: API_LIMIT }]; }; const { data, size, setSize, isValidating, mutate } = useSWRInfinite< SearchResult[] >(getKey, { revalidateFirstPage: true, revalidateOnFocus: true, revalidateAll: false, onError: (error) => { toast.error( `Error fetching tracked objects: ${error.response.data.message}`, { position: "top-center", }, ); if (error.response.status === 404) { // reset all filters if 404 setSearchFilter({}); } }, }); const searchResults = useMemo( () => (data ? ([] as SearchResult[]).concat(...data) : []), [data], ); const isLoadingInitialData = !data && !isValidating; const isLoadingMore = isLoadingInitialData || (size > 0 && data && typeof data[size - 1] === "undefined"); const isEmpty = data?.[0]?.length === 0; const isReachingEnd = isEmpty || (data && data[data.length - 1]?.length < API_LIMIT); const loadMore = useCallback(() => { if (!isReachingEnd && !isLoadingMore) { if (searchQuery) { const [url] = searchQuery; // for embeddings, only load 100 results for description and similarity if (url === "events/search" && searchResults.length >= 100) { return; } } setSize(size + 1); } }, [isReachingEnd, isLoadingMore, setSize, size, searchResults, searchQuery]); // mutation and revalidation const trackedObjectUpdate = useTrackedObjectUpdate(); useEffect(() => { if (trackedObjectUpdate) { mutate(); } // mutate / revalidate when event description updates come in // eslint-disable-next-line react-hooks/exhaustive-deps }, [trackedObjectUpdate]); // embeddings reindex progress const { payload: reindexState } = useEmbeddingsReindexProgress(); const embeddingsReindexing = useMemo(() => { if (reindexState) { switch (reindexState.status) { case "indexing": return true; case "completed": return false; default: return undefined; } } }, [reindexState]); // model states const { payload: textModelState } = useModelState( "jinaai/jina-clip-v1-text_model_fp16.onnx", ); const { payload: textTokenizerState } = useModelState( "jinaai/jina-clip-v1-tokenizer", ); const modelFile = config?.semantic_search.model_size === "large" ? "jinaai/jina-clip-v1-vision_model_fp16.onnx" : "jinaai/jina-clip-v1-vision_model_quantized.onnx"; const { payload: visionModelState } = useModelState(modelFile); const { payload: visionFeatureExtractorState } = useModelState( "jinaai/jina-clip-v1-preprocessor_config.json", ); const allModelsLoaded = useMemo(() => { return ( textModelState === "downloaded" && textTokenizerState === "downloaded" && visionModelState === "downloaded" && visionFeatureExtractorState === "downloaded" ); }, [ textModelState, textTokenizerState, visionModelState, visionFeatureExtractorState, ]); const renderModelStateIcon = (modelState: ModelState) => { if (modelState === "downloading") { return ; } if (modelState === "downloaded") { return ; } if (modelState === "not_downloaded" || modelState === "error") { return ; } return null; }; if ( !defaultViewLoaded || (config?.semantic_search.enabled && (!reindexState || !textModelState || !textTokenizerState || !visionModelState || !visionFeatureExtractorState)) ) { return ( ); } return ( <> {config?.semantic_search.enabled && (!allModelsLoaded || embeddingsReindexing) ? (
Explore is Unavailable
{embeddingsReindexing && allModelsLoaded && ( <>
Explore can be used after tracked object embeddings have finished reindexing.
{reindexState.time_remaining !== null && (
{reindexState.time_remaining === -1 ? "Starting up..." : "Estimated time remaining:"}
{reindexState.time_remaining >= 0 && (formatSecondsToDuration(reindexState.time_remaining) || "Finishing shortly")}
)}
Thumbnails embedded: {reindexState.thumbnails}
Descriptions embedded: {reindexState.descriptions}
Tracked objects processed: {reindexState.processed_objects} /{" "} {reindexState.total_objects}
)} {!allModelsLoaded && ( <>
Frigate is downloading the necessary embeddings models to support the Semantic Search feature. This may take several minutes depending on the speed of your network connection.
{renderModelStateIcon(visionModelState)} Vision model
{renderModelStateIcon(visionFeatureExtractorState)} Vision model feature extractor
{renderModelStateIcon(textModelState)} Text model
{renderModelStateIcon(textTokenizerState)} Text tokenizer
{(textModelState === "error" || textTokenizerState === "error" || visionModelState === "error" || visionFeatureExtractorState === "error") && (
An error has occurred. Check Frigate logs.
)}
You may want to reindex the embeddings of your tracked objects once the models are downloaded.
Read the documentation{" "}
)}
) : ( { setSearchFilter({ ...searchFilter, search_type: ["similarity"], event_id: search.id, }); }} setSearchFilter={setSearchFilter} onUpdateFilter={setSearchFilter} setColumns={setColumnCount} setDefaultView={setDefaultView} loadMore={loadMore} refresh={mutate} /> )} ); }