From 8173cd77761fb740b2f19d5918d9c5145990b821 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Thu, 17 Oct 2024 06:30:52 -0500 Subject: [PATCH] Add score filter to Explore view (#14397) * backend score filtering and sorting * score filter frontend * use input for score filtering * use correct score on search thumbnail * add popover to explain top_score * revert sublabel score calc * update filters logic * fix rounding on score * wait until default view is loaded * don't turn button to selected style for similarity searches * clarify language * fix alert dialog buttons to use correct destructive variant * use root level top_score for very old events * better arrangement of thumbnail footer items on smaller screens --- frigate/api/defs/events_query_parameters.py | 3 + frigate/api/event.py | 23 ++- web/src/components/card/ReviewCard.tsx | 11 +- web/src/components/card/SearchThumbnail.tsx | 6 +- .../components/card/SearchThumbnailFooter.tsx | 188 ++++++++++-------- .../components/filter/CameraGroupSelector.tsx | 7 +- .../components/filter/ReviewActionGroup.tsx | 7 +- .../components/input/DeleteSearchDialog.tsx | 3 +- web/src/components/input/InputWithTags.tsx | 47 ++++- .../overlay/detail/SearchDetailDialog.tsx | 27 ++- .../overlay/dialog/SearchFilterDialog.tsx | 74 ++++++- web/src/components/settings/PolygonItem.tsx | 6 +- .../components/settings/SearchSettings.tsx | 48 +++-- web/src/pages/Explore.tsx | 27 ++- web/src/types/search.ts | 5 + web/src/views/search/SearchView.tsx | 7 +- 16 files changed, 353 insertions(+), 136 deletions(-) diff --git a/frigate/api/defs/events_query_parameters.py b/frigate/api/defs/events_query_parameters.py index 02bbc31ea..c4e40bd4e 100644 --- a/frigate/api/defs/events_query_parameters.py +++ b/frigate/api/defs/events_query_parameters.py @@ -45,6 +45,9 @@ class EventsSearchQueryParams(BaseModel): before: Optional[float] = None time_range: Optional[str] = DEFAULT_TIME_RANGE timezone: Optional[str] = "utc" + min_score: Optional[float] = None + max_score: Optional[float] = None + sort: Optional[str] = None class EventsSummaryQueryParams(BaseModel): diff --git a/frigate/api/event.py b/frigate/api/event.py index c716bba13..892624e53 100644 --- a/frigate/api/event.py +++ b/frigate/api/event.py @@ -348,6 +348,7 @@ def events_search(request: Request, params: EventsSearchQueryParams = Depends()) search_type = params.search_type include_thumbnails = params.include_thumbnails limit = params.limit + sort = params.sort # Filters cameras = params.cameras @@ -355,6 +356,8 @@ def events_search(request: Request, params: EventsSearchQueryParams = Depends()) zones = params.zones after = params.after before = params.before + min_score = params.min_score + max_score = params.max_score time_range = params.time_range # for similarity search @@ -430,6 +433,14 @@ def events_search(request: Request, params: EventsSearchQueryParams = Depends()) if before: event_filters.append((Event.start_time < before)) + if min_score is not None and max_score is not None: + event_filters.append((Event.data["score"].between(min_score, max_score))) + else: + if min_score is not None: + event_filters.append((Event.data["score"] >= min_score)) + if max_score is not None: + event_filters.append((Event.data["score"] <= max_score)) + if time_range != DEFAULT_TIME_RANGE: tz_name = params.timezone hour_modifier, minute_modifier, _ = get_tz_modifiers(tz_name) @@ -554,11 +565,19 @@ def events_search(request: Request, params: EventsSearchQueryParams = Depends()) processed_events.append(processed_event) - # Sort by search distance if search_results are available, otherwise by start_time + # Sort by search distance if search_results are available, otherwise by start_time as default if search_results: processed_events.sort(key=lambda x: x.get("search_distance", float("inf"))) else: - processed_events.sort(key=lambda x: x["start_time"], reverse=True) + if sort == "score_asc": + processed_events.sort(key=lambda x: x["score"]) + elif sort == "score_desc": + processed_events.sort(key=lambda x: x["score"], reverse=True) + elif sort == "date_asc": + processed_events.sort(key=lambda x: x["start_time"]) + else: + # "date_desc" default + processed_events.sort(key=lambda x: x["start_time"], reverse=True) # Limit the number of events returned processed_events = processed_events[:limit] diff --git a/web/src/components/card/ReviewCard.tsx b/web/src/components/card/ReviewCard.tsx index a28b89783..e10e009fb 100644 --- a/web/src/components/card/ReviewCard.tsx +++ b/web/src/components/card/ReviewCard.tsx @@ -34,6 +34,7 @@ import { toast } from "sonner"; import useKeyboardListener from "@/hooks/use-keyboard-listener"; import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip"; import { capitalizeFirstLetter } from "@/utils/stringUtil"; +import { buttonVariants } from "../ui/button"; type ReviewCardProps = { event: ReviewSegment; @@ -228,7 +229,10 @@ export default function ReviewCard({ setOptionsOpen(false)}> Cancel - + Delete @@ -295,7 +299,10 @@ export default function ReviewCard({ setOptionsOpen(false)}> Cancel - + Delete diff --git a/web/src/components/card/SearchThumbnail.tsx b/web/src/components/card/SearchThumbnail.tsx index 4ad7d7c5c..72a560deb 100644 --- a/web/src/components/card/SearchThumbnail.tsx +++ b/web/src/components/card/SearchThumbnail.tsx @@ -90,8 +90,10 @@ export default function SearchThumbnail({ onClick={() => onClick(searchResult)} > {getIconForLabel(objectLabel, "size-3 text-white")} - {Math.floor( - searchResult.score ?? searchResult.data.top_score * 100, + {Math.round( + (searchResult.data.score ?? + searchResult.data.top_score ?? + searchResult.top_score) * 100, )} % diff --git a/web/src/components/card/SearchThumbnailFooter.tsx b/web/src/components/card/SearchThumbnailFooter.tsx index f66aca516..b21361b18 100644 --- a/web/src/components/card/SearchThumbnailFooter.tsx +++ b/web/src/components/card/SearchThumbnailFooter.tsx @@ -32,9 +32,12 @@ import axios from "axios"; import { toast } from "sonner"; import { MdImageSearch } from "react-icons/md"; import { isMobileOnly } from "react-device-detect"; +import { buttonVariants } from "../ui/button"; +import { cn } from "@/lib/utils"; type SearchThumbnailProps = { searchResult: SearchResult; + columns: number; findSimilar: () => void; refreshResults: () => void; showObjectLifecycle: () => void; @@ -42,6 +45,7 @@ type SearchThumbnailProps = { export default function SearchThumbnailFooter({ searchResult, + columns, findSimilar, refreshResults, showObjectLifecycle, @@ -95,7 +99,7 @@ export default function SearchThumbnailFooter({ Cancel Delete @@ -113,104 +117,112 @@ export default function SearchThumbnailFooter({ }} /> -
- {searchResult.end_time ? ( - - ) : ( -
- -
+
4 && + "items-start sm:flex-col sm:gap-2 lg:flex-row lg:items-center lg:gap-1", )} - {formattedDate} -
-
- {!isMobileOnly && - config?.plus?.enabled && - searchResult.has_snapshot && - searchResult.end_time && - !searchResult.plus_id && ( + > +
+ {searchResult.end_time ? ( + + ) : ( +
+ +
+ )} + {formattedDate} +
+
+ {!isMobileOnly && + config?.plus?.enabled && + searchResult.has_snapshot && + searchResult.end_time && + !searchResult.plus_id && ( + + + setShowFrigatePlus(true)} + /> + + Submit to Frigate+ + + )} + + {config?.semantic_search?.enabled && ( - setShowFrigatePlus(true)} + onClick={findSimilar} /> - Submit to Frigate+ + Find similar )} - {config?.semantic_search?.enabled && ( - - - - - Find similar - - )} - - - - - - - {searchResult.has_clip && ( - - - - Download video - - - )} - {searchResult.has_snapshot && ( - - - - Download snapshot - - - )} - - - View object lifecycle - - - {isMobileOnly && - config?.plus?.enabled && - searchResult.has_snapshot && - searchResult.end_time && - !searchResult.plus_id && ( - setShowFrigatePlus(true)} - > - - Submit to Frigate+ + + + + + + {searchResult.has_clip && ( + + + + Download video + )} - setDeleteDialogOpen(true)} - > - - Delete - - - + {searchResult.has_snapshot && ( + + + + Download snapshot + + + )} + + + View object lifecycle + + + {isMobileOnly && + config?.plus?.enabled && + searchResult.has_snapshot && + searchResult.end_time && + !searchResult.plus_id && ( + setShowFrigatePlus(true)} + > + + Submit to Frigate+ + + )} + setDeleteDialogOpen(true)} + > + + Delete + + + +
); diff --git a/web/src/components/filter/CameraGroupSelector.tsx b/web/src/components/filter/CameraGroupSelector.tsx index d63fcd9bf..28568514e 100644 --- a/web/src/components/filter/CameraGroupSelector.tsx +++ b/web/src/components/filter/CameraGroupSelector.tsx @@ -3,7 +3,7 @@ import { isDesktop, isMobile } from "react-device-detect"; import useSWR from "swr"; import { MdHome } from "react-icons/md"; import { usePersistedOverlayState } from "@/hooks/use-overlay-state"; -import { Button } from "../ui/button"; +import { Button, buttonVariants } from "../ui/button"; import { useCallback, useMemo, useState } from "react"; import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip"; import { LuPencil, LuPlus } from "react-icons/lu"; @@ -518,7 +518,10 @@ export function CameraGroupRow({ Cancel - + Delete diff --git a/web/src/components/filter/ReviewActionGroup.tsx b/web/src/components/filter/ReviewActionGroup.tsx index c637b1e35..f2d67efd7 100644 --- a/web/src/components/filter/ReviewActionGroup.tsx +++ b/web/src/components/filter/ReviewActionGroup.tsx @@ -1,7 +1,7 @@ import { FaCircleCheck } from "react-icons/fa6"; import { useCallback, useState } from "react"; import axios from "axios"; -import { Button } from "../ui/button"; +import { Button, buttonVariants } from "../ui/button"; import { isDesktop } from "react-device-detect"; import { FaCompactDisc } from "react-icons/fa"; import { HiTrash } from "react-icons/hi"; @@ -79,7 +79,10 @@ export default function ReviewActionGroup({ Cancel - + Delete diff --git a/web/src/components/input/DeleteSearchDialog.tsx b/web/src/components/input/DeleteSearchDialog.tsx index 0aaabdde5..735f52b26 100644 --- a/web/src/components/input/DeleteSearchDialog.tsx +++ b/web/src/components/input/DeleteSearchDialog.tsx @@ -8,6 +8,7 @@ import { AlertDialogHeader, AlertDialogTitle, } from "@/components/ui/alert-dialog"; +import { buttonVariants } from "../ui/button"; type DeleteSearchDialogProps = { isOpen: boolean; @@ -35,7 +36,7 @@ export function DeleteSearchDialog({ Cancel Delete diff --git a/web/src/components/input/InputWithTags.tsx b/web/src/components/input/InputWithTags.tsx index 9ca1e4093..45dfd6c32 100644 --- a/web/src/components/input/InputWithTags.tsx +++ b/web/src/components/input/InputWithTags.tsx @@ -201,10 +201,13 @@ export default function InputWithTags({ allSuggestions[type as FilterType]?.includes(value) || type == "before" || type == "after" || - type == "time_range" + type == "time_range" || + type == "min_score" || + type == "max_score" ) { const newFilters = { ...filters }; let timestamp = 0; + let score = 0; switch (type) { case "before": @@ -244,6 +247,40 @@ export default function InputWithTags({ newFilters[type] = timestamp / 1000; } break; + case "min_score": + case "max_score": + score = parseInt(value); + if (score >= 0) { + // Check for conflicts between min_score and max_score + if ( + type === "min_score" && + filters.max_score !== undefined && + score > filters.max_score * 100 + ) { + toast.error( + "The 'min_score' must be less than or equal to the 'max_score'.", + { + position: "top-center", + }, + ); + return; + } + if ( + type === "max_score" && + filters.min_score !== undefined && + score < filters.min_score * 100 + ) { + toast.error( + "The 'max_score' must be greater than or equal to the 'min_score'.", + { + position: "top-center", + }, + ); + return; + } + newFilters[type] = score / 100; + } + break; case "time_range": newFilters[type] = value; break; @@ -302,6 +339,8 @@ export default function InputWithTags({ } - ${ config?.ui.time_format === "24hour" ? endTime : convertTo12Hour(endTime) }`; + } else if (filterType === "min_score" || filterType === "max_score") { + return Math.round(Number(filterValues) * 100).toString() + "%"; } else { return filterValues as string; } @@ -320,7 +359,11 @@ export default function InputWithTags({ isValidTimeRange( trimmedValue.replace("-", ","), config?.ui.time_format, - )) + )) || + ((filterType === "min_score" || filterType === "max_score") && + !isNaN(Number(trimmedValue)) && + Number(trimmedValue) >= 50 && + Number(trimmedValue) <= 100) ) { createFilter( filterType, diff --git a/web/src/components/overlay/detail/SearchDetailDialog.tsx b/web/src/components/overlay/detail/SearchDetailDialog.tsx index 37813645b..fe150bd56 100644 --- a/web/src/components/overlay/detail/SearchDetailDialog.tsx +++ b/web/src/components/overlay/detail/SearchDetailDialog.tsx @@ -62,6 +62,12 @@ import { Card, CardContent } from "@/components/ui/card"; import useImageLoaded from "@/hooks/use-image-loaded"; import ImageLoadingIndicator from "@/components/indicators/ImageLoadingIndicator"; import { GenericVideoPlayer } from "@/components/player/GenericVideoPlayer"; +import { + Popover, + PopoverContent, + PopoverTrigger, +} from "@/components/ui/popover"; +import { LuInfo } from "react-icons/lu"; const SEARCH_TABS = [ "details", @@ -279,7 +285,7 @@ function ObjectDetailsTab({ return 0; } - const value = search.score ?? search.data.top_score; + const value = search.data.top_score; return Math.round(value * 100); }, [search]); @@ -369,7 +375,24 @@ function ObjectDetailsTab({
-
Score
+
+
+ Top Score + + +
+ + Info +
+
+ + The top score is the highest median score for the tracked + object, so this may differ from the score shown on the + search result thumbnail. + +
+
+
{score}%{subLabelScore && ` (${subLabelScore}%)`}
diff --git a/web/src/components/overlay/dialog/SearchFilterDialog.tsx b/web/src/components/overlay/dialog/SearchFilterDialog.tsx index f1fd4c676..ad9fe1c2b 100644 --- a/web/src/components/overlay/dialog/SearchFilterDialog.tsx +++ b/web/src/components/overlay/dialog/SearchFilterDialog.tsx @@ -23,6 +23,8 @@ import { Switch } from "@/components/ui/switch"; import { Label } from "@/components/ui/label"; import { DropdownMenuSeparator } from "@/components/ui/dropdown-menu"; import { cn } from "@/lib/utils"; +import { DualThumbSlider } from "@/components/ui/slider"; +import { Input } from "@/components/ui/input"; type SearchFilterDialogProps = { config?: FrigateConfig; @@ -46,6 +48,12 @@ export default function SearchFilterDialog({ const [currentFilter, setCurrentFilter] = useState(filter ?? {}); const { data: allSubLabels } = useSWR(["sub_labels", { split_joined: 1 }]); + useEffect(() => { + if (filter) { + setCurrentFilter(filter); + } + }, [filter]); + // state const [open, setOpen] = useState(false); @@ -54,9 +62,12 @@ export default function SearchFilterDialog({ () => currentFilter && (currentFilter.time_range || + (currentFilter.min_score ?? 0) > 0.5 || + (currentFilter.max_score ?? 1) < 1 || (currentFilter.zones?.length ?? 0) > 0 || (currentFilter.sub_labels?.length ?? 0) > 0 || - (currentFilter.search_type?.length ?? 2) !== 2), + (!currentFilter.search_type?.includes("similarity") && + (currentFilter.search_type?.length ?? 2) !== 2)), [currentFilter], ); @@ -97,6 +108,13 @@ export default function SearchFilterDialog({ setCurrentFilter({ ...currentFilter, sub_labels: newSubLabels }) } /> + + setCurrentFilter({ ...currentFilter, min_score: min, max_score: max }) + } + /> {config?.semantic_search?.enabled && !currentFilter?.search_type?.includes("similarity") && ( @@ -420,6 +440,58 @@ export function SubFilterContent({ ); } +type ScoreFilterContentProps = { + minScore: number | undefined; + maxScore: number | undefined; + setScoreRange: (min: number | undefined, max: number | undefined) => void; +}; +export function ScoreFilterContent({ + minScore, + maxScore, + setScoreRange, +}: ScoreFilterContentProps) { + return ( +
+ +
Score
+
+ { + const value = e.target.value; + + if (value) { + setScoreRange(parseInt(value) / 100.0, maxScore ?? 1.0); + } + }} + /> + setScoreRange(min, max)} + /> + { + const value = e.target.value; + + if (value) { + setScoreRange(minScore ?? 0.5, parseInt(value) / 100.0); + } + }} + /> +
+
+ ); +} + type SearchTypeContentProps = { searchSources: SearchSource[] | undefined; setSearchSources: (sources: SearchSource[] | undefined) => void; diff --git a/web/src/components/settings/PolygonItem.tsx b/web/src/components/settings/PolygonItem.tsx index 68aa89978..bc1db92c3 100644 --- a/web/src/components/settings/PolygonItem.tsx +++ b/web/src/components/settings/PolygonItem.tsx @@ -35,6 +35,7 @@ import { FrigateConfig } from "@/types/frigateConfig"; import { reviewQueries } from "@/utils/zoneEdutUtil"; import IconWrapper from "../ui/icon-wrapper"; import { StatusBarMessagesContext } from "@/context/statusbar-provider"; +import { buttonVariants } from "../ui/button"; type PolygonItemProps = { polygon: Polygon; @@ -257,7 +258,10 @@ export default function PolygonItem({ Cancel - + Delete diff --git a/web/src/components/settings/SearchSettings.tsx b/web/src/components/settings/SearchSettings.tsx index d1c0856ef..b3a1e89d3 100644 --- a/web/src/components/settings/SearchSettings.tsx +++ b/web/src/components/settings/SearchSettings.tsx @@ -1,6 +1,6 @@ import { Button } from "../ui/button"; import { useState } from "react"; -import { isDesktop } from "react-device-detect"; +import { isDesktop, isMobileOnly } from "react-device-detect"; import { cn } from "@/lib/utils"; import PlatformAwareDialog from "../overlay/dialog/PlatformAwareDialog"; import { FaCog } from "react-icons/fa"; @@ -40,7 +40,7 @@ export default function SearchSettings({
-
Default Search View
+
Default View
When no filters are selected, display a summary of the most recent tracked objects per label, or display an unfiltered grid. @@ -68,26 +68,32 @@ export default function SearchSettings({
- -
-
-
Grid Columns
-
- Select the number of columns in the results grid. + {!isMobileOnly && ( + <> + +
+
+
Grid Columns
+
+ Select the number of columns in the grid view. +
+
+
+ setColumns(value)} + max={6} + min={2} + step={1} + className="flex-grow" + /> + + {columns} + +
-
-
- setColumns(value)} - max={6} - min={2} - step={1} - className="flex-grow" - /> - {columns} -
-
+ + )}
); diff --git a/web/src/pages/Explore.tsx b/web/src/pages/Explore.tsx index 770b45cb8..202b079a6 100644 --- a/web/src/pages/Explore.tsx +++ b/web/src/pages/Explore.tsx @@ -14,6 +14,7 @@ import { ModelState } from "@/types/ws"; import { formatSecondsToDuration } from "@/utils/dateUtil"; import SearchView from "@/views/search/SearchView"; import { useCallback, useEffect, useMemo, useState } from "react"; +import { isMobileOnly } from "react-device-detect"; import { LuCheck, LuExternalLink, LuX } from "react-icons/lu"; import { TbExclamationCircle } from "react-icons/tb"; import { Link } from "react-router-dom"; @@ -32,11 +33,16 @@ export default function Explore() { // grid const [columnCount, setColumnCount] = usePersistence("exploreGridColumns", 4); - const gridColumns = useMemo(() => columnCount ?? 4, [columnCount]); + const gridColumns = useMemo(() => { + if (isMobileOnly) { + return 2; + } + return columnCount ?? 4; + }, [columnCount]); // default layout - const [defaultView, setDefaultView] = usePersistence( + const [defaultView, setDefaultView, defaultViewLoaded] = usePersistence( "exploreDefaultView", "summary", ); @@ -103,6 +109,8 @@ export default function Explore() { after: searchSearchParams["after"], time_range: searchSearchParams["time_range"], search_type: searchSearchParams["search_type"], + min_score: searchSearchParams["min_score"], + max_score: searchSearchParams["max_score"], limit: Object.keys(searchSearchParams).length == 0 ? API_LIMIT : undefined, timezone, @@ -129,6 +137,8 @@ export default function Explore() { after: searchSearchParams["after"], time_range: searchSearchParams["time_range"], search_type: searchSearchParams["search_type"], + min_score: searchSearchParams["min_score"], + max_score: searchSearchParams["max_score"], event_id: searchSearchParams["event_id"], timezone, include_thumbnails: 0, @@ -270,12 +280,13 @@ export default function Explore() { }; if ( - config?.semantic_search.enabled && - (!reindexState || - !textModelState || - !textTokenizerState || - !visionModelState || - !visionFeatureExtractorState) + !defaultViewLoaded || + (config?.semantic_search.enabled && + (!reindexState || + !textModelState || + !textTokenizerState || + !visionModelState || + !visionFeatureExtractorState)) ) { return ( diff --git a/web/src/types/search.ts b/web/src/types/search.ts index 50521878e..d7f3fb97d 100644 --- a/web/src/types/search.ts +++ b/web/src/types/search.ts @@ -35,6 +35,7 @@ export type SearchResult = { zones: string[]; search_source: SearchSource; search_distance: number; + top_score: number; // for old events data: { top_score: number; score: number; @@ -56,6 +57,8 @@ export type SearchFilter = { zones?: string[]; before?: number; after?: number; + min_score?: number; + max_score?: number; time_range?: string; search_type?: SearchSource[]; event_id?: string; @@ -71,6 +74,8 @@ export type SearchQueryParams = { zones?: string[]; before?: string; after?: string; + min_score?: number; + max_score?: number; search_type?: string; limit?: number; in_progress?: number; diff --git a/web/src/views/search/SearchView.tsx b/web/src/views/search/SearchView.tsx index b22a5248a..665f7a4fd 100644 --- a/web/src/views/search/SearchView.tsx +++ b/web/src/views/search/SearchView.tsx @@ -144,6 +144,8 @@ export default function SearchView({ : ["12:00AM-11:59PM"], before: [formatDateToLocaleString()], after: [formatDateToLocaleString(-5)], + min_score: ["50"], + max_score: ["100"], }), [config, allLabels, allZones, allSubLabels], ); @@ -385,7 +387,7 @@ export default function SearchView({ key={value.id} ref={(item) => (itemRefs.current[index] = item)} data-start={value.start_time} - className="review-item relative rounded-lg" + className="review-item relative flex flex-col rounded-lg" >
-
+
{ if (config?.semantic_search.enabled) { setSimilaritySearch(value);