Add score filter to Explore view (#14397)

* backend score filtering and sorting

* score filter frontend

* use input for score filtering

* use correct score on search thumbnail

* add popover to explain top_score

* revert sublabel score calc

* update filters logic

* fix rounding on score

* wait until default view is loaded

* don't turn button to selected style for similarity searches

* clarify language

* fix alert dialog buttons to use correct destructive variant

* use root level top_score for very old events

* better arrangement of thumbnail footer items on smaller screens
This commit is contained in:
Josh Hawkins 2024-10-17 06:30:52 -05:00 committed by GitHub
parent edaccd86d6
commit 8173cd7776
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 353 additions and 136 deletions

View File

@ -45,6 +45,9 @@ class EventsSearchQueryParams(BaseModel):
before: Optional[float] = None
time_range: Optional[str] = DEFAULT_TIME_RANGE
timezone: Optional[str] = "utc"
min_score: Optional[float] = None
max_score: Optional[float] = None
sort: Optional[str] = None
class EventsSummaryQueryParams(BaseModel):

View File

@ -348,6 +348,7 @@ def events_search(request: Request, params: EventsSearchQueryParams = Depends())
search_type = params.search_type
include_thumbnails = params.include_thumbnails
limit = params.limit
sort = params.sort
# Filters
cameras = params.cameras
@ -355,6 +356,8 @@ def events_search(request: Request, params: EventsSearchQueryParams = Depends())
zones = params.zones
after = params.after
before = params.before
min_score = params.min_score
max_score = params.max_score
time_range = params.time_range
# for similarity search
@ -430,6 +433,14 @@ def events_search(request: Request, params: EventsSearchQueryParams = Depends())
if before:
event_filters.append((Event.start_time < before))
if min_score is not None and max_score is not None:
event_filters.append((Event.data["score"].between(min_score, max_score)))
else:
if min_score is not None:
event_filters.append((Event.data["score"] >= min_score))
if max_score is not None:
event_filters.append((Event.data["score"] <= max_score))
if time_range != DEFAULT_TIME_RANGE:
tz_name = params.timezone
hour_modifier, minute_modifier, _ = get_tz_modifiers(tz_name)
@ -554,10 +565,18 @@ def events_search(request: Request, params: EventsSearchQueryParams = Depends())
processed_events.append(processed_event)
# Sort by search distance if search_results are available, otherwise by start_time
# Sort by search distance if search_results are available, otherwise by start_time as default
if search_results:
processed_events.sort(key=lambda x: x.get("search_distance", float("inf")))
else:
if sort == "score_asc":
processed_events.sort(key=lambda x: x["score"])
elif sort == "score_desc":
processed_events.sort(key=lambda x: x["score"], reverse=True)
elif sort == "date_asc":
processed_events.sort(key=lambda x: x["start_time"])
else:
# "date_desc" default
processed_events.sort(key=lambda x: x["start_time"], reverse=True)
# Limit the number of events returned

View File

@ -34,6 +34,7 @@ import { toast } from "sonner";
import useKeyboardListener from "@/hooks/use-keyboard-listener";
import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip";
import { capitalizeFirstLetter } from "@/utils/stringUtil";
import { buttonVariants } from "../ui/button";
type ReviewCardProps = {
event: ReviewSegment;
@ -228,7 +229,10 @@ export default function ReviewCard({
<AlertDialogCancel onClick={() => setOptionsOpen(false)}>
Cancel
</AlertDialogCancel>
<AlertDialogAction className="bg-destructive" onClick={onDelete}>
<AlertDialogAction
className={buttonVariants({ variant: "destructive" })}
onClick={onDelete}
>
Delete
</AlertDialogAction>
</AlertDialogFooter>
@ -295,7 +299,10 @@ export default function ReviewCard({
<AlertDialogCancel onClick={() => setOptionsOpen(false)}>
Cancel
</AlertDialogCancel>
<AlertDialogAction className="bg-destructive" onClick={onDelete}>
<AlertDialogAction
className={buttonVariants({ variant: "destructive" })}
onClick={onDelete}
>
Delete
</AlertDialogAction>
</AlertDialogFooter>

View File

@ -90,8 +90,10 @@ export default function SearchThumbnail({
onClick={() => onClick(searchResult)}
>
{getIconForLabel(objectLabel, "size-3 text-white")}
{Math.floor(
searchResult.score ?? searchResult.data.top_score * 100,
{Math.round(
(searchResult.data.score ??
searchResult.data.top_score ??
searchResult.top_score) * 100,
)}
%
</Chip>

View File

@ -32,9 +32,12 @@ import axios from "axios";
import { toast } from "sonner";
import { MdImageSearch } from "react-icons/md";
import { isMobileOnly } from "react-device-detect";
import { buttonVariants } from "../ui/button";
import { cn } from "@/lib/utils";
type SearchThumbnailProps = {
searchResult: SearchResult;
columns: number;
findSimilar: () => void;
refreshResults: () => void;
showObjectLifecycle: () => void;
@ -42,6 +45,7 @@ type SearchThumbnailProps = {
export default function SearchThumbnailFooter({
searchResult,
columns,
findSimilar,
refreshResults,
showObjectLifecycle,
@ -95,7 +99,7 @@ export default function SearchThumbnailFooter({
<AlertDialogFooter>
<AlertDialogCancel>Cancel</AlertDialogCancel>
<AlertDialogAction
className="bg-destructive"
className={buttonVariants({ variant: "destructive" })}
onClick={handleDelete}
>
Delete
@ -113,6 +117,13 @@ export default function SearchThumbnailFooter({
}}
/>
<div
className={cn(
"flex w-full flex-row items-center justify-between",
columns > 4 &&
"items-start sm:flex-col sm:gap-2 lg:flex-row lg:items-center lg:gap-1",
)}
>
<div className="flex flex-col items-start text-xs text-primary-variant">
{searchResult.end_time ? (
<TimeAgo time={searchResult.start_time * 1000} dense />
@ -212,6 +223,7 @@ export default function SearchThumbnailFooter({
</DropdownMenuContent>
</DropdownMenu>
</div>
</div>
</>
);
}

View File

@ -3,7 +3,7 @@ import { isDesktop, isMobile } from "react-device-detect";
import useSWR from "swr";
import { MdHome } from "react-icons/md";
import { usePersistedOverlayState } from "@/hooks/use-overlay-state";
import { Button } from "../ui/button";
import { Button, buttonVariants } from "../ui/button";
import { useCallback, useMemo, useState } from "react";
import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip";
import { LuPencil, LuPlus } from "react-icons/lu";
@ -518,7 +518,10 @@ export function CameraGroupRow({
</AlertDialogDescription>
<AlertDialogFooter>
<AlertDialogCancel>Cancel</AlertDialogCancel>
<AlertDialogAction onClick={onDeleteGroup}>
<AlertDialogAction
className={buttonVariants({ variant: "destructive" })}
onClick={onDeleteGroup}
>
Delete
</AlertDialogAction>
</AlertDialogFooter>

View File

@ -1,7 +1,7 @@
import { FaCircleCheck } from "react-icons/fa6";
import { useCallback, useState } from "react";
import axios from "axios";
import { Button } from "../ui/button";
import { Button, buttonVariants } from "../ui/button";
import { isDesktop } from "react-device-detect";
import { FaCompactDisc } from "react-icons/fa";
import { HiTrash } from "react-icons/hi";
@ -79,7 +79,10 @@ export default function ReviewActionGroup({
</AlertDialogDescription>
<AlertDialogFooter>
<AlertDialogCancel>Cancel</AlertDialogCancel>
<AlertDialogAction className="bg-destructive" onClick={onDelete}>
<AlertDialogAction
className={buttonVariants({ variant: "destructive" })}
onClick={onDelete}
>
Delete
</AlertDialogAction>
</AlertDialogFooter>

View File

@ -8,6 +8,7 @@ import {
AlertDialogHeader,
AlertDialogTitle,
} from "@/components/ui/alert-dialog";
import { buttonVariants } from "../ui/button";
type DeleteSearchDialogProps = {
isOpen: boolean;
@ -35,7 +36,7 @@ export function DeleteSearchDialog({
<AlertDialogCancel onClick={onClose}>Cancel</AlertDialogCancel>
<AlertDialogAction
onClick={onConfirm}
className="bg-destructive text-white"
className={buttonVariants({ variant: "destructive" })}
>
Delete
</AlertDialogAction>

View File

@ -201,10 +201,13 @@ export default function InputWithTags({
allSuggestions[type as FilterType]?.includes(value) ||
type == "before" ||
type == "after" ||
type == "time_range"
type == "time_range" ||
type == "min_score" ||
type == "max_score"
) {
const newFilters = { ...filters };
let timestamp = 0;
let score = 0;
switch (type) {
case "before":
@ -244,6 +247,40 @@ export default function InputWithTags({
newFilters[type] = timestamp / 1000;
}
break;
case "min_score":
case "max_score":
score = parseInt(value);
if (score >= 0) {
// Check for conflicts between min_score and max_score
if (
type === "min_score" &&
filters.max_score !== undefined &&
score > filters.max_score * 100
) {
toast.error(
"The 'min_score' must be less than or equal to the 'max_score'.",
{
position: "top-center",
},
);
return;
}
if (
type === "max_score" &&
filters.min_score !== undefined &&
score < filters.min_score * 100
) {
toast.error(
"The 'max_score' must be greater than or equal to the 'min_score'.",
{
position: "top-center",
},
);
return;
}
newFilters[type] = score / 100;
}
break;
case "time_range":
newFilters[type] = value;
break;
@ -302,6 +339,8 @@ export default function InputWithTags({
} - ${
config?.ui.time_format === "24hour" ? endTime : convertTo12Hour(endTime)
}`;
} else if (filterType === "min_score" || filterType === "max_score") {
return Math.round(Number(filterValues) * 100).toString() + "%";
} else {
return filterValues as string;
}
@ -320,7 +359,11 @@ export default function InputWithTags({
isValidTimeRange(
trimmedValue.replace("-", ","),
config?.ui.time_format,
))
)) ||
((filterType === "min_score" || filterType === "max_score") &&
!isNaN(Number(trimmedValue)) &&
Number(trimmedValue) >= 50 &&
Number(trimmedValue) <= 100)
) {
createFilter(
filterType,

View File

@ -62,6 +62,12 @@ import { Card, CardContent } from "@/components/ui/card";
import useImageLoaded from "@/hooks/use-image-loaded";
import ImageLoadingIndicator from "@/components/indicators/ImageLoadingIndicator";
import { GenericVideoPlayer } from "@/components/player/GenericVideoPlayer";
import {
Popover,
PopoverContent,
PopoverTrigger,
} from "@/components/ui/popover";
import { LuInfo } from "react-icons/lu";
const SEARCH_TABS = [
"details",
@ -279,7 +285,7 @@ function ObjectDetailsTab({
return 0;
}
const value = search.score ?? search.data.top_score;
const value = search.data.top_score;
return Math.round(value * 100);
}, [search]);
@ -369,7 +375,24 @@ function ObjectDetailsTab({
</div>
</div>
<div className="flex flex-col gap-1.5">
<div className="text-sm text-primary/40">Score</div>
<div className="text-sm text-primary/40">
<div className="flex flex-row items-center gap-1">
Top Score
<Popover>
<PopoverTrigger asChild>
<div className="cursor-pointer p-0">
<LuInfo className="size-4" />
<span className="sr-only">Info</span>
</div>
</PopoverTrigger>
<PopoverContent className="w-80">
The top score is the highest median score for the tracked
object, so this may differ from the score shown on the
search result thumbnail.
</PopoverContent>
</Popover>
</div>
</div>
<div className="text-sm">
{score}%{subLabelScore && ` (${subLabelScore}%)`}
</div>

View File

@ -23,6 +23,8 @@ import { Switch } from "@/components/ui/switch";
import { Label } from "@/components/ui/label";
import { DropdownMenuSeparator } from "@/components/ui/dropdown-menu";
import { cn } from "@/lib/utils";
import { DualThumbSlider } from "@/components/ui/slider";
import { Input } from "@/components/ui/input";
type SearchFilterDialogProps = {
config?: FrigateConfig;
@ -46,6 +48,12 @@ export default function SearchFilterDialog({
const [currentFilter, setCurrentFilter] = useState(filter ?? {});
const { data: allSubLabels } = useSWR(["sub_labels", { split_joined: 1 }]);
useEffect(() => {
if (filter) {
setCurrentFilter(filter);
}
}, [filter]);
// state
const [open, setOpen] = useState(false);
@ -54,9 +62,12 @@ export default function SearchFilterDialog({
() =>
currentFilter &&
(currentFilter.time_range ||
(currentFilter.min_score ?? 0) > 0.5 ||
(currentFilter.max_score ?? 1) < 1 ||
(currentFilter.zones?.length ?? 0) > 0 ||
(currentFilter.sub_labels?.length ?? 0) > 0 ||
(currentFilter.search_type?.length ?? 2) !== 2),
(!currentFilter.search_type?.includes("similarity") &&
(currentFilter.search_type?.length ?? 2) !== 2)),
[currentFilter],
);
@ -97,6 +108,13 @@ export default function SearchFilterDialog({
setCurrentFilter({ ...currentFilter, sub_labels: newSubLabels })
}
/>
<ScoreFilterContent
minScore={currentFilter.min_score}
maxScore={currentFilter.max_score}
setScoreRange={(min, max) =>
setCurrentFilter({ ...currentFilter, min_score: min, max_score: max })
}
/>
{config?.semantic_search?.enabled &&
!currentFilter?.search_type?.includes("similarity") && (
<SearchTypeContent
@ -133,6 +151,8 @@ export default function SearchFilterDialog({
zones: undefined,
sub_labels: undefined,
search_type: ["thumbnail", "description"],
min_score: undefined,
max_score: undefined,
}));
}}
>
@ -420,6 +440,58 @@ export function SubFilterContent({
);
}
type ScoreFilterContentProps = {
minScore: number | undefined;
maxScore: number | undefined;
setScoreRange: (min: number | undefined, max: number | undefined) => void;
};
export function ScoreFilterContent({
minScore,
maxScore,
setScoreRange,
}: ScoreFilterContentProps) {
return (
<div className="overflow-x-hidden">
<DropdownMenuSeparator className="mb-3" />
<div className="mb-3 text-lg">Score</div>
<div className="flex items-center gap-1">
<Input
className="w-12"
inputMode="numeric"
value={Math.round((minScore ?? 0.5) * 100)}
onChange={(e) => {
const value = e.target.value;
if (value) {
setScoreRange(parseInt(value) / 100.0, maxScore ?? 1.0);
}
}}
/>
<DualThumbSlider
className="w-full"
min={0.5}
max={1.0}
step={0.01}
value={[minScore ?? 0.5, maxScore ?? 1.0]}
onValueChange={([min, max]) => setScoreRange(min, max)}
/>
<Input
className="w-12"
inputMode="numeric"
value={Math.round((maxScore ?? 1.0) * 100)}
onChange={(e) => {
const value = e.target.value;
if (value) {
setScoreRange(minScore ?? 0.5, parseInt(value) / 100.0);
}
}}
/>
</div>
</div>
);
}
type SearchTypeContentProps = {
searchSources: SearchSource[] | undefined;
setSearchSources: (sources: SearchSource[] | undefined) => void;

View File

@ -35,6 +35,7 @@ import { FrigateConfig } from "@/types/frigateConfig";
import { reviewQueries } from "@/utils/zoneEdutUtil";
import IconWrapper from "../ui/icon-wrapper";
import { StatusBarMessagesContext } from "@/context/statusbar-provider";
import { buttonVariants } from "../ui/button";
type PolygonItemProps = {
polygon: Polygon;
@ -257,7 +258,10 @@ export default function PolygonItem({
</AlertDialogDescription>
<AlertDialogFooter>
<AlertDialogCancel>Cancel</AlertDialogCancel>
<AlertDialogAction onClick={handleDelete}>
<AlertDialogAction
className={buttonVariants({ variant: "destructive" })}
onClick={handleDelete}
>
Delete
</AlertDialogAction>
</AlertDialogFooter>

View File

@ -1,6 +1,6 @@
import { Button } from "../ui/button";
import { useState } from "react";
import { isDesktop } from "react-device-detect";
import { isDesktop, isMobileOnly } from "react-device-detect";
import { cn } from "@/lib/utils";
import PlatformAwareDialog from "../overlay/dialog/PlatformAwareDialog";
import { FaCog } from "react-icons/fa";
@ -40,7 +40,7 @@ export default function SearchSettings({
<div className={cn(className, "my-3 space-y-5 py-3 md:mt-0 md:py-0")}>
<div className="space-y-4">
<div className="space-y-0.5">
<div className="text-md">Default Search View</div>
<div className="text-md">Default View</div>
<div className="space-y-1 text-xs text-muted-foreground">
When no filters are selected, display a summary of the most recent
tracked objects per label, or display an unfiltered grid.
@ -68,12 +68,14 @@ export default function SearchSettings({
</SelectContent>
</Select>
</div>
{!isMobileOnly && (
<>
<DropdownMenuSeparator />
<div className="flex w-full flex-col space-y-4">
<div className="space-y-0.5">
<div className="text-md">Grid Columns</div>
<div className="space-y-1 text-xs text-muted-foreground">
Select the number of columns in the results grid.
Select the number of columns in the grid view.
</div>
</div>
<div className="flex items-center space-x-4">
@ -85,9 +87,13 @@ export default function SearchSettings({
step={1}
className="flex-grow"
/>
<span className="w-9 text-center text-sm font-medium">{columns}</span>
<span className="w-9 text-center text-sm font-medium">
{columns}
</span>
</div>
</div>
</>
)}
</div>
);

View File

@ -14,6 +14,7 @@ import { ModelState } from "@/types/ws";
import { formatSecondsToDuration } from "@/utils/dateUtil";
import SearchView from "@/views/search/SearchView";
import { useCallback, useEffect, useMemo, useState } from "react";
import { isMobileOnly } from "react-device-detect";
import { LuCheck, LuExternalLink, LuX } from "react-icons/lu";
import { TbExclamationCircle } from "react-icons/tb";
import { Link } from "react-router-dom";
@ -32,11 +33,16 @@ export default function Explore() {
// grid
const [columnCount, setColumnCount] = usePersistence("exploreGridColumns", 4);
const gridColumns = useMemo(() => columnCount ?? 4, [columnCount]);
const gridColumns = useMemo(() => {
if (isMobileOnly) {
return 2;
}
return columnCount ?? 4;
}, [columnCount]);
// default layout
const [defaultView, setDefaultView] = usePersistence(
const [defaultView, setDefaultView, defaultViewLoaded] = usePersistence(
"exploreDefaultView",
"summary",
);
@ -103,6 +109,8 @@ export default function Explore() {
after: searchSearchParams["after"],
time_range: searchSearchParams["time_range"],
search_type: searchSearchParams["search_type"],
min_score: searchSearchParams["min_score"],
max_score: searchSearchParams["max_score"],
limit:
Object.keys(searchSearchParams).length == 0 ? API_LIMIT : undefined,
timezone,
@ -129,6 +137,8 @@ export default function Explore() {
after: searchSearchParams["after"],
time_range: searchSearchParams["time_range"],
search_type: searchSearchParams["search_type"],
min_score: searchSearchParams["min_score"],
max_score: searchSearchParams["max_score"],
event_id: searchSearchParams["event_id"],
timezone,
include_thumbnails: 0,
@ -270,12 +280,13 @@ export default function Explore() {
};
if (
config?.semantic_search.enabled &&
!defaultViewLoaded ||
(config?.semantic_search.enabled &&
(!reindexState ||
!textModelState ||
!textTokenizerState ||
!visionModelState ||
!visionFeatureExtractorState)
!visionFeatureExtractorState))
) {
return (
<ActivityIndicator className="absolute left-1/2 top-1/2 -translate-x-1/2 -translate-y-1/2" />

View File

@ -35,6 +35,7 @@ export type SearchResult = {
zones: string[];
search_source: SearchSource;
search_distance: number;
top_score: number; // for old events
data: {
top_score: number;
score: number;
@ -56,6 +57,8 @@ export type SearchFilter = {
zones?: string[];
before?: number;
after?: number;
min_score?: number;
max_score?: number;
time_range?: string;
search_type?: SearchSource[];
event_id?: string;
@ -71,6 +74,8 @@ export type SearchQueryParams = {
zones?: string[];
before?: string;
after?: string;
min_score?: number;
max_score?: number;
search_type?: string;
limit?: number;
in_progress?: number;

View File

@ -144,6 +144,8 @@ export default function SearchView({
: ["12:00AM-11:59PM"],
before: [formatDateToLocaleString()],
after: [formatDateToLocaleString(-5)],
min_score: ["50"],
max_score: ["100"],
}),
[config, allLabels, allZones, allSubLabels],
);
@ -385,7 +387,7 @@ export default function SearchView({
key={value.id}
ref={(item) => (itemRefs.current[index] = item)}
data-start={value.start_time}
className="review-item relative rounded-lg"
className="review-item relative flex flex-col rounded-lg"
>
<div
className={cn(
@ -400,9 +402,10 @@ export default function SearchView({
<div
className={`review-item-ring pointer-events-none absolute inset-0 z-10 size-full rounded-lg outline outline-[3px] -outline-offset-[2.8px] ${selected ? `shadow-selected outline-selected` : "outline-transparent duration-500"}`}
/>
<div className="flex w-full items-center justify-between rounded-b-lg border border-t-0 bg-card p-3 text-card-foreground">
<div className="flex w-full grow items-center justify-between rounded-b-lg border border-t-0 bg-card p-3 text-card-foreground">
<SearchThumbnailFooter
searchResult={value}
columns={columns}
findSimilar={() => {
if (config?.semantic_search.enabled) {
setSimilaritySearch(value);