From e016bd690025d8e888ecc28bb15ffd90b8e1fe33 Mon Sep 17 00:00:00 2001
From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>
Date: Tue, 10 Sep 2024 13:53:25 -0500
Subject: [PATCH] Find similar footage from review item snapshots (#13662)

* Find similar footage from review item snapshots

* Include confidence score for similarity search
---
 .../overlay/detail/ReviewDetailDialog.tsx     | 26 ++++++++++++++++-
 web/src/pages/Search.tsx                      | 28 +++++++++++++++++--
 web/src/types/search.ts                       |  4 +++
 web/src/views/search/SearchView.tsx           | 10 +++++--
 4 files changed, 62 insertions(+), 6 deletions(-)

diff --git a/web/src/components/overlay/detail/ReviewDetailDialog.tsx b/web/src/components/overlay/detail/ReviewDetailDialog.tsx
index 55c94f009..d134d7079 100644
--- a/web/src/components/overlay/detail/ReviewDetailDialog.tsx
+++ b/web/src/components/overlay/detail/ReviewDetailDialog.tsx
@@ -25,7 +25,7 @@ import { cn } from "@/lib/utils";
 import { FrigatePlusDialog } from "../dialog/FrigatePlusDialog";
 import ObjectLifecycle from "./ObjectLifecycle";
 import Chip from "@/components/indicators/Chip";
-import { FaDownload } from "react-icons/fa";
+import { FaDownload, FaImages } from "react-icons/fa";
 import FrigatePlusIcon from "@/components/icons/FrigatePlusIcon";
 import { FaArrowsRotate } from "react-icons/fa6";
 import {
@@ -33,6 +33,7 @@ import {
   TooltipContent,
   TooltipTrigger,
 } from "@/components/ui/tooltip";
+import { useNavigate } from "react-router-dom";
 
 type ReviewDetailDialogProps = {
   review?: ReviewSegment;
@@ -234,6 +235,8 @@ function EventItem({
 
   const [hovered, setHovered] = useState(isMobile);
 
+  const navigate = useNavigate();
+
   return (
     <>
       <div
@@ -328,6 +331,27 @@ function EventItem({
                   <TooltipContent>View Object Lifecycle</TooltipContent>
                 </Tooltip>
               )}
+
+              {event.has_snapshot && config?.semantic_search.enabled && (
+                <Tooltip>
+                  <TooltipTrigger>
+                    <Chip
+                      className="cursor-pointer rounded-md bg-gray-500 bg-gradient-to-br from-gray-400 to-gray-500"
+                      onClick={() => {
+                        const similaritySearchParams = new URLSearchParams({
+                          search_type: "similarity",
+                          event_id: event.id,
+                        }).toString();
+
+                        navigate(`/search?${similaritySearchParams}`);
+                      }}
+                    >
+                      <FaImages className="size-4 text-white" />
+                    </Chip>
+                  </TooltipTrigger>
+                  <TooltipContent>Find Similar</TooltipContent>
+                </Tooltip>
+              )}
             </div>
           </div>
         )}
diff --git a/web/src/pages/Search.tsx b/web/src/pages/Search.tsx
index 80e9846cb..c66cdc30b 100644
--- a/web/src/pages/Search.tsx
+++ b/web/src/pages/Search.tsx
@@ -3,7 +3,11 @@ import { useCameraPreviews } from "@/hooks/use-camera-previews";
 import { useOverlayState } from "@/hooks/use-overlay-state";
 import { FrigateConfig } from "@/types/frigateConfig";
 import { RecordingStartingPoint } from "@/types/record";
-import { SearchFilter, SearchResult } from "@/types/search";
+import {
+  PartialSearchResult,
+  SearchFilter,
+  SearchResult,
+} from "@/types/search";
 import { TimeRange } from "@/types/timeline";
 import { RecordingView } from "@/views/recording/RecordingView";
 import SearchView from "@/views/search/SearchView";
@@ -38,7 +42,27 @@ export default function Search() {
 
   // search api
 
-  const [similaritySearch, setSimilaritySearch] = useState<SearchResult>();
+  const [similaritySearch, setSimilaritySearch] =
+    useState<PartialSearchResult>();
+
+  useEffect(() => {
+    if (
+      config?.semantic_search.enabled &&
+      searchSearchParams["search_type"] == "similarity" &&
+      searchSearchParams["event_id"]?.length != 0 &&
+      searchFilter
+    ) {
+      setSimilaritySearch({
+        id: searchSearchParams["event_id"],
+      });
+
+      // remove event id from url params
+      const { event_id: _event_id, ...newFilter } = searchFilter;
+      setSearchFilter(newFilter);
+    }
+    // only run similarity search with event_id in the url when coming from review
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, []);
 
   useEffect(() => {
     if (similaritySearch) {
diff --git a/web/src/types/search.ts b/web/src/types/search.ts
index 673644e1b..af1567605 100644
--- a/web/src/types/search.ts
+++ b/web/src/types/search.ts
@@ -25,6 +25,9 @@ export type SearchResult = {
   };
 };
 
+
+export type PartialSearchResult = Partial<SearchResult> & { id: string };
+
 export type SearchFilter = {
   cameras?: string[];
   labels?: string[];
@@ -33,4 +36,5 @@ export type SearchFilter = {
   before?: number;
   after?: number;
   search_type?: SearchSource[];
+  event_id?: string;
 };
diff --git a/web/src/views/search/SearchView.tsx b/web/src/views/search/SearchView.tsx
index 06bac13e0..391deceb6 100644
--- a/web/src/views/search/SearchView.tsx
+++ b/web/src/views/search/SearchView.tsx
@@ -13,7 +13,11 @@ import {
 import { cn } from "@/lib/utils";
 import { FrigateConfig } from "@/types/frigateConfig";
 import { Preview } from "@/types/preview";
-import { SearchFilter, SearchResult } from "@/types/search";
+import {
+  PartialSearchResult,
+  SearchFilter,
+  SearchResult,
+} from "@/types/search";
 import { useCallback, useMemo, useState } from "react";
 import { isMobileOnly } from "react-device-detect";
 import { LuImage, LuSearchX, LuText, LuXCircle } from "react-icons/lu";
@@ -26,7 +30,7 @@ type SearchViewProps = {
   searchResults?: SearchResult[];
   allPreviews?: Preview[];
   isLoading: boolean;
-  similaritySearch?: SearchResult;
+  similaritySearch?: PartialSearchResult;
   setSearch: (search: string) => void;
   setSimilaritySearch: (search: SearchResult) => void;
   onUpdateFilter: (filter: SearchFilter) => void;
@@ -186,7 +190,7 @@ export default function SearchView({
                       scrollLock={false}
                       onClick={onSelectSearch}
                     />
-                    {searchTerm && (
+                    {(searchTerm || similaritySearch) && (
                       <div className={cn("absolute right-2 top-2 z-40")}>
                         <Tooltip>
                           <TooltipTrigger>