Add support for filtering history page and add support for creating timeline entries for audio / custom events (#9034)

* Add filter popover

* Add api filter hook and use UI with filtering

* Get history filtering working for cameras and labels

* Allow filtering on detail level

* Save timeline entries for api events

* reset

* fix width

frigate/events/external.py

@@ -52,7 +52,7 @@ class ExternalEventProcessor:
             (
                 EventTypeEnum.api,
                 "new",
-                camera_config,
+                camera,
                 {
                     "id": event_id,
                     "label": label,

frigate/events/maintainer.py

@@ -109,6 +109,16 @@ class EventProcessor(threading.Thread):
 
                 self.handle_object_detection(event_type, camera, event_data)
             elif source_type == EventTypeEnum.api:
+                self.timeline_queue.put(
+                    (
+                        camera,
+                        source_type,
+                        event_type,
+                        {},
+                        event_data,
+                    )
+                )
+
                 self.handle_external_detection(event_type, event_data)
 
         # set an end_time on events without an end_time before exiting

frigate/http.py

@@ -614,20 +614,30 @@ def timeline():
 @bp.route("/timeline/hourly")
 def hourly_timeline():
     """Get hourly summary for timeline."""
-    camera = request.args.get("camera", "all")
+    cameras = request.args.get("cameras", "all")
+    labels = request.args.get("labels", "all")
     before = request.args.get("before", type=float)
+    after = request.args.get("after", type=float)
     limit = request.args.get("limit", 200)
     tz_name = request.args.get("timezone", default="utc", type=str)
     _, minute_modifier, _ = get_tz_modifiers(tz_name)
 
     clauses = []
 
-    if camera != "all":
-        clauses.append((Timeline.camera == camera))
+    if cameras != "all":
+        camera_list = cameras.split(",")
+        clauses.append((Timeline.camera << camera_list))
+
+    if labels != "all":
+        label_list = labels.split(",")
+        clauses.append((Timeline.data["label"] << label_list))
 
     if before:
         clauses.append((Timeline.timestamp < before))
 
+    if after:
+        clauses.append((Timeline.timestamp > after))
+
     if len(clauses) == 0:
         clauses.append((True))
 

frigate/timeline.py

@@ -48,6 +48,8 @@ class TimelineProcessor(threading.Thread):
                 self.handle_object_detection(
                     camera, event_type, prev_event_data, event_data
                 )
+            elif input_type == EventTypeEnum.api:
+                self.handle_api_entry(camera, event_type, event_data)
 
     def insert_or_save(
         self,
@@ -140,3 +142,40 @@ class TimelineProcessor(threading.Thread):
 
         if save:
             self.insert_or_save(timeline_entry, prev_event_data, event_data)
+
+    def handle_api_entry(
+        self,
+        camera: str,
+        event_type: str,
+        event_data: dict[any, any],
+    ) -> bool:
+        if event_type != "new":
+            return False
+
+        if event_data.get("type", "api") == "audio":
+            timeline_entry = {
+                Timeline.class_type: "heard",
+                Timeline.timestamp: event_data["start_time"],
+                Timeline.camera: camera,
+                Timeline.source: "audio",
+                Timeline.source_id: event_data["id"],
+                Timeline.data: {
+                    "label": event_data["label"],
+                    "sub_label": event_data.get("sub_label"),
+                },
+            }
+        else:
+            timeline_entry = {
+                Timeline.class_type: "external",
+                Timeline.timestamp: event_data["start_time"],
+                Timeline.camera: camera,
+                Timeline.source: "api",
+                Timeline.source_id: event_data["id"],
+                Timeline.data: {
+                    "label": event_data["label"],
+                    "sub_label": event_data.get("sub_label"),
+                },
+            }
+
+        Timeline.insert(timeline_entry).execute()
+        return True

web/src/components/filter/HistoryFilterPopover.tsx

@@ -0,0 +1,305 @@
+import { LuCheck, LuFilter } from "react-icons/lu";
+import { Button } from "../ui/button";
+import { Popover, PopoverContent, PopoverTrigger } from "../ui/popover";
+import useSWR from "swr";
+import { FrigateConfig } from "@/types/frigateConfig";
+import { useMemo, useState } from "react";
+import {
+  DropdownMenu,
+  DropdownMenuContent,
+  DropdownMenuLabel,
+  DropdownMenuRadioGroup,
+  DropdownMenuRadioItem,
+  DropdownMenuSeparator,
+  DropdownMenuTrigger,
+} from "../ui/dropdown-menu";
+import { Calendar } from "../ui/calendar";
+
+type HistoryFilterPopoverProps = {
+  filter: HistoryFilter | undefined;
+  onUpdateFilter: (filter: HistoryFilter) => void;
+};
+
+export default function HistoryFilterPopover({
+  filter,
+  onUpdateFilter,
+}: HistoryFilterPopoverProps) {
+  const { data: config } = useSWR<FrigateConfig>("config");
+
+  const [open, setOpen] = useState(false);
+  const disabledDates = useMemo(() => {
+    const tomorrow = new Date();
+    tomorrow.setHours(tomorrow.getHours() + 24, -1, 0, 0);
+    const future = new Date();
+    future.setFullYear(2032);
+    return { from: tomorrow, to: future };
+  }, []);
+
+  const { data: allLabels } = useSWR<string[]>(["labels"], {
+    revalidateOnFocus: false,
+  });
+  const { data: allSubLabels } = useSWR<string[]>(
+    ["sub_labels", { split_joined: 1 }],
+    {
+      revalidateOnFocus: false,
+    }
+  );
+  const filterValues = useMemo(
+    () => ({
+      cameras: Object.keys(config?.cameras || {}),
+      labels: Object.values(allLabels || {}),
+    }),
+    [config, allLabels, allSubLabels]
+  );
+  const [selectedFilters, setSelectedFilters] = useState({
+    cameras: filter == undefined ? ["all"] : filter.cameras,
+    labels: filter == undefined ? ["all"] : filter.labels,
+    before: filter?.before,
+    after: filter?.after,
+    detailLevel: filter?.detailLevel ?? "normal",
+  });
+  const dateRange = useMemo(() => {
+    return selectedFilters?.before == undefined ||
+      selectedFilters?.after == undefined
+      ? undefined
+      : {
+          from: new Date(selectedFilters.after * 1000),
+          to: new Date(selectedFilters.before * 1000),
+        };
+  }, [selectedFilters]);
+
+  const allItems = useMemo(() => {
+    return {
+      cameras:
+        JSON.stringify(selectedFilters.cameras) == JSON.stringify(["all"]),
+      labels: JSON.stringify(selectedFilters.labels) == JSON.stringify(["all"]),
+    };
+  }, [selectedFilters]);
+
+  return (
+    <Popover open={open} onOpenChange={(open) => setOpen(open)}>
+      <PopoverTrigger asChild>
+        <Button>
+          <LuFilter className="mx-1" />
+          Filter
+        </Button>
+      </PopoverTrigger>
+      <PopoverContent className="w-screen sm:w-[340px]">
+        <div className="flex justify-around">
+          <DropdownMenu>
+            <DropdownMenuTrigger asChild>
+              <Button className="capitalize" variant="outline">
+                {allItems.cameras
+                  ? "All Cameras"
+                  : `${selectedFilters.cameras.length} Cameras`}
+              </Button>
+            </DropdownMenuTrigger>
+            <DropdownMenuContent>
+              <DropdownMenuLabel>Filter Cameras</DropdownMenuLabel>
+              <DropdownMenuSeparator />
+              <FilterCheckBox
+                isChecked={allItems.cameras}
+                label="All Cameras"
+                onCheckedChange={(isChecked) => {
+                  if (isChecked) {
+                    setSelectedFilters({
+                      ...selectedFilters,
+                      cameras: ["all"],
+                    });
+                  }
+                }}
+              />
+              <DropdownMenuSeparator />
+              {filterValues.cameras.map((item) => (
+                <FilterCheckBox
+                  key={item}
+                  isChecked={selectedFilters.cameras.includes(item)}
+                  label={item.replaceAll("_", " ")}
+                  onCheckedChange={(isChecked) => {
+                    if (isChecked) {
+                      const selectedCameras = allItems.cameras
+                        ? []
+                        : [...selectedFilters.cameras];
+                      selectedCameras.push(item);
+                      setSelectedFilters({
+                        ...selectedFilters,
+                        cameras: selectedCameras,
+                      });
+                    } else {
+                      const selectedCameraList = [...selectedFilters.cameras];
+
+                      // can not deselect the last item
+                      if (selectedCameraList.length > 1) {
+                        selectedCameraList.splice(
+                          selectedCameraList.indexOf(item),
+                          1
+                        );
+                        setSelectedFilters({
+                          ...selectedFilters,
+                          cameras: selectedCameraList,
+                        });
+                      }
+                    }
+                  }}
+                />
+              ))}
+            </DropdownMenuContent>
+          </DropdownMenu>
+          <DropdownMenu>
+            <DropdownMenuTrigger asChild>
+              <Button className="capitalize" variant="outline">
+                {allItems.labels
+                  ? "All Labels"
+                  : `${selectedFilters.labels.length} Labels`}
+              </Button>
+            </DropdownMenuTrigger>
+            <DropdownMenuContent>
+              <DropdownMenuLabel>Filter Labels</DropdownMenuLabel>
+              <DropdownMenuSeparator />
+              <FilterCheckBox
+                isChecked={allItems.labels}
+                label="All Labels"
+                onCheckedChange={(isChecked) => {
+                  if (isChecked) {
+                    setSelectedFilters({
+                      ...selectedFilters,
+                      labels: ["all"],
+                    });
+                  }
+                }}
+              />
+              <DropdownMenuSeparator />
+              {filterValues.labels.map((item) => (
+                <FilterCheckBox
+                  key={item}
+                  isChecked={
+                    selectedFilters.labels.length == 0 ||
+                    selectedFilters.labels.includes(item)
+                  }
+                  label={item.replaceAll("_", " ")}
+                  onCheckedChange={(isChecked) => {
+                    if (isChecked) {
+                      const selectedLabels = allItems.labels
+                        ? []
+                        : [...selectedFilters.labels];
+                      selectedLabels.push(item);
+                      setSelectedFilters({
+                        ...selectedFilters,
+                        labels: selectedLabels,
+                      });
+                    } else {
+                      const selectedLabelList = [...selectedFilters.labels];
+
+                      // can not deselect the last item
+                      if (selectedLabelList.length > 1) {
+                        selectedLabelList.splice(
+                          selectedLabelList.indexOf(item),
+                          1
+                        );
+                        setSelectedFilters({
+                          ...selectedFilters,
+                          labels: selectedLabelList,
+                        });
+                      }
+                    }
+                  }}
+                />
+              ))}
+            </DropdownMenuContent>
+          </DropdownMenu>
+          <DropdownMenu>
+            <DropdownMenuTrigger asChild>
+              <Button className="capitalize" variant="outline">
+                {selectedFilters.detailLevel}
+              </Button>
+            </DropdownMenuTrigger>
+            <DropdownMenuContent>
+              <DropdownMenuLabel>
+                Detail Level
+              </DropdownMenuLabel>
+              <DropdownMenuSeparator />
+              <DropdownMenuRadioGroup
+                value={selectedFilters.detailLevel}
+                onValueChange={(value) => {
+                  setSelectedFilters({
+                    ...selectedFilters,
+                    // @ts-ignore we know that value is one of the detailLevel
+                    detailLevel: value,
+                  });
+                }}
+              >
+                <DropdownMenuRadioItem value="normal">
+                  Normal
+                </DropdownMenuRadioItem>
+                <DropdownMenuRadioItem value="extra">
+                  Extra
+                </DropdownMenuRadioItem>
+                <DropdownMenuRadioItem value="full">Full</DropdownMenuRadioItem>
+              </DropdownMenuRadioGroup>
+            </DropdownMenuContent>
+          </DropdownMenu>
+        </div>
+        <Calendar
+          mode="range"
+          disabled={disabledDates}
+          selected={dateRange}
+          onSelect={(range) => {
+            let afterTime = undefined;
+            if (range?.from != undefined) {
+              afterTime = range.from.getTime() / 1000;
+            }
+
+            // need to make sure the day selected for before covers the entire day
+            let beforeTime = undefined;
+            if (range?.from != undefined) {
+              const beforeDate = range.to ?? range.from;
+              beforeDate.setHours(beforeDate.getHours() + 24, -1, 0, 0);
+              beforeTime = beforeDate.getTime() / 1000;
+            }
+
+            setSelectedFilters({
+              ...selectedFilters,
+              after: afterTime,
+              before: beforeTime,
+            });
+          }}
+        />
+        <Button
+          onClick={() => {
+            onUpdateFilter(selectedFilters);
+            setOpen(false);
+          }}
+        >
+          Save
+        </Button>
+      </PopoverContent>
+    </Popover>
+  );
+}
+
+type FilterCheckBoxProps = {
+  label: string;
+  isChecked: boolean;
+  onCheckedChange: (isChecked: boolean) => void;
+};
+
+function FilterCheckBox({
+  label,
+  isChecked,
+  onCheckedChange,
+}: FilterCheckBoxProps) {
+  return (
+    <Button
+      className="capitalize flex justify-between items-center cursor-pointer w-full"
+      variant="ghost"
+      onClick={(_) => onCheckedChange(!isChecked)}
+    >
+      {isChecked ? (
+        <LuCheck className="w-6 h-6" />
+      ) : (
+        <div className="w-6 h-6" />
+      )}
+      <div className="ml-1 w-full flex justify-start">{label}</div>
+    </Button>
+  );
+}

web/src/hooks/use-api-filter.ts

@@ -0,0 +1,42 @@
+import { useMemo, useState } from "react";
+
+type useApiFilterReturn<F extends FilterType> = [
+  filter: F | undefined,
+  setFilter: (filter: F) => void,
+  searchParams:
+    | {
+        [key: string]: any;
+      }
+    | undefined,
+];
+
+export default function useApiFilter<
+  F extends FilterType,
+>(): useApiFilterReturn<F> {
+  const [filter, setFilter] = useState<F | undefined>(undefined);
+  const searchParams = useMemo(() => {
+    if (filter == undefined) {
+      return undefined;
+    }
+
+    const search: { [key: string]: string } = {};
+
+    Object.entries(filter).forEach(([key, value]) => {
+      if (Array.isArray(value)) {
+        if (value.length == 0) {
+          // empty array means all so ignore
+        } else {
+          search[key] = value.join(",");
+        }
+      } else {
+        if (value != undefined) {
+          search[key] = `${value}`;
+        }
+      }
+    });
+
+    return search;
+  }, [filter]);
+
+  return [filter, setFilter, searchParams];
+}

web/src/pages/History.tsx

@@ -19,6 +19,8 @@ import {
   AlertDialogHeader,
   AlertDialogTitle,
 } from "@/components/ui/alert-dialog";
+import HistoryFilterPopover from "@/components/filter/HistoryFilterPopover";
+import useApiFilter from "@/hooks/use-api-filter";
 
 const API_LIMIT = 200;
 
@@ -29,20 +31,39 @@ function History() {
       config?.ui?.timezone || Intl.DateTimeFormat().resolvedOptions().timeZone,
     [config]
   );
+
+  const [historyFilter, setHistoryFilter, historySearchParams] =
+    useApiFilter<HistoryFilter>();
+
   const timelineFetcher = useCallback((key: any) => {
     const [path, params] = Array.isArray(key) ? key : [key, undefined];
     return axios.get(path, { params }).then((res) => res.data);
   }, []);
 
-  const getKey = useCallback((index: number, prevData: HourlyTimeline) => {
-    if (index > 0) {
-      const lastDate = prevData.end;
-      const pagedParams = { before: lastDate, timezone, limit: API_LIMIT };
-      return ["timeline/hourly", pagedParams];
-    }
+  const getKey = useCallback(
+    (index: number, prevData: HourlyTimeline) => {
+      if (index > 0) {
+        const lastDate = prevData.end;
+        const pagedParams =
+          historySearchParams == undefined
+            ? { before: lastDate, timezone, limit: API_LIMIT }
+            : {
+                ...historySearchParams,
+                before: lastDate,
+                timezone,
+                limit: API_LIMIT,
+              };
+        return ["timeline/hourly", pagedParams];
+      }
 
-    return ["timeline/hourly", { timezone, limit: API_LIMIT }];
-  }, []);
+      const params =
+        historySearchParams == undefined
+          ? { timezone, limit: API_LIMIT }
+          : { ...historySearchParams, timezone, limit: API_LIMIT };
+      return ["timeline/hourly", params];
+    },
+    [historySearchParams]
+  );
 
   const {
     data: timelinePages,
@@ -59,7 +80,6 @@ function History() {
     { revalidateOnFocus: false }
   );
 
-  const [detailLevel, _] = useState<"normal" | "extra" | "full">("normal");
   const [playback, setPlayback] = useState<Card | undefined>();
 
   const shouldAutoPlay = useMemo(() => {
@@ -71,8 +91,11 @@ function History() {
       return [];
     }
 
-    return getHourlyTimelineData(timelinePages, detailLevel);
-  }, [detailLevel, timelinePages]);
+    return getHourlyTimelineData(
+      timelinePages,
+      historyFilter?.detailLevel ?? "normal"
+    );
+  }, [historyFilter, timelinePages]);
 
   const isDone =
     (timelinePages?.[timelinePages.length - 1]?.count ?? 0) < API_LIMIT;
@@ -137,7 +160,13 @@ function History() {
 
   return (
     <>
-      <Heading as="h2">Review</Heading>
+      <div className="flex justify-between">
+        <Heading as="h2">History</Heading>
+        <HistoryFilterPopover
+          filter={historyFilter}
+          onUpdateFilter={(filter) => setHistoryFilter(filter)}
+        />
+      </div>
 
       <AlertDialog
         open={itemsToDelete != null}
@@ -176,7 +205,7 @@ function History() {
             return (
               <div key={day}>
                 <Heading
-                  className="sticky py-2 -top-4 left-0 bg-background w-full z-10"
+                  className="sticky py-2 -top-4 left-0 bg-background w-full z-20"
                   as="h3"
                 >
                   {formatUnixTimestampToDateTime(parseInt(day), {
@@ -242,7 +271,7 @@ function History() {
                               );
                             })}
                         </div>
-                        {lastRow && <ActivityIndicator />}
+                        {lastRow && !isDone && <ActivityIndicator />}
                       </div>
                     );
                   }

web/src/types/filter.ts

@@ -0,0 +1 @@
+type FilterType = { [searchKey: string]: any };

web/src/types/history.ts

@@ -1,40 +1,57 @@
 type CardsData = {
+  [key: string]: {
     [key: string]: {
-        [key: string]: {
-            [key: string]: Card
-        }
-    }
-}
+      [key: string]: Card;
+    };
+  };
+};
 
 type Card = {
-    camera: string,
-    time: number,
-    entries: Timeline[],
-    uniqueKeys: string[],
-}
+  camera: string;
+  time: number;
+  entries: Timeline[];
+  uniqueKeys: string[];
+};
 
 type Preview = {
-    camera: string,
-    src: string,
-    type: string,
-    start: number,
-    end: number,
-}
+  camera: string;
+  src: string;
+  type: string;
+  start: number;
+  end: number;
+};
 
 type Timeline = {
-    camera: string,
-    timestamp: number,
-    data: {
-        [key: string]: any
-    },
-    class_type: string,
-    source_id: string,
-    source: string,
-}
+  camera: string;
+  timestamp: number;
+  data: {
+    [key: string]: any;
+  };
+  class_type:
+    | "visible"
+    | "gone"
+    | "sub_label"
+    | "entered_zone"
+    | "attribute"
+    | "active"
+    | "stationary"
+    | "heard"
+    | "external";
+  source_id: string;
+  source: string;
+};
 
 type HourlyTimeline = {
-    start: number,
-    end: number,
-    count: number,
-    hours: { [key: string]: Timeline[] };
+  start: number;
+  end: number;
+  count: number;
+  hours: { [key: string]: Timeline[] };
+};
+
+interface HistoryFilter extends FilterType {
+  cameras: string[];
+  labels: string[];
+  before: number | undefined;
+  after: number | undefined;
+  detailLevel: "normal" | "extra" | "full";
 }

web/src/utils/timelineUtil.tsx

@@ -1,4 +1,11 @@
-import { LuCircle, LuPlay, LuPlayCircle, LuTruck } from "react-icons/lu";
+import {
+  LuCircle,
+  LuCircleDot,
+  LuEar,
+  LuPlay,
+  LuPlayCircle,
+  LuTruck,
+} from "react-icons/lu";
 import { IoMdExit } from "react-icons/io";
 import {
   MdFaceUnlock,
@@ -33,7 +40,13 @@ export function getTimelineIcon(timelineItem: Timeline) {
           return <MdFaceUnlock className="w-4 mr-1" />;
         case "car":
           return <MdOutlinePictureInPictureAlt className="w-4 mr-1" />;
+        default:
+          return <LuCircleDot className="w-4 mr-1" />;
       }
+    case "heard":
+      return <LuEar className="w-4 mr-1" />;
+    case "external":
+      return <LuCircleDot className="w-4 mr-1" />;
   }
 }
 
@@ -76,5 +89,9 @@ export function getTimelineItemDescription(timelineItem: Timeline) {
       return `${timelineItem.data.label} recognized as ${timelineItem.data.sub_label}`;
     case "gone":
       return `${label} left`;
+    case "heard":
+      return `${label} heard`;
+    case "external":
+      return `${label} detected`;
   }
 }