Refactor history viewer to show player / timeline for full hour and use preview while scrubbing timeline (#9051)

* Move history card view to separate view and create timeline view

* Get custom time scrubber working

* Add back nav

* Show timeline bounding boxes

* Implement seeking limiter

* Use browser history to allow back button to close timeline viewer

* Fix mobile timeline and add more icons for detections

* Play when item is initially visible

web/package-lock.json

@@ -46,6 +46,7 @@
         "swr": "^2.2.4",
         "tailwind-merge": "^2.1.0",
         "tailwindcss-animate": "^1.0.7",
+        "vaul": "^0.8.0",
         "video.js": "^8.6.1",
         "videojs-playlist": "^5.1.0",
         "vis-timeline": "^7.7.3",
@@ -7758,6 +7759,18 @@
         "node": ">=10.12.0"
       }
     },
+    "node_modules/vaul": {
+      "version": "0.8.0",
+      "resolved": "https://registry.npmjs.org/vaul/-/vaul-0.8.0.tgz",
+      "integrity": "sha512-9nUU2jIObJvJZxeQU1oVr/syKo5XqbRoOMoTEt0hHlWify4QZFlqTh6QSN/yxoKzNrMeEQzxbc3XC/vkPLOIqw==",
+      "dependencies": {
+        "@radix-ui/react-dialog": "^1.0.4"
+      },
+      "peerDependencies": {
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
+      }
+    },
     "node_modules/video.js": {
       "version": "8.6.1",
       "resolved": "https://registry.npmjs.org/video.js/-/video.js-8.6.1.tgz",

web/package.json

@@ -51,6 +51,7 @@
     "swr": "^2.2.4",
     "tailwind-merge": "^2.1.0",
     "tailwindcss-animate": "^1.0.7",
+    "vaul": "^0.8.0",
     "video.js": "^8.6.1",
     "videojs-playlist": "^5.1.0",
     "vis-timeline": "^7.7.3",

web/src/components/card/HistoryCard.tsx

@@ -10,11 +10,12 @@ import {
   getTimelineIcon,
   getTimelineItemDescription,
 } from "@/utils/timelineUtil";
+import { Button } from "../ui/button";
 
 type HistoryCardProps = {
   timeline: Card;
   relevantPreview?: Preview;
-  shouldAutoPlay: boolean;
+  isMobile: boolean;
   onClick?: () => void;
   onDelete?: () => void;
 };
@@ -22,7 +23,7 @@ type HistoryCardProps = {
 export default function HistoryCard({
   relevantPreview,
   timeline,
-  shouldAutoPlay,
+  isMobile,
   onClick,
   onDelete,
 }: HistoryCardProps) {
@@ -42,11 +43,12 @@ export default function HistoryCard({
         relevantPreview={relevantPreview}
         startTs={Object.values(timeline.entries)[0].timestamp}
         eventId={Object.values(timeline.entries)[0].source_id}
-        shouldAutoPlay={shouldAutoPlay}
+        isMobile={isMobile}
+        onClick={onClick}
       />
-      <div className="p-2">
+      <>
         <div className="text-sm flex justify-between items-center">
-          <div>
+          <div className="pl-1 pt-1">
             <LuClock className="h-5 w-5 mr-2 inline" />
             {formatUnixTimestampToDateTime(timeline.time, {
               strftime_fmt:
@@ -55,9 +57,9 @@ export default function HistoryCard({
               date_style: "medium",
             })}
           </div>
+          <Button className="px-2 py-2" variant="ghost" size="xs">
             <LuTrash
-            className="w-5 h-5 m-1 cursor-pointer"
+              className="w-5 h-5 stroke-red-500"
-            stroke="#f87171"
               onClick={(e: Event) => {
                 e.stopPropagation();
 
@@ -66,16 +68,18 @@ export default function HistoryCard({
                 }
               }}
             />
+          </Button>
         </div>
-        <div className="capitalize text-sm flex items-center mt-1">
+        <div className="pl-1 capitalize text-sm flex items-center mt-1">
           <HiOutlineVideoCamera className="h-5 w-5 mr-2 inline" />
           {timeline.camera.replaceAll("_", " ")}
         </div>
-        <div className="my-2 text-sm font-medium">Activity:</div>
+        <div className="pl-1 my-2">
-        {Object.entries(timeline.entries).map(([_, entry]) => {
+          <div className="text-sm font-medium">Activity:</div>
+          {Object.entries(timeline.entries).map(([_, entry], idx) => {
             return (
               <div
-              key={entry.timestamp}
+                key={idx}
                 className="flex text-xs capitalize my-1 items-center"
               >
                 {getTimelineIcon(entry)}
@@ -84,6 +88,7 @@ export default function HistoryCard({
             );
           })}
         </div>
+      </>
     </Card>
   );
 }

web/src/components/player/PreviewThumbnailPlayer.tsx

@@ -1,7 +1,13 @@
 import { FrigateConfig } from "@/types/frigateConfig";
 import VideoPlayer from "./VideoPlayer";
 import useSWR from "swr";
-import { useCallback, useMemo, useRef, useState } from "react";
+import React, {
+  useCallback,
+  useEffect,
+  useMemo,
+  useRef,
+  useState,
+} from "react";
 import { useApiHost } from "@/api";
 import Player from "video.js/dist/types/player";
 import { AspectRatio } from "../ui/aspect-ratio";
@@ -12,7 +18,8 @@ type PreviewPlayerProps = {
   relevantPreview?: Preview;
   startTs: number;
   eventId: string;
-  shouldAutoPlay: boolean;
+  isMobile: boolean;
+  onClick?: () => void;
 };
 
 type Preview = {
@@ -28,20 +35,26 @@ export default function PreviewThumbnailPlayer({
   relevantPreview,
   startTs,
   eventId,
-  shouldAutoPlay,
+  isMobile,
+  onClick,
 }: PreviewPlayerProps) {
   const { data: config } = useSWR("config");
   const playerRef = useRef<Player | null>(null);
-  const apiHost = useApiHost();
   const isSafari = useMemo(() => {
     return /^((?!chrome|android).)*safari/i.test(navigator.userAgent);
   }, []);
 
   const [visible, setVisible] = useState(false);
+  const [isInitiallyVisible, setIsInitiallyVisible] = useState(false);
 
   const onPlayback = useCallback(
     (isHovered: Boolean) => {
-      if (!relevantPreview || !playerRef.current) {
+      if (!relevantPreview) {
+        return;
+      }
+
+      if (!playerRef.current) {
+        setIsInitiallyVisible(true);
         return;
       }
 
@@ -78,7 +91,7 @@ export default function PreviewThumbnailPlayer({
         }
       }
 
-      if (shouldAutoPlay && !autoPlayObserver.current) {
+      if (isMobile && !autoPlayObserver.current) {
         try {
           autoPlayObserver.current = new IntersectionObserver(
             (entries) => {
@@ -92,8 +105,6 @@ export default function PreviewThumbnailPlayer({
             {
               threshold: 1.0,
               root: document.getElementById("pageRoot"),
-              // iOS has bug where poster is empty frame until video starts playing so playback needs to begin earlier
-              rootMargin: isSafari ? "10% 0px 25% 0px" : "0px",
             }
           );
           if (node) autoPlayObserver.current.observe(node);
@@ -105,20 +116,95 @@ export default function PreviewThumbnailPlayer({
     [preloadObserver, autoPlayObserver, onPlayback]
   );
 
-  let content;
+  return (
+    <AspectRatio
+      ref={relevantPreview ? inViewRef : null}
+      ratio={16 / 9}
+      className="bg-black flex justify-center items-center"
+      onMouseEnter={() => onPlayback(true)}
+      onMouseLeave={() => onPlayback(false)}
+    >
+      <PreviewContent
+        playerRef={playerRef}
+        relevantPreview={relevantPreview}
+        isVisible={visible}
+        isInitiallyVisible={isInitiallyVisible}
+        startTs={startTs}
+        camera={camera}
+        config={config}
+        eventId={eventId}
+        isMobile={isMobile}
+        isSafari={isSafari}
+        onClick={onClick}
+      />
+    </AspectRatio>
+  );
+}
 
-  if (relevantPreview && !visible) {
+type PreviewContentProps = {
-    content = <div />;
+  playerRef: React.MutableRefObject<Player | null>;
+  config: FrigateConfig;
+  camera: string;
+  relevantPreview: Preview | undefined;
+  eventId: string;
+  isVisible: boolean;
+  isInitiallyVisible: boolean;
+  startTs: number;
+  isMobile: boolean;
+  isSafari: boolean;
+  onClick?: () => void;
+};
+function PreviewContent({
+  playerRef,
+  config,
+  camera,
+  relevantPreview,
+  eventId,
+  isVisible,
+  isInitiallyVisible,
+  startTs,
+  isMobile,
+  isSafari,
+  onClick,
+}: PreviewContentProps) {
+  const apiHost = useApiHost();
+
+  // handle touchstart -> touchend as click
+  const [touchStart, setTouchStart] = useState(0);
+  const handleTouchStart = useCallback(() => {
+    setTouchStart(new Date().getTime());
+  }, []);
+  useEffect(() => {
+    if (!isMobile || !playerRef.current || !onClick) {
+      return;
+    }
+
+    playerRef.current.on("touchend", () => {
+      if (!onClick) {
+        return;
+      }
+
+      const touchEnd = new Date().getTime();
+
+      // consider tap less than 500 ms
+      if (touchEnd - touchStart < 500) {
+        onClick();
+      }
+    });
+  }, [playerRef, touchStart]);
+
+  if (relevantPreview && !isVisible) {
+    return <div />;
   } else if (!relevantPreview) {
     if (isCurrentHour(startTs)) {
-      content = (
+      return (
         <img
           className={`${getPreviewWidth(camera, config)}`}
           src={`${apiHost}api/preview/${camera}/${startTs}/thumbnail.jpg`}
         />
       );
     } else {
-      content = (
+      return (
         <img
           className="w-[160px]"
           src={`${apiHost}api/events/${eventId}/thumbnail.jpg`}
@@ -126,13 +212,13 @@ export default function PreviewThumbnailPlayer({
       );
     }
   } else {
-    content = (
+    return (
       <>
         <div className={`${getPreviewWidth(camera, config)}`}>
           <VideoPlayer
             options={{
               preload: "auto",
-              autoplay: false,
+              autoplay: true,
               controls: false,
               muted: true,
               loadingSpinner: false,
@@ -146,8 +232,16 @@ export default function PreviewThumbnailPlayer({
             seekOptions={{}}
             onReady={(player) => {
               playerRef.current = player;
+
+              if (!isInitiallyVisible) {
+                player.pause(); // autoplay + pause is required for iOS
+              }
+
               player.playbackRate(isSafari ? 2 : 8);
               player.currentTime(startTs - relevantPreview.start);
+              if (isMobile && onClick) {
+                player.on("touchstart", handleTouchStart);
+              }
             }}
             onDispose={() => {
               playerRef.current = null;
@@ -158,18 +252,6 @@ export default function PreviewThumbnailPlayer({
       </>
     );
   }
-
-  return (
-    <AspectRatio
-      ref={relevantPreview ? inViewRef : null}
-      ratio={16 / 9}
-      className="bg-black flex justify-center items-center"
-      onMouseEnter={() => onPlayback(true)}
-      onMouseLeave={() => onPlayback(false)}
-    >
-      {content}
-    </AspectRatio>
-  );
 }
 
 function isCurrentHour(timestamp: number) {

web/src/components/scrubber/ActivityScrubber.tsx

@@ -4,6 +4,8 @@ import {
   TimelineGroup,
   TimelineItem,
   TimelineOptions,
+  DateType,
+  IdType,
 } from "vis-timeline";
 import type { DataGroup, DataItem, TimelineEvents } from "vis-timeline/types";
 import "./scrubber.css";
@@ -72,13 +74,17 @@ const domEvents: TimelineEventsWithMissing[] = [
 ];
 
 type ActivityScrubberProps = {
-  items: TimelineItem[];
+  className?: string;
+  items?: TimelineItem[];
+  timeBars?: { time: DateType; id?: IdType | undefined }[];
   groups?: TimelineGroup[];
   options?: TimelineOptions;
 } & TimelineEventsHandlers;
 
 function ActivityScrubber({
+  className,
   items,
+  timeBars,
   groups,
   options,
   ...eventHandlers
@@ -123,13 +129,24 @@ function ActivityScrubber({
       return;
     }
 
+    const timelineOptions: TimelineOptions = {
+      ...defaultOptions,
+      ...options,
+    };
+
     const timelineInstance = new VisTimeline(
       divElement,
       items as DataItem[],
       groups as DataGroup[],
-      options
+      timelineOptions
     );
 
+    if (timeBars) {
+      timeBars.forEach((bar) => {
+        timelineInstance.addCustomTime(bar.time, bar.id);
+      });
+    }
+
     domEvents.forEach((event) => {
       const eventHandler = eventHandlers[`${event}Handler`];
       if (typeof eventHandler === "function") {
@@ -139,42 +156,16 @@ function ActivityScrubber({
 
     timelineRef.current.timeline = timelineInstance;
 
-    const timelineOptions: TimelineOptions = {
-      ...defaultOptions,
-      ...options,
-    };
-
-    timelineInstance.setOptions(timelineOptions);
-
     return () => {
       timelineInstance.destroy();
     };
-  }, []);
+  }, [containerRef]);
 
-  useEffect(() => {
+  return (
-    if (!timelineRef.current.timeline) {
+    <div className={className || ""}>
-      return;
+      <div ref={containerRef} />
-    }
+    </div>
-
+  );
-    // If the currentTime updates, adjust the scrubber's end date and max
-    // May not be applicable to all scrubbers, might want to just pass this in
-    // for any scrubbers that we want to dynamically move based on time
-    // const updatedTimeOptions: TimelineOptions = {
-    //   end: currentTime,
-    //   max: currentTime,
-    // };
-
-    const timelineOptions: TimelineOptions = {
-      ...defaultOptions,
-      // ...updatedTimeOptions,
-      ...options,
-    };
-
-    timelineRef.current.timeline.setOptions(timelineOptions);
-    if (items) timelineRef.current.timeline.setItems(items);
-  }, [items, groups, options, currentTime, eventHandlers]);
-
-  return <div ref={containerRef} />;
 }
 
 export default ActivityScrubber;

web/src/components/ui/button.tsx

@@ -21,6 +21,7 @@ const buttonVariants = cva(
       },
       size: {
         default: "h-10 px-4 py-2",
+        xs: "h-6 rounded-md",
         sm: "h-9 rounded-md px-3",
         lg: "h-11 rounded-md px-8",
         icon: "h-10 w-10",

web/src/components/ui/drawer.tsx

@@ -0,0 +1,116 @@
+import * as React from "react"
+import { Drawer as DrawerPrimitive } from "vaul"
+
+import { cn } from "@/lib/utils"
+
+const Drawer = ({
+  shouldScaleBackground = true,
+  ...props
+}: React.ComponentProps<typeof DrawerPrimitive.Root>) => (
+  <DrawerPrimitive.Root
+    shouldScaleBackground={shouldScaleBackground}
+    {...props}
+  />
+)
+Drawer.displayName = "Drawer"
+
+const DrawerTrigger = DrawerPrimitive.Trigger
+
+const DrawerPortal = DrawerPrimitive.Portal
+
+const DrawerClose = DrawerPrimitive.Close
+
+const DrawerOverlay = React.forwardRef<
+  React.ElementRef<typeof DrawerPrimitive.Overlay>,
+  React.ComponentPropsWithoutRef<typeof DrawerPrimitive.Overlay>
+>(({ className, ...props }, ref) => (
+  <DrawerPrimitive.Overlay
+    ref={ref}
+    className={cn("fixed inset-0 z-50 bg-black/80", className)}
+    {...props}
+  />
+))
+DrawerOverlay.displayName = DrawerPrimitive.Overlay.displayName
+
+const DrawerContent = React.forwardRef<
+  React.ElementRef<typeof DrawerPrimitive.Content>,
+  React.ComponentPropsWithoutRef<typeof DrawerPrimitive.Content>
+>(({ className, children, ...props }, ref) => (
+  <DrawerPortal>
+    <DrawerOverlay />
+    <DrawerPrimitive.Content
+      ref={ref}
+      className={cn(
+        "fixed inset-x-0 bottom-0 z-50 mt-24 flex h-auto flex-col rounded-t-[10px] border bg-background",
+        className
+      )}
+      {...props}
+    >
+      <div className="mx-auto mt-4 h-2 w-[100px] rounded-full bg-muted" />
+      {children}
+    </DrawerPrimitive.Content>
+  </DrawerPortal>
+))
+DrawerContent.displayName = "DrawerContent"
+
+const DrawerHeader = ({
+  className,
+  ...props
+}: React.HTMLAttributes<HTMLDivElement>) => (
+  <div
+    className={cn("grid gap-1.5 p-4 text-center sm:text-left", className)}
+    {...props}
+  />
+)
+DrawerHeader.displayName = "DrawerHeader"
+
+const DrawerFooter = ({
+  className,
+  ...props
+}: React.HTMLAttributes<HTMLDivElement>) => (
+  <div
+    className={cn("mt-auto flex flex-col gap-2 p-4", className)}
+    {...props}
+  />
+)
+DrawerFooter.displayName = "DrawerFooter"
+
+const DrawerTitle = React.forwardRef<
+  React.ElementRef<typeof DrawerPrimitive.Title>,
+  React.ComponentPropsWithoutRef<typeof DrawerPrimitive.Title>
+>(({ className, ...props }, ref) => (
+  <DrawerPrimitive.Title
+    ref={ref}
+    className={cn(
+      "text-lg font-semibold leading-none tracking-tight",
+      className
+    )}
+    {...props}
+  />
+))
+DrawerTitle.displayName = DrawerPrimitive.Title.displayName
+
+const DrawerDescription = React.forwardRef<
+  React.ElementRef<typeof DrawerPrimitive.Description>,
+  React.ComponentPropsWithoutRef<typeof DrawerPrimitive.Description>
+>(({ className, ...props }, ref) => (
+  <DrawerPrimitive.Description
+    ref={ref}
+    className={cn("text-sm text-muted-foreground", className)}
+    {...props}
+  />
+))
+DrawerDescription.displayName = DrawerPrimitive.Description.displayName
+
+export {
+  Drawer,
+  DrawerPortal,
+  DrawerOverlay,
+  DrawerTrigger,
+  DrawerClose,
+  DrawerContent,
+  DrawerHeader,
+  DrawerFooter,
+  DrawerTitle,
+  DrawerDescription,
+}

web/src/hooks/use-overlay-state.tsx

@@ -0,0 +1,20 @@
+import { useCallback } from "react";
+import { useLocation, useNavigate } from "react-router-dom";
+
+export default function useOverlayState(key: string) {
+  const location = useLocation();
+  const navigate = useNavigate();
+  const currentLocationState = location.state;
+
+  const setOverlayStateValue = useCallback(
+    (value: string) => {
+      const newLocationState = { ...currentLocationState };
+      newLocationState[key] = value;
+      navigate(location.pathname, { state: newLocationState });
+    },
+    [navigate]
+  );
+
+  const overlayStateValue = location.state && location.state[key];
+  return [overlayStateValue, setOverlayStateValue];
+}

web/src/pages/History.tsx

@@ -1,13 +1,10 @@
-import { useCallback, useMemo, useRef, useState } from "react";
+import { useCallback, useMemo, useState } from "react";
 import useSWR from "swr";
 import useSWRInfinite from "swr/infinite";
 import { FrigateConfig } from "@/types/frigateConfig";
 import Heading from "@/components/ui/heading";
 import ActivityIndicator from "@/components/ui/activity-indicator";
-import HistoryCard from "@/components/card/HistoryCard";
-import { formatUnixTimestampToDateTime } from "@/utils/dateUtil";
 import axios from "axios";
-import TimelinePlayerCard from "@/components/card/TimelinePlayerCard";
 import { getHourlyTimelineData } from "@/utils/historyUtil";
 import {
   AlertDialog,
@@ -21,6 +18,13 @@ import {
 } from "@/components/ui/alert-dialog";
 import HistoryFilterPopover from "@/components/filter/HistoryFilterPopover";
 import useApiFilter from "@/hooks/use-api-filter";
+import HistoryCardView from "@/views/history/HistoryCardView";
+import HistoryTimelineView from "@/views/history/HistoryTimelineView";
+import { Button } from "@/components/ui/button";
+import { IoMdArrowBack } from "react-icons/io";
+import useOverlayState from "@/hooks/use-overlay-state";
+import { useNavigate } from "react-router-dom";
+import { Dialog, DialogContent } from "@/components/ui/dialog";
 
 const API_LIMIT = 200;
 
@@ -80,10 +84,24 @@ function History() {
     { revalidateOnFocus: false }
   );
 
-  const [playback, setPlayback] = useState<Card | undefined>();
+  const navigate = useNavigate();
+  const [playback, setPlayback] = useState<TimelinePlayback | undefined>();
+  const [viewingPlayback, setViewingPlayback] = useOverlayState("timeline");
+  const setPlaybackState = useCallback(
+    (playback: TimelinePlayback | undefined) => {
+      if (playback == undefined) {
+        setPlayback(undefined);
+        navigate(-1);
+      } else {
+        setPlayback(playback);
+        setViewingPlayback(true);
+      }
+    },
+    [navigate]
+  );
 
-  const shouldAutoPlay = useMemo(() => {
+  const isMobile = useMemo(() => {
-    return playback == undefined && window.innerWidth < 480;
+    return window.innerWidth < 768;
   }, [playback]);
 
   const timelineCards: CardsData | never[] = useMemo(() => {
@@ -100,26 +118,6 @@ function History() {
   const isDone =
     (timelinePages?.[timelinePages.length - 1]?.count ?? 0) < API_LIMIT;
 
-  // hooks for infinite scroll
-  const observer = useRef<IntersectionObserver | null>();
-  const lastTimelineRef = useCallback(
-    (node: HTMLElement | null) => {
-      if (isValidating) return;
-      if (observer.current) observer.current.disconnect();
-      try {
-        observer.current = new IntersectionObserver((entries) => {
-          if (entries[0].isIntersecting && !isDone) {
-            setSize(size + 1);
-          }
-        });
-        if (node) observer.current.observe(node);
-      } catch (e) {
-        // no op
-      }
-    },
-    [size, setSize, isValidating, isDone]
-  );
-
   const [itemsToDelete, setItemsToDelete] = useState<string[] | null>(null);
   const onDelete = useCallback(
     async (timeline: Card) => {
@@ -161,11 +159,25 @@ function History() {
   return (
     <>
       <div className="flex justify-between">
+        <div className="flex justify-start">
+          {viewingPlayback && (
+            <Button
+              className="mt-2"
+              size="xs"
+              variant="ghost"
+              onClick={() => setPlaybackState(undefined)}
+            >
+              <IoMdArrowBack className="w-6 h-6" />
+            </Button>
+          )}
           <Heading as="h2">History</Heading>
+        </div>
+        {!playback && (
           <HistoryFilterPopover
             filter={historyFilter}
             onUpdateFilter={(filter) => setHistoryFilter(filter)}
           />
+        )}
       </div>
 
       <AlertDialog
@@ -192,96 +204,51 @@ function History() {
           </AlertDialogFooter>
         </AlertDialogContent>
       </AlertDialog>
-
+      <HistoryCardView
-      <TimelinePlayerCard
+        timelineCards={timelineCards}
-        timeline={playback}
+        allPreviews={allPreviews}
-        onDismiss={() => setPlayback(undefined)}
+        isMobile={isMobile}
-      />
+        isValidating={isValidating}
-
+        isDone={isDone}
-      <div>
+        onNextPage={() => {
-        {Object.entries(timelineCards)
+          setSize(size + 1);
-          .reverse()
-          .map(([day, timelineDay], dayIdx) => {
-            return (
-              <div key={day}>
-                <Heading
-                  className="sticky py-2 -top-4 left-0 bg-background w-full z-20"
-                  as="h3"
-                >
-                  {formatUnixTimestampToDateTime(parseInt(day), {
-                    strftime_fmt: "%A %b %d",
-                    time_style: "medium",
-                    date_style: "medium",
-                  })}
-                </Heading>
-                {Object.entries(timelineDay).map(
-                  ([hour, timelineHour], hourIdx) => {
-                    if (Object.values(timelineHour).length == 0) {
-                      return <div key={hour}></div>;
-                    }
-
-                    const lastRow =
-                      dayIdx == Object.values(timelineCards).length - 1 &&
-                      hourIdx == Object.values(timelineDay).length - 1;
-                    const previewMap: { [key: string]: Preview | undefined } =
-                      {};
-
-                    return (
-                      <div key={hour} ref={lastRow ? lastTimelineRef : null}>
-                        <Heading as="h4">
-                          {formatUnixTimestampToDateTime(parseInt(hour), {
-                            strftime_fmt:
-                              config.ui.time_format == "24hour"
-                                ? "%H:00"
-                                : "%I:00 %p",
-                            time_style: "medium",
-                            date_style: "medium",
-                          })}
-                        </Heading>
-
-                        <div className="flex flex-wrap">
-                          {Object.entries(timelineHour)
-                            .reverse()
-                            .map(([key, timeline]) => {
-                              const startTs = Object.values(timeline.entries)[0]
-                                .timestamp;
-                              let relevantPreview = previewMap[timeline.camera];
-
-                              if (relevantPreview == undefined) {
-                                relevantPreview = previewMap[timeline.camera] =
-                                  Object.values(allPreviews || []).find(
-                                    (preview) =>
-                                      preview.camera == timeline.camera &&
-                                      preview.start < startTs &&
-                                      preview.end > startTs
-                                  );
-                              }
-
-                              return (
-                                <HistoryCard
-                                  key={key}
-                                  timeline={timeline}
-                                  shouldAutoPlay={shouldAutoPlay}
-                                  relevantPreview={relevantPreview}
-                                  onClick={() => {
-                                    setPlayback(timeline);
         }}
-                                  onDelete={() => onDelete(timeline)}
+        onDelete={onDelete}
+        onItemSelected={(item) => setPlaybackState(item)}
+      />
+      <TimelineViewer
+        playback={viewingPlayback ? playback : undefined}
+        isMobile={isMobile}
+        onClose={() => setPlaybackState(undefined)}
       />
-                              );
-                            })}
-                        </div>
-                        {lastRow && !isDone && <ActivityIndicator />}
-                      </div>
-                    );
-                  }
-                )}
-              </div>
-            );
-          })}
-      </div>
     </>
   );
 }
 
+type TimelineViewerProps = {
+  playback: TimelinePlayback | undefined;
+  isMobile: boolean;
+  onClose: () => void;
+};
+
+function TimelineViewer({ playback, isMobile, onClose }: TimelineViewerProps) {
+  if (isMobile) {
+    return playback != undefined ? (
+      <div className="w-screen absolute left-0 top-20 bottom-0 bg-background z-50">
+        <HistoryTimelineView playback={playback} isMobile={isMobile} />
+      </div>
+    ) : null;
+  }
+
+  return (
+    <Dialog open={playback != undefined} onOpenChange={(_) => onClose()}>
+      <DialogContent className="w-3/5 max-w-full">
+        {playback && (
+          <HistoryTimelineView playback={playback} isMobile={isMobile} />
+        )}
+      </DialogContent>
+    </Dialog>
+  );
+}
+
 export default History;

web/src/types/history.ts

@@ -55,3 +55,9 @@ interface HistoryFilter extends FilterType {
   after: number | undefined;
   detailLevel: "normal" | "extra" | "full";
 }
+
+type TimelinePlayback = {
+  camera: string;
+  timelineItems: Timeline[];
+  relevantPreview: Preview | undefined;
+};

web/src/utils/timelineUtil.tsx

@@ -1,17 +1,25 @@
 import {
+  LuCamera,
+  LuCar,
+  LuCat,
   LuCircle,
   LuCircleDot,
+  LuDog,
   LuEar,
+  LuPackage,
+  LuPersonStanding,
   LuPlay,
   LuPlayCircle,
   LuTruck,
 } from "react-icons/lu";
+import { GiDeer } from "react-icons/gi";
 import { IoMdExit } from "react-icons/io";
 import {
   MdFaceUnlock,
   MdOutlineLocationOn,
   MdOutlinePictureInPictureAlt,
 } from "react-icons/md";
+import { FaBicycle } from "react-icons/fa";
 
 export function getTimelineIcon(timelineItem: Timeline) {
   switch (timelineItem.class_type) {
@@ -50,6 +58,32 @@ export function getTimelineIcon(timelineItem: Timeline) {
   }
 }
 
+/**
+ * Get icon representing detection, either label specific or generic detection icon
+ * @param timelineItem timeline item
+ * @returns icon for label
+ */
+export function getTimelineDetectionIcon(timelineItem: Timeline) {
+  switch (timelineItem.data.label) {
+    case "bicycle":
+      return <FaBicycle className="w-4 mr-1" />;
+    case "car":
+      return <LuCar className="w-4 mr-1" />;
+    case "cat":
+      return <LuCat className="w-4 mr-1" />;
+    case "deer":
+      return <GiDeer className="w-4 mr-1" />;
+    case "dog":
+      return <LuDog className="w-4 mr-1" />;
+    case "package":
+      return <LuPackage className="w-4 mr-1" />;
+    case "person":
+      return <LuPersonStanding className="w-4 mr-1" />;
+    default:
+      return <LuCamera className="w-4 mr-1" />;
+  }
+}
+
 export function getTimelineItemDescription(timelineItem: Timeline) {
   const label = (
     (Array.isArray(timelineItem.data.sub_label)

web/src/views/history/HistoryCardView.tsx

@@ -0,0 +1,145 @@
+import HistoryCard from "@/components/card/HistoryCard";
+import ActivityIndicator from "@/components/ui/activity-indicator";
+import Heading from "@/components/ui/heading";
+import { FrigateConfig } from "@/types/frigateConfig";
+import { formatUnixTimestampToDateTime } from "@/utils/dateUtil";
+import { useCallback, useRef } from "react";
+import useSWR from "swr";
+
+type HistoryCardViewProps = {
+  timelineCards: CardsData | never[];
+  allPreviews: Preview[] | undefined;
+  isMobile: boolean;
+  isValidating: boolean;
+  isDone: boolean;
+  onNextPage: () => void;
+  onDelete: (card: Card) => void;
+  onItemSelected: (item: TimelinePlayback) => void;
+};
+
+export default function HistoryCardView({
+  timelineCards,
+  allPreviews,
+  isMobile,
+  isValidating,
+  isDone,
+  onNextPage,
+  onDelete,
+  onItemSelected,
+}: HistoryCardViewProps) {
+  const { data: config } = useSWR<FrigateConfig>("config");
+
+  // hooks for infinite scroll
+  const observer = useRef<IntersectionObserver | null>();
+  const lastTimelineRef = useCallback(
+    (node: HTMLElement | null) => {
+      if (isValidating) return;
+      if (observer.current) observer.current.disconnect();
+      try {
+        observer.current = new IntersectionObserver((entries) => {
+          if (entries[0].isIntersecting && !isDone) {
+            onNextPage();
+          }
+        });
+        if (node) observer.current.observe(node);
+      } catch (e) {
+        // no op
+      }
+    },
+    [isValidating, isDone]
+  );
+
+  return (
+    <>
+      {Object.entries(timelineCards)
+        .reverse()
+        .map(([day, timelineDay], dayIdx) => {
+          return (
+            <div key={day}>
+              <Heading
+                className="sticky py-2 -top-4 left-0 bg-background w-full z-20"
+                as="h3"
+              >
+                {formatUnixTimestampToDateTime(parseInt(day), {
+                  strftime_fmt: "%A %b %d",
+                  time_style: "medium",
+                  date_style: "medium",
+                })}
+              </Heading>
+              {Object.entries(timelineDay).map(
+                ([hour, timelineHour], hourIdx) => {
+                  if (Object.values(timelineHour).length == 0) {
+                    return <div key={hour}></div>;
+                  }
+
+                  const lastRow =
+                    dayIdx == Object.values(timelineCards).length - 1 &&
+                    hourIdx == Object.values(timelineDay).length - 1;
+                  const previewMap: { [key: string]: Preview | undefined } = {};
+
+                  return (
+                    <div key={hour} ref={lastRow ? lastTimelineRef : null}>
+                      <Heading as="h4">
+                        {formatUnixTimestampToDateTime(parseInt(hour), {
+                          strftime_fmt:
+                            config?.ui.time_format == "24hour"
+                              ? "%H:00"
+                              : "%I:00 %p",
+                          time_style: "medium",
+                          date_style: "medium",
+                        })}
+                      </Heading>
+
+                      <div className="flex flex-wrap">
+                        {Object.entries(timelineHour)
+                          .reverse()
+                          .map(([key, timeline]) => {
+                            const startTs = Object.values(timeline.entries)[0]
+                              .timestamp;
+                            let relevantPreview = previewMap[timeline.camera];
+
+                            if (relevantPreview == undefined) {
+                              relevantPreview = previewMap[timeline.camera] =
+                                Object.values(allPreviews || []).find(
+                                  (preview) =>
+                                    preview.camera == timeline.camera &&
+                                    preview.start < startTs &&
+                                    preview.end > startTs
+                                );
+                            }
+
+                            return (
+                              <HistoryCard
+                                key={key}
+                                timeline={timeline}
+                                isMobile={isMobile}
+                                relevantPreview={relevantPreview}
+                                onClick={() => {
+                                  onItemSelected({
+                                    camera: timeline.camera,
+                                    timelineItems: Object.values(
+                                      timelineHour
+                                    ).flatMap((card) =>
+                                      card.camera == timeline.camera
+                                        ? card.entries
+                                        : []
+                                    ),
+                                    relevantPreview: relevantPreview,
+                                  });
+                                }}
+                                onDelete={() => onDelete(timeline)}
+                              />
+                            );
+                          })}
+                      </div>
+                      {lastRow && !isDone && <ActivityIndicator />}
+                    </div>
+                  );
+                }
+              )}
+            </div>
+          );
+        })}
+    </>
+  );
+}

web/src/views/history/HistoryTimelineView.tsx

@@ -0,0 +1,300 @@
+import { useApiHost } from "@/api";
+import TimelineEventOverlay from "@/components/overlay/TimelineDataOverlay";
+import VideoPlayer from "@/components/player/VideoPlayer";
+import ActivityScrubber, {
+  ScrubberItem,
+} from "@/components/scrubber/ActivityScrubber";
+import ActivityIndicator from "@/components/ui/activity-indicator";
+import { FrigateConfig } from "@/types/frigateConfig";
+import {
+  getTimelineDetectionIcon,
+  getTimelineIcon,
+} from "@/utils/timelineUtil";
+import { renderToStaticMarkup } from "react-dom/server";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
+import useSWR from "swr";
+import Player from "video.js/dist/types/player";
+
+type HistoryTimelineViewProps = {
+  playback: TimelinePlayback;
+  isMobile: boolean;
+};
+
+export default function HistoryTimelineView({
+  playback,
+  isMobile,
+}: HistoryTimelineViewProps) {
+  const apiHost = useApiHost();
+  const { data: config } = useSWR<FrigateConfig>("config");
+  const timezone = useMemo(
+    () =>
+      config?.ui?.timezone || Intl.DateTimeFormat().resolvedOptions().timeZone,
+    [config]
+  );
+
+  const hasRelevantPreview = playback.relevantPreview != undefined;
+
+  const playerRef = useRef<Player | undefined>(undefined);
+  const previewRef = useRef<Player | undefined>(undefined);
+
+  const [scrubbing, setScrubbing] = useState(false);
+  const [focusedItem, setFocusedItem] = useState<Timeline | undefined>(
+    undefined
+  );
+
+  const [seeking, setSeeking] = useState(false);
+  const [timeToSeek, setTimeToSeek] = useState<number | undefined>(undefined);
+
+  const annotationOffset = useMemo(() => {
+    if (!config) {
+      return 0;
+    }
+
+    return (
+      (config.cameras[playback.camera]?.detect?.annotation_offset || 0) / 1000
+    );
+  }, [config, playback]);
+
+  const timelineTime = useMemo(() => {
+    if (!playback) {
+      return 0;
+    }
+
+    return playback.timelineItems.at(0)!!.timestamp;
+  }, [playback]);
+  const playbackTimes = useMemo(() => {
+    const date = new Date(timelineTime * 1000);
+    date.setMinutes(0, 0, 0);
+    const startTime = date.getTime() / 1000;
+    date.setHours(date.getHours() + 1);
+    const endTime = date.getTime() / 1000;
+    return {
+      start: parseInt(startTime.toFixed(1)),
+      end: parseInt(endTime.toFixed(1)),
+    };
+  }, [timelineTime]);
+
+  const recordingParams = useMemo(() => {
+    return {
+      before: playbackTimes.end,
+      after: playbackTimes.start,
+    };
+  }, [playbackTimes]);
+  const { data: recordings } = useSWR<Recording[]>(
+    playback ? [`${playback.camera}/recordings`, recordingParams] : null,
+    { revalidateOnFocus: false }
+  );
+
+  const playbackUri = useMemo(() => {
+    if (!playback) {
+      return "";
+    }
+
+    const date = new Date(playbackTimes.start * 1000);
+    return `${apiHost}vod/${date.getFullYear()}-${
+      date.getMonth() + 1
+    }/${date.getDate()}/${date.getHours()}/${
+      playback.camera
+    }/${timezone.replaceAll("/", ",")}/master.m3u8`;
+  }, [playbackTimes]);
+
+  const onSelectItem = useCallback(
+    (data: { items: number[] }) => {
+      if (data.items.length > 0) {
+        const selected = data.items[0];
+        setFocusedItem(
+          playback.timelineItems.find(
+            (timeline) => timeline.timestamp == selected
+          )
+        );
+        playerRef.current?.pause();
+
+        let seekSeconds = 0;
+        (recordings || []).every((segment) => {
+          // if the next segment is past the desired time, stop calculating
+          if (segment.start_time > selected) {
+            return false;
+          }
+
+          if (segment.end_time < selected) {
+            seekSeconds += segment.end_time - segment.start_time;
+            return true;
+          }
+
+          seekSeconds +=
+            segment.end_time -
+            segment.start_time -
+            (segment.end_time - selected);
+          return true;
+        });
+        playerRef.current?.currentTime(seekSeconds);
+      }
+    },
+    [annotationOffset, recordings, playerRef]
+  );
+
+  const onScrubTime = useCallback(
+    (data: { time: Date }) => {
+      if (!hasRelevantPreview) {
+        return;
+      }
+
+      if (playerRef.current?.paused() == false) {
+        setScrubbing(true);
+        playerRef.current?.pause();
+      }
+
+      const seekTimestamp = data.time.getTime() / 1000;
+      const seekTime = seekTimestamp - playback.relevantPreview!!.start;
+      setTimeToSeek(Math.round(seekTime));
+    },
+    [scrubbing, playerRef]
+  );
+
+  const onStopScrubbing = useCallback(
+    (data: { time: Date }) => {
+      const playbackTime = data.time.getTime() / 1000;
+      playerRef.current?.currentTime(playbackTime - playbackTimes.start);
+      setScrubbing(false);
+      playerRef.current?.play();
+    },
+    [playerRef]
+  );
+
+  // handle seeking to next frame when seek is finished
+  useEffect(() => {
+    if (seeking) {
+      return;
+    }
+
+    if (timeToSeek && timeToSeek != previewRef.current?.currentTime()) {
+      setSeeking(true);
+      previewRef.current?.currentTime(timeToSeek);
+    }
+  }, [timeToSeek, seeking]);
+
+  if (!config || !recordings) {
+    return <ActivityIndicator />;
+  }
+
+  return (
+    <div className="w-full">
+      <>
+        <div
+          className={`relative ${
+            hasRelevantPreview && scrubbing ? "hidden" : "visible"
+          }`}
+        >
+          <VideoPlayer
+            options={{
+              preload: "auto",
+              autoplay: true,
+              sources: [
+                {
+                  src: playbackUri,
+                  type: "application/vnd.apple.mpegurl",
+                },
+              ],
+            }}
+            seekOptions={{ forward: 10, backward: 5 }}
+            onReady={(player) => {
+              playerRef.current = player;
+              player.currentTime(timelineTime - playbackTimes.start);
+              player.on("playing", () => {
+                setFocusedItem(undefined);
+              });
+            }}
+            onDispose={() => {
+              playerRef.current = undefined;
+            }}
+          >
+            {config && focusedItem ? (
+              <TimelineEventOverlay
+                timeline={focusedItem}
+                cameraConfig={config.cameras[playback.camera]}
+              />
+            ) : undefined}
+          </VideoPlayer>
+        </div>
+        {hasRelevantPreview && (
+          <div className={`${scrubbing ? "visible" : "hidden"}`}>
+            <VideoPlayer
+              options={{
+                preload: "auto",
+                autoplay: false,
+                controls: false,
+                muted: true,
+                loadingSpinner: false,
+                sources: [
+                  {
+                    src: `${playback.relevantPreview?.src}`,
+                    type: "video/mp4",
+                  },
+                ],
+              }}
+              seekOptions={{}}
+              onReady={(player) => {
+                previewRef.current = player;
+                player.on("seeked", () => setSeeking(false));
+              }}
+              onDispose={() => {
+                previewRef.current = undefined;
+              }}
+            />
+          </div>
+        )}
+      </>
+      <div className="m-1">
+        {playback != undefined && (
+          <ActivityScrubber
+            items={timelineItemsToScrubber(playback.timelineItems)}
+            timeBars={
+              hasRelevantPreview
+                ? [{ time: new Date(timelineTime * 1000), id: "playback" }]
+                : []
+            }
+            options={{
+              ...(isMobile && {
+                start: new Date(
+                  Math.max(playbackTimes.start, timelineTime - 300) * 1000
+                ),
+                end: new Date(
+                  Math.min(playbackTimes.end, timelineTime + 300) * 1000
+                ),
+              }),
+              snap: null,
+              min: new Date(playbackTimes.start * 1000),
+              max: new Date(playbackTimes.end * 1000),
+              timeAxis: isMobile ? { scale: "minute", step: 5 } : {},
+            }}
+            timechangeHandler={onScrubTime}
+            timechangedHandler={onStopScrubbing}
+            selectHandler={onSelectItem}
+          />
+        )}
+      </div>
+    </div>
+  );
+}
+
+function timelineItemsToScrubber(items: Timeline[]): ScrubberItem[] {
+  return items.map((item) => {
+    return {
+      id: item.timestamp,
+      content: getTimelineContentElement(item),
+      start: new Date(item.timestamp * 1000),
+      end: new Date(item.timestamp * 1000),
+      type: "box",
+    };
+  });
+}
+
+function getTimelineContentElement(item: Timeline): HTMLElement {
+  const output = document.createElement(`div-${item.timestamp}`);
+  output.innerHTML = renderToStaticMarkup(
+    <div className="flex items-center">
+      {getTimelineDetectionIcon(item)} : {getTimelineIcon(item)}
+    </div>
+  );
+  return output;
+}

web/tailwind.config.js

@@ -72,7 +72,8 @@ module.exports = {
       },
       screens: {
         "xs": "480px",
-        "2xl": "1400px",
+        "2xl": "1440px",
+        "3xl": "1920px",
       },
     },
   },