From 509e46adc8c3056ba9c11bdc8b967c4ee9b9ecbe Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Wed, 21 Feb 2024 13:07:32 -0700 Subject: [PATCH] Review segment UI (#9945) * Add ui for events * Display data for review items * Use preview thumbnails * Implement paging * Show icons for what was detected * Show progress bar on preview thumbnail * Hide the overlays on hover and update reviewed status * Dim previews that have been reviewed * Show audio icons * Cleanup preview thumb player * initial implementation of review timeline * Use timeout for hover playback * Break apart mobile and desktop views * Show icons for sub labels * autoplay first video on mobile * Only show the last 24 hours by default * Rework scrolling to fix nested scrolling * Final scroll cleanups --------- Co-authored-by: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> --- frigate/http.py | 34 ++ web/package-lock.json | 91 ++++ web/package.json | 3 + web/src/App.tsx | 6 +- web/src/components/Wrapper.tsx | 2 +- web/src/components/card/HistoryCard.tsx | 11 +- .../player/PreviewThumbnailPlayer.tsx | 385 ++++++++--------- .../timeline/EventReviewTimeline.tsx | 4 +- web/src/components/ui/slider.tsx | 17 +- web/src/components/ui/toggle-group.tsx | 59 +++ web/src/components/ui/toggle.tsx | 43 ++ web/src/hooks/use-segment-utils.ts | 2 +- web/src/pages/Events.tsx | 11 + web/src/pages/Live.tsx | 9 +- web/src/pages/Logs.tsx | 6 +- web/src/pages/site-navigation.ts | 62 +-- web/src/types/review.ts | 35 +- web/src/utils/browserUtil.ts | 7 - web/src/utils/iconUtil.tsx | 49 +++ web/src/views/events/DesktopEventView.tsx | 392 ++++++++++++++++++ web/src/views/events/MobileEventView.tsx | 311 ++++++++++++++ 21 files changed, 1262 insertions(+), 277 deletions(-) create mode 100644 web/src/components/ui/toggle-group.tsx create mode 100644 web/src/components/ui/toggle.tsx create mode 100644 web/src/pages/Events.tsx delete mode 100644 web/src/utils/browserUtil.ts create mode 100644 web/src/utils/iconUtil.tsx create mode 100644 web/src/views/events/DesktopEventView.tsx create mode 100644 web/src/views/events/MobileEventView.tsx diff --git a/frigate/http.py b/frigate/http.py index 179094b07..38a16b299 100644 --- a/frigate/http.py +++ b/frigate/http.py @@ -2420,6 +2420,40 @@ def review(): return jsonify([r for r in review]) +@bp.route("/review//viewed", methods=("POST",)) +def set_reviewed(id): + try: + review: ReviewSegment = ReviewSegment.get(ReviewSegment.id == id) + except DoesNotExist: + return make_response( + jsonify({"success": False, "message": "Review " + id + " not found"}), 404 + ) + + review.has_been_reviewed = True + review.save() + + return make_response( + jsonify({"success": True, "message": "Reviewed " + id + " viewed"}), 200 + ) + + +@bp.route("/review//viewed", methods=("DELETE",)) +def set_not_reviewed(id): + try: + review: ReviewSegment = ReviewSegment.get(ReviewSegment.id == id) + except DoesNotExist: + return make_response( + jsonify({"success": False, "message": "Review " + id + " not found"}), 404 + ) + + review.has_been_reviewed = False + review.save() + + return make_response( + jsonify({"success": True, "message": "Reviewed " + id + " not viewed"}), 200 + ) + + @bp.route( "/export//start//end/", methods=["POST"] ) diff --git a/web/package-lock.json b/web/package-lock.json index 27793517d..aa997a927 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -23,6 +23,8 @@ "@radix-ui/react-slot": "^1.0.2", "@radix-ui/react-switch": "^1.0.3", "@radix-ui/react-tabs": "^1.0.4", + "@radix-ui/react-toggle": "^1.0.3", + "@radix-ui/react-toggle-group": "^1.0.4", "@radix-ui/react-tooltip": "^1.0.7", "apexcharts": "^3.45.1", "axios": "^1.6.2", @@ -38,6 +40,7 @@ "react": "^18.2.0", "react-apexcharts": "^1.4.1", "react-day-picker": "^8.9.1", + "react-device-detect": "^2.2.3", "react-dom": "^18.2.0", "react-hook-form": "^7.48.2", "react-icons": "^4.12.0", @@ -1800,6 +1803,60 @@ } } }, + "node_modules/@radix-ui/react-toggle": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-toggle/-/react-toggle-1.0.3.tgz", + "integrity": "sha512-Pkqg3+Bc98ftZGsl60CLANXQBBQ4W3mTFS9EJvNxKMZ7magklKV69/id1mlAlOFDDfHvlCms0fx8fA4CMKDJHg==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/primitive": "1.0.1", + "@radix-ui/react-primitive": "1.0.3", + "@radix-ui/react-use-controllable-state": "1.0.1" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toggle-group": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-toggle-group/-/react-toggle-group-1.0.4.tgz", + "integrity": "sha512-Uaj/M/cMyiyT9Bx6fOZO0SAG4Cls0GptBWiBmBxofmDbNVnYYoyRWj/2M/6VCi/7qcXFWnHhRUfdfZFvvkuu8A==", + "dependencies": { + "@babel/runtime": "^7.13.10", + "@radix-ui/primitive": "1.0.1", + "@radix-ui/react-context": "1.0.1", + "@radix-ui/react-direction": "1.0.1", + "@radix-ui/react-primitive": "1.0.3", + "@radix-ui/react-roving-focus": "1.0.4", + "@radix-ui/react-toggle": "1.0.3", + "@radix-ui/react-use-controllable-state": "1.0.1" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0", + "react-dom": "^16.8 || ^17.0 || ^18.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, "node_modules/@radix-ui/react-tooltip": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/@radix-ui/react-tooltip/-/react-tooltip-1.0.7.tgz", @@ -6748,6 +6805,18 @@ "react": "^16.8.0 || ^17.0.0 || ^18.0.0" } }, + "node_modules/react-device-detect": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/react-device-detect/-/react-device-detect-2.2.3.tgz", + "integrity": "sha512-buYY3qrCnQVlIFHrC5UcUoAj7iANs/+srdkwsnNjI7anr3Tt7UY6MqNxtMLlr0tMBied0O49UZVK8XKs3ZIiPw==", + "dependencies": { + "ua-parser-js": "^1.0.33" + }, + "peerDependencies": { + "react": ">= 0.14.0", + "react-dom": ">= 0.14.0" + } + }, "node_modules/react-dom": { "version": "18.2.0", "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.2.0.tgz", @@ -8062,6 +8131,28 @@ "node": ">=14.17" } }, + "node_modules/ua-parser-js": { + "version": "1.0.37", + "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-1.0.37.tgz", + "integrity": "sha512-bhTyI94tZofjo+Dn8SN6Zv8nBDvyXTymAdM3LDI/0IboIUwTu1rEhW7v2TfiVsoYWgkQ4kOVqnI8APUFbIQIFQ==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/ua-parser-js" + }, + { + "type": "paypal", + "url": "https://paypal.me/faisalman" + }, + { + "type": "github", + "url": "https://github.com/sponsors/faisalman" + } + ], + "engines": { + "node": "*" + } + }, "node_modules/ufo": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/ufo/-/ufo-1.3.2.tgz", diff --git a/web/package.json b/web/package.json index bfc508ff7..61ef26429 100644 --- a/web/package.json +++ b/web/package.json @@ -28,6 +28,8 @@ "@radix-ui/react-slot": "^1.0.2", "@radix-ui/react-switch": "^1.0.3", "@radix-ui/react-tabs": "^1.0.4", + "@radix-ui/react-toggle": "^1.0.3", + "@radix-ui/react-toggle-group": "^1.0.4", "@radix-ui/react-tooltip": "^1.0.7", "apexcharts": "^3.45.1", "axios": "^1.6.2", @@ -43,6 +45,7 @@ "react": "^18.2.0", "react-apexcharts": "^1.4.1", "react-day-picker": "^8.9.1", + "react-device-detect": "^2.2.3", "react-dom": "^18.2.0", "react-hook-form": "^7.48.2", "react-icons": "^4.12.0", diff --git a/web/src/App.tsx b/web/src/App.tsx index 8a22b0842..2ad151c0f 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -14,6 +14,7 @@ import Logs from "@/pages/Logs"; import NoMatch from "@/pages/NoMatch"; import Settings from "@/pages/Settings"; import UIPlayground from "./pages/UIPlayground"; +import Events from "./pages/Events"; function App() { const [sheetOpen, setSheetOpen] = useState(false); @@ -27,14 +28,15 @@ function App() {
-
+
} /> + } /> } /> } /> } /> diff --git a/web/src/components/Wrapper.tsx b/web/src/components/Wrapper.tsx index e9920b76b..0726d280e 100644 --- a/web/src/components/Wrapper.tsx +++ b/web/src/components/Wrapper.tsx @@ -5,7 +5,7 @@ type TWrapperProps = { }; const Wrapper = ({ children }: TWrapperProps) => { - return
{children}
; + return
{children}
; }; export default Wrapper; diff --git a/web/src/components/card/HistoryCard.tsx b/web/src/components/card/HistoryCard.tsx index ce880fc08..11f72c8c7 100644 --- a/web/src/components/card/HistoryCard.tsx +++ b/web/src/components/card/HistoryCard.tsx @@ -1,5 +1,4 @@ import useSWR from "swr"; -import PreviewThumbnailPlayer from "../player/PreviewThumbnailPlayer"; import { Card } from "../ui/card"; import { FrigateConfig } from "@/types/frigateConfig"; import ActivityIndicator from "../ui/activity-indicator"; @@ -21,8 +20,10 @@ type HistoryCardProps = { }; export default function HistoryCard({ + // @ts-ignore relevantPreview, timeline, + // @ts-ignore isMobile, onClick, onDelete, @@ -38,14 +39,6 @@ export default function HistoryCard({ className="cursor-pointer my-2 xs:mr-2 w-full xs:w-[48%] sm:w-[284px]" onClick={onClick} > - <>
diff --git a/web/src/components/player/PreviewThumbnailPlayer.tsx b/web/src/components/player/PreviewThumbnailPlayer.tsx index 068354ce8..97050d3ed 100644 --- a/web/src/components/player/PreviewThumbnailPlayer.tsx +++ b/web/src/components/player/PreviewThumbnailPlayer.tsx @@ -2,23 +2,26 @@ import VideoPlayer from "./VideoPlayer"; import React, { useCallback, useEffect, + useMemo, useRef, useState, } from "react"; import { useApiHost } from "@/api"; import Player from "video.js/dist/types/player"; -import { AspectRatio } from "../ui/aspect-ratio"; -import { LuPlayCircle } from "react-icons/lu"; -import { isCurrentHour } from "@/utils/dateUtil"; -import { isSafari } from "@/utils/browserUtil"; +import { formatUnixTimestampToDateTime } from "@/utils/dateUtil"; +import { ReviewSegment } from "@/types/review"; +import { Slider } from "../ui/slider"; +import { getIconForLabel, getIconForSubLabel } from "@/utils/iconUtil"; +import TimeAgo from "../dynamic/TimeAgo"; +import useSWR from "swr"; +import { FrigateConfig } from "@/types/frigateConfig"; +import { isMobile, isSafari } from "react-device-detect"; type PreviewPlayerProps = { - camera: string; + review: ReviewSegment; relevantPreview?: Preview; - startTs: number; - eventId: string; - isMobile: boolean; - onClick?: () => void; + autoPlayback?: boolean; + setReviewed?: () => void; }; type Preview = { @@ -30,17 +33,43 @@ type Preview = { }; export default function PreviewThumbnailPlayer({ - camera, + review, relevantPreview, - startTs, - eventId, - isMobile, - onClick, + autoPlayback = false, + setReviewed, }: PreviewPlayerProps) { + const apiHost = useApiHost(); + const { data: config } = useSWR("config"); const playerRef = useRef(null); - const [visible, setVisible] = useState(false); - const [isInitiallyVisible, setIsInitiallyVisible] = useState(false); + const [hoverTimeout, setHoverTimeout] = useState(); + const [playback, setPlayback] = useState(false); + const [progress, setProgress] = useState(0); + + const playingBack = useMemo( + () => relevantPreview && playback, + [playback, autoPlayback, relevantPreview] + ); + + useEffect(() => { + if (!autoPlayback) { + setPlayback(false); + + if (hoverTimeout) { + clearTimeout(hoverTimeout); + } + return; + } + + const timeout = setTimeout(() => { + setPlayback(true); + setHoverTimeout(null); + }, 500); + + return () => { + clearTimeout(timeout); + }; + }, [autoPlayback]); const onPlayback = useCallback( (isHovered: Boolean) => { @@ -48,205 +77,181 @@ export default function PreviewThumbnailPlayer({ return; } - if (!playerRef.current) { - if (isHovered) { - setIsInitiallyVisible(true); - } - - return; - } - if (isHovered) { - playerRef.current.play(); + setHoverTimeout( + setTimeout(() => { + setPlayback(true); + setHoverTimeout(null); + }, 500) + ); } else { - playerRef.current.pause(); - playerRef.current.currentTime(startTs - relevantPreview.start); - } - }, - [relevantPreview, startTs, playerRef] - ); - - const autoPlayObserver = useRef(); - const preloadObserver = useRef(); - const inViewRef = useCallback( - (node: HTMLElement | null) => { - if (!preloadObserver.current) { - try { - preloadObserver.current = new IntersectionObserver( - (entries) => { - const [{ isIntersecting }] = entries; - setVisible(isIntersecting); - }, - { - threshold: 0, - root: document.getElementById("pageRoot"), - rootMargin: "10% 0px 25% 0px", - } - ); - if (node) preloadObserver.current.observe(node); - } catch (e) { - // no op + if (hoverTimeout) { + clearTimeout(hoverTimeout); } - } - if (isMobile && !autoPlayObserver.current) { - try { - autoPlayObserver.current = new IntersectionObserver( - (entries) => { - const [{ isIntersecting }] = entries; - if (isIntersecting) { - onPlayback(true); - } else { - onPlayback(false); - } - }, - { - threshold: 1.0, - root: document.getElementById("pageRoot"), - rootMargin: "-10% 0px -25% 0px", - } + setPlayback(false); + setProgress(0); + + if (playerRef.current) { + playerRef.current.pause(); + playerRef.current.currentTime( + review.start_time - relevantPreview.start ); - if (node) autoPlayObserver.current.observe(node); - } catch (e) { - // no op } } }, - [preloadObserver, autoPlayObserver, onPlayback] + [hoverTimeout, relevantPreview, review, playerRef] ); return ( - onPlayback(true)} - onMouseLeave={() => onPlayback(false)} +
onPlayback(true)} + onMouseLeave={isMobile ? undefined : () => onPlayback(false)} > - - + {playingBack ? ( + + ) : ( + + )} + {!playingBack && + (review.severity == "alert" || review.severity == "detection") && ( +
+ {review.data.objects.map((object) => { + return getIconForLabel(object, "w-3 h-3 text-white"); + })} + {review.data.audio.map((audio) => { + return getIconForLabel(audio, "w-3 h-3 text-white"); + })} + {review.data.sub_labels?.map((sub) => { + return getIconForSubLabel(sub, "w-3 h-3 text-white"); + })} +
+ )} + {!playingBack && ( +
+ + {config && + formatUnixTimestampToDateTime(review.start_time, { + strftime_fmt: + config.ui.time_format == "24hour" + ? "%b %-d, %H:%M" + : "%b %-d, %I:%M %p", + })} +
+ )} +
+
+ {playingBack && ( + + )} + {!playingBack && review.has_been_reviewed && ( +
+ )} +
); } type PreviewContentProps = { playerRef: React.MutableRefObject; - camera: string; + review: ReviewSegment; relevantPreview: Preview | undefined; - eventId: string; - isVisible: boolean; - isInitiallyVisible: boolean; - startTs: number; - isMobile: boolean; - onClick?: () => void; + playback: boolean; + setProgress?: (progress: number) => void; + setReviewed?: () => void; }; function PreviewContent({ playerRef, - camera, + review, relevantPreview, - eventId, - isVisible, - isInitiallyVisible, - startTs, - isMobile, - onClick, + playback, + setProgress, + setReviewed, }: PreviewContentProps) { - const apiHost = useApiHost(); - const slowPlayack = isSafari(); - - // handle touchstart -> touchend as click - const [touchStart, setTouchStart] = useState(0); - const handleTouchStart = useCallback(() => { - setTouchStart(new Date().getTime()); - }, []); - useEffect(() => { - if (!isMobile || !playerRef.current || !onClick) { - return; - } - - playerRef.current.on("touchend", () => { - if (!onClick) { - return; - } - - const touchEnd = new Date().getTime(); - - // consider tap less than 100 ms - if (touchEnd - touchStart < 100) { - onClick(); - } - }); - }, [playerRef, touchStart]); - - if (relevantPreview && !isVisible) { - return
; - } else if (!relevantPreview && !isCurrentHour(startTs)) { + if (relevantPreview && playback) { return ( - { + playerRef.current = player; + + if (!relevantPreview) { + return; + } + + // start with a bit of padding + const playerStartTime = Math.max( + 0, + review.start_time - relevantPreview.start - 8 + ); + + player.playbackRate(isSafari ? 2 : 8); + player.currentTime(playerStartTime); + player.on("timeupdate", () => { + if (!setProgress || playerRef.current?.paused()) { + return; + } + + const playerProgress = + (player.currentTime() || 0) - playerStartTime; + + // end with a bit of padding + const playerDuration = review.end_time - review.start_time + 8; + const playerPercent = (playerProgress / playerDuration) * 100; + + if ( + setReviewed && + !review.has_been_reviewed && + playerPercent > 50 + ) { + setReviewed(); + } + + if (playerPercent > 100) { + playerRef.current?.pause(); + setProgress(100.0); + } else { + setProgress(playerPercent); + } + }); + }} + onDispose={() => { + playerRef.current = null; + }} /> ); - } else { - return ( - <> -
- { - playerRef.current = player; - - if (!relevantPreview) { - return; - } - - if (!isInitiallyVisible) { - player.pause(); // autoplay + pause is required for iOS - } - - player.playbackRate(slowPlayack ? 2 : 8); - player.currentTime(startTs - relevantPreview.start); - if (isMobile && onClick) { - player.on("touchstart", handleTouchStart); - } - }} - onDispose={() => { - playerRef.current = null; - }} - /> -
- {relevantPreview && ( - - )} - - ); } } diff --git a/web/src/components/timeline/EventReviewTimeline.tsx b/web/src/components/timeline/EventReviewTimeline.tsx index 73d5fae10..725d0de3c 100644 --- a/web/src/components/timeline/EventReviewTimeline.tsx +++ b/web/src/components/timeline/EventReviewTimeline.tsx @@ -47,7 +47,7 @@ export function EventReviewTimeline({ const currentTimeRef = useRef(null); const observer = useRef(null); const timelineDuration = useMemo( - () => timelineEnd - timelineStart, + () => timelineStart - timelineEnd, [timelineEnd, timelineStart] ); @@ -208,7 +208,7 @@ export function EventReviewTimeline({ return (
diff --git a/web/src/components/ui/slider.tsx b/web/src/components/ui/slider.tsx index e161daec0..2ed9fa769 100644 --- a/web/src/components/ui/slider.tsx +++ b/web/src/components/ui/slider.tsx @@ -1,7 +1,7 @@ -import * as React from "react" -import * as SliderPrimitive from "@radix-ui/react-slider" +import * as React from "react"; +import * as SliderPrimitive from "@radix-ui/react-slider"; -import { cn } from "@/lib/utils" +import { cn } from "@/lib/utils"; const Slider = React.forwardRef< React.ElementRef, @@ -15,12 +15,11 @@ const Slider = React.forwardRef< )} {...props} > - - + + - -)) -Slider.displayName = SliderPrimitive.Root.displayName +)); +Slider.displayName = SliderPrimitive.Root.displayName; -export { Slider } +export { Slider }; diff --git a/web/src/components/ui/toggle-group.tsx b/web/src/components/ui/toggle-group.tsx new file mode 100644 index 000000000..19505f9a4 --- /dev/null +++ b/web/src/components/ui/toggle-group.tsx @@ -0,0 +1,59 @@ +import * as React from "react" +import * as ToggleGroupPrimitive from "@radix-ui/react-toggle-group" +import { VariantProps } from "class-variance-authority" + +import { cn } from "@/lib/utils" +import { toggleVariants } from "@/components/ui/toggle" + +const ToggleGroupContext = React.createContext< + VariantProps +>({ + size: "default", + variant: "default", +}) + +const ToggleGroup = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef & + VariantProps +>(({ className, variant, size, children, ...props }, ref) => ( + + + {children} + + +)) + +ToggleGroup.displayName = ToggleGroupPrimitive.Root.displayName + +const ToggleGroupItem = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef & + VariantProps +>(({ className, children, variant, size, ...props }, ref) => { + const context = React.useContext(ToggleGroupContext) + + return ( + + {children} + + ) +}) + +ToggleGroupItem.displayName = ToggleGroupPrimitive.Item.displayName + +export { ToggleGroup, ToggleGroupItem } diff --git a/web/src/components/ui/toggle.tsx b/web/src/components/ui/toggle.tsx new file mode 100644 index 000000000..9ecac28ee --- /dev/null +++ b/web/src/components/ui/toggle.tsx @@ -0,0 +1,43 @@ +import * as React from "react" +import * as TogglePrimitive from "@radix-ui/react-toggle" +import { cva, type VariantProps } from "class-variance-authority" + +import { cn } from "@/lib/utils" + +const toggleVariants = cva( + "inline-flex items-center justify-center rounded-md text-sm font-medium ring-offset-background transition-colors hover:bg-muted hover:text-muted-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 data-[state=on]:bg-accent data-[state=on]:text-accent-foreground", + { + variants: { + variant: { + default: "bg-transparent", + outline: + "border border-input bg-transparent hover:bg-accent hover:text-accent-foreground", + }, + size: { + default: "h-10 px-3", + sm: "h-9 px-2.5", + lg: "h-11 px-5", + }, + }, + defaultVariants: { + variant: "default", + size: "default", + }, + } +) + +const Toggle = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef & + VariantProps +>(({ className, variant, size, ...props }, ref) => ( + +)) + +Toggle.displayName = TogglePrimitive.Root.displayName + +export { Toggle, toggleVariants } diff --git a/web/src/hooks/use-segment-utils.ts b/web/src/hooks/use-segment-utils.ts index c9d00002d..967bd3586 100644 --- a/web/src/hooks/use-segment-utils.ts +++ b/web/src/hooks/use-segment-utils.ts @@ -164,4 +164,4 @@ export const useSegmentUtils = ( getReviewed, shouldShowRoundedCorners, }; -}; +}; \ No newline at end of file diff --git a/web/src/pages/Events.tsx b/web/src/pages/Events.tsx new file mode 100644 index 000000000..0a465afe8 --- /dev/null +++ b/web/src/pages/Events.tsx @@ -0,0 +1,11 @@ +import DesktopEventView from "@/views/events/DesktopEventView"; +import MobileEventView from "@/views/events/MobileEventView"; +import { isMobile } from 'react-device-detect'; + +export default function Events() { + if (isMobile) { + return ; + } + + return ; +} diff --git a/web/src/pages/Live.tsx b/web/src/pages/Live.tsx index 7042d48fd..fd13a4a9c 100644 --- a/web/src/pages/Live.tsx +++ b/web/src/pages/Live.tsx @@ -5,8 +5,8 @@ import { ScrollArea, ScrollBar } from "@/components/ui/scroll-area"; import { TooltipProvider } from "@/components/ui/tooltip"; import { Event as FrigateEvent } from "@/types/event"; import { FrigateConfig } from "@/types/frigateConfig"; -import { isSafari } from "@/utils/browserUtil"; import { useCallback, useEffect, useMemo, useState } from "react"; +import { isSafari } from "react-device-detect"; import useSWR from "swr"; function Live() { @@ -65,7 +65,6 @@ function Live() { .sort((aConf, bConf) => aConf.ui.order - bConf.ui.order); }, [config]); - const safari = isSafari(); const [windowVisible, setWindowVisible] = useState(true); const visibilityListener = useCallback(() => { setWindowVisible(document.visibilityState == "visible"); @@ -80,7 +79,7 @@ function Live() { }, []); return ( - <> +
{events && events.length > 0 && ( @@ -111,12 +110,12 @@ function Live() { className={`mb-2 md:mb-0 rounded-2xl bg-black ${grow}`} windowVisible={windowVisible} cameraConfig={camera} - preferredLiveMode={safari ? "webrtc" : "mse"} + preferredLiveMode={isSafari ? "webrtc" : "mse"} /> ); })}
- +
); } diff --git a/web/src/pages/Logs.tsx b/web/src/pages/Logs.tsx index 26966faec..00c20c18c 100644 --- a/web/src/pages/Logs.tsx +++ b/web/src/pages/Logs.tsx @@ -41,7 +41,7 @@ function Logs() { }, [logs]); return ( - <> +
Logs @@ -76,10 +76,10 @@ function Logs() {
-
+
{logs}
- +
); } diff --git a/web/src/pages/site-navigation.ts b/web/src/pages/site-navigation.ts index ab94a9cd7..fab2bbb3c 100644 --- a/web/src/pages/site-navigation.ts +++ b/web/src/pages/site-navigation.ts @@ -1,34 +1,34 @@ import { - LuConstruction, - LuFileUp, - LuFilm, - LuVideo, - } from "react-icons/lu"; + LuConstruction, + LuFileUp, + LuFlag, + LuVideo, +} from "react-icons/lu"; export const navbarLinks = [ - { - id: 1, - icon: LuVideo, - title: "Live", - url: "/", - }, - { - id: 2, - icon: LuFilm, - title: "History", - url: "/history", - }, - { - id: 3, - icon: LuFileUp, - title: "Export", - url: "/export", - }, - { - id: 4, - icon: LuConstruction, - title: "UI Playground", - url: "/playground", - dev: true, - }, - ]; \ No newline at end of file + { + id: 1, + icon: LuVideo, + title: "Live", + url: "/", + }, + { + id: 2, + icon: LuFlag, + title: "Events", + url: "/events", + }, + { + id: 3, + icon: LuFileUp, + title: "Export", + url: "/export", + }, + { + id: 4, + icon: LuConstruction, + title: "UI Playground", + url: "/playground", + dev: true, + }, +]; diff --git a/web/src/types/review.ts b/web/src/types/review.ts index 16dc2defa..55a08a819 100644 --- a/web/src/types/review.ts +++ b/web/src/types/review.ts @@ -1,20 +1,21 @@ export interface ReviewSegment { - id: string; - camera: string; - severity: ReviewSeverity; - start_time: number; - end_time: number; - thumb_path: string; - has_been_reviewed: boolean; - data: ReviewData; - } + id: string; + camera: string; + severity: ReviewSeverity; + start_time: number; + end_time: number; + thumb_path: string; + has_been_reviewed: boolean; + data: ReviewData; +} - export type ReviewSeverity = "alert" | "detection" | "significant_motion"; +export type ReviewSeverity = "alert" | "detection" | "significant_motion"; - export type ReviewData = { - audio: string[]; - detections: string[]; - objects: string[]; - significant_motion_areas: number[]; - zones: string[]; - }; \ No newline at end of file +export type ReviewData = { + audio: string[]; + detections: string[]; + objects: string[]; + sub_labels?: string[]; + significant_motion_areas: number[]; + zones: string[]; +}; diff --git a/web/src/utils/browserUtil.ts b/web/src/utils/browserUtil.ts deleted file mode 100644 index ca788309d..000000000 --- a/web/src/utils/browserUtil.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { useMemo } from "react"; - -export function isSafari() { - return useMemo(() => { - return /^((?!chrome|android).)*safari/i.test(navigator.userAgent); - }, []); -} diff --git a/web/src/utils/iconUtil.tsx b/web/src/utils/iconUtil.tsx new file mode 100644 index 000000000..f31aec318 --- /dev/null +++ b/web/src/utils/iconUtil.tsx @@ -0,0 +1,49 @@ +import { BsPersonWalking } from "react-icons/bs"; +import { + FaAmazon, + FaCarSide, + FaCat, + FaDog, + FaFedex, + FaFire, + FaUps, +} from "react-icons/fa"; +import { LuBox, LuLassoSelect } from "react-icons/lu"; +import { MdRecordVoiceOver } from "react-icons/md"; + +export function getIconForLabel(label: string, className?: string) { + switch (label) { + case "car": + return ; + case "cat": + return ; + case "bark": + case "dog": + return ; + case "fire_alarm": + return ; + case "package": + return ; + case "person": + return ; + case "crying": + case "laughter": + case "scream": + case "speech": + case "yell": + return ; + default: + return ; + } +} + +export function getIconForSubLabel(label: string, className?: string) { + switch (label) { + case "amazon": + return ; + case "fedex": + return ; + case "ups": + return ; + } +} diff --git a/web/src/views/events/DesktopEventView.tsx b/web/src/views/events/DesktopEventView.tsx new file mode 100644 index 000000000..78e42e450 --- /dev/null +++ b/web/src/views/events/DesktopEventView.tsx @@ -0,0 +1,392 @@ +import PreviewThumbnailPlayer from "@/components/player/PreviewThumbnailPlayer"; +import EventReviewTimeline from "@/components/timeline/EventReviewTimeline"; +import ActivityIndicator from "@/components/ui/activity-indicator"; +import { Button } from "@/components/ui/button"; +import { Calendar } from "@/components/ui/calendar"; +import { + Popover, + PopoverContent, + PopoverTrigger, +} from "@/components/ui/popover"; +import { ToggleGroup, ToggleGroupItem } from "@/components/ui/toggle-group"; +import { FrigateConfig } from "@/types/frigateConfig"; +import { ReviewSegment, ReviewSeverity } from "@/types/review"; +import { formatUnixTimestampToDateTime } from "@/utils/dateUtil"; +import axios from "axios"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { LuCalendar, LuFilter, LuVideo } from "react-icons/lu"; +import { MdCircle } from "react-icons/md"; +import useSWR from "swr"; +import useSWRInfinite from "swr/infinite"; + +const API_LIMIT = 250; + +export default function DesktopEventView() { + const { data: config } = useSWR("config"); + const [severity, setSeverity] = useState("alert"); + const contentRef = useRef(null); + + // review paging + + const [after, setAfter] = useState(0); + useEffect(() => { + const now = new Date(); + now.setHours(now.getHours() - 24); + setAfter(now.getTime() / 1000); + + const intervalId: NodeJS.Timeout = setInterval(() => { + const now = new Date(); + now.setHours(now.getHours() - 24); + setAfter(now.getTime() / 1000); + }, 60000); + return () => clearInterval(intervalId); + }, [60000]); + + const reviewSearchParams = {}; + const reviewSegmentFetcher = useCallback((key: any) => { + const [path, params] = Array.isArray(key) ? key : [key, undefined]; + return axios.get(path, { params }).then((res) => res.data); + }, []); + + const getKey = useCallback( + (index: number, prevData: ReviewSegment[]) => { + if (index > 0) { + const lastDate = prevData[prevData.length - 1].start_time; + const pagedParams = reviewSearchParams + ? { before: lastDate, after: after, limit: API_LIMIT } + : { + ...reviewSearchParams, + before: lastDate, + after: after, + limit: API_LIMIT, + }; + return ["review", pagedParams]; + } + + const params = reviewSearchParams + ? { limit: API_LIMIT, after: after } + : { ...reviewSearchParams, limit: API_LIMIT, after: after }; + return ["review", params]; + }, + [reviewSearchParams] + ); + + const { + data: reviewPages, + mutate: updateSegments, + size, + setSize, + isValidating, + } = useSWRInfinite(getKey, reviewSegmentFetcher); + + const reviewItems = useMemo(() => { + const all: ReviewSegment[] = []; + const alerts: ReviewSegment[] = []; + const detections: ReviewSegment[] = []; + const motion: ReviewSegment[] = []; + + reviewPages?.forEach((page) => { + page.forEach((segment) => { + all.push(segment); + + switch (segment.severity) { + case "alert": + alerts.push(segment); + break; + case "detection": + detections.push(segment); + break; + default: + motion.push(segment); + break; + } + }); + }); + + return { + all: all, + alert: alerts, + detection: detections, + significant_motion: motion, + }; + }, [reviewPages]); + + const isDone = useMemo( + () => (reviewPages?.at(-1)?.length ?? 0) < API_LIMIT, + [reviewPages] + ); + + // review interaction + + const pagingObserver = useRef(); + const lastReviewRef = useCallback( + (node: HTMLElement | null) => { + if (isValidating) return; + if (pagingObserver.current) pagingObserver.current.disconnect(); + try { + pagingObserver.current = new IntersectionObserver((entries) => { + if (entries[0].isIntersecting && !isDone) { + setSize(size + 1); + } + }); + if (node) pagingObserver.current.observe(node); + } catch (e) { + // no op + } + }, + [isValidating, isDone] + ); + + const [minimap, setMinimap] = useState([]); + const minimapObserver = useRef(); + useEffect(() => { + if (!contentRef.current) { + return; + } + + const visibleTimestamps = new Set(); + minimapObserver.current = new IntersectionObserver( + (entries) => { + entries.forEach((entry) => { + const start = (entry.target as HTMLElement).dataset.start; + + if (!start) { + return; + } + + if (entry.isIntersecting) { + visibleTimestamps.add(start); + } else { + visibleTimestamps.delete(start); + } + + setMinimap([...visibleTimestamps]); + }); + }, + { root: contentRef.current, threshold: 0.5 } + ); + + return () => { + minimapObserver.current?.disconnect(); + }; + }, [contentRef]); + const minimapRef = useCallback( + (node: HTMLElement | null) => { + if (!minimapObserver.current) { + return; + } + + try { + if (node) minimapObserver.current.observe(node); + } catch (e) { + // no op + } + }, + [minimapObserver.current] + ); + const minimapBounds = useMemo(() => { + const data = { + start: Math.floor(Date.now() / 1000) - 35 * 60, + end: Math.floor(Date.now() / 1000) - 21 * 60, + }; + const list = minimap.sort(); + + if (list.length > 0) { + data.end = parseFloat(list.at(-1)!!); + data.start = parseFloat(list[0]); + } + + return data; + }, [minimap]); + + // review status + + const setReviewed = useCallback( + async (id: string) => { + const resp = await axios.post(`review/${id}/viewed`); + + if (resp.status == 200) { + updateSegments(); + } + }, + [updateSegments] + ); + + // preview videos + + const previewTimes = useMemo(() => { + if ( + !reviewPages || + reviewPages.length == 0 || + reviewPages.at(-1)!!.length == 0 + ) { + return undefined; + } + + const startDate = new Date(); + startDate.setMinutes(0, 0, 0); + + const endDate = new Date(reviewPages.at(-1)!!.at(-1)!!.end_time); + endDate.setHours(0, 0, 0, 0); + return { + start: startDate.getTime() / 1000, + end: endDate.getTime() / 1000, + }; + }, [reviewPages]); + const { data: allPreviews } = useSWR( + previewTimes + ? `preview/all/start/${previewTimes.start}/end/${previewTimes.end}` + : null, + { revalidateOnFocus: false } + ); + + if (!config) { + return ; + } + + console.log("end of the timeline is " + after + " vs " + (Math.floor(Date.now() / 1000) + 2 * 60 * 60)) + + return ( +
+
+ setSeverity(value)} + > + + + Alerts + + + + Detections + + + + Motion + + +
+ + + +
+
+ +
+ {reviewItems[severity]?.map((value, segIdx) => { + const lastRow = segIdx == reviewItems[severity].length - 1; + const relevantPreview = Object.values(allPreviews || []).find( + (preview) => + preview.camera == value.camera && + preview.start < value.start_time && + preview.end > value.end_time + ); + + return ( +
+
+ setReviewed(value.id)} + /> +
+ {lastRow && !isDone && } +
+ ); + })} +
+
+ {after != 0 && ()} +
+
+ ); +} + +/** + * + */ + +function ReviewCalendarButton() { + const disabledDates = useMemo(() => { + const tomorrow = new Date(); + tomorrow.setHours(tomorrow.getHours() + 24, -1, 0, 0); + const future = new Date(); + future.setFullYear(tomorrow.getFullYear() + 10); + return { from: tomorrow, to: future }; + }, []); + + return ( + + + + + + + + + ); +} diff --git a/web/src/views/events/MobileEventView.tsx b/web/src/views/events/MobileEventView.tsx new file mode 100644 index 000000000..eb4c9fb29 --- /dev/null +++ b/web/src/views/events/MobileEventView.tsx @@ -0,0 +1,311 @@ +import PreviewThumbnailPlayer from "@/components/player/PreviewThumbnailPlayer"; +import ActivityIndicator from "@/components/ui/activity-indicator"; +import { ToggleGroup, ToggleGroupItem } from "@/components/ui/toggle-group"; +import { FrigateConfig } from "@/types/frigateConfig"; +import { ReviewSegment, ReviewSeverity } from "@/types/review"; +import axios from "axios"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { MdCircle } from "react-icons/md"; +import useSWR from "swr"; +import useSWRInfinite from "swr/infinite"; + +const API_LIMIT = 250; + +export default function MobileEventView() { + const { data: config } = useSWR("config"); + const [severity, setSeverity] = useState("alert"); + const contentRef = useRef(null); + + // review paging + + const [after, setAfter] = useState(0); + useEffect(() => { + const now = new Date(); + now.setHours(now.getHours() - 24); + setAfter(now.getTime() / 1000); + + const intervalId: NodeJS.Timeout = setInterval(() => { + const now = new Date(); + now.setHours(now.getHours() - 24); + setAfter(now.getTime() / 1000); + }, 60000); + return () => clearInterval(intervalId); + }, [60000]); + + const reviewSearchParams = {}; + const reviewSegmentFetcher = useCallback((key: any) => { + const [path, params] = Array.isArray(key) ? key : [key, undefined]; + return axios.get(path, { params }).then((res) => res.data); + }, []); + + const getKey = useCallback( + (index: number, prevData: ReviewSegment[]) => { + if (index > 0) { + const lastDate = prevData[prevData.length - 1].start_time; + const pagedParams = reviewSearchParams + ? { before: lastDate, after: after, limit: API_LIMIT } + : { + ...reviewSearchParams, + before: lastDate, + after: after, + limit: API_LIMIT, + }; + return ["review", pagedParams]; + } + + const params = reviewSearchParams + ? { limit: API_LIMIT, after: after } + : { ...reviewSearchParams, limit: API_LIMIT, after: after }; + return ["review", params]; + }, + [reviewSearchParams] + ); + + const { + data: reviewPages, + mutate: updateSegments, + size, + setSize, + isValidating, + } = useSWRInfinite(getKey, reviewSegmentFetcher); + + const reviewItems = useMemo(() => { + const all: ReviewSegment[] = []; + const alerts: ReviewSegment[] = []; + const detections: ReviewSegment[] = []; + const motion: ReviewSegment[] = []; + + reviewPages?.forEach((page) => { + page.forEach((segment) => { + all.push(segment); + + switch (segment.severity) { + case "alert": + alerts.push(segment); + break; + case "detection": + detections.push(segment); + break; + default: + motion.push(segment); + break; + } + }); + }); + + return { + all: all, + alert: alerts, + detection: detections, + significant_motion: motion, + }; + }, [reviewPages]); + + const isDone = useMemo( + () => (reviewPages?.at(-1)?.length ?? 0) < API_LIMIT, + [reviewPages] + ); + + // review interaction + + const pagingObserver = useRef(); + const lastReviewRef = useCallback( + (node: HTMLElement | null) => { + if (isValidating) return; + if (pagingObserver.current) pagingObserver.current.disconnect(); + try { + pagingObserver.current = new IntersectionObserver((entries) => { + if (entries[0].isIntersecting && !isDone) { + setSize(size + 1); + } + }); + if (node) pagingObserver.current.observe(node); + } catch (e) { + // no op + } + }, + [isValidating, isDone] + ); + + const [minimap, setMinimap] = useState([]); + const minimapObserver = useRef(); + useEffect(() => { + if (!contentRef.current) { + return; + } + + const visibleTimestamps = new Set(); + minimapObserver.current = new IntersectionObserver( + (entries) => { + entries.forEach((entry) => { + const start = (entry.target as HTMLElement).dataset.start; + + if (!start) { + return; + } + + if (entry.isIntersecting) { + visibleTimestamps.add(start); + } else { + visibleTimestamps.delete(start); + } + + setMinimap([...visibleTimestamps]); + }); + }, + { threshold: 0.5 } + ); + + return () => { + minimapObserver.current?.disconnect(); + }; + }, [contentRef]); + const minimapRef = useCallback( + (node: HTMLElement | null) => { + if (!minimapObserver.current) { + return; + } + + try { + if (node) minimapObserver.current.observe(node); + } catch (e) { + // no op + } + }, + [minimapObserver.current] + ); + const minimapBounds = useMemo(() => { + const data = { + start: Math.floor(Date.now() / 1000) - 35 * 60, + end: Math.floor(Date.now() / 1000) - 21 * 60, + }; + const list = minimap.sort(); + + if (list.length > 0) { + data.end = parseFloat(list.at(-1)!!); + data.start = parseFloat(list[0]); + } + + return data; + }, [minimap]); + + // review status + + const setReviewed = useCallback( + async (id: string) => { + const resp = await axios.post(`review/${id}/viewed`); + + if (resp.status == 200) { + updateSegments(); + } + }, + [updateSegments] + ); + + // preview videos + + const previewTimes = useMemo(() => { + if ( + !reviewPages || + reviewPages.length == 0 || + reviewPages.at(-1)!!.length == 0 + ) { + return undefined; + } + + const startDate = new Date(); + startDate.setMinutes(0, 0, 0); + + const endDate = new Date(reviewPages.at(-1)!!.at(-1)!!.end_time); + endDate.setHours(0, 0, 0, 0); + return { + start: startDate.getTime() / 1000, + end: endDate.getTime() / 1000, + }; + }, [reviewPages]); + const { data: allPreviews } = useSWR( + previewTimes + ? `preview/all/start/${previewTimes.start}/end/${previewTimes.end}` + : null, + { revalidateOnFocus: false } + ); + + if (!config) { + return ; + } + + return ( + <> + setSeverity(value)} + > + + + Alerts + + + + Detections + + + + Motion + + + +
+ {reviewItems[severity]?.map((value, segIdx) => { + const lastRow = segIdx == reviewItems[severity].length - 1; + const relevantPreview = Object.values(allPreviews || []).find( + (preview) => + preview.camera == value.camera && + preview.start < value.start_time && + preview.end > value.end_time + ); + + return ( +
+
+ setReviewed(value.id)} + /> +
+ {lastRow && !isDone && } +
+ ); + })} +
+ + ); +}