From af3f6dadcb955d9fe2ee56e69c1c5e79f782dcc0 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Wed, 31 Jan 2024 05:29:18 -0700 Subject: [PATCH] Improve graph using pandas (#9234) * Ensure viewport is always full screen * Protect against hour with no cards and ensure data is consistent * Reduce grouped up image refreshes * Include current hour and fix scrubbing bugginess * Scroll initially selected timeline in to view * Expand timelne class type * Use poster image for preview on video player instead of using separate image view * Fix available streaming modes * Incrase timing for grouping timline items * Fix audio activity listener * Fix player not switching views correctly * Use player time to convert to timeline time * Update sub labels for previous timeline items * Show mini timeline bar for non selected items * Rewrite desktop timeline to use separate dynamic video player component * Extend improvements to mobile as well * Improve time formatting * Fix scroll * Fix no preview case * Mobile fixes * Audio toggle fixes * More fixes for mobile * Improve scaling of graph motion activity * Add keyboard shortcut hook and support shortcuts for playback page * Fix sizing of dialog * Improve height scaling of dialog * simplify and fix layout system for timeline * Fix timeilne items not working * Implement basic Frigate+ submitting from timeline --- docker/main/requirements-wheels.txt | 1 + frigate/http.py | 79 +++- frigate/timeline.py | 21 +- frigate/util/services.py | 2 +- web/src/components/Wrapper.tsx | 2 +- web/src/components/bar/TimelineBar.tsx | 191 ++++++++ .../components/camera/DynamicCameraImage.tsx | 24 +- web/src/components/card/TimelineItemCard.tsx | 140 ++++-- web/src/components/graph/TimelineGraph.tsx | 44 +- .../overlay/TimelineDataOverlay.tsx | 16 + .../components/player/DynamicVideoPlayer.tsx | 411 ++++++++++++++++ .../player/PreviewThumbnailPlayer.tsx | 81 ++-- .../components/scrubber/ActivityScrubber.tsx | 18 +- web/src/hooks/use-keyboard-listener.tsx | 43 ++ web/src/pages/ConfigEditor.tsx | 2 +- web/src/pages/Dashboard.tsx | 2 +- web/src/pages/History.tsx | 2 +- web/src/pages/Live.tsx | 118 +++-- web/src/types/history.ts | 27 -- web/src/types/playback.ts | 5 + web/src/types/record.ts | 4 +- web/src/types/timeline.ts | 31 ++ web/src/utils/dateUtil.ts | 10 +- web/src/utils/historyUtil.ts | 248 +++++----- web/src/utils/timelineUtil.tsx | 11 - web/src/views/history/DesktopTimelineView.tsx | 447 ++++++------------ web/src/views/history/MobileTimelineView.tsx | 250 ++-------- web/tailwind.config.js | 1 + 28 files changed, 1379 insertions(+), 852 deletions(-) create mode 100644 web/src/components/bar/TimelineBar.tsx create mode 100644 web/src/components/player/DynamicVideoPlayer.tsx create mode 100644 web/src/hooks/use-keyboard-listener.tsx create mode 100644 web/src/types/playback.ts create mode 100644 web/src/types/timeline.ts diff --git a/docker/main/requirements-wheels.txt b/docker/main/requirements-wheels.txt index f4167744e..5c98571be 100644 --- a/docker/main/requirements-wheels.txt +++ b/docker/main/requirements-wheels.txt @@ -7,6 +7,7 @@ numpy == 1.23.* onvif_zeep == 0.2.12 opencv-python-headless == 4.7.0.* paho-mqtt == 1.6.* +pandas == 2.1.4 peewee == 3.17.* peewee_migrate == 1.12.* psutil == 5.9.* diff --git a/frigate/http.py b/frigate/http.py index 4c7a01b67..0a671e92c 100644 --- a/frigate/http.py +++ b/frigate/http.py @@ -16,6 +16,7 @@ from urllib.parse import unquote import cv2 import numpy as np +import pandas as pd import pytz import requests from flask import ( @@ -390,6 +391,17 @@ def set_sub_label(id): new_sub_label = json.get("subLabel") new_score = json.get("subLabelScore") + if new_sub_label is None: + return make_response( + jsonify( + { + "success": False, + "message": "A sub label must be supplied", + } + ), + 400, + ) + if new_sub_label and len(new_sub_label) > 100: return make_response( jsonify( @@ -415,6 +427,7 @@ def set_sub_label(id): ) if not event.end_time: + # update tracked object tracked_obj: TrackedObject = ( current_app.detected_frames_processor.camera_states[ event.camera @@ -424,6 +437,11 @@ def set_sub_label(id): if tracked_obj: tracked_obj.obj_data["sub_label"] = (new_sub_label, new_score) + # update timeline items + Timeline.update( + data=Timeline.data.update({"sub_label": (new_sub_label, new_score)}) + ).where(Timeline.source_id == id).execute() + event.sub_label = new_sub_label if new_score: @@ -739,41 +757,59 @@ def hourly_timeline_activity(camera_name: str): # set initial start so data is representative of full hour hours[int(key.timestamp())].append( - { - "date": key.timestamp(), - "count": 0, - "type": "motion", - } + [ + key.timestamp(), + 0, + False, + ] ) for recording in all_recordings: if recording.start_time > check: hours[int(key.timestamp())].append( - { - "date": (key + timedelta(hours=1)).timestamp(), - "count": 0, - "type": "motion", - } + [ + (key + timedelta(minutes=59, seconds=59)).timestamp(), + 0, + False, + ] ) key = key + timedelta(hours=1) check = (key + timedelta(hours=1)).timestamp() hours[int(key.timestamp())].append( - { - "date": key.timestamp(), - "count": 0, - "type": "motion", - } + [ + key.timestamp(), + 0, + False, + ] ) - data_type = "motion" if recording.objects == 0 else "objects" + data_type = recording.objects > 0 + count = recording.motion + recording.objects hours[int(key.timestamp())].append( - { - "date": recording.start_time + (recording.duration / 2), - "count": recording.motion, - "type": data_type, - } + [ + recording.start_time + (recording.duration / 2), + 0 if count == 0 else np.log2(count), + data_type, + ] ) + # resample data using pandas to get activity on minute to minute basis + for key, data in hours.items(): + df = pd.DataFrame(data, columns=["date", "count", "hasObjects"]) + + # set date as datetime index + df["date"] = pd.to_datetime(df["date"], unit="s") + df.set_index(["date"], inplace=True) + + # normalize data + df = df.resample("T").mean().fillna(0) + + # change types for output + df.index = df.index.astype(int) // (10**9) + df["count"] = df["count"].astype(int) + df["hasObjects"] = df["hasObjects"].astype(bool) + hours[key] = df.reset_index().to_dict("records") + return jsonify(hours) @@ -1840,6 +1876,7 @@ def recordings(camera_name): Recordings.segment_size, Recordings.motion, Recordings.objects, + Recordings.duration, ) .where( Recordings.camera == camera_name, diff --git a/frigate/timeline.py b/frigate/timeline.py index 927fd6845..952bb36f4 100644 --- a/frigate/timeline.py +++ b/frigate/timeline.py @@ -7,7 +7,6 @@ from multiprocessing import Queue from multiprocessing.synchronize import Event as MpEvent from frigate.config import FrigateConfig -from frigate.const import ALL_ATTRIBUTE_LABELS from frigate.events.maintainer import EventTypeEnum from frigate.models import Timeline from frigate.util.builtin import to_relative_box @@ -85,12 +84,13 @@ class TimelineProcessor(threading.Thread): """Handle object detection.""" save = False camera_config = self.config.cameras[camera] + event_id = event_data["id"] timeline_entry = { Timeline.timestamp: event_data["frame_time"], Timeline.camera: camera, Timeline.source: "tracked_object", - Timeline.source_id: event_data["id"], + Timeline.source_id: event_id, Timeline.data: { "box": to_relative_box( camera_config.detect.width, @@ -107,6 +107,16 @@ class TimelineProcessor(threading.Thread): "attribute": "", }, } + + # update sub labels for existing entries that haven't been added yet + if ( + prev_event_data != None + and prev_event_data["sub_label"] != event_data["sub_label"] + and event_id in self.pre_event_cache.keys() + ): + for e in self.pre_event_cache[event_id]: + e[Timeline.data]["sub_label"] = event_data["sub_label"] + if event_type == "start": timeline_entry[Timeline.class_type] = "visible" save = True @@ -129,13 +139,6 @@ class TimelineProcessor(threading.Thread): event_data["attributes"].keys() )[0] save = True - elif not prev_event_data.get("sub_label") and event_data.get("sub_label"): - sub_label = event_data["sub_label"][0] - - if sub_label not in ALL_ATTRIBUTE_LABELS: - timeline_entry[Timeline.class_type] = "sub_label" - timeline_entry[Timeline.data]["sub_label"] = sub_label - save = True elif event_type == "end": timeline_entry[Timeline.class_type] = "gone" save = True diff --git a/frigate/util/services.py b/frigate/util/services.py index 1c3adbde0..4c22bab2c 100644 --- a/frigate/util/services.py +++ b/frigate/util/services.py @@ -378,7 +378,7 @@ def auto_detect_hwaccel() -> str: try: cuda = False vaapi = False - resp = requests.get("http://192.168.50.106:1984/api/ffmpeg/hardware", timeout=3) + resp = requests.get("http://127.0.0.1:1984/api/ffmpeg/hardware", timeout=3) if resp.status_code == 200: data: dict[str, list[dict[str, str]]] = resp.json() diff --git a/web/src/components/Wrapper.tsx b/web/src/components/Wrapper.tsx index 4a156f024..e9920b76b 100644 --- a/web/src/components/Wrapper.tsx +++ b/web/src/components/Wrapper.tsx @@ -5,7 +5,7 @@ type TWrapperProps = { }; const Wrapper = ({ children }: TWrapperProps) => { - return
{children}
; + return
{children}
; }; export default Wrapper; diff --git a/web/src/components/bar/TimelineBar.tsx b/web/src/components/bar/TimelineBar.tsx new file mode 100644 index 000000000..6b1b87ce3 --- /dev/null +++ b/web/src/components/bar/TimelineBar.tsx @@ -0,0 +1,191 @@ +import { FrigateConfig } from "@/types/frigateConfig"; +import { GraphDataPoint } from "@/types/graph"; +import { formatUnixTimestampToDateTime } from "@/utils/dateUtil"; +import useSWR from "swr"; +import ActivityIndicator from "../ui/activity-indicator"; + +type TimelineBarProps = { + startTime: number; + graphData: + | { + objects: number[]; + motion: GraphDataPoint[]; + } + | undefined; + onClick?: () => void; +}; +export default function TimelineBar({ + startTime, + graphData, + onClick, +}: TimelineBarProps) { + const { data: config } = useSWR("config"); + + if (!config) { + return ; + } + + return ( +
+ {graphData != undefined && ( +
+ {getHourBlocks().map((idx) => { + return ( +
+ ); + })} +
+
+ {formatUnixTimestampToDateTime(startTime, { + strftime_fmt: + config?.ui.time_format == "24hour" ? "%H:00" : "%I:00%P", + time_style: "medium", + date_style: "medium", + })} +
+
+
+
+ {formatUnixTimestampToDateTime(startTime, { + strftime_fmt: + config?.ui.time_format == "24hour" ? "%H:05" : "%I:05%P", + time_style: "medium", + date_style: "medium", + })} +
+
+
+
+ {formatUnixTimestampToDateTime(startTime, { + strftime_fmt: + config?.ui.time_format == "24hour" ? "%H:10" : "%I:10%P", + time_style: "medium", + date_style: "medium", + })} +
+
+
+
+ {formatUnixTimestampToDateTime(startTime, { + strftime_fmt: + config?.ui.time_format == "24hour" ? "%H:15" : "%I:15%P", + time_style: "medium", + date_style: "medium", + })} +
+
+
+
+ {formatUnixTimestampToDateTime(startTime, { + strftime_fmt: + config?.ui.time_format == "24hour" ? "%H:20" : "%I:20%P", + time_style: "medium", + date_style: "medium", + })} +
+
+
+
+ {formatUnixTimestampToDateTime(startTime, { + strftime_fmt: + config?.ui.time_format == "24hour" ? "%H:25" : "%I:25%P", + time_style: "medium", + date_style: "medium", + })} +
+
+
+
+ {formatUnixTimestampToDateTime(startTime, { + strftime_fmt: + config?.ui.time_format == "24hour" ? "%H:30" : "%I:30%P", + time_style: "medium", + date_style: "medium", + })} +
+
+
+
+ {formatUnixTimestampToDateTime(startTime, { + strftime_fmt: + config?.ui.time_format == "24hour" ? "%H:35" : "%I:35%P", + time_style: "medium", + date_style: "medium", + })} +
+
+
+
+ {formatUnixTimestampToDateTime(startTime, { + strftime_fmt: + config?.ui.time_format == "24hour" ? "%H:40" : "%I:40%P", + time_style: "medium", + date_style: "medium", + })} +
+
+
+
+ {formatUnixTimestampToDateTime(startTime, { + strftime_fmt: + config?.ui.time_format == "24hour" ? "%H:45" : "%I:45%P", + time_style: "medium", + date_style: "medium", + })} +
+
+
+
+ {formatUnixTimestampToDateTime(startTime, { + strftime_fmt: + config?.ui.time_format == "24hour" ? "%H:50" : "%I:50%P", + time_style: "medium", + date_style: "medium", + })} +
+
+
+
+ {formatUnixTimestampToDateTime(startTime, { + strftime_fmt: + config?.ui.time_format == "24hour" ? "%H:55" : "%I:55%P", + time_style: "medium", + date_style: "medium", + })} +
+
+
+ )} +
+ {formatUnixTimestampToDateTime(startTime, { + strftime_fmt: + config.ui.time_format == "24hour" ? "%m/%d %H:%M" : "%m/%d %I:%M%P", + time_style: "medium", + date_style: "medium", + })} +
+
+ ); +} + +function getHourBlocks() { + const arr = []; + + for (let x = 0; x <= 59; x++) { + arr.push(x); + } + + return arr; +} diff --git a/web/src/components/camera/DynamicCameraImage.tsx b/web/src/components/camera/DynamicCameraImage.tsx index 091fcda2e..5bc05f83f 100644 --- a/web/src/components/camera/DynamicCameraImage.tsx +++ b/web/src/components/camera/DynamicCameraImage.tsx @@ -1,11 +1,15 @@ -import { useCallback, useEffect, useState } from "react"; +import { useCallback, useEffect, useMemo, useState } from "react"; import { AspectRatio } from "../ui/aspect-ratio"; import CameraImage from "./CameraImage"; import { LuEar } from "react-icons/lu"; import { CameraConfig } from "@/types/frigateConfig"; import { TbUserScan } from "react-icons/tb"; import { MdLeakAdd } from "react-icons/md"; -import { useFrigateEvents, useMotionActivity } from "@/api/ws"; +import { + useAudioActivity, + useFrigateEvents, + useMotionActivity, +} from "@/api/ws"; type DynamicCameraImageProps = { camera: CameraConfig; @@ -21,10 +25,14 @@ export default function DynamicCameraImage({ }: DynamicCameraImageProps) { const [key, setKey] = useState(Date.now()); const [activeObjects, setActiveObjects] = useState([]); + const hasActiveObjects = useMemo( + () => activeObjects.length > 0, + [activeObjects] + ); const { payload: detectingMotion } = useMotionActivity(camera.name); const { payload: event } = useFrigateEvents(); - const { payload: audioRms } = useMotionActivity(camera.name); + const { payload: audioRms } = useAudioActivity(camera.name); useEffect(() => { if (!event) { @@ -50,7 +58,6 @@ export default function DynamicCameraImage({ if (eventIndex == -1) { const newActiveObjects = [...activeObjects, event.after.id]; setActiveObjects(newActiveObjects); - setKey(Date.now()); } } } @@ -58,8 +65,9 @@ export default function DynamicCameraImage({ const handleLoad = useCallback(() => { const loadTime = Date.now() - key; - const loadInterval = - activeObjects.length > 0 ? INTERVAL_ACTIVE_MS : INTERVAL_INACTIVE_MS; + const loadInterval = hasActiveObjects + ? INTERVAL_ACTIVE_MS + : INTERVAL_INACTIVE_MS; setTimeout( () => { @@ -67,7 +75,7 @@ export default function DynamicCameraImage({ }, loadTime > loadInterval ? 1 : loadInterval ); - }, [activeObjects, key]); + }, [key]); return ( 0 ? "text-object" : "text-gray-600" }`} /> - {camera.audio.enabled && ( + {camera.audio.enabled_in_config && ( = camera.audio.min_volume diff --git a/web/src/components/card/TimelineItemCard.tsx b/web/src/components/card/TimelineItemCard.tsx index 9aa13c9e4..000a29054 100644 --- a/web/src/components/card/TimelineItemCard.tsx +++ b/web/src/components/card/TimelineItemCard.tsx @@ -6,6 +6,20 @@ import useSWR from "swr"; import { FrigateConfig } from "@/types/frigateConfig"; import VideoPlayer from "../player/VideoPlayer"; import { Card } from "../ui/card"; +import { useApiHost } from "@/api"; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, + AlertDialogTrigger, +} from "../ui/alert-dialog"; +import { useCallback } from "react"; +import axios from "axios"; type TimelineItemCardProps = { timeline: Timeline; @@ -18,37 +32,55 @@ export default function TimelineItemCard({ onSelect, }: TimelineItemCardProps) { const { data: config } = useSWR("config"); + const apiHost = useApiHost(); + + const onSubmitToPlus = useCallback( + async (falsePositive: boolean) => { + falsePositive + ? await axios.put(`events/${timeline.source_id}/false_positive`) + : await axios.post(`events/${timeline.source_id}/plus`, { + include_annotation: 1, + }); + }, + [timeline] + ); return ( - -
- {relevantPreview && ( - { + +
+ { + if (relevantPreview) { player.pause(); // autoplay + pause is required for iOS player.currentTime(timeline.timestamp - relevantPreview.start); - }} - /> - )} + } + }} + />
-
+
{getTimelineItemDescription(timeline)}
@@ -60,16 +92,52 @@ export default function TimelineItemCard({ date_style: "medium", })}
- + {timeline.source == "tracked_object" && ( + + + + + + + Submit To Frigate+ + + Objects in locations you want to avoid are not false + positives. Submitting them as false positives will confuse the + model. + + + {`${timeline.data.label}`} + + Cancel + onSubmitToPlus(false)} + > + This is a {timeline.data.label} + + onSubmitToPlus(true)} + > + This is not a {timeline.data.label} + + + + + )}
); diff --git a/web/src/components/graph/TimelineGraph.tsx b/web/src/components/graph/TimelineGraph.tsx index ddffe59ac..73b9b5fb6 100644 --- a/web/src/components/graph/TimelineGraph.tsx +++ b/web/src/components/graph/TimelineGraph.tsx @@ -4,17 +4,34 @@ import Chart from "react-apexcharts"; type TimelineGraphProps = { id: string; data: GraphData[]; + start: number; + end: number; + objects: number[]; }; /** * A graph meant to be overlaid on top of a timeline */ -export default function TimelineGraph({ id, data }: TimelineGraphProps) { +export default function TimelineGraph({ + id, + data, + start, + end, + objects, +}: TimelineGraphProps) { return ( { + if (objects.includes(dataPointIndex)) { + return "#06b6d4"; + } else { + return "#991b1b"; + } + }, + ], chart: { id: id, selection: { @@ -30,11 +47,27 @@ export default function TimelineGraph({ id, data }: TimelineGraphProps) { dataLabels: { enabled: false }, grid: { show: false, + padding: { + bottom: 2, + top: -12, + left: -20, + right: 0, + }, }, legend: { show: false, position: "top", }, + plotOptions: { + bar: { + columnWidth: "100%", + barHeight: "100%", + hideZeroBarsWhenGrouped: true, + }, + }, + stroke: { + width: 0, + }, tooltip: { enabled: false, }, @@ -49,13 +82,16 @@ export default function TimelineGraph({ id, data }: TimelineGraphProps) { labels: { show: false, }, + min: start, + max: end, }, yaxis: { + axisBorder: { + show: false, + }, labels: { show: false, }, - logarithmic: true, - logBase: 10, }, }} series={data} diff --git a/web/src/components/overlay/TimelineDataOverlay.tsx b/web/src/components/overlay/TimelineDataOverlay.tsx index 9ab08bc16..a020e6b5d 100644 --- a/web/src/components/overlay/TimelineDataOverlay.tsx +++ b/web/src/components/overlay/TimelineDataOverlay.tsx @@ -14,6 +14,10 @@ export default function TimelineEventOverlay({ timeline, cameraConfig, }: TimelineEventOverlayProps) { + if (!timeline.data.box) { + return null; + } + const boxLeftEdge = Math.round(timeline.data.box[0] * 100); const boxTopEdge = Math.round(timeline.data.box[1] * 100); const boxRightEdge = Math.round( @@ -25,6 +29,10 @@ export default function TimelineEventOverlay({ const [isHovering, setIsHovering] = useState(false); const getHoverStyle = () => { + if (!timeline.data.box) { + return {}; + } + if (boxLeftEdge < 15) { // show object stats on right side return { @@ -40,12 +48,20 @@ export default function TimelineEventOverlay({ }; const getObjectArea = () => { + if (!timeline.data.box) { + return 0; + } + const width = timeline.data.box[2] * cameraConfig.detect.width; const height = timeline.data.box[3] * cameraConfig.detect.height; return Math.round(width * height); }; const getObjectRatio = () => { + if (!timeline.data.box) { + return 0.0; + } + const width = timeline.data.box[2] * cameraConfig.detect.width; const height = timeline.data.box[3] * cameraConfig.detect.height; return Math.round(100 * (width / height)) / 100; diff --git a/web/src/components/player/DynamicVideoPlayer.tsx b/web/src/components/player/DynamicVideoPlayer.tsx new file mode 100644 index 000000000..9f9c41076 --- /dev/null +++ b/web/src/components/player/DynamicVideoPlayer.tsx @@ -0,0 +1,411 @@ +import { + MutableRefObject, + useCallback, + useEffect, + useMemo, + useRef, + useState, +} from "react"; +import VideoPlayer from "./VideoPlayer"; +import Player from "video.js/dist/types/player"; +import TimelineEventOverlay from "../overlay/TimelineDataOverlay"; +import { useApiHost } from "@/api"; +import useSWR from "swr"; +import { FrigateConfig } from "@/types/frigateConfig"; +import ActivityIndicator from "../ui/activity-indicator"; +import useKeyboardListener from "@/hooks/use-keyboard-listener"; + +/** + * Dynamically switches between video playback and scrubbing preview player. + */ +type DynamicVideoPlayerProps = { + className?: string; + camera: string; + timeRange: { start: number; end: number }; + cameraPreviews: Preview[]; + onControllerReady?: (controller: DynamicVideoController) => void; +}; +export default function DynamicVideoPlayer({ + className, + camera, + timeRange, + cameraPreviews, + onControllerReady, +}: DynamicVideoPlayerProps) { + const apiHost = useApiHost(); + const { data: config } = useSWR("config"); + const timezone = useMemo( + () => + config?.ui?.timezone || Intl.DateTimeFormat().resolvedOptions().timeZone, + [config] + ); + + // controlling playback + + const playerRef = useRef(undefined); + const previewRef = useRef(undefined); + const [isScrubbing, setIsScrubbing] = useState(false); + const [hasPreview, setHasPreview] = useState(false); + const [focusedItem, setFocusedItem] = useState( + undefined + ); + const controller = useMemo(() => { + if (!config) { + return undefined; + } + + return new DynamicVideoController( + playerRef, + previewRef, + (config.cameras[camera]?.detect?.annotation_offset || 0) / 1000, + setIsScrubbing, + setFocusedItem + ); + }, [config]); + + // keyboard control + const onKeyboardShortcut = useCallback( + (key: string, down: boolean, repeat: boolean) => { + switch (key) { + case "ArrowLeft": + if (down) { + const currentTime = playerRef.current?.currentTime(); + + if (currentTime) { + playerRef.current?.currentTime(Math.max(0, currentTime - 5)); + } + } + break; + case "ArrowRight": + if (down) { + const currentTime = playerRef.current?.currentTime(); + + if (currentTime) { + playerRef.current?.currentTime(currentTime + 5); + } + } + break; + case "m": + if (down && !repeat && playerRef.current) { + playerRef.current.muted(!playerRef.current.muted()); + } + break; + case " ": + if (down && playerRef.current) { + if (playerRef.current.paused()) { + playerRef.current.play(); + } else { + playerRef.current.pause(); + } + } + break; + } + }, + [playerRef] + ); + useKeyboardListener( + ["ArrowLeft", "ArrowRight", "m", " "], + onKeyboardShortcut + ); + + // initial state + + const initialPlaybackSource = useMemo(() => { + const date = new Date(timeRange.start * 1000); + return { + src: `${apiHost}vod/${date.getFullYear()}-${ + date.getMonth() + 1 + }/${date.getDate()}/${date.getHours()}/${camera}/${timezone.replaceAll( + "/", + "," + )}/master.m3u8`, + type: "application/vnd.apple.mpegurl", + }; + }, []); + const initialPreviewSource = useMemo(() => { + const preview = cameraPreviews.find( + (preview) => + Math.round(preview.start) >= timeRange.start && + Math.floor(preview.end) <= timeRange.end + ); + + if (preview) { + setHasPreview(true); + return { + src: preview.src, + type: preview.type, + }; + } else { + setHasPreview(false); + return undefined; + } + }, []); + + // state of playback player + + const recordingParams = useMemo(() => { + return { + before: timeRange.end, + after: timeRange.start, + }; + }, [timeRange]); + const { data: recordings } = useSWR( + [`${camera}/recordings`, recordingParams], + { revalidateOnFocus: false } + ); + + useEffect(() => { + if (!controller || !recordings || recordings.length == 0) { + return; + } + + const date = new Date(timeRange.start * 1000); + const playbackUri = `${apiHost}vod/${date.getFullYear()}-${ + date.getMonth() + 1 + }/${date.getDate()}/${date.getHours()}/${camera}/${timezone.replaceAll( + "/", + "," + )}/master.m3u8`; + + const preview = cameraPreviews.find( + (preview) => + Math.round(preview.start) >= timeRange.start && + Math.floor(preview.end) <= timeRange.end + ); + setHasPreview(preview != undefined); + + controller.newPlayback({ + recordings, + playbackUri, + preview, + }); + }, [controller, recordings]); + + if (!controller) { + return ; + } + + return ( +
+
+ { + playerRef.current = player; + player.on("playing", () => setFocusedItem(undefined)); + player.on("timeupdate", () => { + controller.updateProgress(player.currentTime() || 0); + }); + + if (onControllerReady) { + onControllerReady(controller); + } + }} + onDispose={() => { + playerRef.current = undefined; + }} + > + {config && focusedItem && ( + + )} + +
+
+ { + previewRef.current = player; + player.pause(); + player.on("seeked", () => controller.finishedSeeking()); + }} + onDispose={() => { + previewRef.current = undefined; + }} + /> +
+
+ ); +} + +export class DynamicVideoController { + // main state + private playerRef: MutableRefObject; + private previewRef: MutableRefObject; + private setScrubbing: (isScrubbing: boolean) => void; + private setFocusedItem: (timeline: Timeline) => void; + private playerMode: "playback" | "scrubbing" = "playback"; + + // playback + private recordings: Recording[] = []; + private onPlaybackTimestamp: ((time: number) => void) | undefined = undefined; + private annotationOffset: number; + private timeToStart: number | undefined = undefined; + + // preview + private preview: Preview | undefined = undefined; + private timeToSeek: number | undefined = undefined; + private seeking = false; + + constructor( + playerRef: MutableRefObject, + previewRef: MutableRefObject, + annotationOffset: number, + setScrubbing: (isScrubbing: boolean) => void, + setFocusedItem: (timeline: Timeline) => void + ) { + this.playerRef = playerRef; + this.previewRef = previewRef; + this.annotationOffset = annotationOffset; + this.setScrubbing = setScrubbing; + this.setFocusedItem = setFocusedItem; + } + + newPlayback(newPlayback: DynamicPlayback) { + this.recordings = newPlayback.recordings; + + this.playerRef.current?.src({ + src: newPlayback.playbackUri, + type: "application/vnd.apple.mpegurl", + }); + + if (this.timeToStart) { + this.seekToTimestamp(this.timeToStart); + this.timeToStart = undefined; + } + + this.preview = newPlayback.preview; + if (this.preview && this.previewRef.current) { + this.previewRef.current.src({ + src: this.preview.src, + type: this.preview.type, + }); + } + } + + seekToTimestamp(time: number, play: boolean = false) { + if (this.playerMode != "playback") { + this.playerMode = "playback"; + this.setScrubbing(false); + this.timeToSeek = undefined; + this.seeking = false; + } + + if (this.recordings.length == 0) { + this.timeToStart = time; + } + + let seekSeconds = 0; + (this.recordings || []).every((segment) => { + // if the next segment is past the desired time, stop calculating + if (segment.start_time > time) { + return false; + } + + if (segment.end_time < time) { + seekSeconds += segment.end_time - segment.start_time; + return true; + } + + seekSeconds += + segment.end_time - segment.start_time - (segment.end_time - time); + return true; + }); + this.playerRef.current?.currentTime(seekSeconds); + + if (play) { + this.playerRef.current?.play(); + } + } + + seekToTimelineItem(timeline: Timeline) { + this.playerRef.current?.pause(); + this.seekToTimestamp(timeline.timestamp + this.annotationOffset); + this.setFocusedItem(timeline); + } + + updateProgress(playerTime: number) { + if (this.onPlaybackTimestamp) { + // take a player time in seconds and convert to timestamp in timeline + let timestamp = 0; + let totalTime = 0; + (this.recordings || []).every((segment) => { + if (totalTime + segment.duration > playerTime) { + // segment is here + timestamp = segment.start_time + (playerTime - totalTime); + return false; + } else { + totalTime += segment.duration; + return true; + } + }); + + this.onPlaybackTimestamp(timestamp); + } + } + + onPlayerTimeUpdate(listener: (timestamp: number) => void) { + this.onPlaybackTimestamp = listener; + } + + scrubToTimestamp(time: number) { + if (this.playerMode != "scrubbing") { + this.playerMode = "scrubbing"; + this.playerRef.current?.pause(); + this.setScrubbing(true); + } + + if (this.preview) { + if (this.seeking) { + this.timeToSeek = time; + } else { + this.previewRef.current?.currentTime(time - this.preview.start); + this.seeking = true; + } + } + } + + finishedSeeking() { + if (!this.preview || this.playerMode == "playback") { + return; + } + + if ( + this.timeToSeek && + this.timeToSeek != this.previewRef.current?.currentTime() + ) { + this.previewRef.current?.currentTime( + this.timeToSeek - this.preview.start + ); + } else { + this.seeking = false; + } + } +} diff --git a/web/src/components/player/PreviewThumbnailPlayer.tsx b/web/src/components/player/PreviewThumbnailPlayer.tsx index 1fdcaa8b5..f35ab0dcc 100644 --- a/web/src/components/player/PreviewThumbnailPlayer.tsx +++ b/web/src/components/player/PreviewThumbnailPlayer.tsx @@ -1,6 +1,4 @@ -import { FrigateConfig } from "@/types/frigateConfig"; import VideoPlayer from "./VideoPlayer"; -import useSWR from "swr"; import React, { useCallback, useEffect, @@ -12,6 +10,7 @@ import { useApiHost } from "@/api"; import Player from "video.js/dist/types/player"; import { AspectRatio } from "../ui/aspect-ratio"; import { LuPlayCircle } from "react-icons/lu"; +import { isCurrentHour } from "@/utils/dateUtil"; type PreviewPlayerProps = { camera: string; @@ -38,7 +37,6 @@ export default function PreviewThumbnailPlayer({ isMobile, onClick, }: PreviewPlayerProps) { - const { data: config } = useSWR("config"); const playerRef = useRef(null); const isSafari = useMemo(() => { return /^((?!chrome|android).)*safari/i.test(navigator.userAgent); @@ -54,7 +52,10 @@ export default function PreviewThumbnailPlayer({ } if (!playerRef.current) { - setIsInitiallyVisible(true); + if (isHovered) { + setIsInitiallyVisible(true); + } + return; } @@ -105,6 +106,7 @@ export default function PreviewThumbnailPlayer({ { threshold: 1.0, root: document.getElementById("pageRoot"), + rootMargin: "-10% 0px -25% 0px", } ); if (node) autoPlayObserver.current.observe(node); @@ -131,7 +133,6 @@ export default function PreviewThumbnailPlayer({ isInitiallyVisible={isInitiallyVisible} startTs={startTs} camera={camera} - config={config} eventId={eventId} isMobile={isMobile} isSafari={isSafari} @@ -143,7 +144,6 @@ export default function PreviewThumbnailPlayer({ type PreviewContentProps = { playerRef: React.MutableRefObject; - config: FrigateConfig; camera: string; relevantPreview: Preview | undefined; eventId: string; @@ -156,7 +156,6 @@ type PreviewContentProps = { }; function PreviewContent({ playerRef, - config, camera, relevantPreview, eventId, @@ -195,22 +194,13 @@ function PreviewContent({ if (relevantPreview && !isVisible) { return
; - } else if (!relevantPreview) { - if (isCurrentHour(startTs)) { - return ( - - ); - } else { - return ( - - ); - } + } else if (!relevantPreview && !isCurrentHour(startTs)) { + return ( + + ); } else { return ( <> @@ -223,17 +213,26 @@ function PreviewContent({ controls: false, muted: true, loadingSpinner: false, - sources: [ - { - src: `${relevantPreview.src}`, - type: "video/mp4", - }, - ], + poster: relevantPreview + ? "" + : `${apiHost}api/preview/${camera}/${startTs}/thumbnail.jpg`, + sources: relevantPreview + ? [ + { + src: `${relevantPreview.src}`, + type: "video/mp4", + }, + ] + : [], }} seekOptions={{}} onReady={(player) => { playerRef.current = player; + if (!relevantPreview) { + return; + } + if (!isInitiallyVisible) { player.pause(); // autoplay + pause is required for iOS } @@ -249,28 +248,10 @@ function PreviewContent({ }} />
- + {relevantPreview && ( + + )} ); } } - -function isCurrentHour(timestamp: number) { - const now = new Date(); - now.setMinutes(0, 0, 0); - return timestamp > now.getTime() / 1000; -} - -function getPreviewWidth(camera: string, config: FrigateConfig) { - const detect = config.cameras[camera].detect; - - if (detect.width / detect.height < 1) { - return "w-1/2"; - } - - if (detect.width / detect.height < 16 / 9) { - return "w-2/3"; - } - - return "w-full"; -} diff --git a/web/src/components/scrubber/ActivityScrubber.tsx b/web/src/components/scrubber/ActivityScrubber.tsx index de8098aa7..64debc320 100644 --- a/web/src/components/scrubber/ActivityScrubber.tsx +++ b/web/src/components/scrubber/ActivityScrubber.tsx @@ -9,6 +9,8 @@ import { } from "vis-timeline"; import type { DataGroup, DataItem, TimelineEvents } from "vis-timeline/types"; import "./scrubber.css"; +import useSWR from "swr"; +import { FrigateConfig } from "@/types/frigateConfig"; export type TimelineEventsWithMissing = | TimelineEvents @@ -89,14 +91,13 @@ function ActivityScrubber({ options, ...eventHandlers }: ActivityScrubberProps) { + const { data: config } = useSWR("config"); const containerRef = useRef(null); const timelineRef = useRef<{ timeline: VisTimeline | null }>({ timeline: null, }); const [currentTime, setCurrentTime] = useState(Date.now()); - const [_, setCustomTimes] = useState< - { id: IdType; time: DateType }[] - >([]); + const [_, setCustomTimes] = useState<{ id: IdType; time: DateType }[]>([]); const defaultOptions: TimelineOptions = { width: "100%", @@ -110,8 +111,11 @@ function ActivityScrubber({ max: currentTime, format: { minorLabels: { - minute: "h:mma", - hour: "ha", + minute: config?.ui.time_format == "24hour" ? "HH:mm" : "hh:mma", + }, + majorLabels: { + minute: + config?.ui.time_format == "24hour" ? "MM/DD HH:mm" : "MM/DD hh:mma", }, }, }; @@ -139,8 +143,8 @@ function ActivityScrubber({ const timelineInstance = new VisTimeline( divElement, - items as DataItem[], - groups as DataGroup[], + (items || []) as DataItem[], + (groups || []) as DataGroup[], timelineOptions ); diff --git a/web/src/hooks/use-keyboard-listener.tsx b/web/src/hooks/use-keyboard-listener.tsx new file mode 100644 index 000000000..a68cb3a7e --- /dev/null +++ b/web/src/hooks/use-keyboard-listener.tsx @@ -0,0 +1,43 @@ +import { useCallback, useEffect } from "react"; + +export default function useKeyboardListener( + keys: string[], + listener: (key: string, down: boolean, repeat: boolean) => void +) { + const keyDownListener = useCallback( + (e: KeyboardEvent) => { + if (!e) { + return; + } + + if (keys.includes(e.key)) { + e.preventDefault(); + listener(e.key, true, e.repeat); + } + }, + [listener] + ); + + const keyUpListener = useCallback( + (e: KeyboardEvent) => { + if (!e) { + return; + } + + if (keys.includes(e.key)) { + e.preventDefault(); + listener(e.key, false, false); + } + }, + [listener] + ); + + useEffect(() => { + document.addEventListener("keydown", keyDownListener); + document.addEventListener("keyup", keyUpListener); + return () => { + document.removeEventListener("keydown", keyDownListener); + document.removeEventListener("keyup", keyUpListener); + }; + }, [listener]); +} diff --git a/web/src/pages/ConfigEditor.tsx b/web/src/pages/ConfigEditor.tsx index 332e31855..629570d91 100644 --- a/web/src/pages/ConfigEditor.tsx +++ b/web/src/pages/ConfigEditor.tsx @@ -122,7 +122,7 @@ function ConfigEditor() { } return ( -
+
Config
diff --git a/web/src/pages/Dashboard.tsx b/web/src/pages/Dashboard.tsx index 9091a9584..6eaf1bff7 100644 --- a/web/src/pages/Dashboard.tsx +++ b/web/src/pages/Dashboard.tsx @@ -159,7 +159,7 @@ function Camera({ camera }: { camera: CameraConfig }) { onClick={(e) => { e.stopPropagation(); e.preventDefault(); - sendAudio(detectValue == "ON" ? "OFF" : "ON"); + sendAudio(audioValue == "ON" ? "OFF" : "ON"); }} > diff --git a/web/src/pages/History.tsx b/web/src/pages/History.tsx index 66f1d2314..5bfff0426 100644 --- a/web/src/pages/History.tsx +++ b/web/src/pages/History.tsx @@ -268,7 +268,7 @@ function TimelineViewer({ return ( onClose()}> - + {timelineData && playback && ( { if (cameraConfig) { if (restreamEnabled) { - return cameraConfig.ui.live_mode; + return cameraConfig.ui.live_mode || config?.ui.live_mode; } return "jsmpeg"; @@ -65,55 +65,75 @@ function Live() {
Live -
- - - - - - Select A Camera - - - {sortedCameras.map((item) => ( - - {item.name.replaceAll("_", " ")} +
+
+ + + + + + Select A Camera + + + {config?.birdseye.enabled && ( + + Birdseye + + )} + {sortedCameras.map((item) => ( + + {item.name.replaceAll("_", " ")} + + ))} + + + +
+
+ + + + + + Select A Live Mode + + + {restreamEnabled && ( + + Webrtc + + )} + {restreamEnabled && ( + + MSE + + )} + + Jsmpeg - ))} - - - - - - - - - Select A Live Mode - - - - Webrtc - - MSE - - Jsmpeg - - - Debug - - - - + {camera != "birdseye" && ( + + Debug + + )} + + + +
{config && camera == "birdseye" && sourceIsLoaded && ( diff --git a/web/src/types/history.ts b/web/src/types/history.ts index e925de2e1..23e30c48c 100644 --- a/web/src/types/history.ts +++ b/web/src/types/history.ts @@ -21,33 +21,6 @@ type Preview = { end: number; }; -type Timeline = { - camera: string; - timestamp: number; - data: { - [key: string]: any; - }; - class_type: - | "visible" - | "gone" - | "sub_label" - | "entered_zone" - | "attribute" - | "active" - | "stationary" - | "heard" - | "external"; - source_id: string; - source: string; -}; - -type HourlyTimeline = { - start: number; - end: number; - count: number; - hours: { [key: string]: Timeline[] }; -}; - interface HistoryFilter extends FilterType { cameras: string[]; labels: string[]; diff --git a/web/src/types/playback.ts b/web/src/types/playback.ts new file mode 100644 index 000000000..07867b0f9 --- /dev/null +++ b/web/src/types/playback.ts @@ -0,0 +1,5 @@ +type DynamicPlayback = { + recordings: Recording[]; + playbackUri: string; + preview: Preview | undefined; +}; diff --git a/web/src/types/record.ts b/web/src/types/record.ts index 614d223a8..a6979aea8 100644 --- a/web/src/types/record.ts +++ b/web/src/types/record.ts @@ -5,6 +5,7 @@ type Recording = { end_time: number; path: string; segment_size: number; + duration: number; motion: number; objects: number; dBFS: number; @@ -17,6 +18,7 @@ type RecordingSegment = { motion: number; objects: number; segment_size: number; + duration: number; }; type RecordingActivity = { @@ -26,5 +28,5 @@ type RecordingActivity = { type RecordingSegmentActivity = { date: number; count: number; - type: "motion" | "objects"; + hasObjects: boolean; }; diff --git a/web/src/types/timeline.ts b/web/src/types/timeline.ts new file mode 100644 index 000000000..3698a6ebe --- /dev/null +++ b/web/src/types/timeline.ts @@ -0,0 +1,31 @@ +type Timeline = { + camera: string; + timestamp: number; + data: { + camera: string; + label: string; + sub_label: string; + box?: [number, number, number, number]; + region: [number, number, number, number]; + attribute: string; + zones: string[]; + }; + class_type: + | "visible" + | "gone" + | "entered_zone" + | "attribute" + | "active" + | "stationary" + | "heard" + | "external"; + source_id: string; + source: string; + }; + + type HourlyTimeline = { + start: number; + end: number; + count: number; + hours: { [key: string]: Timeline[] }; + }; \ No newline at end of file diff --git a/web/src/utils/dateUtil.ts b/web/src/utils/dateUtil.ts index 6f955d6dd..df06a7173 100644 --- a/web/src/utils/dateUtil.ts +++ b/web/src/utils/dateUtil.ts @@ -282,6 +282,14 @@ export function getRangeForTimestamp(timestamp: number) { date.setMinutes(0, 0, 0); const start = date.getTime() / 1000; date.setHours(date.getHours() + 1); - const end = date.getTime() / 1000; + + // ensure not to go past current time + const end = Math.min(new Date().getTime() / 1000, date.getTime() / 1000); return { start, end }; } + +export function isCurrentHour(timestamp: number) { + const now = new Date(); + now.setMinutes(0, 0, 0); + return timestamp > now.getTime() / 1000; +} diff --git a/web/src/utils/historyUtil.ts b/web/src/utils/historyUtil.ts index d9c6b88c0..38b27126d 100644 --- a/web/src/utils/historyUtil.ts +++ b/web/src/utils/historyUtil.ts @@ -1,158 +1,178 @@ -// group history cards by 60 seconds of activity -const GROUP_SECONDS = 60; +// group history cards by 120 seconds of activity +const GROUP_SECONDS = 120; export function getHourlyTimelineData( timelinePages: HourlyTimeline[], detailLevel: string ): CardsData { const cards: CardsData = {}; + const allHours: { [key: string]: Timeline[] } = {}; + timelinePages.forEach((hourlyTimeline) => { - Object.keys(hourlyTimeline["hours"]) - .reverse() - .forEach((hour) => { - const day = new Date(parseInt(hour) * 1000); - day.setHours(0, 0, 0, 0); - const dayKey = (day.getTime() / 1000).toString(); - - // build a map of course to the types that are included in this hour - // which allows us to know what items to keep depending on detail level - const source_to_types: { [key: string]: string[] } = {}; - let cardTypeStart: { [camera: string]: number } = {}; - Object.values(hourlyTimeline["hours"][hour]).forEach((i) => { - if (i.timestamp > (cardTypeStart[i.camera] ?? 0) + GROUP_SECONDS) { - cardTypeStart[i.camera] = i.timestamp; - } - - const groupKey = `${i.source_id}-${cardTypeStart[i.camera]}`; - - if (groupKey in source_to_types) { - source_to_types[groupKey].push(i.class_type); - } else { - source_to_types[groupKey] = [i.class_type]; - } - }); - - if (!(dayKey in cards)) { - cards[dayKey] = {}; - } - - if (!(hour in cards[dayKey])) { - cards[dayKey][hour] = {}; - } - - let cardStart: { [camera: string]: number } = {}; - Object.values(hourlyTimeline["hours"][hour]).forEach((i) => { - if (i.timestamp > (cardStart[i.camera] ?? 0) + GROUP_SECONDS) { - cardStart[i.camera] = i.timestamp; - } - - const time = new Date(i.timestamp * 1000); - const groupKey = `${i.camera}-${cardStart[i.camera]}`; - const sourceKey = `${i.source_id}-${cardStart[i.camera]}`; - const uniqueKey = `${i.source_id}-${i.class_type}`; - - // detail level for saving items - // detail level determines which timeline items for each moment is returned - // values can be normal, extra, or full - // normal: return all items except active / attribute / gone / stationary / visible unless that is the only item. - // extra: return all items except attribute / gone / visible unless that is the only item - // full: return all items - - let add = true; - if (detailLevel == "normal") { - if ( - source_to_types[sourceKey].length > 1 && - ["active", "attribute", "gone", "stationary", "visible"].includes( - i.class_type - ) - ) { - add = false; - } - } else if (detailLevel == "extra") { - if ( - source_to_types[sourceKey].length > 1 && - i.class_type in ["attribute", "gone", "visible"] - ) { - add = false; - } - } - - if (add) { - if (groupKey in cards[dayKey][hour]) { - if ( - !cards[dayKey][hour][groupKey].uniqueKeys.includes(uniqueKey) || - detailLevel == "full" - ) { - cards[dayKey][hour][groupKey].entries.push(i); - cards[dayKey][hour][groupKey].uniqueKeys.push(uniqueKey); - } - } else { - cards[dayKey][hour][groupKey] = { - camera: i.camera, - time: time.getTime() / 1000, - entries: [i], - uniqueKeys: [uniqueKey], - }; - } - } - }); - }); + Object.entries(hourlyTimeline.hours).forEach(([key, values]) => { + if (key in allHours) { + // only occurs when multiple pages contain elements in the same hour + allHours[key] = allHours[key] + .concat(values) + .sort((a, b) => a.timestamp - b.timestamp); + } else { + allHours[key] = values; + } + }); }); + Object.keys(allHours) + .sort((a, b) => a.localeCompare(b)) + .reverse() + .forEach((hour) => { + const day = new Date(parseInt(hour) * 1000); + day.setHours(0, 0, 0, 0); + const dayKey = (day.getTime() / 1000).toString(); + + // build a map of course to the types that are included in this hour + // which allows us to know what items to keep depending on detail level + const sourceToTypes: { [key: string]: string[] } = {}; + let cardTypeStart: { [camera: string]: number } = {}; + Object.values(allHours[hour]).forEach((i) => { + if (i.timestamp > (cardTypeStart[i.camera] ?? 0) + GROUP_SECONDS) { + cardTypeStart[i.camera] = i.timestamp; + } + + const groupKey = `${i.source_id}-${cardTypeStart[i.camera]}`; + + if (groupKey in sourceToTypes) { + sourceToTypes[groupKey].push(i.class_type); + } else { + sourceToTypes[groupKey] = [i.class_type]; + } + }); + + if (!(dayKey in cards)) { + cards[dayKey] = {}; + } + + if (!(hour in cards[dayKey])) { + cards[dayKey][hour] = {}; + } + + let cardStart: { [camera: string]: number } = {}; + Object.values(allHours[hour]).forEach((i) => { + if (i.timestamp > (cardStart[i.camera] ?? 0) + GROUP_SECONDS) { + cardStart[i.camera] = i.timestamp; + } + + const time = new Date(i.timestamp * 1000); + const groupKey = `${i.camera}-${cardStart[i.camera]}`; + const sourceKey = `${i.source_id}-${cardStart[i.camera]}`; + const uniqueKey = `${i.source_id}-${i.class_type}`; + + // detail level for saving items + // detail level determines which timeline items for each moment is returned + // values can be normal, extra, or full + // normal: return all items except active / attribute / gone / stationary / visible unless that is the only item. + // extra: return all items except attribute / gone / visible unless that is the only item + // full: return all items + + let add = true; + const sourceType = sourceToTypes[sourceKey]; + let hiddenItems: string[] = []; + if (detailLevel == "normal") { + hiddenItems = [ + "active", + "attribute", + "gone", + "stationary", + "visible", + ]; + } else if (detailLevel == "extra") { + hiddenItems = ["attribute", "gone", "visible"]; + } + + if (sourceType.length > 1) { + // we have multiple timeline items for this card + + if ( + sourceType.find((type) => hiddenItems.includes(type) == false) == + undefined + ) { + // all of the attribute items for this card make it hidden, but we need to show one + if (sourceType.indexOf(i.class_type) != 0) { + add = false; + } + } else if (hiddenItems.includes(i.class_type)) { + add = false; + } + } + + if (add) { + if (groupKey in cards[dayKey][hour]) { + if ( + !cards[dayKey][hour][groupKey].uniqueKeys.includes(uniqueKey) || + detailLevel == "full" + ) { + cards[dayKey][hour][groupKey].entries.push(i); + cards[dayKey][hour][groupKey].uniqueKeys.push(uniqueKey); + } + } else { + cards[dayKey][hour][groupKey] = { + camera: i.camera, + time: time.getTime() / 1000, + entries: [i], + uniqueKeys: [uniqueKey], + }; + } + } + }); + }); + return cards; } export function getTimelineHoursForDay( camera: string, cards: CardsData, - allPreviews: Preview[], + cameraPreviews: Preview[], timestamp: number ): HistoryTimeline { - const now = new Date(); + const endOfThisHour = new Date(); + endOfThisHour.setHours(endOfThisHour.getHours() + 1, 0, 0, 0); const data: TimelinePlayback[] = []; const startDay = new Date(timestamp * 1000); startDay.setHours(23, 59, 59, 999); - const dayEnd = startDay.getTime() / 1000; startDay.setHours(0, 0, 0, 0); const startTimestamp = startDay.getTime() / 1000; let start = startDay.getTime() / 1000; let end = 0; - const relevantPreviews = allPreviews.filter((preview) => { - return ( - preview.camera == camera && - preview.start >= start && - Math.floor(preview.end - 1) <= dayEnd - ); - }); - const dayIdx = Object.keys(cards).find((day) => { - if (parseInt(day) > start) { + if (parseInt(day) < start) { return false; } return true; }); - if (dayIdx == undefined) { - return { start: 0, end: 0, playbackItems: [] }; - } + let day: { + [hour: string]: { + [groupKey: string]: Card; + }; + } = {}; - const day = cards[dayIdx]; + if (dayIdx != undefined) { + day = cards[dayIdx]; + } for (let i = 0; i < 24; i++) { startDay.setHours(startDay.getHours() + 1); - if (startDay > now) { + if (startDay > endOfThisHour) { break; } end = startDay.getTime() / 1000; const hour = Object.values(day).find((cards) => { - if ( - Object.values(cards)[0].time < start || - Object.values(cards)[0].time > end - ) { + const card = Object.values(cards)[0]; + if (card == undefined || card.time < start || card.time > end) { return false; } @@ -167,7 +187,7 @@ export function getTimelineHoursForDay( return []; }) : []; - const relevantPreview = relevantPreviews.find( + const relevantPreview = cameraPreviews.find( (preview) => Math.round(preview.start) >= start && Math.floor(preview.end) <= end ); diff --git a/web/src/utils/timelineUtil.tsx b/web/src/utils/timelineUtil.tsx index 1820df5f5..483d63231 100644 --- a/web/src/utils/timelineUtil.tsx +++ b/web/src/utils/timelineUtil.tsx @@ -42,15 +42,6 @@ export function getTimelineIcon(timelineItem: Timeline) { default: return ; } - case "sub_label": - switch (timelineItem.data.label) { - case "person": - return ; - case "car": - return ; - default: - return ; - } case "heard": return ; case "external": @@ -119,8 +110,6 @@ export function getTimelineItemDescription(timelineItem: Timeline) { } return title; } - case "sub_label": - return `${timelineItem.data.label} recognized as ${timelineItem.data.sub_label}`; case "gone": return `${label} left`; case "heard": diff --git a/web/src/views/history/DesktopTimelineView.tsx b/web/src/views/history/DesktopTimelineView.tsx index bbdd9625d..06151f060 100644 --- a/web/src/views/history/DesktopTimelineView.tsx +++ b/web/src/views/history/DesktopTimelineView.tsx @@ -1,17 +1,17 @@ -import { useApiHost } from "@/api"; -import TimelineEventOverlay from "@/components/overlay/TimelineDataOverlay"; -import VideoPlayer from "@/components/player/VideoPlayer"; import ActivityScrubber from "@/components/scrubber/ActivityScrubber"; import ActivityIndicator from "@/components/ui/activity-indicator"; import { FrigateConfig } from "@/types/frigateConfig"; -import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { useEffect, useMemo, useRef, useState } from "react"; import useSWR from "swr"; -import Player from "video.js/dist/types/player"; import TimelineItemCard from "@/components/card/TimelineItemCard"; import { getTimelineHoursForDay } from "@/utils/historyUtil"; import { GraphDataPoint } from "@/types/graph"; import TimelineGraph from "@/components/graph/TimelineGraph"; +import TimelineBar from "@/components/bar/TimelineBar"; +import DynamicVideoPlayer, { + DynamicVideoController, +} from "@/components/player/DynamicVideoPlayer"; type DesktopTimelineViewProps = { timelineData: CardsData; @@ -24,7 +24,6 @@ export default function DesktopTimelineView({ allPreviews, initialPlayback, }: DesktopTimelineViewProps) { - const apiHost = useApiHost(); const { data: config } = useSWR("config"); const timezone = useMemo( () => @@ -32,137 +31,31 @@ export default function DesktopTimelineView({ [config] ); + const controllerRef = useRef(undefined); + const initialScrollRef = useRef(null); + const [selectedPlayback, setSelectedPlayback] = useState(initialPlayback); + const [timelineTime, setTimelineTime] = useState(0); - const playerRef = useRef(undefined); - const previewRef = useRef(undefined); - - const [scrubbing, setScrubbing] = useState(false); - const [focusedItem, setFocusedItem] = useState( - undefined - ); - - const [seeking, setSeeking] = useState(false); - const [timeToSeek, setTimeToSeek] = useState(undefined); - const [timelineTime, setTimelineTime] = useState( - initialPlayback.timelineItems.length > 0 - ? initialPlayback.timelineItems[0].timestamp - initialPlayback.range.start - : 0 - ); - - const annotationOffset = useMemo(() => { - if (!config) { - return 0; - } - - return ( - (config.cameras[initialPlayback.camera]?.detect?.annotation_offset || 0) / - 1000 - ); - }, [config]); - - const recordingParams = useMemo(() => { - return { - before: selectedPlayback.range.end, - after: selectedPlayback.range.start, - }; - }, [selectedPlayback]); - const { data: recordings } = useSWR( - selectedPlayback - ? [`${selectedPlayback.camera}/recordings`, recordingParams] - : null, - { revalidateOnFocus: false } - ); - - const playbackUri = useMemo(() => { - if (!selectedPlayback) { - return ""; - } - - const date = new Date(selectedPlayback.range.start * 1000); - return `${apiHost}vod/${date.getFullYear()}-${ - date.getMonth() + 1 - }/${date.getDate()}/${date.getHours()}/${ - selectedPlayback.camera - }/${timezone.replaceAll("/", ",")}/master.m3u8`; - }, [selectedPlayback]); - - const onSelectItem = useCallback( - (timeline: Timeline | undefined) => { - if (timeline) { - setFocusedItem(timeline); - const selected = timeline.timestamp; - playerRef.current?.pause(); - - let seekSeconds = 0; - (recordings || []).every((segment) => { - // if the next segment is past the desired time, stop calculating - if (segment.start_time > selected) { - return false; - } - - if (segment.end_time < selected) { - seekSeconds += segment.end_time - segment.start_time; - return true; - } - - seekSeconds += - segment.end_time - - segment.start_time - - (segment.end_time - selected); - return true; - }); - playerRef.current?.currentTime(seekSeconds); - } else { - setFocusedItem(undefined); - } - }, - [annotationOffset, recordings, playerRef] - ); - - // handle seeking to next frame when seek is finished + // handle scrolling to initial timeline item useEffect(() => { - if (seeking) { - return; + if (initialScrollRef.current != null) { + initialScrollRef.current.scrollIntoView(); } + }, [initialScrollRef]); - if (timeToSeek && timeToSeek != previewRef.current?.currentTime()) { - setSeeking(true); - previewRef.current?.currentTime(timeToSeek); - } - }, [timeToSeek, seeking]); - - // handle loading main / preview playback when selected hour changes - useEffect(() => { - if (!playerRef.current || !previewRef.current) { - return; - } - - setTimelineTime( - selectedPlayback.timelineItems.length > 0 - ? selectedPlayback.timelineItems[0].timestamp - : selectedPlayback.range.start - ); - - playerRef.current.src({ - src: playbackUri, - type: "application/vnd.apple.mpegurl", + const cameraPreviews = useMemo(() => { + return allPreviews.filter((preview) => { + return preview.camera == initialPlayback.camera; }); - - if (selectedPlayback.relevantPreview) { - previewRef.current.src({ - src: selectedPlayback.relevantPreview.src, - type: selectedPlayback.relevantPreview.type, - }); - } - }, [playerRef, previewRef, selectedPlayback]); + }, []); const timelineStack = useMemo( () => getTimelineHoursForDay( selectedPlayback.camera, timelineData, - allPreviews, + cameraPreviews, selectedPlayback.range.start + 60 ), [] @@ -179,31 +72,29 @@ export default function DesktopTimelineView({ ], { revalidateOnFocus: false } ); + const timelineGraphData = useMemo(() => { if (!activity) { return {}; } const graphData: { - [hour: string]: { objects: GraphDataPoint[]; motion: GraphDataPoint[] }; + [hour: string]: { objects: number[]; motion: GraphDataPoint[] }; } = {}; Object.entries(activity).forEach(([hour, data]) => { - const objects: GraphDataPoint[] = []; + const objects: number[] = []; const motion: GraphDataPoint[] = []; - data.forEach((seg) => { - if (seg.type == "objects") { - objects.push({ - x: new Date(seg.date * 1000), - y: seg.count, - }); - } else { - motion.push({ - x: new Date(seg.date * 1000), - y: seg.count, - }); + data.forEach((seg, idx) => { + if (seg.hasObjects) { + objects.push(idx); } + + motion.push({ + x: new Date(seg.date * 1000), + y: seg.count, + }); }); graphData[hour] = { objects, motion }; @@ -217,191 +108,133 @@ export default function DesktopTimelineView({ } return ( -
-
- <> -
-
- { - playerRef.current = player; +
+
+ { + controllerRef.current = controller; + controllerRef.current.onPlayerTimeUpdate((timestamp: number) => { + setTimelineTime(timestamp); + }); - if (selectedPlayback.timelineItems.length > 0) { - player.currentTime( - selectedPlayback.timelineItems[0].timestamp - - selectedPlayback.range.start - ); - } else { - player.currentTime(0); - } - player.on("playing", () => onSelectItem(undefined)); - player.on("timeupdate", () => { - setTimelineTime(Math.floor(player.currentTime() || 0)); - }); - }} - onDispose={() => { - playerRef.current = undefined; - }} - > - {focusedItem && ( - - )} - -
- {selectedPlayback.relevantPreview && ( -
- { - previewRef.current = player; - player.on("seeked", () => setSeeking(false)); - }} - onDispose={() => { - previewRef.current = undefined; - }} - /> -
- )} -
- -
+ if (initialPlayback.timelineItems.length > 0) { + controllerRef.current?.seekToTimestamp( + selectedPlayback.timelineItems[0].timestamp, + true + ); + } + }} + /> +
{selectedPlayback.timelineItems.map((timeline) => { return ( onSelectItem(timeline)} + onSelect={() => { + controllerRef.current?.seekToTimelineItem(timeline); + }} /> ); })}
-
- {timelineStack.playbackItems.map((timeline) => { - const isSelected = - timeline.range.start == selectedPlayback.range.start; - const graphData = timelineGraphData[timeline.range.start]; +
+
+ {timelineStack.playbackItems.map((timeline) => { + const isInitiallySelected = + initialPlayback.range.start == timeline.range.start; + const isSelected = + timeline.range.start == selectedPlayback.range.start; + const graphData = timelineGraphData[timeline.range.start]; - return ( -
- { - if (!timeline.relevantPreview) { - return; - } + return ( +
+ {isSelected ? ( +
+ { + controllerRef.current?.scrubToTimestamp( + data.time.getTime() / 1000 + ); + setTimelineTime(data.time.getTime() / 1000); + }} + timechangedHandler={(data) => { + controllerRef.current?.seekToTimestamp( + data.time.getTime() / 1000, + true + ); + }} + /> + {isSelected && graphData && ( +
+ +
+ )} +
+ ) : ( + { + setSelectedPlayback(timeline); - if (playerRef.current?.paused() == false) { - setScrubbing(true); - playerRef.current?.pause(); - } + let startTs; + if (timeline.timelineItems.length > 0) { + startTs = selectedPlayback.timelineItems[0].timestamp; + } else { + startTs = timeline.range.start; + } - const seekTimestamp = data.time.getTime() / 1000; - const seekTime = - seekTimestamp - timeline.relevantPreview.start; - setTimelineTime(seekTimestamp - timeline.range.start); - setTimeToSeek(Math.round(seekTime)); - }} - timechangedHandler={(data) => { - const playbackTime = data.time.getTime() / 1000; - playerRef.current?.currentTime( - playbackTime - timeline.range.start - ); - setScrubbing(false); - playerRef.current?.play(); - }} - selectHandler={(data) => { - if (data.items.length > 0) { - const selected = data.items[0]; - onSelectItem( - selectedPlayback.timelineItems.find( - (timeline) => timeline.timestamp == selected - ) - ); - } - }} - doubleClickHandler={() => setSelectedPlayback(timeline)} - /> - {isSelected && graphData && ( -
- -
- )} -
- ); - })} + )} +
+ ); + })} +
); diff --git a/web/src/views/history/MobileTimelineView.tsx b/web/src/views/history/MobileTimelineView.tsx index 7577336d1..48a46dd3d 100644 --- a/web/src/views/history/MobileTimelineView.tsx +++ b/web/src/views/history/MobileTimelineView.tsx @@ -1,6 +1,3 @@ -import { useApiHost } from "@/api"; -import TimelineEventOverlay from "@/components/overlay/TimelineDataOverlay"; -import VideoPlayer from "@/components/player/VideoPlayer"; import ActivityScrubber, { ScrubberItem, } from "@/components/scrubber/ActivityScrubber"; @@ -11,9 +8,11 @@ import { getTimelineIcon, } from "@/utils/timelineUtil"; import { renderToStaticMarkup } from "react-dom/server"; -import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { useMemo, useRef, useState } from "react"; import useSWR from "swr"; -import Player from "video.js/dist/types/player"; +import DynamicVideoPlayer, { + DynamicVideoController, +} from "@/components/player/DynamicVideoPlayer"; type MobileTimelineViewProps = { playback: TimelinePlayback; @@ -22,34 +21,9 @@ type MobileTimelineViewProps = { export default function MobileTimelineView({ playback, }: MobileTimelineViewProps) { - const apiHost = useApiHost(); const { data: config } = useSWR("config"); - const timezone = useMemo( - () => - config?.ui?.timezone || Intl.DateTimeFormat().resolvedOptions().timeZone, - [config] - ); - const playerRef = useRef(undefined); - const previewRef = useRef(undefined); - - const [scrubbing, setScrubbing] = useState(false); - const [focusedItem, setFocusedItem] = useState( - undefined - ); - - const [seeking, setSeeking] = useState(false); - const [timeToSeek, setTimeToSeek] = useState(undefined); - - const annotationOffset = useMemo(() => { - if (!config) { - return 0; - } - - return ( - (config.cameras[playback.camera]?.detect?.annotation_offset || 0) / 1000 - ); - }, [config]); + const controllerRef = useRef(undefined); const [timelineTime, setTimelineTime] = useState( playback.timelineItems.length > 0 @@ -68,197 +42,69 @@ export default function MobileTimelineView({ { revalidateOnFocus: false } ); - const playbackUri = useMemo(() => { - if (!playback) { - return ""; - } - - const date = new Date(playback.range.start * 1000); - return `${apiHost}vod/${date.getFullYear()}-${ - date.getMonth() + 1 - }/${date.getDate()}/${date.getHours()}/${ - playback.camera - }/${timezone.replaceAll("/", ",")}/master.m3u8`; - }, [playback]); - - const onSelectItem = useCallback( - (timeline: Timeline | undefined) => { - if (timeline) { - setFocusedItem(timeline); - const selected = timeline.timestamp; - playerRef.current?.pause(); - - let seekSeconds = 0; - (recordings || []).every((segment) => { - // if the next segment is past the desired time, stop calculating - if (segment.start_time > selected) { - return false; - } - - if (segment.end_time < selected) { - seekSeconds += segment.end_time - segment.start_time; - return true; - } - - seekSeconds += - segment.end_time - - segment.start_time - - (segment.end_time - selected); - return true; - }); - playerRef.current?.currentTime(seekSeconds); - } else { - setFocusedItem(undefined); - } - }, - [annotationOffset, recordings, playerRef] - ); - - const onScrubTime = useCallback( - (data: { time: Date }) => { - if (!playback.relevantPreview) { - return; - } - - if (playerRef.current?.paused() == false) { - setScrubbing(true); - playerRef.current?.pause(); - } - - const seekTimestamp = data.time.getTime() / 1000; - const seekTime = seekTimestamp - playback.relevantPreview.start; - setTimelineTime(seekTimestamp); - setTimeToSeek(Math.round(seekTime)); - }, - [scrubbing, playerRef, playback] - ); - - const onStopScrubbing = useCallback( - (data: { time: Date }) => { - const playbackTime = data.time.getTime() / 1000; - playerRef.current?.currentTime(playbackTime - playback.range.start); - setScrubbing(false); - playerRef.current?.play(); - }, - [playback, playerRef] - ); - - // handle seeking to next frame when seek is finished - useEffect(() => { - if (seeking) { - return; - } - - if (timeToSeek && timeToSeek != previewRef.current?.currentTime()) { - setSeeking(true); - previewRef.current?.currentTime(timeToSeek); - } - }, [timeToSeek, seeking]); - if (!config || !recordings) { return ; } return (
- <> -
- { - playerRef.current = player; - player.currentTime(timelineTime - playback.range.start); - player.on("playing", () => { - onSelectItem(undefined); - }); - }} - onDispose={() => { - playerRef.current = undefined; - }} - > - {config && focusedItem ? ( - - ) : undefined} - -
- {playback.relevantPreview && ( -
- { - previewRef.current = player; - player.pause(); - player.on("seeked", () => setSeeking(false)); - }} - onDispose={() => { - previewRef.current = undefined; - }} - /> -
- )} - + { + controllerRef.current = controller; + controllerRef.current.onPlayerTimeUpdate((timestamp: number) => { + setTimelineTime(timestamp); + }); + + if (playback.timelineItems.length > 0) { + controllerRef.current?.seekToTimestamp( + playback.timelineItems[0].timestamp, + true + ); + } + }} + />
{playback != undefined && ( { + controllerRef.current?.scrubToTimestamp( + data.time.getTime() / 1000 + ); + setTimelineTime(data.time.getTime() / 1000); + }} + timechangedHandler={(data) => { + controllerRef.current?.seekToTimestamp( + data.time.getTime() / 1000, + true + ); }} - timechangeHandler={onScrubTime} - timechangedHandler={onStopScrubbing} selectHandler={(data) => { if (data.items.length > 0) { const selected = parseFloat(data.items[0].split("-")[0]); - onSelectItem( - playback.timelineItems.find( - (timeline) => timeline.timestamp == selected - ) + const timeline = playback.timelineItems.find( + (timeline) => timeline.timestamp == selected ); + + if (timeline) { + controllerRef.current?.seekToTimelineItem(timeline); + } } }} /> diff --git a/web/tailwind.config.js b/web/tailwind.config.js index 696829f5e..dc84c91b1 100644 --- a/web/tailwind.config.js +++ b/web/tailwind.config.js @@ -80,6 +80,7 @@ module.exports = { "xs": "480px", "2xl": "1440px", "3xl": "1920px", + "4xl": "2560px", }, }, },