Fix motion and recording views (#10318)

* Fix switching between different hours

* Simplify book keeping of different time ranges

* Rewrite player to handle changing of previews outside of video player

* Simplify switching between cameras

* Fix previews causing cameras to not load

* Simplify listeners

* Always clear existing listener
This commit is contained in:
Nicolas Mowen 2024-03-07 17:31:43 -07:00 committed by GitHub
parent fc6d6a4e9a
commit 90a40d2509
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 231 additions and 153 deletions

View File

@ -12,7 +12,6 @@ import TimelineEventOverlay from "../overlay/TimelineDataOverlay";
import { useApiHost } from "@/api";
import useSWR from "swr";
import { FrigateConfig } from "@/types/frigateConfig";
import ActivityIndicator from "../indicators/activity-indicator";
import useKeyboardListener from "@/hooks/use-keyboard-listener";
import { Recording } from "@/types/record";
import { Preview } from "@/types/preview";
@ -29,6 +28,7 @@ type DynamicVideoPlayerProps = {
timeRange: { start: number; end: number };
cameraPreviews: Preview[];
previewOnly?: boolean;
preloadRecordings: boolean;
onControllerReady: (controller: DynamicVideoController) => void;
onClick?: () => void;
};
@ -38,6 +38,7 @@ export default function DynamicVideoPlayer({
timeRange,
cameraPreviews,
previewOnly = false,
preloadRecordings = true,
onControllerReady,
onClick,
}: DynamicVideoPlayerProps) {
@ -64,18 +65,19 @@ export default function DynamicVideoPlayer({
// controlling playback
const playerRef = useRef<Player | undefined>(undefined);
const [playerRef, setPlayerRef] = useState<Player | undefined>(undefined);
const previewRef = useRef<HTMLVideoElement | null>(null);
const [isScrubbing, setIsScrubbing] = useState(previewOnly);
const [focusedItem, setFocusedItem] = useState<Timeline | undefined>(
undefined,
);
const controller = useMemo(() => {
if (!config) {
if (!config || !playerRef || !previewRef.current) {
return undefined;
}
return new DynamicVideoController(
camera,
playerRef,
previewRef,
(config.cameras[camera]?.detect?.annotation_offset || 0) / 1000,
@ -83,10 +85,12 @@ export default function DynamicVideoPlayer({
setIsScrubbing,
setFocusedItem,
);
}, [camera, config, previewOnly]);
// we only want to fire once when players are ready
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [camera, config, playerRef, previewRef]);
useEffect(() => {
if (!playerRef.current && !previewRef.current) {
if (!controller) {
return;
}
@ -96,7 +100,29 @@ export default function DynamicVideoPlayer({
// we only want to fire once when players are ready
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [playerRef, previewRef]);
}, [controller]);
const [initPreviewOnly, setInitPreviewOnly] = useState(previewOnly);
useEffect(() => {
if (!controller || !playerRef) {
return;
}
if (previewOnly == initPreviewOnly) {
return;
}
if (previewOnly) {
playerRef.autoplay(false);
} else {
controller.seekToTimestamp(playerRef.currentTime() || 0, true);
}
setInitPreviewOnly(previewOnly);
// we only want to fire once when players are ready
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [controller, previewOnly]);
const [hasRecordingAtTime, setHasRecordingAtTime] = useState(true);
@ -107,33 +133,33 @@ export default function DynamicVideoPlayer({
switch (key) {
case "ArrowLeft":
if (down) {
const currentTime = playerRef.current?.currentTime();
const currentTime = playerRef?.currentTime();
if (currentTime) {
playerRef.current?.currentTime(Math.max(0, currentTime - 5));
playerRef?.currentTime(Math.max(0, currentTime - 5));
}
}
break;
case "ArrowRight":
if (down) {
const currentTime = playerRef.current?.currentTime();
const currentTime = playerRef?.currentTime();
if (currentTime) {
playerRef.current?.currentTime(currentTime + 5);
playerRef?.currentTime(currentTime + 5);
}
}
break;
case "m":
if (down && !repeat && playerRef.current) {
playerRef.current.muted(!playerRef.current.muted());
if (down && !repeat && playerRef) {
playerRef.muted(!playerRef.muted());
}
break;
case " ":
if (down && playerRef.current) {
if (playerRef.current.paused()) {
playerRef.current.play();
if (down && playerRef) {
if (playerRef.paused()) {
playerRef.play();
} else {
playerRef.current.pause();
playerRef.pause();
}
}
break;
@ -236,31 +262,30 @@ export default function DynamicVideoPlayer({
recordings: recordings ?? [],
playbackUri,
preview,
timeRange,
});
// we only want this to change when recordings update
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [controller, recordings]);
if (!controller) {
return <ActivityIndicator />;
}
return (
<div
className={`relative ${className ?? ""} ${onClick ? (hasRecordingAtTime ? "cursor-pointer" : "") : ""}`}
onClick={onClick}
>
{!previewOnly && (
{preloadRecordings && (
<div
className={`w-full relative ${
currentPreview != undefined && isScrubbing ? "hidden" : "visible"
previewOnly || (currentPreview != undefined && isScrubbing)
? "hidden"
: "visible"
}`}
>
<VideoPlayer
options={{
preload: "auto",
autoplay: true,
autoplay: !previewOnly,
sources: [initialPlaybackSource],
aspectRatio: tallVideo ? "16:9" : undefined,
controlBar: {
@ -272,17 +297,10 @@ export default function DynamicVideoPlayer({
}}
seekOptions={{ forward: 10, backward: 5 }}
onReady={(player) => {
playerRef.current = player;
player.on("playing", () => setFocusedItem(undefined));
player.on("timeupdate", () => {
controller.updateProgress(player.currentTime() || 0);
});
player.on("ended", () =>
controller.fireClipChangeEvent("forward"),
);
setPlayerRef(player);
}}
onDispose={() => {
playerRef.current = undefined;
setPlayerRef(undefined);
}}
>
{config && focusedItem && (
@ -303,7 +321,7 @@ export default function DynamicVideoPlayer({
muted
disableRemotePlayback
onSeeked={onPreviewSeeked}
onLoadedData={() => controller.previewReady()}
onLoadedData={() => controller?.previewReady()}
>
{currentPreview != undefined && (
<source src={currentPreview.src} type={currentPreview.type} />
@ -315,35 +333,39 @@ export default function DynamicVideoPlayer({
export class DynamicVideoController {
// main state
private playerRef: MutableRefObject<Player | undefined>;
public camera = "";
private playerRef: Player;
private previewRef: MutableRefObject<HTMLVideoElement | null>;
private setScrubbing: (isScrubbing: boolean) => void;
private setFocusedItem: (timeline: Timeline) => void;
private playerMode: PlayerMode = "playback";
private timeRange: { start: number; end: number } | undefined = undefined;
// playback
private recordings: Recording[] = [];
private onPlaybackTimestamp: ((time: number) => void) | undefined = undefined;
private onClipChange: ((dir: "forward" | "backward") => void) | undefined =
undefined;
private annotationOffset: number;
private timeToStart: number | undefined = undefined;
private clipChangeLockout: boolean = true;
// preview
private preview: Preview | undefined = undefined;
private timeToSeek: number | undefined = undefined;
private seeking = false;
private readyToScrub = true;
// listeners
private playerProgressListener: (() => void) | null = null;
private playerEndedListener: (() => void) | null = null;
private canPlayListener: (() => void) | null = null;
constructor(
playerRef: MutableRefObject<Player | undefined>,
camera: string,
playerRef: Player,
previewRef: MutableRefObject<HTMLVideoElement | null>,
annotationOffset: number,
defaultMode: PlayerMode,
setScrubbing: (isScrubbing: boolean) => void,
setFocusedItem: (timeline: Timeline) => void,
) {
this.camera = camera;
this.playerRef = playerRef;
this.previewRef = previewRef;
this.annotationOffset = annotationOffset;
@ -355,7 +377,7 @@ export class DynamicVideoController {
newPlayback(newPlayback: DynamicPlayback) {
this.recordings = newPlayback.recordings;
this.playerRef.current?.src({
this.playerRef.src({
src: newPlayback.playbackUri,
type: "application/vnd.apple.mpegurl",
});
@ -366,6 +388,13 @@ export class DynamicVideoController {
}
this.preview = newPlayback.preview;
if (this.preview) {
this.seeking = false;
this.timeToSeek = undefined;
}
this.timeRange = newPlayback.timeRange;
}
seekToTimestamp(time: number, play: boolean = false) {
@ -396,99 +425,86 @@ export class DynamicVideoController {
segment.end_time - segment.start_time - (segment.end_time - time);
return true;
});
this.playerRef.current?.currentTime(seekSeconds);
this.playerRef.currentTime(seekSeconds);
if (play) {
this.playerRef.current?.play();
this.playerRef.play();
} else {
this.playerRef.pause();
}
}
onCanPlay(listener: (() => void) | null) {
if (listener) {
this.canPlayListener = listener;
this.playerRef.on("canplay", this.canPlayListener);
} else {
if (this.canPlayListener) {
this.playerRef.off("canplay", this.canPlayListener);
this.canPlayListener = null;
}
}
}
seekToTimelineItem(timeline: Timeline) {
this.playerRef.current?.pause();
this.playerRef.pause();
this.seekToTimestamp(timeline.timestamp + this.annotationOffset);
this.setFocusedItem(timeline);
}
updateProgress(playerTime: number) {
if (this.onPlaybackTimestamp) {
// take a player time in seconds and convert to timestamp in timeline
let timestamp = 0;
let totalTime = 0;
(this.recordings || []).every((segment) => {
if (totalTime + segment.duration > playerTime) {
// segment is here
timestamp = segment.start_time + (playerTime - totalTime);
return false;
} else {
totalTime += segment.duration;
return true;
}
});
getProgress(playerTime: number): number {
// take a player time in seconds and convert to timestamp in timeline
let timestamp = 0;
let totalTime = 0;
(this.recordings || []).every((segment) => {
if (totalTime + segment.duration > playerTime) {
// segment is here
timestamp = segment.start_time + (playerTime - totalTime);
return false;
} else {
totalTime += segment.duration;
return true;
}
});
this.onPlaybackTimestamp(timestamp);
}
return timestamp;
}
onPlayerTimeUpdate(listener: ((timestamp: number) => void) | undefined) {
this.onPlaybackTimestamp = listener;
if (this.playerProgressListener) {
this.playerRef.off("timeupdate", this.playerProgressListener);
}
if (listener) {
this.playerProgressListener = () =>
listener(this.getProgress(this.playerRef.currentTime() || 0));
this.playerRef.on("timeupdate", this.playerProgressListener);
}
}
onClipChangedEvent(listener: (dir: "forward" | "backward") => void) {
this.onClipChange = listener;
}
onClipChangedEvent(listener: ((dir: "forward") => void) | undefined) {
if (this.playerEndedListener) {
this.playerRef.off("ended", this.playerEndedListener);
}
fireClipChangeEvent(dir: "forward" | "backward") {
if (this.onClipChange) {
this.onClipChange(dir);
if (listener) {
this.playerEndedListener = () => listener("forward");
this.playerRef.on("ended", this.playerEndedListener);
}
}
scrubToTimestamp(time: number) {
if (!this.preview) {
if (!this.preview || !this.timeRange) {
return;
}
if (!this.readyToScrub) {
return;
}
if (time > this.preview.end) {
if (this.clipChangeLockout && time - this.preview.end < 30) {
return;
}
if (this.playerMode == "scrubbing") {
this.playerMode = "playback";
this.setScrubbing(false);
this.timeToSeek = undefined;
this.seeking = false;
this.readyToScrub = false;
this.clipChangeLockout = true;
this.fireClipChangeEvent("forward");
}
return;
}
if (time < this.preview.start) {
if (this.clipChangeLockout && this.preview.start - time < 30) {
return;
}
if (this.playerMode == "scrubbing") {
this.playerMode = "playback";
this.setScrubbing(false);
this.timeToSeek = undefined;
this.seeking = false;
this.readyToScrub = false;
this.clipChangeLockout = true;
this.fireClipChangeEvent("backward");
}
if (time < this.preview.start || time > this.preview.end) {
return;
}
if (this.playerMode != "scrubbing") {
this.playerMode = "scrubbing";
this.playerRef.current?.pause();
this.playerRef.pause();
this.setScrubbing(true);
}
@ -514,8 +530,6 @@ export class DynamicVideoController {
return;
}
this.clipChangeLockout = false;
if (
this.timeToSeek &&
this.timeToSeek != this.previewRef.current?.currentTime
@ -529,7 +543,6 @@ export class DynamicVideoController {
previewReady() {
this.previewRef.current?.pause();
this.readyToScrub = true;
}
hasRecordingAtTime(time: number): boolean {

View File

@ -5,4 +5,5 @@ export type DynamicPlayback = {
recordings: Recording[];
playbackUri: string;
preview: Preview | undefined;
timeRange: { end: number; start: number };
};

View File

@ -579,12 +579,17 @@ function MotionReview({
() => getChunkedTimeRange(timeRange.after, lastFullHour),
[lastFullHour, timeRange],
);
const [selectedRangeIdx, setSelectedRangeIdx] = useState(
timeRangeSegments.ranges.length - 1,
);
const [currentTime, setCurrentTime] = useState<number>(
timeRangeSegments.ranges[selectedRangeIdx].start,
);
const currentTimeRange = useMemo(
() => timeRangeSegments.ranges[selectedRangeIdx],
[selectedRangeIdx, timeRangeSegments],
);
// move to next clip
@ -600,23 +605,38 @@ function MotionReview({
if (firstController) {
firstController.onClipChangedEvent((dir) => {
if (
dir == "forward" &&
selectedRangeIdx < timeRangeSegments.ranges.length - 1
) {
setSelectedRangeIdx(selectedRangeIdx + 1);
} else if (selectedRangeIdx > 0) {
setSelectedRangeIdx(selectedRangeIdx - 1);
if (dir == "forward") {
if (selectedRangeIdx < timeRangeSegments.ranges.length - 1) {
setSelectedRangeIdx(selectedRangeIdx + 1);
}
}
});
}
}, [selectedRangeIdx, timeRangeSegments, videoPlayersRef, playerReady]);
useEffect(() => {
if (
currentTime > currentTimeRange.end + 60 ||
currentTime < currentTimeRange.start - 60
) {
const index = timeRangeSegments.ranges.findIndex(
(seg) => seg.start <= currentTime && seg.end >= currentTime,
);
if (index != -1) {
setSelectedRangeIdx(index);
}
return;
}
Object.values(videoPlayersRef.current).forEach((controller) => {
controller.scrubToTimestamp(currentTime);
});
}, [currentTime]);
}, [currentTime, currentTimeRange, timeRangeSegments]);
if (!relevantPreviews) {
return <ActivityIndicator />;
}
return (
<>
@ -640,9 +660,10 @@ function MotionReview({
key={camera.name}
className={`${grow}`}
camera={camera.name}
timeRange={timeRangeSegments.ranges[selectedRangeIdx]}
timeRange={currentTimeRange}
cameraPreviews={relevantPreviews || []}
previewOnly
preloadRecordings={false}
onControllerReady={(controller) => {
videoPlayersRef.current[camera.name] = controller;
setPlayerReady(true);

View File

@ -49,6 +49,10 @@ export function DesktopRecordingView({
return chunk.start <= startTime && chunk.end >= startTime;
}),
);
const currentTimeRange = useMemo(
() => timeRange.ranges[selectedRangeIdx],
[selectedRangeIdx, timeRange],
);
// move to next clip
useEffect(() => {
@ -59,21 +63,18 @@ export function DesktopRecordingView({
return;
}
const firstController = Object.values(videoPlayersRef.current)[0];
const mainController = videoPlayersRef.current[mainCamera];
if (firstController) {
firstController.onClipChangedEvent((dir) => {
if (
dir == "forward" &&
selectedRangeIdx < timeRange.ranges.length - 1
) {
setSelectedRangeIdx(selectedRangeIdx + 1);
} else if (selectedRangeIdx > 0) {
setSelectedRangeIdx(selectedRangeIdx - 1);
if (mainController) {
mainController.onClipChangedEvent((dir) => {
if (dir == "forward") {
if (selectedRangeIdx < timeRange.ranges.length - 1) {
setSelectedRangeIdx(selectedRangeIdx + 1);
}
}
});
}
}, [selectedRangeIdx, timeRange, videoPlayersRef, playerReady]);
}, [selectedRangeIdx, timeRange, videoPlayersRef, playerReady, mainCamera]);
// scrubbing and timeline state
@ -82,11 +83,25 @@ export function DesktopRecordingView({
useEffect(() => {
if (scrubbing) {
if (
currentTime > currentTimeRange.end + 60 ||
currentTime < currentTimeRange.start - 60
) {
const index = timeRange.ranges.findIndex(
(seg) => seg.start <= currentTime && seg.end >= currentTime,
);
if (index != -1) {
setSelectedRangeIdx(index);
}
return;
}
Object.values(videoPlayersRef.current).forEach((controller) => {
controller.scrubToTimestamp(currentTime);
});
}
}, [currentTime, scrubbing]);
}, [currentTime, scrubbing, timeRange, currentTimeRange]);
useEffect(() => {
if (!scrubbing) {
@ -99,22 +114,26 @@ export function DesktopRecordingView({
const onSelectCamera = useCallback(
(newCam: string) => {
videoPlayersRef.current[mainCamera].onPlayerTimeUpdate(undefined);
videoPlayersRef.current[mainCamera].scrubToTimestamp(currentTime);
videoPlayersRef.current[newCam].seekToTimestamp(currentTime, true);
videoPlayersRef.current[newCam].onPlayerTimeUpdate(
(timestamp: number) => {
setCurrentTime(timestamp);
const lastController = videoPlayersRef.current[mainCamera];
const newController = videoPlayersRef.current[newCam];
lastController.onPlayerTimeUpdate(undefined);
lastController.onClipChangedEvent(undefined);
lastController.scrubToTimestamp(currentTime);
newController.onCanPlay(() => {
newController.seekToTimestamp(currentTime, true);
newController.onCanPlay(null);
});
newController.onPlayerTimeUpdate((timestamp: number) => {
setCurrentTime(timestamp);
allCameras.forEach((cam) => {
if (cam != newCam) {
videoPlayersRef.current[cam]?.scrubToTimestamp(
Math.floor(timestamp),
);
}
});
},
);
allCameras.forEach((cam) => {
if (cam != newCam) {
videoPlayersRef.current[cam]?.scrubToTimestamp(
Math.floor(timestamp),
);
}
});
});
setMainCamera(newCam);
},
[allCameras, currentTime, mainCamera],
@ -155,8 +174,9 @@ export function DesktopRecordingView({
>
<DynamicVideoPlayer
camera={cam}
timeRange={timeRange.ranges[selectedRangeIdx]}
timeRange={currentTimeRange}
cameraPreviews={allPreviews ?? []}
preloadRecordings
onControllerReady={(controller) => {
videoPlayersRef.current[cam] = controller;
setPlayerReady(true);
@ -172,7 +192,10 @@ export function DesktopRecordingView({
});
});
controller.seekToTimestamp(startTime, true);
controller.onCanPlay(() => {
controller.seekToTimestamp(startTime, true);
controller.onCanPlay(null);
});
}}
/>
</div>
@ -184,9 +207,10 @@ export function DesktopRecordingView({
<DynamicVideoPlayer
className="size-full"
camera={cam}
timeRange={timeRange.ranges[selectedRangeIdx]}
timeRange={currentTimeRange}
cameraPreviews={allPreviews ?? []}
previewOnly
preloadRecordings
onControllerReady={(controller) => {
videoPlayersRef.current[cam] = controller;
setPlayerReady(true);
@ -265,6 +289,10 @@ export function MobileRecordingView({
return chunk.start <= startTime && chunk.end >= startTime;
}),
);
const currentTimeRange = useMemo(
() => timeRange.ranges[selectedRangeIdx],
[selectedRangeIdx, timeRange],
);
// move to next clip
useEffect(() => {
@ -273,10 +301,10 @@ export function MobileRecordingView({
}
controllerRef.current.onClipChangedEvent((dir) => {
if (dir == "forward" && selectedRangeIdx < timeRange.ranges.length - 1) {
setSelectedRangeIdx(selectedRangeIdx + 1);
} else if (selectedRangeIdx > 0) {
setSelectedRangeIdx(selectedRangeIdx - 1);
if (dir == "forward") {
if (selectedRangeIdx < timeRange.ranges.length - 1) {
setSelectedRangeIdx(selectedRangeIdx + 1);
}
}
});
}, [playerReady, selectedRangeIdx, timeRange]);
@ -290,9 +318,23 @@ export function MobileRecordingView({
useEffect(() => {
if (scrubbing) {
if (
currentTime > currentTimeRange.end + 60 ||
currentTime < currentTimeRange.start - 60
) {
const index = timeRange.ranges.findIndex(
(seg) => seg.start <= currentTime && seg.end >= currentTime,
);
if (index != -1) {
setSelectedRangeIdx(index);
}
return;
}
controllerRef.current?.scrubToTimestamp(currentTime);
}
}, [currentTime, scrubbing]);
}, [currentTime, scrubbing, currentTimeRange, timeRange]);
useEffect(() => {
if (!scrubbing) {
@ -328,8 +370,9 @@ export function MobileRecordingView({
<div>
<DynamicVideoPlayer
camera={startCamera}
timeRange={timeRange.ranges[selectedRangeIdx]}
timeRange={currentTimeRange}
cameraPreviews={relevantPreviews || []}
preloadRecordings
onControllerReady={(controller) => {
controllerRef.current = controller;
setPlayerReady(true);