Preview player upgrades (#10152)

* Implement manual slider control for previews

* Automatically end preview video on mobile

* Show current time on timeline

* remove z height
This commit is contained in:
Nicolas Mowen 2024-02-29 17:58:50 -07:00 committed by GitHub
parent cb30450060
commit e7f8bca2c3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 244 additions and 56 deletions

View File

@ -2,7 +2,7 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { useApiHost } from "@/api"; import { useApiHost } from "@/api";
import { isCurrentHour } from "@/utils/dateUtil"; import { isCurrentHour } from "@/utils/dateUtil";
import { ReviewSegment } from "@/types/review"; import { ReviewSegment } from "@/types/review";
import { Slider } from "../ui/slider"; import { Slider } from "../ui/slider-no-thumb";
import { getIconForLabel, getIconForSubLabel } from "@/utils/iconUtil"; import { getIconForLabel, getIconForSubLabel } from "@/utils/iconUtil";
import TimeAgo from "../dynamic/TimeAgo"; import TimeAgo from "../dynamic/TimeAgo";
import useSWR from "swr"; import useSWR from "swr";
@ -29,6 +29,7 @@ type PreviewPlayerProps = {
autoPlayback?: boolean; autoPlayback?: boolean;
setReviewed?: (reviewId: string) => void; setReviewed?: (reviewId: string) => void;
onClick?: (reviewId: string) => void; onClick?: (reviewId: string) => void;
onTimeUpdate?: (time: number | undefined) => void;
}; };
type Preview = { type Preview = {
@ -44,25 +45,26 @@ export default function PreviewThumbnailPlayer({
relevantPreview, relevantPreview,
setReviewed, setReviewed,
onClick, onClick,
onTimeUpdate,
}: PreviewPlayerProps) { }: PreviewPlayerProps) {
const apiHost = useApiHost(); const apiHost = useApiHost();
const { data: config } = useSWR<FrigateConfig>("config"); const { data: config } = useSWR<FrigateConfig>("config");
const [hoverTimeout, setHoverTimeout] = useState<NodeJS.Timeout | null>(); const [hoverTimeout, setHoverTimeout] = useState<NodeJS.Timeout | null>();
const [playback, setPlayback] = useState(false); const [playback, setPlayback] = useState(false);
const [progress, setProgress] = useState(0); const [ignoreClick, setIgnoreClick] = useState(false);
const [imgRef, imgLoaded, onImgLoad] = useImageLoaded(); const [imgRef, imgLoaded, onImgLoad] = useImageLoaded();
// interaction // interaction
const handleOnClick = useCallback(() => { const handleOnClick = useCallback(() => {
if (onClick) { if (onClick && !ignoreClick) {
onClick(review.id); onClick(review.id);
} }
}, [review, onClick]); }, [ignoreClick, review, onClick]);
const swipeHandlers = useSwipeable({ const swipeHandlers = useSwipeable({
onSwipedLeft: () => setPlayback(false), onSwipedLeft: () => (setReviewed ? setReviewed(review.id) : null),
onSwipedRight: () => setPlayback(true), onSwipedRight: () => setPlayback(true),
preventScrollOnSwipe: true, preventScrollOnSwipe: true,
}); });
@ -92,7 +94,10 @@ export default function PreviewThumbnailPlayer({
} }
setPlayback(false); setPlayback(false);
setProgress(0);
if (onTimeUpdate) {
onTimeUpdate(undefined);
}
} }
}, },
@ -123,8 +128,10 @@ export default function PreviewThumbnailPlayer({
<PreviewContent <PreviewContent
review={review} review={review}
relevantPreview={relevantPreview} relevantPreview={relevantPreview}
setProgress={setProgress}
setReviewed={handleSetReviewed} setReviewed={handleSetReviewed}
setIgnoreClick={setIgnoreClick}
isPlayingBack={setPlayback}
onTimeUpdate={onTimeUpdate}
/> />
</div> </div>
)} )}
@ -174,15 +181,6 @@ export default function PreviewThumbnailPlayer({
</> </>
)} )}
</div> </div>
{playingBack && (
<Slider
className="absolute inset-x-0 bottom-0 z-10"
value={[progress]}
min={0}
step={1}
max={100}
/>
)}
{!playingBack && imgLoaded && review.has_been_reviewed && ( {!playingBack && imgLoaded && review.has_been_reviewed && (
<div className="absolute inset-0 z-10 bg-black bg-opacity-60" /> <div className="absolute inset-0 z-10 bg-black bg-opacity-60" />
)} )}
@ -193,23 +191,70 @@ export default function PreviewThumbnailPlayer({
); );
} }
const PREVIEW_PADDING = 16;
type PreviewContentProps = { type PreviewContentProps = {
review: ReviewSegment; review: ReviewSegment;
relevantPreview: Preview | undefined; relevantPreview: Preview | undefined;
setProgress?: (progress: number) => void;
setReviewed?: () => void; setReviewed?: () => void;
setIgnoreClick: (ignore: boolean) => void;
isPlayingBack: (ended: boolean) => void;
onTimeUpdate?: (time: number | undefined) => void;
}; };
function PreviewContent({ function PreviewContent({
review, review,
relevantPreview, relevantPreview,
setProgress,
setReviewed, setReviewed,
setIgnoreClick,
isPlayingBack,
onTimeUpdate,
}: PreviewContentProps) { }: PreviewContentProps) {
// preview
if (relevantPreview) {
return (
<VideoPreview
review={review}
relevantPreview={relevantPreview}
setReviewed={setReviewed}
setIgnoreClick={setIgnoreClick}
isPlayingBack={isPlayingBack}
onTimeUpdate={onTimeUpdate}
/>
);
} else if (isCurrentHour(review.start_time)) {
return (
<InProgressPreview
review={review}
setReviewed={setReviewed}
setIgnoreClick={setIgnoreClick}
isPlayingBack={isPlayingBack}
onTimeUpdate={onTimeUpdate}
/>
);
}
}
const PREVIEW_PADDING = 16;
type VideoPreviewProps = {
review: ReviewSegment;
relevantPreview: Preview;
setReviewed?: () => void;
setIgnoreClick: (ignore: boolean) => void;
isPlayingBack: (ended: boolean) => void;
onTimeUpdate?: (time: number | undefined) => void;
};
function VideoPreview({
review,
relevantPreview,
setReviewed,
setIgnoreClick,
isPlayingBack,
onTimeUpdate,
}: VideoPreviewProps) {
const playerRef = useRef<HTMLVideoElement | null>(null); const playerRef = useRef<HTMLVideoElement | null>(null);
// keep track of playback state // keep track of playback state
const [progress, setProgress] = useState(0);
const playerStartTime = useMemo(() => { const playerStartTime = useMemo(() => {
if (!relevantPreview) { if (!relevantPreview) {
return 0; return 0;
@ -224,6 +269,12 @@ function PreviewContent({
// we know that these deps are correct // we know that these deps are correct
// eslint-disable-next-line react-hooks/exhaustive-deps // eslint-disable-next-line react-hooks/exhaustive-deps
}, []); }, []);
const playerDuration = useMemo(
() => review.end_time - review.start_time + PREVIEW_PADDING,
// we know that these deps are correct
// eslint-disable-next-line react-hooks/exhaustive-deps
[],
);
const [lastPercent, setLastPercent] = useState(0.0); const [lastPercent, setLastPercent] = useState(0.0);
// initialize player correctly // initialize player correctly
@ -248,16 +299,16 @@ function PreviewContent({
// time progress update // time progress update
const onProgress = useCallback(() => { const onProgress = useCallback(() => {
if (!setProgress) { if (onTimeUpdate) {
return; onTimeUpdate(
relevantPreview.start + (playerRef.current?.currentTime || 0),
);
} }
const playerProgress = const playerProgress =
(playerRef.current?.currentTime || 0) - playerStartTime; (playerRef.current?.currentTime || 0) - playerStartTime;
// end with a bit of padding // end with a bit of padding
const playerDuration =
review.end_time - review.start_time + PREVIEW_PADDING;
const playerPercent = (playerProgress / playerDuration) * 100; const playerPercent = (playerProgress / playerDuration) * 100;
if ( if (
@ -272,7 +323,16 @@ function PreviewContent({
setLastPercent(playerPercent); setLastPercent(playerPercent);
if (playerPercent > 100) { if (playerPercent > 100) {
if (isMobile) {
isPlayingBack(false);
if (onTimeUpdate) {
onTimeUpdate(undefined);
}
} else {
playerRef.current?.pause(); playerRef.current?.pause();
}
setManualPlayback(false); setManualPlayback(false);
setProgress(100.0); setProgress(100.0);
} else { } else {
@ -306,10 +366,53 @@ function PreviewContent({
// eslint-disable-next-line react-hooks/exhaustive-deps // eslint-disable-next-line react-hooks/exhaustive-deps
}, [manualPlayback, playerRef]); }, [manualPlayback, playerRef]);
// preview // user interaction
const onManualSeek = useCallback(
(values: number[]) => {
const value = values[0];
if (!playerRef.current) {
return;
}
if (manualPlayback) {
setManualPlayback(false);
setIgnoreClick(true);
}
if (playerRef.current.paused == false) {
playerRef.current.pause();
setIgnoreClick(true);
}
setProgress(value);
playerRef.current.currentTime =
playerStartTime + (value / 100.0) * playerDuration;
},
[
manualPlayback,
playerDuration,
playerRef,
playerStartTime,
setIgnoreClick,
],
);
const onStopManualSeek = useCallback(() => {
setTimeout(() => {
setIgnoreClick(false);
if (isSafari || (isFirefox && isMobile)) {
setManualPlayback(true);
} else {
playerRef.current?.play();
}
}, 500);
}, [playerRef, setIgnoreClick]);
if (relevantPreview) {
return ( return (
<div className="relative size-full aspect-video bg-black">
<video <video
ref={playerRef} ref={playerRef}
className="size-full aspect-video bg-black" className="size-full aspect-video bg-black"
@ -321,35 +424,41 @@ function PreviewContent({
> >
<source src={relevantPreview.src} type={relevantPreview.type} /> <source src={relevantPreview.src} type={relevantPreview.type} />
</video> </video>
); <Slider
} else if (isCurrentHour(review.start_time)) { className="absolute inset-x-0 bottom-0"
return ( value={[progress]}
<InProgressPreview onValueChange={onManualSeek}
review={review} onValueCommit={onStopManualSeek}
setProgress={setProgress} min={0}
setReviewed={setReviewed} step={1}
max={100}
/> />
</div>
); );
} }
}
const MIN_LOAD_TIMEOUT_MS = 200; const MIN_LOAD_TIMEOUT_MS = 200;
type InProgressPreviewProps = { type InProgressPreviewProps = {
review: ReviewSegment; review: ReviewSegment;
setProgress?: (progress: number) => void;
setReviewed?: (reviewId: string) => void; setReviewed?: (reviewId: string) => void;
setIgnoreClick: (ignore: boolean) => void;
isPlayingBack: (ended: boolean) => void;
onTimeUpdate?: (time: number | undefined) => void;
}; };
function InProgressPreview({ function InProgressPreview({
review, review,
setProgress,
setReviewed, setReviewed,
setIgnoreClick,
isPlayingBack,
onTimeUpdate,
}: InProgressPreviewProps) { }: InProgressPreviewProps) {
const apiHost = useApiHost(); const apiHost = useApiHost();
const { data: previewFrames } = useSWR<string[]>( const { data: previewFrames } = useSWR<string[]>(
`preview/${review.camera}/start/${Math.floor(review.start_time) - 4}/end/${ `preview/${review.camera}/start/${Math.floor(review.start_time) - PREVIEW_PADDING}/end/${
Math.ceil(review.end_time) + 4 Math.ceil(review.end_time) + PREVIEW_PADDING
}/frames`, }/frames`,
); );
const [manualFrame, setManualFrame] = useState(false);
const [key, setKey] = useState(0); const [key, setKey] = useState(0);
const handleLoad = useCallback(() => { const handleLoad = useCallback(() => {
@ -357,19 +466,27 @@ function InProgressPreview({
return; return;
} }
if (onTimeUpdate) {
onTimeUpdate(review.start_time - PREVIEW_PADDING + key);
}
if (manualFrame) {
return;
}
if (key == previewFrames.length - 1) { if (key == previewFrames.length - 1) {
if (setProgress) { if (isMobile) {
setProgress(100); isPlayingBack(false);
if (onTimeUpdate) {
onTimeUpdate(undefined);
}
} }
return; return;
} }
setTimeout(() => { setTimeout(() => {
if (setProgress) {
setProgress((key / (previewFrames.length - 1)) * 100);
}
if (setReviewed && key == Math.floor(previewFrames.length / 2)) { if (setReviewed && key == Math.floor(previewFrames.length / 2)) {
setReviewed(review.id); setReviewed(review.id);
} }
@ -379,7 +496,35 @@ function InProgressPreview({
// we know that these deps are correct // we know that these deps are correct
// eslint-disable-next-line react-hooks/exhaustive-deps // eslint-disable-next-line react-hooks/exhaustive-deps
}, [key, previewFrames]); }, [key, manualFrame, previewFrames]);
// user interaction
const onManualSeek = useCallback(
(values: number[]) => {
const value = values[0];
if (!manualFrame) {
setManualFrame(true);
setIgnoreClick(true);
}
setKey(value);
},
[manualFrame, setIgnoreClick, setManualFrame, setKey],
);
const onStopManualSeek = useCallback(
(values: number[]) => {
const value = values[0];
setTimeout(() => {
setIgnoreClick(false);
setManualFrame(false);
setKey(value - 1);
}, 500);
},
[setManualFrame, setIgnoreClick],
);
if (!previewFrames || previewFrames.length == 0) { if (!previewFrames || previewFrames.length == 0) {
return ( return (
@ -391,12 +536,21 @@ function InProgressPreview({
} }
return ( return (
<div className="size-full flex items-center bg-black"> <div className="relative size-full flex items-center bg-black">
<img <img
className="size-full object-contain" className="size-full object-contain"
src={`${apiHost}api/preview/${previewFrames[key]}/thumbnail.jpg`} src={`${apiHost}api/preview/${previewFrames[key]}/thumbnail.jpg`}
onLoad={handleLoad} onLoad={handleLoad}
/> />
<Slider
className="absolute inset-x-0 bottom-0"
value={[key]}
onValueChange={onManualSeek}
onValueCommit={onStopManualSeek}
min={0}
step={1}
max={previewFrames.length - 1}
/>
</div> </div>
); );
} }

View File

@ -0,0 +1,26 @@
import * as React from "react";
import * as SliderPrimitive from "@radix-ui/react-slider";
import { cn } from "@/lib/utils";
const Slider = React.forwardRef<
React.ElementRef<typeof SliderPrimitive.Root>,
React.ComponentPropsWithoutRef<typeof SliderPrimitive.Root>
>(({ className, ...props }, ref) => (
<SliderPrimitive.Root
ref={ref}
className={cn(
"relative flex w-full touch-none select-none items-center",
className,
)}
{...props}
>
<SliderPrimitive.Track className="relative h-1 w-full grow overflow-hidden rounded-full">
<SliderPrimitive.Range className="absolute h-full bg-blue-500" />
</SliderPrimitive.Track>
<SliderPrimitive.Thumb className="block h-2 w-12 rounded-full border-2 border-transparent bg-transparent ring-offset-transparent focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50" />
</SliderPrimitive.Root>
));
Slider.displayName = SliderPrimitive.Root.displayName;
export { Slider };

View File

@ -1,7 +1,7 @@
import * as React from "react"; import * as React from "react"
import * as SliderPrimitive from "@radix-ui/react-slider"; import * as SliderPrimitive from "@radix-ui/react-slider"
import { cn } from "@/lib/utils"; import { cn } from "@/lib/utils"
const Slider = React.forwardRef< const Slider = React.forwardRef<
React.ElementRef<typeof SliderPrimitive.Root>, React.ElementRef<typeof SliderPrimitive.Root>,
@ -15,11 +15,12 @@ const Slider = React.forwardRef<
)} )}
{...props} {...props}
> >
<SliderPrimitive.Track className="relative h-1 w-full grow overflow-hidden rounded-full"> <SliderPrimitive.Track className="relative h-2 w-full grow overflow-hidden rounded-full bg-secondary">
<SliderPrimitive.Range className="absolute h-full bg-blue-500" /> <SliderPrimitive.Range className="absolute h-full bg-primary" />
</SliderPrimitive.Track> </SliderPrimitive.Track>
<SliderPrimitive.Thumb className="block h-5 w-5 rounded-full border-2 border-primary bg-background ring-offset-background transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50" />
</SliderPrimitive.Root> </SliderPrimitive.Root>
)); ))
Slider.displayName = SliderPrimitive.Root.displayName; Slider.displayName = SliderPrimitive.Root.displayName
export { Slider }; export { Slider }

View File

@ -187,6 +187,10 @@ export default function EventView({
return data; return data;
}, [minimap]); }, [minimap]);
// preview playback
const [previewTime, setPreviewTime] = useState<number>();
if (!config) { if (!config) {
return <ActivityIndicator />; return <ActivityIndicator />;
} }
@ -287,6 +291,7 @@ export default function EventView({
review={value} review={value}
relevantPreview={relevantPreview} relevantPreview={relevantPreview}
setReviewed={markItemAsReviewed} setReviewed={markItemAsReviewed}
onTimeUpdate={setPreviewTime}
onClick={onSelectReview} onClick={onSelectReview}
/> />
</div> </div>
@ -308,6 +313,8 @@ export default function EventView({
showMinimap={showMinimap} showMinimap={showMinimap}
minimapStartTime={minimapBounds.start} minimapStartTime={minimapBounds.start}
minimapEndTime={minimapBounds.end} minimapEndTime={minimapBounds.end}
showHandlebar={previewTime != undefined}
handlebarTime={previewTime}
events={reviewItems.all} events={reviewItems.all}
severityType={severity} severityType={severity}
contentRef={contentRef} contentRef={contentRef}