Improve live streaming (#16447)

* config file changes

* config migrator

* stream selection on single camera live view

* camera streaming settings dialog

* manage persistent group streaming settings

* apply streaming settings in camera groups

* add ability to clear all streaming settings from settings

* docs

* update reference config

* fixes

* clarify docs

* use first stream as default in dialog

* ensure still image is visible after switching stream type to none

* docs

* clarify docs

* add ability to continue playing stream in background

* fix props

* put stream selection inside dropdown on desktop

* add capabilities to live mode hook

* live context menu component

* resize observer: only return new dimensions if they've actually changed

* pass volume prop to players

* fix slider bug, https://github.com/shadcn-ui/ui/issues/1448

* update react-grid-layout

* prevent animated transitions on draggable grid layout

* add context menu to dashboards

* use provider

* streaming dialog from context menu

* docs

* add jsmpeg warning to context menu

* audio and two way talk indicators in single camera view

* add link to debug view

* don't use hook

* create manual events from live camera view

* maintain grow classes on grid items

* fix initial volume state on default dashboard

* fix pointer events causing context menu to end up underneath image on iOS

* mobile drawer tweaks

* stream stats

* show settings menu for non-restreamed cameras

* consistent settings icon

* tweaks

* optional stats to fix birdseye player

* add toaster to live camera view

* fix crash on initial save in streaming dialog

* don't require restreaming for context menu streaming settings

* add debug view to context menu

* stats fixes

* update docs

* always show stream info when restreamed

* update camera streaming dialog

* make note of no h265 support for webrtc

* docs clarity

* ensure docs show streams as a dict

* docs clarity

* fix css file

* tweaks
This commit is contained in:
Josh Hawkins
2025-02-10 10:42:35 -06:00
committed by GitHub
parent 2a28964e63
commit dd7820e4ee
31 changed files with 2681 additions and 219 deletions

View File

@@ -58,6 +58,7 @@ export default function BirdseyeLivePlayer({
height={birdseyeConfig.height}
containerRef={containerRef}
playbackEnabled={true}
useWebGL={true}
/>
);
} else {

View File

@@ -1,6 +1,7 @@
import { baseUrl } from "@/api/baseUrl";
import { useResizeObserver } from "@/hooks/resize-observer";
import { cn } from "@/lib/utils";
import { PlayerStatsType } from "@/types/live";
// @ts-expect-error we know this doesn't have types
import JSMpeg from "@cycjimmy/jsmpeg-player";
import React, { useEffect, useMemo, useRef, useState } from "react";
@@ -12,6 +13,8 @@ type JSMpegPlayerProps = {
height: number;
containerRef: React.MutableRefObject<HTMLDivElement | null>;
playbackEnabled: boolean;
useWebGL: boolean;
setStats?: (stats: PlayerStatsType) => void;
onPlaying?: () => void;
};
@@ -22,6 +25,8 @@ export default function JSMpegPlayer({
className,
containerRef,
playbackEnabled,
useWebGL = false,
setStats,
onPlaying,
}: JSMpegPlayerProps) {
const url = `${baseUrl.replace(/^http/, "ws")}live/jsmpeg/${camera}`;
@@ -33,6 +38,9 @@ export default function JSMpegPlayer({
const [hasData, setHasData] = useState(false);
const hasDataRef = useRef(hasData);
const [dimensionsReady, setDimensionsReady] = useState(false);
const bytesReceivedRef = useRef(0);
const lastTimestampRef = useRef(Date.now());
const statsIntervalRef = useRef<NodeJS.Timeout | null>(null);
const selectedContainerRef = useMemo(
() => (containerRef.current ? containerRef : internalContainerRef),
@@ -111,6 +119,8 @@ export default function JSMpegPlayer({
const canvas = canvasRef.current;
let videoElement: JSMpeg.VideoElement | null = null;
let frameCount = 0;
setHasData(false);
if (videoWrapper && playbackEnabled) {
@@ -123,21 +133,68 @@ export default function JSMpegPlayer({
{
protocols: [],
audio: false,
disableGl: camera != "birdseye",
disableWebAssembly: camera != "birdseye",
disableGl: !useWebGL,
disableWebAssembly: !useWebGL,
videoBufferSize: 1024 * 1024 * 4,
onVideoDecode: () => {
if (!hasDataRef.current) {
setHasData(true);
onPlayingRef.current?.();
}
frameCount++;
},
},
);
// Set up WebSocket message handler
if (
videoElement.player &&
videoElement.player.source &&
videoElement.player.source.socket
) {
const socket = videoElement.player.source.socket;
socket.addEventListener("message", (event: MessageEvent) => {
if (event.data instanceof ArrayBuffer) {
bytesReceivedRef.current += event.data.byteLength;
}
});
}
// Update stats every second
statsIntervalRef.current = setInterval(() => {
const currentTimestamp = Date.now();
const timeDiff = (currentTimestamp - lastTimestampRef.current) / 1000; // in seconds
const bitrate = (bytesReceivedRef.current * 8) / timeDiff / 1000; // in kbps
setStats?.({
streamType: "jsmpeg",
bandwidth: Math.round(bitrate),
totalFrames: frameCount,
latency: undefined,
droppedFrames: undefined,
decodedFrames: undefined,
droppedFrameRate: undefined,
});
bytesReceivedRef.current = 0;
lastTimestampRef.current = currentTimestamp;
}, 1000);
return () => {
if (statsIntervalRef.current) {
clearInterval(statsIntervalRef.current);
frameCount = 0;
statsIntervalRef.current = null;
}
};
}, 0);
return () => {
clearTimeout(initPlayer);
if (statsIntervalRef.current) {
clearInterval(statsIntervalRef.current);
statsIntervalRef.current = null;
}
if (videoElement) {
try {
// this causes issues in react strict mode

View File

@@ -11,6 +11,7 @@ import { useCameraActivity } from "@/hooks/use-camera-activity";
import {
LivePlayerError,
LivePlayerMode,
PlayerStatsType,
VideoResolutionType,
} from "@/types/live";
import { getIconForLabel } from "@/utils/iconUtil";
@@ -20,20 +21,26 @@ import { cn } from "@/lib/utils";
import { TbExclamationCircle } from "react-icons/tb";
import { TooltipPortal } from "@radix-ui/react-tooltip";
import { baseUrl } from "@/api/baseUrl";
import { PlayerStats } from "./PlayerStats";
type LivePlayerProps = {
cameraRef?: (ref: HTMLDivElement | null) => void;
containerRef?: React.MutableRefObject<HTMLDivElement | null>;
className?: string;
cameraConfig: CameraConfig;
streamName: string;
preferredLiveMode: LivePlayerMode;
showStillWithoutActivity?: boolean;
useWebGL: boolean;
windowVisible?: boolean;
playAudio?: boolean;
volume?: number;
playInBackground: boolean;
micEnabled?: boolean; // only webrtc supports mic
iOSCompatFullScreen?: boolean;
pip?: boolean;
autoLive?: boolean;
showStats?: boolean;
onClick?: () => void;
setFullResolution?: React.Dispatch<React.SetStateAction<VideoResolutionType>>;
onError?: (error: LivePlayerError) => void;
@@ -45,14 +52,19 @@ export default function LivePlayer({
containerRef,
className,
cameraConfig,
streamName,
preferredLiveMode,
showStillWithoutActivity = true,
useWebGL = false,
windowVisible = true,
playAudio = false,
volume,
playInBackground = false,
micEnabled = false,
iOSCompatFullScreen = false,
pip,
autoLive = true,
showStats = false,
onClick,
setFullResolution,
onError,
@@ -60,6 +72,18 @@ export default function LivePlayer({
}: LivePlayerProps) {
const internalContainerRef = useRef<HTMLDivElement | null>(null);
// stats
const [stats, setStats] = useState<PlayerStatsType>({
streamType: "-",
bandwidth: 0, // in kbps
latency: undefined, // in seconds
totalFrames: 0,
droppedFrames: undefined,
decodedFrames: 0,
droppedFrameRate: 0, // percentage
});
// camera activity
const { activeMotion, activeTracking, objects, offline } =
@@ -144,6 +168,25 @@ export default function LivePlayer({
setLiveReady(false);
}, [preferredLiveMode]);
const [key, setKey] = useState(0);
const resetPlayer = () => {
setLiveReady(false);
setKey((prevKey) => prevKey + 1);
};
useEffect(() => {
if (streamName) {
resetPlayer();
}
}, [streamName]);
useEffect(() => {
if (showStillWithoutActivity && !autoLive) {
setLiveReady(false);
}
}, [showStillWithoutActivity, autoLive]);
const playerIsPlaying = useCallback(() => {
setLiveReady(true);
}, []);
@@ -153,15 +196,19 @@ export default function LivePlayer({
}
let player;
if (!autoLive) {
if (!autoLive || !streamName) {
player = null;
} else if (preferredLiveMode == "webrtc") {
player = (
<WebRtcPlayer
key={"webrtc_" + key}
className={`size-full rounded-lg md:rounded-2xl ${liveReady ? "" : "hidden"}`}
camera={cameraConfig.live.stream_name}
camera={streamName}
playbackEnabled={cameraActive || liveReady}
getStats={showStats}
setStats={setStats}
audioEnabled={playAudio}
volume={volume}
microphoneEnabled={micEnabled}
iOSCompatFullScreen={iOSCompatFullScreen}
onPlaying={playerIsPlaying}
@@ -173,10 +220,15 @@ export default function LivePlayer({
if ("MediaSource" in window || "ManagedMediaSource" in window) {
player = (
<MSEPlayer
key={"mse_" + key}
className={`size-full rounded-lg md:rounded-2xl ${liveReady ? "" : "hidden"}`}
camera={cameraConfig.live.stream_name}
camera={streamName}
playbackEnabled={cameraActive || liveReady}
audioEnabled={playAudio}
volume={volume}
playInBackground={playInBackground}
getStats={showStats}
setStats={setStats}
onPlaying={playerIsPlaying}
pip={pip}
setFullResolution={setFullResolution}
@@ -194,6 +246,7 @@ export default function LivePlayer({
if (cameraActive || !showStillWithoutActivity || liveReady) {
player = (
<JSMpegPlayer
key={"jsmpeg_" + key}
className="flex justify-center overflow-hidden rounded-lg md:rounded-2xl"
camera={cameraConfig.name}
width={cameraConfig.detect.width}
@@ -201,6 +254,8 @@ export default function LivePlayer({
playbackEnabled={
cameraActive || !showStillWithoutActivity || liveReady
}
useWebGL={useWebGL}
setStats={setStats}
containerRef={containerRef ?? internalContainerRef}
onPlaying={playerIsPlaying}
/>
@@ -293,7 +348,7 @@ export default function LivePlayer({
)}
>
<AutoUpdatingCameraImage
className="size-full"
className="pointer-events-none size-full"
cameraClasses="relative size-full flex justify-center"
camera={cameraConfig.name}
showFps={false}
@@ -331,6 +386,9 @@ export default function LivePlayer({
</Chip>
)}
</div>
{showStats && (
<PlayerStats stats={stats} minimal={cameraRef !== undefined} />
)}
</div>
);
}

View File

@@ -1,5 +1,9 @@
import { baseUrl } from "@/api/baseUrl";
import { LivePlayerError, VideoResolutionType } from "@/types/live";
import {
LivePlayerError,
PlayerStatsType,
VideoResolutionType,
} from "@/types/live";
import {
SetStateAction,
useCallback,
@@ -15,7 +19,11 @@ type MSEPlayerProps = {
className?: string;
playbackEnabled?: boolean;
audioEnabled?: boolean;
volume?: number;
playInBackground?: boolean;
pip?: boolean;
getStats?: boolean;
setStats?: (stats: PlayerStatsType) => void;
onPlaying?: () => void;
setFullResolution?: React.Dispatch<SetStateAction<VideoResolutionType>>;
onError?: (error: LivePlayerError) => void;
@@ -26,7 +34,11 @@ function MSEPlayer({
className,
playbackEnabled = true,
audioEnabled = false,
volume,
playInBackground = false,
pip = false,
getStats = false,
setStats,
onPlaying,
setFullResolution,
onError,
@@ -57,6 +69,7 @@ function MSEPlayer({
const [connectTS, setConnectTS] = useState<number>(0);
const [bufferTimeout, setBufferTimeout] = useState<NodeJS.Timeout>();
const [errorCount, setErrorCount] = useState<number>(0);
const totalBytesLoaded = useRef(0);
const videoRef = useRef<HTMLVideoElement>(null);
const wsRef = useRef<WebSocket | null>(null);
@@ -316,6 +329,8 @@ function MSEPlayer({
let bufLen = 0;
ondataRef.current = (data) => {
totalBytesLoaded.current += data.byteLength;
if (sb?.updating || bufLen > 0) {
const b = new Uint8Array(data);
buf.set(b, bufLen);
@@ -508,12 +523,22 @@ function MSEPlayer({
}
};
document.addEventListener("visibilitychange", listener);
if (!playInBackground) {
document.addEventListener("visibilitychange", listener);
}
return () => {
document.removeEventListener("visibilitychange", listener);
if (!playInBackground) {
document.removeEventListener("visibilitychange", listener);
}
};
}, [playbackEnabled, visibilityCheck, onConnect, onDisconnect]);
}, [
playbackEnabled,
visibilityCheck,
playInBackground,
onConnect,
onDisconnect,
]);
// control pip
@@ -525,6 +550,16 @@ function MSEPlayer({
videoRef.current.requestPictureInPicture();
}, [pip, videoRef]);
// control volume
useEffect(() => {
if (!videoRef.current || volume == undefined) {
return;
}
videoRef.current.volume = volume;
}, [volume, videoRef]);
// ensure we disconnect for slower connections
useEffect(() => {
@@ -542,6 +577,68 @@ function MSEPlayer({
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [playbackEnabled]);
// stats
useEffect(() => {
const video = videoRef.current;
let lastLoadedBytes = totalBytesLoaded.current;
let lastTimestamp = Date.now();
if (!getStats) return;
const updateStats = () => {
if (video) {
const now = Date.now();
const bytesLoaded = totalBytesLoaded.current;
const timeElapsed = (now - lastTimestamp) / 1000; // seconds
const bandwidth = (bytesLoaded - lastLoadedBytes) / timeElapsed / 1024; // kbps
lastLoadedBytes = bytesLoaded;
lastTimestamp = now;
const latency =
video.seekable.length > 0
? Math.max(
0,
video.seekable.end(video.seekable.length - 1) -
video.currentTime,
)
: 0;
const videoQuality = video.getVideoPlaybackQuality();
const { totalVideoFrames, droppedVideoFrames } = videoQuality;
const droppedFrameRate = totalVideoFrames
? (droppedVideoFrames / totalVideoFrames) * 100
: 0;
setStats?.({
streamType: "MSE",
bandwidth,
latency,
totalFrames: totalVideoFrames,
droppedFrames: droppedVideoFrames || undefined,
decodedFrames: totalVideoFrames - droppedVideoFrames,
droppedFrameRate,
});
}
};
const interval = setInterval(updateStats, 1000); // Update every second
return () => {
clearInterval(interval);
setStats?.({
streamType: "-",
bandwidth: 0,
latency: undefined,
totalFrames: 0,
droppedFrames: undefined,
decodedFrames: 0,
droppedFrameRate: 0,
});
};
}, [setStats, getStats]);
return (
<video
ref={videoRef}

View File

@@ -0,0 +1,100 @@
import { cn } from "@/lib/utils";
import { PlayerStatsType } from "@/types/live";
type PlayerStatsProps = {
stats: PlayerStatsType;
minimal: boolean;
};
export function PlayerStats({ stats, minimal }: PlayerStatsProps) {
const fullStatsContent = (
<>
<p>
<span className="text-white/70">Stream Type:</span>{" "}
<span className="text-white">{stats.streamType}</span>
</p>
<p>
<span className="text-white/70">Bandwidth:</span>{" "}
<span className="text-white">{stats.bandwidth.toFixed(2)} kbps</span>
</p>
{stats.latency != undefined && (
<p>
<span className="text-white/70">Latency:</span>{" "}
<span
className={`text-white ${stats.latency > 2 ? "text-danger" : ""}`}
>
{stats.latency.toFixed(2)} seconds
</span>
</p>
)}
<p>
<span className="text-white/70">Total Frames:</span>{" "}
<span className="text-white">{stats.totalFrames}</span>
</p>
{stats.droppedFrames != undefined && (
<p>
<span className="text-white/70">Dropped Frames:</span>{" "}
<span className="text-white">{stats.droppedFrames}</span>
</p>
)}
{stats.decodedFrames != undefined && (
<p>
<span className="text-white/70">Decoded Frames:</span>{" "}
<span className="text-white">{stats.decodedFrames}</span>
</p>
)}
{stats.droppedFrameRate != undefined && (
<p>
<span className="text-white/70">Dropped Frame Rate:</span>{" "}
<span className="text-white">
{stats.droppedFrameRate.toFixed(2)}%
</span>
</p>
)}
</>
);
const minimalStatsContent = (
<div className="flex flex-row items-center justify-center gap-4">
<div className="flex flex-col items-center justify-start gap-1">
<span className="text-white/70">Type</span>
<span className="text-white">{stats.streamType}</span>
</div>
<div className="flex flex-col items-center gap-1">
<span className="text-white/70">Bandwidth</span>{" "}
<span className="text-white">{stats.bandwidth.toFixed(2)} kbps</span>
</div>
{stats.latency != undefined && (
<div className="hidden flex-col items-center gap-1 md:flex">
<span className="text-white/70">Latency</span>
<span
className={`text-white ${stats.latency >= 2 ? "text-danger" : ""}`}
>
{stats.latency.toFixed(2)} sec
</span>
</div>
)}
{stats.droppedFrames != undefined && (
<div className="flex flex-col items-center justify-end gap-1">
<span className="text-white/70">Dropped</span>
<span className="text-white">{stats.droppedFrames} frames</span>
</div>
)}
</div>
);
return (
<>
<div
className={cn(
minimal
? "absolute bottom-0 left-0 max-h-[50%] w-full overflow-y-auto rounded-b-lg p-1 md:rounded-b-xl md:p-3"
: "absolute bottom-2 right-2 min-w-52 rounded-2xl p-4",
"z-50 flex flex-col gap-1 bg-black/70 text-[9px] duration-300 animate-in fade-in md:text-xs",
)}
>
{minimal ? minimalStatsContent : fullStatsContent}
</div>
</>
);
}

View File

@@ -1,5 +1,5 @@
import { baseUrl } from "@/api/baseUrl";
import { LivePlayerError } from "@/types/live";
import { LivePlayerError, PlayerStatsType } from "@/types/live";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
type WebRtcPlayerProps = {
@@ -7,9 +7,12 @@ type WebRtcPlayerProps = {
camera: string;
playbackEnabled?: boolean;
audioEnabled?: boolean;
volume?: number;
microphoneEnabled?: boolean;
iOSCompatFullScreen?: boolean; // ios doesn't support fullscreen divs so we must support the video element
pip?: boolean;
getStats?: boolean;
setStats?: (stats: PlayerStatsType) => void;
onPlaying?: () => void;
onError?: (error: LivePlayerError) => void;
};
@@ -19,9 +22,12 @@ export default function WebRtcPlayer({
camera,
playbackEnabled = true,
audioEnabled = false,
volume,
microphoneEnabled = false,
iOSCompatFullScreen = false,
pip = false,
getStats = false,
setStats,
onPlaying,
onError,
}: WebRtcPlayerProps) {
@@ -194,6 +200,16 @@ export default function WebRtcPlayer({
videoRef.current.requestPictureInPicture();
}, [pip, videoRef]);
// control volume
useEffect(() => {
if (!videoRef.current || volume == undefined) {
return;
}
videoRef.current.volume = volume;
}, [volume, videoRef]);
useEffect(() => {
videoLoadTimeoutRef.current = setTimeout(() => {
onError?.("stalled");
@@ -215,6 +231,75 @@ export default function WebRtcPlayer({
onPlaying?.();
};
// stats
useEffect(() => {
if (!pcRef.current || !getStats) return;
let lastBytesReceived = 0;
let lastTimestamp = 0;
const interval = setInterval(async () => {
if (pcRef.current && videoRef.current && !videoRef.current.paused) {
const report = await pcRef.current.getStats();
let bytesReceived = 0;
let timestamp = 0;
let roundTripTime = 0;
let framesReceived = 0;
let framesDropped = 0;
let framesDecoded = 0;
report.forEach((stat) => {
if (stat.type === "inbound-rtp" && stat.kind === "video") {
bytesReceived = stat.bytesReceived;
timestamp = stat.timestamp;
framesReceived = stat.framesReceived;
framesDropped = stat.framesDropped;
framesDecoded = stat.framesDecoded;
}
if (stat.type === "candidate-pair" && stat.state === "succeeded") {
roundTripTime = stat.currentRoundTripTime;
}
});
const timeDiff = (timestamp - lastTimestamp) / 1000; // in seconds
const bitrate =
timeDiff > 0
? (bytesReceived - lastBytesReceived) / timeDiff / 1000
: 0; // in kbps
setStats?.({
streamType: "WebRTC",
bandwidth: Math.round(bitrate),
latency: roundTripTime,
totalFrames: framesReceived,
droppedFrames: framesDropped,
decodedFrames: framesDecoded,
droppedFrameRate:
framesReceived > 0 ? (framesDropped / framesReceived) * 100 : 0,
});
lastBytesReceived = bytesReceived;
lastTimestamp = timestamp;
}
}, 1000);
return () => {
clearInterval(interval);
setStats?.({
streamType: "-",
bandwidth: 0,
latency: undefined,
totalFrames: 0,
droppedFrames: undefined,
decodedFrames: 0,
droppedFrameRate: 0,
});
};
// we need to listen on the value of the ref
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [pcRef, pcRef.current, getStats]);
return (
<video
ref={videoRef}