Improve handling of backchannel audio in camera wizard (#21250)

* Improve handling of backchannel audio in camera wizard

* Cleanup

* look for backchannel on all registered streams on save

avoids potential issues with a timeout in stream registration

---------

Co-authored-by: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>

web/src/components/settings/CameraStreamingDialog.tsx

@@ -34,6 +34,7 @@ import { LiveStreamMetadata } from "@/types/live";
 import { Trans, useTranslation } from "react-i18next";
 import { useDocDomain } from "@/hooks/use-doc-domain";
 import { useCameraFriendlyName } from "@/hooks/use-camera-friendly-name";
+import { detectCameraAudioFeatures } from "@/utils/cameraUtil";
 
 type CameraStreamingDialogProps = {
   camera: string;
@@ -80,20 +81,10 @@ export function CameraStreamingDialog({
 
   const cameraMetadata = streamName ? streamMetadata?.[streamName] : undefined;
 
-  const supportsAudioOutput = useMemo(() => {
-    if (!cameraMetadata) {
-      return false;
-    }
-
-    return (
-      cameraMetadata.producers.find(
-        (prod) =>
-          prod.medias &&
-          prod.medias.find((media) => media.includes("audio, recvonly")) !=
-            undefined,
-      ) != undefined
-    );
-  }, [cameraMetadata]);
+  const { audioOutput: supportsAudioOutput } = useMemo(
+    () => detectCameraAudioFeatures(cameraMetadata),
+    [cameraMetadata],
+  );
 
   // handlers
 

web/src/components/settings/CameraWizardDialog.tsx

@@ -237,7 +237,18 @@ export default function CameraWizardDialog({
                 const streamUrl = stream.useFfmpeg
                   ? `ffmpeg:${stream.url}`
                   : stream.url;
-                go2rtcStreams[streamName] = [streamUrl];
+
+                if (wizardData.hasBackchannel ?? false) {
+                  // Add two streams: one with #backchannel=0 and one without
+                  // in order to avoid taking control of the microphone during connections
+                  go2rtcStreams[streamName] = [
+                    `${streamUrl}#backchannel=0`,
+                    streamUrl,
+                  ];
+                } else {
+                  // Add single stream as normal
+                  go2rtcStreams[streamName] = [streamUrl];
+                }
               });
 
               if (Object.keys(go2rtcStreams).length > 0) {

web/src/components/settings/wizard/Step4Validation.tsx

@@ -14,7 +14,8 @@ import axios from "axios";
 import { toast } from "sonner";
 import MSEPlayer from "@/components/player/MsePlayer";
 import { WizardFormData, StreamConfig, TestResult } from "@/types/cameraWizard";
-import { PlayerStatsType } from "@/types/live";
+import { PlayerStatsType, LiveStreamMetadata } from "@/types/live";
+import { detectCameraAudioFeatures } from "@/utils/cameraUtil";
 import { FaCircleCheck, FaTriangleExclamation } from "react-icons/fa6";
 import { LuX } from "react-icons/lu";
 import { Card, CardContent } from "../../ui/card";
@@ -41,6 +42,9 @@ export default function Step4Validation({
   const [measuredBandwidth, setMeasuredBandwidth] = useState<
     Map<string, number>
   >(new Map());
+  const [registeredStreamIds, setRegisteredStreamIds] = useState<
+    Map<string, string>
+  >(new Map());
 
   const streams = useMemo(() => wizardData.streams || [], [wizardData.streams]);
 
@@ -125,6 +129,27 @@ export default function Step4Validation({
     [],
   );
 
+  const checkBackchannel = useCallback(
+    async (go2rtcStreamId: string, useFfmpeg: boolean): Promise<boolean> => {
+      // ffmpeg compatibility mode guarantees no backchannel connection
+      if (useFfmpeg) {
+        return false;
+      }
+
+      try {
+        const response = await axios.get<LiveStreamMetadata>(
+          `go2rtc/streams/${go2rtcStreamId}`,
+        );
+
+        const audioFeatures = detectCameraAudioFeatures(response.data, false);
+        return audioFeatures.twoWayAudio;
+      } catch {
+        return false;
+      }
+    },
+    [],
+  );
+
   const validateStream = useCallback(
     async (stream: StreamConfig) => {
       if (!stream.url.trim()) {
@@ -208,12 +233,31 @@ export default function Step4Validation({
     }
   }, [streams, onUpdate, t, performStreamValidation]);
 
-  const handleSave = useCallback(() => {
+  const handleSave = useCallback(async () => {
     if (!wizardData.cameraName || !wizardData.streams?.length) {
       toast.error(t("cameraWizard.step4.saveError"));
       return;
     }
 
+    const candidateStreams =
+      wizardData.streams?.filter(
+        (s) => s.testResult?.success && !(s.useFfmpeg ?? false),
+      ) || [];
+
+    let hasBackchannelResult = false;
+    if (candidateStreams.length > 0) {
+      // Check all candidate streams for backchannel support
+      const backchanelChecks = candidateStreams.map((stream) => {
+        const actualStreamId = registeredStreamIds.get(stream.id);
+        return actualStreamId
+          ? checkBackchannel(actualStreamId, stream.useFfmpeg ?? false)
+          : Promise.resolve(false);
+      });
+      const results = await Promise.all(backchanelChecks);
+      hasBackchannelResult = results.some((result) => result);
+    }
+    onUpdate({ hasBackchannel: hasBackchannelResult });
+
     // Convert wizard data to final config format
     const configData = {
       cameraName: wizardData.cameraName,
@@ -223,10 +267,11 @@ export default function Step4Validation({
       brandTemplate: wizardData.brandTemplate,
       customUrl: wizardData.customUrl,
       streams: wizardData.streams,
+      hasBackchannel: wizardData.hasBackchannel,
     };
 
     onSave(configData);
-  }, [wizardData, onSave, t]);
+  }, [wizardData, onSave, t, onUpdate, checkBackchannel, registeredStreamIds]);
 
   const canSave = useMemo(() => {
     return (
@@ -324,6 +369,11 @@ export default function Step4Validation({
                       <StreamPreview
                         stream={stream}
                         onBandwidthUpdate={handleBandwidthUpdate}
+                        onStreamRegistered={(go2rtcStreamId) => {
+                          setRegisteredStreamIds((prev) =>
+                            new Map(prev).set(stream.id, go2rtcStreamId),
+                          );
+                        }}
                       />
                     </div>
                   )}
@@ -683,10 +733,15 @@ function BandwidthDisplay({
 type StreamPreviewProps = {
   stream: StreamConfig;
   onBandwidthUpdate?: (streamId: string, bandwidth: number) => void;
+  onStreamRegistered?: (go2rtcStreamId: string) => void;
 };
 
 // live stream preview using MSEPlayer with temp go2rtc streams
-function StreamPreview({ stream, onBandwidthUpdate }: StreamPreviewProps) {
+function StreamPreview({
+  stream,
+  onBandwidthUpdate,
+  onStreamRegistered,
+}: StreamPreviewProps) {
   const { t } = useTranslation(["views/settings"]);
   const [streamId, setStreamId] = useState(`wizard_${stream.id}_${Date.now()}`);
   const [registered, setRegistered] = useState(false);
@@ -722,10 +777,11 @@ function StreamPreview({ stream, onBandwidthUpdate }: StreamPreviewProps) {
       .put(`go2rtc/streams/${streamId}`, null, {
         params: { src: streamUrl },
       })
-      .then(() => {
+      .then(async () => {
         // Add small delay to allow go2rtc api to run and initialize the stream
         setTimeout(() => {
           setRegistered(true);
+          onStreamRegistered?.(streamId);
         }, 500);
       })
       .catch(() => {
@@ -738,6 +794,7 @@ function StreamPreview({ stream, onBandwidthUpdate }: StreamPreviewProps) {
         // do nothing on cleanup errors - go2rtc won't consume the streams
       });
     };
+    // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [stream.url, stream.useFfmpeg, streamId]);
 
   const resolution = stream.testResult?.resolution;