Fixes (#19708)

* use custom swr fetcher to check for audio support

The go2rtc API doesn't always return stream data for anything not being actively consumed, so audio support was not always being correctly deduced. So we can use a custom swr fetcher to call the endpoint that probes the streams, which returns the correct producers data.

* return correct mime type for thumbnail and latest frame endpoints

follow up to https://github.com/blakeblackshear/frigate/pull/19555

frigate/api/defs/query/media_query_parameters.py

@@ -10,6 +10,11 @@ class Extension(str, Enum):
     jpg = "jpg"
     jpeg = "jpeg"
 
+    def get_mime_type(self) -> str:
+        if self in (Extension.jpg, Extension.jpeg):
+            return "image/jpeg"
+        return f"image/{self.value}"
+
 
 class MediaLatestFrameQueryParams(BaseModel):
     bbox: Optional[int] = None

frigate/api/media.py

@@ -194,7 +194,7 @@ def latest_frame(
         _, img = cv2.imencode(f".{extension.value}", frame, quality_params)
         return Response(
             content=img.tobytes(),
-            media_type=f"image/{extension.value}",
+            media_type=extension.get_mime_type(),
             headers={
                 "Cache-Control": "no-store"
                 if not params.store
@@ -219,7 +219,7 @@ def latest_frame(
         _, img = cv2.imencode(f".{extension.value}", frame, quality_params)
         return Response(
             content=img.tobytes(),
-            media_type=f"image/{extension.value}",
+            media_type=extension.get_mime_type(),
             headers={
                 "Cache-Control": "no-store"
                 if not params.store
@@ -878,7 +878,7 @@ def event_snapshot(
 def event_thumbnail(
     request: Request,
     event_id: str,
-    extension: str,
+    extension: Extension,
     max_cache_age: int = Query(
         2592000, description="Max cache age in seconds. Default 30 days in seconds."
     ),
@@ -903,7 +903,7 @@ def event_thumbnail(
                 if event_id in camera_state.tracked_objects:
                     tracked_obj = camera_state.tracked_objects.get(event_id)
                     if tracked_obj is not None:
-                        thumbnail_bytes = tracked_obj.get_thumbnail(extension)
+                        thumbnail_bytes = tracked_obj.get_thumbnail(extension.value)
         except Exception:
             return JSONResponse(
                 content={"success": False, "message": "Event not found"},
@@ -931,23 +931,21 @@ def event_thumbnail(
         )
 
         quality_params = None
-
-        if extension == "jpg" or extension == "jpeg":
+        if extension in (Extension.jpg, Extension.jpeg):
             quality_params = [int(cv2.IMWRITE_JPEG_QUALITY), 70]
-        elif extension == "webp":
+        elif extension == Extension.webp:
             quality_params = [int(cv2.IMWRITE_WEBP_QUALITY), 60]
 
-        _, img = cv2.imencode(f".{extension}", thumbnail, quality_params)
+        _, img = cv2.imencode(f".{extension.value}", thumbnail, quality_params)
         thumbnail_bytes = img.tobytes()
 
     return Response(
         thumbnail_bytes,
-        media_type=f"image/{extension}",
+        media_type=extension.get_mime_type(),
         headers={
             "Cache-Control": f"private, max-age={max_cache_age}"
             if event_complete
             else "no-store",
-            "Content-Type": f"image/{extension}",
         },
     )
 

web/src/hooks/use-camera-live-mode.ts

@@ -1,5 +1,5 @@
 import { CameraConfig, FrigateConfig } from "@/types/frigateConfig";
-import { useCallback, useEffect, useState } from "react";
+import { useCallback, useEffect, useState, useMemo } from "react";
 import useSWR from "swr";
 import { LivePlayerMode, LiveStreamMetadata } from "@/types/live";
 
@@ -8,9 +8,54 @@ export default function useCameraLiveMode(
   windowVisible: boolean,
 ) {
   const { data: config } = useSWR<FrigateConfig>("config");
-  const { data: allStreamMetadata } = useSWR<{
+
+  // Get comma-separated list of restreamed stream names for SWR key
+  const restreamedStreamsKey = useMemo(() => {
+    if (!cameras || !config) return null;
+
+    const streamNames = new Set<string>();
+    cameras.forEach((camera) => {
+      const isRestreamed = Object.keys(config.go2rtc.streams || {}).includes(
+        Object.values(camera.live.streams)[0],
+      );
+
+      if (isRestreamed) {
+        Object.values(camera.live.streams).forEach((streamName) => {
+          streamNames.add(streamName);
+        });
+      }
+    });
+
+    return streamNames.size > 0
+      ? Array.from(streamNames).sort().join(",")
+      : null;
+  }, [cameras, config]);
+
+  const streamsFetcher = useCallback(async (key: string) => {
+    const streamNames = key.split(",");
+    const metadata: { [key: string]: LiveStreamMetadata } = {};
+
+    await Promise.all(
+      streamNames.map(async (streamName) => {
+        try {
+          const response = await fetch(`/api/go2rtc/streams/${streamName}`);
+          if (response.ok) {
+            const data = await response.json();
+            metadata[streamName] = data;
+          }
+        } catch (error) {
+          // eslint-disable-next-line no-console
+          console.error(`Failed to fetch metadata for ${streamName}:`, error);
+        }
+      }),
+    );
+
+    return metadata;
+  }, []);
+
+  const { data: allStreamMetadata = {} } = useSWR<{
     [key: string]: LiveStreamMetadata;
-  }>(config ? "go2rtc/streams" : null, { revalidateOnFocus: false });
+  }>(restreamedStreamsKey, streamsFetcher, { revalidateOnFocus: false });
 
   const [preferredLiveModes, setPreferredLiveModes] = useState<{
     [key: string]: LivePlayerMode;