From 2a28964e63db0556597710873a0617f09075dc13 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Mon, 10 Feb 2025 09:38:56 -0600 Subject: [PATCH] Improve UI logs (#16434) * use react-logviewer and backend streaming * layout adjustments * readd copy handler * reorder and fix key * add loading state * handle frigate log consolidation * handle newlines in sheet * update react-logviewer * fix scrolling and use chunked log download * don't combine frigate log lines with timestamp * basic deduplication * use react-logviewer and backend streaming * layout adjustments * readd copy handler * reorder and fix key * add loading state * handle frigate log consolidation * handle newlines in sheet * update react-logviewer * fix scrolling and use chunked log download * don't combine frigate log lines with timestamp * basic deduplication * move process logs function to services util * improve layout and scrolling behavior * clean up --- docker/main/requirements-wheels.txt | 1 + frigate/api/app.py | 76 +- frigate/util/services.py | 54 +- web/package-lock.json | 71 ++ web/package.json | 1 + .../dynamic/EnhancedScrollFollow.tsx | 91 +++ ...gLevelFilter.tsx => LogSettingsButton.tsx} | 64 +- web/src/components/indicators/Chip.tsx | 22 +- web/src/components/overlay/LogInfoDialog.tsx | 9 +- web/src/index.css | 5 + web/src/pages/Logs.tsx | 687 ++++++++++-------- web/src/types/log.ts | 4 + web/src/utils/logUtil.ts | 105 ++- 13 files changed, 813 insertions(+), 377 deletions(-) create mode 100644 web/src/components/dynamic/EnhancedScrollFollow.tsx rename web/src/components/filter/{LogLevelFilter.tsx => LogSettingsButton.tsx} (64%) diff --git a/docker/main/requirements-wheels.txt b/docker/main/requirements-wheels.txt index 905769b49..40a2f1d8b 100644 --- a/docker/main/requirements-wheels.txt +++ b/docker/main/requirements-wheels.txt @@ -1,3 +1,4 @@ +aiofiles == 24.1.* click == 8.1.* # FastAPI aiohttp == 3.11.2 diff --git a/frigate/api/app.py b/frigate/api/app.py index c7c76c632..52e686af1 100644 --- a/frigate/api/app.py +++ b/frigate/api/app.py @@ -1,5 +1,6 @@ """Main api runner.""" +import asyncio import copy import json import logging @@ -10,12 +11,13 @@ from functools import reduce from io import StringIO from typing import Any, Optional +import aiofiles import requests import ruamel.yaml from fastapi import APIRouter, Body, Path, Request, Response from fastapi.encoders import jsonable_encoder from fastapi.params import Depends -from fastapi.responses import JSONResponse, PlainTextResponse +from fastapi.responses import JSONResponse, PlainTextResponse, StreamingResponse from markupsafe import escape from peewee import operator from prometheus_client import CONTENT_TYPE_LATEST, generate_latest @@ -35,6 +37,7 @@ from frigate.util.config import find_config_file from frigate.util.services import ( ffprobe_stream, get_nvidia_driver_info, + process_logs, restart_frigate, vainfo_hwaccel, ) @@ -455,9 +458,10 @@ def nvinfo(): @router.get("/logs/{service}", tags=[Tags.logs]) -def logs( +async def logs( service: str = Path(enum=["frigate", "nginx", "go2rtc"]), download: Optional[str] = None, + stream: Optional[bool] = False, start: Optional[int] = 0, end: Optional[int] = None, ): @@ -476,6 +480,27 @@ def logs( status_code=500, ) + async def stream_logs(file_path: str): + """Asynchronously stream log lines.""" + buffer = "" + try: + async with aiofiles.open(file_path, "r") as file: + await file.seek(0, 2) + while True: + line = await file.readline() + if line: + buffer += line + # Process logs only when there are enough lines in the buffer + if "\n" in buffer: + _, processed_lines = process_logs(buffer, service) + buffer = "" + for processed_line in processed_lines: + yield f"{processed_line}\n" + else: + await asyncio.sleep(0.1) + except FileNotFoundError: + yield "Log file not found.\n" + log_locations = { "frigate": "/dev/shm/logs/frigate/current", "go2rtc": "/dev/shm/logs/go2rtc/current", @@ -492,48 +517,17 @@ def logs( if download: return download_logs(service_location) + if stream: + return StreamingResponse(stream_logs(service_location), media_type="text/plain") + + # For full logs initially try: - file = open(service_location, "r") - contents = file.read() - file.close() - - # use the start timestamp to group logs together`` - logLines = [] - keyLength = 0 - dateEnd = 0 - currentKey = "" - currentLine = "" - - for rawLine in contents.splitlines(): - cleanLine = rawLine.strip() - - if len(cleanLine) < 10: - continue - - # handle cases where S6 does not include date in log line - if " " not in cleanLine: - cleanLine = f"{datetime.now()} {cleanLine}" - - if dateEnd == 0: - dateEnd = cleanLine.index(" ") - keyLength = dateEnd - (6 if service_location == "frigate" else 0) - - newKey = cleanLine[0:keyLength] - - if newKey == currentKey: - currentLine += f"\n{cleanLine[dateEnd:].strip()}" - continue - else: - if len(currentLine) > 0: - logLines.append(currentLine) - - currentKey = newKey - currentLine = cleanLine - - logLines.append(currentLine) + async with aiofiles.open(service_location, "r") as file: + contents = await file.read() + total_lines, log_lines = process_logs(contents, service, start, end) return JSONResponse( - content={"totalLines": len(logLines), "lines": logLines[start:end]}, + content={"totalLines": total_lines, "lines": log_lines}, status_code=200, ) except FileNotFoundError as e: diff --git a/frigate/util/services.py b/frigate/util/services.py index d54d1beb0..d7966bd00 100644 --- a/frigate/util/services.py +++ b/frigate/util/services.py @@ -8,7 +8,8 @@ import re import signal import subprocess as sp import traceback -from typing import Optional +from datetime import datetime +from typing import List, Optional, Tuple import cv2 import psutil @@ -635,3 +636,54 @@ async def get_video_properties( result["fourcc"] = fourcc return result + + +def process_logs( + contents: str, + service: Optional[str] = None, + start: Optional[int] = None, + end: Optional[int] = None, +) -> Tuple[int, List[str]]: + log_lines = [] + last_message = None + last_timestamp = None + repeat_count = 0 + + for raw_line in contents.splitlines(): + clean_line = raw_line.strip() + + if len(clean_line) < 10: + continue + + # Handle cases where S6 does not include date in log line + if " " not in clean_line: + clean_line = f"{datetime.now()} {clean_line}" + + # Find the position of the first double space to extract timestamp and message + date_end = clean_line.index(" ") + timestamp = clean_line[:date_end] + message_part = clean_line[date_end:].strip() + + if message_part == last_message: + repeat_count += 1 + continue + else: + if repeat_count > 0: + # Insert a deduplication message formatted the same way as logs + dedup_message = f"{last_timestamp} [LOGGING] Last message repeated {repeat_count} times" + log_lines.append(dedup_message) + repeat_count = 0 + + log_lines.append(clean_line) + last_timestamp = timestamp + + last_message = message_part + + # If there were repeated messages at the end, log the count + if repeat_count > 0: + dedup_message = ( + f"{last_timestamp} [LOGGING] Last message repeated {repeat_count} times" + ) + log_lines.append(dedup_message) + + return len(log_lines), log_lines[start:end] diff --git a/web/package-lock.json b/web/package-lock.json index 7ce6345af..3ced33ffe 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -10,6 +10,7 @@ "dependencies": { "@cycjimmy/jsmpeg-player": "^6.1.1", "@hookform/resolvers": "^3.9.0", + "@melloware/react-logviewer": "^6.1.2", "@radix-ui/react-alert-dialog": "^1.1.2", "@radix-ui/react-aspect-ratio": "^1.1.0", "@radix-ui/react-checkbox": "^1.1.2", @@ -1002,6 +1003,22 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@melloware/react-logviewer": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/@melloware/react-logviewer/-/react-logviewer-6.1.2.tgz", + "integrity": "sha512-WDw3VIGqhoXxDn93HFDicwRhi4+FQyaKiVTB07bWerT82gTgyWV7bOciVV33z25N3WJrz62j5FKVzvFZCu17/A==", + "license": "MPL-2.0", + "dependencies": { + "hotkeys-js": "3.13.9", + "mitt": "3.0.1", + "react-string-replace": "1.1.1", + "virtua": "0.39.3" + }, + "peerDependencies": { + "react": ">=17.0.0", + "react-dom": ">=17.0.0" + } + }, "node_modules/@mswjs/interceptors": { "version": "0.29.1", "resolved": "https://registry.npmjs.org/@mswjs/interceptors/-/interceptors-0.29.1.tgz", @@ -5511,6 +5528,15 @@ "integrity": "sha512-wA66nnYFvQa1o4DO/BFgLNRKnBTVXpNeldGRBJ2Y0SvFtdwvFKCbqa9zhHoZLoxHhZ+jYsj3aIBkWQQCPNOhMw==", "license": "Apache-2.0" }, + "node_modules/hotkeys-js": { + "version": "3.13.9", + "resolved": "https://registry.npmjs.org/hotkeys-js/-/hotkeys-js-3.13.9.tgz", + "integrity": "sha512-3TRCj9u9KUH6cKo25w4KIdBfdBfNRjfUwrljCLDC2XhmPDG0SjAZFcFZekpUZFmXzfYoGhFDcdx2gX/vUVtztQ==", + "license": "MIT", + "funding": { + "url": "https://jaywcjlove.github.io/#/sponsor" + } + }, "node_modules/html-encoding-sniffer": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-4.0.0.tgz", @@ -6273,6 +6299,12 @@ "node": ">=16 || 14 >=14.17" } }, + "node_modules/mitt": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", + "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==", + "license": "MIT" + }, "node_modules/mock-socket": { "version": "9.3.1", "resolved": "https://registry.npmjs.org/mock-socket/-/mock-socket-9.3.1.tgz", @@ -7425,6 +7457,15 @@ "react-dom": ">=16.8" } }, + "node_modules/react-string-replace": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/react-string-replace/-/react-string-replace-1.1.1.tgz", + "integrity": "sha512-26TUbLzLfHQ5jO5N7y3Mx88eeKo0Ml0UjCQuX4BMfOd/JX+enQqlKpL1CZnmjeBRvQE8TR+ds9j1rqx9CxhKHQ==", + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, "node_modules/react-style-singleton": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/react-style-singleton/-/react-style-singleton-2.2.1.tgz", @@ -8766,6 +8807,36 @@ "react-dom": "^16.8 || ^17.0 || ^18.0" } }, + "node_modules/virtua": { + "version": "0.39.3", + "resolved": "https://registry.npmjs.org/virtua/-/virtua-0.39.3.tgz", + "integrity": "sha512-Ep3aiJXSGPm1UUniThr5mGDfG0upAleP7pqQs5mvvCgM1wPhII1ZKa7eNCWAJRLkC+InpXKokKozyaaj/aMYOQ==", + "license": "MIT", + "peerDependencies": { + "react": ">=16.14.0", + "react-dom": ">=16.14.0", + "solid-js": ">=1.0", + "svelte": ">=5.0", + "vue": ">=3.2" + }, + "peerDependenciesMeta": { + "react": { + "optional": true + }, + "react-dom": { + "optional": true + }, + "solid-js": { + "optional": true + }, + "svelte": { + "optional": true + }, + "vue": { + "optional": true + } + } + }, "node_modules/vite": { "version": "5.4.0", "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.0.tgz", diff --git a/web/package.json b/web/package.json index d76e6ad10..6b3ec1e44 100644 --- a/web/package.json +++ b/web/package.json @@ -16,6 +16,7 @@ "dependencies": { "@cycjimmy/jsmpeg-player": "^6.1.1", "@hookform/resolvers": "^3.9.0", + "@melloware/react-logviewer": "^6.1.2", "@radix-ui/react-alert-dialog": "^1.1.2", "@radix-ui/react-aspect-ratio": "^1.1.0", "@radix-ui/react-checkbox": "^1.1.2", diff --git a/web/src/components/dynamic/EnhancedScrollFollow.tsx b/web/src/components/dynamic/EnhancedScrollFollow.tsx new file mode 100644 index 000000000..35673c80e --- /dev/null +++ b/web/src/components/dynamic/EnhancedScrollFollow.tsx @@ -0,0 +1,91 @@ +import { useRef, useCallback, useEffect, type ReactNode } from "react"; +import { ScrollFollow } from "@melloware/react-logviewer"; + +export type ScrollFollowProps = { + startFollowing?: boolean; + render: (renderProps: ScrollFollowRenderProps) => ReactNode; + onCustomScroll?: ( + scrollTop: number, + scrollHeight: number, + clientHeight: number, + ) => void; +}; + +export type ScrollFollowRenderProps = { + follow: boolean; + onScroll: (args: { + scrollTop: number; + scrollHeight: number; + clientHeight: number; + }) => void; + startFollowing: () => void; + stopFollowing: () => void; + onCustomScroll?: ( + scrollTop: number, + scrollHeight: number, + clientHeight: number, + ) => void; +}; + +const SCROLL_BUFFER = 5; + +export default function EnhancedScrollFollow(props: ScrollFollowProps) { + const followRef = useRef(props.startFollowing || false); + const prevScrollTopRef = useRef(undefined); + + useEffect(() => { + prevScrollTopRef.current = undefined; + }, []); + + const wrappedRender = useCallback( + (renderProps: ScrollFollowRenderProps) => { + const wrappedOnScroll = (args: { + scrollTop: number; + scrollHeight: number; + clientHeight: number; + }) => { + // Check if scrolling up and immediately stop following + if ( + prevScrollTopRef.current !== undefined && + args.scrollTop < prevScrollTopRef.current + ) { + if (followRef.current) { + renderProps.stopFollowing(); + followRef.current = false; + } + } + + const bottomThreshold = + args.scrollHeight - args.clientHeight - SCROLL_BUFFER; + const isNearBottom = args.scrollTop >= bottomThreshold; + + if (isNearBottom && !followRef.current) { + renderProps.startFollowing(); + followRef.current = true; + } else if (!isNearBottom && followRef.current) { + renderProps.stopFollowing(); + followRef.current = false; + } + + prevScrollTopRef.current = args.scrollTop; + renderProps.onScroll(args); + if (props.onCustomScroll) { + props.onCustomScroll( + args.scrollTop, + args.scrollHeight, + args.clientHeight, + ); + } + }; + + return props.render({ + ...renderProps, + onScroll: wrappedOnScroll, + follow: followRef.current, + }); + }, + [props], + ); + + return ; +} diff --git a/web/src/components/filter/LogLevelFilter.tsx b/web/src/components/filter/LogSettingsButton.tsx similarity index 64% rename from web/src/components/filter/LogLevelFilter.tsx rename to web/src/components/filter/LogSettingsButton.tsx index 9f08c51b5..e9465bf1d 100644 --- a/web/src/components/filter/LogLevelFilter.tsx +++ b/web/src/components/filter/LogSettingsButton.tsx @@ -1,36 +1,73 @@ import { Button } from "../ui/button"; -import { FaFilter } from "react-icons/fa"; +import { FaCog } from "react-icons/fa"; import { isMobile } from "react-device-detect"; import { Drawer, DrawerContent, DrawerTrigger } from "../ui/drawer"; import { Popover, PopoverContent, PopoverTrigger } from "../ui/popover"; -import { LogSeverity } from "@/types/log"; +import { LogSettingsType, LogSeverity } from "@/types/log"; import { Label } from "../ui/label"; import { Switch } from "../ui/switch"; import { DropdownMenuSeparator } from "../ui/dropdown-menu"; +import { cn } from "@/lib/utils"; +import FilterSwitch from "./FilterSwitch"; -type LogLevelFilterButtonProps = { +type LogSettingsButtonProps = { selectedLabels?: LogSeverity[]; updateLabelFilter: (labels: LogSeverity[] | undefined) => void; + logSettings?: LogSettingsType; + setLogSettings: (logSettings: LogSettingsType) => void; }; -export function LogLevelFilterButton({ +export function LogSettingsButton({ selectedLabels, updateLabelFilter, -}: LogLevelFilterButtonProps) { + logSettings, + setLogSettings, +}: LogSettingsButtonProps) { const trigger = ( ); const content = ( - +
+
+
+
Filter
+
+ Filter logs by severity. +
+
+ +
+ +
+
+
Loading
+
+
+ When the log pane is scrolled to the bottom, new logs + automatically stream as they are added. +
+ { + setLogSettings({ + disableStreaming: isChecked, + }); + }} + /> +
+
+
+
); if (isMobile) { @@ -63,7 +100,7 @@ export function GeneralFilterContent({ return ( <>
-
+
-
{["debug", "info", "warning", "error"].map((item) => ( -
+
Message
-
{logLine.content}
+
+ {logLine.content.split("\n").map((line) => ( + <> + {line} +
+ + ))} +
{helpfulLinks.length > 0 && (
diff --git a/web/src/index.css b/web/src/index.css index 5c78fe925..c657f22eb 100644 --- a/web/src/index.css +++ b/web/src/index.css @@ -179,3 +179,8 @@ html { border: 3px solid #a00000 !important; opacity: 0.5 !important; } + +.react-lazylog, +.react-lazylog-searchbar { + background-color: transparent !important; +} diff --git a/web/src/pages/Logs.tsx b/web/src/pages/Logs.tsx index 949fffb8a..a4b67f441 100644 --- a/web/src/pages/Logs.tsx +++ b/web/src/pages/Logs.tsx @@ -1,35 +1,47 @@ import { Button } from "@/components/ui/button"; import { ToggleGroup, ToggleGroupItem } from "@/components/ui/toggle-group"; -import { LogData, LogLine, LogSeverity, LogType, logTypes } from "@/types/log"; +import { + LogLine, + LogSettingsType, + LogSeverity, + LogType, + logTypes, +} from "@/types/log"; import copy from "copy-to-clipboard"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import axios from "axios"; import LogInfoDialog from "@/components/overlay/LogInfoDialog"; import { LogChip } from "@/components/indicators/Chip"; -import { LogLevelFilterButton } from "@/components/filter/LogLevelFilter"; -import { FaCopy } from "react-icons/fa6"; +import { LogSettingsButton } from "@/components/filter/LogSettingsButton"; +import { FaCopy, FaDownload } from "react-icons/fa"; import { Toaster } from "@/components/ui/sonner"; import { toast } from "sonner"; -import { - isDesktop, - isMobile, - isMobileOnly, - isTablet, -} from "react-device-detect"; import ActivityIndicator from "@/components/indicators/activity-indicator"; import { cn } from "@/lib/utils"; -import { MdVerticalAlignBottom } from "react-icons/md"; import { parseLogLines } from "@/utils/logUtil"; -import useKeyboardListener from "@/hooks/use-keyboard-listener"; import { ScrollArea, ScrollBar } from "@/components/ui/scroll-area"; import scrollIntoView from "scroll-into-view-if-needed"; -import { FaDownload } from "react-icons/fa"; - -type LogRange = { start: number; end: number }; +import { LazyLog } from "@melloware/react-logviewer"; +import useKeyboardListener from "@/hooks/use-keyboard-listener"; +import EnhancedScrollFollow from "@/components/dynamic/EnhancedScrollFollow"; +import { MdCircle } from "react-icons/md"; +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from "@/components/ui/tooltip"; +import { debounce } from "lodash"; function Logs() { const [logService, setLogService] = useState("frigate"); const tabsRef = useRef(null); + const lazyLogWrapperRef = useRef(null); + const [logs, setLogs] = useState([]); + const [filterSeverity, setFilterSeverity] = useState(); + const [selectedLog, setSelectedLog] = useState(); + const lazyLogRef = useRef(null); + const [isLoading, setIsLoading] = useState(true); + const lastFetchedIndexRef = useRef(-1); useEffect(() => { document.title = `${logService[0].toUpperCase()}${logService.substring(1)} Logs - Frigate`; @@ -49,92 +61,233 @@ function Logs() { } }, [tabsRef, logService]); - // log data handling + // log settings - const logPageSize = useMemo(() => { - if (isMobileOnly) { - return 15; - } + const [logSettings, setLogSettings] = useState({ + disableStreaming: false, + }); - if (isTablet) { - return 25; - } + // filter - return 40; - }, []); + const filterLines = useCallback( + (lines: string[]) => { + if (!filterSeverity?.length) return lines; - const [logRange, setLogRange] = useState({ start: 0, end: 0 }); - const [logs, setLogs] = useState([]); - const [logLines, setLogLines] = useState([]); + return lines.filter((line) => { + const parsedLine = parseLogLines(logService, [line])[0]; + return filterSeverity.includes(parsedLine.severity); + }); + }, + [filterSeverity, logService], + ); - useEffect(() => { - axios - .get(`logs/${logService}?start=-${logPageSize}`) - .then((resp) => { - if (resp.status == 200) { - const data = resp.data as LogData; - setLogRange({ - start: Math.max(0, data.totalLines - logPageSize), - end: data.totalLines, - }); - setLogs(data.lines); - setLogLines(parseLogLines(logService, data.lines)); + // fetchers + + const fetchLogRange = useCallback( + async (start: number, end: number) => { + try { + const response = await axios.get(`logs/${logService}`, { + params: { start, end }, + }); + if ( + response.status === 200 && + response.data && + Array.isArray(response.data.lines) + ) { + const filteredLines = filterLines(response.data.lines); + return filteredLines; } + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : "An unknown error occurred"; + toast.error(`Error fetching logs: ${errorMessage}`, { + position: "top-center", + }); + } + return []; + }, + [logService, filterLines], + ); + + const fetchInitialLogs = useCallback(async () => { + setIsLoading(true); + try { + const response = await axios.get(`logs/${logService}`, { + params: { start: filterSeverity ? 0 : -100 }, + }); + if ( + response.status === 200 && + response.data && + Array.isArray(response.data.lines) + ) { + const filteredLines = filterLines(response.data.lines); + setLogs(filteredLines); + lastFetchedIndexRef.current = + response.data.totalLines - filteredLines.length; + } + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : "An unknown error occurred"; + toast.error(`Error fetching logs: ${errorMessage}`, { + position: "top-center", + }); + } finally { + setIsLoading(false); + } + }, [logService, filterLines, filterSeverity]); + + const abortControllerRef = useRef(null); + + const fetchLogsStream = useCallback(() => { + // Cancel any existing stream + abortControllerRef.current?.abort(); + const abortController = new AbortController(); + abortControllerRef.current = abortController; + let buffer = ""; + const decoder = new TextDecoder(); + + const processStreamChunk = ( + reader: ReadableStreamDefaultReader, + ): Promise => { + return reader.read().then(({ done, value }) => { + if (done) return; + + // Decode the chunk and add it to our buffer + buffer += decoder.decode(value, { stream: true }); + + // Split on newlines, keeping any partial line in the buffer + const lines = buffer.split("\n"); + + // Keep the last partial line + buffer = lines.pop() || ""; + + // Filter and append complete lines + if (lines.length > 0) { + const filteredLines = filterSeverity?.length + ? lines.filter((line) => { + const parsedLine = parseLogLines(logService, [line])[0]; + return filterSeverity.includes(parsedLine.severity); + }) + : lines; + if (filteredLines.length > 0) { + lazyLogRef.current?.appendLines(filteredLines); + } + } + // Process next chunk + return processStreamChunk(reader); + }); + }; + + fetch(`api/logs/${logService}?stream=true`, { + signal: abortController.signal, + }) + .then((response): Promise => { + if (!response.ok) { + throw new Error( + `Error while fetching log stream, status: ${response.status}`, + ); + } + const reader = response.body?.getReader(); + if (!reader) { + throw new Error("No reader available"); + } + return processStreamChunk(reader); }) - .catch(() => {}); - }, [logPageSize, logService]); + .catch((error) => { + if (error.name !== "AbortError") { + const errorMessage = + error instanceof Error + ? error.message + : "An unknown error occurred"; + toast.error(`Error while streaming logs: ${errorMessage}`); + } + }); + }, [logService, filterSeverity]); useEffect(() => { - if (!logs || logs.length == 0) { - return; - } - - const id = setTimeout(() => { - axios - .get(`logs/${logService}?start=${logRange.end}`) - .then((resp) => { - if (resp.status == 200) { - const data = resp.data as LogData; - - if (data.lines.length > 0) { - setLogRange({ - start: logRange.start, - end: data.totalLines, - }); - setLogs([...logs, ...data.lines]); - setLogLines([ - ...logLines, - ...parseLogLines(logService, data.lines), - ]); - } - } - }) - .catch(() => {}); - }, 5000); + setIsLoading(true); + setLogs([]); + lastFetchedIndexRef.current = -1; + fetchInitialLogs().then(() => { + // Start streaming after initial load + if (!logSettings.disableStreaming) { + fetchLogsStream(); + } + }); return () => { - if (id) { - clearTimeout(id); - } + abortControllerRef.current?.abort(); }; - // we need to listen on the current range of visible items + // we know that these deps are correct // eslint-disable-next-line react-hooks/exhaustive-deps - }, [logLines, logService, logRange]); + }, [logService, filterSeverity]); - // convert to log data + // handlers + + const prependLines = useCallback((newLines: string[]) => { + if (!lazyLogRef.current) return; + + const newLinesArray = newLines.map( + (line) => new Uint8Array(new TextEncoder().encode(line + "\n")), + ); + + lazyLogRef.current.setState((prevState) => ({ + ...prevState, + lines: prevState.lines.unshift(...newLinesArray), + count: prevState.count + newLines.length, + })); + }, []); + + // debounced + const handleScroll = useMemo( + () => + debounce(() => { + const scrollThreshold = + lazyLogRef.current?.listRef.current?.findEndIndex() ?? 10; + const startIndex = + lazyLogRef.current?.listRef.current?.findStartIndex() ?? 0; + const endIndex = + lazyLogRef.current?.listRef.current?.findEndIndex() ?? 0; + const pageSize = endIndex - startIndex; + if ( + scrollThreshold < pageSize + pageSize / 2 && + lastFetchedIndexRef.current > 0 && + !isLoading + ) { + const nextEnd = lastFetchedIndexRef.current; + const nextStart = Math.max(0, nextEnd - (pageSize || 100)); + setIsLoading(true); + + fetchLogRange(nextStart, nextEnd).then((newLines) => { + if (newLines.length > 0) { + prependLines(newLines); + lastFetchedIndexRef.current = nextStart; + + lazyLogRef.current?.listRef.current?.scrollTo( + newLines.length * + lazyLogRef.current?.listRef.current?.getItemSize(1), + ); + } + }); + + setIsLoading(false); + } + }, 50), + [fetchLogRange, isLoading, prependLines], + ); const handleCopyLogs = useCallback(() => { - if (logs) { - copy(logs.join("\n")); - toast.success( - logRange.start == 0 - ? "Copied logs to clipboard" - : "Copied visible logs to clipboard", - ); - } else { - toast.error("Could not copy logs to clipboard"); + if (logs.length) { + fetchInitialLogs() + .then(() => { + copy(logs.join("\n")); + toast.success("Copied logs to clipboard"); + }) + .catch(() => { + toast.error("Could not copy logs to clipboard"); + }); } - }, [logs, logRange]); + }, [logs, fetchInitialLogs]); const handleDownloadLogs = useCallback(() => { axios @@ -157,153 +310,76 @@ function Logs() { .catch(() => {}); }, [logService]); - // scroll to bottom - - const [initialScroll, setInitialScroll] = useState(false); - - const contentRef = useRef(null); - const [endVisible, setEndVisible] = useState(true); - const endObserver = useRef(null); - const endLogRef = useCallback( - (node: HTMLElement | null) => { - if (endObserver.current) endObserver.current.disconnect(); - try { - endObserver.current = new IntersectionObserver((entries) => { - setEndVisible(entries[0].isIntersecting); - }); - if (node) endObserver.current.observe(node); - } catch (e) { - // no op - } + const handleRowClick = useCallback( + (rowInfo: { lineNumber: number; rowIndex: number }) => { + const clickedLine = parseLogLines(logService, [ + logs[rowInfo.rowIndex], + ])[0]; + setSelectedLog(clickedLine); }, - [setEndVisible], - ); - const startObserver = useRef(null); - const startLogRef = useCallback( - (node: HTMLElement | null) => { - if (startObserver.current) startObserver.current.disconnect(); - - if (logs.length == 0 || !initialScroll) { - return; - } - - try { - startObserver.current = new IntersectionObserver( - (entries) => { - if (entries[0].isIntersecting && logRange.start > 0) { - const start = Math.max(0, logRange.start - logPageSize); - - axios - .get(`logs/${logService}?start=${start}&end=${logRange.start}`) - .then((resp) => { - if (resp.status == 200) { - const data = resp.data as LogData; - - if (data.lines.length > 0) { - setLogRange({ - start: start, - end: logRange.end, - }); - setLogs([...data.lines, ...logs]); - setLogLines([ - ...parseLogLines(logService, data.lines), - ...logLines, - ]); - } - } - }) - .catch(() => {}); - contentRef.current?.scrollBy({ - top: 10, - }); - } - }, - { rootMargin: `${10 * (isMobile ? 64 : 48)}px 0px 0px 0px` }, - ); - if (node) startObserver.current.observe(node); - } catch (e) { - // no op - } - }, - // we need to listen on the current range of visible items - // eslint-disable-next-line react-hooks/exhaustive-deps - [logRange, initialScroll], + [logs, logService], ); - useEffect(() => { - if (logLines.length == 0) { - setInitialScroll(false); - return; - } - - if (initialScroll) { - return; - } - - if (!contentRef.current) { - return; - } - - if (contentRef.current.scrollHeight <= contentRef.current.clientHeight) { - setInitialScroll(true); - return; - } - - contentRef.current?.scrollTo({ - top: contentRef.current?.scrollHeight, - behavior: "instant", - }); - setTimeout(() => setInitialScroll(true), 300); - // we need to listen on the current range of visible items - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [logLines, logService]); - - // log filtering - - const [filterSeverity, setFilterSeverity] = useState(); - - // log selection - - const [selectedLog, setSelectedLog] = useState(); - - // interaction + // keyboard listener useKeyboardListener( ["PageDown", "PageUp", "ArrowDown", "ArrowUp"], (key, modifiers) => { - if (!modifiers.down) { + if (!key || !modifiers.down || !lazyLogWrapperRef.current) { return; } - switch (key) { - case "PageDown": - contentRef.current?.scrollBy({ - top: 480, - }); - break; - case "PageUp": - contentRef.current?.scrollBy({ - top: -480, - }); - break; - case "ArrowDown": - contentRef.current?.scrollBy({ - top: 48, - }); - break; - case "ArrowUp": - contentRef.current?.scrollBy({ - top: -48, - }); - break; + const container = + lazyLogWrapperRef.current.querySelector(".react-lazylog"); + + const logLineHeight = container?.querySelector(".log-line")?.clientHeight; + + if (!logLineHeight) { + return; } + + const scrollAmount = key.includes("Page") + ? logLineHeight * 10 + : logLineHeight; + const direction = key.includes("Down") ? 1 : -1; + container?.scrollBy({ top: scrollAmount * direction }); }, ); + // format lines + + const lineBufferRef = useRef(""); + + const formatPart = useCallback( + (text: string) => { + lineBufferRef.current += text; + + if (text.endsWith("\n")) { + const completeLine = lineBufferRef.current.trim(); + lineBufferRef.current = ""; + + if (completeLine) { + const parsedLine = parseLogLines(logService, [completeLine])[0]; + return ( + setFilterSeverity([parsedLine.severity])} + onSelect={() => setSelectedLog(parsedLine)} + /> + ); + } + } + + return null; + }, + [logService, setFilterSeverity, setSelectedLog], + ); + useEffect(() => { const handleCopy = (e: ClipboardEvent) => { e.preventDefault(); - if (!contentRef.current) return; + if (!lazyLogWrapperRef.current) return; const selection = window.getSelection(); if (!selection) return; @@ -371,7 +447,7 @@ function Logs() { e.clipboardData?.setData("text/plain", copyText); }; - const content = contentRef.current; + const content = lazyLogWrapperRef.current; content?.addEventListener("copy", handleCopy); return () => { content?.removeEventListener("copy", handleCopy); @@ -393,11 +469,10 @@ function Logs() { onValueChange={(value: LogType) => { if (value) { setLogs([]); - setLogLines([]); setFilterSeverity(undefined); setLogService(value); } - }} // don't allow the severity to be unselected + }} > {Object.values(logTypes).map((item) => (
Download
-
- {initialScroll && !endVisible && ( - - )} - -
-
-
Type
-
- Timestamp +
+
+
+
+
Type
+
Timestamp
+
-
Tag
-
- Message +
+ Tag +
+
+
Message
-
- {logLines.length > 0 && - [...Array(logRange.end).keys()].map((idx) => { - const logLine = - idx >= logRange.start - ? logLines[idx - logRange.start] - : undefined; - if (logLine) { - const line = logLines[idx - logRange.start]; - if (filterSeverity && !filterSeverity.includes(line.severity)) { - return ( -
- ); - } - - return ( - + {isLoading ? ( + + ) : ( + ( + <> + {follow && !logSettings.disableStreaming && ( +
+ + + + + + Logs are streaming from the server + + +
+ )} + } - className={initialScroll ? "" : "invisible"} - line={line} - onClickSeverity={() => setFilterSeverity([line.severity])} - onSelect={() => setSelectedLog(line)} + loading={isLoading} /> - ); - } - - return ( -
- ); - })} - {logLines.length > 0 &&
} + + )} + /> + )}
- {logLines.length == 0 && ( - - )}
); } type LogLineDataProps = { - startRef?: (node: HTMLDivElement | null) => void; - className: string; + className?: string; line: LogLine; + logService: string; onClickSeverity: () => void; onSelect: () => void; }; + function LogLineData({ - startRef, className, line, + logService, onClickSeverity, onSelect, }: LogLineDataProps) { return (
-
- +
+
+
+ +
+
+ {line.dateStamp} +
+
-
- {line.dateStamp} -
-
+ +
{line.section}
-
+
{line.content}
diff --git a/web/src/types/log.ts b/web/src/types/log.ts index 407f67e6d..2e856f574 100644 --- a/web/src/types/log.ts +++ b/web/src/types/log.ts @@ -14,3 +14,7 @@ export type LogLine = { export const logTypes = ["frigate", "go2rtc", "nginx"] as const; export type LogType = (typeof logTypes)[number]; + +export type LogSettingsType = { + disableStreaming: boolean; +}; diff --git a/web/src/utils/logUtil.ts b/web/src/utils/logUtil.ts index 569d417be..ac6eaaec2 100644 --- a/web/src/utils/logUtil.ts +++ b/web/src/utils/logUtil.ts @@ -18,13 +18,29 @@ export function parseLogLines(logService: LogType, logs: string[]) { if (!match) { const infoIndex = line.indexOf("[INFO]"); + const loggingIndex = line.indexOf("[LOGGING]"); + + if (loggingIndex != -1) { + return { + dateStamp: line.substring(0, 19), + severity: "info", + section: "logging", + content: line + .substring(loggingIndex + 9) + .trim() + .replace(/\u200b/g, "\n"), + }; + } if (infoIndex != -1) { return { dateStamp: line.substring(0, 19), severity: "info", section: "startup", - content: line.substring(infoIndex + 6).trim(), + content: line + .substring(infoIndex + 6) + .trim() + .replace(/\u200b/g, "\n"), }; } @@ -32,7 +48,10 @@ export function parseLogLines(logService: LogType, logs: string[]) { dateStamp: line.substring(0, 19), severity: "unknown", section: "unknown", - content: line.substring(30).trim(), + content: line + .substring(30) + .trim() + .replace(/\u200b/g, "\n"), }; } @@ -54,7 +73,8 @@ export function parseLogLines(logService: LogType, logs: string[]) { section: sectionMatch.toString(), content: line .substring(line.indexOf(":", match.index + match[0].length) + 2) - .trim(), + .trim() + .replace(/\u200b/g, "\n"), }; }) .filter((value) => value != null) as LogLine[]; @@ -86,6 +106,15 @@ export function parseLogLines(logService: LogType, logs: string[]) { contentStart = line.indexOf(section) + section.length + 2; } + if (line.includes("[LOGGING]")) { + return { + dateStamp: line.substring(0, 19), + severity: "info", + section: "logging", + content: line.substring(line.indexOf("[LOGGING]") + 9).trim(), + }; + } + let severityCat: LogSeverity; switch (severity?.at(0)?.toString().trim()) { case "INF": @@ -116,18 +145,68 @@ export function parseLogLines(logService: LogType, logs: string[]) { } else if (logService == "nginx") { return logs .map((line) => { - if (line.length == 0) { - return null; - } + if (line.trim().length === 0) return null; - return { - dateStamp: line.substring(0, 19), - severity: "info", - section: httpMethods.exec(line)?.at(0)?.toString() ?? "META", - content: line.substring(line.indexOf(" ", 20)).trim(), - }; + // Match full timestamp including nanoseconds + const timestampRegex = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+/; + const timestampMatch = timestampRegex.exec(line); + const fullTimestamp = timestampMatch ? timestampMatch[0] : ""; + // Remove nanoseconds from the final output + const dateStamp = fullTimestamp.split(".")[0]; + + if (line.includes("[LOGGING]")) { + return { + dateStamp, + severity: "info", + section: "logging", + content: line.slice(line.indexOf("[LOGGING]") + 9).trim(), + }; + } else if (line.includes("[INFO]")) { + return { + dateStamp, + severity: "info", + section: "startup", + content: line.slice(fullTimestamp.length).trim(), + }; + } else if (line.includes("[error]")) { + // Error log + const errorMatch = line.match(/(\[error\].*?,.*request: "[^"]*")/); + const content = errorMatch ? errorMatch[1] : line; + return { + dateStamp, + severity: "error", + section: "error", + content, + }; + } else if ( + line.includes("GET") || + line.includes("POST") || + line.includes("HTTP") + ) { + // HTTP request log + const httpMethodMatch = httpMethods.exec(line); + const section = httpMethodMatch ? httpMethodMatch[0] : "META"; + const contentStart = line.indexOf('"', fullTimestamp.length); + const content = + contentStart !== -1 ? line.slice(contentStart).trim() : line; + + return { + dateStamp, + severity: "info", + section, + content, + }; + } else { + // Fallback: unknown format + return { + dateStamp, + severity: "unknown", + section: "unknown", + content: line.slice(fullTimestamp.length).trim(), + }; + } }) - .filter((value) => value != null) as LogLine[]; + .filter((value) => value !== null) as LogLine[]; } return [];