mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-04-19 23:08:08 +02:00
Add GenAI Backend Streaming and Chat (#22152)
* Add basic chat page with entry * Add chat history * processing * Add markdown * Improvements * Adjust timing format * Reduce fields in response * More time parsing improvements * Show tool calls separately from message * Add title * Improve UI handling * Support streaming * Full streaming support * Fix tool calling * Add copy button * Improvements to UI * Improve default behavior * Implement message editing * Add sub label to event tool filtering * Cleanup * Cleanup UI and prompt * Cleanup UI bubbles * Fix loading * Add support for markdown tables * Add thumbnail images to object results * Add a starting state for chat * Clenaup
This commit is contained in:
Nicolas Mowen
GitHub
42
web/src/components/chat/ChatEventThumbnailsRow.tsx
Normal file
42
web/src/components/chat/ChatEventThumbnailsRow.tsx
Normal file
@@ -0,0 +1,42 @@
|
||||
import { useApiHost } from "@/api";
|
||||
|
||||
type ChatEventThumbnailsRowProps = {
|
||||
events: { id: string }[];
|
||||
};
|
||||
|
||||
/**
|
||||
* Horizontal scroll row of event thumbnail images for chat (e.g. after search_objects).
|
||||
* Renders nothing when events is empty.
|
||||
*/
|
||||
export function ChatEventThumbnailsRow({
|
||||
events,
|
||||
}: ChatEventThumbnailsRowProps) {
|
||||
const apiHost = useApiHost();
|
||||
|
||||
if (events.length === 0) return null;
|
||||
|
||||
return (
|
||||
<div className="flex min-w-0 max-w-full flex-col gap-1 self-start">
|
||||
<div className="scrollbar-container min-w-0 overflow-x-auto">
|
||||
<div className="flex w-max gap-2">
|
||||
{events.map((event) => (
|
||||
<a
|
||||
key={event.id}
|
||||
href={`/explore?event_id=${event.id}`}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="relative aspect-square size-32 shrink-0 overflow-hidden rounded-lg"
|
||||
>
|
||||
<img
|
||||
className="size-full object-cover"
|
||||
src={`${apiHost}api/events/${event.id}/thumbnail.webp`}
|
||||
alt=""
|
||||
loading="lazy"
|
||||
/>
|
||||
</a>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
208
web/src/components/chat/ChatMessage.tsx
Normal file
208
web/src/components/chat/ChatMessage.tsx
Normal file
@@ -0,0 +1,208 @@
|
||||
import { useState, useEffect, useRef } from "react";
|
||||
import ReactMarkdown from "react-markdown";
|
||||
import remarkGfm from "remark-gfm";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import copy from "copy-to-clipboard";
|
||||
import { toast } from "sonner";
|
||||
import { FaCopy, FaPencilAlt } from "react-icons/fa";
|
||||
import { FaArrowUpLong } from "react-icons/fa6";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Textarea } from "@/components/ui/textarea";
|
||||
import {
|
||||
Tooltip,
|
||||
TooltipContent,
|
||||
TooltipTrigger,
|
||||
} from "@/components/ui/tooltip";
|
||||
import { cn } from "@/lib/utils";
|
||||
|
||||
type MessageBubbleProps = {
|
||||
role: "user" | "assistant";
|
||||
content: string;
|
||||
messageIndex?: number;
|
||||
onEditSubmit?: (messageIndex: number, newContent: string) => void;
|
||||
isComplete?: boolean;
|
||||
};
|
||||
|
||||
export function MessageBubble({
|
||||
role,
|
||||
content,
|
||||
messageIndex = 0,
|
||||
onEditSubmit,
|
||||
isComplete = true,
|
||||
}: MessageBubbleProps) {
|
||||
const { t } = useTranslation(["views/chat", "common"]);
|
||||
const isUser = role === "user";
|
||||
const [isEditing, setIsEditing] = useState(false);
|
||||
const [draftContent, setDraftContent] = useState(content);
|
||||
const editInputRef = useRef<HTMLTextAreaElement>(null);
|
||||
|
||||
useEffect(() => {
|
||||
setDraftContent(content);
|
||||
}, [content]);
|
||||
|
||||
useEffect(() => {
|
||||
if (isEditing) {
|
||||
editInputRef.current?.focus();
|
||||
editInputRef.current?.setSelectionRange(
|
||||
editInputRef.current.value.length,
|
||||
editInputRef.current.value.length,
|
||||
);
|
||||
}
|
||||
}, [isEditing]);
|
||||
|
||||
const handleCopy = () => {
|
||||
const text = content?.trim() || "";
|
||||
if (!text) return;
|
||||
if (copy(text)) {
|
||||
toast.success(t("button.copiedToClipboard", { ns: "common" }));
|
||||
}
|
||||
};
|
||||
|
||||
const handleEditClick = () => {
|
||||
setDraftContent(content);
|
||||
setIsEditing(true);
|
||||
};
|
||||
|
||||
const handleEditSubmit = () => {
|
||||
const trimmed = draftContent.trim();
|
||||
if (!trimmed || onEditSubmit == null) return;
|
||||
onEditSubmit(messageIndex, trimmed);
|
||||
setIsEditing(false);
|
||||
};
|
||||
|
||||
const handleEditCancel = () => {
|
||||
setDraftContent(content);
|
||||
setIsEditing(false);
|
||||
};
|
||||
|
||||
const handleEditKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
|
||||
if (e.key === "Enter" && !e.shiftKey) {
|
||||
e.preventDefault();
|
||||
handleEditSubmit();
|
||||
}
|
||||
if (e.key === "Escape") {
|
||||
handleEditCancel();
|
||||
}
|
||||
};
|
||||
|
||||
if (isUser && isEditing) {
|
||||
return (
|
||||
<div className="flex w-full max-w-full flex-col gap-2 self-end">
|
||||
<Textarea
|
||||
ref={editInputRef}
|
||||
value={draftContent}
|
||||
onChange={(e) => setDraftContent(e.target.value)}
|
||||
onKeyDown={handleEditKeyDown}
|
||||
className="min-h-[80px] w-full resize-y rounded-lg bg-primary px-3 py-2 text-primary-foreground placeholder:text-primary-foreground/60"
|
||||
placeholder={t("placeholder")}
|
||||
rows={3}
|
||||
/>
|
||||
<div className="flex items-center gap-2 self-end">
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
className="text-muted-foreground hover:text-foreground"
|
||||
onClick={handleEditCancel}
|
||||
>
|
||||
{t("button.cancel", { ns: "common" })}
|
||||
</Button>
|
||||
<Button
|
||||
variant="select"
|
||||
size="icon"
|
||||
className="size-9 rounded-full"
|
||||
disabled={!draftContent.trim()}
|
||||
onClick={handleEditSubmit}
|
||||
aria-label={t("send")}
|
||||
>
|
||||
<FaArrowUpLong size="16" />
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div
|
||||
className={cn(
|
||||
"flex flex-col gap-1",
|
||||
isUser ? "items-end self-end" : "items-start self-start",
|
||||
)}
|
||||
>
|
||||
<div
|
||||
className={cn(
|
||||
"rounded-lg px-3 py-2",
|
||||
isUser ? "bg-primary text-primary-foreground" : "bg-muted",
|
||||
)}
|
||||
>
|
||||
{isUser ? (
|
||||
content
|
||||
) : (
|
||||
<ReactMarkdown
|
||||
remarkPlugins={[remarkGfm]}
|
||||
components={{
|
||||
table: ({ node: _n, ...props }) => (
|
||||
<table
|
||||
className="my-2 w-full border-collapse border border-border"
|
||||
{...props}
|
||||
/>
|
||||
),
|
||||
th: ({ node: _n, ...props }) => (
|
||||
<th
|
||||
className="border border-border bg-muted/50 px-2 py-1 text-left text-sm font-medium"
|
||||
{...props}
|
||||
/>
|
||||
),
|
||||
td: ({ node: _n, ...props }) => (
|
||||
<td
|
||||
className="border border-border px-2 py-1 text-sm"
|
||||
{...props}
|
||||
/>
|
||||
),
|
||||
}}
|
||||
>
|
||||
{content}
|
||||
</ReactMarkdown>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex items-center gap-0.5">
|
||||
{isUser && onEditSubmit != null && (
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="size-7 text-muted-foreground hover:text-foreground"
|
||||
onClick={handleEditClick}
|
||||
aria-label={t("button.edit", { ns: "common" })}
|
||||
>
|
||||
<FaPencilAlt className="size-3" />
|
||||
</Button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>
|
||||
{t("button.edit", { ns: "common" })}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
)}
|
||||
{isComplete && (
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="size-7 text-muted-foreground hover:text-foreground"
|
||||
onClick={handleCopy}
|
||||
disabled={!content?.trim()}
|
||||
aria-label={t("button.copy", { ns: "common" })}
|
||||
>
|
||||
<FaCopy className="size-3" />
|
||||
</Button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>
|
||||
{t("button.copy", { ns: "common" })}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
89
web/src/components/chat/ChatStartingState.tsx
Normal file
89
web/src/components/chat/ChatStartingState.tsx
Normal file
@@ -0,0 +1,89 @@
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { FaArrowUpLong } from "react-icons/fa6";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { useState } from "react";
|
||||
import type { StartingRequest } from "@/types/chat";
|
||||
|
||||
type ChatStartingStateProps = {
|
||||
onSendMessage: (message: string) => void;
|
||||
};
|
||||
|
||||
export function ChatStartingState({ onSendMessage }: ChatStartingStateProps) {
|
||||
const { t } = useTranslation(["views/chat"]);
|
||||
const [input, setInput] = useState("");
|
||||
|
||||
const defaultRequests: StartingRequest[] = [
|
||||
{
|
||||
label: t("starting_requests.show_recent_events"),
|
||||
prompt: t("starting_requests_prompts.show_recent_events"),
|
||||
},
|
||||
{
|
||||
label: t("starting_requests.show_camera_status"),
|
||||
prompt: t("starting_requests_prompts.show_camera_status"),
|
||||
},
|
||||
];
|
||||
|
||||
const handleRequestClick = (prompt: string) => {
|
||||
onSendMessage(prompt);
|
||||
};
|
||||
|
||||
const handleSubmit = () => {
|
||||
const text = input.trim();
|
||||
if (!text) return;
|
||||
onSendMessage(text);
|
||||
setInput("");
|
||||
};
|
||||
|
||||
const handleKeyDown = (e: React.KeyboardEvent<HTMLInputElement>) => {
|
||||
if (e.key === "Enter" && !e.shiftKey) {
|
||||
e.preventDefault();
|
||||
handleSubmit();
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="flex size-full flex-col items-center justify-center gap-6 p-8">
|
||||
<div className="flex flex-col items-center gap-2">
|
||||
<h1 className="text-4xl font-bold text-foreground">{t("title")}</h1>
|
||||
<p className="text-muted-foreground">{t("subtitle")}</p>
|
||||
</div>
|
||||
|
||||
<div className="flex w-full max-w-2xl flex-col items-center gap-4">
|
||||
<p className="text-center text-sm text-muted-foreground">
|
||||
{t("suggested_requests")}
|
||||
</p>
|
||||
<div className="flex w-full flex-wrap justify-center gap-2">
|
||||
{defaultRequests.map((request, idx) => (
|
||||
<Button
|
||||
key={idx}
|
||||
variant="outline"
|
||||
className="max-w-sm text-sm"
|
||||
onClick={() => handleRequestClick(request.prompt)}
|
||||
>
|
||||
{request.label}
|
||||
</Button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex w-full max-w-2xl flex-row items-center gap-2 rounded-xl bg-secondary p-4">
|
||||
<Input
|
||||
className="h-12 w-full flex-1 border-transparent bg-transparent text-base shadow-none focus-visible:ring-0 dark:bg-transparent"
|
||||
placeholder={t("placeholder")}
|
||||
value={input}
|
||||
onChange={(e) => setInput(e.target.value)}
|
||||
onKeyDown={handleKeyDown}
|
||||
/>
|
||||
<Button
|
||||
variant="select"
|
||||
className="size-10 shrink-0 rounded-full"
|
||||
disabled={!input.trim()}
|
||||
onClick={handleSubmit}
|
||||
>
|
||||
<FaArrowUpLong size="18" />
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
88
web/src/components/chat/ToolCallBubble.tsx
Normal file
88
web/src/components/chat/ToolCallBubble.tsx
Normal file
@@ -0,0 +1,88 @@
|
||||
import { useState } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import {
|
||||
Collapsible,
|
||||
CollapsibleContent,
|
||||
CollapsibleTrigger,
|
||||
} from "@/components/ui/collapsible";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { ChevronDown, ChevronRight } from "lucide-react";
|
||||
|
||||
type ToolCallBubbleProps = {
|
||||
name: string;
|
||||
arguments?: Record<string, unknown>;
|
||||
response?: string;
|
||||
side: "left" | "right";
|
||||
};
|
||||
|
||||
export function ToolCallBubble({
|
||||
name,
|
||||
arguments: args,
|
||||
response,
|
||||
side,
|
||||
}: ToolCallBubbleProps) {
|
||||
const { t } = useTranslation(["views/chat"]);
|
||||
const [open, setOpen] = useState(false);
|
||||
const isLeft = side === "left";
|
||||
const normalizedName = name
|
||||
.replace(/_/g, " ")
|
||||
.split(" ")
|
||||
.map((word) => word.charAt(0).toUpperCase() + word.slice(1).toLowerCase())
|
||||
.join(" ");
|
||||
|
||||
return (
|
||||
<div
|
||||
className={cn(
|
||||
"rounded-lg px-3 py-2",
|
||||
isLeft
|
||||
? "self-start bg-muted"
|
||||
: "self-end bg-primary text-primary-foreground",
|
||||
)}
|
||||
>
|
||||
<Collapsible open={open} onOpenChange={setOpen}>
|
||||
<CollapsibleTrigger asChild>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
className={cn(
|
||||
"h-auto w-full min-w-0 justify-start gap-2 whitespace-normal p-0 text-left text-xs hover:bg-transparent",
|
||||
!isLeft && "hover:text-primary-foreground",
|
||||
)}
|
||||
>
|
||||
{open ? (
|
||||
<ChevronDown size={12} className="shrink-0" />
|
||||
) : (
|
||||
<ChevronRight size={12} className="shrink-0" />
|
||||
)}
|
||||
<span className="break-words font-medium">
|
||||
{isLeft ? t("call") : t("result")} {normalizedName}
|
||||
</span>
|
||||
</Button>
|
||||
</CollapsibleTrigger>
|
||||
<CollapsibleContent>
|
||||
<div className="mt-2 space-y-2">
|
||||
{isLeft && args && Object.keys(args).length > 0 && (
|
||||
<div className="text-xs">
|
||||
<div className="font-medium text-muted-foreground">
|
||||
{t("arguments")}
|
||||
</div>
|
||||
<pre className="scrollbar-container mt-1 max-h-32 overflow-auto whitespace-pre-wrap break-words rounded bg-muted/50 p-2 text-[10px]">
|
||||
{JSON.stringify(args, null, 2)}
|
||||
</pre>
|
||||
</div>
|
||||
)}
|
||||
{!isLeft && response && response !== "" && (
|
||||
<div className="text-xs">
|
||||
<div className="font-medium opacity-80">{t("response")}</div>
|
||||
<pre className="scrollbar-container mt-1 max-h-32 overflow-auto whitespace-pre-wrap break-words rounded bg-primary/20 p-2 text-[10px]">
|
||||
{response}
|
||||
</pre>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</CollapsibleContent>
|
||||
</Collapsible>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user