Ongoing review segments (#10924)

* Update review maintainer to save events when ongoing

* Handle previews for in progress review items

* Reset DB items in app

* Handle in progress review items

* Scroll back down to selected event item

* Handle undefined end time

* Formatting

* remove unused

* Make export handles have full resolution

* reduce preview thumbnail props

* fix missing return

Co-authored-by: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>

---------

Co-authored-by: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>
This commit is contained in:
Nicolas Mowen 2024-04-11 06:42:16 -06:00 committed by GitHub
parent cf7698e7e1
commit 049f27d710
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 160 additions and 64 deletions

View File

@ -1333,7 +1333,9 @@ def review_preview(id: str):
padding = 8
start_ts = review.start_time - padding
end_ts = review.end_time + padding
end_ts = (
review.end_time + padding if review.end_time else datetime.now().timestamp()
)
return preview_gif(review.camera, start_ts, end_ts)
@ -1344,8 +1346,15 @@ def preview_thumbnail(file_name: str):
safe_file_name_current = secure_filename(file_name)
preview_dir = os.path.join(CACHE_DIR, "preview_frames")
with open(os.path.join(preview_dir, safe_file_name_current), "rb") as image_file:
try:
with open(
os.path.join(preview_dir, safe_file_name_current), "rb"
) as image_file:
jpg_bytes = image_file.read()
except FileNotFoundError:
return make_response(
jsonify({"success": False, "message": "Image file not found"}), 404
)
response = make_response(jpg_bytes)
response.headers["Content-Type"] = "image/jpeg"

View File

@ -27,10 +27,18 @@ def review():
before = request.args.get("before", type=float, default=datetime.now().timestamp())
after = request.args.get(
"after", type=float, default=(datetime.now() - timedelta(hours=18)).timestamp()
"after", type=float, default=(datetime.now() - timedelta(hours=24)).timestamp()
)
clauses = [((ReviewSegment.start_time > after) & (ReviewSegment.end_time < before))]
clauses = [
(
(ReviewSegment.start_time > after)
& (
(ReviewSegment.end_time.is_null(True))
| (ReviewSegment.end_time < before)
)
)
]
if cameras != "all":
camera_list = cameras.split(",")
@ -45,6 +53,7 @@ def review():
for label in filtered_labels:
label_clauses.append(
(ReviewSegment.data["objects"].cast("text") % f'*"{label}"*')
| (ReviewSegment.data["audio"].cast("text") % f'*"{label}"*')
)
label_clause = reduce(operator.or_, label_clauses)
@ -94,6 +103,7 @@ def review_summary():
for label in filtered_labels:
label_clauses.append(
(ReviewSegment.data["objects"].cast("text") % f'*"{label}"*')
| (ReviewSegment.data["audio"].cast("text") % f'*"{label}"*')
)
label_clause = reduce(operator.or_, label_clauses)

View File

@ -667,6 +667,14 @@ class FrigateApp:
logger.info("Stopping...")
self.stop_event.set()
# set an end_time on entries without an end_time before exiting
Event.update(end_time=datetime.datetime.now().timestamp()).where(
Event.end_time == None
).execute()
ReviewSegment.update(end_time=datetime.datetime.now().timestamp()).where(
ReviewSegment.end_time == None
).execute()
# Stop Communicators
self.inter_process_communicator.stop()
self.inter_config_updater.stop()

View File

@ -1,4 +1,3 @@
import datetime
import logging
import threading
from multiprocessing import Queue
@ -112,10 +111,6 @@ class EventProcessor(threading.Thread):
self.handle_external_detection(event_type, event_data)
# set an end_time on events without an end_time before exiting
Event.update(end_time=datetime.datetime.now().timestamp()).where(
Event.end_time == None
).execute()
self.event_receiver.stop()
self.event_end_publisher.stop()
logger.info("Exiting event processor...")

View File

@ -66,6 +66,7 @@ class PendingReviewSegment:
# thumbnail
self.frame = np.zeros((THUMB_HEIGHT * 3 // 2, THUMB_WIDTH), np.uint8)
self.frame_active_count = 0
self.frame_path = os.path.join(CLIPS_DIR, f"thumb-{self.camera}-{self.id}.jpg")
def update_frame(
self, camera_config: CameraConfig, frame, objects: list[TrackedObject]
@ -98,19 +99,19 @@ class PendingReviewSegment:
color_frame, dsize=(width, THUMB_HEIGHT), interpolation=cv2.INTER_AREA
)
def end(self) -> dict:
path = os.path.join(CLIPS_DIR, f"thumb-{self.camera}-{self.id}.jpg")
if self.frame is not None:
cv2.imwrite(path, self.frame, [int(cv2.IMWRITE_WEBP_QUALITY), 60])
cv2.imwrite(
self.frame_path, self.frame, [int(cv2.IMWRITE_WEBP_QUALITY), 60]
)
def get_data(self, ended: bool) -> dict:
return {
ReviewSegment.id: self.id,
ReviewSegment.camera: self.camera,
ReviewSegment.start_time: self.start_time,
ReviewSegment.end_time: self.last_update,
ReviewSegment.end_time: self.last_update if ended else None,
ReviewSegment.severity: self.severity.value,
ReviewSegment.thumb_path: path,
ReviewSegment.thumb_path: self.frame_path,
ReviewSegment.data: {
"detections": list(set(self.detections.keys())),
"objects": list(set(self.detections.values())),
@ -141,9 +142,20 @@ class ReviewSegmentMaintainer(threading.Thread):
self.stop_event = stop_event
def update_segment(self, segment: PendingReviewSegment) -> None:
"""Update segment."""
seg_data = segment.get_data(ended=False)
self.requestor.send_data(UPSERT_REVIEW_SEGMENT, seg_data)
self.requestor.send_data(
"reviews",
json.dumps(
{"type": "update", "review": {k.name: v for k, v in seg_data.items()}}
),
)
def end_segment(self, segment: PendingReviewSegment) -> None:
"""End segment."""
seg_data = segment.end()
seg_data = segment.get_data(ended=True)
self.requestor.send_data(UPSERT_REVIEW_SEGMENT, seg_data)
self.requestor.send_data(
"reviews",
@ -179,6 +191,7 @@ class ReviewSegmentMaintainer(threading.Thread):
)
segment.update_frame(camera_config, yuv_frame, active_objects)
self.frame_manager.close(frame_id)
self.update_segment(segment)
for object in active_objects:
if not object["sub_label"]:
@ -263,6 +276,7 @@ class ReviewSegmentMaintainer(threading.Thread):
camera_config, yuv_frame, active_objects
)
self.frame_manager.close(frame_id)
self.update_segment(self.active_review_segments[camera])
elif len(motion) >= 20:
self.active_review_segments[camera] = PendingReviewSegment(
camera,
@ -398,6 +412,11 @@ class ReviewSegmentMaintainer(threading.Thread):
"end_time"
]
self.config_subscriber.stop()
self.requestor.stop()
self.detection_subscriber.stop()
logger.info("Exiting review maintainer...")
def get_active_objects(
frame_time: float, camera_config: CameraConfig, all_objects: list[TrackedObject]

View File

@ -27,7 +27,9 @@ export default function ReviewCard({
config?.ui.time_format == "24hour" ? "%H:%M" : "%I:%M %p",
);
const isSelected = useMemo(
() => event.start_time <= currentTime && event.end_time >= currentTime,
() =>
event.start_time <= currentTime &&
(event.end_time ?? Date.now() / 1000) >= currentTime,
[event, currentTime],
);

View File

@ -21,11 +21,14 @@ import { useSwipeable } from "react-swipeable";
import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip";
import ImageLoadingIndicator from "../indicators/ImageLoadingIndicator";
import useContextMenu from "@/hooks/use-contextmenu";
import ActivityIndicator from "../indicators/activity-indicator";
import { TimeRange } from "@/types/timeline";
type PreviewPlayerProps = {
review: ReviewSegment;
allPreviews?: Preview[];
scrollLock?: boolean;
timeRange: TimeRange;
onTimeUpdate?: (time: number | undefined) => void;
setReviewed: (review: ReviewSegment) => void;
onClick: (review: ReviewSegment, ctrl: boolean) => void;
@ -43,6 +46,7 @@ export default function PreviewThumbnailPlayer({
review,
allPreviews,
scrollLock = false,
timeRange,
setReviewed,
onClick,
onTimeUpdate,
@ -70,8 +74,10 @@ export default function PreviewThumbnailPlayer({
});
const handleSetReviewed = useCallback(() => {
if (review.end_time && !review.has_been_reviewed) {
review.has_been_reviewed = true;
setReviewed(review);
}
}, [review, setReviewed]);
useContextMenu(imgRef, () => {
@ -91,7 +97,7 @@ export default function PreviewThumbnailPlayer({
return false;
}
if (review.end_time > preview.end) {
if ((review.end_time ?? timeRange.before) > preview.end) {
multiHour = true;
}
@ -108,7 +114,8 @@ export default function PreviewThumbnailPlayer({
const firstPrev = allPreviews[firstIndex];
const firstDuration = firstPrev.end - review.start_time;
const secondDuration = review.end_time - firstPrev.end;
const secondDuration =
(review.end_time ?? timeRange.before) - firstPrev.end;
if (firstDuration > secondDuration) {
// the first preview is longer than the second, return the first
@ -123,7 +130,7 @@ export default function PreviewThumbnailPlayer({
return undefined;
}
}, [allPreviews, review]);
}, [allPreviews, review, timeRange]);
// Hover Playback
@ -183,6 +190,7 @@ export default function PreviewThumbnailPlayer({
<PreviewContent
review={review}
relevantPreview={relevantPreview}
timeRange={timeRange}
setReviewed={handleSetReviewed}
setIgnoreClick={setIgnoreClick}
isPlayingBack={setPlayback}
@ -256,7 +264,13 @@ export default function PreviewThumbnailPlayer({
<div className="absolute top-0 inset-x-0 rounded-t-l z-10 w-full h-[30%] bg-gradient-to-b from-black/60 to-transparent pointer-events-none"></div>
<div className="absolute bottom-0 inset-x-0 rounded-b-l z-10 w-full h-[20%] bg-gradient-to-t from-black/60 to-transparent pointer-events-none">
<div className="flex h-full justify-between items-end mx-3 pb-1 text-white text-sm">
{review.end_time ? (
<TimeAgo time={review.start_time * 1000} dense />
) : (
<div>
<ActivityIndicator size={24} />
</div>
)}
{formattedDate}
</div>
</div>
@ -270,6 +284,7 @@ export default function PreviewThumbnailPlayer({
type PreviewContentProps = {
review: ReviewSegment;
relevantPreview: Preview | undefined;
timeRange: TimeRange;
setReviewed: () => void;
setIgnoreClick: (ignore: boolean) => void;
isPlayingBack: (ended: boolean) => void;
@ -278,6 +293,7 @@ type PreviewContentProps = {
function PreviewContent({
review,
relevantPreview,
timeRange,
setReviewed,
setIgnoreClick,
isPlayingBack,
@ -288,8 +304,9 @@ function PreviewContent({
if (relevantPreview) {
return (
<VideoPreview
review={review}
relevantPreview={relevantPreview}
startTime={review.start_time}
endTime={review.end_time}
setReviewed={setReviewed}
setIgnoreClick={setIgnoreClick}
isPlayingBack={isPlayingBack}
@ -300,6 +317,7 @@ function PreviewContent({
return (
<InProgressPreview
review={review}
timeRange={timeRange}
setReviewed={setReviewed}
setIgnoreClick={setIgnoreClick}
isPlayingBack={isPlayingBack}
@ -311,16 +329,18 @@ function PreviewContent({
const PREVIEW_PADDING = 16;
type VideoPreviewProps = {
review: ReviewSegment;
relevantPreview: Preview;
startTime: number;
endTime?: number;
setReviewed: () => void;
setIgnoreClick: (ignore: boolean) => void;
isPlayingBack: (ended: boolean) => void;
onTimeUpdate?: (time: number | undefined) => void;
};
function VideoPreview({
review,
relevantPreview,
startTime,
endTime,
setReviewed,
setIgnoreClick,
isPlayingBack,
@ -339,16 +359,13 @@ function VideoPreview({
}
// start with a bit of padding
return Math.max(
0,
review.start_time - relevantPreview.start - PREVIEW_PADDING,
);
return Math.max(0, startTime - relevantPreview.start - PREVIEW_PADDING);
// we know that these deps are correct
// eslint-disable-next-line react-hooks/exhaustive-deps
}, []);
const playerDuration = useMemo(
() => review.end_time - review.start_time + PREVIEW_PADDING,
() => (endTime ?? relevantPreview.end) - startTime + PREVIEW_PADDING,
// we know that these deps are correct
// eslint-disable-next-line react-hooks/exhaustive-deps
[],
@ -389,21 +406,14 @@ function VideoPreview({
// end with a bit of padding
const playerPercent = (playerProgress / playerDuration) * 100;
if (
setReviewed &&
!review.has_been_reviewed &&
lastPercent < 50 &&
playerPercent > 50
) {
if (setReviewed && lastPercent < 50 && playerPercent > 50) {
setReviewed();
}
setLastPercent(playerPercent);
if (playerPercent > 100) {
if (!review.has_been_reviewed) {
setReviewed();
}
if (isMobile) {
isPlayingBack(false);
@ -468,7 +478,7 @@ function VideoPreview({
setIgnoreClick(true);
}
if (setReviewed && !review.has_been_reviewed) {
if (setReviewed) {
setReviewed();
}
@ -551,6 +561,7 @@ function VideoPreview({
const MIN_LOAD_TIMEOUT_MS = 200;
type InProgressPreviewProps = {
review: ReviewSegment;
timeRange: TimeRange;
setReviewed: (reviewId: string) => void;
setIgnoreClick: (ignore: boolean) => void;
isPlayingBack: (ended: boolean) => void;
@ -558,6 +569,7 @@ type InProgressPreviewProps = {
};
function InProgressPreview({
review,
timeRange,
setReviewed,
setIgnoreClick,
isPlayingBack,
@ -567,7 +579,7 @@ function InProgressPreview({
const sliderRef = useRef<HTMLDivElement | null>(null);
const { data: previewFrames } = useSWR<string[]>(
`preview/${review.camera}/start/${Math.floor(review.start_time) - PREVIEW_PADDING}/end/${
Math.ceil(review.end_time) + PREVIEW_PADDING
Math.ceil(review.end_time ?? timeRange.before) + PREVIEW_PADDING
}/frames`,
{ revalidateOnFocus: false },
);

View File

@ -100,8 +100,10 @@ export function MotionReviewTimeline({
const overlappingReviewItems = events.some(
(item) =>
(item.start_time >= motionStart && item.start_time < motionEnd) ||
(item.end_time > motionStart && item.end_time <= motionEnd) ||
(item.start_time <= motionStart && item.end_time >= motionEnd),
((item.end_time ?? timelineStart) > motionStart &&
(item.end_time ?? timelineStart) <= motionEnd) ||
(item.start_time <= motionStart &&
(item.end_time ?? timelineStart) >= motionEnd),
);
if ((!segmentMotion || overlappingReviewItems) && motionOnly) {

View File

@ -132,7 +132,6 @@ export function ReviewTimeline({
draggableElementTime: exportStartTime,
draggableElementLatestTime: paddedExportEndTime,
setDraggableElementTime: setExportStartTime,
alignSetTimeToSegment: true,
timelineDuration,
timelineStartAligned,
isDragging: isDraggingExportStart,
@ -157,7 +156,6 @@ export function ReviewTimeline({
draggableElementTime: exportEndTime,
draggableElementEarliestTime: paddedExportStartTime,
setDraggableElementTime: setExportEndTime,
alignSetTimeToSegment: true,
timelineDuration,
timelineStartAligned,
isDragging: isDraggingExportEnd,

View File

@ -107,8 +107,10 @@ export function useCameraMotionNextTimestamp(
const overlappingReviewItems = reviewItems.some(
(item) =>
(item.start_time >= motionStart && item.start_time < motionEnd) ||
(item.end_time > motionStart && item.end_time <= motionEnd) ||
(item.start_time <= motionStart && item.end_time >= motionEnd),
((item.end_time ?? Date.now() / 1000) > motionStart &&
(item.end_time ?? Date.now() / 1000) <= motionEnd) ||
(item.start_time <= motionStart &&
(item.end_time ?? Date.now() / 1000) >= motionEnd),
);
if (!segmentMotion || overlappingReviewItems) {

View File

@ -204,7 +204,7 @@ export default function Events() {
const newData = [...data];
newData.forEach((seg) => {
if (seg.severity == severity) {
if (seg.end_time && seg.severity == severity) {
seg.has_been_reviewed = true;
}
});
@ -214,10 +214,16 @@ export default function Events() {
{ revalidate: false, populateCache: true },
);
const itemsToMarkReviewed = currentItems
?.filter((seg) => seg.end_time)
?.map((seg) => seg.id);
if (itemsToMarkReviewed.length > 0) {
await axios.post(`reviews/viewed`, {
ids: currentItems?.map((seg) => seg.id),
ids: itemsToMarkReviewed,
});
reloadData();
}
},
[reloadData, updateSegments],
);

View File

@ -3,7 +3,7 @@ export interface ReviewSegment {
camera: string;
severity: ReviewSeverity;
start_time: number;
end_time: number;
end_time?: number;
thumb_path: string;
has_been_reviewed: boolean;
data: ReviewData;

View File

@ -43,6 +43,7 @@ import { TimeRange } from "@/types/timeline";
import { useCameraMotionNextTimestamp } from "@/hooks/use-camera-activity";
import useOptimisticState from "@/hooks/use-optimistic-state";
import { Skeleton } from "@/components/ui/skeleton";
import scrollIntoView from "scroll-into-view-if-needed";
type EventViewProps = {
reviews?: ReviewSegment[];
@ -293,6 +294,7 @@ export default function EventView({
severity={severity}
filter={filter}
timeRange={timeRange}
startTime={startTime}
markItemAsReviewed={markItemAsReviewed}
markAllItemsAsReviewed={markAllItemsAsReviewed}
onSelectReview={onSelectReview}
@ -331,6 +333,7 @@ type DetectionReviewProps = {
severity: ReviewSeverity;
filter?: ReviewFilter;
timeRange: { before: number; after: number };
startTime?: number;
markItemAsReviewed: (review: ReviewSegment) => void;
markAllItemsAsReviewed: (currentItems: ReviewSegment[]) => void;
onSelectReview: (review: ReviewSegment, ctrl: boolean) => void;
@ -345,6 +348,7 @@ function DetectionReview({
severity,
filter,
timeRange,
startTime,
markItemAsReviewed,
markAllItemsAsReviewed,
onSelectReview,
@ -495,6 +499,26 @@ function DetectionReview({
[minimap],
);
// existing review item
useEffect(() => {
if (!startTime || !currentItems || currentItems.length == 0) {
return;
}
const element = contentRef.current?.querySelector(
`[data-start="${startTime}"]`,
);
if (element) {
scrollIntoView(element, {
scrollMode: "if-needed",
behavior: "smooth",
});
}
// only run when start time changes
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [startTime]);
return (
<>
<div
@ -546,6 +570,7 @@ function DetectionReview({
<PreviewThumbnailPlayer
review={value}
allPreviews={relevantPreviews}
timeRange={timeRange}
setReviewed={markItemAsReviewed}
scrollLock={scrollLock}
onTimeUpdate={onPreviewTimeUpdate}
@ -787,16 +812,23 @@ function MotionReview({
} else {
const segmentStartTime = alignStartDateToTimeline(currentTime);
const segmentEndTime = segmentStartTime + segmentDuration;
const matchingItem = reviewItems?.all.find(
(item) =>
const matchingItem = reviewItems?.all.find((item) => {
const endTime = item.end_time ?? timeRange.before;
return (
((item.start_time >= segmentStartTime &&
item.start_time < segmentEndTime) ||
(item.end_time > segmentStartTime &&
item.end_time <= segmentEndTime) ||
(endTime > segmentStartTime && endTime <= segmentEndTime) ||
(item.start_time <= segmentStartTime &&
item.end_time >= segmentEndTime)) &&
item.camera === cameraName,
endTime >= segmentEndTime)) &&
item.camera === cameraName
);
item.start_time < segmentEndTime) ||
(endTime > segmentStartTime && endTime <= segmentEndTime) ||
(item.start_time <= segmentStartTime &&
endTime >= segmentEndTime)) &&
item.camera === cameraName;
});
return matchingItem ? matchingItem.severity : null;
}
@ -805,6 +837,7 @@ function MotionReview({
reviewItems,
motionData,
currentTime,
timeRange,
motionOnly,
alignStartDateToTimeline,
],

View File

@ -47,8 +47,8 @@ export default function LiveDashboardView({
}
// if event is ended and was saved, update events list
if (eventUpdate.type == "end" && eventUpdate.review.severity == "alert") {
setTimeout(() => updateEvents(), 1000);
if (eventUpdate.review.severity == "alert") {
setTimeout(() => updateEvents(), eventUpdate.type == "end" ? 1000 : 6000);
return;
}
}, [eventUpdate, updateEvents]);