Mirrors/blakeblackshear.frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2025-09-05 17:51:36 +02:00
Code Issues Projects Releases Wiki Activity

Improve review segmentation behavior (#19850)

Browse Source 
* Refactor active objects to class

* Keep segment going when detection is newer than end of alert

* Cleanup logic

* Fix

* Cleanup ending

* Adjust timing

* Improve detection saving

* Don't have padding at end for in progress reviews

* Add review config for cutoff times
This commit is contained in:
Nicolas Mowen 2025-08-31 15:36:12 -06:00 committed by GitHub
parent a478c38f8a
commit 3a1e1d0841
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 242 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs/docs/configuration/reference.md
View File

@ -390,6 +390,8 @@ review:
labels:
- car
- person
# Time to cutoff alerts after no alert-causing activity has occurred (default: shown below)
cutoff_time: 40
# Optional: required zones for an object to be marked as an alert (default: none)
# NOTE: when settings required zones globally, this zone must exist on all cameras
# or the config will be considered invalid. In that case the required_zones
@ -404,6 +406,8 @@ review:
labels:
- car
- person
# Time to cutoff detections after no detection-causing activity has occurred (default: shown below)
cutoff_time: 30
# Optional: required zones for an object to be marked as a detection (default: none)
# NOTE: when settings required zones globally, this zone must exist on all cameras
# or the config will be considered invalid. In that case the required_zones

8
frigate/config/camera/review.py
View File

@ -26,6 +26,10 @@ class AlertsConfig(FrigateBaseModel):
enabled_in_config: Optional[bool] = Field(
default=None, title="Keep track of original state of alerts."
)
cutoff_time: int = Field(
default=40,
title="Time to cutoff alerts after no alert-causing activity has occurred.",
)
@field_validator("required_zones", mode="before")
@classmethod
@ -48,6 +52,10 @@ class DetectionsConfig(FrigateBaseModel):
default_factory=list,
title="List of required zones to be entered in order to save the event as a detection.",
)
cutoff_time: int = Field(
default=30,
title="Time to cutoff detection after no detection-causing activity has occurred.",
)
enabled_in_config: Optional[bool] = Field(
default=None, title="Keep track of original state of detections."

369
frigate/review/maintainer.py
View File

@ -40,9 +40,6 @@ logger = logging.getLogger(__name__)
THUMB_HEIGHT = 180
THUMB_WIDTH = 320
THRESHOLD_ALERT_ACTIVITY = 120
THRESHOLD_DETECTION_ACTIVITY = 30
class PendingReviewSegment:
def __init__(
@ -64,8 +61,12 @@ class PendingReviewSegment:
self.sub_labels = sub_labels
self.zones = zones
self.audio = audio
self.last_update = frame_time
self.thumb_time: float | None = None
self.last_alert_time: float | None = None
self.last_detection_time: float = frame_time
if severity == SeverityEnum.alert:
self.last_alert_time = frame_time
# thumbnail
self._frame = np.zeros((THUMB_HEIGHT * 3 // 2, THUMB_WIDTH), np.uint8)
@ -127,12 +128,20 @@ class PendingReviewSegment:
)
def get_data(self, ended: bool) -> dict:
end_time = None
if ended:
if self.severity == SeverityEnum.alert:
end_time = self.last_alert_time
else:
end_time = self.last_detection_time
return copy.deepcopy(
{
ReviewSegment.id.name: self.id,
ReviewSegment.camera.name: self.camera,
ReviewSegment.start_time.name: self.start_time,
ReviewSegment.end_time.name: self.last_update if ended else None,
ReviewSegment.end_time.name: end_time,
ReviewSegment.severity.name: self.severity.value,
ReviewSegment.thumb_path.name: self.frame_path,
ReviewSegment.data.name: {
@ -148,6 +157,102 @@ class PendingReviewSegment:
)
class ActiveObjects:
def __init__(
self,
frame_time: float,
camera_config: CameraConfig,
all_objects: list[TrackedObject],
):
self.camera_config = camera_config
# get current categorization of objects to know if
# these objects are currently being categorized
self.categorized_objects = {
"alerts": [],
"detections": [],
}
for o in all_objects:
if (
o["motionless_count"] >= camera_config.detect.stationary.threshold
and not o["pending_loitering"]
):
# no stationary objects unless loitering
continue
if o["position_changes"] == 0:
# object must have moved at least once
continue
if o["frame_time"] != frame_time:
# object must be detected in this frame
continue
if o["false_positive"]:
# object must not be a false positive
continue
if (
o["label"] in camera_config.review.alerts.labels
and (
not camera_config.review.alerts.required_zones
or (
len(o["current_zones"]) > 0
and set(o["current_zones"])
& set(camera_config.review.alerts.required_zones)
)
)
and camera_config.review.alerts.enabled
):
self.categorized_objects["alerts"].append(o)
continue
if (
(
camera_config.review.detections.labels is None
or o["label"] in camera_config.review.detections.labels
)
and (
not camera_config.review.detections.required_zones
or (
len(o["current_zones"]) > 0
and set(o["current_zones"])
& set(camera_config.review.detections.required_zones)
)
)
and camera_config.review.detections.enabled
):
self.categorized_objects["detections"].append(o)
continue
def has_active_objects(self) -> bool:
return (
len(self.categorized_objects["alerts"]) > 0
or len(self.categorized_objects["detections"]) > 0
)
def has_activity_category(self, severity: SeverityEnum) -> bool:
if (
severity == SeverityEnum.alert
and len(self.categorized_objects["alerts"]) > 0
):
return True
if (
severity == SeverityEnum.detection
and len(self.categorized_objects["detections"]) > 0
):
return True
return False
def get_all_objects(self) -> list[TrackedObject]:
return (
self.categorized_objects["alerts"] + self.categorized_objects["detections"]
)
class ReviewSegmentMaintainer(threading.Thread):
"""Maintain review segments."""
@ -238,9 +343,10 @@ class ReviewSegmentMaintainer(threading.Thread):
self,
segment: PendingReviewSegment,
prev_data: dict[str, Any],
) -> None:
) -> float:
"""End segment."""
final_data = segment.get_data(ended=True)
end_time = final_data[ReviewSegment.end_time.name]
self.requestor.send_data(UPSERT_REVIEW_SEGMENT, final_data)
review_update = {
"type": "end",
@ -254,13 +360,14 @@ class ReviewSegmentMaintainer(threading.Thread):
self.review_publisher.publish(review_update, segment.camera)
self.requestor.send_data(f"{segment.camera}/review_status", "NONE")
self.active_review_segments[segment.camera] = None
return end_time
def end_segment(self, camera: str) -> None:
"""End the pending segment for a camera."""
def forcibly_end_segment(self, camera: str) -> float:
"""Forcibly end the pending segment for a camera."""
segment = self.active_review_segments.get(camera)
if segment:
prev_data = segment.get_data(False)
self._publish_segment_end(segment, prev_data)
return self._publish_segment_end(segment, prev_data)
def update_existing_segment(
self,
@ -273,21 +380,30 @@ class ReviewSegmentMaintainer(threading.Thread):
camera_config = self.config.cameras[segment.camera]
# get active objects + objects loitering in loitering zones
active_objects = get_active_objects(
frame_time, camera_config, objects
) + get_loitering_objects(frame_time, camera_config, objects)
activity = ActiveObjects(frame_time, camera_config, objects)
prev_data = segment.get_data(False)
has_activity = False
if len(active_objects) > 0:
if activity.has_active_objects():
has_activity = True
should_update_image = False
should_update_state = False
if frame_time > segment.last_update:
segment.last_update = frame_time
if activity.has_activity_category(SeverityEnum.alert):
# update current time for last alert activity
segment.last_alert_time = frame_time
for object in active_objects:
if segment.severity != SeverityEnum.alert:
# if segment is not alert category but current activity is
# update this segment to be an alert
segment.severity = SeverityEnum.alert
should_update_state = True
should_update_image = True
if activity.has_activity_category(SeverityEnum.detection):
segment.last_detection_time = frame_time
for object in activity.get_all_objects():
if not object["sub_label"]:
segment.detections[object["id"]] = object["label"]
elif object["sub_label"][0] in self.config.model.all_attributes:
@ -296,33 +412,13 @@ class ReviewSegmentMaintainer(threading.Thread):
segment.detections[object["id"]] = f"{object['label']}-verified"
segment.sub_labels[object["id"]] = object["sub_label"][0]
# if object is alert label
# and has entered required zones or required zones is not set
# mark this review as alert
if (
segment.severity != SeverityEnum.alert
and object["label"] in camera_config.review.alerts.labels
and (
not camera_config.review.alerts.required_zones
or (
len(object["current_zones"]) > 0
and set(object["current_zones"])
& set(camera_config.review.alerts.required_zones)
)
)
and camera_config.review.alerts.enabled
):
segment.severity = SeverityEnum.alert
should_update_state = True
should_update_image = True
# keep zones up to date
if len(object["current_zones"]) > 0:
for zone in object["current_zones"]:
if zone not in segment.zones:
segment.zones.append(zone)
if len(active_objects) > segment.frame_active_count:
if len(activity.get_all_objects()) > segment.frame_active_count:
should_update_state = True
should_update_image = True
@ -343,7 +439,11 @@ class ReviewSegmentMaintainer(threading.Thread):
yuv_frame = None
self._publish_segment_update(
segment, camera_config, yuv_frame, active_objects, prev_data
segment,
camera_config,
yuv_frame,
activity.get_all_objects(),
prev_data,
)
self.frame_manager.close(frame_name)
except FileNotFoundError:
@ -369,10 +469,50 @@ class ReviewSegmentMaintainer(threading.Thread):
return
if segment.severity == SeverityEnum.alert and frame_time > (
segment.last_update + THRESHOLD_ALERT_ACTIVITY
segment.last_alert_time + camera_config.review.alerts.cutoff_time
):
needs_new_detection = (
segment.last_detection_time > segment.last_alert_time
and (
segment.last_detection_time
+ camera_config.review.detections.cutoff_time
)
> frame_time
)
last_detection_time = segment.last_detection_time
end_time = self._publish_segment_end(segment, prev_data)
if needs_new_detection:
new_detections: dict[str, str] = {}
new_zones = set()
for o in activity.categorized_objects["detections"]:
new_detections[o["id"]] = o["label"]
new_zones.update(o["current_zones"])
if new_detections:
self.active_review_segments[activity.camera_config.name] = (
PendingReviewSegment(
activity.camera_config.name,
end_time,
SeverityEnum.detection,
new_detections,
sub_labels={},
audio=set(),
zones=list(new_zones),
)
)
self._publish_segment_start(
self.active_review_segments[activity.camera_config.name]
)
self.active_review_segments[
activity.camera_config.name
].last_detection_time = last_detection_time
elif segment.severity == SeverityEnum.detection and frame_time > (
segment.last_detection_time
+ camera_config.review.detections.cutoff_time
):
self._publish_segment_end(segment, prev_data)
elif frame_time > (segment.last_update + THRESHOLD_DETECTION_ACTIVITY):
self._publish_segment_end(segment, prev_data)
def check_if_new_segment(
@ -384,15 +524,26 @@ class ReviewSegmentMaintainer(threading.Thread):
) -> None:
"""Check if a new review segment should be created."""
camera_config = self.config.cameras[camera]
active_objects = get_active_objects(frame_time, camera_config, objects)
activity = ActiveObjects(frame_time, camera_config, objects)
if len(active_objects) > 0:
if activity.has_active_objects():
detections: dict[str, str] = {}
sub_labels: dict[str, str] = {}
zones: list[str] = []
severity = None
severity: SeverityEnum | None = None
for object in active_objects:
# if activity is alert category mark this review as alert
if severity != SeverityEnum.alert and activity.has_activity_category(
SeverityEnum.alert
):
severity = SeverityEnum.alert
# if object is detection label and not already higher severity
# mark this review as detection
if not severity and activity.has_activity_category(SeverityEnum.detection):
severity = SeverityEnum.detection
for object in activity.get_all_objects():
if not object["sub_label"]:
detections[object["id"]] = object["label"]
elif object["sub_label"][0] in self.config.model.all_attributes:
@ -401,46 +552,6 @@ class ReviewSegmentMaintainer(threading.Thread):
detections[object["id"]] = f"{object['label']}-verified"
sub_labels[object["id"]] = object["sub_label"][0]
# if object is alert label
# and has entered required zones or required zones is not set
# mark this review as alert
if (
severity != SeverityEnum.alert
and object["label"] in camera_config.review.alerts.labels
and (
not camera_config.review.alerts.required_zones
or (
len(object["current_zones"]) > 0
and set(object["current_zones"])
& set(camera_config.review.alerts.required_zones)
)
)
and camera_config.review.alerts.enabled
):
severity = SeverityEnum.alert
# if object is detection label
# and review is not already a detection or alert
# and has entered required zones or required zones is not set
# mark this review as detection
if (
not severity
and (
camera_config.review.detections.labels is None
or object["label"] in (camera_config.review.detections.labels)
)
and (
not camera_config.review.detections.required_zones
or (
len(object["current_zones"]) > 0
and set(object["current_zones"])
& set(camera_config.review.detections.required_zones)
)
)
and camera_config.review.detections.enabled
):
severity = SeverityEnum.detection
for zone in object["current_zones"]:
if zone not in zones:
zones.append(zone)
@ -466,7 +577,7 @@ class ReviewSegmentMaintainer(threading.Thread):
return
self.active_review_segments[camera].update_frame(
camera_config, yuv_frame, active_objects
camera_config, yuv_frame, activity.get_all_objects()
)
self.frame_manager.close(frame_name)
self._publish_segment_start(self.active_review_segments[camera])
@ -480,11 +591,11 @@ class ReviewSegmentMaintainer(threading.Thread):
if "record" in updated_topics:
for camera in updated_topics["record"]:
self.end_segment(camera)
self.forcibly_end_segment(camera)
if "enabled" in updated_topics:
for camera in updated_topics["enabled"]:
self.end_segment(camera)
self.forcibly_end_segment(camera)
(topic, data) = self.detection_subscriber.check_for_update(timeout=1)
@ -534,7 +645,7 @@ class ReviewSegmentMaintainer(threading.Thread):
current_segment.severity == SeverityEnum.detection
and not self.config.cameras[camera].review.detections.enabled
):
self.end_segment(camera)
self.forcibly_end_segment(camera)
continue
# If we reach here, the segment can be processed (if it exists)
@ -549,9 +660,6 @@ class ReviewSegmentMaintainer(threading.Thread):
elif topic == DetectionTypeEnum.audio and len(audio_detections) > 0:
camera_config = self.config.cameras[camera]
if frame_time > current_segment.last_update:
current_segment.last_update = frame_time
for audio in audio_detections:
if (
audio in camera_config.review.alerts.labels
@ -559,11 +667,13 @@ class ReviewSegmentMaintainer(threading.Thread):
):
current_segment.audio.add(audio)
current_segment.severity = SeverityEnum.alert
current_segment.last_alert_time = frame_time
elif (
camera_config.review.detections.labels is None
or audio in camera_config.review.detections.labels
) and camera_config.review.detections.enabled:
current_segment.audio.add(audio)
current_segment.last_detection_time = frame_time
elif topic == DetectionTypeEnum.api or topic == DetectionTypeEnum.lpr:
if manual_info["state"] == ManualEventState.complete:
current_segment.detections[manual_info["event_id"]] = (
@ -579,7 +689,7 @@ class ReviewSegmentMaintainer(threading.Thread):
and self.config.cameras[camera].review.detections.enabled
):
current_segment.severity = SeverityEnum.detection
current_segment.last_update = manual_info["end_time"]
current_segment.last_alert_time = manual_info["end_time"]
elif manual_info["state"] == ManualEventState.start:
self.indefinite_events[camera][manual_info["event_id"]] = (
manual_info["label"]
@ -599,7 +709,8 @@ class ReviewSegmentMaintainer(threading.Thread):
current_segment.severity = SeverityEnum.detection
# temporarily make it so this event can not end
current_segment.last_update = sys.maxsize
current_segment.last_alert_time = sys.maxsize
current_segment.last_detection_time = sys.maxsize
elif manual_info["state"] == ManualEventState.end:
event_id = manual_info["event_id"]
@ -607,7 +718,12 @@ class ReviewSegmentMaintainer(threading.Thread):
self.indefinite_events[camera].pop(event_id)
if len(self.indefinite_events[camera]) == 0:
current_segment.last_update = manual_info["end_time"]
current_segment.last_alert_time = manual_info[
"end_time"
]
current_segment.last_detection_time = manual_info[
"end_time"
]
else:
logger.error(
f"Event with ID {event_id} has a set duration and can not be ended manually."
@ -675,11 +791,17 @@ class ReviewSegmentMaintainer(threading.Thread):
# temporarily make it so this event can not end
self.active_review_segments[
camera
].last_update = sys.maxsize
].last_alert_time = sys.maxsize
self.active_review_segments[
camera
].last_detection_time = sys.maxsize
elif manual_info["state"] == ManualEventState.complete:
self.active_review_segments[
camera
].last_update = manual_info["end_time"]
].last_alert_time = manual_info["end_time"]
self.active_review_segments[
camera
].last_detection_time = manual_info["end_time"]
else:
logger.warning(
f"Manual event API has been called for {camera}, but alerts are disabled. This manual event will not appear as an alert."
@ -703,11 +825,17 @@ class ReviewSegmentMaintainer(threading.Thread):
# temporarily make it so this event can not end
self.active_review_segments[
camera
].last_update = sys.maxsize
].last_alert_time = sys.maxsize
self.active_review_segments[
camera
].last_detection_time = sys.maxsize
elif manual_info["state"] == ManualEventState.complete:
self.active_review_segments[
camera
].last_update = manual_info["end_time"]
].last_alert_time = manual_info["end_time"]
self.active_review_segments[
camera
].last_detection_time = manual_info["end_time"]
else:
logger.warning(
f"Dedicated LPR camera API has been called for {camera}, but detections are disabled. LPR events will not appear as a detection."
@ -717,46 +845,3 @@ class ReviewSegmentMaintainer(threading.Thread):
self.requestor.stop()
self.detection_subscriber.stop()
logger.info("Exiting review maintainer...")
def get_active_objects(
frame_time: float, camera_config: CameraConfig, all_objects: list[TrackedObject]
) -> list[TrackedObject]:
"""get active objects for detection."""
return [
o
for o in all_objects
if o["motionless_count"]
< camera_config.detect.stationary.threshold # no stationary objects
and o["position_changes"] > 0 # object must have moved at least once
and o["frame_time"] == frame_time # object must be detected in this frame
and not o["false_positive"] # object must not be a false positive
and (
o["label"] in camera_config.review.alerts.labels
or (
camera_config.review.detections.labels is None
or o["label"] in camera_config.review.detections.labels
)
) # object must be in the alerts or detections label list
]
def get_loitering_objects(
frame_time: float, camera_config: CameraConfig, all_objects: list[TrackedObject]
) -> list[TrackedObject]:
"""get loitering objects for detection."""
return [
o
for o in all_objects
if o["pending_loitering"] # object must be pending loitering
and o["position_changes"] > 0 # object must have moved at least once
and o["frame_time"] == frame_time # object must be detected in this frame
and not o["false_positive"] # object must not be a false positive
and (
o["label"] in camera_config.review.alerts.labels
or (
camera_config.review.detections.labels is None
or o["label"] in camera_config.review.detections.labels
)
) # object must be in the alerts or detections label list
]

6
web/src/components/preview/ScrubbablePreview.tsx
View File

@ -315,9 +315,9 @@ export function InProgressPreview({
const apiHost = useApiHost();
const sliderRef = useRef<HTMLDivElement | null>(null);
const { data: previewFrames } = useSWR<string[]>(
`preview/${camera}/start/${Math.floor(startTime) - PREVIEW_PADDING}/end/${
Math.ceil(endTime ?? timeRange.before) + PREVIEW_PADDING
}/frames`,
`preview/${camera}/start/${Math.floor(startTime) - PREVIEW_PADDING}/end/${Math.ceil(
endTime ?? timeRange.before,
)}/frames`,
{ revalidateOnFocus: false },
);