Mirrors/blakeblackshear.frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2024-11-21 19:07:46 +01:00
Code Issues Projects Releases Wiki Activity

Label attributes (#6829)

Browse Source 
* pass attribute labels as attributes

* add label attrs to events and snapshots

* incorporate area of license_plate and face into snapshot selection

* populate sublabels for cars with logos
This commit is contained in:
Blake Blackshear 2023-06-17 09:56:22 -05:00 committed by GitHub
parent 2be2050d57
commit 793fe251b9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 177 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
docs/docs/configuration/advanced.md
View File

@ -96,6 +96,16 @@ model:
Note that if you rename objects in the labelmap, you will also need to update your `objects -> track` list as well.
:::caution
Some labels have special handling and modifications can disable functionality.
`person` objects are associated with `face` and `amazon`
`car` objects are associated with `license_plate`, `ups`, `fedex`, `amazon`
:::
## Custom ffmpeg build
Included with Frigate is a build of ffmpeg that works for the vast majority of users. However, there exists some hardware setups which have incompatibilities with the included build. In this case, a docker volume mapping can be used to overwrite the included ffmpeg build with an ffmpeg build that works for your specific hardware setup.

21
docs/docs/integrations/mqtt.md
View File

@ -62,7 +62,9 @@ Message published for each changed event. The first message is published when th
"has_clip": false,
"stationary": false, // whether or not the object is considered stationary
"motionless_count": 0, // number of frames the object has been motionless
"position_changes": 2 // number of times the object has moved from a stationary position
"position_changes": 2, // number of times the object has moved from a stationary position
"attributes": [], // set of unique attributes that have been identified on the object
"current_attributes": [] // detailed data about the current attributes in this frame
},
"after": {
"id": "1607123955.475377-mxklsc",
@ -87,7 +89,16 @@ Message published for each changed event. The first message is published when th
"has_clip": false,
"stationary": false, // whether or not the object is considered stationary
"motionless_count": 0, // number of frames the object has been motionless
"position_changes": 2 // number of times the object has changed position
"position_changes": 2, // number of times the object has changed position
"attributes": ["face"], // set of unique attributes that have been identified on the object
"current_attributes": [
// detailed data about the current attributes in this frame
{
"label": "face",
"box": [442, 506, 534, 524],
"score": 0.64
}
]
}
}
```
@ -163,9 +174,9 @@ Topic with current motion contour area for a camera. Published value is an integ
Topic to send PTZ commands to camera.
| Command | Description |
| ---------------------- | --------------------------------------------------------------------------------------- |
| Command | Description |
| ---------------------- | ----------------------------------------------------------------------------------------- |
| `preset-<preset_name>` | send command to move to preset with name `<preset_name>` |
| `MOVE_<dir>` | send command to continuously move in `<dir>`, possible values are [UP, DOWN, LEFT, RIGHT] |
| `ZOOM_<dir>` | send command to continuously zoom `<dir>`, possible values are [IN, OUT] |
| `STOP` | send command to stop moving |
| `STOP` | send command to stop moving |

22
frigate/events/maintainer.py
View File

@ -147,6 +147,23 @@ class EventProcessor(threading.Thread):
)
)
attributes = [
(
None
if event_data["snapshot"] is None
else {
"box": to_relative_box(
width,
height,
a["box"],
),
"label": a["label"],
"score": a["score"],
}
)
for a in event_data["snapshot"]["attributes"]
]
# keep these from being set back to false because the event
# may have started while recordings and snapshots were enabled
# this would be an issue for long running events
@ -173,9 +190,14 @@ class EventProcessor(threading.Thread):
"region": region,
"score": score,
"top_score": event_data["top_score"],
"attributes": attributes,
},
}
# only overwrite the sub_label in the database if it's set
if event_data.get("sub_label") is not None:
event[Event.sub_label] = event_data["sub_label"]
(
Event.insert(event)
.on_conflict(

90
frigate/object_processing.py
View File

@ -24,6 +24,7 @@ from frigate.const import CLIPS_DIR
from frigate.events.maintainer import EventTypeEnum
from frigate.util import (
SharedMemoryFrameManager,
area,
calculate_region,
draw_box_with_label,
draw_timestamp,
@ -42,11 +43,45 @@ def on_edge(box, frame_shape):
return True
def is_better_thumbnail(current_thumb, new_obj, frame_shape) -> bool:
def has_better_attr(current_thumb, new_obj, attr_label) -> bool:
max_new_attr = max(
[0]
+ [area(a["box"]) for a in new_obj["attributes"] if a["label"] == attr_label]
)
max_current_attr = max(
[0]
+ [
area(a["box"])
for a in current_thumb["attributes"]
if a["label"] == attr_label
]
)
# if the thumb has a higher scoring attr
return max_new_attr > max_current_attr
def is_better_thumbnail(label, current_thumb, new_obj, frame_shape) -> bool:
# larger is better
# cutoff images are less ideal, but they should also be smaller?
# better scores are obviously better too
# check face on person
if label == "person":
if has_better_attr(current_thumb, new_obj, "face"):
return True
# if the current thumb has a face attr, dont update unless it gets better
if any([a["label"] == "face" for a in current_thumb["attributes"]]):
return False
# check license_plate on car
if label == "car":
if has_better_attr(current_thumb, new_obj, "license_plate"):
return True
# if the current thumb has a license_plate attr, dont update unless it gets better
if any([a["label"] == "license_plate" for a in current_thumb["attributes"]]):
return False
# if the new_thumb is on an edge, and the current thumb is not
if on_edge(new_obj["box"], frame_shape) and not on_edge(
current_thumb["box"], frame_shape
@ -76,6 +111,7 @@ class TrackedObject:
self.zone_presence = {}
self.current_zones = []
self.entered_zones = []
self.attributes = set()
self.false_positive = True
self.has_clip = False
self.has_snapshot = False
@ -125,7 +161,10 @@ class TrackedObject:
if not self.false_positive:
# determine if this frame is a better thumbnail
if self.thumbnail_data is None or is_better_thumbnail(
self.thumbnail_data, obj_data, self.camera_config.frame_shape
self.obj_data["label"],
self.thumbnail_data,
obj_data,
self.camera_config.frame_shape,
):
self.thumbnail_data = {
"frame_time": obj_data["frame_time"],
@ -133,6 +172,7 @@ class TrackedObject:
"area": obj_data["area"],
"region": obj_data["region"],
"score": obj_data["score"],
"attributes": obj_data["attributes"],
}
thumb_update = True
@ -164,6 +204,19 @@ class TrackedObject:
if 0 < zone_score < zone.inertia:
self.zone_presence[name] = zone_score - 1
# maintain attributes
for attr in obj_data["attributes"]:
self.attributes.add(attr["label"])
# populate the sub_label for car with first logo if it exists
if self.obj_data["label"] == "car" and "sub_label" not in self.obj_data:
recognized_logos = self.attributes.intersection(
set(["ups", "fedex", "amazon"])
)
if len(recognized_logos) > 0:
self.obj_data["sub_label"] = recognized_logos.pop()
# check for significant change
if not self.false_positive:
# if the zones changed, signal an update
if set(self.current_zones) != set(current_zones):
@ -214,6 +267,8 @@ class TrackedObject:
"entered_zones": self.entered_zones.copy(),
"has_clip": self.has_clip,
"has_snapshot": self.has_snapshot,
"attributes": list(self.attributes),
"current_attributes": self.obj_data["attributes"],
}
if include_thumbnail:
@ -294,6 +349,21 @@ class TrackedObject:
color=color,
)
# draw any attributes
for attribute in self.thumbnail_data["attributes"]:
box = attribute["box"]
draw_box_with_label(
best_frame,
box[0],
box[1],
box[2],
box[3],
attribute["label"],
f"{attribute['score']:.0%}",
thickness=thickness,
color=color,
)
if crop:
box = self.thumbnail_data["box"]
box_size = 300
@ -421,6 +491,21 @@ class CameraState:
color=color,
)
# draw any attributes
for attribute in obj["current_attributes"]:
box = attribute["box"]
draw_box_with_label(
frame_copy,
box[0],
box[1],
box[2],
box[3],
attribute["label"],
f"{attribute['score']:.0%}",
thickness=thickness,
color=color,
)
if draw_options.get("regions"):
for region in regions:
cv2.rectangle(
@ -553,6 +638,7 @@ class CameraState:
# or the current object is older than desired, use the new object
if (
is_better_thumbnail(
object_type,
current_best.thumbnail_data,
obj.thumbnail_data,
self.camera_config.frame_shape,

43
frigate/video.py
View File

@ -723,6 +723,14 @@ def process_frames(
stop_event,
exit_on_empty: bool = False,
):
# attribute labels are not tracked and are not assigned regions
attribute_label_map = {
"person": ["face", "amazon"],
"car": ["ups", "fedex", "amazon", "license_plate"],
}
all_attribute_labels = [
item for sublist in attribute_label_map.values() for item in sublist
]
fps = process_info["process_fps"]
detection_fps = process_info["detection_fps"]
current_frame_time = process_info["detection_frame"]
@ -758,6 +766,7 @@ def process_frames(
motion_boxes = motion_detector.detect(frame) if motion_enabled.value else []
regions = []
consolidated_detections = []
# if detection is disabled
if not detection_enabled.value:
@ -894,12 +903,42 @@ def process_frames(
consolidated_detections = get_consolidated_object_detections(
detected_object_groups
)
tracked_detections = [
d
for d in consolidated_detections
if d[0] not in all_attribute_labels
]
# now that we have refined our detections, we need to track objects
object_tracker.match_and_update(frame_time, consolidated_detections)
object_tracker.match_and_update(frame_time, tracked_detections)
# else, just update the frame times for the stationary objects
else:
object_tracker.update_frame_times(frame_time)
# group the attribute detections based on what label they apply to
attribute_detections = {}
for label, attribute_labels in attribute_label_map.items():
attribute_detections[label] = [
d for d in consolidated_detections if d[0] in attribute_labels
]
# build detections and add attributes
detections = {}
for obj in object_tracker.tracked_objects.values():
attributes = []
# if the objects label has associated attribute detections
if obj["label"] in attribute_detections.keys():
# add them to attributes if they intersect
for attribute_detection in attribute_detections[obj["label"]]:
if box_inside(obj["box"], (attribute_detection[2])):
attributes.append(
{
"label": attribute_detection[0],
"score": attribute_detection[1],
"box": attribute_detection[2],
}
)
detections[obj["id"]] = {**obj, "attributes": attributes}
# debug object tracking
if False:
bgr_frame = cv2.cvtColor(
@ -982,7 +1021,7 @@ def process_frames(
(
camera_name,
frame_time,
object_tracker.tracked_objects,
detections,
motion_boxes,
regions,
)