diff --git a/docs/docs/configuration/advanced.md b/docs/docs/configuration/advanced.md index cbec076d4..af2d7872f 100644 --- a/docs/docs/configuration/advanced.md +++ b/docs/docs/configuration/advanced.md @@ -96,6 +96,16 @@ model: Note that if you rename objects in the labelmap, you will also need to update your `objects -> track` list as well. +:::caution + +Some labels have special handling and modifications can disable functionality. + +`person` objects are associated with `face` and `amazon` + +`car` objects are associated with `license_plate`, `ups`, `fedex`, `amazon` + +::: + ## Custom ffmpeg build Included with Frigate is a build of ffmpeg that works for the vast majority of users. However, there exists some hardware setups which have incompatibilities with the included build. In this case, a docker volume mapping can be used to overwrite the included ffmpeg build with an ffmpeg build that works for your specific hardware setup. diff --git a/docs/docs/integrations/mqtt.md b/docs/docs/integrations/mqtt.md index 814656258..cde760559 100644 --- a/docs/docs/integrations/mqtt.md +++ b/docs/docs/integrations/mqtt.md @@ -62,7 +62,9 @@ Message published for each changed event. The first message is published when th "has_clip": false, "stationary": false, // whether or not the object is considered stationary "motionless_count": 0, // number of frames the object has been motionless - "position_changes": 2 // number of times the object has moved from a stationary position + "position_changes": 2, // number of times the object has moved from a stationary position + "attributes": [], // set of unique attributes that have been identified on the object + "current_attributes": [] // detailed data about the current attributes in this frame }, "after": { "id": "1607123955.475377-mxklsc", @@ -87,7 +89,16 @@ Message published for each changed event. The first message is published when th "has_clip": false, "stationary": false, // whether or not the object is considered stationary "motionless_count": 0, // number of frames the object has been motionless - "position_changes": 2 // number of times the object has changed position + "position_changes": 2, // number of times the object has changed position + "attributes": ["face"], // set of unique attributes that have been identified on the object + "current_attributes": [ + // detailed data about the current attributes in this frame + { + "label": "face", + "box": [442, 506, 534, 524], + "score": 0.64 + } + ] } } ``` @@ -163,9 +174,9 @@ Topic with current motion contour area for a camera. Published value is an integ Topic to send PTZ commands to camera. -| Command | Description | -| ---------------------- | --------------------------------------------------------------------------------------- | +| Command | Description | +| ---------------------- | ----------------------------------------------------------------------------------------- | | `preset-` | send command to move to preset with name `` | | `MOVE_` | send command to continuously move in ``, possible values are [UP, DOWN, LEFT, RIGHT] | | `ZOOM_` | send command to continuously zoom ``, possible values are [IN, OUT] | -| `STOP` | send command to stop moving | +| `STOP` | send command to stop moving | diff --git a/frigate/events/maintainer.py b/frigate/events/maintainer.py index 8ae6aee07..264ab1142 100644 --- a/frigate/events/maintainer.py +++ b/frigate/events/maintainer.py @@ -147,6 +147,23 @@ class EventProcessor(threading.Thread): ) ) + attributes = [ + ( + None + if event_data["snapshot"] is None + else { + "box": to_relative_box( + width, + height, + a["box"], + ), + "label": a["label"], + "score": a["score"], + } + ) + for a in event_data["snapshot"]["attributes"] + ] + # keep these from being set back to false because the event # may have started while recordings and snapshots were enabled # this would be an issue for long running events @@ -173,9 +190,14 @@ class EventProcessor(threading.Thread): "region": region, "score": score, "top_score": event_data["top_score"], + "attributes": attributes, }, } + # only overwrite the sub_label in the database if it's set + if event_data.get("sub_label") is not None: + event[Event.sub_label] = event_data["sub_label"] + ( Event.insert(event) .on_conflict( diff --git a/frigate/object_processing.py b/frigate/object_processing.py index 6455f6eba..6d31c3cdd 100644 --- a/frigate/object_processing.py +++ b/frigate/object_processing.py @@ -24,6 +24,7 @@ from frigate.const import CLIPS_DIR from frigate.events.maintainer import EventTypeEnum from frigate.util import ( SharedMemoryFrameManager, + area, calculate_region, draw_box_with_label, draw_timestamp, @@ -42,11 +43,45 @@ def on_edge(box, frame_shape): return True -def is_better_thumbnail(current_thumb, new_obj, frame_shape) -> bool: +def has_better_attr(current_thumb, new_obj, attr_label) -> bool: + max_new_attr = max( + [0] + + [area(a["box"]) for a in new_obj["attributes"] if a["label"] == attr_label] + ) + max_current_attr = max( + [0] + + [ + area(a["box"]) + for a in current_thumb["attributes"] + if a["label"] == attr_label + ] + ) + + # if the thumb has a higher scoring attr + return max_new_attr > max_current_attr + + +def is_better_thumbnail(label, current_thumb, new_obj, frame_shape) -> bool: # larger is better # cutoff images are less ideal, but they should also be smaller? # better scores are obviously better too + # check face on person + if label == "person": + if has_better_attr(current_thumb, new_obj, "face"): + return True + # if the current thumb has a face attr, dont update unless it gets better + if any([a["label"] == "face" for a in current_thumb["attributes"]]): + return False + + # check license_plate on car + if label == "car": + if has_better_attr(current_thumb, new_obj, "license_plate"): + return True + # if the current thumb has a license_plate attr, dont update unless it gets better + if any([a["label"] == "license_plate" for a in current_thumb["attributes"]]): + return False + # if the new_thumb is on an edge, and the current thumb is not if on_edge(new_obj["box"], frame_shape) and not on_edge( current_thumb["box"], frame_shape @@ -76,6 +111,7 @@ class TrackedObject: self.zone_presence = {} self.current_zones = [] self.entered_zones = [] + self.attributes = set() self.false_positive = True self.has_clip = False self.has_snapshot = False @@ -125,7 +161,10 @@ class TrackedObject: if not self.false_positive: # determine if this frame is a better thumbnail if self.thumbnail_data is None or is_better_thumbnail( - self.thumbnail_data, obj_data, self.camera_config.frame_shape + self.obj_data["label"], + self.thumbnail_data, + obj_data, + self.camera_config.frame_shape, ): self.thumbnail_data = { "frame_time": obj_data["frame_time"], @@ -133,6 +172,7 @@ class TrackedObject: "area": obj_data["area"], "region": obj_data["region"], "score": obj_data["score"], + "attributes": obj_data["attributes"], } thumb_update = True @@ -164,6 +204,19 @@ class TrackedObject: if 0 < zone_score < zone.inertia: self.zone_presence[name] = zone_score - 1 + # maintain attributes + for attr in obj_data["attributes"]: + self.attributes.add(attr["label"]) + + # populate the sub_label for car with first logo if it exists + if self.obj_data["label"] == "car" and "sub_label" not in self.obj_data: + recognized_logos = self.attributes.intersection( + set(["ups", "fedex", "amazon"]) + ) + if len(recognized_logos) > 0: + self.obj_data["sub_label"] = recognized_logos.pop() + + # check for significant change if not self.false_positive: # if the zones changed, signal an update if set(self.current_zones) != set(current_zones): @@ -214,6 +267,8 @@ class TrackedObject: "entered_zones": self.entered_zones.copy(), "has_clip": self.has_clip, "has_snapshot": self.has_snapshot, + "attributes": list(self.attributes), + "current_attributes": self.obj_data["attributes"], } if include_thumbnail: @@ -294,6 +349,21 @@ class TrackedObject: color=color, ) + # draw any attributes + for attribute in self.thumbnail_data["attributes"]: + box = attribute["box"] + draw_box_with_label( + best_frame, + box[0], + box[1], + box[2], + box[3], + attribute["label"], + f"{attribute['score']:.0%}", + thickness=thickness, + color=color, + ) + if crop: box = self.thumbnail_data["box"] box_size = 300 @@ -421,6 +491,21 @@ class CameraState: color=color, ) + # draw any attributes + for attribute in obj["current_attributes"]: + box = attribute["box"] + draw_box_with_label( + frame_copy, + box[0], + box[1], + box[2], + box[3], + attribute["label"], + f"{attribute['score']:.0%}", + thickness=thickness, + color=color, + ) + if draw_options.get("regions"): for region in regions: cv2.rectangle( @@ -553,6 +638,7 @@ class CameraState: # or the current object is older than desired, use the new object if ( is_better_thumbnail( + object_type, current_best.thumbnail_data, obj.thumbnail_data, self.camera_config.frame_shape, diff --git a/frigate/video.py b/frigate/video.py index 25ae35664..c02ad15c4 100755 --- a/frigate/video.py +++ b/frigate/video.py @@ -723,6 +723,14 @@ def process_frames( stop_event, exit_on_empty: bool = False, ): + # attribute labels are not tracked and are not assigned regions + attribute_label_map = { + "person": ["face", "amazon"], + "car": ["ups", "fedex", "amazon", "license_plate"], + } + all_attribute_labels = [ + item for sublist in attribute_label_map.values() for item in sublist + ] fps = process_info["process_fps"] detection_fps = process_info["detection_fps"] current_frame_time = process_info["detection_frame"] @@ -758,6 +766,7 @@ def process_frames( motion_boxes = motion_detector.detect(frame) if motion_enabled.value else [] regions = [] + consolidated_detections = [] # if detection is disabled if not detection_enabled.value: @@ -894,12 +903,42 @@ def process_frames( consolidated_detections = get_consolidated_object_detections( detected_object_groups ) + tracked_detections = [ + d + for d in consolidated_detections + if d[0] not in all_attribute_labels + ] # now that we have refined our detections, we need to track objects - object_tracker.match_and_update(frame_time, consolidated_detections) + object_tracker.match_and_update(frame_time, tracked_detections) # else, just update the frame times for the stationary objects else: object_tracker.update_frame_times(frame_time) + # group the attribute detections based on what label they apply to + attribute_detections = {} + for label, attribute_labels in attribute_label_map.items(): + attribute_detections[label] = [ + d for d in consolidated_detections if d[0] in attribute_labels + ] + + # build detections and add attributes + detections = {} + for obj in object_tracker.tracked_objects.values(): + attributes = [] + # if the objects label has associated attribute detections + if obj["label"] in attribute_detections.keys(): + # add them to attributes if they intersect + for attribute_detection in attribute_detections[obj["label"]]: + if box_inside(obj["box"], (attribute_detection[2])): + attributes.append( + { + "label": attribute_detection[0], + "score": attribute_detection[1], + "box": attribute_detection[2], + } + ) + detections[obj["id"]] = {**obj, "attributes": attributes} + # debug object tracking if False: bgr_frame = cv2.cvtColor( @@ -982,7 +1021,7 @@ def process_frames( ( camera_name, frame_time, - object_tracker.tracked_objects, + detections, motion_boxes, regions, )