mirror of
https://github.com/blakeblackshear/frigate.git
synced 2024-10-15 20:07:23 +02:00
528 lines
23 KiB
Python
528 lines
23 KiB
Python
import time
|
|
import datetime
|
|
import threading
|
|
import cv2
|
|
import prctl
|
|
import itertools
|
|
import numpy as np
|
|
from scipy.spatial import distance as dist
|
|
from frigate.util import draw_box_with_label, LABELS, compute_intersection_rectangle, compute_intersection_over_union, calculate_region
|
|
|
|
class ObjectCleaner(threading.Thread):
|
|
def __init__(self, objects_parsed, detected_objects):
|
|
threading.Thread.__init__(self)
|
|
self._objects_parsed = objects_parsed
|
|
self._detected_objects = detected_objects
|
|
|
|
def run(self):
|
|
prctl.set_name("ObjectCleaner")
|
|
while True:
|
|
|
|
# wait a bit before checking for expired frames
|
|
time.sleep(0.2)
|
|
|
|
# expire the objects that are more than 1 second old
|
|
now = datetime.datetime.now().timestamp()
|
|
# look for the first object found within the last second
|
|
# (newest objects are appended to the end)
|
|
detected_objects = self._detected_objects.copy()
|
|
|
|
objects_removed = False
|
|
for frame_time in detected_objects.keys():
|
|
if now-frame_time>2:
|
|
del self._detected_objects[frame_time]
|
|
objects_removed = True
|
|
|
|
if objects_removed:
|
|
# notify that parsed objects were changed
|
|
with self._objects_parsed:
|
|
self._objects_parsed.notify_all()
|
|
|
|
class DetectedObjectsProcessor(threading.Thread):
|
|
def __init__(self, camera):
|
|
threading.Thread.__init__(self)
|
|
self.camera = camera
|
|
|
|
def run(self):
|
|
prctl.set_name(self.__class__.__name__)
|
|
while True:
|
|
frame = self.camera.detected_objects_queue.get()
|
|
|
|
objects = frame['detected_objects']
|
|
|
|
# print(f"Processing objects for: {frame['size']} {frame['x_offset']} {frame['y_offset']}")
|
|
|
|
# if len(objects) == 0:
|
|
# continue
|
|
|
|
for raw_obj in objects:
|
|
obj = {
|
|
'name': str(LABELS[raw_obj.label_id]),
|
|
'score': float(raw_obj.score),
|
|
'box': {
|
|
'xmin': int((raw_obj.bounding_box[0][0] * frame['size']) + frame['x_offset']),
|
|
'ymin': int((raw_obj.bounding_box[0][1] * frame['size']) + frame['y_offset']),
|
|
'xmax': int((raw_obj.bounding_box[1][0] * frame['size']) + frame['x_offset']),
|
|
'ymax': int((raw_obj.bounding_box[1][1] * frame['size']) + frame['y_offset'])
|
|
},
|
|
'region': {
|
|
'xmin': frame['x_offset'],
|
|
'ymin': frame['y_offset'],
|
|
'xmax': frame['x_offset']+frame['size'],
|
|
'ymax': frame['y_offset']+frame['size']
|
|
},
|
|
'frame_time': frame['frame_time'],
|
|
'region_id': frame['region_id']
|
|
}
|
|
|
|
if not obj['name'] == 'bicycle':
|
|
continue
|
|
|
|
# if the object is within 5 pixels of the region border, and the region is not on the edge
|
|
# consider the object to be clipped
|
|
obj['clipped'] = False
|
|
if ((obj['region']['xmin'] > 5 and obj['box']['xmin']-obj['region']['xmin'] <= 5) or
|
|
(obj['region']['ymin'] > 5 and obj['box']['ymin']-obj['region']['ymin'] <= 5) or
|
|
(self.camera.frame_shape[1]-obj['region']['xmax'] > 5 and obj['region']['xmax']-obj['box']['xmax'] <= 5) or
|
|
(self.camera.frame_shape[0]-obj['region']['ymax'] > 5 and obj['region']['ymax']-obj['box']['ymax'] <= 5)):
|
|
obj['clipped'] = True
|
|
|
|
# Compute the area
|
|
obj['area'] = (obj['box']['xmax']-obj['box']['xmin'])*(obj['box']['ymax']-obj['box']['ymin'])
|
|
|
|
self.camera.detected_objects[frame['frame_time']].append(obj)
|
|
|
|
with self.camera.regions_in_process_lock:
|
|
self.camera.regions_in_process[frame['frame_time']] -= 1
|
|
# print(f"{frame['frame_time']} remaining regions {self.camera.regions_in_process[frame['frame_time']]}")
|
|
|
|
if self.camera.regions_in_process[frame['frame_time']] == 0:
|
|
del self.camera.regions_in_process[frame['frame_time']]
|
|
# print(f"{frame['frame_time']} no remaining regions")
|
|
self.camera.finished_frame_queue.put(frame['frame_time'])
|
|
|
|
# Thread that checks finished frames for clipped objects and sends back
|
|
# for processing if needed
|
|
class RegionRefiner(threading.Thread):
|
|
def __init__(self, camera):
|
|
threading.Thread.__init__(self)
|
|
self.camera = camera
|
|
|
|
def run(self):
|
|
prctl.set_name(self.__class__.__name__)
|
|
while True:
|
|
frame_time = self.camera.finished_frame_queue.get()
|
|
|
|
detected_objects = self.camera.detected_objects[frame_time].copy()
|
|
# print(f"{frame_time} finished")
|
|
|
|
# apply non-maxima suppression to suppress weak, overlapping bounding boxes
|
|
boxes = [(o['box']['xmin'], o['box']['ymin'], o['box']['xmax']-o['box']['xmin'], o['box']['ymax']-o['box']['ymin'])
|
|
for o in detected_objects]
|
|
confidences = [o['score'] for o in detected_objects]
|
|
idxs = cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.4)
|
|
|
|
# print(f"{frame_time} - NMS reduced objects from {len(detected_objects)} to {len(idxs)}")
|
|
|
|
look_again = False
|
|
# get selected objects
|
|
selected_objects = []
|
|
for index in idxs:
|
|
obj = detected_objects[index[0]]
|
|
selected_objects.append(obj)
|
|
if obj['clipped']:
|
|
box = obj['box']
|
|
# calculate a new region that will hopefully get the entire object
|
|
(size, x_offset, y_offset) = calculate_region(self.camera.frame_shape,
|
|
box['xmin'], box['ymin'],
|
|
box['xmax'], box['ymax'])
|
|
# print(f"{frame_time} new region: {size} {x_offset} {y_offset}")
|
|
|
|
with self.camera.regions_in_process_lock:
|
|
if not frame_time in self.camera.regions_in_process:
|
|
self.camera.regions_in_process[frame_time] = 1
|
|
else:
|
|
self.camera.regions_in_process[frame_time] += 1
|
|
|
|
# add it to the queue
|
|
self.camera.resize_queue.put({
|
|
'camera_name': self.camera.name,
|
|
'frame_time': frame_time,
|
|
'region_id': -1,
|
|
'size': size,
|
|
'x_offset': x_offset,
|
|
'y_offset': y_offset
|
|
})
|
|
self.camera.dynamic_region_fps.update()
|
|
look_again = True
|
|
# TODO: zoom in on unclipped low confidence objects
|
|
# else: ...
|
|
|
|
# if we are looking again, then this frame is not ready for processing
|
|
if look_again:
|
|
# remove the clipped objects
|
|
self.camera.detected_objects[frame_time] = [o for o in selected_objects if not o['clipped']]
|
|
continue
|
|
|
|
# filter objects based on camera settings
|
|
selected_objects = [o for o in selected_objects if not self.filtered(o)]
|
|
|
|
self.camera.detected_objects[frame_time] = selected_objects
|
|
|
|
with self.camera.objects_parsed:
|
|
self.camera.objects_parsed.notify_all()
|
|
|
|
# print(f"{frame_time} is actually finished")
|
|
|
|
# keep adding frames to the refined queue as long as they are finished
|
|
with self.camera.regions_in_process_lock:
|
|
while self.camera.frame_queue.qsize() > 0 and self.camera.frame_queue.queue[0] not in self.camera.regions_in_process:
|
|
self.camera.last_processed_frame = self.camera.frame_queue.get()
|
|
self.camera.refined_frame_queue.put(self.camera.last_processed_frame)
|
|
|
|
def filtered(self, obj):
|
|
object_name = obj['name']
|
|
|
|
if object_name in self.camera.object_filters:
|
|
obj_settings = self.camera.object_filters[object_name]
|
|
|
|
# if the min area is larger than the
|
|
# detected object, don't add it to detected objects
|
|
if obj_settings.get('min_area',-1) > obj['area']:
|
|
return True
|
|
|
|
# if the detected object is larger than the
|
|
# max area, don't add it to detected objects
|
|
if obj_settings.get('max_area', self.camera.frame_shape[0]*self.camera.frame_shape[1]) < obj['area']:
|
|
return True
|
|
|
|
# if the score is lower than the threshold, skip
|
|
if obj_settings.get('threshold', 0) > obj['score']:
|
|
return True
|
|
|
|
# compute the coordinates of the object and make sure
|
|
# the location isnt outside the bounds of the image (can happen from rounding)
|
|
y_location = min(int(obj['ymax']), len(self.camera.mask)-1)
|
|
x_location = min(int((obj['xmax']-obj['xmin'])/2.0)+obj['xmin'], len(self.camera.mask[0])-1)
|
|
|
|
# if the object is in a masked location, don't add it to detected objects
|
|
if self.camera.mask[y_location][x_location] == [0]:
|
|
return True
|
|
|
|
return False
|
|
|
|
def has_overlap(self, new_obj, obj, overlap=.7):
|
|
# compute intersection rectangle with existing object and new objects region
|
|
existing_obj_current_region = compute_intersection_rectangle(obj['box'], new_obj['region'])
|
|
|
|
# compute intersection rectangle with new object and existing objects region
|
|
new_obj_existing_region = compute_intersection_rectangle(new_obj['box'], obj['region'])
|
|
|
|
# compute iou for the two intersection rectangles that were just computed
|
|
iou = compute_intersection_over_union(existing_obj_current_region, new_obj_existing_region)
|
|
|
|
# if intersection is greater than overlap
|
|
if iou > overlap:
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
def find_group(self, new_obj, groups):
|
|
for index, group in enumerate(groups):
|
|
for obj in group:
|
|
if self.has_overlap(new_obj, obj):
|
|
return index
|
|
return None
|
|
|
|
class ObjectTracker(threading.Thread):
|
|
def __init__(self, camera, max_disappeared):
|
|
threading.Thread.__init__(self)
|
|
self.camera = camera
|
|
self.tracked_objects = {}
|
|
self.disappeared = {}
|
|
self.max_disappeared = max_disappeared
|
|
|
|
def run(self):
|
|
prctl.set_name(self.__class__.__name__)
|
|
while True:
|
|
# TODO: track objects
|
|
frame_time = self.camera.refined_frame_queue.get()
|
|
# f = open(f"/debug/{str(frame_time)}.jpg", 'wb')
|
|
# f.write(self.camera.frame_with_objects(frame_time))
|
|
# f.close()
|
|
|
|
|
|
def register(self, index, obj):
|
|
id = f"{str(obj.frame_time)}-{index}"
|
|
self.tracked_objects[id] = obj
|
|
self.disappeared[id] = 0
|
|
|
|
def deregister(self, id):
|
|
del self.disappeared[id]
|
|
del self.tracked_objects[id]
|
|
|
|
def update(self, id, new_obj):
|
|
new_obj.detections = self.tracked_objects[id].detections
|
|
new_obj.detections.append({
|
|
|
|
})
|
|
|
|
def match_and_update(self, new_objects):
|
|
# check to see if the list of input bounding box rectangles
|
|
# is empty
|
|
if len(new_objects) == 0:
|
|
# loop over any existing tracked objects and mark them
|
|
# as disappeared
|
|
for objectID in list(self.disappeared.keys()):
|
|
self.disappeared[objectID] += 1
|
|
|
|
# if we have reached a maximum number of consecutive
|
|
# frames where a given object has been marked as
|
|
# missing, deregister it
|
|
if self.disappeared[objectID] > self.max_disappeared:
|
|
self.deregister(objectID)
|
|
|
|
# return early as there are no centroids or tracking info
|
|
# to update
|
|
return
|
|
|
|
# compute centroids
|
|
for obj in new_objects:
|
|
centroid_x = int((obj['box']['xmin']+obj['box']['xmax']) / 2.0)
|
|
centroid_y = int((obj['box']['ymin']+obj['box']['ymax']) / 2.0)
|
|
obj.centroid = (centroid_x, centroid_y)
|
|
|
|
if len(self.tracked_objects) == 0:
|
|
for index, obj in enumerate(new_objects):
|
|
self.register(index, obj)
|
|
return
|
|
|
|
new_centroids = np.array([o.centroid for o in new_objects])
|
|
current_ids = list(self.tracked_objects.keys())
|
|
current_centroids = np.array([o.centroid for o in self.tracked_objects])
|
|
|
|
# compute the distance between each pair of tracked
|
|
# centroids and new centroids, respectively -- our
|
|
# goal will be to match each new centroid to an existing
|
|
# object centroid
|
|
D = dist.cdist(current_centroids, new_centroids)
|
|
|
|
# in order to perform this matching we must (1) find the
|
|
# smallest value in each row and then (2) sort the row
|
|
# indexes based on their minimum values so that the row
|
|
# with the smallest value is at the *front* of the index
|
|
# list
|
|
rows = D.min(axis=1).argsort()
|
|
|
|
# next, we perform a similar process on the columns by
|
|
# finding the smallest value in each column and then
|
|
# sorting using the previously computed row index list
|
|
cols = D.argmin(axis=1)[rows]
|
|
|
|
# in order to determine if we need to update, register,
|
|
# or deregister an object we need to keep track of which
|
|
# of the rows and column indexes we have already examined
|
|
usedRows = set()
|
|
usedCols = set()
|
|
|
|
# loop over the combination of the (row, column) index
|
|
# tuples
|
|
for (row, col) in zip(rows, cols):
|
|
# if we have already examined either the row or
|
|
# column value before, ignore it
|
|
# val
|
|
if row in usedRows or col in usedCols:
|
|
continue
|
|
|
|
# otherwise, grab the object ID for the current row,
|
|
# set its new centroid, and reset the disappeared
|
|
# counter
|
|
objectID = current_ids[row]
|
|
self.update(objectID, new_objects[col])
|
|
self.disappeared[objectID] = 0
|
|
|
|
# indicate that we have examined each of the row and
|
|
# column indexes, respectively
|
|
usedRows.add(row)
|
|
usedCols.add(col)
|
|
|
|
# compute both the row and column index we have NOT yet
|
|
# examined
|
|
unusedRows = set(range(0, D.shape[0])).difference(usedRows)
|
|
unusedCols = set(range(0, D.shape[1])).difference(usedCols)
|
|
|
|
# in the event that the number of object centroids is
|
|
# equal or greater than the number of input centroids
|
|
# we need to check and see if some of these objects have
|
|
# potentially disappeared
|
|
if D.shape[0] >= D.shape[1]:
|
|
# loop over the unused row indexes
|
|
for row in unusedRows:
|
|
# grab the object ID for the corresponding row
|
|
# index and increment the disappeared counter
|
|
objectID = current_ids[row]
|
|
self.disappeared[objectID] += 1
|
|
|
|
# check to see if the number of consecutive
|
|
# frames the object has been marked "disappeared"
|
|
# for warrants deregistering the object
|
|
if self.disappeared[objectID] > self.max_disappeared:
|
|
self.deregister(objectID)
|
|
|
|
# otherwise, if the number of input centroids is greater
|
|
# than the number of existing object centroids we need to
|
|
# register each new input centroid as a trackable object
|
|
else:
|
|
for col in unusedCols:
|
|
self.register(col, new_objects[col])
|
|
|
|
|
|
# -------------
|
|
|
|
# # initialize an array of input centroids for the current frame
|
|
# inputCentroids = np.zeros((len(rects), 2), dtype="int")
|
|
|
|
# # loop over the bounding box rectangles
|
|
# for (i, (startX, startY, endX, endY)) in enumerate(rects):
|
|
# # use the bounding box coordinates to derive the centroid
|
|
# cX = int((startX + endX) / 2.0)
|
|
# cY = int((startY + endY) / 2.0)
|
|
# inputCentroids[i] = (cX, cY)
|
|
|
|
# # if we are currently not tracking any objects take the input
|
|
# # centroids and register each of them
|
|
# if len(self.objects) == 0:
|
|
# for i in range(0, len(inputCentroids)):
|
|
# self.register(inputCentroids[i])
|
|
# # otherwise, are are currently tracking objects so we need to
|
|
# # try to match the input centroids to existing object
|
|
# # centroids
|
|
# else:
|
|
# # grab the set of object IDs and corresponding centroids
|
|
# objectIDs = list(self.objects.keys())
|
|
# objectCentroids = list(self.objects.values())
|
|
|
|
# # compute the distance between each pair of object
|
|
# # centroids and input centroids, respectively -- our
|
|
# # goal will be to match an input centroid to an existing
|
|
# # object centroid
|
|
# D = dist.cdist(np.array(objectCentroids), inputCentroids)
|
|
|
|
# # in order to perform this matching we must (1) find the
|
|
# # smallest value in each row and then (2) sort the row
|
|
# # indexes based on their minimum values so that the row
|
|
# # with the smallest value is at the *front* of the index
|
|
# # list
|
|
# rows = D.min(axis=1).argsort()
|
|
|
|
# # next, we perform a similar process on the columns by
|
|
# # finding the smallest value in each column and then
|
|
# # sorting using the previously computed row index list
|
|
# cols = D.argmin(axis=1)[rows]
|
|
|
|
# # in order to determine if we need to update, register,
|
|
# # or deregister an object we need to keep track of which
|
|
# # of the rows and column indexes we have already examined
|
|
# usedRows = set()
|
|
# usedCols = set()
|
|
|
|
# # loop over the combination of the (row, column) index
|
|
# # tuples
|
|
# for (row, col) in zip(rows, cols):
|
|
# # if we have already examined either the row or
|
|
# # column value before, ignore it
|
|
# # val
|
|
# if row in usedRows or col in usedCols:
|
|
# continue
|
|
|
|
# # otherwise, grab the object ID for the current row,
|
|
# # set its new centroid, and reset the disappeared
|
|
# # counter
|
|
# objectID = objectIDs[row]
|
|
# self.objects[objectID] = inputCentroids[col]
|
|
# self.disappeared[objectID] = 0
|
|
|
|
# # indicate that we have examined each of the row and
|
|
# # column indexes, respectively
|
|
# usedRows.add(row)
|
|
# usedCols.add(col)
|
|
|
|
# # compute both the row and column index we have NOT yet
|
|
# # examined
|
|
# unusedRows = set(range(0, D.shape[0])).difference(usedRows)
|
|
# unusedCols = set(range(0, D.shape[1])).difference(usedCols)
|
|
|
|
# # in the event that the number of object centroids is
|
|
# # equal or greater than the number of input centroids
|
|
# # we need to check and see if some of these objects have
|
|
# # potentially disappeared
|
|
# if D.shape[0] >= D.shape[1]:
|
|
# # loop over the unused row indexes
|
|
# for row in unusedRows:
|
|
# # grab the object ID for the corresponding row
|
|
# # index and increment the disappeared counter
|
|
# objectID = objectIDs[row]
|
|
# self.disappeared[objectID] += 1
|
|
|
|
# # check to see if the number of consecutive
|
|
# # frames the object has been marked "disappeared"
|
|
# # for warrants deregistering the object
|
|
# if self.disappeared[objectID] > self.maxDisappeared:
|
|
# self.deregister(objectID)
|
|
|
|
# # otherwise, if the number of input centroids is greater
|
|
# # than the number of existing object centroids we need to
|
|
# # register each new input centroid as a trackable object
|
|
# else:
|
|
# for col in unusedCols:
|
|
# self.register(inputCentroids[col])
|
|
|
|
# # return the set of trackable objects
|
|
# return self.objects
|
|
|
|
# Maintains the frame and object with the highest score
|
|
class BestFrames(threading.Thread):
|
|
def __init__(self, objects_parsed, recent_frames, detected_objects):
|
|
threading.Thread.__init__(self)
|
|
self.objects_parsed = objects_parsed
|
|
self.recent_frames = recent_frames
|
|
self.detected_objects = detected_objects
|
|
self.best_objects = {}
|
|
self.best_frames = {}
|
|
|
|
def run(self):
|
|
prctl.set_name("BestFrames")
|
|
while True:
|
|
|
|
# wait until objects have been parsed
|
|
with self.objects_parsed:
|
|
self.objects_parsed.wait()
|
|
|
|
# make a copy of detected objects
|
|
detected_objects = self.detected_objects.copy()
|
|
|
|
for obj in itertools.chain.from_iterable(detected_objects.values()):
|
|
if obj['name'] in self.best_objects:
|
|
now = datetime.datetime.now().timestamp()
|
|
# if the object is a higher score than the current best score
|
|
# or the current object is more than 1 minute old, use the new object
|
|
if obj['score'] > self.best_objects[obj['name']]['score'] or (now - self.best_objects[obj['name']]['frame_time']) > 60:
|
|
self.best_objects[obj['name']] = obj
|
|
else:
|
|
self.best_objects[obj['name']] = obj
|
|
|
|
# make a copy of the recent frames
|
|
recent_frames = self.recent_frames.copy()
|
|
|
|
for name, obj in self.best_objects.items():
|
|
if obj['frame_time'] in recent_frames:
|
|
best_frame = recent_frames[obj['frame_time']] #, np.zeros((720,1280,3), np.uint8))
|
|
|
|
draw_box_with_label(best_frame, obj['box']['xmin'], obj['box']['ymin'],
|
|
obj['box']['xmax'], obj['box']['ymax'], obj['name'], f"{int(obj['score']*100)}% {obj['area']}")
|
|
|
|
# print a timestamp
|
|
time_to_show = datetime.datetime.fromtimestamp(obj['frame_time']).strftime("%m/%d/%Y %H:%M:%S")
|
|
cv2.putText(best_frame, time_to_show, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
|
|
|
|
self.best_frames[name] = best_frame |