Mirrors/blakeblackshear.frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2024-11-26 19:06:11 +01:00
Code Issues Projects Releases Wiki Activity

switch everything to run off of tracked objects

Browse Source
This commit is contained in:
Blake Blackshear 2020-01-09 20:53:04 -06:00
parent 191f293037
commit e580aca440
4 changed files with 72 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
frigate/mqtt.py
View File

@ -6,39 +6,35 @@ from collections import Counter, defaultdict
import itertools import itertools
class MqttObjectPublisher(threading.Thread): class MqttObjectPublisher(threading.Thread):
def __init__(self, client, topic_prefix, objects_parsed, detected_objects, best_frames): def __init__(self, client, topic_prefix, camera):
threading.Thread.__init__(self) threading.Thread.__init__(self)
self.client = client self.client = client
self.topic_prefix = topic_prefix self.topic_prefix = topic_prefix
self.objects_parsed = objects_parsed self.camera = camera
self._detected_objects = detected_objects
self.best_frames = best_frames
def run(self): def run(self):
prctl.set_name("MqttObjectPublisher") prctl.set_name(self.__class__.__name__)
current_object_status = defaultdict(lambda: 'OFF') current_object_status = defaultdict(lambda: 'OFF')
while True: while True:
# wait until objects have been parsed # wait until objects have been tracked
with self.objects_parsed: with self.camera.objects_tracked:
self.objects_parsed.wait() self.camera.objects_tracked.wait()
# make a copy of detected objects # count objects with more than 2 entries in history by type
detected_objects = self._detected_objects.copy()
# total up all scores by object type
obj_counter = Counter() obj_counter = Counter()
for obj in itertools.chain.from_iterable(detected_objects.values()): for obj in self.camera.object_tracker.tracked_objects.values():
obj_counter[obj['name']] += obj['score'] if len(obj['history']) > 1:
obj_counter[obj['name']] += 1
# report on detected objects # report on detected objects
for obj_name, total_score in obj_counter.items(): for obj_name, count in obj_counter.items():
new_status = 'ON' if int(total_score*100) > 100 else 'OFF' new_status = 'ON' if count > 0 else 'OFF'
if new_status != current_object_status[obj_name]: if new_status != current_object_status[obj_name]:
current_object_status[obj_name] = new_status current_object_status[obj_name] = new_status
self.client.publish(self.topic_prefix+'/'+obj_name, new_status, retain=False) self.client.publish(self.topic_prefix+'/'+obj_name, new_status, retain=False)
# send the snapshot over mqtt if we have it as well # send the snapshot over mqtt if we have it as well
if obj_name in self.best_frames.best_frames: if obj_name in self.camera.best_frames.best_frames:
best_frame = cv2.cvtColor(self.best_frames.best_frames[obj_name], cv2.COLOR_RGB2BGR) best_frame = cv2.cvtColor(self.camera.best_frames.best_frames[obj_name], cv2.COLOR_RGB2BGR)
ret, jpg = cv2.imencode('.jpg', best_frame) ret, jpg = cv2.imencode('.jpg', best_frame)
if ret: if ret:
jpg_bytes = jpg.tobytes() jpg_bytes = jpg.tobytes()

8
frigate/object_detection.py
View File

@ -23,7 +23,7 @@ class PreppedQueueProcessor(threading.Thread):
self.avg_inference_speed = 10 self.avg_inference_speed = 10
def run(self): def run(self):
prctl.set_name("PreppedQueueProcessor") prctl.set_name(self.__class__.__name__)
# process queue... # process queue...
while True: while True:
if self.prepped_frame_queue.full(): if self.prepped_frame_queue.full():
@ -44,7 +44,7 @@ class RegionRequester(threading.Thread):
self.camera = camera self.camera = camera
def run(self): def run(self):
prctl.set_name("RegionRequester") prctl.set_name(self.__class__.__name__)
frame_time = 0.0 frame_time = 0.0
while True: while True:
now = datetime.datetime.now().timestamp() now = datetime.datetime.now().timestamp()
@ -58,7 +58,7 @@ class RegionRequester(threading.Thread):
frame_time = self.camera.frame_time.value frame_time = self.camera.frame_time.value
# grab the current tracked objects # grab the current tracked objects
tracked_objects = self.camera.object_tracker.tracked_objects.values().copy() tracked_objects = list(self.camera.object_tracker.tracked_objects.values()).copy()
with self.camera.regions_in_process_lock: with self.camera.regions_in_process_lock:
self.camera.regions_in_process[frame_time] = len(self.camera.config['regions']) self.camera.regions_in_process[frame_time] = len(self.camera.config['regions'])
@ -100,7 +100,7 @@ class RegionPrepper(threading.Thread):
self.prepped_frame_queue = prepped_frame_queue self.prepped_frame_queue = prepped_frame_queue
def run(self): def run(self):
prctl.set_name("RegionPrepper") prctl.set_name(self.__class__.__name__)
while True: while True:
resize_request = self.resize_request_queue.get() resize_request = self.resize_request_queue.get()

77
frigate/objects.py
View File

@ -10,10 +10,9 @@ from scipy.spatial import distance as dist
from frigate.util import draw_box_with_label, LABELS, compute_intersection_rectangle, compute_intersection_over_union, calculate_region from frigate.util import draw_box_with_label, LABELS, compute_intersection_rectangle, compute_intersection_over_union, calculate_region
class ObjectCleaner(threading.Thread): class ObjectCleaner(threading.Thread):
def __init__(self, objects_parsed, detected_objects): def __init__(self, camera):
threading.Thread.__init__(self) threading.Thread.__init__(self)
self._objects_parsed = objects_parsed self.camera = camera
self._detected_objects = detected_objects
def run(self): def run(self):
prctl.set_name("ObjectCleaner") prctl.set_name("ObjectCleaner")
@ -22,22 +21,9 @@ class ObjectCleaner(threading.Thread):
# wait a bit before checking for expired frames # wait a bit before checking for expired frames
time.sleep(0.2) time.sleep(0.2)
# expire the objects that are more than 1 second old for frame_time in list(self.camera.detected_objects.keys()).copy():
now = datetime.datetime.now().timestamp() if not frame_time in self.camera.frame_cache:
# look for the first object found within the last second del self.camera.detected_objects[frame_time]
# (newest objects are appended to the end)
detected_objects = self._detected_objects.copy()
objects_removed = False
for frame_time in detected_objects.keys():
if now-frame_time>2:
del self._detected_objects[frame_time]
objects_removed = True
if objects_removed:
# notify that parsed objects were changed
with self._objects_parsed:
self._objects_parsed.notify_all()
class DetectedObjectsProcessor(threading.Thread): class DetectedObjectsProcessor(threading.Thread):
def __init__(self, camera): def __init__(self, camera):
@ -169,9 +155,6 @@ class RegionRefiner(threading.Thread):
self.camera.detected_objects[frame_time] = selected_objects self.camera.detected_objects[frame_time] = selected_objects
with self.camera.objects_parsed:
self.camera.objects_parsed.notify_all()
# print(f"{frame_time} is actually finished") # print(f"{frame_time} is actually finished")
# keep adding frames to the refined queue as long as they are finished # keep adding frames to the refined queue as long as they are finished
@ -247,11 +230,16 @@ class ObjectTracker(threading.Thread):
while True: while True:
frame_time = self.camera.refined_frame_queue.get() frame_time = self.camera.refined_frame_queue.get()
self.match_and_update(self.camera.detected_objects[frame_time]) self.match_and_update(self.camera.detected_objects[frame_time])
self.camera.frame_tracked_queue.put(frame_time) self.camera.frame_output_queue.put(frame_time)
if len(self.tracked_objects) > 0:
with self.camera.objects_tracked:
self.camera.objects_tracked.notify_all()
def register(self, index, obj): def register(self, index, obj):
id = f"{str(obj['frame_time'])}-{index}" id = f"{str(obj['frame_time'])}-{index}"
obj['id'] = id obj['id'] = id
obj['top_score'] = obj['score']
self.add_history(obj)
self.tracked_objects[id] = obj self.tracked_objects[id] = obj
self.disappeared[id] = 0 self.disappeared[id] = 0
@ -261,7 +249,22 @@ class ObjectTracker(threading.Thread):
def update(self, id, new_obj): def update(self, id, new_obj):
self.tracked_objects[id].update(new_obj) self.tracked_objects[id].update(new_obj)
# TODO: am i missing anything? history? self.add_history(self.tracked_objects[id])
if self.tracked_objects[id]['score'] > self.tracked_objects[id]['top_score']:
self.tracked_objects[id]['top_score'] = self.tracked_objects[id]['score']
def add_history(self, obj):
entry = {
'score': obj['score'],
'box': obj['box'],
'region': obj['region'],
'centroid': obj['centroid'],
'frame_time': obj['frame_time']
}
if 'history' in obj:
obj['history'].append(entry)
else:
obj['history'] = [entry]
def match_and_update(self, new_objects): def match_and_update(self, new_objects):
# check to see if the list of input bounding box rectangles # check to see if the list of input bounding box rectangles
@ -384,26 +387,23 @@ class ObjectTracker(threading.Thread):
# Maintains the frame and object with the highest score # Maintains the frame and object with the highest score
class BestFrames(threading.Thread): class BestFrames(threading.Thread):
def __init__(self, objects_parsed, recent_frames, detected_objects): def __init__(self, camera):
threading.Thread.__init__(self) threading.Thread.__init__(self)
self.objects_parsed = objects_parsed self.camera = camera
self.recent_frames = recent_frames
self.detected_objects = detected_objects
self.best_objects = {} self.best_objects = {}
self.best_frames = {} self.best_frames = {}
def run(self): def run(self):
prctl.set_name("BestFrames") prctl.set_name(self.__class__.__name__)
while True: while True:
# wait until objects have been tracked
# wait until objects have been parsed with self.camera.objects_tracked:
with self.objects_parsed: self.camera.objects_tracked.wait()
self.objects_parsed.wait()
# make a copy of detected objects # make a copy of detected objects
detected_objects = self.detected_objects.copy() detected_objects = list(self.camera.object_tracker.tracked_objects.values()).copy()
for obj in itertools.chain.from_iterable(detected_objects.values()): for obj in detected_objects:
if obj['name'] in self.best_objects: if obj['name'] in self.best_objects:
now = datetime.datetime.now().timestamp() now = datetime.datetime.now().timestamp()
# if the object is a higher score than the current best score # if the object is a higher score than the current best score
@ -413,12 +413,9 @@ class BestFrames(threading.Thread):
else: else:
self.best_objects[obj['name']] = obj self.best_objects[obj['name']] = obj
# make a copy of the recent frames
recent_frames = self.recent_frames.copy()
for name, obj in self.best_objects.items(): for name, obj in self.best_objects.items():
if obj['frame_time'] in recent_frames: if obj['frame_time'] in self.camera.frame_cache:
best_frame = recent_frames[obj['frame_time']] #, np.zeros((720,1280,3), np.uint8)) best_frame = self.camera.frame_cache[obj['frame_time']]
draw_box_with_label(best_frame, obj['box']['xmin'], obj['box']['ymin'], draw_box_with_label(best_frame, obj['box']['xmin'], obj['box']['ymin'],
obj['box']['xmax'], obj['box']['ymax'], obj['name'], f"{int(obj['score']*100)}% {obj['area']}") obj['box']['xmax'], obj['box']['ymax'], obj['name'], f"{int(obj['score']*100)}% {obj['area']}")

33
frigate/video.py
View File

@ -26,17 +26,18 @@ class FrameTracker(threading.Thread):
self.recent_frames = recent_frames self.recent_frames = recent_frames
def run(self): def run(self):
prctl.set_name("FrameTracker") prctl.set_name(self.__class__.__name__)
while True: while True:
# wait for a frame # wait for a frame
with self.frame_ready: with self.frame_ready:
self.frame_ready.wait() self.frame_ready.wait()
now = datetime.datetime.now().timestamp()
# delete any old frames # delete any old frames
stored_frame_times = list(self.recent_frames.keys()) stored_frame_times = list(self.recent_frames.keys())
for k in stored_frame_times: stored_frame_times.sort(reverse=True)
if (now - k) > 10: if len(stored_frame_times) > 100:
frames_to_delete = stored_frame_times[50:]
for k in frames_to_delete:
del self.recent_frames[k] del self.recent_frames[k]
def get_frame_shape(source): def get_frame_shape(source):
@ -58,7 +59,7 @@ class CameraWatchdog(threading.Thread):
self.camera = camera self.camera = camera
def run(self): def run(self):
prctl.set_name("CameraWatchdog") prctl.set_name(self.__class__.__name__)
while True: while True:
# wait a bit before checking # wait a bit before checking
time.sleep(10) time.sleep(10)
@ -75,7 +76,7 @@ class CameraCapture(threading.Thread):
self.camera = camera self.camera = camera
def run(self): def run(self):
prctl.set_name("CameraCapture") prctl.set_name(self.__class__.__name__)
frame_num = 0 frame_num = 0
while True: while True:
if self.camera.ffmpeg_process.poll() != None: if self.camera.ffmpeg_process.poll() != None:
@ -113,10 +114,10 @@ class VideoWriter(threading.Thread):
self.camera = camera self.camera = camera
def run(self): def run(self):
prctl.set_name("VideoWriter") prctl.set_name(self.__class__.__name__)
while True: while True:
frame_time = self.camera.frame_tracked_queue.get() frame_time = self.camera.frame_output_queue.get()
if len(self.camera.detected_objects[frame_time]) == 0: if len(self.camera.object_tracker.tracked_objects) == 0:
continue continue
f = open(f"/debug/{self.camera.name}-{str(frame_time)}.jpg", 'wb') f = open(f"/debug/{self.camera.name}-{str(frame_time)}.jpg", 'wb')
f.write(self.camera.frame_with_objects(frame_time)) f.write(self.camera.frame_with_objects(frame_time))
@ -137,7 +138,7 @@ class Camera:
self.regions_in_process_lock = mp.Lock() self.regions_in_process_lock = mp.Lock()
self.finished_frame_queue = queue.Queue() self.finished_frame_queue = queue.Queue()
self.refined_frame_queue = queue.Queue() self.refined_frame_queue = queue.Queue()
self.frame_tracked_queue = queue.Queue() self.frame_output_queue = queue.Queue()
self.ffmpeg = config.get('ffmpeg', {}) self.ffmpeg = config.get('ffmpeg', {})
self.ffmpeg_input = get_ffmpeg_input(self.ffmpeg['input']) self.ffmpeg_input = get_ffmpeg_input(self.ffmpeg['input'])
@ -161,8 +162,8 @@ class Camera:
self.frame_lock = mp.Lock() self.frame_lock = mp.Lock()
# Condition for notifying that a new frame is ready # Condition for notifying that a new frame is ready
self.frame_ready = mp.Condition() self.frame_ready = mp.Condition()
# Condition for notifying that objects were parsed # Condition for notifying that objects were tracked
self.objects_parsed = mp.Condition() self.objects_tracked = mp.Condition()
# Queue for prepped frames, max size set to (number of regions * 5) # Queue for prepped frames, max size set to (number of regions * 5)
max_queue_size = len(self.config['regions'])*5 max_queue_size = len(self.config['regions'])*5
@ -208,11 +209,11 @@ class Camera:
self.region_prepper.start() self.region_prepper.start()
# start a thread to store the highest scoring recent frames for monitored object types # start a thread to store the highest scoring recent frames for monitored object types
self.best_frames = BestFrames(self.objects_parsed, self.frame_cache, self.detected_objects) self.best_frames = BestFrames(self)
self.best_frames.start() self.best_frames.start()
# start a thread to expire objects from the detected objects list # start a thread to expire objects from the detected objects list
self.object_cleaner = ObjectCleaner(self.objects_parsed, self.detected_objects) self.object_cleaner = ObjectCleaner(self)
self.object_cleaner.start() self.object_cleaner.start()
# start a thread to refine regions when objects are clipped # start a thread to refine regions when objects are clipped
@ -230,7 +231,7 @@ class Camera:
self.video_writer.start() self.video_writer.start()
# start a thread to publish object scores # start a thread to publish object scores
mqtt_publisher = MqttObjectPublisher(self.mqtt_client, self.mqtt_topic_prefix, self.objects_parsed, self.detected_objects, self.best_frames) mqtt_publisher = MqttObjectPublisher(self.mqtt_client, self.mqtt_topic_prefix, self)
mqtt_publisher.start() mqtt_publisher.start()
# create a watchdog thread for capture process # create a watchdog thread for capture process
@ -321,7 +322,7 @@ class Camera:
color, 2) color, 2)
# draw the bounding boxes on the screen # draw the bounding boxes on the screen
for id, obj in self.object_tracker.tracked_objects.items(): for id, obj in list(self.object_tracker.tracked_objects.items()):
# for obj in detected_objects[frame_time]: # for obj in detected_objects[frame_time]:
cv2.rectangle(frame, (obj['region']['xmin'], obj['region']['ymin']), cv2.rectangle(frame, (obj['region']['xmin'], obj['region']['ymin']),
(obj['region']['xmax'], obj['region']['ymax']), (obj['region']['xmax'], obj['region']['ymax']),