Mirrors/blakeblackshear.frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2024-11-21 19:07:46 +01:00
Code Issues Projects Releases Wiki Activity

split into separate processes

Browse Source
This commit is contained in:
Blake Blackshear 2020-02-15 21:07:54 -06:00
parent ffa9534549
commit 569e07949f
10 changed files with 1234 additions and 560 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Dockerfile
View File

@ -26,10 +26,11 @@ RUN apt -qq update && apt -qq install --no-install-recommends -y \
scipy \ scipy \
&& python3.7 -m pip install -U \ && python3.7 -m pip install -U \
SharedArray \ SharedArray \
# Flask \ Flask \
# paho-mqtt \ paho-mqtt \
# PyYAML \ PyYAML \
# matplotlib \ matplotlib \
pyarrow \
&& echo "deb https://packages.cloud.google.com/apt coral-edgetpu-stable main" > /etc/apt/sources.list.d/coral-edgetpu.list \ && echo "deb https://packages.cloud.google.com/apt coral-edgetpu-stable main" > /etc/apt/sources.list.d/coral-edgetpu.list \
&& wget -q -O - https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \ && wget -q -O - https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \
&& apt -qq update \ && apt -qq update \

87
detect_objects.py
View File

@ -2,13 +2,16 @@ import cv2
import time import time
import queue import queue
import yaml import yaml
import multiprocessing as mp
import subprocess as sp
import numpy as np import numpy as np
from flask import Flask, Response, make_response, jsonify from flask import Flask, Response, make_response, jsonify
import paho.mqtt.client as mqtt import paho.mqtt.client as mqtt
from frigate.video import Camera from frigate.video import track_camera
from frigate.object_detection import PreppedQueueProcessor from frigate.object_processing import TrackedObjectProcessor
from frigate.util import EventsPerSecond from frigate.util import EventsPerSecond
from frigate.edgetpu import EdgeTPUProcess
with open('/config/config.yml') as f: with open('/config/config.yml') as f:
CONFIG = yaml.safe_load(f) CONFIG = yaml.safe_load(f)
@ -38,8 +41,7 @@ FFMPEG_DEFAULT_CONFIG = {
'-stimeout', '5000000', '-stimeout', '5000000',
'-use_wallclock_as_timestamps', '1']), '-use_wallclock_as_timestamps', '1']),
'output_args': FFMPEG_CONFIG.get('output_args', 'output_args': FFMPEG_CONFIG.get('output_args',
['-vf', 'mpdecimate', ['-f', 'rawvideo',
'-f', 'rawvideo',
'-pix_fmt', 'rgb24']) '-pix_fmt', 'rgb24'])
} }
@ -48,6 +50,10 @@ GLOBAL_OBJECT_CONFIG = CONFIG.get('objects', {})
WEB_PORT = CONFIG.get('web_port', 5000) WEB_PORT = CONFIG.get('web_port', 5000)
DEBUG = (CONFIG.get('debug', '0') == '1') DEBUG = (CONFIG.get('debug', '0') == '1')
# MODEL_PATH = CONFIG.get('tflite_model', '/lab/mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite')
MODEL_PATH = CONFIG.get('tflite_model', '/lab/detect.tflite')
LABEL_MAP = CONFIG.get('label_map', '/lab/labelmap.txt')
def main(): def main():
# connect to mqtt and setup last will # connect to mqtt and setup last will
def on_connect(client, userdata, flags, rc): def on_connect(client, userdata, flags, rc):
@ -71,27 +77,43 @@ def main():
client.connect(MQTT_HOST, MQTT_PORT, 60) client.connect(MQTT_HOST, MQTT_PORT, 60)
client.loop_start() client.loop_start()
# Queue for prepped frames, max size set to number of regions * 3 # start plasma store
prepped_frame_queue = queue.Queue() plasma_cmd = ['plasma_store', '-m', '400000000', '-s', '/tmp/plasma']
plasma_process = sp.Popen(plasma_cmd, stdout=sp.DEVNULL, stderr=sp.DEVNULL)
cameras = {} ##
# Setup config defaults for cameras
##
for name, config in CONFIG['cameras'].items(): for name, config in CONFIG['cameras'].items():
cameras[name] = Camera(name, FFMPEG_DEFAULT_CONFIG, GLOBAL_OBJECT_CONFIG, config, config['snapshots'] = {
prepped_frame_queue, client, MQTT_TOPIC_PREFIX) 'show_timestamp': config.get('snapshots', {}).get('show_timestamp', True)
}
fps_tracker = EventsPerSecond() # Queue for cameras to push tracked objects to
tracked_objects_queue = mp.Queue()
prepped_queue_processor = PreppedQueueProcessor( # Start the shared tflite process
cameras, tflite_process = EdgeTPUProcess(MODEL_PATH)
prepped_frame_queue,
fps_tracker
)
prepped_queue_processor.start()
fps_tracker.start()
for name, camera in cameras.items(): camera_processes = []
camera.start() camera_stats_values = {}
print("Capture process for {}: {}".format(name, camera.get_capture_pid())) for name, config in CONFIG['cameras'].items():
camera_stats_values[name] = {
'fps': mp.Value('d', 10.0),
'avg_wait': mp.Value('d', 0.0)
}
camera_process = mp.Process(target=track_camera, args=(name, config, FFMPEG_DEFAULT_CONFIG, GLOBAL_OBJECT_CONFIG,
tflite_process.detect_lock, tflite_process.detect_ready, tflite_process.frame_ready, tracked_objects_queue,
camera_stats_values[name]['fps'], camera_stats_values[name]['avg_wait']))
camera_process.daemon = True
camera_processes.append(camera_process)
for camera_process in camera_processes:
camera_process.start()
print(f"Camera_process started {camera_process.pid}")
object_processor = TrackedObjectProcessor(CONFIG['cameras'], client, MQTT_TOPIC_PREFIX, tracked_objects_queue)
object_processor.start()
# create a flask app that encodes frames a mjpeg on demand # create a flask app that encodes frames a mjpeg on demand
app = Flask(__name__) app = Flask(__name__)
@ -105,21 +127,23 @@ def main():
def stats(): def stats():
stats = { stats = {
'coral': { 'coral': {
'fps': fps_tracker.eps(), 'fps': tflite_process.fps.value,
'inference_speed': prepped_queue_processor.avg_inference_speed, 'inference_speed': tflite_process.avg_inference_speed.value
'queue_length': prepped_frame_queue.qsize()
} }
} }
for name, camera in cameras.items(): for name, camera_stats in camera_stats_values.items():
stats[name] = camera.stats() stats[name] = {
'fps': camera_stats['fps'].value,
'avg_wait': camera_stats['avg_wait'].value
}
return jsonify(stats) return jsonify(stats)
@app.route('/<camera_name>/<label>/best.jpg') @app.route('/<camera_name>/<label>/best.jpg')
def best(camera_name, label): def best(camera_name, label):
if camera_name in cameras: if camera_name in CONFIG['cameras']:
best_frame = cameras[camera_name].get_best(label) best_frame = object_processor.get_best(camera_name, label)
if best_frame is None: if best_frame is None:
best_frame = np.zeros((720,1280,3), np.uint8) best_frame = np.zeros((720,1280,3), np.uint8)
best_frame = cv2.cvtColor(best_frame, cv2.COLOR_RGB2BGR) best_frame = cv2.cvtColor(best_frame, cv2.COLOR_RGB2BGR)
@ -132,7 +156,7 @@ def main():
@app.route('/<camera_name>') @app.route('/<camera_name>')
def mjpeg_feed(camera_name): def mjpeg_feed(camera_name):
if camera_name in cameras: if camera_name in CONFIG['cameras']:
# return a multipart response # return a multipart response
return Response(imagestream(camera_name), return Response(imagestream(camera_name),
mimetype='multipart/x-mixed-replace; boundary=frame') mimetype='multipart/x-mixed-replace; boundary=frame')
@ -143,13 +167,16 @@ def main():
while True: while True:
# max out at 1 FPS # max out at 1 FPS
time.sleep(1) time.sleep(1)
frame = cameras[camera_name].get_current_frame_with_objects() frame = object_processor.current_frame_with_objects(camera_name)
yield (b'--frame\r\n' yield (b'--frame\r\n'
b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n\r\n') b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n\r\n')
app.run(host='0.0.0.0', port=WEB_PORT, debug=False) app.run(host='0.0.0.0', port=WEB_PORT, debug=False)
camera.join() for camera_process in camera_processes:
camera_process.join()
plasma_process.terminate()
if __name__ == '__main__': if __name__ == '__main__':
main() main()

18
frigate/edgetpu.py
View File

@ -1,8 +1,11 @@
import os
import datetime
import multiprocessing as mp import multiprocessing as mp
import numpy as np import numpy as np
import SharedArray as sa import SharedArray as sa
import tflite_runtime.interpreter as tflite import tflite_runtime.interpreter as tflite
from tflite_runtime.interpreter import load_delegate from tflite_runtime.interpreter import load_delegate
from frigate.util import EventsPerSecond
def load_labels(path, encoding='utf-8'): def load_labels(path, encoding='utf-8'):
"""Loads labels from file (with or without index numbers). """Loads labels from file (with or without index numbers).
@ -59,6 +62,7 @@ class ObjectDetector():
class EdgeTPUProcess(): class EdgeTPUProcess():
def __init__(self, model): def __init__(self, model):
# TODO: see if we can use the plasma store with a queue and maintain the same speeds
try: try:
sa.delete("frame") sa.delete("frame")
except: except:
@ -74,22 +78,32 @@ class EdgeTPUProcess():
self.detect_lock = mp.Lock() self.detect_lock = mp.Lock()
self.detect_ready = mp.Event() self.detect_ready = mp.Event()
self.frame_ready = mp.Event() self.frame_ready = mp.Event()
self.fps = mp.Value('d', 0.0)
self.avg_inference_speed = mp.Value('d', 10.0)
def run_detector(model, detect_ready, frame_ready): def run_detector(model, detect_ready, frame_ready, fps, avg_speed):
print(f"Starting detection process: {os.getpid()}")
object_detector = ObjectDetector(model) object_detector = ObjectDetector(model)
input_frame = sa.attach("frame") input_frame = sa.attach("frame")
detections = sa.attach("detections") detections = sa.attach("detections")
fps_tracker = EventsPerSecond()
fps_tracker.start()
while True: while True:
# wait until a frame is ready # wait until a frame is ready
frame_ready.wait() frame_ready.wait()
start = datetime.datetime.now().timestamp()
# signal that the process is busy # signal that the process is busy
frame_ready.clear() frame_ready.clear()
detections[:] = object_detector.detect_raw(input_frame) detections[:] = object_detector.detect_raw(input_frame)
# signal that the process is ready to detect # signal that the process is ready to detect
detect_ready.set() detect_ready.set()
fps_tracker.update()
fps.value = fps_tracker.eps()
duration = datetime.datetime.now().timestamp()-start
avg_speed.value = (avg_speed.value*9 + duration)/10
self.detect_process = mp.Process(target=run_detector, args=(model, self.detect_ready, self.frame_ready)) self.detect_process = mp.Process(target=run_detector, args=(model, self.detect_ready, self.frame_ready, self.fps, self.avg_inference_speed))
self.detect_process.daemon = True self.detect_process.daemon = True
self.detect_process.start() self.detect_process.start()

9
frigate/motion.py
View File

@ -3,14 +3,15 @@ import imutils
import numpy as np import numpy as np
class MotionDetector(): class MotionDetector():
# TODO: add motion masking def __init__(self, frame_shape, mask, resize_factor=4):
def __init__(self, frame_shape, resize_factor=4):
self.resize_factor = resize_factor self.resize_factor = resize_factor
self.motion_frame_size = (int(frame_shape[0]/resize_factor), int(frame_shape[1]/resize_factor)) self.motion_frame_size = (int(frame_shape[0]/resize_factor), int(frame_shape[1]/resize_factor))
self.avg_frame = np.zeros(self.motion_frame_size, np.float) self.avg_frame = np.zeros(self.motion_frame_size, np.float)
self.avg_delta = np.zeros(self.motion_frame_size, np.float) self.avg_delta = np.zeros(self.motion_frame_size, np.float)
self.motion_frame_count = 0 self.motion_frame_count = 0
self.frame_counter = 0 self.frame_counter = 0
resized_mask = cv2.resize(mask, dsize=(self.motion_frame_size[1], self.motion_frame_size[0]), interpolation=cv2.INTER_LINEAR)
self.mask = np.where(resized_mask==[0])
def detect(self, frame): def detect(self, frame):
motion_boxes = [] motion_boxes = []
@ -21,6 +22,9 @@ class MotionDetector():
# convert to grayscale # convert to grayscale
gray = cv2.cvtColor(resized_frame, cv2.COLOR_BGR2GRAY) gray = cv2.cvtColor(resized_frame, cv2.COLOR_BGR2GRAY)
# mask frame
gray[self.mask] = [255]
# it takes ~30 frames to establish a baseline # it takes ~30 frames to establish a baseline
# dont bother looking for motion # dont bother looking for motion
if self.frame_counter < 30: if self.frame_counter < 30:
@ -58,7 +62,6 @@ class MotionDetector():
# if the contour is big enough, count it as motion # if the contour is big enough, count it as motion
contour_area = cv2.contourArea(c) contour_area = cv2.contourArea(c)
if contour_area > 100: if contour_area > 100:
# cv2.drawContours(resized_frame, [c], -1, (255,255,255), 2)
x, y, w, h = cv2.boundingRect(c) x, y, w, h = cv2.boundingRect(c)
motion_boxes.append((x*self.resize_factor, y*self.resize_factor, (x+w)*self.resize_factor, (y+h)*self.resize_factor)) motion_boxes.append((x*self.resize_factor, y*self.resize_factor, (x+w)*self.resize_factor, (y+h)*self.resize_factor))

54
frigate/mqtt.py
View File

@ -1,54 +0,0 @@
import json
import cv2
import threading
import prctl
from collections import Counter, defaultdict
import itertools
class MqttObjectPublisher(threading.Thread):
def __init__(self, client, topic_prefix, camera):
threading.Thread.__init__(self)
self.client = client
self.topic_prefix = topic_prefix
self.camera = camera
def run(self):
prctl.set_name(self.__class__.__name__)
current_object_status = defaultdict(lambda: 'OFF')
while True:
# wait until objects have been tracked
with self.camera.objects_tracked:
self.camera.objects_tracked.wait()
# count objects with more than 2 entries in history by type
obj_counter = Counter()
for obj in self.camera.object_tracker.tracked_objects.values():
if len(obj['history']) > 1:
obj_counter[obj['name']] += 1
# report on detected objects
for obj_name, count in obj_counter.items():
new_status = 'ON' if count > 0 else 'OFF'
if new_status != current_object_status[obj_name]:
current_object_status[obj_name] = new_status
self.client.publish(self.topic_prefix+'/'+obj_name, new_status, retain=False)
# send the snapshot over mqtt if we have it as well
if obj_name in self.camera.best_frames.best_frames:
best_frame = cv2.cvtColor(self.camera.best_frames.best_frames[obj_name], cv2.COLOR_RGB2BGR)
ret, jpg = cv2.imencode('.jpg', best_frame)
if ret:
jpg_bytes = jpg.tobytes()
self.client.publish(self.topic_prefix+'/'+obj_name+'/snapshot', jpg_bytes, retain=True)
# expire any objects that are ON and no longer detected
expired_objects = [obj_name for obj_name, status in current_object_status.items() if status == 'ON' and not obj_name in obj_counter]
for obj_name in expired_objects:
current_object_status[obj_name] = 'OFF'
self.client.publish(self.topic_prefix+'/'+obj_name, 'OFF', retain=False)
# send updated snapshot snapshot over mqtt if we have it as well
if obj_name in self.camera.best_frames.best_frames:
best_frame = cv2.cvtColor(self.camera.best_frames.best_frames[obj_name], cv2.COLOR_RGB2BGR)
ret, jpg = cv2.imencode('.jpg', best_frame)
if ret:
jpg_bytes = jpg.tobytes()
self.client.publish(self.topic_prefix+'/'+obj_name+'/snapshot', jpg_bytes, retain=True)

2
frigate/object_detection.py
View File

@ -3,7 +3,7 @@ import time
import cv2 import cv2
import threading import threading
import copy import copy
import prctl # import prctl
import numpy as np import numpy as np
from edgetpu.detection.engine import DetectionEngine from edgetpu.detection.engine import DetectionEngine

146
frigate/object_processing.py Normal file
View File

@ -0,0 +1,146 @@
import json
import hashlib
import datetime
import copy
import cv2
import threading
import numpy as np
from collections import Counter, defaultdict
import itertools
import pyarrow.plasma as plasma
import SharedArray as sa
import matplotlib.pyplot as plt
from frigate.util import draw_box_with_label, ReadLabelFile
PATH_TO_LABELS = '/lab/labelmap.txt'
LABELS = ReadLabelFile(PATH_TO_LABELS)
cmap = plt.cm.get_cmap('tab10', len(LABELS.keys()))
COLOR_MAP = {}
for key, val in LABELS.items():
COLOR_MAP[val] = tuple(int(round(255 * c)) for c in cmap(key)[:3])
class TrackedObjectProcessor(threading.Thread):
def __init__(self, config, client, topic_prefix, tracked_objects_queue):
threading.Thread.__init__(self)
self.config = config
self.client = client
self.topic_prefix = topic_prefix
self.tracked_objects_queue = tracked_objects_queue
self.plasma_client = plasma.connect("/tmp/plasma")
self.camera_data = defaultdict(lambda: {
'best_objects': {},
'object_status': defaultdict(lambda: defaultdict(lambda: 'OFF')),
'tracked_objects': {}
})
def get_best(self, camera, label):
if label in self.camera_data[camera]['best_objects']:
return self.camera_data[camera]['best_objects'][label]['frame']
else:
return None
def get_frame(self, config, camera, obj):
object_id_hash = hashlib.sha1(str.encode(f"{camera}{obj['frame_time']}"))
object_id_bytes = object_id_hash.digest()
object_id = plasma.ObjectID(object_id_bytes)
best_frame = self.plasma_client.get(object_id)
box = obj['box']
draw_box_with_label(best_frame, box[0], box[1], box[2], box[3], obj['label'], f"{int(obj['score']*100)}% {int(obj['area'])}")
# print a timestamp
if config['snapshots']['show_timestamp']:
time_to_show = datetime.datetime.fromtimestamp(obj['frame_time']).strftime("%m/%d/%Y %H:%M:%S")
cv2.putText(best_frame, time_to_show, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
return best_frame
def current_frame_with_objects(self, camera):
frame_time = self.camera_data[camera]['current_frame']
object_id_hash = hashlib.sha1(str.encode(f"{camera}{frame_time}"))
object_id_bytes = object_id_hash.digest()
object_id = plasma.ObjectID(object_id_bytes)
current_frame = self.plasma_client.get(object_id)
tracked_objects = copy.deepcopy(self.camera_data[camera]['tracked_objects'])
# draw the bounding boxes on the screen
for obj in tracked_objects.values():
thickness = 2
color = COLOR_MAP[obj['label']]
if obj['frame_time'] != frame_time:
thickness = 1
color = (255,0,0)
box = obj['box']
draw_box_with_label(current_frame, box[0], box[1], box[2], box[3], obj['label'], f"{int(obj['score']*100)}% {int(obj['area'])}", thickness=thickness, color=color)
# # print fps
# cv2.putText(frame, str(self.fps.eps())+'FPS', (10, 60), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
# convert to BGR
frame = cv2.cvtColor(current_frame, cv2.COLOR_RGB2BGR)
# encode the image into a jpg
ret, jpg = cv2.imencode('.jpg', frame)
return jpg.tobytes()
def run(self):
while True:
camera, frame_time, tracked_objects = self.tracked_objects_queue.get()
config = self.config[camera]
best_objects = self.camera_data[camera]['best_objects']
current_object_status = self.camera_data[camera]['object_status']
self.camera_data[camera]['tracked_objects'] = tracked_objects
self.camera_data[camera]['current_frame'] = frame_time
###
# Maintain the highest scoring recent object and frame for each label
###
for obj in tracked_objects.values():
if obj['label'] in best_objects:
now = datetime.datetime.now().timestamp()
# if the object is a higher score than the current best score
# or the current object is more than 1 minute old, use the new object
if obj['score'] > best_objects[obj['label']]['score'] or (now - best_objects[obj['label']]['frame_time']) > 60:
obj['frame'] = self.get_frame(config, camera, obj)
best_objects[obj['label']] = obj
else:
obj['frame'] = self.get_frame(config, camera, obj)
best_objects[obj['label']] = obj
###
# Report over MQTT
###
# count objects with more than 2 entries in history by type
obj_counter = Counter()
for obj in tracked_objects.values():
if len(obj['history']) > 1:
obj_counter[obj['label']] += 1
# report on detected objects
for obj_name, count in obj_counter.items():
new_status = 'ON' if count > 0 else 'OFF'
if new_status != current_object_status[obj_name]:
current_object_status[obj_name] = new_status
self.client.publish(f"{self.topic_prefix}/{camera}/{obj_name}", new_status, retain=False)
# send the best snapshot over mqtt
best_frame = cv2.cvtColor(best_objects[obj_name]['frame'], cv2.COLOR_RGB2BGR)
ret, jpg = cv2.imencode('.jpg', best_frame)
if ret:
jpg_bytes = jpg.tobytes()
self.client.publish(f"{self.topic_prefix}/{camera}/{obj_name}/snapshot", jpg_bytes, retain=True)
# expire any objects that are ON and no longer detected
expired_objects = [obj_name for obj_name, status in current_object_status.items() if status == 'ON' and not obj_name in obj_counter]
for obj_name in expired_objects:
current_object_status[obj_name] = 'OFF'
self.client.publish(f"{self.topic_prefix}/{camera}/{obj_name}", 'OFF', retain=False)
# send updated snapshot over mqtt
best_frame = cv2.cvtColor(best_objects[obj_name]['frame'], cv2.COLOR_RGB2BGR)
ret, jpg = cv2.imencode('.jpg', best_frame)
if ret:
jpg_bytes = jpg.tobytes()
self.client.publish(f"{self.topic_prefix}/{camera}/{obj_name}/snapshot", jpg_bytes, retain=True)

511
frigate/objects.py
View File

@ -2,277 +2,266 @@ import time
import datetime import datetime
import threading import threading
import cv2 import cv2
import prctl # import prctl
import itertools import itertools
import copy import copy
import numpy as np import numpy as np
import multiprocessing as mp import multiprocessing as mp
from collections import defaultdict from collections import defaultdict
from scipy.spatial import distance as dist from scipy.spatial import distance as dist
from frigate.util import draw_box_with_label, LABELS, compute_intersection_rectangle, compute_intersection_over_union, calculate_region from frigate.util import draw_box_with_label, LABELS, calculate_region
class ObjectCleaner(threading.Thread): # class ObjectCleaner(threading.Thread):
def __init__(self, camera): # def __init__(self, camera):
threading.Thread.__init__(self) # threading.Thread.__init__(self)
self.camera = camera # self.camera = camera
def run(self): # def run(self):
prctl.set_name("ObjectCleaner") # prctl.set_name("ObjectCleaner")
while True: # while True:
# wait a bit before checking for expired frames # # wait a bit before checking for expired frames
time.sleep(0.2) # time.sleep(0.2)
for frame_time in list(self.camera.detected_objects.keys()).copy(): # for frame_time in list(self.camera.detected_objects.keys()).copy():
if not frame_time in self.camera.frame_cache: # if not frame_time in self.camera.frame_cache:
del self.camera.detected_objects[frame_time] # del self.camera.detected_objects[frame_time]
objects_deregistered = False # objects_deregistered = False
with self.camera.object_tracker.tracked_objects_lock: # with self.camera.object_tracker.tracked_objects_lock:
now = datetime.datetime.now().timestamp() # now = datetime.datetime.now().timestamp()
for id, obj in list(self.camera.object_tracker.tracked_objects.items()): # for id, obj in list(self.camera.object_tracker.tracked_objects.items()):
# if the object is more than 10 seconds old # # if the object is more than 10 seconds old
# and not in the most recent frame, deregister # # and not in the most recent frame, deregister
if (now - obj['frame_time']) > 10 and self.camera.object_tracker.most_recent_frame_time > obj['frame_time']: # if (now - obj['frame_time']) > 10 and self.camera.object_tracker.most_recent_frame_time > obj['frame_time']:
self.camera.object_tracker.deregister(id) # self.camera.object_tracker.deregister(id)
objects_deregistered = True # objects_deregistered = True
if objects_deregistered: # if objects_deregistered:
with self.camera.objects_tracked: # with self.camera.objects_tracked:
self.camera.objects_tracked.notify_all() # self.camera.objects_tracked.notify_all()
class DetectedObjectsProcessor(threading.Thread): # class DetectedObjectsProcessor(threading.Thread):
def __init__(self, camera): # def __init__(self, camera):
threading.Thread.__init__(self) # threading.Thread.__init__(self)
self.camera = camera # self.camera = camera
def run(self): # def run(self):
prctl.set_name(self.__class__.__name__) # prctl.set_name(self.__class__.__name__)
while True: # while True:
frame = self.camera.detected_objects_queue.get() # frame = self.camera.detected_objects_queue.get()
objects = frame['detected_objects'] # objects = frame['detected_objects']
for raw_obj in objects: # for raw_obj in objects:
name = str(LABELS[raw_obj.label_id]) # name = str(LABELS[raw_obj.label_id])
if not name in self.camera.objects_to_track: # if not name in self.camera.objects_to_track:
continue # continue
obj = { # obj = {
'name': name, # 'name': name,
'score': float(raw_obj.score), # 'score': float(raw_obj.score),
'box': { # 'box': {
'xmin': int((raw_obj.bounding_box[0][0] * frame['size']) + frame['x_offset']), # 'xmin': int((raw_obj.bounding_box[0][0] * frame['size']) + frame['x_offset']),
'ymin': int((raw_obj.bounding_box[0][1] * frame['size']) + frame['y_offset']), # 'ymin': int((raw_obj.bounding_box[0][1] * frame['size']) + frame['y_offset']),
'xmax': int((raw_obj.bounding_box[1][0] * frame['size']) + frame['x_offset']), # 'xmax': int((raw_obj.bounding_box[1][0] * frame['size']) + frame['x_offset']),
'ymax': int((raw_obj.bounding_box[1][1] * frame['size']) + frame['y_offset']) # 'ymax': int((raw_obj.bounding_box[1][1] * frame['size']) + frame['y_offset'])
}, # },
'region': { # 'region': {
'xmin': frame['x_offset'], # 'xmin': frame['x_offset'],
'ymin': frame['y_offset'], # 'ymin': frame['y_offset'],
'xmax': frame['x_offset']+frame['size'], # 'xmax': frame['x_offset']+frame['size'],
'ymax': frame['y_offset']+frame['size'] # 'ymax': frame['y_offset']+frame['size']
}, # },
'frame_time': frame['frame_time'], # 'frame_time': frame['frame_time'],
'region_id': frame['region_id'] # 'region_id': frame['region_id']
} # }
# if the object is within 5 pixels of the region border, and the region is not on the edge # # if the object is within 5 pixels of the region border, and the region is not on the edge
# consider the object to be clipped # # consider the object to be clipped
obj['clipped'] = False # obj['clipped'] = False
if ((obj['region']['xmin'] > 5 and obj['box']['xmin']-obj['region']['xmin'] <= 5) or # if ((obj['region']['xmin'] > 5 and obj['box']['xmin']-obj['region']['xmin'] <= 5) or
(obj['region']['ymin'] > 5 and obj['box']['ymin']-obj['region']['ymin'] <= 5) or # (obj['region']['ymin'] > 5 and obj['box']['ymin']-obj['region']['ymin'] <= 5) or
(self.camera.frame_shape[1]-obj['region']['xmax'] > 5 and obj['region']['xmax']-obj['box']['xmax'] <= 5) or # (self.camera.frame_shape[1]-obj['region']['xmax'] > 5 and obj['region']['xmax']-obj['box']['xmax'] <= 5) or
(self.camera.frame_shape[0]-obj['region']['ymax'] > 5 and obj['region']['ymax']-obj['box']['ymax'] <= 5)): # (self.camera.frame_shape[0]-obj['region']['ymax'] > 5 and obj['region']['ymax']-obj['box']['ymax'] <= 5)):
obj['clipped'] = True # obj['clipped'] = True
# Compute the area # # Compute the area
# TODO: +1 right? # # TODO: +1 right?
obj['area'] = (obj['box']['xmax']-obj['box']['xmin'])*(obj['box']['ymax']-obj['box']['ymin']) # obj['area'] = (obj['box']['xmax']-obj['box']['xmin'])*(obj['box']['ymax']-obj['box']['ymin'])
self.camera.detected_objects[frame['frame_time']].append(obj) # self.camera.detected_objects[frame['frame_time']].append(obj)
# TODO: use in_process and processed counts instead to avoid lock # # TODO: use in_process and processed counts instead to avoid lock
with self.camera.regions_in_process_lock: # with self.camera.regions_in_process_lock:
if frame['frame_time'] in self.camera.regions_in_process: # if frame['frame_time'] in self.camera.regions_in_process:
self.camera.regions_in_process[frame['frame_time']] -= 1 # self.camera.regions_in_process[frame['frame_time']] -= 1
# print(f"{frame['frame_time']} remaining regions {self.camera.regions_in_process[frame['frame_time']]}") # # print(f"{frame['frame_time']} remaining regions {self.camera.regions_in_process[frame['frame_time']]}")
if self.camera.regions_in_process[frame['frame_time']] == 0: # if self.camera.regions_in_process[frame['frame_time']] == 0:
del self.camera.regions_in_process[frame['frame_time']] # del self.camera.regions_in_process[frame['frame_time']]
# print(f"{frame['frame_time']} no remaining regions") # # print(f"{frame['frame_time']} no remaining regions")
self.camera.finished_frame_queue.put(frame['frame_time']) # self.camera.finished_frame_queue.put(frame['frame_time'])
else: # else:
self.camera.finished_frame_queue.put(frame['frame_time']) # self.camera.finished_frame_queue.put(frame['frame_time'])
# Thread that checks finished frames for clipped objects and sends back # # Thread that checks finished frames for clipped objects and sends back
# for processing if needed # # for processing if needed
# TODO: evaluate whether or not i really need separate threads/queues for each step # # TODO: evaluate whether or not i really need separate threads/queues for each step
# given that only 1 thread will really be able to run at a time. you need a # # given that only 1 thread will really be able to run at a time. you need a
# separate process to actually do things in parallel for when you are CPU bound. # # separate process to actually do things in parallel for when you are CPU bound.
# threads are good when you are waiting and could be processing while you wait # # threads are good when you are waiting and could be processing while you wait
class RegionRefiner(threading.Thread): # class RegionRefiner(threading.Thread):
def __init__(self, camera): # def __init__(self, camera):
threading.Thread.__init__(self) # threading.Thread.__init__(self)
self.camera = camera # self.camera = camera
def run(self): # def run(self):
prctl.set_name(self.__class__.__name__) # prctl.set_name(self.__class__.__name__)
while True: # while True:
frame_time = self.camera.finished_frame_queue.get() # frame_time = self.camera.finished_frame_queue.get()
detected_objects = self.camera.detected_objects[frame_time].copy() # detected_objects = self.camera.detected_objects[frame_time].copy()
# print(f"{frame_time} finished") # # print(f"{frame_time} finished")
# group by name # # group by name
detected_object_groups = defaultdict(lambda: []) # detected_object_groups = defaultdict(lambda: [])
for obj in detected_objects: # for obj in detected_objects:
detected_object_groups[obj['name']].append(obj) # detected_object_groups[obj['name']].append(obj)
look_again = False # look_again = False
selected_objects = [] # selected_objects = []
for group in detected_object_groups.values(): # for group in detected_object_groups.values():
# apply non-maxima suppression to suppress weak, overlapping bounding boxes # # apply non-maxima suppression to suppress weak, overlapping bounding boxes
boxes = [(o['box']['xmin'], o['box']['ymin'], o['box']['xmax']-o['box']['xmin'], o['box']['ymax']-o['box']['ymin']) # boxes = [(o['box']['xmin'], o['box']['ymin'], o['box']['xmax']-o['box']['xmin'], o['box']['ymax']-o['box']['ymin'])
for o in group] # for o in group]
confidences = [o['score'] for o in group] # confidences = [o['score'] for o in group]
idxs = cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.4) # idxs = cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.4)
for index in idxs: # for index in idxs:
obj = group[index[0]] # obj = group[index[0]]
selected_objects.append(obj) # selected_objects.append(obj)
if obj['clipped']: # if obj['clipped']:
box = obj['box'] # box = obj['box']
# calculate a new region that will hopefully get the entire object # # calculate a new region that will hopefully get the entire object
(size, x_offset, y_offset) = calculate_region(self.camera.frame_shape, # (size, x_offset, y_offset) = calculate_region(self.camera.frame_shape,
box['xmin'], box['ymin'], # box['xmin'], box['ymin'],
box['xmax'], box['ymax']) # box['xmax'], box['ymax'])
# print(f"{frame_time} new region: {size} {x_offset} {y_offset}") # # print(f"{frame_time} new region: {size} {x_offset} {y_offset}")
with self.camera.regions_in_process_lock: # with self.camera.regions_in_process_lock:
if not frame_time in self.camera.regions_in_process: # if not frame_time in self.camera.regions_in_process:
self.camera.regions_in_process[frame_time] = 1 # self.camera.regions_in_process[frame_time] = 1
else: # else:
self.camera.regions_in_process[frame_time] += 1 # self.camera.regions_in_process[frame_time] += 1
# add it to the queue # # add it to the queue
self.camera.resize_queue.put({ # self.camera.resize_queue.put({
'camera_name': self.camera.name, # 'camera_name': self.camera.name,
'frame_time': frame_time, # 'frame_time': frame_time,
'region_id': -1, # 'region_id': -1,
'size': size, # 'size': size,
'x_offset': x_offset, # 'x_offset': x_offset,
'y_offset': y_offset # 'y_offset': y_offset
}) # })
self.camera.dynamic_region_fps.update() # self.camera.dynamic_region_fps.update()
look_again = True # look_again = True
# if we are looking again, then this frame is not ready for processing # # if we are looking again, then this frame is not ready for processing
if look_again: # if look_again:
# remove the clipped objects # # remove the clipped objects
self.camera.detected_objects[frame_time] = [o for o in selected_objects if not o['clipped']] # self.camera.detected_objects[frame_time] = [o for o in selected_objects if not o['clipped']]
continue # continue
# filter objects based on camera settings # # filter objects based on camera settings
selected_objects = [o for o in selected_objects if not self.filtered(o)] # selected_objects = [o for o in selected_objects if not self.filtered(o)]
self.camera.detected_objects[frame_time] = selected_objects # self.camera.detected_objects[frame_time] = selected_objects
# print(f"{frame_time} is actually finished") # # print(f"{frame_time} is actually finished")
# keep adding frames to the refined queue as long as they are finished # # keep adding frames to the refined queue as long as they are finished
with self.camera.regions_in_process_lock: # with self.camera.regions_in_process_lock:
while self.camera.frame_queue.qsize() > 0 and self.camera.frame_queue.queue[0] not in self.camera.regions_in_process: # while self.camera.frame_queue.qsize() > 0 and self.camera.frame_queue.queue[0] not in self.camera.regions_in_process:
self.camera.last_processed_frame = self.camera.frame_queue.get() # self.camera.last_processed_frame = self.camera.frame_queue.get()
self.camera.refined_frame_queue.put(self.camera.last_processed_frame) # self.camera.refined_frame_queue.put(self.camera.last_processed_frame)
def filtered(self, obj): # def filtered(self, obj):
object_name = obj['name'] # object_name = obj['name']
if object_name in self.camera.object_filters: # if object_name in self.camera.object_filters:
obj_settings = self.camera.object_filters[object_name] # obj_settings = self.camera.object_filters[object_name]
# if the min area is larger than the # # if the min area is larger than the
# detected object, don't add it to detected objects # # detected object, don't add it to detected objects
if obj_settings.get('min_area',-1) > obj['area']: # if obj_settings.get('min_area',-1) > obj['area']:
return True # return True
# if the detected object is larger than the # # if the detected object is larger than the
# max area, don't add it to detected objects # # max area, don't add it to detected objects
if obj_settings.get('max_area', self.camera.frame_shape[0]*self.camera.frame_shape[1]) < obj['area']: # if obj_settings.get('max_area', self.camera.frame_shape[0]*self.camera.frame_shape[1]) < obj['area']:
return True # return True
# if the score is lower than the threshold, skip # # if the score is lower than the threshold, skip
if obj_settings.get('threshold', 0) > obj['score']: # if obj_settings.get('threshold', 0) > obj['score']:
return True # return True
# compute the coordinates of the object and make sure # # compute the coordinates of the object and make sure
# the location isnt outside the bounds of the image (can happen from rounding) # # the location isnt outside the bounds of the image (can happen from rounding)
y_location = min(int(obj['box']['ymax']), len(self.camera.mask)-1) # y_location = min(int(obj['box']['ymax']), len(self.camera.mask)-1)
x_location = min(int((obj['box']['xmax']-obj['box']['xmin'])/2.0)+obj['box']['xmin'], len(self.camera.mask[0])-1) # x_location = min(int((obj['box']['xmax']-obj['box']['xmin'])/2.0)+obj['box']['xmin'], len(self.camera.mask[0])-1)
# if the object is in a masked location, don't add it to detected objects # # if the object is in a masked location, don't add it to detected objects
if self.camera.mask[y_location][x_location] == [0]: # if self.camera.mask[y_location][x_location] == [0]:
return True # return True
return False # return False
def has_overlap(self, new_obj, obj, overlap=.7): # def has_overlap(self, new_obj, obj, overlap=.7):
# compute intersection rectangle with existing object and new objects region # # compute intersection rectangle with existing object and new objects region
existing_obj_current_region = compute_intersection_rectangle(obj['box'], new_obj['region']) # existing_obj_current_region = compute_intersection_rectangle(obj['box'], new_obj['region'])
# compute intersection rectangle with new object and existing objects region # # compute intersection rectangle with new object and existing objects region
new_obj_existing_region = compute_intersection_rectangle(new_obj['box'], obj['region']) # new_obj_existing_region = compute_intersection_rectangle(new_obj['box'], obj['region'])
# compute iou for the two intersection rectangles that were just computed # # compute iou for the two intersection rectangles that were just computed
iou = compute_intersection_over_union(existing_obj_current_region, new_obj_existing_region) # iou = compute_intersection_over_union(existing_obj_current_region, new_obj_existing_region)
# if intersection is greater than overlap # # if intersection is greater than overlap
if iou > overlap: # if iou > overlap:
return True # return True
else: # else:
return False # return False
def find_group(self, new_obj, groups): # def find_group(self, new_obj, groups):
for index, group in enumerate(groups): # for index, group in enumerate(groups):
for obj in group: # for obj in group:
if self.has_overlap(new_obj, obj): # if self.has_overlap(new_obj, obj):
return index # return index
return None # return None
class ObjectTracker(threading.Thread): class ObjectTracker():
def __init__(self, camera, max_disappeared): def __init__(self, max_disappeared):
threading.Thread.__init__(self)
self.camera = camera
self.tracked_objects = {} self.tracked_objects = {}
self.tracked_objects_lock = mp.Lock() self.disappeared = {}
self.most_recent_frame_time = None self.max_disappeared = max_disappeared
def run(self):
prctl.set_name(self.__class__.__name__)
while True:
frame_time = self.camera.refined_frame_queue.get()
with self.tracked_objects_lock:
self.match_and_update(self.camera.detected_objects[frame_time])
self.most_recent_frame_time = frame_time
self.camera.frame_output_queue.put((frame_time, copy.deepcopy(self.tracked_objects)))
if len(self.tracked_objects) > 0:
with self.camera.objects_tracked:
self.camera.objects_tracked.notify_all()
def register(self, index, obj): def register(self, index, obj):
id = "{}-{}".format(str(obj['frame_time']), index) id = f"{obj['frame_time']}-{index}"
obj['id'] = id obj['id'] = id
obj['top_score'] = obj['score'] obj['top_score'] = obj['score']
self.add_history(obj) self.add_history(obj)
self.tracked_objects[id] = obj self.tracked_objects[id] = obj
self.disappeared[id] = 0
def deregister(self, id): def deregister(self, id):
del self.tracked_objects[id] del self.tracked_objects[id]
del self.disappeared[id]
def update(self, id, new_obj): def update(self, id, new_obj):
self.disappeared[id] = 0
self.tracked_objects[id].update(new_obj) self.tracked_objects[id].update(new_obj)
self.add_history(self.tracked_objects[id]) self.add_history(self.tracked_objects[id])
if self.tracked_objects[id]['score'] > self.tracked_objects[id]['top_score']: if self.tracked_objects[id]['score'] > self.tracked_objects[id]['top_score']:
@ -291,25 +280,37 @@ class ObjectTracker(threading.Thread):
else: else:
obj['history'] = [entry] obj['history'] = [entry]
def match_and_update(self, new_objects): def match_and_update(self, frame_time, new_objects):
if len(new_objects) == 0: if len(new_objects) == 0:
for id in list(self.tracked_objects.keys()):
if self.disappeared[id] >= self.max_disappeared:
self.deregister(id)
else:
self.disappeared[id] += 1
return return
# group by name # group by name
new_object_groups = defaultdict(lambda: []) new_object_groups = defaultdict(lambda: [])
for obj in new_objects: for obj in new_objects:
new_object_groups[obj['name']].append(obj) new_object_groups[obj[0]].append({
'label': obj[0],
'score': obj[1],
'box': obj[2],
'area': obj[3],
'region': obj[4],
'frame_time': frame_time
})
# track objects for each label type # track objects for each label type
for label, group in new_object_groups.items(): for label, group in new_object_groups.items():
current_objects = [o for o in self.tracked_objects.values() if o['name'] == label] current_objects = [o for o in self.tracked_objects.values() if o['label'] == label]
current_ids = [o['id'] for o in current_objects] current_ids = [o['id'] for o in current_objects]
current_centroids = np.array([o['centroid'] for o in current_objects]) current_centroids = np.array([o['centroid'] for o in current_objects])
# compute centroids of new objects # compute centroids of new objects
for obj in group: for obj in group:
centroid_x = int((obj['box']['xmin']+obj['box']['xmax']) / 2.0) centroid_x = int((obj['box'][0]+obj['box'][2]) / 2.0)
centroid_y = int((obj['box']['ymin']+obj['box']['ymax']) / 2.0) centroid_y = int((obj['box'][1]+obj['box'][3]) / 2.0)
obj['centroid'] = (centroid_x, centroid_y) obj['centroid'] = (centroid_x, centroid_y)
if len(current_objects) == 0: if len(current_objects) == 0:
@ -363,56 +364,66 @@ class ObjectTracker(threading.Thread):
usedCols.add(col) usedCols.add(col)
# compute the column index we have NOT yet examined # compute the column index we have NOT yet examined
unusedRows = set(range(0, D.shape[0])).difference(usedRows)
unusedCols = set(range(0, D.shape[1])).difference(usedCols) unusedCols = set(range(0, D.shape[1])).difference(usedCols)
# in the event that the number of object centroids is
# equal or greater than the number of input centroids
# we need to check and see if some of these objects have
# potentially disappeared
if D.shape[0] >= D.shape[1]:
for row in unusedRows:
id = current_ids[row]
if self.disappeared[id] >= self.max_disappeared:
self.deregister(id)
else:
self.disappeared[id] += 1
# if the number of input centroids is greater # if the number of input centroids is greater
# than the number of existing object centroids we need to # than the number of existing object centroids we need to
# register each new input centroid as a trackable object # register each new input centroid as a trackable object
# if D.shape[0] < D.shape[1]: else:
# TODO: rather than assuming these are new objects, we could
# look to see if any of the remaining boxes have a large amount
# of overlap...
for col in unusedCols: for col in unusedCols:
self.register(col, group[col]) self.register(col, group[col])
# Maintains the frame and object with the highest score # Maintains the frame and object with the highest score
class BestFrames(threading.Thread): # class BestFrames(threading.Thread):
def __init__(self, camera): # def __init__(self, camera):
threading.Thread.__init__(self) # threading.Thread.__init__(self)
self.camera = camera # self.camera = camera
self.best_objects = {} # self.best_objects = {}
self.best_frames = {} # self.best_frames = {}
def run(self): # def run(self):
prctl.set_name(self.__class__.__name__) # prctl.set_name(self.__class__.__name__)
while True: # while True:
# wait until objects have been tracked # # wait until objects have been tracked
with self.camera.objects_tracked: # with self.camera.objects_tracked:
self.camera.objects_tracked.wait() # self.camera.objects_tracked.wait()
# make a copy of tracked objects # # make a copy of tracked objects
tracked_objects = list(self.camera.object_tracker.tracked_objects.values()) # tracked_objects = list(self.camera.object_tracker.tracked_objects.values())
for obj in tracked_objects: # for obj in tracked_objects:
if obj['name'] in self.best_objects: # if obj['name'] in self.best_objects:
now = datetime.datetime.now().timestamp() # now = datetime.datetime.now().timestamp()
# if the object is a higher score than the current best score # # if the object is a higher score than the current best score
# or the current object is more than 1 minute old, use the new object # # or the current object is more than 1 minute old, use the new object
if obj['score'] > self.best_objects[obj['name']]['score'] or (now - self.best_objects[obj['name']]['frame_time']) > 60: # if obj['score'] > self.best_objects[obj['name']]['score'] or (now - self.best_objects[obj['name']]['frame_time']) > 60:
self.best_objects[obj['name']] = copy.deepcopy(obj) # self.best_objects[obj['name']] = copy.deepcopy(obj)
else: # else:
self.best_objects[obj['name']] = copy.deepcopy(obj) # self.best_objects[obj['name']] = copy.deepcopy(obj)
for name, obj in self.best_objects.items(): # for name, obj in self.best_objects.items():
if obj['frame_time'] in self.camera.frame_cache: # if obj['frame_time'] in self.camera.frame_cache:
best_frame = self.camera.frame_cache[obj['frame_time']] # best_frame = self.camera.frame_cache[obj['frame_time']]
draw_box_with_label(best_frame, obj['box']['xmin'], obj['box']['ymin'], # draw_box_with_label(best_frame, obj['box']['xmin'], obj['box']['ymin'],
obj['box']['xmax'], obj['box']['ymax'], obj['name'], "{}% {}".format(int(obj['score']*100), obj['area'])) # obj['box']['xmax'], obj['box']['ymax'], obj['name'], "{}% {}".format(int(obj['score']*100), obj['area']))
# print a timestamp # # print a timestamp
if self.camera.snapshot_config['show_timestamp']: # if self.camera.snapshot_config['show_timestamp']:
time_to_show = datetime.datetime.fromtimestamp(obj['frame_time']).strftime("%m/%d/%Y %H:%M:%S") # time_to_show = datetime.datetime.fromtimestamp(obj['frame_time']).strftime("%m/%d/%Y %H:%M:%S")
cv2.putText(best_frame, time_to_show, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2) # cv2.putText(best_frame, time_to_show, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
self.best_frames[name] = best_frame # self.best_frames[name] = best_frame

160
frigate/util.py Normal file → Executable file
View File

@ -15,73 +15,11 @@ def ReadLabelFile(file_path):
ret[int(pair[0])] = pair[1].strip() ret[int(pair[0])] = pair[1].strip()
return ret return ret
def calculate_region(frame_shape, xmin, ymin, xmax, ymax):
# size is larger than longest edge
size = int(max(xmax-xmin, ymax-ymin)*2)
# if the size is too big to fit in the frame
if size > min(frame_shape[0], frame_shape[1]):
size = min(frame_shape[0], frame_shape[1])
# x_offset is midpoint of bounding box minus half the size
x_offset = int((xmax-xmin)/2.0+xmin-size/2.0)
# if outside the image
if x_offset < 0:
x_offset = 0
elif x_offset > (frame_shape[1]-size):
x_offset = (frame_shape[1]-size)
# y_offset is midpoint of bounding box minus half the size
y_offset = int((ymax-ymin)/2.0+ymin-size/2.0)
# if outside the image
if y_offset < 0:
y_offset = 0
elif y_offset > (frame_shape[0]-size):
y_offset = (frame_shape[0]-size)
return (size, x_offset, y_offset)
def compute_intersection_rectangle(box_a, box_b):
return {
'xmin': max(box_a['xmin'], box_b['xmin']),
'ymin': max(box_a['ymin'], box_b['ymin']),
'xmax': min(box_a['xmax'], box_b['xmax']),
'ymax': min(box_a['ymax'], box_b['ymax'])
}
def compute_intersection_over_union(box_a, box_b):
# determine the (x, y)-coordinates of the intersection rectangle
intersect = compute_intersection_rectangle(box_a, box_b)
# compute the area of intersection rectangle
inter_area = max(0, intersect['xmax'] - intersect['xmin'] + 1) * max(0, intersect['ymax'] - intersect['ymin'] + 1)
if inter_area == 0:
return 0.0
# compute the area of both the prediction and ground-truth
# rectangles
box_a_area = (box_a['xmax'] - box_a['xmin'] + 1) * (box_a['ymax'] - box_a['ymin'] + 1)
box_b_area = (box_b['xmax'] - box_b['xmin'] + 1) * (box_b['ymax'] - box_b['ymin'] + 1)
# compute the intersection over union by taking the intersection
# area and dividing it by the sum of prediction + ground-truth
# areas - the interesection area
iou = inter_area / float(box_a_area + box_b_area - inter_area)
# return the intersection over union value
return iou
# convert shared memory array into numpy array
def tonumpyarray(mp_arr):
return np.frombuffer(mp_arr.get_obj(), dtype=np.uint8)
def draw_box_with_label(frame, x_min, y_min, x_max, y_max, label, info, thickness=2, color=None, position='ul'): def draw_box_with_label(frame, x_min, y_min, x_max, y_max, label, info, thickness=2, color=None, position='ul'):
if color is None: if color is None:
color = COLOR_MAP[label] color = (0,0,255)
display_text = "{}: {}".format(label, info) display_text = "{}: {}".format(label, info)
cv2.rectangle(frame, (x_min, y_min), cv2.rectangle(frame, (x_min, y_min), (x_max, y_max), color, thickness)
(x_max, y_max),
color, thickness)
font_scale = 0.5 font_scale = 0.5
font = cv2.FONT_HERSHEY_SIMPLEX font = cv2.FONT_HERSHEY_SIMPLEX
# get the width and height of the text box # get the width and height of the text box
@ -107,37 +45,81 @@ def draw_box_with_label(frame, x_min, y_min, x_max, y_max, label, info, thicknes
cv2.rectangle(frame, textbox_coords[0], textbox_coords[1], color, cv2.FILLED) cv2.rectangle(frame, textbox_coords[0], textbox_coords[1], color, cv2.FILLED)
cv2.putText(frame, display_text, (text_offset_x, text_offset_y + line_height - 3), font, fontScale=font_scale, color=(0, 0, 0), thickness=2) cv2.putText(frame, display_text, (text_offset_x, text_offset_y + line_height - 3), font, fontScale=font_scale, color=(0, 0, 0), thickness=2)
# Path to frozen detection graph. This is the actual model that is used for the object detection. def calculate_region(frame_shape, xmin, ymin, xmax, ymax, multiplier=2):
PATH_TO_CKPT = '/frozen_inference_graph.pb' # size is larger than longest edge
# List of the strings that is used to add correct label for each box. size = int(max(xmax-xmin, ymax-ymin)*multiplier)
PATH_TO_LABELS = '/label_map.pbtext' # if the size is too big to fit in the frame
if size > min(frame_shape[0], frame_shape[1]):
size = min(frame_shape[0], frame_shape[1])
LABELS = ReadLabelFile(PATH_TO_LABELS) # x_offset is midpoint of bounding box minus half the size
cmap = plt.cm.get_cmap('tab10', len(LABELS.keys())) x_offset = int((xmax-xmin)/2.0+xmin-size/2.0)
# if outside the image
if x_offset < 0:
x_offset = 0
elif x_offset > (frame_shape[1]-size):
x_offset = (frame_shape[1]-size)
COLOR_MAP = {} # y_offset is midpoint of bounding box minus half the size
for key, val in LABELS.items(): y_offset = int((ymax-ymin)/2.0+ymin-size/2.0)
COLOR_MAP[val] = tuple(int(round(255 * c)) for c in cmap(key)[:3]) # if outside the image
if y_offset < 0:
y_offset = 0
elif y_offset > (frame_shape[0]-size):
y_offset = (frame_shape[0]-size)
class QueueMerger(): return (x_offset, y_offset, x_offset+size, y_offset+size)
def __init__(self, from_queues, to_queue):
self.from_queues = from_queues
self.to_queue = to_queue
self.merge_threads = []
def start(self): def intersection(box_a, box_b):
for from_q in self.from_queues: return (
self.merge_threads.append(QueueTransfer(from_q,self.to_queue)) max(box_a[0], box_b[0]),
max(box_a[1], box_b[1]),
min(box_a[2], box_b[2]),
min(box_a[3], box_b[3])
)
class QueueTransfer(threading.Thread): def area(box):
def __init__(self, from_queue, to_queue): return (box[2]-box[0] + 1)*(box[3]-box[1] + 1)
threading.Thread.__init__(self)
self.from_queue = from_queue
self.to_queue = to_queue
def run(self): def intersection_over_union(box_a, box_b):
while True: # determine the (x, y)-coordinates of the intersection rectangle
self.to_queue.put(self.from_queue.get()) intersect = intersection(box_a, box_b)
# compute the area of intersection rectangle
inter_area = max(0, intersect[2] - intersect[0] + 1) * max(0, intersect[3] - intersect[1] + 1)
if inter_area == 0:
return 0.0
# compute the area of both the prediction and ground-truth
# rectangles
box_a_area = (box_a[2] - box_a[0] + 1) * (box_a[3] - box_a[1] + 1)
box_b_area = (box_b[2] - box_b[0] + 1) * (box_b[3] - box_b[1] + 1)
# compute the intersection over union by taking the intersection
# area and dividing it by the sum of prediction + ground-truth
# areas - the interesection area
iou = inter_area / float(box_a_area + box_b_area - inter_area)
# return the intersection over union value
return iou
def clipped(obj, frame_shape):
# if the object is within 5 pixels of the region border, and the region is not on the edge
# consider the object to be clipped
box = obj[2]
region = obj[4]
if ((region[0] > 5 and box[0]-region[0] <= 5) or
(region[1] > 5 and box[1]-region[1] <= 5) or
(frame_shape[1]-region[2] > 5 and region[2]-box[2] <= 5) or
(frame_shape[0]-region[3] > 5 and region[3]-box[3] <= 5)):
return True
else:
return False
# convert shared memory array into numpy array
def tonumpyarray(mp_arr):
return np.frombuffer(mp_arr.get_obj(), dtype=np.uint8)
class EventsPerSecond: class EventsPerSecond:
def __init__(self, max_events=1000): def __init__(self, max_events=1000):

748
frigate/video.py Normal file → Executable file
View File

@ -8,40 +8,47 @@ import ctypes
import multiprocessing as mp import multiprocessing as mp
import subprocess as sp import subprocess as sp
import numpy as np import numpy as np
import prctl import hashlib
import pyarrow.plasma as plasma
import SharedArray as sa
# import prctl
import copy import copy
import itertools import itertools
import json import json
from collections import defaultdict from collections import defaultdict
from frigate.util import tonumpyarray, LABELS, draw_box_with_label, calculate_region, EventsPerSecond from frigate.util import tonumpyarray, LABELS, draw_box_with_label, area, calculate_region, clipped, intersection_over_union, intersection, EventsPerSecond
from frigate.object_detection import RegionPrepper, RegionRequester # from frigate.object_detection import RegionPrepper, RegionRequester
from frigate.objects import ObjectCleaner, BestFrames, DetectedObjectsProcessor, RegionRefiner, ObjectTracker from frigate.objects import ObjectTracker
from frigate.mqtt import MqttObjectPublisher # from frigate.mqtt import MqttObjectPublisher
from frigate.edgetpu import RemoteObjectDetector
from frigate.motion import MotionDetector
# Stores 2 seconds worth of frames so they can be used for other threads # Stores 2 seconds worth of frames so they can be used for other threads
class FrameTracker(threading.Thread): # TODO: we do actually know when these frames are no longer needed
def __init__(self, frame_time, frame_ready, frame_lock, recent_frames): # class FrameTracker(threading.Thread):
threading.Thread.__init__(self) # def __init__(self, frame_time, frame_ready, frame_lock, recent_frames):
self.frame_time = frame_time # threading.Thread.__init__(self)
self.frame_ready = frame_ready # self.frame_time = frame_time
self.frame_lock = frame_lock # self.frame_ready = frame_ready
self.recent_frames = recent_frames # self.frame_lock = frame_lock
# self.recent_frames = recent_frames
def run(self): # def run(self):
prctl.set_name(self.__class__.__name__) # prctl.set_name(self.__class__.__name__)
while True: # while True:
# wait for a frame # # wait for a frame
with self.frame_ready: # with self.frame_ready:
self.frame_ready.wait() # self.frame_ready.wait()
# delete any old frames # # delete any old frames
stored_frame_times = list(self.recent_frames.keys()) # stored_frame_times = list(self.recent_frames.keys())
stored_frame_times.sort(reverse=True) # stored_frame_times.sort(reverse=True)
if len(stored_frame_times) > 100: # if len(stored_frame_times) > 100:
frames_to_delete = stored_frame_times[50:] # frames_to_delete = stored_frame_times[50:]
for k in frames_to_delete: # for k in frames_to_delete:
del self.recent_frames[k] # del self.recent_frames[k]
# TODO: add back opencv fallback
def get_frame_shape(source): def get_frame_shape(source):
ffprobe_cmd = " ".join([ ffprobe_cmd = " ".join([
'ffprobe', 'ffprobe',
@ -76,6 +83,7 @@ def get_ffmpeg_input(ffmpeg_input):
frigate_vars = {k: v for k, v in os.environ.items() if k.startswith('FRIGATE_')} frigate_vars = {k: v for k, v in os.environ.items() if k.startswith('FRIGATE_')}
return ffmpeg_input.format(**frigate_vars) return ffmpeg_input.format(**frigate_vars)
<<<<<<< HEAD
class CameraWatchdog(threading.Thread): class CameraWatchdog(threading.Thread):
def __init__(self, camera): def __init__(self, camera):
threading.Thread.__init__(self) threading.Thread.__init__(self)
@ -294,112 +302,648 @@ class Camera:
self.capture_thread.join() self.capture_thread.join()
self.ffmpeg_process = None self.ffmpeg_process = None
self.capture_thread = None self.capture_thread = None
=======
# class CameraWatchdog(threading.Thread):
# def __init__(self, camera):
# threading.Thread.__init__(self)
# self.camera = camera
# create the process to capture frames from the input stream and store in a shared array # def run(self):
print("Creating a new ffmpeg process...") # prctl.set_name(self.__class__.__name__)
self.start_ffmpeg() # while True:
# # wait a bit before checking
# time.sleep(10)
print("Creating a new capture thread...") # if self.camera.frame_time.value != 0.0 and (datetime.datetime.now().timestamp() - self.camera.frame_time.value) > self.camera.watchdog_timeout:
self.capture_thread = CameraCapture(self) # print(self.camera.name + ": last frame is more than 5 minutes old, restarting camera capture...")
print("Starting a new capture thread...") # self.camera.start_or_restart_capture()
self.capture_thread.start() # time.sleep(5)
self.fps.start()
self.skipped_region_tracker.start() # # Thread to read the stdout of the ffmpeg process and update the current frame
# class CameraCapture(threading.Thread):
# def __init__(self, camera):
# threading.Thread.__init__(self)
# self.camera = camera
# def run(self):
# prctl.set_name(self.__class__.__name__)
# frame_num = 0
# while True:
# if self.camera.ffmpeg_process.poll() != None:
# print(self.camera.name + ": ffmpeg process is not running. exiting capture thread...")
# break
# raw_image = self.camera.ffmpeg_process.stdout.read(self.camera.frame_size)
# if len(raw_image) == 0:
# print(self.camera.name + ": ffmpeg didnt return a frame. something is wrong. exiting capture thread...")
# break
# frame_num += 1
# if (frame_num % self.camera.take_frame) != 0:
# continue
# with self.camera.frame_lock:
# # TODO: use frame_queue instead
# self.camera.frame_time.value = datetime.datetime.now().timestamp()
# self.camera.frame_cache[self.camera.frame_time.value] = (
# np
# .frombuffer(raw_image, np.uint8)
# .reshape(self.camera.frame_shape)
# )
# self.camera.frame_queue.put(self.camera.frame_time.value)
# # Notify with the condition that a new frame is ready
# with self.camera.frame_ready:
# self.camera.frame_ready.notify_all()
# self.camera.fps.update()
# class VideoWriter(threading.Thread):
# def __init__(self, camera):
# threading.Thread.__init__(self)
# self.camera = camera
# def run(self):
# prctl.set_name(self.__class__.__name__)
# while True:
# (frame_time, tracked_objects) = self.camera.frame_output_queue.get()
# # if len(tracked_objects) == 0:
# # continue
# # f = open(f"/debug/output/{self.camera.name}-{str(format(frame_time, '.8f'))}.jpg", 'wb')
# # f.write(self.camera.frame_with_objects(frame_time, tracked_objects))
# # f.close()
# class Camera:
# def __init__(self, name, ffmpeg_config, global_objects_config, config, tflite_process, mqtt_client, mqtt_prefix):
# self.name = name
# self.config = config
# self.detected_objects = defaultdict(lambda: [])
# self.frame_cache = {}
# self.last_processed_frame = None
# # queue for re-assembling frames in order
# self.frame_queue = queue.Queue()
# # track how many regions have been requested for a frame so we know when a frame is complete
# self.regions_in_process = {}
# # Lock to control access
# self.regions_in_process_lock = mp.Lock()
# self.finished_frame_queue = queue.Queue()
# self.refined_frame_queue = queue.Queue()
# self.frame_output_queue = queue.Queue()
# self.ffmpeg = config.get('ffmpeg', {})
# self.ffmpeg_input = get_ffmpeg_input(self.ffmpeg['input'])
# self.ffmpeg_global_args = self.ffmpeg.get('global_args', ffmpeg_config['global_args'])
# self.ffmpeg_hwaccel_args = self.ffmpeg.get('hwaccel_args', ffmpeg_config['hwaccel_args'])
# self.ffmpeg_input_args = self.ffmpeg.get('input_args', ffmpeg_config['input_args'])
# self.ffmpeg_output_args = self.ffmpeg.get('output_args', ffmpeg_config['output_args'])
# camera_objects_config = config.get('objects', {})
# self.take_frame = self.config.get('take_frame', 1)
# self.watchdog_timeout = self.config.get('watchdog_timeout', 300)
# self.snapshot_config = {
# 'show_timestamp': self.config.get('snapshots', {}).get('show_timestamp', True)
# }
# self.regions = self.config['regions']
# self.frame_shape = get_frame_shape(self.ffmpeg_input)
# self.frame_size = self.frame_shape[0] * self.frame_shape[1] * self.frame_shape[2]
# self.mqtt_client = mqtt_client
# self.mqtt_topic_prefix = '{}/{}'.format(mqtt_prefix, self.name)
# # create shared value for storing the frame_time
# self.frame_time = mp.Value('d', 0.0)
# # Lock to control access to the frame
# self.frame_lock = mp.Lock()
# # Condition for notifying that a new frame is ready
# self.frame_ready = mp.Condition()
# # Condition for notifying that objects were tracked
# self.objects_tracked = mp.Condition()
# # Queue for prepped frames, max size set to (number of regions * 5)
# self.resize_queue = queue.Queue()
# # Queue for raw detected objects
# self.detected_objects_queue = queue.Queue()
# self.detected_objects_processor = DetectedObjectsProcessor(self)
# self.detected_objects_processor.start()
# # initialize the frame cache
# self.cached_frame_with_objects = {
# 'frame_bytes': [],
# 'frame_time': 0
# }
# self.ffmpeg_process = None
# self.capture_thread = None
# self.fps = EventsPerSecond()
# self.skipped_region_tracker = EventsPerSecond()
# # combine tracked objects lists
# self.objects_to_track = set().union(global_objects_config.get('track', ['person', 'car', 'truck']), camera_objects_config.get('track', []))
# # merge object filters
# global_object_filters = global_objects_config.get('filters', {})
# camera_object_filters = camera_objects_config.get('filters', {})
# objects_with_config = set().union(global_object_filters.keys(), camera_object_filters.keys())
# self.object_filters = {}
# for obj in objects_with_config:
# self.object_filters[obj] = {**global_object_filters.get(obj, {}), **camera_object_filters.get(obj, {})}
# # start a thread to track objects
# self.object_tracker = ObjectTracker(self, 10)
# self.object_tracker.start()
# # start a thread to write tracked frames to disk
# self.video_writer = VideoWriter(self)
# self.video_writer.start()
# # start a thread to queue resize requests for regions
# self.region_requester = RegionRequester(self)
# self.region_requester.start()
# # start a thread to cache recent frames for processing
# self.frame_tracker = FrameTracker(self.frame_time,
# self.frame_ready, self.frame_lock, self.frame_cache)
# self.frame_tracker.start()
# # start a thread to resize regions
# self.region_prepper = RegionPrepper(self, self.frame_cache, self.resize_queue, prepped_frame_queue)
# self.region_prepper.start()
# # start a thread to store the highest scoring recent frames for monitored object types
# self.best_frames = BestFrames(self)
# self.best_frames.start()
# # start a thread to expire objects from the detected objects list
# self.object_cleaner = ObjectCleaner(self)
# self.object_cleaner.start()
# # start a thread to refine regions when objects are clipped
# self.dynamic_region_fps = EventsPerSecond()
# self.region_refiner = RegionRefiner(self)
# self.region_refiner.start()
# self.dynamic_region_fps.start()
# # start a thread to publish object scores
# mqtt_publisher = MqttObjectPublisher(self.mqtt_client, self.mqtt_topic_prefix, self)
# mqtt_publisher.start()
# # create a watchdog thread for capture process
# self.watchdog = CameraWatchdog(self)
# # load in the mask for object detection
# if 'mask' in self.config:
# self.mask = cv2.imread("/config/{}".format(self.config['mask']), cv2.IMREAD_GRAYSCALE)
# else:
# self.mask = None
# if self.mask is None:
# self.mask = np.zeros((self.frame_shape[0], self.frame_shape[1], 1), np.uint8)
# self.mask[:] = 255
# def start_or_restart_capture(self):
# if not self.ffmpeg_process is None:
# print("Terminating the existing ffmpeg process...")
# self.ffmpeg_process.terminate()
# try:
# print("Waiting for ffmpeg to exit gracefully...")
# self.ffmpeg_process.wait(timeout=30)
# except sp.TimeoutExpired:
# print("FFmpeg didnt exit. Force killing...")
# self.ffmpeg_process.kill()
# self.ffmpeg_process.wait()
# print("Waiting for the capture thread to exit...")
# self.capture_thread.join()
# self.ffmpeg_process = None
# self.capture_thread = None
>>>>>>> 9b1c7e9... split into separate processes
# # create the process to capture frames from the input stream and store in a shared array
# print("Creating a new ffmpeg process...")
# self.start_ffmpeg()
# print("Creating a new capture thread...")
# self.capture_thread = CameraCapture(self)
# print("Starting a new capture thread...")
# self.capture_thread.start()
# self.fps.start()
# self.skipped_region_tracker.start()
# def start_ffmpeg(self):
# ffmpeg_cmd = (['ffmpeg'] +
# self.ffmpeg_global_args +
# self.ffmpeg_hwaccel_args +
# self.ffmpeg_input_args +
# ['-i', self.ffmpeg_input] +
# self.ffmpeg_output_args +
# ['pipe:'])
# print(" ".join(ffmpeg_cmd))
# self.ffmpeg_process = sp.Popen(ffmpeg_cmd, stdout = sp.PIPE, bufsize=self.frame_size)
# def start(self):
# self.start_or_restart_capture()
# self.watchdog.start()
# def join(self):
# self.capture_thread.join()
# def get_capture_pid(self):
# return self.ffmpeg_process.pid
# def get_best(self, label):
# return self.best_frames.best_frames.get(label)
# def stats(self):
# # TODO: anything else?
# return {
# 'camera_fps': self.fps.eps(60),
# 'resize_queue': self.resize_queue.qsize(),
# 'frame_queue': self.frame_queue.qsize(),
# 'finished_frame_queue': self.finished_frame_queue.qsize(),
# 'refined_frame_queue': self.refined_frame_queue.qsize(),
# 'regions_in_process': self.regions_in_process,
# 'dynamic_regions_per_sec': self.dynamic_region_fps.eps(),
# 'skipped_regions_per_sec': self.skipped_region_tracker.eps(60)
# }
# def frame_with_objects(self, frame_time, tracked_objects=None):
# if not frame_time in self.frame_cache:
# frame = np.zeros(self.frame_shape, np.uint8)
# else:
# frame = self.frame_cache[frame_time].copy()
# detected_objects = self.detected_objects[frame_time].copy()
# for region in self.regions:
# color = (255,255,255)
# cv2.rectangle(frame, (region['x_offset'], region['y_offset']),
# (region['x_offset']+region['size'], region['y_offset']+region['size']),
# color, 2)
# # draw the bounding boxes on the screen
# if tracked_objects is None:
# with self.object_tracker.tracked_objects_lock:
# tracked_objects = copy.deepcopy(self.object_tracker.tracked_objects)
# for obj in detected_objects:
# draw_box_with_label(frame, obj['box']['xmin'], obj['box']['ymin'], obj['box']['xmax'], obj['box']['ymax'], obj['name'], "{}% {}".format(int(obj['score']*100), obj['area']), thickness=3)
# for id, obj in tracked_objects.items():
# color = (0, 255,0) if obj['frame_time'] == frame_time else (255, 0, 0)
# draw_box_with_label(frame, obj['box']['xmin'], obj['box']['ymin'], obj['box']['xmax'], obj['box']['ymax'], obj['name'], id, color=color, thickness=1, position='bl')
# # print a timestamp
# time_to_show = datetime.datetime.fromtimestamp(frame_time).strftime("%m/%d/%Y %H:%M:%S")
# cv2.putText(frame, time_to_show, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
# # print fps
# cv2.putText(frame, str(self.fps.eps())+'FPS', (10, 60), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
# # convert to BGR
# frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
# # encode the image into a jpg
# ret, jpg = cv2.imencode('.jpg', frame)
# return jpg.tobytes()
# def get_current_frame_with_objects(self):
# frame_time = self.last_processed_frame
# if frame_time == self.cached_frame_with_objects['frame_time']:
# return self.cached_frame_with_objects['frame_bytes']
# frame_bytes = self.frame_with_objects(frame_time)
# self.cached_frame_with_objects = {
# 'frame_bytes': frame_bytes,
# 'frame_time': frame_time
# }
# return frame_bytes
def filtered(obj, objects_to_track, object_filters, mask):
object_name = obj[0]
if not object_name in objects_to_track:
return True
if object_name in object_filters:
obj_settings = object_filters[object_name]
# if the min area is larger than the
# detected object, don't add it to detected objects
if obj_settings.get('min_area',-1) > obj[3]:
return True
# if the detected object is larger than the
# max area, don't add it to detected objects
if obj_settings.get('max_area', 24000000) < obj[3]:
return True
# if the score is lower than the threshold, skip
if obj_settings.get('threshold', 0) > obj[1]:
return True
# compute the coordinates of the object and make sure
# the location isnt outside the bounds of the image (can happen from rounding)
y_location = min(int(obj[2][3]), len(mask)-1)
x_location = min(int((obj[2][2]-obj[2][0])/2.0)+obj[2][0], len(mask[0])-1)
# if the object is in a masked location, don't add it to detected objects
if mask[y_location][x_location] == [0]:
return True
return False
def create_tensor_input(frame, region):
cropped_frame = frame[region[1]:region[3], region[0]:region[2]]
# Resize to 300x300 if needed
if cropped_frame.shape != (300, 300, 3):
cropped_frame = cv2.resize(cropped_frame, dsize=(300, 300), interpolation=cv2.INTER_LINEAR)
# Expand dimensions since the model expects images to have shape: [1, 300, 300, 3]
return np.expand_dims(cropped_frame, axis=0)
def track_camera(name, config, ffmpeg_global_config, global_objects_config, detect_lock, detect_ready, frame_ready, detected_objects_queue, fps, avg_wait):
print(f"Starting process for {name}: {os.getpid()}")
# Merge the ffmpeg config with the global config
ffmpeg = config.get('ffmpeg', {})
ffmpeg_input = get_ffmpeg_input(ffmpeg['input'])
ffmpeg_global_args = ffmpeg.get('global_args', ffmpeg_global_config['global_args'])
ffmpeg_hwaccel_args = ffmpeg.get('hwaccel_args', ffmpeg_global_config['hwaccel_args'])
ffmpeg_input_args = ffmpeg.get('input_args', ffmpeg_global_config['input_args'])
ffmpeg_output_args = ffmpeg.get('output_args', ffmpeg_global_config['output_args'])
# Merge the tracked object config with the global config
camera_objects_config = config.get('objects', {})
# combine tracked objects lists
objects_to_track = set().union(global_objects_config.get('track', ['person', 'car', 'truck']), camera_objects_config.get('track', []))
# merge object filters
global_object_filters = global_objects_config.get('filters', {})
camera_object_filters = camera_objects_config.get('filters', {})
objects_with_config = set().union(global_object_filters.keys(), camera_object_filters.keys())
object_filters = {}
for obj in objects_with_config:
object_filters[obj] = {**global_object_filters.get(obj, {}), **camera_object_filters.get(obj, {})}
take_frame = config.get('take_frame', 1)
# watchdog_timeout = config.get('watchdog_timeout', 300)
frame_shape = get_frame_shape(ffmpeg_input)
frame_size = frame_shape[0] * frame_shape[1] * frame_shape[2]
try:
sa.delete(name)
except:
pass
frame = sa.create(name, shape=frame_shape, dtype=np.uint8)
# load in the mask for object detection
if 'mask' in config:
mask = cv2.imread("/config/{}".format(config['mask']), cv2.IMREAD_GRAYSCALE)
else:
mask = None
if mask is None:
mask = np.zeros((frame_shape[0], frame_shape[1], 1), np.uint8)
mask[:] = 255
motion_detector = MotionDetector(frame_shape, mask, resize_factor=6)
object_detector = RemoteObjectDetector('/lab/labelmap.txt', detect_lock, detect_ready, frame_ready)
object_tracker = ObjectTracker(10)
def start_ffmpeg(self):
ffmpeg_cmd = (['ffmpeg'] + ffmpeg_cmd = (['ffmpeg'] +
self.ffmpeg_global_args + ffmpeg_global_args +
self.ffmpeg_hwaccel_args + ffmpeg_hwaccel_args +
self.ffmpeg_input_args + ffmpeg_input_args +
['-i', self.ffmpeg_input] + ['-i', ffmpeg_input] +
self.ffmpeg_output_args + ffmpeg_output_args +
['pipe:']) ['pipe:'])
print(" ".join(ffmpeg_cmd)) print(" ".join(ffmpeg_cmd))
self.ffmpeg_process = sp.Popen(ffmpeg_cmd, stdout = sp.PIPE, bufsize=self.frame_size) ffmpeg_process = sp.Popen(ffmpeg_cmd, stdout = sp.PIPE, bufsize=frame_size)
def start(self): plasma_client = plasma.connect("/tmp/plasma")
self.start_or_restart_capture() frame_num = 0
self.watchdog.start() fps_tracker = EventsPerSecond()
fps_tracker.start()
while True:
# TODO: implement something to determine if it had to wait for a frame at all
# to determine if it might be behind and the buffer is filling up
start = datetime.datetime.now().timestamp()
frame_bytes = ffmpeg_process.stdout.read(frame_size)
duration = datetime.datetime.now().timestamp()-start
avg_wait.value = (avg_wait.value*9 + duration)/10
def join(self): if not frame_bytes:
self.capture_thread.join() # TODO: restart the ffmpeg process and track number of restarts
break
def get_capture_pid(self): # limit frame rate
return self.ffmpeg_process.pid frame_num += 1
if (frame_num % take_frame) != 0:
continue
def get_best(self, label): fps_tracker.update()
return self.best_frames.best_frames.get(label) fps.value = fps_tracker.eps()
def stats(self): frame_time = datetime.datetime.now().timestamp()
return {
'camera_fps': self.fps.eps(60),
'resize_queue': self.resize_queue.qsize(),
'frame_queue': self.frame_queue.qsize(),
'finished_frame_queue': self.finished_frame_queue.qsize(),
'refined_frame_queue': self.refined_frame_queue.qsize(),
'regions_in_process': self.regions_in_process,
'dynamic_regions_per_sec': self.dynamic_region_fps.eps(),
'skipped_regions_per_sec': self.skipped_region_tracker.eps(60)
}
def frame_with_objects(self, frame_time, tracked_objects=None): # Store frame in numpy array
if not frame_time in self.frame_cache: frame[:] = (np
frame = np.zeros(self.frame_shape, np.uint8) .frombuffer(frame_bytes, np.uint8)
.reshape(frame_shape))
# look for motion
motion_boxes = motion_detector.detect(frame)
tracked_objects = object_tracker.tracked_objects.values()
# merge areas of motion that intersect with a known tracked object into a single area to look at
areas_of_interest = []
used_motion_boxes = []
for obj in tracked_objects:
x_min, y_min, x_max, y_max = obj['box']
for m_index, motion_box in enumerate(motion_boxes):
if area(intersection(obj['box'], motion_box))/area(motion_box) > .5:
used_motion_boxes.append(m_index)
x_min = min(obj['box'][0], motion_box[0])
y_min = min(obj['box'][1], motion_box[1])
x_max = max(obj['box'][2], motion_box[2])
y_max = max(obj['box'][3], motion_box[3])
areas_of_interest.append((x_min, y_min, x_max, y_max))
unused_motion_boxes = set(range(0, len(motion_boxes))).difference(used_motion_boxes)
# compute motion regions
motion_regions = [calculate_region(frame_shape, motion_boxes[i][0], motion_boxes[i][1], motion_boxes[i][2], motion_boxes[i][3], 1.2)
for i in unused_motion_boxes]
# compute tracked object regions
object_regions = [calculate_region(frame_shape, a[0], a[1], a[2], a[3], 1.2)
for a in areas_of_interest]
# merge regions with high IOU
merged_regions = motion_regions+object_regions
while True:
max_iou = 0.0
max_indices = None
region_indices = range(len(merged_regions))
for a, b in itertools.combinations(region_indices, 2):
iou = intersection_over_union(merged_regions[a], merged_regions[b])
if iou > max_iou:
max_iou = iou
max_indices = (a, b)
if max_iou > 0.1:
a = merged_regions[max_indices[0]]
b = merged_regions[max_indices[1]]
merged_regions.append(calculate_region(frame_shape,
min(a[0], b[0]),
min(a[1], b[1]),
max(a[2], b[2]),
max(a[3], b[3]),
1
))
del merged_regions[max(max_indices[0], max_indices[1])]
del merged_regions[min(max_indices[0], max_indices[1])]
else: else:
frame = self.frame_cache[frame_time].copy() break
detected_objects = self.detected_objects[frame_time].copy() # resize regions and detect
detections = []
for region in merged_regions:
for region in self.regions: tensor_input = create_tensor_input(frame, region)
color = (255,255,255)
cv2.rectangle(frame, (region['x_offset'], region['y_offset']),
(region['x_offset']+region['size'], region['y_offset']+region['size']),
color, 2)
# draw the bounding boxes on the screen region_detections = object_detector.detect(tensor_input)
if tracked_objects is None: for d in region_detections:
with self.object_tracker.tracked_objects_lock: box = d[2]
tracked_objects = copy.deepcopy(self.object_tracker.tracked_objects) size = region[2]-region[0]
x_min = int((box[1] * size) + region[0])
y_min = int((box[0] * size) + region[1])
x_max = int((box[3] * size) + region[0])
y_max = int((box[2] * size) + region[1])
det = (d[0],
d[1],
(x_min, y_min, x_max, y_max),
(x_max-x_min)*(y_max-y_min),
region)
if filtered(det, objects_to_track, object_filters, mask):
continue
detections.append(det)
for obj in detected_objects: #########
draw_box_with_label(frame, obj['box']['xmin'], obj['box']['ymin'], obj['box']['xmax'], obj['box']['ymax'], obj['name'], "{}% {}".format(int(obj['score']*100), obj['area']), thickness=3) # merge objects, check for clipped objects and look again up to N times
#########
refining = True
refine_count = 0
while refining and refine_count < 4:
refining = False
for id, obj in tracked_objects.items(): # group by name
color = (0, 255,0) if obj['frame_time'] == frame_time else (255, 0, 0) detected_object_groups = defaultdict(lambda: [])
draw_box_with_label(frame, obj['box']['xmin'], obj['box']['ymin'], obj['box']['xmax'], obj['box']['ymax'], obj['name'], id, color=color, thickness=1, position='bl') for detection in detections:
detected_object_groups[detection[0]].append(detection)
# print a timestamp selected_objects = []
time_to_show = datetime.datetime.fromtimestamp(frame_time).strftime("%m/%d/%Y %H:%M:%S") for group in detected_object_groups.values():
cv2.putText(frame, time_to_show, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
# print fps # apply non-maxima suppression to suppress weak, overlapping bounding boxes
cv2.putText(frame, str(self.fps.eps())+'FPS', (10, 60), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2) boxes = [(o[2][0], o[2][1], o[2][2]-o[2][0], o[2][3]-o[2][1])
for o in group]
confidences = [o[1] for o in group]
idxs = cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.4)
# convert to BGR for index in idxs:
frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR) obj = group[index[0]]
if clipped(obj, frame_shape): #obj['clipped']:
box = obj[2]
# calculate a new region that will hopefully get the entire object
region = calculate_region(frame_shape,
box[0], box[1],
box[2], box[3])
# encode the image into a jpg tensor_input = create_tensor_input(frame, region)
ret, jpg = cv2.imencode('.jpg', frame) # run detection on new region
refined_detections = object_detector.detect(tensor_input)
for d in refined_detections:
box = d[2]
size = region[2]-region[0]
x_min = int((box[1] * size) + region[0])
y_min = int((box[0] * size) + region[1])
x_max = int((box[3] * size) + region[0])
y_max = int((box[2] * size) + region[1])
det = (d[0],
d[1],
(x_min, y_min, x_max, y_max),
(x_max-x_min)*(y_max-y_min),
region)
if filtered(det, objects_to_track, object_filters, mask):
continue
selected_objects.append(det)
return jpg.tobytes() refining = True
else:
selected_objects.append(obj)
def get_current_frame_with_objects(self): # set the detections list to only include top, complete objects
frame_time = self.last_processed_frame # and new detections
if frame_time == self.cached_frame_with_objects['frame_time']: detections = selected_objects
return self.cached_frame_with_objects['frame_bytes']
frame_bytes = self.frame_with_objects(frame_time) if refining:
refine_count += 1
self.cached_frame_with_objects = {
'frame_bytes': frame_bytes,
'frame_time': frame_time
}
return frame_bytes
# now that we have refined our detections, we need to track objects
object_tracker.match_and_update(frame_time, detections)
# put the frame in the plasma store
object_id = hashlib.sha1(str.encode(f"{name}{frame_time}")).digest()
plasma_client.put(frame, plasma.ObjectID(object_id))
# add to the queue
detected_objects_queue.put((name, frame_time, object_tracker.tracked_objects))
# if (frames >= 700 and frames <= 1635) or (frames >= 2500):
# if (frames >= 300 and frames <= 600):
# if (frames >= 0):
# row1 = cv2.hconcat([gray, cv2.convertScaleAbs(avg_frame)])
# row2 = cv2.hconcat([frameDelta, thresh])
# cv2.imwrite(f"/lab/debug/output/{frames}.jpg", cv2.vconcat([row1, row2]))
# # cv2.imwrite(f"/lab/debug/output/resized-frame-{frames}.jpg", resized_frame)
# for region in motion_regions:
# cv2.rectangle(frame, (region[0], region[1]), (region[2], region[3]), (255,128,0), 2)
# for region in object_regions:
# cv2.rectangle(frame, (region[0], region[1]), (region[2], region[3]), (0,128,255), 2)
# for region in merged_regions:
# cv2.rectangle(frame, (region[0], region[1]), (region[2], region[3]), (0,255,0), 2)
# for box in motion_boxes:
# cv2.rectangle(frame, (box[0], box[1]), (box[2], box[3]), (255,0,0), 2)
# for detection in detections:
# box = detection[2]
# draw_box_with_label(frame, box[0], box[1], box[2], box[3], detection[0], f"{detection[1]*100}%")
# for obj in object_tracker.tracked_objects.values():
# box = obj['box']
# draw_box_with_label(frame, box[0], box[1], box[2], box[3], obj['label'], obj['id'], thickness=1, color=(0,0,255), position='bl')
# cv2.putText(frame, str(total_detections), (10, 10), cv2.FONT_HERSHEY_SIMPLEX, fontScale=0.5, color=(0, 0, 0), thickness=2)
# cv2.putText(frame, str(frame_detections), (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=0.5, color=(0, 0, 0), thickness=2)
# cv2.imwrite(f"/lab/debug/output/frame-{frames}.jpg", frame)
# break
# start a thread to publish object scores
# mqtt_publisher = MqttObjectPublisher(self.mqtt_client, self.mqtt_topic_prefix, self)
# mqtt_publisher.start()
# create a watchdog thread for capture process
# self.watchdog = CameraWatchdog(self)