mirror of
https://github.com/blakeblackshear/frigate.git
synced 2024-11-21 19:07:46 +01:00
289 lines
11 KiB
Python
289 lines
11 KiB
Python
import os
|
|
import time
|
|
import datetime
|
|
import cv2
|
|
import threading
|
|
import ctypes
|
|
import multiprocessing as mp
|
|
import subprocess as sp
|
|
import numpy as np
|
|
from . util import tonumpyarray, draw_box_with_label
|
|
from . object_detection import FramePrepper
|
|
from . objects import ObjectCleaner, BestPersonFrame
|
|
from . mqtt import MqttObjectPublisher
|
|
|
|
# fetch the frames as fast a possible and store current frame in a shared memory array
|
|
def fetch_frames(shared_arr, shared_frame_time, frame_lock, frame_ready, frame_shape, rtsp_url):
|
|
# convert shared memory array into numpy and shape into image array
|
|
arr = tonumpyarray(shared_arr).reshape(frame_shape)
|
|
frame_size = frame_shape[0] * frame_shape[1] * frame_shape[2]
|
|
|
|
ffmpeg_cmd = ['ffmpeg',
|
|
'-hide_banner', '-loglevel', 'panic',
|
|
'-avoid_negative_ts', 'make_zero',
|
|
'-fflags', '+genpts',
|
|
'-rtsp_transport', 'tcp',
|
|
'-stimeout', '5000000',
|
|
'-use_wallclock_as_timestamps', '1',
|
|
'-i', rtsp_url,
|
|
'-f', 'rawvideo',
|
|
'-pix_fmt', 'rgb24',
|
|
'pipe:']
|
|
|
|
pipe = sp.Popen(ffmpeg_cmd, stdout = sp.PIPE, bufsize=frame_size)
|
|
|
|
while True:
|
|
raw_image = pipe.stdout.read(frame_size)
|
|
frame = (
|
|
np
|
|
.frombuffer(raw_image, np.uint8)
|
|
.reshape(frame_shape)
|
|
)
|
|
|
|
with frame_lock:
|
|
shared_frame_time.value = datetime.datetime.now().timestamp()
|
|
arr[:] = frame
|
|
# Notify with the condition that a new frame is ready
|
|
with frame_ready:
|
|
frame_ready.notify_all()
|
|
|
|
pipe.stdout.flush()
|
|
|
|
# Stores 2 seconds worth of frames when motion is detected so they can be used for other threads
|
|
class FrameTracker(threading.Thread):
|
|
def __init__(self, shared_frame, frame_time, frame_ready, frame_lock, recent_frames):
|
|
threading.Thread.__init__(self)
|
|
self.shared_frame = shared_frame
|
|
self.frame_time = frame_time
|
|
self.frame_ready = frame_ready
|
|
self.frame_lock = frame_lock
|
|
self.recent_frames = recent_frames
|
|
|
|
def run(self):
|
|
frame_time = 0.0
|
|
while True:
|
|
now = datetime.datetime.now().timestamp()
|
|
# wait for a frame
|
|
with self.frame_ready:
|
|
# if there isnt a frame ready for processing or it is old, wait for a signal
|
|
if self.frame_time.value == frame_time or (now - self.frame_time.value) > 0.5:
|
|
self.frame_ready.wait()
|
|
|
|
# lock and make a copy of the frame
|
|
with self.frame_lock:
|
|
frame = self.shared_frame.copy()
|
|
frame_time = self.frame_time.value
|
|
|
|
# add the frame to recent frames
|
|
self.recent_frames[frame_time] = frame
|
|
|
|
# delete any old frames
|
|
stored_frame_times = list(self.recent_frames.keys())
|
|
for k in stored_frame_times:
|
|
if (now - k) > 2:
|
|
del self.recent_frames[k]
|
|
|
|
def get_frame_shape(rtsp_url):
|
|
# capture a single frame and check the frame shape so the correct array
|
|
# size can be allocated in memory
|
|
video = cv2.VideoCapture(rtsp_url)
|
|
ret, frame = video.read()
|
|
frame_shape = frame.shape
|
|
video.release()
|
|
return frame_shape
|
|
|
|
def get_rtsp_url(rtsp_config):
|
|
if (rtsp_config['password'].startswith('$')):
|
|
rtsp_config['password'] = os.getenv(rtsp_config['password'][1:])
|
|
return 'rtsp://{}:{}@{}:{}{}'.format(rtsp_config['user'],
|
|
rtsp_config['password'], rtsp_config['host'], rtsp_config['port'],
|
|
rtsp_config['path'])
|
|
|
|
class CameraWatchdog(threading.Thread):
|
|
def __init__(self, camera):
|
|
threading.Thread.__init__(self)
|
|
self.camera = camera
|
|
|
|
def run(self):
|
|
|
|
while True:
|
|
# wait a bit before checking
|
|
time.sleep(60)
|
|
|
|
if (datetime.datetime.now().timestamp() - self.camera.shared_frame_time.value) > 2:
|
|
print("last frame is more than 2 seconds old, restarting camera capture...")
|
|
self.camera.start_or_restart_capture()
|
|
time.sleep(5)
|
|
|
|
class Camera:
|
|
def __init__(self, name, config, prepped_frame_queue, mqtt_client, mqtt_prefix):
|
|
self.name = name
|
|
self.config = config
|
|
self.detected_objects = []
|
|
self.recent_frames = {}
|
|
self.rtsp_url = get_rtsp_url(self.config['rtsp'])
|
|
self.regions = self.config['regions']
|
|
self.frame_shape = get_frame_shape(self.rtsp_url)
|
|
self.mqtt_client = mqtt_client
|
|
self.mqtt_topic_prefix = '{}/{}'.format(mqtt_prefix, self.name)
|
|
|
|
# compute the flattened array length from the shape of the frame
|
|
flat_array_length = self.frame_shape[0] * self.frame_shape[1] * self.frame_shape[2]
|
|
# create shared array for storing the full frame image data
|
|
self.shared_frame_array = mp.Array(ctypes.c_uint8, flat_array_length)
|
|
# create shared value for storing the frame_time
|
|
self.shared_frame_time = mp.Value('d', 0.0)
|
|
# Lock to control access to the frame
|
|
self.frame_lock = mp.Lock()
|
|
# Condition for notifying that a new frame is ready
|
|
self.frame_ready = mp.Condition()
|
|
# Condition for notifying that objects were parsed
|
|
self.objects_parsed = mp.Condition()
|
|
|
|
# shape current frame so it can be treated as a numpy image
|
|
self.shared_frame_np = tonumpyarray(self.shared_frame_array).reshape(self.frame_shape)
|
|
|
|
self.capture_process = None
|
|
|
|
# for each region, create a separate thread to resize the region and prep for detection
|
|
self.detection_prep_threads = []
|
|
for region in self.config['regions']:
|
|
# set a default threshold of 0.5 if not defined
|
|
if not 'threshold' in region:
|
|
region['threshold'] = 0.5
|
|
if not isinstance(region['threshold'], float):
|
|
print('Threshold is not a float. Setting to 0.5 default.')
|
|
region['threshold'] = 0.5
|
|
self.detection_prep_threads.append(FramePrepper(
|
|
self.name,
|
|
self.shared_frame_np,
|
|
self.shared_frame_time,
|
|
self.frame_ready,
|
|
self.frame_lock,
|
|
region['size'], region['x_offset'], region['y_offset'], region['threshold'],
|
|
prepped_frame_queue
|
|
))
|
|
|
|
# start a thread to store recent motion frames for processing
|
|
self.frame_tracker = FrameTracker(self.shared_frame_np, self.shared_frame_time,
|
|
self.frame_ready, self.frame_lock, self.recent_frames)
|
|
self.frame_tracker.start()
|
|
|
|
# start a thread to store the highest scoring recent person frame
|
|
self.best_person_frame = BestPersonFrame(self.objects_parsed, self.recent_frames, self.detected_objects)
|
|
self.best_person_frame.start()
|
|
|
|
# start a thread to expire objects from the detected objects list
|
|
self.object_cleaner = ObjectCleaner(self.objects_parsed, self.detected_objects)
|
|
self.object_cleaner.start()
|
|
|
|
# start a thread to publish object scores (currently only person)
|
|
mqtt_publisher = MqttObjectPublisher(self.mqtt_client, self.mqtt_topic_prefix, self.objects_parsed, self.detected_objects)
|
|
mqtt_publisher.start()
|
|
|
|
# create a watchdog thread for capture process
|
|
self.watchdog = CameraWatchdog(self)
|
|
|
|
# load in the mask for person detection
|
|
if 'mask' in self.config:
|
|
self.mask = cv2.imread("/config/{}".format(self.config['mask']), cv2.IMREAD_GRAYSCALE)
|
|
else:
|
|
self.mask = np.zeros((self.frame_shape[0], self.frame_shape[1], 1), np.uint8)
|
|
self.mask[:] = 255
|
|
|
|
|
|
def start_or_restart_capture(self):
|
|
if not self.capture_process is None:
|
|
print("Terminating the existing capture process...")
|
|
self.capture_process.terminate()
|
|
del self.capture_process
|
|
self.capture_process = None
|
|
|
|
# create the process to capture frames from the RTSP stream and store in a shared array
|
|
print("Creating a new capture process...")
|
|
self.capture_process = mp.Process(target=fetch_frames, args=(self.shared_frame_array,
|
|
self.shared_frame_time, self.frame_lock, self.frame_ready, self.frame_shape, self.rtsp_url))
|
|
self.capture_process.daemon = True
|
|
print("Starting a new capture process...")
|
|
self.capture_process.start()
|
|
|
|
def start(self):
|
|
self.start_or_restart_capture()
|
|
# start the object detection prep threads
|
|
for detection_prep_thread in self.detection_prep_threads:
|
|
detection_prep_thread.start()
|
|
self.watchdog.start()
|
|
|
|
def join(self):
|
|
self.capture_process.join()
|
|
|
|
def get_capture_pid(self):
|
|
return self.capture_process.pid
|
|
|
|
def add_objects(self, objects):
|
|
if len(objects) == 0:
|
|
return
|
|
|
|
for obj in objects:
|
|
if obj['name'] == 'person':
|
|
person_area = (obj['xmax']-obj['xmin'])*(obj['ymax']-obj['ymin'])
|
|
# find the matching region
|
|
region = None
|
|
for r in self.regions:
|
|
if (
|
|
obj['xmin'] >= r['x_offset'] and
|
|
obj['ymin'] >= r['y_offset'] and
|
|
obj['xmax'] <= r['x_offset']+r['size'] and
|
|
obj['ymax'] <= r['y_offset']+r['size']
|
|
):
|
|
region = r
|
|
break
|
|
|
|
# if the min person area is larger than the
|
|
# detected person, don't add it to detected objects
|
|
if region and region['min_person_area'] > person_area:
|
|
continue
|
|
|
|
# compute the coordinates of the person and make sure
|
|
# the location isnt outide the bounds of the image (can happen from rounding)
|
|
y_location = min(int(obj['ymax']), len(self.mask)-1)
|
|
x_location = min(int((obj['xmax']-obj['xmin'])/2.0), len(self.mask[0])-1)
|
|
|
|
# if the person is in a masked location, continue
|
|
if self.mask[y_location][x_location] == [0]:
|
|
continue
|
|
|
|
self.detected_objects.append(obj)
|
|
|
|
with self.objects_parsed:
|
|
self.objects_parsed.notify_all()
|
|
|
|
def get_best_person(self):
|
|
return self.best_person_frame.best_frame
|
|
|
|
def get_current_frame_with_objects(self):
|
|
# make a copy of the current detected objects
|
|
detected_objects = self.detected_objects.copy()
|
|
# lock and make a copy of the current frame
|
|
with self.frame_lock:
|
|
frame = self.shared_frame_np.copy()
|
|
|
|
# draw the bounding boxes on the screen
|
|
for obj in detected_objects:
|
|
label = "{}: {}%".format(obj['name'],int(obj['score']*100))
|
|
draw_box_with_label(frame, obj['xmin'], obj['ymin'], obj['xmax'], obj['ymax'], label)
|
|
|
|
for region in self.regions:
|
|
color = (255,255,255)
|
|
cv2.rectangle(frame, (region['x_offset'], region['y_offset']),
|
|
(region['x_offset']+region['size'], region['y_offset']+region['size']),
|
|
color, 2)
|
|
|
|
# convert to BGR
|
|
frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
|
|
|
|
return frame
|
|
|
|
|
|
|
|
|