diff --git a/.gitignore b/.gitignore index 8456d9be0..0b71ba313 100644 --- a/.gitignore +++ b/.gitignore @@ -5,13 +5,13 @@ __pycache__ debug .vscode/* !.vscode/launch.json -config/* +# config/* !config/*.example models *.mp4 *.db *.csv -frigate/version.py +# frigate/version.py web/build web/node_modules web/coverage diff --git a/config/.exports b/config/.exports new file mode 100644 index 000000000..7bb6da740 --- /dev/null +++ b/config/.exports @@ -0,0 +1 @@ +1744058517.724104 \ No newline at end of file diff --git a/config/.jwt_secret b/config/.jwt_secret new file mode 100644 index 000000000..08fcbc3ca --- /dev/null +++ b/config/.jwt_secret @@ -0,0 +1 @@ +c6dcc36e80f0d9f7090c478197acd9b1ac48a1e6312ce70809327a72b1c2b537666cc46a613e40ea7f50993a963f10eadd0e57a1ee7c529a9a907061eac57a57 \ No newline at end of file diff --git a/config/.timeline b/config/.timeline new file mode 100644 index 000000000..35b1a7a14 --- /dev/null +++ b/config/.timeline @@ -0,0 +1 @@ +1744058517.683821 \ No newline at end of file diff --git a/config/.vacuum b/config/.vacuum new file mode 100644 index 000000000..489dfffb1 --- /dev/null +++ b/config/.vacuum @@ -0,0 +1 @@ +1744058517.699165 \ No newline at end of file diff --git a/config/config.yaml b/config/config.yaml new file mode 100644 index 000000000..39696ba95 --- /dev/null +++ b/config/config.yaml @@ -0,0 +1,83 @@ +mqtt: + enabled: false # Set this to True if using MQTT for event triggers + +detectors: + memryx: + type: memryx + device: PCIe + +# model: +# model_type: yolov9 +# width: 640 +# height: 640 +# path: /config/model_cache/memryx_cache/YOLO_v9_small_640_640_3_onnx.dfp +# labelmap_path: /config/model_cache/memryx_cache/labelmap.txt + +# model: +# model_type: yolov8 +# width: 640 +# height: 640 +# path: /config/model_cache/memryx_cache/YOLO_v8_small_640_640_3_onnx.dfp +# labelmap_path: /config/model_cache/memryx_cache/labelmap.txt + +# model: +# model_type: yolonas +# width: 320 +# height: 320 +# path: /config/model_cache/memryx_cache/yolo_nas_s.dfp +# labelmap_path: /config/model_cache/memryx_cache/labelmap.txt + +model: + model_type: yolox + width: 640 + height: 640 + path: /config/model_cache/memryx_cache/YOLOX_640_640_3_onnx.dfp + labelmap_path: /config/model_cache/memryx_cache/labelmap.txt + +# model: +# model_type: ssd +# width: 320 +# height: 320 +# path: /config/model_cache/memryx_cache/SSDlite_MobileNet_v2_320_320_3_onnx.dfp +# labelmap_path: /config/model_cache/memryx_cache/labelmap.txt + +cameras: + Cam1: + ffmpeg: + inputs: + - path: rtsp://admin:NoPa$$word@192.168.56.22:554/cam/realmonitor?channel=1&subtype=0 + roles: + - detect + - record + detect: + width: 640 + height: 480 + fps: 30 + enabled: true + + objects: + track: + - person + - cup + - bottle + - keyboard + - cell phone + + snapshots: + enabled: false + bounding_box: true + retain: + default: 1 # Keep snapshots for 2 days + record: + enabled: false + retain: + days: 1 # Keep recordings for 7 days + alerts: + retain: + days: 1 + detections: + retain: + days: 1 + +version: 0.16-0 + diff --git a/config/frigate.db-shm b/config/frigate.db-shm new file mode 100644 index 000000000..fe9ac2845 Binary files /dev/null and b/config/frigate.db-shm differ diff --git a/config/frigate.db-wal b/config/frigate.db-wal new file mode 100644 index 000000000..e69de29bb diff --git a/config/model_cache/memryx_cache/SSDlite_MobileNet_v2_320_320_3_onnx.dfp b/config/model_cache/memryx_cache/SSDlite_MobileNet_v2_320_320_3_onnx.dfp new file mode 100644 index 000000000..2925f6a6c Binary files /dev/null and b/config/model_cache/memryx_cache/SSDlite_MobileNet_v2_320_320_3_onnx.dfp differ diff --git a/config/model_cache/memryx_cache/SSDlite_MobileNet_v2_320_320_3_onnx_post.onnx b/config/model_cache/memryx_cache/SSDlite_MobileNet_v2_320_320_3_onnx_post.onnx new file mode 100644 index 000000000..123d76a79 Binary files /dev/null and b/config/model_cache/memryx_cache/SSDlite_MobileNet_v2_320_320_3_onnx_post.onnx differ diff --git a/config/model_cache/memryx_cache/YOLOX_640_640_3_onnx.dfp b/config/model_cache/memryx_cache/YOLOX_640_640_3_onnx.dfp new file mode 100644 index 000000000..941a6a798 Binary files /dev/null and b/config/model_cache/memryx_cache/YOLOX_640_640_3_onnx.dfp differ diff --git a/config/model_cache/memryx_cache/YOLO_v8_small_640_640_3_onnx.dfp b/config/model_cache/memryx_cache/YOLO_v8_small_640_640_3_onnx.dfp new file mode 100644 index 000000000..a32bbc5b3 Binary files /dev/null and b/config/model_cache/memryx_cache/YOLO_v8_small_640_640_3_onnx.dfp differ diff --git a/config/model_cache/memryx_cache/YOLO_v9_small_640_640_3_onnx.dfp b/config/model_cache/memryx_cache/YOLO_v9_small_640_640_3_onnx.dfp new file mode 100644 index 000000000..b0eccf32b Binary files /dev/null and b/config/model_cache/memryx_cache/YOLO_v9_small_640_640_3_onnx.dfp differ diff --git a/config/model_cache/memryx_cache/_model_22_Constant_10_output_0.npy b/config/model_cache/memryx_cache/_model_22_Constant_10_output_0.npy new file mode 100644 index 000000000..db2228469 Binary files /dev/null and b/config/model_cache/memryx_cache/_model_22_Constant_10_output_0.npy differ diff --git a/config/model_cache/memryx_cache/_model_22_Constant_12_output_0.npy b/config/model_cache/memryx_cache/_model_22_Constant_12_output_0.npy new file mode 100644 index 000000000..051d51c93 Binary files /dev/null and b/config/model_cache/memryx_cache/_model_22_Constant_12_output_0.npy differ diff --git a/config/model_cache/memryx_cache/_model_22_Constant_9_output_0.npy b/config/model_cache/memryx_cache/_model_22_Constant_9_output_0.npy new file mode 100644 index 000000000..db2228469 Binary files /dev/null and b/config/model_cache/memryx_cache/_model_22_Constant_9_output_0.npy differ diff --git a/config/model_cache/memryx_cache/labelmap.txt b/config/model_cache/memryx_cache/labelmap.txt new file mode 100644 index 000000000..51dba82b4 --- /dev/null +++ b/config/model_cache/memryx_cache/labelmap.txt @@ -0,0 +1,80 @@ +0 person +1 bicycle +2 car +3 motorcycle +4 airplane +5 bus +6 train +7 truck +8 boat +9 traffic light +10 fire hydrant +11 stop sign +12 parking meter +13 bench +14 bird +15 cat +16 dog +17 horse +18 sheep +19 cow +20 elephant +21 bear +22 zebra +23 giraffe +24 backpack +25 umbrella +26 handbag +27 tie +28 suitcase +29 frisbee +30 skis +31 snowboard +32 sports ball +33 kite +34 baseball bat +35 baseball glove +36 skateboard +37 surfboard +38 tennis racket +39 bottle +40 wine glass +41 cup +42 fork +43 knife +44 spoon +45 bowl +46 banana +47 apple +48 sandwich +49 orange +50 broccoli +51 carrot +52 hot dog +53 pizza +54 donut +55 cake +56 chair +57 couch +58 potted plant +59 bed +60 dining table +61 toilet +62 tv +63 laptop +64 mouse +65 remote +66 keyboard +67 cell phone +68 microwave +69 oven +70 toaster +71 sink +72 refrigerator +73 book +74 clock +75 vase +76 scissors +77 teddy bear +78 hair drier +79 toothbrush \ No newline at end of file diff --git a/config/model_cache/memryx_cache/model_22_dfl_conv_weight.npy b/config/model_cache/memryx_cache/model_22_dfl_conv_weight.npy new file mode 100644 index 000000000..3348cca5b Binary files /dev/null and b/config/model_cache/memryx_cache/model_22_dfl_conv_weight.npy differ diff --git a/config/model_cache/memryx_cache/yolo_nas_s.dfp b/config/model_cache/memryx_cache/yolo_nas_s.dfp new file mode 100644 index 000000000..96716131c Binary files /dev/null and b/config/model_cache/memryx_cache/yolo_nas_s.dfp differ diff --git a/config/model_cache/memryx_cache/yolo_nas_s_post.onnx b/config/model_cache/memryx_cache/yolo_nas_s_post.onnx new file mode 100644 index 000000000..f246f028e Binary files /dev/null and b/config/model_cache/memryx_cache/yolo_nas_s_post.onnx differ diff --git a/docker/main/Dockerfile b/docker/main/Dockerfile index a71250813..95664e302 100644 --- a/docker/main/Dockerfile +++ b/docker/main/Dockerfile @@ -245,6 +245,40 @@ RUN --mount=type=bind,from=wheels,source=/wheels,target=/deps/wheels \ COPY --from=deps-rootfs / / +#### +# +# MemryX Support +# +# 1. Install system dependencies and Python packages +# 2. Add MemryX repo and install memx-accl +# +#### +# Install system dependencies +RUN apt-get -qq update && \ + apt-get -qq install -y --no-install-recommends \ + libhdf5-dev \ + python3-dev \ + cmake \ + python3-venv \ + build-essential \ + curl \ + wget \ + gnupg + +RUN python3 -m pip install --upgrade pip && \ + python3 -m pip install --extra-index-url https://developer.memryx.com/pip memryx + +# Add MemryX repo + key +RUN curl -fsSL https://developer.memryx.com/deb/memryx.asc | tee /etc/apt/trusted.gpg.d/memryx.asc && \ + echo "deb https://developer.memryx.com/deb stable main" > /etc/apt/sources.list.d/memryx.list && \ + apt-get update -qq + +# Install memx-accl from MemryX repo +RUN apt-get install -y --no-install-recommends memx-accl + +# Debug messages +RUN echo "Hello from inside MemryX Docker image!" + RUN ldconfig EXPOSE 5000 diff --git a/docker/memryx/user_installation.sh b/docker/memryx/user_installation.sh new file mode 100755 index 000000000..17ac8669d --- /dev/null +++ b/docker/memryx/user_installation.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +set -e # Exit on error +set -o pipefail + +echo "Starting MemryX driver and runtime installation..." + +# Detect architecture +arch=$(uname -m) + +if [[ -d /sys/memx0/ ]]; then + echo "Existing functional MX3 driver found. Skipping driver re-install." +else + + # Purge existing packages and repo + echo "Removing old MemryX installations..." + sudo apt purge -y memx-* || true + sudo rm -f /etc/apt/sources.list.d/memryx.list /etc/apt/trusted.gpg.d/memryx.asc + + # Install kernel headers + echo "Installing kernel headers for: $(uname -r)" + sudo apt update + sudo apt install -y linux-headers-$(uname -r) + + # Add MemryX key and repo + echo "Adding MemryX GPG key and repository..." + wget -qO- https://developer.memryx.com/deb/memryx.asc | sudo tee /etc/apt/trusted.gpg.d/memryx.asc >/dev/null + echo 'deb https://developer.memryx.com/deb stable main' | sudo tee /etc/apt/sources.list.d/memryx.list >/dev/null + + # Update and install packages + echo "Installing memx-drivers..." + sudo apt update + sudo apt install -y memx-drivers + + # ARM-specific board setup + if [[ "$arch" == "aarch64" || "$arch" == "arm64" ]]; then + echo " Running ARM board setup..." + sudo mx_arm_setup + fi + + echo -e "\n\n\033[1;31mYOU MUST RESTART YOUR COMPUTER NOW\033[0m\n\n" +fi + +# Install mxa-manager +echo "Installing mxa-manager..." +sudo apt install -y memx-accl mxa-manager + + +echo "MemryX installation complete!" diff --git a/frigate/detectors/plugins/memryx.py b/frigate/detectors/plugins/memryx.py new file mode 100644 index 000000000..947807a47 --- /dev/null +++ b/frigate/detectors/plugins/memryx.py @@ -0,0 +1,519 @@ +import logging +import numpy as np +import cv2 +import os +import urllib.request +import zipfile +from queue import Queue +import time + +try: + # from memryx import AsyncAccl # Import MemryX SDK + from memryx import AsyncAccl +except ModuleNotFoundError: + raise ImportError("MemryX SDK is not installed. Install it and set up MIX environment.") + +from pydantic import BaseModel, Field +from typing_extensions import Literal +from frigate.detectors.detection_api import DetectionApi +from frigate.detectors.detector_config import BaseDetectorConfig, ModelTypeEnum +from frigate.util.model import post_process_yolov9 + +logger = logging.getLogger(__name__) + +DETECTOR_KEY = "memryx" + +# Configuration class for model settings +class ModelConfig(BaseModel): + path: str = Field(default=None, title="Model Path") # Path to the DFP file + labelmap_path: str = Field(default=None, title="Path to Label Map") + +class MemryXDetectorConfig(BaseDetectorConfig): + type: Literal[DETECTOR_KEY] + device: str = Field(default="PCIe", title="Device Path") + +class MemryXDetector(DetectionApi): + type_key = DETECTOR_KEY # Set the type key + supported_models = [ + ModelTypeEnum.ssd, + ModelTypeEnum.yolonas, + ModelTypeEnum.yolov9, + ModelTypeEnum.yolox, + ] + + def __init__(self, detector_config): + + self.capture_queue = Queue(maxsize=10) + self.output_queue = Queue(maxsize=10) + self.capture_id_queue = Queue(maxsize=10) + self.logger = logger + + """Initialize MemryX detector with the provided configuration.""" + self.memx_model_path = detector_config.model.path # Path to .dfp file + self.memx_post_model = None # Path to .post file + self.expected_post_model = None + self.memx_device_path = detector_config.device # Device path + self.memx_model_height = detector_config.model.height + self.memx_model_width = detector_config.model.width + self.memx_model_type = detector_config.model.model_type + + self.cache_dir = "/config/model_cache/memryx_cache" + + if self.memx_model_type == ModelTypeEnum.yolov9: + self.model_url = "https://developer.memryx.com/model_explorer/1p2/YOLO_v9_small_640_640_3_onnx.zip" + # self.expected_post_model = "YOLO_v9_small_640_640_3_onnx_post.onnx" + self.const_A = np.load("/config/model_cache/memryx_cache/_model_22_Constant_9_output_0.npy") + self.const_B = np.load("/config/model_cache/memryx_cache/_model_22_Constant_10_output_0.npy") + self.const_C = np.load("/config/model_cache/memryx_cache/_model_22_Constant_12_output_0.npy") + + elif self.memx_model_type == ModelTypeEnum.yolonas: + self.model_url = "" + self.expected_post_model = "yolo_nas_s_post.onnx" + + elif self.memx_model_type == ModelTypeEnum.yolox: + self.model_url = "https://developer.memryx.com/model_explorer/1p2/YOLOX_640_640_3_onnx.zip" + # self.expected_post_model = "YOLOX_640_640_3_onnx_post.onnx" + self.set_strides_grids() + + elif self.memx_model_type == ModelTypeEnum.ssd: + self.model_url = "https://developer.memryx.com/model_explorer/1p2/SSDlite_MobileNet_v2_320_320_3_onnx.zip" + self.expected_post_model = "SSDlite_MobileNet_v2_320_320_3_onnx_post.onnx" + + self.check_and_prepare_model() + logger.info(f"Initializing MemryX with model: {self.memx_model_path} on device {self.memx_device_path}") + + try: + # Load MemryX Model + logger.info(f"dfp path: {self.memx_model_path}") + + # Your initialization code + self.accl = AsyncAccl(self.memx_model_path, mxserver_addr="host.docker.internal") + if self.memx_post_model: + self.accl.set_postprocessing_model(self.memx_post_model, model_idx=0) + self.accl.connect_input(self.process_input) + self.accl.connect_output(self.process_output) + # self.accl.wait() # Wait for the accelerator to finish + + logger.info(f"Loaded MemryX model from {self.memx_model_path} and {self.memx_post_model}") + + except Exception as e: + logger.error(f"Failed to initialize MemryX model: {e}") + raise + + def check_and_prepare_model(self): + """Check if both models exist; if not, download and extract them.""" + if not os.path.exists(self.cache_dir): + os.makedirs(self.cache_dir) + + if not self.expected_post_model: + logger.info(f"Assigned Model Path: {self.memx_model_path}") + + else: + post_model_file_path = os.path.join(self.cache_dir, self.expected_post_model) + # model_file_path_tflite = os.path.join(self.cache_dir, self.expected_model_filename_tflite) + + # Check if both required model files exist + if os.path.isfile(post_model_file_path): + self.memx_post_model = post_model_file_path + logger.info(f"Post-processing model found at {post_model_file_path}, skipping download.") + else: + logger.info(f"Model files not found. Downloading from {self.model_url}...") + zip_path = os.path.join(self.cache_dir, "memryx_model.zip") + + # Download the ZIP file + urllib.request.urlretrieve(self.model_url, zip_path) + logger.info(f"Model ZIP downloaded to {zip_path}. Extracting...") + + # Extract ZIP file + with zipfile.ZipFile(zip_path, "r") as zip_ref: + zip_ref.extractall(self.cache_dir) + + logger.info(f"Model extracted to {self.cache_dir}.") + + # Assign extracted files to correct paths + for file in os.listdir(self.cache_dir): + if file == self.expected_post_model: + self.memx_post_model = os.path.join(self.cache_dir, file) + + logger.info(f"Assigned Model Path: {self.memx_model_path}") + logger.info(f"Assigned Post-processing Model Path: {self.memx_post_model}") + + # Cleanup: Remove the ZIP file after extraction + os.remove(zip_path) + logger.info("Cleaned up ZIP file after extraction.") + + def send_input(self, connection_id, input_frame): + """Send frame directly to MemryX processing.""" + # logging.info(f"Processing frame for connection ID: {connection_id}") + + if input_frame is None: + raise ValueError("[send_input] No image data provided for inference") + + # Send frame to MemryX for processing + self.capture_queue.put(input_frame) # MemryX will process this + self.capture_id_queue.put(connection_id) # Keep track of connection ID + + def process_input(self): + """ + Wait for frames in the queue, preprocess the image, and return it. + """ + while True: + try: + # Wait for a frame from the queue (blocking call) + frame = self.capture_queue.get(block=True) # Blocks until data is available + + return frame + + except Exception as e: + logger.info(f"[process_input] Error processing input: {e}") + time.sleep(0.1) # Prevent busy waiting in case of error + + def receive_output(self): + + """Retrieve processed results directly from MemryX.""" + connection_id = self.capture_id_queue.get() # Get the corresponding connection ID + detections = self.output_queue.get() # Get detections from MemryX + + return connection_id, detections + + def post_process_yolonas(self, output): + predictions = output[0] + + detections = np.zeros((20, 6), np.float32) + + for i, prediction in enumerate(predictions): + if i == 20: + break + + (_, x_min, y_min, x_max, y_max, confidence, class_id) = prediction + + if class_id < 0: + break + + detections[i] = [ + class_id, + confidence, + y_min / self.memx_model_height, + x_min / self.memx_model_width, + y_max / self.memx_model_height, + x_max / self.memx_model_width, + ] + + # Return the list of final detections + self.output_queue.put(detections) + + ## Takes in class ID, confidence score, and array of [x, y, w, h] that describes detection position, + ## returns an array that's easily passable back to Frigate. + def process_yolo(self, class_id, conf, pos): + return [ + class_id, # class ID + conf, # confidence score + (pos[1] - (pos[3] / 2)) / self.memx_model_height, # y_min + (pos[0] - (pos[2] / 2)) / self.memx_model_width, # x_min + (pos[1] + (pos[3] / 2)) / self.memx_model_height, # y_max + (pos[0] + (pos[2] / 2)) / self.memx_model_width, # x_max + + ] + + def set_strides_grids(self): + grids = [] + expanded_strides = [] + + strides = [8, 16, 32] + + hsize_list = [self.memx_model_height // stride for stride in strides] + wsize_list = [self.memx_model_width // stride for stride in strides] + + for hsize, wsize, stride in zip(hsize_list, wsize_list, strides): + xv, yv = np.meshgrid(np.arange(wsize), np.arange(hsize)) + grid = np.stack((xv, yv), 2).reshape(1, -1, 2) + grids.append(grid) + shape = grid.shape[:2] + expanded_strides.append(np.full((*shape, 1), stride)) + self.grids = np.concatenate(grids, 1) + self.expanded_strides = np.concatenate(expanded_strides, 1) + + def sigmoid(self, x: np.ndarray) -> np.ndarray: + + return 1 / (1 + np.exp(-x)) + + def onnx_concat(self, inputs: list, axis: int) -> np.ndarray: + + # Ensure all inputs are numpy arrays + if not all(isinstance(x, np.ndarray) for x in inputs): + raise TypeError("All inputs must be numpy arrays.") + + # Ensure shapes match on non-concat axes + ref_shape = list(inputs[0].shape) + for i, tensor in enumerate(inputs[1:], start=1): + for ax in range(len(ref_shape)): + if ax == axis: + continue + if tensor.shape[ax] != ref_shape[ax]: + raise ValueError(f"Shape mismatch at axis {ax} between input[0] and input[{i}]") + + return np.concatenate(inputs, axis=axis) + + def onnx_reshape(self, data: np.ndarray, shape: np.ndarray) -> np.ndarray: + + # Ensure shape is a 1D array of integers + target_shape = shape.astype(int).tolist() + + # Use NumPy reshape with dynamic handling of -1 + reshaped = np.reshape(data, target_shape) + + return reshaped + + def post_process_yolox(self, output): + + output = [np.expand_dims(tensor, axis=0) for tensor in output] # Shape: (1, H, W, C) + + # Move channel axis from 3rd (last) position to 1st position → (1, C, H, W) + output = [np.transpose(tensor, (0, 3, 1, 2)) for tensor in output] + + output_785 = output[0] # 785 + output_794 = output[1] # 794 + output_795 = output[2] # 795 + output_811 = output[3] # 811 + output_820 = output[4] # 820 + output_821 = output[5] # 821 + output_837 = output[6] # 837 + output_846 = output[7] # 846 + output_847 = output[8] # 847 + + output_795 = self.sigmoid(output_795) + output_785 = self.sigmoid(output_785) + output_821 = self.sigmoid(output_821) + output_811 = self.sigmoid(output_811) + output_847 = self.sigmoid(output_847) + output_837 = self.sigmoid(output_837) + + concat_1 = self.onnx_concat([output_794, output_795, output_785], axis=1) + concat_2 = self.onnx_concat([output_820, output_821, output_811], axis=1) + concat_3 = self.onnx_concat([output_846, output_847, output_837], axis=1) + + shape = np.array([1, 85, -1], dtype=np.int64) + + reshape_1 = self.onnx_reshape(concat_1, shape) + reshape_2 = self.onnx_reshape(concat_2, shape) + reshape_3 = self.onnx_reshape(concat_3, shape) + + concat_out = self.onnx_concat([reshape_1, reshape_2, reshape_3], axis=2) + + output = concat_out.transpose(0,2,1) #1, 840, 85 + + self.num_classes = output.shape[2] - 5 + + # [x, y, h, w, box_score, class_no_1, ..., class_no_80], + results = output + + results[..., :2] = (results[..., :2] + self.grids) * self.expanded_strides + results[..., 2:4] = np.exp(results[..., 2:4]) * self.expanded_strides + image_pred = results[0, ...] + + class_conf = np.max(image_pred[:, 5:5 + self.num_classes], axis=1, keepdims=True) + class_pred = np.argmax(image_pred[:, 5:5 + self.num_classes], axis=1) + class_pred = np.expand_dims(class_pred, axis=1) + + conf_mask = (image_pred[:, 4] * class_conf.squeeze() >= 0.3).squeeze() + # Detections ordered as (x1, y1, x2, y2, obj_conf, class_conf, class_pred) + detections = np.concatenate((image_pred[:, :5], class_conf, class_pred), axis=1) + detections = detections[conf_mask] + + # Sort by class confidence (index 5) and keep top 20 detections + ordered = detections[detections[:, 5].argsort()[::-1]][:20] + + # Prepare a final detections array of shape (20, 6) + final_detections = np.zeros((20, 6), np.float32) + for i, object_detected in enumerate(ordered): + final_detections[i] = self.process_yolo( + object_detected[6], object_detected[5], object_detected[:4] + ) + + self.output_queue.put(final_detections) + + + def post_process_ssdlite(self, outputs): + dets = outputs[0].squeeze(0) # Shape: (1, num_dets, 5) + labels = outputs[1].squeeze(0) + + detections = [] + + for i in range(dets.shape[0]): + x_min, y_min, x_max, y_max, confidence = dets[i] + class_id = int(labels[i]) # Convert label to integer + + if confidence < 0.45: + continue # Skip detections below threshold + + # Convert coordinates to integers + x_min, y_min, x_max, y_max = map(int, [x_min, y_min, x_max, y_max]) + + # Append valid detections [class_id, confidence, x, y, width, height] + detections.append([class_id, confidence, x_min, y_min, x_max, y_max]) + + final_detections = np.zeros((20, 6), np.float32) + + if len(detections) == 0: + # logger.info("No detections found.") + self.output_queue.put(final_detections) + return + + # Convert to NumPy array + detections = np.array(detections, dtype=np.float32) + + # Apply Non-Maximum Suppression (NMS) + bboxes = detections[:, 2:6].tolist() # (x_min, y_min, width, height) + scores = detections[:, 1].tolist() # Confidence scores + + indices = cv2.dnn.NMSBoxes(bboxes, scores, 0.45, 0.5) + + if len(indices) > 0: + indices = indices.flatten()[:20] # Keep only the top 20 detections + selected_detections = detections[indices] + + # Normalize coordinates AFTER NMS + for i, det in enumerate(selected_detections): + class_id, confidence, x_min, y_min, x_max, y_max = det + + # Normalize coordinates + x_min /= self.memx_model_width + y_min /= self.memx_model_height + x_max /= self.memx_model_width + y_max /= self.memx_model_height + + final_detections[i] = [class_id, confidence, y_min, x_min, y_max, x_max] + + # logger.info(f"Final detections: {final_detections}") + self.output_queue.put(final_detections) + + def onnx_reshape_with_allowzero(self, data: np.ndarray, shape: np.ndarray, allowzero: int = 0) -> np.ndarray: + + shape = shape.astype(int) + input_shape = data.shape + output_shape = [] + + for i, dim in enumerate(shape): + if dim == 0 and allowzero == 0: + output_shape.append(input_shape[i]) # Copy dimension from input + else: + output_shape.append(dim) + + # Now let NumPy infer any -1 if needed + reshaped = np.reshape(data, output_shape) + + return reshaped + + def process_output(self, *outputs): + + if self.memx_model_type == ModelTypeEnum.yolov9: + outputs = [np.expand_dims(tensor, axis=0) for tensor in outputs] # Shape: (1, H, W, C) + + # Move channel axis from 3rd (last) position to 1st position → (1, C, H, W) + outputs = [np.transpose(tensor, (0, 3, 1, 2)) for tensor in outputs] + + conv_out1 = outputs[0] + conv_out2 = outputs[1] + conv_out3 = outputs[2] + conv_out4 = outputs[3] + conv_out5 = outputs[4] + conv_out6 = outputs[5] + + concat_1 = self.onnx_concat([conv_out1, conv_out2], axis=1) + concat_2 = self.onnx_concat([conv_out3, conv_out4], axis=1) + concat_3 = self.onnx_concat([conv_out5, conv_out6], axis=1) + + shape = np.array([1, 144, -1], dtype=np.int64) + + reshaped_1 = self.onnx_reshape_with_allowzero(concat_1, shape, allowzero=0) + reshaped_2 = self.onnx_reshape_with_allowzero(concat_2, shape, allowzero=0) + reshaped_3 = self.onnx_reshape_with_allowzero(concat_3, shape, allowzero=0) + + concat_4 = self.onnx_concat([reshaped_1, reshaped_2, reshaped_3], 2) + + axis = 1 + split_sizes = [64, 80] + + # Calculate indices at which to split + indices = np.cumsum(split_sizes)[:-1] # [64] — split before the second chunk + + # Perform split along axis 1 + split_0, split_1 = np.split(concat_4, indices, axis=axis) + + shape1 = np.array([1,4,16,8400]) + reshape_4 = self.onnx_reshape_with_allowzero(split_0, shape1, allowzero=0) + + transpose_1 = reshape_4.transpose(0,2,1,3) + + axis = 1 # As per ONNX softmax node + + # Subtract max for numerical stability + x_max = np.max(transpose_1, axis=axis, keepdims=True) + x_exp = np.exp(transpose_1 - x_max) + x_sum = np.sum(x_exp, axis=axis, keepdims=True) + softmax_output = x_exp / x_sum + + # Weight W from the ONNX initializer (1, 16, 1, 1) with values 0 to 15 + W = np.arange(16, dtype=np.float32).reshape(1, 16, 1, 1) # (1, 16, 1, 1) + + # Apply 1x1 convolution: this is a weighted sum over channels + conv_output = np.sum(softmax_output * W, axis=1, keepdims=True) # shape: (1, 1, 4, 8400) + + shape2 = np.array([1,4,8400]) + reshape_5 = self.onnx_reshape_with_allowzero(conv_output, shape2, allowzero=0) + + # ONNX Slice — get first 2 channels: [0:2] along axis 1 + slice_output1 = reshape_5[:, 0:2, :] # Result: (1, 2, 8400) + + # Slice channels 2 to 4 → axis = 1 + slice_output2 = reshape_5[:, 2:4, :] + + # Perform Subtraction + sub_output = self.const_A - slice_output1 # Equivalent to ONNX Sub + + # Perform the ONNX-style Add + add_output = self.const_B + slice_output2 + + sub1 = add_output - sub_output + + add1 = sub_output + add_output + + div_output = add1 / 2.0 + + concat_5 = self.onnx_concat([div_output, sub1], axis=1) + + # const_C = np.load("_model_22_Constant_12_output_0.npy") # Shape: (1, 8400) + + # Expand B to (1, 1, 8400) so it can broadcast across axis=1 (4 channels) + const_C_expanded = self.const_C[:, np.newaxis, :] # Shape: (1, 1, 8400) + + # Perform ONNX-style element-wise multiplication + mul_output = concat_5 * const_C_expanded # Result: (1, 4, 8400) + + sigmoid_output = self.sigmoid(split_1) + outputs = self.onnx_concat([mul_output, sigmoid_output], axis=1) + + final_detections = post_process_yolov9(outputs, self.memx_model_width, self.memx_model_height) + self.output_queue.put(final_detections) + + elif self.memx_model_type == ModelTypeEnum.yolonas: + return self.post_process_yolonas(outputs) + + elif self.memx_model_type == ModelTypeEnum.yolox: + return self.post_process_yolox(outputs) + + elif self.memx_model_type == ModelTypeEnum.ssd: + return self.post_process_ssdlite(outputs) + + else: + raise Exception( + f"{self.memx_model_type} is currently not supported for memryx. See the docs for more info on supported models." + ) + + def detect_raw(self, tensor_input: np.ndarray): + """ + Run inference on the input image and return raw results. + tensor_input: Preprocessed image (normalized & resized) + """ + # logger.info("[detect_raw] Running inference on MemryX") + return 0 diff --git a/frigate/object_detection.py b/frigate/object_detection.py index 8e88ae578..d409e49c1 100644 --- a/frigate/object_detection.py +++ b/frigate/object_detection.py @@ -8,6 +8,8 @@ import threading from abc import ABC, abstractmethod import numpy as np +import cv2 +import time from setproctitle import setproctitle import frigate.util as util @@ -16,6 +18,7 @@ from frigate.detectors.detector_config import ( BaseDetectorConfig, InputDTypeEnum, InputTensorEnum, + ModelTypeEnum ) from frigate.util.builtin import EventsPerSecond, load_labels from frigate.util.image import SharedMemoryFrameManager, UntrackedSharedMemory @@ -49,6 +52,8 @@ class LocalObjectDetector(ObjectDetector): self.labels = {} else: self.labels = load_labels(labels) + + self.model_type = detector_config.model.model_type if detector_config: self.input_transform = tensor_transform(detector_config.model.input_tensor) @@ -86,7 +91,131 @@ class LocalObjectDetector(ObjectDetector): tensor_input /= 255 return self.detect_api.detect_raw(tensor_input=tensor_input) + + def detect_raw_memx(self, tensor_input: np.ndarray): + if self.model_type == ModelTypeEnum.yolox: + + tensor_input = tensor_input.squeeze(0) + + padded_img = np.ones((640, 640, 3), + dtype=np.uint8) * 114 + + scale = min(640 / float(tensor_input.shape[0]), + 640 / float(tensor_input.shape[1])) + sx,sy = int(tensor_input.shape[1] * scale), int(tensor_input.shape[0] * scale) + + resized_img = cv2.resize(tensor_input, (sx,sy), interpolation=cv2.INTER_LINEAR) + padded_img[:sy, :sx] = resized_img.astype(np.uint8) + + + # Step 4: Slice the padded image into 4 quadrants and concatenate them into 12 channels + x0 = padded_img[0::2, 0::2, :] # Top-left + x1 = padded_img[1::2, 0::2, :] # Bottom-left + x2 = padded_img[0::2, 1::2, :] # Top-right + x3 = padded_img[1::2, 1::2, :] # Bottom-right + + # Step 5: Concatenate along the channel dimension (axis 2) + concatenated_img = np.concatenate([x0, x1, x2, x3], axis=2) + + # Step 6: Return the processed image as a contiguous array of type float32 + return np.ascontiguousarray(concatenated_img).astype(np.float32) + + # if self.dtype == InputDTypeEnum.float: + tensor_input = tensor_input.astype(np.float32) + tensor_input /= 255 + + tensor_input = tensor_input.transpose(1,2,0,3) #NHWC --> HWNC(dfp input shape) + + return tensor_input + + +def async_run_detector( + name: str, + detection_queue: mp.Queue, + out_events: dict[str, mp.Event], + avg_speed, + start, + detector_config, +): + threading.current_thread().name = f"detector:{name}" + logger.info(f"Starting MemryX Async detection process: {os.getpid()}") + setproctitle(f"frigate.detector.{name}") + + stop_event = mp.Event() + + def receiveSignal(signalNumber, frame): + stop_event.set() + + signal.signal(signal.SIGTERM, receiveSignal) + signal.signal(signal.SIGINT, receiveSignal) + + frame_manager = SharedMemoryFrameManager() + object_detector = LocalObjectDetector(detector_config=detector_config) + + outputs = {} + for name in out_events.keys(): + out_shm = UntrackedSharedMemory(name=f"out-{name}", create=False) + out_np = np.ndarray((20, 6), dtype=np.float32, buffer=out_shm.buf) + outputs[name] = {"shm": out_shm, "np": out_np} + + def detect_worker(): + """ Continuously fetch frames and send them to MemryX """ + logger.info(f"Starting Detect Worker Thread") + while not stop_event.is_set(): + try: + connection_id = detection_queue.get(timeout=1) + except queue.Empty: + continue + + input_frame = frame_manager.get( + connection_id, + (1, detector_config.model.height, detector_config.model.width, 3), + ) + + if input_frame is None: + logger.warning(f"Failed to get frame {connection_id} from SHM") + continue + + input_frame = object_detector.detect_raw_memx(input_frame) + + # Start measuring inference time + start.value = datetime.datetime.now().timestamp() + + # Send frame directly to MemryX processing + object_detector.detect_api.send_input(connection_id, input_frame) + + def result_worker(): + """ Continuously fetch results from MemryX and update outputs """ + logger.info(f"Starting Result Worker Thread") + while not stop_event.is_set(): + connection_id, detections = object_detector.detect_api.receive_output() + + # Calculate processing time + duration = datetime.datetime.now().timestamp() - start.value + frame_manager.close(connection_id) + + # Update average inference speed + avg_speed.value = (avg_speed.value * 9 + duration) / 10 + + if connection_id in outputs and detections is not None: + outputs[connection_id]["np"][:] = detections[:] + out_events[connection_id].set() + + # Initialize avg_speed + start.value = 0.0 + avg_speed.value = 0.0 # Start with an initial value + + # Start worker threads + detect_thread = threading.Thread(target=detect_worker, daemon=True) + result_thread = threading.Thread(target=result_worker, daemon=True) + detect_thread.start() + result_thread.start() + + while not stop_event.is_set(): + time.sleep(1) # Keep process alive + + logger.info("Exited MemryX detection process...") def run_detector( name: str, @@ -181,17 +310,31 @@ class ObjectDetectProcess: self.detection_start.value = 0.0 if (self.detect_process is not None) and self.detect_process.is_alive(): self.stop() - self.detect_process = util.Process( - target=run_detector, - name=f"detector:{self.name}", - args=( - self.name, - self.detection_queue, - self.out_events, - self.avg_inference_speed, - self.detection_start, - self.detector_config, - ), + if (self.detector_config.type == 'memryx'): + self.detect_process = util.Process( + target=async_run_detector, + name=f"detector:{self.name}", + args=( + self.name, + self.detection_queue, + self.out_events, + self.avg_inference_speed, + self.detection_start, + self.detector_config, + ), + ) + else: + self.detect_process = util.Process( + target=run_detector, + name=f"detector:{self.name}", + args=( + self.name, + self.detection_queue, + self.out_events, + self.avg_inference_speed, + self.detection_start, + self.detector_config, + ), ) self.detect_process.daemon = True self.detect_process.start() diff --git a/frigate/version.py b/frigate/version.py new file mode 100644 index 000000000..584d821ca --- /dev/null +++ b/frigate/version.py @@ -0,0 +1 @@ +VERSION = "0.16.0-2458f667" \ No newline at end of file diff --git a/startdocker.sh b/startdocker.sh new file mode 100755 index 000000000..0af2d26d3 --- /dev/null +++ b/startdocker.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Stop and remove existing container +docker stop frigate-memx +docker rm frigate-memx + +# Build the new Docker image +sudo docker build -t frigate-memx -f docker/main/Dockerfile . + +# Run the new container +sudo docker run -d \ + --name frigate-memx \ + --restart=unless-stopped \ + --mount type=tmpfs,target=/tmp/cache,tmpfs-size=1000000000 \ + --shm-size=256m \ + -v /home/memryx/final/Frigate_MemryX/config:/config \ + -e FRIGATE_RTSP_PASSWORD='password' \ + --add-host=host.docker.internal:host-gateway \ + --privileged=true \ + -p 8971:8971 \ + -p 8554:8554 \ + -p 5000:5000 \ + -p 8555:8555/tcp \ + -p 8555:8555/udp \ + --device /dev/memx0 frigate-memx + +echo "Frigate container restarted successfully."