Merge pull request #1 from abinila4/memryx

memryx-support-added

.gitignore

@@ -5,13 +5,13 @@ __pycache__
 debug
 .vscode/*
 !.vscode/launch.json
-config/*
+# config/*
 !config/*.example
 models
 *.mp4
 *.db
 *.csv
-frigate/version.py
+# frigate/version.py
 web/build
 web/node_modules
 web/coverage

config/.exports

@@ -0,0 +1 @@
+1744058517.724104

config/.jwt_secret

@@ -0,0 +1 @@
+c6dcc36e80f0d9f7090c478197acd9b1ac48a1e6312ce70809327a72b1c2b537666cc46a613e40ea7f50993a963f10eadd0e57a1ee7c529a9a907061eac57a57

config/.timeline

@@ -0,0 +1 @@
+1744058517.683821

config/.vacuum

@@ -0,0 +1 @@
+1744058517.699165

config/config.yaml

@@ -0,0 +1,83 @@
+mqtt:
+  enabled: false  # Set this to True if using MQTT for event triggers
+
+detectors:
+  memryx:
+    type: memryx
+    device: PCIe
+
+# model:
+#   model_type: yolov9
+#   width: 640
+#   height: 640
+#   path: /config/model_cache/memryx_cache/YOLO_v9_small_640_640_3_onnx.dfp
+#   labelmap_path: /config/model_cache/memryx_cache/labelmap.txt
+
+# model:
+#   model_type: yolov8
+#   width: 640
+#   height: 640
+#   path: /config/model_cache/memryx_cache/YOLO_v8_small_640_640_3_onnx.dfp
+#   labelmap_path: /config/model_cache/memryx_cache/labelmap.txt
+
+# model:
+#   model_type: yolonas
+#   width: 320
+#   height: 320
+#   path: /config/model_cache/memryx_cache/yolo_nas_s.dfp
+#   labelmap_path: /config/model_cache/memryx_cache/labelmap.txt
+
+model:
+  model_type: yolox
+  width: 640
+  height: 640
+  path: /config/model_cache/memryx_cache/YOLOX_640_640_3_onnx.dfp
+  labelmap_path: /config/model_cache/memryx_cache/labelmap.txt
+
+# model:
+#   model_type: ssd
+#   width: 320
+#   height: 320
+#   path: /config/model_cache/memryx_cache/SSDlite_MobileNet_v2_320_320_3_onnx.dfp
+#   labelmap_path: /config/model_cache/memryx_cache/labelmap.txt
+
+cameras:
+  Cam1:
+    ffmpeg:
+      inputs:
+        - path: rtsp://admin:NoPa$$word@192.168.56.22:554/cam/realmonitor?channel=1&subtype=0
+          roles:
+            - detect
+            - record
+    detect:
+      width: 640  
+      height: 480
+      fps: 30  
+      enabled: true
+
+    objects:
+      track:
+        - person
+        - cup
+        - bottle
+        - keyboard
+        - cell phone
+
+    snapshots:
+      enabled: false
+      bounding_box: true
+      retain:
+        default: 1  # Keep snapshots for 2 days
+    record:
+      enabled: false
+      retain:
+        days: 1  # Keep recordings for 7 days
+      alerts:
+        retain:
+          days: 1
+      detections:
+        retain:
+          days: 1
+
+version: 0.16-0
+

config/model_cache/memryx_cache/labelmap.txt

@@ -0,0 +1,80 @@
+0 person
+1 bicycle
+2 car
+3 motorcycle
+4 airplane
+5 bus
+6 train
+7 truck
+8 boat
+9 traffic light
+10 fire hydrant
+11 stop sign
+12 parking meter
+13 bench
+14 bird
+15 cat
+16 dog
+17 horse
+18 sheep
+19 cow
+20 elephant
+21 bear
+22 zebra
+23 giraffe
+24 backpack
+25 umbrella
+26 handbag
+27 tie
+28 suitcase
+29 frisbee
+30 skis
+31 snowboard
+32 sports ball
+33 kite
+34 baseball bat
+35 baseball glove
+36 skateboard
+37 surfboard
+38 tennis racket
+39 bottle
+40 wine glass
+41 cup
+42 fork
+43 knife
+44 spoon
+45 bowl
+46 banana
+47 apple
+48 sandwich
+49 orange
+50 broccoli
+51 carrot
+52 hot dog
+53 pizza
+54 donut
+55 cake
+56 chair
+57 couch
+58 potted plant
+59 bed
+60 dining table
+61 toilet
+62 tv
+63 laptop
+64 mouse
+65 remote
+66 keyboard
+67 cell phone
+68 microwave
+69 oven
+70 toaster
+71 sink
+72 refrigerator
+73 book
+74 clock
+75 vase
+76 scissors
+77 teddy bear
+78 hair drier
+79 toothbrush

docker/main/Dockerfile

@@ -245,6 +245,40 @@ RUN --mount=type=bind,from=wheels,source=/wheels,target=/deps/wheels \
 
 COPY --from=deps-rootfs / /
 
+####
+#
+# MemryX Support
+#
+# 1.  Install system dependencies and Python packages
+# 2. Add MemryX repo and install memx-accl
+#
+####
+# Install system dependencies
+RUN apt-get -qq update && \
+    apt-get -qq install -y --no-install-recommends \
+    libhdf5-dev \
+    python3-dev \
+    cmake \
+    python3-venv \
+    build-essential \
+    curl \
+    wget \
+    gnupg 
+
+RUN python3 -m pip install --upgrade pip && \
+    python3 -m pip install --extra-index-url https://developer.memryx.com/pip memryx
+
+# Add MemryX repo + key
+RUN curl -fsSL https://developer.memryx.com/deb/memryx.asc | tee /etc/apt/trusted.gpg.d/memryx.asc && \
+    echo "deb https://developer.memryx.com/deb stable main" > /etc/apt/sources.list.d/memryx.list && \
+    apt-get update -qq
+
+# Install memx-accl from MemryX repo
+RUN apt-get install -y --no-install-recommends memx-accl
+
+# Debug messages
+RUN echo "Hello from inside MemryX Docker image!"
+
 RUN ldconfig
 
 EXPOSE 5000

docker/memryx/user_installation.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+set -e  # Exit on error
+set -o pipefail
+
+echo "Starting MemryX driver and runtime installation..."
+
+# Detect architecture
+arch=$(uname -m)
+
+if [[ -d /sys/memx0/ ]]; then
+    echo "Existing functional MX3 driver found. Skipping driver re-install."
+else
+
+    # Purge existing packages and repo
+    echo "Removing old MemryX installations..."
+    sudo apt purge -y memx-* || true
+    sudo rm -f /etc/apt/sources.list.d/memryx.list /etc/apt/trusted.gpg.d/memryx.asc
+
+    # Install kernel headers
+    echo "Installing kernel headers for: $(uname -r)"
+    sudo apt update
+    sudo apt install -y linux-headers-$(uname -r)
+
+    # Add MemryX key and repo
+    echo "Adding MemryX GPG key and repository..."
+    wget -qO- https://developer.memryx.com/deb/memryx.asc | sudo tee /etc/apt/trusted.gpg.d/memryx.asc >/dev/null
+    echo 'deb https://developer.memryx.com/deb stable main' | sudo tee /etc/apt/sources.list.d/memryx.list >/dev/null
+
+    # Update and install packages
+    echo "Installing memx-drivers..."
+    sudo apt update
+    sudo apt install -y memx-drivers
+
+    # ARM-specific board setup
+    if [[ "$arch" == "aarch64" || "$arch" == "arm64" ]]; then
+        echo " Running ARM board setup..."
+        sudo mx_arm_setup
+    fi
+
+    echo -e "\n\n\033[1;31mYOU MUST RESTART YOUR COMPUTER NOW\033[0m\n\n"
+fi
+
+# Install mxa-manager
+echo "Installing mxa-manager..."
+sudo apt install -y memx-accl mxa-manager
+
+
+echo "MemryX installation complete!"

frigate/detectors/plugins/memryx.py

@@ -0,0 +1,519 @@
+import logging
+import numpy as np
+import cv2
+import os
+import urllib.request
+import zipfile
+from queue import Queue
+import time
+
+try:
+    # from memryx import AsyncAccl  # Import MemryX SDK
+    from memryx import AsyncAccl
+except ModuleNotFoundError:
+    raise ImportError("MemryX SDK is not installed. Install it and set up MIX environment.")
+
+from pydantic import BaseModel, Field
+from typing_extensions import Literal
+from frigate.detectors.detection_api import DetectionApi
+from frigate.detectors.detector_config import BaseDetectorConfig, ModelTypeEnum
+from frigate.util.model import post_process_yolov9
+
+logger = logging.getLogger(__name__)
+
+DETECTOR_KEY = "memryx"
+
+# Configuration class for model settings
+class ModelConfig(BaseModel):
+    path: str = Field(default=None, title="Model Path")  # Path to the DFP file
+    labelmap_path: str = Field(default=None, title="Path to Label Map") 
+
+class MemryXDetectorConfig(BaseDetectorConfig):
+    type: Literal[DETECTOR_KEY]
+    device: str = Field(default="PCIe", title="Device Path")
+
+class MemryXDetector(DetectionApi):
+    type_key = DETECTOR_KEY  # Set the type key 
+    supported_models = [
+        ModelTypeEnum.ssd,
+        ModelTypeEnum.yolonas,
+        ModelTypeEnum.yolov9,
+        ModelTypeEnum.yolox,
+    ]
+
+    def __init__(self, detector_config):
+
+        self.capture_queue = Queue(maxsize=10)
+        self.output_queue = Queue(maxsize=10)
+        self.capture_id_queue = Queue(maxsize=10)
+        self.logger = logger
+
+        """Initialize MemryX detector with the provided configuration."""
+        self.memx_model_path = detector_config.model.path  # Path to .dfp file
+        self.memx_post_model =  None # Path to .post file
+        self.expected_post_model = None
+        self.memx_device_path = detector_config.device  # Device path
+        self.memx_model_height = detector_config.model.height
+        self.memx_model_width = detector_config.model.width
+        self.memx_model_type = detector_config.model.model_type
+
+        self.cache_dir = "/config/model_cache/memryx_cache"
+
+        if self.memx_model_type == ModelTypeEnum.yolov9:
+            self.model_url = "https://developer.memryx.com/model_explorer/1p2/YOLO_v9_small_640_640_3_onnx.zip"
+            # self.expected_post_model = "YOLO_v9_small_640_640_3_onnx_post.onnx"
+            self.const_A = np.load("/config/model_cache/memryx_cache/_model_22_Constant_9_output_0.npy")
+            self.const_B = np.load("/config/model_cache/memryx_cache/_model_22_Constant_10_output_0.npy")
+            self.const_C = np.load("/config/model_cache/memryx_cache/_model_22_Constant_12_output_0.npy")  
+
+        elif self.memx_model_type == ModelTypeEnum.yolonas:
+            self.model_url = ""
+            self.expected_post_model = "yolo_nas_s_post.onnx"
+
+        elif self.memx_model_type == ModelTypeEnum.yolox:
+            self.model_url = "https://developer.memryx.com/model_explorer/1p2/YOLOX_640_640_3_onnx.zip"
+            # self.expected_post_model = "YOLOX_640_640_3_onnx_post.onnx"
+            self.set_strides_grids()
+
+        elif self.memx_model_type == ModelTypeEnum.ssd:
+            self.model_url = "https://developer.memryx.com/model_explorer/1p2/SSDlite_MobileNet_v2_320_320_3_onnx.zip"
+            self.expected_post_model = "SSDlite_MobileNet_v2_320_320_3_onnx_post.onnx"
+
+        self.check_and_prepare_model()
+        logger.info(f"Initializing MemryX with model: {self.memx_model_path} on device {self.memx_device_path}")
+
+        try:
+            # Load MemryX Model
+            logger.info(f"dfp path: {self.memx_model_path}")
+
+            # Your initialization code
+            self.accl = AsyncAccl(self.memx_model_path, mxserver_addr="host.docker.internal")
+            if self.memx_post_model:
+                self.accl.set_postprocessing_model(self.memx_post_model, model_idx=0)
+            self.accl.connect_input(self.process_input)
+            self.accl.connect_output(self.process_output)
+            # self.accl.wait()  # Wait for the accelerator to finish
+
+            logger.info(f"Loaded MemryX model from {self.memx_model_path} and {self.memx_post_model}")
+
+        except Exception as e:
+            logger.error(f"Failed to initialize MemryX model: {e}")
+            raise
+
+    def check_and_prepare_model(self):
+        """Check if both models exist; if not, download and extract them."""
+        if not os.path.exists(self.cache_dir):
+            os.makedirs(self.cache_dir)
+
+        if not self.expected_post_model:
+            logger.info(f"Assigned Model Path: {self.memx_model_path}")
+
+        else:
+            post_model_file_path = os.path.join(self.cache_dir, self.expected_post_model)
+            # model_file_path_tflite = os.path.join(self.cache_dir, self.expected_model_filename_tflite)
+
+            # Check if both required model files exist
+            if os.path.isfile(post_model_file_path):
+                self.memx_post_model = post_model_file_path
+                logger.info(f"Post-processing model found at {post_model_file_path}, skipping download.")
+            else:
+                logger.info(f"Model files not found. Downloading from {self.model_url}...")
+                zip_path = os.path.join(self.cache_dir, "memryx_model.zip")
+
+                # Download the ZIP file
+                urllib.request.urlretrieve(self.model_url, zip_path)
+                logger.info(f"Model ZIP downloaded to {zip_path}. Extracting...")
+
+                # Extract ZIP file
+                with zipfile.ZipFile(zip_path, "r") as zip_ref:
+                    zip_ref.extractall(self.cache_dir)
+
+                logger.info(f"Model extracted to {self.cache_dir}.")
+
+                # Assign extracted files to correct paths
+                for file in os.listdir(self.cache_dir):
+                    if file == self.expected_post_model:
+                        self.memx_post_model = os.path.join(self.cache_dir, file)
+
+                logger.info(f"Assigned Model Path: {self.memx_model_path}")
+                logger.info(f"Assigned Post-processing Model Path: {self.memx_post_model}")
+
+                # Cleanup: Remove the ZIP file after extraction
+                os.remove(zip_path)
+                logger.info("Cleaned up ZIP file after extraction.")
+
+    def send_input(self, connection_id, input_frame):
+        """Send frame directly to MemryX processing."""
+        # logging.info(f"Processing frame for connection ID: {connection_id}")
+
+        if input_frame is None:
+            raise ValueError("[send_input] No image data provided for inference")
+
+        # Send frame to MemryX for processing
+        self.capture_queue.put(input_frame)  # MemryX will process this
+        self.capture_id_queue.put(connection_id)  # Keep track of connection ID
+
+    def process_input(self):
+        """
+        Wait for frames in the queue, preprocess the image, and return it.
+        """
+        while True:
+            try:
+                # Wait for a frame from the queue (blocking call)
+                frame = self.capture_queue.get(block=True)  # Blocks until data is available
+                
+                return frame
+
+            except Exception as e:
+                logger.info(f"[process_input] Error processing input: {e}")
+                time.sleep(0.1)  # Prevent busy waiting in case of error
+
+    def receive_output(self):
+
+        """Retrieve processed results directly from MemryX."""
+        connection_id = self.capture_id_queue.get()  # Get the corresponding connection ID
+        detections = self.output_queue.get()  # Get detections from MemryX
+
+        return connection_id, detections
+
+    def post_process_yolonas(self, output):
+        predictions = output[0]
+
+        detections = np.zeros((20, 6), np.float32)
+
+        for i, prediction in enumerate(predictions):
+            if i == 20:
+                break
+
+            (_, x_min, y_min, x_max, y_max, confidence, class_id) = prediction
+            
+            if class_id < 0:
+                break
+            
+            detections[i] = [
+                class_id,
+                confidence,
+                y_min / self.memx_model_height,
+                x_min / self.memx_model_width,
+                y_max / self.memx_model_height,
+                x_max / self.memx_model_width,
+            ]
+        
+        # Return the list of final detections
+        self.output_queue.put(detections)
+
+    ## Takes in class ID, confidence score, and array of [x, y, w, h] that describes detection position,
+    ## returns an array that's easily passable back to Frigate.
+    def process_yolo(self, class_id, conf, pos):
+        return [
+            class_id,  # class ID
+            conf,  # confidence score
+            (pos[1] - (pos[3] / 2)) / self.memx_model_height,  # y_min
+            (pos[0] - (pos[2] / 2)) / self.memx_model_width,  # x_min
+            (pos[1] + (pos[3] / 2)) / self.memx_model_height,  # y_max
+            (pos[0] + (pos[2] / 2)) / self.memx_model_width,  # x_max
+            
+        ]
+    
+    def set_strides_grids(self):
+        grids = []
+        expanded_strides = []
+
+        strides = [8, 16, 32]
+
+        hsize_list = [self.memx_model_height // stride for stride in strides]
+        wsize_list = [self.memx_model_width // stride for stride in strides]
+
+        for hsize, wsize, stride in zip(hsize_list, wsize_list, strides):
+            xv, yv = np.meshgrid(np.arange(wsize), np.arange(hsize))
+            grid = np.stack((xv, yv), 2).reshape(1, -1, 2)
+            grids.append(grid)
+            shape = grid.shape[:2]
+            expanded_strides.append(np.full((*shape, 1), stride))
+        self.grids = np.concatenate(grids, 1)
+        self.expanded_strides = np.concatenate(expanded_strides, 1)
+
+    def sigmoid(self, x: np.ndarray) -> np.ndarray:
+
+        return 1 / (1 + np.exp(-x))
+
+    def onnx_concat(self, inputs: list, axis: int) -> np.ndarray:
+
+        # Ensure all inputs are numpy arrays
+        if not all(isinstance(x, np.ndarray) for x in inputs):
+            raise TypeError("All inputs must be numpy arrays.")
+        
+        # Ensure shapes match on non-concat axes
+        ref_shape = list(inputs[0].shape)
+        for i, tensor in enumerate(inputs[1:], start=1):
+            for ax in range(len(ref_shape)):
+                if ax == axis:
+                    continue
+                if tensor.shape[ax] != ref_shape[ax]:
+                    raise ValueError(f"Shape mismatch at axis {ax} between input[0] and input[{i}]")
+
+        return np.concatenate(inputs, axis=axis)
+
+    def onnx_reshape(self, data: np.ndarray, shape: np.ndarray) -> np.ndarray:
+
+        # Ensure shape is a 1D array of integers
+        target_shape = shape.astype(int).tolist()
+
+        # Use NumPy reshape with dynamic handling of -1
+        reshaped = np.reshape(data, target_shape)
+
+        return reshaped
+    
+    def post_process_yolox(self, output):
+
+        output = [np.expand_dims(tensor, axis=0) for tensor in output]  # Shape: (1, H, W, C)
+
+        # Move channel axis from 3rd (last) position to 1st position → (1, C, H, W)
+        output = [np.transpose(tensor, (0, 3, 1, 2)) for tensor in output]
+
+        output_785 = output[0]     # 785
+        output_794 = output[1]     # 794   
+        output_795 = output[2]     # 795
+        output_811 = output[3]     # 811
+        output_820 = output[4]     # 820
+        output_821 = output[5]     # 821
+        output_837 = output[6]     # 837
+        output_846 = output[7]     # 846
+        output_847 = output[8]     # 847
+
+        output_795 = self.sigmoid(output_795)
+        output_785 = self.sigmoid(output_785)
+        output_821 = self.sigmoid(output_821)
+        output_811 = self.sigmoid(output_811)
+        output_847 = self.sigmoid(output_847)
+        output_837 = self.sigmoid(output_837)
+
+        concat_1 = self.onnx_concat([output_794, output_795, output_785], axis=1)
+        concat_2 = self.onnx_concat([output_820, output_821, output_811], axis=1)
+        concat_3 = self.onnx_concat([output_846, output_847, output_837], axis=1)
+
+        shape = np.array([1, 85, -1], dtype=np.int64)   
+
+        reshape_1 = self.onnx_reshape(concat_1, shape)
+        reshape_2 = self.onnx_reshape(concat_2, shape)
+        reshape_3 = self.onnx_reshape(concat_3, shape)
+
+        concat_out = self.onnx_concat([reshape_1, reshape_2, reshape_3], axis=2)
+
+        output = concat_out.transpose(0,2,1)  #1, 840, 85
+
+        self.num_classes = output.shape[2] - 5
+
+        # [x, y, h, w, box_score, class_no_1, ..., class_no_80],
+        results = output
+
+        results[..., :2] = (results[..., :2] + self.grids) * self.expanded_strides
+        results[..., 2:4] = np.exp(results[..., 2:4]) * self.expanded_strides
+        image_pred = results[0, ...]
+
+        class_conf = np.max(image_pred[:, 5:5 + self.num_classes], axis=1, keepdims=True)
+        class_pred = np.argmax(image_pred[:, 5:5 + self.num_classes], axis=1)
+        class_pred = np.expand_dims(class_pred, axis=1)
+
+        conf_mask = (image_pred[:, 4] * class_conf.squeeze() >= 0.3).squeeze()
+        # Detections ordered as (x1, y1, x2, y2, obj_conf, class_conf, class_pred)
+        detections = np.concatenate((image_pred[:, :5], class_conf, class_pred), axis=1)
+        detections = detections[conf_mask]
+
+        # Sort by class confidence (index 5) and keep top 20 detections
+        ordered = detections[detections[:, 5].argsort()[::-1]][:20]
+
+        # Prepare a final detections array of shape (20, 6)
+        final_detections = np.zeros((20, 6), np.float32)
+        for i, object_detected in enumerate(ordered):
+            final_detections[i] = self.process_yolo(
+                object_detected[6], object_detected[5], object_detected[:4]
+            )
+
+        self.output_queue.put(final_detections)
+
+
+    def post_process_ssdlite(self, outputs):
+        dets = outputs[0].squeeze(0)  # Shape: (1, num_dets, 5)
+        labels = outputs[1].squeeze(0)
+
+        detections = []
+
+        for i in range(dets.shape[0]):
+            x_min, y_min, x_max, y_max, confidence = dets[i]
+            class_id = int(labels[i])  # Convert label to integer
+
+            if confidence < 0.45:
+                continue  # Skip detections below threshold
+
+            # Convert coordinates to integers
+            x_min, y_min, x_max, y_max = map(int, [x_min, y_min, x_max, y_max])
+
+            # Append valid detections [class_id, confidence, x, y, width, height]
+            detections.append([class_id, confidence, x_min, y_min, x_max, y_max])
+
+        final_detections = np.zeros((20, 6), np.float32)
+
+        if len(detections) == 0:
+            # logger.info("No detections found.")
+            self.output_queue.put(final_detections)
+            return
+
+        # Convert to NumPy array
+        detections = np.array(detections, dtype=np.float32)
+
+        # Apply Non-Maximum Suppression (NMS)
+        bboxes = detections[:, 2:6].tolist()  # (x_min, y_min, width, height)
+        scores = detections[:, 1].tolist()  # Confidence scores
+
+        indices = cv2.dnn.NMSBoxes(bboxes, scores, 0.45, 0.5)
+
+        if len(indices) > 0:
+            indices = indices.flatten()[:20]  # Keep only the top 20 detections
+            selected_detections = detections[indices]
+
+            # Normalize coordinates AFTER NMS
+            for i, det in enumerate(selected_detections):
+                class_id, confidence, x_min, y_min, x_max, y_max = det
+
+                # Normalize coordinates
+                x_min /= self.memx_model_width
+                y_min /= self.memx_model_height
+                x_max /= self.memx_model_width
+                y_max /= self.memx_model_height
+
+                final_detections[i] = [class_id, confidence,  y_min, x_min, y_max, x_max]
+
+        # logger.info(f"Final detections: {final_detections}")
+        self.output_queue.put(final_detections)
+
+    def onnx_reshape_with_allowzero(self, data: np.ndarray, shape: np.ndarray, allowzero: int = 0) -> np.ndarray:
+
+        shape = shape.astype(int)
+        input_shape = data.shape
+        output_shape = []
+
+        for i, dim in enumerate(shape):
+            if dim == 0 and allowzero == 0:
+                output_shape.append(input_shape[i])  # Copy dimension from input
+            else:
+                output_shape.append(dim)
+
+        # Now let NumPy infer any -1 if needed
+        reshaped = np.reshape(data, output_shape)
+
+        return reshaped
+
+    def process_output(self, *outputs):
+        
+        if self.memx_model_type == ModelTypeEnum.yolov9:
+            outputs = [np.expand_dims(tensor, axis=0) for tensor in outputs]  # Shape: (1, H, W, C)
+
+            # Move channel axis from 3rd (last) position to 1st position → (1, C, H, W)
+            outputs = [np.transpose(tensor, (0, 3, 1, 2)) for tensor in outputs]
+
+            conv_out1 = outputs[0]
+            conv_out2 = outputs[1]
+            conv_out3 = outputs[2]
+            conv_out4 = outputs[3]
+            conv_out5 = outputs[4]
+            conv_out6 = outputs[5]
+
+            concat_1 = self.onnx_concat([conv_out1, conv_out2], axis=1)
+            concat_2 = self.onnx_concat([conv_out3, conv_out4], axis=1)
+            concat_3 = self.onnx_concat([conv_out5, conv_out6], axis=1)
+
+            shape = np.array([1, 144, -1], dtype=np.int64)
+
+            reshaped_1 = self.onnx_reshape_with_allowzero(concat_1, shape, allowzero=0)
+            reshaped_2 = self.onnx_reshape_with_allowzero(concat_2, shape, allowzero=0)
+            reshaped_3 = self.onnx_reshape_with_allowzero(concat_3, shape, allowzero=0)
+
+            concat_4 = self.onnx_concat([reshaped_1, reshaped_2, reshaped_3], 2)
+
+            axis = 1
+            split_sizes = [64, 80]
+
+            # Calculate indices at which to split
+            indices = np.cumsum(split_sizes)[:-1]  # [64] — split before the second chunk
+
+            # Perform split along axis 1
+            split_0, split_1 = np.split(concat_4, indices, axis=axis)
+
+            shape1 = np.array([1,4,16,8400])
+            reshape_4 = self.onnx_reshape_with_allowzero(split_0, shape1, allowzero=0)
+
+            transpose_1 = reshape_4.transpose(0,2,1,3)
+
+            axis = 1  # As per ONNX softmax node
+
+            # Subtract max for numerical stability
+            x_max = np.max(transpose_1, axis=axis, keepdims=True)
+            x_exp = np.exp(transpose_1 - x_max)
+            x_sum = np.sum(x_exp, axis=axis, keepdims=True)
+            softmax_output = x_exp / x_sum
+
+            # Weight W from the ONNX initializer (1, 16, 1, 1) with values 0 to 15
+            W = np.arange(16, dtype=np.float32).reshape(1, 16, 1, 1)  # (1, 16, 1, 1)
+
+            # Apply 1x1 convolution: this is a weighted sum over channels
+            conv_output = np.sum(softmax_output * W, axis=1, keepdims=True)  # shape: (1, 1, 4, 8400)
+
+            shape2 = np.array([1,4,8400])
+            reshape_5 = self.onnx_reshape_with_allowzero(conv_output, shape2, allowzero=0)
+
+            # ONNX Slice — get first 2 channels: [0:2] along axis 1
+            slice_output1 = reshape_5[:, 0:2, :]  # Result: (1, 2, 8400)
+
+            # Slice channels 2 to 4 → axis = 1
+            slice_output2 = reshape_5[:, 2:4, :]
+
+            # Perform Subtraction
+            sub_output = self.const_A - slice_output1  # Equivalent to ONNX Sub
+
+            # Perform the ONNX-style Add
+            add_output = self.const_B + slice_output2
+
+            sub1 = add_output - sub_output
+
+            add1 = sub_output + add_output
+
+            div_output = add1 / 2.0
+
+            concat_5 = self.onnx_concat([div_output, sub1], axis=1)  
+
+            # const_C = np.load("_model_22_Constant_12_output_0.npy")  # Shape: (1, 8400)
+
+            # Expand B to (1, 1, 8400) so it can broadcast across axis=1 (4 channels)
+            const_C_expanded = self.const_C[:, np.newaxis, :]  # Shape: (1, 1, 8400)
+
+            # Perform ONNX-style element-wise multiplication
+            mul_output = concat_5 * const_C_expanded  # Result: (1, 4, 8400)
+
+            sigmoid_output = self.sigmoid(split_1)
+            outputs = self.onnx_concat([mul_output, sigmoid_output], axis=1)
+
+            final_detections = post_process_yolov9(outputs, self.memx_model_width, self.memx_model_height)
+            self.output_queue.put(final_detections)
+        
+        elif self.memx_model_type == ModelTypeEnum.yolonas:
+            return self.post_process_yolonas(outputs)
+        
+        elif self.memx_model_type == ModelTypeEnum.yolox:
+            return self.post_process_yolox(outputs)
+        
+        elif self.memx_model_type == ModelTypeEnum.ssd:
+            return self.post_process_ssdlite(outputs)
+        
+        else:
+            raise Exception(
+                f"{self.memx_model_type} is currently not supported for memryx. See the docs for more info on supported models."
+            )
+        
+    def detect_raw(self, tensor_input: np.ndarray):
+        """
+        Run inference on the input image and return raw results.
+        tensor_input: Preprocessed image (normalized & resized)
+        """
+        # logger.info("[detect_raw] Running inference on MemryX")
+        return 0

frigate/object_detection.py

@@ -8,6 +8,8 @@ import threading
 from abc import ABC, abstractmethod
 
 import numpy as np
+import cv2
+import time
 from setproctitle import setproctitle
 
 import frigate.util as util
@@ -16,6 +18,7 @@ from frigate.detectors.detector_config import (
     BaseDetectorConfig,
     InputDTypeEnum,
     InputTensorEnum,
+    ModelTypeEnum
 )
 from frigate.util.builtin import EventsPerSecond, load_labels
 from frigate.util.image import SharedMemoryFrameManager, UntrackedSharedMemory
@@ -49,6 +52,8 @@ class LocalObjectDetector(ObjectDetector):
             self.labels = {}
         else:
             self.labels = load_labels(labels)
+        
+        self.model_type = detector_config.model.model_type
 
         if detector_config:
             self.input_transform = tensor_transform(detector_config.model.input_tensor)
@@ -86,7 +91,131 @@ class LocalObjectDetector(ObjectDetector):
             tensor_input /= 255
 
         return self.detect_api.detect_raw(tensor_input=tensor_input)
+    
+    def detect_raw_memx(self, tensor_input: np.ndarray):
 
+        if self.model_type == ModelTypeEnum.yolox:
+            
+            tensor_input = tensor_input.squeeze(0)
+
+            padded_img = np.ones((640, 640, 3),
+                    dtype=np.uint8) * 114
+
+            scale = min(640 / float(tensor_input.shape[0]),
+                    640 / float(tensor_input.shape[1]))
+            sx,sy = int(tensor_input.shape[1] * scale), int(tensor_input.shape[0] * scale)
+
+            resized_img = cv2.resize(tensor_input, (sx,sy), interpolation=cv2.INTER_LINEAR)
+            padded_img[:sy, :sx] = resized_img.astype(np.uint8)
+            
+
+            # Step 4: Slice the padded image into 4 quadrants and concatenate them into 12 channels
+            x0 = padded_img[0::2, 0::2, :]  # Top-left
+            x1 = padded_img[1::2, 0::2, :]  # Bottom-left
+            x2 = padded_img[0::2, 1::2, :]  # Top-right
+            x3 = padded_img[1::2, 1::2, :]  # Bottom-right
+
+            # Step 5: Concatenate along the channel dimension (axis 2)
+            concatenated_img = np.concatenate([x0, x1, x2, x3], axis=2)
+
+            # Step 6: Return the processed image as a contiguous array of type float32
+            return np.ascontiguousarray(concatenated_img).astype(np.float32)
+
+        # if self.dtype == InputDTypeEnum.float:
+        tensor_input = tensor_input.astype(np.float32)
+        tensor_input /= 255
+
+        tensor_input = tensor_input.transpose(1,2,0,3)    #NHWC --> HWNC(dfp input shape)
+        
+        return tensor_input
+    
+
+def async_run_detector(
+    name: str,
+    detection_queue: mp.Queue,
+    out_events: dict[str, mp.Event],
+    avg_speed,
+    start,
+    detector_config,
+):
+    threading.current_thread().name = f"detector:{name}"
+    logger.info(f"Starting MemryX Async detection process: {os.getpid()}")
+    setproctitle(f"frigate.detector.{name}")
+
+    stop_event = mp.Event()
+
+    def receiveSignal(signalNumber, frame):
+        stop_event.set()
+
+    signal.signal(signal.SIGTERM, receiveSignal)
+    signal.signal(signal.SIGINT, receiveSignal)
+
+    frame_manager = SharedMemoryFrameManager()
+    object_detector = LocalObjectDetector(detector_config=detector_config)
+
+    outputs = {}
+    for name in out_events.keys():
+        out_shm = UntrackedSharedMemory(name=f"out-{name}", create=False)
+        out_np = np.ndarray((20, 6), dtype=np.float32, buffer=out_shm.buf)
+        outputs[name] = {"shm": out_shm, "np": out_np}
+
+    def detect_worker():
+        """ Continuously fetch frames and send them to MemryX """
+        logger.info(f"Starting Detect Worker Thread")
+        while not stop_event.is_set():
+            try:
+                connection_id = detection_queue.get(timeout=1)
+            except queue.Empty:
+                continue
+
+            input_frame = frame_manager.get(
+                connection_id,
+                (1, detector_config.model.height, detector_config.model.width, 3),
+            )
+
+            if input_frame is None:
+                logger.warning(f"Failed to get frame {connection_id} from SHM")
+                continue
+            
+            input_frame = object_detector.detect_raw_memx(input_frame)
+
+            # Start measuring inference time
+            start.value = datetime.datetime.now().timestamp()
+
+            # Send frame directly to MemryX processing
+            object_detector.detect_api.send_input(connection_id, input_frame)
+
+    def result_worker():
+        """ Continuously fetch results from MemryX and update outputs """
+        logger.info(f"Starting Result Worker Thread")
+        while not stop_event.is_set():
+            connection_id, detections = object_detector.detect_api.receive_output()
+
+            # Calculate processing time
+            duration = datetime.datetime.now().timestamp() - start.value
+            frame_manager.close(connection_id)
+
+            # Update average inference speed
+            avg_speed.value = (avg_speed.value * 9 + duration) / 10
+
+            if connection_id in outputs and detections is not None:
+                outputs[connection_id]["np"][:] = detections[:]
+                out_events[connection_id].set()
+
+    # Initialize avg_speed
+    start.value = 0.0
+    avg_speed.value = 0.0  # Start with an initial value
+
+    # Start worker threads
+    detect_thread = threading.Thread(target=detect_worker, daemon=True)
+    result_thread = threading.Thread(target=result_worker, daemon=True)
+    detect_thread.start()
+    result_thread.start()
+
+    while not stop_event.is_set():
+        time.sleep(1)  # Keep process alive
+
+    logger.info("Exited MemryX detection process...")
 
 def run_detector(
     name: str,
@@ -181,17 +310,31 @@ class ObjectDetectProcess:
         self.detection_start.value = 0.0
         if (self.detect_process is not None) and self.detect_process.is_alive():
             self.stop()
-        self.detect_process = util.Process(
+        if (self.detector_config.type == 'memryx'):
-            target=run_detector,
+            self.detect_process = util.Process(
-            name=f"detector:{self.name}",
+                target=async_run_detector,
-            args=(
+                name=f"detector:{self.name}",
-                self.name,
+                args=(
-                self.detection_queue,
+                    self.name,
-                self.out_events,
+                    self.detection_queue,
-                self.avg_inference_speed,
+                    self.out_events,
-                self.detection_start,
+                    self.avg_inference_speed,
-                self.detector_config,
+                    self.detection_start,
-            ),
+                    self.detector_config,
+                ),
+            )
+        else:
+            self.detect_process = util.Process(
+                target=run_detector,
+                name=f"detector:{self.name}",
+                args=(
+                    self.name,
+                    self.detection_queue,
+                    self.out_events,
+                    self.avg_inference_speed,
+                    self.detection_start,
+                    self.detector_config,
+                ),
         )
         self.detect_process.daemon = True
         self.detect_process.start()

frigate/version.py

@@ -0,0 +1 @@
+VERSION = "0.16.0-2458f667"

startdocker.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Stop and remove existing container
+docker stop frigate-memx
+docker rm frigate-memx
+
+# Build the new Docker image
+sudo docker build -t frigate-memx -f docker/main/Dockerfile .
+
+# Run the new container
+sudo docker run -d \
+  --name frigate-memx \
+  --restart=unless-stopped \
+  --mount type=tmpfs,target=/tmp/cache,tmpfs-size=1000000000 \
+  --shm-size=256m \
+  -v /home/memryx/final/Frigate_MemryX/config:/config \
+  -e FRIGATE_RTSP_PASSWORD='password' \
+  --add-host=host.docker.internal:host-gateway \
+  --privileged=true \
+  -p 8971:8971 \
+  -p 8554:8554 \
+  -p 5000:5000 \
+  -p 8555:8555/tcp \
+  -p 8555:8555/udp \
+  --device /dev/memx0 frigate-memx
+
+echo "Frigate container restarted successfully."