From 662025a961a8110bf47488823f1d76d6093fb24b Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Sun, 16 Jul 2023 06:42:56 -0600 Subject: [PATCH] Remove faster_fifo (#7181) * Remove faster_fifo * Remove const --- Dockerfile | 4 +--- frigate/app.py | 35 +++++++++------------------------- frigate/comms/inter_process.py | 3 +-- frigate/const.py | 4 ---- frigate/events/audio.py | 5 ++--- frigate/events/external.py | 2 +- frigate/events/maintainer.py | 3 +-- frigate/log.py | 2 +- frigate/object_detection.py | 3 +-- frigate/record/maintainer.py | 6 +++--- frigate/record/record.py | 5 ++--- frigate/timeline.py | 3 +-- frigate/types.py | 3 +-- frigate/video.py | 5 ++--- requirements-wheels.txt | 1 - 15 files changed, 26 insertions(+), 58 deletions(-) diff --git a/Dockerfile b/Dockerfile index 63ae4e212..89e5bc3a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -134,9 +134,7 @@ RUN apt-get -qq update \ libtbb2 libtbb-dev libdc1394-22-dev libopenexr-dev \ libgstreamer-plugins-base1.0-dev libgstreamer1.0-dev \ # scipy dependencies - gcc gfortran libopenblas-dev liblapack-dev \ - # faster-fifo dependencies - g++ cython3 && \ + gcc gfortran libopenblas-dev liblapack-dev && \ rm -rf /var/lib/apt/lists/* RUN wget -q https://bootstrap.pypa.io/get-pip.py -O get-pip.py \ diff --git a/frigate/app.py b/frigate/app.py index 6296134eb..7a9f7538b 100644 --- a/frigate/app.py +++ b/frigate/app.py @@ -6,13 +6,12 @@ import shutil import signal import sys import traceback +from multiprocessing import Queue from multiprocessing.synchronize import Event as MpEvent from types import FrameType from typing import Optional -import faster_fifo as ff import psutil -from faster_fifo import Queue from peewee_migrate import Router from playhouse.sqlite_ext import SqliteExtDatabase from playhouse.sqliteq import SqliteQueueDatabase @@ -27,7 +26,6 @@ from frigate.const import ( CLIPS_DIR, CONFIG_DIR, DEFAULT_DB_PATH, - DEFAULT_QUEUE_BUFFER_SIZE, EXPORT_DIR, MODEL_CACHE_DIR, RECORD_DIR, @@ -60,11 +58,11 @@ logger = logging.getLogger(__name__) class FrigateApp: def __init__(self) -> None: self.stop_event: MpEvent = mp.Event() - self.detection_queue: Queue = ff.Queue() + self.detection_queue: Queue = mp.Queue() self.detectors: dict[str, ObjectDetectProcess] = {} self.detection_out_events: dict[str, MpEvent] = {} self.detection_shms: list[mp.shared_memory.SharedMemory] = [] - self.log_queue: Queue = ff.Queue() + self.log_queue: Queue = mp.Queue() self.plus_api = PlusApi() self.camera_metrics: dict[str, CameraMetricsTypes] = {} self.feature_metrics: dict[str, FeatureMetricsTypes] = {} @@ -210,14 +208,8 @@ class FrigateApp: def init_queues(self) -> None: # Queues for clip processing - self.event_queue: Queue = ff.Queue( - DEFAULT_QUEUE_BUFFER_SIZE - * sum(camera.enabled for camera in self.config.cameras.values()) - ) - self.event_processed_queue: Queue = ff.Queue( - DEFAULT_QUEUE_BUFFER_SIZE - * sum(camera.enabled for camera in self.config.cameras.values()) - ) + self.event_queue: Queue = mp.Queue() + self.event_processed_queue: Queue = mp.Queue() self.video_output_queue: Queue = mp.Queue( maxsize=sum(camera.enabled for camera in self.config.cameras.values()) * 2 ) @@ -228,29 +220,20 @@ class FrigateApp: ) # Queue for object recordings info - self.object_recordings_info_queue: Queue = ff.Queue( - DEFAULT_QUEUE_BUFFER_SIZE - * sum(camera.enabled for camera in self.config.cameras.values()) - ) + self.object_recordings_info_queue: Queue = mp.Queue() # Queue for audio recordings info if enabled self.audio_recordings_info_queue: Optional[Queue] = ( - ff.Queue( - DEFAULT_QUEUE_BUFFER_SIZE - * sum(camera.audio.enabled for camera in self.config.cameras.values()) - ) + mp.Queue() if len([c for c in self.config.cameras.values() if c.audio.enabled]) > 0 else None ) # Queue for timeline events - self.timeline_queue: Queue = ff.Queue( - DEFAULT_QUEUE_BUFFER_SIZE - * sum(camera.enabled for camera in self.config.cameras.values()) - ) + self.timeline_queue: Queue = mp.Queue() # Queue for inter process communication - self.inter_process_queue: Queue = ff.Queue(DEFAULT_QUEUE_BUFFER_SIZE) + self.inter_process_queue: Queue = mp.Queue() def init_database(self) -> None: def vacuum_db(db: SqliteExtDatabase) -> None: diff --git a/frigate/comms/inter_process.py b/frigate/comms/inter_process.py index ff4a1180a..74ce9bc0c 100644 --- a/frigate/comms/inter_process.py +++ b/frigate/comms/inter_process.py @@ -1,11 +1,10 @@ import multiprocessing as mp import queue import threading +from multiprocessing import Queue from multiprocessing.synchronize import Event as MpEvent from typing import Callable -from faster_fifo import Queue - from frigate.comms.dispatcher import Communicator diff --git a/frigate/const.py b/frigate/const.py index c7c75c6b3..b6b0e44bd 100644 --- a/frigate/const.py +++ b/frigate/const.py @@ -46,7 +46,3 @@ DRIVER_INTEL_iHD = "iHD" MAX_SEGMENT_DURATION = 600 MAX_PLAYLIST_SECONDS = 7200 # support 2 hour segments for a single playlist to account for cameras with inconsistent segment times - -# Queue Values - -DEFAULT_QUEUE_BUFFER_SIZE = 1000 * 1000 # 1MB diff --git a/frigate/events/audio.py b/frigate/events/audio.py index 1c39e4d59..d2b3d8298 100644 --- a/frigate/events/audio.py +++ b/frigate/events/audio.py @@ -9,7 +9,6 @@ import threading from types import FrameType from typing import Optional, Tuple -import faster_fifo as ff import numpy as np import requests from setproctitle import setproctitle @@ -52,7 +51,7 @@ def get_ffmpeg_command(input_args: list[str], input_path: str, pipe: str) -> lis def listen_to_audio( config: FrigateConfig, - recordings_info_queue: ff.Queue, + recordings_info_queue: mp.Queue, process_info: dict[str, FeatureMetricsTypes], inter_process_communicator: InterProcessCommunicator, ) -> None: @@ -152,7 +151,7 @@ class AudioEventMaintainer(threading.Thread): def __init__( self, camera: CameraConfig, - recordings_info_queue: ff.Queue, + recordings_info_queue: mp.Queue, feature_metrics: dict[str, FeatureMetricsTypes], stop_event: mp.Event, inter_process_communicator: InterProcessCommunicator, diff --git a/frigate/events/external.py b/frigate/events/external.py index a801e6d24..376f47211 100644 --- a/frigate/events/external.py +++ b/frigate/events/external.py @@ -6,10 +6,10 @@ import logging import os import random import string +from multiprocessing import Queue from typing import Optional import cv2 -from faster_fifo import Queue from frigate.config import CameraConfig, FrigateConfig from frigate.const import CLIPS_DIR diff --git a/frigate/events/maintainer.py b/frigate/events/maintainer.py index d92bb0a44..0c9986693 100644 --- a/frigate/events/maintainer.py +++ b/frigate/events/maintainer.py @@ -3,11 +3,10 @@ import logging import queue import threading from enum import Enum +from multiprocessing import Queue from multiprocessing.synchronize import Event as MpEvent from typing import Dict -from faster_fifo import Queue - from frigate.config import EventsConfig, FrigateConfig from frigate.models import Event from frigate.types import CameraMetricsTypes diff --git a/frigate/log.py b/frigate/log.py index e839de76d..387db0217 100644 --- a/frigate/log.py +++ b/frigate/log.py @@ -7,10 +7,10 @@ import signal import threading from collections import deque from logging import handlers +from multiprocessing import Queue from types import FrameType from typing import Deque, Optional -from faster_fifo import Queue from setproctitle import setproctitle from frigate.util.builtin import clean_camera_user_pass diff --git a/frigate/object_detection.py b/frigate/object_detection.py index 22adb954e..9d00dca46 100644 --- a/frigate/object_detection.py +++ b/frigate/object_detection.py @@ -7,7 +7,6 @@ import signal import threading from abc import ABC, abstractmethod -import faster_fifo as ff import numpy as np from setproctitle import setproctitle @@ -78,7 +77,7 @@ class LocalObjectDetector(ObjectDetector): def run_detector( name: str, - detection_queue: ff.Queue, + detection_queue: mp.Queue, out_events: dict[str, mp.Event], avg_speed, start, diff --git a/frigate/record/maintainer.py b/frigate/record/maintainer.py index 044a8e0d7..575a5b91e 100644 --- a/frigate/record/maintainer.py +++ b/frigate/record/maintainer.py @@ -3,6 +3,7 @@ import asyncio import datetime import logging +import multiprocessing as mp import os import queue import random @@ -14,7 +15,6 @@ from multiprocessing.synchronize import Event as MpEvent from pathlib import Path from typing import Any, Optional, Tuple -import faster_fifo as ff import numpy as np import psutil @@ -32,8 +32,8 @@ class RecordingMaintainer(threading.Thread): def __init__( self, config: FrigateConfig, - object_recordings_info_queue: ff.Queue, - audio_recordings_info_queue: Optional[ff.Queue], + object_recordings_info_queue: mp.Queue, + audio_recordings_info_queue: Optional[mp.Queue], process_info: dict[str, FeatureMetricsTypes], stop_event: MpEvent, ): diff --git a/frigate/record/record.py b/frigate/record/record.py index 1a9edada7..c76ff3d63 100644 --- a/frigate/record/record.py +++ b/frigate/record/record.py @@ -7,7 +7,6 @@ import threading from types import FrameType from typing import Optional -import faster_fifo as ff from playhouse.sqliteq import SqliteQueueDatabase from setproctitle import setproctitle @@ -23,8 +22,8 @@ logger = logging.getLogger(__name__) def manage_recordings( config: FrigateConfig, - object_recordings_info_queue: ff.Queue, - audio_recordings_info_queue: ff.Queue, + object_recordings_info_queue: mp.Queue, + audio_recordings_info_queue: mp.Queue, process_info: dict[str, FeatureMetricsTypes], ) -> None: stop_event = mp.Event() diff --git a/frigate/timeline.py b/frigate/timeline.py index 48392ad96..861dbe411 100644 --- a/frigate/timeline.py +++ b/frigate/timeline.py @@ -3,10 +3,9 @@ import logging import queue import threading +from multiprocessing import Queue from multiprocessing.synchronize import Event as MpEvent -from faster_fifo import Queue - from frigate.config import FrigateConfig from frigate.events.maintainer import EventTypeEnum from frigate.models import Timeline diff --git a/frigate/types.py b/frigate/types.py index 05181d714..5637c325e 100644 --- a/frigate/types.py +++ b/frigate/types.py @@ -1,10 +1,9 @@ +from multiprocessing import Queue from multiprocessing.context import Process from multiprocessing.sharedctypes import Synchronized from multiprocessing.synchronize import Event from typing import Optional, TypedDict -from faster_fifo import Queue - from frigate.object_detection import ObjectDetectProcess diff --git a/frigate/video.py b/frigate/video.py index d9dc97acf..075e53a0c 100755 --- a/frigate/video.py +++ b/frigate/video.py @@ -11,7 +11,6 @@ import time from collections import defaultdict import cv2 -import faster_fifo as ff import numpy as np from setproctitle import setproctitle @@ -731,7 +730,7 @@ def get_consolidated_object_detections(detected_object_groups): def process_frames( camera_name: str, - frame_queue: ff.Queue, + frame_queue: mp.Queue, frame_shape, model_config: ModelConfig, detect_config: DetectConfig, @@ -739,7 +738,7 @@ def process_frames( motion_detector: MotionDetector, object_detector: RemoteObjectDetector, object_tracker: ObjectTracker, - detected_objects_queue: ff.Queue, + detected_objects_queue: mp.Queue, process_info: dict, objects_to_track: list[str], object_filters, diff --git a/requirements-wheels.txt b/requirements-wheels.txt index 3232e8f31..30f8908b5 100644 --- a/requirements-wheels.txt +++ b/requirements-wheels.txt @@ -1,6 +1,5 @@ click == 8.1.* Flask == 2.3.* -faster-fifo == 1.4.* imutils == 0.5.* matplotlib == 3.7.* mypy == 1.4.1