Optimize nginx & recordings (#4688)

* Add segment duration metadata

* Use faststart only for kept segments

* Add more options for performance

* Build nginx locally

* Build nginx in dockerfile and enable threaded vod handling

* Use DASH instead of hls

* Allow player to continue on error

* Undo DASH change

* Fix typo

* Correct log

* Fix bad comments

* Fix indentation

* Preload stream

* remove unused

* Fix spacing

* Fix tabs / sspaces

* Retab

* More cleanup

Dockerfile

@@ -9,9 +9,6 @@ FROM --platform=linux/amd64 debian:11 AS base_amd64
 
 FROM debian:11-slim AS slim-base
 
-FROM blakeblackshear/frigate-nginx:1.0.2 AS nginx
-
-
 FROM slim-base AS wget
 ARG DEBIAN_FRONTEND
 RUN apt-get update \
@@ -19,6 +16,53 @@ RUN apt-get update \
     && rm -rf /var/lib/apt/lists/*
 WORKDIR /rootfs
 
+FROM ubuntu:20.04 AS nginx
+ARG DEBIAN_FRONTEND
+ARG NGINX_VERSION=1.22.1
+ARG VOD_MODULE_VERSION=1.30
+ARG SECURE_TOKEN_MODULE_VERSION=1.4
+ARG RTMP_MODULE_VERSION=1.2.1
+
+RUN cp /etc/apt/sources.list /etc/apt/sources.list~ \
+    && sed -Ei 's/^# deb-src /deb-src /' /etc/apt/sources.list \
+    && apt-get update
+
+RUN apt-get -yqq build-dep nginx
+
+RUN apt-get -yqq install --no-install-recommends ca-certificates wget \
+    && mkdir /tmp/nginx \
+    && wget https://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz \
+    && tar -zxf nginx-${NGINX_VERSION}.tar.gz -C /tmp/nginx --strip-components=1 \
+    && rm nginx-${NGINX_VERSION}.tar.gz \
+    && mkdir /tmp/nginx-vod-module \
+    && wget https://github.com/kaltura/nginx-vod-module/archive/refs/tags/${VOD_MODULE_VERSION}.tar.gz \
+    && tar -zxf ${VOD_MODULE_VERSION}.tar.gz -C /tmp/nginx-vod-module --strip-components=1 \
+    && rm ${VOD_MODULE_VERSION}.tar.gz \
+    # Patch MAX_CLIPS to allow more clips to be added than the default 128
+    && sed -i 's/MAX_CLIPS (128)/MAX_CLIPS (1080)/g' /tmp/nginx-vod-module/vod/media_set.h \
+    && mkdir /tmp/nginx-secure-token-module \
+    && wget https://github.com/kaltura/nginx-secure-token-module/archive/refs/tags/${SECURE_TOKEN_MODULE_VERSION}.tar.gz \
+    && tar -zxf ${SECURE_TOKEN_MODULE_VERSION}.tar.gz -C /tmp/nginx-secure-token-module --strip-components=1 \
+    && rm ${SECURE_TOKEN_MODULE_VERSION}.tar.gz \
+    && mkdir /tmp/nginx-rtmp-module \
+    && wget https://github.com/arut/nginx-rtmp-module/archive/refs/tags/v${RTMP_MODULE_VERSION}.tar.gz \
+    && tar -zxf v${RTMP_MODULE_VERSION}.tar.gz -C /tmp/nginx-rtmp-module --strip-components=1 \
+    && rm v${RTMP_MODULE_VERSION}.tar.gz
+
+WORKDIR /tmp/nginx
+
+RUN ./configure --prefix=/usr/local/nginx \
+    --with-file-aio \
+    --with-http_sub_module \
+    --with-http_ssl_module \
+    --with-threads \
+    --add-module=../nginx-vod-module \
+    --add-module=../nginx-secure-token-module \
+    --add-module=../nginx-rtmp-module \
+    --with-cc-opt="-O3 -Wno-error=implicit-fallthrough"
+
+RUN make && make install
+RUN rm -rf /usr/local/nginx/html /usr/local/nginx/conf/*.default
 
 FROM wget AS go2rtc
 ARG TARGETARCH

docker/rootfs/usr/local/nginx/conf/nginx.conf

@@ -1,27 +1,30 @@
 daemon off;
 user root;
-worker_processes  1;
+worker_processes auto;
 
-error_log  /dev/stdout warn;
+error_log /dev/stdout warn;
-pid        /var/run/nginx.pid;
+pid /var/run/nginx.pid;
 
 events {
-    worker_connections  1024;
+    worker_connections 1024;
 }
 
 http {
-    include       mime.types;
+    include mime.types;
-    default_type  application/octet-stream;
+    default_type application/octet-stream;
 
-    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
                         '$status $body_bytes_sent "$http_referer" '
                         '"$http_user_agent" "$http_x_forwarded_for"';
 
-    access_log  /dev/stdout  main;
+    access_log /dev/stdout main;
 
-    sendfile        on;
+    # send headers in one piece, it is better than sending them one by one
+    tcp_nopush on;
 
-    keepalive_timeout  65;
+    sendfile on;
+
+    keepalive_timeout 65;
 
     gzip on;
     gzip_comp_level 6;
@@ -30,18 +33,18 @@ http {
     gzip_vary on;
 
     upstream frigate_api {
-      server 127.0.0.1:5001;
+        server 127.0.0.1:5001;
-      keepalive 1024;
+        keepalive 1024;
     }
 
     upstream mqtt_ws {
-      server 127.0.0.1:5002;
+        server 127.0.0.1:5002;
-      keepalive 1024;
+        keepalive 1024;
     }
 
     upstream jsmpeg {
-      server 127.0.0.1:8082;
+        server 127.0.0.1:8082;
-      keepalive 1024;
+        keepalive 1024;
     }
 
     upstream go2rtc {
@@ -61,6 +64,19 @@ http {
         vod_align_segments_to_key_frames on;
         vod_manifest_segment_durations_mode accurate;
         vod_ignore_edit_list on;
+        vod_segment_duration 10000;
+        vod_hls_mpegts_align_frames off;
+        vod_hls_mpegts_interleave_frames on;
+
+        # file handle caching / aio
+        open_file_cache max=1000 inactive=5m;
+        open_file_cache_valid 2m;
+        open_file_cache_min_uses 1;
+        open_file_cache_errors on;
+        aio on;
+
+        # https://github.com/kaltura/nginx-vod-module#vod_open_file_thread_pool
+        vod_open_file_thread_pool default;
 
         # vod caches
         vod_metadata_cache metadata_cache 512m;
@@ -70,18 +86,12 @@ http {
         gzip on;
         gzip_types application/vnd.apple.mpegurl;
 
-        # file handle caching / aio
-        open_file_cache          max=1000 inactive=5m;
-        open_file_cache_valid    2m;
-        open_file_cache_min_uses 1;
-        open_file_cache_errors   on;
-        aio on;
-
         location /vod/ {
+            aio threads;
             vod hls;
 
             secure_token $args;
-			secure_token_types application/vnd.apple.mpegurl;
+            secure_token_types application/vnd.apple.mpegurl;
 
             add_header Access-Control-Allow-Headers '*';
             add_header Access-Control-Expose-Headers 'Server,range,Content-Length,Content-Range';
@@ -137,8 +147,8 @@ http {
         }
 
         location /cache/ {
-                internal; # This tells nginx it's not accessible from the outside
+            internal; # This tells nginx it's not accessible from the outside
-                alias /tmp/cache/;
+            alias /tmp/cache/;
         }
 
         location /recordings/ {
@@ -257,4 +267,4 @@ rtmp {
             meta copy;
         }
     }
-}
+}

frigate/const.py

@@ -5,6 +5,7 @@ CACHE_DIR = "/tmp/cache"
 YAML_EXT = (".yaml", ".yml")
 PLUS_ENV_VAR = "PLUS_API_KEY"
 PLUS_API_HOST = "https://api.frigate.video"
+MAX_SEGMENT_DURATION = 600
 
 # Regex Consts
 

frigate/http.py

@@ -32,7 +32,7 @@ from peewee import SqliteDatabase, operator, fn, DoesNotExist
 from playhouse.shortcuts import model_to_dict
 
 from frigate.config import FrigateConfig
-from frigate.const import CLIPS_DIR, RECORD_DIR
+from frigate.const import CLIPS_DIR, MAX_SEGMENT_DURATION, RECORD_DIR
 from frigate.models import Event, Recordings
 from frigate.object_processing import TrackedObject
 from frigate.stats import stats_snapshot
@@ -1050,6 +1050,7 @@ def vod_ts(camera_name, start_ts, end_ts):
 
     clips = []
     durations = []
+    max_duration_ms = MAX_SEGMENT_DURATION * 1000
 
     recording: Recordings
     for recording in recordings:
@@ -1060,7 +1061,7 @@ def vod_ts(camera_name, start_ts, end_ts):
         if recording.end_time > end_ts:
             duration -= int((recording.end_time - end_ts) * 1000)
 
-        if duration > 0:
+        if 0 < duration < max_duration_ms:
             clip["keyFrameDurations"] = [duration]
             clips.append(clip)
             durations.append(duration)
@@ -1076,7 +1077,9 @@ def vod_ts(camera_name, start_ts, end_ts):
         {
             "cache": hour_ago.timestamp() > start_ts,
             "discontinuity": False,
+            "consistentSequenceMediaInfo": True,
             "durations": durations,
+            "segment_duration": max(durations),
             "sequences": [{"clips": clips}],
         }
     )

frigate/record.py

@@ -5,7 +5,6 @@ import multiprocessing as mp
 import os
 import queue
 import random
-import shutil
 import string
 import subprocess as sp
 import threading
@@ -16,7 +15,7 @@ import psutil
 from peewee import JOIN, DoesNotExist
 
 from frigate.config import RetainModeEnum, FrigateConfig
-from frigate.const import CACHE_DIR, RECORD_DIR
+from frigate.const import CACHE_DIR, MAX_SEGMENT_DURATION, RECORD_DIR
 from frigate.models import Event, Recordings
 from frigate.util import area
 
@@ -173,7 +172,7 @@ class RecordingMaintainer(threading.Thread):
                         duration = -1
 
                     # ensure duration is within expected length
-                    if 0 < duration < 600:
+                    if 0 < duration < MAX_SEGMENT_DURATION:
                         end_time = start_time + datetime.timedelta(seconds=duration)
                         self.end_time_cache[cache_path] = (end_time, duration)
                     else:
@@ -296,13 +295,38 @@ class RecordingMaintainer(threading.Thread):
         try:
             if not os.path.exists(file_path):
                 start_frame = datetime.datetime.now().timestamp()
-                # copy then delete is required when recordings are stored on some network drives
+
-                shutil.copyfile(cache_path, file_path)
+                # add faststart to kept segments to improve metadata reading
-                logger.debug(
+                ffmpeg_cmd = [
-                    f"Copied {file_path} in {datetime.datetime.now().timestamp()-start_frame} seconds."
+                    "ffmpeg",
+                    "-y",
+                    "-i",
+                    cache_path,
+                    "-c",
+                    "copy",
+                    "-movflags",
+                    "+faststart",
+                    file_path,
+                ]
+
+                p = sp.run(
+                    ffmpeg_cmd,
+                    encoding="ascii",
+                    capture_output=True,
                 )
 
+                if p.returncode != 0:
+                    logger.error(f"Unable to convert {cache_path} to {file_path}")
+                    logger.error(p.stderr)
+                    return
+                else:
+                    logger.debug(
+                        f"Copied {file_path} in {datetime.datetime.now().timestamp()-start_frame} seconds."
+                    )
+
                 try:
+                    # get the segment size of the cache file
+                    # file without faststart is same size
                     segment_size = round(
                         float(os.path.getsize(cache_path)) / 1000000, 1
                     )

web/src/components/VideoPlayer.jsx

@@ -33,6 +33,9 @@ export default function VideoPlayer({ children, options, seekOptions = {}, onRea
       ...seekOptions,
     });
 
+    // Allows player to continue on error
+    player.reloadSourceOnError();
+
     // Disable fullscreen on iOS if we have children
     if (
       children &&

web/src/routes/Events.jsx

@@ -561,7 +561,7 @@ export default function Events({ path, ...props }) {
                                 autoplay: true,
                                 sources: [
                                   {
-                                    src: `${apiHost}/vod/event/${event.id}/master.m3u8`,
+                                    src: `${apiHost}vod/event/${event.id}/master.m3u8`,
                                     type: 'application/vnd.apple.mpegurl',
                                   },
                                 ],

web/src/routes/Recording.jsx

@@ -69,7 +69,7 @@ export default function Recording({ camera, date, hour = '00', minute = '00', se
           description: `${camera} recording @ ${h.hour}:00.`,
           sources: [
             {
-              src: `${apiHost}/vod/${year}-${month}/${day}/${h.hour}/${camera}/${timezone.replaceAll(
+              src: `${apiHost}vod/${year}-${month}/${day}/${h.hour}/${camera}/${timezone.replaceAll(
                 '/',
                 '_'
               )}/master.m3u8`,
@@ -135,6 +135,9 @@ export default function Recording({ camera, date, hour = '00', minute = '00', se
       <div className="text-xs">Dates and times are based on the browser's timezone {timezone}</div>
 
       <VideoPlayer
+        options={{
+          preload: 'auto',
+        }}
         onReady={(player) => {
           player.on('ratechange', () => player.defaultPlaybackRate(player.playbackRate()));
           if (player.playlist) {