Mirrors/blakeblackshear.frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2024-11-21 19:07:46 +01:00
Code Issues Projects Releases Wiki Activity

Optimize nginx & recordings (#4688)

Browse Source 
* Add segment duration metadata

* Use faststart only for kept segments

* Add more options for performance

* Build nginx locally

* Build nginx in dockerfile and enable threaded vod handling

* Use DASH instead of hls

* Allow player to continue on error

* Undo DASH change

* Fix typo

* Correct log

* Fix bad comments

* Fix indentation

* Preload stream

* remove unused

* Fix spacing

* Fix tabs / sspaces

* Retab

* More cleanup
This commit is contained in:
Nicolas Mowen 2022-12-17 16:53:34 -07:00 committed by GitHub
parent 9b99ba81e5
commit 369299315f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 129 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
Dockerfile
View File

@ -9,9 +9,6 @@ FROM --platform=linux/amd64 debian:11 AS base_amd64
FROM debian:11-slim AS slim-base
FROM blakeblackshear/frigate-nginx:1.0.2 AS nginx
FROM slim-base AS wget
ARG DEBIAN_FRONTEND
RUN apt-get update \
@ -19,6 +16,53 @@ RUN apt-get update \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /rootfs
FROM ubuntu:20.04 AS nginx
ARG DEBIAN_FRONTEND
ARG NGINX_VERSION=1.22.1
ARG VOD_MODULE_VERSION=1.30
ARG SECURE_TOKEN_MODULE_VERSION=1.4
ARG RTMP_MODULE_VERSION=1.2.1
RUN cp /etc/apt/sources.list /etc/apt/sources.list~ \
&& sed -Ei 's/^# deb-src /deb-src /' /etc/apt/sources.list \
&& apt-get update
RUN apt-get -yqq build-dep nginx
RUN apt-get -yqq install --no-install-recommends ca-certificates wget \
&& mkdir /tmp/nginx \
&& wget https://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz \
&& tar -zxf nginx-${NGINX_VERSION}.tar.gz -C /tmp/nginx --strip-components=1 \
&& rm nginx-${NGINX_VERSION}.tar.gz \
&& mkdir /tmp/nginx-vod-module \
&& wget https://github.com/kaltura/nginx-vod-module/archive/refs/tags/${VOD_MODULE_VERSION}.tar.gz \
&& tar -zxf ${VOD_MODULE_VERSION}.tar.gz -C /tmp/nginx-vod-module --strip-components=1 \
&& rm ${VOD_MODULE_VERSION}.tar.gz \
# Patch MAX_CLIPS to allow more clips to be added than the default 128
&& sed -i 's/MAX_CLIPS (128)/MAX_CLIPS (1080)/g' /tmp/nginx-vod-module/vod/media_set.h \
&& mkdir /tmp/nginx-secure-token-module \
&& wget https://github.com/kaltura/nginx-secure-token-module/archive/refs/tags/${SECURE_TOKEN_MODULE_VERSION}.tar.gz \
&& tar -zxf ${SECURE_TOKEN_MODULE_VERSION}.tar.gz -C /tmp/nginx-secure-token-module --strip-components=1 \
&& rm ${SECURE_TOKEN_MODULE_VERSION}.tar.gz \
&& mkdir /tmp/nginx-rtmp-module \
&& wget https://github.com/arut/nginx-rtmp-module/archive/refs/tags/v${RTMP_MODULE_VERSION}.tar.gz \
&& tar -zxf v${RTMP_MODULE_VERSION}.tar.gz -C /tmp/nginx-rtmp-module --strip-components=1 \
&& rm v${RTMP_MODULE_VERSION}.tar.gz
WORKDIR /tmp/nginx
RUN ./configure --prefix=/usr/local/nginx \
--with-file-aio \
--with-http_sub_module \
--with-http_ssl_module \
--with-threads \
--add-module=../nginx-vod-module \
--add-module=../nginx-secure-token-module \
--add-module=../nginx-rtmp-module \
--with-cc-opt="-O3 -Wno-error=implicit-fallthrough"
RUN make && make install
RUN rm -rf /usr/local/nginx/html /usr/local/nginx/conf/*.default
FROM wget AS go2rtc
ARG TARGETARCH

64
docker/rootfs/usr/local/nginx/conf/nginx.conf
View File

@ -1,27 +1,30 @@
daemon off;
user root;
worker_processes 1;
worker_processes auto;
error_log /dev/stdout warn;
pid /var/run/nginx.pid;
error_log /dev/stdout warn;
pid /var/run/nginx.pid;
events {
worker_connections 1024;
worker_connections 1024;
}
http {
include mime.types;
default_type application/octet-stream;
include mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /dev/stdout main;
access_log /dev/stdout main;
sendfile on;
# send headers in one piece, it is better than sending them one by one
tcp_nopush on;
keepalive_timeout 65;
sendfile on;
keepalive_timeout 65;
gzip on;
gzip_comp_level 6;
@ -30,18 +33,18 @@ http {
gzip_vary on;
upstream frigate_api {
server 127.0.0.1:5001;
keepalive 1024;
server 127.0.0.1:5001;
keepalive 1024;
}
upstream mqtt_ws {
server 127.0.0.1:5002;
keepalive 1024;
server 127.0.0.1:5002;
keepalive 1024;
}
upstream jsmpeg {
server 127.0.0.1:8082;
keepalive 1024;
server 127.0.0.1:8082;
keepalive 1024;
}
upstream go2rtc {
@ -61,6 +64,19 @@ http {
vod_align_segments_to_key_frames on;
vod_manifest_segment_durations_mode accurate;
vod_ignore_edit_list on;
vod_segment_duration 10000;
vod_hls_mpegts_align_frames off;
vod_hls_mpegts_interleave_frames on;
# file handle caching / aio
open_file_cache max=1000 inactive=5m;
open_file_cache_valid 2m;
open_file_cache_min_uses 1;
open_file_cache_errors on;
aio on;
# https://github.com/kaltura/nginx-vod-module#vod_open_file_thread_pool
vod_open_file_thread_pool default;
# vod caches
vod_metadata_cache metadata_cache 512m;
@ -70,18 +86,12 @@ http {
gzip on;
gzip_types application/vnd.apple.mpegurl;
# file handle caching / aio
open_file_cache max=1000 inactive=5m;
open_file_cache_valid 2m;
open_file_cache_min_uses 1;
open_file_cache_errors on;
aio on;
location /vod/ {
aio threads;
vod hls;
secure_token $args;
secure_token_types application/vnd.apple.mpegurl;
secure_token_types application/vnd.apple.mpegurl;
add_header Access-Control-Allow-Headers '*';
add_header Access-Control-Expose-Headers 'Server,range,Content-Length,Content-Range';
@ -137,8 +147,8 @@ http {
}
location /cache/ {
internal; # This tells nginx it's not accessible from the outside
alias /tmp/cache/;
internal; # This tells nginx it's not accessible from the outside
alias /tmp/cache/;
}
location /recordings/ {
@ -257,4 +267,4 @@ rtmp {
meta copy;
}
}
}
}

1
frigate/const.py
View File

@ -5,6 +5,7 @@ CACHE_DIR = "/tmp/cache"
YAML_EXT = (".yaml", ".yml")
PLUS_ENV_VAR = "PLUS_API_KEY"
PLUS_API_HOST = "https://api.frigate.video"
MAX_SEGMENT_DURATION = 600
# Regex Consts

7
frigate/http.py
View File

@ -32,7 +32,7 @@ from peewee import SqliteDatabase, operator, fn, DoesNotExist
from playhouse.shortcuts import model_to_dict
from frigate.config import FrigateConfig
from frigate.const import CLIPS_DIR, RECORD_DIR
from frigate.const import CLIPS_DIR, MAX_SEGMENT_DURATION, RECORD_DIR
from frigate.models import Event, Recordings
from frigate.object_processing import TrackedObject
from frigate.stats import stats_snapshot
@ -1050,6 +1050,7 @@ def vod_ts(camera_name, start_ts, end_ts):
clips = []
durations = []
max_duration_ms = MAX_SEGMENT_DURATION * 1000
recording: Recordings
for recording in recordings:
@ -1060,7 +1061,7 @@ def vod_ts(camera_name, start_ts, end_ts):
if recording.end_time > end_ts:
duration -= int((recording.end_time - end_ts) * 1000)
if duration > 0:
if 0 < duration < max_duration_ms:
clip["keyFrameDurations"] = [duration]
clips.append(clip)
durations.append(duration)
@ -1076,7 +1077,9 @@ def vod_ts(camera_name, start_ts, end_ts):
{
"cache": hour_ago.timestamp() > start_ts,
"discontinuity": False,
"consistentSequenceMediaInfo": True,
"durations": durations,
"segment_duration": max(durations),
"sequences": [{"clips": clips}],
}
)

38
frigate/record.py
View File

@ -5,7 +5,6 @@ import multiprocessing as mp
import os
import queue
import random
import shutil
import string
import subprocess as sp
import threading
@ -16,7 +15,7 @@ import psutil
from peewee import JOIN, DoesNotExist
from frigate.config import RetainModeEnum, FrigateConfig
from frigate.const import CACHE_DIR, RECORD_DIR
from frigate.const import CACHE_DIR, MAX_SEGMENT_DURATION, RECORD_DIR
from frigate.models import Event, Recordings
from frigate.util import area
@ -173,7 +172,7 @@ class RecordingMaintainer(threading.Thread):
duration = -1
# ensure duration is within expected length
if 0 < duration < 600:
if 0 < duration < MAX_SEGMENT_DURATION:
end_time = start_time + datetime.timedelta(seconds=duration)
self.end_time_cache[cache_path] = (end_time, duration)
else:
@ -296,13 +295,38 @@ class RecordingMaintainer(threading.Thread):
try:
if not os.path.exists(file_path):
start_frame = datetime.datetime.now().timestamp()
# copy then delete is required when recordings are stored on some network drives
shutil.copyfile(cache_path, file_path)
logger.debug(
f"Copied {file_path} in {datetime.datetime.now().timestamp()-start_frame} seconds."
# add faststart to kept segments to improve metadata reading
ffmpeg_cmd = [
"ffmpeg",
"-y",
"-i",
cache_path,
"-c",
"copy",
"-movflags",
"+faststart",
file_path,
]
p = sp.run(
ffmpeg_cmd,
encoding="ascii",
capture_output=True,
)
if p.returncode != 0:
logger.error(f"Unable to convert {cache_path} to {file_path}")
logger.error(p.stderr)
return
else:
logger.debug(
f"Copied {file_path} in {datetime.datetime.now().timestamp()-start_frame} seconds."
)
try:
# get the segment size of the cache file
# file without faststart is same size
segment_size = round(
float(os.path.getsize(cache_path)) / 1000000, 1
)

3
web/src/components/VideoPlayer.jsx
View File

@ -33,6 +33,9 @@ export default function VideoPlayer({ children, options, seekOptions = {}, onRea
...seekOptions,
});
// Allows player to continue on error
player.reloadSourceOnError();
// Disable fullscreen on iOS if we have children
if (
children &&

2
web/src/routes/Events.jsx
View File

@ -561,7 +561,7 @@ export default function Events({ path, ...props }) {
autoplay: true,
sources: [
{
src: `${apiHost}/vod/event/${event.id}/master.m3u8`,
src: `${apiHost}vod/event/${event.id}/master.m3u8`,
type: 'application/vnd.apple.mpegurl',
},
],

5
web/src/routes/Recording.jsx
View File

@ -69,7 +69,7 @@ export default function Recording({ camera, date, hour = '00', minute = '00', se
description: `${camera} recording @ ${h.hour}:00.`,
sources: [
{
src: `${apiHost}/vod/${year}-${month}/${day}/${h.hour}/${camera}/${timezone.replaceAll(
src: `${apiHost}vod/${year}-${month}/${day}/${h.hour}/${camera}/${timezone.replaceAll(
'/',
'_'
)}/master.m3u8`,
@ -135,6 +135,9 @@ export default function Recording({ camera, date, hour = '00', minute = '00', se
<div className="text-xs">Dates and times are based on the browser's timezone {timezone}</div>
<VideoPlayer
options={{
preload: 'auto',
}}
onReady={(player) => {
player.on('ratechange', () => player.defaultPlaybackRate(player.playbackRate()));
if (player.playlist) {