From a8c567d877e9874312cf1cb525da6cfa717b0a04 Mon Sep 17 00:00:00 2001
From: Felipe Santos <felipecassiors@gmail.com>
Date: Sun, 19 Feb 2023 16:11:12 -0300
Subject: [PATCH] Add healthcheck for go2rtc service (#5545)

* Upgrade s6-overlay from 3.1.3.0 to 3.1.4.0

* Add go2rtc healthcheck service

Also don't make go2rtc exits cause the container to fail.

* Reword healthcheck message

Co-authored-by: Nicolas Mowen <nickmowen213@gmail.com>

* Add timeout to go2rtc healthcheck

* Update healthcheck message

Co-authored-by: Nicolas Mowen <nickmowen213@gmail.com>

* Give additional time for go2rtc start/restart

* Fix typo

* Avoid creating go2rtc config multiple times

* Fix healthcheck not starting

* Fix sleep

* Fix more hidden logs

* Decrease time window and use curl's timeout flag

---------

Co-authored-by: Nicolas Mowen <nickmowen213@gmail.com>
---
 Dockerfile                                    |  4 +++
 docker/install_s6_overlay.sh                  |  2 +-
 .../etc/s6-overlay/s6-rc.d/frigate/finish     | 12 ++++----
 .../rootfs/etc/s6-overlay/s6-rc.d/frigate/run |  6 ++--
 .../go2rtc-healthcheck/dependencies.d/go2rtc  |  0
 .../s6-rc.d/go2rtc-healthcheck/finish         | 12 ++++++++
 .../s6-rc.d/go2rtc-healthcheck/producer-for   |  1 +
 .../s6-overlay/s6-rc.d/go2rtc-healthcheck/run | 22 ++++++++++++++
 .../s6-rc.d/go2rtc-healthcheck/timeout-kill   |  1 +
 .../s6-rc.d/go2rtc-healthcheck/type           |  1 +
 .../s6-rc.d/go2rtc-log/consumer-for           |  1 +
 .../etc/s6-overlay/s6-rc.d/go2rtc/finish      | 26 ++---------------
 .../rootfs/etc/s6-overlay/s6-rc.d/go2rtc/run  | 29 ++++++++++---------
 .../etc/s6-overlay/s6-rc.d/nginx/finish       |  8 +++--
 .../rootfs/etc/s6-overlay/s6-rc.d/nginx/run   |  4 ++-
 docker/rootfs/usr/local/bin/frigate           |  5 ----
 .../rootfs/usr/local/go2rtc/create_config.py  |  7 +++--
 17 files changed, 82 insertions(+), 59 deletions(-)
 create mode 100644 docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/dependencies.d/go2rtc
 create mode 100755 docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/finish
 create mode 100644 docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/producer-for
 create mode 100755 docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/run
 create mode 100644 docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/timeout-kill
 create mode 100644 docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/type
 delete mode 100644 docker/rootfs/usr/local/bin/frigate

diff --git a/Dockerfile b/Dockerfile
index 448d5ebf8..204fc7c81 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -207,6 +207,10 @@ FROM deps AS devcontainer
 # But start a fake service for simulating the logs
 COPY docker/fake_frigate_run /etc/s6-overlay/s6-rc.d/frigate/run
 
+# Create symbolic link to the frigate source code, as go2rtc's create_config.sh uses it
+RUN mkdir -p /opt/frigate \
+    && ln -svf /workspace/frigate/frigate /opt/frigate/frigate
+
 # Install Node 16
 RUN apt-get update \
     && apt-get install wget -y \
diff --git a/docker/install_s6_overlay.sh b/docker/install_s6_overlay.sh
index 5bee1c65f..6018c20f5 100755
--- a/docker/install_s6_overlay.sh
+++ b/docker/install_s6_overlay.sh
@@ -2,7 +2,7 @@
 
 set -euxo pipefail
 
-s6_version="3.1.3.0"
+s6_version="3.1.4.0"
 
 if [[ "${TARGETARCH}" == "amd64" ]]; then
     s6_arch="x86_64"
diff --git a/docker/rootfs/etc/s6-overlay/s6-rc.d/frigate/finish b/docker/rootfs/etc/s6-overlay/s6-rc.d/frigate/finish
index d83c1a6e6..75869b5e7 100755
--- a/docker/rootfs/etc/s6-overlay/s6-rc.d/frigate/finish
+++ b/docker/rootfs/etc/s6-overlay/s6-rc.d/frigate/finish
@@ -4,6 +4,8 @@
 
 set -o errexit -o nounset -o pipefail
 
+# Logs should be sent to stdout so that s6 can collect them
+
 declare exit_code_container
 exit_code_container=$(cat /run/s6-linux-init-container-results/exitcode)
 readonly exit_code_container
@@ -11,20 +13,16 @@ readonly exit_code_service="${1}"
 readonly exit_code_signal="${2}"
 readonly service="Frigate"
 
-echo "Service ${service} exited with code ${exit_code_service} (by signal ${exit_code_signal})" >&2
+echo "[INFO] Service ${service} exited with code ${exit_code_service} (by signal ${exit_code_signal})"
 
 if [[ "${exit_code_service}" -eq 256 ]]; then
   if [[ "${exit_code_container}" -eq 0 ]]; then
-    echo $((128 + exit_code_signal)) > /run/s6-linux-init-container-results/exitcode
+    echo $((128 + exit_code_signal)) >/run/s6-linux-init-container-results/exitcode
   fi
 elif [[ "${exit_code_service}" -ne 0 ]]; then
   if [[ "${exit_code_container}" -eq 0 ]]; then
-    echo "${exit_code_service}" > /run/s6-linux-init-container-results/exitcode
+    echo "${exit_code_service}" >/run/s6-linux-init-container-results/exitcode
   fi
-else
-  # Exit code 0 is expected when Frigate is restarted by the user. In this case,
-  # we create a signal for the go2rtc finish script to tolerate the restart.
-  touch /dev/shm/restarting-frigate
 fi
 
 exec /run/s6/basedir/bin/halt
diff --git a/docker/rootfs/etc/s6-overlay/s6-rc.d/frigate/run b/docker/rootfs/etc/s6-overlay/s6-rc.d/frigate/run
index 1adf1a0cb..562081fc5 100755
--- a/docker/rootfs/etc/s6-overlay/s6-rc.d/frigate/run
+++ b/docker/rootfs/etc/s6-overlay/s6-rc.d/frigate/run
@@ -4,12 +4,14 @@
 
 set -o errexit -o nounset -o pipefail
 
+# Logs should be sent to stdout so that s6 can collect them
+
 # Tell S6-Overlay not to restart this service
 s6-svc -O .
 
-echo "[INFO] Starting Frigate..." >&2
+echo "[INFO] Starting Frigate..."
 
-cd /opt/frigate || echo "[ERROR] Failed to change working directory to /opt/frigate" >&2
+cd /opt/frigate || echo "[ERROR] Failed to change working directory to /opt/frigate"
 
 # Replace the bash process with the Frigate process, redirecting stderr to stdout
 exec 2>&1
diff --git a/docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/dependencies.d/go2rtc b/docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/dependencies.d/go2rtc
new file mode 100644
index 000000000..e69de29bb
diff --git a/docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/finish b/docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/finish
new file mode 100755
index 000000000..f83421640
--- /dev/null
+++ b/docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/finish
@@ -0,0 +1,12 @@
+#!/command/with-contenv bash
+# shellcheck shell=bash
+
+set -o errexit -o nounset -o pipefail
+
+# Logs should be sent to stdout so that s6 can collect them
+
+readonly exit_code_service="${1}"
+readonly exit_code_signal="${2}"
+readonly service="go2rtc-healthcheck"
+
+echo "[INFO] The ${service} service exited with code ${exit_code_service} (by signal ${exit_code_signal})"
diff --git a/docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/producer-for b/docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/producer-for
new file mode 100644
index 000000000..20fbc45f8
--- /dev/null
+++ b/docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/producer-for
@@ -0,0 +1 @@
+go2rtc-log
diff --git a/docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/run b/docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/run
new file mode 100755
index 000000000..3a6e423e5
--- /dev/null
+++ b/docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/run
@@ -0,0 +1,22 @@
+#!/command/with-contenv bash
+# shellcheck shell=bash
+# Start the go2rtc-healthcheck service
+
+set -o errexit -o nounset -o pipefail
+
+# Logs should be sent to stdout so that s6 can collect them
+
+# Give some additional time for go2rtc to start before start pinging
+sleep 10s
+echo "[INFO] Starting go2rtc healthcheck service..."
+
+while sleep 30s; do
+    # Check if the service is running
+    if ! curl --connect-timeout 10 --fail --silent --show-error --output /dev/null http://127.0.0.1:1984/api/streams 2>&1; then
+        echo "[ERROR] The go2rtc service is not responding to ping, restarting..."
+        # We can also use -r instead of -t to send kill signal rather than term
+        s6-svc -t /var/run/service/go2rtc 2>&1
+        # Give some additional time to go2rtc to restart before start pinging again
+        sleep 10s
+    fi
+done
diff --git a/docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/timeout-kill b/docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/timeout-kill
new file mode 100644
index 000000000..e9c02dad1
--- /dev/null
+++ b/docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/timeout-kill
@@ -0,0 +1 @@
+5000
diff --git a/docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/type b/docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/type
new file mode 100644
index 000000000..5883cff0c
--- /dev/null
+++ b/docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/type
@@ -0,0 +1 @@
+longrun
diff --git a/docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-log/consumer-for b/docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-log/consumer-for
index 02b16a879..bdd482e42 100644
--- a/docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-log/consumer-for
+++ b/docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-log/consumer-for
@@ -1 +1,2 @@
 go2rtc
+go2rtc-healthcheck
diff --git a/docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc/finish b/docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc/finish
index 26b34294a..e95ba75c8 100755
--- a/docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc/finish
+++ b/docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc/finish
@@ -1,32 +1,12 @@
 #!/command/with-contenv bash
 # shellcheck shell=bash
-# Take down the S6 supervision tree when the service exits
 
 set -o errexit -o nounset -o pipefail
 
-declare exit_code_container
-exit_code_container=$(cat /run/s6-linux-init-container-results/exitcode)
-readonly exit_code_container
+# Logs should be sent to stdout so that s6 can collect them
+
 readonly exit_code_service="${1}"
 readonly exit_code_signal="${2}"
 readonly service="go2rtc"
 
-echo "Service ${service} exited with code ${exit_code_service} (by signal ${exit_code_signal})" >&2
-
-if [[ "${exit_code_service}" -eq 256 ]]; then
-  if [[ "${exit_code_container}" -eq 0 ]]; then
-    echo $((128 + exit_code_signal)) > /run/s6-linux-init-container-results/exitcode
-  fi
-elif [[ "${exit_code_service}" -ne 0 ]]; then
-  if [[ "${exit_code_container}" -eq 0 ]]; then
-    echo "${exit_code_service}" > /run/s6-linux-init-container-results/exitcode
-  fi
-else
-  # go2rtc is not supposed to exit, so even when it exits with 0 we make the
-  # container with 1. We only tolerate it when Frigate is restarting.
-  if [[ "${exit_code_container}" -eq 0 && ! -f /dev/shm/restarting-frigate ]]; then
-    echo "1" > /run/s6-linux-init-container-results/exitcode
-  fi
-fi
-
-exec /run/s6/basedir/bin/halt
+echo "[INFO] The ${service} service exited with code ${exit_code_service} (by signal ${exit_code_signal})"
diff --git a/docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc/run b/docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc/run
index e84e645f2..1f46194a3 100755
--- a/docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc/run
+++ b/docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc/run
@@ -4,8 +4,7 @@
 
 set -o errexit -o nounset -o pipefail
 
-# Tell S6-Overlay not to restart this service
-s6-svc -O .
+# Logs should be sent to stdout so that s6 can collect them
 
 function get_ip_and_port_from_supervisor() {
     local ip_address
@@ -19,9 +18,9 @@ function get_ip_and_port_from_supervisor() {
             jq --exit-status --raw-output '.data.ipv4.address[0]'
     ) && [[ "${ip_address}" =~ ${ip_regex} ]]; then
         ip_address="${BASH_REMATCH[1]}"
-        echo "[INFO] Got IP address from supervisor: ${ip_address}" >&2
+        echo "[INFO] Got IP address from supervisor: ${ip_address}"
     else
-        echo "[WARN] Failed to get IP address from supervisor" >&2
+        echo "[WARN] Failed to get IP address from supervisor"
         return 0
     fi
 
@@ -35,26 +34,28 @@ function get_ip_and_port_from_supervisor() {
             jq --exit-status --raw-output '.data.network["8555/tcp"]'
     ) && [[ "${webrtc_port}" =~ ${port_regex} ]]; then
         webrtc_port="${BASH_REMATCH[1]}"
-        echo "[INFO] Got WebRTC port from supervisor: ${webrtc_port}" >&2
+        echo "[INFO] Got WebRTC port from supervisor: ${webrtc_port}"
     else
-        echo "[WARN] Failed to get WebRTC port from supervisor" >&2
+        echo "[WARN] Failed to get WebRTC port from supervisor"
         return 0
     fi
 
     export FRIGATE_GO2RTC_WEBRTC_CANDIDATE_INTERNAL="${ip_address}:${webrtc_port}"
 }
 
-echo "[INFO] Preparing go2rtc config..." >&2
+if [[ ! -f "/dev/shm/go2rtc.yaml" ]]; then
+    echo "[INFO] Preparing go2rtc config..."
 
-if [[ -n "${SUPERVISOR_TOKEN:-}" ]]; then
-    # Running as a Home Assistant add-on, infer the IP address and port
-    get_ip_and_port_from_supervisor
+    if [[ -n "${SUPERVISOR_TOKEN:-}" ]]; then
+        # Running as a Home Assistant add-on, infer the IP address and port
+        get_ip_and_port_from_supervisor
+    fi
+
+    python3 /usr/local/go2rtc/create_config.py
 fi
 
-raw_config=$(python3 /usr/local/go2rtc/create_config.py)
-
-echo "[INFO] Starting go2rtc..." >&2
+echo "[INFO] Starting go2rtc..."
 
 # Replace the bash process with the go2rtc process, redirecting stderr to stdout
 exec 2>&1
-exec go2rtc -config="${raw_config}"
+exec go2rtc -config=/dev/shm/go2rtc.yaml
diff --git a/docker/rootfs/etc/s6-overlay/s6-rc.d/nginx/finish b/docker/rootfs/etc/s6-overlay/s6-rc.d/nginx/finish
index 01c37d192..d147d74bd 100755
--- a/docker/rootfs/etc/s6-overlay/s6-rc.d/nginx/finish
+++ b/docker/rootfs/etc/s6-overlay/s6-rc.d/nginx/finish
@@ -4,6 +4,8 @@
 
 set -o errexit -o nounset -o pipefail
 
+# Logs should be sent to stdout so that s6 can collect them
+
 declare exit_code_container
 exit_code_container=$(cat /run/s6-linux-init-container-results/exitcode)
 readonly exit_code_container
@@ -11,18 +13,18 @@ readonly exit_code_service="${1}"
 readonly exit_code_signal="${2}"
 readonly service="NGINX"
 
-echo "Service ${service} exited with code ${exit_code_service} (by signal ${exit_code_signal})" >&2
+echo "[INFO] Service ${service} exited with code ${exit_code_service} (by signal ${exit_code_signal})"
 
 if [[ "${exit_code_service}" -eq 256 ]]; then
   if [[ "${exit_code_container}" -eq 0 ]]; then
-    echo $((128 + exit_code_signal)) > /run/s6-linux-init-container-results/exitcode
+    echo $((128 + exit_code_signal)) >/run/s6-linux-init-container-results/exitcode
   fi
   if [[ "${exit_code_signal}" -eq 15 ]]; then
     exec /run/s6/basedir/bin/halt
   fi
 elif [[ "${exit_code_service}" -ne 0 ]]; then
   if [[ "${exit_code_container}" -eq 0 ]]; then
-    echo "${exit_code_service}" > /run/s6-linux-init-container-results/exitcode
+    echo "${exit_code_service}" >/run/s6-linux-init-container-results/exitcode
   fi
   exec /run/s6/basedir/bin/halt
 fi
diff --git a/docker/rootfs/etc/s6-overlay/s6-rc.d/nginx/run b/docker/rootfs/etc/s6-overlay/s6-rc.d/nginx/run
index f7a5a6d0a..2754c0d09 100755
--- a/docker/rootfs/etc/s6-overlay/s6-rc.d/nginx/run
+++ b/docker/rootfs/etc/s6-overlay/s6-rc.d/nginx/run
@@ -4,7 +4,9 @@
 
 set -o errexit -o nounset -o pipefail
 
-echo "[INFO] Starting NGINX..." >&2
+# Logs should be sent to stdout so that s6 can collect them
+
+echo "[INFO] Starting NGINX..."
 
 # Replace the bash process with the NGINX process, redirecting stderr to stdout
 exec 2>&1
diff --git a/docker/rootfs/usr/local/bin/frigate b/docker/rootfs/usr/local/bin/frigate
deleted file mode 100644
index 7e431b11c..000000000
--- a/docker/rootfs/usr/local/bin/frigate
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/command/with-contenv bash
-# shellcheck shell=bash
-
-exec 2>&1
-exec python3 -u -m frigate "${@}"
diff --git a/docker/rootfs/usr/local/go2rtc/create_config.py b/docker/rootfs/usr/local/go2rtc/create_config.py
index 28f884fe6..d034e044b 100644
--- a/docker/rootfs/usr/local/go2rtc/create_config.py
+++ b/docker/rootfs/usr/local/go2rtc/create_config.py
@@ -8,6 +8,7 @@ import yaml
 sys.path.insert(0, "/opt/frigate")
 from frigate.const import BIRDSEYE_PIPE, BTBN_PATH
 from frigate.ffmpeg_presets import parse_preset_hardware_acceleration_encode
+
 sys.path.remove("/opt/frigate")
 
 
@@ -50,7 +51,6 @@ if not go2rtc_config.get("webrtc", {}).get("candidates", []):
 else:
     print(
         "[INFO] Not injecting WebRTC candidates into go2rtc config as it has been set manually",
-        file=sys.stderr,
     )
 
 # sets default RTSP response to be equivalent to ?video=h264,h265&audio=aac
@@ -95,5 +95,6 @@ if config.get("birdseye", {}).get("restream", False):
     else:
         go2rtc_config["streams"] = {"birdseye": ffmpeg_cmd}
 
-
-print(json.dumps(go2rtc_config))
+# Write go2rtc_config to /dev/shm/go2rtc.yaml
+with open("/dev/shm/go2rtc.yaml", "w") as f:
+    yaml.dump(go2rtc_config, f)