Add healthcheck for go2rtc service (#5545)

* Upgrade s6-overlay from 3.1.3.0 to 3.1.4.0

* Add go2rtc healthcheck service

Also don't make go2rtc exits cause the container to fail.

* Reword healthcheck message

Co-authored-by: Nicolas Mowen <nickmowen213@gmail.com>

* Add timeout to go2rtc healthcheck

* Update healthcheck message

Co-authored-by: Nicolas Mowen <nickmowen213@gmail.com>

* Give additional time for go2rtc start/restart

* Fix typo

* Avoid creating go2rtc config multiple times

* Fix healthcheck not starting

* Fix sleep

* Fix more hidden logs

* Decrease time window and use curl's timeout flag

---------

Co-authored-by: Nicolas Mowen <nickmowen213@gmail.com>

Dockerfile

@@ -207,6 +207,10 @@ FROM deps AS devcontainer
 # But start a fake service for simulating the logs
 COPY docker/fake_frigate_run /etc/s6-overlay/s6-rc.d/frigate/run
 
+# Create symbolic link to the frigate source code, as go2rtc's create_config.sh uses it
+RUN mkdir -p /opt/frigate \
+    && ln -svf /workspace/frigate/frigate /opt/frigate/frigate
+
 # Install Node 16
 RUN apt-get update \
     && apt-get install wget -y \

docker/install_s6_overlay.sh

@@ -2,7 +2,7 @@
 
 set -euxo pipefail
 
-s6_version="3.1.3.0"
+s6_version="3.1.4.0"
 
 if [[ "${TARGETARCH}" == "amd64" ]]; then
     s6_arch="x86_64"

docker/rootfs/etc/s6-overlay/s6-rc.d/frigate/finish

@@ -4,6 +4,8 @@
 
 set -o errexit -o nounset -o pipefail
 
+# Logs should be sent to stdout so that s6 can collect them
+
 declare exit_code_container
 exit_code_container=$(cat /run/s6-linux-init-container-results/exitcode)
 readonly exit_code_container
@@ -11,20 +13,16 @@ readonly exit_code_service="${1}"
 readonly exit_code_signal="${2}"
 readonly service="Frigate"
 
-echo "Service ${service} exited with code ${exit_code_service} (by signal ${exit_code_signal})" >&2
+echo "[INFO] Service ${service} exited with code ${exit_code_service} (by signal ${exit_code_signal})"
 
 if [[ "${exit_code_service}" -eq 256 ]]; then
   if [[ "${exit_code_container}" -eq 0 ]]; then
-    echo $((128 + exit_code_signal)) > /run/s6-linux-init-container-results/exitcode
+    echo $((128 + exit_code_signal)) >/run/s6-linux-init-container-results/exitcode
   fi
 elif [[ "${exit_code_service}" -ne 0 ]]; then
   if [[ "${exit_code_container}" -eq 0 ]]; then
-    echo "${exit_code_service}" > /run/s6-linux-init-container-results/exitcode
+    echo "${exit_code_service}" >/run/s6-linux-init-container-results/exitcode
   fi
-else
-  # Exit code 0 is expected when Frigate is restarted by the user. In this case,
-  # we create a signal for the go2rtc finish script to tolerate the restart.
-  touch /dev/shm/restarting-frigate
 fi
 
 exec /run/s6/basedir/bin/halt

docker/rootfs/etc/s6-overlay/s6-rc.d/frigate/run

@@ -4,12 +4,14 @@
 
 set -o errexit -o nounset -o pipefail
 
+# Logs should be sent to stdout so that s6 can collect them
+
 # Tell S6-Overlay not to restart this service
 s6-svc -O .
 
-echo "[INFO] Starting Frigate..." >&2
+echo "[INFO] Starting Frigate..."
 
-cd /opt/frigate || echo "[ERROR] Failed to change working directory to /opt/frigate" >&2
+cd /opt/frigate || echo "[ERROR] Failed to change working directory to /opt/frigate"
 
 # Replace the bash process with the Frigate process, redirecting stderr to stdout
 exec 2>&1

docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/finish

@@ -0,0 +1,12 @@
+#!/command/with-contenv bash
+# shellcheck shell=bash
+
+set -o errexit -o nounset -o pipefail
+
+# Logs should be sent to stdout so that s6 can collect them
+
+readonly exit_code_service="${1}"
+readonly exit_code_signal="${2}"
+readonly service="go2rtc-healthcheck"
+
+echo "[INFO] The ${service} service exited with code ${exit_code_service} (by signal ${exit_code_signal})"

docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/producer-for

@@ -0,0 +1 @@
+go2rtc-log

docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/run

@@ -0,0 +1,22 @@
+#!/command/with-contenv bash
+# shellcheck shell=bash
+# Start the go2rtc-healthcheck service
+
+set -o errexit -o nounset -o pipefail
+
+# Logs should be sent to stdout so that s6 can collect them
+
+# Give some additional time for go2rtc to start before start pinging
+sleep 10s
+echo "[INFO] Starting go2rtc healthcheck service..."
+
+while sleep 30s; do
+    # Check if the service is running
+    if ! curl --connect-timeout 10 --fail --silent --show-error --output /dev/null http://127.0.0.1:1984/api/streams 2>&1; then
+        echo "[ERROR] The go2rtc service is not responding to ping, restarting..."
+        # We can also use -r instead of -t to send kill signal rather than term
+        s6-svc -t /var/run/service/go2rtc 2>&1
+        # Give some additional time to go2rtc to restart before start pinging again
+        sleep 10s
+    fi
+done

docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/timeout-kill

@@ -0,0 +1 @@
+5000

docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/type

@@ -0,0 +1 @@
+longrun

docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-log/consumer-for

@@ -1 +1,2 @@
 go2rtc
+go2rtc-healthcheck

docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc/finish

@@ -1,32 +1,12 @@
 #!/command/with-contenv bash
 # shellcheck shell=bash
-# Take down the S6 supervision tree when the service exits
 
 set -o errexit -o nounset -o pipefail
 
-declare exit_code_container
-exit_code_container=$(cat /run/s6-linux-init-container-results/exitcode)
-readonly exit_code_container
+# Logs should be sent to stdout so that s6 can collect them
+
 readonly exit_code_service="${1}"
 readonly exit_code_signal="${2}"
 readonly service="go2rtc"
 
-echo "Service ${service} exited with code ${exit_code_service} (by signal ${exit_code_signal})" >&2
-
-if [[ "${exit_code_service}" -eq 256 ]]; then
-  if [[ "${exit_code_container}" -eq 0 ]]; then
-    echo $((128 + exit_code_signal)) > /run/s6-linux-init-container-results/exitcode
-  fi
-elif [[ "${exit_code_service}" -ne 0 ]]; then
-  if [[ "${exit_code_container}" -eq 0 ]]; then
-    echo "${exit_code_service}" > /run/s6-linux-init-container-results/exitcode
-  fi
-else
-  # go2rtc is not supposed to exit, so even when it exits with 0 we make the
-  # container with 1. We only tolerate it when Frigate is restarting.
-  if [[ "${exit_code_container}" -eq 0 && ! -f /dev/shm/restarting-frigate ]]; then
-    echo "1" > /run/s6-linux-init-container-results/exitcode
-  fi
-fi
-
-exec /run/s6/basedir/bin/halt
+echo "[INFO] The ${service} service exited with code ${exit_code_service} (by signal ${exit_code_signal})"

docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc/run

@@ -4,8 +4,7 @@
 
 set -o errexit -o nounset -o pipefail
 
-# Tell S6-Overlay not to restart this service
-s6-svc -O .
+# Logs should be sent to stdout so that s6 can collect them
 
 function get_ip_and_port_from_supervisor() {
     local ip_address
@@ -19,9 +18,9 @@ function get_ip_and_port_from_supervisor() {
             jq --exit-status --raw-output '.data.ipv4.address[0]'
     ) && [[ "${ip_address}" =~ ${ip_regex} ]]; then
         ip_address="${BASH_REMATCH[1]}"
-        echo "[INFO] Got IP address from supervisor: ${ip_address}" >&2
+        echo "[INFO] Got IP address from supervisor: ${ip_address}"
     else
-        echo "[WARN] Failed to get IP address from supervisor" >&2
+        echo "[WARN] Failed to get IP address from supervisor"
         return 0
     fi
 
@@ -35,26 +34,28 @@ function get_ip_and_port_from_supervisor() {
             jq --exit-status --raw-output '.data.network["8555/tcp"]'
     ) && [[ "${webrtc_port}" =~ ${port_regex} ]]; then
         webrtc_port="${BASH_REMATCH[1]}"
-        echo "[INFO] Got WebRTC port from supervisor: ${webrtc_port}" >&2
+        echo "[INFO] Got WebRTC port from supervisor: ${webrtc_port}"
     else
-        echo "[WARN] Failed to get WebRTC port from supervisor" >&2
+        echo "[WARN] Failed to get WebRTC port from supervisor"
         return 0
     fi
 
     export FRIGATE_GO2RTC_WEBRTC_CANDIDATE_INTERNAL="${ip_address}:${webrtc_port}"
 }
 
-echo "[INFO] Preparing go2rtc config..." >&2
+if [[ ! -f "/dev/shm/go2rtc.yaml" ]]; then
+    echo "[INFO] Preparing go2rtc config..."
 
-if [[ -n "${SUPERVISOR_TOKEN:-}" ]]; then
-    # Running as a Home Assistant add-on, infer the IP address and port
-    get_ip_and_port_from_supervisor
+    if [[ -n "${SUPERVISOR_TOKEN:-}" ]]; then
+        # Running as a Home Assistant add-on, infer the IP address and port
+        get_ip_and_port_from_supervisor
+    fi
+
+    python3 /usr/local/go2rtc/create_config.py
 fi
 
-raw_config=$(python3 /usr/local/go2rtc/create_config.py)
-
-echo "[INFO] Starting go2rtc..." >&2
+echo "[INFO] Starting go2rtc..."
 
 # Replace the bash process with the go2rtc process, redirecting stderr to stdout
 exec 2>&1
-exec go2rtc -config="${raw_config}"
+exec go2rtc -config=/dev/shm/go2rtc.yaml

docker/rootfs/etc/s6-overlay/s6-rc.d/nginx/finish

@@ -4,6 +4,8 @@
 
 set -o errexit -o nounset -o pipefail
 
+# Logs should be sent to stdout so that s6 can collect them
+
 declare exit_code_container
 exit_code_container=$(cat /run/s6-linux-init-container-results/exitcode)
 readonly exit_code_container
@@ -11,18 +13,18 @@ readonly exit_code_service="${1}"
 readonly exit_code_signal="${2}"
 readonly service="NGINX"
 
-echo "Service ${service} exited with code ${exit_code_service} (by signal ${exit_code_signal})" >&2
+echo "[INFO] Service ${service} exited with code ${exit_code_service} (by signal ${exit_code_signal})"
 
 if [[ "${exit_code_service}" -eq 256 ]]; then
   if [[ "${exit_code_container}" -eq 0 ]]; then
-    echo $((128 + exit_code_signal)) > /run/s6-linux-init-container-results/exitcode
+    echo $((128 + exit_code_signal)) >/run/s6-linux-init-container-results/exitcode
   fi
   if [[ "${exit_code_signal}" -eq 15 ]]; then
     exec /run/s6/basedir/bin/halt
   fi
 elif [[ "${exit_code_service}" -ne 0 ]]; then
   if [[ "${exit_code_container}" -eq 0 ]]; then
-    echo "${exit_code_service}" > /run/s6-linux-init-container-results/exitcode
+    echo "${exit_code_service}" >/run/s6-linux-init-container-results/exitcode
   fi
   exec /run/s6/basedir/bin/halt
 fi

docker/rootfs/etc/s6-overlay/s6-rc.d/nginx/run

@@ -4,7 +4,9 @@
 
 set -o errexit -o nounset -o pipefail
 
-echo "[INFO] Starting NGINX..." >&2
+# Logs should be sent to stdout so that s6 can collect them
+
+echo "[INFO] Starting NGINX..."
 
 # Replace the bash process with the NGINX process, redirecting stderr to stdout
 exec 2>&1

docker/rootfs/usr/local/bin/frigate

@@ -1,5 +0,0 @@
-#!/command/with-contenv bash
-# shellcheck shell=bash
-
-exec 2>&1
-exec python3 -u -m frigate "${@}"

docker/rootfs/usr/local/go2rtc/create_config.py

@@ -8,6 +8,7 @@ import yaml
 sys.path.insert(0, "/opt/frigate")
 from frigate.const import BIRDSEYE_PIPE, BTBN_PATH
 from frigate.ffmpeg_presets import parse_preset_hardware_acceleration_encode
+
 sys.path.remove("/opt/frigate")
 
 
@@ -50,7 +51,6 @@ if not go2rtc_config.get("webrtc", {}).get("candidates", []):
 else:
     print(
         "[INFO] Not injecting WebRTC candidates into go2rtc config as it has been set manually",
-        file=sys.stderr,
     )
 
 # sets default RTSP response to be equivalent to ?video=h264,h265&audio=aac
@@ -95,5 +95,6 @@ if config.get("birdseye", {}).get("restream", False):
     else:
         go2rtc_config["streams"] = {"birdseye": ffmpeg_cmd}
 
-
-print(json.dumps(go2rtc_config))
+# Write go2rtc_config to /dev/shm/go2rtc.yaml
+with open("/dev/shm/go2rtc.yaml", "w") as f:
+    yaml.dump(go2rtc_config, f)