Merge pull request #331 from kerberos-io/feature/add-encrypted-metadata-field

feature/add-encrypted-metadata-field

.circleci/config.yml

@@ -1,32 +0,0 @@
-# Golang CircleCI 2.0 configuration file
-#
-# Check https://circleci.com/docs/2.0/language-go/ for more details
-version: 2
-jobs:
-  machinery:
-    docker:
-      - image: kerberos/base:0a50dc9
-    working_directory: /go/src/github.com/{{ORG_NAME}}/{{REPO_NAME}}
-    steps:
-      - checkout
-      - run: apt-get update && apt-get install -y --no-install-recommends libavcodec-dev libavformat-dev libswscale-dev libjpeg62-turbo-dev
-      - run: cd machinery && go mod download
-      - run: cd machinery && go test -v ./...
-      - run: cd machinery && go vet
-
-  ui:
-    docker:
-      - image: cimg/python:3.9.13-node
-    steps:
-      - checkout
-      - run: node --version
-      - run: cd ui && yarn
-      - run: cd ui && yarn test --passWithNoTests
-      - run: cd ui && yarn build
-
-workflows:
-  version: 2
-  build:
-    jobs:
-      - machinery
-      - ui

.devcontainer/Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/devcontainers/go:1.24-trixie
+FROM mcr.microsoft.com/devcontainers/go:1.25-trixie
 
 # Install node environment
 RUN apt-get update && \

.github/workflows/go.yml

@@ -10,24 +10,27 @@ jobs:
   build:
     name: Build
     runs-on: ubuntu-latest
-    container:
-      image: kerberos/base:eb6b088
-
-    strategy:
-      matrix:
-        #No longer supported Go versions.
-        #go-version: ['1.17', '1.18', '1.19', '1.20', '1.21']
-        go-version: ["1.24"]
 
     steps:
-      - name: Set up Go ${{ matrix.go-version }}
-        uses: actions/setup-go@v2
-        with:
-          go-version: ${{ matrix.go-version }}
       - name: Check out code into the Go module directory
         uses: actions/checkout@v4
-      - name: Set up git ownershi
-        run: git config --system --add safe.directory /__w/agent/agent
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.25.x"
+          check-latest: true
+          cache: true
+          cache-dependency-path: machinery/go.sum
+      - name: Install native dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential \
+            libavcodec-dev \
+            libavutil-dev \
+            libswresample-dev \
+            libswscale-dev \
+            pkg-config
       - name: Get dependencies
         run: cd machinery && go mod download
       - name: Build

.github/workflows/nightly-build.yml

@@ -15,7 +15,7 @@ jobs:
         architecture: [amd64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
@@ -24,13 +24,14 @@ jobs:
         with:
           ref: master
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        id: buildx
+        uses: docker/setup-buildx-action@v3
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
       - name: Run Buildx
-        run: docker buildx build --platform linux/${{matrix.architecture}} -t kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7) --push .
+        run: docker buildx build --build-arg VERSION=$(echo $GITHUB_SHA | cut -c1-7) --platform linux/${{matrix.architecture}} -t kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7) --push .
       - name: Create new and append to manifest
         run: docker buildx imagetools create -t kerberos/agent-nightly:$(echo $GITHUB_SHA | cut -c1-7) kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7)
   nightly-build-other:
@@ -40,7 +41,7 @@ jobs:
         architecture: [arm64, arm/v7, arm/v6]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
@@ -49,12 +50,13 @@ jobs:
         with:
           ref: master
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        id: buildx
+        uses: docker/setup-buildx-action@v3
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
       - name: Run Buildx
-        run: docker buildx build --platform linux/${{matrix.architecture}} -t kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7) --push .
+        run: docker buildx build --build-arg VERSION=$(echo $GITHUB_SHA | cut -c1-7) --platform linux/${{matrix.architecture}} -t kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7) --push .
       - name: Create new and append to manifest
         run: docker buildx imagetools create --append -t kerberos/agent-nightly:$(echo $GITHUB_SHA | cut -c1-7) kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7)

.github/workflows/pr-build.yml

@@ -20,19 +20,15 @@ jobs:
             runner: ubuntu-24.04-arm
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-      - uses: benjlevesque/short-sha@v2.1
-        id: short-sha
-        with:
-          length: 7
       - name: Run Build
         run: |
-          docker build -t ${{ matrix.architecture }} .
+          docker build --build-arg VERSION=$(echo "${{ github.event.pull_request.head.sha }}" | cut -c1-7) -t ${{ matrix.architecture }} .
           CID=$(docker create ${{matrix.architecture}})
           docker cp ${CID}:/home/agent ./output-${{matrix.architecture}}
           docker rm ${CID}
@@ -43,4 +39,3 @@ jobs:
         with:
           name: agent-${{matrix.architecture}}.tar
           path: agent-${{matrix.architecture}}.tar
-

.github/workflows/react.yml

@@ -11,21 +11,22 @@ on:
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     strategy:
       matrix:
-        node-version: [14.x, 16.x]
+        node-version: [22.x]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v2
+      uses: actions/setup-node@v4
       with:
         node-version: ${{ matrix.node-version }}
+        cache: yarn
+        cache-dependency-path: ui/yarn.lock
     - name: Yarn install
-      run: cd ui && yarn
+      run: cd ui && yarn --frozen-lockfile
     - name: Yarn test
       run: cd ui && yarn test --passWithNoTests
     - name: Yarn build

.github/workflows/release-bump.yml

@@ -58,16 +58,12 @@ jobs:
         architecture: [amd64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Checkout
-        uses: actions/checkout@v3
-      - uses: benjlevesque/short-sha@v2.1
-        id: short-sha
-        with:
-          length: 7
+        uses: actions/checkout@v4
       - name: Run Build
         run: |
           docker build --provenance=false --build-arg VERSION=${{ needs.bump-release.outputs.tag }} -t ${{matrix.architecture}} .
@@ -96,16 +92,12 @@ jobs:
         architecture: [arm64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Checkout
-        uses: actions/checkout@v3
-      - uses: benjlevesque/short-sha@v2.1
-        id: short-sha
-        with:
-          length: 7
+        uses: actions/checkout@v4
       - name: Run Build
         run: |
           docker build --provenance=false --build-arg VERSION=${{ needs.bump-release.outputs.tag }} -t ${{matrix.architecture}} .
@@ -129,7 +121,7 @@ jobs:
     needs: [bump-release, build-amd64, build-arm64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/release-create.yml

@@ -22,16 +22,12 @@ jobs:
         architecture: [amd64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Checkout
-        uses: actions/checkout@v3
-      - uses: benjlevesque/short-sha@v2.1
-        id: short-sha
-        with:
-          length: 7
+        uses: actions/checkout@v4
       - name: Run Build
         run: |
           docker build --provenance=false --build-arg VERSION=${{github.event.inputs.tag || github.ref_name}} -t ${{matrix.architecture}} .
@@ -59,16 +55,12 @@ jobs:
         architecture: [arm64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Checkout
-        uses: actions/checkout@v3
-      - uses: benjlevesque/short-sha@v2.1
-        id: short-sha
-        with:
-          length: 7
+        uses: actions/checkout@v4
       - name: Run Build
         run: |
           docker build --provenance=false --build-arg VERSION=${{github.event.inputs.tag || github.ref_name}} -t ${{matrix.architecture}} .
@@ -92,7 +84,7 @@ jobs:
     needs: [build-amd64, build-arm64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}

.vscode/launch.json

@@ -17,7 +17,7 @@
                 "8080"
             ],
             "envFile": "${workspaceFolder}/machinery/.env.local",
-            "buildFlags": "--tags dynamic",
+            "buildFlags": "--tags dynamic,moq",
             "env": {
                 "GOWORK": "off"
             },

Dockerfile

@@ -1,5 +1,5 @@
 
-ARG GO_IMAGE=golang:1.24-trixie
+ARG GO_IMAGE=golang:1.25-trixie
 ARG RUNTIME_IMAGE=debian:trixie-slim
 ARG VERSION=0.0.0
 FROM ${GO_IMAGE} AS build-machinery
@@ -26,29 +26,27 @@ RUN apt-get update && apt-get install -y --fix-missing --no-install-recommends \
 	rm -rf /var/lib/apt/lists/*
 
 ##############################################################################
-# Copy all the relevant source code in the Docker image, so we can build this.
+# Copy dependency metadata first so module downloads can be cached separately.
 
 RUN mkdir -p /go/src/github.com/kerberos-io/agent
-COPY machinery /go/src/github.com/kerberos-io/agent/machinery
-RUN rm -rf /go/src/github.com/kerberos-io/agent/machinery/.env
+WORKDIR /go/src/github.com/kerberos-io/agent/machinery
+COPY machinery/go.mod machinery/go.sum ./
+RUN go mod download
 
-##################################################################
-# Get the latest commit hash, so we know which version we're running
-COPY .git /go/src/github.com/kerberos-io/agent/.git
-RUN cd /go/src/github.com/kerberos-io/agent/.git && git log --format="%H" -n 1 | head -c7 > /go/src/github.com/kerberos-io/agent/machinery/version
-RUN cat /go/src/github.com/kerberos-io/agent/machinery/version
+##############################################################################
+# Copy the rest of the source after dependencies are primed.
+
+COPY machinery ./
+RUN rm -rf .env
 
 ##################
 # Build Machinery
 
-RUN cd /go/src/github.com/kerberos-io/agent/machinery && \
-	go mod download && \
-	if [ -z "${VERSION}" ] || [ "${VERSION}" = "0.0.0" ]; then \
-		VERSION=$(cd /go/src/github.com/kerberos-io/agent && git describe --tags --always 2>/dev/null || echo "0.0.0"); \
-	fi && \
+RUN RESOLVED_VERSION="${VERSION:-0.0.0}" && \
+	printf '%s' "${RESOLVED_VERSION}" > version && \
 	BUILD_TAGS=timetzdata,netgo,osusergo && \
 	case "${TARGETARCH:-$(go env GOARCH)}" in amd64|arm64) BUILD_TAGS="moq,${BUILD_TAGS}" ;; esac && \
-	go build -tags "${BUILD_TAGS}" --ldflags "-s -w -X github.com/kerberos-io/agent/machinery/src/utils.VERSION=${VERSION}" main.go && \
+	go build -o main -tags "${BUILD_TAGS}" --ldflags "-s -w -X github.com/kerberos-io/agent/machinery/src/utils.VERSION=${RESOLVED_VERSION}" . && \
 	mkdir -p /agent && \
 	mv main /agent && \
 	mv version /agent && \
@@ -80,9 +78,11 @@ RUN apk update && apk upgrade --available && sync
 # Build Web (React app)
 
 RUN mkdir -p /go/src/github.com/kerberos-io/agent/machinery/www
-COPY ui /go/src/github.com/kerberos-io/agent/ui
-RUN cd /go/src/github.com/kerberos-io/agent/ui && rm -rf yarn.lock && yarn config set network-timeout 300000 && \
-	yarn && yarn build
+WORKDIR /go/src/github.com/kerberos-io/agent/ui
+COPY ui/package.json ui/yarn.lock ./
+RUN yarn config set network-timeout 300000 && yarn --frozen-lockfile
+COPY ui ./
+RUN yarn build
 
 ####################################
 # Let's create a /dist folder containing just the files necessary for runtime.
@@ -103,7 +103,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 	ca-certificates curl ffmpeg libatomic1 libcap2-bin libstdc++6 && \
 	rm -rf /var/lib/apt/lists/* && \
 	groupadd --system kerberosio && \
-	useradd --system --gid kerberosio --groups video --create-home agent
+	useradd --system --gid kerberosio --groups video --create-home agent && \
+	chmod 0755 /home/agent
 
 #################################
 # Copy files from previous images
@@ -145,10 +146,10 @@ EXPOSE 80
 ######################################
 # Check if agent is still running
 
-HEALTHCHECK CMD curl --fail http://localhost:80 || exit 1   
+HEALTHCHECK CMD curl --fail "http://localhost:${AGENT_PORT:-80}/health" || exit 1
 
 ###################################################
 # Leeeeettttt'ssss goooooo!!!
 # Run the shizzle from the right working directory.
 WORKDIR /home/agent
-CMD ["./main", "-action", "run", "-port", "80"]
+CMD ["./main", "-action", "run"]

README-RTSPS-TLS.md

@@ -0,0 +1,645 @@
+# RTSPS and TLS certificates
+
+This guide explains how Kerberos Agent connects to an IP camera over RTSPS,
+how to issue a camera certificate with a private CA, and how to validate the
+complete trust path. It also explains why some apparently corrupted trust
+bundles can still allow a connection.
+
+The camera-specific steps were verified with a Bosch FLEXIDOME micro 3100i.
+Other Bosch firmware versions may use different labels or ports.
+
+The commands were tested with Smallstep CLI `0.30.6` and OpenSSL `3.5.6` on
+Debian. Check `step certificate sign --help` when using an older Smallstep CLI.
+The OpenSSL isolation flags `-no-CApath` and `-no-CAstore` require a version that
+lists them in `openssl s_client -help`.
+
+## Tested configuration
+
+| Setting | Value |
+| --- | --- |
+| Camera | Bosch FLEXIDOME micro 3100i |
+| Example camera address | `10.0.30.11` |
+| RTSPS port | `9554` |
+| Main stream | `rtsps://<user>:<password>@10.0.30.11:9554/?inst=1` |
+| Sub stream | `rtsps://<user>:<password>@10.0.30.11:9554/?inst=2` |
+| Certificate SAN | `IP Address:10.0.30.11` |
+| Bosch certificate usage | `HTTPS` |
+| Agent trust input | Issuing intermediate plus root CA |
+
+Replace the example address and certificate names throughout this guide. Keep
+camera credentials out of source control and percent-encode reserved URL
+characters in usernames and passwords.
+
+## Mental model
+
+### RTSPS, SRTSP, TLS, and SRTP
+
+- The standard URL scheme is `rtsps://`. Do not use `srtsp://`.
+- Bosch interfaces and documentation may use SRTSP or Secure RTSP as product
+  terminology.
+- RTSPS carries the RTSP control connection over TLS. With gortsplib, media is
+  normally interleaved over the same TCP/TLS connection for this camera.
+- SRTP is a separate media protection mechanism and is negotiated only when the
+  camera advertises an appropriate secure RTP profile.
+
+Encryption alone does not prove which camera the Agent reached. Verified TLS
+also checks that:
+
+1. The camera certificate is signed by a trusted authority.
+2. The certificate is valid at the current time.
+3. The URL host matches a certificate Subject Alternative Name (SAN).
+
+Modern Go verification uses SANs for identity. A Common Name alone is not
+sufficient. Connecting to `10.0.30.11` requires an IP SAN with that exact value,
+not `DNS:10.0.30.11` and not only a device-name DNS SAN.
+
+### Agent behavior
+
+Kerberos Agent uses gortsplib for RTSP and RTSPS. With the normal configuration,
+gortsplib receives a nil custom TLS configuration and Go performs standard
+certificate and hostname verification with the process trust pool.
+
+`AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE=true` is an explicit escape hatch that
+sets `InsecureSkipVerify` for camera clients. It should be false in a verified
+deployment.
+
+## Communication and certificate flow
+
+The certificate is used during the TLS handshake, before the first RTSP command
+is exchanged. It is not attached to `DESCRIBE`, `SETUP`, or `PLAY`, and the CA
+trust bundle is never sent to the camera.
+
+```mermaid
+sequenceDiagram
+  participant Agent as Kerberos Agent
+  participant Trust as Go trust pool
+  participant Camera as Camera RTSPS :9554
+
+  Agent->>Trust: Load system roots and append AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE
+  Agent->>Camera: Open TCP connection
+  Agent->>Camera: Send TLS ClientHello
+  Camera-->>Agent: Send TLS ServerHello and camera certificate
+  Agent->>Trust: Verify chain, validity, serverAuth, and URL host against SAN
+  Trust-->>Agent: Accept or reject the camera identity
+  Agent->>Camera: Complete TLS handshake
+  Note over Agent,Camera: All following traffic is encrypted by TLS
+  Agent->>Camera: DESCRIBE with RTSP authentication
+  Camera-->>Agent: Return SDP and available media tracks
+  Agent->>Camera: SETUP selected video and audio tracks over TCP
+  Agent->>Camera: PLAY
+  Camera-->>Agent: Send interleaved RTP and RTCP media over TLS
+```
+
+The files and keys have distinct roles:
+
+| Material | Location | Purpose | Sent over the connection |
+| --- | --- | --- | --- |
+| Camera leaf certificate | Camera | Identifies the camera and binds its public key to its SAN | Yes, by the camera during the TLS handshake |
+| Camera private key | Camera | Proves that the camera owns the presented certificate | No |
+| Intermediate and root CA PEM bundle | Agent | Lets Go build and trust the camera certificate chain | No |
+| RTSP username and password | Agent configuration or URL | Authenticates the Agent to the RTSP service after TLS succeeds | An authentication response is sent inside TLS; its form depends on the RTSP authentication method |
+
+For an `rtsps://` URL, the Agent parses the URL and gives gortsplib the host and
+TLS settings. gortsplib opens the TCP connection and starts TLS. Go compares the
+certificate presented by the camera with the local trust pool, checks its
+validity period and server usage, and matches the URL hostname or IP address to
+the certificate SAN. Only a successful handshake creates the encrypted channel
+needed for the RTSP exchange.
+
+The Agent then sends `DESCRIBE`, selects the advertised video and audio tracks,
+sends `SETUP`, and starts delivery with `PLAY`. For the tested camera, gortsplib
+uses interleaved TCP, so the RTSP control messages and RTP/RTCP media remain
+inside the same encrypted TLS connection. Main stream, sub stream, and enabled
+audio backchannel clients each establish and verify their own connection.
+
+If certificate verification fails, the TLS handshake does not complete and no
+usable RTSP session is established. Setting
+`AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE=true` keeps traffic encrypted but skips
+certificate-chain and hostname verification, so an attacker could impersonate
+the camera. It is not equivalent to trusting the camera certificate.
+
+## Decide the certificate identity first
+
+Choose the stable name used in every Agent URL before creating the certificate:
+
+- For an IP URL, add that address as an IP SAN.
+- For a DNS URL, add the exact hostname as a DNS SAN.
+- Add both when clients legitimately use both forms.
+
+A certificate stops matching if the camera IP changes. Use a static address,
+DHCP reservation, or stable DNS name.
+
+## Configure RTSPS in the Bosch UI
+
+1. Sign in to the camera as an administrator.
+2. Open **Configuration**.
+3. Open **Network > Network Services**.
+4. Enable **RTSPS**.
+5. Confirm port `9554`, or record the configured alternative.
+6. Click **Set**.
+
+RTSP on port `554` and RTSPS on port `9554` are separate services. Enabling
+RTSPS does not make an `rtsp://` URL secure.
+
+## Generate the private key and CSR on the camera
+
+Keeping the TLS private key on the camera avoids exporting it to an operator
+workstation or deployment system.
+
+1. Open **Service > Certificates**.
+2. Click **Add**.
+3. Select **Generate signing request**.
+4. Select `RSA 2048bit` or the stronger option supported by all clients.
+5. Enter a unique file name, such as `agent-rtsps`.
+6. Enter a descriptive Common Name and any required organization fields.
+7. Click **Generate**.
+8. Download the resulting CSR from the certificate table.
+
+On the tested firmware, this form contains no SAN field. The downloaded CSR
+therefore has no IP SAN. The CA must add the SAN while signing.
+
+Inspect the CSR before signing:
+
+```bash
+openssl req -in camera.csr.pem -noout -verify -subject
+openssl req -in camera.csr.pem -noout -text
+```
+
+The first command must report `Certificate request self-signature verify OK`.
+An absent `Subject Alternative Name` section is expected for this firmware.
+
+## Prepare Smallstep
+
+Use an existing organizational CA when one is available. Creating a new CA
+creates a new long-lived trust domain that must be distributed, protected,
+backed up, and eventually rotated.
+
+### Install the CLI on Debian amd64
+
+```bash
+curl -fsSL \
+  https://dl.smallstep.com/cli/docs-ca-install/latest/step-cli_amd64.deb \
+  -o /tmp/step-cli_amd64.deb
+sudo dpkg -i /tmp/step-cli_amd64.deb
+rm /tmp/step-cli_amd64.deb
+step version
+```
+
+Use the official package matching the host architecture on other systems.
+
+### Create a dedicated offline CA
+
+Skip this section when using an existing CA.
+
+```bash
+umask 077
+mkdir -p "$HOME/.step/secrets" "$HOME/.step/camera"
+
+openssl rand -base64 48 > "$HOME/.step/secrets/camera_ca_password"
+chmod 600 "$HOME/.step/secrets/camera_ca_password"
+
+step ca init \
+  --pki \
+  --name "UUG Camera CA" \
+  --password-file "$HOME/.step/secrets/camera_ca_password"
+```
+
+This produces:
+
+```text
+$HOME/.step/certs/root_ca.crt
+$HOME/.step/certs/intermediate_ca.crt
+$HOME/.step/secrets/root_ca_key
+$HOME/.step/secrets/intermediate_ca_key
+$HOME/.step/secrets/camera_ca_password
+```
+
+The files under `secrets/` are sensitive. Keep them mode `600`, never commit
+them, and back them up to encrypted persistent storage. A devcontainer can be
+rebuilt or deleted; it is not sufficient as the only CA backup.
+
+## Add the SAN while signing
+
+Copy the camera CSR into a protected working directory:
+
+```bash
+cp camera.csr.pem "$HOME/.step/camera/camera.csr.pem"
+```
+
+Create `$HOME/.step/camera/bosch-rtsps.tpl`:
+
+```json
+{
+  "subject": {
+    "commonName": {{ toJson .Insecure.CR.Subject.CommonName }}
+  },
+  "ipAddresses": ["10.0.30.11"],
+  "keyUsage": ["keyEncipherment", "digitalSignature"],
+  "extKeyUsage": ["serverAuth", "clientAuth"]
+}
+```
+
+The template preserves the camera CSR public key, sets the IP identity, and
+creates a TLS leaf rather than a CA certificate.
+
+Sign it with a validity period that ends before the intermediate CA expires.
+A one-year leaf is preferable to a ten-year leaf when automated renewal is
+available:
+
+```bash
+step certificate sign \
+  --template "$HOME/.step/camera/bosch-rtsps.tpl" \
+  --bundle \
+  --not-after 8760h \
+  --password-file "$HOME/.step/secrets/camera_ca_password" \
+  "$HOME/.step/camera/camera.csr.pem" \
+  "$HOME/.step/certs/intermediate_ca.crt" \
+  "$HOME/.step/secrets/intermediate_ca_key" \
+  > "$HOME/.step/camera/bosch-rtsps-chain.pem"
+```
+
+For an online `step-ca`, do not assume `step ca sign` accepts a `--san` flag. It
+does not. Authorize SANs in the one-time token or configure a provisioner
+template that produces the required SANs.
+
+## Validate before upload
+
+Inspect the leaf certificate, which is the first PEM block in the chain file:
+
+```bash
+openssl x509 \
+  -in "$HOME/.step/camera/bosch-rtsps-chain.pem" \
+  -noout -subject -issuer -dates -ext subjectAltName -ext extendedKeyUsage
+```
+
+Confirm the SAN separately because some OpenSSL versions display only the last
+requested extension:
+
+```bash
+openssl x509 \
+  -in "$HOME/.step/camera/bosch-rtsps-chain.pem" \
+  -noout -ext subjectAltName
+```
+
+Verify the path and IP identity:
+
+```bash
+openssl verify \
+  -CAfile "$HOME/.step/certs/root_ca.crt" \
+  -untrusted "$HOME/.step/certs/intermediate_ca.crt" \
+  -verify_ip 10.0.30.11 \
+  "$HOME/.step/camera/bosch-rtsps-chain.pem"
+```
+
+Confirm that the signed leaf uses the exact public key from the camera CSR:
+
+```bash
+csr_key=$(
+  openssl req -in "$HOME/.step/camera/camera.csr.pem" -pubkey -noout |
+    openssl pkey -pubin -outform DER 2>/dev/null |
+    sha256sum | cut -d' ' -f1
+)
+
+cert_key=$(
+  openssl x509 -in "$HOME/.step/camera/bosch-rtsps-chain.pem" -pubkey -noout |
+    openssl pkey -pubin -outform DER 2>/dev/null |
+    sha256sum | cut -d' ' -f1
+)
+
+test "$csr_key" = "$cert_key"
+```
+
+Do not upload a certificate when any of these checks fail.
+
+## Upload and assign the certificate
+
+1. Return to **Service > Certificates**.
+2. Click **Add > Upload certificate**.
+3. Select the leaf-plus-intermediate PEM chain.
+4. Click **Upload** and wait for `100%`.
+5. Confirm that the former CSR row is now a `Certificate`.
+6. Confirm that the key icon is present. It proves that the camera associated
+   the certificate with its retained private key.
+7. Open the new certificate's **Usage** selector.
+8. Select only **HTTPS**.
+9. Leave **CBS client** assigned to the original Bosch `DeviceCertificate`.
+10. Click **Set** and wait for the table to reload.
+
+On the tested firmware, there is no separate SRTSP usage. RTSPS presents the
+certificate assigned to HTTPS. Reassigning HTTPS therefore changes both the
+web interface and RTSPS certificate.
+
+After saving, the expected split is:
+
+| Certificate | Usage |
+| --- | --- |
+| Private-CA camera certificate | `HTTPS` |
+| Bosch `DeviceCertificate` | `CBS client` |
+
+The browser may warn about the new HTTPS certificate until the private root CA
+is trusted by the workstation.
+
+## Account for the Bosch chain behavior
+
+The tested firmware served only the leaf certificate on ports `443` and `9554`,
+even when the uploaded file contained the leaf and intermediate. Uploading the
+intermediate separately as a trusted camera certificate did not change the
+served chain.
+
+Confirm the behavior:
+
+```bash
+openssl s_client \
+  -connect 10.0.30.11:9554 \
+  -showcerts </dev/null 2>/dev/null |
+  grep -c '^-----BEGIN CERTIFICATE-----$'
+```
+
+A result of `1` means the client must already have the issuing intermediate.
+Create a portable trust bundle containing the intermediate and root:
+
+```bash
+step certificate bundle \
+  "$HOME/.step/certs/intermediate_ca.crt" \
+  "$HOME/.step/certs/root_ca.crt" \
+  "$HOME/.step/camera/uug-camera-trust-bundle.pem"
+
+chmod 644 "$HOME/.step/camera/uug-camera-trust-bundle.pem"
+```
+
+The trust bundle is public material. The CA private keys and password are not.
+
+## Configure Kerberos Agent
+
+For a process running directly in the same environment:
+
+```dotenv
+AGENT_CAPTURE_IPCAMERA_RTSP="rtsps://<user>:<password>@10.0.30.11:9554/?inst=1"
+AGENT_CAPTURE_IPCAMERA_SUB_RTSP="rtsps://<user>:<password>@10.0.30.11:9554/?inst=2"
+AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE=false
+AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE=/home/agent/data/config/uug-camera-trust-bundle.pem
+```
+
+For a container, mount the public bundle read-only at the exact path visible
+inside the container. The Agent image creates `/home/agent/data/config` and
+includes Debian's `ca-certificates` package:
+
+```bash
+docker run \
+  -v /secure/config/uug-camera-trust-bundle.pem:/home/agent/data/config/uug-camera-trust-bundle.pem:ro \
+  -e AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE=/home/agent/data/config/uug-camera-trust-bundle.pem \
+  -e AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE=false \
+  -e 'AGENT_CAPTURE_IPCAMERA_RTSP=rtsps://<user>:<password>@10.0.30.11:9554/?inst=1' \
+  -e 'AGENT_CAPTURE_IPCAMERA_SUB_RTSP=rtsps://<user>:<password>@10.0.30.11:9554/?inst=2' \
+  kerberos/agent:latest
+```
+
+`AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE` starts with the operating system's roots
+and appends the camera bundle only to the gortsplib TLS configuration. Other
+clients, including MoQ, Hub, and Vault, retain the normal public CA chain.
+Restart the Agent after changing trust files.
+
+Do not set `SSL_CERT_FILE` or `SSL_CERT_DIR` in production solely for camera
+trust. They are process-wide and can prevent other clients from validating
+public services. For a deliberate process-wide isolation test, mount an empty
+directory and set `SSL_CERT_DIR` to its path:
+
+```bash
+-v /secure/config/empty-ca-dir:/home/agent/data/config/empty-ca-dir:ro \
+-e SSL_CERT_DIR=/home/agent/data/config/empty-ca-dir
+```
+
+Do not use `SSL_CERT_DIR=`. Go treats an empty value as unset and scans its
+default certificate directories.
+
+Only use that mode when the Agent does not need public roots for other TLS
+connections.
+
+## Validate the live endpoints
+
+### Strict TLS and identity check
+
+Use only the specified bundle, without OpenSSL's default CA locations:
+
+```bash
+openssl s_client \
+  -brief \
+  -connect 10.0.30.11:9554 \
+  -verify_ip 10.0.30.11 \
+  -verify_return_error \
+  -CAfile "$HOME/.step/camera/uug-camera-trust-bundle.pem" \
+  -no-CApath \
+  -no-CAstore \
+  </dev/null
+```
+
+Repeat with port `443`. Both must report `Verification: OK`.
+
+Confirm that identity checking is active by repeating the command with a wrong
+address, such as `-verify_ip 10.0.30.12`. It must fail with an IP address
+mismatch.
+
+### Confirm the live leaf is the generated leaf
+
+```bash
+live_fingerprint=$(
+  openssl s_client -connect 10.0.30.11:9554 -showcerts </dev/null 2>/dev/null |
+    openssl x509 -noout -fingerprint -sha256 |
+    cut -d= -f2
+)
+
+local_fingerprint=$(
+  openssl x509 \
+    -in "$HOME/.step/camera/bosch-rtsps-chain.pem" \
+    -noout -fingerprint -sha256 |
+    cut -d= -f2
+)
+
+test "$live_fingerprint" = "$local_fingerprint"
+```
+
+### Validate the media path
+
+A successful TLS handshake does not prove that RTSP authentication, DESCRIBE,
+SETUP, PLAY, and RTP delivery work. Start a fresh Agent with verified TLS and
+confirm that it connects without an x509 error and receives frames. During the
+verified setup described here, a gortsplib probe completed all RTSP operations
+and received an RTP packet over TCP.
+
+## Why a tampered bundle may still connect
+
+Editing PEM text is not always a useful negative TLS test.
+
+### A certificate can still parse after a byte change
+
+Base64 can remain syntactically valid when one character changes. OpenSSL may
+still list the certificate subject and issuer even though a signature is now
+invalid. Parsing and signature verification are different operations.
+
+### Trust anchors are not validated through a parent
+
+Every certificate loaded into Go's root pool is a trust anchor, including a
+non-self-signed intermediate CA. Verification can terminate at that certificate.
+
+If tampering changes only the intermediate's signature from its parent root,
+but does not change its public key, that intermediate can still validate the
+camera leaf when it is trusted directly. Its now-invalid parent signature is
+not consulted at the trust boundary.
+
+This is equivalent to OpenSSL's partial-chain behavior:
+
+```bash
+openssl s_client \
+  -connect 10.0.30.11:9554 \
+  -verify_ip 10.0.30.11 \
+  -verify_return_error \
+  -partial_chain \
+  -CAfile tampered-bundle.pem \
+  -no-CApath \
+  -no-CAstore \
+  </dev/null
+```
+
+### `SSL_CERT_FILE` does not isolate Go from CA directories
+
+On Unix, Go uses `SSL_CERT_FILE` instead of its default aggregate CA file, but it
+still scans default certificate directories such as `/etc/ssl/certs`. Setting
+`SSL_CERT_FILE` alone therefore does not remove CA certificates installed with
+`update-ca-certificates`. `AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE` is appended
+after this system pool is loaded; it does not replace the system roots.
+
+Blank values do not select empty trust sources. Both `SSL_CERT_FILE=` and
+`SSL_CERT_DIR=` are treated as unset, so Go falls back to its default aggregate
+CA file and certificate directories. To test with no trusted certificates on
+Linux, use a non-empty file path that contains no certificates and a non-empty
+directory path that contains no certificates:
+
+```bash
+mkdir -p /tmp/empty-ca-dir
+SSL_CERT_FILE=/dev/null \
+SSL_CERT_DIR=/tmp/empty-ca-dir \
+AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE= \
+AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE=false \
+GOWORK=off \
+go run -tags moq . -action run -port 8080
+```
+
+That fresh process must fail with `x509: certificate signed by unknown
+authority`.
+
+Use exactly one camera trust-distribution approach when possible:
+
+1. Mount a private trust bundle and set
+  `AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE`; or
+2. Install the CA certificates into the operating-system trust store.
+
+Using both is valid, but makes isolation tests less obvious.
+
+### Running processes can retain old roots
+
+A long-running Go process may already have loaded and cached the trust pool.
+Always start a new process after changing trust configuration during a negative
+test.
+
+## Perform a meaningful negative test
+
+Do not corrupt only the root or intermediate signature. Instead, give a fresh
+Agent process a completely unrelated CA and hide the default CA directories.
+
+```bash
+mkdir -p /tmp/empty-ca-dir
+
+openssl req \
+  -x509 -newkey rsa:2048 -nodes -days 1 \
+  -subj '/CN=Unrelated Test Root' \
+  -keyout /tmp/unrelated-test-root.key \
+  -out /tmp/unrelated-test-root.crt
+
+SSL_CERT_FILE=/dev/null \
+SSL_CERT_DIR=/tmp/empty-ca-dir \
+AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE=/tmp/unrelated-test-root.crt \
+AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE=false \
+GOWORK=off \
+go run -tags moq . -action run -port 8080
+```
+
+The connection must fail with an unknown-authority or chain-building error.
+Delete the temporary test key and certificate afterward.
+
+To test bundle integrity rather than client distrust, validate the intermediate
+against the root explicitly:
+
+```bash
+openssl verify \
+  -CAfile "$HOME/.step/certs/root_ca.crt" \
+  -no-CApath \
+  -no-CAstore \
+  "$HOME/.step/certs/intermediate_ca.crt"
+```
+
+Store and compare approved SHA-256 fingerprints when detecting unauthorized
+certificate-file changes is a requirement.
+
+Restore an accidentally edited bundle from the protected CA certificates, then
+restart the Agent:
+
+```bash
+step certificate bundle -f \
+  "$HOME/.step/certs/intermediate_ca.crt" \
+  "$HOME/.step/certs/root_ca.crt" \
+  "$HOME/.step/camera/uug-camera-trust-bundle.pem"
+
+openssl verify \
+  -CAfile "$HOME/.step/certs/root_ca.crt" \
+  -no-CApath \
+  -no-CAstore \
+  "$HOME/.step/certs/intermediate_ca.crt"
+```
+
+## Optional system trust installation
+
+On Debian, install both public CA certificates when every process in the system
+should trust this camera PKI:
+
+```bash
+sudo install -m 0644 \
+  "$HOME/.step/certs/root_ca.crt" \
+  /usr/local/share/ca-certificates/uug-camera-ca.crt
+
+sudo install -m 0644 \
+  "$HOME/.step/certs/intermediate_ca.crt" \
+  /usr/local/share/ca-certificates/uug-camera-intermediate-ca.crt
+
+sudo update-ca-certificates
+```
+
+This creates links below `/etc/ssl/certs`. Remove those files and rerun
+`update-ca-certificates` before attempting an isolated trust-bundle test.
+
+## Renewal and recovery
+
+- Renew before the leaf or intermediate expires.
+- Generate a new camera CSR if the firmware cannot renew the existing key.
+- Sign the new CSR with all required SANs.
+- Upload and validate the new certificate before deleting the old one.
+- Preserve an alternate administrative access path while changing HTTPS usage.
+- Back up the CA certificates, encrypted CA keys, and password separately.
+- If the CA private keys are lost, create a new CA and redistribute its trust
+  before replacing camera certificates.
+
+## Production checklist
+
+- [ ] The Agent URL uses `rtsps://`, not `rtsp://` or `srtsp://`.
+- [ ] RTSPS is enabled on the camera and the configured port is reachable.
+- [ ] The certificate SAN exactly matches the Agent URL host.
+- [ ] The leaf public key matches the camera-generated CSR.
+- [ ] The certificate has `serverAuth` extended key usage.
+- [ ] The certificate expires before its issuer.
+- [ ] HTTPS is assigned to the private-CA certificate.
+- [ ] CBS client remains assigned to the Bosch device certificate.
+- [ ] The Agent has the intermediate and root CA certificates it needs.
+- [ ] `AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE=false`.
+- [ ] The Agent was restarted after trust changes.
+- [ ] A strict TLS check reports `Verification: OK`.
+- [ ] A real Agent connection receives RTP packets.
+- [ ] CA private keys and passwords are backed up outside the devcontainer.

README.md

@@ -3,7 +3,6 @@
 <a target="_blank" href="https://kerberos.io"><img src="https://img.shields.io/badge/kerberos-website-gray.svg?longCache=true&colorB=brightgreen" alt="Kerberos Agent"></a>
 <a target="_blank" href="https://doc.kerberos.io"><img src="https://img.shields.io/badge/kerberos-documentation-gray.svg?longCache=true&colorB=brightgreen" alt="Kerberos Agent"></a>
 
-<a target="_blank" href="https://circleci.com/gh/kerberos-io/agent"><img src="https://circleci.com/gh/kerberos-io/agent.svg?style=svg"/></a>
 <img src="https://github.com/kerberos-io/agent/workflows/Go/badge.svg"/>
 <img src="https://github.com/kerberos-io/agent/workflows/React/badge.svg"/>
 <img src="https://github.com/kerberos-io/agent/workflows/CodeQL/badge.svg"/>
@@ -189,10 +188,123 @@ Next to attaching the configuration file, it is also possible to override the co
     -e AGENT_CAPTURE_CONTINUOUS=true \
     -d --restart=always kerberos/agent:latest
 
+Set `AGENT_PORT` to change the Agent's internal HTTP port. The environment
+variable takes precedence over `-port`. When using Docker, publish the same
+container port:
+
+```bash
+docker run -p 8082:8082 \
+  -e AGENT_PORT=8082 \
+  --name mycamera \
+  -d --restart=always kerberos/agent:latest
+```
+
+### Health checks and API responses
+
+`GET /health` is an unauthenticated liveness and diagnostics endpoint for
+container orchestrators and external monitors. `healthy` means that the Agent
+HTTP process can serve requests. Camera streams and Hub connectivity are
+reported independently in the response, so an external camera or Hub outage
+does not make Docker restart an otherwise functioning Agent process. Successful
+probes return `200 OK` and are logged at debug level to avoid periodic health
+checks filling production logs.
+
+```bash
+curl http://localhost:8082/health
+```
+
+```json
+{
+  "httpStatusCode": 200,
+  "applicationStatusCode": "get_success",
+  "entityStatusCode": "healthy",
+  "message": "Healthy",
+  "metadata": {
+    "applicationName": "agent",
+    "applicationVersion": "0.0.0",
+    "timestamp": 1788710400,
+    "path": "/health"
+  },
+  "data": {
+    "health": {
+      "description": "Agent HTTP service is healthy",
+      "cameraConnected": true,
+      "mainStream": {
+        "configured": true,
+        "connected": true,
+        "packagesProcessed": 183421,
+        "fps": 29.97,
+        "resolution": {
+          "width": 1920,
+          "height": 1080
+        },
+        "lastPacketAt": 1788710399
+      },
+      "subStream": {
+        "configured": true,
+        "connected": true,
+        "packagesProcessed": 91710,
+        "fps": 15,
+        "resolution": {
+          "width": 640,
+          "height": 360
+        },
+        "lastPacketAt": 1788710399
+      },
+      "hub": {
+        "configured": true,
+        "connected": true,
+        "lastHeartbeatAttemptAt": 1788710398,
+        "lastSuccessfulHeartbeatAt": 1788710398
+      }
+    }
+  }
+}
+```
+
+Stream `packagesProcessed` counts complete encoded video access units processed
+since the Agent process started; it does not count individual fragmented RTP
+packets. `fps` is the latest PTS-derived frame-rate estimate. Resolution is the
+most recently observed encoded width and height. `lastPacketAt`, Hub heartbeat
+timestamps, and response metadata timestamps are Unix seconds; `0` means that
+no value has been observed yet. A stream can be configured but temporarily
+disconnected.
+
+Hub `configured` means heartbeat delivery is enabled and has the required URI
+and key. Hub `connected` means the most recent heartbeat succeeded and the last
+success is no more than three minutes old. A failed heartbeat marks it
+disconnected immediately while retaining the last-success timestamp for
+diagnosis.
+
+New JSON endpoints should use this Hub-compatible response envelope rather than
+adding unrelated top-level response shapes. HTTP status describes the transport
+result, `applicationStatusCode` describes the operation, `entityStatusCode` and
+`message` describe the domain outcome, `metadata` carries request/application
+context, and endpoint-specific content belongs in a typed `data` object. Client
+responses must contain safe messages only; detailed internal errors belong in
+structured logs. Existing endpoints retain their legacy response shapes until
+they are migrated deliberately.
+
+### Secure camera streams (RTSPS)
+
+The Agent accepts `rtsps://` camera URLs. Do not use `srtsp://`; RTSPS is RTSP over TLS. For a Bosch FLEXIDOME micro 3100i, enable **Secure RTSP** under **Network > Network Services** and use port `9554`:
+
+```bash
+AGENT_CAPTURE_IPCAMERA_RTSP='rtsps://username:password@camera.example:9554/?inst=1'
+AGENT_CAPTURE_IPCAMERA_SUB_RTSP='rtsps://username:password@camera.example:9554/?inst=2'
+```
+
+Certificate verification is enabled by default. The URL hostname or IP address must match the camera certificate SAN. On this Bosch firmware, RTSPS presents the certificate assigned to **HTTPS**; there is no separate SRTSP certificate usage. Leave **CBS client** assigned to the Bosch device certificate.
+
+For a private CA, mount a PEM trust bundle containing every CA certificate needed to build the camera certificate chain and set `AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE` to its path inside the Agent. The bundle is appended to the system roots for camera RTSPS connections only. This Bosch firmware presents only its leaf certificate, so include both the issuing intermediate and root certificates in the bundle. As a temporary fallback, `AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE=true` disables certificate verification for camera streams only.
+
+See [RTSPS and TLS certificates](README-RTSPS-TLS.md) for the complete Bosch UI, private-CA, deployment, validation, and troubleshooting procedure.
+
 | Name                                    | Description                                                                                     | Default Value                  |
 | --------------------------------------- | ----------------------------------------------------------------------------------------------- | ------------------------------ |
 | `LOG_LEVEL`                                 | Level for logging, could be "info", "warning", "debug", "error" or "fatal".                     | "info"                         |
 | `LOG_OUTPUT`                                | Logging output format "json" or "text".                                                         | "text"                         |
+| `AGENT_PORT`                                | HTTP web server port. Overrides the `-port` command-line value and must be between 1 and 65535. | "80"                           |
 | `AGENT_MODE`                                | You can choose to run this in 'release' for production, and or 'demo' for showcasing.           | "release"                      |
 | `AGENT_TLS_INSECURE`                        | Specify if you want to use `InsecureSkipVerify` for the internal HTTP client.                   | "false"                        |
 | `AGENT_USERNAME`                            | The username used to authenticate against the Kerberos Agent login page.                        | "root"                         |
@@ -208,8 +320,10 @@ Next to attaching the configuration file, it is also possible to override the co
 | `AGENT_TIME`                                | Enable the timetable for Kerberos Agent                                                         | "false"                        |
 | `AGENT_TIMETABLE`                           | A (weekly) time table to specify when to make recordings "start1,end1,start2,end2;start1..      | ""                             |
 | `AGENT_REGION_POLYGON`                      | A single polygon set for motion detection: "x1,y1;x2,y2;x3,y3;...                               | ""                             |
-| `AGENT_CAPTURE_IPCAMERA_RTSP`               | Full-HD RTSP endpoint to the camera you're targetting.                                          | ""                             |
-| `AGENT_CAPTURE_IPCAMERA_SUB_RTSP`           | Sub-stream RTSP endpoint used for livestreaming (WebRTC).                                       | ""                             |
+| `AGENT_CAPTURE_IPCAMERA_RTSP`               | Full-HD RTSP or RTSPS endpoint for the target camera.                                           | ""                             |
+| `AGENT_CAPTURE_IPCAMERA_SUB_RTSP`           | RTSP or RTSPS sub-stream endpoint used for livestreaming (WebRTC).                              | ""                             |
+| `AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE`      | PEM CA bundle appended to the system roots for RTSPS camera certificate verification.           | ""                             |
+| `AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE`     | Disable RTSPS camera certificate verification; use only when a trusted CA cannot be installed.  | "false"                        |
 | `AGENT_CAPTURE_IPCAMERA_BASE_WIDTH`         | Force a specific width resolution for live view processing.                                     | ""                             |
 | `AGENT_CAPTURE_IPCAMERA_BASE_HEIGHT`        | Force a specific height resolution for live view processing.                                    | ""                             |
 | `AGENT_CAPTURE_IPCAMERA_ONVIF`              | Mark as a compliant ONVIF device.                                                               | ""                             |
@@ -227,9 +341,10 @@ Next to attaching the configuration file, it is also possible to override the co
 | `AGENT_CAPTURE_PIXEL_CHANGE`                | If `CONTINUOUS` set to `false`, the number of pixel require to change before motion triggers.   | "150"                          |
 | `AGENT_CAPTURE_FRAGMENTED`                  | Set the format of the recorded MP4 to fragmented (suitable for HLS).                            | "false"                        |
 | `AGENT_CAPTURE_FRAGMENTED_DURATION`         | If `AGENT_CAPTURE_FRAGMENTED` set to `true`, define the duration (seconds) of a fragment.       | "8"                            |
-| `AGENT_MQTT_URI`                            | An MQTT broker endpoint that is used for bi-directional communication (live view, onvif, etc)   | "tcp://mqtt.kerberos.io:1883"  |
+| `AGENT_MQTT_URI`                            | MQTT broker endpoint for bi-directional communication. Accepts ActiveMQ `mqtt+ssl://` URLs.     | "tcp://mqtt.kerberos.io:1883"  |
 | `AGENT_MQTT_USERNAME`                       | Username of the MQTT broker.                                                                    | ""                             |
 | `AGENT_MQTT_PASSWORD`                       | Password of the MQTT broker.                                                                    | ""                             |
+| `AGENT_REMOTE_ACCESS_ENABLED`               | Allow encrypted Hub MQTT sessions to stream Agent logs and open an interactive shell. Enable only for trusted deployments. | "false"                        |
 | `AGENT_REALTIME_PROCESSING`                 | If `AGENT_REALTIME_PROCESSING` set to `true`, the agent will send key frames to the topic       | ""                             |
 | `AGENT_REALTIME_PROCESSING_TOPIC`           | The topic to which keyframes will be sent in base64 encoded format.                             | ""                             |
 | `AGENT_STUN_URI`                            | When using WebRTC, you'll need to provide a STUN server.                                        | "stun:turn-fra1.kerberos.io:3478"|
@@ -244,6 +359,7 @@ Next to attaching the configuration file, it is also possible to override the co
 | `AGENT_HUB_PRIVATE_KEY`                     | The secret access key linked to your account in Kerberos Hub.                                   | ""                             |
 | `AGENT_HUB_REGION`                          | The Kerberos Hub region, to which you want to upload.                                           | ""                             |
 | `AGENT_HUB_SITE`                            | The site ID of a site you've created in your Kerberos Hub account.                              | ""                             |
+| `AGENT_TUS_CHUNK_SIZE_BYTES`                | Bytes sent in each resumable-upload PATCH. Set to `0` or a negative value to send all remaining bytes in one PATCH. | "8388608" (8 MiB)              |
 | `AGENT_KERBEROSVAULT_URI`                   | The Kerberos Vault API url.                                                                     | "https://vault.domain.com/api" |
 | `AGENT_KERBEROSVAULT_ACCESS_KEY`            | The access key of a Kerberos Vault account.                                                     | ""                             |
 | `AGENT_KERBEROSVAULT_SECRET_KEY`            | The secret key of a Kerberos Vault account.                                                     | ""                             |
@@ -264,6 +380,34 @@ Next to attaching the configuration file, it is also possible to override the co
 | `AGENT_SIGNING`                             | Enable 'true' or disable 'false' for signing recordings.                                        | "true"                         |
 | `AGENT_SIGNING_PRIVATE_KEY`                 | The private key (RSA) to sign the recordings fingerprint to validate origin.                    | "" - uses default one if empty |
 
+Remote console access is disabled unless `AGENT_REMOTE_ACCESS_ENABLED=true`.
+The Agent also rejects remote session messages unless Hub encryption or
+end-to-end MQTT encryption is configured and used. A remote shell runs inside
+the Agent process environment as the Agent operating-system user; it is not an
+SSH server and does not expose a new network port. Keep the feature disabled on
+deployments where Hub owners should not have operating-system access.
+
+### Resumable upload chunk size
+
+Hub and Vault resumable uploads use `AGENT_TUS_CHUNK_SIZE_BYTES` as the maximum
+body size of each tus `PATCH` request. The value is a number of bytes, not a
+number of chunks. If the variable is unset or invalid, the Agent uses 8 MiB:
+
+```dotenv
+# 8 MiB (default)
+AGENT_TUS_CHUNK_SIZE_BYTES=8388608
+
+# 4 MiB: more frequent checkpoints on unstable connections
+AGENT_TUS_CHUNK_SIZE_BYTES=4194304
+
+# Disable chunking and send all remaining bytes in one PATCH
+AGENT_TUS_CHUNK_SIZE_BYTES=0
+```
+
+Smaller chunks provide more frequent resumable checkpoints but create more HTTP
+requests. Larger chunks reduce request overhead but require more data to be
+retransmitted when a request fails.
+
 
 ## Encryption
 
@@ -429,10 +573,35 @@ uses Debian Trixie. The publisher is disabled unless explicitly enabled at runti
       -e AGENT_LIVE_MOQ_URL=https://relay.uug.ai/anon \
       kerberos/agent
 
-`AGENT_LIVE_MOQ_BROADCAST_PREFIX` defaults to `devices`, producing the broadcast
-`devices/<agent-key>/live.hang`. `AGENT_LIVE_MOQ_QUALITY` accepts `auto` (the
-default), `high`, or `low` and selects the main or sub camera stream when the
-Agent starts. The initial implementation publishes H.264 video only.
+`AGENT_LIVE_MOQ_BROADCAST_PREFIX` defaults to `devices`. MoQ viewers subscribe to
+a relay and never negotiate with the Agent, so every quality tier is published as
+its own broadcast and switching quality is simply a resubscribe:
+
+| Tier   | Broadcast                             | Source                                     |
+| ------ | ------------------------------------- | ------------------------------------------ |
+| `high` | `devices/<agent-key>/live.hang`       | highest-resolution camera stream           |
+| `low`  | `devices/<agent-key>/live-low.hang`   | sub stream (main stream when none is set)  |
+
+Each tier uploads every frame while it has subscribers. While idle, it continues
+to upload fresh keyframes only. This keeps the relay's latest cached GOP current,
+so a new viewer does not initially render the final frame from the previous
+viewer session, while avoiding the bandwidth cost of continuously sending every
+delta frame. `AGENT_LIVE_MOQ_QUALITY` accepts `high` or `low` to pin the Agent to
+a single tier; viewers requesting the other tier then find no broadcast. Any
+other value (including the default) publishes both. The initial implementation
+publishes H.264 video only.
+
+Two bounded duration settings tune recovery for unusual network conditions:
+
+| Variable | Default | Allowed range |
+| -------- | ------- | ------------- |
+| `AGENT_LIVE_MOQ_MAX_PACKET_AGE` | `1.5s` | `250ms` to `30s` |
+| `AGENT_LIVE_MOQ_WRITE_TIMEOUT` | `5s` | `1s` to `1m` |
+
+Values use Go duration syntax and are clamped to the documented range. The
+dashboard API's `recovery` object reports per-tier reconnects, last successful
+frame/write timing, native write timeouts, watchdog restarts/cooldown, dropped
+run-channel events, and run-worker shutdown timeouts.
 
 The `/anon` relay route is intended for interoperability testing. Production
 deployments must set `AGENT_LIVE_MOQ_URL` to a short-lived, device-scoped
@@ -445,8 +614,9 @@ monorepo devcontainer so it uses the Trixie base, then run the VS Code task
     cd machinery
     bash ./verify-moq-devcontainer.sh
 
-The check requires glibc 2.38 or newer, runs the tagged package tests, links the
-complete Agent with `-tags moq`, and executes the resulting binary's version
+The check requires glibc 2.38 or newer, runs the tagged package and in-process
+native QUIC lifecycle tests, links the complete Agent with `-tags moq`, and
+executes the resulting binary's version
 command. Both devcontainers also run this check during their post-create setup.
 
 ## What is new?

machinery/README.md

@@ -5,7 +5,6 @@
 <a target="_blank" href="https://twitter.com/kerberosio?ref_src=twsrc%5Etfw"><img src="https://img.shields.io/twitter/url.svg?label=Follow%20%40kerberosio&style=social&url=https%3A%2F%2Ftwitter.com%2Fkerberosio" alt="Twitter Widget"></a>
 <a target="_blank" href="https://join.slack.com/t/kerberosio/shared_invite/zt-1a5oj4pwm-O4qCAN9c5r2um0Ns0ge8ww"><img src="https://img.shields.io/badge/slack-@kerberosio-yellow.svg?logo=slack " alt="Kerberos.io"></a>
 
-<a target="_blank" href="https://circleci.com/gh/kerberos-io/agent"><img src="https://circleci.com/gh/kerberos-io/agent.svg?style=svg"/></a>
 <img src="https://github.com/kerberos-io/agent/workflows/Go/badge.svg"/>
 <img src="https://github.com/kerberos-io/agent/workflows/CodeQL/badge.svg"/>
 

machinery/data/tus/1786715908_3-737_encrypted_0-0-0-0_-1_25498.mp4.primary.json

@@ -0,0 +1 @@
+{"upload_url":"https://vault.kerberos.io/api/storage/tus/19e42fbc666a38064904caf8c46d182a","vault_uri":"https://vault.kerberos.io/api/storage/tus/","size":1591581}

machinery/data/tus/1788718145_3-694_encrypted_0-0-0-0_-1_25497.mp4.primary.json

@@ -0,0 +1 @@
+{"upload_url":"https://vault.kerberos.io/api/storage/tus/bd3be0f9ecd3873d381f57cb8b4a7e96","vault_uri":"https://vault.kerberos.io/api/storage/tus/","size":2560480}

machinery/docs/docs.go

@@ -24,6 +24,38 @@ const docTemplate = `{
     "host": "{{.Host}}",
     "basePath": "{{.BasePath}}",
     "paths": {
+        "/api/camera/discover": {
+            "get": {
+                "description": "Runs an advanced Fing/WiFiman-style scan (ONVIF WS-Discovery + TCP port scan + MAC/vendor lookup) and returns the devices found on the local network.",
+                "tags": [
+                    "onvif"
+                ],
+                "summary": "Discover cameras and other devices on the local network.",
+                "operationId": "camera-discover",
+                "parameters": [
+                    {
+                        "type": "integer",
+                        "description": "Discovery timeout in milliseconds (default 2000)",
+                        "name": "timeout",
+                        "in": "query"
+                    },
+                    {
+                        "type": "string",
+                        "description": "Optional subnet(s) to scan, e.g. '192.168.1.0/24' (comma-separated). Defaults to the local interfaces.",
+                        "name": "subnet",
+                        "in": "query"
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/models.APIResponse"
+                        }
+                    }
+                }
+            }
+        },
         "/api/camera/onvif/capabilities": {
             "post": {
                 "description": "Will return the ONVIF capabilities for the specific camera.",
@@ -623,6 +655,40 @@ const docTemplate = `{
                 }
             }
         },
+        "/api/persistence/secondary/verify": {
+            "post": {
+                "security": [
+                    {
+                        "Bearer": []
+                    }
+                ],
+                "description": "Will verify the secondary persistence.",
+                "tags": [
+                    "persistence"
+                ],
+                "summary": "Will verify the secondary persistence.",
+                "operationId": "verify-secondary-persistence",
+                "parameters": [
+                    {
+                        "description": "Config",
+                        "name": "config",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/models.Config"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/models.APIResponse"
+                        }
+                    }
+                }
+            }
+        },
         "/api/persistence/verify": {
             "post": {
                 "security": [
@@ -656,9 +722,151 @@ const docTemplate = `{
                     }
                 }
             }
+        },
+        "/health": {
+            "get": {
+                "description": "Confirms that the Agent HTTP process can serve requests and reports current camera stream and Hub heartbeat diagnostics. Operational dependency failures do not change the liveness HTTP status.",
+                "produces": [
+                    "application/json"
+                ],
+                "tags": [
+                    "health"
+                ],
+                "summary": "Check Agent health",
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/http.HealthResponse"
+                        }
+                    }
+                }
+            }
         }
     },
     "definitions": {
+        "http.Health": {
+            "type": "object",
+            "properties": {
+                "cameraConnected": {
+                    "type": "boolean"
+                },
+                "description": {
+                    "type": "string"
+                },
+                "hub": {
+                    "$ref": "#/definitions/http.HubHealth"
+                },
+                "mainStream": {
+                    "$ref": "#/definitions/http.StreamHealth"
+                },
+                "subStream": {
+                    "$ref": "#/definitions/http.StreamHealth"
+                }
+            }
+        },
+        "http.HealthResponse": {
+            "type": "object",
+            "properties": {
+                "applicationStatusCode": {
+                    "type": "string"
+                },
+                "data": {
+                    "$ref": "#/definitions/http.HealthResponseData"
+                },
+                "entityStatusCode": {
+                    "type": "string"
+                },
+                "httpStatusCode": {
+                    "type": "integer"
+                },
+                "message": {
+                    "type": "string"
+                },
+                "metadata": {
+                    "$ref": "#/definitions/http.ResponseMetadata"
+                }
+            }
+        },
+        "http.HealthResponseData": {
+            "type": "object",
+            "properties": {
+                "health": {
+                    "$ref": "#/definitions/http.Health"
+                }
+            }
+        },
+        "http.HubHealth": {
+            "type": "object",
+            "properties": {
+                "configured": {
+                    "type": "boolean"
+                },
+                "connected": {
+                    "type": "boolean"
+                },
+                "lastHeartbeatAttemptAt": {
+                    "type": "integer"
+                },
+                "lastSuccessfulHeartbeatAt": {
+                    "type": "integer"
+                }
+            }
+        },
+        "http.ResponseMetadata": {
+            "type": "object",
+            "properties": {
+                "applicationName": {
+                    "type": "string"
+                },
+                "applicationVersion": {
+                    "type": "string"
+                },
+                "path": {
+                    "type": "string"
+                },
+                "timestamp": {
+                    "type": "integer"
+                },
+                "traceId": {
+                    "type": "string"
+                }
+            }
+        },
+        "http.StreamHealth": {
+            "type": "object",
+            "properties": {
+                "configured": {
+                    "type": "boolean"
+                },
+                "connected": {
+                    "type": "boolean"
+                },
+                "fps": {
+                    "type": "number"
+                },
+                "lastPacketAt": {
+                    "type": "integer"
+                },
+                "packagesProcessed": {
+                    "type": "integer"
+                },
+                "resolution": {
+                    "$ref": "#/definitions/http.StreamResolution"
+                }
+            }
+        },
+        "http.StreamResolution": {
+            "type": "object",
+            "properties": {
+                "height": {
+                    "type": "integer"
+                },
+                "width": {
+                    "type": "integer"
+                }
+            }
+        },
         "models.APIResponse": {
             "type": "object",
             "properties": {
@@ -730,12 +938,19 @@ const docTemplate = `{
                 "fragmentedduration": {
                     "type": "integer"
                 },
+                "gopsize": {
+                    "description": "GOP size in seconds, used for pre-recording",
+                    "type": "integer"
+                },
                 "ipcamera": {
                     "$ref": "#/definitions/models.IPCamera"
                 },
                 "liveview": {
                     "type": "string"
                 },
+                "liveview_chunking": {
+                    "type": "string"
+                },
                 "maxlengthrecording": {
                     "type": "integer"
                 },
@@ -745,6 +960,10 @@ const docTemplate = `{
                 "name": {
                     "type": "string"
                 },
+                "onvif_motion": {
+                    "description": "ONVIFMotion routes the camera's ONVIF motion events into the\nagent's motion-triggered recording pipeline. When \"true\" the\nagent opens an event/stream against the configured ONVIF\nendpoint and forwards Motion+Active events to HandleMotion.\nRequires Capture.IPCamera.ONVIFXAddr / ONVIFUsername /\nONVIFPassword to be set. Default empty (disabled) keeps the\nexisting pixel-diff motion detection as the only source.",
+                    "type": "string"
+                },
                 "pixelChangeThreshold": {
                     "type": "integer"
                 },
@@ -823,9 +1042,15 @@ const docTemplate = `{
                 "kstorage": {
                     "$ref": "#/definitions/models.KStorage"
                 },
+                "kstorage_secondary": {
+                    "$ref": "#/definitions/models.KStorage"
+                },
                 "max_directory_size": {
                     "type": "integer"
                 },
+                "min_free_space": {
+                    "type": "integer"
+                },
                 "mqtt_password": {
                     "type": "string"
                 },
@@ -856,6 +1081,9 @@ const docTemplate = `{
                 "s3": {
                     "$ref": "#/definitions/models.S3"
                 },
+                "signing": {
+                    "$ref": "#/definitions/models.Signing"
+                },
                 "stunuri": {
                     "type": "string"
                 },
@@ -947,6 +1175,15 @@ const docTemplate = `{
         "models.IPCamera": {
             "type": "object",
             "properties": {
+                "base_height": {
+                    "type": "integer"
+                },
+                "base_width": {
+                    "type": "integer"
+                },
+                "channels": {
+                    "type": "integer"
+                },
                 "fps": {
                     "type": "string"
                 },
@@ -965,21 +1202,85 @@ const docTemplate = `{
                 "onvif_xaddr": {
                     "type": "string"
                 },
+                "pps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "rtsp": {
                     "type": "string"
                 },
+                "sample_rate": {
+                    "type": "integer"
+                },
+                "sps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_fps": {
                     "type": "string"
                 },
                 "sub_height": {
                     "type": "integer"
                 },
+                "sub_pps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_rtsp": {
                     "type": "string"
                 },
+                "sub_sps_nalus": {
+                    "description": "Sub stream parameter sets, captured separately from the main stream so the\nlive HLS muxer can build a correct init segment when a viewer switches the\nlive view to the sub (low-resolution) stream.",
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
+                "sub_vps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_width": {
                     "type": "integer"
                 },
+                "vps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "width": {
                     "type": "integer"
                 }
@@ -998,12 +1299,18 @@ const docTemplate = `{
                 "directory": {
                     "type": "string"
                 },
+                "max_retries": {
+                    "type": "integer"
+                },
                 "provider": {
                     "type": "string"
                 },
                 "secret_access_key": {
                     "type": "string"
                 },
+                "timeout": {
+                    "type": "integer"
+                },
                 "uri": {
                     "type": "string"
                 }
@@ -1141,6 +1448,17 @@ const docTemplate = `{
                 }
             }
         },
+        "models.Signing": {
+            "type": "object",
+            "properties": {
+                "enabled": {
+                    "type": "string"
+                },
+                "private_key": {
+                    "type": "string"
+                }
+            }
+        },
         "models.Timetable": {
             "type": "object",
             "properties": {

machinery/docs/swagger.json

@@ -17,6 +17,38 @@
     },
     "basePath": "/",
     "paths": {
+        "/api/camera/discover": {
+            "get": {
+                "description": "Runs an advanced Fing/WiFiman-style scan (ONVIF WS-Discovery + TCP port scan + MAC/vendor lookup) and returns the devices found on the local network.",
+                "tags": [
+                    "onvif"
+                ],
+                "summary": "Discover cameras and other devices on the local network.",
+                "operationId": "camera-discover",
+                "parameters": [
+                    {
+                        "type": "integer",
+                        "description": "Discovery timeout in milliseconds (default 2000)",
+                        "name": "timeout",
+                        "in": "query"
+                    },
+                    {
+                        "type": "string",
+                        "description": "Optional subnet(s) to scan, e.g. '192.168.1.0/24' (comma-separated). Defaults to the local interfaces.",
+                        "name": "subnet",
+                        "in": "query"
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/models.APIResponse"
+                        }
+                    }
+                }
+            }
+        },
         "/api/camera/onvif/capabilities": {
             "post": {
                 "description": "Will return the ONVIF capabilities for the specific camera.",
@@ -616,6 +648,40 @@
                 }
             }
         },
+        "/api/persistence/secondary/verify": {
+            "post": {
+                "security": [
+                    {
+                        "Bearer": []
+                    }
+                ],
+                "description": "Will verify the secondary persistence.",
+                "tags": [
+                    "persistence"
+                ],
+                "summary": "Will verify the secondary persistence.",
+                "operationId": "verify-secondary-persistence",
+                "parameters": [
+                    {
+                        "description": "Config",
+                        "name": "config",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/models.Config"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/models.APIResponse"
+                        }
+                    }
+                }
+            }
+        },
         "/api/persistence/verify": {
             "post": {
                 "security": [
@@ -649,9 +715,151 @@
                     }
                 }
             }
+        },
+        "/health": {
+            "get": {
+                "description": "Confirms that the Agent HTTP process can serve requests and reports current camera stream and Hub heartbeat diagnostics. Operational dependency failures do not change the liveness HTTP status.",
+                "produces": [
+                    "application/json"
+                ],
+                "tags": [
+                    "health"
+                ],
+                "summary": "Check Agent health",
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/http.HealthResponse"
+                        }
+                    }
+                }
+            }
         }
     },
     "definitions": {
+        "http.Health": {
+            "type": "object",
+            "properties": {
+                "cameraConnected": {
+                    "type": "boolean"
+                },
+                "description": {
+                    "type": "string"
+                },
+                "hub": {
+                    "$ref": "#/definitions/http.HubHealth"
+                },
+                "mainStream": {
+                    "$ref": "#/definitions/http.StreamHealth"
+                },
+                "subStream": {
+                    "$ref": "#/definitions/http.StreamHealth"
+                }
+            }
+        },
+        "http.HealthResponse": {
+            "type": "object",
+            "properties": {
+                "applicationStatusCode": {
+                    "type": "string"
+                },
+                "data": {
+                    "$ref": "#/definitions/http.HealthResponseData"
+                },
+                "entityStatusCode": {
+                    "type": "string"
+                },
+                "httpStatusCode": {
+                    "type": "integer"
+                },
+                "message": {
+                    "type": "string"
+                },
+                "metadata": {
+                    "$ref": "#/definitions/http.ResponseMetadata"
+                }
+            }
+        },
+        "http.HealthResponseData": {
+            "type": "object",
+            "properties": {
+                "health": {
+                    "$ref": "#/definitions/http.Health"
+                }
+            }
+        },
+        "http.HubHealth": {
+            "type": "object",
+            "properties": {
+                "configured": {
+                    "type": "boolean"
+                },
+                "connected": {
+                    "type": "boolean"
+                },
+                "lastHeartbeatAttemptAt": {
+                    "type": "integer"
+                },
+                "lastSuccessfulHeartbeatAt": {
+                    "type": "integer"
+                }
+            }
+        },
+        "http.ResponseMetadata": {
+            "type": "object",
+            "properties": {
+                "applicationName": {
+                    "type": "string"
+                },
+                "applicationVersion": {
+                    "type": "string"
+                },
+                "path": {
+                    "type": "string"
+                },
+                "timestamp": {
+                    "type": "integer"
+                },
+                "traceId": {
+                    "type": "string"
+                }
+            }
+        },
+        "http.StreamHealth": {
+            "type": "object",
+            "properties": {
+                "configured": {
+                    "type": "boolean"
+                },
+                "connected": {
+                    "type": "boolean"
+                },
+                "fps": {
+                    "type": "number"
+                },
+                "lastPacketAt": {
+                    "type": "integer"
+                },
+                "packagesProcessed": {
+                    "type": "integer"
+                },
+                "resolution": {
+                    "$ref": "#/definitions/http.StreamResolution"
+                }
+            }
+        },
+        "http.StreamResolution": {
+            "type": "object",
+            "properties": {
+                "height": {
+                    "type": "integer"
+                },
+                "width": {
+                    "type": "integer"
+                }
+            }
+        },
         "models.APIResponse": {
             "type": "object",
             "properties": {
@@ -723,12 +931,19 @@
                 "fragmentedduration": {
                     "type": "integer"
                 },
+                "gopsize": {
+                    "description": "GOP size in seconds, used for pre-recording",
+                    "type": "integer"
+                },
                 "ipcamera": {
                     "$ref": "#/definitions/models.IPCamera"
                 },
                 "liveview": {
                     "type": "string"
                 },
+                "liveview_chunking": {
+                    "type": "string"
+                },
                 "maxlengthrecording": {
                     "type": "integer"
                 },
@@ -738,6 +953,10 @@
                 "name": {
                     "type": "string"
                 },
+                "onvif_motion": {
+                    "description": "ONVIFMotion routes the camera's ONVIF motion events into the\nagent's motion-triggered recording pipeline. When \"true\" the\nagent opens an event/stream against the configured ONVIF\nendpoint and forwards Motion+Active events to HandleMotion.\nRequires Capture.IPCamera.ONVIFXAddr / ONVIFUsername /\nONVIFPassword to be set. Default empty (disabled) keeps the\nexisting pixel-diff motion detection as the only source.",
+                    "type": "string"
+                },
                 "pixelChangeThreshold": {
                     "type": "integer"
                 },
@@ -816,9 +1035,15 @@
                 "kstorage": {
                     "$ref": "#/definitions/models.KStorage"
                 },
+                "kstorage_secondary": {
+                    "$ref": "#/definitions/models.KStorage"
+                },
                 "max_directory_size": {
                     "type": "integer"
                 },
+                "min_free_space": {
+                    "type": "integer"
+                },
                 "mqtt_password": {
                     "type": "string"
                 },
@@ -849,6 +1074,9 @@
                 "s3": {
                     "$ref": "#/definitions/models.S3"
                 },
+                "signing": {
+                    "$ref": "#/definitions/models.Signing"
+                },
                 "stunuri": {
                     "type": "string"
                 },
@@ -940,6 +1168,15 @@
         "models.IPCamera": {
             "type": "object",
             "properties": {
+                "base_height": {
+                    "type": "integer"
+                },
+                "base_width": {
+                    "type": "integer"
+                },
+                "channels": {
+                    "type": "integer"
+                },
                 "fps": {
                     "type": "string"
                 },
@@ -958,21 +1195,85 @@
                 "onvif_xaddr": {
                     "type": "string"
                 },
+                "pps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "rtsp": {
                     "type": "string"
                 },
+                "sample_rate": {
+                    "type": "integer"
+                },
+                "sps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_fps": {
                     "type": "string"
                 },
                 "sub_height": {
                     "type": "integer"
                 },
+                "sub_pps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_rtsp": {
                     "type": "string"
                 },
+                "sub_sps_nalus": {
+                    "description": "Sub stream parameter sets, captured separately from the main stream so the\nlive HLS muxer can build a correct init segment when a viewer switches the\nlive view to the sub (low-resolution) stream.",
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
+                "sub_vps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_width": {
                     "type": "integer"
                 },
+                "vps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "width": {
                     "type": "integer"
                 }
@@ -991,12 +1292,18 @@
                 "directory": {
                     "type": "string"
                 },
+                "max_retries": {
+                    "type": "integer"
+                },
                 "provider": {
                     "type": "string"
                 },
                 "secret_access_key": {
                     "type": "string"
                 },
+                "timeout": {
+                    "type": "integer"
+                },
                 "uri": {
                     "type": "string"
                 }
@@ -1134,6 +1441,17 @@
                 }
             }
         },
+        "models.Signing": {
+            "type": "object",
+            "properties": {
+                "enabled": {
+                    "type": "string"
+                },
+                "private_key": {
+                    "type": "string"
+                }
+            }
+        },
         "models.Timetable": {
             "type": "object",
             "properties": {

machinery/docs/swagger.yaml

@@ -1,5 +1,84 @@
 basePath: /
 definitions:
+  http.Health:
+    properties:
+      cameraConnected:
+        type: boolean
+      description:
+        type: string
+      hub:
+        $ref: '#/definitions/http.HubHealth'
+      mainStream:
+        $ref: '#/definitions/http.StreamHealth'
+      subStream:
+        $ref: '#/definitions/http.StreamHealth'
+    type: object
+  http.HealthResponse:
+    properties:
+      applicationStatusCode:
+        type: string
+      data:
+        $ref: '#/definitions/http.HealthResponseData'
+      entityStatusCode:
+        type: string
+      httpStatusCode:
+        type: integer
+      message:
+        type: string
+      metadata:
+        $ref: '#/definitions/http.ResponseMetadata'
+    type: object
+  http.HealthResponseData:
+    properties:
+      health:
+        $ref: '#/definitions/http.Health'
+    type: object
+  http.HubHealth:
+    properties:
+      configured:
+        type: boolean
+      connected:
+        type: boolean
+      lastHeartbeatAttemptAt:
+        type: integer
+      lastSuccessfulHeartbeatAt:
+        type: integer
+    type: object
+  http.ResponseMetadata:
+    properties:
+      applicationName:
+        type: string
+      applicationVersion:
+        type: string
+      path:
+        type: string
+      timestamp:
+        type: integer
+      traceId:
+        type: string
+    type: object
+  http.StreamHealth:
+    properties:
+      configured:
+        type: boolean
+      connected:
+        type: boolean
+      fps:
+        type: number
+      lastPacketAt:
+        type: integer
+      packagesProcessed:
+        type: integer
+      resolution:
+        $ref: '#/definitions/http.StreamResolution'
+    type: object
+  http.StreamResolution:
+    properties:
+      height:
+        type: integer
+      width:
+        type: integer
+    type: object
   models.APIResponse:
     properties:
       can_pan_tilt:
@@ -47,16 +126,31 @@ definitions:
         type: string
       fragmentedduration:
         type: integer
+      gopsize:
+        description: GOP size in seconds, used for pre-recording
+        type: integer
       ipcamera:
         $ref: '#/definitions/models.IPCamera'
       liveview:
         type: string
+      liveview_chunking:
+        type: string
       maxlengthrecording:
         type: integer
       motion:
         type: string
       name:
         type: string
+      onvif_motion:
+        description: |-
+          ONVIFMotion routes the camera's ONVIF motion events into the
+          agent's motion-triggered recording pipeline. When "true" the
+          agent opens an event/stream against the configured ONVIF
+          endpoint and forwards Motion+Active events to HandleMotion.
+          Requires Capture.IPCamera.ONVIFXAddr / ONVIFUsername /
+          ONVIFPassword to be set. Default empty (disabled) keeps the
+          existing pixel-diff motion detection as the only source.
+        type: string
       pixelChangeThreshold:
         type: integer
       postrecording:
@@ -109,8 +203,12 @@ definitions:
         type: string
       kstorage:
         $ref: '#/definitions/models.KStorage'
+      kstorage_secondary:
+        $ref: '#/definitions/models.KStorage'
       max_directory_size:
         type: integer
+      min_free_space:
+        type: integer
       mqtt_password:
         type: string
       mqtt_username:
@@ -131,6 +229,8 @@ definitions:
         type: string
       s3:
         $ref: '#/definitions/models.S3'
+      signing:
+        $ref: '#/definitions/models.Signing'
       stunuri:
         type: string
       time:
@@ -190,6 +290,12 @@ definitions:
     type: object
   models.IPCamera:
     properties:
+      base_height:
+        type: integer
+      base_width:
+        type: integer
+      channels:
+        type: integer
       fps:
         type: string
       height:
@@ -202,16 +308,64 @@ definitions:
         type: string
       onvif_xaddr:
         type: string
+      pps_nalus:
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
       rtsp:
         type: string
+      sample_rate:
+        type: integer
+      sps_nalus:
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
       sub_fps:
         type: string
       sub_height:
         type: integer
+      sub_pps_nalus:
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
       sub_rtsp:
         type: string
+      sub_sps_nalus:
+        description: |-
+          Sub stream parameter sets, captured separately from the main stream so the
+          live HLS muxer can build a correct init segment when a viewer switches the
+          live view to the sub (low-resolution) stream.
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
+      sub_vps_nalus:
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
       sub_width:
         type: integer
+      vps_nalus:
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
       width:
         type: integer
     type: object
@@ -224,10 +378,14 @@ definitions:
         type: string
       directory:
         type: string
+      max_retries:
+        type: integer
       provider:
         type: string
       secret_access_key:
         type: string
+      timeout:
+        type: integer
       uri:
         type: string
     type: object
@@ -316,6 +474,13 @@ definitions:
       username:
         type: string
     type: object
+  models.Signing:
+    properties:
+      enabled:
+        type: string
+      private_key:
+        type: string
+    type: object
   models.Timetable:
     properties:
       end1:
@@ -345,6 +510,30 @@ info:
   title: Swagger Kerberos Agent API
   version: "1.0"
 paths:
+  /api/camera/discover:
+    get:
+      description: Runs an advanced Fing/WiFiman-style scan (ONVIF WS-Discovery +
+        TCP port scan + MAC/vendor lookup) and returns the devices found on the local
+        network.
+      operationId: camera-discover
+      parameters:
+      - description: Discovery timeout in milliseconds (default 2000)
+        in: query
+        name: timeout
+        type: integer
+      - description: Optional subnet(s) to scan, e.g. '192.168.1.0/24' (comma-separated).
+          Defaults to the local interfaces.
+        in: query
+        name: subnet
+        type: string
+      responses:
+        "200":
+          description: OK
+          schema:
+            $ref: '#/definitions/models.APIResponse'
+      summary: Discover cameras and other devices on the local network.
+      tags:
+      - onvif
   /api/camera/onvif/capabilities:
     post:
       description: Will return the ONVIF capabilities for the specific camera.
@@ -735,6 +924,27 @@ paths:
       summary: Get Authorization token.
       tags:
       - authentication
+  /api/persistence/secondary/verify:
+    post:
+      description: Will verify the secondary persistence.
+      operationId: verify-secondary-persistence
+      parameters:
+      - description: Config
+        in: body
+        name: config
+        required: true
+        schema:
+          $ref: '#/definitions/models.Config'
+      responses:
+        "200":
+          description: OK
+          schema:
+            $ref: '#/definitions/models.APIResponse'
+      security:
+      - Bearer: []
+      summary: Will verify the secondary persistence.
+      tags:
+      - persistence
   /api/persistence/verify:
     post:
       description: Will verify the persistence.
@@ -756,6 +966,21 @@ paths:
       summary: Will verify the persistence.
       tags:
       - persistence
+  /health:
+    get:
+      description: Confirms that the Agent HTTP process can serve requests and reports
+        current camera stream and Hub heartbeat diagnostics. Operational dependency
+        failures do not change the liveness HTTP status.
+      produces:
+      - application/json
+      responses:
+        "200":
+          description: OK
+          schema:
+            $ref: '#/definitions/http.HealthResponse'
+      summary: Check Agent health
+      tags:
+      - health
 securityDefinitions:
   Bearer:
     in: header

machinery/go.mod

@@ -1,6 +1,6 @@
 module github.com/kerberos-io/agent/machinery
 
-go 1.24.2
+go 1.25.0
 
 replace google.golang.org/genproto => google.golang.org/genproto v0.0.0-20250519155744-55703ea1f237
 
@@ -8,9 +8,10 @@ require (
 	github.com/Eyevinn/mp4ff v0.48.0
 	github.com/InVisionApp/conjungo v1.1.0
 	github.com/appleboy/gin-jwt/v2 v2.10.3
-	github.com/bluenviron/gortsplib/v4 v4.14.1
+	github.com/bluenviron/gortsplib/v5 v5.6.3
 	github.com/bluenviron/mediacommon v1.14.0
 	github.com/cedricve/go-onvif v0.0.0-20200222191200-567e8ce298f6
+	github.com/creack/pty v1.1.24
 	github.com/dromara/carbon/v2 v2.6.8
 	github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5
 	github.com/eclipse/paho.mqtt.golang v1.5.0
@@ -28,12 +29,11 @@ require (
 	github.com/minio/minio-go/v6 v6.0.57
 	github.com/moq-dev/moq-go v0.5.7
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
-	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
-	github.com/pion/interceptor v0.1.40
-	github.com/pion/rtp v1.8.19
-	github.com/pion/webrtc/v4 v4.1.2
+	github.com/pion/interceptor v0.1.47
+	github.com/pion/rtp v1.10.5
+	github.com/pion/webrtc/v4 v4.2.18
 	github.com/sirupsen/logrus v1.9.3
-	github.com/stretchr/testify v1.10.0
+	github.com/stretchr/testify v1.11.1
 	github.com/swaggo/files v1.0.1
 	github.com/swaggo/gin-swagger v1.6.0
 	github.com/swaggo/swag v1.16.4
@@ -45,7 +45,6 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.36.0
 	go.opentelemetry.io/otel/sdk v1.36.0
 	go.opentelemetry.io/otel/trace v1.36.0
-	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 )
 
 require (
@@ -53,7 +52,7 @@ require (
 	github.com/PuerkitoBio/purell v1.1.1 // indirect
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/beevik/etree v1.2.0 // indirect
-	github.com/bluenviron/mediacommon/v2 v2.2.0 // indirect
+	github.com/bluenviron/mediacommon/v2 v2.9.2 // indirect
 	github.com/bytedance/sonic v1.13.2 // indirect
 	github.com/bytedance/sonic/loader v0.2.4 // indirect
 	github.com/cenkalti/backoff/v5 v5.0.2 // indirect
@@ -99,19 +98,19 @@ require (
 	github.com/moq-dev/moq-go-ffi v0.3.7 // indirect
 	github.com/nxadm/tail v1.4.11 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
-	github.com/pion/datachannel v1.5.10 // indirect
-	github.com/pion/dtls/v3 v3.0.6 // indirect
-	github.com/pion/ice/v4 v4.0.10 // indirect
-	github.com/pion/logging v0.2.3 // indirect
-	github.com/pion/mdns/v2 v2.0.7 // indirect
+	github.com/pion/datachannel v1.6.2 // indirect
+	github.com/pion/dtls/v3 v3.1.5 // indirect
+	github.com/pion/ice/v4 v4.4.0 // indirect
+	github.com/pion/logging v0.2.4 // indirect
+	github.com/pion/mdns/v2 v2.1.0 // indirect
 	github.com/pion/randutil v0.1.0 // indirect
-	github.com/pion/rtcp v1.2.15 // indirect
-	github.com/pion/sctp v1.8.39 // indirect
-	github.com/pion/sdp/v3 v3.0.13 // indirect
-	github.com/pion/srtp/v3 v3.0.5 // indirect
-	github.com/pion/stun/v3 v3.0.0 // indirect
-	github.com/pion/transport/v3 v3.0.7 // indirect
-	github.com/pion/turn/v4 v4.0.0 // indirect
+	github.com/pion/rtcp v1.2.17 // indirect
+	github.com/pion/sctp v1.11.1 // indirect
+	github.com/pion/sdp/v3 v3.0.19 // indirect
+	github.com/pion/srtp/v3 v3.0.12 // indirect
+	github.com/pion/stun/v3 v3.1.6 // indirect
+	github.com/pion/transport/v4 v4.0.2 // indirect
+	github.com/pion/turn/v5 v5.0.12 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
@@ -126,13 +125,14 @@ require (
 	go.opentelemetry.io/otel/metric v1.36.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.6.0 // indirect
 	golang.org/x/arch v0.16.0 // indirect
-	golang.org/x/crypto v0.38.0 // indirect
-	golang.org/x/net v0.40.0 // indirect
+	golang.org/x/crypto v0.54.0 // indirect
+	golang.org/x/net v0.57.0 // indirect
 	golang.org/x/oauth2 v0.30.0 // indirect
-	golang.org/x/sync v0.14.0 // indirect
-	golang.org/x/sys v0.33.0 // indirect
-	golang.org/x/text v0.25.0 // indirect
-	golang.org/x/tools v0.30.0 // indirect
+	golang.org/x/sync v0.22.0 // indirect
+	golang.org/x/sys v0.47.0 // indirect
+	golang.org/x/text v0.40.0 // indirect
+	golang.org/x/time v0.14.0 // indirect
+	golang.org/x/tools v0.47.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20250519155744-55703ea1f237 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237 // indirect
 	google.golang.org/grpc v1.72.1 // indirect

machinery/go.sum

@@ -393,12 +393,12 @@ github.com/appleboy/gofight/v2 v2.1.2/go.mod h1:frW+U1QZEdDgixycTj4CygQ48yLTUhpl
 github.com/bazelbuild/rules_go v0.49.0/go.mod h1:Dhcz716Kqg1RHNWos+N6MlXNkjNP2EwZQ0LukRKJfMs=
 github.com/beevik/etree v1.2.0 h1:l7WETslUG/T+xOPs47dtd6jov2Ii/8/OjCldk5fYfQw=
 github.com/beevik/etree v1.2.0/go.mod h1:aiPf89g/1k3AShMVAzriilpcE4R/Vuor90y83zVZWFc=
-github.com/bluenviron/gortsplib/v4 v4.14.1 h1:v99NmXeeJFfbrO+ipPzPxYGibQaR5ZOUESOA9UQZhsI=
-github.com/bluenviron/gortsplib/v4 v4.14.1/go.mod h1:3LaEcg0d47+kfXju5KSlsSxCiZ3IKBI/sqIrBPcsS64=
+github.com/bluenviron/gortsplib/v5 v5.6.3 h1:OXvHthQZ9fZbLh6r3Go2wuF4XQ4/QW4WTIM2f4bv/W4=
+github.com/bluenviron/gortsplib/v5 v5.6.3/go.mod h1:kzHgUtvl8NWNsQ5Vsez6Vuugk6ItFT4ByCPm8J/kbSQ=
 github.com/bluenviron/mediacommon v1.14.0 h1:lWCwOBKNKgqmspRpwpvvg3CidYm+XOc2+z/Jw7LM5dQ=
 github.com/bluenviron/mediacommon v1.14.0/go.mod h1:z5LP9Tm1ZNfQV5Co54PyOzaIhGMusDfRKmh42nQSnyo=
-github.com/bluenviron/mediacommon/v2 v2.2.0 h1:fGXEX0OEvv5VhGHOv3Q2ABzOtSkIpl9UbwOHrnKWNTk=
-github.com/bluenviron/mediacommon/v2 v2.2.0/go.mod h1:a6MbPmXtYda9mKibKVMZlW20GYLLrX2R7ZkUE+1pwV0=
+github.com/bluenviron/mediacommon/v2 v2.9.2 h1:jvYeBjvhHKFOBRMTMm4hvrSjyOlCelkOkx6708DidQM=
+github.com/bluenviron/mediacommon/v2 v2.9.2/go.mod h1:jMf/OJDaJl02xRgkLM2zbidUHnDYqLnO1dMMveCmyyU=
 github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
 github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
 github.com/bytedance/sonic v1.13.2 h1:8/H1FempDZqC4VqjptGo14QQlJx8VdZJegxs6wwfqpQ=
@@ -456,6 +456,8 @@ github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78/go.mod h1:W+zGtBO5Y1Ig
 github.com/cncf/xds/go v0.0.0-20241223141626-cff3c89139a3/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
 github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
+github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -858,46 +860,46 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
 github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE=
 github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
-github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
-github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
 github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
 github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY=
 github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI=
 github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI=
 github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
-github.com/pion/datachannel v1.5.10 h1:ly0Q26K1i6ZkGf42W7D4hQYR90pZwzFOjTq5AuCKk4o=
-github.com/pion/datachannel v1.5.10/go.mod h1:p/jJfC9arb29W7WrxyKbepTU20CFgyx5oLo8Rs4Py/M=
-github.com/pion/dtls/v3 v3.0.6 h1:7Hkd8WhAJNbRgq9RgdNh1aaWlZlGpYTzdqjy9x9sK2E=
-github.com/pion/dtls/v3 v3.0.6/go.mod h1:iJxNQ3Uhn1NZWOMWlLxEEHAN5yX7GyPvvKw04v9bzYU=
-github.com/pion/ice/v4 v4.0.10 h1:P59w1iauC/wPk9PdY8Vjl4fOFL5B+USq1+xbDcN6gT4=
-github.com/pion/ice/v4 v4.0.10/go.mod h1:y3M18aPhIxLlcO/4dn9X8LzLLSma84cx6emMSu14FGw=
-github.com/pion/interceptor v0.1.40 h1:e0BjnPcGpr2CFQgKhrQisBU7V3GXK6wrfYrGYaU6Jq4=
-github.com/pion/interceptor v0.1.40/go.mod h1:Z6kqH7M/FYirg3frjGJ21VLSRJGBXB/KqaTIrdqnOic=
-github.com/pion/logging v0.2.3 h1:gHuf0zpoh1GW67Nr6Gj4cv5Z9ZscU7g/EaoC/Ke/igI=
-github.com/pion/logging v0.2.3/go.mod h1:z8YfknkquMe1csOrxK5kc+5/ZPAzMxbKLX5aXpbpC90=
-github.com/pion/mdns/v2 v2.0.7 h1:c9kM8ewCgjslaAmicYMFQIde2H9/lrZpjBkN8VwoVtM=
-github.com/pion/mdns/v2 v2.0.7/go.mod h1:vAdSYNAT0Jy3Ru0zl2YiW3Rm/fJCwIeM0nToenfOJKA=
+github.com/pion/datachannel v1.6.2 h1:7EXQ8TH3vTouBUdRWYbcX2edSx9Yj6k5zl5P+qyxEPc=
+github.com/pion/datachannel v1.6.2/go.mod h1:pzbdAZvyGtXbcHM1hBbsFaOTf40lZizU/dNlvVOak6E=
+github.com/pion/dtls/v3 v3.1.5 h1:9xJtVsHwMYeSjPp5Hh1FTis4DchnQWtnOa5o+6ygqfc=
+github.com/pion/dtls/v3 v3.1.5/go.mod h1:gz1K4jg6c+fq86oQMH4pilpCEOEPwmEr2jY+VcF/mkU=
+github.com/pion/ice/v4 v4.4.0 h1:wvHDDqimaC38Y7MVpD46Y63p246ChvXd87VKoLYS5b4=
+github.com/pion/ice/v4 v4.4.0/go.mod h1:obAyD+J+Hzs7QA7Y8YXHp5uIn6gb7z87pKedXZkrcFU=
+github.com/pion/interceptor v0.1.47 h1:yw8t5pJ2f8t78NgU+8EmxhaqYLXS7uFCC/tAGOaSDBo=
+github.com/pion/interceptor v0.1.47/go.mod h1:7yoRBzaIDETPC6cIN8Zj9EyGqHv1ImOpcTFPha6MuOM=
+github.com/pion/logging v0.2.4 h1:tTew+7cmQ+Mc1pTBLKH2puKsOvhm32dROumOZ655zB8=
+github.com/pion/logging v0.2.4/go.mod h1:DffhXTKYdNZU+KtJ5pyQDjvOAh/GsNSyv1lbkFbe3so=
+github.com/pion/mdns/v2 v2.1.0 h1:3IJ9+Xio6tWYjhN6WwuY142P/1jA0D5ERaIqawg/fOY=
+github.com/pion/mdns/v2 v2.1.0/go.mod h1:pcez23GdynwcfRU1977qKU0mDxSeucttSHbCSfFOd9A=
 github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA=
 github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8=
-github.com/pion/rtcp v1.2.15 h1:LZQi2JbdipLOj4eBjK4wlVoQWfrZbh3Q6eHtWtJBZBo=
-github.com/pion/rtcp v1.2.15/go.mod h1:jlGuAjHMEXwMUHK78RgX0UmEJFV4zUKOFHR7OP+D3D0=
-github.com/pion/rtp v1.8.19 h1:jhdO/3XhL/aKm/wARFVmvTfq0lC/CvN1xwYKmduly3c=
-github.com/pion/rtp v1.8.19/go.mod h1:bAu2UFKScgzyFqvUKmbvzSdPr+NGbZtv6UB2hesqXBk=
-github.com/pion/sctp v1.8.39 h1:PJma40vRHa3UTO3C4MyeJDQ+KIobVYRZQZ0Nt7SjQnE=
-github.com/pion/sctp v1.8.39/go.mod h1:cNiLdchXra8fHQwmIoqw0MbLLMs+f7uQ+dGMG2gWebE=
-github.com/pion/sdp/v3 v3.0.13 h1:uN3SS2b+QDZnWXgdr69SM8KB4EbcnPnPf2Laxhty/l4=
-github.com/pion/sdp/v3 v3.0.13/go.mod h1:88GMahN5xnScv1hIMTqLdu/cOcUkj6a9ytbncwMCq2E=
-github.com/pion/srtp/v3 v3.0.5 h1:8XLB6Dt3QXkMkRFpoqC3314BemkpMQK2mZeJc4pUKqo=
-github.com/pion/srtp/v3 v3.0.5/go.mod h1:r1G7y5r1scZRLe2QJI/is+/O83W2d+JoEsuIexpw+uM=
-github.com/pion/stun/v3 v3.0.0 h1:4h1gwhWLWuZWOJIJR9s2ferRO+W3zA/b6ijOI6mKzUw=
-github.com/pion/stun/v3 v3.0.0/go.mod h1:HvCN8txt8mwi4FBvS3EmDghW6aQJ24T+y+1TKjB5jyU=
-github.com/pion/transport/v3 v3.0.7 h1:iRbMH05BzSNwhILHoBoAPxoB9xQgOaJk+591KC9P1o0=
-github.com/pion/transport/v3 v3.0.7/go.mod h1:YleKiTZ4vqNxVwh77Z0zytYi7rXHl7j6uPLGhhz9rwo=
-github.com/pion/turn/v4 v4.0.0 h1:qxplo3Rxa9Yg1xXDxxH8xaqcyGUtbHYw4QSCvmFWvhM=
-github.com/pion/turn/v4 v4.0.0/go.mod h1:MuPDkm15nYSklKpN8vWJ9W2M0PlyQZqYt1McGuxG7mA=
-github.com/pion/webrtc/v4 v4.1.2 h1:mpuUo/EJ1zMNKGE79fAdYNFZBX790KE7kQQpLMjjR54=
-github.com/pion/webrtc/v4 v4.1.2/go.mod h1:xsCXiNAmMEjIdFxAYU0MbB3RwRieJsegSB2JZsGN+8U=
+github.com/pion/rtcp v1.2.17 h1:PxiT6L79yPZKtXIsXdG1eakBl6dtBj4x+4oVEL0DlSw=
+github.com/pion/rtcp v1.2.17/go.mod h1:7kBpuBJaWwax4hzc/pgexY8vkOpvh8atgYDbaKZq0iU=
+github.com/pion/rtp v1.10.5 h1:ip0HhO/wYZqQ4bKS+R99KnZh/GRCmIT0jDXikub7vlE=
+github.com/pion/rtp v1.10.5/go.mod h1:Au8fc6cEByy8RLTwKTQTEeQqDB/SJDxwL4mZuxYA5Pk=
+github.com/pion/sctp v1.11.1 h1:O4dIFyURw1KTST7w+gtD4gLeYXkhPa0xXLHMMoe/OSA=
+github.com/pion/sctp v1.11.1/go.mod h1:7KFmTwLcoYgJs/Z+99nJvsWL0qDpuyloSI0RbAqlrz0=
+github.com/pion/sdp/v3 v3.0.19 h1:1VMKs3gIkTQV5M3hNKfTAPrDXSNrYtOlmOD8+mSZUGQ=
+github.com/pion/sdp/v3 v3.0.19/go.mod h1:dE5WOSlzXrtiE/iuZqe9n+AcEbOjtAd3k5m5NtlV/qU=
+github.com/pion/srtp/v3 v3.0.12 h1:U7V17bckl7sI4mb3sepiojByDuBY0wNCqQE+6IlQBbc=
+github.com/pion/srtp/v3 v3.0.12/go.mod h1:EeZOi/sd6glM1EXapg051gdNWO9yWT1YSsgQ4SlJkns=
+github.com/pion/stun/v3 v3.1.6 h1:WnhsD0eHCiwCfKNkVx0VJJwr2Y3eV4Ueih3KJ+dfZy8=
+github.com/pion/stun/v3 v3.1.6/go.mod h1:zRUghXSQU32Lx5orJsz3uYMkIihweXb3mu5gIns02fs=
+github.com/pion/transport/v3 v3.1.1 h1:Tr684+fnnKlhPceU+ICdrw6KKkTms+5qHMgw6bIkYOM=
+github.com/pion/transport/v3 v3.1.1/go.mod h1:+c2eewC5WJQHiAA46fkMMzoYZSuGzA/7E2FPrOYHctQ=
+github.com/pion/transport/v4 v4.0.2 h1:ifYlPqNwsy6aKQ9y8yzxXlHae5431ZrH2avkD/Rn6Tk=
+github.com/pion/transport/v4 v4.0.2/go.mod h1:06hFI+jCFcok2X2MekVufNZ/uzNZXivGBPfviSVcjgM=
+github.com/pion/turn/v5 v5.0.12 h1:6+b69ivQQXSlyfkp2AKripqD2k3W32qXK8QzCzpJWPI=
+github.com/pion/turn/v5 v5.0.12/go.mod h1:CQACsRDJtjQ+6RSrGHrS2PCIerLwbW3uqXRqOvtjAFg=
+github.com/pion/webrtc/v4 v4.2.18 h1:smA/3g6Gy4RohM0VIZ5KKY/12TQbxv3XFgpUMyb2EUI=
+github.com/pion/webrtc/v4 v4.2.18/go.mod h1:vmzi6s+rvhoIuT94DPqivB+0xJXs9rG4QRD+4MgBtlY=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -957,8 +959,9 @@ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o
 github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 github.com/substrait-io/substrait-go v0.4.2/go.mod h1:qhpnLmrcvAnlZsUyPXZRqldiHapPTXC3t7xFgDi3aQg=
 github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE=
 github.com/swaggo/files v1.0.1/go.mod h1:0qXmMNH6sXNf+73t65aKeB+ApmgxdnkQzVTAj2uaMUg=
@@ -1166,8 +1169,8 @@ golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5
 golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ=
 golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=
 golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
-golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8=
-golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw=
+golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
+golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -1236,8 +1239,9 @@ golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM=
 golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
+golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
+golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1325,8 +1329,8 @@ golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
 golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk=
 golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
 golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E=
-golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY=
-golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds=
+golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
+golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1405,8 +1409,9 @@ golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
-golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ=
 golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
+golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1515,8 +1520,8 @@ golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
-golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
-golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
+golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
 golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -1583,8 +1588,8 @@ golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
 golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
 golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
 golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
-golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
-golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
+golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
+golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1596,6 +1601,8 @@ golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
+golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
+golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -1669,8 +1676,9 @@ golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk
 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
 golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c=
 golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI=
-golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY=
 golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY=
+golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
+golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -2098,8 +2106,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk=
 gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
-gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

machinery/logging.go

@@ -0,0 +1,160 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	defaultLogLevel  = log.InfoLevel
+	defaultLogOutput = "text"
+)
+
+type localTimeFormatter struct {
+	timezone  *time.Location
+	formatter log.Formatter
+}
+
+func (f localTimeFormatter) Format(entry *log.Entry) ([]byte, error) {
+	entry.Time = entry.Time.In(f.timezone)
+	return f.formatter.Format(entry)
+}
+
+type componentHook struct{}
+
+func (componentHook) Levels() []log.Level {
+	return log.AllLevels
+}
+
+func (componentHook) Fire(entry *log.Entry) error {
+	if _, exists := entry.Data["component"]; exists {
+		return nil
+	}
+	entry.Data["component"] = componentFromCaller(entry.Caller)
+	return nil
+}
+
+func configureLogging(levelValue string, outputValue string, timezone *time.Location) {
+	configureLogger(log.StandardLogger(), levelValue, outputValue, timezone)
+}
+
+func configureLogger(logger *log.Logger, levelValue string, outputValue string, timezone *time.Location) {
+	if timezone == nil {
+		timezone = time.Local
+	}
+
+	level, levelErr := parseLogLevel(levelValue)
+	output, outputErr := parseLogOutput(outputValue)
+
+	logger.SetOutput(os.Stdout)
+	logger.SetLevel(level)
+	logger.SetReportCaller(true)
+	logger.SetFormatter(localTimeFormatter{
+		timezone:  timezone,
+		formatter: newLogFormatter(output),
+	})
+	installComponentHook(logger)
+
+	if levelErr != nil {
+		logger.WithFields(log.Fields{
+			"configured_level": levelValue,
+			"effective_level":  level.String(),
+		}).WithError(levelErr).Warn("invalid log level; using default")
+	}
+	if outputErr != nil {
+		logger.WithFields(log.Fields{
+			"configured_output": outputValue,
+			"effective_output":  output,
+		}).WithError(outputErr).Warn("invalid log output; using default")
+	}
+
+	logger.WithFields(log.Fields{
+		"event":         "logger_configured",
+		"log_level":     level.String(),
+		"output":        output,
+		"report_caller": logger.ReportCaller,
+		"timezone":      timezone.String(),
+	}).Debug("logging configured")
+}
+
+func installComponentHook(logger *log.Logger) {
+	for _, hooks := range logger.Hooks {
+		for _, hook := range hooks {
+			if _, ok := hook.(componentHook); ok {
+				return
+			}
+		}
+	}
+	logger.AddHook(componentHook{})
+}
+
+func componentFromCaller(frame *runtime.Frame) string {
+	if frame == nil {
+		return "unknown"
+	}
+
+	const sourceMarker = "/machinery/src/"
+	normalizedFile := filepath.ToSlash(frame.File)
+	if markerIndex := strings.Index(normalizedFile, sourceMarker); markerIndex >= 0 {
+		relativeFile := normalizedFile[markerIndex+len(sourceMarker):]
+		if directory := filepath.ToSlash(filepath.Dir(relativeFile)); directory != "." {
+			return directory
+		}
+	}
+	if strings.Contains(normalizedFile, "/machinery/") {
+		return "agent"
+	}
+	return "unknown"
+}
+
+func parseLogLevel(value string) (log.Level, error) {
+	normalized := strings.ToLower(strings.TrimSpace(value))
+	if normalized == "" {
+		return defaultLogLevel, nil
+	}
+	if normalized == "warning" {
+		normalized = "warn"
+	}
+
+	level, err := log.ParseLevel(normalized)
+	if err != nil {
+		return defaultLogLevel, fmt.Errorf("parse LOG_LEVEL: %w", err)
+	}
+	return level, nil
+}
+
+func parseLogOutput(value string) (string, error) {
+	normalized := strings.ToLower(strings.TrimSpace(value))
+	if normalized == "" {
+		return defaultLogOutput, nil
+	}
+	switch normalized {
+	case "json", "text":
+		return normalized, nil
+	default:
+		return defaultLogOutput, fmt.Errorf("unsupported LOG_OUTPUT %q", value)
+	}
+}
+
+func newLogFormatter(output string) log.Formatter {
+	callerPrettyfier := func(frame *runtime.Frame) (string, string) {
+		return filepath.Base(frame.Function), fmt.Sprintf("%s:%d", filepath.Base(frame.File), frame.Line)
+	}
+	if output == "json" {
+		return &log.JSONFormatter{
+			CallerPrettyfier: callerPrettyfier,
+			TimestampFormat:  time.RFC3339Nano,
+		}
+	}
+	return &log.TextFormatter{
+		CallerPrettyfier: callerPrettyfier,
+		FullTimestamp:    true,
+		TimestampFormat:  time.RFC3339Nano,
+	}
+}

machinery/logging_test.go

@@ -0,0 +1,165 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func TestConfigureLoggingDebugIncludesStructuredContext(t *testing.T) {
+	logger := log.New()
+	var output bytes.Buffer
+	configureLogger(logger, "debug", "json", time.UTC)
+	logger.SetOutput(&output)
+	logger.WithField("event", "test_event").Debug("structured debug test")
+
+	lines := strings.Split(strings.TrimSpace(output.String()), "\n")
+	var entry map[string]interface{}
+	if err := json.Unmarshal([]byte(lines[len(lines)-1]), &entry); err != nil {
+		t.Fatalf("decode debug log: %v; output=%q", err, output.String())
+	}
+	for key, want := range map[string]interface{}{
+		"component": "agent",
+		"event":     "test_event",
+		"level":     "debug",
+		"msg":       "structured debug test",
+	} {
+		if got := entry[key]; got != want {
+			t.Fatalf("%s = %v, want %v", key, got, want)
+		}
+	}
+	if entry["file"] == nil || entry["func"] == nil {
+		t.Fatalf("debug log is missing caller metadata: %v", entry)
+	}
+}
+
+func TestConfigureLoggerInstallsComponentHookOnce(t *testing.T) {
+	logger := log.New()
+	configureLogger(logger, "info", "text", time.UTC)
+	configureLogger(logger, "debug", "json", time.UTC)
+
+	var componentHooks int
+	for _, hooks := range logger.Hooks {
+		for _, hook := range hooks {
+			if _, ok := hook.(componentHook); ok {
+				componentHooks++
+			}
+		}
+	}
+	if componentHooks != len(log.AllLevels) {
+		t.Fatalf("component hook registrations = %d, want %d", componentHooks, len(log.AllLevels))
+	}
+}
+
+func TestComponentFromCaller(t *testing.T) {
+	tests := []struct {
+		name  string
+		frame *runtime.Frame
+		want  string
+	}{
+		{
+			name:  "nested runtime package",
+			frame: &runtime.Frame{File: "/workspace/agent/machinery/src/routers/mqtt/main.go"},
+			want:  "routers/mqtt",
+		},
+		{
+			name:  "top-level runtime package",
+			frame: &runtime.Frame{File: "/workspace/agent/machinery/src/capture/main.go"},
+			want:  "capture",
+		},
+		{
+			name:  "executable",
+			frame: &runtime.Frame{File: "/workspace/agent/machinery/main.go"},
+			want:  "agent",
+		},
+		{name: "missing caller", want: "unknown"},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if got := componentFromCaller(test.frame); got != test.want {
+				t.Fatalf("componentFromCaller() = %q, want %q", got, test.want)
+			}
+		})
+	}
+}
+
+func TestComponentHookAddsAndPreservesComponent(t *testing.T) {
+	hook := componentHook{}
+
+	entry := log.NewEntry(log.New())
+	entry.Caller = &runtime.Frame{File: "/workspace/agent/machinery/src/cloud/livehls/session.go"}
+	if err := hook.Fire(entry); err != nil {
+		t.Fatalf("componentHook.Fire() error = %v", err)
+	}
+	if got := entry.Data["component"]; got != "cloud/livehls" {
+		t.Fatalf("component = %v, want cloud/livehls", got)
+	}
+
+	entry.Data["component"] = "explicit"
+	if err := hook.Fire(entry); err != nil {
+		t.Fatalf("componentHook.Fire() preserving field error = %v", err)
+	}
+	if got := entry.Data["component"]; got != "explicit" {
+		t.Fatalf("component = %v, want explicit", got)
+	}
+}
+
+func TestParseLogLevel(t *testing.T) {
+	tests := []struct {
+		name    string
+		value   string
+		want    log.Level
+		wantErr bool
+	}{
+		{name: "default", want: log.InfoLevel},
+		{name: "info", value: "INFO", want: log.InfoLevel},
+		{name: "warning alias", value: "warning", want: log.WarnLevel},
+		{name: "debug", value: "debug", want: log.DebugLevel},
+		{name: "trace", value: "trace", want: log.TraceLevel},
+		{name: "invalid", value: "verbose", want: log.InfoLevel, wantErr: true},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got, err := parseLogLevel(test.value)
+			if (err != nil) != test.wantErr {
+				t.Fatalf("parseLogLevel(%q) error = %v, wantErr %t", test.value, err, test.wantErr)
+			}
+			if got != test.want {
+				t.Fatalf("parseLogLevel(%q) = %s, want %s", test.value, got, test.want)
+			}
+		})
+	}
+}
+
+func TestParseLogOutput(t *testing.T) {
+	tests := []struct {
+		name    string
+		value   string
+		want    string
+		wantErr bool
+	}{
+		{name: "default", want: "text"},
+		{name: "text", value: "TEXT", want: "text"},
+		{name: "json", value: "json", want: "json"},
+		{name: "invalid", value: "console", want: "text", wantErr: true},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got, err := parseLogOutput(test.value)
+			if (err != nil) != test.wantErr {
+				t.Fatalf("parseLogOutput(%q) error = %v, wantErr %t", test.value, err, test.wantErr)
+			}
+			if got != test.want {
+				t.Fatalf("parseLogOutput(%q) = %q, want %q", test.value, got, test.want)
+			}
+		})
+	}
+}

machinery/main.go

@@ -5,14 +5,15 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/capture"
 	"github.com/kerberos-io/agent/machinery/src/components"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/onvif"
+	log "github.com/sirupsen/logrus"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
@@ -28,6 +29,22 @@ import (
 
 var VERSION = utils.VERSION
 
+func resolveServerPort(flagValue, environmentValue string) (string, error) {
+	value := strings.TrimSpace(environmentValue)
+	if value == "" {
+		value = strings.TrimSpace(flagValue)
+	}
+	if value == "" {
+		value = "80"
+	}
+
+	port, err := strconv.Atoi(value)
+	if err != nil || port < 1 || port > 65535 {
+		return "", fmt.Errorf("port must be an integer between 1 and 65535, got %q", value)
+	}
+	return strconv.Itoa(port), nil
+}
+
 func startTracing(agentKey string, otelEndpoint string) (*trace.TracerProvider, error) {
 	serviceName := "agent-" + agentKey
 	headers := map[string]string{
@@ -99,20 +116,44 @@ func main() {
 	}
 	// Specify the timezone of the log: "UTC" or "Local".
 	timezone, _ := time.LoadLocation("CET")
-	log.Log.Init(logLevel, logOutput, configDirectory, timezone)
+	configureLogging(logLevel, logOutput, timezone)
+	if action == "run" {
+		resolvedPort, err := resolveServerPort(port, os.Getenv("AGENT_PORT"))
+		if err != nil {
+			log.WithError(err).WithFields(log.Fields{
+				"component": "http",
+				"event":     "server_port_invalid",
+			}).Fatal("Invalid HTTP server port")
+			return
+		}
+		port = resolvedPort
+	}
+	log.WithFields(log.Fields{
+		"action":           action,
+		"component":        "agent",
+		"config_directory": configDirectory,
+		"event":            "command_parsed",
+		"port":             port,
+		"version":          VERSION,
+	}).Debug("Agent command parsed")
 
 	switch action {
 
 	case "version":
 		{
-			log.Log.Info("main.Main(): You are currrently running Kerberos Agent " + VERSION)
+			log.WithFields(log.Fields{
+				"component": "agent",
+				"event":     "version",
+				"version":   VERSION,
+			}).Info("Kerberos Agent version")
 		}
 	case "discover":
 		{
 			// Convert duration to int
 			timeout, err := time.ParseDuration(timeout + "ms")
 			if err != nil {
-				log.Log.Fatal("main.Main(): could not parse timeout: " + err.Error())
+				log.WithError(err).WithField("component", "onvif").
+					Fatal("invalid ONVIF discovery timeout")
 				return
 			}
 			var subnets []string
@@ -125,15 +166,19 @@ func main() {
 		}
 	case "decrypt":
 		{
-			log.Log.Info("main.Main(): Decrypting: " + flag.Arg(0) + " with key: " + flag.Arg(1))
+			log.WithFields(log.Fields{
+				"component": "encryption",
+				"event":     "decrypt_started",
+				"path":      flag.Arg(0),
+			}).Info("Decrypting recording")
 			symmetricKey := []byte(flag.Arg(1))
 
 			if len(symmetricKey) == 0 {
-				log.Log.Fatal("main.Main(): symmetric key should not be empty")
+				log.Fatal("main.Main(): symmetric key should not be empty")
 				return
 			}
 			if len(symmetricKey) != 32 {
-				log.Log.Fatal("main.Main(): symmetric key should be 32 bytes")
+				log.Fatal("main.Main(): symmetric key should be 32 bytes")
 				return
 			}
 
@@ -163,19 +208,28 @@ func main() {
 
 			// Start OpenTelemetry tracing
 			if otelEndpoint := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"); otelEndpoint == "" {
-				log.Log.Info("main.Main(): No OpenTelemetry endpoint provided, skipping tracing")
+				log.WithFields(log.Fields{
+					"component": "tracing",
+					"event":     "tracing_disabled",
+				}).Debug("OpenTelemetry tracing disabled")
 			} else {
-				log.Log.Info("main.Main(): Starting OpenTelemetry tracing with endpoint: " + otelEndpoint)
+				log.WithFields(log.Fields{
+					"component": "tracing",
+					"event":     "tracing_starting",
+				}).Info("Starting OpenTelemetry tracing")
 				agentKey := configuration.Config.Key
 				traceProvider, err := startTracing(agentKey, otelEndpoint)
 				if err != nil {
-					log.Log.Error("traceprovider: " + err.Error())
+					log.WithError(err).WithField("component", "tracing").
+						Error("Failed to start OpenTelemetry tracing")
+				} else {
+					defer func() {
+						if err := traceProvider.Shutdown(context.Background()); err != nil {
+							log.WithError(err).WithField("component", "tracing").
+								Error("Failed to shut down OpenTelemetry tracing")
+						}
+					}()
 				}
-				defer func() {
-					if err := traceProvider.Shutdown(context.Background()); err != nil {
-						log.Log.Error("traceprovider: " + err.Error())
-					}
-				}()
 			}
 
 			// Printing final configuration
@@ -186,8 +240,13 @@ func main() {
 			utils.CheckDataDirectoryPermissions(configDirectory)
 
 			// Set timezone
-			timezone, _ := time.LoadLocation(configuration.Config.Timezone)
-			log.Log.Init(logLevel, logOutput, configDirectory, timezone)
+			timezone, err := time.LoadLocation(configuration.Config.Timezone)
+			if err != nil {
+				log.WithError(err).WithField("timezone", configuration.Config.Timezone).
+					Warn("invalid Agent timezone; using the host timezone for logs")
+				timezone = time.Local
+			}
+			configureLogging(logLevel, logOutput, timezone)
 
 			// Check if we have a device Key or not, if not
 			// we will generate one.
@@ -196,30 +255,37 @@ func main() {
 				configuration.Config.Key = key
 				err := configService.StoreConfig(configDirectory, configuration.Config)
 				if err == nil {
-					log.Log.Info("main.Main(): updated unique key for agent to: " + key)
+					log.WithFields(log.Fields{
+						"component": "configuration",
+						"event":     "agent_key_generated",
+					}).Info("Generated and stored a unique Agent key")
 				} else {
-					log.Log.Info("main.Main(): something went wrong while trying to store key: " + key)
+					log.WithError(err).WithFields(log.Fields{
+						"component": "configuration",
+						"event":     "agent_key_store_failed",
+					}).Error("Failed to store the generated Agent key")
 				}
 			}
 
 			// Create a cancelable context, which will be used to cancel and restart.
 			// This is used to restart the agent when the configuration is updated.
 			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
 
 			// We create a capture object, this will contain all the streaming clients.
 			// And allow us to extract media from within difference places in the agent.
-			capture := capture.Capture{
-				RTSPClient:    nil,
-				RTSPSubClient: nil,
-			}
+			capture := capture.Capture{}
 
 			// Bootstrapping the agent
 			communication := models.Communication{
-				Context:         &ctx,
-				CancelContext:   &cancel,
 				HandleBootstrap: make(chan string, 1),
 			}
 
+			log.WithFields(log.Fields{
+				"component": "agent",
+				"event":     "runtime_starting",
+				"port":      configuration.Port,
+			}).Info("Starting Agent runtime")
 			go components.Bootstrap(ctx, configDirectory, &configuration, &communication, &capture)
 
 			// Start the REST API.
@@ -227,7 +293,7 @@ func main() {
 		}
 	default:
 		{
-			log.Log.Error("main.Main(): Sorry I don't understand :(")
+			log.Error("main.Main(): Sorry I don't understand :(")
 		}
 	}
 }

machinery/main_test.go

@@ -0,0 +1,33 @@
+package main
+
+import "testing"
+
+func TestResolveServerPort(t *testing.T) {
+	tests := []struct {
+		name      string
+		flagValue string
+		envValue  string
+		want      string
+		wantError bool
+	}{
+		{name: "flag default", flagValue: "80", want: "80"},
+		{name: "environment overrides flag", flagValue: "80", envValue: "8082", want: "8082"},
+		{name: "trims environment", flagValue: "80", envValue: " 9090 ", want: "9090"},
+		{name: "empty values use default", want: "80"},
+		{name: "invalid text", flagValue: "80", envValue: "http", wantError: true},
+		{name: "zero", flagValue: "80", envValue: "0", wantError: true},
+		{name: "above maximum", flagValue: "80", envValue: "65536", wantError: true},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got, err := resolveServerPort(test.flagValue, test.envValue)
+			if (err != nil) != test.wantError {
+				t.Fatalf("resolveServerPort(%q, %q) error = %v, wantError %t", test.flagValue, test.envValue, err, test.wantError)
+			}
+			if got != test.want {
+				t.Fatalf("resolveServerPort(%q, %q) = %q, want %q", test.flagValue, test.envValue, got, test.want)
+			}
+		})
+	}
+}

machinery/src/capture/gortsplib.go

@@ -8,42 +8,97 @@ import "C"
 
 import (
 	"context"
+	"crypto/tls"
+	"crypto/x509"
 	"errors"
 	"fmt"
 	"image"
+	"math"
+	"net/url"
+	"os"
 	"reflect"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 	"unsafe"
 
-	"github.com/bluenviron/gortsplib/v4"
-	"github.com/bluenviron/gortsplib/v4/pkg/base"
-	"github.com/bluenviron/gortsplib/v4/pkg/description"
-	"github.com/bluenviron/gortsplib/v4/pkg/format"
-	"github.com/bluenviron/gortsplib/v4/pkg/format/rtph264"
-	"github.com/bluenviron/gortsplib/v4/pkg/format/rtph265"
-	"github.com/bluenviron/gortsplib/v4/pkg/format/rtplpcm"
-	"github.com/bluenviron/gortsplib/v4/pkg/format/rtpmpeg4audio"
-	"github.com/bluenviron/gortsplib/v4/pkg/format/rtpsimpleaudio"
+	"github.com/bluenviron/gortsplib/v5"
+	"github.com/bluenviron/gortsplib/v5/pkg/base"
+	"github.com/bluenviron/gortsplib/v5/pkg/description"
+	"github.com/bluenviron/gortsplib/v5/pkg/format"
+	"github.com/bluenviron/gortsplib/v5/pkg/format/rtph264"
+	"github.com/bluenviron/gortsplib/v5/pkg/format/rtph265"
+	"github.com/bluenviron/gortsplib/v5/pkg/format/rtplpcm"
+	"github.com/bluenviron/gortsplib/v5/pkg/format/rtpmpeg4audio"
+	"github.com/bluenviron/gortsplib/v5/pkg/format/rtpsimpleaudio"
 	"github.com/bluenviron/mediacommon/pkg/codecs/h264"
 	"github.com/bluenviron/mediacommon/pkg/codecs/h265"
 	"github.com/bluenviron/mediacommon/pkg/codecs/mpeg4audio"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/pion/rtp"
+	log "github.com/sirupsen/logrus"
 	"go.opentelemetry.io/otel"
 )
 
 var tracer = otel.Tracer("github.com/kerberos-io/agent/machinery/src/capture")
 
+const (
+	rtspsCAFileEnv   = "AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE"
+	rtspsInsecureEnv = "AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE"
+)
+
+func rtspsTLSConfig() (*tls.Config, error) {
+	if os.Getenv(rtspsInsecureEnv) == "true" {
+		return &tls.Config{InsecureSkipVerify: true}, nil // #nosec G402 -- explicit opt-in for cameras with self-signed certificates
+	}
+
+	caFile := os.Getenv(rtspsCAFileEnv)
+	if caFile == "" {
+		return nil, nil
+	}
+
+	rootCAs, err := x509.SystemCertPool()
+	if err != nil {
+		return nil, fmt.Errorf("load system CA pool: %w", err)
+	}
+	caPEM, err := os.ReadFile(caFile)
+	if err != nil {
+		return nil, fmt.Errorf("read RTSPS CA file %q: %w", caFile, err)
+	}
+	if !rootCAs.AppendCertsFromPEM(caPEM) {
+		return nil, fmt.Errorf("RTSPS CA file %q contains no certificates", caFile)
+	}
+
+	return &tls.Config{RootCAs: rootCAs}, nil
+}
+
+func sanitizeRTSPError(err error, rawURL string) error {
+	if err == nil {
+		return nil
+	}
+	if rawURL == "" || !strings.Contains(err.Error(), rawURL) {
+		return err
+	}
+
+	replacement := "<redacted-rtsp-url>"
+	if parsed, parseErr := url.Parse(rawURL); parseErr == nil {
+		parsed.User = nil
+		parsed.RawQuery = ""
+		parsed.Fragment = ""
+		replacement = parsed.String()
+	}
+	return errors.New(strings.ReplaceAll(err.Error(), rawURL, replacement))
+}
+
 // Implements the RTSPClient interface.
 type Golibrtsp struct {
 	RTSPClient
 	Url string
 
 	Client            gortsplib.Client
+	clientStarted     bool
 	VideoDecoderMutex *sync.Mutex
 
 	VideoH264Index        int8
@@ -164,14 +219,20 @@ func (h *streamHealth) observePacket(streamType string, writeDur time.Duration)
 		h.gapMax = gap
 	}
 	if writeDur >= streamHealthWriteWarn {
-		log.Log.Warning(fmt.Sprintf(
-			"capture.golibrtsp.health(%s): WritePacket blocked %dms — downstream back-pressure / CPU starvation",
-			streamType, writeDur.Milliseconds()))
+		log.WithFields(log.Fields{
+			"component":   "capture",
+			"duration_ms": writeDur.Milliseconds(),
+			"event":       "packet_queue_write_slow",
+			"stream":      streamType,
+		}).Warn("Packet queue write was slow")
 	}
 	if gap >= streamHealthGapWarn {
-		log.Log.Warning(fmt.Sprintf(
-			"capture.golibrtsp.health(%s): %dms since previous frame — upstream network / camera stall",
-			streamType, gap.Milliseconds()))
+		log.WithFields(log.Fields{
+			"component":    "capture",
+			"event":        "frame_gap_detected",
+			"frame_gap_ms": gap.Milliseconds(),
+			"stream":       streamType,
+		}).Warn("Large gap detected between camera frames")
 	}
 	if now.Sub(h.windowStart) >= streamHealthWindow {
 		elapsed := now.Sub(h.windowStart).Seconds()
@@ -179,10 +240,20 @@ func (h *streamHealth) observePacket(streamType string, writeDur time.Duration)
 		if h.frames > 0 {
 			avgWriteMs = float64(h.writeSum.Milliseconds()) / float64(h.frames)
 		}
-		log.Log.Info(fmt.Sprintf(
-			"capture.golibrtsp.health(%s): %.0fs window — frames=%d (%.1f/s) writeAvg=%.1fms writeMax=%dms gapMax=%dms lost=%d decodeErrs=%d",
-			streamType, elapsed, h.frames, float64(h.frames)/elapsed, avgWriteMs,
-			h.writeMax.Milliseconds(), h.gapMax.Milliseconds(), h.lost, h.decodeErrs))
+		log.WithFields(log.Fields{
+			"average_write_ms": avgWriteMs,
+			"component":        "capture",
+			"decode_errors":    h.decodeErrs,
+			"event":            "stream_health_sampled",
+			"frame_rate":       float64(h.frames) / elapsed,
+			"frames":           h.frames,
+			"lost_packets":     h.lost,
+			"max_frame_gap_ms": h.gapMax.Milliseconds(),
+			"max_write_ms":     h.writeMax.Milliseconds(),
+			"stream":           streamType,
+			"window_seconds":   elapsed,
+		}).Debug("RTSP stream health sampled")
+
 		h.windowStart = now
 		h.frames = 0
 		h.writeSum = 0
@@ -203,9 +274,12 @@ func (h *streamHealth) observeLost(streamType string, lost uint64) {
 	h.mu.Lock()
 	h.lost += lost
 	h.mu.Unlock()
-	log.Log.Warning(fmt.Sprintf(
-		"capture.golibrtsp.health(%s): %d RTP packet(s) lost — sender-side gap (receiver not draining TCP fast enough)",
-		streamType, lost))
+	log.WithFields(log.Fields{
+		"component":    "capture",
+		"event":        "rtp_packets_lost",
+		"lost_packets": lost,
+		"stream":       streamType,
+	}).Warn("RTSP sender reported lost RTP packets")
 }
 
 // observeDecodeError is invoked by gortsplib on incomplete/invalid access units,
@@ -217,7 +291,11 @@ func (h *streamHealth) observeDecodeError(streamType string, err error) {
 	h.mu.Lock()
 	h.decodeErrs++
 	h.mu.Unlock()
-	log.Log.Debug(fmt.Sprintf("capture.golibrtsp.health(%s): decode error: %s", streamType, err.Error()))
+	log.WithError(err).WithFields(log.Fields{
+		"component": "capture",
+		"event":     "access_unit_decode_failed",
+		"stream":    streamType,
+	}).Debug("Failed to decode RTSP access unit")
 }
 
 // fpsTracker holds per-stream state for PTS-based FPS calculation.
@@ -301,13 +379,13 @@ func init() {
 	// setup H264 -> raw frames decoder
 	H264FrameDecoder, err = newDecoder("H264")
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.init(): " + err.Error())
+		log.Error("capture.golibrtsp.init(): " + err.Error())
 	}
 
 	// setup H265 -> raw frames decoder
 	H265FrameDecoder, err = newDecoder("H265")
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.init(): " + err.Error())
+		log.Error("capture.golibrtsp.init(): " + err.Error())
 	}
 }
 
@@ -317,11 +395,17 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 	_, span := tracer.Start(ctxOtel, "Connect")
 	defer span.End()
 
-	transport := gortsplib.TransportTCP
+	tlsConfig, err := rtspsTLSConfig()
+	if err != nil {
+		return fmt.Errorf("configure RTSPS TLS: %w", err)
+	}
+
+	protocol := gortsplib.ProtocolTCP
 	g.health = newStreamHealth()
 	g.Client = gortsplib.Client{
 		RequestBackChannels: false,
-		Transport:           &transport,
+		Protocol:            &protocol,
+		TLSConfig:           tlsConfig,
 		// Route gortsplib's packet-loss / decode-error reporting through our
 		// structured logger with stream context (replaces its plain stdout
 		// logging). These hooks are what let us tell whether the camera is
@@ -337,21 +421,22 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 	// parse URL
 	u, err := base.ParseURL(g.Url)
 	if err != nil {
-		log.Log.Debug("capture.golibrtsp.Connect(ParseURL): " + err.Error())
-		return
+		return sanitizeRTSPError(err, g.Url)
 	}
 
 	// connect to the server
-	err = g.Client.Start(u.Scheme, u.Host)
+	g.Client.Scheme = u.Scheme
+	g.Client.Host = u.Host
+	err = g.Client.Start()
 	if err != nil {
-		log.Log.Debug("capture.golibrtsp.Connect(Start): " + err.Error())
+		return sanitizeRTSPError(err, g.Url)
 	}
+	g.clientStarted = true
 
 	// find published medias
 	desc, _, err := g.Client.Describe(u)
 	if err != nil {
-		log.Log.Debug("capture.golibrtsp.Connect(Describe): " + err.Error())
-		return
+		return sanitizeRTSPError(err, g.Url)
 	}
 
 	// Initialize the mutex and FPS calculation.
@@ -364,13 +449,13 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 	g.VideoH264Media = mediH264
 	g.VideoH264Forma = formaH264
 	if mediH264 == nil {
-		log.Log.Debug("capture.golibrtsp.Connect(H264): " + "video media not found")
+		log.Debug("capture.golibrtsp.Connect(H264): " + "video media not found")
 	} else {
 		// setup a video media
 		_, err = g.Client.Setup(desc.BaseURL, mediH264, 0, 0)
 		if err != nil {
 			// Something went wrong .. Do something
-			log.Log.Error("capture.golibrtsp.Connect(H264): " + err.Error())
+			log.Error("capture.golibrtsp.Connect(H264): " + err.Error())
 		} else {
 			// Get SPS and PPS from the SDP
 			// Calculate the width and height of the video
@@ -379,7 +464,7 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 			// It might be that the SPS is not available yet, so we'll proceed,
 			// but try to fetch it later on.
 			if errSPS != nil {
-				log.Log.Debug("capture.golibrtsp.Connect(H264): " + errSPS.Error())
+				log.Debug("capture.golibrtsp.Connect(H264): " + errSPS.Error())
 				streamIndex := len(g.Streams)
 				g.Streams = append(g.Streams, packets.Stream{
 					Index:         streamIndex,
@@ -415,7 +500,7 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 			// setup RTP/H264 -> H264 decoder
 			rtpDec, err := formaH264.CreateDecoder()
 			if err != nil {
-				log.Log.Error("capture.golibrtsp.Connect(H264): " + err.Error())
+				log.Error("capture.golibrtsp.Connect(H264): " + err.Error())
 			}
 			g.VideoH264Decoder = rtpDec
 			g.VideoH264FrameDecoder = H264FrameDecoder
@@ -428,20 +513,20 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 	g.VideoH265Media = mediH265
 	g.VideoH265Forma = formaH265
 	if mediH265 == nil {
-		log.Log.Debug("capture.golibrtsp.Connect(H265): " + "video media not found")
+		log.Debug("capture.golibrtsp.Connect(H265): " + "video media not found")
 	} else {
 		// setup a video media
 		_, err = g.Client.Setup(desc.BaseURL, mediH265, 0, 0)
 		if err != nil {
 			// Something went wrong .. Do something
-			log.Log.Error("capture.golibrtsp.Connect(H265): " + err.Error())
+			log.Error("capture.golibrtsp.Connect(H265): " + err.Error())
 		} else {
 			// Get SPS from the SDP
 			// Calculate the width and height of the video
 			var sps h265.SPS
 			err = sps.Unmarshal(formaH265.SPS)
 			if err != nil {
-				log.Log.Info("capture.golibrtsp.Connect(H265): " + err.Error())
+				log.Info("capture.golibrtsp.Connect(H265): " + err.Error())
 				return
 			}
 			streamIndex := len(g.Streams)
@@ -465,7 +550,7 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 			// setup RTP/H265 -> H265 decoder
 			rtpDec, err := formaH265.CreateDecoder()
 			if err != nil {
-				log.Log.Error("capture.golibrtsp.Connect(H265): " + err.Error())
+				log.Error("capture.golibrtsp.Connect(H265): " + err.Error())
 			}
 			g.VideoH265Decoder = rtpDec
 
@@ -479,19 +564,19 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 	g.AudioG711Media = audioMedi
 	g.AudioG711Forma = audioForma
 	if audioMedi == nil {
-		log.Log.Debug("capture.golibrtsp.Connect(G711): " + "audio media not found")
+		log.Debug("capture.golibrtsp.Connect(G711): " + "audio media not found")
 	} else {
 		// setup a audio media
 		_, err = g.Client.Setup(desc.BaseURL, audioMedi, 0, 0)
 		if err != nil {
 			// Something went wrong .. Do something
-			log.Log.Error("capture.golibrtsp.Connect(G711): " + err.Error())
+			log.Error("capture.golibrtsp.Connect(G711): " + err.Error())
 		} else {
 			// create decoder
 			audiortpDec, err := audioForma.CreateDecoder()
 			if err != nil {
 				// Something went wrong .. Do something
-				log.Log.Error("capture.golibrtsp.Connect(G711): " + err.Error())
+				log.Error("capture.golibrtsp.Connect(G711): " + err.Error())
 			} else {
 				g.AudioG711Decoder = audiortpDec
 				streamIndex := len(g.Streams)
@@ -515,19 +600,19 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 	g.AudioOpusMedia = audioMediOpus
 	g.AudioOpusForma = audioFormaOpus
 	if audioMediOpus == nil {
-		log.Log.Debug("capture.golibrtsp.Connect(Opus): " + "audio media not found")
+		log.Debug("capture.golibrtsp.Connect(Opus): " + "audio media not found")
 	} else {
 		// setup a audio media
 		_, err = g.Client.Setup(desc.BaseURL, audioMediOpus, 0, 0)
 		if err != nil {
 			// Something went wrong .. Do something
-			log.Log.Error("capture.golibrtsp.Connect(Opus): " + err.Error())
+			log.Error("capture.golibrtsp.Connect(Opus): " + err.Error())
 		} else {
 			// create decoder
 			audiortpDec, err := audioFormaOpus.CreateDecoder()
 			if err != nil {
 				// Something went wrong .. Do something
-				log.Log.Error("capture.golibrtsp.Connect(Opus): " + err.Error())
+				log.Error("capture.golibrtsp.Connect(Opus): " + err.Error())
 			} else {
 				g.AudioOpusDecoder = audiortpDec
 				streamIndex := len(g.Streams)
@@ -551,13 +636,13 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 	g.AudioMPEG4Media = audioMediMPEG4
 	g.AudioMPEG4Forma = audioFormaMPEG4
 	if audioMediMPEG4 == nil {
-		log.Log.Debug("capture.golibrtsp.Connect(MPEG4): " + "audio media not found")
+		log.Debug("capture.golibrtsp.Connect(MPEG4): " + "audio media not found")
 	} else {
 		// setup a audio media
 		_, err = g.Client.Setup(desc.BaseURL, audioMediMPEG4, 0, 0)
 		if err != nil {
 			// Something went wrong .. Do something
-			log.Log.Error("capture.golibrtsp.Connect(MPEG4): " + err.Error())
+			log.Error("capture.golibrtsp.Connect(MPEG4): " + err.Error())
 		} else {
 			streamIndex := len(g.Streams)
 			g.Streams = append(g.Streams, packets.Stream{
@@ -577,7 +662,7 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 			audiortpDec, err := audioFormaMPEG4.CreateDecoder()
 			if err != nil {
 				// Something went wrong .. Do something
-				log.Log.Error("capture.golibrtsp.Connect(MPEG4): " + err.Error())
+				log.Error("capture.golibrtsp.Connect(MPEG4): " + err.Error())
 			}
 			g.AudioMPEG4Decoder = audiortpDec
 
@@ -592,30 +677,37 @@ func (g *Golibrtsp) ConnectBackChannel(ctx context.Context, ctxRunAgent context.
 	_, span := tracer.Start(ctxRunAgent, "ConnectBackChannel")
 	defer span.End()
 
+	tlsConfig, err := rtspsTLSConfig()
+	if err != nil {
+		return fmt.Errorf("configure RTSPS TLS: %w", err)
+	}
+
 	// Transport TCP
-	transport := gortsplib.TransportTCP
+	protocol := gortsplib.ProtocolTCP
 	g.Client = gortsplib.Client{
 		RequestBackChannels: true,
-		Transport:           &transport,
+		Protocol:            &protocol,
+		TLSConfig:           tlsConfig,
 	}
 	// parse URL
 	u, err := base.ParseURL(g.Url)
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.ConnectBackChannel(): " + err.Error())
-		return
+		return sanitizeRTSPError(err, g.Url)
 	}
 
 	// connect to the server
-	err = g.Client.Start(u.Scheme, u.Host)
+	g.Client.Scheme = u.Scheme
+	g.Client.Host = u.Host
+	err = g.Client.Start()
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.ConnectBackChannel(): " + err.Error())
+		return sanitizeRTSPError(err, g.Url)
 	}
+	g.clientStarted = true
 
 	// find published medias
 	desc, _, err := g.Client.Describe(u)
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.ConnectBackChannel(): " + err.Error())
-		return
+		return sanitizeRTSPError(err, g.Url)
 	}
 
 	// Look for audio back channel.
@@ -625,15 +717,17 @@ func (g *Golibrtsp) ConnectBackChannel(ctx context.Context, ctxRunAgent context.
 	g.AudioG711MediaBackChannel = audioMediBackChannel
 	g.AudioG711FormaBackChannel = audioFormaBackChannel
 	if audioMediBackChannel == nil {
-		log.Log.Error("capture.golibrtsp.ConnectBackChannel(): audio backchannel not found, not a real error, however you might expect a backchannel. One of the reasons might be that the device already has an active client connected to the backchannel.")
+		log.WithFields(log.Fields{
+			"component": "capture",
+			"event":     "backchannel_unavailable",
+		}).Debug("Optional camera audio backchannel unavailable")
 		err = errors.New("no audio backchannel found")
 	} else {
 		// setup a audio media
 		_, err = g.Client.Setup(desc.BaseURL, audioMediBackChannel, 0, 0)
 		if err != nil {
-			// Something went wrong .. Do something
-			log.Log.Error("capture.golibrtsp.ConnectBackChannel(): " + err.Error())
 			g.HasBackChannel = false
+			return sanitizeRTSPError(err, g.Url)
 		} else {
 			g.HasBackChannel = true
 			streamIndex := len(g.Streams)
@@ -680,9 +774,32 @@ func compositionOffsetMs(ext dtsExtractor, au [][]byte, pts int64, clockRate int
 	return offset * 1000 / int64(clockRate)
 }
 
+func ptsToDuration(pts int64, clockRate int) time.Duration {
+	rate := int64(clockRate)
+	return time.Duration(pts/rate)*time.Second +
+		time.Duration(pts%rate)*time.Second/time.Duration(rate)
+}
+
+func preRecordingGOPCount(preRecording int64, gopDuration float64) (int, bool) {
+	if preRecording <= 0 ||
+		gopDuration < 1 ||
+		math.IsNaN(gopDuration) ||
+		math.IsInf(gopDuration, 0) ||
+		gopDuration >= float64(math.MaxInt64) {
+		return 0, false
+	}
+
+	count := preRecording / int64(gopDuration)
+	maxInt := int64(^uint(0) >> 1)
+	if count >= maxInt {
+		return 0, false
+	}
+	return int(count) + 1, true
+}
+
 // Start the RTSP client, and start reading packets.
 func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets.Queue, configuration *models.Configuration, communication *models.Communication) (err error) {
-	log.Log.Debug("capture.golibrtsp.Start(): started")
+	log.Debug("capture.golibrtsp.Start(): started")
 
 	// Label this client's loss/decode/health logging with the stream type.
 	g.streamLabel = streamType
@@ -693,18 +810,18 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 	// called when a MULAW audio RTP packet arrives
 	if g.AudioG711Media != nil && g.AudioG711Forma != nil {
 		g.Client.OnPacketRTP(g.AudioG711Media, g.AudioG711Forma, func(rtppkt *rtp.Packet) {
-			pts, ok := g.Client.PacketPTS(g.AudioG711Media, rtppkt)
 			// decode timestamp
-			pts2, ok := g.Client.PacketPTS2(g.AudioG711Media, rtppkt)
+			pts2, ok := g.Client.PacketPTS(g.AudioG711Media, rtppkt)
 			if !ok {
-				log.Log.Debug("capture.golibrtsp.Start(): " + "unable to get PTS")
+				log.Debug("capture.golibrtsp.Start(): " + "unable to get PTS")
 				return
 			}
+			pts := ptsToDuration(pts2, g.AudioG711Forma.ClockRate())
 
 			// extract LPCM samples from RTP packets
 			op, err := g.AudioG711Decoder.Decode(rtppkt)
 			if err != nil {
-				log.Log.Error("capture.golibrtsp.Start(): " + err.Error())
+				log.Error("capture.golibrtsp.Start(): " + err.Error())
 				return
 			}
 
@@ -729,24 +846,24 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 	if g.AudioMPEG4Media != nil && g.AudioMPEG4Forma != nil {
 		g.Client.OnPacketRTP(g.AudioMPEG4Media, g.AudioMPEG4Forma, func(rtppkt *rtp.Packet) {
 			// decode timestamp
-			pts, ok := g.Client.PacketPTS(g.AudioMPEG4Media, rtppkt)
-			pts2, ok := g.Client.PacketPTS2(g.AudioMPEG4Media, rtppkt)
+			pts2, ok := g.Client.PacketPTS(g.AudioMPEG4Media, rtppkt)
 			if !ok {
-				log.Log.Error("capture.golibrtsp.Start(): " + "unable to get PTS")
+				log.Error("capture.golibrtsp.Start(): " + "unable to get PTS")
 				return
 			}
+			pts := ptsToDuration(pts2, g.AudioMPEG4Forma.ClockRate())
 
 			// Encode the AAC samples from RTP packets
 			// extract access units from RTP packets
 			aus, err := g.AudioMPEG4Decoder.Decode(rtppkt)
 			if err != nil {
-				log.Log.Error("capture.golibrtsp.Start(): " + err.Error())
+				log.Error("capture.golibrtsp.Start(): " + err.Error())
 				return
 			}
 
 			enc, err := WriteMPEG4Audio(g.AudioMPEG4Forma, aus)
 			if err != nil {
-				log.Log.Error("capture.golibrtsp.Start(): " + err.Error())
+				log.Error("capture.golibrtsp.Start(): " + err.Error())
 				return
 			}
 
@@ -788,12 +905,12 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 			if len(rtppkt.Payload) > 0 {
 
 				// decode timestamps — validate each call separately
-				pts, okPTS := g.Client.PacketPTS(g.VideoH264Media, rtppkt)
-				pts2, okPTS2 := g.Client.PacketPTS2(g.VideoH264Media, rtppkt)
+				pts2, okPTS2 := g.Client.PacketPTS(g.VideoH264Media, rtppkt)
 				if !okPTS2 {
-					log.Log.Debug("capture.golibrtsp.Start(): unable to get PTS2 from PacketPTS2")
+					log.Debug("capture.golibrtsp.Start(): unable to get PTS")
 					return
 				}
+				pts := ptsToDuration(pts2, g.VideoH264Forma.ClockRate())
 
 				// Extract access units from RTP packets.
 				// We need a complete access unit to determine whether
@@ -801,21 +918,19 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 				au, errDecode := g.VideoH264Decoder.Decode(rtppkt)
 				if errDecode != nil {
 					if errDecode != rtph264.ErrNonStartingPacketAndNoPrevious && errDecode != rtph264.ErrMorePacketsNeeded {
-						log.Log.Error("capture.golibrtsp.Start(): " + errDecode.Error())
+						log.Error("capture.golibrtsp.Start(): " + errDecode.Error())
 					}
 					return
 				}
 
 				// Frame is complete — update per-stream FPS from PTS.
-				if okPTS {
-					ft := g.fpsTrackers[g.VideoH264Index]
-					if ft == nil {
-						ft = newFPSTracker(30)
-						g.fpsTrackers[g.VideoH264Index] = ft
-					}
-					if ptsFPS := ft.update(pts); ptsFPS > 0 && ptsFPS <= 120 {
-						g.Streams[g.VideoH264Index].FPS = ptsFPS
-					}
+				ft := g.fpsTrackers[g.VideoH264Index]
+				if ft == nil {
+					ft = newFPSTracker(30)
+					g.fpsTrackers[g.VideoH264Index] = ft
+				}
+				if ptsFPS := ft.update(pts); ptsFPS > 0 && ptsFPS <= 120 {
+					g.Streams[g.VideoH264Index].FPS = ptsFPS
 				}
 
 				// We'll need to read out a few things.
@@ -865,7 +980,12 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 							fps := g.getEnhancedFPS(&sps, g.VideoH264Index)
 							g.Streams[g.VideoH264Index].FPS = fps
 							g.persistStreamFPS(configuration, streamType, fps)
-							log.Log.Debug(fmt.Sprintf("capture.golibrtsp.Start(%s): Final FPS=%.2f", streamType, fps))
+							log.WithFields(log.Fields{
+								"component": "capture",
+								"event":     "frame_rate_detected",
+								"fps":       fps,
+								"stream":    streamType,
+							}).Debug("RTSP frame rate detected")
 							g.VideoH264Forma.SPS = nalu
 							if streamType == "main" && len(nalu) > 0 {
 								// Fallback: store SPS from in-band NALUs when SDP was missing it.
@@ -887,14 +1007,14 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 					// Ensure config has parameter sets before recordings start.
 					if len(configuration.Config.Capture.IPCamera.SPSNALUs) == 0 && len(g.VideoH264Forma.SPS) > 0 {
 						configuration.Config.Capture.IPCamera.SPSNALUs = [][]byte{g.VideoH264Forma.SPS}
-						log.Log.Warning("capture.golibrtsp.Start(main): fallback SPS set from keyframe")
+						log.Warn("capture.golibrtsp.Start(main): fallback SPS set from keyframe")
 					}
 					if len(configuration.Config.Capture.IPCamera.PPSNALUs) == 0 && len(g.VideoH264Forma.PPS) > 0 {
 						configuration.Config.Capture.IPCamera.PPSNALUs = [][]byte{g.VideoH264Forma.PPS}
-						log.Log.Warning("capture.golibrtsp.Start(main): fallback PPS set from keyframe")
+						log.Warn("capture.golibrtsp.Start(main): fallback PPS set from keyframe")
 					}
 					if len(configuration.Config.Capture.IPCamera.SPSNALUs) == 0 || len(configuration.Config.Capture.IPCamera.PPSNALUs) == 0 {
-						log.Log.Warning("capture.golibrtsp.Start(main): SPS/PPS still missing after IDR keyframe")
+						log.Warn("capture.golibrtsp.Start(main): SPS/PPS still missing after IDR keyframe")
 					}
 				}
 
@@ -903,14 +1023,18 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 				}
 
 				if idrPresent {
-					log.Log.Debug(fmt.Sprintf("capture.golibrtsp.Start(%s): IDR frame NALUs: [%s]",
-						streamType, fmt.Sprintf("%v", naluTypes)))
+					log.WithFields(log.Fields{
+						"component":  "capture",
+						"event":      "idr_frame_received",
+						"nalu_types": naluTypes,
+						"stream":     streamType,
+					}).Debug("RTSP IDR frame received")
 				}
 
 				// Convert to packet.
 				enc, err := h264.AnnexBMarshal(filteredAU)
 				if err != nil {
-					log.Log.Error("capture.golibrtsp.Start(): " + err.Error())
+					log.Error("capture.golibrtsp.Start(): " + err.Error())
 					return
 				}
 
@@ -944,11 +1068,19 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 					gopDuration := float64(keyframeInterval) / fps
 					gopSize := int(avgInterval) // Store GOP size in a separate variable
 					g.Streams[g.VideoH264Index].GopSize = gopSize
-					log.Log.Debug(fmt.Sprintf("capture.golibrtsp.Start(%s): Keyframe interval=%d packets, Avg=%.1f, GOP=%.1fs, GOPSize=%d",
-						streamType, keyframeInterval, avgInterval, gopDuration, gopSize))
+					log.WithFields(log.Fields{
+						"average_keyframe_interval_packets": avgInterval,
+						"component":                         "capture",
+						"event":                             "keyframe_interval_observed",
+						"gop_duration_seconds":              gopDuration,
+						"gop_size_packets":                  gopSize,
+						"keyframe_interval_packets":         keyframeInterval,
+						"stream":                            streamType,
+					}).Debug("RTSP keyframe interval observed")
+
 					preRecording := configuration.Config.Capture.PreRecording
-					if preRecording > 0 && int(gopDuration) > 0 {
-						queue.SetMaxGopCount(int(preRecording)/int(gopDuration) + 1)
+					if maxGOPCount, ok := preRecordingGOPCount(preRecording, gopDuration); ok {
+						queue.SetMaxGopCount(maxGOPCount)
 					}
 				}
 
@@ -995,20 +1127,36 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 				default:
 				}
 
-				if idrPresent {
-					// Increment packets, so we know the device
-					// is not blocking.
-					if streamType == "main" {
-						r := communication.PackageCounter.Load().(int64)
-						log.Log.Debug("capture.golibrtsp.Start(): packet size " + strconv.Itoa(len(pkt.Data)))
-						communication.PackageCounter.Store((r + 1) % 1000)
-						communication.LastPacketTimer.Store(time.Now().Unix())
-					} else if streamType == "sub" {
-						r := communication.PackageCounterSub.Load().(int64)
-						log.Log.Debug("capture.golibrtsp.Start(): packet size " + strconv.Itoa(len(pkt.Data)))
-						communication.PackageCounterSub.Store((r + 1) % 1000)
-						communication.LastPacketTimerSub.Store(time.Now().Unix())
-					}
+				// Count every complete video access unit. Keyframe-only counters make
+				// healthy cameras with GOPs longer than the watchdog window look stalled.
+				if streamType == "main" {
+					observedAt := time.Now()
+					r := communication.PackageCounter.Load().(int64)
+					log.WithFields(log.Fields{
+						"bytes":     len(pkt.Data),
+						"codec":     "H264",
+						"component": "capture",
+						"event":     "access_unit_received",
+						"keyframe":  pkt.IsKeyFrame,
+						"stream":    streamType,
+					}).Trace("RTSP access unit received")
+					communication.RecordStreamPackage(models.MainStream, g.Streams[g.VideoH264Index].FPS, g.Streams[g.VideoH264Index].Width, g.Streams[g.VideoH264Index].Height, observedAt)
+					communication.PackageCounter.Store((r + 1) % 1000)
+					communication.LastPacketTimer.Store(observedAt.Unix())
+				} else if streamType == "sub" {
+					observedAt := time.Now()
+					r := communication.PackageCounterSub.Load().(int64)
+					log.WithFields(log.Fields{
+						"bytes":     len(pkt.Data),
+						"codec":     "H264",
+						"component": "capture",
+						"event":     "access_unit_received",
+						"keyframe":  pkt.IsKeyFrame,
+						"stream":    streamType,
+					}).Trace("RTSP access unit received")
+					communication.RecordStreamPackage(models.SubStream, g.Streams[g.VideoH264Index].FPS, g.Streams[g.VideoH264Index].Width, g.Streams[g.VideoH264Index].Height, observedAt)
+					communication.PackageCounterSub.Store((r + 1) % 1000)
+					communication.LastPacketTimerSub.Store(observedAt.Unix())
 				}
 			}
 
@@ -1035,12 +1183,12 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 			if len(rtppkt.Payload) > 0 {
 
 				// decode timestamps — validate each call separately
-				pts, okPTS := g.Client.PacketPTS(g.VideoH265Media, rtppkt)
-				pts2, okPTS2 := g.Client.PacketPTS2(g.VideoH265Media, rtppkt)
+				pts2, okPTS2 := g.Client.PacketPTS(g.VideoH265Media, rtppkt)
 				if !okPTS2 {
-					log.Log.Debug("capture.golibrtsp.Start(): unable to get PTS")
+					log.Debug("capture.golibrtsp.Start(): unable to get PTS")
 					return
 				}
+				pts := ptsToDuration(pts2, g.VideoH265Forma.ClockRate())
 
 				// Extract access units from RTP packets.
 				// We need a complete access unit to determine whether
@@ -1048,22 +1196,20 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 				au, errDecode := g.VideoH265Decoder.Decode(rtppkt)
 				if errDecode != nil {
 					if errDecode != rtph265.ErrNonStartingPacketAndNoPrevious && errDecode != rtph265.ErrMorePacketsNeeded {
-						log.Log.Error("capture.golibrtsp.Start(): " + errDecode.Error())
+						log.Error("capture.golibrtsp.Start(): " + errDecode.Error())
 					}
 					return
 				}
 
 				// Frame is complete — update per-stream FPS from PTS.
-				if okPTS {
-					ft := g.fpsTrackers[g.VideoH265Index]
-					if ft == nil {
-						ft = newFPSTracker(30)
-						g.fpsTrackers[g.VideoH265Index] = ft
-					}
-					if ptsFPS := ft.update(pts); ptsFPS > 0 && ptsFPS <= 120 {
-						g.Streams[g.VideoH265Index].FPS = ptsFPS
-						g.persistStreamFPS(configuration, streamType, ptsFPS)
-					}
+				ft := g.fpsTrackers[g.VideoH265Index]
+				if ft == nil {
+					ft = newFPSTracker(30)
+					g.fpsTrackers[g.VideoH265Index] = ft
+				}
+				if ptsFPS := ft.update(pts); ptsFPS > 0 && ptsFPS <= 120 {
+					g.Streams[g.VideoH265Index].FPS = ptsFPS
+					g.persistStreamFPS(configuration, streamType, ptsFPS)
 				}
 
 				// Preserve the decoded access unit (in decode order) for DTS
@@ -1108,7 +1254,7 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 
 				enc, err := h264.AnnexBMarshal(au)
 				if err != nil {
-					log.Log.Error("capture.golibrtsp.Start(): " + err.Error())
+					log.Error("capture.golibrtsp.Start(): " + err.Error())
 					return
 				}
 
@@ -1140,11 +1286,19 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 					gopDuration := float64(keyframeInterval) / fps
 					gopSize := int(avgInterval) // Store GOP size in a separate variable
 					g.Streams[g.VideoH265Index].GopSize = gopSize
-					log.Log.Debug(fmt.Sprintf("capture.golibrtsp.Start(%s): Keyframe interval=%d packets, Avg=%.1f, GOP=%.1fs, GOPSize=%d",
-						streamType, keyframeInterval, avgInterval, gopDuration, gopSize))
+					log.WithFields(log.Fields{
+						"average_keyframe_interval_packets": avgInterval,
+						"component":                         "capture",
+						"event":                             "keyframe_interval_observed",
+						"gop_duration_seconds":              gopDuration,
+						"gop_size_packets":                  gopSize,
+						"keyframe_interval_packets":         keyframeInterval,
+						"stream":                            streamType,
+					}).Debug("RTSP keyframe interval observed")
+
 					preRecording := configuration.Config.Capture.PreRecording
-					if preRecording > 0 && int(gopDuration) > 0 {
-						queue.SetMaxGopCount(int(preRecording)/int(gopDuration) + 1)
+					if maxGOPCount, ok := preRecordingGOPCount(preRecording, gopDuration); ok {
+						queue.SetMaxGopCount(maxGOPCount)
 					}
 				}
 
@@ -1162,20 +1316,36 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 				default:
 				}
 
-				if isRandomAccess {
-					// Increment packets, so we know the device
-					// is not blocking.
-					if streamType == "main" {
-						r := communication.PackageCounter.Load().(int64)
-						log.Log.Debug("capture.golibrtsp.Start(): packet size " + strconv.Itoa(len(pkt.Data)))
-						communication.PackageCounter.Store((r + 1) % 1000)
-						communication.LastPacketTimer.Store(time.Now().Unix())
-					} else if streamType == "sub" {
-						r := communication.PackageCounterSub.Load().(int64)
-						log.Log.Debug("capture.golibrtsp.Start(): packet size " + strconv.Itoa(len(pkt.Data)))
-						communication.PackageCounterSub.Store((r + 1) % 1000)
-						communication.LastPacketTimerSub.Store(time.Now().Unix())
-					}
+				// Count every complete video access unit; random-access frames remain
+				// responsible only for GOP tracking above.
+				if streamType == "main" {
+					observedAt := time.Now()
+					r := communication.PackageCounter.Load().(int64)
+					log.WithFields(log.Fields{
+						"bytes":     len(pkt.Data),
+						"codec":     "H265",
+						"component": "capture",
+						"event":     "access_unit_received",
+						"keyframe":  pkt.IsKeyFrame,
+						"stream":    streamType,
+					}).Trace("RTSP access unit received")
+					communication.RecordStreamPackage(models.MainStream, g.Streams[g.VideoH265Index].FPS, g.Streams[g.VideoH265Index].Width, g.Streams[g.VideoH265Index].Height, observedAt)
+					communication.PackageCounter.Store((r + 1) % 1000)
+					communication.LastPacketTimer.Store(observedAt.Unix())
+				} else if streamType == "sub" {
+					observedAt := time.Now()
+					r := communication.PackageCounterSub.Load().(int64)
+					log.WithFields(log.Fields{
+						"bytes":     len(pkt.Data),
+						"codec":     "H265",
+						"component": "capture",
+						"event":     "access_unit_received",
+						"keyframe":  pkt.IsKeyFrame,
+						"stream":    streamType,
+					}).Trace("RTSP access unit received")
+					communication.RecordStreamPackage(models.SubStream, g.Streams[g.VideoH265Index].FPS, g.Streams[g.VideoH265Index].Width, g.Streams[g.VideoH265Index].Height, observedAt)
+					communication.PackageCounterSub.Store((r + 1) % 1000)
+					communication.LastPacketTimerSub.Store(observedAt.Unix())
 				}
 			}
 
@@ -1187,7 +1357,7 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 	// Play the stream.
 	_, err = g.Client.Play(nil)
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.Start(): " + err.Error())
+		log.Error("capture.golibrtsp.Start(): " + err.Error())
 	}
 
 	return
@@ -1195,13 +1365,13 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 
 // Start the RTSP client, and start reading packets.
 func (g *Golibrtsp) StartBackChannel(ctx context.Context, ctxRunAgent context.Context) (err error) {
-	log.Log.Info("capture.golibrtsp.StartBackChannel(): started")
+	log.Info("capture.golibrtsp.StartBackChannel(): started")
 	// Wait for a second, so we can be sure the stream is playing.
 	time.Sleep(1 * time.Second)
 	// Play the stream.
 	_, err = g.Client.Play(nil)
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.StartBackChannel(): " + err.Error())
+		log.Error("capture.golibrtsp.StartBackChannel(): " + err.Error())
 	}
 	return
 }
@@ -1210,7 +1380,7 @@ func (g *Golibrtsp) WritePacket(pkt packets.Packet) error {
 	if g.HasBackChannel && g.AudioG711MediaBackChannel != nil {
 		err := g.Client.WritePacketRTP(g.AudioG711MediaBackChannel, pkt.Packet)
 		if err != nil {
-			log.Log.Debug("capture.golibrtsp.WritePacket(): " + err.Error())
+			log.Debug("capture.golibrtsp.WritePacket(): " + err.Error())
 			return err
 		}
 	}
@@ -1233,11 +1403,11 @@ func (g *Golibrtsp) DecodePacket(pkt packets.Packet) (image.YCbCr, error) {
 	}
 	g.VideoDecoderMutex.Unlock()
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.DecodePacket(): " + err.Error())
+		log.Error("capture.golibrtsp.DecodePacket(): " + err.Error())
 		return image.YCbCr{}, err
 	}
 	if img.Bounds().Empty() {
-		log.Log.Debug("capture.golibrtsp.DecodePacket(): empty frame")
+		log.Debug("capture.golibrtsp.DecodePacket(): empty frame")
 		return image.YCbCr{}, errors.New("Empty image")
 	}
 	return img, nil
@@ -1259,11 +1429,11 @@ func (g *Golibrtsp) DecodePacketRaw(pkt packets.Packet) (image.Gray, error) {
 	}
 	g.VideoDecoderMutex.Unlock()
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.DecodePacketRaw(): " + err.Error())
+		log.Error("capture.golibrtsp.DecodePacketRaw(): " + err.Error())
 		return image.Gray{}, err
 	}
 	if img.Bounds().Empty() {
-		log.Log.Debug("capture.golibrtsp.DecodePacketRaw(): empty image")
+		log.Debug("capture.golibrtsp.DecodePacketRaw(): empty image")
 		return image.Gray{}, errors.New("Empty image")
 	}
 
@@ -1308,7 +1478,11 @@ func (g *Golibrtsp) Close(ctxOtel context.Context) error {
 	_, span := tracer.Start(ctxOtel, "Close")
 	defer span.End()
 
-	// Close the demuxer.
+	if !g.clientStarted {
+		return nil
+	}
+
+	g.clientStarted = false
 	g.Client.Close()
 
 	// We will have created the decoders globally, so we don't need to close them here.
@@ -1353,13 +1527,13 @@ func newDecoder(codecName string) (*Decoder, error) {
 
 	res := C.avcodec_open2(codecCtx, codec, nil)
 	if res < 0 {
-		C.avcodec_close(codecCtx)
+		C.avcodec_free_context(&codecCtx)
 		return nil, fmt.Errorf("avcodec_open2() failed")
 	}
 
 	srcFrame := C.av_frame_alloc()
 	if srcFrame == nil {
-		C.avcodec_close(codecCtx)
+		C.avcodec_free_context(&codecCtx)
 		return nil, fmt.Errorf("av_frame_alloc() failed")
 	}
 
@@ -1375,7 +1549,7 @@ func (d *Decoder) Close() {
 		C.av_frame_free(&d.srcFrame)
 	}
 	C.av_frame_free(&d.srcFrame)
-	C.avcodec_close(d.codecCtx)
+	C.avcodec_free_context(&d.codecCtx)
 }
 
 func (d *Decoder) decode(nalu []byte) (image.YCbCr, error) {
@@ -1562,7 +1736,12 @@ func (g *Golibrtsp) getEnhancedFPS(sps *h264.SPS, streamIndex int8) float64 {
 
 	// Check if SPS FPS is reasonable (between 1 and 120 fps)
 	if spsFPS > 0 && spsFPS <= 120 {
-		log.Log.Debug(fmt.Sprintf("capture.golibrtsp.getEnhancedFPS(): SPS FPS: %.2f", spsFPS))
+		log.WithFields(log.Fields{
+			"component": "capture",
+			"event":     "frame_rate_selected",
+			"fps":       spsFPS,
+			"source":    "sps",
+		}).Debug("RTSP frame rate selected")
 		return spsFPS
 	}
 
@@ -1570,7 +1749,12 @@ func (g *Golibrtsp) getEnhancedFPS(sps *h264.SPS, streamIndex int8) float64 {
 	if ft := g.fpsTrackers[streamIndex]; ft != nil {
 		ptsFPS := ft.fps()
 		if ptsFPS > 0 && ptsFPS <= 120 {
-			log.Log.Debug(fmt.Sprintf("capture.golibrtsp.getEnhancedFPS(): PTS FPS: %.2f", ptsFPS))
+			log.WithFields(log.Fields{
+				"component": "capture",
+				"event":     "frame_rate_selected",
+				"fps":       ptsFPS,
+				"source":    "presentation_timestamp",
+			}).Debug("RTSP frame rate selected")
 			return ptsFPS
 		}
 	}
@@ -1672,13 +1856,15 @@ func (g *Golibrtsp) getSPSTimingInfo(sps *h264.SPS) (hasVUI bool, timeScale uint
 func (g *Golibrtsp) debugSPSInfo(sps *h264.SPS, streamType string) {
 	hasVUI, timeScale, numUnitsInTick, fps := g.getSPSTimingInfo(sps)
 
-	log.Log.Debug(fmt.Sprintf("capture.golibrtsp.debugSPSInfo(%s): Width=%d, Height=%d",
-		streamType, sps.Width(), sps.Height()))
-	log.Log.Debug(fmt.Sprintf("capture.golibrtsp.debugSPSInfo(%s): HasVUI=%t, FPS=%.2f",
-		streamType, hasVUI, fps))
-
-	if hasVUI {
-		log.Log.Debug(fmt.Sprintf("capture.golibrtsp.debugSPSInfo(%s): TimeScale=%d, NumUnitsInTick=%d",
-			streamType, timeScale, numUnitsInTick))
-	}
+	log.WithFields(log.Fields{
+		"component":         "capture",
+		"event":             "sps_inspected",
+		"fps":               fps,
+		"has_vui":           hasVUI,
+		"height_pixels":     sps.Height(),
+		"num_units_in_tick": numUnitsInTick,
+		"stream":            streamType,
+		"time_scale":        timeScale,
+		"width_pixels":      sps.Width(),
+	}).Debug("RTSP sequence parameter set inspected")
 }

machinery/src/capture/gortsplib_test.go

@@ -0,0 +1,136 @@
+package capture
+
+import (
+	"bytes"
+	"context"
+	"encoding/pem"
+	"errors"
+	"math"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestPreRecordingGOPCount(t *testing.T) {
+	maxInt := int64(^uint(0) >> 1)
+	tests := []struct {
+		name         string
+		preRecording int64
+		gopDuration  float64
+		want         int
+		wantOK       bool
+	}{
+		{name: "normal duration", preRecording: 10, gopDuration: 2.9, want: 6, wantOK: true},
+		{name: "duration longer than buffer", preRecording: 1, gopDuration: 2, want: 1, wantOK: true},
+		{name: "largest representable result", preRecording: maxInt - 1, gopDuration: 1, want: int(maxInt), wantOK: true},
+		{name: "result exceeds int", preRecording: maxInt, gopDuration: 1, wantOK: false},
+		{name: "non-positive pre-recording", preRecording: 0, gopDuration: 1, wantOK: false},
+		{name: "sub-second GOP", preRecording: 10, gopDuration: 0.9, wantOK: false},
+		{name: "NaN GOP", preRecording: 10, gopDuration: math.NaN(), wantOK: false},
+		{name: "infinite GOP", preRecording: 10, gopDuration: math.Inf(1), wantOK: false},
+		{name: "GOP exceeds int64", preRecording: 10, gopDuration: float64(math.MaxInt64), wantOK: false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, ok := preRecordingGOPCount(tt.preRecording, tt.gopDuration)
+			if ok != tt.wantOK || got != tt.want {
+				t.Fatalf("preRecordingGOPCount(%d, %v) = (%d, %t), want (%d, %t)",
+					tt.preRecording, tt.gopDuration, got, ok, tt.want, tt.wantOK)
+			}
+		})
+	}
+}
+
+func TestGolibrtspCloseBeforeClientStart(t *testing.T) {
+	client := &Golibrtsp{}
+
+	if err := client.Close(context.Background()); err != nil {
+		t.Fatalf("Close() error = %v", err)
+	}
+}
+
+func TestSanitizeRTSPErrorRemovesCredentialsAndQuery(t *testing.T) {
+	rawURL := "rtsp://camera-user:camera-password@10.0.20.15/live?access_token=secret"
+	got := sanitizeRTSPError(errors.New("describe "+rawURL+": bad status code"), rawURL)
+
+	for _, secret := range []string{"camera-user", "camera-password", "access_token", "secret"} {
+		if strings.Contains(got.Error(), secret) {
+			t.Fatalf("sanitizeRTSPError() exposed %q in %q", secret, got)
+		}
+	}
+	if !strings.Contains(got.Error(), "rtsp://10.0.20.15/live") {
+		t.Fatalf("sanitizeRTSPError() removed useful host/path context: %q", got)
+	}
+}
+
+func TestRTSPSTLSConfig(t *testing.T) {
+	t.Run("verifies certificates by default", func(t *testing.T) {
+		t.Setenv(rtspsCAFileEnv, "")
+		t.Setenv(rtspsInsecureEnv, "")
+
+		got, err := rtspsTLSConfig()
+		if err != nil {
+			t.Fatalf("rtspsTLSConfig() error = %v", err)
+		}
+		if got != nil {
+			t.Fatalf("rtspsTLSConfig() = %#v, want nil", got)
+		}
+	})
+
+	t.Run("allows explicit insecure mode", func(t *testing.T) {
+		t.Setenv(rtspsCAFileEnv, "/missing/ignored-in-insecure-mode.pem")
+		t.Setenv(rtspsInsecureEnv, "true")
+
+		got, err := rtspsTLSConfig()
+		if err != nil {
+			t.Fatalf("rtspsTLSConfig() error = %v", err)
+		}
+		if got == nil || !got.InsecureSkipVerify {
+			t.Fatalf("rtspsTLSConfig() = %#v, want InsecureSkipVerify enabled", got)
+		}
+	})
+
+	t.Run("adds a camera CA to system roots", func(t *testing.T) {
+		t.Setenv(rtspsInsecureEnv, "")
+		server := httptest.NewTLSServer(nil)
+		defer server.Close()
+
+		certificate := server.Certificate()
+		caFile := filepath.Join(t.TempDir(), "camera-ca.pem")
+		caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certificate.Raw})
+		if err := os.WriteFile(caFile, caPEM, 0o600); err != nil {
+			t.Fatal(err)
+		}
+		t.Setenv(rtspsCAFileEnv, caFile)
+
+		got, err := rtspsTLSConfig()
+		if err != nil {
+			t.Fatalf("rtspsTLSConfig() error = %v", err)
+		}
+		if got == nil || got.RootCAs == nil {
+			t.Fatalf("rtspsTLSConfig() = %#v, want custom RootCAs", got)
+		}
+		for _, subject := range got.RootCAs.Subjects() {
+			if bytes.Equal(subject, certificate.RawSubject) {
+				return
+			}
+		}
+		t.Fatal("camera CA was not added to RootCAs")
+	})
+
+	t.Run("rejects an invalid camera CA file", func(t *testing.T) {
+		t.Setenv(rtspsInsecureEnv, "")
+		caFile := filepath.Join(t.TempDir(), "camera-ca.pem")
+		if err := os.WriteFile(caFile, []byte("not a certificate"), 0o600); err != nil {
+			t.Fatal(err)
+		}
+		t.Setenv(rtspsCAFileEnv, caFile)
+
+		if _, err := rtspsTLSConfig(); err == nil {
+			t.Fatal("rtspsTLSConfig() error = nil, want invalid CA error")
+		}
+	})
+}

machinery/src/capture/main.go

@@ -16,11 +16,11 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/kerberos-io/agent/machinery/src/conditions"
 	"github.com/kerberos-io/agent/machinery/src/encryption"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/kerberos-io/agent/machinery/src/utils"
 	"github.com/kerberos-io/agent/machinery/src/video"
+	log "github.com/sirupsen/logrus"
 	"go.opentelemetry.io/otel/trace"
 )
 
@@ -49,16 +49,17 @@ func publishRecordingState(mqttClient mqtt.Client, hubKey string, configuration
 	if err == nil {
 		mqttClient.Publish("kerberos/hub/"+hubKey, 2, false, payload)
 	} else {
-		log.Log.Error("capture.main.publishRecordingState(): failed to package MQTT message: " + err.Error())
+		log.Error("capture.main.publishRecordingState(): failed to package MQTT message: " + err.Error())
 	}
 }
 
-func recordingUploadMetadata(name, deviceKey string, timestamp int64, mp4Video *video.MP4) models.RecordingUploadMetadata {
+func recordingUploadMetadata(name, deviceKey string, timestamp int64, mp4Video *video.MP4, encrypted bool) models.RecordingUploadMetadata {
 	metadata := models.RecordingUploadMetadata{
 		FileName:  filepath.Base(name),
 		DeviceKey: deviceKey,
 		Timestamp: timestamp,
 		Duration:  mp4Video.VideoTotalDuration,
+		Encrypted: encrypted,
 	}
 	value := mp4Video.AverageFPS()
 	if value > 0 && value <= 240 && !math.IsInf(value, 0) && !math.IsNaN(value) {
@@ -72,7 +73,7 @@ func recordingUploadMetadata(name, deviceKey string, timestamp int64, mp4Video *
 func queueRecordingForUpload(configDirectory string, metadata models.RecordingUploadMetadata) {
 	payload, err := json.Marshal(metadata)
 	if err != nil {
-		log.Log.Error("capture.main.queueRecordingForUpload(): " + err.Error())
+		log.Error("capture.main.queueRecordingForUpload(): " + err.Error())
 		return
 	}
 
@@ -95,7 +96,7 @@ func queueRecordingForUpload(configDirectory string, metadata models.RecordingUp
 		err = os.Rename(marker.Name(), filepath.Join(configDirectory, "data", "cloud", models.RecordingUploadMetadataFileName(metadata.FileName)))
 	}
 	if err != nil {
-		log.Log.Error("capture.main.queueRecordingForUpload(): " + err.Error())
+		log.Error("capture.main.queueRecordingForUpload(): " + err.Error())
 	}
 }
 
@@ -137,9 +138,9 @@ func manualRecordingExpired(communication *models.Communication, now int64) bool
 		return false
 	}
 	if heartbeatExpired {
-		log.Log.Info("capture.main.HandleRecordStream(motiondetection): auto-stopping manual recording, no viewer heartbeat within timeout.")
+		log.Info("capture.main.HandleRecordStream(motiondetection): auto-stopping manual recording, no viewer heartbeat within timeout.")
 	} else {
-		log.Log.Info("capture.main.HandleRecordStream(motiondetection): auto-stopping manual recording, maximum duration reached.")
+		log.Info("capture.main.HandleRecordStream(motiondetection): auto-stopping manual recording, maximum duration reached.")
 	}
 	communication.IsRecordingManual.UnSet()
 	communication.RecordingManualHeartbeat.Store(0)
@@ -151,7 +152,7 @@ func manualRecordingExpired(communication *models.Communication, now int64) bool
 func CleanupRecordingDirectory(configDirectory string, configuration *models.Configuration) {
 	autoClean := configuration.Config.AutoClean
 	if autoClean != "true" {
-		log.Log.Info("HandleRecordStream: Autoclean disabled, nothing to do here.")
+		log.Info("HandleRecordStream: Autoclean disabled, nothing to do here.")
 		return
 	}
 
@@ -164,7 +165,7 @@ func CleanupRecordingDirectory(configDirectory string, configuration *models.Con
 	// disk while keeping a free-space reserve.
 	needsCleanup, err := recordingsNeedCleanup(recordingsDirectory, configuration)
 	if err != nil {
-		log.Log.Info("HandleRecordStream: something went wrong, " + err.Error())
+		log.Info("HandleRecordStream: something went wrong, " + err.Error())
 		return
 	}
 	if !needsCleanup {
@@ -185,12 +186,12 @@ func CleanupRecordingDirectory(configDirectory string, configuration *models.Con
 	// (otherwise a long outage would fill the disk and stop new recordings).
 	name, pending, err := pickRecordingToCleanup(recordingsDirectory, cloudDirectory)
 	if err != nil {
-		log.Log.Info("HandleRecordStream: something went wrong, " + err.Error())
+		log.Info("HandleRecordStream: something went wrong, " + err.Error())
 		return
 	}
 
 	if err := os.Remove(recordingsDirectory + "/" + name); err != nil {
-		log.Log.Info("HandleRecordStream: something went wrong, " + err.Error())
+		log.Info("HandleRecordStream: something went wrong, " + err.Error())
 		return
 	}
 
@@ -200,14 +201,14 @@ func CleanupRecordingDirectory(configDirectory string, configuration *models.Con
 		// that was never uploaded to keep recording new footage. Also remove the
 		// now-dangling upload marker so the upload loop doesn't keep trying to
 		// upload a file that no longer exists.
-		log.Log.Warning("HandleRecordStream: removed oldest recording as part of cleanup, but it was STILL PENDING UPLOAD (disk full of un-uploaded recordings) - " + recordingsDirectory + "/" + name)
+		log.Warn("HandleRecordStream: removed oldest recording as part of cleanup, but it was STILL PENDING UPLOAD (disk full of un-uploaded recordings) - " + recordingsDirectory + "/" + name)
 		for _, markerName := range uploadMarkerNames(name) {
 			if err := os.Remove(filepath.Join(cloudDirectory, markerName)); err != nil && !os.IsNotExist(err) {
-				log.Log.Info("HandleRecordStream: could not remove dangling upload marker " + markerName + ", " + err.Error())
+				log.Info("HandleRecordStream: could not remove dangling upload marker " + markerName + ", " + err.Error())
 			}
 		}
 	} else {
-		log.Log.Info("HandleRecordStream: removed oldest file as part of cleanup - " + recordingsDirectory + "/" + name)
+		log.Info("HandleRecordStream: removed oldest file as part of cleanup - " + recordingsDirectory + "/" + name)
 	}
 }
 
@@ -331,7 +332,7 @@ func recordingPendingUpload(cloudDirectory, recordingName string) bool {
 	return false
 }
 
-func HandleRecordStream(queue *packets.Queue, configDirectory string, configuration *models.Configuration, communication *models.Communication, rtspClient RTSPClient, mqttClient mqtt.Client) {
+func HandleRecordStream(queue *packets.Queue, configDirectory string, configuration *models.Configuration, communication *models.Communication, rtspClient RTSPClient, mqttClient mqtt.Client, motionEvents <-chan models.MotionDataPartial) {
 
 	config := configuration.Config
 	hubKey := config.HubKey
@@ -345,9 +346,9 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 	communication.RecordingManualHeartbeatSeen.UnSet()
 
 	if config.Capture.Recording == "false" {
-		log.Log.Info("capture.main.HandleRecordStream(): disabled, we will not record anything.")
+		log.Info("capture.main.HandleRecordStream(): disabled, we will not record anything.")
 	} else {
-		log.Log.Debug("capture.main.HandleRecordStream(): started")
+		log.Debug("capture.main.HandleRecordStream(): started")
 
 		preRecording := config.Capture.PreRecording * 1000
 		postRecording := config.Capture.PostRecording * 1000           // number of seconds to record.
@@ -360,7 +361,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 		}
 
 		if maxRecordingPeriod < preRecording+postRecording {
-			log.Log.Error("capture.main.HandleRecordStream(): maxRecordingPeriod is less than preRecording + postRecording, this is not allowed. Setting maxRecordingPeriod to preRecording + postRecording.")
+			log.Error("capture.main.HandleRecordStream(): maxRecordingPeriod is less than preRecording + postRecording, this is not allowed. Setting maxRecordingPeriod to preRecording + postRecording.")
 			maxRecordingPeriod = preRecording + postRecording
 		}
 
@@ -394,7 +395,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 			var name string
 
 			// Do not do anything!
-			log.Log.Info("capture.main.HandleRecordStream(continuous): start recording")
+			log.Info("capture.main.HandleRecordStream(continuous): start recording")
 
 			start := false
 
@@ -440,10 +441,10 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					}
 					if (videoCodec == "H264" && (len(mp4Video.SPSNALUs) == 0 || len(mp4Video.PPSNALUs) == 0)) ||
 						(videoCodec == "H265" && (len(mp4Video.VPSNALUs) == 0 || len(mp4Video.SPSNALUs) == 0 || len(mp4Video.PPSNALUs) == 0)) {
-						log.Log.Warning("capture.main.HandleRecordStream(continuous): closing MP4 without full parameter sets, moov may be incomplete")
+						log.Warn("capture.main.HandleRecordStream(continuous): closing MP4 without full parameter sets, moov may be incomplete")
 					}
 					mp4Video.Close(&config)
-					log.Log.Info("capture.main.HandleRecordStream(continuous): recording finished: file save: " + name)
+					log.Info("capture.main.HandleRecordStream(continuous): recording finished: file save: " + name)
 
 					// Cleanup muxer
 					start = false
@@ -467,7 +468,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 						oldName := name
 						name = s + ".mp4"
 						fullName = configDirectory + "/data/recordings/" + name
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
+						log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
 
 						// Rename the file to the new name.
 						err := os.Rename(
@@ -475,12 +476,13 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 							configDirectory+"/data/recordings/"+s+".mp4")
 
 						if err != nil {
-							log.Log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
+							log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
 						}
 					} else {
-						log.Log.Info("capture.main.HandleRecordStream(continuous): no video data recorded, not renaming file.")
+						log.Info("capture.main.HandleRecordStream(continuous): no video data recorded, not renaming file.")
 					}
 
+					encrypted := false
 					// Check if we need to encrypt the recording.
 					if config.Encryption != nil && config.Encryption.Enabled == "true" && config.Encryption.Recordings == "true" && config.Encryption.SymmetricKey != "" {
 						// reopen file into memory 'fullName'
@@ -492,17 +494,19 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 								// write back to file
 								err := os.WriteFile(fullName, []byte(encryptedContents), 0644)
 								if err != nil {
-									log.Log.Error("capture.main.HandleRecordStream(continuous): error writing file: " + err.Error())
+									log.Error("capture.main.HandleRecordStream(continuous): error writing file: " + err.Error())
+								} else {
+									encrypted = true
 								}
 							} else {
-								log.Log.Error("capture.main.HandleRecordStream(continuous): error encrypting file: " + err.Error())
+								log.Error("capture.main.HandleRecordStream(continuous): error encrypting file: " + err.Error())
 							}
 						} else {
-							log.Log.Error("capture.main.HandleRecordStream(continuous): error reading file: " + err.Error())
+							log.Error("capture.main.HandleRecordStream(continuous): error reading file: " + err.Error())
 						}
 					}
 
-					queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, startRecording, mp4Video))
+					queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, startRecording, mp4Video, encrypted))
 
 					recordingStatus = "idle"
 
@@ -520,7 +524,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					// We'll validate those conditions and if not valid we'll not do anything.
 					valid, err := conditions.Validate(loc, configuration)
 					if !valid && err != nil {
-						log.Log.Debug("capture.main.HandleRecordStream(continuous): " + err.Error() + ".")
+						log.Debug("capture.main.HandleRecordStream(continuous): " + err.Error() + ".")
 						time.Sleep(5 * time.Second)
 						continue
 					}
@@ -552,7 +556,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					fullName = configDirectory + "/data/recordings/" + name
 
 					// Running...
-					log.Log.Info("capture.main.HandleRecordStream(continuous): recording started")
+					log.Info("capture.main.HandleRecordStream(continuous): recording started")
 
 					// Get width and height from the camera.
 					width := configuration.Config.Capture.IPCamera.Width
@@ -564,7 +568,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					vpsNALUS := configuration.Config.Capture.IPCamera.VPSNALUs
 
 					if len(spsNALUS) == 0 || len(ppsNALUS) == 0 {
-						log.Log.Warning("capture.main.HandleRecordStream(continuous): missing SPS/PPS at recording start")
+						log.Warn("capture.main.HandleRecordStream(continuous): missing SPS/PPS at recording start")
 					}
 					// Create a video file, and set the dimensions.
 					mp4Video = video.NewMP4(fullName, spsNALUS, ppsNALUS, vpsNALUS, configuration.Config.Capture.MaxLengthRecording)
@@ -579,7 +583,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					if audioCodec == "AAC" {
 						audioTrack = mp4Video.AddAudioTrack("AAC")
 					} else if audioCodec == "PCM_MULAW" {
-						log.Log.Debug("capture.main.HandleRecordStream(continuous): no AAC audio codec detected, skipping audio track.")
+						log.Debug("capture.main.HandleRecordStream(continuous): no AAC audio codec detected, skipping audio track.")
 					}
 
 					writeSampleToMP4(mp4Video, videoTrack, audioTrack, pkt)
@@ -600,7 +604,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 			if cursorError != nil {
 				if recordingStatus == "started" {
 
-					log.Log.Info("capture.main.HandleRecordStream(continuous): Recording finished: file save: " + name)
+					log.Info("capture.main.HandleRecordStream(continuous): Recording finished: file save: " + name)
 
 					// Cleanup muxer
 					start = false
@@ -624,7 +628,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 						oldName := name
 						name = s + ".mp4"
 						fullName = configDirectory + "/data/recordings/" + name
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
+						log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
 
 						// Rename the file to the new name.
 						err := os.Rename(
@@ -632,12 +636,13 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 							configDirectory+"/data/recordings/"+s+".mp4")
 
 						if err != nil {
-							log.Log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
+							log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
 						}
 					} else {
-						log.Log.Info("capture.main.HandleRecordStream(continuous): no video data recorded, not renaming file.")
+						log.Info("capture.main.HandleRecordStream(continuous): no video data recorded, not renaming file.")
 					}
 
+					encrypted := false
 					// Check if we need to encrypt the recording.
 					if config.Encryption != nil && config.Encryption.Enabled == "true" && config.Encryption.Recordings == "true" && config.Encryption.SymmetricKey != "" {
 						// reopen file into memory 'fullName'
@@ -649,17 +654,19 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 								// write back to file
 								err := os.WriteFile(fullName, []byte(encryptedContents), 0644)
 								if err != nil {
-									log.Log.Error("capture.main.HandleRecordStream(motiondetection): error writing file: " + err.Error())
+									log.Error("capture.main.HandleRecordStream(motiondetection): error writing file: " + err.Error())
+								} else {
+									encrypted = true
 								}
 							} else {
-								log.Log.Error("capture.main.HandleRecordStream(motiondetection): error encrypting file: " + err.Error())
+								log.Error("capture.main.HandleRecordStream(motiondetection): error encrypting file: " + err.Error())
 							}
 						} else {
-							log.Log.Error("capture.main.HandleRecordStream(motiondetection): error reading file: " + err.Error())
+							log.Error("capture.main.HandleRecordStream(motiondetection): error reading file: " + err.Error())
 						}
 					}
 
-					queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, startRecording, mp4Video))
+					queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, startRecording, mp4Video, encrypted))
 
 					recordingStatus = "idle"
 
@@ -672,7 +679,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 			}
 		} else {
 
-			log.Log.Info("capture.main.HandleRecordStream(motiondetection): Start motion based recording ")
+			log.Info("capture.main.HandleRecordStream(motiondetection): Start motion based recording ")
 
 			var lastRecordingTime int64 = 0 // last recording timestamp in milliseconds
 			var displayTime int64 = 0       // display time in milliseconds
@@ -680,7 +687,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 			var videoTrack uint32
 			var audioTrack uint32
 
-			for motion := range communication.HandleMotion {
+			for motion := range motionEvents {
 
 				// Get as much packets we need.
 				var cursorError error
@@ -710,7 +717,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 
 				// If startRecording is 0, we will continue as it might be we are in a state of restarting the agent.
 				if startRecording == 0 {
-					log.Log.Info("capture.main.HandleRecordStream(motiondetection): startRecording is 0, we will continue as it might be we are in a state of restarting the agent.")
+					log.Info("capture.main.HandleRecordStream(motiondetection): startRecording is 0, we will continue as it might be we are in a state of restarting the agent.")
 					continue
 				}
 
@@ -748,7 +755,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 				fullName := configDirectory + "/data/recordings/" + name
 
 				// Running...
-				log.Log.Info("capture.main.HandleRecordStream(motiondetection): recording started (" + name + ")" + " at " + strconv.FormatInt(displayTimeSeconds, 10) + " unix")
+				log.Info("capture.main.HandleRecordStream(motiondetection): recording started (" + name + ")" + " at " + strconv.FormatInt(displayTimeSeconds, 10) + " unix")
 
 				// Get width and height from the camera.
 				width := configuration.Config.Capture.IPCamera.Width
@@ -760,7 +767,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 				vpsNALUS := configuration.Config.Capture.IPCamera.VPSNALUs
 
 				if len(spsNALUS) == 0 || len(ppsNALUS) == 0 {
-					log.Log.Warning("capture.main.HandleRecordStream(motiondetection): missing SPS/PPS at recording start")
+					log.Warn("capture.main.HandleRecordStream(motiondetection): missing SPS/PPS at recording start")
 				}
 				// Create the MP4 only once the first keyframe arrives.
 				var mp4Video *video.MP4
@@ -769,16 +776,16 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 
 					nextPkt, cursorError = recordingCursor.ReadPacket()
 					if cursorError != nil {
-						log.Log.Error("capture.main.HandleRecordStream(motiondetection): " + cursorError.Error())
+						log.Error("capture.main.HandleRecordStream(motiondetection): " + cursorError.Error())
 					}
 
 					now = time.Now().UnixMilli()
 					select {
-					case motion := <-communication.HandleMotion:
+					case motion := <-motionEvents:
 						motionTimestamp = now
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): motion detected while recording. Expanding recording.")
+						log.Info("capture.main.HandleRecordStream(motiondetection): motion detected while recording. Expanding recording.")
 						numberOfChanges := motion.NumberOfChanges
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): Received message with recording data, detected changes to save: " + strconv.Itoa(numberOfChanges))
+						log.Info("capture.main.HandleRecordStream(motiondetection): Received message with recording data, detected changes to save: " + strconv.Itoa(numberOfChanges))
 					default:
 					}
 
@@ -794,16 +801,16 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					}
 
 					if start && (motionTimestamp+postRecording-now < 0 || now-startRecording > maxRecordingPeriod-500) && nextPkt.IsKeyFrame {
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): timestamp+postRecording-now < 0  - " + strconv.FormatInt(motionTimestamp+postRecording-now, 10) + " < 0")
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): now-startRecording > maxRecordingPeriod-500 - " + strconv.FormatInt(now-startRecording, 10) + " > " + strconv.FormatInt(maxRecordingPeriod-500, 10))
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): closing recording (timestamp: " + strconv.FormatInt(motionTimestamp, 10) + ", postRecording: " + strconv.FormatInt(postRecording, 10) + ", now: " + strconv.FormatInt(now, 10) + ", startRecording: " + strconv.FormatInt(startRecording, 10) + ", maxRecordingPeriod: " + strconv.FormatInt(maxRecordingPeriod, 10))
+						log.Info("capture.main.HandleRecordStream(motiondetection): timestamp+postRecording-now < 0  - " + strconv.FormatInt(motionTimestamp+postRecording-now, 10) + " < 0")
+						log.Info("capture.main.HandleRecordStream(motiondetection): now-startRecording > maxRecordingPeriod-500 - " + strconv.FormatInt(now-startRecording, 10) + " > " + strconv.FormatInt(maxRecordingPeriod-500, 10))
+						log.Info("capture.main.HandleRecordStream(motiondetection): closing recording (timestamp: " + strconv.FormatInt(motionTimestamp, 10) + ", postRecording: " + strconv.FormatInt(postRecording, 10) + ", now: " + strconv.FormatInt(now, 10) + ", startRecording: " + strconv.FormatInt(startRecording, 10) + ", maxRecordingPeriod: " + strconv.FormatInt(maxRecordingPeriod, 10))
 						break
 					}
 					if pkt.IsKeyFrame && !start && pkt.CurrentTime >= startRecording {
 						// We start the recording if we have a keyframe and the last duration is 0 or less than the current packet time.
 						// It could be start we start from the beginning of the recording.
-						log.Log.Debug("capture.main.HandleRecordStream(motiondetection): write frames")
-						log.Log.Debug("capture.main.HandleRecordStream(motiondetection): recording started on keyframe")
+						log.Debug("capture.main.HandleRecordStream(motiondetection): write frames")
+						log.Debug("capture.main.HandleRecordStream(motiondetection): recording started on keyframe")
 
 						// Align duration timers with the first keyframe.
 						startRecording = pkt.CurrentTime
@@ -821,7 +828,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 						if audioCodec == "AAC" {
 							audioTrack = mp4Video.AddAudioTrack("AAC")
 						} else if audioCodec == "PCM_MULAW" {
-							log.Log.Debug("capture.main.HandleRecordStream(continuous): no AAC audio codec detected, skipping audio track.")
+							log.Debug("capture.main.HandleRecordStream(continuous): no AAC audio codec detected, skipping audio track.")
 						}
 						start = true
 
@@ -840,7 +847,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 				lastRecordingTime = pkt.CurrentTime
 
 				if mp4Video == nil {
-					log.Log.Warning("capture.main.HandleRecordStream(motiondetection): recording closed without keyframe; no MP4 created")
+					log.Warn("capture.main.HandleRecordStream(motiondetection): recording closed without keyframe; no MP4 created")
 					continue
 				}
 
@@ -856,10 +863,10 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 				}
 				if (videoCodec == "H264" && (len(mp4Video.SPSNALUs) == 0 || len(mp4Video.PPSNALUs) == 0)) ||
 					(videoCodec == "H265" && (len(mp4Video.VPSNALUs) == 0 || len(mp4Video.SPSNALUs) == 0 || len(mp4Video.PPSNALUs) == 0)) {
-					log.Log.Warning("capture.main.HandleRecordStream(motiondetection): closing MP4 without full parameter sets, moov may be incomplete")
+					log.Warn("capture.main.HandleRecordStream(motiondetection): closing MP4 without full parameter sets, moov may be incomplete")
 				}
 				mp4Video.Close(&config)
-				log.Log.Info("capture.main.HandleRecordStream(motiondetection): file save: " + name)
+				log.Info("capture.main.HandleRecordStream(motiondetection): file save: " + name)
 
 				// Notify the hub / live-view UI that this camera stopped recording.
 				publishRecordingState(mqttClient, hubKey, configuration, false)
@@ -870,10 +877,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 				// restart when the recording has expired (heartbeat lapsed or max
 				// duration reached), so it ends here instead of recording forever.
 				if communication.IsRecordingManual.IsSet() && !manualRecordingExpired(communication, time.Now().UnixMilli()) {
-					select {
-					case communication.HandleMotion <- models.MotionDataPartial{Timestamp: time.Now().Unix(), NumberOfChanges: 100000000}:
-					default:
-					}
+					communication.TrySendMotion(models.MotionDataPartial{Timestamp: time.Now().Unix(), NumberOfChanges: 100000000})
 				}
 
 				// Update the name of the recording with the duration.
@@ -894,7 +898,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					oldName := name
 					name = s + ".mp4"
 					fullName = configDirectory + "/data/recordings/" + name
-					log.Log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
+					log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
 
 					// Rename the file to the new name.
 					err := os.Rename(
@@ -902,12 +906,13 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 						configDirectory+"/data/recordings/"+s+".mp4")
 
 					if err != nil {
-						log.Log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
+						log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
 					}
 				} else {
-					log.Log.Info("capture.main.HandleRecordStream(motiondetection): no video data recorded, not renaming file.")
+					log.Info("capture.main.HandleRecordStream(motiondetection): no video data recorded, not renaming file.")
 				}
 
+				encrypted := false
 				// Check if we need to encrypt the recording.
 				if config.Encryption != nil && config.Encryption.Enabled == "true" && config.Encryption.Recordings == "true" && config.Encryption.SymmetricKey != "" {
 					// reopen file into memory 'fullName'
@@ -919,24 +924,26 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 							// write back to file
 							err := os.WriteFile(fullName, []byte(encryptedContents), 0644)
 							if err != nil {
-								log.Log.Error("capture.main.HandleRecordStream(motiondetection): error writing file: " + err.Error())
+								log.Error("capture.main.HandleRecordStream(motiondetection): error writing file: " + err.Error())
+							} else {
+								encrypted = true
 							}
 						} else {
-							log.Log.Error("capture.main.HandleRecordStream(motiondetection): error encrypting file: " + err.Error())
+							log.Error("capture.main.HandleRecordStream(motiondetection): error encrypting file: " + err.Error())
 						}
 					} else {
-						log.Log.Error("capture.main.HandleRecordStream(motiondetection): error reading file: " + err.Error())
+						log.Error("capture.main.HandleRecordStream(motiondetection): error reading file: " + err.Error())
 					}
 				}
 
-				queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, displayTime, mp4Video))
+				queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, displayTime, mp4Video, encrypted))
 
 				// Clean up the recording directory if necessary.
 				CleanupRecordingDirectory(configDirectory, configuration)
 			}
 		}
 
-		log.Log.Debug("capture.main.HandleRecordStream(): finished")
+		log.Debug("capture.main.HandleRecordStream(): finished")
 	}
 }
 
@@ -1030,13 +1037,14 @@ func Base64Image(captureDevice *Capture, communication *models.Communication, co
 	var cursor *packets.QueueCursor
 
 	// We'll pick the right client and decoder.
-	rtspClient := captureDevice.RTSPSubClient
+	rtspClient := captureDevice.SubClient()
 	if rtspClient != nil {
-		queue = communication.SubQueue
-		cursor = queue.Latest()
+		queue = communication.SubQueue()
 	} else {
-		rtspClient = captureDevice.RTSPClient
-		queue = communication.Queue
+		rtspClient = captureDevice.MainClient()
+		queue = communication.MainQueue()
+	}
+	if queue != nil {
 		cursor = queue.Latest()
 	}
 
@@ -1076,13 +1084,14 @@ func JpegImage(captureDevice *Capture, communication *models.Communication) imag
 	var cursor *packets.QueueCursor
 
 	// We'll pick the right client and decoder.
-	rtspClient := captureDevice.RTSPSubClient
+	rtspClient := captureDevice.SubClient()
 	if rtspClient != nil {
-		queue = communication.SubQueue
-		cursor = queue.Latest()
+		queue = communication.SubQueue()
 	} else {
-		rtspClient = captureDevice.RTSPClient
-		queue = communication.Queue
+		rtspClient = captureDevice.MainClient()
+		queue = communication.MainQueue()
+	}
+	if queue != nil {
 		cursor = queue.Latest()
 	}
 
@@ -1137,20 +1146,16 @@ func writeSampleToMP4(mp4Video *video.MP4, videoTrack, audioTrack uint32, pkt pa
 			dts = pts - uint64(compositionOffset)
 		}
 		if err := mp4Video.AddSampleToTrack(videoTrack, pkt.IsKeyFrame, pkt.Data, dts, compositionOffset); err != nil {
-			log.Log.Error("capture.main.writeSampleToMP4(): " + err.Error())
+			log.Error("capture.main.writeSampleToMP4(): " + err.Error())
 		}
 	} else if pkt.IsAudio {
 		if pkt.Codec == "AAC" {
 			if err := mp4Video.AddSampleToTrack(audioTrack, pkt.IsKeyFrame, pkt.Data, pts, 0); err != nil {
-				log.Log.Error("capture.main.writeSampleToMP4(): " + err.Error())
+				log.Error("capture.main.writeSampleToMP4(): " + err.Error())
 			}
 		} else if pkt.Codec == "PCM_MULAW" {
 			// TODO: transcode to AAC, some work to do..
-			log.Log.Debug("capture.main.writeSampleToMP4(): no AAC audio codec detected, skipping audio track.")
+			log.Debug("capture.main.writeSampleToMP4(): no AAC audio codec detected, skipping audio track.")
 		}
 	}
 }
-
-/*func convertPTS2(v int64) uint64 {
-	return uint64(v) / 100
-}*/

machinery/src/capture/main_test.go

@@ -6,11 +6,35 @@ import (
 	"os"
 	"path/filepath"
 	"testing"
+	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/video"
 )
 
+func TestPTSToDuration(t *testing.T) {
+	tests := []struct {
+		name      string
+		pts       int64
+		clockRate int
+		want      time.Duration
+	}{
+		{name: "one video second", pts: 90_000, clockRate: 90_000, want: time.Second},
+		{name: "one audio frame", pts: 1_024, clockRate: 8_000, want: 128 * time.Millisecond},
+		{name: "fractional millisecond", pts: 45_045, clockRate: 90_000, want: 500*time.Millisecond + 500*time.Microsecond},
+		{name: "negative timestamp", pts: -45_045, clockRate: 90_000, want: -500*time.Millisecond - 500*time.Microsecond},
+		{name: "large timestamp", pts: 90_000 * 60 * 60 * 24, clockRate: 90_000, want: 24 * time.Hour},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if got := ptsToDuration(test.pts, test.clockRate); got != test.want {
+				t.Fatalf("ptsToDuration(%d, %d) = %s, want %s", test.pts, test.clockRate, got, test.want)
+			}
+		})
+	}
+}
+
 func TestQueueRecordingForUploadStoresFinalizedMetadata(t *testing.T) {
 	configDirectory := t.TempDir()
 	if err := os.MkdirAll(filepath.Join(configDirectory, "data", "cloud"), 0o755); err != nil {
@@ -18,7 +42,7 @@ func TestQueueRecordingForUploadStoresFinalizedMetadata(t *testing.T) {
 	}
 
 	mp4Video := &video.MP4{VideoTotalDuration: 20452, SampleCount: 613}
-	metadata := recordingUploadMetadata("recording.mp4", "device-key", 1785934709414, mp4Video)
+	metadata := recordingUploadMetadata("recording.mp4", "device-key", 1785934709414, mp4Video, true)
 	queueRecordingForUpload(configDirectory, metadata)
 
 	got, err := os.ReadFile(filepath.Join(configDirectory, "data", "cloud", "recording.metadata"))
@@ -30,7 +54,7 @@ func TestQueueRecordingForUploadStoresFinalizedMetadata(t *testing.T) {
 		t.Fatalf("decode upload marker: %v", err)
 	}
 	expectedFPS := mp4Video.AverageFPS()
-	if stored.FileName != "recording.mp4" || stored.DeviceKey != "device-key" || stored.Timestamp != 1785934709414 || stored.Duration != 20452 || math.Abs(stored.FPS-expectedFPS) > 1e-9 {
+	if stored.FileName != "recording.mp4" || stored.DeviceKey != "device-key" || stored.Timestamp != 1785934709414 || stored.Duration != 20452 || math.Abs(stored.FPS-expectedFPS) > 1e-9 || !stored.Encrypted {
 		t.Fatalf("upload marker = %+v", stored)
 	}
 	if stored.FPS == math.Floor(stored.FPS) {

machinery/src/capture/rtsp_client.go

@@ -3,36 +3,73 @@ package capture
 import (
 	"context"
 	"image"
+	"sync"
 
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 )
 
 type Capture struct {
-	RTSPClient            *Golibrtsp
-	RTSPSubClient         *Golibrtsp
-	RTSPBackChannelClient *Golibrtsp
+	clientsMu             sync.RWMutex
+	rtspClient            *Golibrtsp
+	rtspSubClient         *Golibrtsp
+	rtspBackChannelClient *Golibrtsp
 }
 
 func (c *Capture) SetMainClient(rtspUrl string) *Golibrtsp {
-	c.RTSPClient = &Golibrtsp{
+	client := &Golibrtsp{
 		Url: rtspUrl,
 	}
-	return c.RTSPClient
+	c.clientsMu.Lock()
+	c.rtspClient = client
+	c.clientsMu.Unlock()
+	return client
 }
 
 func (c *Capture) SetSubClient(rtspUrl string) *Golibrtsp {
-	c.RTSPSubClient = &Golibrtsp{
+	client := &Golibrtsp{
 		Url: rtspUrl,
 	}
-	return c.RTSPSubClient
+	c.clientsMu.Lock()
+	c.rtspSubClient = client
+	c.clientsMu.Unlock()
+	return client
 }
 
 func (c *Capture) SetBackChannelClient(rtspUrl string) *Golibrtsp {
-	c.RTSPBackChannelClient = &Golibrtsp{
+	client := &Golibrtsp{
 		Url: rtspUrl,
 	}
-	return c.RTSPBackChannelClient
+	c.clientsMu.Lock()
+	c.rtspBackChannelClient = client
+	c.clientsMu.Unlock()
+	return client
+}
+
+func (c *Capture) MainClient() *Golibrtsp {
+	c.clientsMu.RLock()
+	defer c.clientsMu.RUnlock()
+	return c.rtspClient
+}
+
+func (c *Capture) SubClient() *Golibrtsp {
+	c.clientsMu.RLock()
+	defer c.clientsMu.RUnlock()
+	return c.rtspSubClient
+}
+
+func (c *Capture) ClearClients(main, sub, backchannel *Golibrtsp) {
+	c.clientsMu.Lock()
+	defer c.clientsMu.Unlock()
+	if c.rtspClient == main {
+		c.rtspClient = nil
+	}
+	if c.rtspSubClient == sub {
+		c.rtspSubClient = nil
+	}
+	if c.rtspBackChannelClient == backchannel {
+		c.rtspBackChannelClient = nil
+	}
 }
 
 // RTSPClient is a interface that abstracts the RTSP client implementation.

machinery/src/capture/rtsp_client_test.go

@@ -0,0 +1,62 @@
+package capture
+
+import (
+	"strconv"
+	"sync"
+	"testing"
+)
+
+func TestCaptureClientAccessCanRaceReplacement(t *testing.T) {
+	captureDevice := &Capture{}
+	captureDevice.SetMainClient("rtsp://main/0")
+	captureDevice.SetSubClient("rtsp://sub/0")
+
+	var workers sync.WaitGroup
+	workers.Add(2)
+	go func() {
+		defer workers.Done()
+		for replacement := 1; replacement <= 1000; replacement++ {
+			suffix := strconv.Itoa(replacement)
+			captureDevice.SetMainClient("rtsp://main/" + suffix)
+			captureDevice.SetSubClient("rtsp://sub/" + suffix)
+		}
+	}()
+	go func() {
+		defer workers.Done()
+		for snapshot := 0; snapshot < 1000; snapshot++ {
+			if captureDevice.MainClient() == nil {
+				t.Error("MainClient() returned nil")
+				return
+			}
+			if captureDevice.SubClient() == nil {
+				t.Error("SubClient() returned nil")
+				return
+			}
+		}
+	}()
+	workers.Wait()
+}
+
+func TestCaptureClearClientsOnlyClearsMatchingRun(t *testing.T) {
+	captureDevice := &Capture{}
+	oldMain := captureDevice.SetMainClient("rtsp://main/old")
+	oldSub := captureDevice.SetSubClient("rtsp://sub/old")
+	oldBackchannel := captureDevice.SetBackChannelClient("rtsp://back/old")
+
+	newMain := captureDevice.SetMainClient("rtsp://main/new")
+	newSub := captureDevice.SetSubClient("rtsp://sub/new")
+	newBackchannel := captureDevice.SetBackChannelClient("rtsp://back/new")
+
+	captureDevice.ClearClients(oldMain, oldSub, oldBackchannel)
+	if captureDevice.MainClient() != newMain {
+		t.Fatal("stale cleanup cleared the new main client")
+	}
+	if captureDevice.SubClient() != newSub {
+		t.Fatal("stale cleanup cleared the new sub client")
+	}
+
+	captureDevice.ClearClients(newMain, newSub, newBackchannel)
+	if captureDevice.MainClient() != nil || captureDevice.SubClient() != nil {
+		t.Fatal("matching cleanup did not clear current clients")
+	}
+}

machinery/src/cloud/cloud_test.go

@@ -0,0 +1,592 @@
+package cloud
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/kerberos-io/agent/machinery/src/models"
+	agentonvif "github.com/kerberos-io/agent/machinery/src/onvif"
+	goonvif "github.com/kerberos-io/onvif"
+	goonvifdevice "github.com/kerberos-io/onvif/device"
+	goonvifptz "github.com/kerberos-io/onvif/ptz"
+)
+
+func TestHeartbeatFailureLogOmitsHubResponseBody(t *testing.T) {
+	response := &http.Response{
+		StatusCode: http.StatusBadRequest,
+		Status:     "400 Bad Request",
+		Body:       io.NopCloser(strings.NewReader(`{"error":"invalid heartbeat"}`)),
+	}
+
+	responseBody, truncated, err := readHeartbeatResponseBody(response)
+	if err != nil {
+		t.Fatalf("readHeartbeatResponseBody() error = %v", err)
+	}
+	fields := heartbeatFailureLogFields(response, responseBody, truncated, 125*time.Millisecond)
+
+	for key, want := range map[string]interface{}{
+		"duration_ms":             int64(125),
+		"response_body_bytes":     len(responseBody),
+		"response_body_truncated": false,
+		"status_code":             http.StatusBadRequest,
+	} {
+		if got := fields[key]; got != want {
+			t.Errorf("%s = %v, want %v", key, got, want)
+		}
+	}
+	for key, value := range fields {
+		if strings.Contains(key, "response_body") && key != "response_body_bytes" && key != "response_body_truncated" {
+			t.Fatalf("unexpected response body field %q=%v", key, value)
+		}
+		if strings.Contains(fmt.Sprint(value), "invalid heartbeat") {
+			t.Fatalf("heartbeat log fields exposed response body in %q=%v", key, value)
+		}
+	}
+}
+
+func TestReadHeartbeatResponseBodyTruncatesLargeBody(t *testing.T) {
+	response := &http.Response{
+		Body: io.NopCloser(strings.NewReader(strings.Repeat("x", heartbeatResponseBodyLogLimit+1))),
+	}
+
+	body, truncated, err := readHeartbeatResponseBody(response)
+	if err != nil {
+		t.Fatalf("readHeartbeatResponseBody() error = %v", err)
+	}
+	if !truncated {
+		t.Fatal("readHeartbeatResponseBody() truncated = false, want true")
+	}
+	if len(body) != heartbeatResponseBodyLogLimit {
+		t.Fatalf("len(body) = %d, want %d", len(body), heartbeatResponseBodyLogLimit)
+	}
+}
+
+func TestHeartbeatONVIFPayloadCachesStaticAndReusesLoopSubscription(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	device := newTestONVIFDevice()
+	camera := models.IPCamera{
+		ONVIFXAddr:    "http://camera/onvif",
+		ONVIFUsername: "operator",
+		ONVIFPassword: "secret",
+	}
+	initialEvents := []agentonvif.ONVIFEvents{{Key: "input-1", Type: "input", Value: "true", Timestamp: 1}}
+	loopEvents := []agentonvif.ONVIFEvents{{Key: "output-1", Type: "output", Value: "false", Timestamp: 2}}
+	wantPresets := mustJSONMarshal(t, []models.OnvifActionPreset{{Name: "Lobby", Token: "1"}})
+	wantInitialEvents := mustJSONMarshal(t, initialEvents)
+	wantLoopEvents := mustJSONMarshal(t, loopEvents)
+
+	var connectCalls, ptzConfigCalls, ptzFunctionCalls, presetCalls, createCalls, eventCalls, unsubscribeCalls int
+
+	heartbeatConnectToONVIFDevice = func(*models.IPCamera) (*goonvif.Device, goonvifdevice.GetCapabilitiesResponse, error) {
+		connectCalls++
+		return device, goonvifdevice.GetCapabilitiesResponse{}, nil
+	}
+	heartbeatGetPTZConfigurationsFromDevice = func(*goonvif.Device) (goonvifptz.GetConfigurationsResponse, error) {
+		ptzConfigCalls++
+		return goonvifptz.GetConfigurationsResponse{}, nil
+	}
+	heartbeatGetPTZFunctionsFromDevice = func(goonvifptz.GetConfigurationsResponse) ([]string, bool, bool) {
+		ptzFunctionCalls++
+		return nil, true, true
+	}
+	heartbeatGetPresetsFromDevice = func(*goonvif.Device) ([]models.OnvifActionPreset, error) {
+		presetCalls++
+		return []models.OnvifActionPreset{{Name: "Lobby", Token: "1"}}, nil
+	}
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		createCalls++
+		switch createCalls {
+		case 1:
+			return "initial-1", nil
+		case 2:
+			return "loop", nil
+		case 3:
+			return "initial-2", nil
+		default:
+			t.Fatalf("unexpected create pull point call %d", createCalls)
+			return "", nil
+		}
+	}
+	heartbeatGetEventMessages = func(_ *goonvif.Device, pullPointAddress string) ([]agentonvif.ONVIFEvents, error) {
+		eventCalls++
+		switch pullPointAddress {
+		case "initial-1", "initial-2":
+			return initialEvents, nil
+		case "loop":
+			return loopEvents, nil
+		default:
+			t.Fatalf("unexpected pull point address %q", pullPointAddress)
+			return nil, nil
+		}
+	}
+	heartbeatUnsubscribePullPoint = func(_ *goonvif.Device, pullPointAddress string) error {
+		unsubscribeCalls++
+		if pullPointAddress != "initial-1" && pullPointAddress != "initial-2" {
+			t.Fatalf("unexpected unsubscribe pull point %q", pullPointAddress)
+		}
+		return nil
+	}
+
+	state := newHeartbeatONVIFState()
+
+	payload := getHeartbeatONVIFPayload(camera, state)
+	if payload.enabled != "true" || payload.zoom != "true" || payload.panTilt != "true" || payload.presets != "true" {
+		t.Fatalf("unexpected static payload: %+v", payload)
+	}
+	if string(payload.presetsList) != string(wantPresets) {
+		t.Fatalf("payload.presetsList = %s, want %s", payload.presetsList, wantPresets)
+	}
+	if string(payload.eventsList) != string(wantInitialEvents) {
+		t.Fatalf("payload.eventsList = %s, want %s", payload.eventsList, wantInitialEvents)
+	}
+	if connectCalls != 1 || ptzConfigCalls != 1 || ptzFunctionCalls != 1 || presetCalls != 1 {
+		t.Fatalf("unexpected static call counts after first cycle: connect=%d ptzConfig=%d ptzFunctions=%d presets=%d", connectCalls, ptzConfigCalls, ptzFunctionCalls, presetCalls)
+	}
+	if createCalls != 2 || eventCalls != 1 || unsubscribeCalls != 1 {
+		t.Fatalf("unexpected event call counts after first cycle: create=%d events=%d unsubscribe=%d", createCalls, eventCalls, unsubscribeCalls)
+	}
+
+	payload = getHeartbeatONVIFPayload(camera, state)
+	if string(payload.eventsList) != string(wantLoopEvents) {
+		t.Fatalf("second payload.eventsList = %s, want %s", payload.eventsList, wantLoopEvents)
+	}
+	if connectCalls != 1 {
+		t.Fatalf("connectCalls = %d, want 1", connectCalls)
+	}
+	if ptzConfigCalls != 1 || ptzFunctionCalls != 1 || presetCalls != 1 {
+		t.Fatalf("static calls were not cached: ptzConfig=%d ptzFunctions=%d presets=%d", ptzConfigCalls, ptzFunctionCalls, presetCalls)
+	}
+	if createCalls != 3 || eventCalls != 3 || unsubscribeCalls != 2 {
+		t.Fatalf("temporary state subscription was not refreshed or loop subscription was not reused: create=%d events=%d unsubscribe=%d", createCalls, eventCalls, unsubscribeCalls)
+	}
+}
+
+func TestHeartbeatONVIFPayloadRefreshesAfterCameraConfigChange(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	deviceA := newTestONVIFDevice()
+	deviceB := newTestONVIFDevice()
+	cameraA := models.IPCamera{ONVIFXAddr: "http://camera-a/onvif", ONVIFUsername: "user", ONVIFPassword: "secret-a"}
+	cameraB := models.IPCamera{ONVIFXAddr: "http://camera-b/onvif", ONVIFUsername: "user", ONVIFPassword: "secret-b"}
+	var connectCalls, ptzConfigCalls, presetCalls, createCalls int
+	var unsubscribed []string
+
+	heartbeatConnectToONVIFDevice = func(camera *models.IPCamera) (*goonvif.Device, goonvifdevice.GetCapabilitiesResponse, error) {
+		connectCalls++
+		switch camera.ONVIFXAddr {
+		case cameraA.ONVIFXAddr:
+			return deviceA, goonvifdevice.GetCapabilitiesResponse{}, nil
+		case cameraB.ONVIFXAddr:
+			return deviceB, goonvifdevice.GetCapabilitiesResponse{}, nil
+		default:
+			t.Fatalf("unexpected camera address %q", camera.ONVIFXAddr)
+			return nil, goonvifdevice.GetCapabilitiesResponse{}, nil
+		}
+	}
+	heartbeatGetPTZConfigurationsFromDevice = func(*goonvif.Device) (goonvifptz.GetConfigurationsResponse, error) {
+		ptzConfigCalls++
+		return goonvifptz.GetConfigurationsResponse{}, nil
+	}
+	heartbeatGetPTZFunctionsFromDevice = func(goonvifptz.GetConfigurationsResponse) ([]string, bool, bool) {
+		return nil, false, true
+	}
+	heartbeatGetPresetsFromDevice = func(*goonvif.Device) ([]models.OnvifActionPreset, error) {
+		presetCalls++
+		return nil, nil
+	}
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		createCalls++
+		switch createCalls {
+		case 1:
+			return "initial-a", nil
+		case 2:
+			return "loop-a", nil
+		case 3:
+			return "initial-b", nil
+		case 4:
+			return "loop-b", nil
+		default:
+			t.Fatalf("unexpected create pull point call %d", createCalls)
+			return "", nil
+		}
+	}
+	heartbeatGetEventMessages = func(_ *goonvif.Device, pullPointAddress string) ([]agentonvif.ONVIFEvents, error) {
+		switch pullPointAddress {
+		case "initial-a":
+			return []agentonvif.ONVIFEvents{{Key: "a", Type: "input", Value: "true", Timestamp: 1}}, nil
+		case "initial-b":
+			return []agentonvif.ONVIFEvents{{Key: "b", Type: "input", Value: "false", Timestamp: 2}}, nil
+		default:
+			t.Fatalf("unexpected pull point address %q", pullPointAddress)
+			return nil, nil
+		}
+	}
+	heartbeatUnsubscribePullPoint = func(_ *goonvif.Device, pullPointAddress string) error {
+		unsubscribed = append(unsubscribed, pullPointAddress)
+		return nil
+	}
+
+	state := newHeartbeatONVIFState()
+	_ = getHeartbeatONVIFPayload(cameraA, state)
+	payload := getHeartbeatONVIFPayload(cameraB, state)
+
+	if connectCalls != 2 {
+		t.Fatalf("connectCalls = %d, want 2", connectCalls)
+	}
+	if ptzConfigCalls != 2 || presetCalls != 2 {
+		t.Fatalf("camera config change did not refresh static state: ptzConfig=%d presets=%d", ptzConfigCalls, presetCalls)
+	}
+	if createCalls != 4 {
+		t.Fatalf("camera config change did not recreate subscriptions: create=%d", createCalls)
+	}
+	if countString(unsubscribed, "loop-a") != 1 {
+		t.Fatalf("unsubscribed loop-a %d times, want 1; unsubscribed=%v", countString(unsubscribed, "loop-a"), unsubscribed)
+	}
+	wantEvents := mustJSONMarshal(t, []agentonvif.ONVIFEvents{{Key: "b", Type: "input", Value: "false", Timestamp: 2}})
+	if string(payload.eventsList) != string(wantEvents) {
+		t.Fatalf("payload.eventsList = %s, want %s", payload.eventsList, wantEvents)
+	}
+}
+
+func TestHeartbeatONVIFPayloadRetriesStaticFetchFailuresWithoutReconnect(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	device := newTestONVIFDevice()
+	camera := models.IPCamera{
+		ONVIFXAddr:    "http://camera/onvif",
+		ONVIFUsername: "operator",
+		ONVIFPassword: "secret",
+	}
+	var connectCalls, ptzConfigCalls, presetCalls, createCalls, eventCalls int
+
+	heartbeatConnectToONVIFDevice = func(*models.IPCamera) (*goonvif.Device, goonvifdevice.GetCapabilitiesResponse, error) {
+		connectCalls++
+		return device, goonvifdevice.GetCapabilitiesResponse{}, nil
+	}
+	heartbeatGetPTZConfigurationsFromDevice = func(*goonvif.Device) (goonvifptz.GetConfigurationsResponse, error) {
+		ptzConfigCalls++
+		return goonvifptz.GetConfigurationsResponse{}, nil
+	}
+	heartbeatGetPTZFunctionsFromDevice = func(goonvifptz.GetConfigurationsResponse) ([]string, bool, bool) {
+		return nil, true, true
+	}
+	heartbeatGetPresetsFromDevice = func(*goonvif.Device) ([]models.OnvifActionPreset, error) {
+		presetCalls++
+		if presetCalls == 1 {
+			return nil, errors.New("temporary preset failure")
+		}
+		return []models.OnvifActionPreset{{Name: "Lobby", Token: "1"}}, nil
+	}
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		createCalls++
+		switch createCalls {
+		case 1:
+			return "initial-1", nil
+		case 2:
+			return "loop", nil
+		case 3:
+			return "initial-2", nil
+		default:
+			t.Fatalf("unexpected create pull point call %d", createCalls)
+			return "", nil
+		}
+	}
+	heartbeatGetEventMessages = func(_ *goonvif.Device, pullPointAddress string) ([]agentonvif.ONVIFEvents, error) {
+		eventCalls++
+		switch pullPointAddress {
+		case "initial-1", "initial-2":
+			return []agentonvif.ONVIFEvents{{Key: "one", Type: "input", Value: "true", Timestamp: 1}}, nil
+		case "loop":
+			return []agentonvif.ONVIFEvents{{Key: "two", Type: "output", Value: "false", Timestamp: 2}}, nil
+		default:
+			t.Fatalf("unexpected pull point address %q", pullPointAddress)
+			return nil, nil
+		}
+	}
+	heartbeatUnsubscribePullPoint = func(*goonvif.Device, string) error { return nil }
+
+	state := newHeartbeatONVIFState()
+	payload := getHeartbeatONVIFPayload(camera, state)
+	if payload.presets != "false" {
+		t.Fatalf("payload.presets = %q, want false on transient preset failure", payload.presets)
+	}
+	payload = getHeartbeatONVIFPayload(camera, state)
+	if payload.presets != "true" {
+		t.Fatalf("payload.presets = %q, want true after retry", payload.presets)
+	}
+	if connectCalls != 1 {
+		t.Fatalf("connectCalls = %d, want 1", connectCalls)
+	}
+	if ptzConfigCalls != 2 || presetCalls != 2 {
+		t.Fatalf("static failures were not retried on heartbeat cadence: ptzConfig=%d presets=%d", ptzConfigCalls, presetCalls)
+	}
+	if createCalls != 3 || eventCalls != 3 {
+		t.Fatalf("unexpected event behavior during retry: create=%d events=%d", createCalls, eventCalls)
+	}
+}
+
+func TestHeartbeatONVIFStateReleasesSubscriptionWhenDisabled(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	device := newTestONVIFDevice()
+	state := newHeartbeatONVIFState()
+	state.cameraConfiguration = models.IPCamera{ONVIFXAddr: "http://camera/onvif"}
+	state.cameraKey = heartbeatONVIFCameraKey(state.cameraConfiguration)
+	state.device = device
+	state.loopPullPoint = "loop"
+
+	var unsubscribed []string
+	heartbeatUnsubscribePullPoint = func(gotDevice *goonvif.Device, pullPointAddress string) error {
+		if gotDevice != device {
+			t.Fatal("unsubscribe used a different ONVIF device")
+		}
+		unsubscribed = append(unsubscribed, pullPointAddress)
+		return nil
+	}
+
+	state.prepare(models.IPCamera{})
+
+	if len(unsubscribed) != 1 || unsubscribed[0] != "loop" {
+		t.Fatalf("unsubscribed = %v, want [loop]", unsubscribed)
+	}
+	if state.device != nil || state.loopPullPoint != "" {
+		t.Fatalf("disabled state retained device or pull point: %+v", state)
+	}
+}
+
+func TestHeartbeatONVIFPayloadDoesNotCreateSubscriptionsWhenConnectFails(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	var createCalls int
+
+	heartbeatConnectToONVIFDevice = func(*models.IPCamera) (*goonvif.Device, goonvifdevice.GetCapabilitiesResponse, error) {
+		return nil, goonvifdevice.GetCapabilitiesResponse{}, errors.New("connect failed")
+	}
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		createCalls++
+		return "unexpected", nil
+	}
+
+	payload := getHeartbeatONVIFPayload(models.IPCamera{
+		ONVIFXAddr:    "http://camera/onvif",
+		ONVIFUsername: "operator",
+		ONVIFPassword: "secret",
+	}, newHeartbeatONVIFState())
+
+	if createCalls != 0 {
+		t.Fatalf("createCalls = %d, want 0", createCalls)
+	}
+	assertDefaultHeartbeatONVIFPayload(t, payload)
+}
+
+func TestHeartbeatONVIFPayloadReconnectFailureInvalidatesCache(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	device1 := newTestONVIFDevice()
+	device2 := newTestONVIFDevice()
+	camera := models.IPCamera{
+		ONVIFXAddr:    "http://camera/onvif",
+		ONVIFUsername: "operator",
+		ONVIFPassword: "secret",
+	}
+	var connectCalls, ptzConfigCalls, presetCalls, createCalls int
+	var unsubscribed []string
+
+	heartbeatConnectToONVIFDevice = func(*models.IPCamera) (*goonvif.Device, goonvifdevice.GetCapabilitiesResponse, error) {
+		connectCalls++
+		switch connectCalls {
+		case 1:
+			return device1, goonvifdevice.GetCapabilitiesResponse{}, nil
+		case 2:
+			return device2, goonvifdevice.GetCapabilitiesResponse{}, nil
+		default:
+			t.Fatalf("unexpected connect call %d", connectCalls)
+			return nil, goonvifdevice.GetCapabilitiesResponse{}, nil
+		}
+	}
+	heartbeatGetPTZConfigurationsFromDevice = func(*goonvif.Device) (goonvifptz.GetConfigurationsResponse, error) {
+		ptzConfigCalls++
+		return goonvifptz.GetConfigurationsResponse{}, nil
+	}
+	heartbeatGetPTZFunctionsFromDevice = func(goonvifptz.GetConfigurationsResponse) ([]string, bool, bool) {
+		return nil, true, false
+	}
+	heartbeatGetPresetsFromDevice = func(*goonvif.Device) ([]models.OnvifActionPreset, error) {
+		presetCalls++
+		return []models.OnvifActionPreset{{Name: "Preset", Token: "1"}}, nil
+	}
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		createCalls++
+		switch createCalls {
+		case 1:
+			return "initial-1", nil
+		case 2:
+			return "loop-1", nil
+		case 3:
+			return "initial-2", nil
+		case 4:
+			return "initial-3", nil
+		case 5:
+			return "loop-3", nil
+		default:
+			t.Fatalf("unexpected create pull point call %d", createCalls)
+			return "", nil
+		}
+	}
+	heartbeatGetEventMessages = func(_ *goonvif.Device, pullPointAddress string) ([]agentonvif.ONVIFEvents, error) {
+		switch pullPointAddress {
+		case "initial-1":
+			return []agentonvif.ONVIFEvents{{Key: "before", Type: "input", Value: "true", Timestamp: 1}}, nil
+		case "initial-2":
+			return []agentonvif.ONVIFEvents{{Key: "during", Type: "input", Value: "true", Timestamp: 2}}, nil
+		case "loop-1":
+			return nil, errors.New("pull failed")
+		case "initial-3":
+			return []agentonvif.ONVIFEvents{{Key: "after", Type: "input", Value: "false", Timestamp: 2}}, nil
+		default:
+			t.Fatalf("unexpected pull point address %q", pullPointAddress)
+			return nil, nil
+		}
+	}
+	heartbeatUnsubscribePullPoint = func(_ *goonvif.Device, pullPointAddress string) error {
+		unsubscribed = append(unsubscribed, pullPointAddress)
+		return nil
+	}
+
+	state := newHeartbeatONVIFState()
+	_ = getHeartbeatONVIFPayload(camera, state)
+	payload := getHeartbeatONVIFPayload(camera, state)
+	wantEventsDuringFailure := mustJSONMarshal(t, []agentonvif.ONVIFEvents{{Key: "during", Type: "input", Value: "true", Timestamp: 2}})
+	if string(payload.eventsList) != string(wantEventsDuringFailure) {
+		t.Fatalf("payload.eventsList after operation failure = %s, want %s", payload.eventsList, wantEventsDuringFailure)
+	}
+	payload = getHeartbeatONVIFPayload(camera, state)
+
+	if ptzConfigCalls != 2 || presetCalls != 2 {
+		t.Fatalf("reconnect did not refresh static state: ptzConfig=%d presets=%d", ptzConfigCalls, presetCalls)
+	}
+	if connectCalls != 2 {
+		t.Fatalf("connectCalls = %d, want 2 after reconnect", connectCalls)
+	}
+	if createCalls != 5 {
+		t.Fatalf("reconnect did not recreate subscriptions: create=%d", createCalls)
+	}
+	if countString(unsubscribed, "loop-1") != 1 {
+		t.Fatalf("operation failure cleanup mismatch, unsubscribed=%v", unsubscribed)
+	}
+	wantEvents := mustJSONMarshal(t, []agentonvif.ONVIFEvents{{Key: "after", Type: "input", Value: "false", Timestamp: 2}})
+	if string(payload.eventsList) != string(wantEvents) {
+		t.Fatalf("payload.eventsList = %s, want %s", payload.eventsList, wantEvents)
+	}
+}
+
+func TestHeartbeatONVIFPayloadKeepsCachedConnectionWhenInitialStateFetchFails(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	device := newTestONVIFDevice()
+	camera := models.IPCamera{
+		ONVIFXAddr:    "http://camera/onvif",
+		ONVIFUsername: "operator",
+		ONVIFPassword: "secret",
+	}
+	loopEvents := []agentonvif.ONVIFEvents{{Key: "input-1", Type: "input", Value: "true", Timestamp: 1}}
+
+	state := newHeartbeatONVIFState()
+	state.cameraConfiguration = camera
+	state.cameraKey = heartbeatONVIFCameraKey(camera)
+	state.device = device
+	state.loopPullPoint = "loop"
+	state.staticLoaded = true
+	state.staticPayload.enabled = "true"
+
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		return "", errors.New("temporary initial-state failure")
+	}
+	heartbeatGetEventMessages = func(_ *goonvif.Device, pullPointAddress string) ([]agentonvif.ONVIFEvents, error) {
+		if pullPointAddress != "loop" {
+			t.Fatalf("unexpected pull point address %q", pullPointAddress)
+		}
+		return loopEvents, nil
+	}
+
+	payload := getHeartbeatONVIFPayload(camera, state)
+
+	wantEvents := mustJSONMarshal(t, loopEvents)
+	if string(payload.eventsList) != string(wantEvents) {
+		t.Fatalf("payload.eventsList = %s, want %s", payload.eventsList, wantEvents)
+	}
+	if state.device != device || state.loopPullPoint != "loop" || !state.staticLoaded {
+		t.Fatalf("temporary failure invalidated healthy cached state: %+v", state)
+	}
+}
+
+func restoreHeartbeatONVIFStubs(t *testing.T) {
+	t.Helper()
+
+	originalConnect := heartbeatConnectToONVIFDevice
+	originalCreate := heartbeatCreatePullPointSubscription
+	originalDigitalInputs := heartbeatGetDigitalInputs
+	originalEvents := heartbeatGetEventMessages
+	originalPresets := heartbeatGetPresetsFromDevice
+	originalPTZConfigurations := heartbeatGetPTZConfigurationsFromDevice
+	originalPTZFunctions := heartbeatGetPTZFunctionsFromDevice
+	originalRelayOutputs := heartbeatGetRelayOutputs
+	originalUnsubscribe := heartbeatUnsubscribePullPoint
+
+	t.Cleanup(func() {
+		heartbeatConnectToONVIFDevice = originalConnect
+		heartbeatCreatePullPointSubscription = originalCreate
+		heartbeatGetDigitalInputs = originalDigitalInputs
+		heartbeatGetEventMessages = originalEvents
+		heartbeatGetPresetsFromDevice = originalPresets
+		heartbeatGetPTZConfigurationsFromDevice = originalPTZConfigurations
+		heartbeatGetPTZFunctionsFromDevice = originalPTZFunctions
+		heartbeatGetRelayOutputs = originalRelayOutputs
+		heartbeatUnsubscribePullPoint = originalUnsubscribe
+	})
+}
+
+func mustJSONMarshal(t *testing.T, v interface{}) []byte {
+	t.Helper()
+
+	b, err := json.Marshal(v)
+	if err != nil {
+		t.Fatalf("json.Marshal() error = %v", err)
+	}
+	return b
+}
+
+func assertDefaultHeartbeatONVIFPayload(t *testing.T, payload heartbeatONVIFPayload) {
+	t.Helper()
+
+	defaultPayload := defaultHeartbeatONVIFPayload()
+	if payload.enabled != defaultPayload.enabled ||
+		payload.zoom != defaultPayload.zoom ||
+		payload.panTilt != defaultPayload.panTilt ||
+		payload.presets != defaultPayload.presets ||
+		string(payload.presetsList) != string(defaultPayload.presetsList) ||
+		string(payload.eventsList) != string(defaultPayload.eventsList) {
+		t.Fatalf("payload = %+v, want default payload", payload)
+	}
+}
+
+func newTestONVIFDevice() *goonvif.Device {
+	return &goonvif.Device{}
+}
+
+func countString(values []string, want string) int {
+	count := 0
+	for _, value := range values {
+		if value == want {
+			count++
+		}
+	}
+	return count
+}

machinery/src/cloud/dropbox.go

@@ -11,8 +11,8 @@ import (
 	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
 	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/users"
 	"github.com/gin-gonic/gin"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 // UploadDropbox uploads the file to your Dropbox account using the access token and directory.
@@ -30,13 +30,13 @@ func UploadDropbox(configuration *models.Configuration, fileName string) (bool,
 
 	if token == "" {
 		err := "UploadDropbox: Dropbox not properly configured"
-		log.Log.Info(err)
+		log.Info(err)
 		return false, true, errors.New(err)
 	}
 
 	// Upload to Dropbox
-	log.Log.Info("UploadDropbox: Uploading to Dropbox")
-	log.Log.Info("UploadDropbox: Upload started for " + fileName)
+	log.Info("UploadDropbox: Uploading to Dropbox")
+	log.Info("UploadDropbox: Upload started for " + fileName)
 	fullname := "data/recordings/" + fileName
 
 	dConfig := dropbox.Config{
@@ -48,7 +48,7 @@ func UploadDropbox(configuration *models.Configuration, fileName string) (bool,
 	if file != nil {
 		defer func() {
 			if cerr := file.Close(); cerr != nil {
-				log.Log.Error("UploadDropbox: Error closing file: " + cerr.Error())
+				log.Error("UploadDropbox: Error closing file: " + cerr.Error())
 			}
 		}()
 	}
@@ -68,15 +68,15 @@ func UploadDropbox(configuration *models.Configuration, fileName string) (bool,
 		}, file)
 
 		if err != nil {
-			log.Log.Error("UploadDropbox: Error uploading file: " + err.Error())
+			log.Error("UploadDropbox: Error uploading file: " + err.Error())
 			return false, false, err
 		}
 
-		log.Log.Info("UploadDropbox: File uploaded successfully, " + res.Name)
+		log.Info("UploadDropbox: File uploaded successfully, " + res.Name)
 		return true, true, nil
 	}
 
-	log.Log.Error("UploadDropbox: Error opening file: " + err.Error())
+	log.Error("UploadDropbox: Error opening file: " + err.Error())
 	return false, true, err
 }
 

machinery/src/cloud/kerberos_hub.go

@@ -3,12 +3,13 @@ package cloud
 import (
 	"crypto/tls"
 	"errors"
+	"fmt"
 	"io/ioutil"
 	"net/http"
 	"os"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 func UploadKerberosHub(configuration *models.Configuration, fileName string) (bool, bool, error) {
@@ -19,7 +20,7 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 		config.HubPrivateKey == "" ||
 		config.S3.Region == "" {
 		err := "UploadKerberosHub: Kerberos Hub not properly configured."
-		log.Log.Info(err)
+		log.Info(err)
 		return false, false, errors.New(err)
 	}
 
@@ -32,8 +33,10 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 	// - Number of changes
 	// - Token
 
-	log.Log.Info("UploadKerberosHub: Uploading to Kerberos Hub (" + config.HubURI + ")")
-	log.Log.Info("UploadKerberosHub: Upload started for " + fileName)
+	log.WithFields(log.Fields{
+		"component": "kerberos_hub",
+		"event":     "upload_started",
+	}).Info("Uploading recording to Kerberos Hub")
 
 	// Prefer the resumable (tus) upload when enabled (the default). Kerberos Hub
 	// authenticates the agent with its Hub public/private key and proxies the
@@ -44,17 +47,35 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 		uploaded, _, supported, body, rerr := uploadHubResumable(&config, fileName, "UploadKerberosHub", "hub")
 		if supported {
 			if uploaded {
-				log.Log.Info("UploadKerberosHub: Upload Finished (resumable), " + body)
+				log.WithFields(log.Fields{
+					"component":      "kerberos_hub",
+					"event":          "upload_completed",
+					"response_bytes": len(body),
+					"transport":      "tus",
+				}).Info("Hub upload completed")
 				return true, true, nil
 			}
 			if rerr != nil {
-				log.Log.Info("UploadKerberosHub: resumable upload failed, " + rerr.Error())
+				log.WithError(rerr).WithFields(log.Fields{
+					"component": "kerberos_hub",
+					"event":     "upload_failed",
+					"transport": "tus",
+				}).Error("Hub upload failed")
 			} else {
-				log.Log.Info("UploadKerberosHub: resumable upload incomplete, " + body)
+				log.WithFields(log.Fields{
+					"component":      "kerberos_hub",
+					"event":          "upload_incomplete",
+					"response_bytes": len(body),
+					"transport":      "tus",
+				}).Warn("Hub upload incomplete")
 			}
 			return false, true, rerr
 		}
-		log.Log.Info("UploadKerberosHub: resumable (tus) endpoint not available, falling back to legacy upload")
+		log.WithFields(log.Fields{
+			"component": "kerberos_hub",
+			"event":     "upload_transport_fallback",
+			"transport": "http",
+		}).Info("Resumable Hub upload unavailable; using legacy upload")
 	}
 
 	fullname := "data/recordings/" + fileName
@@ -66,7 +87,7 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 	}
 	if err != nil {
 		err := "UploadKerberosHub: Upload Failed, file doesn't exists anymore."
-		log.Log.Info(err)
+		log.Info(err)
 		return false, false, errors.New(err)
 	}
 
@@ -74,9 +95,11 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 	// There might be different reasons like (muted, read-only..)
 	req, err := http.NewRequest("HEAD", config.HubURI+"/storage/upload", nil)
 	if err != nil {
-		errorMessage := "UploadKerberosHub: error reading HEAD request, " + config.HubURI + "/storage: " + err.Error()
-		log.Log.Error(errorMessage)
-		return false, true, errors.New(errorMessage)
+		log.WithError(err).WithFields(log.Fields{
+			"component": "kerberos_hub",
+			"event":     "authorization_request_creation_failed",
+		}).Error("Failed to create Hub upload authorization request")
+		return false, true, fmt.Errorf("create Hub upload authorization request: %w", err)
 	}
 
 	req.Header.Set("X-Kerberos-Storage-FileName", fileName)
@@ -102,25 +125,31 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 		defer resp.Body.Close()
 	}
 
-	if err == nil {
-		if resp != nil {
-			if err == nil {
-				if resp.StatusCode == 200 {
-					log.Log.Info("UploadKerberosHub: Upload allowed using the credentials provided (" + config.HubKey + ", " + config.HubPrivateKey + ")")
-				} else {
-					log.Log.Info("UploadKerberosHub: Upload NOT allowed using the credentials provided (" + config.HubKey + ", " + config.HubPrivateKey + ")")
-					return false, true, nil
-				}
-			}
+	if err == nil && resp != nil {
+		if resp.StatusCode == 200 {
+			log.WithFields(log.Fields{
+				"component":   "kerberos_hub",
+				"event":       "upload_authorized",
+				"status_code": resp.StatusCode,
+			}).Debug("Hub upload authorized")
+		} else {
+			log.WithFields(log.Fields{
+				"component":   "kerberos_hub",
+				"event":       "upload_rejected",
+				"status_code": resp.StatusCode,
+			}).Warn("Hub upload rejected")
+			return false, true, nil
 		}
 	}
 
 	// Now we know we are allowed to upload to the hub, we can start uploading.
 	req, err = http.NewRequest("POST", config.HubURI+"/storage/upload", file)
 	if err != nil {
-		errorMessage := "UploadKerberosHub: error reading POST request, " + config.KStorage.URI + "/storage/upload: " + err.Error()
-		log.Log.Error(errorMessage)
-		return false, true, errors.New(errorMessage)
+		log.WithError(err).WithFields(log.Fields{
+			"component": "kerberos_hub",
+			"event":     "upload_request_creation_failed",
+		}).Error("Failed to create Hub upload request")
+		return false, true, fmt.Errorf("create Hub upload request: %w", err)
 	}
 	req.Header.Set("Content-Type", "video/mp4")
 	req.Header.Set("X-Kerberos-Storage-FileName", fileName)
@@ -140,19 +169,36 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 			body, err := ioutil.ReadAll(resp.Body)
 			if err == nil {
 				if resp.StatusCode == 200 {
-					log.Log.Info("UploadKerberosHub: Upload Finished, " + resp.Status + ".")
+					log.WithFields(log.Fields{
+						"component":   "kerberos_hub",
+						"event":       "upload_completed",
+						"status_code": resp.StatusCode,
+						"transport":   "http",
+					}).Info("Hub upload completed")
 					return true, true, nil
 				} else {
-					log.Log.Info("UploadKerberosHub: Upload Failed, " + resp.Status + ", " + string(body))
+					log.WithFields(log.Fields{
+						"component":      "kerberos_hub",
+						"event":          "upload_rejected",
+						"response_bytes": len(body),
+						"status_code":    resp.StatusCode,
+						"transport":      "http",
+					}).Warn("Hub upload rejected")
 					return false, true, nil
 				}
 			}
 		}
 	}
 
-	errorMessage := "UploadKerberosHub: Upload Failed, " + err.Error()
-	log.Log.Info(errorMessage)
-	return false, true, errors.New(errorMessage)
+	if err == nil {
+		err = errors.New("Hub upload failed without a response")
+	}
+	log.WithError(err).WithFields(log.Fields{
+		"component": "kerberos_hub",
+		"event":     "upload_failed",
+		"transport": "http",
+	}).Error("Hub upload failed")
+	return false, true, fmt.Errorf("Hub upload failed: %w", err)
 }
 
 // stripHubCredentialsOnCrossHostRedirect removes the custom Kerberos Hub

machinery/src/cloud/kerberos_vault.go

@@ -9,8 +9,8 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 // We will count the number of retries we have done.
@@ -27,7 +27,7 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string) (
 		config.KStorage.Directory == "" ||
 		config.KStorage.URI == "" {
 		err := "UploadKerberosVault: Kerberos Vault not properly configured"
-		log.Log.Info(err)
+		log.Info(err)
 		return false, false, errors.New(err)
 	}
 
@@ -35,8 +35,13 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string) (
 	// This can happen when the file was already removed (e.g. cleanup, or an
 	// earlier successful upload). Skip it so the watcher drops the marker
 	// instead of retrying indefinitely.
-	if _, err := os.Stat("data/recordings/" + fileName); err != nil {
-		log.Log.Info("UploadKerberosVault: skipping " + fileName + ", file doesn't exist anymore")
+	info, err := os.Stat("data/recordings/" + fileName)
+	if err != nil {
+		log.Info("UploadKerberosVault: skipping " + fileName + ", file doesn't exist anymore")
+		return false, false, nil
+	}
+	if info.Size() == 0 {
+		log.Warn("UploadKerberosVault: skipping " + fileName + ", recording is empty")
 		return false, false, nil
 	}
 
@@ -49,8 +54,11 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string) (
 	// - Number of changes
 	// - Token
 	// KerberosCloud, this means storage is disabled and proxy enabled.
-	log.Log.Info("UploadKerberosVault: Uploading to Kerberos Vault (" + config.KStorage.URI + ")")
-	log.Log.Info("UploadKerberosVault: Upload started for " + fileName)
+	log.WithFields(log.Fields{
+		"component": "kerberos_vault",
+		"event":     "upload_started",
+		"storage":   "primary",
+	}).Info("Vault upload started")
 
 	publicKey := config.KStorage.CloudKey
 	if config.HubKey != "" {
@@ -62,14 +70,27 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string) (
 		uploaded, responded, body, err := sendToVault(*config.KStorage, publicKey, config.Key, fileName, "UploadKerberosVault", "primary")
 		if uploaded {
 			kstorageRetryCount = 0
-			log.Log.Info("UploadKerberosVault: Upload Finished, " + body)
+			log.WithFields(log.Fields{
+				"component": "kerberos_vault",
+				"event":     "upload_completed",
+				"storage":   "primary",
+			}).Info("Vault upload completed")
 			return true, true, nil
 		}
 
 		if err != nil {
-			log.Log.Info("UploadKerberosVault: Upload Failed, " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component": "kerberos_vault",
+				"event":     "upload_failed",
+				"storage":   "primary",
+			}).Error("Vault upload failed")
 		} else {
-			log.Log.Info("UploadKerberosVault: Upload Failed, " + body)
+			log.WithFields(log.Fields{
+				"component":      "kerberos_vault",
+				"event":          "upload_rejected",
+				"response_bytes": len(body),
+				"storage":        "primary",
+			}).Warn("Vault upload rejected")
 		}
 
 		// We only advance the retry policy when the vault gave a definitive
@@ -91,26 +112,43 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string) (
 		config.KStorageSecondary.SecretAccessKey == "" ||
 		config.KStorageSecondary.Directory == "" ||
 		config.KStorageSecondary.URI == "" {
-		log.Log.Info("UploadKerberosVault (Secondary): Secondary Kerberos Vault not properly configured.")
+		log.Info("UploadKerberosVault (Secondary): Secondary Kerberos Vault not properly configured.")
 	} else {
 
 		if kstorageRetryCount < config.KStorage.MaxRetries {
-			log.Log.Info("UploadKerberosVault (Secondary): Do not upload to secondary storage, we are still in retry policy.")
+			log.Info("UploadKerberosVault (Secondary): Do not upload to secondary storage, we are still in retry policy.")
 			return false, true, nil
 		}
 
-		log.Log.Info("UploadKerberosVault (Secondary): Uploading to Secondary Kerberos Vault (" + config.KStorageSecondary.URI + ")")
+		log.WithFields(log.Fields{
+			"component": "kerberos_vault",
+			"event":     "upload_started",
+			"storage":   "secondary",
+		}).Info("Vault upload started")
 
 		uploaded, _, body, err := sendToVault(*config.KStorageSecondary, publicKey, config.Key, fileName, "UploadKerberosVault (Secondary)", "secondary")
 		if uploaded {
-			log.Log.Info("UploadKerberosVault (Secondary): Upload Finished to secondary, " + body)
+			log.WithFields(log.Fields{
+				"component": "kerberos_vault",
+				"event":     "upload_completed",
+				"storage":   "secondary",
+			}).Info("Vault upload completed")
 			return true, true, nil
 		}
 
 		if err != nil {
-			log.Log.Info("UploadKerberosVault (Secondary): Upload Failed to secondary, " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component": "kerberos_vault",
+				"event":     "upload_failed",
+				"storage":   "secondary",
+			}).Error("Vault upload failed")
 		} else {
-			log.Log.Info("UploadKerberosVault (Secondary): Upload Failed to secondary, " + body)
+			log.WithFields(log.Fields{
+				"component":      "kerberos_vault",
+				"event":          "upload_rejected",
+				"response_bytes": len(body),
+				"storage":        "secondary",
+			}).Warn("Vault upload rejected")
 		}
 	}
 
@@ -131,7 +169,7 @@ func sendToVault(vault models.KStorage, publicKey, deviceKey, fileName, label, s
 		if supported {
 			return uploaded, responded, body, err
 		}
-		log.Log.Info(label + ": resumable (tus) endpoint not available, falling back to legacy upload")
+		log.Info(label + ": resumable (tus) endpoint not available, falling back to legacy upload")
 	}
 	return uploadVaultLegacy(vault, publicKey, deviceKey, fileName, label)
 }
@@ -148,7 +186,7 @@ func uploadVaultLegacy(vault models.KStorage, publicKey, deviceKey, fileName, la
 	}
 	if err != nil {
 		msg := label + ": Upload Failed, file doesn't exists anymore"
-		log.Log.Info(msg)
+		log.Info(msg)
 		return false, false, "", errors.New(msg)
 	}
 
@@ -160,7 +198,7 @@ func uploadVaultLegacy(vault models.KStorage, publicKey, deviceKey, fileName, la
 	req, err := http.NewRequest("POST", uri+"/storage", file)
 	if err != nil {
 		errorMessage := label + ": error reading request, " + uri + "/storage: " + err.Error()
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, false, "", errors.New(errorMessage)
 	}
 	req.Header.Set("Content-Type", "video/mp4")

machinery/src/cloud/livehls.go

@@ -7,9 +7,9 @@ import (
 	mqtt "github.com/eclipse/paho.mqtt.golang"
 
 	"github.com/kerberos-io/agent/machinery/src/cloud/livehls"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
+	log "github.com/sirupsen/logrus"
 )
 
 // hlsViewerTimeoutSeconds is how long the agent keeps shipping live HLS segments
@@ -45,22 +45,29 @@ const hlsReadyReannounceSeconds = 2
 // trades a little idle CPU for a near-instant "requesting stream", so viewers no
 // longer wait a full GOP for the first segment to be cut. Set
 // AGENT_LIVE_HLS_PREWARM=false to fall back to the lazy on-demand path above.
-func HandleLiveStreamHLS(configuration *models.Configuration, communication *models.Communication, mqttClient mqtt.Client, subStreamEnabled bool) {
+func HandleLiveStreamHLS(
+	configuration *models.Configuration,
+	communication *models.Communication,
+	mqttClient mqtt.Client,
+	subStreamEnabled bool,
+	mainQueue *packets.Queue,
+	subQueue *packets.Queue,
+) {
 
-	log.Log.Debug("cloud.HandleLiveStreamHLS(): started")
+	log.Debug("cloud.HandleLiveStreamHLS(): started")
 
 	config := configuration.Config
 
 	if config.Offline == "true" {
-		log.Log.Debug("cloud.HandleLiveStreamHLS(): stopping as Offline is enabled.")
+		log.Debug("cloud.HandleLiveStreamHLS(): stopping as Offline is enabled.")
 		return
 	}
 	if config.Capture.Liveview == "false" {
-		log.Log.Debug("cloud.HandleLiveStreamHLS(): stopping as Liveview is disabled.")
+		log.Debug("cloud.HandleLiveStreamHLS(): stopping as Liveview is disabled.")
 		return
 	}
 	if config.HubURI == "" || config.HubKey == "" {
-		log.Log.Debug("cloud.HandleLiveStreamHLS(): stopping as the Hub is not configured (HubURI/HubKey).")
+		log.Debug("cloud.HandleLiveStreamHLS(): stopping as the Hub is not configured (HubURI/HubKey).")
 		return
 	}
 
@@ -88,8 +95,8 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 	// mp4ff's strict parser rejects).
 	requestedQuality := models.StreamQualityAuto
 	useSub := models.SelectSubStreamForQuality(config, requestedQuality, subStreamEnabled)
-	source := buildHLSSource(config, communication, useSub)
-	log.Log.Info("cloud.HandleLiveStreamHLS(): serving live HLS from the " + source.label + " stream")
+	source := buildHLSSource(config, mainQueue, subQueue, useSub)
+	log.Info("cloud.HandleLiveStreamHLS(): serving live HLS from the " + source.label + " stream")
 
 	// prewarm keeps a single long-lived session muxing into an in-memory ring
 	// buffer while idle and flushes it the instant a viewer arrives, eliminating
@@ -97,9 +104,9 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 	// to fall back to the lazy on-demand path.
 	prewarm := os.Getenv("AGENT_LIVE_HLS_PREWARM") != "false"
 	if prewarm {
-		log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS prewarm ENABLED (set AGENT_LIVE_HLS_PREWARM=false to disable)")
+		log.Info("cloud.HandleLiveStreamHLS(): live HLS prewarm ENABLED (set AGENT_LIVE_HLS_PREWARM=false to disable)")
 	} else {
-		log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS prewarm DISABLED (AGENT_LIVE_HLS_PREWARM=false)")
+		log.Info("cloud.HandleLiveStreamHLS(): live HLS prewarm DISABLED (AGENT_LIVE_HLS_PREWARM=false)")
 	}
 
 	// lowLatency enables LL-HLS: each segment is sliced into CMAF parts shipped the
@@ -109,9 +116,9 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 	partTargetMs := uint64(0)
 	if os.Getenv("AGENT_LIVE_HLS_LOW_LATENCY") != "false" {
 		partTargetMs = livehls.DefaultPartTargetMs
-		log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS low-latency (LL-HLS) ENABLED (set AGENT_LIVE_HLS_LOW_LATENCY=false to disable)")
+		log.Info("cloud.HandleLiveStreamHLS(): live HLS low-latency (LL-HLS) ENABLED (set AGENT_LIVE_HLS_LOW_LATENCY=false to disable)")
 	} else {
-		log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS low-latency (LL-HLS) DISABLED (AGENT_LIVE_HLS_LOW_LATENCY=false)")
+		log.Info("cloud.HandleLiveStreamHLS(): live HLS low-latency (LL-HLS) DISABLED (AGENT_LIVE_HLS_LOW_LATENCY=false)")
 	}
 
 	var session *livehls.Session
@@ -123,6 +130,9 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 
 	for cursorError == nil {
 		pkt, cursorError = source.cursor.ReadPacket()
+		if cursorError != nil {
+			break
+		}
 
 		now := time.Now().Unix()
 		select {
@@ -156,9 +166,9 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 				_ = session.Close()
 				session = nil
 			}
-			source = buildHLSSource(config, communication, useSub)
+			source = buildHLSSource(config, mainQueue, subQueue, useSub)
 			lastReadyAnnounce = 0
-			log.Log.Info("cloud.HandleLiveStreamHLS(): switched live HLS to the " + source.label + " stream (quality=" + requestedQuality + ")")
+			log.Info("cloud.HandleLiveStreamHLS(): switched live HLS to the " + source.label + " stream (quality=" + requestedQuality + ")")
 			continue
 		}
 
@@ -184,11 +194,22 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 					StartBuffering: true,
 				})
 				session.SetOnReady(func(sessionID string) {
-					log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS session ready, announcing " + sessionID)
+					log.WithFields(log.Fields{
+						"component":  "cloud/livehls",
+						"event":      "session_ready",
+						"session_id": sessionID,
+					}).Info("Live HLS session ready")
 					publishHLSReady(configuration, mqttClient, hubKey, deviceId, sessionID)
 					lastReadyAnnounce = time.Now().Unix()
 				})
-				log.Log.Info("cloud.HandleLiveStreamHLS(): prewarming live HLS session " + session.SessionID())
+				session.SetOnFailure(func(sessionID, reason string) {
+					publishHLSFailure(configuration, mqttClient, hubKey, deviceId, sessionID, reason)
+				})
+				log.WithFields(log.Fields{
+					"component":  "cloud/livehls",
+					"event":      "session_prewarming",
+					"session_id": session.SessionID(),
+				}).Info("Prewarming live HLS session")
 			}
 
 			if viewerActive {
@@ -207,7 +228,7 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 
 			if len(pkt.Data) > 0 && pkt.IsVideo {
 				if err := session.WritePacket(pkt); err != nil {
-					log.Log.Error("cloud.HandleLiveStreamHLS(): " + err.Error())
+					log.Error("cloud.HandleLiveStreamHLS(): " + err.Error())
 				}
 			}
 			continue
@@ -217,7 +238,12 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 			// No viewer: stop and discard the session so we stop shipping segments.
 			if session != nil {
 				_ = session.Close()
-				log.Log.Info("cloud.HandleLiveStreamHLS(): no active viewers, stopped live HLS session " + session.SessionID())
+				log.WithFields(log.Fields{
+					"component":  "cloud/livehls",
+					"event":      "session_stopped",
+					"reason":     "no_active_viewers",
+					"session_id": session.SessionID(),
+				}).Info("Live HLS session stopped")
 				session = nil
 			}
 			continue
@@ -243,22 +269,33 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 				PartTargetMs: partTargetMs,
 			})
 			session.SetOnReady(func(sessionID string) {
-				log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS session ready, announcing " + sessionID)
+				log.WithFields(log.Fields{
+					"component":  "cloud/livehls",
+					"event":      "session_ready",
+					"session_id": sessionID,
+				}).Info("Live HLS session ready")
 				publishHLSReady(configuration, mqttClient, hubKey, deviceId, sessionID)
 				lastReadyAnnounce = time.Now().Unix()
 			})
-			log.Log.Info("cloud.HandleLiveStreamHLS(): started live HLS session " + session.SessionID())
+			session.SetOnFailure(func(sessionID, reason string) {
+				publishHLSFailure(configuration, mqttClient, hubKey, deviceId, sessionID, reason)
+			})
+			log.WithFields(log.Fields{
+				"component":  "cloud/livehls",
+				"event":      "session_started",
+				"session_id": session.SessionID(),
+			}).Info("Live HLS session started")
 		}
 
 		if err := session.WritePacket(pkt); err != nil {
-			log.Log.Error("cloud.HandleLiveStreamHLS(): " + err.Error())
+			log.Error("cloud.HandleLiveStreamHLS(): " + err.Error())
 		}
 	}
 
 	if session != nil {
 		_ = session.Close()
 	}
-	log.Log.Debug("cloud.HandleLiveStreamHLS(): finished")
+	log.Debug("cloud.HandleLiveStreamHLS(): finished")
 }
 
 // publishHLSReady announces, over MQTT, that a live HLS session is available so
@@ -278,9 +315,48 @@ func publishHLSReady(configuration *models.Configuration, mqttClient mqtt.Client
 	payload, err := models.PackageMQTTMessage(configuration, message)
 	if err == nil {
 		mqttClient.Publish("kerberos/hub/"+hubKey, 0, false, payload)
-		log.Log.Info("cloud.HandleLiveStreamHLS(): announced live HLS session " + sessionID)
+		log.WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "session_announced",
+			"session_id": sessionID,
+		}).Info("Live HLS session announced")
 	} else {
-		log.Log.Error("cloud.HandleLiveStreamHLS(): failed to package receive-hls-ready message: " + err.Error())
+		log.WithError(err).WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "session_announcement_failed",
+			"session_id": sessionID,
+		}).Error("Failed to package live HLS session announcement")
+	}
+}
+
+func publishHLSFailure(configuration *models.Configuration, mqttClient mqtt.Client, hubKey, deviceId, sessionID, reason string) {
+	valueMap := map[string]interface{}{
+		"session": sessionID,
+		"device":  deviceId,
+		"reason":  reason,
+	}
+	message := models.Message{
+		Payload: models.Payload{
+			Action:   "receive-hls-error",
+			DeviceId: deviceId,
+			Value:    valueMap,
+		},
+	}
+	payload, err := models.PackageMQTTMessage(configuration, message)
+	if err == nil {
+		mqttClient.Publish("kerberos/hub/"+hubKey, 0, false, payload)
+		log.WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "session_failure_announced",
+			"reason":     reason,
+			"session_id": sessionID,
+		}).Warn("Live HLS session failure announced")
+	} else {
+		log.WithError(err).WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "session_failure_announcement_failed",
+			"session_id": sessionID,
+		}).Error("Failed to package live HLS session failure")
 	}
 }
 
@@ -302,11 +378,11 @@ type hlsStreamSource struct {
 // is available; otherwise the main (high-resolution) stream is used. A fresh
 // Latest() cursor is created so muxing resumes from the live edge of the chosen
 // stream after a switch.
-func buildHLSSource(config models.Config, communication *models.Communication, useSub bool) hlsStreamSource {
+func buildHLSSource(config models.Config, mainQueue, subQueue *packets.Queue, useSub bool) hlsStreamSource {
 	cam := config.Capture.IPCamera
-	if useSub && communication.SubQueue != nil {
+	if useSub && subQueue != nil {
 		return hlsStreamSource{
-			cursor: communication.SubQueue.Latest(),
+			cursor: subQueue.Latest(),
 			sps:    cam.SubSPSNALUs,
 			pps:    cam.SubPPSNALUs,
 			vps:    cam.SubVPSNALUs,
@@ -316,7 +392,7 @@ func buildHLSSource(config models.Config, communication *models.Communication, u
 		}
 	}
 	return hlsStreamSource{
-		cursor: communication.Queue.Latest(),
+		cursor: mainQueue.Latest(),
 		sps:    cam.SPSNALUs,
 		pps:    cam.PPSNALUs,
 		vps:    cam.VPSNALUs,

machinery/src/cloud/livehls/publisher.go

@@ -24,13 +24,14 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"net/http"
 	"strconv"
 	"strings"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/video"
+	log "github.com/sirupsen/logrus"
 )
 
 const (
@@ -67,6 +68,7 @@ const (
 	// cannot be delivered within roughly its own duration is stale, so the upload
 	// is abandoned (dropped) rather than allowed to back up the pipeline.
 	defaultPublishTimeout = 4 * time.Second
+	maxErrorResponseBytes = 4 << 10
 )
 
 // PublisherConfig carries the hub endpoint and credentials needed to ship live
@@ -214,9 +216,15 @@ func (p *Publisher) post(ctx context.Context, params postParams) error {
 	defer resp.Body.Close()
 
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		return fmt.Errorf("livehls: upload %s rejected: %s", params.name, resp.Status)
+		body, _ := io.ReadAll(io.LimitReader(resp.Body, maxErrorResponseBytes))
+		return fmt.Errorf("livehls: upload %s rejected with status %d (%d response bytes)", params.name, resp.StatusCode, len(body))
 	}
-	log.Log.Debug("livehls.Publisher.post(): shipped " + params.name + " for session " + params.sessionID)
+	log.WithFields(log.Fields{
+		"component":  "cloud/livehls",
+		"event":      "object_uploaded",
+		"object":     params.name,
+		"session_id": params.sessionID,
+	}).Trace("Live HLS object uploaded")
 	return nil
 }
 

machinery/src/cloud/livehls/publisher_test.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"sync"
 	"testing"
 	"time"
@@ -144,12 +145,22 @@ func TestPublisherPublishSegmentSendsSequenceAndDuration(t *testing.T) {
 }
 
 func TestPublisherReturnsErrorOnNon2xx(t *testing.T) {
-	srv, _, _ := newCapturingServer(t, http.StatusInternalServerError)
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusBadRequest)
+		_, _ = w.Write([]byte(`{"error":true,"data":"No user found with this public and private key."}`))
+	}))
+	t.Cleanup(srv.Close)
 	p := testPublisher(srv.URL)
 
 	err := p.PublishSegment(context.Background(), "s", video.LiveSegment{SequenceNumber: 1, Data: []byte("x")})
 	if err == nil {
-		t.Fatal("expected an error on 500 response")
+		t.Fatal("expected an error on 400 response")
+	}
+	if !strings.Contains(err.Error(), "status 400") || !strings.Contains(err.Error(), "71 response bytes") {
+		t.Fatalf("error = %q, want status code and response size", err)
+	}
+	if strings.Contains(err.Error(), "No user found with this public and private key") {
+		t.Fatalf("error exposed Hub response body: %q", err)
 	}
 }
 
@@ -281,7 +292,13 @@ func TestSessionRetriesInitWhenFirstAttemptFails(t *testing.T) {
 	})
 
 	var ready int
+	var failures int
+	var failureReason string
 	sess.SetOnReady(func(string) { ready++ })
+	sess.SetOnFailure(func(_ string, reason string) {
+		failures++
+		failureReason = reason
+	})
 
 	for i := 0; i < 60; i++ {
 		isKey := i%25 == 0
@@ -304,6 +321,9 @@ func TestSessionRetriesInitWhenFirstAttemptFails(t *testing.T) {
 	if ready != 1 {
 		t.Errorf("OnReady fired %d times, want 1", ready)
 	}
+	if failures != 1 || failureReason != "init-upload-failed" {
+		t.Errorf("OnFailure = %d/%q, want 1/init-upload-failed", failures, failureReason)
+	}
 }
 
 // liveTestSPSForSession is the known-good baseline SPS reused across tests.

machinery/src/cloud/livehls/session.go

@@ -8,9 +8,9 @@ import (
 	"sync"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/kerberos-io/agent/machinery/src/video"
+	log "github.com/sirupsen/logrus"
 )
 
 // DefaultTargetSegmentMs is the nominal live segment length. ~2s keeps standard
@@ -46,9 +46,11 @@ type Session struct {
 	// lastInitAt is when the init segment was last (re)uploaded. The init is
 	// re-sent periodically so its short TTL in the hub live window never lapses
 	// mid-session; see refreshInitIfStale.
-	lastInitAt time.Time
-	readyFired bool
-	onReady    func(sessionID string)
+	lastInitAt   time.Time
+	readyFired   bool
+	onReady      func(sessionID string)
+	failureFired bool
+	onFailure    func(sessionID, reason string)
 
 	// uploadsActive gates whether the init and completed segments are shipped to
 	// hub-api. It is true for the default on-demand path. The prewarm path starts
@@ -142,14 +144,25 @@ func NewSession(publisher *Publisher, opts SessionOptions) *Session {
 			return nil
 		}
 		if !s.publishInitIfNeeded() {
-			log.Log.Warning("livehls.Session: dropping segment " +
-				fmt.Sprintf("%d", segment.SequenceNumber) + " because init has not been delivered yet")
+			log.WithFields(log.Fields{
+				"component":        "cloud/livehls",
+				"event":            "segment_dropped",
+				"reason":           "init_unavailable",
+				"segment_sequence": segment.SequenceNumber,
+				"session_id":       s.id,
+			}).Warn("Dropping live HLS segment")
 			return nil
 		}
 		ctx, cancel := s.newContext()
 		defer cancel()
 		if err := s.publisher.PublishSegment(ctx, s.id, segment); err != nil {
-			log.Log.Warning("livehls.Session: " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component":        "cloud/livehls",
+				"event":            "segment_upload_failed",
+				"segment_sequence": segment.SequenceNumber,
+				"session_id":       s.id,
+			}).Warn("Failed to upload live HLS segment")
+			s.fireFailureOnce("segment-upload-failed")
 			return nil
 		}
 		s.fireReadyOnce()
@@ -172,15 +185,27 @@ func NewSession(publisher *Publisher, opts SessionOptions) *Session {
 				return nil
 			}
 			if !s.publishInitIfNeeded() {
-				log.Log.Warning("livehls.Session: dropping part " +
-					fmt.Sprintf("%d.%d", part.SegmentSeq, part.PartIndex) +
-					" because init has not been delivered yet")
+				log.WithFields(log.Fields{
+					"component":        "cloud/livehls",
+					"event":            "part_dropped",
+					"part_index":       part.PartIndex,
+					"reason":           "init_unavailable",
+					"segment_sequence": part.SegmentSeq,
+					"session_id":       s.id,
+				}).Warn("Dropping live HLS part")
 				return nil
 			}
 			ctx, cancel := s.newContext()
 			defer cancel()
 			if err := s.publisher.PublishPart(ctx, s.id, part); err != nil {
-				log.Log.Warning("livehls.Session: " + err.Error())
+				log.WithError(err).WithFields(log.Fields{
+					"component":        "cloud/livehls",
+					"event":            "part_upload_failed",
+					"part_index":       part.PartIndex,
+					"segment_sequence": part.SegmentSeq,
+					"session_id":       s.id,
+				}).Warn("Failed to upload live HLS part")
+				s.fireFailureOnce("part-upload-failed")
 				return nil
 			}
 			s.fireReadyOnce()
@@ -216,6 +241,15 @@ func (s *Session) SetOnReady(fn func(sessionID string)) {
 	s.mu.Unlock()
 }
 
+// SetOnFailure registers a one-shot callback for startup upload failures. The
+// reason is a fixed code rather than an HTTP response body, so credentials or
+// server details cannot leak through MQTT diagnostics.
+func (s *Session) SetOnFailure(fn func(sessionID, reason string)) {
+	s.mu.Lock()
+	s.onFailure = fn
+	s.mu.Unlock()
+}
+
 // prewarmMaxBufferedSegments is how many of the most recent completed segments
 // the prewarm path keeps in memory while idle and flushes to a viewer on arrival.
 // One segment keeps startup instant (the viewer immediately gets a playable
@@ -268,7 +302,13 @@ func (s *Session) SetUploadsActive(active bool) bool {
 		}
 		ctx, cancel := s.newContext()
 		if err := s.publisher.PublishSegment(ctx, s.id, buffered[i]); err != nil {
-			log.Log.Warning("livehls.Session: prewarm flush: " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component":        "cloud/livehls",
+				"event":            "prewarm_segment_upload_failed",
+				"segment_sequence": buffered[i].SequenceNumber,
+				"session_id":       s.id,
+			}).Warn("Failed to upload buffered live HLS segment")
+			s.fireFailureOnce("segment-upload-failed")
 			cancel()
 			continue
 		}
@@ -284,7 +324,14 @@ func (s *Session) SetUploadsActive(active bool) bool {
 		}
 		ctx, cancel := s.newContext()
 		if err := s.publisher.PublishPart(ctx, s.id, bufferedParts[i]); err != nil {
-			log.Log.Warning("livehls.Session: prewarm flush (part): " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component":        "cloud/livehls",
+				"event":            "prewarm_part_upload_failed",
+				"part_index":       bufferedParts[i].PartIndex,
+				"segment_sequence": bufferedParts[i].SegmentSeq,
+				"session_id":       s.id,
+			}).Warn("Failed to upload buffered live HLS part")
+			s.fireFailureOnce("part-upload-failed")
 			cancel()
 			continue
 		}
@@ -381,7 +428,12 @@ func (s *Session) publishInitIfNeeded() bool {
 	ctx, cancel := s.newContext()
 	defer cancel()
 	if err := s.publisher.PublishInit(ctx, s.id, initBytes); err != nil {
-		log.Log.Warning("livehls.Session: init upload failed, will retry: " + err.Error())
+		log.WithError(err).WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "init_upload_failed",
+			"session_id": s.id,
+		}).Warn("Failed to upload live HLS init segment; retrying")
+		s.fireFailureOnce("init-upload-failed")
 		return false
 	}
 
@@ -389,7 +441,11 @@ func (s *Session) publishInitIfNeeded() bool {
 	s.initPublished = true
 	s.lastInitAt = time.Now()
 	s.mu.Unlock()
-	log.Log.Info("livehls.Session: init segment delivered for session " + s.id)
+	log.WithFields(log.Fields{
+		"component":  "cloud/livehls",
+		"event":      "init_uploaded",
+		"session_id": s.id,
+	}).Info("Live HLS init segment uploaded")
 	return true
 }
 
@@ -422,14 +478,22 @@ func (s *Session) refreshInitIfStale() {
 	ctx, cancel := s.newContext()
 	defer cancel()
 	if err := s.publisher.PublishInit(ctx, s.id, initBytes); err != nil {
-		log.Log.Warning("livehls.Session: init refresh failed, will retry: " + err.Error())
+		log.WithError(err).WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "init_refresh_failed",
+			"session_id": s.id,
+		}).Warn("Failed to refresh live HLS init segment; retrying")
 		return
 	}
 
 	s.mu.Lock()
 	s.lastInitAt = time.Now()
 	s.mu.Unlock()
-	log.Log.Debug("livehls.Session: refreshed init segment TTL for session " + s.id)
+	log.WithFields(log.Fields{
+		"component":  "cloud/livehls",
+		"event":      "init_refreshed",
+		"session_id": s.id,
+	}).Debug("Live HLS init segment refreshed")
 }
 
 // fireReadyOnce invokes the OnReady callback the first time it is called.
@@ -445,6 +509,18 @@ func (s *Session) fireReadyOnce() {
 	fn(s.id)
 }
 
+func (s *Session) fireFailureOnce(reason string) {
+	s.mu.Lock()
+	if s.failureFired || s.readyFired || s.onFailure == nil {
+		s.mu.Unlock()
+		return
+	}
+	s.failureFired = true
+	fn := s.onFailure
+	s.mu.Unlock()
+	fn(s.id, reason)
+}
+
 // newSessionID returns a short, unique, URL-safe session identifier of the form
 // <unix-seconds>-<random-hex>.
 func newSessionID() string {

machinery/src/cloud/livemoq/annexb.go

@@ -5,10 +5,16 @@ import (
 	"strings"
 
 	"github.com/bluenviron/mediacommon/pkg/codecs/h264"
+	"github.com/kerberos-io/agent/machinery/src/models"
 )
 
 var annexBStartCode = []byte{0x00, 0x00, 0x00, 0x01}
 
+// H264NormalizationStats reports malformed duplication removed from an access unit.
+type H264NormalizationStats struct {
+	DuplicateIDRNALUs int
+}
+
 // EnsureAnnexB restores the start code stripped by the Agent capture queue.
 func EnsureAnnexB(payload []byte) []byte {
 	if hasAnnexBStartCode(payload) {
@@ -20,20 +26,28 @@ func EnsureAnnexB(payload []byte) []byte {
 	return append(framed, payload...)
 }
 
-// NormalizeH264AccessUnit removes delimiters and duplicate parameter sets that
-// can make older MoQ splitters emit a parameter-only frame before the IDR.
+// NormalizeH264AccessUnit removes delimiters and exact duplicate parameter-set
+// or IDR NALUs that can confuse older MoQ splitters and decoders.
 func NormalizeH264AccessUnit(payload []byte) ([]byte, error) {
+	normalized, _, err := NormalizeH264AccessUnitWithStats(payload)
+	return normalized, err
+}
+
+// NormalizeH264AccessUnitWithStats also reports exact duplicate IDR NALUs.
+func NormalizeH264AccessUnitWithStats(payload []byte) ([]byte, H264NormalizationStats, error) {
 	nalus, err := h264.AnnexBUnmarshal(EnsureAnnexB(payload))
 	if err != nil {
-		return nil, err
+		return nil, H264NormalizationStats{}, err
 	}
 
+	stats := H264NormalizationStats{}
 	normalized := make([][]byte, 0, len(nalus))
 	for _, nalu := range nalus {
 		if len(nalu) == 0 || nalu[0]&0x1f == 9 {
 			continue
 		}
-		if nalu[0]&0x1f == 7 || nalu[0]&0x1f == 8 {
+		naluType := nalu[0] & 0x1f
+		if naluType == 7 || naluType == 8 || naluType == 5 {
 			duplicate := false
 			for _, existing := range normalized {
 				if bytes.Equal(existing, nalu) {
@@ -42,21 +56,35 @@ func NormalizeH264AccessUnit(payload []byte) ([]byte, error) {
 				}
 			}
 			if duplicate {
+				if naluType == 5 {
+					stats.DuplicateIDRNALUs++
+				}
 				continue
 			}
 		}
 		normalized = append(normalized, nalu)
 	}
 
-	return h264.AnnexBMarshal(normalized)
+	result, err := h264.AnnexBMarshal(normalized)
+	return result, stats, err
 }
 
-func BroadcastPath(prefix string, deviceKey string) string {
+// BroadcastPath returns the relay path a quality tier is published on. Every
+// tier gets its own broadcast so a viewer switches between the camera's main and
+// sub stream by resubscribing to another path, without any control channel back
+// to the Agent. The high tier keeps the historical ".../live.hang" path so
+// existing viewers keep working; the low tier lives next to it on
+// ".../live-low.hang".
+func BroadcastPath(prefix string, deviceKey string, quality string) string {
 	prefix = strings.Trim(prefix, "/")
 	if prefix == "" {
 		prefix = "devices"
 	}
-	return prefix + "/" + strings.Trim(deviceKey, "/") + "/live.hang"
+	name := "live.hang"
+	if quality == models.StreamQualityLow {
+		name = "live-low.hang"
+	}
+	return prefix + "/" + strings.Trim(deviceKey, "/") + "/" + name
 }
 
 // TimestampUs converts the capture presentation timestamp from milliseconds.

machinery/src/cloud/livemoq/annexb_test.go

@@ -3,6 +3,8 @@ package livemoq
 import (
 	"bytes"
 	"testing"
+
+	"github.com/kerberos-io/agent/machinery/src/models"
 )
 
 func TestEnsureAnnexB(t *testing.T) {
@@ -64,13 +66,44 @@ func TestNormalizeH264AccessUnit(t *testing.T) {
 	}
 }
 
-func TestBroadcastPath(t *testing.T) {
-	if got := BroadcastPath("/devices/", "/camera-1/"); got != "devices/camera-1/live.hang" {
-		t.Fatalf("BroadcastPath() = %q", got)
+func TestNormalizeH264AccessUnitRemovesOnlyExactDuplicateIDRSlices(t *testing.T) {
+	startCode := []byte{0x00, 0x00, 0x00, 0x01}
+	idrSlice1 := []byte{0x65, 0x88, 0x84}
+	idrSlice2 := []byte{0x65, 0x44, 0x22}
+
+	payload := make([]byte, 0)
+	for _, nalu := range [][]byte{idrSlice1, idrSlice1, idrSlice2} {
+		payload = append(payload, startCode...)
+		payload = append(payload, nalu...)
 	}
-	if got := BroadcastPath("", "camera-1"); got != "devices/camera-1/live.hang" {
+
+	got, stats, err := NormalizeH264AccessUnitWithStats(payload)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := make([]byte, 0)
+	for _, nalu := range [][]byte{idrSlice1, idrSlice2} {
+		want = append(want, startCode...)
+		want = append(want, nalu...)
+	}
+	if !bytes.Equal(got, want) {
+		t.Fatalf("NormalizeH264AccessUnitWithStats() = %x, want %x", got, want)
+	}
+	if stats.DuplicateIDRNALUs != 1 {
+		t.Fatalf("DuplicateIDRNALUs = %d, want 1", stats.DuplicateIDRNALUs)
+	}
+}
+
+func TestBroadcastPath(t *testing.T) {
+	if got := BroadcastPath("/devices/", "/camera-1/", models.StreamQualityHigh); got != "devices/camera-1/live.hang" {
+		t.Fatalf("BroadcastPath() high = %q", got)
+	}
+	if got := BroadcastPath("", "camera-1", models.StreamQualityHigh); got != "devices/camera-1/live.hang" {
 		t.Fatalf("BroadcastPath() default = %q", got)
 	}
+	if got := BroadcastPath("", "camera-1", models.StreamQualityLow); got != "devices/camera-1/live-low.hang" {
+		t.Fatalf("BroadcastPath() low = %q", got)
+	}
 }
 
 func TestTimestampUs(t *testing.T) {

machinery/src/cloud/livemoq/dedup.go

@@ -0,0 +1,41 @@
+package livemoq
+
+import (
+	"crypto/sha256"
+	"time"
+)
+
+type KeyframeDeduplicator struct {
+	hasPrevious  bool
+	timestampMs  int64
+	capturedAtMs int64
+	observedAt   time.Time
+	digest       [sha256.Size]byte
+}
+
+func (d *KeyframeDeduplicator) Reset() {
+	*d = KeyframeDeduplicator{}
+}
+
+// IsDuplicate reports exact repeated keyframe access units observed close
+// together. Distinct IDR slices within one access unit remain untouched.
+func (d *KeyframeDeduplicator) IsDuplicate(timestampMs int64, capturedAtMs int64, payload []byte, observedAt time.Time, window time.Duration) bool {
+	digest := sha256.Sum256(payload)
+	duplicate := d.hasPrevious && d.timestampMs == timestampMs && d.digest == digest
+	if duplicate {
+		if capturedAtMs > 0 && d.capturedAtMs > 0 {
+			gap := time.Duration(capturedAtMs-d.capturedAtMs) * time.Millisecond
+			duplicate = gap >= 0 && gap <= window
+		} else {
+			gap := observedAt.Sub(d.observedAt)
+			duplicate = gap >= 0 && gap <= window
+		}
+	}
+
+	d.hasPrevious = true
+	d.timestampMs = timestampMs
+	d.capturedAtMs = capturedAtMs
+	d.observedAt = observedAt
+	d.digest = digest
+	return duplicate
+}

machinery/src/cloud/livemoq/dedup_test.go

@@ -0,0 +1,60 @@
+package livemoq
+
+import (
+	"testing"
+	"time"
+)
+
+func TestKeyframeDeduplicator(t *testing.T) {
+	now := time.UnixMilli(10_000)
+	window := 500 * time.Millisecond
+	payload := []byte{0x00, 0x00, 0x00, 0x01, 0x65, 0x88}
+	deduplicator := KeyframeDeduplicator{}
+
+	if deduplicator.IsDuplicate(1_000, 10_000, payload, now, window) {
+		t.Fatal("first keyframe reported as duplicate")
+	}
+	if !deduplicator.IsDuplicate(1_000, 10_020, payload, now.Add(20*time.Millisecond), window) {
+		t.Fatal("exact repeated keyframe was not reported as duplicate")
+	}
+	if deduplicator.IsDuplicate(2_000, 11_000, payload, now.Add(time.Second), window) {
+		t.Fatal("same payload with a new timestamp reported as duplicate")
+	}
+	if deduplicator.IsDuplicate(2_000, 11_020, append(payload, 0x01), now.Add(1020*time.Millisecond), window) {
+		t.Fatal("different payload with the same timestamp reported as duplicate")
+	}
+}
+
+func TestKeyframeDeduplicatorAllowsTimestampReuseOutsideWindow(t *testing.T) {
+	now := time.UnixMilli(10_000)
+	payload := []byte{0x00, 0x00, 0x00, 0x01, 0x65, 0x88}
+	deduplicator := KeyframeDeduplicator{}
+
+	deduplicator.IsDuplicate(1_000, 10_000, payload, now, 500*time.Millisecond)
+	if deduplicator.IsDuplicate(1_000, 20_000, payload, now.Add(10*time.Second), 500*time.Millisecond) {
+		t.Fatal("later keyframe after timestamp reset reported as duplicate")
+	}
+}
+
+func TestKeyframeDeduplicatorFallsBackToObservationTime(t *testing.T) {
+	now := time.UnixMilli(10_000)
+	payload := []byte{0x65, 0x88}
+	deduplicator := KeyframeDeduplicator{}
+
+	deduplicator.IsDuplicate(1_000, 0, payload, now, 500*time.Millisecond)
+	if !deduplicator.IsDuplicate(1_000, 0, payload, now.Add(20*time.Millisecond), 500*time.Millisecond) {
+		t.Fatal("duplicate without capture time was not reported")
+	}
+}
+
+func TestKeyframeDeduplicatorReset(t *testing.T) {
+	now := time.UnixMilli(10_000)
+	payload := []byte{0x65, 0x88}
+	deduplicator := KeyframeDeduplicator{}
+
+	deduplicator.IsDuplicate(1_000, 10_000, payload, now, 500*time.Millisecond)
+	deduplicator.Reset()
+	if deduplicator.IsDuplicate(1_000, 10_020, payload, now.Add(20*time.Millisecond), 500*time.Millisecond) {
+		t.Fatal("first keyframe after reset reported as duplicate")
+	}
+}

machinery/src/cloud/livemoq/recovery.go

@@ -0,0 +1,101 @@
+package livemoq
+
+import (
+	"sync/atomic"
+	"time"
+)
+
+type FrameGateEvent uint8
+
+const (
+	FrameGateEventNone FrameGateEvent = iota
+	FrameGateEventStarted
+	FrameGateEventLagging
+	FrameGateEventRecovered
+)
+
+// FrameGate keeps publication on a decodable, recent GOP.
+type FrameGate struct {
+	started    bool
+	recovering bool
+}
+
+// AudienceGate publishes every frame while watched and only keyframes while
+// idle. The idle keyframes keep the relay's latest cached group current without
+// paying the bandwidth cost of a full stream when nobody is watching.
+type AudienceGate struct {
+	idle bool
+}
+
+func (g *AudienceGate) Allow(hasSubscribers bool, isKeyFrame bool) (allowed bool, enteredIdle bool) {
+	if hasSubscribers {
+		g.idle = false
+		return true, false
+	}
+
+	enteredIdle = !g.idle
+	g.idle = true
+	return isKeyFrame, enteredIdle
+}
+
+// WriteWatchdog tracks the single synchronous frame write performed by a MoQ
+// publisher so another goroutine can interrupt a wedged native call.
+type WriteWatchdog struct {
+	startedAt atomic.Int64
+}
+
+func (w *WriteWatchdog) Begin(now time.Time) {
+	w.startedAt.Store(now.UnixNano())
+}
+
+func (w *WriteWatchdog) End() {
+	w.startedAt.Store(0)
+}
+
+func (w *WriteWatchdog) Elapsed(now time.Time) (time.Duration, bool) {
+	startedAt := w.startedAt.Load()
+	if startedAt == 0 {
+		return 0, false
+	}
+	elapsed := now.Sub(time.Unix(0, startedAt))
+	if elapsed < 0 {
+		elapsed = 0
+	}
+	return elapsed, true
+}
+
+// Reset closes the gate so publication resumes on the next keyframe. Entering
+// idle mode uses it to start the relay-cache refresh on a complete GOP.
+func (g *FrameGate) Reset() {
+	g.started = false
+	g.recovering = false
+}
+
+// Allow rejects stale frames and waits for a fresh keyframe before reopening.
+func (g *FrameGate) Allow(isKeyFrame bool, capturedAtMs int64, now time.Time, maxAge time.Duration) (bool, FrameGateEvent) {
+	if capturedAtMs > 0 && now.Sub(time.UnixMilli(capturedAtMs)) > maxAge {
+		event := FrameGateEventNone
+		if g.started {
+			if !g.recovering {
+				event = FrameGateEventLagging
+			}
+			g.started = false
+			g.recovering = true
+		}
+		return false, event
+	}
+
+	if !g.started {
+		if !isKeyFrame {
+			return false, FrameGateEventNone
+		}
+		g.started = true
+		if g.recovering {
+			g.recovering = false
+			return true, FrameGateEventRecovered
+		}
+		return true, FrameGateEventStarted
+	}
+
+	return true, FrameGateEventNone
+}

machinery/src/cloud/livemoq/recovery_test.go

@@ -0,0 +1,91 @@
+package livemoq
+
+import (
+	"testing"
+	"time"
+)
+
+func TestFrameGateRecoversAtFreshKeyframe(t *testing.T) {
+	now := time.UnixMilli(10_000)
+	maxAge := 1500 * time.Millisecond
+	gate := FrameGate{}
+
+	tests := []struct {
+		name         string
+		isKeyFrame   bool
+		capturedAtMs int64
+		wantAllowed  bool
+		wantEvent    FrameGateEvent
+	}{
+		{name: "waits for initial keyframe", capturedAtMs: 10_000},
+		{name: "starts at initial keyframe", isKeyFrame: true, capturedAtMs: 10_000, wantAllowed: true, wantEvent: FrameGateEventStarted},
+		{name: "publishes fresh delta", capturedAtMs: 10_020, wantAllowed: true},
+		{name: "detects stale packet", capturedAtMs: 8_000, wantEvent: FrameGateEventLagging},
+		{name: "rejects fresh delta while recovering", capturedAtMs: 10_040},
+		{name: "rejects stale keyframe without duplicate event", isKeyFrame: true, capturedAtMs: 8_000},
+		{name: "recovers at fresh keyframe", isKeyFrame: true, capturedAtMs: 10_060, wantAllowed: true, wantEvent: FrameGateEventRecovered},
+		{name: "publishes delta after recovery", capturedAtMs: 10_080, wantAllowed: true},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			allowed, event := gate.Allow(test.isKeyFrame, test.capturedAtMs, now, maxAge)
+			if allowed != test.wantAllowed {
+				t.Fatalf("Allow() allowed = %t, want %t", allowed, test.wantAllowed)
+			}
+			if event != test.wantEvent {
+				t.Fatalf("Allow() event = %d, want %d", event, test.wantEvent)
+			}
+		})
+	}
+}
+
+func TestFrameGateAllowsMissingCaptureTime(t *testing.T) {
+	gate := FrameGate{}
+	allowed, event := gate.Allow(true, 0, time.Now(), time.Second)
+	if !allowed || event != FrameGateEventStarted {
+		t.Fatalf("Allow() = (%t, %d), want (true, %d)", allowed, event, FrameGateEventStarted)
+	}
+}
+
+func TestAudienceGateRefreshesOnlyKeyframesWhileIdle(t *testing.T) {
+	gate := AudienceGate{}
+
+	if allowed, enteredIdle := gate.Allow(true, false); !allowed || enteredIdle {
+		t.Fatalf("active delta = (%t, %t), want (true, false)", allowed, enteredIdle)
+	}
+	if allowed, enteredIdle := gate.Allow(false, false); allowed || !enteredIdle {
+		t.Fatalf("first idle delta = (%t, %t), want (false, true)", allowed, enteredIdle)
+	}
+	if allowed, enteredIdle := gate.Allow(false, true); !allowed || enteredIdle {
+		t.Fatalf("idle keyframe = (%t, %t), want (true, false)", allowed, enteredIdle)
+	}
+	if allowed, enteredIdle := gate.Allow(false, false); allowed || enteredIdle {
+		t.Fatalf("later idle delta = (%t, %t), want (false, false)", allowed, enteredIdle)
+	}
+	if allowed, enteredIdle := gate.Allow(true, false); !allowed || enteredIdle {
+		t.Fatalf("resumed delta = (%t, %t), want (true, false)", allowed, enteredIdle)
+	}
+	if allowed, enteredIdle := gate.Allow(false, true); !allowed || !enteredIdle {
+		t.Fatalf("next idle keyframe = (%t, %t), want (true, true)", allowed, enteredIdle)
+	}
+}
+
+func TestWriteWatchdogTracksActiveWrite(t *testing.T) {
+	now := time.Unix(10, 0)
+	watchdog := WriteWatchdog{}
+
+	if elapsed, active := watchdog.Elapsed(now); active || elapsed != 0 {
+		t.Fatalf("Elapsed() before Begin() = (%s, %t), want (0, false)", elapsed, active)
+	}
+
+	watchdog.Begin(now)
+	if elapsed, active := watchdog.Elapsed(now.Add(5 * time.Second)); !active || elapsed != 5*time.Second {
+		t.Fatalf("Elapsed() during write = (%s, %t), want (5s, true)", elapsed, active)
+	}
+
+	watchdog.End()
+	if elapsed, active := watchdog.Elapsed(now.Add(6 * time.Second)); active || elapsed != 0 {
+		t.Fatalf("Elapsed() after End() = (%s, %t), want (0, false)", elapsed, active)
+	}
+}

machinery/src/cloud/livemoq_disabled.go

@@ -2,7 +2,20 @@
 
 package cloud
 
-import "github.com/kerberos-io/agent/machinery/src/models"
+import (
+	"context"
+
+	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/packets"
+)
 
 // StartLiveStreamMoQ is disabled in the standard Agent build.
-func StartLiveStreamMoQ(_ *models.Configuration, _ *models.Communication, _ bool) {}
+func StartLiveStreamMoQ(
+	_ context.Context,
+	_ *models.Configuration,
+	_ *models.Communication,
+	_ bool,
+	_ *packets.Queue,
+	_ *packets.Queue,
+) {
+}

machinery/src/cloud/livemoq_enabled.go

@@ -4,89 +4,226 @@ package cloud
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"net/url"
 	"os"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/cloud/livemoq"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/moq-dev/moq-go/moq"
+	log "github.com/sirupsen/logrus"
 )
 
 const (
-	defaultMoQRelayURL = "https://relay.uug.ai/anon"
-	minMoQRetryDelay   = time.Second
-	maxMoQRetryDelay   = 30 * time.Second
+	defaultMoQRelayURL      = "https://relay.uug.ai/anon"
+	minMoQRetryDelay        = time.Second
+	maxMoQRetryDelay        = 30 * time.Second
+	defaultMoQLivePacketAge = 1500 * time.Millisecond
+	minMoQLivePacketAge     = 250 * time.Millisecond
+	maxMoQLivePacketAge     = 30 * time.Second
+	defaultMoQWriteTimeout  = 5 * time.Second
+	minMoQWriteTimeout      = time.Second
+	maxMoQWriteTimeout      = time.Minute
+	moQWriteWatchInterval   = 250 * time.Millisecond
+	slowMoQWriteThreshold   = 100 * time.Millisecond
+	moQWriteWarningInterval = 10 * time.Second
+	duplicateKeyframeWindow = 500 * time.Millisecond
 )
 
 type liveMoQConfig struct {
-	relayURL    string
-	broadcast   string
-	quality     string
-	sourceLabel string
-	queue       *packets.Queue
+	relayURL      string
+	broadcast     string
+	quality       string
+	sourceLabel   string
+	queue         *packets.Queue
+	communication *models.Communication
+	maxPacketAge  time.Duration
+	writeTimeout  time.Duration
+}
+
+func (c liveMoQConfig) logEntry(event string) *log.Entry {
+	return log.WithFields(log.Fields{
+		"component": "moq",
+		"event":     event,
+		"quality":   c.quality,
+		"stream":    c.sourceLabel,
+	})
+}
+
+func moQRelayHost(relayURL string) string {
+	parsed, err := url.Parse(relayURL)
+	if err != nil {
+		return ""
+	}
+	return parsed.Host
+}
+
+func (c liveMoQConfig) packetAgeLimit() time.Duration {
+	if c.maxPacketAge > 0 {
+		return c.maxPacketAge
+	}
+	return defaultMoQLivePacketAge
+}
+
+func (c liveMoQConfig) writeTimeoutLimit() time.Duration {
+	if c.writeTimeout > 0 {
+		return c.writeTimeout
+	}
+	return defaultMoQWriteTimeout
+}
+
+func boundedMoQDuration(name string, fallback, minimum, maximum time.Duration) time.Duration {
+	raw := strings.TrimSpace(os.Getenv(name))
+	if raw == "" {
+		return fallback
+	}
+
+	value, err := time.ParseDuration(raw)
+	if err != nil {
+		log.WithError(err).WithFields(log.Fields{
+			"component":        "moq",
+			"configured_value": raw,
+			"effective_value":  fallback.String(),
+			"event":            "duration_invalid",
+			"variable":         name,
+		}).Warn("Invalid MoQ duration; using default")
+		return fallback
+	}
+	if value < minimum {
+		log.WithFields(log.Fields{
+			"component":        "moq",
+			"configured_value": value.String(),
+			"effective_value":  minimum.String(),
+			"event":            "duration_clamped",
+			"minimum":          minimum.String(),
+			"variable":         name,
+		}).Warn("MoQ duration is below the supported minimum")
+		return minimum
+	}
+	if value > maximum {
+		log.WithFields(log.Fields{
+			"component":        "moq",
+			"configured_value": value.String(),
+			"effective_value":  maximum.String(),
+			"event":            "duration_clamped",
+			"maximum":          maximum.String(),
+			"variable":         name,
+		}).Warn("MoQ duration exceeds the supported maximum")
+		return maximum
+	}
+	return value
 }
 
 // StartLiveStreamMoQ starts the publisher only in the dedicated MoQ build and
 // only when explicitly enabled by the deployment.
-func StartLiveStreamMoQ(configuration *models.Configuration, communication *models.Communication, subStreamEnabled bool) {
+//
+// Unlike WebRTC and HLS — where a viewer negotiates a session with the Agent and
+// can therefore ask for another quality on the fly — MoQ viewers subscribe to a
+// relay and never talk to the Agent. The quality selector is honoured by
+// publishing each tier as its OWN broadcast (see livemoq.BroadcastPath): the
+// high tier from the camera's highest-resolution stream and the low tier from
+// its sub stream, so switching quality in the frontend is a resubscribe to the
+// other path. Each tier uploads every frame while watched and keyframes only
+// while idle, keeping the relay cache current without continuously sending the
+// full unused stream.
+func StartLiveStreamMoQ(
+	ctx context.Context,
+	configuration *models.Configuration,
+	communication *models.Communication,
+	subStreamEnabled bool,
+	mainQueue *packets.Queue,
+	subQueue *packets.Queue,
+) {
 	if os.Getenv("AGENT_LIVE_MOQ_ENABLED") != "true" {
 		return
 	}
 
 	config := configuration.Config
 	if config.Offline == "true" || config.Capture.Liveview == "false" {
-		log.Log.Info("cloud.StartLiveStreamMoQ(): disabled by Agent live-view configuration")
+		log.WithFields(log.Fields{
+			"component": "moq",
+			"event":     "publisher_disabled",
+			"liveview":  config.Capture.Liveview,
+			"offline":   config.Offline,
+		}).Debug("MoQ publisher disabled by Agent configuration")
 		return
 	}
 	if config.Key == "" {
-		log.Log.Warning("cloud.StartLiveStreamMoQ(): AGENT_KEY is required")
+		log.WithFields(log.Fields{
+			"component": "moq",
+			"event":     "publisher_configuration_invalid",
+			"variable":  "AGENT_KEY",
+		}).Warn("MoQ publisher requires an Agent key")
 		return
 	}
 
-	quality := os.Getenv("AGENT_LIVE_MOQ_QUALITY")
-	if quality == "" {
-		quality = models.StreamQualityAuto
-	}
-	useSub := models.SelectSubStreamForQuality(config, quality, subStreamEnabled)
-	queue := communication.Queue
-	sourceLabel := "main"
-	if useSub && communication.SubQueue != nil {
-		queue = communication.SubQueue
-		sourceLabel = "sub"
-	}
-	if queue == nil {
-		log.Log.Warning("cloud.StartLiveStreamMoQ(): selected packet queue is unavailable")
-		return
+	// Both tiers are published by default. AGENT_LIVE_MOQ_QUALITY pins the Agent
+	// to a single tier for deployments that must never publish the other one
+	// (viewers asking for the pinned-away tier then find no broadcast).
+	qualities := []string{models.StreamQualityHigh, models.StreamQualityLow}
+	switch strings.ToLower(strings.TrimSpace(os.Getenv("AGENT_LIVE_MOQ_QUALITY"))) {
+	case models.StreamQualityHigh:
+		qualities = []string{models.StreamQualityHigh}
+	case models.StreamQualityLow:
+		qualities = []string{models.StreamQualityLow}
 	}
 
 	relayURL := os.Getenv("AGENT_LIVE_MOQ_URL")
 	if relayURL == "" {
 		relayURL = defaultMoQRelayURL
 	}
-	publisherConfig := liveMoQConfig{
-		relayURL:    relayURL,
-		broadcast:   livemoq.BroadcastPath(os.Getenv("AGENT_LIVE_MOQ_BROADCAST_PREFIX"), config.Key),
-		quality:     quality,
-		sourceLabel: sourceLabel,
-		queue:       queue,
-	}
+	broadcastPrefix := os.Getenv("AGENT_LIVE_MOQ_BROADCAST_PREFIX")
+	maxPacketAge := boundedMoQDuration("AGENT_LIVE_MOQ_MAX_PACKET_AGE", defaultMoQLivePacketAge, minMoQLivePacketAge, maxMoQLivePacketAge)
+	writeTimeout := boundedMoQDuration("AGENT_LIVE_MOQ_WRITE_TIMEOUT", defaultMoQWriteTimeout, minMoQWriteTimeout, maxMoQWriteTimeout)
 
-	ctx := context.Background()
-	if communication.Context != nil {
-		ctx = *communication.Context
+	var publishers sync.WaitGroup
+	for _, quality := range qualities {
+		queue := mainQueue
+		sourceLabel := "main"
+		if models.SelectSubStreamForQuality(config, quality, subStreamEnabled) && subQueue != nil {
+			queue = subQueue
+			sourceLabel = "sub"
+		}
+		if queue == nil {
+			log.WithFields(log.Fields{
+				"component": "moq",
+				"event":     "packet_queue_unavailable",
+				"quality":   quality,
+			}).Warn("MoQ packet queue is unavailable")
+			continue
+		}
+		publisherConfig := liveMoQConfig{
+			relayURL:      relayURL,
+			broadcast:     livemoq.BroadcastPath(broadcastPrefix, config.Key, quality),
+			quality:       quality,
+			sourceLabel:   sourceLabel,
+			queue:         queue,
+			communication: communication,
+			maxPacketAge:  maxPacketAge,
+			writeTimeout:  writeTimeout,
+		}
+		publishers.Add(1)
+		go func() {
+			defer publishers.Done()
+			runLiveStreamMoQ(ctx, publisherConfig)
+		}()
 	}
-	go runLiveStreamMoQ(ctx, publisherConfig)
+	publishers.Wait()
 }
 
 func runLiveStreamMoQ(ctx context.Context, config liveMoQConfig) {
-	log.Log.Info(fmt.Sprintf(
-		"cloud.runLiveStreamMoQ(): publishing %s stream (quality=%s) to %s/%s",
-		config.sourceLabel, config.quality, strings.TrimRight(config.relayURL, "/"), config.broadcast,
-	))
+	config.logEntry("publisher_started").Info("MoQ publisher started")
+	config.logEntry("publisher_configuration").WithFields(log.Fields{
+		"max_packet_age_ms": config.packetAgeLimit().Milliseconds(),
+		"relay_host":        moQRelayHost(config.relayURL),
+		"write_timeout_ms":  config.writeTimeoutLimit().Milliseconds(),
+	}).Debug("MoQ publisher configuration")
 
 	retryDelay := minMoQRetryDelay
 	for ctx.Err() == nil {
@@ -95,7 +232,13 @@ func runLiveStreamMoQ(ctx context.Context, config liveMoQConfig) {
 		if ctx.Err() != nil {
 			return
 		}
-		log.Log.Warning("cloud.runLiveStreamMoQ(): publisher stopped: " + err.Error())
+		if config.communication != nil {
+			config.communication.RecordMoQReconnect(config.quality)
+		}
+		config.logEntry("publisher_reconnecting").WithError(err).WithFields(log.Fields{
+			"connected_duration_ms": time.Since(connectedAt).Milliseconds(),
+			"retry_delay_ms":        retryDelay.Milliseconds(),
+		}).Warn("MoQ publisher stopped; reconnecting")
 		if time.Since(connectedAt) >= time.Minute {
 			retryDelay = minMoQRetryDelay
 		}
@@ -122,46 +265,248 @@ func publishLiveStreamMoQ(ctx context.Context, config liveMoQConfig) error {
 		return fmt.Errorf("connect to relay: %w", err)
 	}
 	defer client.Close()
+	config.logEntry("relay_connected").WithField("relay_host", moQRelayHost(config.relayURL)).
+		Info("MoQ relay connected")
+
+	publisherCtx, cancelPublisher := context.WithCancel(ctx)
+	defer cancelPublisher()
+	sessionClosed := make(chan error, 1)
+	sessionWatchDone := make(chan struct{})
+	go func() {
+		defer close(sessionWatchDone)
+		sessionClosed <- client.Session().Closed(publisherCtx)
+		cancelPublisher()
+	}()
 
 	broadcast, err := client.CreateBroadcast(config.broadcast)
 	if err != nil {
 		return fmt.Errorf("create broadcast: %w", err)
 	}
-	defer broadcast.Finish()
+	var finishBroadcastOnce sync.Once
+	var finishBroadcastErr error
+	finishBroadcast := func() error {
+		finishBroadcastOnce.Do(func() {
+			finishBroadcastErr = broadcast.Finish()
+		})
+		return finishBroadcastErr
+	}
+	defer finishBroadcast()
 
 	stream, err := broadcast.PublishMedia("avc3", nil)
 	if err != nil {
 		return fmt.Errorf("create H.264 media stream: %w", err)
 	}
-	defer stream.Finish()
+	var finishStreamOnce sync.Once
+	var finishStreamErr error
+	finishStream := func() error {
+		finishStreamOnce.Do(func() {
+			finishStreamErr = stream.Finish()
+		})
+		return finishStreamErr
+	}
+	defer finishStream()
+
+	// Publish the complete stream while watched and only keyframes while idle.
+	// `publishing` starts true so the track becomes discoverable on the relay
+	// before the first subscriber; after that, the subscriber watcher selects the
+	// full-rate or keyframe-refresh mode.
+	publishing := &atomic.Bool{}
+	publishing.Store(true)
+	subscriberWatchDone := make(chan struct{})
+	go func() {
+		defer close(subscriberWatchDone)
+		watchLiveStreamMoQSubscribers(publisherCtx, stream, publishing, config)
+	}()
+	writeWatchdog := &livemoq.WriteWatchdog{}
+	writeWatchDone := make(chan struct{})
+	closePublisher := func() error {
+		return closeLiveMoQPublisher(finishStream, finishBroadcast, client.Close)
+	}
+	go func() {
+		defer close(writeWatchDone)
+		watchLiveStreamMoQWrites(publisherCtx, closePublisher, writeWatchdog, config)
+	}()
+	defer func() {
+		cancelPublisher()
+		<-writeWatchDone
+		<-subscriberWatchDone
+		<-sessionWatchDone
+	}()
 
 	cursor := config.queue.Latest()
-	writing := false
+	gate := livemoq.FrameGate{}
+	audienceGate := livemoq.AudienceGate{}
+	deduplicator := livemoq.KeyframeDeduplicator{}
+	var lastSlowWriteWarning time.Time
+	var lastDuplicateKeyframeWarning time.Time
 	for {
-		packet, err := cursor.ReadPacket()
+		select {
+		case err := <-sessionClosed:
+			return fmt.Errorf("relay session closed: %w", err)
+		default:
+		}
+
+		packet, err := cursor.ReadPacketContext(publisherCtx)
 		if err != nil {
+			select {
+			case sessionErr := <-sessionClosed:
+				return fmt.Errorf("relay session closed: %w", sessionErr)
+			default:
+			}
+			if ctx.Err() != nil {
+				return ctx.Err()
+			}
 			return fmt.Errorf("read packet: %w", err)
 		}
 		if !packet.IsVideo || len(packet.Data) == 0 || !strings.EqualFold(packet.Codec, "H264") {
 			continue
 		}
-		if !writing {
-			if !packet.IsKeyFrame {
-				continue
-			}
-			writing = true
-			log.Log.Info("cloud.publishLiveStreamMoQ(): first H.264 keyframe received; broadcast is live")
+		allowedForAudience, enteredIdle := audienceGate.Allow(publishing.Load(), packet.IsKeyFrame)
+		if enteredIdle {
+			gate.Reset()
+			deduplicator.Reset()
 		}
-		payload, err := livemoq.NormalizeH264AccessUnit(packet.Data)
+		if !allowedForAudience {
+			continue
+		}
+		allowed, event := gate.Allow(packet.IsKeyFrame, packet.CurrentTime, time.Now(), config.packetAgeLimit())
+		switch event {
+		case livemoq.FrameGateEventStarted:
+			config.logEntry("broadcast_live").WithFields(log.Fields{
+				"codec":        packet.Codec,
+				"timestamp_ms": packet.Time,
+			}).Info("MoQ broadcast is live")
+		case livemoq.FrameGateEventLagging:
+			config.logEntry("stream_lagging").WithFields(log.Fields{
+				"max_packet_age_ms": config.packetAgeLimit().Milliseconds(),
+				"timestamp_ms":      packet.Time,
+			}).Warn("MoQ stream is lagging; dropping packets until a recent keyframe")
+		case livemoq.FrameGateEventRecovered:
+			config.logEntry("stream_recovered").WithField("timestamp_ms", packet.Time).
+				Info("MoQ stream recovered at a recent keyframe")
+		}
+		if !allowed {
+			continue
+		}
+		payload, normalizationStats, err := livemoq.NormalizeH264AccessUnitWithStats(packet.Data)
 		if err != nil {
 			return fmt.Errorf("normalize H.264 access unit: %w", err)
 		}
+		if normalizationStats.DuplicateIDRNALUs > 0 && time.Since(lastDuplicateKeyframeWarning) >= moQWriteWarningInterval {
+			config.logEntry("duplicate_idr_removed").WithFields(log.Fields{
+				"duplicate_idr_count": normalizationStats.DuplicateIDRNALUs,
+				"timestamp_ms":        packet.Time,
+			}).Warn("Removed duplicate IDR NAL units from MoQ keyframe")
+
+			lastDuplicateKeyframeWarning = time.Now()
+		}
+		if packet.IsKeyFrame && deduplicator.IsDuplicate(packet.Time, packet.CurrentTime, payload, time.Now(), duplicateKeyframeWindow) {
+			if time.Since(lastDuplicateKeyframeWarning) >= moQWriteWarningInterval {
+				config.logEntry("duplicate_keyframe_dropped").WithField("timestamp_ms", packet.Time).
+					Warn("Dropped duplicate MoQ keyframe")
+
+				lastDuplicateKeyframeWarning = time.Now()
+			}
+			continue
+		}
 		frame := moq.Frame{
 			Payload:     payload,
 			TimestampUs: livemoq.TimestampUs(packet.Time),
 		}
-		if err := stream.WriteFrame(frame); err != nil {
+		writeStartedAt := time.Now()
+		writeWatchdog.Begin(writeStartedAt)
+		err = stream.WriteFrame(frame)
+		writeWatchdog.End()
+		if err != nil {
 			return fmt.Errorf("write H.264 access unit: %w", err)
 		}
+		writeDuration := time.Since(writeStartedAt)
+		if config.communication != nil {
+			config.communication.RecordMoQWrite(config.quality, writeDuration, time.Now())
+		}
+		if writeDuration >= slowMoQWriteThreshold && time.Since(lastSlowWriteWarning) >= moQWriteWarningInterval {
+			packetAge := time.Duration(0)
+			if packet.CurrentTime > 0 {
+				packetAge = time.Since(time.UnixMilli(packet.CurrentTime))
+				if packetAge < 0 {
+					packetAge = 0
+				}
+			}
+			config.logEntry("slow_frame_write").WithFields(log.Fields{
+				"duration_ms":   writeDuration.Milliseconds(),
+				"keyframe":      packet.IsKeyFrame,
+				"packet_age_ms": packetAge.Milliseconds(),
+			}).Warn("MoQ frame write was slow")
+
+			lastSlowWriteWarning = time.Now()
+		}
+	}
+}
+
+func closeLiveMoQPublisher(finishStream, finishBroadcast, closeClient func() error) error {
+	streamErr := finishStream()
+	broadcastErr := finishBroadcast()
+	clientErr := closeClient()
+	return errors.Join(streamErr, broadcastErr, clientErr)
+}
+
+func watchLiveStreamMoQWrites(ctx context.Context, closePublisher func() error, watchdog *livemoq.WriteWatchdog, config liveMoQConfig) {
+	ticker := time.NewTicker(moQWriteWatchInterval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			if err := closePublisher(); err != nil {
+				config.logEntry("publisher_close_failed").WithError(err).
+					Debug("MoQ publisher reported an error while closing")
+			}
+			return
+		case now := <-ticker.C:
+			writeDuration, active := watchdog.Elapsed(now)
+			if !active || writeDuration < config.writeTimeoutLimit() {
+				continue
+			}
+			config.logEntry("frame_write_timeout").WithFields(log.Fields{
+				"duration_ms": writeDuration.Milliseconds(),
+				"timeout_ms":  config.writeTimeoutLimit().Milliseconds(),
+			}).Warn("MoQ frame write timed out; reconnecting relay client")
+
+			if config.communication != nil {
+				config.communication.RecordMoQWriteTimeout()
+			}
+			if err := closePublisher(); err != nil {
+				config.logEntry("publisher_close_failed").WithError(err).
+					Warn("MoQ publisher reported an error while closing after a write timeout")
+			}
+			return
+		}
+	}
+}
+
+// watchLiveStreamMoQSubscribers flips the publisher between uploading and idling
+// as viewers subscribe to and leave this tier's broadcast. Used and Unused both
+// block, so they are followed from their own goroutine.
+//
+// It deliberately never turns publishing off before the first subscriber has
+// been observed: the relay catalog is only complete once media has flowed, so
+// going idle up front could keep the tier undiscoverable. On any error it fails
+// open (keeps publishing) — a stalled watcher must never take the live view down.
+func watchLiveStreamMoQSubscribers(ctx context.Context, stream *moq.MediaProducer, publishing *atomic.Bool, config liveMoQConfig) {
+	for ctx.Err() == nil {
+		if err := stream.Used(ctx); err != nil {
+			publishing.Store(true)
+			return
+		}
+		if publishing.CompareAndSwap(false, true) {
+			config.logEntry("subscriber_joined").Info("MoQ subscriber joined; resuming broadcast")
+		}
+
+		if err := stream.Unused(ctx); err != nil {
+			publishing.Store(true)
+			return
+		}
+		publishing.Store(false)
+		config.logEntry("subscribers_idle").Info("MoQ broadcast idle; refreshing keyframes only")
 	}
 }

machinery/src/cloud/livemoq_enabled_test.go

@@ -0,0 +1,212 @@
+//go:build moq
+
+package cloud
+
+import (
+	"context"
+	"errors"
+	"reflect"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/kerberos-io/agent/machinery/src/cloud/livemoq"
+	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/moq-dev/moq-go/moq"
+)
+
+func TestCloseLiveMoQPublisherFinishesBeforeClosingClient(t *testing.T) {
+	var order []string
+	streamErr := errors.New("stream finish failed")
+	broadcastErr := errors.New("broadcast finish failed")
+	clientErr := errors.New("client close failed")
+
+	err := closeLiveMoQPublisher(
+		func() error {
+			order = append(order, "stream")
+			return streamErr
+		},
+		func() error {
+			order = append(order, "broadcast")
+			return broadcastErr
+		},
+		func() error {
+			order = append(order, "client")
+			return clientErr
+		},
+	)
+
+	if want := []string{"stream", "broadcast", "client"}; !reflect.DeepEqual(order, want) {
+		t.Fatalf("shutdown order = %v, want %v", order, want)
+	}
+	for _, want := range []error{streamErr, broadcastErr, clientErr} {
+		if !errors.Is(err, want) {
+			t.Errorf("shutdown error %v does not include %v", err, want)
+		}
+	}
+}
+
+func TestWatchLiveStreamMoQWritesClosesClientWhenContextEnds(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	closed := make(chan struct{})
+	var closeCalls atomic.Int32
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		watchLiveStreamMoQWrites(ctx, func() error {
+			if closeCalls.Add(1) == 1 {
+				close(closed)
+			}
+			return nil
+		}, &livemoq.WriteWatchdog{}, liveMoQConfig{quality: "low", sourceLabel: "sub"})
+	}()
+
+	cancel()
+	select {
+	case <-closed:
+	case <-time.After(time.Second):
+		t.Fatal("watchLiveStreamMoQWrites() did not close the client after cancellation")
+	}
+	<-done
+	if got := closeCalls.Load(); got != 1 {
+		t.Fatalf("close calls = %d, want 1", got)
+	}
+}
+
+func TestBoundedMoQDuration(t *testing.T) {
+	const name = "AGENT_LIVE_MOQ_TEST_DURATION"
+	tests := []struct {
+		name string
+		raw  string
+		want time.Duration
+	}{
+		{name: "unset", want: 5 * time.Second},
+		{name: "valid", raw: "12s", want: 12 * time.Second},
+		{name: "invalid", raw: "later", want: 5 * time.Second},
+		{name: "below minimum", raw: "100ms", want: time.Second},
+		{name: "above maximum", raw: "2m", want: time.Minute},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			t.Setenv(name, test.raw)
+			if got := boundedMoQDuration(name, 5*time.Second, time.Second, time.Minute); got != test.want {
+				t.Fatalf("boundedMoQDuration() = %s, want %s", got, test.want)
+			}
+		})
+	}
+}
+
+func TestWatchLiveStreamMoQWritesRecordsTimeout(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	closed := make(chan struct{})
+	communication := &models.Communication{}
+	watchdog := &livemoq.WriteWatchdog{}
+	watchdog.Begin(time.Now().Add(-time.Second))
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		watchLiveStreamMoQWrites(ctx, func() error {
+			close(closed)
+			return nil
+		}, watchdog, liveMoQConfig{
+			quality:       models.StreamQualityLow,
+			sourceLabel:   "sub",
+			communication: communication,
+			writeTimeout:  10 * time.Millisecond,
+		})
+	}()
+
+	select {
+	case <-closed:
+	case <-time.After(time.Second):
+		t.Fatal("watchLiveStreamMoQWrites() did not close the client after a write timeout")
+	}
+	<-done
+	if got := communication.RecoveryTelemetry().MoQWriteTimeouts; got != 1 {
+		t.Fatalf("MoQ write timeouts = %d, want 1", got)
+	}
+}
+
+func TestNativeMoQClientCloseStopsActiveWriter(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	server, err := moq.Listen(ctx, "127.0.0.1:0", moq.WithTLSGenerate("localhost"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	serveDone := make(chan error, 1)
+	go func() {
+		serveDone <- server.Serve(ctx)
+	}()
+
+	client, err := moq.Dial(ctx, "https://"+server.LocalAddr(), moq.WithTLSVerify(false))
+	if err != nil {
+		server.Close()
+		t.Fatal(err)
+	}
+	broadcast, err := client.CreateBroadcast("test/native-close")
+	if err != nil {
+		client.Close()
+		server.Close()
+		t.Fatal(err)
+	}
+	stream, err := broadcast.PublishMedia("avc3", nil)
+	if err != nil {
+		client.Close()
+		server.Close()
+		t.Fatal(err)
+	}
+
+	firstWrite := make(chan struct{})
+	writerDone := make(chan error, 1)
+	go func() {
+		payload := []byte{
+			0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x1e, 0xd9, 0x00, 0xa0, 0x47, 0xfe, 0xc8,
+			0x00, 0x00, 0x00, 0x01, 0x68, 0xce, 0x38, 0x80,
+			0x00, 0x00, 0x00, 0x01, 0x65, 0x88,
+		}
+		for timestamp := uint64(0); ; timestamp++ {
+			err := stream.WriteFrame(moq.Frame{Payload: payload, TimestampUs: timestamp})
+			if err != nil {
+				writerDone <- err
+				return
+			}
+			if timestamp == 0 {
+				close(firstWrite)
+			}
+		}
+	}()
+
+	select {
+	case <-firstWrite:
+	case <-ctx.Done():
+		t.Fatal("native MoQ writer did not start")
+	}
+	if err := stream.Finish(); err != nil {
+		t.Fatal(err)
+	}
+	if err := client.Close(); err != nil {
+		t.Fatal(err)
+	}
+	select {
+	case err := <-writerDone:
+		if err == nil {
+			t.Fatal("native MoQ writer stopped without an error after client close")
+		}
+	case <-ctx.Done():
+		t.Fatal("native MoQ client close did not stop the writer")
+	}
+
+	_ = broadcast.Finish()
+	_ = server.Close()
+	select {
+	case <-serveDone:
+	case <-ctx.Done():
+		t.Fatal("native MoQ server did not stop")
+	}
+}

machinery/src/cloud/livesnapshot/publisher.go

@@ -30,7 +30,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
+	log "github.com/sirupsen/logrus"
 )
 
 const (
@@ -131,7 +131,10 @@ func (p *Publisher) PublishSnapshot(ctx context.Context, jpeg []byte) error {
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
 		return fmt.Errorf("livesnapshot: upload snapshot rejected: %s", resp.Status)
 	}
-	log.Log.Debug("livesnapshot.Publisher.PublishSnapshot(): shipped preview frame for device " + p.cfg.DeviceKey)
+	log.WithFields(log.Fields{
+		"component": "livesnapshot",
+		"event":     "snapshot_published",
+	}).Debug("Live snapshot published")
 	return nil
 }
 

machinery/src/cloud/recording_metadata.go

@@ -15,6 +15,7 @@ import (
 const recordingFPSHeader = "X-Kerberos-Storage-Fps"
 const recordingDurationHeader = "X-Kerberos-Storage-Duration"
 const recordingTimestampHeader = "X-Kerberos-Storage-Timestamp"
+const recordingEncryptedHeader = "X-Kerberos-Storage-Encrypted"
 
 // queuedRecordingFPS reads the FPS snapshot written into the upload marker
 // when the recording was finalized. Historical empty markers intentionally
@@ -84,5 +85,8 @@ func setQueuedRecordingMetadataHeaders(header http.Header, fileName string) {
 		if metadata.Timestamp > 0 {
 			header.Set(recordingTimestampHeader, strconv.FormatInt(metadata.Timestamp, 10))
 		}
+		if metadata.Encrypted {
+			header.Set(recordingEncryptedHeader, "true")
+		}
 	}
 }

machinery/src/cloud/s3.go

@@ -9,9 +9,9 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/minio/minio-go/v6"
+	log "github.com/sirupsen/logrus"
 )
 
 func UploadS3(configuration *models.Configuration, fileName string) (bool, bool, error) {
@@ -29,7 +29,7 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
 
 	if config.S3 == nil {
 		errorMessage := "UploadS3: Uploading Failed, as no settings found"
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, false, errors.New(errorMessage)
 	}
 
@@ -49,14 +49,14 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
 	// Check if we have some credentials otherwise we abort the request.
 	if aws_access_key_id == "" || aws_secret_access_key == "" {
 		errorMessage := "UploadS3: Uploading Failed, as no credentials found"
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, false, errors.New(errorMessage)
 	}
 
 	s3Client, err := minio.NewWithRegion("s3.amazonaws.com", aws_access_key_id, aws_secret_access_key, true, aws_region)
 	if err != nil {
 		errorMessage := "UploadS3: " + err.Error()
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, true, errors.New(errorMessage)
 	}
 
@@ -74,7 +74,7 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
 	fileParts := strings.Split(fileName, "_")
 	if len(fileParts) == 1 {
 		errorMessage := "UploadS3: " + fileName + " is not a valid name."
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, true, errors.New(errorMessage)
 	}
 
@@ -85,7 +85,7 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
 	//numberOfChanges := fileParts[4]
 	token, _ := strconv.Atoi(fileParts[5])
 
-	log.Log.Info("UploadS3: Upload started for " + fileName)
+	log.Info("UploadS3: Upload started for " + fileName)
 	fullname := "data/recordings/" + fileName
 
 	file, err := os.OpenFile(fullname, os.O_RDWR, 0755)
@@ -95,14 +95,14 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
 
 	if err != nil {
 		errorMessage := "UploadS3: " + err.Error()
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, true, errors.New(errorMessage)
 	}
 
 	fileInfo, err := file.Stat()
 	if err != nil {
 		errorMessage := "UploadS3: " + err.Error()
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, true, errors.New(errorMessage)
 	}
 
@@ -128,10 +128,10 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
 
 	if err != nil {
 		errorMessage := "UploadS3: Uploading Failed, " + err.Error()
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, true, errors.New(errorMessage)
 	} else {
-		log.Log.Info("UploadS3: Upload Finished, file has been uploaded to bucket: " + strconv.FormatInt(n, 10))
+		log.Info("UploadS3: Upload Finished, file has been uploaded to bucket: " + strconv.FormatInt(n, 10))
 		return true, true, nil
 	}
 }

machinery/src/cloud/tus_client.go

@@ -15,8 +15,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 // tusResumableVersion is the tus protocol version implemented by this client.
@@ -90,7 +90,14 @@ func logTusUploadProgress(label string, offset, size int64, loggedBucket *int64)
 	if percent > 100 {
 		percent = 100
 	}
-	log.Log.Infof("%s: resumable upload progress %d%% (%d/%d bytes)", label, percent, offset, size)
+	log.WithFields(log.Fields{
+		"bytes_sent":  offset,
+		"bytes_total": size,
+		"component":   "tus",
+		"event":       "upload_progress",
+		"progress":    percent,
+		"target":      label,
+	}).Info("Resumable upload progress")
 }
 
 // tusHeaderFunc sets the authentication and routing headers required on every
@@ -121,7 +128,7 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
 	}
 	if ferr != nil {
 		msg := label + ": resumable upload failed, file doesn't exist anymore"
-		log.Log.Info(msg)
+		log.Info(msg)
 		// The file is gone, so the legacy path cannot help either. Report it as
 		// "supported" to avoid a pointless fallback attempt.
 		return false, false, true, "", errors.New(msg)
@@ -182,7 +189,7 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
 					// The vault does not implement tus; let the caller fall back.
 					return false, false, false, "", cerr
 				}
-				log.Log.Info(label + ": resumable create failed, " + cerr.Error())
+				log.Info(label + ": resumable create failed, " + cerr.Error())
 				tusBackoff(attempt)
 				continue
 			}
@@ -200,7 +207,7 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
 				uploadURL = ""
 				continue
 			}
-			log.Log.Info(label + ": resumable head failed, " + herr.Error())
+			log.Info(label + ": resumable head failed, " + herr.Error())
 			tusBackoff(attempt)
 			continue
 		}
@@ -253,9 +260,9 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
 				if status >= 400 {
 					// Definitive rejection (e.g. provider push failed during finalize).
 					// Re-evaluate via HEAD on the next iteration to decide retry/restart.
-					log.Log.Info(label + ": resumable patch rejected, " + perr.Error())
+					log.Info(label + ": resumable patch rejected, " + perr.Error())
 				} else {
-					log.Log.Info(label + ": resumable patch failed, " + perr.Error())
+					log.Info(label + ": resumable patch failed, " + perr.Error())
 				}
 				tusBackoff(attempt)
 				patchFailed = true
@@ -354,6 +361,9 @@ func addRecordingTusMetadata(values map[string]string, fileName string) {
 	if metadata.Timestamp > 0 {
 		values["timestamp"] = strconv.FormatInt(metadata.Timestamp, 10)
 	}
+	if metadata.Encrypted {
+		values["encrypted"] = "true"
+	}
 }
 
 // tusCreate performs the tus "creation" request (POST). On success it returns

machinery/src/cloud/tus_client_test.go

@@ -264,6 +264,36 @@ func testVault(uri string) models.KStorage {
 	}
 }
 
+func TestUploadKerberosVaultSkipsEmptyRecording(t *testing.T) {
+	fileName := "1787015373_3-654_office-camera17_0-0-0-0_-1_1960.mp4"
+	withRecording(t, fileName, nil)
+
+	requestCount := 0
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requestCount++
+		w.WriteHeader(http.StatusInternalServerError)
+	}))
+	defer server.Close()
+
+	vault := testVault(server.URL)
+	configuration := &models.Configuration{Config: models.Config{
+		Key:               "device-key",
+		KStorage:          &vault,
+		KStorageSecondary: &models.KStorage{},
+	}}
+
+	uploaded, configured, err := UploadKerberosVault(configuration, fileName)
+	if err != nil {
+		t.Fatalf("UploadKerberosVault() error = %v", err)
+	}
+	if uploaded || configured {
+		t.Fatalf("UploadKerberosVault() uploaded/configured = %v/%v, want false/false", uploaded, configured)
+	}
+	if requestCount != 0 {
+		t.Fatalf("Vault received %d requests, want 0", requestCount)
+	}
+}
+
 func TestUploadVaultResumable_HappyPath(t *testing.T) {
 	srv := newFakeTus()
 	ts := httptest.NewServer(srv)
@@ -356,7 +386,7 @@ func TestQueuedRecordingFPSAllowsMissingHistoricalMarker(t *testing.T) {
 func TestQueuedRecordingMetadataHeaders(t *testing.T) {
 	fileName := "recording.mp4"
 	withRecording(t, fileName, []byte("recording"))
-	withQueuedRecordingFPS(t, fileName, `{"filename":"recording.mp4","device_key":"device-key","timestamp":1785934709414,"duration":20452,"fps":25}`)
+	withQueuedRecordingFPS(t, fileName, `{"filename":"recording.mp4","device_key":"device-key","timestamp":1785934709414,"duration":20452,"fps":25,"encrypted":true}`)
 
 	header := make(http.Header)
 	setQueuedRecordingMetadataHeaders(header, fileName)
@@ -369,6 +399,14 @@ func TestQueuedRecordingMetadataHeaders(t *testing.T) {
 	if got := header.Get(recordingTimestampHeader); got != "1785934709414" {
 		t.Fatalf("timestamp header = %q", got)
 	}
+	if got := header.Get(recordingEncryptedHeader); got != "true" {
+		t.Fatalf("encrypted header = %q", got)
+	}
+	metadata := map[string]string{}
+	addRecordingTusMetadata(metadata, fileName)
+	if got := metadata["encrypted"]; got != "true" {
+		t.Fatalf("encrypted TUS metadata = %q", got)
+	}
 }
 
 func TestQueuedRecordingFPSAllowsLegacyMarkerFileName(t *testing.T) {

machinery/src/components/backchannel.go

@@ -2,26 +2,83 @@ package components
 
 import (
 	"bufio"
-	"fmt"
+	"context"
+	"math/rand"
 	"os"
 	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/capture"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/kerberos-io/joy4/av"
 	"github.com/pion/rtp"
+	log "github.com/sirupsen/logrus"
 	"github.com/zaf/g711"
 )
 
+const (
+	backchannelSampleRate       = 8000
+	backchannelTalkspurtGap     = 500 * time.Millisecond
+	backchannelReconnectInitial = time.Second
+	backchannelReconnectMax     = 30 * time.Second
+)
+
+type backchannelClient interface {
+	ConnectBackChannel(ctx context.Context, otelContext context.Context) error
+	StartBackChannel(ctx context.Context, otelContext context.Context) error
+	WritePacket(pkt packets.Packet) error
+	Close(otelContext context.Context) error
+}
+
+type backchannelPacketizer struct {
+	sequenceNumber uint16
+	timestamp      uint32
+	ssrc           uint32
+	lastPacketAt   time.Time
+}
+
+func newBackchannelPacketizer() backchannelPacketizer {
+	return backchannelPacketizer{
+		sequenceNumber: uint16(rand.Uint32()),
+		timestamp:      rand.Uint32(),
+		ssrc:           rand.Uint32(),
+	}
+}
+
+func (p *backchannelPacketizer) packet(audio models.AudioDataPartial, now time.Time) packets.Packet {
+	bufferUlaw := make([]byte, len(audio.Data))
+	for index, sample := range audio.Data {
+		bufferUlaw[index] = g711.EncodeUlawFrame(sample)
+	}
+
+	pkt := packets.Packet{
+		Packet: &rtp.Packet{
+			Header: rtp.Header{
+				Version:        2,
+				Marker:         p.lastPacketAt.IsZero() || now.Sub(p.lastPacketAt) >= backchannelTalkspurtGap,
+				PayloadType:    0,
+				SequenceNumber: p.sequenceNumber,
+				Timestamp:      p.timestamp,
+				SSRC:           p.ssrc,
+			},
+			Payload: bufferUlaw,
+		},
+	}
+
+	p.timestamp += uint32(len(bufferUlaw))
+	p.sequenceNumber++
+	p.lastPacketAt = now
+
+	return pkt
+}
+
 func GetBackChannelAudioCodec(streams []av.CodecData, communication *models.Communication) av.AudioCodecData {
 	for _, stream := range streams {
 		if stream.Type().IsAudio() {
 			if stream.Type().String() == "PCM_MULAW" {
 				pcmuCodec := stream.(av.AudioCodecData)
 				if pcmuCodec.IsBackChannel() {
-					communication.HasBackChannel = true
+					communication.HasBackChannel.Store(true)
 					return pcmuCodec
 				}
 			}
@@ -30,49 +87,123 @@ func GetBackChannelAudioCodec(streams []av.CodecData, communication *models.Comm
 	return nil
 }
 
-func WriteAudioToBackchannel(communication *models.Communication, rtspClient capture.RTSPClient) {
-	log.Log.Info("Audio.WriteAudioToBackchannel(): writing to backchannel audio codec")
-	length := uint32(0)
-	sequenceNumber := uint16(0)
-	for audio := range communication.HandleAudio {
-		// Encode PCM to MULAW
-		var bufferUlaw []byte
-		for _, v := range audio.Data {
-			b := g711.EncodeUlawFrame(v)
-			bufferUlaw = append(bufferUlaw, b)
-		}
+func WriteAudioToBackchannel(ctx context.Context, communication *models.Communication, rtspClient capture.RTSPClient) {
+	writeAudioToBackchannel(ctx, ctx, communication.HandleAudio, rtspClient)
+}
 
-		pkt := packets.Packet{
-			Packet: &rtp.Packet{
-				Header: rtp.Header{
-					Version:        2,
-					Marker:         true, // should be true
-					PayloadType:    0,    //packet.PayloadType, // will be owerwriten
-					SequenceNumber: sequenceNumber,
-					Timestamp:      uint32(length),
-					SSRC:           1293847657,
-				},
-				Payload: bufferUlaw,
-			},
-		}
-		err := rtspClient.WritePacket(pkt)
-		if err != nil {
-			log.Log.Error("Audio.WriteAudioToBackchannel(): error writing packet to backchannel")
-		}
+func writeAudioToBackchannel(ctx context.Context, otelContext context.Context, audioChannel <-chan models.AudioDataPartial, rtspClient backchannelClient) {
+	log.Info("Audio.WriteAudioToBackchannel(): writing to backchannel audio codec")
 
-		length = (length + uint32(len(bufferUlaw))) % 65536
-		sequenceNumber = (sequenceNumber + 1) % 65535
-		time.Sleep(128 * time.Millisecond)
+	if err := rtspClient.StartBackChannel(ctx, otelContext); err != nil {
+		log.Error("Audio.WriteAudioToBackchannel(): error starting backchannel: " + err.Error())
+		if !reconnectBackchannel(ctx, otelContext, rtspClient) {
+			log.Info("Audio.WriteAudioToBackchannel(): stopped while reconnecting")
+			return
+		}
 	}
-	log.Log.Info("Audio.WriteAudioToBackchannel(): finished")
 
+	packetizer := newBackchannelPacketizer()
+	for {
+		select {
+		case <-ctx.Done():
+			log.Info("Audio.WriteAudioToBackchannel(): stopped")
+			return
+		case audio, ok := <-audioChannel:
+			if !ok {
+				log.Info("Audio.WriteAudioToBackchannel(): finished")
+				return
+			}
+
+			audio = latestBackchannelAudio(audio, audioChannel)
+			if len(audio.Data) == 0 {
+				continue
+			}
+
+			pkt := packetizer.packet(audio, time.Now())
+			if err := rtspClient.WritePacket(pkt); err != nil {
+				log.Error("Audio.WriteAudioToBackchannel(): error writing packet to backchannel: " + err.Error())
+				if !reconnectBackchannel(ctx, otelContext, rtspClient) {
+					log.Info("Audio.WriteAudioToBackchannel(): stopped while reconnecting")
+					return
+				}
+				packetizer = newBackchannelPacketizer()
+				continue
+			}
+
+			if !waitForBackchannel(ctx, time.Duration(len(audio.Data))*time.Second/backchannelSampleRate) {
+				log.Info("Audio.WriteAudioToBackchannel(): stopped")
+				return
+			}
+		}
+	}
+}
+
+func latestBackchannelAudio(audio models.AudioDataPartial, audioChannel <-chan models.AudioDataPartial) models.AudioDataPartial {
+	for {
+		select {
+		case next, ok := <-audioChannel:
+			if !ok {
+				return audio
+			}
+			audio = next
+		default:
+			return audio
+		}
+	}
+}
+
+func reconnectBackchannel(ctx context.Context, otelContext context.Context, rtspClient backchannelClient) bool {
+	backoff := backchannelReconnectInitial
+	for {
+		if err := rtspClient.Close(otelContext); err != nil {
+			log.Error("Audio.WriteAudioToBackchannel(): error closing failed backchannel: " + err.Error())
+		}
+		if ctx.Err() != nil {
+			return false
+		}
+
+		err := rtspClient.ConnectBackChannel(ctx, otelContext)
+		if err == nil {
+			err = rtspClient.StartBackChannel(ctx, otelContext)
+		}
+		if err == nil {
+			log.Info("Audio.WriteAudioToBackchannel(): reconnected backchannel")
+			return true
+		}
+
+		log.Error("Audio.WriteAudioToBackchannel(): error reconnecting backchannel: " + err.Error())
+		if !waitForBackchannel(ctx, backoff) {
+			return false
+		}
+		backoff *= 2
+		if backoff > backchannelReconnectMax {
+			backoff = backchannelReconnectMax
+		}
+	}
+}
+
+func waitForBackchannel(ctx context.Context, duration time.Duration) bool {
+	timer := time.NewTimer(duration)
+	defer timer.Stop()
+
+	select {
+	case <-ctx.Done():
+		return false
+	case <-timer.C:
+		return true
+	}
 }
 
 func WriteFileToBackChannel(infile av.DemuxCloser) {
 	// Do the warmup!
 	file, err := os.Open("./audiofile.bye")
 	if err != nil {
-		fmt.Println("WriteFileToBackChannel: error opening audiofile.bye file")
+		log.WithError(err).WithFields(log.Fields{
+			"component": "backchannel",
+			"event":     "audio_file_open_failed",
+			"path":      "./audiofile.bye",
+		}).Error("Failed to open backchannel audio file")
+		return
 	}
 	defer file.Close()
 

machinery/src/components/backchannel_test.go

@@ -0,0 +1,209 @@
+package components
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/packets"
+)
+
+type fakeBackchannelClient struct {
+	mutex           sync.Mutex
+	startErrors     []error
+	connectError    error
+	writeErrors     []error
+	startCalls      int
+	connectCalls    int
+	closeCalls      int
+	writeCalls      int
+	connectAttempt  chan struct{}
+	successfulWrite chan packets.Packet
+}
+
+func (f *fakeBackchannelClient) ConnectBackChannel(context.Context, context.Context) error {
+	f.mutex.Lock()
+	f.connectCalls++
+	err := f.connectError
+	f.mutex.Unlock()
+
+	select {
+	case f.connectAttempt <- struct{}{}:
+	default:
+	}
+	return err
+}
+
+func (f *fakeBackchannelClient) StartBackChannel(context.Context, context.Context) error {
+	f.mutex.Lock()
+	defer f.mutex.Unlock()
+	f.startCalls++
+	if len(f.startErrors) == 0 {
+		return nil
+	}
+	err := f.startErrors[0]
+	f.startErrors = f.startErrors[1:]
+	return err
+}
+
+func (f *fakeBackchannelClient) WritePacket(pkt packets.Packet) error {
+	f.mutex.Lock()
+	f.writeCalls++
+	var err error
+	if len(f.writeErrors) != 0 {
+		err = f.writeErrors[0]
+		f.writeErrors = f.writeErrors[1:]
+	}
+	f.mutex.Unlock()
+
+	if err == nil {
+		select {
+		case f.successfulWrite <- pkt:
+		default:
+		}
+	}
+	return err
+}
+
+func (f *fakeBackchannelClient) Close(context.Context) error {
+	f.mutex.Lock()
+	f.closeCalls++
+	f.mutex.Unlock()
+	return nil
+}
+
+func (f *fakeBackchannelClient) callCounts() (start, connect, close, write int) {
+	f.mutex.Lock()
+	defer f.mutex.Unlock()
+	return f.startCalls, f.connectCalls, f.closeCalls, f.writeCalls
+}
+
+func TestBackchannelPacketizerUsesFullRTPClock(t *testing.T) {
+	packetizer := backchannelPacketizer{ssrc: 1}
+	audio := models.AudioDataPartial{Data: make([]int16, 1024)}
+	startedAt := time.Unix(1, 0)
+
+	var timestamp uint32
+	for index := 0; index <= 64; index++ {
+		pkt := packetizer.packet(audio, startedAt.Add(time.Duration(index)*128*time.Millisecond))
+		timestamp = pkt.Packet.Timestamp
+	}
+
+	if timestamp != 65536 {
+		t.Fatalf("timestamp after 64 frames = %d, want 65536", timestamp)
+	}
+}
+
+func TestBackchannelPacketizerUsesNaturalSequenceRollover(t *testing.T) {
+	packetizer := backchannelPacketizer{sequenceNumber: ^uint16(0), ssrc: 1}
+	audio := models.AudioDataPartial{Data: []int16{0}}
+	startedAt := time.Unix(1, 0)
+
+	last := packetizer.packet(audio, startedAt)
+	firstAfterRollover := packetizer.packet(audio, startedAt.Add(time.Millisecond))
+
+	if last.Packet.SequenceNumber != ^uint16(0) {
+		t.Fatalf("last sequence number = %d, want %d", last.Packet.SequenceNumber, ^uint16(0))
+	}
+	if firstAfterRollover.Packet.SequenceNumber != 0 {
+		t.Fatalf("first sequence number after rollover = %d, want 0", firstAfterRollover.Packet.SequenceNumber)
+	}
+}
+
+func TestBackchannelPacketizerMarksTalkspurtStart(t *testing.T) {
+	packetizer := backchannelPacketizer{ssrc: 1}
+	audio := models.AudioDataPartial{Data: []int16{0}}
+	startedAt := time.Unix(1, 0)
+
+	first := packetizer.packet(audio, startedAt)
+	continuous := packetizer.packet(audio, startedAt.Add(128*time.Millisecond))
+	afterGap := packetizer.packet(audio, startedAt.Add(backchannelTalkspurtGap+128*time.Millisecond))
+
+	if !first.Packet.Marker {
+		t.Fatal("first packet must mark the start of a talkspurt")
+	}
+	if continuous.Packet.Marker {
+		t.Fatal("continuous packet must not carry the marker bit")
+	}
+	if !afterGap.Packet.Marker {
+		t.Fatal("packet after an audio gap must mark a new talkspurt")
+	}
+}
+
+func TestWriteAudioToBackchannelReconnectsAfterWriteFailure(t *testing.T) {
+	writeFailure := errors.New("EOF")
+	client := &fakeBackchannelClient{
+		writeErrors:     []error{writeFailure, nil},
+		connectAttempt:  make(chan struct{}, 1),
+		successfulWrite: make(chan packets.Packet, 1),
+	}
+	audioChannel := make(chan models.AudioDataPartial, 2)
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() {
+		writeAudioToBackchannel(ctx, ctx, audioChannel, client)
+		close(done)
+	}()
+
+	audioChannel <- models.AudioDataPartial{Data: make([]int16, 1024)}
+	select {
+	case <-client.connectAttempt:
+	case <-time.After(time.Second):
+		t.Fatal("backchannel was not reconnected after the write failure")
+	}
+
+	audioChannel <- models.AudioDataPartial{Data: make([]int16, 1024)}
+	select {
+	case pkt := <-client.successfulWrite:
+		if !pkt.Packet.Marker {
+			t.Fatal("first packet after reconnect must mark a new talkspurt")
+		}
+	case <-time.After(time.Second):
+		t.Fatal("fresh audio was not written after reconnect")
+	}
+
+	cancel()
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		t.Fatal("backchannel writer did not stop after cancellation")
+	}
+
+	startCalls, connectCalls, closeCalls, writeCalls := client.callCounts()
+	if startCalls != 2 || connectCalls != 1 || closeCalls != 1 || writeCalls != 2 {
+		t.Fatalf("calls (start, connect, close, write) = (%d, %d, %d, %d), want (2, 1, 1, 2)", startCalls, connectCalls, closeCalls, writeCalls)
+	}
+}
+
+func TestWriteAudioToBackchannelCancellationStopsReconnect(t *testing.T) {
+	client := &fakeBackchannelClient{
+		connectError:    errors.New("camera unavailable"),
+		writeErrors:     []error{errors.New("EOF")},
+		connectAttempt:  make(chan struct{}, 1),
+		successfulWrite: make(chan packets.Packet, 1),
+	}
+	audioChannel := make(chan models.AudioDataPartial, 1)
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() {
+		writeAudioToBackchannel(ctx, ctx, audioChannel, client)
+		close(done)
+	}()
+
+	audioChannel <- models.AudioDataPartial{Data: make([]int16, 1024)}
+	select {
+	case <-client.connectAttempt:
+	case <-time.After(time.Second):
+		t.Fatal("expected a reconnect attempt")
+	}
+	cancel()
+
+	select {
+	case <-done:
+	case <-time.After(250 * time.Millisecond):
+		t.Fatal("cancellation did not interrupt reconnect backoff")
+	}
+}

machinery/src/components/watchdog_test.go

@@ -0,0 +1,82 @@
+package components
+
+import (
+	"context"
+	"testing"
+	"time"
+)
+
+func TestWaitForRunRetryStopsOnCancellation(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	started := time.Now()
+	if waitForRunRetry(ctx) {
+		t.Fatal("waitForRunRetry() completed the retry delay after cancellation")
+	}
+	if elapsed := time.Since(started); elapsed > 100*time.Millisecond {
+		t.Fatalf("waitForRunRetry() took %s after cancellation", elapsed)
+	}
+}
+
+func TestStreamRestartWatchdogCoalescesStallsAndBacksOff(t *testing.T) {
+	now := time.Unix(1_000, 0)
+	watchdog := newStreamRestartWatchdog()
+	if _, restart := watchdog.Observe(now, 1, 1, true, false); restart {
+		t.Fatal("initial observation requested a restart")
+	}
+
+	for check := 1; check < streamWatchdogStallChecks; check++ {
+		now = now.Add(streamWatchdogInterval)
+		if _, restart := watchdog.Observe(now, 1, 1, true, false); restart {
+			t.Fatalf("check %d requested an early restart", check)
+		}
+	}
+	now = now.Add(streamWatchdogInterval)
+	reason, restart := watchdog.Observe(now, 1, 1, true, false)
+	if !restart || reason != "main and sub streams" {
+		t.Fatalf("Observe() = (%q, %t), want coalesced restart", reason, restart)
+	}
+
+	watchdog.MarkRestart(now)
+	if got := watchdog.Backoff(); got != 30*time.Second {
+		t.Fatalf("Backoff() = %s, want 30s", got)
+	}
+	for now = now.Add(streamWatchdogInterval); now.Before(watchdog.nextRestart); now = now.Add(streamWatchdogInterval) {
+		if _, restart := watchdog.Observe(now, 1, 1, true, false); restart {
+			t.Fatal("restart requested during cooldown")
+		}
+	}
+}
+
+func TestStreamRestartWatchdogResetsAfterHealthyMinute(t *testing.T) {
+	now := time.Unix(2_000, 0)
+	watchdog := newStreamRestartWatchdog()
+	watchdog.backoff = streamWatchdogMaxBackoff
+	watchdog.nextRestart = now.Add(streamWatchdogMaxBackoff)
+	watchdog.Observe(now, 1, 1, true, false)
+
+	for elapsed := streamWatchdogInterval; elapsed <= streamWatchdogHealthyReset+streamWatchdogInterval; elapsed += streamWatchdogInterval {
+		now = now.Add(streamWatchdogInterval)
+		watchdog.Observe(now, int64(elapsed), int64(elapsed), true, false)
+	}
+
+	if got := watchdog.Backoff(); got != streamWatchdogBaseBackoff {
+		t.Fatalf("Backoff() = %s, want %s", got, streamWatchdogBaseBackoff)
+	}
+	if !watchdog.nextRestart.IsZero() {
+		t.Fatalf("nextRestart = %s, want zero", watchdog.nextRestart)
+	}
+}
+
+func TestStreamRestartWatchdogPausesWhileConfiguring(t *testing.T) {
+	now := time.Unix(3_000, 0)
+	watchdog := newStreamRestartWatchdog()
+	watchdog.Observe(now, 1, 0, false, false)
+	for check := 0; check < streamWatchdogStallChecks+1; check++ {
+		now = now.Add(streamWatchdogInterval)
+		if _, restart := watchdog.Observe(now, 1, 0, false, true); restart {
+			t.Fatal("restart requested while configuring")
+		}
+	}
+}

machinery/src/computervision/main.go

@@ -8,14 +8,14 @@ import (
 	geo "github.com/kellydunn/golang-geo"
 	"github.com/kerberos-io/agent/machinery/src/capture"
 	"github.com/kerberos-io/agent/machinery/src/conditions"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
+	log "github.com/sirupsen/logrus"
 )
 
 func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Configuration, communication *models.Communication, mqttClient mqtt.Client, rtspClient capture.RTSPClient) {
 
-	log.Log.Debug("computervision.main.ProcessMotion(): start motion detection")
+	log.Debug("computervision.main.ProcessMotion(): start motion detection")
 	config := configuration.Config
 	loc, _ := time.LoadLocation(config.Timezone)
 
@@ -42,14 +42,14 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 
 	if continuousMode && !hasMotionRegion {
 
-		log.Log.Info("computervision.main.ProcessMotion(): continuous recording enabled and no motion region configured, so no motion detection required.")
+		log.Info("computervision.main.ProcessMotion(): continuous recording enabled and no motion region configured, so no motion detection required.")
 
 	} else {
 
 		if continuousMode {
-			log.Log.Info("computervision.main.ProcessMotion(): continuous recording enabled with a motion region, running motion detection for live-view visualisation only (no motion-triggered recording).")
+			log.Info("computervision.main.ProcessMotion(): continuous recording enabled with a motion region, running motion detection for live-view visualisation only (no motion-triggered recording).")
 		} else {
-			log.Log.Info("computervision.main.ProcessMotion(): motion detected is enabled, so starting the motion detection.")
+			log.Info("computervision.main.ProcessMotion(): motion detected is enabled, so starting the motion detection.")
 		}
 
 		hubKey := config.HubKey
@@ -185,7 +185,7 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 				// We'll validate those conditions and if not valid we'll not do anything.
 				detectMotion, err := conditions.Validate(loc, configuration)
 				if !detectMotion && err != nil {
-					log.Log.Debug("computervision.main.ProcessMotion(): " + err.Error() + ".")
+					log.Debug("computervision.main.ProcessMotion(): " + err.Error() + ".")
 				}
 
 				// Run detection when motion is enabled, OR when we're in continuous
@@ -235,7 +235,7 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 										if err == nil {
 											mqttClient.Publish("kerberos/hub/"+hubKey, 2, false, payload)
 										} else {
-											log.Log.Info("computervision.main.ProcessMotion(): failed to package MQTT message: " + err.Error())
+											log.Info("computervision.main.ProcessMotion(): failed to package MQTT message: " + err.Error())
 										}
 									} else {
 										mqttClient.Publish("kerberos/agent/"+deviceKey, 2, false, "motion")
@@ -254,7 +254,9 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 									NumberOfChanges: changesToReturn,
 									Rectangle:       motionRectangle,
 								}
-								communication.HandleMotion <- dataToPass //Save data to the channel
+								if !communication.TrySendMotion(dataToPass) {
+									log.Warn("computervision.main.ProcessMotion(): motion channel unavailable or full, dropping recording trigger")
+								}
 							}
 						}
 					}
@@ -271,7 +273,7 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 		}
 	}
 
-	log.Log.Debug("computervision.main.ProcessMotion(): stop the motion detection.")
+	log.Debug("computervision.main.ProcessMotion(): stop the motion detection.")
 }
 
 func FindMotion(imageArray [3]*image.Gray, coordinatesPerRegion [][]int, pixelChangeThreshold int) (thresholdReached bool, changesDetected int, motionRectangle models.MotionRectangle, motionRectangles []models.MotionRectangle) {
@@ -374,14 +376,28 @@ func AbsDiffBitwiseAndThreshold(img1 *image.Gray, img2 *image.Gray, img3 *image.
 				endY = pixel[1]
 			}
 		}
-		log.Log.Debugf("Rectangle of changes detected: startX: %d, startY: %d, endX: %d, endY: %d", startX, startY, endX, endY)
+		log.WithFields(log.Fields{
+			"component": "computer_vision",
+			"end_x":     endX,
+			"end_y":     endY,
+			"event":     "motion_bounds_detected",
+			"start_x":   startX,
+			"start_y":   startY,
+		}).Debug("Motion bounds detected")
 		motionRectangle = models.MotionRectangle{
 			X:      startX,
 			Y:      startY,
 			Width:  endX - startX,
 			Height: endY - startY,
 		}
-		log.Log.Debugf("Motion rectangle: %+v", motionRectangle)
+		log.WithFields(log.Fields{
+			"component": "computer_vision",
+			"event":     "motion_rectangle_created",
+			"height":    motionRectangle.Height,
+			"width":     motionRectangle.Width,
+			"x":         motionRectangle.X,
+			"y":         motionRectangle.Y,
+		}).Debug("Motion rectangle created")
 	}
 
 	// Cluster the changed pixels into separate bounding boxes so the live view can

machinery/src/conditions/timewindow.go

@@ -3,8 +3,8 @@ package conditions
 import (
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 func IsWithinTimeInterval(loc *time.Location, configuration *models.Configuration) (enabled bool) {
@@ -27,9 +27,9 @@ func IsWithinTimeInterval(loc *time.Location, configuration *models.Configuratio
 				currentTimeInSeconds := hour*60*60 + minute*60 + second
 				if (currentTimeInSeconds >= start1 && currentTimeInSeconds <= end1) ||
 					(currentTimeInSeconds >= start2 && currentTimeInSeconds <= end2) {
-					log.Log.Debug("conditions.timewindow.IsWithinTimeInterval(): time interval valid, enabling recording.")
+					log.Debug("conditions.timewindow.IsWithinTimeInterval(): time interval valid, enabling recording.")
 				} else {
-					log.Log.Info("conditions.timewindow.IsWithinTimeInterval(): time interval not valid, disabling recording.")
+					log.Info("conditions.timewindow.IsWithinTimeInterval(): time interval not valid, disabling recording.")
 					enabled = false
 				}
 			}

machinery/src/conditions/uri.go

@@ -3,13 +3,26 @@ package conditions
 import (
 	"bytes"
 	"crypto/tls"
-	"fmt"
+	"encoding/json"
+	"io"
 	"net/http"
 	"os"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
+)
+
+const conditionHTTPTimeout = 10 * time.Second
+
+var (
+	conditionHTTPClient         = &http.Client{Timeout: conditionHTTPTimeout}
+	conditionInsecureHTTPClient = &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // #nosec G402 -- explicit operator opt-in
+		},
+		Timeout: conditionHTTPTimeout,
+	}
 )
 
 func IsValidUriResponse(configuration *models.Configuration) (enabled bool) {
@@ -17,41 +30,66 @@ func IsValidUriResponse(configuration *models.Configuration) (enabled bool) {
 	conditionURI := config.ConditionURI
 	enabled = true
 	if conditionURI != "" {
-
-		// We will send a POST request to the conditionURI, and expect a 200 response.
-		// In the payload we will send some information, so the other end can decide
-		// if it should enable or disable recording.
-
-		var client *http.Client
+		client := conditionHTTPClient
 		if os.Getenv("AGENT_TLS_INSECURE") == "true" {
-			tr := &http.Transport{
-				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-			}
-			client = &http.Client{Transport: tr}
-		} else {
-			client = &http.Client{}
+			client = conditionInsecureHTTPClient
 		}
 
-		var object = fmt.Sprintf(`{
-			"camera_id" : "%s",
-			"camera_name" : "%s",
-			"site_id" : "%s",
-			"hub_key" : "%s",
-			"timestamp" : "%s",
-		}`, config.Key, config.FriendlyName, config.HubSite, config.HubKey, time.Now().Format("2006-01-02 15:04:05"))
+		payload := struct {
+			CameraID   string `json:"camera_id"`
+			CameraName string `json:"camera_name"`
+			SiteID     string `json:"site_id"`
+			HubKey     string `json:"hub_key"`
+			Timestamp  string `json:"timestamp"`
+		}{
+			CameraID:   config.Key,
+			CameraName: config.FriendlyName,
+			SiteID:     config.HubSite,
+			HubKey:     config.HubKey,
+			Timestamp:  time.Now().Format("2006-01-02 15:04:05"),
+		}
+		jsonBody, err := json.Marshal(payload)
+		if err != nil {
+			log.WithError(err).WithFields(log.Fields{
+				"component": "conditions/uri",
+				"event":     "request_encoding_failed",
+			}).Error("Failed to encode condition request")
+			return false
+		}
 
-		var jsonStr = []byte(object)
-		buffy := bytes.NewBuffer(jsonStr)
-		req, _ := http.NewRequest("POST", conditionURI, buffy)
+		req, err := http.NewRequest(http.MethodPost, conditionURI, bytes.NewReader(jsonBody))
+		if err != nil {
+			log.WithError(err).WithFields(log.Fields{
+				"component": "conditions/uri",
+				"event":     "request_creation_failed",
+			}).Error("Failed to create condition request")
+			return false
+		}
 		req.Header.Set("Content-Type", "application/json")
 		resp, err := client.Do(req)
 		if resp != nil {
+			_, _ = io.Copy(io.Discard, resp.Body)
 			resp.Body.Close()
 		}
-		if err == nil && resp.StatusCode == 200 {
-			log.Log.Info("conditions.uri.IsValidUriResponse(): response 200, enabling recording.")
+		if err == nil && resp != nil && resp.StatusCode == http.StatusOK {
+			log.WithFields(log.Fields{
+				"component":   "conditions/uri",
+				"event":       "recording_enabled",
+				"status_code": resp.StatusCode,
+			}).Info("Condition request enabled recording")
 		} else {
-			log.Log.Info("conditions.uri.IsValidUriResponse(): response not 200, disabling recording.")
+			if err != nil {
+				log.WithError(err).WithFields(log.Fields{
+					"component": "conditions/uri",
+					"event":     "request_failed",
+				}).Error("Condition request failed")
+			} else {
+				log.WithFields(log.Fields{
+					"component":   "conditions/uri",
+					"event":       "recording_disabled",
+					"status_code": resp.StatusCode,
+				}).Info("Condition request disabled recording")
+			}
 			enabled = false
 		}
 	}

machinery/src/conditions/uri_test.go

@@ -0,0 +1,63 @@
+package conditions
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+
+	"github.com/kerberos-io/agent/machinery/src/models"
+)
+
+func TestIsValidUriResponseReusesClientAndEncodesPayload(t *testing.T) {
+	var requests atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		requests.Add(1)
+		if request.Method != http.MethodPost {
+			t.Errorf("method = %s, want POST", request.Method)
+		}
+		if got := request.Header.Get("Content-Type"); got != "application/json" {
+			t.Errorf("Content-Type = %q, want application/json", got)
+		}
+		var payload struct {
+			CameraID   string `json:"camera_id"`
+			CameraName string `json:"camera_name"`
+		}
+		if err := json.NewDecoder(request.Body).Decode(&payload); err != nil {
+			t.Errorf("decode request: %v", err)
+		}
+		if payload.CameraID != "camera-1" || payload.CameraName != `Front "Door"` {
+			t.Errorf("payload = %+v", payload)
+		}
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer server.Close()
+
+	configuration := &models.Configuration{}
+	configuration.Config.ConditionURI = server.URL
+	configuration.Config.Key = "camera-1"
+	configuration.Config.FriendlyName = `Front "Door"`
+
+	for i := 0; i < 2; i++ {
+		if !IsValidUriResponse(configuration) {
+			t.Fatal("IsValidUriResponse() = false, want true")
+		}
+	}
+	if got := requests.Load(); got != 2 {
+		t.Fatalf("requests = %d, want 2", got)
+	}
+}
+
+func TestIsValidUriResponseRejectsNonOK(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		http.Error(w, "unavailable", http.StatusServiceUnavailable)
+	}))
+	defer server.Close()
+
+	configuration := &models.Configuration{}
+	configuration.Config.ConditionURI = server.URL
+	if IsValidUriResponse(configuration) {
+		t.Fatal("IsValidUriResponse() = true, want false")
+	}
+}

machinery/src/config/main.go

@@ -13,11 +13,36 @@ import (
 
 	"github.com/InVisionApp/conjungo"
 	"github.com/kerberos-io/agent/machinery/src/database"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
 )
 
+const factoryConfigRetryDelay = 5 * time.Second
+
+func newFactoryConfigReadContext() (context.Context, context.CancelFunc) {
+	return context.WithTimeout(context.Background(), database.TIMEOUT)
+}
+
+func isRetryableFactoryConfigReadError(err error) bool {
+	return errors.Is(err, context.DeadlineExceeded) || mongo.IsTimeout(err) || mongo.IsNetworkError(err)
+}
+
+func readFactoryConfig(collection *mongo.Collection, filter bson.M, destination *models.Config) error {
+	for {
+		ctx, cancel := newFactoryConfigReadContext()
+		err := collection.FindOne(ctx, filter).Decode(destination)
+		cancel()
+		if !isRetryableFactoryConfigReadError(err) {
+			return err
+		}
+
+		log.Warn("Factory configuration read timed out or lost its database connection; retrying.")
+		time.Sleep(factoryConfigRetryDelay)
+	}
+}
+
 // ReadUserConfig Reads the user configuration of the Kerberos Open Source instance.
 // This will return a models.User struct including the username, password,
 // selected language, and if the installation was completed or not.
@@ -25,14 +50,14 @@ func ReadUserConfig(configDirectory string) (userConfig models.User) {
 	for {
 		jsonFile, err := os.Open(configDirectory + "/data/config/user.json")
 		if err != nil {
-			log.Log.Error("Config file is not found " + configDirectory + "/data/config/user.json, trying again in 5s: " + err.Error())
+			log.Error("Config file is not found " + configDirectory + "/data/config/user.json, trying again in 5s: " + err.Error())
 			time.Sleep(5 * time.Second)
 		} else {
-			log.Log.Info("Successfully Opened user.json")
+			log.Info("Successfully Opened user.json")
 			byteValue, _ := ioutil.ReadAll(jsonFile)
 			err = json.Unmarshal(byteValue, &userConfig)
 			if err != nil {
-				log.Log.Error("JSON file not valid: " + err.Error())
+				log.Error("JSON file not valid: " + err.Error())
 			} else {
 				jsonFile.Close()
 				break
@@ -63,21 +88,15 @@ func OpenConfig(configDirectory string, configuration *models.Configuration) {
 		collection := db.Collection("configuration")
 
 		var globalConfig models.Config
-		res := collection.FindOne(context.Background(), bson.M{
+		err := readFactoryConfig(collection, bson.M{
 			"type": "global",
-		})
-
-		if res.Err() != nil {
-			log.Log.Error("Could not find global configuration, using default configuration.")
-			panic("Could not find global configuration, using default configuration.")
-		}
-		err := res.Decode(&globalConfig)
+		}, &globalConfig)
 		if err != nil {
-			log.Log.Error("Could not find global configuration, using default configuration.")
+			log.Error("Could not find global configuration, using default configuration.")
 			panic("Could not find global configuration, using default configuration.")
 		}
 		if globalConfig.Type != "global" {
-			log.Log.Error("Could not find global configuration, might missed the mongodb connection.")
+			log.Error("Could not find global configuration, might missed the mongodb connection.")
 			panic("Could not find global configuration, might missed the mongodb connection.")
 		}
 
@@ -85,21 +104,16 @@ func OpenConfig(configDirectory string, configuration *models.Configuration) {
 
 		var customConfig models.Config
 		deploymentName := os.Getenv("DEPLOYMENT_NAME")
-		res = collection.FindOne(context.Background(), bson.M{
+		err = readFactoryConfig(collection, bson.M{
 			"type": "config",
 			"name": deploymentName,
-		})
-		if res.Err() != nil {
-			log.Log.Error("Could not find configuration for " + deploymentName + ", using global configuration.")
-		}
-		err = res.Decode(&customConfig)
+		}, &customConfig)
 		if err != nil {
-			log.Log.Error("Could not find configuration for " + deploymentName + ", using global configuration.")
-		}
-
-		if customConfig.Type != "config" {
-			log.Log.Error("Could not find custom configuration, might missed the mongodb connection.")
-			panic("Could not find custom configuration, might missed the mongodb connection.")
+			log.Error("Could not find configuration for " + deploymentName + ", using global configuration.")
+			customConfig = models.Config{}
+		} else if customConfig.Type != "config" {
+			log.Error("Custom configuration has an invalid type, using global configuration.")
+			customConfig = models.Config{}
 		}
 		configuration.CustomConfig = customConfig
 
@@ -166,21 +180,29 @@ func OpenConfig(configDirectory string, configuration *models.Configuration) {
 
 		// Open device config
 		for {
-			jsonFile, err := os.Open(configDirectory + "/data/config/config.json")
+			configPath := configDirectory + "/data/config/config.json"
+			jsonFile, err := os.Open(configPath)
 			if err != nil {
-				log.Log.Error("Config file is not found " + configDirectory + "/data/config/config.json" + ", trying again in 5s.")
+				log.WithError(err).WithFields(log.Fields{
+					"component":      "config",
+					"event":          "configuration_open_failed",
+					"retry_delay_ms": 5000,
+				}).Warn("Configuration file unavailable; retrying")
 				time.Sleep(5 * time.Second)
 			} else {
-				log.Log.Info("Successfully Opened config.json from " + configuration.Name)
+				log.WithFields(log.Fields{
+					"component": "config",
+					"event":     "configuration_opened",
+				}).Info("Configuration file opened")
 				byteValue, _ := ioutil.ReadAll(jsonFile)
 				err = json.Unmarshal(byteValue, &configuration.Config)
 				jsonFile.Close()
 				if err != nil {
-					log.Log.Error("JSON file not valid: " + err.Error())
+					log.Error("JSON file not valid: " + err.Error())
 				} else {
 					err = json.Unmarshal(byteValue, &configuration.CustomConfig)
 					if err != nil {
-						log.Log.Error("JSON file not valid: " + err.Error())
+						log.Error("JSON file not valid: " + err.Error())
 					} else {
 						break
 					}
@@ -687,12 +709,12 @@ func SaveConfig(configDirectory string, config models.Config, configuration *mod
 			return err
 		}
 
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			select {
 			case communication.HandleBootstrap <- "restart":
-				log.Log.Info("config.main.SaveConfig(): update config, restart agent.")
+				log.Info("config.main.SaveConfig(): update config, restart agent.")
 			case <-time.After(1 * time.Second):
-				log.Log.Info("config.main.SaveConfig(): update config, restart agent.")
+				log.Info("config.main.SaveConfig(): update config, restart agent.")
 			}
 		}
 

machinery/src/config/main_test.go

@@ -1,9 +1,14 @@
 package config
 
 import (
+	"context"
+	"errors"
 	"testing"
+	"time"
 
+	"github.com/kerberos-io/agent/machinery/src/database"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	"go.mongodb.org/mongo-driver/mongo"
 )
 
 func TestApplyAgentEnvVarsPixelChangeThresholdDefault(t *testing.T) {
@@ -38,3 +43,36 @@ func TestApplyAgentEnvVarsPixelChangeThresholdDefault(t *testing.T) {
 func intPointer(value int) *int {
 	return &value
 }
+
+func TestNewFactoryConfigReadContextUsesDatabaseTimeout(t *testing.T) {
+	ctx, cancel := newFactoryConfigReadContext()
+	defer cancel()
+
+	deadline, ok := ctx.Deadline()
+	if !ok {
+		t.Fatal("expected read context deadline")
+	}
+
+	remaining := time.Until(deadline)
+	if remaining <= 0 {
+		t.Fatalf("deadline already expired: %v", remaining)
+	}
+	if remaining > database.TIMEOUT {
+		t.Fatalf("remaining deadline = %v, want <= %v", remaining, database.TIMEOUT)
+	}
+	if remaining < database.TIMEOUT-time.Second {
+		t.Fatalf("remaining deadline = %v, want close to %v", remaining, database.TIMEOUT)
+	}
+}
+
+func TestFactoryConfigRetryableErrors(t *testing.T) {
+	if !isRetryableFactoryConfigReadError(context.DeadlineExceeded) {
+		t.Fatal("context deadline should be retryable")
+	}
+	if isRetryableFactoryConfigReadError(mongo.ErrNoDocuments) {
+		t.Fatal("missing configuration should not retry")
+	}
+	if isRetryableFactoryConfigReadError(errors.New("invalid BSON")) {
+		t.Fatal("decode errors should not retry")
+	}
+}

machinery/src/database/main.go

@@ -7,6 +7,7 @@ import (
 	"sync"
 	"time"
 
+	log "github.com/sirupsen/logrus"
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
 )
@@ -44,18 +45,34 @@ func New() *DB {
 		// We can also apply the complete URI
 		// e.g. "mongodb+srv://<username>:<password>@kerberos-hub.shhng.mongodb.net/?retryWrites=true&w=majority&appName=kerberos-hub"
 		if mongodbURI != "" {
+			log.WithFields(log.Fields{
+				"component":      "database",
+				"database":       DatabaseName,
+				"event":          "client_configuring",
+				"uri_configured": true,
+			}).Debug("Configuring MongoDB client")
 			serverAPI := options.ServerAPI(options.ServerAPIVersion1)
 			opts := options.Client().ApplyURI(mongodbURI).SetServerAPIOptions(serverAPI)
 
 			// Create a new client and connect to the server
 			client, err := mongo.Connect(ctx, opts)
 			if err != nil {
-				fmt.Printf("Error setting up mongodb connection: %+v\n", err)
-				os.Exit(1)
+				log.WithError(err).WithFields(log.Fields{
+					"component": "database",
+					"database":  DatabaseName,
+					"event":     "client_configuration_failed",
+				}).Fatal("Failed to configure MongoDB client")
 			}
 			_instance.Client = client
 
 		} else {
+			log.WithFields(log.Fields{
+				"component":             "database",
+				"database":              DatabaseName,
+				"event":                 "client_configuring",
+				"replicaset_configured": replicaset != "",
+				"uri_configured":        false,
+			}).Debug("Configuring MongoDB client")
 
 			// New MongoDB driver
 			mongodbURI := fmt.Sprintf("mongodb://%s:%s@%s", username, password, host)
@@ -69,8 +86,11 @@ func New() *DB {
 				Password:      password,
 			}))
 			if err != nil {
-				fmt.Printf("Error setting up mongodb connection: %+v\n", err)
-				os.Exit(1)
+				log.WithError(err).WithFields(log.Fields{
+					"component": "database",
+					"database":  DatabaseName,
+					"event":     "client_configuration_failed",
+				}).Fatal("Failed to configure MongoDB client")
 			}
 			_instance.Client = client
 		}

machinery/src/lifecycle/supervisor.go

@@ -0,0 +1,336 @@
+package lifecycle
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"runtime/debug"
+	"sort"
+	"sync"
+	"time"
+)
+
+var (
+	ErrSupervisorSealed  = errors.New("supervisor is sealed")
+	ErrSupervisorStopped = errors.New("supervisor is shutting down")
+	ErrTaskExists        = errors.New("task already exists")
+)
+
+type TaskFunc func(context.Context) error
+
+type TaskPolicy struct {
+	Required    bool
+	LongRunning bool
+}
+
+type TaskStatus string
+
+const (
+	TaskRunning   TaskStatus = "running"
+	TaskSucceeded TaskStatus = "succeeded"
+	TaskFailed    TaskStatus = "failed"
+	TaskPanicked  TaskStatus = "panicked"
+)
+
+type TaskSnapshot struct {
+	Name      string
+	Policy    TaskPolicy
+	Status    TaskStatus
+	StartedAt time.Time
+	EndedAt   time.Time
+	Error     string
+	Panic     string
+	Stack     []byte
+}
+
+type Failure struct {
+	Task  string
+	Cause error
+	Panic string
+	Stack []byte
+}
+
+func (f Failure) Error() string {
+	if f.Cause == nil {
+		return fmt.Sprintf("task %q failed", f.Task)
+	}
+	return f.Cause.Error()
+}
+
+type ShutdownReport struct {
+	Complete bool
+	Cause    error
+	Tasks    []TaskSnapshot
+	Running  []TaskSnapshot
+}
+
+type supervisorPhase uint8
+
+const (
+	phaseStarting supervisorPhase = iota
+	phaseRunning
+	phaseDraining
+)
+
+type taskState struct {
+	TaskSnapshot
+}
+
+type Supervisor struct {
+	ctx    context.Context
+	cancel context.CancelCauseFunc
+
+	mu        sync.Mutex
+	phase     supervisorPhase
+	sealed    bool
+	active    int
+	tasks     map[string]*taskState
+	allDone   chan struct{}
+	doneOnce  sync.Once
+	start     chan struct{}
+	startOnce sync.Once
+
+	failures    chan Failure
+	failureOnce sync.Once
+
+	stopParentCancel func() bool
+}
+
+func NewSupervisor(parent context.Context) *Supervisor {
+	if parent == nil {
+		parent = context.Background()
+	}
+
+	ctx, cancel := context.WithCancelCause(parent)
+	supervisor := &Supervisor{
+		ctx:      ctx,
+		cancel:   cancel,
+		phase:    phaseStarting,
+		tasks:    make(map[string]*taskState),
+		allDone:  make(chan struct{}),
+		start:    make(chan struct{}),
+		failures: make(chan Failure, 1),
+	}
+
+	stopParentCancel := context.AfterFunc(parent, func() {
+		supervisor.BeginShutdown(context.Cause(parent))
+	})
+	supervisor.mu.Lock()
+	if supervisor.sealed && supervisor.active == 0 {
+		supervisor.mu.Unlock()
+		stopParentCancel()
+	} else {
+		supervisor.stopParentCancel = stopParentCancel
+		supervisor.mu.Unlock()
+	}
+
+	return supervisor
+}
+
+func (s *Supervisor) Context() context.Context {
+	return s.ctx
+}
+
+func (s *Supervisor) Failures() <-chan Failure {
+	return s.failures
+}
+
+func (s *Supervisor) Go(name string, policy TaskPolicy, task TaskFunc) error {
+	if name == "" {
+		return errors.New("task name is required")
+	}
+	if task == nil {
+		return errors.New("task function is required")
+	}
+
+	s.mu.Lock()
+	if s.sealed {
+		s.mu.Unlock()
+		return fmt.Errorf("%w: %s", ErrSupervisorSealed, name)
+	}
+	if s.ctx.Err() != nil {
+		s.mu.Unlock()
+		return fmt.Errorf("%w: %s", ErrSupervisorStopped, name)
+	}
+	if _, exists := s.tasks[name]; exists {
+		s.mu.Unlock()
+		return fmt.Errorf("%w: %s", ErrTaskExists, name)
+	}
+
+	state := &taskState{TaskSnapshot: TaskSnapshot{
+		Name:      name,
+		Policy:    policy,
+		Status:    TaskRunning,
+		StartedAt: time.Now(),
+	}}
+	s.tasks[name] = state
+	s.active++
+	s.mu.Unlock()
+
+	go s.run(state, task)
+	return nil
+}
+
+func (s *Supervisor) run(state *taskState, task TaskFunc) {
+	var taskErr error
+	panicked := true
+	var panicValue string
+	var panicStack []byte
+
+	defer func() {
+		recovered := recover()
+		if panicked {
+			panicValue = fmt.Sprint(recovered)
+			panicStack = debug.Stack()
+			taskErr = fmt.Errorf("task %q panicked: %s", state.Name, panicValue)
+		}
+		s.finish(state, taskErr, panicked, panicValue, panicStack)
+	}()
+
+	select {
+	case <-s.start:
+	case <-s.ctx.Done():
+		panicked = false
+		return
+	}
+
+	taskErr = task(s.ctx)
+	panicked = false
+}
+
+func (s *Supervisor) finish(
+	state *taskState,
+	taskErr error,
+	panicked bool,
+	panicValue string,
+	panicStack []byte,
+) {
+	var failure *Failure
+
+	s.mu.Lock()
+	state.EndedAt = time.Now()
+	state.Error = ""
+	state.Panic = panicValue
+	state.Stack = append([]byte(nil), panicStack...)
+	switch {
+	case panicked:
+		state.Status = TaskPanicked
+		state.Error = taskErr.Error()
+	case taskErr != nil:
+		state.Status = TaskFailed
+		state.Error = taskErr.Error()
+	default:
+		state.Status = TaskSucceeded
+	}
+
+	shuttingDown := s.phase == phaseDraining || s.ctx.Err() != nil
+	unexpected := panicked || (!shuttingDown &&
+		(state.Policy.Required && (taskErr != nil || state.Policy.LongRunning)))
+	if unexpected {
+		cause := taskErr
+		if cause == nil {
+			cause = fmt.Errorf("required long-running task %q exited", state.Name)
+		}
+		failure = &Failure{
+			Task:  state.Name,
+			Cause: cause,
+			Panic: panicValue,
+			Stack: append([]byte(nil), panicStack...),
+		}
+		s.phase = phaseDraining
+		s.sealed = true
+		s.cancel(failure.Cause)
+		s.failureOnce.Do(func() {
+			s.failures <- *failure
+		})
+	}
+
+	s.active--
+	s.closeDoneIfReadyLocked()
+	s.mu.Unlock()
+}
+
+func (s *Supervisor) Seal() {
+	s.mu.Lock()
+	s.sealed = true
+	if s.phase == phaseStarting {
+		s.phase = phaseRunning
+	}
+	s.startOnce.Do(func() {
+		close(s.start)
+	})
+	s.closeDoneIfReadyLocked()
+	s.mu.Unlock()
+}
+
+func (s *Supervisor) BeginShutdown(cause error) {
+	if cause == nil {
+		cause = context.Canceled
+	}
+
+	s.mu.Lock()
+	s.phase = phaseDraining
+	s.sealed = true
+	s.cancel(cause)
+	s.closeDoneIfReadyLocked()
+	s.mu.Unlock()
+}
+
+func (s *Supervisor) Wait(ctx context.Context) ShutdownReport {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+
+	select {
+	case <-s.allDone:
+	case <-ctx.Done():
+	}
+
+	return s.report()
+}
+
+func (s *Supervisor) Snapshot() []TaskSnapshot {
+	return s.report().Tasks
+}
+
+func (s *Supervisor) closeDoneIfReadyLocked() {
+	if !s.sealed || s.active != 0 {
+		return
+	}
+	if s.stopParentCancel != nil {
+		s.stopParentCancel()
+		s.stopParentCancel = nil
+	}
+	s.doneOnce.Do(func() {
+		close(s.allDone)
+	})
+}
+
+func (s *Supervisor) report() ShutdownReport {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	tasks := make([]TaskSnapshot, 0, len(s.tasks))
+	running := make([]TaskSnapshot, 0)
+	for _, state := range s.tasks {
+		snapshot := state.TaskSnapshot
+		snapshot.Stack = append([]byte(nil), state.Stack...)
+		tasks = append(tasks, snapshot)
+		if snapshot.Status == TaskRunning {
+			running = append(running, snapshot)
+		}
+	}
+	sort.Slice(tasks, func(i, j int) bool {
+		return tasks[i].Name < tasks[j].Name
+	})
+	sort.Slice(running, func(i, j int) bool {
+		return running[i].Name < running[j].Name
+	})
+
+	return ShutdownReport{
+		Complete: s.sealed && s.active == 0,
+		Cause:    context.Cause(s.ctx),
+		Tasks:    tasks,
+		Running:  running,
+	}
+}

machinery/src/lifecycle/supervisor_test.go

@@ -0,0 +1,300 @@
+package lifecycle
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+func TestSupervisorWaitsForSeal(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	started := make(chan struct{})
+	if err := supervisor.Go("quick", TaskPolicy{}, func(context.Context) error {
+		close(started)
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	waitContext, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+	defer cancel()
+	if report := supervisor.Wait(waitContext); report.Complete {
+		t.Fatal("Wait() completed before Seal()")
+	}
+	select {
+	case <-started:
+		t.Fatal("task started before Seal()")
+	default:
+	}
+
+	supervisor.Seal()
+	<-started
+	if report := supervisor.Wait(context.Background()); !report.Complete {
+		t.Fatal("Wait() did not complete after Seal()")
+	}
+}
+
+func TestSupervisorSealsEmptyGroup(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	supervisor.Seal()
+
+	report := supervisor.Wait(context.Background())
+	if !report.Complete || len(report.Tasks) != 0 {
+		t.Fatalf("Wait() report = %+v, want complete empty report", report)
+	}
+}
+
+func TestSupervisorReportsRunningTasksOnTimeout(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	release := make(chan struct{})
+	if err := supervisor.Go("blocked", TaskPolicy{}, func(context.Context) error {
+		<-release
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+
+	waitContext, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+	defer cancel()
+	report := supervisor.Wait(waitContext)
+	if report.Complete {
+		t.Fatal("Wait() unexpectedly completed")
+	}
+	if len(report.Running) != 1 || report.Running[0].Name != "blocked" {
+		t.Fatalf("Wait() running tasks = %+v, want blocked", report.Running)
+	}
+
+	close(release)
+	if report = supervisor.Wait(context.Background()); !report.Complete {
+		t.Fatal("Wait() did not complete after releasing task")
+	}
+}
+
+func TestSupervisorFailsWhenRequiredLongRunningTaskExits(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	release := make(chan struct{})
+	if err := supervisor.Go("recorder", TaskPolicy{Required: true, LongRunning: true}, func(context.Context) error {
+		<-release
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+	close(release)
+
+	select {
+	case failure := <-supervisor.Failures():
+		if failure.Task != "recorder" || !strings.Contains(failure.Error(), "exited") {
+			t.Fatalf("failure = %+v", failure)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("required task exit did not emit a failure")
+	}
+}
+
+func TestSupervisorAllowsRequiredOneShotTaskToSucceed(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	if err := supervisor.Go("setup", TaskPolicy{Required: true}, func(context.Context) error {
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+	if report := supervisor.Wait(context.Background()); !report.Complete {
+		t.Fatal("Wait() did not complete")
+	}
+
+	select {
+	case failure := <-supervisor.Failures():
+		t.Fatalf("successful one-shot task emitted failure: %+v", failure)
+	default:
+	}
+}
+
+func TestSupervisorFailsWhenRequiredOneShotTaskReturnsError(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	wantErr := errors.New("setup failed")
+	if err := supervisor.Go("setup", TaskPolicy{Required: true}, func(context.Context) error {
+		return wantErr
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+
+	select {
+	case failure := <-supervisor.Failures():
+		if !errors.Is(failure.Cause, wantErr) {
+			t.Fatalf("failure cause = %v, want %v", failure.Cause, wantErr)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("required task error did not emit a failure")
+	}
+}
+
+func TestSupervisorRecoversPanicAndCapturesStack(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	if err := supervisor.Go("panic", TaskPolicy{}, func(context.Context) error {
+		panic("boom")
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+
+	select {
+	case failure := <-supervisor.Failures():
+		if failure.Task != "panic" || failure.Panic != "boom" || len(failure.Stack) == 0 {
+			t.Fatalf("failure = %+v, want captured panic and stack", failure)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("panic did not emit a failure")
+	}
+
+	report := supervisor.Wait(context.Background())
+	if !report.Complete || len(report.Tasks) != 1 || len(report.Tasks[0].Stack) == 0 {
+		t.Fatalf("report = %+v, want completed task with captured stack", report)
+	}
+}
+
+func TestSupervisorCapturesEmptyPanicDuringShutdown(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	started := make(chan struct{})
+	if err := supervisor.Go("panic", TaskPolicy{}, func(ctx context.Context) error {
+		close(started)
+		<-ctx.Done()
+		panic("")
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+	<-started
+	supervisor.BeginShutdown(errors.New("requested shutdown"))
+
+	select {
+	case failure := <-supervisor.Failures():
+		if failure.Task != "panic" || len(failure.Stack) == 0 {
+			t.Fatalf("failure = %+v, want captured panic and stack", failure)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("panic during shutdown did not emit a failure")
+	}
+	if report := supervisor.Wait(context.Background()); !report.Complete || report.Tasks[0].Status != TaskPanicked {
+		t.Fatalf("report = %+v, want completed panicked task", report)
+	}
+}
+
+func TestSupervisorPublishesFailureBeforeCompletion(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	wantErr := errors.New("setup failed")
+	if err := supervisor.Go("setup", TaskPolicy{Required: true}, func(context.Context) error {
+		return wantErr
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+
+	report := supervisor.Wait(context.Background())
+	if !report.Complete || !errors.Is(report.Cause, wantErr) {
+		t.Fatalf("report = %+v, want complete report caused by task failure", report)
+	}
+	select {
+	case failure := <-supervisor.Failures():
+		if !errors.Is(failure.Cause, wantErr) {
+			t.Fatalf("failure cause = %v, want %v", failure.Cause, wantErr)
+		}
+	default:
+		t.Fatal("failure was not published before completion")
+	}
+}
+
+func TestSupervisorCompletedWaitWinsOverCanceledContext(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	supervisor.Seal()
+
+	waitContext, cancel := context.WithCancel(context.Background())
+	cancel()
+	if report := supervisor.Wait(waitContext); !report.Complete {
+		t.Fatalf("report = %+v, want authoritative completion state", report)
+	}
+}
+
+func TestSupervisorRejectsTaskAfterParentCancellation(t *testing.T) {
+	parent, cancel := context.WithCancel(context.Background())
+	supervisor := NewSupervisor(parent)
+	cancel()
+
+	err := supervisor.Go("late", TaskPolicy{}, func(context.Context) error { return nil })
+	if !errors.Is(err, ErrSupervisorStopped) && !errors.Is(err, ErrSupervisorSealed) {
+		t.Fatalf("Go() error = %v, want stopped or sealed supervisor", err)
+	}
+	supervisor.BeginShutdown(context.Canceled)
+	if report := supervisor.Wait(context.Background()); !report.Complete {
+		t.Fatalf("report = %+v, want complete canceled supervisor", report)
+	}
+}
+
+func TestSupervisorShutdownBeforeSealDoesNotStartTasks(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	started := make(chan struct{})
+	if err := supervisor.Go("worker", TaskPolicy{}, func(context.Context) error {
+		close(started)
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	supervisor.BeginShutdown(errors.New("startup aborted"))
+	if report := supervisor.Wait(context.Background()); !report.Complete {
+		t.Fatalf("report = %+v, want completed aborted startup", report)
+	}
+	select {
+	case <-started:
+		t.Fatal("task started after startup was aborted")
+	default:
+	}
+}
+
+func TestSupervisorRejectsDuplicateAndLateTasks(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	release := make(chan struct{})
+	if err := supervisor.Go("worker", TaskPolicy{}, func(context.Context) error {
+		<-release
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := supervisor.Go("worker", TaskPolicy{}, func(context.Context) error { return nil }); !errors.Is(err, ErrTaskExists) {
+		t.Fatalf("duplicate Go() error = %v, want ErrTaskExists", err)
+	}
+
+	supervisor.Seal()
+	if err := supervisor.Go("late", TaskPolicy{}, func(context.Context) error { return nil }); !errors.Is(err, ErrSupervisorSealed) {
+		t.Fatalf("late Go() error = %v, want ErrSupervisorSealed", err)
+	}
+	close(release)
+}
+
+func TestSupervisorConcurrentStartAndSeal(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	var starters sync.WaitGroup
+	for index := 0; index < 100; index++ {
+		starters.Add(1)
+		go func(index int) {
+			defer starters.Done()
+			_ = supervisor.Go(
+				"worker-"+time.Unix(0, int64(index)).Format("150405.000000000"),
+				TaskPolicy{},
+				func(context.Context) error { return nil },
+			)
+		}(index)
+	}
+
+	supervisor.Seal()
+	starters.Wait()
+	if report := supervisor.Wait(context.Background()); !report.Complete {
+		t.Fatal("Wait() did not complete after concurrent Start and Seal")
+	}
+}

machinery/src/log/main.go

@@ -1,179 +0,0 @@
-package log
-
-import (
-	"os"
-	"time"
-
-	"github.com/op/go-logging"
-	"github.com/sirupsen/logrus"
-	"gopkg.in/natefinch/lumberjack.v2"
-)
-
-// The logging library being used everywhere.
-var Log = Logging{
-	Logger: "logrus",
-}
-
-// -----------------
-// This a gologging
-// -> github.com/op/go-logging
-
-var gologging = logging.MustGetLogger("gologger")
-
-func ConfigureGoLogging(configDirectory string, timezone *time.Location) {
-	// Logging
-	var format = logging.MustStringFormatter(
-		`%{color}%{time:15:04:05.000} %{shortfunc} ▶ %{level:.4s} %{id:03x}%{color:reset} %{message}`,
-	)
-	var format2 = logging.MustStringFormatter(
-		`%{time:15:04:05.000} %{shortfunc} ▶ %{level:.4s}  %{id:03x}%{color:reset} %{message}`,
-	)
-	stdBackend := logging.NewLogBackend(os.Stderr, "", 0)
-	stdBackendLeveled := logging.NewBackendFormatter(stdBackend, format)
-	fileBackend := logging.NewLogBackend(&lumberjack.Logger{
-		Filename: configDirectory + "/data/log/machinery.txt",
-		MaxSize:  2,    // megabytes
-		Compress: true, // disabled by default
-	}, "", 0)
-	fileBackendLeveled := logging.NewBackendFormatter(fileBackend, format2)
-	logging.SetBackend(stdBackendLeveled, fileBackendLeveled)
-	logging.SetLevel(logging.DEBUG, "")
-}
-
-// -----------------
-// This a logrus
-// -> github.com/sirupsen/logrus
-
-func ConfigureLogrus(level string, output string, timezone *time.Location) {
-
-	if output == "json" {
-		// Log as JSON instead of the default ASCII formatter.
-		logrus.SetFormatter(LocalTimeZoneFormatter{
-			Timezone:  timezone,
-			Formatter: &logrus.JSONFormatter{},
-		})
-	} else if output == "text" {
-		// Log as text with colors.
-		formatter := logrus.TextFormatter{
-			ForceColors:   true,
-			FullTimestamp: true,
-		}
-		logrus.SetFormatter(LocalTimeZoneFormatter{
-			Timezone:  timezone,
-			Formatter: &formatter,
-		})
-	}
-
-	// Use local timezone for providing datetime in logs!
-
-	// Output to stdout instead of the default stderr
-	// Can be any io.Writer, see below for File example
-	logrus.SetOutput(os.Stdout)
-
-	// Only log the warning severity or above.
-	logLevel := logrus.InfoLevel
-	if level == "error" {
-		logLevel = logrus.ErrorLevel
-	} else if level == "debug" {
-		logLevel = logrus.DebugLevel
-	} else if level == "fatal" {
-		logLevel = logrus.FatalLevel
-	} else if level == "warning" {
-		logLevel = logrus.WarnLevel
-	} // Add this line for logging filename and line number!
-	logrus.SetLevel(logLevel)
-}
-
-type LocalTimeZoneFormatter struct {
-	Timezone  *time.Location
-	Formatter logrus.Formatter
-}
-
-func (u LocalTimeZoneFormatter) Format(e *logrus.Entry) ([]byte, error) {
-	e.Time = e.Time.In(u.Timezone)
-	return u.Formatter.Format(e)
-}
-
-type Logging struct {
-	Logger string
-}
-
-func (self *Logging) Init(level string, logoutput string, configDirectory string, timezone *time.Location) {
-	switch self.Logger {
-	case "go-logging":
-		ConfigureGoLogging(configDirectory, timezone)
-	case "logrus":
-		ConfigureLogrus(level, logoutput, timezone)
-	default:
-	}
-}
-
-func (self *Logging) Info(sentence string) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Info(sentence)
-	case "logrus":
-		logrus.Info(sentence)
-	default:
-	}
-}
-
-func (self *Logging) Infof(format string, args ...interface{}) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Infof(format, args...)
-	case "logrus":
-		logrus.Infof(format, args...)
-	default:
-	}
-}
-
-func (self *Logging) Warning(sentence string) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Warning(sentence)
-	case "logrus":
-		logrus.Warn(sentence)
-	default:
-	}
-}
-
-func (self *Logging) Debug(sentence string) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Debug(sentence)
-	case "logrus":
-		logrus.Debug(sentence)
-	default:
-	}
-}
-
-func (self *Logging) Debugf(format string, args ...interface{}) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Debugf(format, args...)
-	case "logrus":
-		logrus.Debugf(format, args...)
-	default:
-	}
-}
-
-func (self *Logging) Error(sentence string) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Error(sentence)
-	case "logrus":
-		logrus.Error(sentence)
-	default:
-	}
-}
-
-func (self *Logging) Fatal(sentence string) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Fatal(sentence)
-	case "logrus":
-		logrus.Fatal(sentence)
-	default:
-	}
-}

machinery/src/models/agent_run.go

@@ -0,0 +1,477 @@
+package models
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/kerberos-io/agent/machinery/src/lifecycle"
+	"github.com/kerberos-io/agent/machinery/src/packets"
+	log "github.com/sirupsen/logrus"
+)
+
+var (
+	ErrAgentRunActive  = errors.New("another agent run is active")
+	ErrAgentRunStarted = errors.New("agent run already started")
+	ErrAgentRunStopped = errors.New("agent run is stopping")
+	nextAgentRunID     atomic.Uint64
+)
+
+type AgentRunClient interface {
+	Close(context.Context) error
+}
+
+type AgentRunResourceError struct {
+	Resource string
+	Err      error
+}
+
+func (e AgentRunResourceError) Error() string {
+	return fmt.Sprintf("close %s: %v", e.Resource, e.Err)
+}
+
+func (e AgentRunResourceError) Unwrap() error {
+	return e.Err
+}
+
+type AgentRunShutdownReport struct {
+	lifecycle.ShutdownReport
+	ResourceErrors      []AgentRunResourceError
+	UploadStopDelivered bool
+	StreamStopDelivered bool
+}
+
+type AgentRun struct {
+	id            uint64
+	ctx           context.Context
+	cancel        context.CancelCauseFunc
+	supervisor    *lifecycle.Supervisor
+	communication *Communication
+	stopUpload    bool
+
+	stateMu   sync.Mutex
+	activated bool
+	stopping  bool
+
+	resourcesMu       sync.RWMutex
+	mainClient        AgentRunClient
+	subClient         AgentRunClient
+	backchannelClient AgentRunClient
+	mainQueue         *packets.Queue
+	subQueue          *packets.Queue
+	releaseClients    func()
+
+	channelsMu       sync.RWMutex
+	channelsClosed   bool
+	liveHDHandshakes chan LiveHDHandshake
+	motionEvents     chan MotionDataPartial
+	onvifActions     chan OnvifAction
+
+	shutdownOnce   sync.Once
+	shutdownReport AgentRunShutdownReport
+}
+
+func NewAgentRun(parent context.Context, communication *Communication, stopUpload bool) *AgentRun {
+	if parent == nil {
+		parent = context.Background()
+	}
+	ctx, cancel := context.WithCancelCause(parent)
+	run := &AgentRun{
+		id:               nextAgentRunID.Add(1),
+		ctx:              ctx,
+		cancel:           cancel,
+		supervisor:       lifecycle.NewSupervisor(ctx),
+		communication:    communication,
+		stopUpload:       stopUpload,
+		liveHDHandshakes: make(chan LiveHDHandshake, 100),
+		motionEvents:     make(chan MotionDataPartial, 10),
+		onvifActions:     make(chan OnvifAction, 10),
+	}
+	log.WithFields(log.Fields{
+		"component":   "agent_run",
+		"event":       "run_created",
+		"run_id":      run.id,
+		"stop_upload": stopUpload,
+	}).Debug("Agent run created")
+	return run
+}
+
+func (r *AgentRun) ID() uint64 {
+	return r.id
+}
+
+func (r *AgentRun) Context() context.Context {
+	return r.ctx
+}
+
+func (r *AgentRun) Go(name string, policy lifecycle.TaskPolicy, task lifecycle.TaskFunc) error {
+	if task == nil {
+		return r.supervisor.Go(name, policy, task)
+	}
+
+	fields := log.Fields{
+		"component":    "agent_run",
+		"event":        "task_lifecycle",
+		"long_running": policy.LongRunning,
+		"required":     policy.Required,
+		"run_id":       r.id,
+		"task":         name,
+	}
+	wrappedTask := func(ctx context.Context) error {
+		startedAt := time.Now()
+		log.WithFields(fields).Debug("Agent task started")
+		err := task(ctx)
+		entry := log.WithFields(fields).
+			WithField("duration_ms", time.Since(startedAt).Milliseconds())
+		if err != nil {
+			entry = entry.WithError(err)
+			if ctx.Err() == nil {
+				entry.Warn("Agent task stopped with an error")
+			} else {
+				entry.Debug("Agent task stopped during shutdown")
+			}
+		} else {
+			entry.Debug("Agent task stopped")
+		}
+		return err
+	}
+
+	if err := r.supervisor.Go(name, policy, wrappedTask); err != nil {
+		log.WithError(err).WithFields(fields).Error("Failed to register Agent task")
+		return err
+	}
+	log.WithFields(fields).Debug("Agent task registered")
+	return nil
+}
+
+func (r *AgentRun) Seal() {
+	r.supervisor.Seal()
+	log.WithFields(log.Fields{
+		"component":  "agent_run",
+		"event":      "run_sealed",
+		"run_id":     r.id,
+		"task_count": len(r.supervisor.Snapshot()),
+	}).Debug("Agent run sealed")
+}
+
+func (r *AgentRun) Failures() <-chan lifecycle.Failure {
+	return r.supervisor.Failures()
+}
+
+func (r *AgentRun) Snapshot() []lifecycle.TaskSnapshot {
+	return r.supervisor.Snapshot()
+}
+
+func (r *AgentRun) Wait(ctx context.Context) lifecycle.ShutdownReport {
+	return r.supervisor.Wait(ctx)
+}
+
+func (r *AgentRun) Activate() error {
+	r.stateMu.Lock()
+	defer r.stateMu.Unlock()
+	if r.stopping {
+		return ErrAgentRunStopped
+	}
+	if r.ctx.Err() != nil {
+		return ErrAgentRunStopped
+	}
+	if r.activated {
+		return ErrAgentRunStarted
+	}
+	if r.communication == nil {
+		return errors.New("agent run communication is required")
+	}
+	if err := r.communication.attachRun(r); err != nil {
+		return err
+	}
+	r.activated = true
+	log.WithFields(log.Fields{
+		"component": "agent_run",
+		"event":     "run_activated",
+		"run_id":    r.id,
+	}).Info("Agent run activated")
+	return nil
+}
+
+func (r *AgentRun) SetMainClient(client AgentRunClient) {
+	r.resourcesMu.Lock()
+	r.mainClient = client
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) SetSubClient(client AgentRunClient) {
+	r.resourcesMu.Lock()
+	r.subClient = client
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) SetBackchannelClient(client AgentRunClient) {
+	r.resourcesMu.Lock()
+	r.backchannelClient = client
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) SetQueues(mainQueue, subQueue *packets.Queue) {
+	r.resourcesMu.Lock()
+	r.mainQueue = mainQueue
+	r.subQueue = subQueue
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) SetMainQueue(queue *packets.Queue) {
+	r.resourcesMu.Lock()
+	r.mainQueue = queue
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) SetSubQueue(queue *packets.Queue) {
+	r.resourcesMu.Lock()
+	r.subQueue = queue
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) SetClientRelease(release func()) {
+	r.resourcesMu.Lock()
+	r.releaseClients = release
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) MainQueue() *packets.Queue {
+	if r.isStopping() {
+		return nil
+	}
+	r.resourcesMu.RLock()
+	defer r.resourcesMu.RUnlock()
+	return r.mainQueue
+}
+
+func (r *AgentRun) SubQueue() *packets.Queue {
+	if r.isStopping() {
+		return nil
+	}
+	r.resourcesMu.RLock()
+	defer r.resourcesMu.RUnlock()
+	return r.subQueue
+}
+
+func (r *AgentRun) LiveHDHandshakes() <-chan LiveHDHandshake {
+	return r.liveHDHandshakes
+}
+
+func (r *AgentRun) MotionEvents() <-chan MotionDataPartial {
+	return r.motionEvents
+}
+
+func (r *AgentRun) ONVIFActions() <-chan OnvifAction {
+	return r.onvifActions
+}
+
+func (r *AgentRun) TrySendLiveHDHandshake(handshake LiveHDHandshake) bool {
+	if r.isStopping() {
+		return false
+	}
+	r.channelsMu.RLock()
+	defer r.channelsMu.RUnlock()
+	if r.channelsClosed {
+		return false
+	}
+	select {
+	case r.liveHDHandshakes <- handshake:
+		return true
+	default:
+		return false
+	}
+}
+
+func (r *AgentRun) PendingLiveHDHandshakes() int {
+	if r.isStopping() {
+		return 0
+	}
+	r.channelsMu.RLock()
+	defer r.channelsMu.RUnlock()
+	if r.channelsClosed {
+		return 0
+	}
+	return len(r.liveHDHandshakes)
+}
+
+func (r *AgentRun) TrySendMotion(motion MotionDataPartial) bool {
+	if r.isStopping() {
+		return false
+	}
+	r.channelsMu.RLock()
+	defer r.channelsMu.RUnlock()
+	if r.channelsClosed {
+		return false
+	}
+	select {
+	case r.motionEvents <- motion:
+		return true
+	default:
+		return false
+	}
+}
+
+func (r *AgentRun) TrySendONVIF(action OnvifAction) bool {
+	if r.isStopping() {
+		return false
+	}
+	r.channelsMu.RLock()
+	defer r.channelsMu.RUnlock()
+	if r.channelsClosed {
+		return false
+	}
+	select {
+	case r.onvifActions <- action:
+		return true
+	default:
+		return false
+	}
+}
+
+func (r *AgentRun) Shutdown(ctx context.Context, cause error) AgentRunShutdownReport {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	if cause == nil {
+		cause = context.Canceled
+	}
+
+	r.shutdownOnce.Do(func() {
+		startedAt := time.Now()
+		log.WithError(cause).WithFields(log.Fields{
+			"component": "agent_run",
+			"event":     "shutdown_started",
+			"run_id":    r.id,
+		}).Info("Agent run shutdown started")
+
+		r.stateMu.Lock()
+		r.stopping = true
+		activated := r.activated
+		r.stateMu.Unlock()
+
+		r.cancel(cause)
+		r.supervisor.BeginShutdown(cause)
+
+		if activated && r.stopUpload {
+			r.shutdownReport.UploadStopDelivered = sendRunStop(ctx, r.communication.HandleUpload)
+		}
+		if activated {
+			r.shutdownReport.StreamStopDelivered = sendRunStop(ctx, r.communication.HandleStream)
+		}
+
+		r.resourcesMu.RLock()
+		mainClient := r.mainClient
+		subClient := r.subClient
+		backchannelClient := r.backchannelClient
+		mainQueue := r.mainQueue
+		subQueue := r.subQueue
+		releaseClients := r.releaseClients
+		r.resourcesMu.RUnlock()
+
+		r.closeClient(ctx, "main RTSP client", mainClient)
+		if mainQueue != nil {
+			_ = mainQueue.Close()
+		}
+		r.closeClient(ctx, "sub RTSP client", subClient)
+		if subQueue != nil {
+			_ = subQueue.Close()
+		}
+		r.closeClient(ctx, "RTSP backchannel client", backchannelClient)
+		r.closeChannels()
+		if releaseClients != nil {
+			releaseClients()
+		}
+
+		r.shutdownReport.ShutdownReport = r.supervisor.Wait(ctx)
+		if r.shutdownReport.Complete && r.communication != nil {
+			r.communication.detachRun(r)
+		}
+
+		runningTasks := make([]string, 0, len(r.shutdownReport.Running))
+		for _, task := range r.shutdownReport.Running {
+			runningTasks = append(runningTasks, task.Name)
+		}
+		entry := log.WithFields(log.Fields{
+			"component":            "agent_run",
+			"duration_ms":          time.Since(startedAt).Milliseconds(),
+			"resource_error_count": len(r.shutdownReport.ResourceErrors),
+			"run_id":               r.id,
+			"running_tasks":        runningTasks,
+			"task_count":           len(r.shutdownReport.Tasks),
+			"upload_stop_sent":     r.shutdownReport.UploadStopDelivered,
+			"stream_stop_sent":     r.shutdownReport.StreamStopDelivered,
+		})
+		if r.shutdownReport.Complete {
+			entry.WithField("event", "shutdown_completed").Info("Agent run shutdown completed")
+		} else {
+			entry.WithField("event", "shutdown_timed_out").Error("Agent run shutdown timed out")
+		}
+	})
+
+	report := r.shutdownReport
+	report.ResourceErrors = append([]AgentRunResourceError(nil), report.ResourceErrors...)
+	return report
+}
+
+func (r *AgentRun) isStopping() bool {
+	r.stateMu.Lock()
+	defer r.stateMu.Unlock()
+	return r.stopping
+}
+
+func (r *AgentRun) closeClient(ctx context.Context, name string, client AgentRunClient) {
+	if client == nil {
+		return
+	}
+	if err := client.Close(ctx); err != nil {
+		r.shutdownReport.ResourceErrors = append(r.shutdownReport.ResourceErrors, AgentRunResourceError{
+			Resource: name,
+			Err:      err,
+		})
+		log.WithError(err).WithFields(log.Fields{
+			"component": "agent_run",
+			"event":     "resource_close_failed",
+			"resource":  name,
+			"run_id":    r.id,
+		}).Warn("Failed to close Agent run resource")
+		return
+	}
+	log.WithFields(log.Fields{
+		"component": "agent_run",
+		"event":     "resource_closed",
+		"resource":  name,
+		"run_id":    r.id,
+	}).Debug("Agent run resource closed")
+}
+
+func (r *AgentRun) closeChannels() {
+	r.channelsMu.Lock()
+	defer r.channelsMu.Unlock()
+	if r.channelsClosed {
+		return
+	}
+	r.channelsClosed = true
+	close(r.liveHDHandshakes)
+	close(r.motionEvents)
+	close(r.onvifActions)
+}
+
+func sendRunStop(ctx context.Context, channel chan<- string) bool {
+	if channel == nil {
+		return false
+	}
+	timer := time.NewTimer(time.Second)
+	defer timer.Stop()
+	select {
+	case channel <- "stop":
+		return true
+	case <-ctx.Done():
+		return false
+	case <-timer.C:
+		return false
+	}
+}

machinery/src/models/agent_run_test.go

@@ -0,0 +1,303 @@
+package models
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/kerberos-io/agent/machinery/src/lifecycle"
+	"github.com/kerberos-io/agent/machinery/src/packets"
+)
+
+type fakeAgentRunClient struct {
+	mu       sync.Mutex
+	name     string
+	order    *[]string
+	calls    int
+	closeErr error
+}
+
+func (c *fakeAgentRunClient) Close(context.Context) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.calls++
+	*c.order = append(*c.order, c.name)
+	return c.closeErr
+}
+
+func (c *fakeAgentRunClient) Calls() int {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.calls
+}
+
+func TestAgentRunOwnsAndShutsDownResources(t *testing.T) {
+	communication := &Communication{
+		HandleStream: make(chan string, 1),
+		HandleUpload: make(chan string, 1),
+	}
+	run := NewAgentRun(context.Background(), communication, true)
+	mainQueue := packets.NewQueue()
+	subQueue := packets.NewQueue()
+	run.SetQueues(mainQueue, subQueue)
+
+	var order []string
+	mainClient := &fakeAgentRunClient{name: "main", order: &order}
+	subClient := &fakeAgentRunClient{name: "sub", order: &order}
+	backchannelClient := &fakeAgentRunClient{name: "backchannel", order: &order}
+	run.SetMainClient(mainClient)
+	run.SetSubClient(subClient)
+	run.SetBackchannelClient(backchannelClient)
+	run.SetClientRelease(func() {
+		order = append(order, "release")
+	})
+
+	taskStarted := make(chan struct{})
+	if err := run.Go("worker", lifecycle.TaskPolicy{}, func(ctx context.Context) error {
+		close(taskStarted)
+		<-ctx.Done()
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := run.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	run.Seal()
+	<-taskStarted
+
+	report := run.Shutdown(context.Background(), errors.New("test shutdown"))
+	if !report.Complete {
+		t.Fatalf("shutdown report = %+v, want complete", report)
+	}
+	if communication.CurrentRun() != nil {
+		t.Fatal("shutdown left the run attached")
+	}
+	if got, want := order, []string{"main", "sub", "backchannel", "release"}; !equalStrings(got, want) {
+		t.Fatalf("resource order = %v, want %v", got, want)
+	}
+	if !report.UploadStopDelivered || !report.StreamStopDelivered {
+		t.Fatalf("stop delivery = upload:%t stream:%t, want both", report.UploadStopDelivered, report.StreamStopDelivered)
+	}
+	select {
+	case value := <-communication.HandleUpload:
+		if value != "stop" {
+			t.Fatalf("upload stop = %q, want stop", value)
+		}
+	default:
+		t.Fatal("upload stop was not delivered")
+	}
+	select {
+	case value := <-communication.HandleStream:
+		if value != "stop" {
+			t.Fatalf("stream stop = %q, want stop", value)
+		}
+	default:
+		t.Fatal("stream stop was not delivered")
+	}
+	if _, err := mainQueue.Latest().ReadPacket(); err == nil {
+		t.Fatal("main queue remained open")
+	}
+	if _, err := subQueue.Latest().ReadPacket(); err == nil {
+		t.Fatal("sub queue remained open")
+	}
+	if _, ok := <-run.LiveHDHandshakes(); ok {
+		t.Fatal("handshake channel remained open")
+	}
+	if _, ok := <-run.MotionEvents(); ok {
+		t.Fatal("motion channel remained open")
+	}
+	if _, ok := <-run.ONVIFActions(); ok {
+		t.Fatal("ONVIF channel remained open")
+	}
+}
+
+func TestAgentRunShutdownIsConcurrentAndIdempotent(t *testing.T) {
+	communication := &Communication{}
+	run := NewAgentRun(context.Background(), communication, false)
+	var order []string
+	client := &fakeAgentRunClient{name: "main", order: &order}
+	run.SetMainClient(client)
+	if err := run.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	run.Seal()
+
+	var callers sync.WaitGroup
+	reports := make(chan AgentRunShutdownReport, 20)
+	for index := 0; index < 20; index++ {
+		callers.Add(1)
+		go func() {
+			defer callers.Done()
+			reports <- run.Shutdown(context.Background(), errors.New("test shutdown"))
+		}()
+	}
+	callers.Wait()
+	close(reports)
+
+	for report := range reports {
+		if !report.Complete {
+			t.Fatalf("shutdown report = %+v, want complete", report)
+		}
+	}
+	if got := client.Calls(); got != 1 {
+		t.Fatalf("client Close() calls = %d, want 1", got)
+	}
+}
+
+func TestAgentRunStaleShutdownDoesNotDetachNewRun(t *testing.T) {
+	communication := &Communication{}
+	oldRun := NewAgentRun(context.Background(), communication, false)
+	if err := oldRun.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	if !communication.detachRun(oldRun) {
+		t.Fatal("failed to detach old run during test setup")
+	}
+
+	newRun := NewAgentRun(context.Background(), communication, false)
+	if err := newRun.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	newRun.Seal()
+	t.Cleanup(func() {
+		newRun.Shutdown(context.Background(), errors.New("test complete"))
+	})
+
+	oldRun.Shutdown(context.Background(), errors.New("stale shutdown"))
+	if got := communication.CurrentRun(); got != newRun {
+		t.Fatalf("current run = %p, want new run %p", got, newRun)
+	}
+}
+
+func TestAgentRunRejectsOverlap(t *testing.T) {
+	communication := &Communication{}
+	first := NewAgentRun(context.Background(), communication, false)
+	if err := first.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	first.Seal()
+	t.Cleanup(func() {
+		first.Shutdown(context.Background(), errors.New("test complete"))
+	})
+
+	second := NewAgentRun(context.Background(), communication, false)
+	if err := second.Activate(); !errors.Is(err, ErrAgentRunActive) {
+		t.Fatalf("Activate() error = %v, want ErrAgentRunActive", err)
+	}
+	second.Shutdown(context.Background(), errors.New("test complete"))
+}
+
+func TestAgentRunRetainsOwnershipUntilWorkersStop(t *testing.T) {
+	communication := &Communication{}
+	run := NewAgentRun(context.Background(), communication, false)
+	release := make(chan struct{})
+	started := make(chan struct{})
+	if err := run.Go("blocked", lifecycle.TaskPolicy{}, func(context.Context) error {
+		close(started)
+		<-release
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := run.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	run.Seal()
+	<-started
+
+	shutdownDone := make(chan AgentRunShutdownReport, 1)
+	go func() {
+		shutdownDone <- run.Shutdown(context.Background(), errors.New("test shutdown"))
+	}()
+
+	deadline := time.Now().Add(time.Second)
+	for run.Context().Err() == nil && time.Now().Before(deadline) {
+		time.Sleep(time.Millisecond)
+	}
+	if run.Context().Err() == nil {
+		t.Fatal("run context was not canceled")
+	}
+	if communication.CurrentRun() != run {
+		t.Fatal("run ownership was released before its worker stopped")
+	}
+	if communication.TrySendMotion(MotionDataPartial{}) {
+		t.Fatal("stopping run accepted new input")
+	}
+	if communication.MainQueue() != nil {
+		t.Fatal("stopping run exposed its queue")
+	}
+
+	replacement := NewAgentRun(context.Background(), communication, false)
+	if err := replacement.Activate(); !errors.Is(err, ErrAgentRunActive) {
+		t.Fatalf("replacement Activate() error = %v, want ErrAgentRunActive", err)
+	}
+	replacement.Shutdown(context.Background(), errors.New("test complete"))
+
+	close(release)
+	report := <-shutdownDone
+	if !report.Complete {
+		t.Fatalf("shutdown report = %+v, want complete", report)
+	}
+	if communication.CurrentRun() != nil {
+		t.Fatal("completed shutdown left the run attached")
+	}
+}
+
+func TestAgentRunShutdownReportsStuckTask(t *testing.T) {
+	communication := &Communication{}
+	run := NewAgentRun(context.Background(), communication, false)
+	release := make(chan struct{})
+	started := make(chan struct{})
+	if err := run.Go("blocked", lifecycle.TaskPolicy{}, func(context.Context) error {
+		close(started)
+		<-release
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := run.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	run.Seal()
+	<-started
+
+	waitContext, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+	defer cancel()
+	report := run.Shutdown(waitContext, errors.New("test shutdown"))
+	if report.Complete {
+		t.Fatal("shutdown unexpectedly completed")
+	}
+	if len(report.Running) != 1 || report.Running[0].Name != "blocked" {
+		t.Fatalf("running tasks = %+v, want blocked", report.Running)
+	}
+	if communication.CurrentRun() != run {
+		t.Fatal("timed-out shutdown released run ownership")
+	}
+	replacement := NewAgentRun(context.Background(), communication, false)
+	if err := replacement.Activate(); !errors.Is(err, ErrAgentRunActive) {
+		t.Fatalf("replacement Activate() error = %v, want ErrAgentRunActive", err)
+	}
+	replacement.Shutdown(context.Background(), errors.New("test complete"))
+	close(release)
+	waitContext, waitCancel := context.WithTimeout(context.Background(), time.Second)
+	defer waitCancel()
+	if followup := run.Wait(waitContext); !followup.Complete {
+		t.Fatalf("follow-up report = %+v, want complete", followup)
+	}
+	communication.detachRun(run)
+}
+
+func equalStrings(left, right []string) bool {
+	if len(left) != len(right) {
+		return false
+	}
+	for index := range left {
+		if left[index] != right[index] {
+			return false
+		}
+	}
+	return true
+}

machinery/src/models/communication.go

@@ -1,8 +1,10 @@
 package models
 
 import (
-	"context"
+	"math"
+	"sync"
 	"sync/atomic"
+	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/tevino/abool"
@@ -19,11 +21,84 @@ type LiveHDHandshake struct {
 	Signaling *LiveHDSignalingCallbacks
 }
 
+type MoQRecoveryTelemetry struct {
+	Reconnects          uint64 `json:"reconnects"`
+	LastFrameUnixMillis int64  `json:"lastFrameUnixMillis"`
+	LastWriteMillis     int64  `json:"lastWriteMillis"`
+}
+
+type RecoveryTelemetry struct {
+	MoQHigh                   MoQRecoveryTelemetry `json:"moqHigh"`
+	MoQLow                    MoQRecoveryTelemetry `json:"moqLow"`
+	MoQWriteTimeouts          uint64               `json:"moqWriteTimeouts"`
+	DroppedLiveHDHandshakes   uint64               `json:"droppedLiveHDHandshakes"`
+	DroppedMotionEvents       uint64               `json:"droppedMotionEvents"`
+	DroppedONVIFActions       uint64               `json:"droppedOnvifActions"`
+	WatchdogRestarts          uint64               `json:"watchdogRestarts"`
+	WatchdogCooldownSeconds   int64                `json:"watchdogCooldownSeconds"`
+	RunWorkerShutdownTimeouts uint64               `json:"runWorkerShutdownTimeouts"`
+}
+
+type StreamKind uint8
+
+const (
+	MainStream StreamKind = iota
+	SubStream
+)
+
+type StreamRuntimeTelemetry struct {
+	Configured        bool
+	PackagesProcessed uint64
+	FPS               float64
+	Width             int64
+	Height            int64
+	LastPacketAt      int64
+}
+
+type streamRuntimeTelemetry struct {
+	configured        atomic.Bool
+	packagesProcessed atomic.Uint64
+	fpsBits           atomic.Uint64
+	width             atomic.Int64
+	height            atomic.Int64
+	lastPacketAt      atomic.Int64
+}
+
+type HubRuntimeTelemetry struct {
+	Configured                bool
+	Connected                 bool
+	LastHeartbeatAttemptAt    int64
+	LastSuccessfulHeartbeatAt int64
+}
+
+type hubRuntimeTelemetry struct {
+	configured                atomic.Bool
+	connected                 atomic.Bool
+	lastHeartbeatAttemptAt    atomic.Int64
+	lastSuccessfulHeartbeatAt atomic.Int64
+}
+
+type recoveryTelemetry struct {
+	moqHighReconnects          atomic.Uint64
+	moqHighLastFrameUnixMillis atomic.Int64
+	moqHighLastWriteMillis     atomic.Int64
+	moqLowReconnects           atomic.Uint64
+	moqLowLastFrameUnixMillis  atomic.Int64
+	moqLowLastWriteMillis      atomic.Int64
+	moqWriteTimeouts           atomic.Uint64
+	droppedLiveHDHandshakes    atomic.Uint64
+	droppedMotionEvents        atomic.Uint64
+	droppedONVIFActions        atomic.Uint64
+	watchdogRestarts           atomic.Uint64
+	watchdogCooldownSeconds    atomic.Int64
+	runWorkerShutdownTimeouts  atomic.Uint64
+}
+
 // The communication struct that is managing
 // all the communication between the different goroutines.
 type Communication struct {
-	Context               *context.Context
-	CancelContext         *context.CancelFunc
+	currentRunMu          sync.RWMutex
+	currentRun            *AgentRun
 	PackageCounter        *atomic.Value
 	LastPacketTimer       *atomic.Value
 	PackageCounterSub     *atomic.Value
@@ -32,20 +107,17 @@ type Communication struct {
 	HandleBootstrap       chan string
 	HandleStream          chan string
 	HandleSubStream       chan string
-	HandleMotion          chan MotionDataPartial
 	HandleAudio           chan AudioDataPartial
 	HandleUpload          chan string
 	HandleHeartBeat       chan string
 	HandleLiveSD          chan int64
 	HandleLiveSDHTTP      chan int64
 	HandleLiveHDKeepalive chan string
-	HandleLiveHDHandshake chan LiveHDHandshake
 	HandleLiveHDPeers     chan string
 	// HandleLiveHLS is the live HLS viewer keepalive. It carries the requested
 	// quality tier ("auto"|"high"|"low"; empty => auto) so the producer can switch
 	// the live session between the main and sub stream on demand.
 	HandleLiveHLS chan string
-	HandleONVIF   chan OnvifAction
 	IsConfiguring *abool.AtomicBool
 	// IsRecordingManual is set while a viewer has requested a manual recording
 	// from the live view (the record button). While set, the motion-based
@@ -71,11 +143,224 @@ type Communication struct {
 	// viewer that starts a recording but never heartbeats (an older frontend)
 	// still records up to the max-duration cap instead of being cut off early.
 	RecordingManualHeartbeatSeen *abool.AtomicBool
-	Queue                        *packets.Queue
-	SubQueue                     *packets.Queue
 	Image                        string
-	CameraConnected              bool
-	MainStreamConnected          bool
-	SubStreamConnected           bool
-	HasBackChannel               bool
+	CameraConnected              atomic.Bool
+	MainStreamConnected          atomic.Bool
+	SubStreamConnected           atomic.Bool
+	HasBackChannel               atomic.Bool
+	mainStreamTelemetry          streamRuntimeTelemetry
+	subStreamTelemetry           streamRuntimeTelemetry
+	hubTelemetry                 hubRuntimeTelemetry
+	recovery                     recoveryTelemetry
+}
+
+func (c *Communication) streamTelemetry(stream StreamKind) *streamRuntimeTelemetry {
+	switch stream {
+	case MainStream:
+		return &c.mainStreamTelemetry
+	case SubStream:
+		return &c.subStreamTelemetry
+	default:
+		panic("unsupported stream kind")
+	}
+}
+
+func (c *Communication) SetStreamConfigured(stream StreamKind, configured bool) {
+	c.streamTelemetry(stream).configured.Store(configured)
+}
+
+func (c *Communication) RecordStreamPackage(stream StreamKind, fps float64, width, height int, at time.Time) {
+	telemetry := c.streamTelemetry(stream)
+	telemetry.packagesProcessed.Add(1)
+	if fps > 0 && !math.IsNaN(fps) && !math.IsInf(fps, 0) {
+		telemetry.fpsBits.Store(math.Float64bits(fps))
+	}
+	if width > 0 {
+		telemetry.width.Store(int64(width))
+	}
+	if height > 0 {
+		telemetry.height.Store(int64(height))
+	}
+	if !at.IsZero() {
+		telemetry.lastPacketAt.Store(at.Unix())
+	}
+}
+
+func (c *Communication) StreamRuntimeTelemetry(stream StreamKind) StreamRuntimeTelemetry {
+	telemetry := c.streamTelemetry(stream)
+	return StreamRuntimeTelemetry{
+		Configured:        telemetry.configured.Load(),
+		PackagesProcessed: telemetry.packagesProcessed.Load(),
+		FPS:               math.Float64frombits(telemetry.fpsBits.Load()),
+		Width:             telemetry.width.Load(),
+		Height:            telemetry.height.Load(),
+		LastPacketAt:      telemetry.lastPacketAt.Load(),
+	}
+}
+
+func (c *Communication) SetHubConfigured(configured bool) {
+	c.hubTelemetry.configured.Store(configured)
+	if !configured {
+		c.hubTelemetry.connected.Store(false)
+	}
+}
+
+func (c *Communication) RecordHubHeartbeatAttempt(at time.Time) {
+	if !at.IsZero() {
+		c.hubTelemetry.lastHeartbeatAttemptAt.Store(at.Unix())
+	}
+}
+
+func (c *Communication) RecordHubHeartbeatSuccess(at time.Time) {
+	if !at.IsZero() {
+		c.hubTelemetry.lastSuccessfulHeartbeatAt.Store(at.Unix())
+	}
+	c.hubTelemetry.connected.Store(true)
+}
+
+func (c *Communication) RecordHubHeartbeatFailure() {
+	c.hubTelemetry.connected.Store(false)
+}
+
+func (c *Communication) HubRuntimeTelemetry() HubRuntimeTelemetry {
+	return HubRuntimeTelemetry{
+		Configured:                c.hubTelemetry.configured.Load(),
+		Connected:                 c.hubTelemetry.connected.Load(),
+		LastHeartbeatAttemptAt:    c.hubTelemetry.lastHeartbeatAttemptAt.Load(),
+		LastSuccessfulHeartbeatAt: c.hubTelemetry.lastSuccessfulHeartbeatAt.Load(),
+	}
+}
+
+func (c *Communication) RecordMoQReconnect(quality string) {
+	if quality == StreamQualityLow {
+		c.recovery.moqLowReconnects.Add(1)
+		return
+	}
+	c.recovery.moqHighReconnects.Add(1)
+}
+
+func (c *Communication) RecordMoQWrite(quality string, duration time.Duration, at time.Time) {
+	if quality == StreamQualityLow {
+		c.recovery.moqLowLastWriteMillis.Store(duration.Milliseconds())
+		c.recovery.moqLowLastFrameUnixMillis.Store(at.UnixMilli())
+		return
+	}
+	c.recovery.moqHighLastWriteMillis.Store(duration.Milliseconds())
+	c.recovery.moqHighLastFrameUnixMillis.Store(at.UnixMilli())
+}
+
+func (c *Communication) RecordMoQWriteTimeout() {
+	c.recovery.moqWriteTimeouts.Add(1)
+}
+
+func (c *Communication) RecordWatchdogRestart(cooldown time.Duration) {
+	c.recovery.watchdogRestarts.Add(1)
+	c.SetWatchdogCooldown(cooldown)
+}
+
+func (c *Communication) SetWatchdogCooldown(cooldown time.Duration) {
+	c.recovery.watchdogCooldownSeconds.Store(int64(cooldown / time.Second))
+}
+
+func (c *Communication) RecordRunWorkerShutdownTimeout() {
+	c.recovery.runWorkerShutdownTimeouts.Add(1)
+}
+
+func (c *Communication) RecoveryTelemetry() RecoveryTelemetry {
+	return RecoveryTelemetry{
+		MoQHigh: MoQRecoveryTelemetry{
+			Reconnects:          c.recovery.moqHighReconnects.Load(),
+			LastFrameUnixMillis: c.recovery.moqHighLastFrameUnixMillis.Load(),
+			LastWriteMillis:     c.recovery.moqHighLastWriteMillis.Load(),
+		},
+		MoQLow: MoQRecoveryTelemetry{
+			Reconnects:          c.recovery.moqLowReconnects.Load(),
+			LastFrameUnixMillis: c.recovery.moqLowLastFrameUnixMillis.Load(),
+			LastWriteMillis:     c.recovery.moqLowLastWriteMillis.Load(),
+		},
+		MoQWriteTimeouts:          c.recovery.moqWriteTimeouts.Load(),
+		DroppedLiveHDHandshakes:   c.recovery.droppedLiveHDHandshakes.Load(),
+		DroppedMotionEvents:       c.recovery.droppedMotionEvents.Load(),
+		DroppedONVIFActions:       c.recovery.droppedONVIFActions.Load(),
+		WatchdogRestarts:          c.recovery.watchdogRestarts.Load(),
+		WatchdogCooldownSeconds:   c.recovery.watchdogCooldownSeconds.Load(),
+		RunWorkerShutdownTimeouts: c.recovery.runWorkerShutdownTimeouts.Load(),
+	}
+}
+
+func (c *Communication) attachRun(run *AgentRun) error {
+	c.currentRunMu.Lock()
+	defer c.currentRunMu.Unlock()
+	if c.currentRun != nil {
+		return ErrAgentRunActive
+	}
+	c.currentRun = run
+	return nil
+}
+
+func (c *Communication) detachRun(run *AgentRun) bool {
+	c.currentRunMu.Lock()
+	defer c.currentRunMu.Unlock()
+	if c.currentRun != run {
+		return false
+	}
+	c.currentRun = nil
+	return true
+}
+
+func (c *Communication) CurrentRun() *AgentRun {
+	c.currentRunMu.RLock()
+	defer c.currentRunMu.RUnlock()
+	return c.currentRun
+}
+
+func (c *Communication) MainQueue() *packets.Queue {
+	run := c.CurrentRun()
+	if run == nil {
+		return nil
+	}
+	return run.MainQueue()
+}
+
+func (c *Communication) SubQueue() *packets.Queue {
+	run := c.CurrentRun()
+	if run == nil {
+		return nil
+	}
+	return run.SubQueue()
+}
+
+func (c *Communication) TrySendLiveHDHandshake(handshake LiveHDHandshake) bool {
+	run := c.CurrentRun()
+	if run == nil || !run.TrySendLiveHDHandshake(handshake) {
+		c.recovery.droppedLiveHDHandshakes.Add(1)
+		return false
+	}
+	return true
+}
+
+func (c *Communication) PendingLiveHDHandshakes() int {
+	run := c.CurrentRun()
+	if run == nil {
+		return 0
+	}
+	return run.PendingLiveHDHandshakes()
+}
+
+func (c *Communication) TrySendMotion(motion MotionDataPartial) bool {
+	run := c.CurrentRun()
+	if run == nil || !run.TrySendMotion(motion) {
+		c.recovery.droppedMotionEvents.Add(1)
+		return false
+	}
+	return true
+}
+
+func (c *Communication) TrySendONVIF(action OnvifAction) bool {
+	run := c.CurrentRun()
+	if run == nil || !run.TrySendONVIF(action) {
+		c.recovery.droppedONVIFActions.Add(1)
+		return false
+	}
+	return true
 }

machinery/src/models/communication_test.go

@@ -0,0 +1,221 @@
+package models
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+	"time"
+)
+
+func TestCommunicationRunChannelDispatchAfterClose(t *testing.T) {
+	communication := &Communication{}
+	run := NewAgentRun(context.Background(), communication, false)
+	if err := run.Activate(); err != nil {
+		t.Fatal(err)
+	}
+
+	if !communication.TrySendLiveHDHandshake(LiveHDHandshake{}) {
+		t.Fatal("TrySendLiveHDHandshake() rejected an available channel")
+	}
+	if !communication.TrySendMotion(MotionDataPartial{}) {
+		t.Fatal("TrySendMotion() rejected an available channel")
+	}
+	if !communication.TrySendONVIF(OnvifAction{}) {
+		t.Fatal("TrySendONVIF() rejected an available channel")
+	}
+
+	report := run.Shutdown(context.Background(), errors.New("test complete"))
+	if !report.Complete {
+		t.Fatalf("shutdown report = %+v, want complete", report)
+	}
+	if communication.TrySendLiveHDHandshake(LiveHDHandshake{}) {
+		t.Fatal("TrySendLiveHDHandshake() accepted a closed run")
+	}
+	if communication.TrySendMotion(MotionDataPartial{}) {
+		t.Fatal("TrySendMotion() accepted a closed run")
+	}
+	if communication.TrySendONVIF(OnvifAction{}) {
+		t.Fatal("TrySendONVIF() accepted a closed run")
+	}
+}
+
+func TestCommunicationDispatchCanRaceRunChannelClose(t *testing.T) {
+	communication := &Communication{}
+	run := NewAgentRun(context.Background(), communication, false)
+	if err := run.Activate(); err != nil {
+		t.Fatal(err)
+	}
+
+	start := make(chan struct{})
+	var senders sync.WaitGroup
+	for sender := 0; sender < 20; sender++ {
+		senders.Add(1)
+		go func() {
+			defer senders.Done()
+			<-start
+			for attempt := 0; attempt < 100; attempt++ {
+				communication.TrySendLiveHDHandshake(LiveHDHandshake{})
+				communication.TrySendMotion(MotionDataPartial{})
+				communication.TrySendONVIF(OnvifAction{})
+			}
+		}()
+	}
+
+	close(start)
+	run.Shutdown(context.Background(), errors.New("test complete"))
+	senders.Wait()
+}
+
+func TestCommunicationRunChannelLifecycleSoak(t *testing.T) {
+	communication := &Communication{}
+	for cycle := 0; cycle < 100; cycle++ {
+		run := NewAgentRun(context.Background(), communication, false)
+		if err := run.Activate(); err != nil {
+			t.Fatalf("cycle %d Activate() error = %v", cycle, err)
+		}
+		handshakes := run.LiveHDHandshakes()
+		motion := run.MotionEvents()
+		onvif := run.ONVIFActions()
+
+		var consumers sync.WaitGroup
+		consumers.Add(3)
+		go func() {
+			defer consumers.Done()
+			for range handshakes {
+			}
+		}()
+		go func() {
+			defer consumers.Done()
+			for range motion {
+			}
+		}()
+		go func() {
+			defer consumers.Done()
+			for range onvif {
+			}
+		}()
+
+		var producers sync.WaitGroup
+		for producer := 0; producer < 4; producer++ {
+			producers.Add(1)
+			go func() {
+				defer producers.Done()
+				for attempt := 0; attempt < 50; attempt++ {
+					communication.TrySendLiveHDHandshake(LiveHDHandshake{})
+					communication.TrySendMotion(MotionDataPartial{})
+					communication.TrySendONVIF(OnvifAction{})
+				}
+			}()
+		}
+
+		run.Shutdown(context.Background(), errors.New("test complete"))
+		producers.Wait()
+
+		done := make(chan struct{})
+		go func() {
+			consumers.Wait()
+			close(done)
+		}()
+		select {
+		case <-done:
+		case <-time.After(time.Second):
+			t.Fatalf("cycle %d consumers did not stop after channel close", cycle)
+		}
+	}
+}
+
+func TestCommunicationRecoveryTelemetry(t *testing.T) {
+	communication := &Communication{}
+	communication.RecordMoQReconnect(StreamQualityHigh)
+	communication.RecordMoQReconnect(StreamQualityLow)
+	communication.RecordMoQWrite(StreamQualityHigh, 1250*time.Millisecond, time.UnixMilli(1234))
+	communication.RecordMoQWrite(StreamQualityLow, 2500*time.Millisecond, time.UnixMilli(5678))
+	communication.RecordMoQWriteTimeout()
+	communication.RecordWatchdogRestart(30 * time.Second)
+	communication.RecordRunWorkerShutdownTimeout()
+
+	if communication.TrySendLiveHDHandshake(LiveHDHandshake{}) {
+		t.Fatal("TrySendLiveHDHandshake() accepted an unavailable channel")
+	}
+	if communication.TrySendMotion(MotionDataPartial{}) {
+		t.Fatal("TrySendMotion() accepted an unavailable channel")
+	}
+	if communication.TrySendONVIF(OnvifAction{}) {
+		t.Fatal("TrySendONVIF() accepted an unavailable channel")
+	}
+
+	got := communication.RecoveryTelemetry()
+	if got.MoQHigh != (MoQRecoveryTelemetry{Reconnects: 1, LastFrameUnixMillis: 1234, LastWriteMillis: 1250}) {
+		t.Fatalf("high MoQ telemetry = %+v", got.MoQHigh)
+	}
+	if got.MoQLow != (MoQRecoveryTelemetry{Reconnects: 1, LastFrameUnixMillis: 5678, LastWriteMillis: 2500}) {
+		t.Fatalf("low MoQ telemetry = %+v", got.MoQLow)
+	}
+	if got.MoQWriteTimeouts != 1 || got.WatchdogRestarts != 1 || got.WatchdogCooldownSeconds != 30 || got.RunWorkerShutdownTimeouts != 1 {
+		t.Fatalf("recovery telemetry = %+v", got)
+	}
+	if got.DroppedLiveHDHandshakes != 1 || got.DroppedMotionEvents != 1 || got.DroppedONVIFActions != 1 {
+		t.Fatalf("drop telemetry = %+v", got)
+	}
+}
+
+func TestCommunicationOperationalTelemetry(t *testing.T) {
+	communication := &Communication{}
+	mainPacketAt := time.Unix(1_788_710_401, 0)
+	subPacketAt := time.Unix(1_788_710_402, 0)
+	hubAttemptAt := time.Unix(1_788_710_403, 0)
+	hubSuccessAt := time.Unix(1_788_710_404, 0)
+
+	communication.SetStreamConfigured(MainStream, true)
+	communication.SetStreamConfigured(SubStream, true)
+	communication.RecordStreamPackage(MainStream, 29.97, 1920, 1080, mainPacketAt)
+	communication.RecordStreamPackage(MainStream, 29.97, 1920, 1080, mainPacketAt)
+	communication.RecordStreamPackage(SubStream, 15, 640, 360, subPacketAt)
+	communication.SetHubConfigured(true)
+	communication.RecordHubHeartbeatAttempt(hubAttemptAt)
+	communication.RecordHubHeartbeatSuccess(hubSuccessAt)
+
+	main := communication.StreamRuntimeTelemetry(MainStream)
+	if main != (StreamRuntimeTelemetry{
+		Configured:        true,
+		PackagesProcessed: 2,
+		FPS:               29.97,
+		Width:             1920,
+		Height:            1080,
+		LastPacketAt:      mainPacketAt.Unix(),
+	}) {
+		t.Fatalf("main stream telemetry = %+v", main)
+	}
+
+	sub := communication.StreamRuntimeTelemetry(SubStream)
+	if sub != (StreamRuntimeTelemetry{
+		Configured:        true,
+		PackagesProcessed: 1,
+		FPS:               15,
+		Width:             640,
+		Height:            360,
+		LastPacketAt:      subPacketAt.Unix(),
+	}) {
+		t.Fatalf("sub stream telemetry = %+v", sub)
+	}
+
+	hub := communication.HubRuntimeTelemetry()
+	if hub != (HubRuntimeTelemetry{
+		Configured:                true,
+		Connected:                 true,
+		LastHeartbeatAttemptAt:    hubAttemptAt.Unix(),
+		LastSuccessfulHeartbeatAt: hubSuccessAt.Unix(),
+	}) {
+		t.Fatalf("Hub telemetry = %+v", hub)
+	}
+
+	communication.RecordHubHeartbeatFailure()
+	if communication.HubRuntimeTelemetry().Connected {
+		t.Fatal("Hub telemetry remained connected after a failed heartbeat")
+	}
+	communication.SetHubConfigured(false)
+	if communication.HubRuntimeTelemetry().Configured {
+		t.Fatal("Hub telemetry remained configured after it was disabled")
+	}
+}

machinery/src/models/mqtt.go

@@ -6,20 +6,96 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"encoding/pem"
-	"io"
-	"strings"
+	"errors"
+	"sync"
 	"time"
 
 	"github.com/gofrs/uuid"
 	"github.com/kerberos-io/agent/machinery/src/encryption"
-	"github.com/kerberos-io/agent/machinery/src/log"
+	log "github.com/sirupsen/logrus"
 )
 
+var (
+	errMQTTPrivateKeyPEMDecode = errors.New("error decoding PEM block containing private key")
+	errMQTTPrivateKeyNotRSA    = errors.New("private key is not RSA")
+
+	rsaPrivateKeyCache   = newRSAPrivateKeyCache()
+	parsePKCS8PrivateKey = x509.ParsePKCS8PrivateKey
+)
+
+type rsaPrivateKeyCacheState struct {
+	mu      sync.Mutex
+	cond    *sync.Cond
+	pem     string
+	key     *rsa.PrivateKey
+	err     error
+	parsing bool
+	ready   bool
+}
+
+func newRSAPrivateKeyCache() *rsaPrivateKeyCacheState {
+	cache := &rsaPrivateKeyCacheState{}
+	cache.cond = sync.NewCond(&cache.mu)
+	return cache
+}
+
+func (cache *rsaPrivateKeyCacheState) get(privateKey string) (*rsa.PrivateKey, error) {
+	cache.mu.Lock()
+	for {
+		if cache.ready && cache.pem == privateKey && !cache.parsing {
+			key, err := cache.key, cache.err
+			cache.mu.Unlock()
+			return key, err
+		}
+		if cache.parsing {
+			cache.cond.Wait()
+			continue
+		}
+
+		cache.pem = privateKey
+		cache.key = nil
+		cache.err = nil
+		cache.parsing = true
+		cache.ready = false
+		cache.mu.Unlock()
+
+		key, err := parseRSAPrivateKey(privateKey)
+
+		cache.mu.Lock()
+		cache.key = key
+		cache.err = err
+		cache.parsing = false
+		cache.ready = true
+		cache.cond.Broadcast()
+		cache.mu.Unlock()
+		return key, err
+	}
+}
+
+func parseRSAPrivateKey(privateKey string) (*rsa.PrivateKey, error) {
+	block, _ := pem.Decode([]byte(privateKey))
+	if block == nil {
+		return nil, errMQTTPrivateKeyPEMDecode
+	}
+
+	key, err := parsePKCS8PrivateKey(block.Bytes)
+	if err != nil {
+		return nil, err
+	}
+
+	rsaKey, ok := key.(*rsa.PrivateKey)
+	if !ok {
+		return nil, errMQTTPrivateKeyNotRSA
+	}
+
+	return rsaKey, nil
+}
+
 func PackageMQTTMessage(configuration *Configuration, msg Message) ([]byte, error) {
 	// Create a Version 4 UUID.
 	u2, err := uuid.NewV4()
 	if err != nil {
-		log.Log.Error("failed to generate UUID: " + err.Error())
+		log.Error("failed to generate UUID: " + err.Error())
 	}
 
 	// We'll generate an unique id, and encrypt / decrypt it using the private key if available.
@@ -45,27 +121,25 @@ func PackageMQTTMessage(configuration *Configuration, msg Message) ([]byte, erro
 		// Pload to base64
 		data, err := json.Marshal(pload)
 		if err != nil {
-			log.Log.Error("models.mqtt.PackageMQTTMessage(): failed to marshal payload: " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component": "models/mqtt",
+				"event":     "payload_marshal_failed",
+			}).Error("Failed to marshal MQTT payload")
 		}
 
 		// Encrypt the value
 		privateKey := configuration.Config.Encryption.PrivateKey
-		r := strings.NewReader(privateKey)
-		pemBytes, _ := io.ReadAll(r)
-		block, _ := pem.Decode(pemBytes)
-		if block == nil {
-			log.Log.Error("models.mqtt.PackageMQTTMessage(): error decoding PEM block containing private key")
-		} else {
-			// Parse private key
-			b := block.Bytes
-			key, err := x509.ParsePKCS8PrivateKey(b)
-			if err != nil {
-				log.Log.Error("models.mqtt.PackageMQTTMessage(): error parsing private key: " + err.Error())
+		rsaKey, err := rsaPrivateKeyCache.get(privateKey)
+		if err != nil {
+			if errors.Is(err, errMQTTPrivateKeyPEMDecode) {
+				log.Error("models.mqtt.PackageMQTTMessage(): error decoding PEM block containing private key")
+			} else {
+				log.WithError(err).WithFields(log.Fields{
+					"component": "models/mqtt",
+					"event":     "private_key_parse_failed",
+				}).Error("Failed to parse MQTT private key")
 			}
-
-			// Conver key to *rsa.PrivateKey
-			rsaKey, _ := key.(*rsa.PrivateKey)
-
+		} else {
 			// Create a 16bit key random
 			if config.Encryption != nil && config.Encryption.SymmetricKey != "" {
 				k := config.Encryption.SymmetricKey
@@ -250,3 +324,23 @@ type TriggerRelay struct {
 	DeviceId  string `json:"device_id"` // device id
 	Token     string `json:"token"`     // token
 }
+
+// RemoteSessionPayload controls an interactive shell or log stream over MQTT.
+// Data is base64 encoded so terminal control bytes remain valid JSON.
+type RemoteSessionPayload struct {
+	Timestamp int64  `json:"timestamp"`
+	SessionID string `json:"session_id"`
+	Kind      string `json:"kind,omitempty"`
+	Data      string `json:"data,omitempty"`
+	Rows      uint16 `json:"rows,omitempty"`
+	Columns   uint16 `json:"columns,omitempty"`
+	Tail      int    `json:"tail,omitempty"`
+}
+
+type RemoteSessionStatus struct {
+	Timestamp int64  `json:"timestamp"`
+	SessionID string `json:"session_id"`
+	Kind      string `json:"kind"`
+	State     string `json:"state"`
+	Error     string `json:"error,omitempty"`
+}

machinery/src/models/mqtt_test.go

@@ -0,0 +1,199 @@
+package models
+
+import (
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/json"
+	"encoding/pem"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestPackageMQTTMessageCachesCurrentSigningKey(t *testing.T) {
+	resetMQTTPrivateKeyCache(t)
+
+	privateKey := generatePKCS8RSAPrivateKeyPEM(t)
+
+	var parseCalls atomic.Int32
+	restore := hookPKCS8PrivateKeyParser(t, func(der []byte) (any, error) {
+		parseCalls.Add(1)
+		return x509.ParsePKCS8PrivateKey(der)
+	})
+	defer restore()
+
+	msg := Message{
+		Payload: Payload{
+			DeviceId: "device-1",
+			Value: map[string]interface{}{
+				"hello": "world",
+			},
+		},
+	}
+
+	first, err := PackageMQTTMessage(encryptedConfiguration(privateKey), msg)
+	if err != nil {
+		t.Fatalf("first PackageMQTTMessage() error = %v", err)
+	}
+	assertEncryptedMQTTMessage(t, first)
+
+	second, err := PackageMQTTMessage(encryptedConfiguration(privateKey), msg)
+	if err != nil {
+		t.Fatalf("second PackageMQTTMessage() error = %v", err)
+	}
+	assertEncryptedMQTTMessage(t, second)
+
+	if got := parseCalls.Load(); got != 1 {
+		t.Fatalf("cached parse calls = %d, want 1", got)
+	}
+}
+
+func TestPackageMQTTMessageReplacesCachedSigningKeyOnRotation(t *testing.T) {
+	resetMQTTPrivateKeyCache(t)
+
+	privateKey := generatePKCS8RSAPrivateKeyPEM(t)
+	rotatedPrivateKey := generatePKCS8RSAPrivateKeyPEM(t)
+
+	var parseCalls atomic.Int32
+	restore := hookPKCS8PrivateKeyParser(t, func(der []byte) (any, error) {
+		parseCalls.Add(1)
+		return x509.ParsePKCS8PrivateKey(der)
+	})
+	defer restore()
+
+	msg := Message{
+		Payload: Payload{
+			DeviceId: "device-1",
+			Value: map[string]interface{}{
+				"hello": "world",
+			},
+		},
+	}
+
+	for _, key := range []string{privateKey, rotatedPrivateKey, privateKey} {
+		payload, err := PackageMQTTMessage(encryptedConfiguration(key), msg)
+		if err != nil {
+			t.Fatalf("PackageMQTTMessage() error = %v", err)
+		}
+		assertEncryptedMQTTMessage(t, payload)
+	}
+
+	if got := parseCalls.Load(); got != 3 {
+		t.Fatalf("parse calls across old/new/old rotation = %d, want 3", got)
+	}
+}
+
+func TestPackageMQTTMessageConcurrentCallsReuseCachedSigningKey(t *testing.T) {
+	resetMQTTPrivateKeyCache(t)
+
+	privateKey := generatePKCS8RSAPrivateKeyPEM(t)
+
+	var parseCalls atomic.Int32
+	restore := hookPKCS8PrivateKeyParser(t, func(der []byte) (any, error) {
+		parseCalls.Add(1)
+		time.Sleep(10 * time.Millisecond)
+		return x509.ParsePKCS8PrivateKey(der)
+	})
+	defer restore()
+
+	msg := Message{
+		Payload: Payload{
+			DeviceId: "device-1",
+			Value: map[string]interface{}{
+				"hello": "world",
+			},
+		},
+	}
+
+	start := make(chan struct{})
+	var workers sync.WaitGroup
+	for i := 0; i < 16; i++ {
+		workers.Add(1)
+		go func() {
+			defer workers.Done()
+			<-start
+
+			payload, err := PackageMQTTMessage(encryptedConfiguration(privateKey), msg)
+			if err != nil {
+				t.Errorf("PackageMQTTMessage() error = %v", err)
+				return
+			}
+			assertEncryptedMQTTMessage(t, payload)
+		}()
+	}
+
+	close(start)
+	workers.Wait()
+
+	if got := parseCalls.Load(); got != 1 {
+		t.Fatalf("concurrent parse calls = %d, want 1", got)
+	}
+}
+
+func encryptedConfiguration(privateKey string) *Configuration {
+	return &Configuration{
+		Config: Config{
+			Encryption: &Encryption{
+				Enabled:      "true",
+				PrivateKey:   privateKey,
+				SymmetricKey: "secret",
+			},
+		},
+	}
+}
+
+func generatePKCS8RSAPrivateKeyPEM(t *testing.T) string {
+	t.Helper()
+
+	privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		t.Fatalf("rsa.GenerateKey() error = %v", err)
+	}
+
+	der, err := x509.MarshalPKCS8PrivateKey(privateKey)
+	if err != nil {
+		t.Fatalf("x509.MarshalPKCS8PrivateKey() error = %v", err)
+	}
+
+	return string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}))
+}
+
+func assertEncryptedMQTTMessage(t *testing.T, payload []byte) {
+	t.Helper()
+
+	var got Message
+	if err := json.Unmarshal(payload, &got); err != nil {
+		t.Fatalf("json.Unmarshal() error = %v", err)
+	}
+
+	if !got.Encrypted {
+		t.Fatal("message is not marked encrypted")
+	}
+	if got.Payload.EncryptedValue == "" {
+		t.Fatal("encrypted value is empty")
+	}
+	if got.Payload.Signature == "" {
+		t.Fatal("signature is empty")
+	}
+	if len(got.Payload.Value) != 0 {
+		t.Fatalf("payload value = %#v, want cleared map", got.Payload.Value)
+	}
+}
+
+func hookPKCS8PrivateKeyParser(t *testing.T, parser func([]byte) (any, error)) func() {
+	t.Helper()
+
+	original := parsePKCS8PrivateKey
+	parsePKCS8PrivateKey = parser
+
+	return func() {
+		parsePKCS8PrivateKey = original
+	}
+}
+
+func resetMQTTPrivateKeyCache(t *testing.T) {
+	t.Helper()
+	rsaPrivateKeyCache = newRSAPrivateKeyCache()
+}

machinery/src/models/recording_upload_metadata.go

@@ -16,6 +16,7 @@ type RecordingUploadMetadata struct {
 	Timestamp int64   `json:"timestamp"` // Unix milliseconds.
 	Duration  uint64  `json:"duration"`  // Milliseconds.
 	FPS       float64 `json:"fps,omitempty"`
+	Encrypted bool    `json:"encrypted,omitempty"`
 }
 
 // RecordingUploadMetadataFileName returns the queue marker name associated

machinery/src/onvif/discovery.go

@@ -13,8 +13,8 @@ import (
 	"time"
 
 	onvifc "github.com/cedricve/go-onvif"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 // scanPort describes a TCP port we probe while scanning the local network,
@@ -105,7 +105,7 @@ func DiscoverDevices(timeout time.Duration, subnets ...string) []models.Discover
 	// 1) ONVIF WS-Discovery. This is quick and reliable for ONVIF cameras.
 	onvifDevices, err := onvifc.StartDiscovery(timeout)
 	if err != nil {
-		log.Log.Error("onvif.DiscoverDevices(): WS-Discovery failed: " + err.Error())
+		log.Error("onvif.DiscoverDevices(): WS-Discovery failed: " + err.Error())
 	} else {
 		for _, onvifDevice := range onvifDevices {
 			ip := hostFromXAddr(onvifDevice.XAddr)
@@ -129,7 +129,7 @@ func DiscoverDevices(timeout time.Duration, subnets ...string) []models.Discover
 	} else {
 		targets = localScanTargets()
 	}
-	log.Log.Info("onvif.DiscoverDevices(): scanning " + strconv.Itoa(len(targets)) + " hosts on the local network(s)")
+	log.Info("onvif.DiscoverDevices(): scanning " + strconv.Itoa(len(targets)) + " hosts on the local network(s)")
 
 	// Bound the amount of concurrent dials so we do not exhaust file
 	// descriptors on constrained devices (e.g. Raspberry Pi).
@@ -319,17 +319,17 @@ func targetsFromSubnets(subnets []string) []string {
 					targets = append(targets, subnet)
 				}
 			} else {
-				log.Log.Error("onvif.targetsFromSubnets(): invalid address '" + subnet + "'")
+				log.Error("onvif.targetsFromSubnets(): invalid address '" + subnet + "'")
 			}
 			continue
 		}
 		_, ipNet, err := net.ParseCIDR(subnet)
 		if err != nil || ipNet.IP.To4() == nil {
-			log.Log.Error("onvif.targetsFromSubnets(): invalid CIDR '" + subnet + "'")
+			log.Error("onvif.targetsFromSubnets(): invalid CIDR '" + subnet + "'")
 			continue
 		}
 		if ones, bits := ipNet.Mask.Size(); bits != 32 || ones < 22 {
-			log.Log.Error("onvif.targetsFromSubnets(): range '" + subnet + "' is too large to scan (use /22 or smaller)")
+			log.Error("onvif.targetsFromSubnets(): range '" + subnet + "' is too large to scan (use /22 or smaller)")
 			continue
 		}
 		for _, host := range hostsInNetwork(ipNet) {
@@ -352,7 +352,7 @@ func localScanTargets() []string {
 
 	interfaces, err := net.Interfaces()
 	if err != nil {
-		log.Log.Error("onvif.localScanTargets(): " + err.Error())
+		log.Error("onvif.localScanTargets(): " + err.Error())
 		return targets
 	}
 

machinery/src/onvif/events.go

@@ -7,9 +7,9 @@ import (
 	"strings"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/onvif/event/stream"
+	log "github.com/sirupsen/logrus"
 )
 
 // The library handles in-stream reconnect; these guards cover the
@@ -29,14 +29,14 @@ const (
 // blip, credential reload) the goroutine retries with exponential
 // backoff. Exits when ctx is cancelled.
 func HandleONVIFEventStream(ctx context.Context, configuration *models.Configuration, communication *models.Communication) {
-	log.Log.Debug("onvif.HandleONVIFEventStream(): started")
-	defer log.Log.Debug("onvif.HandleONVIFEventStream(): finished")
+	log.Debug("onvif.HandleONVIFEventStream(): started")
+	defer log.Debug("onvif.HandleONVIFEventStream(): finished")
 
 	if !isONVIFMotionEnabled(configuration.Config.Capture.ONVIFMotion) {
 		return
 	}
 	if configuration.Config.Capture.IPCamera.ONVIFXAddr == "" {
-		log.Log.Warning("onvif.HandleONVIFEventStream(): ONVIFMotion enabled but ONVIFXAddr is empty; nothing to do")
+		log.Warn("onvif.HandleONVIFEventStream(): ONVIFMotion enabled but ONVIFXAddr is empty; nothing to do")
 		return
 	}
 
@@ -66,23 +66,26 @@ func runStreamOnce(ctx context.Context, configuration *models.Configuration, com
 
 	device, _, err := ConnectToOnvifDevice(&camera)
 	if err != nil {
-		log.Log.Error("onvif.HandleONVIFEventStream(): connect: " + err.Error())
+		log.Error("onvif.HandleONVIFEventStream(): connect: " + err.Error())
 		return true
 	}
 
 	deviceID := resolveDeviceID(configuration.Name, camera.ONVIFXAddr)
 	s, err := stream.NewStream(ctx, device, stream.Options{DeviceID: deviceID})
 	if err != nil {
-		log.Log.Error("onvif.HandleONVIFEventStream(): open stream: " + err.Error())
+		log.Error("onvif.HandleONVIFEventStream(): open stream: " + err.Error())
 		return true
 	}
 	defer func() {
 		if err := s.Close(); err != nil {
-			log.Log.Debug("onvif.HandleONVIFEventStream(): close: " + err.Error())
+			log.Debug("onvif.HandleONVIFEventStream(): close: " + err.Error())
 		}
 	}()
 
-	log.Log.Info("onvif.HandleONVIFEventStream(): consuming events for " + deviceID)
+	log.WithFields(log.Fields{
+		"component": "onvif",
+		"event":     "event_stream_started",
+	}).Info("Consuming ONVIF events")
 
 	// recovering = the first successful event after an error streak
 	// logs a recovery line so on-call operators see the clear-of-
@@ -97,7 +100,10 @@ func runStreamOnce(ctx context.Context, configuration *models.Configuration, com
 				return false
 			}
 			if recovering {
-				log.Log.Info("onvif.HandleONVIFEventStream(): event stream recovered for " + deviceID)
+				log.WithFields(log.Fields{
+					"component": "onvif",
+					"event":     "event_stream_recovered",
+				}).Info("ONVIF event stream recovered")
 				recovering = false
 			}
 			dispatchEvent(ctx, ev, configuration, communication)
@@ -119,14 +125,14 @@ func runStreamOnce(ctx context.Context, configuration *models.Configuration, com
 func dispatchEvent(ctx context.Context, ev stream.Event, configuration *models.Configuration, communication *models.Communication) {
 	topic := sanitiseTopic(ev.Topic)
 	if ev.Kind != stream.KindMotion {
-		log.Log.Debug("onvif.dispatchEvent(): non-motion event " + ev.Kind.String() + " topic=" + topic)
+		log.Debug("onvif.dispatchEvent(): non-motion event " + ev.Kind.String() + " topic=" + topic)
 		return
 	}
 	if ev.State != stream.StateActive {
 		return
 	}
 	if !isTransition(ev.Operation) {
-		log.Log.Debug("onvif.dispatchEvent(): " + ev.Operation.String() + " is not a transition, not a trigger: topic=" + topic)
+		log.Debug("onvif.dispatchEvent(): " + ev.Operation.String() + " is not a transition, not a trigger: topic=" + topic)
 		return
 	}
 	if configuration.Config.Capture.Recording == "false" {
@@ -140,14 +146,15 @@ func dispatchEvent(ctx context.Context, ev stream.Event, configuration *models.C
 		Timestamp:       time.Now().Unix(),
 		NumberOfChanges: 0, // ONVIF does not quantify motion area.
 	}
-	select {
-	case <-ctx.Done():
-	case communication.HandleMotion <- dataToPass:
+	if ctx.Err() != nil {
+		return
+	}
+	if communication.TrySendMotion(dataToPass) {
 		// Logged on the send, not before it: this line records that a
 		// recording started, so a dropped event must not leave one.
-		log.Log.Debug("onvif.dispatchEvent(): recording trigger " + ev.Kind.String() + " topic=" + topic)
-	default:
-		log.Log.Debug("onvif.dispatchEvent(): HandleMotion full, dropping ONVIF motion event")
+		log.Debug("onvif.dispatchEvent(): recording trigger " + ev.Kind.String() + " topic=" + topic)
+	} else {
+		log.Debug("onvif.dispatchEvent(): HandleMotion full, dropping ONVIF motion event")
 	}
 }
 
@@ -185,13 +192,13 @@ func logStreamError(e error) {
 	var renew stream.ErrRenewFailed
 	switch {
 	case errors.As(e, &recreate):
-		log.Log.Error("onvif.HandleONVIFEventStream(): subscription recreate failed (camera may be offline): " + recreate.Err.Error())
+		log.Error("onvif.HandleONVIFEventStream(): subscription recreate failed (camera may be offline): " + recreate.Err.Error())
 	case errors.As(e, &renew):
-		log.Log.Debug("onvif.HandleONVIFEventStream(): renew failed (will recover via pull/recreate): " + renew.Err.Error())
+		log.Debug("onvif.HandleONVIFEventStream(): renew failed (will recover via pull/recreate): " + renew.Err.Error())
 	case errors.As(e, &pull):
-		log.Log.Debug("onvif.HandleONVIFEventStream(): pull failed (will retry): " + pull.Err.Error())
+		log.Debug("onvif.HandleONVIFEventStream(): pull failed (will retry): " + pull.Err.Error())
 	default:
-		log.Log.Info("onvif.HandleONVIFEventStream(): stream error: " + e.Error())
+		log.Info("onvif.HandleONVIFEventStream(): stream error: " + e.Error())
 	}
 }
 

machinery/src/onvif/events_test.go

@@ -3,6 +3,7 @@ package onvif
 import (
 	"bytes"
 	"context"
+	"errors"
 	"strings"
 	"testing"
 	"time"
@@ -26,17 +27,31 @@ func makeConfig(recording, onvifMotion, name string) *models.Configuration {
 	}
 }
 
-func makeCommunication(buffer int) *models.Communication {
-	return &models.Communication{
-		HandleMotion: make(chan models.MotionDataPartial, buffer),
+func makeCommunication(t *testing.T) *models.Communication {
+	t.Helper()
+	communication := &models.Communication{}
+	run := models.NewAgentRun(context.Background(), communication, false)
+	require.NoError(t, run.Activate())
+	t.Cleanup(func() {
+		run.Shutdown(context.Background(), errors.New("test complete"))
+	})
+	return communication
+}
+
+func motionEvents(t *testing.T, communication *models.Communication) <-chan models.MotionDataPartial {
+	t.Helper()
+	run := communication.CurrentRun()
+	if run == nil {
+		t.Fatal("expected active agent run")
 	}
+	return run.MotionEvents()
 }
 
 // --- dispatchEvent ---------------------------------------------------
 
 func TestDispatchEvent_MotionActive_SendsToHandleMotion(t *testing.T) {
 	cfg := makeConfig("true", "true", "cam-1")
-	comm := makeCommunication(1)
+	comm := makeCommunication(t)
 	ev := stream.Event{Kind: stream.KindMotion, State: stream.StateActive}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -44,7 +59,7 @@ func TestDispatchEvent_MotionActive_SendsToHandleMotion(t *testing.T) {
 	dispatchEvent(ctx, ev, cfg, comm)
 
 	select {
-	case m := <-comm.HandleMotion:
+	case m := <-motionEvents(t, comm):
 		assert.NotZero(t, m.Timestamp)
 	case <-time.After(time.Second):
 		t.Fatal("expected motion data on HandleMotion")
@@ -53,7 +68,7 @@ func TestDispatchEvent_MotionActive_SendsToHandleMotion(t *testing.T) {
 
 func TestDispatchEvent_MotionInactive_DoesNotSend(t *testing.T) {
 	cfg := makeConfig("true", "true", "cam-1")
-	comm := makeCommunication(1)
+	comm := makeCommunication(t)
 	ev := stream.Event{Kind: stream.KindMotion, State: stream.StateInactive}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -61,7 +76,7 @@ func TestDispatchEvent_MotionInactive_DoesNotSend(t *testing.T) {
 	dispatchEvent(ctx, ev, cfg, comm)
 
 	select {
-	case <-comm.HandleMotion:
+	case <-motionEvents(t, comm):
 		t.Fatal("inactive motion must not reach HandleMotion (motion-stop is a follow-up)")
 	case <-time.After(100 * time.Millisecond):
 	}
@@ -69,7 +84,7 @@ func TestDispatchEvent_MotionInactive_DoesNotSend(t *testing.T) {
 
 func TestDispatchEvent_NonMotionKindIgnored(t *testing.T) {
 	cfg := makeConfig("true", "true", "cam-1")
-	comm := makeCommunication(1)
+	comm := makeCommunication(t)
 	ev := stream.Event{Kind: stream.KindDigitalInput, State: stream.StateActive}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -77,7 +92,7 @@ func TestDispatchEvent_NonMotionKindIgnored(t *testing.T) {
 	dispatchEvent(ctx, ev, cfg, comm)
 
 	select {
-	case <-comm.HandleMotion:
+	case <-motionEvents(t, comm):
 		t.Fatal("non-motion kinds must not reach HandleMotion")
 	case <-time.After(100 * time.Millisecond):
 	}
@@ -108,7 +123,7 @@ func TestDispatchEvent_LogsTheTriggeringTopic(t *testing.T) {
 	buf := captureDebugLog(t)
 
 	cfg := makeConfig("true", "true", "cam-1")
-	comm := makeCommunication(1)
+	comm := makeCommunication(t)
 	ev := stream.Event{
 		Kind:  stream.KindMotion,
 		State: stream.StateActive,
@@ -147,7 +162,7 @@ func TestDispatchEvent_PropertyOperation(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			cfg := makeConfig("true", "true", "cam-1")
-			comm := makeCommunication(1)
+			comm := makeCommunication(t)
 			ev := stream.Event{
 				Kind:      stream.KindMotion,
 				State:     stream.StateActive,
@@ -160,7 +175,7 @@ func TestDispatchEvent_PropertyOperation(t *testing.T) {
 			dispatchEvent(ctx, ev, cfg, comm)
 
 			select {
-			case <-comm.HandleMotion:
+			case <-motionEvents(t, comm):
 				if !tt.wantSend {
 					t.Fatalf("%v must not trigger a recording", tt.op)
 				}
@@ -175,7 +190,7 @@ func TestDispatchEvent_PropertyOperation(t *testing.T) {
 
 func TestDispatchEvent_RecordingDisabled_DoesNotSend(t *testing.T) {
 	cfg := makeConfig("false", "true", "cam-1")
-	comm := makeCommunication(1)
+	comm := makeCommunication(t)
 	ev := stream.Event{Kind: stream.KindMotion, State: stream.StateActive}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -183,7 +198,7 @@ func TestDispatchEvent_RecordingDisabled_DoesNotSend(t *testing.T) {
 	dispatchEvent(ctx, ev, cfg, comm)
 
 	select {
-	case <-comm.HandleMotion:
+	case <-motionEvents(t, comm):
 		t.Fatal("Recording=false must gate the send (matches computervision behaviour)")
 	case <-time.After(100 * time.Millisecond):
 	}
@@ -191,9 +206,10 @@ func TestDispatchEvent_RecordingDisabled_DoesNotSend(t *testing.T) {
 
 func TestDispatchEvent_HandleMotionFull_DropsRatherThanBlocks(t *testing.T) {
 	cfg := makeConfig("true", "true", "cam-1")
-	// Pre-fill the buffer so the next send would block.
-	comm := &models.Communication{HandleMotion: make(chan models.MotionDataPartial, 1)}
-	comm.HandleMotion <- models.MotionDataPartial{}
+	// Fill the buffer so the next send would be dropped.
+	comm := makeCommunication(t)
+	for comm.TrySendMotion(models.MotionDataPartial{}) {
+	}
 	ev := stream.Event{Kind: stream.KindMotion, State: stream.StateActive}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -219,8 +235,9 @@ func TestDispatchEvent_CtxCancelledAndHandleMotionClosed_DoesNotPanic(t *testing
 	// send would panic. The ctx pre-check must short-circuit before the
 	// send is attempted.
 	cfg := makeConfig("true", "true", "cam-1")
-	comm := &models.Communication{HandleMotion: make(chan models.MotionDataPartial, 1)}
-	close(comm.HandleMotion)
+	comm := makeCommunication(t)
+	run := comm.CurrentRun()
+	run.Shutdown(context.Background(), errors.New("test shutdown"))
 	ev := stream.Event{Kind: stream.KindMotion, State: stream.StateActive}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -299,7 +316,7 @@ func TestDispatchEvent_OnlyRealTransitionsTrigger(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.op.String(), func(t *testing.T) {
 			cfg := makeConfig("true", "true", "cam-1")
-			comm := makeCommunication(1)
+			comm := makeCommunication(t)
 			ev := stream.Event{Kind: stream.KindMotion, State: stream.StateActive, Operation: tt.op}
 
 			ctx, cancel := context.WithCancel(context.Background())
@@ -307,10 +324,10 @@ func TestDispatchEvent_OnlyRealTransitionsTrigger(t *testing.T) {
 			dispatchEvent(ctx, ev, cfg, comm)
 
 			if tt.wantSend {
-				require.Len(t, comm.HandleMotion, 1, "%v must trigger a recording", tt.op)
+				require.Len(t, motionEvents(t, comm), 1, "%v must trigger a recording", tt.op)
 				return
 			}
-			require.Empty(t, comm.HandleMotion, "%v must not trigger a recording", tt.op)
+			require.Empty(t, motionEvents(t, comm), "%v must not trigger a recording", tt.op)
 		})
 	}
 }
@@ -361,8 +378,9 @@ func TestDispatchEvent_LogsTriggerOnlyWhenSent(t *testing.T) {
 	buf := captureDebugLog(t)
 
 	cfg := makeConfig("true", "true", "cam-1")
-	comm := &models.Communication{HandleMotion: make(chan models.MotionDataPartial, 1)}
-	comm.HandleMotion <- models.MotionDataPartial{} // full
+	comm := makeCommunication(t)
+	for comm.TrySendMotion(models.MotionDataPartial{}) {
+	}
 	ev := stream.Event{Kind: stream.KindMotion, State: stream.StateActive, Topic: "tns1:VideoSource/MotionAlarm"}
 
 	ctx, cancel := context.WithCancel(context.Background())

machinery/src/onvif/main.go

@@ -6,12 +6,12 @@ import (
 	"encoding/xml"
 	"errors"
 	"io"
+	"net/http"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/gin-gonic/gin"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/onvif"
 	"github.com/kerberos-io/onvif/device"
@@ -21,6 +21,7 @@ import (
 	"github.com/kerberos-io/onvif/ptz"
 	xsd "github.com/kerberos-io/onvif/xsd"
 	xsdonvif "github.com/kerberos-io/onvif/xsd/onvif"
+	log "github.com/sirupsen/logrus"
 )
 
 // Discover performs an advanced Fing/WiFiman-style scan of the local network
@@ -29,12 +30,12 @@ import (
 // `-action discover` CLI command. Optional subnets (CIDR, e.g.
 // "192.168.1.0/24") override the auto-detected local subnets.
 func Discover(timeout time.Duration, subnets ...string) {
-	log.Log.Info("onvif.Discover(): starting advanced network discovery")
-	log.Log.Info("onvif.Discover(): this may take up to " + timeout.String() + " for the ONVIF probe plus the port scan")
+	log.Info("onvif.Discover(): starting advanced network discovery")
+	log.Info("onvif.Discover(): this may take up to " + timeout.String() + " for the ONVIF probe plus the port scan")
 
 	devices := DiscoverDevices(timeout, subnets...)
 	if len(devices) == 0 {
-		log.Log.Info("onvif.Discover(): no devices discovered on the local network")
+		log.Info("onvif.Discover(): no devices discovered on the local network")
 		return
 	}
 
@@ -44,7 +45,7 @@ func Discover(timeout time.Duration, subnets ...string) {
 			cameraCount++
 		}
 	}
-	log.Log.Info("onvif.Discover(): found " + strconv.Itoa(len(devices)) + " device(s), " + strconv.Itoa(cameraCount) + " likely camera(s)")
+	log.Info("onvif.Discover(): found " + strconv.Itoa(len(devices)) + " device(s), " + strconv.Itoa(cameraCount) + " likely camera(s)")
 
 	for _, device := range devices {
 		label := "device"
@@ -84,7 +85,7 @@ func Discover(timeout time.Duration, subnets ...string) {
 		if device.RTSPURL != "" {
 			summary += " rtsp=" + device.RTSPURL
 		}
-		log.Log.Info(summary)
+		log.Info(summary)
 
 		// Detail the guessed RTSP stream URLs from the brand -> RTSP mapping.
 		for _, stream := range device.RTSPStreams {
@@ -97,15 +98,15 @@ func Discover(timeout time.Duration, subnets ...string) {
 				line += " (auth required)"
 			}
 			line += ": " + stream.URL
-			log.Log.Info(line)
+			log.Info(line)
 		}
 	}
 }
 
-func HandleONVIFActions(configuration *models.Configuration, communication *models.Communication) {
-	log.Log.Debug("onvif.HandleONVIFActions(): started")
+func HandleONVIFActions(configuration *models.Configuration, communication *models.Communication, actions <-chan models.OnvifAction) {
+	log.Debug("onvif.HandleONVIFActions(): started")
 
-	for onvifAction := range communication.HandleONVIF {
+	for onvifAction := range actions {
 
 		// First we'll get the desired PTZ action from the payload
 		// We need to know if we need to move left, right, up, down, zoom in, zoom out, center.
@@ -138,7 +139,7 @@ func HandleONVIFActions(configuration *models.Configuration, communication *mode
 						functions, _, _ := GetPTZFunctionsFromDevice(configurations)
 
 						// Log functions
-						log.Log.Debug("onvif.HandleONVIFActions(): functions: " + strings.Join(functions, ", "))
+						log.Debug("onvif.HandleONVIFActions(): functions: " + strings.Join(functions, ", "))
 
 						// Check if we need to use absolute or continuous move
 						/*canAbsoluteMove := false
@@ -159,20 +160,20 @@ func HandleONVIFActions(configuration *models.Configuration, communication *mode
 						// on the ContinuousPanTiltMove function which is more compatible with more cameras.
 						err = AbsolutePanTiltMoveFake(device, configurations, token, x, y, z)
 						if err != nil {
-							log.Log.Debug("onvif.HandleONVIFActions() - AbsolutePanTitleMoveFake: " + err.Error())
+							log.Debug("onvif.HandleONVIFActions() - AbsolutePanTitleMoveFake: " + err.Error())
 						} else {
-							log.Log.Info("onvif.HandleONVIFActions() - AbsolutePanTitleMoveFake: successfully moved camera.")
+							log.Info("onvif.HandleONVIFActions() - AbsolutePanTitleMoveFake: successfully moved camera.")
 						}
 
 						/*if canAbsoluteMove {
 							err = AbsolutePanTiltMove(device, configurations, token, x, y, z)
 							if err != nil {
-								log.Log.Error("HandleONVIFActions (AbsolutePanTitleMove): " + err.Error())
+								log.Error("HandleONVIFActions (AbsolutePanTitleMove): " + err.Error())
 							}
 						} else if canContinuousMove {
 							err = AbsolutePanTiltMoveFake(device, configurations, token, x, y, z)
 							if err != nil {
-								log.Log.Error("HandleONVIFActions (AbsolutePanTitleMoveFake): " + err.Error())
+								log.Error("HandleONVIFActions (AbsolutePanTitleMoveFake): " + err.Error())
 							}
 						}*/
 
@@ -182,9 +183,9 @@ func HandleONVIFActions(configuration *models.Configuration, communication *mode
 						preset := ptzAction.Preset
 						err := GoToPresetFromDevice(device, preset)
 						if err != nil {
-							log.Log.Debug("onvif.HandleONVIFActions() - GotoPreset: " + err.Error())
+							log.Debug("onvif.HandleONVIFActions() - GotoPreset: " + err.Error())
 						} else {
-							log.Log.Info("onvif.HandleONVIFActions() - GotoPreset: successfully moved camera")
+							log.Info("onvif.HandleONVIFActions() - GotoPreset: successfully moved camera")
 						}
 
 					} else if onvifAction.Action == "ptz" {
@@ -196,9 +197,9 @@ func HandleONVIFActions(configuration *models.Configuration, communication *mode
 								// We will move the camera to zero position.
 								err := AbsolutePanTiltMove(device, configurations, token, 0, 0, 0)
 								if err != nil {
-									log.Log.Debug("onvif.HandleONVIFActions() - AbsolutePanTitleMove: " + err.Error())
+									log.Debug("onvif.HandleONVIFActions() - AbsolutePanTitleMove: " + err.Error())
 								} else {
-									log.Log.Info("onvif.HandleONVIFActions() - AbsolutePanTitleMove: successfully centered camera")
+									log.Info("onvif.HandleONVIFActions() - AbsolutePanTitleMove: successfully centered camera")
 								}
 
 							} else {
@@ -225,9 +226,9 @@ func HandleONVIFActions(configuration *models.Configuration, communication *mode
 
 								err := ContinuousPanTilt(device, configurations, token, x, y)
 								if err != nil {
-									log.Log.Debug("onvif.HandleONVIFActions() - ContinuousPanTilt: " + err.Error())
+									log.Debug("onvif.HandleONVIFActions() - ContinuousPanTilt: " + err.Error())
 								} else {
-									log.Log.Info("onvif.HandleONVIFActions() - ContinuousPanTilt: successfully pan tilted camera")
+									log.Info("onvif.HandleONVIFActions() - ContinuousPanTilt: successfully pan tilted camera")
 								}
 							}
 						}
@@ -237,9 +238,9 @@ func HandleONVIFActions(configuration *models.Configuration, communication *mode
 							zoom := ptzAction.Zoom
 							err := ContinuousZoom(device, configurations, token, zoom)
 							if err != nil {
-								log.Log.Debug("onvif.HandleONVIFActions() - ContinuousZoom: " + err.Error())
+								log.Debug("onvif.HandleONVIFActions() - ContinuousZoom: " + err.Error())
 							} else {
-								log.Log.Info("onvif.HandleONVIFActions() - ContinuousZoom: successfully zoomed camera")
+								log.Info("onvif.HandleONVIFActions() - ContinuousZoom: successfully zoomed camera")
 							}
 						}
 					}
@@ -247,11 +248,11 @@ func HandleONVIFActions(configuration *models.Configuration, communication *mode
 			}
 		}
 	}
-	log.Log.Debug("onvif.HandleONVIFActions(): finished")
+	log.Debug("onvif.HandleONVIFActions(): finished")
 }
 
 func ConnectToOnvifDevice(cameraConfiguration *models.IPCamera) (*onvif.Device, device.GetCapabilitiesResponse, error) {
-	log.Log.Debug("onvif.ConnectToOnvifDevice(): started")
+	log.Debug("onvif.ConnectToOnvifDevice(): started")
 	dev, err := onvif.NewDevice(onvif.DeviceParams{
 		Xaddr:    cameraConfiguration.ONVIFXAddr,
 		Username: cameraConfiguration.ONVIFUsername,
@@ -269,7 +270,7 @@ func ConnectToOnvifDevice(cameraConfiguration *models.IPCamera) (*onvif.Device,
 			AuthMode: "digest",
 		})
 		if err != nil {
-			log.Log.Debug("onvif.ConnectToOnvifDevice(): " + err.Error())
+			log.Debug("onvif.ConnectToOnvifDevice(): " + err.Error())
 		}
 	}
 
@@ -277,7 +278,7 @@ func ConnectToOnvifDevice(cameraConfiguration *models.IPCamera) (*onvif.Device,
 		getCapabilities := device.GetCapabilities{Category: []xsdonvif.CapabilityCategory{"All"}}
 		resp, err := dev.CallMethod(getCapabilities)
 		if err != nil {
-			log.Log.Error("onvif.ConnectToOnvifDevice(): " + err.Error())
+			log.Error("onvif.ConnectToOnvifDevice(): " + err.Error())
 		}
 
 		var b []byte
@@ -285,24 +286,24 @@ func ConnectToOnvifDevice(cameraConfiguration *models.IPCamera) (*onvif.Device,
 			b, err = io.ReadAll(resp.Body)
 			resp.Body.Close() // Ensure the response body is closed
 			if err != nil {
-				log.Log.Error("onvif.ConnectToOnvifDevice(): " + err.Error())
+				log.Error("onvif.ConnectToOnvifDevice(): " + err.Error())
 			}
 		}
 		stringBody := string(b)
 		decodedXML, et, err := getXMLNode(stringBody, "GetCapabilitiesResponse")
 		if err != nil {
-			log.Log.Error("onvif.ConnectToOnvifDevice(): " + err.Error())
+			log.Error("onvif.ConnectToOnvifDevice(): " + err.Error())
 		} else {
 			if err := decodedXML.DecodeElement(&capabilities, et); err != nil {
-				log.Log.Error("onvif.ConnectToOnvifDevice(): " + err.Error())
+				log.Error("onvif.ConnectToOnvifDevice(): " + err.Error())
 			} else {
-				log.Log.Debug("onvif.ConnectToOnvifDevice(): capabilities.")
+				log.Debug("onvif.ConnectToOnvifDevice(): capabilities.")
 			}
 		}
 
-		log.Log.Info("onvif.ConnectToOnvifDevice(): successfully connected to device")
+		log.Info("onvif.ConnectToOnvifDevice(): successfully connected to device")
 	}
-	log.Log.Debug("onvif.ConnectToOnvifDevice(): finished")
+	log.Debug("onvif.ConnectToOnvifDevice(): finished")
 	return dev, capabilities, err
 }
 
@@ -319,13 +320,13 @@ func GetTokenFromProfile(device *onvif.Device, profileId int) (xsdonvif.Referenc
 			resp.Body.Close() // Ensure the response body is closed
 			decodedXML, et, err := getXMLNode(stringBody, "GetProfilesResponse")
 			if err != nil {
-				log.Log.Debug("onvif.GetTokenFromProfile(): " + err.Error())
+				log.Debug("onvif.GetTokenFromProfile(): " + err.Error())
 				return profileToken, err
 			} else {
 				// Decode the profiles from the server
 				var mProfilesResp media.GetProfilesResponse
 				if err := decodedXML.DecodeElement(&mProfilesResp, et); err != nil {
-					log.Log.Debug("onvif.GetTokenFromProfile(): " + err.Error())
+					log.Debug("onvif.GetTokenFromProfile(): " + err.Error())
 				}
 
 				// We'll try to get the token from a preferred profile
@@ -356,11 +357,11 @@ func GetPTZConfigurationsFromDevice(device *onvif.Device) (ptz.GetConfigurations
 		stringBody := string(b)
 		decodedXML, et, err := getXMLNode(stringBody, "GetConfigurationsResponse")
 		if err != nil {
-			log.Log.Debug("onvif.GetPTZConfigurationsFromDevice(): " + err.Error())
+			log.Debug("onvif.GetPTZConfigurationsFromDevice(): " + err.Error())
 			return configurations, err
 		} else {
 			if err := decodedXML.DecodeElement(&configurations, et); err != nil {
-				log.Log.Debug("onvif.GetPTZConfigurationsFromDevice(): " + err.Error())
+				log.Debug("onvif.GetPTZConfigurationsFromDevice(): " + err.Error())
 				return configurations, err
 			}
 		}
@@ -386,22 +387,22 @@ func GetPositionFromDevice(configuration models.Configuration) (xsdonvif.PTZVect
 					x := strconv.FormatFloat(position.PanTilt.X, 'f', 6, 64)
 					y := strconv.FormatFloat(position.PanTilt.Y, 'f', 6, 64)
 					z := strconv.FormatFloat(position.Zoom.X, 'f', 6, 64)
-					log.Log.Info("onvif.GetPositionFromDevice(): successfully got position (" + x + ", " + y + ", " + z + ")")
+					log.Info("onvif.GetPositionFromDevice(): successfully got position (" + x + ", " + y + ", " + z + ")")
 					return position, err
 				} else {
-					log.Log.Debug("onvif.GetPositionFromDevice(): position is nil")
+					log.Debug("onvif.GetPositionFromDevice(): position is nil")
 					return position, errors.New("position is nil")
 				}
 			} else {
-				log.Log.Debug("onvif.GetPositionFromDevice(): " + err.Error())
+				log.Debug("onvif.GetPositionFromDevice(): " + err.Error())
 				return position, err
 			}
 		} else {
-			log.Log.Debug("onvif.GetPositionFromDevice(): " + err.Error())
+			log.Debug("onvif.GetPositionFromDevice(): " + err.Error())
 			return position, err
 		}
 	} else {
-		log.Log.Debug("onvif.GetPositionFromDevice(): " + err.Error())
+		log.Debug("onvif.GetPositionFromDevice(): " + err.Error())
 		return position, err
 	}
 }
@@ -426,11 +427,11 @@ func GetPosition(device *onvif.Device, token xsdonvif.ReferenceToken) (xsdonvif.
 		stringBody := string(b)
 		decodedXML, et, err := getXMLNode(stringBody, "GetStatusResponse")
 		if err != nil {
-			log.Log.Error("GetPositionFromDevice: " + err.Error())
+			log.Error("GetPositionFromDevice: " + err.Error())
 			return position, err
 		} else {
 			if err := decodedXML.DecodeElement(&status, et); err != nil {
-				log.Log.Error("GetPositionFromDevice: " + err.Error())
+				log.Error("GetPositionFromDevice: " + err.Error())
 				return position, err
 			}
 		}
@@ -439,6 +440,36 @@ func GetPosition(device *onvif.Device, token xsdonvif.ReferenceToken) (xsdonvif.
 	return position, err
 }
 
+func handleONVIFResponse(operation string, response *http.Response, requestErr error) error {
+	fields := log.Fields{
+		"component": "onvif",
+		"operation": operation,
+	}
+	var responseErr error
+	if response != nil {
+		fields["status_code"] = response.StatusCode
+		fields["response_bytes"], responseErr = io.Copy(io.Discard, response.Body)
+		response.Body.Close()
+	}
+
+	err := errors.Join(requestErr, responseErr)
+	if response == nil && err == nil {
+		err = errors.New("ONVIF operation returned no response")
+	}
+	if err == nil && (response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices) {
+		err = errors.New("ONVIF operation was rejected")
+	}
+	if err != nil {
+		fields["event"] = "operation_failed"
+		log.WithError(err).WithFields(fields).Error("ONVIF operation failed")
+		return err
+	}
+
+	fields["event"] = "operation_completed"
+	log.WithFields(fields).Debug("ONVIF operation completed")
+	return nil
+}
+
 func AbsolutePanTiltMove(device *onvif.Device, configuration ptz.GetConfigurationsResponse, token xsdonvif.ReferenceToken, pan float64, tilt float64, zoom float64) error {
 
 	absolutePantiltVector := xsdonvif.Vector2D{
@@ -459,18 +490,7 @@ func AbsolutePanTiltMove(device *onvif.Device, configuration ptz.GetConfiguratio
 			Zoom:    &absoluteZoomVector,
 		},
 	})
-
-	var b []byte
-	if resp != nil {
-		b, err = io.ReadAll(resp.Body)
-		resp.Body.Close()
-	}
-	if err != nil {
-		log.Log.Error("AbsoluteMove: " + err.Error())
-	}
-	log.Log.Info("AbsoluteMove: " + string(b))
-
-	return err
+	return handleONVIFResponse("absolute_move", resp, err)
 }
 
 // This function will simulate the AbsolutePanTiltMove function.
@@ -479,7 +499,7 @@ func AbsolutePanTiltMove(device *onvif.Device, configuration ptz.GetConfiguratio
 func AbsolutePanTiltMoveFake(device *onvif.Device, configuration ptz.GetConfigurationsResponse, token xsdonvif.ReferenceToken, pan float64, tilt float64, zoom float64) error {
 	position, err := GetPosition(device, token)
 	if position.PanTilt.X >= pan-0.01 && position.PanTilt.X <= pan+0.01 && position.PanTilt.Y >= tilt-0.01 && position.PanTilt.Y <= tilt+0.01 && position.Zoom.X >= zoom-0.01 && position.Zoom.X <= zoom+0.01 {
-		log.Log.Debug("AbsolutePanTiltMoveFake: already at position")
+		log.Debug("AbsolutePanTiltMoveFake: already at position")
 	} else {
 
 		// The speed of panning, the higher the faster we'll pan the camera
@@ -518,7 +538,7 @@ func ZoomOutCompletely(device *onvif.Device, configuration ptz.GetConfigurations
 		},
 	})
 	if err != nil {
-		log.Log.Error("ZoomOutCompletely: " + err.Error())
+		log.Error("ZoomOutCompletely: " + err.Error())
 	}
 
 	for {
@@ -534,7 +554,7 @@ func ZoomOutCompletely(device *onvif.Device, configuration ptz.GetConfigurations
 		Zoom:         true,
 	})
 	if err != nil {
-		log.Log.Error("ZoomOutCompletely: " + err.Error())
+		log.Error("ZoomOutCompletely: " + err.Error())
 	}
 	return err
 }
@@ -564,16 +584,7 @@ func PanUntilPosition(device *onvif.Device, configuration ptz.GetConfigurationsR
 				PanTilt: panTiltVector,
 			},
 		})
-
-		var b []byte
-		if resp != nil {
-			b, err = io.ReadAll(resp.Body)
-			resp.Body.Close()
-		}
-		if err != nil {
-			log.Log.Error("ContinuousPanTiltMove (Pan): " + err.Error())
-		}
-		log.Log.Debug("ContinuousPanTiltMove (Pan): " + string(b))
+		err = handleONVIFResponse("continuous_pan", resp, err)
 
 		// While moving we'll check if we reached the desired position.
 		// or if we overshot the desired position.
@@ -598,7 +609,7 @@ func PanUntilPosition(device *onvif.Device, configuration ptz.GetConfigurationsR
 		})
 
 		if err != nil {
-			log.Log.Error("ContinuousPanTiltMove (Pan): " + err.Error())
+			log.Error("ContinuousPanTiltMove (Pan): " + err.Error())
 		}
 	}
 	return err
@@ -632,17 +643,7 @@ func TiltUntilPosition(device *onvif.Device, configuration ptz.GetConfigurations
 			ProfileToken: &token,
 			Velocity:     velocity,
 		})
-
-		var b []byte
-		if resp != nil {
-			b, err = io.ReadAll(resp.Body)
-			resp.Body.Close()
-		}
-
-		if err != nil {
-			log.Log.Error("ContinuousPanTiltMove (Tilt): " + err.Error())
-		}
-		log.Log.Debug("ContinuousPanTiltMove (Tilt) " + string(b))
+		err = handleONVIFResponse("continuous_tilt", resp, err)
 
 		// While moving we'll check if we reached the desired position.
 		// or if we overshot the desired position.
@@ -667,7 +668,7 @@ func TiltUntilPosition(device *onvif.Device, configuration ptz.GetConfigurations
 		})
 
 		if err != nil {
-			log.Log.Error("ContinuousPanTiltMove (Tilt): " + err.Error())
+			log.Error("ContinuousPanTiltMove (Tilt): " + err.Error())
 		}
 	}
 	return err
@@ -697,17 +698,7 @@ func ZoomUntilPosition(device *onvif.Device, configuration ptz.GetConfigurations
 				Zoom: zoomVector,
 			},
 		})
-
-		var b []byte
-		if resp != nil {
-			b, err = io.ReadAll(resp.Body)
-			resp.Body.Close()
-		}
-		if err != nil {
-			log.Log.Error("ContinuousPanTiltMove (Zoom): " + err.Error())
-		}
-
-		log.Log.Debug("ContinuousPanTiltMove (Zoom) " + string(b))
+		err = handleONVIFResponse("continuous_zoom_to_position", resp, err)
 
 		// While moving we'll check if we reached the desired position.
 		// or if we overshot the desired position.
@@ -732,7 +723,7 @@ func ZoomUntilPosition(device *onvif.Device, configuration ptz.GetConfigurations
 		})
 
 		if err != nil {
-			log.Log.Error("ContinuousPanTiltMove (Zoom): " + err.Error())
+			log.Error("ContinuousPanTiltMove (Zoom): " + err.Error())
 		}
 	}
 	return err
@@ -752,17 +743,7 @@ func ContinuousPanTilt(device *onvif.Device, configuration ptz.GetConfigurations
 			PanTilt: panTiltVector,
 		},
 	})
-
-	var b []byte
-	if resp != nil {
-		b, err = io.ReadAll(resp.Body)
-		resp.Body.Close()
-	}
-	if err != nil {
-		log.Log.Error("ContinuousPanTiltMove: " + err.Error())
-	}
-
-	log.Log.Debug("ContinuousPanTiltMove: " + string(b))
+	moveErr := handleONVIFResponse("continuous_pan_tilt", resp, err)
 
 	time.Sleep(200 * time.Millisecond)
 
@@ -770,18 +751,8 @@ func ContinuousPanTilt(device *onvif.Device, configuration ptz.GetConfigurations
 		ProfileToken: token,
 		PanTilt:      true,
 	})
-
-	b = []byte{}
-	if resp != nil {
-		b, err = io.ReadAll(resp.Body)
-		resp.Body.Close()
-	}
-
-	if err != nil {
-		log.Log.Error("ContinuousPanTiltMove: " + err.Error())
-	}
-
-	return err
+	stopErr := handleONVIFResponse("continuous_pan_tilt_stop", resp, err)
+	return errors.Join(moveErr, stopErr)
 }
 
 func ContinuousZoom(device *onvif.Device, configuration ptz.GetConfigurationsResponse, token xsdonvif.ReferenceToken, zoom float64) error {
@@ -799,36 +770,26 @@ func ContinuousZoom(device *onvif.Device, configuration ptz.GetConfigurationsRes
 		ProfileToken: &token,
 		Velocity:     &velocity,
 	})
-
-	var b []byte
-	if resp != nil {
-		b, err = io.ReadAll(resp.Body)
-		resp.Body.Close()
-	}
-	if err != nil {
-		log.Log.Error("onvif.main.ContinuousZoom(): " + err.Error())
-	}
-
-	log.Log.Debug("onvif.main.ContinuousZoom(): " + string(b))
+	moveErr := handleONVIFResponse("continuous_zoom", resp, err)
 	time.Sleep(500 * time.Millisecond)
 
-	_, err = device.CallMethod(ptz.Stop{
+	resp, err = device.CallMethod(ptz.Stop{
 		ProfileToken: token,
 		Zoom:         true,
 	})
-
-	if err != nil {
-		log.Log.Error("onvif.main.ContinuousZoom(): " + err.Error())
-	}
-
-	return err
+	stopErr := handleONVIFResponse("continuous_zoom_stop", resp, err)
+	return errors.Join(moveErr, stopErr)
 }
 
 func GetCapabilitiesFromDevice(dev *onvif.Device) []string {
 	var capabilities []string
 	services := dev.GetServices()
 	for key, _ := range services {
-		log.Log.Debug("onvif.main.GetCapabilitiesFromDevice(): has key: " + key)
+		log.WithFields(log.Fields{
+			"capability": key,
+			"component":  "onvif",
+			"event":      "capability_detected",
+		}).Debug("ONVIF capability detected")
 		if key != "" {
 			keyParts := strings.Split(key, "/")
 			if len(keyParts) > 0 {
@@ -859,11 +820,11 @@ func GetPresetsFromDevice(device *onvif.Device) ([]models.OnvifActionPreset, err
 			stringBody := string(b)
 			decodedXML, et, err := getXMLNode(stringBody, "GetPresetsResponse")
 			if err != nil {
-				log.Log.Error("onvif.main.GetPresetsFromDevice(): " + err.Error())
+				log.Error("onvif.main.GetPresetsFromDevice(): " + err.Error())
 				return presets, err
 			} else {
 				if err := decodedXML.DecodeElement(&presetsResponse, et); err != nil {
-					log.Log.Error("onvif.main.GetPresetsFromDevice(): " + err.Error())
+					log.Error("onvif.main.GetPresetsFromDevice(): " + err.Error())
 					return presets, err
 				}
 
@@ -876,15 +837,15 @@ func GetPresetsFromDevice(device *onvif.Device) ([]models.OnvifActionPreset, err
 					presetsList += string(preset.Name) + " (" + string(preset.Token) + "), "
 					presets = append(presets, p)
 				}
-				log.Log.Debug("onvif.main.GetPresetsFromDevice(): " + presetsList)
+				log.Debug("onvif.main.GetPresetsFromDevice(): " + presetsList)
 
 				return presets, err
 			}
 		} else {
-			log.Log.Error("onvif.main.GetPresetsFromDevice(): " + err.Error())
+			log.Error("onvif.main.GetPresetsFromDevice(): " + err.Error())
 		}
 	} else {
-		log.Log.Error("onvif.main.GetPresetsFromDevice(): " + err.Error())
+		log.Error("onvif.main.GetPresetsFromDevice(): " + err.Error())
 	}
 
 	return presets, err
@@ -910,20 +871,20 @@ func GoToPresetFromDevice(device *onvif.Device, presetName string) error {
 			stringBody := string(b)
 			decodedXML, et, err := getXMLNode(stringBody, "GotoPresetResponses")
 			if err != nil {
-				log.Log.Error("onvif.main.GoToPresetFromDevice(): " + err.Error())
+				log.Error("onvif.main.GoToPresetFromDevice(): " + err.Error())
 				return err
 			} else {
 				if err := decodedXML.DecodeElement(&goToPresetResponse, et); err != nil {
-					log.Log.Error("onvif.main.GoToPresetFromDevice(): " + err.Error())
+					log.Error("onvif.main.GoToPresetFromDevice(): " + err.Error())
 					return err
 				}
 				return err
 			}
 		} else {
-			log.Log.Error("onvif.main.GoToPresetFromDevice(): " + err.Error())
+			log.Error("onvif.main.GoToPresetFromDevice(): " + err.Error())
 		}
 	} else {
-		log.Log.Error("onvif.main.GoToPresetFromDevice(): " + err.Error())
+		log.Error("onvif.main.GoToPresetFromDevice(): " + err.Error())
 	}
 
 	return err
@@ -1068,10 +1029,10 @@ func CreatePullPointSubscription(dev *onvif.Device) (string, error) {
 			stringBody := string(b2)
 			decodedXML, et, err := getXMLNode(stringBody, "CreatePullPointSubscriptionResponse")
 			if err != nil {
-				log.Log.Debug("onvif.main.CreatePullPointSubscription(): " + err.Error())
+				log.Debug("onvif.main.CreatePullPointSubscription(): " + err.Error())
 			} else {
 				if err := decodedXML.DecodeElement(&createPullPointSubscriptionResponse, et); err != nil {
-					log.Log.Error("onvif.main.CreatePullPointSubscription(): " + err.Error())
+					log.Error("onvif.main.CreatePullPointSubscription(): " + err.Error())
 				} else {
 					pullPointAdress = string(createPullPointSubscriptionResponse.SubscriptionReference.Address)
 				}
@@ -1087,25 +1048,40 @@ func UnsubscribePullPoint(dev *onvif.Device, pullPointAddress string) error {
 	unsubscribe := event.Unsubscribe{}
 	requestBody, err := xml.Marshal(unsubscribe)
 	if err != nil {
-		log.Log.Error("onvif.main.UnsubscribePullPoint(): " + err.Error())
+		log.WithError(err).WithFields(log.Fields{
+			"component": "onvif",
+			"event":     "unsubscribe_request_encoding_failed",
+		}).Error("Failed to encode ONVIF unsubscribe request")
+		return err
 	}
 
 	res, err := dev.SendSoap(pullPointAddress, string(requestBody))
 	if err != nil {
-		log.Log.Error("onvif.main.UnsubscribePullPoint(): " + err.Error())
+		log.WithError(err).WithFields(log.Fields{
+			"component": "onvif",
+			"event":     "unsubscribe_request_failed",
+		}).Error("Failed to send ONVIF unsubscribe request")
+		return err
 	}
-	if res != nil {
-		b, err := io.ReadAll(res.Body)
-		res.Body.Close() // Ensure the response body is closed
-		if err == nil {
-			stringBody := string(b)
-			log.Log.Debug("onvif.main.UnsubscribePullPoint(): " + stringBody)
-		}
-		if err != nil {
-			log.Log.Error("onvif.main.UnsubscribePullPoint(): " + err.Error())
-		}
+	if res == nil {
+		return errors.New("ONVIF unsubscribe returned no response")
 	}
-	return err
+	defer res.Body.Close()
+	responseBytes, err := io.Copy(io.Discard, res.Body)
+	if err != nil {
+		log.WithError(err).WithFields(log.Fields{
+			"component": "onvif",
+			"event":     "unsubscribe_response_read_failed",
+		}).Error("Failed to read ONVIF unsubscribe response")
+		return err
+	}
+	log.WithFields(log.Fields{
+		"component":      "onvif",
+		"event":          "unsubscribe_completed",
+		"response_bytes": responseBytes,
+		"status_code":    res.StatusCode,
+	}).Debug("ONVIF unsubscribe completed")
+	return nil
 }
 
 // Look for Source of input and output
@@ -1126,7 +1102,13 @@ func GetInputOutputs() ([]ONVIFEvents, error) {
 		eventsArray = append(eventsArray, *value)
 	}
 	for _, value := range eventsArray {
-		log.Log.Debug("onvif.main.GetInputOutputs(): " + value.Key + " - " + value.Value + " (" + strconv.FormatInt(value.Timestamp, 10) + ")")
+		log.WithFields(log.Fields{
+			"component":      "onvif",
+			"event":          "input_output_state",
+			"input_output":   value.Key,
+			"state":          value.Value,
+			"timestamp_unix": value.Timestamp,
+		}).Debug("ONVIF input/output state")
 	}
 	return eventsArray, nil
 }
@@ -1143,7 +1125,7 @@ func GetEventMessages(dev *onvif.Device, pullPointAddress string) ([]ONVIFEvents
 		// We were able to create a subscription to the device. Now pull some messages from the subscription.
 		subscriptionURI := pullPointAddress
 		if subscriptionURI == "" {
-			log.Log.Error("onvif.main.GetEventMessages(): subscriptionURI is empty")
+			log.Error("onvif.main.GetEventMessages(): subscriptionURI is empty")
 		} else {
 			// Pull message
 			pullMessage := event.PullMessages{
@@ -1152,12 +1134,12 @@ func GetEventMessages(dev *onvif.Device, pullPointAddress string) ([]ONVIFEvents
 			}
 			requestBody, err := xml.Marshal(pullMessage)
 			if err != nil {
-				log.Log.Error("onvif.main.GetEventMessages(pullMessages): " + err.Error())
+				log.Error("onvif.main.GetEventMessages(pullMessages): " + err.Error())
 				return eventsArray, err
 			}
 			res, err := dev.SendSoap(string(subscriptionURI), string(requestBody))
 			if err != nil {
-				log.Log.Error("onvif.main.GetEventMessages(pullMessages): " + err.Error())
+				log.Error("onvif.main.GetEventMessages(pullMessages): " + err.Error())
 				return eventsArray, err
 			}
 
@@ -1169,11 +1151,11 @@ func GetEventMessages(dev *onvif.Device, pullPointAddress string) ([]ONVIFEvents
 					stringBody := string(bs)
 					decodedXML, et, err := getXMLNode(stringBody, "PullMessagesResponse")
 					if err != nil {
-						log.Log.Error("onvif.main.GetEventMessages(pullMessages): " + err.Error())
+						log.Error("onvif.main.GetEventMessages(pullMessages): " + err.Error())
 						return eventsArray, err
 					} else {
 						if err := decodedXML.DecodeElement(&pullMessagesResponse, et); err != nil {
-							log.Log.Error("onvif.main.GetEventMessages(pullMessages): " + err.Error())
+							log.Error("onvif.main.GetEventMessages(pullMessages): " + err.Error())
 							return eventsArray, err
 						}
 					}
@@ -1181,9 +1163,9 @@ func GetEventMessages(dev *onvif.Device, pullPointAddress string) ([]ONVIFEvents
 			}
 
 			for _, message := range pullMessagesResponse.NotificationMessage {
-				log.Log.Debug("onvif.main.GetEventMessages(pullMessages): " + string(message.Topic.TopicKinds))
+				log.Debug("onvif.main.GetEventMessages(pullMessages): " + string(message.Topic.TopicKinds))
 				//if len(message.Message.Message.Data.SimpleItem) > 0 {
-				//	log.Log.Debug("onvif.main.GetEventMessages(pullMessages): " + string(message.Message.Message.Data.SimpleItem[0].Name) + " " + string(message.Message.Message.Data.SimpleItem[0].Value))
+				// log.Debug("onvif.main.GetEventMessages(pullMessages): " + string(message.Message.Message.Data.SimpleItem[0].Name) + " " + string(message.Message.Message.Data.SimpleItem[0].Value))
 				//}
 				if message.Topic.TopicKinds == "tns1:Device/Trigger/Relay" ||
 					message.Topic.TopicKinds == "tns1:Device/tns1:Trigger/tns1:Relay" { // This is for avigilon cameras
@@ -1193,7 +1175,13 @@ func GetEventMessages(dev *onvif.Device, pullPointAddress string) ([]ONVIFEvents
 							key := string(message.Message.Message.Source.SimpleItem[0].Value)
 							value := string(message.Message.Message.Data.SimpleItem[0].Value)
 							propertyOperation := string(message.Message.Message.PropertyOperation)
-							log.Log.Debug("onvif.main.GetEventMessages(pullMessages) output: " + key + " " + value + " (" + propertyOperation + ")")
+							log.WithFields(log.Fields{
+								"component":          "onvif",
+								"event":              "event_property_output",
+								"property":           key,
+								"property_operation": propertyOperation,
+								"value":              value,
+							}).Debug("ONVIF event property output")
 
 							// Depending on the onvif library they might use different values for active and inactive.
 							if value == "active" || value == "1" {
@@ -1227,7 +1215,13 @@ func GetEventMessages(dev *onvif.Device, pullPointAddress string) ([]ONVIFEvents
 							key := string(message.Message.Message.Source.SimpleItem[0].Value)
 							value := string(message.Message.Message.Data.SimpleItem[0].Value)
 							propertyOperation := string(message.Message.Message.PropertyOperation)
-							log.Log.Debug("onvif.main.GetEventMessages(pullMessages) input: " + key + " " + value + " (" + propertyOperation + ")")
+							log.WithFields(log.Fields{
+								"component":          "onvif",
+								"event":              "event_property_input",
+								"property":           key,
+								"property_operation": propertyOperation,
+								"value":              value,
+							}).Debug("ONVIF event property input")
 
 							// Depending on the onvif library they might use different values for active and inactive.
 							if value == "active" || value == "1" {
@@ -1281,11 +1275,11 @@ func GetDigitalInputs(dev *onvif.Device) (device.GetDigitalInputsResponse, error
 			stringBody := string(b)
 			decodedXML, et, err := getXMLNode(stringBody, "GetDigitalInputsResponse")
 			if err != nil {
-				log.Log.Error("onvif.main.GetDigitalInputs(): " + err.Error())
+				log.Error("onvif.main.GetDigitalInputs(): " + err.Error())
 				return digitalinputs, err
 			} else {
 				if err := decodedXML.DecodeElement(&digitalinputs, et); err != nil {
-					log.Log.Debug("onvif.main.GetDigitalInputs(): " + err.Error())
+					log.Debug("onvif.main.GetDigitalInputs(): " + err.Error())
 					return digitalinputs, err
 				}
 			}
@@ -1312,11 +1306,11 @@ func GetRelayOutputs(dev *onvif.Device) (device.GetRelayOutputsResponse, error)
 		stringBody := string(b)
 		decodedXML, et, err := getXMLNode(stringBody, "GetRelayOutputsResponse")
 		if err != nil {
-			log.Log.Error("onvif.main.GetRelayOutputs(): " + err.Error())
+			log.Error("onvif.main.GetRelayOutputs(): " + err.Error())
 			return relayoutputs, err
 		} else {
 			if err := decodedXML.DecodeElement(&relayoutputs, et); err != nil {
-				log.Log.Debug("onvif.main.GetRelayOutputs(): " + err.Error())
+				log.Debug("onvif.main.GetRelayOutputs(): " + err.Error())
 				return relayoutputs, err
 			}
 		}
@@ -1325,42 +1319,79 @@ func GetRelayOutputs(dev *onvif.Device) (device.GetRelayOutputsResponse, error)
 	return relayoutputs, err
 }
 
-func TriggerRelayOutput(dev *onvif.Device, output string) (err error) {
-	err = nil
-
+func TriggerRelayOutput(dev *onvif.Device, output string) error {
 	// Get all outputs
-	relayoutputs, err := GetRelayOutputs(dev)
-
-	// For the moment we expect a single output
-	// However in theory there might be multiple outputs. We might need to change
-	// this in the future "kerberos-io/onvif" library.
-	if err == nil {
-		token := relayoutputs.RelayOutputs[0].Token
-		if output == string(token+"-output") {
-			outputState := device.SetRelayOutputState{
-				RelayOutputToken: token,
-				LogicalState:     "active",
-			}
-
-			resp, errResp := dev.CallMethod(outputState)
-			var b []byte
-			if errResp != nil {
-				b, err = io.ReadAll(resp.Body)
-				resp.Body.Close() // Ensure the response body is closed
-			}
-			stringBody := string(b)
-			if err == nil && resp.StatusCode == 200 {
-				log.Log.Info("onvif.main.TriggerRelayOutput(): triggered relay output (" + string(token) + ")")
-			} else {
-				log.Log.Error("onvif.main.TriggerRelayOutput(): " + stringBody)
-			}
-		} else {
-			log.Log.Error("onvif.main.TriggerRelayOutput(): could not find relay output (" + output + ")")
-		}
-	} else {
-		log.Log.Error("onvif.main.TriggerRelayOutput(): something went wrong while getting the relay outputs " + err.Error())
+	relayOutputs, err := GetRelayOutputs(dev)
+	if err != nil {
+		log.WithError(err).WithFields(log.Fields{
+			"component": "onvif",
+			"event":     "relay_outputs_query_failed",
+		}).Error("Failed to query ONVIF relay outputs")
+		return err
 	}
-	return
+	if len(relayOutputs.RelayOutputs) == 0 {
+		err := errors.New("ONVIF device has no relay outputs")
+		log.WithError(err).WithFields(log.Fields{
+			"component": "onvif",
+			"event":     "relay_output_unavailable",
+		}).Error("ONVIF relay output unavailable")
+		return err
+	}
+
+	// For the moment we expect a single output. Supporting multiple outputs
+	// requires matching the requested output against every returned token.
+	token := relayOutputs.RelayOutputs[0].Token
+	if output != string(token+"-output") {
+		err := errors.New("requested ONVIF relay output was not found")
+		log.WithError(err).WithFields(log.Fields{
+			"component": "onvif",
+			"event":     "relay_output_unavailable",
+		}).Error("ONVIF relay output unavailable")
+		return err
+	}
+
+	resp, err := dev.CallMethod(device.SetRelayOutputState{
+		RelayOutputToken: token,
+		LogicalState:     "active",
+	})
+	if err != nil {
+		log.WithError(err).WithFields(log.Fields{
+			"component": "onvif",
+			"event":     "relay_output_request_failed",
+		}).Error("Failed to trigger ONVIF relay output")
+		return err
+	}
+	if resp == nil {
+		err := errors.New("ONVIF relay request returned no response")
+		log.WithError(err).WithFields(log.Fields{
+			"component": "onvif",
+			"event":     "relay_output_request_failed",
+		}).Error("Failed to trigger ONVIF relay output")
+		return err
+	}
+	defer resp.Body.Close()
+	if _, err := io.Copy(io.Discard, resp.Body); err != nil {
+		log.WithError(err).WithFields(log.Fields{
+			"component": "onvif",
+			"event":     "relay_output_response_read_failed",
+		}).Warn("Failed to read ONVIF relay response")
+		return err
+	}
+	if resp.StatusCode != 200 {
+		err := errors.New("ONVIF relay request was rejected")
+		log.WithError(err).WithFields(log.Fields{
+			"component":   "onvif",
+			"event":       "relay_output_rejected",
+			"status_code": resp.StatusCode,
+		}).Error("ONVIF relay request rejected")
+		return err
+	}
+
+	log.WithFields(log.Fields{
+		"component": "onvif",
+		"event":     "relay_output_triggered",
+	}).Info("ONVIF relay output triggered")
+	return nil
 }
 
 func getXMLNode(xmlBody string, nodeName string) (*xml.Decoder, *xml.StartElement, error) {

machinery/src/onvif/main_test.go

@@ -0,0 +1,42 @@
+package onvif
+
+import (
+	"bytes"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func TestHandleONVIFResponseDoesNotLogBody(t *testing.T) {
+	logger := log.StandardLogger()
+	originalOutput := logger.Out
+	originalFormatter := logger.Formatter
+	originalLevel := logger.Level
+	defer func() {
+		logger.SetOutput(originalOutput)
+		logger.SetFormatter(originalFormatter)
+		logger.SetLevel(originalLevel)
+	}()
+
+	var output bytes.Buffer
+	logger.SetOutput(&output)
+	logger.SetFormatter(&log.JSONFormatter{})
+	logger.SetLevel(log.DebugLevel)
+
+	response := &http.Response{
+		StatusCode: http.StatusOK,
+		Body:       io.NopCloser(strings.NewReader(`{"credential":"do-not-log"}`)),
+	}
+	if err := handleONVIFResponse("test_operation", response, nil); err != nil {
+		t.Fatalf("handleONVIFResponse() error = %v", err)
+	}
+	if strings.Contains(output.String(), "do-not-log") {
+		t.Fatalf("ONVIF response log exposed response body: %q", output.String())
+	}
+	if !strings.Contains(output.String(), `"response_bytes":27`) {
+		t.Fatalf("ONVIF response log omitted response byte count: %q", output.String())
+	}
+}

machinery/src/outputs/main.go

@@ -1,8 +1,8 @@
 package outputs
 
 import (
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 type Output interface {
@@ -20,36 +20,36 @@ func Execute(message *models.OutputMessage) (err error) {
 			slack := &SlackOutput{}
 			err := slack.Trigger(message)
 			if err == nil {
-				log.Log.Debug("outputs.main.Execute(slack): message was processed by output.")
+				log.Debug("outputs.main.Execute(slack): message was processed by output.")
 			} else {
-				log.Log.Error("outputs.main.Execute(slack): " + err.Error())
+				log.Error("outputs.main.Execute(slack): " + err.Error())
 			}
 			break
 		case "webhook":
 			webhook := &WebhookOutput{}
 			err := webhook.Trigger(message)
 			if err == nil {
-				log.Log.Debug("outputs.main.Execute(webhook): message was processed by output.")
+				log.Debug("outputs.main.Execute(webhook): message was processed by output.")
 			} else {
-				log.Log.Error("outputs.main.Execute(webhook): " + err.Error())
+				log.Error("outputs.main.Execute(webhook): " + err.Error())
 			}
 			break
 		case "onvif_relay":
 			onvif := &OnvifRelayOutput{}
 			err := onvif.Trigger(message)
 			if err == nil {
-				log.Log.Debug("outputs.main.Execute(onvif): message was processed by output.")
+				log.Debug("outputs.main.Execute(onvif): message was processed by output.")
 			} else {
-				log.Log.Error("outputs.main.Execute(onvif): " + err.Error())
+				log.Error("outputs.main.Execute(onvif): " + err.Error())
 			}
 			break
 		case "script":
 			script := &ScriptOutput{}
 			err := script.Trigger(message)
 			if err == nil {
-				log.Log.Debug("outputs.main.Execute(script): message was processed by output.")
+				log.Debug("outputs.main.Execute(script): message was processed by output.")
 			} else {
-				log.Log.Error("outputs.main.Execute(script): " + err.Error())
+				log.Error("outputs.main.Execute(script): " + err.Error())
 			}
 			break
 		}

machinery/src/packets/queue.go

@@ -2,6 +2,7 @@
 package packets
 
 import (
+	"context"
 	"io"
 	"sync"
 )
@@ -201,7 +202,26 @@ func (self *QueueCursor) Streams() (streams []Stream, err error) {
 
 // ReadPacket will not consume packets in Queue, it's just a cursor.
 func (self *QueueCursor) ReadPacket() (pkt Packet, err error) {
+	return self.readPacket(context.Background(), false)
+}
+
+// ReadPacketContext waits for the next packet or returns when ctx is cancelled.
+func (self *QueueCursor) ReadPacketContext(ctx context.Context) (pkt Packet, err error) {
+	if err := ctx.Err(); err != nil {
+		return Packet{}, err
+	}
+	return self.readPacket(ctx, true)
+}
+
+func (self *QueueCursor) readPacket(ctx context.Context, cancellable bool) (pkt Packet, err error) {
 	self.que.cond.L.Lock()
+	var stop func() bool
+	defer func() {
+		self.que.cond.L.Unlock()
+		if stop != nil {
+			stop()
+		}
+	}()
 	buf := self.que.buf
 	if !self.gotpos {
 		self.pos = self.init(buf, self.que.videoidx)
@@ -222,8 +242,21 @@ func (self *QueueCursor) ReadPacket() (pkt Packet, err error) {
 			err = io.EOF
 			break
 		}
+		if cancellable {
+			if err = ctx.Err(); err != nil {
+				break
+			}
+			if stop == nil {
+				stop = context.AfterFunc(ctx, func() {
+					// The write lock prevents cancellation from being signalled
+					// between the context check above and cond.Wait.
+					self.que.lock.Lock()
+					self.que.cond.Broadcast()
+					self.que.lock.Unlock()
+				})
+			}
+		}
 		self.que.cond.Wait()
 	}
-	self.que.cond.L.Unlock()
 	return
 }

machinery/src/packets/queue_test.go

@@ -0,0 +1,109 @@
+package packets
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+type observedCancelContext struct {
+	done        chan struct{}
+	cancelled   atomic.Bool
+	errCalls    atomic.Int32
+	waiting     chan struct{}
+	waitingOnce sync.Once
+}
+
+func newObservedCancelContext() *observedCancelContext {
+	return &observedCancelContext{
+		done:    make(chan struct{}),
+		waiting: make(chan struct{}),
+	}
+}
+
+func (c *observedCancelContext) Deadline() (time.Time, bool) { return time.Time{}, false }
+func (c *observedCancelContext) Done() <-chan struct{}       { return c.done }
+func (c *observedCancelContext) Value(any) any               { return nil }
+
+func (c *observedCancelContext) Err() error {
+	if c.errCalls.Add(1) >= 2 {
+		c.waitingOnce.Do(func() { close(c.waiting) })
+	}
+	if c.cancelled.Load() {
+		return context.Canceled
+	}
+	return nil
+}
+
+func (c *observedCancelContext) Cancel() {
+	c.cancelled.Store(true)
+	close(c.done)
+}
+
+func TestQueueCursorReadPacketContextCancelsWhileQueueIsIdle(t *testing.T) {
+	queue := NewQueue()
+	cursor := queue.Latest()
+	ctx := newObservedCancelContext()
+	done := make(chan error, 1)
+
+	go func() {
+		_, err := cursor.ReadPacketContext(ctx)
+		done <- err
+	}()
+
+	select {
+	case <-ctx.waiting:
+	case <-time.After(time.Second):
+		t.Fatal("ReadPacketContext() did not reach the queue wait")
+	}
+	ctx.Cancel()
+	select {
+	case err := <-done:
+		if !errors.Is(err, context.Canceled) {
+			t.Fatalf("ReadPacketContext() error = %v, want %v", err, context.Canceled)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("ReadPacketContext() did not return after cancellation")
+	}
+}
+
+func TestQueueCursorReadPacketDoesNotAllocate(t *testing.T) {
+	queue := NewQueue()
+	for i := 0; i < 200; i++ {
+		if err := queue.WritePacket(Packet{Data: []byte{1}}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	cursor := queue.Oldest()
+
+	allocations := testing.AllocsPerRun(100, func() {
+		if _, err := cursor.ReadPacket(); err != nil {
+			t.Fatal(err)
+		}
+	})
+	if allocations != 0 {
+		t.Fatalf("ReadPacket() allocations = %.2f, want 0", allocations)
+	}
+}
+
+func TestQueueCursorReadPacketContextDoesNotAllocateWhenPacketIsAvailable(t *testing.T) {
+	queue := NewQueue()
+	for i := 0; i < 200; i++ {
+		if err := queue.WritePacket(Packet{Data: []byte{1}}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	cursor := queue.Oldest()
+
+	allocations := testing.AllocsPerRun(100, func() {
+		if _, err := cursor.ReadPacketContext(context.Background()); err != nil {
+			t.Fatal(err)
+		}
+	})
+	if allocations != 0 {
+		t.Fatalf("ReadPacketContext() allocations = %.2f, want 0", allocations)
+	}
+}

machinery/src/routers/http/health.go

@@ -0,0 +1,126 @@
+package http
+
+import (
+	"math"
+	stdhttp "net/http"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/utils"
+)
+
+const (
+	healthApplicationStatus = "get_success"
+	healthEntityStatus      = "healthy"
+	hubConnectionFreshness  = 3 * time.Minute
+)
+
+// StreamResolution is the most recently observed encoded video size in pixels.
+type StreamResolution struct {
+	Width  int64 `json:"width"`
+	Height int64 `json:"height"`
+}
+
+// StreamHealth describes the most recently observed state of one camera stream.
+type StreamHealth struct {
+	Configured        bool             `json:"configured"`
+	Connected         bool             `json:"connected"`
+	PackagesProcessed uint64           `json:"packagesProcessed"`
+	FPS               float64          `json:"fps"`
+	Resolution        StreamResolution `json:"resolution"`
+	LastPacketAt      int64            `json:"lastPacketAt"`
+}
+
+// HubHealth describes the Agent's periodic HTTP heartbeat connection to Hub.
+type HubHealth struct {
+	Configured                bool  `json:"configured"`
+	Connected                 bool  `json:"connected"`
+	LastHeartbeatAttemptAt    int64 `json:"lastHeartbeatAttemptAt"`
+	LastSuccessfulHeartbeatAt int64 `json:"lastSuccessfulHeartbeatAt"`
+}
+
+// Health describes the Agent process health exposed to API clients.
+type Health struct {
+	Description     string       `json:"description"`
+	CameraConnected bool         `json:"cameraConnected"`
+	MainStream      StreamHealth `json:"mainStream"`
+	SubStream       StreamHealth `json:"subStream"`
+	Hub             HubHealth    `json:"hub"`
+}
+
+// HealthResponseData contains the typed payload of a health response.
+type HealthResponseData struct {
+	Health Health `json:"health"`
+}
+
+// HealthResponse is the standard successful response returned by GET /health.
+type HealthResponse struct {
+	SuccessResponse
+	Data HealthResponseData `json:"data"`
+}
+
+// HealthCheck godoc
+// @Summary Check Agent health
+// @Description Confirms that the Agent HTTP process can serve requests and reports current camera stream and Hub heartbeat diagnostics. Operational dependency failures do not change the liveness HTTP status.
+// @Tags health
+// @Produce json
+// @Success 200 {object} HealthResponse
+// @Router /health [get]
+func HealthCheck(c *gin.Context, communication *models.Communication) {
+	c.Header("Cache-Control", "no-store")
+	c.JSON(stdhttp.StatusOK, HealthResponse{
+		SuccessResponse: NewSuccessResponse(
+			stdhttp.StatusOK,
+			healthApplicationStatus,
+			healthEntityStatus,
+			"Healthy",
+			ResponseMetadata{
+				ApplicationName:    "agent",
+				ApplicationVersion: utils.VERSION,
+				Path:               c.Request.URL.Path,
+			},
+		),
+		Data: HealthResponseData{
+			Health: buildHealth(communication, time.Now()),
+		},
+	})
+}
+
+func buildHealth(communication *models.Communication, now time.Time) Health {
+	mainTelemetry := communication.StreamRuntimeTelemetry(models.MainStream)
+	subTelemetry := communication.StreamRuntimeTelemetry(models.SubStream)
+	hubTelemetry := communication.HubRuntimeTelemetry()
+
+	hubConnected := hubTelemetry.Configured &&
+		hubTelemetry.Connected &&
+		hubTelemetry.LastSuccessfulHeartbeatAt > 0 &&
+		now.Unix()-hubTelemetry.LastSuccessfulHeartbeatAt <= int64(hubConnectionFreshness/time.Second)
+
+	return Health{
+		Description:     "Agent HTTP service is healthy",
+		CameraConnected: communication.CameraConnected.Load(),
+		MainStream:      newStreamHealth(mainTelemetry, communication.MainStreamConnected.Load()),
+		SubStream:       newStreamHealth(subTelemetry, communication.SubStreamConnected.Load()),
+		Hub: HubHealth{
+			Configured:                hubTelemetry.Configured,
+			Connected:                 hubConnected,
+			LastHeartbeatAttemptAt:    hubTelemetry.LastHeartbeatAttemptAt,
+			LastSuccessfulHeartbeatAt: hubTelemetry.LastSuccessfulHeartbeatAt,
+		},
+	}
+}
+
+func newStreamHealth(telemetry models.StreamRuntimeTelemetry, connected bool) StreamHealth {
+	return StreamHealth{
+		Configured:        telemetry.Configured,
+		Connected:         connected,
+		PackagesProcessed: telemetry.PackagesProcessed,
+		FPS:               math.Round(telemetry.FPS*100) / 100,
+		Resolution: StreamResolution{
+			Width:  telemetry.Width,
+			Height: telemetry.Height,
+		},
+		LastPacketAt: telemetry.LastPacketAt,
+	}
+}

machinery/src/routers/http/health_test.go

@@ -0,0 +1,117 @@
+package http
+
+import (
+	"encoding/json"
+	stdhttp "net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	jwt "github.com/appleboy/gin-jwt/v2"
+	"github.com/gin-gonic/gin"
+	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/utils"
+)
+
+func TestHealthCheckReturnsStandardPublicResponse(t *testing.T) {
+	gin.SetMode(gin.TestMode)
+
+	middlewareConfig := JWTMiddleWare()
+	authMiddleware, err := jwt.New(&middlewareConfig)
+	if err != nil {
+		t.Fatalf("initialize JWT middleware: %v", err)
+	}
+
+	now := time.Now()
+	communication := &models.Communication{}
+	communication.SetStreamConfigured(models.MainStream, true)
+	communication.SetStreamConfigured(models.SubStream, true)
+	communication.RecordStreamPackage(models.MainStream, 29.969, 1920, 1080, now.Add(-2*time.Second))
+	communication.RecordStreamPackage(models.MainStream, 29.969, 1920, 1080, now.Add(-time.Second))
+	communication.RecordStreamPackage(models.SubStream, 15, 640, 360, now.Add(-time.Second))
+	communication.CameraConnected.Store(true)
+	communication.MainStreamConnected.Store(true)
+	communication.SubStreamConnected.Store(true)
+	communication.SetHubConfigured(true)
+	communication.RecordHubHeartbeatAttempt(now.Add(-2 * time.Second))
+	communication.RecordHubHeartbeatSuccess(now.Add(-time.Second))
+
+	router := gin.New()
+	AddRoutes(router, authMiddleware, "", nil, communication, nil)
+
+	response := httptest.NewRecorder()
+	request := httptest.NewRequest(stdhttp.MethodGet, "/health", nil)
+	router.ServeHTTP(response, request)
+
+	if response.Code != stdhttp.StatusOK {
+		t.Fatalf("status = %d, want %d; body=%s", response.Code, stdhttp.StatusOK, response.Body.String())
+	}
+	if got := response.Header().Get("Cache-Control"); got != "no-store" {
+		t.Fatalf("Cache-Control = %q, want %q", got, "no-store")
+	}
+
+	var body HealthResponse
+	if err := json.Unmarshal(response.Body.Bytes(), &body); err != nil {
+		t.Fatalf("decode health response: %v; body=%s", err, response.Body.String())
+	}
+
+	if body.HTTPStatusCode != stdhttp.StatusOK {
+		t.Errorf("httpStatusCode = %d, want %d", body.HTTPStatusCode, stdhttp.StatusOK)
+	}
+	if body.ApplicationStatusCode != healthApplicationStatus {
+		t.Errorf("applicationStatusCode = %q, want %q", body.ApplicationStatusCode, healthApplicationStatus)
+	}
+	if body.EntityStatusCode != healthEntityStatus {
+		t.Errorf("entityStatusCode = %q, want %q", body.EntityStatusCode, healthEntityStatus)
+	}
+	if body.Message != "Healthy" {
+		t.Errorf("message = %q, want %q", body.Message, "Healthy")
+	}
+	if body.Metadata.ApplicationName != "agent" {
+		t.Errorf("metadata.applicationName = %q, want %q", body.Metadata.ApplicationName, "agent")
+	}
+	if body.Metadata.ApplicationVersion != utils.VERSION {
+		t.Errorf("metadata.applicationVersion = %q, want %q", body.Metadata.ApplicationVersion, utils.VERSION)
+	}
+	if body.Metadata.Timestamp <= 0 {
+		t.Errorf("metadata.timestamp = %d, want a positive Unix timestamp", body.Metadata.Timestamp)
+	}
+	if body.Metadata.Path != "/health" {
+		t.Errorf("metadata.path = %q, want %q", body.Metadata.Path, "/health")
+	}
+	health := body.Data.Health
+	if health.Description != "Agent HTTP service is healthy" {
+		t.Errorf("data.health.description = %q, want %q", health.Description, "Agent HTTP service is healthy")
+	}
+	if !health.CameraConnected {
+		t.Error("data.health.cameraConnected = false, want true")
+	}
+	if health.MainStream.PackagesProcessed != 2 || health.MainStream.FPS != 29.97 {
+		t.Errorf("data.health.mainStream = %+v, want 2 packages at 29.97 FPS", health.MainStream)
+	}
+	if health.MainStream.Resolution != (StreamResolution{Width: 1920, Height: 1080}) {
+		t.Errorf("data.health.mainStream.resolution = %+v", health.MainStream.Resolution)
+	}
+	if health.SubStream.PackagesProcessed != 1 || health.SubStream.FPS != 15 {
+		t.Errorf("data.health.subStream = %+v, want 1 package at 15 FPS", health.SubStream)
+	}
+	if health.SubStream.Resolution != (StreamResolution{Width: 640, Height: 360}) {
+		t.Errorf("data.health.subStream.resolution = %+v", health.SubStream.Resolution)
+	}
+	if !health.Hub.Configured || !health.Hub.Connected {
+		t.Errorf("data.health.hub = %+v, want configured and connected", health.Hub)
+	}
+}
+
+func TestBuildHealthMarksStaleHubHeartbeatDisconnected(t *testing.T) {
+	now := time.Unix(1_788_710_500, 0)
+	communication := &models.Communication{}
+	communication.SetHubConfigured(true)
+	communication.RecordHubHeartbeatAttempt(now.Add(-4 * time.Minute))
+	communication.RecordHubHeartbeatSuccess(now.Add(-4 * time.Minute))
+
+	health := buildHealth(communication, now)
+	if health.Hub.Connected {
+		t.Fatalf("Hub health = %+v, want stale heartbeat to be disconnected", health.Hub)
+	}
+}

machinery/src/routers/http/methods.go

@@ -6,9 +6,9 @@ import (
 	"time"
 
 	"github.com/gin-gonic/gin"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/onvif"
+	log "github.com/sirupsen/logrus"
 )
 
 // Login godoc
@@ -598,27 +598,27 @@ func DoTriggerRelayOutput(c *gin.Context) {
 			err := onvif.TriggerRelayOutput(device, output)
 			if err == nil {
 				msg := "relay output triggered: " + output
-				log.Log.Info("routers.http.methods.DoTriggerRelayOutput(): " + msg)
+				log.Info("routers.http.methods.DoTriggerRelayOutput(): " + msg)
 				c.JSON(200, gin.H{
 					"data": msg,
 				})
 			} else {
 				msg := "something went wrong: " + err.Error()
-				log.Log.Error("routers.http.methods.DoTriggerRelayOutput(): " + msg)
+				log.Error("routers.http.methods.DoTriggerRelayOutput(): " + msg)
 				c.JSON(400, gin.H{
 					"data": msg,
 				})
 			}
 		} else {
 			msg := "something went wrong: " + err.Error()
-			log.Log.Error("routers.http.methods.DoTriggerRelayOutput(): " + msg)
+			log.Error("routers.http.methods.DoTriggerRelayOutput(): " + msg)
 			c.JSON(400, gin.H{
 				"data": msg,
 			})
 		}
 	} else {
 		msg := "something went wrong: " + err.Error()
-		log.Log.Error("routers.http.methods.DoTriggerRelayOutput(): " + msg)
+		log.Error("routers.http.methods.DoTriggerRelayOutput(): " + msg)
 		c.JSON(400, gin.H{
 			"data": msg,
 		})

machinery/src/routers/http/response.go

@@ -0,0 +1,41 @@
+package http
+
+import "time"
+
+// ResponseMetadata contains request and application context shared by API responses.
+type ResponseMetadata struct {
+	ApplicationName    string `json:"applicationName,omitempty"`
+	ApplicationVersion string `json:"applicationVersion,omitempty"`
+	Timestamp          int64  `json:"timestamp,omitempty"`
+	Path               string `json:"path,omitempty"`
+	TraceID            string `json:"traceId,omitempty"`
+}
+
+// SuccessResponse is the common envelope for successful Agent API responses.
+type SuccessResponse struct {
+	HTTPStatusCode        int              `json:"httpStatusCode"`
+	ApplicationStatusCode string           `json:"applicationStatusCode"`
+	EntityStatusCode      string           `json:"entityStatusCode"`
+	Message               string           `json:"message"`
+	Metadata              ResponseMetadata `json:"metadata"`
+}
+
+// NewSuccessResponse creates the Hub-compatible success envelope used by new
+// Agent API endpoints.
+func NewSuccessResponse(
+	httpStatusCode int,
+	applicationStatusCode string,
+	entityStatusCode string,
+	message string,
+	metadata ResponseMetadata,
+) SuccessResponse {
+	metadata.Timestamp = time.Now().Unix()
+
+	return SuccessResponse{
+		HTTPStatusCode:        httpStatusCode,
+		ApplicationStatusCode: applicationStatusCode,
+		EntityStatusCode:      entityStatusCode,
+		Message:               message,
+		Metadata:              metadata,
+	}
+}

machinery/src/routers/http/routes.go

@@ -14,6 +14,10 @@ import (
 
 func AddRoutes(r *gin.Engine, authMiddleware *jwt.GinJWTMiddleware, configDirectory string, configuration *models.Configuration, communication *models.Communication, captureDevice *capture.Capture) *gin.RouterGroup {
 
+	r.GET("/health", func(c *gin.Context) {
+		HealthCheck(c, communication)
+	})
+
 	r.GET("/ws", func(c *gin.Context) {
 		websocket.WebsocketHandler(c, configuration, communication, captureDevice)
 	})

machinery/src/routers/http/server.go

@@ -1,9 +1,11 @@
 package http
 
 import (
+	"fmt"
 	"io"
 	"os"
 	"strconv"
+	"time"
 
 	jwt "github.com/appleboy/gin-jwt/v2"
 	"github.com/gin-contrib/pprof"
@@ -11,7 +13,7 @@ import (
 	"github.com/gin-gonic/gin"
 
 	//Swagger documentantion
-	"log"
+	log "github.com/sirupsen/logrus"
 
 	_ "github.com/kerberos-io/agent/machinery/docs"
 	"github.com/kerberos-io/agent/machinery/src/capture"
@@ -45,7 +47,17 @@ func StartServer(configDirectory string, configuration *models.Configuration, co
 	gin.SetMode(gin.ReleaseMode)
 
 	// Initialize REST API
-	r := gin.Default()
+	r := gin.New()
+	r.Use(requestLogger(), gin.CustomRecovery(func(c *gin.Context, recovered interface{}) {
+		log.WithFields(log.Fields{
+			"component": "http",
+			"event":     "request_panic",
+			"method":    c.Request.Method,
+			"path":      c.Request.URL.Path,
+			"panic":     fmt.Sprint(recovered),
+		}).Error("HTTP request panicked")
+		c.AbortWithStatus(500)
+	}))
 
 	// Profiler
 	pprof.Register(r)
@@ -60,7 +72,10 @@ func StartServer(configDirectory string, configuration *models.Configuration, co
 	middleWare := JWTMiddleWare()
 	authMiddleware, err := jwt.New(&middleWare)
 	if err != nil {
-		log.Fatal("JWT Error:" + err.Error())
+		log.WithError(err).WithFields(log.Fields{
+			"component": "http",
+			"event":     "jwt_initialization_failed",
+		}).Fatal("Failed to initialize JWT middleware")
 	}
 
 	// Add all routes
@@ -73,7 +88,10 @@ func StartServer(configDirectory string, configuration *models.Configuration, co
 		// Move demo environment variables to environment variables
 		err := os.Rename(demoEnvironmentVariables, environmentVariables)
 		if err != nil {
-			log.Fatal(err)
+			log.WithError(err).WithFields(log.Fields{
+				"component": "http",
+				"event":     "demo_environment_move_failed",
+			}).Fatal("Failed to activate demo UI environment")
 		}
 	}
 
@@ -88,9 +106,54 @@ func StartServer(configDirectory string, configuration *models.Configuration, co
 	})
 
 	// Run the api on port
+	log.WithFields(log.Fields{
+		"component": "http",
+		"event":     "server_starting",
+		"port":      configuration.Port,
+	}).Info("HTTP server starting")
 	err = r.Run(":" + configuration.Port)
 	if err != nil {
-		log.Fatal(err)
+		log.WithError(err).WithFields(log.Fields{
+			"component": "http",
+			"event":     "server_failed",
+			"port":      configuration.Port,
+		}).Fatal("HTTP server stopped")
+	}
+}
+
+func requestLogger() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		startedAt := time.Now()
+		c.Next()
+
+		fields := log.Fields{
+			"component":      "http",
+			"duration_ms":    time.Since(startedAt).Milliseconds(),
+			"event":          "request_completed",
+			"method":         c.Request.Method,
+			"path":           c.Request.URL.Path,
+			"response_bytes": c.Writer.Size(),
+			"status":         c.Writer.Status(),
+		}
+		if log.IsLevelEnabled(log.DebugLevel) {
+			fields["client_ip"] = c.ClientIP()
+			fields["user_agent"] = c.Request.UserAgent()
+		}
+		entry := log.WithFields(fields)
+		if len(c.Errors) > 0 {
+			entry = entry.WithField("request_errors", c.Errors.String())
+		}
+
+		switch status := c.Writer.Status(); {
+		case status >= 500:
+			entry.Error("HTTP request completed")
+		case status >= 400:
+			entry.Warn("HTTP request completed")
+		case c.Request.URL.Path == "/health":
+			entry.Debug("HTTP request completed")
+		default:
+			entry.Info("HTTP request completed")
+		}
 	}
 }
 

machinery/src/routers/http/server_test.go

@@ -0,0 +1,89 @@
+package http
+
+import (
+	"bytes"
+	"encoding/json"
+	stdhttp "net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/gin-gonic/gin"
+	log "github.com/sirupsen/logrus"
+)
+
+func TestRequestLoggerEmitsStructuredFields(t *testing.T) {
+	originalOutput := log.StandardLogger().Out
+	originalFormatter := log.StandardLogger().Formatter
+	originalLevel := log.GetLevel()
+	defer func() {
+		log.SetOutput(originalOutput)
+		log.SetFormatter(originalFormatter)
+		log.SetLevel(originalLevel)
+	}()
+
+	var output bytes.Buffer
+	log.SetOutput(&output)
+	log.SetFormatter(&log.JSONFormatter{})
+	log.SetLevel(log.InfoLevel)
+	gin.SetMode(gin.TestMode)
+
+	router := gin.New()
+	router.Use(requestLogger())
+	router.GET("/status", func(c *gin.Context) {
+		c.Status(stdhttp.StatusNoContent)
+	})
+
+	response := httptest.NewRecorder()
+	request := httptest.NewRequest(stdhttp.MethodGet, "/status?token=do-not-log", nil)
+	router.ServeHTTP(response, request)
+
+	var entry map[string]interface{}
+	if err := json.Unmarshal(output.Bytes(), &entry); err != nil {
+		t.Fatalf("decode request log: %v; output=%q", err, output.String())
+	}
+	for key, want := range map[string]interface{}{
+		"component": "http",
+		"event":     "request_completed",
+		"method":    stdhttp.MethodGet,
+		"path":      "/status",
+		"status":    float64(stdhttp.StatusNoContent),
+	} {
+		if got := entry[key]; got != want {
+			t.Fatalf("%s = %v, want %v", key, got, want)
+		}
+	}
+	if bytes.Contains(output.Bytes(), []byte("do-not-log")) {
+		t.Fatalf("request log exposed query parameters: %q", output.String())
+	}
+}
+
+func TestRequestLoggerKeepsSuccessfulHealthChecksBelowInfo(t *testing.T) {
+	originalOutput := log.StandardLogger().Out
+	originalFormatter := log.StandardLogger().Formatter
+	originalLevel := log.GetLevel()
+	defer func() {
+		log.SetOutput(originalOutput)
+		log.SetFormatter(originalFormatter)
+		log.SetLevel(originalLevel)
+	}()
+
+	var output bytes.Buffer
+	log.SetOutput(&output)
+	log.SetFormatter(&log.JSONFormatter{})
+	log.SetLevel(log.InfoLevel)
+	gin.SetMode(gin.TestMode)
+
+	router := gin.New()
+	router.Use(requestLogger())
+	router.GET("/health", func(c *gin.Context) {
+		c.Status(stdhttp.StatusNoContent)
+	})
+
+	response := httptest.NewRecorder()
+	request := httptest.NewRequest(stdhttp.MethodGet, "/health", nil)
+	router.ServeHTTP(response, request)
+
+	if output.Len() != 0 {
+		t.Fatalf("successful health check emitted an info log: %q", output.String())
+	}
+}

machinery/src/routers/mqtt/main.go

@@ -1,7 +1,9 @@
 package mqtt
 
 import (
+	"context"
 	"crypto/rsa"
+	"crypto/tls"
 	"crypto/x509"
 	"encoding/base64"
 	"encoding/json"
@@ -9,21 +11,23 @@ import (
 	"fmt"
 	"io/ioutil"
 	"math/rand"
+	"net"
+	"net/url"
+	"os"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
-	"context"
-
 	mqtt "github.com/eclipse/paho.mqtt.golang"
 	"github.com/kerberos-io/agent/machinery/src/capture"
 	configService "github.com/kerberos-io/agent/machinery/src/config"
 	"github.com/kerberos-io/agent/machinery/src/encryption"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/onvif"
 	"github.com/kerberos-io/agent/machinery/src/webrtc"
+	log "github.com/sirupsen/logrus"
+	"golang.org/x/net/proxy"
 )
 
 // We'll cache the MQTT settings to know if we need to reinitialize the MQTT client connection.
@@ -34,6 +38,177 @@ var PREV_MQTTPassword string
 var PREV_HubKey string
 var PREV_AgentKey string
 
+type pahoErrorLogger struct{}
+
+func (pahoErrorLogger) Println(values ...interface{}) {
+	log.WithFields(log.Fields{
+		"component": "routers/mqtt",
+		"event":     "paho_error",
+	}).Error(strings.TrimSpace(fmt.Sprintln(values...)))
+}
+
+func (pahoErrorLogger) Printf(format string, values ...interface{}) {
+	log.WithFields(log.Fields{
+		"component": "routers/mqtt",
+		"event":     "paho_error",
+	}).Errorf(strings.TrimSpace(format), values...)
+}
+
+func init() {
+	mqtt.ERROR = pahoErrorLogger{}
+}
+
+func enableMQTTConnectionDiagnostics(options *mqtt.ClientOptions, brokerURL string) {
+	if !strings.Contains(brokerURL, "://") {
+		options.SetCustomOpenConnectionFn(openMQTTConnection)
+		return
+	}
+
+	parsedURL, err := url.Parse(brokerURL)
+	if err != nil {
+		return
+	}
+
+	switch strings.ToLower(parsedURL.Scheme) {
+	case "", "mqtt", "tcp", "ssl", "tls", "mqtts", "mqtt+ssl", "tcps":
+		options.SetCustomOpenConnectionFn(openMQTTConnection)
+	}
+}
+
+func openMQTTConnection(uri *url.URL, options mqtt.ClientOptions) (net.Conn, error) {
+	host := uri.Hostname()
+	fields := log.Fields{
+		"component": "routers/mqtt",
+		"host":      host,
+		"port":      uri.Port(),
+		"scheme":    uri.Scheme,
+	}
+	logMQTTDNSResolution(host, options.ConnectTimeout, fields)
+
+	connectionStartedAt := time.Now()
+	dialer := options.Dialer
+	if dialer == nil {
+		dialer = &net.Dialer{Timeout: options.ConnectTimeout}
+	}
+
+	proxyConfigured := os.Getenv("all_proxy") != ""
+	proxyMode := "direct"
+	if proxyConfigured {
+		proxyMode = "socks"
+	}
+	fields["proxy_mode"] = proxyMode
+	log.WithFields(fields).Info("Opening MQTT TCP connection")
+
+	var (
+		connection net.Conn
+		err        error
+	)
+	if proxyConfigured {
+		connection, err = proxy.FromEnvironment().Dial("tcp", uri.Host)
+	} else {
+		connection, err = dialer.Dial("tcp", uri.Host)
+	}
+	fields["duration_ms"] = time.Since(connectionStartedAt).Milliseconds()
+	if err != nil {
+		logMQTTNetworkError("MQTT TCP connection failed", err, fields)
+		return nil, err
+	}
+
+	fields["local_address"] = connection.LocalAddr().String()
+	fields["remote_address"] = connection.RemoteAddr().String()
+	log.WithFields(fields).Info("MQTT TCP connection established")
+
+	if !isSecureMQTTScheme(uri.Scheme) {
+		return connection, nil
+	}
+
+	tlsConfig := options.TLSConfig
+	if tlsConfig == nil {
+		tlsConfig = &tls.Config{}
+	} else {
+		tlsConfig = tlsConfig.Clone()
+	}
+	if tlsConfig.ServerName == "" {
+		tlsConfig.ServerName = host
+	}
+
+	tlsConnection := tls.Client(connection, tlsConfig)
+	tlsStartedAt := time.Now()
+	if options.ConnectTimeout > 0 {
+		_ = tlsConnection.SetDeadline(connectionStartedAt.Add(options.ConnectTimeout))
+	}
+	if err = tlsConnection.Handshake(); err != nil {
+		_ = connection.Close()
+		fields["duration_ms"] = time.Since(tlsStartedAt).Milliseconds()
+		fields["server_name"] = tlsConfig.ServerName
+		logMQTTNetworkError("MQTT TLS handshake failed", err, fields)
+		return nil, err
+	}
+	_ = tlsConnection.SetDeadline(time.Time{})
+
+	state := tlsConnection.ConnectionState()
+	fields["cipher_suite"] = tls.CipherSuiteName(state.CipherSuite)
+	fields["duration_ms"] = time.Since(tlsStartedAt).Milliseconds()
+	fields["server_name"] = tlsConfig.ServerName
+	fields["tls_version"] = tls.VersionName(state.Version)
+	log.WithFields(fields).Info("MQTT TLS handshake established")
+	return tlsConnection, nil
+}
+
+func logMQTTDNSResolution(host string, timeout time.Duration, fields log.Fields) {
+	if host == "" || net.ParseIP(host) != nil {
+		return
+	}
+
+	lookupTimeout := timeout
+	if lookupTimeout <= 0 || lookupTimeout > 5*time.Second {
+		lookupTimeout = 5 * time.Second
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), lookupTimeout)
+	defer cancel()
+
+	startedAt := time.Now()
+	addresses, err := net.DefaultResolver.LookupHost(ctx, host)
+	dnsFields := cloneLogFields(fields)
+	dnsFields["duration_ms"] = time.Since(startedAt).Milliseconds()
+	if err != nil {
+		logMQTTNetworkError("MQTT broker DNS resolution failed", err, dnsFields)
+		return
+	}
+
+	dnsFields["resolved_addresses"] = addresses
+	log.WithFields(dnsFields).Info("MQTT broker DNS resolved")
+}
+
+func logMQTTNetworkError(message string, err error, fields log.Fields) {
+	errorFields := cloneLogFields(fields)
+	if networkError, ok := err.(net.Error); ok {
+		errorFields["network_timeout"] = networkError.Timeout()
+	}
+	if operationError, ok := err.(*net.OpError); ok {
+		errorFields["network"] = operationError.Net
+		errorFields["operation"] = operationError.Op
+	}
+	log.WithError(err).WithFields(errorFields).Error(message)
+}
+
+func cloneLogFields(fields log.Fields) log.Fields {
+	cloned := make(log.Fields, len(fields))
+	for key, value := range fields {
+		cloned[key] = value
+	}
+	return cloned
+}
+
+func isSecureMQTTScheme(scheme string) bool {
+	switch strings.ToLower(scheme) {
+	case "ssl", "tls", "mqtts", "mqtt+ssl", "tcps":
+		return true
+	default:
+		return false
+	}
+}
+
 func HasMQTTClientModified(configuration *models.Configuration) bool {
 	MTTURI := configuration.Config.MQTTURI
 	MTTUsername := configuration.Config.MQTTUsername
@@ -41,7 +216,7 @@ func HasMQTTClientModified(configuration *models.Configuration) bool {
 	HubKey := configuration.Config.HubKey
 	AgentKey := configuration.Config.Key
 	if PREV_MQTTURI != MTTURI || PREV_MQTTUsername != MTTUsername || PREV_MQTTPassword != MQTTPassword || PREV_HubKey != HubKey || PREV_AgentKey != AgentKey {
-		log.Log.Info("HasMQTTClientModified: MQTT settings have been modified, restarting MQTT client.")
+		log.Info("HasMQTTClientModified: MQTT settings have been modified, restarting MQTT client.")
 		return true
 	}
 	return false
@@ -59,6 +234,7 @@ func HasMQTTClientModified(configuration *models.Configuration) bool {
 // - kerberos/{hubkey}/device/{devicekey}/motion: a motion signal
 
 func ConfigureMQTT(configDirectory string, configuration *models.Configuration, communication *models.Communication) mqtt.Client {
+	installRemoteAccessHook()
 
 	config := configuration.Config
 
@@ -70,8 +246,25 @@ func ConfigureMQTT(configDirectory string, configuration *models.Configuration,
 	PREV_AgentKey = configuration.Config.Key
 
 	if config.Offline == "true" {
-		log.Log.Info("routers.mqtt.main.ConfigureMQTT(): not starting as running in Offline mode.")
+		log.Info("routers.mqtt.main.ConfigureMQTT(): not starting as running in Offline mode.")
 	} else {
+		hubKey := ""
+		if config.Cloud == "s3" && config.S3 != nil && config.S3.Publickey != "" {
+			hubKey = config.S3.Publickey
+		} else if config.Cloud == "kstorage" && config.KStorage != nil && config.KStorage.CloudKey != "" {
+			hubKey = config.KStorage.CloudKey
+		}
+		if config.HubKey != "" {
+			hubKey = config.HubKey
+		}
+		if hubKey == "" {
+			log.Warn("routers.mqtt.main.ConfigureMQTT(): not starting without a Hub key")
+			return nil
+		}
+		if config.Key == "" {
+			log.Warn("routers.mqtt.main.ConfigureMQTT(): not starting without an Agent key")
+			return nil
+		}
 
 		opts := mqtt.NewClientOptions()
 
@@ -79,7 +272,6 @@ func ConfigureMQTT(configDirectory string, configuration *models.Configuration,
 		// and share and receive messages to/from.
 		mqttURL := config.MQTTURI
 		opts.AddBroker(mqttURL)
-		log.Log.Debug("routers.mqtt.main.ConfigureMQTT(): Set broker uri " + mqttURL)
 
 		// Our MQTT broker can have username/password credentials
 		// to protect it from the outside.
@@ -88,13 +280,19 @@ func ConfigureMQTT(configDirectory string, configuration *models.Configuration,
 		if mqtt_username != "" || mqtt_password != "" {
 			opts.SetUsername(mqtt_username)
 			opts.SetPassword(mqtt_password)
-			log.Log.Debug("routers.mqtt.main.ConfigureMQTT(): Set username " + mqtt_username)
-			log.Log.Debug("routers.mqtt.main.ConfigureMQTT(): Set password " + mqtt_password)
 		}
+		log.WithFields(log.Fields{
+			"broker_configured":   mqttURL != "",
+			"component":           "routers/mqtt",
+			"event":               "credentials_configured",
+			"password_configured": mqtt_password != "",
+			"username_configured": mqtt_username != "",
+		}).Debug("MQTT connection configuration loaded")
 
 		// Some extra options to make sure the connection behaves
 		// properly. More information here: github.com/eclipse/paho.mqtt.golang.
 		//opts.SetCleanSession(true)
+		enableMQTTConnectionDiagnostics(opts, mqttURL)
 		opts.SetCleanSession(false)
 		opts.SetResumeSubs(true)
 		opts.SetStore(mqtt.NewMemoryStore())
@@ -108,65 +306,81 @@ func ConfigureMQTT(configDirectory string, configuration *models.Configuration,
 		opts.SetOrderMatters(false)
 		opts.SetConnectTimeout(30 * time.Second)
 		opts.SetConnectionLostHandler(func(client mqtt.Client, err error) {
+			entry := log.WithFields(log.Fields{
+				"component": "routers/mqtt",
+				"event":     "connection_lost",
+			})
 			if err != nil {
-				log.Log.Error("routers.mqtt.main.ConfigureMQTT(): MQTT connection lost: " + err.Error())
-			} else {
-				log.Log.Error("routers.mqtt.main.ConfigureMQTT(): MQTT connection lost")
+				entry = entry.WithError(err)
 			}
+			entry.Error("MQTT connection lost")
 		})
 		opts.SetReconnectingHandler(func(client mqtt.Client, options *mqtt.ClientOptions) {
-			log.Log.Warning("routers.mqtt.main.ConfigureMQTT(): reconnecting to MQTT broker")
+			log.WithFields(log.Fields{
+				"component": "routers/mqtt",
+				"event":     "reconnecting",
+			}).Warn("Reconnecting to MQTT broker")
 		})
 		opts.SetOnConnectHandler(func(c mqtt.Client) {
-			log.Log.Info("routers.mqtt.main.ConfigureMQTT(): MQTT session is online")
+			log.WithFields(log.Fields{
+				"component": "routers/mqtt",
+				"event":     "session_online",
+			}).Info("MQTT session is online")
 		})
 
-		hubKey := ""
-		// This is the old way ;)
-		if config.Cloud == "s3" && config.S3 != nil && config.S3.Publickey != "" {
-			hubKey = config.S3.Publickey
-		} else if config.Cloud == "kstorage" && config.KStorage != nil && config.KStorage.CloudKey != "" {
-			hubKey = config.KStorage.CloudKey
-		}
-		// This is the new way ;)
-		if config.HubKey != "" {
-			hubKey = config.HubKey
+		rand.Seed(time.Now().UnixNano())
+		random := rand.Intn(100)
+		mqttClientID := config.Key + strconv.Itoa(random) // this random int is to avoid conflicts.
+
+		// This is a worked-around.
+		// current S3 (Kerberos Hub SAAS) is using a secured MQTT, where the client id,
+		// should match the kerberos hub key.
+		if config.Cloud == "s3" {
+			mqttClientID = config.Key
 		}
 
-		if hubKey != "" {
+		opts.SetClientID(mqttClientID)
+		log.WithFields(log.Fields{
+			"component":            "routers/mqtt",
+			"event":                "client_configured",
+			"randomized_client_id": config.Cloud != "s3",
+		}).Debug("MQTT client configured")
+		rand.Seed(time.Now().UnixNano())
 
-			rand.Seed(time.Now().UnixNano())
-			random := rand.Intn(100)
-			mqttClientID := config.Key + strconv.Itoa(random) // this random int is to avoid conflicts.
+		opts.OnConnect = func(c mqtt.Client) {
+			// We managed to connect to the MQTT broker, hurray!
+			log.WithFields(log.Fields{
+				"component": "routers/mqtt",
+				"event":     "connected",
+			}).Info("MQTT broker connected")
+			log.WithFields(log.Fields{
+				"broker_configured": mqttURL != "",
+				"component":         "routers/mqtt",
+				"event":             "connection_details",
+			}).Debug("MQTT connection details")
 
-			// This is a worked-around.
-			// current S3 (Kerberos Hub SAAS) is using a secured MQTT, where the client id,
-			// should match the kerberos hub key.
-			if config.Cloud == "s3" {
-				mqttClientID = config.Key
-			}
-
-			opts.SetClientID(mqttClientID)
-			log.Log.Info("routers.mqtt.main.ConfigureMQTT(): Set ClientID " + mqttClientID)
-			rand.Seed(time.Now().UnixNano())
-
-			opts.OnConnect = func(c mqtt.Client) {
-				// We managed to connect to the MQTT broker, hurray!
-				log.Log.Info("routers.mqtt.main.ConfigureMQTT(): " + mqttClientID + " connected to " + mqttURL)
-
-				// Create a susbcription for listen and reply
-				MQTTListenerHandler(c, hubKey, configDirectory, configuration, communication)
-			}
+			// Create a susbcription for listen and reply
+			MQTTListenerHandler(c, hubKey, configDirectory, configuration, communication)
 		}
 		mqc := mqtt.NewClient(opts)
 		if token := mqc.Connect(); token.WaitTimeout(30 * time.Second) {
 			if token.Error() != nil {
-				log.Log.Error("routers.mqtt.main.ConfigureMQTT(): unable to establish mqtt broker connection, error was: " + token.Error().Error())
+				log.WithError(token.Error()).WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "initial_connection_failed",
+				}).Error("Failed to establish initial MQTT connection")
 			} else {
-				log.Log.Info("routers.mqtt.main.ConfigureMQTT(): initial MQTT connection established")
+				log.WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "initial_connection_established",
+				}).Info("Initial MQTT connection established")
 			}
 		} else {
-			log.Log.Error("routers.mqtt.main.ConfigureMQTT(): timed out while establishing mqtt broker connection")
+			log.WithFields(log.Fields{
+				"component":  "routers/mqtt",
+				"event":      "initial_connection_timeout",
+				"timeout_ms": (30 * time.Second).Milliseconds(),
+			}).Warn("Initial MQTT connection is still retrying")
 		}
 		return mqc
 	}
@@ -216,7 +430,7 @@ func markHDSessionSeen(sessionID string) bool {
 
 func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory string, configuration *models.Configuration, communication *models.Communication) {
 	if hubKey == "" {
-		log.Log.Info("routers.mqtt.main.MQTTListenerHandler(): no hub key provided, not subscribing to kerberos/hub/{hubkey}")
+		log.Info("routers.mqtt.main.MQTTListenerHandler(): no hub key provided, not subscribing to kerberos/hub/{hubkey}")
 	} else {
 		agentListener := fmt.Sprintf("kerberos/agent/%s", hubKey)
 		token := mqttClient.Subscribe(agentListener, 1, func(c mqtt.Client, msg mqtt.Message) {
@@ -236,6 +450,7 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 			// We will receive all messages from our hub, so we'll need to filter to the relevant device.
 			if message.Mid != "" && message.Timestamp != 0 && message.DeviceId == configuration.Config.Key {
 				var payload models.Payload
+				remoteAuthenticated := false
 
 				// Messages might be hidden, if so we'll need to decrypt them using the Kerberos Hub private key.
 				if message.Hidden && configuration.Config.HubEncryption == "true" {
@@ -249,13 +464,15 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 							}
 							visibleValue, err := encryption.AesDecrypt(data, privateKey)
 							if err != nil {
-								log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message: " + err.Error())
+								log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message: " + err.Error())
 								return
 							}
-							json.Unmarshal(visibleValue, &payload)
-							message.Payload = payload
+							if err := json.Unmarshal(visibleValue, &payload); err == nil {
+								message.Payload = payload
+								remoteAuthenticated = true
+							}
 						} else {
-							log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message, no private key provided.")
+							log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message, no private key provided.")
 						}
 					}
 				}
@@ -271,14 +488,14 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 						pemBytes, _ := ioutil.ReadAll(r)
 						block, _ := pem.Decode(pemBytes)
 						if block == nil {
-							log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error decoding PEM block containing private key")
+							log.Error("routers.mqtt.main.MQTTListenerHandler(): error decoding PEM block containing private key")
 							return
 						} else {
 							// Parse private key
 							b := block.Bytes
 							key, err := x509.ParsePKCS8PrivateKey(b)
 							if err != nil {
-								log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error parsing private key: " + err.Error())
+								log.Error("routers.mqtt.main.MQTTListenerHandler(): error parsing private key: " + err.Error())
 								return
 							} else {
 								// Conver key to *rsa.PrivateKey
@@ -298,16 +515,18 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 										}
 										decryptedValue, err := encryption.AesDecrypt(data, string(decryptedKey))
 										if err != nil {
-											log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message: " + err.Error())
+											log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message: " + err.Error())
 											return
 										}
-										json.Unmarshal(decryptedValue, &payload)
+										if err := json.Unmarshal(decryptedValue, &payload); err == nil {
+											remoteAuthenticated = true
+										}
 									} else {
-										log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message, assymetric keys do not match.")
+										log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message, assymetric keys do not match.")
 										return
 									}
 								} else if err != nil {
-									log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message: " + err.Error())
+									log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message: " + err.Error())
 									return
 								}
 							}
@@ -318,7 +537,11 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 				}
 
 				// We'll find out which message we received, and act accordingly.
-				log.Log.Info("routers.mqtt.main.MQTTListenerHandler(): received message with action: " + payload.Action)
+				log.WithFields(log.Fields{
+					"action":    payload.Action,
+					"component": "routers/mqtt",
+					"event":     "message_received",
+				}).Debug("MQTT message received")
 
 				// NOTE: We intentionally do NOT discard request-hd-stream /
 				// receive-hd-candidates messages based on a wall-clock age. The
@@ -332,7 +555,7 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 				case "record":
 					go HandleRecording(mqttClient, hubKey, payload, configuration, communication)
 				case "get-audio-backchannel":
-					go HandleAudio(mqttClient, hubKey, payload, configuration, communication)
+					HandleAudio(mqttClient, hubKey, payload, configuration, communication)
 				case "get-ptz-position":
 					go HandleGetPTZPosition(mqttClient, hubKey, payload, configuration, communication)
 				case "update-ptz-position":
@@ -355,6 +578,14 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 					go HandleReceiveHDCandidates(mqttClient, hubKey, payload, configuration, communication)
 				case "trigger-relay":
 					go HandleTriggerRelay(mqttClient, hubKey, payload, configuration, communication)
+				case "remote-session-open":
+					go HandleRemoteSessionOpen(mqttClient, hubKey, payload, remoteAuthenticated, configuration)
+				case "remote-session-input":
+					go HandleRemoteSessionInput(mqttClient, hubKey, payload, remoteAuthenticated, configuration)
+				case "remote-session-resize":
+					go HandleRemoteSessionResize(payload, remoteAuthenticated)
+				case "remote-session-close":
+					go HandleRemoteSessionClose(payload, remoteAuthenticated)
 				}
 
 			}
@@ -362,12 +593,15 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 
 		if token.WaitTimeout(10 * time.Second) {
 			if token.Error() != nil {
-				log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): failed to subscribe to " + agentListener + ": " + token.Error().Error())
+				log.WithError(token.Error()).WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "subscription_failed",
+				}).Error("Failed to subscribe to Agent MQTT topic")
 			} else {
-				log.Log.Info("routers.mqtt.main.MQTTListenerHandler(): subscribed to " + agentListener)
+				log.Info("routers.mqtt.main.MQTTListenerHandler(): subscribed to " + agentListener)
 			}
 		} else {
-			log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): timed out while subscribing to " + agentListener)
+			log.Error("routers.mqtt.main.MQTTListenerHandler(): timed out while subscribing to " + agentListener)
 		}
 	}
 }
@@ -396,9 +630,9 @@ func HandleRecording(mqttClient mqtt.Client, hubKey string, payload models.Paylo
 			if communication.IsRecordingManual.IsSet() {
 				communication.RecordingManualHeartbeat.Store(now)
 				communication.RecordingManualHeartbeatSeen.Set()
-				log.Log.Debug("routers.mqtt.main.HandleRecording(): manual recording heartbeat received.")
+				log.Debug("routers.mqtt.main.HandleRecording(): manual recording heartbeat received.")
 			} else {
-				log.Log.Debug("routers.mqtt.main.HandleRecording(): ignoring heartbeat, no active manual recording.")
+				log.Debug("routers.mqtt.main.HandleRecording(): ignoring heartbeat, no active manual recording.")
 			}
 		} else {
 			// Explicit start from the live view (record button). Start a manual
@@ -410,18 +644,16 @@ func HandleRecording(mqttClient mqtt.Client, hubKey string, payload models.Paylo
 			if communication.IsRecordingManual.SetToIf(false, true) {
 				communication.RecordingManualStart.Store(now)
 				communication.RecordingManualHeartbeatSeen.UnSet()
-				log.Log.Info("routers.mqtt.main.HandleRecording(): manual recording started.")
-				select {
-				case communication.HandleMotion <- models.MotionDataPartial{Timestamp: timestamp, NumberOfChanges: 100000000}:
-				default:
-					log.Log.Warning("routers.mqtt.main.HandleRecording(): motion channel full, manual recording start not queued.")
+				log.Info("routers.mqtt.main.HandleRecording(): manual recording started.")
+				if !communication.TrySendMotion(models.MotionDataPartial{Timestamp: timestamp, NumberOfChanges: 100000000}) {
+					log.Warn("routers.mqtt.main.HandleRecording(): motion channel full, manual recording start not queued.")
 				}
 			}
 		}
 	} else {
 		// Stop the manual recording; the motion recorder closes the clip once the
 		// post-recording window elapses. Clear the heartbeat/start markers too.
-		log.Log.Info("routers.mqtt.main.HandleRecording(): manual recording stopped.")
+		log.Info("routers.mqtt.main.HandleRecording(): manual recording stopped.")
 		communication.IsRecordingManual.UnSet()
 		communication.RecordingManualHeartbeat.Store(0)
 		communication.RecordingManualStart.Store(0)
@@ -442,10 +674,31 @@ func HandleAudio(mqttClient mqtt.Client, hubKey string, payload models.Payload,
 			Timestamp: audioPayload.Timestamp,
 			Data:      audioPayload.Data,
 		}
-		communication.HandleAudio <- audioDataPartial
+		if enqueueLatestAudio(communication.HandleAudio, audioDataPartial) {
+			log.Debug("routers.mqtt.main.HandleAudio(): dropped stale audio because the backchannel queue was full")
+		}
 	}
 }
 
+func enqueueLatestAudio(audioChannel chan models.AudioDataPartial, audio models.AudioDataPartial) bool {
+	select {
+	case audioChannel <- audio:
+		return false
+	default:
+	}
+
+	select {
+	case <-audioChannel:
+	default:
+	}
+
+	select {
+	case audioChannel <- audio:
+	default:
+	}
+	return true
+}
+
 func HandleGetPTZPosition(mqttClient mqtt.Client, hubKey string, payload models.Payload, configuration *models.Configuration, communication *models.Communication) {
 	value := payload.Value
 
@@ -458,7 +711,7 @@ func HandleGetPTZPosition(mqttClient mqtt.Client, hubKey string, payload models.
 		// Get Position from device
 		pos, err := onvif.GetPositionFromDevice(*configuration)
 		if err != nil {
-			log.Log.Error("routers.mqtt.main.HandlePTZPosition(): error getting position from device: " + err.Error())
+			log.Error("routers.mqtt.main.HandlePTZPosition(): error getting position from device: " + err.Error())
 		} else {
 			// Needs to wrapped!
 			posString := fmt.Sprintf("%f,%f,%f", pos.PanTilt.X, pos.PanTilt.Y, pos.Zoom.X)
@@ -476,7 +729,10 @@ func HandleGetPTZPosition(mqttClient mqtt.Client, hubKey string, payload models.
 			if err == nil {
 				mqttClient.Publish("kerberos/hub/"+hubKey, 2, false, payload)
 			} else {
-				log.Log.Info("routers.mqtt.main.HandlePTZPosition(): something went wrong while sending position to hub: " + string(payload))
+				log.WithError(err).WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "ptz_position_packaging_failed",
+				}).Error("Failed to package PTZ position message")
 			}
 		}
 	}
@@ -491,11 +747,14 @@ func HandleUpdatePTZPosition(mqttClient mqtt.Client, hubKey string, payload mode
 	json.Unmarshal(jsonData, &onvifAction)
 
 	if onvifAction.Action != "" {
-		if communication.CameraConnected {
-			communication.HandleONVIF <- onvifAction
-			log.Log.Info("routers.mqtt.main.MQTTListenerHandleONVIF(): Received an action - " + onvifAction.Action)
+		if communication.CameraConnected.Load() {
+			if communication.TrySendONVIF(onvifAction) {
+				log.Info("routers.mqtt.main.MQTTListenerHandleONVIF(): Received an action - " + onvifAction.Action)
+			} else {
+				log.Warn("routers.mqtt.main.MQTTListenerHandleONVIF(): action channel unavailable or full")
+			}
 		} else {
-			log.Log.Info("routers.mqtt.main.MQTTListenerHandleONVIF(): received action, but camera is not connected.")
+			log.Info("routers.mqtt.main.MQTTListenerHandleONVIF(): received action, but camera is not connected.")
 		}
 	}
 }
@@ -540,14 +799,17 @@ func HandleRequestConfig(mqttClient mqtt.Client, hubKey string, payload models.P
 			if err == nil {
 				mqttClient.Publish("kerberos/hub/"+hubKey, 2, false, payload)
 			} else {
-				log.Log.Info("routers.mqtt.main.HandleRequestConfig(): something went wrong while sending config to hub: " + string(payload))
+				log.WithError(err).WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "configuration_packaging_failed",
+				}).Error("Failed to package configuration message")
 			}
 
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleRequestConfig(): no config available")
+			log.Info("routers.mqtt.main.HandleRequestConfig(): no config available")
 		}
 
-		log.Log.Info("routers.mqtt.main.HandleRequestConfig(): Received a request for the config")
+		log.Info("routers.mqtt.main.HandleRequestConfig(): Received a request for the config")
 	}
 }
 
@@ -639,7 +901,10 @@ func HandleVerifyStream(mqttClient mqtt.Client, hubKey string, payload models.Pa
 	if err == nil {
 		mqttClient.Publish("kerberos/hub/"+hubKey, 2, false, packagedPayload)
 	} else {
-		log.Log.Info("routers.mqtt.main.HandleVerifyStream(): something went wrong while sending result to hub: " + string(packagedPayload))
+		log.WithError(err).WithFields(log.Fields{
+			"component": "routers/mqtt",
+			"event":     "stream_verification_packaging_failed",
+		}).Error("Failed to package stream verification result")
 	}
 }
 
@@ -660,7 +925,7 @@ func HandleUpdateConfig(mqttClient mqtt.Client, hubKey string, payload models.Pa
 
 		err := configService.SaveConfig(configDirectory, config, configuration, communication)
 		if err == nil {
-			log.Log.Info("routers.mqtt.main.HandleUpdateConfig(): Config updated")
+			log.Info("routers.mqtt.main.HandleUpdateConfig(): Config updated")
 			message := models.Message{
 				Payload: models.Payload{
 					Action:   "acknowledge-update-config",
@@ -671,10 +936,13 @@ func HandleUpdateConfig(mqttClient mqtt.Client, hubKey string, payload models.Pa
 			if err == nil {
 				mqttClient.Publish("kerberos/hub/"+hubKey, 2, false, payload)
 			} else {
-				log.Log.Info("routers.mqtt.main.HandleUpdateConfig(): something went wrong while sending acknowledge config to hub: " + string(payload))
+				log.WithError(err).WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "configuration_acknowledgement_packaging_failed",
+				}).Error("Failed to package configuration acknowledgement")
 			}
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleUpdateConfig(): Config update failed")
+			log.Info("routers.mqtt.main.HandleUpdateConfig(): Config update failed")
 		}
 	}
 }
@@ -687,7 +955,7 @@ func HandleRequestSDStream(mqttClient mqtt.Client, hubKey string, payload models
 	json.Unmarshal(jsonData, &requestSDStreamPayload)
 
 	if requestSDStreamPayload.Timestamp != 0 {
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			// A viewer that opted into the HTTP transport is signalled on a separate
 			// channel so the producer ships its frames to hub-api over HTTP instead of
 			// publishing them over MQTT. Any other (or absent) transport keeps the
@@ -703,9 +971,9 @@ func HandleRequestSDStream(mqttClient mqtt.Client, hubKey string, payload models
 				default:
 				}
 			}
-			log.Log.Info("routers.mqtt.main.HandleRequestSDStream(): received request to livestream.")
+			log.Info("routers.mqtt.main.HandleRequestSDStream(): received request to livestream.")
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleRequestSDStream(): received request to livestream, but camera is not connected.")
+			log.Info("routers.mqtt.main.HandleRequestSDStream(): received request to livestream, but camera is not connected.")
 		}
 	}
 }
@@ -722,7 +990,7 @@ func HandleRequestHLSStream(mqttClient mqtt.Client, hubKey string, payload model
 	json.Unmarshal(jsonData, &requestHLSStreamPayload)
 
 	if requestHLSStreamPayload.Timestamp != 0 {
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			// Forward the requested quality ("auto"|"high"|"low"; empty => auto) so
 			// the producer can switch the live session between the main and sub
 			// stream on demand. The send doubles as the viewer keepalive.
@@ -730,9 +998,9 @@ func HandleRequestHLSStream(mqttClient mqtt.Client, hubKey string, payload model
 			case communication.HandleLiveHLS <- requestHLSStreamPayload.Quality:
 			default:
 			}
-			log.Log.Info("routers.mqtt.main.HandleRequestHLSStream(): received request to livestream over HLS.")
+			log.Info("routers.mqtt.main.HandleRequestHLSStream(): received request to livestream over HLS.")
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleRequestHLSStream(): received request to livestream over HLS, but camera is not connected.")
+			log.Info("routers.mqtt.main.HandleRequestHLSStream(): received request to livestream over HLS, but camera is not connected.")
 		}
 	}
 }
@@ -745,29 +1013,30 @@ func HandleRequestHDStream(mqttClient mqtt.Client, hubKey string, payload models
 	json.Unmarshal(jsonData, &requestHDStreamPayload)
 
 	if requestHDStreamPayload.Timestamp != 0 {
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			// Dedupe by session_id: the viewer republishes its offer while
 			// waiting for an answer (and the broker may redeliver), and we
 			// don't want to spawn multiple peer connections for the same
 			// browser session.
 			if markHDSessionSeen(requestHDStreamPayload.SessionID) {
-				log.Log.Info("routers.mqtt.main.HandleRequestHDStream(): duplicate request for session " +
-					requestHDStreamPayload.SessionID + ", ignoring")
+				log.WithFields(log.Fields{
+					"component":  "routers/mqtt",
+					"event":      "duplicate_hd_stream_request",
+					"session_id": requestHDStreamPayload.SessionID,
+				}).Debug("Ignoring duplicate HD stream request")
 				return
 			}
 			// Set the Hub key, so we can send back the answer.
 			requestHDStreamPayload.HubKey = hubKey
-			if communication.HandleLiveHDHandshake == nil {
-				log.Log.Error("routers.mqtt.main.HandleRequestHDStream(): handshake channel is nil, dropping request")
+			if !communication.TrySendLiveHDHandshake(models.LiveHDHandshake{
+				Payload: requestHDStreamPayload,
+			}) {
+				log.Error("routers.mqtt.main.HandleRequestHDStream(): handshake channel unavailable or full, dropping request")
 				return
 			}
-
-			communication.HandleLiveHDHandshake <- models.LiveHDHandshake{
-				Payload: requestHDStreamPayload,
-			}
-			log.Log.Info("routers.mqtt.main.HandleRequestHDStream(): received request to setup webrtc.")
+			log.Info("routers.mqtt.main.HandleRequestHDStream(): received request to setup webrtc.")
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleRequestHDStream(): received request to setup webrtc, but camera is not connected.")
+			log.Info("routers.mqtt.main.HandleRequestHDStream(): received request to setup webrtc, but camera is not connected.")
 		}
 	}
 }
@@ -780,12 +1049,12 @@ func HandleReceiveHDCandidates(mqttClient mqtt.Client, hubKey string, payload mo
 	json.Unmarshal(jsonData, &receiveHDCandidatesPayload)
 
 	if receiveHDCandidatesPayload.Timestamp != 0 {
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			// Register candidate channel
 			key := configuration.Config.Key + "/" + receiveHDCandidatesPayload.SessionID
 			go webrtc.RegisterCandidates(key, receiveHDCandidatesPayload)
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleReceiveHDCandidates(): received candidate, but camera is not connected.")
+			log.Info("routers.mqtt.main.HandleReceiveHDCandidates(): received candidate, but camera is not connected.")
 		}
 	}
 }
@@ -797,14 +1066,17 @@ func HandleNavigatePTZ(mqttClient mqtt.Client, hubKey string, payload models.Pay
 	json.Unmarshal(jsonData, &navigatePTZPayload)
 
 	if navigatePTZPayload.Timestamp != 0 {
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			action := navigatePTZPayload.Action
 			var onvifAction models.OnvifAction
 			json.Unmarshal([]byte(action), &onvifAction)
-			communication.HandleONVIF <- onvifAction
-			log.Log.Info("routers.mqtt.main.HandleNavigatePTZ(): Received an action - " + onvifAction.Action)
+			if communication.TrySendONVIF(onvifAction) {
+				log.Info("routers.mqtt.main.HandleNavigatePTZ(): Received an action - " + onvifAction.Action)
+			} else {
+				log.Warn("routers.mqtt.main.HandleNavigatePTZ(): action channel unavailable or full")
+			}
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleNavigatePTZ(): received action, but camera is not connected.")
+			log.Info("routers.mqtt.main.HandleNavigatePTZ(): received action, but camera is not connected.")
 		}
 	}
 }
@@ -816,7 +1088,7 @@ func HandleTriggerRelay(mqttClient mqtt.Client, hubKey string, payload models.Pa
 	json.Unmarshal(jsonData, &triggerRelayPayload)
 
 	if triggerRelayPayload.Timestamp != 0 {
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			// Get token (name of relay)
 			token := triggerRelayPayload.Token
 			// Connect to Onvif device
@@ -826,16 +1098,25 @@ func HandleTriggerRelay(mqttClient mqtt.Client, hubKey string, payload models.Pa
 				// Trigger relay output
 				err := onvif.TriggerRelayOutput(device, token)
 				if err != nil {
-					log.Log.Error("routers.mqtt.main.HandleTriggerRelay(): error triggering relay: " + err.Error())
+					log.WithError(err).WithFields(log.Fields{
+						"component": "routers/mqtt",
+						"event":     "relay_output_failed",
+					}).Error("Failed to trigger relay output")
 				} else {
-					log.Log.Info("routers.mqtt.main.HandleTriggerRelay(): trigger (" + token + ") relay output.")
+					log.WithFields(log.Fields{
+						"component": "routers/mqtt",
+						"event":     "relay_output_triggered",
+					}).Info("Relay output triggered")
 				}
 			} else {
-				log.Log.Error("routers.mqtt.main.HandleTriggerRelay(): error connecting to device: " + err.Error())
+				log.WithError(err).WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "relay_device_connection_failed",
+				}).Error("Failed to connect to relay device")
 			}
 
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleTriggerRelay(): received trigger, but camera is not connected.")
+			log.Info("routers.mqtt.main.HandleTriggerRelay(): received trigger, but camera is not connected.")
 		}
 	}
 }
@@ -847,6 +1128,6 @@ func DisconnectMQTT(mqttClient mqtt.Client, config *models.Config) {
 		mqttClient.Unsubscribe("kerberos/agent/" + PREV_HubKey)
 		mqttClient.Disconnect(1000)
 		mqttClient = nil
-		log.Log.Info("routers.mqtt.main.DisconnectMQTT(): MQTT client disconnected.")
+		log.Info("routers.mqtt.main.DisconnectMQTT(): MQTT client disconnected.")
 	}
 }

machinery/src/routers/mqtt/main_test.go

@@ -0,0 +1,208 @@
+package mqtt
+
+import (
+	"crypto/tls"
+	"errors"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"testing"
+	"time"
+
+	mqtt "github.com/eclipse/paho.mqtt.golang"
+	"github.com/kerberos-io/agent/machinery/src/models"
+)
+
+func TestEnableMQTTConnectionDiagnostics(t *testing.T) {
+	tests := []struct {
+		name      string
+		brokerURL string
+		want      bool
+	}{
+		{name: "ActiveMQ TLS", brokerURL: "mqtt+ssl://broker.example:8883", want: true},
+		{name: "TCP", brokerURL: "tcp://broker.example:1883", want: true},
+		{name: "default TCP", brokerURL: "broker.example:1883", want: true},
+		{name: "WebSocket", brokerURL: "wss://broker.example/mqtt", want: false},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			options := mqtt.NewClientOptions()
+			enableMQTTConnectionDiagnostics(options, test.brokerURL)
+			if got := options.CustomOpenConnectionFn != nil; got != test.want {
+				t.Fatalf("CustomOpenConnectionFn configured = %t, want %t", got, test.want)
+			}
+		})
+	}
+}
+
+func TestOpenMQTTConnectionReturnsTCPError(t *testing.T) {
+	options := *mqtt.NewClientOptions().SetConnectTimeout(100 * time.Millisecond)
+	brokerURL, err := url.Parse("tcp://127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	connection, err := openMQTTConnection(brokerURL, options)
+	if connection != nil {
+		connection.Close()
+		t.Fatal("openMQTTConnection() returned a connection for an unavailable endpoint")
+	}
+	if err == nil {
+		t.Fatal("openMQTTConnection() returned no TCP error")
+	}
+}
+
+func TestOpenMQTTConnectionEstablishesActiveMQTLS(t *testing.T) {
+	server := httptest.NewTLSServer(nil)
+	defer server.Close()
+
+	serverURL, err := url.Parse(server.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	brokerURL, err := url.Parse("mqtt+ssl://" + serverURL.Host)
+	if err != nil {
+		t.Fatal(err)
+	}
+	options := *mqtt.NewClientOptions().
+		SetConnectTimeout(time.Second).
+		SetTLSConfig(&tls.Config{InsecureSkipVerify: true}) // #nosec G402 -- local test server
+
+	connection, err := openMQTTConnection(brokerURL, options)
+	if err != nil {
+		t.Fatalf("openMQTTConnection() error = %v", err)
+	}
+	defer connection.Close()
+	if _, ok := connection.(*tls.Conn); !ok {
+		t.Fatalf("openMQTTConnection() connection type = %T, want *tls.Conn", connection)
+	}
+}
+
+func TestIsSecureMQTTScheme(t *testing.T) {
+	for _, scheme := range []string{"ssl", "tls", "mqtts", "mqtt+ssl", "tcps"} {
+		if !isSecureMQTTScheme(scheme) {
+			t.Errorf("isSecureMQTTScheme(%q) = false, want true", scheme)
+		}
+	}
+	if isSecureMQTTScheme("tcp") {
+		t.Error("isSecureMQTTScheme(\"tcp\") = true, want false")
+	}
+}
+
+func TestConfigureMQTTRequiresHubKey(t *testing.T) {
+	configuration := &models.Configuration{Config: models.Config{Key: "agent-key"}}
+
+	if client := ConfigureMQTT("", configuration, &models.Communication{}); client != nil {
+		t.Fatal("ConfigureMQTT() returned a client without a Hub key")
+	}
+}
+
+func TestConfigureMQTTRequiresAgentKey(t *testing.T) {
+	configuration := &models.Configuration{Config: models.Config{HubKey: "hub-key"}}
+
+	if client := ConfigureMQTT("", configuration, &models.Communication{}); client != nil {
+		t.Fatal("ConfigureMQTT() returned a client without an Agent key")
+	}
+}
+
+func TestEnqueueLatestAudioReplacesOldestFrameWhenFull(t *testing.T) {
+	audioChannel := make(chan models.AudioDataPartial, 2)
+	audioChannel <- models.AudioDataPartial{Timestamp: 1}
+	audioChannel <- models.AudioDataPartial{Timestamp: 2}
+
+	dropped := enqueueLatestAudio(audioChannel, models.AudioDataPartial{Timestamp: 3})
+	if !dropped {
+		t.Fatal("enqueueLatestAudio() dropped = false, want true")
+	}
+
+	first := <-audioChannel
+	second := <-audioChannel
+	if first.Timestamp != 2 || second.Timestamp != 3 {
+		t.Fatalf("queued timestamps = (%d, %d), want (2, 3)", first.Timestamp, second.Timestamp)
+	}
+}
+
+func TestEnqueueLatestAudioDoesNotBlockNilChannel(t *testing.T) {
+	done := make(chan struct{})
+	go func() {
+		enqueueLatestAudio(nil, models.AudioDataPartial{Timestamp: 1})
+		close(done)
+	}()
+
+	select {
+	case <-done:
+	case <-time.After(100 * time.Millisecond):
+		t.Fatal("enqueueLatestAudio() blocked on a nil channel")
+	}
+}
+
+func TestRemoteAccessRequiresExplicitOptIn(t *testing.T) {
+	previous, present := os.LookupEnv(remoteAccessEnvironment)
+	t.Cleanup(func() {
+		if present {
+			_ = os.Setenv(remoteAccessEnvironment, previous)
+		} else {
+			_ = os.Unsetenv(remoteAccessEnvironment)
+		}
+	})
+
+	_ = os.Unsetenv(remoteAccessEnvironment)
+	if remoteAccessEnabled() {
+		t.Fatal("remoteAccessEnabled() = true without opt-in")
+	}
+	_ = os.Setenv(remoteAccessEnvironment, "true")
+	if !remoteAccessEnabled() {
+		t.Fatal("remoteAccessEnabled() = false after opt-in")
+	}
+}
+
+func TestNormalizeTerminalSize(t *testing.T) {
+	rows, columns := normalizeTerminalSize(0, 0)
+	if rows != 24 || columns != 80 {
+		t.Fatalf("normalizeTerminalSize(0, 0) = (%d, %d), want (24, 80)", rows, columns)
+	}
+
+	rows, columns = normalizeTerminalSize(500, 500)
+	if rows != 200 || columns != 400 {
+		t.Fatalf("normalizeTerminalSize(500, 500) = (%d, %d), want (200, 400)", rows, columns)
+	}
+}
+
+func TestDecodeRemotePayloadRejectsMissingSession(t *testing.T) {
+	_, err := decodeRemotePayload(models.Payload{Value: map[string]interface{}{
+		"kind": "shell",
+	}})
+	if err == nil {
+		t.Fatal("decodeRemotePayload() accepted a missing session id")
+	}
+}
+
+func TestRemoteSessionOpenRejectsUnprovenEncryption(t *testing.T) {
+	previous, present := os.LookupEnv(remoteAccessEnvironment)
+	t.Cleanup(func() {
+		if present {
+			_ = os.Setenv(remoteAccessEnvironment, previous)
+		} else {
+			_ = os.Unsetenv(remoteAccessEnvironment)
+		}
+	})
+	_ = os.Setenv(remoteAccessEnvironment, "true")
+
+	// The listener passes false when an envelope merely claims to be hidden but
+	// no ciphertext was successfully decrypted. The remote handler must reject it.
+	if err := validateRemoteAccess(false); !errors.Is(err, errRemoteUnauthenticated) {
+		t.Fatalf("validateRemoteAccess(false) error = %v, want %v", err, errRemoteUnauthenticated)
+	}
+}
+
+func TestRemoteSessionReservationIsIdempotent(t *testing.T) {
+	manager := newRemoteAccessManager()
+	session := &remoteSession{id: "session-1", kind: "logs"}
+	if _, err := manager.reserve(session); err != nil {
+		t.Fatalf("first reserve() failed: %v", err)
+	}
+	if _, err := manager.reserve(session); !errors.Is(err, errRemoteSessionExists) {
+		t.Fatalf("duplicate reserve() error = %v, want %v", err, errRemoteSessionExists)
+	}
+}

machinery/src/routers/mqtt/remote_access.go

@@ -0,0 +1,397 @@
+package mqtt
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"io"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/creack/pty"
+	paho "github.com/eclipse/paho.mqtt.golang"
+	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	remoteAccessEnvironment = "AGENT_REMOTE_ACCESS_ENABLED"
+	remoteHistoryLimit      = 500
+	remoteSessionLimit      = 5
+	remoteOutputChunkSize   = 4096
+	remoteInputLimit        = 64 * 1024
+	remoteSessionLifetime   = time.Hour
+)
+
+type remoteSession struct {
+	id     string
+	kind   string
+	pty    *os.File
+	cancel context.CancelFunc
+	logs   chan string
+	done   chan struct{}
+	timer  *time.Timer
+}
+
+type remoteAccessManager struct {
+	mu       sync.Mutex
+	sessions map[string]*remoteSession
+	history  []string
+}
+
+var (
+	remoteAccess             = newRemoteAccessManager()
+	remoteHookOnce           sync.Once
+	errRemoteDisabled        = errors.New("remote access is disabled on this agent")
+	errRemoteUnauthenticated = errors.New("remote access requires encrypted MQTT")
+	errRemoteSessionExists   = errors.New("remote session already exists")
+	errRemoteSessionLimit    = errors.New("remote session limit reached")
+)
+
+func newRemoteAccessManager() *remoteAccessManager {
+	return &remoteAccessManager{sessions: make(map[string]*remoteSession)}
+}
+
+func installRemoteAccessHook() {
+	remoteHookOnce.Do(func() {
+		log.AddHook(remoteAccess)
+	})
+}
+
+func (manager *remoteAccessManager) Levels() []log.Level {
+	return log.AllLevels
+}
+
+func (manager *remoteAccessManager) Fire(entry *log.Entry) error {
+	line, err := json.Marshal(map[string]interface{}{
+		"timestamp": entry.Time.Format(time.RFC3339Nano),
+		"level":     entry.Level.String(),
+		"message":   entry.Message,
+		"fields":    entry.Data,
+	})
+	if err != nil {
+		return nil
+	}
+	encoded := base64.StdEncoding.EncodeToString(append(line, '\n'))
+
+	manager.mu.Lock()
+	manager.history = append(manager.history, encoded)
+	if len(manager.history) > remoteHistoryLimit {
+		manager.history = manager.history[len(manager.history)-remoteHistoryLimit:]
+	}
+	for _, session := range manager.sessions {
+		if session.kind != "logs" {
+			continue
+		}
+		select {
+		case session.logs <- encoded:
+		default:
+		}
+	}
+	manager.mu.Unlock()
+	return nil
+}
+
+func remoteAccessEnabled() bool {
+	enabled, err := strconv.ParseBool(strings.TrimSpace(os.Getenv(remoteAccessEnvironment)))
+	return err == nil && enabled
+}
+
+func validateRemoteAccess(authenticated bool) error {
+	if !authenticated {
+		return errRemoteUnauthenticated
+	}
+	if !remoteAccessEnabled() {
+		return errRemoteDisabled
+	}
+	return nil
+}
+
+func decodeRemotePayload(payload models.Payload) (models.RemoteSessionPayload, error) {
+	data, err := json.Marshal(payload.Value)
+	if err != nil {
+		return models.RemoteSessionPayload{}, err
+	}
+	var request models.RemoteSessionPayload
+	if err := json.Unmarshal(data, &request); err != nil {
+		return models.RemoteSessionPayload{}, err
+	}
+	if request.SessionID == "" || len(request.SessionID) > 128 {
+		return models.RemoteSessionPayload{}, errors.New("invalid remote session id")
+	}
+	return request, nil
+}
+
+func normalizeTerminalSize(rows uint16, columns uint16) (uint16, uint16) {
+	if rows < 5 {
+		rows = 24
+	}
+	if rows > 200 {
+		rows = 200
+	}
+	if columns < 20 {
+		columns = 80
+	}
+	if columns > 400 {
+		columns = 400
+	}
+	return rows, columns
+}
+
+func HandleRemoteSessionOpen(client paho.Client, hubKey string, payload models.Payload, authenticated bool, configuration *models.Configuration) {
+	request, err := decodeRemotePayload(payload)
+	if err != nil {
+		return
+	}
+	if accessErr := validateRemoteAccess(authenticated); accessErr != nil {
+		publishRemoteStatus(client, hubKey, configuration, request.SessionID, request.Kind, "error", accessErr.Error())
+		return
+	}
+
+	switch request.Kind {
+	case "logs":
+		err = remoteAccess.openLogs(client, hubKey, configuration, request)
+	case "shell":
+		err = remoteAccess.openShell(client, hubKey, configuration, request)
+	default:
+		err = errors.New("unsupported remote session kind")
+	}
+	if err != nil {
+		if errors.Is(err, errRemoteSessionExists) {
+			publishRemoteStatus(client, hubKey, configuration, request.SessionID, request.Kind, "opened", "")
+			return
+		}
+		publishRemoteStatus(client, hubKey, configuration, request.SessionID, request.Kind, "error", err.Error())
+	}
+}
+
+func HandleRemoteSessionInput(client paho.Client, hubKey string, payload models.Payload, authenticated bool, configuration *models.Configuration) {
+	if validateRemoteAccess(authenticated) != nil {
+		return
+	}
+	request, err := decodeRemotePayload(payload)
+	if err != nil || len(request.Data) > remoteInputLimit*2 {
+		return
+	}
+	data, err := base64.StdEncoding.DecodeString(request.Data)
+	if err != nil || len(data) > remoteInputLimit {
+		return
+	}
+	remoteAccess.mu.Lock()
+	session := remoteAccess.sessions[request.SessionID]
+	remoteAccess.mu.Unlock()
+	if session == nil || session.kind != "shell" || session.pty == nil {
+		publishRemoteStatus(client, hubKey, configuration, request.SessionID, "shell", "error", "remote session is not open")
+		return
+	}
+	if _, err := session.pty.Write(data); err != nil {
+		publishRemoteStatus(client, hubKey, configuration, request.SessionID, "shell", "error", "failed to write terminal input")
+	}
+}
+
+func HandleRemoteSessionResize(payload models.Payload, authenticated bool) {
+	if validateRemoteAccess(authenticated) != nil {
+		return
+	}
+	request, err := decodeRemotePayload(payload)
+	if err != nil {
+		return
+	}
+	rows, columns := normalizeTerminalSize(request.Rows, request.Columns)
+	remoteAccess.mu.Lock()
+	session := remoteAccess.sessions[request.SessionID]
+	remoteAccess.mu.Unlock()
+	if session != nil && session.kind == "shell" && session.pty != nil {
+		_ = pty.Setsize(session.pty, &pty.Winsize{Rows: rows, Cols: columns})
+	}
+}
+
+func HandleRemoteSessionClose(payload models.Payload, authenticated bool) {
+	if !authenticated {
+		return
+	}
+	request, err := decodeRemotePayload(payload)
+	if err == nil {
+		remoteAccess.close(request.SessionID)
+	}
+}
+
+func (manager *remoteAccessManager) reserve(session *remoteSession) ([]string, error) {
+	manager.mu.Lock()
+	defer manager.mu.Unlock()
+	if _, exists := manager.sessions[session.id]; exists {
+		return nil, errRemoteSessionExists
+	}
+	if len(manager.sessions) >= remoteSessionLimit {
+		return nil, errRemoteSessionLimit
+	}
+	manager.sessions[session.id] = session
+	history := append([]string(nil), manager.history...)
+	return history, nil
+}
+
+func (manager *remoteAccessManager) expire(client paho.Client, hubKey string, configuration *models.Configuration, session *remoteSession) {
+	session.timer = time.AfterFunc(remoteSessionLifetime, func() {
+		manager.close(session.id)
+		if session.kind == "logs" {
+			publishRemoteStatus(client, hubKey, configuration, session.id, session.kind, "closed", "session lifetime reached")
+		}
+	})
+}
+
+func (manager *remoteAccessManager) openLogs(client paho.Client, hubKey string, configuration *models.Configuration, request models.RemoteSessionPayload) error {
+	session := &remoteSession{
+		id:   request.SessionID,
+		kind: "logs",
+		logs: make(chan string, 256),
+		done: make(chan struct{}),
+	}
+	history, err := manager.reserve(session)
+	if err != nil {
+		return err
+	}
+	manager.expire(client, hubKey, configuration, session)
+	tail := request.Tail
+	if tail <= 0 || tail > remoteHistoryLimit {
+		tail = 200
+	}
+	if len(history) > tail {
+		history = history[len(history)-tail:]
+	}
+
+	publishRemoteStatus(client, hubKey, configuration, session.id, session.kind, "opened", "")
+	go func() {
+		for _, line := range history {
+			publishRemoteOutput(client, hubKey, configuration, session.id, session.kind, line)
+		}
+		for {
+			select {
+			case line := <-session.logs:
+				publishRemoteOutput(client, hubKey, configuration, session.id, session.kind, line)
+			case <-session.done:
+				return
+			}
+		}
+	}()
+	return nil
+}
+
+func (manager *remoteAccessManager) openShell(client paho.Client, hubKey string, configuration *models.Configuration, request models.RemoteSessionPayload) error {
+	rows, columns := normalizeTerminalSize(request.Rows, request.Columns)
+	ctx, cancel := context.WithCancel(context.Background())
+	session := &remoteSession{
+		id:     request.SessionID,
+		kind:   "shell",
+		cancel: cancel,
+		done:   make(chan struct{}),
+	}
+	if _, err := manager.reserve(session); err != nil {
+		cancel()
+		return err
+	}
+	manager.expire(client, hubKey, configuration, session)
+	command := exec.CommandContext(ctx, "/bin/sh")
+	command.Env = append(os.Environ(), "TERM=xterm-256color", "HISTFILE=/dev/null")
+	terminal, err := pty.StartWithSize(command, &pty.Winsize{Rows: rows, Cols: columns})
+	if err != nil {
+		manager.remove(session.id)
+		cancel()
+		return err
+	}
+	session.pty = terminal
+
+	publishRemoteStatus(client, hubKey, configuration, session.id, session.kind, "opened", "")
+	go manager.forwardShell(client, hubKey, configuration, session, command)
+	return nil
+}
+
+func (manager *remoteAccessManager) forwardShell(client paho.Client, hubKey string, configuration *models.Configuration, session *remoteSession, command *exec.Cmd) {
+	buffer := make([]byte, remoteOutputChunkSize)
+	for {
+		count, err := session.pty.Read(buffer)
+		if count > 0 {
+			publishRemoteOutput(client, hubKey, configuration, session.id, session.kind, base64.StdEncoding.EncodeToString(buffer[:count]))
+		}
+		if err != nil {
+			if !errors.Is(err, io.EOF) && !errors.Is(err, os.ErrClosed) {
+				publishRemoteStatus(client, hubKey, configuration, session.id, session.kind, "error", "terminal stream closed unexpectedly")
+			}
+			break
+		}
+	}
+	_ = command.Wait()
+	manager.remove(session.id)
+	publishRemoteStatus(client, hubKey, configuration, session.id, session.kind, "closed", "")
+}
+
+func (manager *remoteAccessManager) close(sessionID string) {
+	manager.mu.Lock()
+	session := manager.sessions[sessionID]
+	delete(manager.sessions, sessionID)
+	manager.mu.Unlock()
+	if session == nil {
+		return
+	}
+	if session.cancel != nil {
+		session.cancel()
+	}
+	if session.pty != nil {
+		_ = session.pty.Close()
+	}
+	if session.logs != nil {
+		close(session.done)
+	}
+	if session.timer != nil {
+		session.timer.Stop()
+	}
+}
+
+func (manager *remoteAccessManager) remove(sessionID string) {
+	manager.mu.Lock()
+	session := manager.sessions[sessionID]
+	delete(manager.sessions, sessionID)
+	manager.mu.Unlock()
+	if session != nil && session.timer != nil {
+		session.timer.Stop()
+	}
+}
+
+func publishRemoteStatus(client paho.Client, hubKey string, configuration *models.Configuration, sessionID string, kind string, state string, errorMessage string) {
+	status := models.RemoteSessionStatus{
+		Timestamp: time.Now().Unix(),
+		SessionID: sessionID,
+		Kind:      kind,
+		State:     state,
+		Error:     errorMessage,
+	}
+	value, _ := json.Marshal(status)
+	var statusValue map[string]interface{}
+	_ = json.Unmarshal(value, &statusValue)
+	publishRemote(client, hubKey, configuration, "remote-session-status", statusValue, 1)
+}
+
+func publishRemoteOutput(client paho.Client, hubKey string, configuration *models.Configuration, sessionID string, kind string, data string) {
+	publishRemote(client, hubKey, configuration, "remote-session-output", map[string]interface{}{
+		"timestamp":  time.Now().Unix(),
+		"session_id": sessionID,
+		"kind":       kind,
+		"data":       data,
+	}, 0)
+}
+
+func publishRemote(client paho.Client, hubKey string, configuration *models.Configuration, action string, value map[string]interface{}, qos byte) {
+	message := models.Message{Payload: models.Payload{
+		Action:   action,
+		DeviceId: configuration.Config.Key,
+		Value:    value,
+	}}
+	payload, err := models.PackageMQTTMessage(configuration, message)
+	if err == nil {
+		client.Publish("kerberos/hub/"+hubKey, qos, false, payload)
+	}
+}

machinery/src/routers/websocket/main.go

@@ -11,11 +11,11 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/gorilla/websocket"
 	"github.com/kerberos-io/agent/machinery/src/capture"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/kerberos-io/agent/machinery/src/utils"
 	"github.com/kerberos-io/agent/machinery/src/webrtc"
+	log "github.com/sirupsen/logrus"
 )
 
 type Message struct {
@@ -43,7 +43,7 @@ func writeWebRTCError(connection *Connection, clientID string, sessionID string,
 			"message":    errorMessage,
 		},
 	}); err != nil {
-		log.Log.Error("routers.websocket.main.writeWebRTCError(): " + err.Error())
+		log.Error("routers.websocket.main.writeWebRTCError(): " + err.Error())
 	}
 }
 
@@ -80,7 +80,7 @@ func WebsocketHandler(c *gin.Context, configuration *models.Configuration, commu
 		var message Message
 		err = conn.ReadJSON(&message)
 		if err != nil {
-			log.Log.Error("routers.websocket.main.WebsocketHandler(): " + err.Error())
+			log.Error("routers.websocket.main.WebsocketHandler(): " + err.Error())
 			return
 		}
 		clientID := message.ClientID
@@ -89,7 +89,7 @@ func WebsocketHandler(c *gin.Context, configuration *models.Configuration, commu
 			connection.Socket = conn
 			sockets[clientID] = connection
 			sockets[clientID].Cancels = make(map[string]context.CancelFunc)
-			log.Log.Info("routers.websocket.main.WebsocketHandler(): " + clientID + ": connected.")
+			log.Info("routers.websocket.main.WebsocketHandler(): " + clientID + ": connected.")
 		}
 
 		// Continuously read messages
@@ -111,14 +111,14 @@ func WebsocketHandler(c *gin.Context, configuration *models.Configuration, commu
 				if exists {
 					sockets[clientID].Cancels["stream-sd"]()
 				} else {
-					log.Log.Error("routers.websocket.main.WebsocketHandler(): streaming sd does not exists for " + clientID)
+					log.Error("routers.websocket.main.WebsocketHandler(): streaming sd does not exists for " + clientID)
 				}
 
 			case "stream-sd":
-				if communication.CameraConnected {
+				if communication.CameraConnected.Load() {
 					_, exists := sockets[clientID].Cancels["stream-sd"]
 					if exists {
-						log.Log.Debug("routers.websocket.main.WebsocketHandler(): already streaming sd for " + clientID)
+						log.Debug("routers.websocket.main.WebsocketHandler(): already streaming sd for " + clientID)
 					} else {
 						startStream := Message{
 							ClientID:    clientID,
@@ -144,16 +144,11 @@ func WebsocketHandler(c *gin.Context, configuration *models.Configuration, commu
 					break
 				}
 
-				if !communication.CameraConnected {
+				if !communication.CameraConnected.Load() {
 					writeWebRTCError(sockets[clientID], clientID, sessionID, "camera is not connected")
 					break
 				}
 
-				if communication.HandleLiveHDHandshake == nil {
-					writeWebRTCError(sockets[clientID], clientID, sessionID, "webrtc liveview is not available")
-					break
-				}
-
 				handshake := models.LiveHDHandshake{
 					Payload: models.RequestHDStreamPayload{
 						Timestamp:          time.Now().Unix(),
@@ -188,7 +183,9 @@ func WebsocketHandler(c *gin.Context, configuration *models.Configuration, commu
 					},
 				}
 
-				communication.HandleLiveHDHandshake <- handshake
+				if !communication.TrySendLiveHDHandshake(handshake) {
+					writeWebRTCError(sockets[clientID], clientID, handshake.Payload.SessionID, "camera is restarting or live-view queue is full")
+				}
 
 			case "webrtc-candidate":
 				sessionID := message.Message["session_id"]
@@ -199,7 +196,7 @@ func WebsocketHandler(c *gin.Context, configuration *models.Configuration, commu
 					break
 				}
 
-				if !communication.CameraConnected {
+				if !communication.CameraConnected.Load() {
 					writeWebRTCError(sockets[clientID], clientID, sessionID, "camera is not connected")
 					break
 				}
@@ -221,7 +218,7 @@ func WebsocketHandler(c *gin.Context, configuration *models.Configuration, commu
 		_, exists := sockets[clientID]
 		if exists {
 			delete(sockets, clientID)
-			log.Log.Info("routers.websocket.main.WebsocketHandler(): " + clientID + ": terminated and disconnected websocket connection.")
+			log.Info("routers.websocket.main.WebsocketHandler(): " + clientID + ": terminated and disconnected websocket connection.")
 		}
 	}
 }
@@ -232,13 +229,14 @@ func ForwardSDStream(ctx context.Context, clientID string, connection *Connectio
 	var cursor *packets.QueueCursor
 
 	// We'll pick the right client and decoder.
-	rtspClient := captureDevice.RTSPSubClient
+	rtspClient := captureDevice.SubClient()
 	if rtspClient != nil {
-		queue = communication.SubQueue
-		cursor = queue.Latest()
+		queue = communication.SubQueue()
 	} else {
-		rtspClient = captureDevice.RTSPClient
-		queue = communication.Queue
+		rtspClient = captureDevice.MainClient()
+		queue = communication.MainQueue()
+	}
+	if queue != nil {
 		cursor = queue.Latest()
 	}
 
@@ -263,7 +261,7 @@ logreader:
 					continue
 				}
 			} else {
-				log.Log.Error("routers.websocket.main.ForwardSDStream():" + err.Error())
+				log.Error("routers.websocket.main.ForwardSDStream():" + err.Error())
 				break logreader
 			}
 		}
@@ -277,7 +275,7 @@ logreader:
 		}
 		err := connection.WriteJson(startStrean)
 		if err != nil {
-			log.Log.Error("routers.websocket.main.ForwardSDStream():" + err.Error())
+			log.Error("routers.websocket.main.ForwardSDStream():" + err.Error())
 			break logreader
 		}
 		select {
@@ -292,9 +290,9 @@ logreader:
 	if exists {
 		delete(connection.Cancels, "stream-sd")
 	} else {
-		log.Log.Error("routers.websocket.main.ForwardSDStream(): streaming sd does not exists for " + clientID)
+		log.Error("routers.websocket.main.ForwardSDStream(): streaming sd does not exists for " + clientID)
 	}
 
 	// Send stop streaming message
-	log.Log.Info("routers.websocket.main.ForwardSDStream(): stop sending streaming over websocket")
+	log.Info("routers.websocket.main.ForwardSDStream(): stop sending streaming over websocket")
 }

machinery/src/utils/main.go

@@ -1,7 +1,6 @@
 package utils
 
 import (
-	"bufio"
 	"bytes"
 	"errors"
 	"fmt"
@@ -12,15 +11,14 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
-	"reflect"
 	"sort"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/encryption"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 
 	"github.com/nfnt/resize"
 )
@@ -140,11 +138,11 @@ func CheckDataDirectoryPermissions(configDirectory string) error {
 	}
 
 	if err != nil {
-		log.Log.Error("Checking data directory permissions: " + err.Error())
+		log.Error("Checking data directory permissions: " + err.Error())
 		return err
 	}
 
-	log.Log.Info("Checking data directory permissions: OK")
+	log.Info("Checking data directory permissions: OK")
 	return nil
 }
 
@@ -301,16 +299,16 @@ func CreateFragmentedMP4(fullName string, fragmentedDuration int64) {
 	// This timescale is crucial, as it should be the same as the one defined in JOY4.
 	cmd := exec.Command("mp4fragment", "--timescale", "10000000", "--fragment-duration", strconv.FormatInt(duration, 10), fullName, fullName+"f.mp4")
 	cmd.Dir = path
-	log.Log.Info(cmd.String())
+	log.Info(cmd.String())
 	var out bytes.Buffer
 	var stderr bytes.Buffer
 	cmd.Stdout = &out
 	cmd.Stderr = &stderr
 	err := cmd.Run()
 	if err != nil {
-		log.Log.Error(fmt.Sprint(err) + ": " + stderr.String())
+		log.Error(fmt.Sprint(err) + ": " + stderr.String())
 	} else {
-		log.Log.Info("Created Fragmented: " + out.String())
+		log.Info("Created Fragmented: " + out.String())
 	}
 
 	// We will swap the files.
@@ -319,42 +317,93 @@ func CreateFragmentedMP4(fullName string, fragmentedDuration int64) {
 }
 
 func PrintEnvironmentVariables() {
-	// Print environment variables that include "AGENT_" as a prefix.
-	environmentVariables := ""
-	for _, e := range os.Environ() {
-		if strings.Contains(e, "AGENT_") {
-			pair := strings.Split(e, "=")
-			environmentVariables = environmentVariables + pair[0] + "=" + pair[1] + " "
+	names := agentEnvironmentVariableNames(os.Environ())
+	log.WithFields(log.Fields{
+		"component":      "configuration",
+		"event":          "environment_loaded",
+		"variable_count": len(names),
+	}).Info("Agent environment loaded")
+	log.WithFields(log.Fields{
+		"component": "configuration",
+		"event":     "environment_details",
+		"variables": names,
+	}).Debug("Agent environment details")
+}
+
+func agentEnvironmentVariableNames(environment []string) []string {
+	names := make([]string, 0)
+	for _, entry := range environment {
+		name, _, found := strings.Cut(entry, "=")
+		if found && strings.HasPrefix(name, "AGENT_") {
+			names = append(names, name)
 		}
 	}
-	log.Log.Info("Printing out environmentVariables (AGENT_...): " + environmentVariables)
+	sort.Strings(names)
+	return names
 }
 
 func PrintConfiguration(configuration *models.Configuration) {
-	// We will print out the struct.
 	if configuration == nil {
-		log.Log.Info("Configuration is nil")
+		log.WithFields(log.Fields{
+			"component": "configuration",
+			"event":     "configuration_missing",
+		}).Warn("Agent configuration is nil")
 		return
 	}
-	config := configuration.Config
-	// Iterate over the struct and printout the values.
-	v := reflect.ValueOf(config)
-	typeOfS := v.Type()
-	configurationVariables := ""
-	for i := 0; i < v.NumField(); i++ {
-		key := typeOfS.Field(i).Name
-		value := v.Field(i).Interface()
-		// Convert to string.
-		configurationVariables = configurationVariables + key + ": " + fmt.Sprintf("%v", value) + " "
+
+	log.WithFields(configurationLogFields(configuration.Config)).
+		Info("Agent configuration loaded")
+	log.WithFields(configurationDebugLogFields(configuration.Config)).
+		Debug("Agent configuration details")
+}
+
+func configurationLogFields(config models.Config) log.Fields {
+	return log.Fields{
+		"agent_name":     config.Name,
+		"camera_type":    config.Capture.Name,
+		"cloud_provider": config.Cloud,
+		"component":      "configuration",
+		"event":          "configuration_loaded",
+		"friendly_name":  config.FriendlyName,
+		"liveview":       config.Capture.Liveview,
+		"offline":        config.Offline,
+		"recording":      config.Capture.Recording,
+		"timezone":       config.Timezone,
+	}
+}
+
+func configurationDebugLogFields(config models.Config) log.Fields {
+	return log.Fields{
+		"auto_clean":             config.AutoClean,
+		"component":              "configuration",
+		"continuous":             config.Capture.Continuous,
+		"event":                  "configuration_details",
+		"gop_size":               config.Capture.GopSize,
+		"main_fps":               config.Capture.IPCamera.FPS,
+		"main_height":            config.Capture.IPCamera.Height,
+		"main_stream_configured": config.Capture.IPCamera.RTSP != "",
+		"main_width":             config.Capture.IPCamera.Width,
+		"max_recording_seconds":  config.Capture.MaxLengthRecording,
+		"motion":                 config.Capture.Motion,
+		"onvif_configured":       config.Capture.IPCamera.ONVIFXAddr != "",
+		"onvif_motion":           config.Capture.ONVIFMotion,
+		"post_recording_seconds": config.Capture.PostRecording,
+		"pre_recording_seconds":  config.Capture.PreRecording,
+		"remove_after_upload":    config.RemoveAfterUpload,
+		"snapshots":              config.Capture.Snapshots,
+		"sub_fps":                config.Capture.IPCamera.SubFPS,
+		"sub_height":             config.Capture.IPCamera.SubHeight,
+		"sub_stream_configured":  config.Capture.IPCamera.SubRTSP != "",
+		"sub_width":              config.Capture.IPCamera.SubWidth,
+		"transcoding_webrtc":     config.Capture.TranscodingWebRTC,
 	}
-	log.Log.Info("Printing our configuration (config.json): " + configurationVariables)
 }
 
 func Decrypt(directoryOrFile string, symmetricKey []byte) {
 	// Check if file or directory
 	fileInfo, err := os.Stat(directoryOrFile)
 	if err != nil {
-		log.Log.Fatal(err.Error())
+		log.Fatal(err.Error())
 		return
 	}
 
@@ -363,12 +412,12 @@ func Decrypt(directoryOrFile string, symmetricKey []byte) {
 		// Create decrypted directory
 		err = os.MkdirAll(directoryOrFile+"/decrypted", 0755)
 		if err != nil {
-			log.Log.Fatal(err.Error())
+			log.Fatal(err.Error())
 			return
 		}
 		dir, err := os.ReadDir(directoryOrFile)
 		if err != nil {
-			log.Log.Fatal(err.Error())
+			log.Fatal(err.Error())
 			return
 		}
 		for _, file := range dir {
@@ -390,13 +439,13 @@ func Decrypt(directoryOrFile string, symmetricKey []byte) {
 		// Read file
 		content, err := os.ReadFile(file)
 		if err != nil {
-			log.Log.Fatal(err.Error())
+			log.Fatal(err.Error())
 			return
 		}
 		// Decrypt using AES key
 		decrypted, err := encryption.AesDecrypt(content, string(symmetricKey))
 		if err != nil {
-			log.Log.Fatal("Something went wrong while decrypting: " + err.Error())
+			log.Fatal("Something went wrong while decrypting: " + err.Error())
 			return
 		}
 
@@ -408,7 +457,7 @@ func Decrypt(directoryOrFile string, symmetricKey []byte) {
 
 		err = os.WriteFile(pathToFile+"/decrypted/"+fileName, []byte(decrypted), 0644)
 		if err != nil {
-			log.Log.Fatal(err.Error())
+			log.Fatal(err.Error())
 			return
 		}
 	}
@@ -416,9 +465,12 @@ func Decrypt(directoryOrFile string, symmetricKey []byte) {
 
 func ImageToBytes(img *image.Image) ([]byte, error) {
 	buffer := new(bytes.Buffer)
-	w := bufio.NewWriter(buffer)
-	err := jpeg.Encode(w, *img, &jpeg.Options{Quality: 35})
-	log.Log.Debug("ImageToBytes() - buffer size: " + strconv.Itoa(buffer.Len()))
+	err := jpeg.Encode(buffer, *img, &jpeg.Options{Quality: 35})
+	log.WithFields(log.Fields{
+		"bytes":     buffer.Len(),
+		"component": "image",
+		"event":     "jpeg_encoded",
+	}).Debug("Image encoded as JPEG")
 	return buffer.Bytes(), err
 }
 

machinery/src/utils/main_test.go

@@ -1,13 +1,112 @@
 package utils
 
 import (
+	"bytes"
+	"fmt"
+	"image"
+	"image/color"
+	"image/jpeg"
 	"os"
+	"reflect"
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/models"
 )
 
+func TestImageToBytesReturnsCompleteJPEG(t *testing.T) {
+	source := image.NewRGBA(image.Rect(0, 0, 16, 12))
+	source.Set(8, 6, color.RGBA{R: 255, A: 255})
+	var input image.Image = source
+
+	encoded, err := ImageToBytes(&input)
+	if err != nil {
+		t.Fatalf("ImageToBytes() error = %v", err)
+	}
+	if len(encoded) == 0 {
+		t.Fatal("ImageToBytes() returned an empty JPEG")
+	}
+
+	decoded, err := jpeg.Decode(bytes.NewReader(encoded))
+	if err != nil {
+		t.Fatalf("decoding ImageToBytes() output: %v", err)
+	}
+	if got := decoded.Bounds().Size(); got.X != 16 || got.Y != 12 {
+		t.Fatalf("decoded JPEG size = %dx%d, want 16x12", got.X, got.Y)
+	}
+}
+
+func TestAgentEnvironmentVariableNamesOmitsValues(t *testing.T) {
+	got := agentEnvironmentVariableNames([]string{
+		"AGENT_HUB_PRIVATE_KEY=do-not-log",
+		"PATH=/usr/bin",
+		"NOT_AGENT_SECRET=also-do-not-log",
+		"AGENT_CAPTURE_LIVEVIEW=true",
+	})
+	want := []string{"AGENT_CAPTURE_LIVEVIEW", "AGENT_HUB_PRIVATE_KEY"}
+
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("agentEnvironmentVariableNames() = %v, want %v", got, want)
+	}
+}
+
+func TestConfigurationLogFieldsOmitCredentialsAndEndpoints(t *testing.T) {
+	config := models.Config{
+		Name:          "Front Door",
+		FriendlyName:  "Entrance",
+		Cloud:         "kstorage",
+		HubKey:        "hub-key-secret",
+		HubPrivateKey: "hub-private-secret",
+		MQTTURI:       "mqtt://internal.example",
+		MQTTUsername:  "mqtt-user",
+		MQTTPassword:  "mqtt-password-secret",
+		Capture: models.Capture{
+			Liveview:           "true",
+			MaxLengthRecording: 20,
+			IPCamera: models.IPCamera{
+				RTSP:          "rtsp://camera-user:camera-password@10.0.30.11/live",
+				ONVIFUsername: "onvif-user",
+				ONVIFPassword: "onvif-password-secret",
+			},
+		},
+	}
+
+	fields := []map[string]interface{}{
+		configurationLogFields(config),
+		configurationDebugLogFields(config),
+	}
+	summary := fmt.Sprint(fields)
+	for _, secret := range []string{
+		config.HubKey,
+		config.HubPrivateKey,
+		config.MQTTURI,
+		config.MQTTUsername,
+		config.MQTTPassword,
+		config.Capture.IPCamera.RTSP,
+		config.Capture.IPCamera.ONVIFUsername,
+		config.Capture.IPCamera.ONVIFPassword,
+	} {
+		if strings.Contains(summary, secret) {
+			t.Fatalf("configuration log fields exposed sensitive value %q in %q", secret, summary)
+		}
+	}
+	infoFields := configurationLogFields(config)
+	if got := infoFields["agent_name"]; got != "Front Door" {
+		t.Fatalf("agent_name = %v, want Front Door", got)
+	}
+	if got := infoFields["cloud_provider"]; got != "kstorage" {
+		t.Fatalf("cloud_provider = %v, want kstorage", got)
+	}
+	debugFields := configurationDebugLogFields(config)
+	if got := debugFields["max_recording_seconds"]; got != int64(20) {
+		t.Fatalf("max_recording_seconds = %v, want 20", got)
+	}
+	if got := debugFields["main_stream_configured"]; got != true {
+		t.Fatalf("main_stream_configured = %v, want true", got)
+	}
+}
+
 type stubFileInfo struct {
 	name string
 }

machinery/src/video/livehls.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 
 	mp4ff "github.com/Eyevinn/mp4ff/mp4"
-	"github.com/kerberos-io/agent/machinery/src/log"
+	log "github.com/sirupsen/logrus"
 )
 
 // LiveSegmenter turns a live stream of Annex B video samples into HLS-ready
@@ -217,7 +217,7 @@ func (ls *LiveSegmenter) buildInit() error {
 		// fall back to a manually built avcC just like the recording muxer does so
 		// those cameras still produce a valid init segment.
 		if err := trak.SetAVCDescriptor("avc1", sps, pps, true); err != nil {
-			log.Log.Warning("livehls: SetAVCDescriptor failed, using manual avcC fallback: " + err.Error())
+			log.Warn("livehls: SetAVCDescriptor failed, using manual avcC fallback: " + err.Error())
 			if fbErr := addAVCDescriptorFallback(trak, sps, pps, ls.width, ls.height); fbErr != nil {
 				return fmt.Errorf("livehls: AVC descriptor fallback: %w", fbErr)
 			}
@@ -279,7 +279,7 @@ func (ls *LiveSegmenter) WriteSample(isKeyframe bool, annexB []byte, ptsMs uint6
 	// A session must open on a random-access point; otherwise the first segment
 	// would reference frames that never arrived.
 	if !ls.started && !isKeyframe {
-		log.Log.Debug("LiveSegmenter.WriteSample(): dropping leading non-keyframe before first IDR")
+		log.Debug("LiveSegmenter.WriteSample(): dropping leading non-keyframe before first IDR")
 		return nil
 	}
 
@@ -352,7 +352,7 @@ func (ls *LiveSegmenter) openSegment(startPTS uint64) {
 	ls.seg = mp4ff.NewMediaSegment() // includes a CMAF styp box by default
 	frag, err := mp4ff.CreateFragment(ls.seqNr, ls.videoTrackID)
 	if err != nil {
-		log.Log.Error("LiveSegmenter.openSegment(): CreateFragment failed: " + err.Error())
+		log.Error("LiveSegmenter.openSegment(): CreateFragment failed: " + err.Error())
 		return
 	}
 	ls.seg.AddFragment(frag)
@@ -543,7 +543,7 @@ func (ls *LiveSegmenter) openPartFragment() {
 	ls.fragSeq++
 	frag, err := mp4ff.CreateFragment(ls.fragSeq, ls.videoTrackID)
 	if err != nil {
-		log.Log.Error("LiveSegmenter.openPartFragment(): CreateFragment failed: " + err.Error())
+		log.Error("LiveSegmenter.openPartFragment(): CreateFragment failed: " + err.Error())
 		return
 	}
 	ls.partFrag = frag

machinery/src/video/mp4.go

@@ -16,9 +16,9 @@ import (
 	"github.com/Eyevinn/mp4ff/avc"
 	mp4ff "github.com/Eyevinn/mp4ff/mp4"
 	"github.com/kerberos-io/agent/machinery/src/encryption"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/utils"
+	log "github.com/sirupsen/logrus"
 )
 
 var LastPTS uint64 = 0 // Last PTS for the current segment
@@ -216,13 +216,13 @@ func (mp4 *MP4) updateVideoParameterSetsFromAnnexB(data []byte) {
 				if needSPS {
 					mp4.SPSNALUs = [][]byte{nalu}
 					needSPS = false
-					log.Log.Warning("mp4.updateVideoParameterSetsFromAnnexB(): SPS recovered from in-band NALU")
+					log.Warn("mp4.updateVideoParameterSetsFromAnnexB(): SPS recovered from in-band NALU")
 				}
 			case 8: // PPS
 				if needPPS {
 					mp4.PPSNALUs = [][]byte{nalu}
 					needPPS = false
-					log.Log.Warning("mp4.updateVideoParameterSetsFromAnnexB(): PPS recovered from in-band NALU")
+					log.Warn("mp4.updateVideoParameterSetsFromAnnexB(): PPS recovered from in-band NALU")
 				}
 			}
 		case "H265", "HVC1":
@@ -232,19 +232,19 @@ func (mp4 *MP4) updateVideoParameterSetsFromAnnexB(data []byte) {
 				if needVPS {
 					mp4.VPSNALUs = [][]byte{nalu}
 					needVPS = false
-					log.Log.Warning("mp4.updateVideoParameterSetsFromAnnexB(): VPS recovered from in-band NALU")
+					log.Warn("mp4.updateVideoParameterSetsFromAnnexB(): VPS recovered from in-band NALU")
 				}
 			case 33: // SPS
 				if needSPS {
 					mp4.SPSNALUs = [][]byte{nalu}
 					needSPS = false
-					log.Log.Warning("mp4.updateVideoParameterSetsFromAnnexB(): SPS recovered from in-band NALU")
+					log.Warn("mp4.updateVideoParameterSetsFromAnnexB(): SPS recovered from in-band NALU")
 				}
 			case 34: // PPS
 				if needPPS {
 					mp4.PPSNALUs = [][]byte{nalu}
 					needPPS = false
-					log.Log.Warning("mp4.updateVideoParameterSetsFromAnnexB(): PPS recovered from in-band NALU")
+					log.Warn("mp4.updateVideoParameterSetsFromAnnexB(): PPS recovered from in-band NALU")
 				}
 			}
 		}
@@ -266,7 +266,12 @@ func (mp4 *MP4) flushPendingVideoSample(nextPTS uint64) bool {
 	} else {
 		// No valid nextPTS (Close case) or PTS went backwards (jitter/discontinuity)
 		if nextPTS > 0 {
-			log.Log.Warning(fmt.Sprintf("mp4.flushPendingVideoSample(): video PTS went backwards or zero duration (nextPTS=%d, prevDTS=%d), using last known duration", nextPTS, mp4.VideoFullSample.DecodeTime))
+			log.WithFields(log.Fields{
+				"component":                 "video/mp4",
+				"event":                     "invalid_video_timestamp",
+				"next_presentation_time_ms": nextPTS,
+				"previous_decode_time_ms":   mp4.VideoFullSample.DecodeTime,
+			}).Warn("Video timestamp moved backwards or produced zero duration")
 		}
 		duration = mp4.LastVideoSampleDTS
 		if duration == 0 {
@@ -282,15 +287,23 @@ func (mp4 *MP4) flushPendingVideoSample(nextPTS uint64) bool {
 	isKF := mp4.PendingSampleIsKeyframe
 	err := mp4.MultiTrackFragment.AddFullSampleToTrack(*mp4.VideoFullSample, uint32(mp4.VideoTrack))
 	if err != nil {
-		log.Log.Error("mp4.flushPendingVideoSample(): error adding sample: " + err.Error())
+		log.Error("mp4.flushPendingVideoSample(): error adding sample: " + err.Error())
 	} else {
 		mp4.SampleCount++
 	}
 	if isKF {
 		mp4.TotalKeyframesWritten++
 		mp4.FragmentKeyframeCount++
-		log.Log.Debug(fmt.Sprintf("mp4.flushPendingVideoSample(): KEYFRAME WRITTEN to trun - totalWritten=%d, fragmentKF=%d, flags=0x%08x, dur=%d, DTS=%d",
-			mp4.TotalKeyframesWritten, mp4.FragmentKeyframeCount, mp4.VideoFullSample.Sample.Flags, duration, mp4.VideoFullSample.DecodeTime))
+		log.WithFields(log.Fields{
+			"component":               "video/mp4",
+			"decode_time_ms":          mp4.VideoFullSample.DecodeTime,
+			"duration_ms":             duration,
+			"event":                   "keyframe_written",
+			"fragment_keyframe_count": mp4.FragmentKeyframeCount,
+			"sample_flags":            mp4.VideoFullSample.Sample.Flags,
+			"total_keyframes_written": mp4.TotalKeyframesWritten,
+		}).Debug("MP4 keyframe written")
+
 	}
 
 	mp4.VideoFullSample = nil
@@ -378,7 +391,14 @@ func (mp4 *MP4) AddSampleToTrack(trackID uint32, isKeyframe bool, data []byte, p
 		truncatedTail := fullGopFrames > 0 && bufferedVideo*2 < fullGopFrames
 		if closeKeyframe && truncatedTail {
 			seam = true
-			log.Log.Warning(fmt.Sprintf("mp4.AddSampleToTrack(): dropping truncated GOP at premature keyframe (interval=%d ms, min interval=%d ms, buffered video frames=%d of ~%d) - likely upstream loop/restart discontinuity", gap, mp4.MinKeyframeGapMs, bufferedVideo, fullGopFrames))
+			log.WithFields(log.Fields{
+				"buffered_video_frames":   bufferedVideo,
+				"component":               "video/mp4",
+				"event":                   "truncated_gop_dropped",
+				"expected_gop_frames":     fullGopFrames,
+				"keyframe_interval_ms":    gap,
+				"minimum_keyframe_gap_ms": mp4.MinKeyframeGapMs,
+			}).Warn("Dropping truncated GOP at premature keyframe")
 		}
 		mp4.LastKeyframeGapMs = gap
 		if !seam && (mp4.MinKeyframeGapMs == 0 || gap < mp4.MinKeyframeGapMs) {
@@ -483,7 +503,7 @@ func (mp4 *MP4) commitBufferedGOP() {
 	mp4.gopBuffer = nil // detach so commitSampleToTrack never observes a half-cleared buffer
 	for _, s := range buffered {
 		if err := mp4.commitSampleToTrack(s.trackID, s.isKeyframe, s.data, s.pts, s.compositionOffset); err != nil {
-			log.Log.Error("mp4.commitBufferedGOP(): " + err.Error())
+			log.Error("mp4.commitBufferedGOP(): " + err.Error())
 		}
 	}
 }
@@ -499,8 +519,18 @@ func (mp4 *MP4) commitSampleToTrack(trackID uint32, isKeyframe bool, data []byte
 		if mp4.Start {
 			elapsedDbg = pts - mp4.FragmentStartRawPTS
 		}
-		log.Log.Debug(fmt.Sprintf("mp4.AddSampleToTrack(): KEYFRAME #%d received - PTS=%d, size=%d, elapsed=%dms, started=%t, segment=%d, fragKF=%d",
-			mp4.TotalKeyframesReceived, pts, len(data), elapsedDbg, mp4.Start, mp4.SegmentCount, mp4.FragmentKeyframeCount))
+		log.WithFields(log.Fields{
+			"component":                "video/mp4",
+			"elapsed_ms":               elapsedDbg,
+			"event":                    "keyframe_received",
+			"fragment_keyframe_count":  mp4.FragmentKeyframeCount,
+			"presentation_time_ms":     pts,
+			"recording_started":        mp4.Start,
+			"sample_bytes":             len(data),
+			"segment_count":            mp4.SegmentCount,
+			"total_keyframes_received": mp4.TotalKeyframesReceived,
+		}).Debug("MP4 keyframe received")
+
 	}
 
 	if isKeyframe {
@@ -524,8 +554,15 @@ func (mp4 *MP4) commitSampleToTrack(trackID uint32, isKeyframe bool, data []byte
 					mp4.flushPendingVideoSample(pts)
 				}
 
-				log.Log.Debug(fmt.Sprintf("mp4.AddSampleToTrack(): FLUSHING segment #%d - keyframes_in_fragment=%d, totalKF_received=%d, totalKF_written=%d",
-					mp4.SegmentCount, mp4.FragmentKeyframeCount, mp4.TotalKeyframesReceived, mp4.TotalKeyframesWritten))
+				log.WithFields(log.Fields{
+					"component":                "video/mp4",
+					"event":                    "segment_flush_started",
+					"fragment_keyframe_count":  mp4.FragmentKeyframeCount,
+					"segment_count":            mp4.SegmentCount,
+					"total_keyframes_received": mp4.TotalKeyframesReceived,
+					"total_keyframes_written":  mp4.TotalKeyframesWritten,
+				}).Debug("Flushing MP4 segment")
+
 				mp4.MoofBoxes = mp4.MoofBoxes + 1
 				mp4.MoofBoxSizes = append(mp4.MoofBoxSizes, int64(mp4.Segment.Size()))
 				// Track the segment's duration and base decode time for sidx.
@@ -536,7 +573,7 @@ func (mp4 *MP4) commitSampleToTrack(trackID uint32, isKeyframe bool, data []byte
 				mp4.SegmentBaseDecTimes = append(mp4.SegmentBaseDecTimes, mp4.FragmentStartDTS)
 				err := mp4.Segment.Encode(mp4.Writer)
 				if err != nil {
-					log.Log.Error("mp4.AddSampleToTrack(): error encoding segment: " + err.Error())
+					log.Error("mp4.AddSampleToTrack(): error encoding segment: " + err.Error())
 				}
 				mp4.Segments = append(mp4.Segments, mp4.Segment)
 			}
@@ -552,7 +589,7 @@ func (mp4 *MP4) commitSampleToTrack(trackID uint32, isKeyframe bool, data []byte
 			// Create a video fragment
 			multiTrackFragment, err := mp4ff.CreateMultiTrackFragment(uint32(mp4.SegmentCount), mp4.TrackIDs)
 			if err != nil {
-				log.Log.Error("mp4.AddSampleToTrack(): error creating multi track fragment: " + err.Error())
+				log.Error("mp4.AddSampleToTrack(): error creating multi track fragment: " + err.Error())
 			}
 			mp4.MultiTrackFragment = multiTrackFragment
 			seg.AddFragment(multiTrackFragment)
@@ -585,7 +622,14 @@ func (mp4 *MP4) commitSampleToTrack(trackID uint32, isKeyframe bool, data []byte
 			if err == nil {
 				// Flush previous pending sample before storing the new one
 				if mp4.VideoFullSample != nil {
-					log.Log.Debug("Adding sample to track " + fmt.Sprintf("%d, PTS: %d, size: %d, Keyframe: %t", trackID, pts, len(lengthPrefixed), isKeyframe))
+					log.WithFields(log.Fields{
+						"bytes":     len(lengthPrefixed),
+						"component": "mp4",
+						"event":     "video_sample_added",
+						"keyframe":  isKeyframe,
+						"pts":       pts,
+						"track_id":  trackID,
+					}).Trace("Adding MP4 video sample")
 					mp4.flushPendingVideoSample(pts)
 				}
 
@@ -628,7 +672,11 @@ func (mp4 *MP4) commitSampleToTrack(trackID uint32, isKeyframe bool, data []byte
 					sampleToAdd.Sample.Size = uint32(len(aac[7:]))
 					err := mp4.MultiTrackFragment.AddFullSampleToTrack(sampleToAdd, trackID)
 					if err != nil {
-						log.Log.Error("mp4.AddSampleToTrack(): error adding sample to track " + fmt.Sprintf("%d: %v", trackID, err))
+						log.WithError(err).WithFields(log.Fields{
+							"component": "video/mp4",
+							"event":     "audio_sample_write_failed",
+							"track_id":  trackID,
+						}).Error("Failed to add audio sample to MP4 track")
 					}
 				})
 			}
@@ -657,11 +705,17 @@ func (mp4 *MP4) Close(config *models.Config) {
 	// a recording is never a loop seam, so it must always be written out.
 	mp4.commitBufferedGOP()
 
-	log.Log.Info(fmt.Sprintf("mp4.Close(): KEYFRAME SUMMARY - totalReceived=%d, totalWritten=%d, segments=%d, lastFragmentKF=%d",
-		mp4.TotalKeyframesReceived, mp4.TotalKeyframesWritten, mp4.SegmentCount, mp4.FragmentKeyframeCount))
+	log.WithFields(log.Fields{
+		"component":                "video/mp4",
+		"event":                    "recording_keyframe_summary",
+		"fragment_keyframe_count":  mp4.FragmentKeyframeCount,
+		"segment_count":            mp4.SegmentCount,
+		"total_keyframes_received": mp4.TotalKeyframesReceived,
+		"total_keyframes_written":  mp4.TotalKeyframesWritten,
+	}).Info("MP4 recording keyframe summary")
 
 	if mp4.VideoTotalDuration == 0 && mp4.AudioTotalDuration == 0 {
-		log.Log.Error("mp4.Close(): no video or audio samples added, removing empty MP4 file")
+		log.Error("mp4.Close(): no video or audio samples added, removing empty MP4 file")
 		mp4.Writer.Flush()
 		_ = mp4.FileWriter.Sync()
 		_ = mp4.FileWriter.Close()
@@ -690,7 +744,7 @@ func (mp4 *MP4) Close(config *models.Config) {
 				sampleToAdd.Sample.Size = uint32(len(aac[7:]))
 				err := mp4.MultiTrackFragment.AddFullSampleToTrack(sampleToAdd, uint32(mp4.AudioTrack))
 				if err != nil {
-					log.Log.Error("mp4.Close(): error adding final audio sample: " + err.Error())
+					log.Error("mp4.Close(): error adding final audio sample: " + err.Error())
 				}
 			})
 			mp4.AudioFullSample = nil
@@ -712,14 +766,14 @@ func (mp4 *MP4) Close(config *models.Config) {
 
 		err := mp4.Segment.Encode(mp4.Writer)
 		if err != nil {
-			log.Log.Error("mp4.Close(): error encoding last segment: " + err.Error())
+			log.Error("mp4.Close(): error encoding last segment: " + err.Error())
 		}
 	}
 
 	mp4.Writer.Flush()
 	// Ensure all segment data is on disk before we overwrite the placeholder at offset 0.
 	if err := mp4.FileWriter.Sync(); err != nil {
-		log.Log.Error("mp4.Close(): error syncing file: " + err.Error())
+		log.Error("mp4.Close(): error syncing file: " + err.Error())
 	}
 
 	// Now we have all the moof and mdat boxes written to the file.
@@ -747,8 +801,14 @@ func (mp4 *MP4) Close(config *models.Config) {
 		actualVideoDuration += d
 	}
 	if actualVideoDuration != mp4.VideoTotalDuration {
-		log.Log.Warning(fmt.Sprintf("mp4.Close(): duration mismatch: accumulated VideoTotalDuration=%d, sum of segment durations=%d (diff=%d ms)",
-			mp4.VideoTotalDuration, actualVideoDuration, int64(mp4.VideoTotalDuration)-int64(actualVideoDuration)))
+		log.WithFields(log.Fields{
+			"accumulated_duration_ms": mp4.VideoTotalDuration,
+			"component":               "video/mp4",
+			"difference_ms":           int64(mp4.VideoTotalDuration) - int64(actualVideoDuration),
+			"event":                   "duration_mismatch",
+			"segment_duration_ms":     actualVideoDuration,
+		}).Warn("MP4 video duration does not match segment durations")
+
 	}
 
 	// Set the creation time and modification time for the moov box.
@@ -790,21 +850,40 @@ func (mp4 *MP4) Close(config *models.Config) {
 		init.AddEmptyTrack(videoTimescale, "video", "und")
 		includePS := true
 		spsNALUs, ppsNALUs := normalizeH264ParameterSets(mp4.SPSNALUs, mp4.PPSNALUs)
-		log.Log.Debug("mp4.Close(): AVC parameter sets: SPS=" + formatNaluDebug(spsNALUs) + ", PPS=" + formatNaluDebug(ppsNALUs))
+		log.WithFields(log.Fields{
+			"component": "video/mp4",
+			"event":     "avc_parameter_sets",
+			"pps":       formatNaluDebug(ppsNALUs),
+			"sps":       formatNaluDebug(spsNALUs),
+		}).Debug("AVC parameter sets normalized")
 		if len(spsNALUs) == 0 || len(ppsNALUs) == 0 {
 			// An avcC without both SPS and PPS is invalid: downstream FFmpeg-based
 			// pipelines decoding this file will report "non-existing PPS 0 referenced"
 			// and fail to extract any frame. Surface it loudly so the capture-side
 			// parameter-set handling can be diagnosed.
-			log.Log.Error(fmt.Sprintf("mp4.Close(): incomplete H264 parameter sets (SPS=%d, PPS=%d) - the avcC will be invalid and downstream decoders will report 'non-existing PPS 0 referenced'", len(spsNALUs), len(ppsNALUs)))
+			log.WithFields(log.Fields{
+				"component": "video/mp4",
+				"event":     "avc_parameter_sets_incomplete",
+				"pps_count": len(ppsNALUs),
+				"sps_count": len(spsNALUs),
+			}).Error("Incomplete H264 parameter sets will produce an invalid AVC descriptor")
 		}
 		err := init.Moov.Traks[0].SetAVCDescriptor("avc1", spsNALUs, ppsNALUs, includePS)
 		if err != nil {
-			log.Log.Error("mp4.Close(): error setting AVC descriptor: " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component": "video/mp4",
+				"event":     "avc_descriptor_failed",
+			}).Error("Failed to set AVC descriptor")
 			if fallbackErr := addAVCDescriptorFallback(init.Moov.Traks[0], spsNALUs, ppsNALUs, uint16(mp4.width), uint16(mp4.height)); fallbackErr != nil {
-				log.Log.Error("mp4.Close(): error setting AVC descriptor fallback: " + fallbackErr.Error())
+				log.WithError(fallbackErr).WithFields(log.Fields{
+					"component": "video/mp4",
+					"event":     "avc_descriptor_fallback_failed",
+				}).Error("Failed to set fallback AVC descriptor")
 			} else {
-				log.Log.Warning("mp4.Close(): AVC descriptor fallback used due to SPS parse error")
+				log.WithFields(log.Fields{
+					"component": "video/mp4",
+					"event":     "avc_descriptor_fallback_used",
+				}).Warn("Using fallback AVC descriptor")
 			}
 		}
 		init.Moov.Traks[0].Tkhd.Duration = actualVideoDuration
@@ -823,15 +902,30 @@ func (mp4 *MP4) Close(config *models.Config) {
 		init.AddEmptyTrack(videoTimescale, "video", "und")
 		includePS := true
 		vpsNALUs, spsNALUs, ppsNALUs := normalizeH265ParameterSets(mp4.VPSNALUs, mp4.SPSNALUs, mp4.PPSNALUs)
-		log.Log.Debug("mp4.Close(): HEVC parameter sets: VPS=" + formatNaluDebug(vpsNALUs) + ", SPS=" + formatNaluDebug(spsNALUs) + ", PPS=" + formatNaluDebug(ppsNALUs))
+		log.WithFields(log.Fields{
+			"component": "video/mp4",
+			"event":     "hevc_parameter_sets",
+			"pps":       formatNaluDebug(ppsNALUs),
+			"sps":       formatNaluDebug(spsNALUs),
+			"vps":       formatNaluDebug(vpsNALUs),
+		}).Debug("HEVC parameter sets normalized")
 		if len(vpsNALUs) == 0 || len(spsNALUs) == 0 || len(ppsNALUs) == 0 {
 			// An hvcC missing VPS/SPS/PPS is invalid and downstream FFmpeg-based
 			// pipelines will fail to decode the recording. Surface it loudly.
-			log.Log.Error(fmt.Sprintf("mp4.Close(): incomplete H265 parameter sets (VPS=%d, SPS=%d, PPS=%d) - the hvcC will be invalid and downstream decoders will fail to process the recording", len(vpsNALUs), len(spsNALUs), len(ppsNALUs)))
+			log.WithFields(log.Fields{
+				"component": "video/mp4",
+				"event":     "hevc_parameter_sets_incomplete",
+				"pps_count": len(ppsNALUs),
+				"sps_count": len(spsNALUs),
+				"vps_count": len(vpsNALUs),
+			}).Error("Incomplete H265 parameter sets will produce an invalid HEVC descriptor")
 		}
 		err := init.Moov.Traks[0].SetHEVCDescriptor("hvc1", vpsNALUs, spsNALUs, ppsNALUs, [][]byte{}, includePS)
 		if err != nil {
-			log.Log.Error("mp4.Close(): error setting HEVC descriptor: " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component": "video/mp4",
+				"event":     "hevc_descriptor_failed",
+			}).Error("Failed to set HEVC descriptor")
 		}
 		init.Moov.Traks[0].Tkhd.Duration = actualVideoDuration
 		init.Moov.Traks[0].Tkhd.Width = mp4ff.Fixed32(uint32(mp4.width) << 16)
@@ -876,7 +970,7 @@ func (mp4 *MP4) Close(config *models.Config) {
 		// Set the subtitle descriptor
 		err := init.Moov.Traks[2].SetWvttDescriptor("")
 		if err != nil {
-			//log.Log.Error("mp4.Close(): error setting VTT descriptor: " + err.Error())
+			// log.Error("mp4.Close(): error setting VTT descriptor: " + err.Error())
 			//return
 		}
 		init.Moov.Traks[2].Mdia.Hdlr.Name = "agent " + utils.VERSION
@@ -924,14 +1018,14 @@ func (mp4 *MP4) Close(config *models.Config) {
 	block, _ := pem.Decode(pemBytes)
 
 	if block == nil {
-		//log.Log.Error("mp4.Close(): error decoding PEM block containing private key")
+		// log.Error("mp4.Close(): error decoding PEM block containing private key")
 		//return
 	} else {
 		// Parse private key
 		b := block.Bytes
 		key, err := x509.ParsePKCS8PrivateKey(b)
 		if err != nil {
-			//log.Log.Error("mp4.Close(): error parsing private key: " + err.Error())
+			// log.Error("mp4.Close(): error parsing private key: " + err.Error())
 			//return
 		} else {
 			// Conver key to *rsa.PrivateKey
@@ -944,7 +1038,7 @@ func (mp4 *MP4) Close(config *models.Config) {
 				uuid.UnknownPayload = signature
 				init.Moov.AddChild(uuid)
 			} else {
-				//log.Log.Error("mp4.Close(): error signing fingerprint: " + err.Error())
+				// log.Error("mp4.Close(): error signing fingerprint: " + err.Error())
 			}
 		}
 	}
@@ -979,7 +1073,7 @@ func (mp4 *MP4) Close(config *models.Config) {
 	// and re-encode with the corrected value.
 	var initBuf bytes.Buffer
 	if err := init.Encode(&initBuf); err != nil {
-		log.Log.Error("mp4.Close(): error encoding init segment: " + err.Error())
+		log.Error("mp4.Close(): error encoding init segment: " + err.Error())
 	}
 
 	initSize := int64(initBuf.Len())
@@ -991,7 +1085,7 @@ func (mp4 *MP4) Close(config *models.Config) {
 	if len(mp4.SegmentDurations) > 0 {
 		if mp4.FreeBoxSize < initSize {
 			// Avoid computing a negative offset and wrapping it to uint64.
-			log.Log.Error("mp4.Close(): FreeBoxSize is smaller than initSize; skipping sidx FirstOffset adjustment")
+			log.Error("mp4.Close(): FreeBoxSize is smaller than initSize; skipping sidx FirstOffset adjustment")
 		} else {
 			firstOffset := uint64(mp4.FreeBoxSize - initSize)
 			// Find the sidx we added and update its FirstOffset
@@ -1004,19 +1098,24 @@ func (mp4 *MP4) Close(config *models.Config) {
 			// Re-encode with the corrected FirstOffset (same size, no layout change)
 			initBuf.Reset()
 			if err := init.Encode(&initBuf); err != nil {
-				log.Log.Error("mp4.Close(): error re-encoding init segment: " + err.Error())
+				log.Error("mp4.Close(): error re-encoding init segment: " + err.Error())
 			}
 			initSize = int64(initBuf.Len())
 		}
 	}
 
 	if initSize > mp4.FreeBoxSize {
-		log.Log.Error(fmt.Sprintf("mp4.Close(): init segment (%d bytes) exceeds reserved space (%d bytes), file may be corrupt", initSize, mp4.FreeBoxSize))
+		log.WithFields(log.Fields{
+			"component":      "video/mp4",
+			"event":          "init_segment_oversized",
+			"init_bytes":     initSize,
+			"reserved_bytes": mp4.FreeBoxSize,
+		}).Error("MP4 init segment exceeds reserved space")
 	}
 
 	// Write the init segment at the beginning of the file, overwriting the free box placeholder.
 	if _, err := mp4.FileWriter.WriteAt(initBuf.Bytes(), 0); err != nil {
-		log.Log.Error("mp4.Close(): error writing init segment: " + err.Error())
+		log.Error("mp4.Close(): error writing init segment: " + err.Error())
 	}
 
 	// Fill any remaining reserved space with a new (smaller) free box so
@@ -1026,15 +1125,15 @@ func (mp4 *MP4) Close(config *models.Config) {
 		newFree := mp4ff.NewFreeBox(make([]byte, remainingSize-8))
 		var freeBuf bytes.Buffer
 		if err := newFree.Encode(&freeBuf); err != nil {
-			log.Log.Error("mp4.Close(): error encoding free box: " + err.Error())
+			log.Error("mp4.Close(): error encoding free box: " + err.Error())
 		}
 		if _, err := mp4.FileWriter.WriteAt(freeBuf.Bytes(), initSize); err != nil {
-			log.Log.Error("mp4.Close(): error writing free box: " + err.Error())
+			log.Error("mp4.Close(): error writing free box: " + err.Error())
 		}
 	}
 
 	if err := mp4.FileWriter.Sync(); err != nil {
-		log.Log.Error("mp4.Close(): error syncing file: " + err.Error())
+		log.Error("mp4.Close(): error syncing file: " + err.Error())
 	}
 	mp4.FileWriter.Close()
 }

machinery/src/webrtc/aac_transcoder_ffmpeg.go

@@ -193,10 +193,9 @@ import "C"
 
 import (
 	"errors"
-	"fmt"
 	"unsafe"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
+	log "github.com/sirupsen/logrus"
 	"github.com/zaf/g711"
 )
 
@@ -216,7 +215,7 @@ func NewAACTranscoder() (*AACTranscoder, error) {
 	if h == nil {
 		return nil, errors.New("failed to create AAC transcoder (FFmpeg AAC decoder not available?)")
 	}
-	log.Log.Info("webrtc.aac_transcoder: AAC → G.711 µ-law transcoder initialised (FFmpeg)")
+	log.Info("webrtc.aac_transcoder: AAC → G.711 µ-law transcoder initialised (FFmpeg)")
 	return &AACTranscoder{handle: h}, nil
 }
 
@@ -250,9 +249,16 @@ func (t *AACTranscoder) Transcode(adtsData []byte) ([]byte, error) {
 
 	// Log resampler details once.
 	if t.handle.swr_initialized == 1 && t.handle.in_sample_rate != 0 {
-		log.Log.Info(fmt.Sprintf(
-			"webrtc.aac_transcoder: first output – resampling %d Hz / %d ch → 8000 Hz mono → µ-law",
-			int(t.handle.in_sample_rate), int(t.handle.in_channels)))
+		log.WithFields(log.Fields{
+			"component":        "webrtc",
+			"event":            "aac_resampler_initialized",
+			"input_channels":   int(t.handle.in_channels),
+			"input_sample_hz":  int(t.handle.in_sample_rate),
+			"output_channels":  1,
+			"output_codec":     "PCMU",
+			"output_sample_hz": 8000,
+		}).Info("AAC resampler initialized")
+
 		// Prevent repeated logging by zeroing the field we check.
 		t.handle.in_sample_rate = 0
 	}
@@ -265,6 +271,6 @@ func (t *AACTranscoder) Close() {
 	if t != nil && t.handle != nil {
 		C.aac_transcoder_destroy(t.handle)
 		t.handle = nil
-		log.Log.Info("webrtc.aac_transcoder: transcoder closed")
+		log.Info("webrtc.aac_transcoder: transcoder closed")
 	}
 }

machinery/src/webrtc/aac_transcoder_stub.go

@@ -15,7 +15,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
+	log "github.com/sirupsen/logrus"
 )
 
 // AACTranscodingAvailable reports whether AAC→PCMU transcoding
@@ -45,7 +45,7 @@ func NewAACTranscoder() (*AACTranscoder, error) {
 	if err != nil {
 		return nil, errors.New("AAC transcoding not available: ffmpeg binary not found in PATH")
 	}
-	log.Log.Info("webrtc.aac_transcoder: using ffmpeg binary at " + ffmpegPath)
+	log.Info("webrtc.aac_transcoder: using ffmpeg binary at " + ffmpegPath)
 
 	cmd := exec.Command(
 		ffmpegPath,
@@ -96,19 +96,19 @@ func NewAACTranscoder() (*AACTranscoder, error) {
 				buffered := t.outBuf.Len()
 				t.outMu.Unlock()
 				if buffered <= 8192 || buffered%16000 == 0 {
-					log.Log.Debug("webrtc.aac_transcoder: ffmpeg produced PCMU bytes, buffered=" + strconv.Itoa(buffered))
+					log.Debug("webrtc.aac_transcoder: ffmpeg produced PCMU bytes, buffered=" + strconv.Itoa(buffered))
 				}
 			}
 			if readErr != nil {
 				if readErr != io.EOF {
-					log.Log.Warning("webrtc.aac_transcoder: stdout reader stopped: " + readErr.Error())
+					log.Warn("webrtc.aac_transcoder: stdout reader stopped: " + readErr.Error())
 				}
 				return
 			}
 		}
 	}()
 
-	log.Log.Info("webrtc.aac_transcoder: AAC → PCMU transcoder initialised (ffmpeg process)")
+	log.Info("webrtc.aac_transcoder: AAC → PCMU transcoder initialised (ffmpeg process)")
 	return t, nil
 }
 
@@ -129,22 +129,22 @@ func (t *AACTranscoder) Transcode(adtsData []byte) ([]byte, error) {
 		return nil, err
 	}
 	if len(adtsData) <= 512 || len(adtsData)%1024 == 0 {
-		log.Log.Debug("webrtc.aac_transcoder: wrote AAC bytes to ffmpeg, input=" + strconv.Itoa(len(adtsData)))
+		log.Debug("webrtc.aac_transcoder: wrote AAC bytes to ffmpeg, input=" + strconv.Itoa(len(adtsData)))
 	}
 
 	deadline := time.Now().Add(75 * time.Millisecond)
 	for {
 		data := t.readAvailable()
 		if len(data) > 0 {
-			log.Log.Debug("webrtc.aac_transcoder: returning PCMU bytes=" + strconv.Itoa(len(data)))
+			log.Debug("webrtc.aac_transcoder: returning PCMU bytes=" + strconv.Itoa(len(data)))
 			return data, nil
 		}
 
 		if time.Now().After(deadline) {
 			if stderr := t.stderrString(); stderr != "" {
-				log.Log.Warning("webrtc.aac_transcoder: no output before deadline, ffmpeg stderr: " + stderr)
+				log.Warn("webrtc.aac_transcoder: no output before deadline, ffmpeg stderr: " + stderr)
 			} else {
-				log.Log.Debug("webrtc.aac_transcoder: no PCMU output before deadline")
+				log.Debug("webrtc.aac_transcoder: no PCMU output before deadline")
 			}
 			return nil, nil
 		}
@@ -198,7 +198,7 @@ func (t *AACTranscoder) Close() {
 			_ = t.cmd.Process.Kill()
 			_, _ = t.cmd.Process.Wait()
 			if stderr := t.stderrString(); stderr != "" {
-				log.Log.Info("webrtc.aac_transcoder: ffmpeg stderr on close: " + stderr)
+				log.Info("webrtc.aac_transcoder: ffmpeg stderr on close: " + stderr)
 			}
 		}
 	})

machinery/src/webrtc/broadcaster.go

@@ -4,9 +4,9 @@ import (
 	"io"
 	"sync"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	pionWebRTC "github.com/pion/webrtc/v4"
 	pionMedia "github.com/pion/webrtc/v4/pkg/media"
+	log "github.com/sirupsen/logrus"
 )
 
 const (
@@ -74,12 +74,20 @@ func (b *TrackBroadcaster) AddPeer(sessionKey string) (*pionWebRTC.TrackLocalSta
 				if err == io.ErrClosedPipe {
 					return
 				}
-				log.Log.Error("webrtc.broadcaster.peerWriter(): error writing sample for " + sessionKey + ": " + err.Error())
+				log.WithError(err).WithFields(log.Fields{
+					"component":  "webrtc",
+					"event":      "sample_write_failed",
+					"session_id": sessionKey,
+				}).Error("Failed to write WebRTC sample")
 			}
 		}
 	}()
 
-	log.Log.Info("webrtc.broadcaster.AddPeer(): added peer track for " + sessionKey)
+	log.WithFields(log.Fields{
+		"component":  "webrtc",
+		"event":      "peer_added",
+		"session_id": sessionKey,
+	}).Info("WebRTC peer added")
 	return track, nil
 }
 
@@ -95,7 +103,11 @@ func (b *TrackBroadcaster) RemovePeer(sessionKey string) {
 	if exists {
 		close(pt.samples)
 		<-pt.done // wait for writer goroutine to finish
-		log.Log.Info("webrtc.broadcaster.RemovePeer(): removed peer track for " + sessionKey)
+		log.WithFields(log.Fields{
+			"component":  "webrtc",
+			"event":      "peer_removed",
+			"session_id": sessionKey,
+		}).Info("WebRTC peer removed")
 	}
 }
 
@@ -110,7 +122,11 @@ func (b *TrackBroadcaster) WriteSample(sample pionMedia.Sample) {
 		select {
 		case pt.samples <- sample:
 		default:
-			log.Log.Warning("webrtc.broadcaster.WriteSample(): dropping sample for slow peer " + sessionKey)
+			log.WithFields(log.Fields{
+				"component":  "webrtc",
+				"event":      "slow_peer_sample_dropped",
+				"session_id": sessionKey,
+			}).Warn("Dropping WebRTC sample for slow peer")
 		}
 	}
 }