Merge pull request #336 from 4alvit/fix/snapshot-closed-queue

fix(capture): stop snapshot readers when the packet queue closes

.circleci/config.yml

@@ -1,32 +0,0 @@
-# Golang CircleCI 2.0 configuration file
-#
-# Check https://circleci.com/docs/2.0/language-go/ for more details
-version: 2
-jobs:
-  machinery:
-    docker:
-      - image: kerberos/base:0a50dc9
-    working_directory: /go/src/github.com/{{ORG_NAME}}/{{REPO_NAME}}
-    steps:
-      - checkout
-      - run: apt-get update && apt-get install -y --no-install-recommends libavcodec-dev libavformat-dev libswscale-dev libjpeg62-turbo-dev
-      - run: cd machinery && go mod download
-      - run: cd machinery && go test -v ./...
-      - run: cd machinery && go vet
-
-  ui:
-    docker:
-      - image: cimg/python:3.9.13-node
-    steps:
-      - checkout
-      - run: node --version
-      - run: cd ui && yarn
-      - run: cd ui && yarn test --passWithNoTests
-      - run: cd ui && yarn build
-
-workflows:
-  version: 2
-  build:
-    jobs:
-      - machinery
-      - ui

.github/workflows/go.yml

@@ -10,24 +10,27 @@ jobs:
   build:
     name: Build
     runs-on: ubuntu-latest
-    container:
-      image: kerberos/base:eb6b088
-
-    strategy:
-      matrix:
-        #No longer supported Go versions.
-        #go-version: ['1.17', '1.18', '1.19', '1.20', '1.21']
-        go-version: ["1.24"]
 
     steps:
-      - name: Set up Go ${{ matrix.go-version }}
-        uses: actions/setup-go@v2
-        with:
-          go-version: ${{ matrix.go-version }}
       - name: Check out code into the Go module directory
         uses: actions/checkout@v4
-      - name: Set up git ownershi
-        run: git config --system --add safe.directory /__w/agent/agent
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.25.x"
+          check-latest: true
+          cache: true
+          cache-dependency-path: machinery/go.sum
+      - name: Install native dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential \
+            libavcodec-dev \
+            libavutil-dev \
+            libswresample-dev \
+            libswscale-dev \
+            pkg-config
       - name: Get dependencies
         run: cd machinery && go mod download
       - name: Build

.github/workflows/nightly-build.yml

@@ -15,7 +15,7 @@ jobs:
         architecture: [amd64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
@@ -24,13 +24,14 @@ jobs:
         with:
           ref: master
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        id: buildx
+        uses: docker/setup-buildx-action@v3
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
       - name: Run Buildx
-        run: docker buildx build --platform linux/${{matrix.architecture}} -t kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7) --push .
+        run: docker buildx build --build-arg VERSION=$(echo $GITHUB_SHA | cut -c1-7) --platform linux/${{matrix.architecture}} -t kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7) --push .
       - name: Create new and append to manifest
         run: docker buildx imagetools create -t kerberos/agent-nightly:$(echo $GITHUB_SHA | cut -c1-7) kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7)
   nightly-build-other:
@@ -40,7 +41,7 @@ jobs:
         architecture: [arm64, arm/v7, arm/v6]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
@@ -49,12 +50,13 @@ jobs:
         with:
           ref: master
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        id: buildx
+        uses: docker/setup-buildx-action@v3
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
       - name: Run Buildx
-        run: docker buildx build --platform linux/${{matrix.architecture}} -t kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7) --push .
+        run: docker buildx build --build-arg VERSION=$(echo $GITHUB_SHA | cut -c1-7) --platform linux/${{matrix.architecture}} -t kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7) --push .
       - name: Create new and append to manifest
         run: docker buildx imagetools create --append -t kerberos/agent-nightly:$(echo $GITHUB_SHA | cut -c1-7) kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7)

.github/workflows/pr-build.yml

@@ -20,19 +20,15 @@ jobs:
             runner: ubuntu-24.04-arm
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-      - uses: benjlevesque/short-sha@v2.1
-        id: short-sha
-        with:
-          length: 7
       - name: Run Build
         run: |
-          docker build -t ${{ matrix.architecture }} .
+          docker build --build-arg VERSION=$(echo "${{ github.event.pull_request.head.sha }}" | cut -c1-7) -t ${{ matrix.architecture }} .
           CID=$(docker create ${{matrix.architecture}})
           docker cp ${CID}:/home/agent ./output-${{matrix.architecture}}
           docker rm ${CID}
@@ -43,4 +39,3 @@ jobs:
         with:
           name: agent-${{matrix.architecture}}.tar
           path: agent-${{matrix.architecture}}.tar
-

.github/workflows/react.yml

@@ -11,21 +11,22 @@ on:
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     strategy:
       matrix:
-        node-version: [14.x, 16.x]
+        node-version: [22.x]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v2
+      uses: actions/setup-node@v4
       with:
         node-version: ${{ matrix.node-version }}
+        cache: yarn
+        cache-dependency-path: ui/yarn.lock
     - name: Yarn install
-      run: cd ui && yarn
+      run: cd ui && yarn --frozen-lockfile
     - name: Yarn test
       run: cd ui && yarn test --passWithNoTests
     - name: Yarn build

.github/workflows/release-bump.yml

@@ -58,16 +58,12 @@ jobs:
         architecture: [amd64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Checkout
-        uses: actions/checkout@v3
-      - uses: benjlevesque/short-sha@v2.1
-        id: short-sha
-        with:
-          length: 7
+        uses: actions/checkout@v4
       - name: Run Build
         run: |
           docker build --provenance=false --build-arg VERSION=${{ needs.bump-release.outputs.tag }} -t ${{matrix.architecture}} .
@@ -96,16 +92,12 @@ jobs:
         architecture: [arm64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Checkout
-        uses: actions/checkout@v3
-      - uses: benjlevesque/short-sha@v2.1
-        id: short-sha
-        with:
-          length: 7
+        uses: actions/checkout@v4
       - name: Run Build
         run: |
           docker build --provenance=false --build-arg VERSION=${{ needs.bump-release.outputs.tag }} -t ${{matrix.architecture}} .
@@ -129,7 +121,7 @@ jobs:
     needs: [bump-release, build-amd64, build-arm64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/release-create.yml

@@ -22,16 +22,12 @@ jobs:
         architecture: [amd64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Checkout
-        uses: actions/checkout@v3
-      - uses: benjlevesque/short-sha@v2.1
-        id: short-sha
-        with:
-          length: 7
+        uses: actions/checkout@v4
       - name: Run Build
         run: |
           docker build --provenance=false --build-arg VERSION=${{github.event.inputs.tag || github.ref_name}} -t ${{matrix.architecture}} .
@@ -59,16 +55,12 @@ jobs:
         architecture: [arm64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Checkout
-        uses: actions/checkout@v3
-      - uses: benjlevesque/short-sha@v2.1
-        id: short-sha
-        with:
-          length: 7
+        uses: actions/checkout@v4
       - name: Run Build
         run: |
           docker build --provenance=false --build-arg VERSION=${{github.event.inputs.tag || github.ref_name}} -t ${{matrix.architecture}} .
@@ -92,7 +84,7 @@ jobs:
     needs: [build-amd64, build-arm64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}

Dockerfile

@@ -26,29 +26,27 @@ RUN apt-get update && apt-get install -y --fix-missing --no-install-recommends \
 	rm -rf /var/lib/apt/lists/*
 
 ##############################################################################
-# Copy all the relevant source code in the Docker image, so we can build this.
+# Copy dependency metadata first so module downloads can be cached separately.
 
 RUN mkdir -p /go/src/github.com/kerberos-io/agent
-COPY machinery /go/src/github.com/kerberos-io/agent/machinery
-RUN rm -rf /go/src/github.com/kerberos-io/agent/machinery/.env
+WORKDIR /go/src/github.com/kerberos-io/agent/machinery
+COPY machinery/go.mod machinery/go.sum ./
+RUN go mod download
 
-##################################################################
-# Get the latest commit hash, so we know which version we're running
-COPY .git /go/src/github.com/kerberos-io/agent/.git
-RUN cd /go/src/github.com/kerberos-io/agent/.git && git log --format="%H" -n 1 | head -c7 > /go/src/github.com/kerberos-io/agent/machinery/version
-RUN cat /go/src/github.com/kerberos-io/agent/machinery/version
+##############################################################################
+# Copy the rest of the source after dependencies are primed.
+
+COPY machinery ./
+RUN rm -rf .env
 
 ##################
 # Build Machinery
 
-RUN cd /go/src/github.com/kerberos-io/agent/machinery && \
-	go mod download && \
-	if [ -z "${VERSION}" ] || [ "${VERSION}" = "0.0.0" ]; then \
-		VERSION=$(cd /go/src/github.com/kerberos-io/agent && git describe --tags --always 2>/dev/null || echo "0.0.0"); \
-	fi && \
+RUN RESOLVED_VERSION="${VERSION:-0.0.0}" && \
+	printf '%s' "${RESOLVED_VERSION}" > version && \
 	BUILD_TAGS=timetzdata,netgo,osusergo && \
 	case "${TARGETARCH:-$(go env GOARCH)}" in amd64|arm64) BUILD_TAGS="moq,${BUILD_TAGS}" ;; esac && \
-	go build -tags "${BUILD_TAGS}" --ldflags "-s -w -X github.com/kerberos-io/agent/machinery/src/utils.VERSION=${VERSION}" main.go && \
+	go build -o main -tags "${BUILD_TAGS}" --ldflags "-s -w -X github.com/kerberos-io/agent/machinery/src/utils.VERSION=${RESOLVED_VERSION}" . && \
 	mkdir -p /agent && \
 	mv main /agent && \
 	mv version /agent && \
@@ -80,9 +78,11 @@ RUN apk update && apk upgrade --available && sync
 # Build Web (React app)
 
 RUN mkdir -p /go/src/github.com/kerberos-io/agent/machinery/www
-COPY ui /go/src/github.com/kerberos-io/agent/ui
-RUN cd /go/src/github.com/kerberos-io/agent/ui && rm -rf yarn.lock && yarn config set network-timeout 300000 && \
-	yarn && yarn build
+WORKDIR /go/src/github.com/kerberos-io/agent/ui
+COPY ui/package.json ui/yarn.lock ./
+RUN yarn config set network-timeout 300000 && yarn --frozen-lockfile
+COPY ui ./
+RUN yarn build
 
 ####################################
 # Let's create a /dist folder containing just the files necessary for runtime.
@@ -146,10 +146,10 @@ EXPOSE 80
 ######################################
 # Check if agent is still running
 
-HEALTHCHECK CMD curl --fail http://localhost:80 || exit 1   
+HEALTHCHECK CMD curl --fail "http://localhost:${AGENT_PORT:-80}/health" || exit 1
 
 ###################################################
 # Leeeeettttt'ssss goooooo!!!
 # Run the shizzle from the right working directory.
 WORKDIR /home/agent
-CMD ["./main", "-action", "run", "-port", "80"]
+CMD ["./main", "-action", "run"]

README.md

@@ -3,7 +3,6 @@
 <a target="_blank" href="https://kerberos.io"><img src="https://img.shields.io/badge/kerberos-website-gray.svg?longCache=true&colorB=brightgreen" alt="Kerberos Agent"></a>
 <a target="_blank" href="https://doc.kerberos.io"><img src="https://img.shields.io/badge/kerberos-documentation-gray.svg?longCache=true&colorB=brightgreen" alt="Kerberos Agent"></a>
 
-<a target="_blank" href="https://circleci.com/gh/kerberos-io/agent"><img src="https://circleci.com/gh/kerberos-io/agent.svg?style=svg"/></a>
 <img src="https://github.com/kerberos-io/agent/workflows/Go/badge.svg"/>
 <img src="https://github.com/kerberos-io/agent/workflows/React/badge.svg"/>
 <img src="https://github.com/kerberos-io/agent/workflows/CodeQL/badge.svg"/>
@@ -189,6 +188,103 @@ Next to attaching the configuration file, it is also possible to override the co
     -e AGENT_CAPTURE_CONTINUOUS=true \
     -d --restart=always kerberos/agent:latest
 
+Set `AGENT_PORT` to change the Agent's internal HTTP port. The environment
+variable takes precedence over `-port`. When using Docker, publish the same
+container port:
+
+```bash
+docker run -p 8082:8082 \
+  -e AGENT_PORT=8082 \
+  --name mycamera \
+  -d --restart=always kerberos/agent:latest
+```
+
+### Health checks and API responses
+
+`GET /health` is an unauthenticated liveness and diagnostics endpoint for
+container orchestrators and external monitors. `healthy` means that the Agent
+HTTP process can serve requests. Camera streams and Hub connectivity are
+reported independently in the response, so an external camera or Hub outage
+does not make Docker restart an otherwise functioning Agent process. Successful
+probes return `200 OK` and are logged at debug level to avoid periodic health
+checks filling production logs.
+
+```bash
+curl http://localhost:8082/health
+```
+
+```json
+{
+  "httpStatusCode": 200,
+  "applicationStatusCode": "get_success",
+  "entityStatusCode": "healthy",
+  "message": "Healthy",
+  "metadata": {
+    "applicationName": "agent",
+    "applicationVersion": "0.0.0",
+    "timestamp": 1788710400,
+    "path": "/health"
+  },
+  "data": {
+    "health": {
+      "description": "Agent HTTP service is healthy",
+      "cameraConnected": true,
+      "mainStream": {
+        "configured": true,
+        "connected": true,
+        "packagesProcessed": 183421,
+        "fps": 29.97,
+        "resolution": {
+          "width": 1920,
+          "height": 1080
+        },
+        "lastPacketAt": 1788710399
+      },
+      "subStream": {
+        "configured": true,
+        "connected": true,
+        "packagesProcessed": 91710,
+        "fps": 15,
+        "resolution": {
+          "width": 640,
+          "height": 360
+        },
+        "lastPacketAt": 1788710399
+      },
+      "hub": {
+        "configured": true,
+        "connected": true,
+        "lastHeartbeatAttemptAt": 1788710398,
+        "lastSuccessfulHeartbeatAt": 1788710398
+      }
+    }
+  }
+}
+```
+
+Stream `packagesProcessed` counts complete encoded video access units processed
+since the Agent process started; it does not count individual fragmented RTP
+packets. `fps` is the latest PTS-derived frame-rate estimate. Resolution is the
+most recently observed encoded width and height. `lastPacketAt`, Hub heartbeat
+timestamps, and response metadata timestamps are Unix seconds; `0` means that
+no value has been observed yet. A stream can be configured but temporarily
+disconnected.
+
+Hub `configured` means heartbeat delivery is enabled and has the required URI
+and key. Hub `connected` means the most recent heartbeat succeeded and the last
+success is no more than three minutes old. A failed heartbeat marks it
+disconnected immediately while retaining the last-success timestamp for
+diagnosis.
+
+New JSON endpoints should use this Hub-compatible response envelope rather than
+adding unrelated top-level response shapes. HTTP status describes the transport
+result, `applicationStatusCode` describes the operation, `entityStatusCode` and
+`message` describe the domain outcome, `metadata` carries request/application
+context, and endpoint-specific content belongs in a typed `data` object. Client
+responses must contain safe messages only; detailed internal errors belong in
+structured logs. Existing endpoints retain their legacy response shapes until
+they are migrated deliberately.
+
 ### Secure camera streams (RTSPS)
 
 The Agent accepts `rtsps://` camera URLs. Do not use `srtsp://`; RTSPS is RTSP over TLS. For a Bosch FLEXIDOME micro 3100i, enable **Secure RTSP** under **Network > Network Services** and use port `9554`:
@@ -208,6 +304,7 @@ See [RTSPS and TLS certificates](README-RTSPS-TLS.md) for the complete Bosch UI,
 | --------------------------------------- | ----------------------------------------------------------------------------------------------- | ------------------------------ |
 | `LOG_LEVEL`                                 | Level for logging, could be "info", "warning", "debug", "error" or "fatal".                     | "info"                         |
 | `LOG_OUTPUT`                                | Logging output format "json" or "text".                                                         | "text"                         |
+| `AGENT_PORT`                                | HTTP web server port. Overrides the `-port` command-line value and must be between 1 and 65535. | "80"                           |
 | `AGENT_MODE`                                | You can choose to run this in 'release' for production, and or 'demo' for showcasing.           | "release"                      |
 | `AGENT_TLS_INSECURE`                        | Specify if you want to use `InsecureSkipVerify` for the internal HTTP client.                   | "false"                        |
 | `AGENT_USERNAME`                            | The username used to authenticate against the Kerberos Agent login page.                        | "root"                         |
@@ -244,9 +341,10 @@ See [RTSPS and TLS certificates](README-RTSPS-TLS.md) for the complete Bosch UI,
 | `AGENT_CAPTURE_PIXEL_CHANGE`                | If `CONTINUOUS` set to `false`, the number of pixel require to change before motion triggers.   | "150"                          |
 | `AGENT_CAPTURE_FRAGMENTED`                  | Set the format of the recorded MP4 to fragmented (suitable for HLS).                            | "false"                        |
 | `AGENT_CAPTURE_FRAGMENTED_DURATION`         | If `AGENT_CAPTURE_FRAGMENTED` set to `true`, define the duration (seconds) of a fragment.       | "8"                            |
-| `AGENT_MQTT_URI`                            | An MQTT broker endpoint that is used for bi-directional communication (live view, onvif, etc)   | "tcp://mqtt.kerberos.io:1883"  |
+| `AGENT_MQTT_URI`                            | MQTT broker endpoint for bi-directional communication. Accepts ActiveMQ `mqtt+ssl://` URLs.     | "tcp://mqtt.kerberos.io:1883"  |
 | `AGENT_MQTT_USERNAME`                       | Username of the MQTT broker.                                                                    | ""                             |
 | `AGENT_MQTT_PASSWORD`                       | Password of the MQTT broker.                                                                    | ""                             |
+| `AGENT_REMOTE_ACCESS_ENABLED`               | Allow encrypted Hub MQTT sessions to stream Agent logs and open an interactive shell. Enable only for trusted deployments. | "false"                        |
 | `AGENT_REALTIME_PROCESSING`                 | If `AGENT_REALTIME_PROCESSING` set to `true`, the agent will send key frames to the topic       | ""                             |
 | `AGENT_REALTIME_PROCESSING_TOPIC`           | The topic to which keyframes will be sent in base64 encoded format.                             | ""                             |
 | `AGENT_STUN_URI`                            | When using WebRTC, you'll need to provide a STUN server.                                        | "stun:turn-fra1.kerberos.io:3478"|
@@ -261,16 +359,19 @@ See [RTSPS and TLS certificates](README-RTSPS-TLS.md) for the complete Bosch UI,
 | `AGENT_HUB_PRIVATE_KEY`                     | The secret access key linked to your account in Kerberos Hub.                                   | ""                             |
 | `AGENT_HUB_REGION`                          | The Kerberos Hub region, to which you want to upload.                                           | ""                             |
 | `AGENT_HUB_SITE`                            | The site ID of a site you've created in your Kerberos Hub account.                              | ""                             |
+| `AGENT_TUS_CHUNK_SIZE_BYTES`                | Bytes sent in each resumable-upload PATCH. Set to `0` or a negative value to send all remaining bytes in one PATCH. | "8388608" (8 MiB)              |
 | `AGENT_KERBEROSVAULT_URI`                   | The Kerberos Vault API url.                                                                     | "https://vault.domain.com/api" |
 | `AGENT_KERBEROSVAULT_ACCESS_KEY`            | The access key of a Kerberos Vault account.                                                     | ""                             |
 | `AGENT_KERBEROSVAULT_SECRET_KEY`            | The secret key of a Kerberos Vault account.                                                     | ""                             |
 | `AGENT_KERBEROSVAULT_PROVIDER`              | A Kerberos Vault provider you have created (optional).                                          | ""                             |
 | `AGENT_KERBEROSVAULT_DIRECTORY`             | The directory, in the Kerberos vault, where the recordings will be stored.                      | ""                             |
+| `AGENT_KERBEROSVAULT_CUSTOM_HEADERS`        | JSON object of custom upload headers that Vault includes as integration metadata.               | `{"site_id":"site-1"}`         |
 | `AGENT_KERBEROSVAULT_SECONDARY_URI`         | The Kerberos Vault API url.                                                                     | "https://vault.domain.com/api" |
 | `AGENT_KERBEROSVAULT_SECONDARY_ACCESS_KEY`  | The access key of a secondary Kerberos Vault account.                                           | ""                             |
 | `AGENT_KERBEROSVAULT_SECONDARY_SECRET_KEY`  | The secret key of a secondary Kerberos Vault account.                                           | ""                             |
 | `AGENT_KERBEROSVAULT_SECONDARY_PROVIDER`    | A secondary Kerberos Vault provider you have created (optional).                                | ""                             |
 | `AGENT_KERBEROSVAULT_SECONDARY_DIRECTORY`   | The directory, in the secondary Kerberos vault, where the recordings will be stored.            | ""                             |
+| `AGENT_KERBEROSVAULT_SECONDARY_CUSTOM_HEADERS` | JSON object of custom upload headers for the secondary Vault.                                | `{"site_id":"site-1"}`         |
 | `AGENT_DROPBOX_ACCESS_TOKEN`                | The Access Token from your Dropbox app, that is used to leverage the Dropbox SDK.               | ""                             |
 | `AGENT_DROPBOX_DIRECTORY`                   | The directory, in Dropbox, where the recordings will be stored.                                 | ""                             |
 | `AGENT_ENCRYPTION`                          | Enable 'true' or disable 'false' end-to-end encryption for MQTT messages.                       | "false"                        |
@@ -281,6 +382,122 @@ See [RTSPS and TLS certificates](README-RTSPS-TLS.md) for the complete Bosch UI,
 | `AGENT_SIGNING`                             | Enable 'true' or disable 'false' for signing recordings.                                        | "true"                         |
 | `AGENT_SIGNING_PRIVATE_KEY`                 | The private key (RSA) to sign the recordings fingerprint to validate origin.                    | "" - uses default one if empty |
 
+Remote console access is disabled unless `AGENT_REMOTE_ACCESS_ENABLED=true`.
+The Agent also rejects remote session messages unless Hub encryption or
+end-to-end MQTT encryption is configured and used. A remote shell runs inside
+the Agent process environment as the Agent operating-system user; it is not an
+SSH server and does not expose a new network port. Keep the feature disabled on
+deployments where Hub owners should not have operating-system access.
+
+### Resumable upload chunk size
+
+Hub and Vault resumable uploads use `AGENT_TUS_CHUNK_SIZE_BYTES` as the maximum
+body size of each tus `PATCH` request. The value is a number of bytes, not a
+number of chunks. If the variable is unset or invalid, the Agent uses 8 MiB:
+
+```dotenv
+# 8 MiB (default)
+AGENT_TUS_CHUNK_SIZE_BYTES=8388608
+
+# 4 MiB: more frequent checkpoints on unstable connections
+AGENT_TUS_CHUNK_SIZE_BYTES=4194304
+
+# Disable chunking and send all remaining bytes in one PATCH
+AGENT_TUS_CHUNK_SIZE_BYTES=0
+```
+
+Smaller chunks provide more frequent resumable checkpoints but create more HTTP
+requests. Larger chunks reduce request overhead but require more data to be
+retransmitted when a request fails.
+
+### Custom metadata headers for Kerberos Vault
+
+Use `AGENT_KERBEROSVAULT_CUSTOM_HEADERS` to attach your own metadata (for
+example a site, line, or equipment identifier) to every recording uploaded to
+Kerberos Vault. The value is a JSON object whose keys are header names and whose
+values are strings. The Agent sends these as HTTP headers on every upload, and
+Vault adds them to the `metadata` object of its integration messages (Kafka,
+MQTT, SQS, RabbitMQ, etc.):
+
+```json
+{
+  "timestamp": "2026-09-24T15:06:45Z",
+  "key": "agent1/1790262405_3-425_agent1_0-0-0-0_-1_10000.mp4",
+  "device": "agent1",
+  "metadata": {
+    "site_id": "site-1",
+    "line_id": "line-2",
+    "equipment_id": "equipment-3"
+  }
+}
+```
+
+The value must be valid JSON on a single line, so wrap it in single quotes in a
+shell:
+
+```bash
+docker run -p 80:80 --name mycamera \
+  -e AGENT_CLOUD=kstorage \
+  -e AGENT_KERBEROSVAULT_URI=https://vault.domain.com/api \
+  -e AGENT_KERBEROSVAULT_ACCESS_KEY=xxx \
+  -e AGENT_KERBEROSVAULT_SECRET_KEY=xxx \
+  -e AGENT_KERBEROSVAULT_DIRECTORY=agent1 \
+  -e AGENT_KERBEROSVAULT_CUSTOM_HEADERS='{"site_id":"site-1","line_id":"line-2","equipment_id":"equipment-3"}' \
+  -d --restart=always kerberos/agent:latest
+```
+
+In `docker compose` or an `.env` file:
+
+```yaml
+services:
+  agent:
+    image: kerberos/agent:latest
+    environment:
+      AGENT_CLOUD: kstorage
+      AGENT_KERBEROSVAULT_CUSTOM_HEADERS: '{"site_id":"site-1","line_id":"line-2","equipment_id":"equipment-3"}'
+```
+
+```dotenv
+AGENT_KERBEROSVAULT_CUSTOM_HEADERS={"site_id":"site-1","line_id":"line-2","equipment_id":"equipment-3"}
+```
+
+In Kubernetes, quote the value so YAML keeps it as a string:
+
+```yaml
+env:
+  - name: AGENT_KERBEROSVAULT_CUSTOM_HEADERS
+    value: '{"site_id":"site-1","line_id":"line-2","equipment_id":"equipment-3"}'
+```
+
+Use `AGENT_KERBEROSVAULT_SECONDARY_CUSTOM_HEADERS` for the secondary Vault. In
+ConfigMap deployments you can set shared defaults with the `GLOBAL_` prefix
+(for example `GLOBAL_AGENT_KERBEROSVAULT_CUSTOM_HEADERS`) and override them per
+agent with `AGENT_KERBEROSVAULT_CUSTOM_HEADERS`. You can also set the same JSON
+under **Settings → Persistence → Custom upload headers** in the Agent UI, which
+stores it as `kstorage.custom_headers` in `config.json`.
+
+The rules for this value are:
+
+- It must be a JSON object with string values. Numbers, `null`, arrays, and
+  nested objects are rejected.
+- It can contain up to 32 headers, with names up to 128 characters and values
+  up to 4096 bytes.
+- Header names must be valid HTTP tokens (letters, digits, and ``!#$%&'*+-.^_`|~``).
+  Keys are matched case-insensitively, so `site_id` and `SITE_ID` cannot both
+  be used.
+- Reserved names are rejected: anything starting with `X-Kerberos-`,
+  `Authorization`, `Cookie`, `Host`, `Content-Type`, `Content-Length`,
+  `Trace-Id`, tus protocol headers (`Tus-Resumable`, `Upload-*`), and
+  hop-by-hop headers.
+- Values cannot contain line breaks, tabs, or other control characters.
+
+If the value is invalid, the Agent logs an error and does not upload to that
+Vault until the configuration is fixed. The recording stays queued, so it will
+upload once the value is corrected. Header names reach Vault unchanged, but
+proxies may normalise their casing. If Vault redirects an upload to another
+host, the Agent removes the custom headers and Vault credentials before
+following the redirect.
+
 
 ## Encryption
 
@@ -455,11 +672,26 @@ its own broadcast and switching quality is simply a resubscribe:
 | `high` | `devices/<agent-key>/live.hang`       | highest-resolution camera stream           |
 | `low`  | `devices/<agent-key>/live-low.hang`   | sub stream (main stream when none is set)  |
 
-Each tier only uploads while it has at least one subscriber, so the tier nobody
-watches costs virtually no bandwidth. `AGENT_LIVE_MOQ_QUALITY` accepts `high` or
-`low` to pin the Agent to a single tier; viewers requesting the other tier then
-find no broadcast. Any other value (including the default) publishes both. The
-initial implementation publishes H.264 video only.
+Each tier uploads every frame while it has subscribers. While idle, it continues
+to upload fresh keyframes only. This keeps the relay's latest cached GOP current,
+so a new viewer does not initially render the final frame from the previous
+viewer session, while avoiding the bandwidth cost of continuously sending every
+delta frame. `AGENT_LIVE_MOQ_QUALITY` accepts `high` or `low` to pin the Agent to
+a single tier; viewers requesting the other tier then find no broadcast. Any
+other value (including the default) publishes both. The initial implementation
+publishes H.264 video only.
+
+Two bounded duration settings tune recovery for unusual network conditions:
+
+| Variable | Default | Allowed range |
+| -------- | ------- | ------------- |
+| `AGENT_LIVE_MOQ_MAX_PACKET_AGE` | `1.5s` | `250ms` to `30s` |
+| `AGENT_LIVE_MOQ_WRITE_TIMEOUT` | `5s` | `1s` to `1m` |
+
+Values use Go duration syntax and are clamped to the documented range. The
+dashboard API's `recovery` object reports per-tier reconnects, last successful
+frame/write timing, native write timeouts, watchdog restarts/cooldown, dropped
+run-channel events, and run-worker shutdown timeouts.
 
 The `/anon` relay route is intended for interoperability testing. Production
 deployments must set `AGENT_LIVE_MOQ_URL` to a short-lived, device-scoped
@@ -472,8 +704,9 @@ monorepo devcontainer so it uses the Trixie base, then run the VS Code task
     cd machinery
     bash ./verify-moq-devcontainer.sh
 
-The check requires glibc 2.38 or newer, runs the tagged package tests, links the
-complete Agent with `-tags moq`, and executes the resulting binary's version
+The check requires glibc 2.38 or newer, runs the tagged package and in-process
+native QUIC lifecycle tests, links the complete Agent with `-tags moq`, and
+executes the resulting binary's version
 command. Both devcontainers also run this check during their post-create setup.
 
 ## What is new?

machinery/README.md

@@ -5,7 +5,6 @@
 <a target="_blank" href="https://twitter.com/kerberosio?ref_src=twsrc%5Etfw"><img src="https://img.shields.io/twitter/url.svg?label=Follow%20%40kerberosio&style=social&url=https%3A%2F%2Ftwitter.com%2Fkerberosio" alt="Twitter Widget"></a>
 <a target="_blank" href="https://join.slack.com/t/kerberosio/shared_invite/zt-1a5oj4pwm-O4qCAN9c5r2um0Ns0ge8ww"><img src="https://img.shields.io/badge/slack-@kerberosio-yellow.svg?logo=slack " alt="Kerberos.io"></a>
 
-<a target="_blank" href="https://circleci.com/gh/kerberos-io/agent"><img src="https://circleci.com/gh/kerberos-io/agent.svg?style=svg"/></a>
 <img src="https://github.com/kerberos-io/agent/workflows/Go/badge.svg"/>
 <img src="https://github.com/kerberos-io/agent/workflows/CodeQL/badge.svg"/>
 

machinery/data/tus/1788718145_3-694_encrypted_0-0-0-0_-1_25497.mp4.primary.json

@@ -0,0 +1 @@
+{"upload_url":"https://vault.kerberos.io/api/storage/tus/bd3be0f9ecd3873d381f57cb8b4a7e96","vault_uri":"https://vault.kerberos.io/api/storage/tus/","size":2560480}

machinery/docs/docs.go

@@ -24,6 +24,38 @@ const docTemplate = `{
     "host": "{{.Host}}",
     "basePath": "{{.BasePath}}",
     "paths": {
+        "/api/camera/discover": {
+            "get": {
+                "description": "Runs an advanced Fing/WiFiman-style scan (ONVIF WS-Discovery + TCP port scan + MAC/vendor lookup) and returns the devices found on the local network.",
+                "tags": [
+                    "onvif"
+                ],
+                "summary": "Discover cameras and other devices on the local network.",
+                "operationId": "camera-discover",
+                "parameters": [
+                    {
+                        "type": "integer",
+                        "description": "Discovery timeout in milliseconds (default 2000)",
+                        "name": "timeout",
+                        "in": "query"
+                    },
+                    {
+                        "type": "string",
+                        "description": "Optional subnet(s) to scan, e.g. '192.168.1.0/24' (comma-separated). Defaults to the local interfaces.",
+                        "name": "subnet",
+                        "in": "query"
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/models.APIResponse"
+                        }
+                    }
+                }
+            }
+        },
         "/api/camera/onvif/capabilities": {
             "post": {
                 "description": "Will return the ONVIF capabilities for the specific camera.",
@@ -623,6 +655,40 @@ const docTemplate = `{
                 }
             }
         },
+        "/api/persistence/secondary/verify": {
+            "post": {
+                "security": [
+                    {
+                        "Bearer": []
+                    }
+                ],
+                "description": "Will verify the secondary persistence.",
+                "tags": [
+                    "persistence"
+                ],
+                "summary": "Will verify the secondary persistence.",
+                "operationId": "verify-secondary-persistence",
+                "parameters": [
+                    {
+                        "description": "Config",
+                        "name": "config",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/models.Config"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/models.APIResponse"
+                        }
+                    }
+                }
+            }
+        },
         "/api/persistence/verify": {
             "post": {
                 "security": [
@@ -656,9 +722,151 @@ const docTemplate = `{
                     }
                 }
             }
+        },
+        "/health": {
+            "get": {
+                "description": "Confirms that the Agent HTTP process can serve requests and reports current camera stream and Hub heartbeat diagnostics. Operational dependency failures do not change the liveness HTTP status.",
+                "produces": [
+                    "application/json"
+                ],
+                "tags": [
+                    "health"
+                ],
+                "summary": "Check Agent health",
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/http.HealthResponse"
+                        }
+                    }
+                }
+            }
         }
     },
     "definitions": {
+        "http.Health": {
+            "type": "object",
+            "properties": {
+                "cameraConnected": {
+                    "type": "boolean"
+                },
+                "description": {
+                    "type": "string"
+                },
+                "hub": {
+                    "$ref": "#/definitions/http.HubHealth"
+                },
+                "mainStream": {
+                    "$ref": "#/definitions/http.StreamHealth"
+                },
+                "subStream": {
+                    "$ref": "#/definitions/http.StreamHealth"
+                }
+            }
+        },
+        "http.HealthResponse": {
+            "type": "object",
+            "properties": {
+                "applicationStatusCode": {
+                    "type": "string"
+                },
+                "data": {
+                    "$ref": "#/definitions/http.HealthResponseData"
+                },
+                "entityStatusCode": {
+                    "type": "string"
+                },
+                "httpStatusCode": {
+                    "type": "integer"
+                },
+                "message": {
+                    "type": "string"
+                },
+                "metadata": {
+                    "$ref": "#/definitions/http.ResponseMetadata"
+                }
+            }
+        },
+        "http.HealthResponseData": {
+            "type": "object",
+            "properties": {
+                "health": {
+                    "$ref": "#/definitions/http.Health"
+                }
+            }
+        },
+        "http.HubHealth": {
+            "type": "object",
+            "properties": {
+                "configured": {
+                    "type": "boolean"
+                },
+                "connected": {
+                    "type": "boolean"
+                },
+                "lastHeartbeatAttemptAt": {
+                    "type": "integer"
+                },
+                "lastSuccessfulHeartbeatAt": {
+                    "type": "integer"
+                }
+            }
+        },
+        "http.ResponseMetadata": {
+            "type": "object",
+            "properties": {
+                "applicationName": {
+                    "type": "string"
+                },
+                "applicationVersion": {
+                    "type": "string"
+                },
+                "path": {
+                    "type": "string"
+                },
+                "timestamp": {
+                    "type": "integer"
+                },
+                "traceId": {
+                    "type": "string"
+                }
+            }
+        },
+        "http.StreamHealth": {
+            "type": "object",
+            "properties": {
+                "configured": {
+                    "type": "boolean"
+                },
+                "connected": {
+                    "type": "boolean"
+                },
+                "fps": {
+                    "type": "number"
+                },
+                "lastPacketAt": {
+                    "type": "integer"
+                },
+                "packagesProcessed": {
+                    "type": "integer"
+                },
+                "resolution": {
+                    "$ref": "#/definitions/http.StreamResolution"
+                }
+            }
+        },
+        "http.StreamResolution": {
+            "type": "object",
+            "properties": {
+                "height": {
+                    "type": "integer"
+                },
+                "width": {
+                    "type": "integer"
+                }
+            }
+        },
         "models.APIResponse": {
             "type": "object",
             "properties": {
@@ -730,12 +938,19 @@ const docTemplate = `{
                 "fragmentedduration": {
                     "type": "integer"
                 },
+                "gopsize": {
+                    "description": "GOP size in seconds, used for pre-recording",
+                    "type": "integer"
+                },
                 "ipcamera": {
                     "$ref": "#/definitions/models.IPCamera"
                 },
                 "liveview": {
                     "type": "string"
                 },
+                "liveview_chunking": {
+                    "type": "string"
+                },
                 "maxlengthrecording": {
                     "type": "integer"
                 },
@@ -745,6 +960,10 @@ const docTemplate = `{
                 "name": {
                     "type": "string"
                 },
+                "onvif_motion": {
+                    "description": "ONVIFMotion routes the camera's ONVIF motion events into the\nagent's motion-triggered recording pipeline. When \"true\" the\nagent opens an event/stream against the configured ONVIF\nendpoint and forwards Motion+Active events to HandleMotion.\nRequires Capture.IPCamera.ONVIFXAddr / ONVIFUsername /\nONVIFPassword to be set. Default empty (disabled) keeps the\nexisting pixel-diff motion detection as the only source.",
+                    "type": "string"
+                },
                 "pixelChangeThreshold": {
                     "type": "integer"
                 },
@@ -823,9 +1042,15 @@ const docTemplate = `{
                 "kstorage": {
                     "$ref": "#/definitions/models.KStorage"
                 },
+                "kstorage_secondary": {
+                    "$ref": "#/definitions/models.KStorage"
+                },
                 "max_directory_size": {
                     "type": "integer"
                 },
+                "min_free_space": {
+                    "type": "integer"
+                },
                 "mqtt_password": {
                     "type": "string"
                 },
@@ -856,6 +1081,9 @@ const docTemplate = `{
                 "s3": {
                     "$ref": "#/definitions/models.S3"
                 },
+                "signing": {
+                    "$ref": "#/definitions/models.Signing"
+                },
                 "stunuri": {
                     "type": "string"
                 },
@@ -947,6 +1175,15 @@ const docTemplate = `{
         "models.IPCamera": {
             "type": "object",
             "properties": {
+                "base_height": {
+                    "type": "integer"
+                },
+                "base_width": {
+                    "type": "integer"
+                },
+                "channels": {
+                    "type": "integer"
+                },
                 "fps": {
                     "type": "string"
                 },
@@ -965,21 +1202,85 @@ const docTemplate = `{
                 "onvif_xaddr": {
                     "type": "string"
                 },
+                "pps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "rtsp": {
                     "type": "string"
                 },
+                "sample_rate": {
+                    "type": "integer"
+                },
+                "sps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_fps": {
                     "type": "string"
                 },
                 "sub_height": {
                     "type": "integer"
                 },
+                "sub_pps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_rtsp": {
                     "type": "string"
                 },
+                "sub_sps_nalus": {
+                    "description": "Sub stream parameter sets, captured separately from the main stream so the\nlive HLS muxer can build a correct init segment when a viewer switches the\nlive view to the sub (low-resolution) stream.",
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
+                "sub_vps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_width": {
                     "type": "integer"
                 },
+                "vps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "width": {
                     "type": "integer"
                 }
@@ -995,15 +1296,24 @@ const docTemplate = `{
                     "description": "old way, remove this",
                     "type": "string"
                 },
+                "custom_headers": {
+                    "type": "string"
+                },
                 "directory": {
                     "type": "string"
                 },
+                "max_retries": {
+                    "type": "integer"
+                },
                 "provider": {
                     "type": "string"
                 },
                 "secret_access_key": {
                     "type": "string"
                 },
+                "timeout": {
+                    "type": "integer"
+                },
                 "uri": {
                     "type": "string"
                 }
@@ -1141,6 +1451,17 @@ const docTemplate = `{
                 }
             }
         },
+        "models.Signing": {
+            "type": "object",
+            "properties": {
+                "enabled": {
+                    "type": "string"
+                },
+                "private_key": {
+                    "type": "string"
+                }
+            }
+        },
         "models.Timetable": {
             "type": "object",
             "properties": {

machinery/docs/swagger.json

@@ -17,6 +17,38 @@
     },
     "basePath": "/",
     "paths": {
+        "/api/camera/discover": {
+            "get": {
+                "description": "Runs an advanced Fing/WiFiman-style scan (ONVIF WS-Discovery + TCP port scan + MAC/vendor lookup) and returns the devices found on the local network.",
+                "tags": [
+                    "onvif"
+                ],
+                "summary": "Discover cameras and other devices on the local network.",
+                "operationId": "camera-discover",
+                "parameters": [
+                    {
+                        "type": "integer",
+                        "description": "Discovery timeout in milliseconds (default 2000)",
+                        "name": "timeout",
+                        "in": "query"
+                    },
+                    {
+                        "type": "string",
+                        "description": "Optional subnet(s) to scan, e.g. '192.168.1.0/24' (comma-separated). Defaults to the local interfaces.",
+                        "name": "subnet",
+                        "in": "query"
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/models.APIResponse"
+                        }
+                    }
+                }
+            }
+        },
         "/api/camera/onvif/capabilities": {
             "post": {
                 "description": "Will return the ONVIF capabilities for the specific camera.",
@@ -616,6 +648,40 @@
                 }
             }
         },
+        "/api/persistence/secondary/verify": {
+            "post": {
+                "security": [
+                    {
+                        "Bearer": []
+                    }
+                ],
+                "description": "Will verify the secondary persistence.",
+                "tags": [
+                    "persistence"
+                ],
+                "summary": "Will verify the secondary persistence.",
+                "operationId": "verify-secondary-persistence",
+                "parameters": [
+                    {
+                        "description": "Config",
+                        "name": "config",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/models.Config"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/models.APIResponse"
+                        }
+                    }
+                }
+            }
+        },
         "/api/persistence/verify": {
             "post": {
                 "security": [
@@ -649,9 +715,151 @@
                     }
                 }
             }
+        },
+        "/health": {
+            "get": {
+                "description": "Confirms that the Agent HTTP process can serve requests and reports current camera stream and Hub heartbeat diagnostics. Operational dependency failures do not change the liveness HTTP status.",
+                "produces": [
+                    "application/json"
+                ],
+                "tags": [
+                    "health"
+                ],
+                "summary": "Check Agent health",
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/http.HealthResponse"
+                        }
+                    }
+                }
+            }
         }
     },
     "definitions": {
+        "http.Health": {
+            "type": "object",
+            "properties": {
+                "cameraConnected": {
+                    "type": "boolean"
+                },
+                "description": {
+                    "type": "string"
+                },
+                "hub": {
+                    "$ref": "#/definitions/http.HubHealth"
+                },
+                "mainStream": {
+                    "$ref": "#/definitions/http.StreamHealth"
+                },
+                "subStream": {
+                    "$ref": "#/definitions/http.StreamHealth"
+                }
+            }
+        },
+        "http.HealthResponse": {
+            "type": "object",
+            "properties": {
+                "applicationStatusCode": {
+                    "type": "string"
+                },
+                "data": {
+                    "$ref": "#/definitions/http.HealthResponseData"
+                },
+                "entityStatusCode": {
+                    "type": "string"
+                },
+                "httpStatusCode": {
+                    "type": "integer"
+                },
+                "message": {
+                    "type": "string"
+                },
+                "metadata": {
+                    "$ref": "#/definitions/http.ResponseMetadata"
+                }
+            }
+        },
+        "http.HealthResponseData": {
+            "type": "object",
+            "properties": {
+                "health": {
+                    "$ref": "#/definitions/http.Health"
+                }
+            }
+        },
+        "http.HubHealth": {
+            "type": "object",
+            "properties": {
+                "configured": {
+                    "type": "boolean"
+                },
+                "connected": {
+                    "type": "boolean"
+                },
+                "lastHeartbeatAttemptAt": {
+                    "type": "integer"
+                },
+                "lastSuccessfulHeartbeatAt": {
+                    "type": "integer"
+                }
+            }
+        },
+        "http.ResponseMetadata": {
+            "type": "object",
+            "properties": {
+                "applicationName": {
+                    "type": "string"
+                },
+                "applicationVersion": {
+                    "type": "string"
+                },
+                "path": {
+                    "type": "string"
+                },
+                "timestamp": {
+                    "type": "integer"
+                },
+                "traceId": {
+                    "type": "string"
+                }
+            }
+        },
+        "http.StreamHealth": {
+            "type": "object",
+            "properties": {
+                "configured": {
+                    "type": "boolean"
+                },
+                "connected": {
+                    "type": "boolean"
+                },
+                "fps": {
+                    "type": "number"
+                },
+                "lastPacketAt": {
+                    "type": "integer"
+                },
+                "packagesProcessed": {
+                    "type": "integer"
+                },
+                "resolution": {
+                    "$ref": "#/definitions/http.StreamResolution"
+                }
+            }
+        },
+        "http.StreamResolution": {
+            "type": "object",
+            "properties": {
+                "height": {
+                    "type": "integer"
+                },
+                "width": {
+                    "type": "integer"
+                }
+            }
+        },
         "models.APIResponse": {
             "type": "object",
             "properties": {
@@ -723,12 +931,19 @@
                 "fragmentedduration": {
                     "type": "integer"
                 },
+                "gopsize": {
+                    "description": "GOP size in seconds, used for pre-recording",
+                    "type": "integer"
+                },
                 "ipcamera": {
                     "$ref": "#/definitions/models.IPCamera"
                 },
                 "liveview": {
                     "type": "string"
                 },
+                "liveview_chunking": {
+                    "type": "string"
+                },
                 "maxlengthrecording": {
                     "type": "integer"
                 },
@@ -738,6 +953,10 @@
                 "name": {
                     "type": "string"
                 },
+                "onvif_motion": {
+                    "description": "ONVIFMotion routes the camera's ONVIF motion events into the\nagent's motion-triggered recording pipeline. When \"true\" the\nagent opens an event/stream against the configured ONVIF\nendpoint and forwards Motion+Active events to HandleMotion.\nRequires Capture.IPCamera.ONVIFXAddr / ONVIFUsername /\nONVIFPassword to be set. Default empty (disabled) keeps the\nexisting pixel-diff motion detection as the only source.",
+                    "type": "string"
+                },
                 "pixelChangeThreshold": {
                     "type": "integer"
                 },
@@ -816,9 +1035,15 @@
                 "kstorage": {
                     "$ref": "#/definitions/models.KStorage"
                 },
+                "kstorage_secondary": {
+                    "$ref": "#/definitions/models.KStorage"
+                },
                 "max_directory_size": {
                     "type": "integer"
                 },
+                "min_free_space": {
+                    "type": "integer"
+                },
                 "mqtt_password": {
                     "type": "string"
                 },
@@ -849,6 +1074,9 @@
                 "s3": {
                     "$ref": "#/definitions/models.S3"
                 },
+                "signing": {
+                    "$ref": "#/definitions/models.Signing"
+                },
                 "stunuri": {
                     "type": "string"
                 },
@@ -940,6 +1168,15 @@
         "models.IPCamera": {
             "type": "object",
             "properties": {
+                "base_height": {
+                    "type": "integer"
+                },
+                "base_width": {
+                    "type": "integer"
+                },
+                "channels": {
+                    "type": "integer"
+                },
                 "fps": {
                     "type": "string"
                 },
@@ -958,21 +1195,85 @@
                 "onvif_xaddr": {
                     "type": "string"
                 },
+                "pps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "rtsp": {
                     "type": "string"
                 },
+                "sample_rate": {
+                    "type": "integer"
+                },
+                "sps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_fps": {
                     "type": "string"
                 },
                 "sub_height": {
                     "type": "integer"
                 },
+                "sub_pps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_rtsp": {
                     "type": "string"
                 },
+                "sub_sps_nalus": {
+                    "description": "Sub stream parameter sets, captured separately from the main stream so the\nlive HLS muxer can build a correct init segment when a viewer switches the\nlive view to the sub (low-resolution) stream.",
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
+                "sub_vps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_width": {
                     "type": "integer"
                 },
+                "vps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "width": {
                     "type": "integer"
                 }
@@ -988,15 +1289,24 @@
                     "description": "old way, remove this",
                     "type": "string"
                 },
+                "custom_headers": {
+                    "type": "string"
+                },
                 "directory": {
                     "type": "string"
                 },
+                "max_retries": {
+                    "type": "integer"
+                },
                 "provider": {
                     "type": "string"
                 },
                 "secret_access_key": {
                     "type": "string"
                 },
+                "timeout": {
+                    "type": "integer"
+                },
                 "uri": {
                     "type": "string"
                 }
@@ -1134,6 +1444,17 @@
                 }
             }
         },
+        "models.Signing": {
+            "type": "object",
+            "properties": {
+                "enabled": {
+                    "type": "string"
+                },
+                "private_key": {
+                    "type": "string"
+                }
+            }
+        },
         "models.Timetable": {
             "type": "object",
             "properties": {

machinery/docs/swagger.yaml

@@ -1,5 +1,84 @@
 basePath: /
 definitions:
+  http.Health:
+    properties:
+      cameraConnected:
+        type: boolean
+      description:
+        type: string
+      hub:
+        $ref: '#/definitions/http.HubHealth'
+      mainStream:
+        $ref: '#/definitions/http.StreamHealth'
+      subStream:
+        $ref: '#/definitions/http.StreamHealth'
+    type: object
+  http.HealthResponse:
+    properties:
+      applicationStatusCode:
+        type: string
+      data:
+        $ref: '#/definitions/http.HealthResponseData'
+      entityStatusCode:
+        type: string
+      httpStatusCode:
+        type: integer
+      message:
+        type: string
+      metadata:
+        $ref: '#/definitions/http.ResponseMetadata'
+    type: object
+  http.HealthResponseData:
+    properties:
+      health:
+        $ref: '#/definitions/http.Health'
+    type: object
+  http.HubHealth:
+    properties:
+      configured:
+        type: boolean
+      connected:
+        type: boolean
+      lastHeartbeatAttemptAt:
+        type: integer
+      lastSuccessfulHeartbeatAt:
+        type: integer
+    type: object
+  http.ResponseMetadata:
+    properties:
+      applicationName:
+        type: string
+      applicationVersion:
+        type: string
+      path:
+        type: string
+      timestamp:
+        type: integer
+      traceId:
+        type: string
+    type: object
+  http.StreamHealth:
+    properties:
+      configured:
+        type: boolean
+      connected:
+        type: boolean
+      fps:
+        type: number
+      lastPacketAt:
+        type: integer
+      packagesProcessed:
+        type: integer
+      resolution:
+        $ref: '#/definitions/http.StreamResolution'
+    type: object
+  http.StreamResolution:
+    properties:
+      height:
+        type: integer
+      width:
+        type: integer
+    type: object
   models.APIResponse:
     properties:
       can_pan_tilt:
@@ -47,16 +126,31 @@ definitions:
         type: string
       fragmentedduration:
         type: integer
+      gopsize:
+        description: GOP size in seconds, used for pre-recording
+        type: integer
       ipcamera:
         $ref: '#/definitions/models.IPCamera'
       liveview:
         type: string
+      liveview_chunking:
+        type: string
       maxlengthrecording:
         type: integer
       motion:
         type: string
       name:
         type: string
+      onvif_motion:
+        description: |-
+          ONVIFMotion routes the camera's ONVIF motion events into the
+          agent's motion-triggered recording pipeline. When "true" the
+          agent opens an event/stream against the configured ONVIF
+          endpoint and forwards Motion+Active events to HandleMotion.
+          Requires Capture.IPCamera.ONVIFXAddr / ONVIFUsername /
+          ONVIFPassword to be set. Default empty (disabled) keeps the
+          existing pixel-diff motion detection as the only source.
+        type: string
       pixelChangeThreshold:
         type: integer
       postrecording:
@@ -109,8 +203,12 @@ definitions:
         type: string
       kstorage:
         $ref: '#/definitions/models.KStorage'
+      kstorage_secondary:
+        $ref: '#/definitions/models.KStorage'
       max_directory_size:
         type: integer
+      min_free_space:
+        type: integer
       mqtt_password:
         type: string
       mqtt_username:
@@ -131,6 +229,8 @@ definitions:
         type: string
       s3:
         $ref: '#/definitions/models.S3'
+      signing:
+        $ref: '#/definitions/models.Signing'
       stunuri:
         type: string
       time:
@@ -190,6 +290,12 @@ definitions:
     type: object
   models.IPCamera:
     properties:
+      base_height:
+        type: integer
+      base_width:
+        type: integer
+      channels:
+        type: integer
       fps:
         type: string
       height:
@@ -202,16 +308,64 @@ definitions:
         type: string
       onvif_xaddr:
         type: string
+      pps_nalus:
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
       rtsp:
         type: string
+      sample_rate:
+        type: integer
+      sps_nalus:
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
       sub_fps:
         type: string
       sub_height:
         type: integer
+      sub_pps_nalus:
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
       sub_rtsp:
         type: string
+      sub_sps_nalus:
+        description: |-
+          Sub stream parameter sets, captured separately from the main stream so the
+          live HLS muxer can build a correct init segment when a viewer switches the
+          live view to the sub (low-resolution) stream.
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
+      sub_vps_nalus:
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
       sub_width:
         type: integer
+      vps_nalus:
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
       width:
         type: integer
     type: object
@@ -224,10 +378,14 @@ definitions:
         type: string
       directory:
         type: string
+      max_retries:
+        type: integer
       provider:
         type: string
       secret_access_key:
         type: string
+      timeout:
+        type: integer
       uri:
         type: string
     type: object
@@ -316,6 +474,13 @@ definitions:
       username:
         type: string
     type: object
+  models.Signing:
+    properties:
+      enabled:
+        type: string
+      private_key:
+        type: string
+    type: object
   models.Timetable:
     properties:
       end1:
@@ -345,6 +510,30 @@ info:
   title: Swagger Kerberos Agent API
   version: "1.0"
 paths:
+  /api/camera/discover:
+    get:
+      description: Runs an advanced Fing/WiFiman-style scan (ONVIF WS-Discovery +
+        TCP port scan + MAC/vendor lookup) and returns the devices found on the local
+        network.
+      operationId: camera-discover
+      parameters:
+      - description: Discovery timeout in milliseconds (default 2000)
+        in: query
+        name: timeout
+        type: integer
+      - description: Optional subnet(s) to scan, e.g. '192.168.1.0/24' (comma-separated).
+          Defaults to the local interfaces.
+        in: query
+        name: subnet
+        type: string
+      responses:
+        "200":
+          description: OK
+          schema:
+            $ref: '#/definitions/models.APIResponse'
+      summary: Discover cameras and other devices on the local network.
+      tags:
+      - onvif
   /api/camera/onvif/capabilities:
     post:
       description: Will return the ONVIF capabilities for the specific camera.
@@ -735,6 +924,27 @@ paths:
       summary: Get Authorization token.
       tags:
       - authentication
+  /api/persistence/secondary/verify:
+    post:
+      description: Will verify the secondary persistence.
+      operationId: verify-secondary-persistence
+      parameters:
+      - description: Config
+        in: body
+        name: config
+        required: true
+        schema:
+          $ref: '#/definitions/models.Config'
+      responses:
+        "200":
+          description: OK
+          schema:
+            $ref: '#/definitions/models.APIResponse'
+      security:
+      - Bearer: []
+      summary: Will verify the secondary persistence.
+      tags:
+      - persistence
   /api/persistence/verify:
     post:
       description: Will verify the persistence.
@@ -756,6 +966,21 @@ paths:
       summary: Will verify the persistence.
       tags:
       - persistence
+  /health:
+    get:
+      description: Confirms that the Agent HTTP process can serve requests and reports
+        current camera stream and Hub heartbeat diagnostics. Operational dependency
+        failures do not change the liveness HTTP status.
+      produces:
+      - application/json
+      responses:
+        "200":
+          description: OK
+          schema:
+            $ref: '#/definitions/http.HealthResponse'
+      summary: Check Agent health
+      tags:
+      - health
 securityDefinitions:
   Bearer:
     in: header

machinery/go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/bluenviron/gortsplib/v5 v5.6.3
 	github.com/bluenviron/mediacommon v1.14.0
 	github.com/cedricve/go-onvif v0.0.0-20200222191200-567e8ce298f6
+	github.com/creack/pty v1.1.24
 	github.com/dromara/carbon/v2 v2.6.8
 	github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5
 	github.com/eclipse/paho.mqtt.golang v1.5.0
@@ -28,7 +29,6 @@ require (
 	github.com/minio/minio-go/v6 v6.0.57
 	github.com/moq-dev/moq-go v0.5.7
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
-	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/pion/interceptor v0.1.47
 	github.com/pion/rtp v1.10.5
 	github.com/pion/webrtc/v4 v4.2.18
@@ -45,7 +45,6 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.36.0
 	go.opentelemetry.io/otel/sdk v1.36.0
 	go.opentelemetry.io/otel/trace v1.36.0
-	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 )
 
 require (

machinery/go.sum

@@ -456,6 +456,8 @@ github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78/go.mod h1:W+zGtBO5Y1Ig
 github.com/cncf/xds/go v0.0.0-20241223141626-cff3c89139a3/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
 github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
+github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -858,8 +860,6 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
 github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE=
 github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
-github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
-github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
 github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
 github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY=
@@ -2106,8 +2106,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk=
 gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
-gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

machinery/logging.go

@@ -0,0 +1,160 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	defaultLogLevel  = log.InfoLevel
+	defaultLogOutput = "text"
+)
+
+type localTimeFormatter struct {
+	timezone  *time.Location
+	formatter log.Formatter
+}
+
+func (f localTimeFormatter) Format(entry *log.Entry) ([]byte, error) {
+	entry.Time = entry.Time.In(f.timezone)
+	return f.formatter.Format(entry)
+}
+
+type componentHook struct{}
+
+func (componentHook) Levels() []log.Level {
+	return log.AllLevels
+}
+
+func (componentHook) Fire(entry *log.Entry) error {
+	if _, exists := entry.Data["component"]; exists {
+		return nil
+	}
+	entry.Data["component"] = componentFromCaller(entry.Caller)
+	return nil
+}
+
+func configureLogging(levelValue string, outputValue string, timezone *time.Location) {
+	configureLogger(log.StandardLogger(), levelValue, outputValue, timezone)
+}
+
+func configureLogger(logger *log.Logger, levelValue string, outputValue string, timezone *time.Location) {
+	if timezone == nil {
+		timezone = time.Local
+	}
+
+	level, levelErr := parseLogLevel(levelValue)
+	output, outputErr := parseLogOutput(outputValue)
+
+	logger.SetOutput(os.Stdout)
+	logger.SetLevel(level)
+	logger.SetReportCaller(true)
+	logger.SetFormatter(localTimeFormatter{
+		timezone:  timezone,
+		formatter: newLogFormatter(output),
+	})
+	installComponentHook(logger)
+
+	if levelErr != nil {
+		logger.WithFields(log.Fields{
+			"configured_level": levelValue,
+			"effective_level":  level.String(),
+		}).WithError(levelErr).Warn("invalid log level; using default")
+	}
+	if outputErr != nil {
+		logger.WithFields(log.Fields{
+			"configured_output": outputValue,
+			"effective_output":  output,
+		}).WithError(outputErr).Warn("invalid log output; using default")
+	}
+
+	logger.WithFields(log.Fields{
+		"event":         "logger_configured",
+		"log_level":     level.String(),
+		"output":        output,
+		"report_caller": logger.ReportCaller,
+		"timezone":      timezone.String(),
+	}).Debug("logging configured")
+}
+
+func installComponentHook(logger *log.Logger) {
+	for _, hooks := range logger.Hooks {
+		for _, hook := range hooks {
+			if _, ok := hook.(componentHook); ok {
+				return
+			}
+		}
+	}
+	logger.AddHook(componentHook{})
+}
+
+func componentFromCaller(frame *runtime.Frame) string {
+	if frame == nil {
+		return "unknown"
+	}
+
+	const sourceMarker = "/machinery/src/"
+	normalizedFile := filepath.ToSlash(frame.File)
+	if markerIndex := strings.Index(normalizedFile, sourceMarker); markerIndex >= 0 {
+		relativeFile := normalizedFile[markerIndex+len(sourceMarker):]
+		if directory := filepath.ToSlash(filepath.Dir(relativeFile)); directory != "." {
+			return directory
+		}
+	}
+	if strings.Contains(normalizedFile, "/machinery/") {
+		return "agent"
+	}
+	return "unknown"
+}
+
+func parseLogLevel(value string) (log.Level, error) {
+	normalized := strings.ToLower(strings.TrimSpace(value))
+	if normalized == "" {
+		return defaultLogLevel, nil
+	}
+	if normalized == "warning" {
+		normalized = "warn"
+	}
+
+	level, err := log.ParseLevel(normalized)
+	if err != nil {
+		return defaultLogLevel, fmt.Errorf("parse LOG_LEVEL: %w", err)
+	}
+	return level, nil
+}
+
+func parseLogOutput(value string) (string, error) {
+	normalized := strings.ToLower(strings.TrimSpace(value))
+	if normalized == "" {
+		return defaultLogOutput, nil
+	}
+	switch normalized {
+	case "json", "text":
+		return normalized, nil
+	default:
+		return defaultLogOutput, fmt.Errorf("unsupported LOG_OUTPUT %q", value)
+	}
+}
+
+func newLogFormatter(output string) log.Formatter {
+	callerPrettyfier := func(frame *runtime.Frame) (string, string) {
+		return filepath.Base(frame.Function), fmt.Sprintf("%s:%d", filepath.Base(frame.File), frame.Line)
+	}
+	if output == "json" {
+		return &log.JSONFormatter{
+			CallerPrettyfier: callerPrettyfier,
+			TimestampFormat:  time.RFC3339Nano,
+		}
+	}
+	return &log.TextFormatter{
+		CallerPrettyfier: callerPrettyfier,
+		FullTimestamp:    true,
+		TimestampFormat:  time.RFC3339Nano,
+	}
+}

machinery/logging_test.go

@@ -0,0 +1,165 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func TestConfigureLoggingDebugIncludesStructuredContext(t *testing.T) {
+	logger := log.New()
+	var output bytes.Buffer
+	configureLogger(logger, "debug", "json", time.UTC)
+	logger.SetOutput(&output)
+	logger.WithField("event", "test_event").Debug("structured debug test")
+
+	lines := strings.Split(strings.TrimSpace(output.String()), "\n")
+	var entry map[string]interface{}
+	if err := json.Unmarshal([]byte(lines[len(lines)-1]), &entry); err != nil {
+		t.Fatalf("decode debug log: %v; output=%q", err, output.String())
+	}
+	for key, want := range map[string]interface{}{
+		"component": "agent",
+		"event":     "test_event",
+		"level":     "debug",
+		"msg":       "structured debug test",
+	} {
+		if got := entry[key]; got != want {
+			t.Fatalf("%s = %v, want %v", key, got, want)
+		}
+	}
+	if entry["file"] == nil || entry["func"] == nil {
+		t.Fatalf("debug log is missing caller metadata: %v", entry)
+	}
+}
+
+func TestConfigureLoggerInstallsComponentHookOnce(t *testing.T) {
+	logger := log.New()
+	configureLogger(logger, "info", "text", time.UTC)
+	configureLogger(logger, "debug", "json", time.UTC)
+
+	var componentHooks int
+	for _, hooks := range logger.Hooks {
+		for _, hook := range hooks {
+			if _, ok := hook.(componentHook); ok {
+				componentHooks++
+			}
+		}
+	}
+	if componentHooks != len(log.AllLevels) {
+		t.Fatalf("component hook registrations = %d, want %d", componentHooks, len(log.AllLevels))
+	}
+}
+
+func TestComponentFromCaller(t *testing.T) {
+	tests := []struct {
+		name  string
+		frame *runtime.Frame
+		want  string
+	}{
+		{
+			name:  "nested runtime package",
+			frame: &runtime.Frame{File: "/workspace/agent/machinery/src/routers/mqtt/main.go"},
+			want:  "routers/mqtt",
+		},
+		{
+			name:  "top-level runtime package",
+			frame: &runtime.Frame{File: "/workspace/agent/machinery/src/capture/main.go"},
+			want:  "capture",
+		},
+		{
+			name:  "executable",
+			frame: &runtime.Frame{File: "/workspace/agent/machinery/main.go"},
+			want:  "agent",
+		},
+		{name: "missing caller", want: "unknown"},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if got := componentFromCaller(test.frame); got != test.want {
+				t.Fatalf("componentFromCaller() = %q, want %q", got, test.want)
+			}
+		})
+	}
+}
+
+func TestComponentHookAddsAndPreservesComponent(t *testing.T) {
+	hook := componentHook{}
+
+	entry := log.NewEntry(log.New())
+	entry.Caller = &runtime.Frame{File: "/workspace/agent/machinery/src/cloud/livehls/session.go"}
+	if err := hook.Fire(entry); err != nil {
+		t.Fatalf("componentHook.Fire() error = %v", err)
+	}
+	if got := entry.Data["component"]; got != "cloud/livehls" {
+		t.Fatalf("component = %v, want cloud/livehls", got)
+	}
+
+	entry.Data["component"] = "explicit"
+	if err := hook.Fire(entry); err != nil {
+		t.Fatalf("componentHook.Fire() preserving field error = %v", err)
+	}
+	if got := entry.Data["component"]; got != "explicit" {
+		t.Fatalf("component = %v, want explicit", got)
+	}
+}
+
+func TestParseLogLevel(t *testing.T) {
+	tests := []struct {
+		name    string
+		value   string
+		want    log.Level
+		wantErr bool
+	}{
+		{name: "default", want: log.InfoLevel},
+		{name: "info", value: "INFO", want: log.InfoLevel},
+		{name: "warning alias", value: "warning", want: log.WarnLevel},
+		{name: "debug", value: "debug", want: log.DebugLevel},
+		{name: "trace", value: "trace", want: log.TraceLevel},
+		{name: "invalid", value: "verbose", want: log.InfoLevel, wantErr: true},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got, err := parseLogLevel(test.value)
+			if (err != nil) != test.wantErr {
+				t.Fatalf("parseLogLevel(%q) error = %v, wantErr %t", test.value, err, test.wantErr)
+			}
+			if got != test.want {
+				t.Fatalf("parseLogLevel(%q) = %s, want %s", test.value, got, test.want)
+			}
+		})
+	}
+}
+
+func TestParseLogOutput(t *testing.T) {
+	tests := []struct {
+		name    string
+		value   string
+		want    string
+		wantErr bool
+	}{
+		{name: "default", want: "text"},
+		{name: "text", value: "TEXT", want: "text"},
+		{name: "json", value: "json", want: "json"},
+		{name: "invalid", value: "console", want: "text", wantErr: true},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got, err := parseLogOutput(test.value)
+			if (err != nil) != test.wantErr {
+				t.Fatalf("parseLogOutput(%q) error = %v, wantErr %t", test.value, err, test.wantErr)
+			}
+			if got != test.want {
+				t.Fatalf("parseLogOutput(%q) = %q, want %q", test.value, got, test.want)
+			}
+		})
+	}
+}

machinery/main.go

@@ -5,14 +5,15 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/capture"
 	"github.com/kerberos-io/agent/machinery/src/components"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/onvif"
+	log "github.com/sirupsen/logrus"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
@@ -28,6 +29,22 @@ import (
 
 var VERSION = utils.VERSION
 
+func resolveServerPort(flagValue, environmentValue string) (string, error) {
+	value := strings.TrimSpace(environmentValue)
+	if value == "" {
+		value = strings.TrimSpace(flagValue)
+	}
+	if value == "" {
+		value = "80"
+	}
+
+	port, err := strconv.Atoi(value)
+	if err != nil || port < 1 || port > 65535 {
+		return "", fmt.Errorf("port must be an integer between 1 and 65535, got %q", value)
+	}
+	return strconv.Itoa(port), nil
+}
+
 func startTracing(agentKey string, otelEndpoint string) (*trace.TracerProvider, error) {
 	serviceName := "agent-" + agentKey
 	headers := map[string]string{
@@ -99,20 +116,44 @@ func main() {
 	}
 	// Specify the timezone of the log: "UTC" or "Local".
 	timezone, _ := time.LoadLocation("CET")
-	log.Log.Init(logLevel, logOutput, configDirectory, timezone)
+	configureLogging(logLevel, logOutput, timezone)
+	if action == "run" {
+		resolvedPort, err := resolveServerPort(port, os.Getenv("AGENT_PORT"))
+		if err != nil {
+			log.WithError(err).WithFields(log.Fields{
+				"component": "http",
+				"event":     "server_port_invalid",
+			}).Fatal("Invalid HTTP server port")
+			return
+		}
+		port = resolvedPort
+	}
+	log.WithFields(log.Fields{
+		"action":           action,
+		"component":        "agent",
+		"config_directory": configDirectory,
+		"event":            "command_parsed",
+		"port":             port,
+		"version":          VERSION,
+	}).Debug("Agent command parsed")
 
 	switch action {
 
 	case "version":
 		{
-			log.Log.Info("main.Main(): You are currrently running Kerberos Agent " + VERSION)
+			log.WithFields(log.Fields{
+				"component": "agent",
+				"event":     "version",
+				"version":   VERSION,
+			}).Info("Kerberos Agent version")
 		}
 	case "discover":
 		{
 			// Convert duration to int
 			timeout, err := time.ParseDuration(timeout + "ms")
 			if err != nil {
-				log.Log.Fatal("main.Main(): could not parse timeout: " + err.Error())
+				log.WithError(err).WithField("component", "onvif").
+					Fatal("invalid ONVIF discovery timeout")
 				return
 			}
 			var subnets []string
@@ -125,15 +166,19 @@ func main() {
 		}
 	case "decrypt":
 		{
-			log.Log.Info("main.Main(): Decrypting: " + flag.Arg(0) + " with key: " + flag.Arg(1))
+			log.WithFields(log.Fields{
+				"component": "encryption",
+				"event":     "decrypt_started",
+				"path":      flag.Arg(0),
+			}).Info("Decrypting recording")
 			symmetricKey := []byte(flag.Arg(1))
 
 			if len(symmetricKey) == 0 {
-				log.Log.Fatal("main.Main(): symmetric key should not be empty")
+				log.Fatal("main.Main(): symmetric key should not be empty")
 				return
 			}
 			if len(symmetricKey) != 32 {
-				log.Log.Fatal("main.Main(): symmetric key should be 32 bytes")
+				log.Fatal("main.Main(): symmetric key should be 32 bytes")
 				return
 			}
 
@@ -163,19 +208,28 @@ func main() {
 
 			// Start OpenTelemetry tracing
 			if otelEndpoint := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"); otelEndpoint == "" {
-				log.Log.Info("main.Main(): No OpenTelemetry endpoint provided, skipping tracing")
+				log.WithFields(log.Fields{
+					"component": "tracing",
+					"event":     "tracing_disabled",
+				}).Debug("OpenTelemetry tracing disabled")
 			} else {
-				log.Log.Info("main.Main(): Starting OpenTelemetry tracing with endpoint: " + otelEndpoint)
+				log.WithFields(log.Fields{
+					"component": "tracing",
+					"event":     "tracing_starting",
+				}).Info("Starting OpenTelemetry tracing")
 				agentKey := configuration.Config.Key
 				traceProvider, err := startTracing(agentKey, otelEndpoint)
 				if err != nil {
-					log.Log.Error("traceprovider: " + err.Error())
+					log.WithError(err).WithField("component", "tracing").
+						Error("Failed to start OpenTelemetry tracing")
+				} else {
+					defer func() {
+						if err := traceProvider.Shutdown(context.Background()); err != nil {
+							log.WithError(err).WithField("component", "tracing").
+								Error("Failed to shut down OpenTelemetry tracing")
+						}
+					}()
 				}
-				defer func() {
-					if err := traceProvider.Shutdown(context.Background()); err != nil {
-						log.Log.Error("traceprovider: " + err.Error())
-					}
-				}()
 			}
 
 			// Printing final configuration
@@ -186,8 +240,13 @@ func main() {
 			utils.CheckDataDirectoryPermissions(configDirectory)
 
 			// Set timezone
-			timezone, _ := time.LoadLocation(configuration.Config.Timezone)
-			log.Log.Init(logLevel, logOutput, configDirectory, timezone)
+			timezone, err := time.LoadLocation(configuration.Config.Timezone)
+			if err != nil {
+				log.WithError(err).WithField("timezone", configuration.Config.Timezone).
+					Warn("invalid Agent timezone; using the host timezone for logs")
+				timezone = time.Local
+			}
+			configureLogging(logLevel, logOutput, timezone)
 
 			// Check if we have a device Key or not, if not
 			// we will generate one.
@@ -196,30 +255,37 @@ func main() {
 				configuration.Config.Key = key
 				err := configService.StoreConfig(configDirectory, configuration.Config)
 				if err == nil {
-					log.Log.Info("main.Main(): updated unique key for agent to: " + key)
+					log.WithFields(log.Fields{
+						"component": "configuration",
+						"event":     "agent_key_generated",
+					}).Info("Generated and stored a unique Agent key")
 				} else {
-					log.Log.Info("main.Main(): something went wrong while trying to store key: " + key)
+					log.WithError(err).WithFields(log.Fields{
+						"component": "configuration",
+						"event":     "agent_key_store_failed",
+					}).Error("Failed to store the generated Agent key")
 				}
 			}
 
 			// Create a cancelable context, which will be used to cancel and restart.
 			// This is used to restart the agent when the configuration is updated.
 			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
 
 			// We create a capture object, this will contain all the streaming clients.
 			// And allow us to extract media from within difference places in the agent.
-			capture := capture.Capture{
-				RTSPClient:    nil,
-				RTSPSubClient: nil,
-			}
+			capture := capture.Capture{}
 
 			// Bootstrapping the agent
 			communication := models.Communication{
-				Context:         &ctx,
-				CancelContext:   &cancel,
 				HandleBootstrap: make(chan string, 1),
 			}
 
+			log.WithFields(log.Fields{
+				"component": "agent",
+				"event":     "runtime_starting",
+				"port":      configuration.Port,
+			}).Info("Starting Agent runtime")
 			go components.Bootstrap(ctx, configDirectory, &configuration, &communication, &capture)
 
 			// Start the REST API.
@@ -227,7 +293,7 @@ func main() {
 		}
 	default:
 		{
-			log.Log.Error("main.Main(): Sorry I don't understand :(")
+			log.Error("main.Main(): Sorry I don't understand :(")
 		}
 	}
 }

machinery/main_test.go

@@ -0,0 +1,33 @@
+package main
+
+import "testing"
+
+func TestResolveServerPort(t *testing.T) {
+	tests := []struct {
+		name      string
+		flagValue string
+		envValue  string
+		want      string
+		wantError bool
+	}{
+		{name: "flag default", flagValue: "80", want: "80"},
+		{name: "environment overrides flag", flagValue: "80", envValue: "8082", want: "8082"},
+		{name: "trims environment", flagValue: "80", envValue: " 9090 ", want: "9090"},
+		{name: "empty values use default", want: "80"},
+		{name: "invalid text", flagValue: "80", envValue: "http", wantError: true},
+		{name: "zero", flagValue: "80", envValue: "0", wantError: true},
+		{name: "above maximum", flagValue: "80", envValue: "65536", wantError: true},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got, err := resolveServerPort(test.flagValue, test.envValue)
+			if (err != nil) != test.wantError {
+				t.Fatalf("resolveServerPort(%q, %q) error = %v, wantError %t", test.flagValue, test.envValue, err, test.wantError)
+			}
+			if got != test.want {
+				t.Fatalf("resolveServerPort(%q, %q) = %q, want %q", test.flagValue, test.envValue, got, test.want)
+			}
+		})
+	}
+}

machinery/src/capture/gortsplib.go

@@ -13,9 +13,12 @@ import (
 	"errors"
 	"fmt"
 	"image"
+	"math"
+	"net/url"
 	"os"
 	"reflect"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 	"unsafe"
@@ -32,10 +35,10 @@ import (
 	"github.com/bluenviron/mediacommon/pkg/codecs/h264"
 	"github.com/bluenviron/mediacommon/pkg/codecs/h265"
 	"github.com/bluenviron/mediacommon/pkg/codecs/mpeg4audio"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/pion/rtp"
+	log "github.com/sirupsen/logrus"
 	"go.opentelemetry.io/otel"
 )
 
@@ -71,12 +74,31 @@ func rtspsTLSConfig() (*tls.Config, error) {
 	return &tls.Config{RootCAs: rootCAs}, nil
 }
 
+func sanitizeRTSPError(err error, rawURL string) error {
+	if err == nil {
+		return nil
+	}
+	if rawURL == "" || !strings.Contains(err.Error(), rawURL) {
+		return err
+	}
+
+	replacement := "<redacted-rtsp-url>"
+	if parsed, parseErr := url.Parse(rawURL); parseErr == nil {
+		parsed.User = nil
+		parsed.RawQuery = ""
+		parsed.Fragment = ""
+		replacement = parsed.String()
+	}
+	return errors.New(strings.ReplaceAll(err.Error(), rawURL, replacement))
+}
+
 // Implements the RTSPClient interface.
 type Golibrtsp struct {
 	RTSPClient
 	Url string
 
 	Client            gortsplib.Client
+	clientStarted     bool
 	VideoDecoderMutex *sync.Mutex
 
 	VideoH264Index        int8
@@ -197,14 +219,20 @@ func (h *streamHealth) observePacket(streamType string, writeDur time.Duration)
 		h.gapMax = gap
 	}
 	if writeDur >= streamHealthWriteWarn {
-		log.Log.Warning(fmt.Sprintf(
-			"capture.golibrtsp.health(%s): WritePacket blocked %dms — downstream back-pressure / CPU starvation",
-			streamType, writeDur.Milliseconds()))
+		log.WithFields(log.Fields{
+			"component":   "capture",
+			"duration_ms": writeDur.Milliseconds(),
+			"event":       "packet_queue_write_slow",
+			"stream":      streamType,
+		}).Warn("Packet queue write was slow")
 	}
 	if gap >= streamHealthGapWarn {
-		log.Log.Warning(fmt.Sprintf(
-			"capture.golibrtsp.health(%s): %dms since previous frame — upstream network / camera stall",
-			streamType, gap.Milliseconds()))
+		log.WithFields(log.Fields{
+			"component":    "capture",
+			"event":        "frame_gap_detected",
+			"frame_gap_ms": gap.Milliseconds(),
+			"stream":       streamType,
+		}).Warn("Large gap detected between camera frames")
 	}
 	if now.Sub(h.windowStart) >= streamHealthWindow {
 		elapsed := now.Sub(h.windowStart).Seconds()
@@ -212,10 +240,20 @@ func (h *streamHealth) observePacket(streamType string, writeDur time.Duration)
 		if h.frames > 0 {
 			avgWriteMs = float64(h.writeSum.Milliseconds()) / float64(h.frames)
 		}
-		log.Log.Info(fmt.Sprintf(
-			"capture.golibrtsp.health(%s): %.0fs window — frames=%d (%.1f/s) writeAvg=%.1fms writeMax=%dms gapMax=%dms lost=%d decodeErrs=%d",
-			streamType, elapsed, h.frames, float64(h.frames)/elapsed, avgWriteMs,
-			h.writeMax.Milliseconds(), h.gapMax.Milliseconds(), h.lost, h.decodeErrs))
+		log.WithFields(log.Fields{
+			"average_write_ms": avgWriteMs,
+			"component":        "capture",
+			"decode_errors":    h.decodeErrs,
+			"event":            "stream_health_sampled",
+			"frame_rate":       float64(h.frames) / elapsed,
+			"frames":           h.frames,
+			"lost_packets":     h.lost,
+			"max_frame_gap_ms": h.gapMax.Milliseconds(),
+			"max_write_ms":     h.writeMax.Milliseconds(),
+			"stream":           streamType,
+			"window_seconds":   elapsed,
+		}).Debug("RTSP stream health sampled")
+
 		h.windowStart = now
 		h.frames = 0
 		h.writeSum = 0
@@ -236,9 +274,12 @@ func (h *streamHealth) observeLost(streamType string, lost uint64) {
 	h.mu.Lock()
 	h.lost += lost
 	h.mu.Unlock()
-	log.Log.Warning(fmt.Sprintf(
-		"capture.golibrtsp.health(%s): %d RTP packet(s) lost — sender-side gap (receiver not draining TCP fast enough)",
-		streamType, lost))
+	log.WithFields(log.Fields{
+		"component":    "capture",
+		"event":        "rtp_packets_lost",
+		"lost_packets": lost,
+		"stream":       streamType,
+	}).Warn("RTSP sender reported lost RTP packets")
 }
 
 // observeDecodeError is invoked by gortsplib on incomplete/invalid access units,
@@ -250,7 +291,11 @@ func (h *streamHealth) observeDecodeError(streamType string, err error) {
 	h.mu.Lock()
 	h.decodeErrs++
 	h.mu.Unlock()
-	log.Log.Debug(fmt.Sprintf("capture.golibrtsp.health(%s): decode error: %s", streamType, err.Error()))
+	log.WithError(err).WithFields(log.Fields{
+		"component": "capture",
+		"event":     "access_unit_decode_failed",
+		"stream":    streamType,
+	}).Debug("Failed to decode RTSP access unit")
 }
 
 // fpsTracker holds per-stream state for PTS-based FPS calculation.
@@ -334,13 +379,13 @@ func init() {
 	// setup H264 -> raw frames decoder
 	H264FrameDecoder, err = newDecoder("H264")
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.init(): " + err.Error())
+		log.Error("capture.golibrtsp.init(): " + err.Error())
 	}
 
 	// setup H265 -> raw frames decoder
 	H265FrameDecoder, err = newDecoder("H265")
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.init(): " + err.Error())
+		log.Error("capture.golibrtsp.init(): " + err.Error())
 	}
 }
 
@@ -376,8 +421,7 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 	// parse URL
 	u, err := base.ParseURL(g.Url)
 	if err != nil {
-		log.Log.Debug("capture.golibrtsp.Connect(ParseURL): " + err.Error())
-		return
+		return sanitizeRTSPError(err, g.Url)
 	}
 
 	// connect to the server
@@ -385,14 +429,14 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 	g.Client.Host = u.Host
 	err = g.Client.Start()
 	if err != nil {
-		log.Log.Debug("capture.golibrtsp.Connect(Start): " + err.Error())
+		return sanitizeRTSPError(err, g.Url)
 	}
+	g.clientStarted = true
 
 	// find published medias
 	desc, _, err := g.Client.Describe(u)
 	if err != nil {
-		log.Log.Debug("capture.golibrtsp.Connect(Describe): " + err.Error())
-		return
+		return sanitizeRTSPError(err, g.Url)
 	}
 
 	// Initialize the mutex and FPS calculation.
@@ -405,13 +449,13 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 	g.VideoH264Media = mediH264
 	g.VideoH264Forma = formaH264
 	if mediH264 == nil {
-		log.Log.Debug("capture.golibrtsp.Connect(H264): " + "video media not found")
+		log.Debug("capture.golibrtsp.Connect(H264): " + "video media not found")
 	} else {
 		// setup a video media
 		_, err = g.Client.Setup(desc.BaseURL, mediH264, 0, 0)
 		if err != nil {
 			// Something went wrong .. Do something
-			log.Log.Error("capture.golibrtsp.Connect(H264): " + err.Error())
+			log.Error("capture.golibrtsp.Connect(H264): " + err.Error())
 		} else {
 			// Get SPS and PPS from the SDP
 			// Calculate the width and height of the video
@@ -420,7 +464,7 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 			// It might be that the SPS is not available yet, so we'll proceed,
 			// but try to fetch it later on.
 			if errSPS != nil {
-				log.Log.Debug("capture.golibrtsp.Connect(H264): " + errSPS.Error())
+				log.Debug("capture.golibrtsp.Connect(H264): " + errSPS.Error())
 				streamIndex := len(g.Streams)
 				g.Streams = append(g.Streams, packets.Stream{
 					Index:         streamIndex,
@@ -456,7 +500,7 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 			// setup RTP/H264 -> H264 decoder
 			rtpDec, err := formaH264.CreateDecoder()
 			if err != nil {
-				log.Log.Error("capture.golibrtsp.Connect(H264): " + err.Error())
+				log.Error("capture.golibrtsp.Connect(H264): " + err.Error())
 			}
 			g.VideoH264Decoder = rtpDec
 			g.VideoH264FrameDecoder = H264FrameDecoder
@@ -469,20 +513,20 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 	g.VideoH265Media = mediH265
 	g.VideoH265Forma = formaH265
 	if mediH265 == nil {
-		log.Log.Debug("capture.golibrtsp.Connect(H265): " + "video media not found")
+		log.Debug("capture.golibrtsp.Connect(H265): " + "video media not found")
 	} else {
 		// setup a video media
 		_, err = g.Client.Setup(desc.BaseURL, mediH265, 0, 0)
 		if err != nil {
 			// Something went wrong .. Do something
-			log.Log.Error("capture.golibrtsp.Connect(H265): " + err.Error())
+			log.Error("capture.golibrtsp.Connect(H265): " + err.Error())
 		} else {
 			// Get SPS from the SDP
 			// Calculate the width and height of the video
 			var sps h265.SPS
 			err = sps.Unmarshal(formaH265.SPS)
 			if err != nil {
-				log.Log.Info("capture.golibrtsp.Connect(H265): " + err.Error())
+				log.Info("capture.golibrtsp.Connect(H265): " + err.Error())
 				return
 			}
 			streamIndex := len(g.Streams)
@@ -506,7 +550,7 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 			// setup RTP/H265 -> H265 decoder
 			rtpDec, err := formaH265.CreateDecoder()
 			if err != nil {
-				log.Log.Error("capture.golibrtsp.Connect(H265): " + err.Error())
+				log.Error("capture.golibrtsp.Connect(H265): " + err.Error())
 			}
 			g.VideoH265Decoder = rtpDec
 
@@ -520,19 +564,19 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 	g.AudioG711Media = audioMedi
 	g.AudioG711Forma = audioForma
 	if audioMedi == nil {
-		log.Log.Debug("capture.golibrtsp.Connect(G711): " + "audio media not found")
+		log.Debug("capture.golibrtsp.Connect(G711): " + "audio media not found")
 	} else {
 		// setup a audio media
 		_, err = g.Client.Setup(desc.BaseURL, audioMedi, 0, 0)
 		if err != nil {
 			// Something went wrong .. Do something
-			log.Log.Error("capture.golibrtsp.Connect(G711): " + err.Error())
+			log.Error("capture.golibrtsp.Connect(G711): " + err.Error())
 		} else {
 			// create decoder
 			audiortpDec, err := audioForma.CreateDecoder()
 			if err != nil {
 				// Something went wrong .. Do something
-				log.Log.Error("capture.golibrtsp.Connect(G711): " + err.Error())
+				log.Error("capture.golibrtsp.Connect(G711): " + err.Error())
 			} else {
 				g.AudioG711Decoder = audiortpDec
 				streamIndex := len(g.Streams)
@@ -556,19 +600,19 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 	g.AudioOpusMedia = audioMediOpus
 	g.AudioOpusForma = audioFormaOpus
 	if audioMediOpus == nil {
-		log.Log.Debug("capture.golibrtsp.Connect(Opus): " + "audio media not found")
+		log.Debug("capture.golibrtsp.Connect(Opus): " + "audio media not found")
 	} else {
 		// setup a audio media
 		_, err = g.Client.Setup(desc.BaseURL, audioMediOpus, 0, 0)
 		if err != nil {
 			// Something went wrong .. Do something
-			log.Log.Error("capture.golibrtsp.Connect(Opus): " + err.Error())
+			log.Error("capture.golibrtsp.Connect(Opus): " + err.Error())
 		} else {
 			// create decoder
 			audiortpDec, err := audioFormaOpus.CreateDecoder()
 			if err != nil {
 				// Something went wrong .. Do something
-				log.Log.Error("capture.golibrtsp.Connect(Opus): " + err.Error())
+				log.Error("capture.golibrtsp.Connect(Opus): " + err.Error())
 			} else {
 				g.AudioOpusDecoder = audiortpDec
 				streamIndex := len(g.Streams)
@@ -592,13 +636,13 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 	g.AudioMPEG4Media = audioMediMPEG4
 	g.AudioMPEG4Forma = audioFormaMPEG4
 	if audioMediMPEG4 == nil {
-		log.Log.Debug("capture.golibrtsp.Connect(MPEG4): " + "audio media not found")
+		log.Debug("capture.golibrtsp.Connect(MPEG4): " + "audio media not found")
 	} else {
 		// setup a audio media
 		_, err = g.Client.Setup(desc.BaseURL, audioMediMPEG4, 0, 0)
 		if err != nil {
 			// Something went wrong .. Do something
-			log.Log.Error("capture.golibrtsp.Connect(MPEG4): " + err.Error())
+			log.Error("capture.golibrtsp.Connect(MPEG4): " + err.Error())
 		} else {
 			streamIndex := len(g.Streams)
 			g.Streams = append(g.Streams, packets.Stream{
@@ -618,7 +662,7 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 			audiortpDec, err := audioFormaMPEG4.CreateDecoder()
 			if err != nil {
 				// Something went wrong .. Do something
-				log.Log.Error("capture.golibrtsp.Connect(MPEG4): " + err.Error())
+				log.Error("capture.golibrtsp.Connect(MPEG4): " + err.Error())
 			}
 			g.AudioMPEG4Decoder = audiortpDec
 
@@ -648,8 +692,7 @@ func (g *Golibrtsp) ConnectBackChannel(ctx context.Context, ctxRunAgent context.
 	// parse URL
 	u, err := base.ParseURL(g.Url)
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.ConnectBackChannel(): " + err.Error())
-		return
+		return sanitizeRTSPError(err, g.Url)
 	}
 
 	// connect to the server
@@ -657,14 +700,14 @@ func (g *Golibrtsp) ConnectBackChannel(ctx context.Context, ctxRunAgent context.
 	g.Client.Host = u.Host
 	err = g.Client.Start()
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.ConnectBackChannel(): " + err.Error())
+		return sanitizeRTSPError(err, g.Url)
 	}
+	g.clientStarted = true
 
 	// find published medias
 	desc, _, err := g.Client.Describe(u)
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.ConnectBackChannel(): " + err.Error())
-		return
+		return sanitizeRTSPError(err, g.Url)
 	}
 
 	// Look for audio back channel.
@@ -674,15 +717,17 @@ func (g *Golibrtsp) ConnectBackChannel(ctx context.Context, ctxRunAgent context.
 	g.AudioG711MediaBackChannel = audioMediBackChannel
 	g.AudioG711FormaBackChannel = audioFormaBackChannel
 	if audioMediBackChannel == nil {
-		log.Log.Error("capture.golibrtsp.ConnectBackChannel(): audio backchannel not found, not a real error, however you might expect a backchannel. One of the reasons might be that the device already has an active client connected to the backchannel.")
+		log.WithFields(log.Fields{
+			"component": "capture",
+			"event":     "backchannel_unavailable",
+		}).Debug("Optional camera audio backchannel unavailable")
 		err = errors.New("no audio backchannel found")
 	} else {
 		// setup a audio media
 		_, err = g.Client.Setup(desc.BaseURL, audioMediBackChannel, 0, 0)
 		if err != nil {
-			// Something went wrong .. Do something
-			log.Log.Error("capture.golibrtsp.ConnectBackChannel(): " + err.Error())
 			g.HasBackChannel = false
+			return sanitizeRTSPError(err, g.Url)
 		} else {
 			g.HasBackChannel = true
 			streamIndex := len(g.Streams)
@@ -735,9 +780,26 @@ func ptsToDuration(pts int64, clockRate int) time.Duration {
 		time.Duration(pts%rate)*time.Second/time.Duration(rate)
 }
 
+func preRecordingGOPCount(preRecording int64, gopDuration float64) (int, bool) {
+	if preRecording <= 0 ||
+		gopDuration < 1 ||
+		math.IsNaN(gopDuration) ||
+		math.IsInf(gopDuration, 0) ||
+		gopDuration >= float64(math.MaxInt64) {
+		return 0, false
+	}
+
+	count := preRecording / int64(gopDuration)
+	maxInt := int64(^uint(0) >> 1)
+	if count >= maxInt {
+		return 0, false
+	}
+	return int(count) + 1, true
+}
+
 // Start the RTSP client, and start reading packets.
 func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets.Queue, configuration *models.Configuration, communication *models.Communication) (err error) {
-	log.Log.Debug("capture.golibrtsp.Start(): started")
+	log.Debug("capture.golibrtsp.Start(): started")
 
 	// Label this client's loss/decode/health logging with the stream type.
 	g.streamLabel = streamType
@@ -751,7 +813,7 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 			// decode timestamp
 			pts2, ok := g.Client.PacketPTS(g.AudioG711Media, rtppkt)
 			if !ok {
-				log.Log.Debug("capture.golibrtsp.Start(): " + "unable to get PTS")
+				log.Debug("capture.golibrtsp.Start(): " + "unable to get PTS")
 				return
 			}
 			pts := ptsToDuration(pts2, g.AudioG711Forma.ClockRate())
@@ -759,7 +821,7 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 			// extract LPCM samples from RTP packets
 			op, err := g.AudioG711Decoder.Decode(rtppkt)
 			if err != nil {
-				log.Log.Error("capture.golibrtsp.Start(): " + err.Error())
+				log.Error("capture.golibrtsp.Start(): " + err.Error())
 				return
 			}
 
@@ -786,7 +848,7 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 			// decode timestamp
 			pts2, ok := g.Client.PacketPTS(g.AudioMPEG4Media, rtppkt)
 			if !ok {
-				log.Log.Error("capture.golibrtsp.Start(): " + "unable to get PTS")
+				log.Error("capture.golibrtsp.Start(): " + "unable to get PTS")
 				return
 			}
 			pts := ptsToDuration(pts2, g.AudioMPEG4Forma.ClockRate())
@@ -795,13 +857,13 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 			// extract access units from RTP packets
 			aus, err := g.AudioMPEG4Decoder.Decode(rtppkt)
 			if err != nil {
-				log.Log.Error("capture.golibrtsp.Start(): " + err.Error())
+				log.Error("capture.golibrtsp.Start(): " + err.Error())
 				return
 			}
 
 			enc, err := WriteMPEG4Audio(g.AudioMPEG4Forma, aus)
 			if err != nil {
-				log.Log.Error("capture.golibrtsp.Start(): " + err.Error())
+				log.Error("capture.golibrtsp.Start(): " + err.Error())
 				return
 			}
 
@@ -845,7 +907,7 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 				// decode timestamps — validate each call separately
 				pts2, okPTS2 := g.Client.PacketPTS(g.VideoH264Media, rtppkt)
 				if !okPTS2 {
-					log.Log.Debug("capture.golibrtsp.Start(): unable to get PTS")
+					log.Debug("capture.golibrtsp.Start(): unable to get PTS")
 					return
 				}
 				pts := ptsToDuration(pts2, g.VideoH264Forma.ClockRate())
@@ -856,7 +918,7 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 				au, errDecode := g.VideoH264Decoder.Decode(rtppkt)
 				if errDecode != nil {
 					if errDecode != rtph264.ErrNonStartingPacketAndNoPrevious && errDecode != rtph264.ErrMorePacketsNeeded {
-						log.Log.Error("capture.golibrtsp.Start(): " + errDecode.Error())
+						log.Error("capture.golibrtsp.Start(): " + errDecode.Error())
 					}
 					return
 				}
@@ -918,7 +980,12 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 							fps := g.getEnhancedFPS(&sps, g.VideoH264Index)
 							g.Streams[g.VideoH264Index].FPS = fps
 							g.persistStreamFPS(configuration, streamType, fps)
-							log.Log.Debug(fmt.Sprintf("capture.golibrtsp.Start(%s): Final FPS=%.2f", streamType, fps))
+							log.WithFields(log.Fields{
+								"component": "capture",
+								"event":     "frame_rate_detected",
+								"fps":       fps,
+								"stream":    streamType,
+							}).Debug("RTSP frame rate detected")
 							g.VideoH264Forma.SPS = nalu
 							if streamType == "main" && len(nalu) > 0 {
 								// Fallback: store SPS from in-band NALUs when SDP was missing it.
@@ -940,14 +1007,14 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 					// Ensure config has parameter sets before recordings start.
 					if len(configuration.Config.Capture.IPCamera.SPSNALUs) == 0 && len(g.VideoH264Forma.SPS) > 0 {
 						configuration.Config.Capture.IPCamera.SPSNALUs = [][]byte{g.VideoH264Forma.SPS}
-						log.Log.Warning("capture.golibrtsp.Start(main): fallback SPS set from keyframe")
+						log.Warn("capture.golibrtsp.Start(main): fallback SPS set from keyframe")
 					}
 					if len(configuration.Config.Capture.IPCamera.PPSNALUs) == 0 && len(g.VideoH264Forma.PPS) > 0 {
 						configuration.Config.Capture.IPCamera.PPSNALUs = [][]byte{g.VideoH264Forma.PPS}
-						log.Log.Warning("capture.golibrtsp.Start(main): fallback PPS set from keyframe")
+						log.Warn("capture.golibrtsp.Start(main): fallback PPS set from keyframe")
 					}
 					if len(configuration.Config.Capture.IPCamera.SPSNALUs) == 0 || len(configuration.Config.Capture.IPCamera.PPSNALUs) == 0 {
-						log.Log.Warning("capture.golibrtsp.Start(main): SPS/PPS still missing after IDR keyframe")
+						log.Warn("capture.golibrtsp.Start(main): SPS/PPS still missing after IDR keyframe")
 					}
 				}
 
@@ -956,14 +1023,18 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 				}
 
 				if idrPresent {
-					log.Log.Debug(fmt.Sprintf("capture.golibrtsp.Start(%s): IDR frame NALUs: [%s]",
-						streamType, fmt.Sprintf("%v", naluTypes)))
+					log.WithFields(log.Fields{
+						"component":  "capture",
+						"event":      "idr_frame_received",
+						"nalu_types": naluTypes,
+						"stream":     streamType,
+					}).Debug("RTSP IDR frame received")
 				}
 
 				// Convert to packet.
 				enc, err := h264.AnnexBMarshal(filteredAU)
 				if err != nil {
-					log.Log.Error("capture.golibrtsp.Start(): " + err.Error())
+					log.Error("capture.golibrtsp.Start(): " + err.Error())
 					return
 				}
 
@@ -997,11 +1068,19 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 					gopDuration := float64(keyframeInterval) / fps
 					gopSize := int(avgInterval) // Store GOP size in a separate variable
 					g.Streams[g.VideoH264Index].GopSize = gopSize
-					log.Log.Debug(fmt.Sprintf("capture.golibrtsp.Start(%s): Keyframe interval=%d packets, Avg=%.1f, GOP=%.1fs, GOPSize=%d",
-						streamType, keyframeInterval, avgInterval, gopDuration, gopSize))
+					log.WithFields(log.Fields{
+						"average_keyframe_interval_packets": avgInterval,
+						"component":                         "capture",
+						"event":                             "keyframe_interval_observed",
+						"gop_duration_seconds":              gopDuration,
+						"gop_size_packets":                  gopSize,
+						"keyframe_interval_packets":         keyframeInterval,
+						"stream":                            streamType,
+					}).Debug("RTSP keyframe interval observed")
+
 					preRecording := configuration.Config.Capture.PreRecording
-					if preRecording > 0 && int(gopDuration) > 0 {
-						queue.SetMaxGopCount(int(preRecording)/int(gopDuration) + 1)
+					if maxGOPCount, ok := preRecordingGOPCount(preRecording, gopDuration); ok {
+						queue.SetMaxGopCount(maxGOPCount)
 					}
 				}
 
@@ -1048,20 +1127,36 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 				default:
 				}
 
-				if idrPresent {
-					// Increment packets, so we know the device
-					// is not blocking.
-					if streamType == "main" {
-						r := communication.PackageCounter.Load().(int64)
-						log.Log.Debug("capture.golibrtsp.Start(): packet size " + strconv.Itoa(len(pkt.Data)))
-						communication.PackageCounter.Store((r + 1) % 1000)
-						communication.LastPacketTimer.Store(time.Now().Unix())
-					} else if streamType == "sub" {
-						r := communication.PackageCounterSub.Load().(int64)
-						log.Log.Debug("capture.golibrtsp.Start(): packet size " + strconv.Itoa(len(pkt.Data)))
-						communication.PackageCounterSub.Store((r + 1) % 1000)
-						communication.LastPacketTimerSub.Store(time.Now().Unix())
-					}
+				// Count every complete video access unit. Keyframe-only counters make
+				// healthy cameras with GOPs longer than the watchdog window look stalled.
+				if streamType == "main" {
+					observedAt := time.Now()
+					r := communication.PackageCounter.Load().(int64)
+					log.WithFields(log.Fields{
+						"bytes":     len(pkt.Data),
+						"codec":     "H264",
+						"component": "capture",
+						"event":     "access_unit_received",
+						"keyframe":  pkt.IsKeyFrame,
+						"stream":    streamType,
+					}).Trace("RTSP access unit received")
+					communication.RecordStreamPackage(models.MainStream, g.Streams[g.VideoH264Index].FPS, g.Streams[g.VideoH264Index].Width, g.Streams[g.VideoH264Index].Height, observedAt)
+					communication.PackageCounter.Store((r + 1) % 1000)
+					communication.LastPacketTimer.Store(observedAt.Unix())
+				} else if streamType == "sub" {
+					observedAt := time.Now()
+					r := communication.PackageCounterSub.Load().(int64)
+					log.WithFields(log.Fields{
+						"bytes":     len(pkt.Data),
+						"codec":     "H264",
+						"component": "capture",
+						"event":     "access_unit_received",
+						"keyframe":  pkt.IsKeyFrame,
+						"stream":    streamType,
+					}).Trace("RTSP access unit received")
+					communication.RecordStreamPackage(models.SubStream, g.Streams[g.VideoH264Index].FPS, g.Streams[g.VideoH264Index].Width, g.Streams[g.VideoH264Index].Height, observedAt)
+					communication.PackageCounterSub.Store((r + 1) % 1000)
+					communication.LastPacketTimerSub.Store(observedAt.Unix())
 				}
 			}
 
@@ -1090,7 +1185,7 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 				// decode timestamps — validate each call separately
 				pts2, okPTS2 := g.Client.PacketPTS(g.VideoH265Media, rtppkt)
 				if !okPTS2 {
-					log.Log.Debug("capture.golibrtsp.Start(): unable to get PTS")
+					log.Debug("capture.golibrtsp.Start(): unable to get PTS")
 					return
 				}
 				pts := ptsToDuration(pts2, g.VideoH265Forma.ClockRate())
@@ -1101,7 +1196,7 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 				au, errDecode := g.VideoH265Decoder.Decode(rtppkt)
 				if errDecode != nil {
 					if errDecode != rtph265.ErrNonStartingPacketAndNoPrevious && errDecode != rtph265.ErrMorePacketsNeeded {
-						log.Log.Error("capture.golibrtsp.Start(): " + errDecode.Error())
+						log.Error("capture.golibrtsp.Start(): " + errDecode.Error())
 					}
 					return
 				}
@@ -1159,7 +1254,7 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 
 				enc, err := h264.AnnexBMarshal(au)
 				if err != nil {
-					log.Log.Error("capture.golibrtsp.Start(): " + err.Error())
+					log.Error("capture.golibrtsp.Start(): " + err.Error())
 					return
 				}
 
@@ -1191,11 +1286,19 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 					gopDuration := float64(keyframeInterval) / fps
 					gopSize := int(avgInterval) // Store GOP size in a separate variable
 					g.Streams[g.VideoH265Index].GopSize = gopSize
-					log.Log.Debug(fmt.Sprintf("capture.golibrtsp.Start(%s): Keyframe interval=%d packets, Avg=%.1f, GOP=%.1fs, GOPSize=%d",
-						streamType, keyframeInterval, avgInterval, gopDuration, gopSize))
+					log.WithFields(log.Fields{
+						"average_keyframe_interval_packets": avgInterval,
+						"component":                         "capture",
+						"event":                             "keyframe_interval_observed",
+						"gop_duration_seconds":              gopDuration,
+						"gop_size_packets":                  gopSize,
+						"keyframe_interval_packets":         keyframeInterval,
+						"stream":                            streamType,
+					}).Debug("RTSP keyframe interval observed")
+
 					preRecording := configuration.Config.Capture.PreRecording
-					if preRecording > 0 && int(gopDuration) > 0 {
-						queue.SetMaxGopCount(int(preRecording)/int(gopDuration) + 1)
+					if maxGOPCount, ok := preRecordingGOPCount(preRecording, gopDuration); ok {
+						queue.SetMaxGopCount(maxGOPCount)
 					}
 				}
 
@@ -1213,20 +1316,36 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 				default:
 				}
 
-				if isRandomAccess {
-					// Increment packets, so we know the device
-					// is not blocking.
-					if streamType == "main" {
-						r := communication.PackageCounter.Load().(int64)
-						log.Log.Debug("capture.golibrtsp.Start(): packet size " + strconv.Itoa(len(pkt.Data)))
-						communication.PackageCounter.Store((r + 1) % 1000)
-						communication.LastPacketTimer.Store(time.Now().Unix())
-					} else if streamType == "sub" {
-						r := communication.PackageCounterSub.Load().(int64)
-						log.Log.Debug("capture.golibrtsp.Start(): packet size " + strconv.Itoa(len(pkt.Data)))
-						communication.PackageCounterSub.Store((r + 1) % 1000)
-						communication.LastPacketTimerSub.Store(time.Now().Unix())
-					}
+				// Count every complete video access unit; random-access frames remain
+				// responsible only for GOP tracking above.
+				if streamType == "main" {
+					observedAt := time.Now()
+					r := communication.PackageCounter.Load().(int64)
+					log.WithFields(log.Fields{
+						"bytes":     len(pkt.Data),
+						"codec":     "H265",
+						"component": "capture",
+						"event":     "access_unit_received",
+						"keyframe":  pkt.IsKeyFrame,
+						"stream":    streamType,
+					}).Trace("RTSP access unit received")
+					communication.RecordStreamPackage(models.MainStream, g.Streams[g.VideoH265Index].FPS, g.Streams[g.VideoH265Index].Width, g.Streams[g.VideoH265Index].Height, observedAt)
+					communication.PackageCounter.Store((r + 1) % 1000)
+					communication.LastPacketTimer.Store(observedAt.Unix())
+				} else if streamType == "sub" {
+					observedAt := time.Now()
+					r := communication.PackageCounterSub.Load().(int64)
+					log.WithFields(log.Fields{
+						"bytes":     len(pkt.Data),
+						"codec":     "H265",
+						"component": "capture",
+						"event":     "access_unit_received",
+						"keyframe":  pkt.IsKeyFrame,
+						"stream":    streamType,
+					}).Trace("RTSP access unit received")
+					communication.RecordStreamPackage(models.SubStream, g.Streams[g.VideoH265Index].FPS, g.Streams[g.VideoH265Index].Width, g.Streams[g.VideoH265Index].Height, observedAt)
+					communication.PackageCounterSub.Store((r + 1) % 1000)
+					communication.LastPacketTimerSub.Store(observedAt.Unix())
 				}
 			}
 
@@ -1238,7 +1357,7 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 	// Play the stream.
 	_, err = g.Client.Play(nil)
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.Start(): " + err.Error())
+		log.Error("capture.golibrtsp.Start(): " + err.Error())
 	}
 
 	return
@@ -1246,13 +1365,13 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 
 // Start the RTSP client, and start reading packets.
 func (g *Golibrtsp) StartBackChannel(ctx context.Context, ctxRunAgent context.Context) (err error) {
-	log.Log.Info("capture.golibrtsp.StartBackChannel(): started")
+	log.Info("capture.golibrtsp.StartBackChannel(): started")
 	// Wait for a second, so we can be sure the stream is playing.
 	time.Sleep(1 * time.Second)
 	// Play the stream.
 	_, err = g.Client.Play(nil)
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.StartBackChannel(): " + err.Error())
+		log.Error("capture.golibrtsp.StartBackChannel(): " + err.Error())
 	}
 	return
 }
@@ -1261,7 +1380,7 @@ func (g *Golibrtsp) WritePacket(pkt packets.Packet) error {
 	if g.HasBackChannel && g.AudioG711MediaBackChannel != nil {
 		err := g.Client.WritePacketRTP(g.AudioG711MediaBackChannel, pkt.Packet)
 		if err != nil {
-			log.Log.Debug("capture.golibrtsp.WritePacket(): " + err.Error())
+			log.Debug("capture.golibrtsp.WritePacket(): " + err.Error())
 			return err
 		}
 	}
@@ -1284,11 +1403,11 @@ func (g *Golibrtsp) DecodePacket(pkt packets.Packet) (image.YCbCr, error) {
 	}
 	g.VideoDecoderMutex.Unlock()
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.DecodePacket(): " + err.Error())
+		log.Error("capture.golibrtsp.DecodePacket(): " + err.Error())
 		return image.YCbCr{}, err
 	}
 	if img.Bounds().Empty() {
-		log.Log.Debug("capture.golibrtsp.DecodePacket(): empty frame")
+		log.Debug("capture.golibrtsp.DecodePacket(): empty frame")
 		return image.YCbCr{}, errors.New("Empty image")
 	}
 	return img, nil
@@ -1310,11 +1429,11 @@ func (g *Golibrtsp) DecodePacketRaw(pkt packets.Packet) (image.Gray, error) {
 	}
 	g.VideoDecoderMutex.Unlock()
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.DecodePacketRaw(): " + err.Error())
+		log.Error("capture.golibrtsp.DecodePacketRaw(): " + err.Error())
 		return image.Gray{}, err
 	}
 	if img.Bounds().Empty() {
-		log.Log.Debug("capture.golibrtsp.DecodePacketRaw(): empty image")
+		log.Debug("capture.golibrtsp.DecodePacketRaw(): empty image")
 		return image.Gray{}, errors.New("Empty image")
 	}
 
@@ -1359,7 +1478,11 @@ func (g *Golibrtsp) Close(ctxOtel context.Context) error {
 	_, span := tracer.Start(ctxOtel, "Close")
 	defer span.End()
 
-	// Close the demuxer.
+	if !g.clientStarted {
+		return nil
+	}
+
+	g.clientStarted = false
 	g.Client.Close()
 
 	// We will have created the decoders globally, so we don't need to close them here.
@@ -1404,13 +1527,13 @@ func newDecoder(codecName string) (*Decoder, error) {
 
 	res := C.avcodec_open2(codecCtx, codec, nil)
 	if res < 0 {
-		C.avcodec_close(codecCtx)
+		C.avcodec_free_context(&codecCtx)
 		return nil, fmt.Errorf("avcodec_open2() failed")
 	}
 
 	srcFrame := C.av_frame_alloc()
 	if srcFrame == nil {
-		C.avcodec_close(codecCtx)
+		C.avcodec_free_context(&codecCtx)
 		return nil, fmt.Errorf("av_frame_alloc() failed")
 	}
 
@@ -1426,7 +1549,7 @@ func (d *Decoder) Close() {
 		C.av_frame_free(&d.srcFrame)
 	}
 	C.av_frame_free(&d.srcFrame)
-	C.avcodec_close(d.codecCtx)
+	C.avcodec_free_context(&d.codecCtx)
 }
 
 func (d *Decoder) decode(nalu []byte) (image.YCbCr, error) {
@@ -1613,7 +1736,12 @@ func (g *Golibrtsp) getEnhancedFPS(sps *h264.SPS, streamIndex int8) float64 {
 
 	// Check if SPS FPS is reasonable (between 1 and 120 fps)
 	if spsFPS > 0 && spsFPS <= 120 {
-		log.Log.Debug(fmt.Sprintf("capture.golibrtsp.getEnhancedFPS(): SPS FPS: %.2f", spsFPS))
+		log.WithFields(log.Fields{
+			"component": "capture",
+			"event":     "frame_rate_selected",
+			"fps":       spsFPS,
+			"source":    "sps",
+		}).Debug("RTSP frame rate selected")
 		return spsFPS
 	}
 
@@ -1621,7 +1749,12 @@ func (g *Golibrtsp) getEnhancedFPS(sps *h264.SPS, streamIndex int8) float64 {
 	if ft := g.fpsTrackers[streamIndex]; ft != nil {
 		ptsFPS := ft.fps()
 		if ptsFPS > 0 && ptsFPS <= 120 {
-			log.Log.Debug(fmt.Sprintf("capture.golibrtsp.getEnhancedFPS(): PTS FPS: %.2f", ptsFPS))
+			log.WithFields(log.Fields{
+				"component": "capture",
+				"event":     "frame_rate_selected",
+				"fps":       ptsFPS,
+				"source":    "presentation_timestamp",
+			}).Debug("RTSP frame rate selected")
 			return ptsFPS
 		}
 	}
@@ -1723,13 +1856,15 @@ func (g *Golibrtsp) getSPSTimingInfo(sps *h264.SPS) (hasVUI bool, timeScale uint
 func (g *Golibrtsp) debugSPSInfo(sps *h264.SPS, streamType string) {
 	hasVUI, timeScale, numUnitsInTick, fps := g.getSPSTimingInfo(sps)
 
-	log.Log.Debug(fmt.Sprintf("capture.golibrtsp.debugSPSInfo(%s): Width=%d, Height=%d",
-		streamType, sps.Width(), sps.Height()))
-	log.Log.Debug(fmt.Sprintf("capture.golibrtsp.debugSPSInfo(%s): HasVUI=%t, FPS=%.2f",
-		streamType, hasVUI, fps))
-
-	if hasVUI {
-		log.Log.Debug(fmt.Sprintf("capture.golibrtsp.debugSPSInfo(%s): TimeScale=%d, NumUnitsInTick=%d",
-			streamType, timeScale, numUnitsInTick))
-	}
+	log.WithFields(log.Fields{
+		"component":         "capture",
+		"event":             "sps_inspected",
+		"fps":               fps,
+		"has_vui":           hasVUI,
+		"height_pixels":     sps.Height(),
+		"num_units_in_tick": numUnitsInTick,
+		"stream":            streamType,
+		"time_scale":        timeScale,
+		"width_pixels":      sps.Width(),
+	}).Debug("RTSP sequence parameter set inspected")
 }

machinery/src/capture/gortsplib_test.go

@@ -2,13 +2,70 @@ package capture
 
 import (
 	"bytes"
+	"context"
 	"encoding/pem"
+	"errors"
+	"math"
 	"net/http/httptest"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 )
 
+func TestPreRecordingGOPCount(t *testing.T) {
+	maxInt := int64(^uint(0) >> 1)
+	tests := []struct {
+		name         string
+		preRecording int64
+		gopDuration  float64
+		want         int
+		wantOK       bool
+	}{
+		{name: "normal duration", preRecording: 10, gopDuration: 2.9, want: 6, wantOK: true},
+		{name: "duration longer than buffer", preRecording: 1, gopDuration: 2, want: 1, wantOK: true},
+		{name: "largest representable result", preRecording: maxInt - 1, gopDuration: 1, want: int(maxInt), wantOK: true},
+		{name: "result exceeds int", preRecording: maxInt, gopDuration: 1, wantOK: false},
+		{name: "non-positive pre-recording", preRecording: 0, gopDuration: 1, wantOK: false},
+		{name: "sub-second GOP", preRecording: 10, gopDuration: 0.9, wantOK: false},
+		{name: "NaN GOP", preRecording: 10, gopDuration: math.NaN(), wantOK: false},
+		{name: "infinite GOP", preRecording: 10, gopDuration: math.Inf(1), wantOK: false},
+		{name: "GOP exceeds int64", preRecording: 10, gopDuration: float64(math.MaxInt64), wantOK: false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, ok := preRecordingGOPCount(tt.preRecording, tt.gopDuration)
+			if ok != tt.wantOK || got != tt.want {
+				t.Fatalf("preRecordingGOPCount(%d, %v) = (%d, %t), want (%d, %t)",
+					tt.preRecording, tt.gopDuration, got, ok, tt.want, tt.wantOK)
+			}
+		})
+	}
+}
+
+func TestGolibrtspCloseBeforeClientStart(t *testing.T) {
+	client := &Golibrtsp{}
+
+	if err := client.Close(context.Background()); err != nil {
+		t.Fatalf("Close() error = %v", err)
+	}
+}
+
+func TestSanitizeRTSPErrorRemovesCredentialsAndQuery(t *testing.T) {
+	rawURL := "rtsp://camera-user:camera-password@10.0.20.15/live?access_token=secret"
+	got := sanitizeRTSPError(errors.New("describe "+rawURL+": bad status code"), rawURL)
+
+	for _, secret := range []string{"camera-user", "camera-password", "access_token", "secret"} {
+		if strings.Contains(got.Error(), secret) {
+			t.Fatalf("sanitizeRTSPError() exposed %q in %q", secret, got)
+		}
+	}
+	if !strings.Contains(got.Error(), "rtsp://10.0.20.15/live") {
+		t.Fatalf("sanitizeRTSPError() removed useful host/path context: %q", got)
+	}
+}
+
 func TestRTSPSTLSConfig(t *testing.T) {
 	t.Run("verifies certificates by default", func(t *testing.T) {
 		t.Setenv(rtspsCAFileEnv, "")

machinery/src/capture/main.go

@@ -16,11 +16,11 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/kerberos-io/agent/machinery/src/conditions"
 	"github.com/kerberos-io/agent/machinery/src/encryption"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/kerberos-io/agent/machinery/src/utils"
 	"github.com/kerberos-io/agent/machinery/src/video"
+	log "github.com/sirupsen/logrus"
 	"go.opentelemetry.io/otel/trace"
 )
 
@@ -49,16 +49,17 @@ func publishRecordingState(mqttClient mqtt.Client, hubKey string, configuration
 	if err == nil {
 		mqttClient.Publish("kerberos/hub/"+hubKey, 2, false, payload)
 	} else {
-		log.Log.Error("capture.main.publishRecordingState(): failed to package MQTT message: " + err.Error())
+		log.Error("capture.main.publishRecordingState(): failed to package MQTT message: " + err.Error())
 	}
 }
 
-func recordingUploadMetadata(name, deviceKey string, timestamp int64, mp4Video *video.MP4) models.RecordingUploadMetadata {
+func recordingUploadMetadata(name, deviceKey string, timestamp int64, mp4Video *video.MP4, encrypted bool) models.RecordingUploadMetadata {
 	metadata := models.RecordingUploadMetadata{
 		FileName:  filepath.Base(name),
 		DeviceKey: deviceKey,
 		Timestamp: timestamp,
 		Duration:  mp4Video.VideoTotalDuration,
+		Encrypted: encrypted,
 	}
 	value := mp4Video.AverageFPS()
 	if value > 0 && value <= 240 && !math.IsInf(value, 0) && !math.IsNaN(value) {
@@ -72,7 +73,7 @@ func recordingUploadMetadata(name, deviceKey string, timestamp int64, mp4Video *
 func queueRecordingForUpload(configDirectory string, metadata models.RecordingUploadMetadata) {
 	payload, err := json.Marshal(metadata)
 	if err != nil {
-		log.Log.Error("capture.main.queueRecordingForUpload(): " + err.Error())
+		log.Error("capture.main.queueRecordingForUpload(): " + err.Error())
 		return
 	}
 
@@ -95,7 +96,7 @@ func queueRecordingForUpload(configDirectory string, metadata models.RecordingUp
 		err = os.Rename(marker.Name(), filepath.Join(configDirectory, "data", "cloud", models.RecordingUploadMetadataFileName(metadata.FileName)))
 	}
 	if err != nil {
-		log.Log.Error("capture.main.queueRecordingForUpload(): " + err.Error())
+		log.Error("capture.main.queueRecordingForUpload(): " + err.Error())
 	}
 }
 
@@ -137,9 +138,9 @@ func manualRecordingExpired(communication *models.Communication, now int64) bool
 		return false
 	}
 	if heartbeatExpired {
-		log.Log.Info("capture.main.HandleRecordStream(motiondetection): auto-stopping manual recording, no viewer heartbeat within timeout.")
+		log.Info("capture.main.HandleRecordStream(motiondetection): auto-stopping manual recording, no viewer heartbeat within timeout.")
 	} else {
-		log.Log.Info("capture.main.HandleRecordStream(motiondetection): auto-stopping manual recording, maximum duration reached.")
+		log.Info("capture.main.HandleRecordStream(motiondetection): auto-stopping manual recording, maximum duration reached.")
 	}
 	communication.IsRecordingManual.UnSet()
 	communication.RecordingManualHeartbeat.Store(0)
@@ -151,7 +152,7 @@ func manualRecordingExpired(communication *models.Communication, now int64) bool
 func CleanupRecordingDirectory(configDirectory string, configuration *models.Configuration) {
 	autoClean := configuration.Config.AutoClean
 	if autoClean != "true" {
-		log.Log.Info("HandleRecordStream: Autoclean disabled, nothing to do here.")
+		log.Info("HandleRecordStream: Autoclean disabled, nothing to do here.")
 		return
 	}
 
@@ -164,7 +165,7 @@ func CleanupRecordingDirectory(configDirectory string, configuration *models.Con
 	// disk while keeping a free-space reserve.
 	needsCleanup, err := recordingsNeedCleanup(recordingsDirectory, configuration)
 	if err != nil {
-		log.Log.Info("HandleRecordStream: something went wrong, " + err.Error())
+		log.Info("HandleRecordStream: something went wrong, " + err.Error())
 		return
 	}
 	if !needsCleanup {
@@ -185,12 +186,12 @@ func CleanupRecordingDirectory(configDirectory string, configuration *models.Con
 	// (otherwise a long outage would fill the disk and stop new recordings).
 	name, pending, err := pickRecordingToCleanup(recordingsDirectory, cloudDirectory)
 	if err != nil {
-		log.Log.Info("HandleRecordStream: something went wrong, " + err.Error())
+		log.Info("HandleRecordStream: something went wrong, " + err.Error())
 		return
 	}
 
 	if err := os.Remove(recordingsDirectory + "/" + name); err != nil {
-		log.Log.Info("HandleRecordStream: something went wrong, " + err.Error())
+		log.Info("HandleRecordStream: something went wrong, " + err.Error())
 		return
 	}
 
@@ -200,14 +201,14 @@ func CleanupRecordingDirectory(configDirectory string, configuration *models.Con
 		// that was never uploaded to keep recording new footage. Also remove the
 		// now-dangling upload marker so the upload loop doesn't keep trying to
 		// upload a file that no longer exists.
-		log.Log.Warning("HandleRecordStream: removed oldest recording as part of cleanup, but it was STILL PENDING UPLOAD (disk full of un-uploaded recordings) - " + recordingsDirectory + "/" + name)
+		log.Warn("HandleRecordStream: removed oldest recording as part of cleanup, but it was STILL PENDING UPLOAD (disk full of un-uploaded recordings) - " + recordingsDirectory + "/" + name)
 		for _, markerName := range uploadMarkerNames(name) {
 			if err := os.Remove(filepath.Join(cloudDirectory, markerName)); err != nil && !os.IsNotExist(err) {
-				log.Log.Info("HandleRecordStream: could not remove dangling upload marker " + markerName + ", " + err.Error())
+				log.Info("HandleRecordStream: could not remove dangling upload marker " + markerName + ", " + err.Error())
 			}
 		}
 	} else {
-		log.Log.Info("HandleRecordStream: removed oldest file as part of cleanup - " + recordingsDirectory + "/" + name)
+		log.Info("HandleRecordStream: removed oldest file as part of cleanup - " + recordingsDirectory + "/" + name)
 	}
 }
 
@@ -331,7 +332,7 @@ func recordingPendingUpload(cloudDirectory, recordingName string) bool {
 	return false
 }
 
-func HandleRecordStream(queue *packets.Queue, configDirectory string, configuration *models.Configuration, communication *models.Communication, rtspClient RTSPClient, mqttClient mqtt.Client) {
+func HandleRecordStream(queue *packets.Queue, configDirectory string, configuration *models.Configuration, communication *models.Communication, rtspClient RTSPClient, mqttClient mqtt.Client, motionEvents <-chan models.MotionDataPartial) {
 
 	config := configuration.Config
 	hubKey := config.HubKey
@@ -345,9 +346,9 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 	communication.RecordingManualHeartbeatSeen.UnSet()
 
 	if config.Capture.Recording == "false" {
-		log.Log.Info("capture.main.HandleRecordStream(): disabled, we will not record anything.")
+		log.Info("capture.main.HandleRecordStream(): disabled, we will not record anything.")
 	} else {
-		log.Log.Debug("capture.main.HandleRecordStream(): started")
+		log.Debug("capture.main.HandleRecordStream(): started")
 
 		preRecording := config.Capture.PreRecording * 1000
 		postRecording := config.Capture.PostRecording * 1000           // number of seconds to record.
@@ -360,7 +361,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 		}
 
 		if maxRecordingPeriod < preRecording+postRecording {
-			log.Log.Error("capture.main.HandleRecordStream(): maxRecordingPeriod is less than preRecording + postRecording, this is not allowed. Setting maxRecordingPeriod to preRecording + postRecording.")
+			log.Error("capture.main.HandleRecordStream(): maxRecordingPeriod is less than preRecording + postRecording, this is not allowed. Setting maxRecordingPeriod to preRecording + postRecording.")
 			maxRecordingPeriod = preRecording + postRecording
 		}
 
@@ -394,7 +395,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 			var name string
 
 			// Do not do anything!
-			log.Log.Info("capture.main.HandleRecordStream(continuous): start recording")
+			log.Info("capture.main.HandleRecordStream(continuous): start recording")
 
 			start := false
 
@@ -440,10 +441,10 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					}
 					if (videoCodec == "H264" && (len(mp4Video.SPSNALUs) == 0 || len(mp4Video.PPSNALUs) == 0)) ||
 						(videoCodec == "H265" && (len(mp4Video.VPSNALUs) == 0 || len(mp4Video.SPSNALUs) == 0 || len(mp4Video.PPSNALUs) == 0)) {
-						log.Log.Warning("capture.main.HandleRecordStream(continuous): closing MP4 without full parameter sets, moov may be incomplete")
+						log.Warn("capture.main.HandleRecordStream(continuous): closing MP4 without full parameter sets, moov may be incomplete")
 					}
 					mp4Video.Close(&config)
-					log.Log.Info("capture.main.HandleRecordStream(continuous): recording finished: file save: " + name)
+					log.Info("capture.main.HandleRecordStream(continuous): recording finished: file save: " + name)
 
 					// Cleanup muxer
 					start = false
@@ -467,7 +468,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 						oldName := name
 						name = s + ".mp4"
 						fullName = configDirectory + "/data/recordings/" + name
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
+						log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
 
 						// Rename the file to the new name.
 						err := os.Rename(
@@ -475,12 +476,13 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 							configDirectory+"/data/recordings/"+s+".mp4")
 
 						if err != nil {
-							log.Log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
+							log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
 						}
 					} else {
-						log.Log.Info("capture.main.HandleRecordStream(continuous): no video data recorded, not renaming file.")
+						log.Info("capture.main.HandleRecordStream(continuous): no video data recorded, not renaming file.")
 					}
 
+					encrypted := false
 					// Check if we need to encrypt the recording.
 					if config.Encryption != nil && config.Encryption.Enabled == "true" && config.Encryption.Recordings == "true" && config.Encryption.SymmetricKey != "" {
 						// reopen file into memory 'fullName'
@@ -492,17 +494,19 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 								// write back to file
 								err := os.WriteFile(fullName, []byte(encryptedContents), 0644)
 								if err != nil {
-									log.Log.Error("capture.main.HandleRecordStream(continuous): error writing file: " + err.Error())
+									log.Error("capture.main.HandleRecordStream(continuous): error writing file: " + err.Error())
+								} else {
+									encrypted = true
 								}
 							} else {
-								log.Log.Error("capture.main.HandleRecordStream(continuous): error encrypting file: " + err.Error())
+								log.Error("capture.main.HandleRecordStream(continuous): error encrypting file: " + err.Error())
 							}
 						} else {
-							log.Log.Error("capture.main.HandleRecordStream(continuous): error reading file: " + err.Error())
+							log.Error("capture.main.HandleRecordStream(continuous): error reading file: " + err.Error())
 						}
 					}
 
-					queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, startRecording, mp4Video))
+					queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, startRecording, mp4Video, encrypted))
 
 					recordingStatus = "idle"
 
@@ -520,7 +524,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					// We'll validate those conditions and if not valid we'll not do anything.
 					valid, err := conditions.Validate(loc, configuration)
 					if !valid && err != nil {
-						log.Log.Debug("capture.main.HandleRecordStream(continuous): " + err.Error() + ".")
+						log.Debug("capture.main.HandleRecordStream(continuous): " + err.Error() + ".")
 						time.Sleep(5 * time.Second)
 						continue
 					}
@@ -552,7 +556,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					fullName = configDirectory + "/data/recordings/" + name
 
 					// Running...
-					log.Log.Info("capture.main.HandleRecordStream(continuous): recording started")
+					log.Info("capture.main.HandleRecordStream(continuous): recording started")
 
 					// Get width and height from the camera.
 					width := configuration.Config.Capture.IPCamera.Width
@@ -564,7 +568,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					vpsNALUS := configuration.Config.Capture.IPCamera.VPSNALUs
 
 					if len(spsNALUS) == 0 || len(ppsNALUS) == 0 {
-						log.Log.Warning("capture.main.HandleRecordStream(continuous): missing SPS/PPS at recording start")
+						log.Warn("capture.main.HandleRecordStream(continuous): missing SPS/PPS at recording start")
 					}
 					// Create a video file, and set the dimensions.
 					mp4Video = video.NewMP4(fullName, spsNALUS, ppsNALUS, vpsNALUS, configuration.Config.Capture.MaxLengthRecording)
@@ -579,7 +583,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					if audioCodec == "AAC" {
 						audioTrack = mp4Video.AddAudioTrack("AAC")
 					} else if audioCodec == "PCM_MULAW" {
-						log.Log.Debug("capture.main.HandleRecordStream(continuous): no AAC audio codec detected, skipping audio track.")
+						log.Debug("capture.main.HandleRecordStream(continuous): no AAC audio codec detected, skipping audio track.")
 					}
 
 					writeSampleToMP4(mp4Video, videoTrack, audioTrack, pkt)
@@ -600,7 +604,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 			if cursorError != nil {
 				if recordingStatus == "started" {
 
-					log.Log.Info("capture.main.HandleRecordStream(continuous): Recording finished: file save: " + name)
+					log.Info("capture.main.HandleRecordStream(continuous): Recording finished: file save: " + name)
 
 					// Cleanup muxer
 					start = false
@@ -624,7 +628,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 						oldName := name
 						name = s + ".mp4"
 						fullName = configDirectory + "/data/recordings/" + name
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
+						log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
 
 						// Rename the file to the new name.
 						err := os.Rename(
@@ -632,12 +636,13 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 							configDirectory+"/data/recordings/"+s+".mp4")
 
 						if err != nil {
-							log.Log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
+							log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
 						}
 					} else {
-						log.Log.Info("capture.main.HandleRecordStream(continuous): no video data recorded, not renaming file.")
+						log.Info("capture.main.HandleRecordStream(continuous): no video data recorded, not renaming file.")
 					}
 
+					encrypted := false
 					// Check if we need to encrypt the recording.
 					if config.Encryption != nil && config.Encryption.Enabled == "true" && config.Encryption.Recordings == "true" && config.Encryption.SymmetricKey != "" {
 						// reopen file into memory 'fullName'
@@ -649,17 +654,19 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 								// write back to file
 								err := os.WriteFile(fullName, []byte(encryptedContents), 0644)
 								if err != nil {
-									log.Log.Error("capture.main.HandleRecordStream(motiondetection): error writing file: " + err.Error())
+									log.Error("capture.main.HandleRecordStream(motiondetection): error writing file: " + err.Error())
+								} else {
+									encrypted = true
 								}
 							} else {
-								log.Log.Error("capture.main.HandleRecordStream(motiondetection): error encrypting file: " + err.Error())
+								log.Error("capture.main.HandleRecordStream(motiondetection): error encrypting file: " + err.Error())
 							}
 						} else {
-							log.Log.Error("capture.main.HandleRecordStream(motiondetection): error reading file: " + err.Error())
+							log.Error("capture.main.HandleRecordStream(motiondetection): error reading file: " + err.Error())
 						}
 					}
 
-					queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, startRecording, mp4Video))
+					queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, startRecording, mp4Video, encrypted))
 
 					recordingStatus = "idle"
 
@@ -672,7 +679,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 			}
 		} else {
 
-			log.Log.Info("capture.main.HandleRecordStream(motiondetection): Start motion based recording ")
+			log.Info("capture.main.HandleRecordStream(motiondetection): Start motion based recording ")
 
 			var lastRecordingTime int64 = 0 // last recording timestamp in milliseconds
 			var displayTime int64 = 0       // display time in milliseconds
@@ -680,7 +687,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 			var videoTrack uint32
 			var audioTrack uint32
 
-			for motion := range communication.HandleMotion {
+			for motion := range motionEvents {
 
 				// Get as much packets we need.
 				var cursorError error
@@ -710,7 +717,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 
 				// If startRecording is 0, we will continue as it might be we are in a state of restarting the agent.
 				if startRecording == 0 {
-					log.Log.Info("capture.main.HandleRecordStream(motiondetection): startRecording is 0, we will continue as it might be we are in a state of restarting the agent.")
+					log.Info("capture.main.HandleRecordStream(motiondetection): startRecording is 0, we will continue as it might be we are in a state of restarting the agent.")
 					continue
 				}
 
@@ -748,7 +755,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 				fullName := configDirectory + "/data/recordings/" + name
 
 				// Running...
-				log.Log.Info("capture.main.HandleRecordStream(motiondetection): recording started (" + name + ")" + " at " + strconv.FormatInt(displayTimeSeconds, 10) + " unix")
+				log.Info("capture.main.HandleRecordStream(motiondetection): recording started (" + name + ")" + " at " + strconv.FormatInt(displayTimeSeconds, 10) + " unix")
 
 				// Get width and height from the camera.
 				width := configuration.Config.Capture.IPCamera.Width
@@ -760,7 +767,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 				vpsNALUS := configuration.Config.Capture.IPCamera.VPSNALUs
 
 				if len(spsNALUS) == 0 || len(ppsNALUS) == 0 {
-					log.Log.Warning("capture.main.HandleRecordStream(motiondetection): missing SPS/PPS at recording start")
+					log.Warn("capture.main.HandleRecordStream(motiondetection): missing SPS/PPS at recording start")
 				}
 				// Create the MP4 only once the first keyframe arrives.
 				var mp4Video *video.MP4
@@ -769,16 +776,16 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 
 					nextPkt, cursorError = recordingCursor.ReadPacket()
 					if cursorError != nil {
-						log.Log.Error("capture.main.HandleRecordStream(motiondetection): " + cursorError.Error())
+						log.Error("capture.main.HandleRecordStream(motiondetection): " + cursorError.Error())
 					}
 
 					now = time.Now().UnixMilli()
 					select {
-					case motion := <-communication.HandleMotion:
+					case motion := <-motionEvents:
 						motionTimestamp = now
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): motion detected while recording. Expanding recording.")
+						log.Info("capture.main.HandleRecordStream(motiondetection): motion detected while recording. Expanding recording.")
 						numberOfChanges := motion.NumberOfChanges
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): Received message with recording data, detected changes to save: " + strconv.Itoa(numberOfChanges))
+						log.Info("capture.main.HandleRecordStream(motiondetection): Received message with recording data, detected changes to save: " + strconv.Itoa(numberOfChanges))
 					default:
 					}
 
@@ -794,16 +801,16 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					}
 
 					if start && (motionTimestamp+postRecording-now < 0 || now-startRecording > maxRecordingPeriod-500) && nextPkt.IsKeyFrame {
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): timestamp+postRecording-now < 0  - " + strconv.FormatInt(motionTimestamp+postRecording-now, 10) + " < 0")
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): now-startRecording > maxRecordingPeriod-500 - " + strconv.FormatInt(now-startRecording, 10) + " > " + strconv.FormatInt(maxRecordingPeriod-500, 10))
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): closing recording (timestamp: " + strconv.FormatInt(motionTimestamp, 10) + ", postRecording: " + strconv.FormatInt(postRecording, 10) + ", now: " + strconv.FormatInt(now, 10) + ", startRecording: " + strconv.FormatInt(startRecording, 10) + ", maxRecordingPeriod: " + strconv.FormatInt(maxRecordingPeriod, 10))
+						log.Info("capture.main.HandleRecordStream(motiondetection): timestamp+postRecording-now < 0  - " + strconv.FormatInt(motionTimestamp+postRecording-now, 10) + " < 0")
+						log.Info("capture.main.HandleRecordStream(motiondetection): now-startRecording > maxRecordingPeriod-500 - " + strconv.FormatInt(now-startRecording, 10) + " > " + strconv.FormatInt(maxRecordingPeriod-500, 10))
+						log.Info("capture.main.HandleRecordStream(motiondetection): closing recording (timestamp: " + strconv.FormatInt(motionTimestamp, 10) + ", postRecording: " + strconv.FormatInt(postRecording, 10) + ", now: " + strconv.FormatInt(now, 10) + ", startRecording: " + strconv.FormatInt(startRecording, 10) + ", maxRecordingPeriod: " + strconv.FormatInt(maxRecordingPeriod, 10))
 						break
 					}
 					if pkt.IsKeyFrame && !start && pkt.CurrentTime >= startRecording {
 						// We start the recording if we have a keyframe and the last duration is 0 or less than the current packet time.
 						// It could be start we start from the beginning of the recording.
-						log.Log.Debug("capture.main.HandleRecordStream(motiondetection): write frames")
-						log.Log.Debug("capture.main.HandleRecordStream(motiondetection): recording started on keyframe")
+						log.Debug("capture.main.HandleRecordStream(motiondetection): write frames")
+						log.Debug("capture.main.HandleRecordStream(motiondetection): recording started on keyframe")
 
 						// Align duration timers with the first keyframe.
 						startRecording = pkt.CurrentTime
@@ -821,7 +828,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 						if audioCodec == "AAC" {
 							audioTrack = mp4Video.AddAudioTrack("AAC")
 						} else if audioCodec == "PCM_MULAW" {
-							log.Log.Debug("capture.main.HandleRecordStream(continuous): no AAC audio codec detected, skipping audio track.")
+							log.Debug("capture.main.HandleRecordStream(continuous): no AAC audio codec detected, skipping audio track.")
 						}
 						start = true
 
@@ -840,7 +847,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 				lastRecordingTime = pkt.CurrentTime
 
 				if mp4Video == nil {
-					log.Log.Warning("capture.main.HandleRecordStream(motiondetection): recording closed without keyframe; no MP4 created")
+					log.Warn("capture.main.HandleRecordStream(motiondetection): recording closed without keyframe; no MP4 created")
 					continue
 				}
 
@@ -856,10 +863,10 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 				}
 				if (videoCodec == "H264" && (len(mp4Video.SPSNALUs) == 0 || len(mp4Video.PPSNALUs) == 0)) ||
 					(videoCodec == "H265" && (len(mp4Video.VPSNALUs) == 0 || len(mp4Video.SPSNALUs) == 0 || len(mp4Video.PPSNALUs) == 0)) {
-					log.Log.Warning("capture.main.HandleRecordStream(motiondetection): closing MP4 without full parameter sets, moov may be incomplete")
+					log.Warn("capture.main.HandleRecordStream(motiondetection): closing MP4 without full parameter sets, moov may be incomplete")
 				}
 				mp4Video.Close(&config)
-				log.Log.Info("capture.main.HandleRecordStream(motiondetection): file save: " + name)
+				log.Info("capture.main.HandleRecordStream(motiondetection): file save: " + name)
 
 				// Notify the hub / live-view UI that this camera stopped recording.
 				publishRecordingState(mqttClient, hubKey, configuration, false)
@@ -870,10 +877,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 				// restart when the recording has expired (heartbeat lapsed or max
 				// duration reached), so it ends here instead of recording forever.
 				if communication.IsRecordingManual.IsSet() && !manualRecordingExpired(communication, time.Now().UnixMilli()) {
-					select {
-					case communication.HandleMotion <- models.MotionDataPartial{Timestamp: time.Now().Unix(), NumberOfChanges: 100000000}:
-					default:
-					}
+					communication.TrySendMotion(models.MotionDataPartial{Timestamp: time.Now().Unix(), NumberOfChanges: 100000000})
 				}
 
 				// Update the name of the recording with the duration.
@@ -894,7 +898,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					oldName := name
 					name = s + ".mp4"
 					fullName = configDirectory + "/data/recordings/" + name
-					log.Log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
+					log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
 
 					// Rename the file to the new name.
 					err := os.Rename(
@@ -902,12 +906,13 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 						configDirectory+"/data/recordings/"+s+".mp4")
 
 					if err != nil {
-						log.Log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
+						log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
 					}
 				} else {
-					log.Log.Info("capture.main.HandleRecordStream(motiondetection): no video data recorded, not renaming file.")
+					log.Info("capture.main.HandleRecordStream(motiondetection): no video data recorded, not renaming file.")
 				}
 
+				encrypted := false
 				// Check if we need to encrypt the recording.
 				if config.Encryption != nil && config.Encryption.Enabled == "true" && config.Encryption.Recordings == "true" && config.Encryption.SymmetricKey != "" {
 					// reopen file into memory 'fullName'
@@ -919,24 +924,26 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 							// write back to file
 							err := os.WriteFile(fullName, []byte(encryptedContents), 0644)
 							if err != nil {
-								log.Log.Error("capture.main.HandleRecordStream(motiondetection): error writing file: " + err.Error())
+								log.Error("capture.main.HandleRecordStream(motiondetection): error writing file: " + err.Error())
+							} else {
+								encrypted = true
 							}
 						} else {
-							log.Log.Error("capture.main.HandleRecordStream(motiondetection): error encrypting file: " + err.Error())
+							log.Error("capture.main.HandleRecordStream(motiondetection): error encrypting file: " + err.Error())
 						}
 					} else {
-						log.Log.Error("capture.main.HandleRecordStream(motiondetection): error reading file: " + err.Error())
+						log.Error("capture.main.HandleRecordStream(motiondetection): error reading file: " + err.Error())
 					}
 				}
 
-				queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, displayTime, mp4Video))
+				queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, displayTime, mp4Video, encrypted))
 
 				// Clean up the recording directory if necessary.
 				CleanupRecordingDirectory(configDirectory, configuration)
 			}
 		}
 
-		log.Log.Debug("capture.main.HandleRecordStream(): finished")
+		log.Debug("capture.main.HandleRecordStream(): finished")
 	}
 }
 
@@ -1030,13 +1037,14 @@ func Base64Image(captureDevice *Capture, communication *models.Communication, co
 	var cursor *packets.QueueCursor
 
 	// We'll pick the right client and decoder.
-	rtspClient := captureDevice.RTSPSubClient
+	rtspClient := captureDevice.SubClient()
 	if rtspClient != nil {
-		queue = communication.SubQueue
-		cursor = queue.Latest()
+		queue = communication.SubQueue()
 	} else {
-		rtspClient = captureDevice.RTSPClient
-		queue = communication.Queue
+		rtspClient = captureDevice.MainClient()
+		queue = communication.MainQueue()
+	}
+	if queue != nil {
 		cursor = queue.Latest()
 	}
 
@@ -1062,6 +1070,10 @@ func Base64Image(captureDevice *Capture, communication *models.Communication, co
 					count++
 					continue
 				}
+			} else {
+				// A stopped run closes this queue permanently. Retrying its
+				// cursor would spin on EOF even after the HTTP client leaves.
+				break
 			}
 		} else {
 			break
@@ -1076,13 +1088,14 @@ func JpegImage(captureDevice *Capture, communication *models.Communication) imag
 	var cursor *packets.QueueCursor
 
 	// We'll pick the right client and decoder.
-	rtspClient := captureDevice.RTSPSubClient
+	rtspClient := captureDevice.SubClient()
 	if rtspClient != nil {
-		queue = communication.SubQueue
-		cursor = queue.Latest()
+		queue = communication.SubQueue()
 	} else {
-		rtspClient = captureDevice.RTSPClient
-		queue = communication.Queue
+		rtspClient = captureDevice.MainClient()
+		queue = communication.MainQueue()
+	}
+	if queue != nil {
 		cursor = queue.Latest()
 	}
 
@@ -1104,6 +1117,10 @@ func JpegImage(captureDevice *Capture, communication *models.Communication) imag
 				} else {
 					break
 				}
+			} else {
+				// A stopped run closes this queue permanently. Retrying its
+				// cursor would spin on EOF even after the HTTP client leaves.
+				break
 			}
 		} else {
 			break
@@ -1137,20 +1154,16 @@ func writeSampleToMP4(mp4Video *video.MP4, videoTrack, audioTrack uint32, pkt pa
 			dts = pts - uint64(compositionOffset)
 		}
 		if err := mp4Video.AddSampleToTrack(videoTrack, pkt.IsKeyFrame, pkt.Data, dts, compositionOffset); err != nil {
-			log.Log.Error("capture.main.writeSampleToMP4(): " + err.Error())
+			log.Error("capture.main.writeSampleToMP4(): " + err.Error())
 		}
 	} else if pkt.IsAudio {
 		if pkt.Codec == "AAC" {
 			if err := mp4Video.AddSampleToTrack(audioTrack, pkt.IsKeyFrame, pkt.Data, pts, 0); err != nil {
-				log.Log.Error("capture.main.writeSampleToMP4(): " + err.Error())
+				log.Error("capture.main.writeSampleToMP4(): " + err.Error())
 			}
 		} else if pkt.Codec == "PCM_MULAW" {
 			// TODO: transcode to AAC, some work to do..
-			log.Log.Debug("capture.main.writeSampleToMP4(): no AAC audio codec detected, skipping audio track.")
+			log.Debug("capture.main.writeSampleToMP4(): no AAC audio codec detected, skipping audio track.")
 		}
 	}
 }
-
-/*func convertPTS2(v int64) uint64 {
-	return uint64(v) / 100
-}*/

machinery/src/capture/main_test.go

@@ -42,7 +42,7 @@ func TestQueueRecordingForUploadStoresFinalizedMetadata(t *testing.T) {
 	}
 
 	mp4Video := &video.MP4{VideoTotalDuration: 20452, SampleCount: 613}
-	metadata := recordingUploadMetadata("recording.mp4", "device-key", 1785934709414, mp4Video)
+	metadata := recordingUploadMetadata("recording.mp4", "device-key", 1785934709414, mp4Video, true)
 	queueRecordingForUpload(configDirectory, metadata)
 
 	got, err := os.ReadFile(filepath.Join(configDirectory, "data", "cloud", "recording.metadata"))
@@ -54,7 +54,7 @@ func TestQueueRecordingForUploadStoresFinalizedMetadata(t *testing.T) {
 		t.Fatalf("decode upload marker: %v", err)
 	}
 	expectedFPS := mp4Video.AverageFPS()
-	if stored.FileName != "recording.mp4" || stored.DeviceKey != "device-key" || stored.Timestamp != 1785934709414 || stored.Duration != 20452 || math.Abs(stored.FPS-expectedFPS) > 1e-9 {
+	if stored.FileName != "recording.mp4" || stored.DeviceKey != "device-key" || stored.Timestamp != 1785934709414 || stored.Duration != 20452 || math.Abs(stored.FPS-expectedFPS) > 1e-9 || !stored.Encrypted {
 		t.Fatalf("upload marker = %+v", stored)
 	}
 	if stored.FPS == math.Floor(stored.FPS) {

machinery/src/capture/rtsp_client.go

@@ -3,36 +3,73 @@ package capture
 import (
 	"context"
 	"image"
+	"sync"
 
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 )
 
 type Capture struct {
-	RTSPClient            *Golibrtsp
-	RTSPSubClient         *Golibrtsp
-	RTSPBackChannelClient *Golibrtsp
+	clientsMu             sync.RWMutex
+	rtspClient            *Golibrtsp
+	rtspSubClient         *Golibrtsp
+	rtspBackChannelClient *Golibrtsp
 }
 
 func (c *Capture) SetMainClient(rtspUrl string) *Golibrtsp {
-	c.RTSPClient = &Golibrtsp{
+	client := &Golibrtsp{
 		Url: rtspUrl,
 	}
-	return c.RTSPClient
+	c.clientsMu.Lock()
+	c.rtspClient = client
+	c.clientsMu.Unlock()
+	return client
 }
 
 func (c *Capture) SetSubClient(rtspUrl string) *Golibrtsp {
-	c.RTSPSubClient = &Golibrtsp{
+	client := &Golibrtsp{
 		Url: rtspUrl,
 	}
-	return c.RTSPSubClient
+	c.clientsMu.Lock()
+	c.rtspSubClient = client
+	c.clientsMu.Unlock()
+	return client
 }
 
 func (c *Capture) SetBackChannelClient(rtspUrl string) *Golibrtsp {
-	c.RTSPBackChannelClient = &Golibrtsp{
+	client := &Golibrtsp{
 		Url: rtspUrl,
 	}
-	return c.RTSPBackChannelClient
+	c.clientsMu.Lock()
+	c.rtspBackChannelClient = client
+	c.clientsMu.Unlock()
+	return client
+}
+
+func (c *Capture) MainClient() *Golibrtsp {
+	c.clientsMu.RLock()
+	defer c.clientsMu.RUnlock()
+	return c.rtspClient
+}
+
+func (c *Capture) SubClient() *Golibrtsp {
+	c.clientsMu.RLock()
+	defer c.clientsMu.RUnlock()
+	return c.rtspSubClient
+}
+
+func (c *Capture) ClearClients(main, sub, backchannel *Golibrtsp) {
+	c.clientsMu.Lock()
+	defer c.clientsMu.Unlock()
+	if c.rtspClient == main {
+		c.rtspClient = nil
+	}
+	if c.rtspSubClient == sub {
+		c.rtspSubClient = nil
+	}
+	if c.rtspBackChannelClient == backchannel {
+		c.rtspBackChannelClient = nil
+	}
 }
 
 // RTSPClient is a interface that abstracts the RTSP client implementation.

machinery/src/capture/rtsp_client_test.go

@@ -0,0 +1,62 @@
+package capture
+
+import (
+	"strconv"
+	"sync"
+	"testing"
+)
+
+func TestCaptureClientAccessCanRaceReplacement(t *testing.T) {
+	captureDevice := &Capture{}
+	captureDevice.SetMainClient("rtsp://main/0")
+	captureDevice.SetSubClient("rtsp://sub/0")
+
+	var workers sync.WaitGroup
+	workers.Add(2)
+	go func() {
+		defer workers.Done()
+		for replacement := 1; replacement <= 1000; replacement++ {
+			suffix := strconv.Itoa(replacement)
+			captureDevice.SetMainClient("rtsp://main/" + suffix)
+			captureDevice.SetSubClient("rtsp://sub/" + suffix)
+		}
+	}()
+	go func() {
+		defer workers.Done()
+		for snapshot := 0; snapshot < 1000; snapshot++ {
+			if captureDevice.MainClient() == nil {
+				t.Error("MainClient() returned nil")
+				return
+			}
+			if captureDevice.SubClient() == nil {
+				t.Error("SubClient() returned nil")
+				return
+			}
+		}
+	}()
+	workers.Wait()
+}
+
+func TestCaptureClearClientsOnlyClearsMatchingRun(t *testing.T) {
+	captureDevice := &Capture{}
+	oldMain := captureDevice.SetMainClient("rtsp://main/old")
+	oldSub := captureDevice.SetSubClient("rtsp://sub/old")
+	oldBackchannel := captureDevice.SetBackChannelClient("rtsp://back/old")
+
+	newMain := captureDevice.SetMainClient("rtsp://main/new")
+	newSub := captureDevice.SetSubClient("rtsp://sub/new")
+	newBackchannel := captureDevice.SetBackChannelClient("rtsp://back/new")
+
+	captureDevice.ClearClients(oldMain, oldSub, oldBackchannel)
+	if captureDevice.MainClient() != newMain {
+		t.Fatal("stale cleanup cleared the new main client")
+	}
+	if captureDevice.SubClient() != newSub {
+		t.Fatal("stale cleanup cleared the new sub client")
+	}
+
+	captureDevice.ClearClients(newMain, newSub, newBackchannel)
+	if captureDevice.MainClient() != nil || captureDevice.SubClient() != nil {
+		t.Fatal("matching cleanup did not clear current clients")
+	}
+}

machinery/src/capture/snapshot_test.go

@@ -0,0 +1,68 @@
+package capture
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/packets"
+)
+
+func TestSnapshotReturnsAfterQueueClosure(t *testing.T) {
+	for _, stream := range []string{"main", "sub"} {
+		for _, format := range []string{"jpeg", "base64"} {
+			for _, alreadyClosed := range []bool{true, false} {
+				name := stream + "/" + format + "/closed_during_read"
+				if alreadyClosed {
+					name = stream + "/" + format + "/already_closed"
+				}
+				t.Run(name, func(t *testing.T) {
+					communication := &models.Communication{}
+					run := models.NewAgentRun(context.Background(), communication, false)
+					queue := packets.NewQueue()
+					captureDevice := &Capture{}
+					if stream == "main" {
+						captureDevice.SetMainClient("rtsp://unused.invalid/main")
+						run.SetMainQueue(queue)
+					} else {
+						captureDevice.SetSubClient("rtsp://unused.invalid/sub")
+						run.SetSubQueue(queue)
+					}
+					if err := run.Activate(); err != nil {
+						t.Fatal(err)
+					}
+					defer queue.Close()
+					if alreadyClosed {
+						queue.Close()
+					}
+					done := make(chan bool, 1)
+					go func() {
+						if format == "jpeg" {
+							image := JpegImage(captureDevice, communication)
+							done <- image.Rect.Empty()
+						} else {
+							done <- Base64Image(captureDevice, communication, &models.Configuration{}) == ""
+						}
+					}()
+					if !alreadyClosed {
+						select {
+						case <-done:
+							t.Fatal("snapshot returned before a frame or queue closure")
+						case <-time.After(10 * time.Millisecond):
+						}
+						queue.Close()
+					}
+					select {
+					case empty := <-done:
+						if !empty {
+							t.Fatal("closed empty queue produced a snapshot")
+						}
+					case <-time.After(time.Second):
+						t.Fatal("snapshot did not return after its packet queue closed")
+					}
+				})
+			}
+		}
+	}
+}

machinery/src/cloud/cloud_test.go

@@ -1,14 +1,23 @@
 package cloud
 
 import (
+	"encoding/json"
+	"errors"
+	"fmt"
 	"io"
 	"net/http"
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/kerberos-io/agent/machinery/src/models"
+	agentonvif "github.com/kerberos-io/agent/machinery/src/onvif"
+	goonvif "github.com/kerberos-io/onvif"
+	goonvifdevice "github.com/kerberos-io/onvif/device"
+	goonvifptz "github.com/kerberos-io/onvif/ptz"
 )
 
-func TestHeartbeatFailureLogIncludesHubResponse(t *testing.T) {
+func TestHeartbeatFailureLogOmitsHubResponseBody(t *testing.T) {
 	response := &http.Response{
 		StatusCode: http.StatusBadRequest,
 		Status:     "400 Bad Request",
@@ -19,16 +28,24 @@ func TestHeartbeatFailureLogIncludesHubResponse(t *testing.T) {
 	if err != nil {
 		t.Fatalf("readHeartbeatResponseBody() error = %v", err)
 	}
-	message := formatHeartbeatFailureLog(response, nil, responseBody, truncated, nil, 125*time.Millisecond)
+	fields := heartbeatFailureLogFields(response, responseBody, truncated, 125*time.Millisecond)
 
-	for _, expected := range []string{
-		"status_code=400",
-		`status="400 Bad Request"`,
-		"duration=125ms",
-		`response_body="{\"error\":\"invalid heartbeat\"}"`,
+	for key, want := range map[string]interface{}{
+		"duration_ms":             int64(125),
+		"response_body_bytes":     len(responseBody),
+		"response_body_truncated": false,
+		"status_code":             http.StatusBadRequest,
 	} {
-		if !strings.Contains(message, expected) {
-			t.Errorf("formatHeartbeatFailureLog() = %q, want it to contain %q", message, expected)
+		if got := fields[key]; got != want {
+			t.Errorf("%s = %v, want %v", key, got, want)
+		}
+	}
+	for key, value := range fields {
+		if strings.Contains(key, "response_body") && key != "response_body_bytes" && key != "response_body_truncated" {
+			t.Fatalf("unexpected response body field %q=%v", key, value)
+		}
+		if strings.Contains(fmt.Sprint(value), "invalid heartbeat") {
+			t.Fatalf("heartbeat log fields exposed response body in %q=%v", key, value)
 		}
 	}
 }
@@ -49,3 +66,527 @@ func TestReadHeartbeatResponseBodyTruncatesLargeBody(t *testing.T) {
 		t.Fatalf("len(body) = %d, want %d", len(body), heartbeatResponseBodyLogLimit)
 	}
 }
+
+func TestHeartbeatONVIFPayloadCachesStaticAndReusesLoopSubscription(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	device := newTestONVIFDevice()
+	camera := models.IPCamera{
+		ONVIFXAddr:    "http://camera/onvif",
+		ONVIFUsername: "operator",
+		ONVIFPassword: "secret",
+	}
+	initialEvents := []agentonvif.ONVIFEvents{{Key: "input-1", Type: "input", Value: "true", Timestamp: 1}}
+	loopEvents := []agentonvif.ONVIFEvents{{Key: "output-1", Type: "output", Value: "false", Timestamp: 2}}
+	wantPresets := mustJSONMarshal(t, []models.OnvifActionPreset{{Name: "Lobby", Token: "1"}})
+	wantInitialEvents := mustJSONMarshal(t, initialEvents)
+	wantLoopEvents := mustJSONMarshal(t, loopEvents)
+
+	var connectCalls, ptzConfigCalls, ptzFunctionCalls, presetCalls, createCalls, eventCalls, unsubscribeCalls int
+
+	heartbeatConnectToONVIFDevice = func(*models.IPCamera) (*goonvif.Device, goonvifdevice.GetCapabilitiesResponse, error) {
+		connectCalls++
+		return device, goonvifdevice.GetCapabilitiesResponse{}, nil
+	}
+	heartbeatGetPTZConfigurationsFromDevice = func(*goonvif.Device) (goonvifptz.GetConfigurationsResponse, error) {
+		ptzConfigCalls++
+		return goonvifptz.GetConfigurationsResponse{}, nil
+	}
+	heartbeatGetPTZFunctionsFromDevice = func(goonvifptz.GetConfigurationsResponse) ([]string, bool, bool) {
+		ptzFunctionCalls++
+		return nil, true, true
+	}
+	heartbeatGetPresetsFromDevice = func(*goonvif.Device) ([]models.OnvifActionPreset, error) {
+		presetCalls++
+		return []models.OnvifActionPreset{{Name: "Lobby", Token: "1"}}, nil
+	}
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		createCalls++
+		switch createCalls {
+		case 1:
+			return "initial-1", nil
+		case 2:
+			return "loop", nil
+		case 3:
+			return "initial-2", nil
+		default:
+			t.Fatalf("unexpected create pull point call %d", createCalls)
+			return "", nil
+		}
+	}
+	heartbeatGetEventMessages = func(_ *goonvif.Device, pullPointAddress string) ([]agentonvif.ONVIFEvents, error) {
+		eventCalls++
+		switch pullPointAddress {
+		case "initial-1", "initial-2":
+			return initialEvents, nil
+		case "loop":
+			return loopEvents, nil
+		default:
+			t.Fatalf("unexpected pull point address %q", pullPointAddress)
+			return nil, nil
+		}
+	}
+	heartbeatUnsubscribePullPoint = func(_ *goonvif.Device, pullPointAddress string) error {
+		unsubscribeCalls++
+		if pullPointAddress != "initial-1" && pullPointAddress != "initial-2" {
+			t.Fatalf("unexpected unsubscribe pull point %q", pullPointAddress)
+		}
+		return nil
+	}
+
+	state := newHeartbeatONVIFState()
+
+	payload := getHeartbeatONVIFPayload(camera, state)
+	if payload.enabled != "true" || payload.zoom != "true" || payload.panTilt != "true" || payload.presets != "true" {
+		t.Fatalf("unexpected static payload: %+v", payload)
+	}
+	if string(payload.presetsList) != string(wantPresets) {
+		t.Fatalf("payload.presetsList = %s, want %s", payload.presetsList, wantPresets)
+	}
+	if string(payload.eventsList) != string(wantInitialEvents) {
+		t.Fatalf("payload.eventsList = %s, want %s", payload.eventsList, wantInitialEvents)
+	}
+	if connectCalls != 1 || ptzConfigCalls != 1 || ptzFunctionCalls != 1 || presetCalls != 1 {
+		t.Fatalf("unexpected static call counts after first cycle: connect=%d ptzConfig=%d ptzFunctions=%d presets=%d", connectCalls, ptzConfigCalls, ptzFunctionCalls, presetCalls)
+	}
+	if createCalls != 2 || eventCalls != 1 || unsubscribeCalls != 1 {
+		t.Fatalf("unexpected event call counts after first cycle: create=%d events=%d unsubscribe=%d", createCalls, eventCalls, unsubscribeCalls)
+	}
+
+	payload = getHeartbeatONVIFPayload(camera, state)
+	if string(payload.eventsList) != string(wantLoopEvents) {
+		t.Fatalf("second payload.eventsList = %s, want %s", payload.eventsList, wantLoopEvents)
+	}
+	if connectCalls != 1 {
+		t.Fatalf("connectCalls = %d, want 1", connectCalls)
+	}
+	if ptzConfigCalls != 1 || ptzFunctionCalls != 1 || presetCalls != 1 {
+		t.Fatalf("static calls were not cached: ptzConfig=%d ptzFunctions=%d presets=%d", ptzConfigCalls, ptzFunctionCalls, presetCalls)
+	}
+	if createCalls != 3 || eventCalls != 3 || unsubscribeCalls != 2 {
+		t.Fatalf("temporary state subscription was not refreshed or loop subscription was not reused: create=%d events=%d unsubscribe=%d", createCalls, eventCalls, unsubscribeCalls)
+	}
+}
+
+func TestHeartbeatONVIFPayloadRefreshesAfterCameraConfigChange(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	deviceA := newTestONVIFDevice()
+	deviceB := newTestONVIFDevice()
+	cameraA := models.IPCamera{ONVIFXAddr: "http://camera-a/onvif", ONVIFUsername: "user", ONVIFPassword: "secret-a"}
+	cameraB := models.IPCamera{ONVIFXAddr: "http://camera-b/onvif", ONVIFUsername: "user", ONVIFPassword: "secret-b"}
+	var connectCalls, ptzConfigCalls, presetCalls, createCalls int
+	var unsubscribed []string
+
+	heartbeatConnectToONVIFDevice = func(camera *models.IPCamera) (*goonvif.Device, goonvifdevice.GetCapabilitiesResponse, error) {
+		connectCalls++
+		switch camera.ONVIFXAddr {
+		case cameraA.ONVIFXAddr:
+			return deviceA, goonvifdevice.GetCapabilitiesResponse{}, nil
+		case cameraB.ONVIFXAddr:
+			return deviceB, goonvifdevice.GetCapabilitiesResponse{}, nil
+		default:
+			t.Fatalf("unexpected camera address %q", camera.ONVIFXAddr)
+			return nil, goonvifdevice.GetCapabilitiesResponse{}, nil
+		}
+	}
+	heartbeatGetPTZConfigurationsFromDevice = func(*goonvif.Device) (goonvifptz.GetConfigurationsResponse, error) {
+		ptzConfigCalls++
+		return goonvifptz.GetConfigurationsResponse{}, nil
+	}
+	heartbeatGetPTZFunctionsFromDevice = func(goonvifptz.GetConfigurationsResponse) ([]string, bool, bool) {
+		return nil, false, true
+	}
+	heartbeatGetPresetsFromDevice = func(*goonvif.Device) ([]models.OnvifActionPreset, error) {
+		presetCalls++
+		return nil, nil
+	}
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		createCalls++
+		switch createCalls {
+		case 1:
+			return "initial-a", nil
+		case 2:
+			return "loop-a", nil
+		case 3:
+			return "initial-b", nil
+		case 4:
+			return "loop-b", nil
+		default:
+			t.Fatalf("unexpected create pull point call %d", createCalls)
+			return "", nil
+		}
+	}
+	heartbeatGetEventMessages = func(_ *goonvif.Device, pullPointAddress string) ([]agentonvif.ONVIFEvents, error) {
+		switch pullPointAddress {
+		case "initial-a":
+			return []agentonvif.ONVIFEvents{{Key: "a", Type: "input", Value: "true", Timestamp: 1}}, nil
+		case "initial-b":
+			return []agentonvif.ONVIFEvents{{Key: "b", Type: "input", Value: "false", Timestamp: 2}}, nil
+		default:
+			t.Fatalf("unexpected pull point address %q", pullPointAddress)
+			return nil, nil
+		}
+	}
+	heartbeatUnsubscribePullPoint = func(_ *goonvif.Device, pullPointAddress string) error {
+		unsubscribed = append(unsubscribed, pullPointAddress)
+		return nil
+	}
+
+	state := newHeartbeatONVIFState()
+	_ = getHeartbeatONVIFPayload(cameraA, state)
+	payload := getHeartbeatONVIFPayload(cameraB, state)
+
+	if connectCalls != 2 {
+		t.Fatalf("connectCalls = %d, want 2", connectCalls)
+	}
+	if ptzConfigCalls != 2 || presetCalls != 2 {
+		t.Fatalf("camera config change did not refresh static state: ptzConfig=%d presets=%d", ptzConfigCalls, presetCalls)
+	}
+	if createCalls != 4 {
+		t.Fatalf("camera config change did not recreate subscriptions: create=%d", createCalls)
+	}
+	if countString(unsubscribed, "loop-a") != 1 {
+		t.Fatalf("unsubscribed loop-a %d times, want 1; unsubscribed=%v", countString(unsubscribed, "loop-a"), unsubscribed)
+	}
+	wantEvents := mustJSONMarshal(t, []agentonvif.ONVIFEvents{{Key: "b", Type: "input", Value: "false", Timestamp: 2}})
+	if string(payload.eventsList) != string(wantEvents) {
+		t.Fatalf("payload.eventsList = %s, want %s", payload.eventsList, wantEvents)
+	}
+}
+
+func TestHeartbeatONVIFPayloadRetriesStaticFetchFailuresWithoutReconnect(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	device := newTestONVIFDevice()
+	camera := models.IPCamera{
+		ONVIFXAddr:    "http://camera/onvif",
+		ONVIFUsername: "operator",
+		ONVIFPassword: "secret",
+	}
+	var connectCalls, ptzConfigCalls, presetCalls, createCalls, eventCalls int
+
+	heartbeatConnectToONVIFDevice = func(*models.IPCamera) (*goonvif.Device, goonvifdevice.GetCapabilitiesResponse, error) {
+		connectCalls++
+		return device, goonvifdevice.GetCapabilitiesResponse{}, nil
+	}
+	heartbeatGetPTZConfigurationsFromDevice = func(*goonvif.Device) (goonvifptz.GetConfigurationsResponse, error) {
+		ptzConfigCalls++
+		return goonvifptz.GetConfigurationsResponse{}, nil
+	}
+	heartbeatGetPTZFunctionsFromDevice = func(goonvifptz.GetConfigurationsResponse) ([]string, bool, bool) {
+		return nil, true, true
+	}
+	heartbeatGetPresetsFromDevice = func(*goonvif.Device) ([]models.OnvifActionPreset, error) {
+		presetCalls++
+		if presetCalls == 1 {
+			return nil, errors.New("temporary preset failure")
+		}
+		return []models.OnvifActionPreset{{Name: "Lobby", Token: "1"}}, nil
+	}
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		createCalls++
+		switch createCalls {
+		case 1:
+			return "initial-1", nil
+		case 2:
+			return "loop", nil
+		case 3:
+			return "initial-2", nil
+		default:
+			t.Fatalf("unexpected create pull point call %d", createCalls)
+			return "", nil
+		}
+	}
+	heartbeatGetEventMessages = func(_ *goonvif.Device, pullPointAddress string) ([]agentonvif.ONVIFEvents, error) {
+		eventCalls++
+		switch pullPointAddress {
+		case "initial-1", "initial-2":
+			return []agentonvif.ONVIFEvents{{Key: "one", Type: "input", Value: "true", Timestamp: 1}}, nil
+		case "loop":
+			return []agentonvif.ONVIFEvents{{Key: "two", Type: "output", Value: "false", Timestamp: 2}}, nil
+		default:
+			t.Fatalf("unexpected pull point address %q", pullPointAddress)
+			return nil, nil
+		}
+	}
+	heartbeatUnsubscribePullPoint = func(*goonvif.Device, string) error { return nil }
+
+	state := newHeartbeatONVIFState()
+	payload := getHeartbeatONVIFPayload(camera, state)
+	if payload.presets != "false" {
+		t.Fatalf("payload.presets = %q, want false on transient preset failure", payload.presets)
+	}
+	payload = getHeartbeatONVIFPayload(camera, state)
+	if payload.presets != "true" {
+		t.Fatalf("payload.presets = %q, want true after retry", payload.presets)
+	}
+	if connectCalls != 1 {
+		t.Fatalf("connectCalls = %d, want 1", connectCalls)
+	}
+	if ptzConfigCalls != 2 || presetCalls != 2 {
+		t.Fatalf("static failures were not retried on heartbeat cadence: ptzConfig=%d presets=%d", ptzConfigCalls, presetCalls)
+	}
+	if createCalls != 3 || eventCalls != 3 {
+		t.Fatalf("unexpected event behavior during retry: create=%d events=%d", createCalls, eventCalls)
+	}
+}
+
+func TestHeartbeatONVIFStateReleasesSubscriptionWhenDisabled(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	device := newTestONVIFDevice()
+	state := newHeartbeatONVIFState()
+	state.cameraConfiguration = models.IPCamera{ONVIFXAddr: "http://camera/onvif"}
+	state.cameraKey = heartbeatONVIFCameraKey(state.cameraConfiguration)
+	state.device = device
+	state.loopPullPoint = "loop"
+
+	var unsubscribed []string
+	heartbeatUnsubscribePullPoint = func(gotDevice *goonvif.Device, pullPointAddress string) error {
+		if gotDevice != device {
+			t.Fatal("unsubscribe used a different ONVIF device")
+		}
+		unsubscribed = append(unsubscribed, pullPointAddress)
+		return nil
+	}
+
+	state.prepare(models.IPCamera{})
+
+	if len(unsubscribed) != 1 || unsubscribed[0] != "loop" {
+		t.Fatalf("unsubscribed = %v, want [loop]", unsubscribed)
+	}
+	if state.device != nil || state.loopPullPoint != "" {
+		t.Fatalf("disabled state retained device or pull point: %+v", state)
+	}
+}
+
+func TestHeartbeatONVIFPayloadDoesNotCreateSubscriptionsWhenConnectFails(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	var createCalls int
+
+	heartbeatConnectToONVIFDevice = func(*models.IPCamera) (*goonvif.Device, goonvifdevice.GetCapabilitiesResponse, error) {
+		return nil, goonvifdevice.GetCapabilitiesResponse{}, errors.New("connect failed")
+	}
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		createCalls++
+		return "unexpected", nil
+	}
+
+	payload := getHeartbeatONVIFPayload(models.IPCamera{
+		ONVIFXAddr:    "http://camera/onvif",
+		ONVIFUsername: "operator",
+		ONVIFPassword: "secret",
+	}, newHeartbeatONVIFState())
+
+	if createCalls != 0 {
+		t.Fatalf("createCalls = %d, want 0", createCalls)
+	}
+	assertDefaultHeartbeatONVIFPayload(t, payload)
+}
+
+func TestHeartbeatONVIFPayloadReconnectFailureInvalidatesCache(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	device1 := newTestONVIFDevice()
+	device2 := newTestONVIFDevice()
+	camera := models.IPCamera{
+		ONVIFXAddr:    "http://camera/onvif",
+		ONVIFUsername: "operator",
+		ONVIFPassword: "secret",
+	}
+	var connectCalls, ptzConfigCalls, presetCalls, createCalls int
+	var unsubscribed []string
+
+	heartbeatConnectToONVIFDevice = func(*models.IPCamera) (*goonvif.Device, goonvifdevice.GetCapabilitiesResponse, error) {
+		connectCalls++
+		switch connectCalls {
+		case 1:
+			return device1, goonvifdevice.GetCapabilitiesResponse{}, nil
+		case 2:
+			return device2, goonvifdevice.GetCapabilitiesResponse{}, nil
+		default:
+			t.Fatalf("unexpected connect call %d", connectCalls)
+			return nil, goonvifdevice.GetCapabilitiesResponse{}, nil
+		}
+	}
+	heartbeatGetPTZConfigurationsFromDevice = func(*goonvif.Device) (goonvifptz.GetConfigurationsResponse, error) {
+		ptzConfigCalls++
+		return goonvifptz.GetConfigurationsResponse{}, nil
+	}
+	heartbeatGetPTZFunctionsFromDevice = func(goonvifptz.GetConfigurationsResponse) ([]string, bool, bool) {
+		return nil, true, false
+	}
+	heartbeatGetPresetsFromDevice = func(*goonvif.Device) ([]models.OnvifActionPreset, error) {
+		presetCalls++
+		return []models.OnvifActionPreset{{Name: "Preset", Token: "1"}}, nil
+	}
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		createCalls++
+		switch createCalls {
+		case 1:
+			return "initial-1", nil
+		case 2:
+			return "loop-1", nil
+		case 3:
+			return "initial-2", nil
+		case 4:
+			return "initial-3", nil
+		case 5:
+			return "loop-3", nil
+		default:
+			t.Fatalf("unexpected create pull point call %d", createCalls)
+			return "", nil
+		}
+	}
+	heartbeatGetEventMessages = func(_ *goonvif.Device, pullPointAddress string) ([]agentonvif.ONVIFEvents, error) {
+		switch pullPointAddress {
+		case "initial-1":
+			return []agentonvif.ONVIFEvents{{Key: "before", Type: "input", Value: "true", Timestamp: 1}}, nil
+		case "initial-2":
+			return []agentonvif.ONVIFEvents{{Key: "during", Type: "input", Value: "true", Timestamp: 2}}, nil
+		case "loop-1":
+			return nil, errors.New("pull failed")
+		case "initial-3":
+			return []agentonvif.ONVIFEvents{{Key: "after", Type: "input", Value: "false", Timestamp: 2}}, nil
+		default:
+			t.Fatalf("unexpected pull point address %q", pullPointAddress)
+			return nil, nil
+		}
+	}
+	heartbeatUnsubscribePullPoint = func(_ *goonvif.Device, pullPointAddress string) error {
+		unsubscribed = append(unsubscribed, pullPointAddress)
+		return nil
+	}
+
+	state := newHeartbeatONVIFState()
+	_ = getHeartbeatONVIFPayload(camera, state)
+	payload := getHeartbeatONVIFPayload(camera, state)
+	wantEventsDuringFailure := mustJSONMarshal(t, []agentonvif.ONVIFEvents{{Key: "during", Type: "input", Value: "true", Timestamp: 2}})
+	if string(payload.eventsList) != string(wantEventsDuringFailure) {
+		t.Fatalf("payload.eventsList after operation failure = %s, want %s", payload.eventsList, wantEventsDuringFailure)
+	}
+	payload = getHeartbeatONVIFPayload(camera, state)
+
+	if ptzConfigCalls != 2 || presetCalls != 2 {
+		t.Fatalf("reconnect did not refresh static state: ptzConfig=%d presets=%d", ptzConfigCalls, presetCalls)
+	}
+	if connectCalls != 2 {
+		t.Fatalf("connectCalls = %d, want 2 after reconnect", connectCalls)
+	}
+	if createCalls != 5 {
+		t.Fatalf("reconnect did not recreate subscriptions: create=%d", createCalls)
+	}
+	if countString(unsubscribed, "loop-1") != 1 {
+		t.Fatalf("operation failure cleanup mismatch, unsubscribed=%v", unsubscribed)
+	}
+	wantEvents := mustJSONMarshal(t, []agentonvif.ONVIFEvents{{Key: "after", Type: "input", Value: "false", Timestamp: 2}})
+	if string(payload.eventsList) != string(wantEvents) {
+		t.Fatalf("payload.eventsList = %s, want %s", payload.eventsList, wantEvents)
+	}
+}
+
+func TestHeartbeatONVIFPayloadKeepsCachedConnectionWhenInitialStateFetchFails(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	device := newTestONVIFDevice()
+	camera := models.IPCamera{
+		ONVIFXAddr:    "http://camera/onvif",
+		ONVIFUsername: "operator",
+		ONVIFPassword: "secret",
+	}
+	loopEvents := []agentonvif.ONVIFEvents{{Key: "input-1", Type: "input", Value: "true", Timestamp: 1}}
+
+	state := newHeartbeatONVIFState()
+	state.cameraConfiguration = camera
+	state.cameraKey = heartbeatONVIFCameraKey(camera)
+	state.device = device
+	state.loopPullPoint = "loop"
+	state.staticLoaded = true
+	state.staticPayload.enabled = "true"
+
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		return "", errors.New("temporary initial-state failure")
+	}
+	heartbeatGetEventMessages = func(_ *goonvif.Device, pullPointAddress string) ([]agentonvif.ONVIFEvents, error) {
+		if pullPointAddress != "loop" {
+			t.Fatalf("unexpected pull point address %q", pullPointAddress)
+		}
+		return loopEvents, nil
+	}
+
+	payload := getHeartbeatONVIFPayload(camera, state)
+
+	wantEvents := mustJSONMarshal(t, loopEvents)
+	if string(payload.eventsList) != string(wantEvents) {
+		t.Fatalf("payload.eventsList = %s, want %s", payload.eventsList, wantEvents)
+	}
+	if state.device != device || state.loopPullPoint != "loop" || !state.staticLoaded {
+		t.Fatalf("temporary failure invalidated healthy cached state: %+v", state)
+	}
+}
+
+func restoreHeartbeatONVIFStubs(t *testing.T) {
+	t.Helper()
+
+	originalConnect := heartbeatConnectToONVIFDevice
+	originalCreate := heartbeatCreatePullPointSubscription
+	originalDigitalInputs := heartbeatGetDigitalInputs
+	originalEvents := heartbeatGetEventMessages
+	originalPresets := heartbeatGetPresetsFromDevice
+	originalPTZConfigurations := heartbeatGetPTZConfigurationsFromDevice
+	originalPTZFunctions := heartbeatGetPTZFunctionsFromDevice
+	originalRelayOutputs := heartbeatGetRelayOutputs
+	originalUnsubscribe := heartbeatUnsubscribePullPoint
+
+	t.Cleanup(func() {
+		heartbeatConnectToONVIFDevice = originalConnect
+		heartbeatCreatePullPointSubscription = originalCreate
+		heartbeatGetDigitalInputs = originalDigitalInputs
+		heartbeatGetEventMessages = originalEvents
+		heartbeatGetPresetsFromDevice = originalPresets
+		heartbeatGetPTZConfigurationsFromDevice = originalPTZConfigurations
+		heartbeatGetPTZFunctionsFromDevice = originalPTZFunctions
+		heartbeatGetRelayOutputs = originalRelayOutputs
+		heartbeatUnsubscribePullPoint = originalUnsubscribe
+	})
+}
+
+func mustJSONMarshal(t *testing.T, v interface{}) []byte {
+	t.Helper()
+
+	b, err := json.Marshal(v)
+	if err != nil {
+		t.Fatalf("json.Marshal() error = %v", err)
+	}
+	return b
+}
+
+func assertDefaultHeartbeatONVIFPayload(t *testing.T, payload heartbeatONVIFPayload) {
+	t.Helper()
+
+	defaultPayload := defaultHeartbeatONVIFPayload()
+	if payload.enabled != defaultPayload.enabled ||
+		payload.zoom != defaultPayload.zoom ||
+		payload.panTilt != defaultPayload.panTilt ||
+		payload.presets != defaultPayload.presets ||
+		string(payload.presetsList) != string(defaultPayload.presetsList) ||
+		string(payload.eventsList) != string(defaultPayload.eventsList) {
+		t.Fatalf("payload = %+v, want default payload", payload)
+	}
+}
+
+func newTestONVIFDevice() *goonvif.Device {
+	return &goonvif.Device{}
+}
+
+func countString(values []string, want string) int {
+	count := 0
+	for _, value := range values {
+		if value == want {
+			count++
+		}
+	}
+	return count
+}

machinery/src/cloud/custom_headers.go

@@ -0,0 +1,195 @@
+package cloud
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"sort"
+	"strings"
+)
+
+const (
+	customHeadersManifestHeader = "X-Kerberos-Storage-Custom-Headers"
+	maxCustomHeaders            = 32
+	maxCustomHeaderNameLength   = 128
+	maxCustomHeaderValueLength  = 4096
+)
+
+var disallowedCustomHeaders = map[string]struct{}{
+	"authorization":       {},
+	"connection":          {},
+	"content-length":      {},
+	"content-type":        {},
+	"cookie":              {},
+	"expect":              {},
+	"host":                {},
+	"keep-alive":          {},
+	"proxy-authorization": {},
+	"proxy-connection":    {},
+	"te":                  {},
+	"trailer":             {},
+	"trace-id":            {},
+	"transfer-encoding":   {},
+	"tus-resumable":       {},
+	"upgrade":             {},
+}
+
+func parseCustomVaultHeaders(value string) (map[string]string, error) {
+	if strings.TrimSpace(value) == "" {
+		return nil, nil
+	}
+
+	// Decode into pointers so JSON null values can be told apart from strings.
+	var rawHeaders map[string]*string
+	if err := json.Unmarshal([]byte(value), &rawHeaders); err != nil {
+		return nil, fmt.Errorf("custom Vault headers must be a JSON object of string values: %w", err)
+	}
+	if rawHeaders == nil {
+		return nil, errors.New("custom Vault headers must be a JSON object of string values")
+	}
+	if len(rawHeaders) > maxCustomHeaders {
+		return nil, fmt.Errorf("custom Vault headers exceed the limit of %d", maxCustomHeaders)
+	}
+	headers := make(map[string]string, len(rawHeaders))
+	seen := make(map[string]struct{}, len(rawHeaders))
+	for name, rawValue := range rawHeaders {
+		if rawValue == nil {
+			return nil, fmt.Errorf("custom Vault header %q must be a string, not null", name)
+		}
+		value := *rawValue
+		headers[name] = value
+		if err := validateCustomVaultHeader(name, value); err != nil {
+			return nil, err
+		}
+		lowerName := strings.ToLower(name)
+		if _, duplicate := seen[lowerName]; duplicate {
+			return nil, fmt.Errorf("custom Vault header %q is duplicated with different casing", name)
+		}
+		seen[lowerName] = struct{}{}
+	}
+	return headers, nil
+}
+
+func validateCustomVaultHeader(name, value string) error {
+	if name == "" || len(name) > maxCustomHeaderNameLength || !isHTTPToken(name) {
+		return fmt.Errorf("invalid custom Vault header name %q", name)
+	}
+	lowerName := strings.ToLower(name)
+	if strings.HasPrefix(lowerName, "x-kerberos-") {
+		return fmt.Errorf("custom Vault header %q uses the reserved X-Kerberos namespace", name)
+	}
+	if strings.HasPrefix(lowerName, "upload-") {
+		return fmt.Errorf("custom Vault header %q uses the reserved tus Upload namespace", name)
+	}
+	if _, disallowed := disallowedCustomHeaders[lowerName]; disallowed {
+		return fmt.Errorf("custom Vault header %q is reserved", name)
+	}
+	if len(value) > maxCustomHeaderValueLength {
+		return fmt.Errorf("custom Vault header %q exceeds the %d byte value limit", name, maxCustomHeaderValueLength)
+	}
+	for _, character := range value {
+		if character < 0x20 || character == 0x7f {
+			return fmt.Errorf("custom Vault header %q contains an invalid value", name)
+		}
+	}
+	return nil
+}
+
+func isHTTPToken(value string) bool {
+	if value == "" {
+		return false
+	}
+	for _, character := range value {
+		if (character >= 'a' && character <= 'z') ||
+			(character >= 'A' && character <= 'Z') ||
+			(character >= '0' && character <= '9') ||
+			strings.ContainsRune("!#$%&'*+-.^_`|~", character) {
+			continue
+		}
+		return false
+	}
+	return true
+}
+
+func setCustomVaultHeaders(headers http.Header, customHeaders map[string]string) error {
+	if len(customHeaders) == 0 {
+		return nil
+	}
+
+	names := make([]string, 0, len(customHeaders))
+	for name, value := range customHeaders {
+		if err := validateCustomVaultHeader(name, value); err != nil {
+			return err
+		}
+		names = append(names, name)
+		// Assign directly so the configured name is not canonicalised by Set.
+		headers[name] = []string{value}
+	}
+	sort.Strings(names)
+	manifest, err := json.Marshal(names)
+	if err != nil {
+		return err
+	}
+	headers.Set(customHeadersManifestHeader, base64.RawURLEncoding.EncodeToString(manifest))
+	return nil
+}
+
+const maxVaultRedirects = 10
+
+// stripVaultHeadersOnCrossHostRedirect removes Vault credentials and the
+// customer-supplied metadata (custom headers, their manifest and the tus
+// Upload-Metadata that embeds them) before following a redirect to another host.
+func stripVaultHeadersOnCrossHostRedirect(req *http.Request, via []*http.Request) error {
+	if len(via) >= maxVaultRedirects {
+		return fmt.Errorf("stopped after %d redirects", maxVaultRedirects)
+	}
+	if len(via) == 0 || req.URL.Host == via[0].URL.Host {
+		return nil
+	}
+
+	customNames := customVaultHeaderNamesFromManifest(req.Header)
+	for name := range req.Header {
+		canonicalName := http.CanonicalHeaderKey(name)
+		_, custom := customNames[strings.ToLower(name)]
+		if custom || strings.HasPrefix(canonicalName, "X-Kerberos-") || canonicalName == "Upload-Metadata" {
+			delete(req.Header, name)
+		}
+	}
+	return nil
+}
+
+func customVaultHeaderNamesFromManifest(headers http.Header) map[string]struct{} {
+	encoded := headers.Get(customHeadersManifestHeader)
+	if encoded == "" {
+		return nil
+	}
+	manifest, err := base64.RawURLEncoding.DecodeString(encoded)
+	if err != nil {
+		return nil
+	}
+	var names []string
+	if err := json.Unmarshal(manifest, &names); err != nil {
+		return nil
+	}
+	lowerNames := make(map[string]struct{}, len(names))
+	for _, name := range names {
+		lowerNames[strings.ToLower(name)] = struct{}{}
+	}
+	return lowerNames
+}
+
+func encodeCustomVaultHeaders(customHeaders map[string]string) (string, error) {
+	if len(customHeaders) == 0 {
+		return "", nil
+	}
+	encoded, err := json.Marshal(customHeaders)
+	if err != nil {
+		return "", err
+	}
+	if len(encoded) == 0 {
+		return "", errors.New("custom Vault headers could not be encoded")
+	}
+	return string(encoded), nil
+}

machinery/src/cloud/custom_headers_test.go

@@ -0,0 +1,100 @@
+package cloud
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"net/http"
+	"reflect"
+	"testing"
+)
+
+func TestCustomVaultHeadersAreValidatedAndAdded(t *testing.T) {
+	customHeaders, err := parseCustomVaultHeaders(`{"site_id":"site-1","line_id":"line-2"}`)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	headers := http.Header{}
+	if err := setCustomVaultHeaders(headers, customHeaders); err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(headers["site_id"], []string{"site-1"}) || !reflect.DeepEqual(headers["line_id"], []string{"line-2"}) {
+		t.Fatalf("custom header names were not preserved: %#v", headers)
+	}
+	if _, canonicalised := headers["Site_id"]; canonicalised {
+		t.Fatalf("custom header name was canonicalised: %#v", headers)
+	}
+
+	encodedManifest, err := base64.RawURLEncoding.DecodeString(headers.Get(customHeadersManifestHeader))
+	if err != nil {
+		t.Fatal(err)
+	}
+	var names []string
+	if err := json.Unmarshal(encodedManifest, &names); err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(names, []string{"line_id", "site_id"}) {
+		t.Fatalf("manifest = %#v", names)
+	}
+}
+
+func TestCustomVaultHeadersRejectReservedAndInvalidValues(t *testing.T) {
+	for _, value := range []string{
+		`{"X-Kerberos-Storage-AccessKey":"secret"}`,
+		`{"Authorization":"secret"}`,
+		`{"site_id":"one\nInjected: value"}`,
+		`{"bad header":"value"}`,
+		`{"site_id":1}`,
+		`{"Content-Type":"text/plain"}`,
+		`{"site_id":"one","SITE_ID":"two"}`,
+		`{"site_id":null}`,
+		`{"site_id":"one\ttwo"}`,
+		`{"Upload-Checksum":"sha1 abc"}`,
+		`{"upload-concat":"final;/files/a"}`,
+		`{"Upload-Defer-Length":"1"}`,
+		`null`,
+	} {
+		if _, err := parseCustomVaultHeaders(value); err == nil {
+			t.Fatalf("parseCustomVaultHeaders(%q) succeeded", value)
+		}
+	}
+}
+
+func TestVaultRedirectStripsCustomHeadersOnHostChange(t *testing.T) {
+	newRedirect := func(t *testing.T, host string) (*http.Request, []*http.Request) {
+		t.Helper()
+		original, err := http.NewRequest(http.MethodPost, "https://vault.example.com/api/storage", nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		redirected, err := http.NewRequest(http.MethodPost, "https://"+host+"/api/storage", nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		redirected.Header.Set("X-Kerberos-Storage-AccessKey", "secret")
+		redirected.Header.Set("Upload-Metadata", "custom_headers abc")
+		redirected.Header.Set("Tus-Resumable", "1.0.0")
+		if err := setCustomVaultHeaders(redirected.Header, map[string]string{"site_id": "site-1", "Line-Id": "line-2"}); err != nil {
+			t.Fatal(err)
+		}
+		return redirected, []*http.Request{original}
+	}
+
+	crossHost, via := newRedirect(t, "other.example.com")
+	if err := stripVaultHeadersOnCrossHostRedirect(crossHost, via); err != nil {
+		t.Fatal(err)
+	}
+	if len(crossHost.Header) != 1 || crossHost.Header.Get("Tus-Resumable") != "1.0.0" {
+		t.Fatalf("cross-host redirect headers = %#v", crossHost.Header)
+	}
+
+	sameHost, via := newRedirect(t, "vault.example.com")
+	if err := stripVaultHeadersOnCrossHostRedirect(sameHost, via); err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(sameHost.Header["site_id"], []string{"site-1"}) ||
+		sameHost.Header.Get("X-Kerberos-Storage-AccessKey") != "secret" ||
+		sameHost.Header.Get(customHeadersManifestHeader) == "" {
+		t.Fatalf("same-host redirect headers = %#v", sameHost.Header)
+	}
+}

machinery/src/cloud/dropbox.go

@@ -11,8 +11,8 @@ import (
 	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
 	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/users"
 	"github.com/gin-gonic/gin"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 // UploadDropbox uploads the file to your Dropbox account using the access token and directory.
@@ -30,13 +30,13 @@ func UploadDropbox(configuration *models.Configuration, fileName string) (bool,
 
 	if token == "" {
 		err := "UploadDropbox: Dropbox not properly configured"
-		log.Log.Info(err)
+		log.Info(err)
 		return false, true, errors.New(err)
 	}
 
 	// Upload to Dropbox
-	log.Log.Info("UploadDropbox: Uploading to Dropbox")
-	log.Log.Info("UploadDropbox: Upload started for " + fileName)
+	log.Info("UploadDropbox: Uploading to Dropbox")
+	log.Info("UploadDropbox: Upload started for " + fileName)
 	fullname := "data/recordings/" + fileName
 
 	dConfig := dropbox.Config{
@@ -48,7 +48,7 @@ func UploadDropbox(configuration *models.Configuration, fileName string) (bool,
 	if file != nil {
 		defer func() {
 			if cerr := file.Close(); cerr != nil {
-				log.Log.Error("UploadDropbox: Error closing file: " + cerr.Error())
+				log.Error("UploadDropbox: Error closing file: " + cerr.Error())
 			}
 		}()
 	}
@@ -68,15 +68,15 @@ func UploadDropbox(configuration *models.Configuration, fileName string) (bool,
 		}, file)
 
 		if err != nil {
-			log.Log.Error("UploadDropbox: Error uploading file: " + err.Error())
+			log.Error("UploadDropbox: Error uploading file: " + err.Error())
 			return false, false, err
 		}
 
-		log.Log.Info("UploadDropbox: File uploaded successfully, " + res.Name)
+		log.Info("UploadDropbox: File uploaded successfully, " + res.Name)
 		return true, true, nil
 	}
 
-	log.Log.Error("UploadDropbox: Error opening file: " + err.Error())
+	log.Error("UploadDropbox: Error opening file: " + err.Error())
 	return false, true, err
 }
 

machinery/src/cloud/kerberos_hub.go

@@ -3,12 +3,13 @@ package cloud
 import (
 	"crypto/tls"
 	"errors"
+	"fmt"
 	"io/ioutil"
 	"net/http"
 	"os"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 func UploadKerberosHub(configuration *models.Configuration, fileName string) (bool, bool, error) {
@@ -19,7 +20,7 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 		config.HubPrivateKey == "" ||
 		config.S3.Region == "" {
 		err := "UploadKerberosHub: Kerberos Hub not properly configured."
-		log.Log.Info(err)
+		log.Info(err)
 		return false, false, errors.New(err)
 	}
 
@@ -32,8 +33,10 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 	// - Number of changes
 	// - Token
 
-	log.Log.Info("UploadKerberosHub: Uploading to Kerberos Hub (" + config.HubURI + ")")
-	log.Log.Info("UploadKerberosHub: Upload started for " + fileName)
+	log.WithFields(log.Fields{
+		"component": "kerberos_hub",
+		"event":     "upload_started",
+	}).Info("Uploading recording to Kerberos Hub")
 
 	// Prefer the resumable (tus) upload when enabled (the default). Kerberos Hub
 	// authenticates the agent with its Hub public/private key and proxies the
@@ -44,17 +47,35 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 		uploaded, _, supported, body, rerr := uploadHubResumable(&config, fileName, "UploadKerberosHub", "hub")
 		if supported {
 			if uploaded {
-				log.Log.Info("UploadKerberosHub: Upload Finished (resumable), " + body)
+				log.WithFields(log.Fields{
+					"component":      "kerberos_hub",
+					"event":          "upload_completed",
+					"response_bytes": len(body),
+					"transport":      "tus",
+				}).Info("Hub upload completed")
 				return true, true, nil
 			}
 			if rerr != nil {
-				log.Log.Info("UploadKerberosHub: resumable upload failed, " + rerr.Error())
+				log.WithError(rerr).WithFields(log.Fields{
+					"component": "kerberos_hub",
+					"event":     "upload_failed",
+					"transport": "tus",
+				}).Error("Hub upload failed")
 			} else {
-				log.Log.Info("UploadKerberosHub: resumable upload incomplete, " + body)
+				log.WithFields(log.Fields{
+					"component":      "kerberos_hub",
+					"event":          "upload_incomplete",
+					"response_bytes": len(body),
+					"transport":      "tus",
+				}).Warn("Hub upload incomplete")
 			}
 			return false, true, rerr
 		}
-		log.Log.Info("UploadKerberosHub: resumable (tus) endpoint not available, falling back to legacy upload")
+		log.WithFields(log.Fields{
+			"component": "kerberos_hub",
+			"event":     "upload_transport_fallback",
+			"transport": "http",
+		}).Info("Resumable Hub upload unavailable; using legacy upload")
 	}
 
 	fullname := "data/recordings/" + fileName
@@ -66,7 +87,7 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 	}
 	if err != nil {
 		err := "UploadKerberosHub: Upload Failed, file doesn't exists anymore."
-		log.Log.Info(err)
+		log.Info(err)
 		return false, false, errors.New(err)
 	}
 
@@ -74,9 +95,11 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 	// There might be different reasons like (muted, read-only..)
 	req, err := http.NewRequest("HEAD", config.HubURI+"/storage/upload", nil)
 	if err != nil {
-		errorMessage := "UploadKerberosHub: error reading HEAD request, " + config.HubURI + "/storage: " + err.Error()
-		log.Log.Error(errorMessage)
-		return false, true, errors.New(errorMessage)
+		log.WithError(err).WithFields(log.Fields{
+			"component": "kerberos_hub",
+			"event":     "authorization_request_creation_failed",
+		}).Error("Failed to create Hub upload authorization request")
+		return false, true, fmt.Errorf("create Hub upload authorization request: %w", err)
 	}
 
 	req.Header.Set("X-Kerberos-Storage-FileName", fileName)
@@ -102,25 +125,31 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 		defer resp.Body.Close()
 	}
 
-	if err == nil {
-		if resp != nil {
-			if err == nil {
-				if resp.StatusCode == 200 {
-					log.Log.Info("UploadKerberosHub: Upload allowed using the credentials provided (" + config.HubKey + ", " + config.HubPrivateKey + ")")
-				} else {
-					log.Log.Info("UploadKerberosHub: Upload NOT allowed using the credentials provided (" + config.HubKey + ", " + config.HubPrivateKey + ")")
-					return false, true, nil
-				}
-			}
+	if err == nil && resp != nil {
+		if resp.StatusCode == 200 {
+			log.WithFields(log.Fields{
+				"component":   "kerberos_hub",
+				"event":       "upload_authorized",
+				"status_code": resp.StatusCode,
+			}).Debug("Hub upload authorized")
+		} else {
+			log.WithFields(log.Fields{
+				"component":   "kerberos_hub",
+				"event":       "upload_rejected",
+				"status_code": resp.StatusCode,
+			}).Warn("Hub upload rejected")
+			return false, true, nil
 		}
 	}
 
 	// Now we know we are allowed to upload to the hub, we can start uploading.
 	req, err = http.NewRequest("POST", config.HubURI+"/storage/upload", file)
 	if err != nil {
-		errorMessage := "UploadKerberosHub: error reading POST request, " + config.KStorage.URI + "/storage/upload: " + err.Error()
-		log.Log.Error(errorMessage)
-		return false, true, errors.New(errorMessage)
+		log.WithError(err).WithFields(log.Fields{
+			"component": "kerberos_hub",
+			"event":     "upload_request_creation_failed",
+		}).Error("Failed to create Hub upload request")
+		return false, true, fmt.Errorf("create Hub upload request: %w", err)
 	}
 	req.Header.Set("Content-Type", "video/mp4")
 	req.Header.Set("X-Kerberos-Storage-FileName", fileName)
@@ -140,19 +169,36 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 			body, err := ioutil.ReadAll(resp.Body)
 			if err == nil {
 				if resp.StatusCode == 200 {
-					log.Log.Info("UploadKerberosHub: Upload Finished, " + resp.Status + ".")
+					log.WithFields(log.Fields{
+						"component":   "kerberos_hub",
+						"event":       "upload_completed",
+						"status_code": resp.StatusCode,
+						"transport":   "http",
+					}).Info("Hub upload completed")
 					return true, true, nil
 				} else {
-					log.Log.Info("UploadKerberosHub: Upload Failed, " + resp.Status + ", " + string(body))
+					log.WithFields(log.Fields{
+						"component":      "kerberos_hub",
+						"event":          "upload_rejected",
+						"response_bytes": len(body),
+						"status_code":    resp.StatusCode,
+						"transport":      "http",
+					}).Warn("Hub upload rejected")
 					return false, true, nil
 				}
 			}
 		}
 	}
 
-	errorMessage := "UploadKerberosHub: Upload Failed, " + err.Error()
-	log.Log.Info(errorMessage)
-	return false, true, errors.New(errorMessage)
+	if err == nil {
+		err = errors.New("Hub upload failed without a response")
+	}
+	log.WithError(err).WithFields(log.Fields{
+		"component": "kerberos_hub",
+		"event":     "upload_failed",
+		"transport": "http",
+	}).Error("Hub upload failed")
+	return false, true, fmt.Errorf("Hub upload failed: %w", err)
 }
 
 // stripHubCredentialsOnCrossHostRedirect removes the custom Kerberos Hub

machinery/src/cloud/kerberos_vault.go

@@ -9,8 +9,8 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 // We will count the number of retries we have done.
@@ -27,7 +27,7 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string) (
 		config.KStorage.Directory == "" ||
 		config.KStorage.URI == "" {
 		err := "UploadKerberosVault: Kerberos Vault not properly configured"
-		log.Log.Info(err)
+		log.Info(err)
 		return false, false, errors.New(err)
 	}
 
@@ -35,8 +35,13 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string) (
 	// This can happen when the file was already removed (e.g. cleanup, or an
 	// earlier successful upload). Skip it so the watcher drops the marker
 	// instead of retrying indefinitely.
-	if _, err := os.Stat("data/recordings/" + fileName); err != nil {
-		log.Log.Info("UploadKerberosVault: skipping " + fileName + ", file doesn't exist anymore")
+	info, err := os.Stat("data/recordings/" + fileName)
+	if err != nil {
+		log.Info("UploadKerberosVault: skipping " + fileName + ", file doesn't exist anymore")
+		return false, false, nil
+	}
+	if info.Size() == 0 {
+		log.Warn("UploadKerberosVault: skipping " + fileName + ", recording is empty")
 		return false, false, nil
 	}
 
@@ -49,8 +54,11 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string) (
 	// - Number of changes
 	// - Token
 	// KerberosCloud, this means storage is disabled and proxy enabled.
-	log.Log.Info("UploadKerberosVault: Uploading to Kerberos Vault (" + config.KStorage.URI + ")")
-	log.Log.Info("UploadKerberosVault: Upload started for " + fileName)
+	log.WithFields(log.Fields{
+		"component": "kerberos_vault",
+		"event":     "upload_started",
+		"storage":   "primary",
+	}).Info("Vault upload started")
 
 	publicKey := config.KStorage.CloudKey
 	if config.HubKey != "" {
@@ -62,14 +70,27 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string) (
 		uploaded, responded, body, err := sendToVault(*config.KStorage, publicKey, config.Key, fileName, "UploadKerberosVault", "primary")
 		if uploaded {
 			kstorageRetryCount = 0
-			log.Log.Info("UploadKerberosVault: Upload Finished, " + body)
+			log.WithFields(log.Fields{
+				"component": "kerberos_vault",
+				"event":     "upload_completed",
+				"storage":   "primary",
+			}).Info("Vault upload completed")
 			return true, true, nil
 		}
 
 		if err != nil {
-			log.Log.Info("UploadKerberosVault: Upload Failed, " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component": "kerberos_vault",
+				"event":     "upload_failed",
+				"storage":   "primary",
+			}).Error("Vault upload failed")
 		} else {
-			log.Log.Info("UploadKerberosVault: Upload Failed, " + body)
+			log.WithFields(log.Fields{
+				"component":      "kerberos_vault",
+				"event":          "upload_rejected",
+				"response_bytes": len(body),
+				"storage":        "primary",
+			}).Warn("Vault upload rejected")
 		}
 
 		// We only advance the retry policy when the vault gave a definitive
@@ -91,26 +112,43 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string) (
 		config.KStorageSecondary.SecretAccessKey == "" ||
 		config.KStorageSecondary.Directory == "" ||
 		config.KStorageSecondary.URI == "" {
-		log.Log.Info("UploadKerberosVault (Secondary): Secondary Kerberos Vault not properly configured.")
+		log.Info("UploadKerberosVault (Secondary): Secondary Kerberos Vault not properly configured.")
 	} else {
 
 		if kstorageRetryCount < config.KStorage.MaxRetries {
-			log.Log.Info("UploadKerberosVault (Secondary): Do not upload to secondary storage, we are still in retry policy.")
+			log.Info("UploadKerberosVault (Secondary): Do not upload to secondary storage, we are still in retry policy.")
 			return false, true, nil
 		}
 
-		log.Log.Info("UploadKerberosVault (Secondary): Uploading to Secondary Kerberos Vault (" + config.KStorageSecondary.URI + ")")
+		log.WithFields(log.Fields{
+			"component": "kerberos_vault",
+			"event":     "upload_started",
+			"storage":   "secondary",
+		}).Info("Vault upload started")
 
 		uploaded, _, body, err := sendToVault(*config.KStorageSecondary, publicKey, config.Key, fileName, "UploadKerberosVault (Secondary)", "secondary")
 		if uploaded {
-			log.Log.Info("UploadKerberosVault (Secondary): Upload Finished to secondary, " + body)
+			log.WithFields(log.Fields{
+				"component": "kerberos_vault",
+				"event":     "upload_completed",
+				"storage":   "secondary",
+			}).Info("Vault upload completed")
 			return true, true, nil
 		}
 
 		if err != nil {
-			log.Log.Info("UploadKerberosVault (Secondary): Upload Failed to secondary, " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component": "kerberos_vault",
+				"event":     "upload_failed",
+				"storage":   "secondary",
+			}).Error("Vault upload failed")
 		} else {
-			log.Log.Info("UploadKerberosVault (Secondary): Upload Failed to secondary, " + body)
+			log.WithFields(log.Fields{
+				"component":      "kerberos_vault",
+				"event":          "upload_rejected",
+				"response_bytes": len(body),
+				"storage":        "secondary",
+			}).Warn("Vault upload rejected")
 		}
 	}
 
@@ -131,7 +169,7 @@ func sendToVault(vault models.KStorage, publicKey, deviceKey, fileName, label, s
 		if supported {
 			return uploaded, responded, body, err
 		}
-		log.Log.Info(label + ": resumable (tus) endpoint not available, falling back to legacy upload")
+		log.Info(label + ": resumable (tus) endpoint not available, falling back to legacy upload")
 	}
 	return uploadVaultLegacy(vault, publicKey, deviceKey, fileName, label)
 }
@@ -140,6 +178,10 @@ func sendToVault(vault models.KStorage, publicKey, deviceKey, fileName, label, s
 // is sent as the body of a POST to {URI}/storage. Kept for backwards
 // compatibility with vault deployments that do not support resumable uploads.
 func uploadVaultLegacy(vault models.KStorage, publicKey, deviceKey, fileName, label string) (bool, bool, string, error) {
+	customHeaders, err := parseCustomVaultHeaders(vault.CustomHeaders)
+	if err != nil {
+		return false, false, "", err
+	}
 	fullname := "data/recordings/" + fileName
 
 	file, err := os.Open(fullname)
@@ -148,7 +190,7 @@ func uploadVaultLegacy(vault models.KStorage, publicKey, deviceKey, fileName, la
 	}
 	if err != nil {
 		msg := label + ": Upload Failed, file doesn't exists anymore"
-		log.Log.Info(msg)
+		log.Info(msg)
 		return false, false, "", errors.New(msg)
 	}
 
@@ -160,11 +202,14 @@ func uploadVaultLegacy(vault models.KStorage, publicKey, deviceKey, fileName, la
 	req, err := http.NewRequest("POST", uri+"/storage", file)
 	if err != nil {
 		errorMessage := label + ": error reading request, " + uri + "/storage: " + err.Error()
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, false, "", errors.New(errorMessage)
 	}
 	req.Header.Set("Content-Type", "video/mp4")
 	setVaultHeaders(req.Header, vault, publicKey, deviceKey, fileName)
+	if err := setCustomVaultHeaders(req.Header, customHeaders); err != nil {
+		return false, false, "", err
+	}
 	setQueuedRecordingMetadataHeaders(req.Header, fileName)
 
 	client := newVaultHTTPClient(0)
@@ -229,7 +274,7 @@ func newVaultHTTPClient(timeout time.Duration) *http.Client {
 		transport.TLSClientConfig.InsecureSkipVerify = true
 	}
 
-	client := &http.Client{Transport: transport}
+	client := &http.Client{Transport: transport, CheckRedirect: stripVaultHeadersOnCrossHostRedirect}
 	if timeout > 0 {
 		client.Timeout = timeout
 	}

machinery/src/cloud/livehls.go

@@ -7,9 +7,9 @@ import (
 	mqtt "github.com/eclipse/paho.mqtt.golang"
 
 	"github.com/kerberos-io/agent/machinery/src/cloud/livehls"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
+	log "github.com/sirupsen/logrus"
 )
 
 // hlsViewerTimeoutSeconds is how long the agent keeps shipping live HLS segments
@@ -45,22 +45,29 @@ const hlsReadyReannounceSeconds = 2
 // trades a little idle CPU for a near-instant "requesting stream", so viewers no
 // longer wait a full GOP for the first segment to be cut. Set
 // AGENT_LIVE_HLS_PREWARM=false to fall back to the lazy on-demand path above.
-func HandleLiveStreamHLS(configuration *models.Configuration, communication *models.Communication, mqttClient mqtt.Client, subStreamEnabled bool) {
+func HandleLiveStreamHLS(
+	configuration *models.Configuration,
+	communication *models.Communication,
+	mqttClient mqtt.Client,
+	subStreamEnabled bool,
+	mainQueue *packets.Queue,
+	subQueue *packets.Queue,
+) {
 
-	log.Log.Debug("cloud.HandleLiveStreamHLS(): started")
+	log.Debug("cloud.HandleLiveStreamHLS(): started")
 
 	config := configuration.Config
 
 	if config.Offline == "true" {
-		log.Log.Debug("cloud.HandleLiveStreamHLS(): stopping as Offline is enabled.")
+		log.Debug("cloud.HandleLiveStreamHLS(): stopping as Offline is enabled.")
 		return
 	}
 	if config.Capture.Liveview == "false" {
-		log.Log.Debug("cloud.HandleLiveStreamHLS(): stopping as Liveview is disabled.")
+		log.Debug("cloud.HandleLiveStreamHLS(): stopping as Liveview is disabled.")
 		return
 	}
 	if config.HubURI == "" || config.HubKey == "" {
-		log.Log.Debug("cloud.HandleLiveStreamHLS(): stopping as the Hub is not configured (HubURI/HubKey).")
+		log.Debug("cloud.HandleLiveStreamHLS(): stopping as the Hub is not configured (HubURI/HubKey).")
 		return
 	}
 
@@ -88,8 +95,8 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 	// mp4ff's strict parser rejects).
 	requestedQuality := models.StreamQualityAuto
 	useSub := models.SelectSubStreamForQuality(config, requestedQuality, subStreamEnabled)
-	source := buildHLSSource(config, communication, useSub)
-	log.Log.Info("cloud.HandleLiveStreamHLS(): serving live HLS from the " + source.label + " stream")
+	source := buildHLSSource(config, mainQueue, subQueue, useSub)
+	log.Info("cloud.HandleLiveStreamHLS(): serving live HLS from the " + source.label + " stream")
 
 	// prewarm keeps a single long-lived session muxing into an in-memory ring
 	// buffer while idle and flushes it the instant a viewer arrives, eliminating
@@ -97,9 +104,9 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 	// to fall back to the lazy on-demand path.
 	prewarm := os.Getenv("AGENT_LIVE_HLS_PREWARM") != "false"
 	if prewarm {
-		log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS prewarm ENABLED (set AGENT_LIVE_HLS_PREWARM=false to disable)")
+		log.Info("cloud.HandleLiveStreamHLS(): live HLS prewarm ENABLED (set AGENT_LIVE_HLS_PREWARM=false to disable)")
 	} else {
-		log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS prewarm DISABLED (AGENT_LIVE_HLS_PREWARM=false)")
+		log.Info("cloud.HandleLiveStreamHLS(): live HLS prewarm DISABLED (AGENT_LIVE_HLS_PREWARM=false)")
 	}
 
 	// lowLatency enables LL-HLS: each segment is sliced into CMAF parts shipped the
@@ -109,9 +116,9 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 	partTargetMs := uint64(0)
 	if os.Getenv("AGENT_LIVE_HLS_LOW_LATENCY") != "false" {
 		partTargetMs = livehls.DefaultPartTargetMs
-		log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS low-latency (LL-HLS) ENABLED (set AGENT_LIVE_HLS_LOW_LATENCY=false to disable)")
+		log.Info("cloud.HandleLiveStreamHLS(): live HLS low-latency (LL-HLS) ENABLED (set AGENT_LIVE_HLS_LOW_LATENCY=false to disable)")
 	} else {
-		log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS low-latency (LL-HLS) DISABLED (AGENT_LIVE_HLS_LOW_LATENCY=false)")
+		log.Info("cloud.HandleLiveStreamHLS(): live HLS low-latency (LL-HLS) DISABLED (AGENT_LIVE_HLS_LOW_LATENCY=false)")
 	}
 
 	var session *livehls.Session
@@ -123,6 +130,9 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 
 	for cursorError == nil {
 		pkt, cursorError = source.cursor.ReadPacket()
+		if cursorError != nil {
+			break
+		}
 
 		now := time.Now().Unix()
 		select {
@@ -156,9 +166,9 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 				_ = session.Close()
 				session = nil
 			}
-			source = buildHLSSource(config, communication, useSub)
+			source = buildHLSSource(config, mainQueue, subQueue, useSub)
 			lastReadyAnnounce = 0
-			log.Log.Info("cloud.HandleLiveStreamHLS(): switched live HLS to the " + source.label + " stream (quality=" + requestedQuality + ")")
+			log.Info("cloud.HandleLiveStreamHLS(): switched live HLS to the " + source.label + " stream (quality=" + requestedQuality + ")")
 			continue
 		}
 
@@ -184,11 +194,22 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 					StartBuffering: true,
 				})
 				session.SetOnReady(func(sessionID string) {
-					log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS session ready, announcing " + sessionID)
+					log.WithFields(log.Fields{
+						"component":  "cloud/livehls",
+						"event":      "session_ready",
+						"session_id": sessionID,
+					}).Info("Live HLS session ready")
 					publishHLSReady(configuration, mqttClient, hubKey, deviceId, sessionID)
 					lastReadyAnnounce = time.Now().Unix()
 				})
-				log.Log.Info("cloud.HandleLiveStreamHLS(): prewarming live HLS session " + session.SessionID())
+				session.SetOnFailure(func(sessionID, reason string) {
+					publishHLSFailure(configuration, mqttClient, hubKey, deviceId, sessionID, reason)
+				})
+				log.WithFields(log.Fields{
+					"component":  "cloud/livehls",
+					"event":      "session_prewarming",
+					"session_id": session.SessionID(),
+				}).Info("Prewarming live HLS session")
 			}
 
 			if viewerActive {
@@ -207,7 +228,7 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 
 			if len(pkt.Data) > 0 && pkt.IsVideo {
 				if err := session.WritePacket(pkt); err != nil {
-					log.Log.Error("cloud.HandleLiveStreamHLS(): " + err.Error())
+					log.Error("cloud.HandleLiveStreamHLS(): " + err.Error())
 				}
 			}
 			continue
@@ -217,7 +238,12 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 			// No viewer: stop and discard the session so we stop shipping segments.
 			if session != nil {
 				_ = session.Close()
-				log.Log.Info("cloud.HandleLiveStreamHLS(): no active viewers, stopped live HLS session " + session.SessionID())
+				log.WithFields(log.Fields{
+					"component":  "cloud/livehls",
+					"event":      "session_stopped",
+					"reason":     "no_active_viewers",
+					"session_id": session.SessionID(),
+				}).Info("Live HLS session stopped")
 				session = nil
 			}
 			continue
@@ -243,22 +269,33 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 				PartTargetMs: partTargetMs,
 			})
 			session.SetOnReady(func(sessionID string) {
-				log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS session ready, announcing " + sessionID)
+				log.WithFields(log.Fields{
+					"component":  "cloud/livehls",
+					"event":      "session_ready",
+					"session_id": sessionID,
+				}).Info("Live HLS session ready")
 				publishHLSReady(configuration, mqttClient, hubKey, deviceId, sessionID)
 				lastReadyAnnounce = time.Now().Unix()
 			})
-			log.Log.Info("cloud.HandleLiveStreamHLS(): started live HLS session " + session.SessionID())
+			session.SetOnFailure(func(sessionID, reason string) {
+				publishHLSFailure(configuration, mqttClient, hubKey, deviceId, sessionID, reason)
+			})
+			log.WithFields(log.Fields{
+				"component":  "cloud/livehls",
+				"event":      "session_started",
+				"session_id": session.SessionID(),
+			}).Info("Live HLS session started")
 		}
 
 		if err := session.WritePacket(pkt); err != nil {
-			log.Log.Error("cloud.HandleLiveStreamHLS(): " + err.Error())
+			log.Error("cloud.HandleLiveStreamHLS(): " + err.Error())
 		}
 	}
 
 	if session != nil {
 		_ = session.Close()
 	}
-	log.Log.Debug("cloud.HandleLiveStreamHLS(): finished")
+	log.Debug("cloud.HandleLiveStreamHLS(): finished")
 }
 
 // publishHLSReady announces, over MQTT, that a live HLS session is available so
@@ -278,9 +315,48 @@ func publishHLSReady(configuration *models.Configuration, mqttClient mqtt.Client
 	payload, err := models.PackageMQTTMessage(configuration, message)
 	if err == nil {
 		mqttClient.Publish("kerberos/hub/"+hubKey, 0, false, payload)
-		log.Log.Info("cloud.HandleLiveStreamHLS(): announced live HLS session " + sessionID)
+		log.WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "session_announced",
+			"session_id": sessionID,
+		}).Info("Live HLS session announced")
 	} else {
-		log.Log.Error("cloud.HandleLiveStreamHLS(): failed to package receive-hls-ready message: " + err.Error())
+		log.WithError(err).WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "session_announcement_failed",
+			"session_id": sessionID,
+		}).Error("Failed to package live HLS session announcement")
+	}
+}
+
+func publishHLSFailure(configuration *models.Configuration, mqttClient mqtt.Client, hubKey, deviceId, sessionID, reason string) {
+	valueMap := map[string]interface{}{
+		"session": sessionID,
+		"device":  deviceId,
+		"reason":  reason,
+	}
+	message := models.Message{
+		Payload: models.Payload{
+			Action:   "receive-hls-error",
+			DeviceId: deviceId,
+			Value:    valueMap,
+		},
+	}
+	payload, err := models.PackageMQTTMessage(configuration, message)
+	if err == nil {
+		mqttClient.Publish("kerberos/hub/"+hubKey, 0, false, payload)
+		log.WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "session_failure_announced",
+			"reason":     reason,
+			"session_id": sessionID,
+		}).Warn("Live HLS session failure announced")
+	} else {
+		log.WithError(err).WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "session_failure_announcement_failed",
+			"session_id": sessionID,
+		}).Error("Failed to package live HLS session failure")
 	}
 }
 
@@ -302,11 +378,11 @@ type hlsStreamSource struct {
 // is available; otherwise the main (high-resolution) stream is used. A fresh
 // Latest() cursor is created so muxing resumes from the live edge of the chosen
 // stream after a switch.
-func buildHLSSource(config models.Config, communication *models.Communication, useSub bool) hlsStreamSource {
+func buildHLSSource(config models.Config, mainQueue, subQueue *packets.Queue, useSub bool) hlsStreamSource {
 	cam := config.Capture.IPCamera
-	if useSub && communication.SubQueue != nil {
+	if useSub && subQueue != nil {
 		return hlsStreamSource{
-			cursor: communication.SubQueue.Latest(),
+			cursor: subQueue.Latest(),
 			sps:    cam.SubSPSNALUs,
 			pps:    cam.SubPPSNALUs,
 			vps:    cam.SubVPSNALUs,
@@ -316,7 +392,7 @@ func buildHLSSource(config models.Config, communication *models.Communication, u
 		}
 	}
 	return hlsStreamSource{
-		cursor: communication.Queue.Latest(),
+		cursor: mainQueue.Latest(),
 		sps:    cam.SPSNALUs,
 		pps:    cam.PPSNALUs,
 		vps:    cam.VPSNALUs,

machinery/src/cloud/livehls/publisher.go

@@ -24,13 +24,14 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"net/http"
 	"strconv"
 	"strings"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/video"
+	log "github.com/sirupsen/logrus"
 )
 
 const (
@@ -67,6 +68,7 @@ const (
 	// cannot be delivered within roughly its own duration is stale, so the upload
 	// is abandoned (dropped) rather than allowed to back up the pipeline.
 	defaultPublishTimeout = 4 * time.Second
+	maxErrorResponseBytes = 4 << 10
 )
 
 // PublisherConfig carries the hub endpoint and credentials needed to ship live
@@ -214,9 +216,15 @@ func (p *Publisher) post(ctx context.Context, params postParams) error {
 	defer resp.Body.Close()
 
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		return fmt.Errorf("livehls: upload %s rejected: %s", params.name, resp.Status)
+		body, _ := io.ReadAll(io.LimitReader(resp.Body, maxErrorResponseBytes))
+		return fmt.Errorf("livehls: upload %s rejected with status %d (%d response bytes)", params.name, resp.StatusCode, len(body))
 	}
-	log.Log.Debug("livehls.Publisher.post(): shipped " + params.name + " for session " + params.sessionID)
+	log.WithFields(log.Fields{
+		"component":  "cloud/livehls",
+		"event":      "object_uploaded",
+		"object":     params.name,
+		"session_id": params.sessionID,
+	}).Trace("Live HLS object uploaded")
 	return nil
 }
 

machinery/src/cloud/livehls/publisher_test.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"sync"
 	"testing"
 	"time"
@@ -144,12 +145,22 @@ func TestPublisherPublishSegmentSendsSequenceAndDuration(t *testing.T) {
 }
 
 func TestPublisherReturnsErrorOnNon2xx(t *testing.T) {
-	srv, _, _ := newCapturingServer(t, http.StatusInternalServerError)
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusBadRequest)
+		_, _ = w.Write([]byte(`{"error":true,"data":"No user found with this public and private key."}`))
+	}))
+	t.Cleanup(srv.Close)
 	p := testPublisher(srv.URL)
 
 	err := p.PublishSegment(context.Background(), "s", video.LiveSegment{SequenceNumber: 1, Data: []byte("x")})
 	if err == nil {
-		t.Fatal("expected an error on 500 response")
+		t.Fatal("expected an error on 400 response")
+	}
+	if !strings.Contains(err.Error(), "status 400") || !strings.Contains(err.Error(), "71 response bytes") {
+		t.Fatalf("error = %q, want status code and response size", err)
+	}
+	if strings.Contains(err.Error(), "No user found with this public and private key") {
+		t.Fatalf("error exposed Hub response body: %q", err)
 	}
 }
 
@@ -281,7 +292,13 @@ func TestSessionRetriesInitWhenFirstAttemptFails(t *testing.T) {
 	})
 
 	var ready int
+	var failures int
+	var failureReason string
 	sess.SetOnReady(func(string) { ready++ })
+	sess.SetOnFailure(func(_ string, reason string) {
+		failures++
+		failureReason = reason
+	})
 
 	for i := 0; i < 60; i++ {
 		isKey := i%25 == 0
@@ -304,6 +321,9 @@ func TestSessionRetriesInitWhenFirstAttemptFails(t *testing.T) {
 	if ready != 1 {
 		t.Errorf("OnReady fired %d times, want 1", ready)
 	}
+	if failures != 1 || failureReason != "init-upload-failed" {
+		t.Errorf("OnFailure = %d/%q, want 1/init-upload-failed", failures, failureReason)
+	}
 }
 
 // liveTestSPSForSession is the known-good baseline SPS reused across tests.

machinery/src/cloud/livehls/session.go

@@ -8,9 +8,9 @@ import (
 	"sync"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/kerberos-io/agent/machinery/src/video"
+	log "github.com/sirupsen/logrus"
 )
 
 // DefaultTargetSegmentMs is the nominal live segment length. ~2s keeps standard
@@ -46,9 +46,11 @@ type Session struct {
 	// lastInitAt is when the init segment was last (re)uploaded. The init is
 	// re-sent periodically so its short TTL in the hub live window never lapses
 	// mid-session; see refreshInitIfStale.
-	lastInitAt time.Time
-	readyFired bool
-	onReady    func(sessionID string)
+	lastInitAt   time.Time
+	readyFired   bool
+	onReady      func(sessionID string)
+	failureFired bool
+	onFailure    func(sessionID, reason string)
 
 	// uploadsActive gates whether the init and completed segments are shipped to
 	// hub-api. It is true for the default on-demand path. The prewarm path starts
@@ -142,14 +144,25 @@ func NewSession(publisher *Publisher, opts SessionOptions) *Session {
 			return nil
 		}
 		if !s.publishInitIfNeeded() {
-			log.Log.Warning("livehls.Session: dropping segment " +
-				fmt.Sprintf("%d", segment.SequenceNumber) + " because init has not been delivered yet")
+			log.WithFields(log.Fields{
+				"component":        "cloud/livehls",
+				"event":            "segment_dropped",
+				"reason":           "init_unavailable",
+				"segment_sequence": segment.SequenceNumber,
+				"session_id":       s.id,
+			}).Warn("Dropping live HLS segment")
 			return nil
 		}
 		ctx, cancel := s.newContext()
 		defer cancel()
 		if err := s.publisher.PublishSegment(ctx, s.id, segment); err != nil {
-			log.Log.Warning("livehls.Session: " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component":        "cloud/livehls",
+				"event":            "segment_upload_failed",
+				"segment_sequence": segment.SequenceNumber,
+				"session_id":       s.id,
+			}).Warn("Failed to upload live HLS segment")
+			s.fireFailureOnce("segment-upload-failed")
 			return nil
 		}
 		s.fireReadyOnce()
@@ -172,15 +185,27 @@ func NewSession(publisher *Publisher, opts SessionOptions) *Session {
 				return nil
 			}
 			if !s.publishInitIfNeeded() {
-				log.Log.Warning("livehls.Session: dropping part " +
-					fmt.Sprintf("%d.%d", part.SegmentSeq, part.PartIndex) +
-					" because init has not been delivered yet")
+				log.WithFields(log.Fields{
+					"component":        "cloud/livehls",
+					"event":            "part_dropped",
+					"part_index":       part.PartIndex,
+					"reason":           "init_unavailable",
+					"segment_sequence": part.SegmentSeq,
+					"session_id":       s.id,
+				}).Warn("Dropping live HLS part")
 				return nil
 			}
 			ctx, cancel := s.newContext()
 			defer cancel()
 			if err := s.publisher.PublishPart(ctx, s.id, part); err != nil {
-				log.Log.Warning("livehls.Session: " + err.Error())
+				log.WithError(err).WithFields(log.Fields{
+					"component":        "cloud/livehls",
+					"event":            "part_upload_failed",
+					"part_index":       part.PartIndex,
+					"segment_sequence": part.SegmentSeq,
+					"session_id":       s.id,
+				}).Warn("Failed to upload live HLS part")
+				s.fireFailureOnce("part-upload-failed")
 				return nil
 			}
 			s.fireReadyOnce()
@@ -216,6 +241,15 @@ func (s *Session) SetOnReady(fn func(sessionID string)) {
 	s.mu.Unlock()
 }
 
+// SetOnFailure registers a one-shot callback for startup upload failures. The
+// reason is a fixed code rather than an HTTP response body, so credentials or
+// server details cannot leak through MQTT diagnostics.
+func (s *Session) SetOnFailure(fn func(sessionID, reason string)) {
+	s.mu.Lock()
+	s.onFailure = fn
+	s.mu.Unlock()
+}
+
 // prewarmMaxBufferedSegments is how many of the most recent completed segments
 // the prewarm path keeps in memory while idle and flushes to a viewer on arrival.
 // One segment keeps startup instant (the viewer immediately gets a playable
@@ -268,7 +302,13 @@ func (s *Session) SetUploadsActive(active bool) bool {
 		}
 		ctx, cancel := s.newContext()
 		if err := s.publisher.PublishSegment(ctx, s.id, buffered[i]); err != nil {
-			log.Log.Warning("livehls.Session: prewarm flush: " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component":        "cloud/livehls",
+				"event":            "prewarm_segment_upload_failed",
+				"segment_sequence": buffered[i].SequenceNumber,
+				"session_id":       s.id,
+			}).Warn("Failed to upload buffered live HLS segment")
+			s.fireFailureOnce("segment-upload-failed")
 			cancel()
 			continue
 		}
@@ -284,7 +324,14 @@ func (s *Session) SetUploadsActive(active bool) bool {
 		}
 		ctx, cancel := s.newContext()
 		if err := s.publisher.PublishPart(ctx, s.id, bufferedParts[i]); err != nil {
-			log.Log.Warning("livehls.Session: prewarm flush (part): " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component":        "cloud/livehls",
+				"event":            "prewarm_part_upload_failed",
+				"part_index":       bufferedParts[i].PartIndex,
+				"segment_sequence": bufferedParts[i].SegmentSeq,
+				"session_id":       s.id,
+			}).Warn("Failed to upload buffered live HLS part")
+			s.fireFailureOnce("part-upload-failed")
 			cancel()
 			continue
 		}
@@ -381,7 +428,12 @@ func (s *Session) publishInitIfNeeded() bool {
 	ctx, cancel := s.newContext()
 	defer cancel()
 	if err := s.publisher.PublishInit(ctx, s.id, initBytes); err != nil {
-		log.Log.Warning("livehls.Session: init upload failed, will retry: " + err.Error())
+		log.WithError(err).WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "init_upload_failed",
+			"session_id": s.id,
+		}).Warn("Failed to upload live HLS init segment; retrying")
+		s.fireFailureOnce("init-upload-failed")
 		return false
 	}
 
@@ -389,7 +441,11 @@ func (s *Session) publishInitIfNeeded() bool {
 	s.initPublished = true
 	s.lastInitAt = time.Now()
 	s.mu.Unlock()
-	log.Log.Info("livehls.Session: init segment delivered for session " + s.id)
+	log.WithFields(log.Fields{
+		"component":  "cloud/livehls",
+		"event":      "init_uploaded",
+		"session_id": s.id,
+	}).Info("Live HLS init segment uploaded")
 	return true
 }
 
@@ -422,14 +478,22 @@ func (s *Session) refreshInitIfStale() {
 	ctx, cancel := s.newContext()
 	defer cancel()
 	if err := s.publisher.PublishInit(ctx, s.id, initBytes); err != nil {
-		log.Log.Warning("livehls.Session: init refresh failed, will retry: " + err.Error())
+		log.WithError(err).WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "init_refresh_failed",
+			"session_id": s.id,
+		}).Warn("Failed to refresh live HLS init segment; retrying")
 		return
 	}
 
 	s.mu.Lock()
 	s.lastInitAt = time.Now()
 	s.mu.Unlock()
-	log.Log.Debug("livehls.Session: refreshed init segment TTL for session " + s.id)
+	log.WithFields(log.Fields{
+		"component":  "cloud/livehls",
+		"event":      "init_refreshed",
+		"session_id": s.id,
+	}).Debug("Live HLS init segment refreshed")
 }
 
 // fireReadyOnce invokes the OnReady callback the first time it is called.
@@ -445,6 +509,18 @@ func (s *Session) fireReadyOnce() {
 	fn(s.id)
 }
 
+func (s *Session) fireFailureOnce(reason string) {
+	s.mu.Lock()
+	if s.failureFired || s.readyFired || s.onFailure == nil {
+		s.mu.Unlock()
+		return
+	}
+	s.failureFired = true
+	fn := s.onFailure
+	s.mu.Unlock()
+	fn(s.id, reason)
+}
+
 // newSessionID returns a short, unique, URL-safe session identifier of the form
 // <unix-seconds>-<random-hex>.
 func newSessionID() string {

machinery/src/cloud/livemoq/recovery.go

@@ -1,6 +1,9 @@
 package livemoq
 
-import "time"
+import (
+	"sync/atomic"
+	"time"
+)
 
 type FrameGateEvent uint8
 
@@ -17,9 +20,52 @@ type FrameGate struct {
 	recovering bool
 }
 
-// Reset closes the gate so publication resumes on the next keyframe. It is used
-// when the publisher stopped writing for a reason unrelated to the stream health
-// (no subscribers), so the next viewer never receives a partial GOP.
+// AudienceGate publishes every frame while watched and only keyframes while
+// idle. The idle keyframes keep the relay's latest cached group current without
+// paying the bandwidth cost of a full stream when nobody is watching.
+type AudienceGate struct {
+	idle bool
+}
+
+func (g *AudienceGate) Allow(hasSubscribers bool, isKeyFrame bool) (allowed bool, enteredIdle bool) {
+	if hasSubscribers {
+		g.idle = false
+		return true, false
+	}
+
+	enteredIdle = !g.idle
+	g.idle = true
+	return isKeyFrame, enteredIdle
+}
+
+// WriteWatchdog tracks the single synchronous frame write performed by a MoQ
+// publisher so another goroutine can interrupt a wedged native call.
+type WriteWatchdog struct {
+	startedAt atomic.Int64
+}
+
+func (w *WriteWatchdog) Begin(now time.Time) {
+	w.startedAt.Store(now.UnixNano())
+}
+
+func (w *WriteWatchdog) End() {
+	w.startedAt.Store(0)
+}
+
+func (w *WriteWatchdog) Elapsed(now time.Time) (time.Duration, bool) {
+	startedAt := w.startedAt.Load()
+	if startedAt == 0 {
+		return 0, false
+	}
+	elapsed := now.Sub(time.Unix(0, startedAt))
+	if elapsed < 0 {
+		elapsed = 0
+	}
+	return elapsed, true
+}
+
+// Reset closes the gate so publication resumes on the next keyframe. Entering
+// idle mode uses it to start the relay-cache refresh on a complete GOP.
 func (g *FrameGate) Reset() {
 	g.started = false
 	g.recovering = false

machinery/src/cloud/livemoq/recovery_test.go

@@ -47,3 +47,45 @@ func TestFrameGateAllowsMissingCaptureTime(t *testing.T) {
 		t.Fatalf("Allow() = (%t, %d), want (true, %d)", allowed, event, FrameGateEventStarted)
 	}
 }
+
+func TestAudienceGateRefreshesOnlyKeyframesWhileIdle(t *testing.T) {
+	gate := AudienceGate{}
+
+	if allowed, enteredIdle := gate.Allow(true, false); !allowed || enteredIdle {
+		t.Fatalf("active delta = (%t, %t), want (true, false)", allowed, enteredIdle)
+	}
+	if allowed, enteredIdle := gate.Allow(false, false); allowed || !enteredIdle {
+		t.Fatalf("first idle delta = (%t, %t), want (false, true)", allowed, enteredIdle)
+	}
+	if allowed, enteredIdle := gate.Allow(false, true); !allowed || enteredIdle {
+		t.Fatalf("idle keyframe = (%t, %t), want (true, false)", allowed, enteredIdle)
+	}
+	if allowed, enteredIdle := gate.Allow(false, false); allowed || enteredIdle {
+		t.Fatalf("later idle delta = (%t, %t), want (false, false)", allowed, enteredIdle)
+	}
+	if allowed, enteredIdle := gate.Allow(true, false); !allowed || enteredIdle {
+		t.Fatalf("resumed delta = (%t, %t), want (true, false)", allowed, enteredIdle)
+	}
+	if allowed, enteredIdle := gate.Allow(false, true); !allowed || !enteredIdle {
+		t.Fatalf("next idle keyframe = (%t, %t), want (true, true)", allowed, enteredIdle)
+	}
+}
+
+func TestWriteWatchdogTracksActiveWrite(t *testing.T) {
+	now := time.Unix(10, 0)
+	watchdog := WriteWatchdog{}
+
+	if elapsed, active := watchdog.Elapsed(now); active || elapsed != 0 {
+		t.Fatalf("Elapsed() before Begin() = (%s, %t), want (0, false)", elapsed, active)
+	}
+
+	watchdog.Begin(now)
+	if elapsed, active := watchdog.Elapsed(now.Add(5 * time.Second)); !active || elapsed != 5*time.Second {
+		t.Fatalf("Elapsed() during write = (%s, %t), want (5s, true)", elapsed, active)
+	}
+
+	watchdog.End()
+	if elapsed, active := watchdog.Elapsed(now.Add(6 * time.Second)); active || elapsed != 0 {
+		t.Fatalf("Elapsed() after End() = (%s, %t), want (0, false)", elapsed, active)
+	}
+}

machinery/src/cloud/livemoq_disabled.go

@@ -2,7 +2,20 @@
 
 package cloud
 
-import "github.com/kerberos-io/agent/machinery/src/models"
+import (
+	"context"
+
+	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/packets"
+)
 
 // StartLiveStreamMoQ is disabled in the standard Agent build.
-func StartLiveStreamMoQ(_ *models.Configuration, _ *models.Communication, _ bool) {}
+func StartLiveStreamMoQ(
+	_ context.Context,
+	_ *models.Configuration,
+	_ *models.Communication,
+	_ bool,
+	_ *packets.Queue,
+	_ *packets.Queue,
+) {
+}

machinery/src/cloud/livemoq_enabled.go

@@ -4,41 +4,120 @@ package cloud
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"net/url"
 	"os"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/cloud/livemoq"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/moq-dev/moq-go/moq"
+	log "github.com/sirupsen/logrus"
 )
 
 const (
 	defaultMoQRelayURL      = "https://relay.uug.ai/anon"
 	minMoQRetryDelay        = time.Second
 	maxMoQRetryDelay        = 30 * time.Second
-	maxMoQLivePacketAge     = 1500 * time.Millisecond
+	defaultMoQLivePacketAge = 1500 * time.Millisecond
+	minMoQLivePacketAge     = 250 * time.Millisecond
+	maxMoQLivePacketAge     = 30 * time.Second
+	defaultMoQWriteTimeout  = 5 * time.Second
+	minMoQWriteTimeout      = time.Second
+	maxMoQWriteTimeout      = time.Minute
+	moQWriteWatchInterval   = 250 * time.Millisecond
 	slowMoQWriteThreshold   = 100 * time.Millisecond
 	moQWriteWarningInterval = 10 * time.Second
 	duplicateKeyframeWindow = 500 * time.Millisecond
 )
 
 type liveMoQConfig struct {
-	relayURL    string
-	broadcast   string
-	quality     string
-	sourceLabel string
-	queue       *packets.Queue
+	relayURL      string
+	broadcast     string
+	quality       string
+	sourceLabel   string
+	queue         *packets.Queue
+	communication *models.Communication
+	maxPacketAge  time.Duration
+	writeTimeout  time.Duration
 }
 
-// label identifies the tier in log lines, since one Agent runs a publisher per
-// quality tier.
-func (c liveMoQConfig) label() string {
-	return c.quality + " (" + c.sourceLabel + " stream)"
+func (c liveMoQConfig) logEntry(event string) *log.Entry {
+	return log.WithFields(log.Fields{
+		"component": "moq",
+		"event":     event,
+		"quality":   c.quality,
+		"stream":    c.sourceLabel,
+	})
+}
+
+func moQRelayHost(relayURL string) string {
+	parsed, err := url.Parse(relayURL)
+	if err != nil {
+		return ""
+	}
+	return parsed.Host
+}
+
+func (c liveMoQConfig) packetAgeLimit() time.Duration {
+	if c.maxPacketAge > 0 {
+		return c.maxPacketAge
+	}
+	return defaultMoQLivePacketAge
+}
+
+func (c liveMoQConfig) writeTimeoutLimit() time.Duration {
+	if c.writeTimeout > 0 {
+		return c.writeTimeout
+	}
+	return defaultMoQWriteTimeout
+}
+
+func boundedMoQDuration(name string, fallback, minimum, maximum time.Duration) time.Duration {
+	raw := strings.TrimSpace(os.Getenv(name))
+	if raw == "" {
+		return fallback
+	}
+
+	value, err := time.ParseDuration(raw)
+	if err != nil {
+		log.WithError(err).WithFields(log.Fields{
+			"component":        "moq",
+			"configured_value": raw,
+			"effective_value":  fallback.String(),
+			"event":            "duration_invalid",
+			"variable":         name,
+		}).Warn("Invalid MoQ duration; using default")
+		return fallback
+	}
+	if value < minimum {
+		log.WithFields(log.Fields{
+			"component":        "moq",
+			"configured_value": value.String(),
+			"effective_value":  minimum.String(),
+			"event":            "duration_clamped",
+			"minimum":          minimum.String(),
+			"variable":         name,
+		}).Warn("MoQ duration is below the supported minimum")
+		return minimum
+	}
+	if value > maximum {
+		log.WithFields(log.Fields{
+			"component":        "moq",
+			"configured_value": value.String(),
+			"effective_value":  maximum.String(),
+			"event":            "duration_clamped",
+			"maximum":          maximum.String(),
+			"variable":         name,
+		}).Warn("MoQ duration exceeds the supported maximum")
+		return maximum
+	}
+	return value
 }
 
 // StartLiveStreamMoQ starts the publisher only in the dedicated MoQ build and
@@ -50,20 +129,37 @@ func (c liveMoQConfig) label() string {
 // publishing each tier as its OWN broadcast (see livemoq.BroadcastPath): the
 // high tier from the camera's highest-resolution stream and the low tier from
 // its sub stream, so switching quality in the frontend is a resubscribe to the
-// other path. Each tier only uploads while it actually has subscribers, so the
-// second broadcast is close to free when nobody watches it.
-func StartLiveStreamMoQ(configuration *models.Configuration, communication *models.Communication, subStreamEnabled bool) {
+// other path. Each tier uploads every frame while watched and keyframes only
+// while idle, keeping the relay cache current without continuously sending the
+// full unused stream.
+func StartLiveStreamMoQ(
+	ctx context.Context,
+	configuration *models.Configuration,
+	communication *models.Communication,
+	subStreamEnabled bool,
+	mainQueue *packets.Queue,
+	subQueue *packets.Queue,
+) {
 	if os.Getenv("AGENT_LIVE_MOQ_ENABLED") != "true" {
 		return
 	}
 
 	config := configuration.Config
 	if config.Offline == "true" || config.Capture.Liveview == "false" {
-		log.Log.Info("cloud.StartLiveStreamMoQ(): disabled by Agent live-view configuration")
+		log.WithFields(log.Fields{
+			"component": "moq",
+			"event":     "publisher_disabled",
+			"liveview":  config.Capture.Liveview,
+			"offline":   config.Offline,
+		}).Debug("MoQ publisher disabled by Agent configuration")
 		return
 	}
 	if config.Key == "" {
-		log.Log.Warning("cloud.StartLiveStreamMoQ(): AGENT_KEY is required")
+		log.WithFields(log.Fields{
+			"component": "moq",
+			"event":     "publisher_configuration_invalid",
+			"variable":  "AGENT_KEY",
+		}).Warn("MoQ publisher requires an Agent key")
 		return
 	}
 
@@ -83,38 +179,51 @@ func StartLiveStreamMoQ(configuration *models.Configuration, communication *mode
 		relayURL = defaultMoQRelayURL
 	}
 	broadcastPrefix := os.Getenv("AGENT_LIVE_MOQ_BROADCAST_PREFIX")
+	maxPacketAge := boundedMoQDuration("AGENT_LIVE_MOQ_MAX_PACKET_AGE", defaultMoQLivePacketAge, minMoQLivePacketAge, maxMoQLivePacketAge)
+	writeTimeout := boundedMoQDuration("AGENT_LIVE_MOQ_WRITE_TIMEOUT", defaultMoQWriteTimeout, minMoQWriteTimeout, maxMoQWriteTimeout)
 
-	ctx := context.Background()
-	if communication.Context != nil {
-		ctx = *communication.Context
-	}
-
+	var publishers sync.WaitGroup
 	for _, quality := range qualities {
-		queue := communication.Queue
+		queue := mainQueue
 		sourceLabel := "main"
-		if models.SelectSubStreamForQuality(config, quality, subStreamEnabled) && communication.SubQueue != nil {
-			queue = communication.SubQueue
+		if models.SelectSubStreamForQuality(config, quality, subStreamEnabled) && subQueue != nil {
+			queue = subQueue
 			sourceLabel = "sub"
 		}
 		if queue == nil {
-			log.Log.Warning("cloud.StartLiveStreamMoQ(): packet queue for the " + quality + " tier is unavailable")
+			log.WithFields(log.Fields{
+				"component": "moq",
+				"event":     "packet_queue_unavailable",
+				"quality":   quality,
+			}).Warn("MoQ packet queue is unavailable")
 			continue
 		}
-		go runLiveStreamMoQ(ctx, liveMoQConfig{
-			relayURL:    relayURL,
-			broadcast:   livemoq.BroadcastPath(broadcastPrefix, config.Key, quality),
-			quality:     quality,
-			sourceLabel: sourceLabel,
-			queue:       queue,
-		})
+		publisherConfig := liveMoQConfig{
+			relayURL:      relayURL,
+			broadcast:     livemoq.BroadcastPath(broadcastPrefix, config.Key, quality),
+			quality:       quality,
+			sourceLabel:   sourceLabel,
+			queue:         queue,
+			communication: communication,
+			maxPacketAge:  maxPacketAge,
+			writeTimeout:  writeTimeout,
+		}
+		publishers.Add(1)
+		go func() {
+			defer publishers.Done()
+			runLiveStreamMoQ(ctx, publisherConfig)
+		}()
 	}
+	publishers.Wait()
 }
 
 func runLiveStreamMoQ(ctx context.Context, config liveMoQConfig) {
-	log.Log.Info(fmt.Sprintf(
-		"cloud.runLiveStreamMoQ(): publishing %s stream (quality=%s) to %s/%s",
-		config.sourceLabel, config.quality, strings.TrimRight(config.relayURL, "/"), config.broadcast,
-	))
+	config.logEntry("publisher_started").Info("MoQ publisher started")
+	config.logEntry("publisher_configuration").WithFields(log.Fields{
+		"max_packet_age_ms": config.packetAgeLimit().Milliseconds(),
+		"relay_host":        moQRelayHost(config.relayURL),
+		"write_timeout_ms":  config.writeTimeoutLimit().Milliseconds(),
+	}).Debug("MoQ publisher configuration")
 
 	retryDelay := minMoQRetryDelay
 	for ctx.Err() == nil {
@@ -123,7 +232,13 @@ func runLiveStreamMoQ(ctx context.Context, config liveMoQConfig) {
 		if ctx.Err() != nil {
 			return
 		}
-		log.Log.Warning("cloud.runLiveStreamMoQ(): publisher stopped: " + err.Error())
+		if config.communication != nil {
+			config.communication.RecordMoQReconnect(config.quality)
+		}
+		config.logEntry("publisher_reconnecting").WithError(err).WithFields(log.Fields{
+			"connected_duration_ms": time.Since(connectedAt).Milliseconds(),
+			"retry_delay_ms":        retryDelay.Milliseconds(),
+		}).Warn("MoQ publisher stopped; reconnecting")
 		if time.Since(connectedAt) >= time.Minute {
 			retryDelay = minMoQRetryDelay
 		}
@@ -150,42 +265,80 @@ func publishLiveStreamMoQ(ctx context.Context, config liveMoQConfig) error {
 		return fmt.Errorf("connect to relay: %w", err)
 	}
 	defer client.Close()
+	config.logEntry("relay_connected").WithField("relay_host", moQRelayHost(config.relayURL)).
+		Info("MoQ relay connected")
 
-	sessionCtx, cancelSessionWatch := context.WithCancel(ctx)
-	defer cancelSessionWatch()
+	publisherCtx, cancelPublisher := context.WithCancel(ctx)
+	defer cancelPublisher()
 	sessionClosed := make(chan error, 1)
+	sessionWatchDone := make(chan struct{})
 	go func() {
-		sessionClosed <- client.Session().Closed(sessionCtx)
+		defer close(sessionWatchDone)
+		sessionClosed <- client.Session().Closed(publisherCtx)
+		cancelPublisher()
 	}()
 
 	broadcast, err := client.CreateBroadcast(config.broadcast)
 	if err != nil {
 		return fmt.Errorf("create broadcast: %w", err)
 	}
-	defer broadcast.Finish()
+	var finishBroadcastOnce sync.Once
+	var finishBroadcastErr error
+	finishBroadcast := func() error {
+		finishBroadcastOnce.Do(func() {
+			finishBroadcastErr = broadcast.Finish()
+		})
+		return finishBroadcastErr
+	}
+	defer finishBroadcast()
 
 	stream, err := broadcast.PublishMedia("avc3", nil)
 	if err != nil {
 		return fmt.Errorf("create H.264 media stream: %w", err)
 	}
-	defer stream.Finish()
+	var finishStreamOnce sync.Once
+	var finishStreamErr error
+	finishStream := func() error {
+		finishStreamOnce.Do(func() {
+			finishStreamErr = stream.Finish()
+		})
+		return finishStreamErr
+	}
+	defer finishStream()
 
-	// Only upload while this tier is actually being watched. `publishing` starts
-	// true so the track becomes discoverable on the relay even before the first
-	// subscriber ever arrives; from the moment a viewer has attached once, the
-	// subscriber watcher takes over and idles the tier again when everybody left.
-	watchCtx, cancelWatch := context.WithCancel(ctx)
-	defer cancelWatch()
+	// Publish the complete stream while watched and only keyframes while idle.
+	// `publishing` starts true so the track becomes discoverable on the relay
+	// before the first subscriber; after that, the subscriber watcher selects the
+	// full-rate or keyframe-refresh mode.
 	publishing := &atomic.Bool{}
 	publishing.Store(true)
-	go watchLiveStreamMoQSubscribers(watchCtx, stream, publishing, config)
+	subscriberWatchDone := make(chan struct{})
+	go func() {
+		defer close(subscriberWatchDone)
+		watchLiveStreamMoQSubscribers(publisherCtx, stream, publishing, config)
+	}()
+	writeWatchdog := &livemoq.WriteWatchdog{}
+	writeWatchDone := make(chan struct{})
+	closePublisher := func() error {
+		return closeLiveMoQPublisher(finishStream, finishBroadcast, client.Close)
+	}
+	go func() {
+		defer close(writeWatchDone)
+		watchLiveStreamMoQWrites(publisherCtx, closePublisher, writeWatchdog, config)
+	}()
+	defer func() {
+		cancelPublisher()
+		<-writeWatchDone
+		<-subscriberWatchDone
+		<-sessionWatchDone
+	}()
 
 	cursor := config.queue.Latest()
 	gate := livemoq.FrameGate{}
+	audienceGate := livemoq.AudienceGate{}
 	deduplicator := livemoq.KeyframeDeduplicator{}
 	var lastSlowWriteWarning time.Time
 	var lastDuplicateKeyframeWarning time.Time
-	idle := false
 	for {
 		select {
 		case err := <-sessionClosed:
@@ -193,32 +346,44 @@ func publishLiveStreamMoQ(ctx context.Context, config liveMoQConfig) error {
 		default:
 		}
 
-		packet, err := cursor.ReadPacket()
+		packet, err := cursor.ReadPacketContext(publisherCtx)
 		if err != nil {
+			select {
+			case sessionErr := <-sessionClosed:
+				return fmt.Errorf("relay session closed: %w", sessionErr)
+			default:
+			}
+			if ctx.Err() != nil {
+				return ctx.Err()
+			}
 			return fmt.Errorf("read packet: %w", err)
 		}
-		if !publishing.Load() {
-			// Keep draining the cursor so we stay at the live edge, but publish
-			// nothing. The gate is closed so the next viewer resumes on a keyframe.
-			if !idle {
-				gate.Reset()
-				deduplicator.Reset()
-				idle = true
-			}
-			continue
-		}
-		idle = false
 		if !packet.IsVideo || len(packet.Data) == 0 || !strings.EqualFold(packet.Codec, "H264") {
 			continue
 		}
-		allowed, event := gate.Allow(packet.IsKeyFrame, packet.CurrentTime, time.Now(), maxMoQLivePacketAge)
+		allowedForAudience, enteredIdle := audienceGate.Allow(publishing.Load(), packet.IsKeyFrame)
+		if enteredIdle {
+			gate.Reset()
+			deduplicator.Reset()
+		}
+		if !allowedForAudience {
+			continue
+		}
+		allowed, event := gate.Allow(packet.IsKeyFrame, packet.CurrentTime, time.Now(), config.packetAgeLimit())
 		switch event {
 		case livemoq.FrameGateEventStarted:
-			log.Log.Info("cloud.publishLiveStreamMoQ(): first H.264 keyframe received; " + config.label() + " broadcast is live")
+			config.logEntry("broadcast_live").WithFields(log.Fields{
+				"codec":        packet.Codec,
+				"timestamp_ms": packet.Time,
+			}).Info("MoQ broadcast is live")
 		case livemoq.FrameGateEventLagging:
-			log.Log.Warning("cloud.publishLiveStreamMoQ(): " + config.label() + " stream is lagging; dropping packets until a recent keyframe")
+			config.logEntry("stream_lagging").WithFields(log.Fields{
+				"max_packet_age_ms": config.packetAgeLimit().Milliseconds(),
+				"timestamp_ms":      packet.Time,
+			}).Warn("MoQ stream is lagging; dropping packets until a recent keyframe")
 		case livemoq.FrameGateEventRecovered:
-			log.Log.Info("cloud.publishLiveStreamMoQ(): caught up with the " + config.label() + " live stream at a recent keyframe")
+			config.logEntry("stream_recovered").WithField("timestamp_ms", packet.Time).
+				Info("MoQ stream recovered at a recent keyframe")
 		}
 		if !allowed {
 			continue
@@ -228,18 +393,18 @@ func publishLiveStreamMoQ(ctx context.Context, config liveMoQConfig) error {
 			return fmt.Errorf("normalize H.264 access unit: %w", err)
 		}
 		if normalizationStats.DuplicateIDRNALUs > 0 && time.Since(lastDuplicateKeyframeWarning) >= moQWriteWarningInterval {
-			log.Log.Warning(fmt.Sprintf(
-				"cloud.publishLiveStreamMoQ(): %s removed %d duplicate IDR NALU(s) from H.264 keyframe (timestamp_ms=%d)",
-				config.label(), normalizationStats.DuplicateIDRNALUs, packet.Time,
-			))
+			config.logEntry("duplicate_idr_removed").WithFields(log.Fields{
+				"duplicate_idr_count": normalizationStats.DuplicateIDRNALUs,
+				"timestamp_ms":        packet.Time,
+			}).Warn("Removed duplicate IDR NAL units from MoQ keyframe")
+
 			lastDuplicateKeyframeWarning = time.Now()
 		}
 		if packet.IsKeyFrame && deduplicator.IsDuplicate(packet.Time, packet.CurrentTime, payload, time.Now(), duplicateKeyframeWindow) {
 			if time.Since(lastDuplicateKeyframeWarning) >= moQWriteWarningInterval {
-				log.Log.Warning(fmt.Sprintf(
-					"cloud.publishLiveStreamMoQ(): %s dropping duplicate H.264 keyframe (timestamp_ms=%d)",
-					config.label(), packet.Time,
-				))
+				config.logEntry("duplicate_keyframe_dropped").WithField("timestamp_ms", packet.Time).
+					Warn("Dropped duplicate MoQ keyframe")
+
 				lastDuplicateKeyframeWarning = time.Now()
 			}
 			continue
@@ -249,10 +414,16 @@ func publishLiveStreamMoQ(ctx context.Context, config liveMoQConfig) error {
 			TimestampUs: livemoq.TimestampUs(packet.Time),
 		}
 		writeStartedAt := time.Now()
-		if err := stream.WriteFrame(frame); err != nil {
+		writeWatchdog.Begin(writeStartedAt)
+		err = stream.WriteFrame(frame)
+		writeWatchdog.End()
+		if err != nil {
 			return fmt.Errorf("write H.264 access unit: %w", err)
 		}
 		writeDuration := time.Since(writeStartedAt)
+		if config.communication != nil {
+			config.communication.RecordMoQWrite(config.quality, writeDuration, time.Now())
+		}
 		if writeDuration >= slowMoQWriteThreshold && time.Since(lastSlowWriteWarning) >= moQWriteWarningInterval {
 			packetAge := time.Duration(0)
 			if packet.CurrentTime > 0 {
@@ -261,15 +432,58 @@ func publishLiveStreamMoQ(ctx context.Context, config liveMoQConfig) error {
 					packetAge = 0
 				}
 			}
-			log.Log.Warning(fmt.Sprintf(
-				"cloud.publishLiveStreamMoQ(): %s WriteFrame blocked for %s (packet_age=%s keyframe=%t)",
-				config.label(), writeDuration.Round(time.Millisecond), packetAge.Round(time.Millisecond), packet.IsKeyFrame,
-			))
+			config.logEntry("slow_frame_write").WithFields(log.Fields{
+				"duration_ms":   writeDuration.Milliseconds(),
+				"keyframe":      packet.IsKeyFrame,
+				"packet_age_ms": packetAge.Milliseconds(),
+			}).Warn("MoQ frame write was slow")
+
 			lastSlowWriteWarning = time.Now()
 		}
 	}
 }
 
+func closeLiveMoQPublisher(finishStream, finishBroadcast, closeClient func() error) error {
+	streamErr := finishStream()
+	broadcastErr := finishBroadcast()
+	clientErr := closeClient()
+	return errors.Join(streamErr, broadcastErr, clientErr)
+}
+
+func watchLiveStreamMoQWrites(ctx context.Context, closePublisher func() error, watchdog *livemoq.WriteWatchdog, config liveMoQConfig) {
+	ticker := time.NewTicker(moQWriteWatchInterval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			if err := closePublisher(); err != nil {
+				config.logEntry("publisher_close_failed").WithError(err).
+					Debug("MoQ publisher reported an error while closing")
+			}
+			return
+		case now := <-ticker.C:
+			writeDuration, active := watchdog.Elapsed(now)
+			if !active || writeDuration < config.writeTimeoutLimit() {
+				continue
+			}
+			config.logEntry("frame_write_timeout").WithFields(log.Fields{
+				"duration_ms": writeDuration.Milliseconds(),
+				"timeout_ms":  config.writeTimeoutLimit().Milliseconds(),
+			}).Warn("MoQ frame write timed out; reconnecting relay client")
+
+			if config.communication != nil {
+				config.communication.RecordMoQWriteTimeout()
+			}
+			if err := closePublisher(); err != nil {
+				config.logEntry("publisher_close_failed").WithError(err).
+					Warn("MoQ publisher reported an error while closing after a write timeout")
+			}
+			return
+		}
+	}
+}
+
 // watchLiveStreamMoQSubscribers flips the publisher between uploading and idling
 // as viewers subscribe to and leave this tier's broadcast. Used and Unused both
 // block, so they are followed from their own goroutine.
@@ -285,7 +499,7 @@ func watchLiveStreamMoQSubscribers(ctx context.Context, stream *moq.MediaProduce
 			return
 		}
 		if publishing.CompareAndSwap(false, true) {
-			log.Log.Info("cloud.watchLiveStreamMoQSubscribers(): viewer subscribed, resuming the " + config.label() + " broadcast")
+			config.logEntry("subscriber_joined").Info("MoQ subscriber joined; resuming broadcast")
 		}
 
 		if err := stream.Unused(ctx); err != nil {
@@ -293,6 +507,6 @@ func watchLiveStreamMoQSubscribers(ctx context.Context, stream *moq.MediaProduce
 			return
 		}
 		publishing.Store(false)
-		log.Log.Info("cloud.watchLiveStreamMoQSubscribers(): no viewers left, idling the " + config.label() + " broadcast")
+		config.logEntry("subscribers_idle").Info("MoQ broadcast idle; refreshing keyframes only")
 	}
 }

machinery/src/cloud/livemoq_enabled_test.go

@@ -0,0 +1,212 @@
+//go:build moq
+
+package cloud
+
+import (
+	"context"
+	"errors"
+	"reflect"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/kerberos-io/agent/machinery/src/cloud/livemoq"
+	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/moq-dev/moq-go/moq"
+)
+
+func TestCloseLiveMoQPublisherFinishesBeforeClosingClient(t *testing.T) {
+	var order []string
+	streamErr := errors.New("stream finish failed")
+	broadcastErr := errors.New("broadcast finish failed")
+	clientErr := errors.New("client close failed")
+
+	err := closeLiveMoQPublisher(
+		func() error {
+			order = append(order, "stream")
+			return streamErr
+		},
+		func() error {
+			order = append(order, "broadcast")
+			return broadcastErr
+		},
+		func() error {
+			order = append(order, "client")
+			return clientErr
+		},
+	)
+
+	if want := []string{"stream", "broadcast", "client"}; !reflect.DeepEqual(order, want) {
+		t.Fatalf("shutdown order = %v, want %v", order, want)
+	}
+	for _, want := range []error{streamErr, broadcastErr, clientErr} {
+		if !errors.Is(err, want) {
+			t.Errorf("shutdown error %v does not include %v", err, want)
+		}
+	}
+}
+
+func TestWatchLiveStreamMoQWritesClosesClientWhenContextEnds(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	closed := make(chan struct{})
+	var closeCalls atomic.Int32
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		watchLiveStreamMoQWrites(ctx, func() error {
+			if closeCalls.Add(1) == 1 {
+				close(closed)
+			}
+			return nil
+		}, &livemoq.WriteWatchdog{}, liveMoQConfig{quality: "low", sourceLabel: "sub"})
+	}()
+
+	cancel()
+	select {
+	case <-closed:
+	case <-time.After(time.Second):
+		t.Fatal("watchLiveStreamMoQWrites() did not close the client after cancellation")
+	}
+	<-done
+	if got := closeCalls.Load(); got != 1 {
+		t.Fatalf("close calls = %d, want 1", got)
+	}
+}
+
+func TestBoundedMoQDuration(t *testing.T) {
+	const name = "AGENT_LIVE_MOQ_TEST_DURATION"
+	tests := []struct {
+		name string
+		raw  string
+		want time.Duration
+	}{
+		{name: "unset", want: 5 * time.Second},
+		{name: "valid", raw: "12s", want: 12 * time.Second},
+		{name: "invalid", raw: "later", want: 5 * time.Second},
+		{name: "below minimum", raw: "100ms", want: time.Second},
+		{name: "above maximum", raw: "2m", want: time.Minute},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			t.Setenv(name, test.raw)
+			if got := boundedMoQDuration(name, 5*time.Second, time.Second, time.Minute); got != test.want {
+				t.Fatalf("boundedMoQDuration() = %s, want %s", got, test.want)
+			}
+		})
+	}
+}
+
+func TestWatchLiveStreamMoQWritesRecordsTimeout(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	closed := make(chan struct{})
+	communication := &models.Communication{}
+	watchdog := &livemoq.WriteWatchdog{}
+	watchdog.Begin(time.Now().Add(-time.Second))
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		watchLiveStreamMoQWrites(ctx, func() error {
+			close(closed)
+			return nil
+		}, watchdog, liveMoQConfig{
+			quality:       models.StreamQualityLow,
+			sourceLabel:   "sub",
+			communication: communication,
+			writeTimeout:  10 * time.Millisecond,
+		})
+	}()
+
+	select {
+	case <-closed:
+	case <-time.After(time.Second):
+		t.Fatal("watchLiveStreamMoQWrites() did not close the client after a write timeout")
+	}
+	<-done
+	if got := communication.RecoveryTelemetry().MoQWriteTimeouts; got != 1 {
+		t.Fatalf("MoQ write timeouts = %d, want 1", got)
+	}
+}
+
+func TestNativeMoQClientCloseStopsActiveWriter(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	server, err := moq.Listen(ctx, "127.0.0.1:0", moq.WithTLSGenerate("localhost"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	serveDone := make(chan error, 1)
+	go func() {
+		serveDone <- server.Serve(ctx)
+	}()
+
+	client, err := moq.Dial(ctx, "https://"+server.LocalAddr(), moq.WithTLSVerify(false))
+	if err != nil {
+		server.Close()
+		t.Fatal(err)
+	}
+	broadcast, err := client.CreateBroadcast("test/native-close")
+	if err != nil {
+		client.Close()
+		server.Close()
+		t.Fatal(err)
+	}
+	stream, err := broadcast.PublishMedia("avc3", nil)
+	if err != nil {
+		client.Close()
+		server.Close()
+		t.Fatal(err)
+	}
+
+	firstWrite := make(chan struct{})
+	writerDone := make(chan error, 1)
+	go func() {
+		payload := []byte{
+			0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x1e, 0xd9, 0x00, 0xa0, 0x47, 0xfe, 0xc8,
+			0x00, 0x00, 0x00, 0x01, 0x68, 0xce, 0x38, 0x80,
+			0x00, 0x00, 0x00, 0x01, 0x65, 0x88,
+		}
+		for timestamp := uint64(0); ; timestamp++ {
+			err := stream.WriteFrame(moq.Frame{Payload: payload, TimestampUs: timestamp})
+			if err != nil {
+				writerDone <- err
+				return
+			}
+			if timestamp == 0 {
+				close(firstWrite)
+			}
+		}
+	}()
+
+	select {
+	case <-firstWrite:
+	case <-ctx.Done():
+		t.Fatal("native MoQ writer did not start")
+	}
+	if err := stream.Finish(); err != nil {
+		t.Fatal(err)
+	}
+	if err := client.Close(); err != nil {
+		t.Fatal(err)
+	}
+	select {
+	case err := <-writerDone:
+		if err == nil {
+			t.Fatal("native MoQ writer stopped without an error after client close")
+		}
+	case <-ctx.Done():
+		t.Fatal("native MoQ client close did not stop the writer")
+	}
+
+	_ = broadcast.Finish()
+	_ = server.Close()
+	select {
+	case <-serveDone:
+	case <-ctx.Done():
+		t.Fatal("native MoQ server did not stop")
+	}
+}

machinery/src/cloud/livesnapshot/publisher.go

@@ -30,7 +30,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
+	log "github.com/sirupsen/logrus"
 )
 
 const (
@@ -131,7 +131,10 @@ func (p *Publisher) PublishSnapshot(ctx context.Context, jpeg []byte) error {
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
 		return fmt.Errorf("livesnapshot: upload snapshot rejected: %s", resp.Status)
 	}
-	log.Log.Debug("livesnapshot.Publisher.PublishSnapshot(): shipped preview frame for device " + p.cfg.DeviceKey)
+	log.WithFields(log.Fields{
+		"component": "livesnapshot",
+		"event":     "snapshot_published",
+	}).Debug("Live snapshot published")
 	return nil
 }
 

machinery/src/cloud/recording_metadata.go

@@ -15,6 +15,7 @@ import (
 const recordingFPSHeader = "X-Kerberos-Storage-Fps"
 const recordingDurationHeader = "X-Kerberos-Storage-Duration"
 const recordingTimestampHeader = "X-Kerberos-Storage-Timestamp"
+const recordingEncryptedHeader = "X-Kerberos-Storage-Encrypted"
 
 // queuedRecordingFPS reads the FPS snapshot written into the upload marker
 // when the recording was finalized. Historical empty markers intentionally
@@ -84,5 +85,8 @@ func setQueuedRecordingMetadataHeaders(header http.Header, fileName string) {
 		if metadata.Timestamp > 0 {
 			header.Set(recordingTimestampHeader, strconv.FormatInt(metadata.Timestamp, 10))
 		}
+		if metadata.Encrypted {
+			header.Set(recordingEncryptedHeader, "true")
+		}
 	}
 }

machinery/src/cloud/s3.go

@@ -9,9 +9,9 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/minio/minio-go/v6"
+	log "github.com/sirupsen/logrus"
 )
 
 func UploadS3(configuration *models.Configuration, fileName string) (bool, bool, error) {
@@ -29,7 +29,7 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
 
 	if config.S3 == nil {
 		errorMessage := "UploadS3: Uploading Failed, as no settings found"
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, false, errors.New(errorMessage)
 	}
 
@@ -49,14 +49,14 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
 	// Check if we have some credentials otherwise we abort the request.
 	if aws_access_key_id == "" || aws_secret_access_key == "" {
 		errorMessage := "UploadS3: Uploading Failed, as no credentials found"
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, false, errors.New(errorMessage)
 	}
 
 	s3Client, err := minio.NewWithRegion("s3.amazonaws.com", aws_access_key_id, aws_secret_access_key, true, aws_region)
 	if err != nil {
 		errorMessage := "UploadS3: " + err.Error()
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, true, errors.New(errorMessage)
 	}
 
@@ -74,7 +74,7 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
 	fileParts := strings.Split(fileName, "_")
 	if len(fileParts) == 1 {
 		errorMessage := "UploadS3: " + fileName + " is not a valid name."
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, true, errors.New(errorMessage)
 	}
 
@@ -85,7 +85,7 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
 	//numberOfChanges := fileParts[4]
 	token, _ := strconv.Atoi(fileParts[5])
 
-	log.Log.Info("UploadS3: Upload started for " + fileName)
+	log.Info("UploadS3: Upload started for " + fileName)
 	fullname := "data/recordings/" + fileName
 
 	file, err := os.OpenFile(fullname, os.O_RDWR, 0755)
@@ -95,14 +95,14 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
 
 	if err != nil {
 		errorMessage := "UploadS3: " + err.Error()
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, true, errors.New(errorMessage)
 	}
 
 	fileInfo, err := file.Stat()
 	if err != nil {
 		errorMessage := "UploadS3: " + err.Error()
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, true, errors.New(errorMessage)
 	}
 
@@ -128,10 +128,10 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
 
 	if err != nil {
 		errorMessage := "UploadS3: Uploading Failed, " + err.Error()
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, true, errors.New(errorMessage)
 	} else {
-		log.Log.Info("UploadS3: Upload Finished, file has been uploaded to bucket: " + strconv.FormatInt(n, 10))
+		log.Info("UploadS3: Upload Finished, file has been uploaded to bucket: " + strconv.FormatInt(n, 10))
 		return true, true, nil
 	}
 }

machinery/src/cloud/tus_client.go

@@ -15,8 +15,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 // tusResumableVersion is the tus protocol version implemented by this client.
@@ -90,7 +90,14 @@ func logTusUploadProgress(label string, offset, size int64, loggedBucket *int64)
 	if percent > 100 {
 		percent = 100
 	}
-	log.Log.Infof("%s: resumable upload progress %d%% (%d/%d bytes)", label, percent, offset, size)
+	log.WithFields(log.Fields{
+		"bytes_sent":  offset,
+		"bytes_total": size,
+		"component":   "tus",
+		"event":       "upload_progress",
+		"progress":    percent,
+		"target":      label,
+	}).Info("Resumable upload progress")
 }
 
 // tusHeaderFunc sets the authentication and routing headers required on every
@@ -121,7 +128,7 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
 	}
 	if ferr != nil {
 		msg := label + ": resumable upload failed, file doesn't exist anymore"
-		log.Log.Info(msg)
+		log.Info(msg)
 		// The file is gone, so the legacy path cannot help either. Report it as
 		// "supported" to avoid a pointless fallback attempt.
 		return false, false, true, "", errors.New(msg)
@@ -134,19 +141,6 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
 	size := info.Size()
 
 	client := newVaultHTTPClient(0)
-	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
-		if len(via) == 0 {
-			return nil
-		}
-		if req.URL.Host != via[0].URL.Host {
-			for k := range req.Header {
-				if strings.HasPrefix(http.CanonicalHeaderKey(k), "X-Kerberos-") {
-					req.Header.Del(k)
-				}
-			}
-		}
-		return nil
-	}
 
 	sidecar := tusSidecarPath(fileName, slot)
 	uploadURL := loadTusResumeState(sidecar, baseURL)
@@ -182,7 +176,7 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
 					// The vault does not implement tus; let the caller fall back.
 					return false, false, false, "", cerr
 				}
-				log.Log.Info(label + ": resumable create failed, " + cerr.Error())
+				log.Info(label + ": resumable create failed, " + cerr.Error())
 				tusBackoff(attempt)
 				continue
 			}
@@ -200,7 +194,7 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
 				uploadURL = ""
 				continue
 			}
-			log.Log.Info(label + ": resumable head failed, " + herr.Error())
+			log.Info(label + ": resumable head failed, " + herr.Error())
 			tusBackoff(attempt)
 			continue
 		}
@@ -253,9 +247,9 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
 				if status >= 400 {
 					// Definitive rejection (e.g. provider push failed during finalize).
 					// Re-evaluate via HEAD on the next iteration to decide retry/restart.
-					log.Log.Info(label + ": resumable patch rejected, " + perr.Error())
+					log.Info(label + ": resumable patch rejected, " + perr.Error())
 				} else {
-					log.Log.Info(label + ": resumable patch failed, " + perr.Error())
+					log.Info(label + ": resumable patch failed, " + perr.Error())
 				}
 				tusBackoff(attempt)
 				patchFailed = true
@@ -304,6 +298,10 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
 // X-Kerberos-Storage-* headers on every request and routing (directory/provider)
 // is additionally carried in the tus Upload-Metadata.
 func uploadVaultResumable(vault models.KStorage, publicKey, deviceKey, fileName, label, slot string) (bool, bool, bool, string, error) {
+	customHeaders, err := parseCustomVaultHeaders(vault.CustomHeaders)
+	if err != nil {
+		return false, false, true, "", err
+	}
 	baseURL := strings.TrimRight(vault.URI, "/") + tusUploadPath
 	metadataValues := map[string]string{
 		"filename":  fileName,
@@ -314,10 +312,22 @@ func uploadVaultResumable(vault models.KStorage, publicKey, deviceKey, fileName,
 		"cloudkey":  publicKey,
 		"fps":       queuedRecordingFPS(fileName),
 	}
+	if encodedHeaders, err := encodeCustomVaultHeaders(customHeaders); err != nil {
+		return false, false, true, "", err
+	} else if encodedHeaders != "" {
+		metadataValues["custom_headers"] = encodedHeaders
+	}
+	customRequestHeaders := http.Header{}
+	if err := setCustomVaultHeaders(customRequestHeaders, customHeaders); err != nil {
+		return false, false, true, "", err
+	}
 	addRecordingTusMetadata(metadataValues, fileName)
 	metadata := encodeTusMetadata(metadataValues)
 	setHeaders := func(h http.Header, fn string) {
 		setVaultTusHeaders(h, vault, publicKey, deviceKey, fn)
+		for name, values := range customRequestHeaders {
+			h[name] = append([]string(nil), values...)
+		}
 	}
 	return runTusUpload(baseURL, metadata, fileName, label, slot, setHeaders)
 }
@@ -354,6 +364,9 @@ func addRecordingTusMetadata(values map[string]string, fileName string) {
 	if metadata.Timestamp > 0 {
 		values["timestamp"] = strconv.FormatInt(metadata.Timestamp, 10)
 	}
+	if metadata.Encrypted {
+		values["encrypted"] = "true"
+	}
 }
 
 // tusCreate performs the tus "creation" request (POST). On success it returns

machinery/src/cloud/tus_client_test.go

@@ -3,6 +3,7 @@ package cloud
 import (
 	"bytes"
 	"encoding/base64"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net"
@@ -264,6 +265,36 @@ func testVault(uri string) models.KStorage {
 	}
 }
 
+func TestUploadKerberosVaultSkipsEmptyRecording(t *testing.T) {
+	fileName := "1787015373_3-654_office-camera17_0-0-0-0_-1_1960.mp4"
+	withRecording(t, fileName, nil)
+
+	requestCount := 0
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requestCount++
+		w.WriteHeader(http.StatusInternalServerError)
+	}))
+	defer server.Close()
+
+	vault := testVault(server.URL)
+	configuration := &models.Configuration{Config: models.Config{
+		Key:               "device-key",
+		KStorage:          &vault,
+		KStorageSecondary: &models.KStorage{},
+	}}
+
+	uploaded, configured, err := UploadKerberosVault(configuration, fileName)
+	if err != nil {
+		t.Fatalf("UploadKerberosVault() error = %v", err)
+	}
+	if uploaded || configured {
+		t.Fatalf("UploadKerberosVault() uploaded/configured = %v/%v, want false/false", uploaded, configured)
+	}
+	if requestCount != 0 {
+		t.Fatalf("Vault received %d requests, want 0", requestCount)
+	}
+}
+
 func TestUploadVaultResumable_HappyPath(t *testing.T) {
 	srv := newFakeTus()
 	ts := httptest.NewServer(srv)
@@ -274,7 +305,9 @@ func TestUploadVaultResumable_HappyPath(t *testing.T) {
 	withRecording(t, fileName, payload)
 	withQueuedRecordingFPS(t, fileName, `{"filename":"recording.mp4","device_key":"device-key","timestamp":1785934709414,"duration":20452,"fps":29.97}`)
 
-	uploaded, responded, supported, _, err := uploadVaultResumable(testVault(ts.URL), "pk", "dev", fileName, "test", "primary")
+	vault := testVault(ts.URL)
+	vault.CustomHeaders = `{"site_id":"site-1","equipment_id":"equipment-3"}`
+	uploaded, responded, supported, _, err := uploadVaultResumable(vault, "pk", "dev", fileName, "test", "primary")
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -298,6 +331,23 @@ func TestUploadVaultResumable_HappyPath(t *testing.T) {
 	if got := metadata["timestamp"]; got != "1785934709414" {
 		t.Fatalf("POST metadata timestamp = %q, want %q", got, "1785934709414")
 	}
+	var customHeaders map[string]string
+	if err := json.Unmarshal([]byte(metadata["custom_headers"]), &customHeaders); err != nil {
+		t.Fatal(err)
+	}
+	if customHeaders["site_id"] != "site-1" || customHeaders["equipment_id"] != "equipment-3" {
+		t.Fatalf("POST custom header metadata = %#v", customHeaders)
+	}
+	for _, method := range []string{http.MethodPost, http.MethodHead, http.MethodPatch} {
+		for _, request := range srv.requestsForMethod(method) {
+			if request.header.Get("site_id") != "site-1" || request.header.Get("equipment_id") != "equipment-3" {
+				t.Fatalf("%s custom headers = %#v", method, request.header)
+			}
+			if request.header.Get(customHeadersManifestHeader) == "" {
+				t.Fatalf("%s is missing the custom header manifest", method)
+			}
+		}
+	}
 }
 
 func TestQueuedRecordingFPSValidation(t *testing.T) {
@@ -356,7 +406,7 @@ func TestQueuedRecordingFPSAllowsMissingHistoricalMarker(t *testing.T) {
 func TestQueuedRecordingMetadataHeaders(t *testing.T) {
 	fileName := "recording.mp4"
 	withRecording(t, fileName, []byte("recording"))
-	withQueuedRecordingFPS(t, fileName, `{"filename":"recording.mp4","device_key":"device-key","timestamp":1785934709414,"duration":20452,"fps":25}`)
+	withQueuedRecordingFPS(t, fileName, `{"filename":"recording.mp4","device_key":"device-key","timestamp":1785934709414,"duration":20452,"fps":25,"encrypted":true}`)
 
 	header := make(http.Header)
 	setQueuedRecordingMetadataHeaders(header, fileName)
@@ -369,6 +419,14 @@ func TestQueuedRecordingMetadataHeaders(t *testing.T) {
 	if got := header.Get(recordingTimestampHeader); got != "1785934709414" {
 		t.Fatalf("timestamp header = %q", got)
 	}
+	if got := header.Get(recordingEncryptedHeader); got != "true" {
+		t.Fatalf("encrypted header = %q", got)
+	}
+	metadata := map[string]string{}
+	addRecordingTusMetadata(metadata, fileName)
+	if got := metadata["encrypted"]; got != "true" {
+		t.Fatalf("encrypted TUS metadata = %q", got)
+	}
 }
 
 func TestQueuedRecordingFPSAllowsLegacyMarkerFileName(t *testing.T) {

machinery/src/components/backchannel.go

@@ -3,17 +3,16 @@ package components
 import (
 	"bufio"
 	"context"
-	"fmt"
 	"math/rand"
 	"os"
 	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/capture"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/kerberos-io/joy4/av"
 	"github.com/pion/rtp"
+	log "github.com/sirupsen/logrus"
 	"github.com/zaf/g711"
 )
 
@@ -79,7 +78,7 @@ func GetBackChannelAudioCodec(streams []av.CodecData, communication *models.Comm
 			if stream.Type().String() == "PCM_MULAW" {
 				pcmuCodec := stream.(av.AudioCodecData)
 				if pcmuCodec.IsBackChannel() {
-					communication.HasBackChannel = true
+					communication.HasBackChannel.Store(true)
 					return pcmuCodec
 				}
 			}
@@ -88,22 +87,17 @@ func GetBackChannelAudioCodec(streams []av.CodecData, communication *models.Comm
 	return nil
 }
 
-func WriteAudioToBackchannel(communication *models.Communication, rtspClient capture.RTSPClient) {
-	ctx := context.Background()
-	if communication.Context != nil {
-		ctx = *communication.Context
-	}
-
+func WriteAudioToBackchannel(ctx context.Context, communication *models.Communication, rtspClient capture.RTSPClient) {
 	writeAudioToBackchannel(ctx, ctx, communication.HandleAudio, rtspClient)
 }
 
 func writeAudioToBackchannel(ctx context.Context, otelContext context.Context, audioChannel <-chan models.AudioDataPartial, rtspClient backchannelClient) {
-	log.Log.Info("Audio.WriteAudioToBackchannel(): writing to backchannel audio codec")
+	log.Info("Audio.WriteAudioToBackchannel(): writing to backchannel audio codec")
 
 	if err := rtspClient.StartBackChannel(ctx, otelContext); err != nil {
-		log.Log.Error("Audio.WriteAudioToBackchannel(): error starting backchannel: " + err.Error())
+		log.Error("Audio.WriteAudioToBackchannel(): error starting backchannel: " + err.Error())
 		if !reconnectBackchannel(ctx, otelContext, rtspClient) {
-			log.Log.Info("Audio.WriteAudioToBackchannel(): stopped while reconnecting")
+			log.Info("Audio.WriteAudioToBackchannel(): stopped while reconnecting")
 			return
 		}
 	}
@@ -112,11 +106,11 @@ func writeAudioToBackchannel(ctx context.Context, otelContext context.Context, a
 	for {
 		select {
 		case <-ctx.Done():
-			log.Log.Info("Audio.WriteAudioToBackchannel(): stopped")
+			log.Info("Audio.WriteAudioToBackchannel(): stopped")
 			return
 		case audio, ok := <-audioChannel:
 			if !ok {
-				log.Log.Info("Audio.WriteAudioToBackchannel(): finished")
+				log.Info("Audio.WriteAudioToBackchannel(): finished")
 				return
 			}
 
@@ -127,9 +121,9 @@ func writeAudioToBackchannel(ctx context.Context, otelContext context.Context, a
 
 			pkt := packetizer.packet(audio, time.Now())
 			if err := rtspClient.WritePacket(pkt); err != nil {
-				log.Log.Error("Audio.WriteAudioToBackchannel(): error writing packet to backchannel: " + err.Error())
+				log.Error("Audio.WriteAudioToBackchannel(): error writing packet to backchannel: " + err.Error())
 				if !reconnectBackchannel(ctx, otelContext, rtspClient) {
-					log.Log.Info("Audio.WriteAudioToBackchannel(): stopped while reconnecting")
+					log.Info("Audio.WriteAudioToBackchannel(): stopped while reconnecting")
 					return
 				}
 				packetizer = newBackchannelPacketizer()
@@ -137,7 +131,7 @@ func writeAudioToBackchannel(ctx context.Context, otelContext context.Context, a
 			}
 
 			if !waitForBackchannel(ctx, time.Duration(len(audio.Data))*time.Second/backchannelSampleRate) {
-				log.Log.Info("Audio.WriteAudioToBackchannel(): stopped")
+				log.Info("Audio.WriteAudioToBackchannel(): stopped")
 				return
 			}
 		}
@@ -162,7 +156,7 @@ func reconnectBackchannel(ctx context.Context, otelContext context.Context, rtsp
 	backoff := backchannelReconnectInitial
 	for {
 		if err := rtspClient.Close(otelContext); err != nil {
-			log.Log.Error("Audio.WriteAudioToBackchannel(): error closing failed backchannel: " + err.Error())
+			log.Error("Audio.WriteAudioToBackchannel(): error closing failed backchannel: " + err.Error())
 		}
 		if ctx.Err() != nil {
 			return false
@@ -173,11 +167,11 @@ func reconnectBackchannel(ctx context.Context, otelContext context.Context, rtsp
 			err = rtspClient.StartBackChannel(ctx, otelContext)
 		}
 		if err == nil {
-			log.Log.Info("Audio.WriteAudioToBackchannel(): reconnected backchannel")
+			log.Info("Audio.WriteAudioToBackchannel(): reconnected backchannel")
 			return true
 		}
 
-		log.Log.Error("Audio.WriteAudioToBackchannel(): error reconnecting backchannel: " + err.Error())
+		log.Error("Audio.WriteAudioToBackchannel(): error reconnecting backchannel: " + err.Error())
 		if !waitForBackchannel(ctx, backoff) {
 			return false
 		}
@@ -204,7 +198,12 @@ func WriteFileToBackChannel(infile av.DemuxCloser) {
 	// Do the warmup!
 	file, err := os.Open("./audiofile.bye")
 	if err != nil {
-		fmt.Println("WriteFileToBackChannel: error opening audiofile.bye file")
+		log.WithError(err).WithFields(log.Fields{
+			"component": "backchannel",
+			"event":     "audio_file_open_failed",
+			"path":      "./audiofile.bye",
+		}).Error("Failed to open backchannel audio file")
+		return
 	}
 	defer file.Close()
 

machinery/src/components/watchdog_test.go

@@ -0,0 +1,82 @@
+package components
+
+import (
+	"context"
+	"testing"
+	"time"
+)
+
+func TestWaitForRunRetryStopsOnCancellation(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	started := time.Now()
+	if waitForRunRetry(ctx) {
+		t.Fatal("waitForRunRetry() completed the retry delay after cancellation")
+	}
+	if elapsed := time.Since(started); elapsed > 100*time.Millisecond {
+		t.Fatalf("waitForRunRetry() took %s after cancellation", elapsed)
+	}
+}
+
+func TestStreamRestartWatchdogCoalescesStallsAndBacksOff(t *testing.T) {
+	now := time.Unix(1_000, 0)
+	watchdog := newStreamRestartWatchdog()
+	if _, restart := watchdog.Observe(now, 1, 1, true, false); restart {
+		t.Fatal("initial observation requested a restart")
+	}
+
+	for check := 1; check < streamWatchdogStallChecks; check++ {
+		now = now.Add(streamWatchdogInterval)
+		if _, restart := watchdog.Observe(now, 1, 1, true, false); restart {
+			t.Fatalf("check %d requested an early restart", check)
+		}
+	}
+	now = now.Add(streamWatchdogInterval)
+	reason, restart := watchdog.Observe(now, 1, 1, true, false)
+	if !restart || reason != "main and sub streams" {
+		t.Fatalf("Observe() = (%q, %t), want coalesced restart", reason, restart)
+	}
+
+	watchdog.MarkRestart(now)
+	if got := watchdog.Backoff(); got != 30*time.Second {
+		t.Fatalf("Backoff() = %s, want 30s", got)
+	}
+	for now = now.Add(streamWatchdogInterval); now.Before(watchdog.nextRestart); now = now.Add(streamWatchdogInterval) {
+		if _, restart := watchdog.Observe(now, 1, 1, true, false); restart {
+			t.Fatal("restart requested during cooldown")
+		}
+	}
+}
+
+func TestStreamRestartWatchdogResetsAfterHealthyMinute(t *testing.T) {
+	now := time.Unix(2_000, 0)
+	watchdog := newStreamRestartWatchdog()
+	watchdog.backoff = streamWatchdogMaxBackoff
+	watchdog.nextRestart = now.Add(streamWatchdogMaxBackoff)
+	watchdog.Observe(now, 1, 1, true, false)
+
+	for elapsed := streamWatchdogInterval; elapsed <= streamWatchdogHealthyReset+streamWatchdogInterval; elapsed += streamWatchdogInterval {
+		now = now.Add(streamWatchdogInterval)
+		watchdog.Observe(now, int64(elapsed), int64(elapsed), true, false)
+	}
+
+	if got := watchdog.Backoff(); got != streamWatchdogBaseBackoff {
+		t.Fatalf("Backoff() = %s, want %s", got, streamWatchdogBaseBackoff)
+	}
+	if !watchdog.nextRestart.IsZero() {
+		t.Fatalf("nextRestart = %s, want zero", watchdog.nextRestart)
+	}
+}
+
+func TestStreamRestartWatchdogPausesWhileConfiguring(t *testing.T) {
+	now := time.Unix(3_000, 0)
+	watchdog := newStreamRestartWatchdog()
+	watchdog.Observe(now, 1, 0, false, false)
+	for check := 0; check < streamWatchdogStallChecks+1; check++ {
+		now = now.Add(streamWatchdogInterval)
+		if _, restart := watchdog.Observe(now, 1, 0, false, true); restart {
+			t.Fatal("restart requested while configuring")
+		}
+	}
+}

machinery/src/computervision/main.go

@@ -8,14 +8,14 @@ import (
 	geo "github.com/kellydunn/golang-geo"
 	"github.com/kerberos-io/agent/machinery/src/capture"
 	"github.com/kerberos-io/agent/machinery/src/conditions"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
+	log "github.com/sirupsen/logrus"
 )
 
 func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Configuration, communication *models.Communication, mqttClient mqtt.Client, rtspClient capture.RTSPClient) {
 
-	log.Log.Debug("computervision.main.ProcessMotion(): start motion detection")
+	log.Debug("computervision.main.ProcessMotion(): start motion detection")
 	config := configuration.Config
 	loc, _ := time.LoadLocation(config.Timezone)
 
@@ -42,14 +42,14 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 
 	if continuousMode && !hasMotionRegion {
 
-		log.Log.Info("computervision.main.ProcessMotion(): continuous recording enabled and no motion region configured, so no motion detection required.")
+		log.Info("computervision.main.ProcessMotion(): continuous recording enabled and no motion region configured, so no motion detection required.")
 
 	} else {
 
 		if continuousMode {
-			log.Log.Info("computervision.main.ProcessMotion(): continuous recording enabled with a motion region, running motion detection for live-view visualisation only (no motion-triggered recording).")
+			log.Info("computervision.main.ProcessMotion(): continuous recording enabled with a motion region, running motion detection for live-view visualisation only (no motion-triggered recording).")
 		} else {
-			log.Log.Info("computervision.main.ProcessMotion(): motion detected is enabled, so starting the motion detection.")
+			log.Info("computervision.main.ProcessMotion(): motion detected is enabled, so starting the motion detection.")
 		}
 
 		hubKey := config.HubKey
@@ -185,7 +185,7 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 				// We'll validate those conditions and if not valid we'll not do anything.
 				detectMotion, err := conditions.Validate(loc, configuration)
 				if !detectMotion && err != nil {
-					log.Log.Debug("computervision.main.ProcessMotion(): " + err.Error() + ".")
+					log.Debug("computervision.main.ProcessMotion(): " + err.Error() + ".")
 				}
 
 				// Run detection when motion is enabled, OR when we're in continuous
@@ -235,7 +235,7 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 										if err == nil {
 											mqttClient.Publish("kerberos/hub/"+hubKey, 2, false, payload)
 										} else {
-											log.Log.Info("computervision.main.ProcessMotion(): failed to package MQTT message: " + err.Error())
+											log.Info("computervision.main.ProcessMotion(): failed to package MQTT message: " + err.Error())
 										}
 									} else {
 										mqttClient.Publish("kerberos/agent/"+deviceKey, 2, false, "motion")
@@ -254,7 +254,9 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 									NumberOfChanges: changesToReturn,
 									Rectangle:       motionRectangle,
 								}
-								communication.HandleMotion <- dataToPass //Save data to the channel
+								if !communication.TrySendMotion(dataToPass) {
+									log.Warn("computervision.main.ProcessMotion(): motion channel unavailable or full, dropping recording trigger")
+								}
 							}
 						}
 					}
@@ -271,7 +273,7 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 		}
 	}
 
-	log.Log.Debug("computervision.main.ProcessMotion(): stop the motion detection.")
+	log.Debug("computervision.main.ProcessMotion(): stop the motion detection.")
 }
 
 func FindMotion(imageArray [3]*image.Gray, coordinatesPerRegion [][]int, pixelChangeThreshold int) (thresholdReached bool, changesDetected int, motionRectangle models.MotionRectangle, motionRectangles []models.MotionRectangle) {
@@ -374,14 +376,28 @@ func AbsDiffBitwiseAndThreshold(img1 *image.Gray, img2 *image.Gray, img3 *image.
 				endY = pixel[1]
 			}
 		}
-		log.Log.Debugf("Rectangle of changes detected: startX: %d, startY: %d, endX: %d, endY: %d", startX, startY, endX, endY)
+		log.WithFields(log.Fields{
+			"component": "computer_vision",
+			"end_x":     endX,
+			"end_y":     endY,
+			"event":     "motion_bounds_detected",
+			"start_x":   startX,
+			"start_y":   startY,
+		}).Debug("Motion bounds detected")
 		motionRectangle = models.MotionRectangle{
 			X:      startX,
 			Y:      startY,
 			Width:  endX - startX,
 			Height: endY - startY,
 		}
-		log.Log.Debugf("Motion rectangle: %+v", motionRectangle)
+		log.WithFields(log.Fields{
+			"component": "computer_vision",
+			"event":     "motion_rectangle_created",
+			"height":    motionRectangle.Height,
+			"width":     motionRectangle.Width,
+			"x":         motionRectangle.X,
+			"y":         motionRectangle.Y,
+		}).Debug("Motion rectangle created")
 	}
 
 	// Cluster the changed pixels into separate bounding boxes so the live view can

machinery/src/conditions/timewindow.go

@@ -3,8 +3,8 @@ package conditions
 import (
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 func IsWithinTimeInterval(loc *time.Location, configuration *models.Configuration) (enabled bool) {
@@ -27,9 +27,9 @@ func IsWithinTimeInterval(loc *time.Location, configuration *models.Configuratio
 				currentTimeInSeconds := hour*60*60 + minute*60 + second
 				if (currentTimeInSeconds >= start1 && currentTimeInSeconds <= end1) ||
 					(currentTimeInSeconds >= start2 && currentTimeInSeconds <= end2) {
-					log.Log.Debug("conditions.timewindow.IsWithinTimeInterval(): time interval valid, enabling recording.")
+					log.Debug("conditions.timewindow.IsWithinTimeInterval(): time interval valid, enabling recording.")
 				} else {
-					log.Log.Info("conditions.timewindow.IsWithinTimeInterval(): time interval not valid, disabling recording.")
+					log.Info("conditions.timewindow.IsWithinTimeInterval(): time interval not valid, disabling recording.")
 					enabled = false
 				}
 			}

machinery/src/conditions/uri.go

@@ -3,13 +3,26 @@ package conditions
 import (
 	"bytes"
 	"crypto/tls"
-	"fmt"
+	"encoding/json"
+	"io"
 	"net/http"
 	"os"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
+)
+
+const conditionHTTPTimeout = 10 * time.Second
+
+var (
+	conditionHTTPClient         = &http.Client{Timeout: conditionHTTPTimeout}
+	conditionInsecureHTTPClient = &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // #nosec G402 -- explicit operator opt-in
+		},
+		Timeout: conditionHTTPTimeout,
+	}
 )
 
 func IsValidUriResponse(configuration *models.Configuration) (enabled bool) {
@@ -17,41 +30,66 @@ func IsValidUriResponse(configuration *models.Configuration) (enabled bool) {
 	conditionURI := config.ConditionURI
 	enabled = true
 	if conditionURI != "" {
-
-		// We will send a POST request to the conditionURI, and expect a 200 response.
-		// In the payload we will send some information, so the other end can decide
-		// if it should enable or disable recording.
-
-		var client *http.Client
+		client := conditionHTTPClient
 		if os.Getenv("AGENT_TLS_INSECURE") == "true" {
-			tr := &http.Transport{
-				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-			}
-			client = &http.Client{Transport: tr}
-		} else {
-			client = &http.Client{}
+			client = conditionInsecureHTTPClient
 		}
 
-		var object = fmt.Sprintf(`{
-			"camera_id" : "%s",
-			"camera_name" : "%s",
-			"site_id" : "%s",
-			"hub_key" : "%s",
-			"timestamp" : "%s",
-		}`, config.Key, config.FriendlyName, config.HubSite, config.HubKey, time.Now().Format("2006-01-02 15:04:05"))
+		payload := struct {
+			CameraID   string `json:"camera_id"`
+			CameraName string `json:"camera_name"`
+			SiteID     string `json:"site_id"`
+			HubKey     string `json:"hub_key"`
+			Timestamp  string `json:"timestamp"`
+		}{
+			CameraID:   config.Key,
+			CameraName: config.FriendlyName,
+			SiteID:     config.HubSite,
+			HubKey:     config.HubKey,
+			Timestamp:  time.Now().Format("2006-01-02 15:04:05"),
+		}
+		jsonBody, err := json.Marshal(payload)
+		if err != nil {
+			log.WithError(err).WithFields(log.Fields{
+				"component": "conditions/uri",
+				"event":     "request_encoding_failed",
+			}).Error("Failed to encode condition request")
+			return false
+		}
 
-		var jsonStr = []byte(object)
-		buffy := bytes.NewBuffer(jsonStr)
-		req, _ := http.NewRequest("POST", conditionURI, buffy)
+		req, err := http.NewRequest(http.MethodPost, conditionURI, bytes.NewReader(jsonBody))
+		if err != nil {
+			log.WithError(err).WithFields(log.Fields{
+				"component": "conditions/uri",
+				"event":     "request_creation_failed",
+			}).Error("Failed to create condition request")
+			return false
+		}
 		req.Header.Set("Content-Type", "application/json")
 		resp, err := client.Do(req)
 		if resp != nil {
+			_, _ = io.Copy(io.Discard, resp.Body)
 			resp.Body.Close()
 		}
-		if err == nil && resp.StatusCode == 200 {
-			log.Log.Info("conditions.uri.IsValidUriResponse(): response 200, enabling recording.")
+		if err == nil && resp != nil && resp.StatusCode == http.StatusOK {
+			log.WithFields(log.Fields{
+				"component":   "conditions/uri",
+				"event":       "recording_enabled",
+				"status_code": resp.StatusCode,
+			}).Info("Condition request enabled recording")
 		} else {
-			log.Log.Info("conditions.uri.IsValidUriResponse(): response not 200, disabling recording.")
+			if err != nil {
+				log.WithError(err).WithFields(log.Fields{
+					"component": "conditions/uri",
+					"event":     "request_failed",
+				}).Error("Condition request failed")
+			} else {
+				log.WithFields(log.Fields{
+					"component":   "conditions/uri",
+					"event":       "recording_disabled",
+					"status_code": resp.StatusCode,
+				}).Info("Condition request disabled recording")
+			}
 			enabled = false
 		}
 	}

machinery/src/conditions/uri_test.go

@@ -0,0 +1,63 @@
+package conditions
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+
+	"github.com/kerberos-io/agent/machinery/src/models"
+)
+
+func TestIsValidUriResponseReusesClientAndEncodesPayload(t *testing.T) {
+	var requests atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		requests.Add(1)
+		if request.Method != http.MethodPost {
+			t.Errorf("method = %s, want POST", request.Method)
+		}
+		if got := request.Header.Get("Content-Type"); got != "application/json" {
+			t.Errorf("Content-Type = %q, want application/json", got)
+		}
+		var payload struct {
+			CameraID   string `json:"camera_id"`
+			CameraName string `json:"camera_name"`
+		}
+		if err := json.NewDecoder(request.Body).Decode(&payload); err != nil {
+			t.Errorf("decode request: %v", err)
+		}
+		if payload.CameraID != "camera-1" || payload.CameraName != `Front "Door"` {
+			t.Errorf("payload = %+v", payload)
+		}
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer server.Close()
+
+	configuration := &models.Configuration{}
+	configuration.Config.ConditionURI = server.URL
+	configuration.Config.Key = "camera-1"
+	configuration.Config.FriendlyName = `Front "Door"`
+
+	for i := 0; i < 2; i++ {
+		if !IsValidUriResponse(configuration) {
+			t.Fatal("IsValidUriResponse() = false, want true")
+		}
+	}
+	if got := requests.Load(); got != 2 {
+		t.Fatalf("requests = %d, want 2", got)
+	}
+}
+
+func TestIsValidUriResponseRejectsNonOK(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		http.Error(w, "unavailable", http.StatusServiceUnavailable)
+	}))
+	defer server.Close()
+
+	configuration := &models.Configuration{}
+	configuration.Config.ConditionURI = server.URL
+	if IsValidUriResponse(configuration) {
+		t.Fatal("IsValidUriResponse() = true, want false")
+	}
+}

machinery/src/config/main.go

@@ -13,11 +13,36 @@ import (
 
 	"github.com/InVisionApp/conjungo"
 	"github.com/kerberos-io/agent/machinery/src/database"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
 )
 
+const factoryConfigRetryDelay = 5 * time.Second
+
+func newFactoryConfigReadContext() (context.Context, context.CancelFunc) {
+	return context.WithTimeout(context.Background(), database.TIMEOUT)
+}
+
+func isRetryableFactoryConfigReadError(err error) bool {
+	return errors.Is(err, context.DeadlineExceeded) || mongo.IsTimeout(err) || mongo.IsNetworkError(err)
+}
+
+func readFactoryConfig(collection *mongo.Collection, filter bson.M, destination *models.Config) error {
+	for {
+		ctx, cancel := newFactoryConfigReadContext()
+		err := collection.FindOne(ctx, filter).Decode(destination)
+		cancel()
+		if !isRetryableFactoryConfigReadError(err) {
+			return err
+		}
+
+		log.Warn("Factory configuration read timed out or lost its database connection; retrying.")
+		time.Sleep(factoryConfigRetryDelay)
+	}
+}
+
 // ReadUserConfig Reads the user configuration of the Kerberos Open Source instance.
 // This will return a models.User struct including the username, password,
 // selected language, and if the installation was completed or not.
@@ -25,14 +50,14 @@ func ReadUserConfig(configDirectory string) (userConfig models.User) {
 	for {
 		jsonFile, err := os.Open(configDirectory + "/data/config/user.json")
 		if err != nil {
-			log.Log.Error("Config file is not found " + configDirectory + "/data/config/user.json, trying again in 5s: " + err.Error())
+			log.Error("Config file is not found " + configDirectory + "/data/config/user.json, trying again in 5s: " + err.Error())
 			time.Sleep(5 * time.Second)
 		} else {
-			log.Log.Info("Successfully Opened user.json")
+			log.Info("Successfully Opened user.json")
 			byteValue, _ := ioutil.ReadAll(jsonFile)
 			err = json.Unmarshal(byteValue, &userConfig)
 			if err != nil {
-				log.Log.Error("JSON file not valid: " + err.Error())
+				log.Error("JSON file not valid: " + err.Error())
 			} else {
 				jsonFile.Close()
 				break
@@ -63,21 +88,15 @@ func OpenConfig(configDirectory string, configuration *models.Configuration) {
 		collection := db.Collection("configuration")
 
 		var globalConfig models.Config
-		res := collection.FindOne(context.Background(), bson.M{
+		err := readFactoryConfig(collection, bson.M{
 			"type": "global",
-		})
-
-		if res.Err() != nil {
-			log.Log.Error("Could not find global configuration, using default configuration.")
-			panic("Could not find global configuration, using default configuration.")
-		}
-		err := res.Decode(&globalConfig)
+		}, &globalConfig)
 		if err != nil {
-			log.Log.Error("Could not find global configuration, using default configuration.")
+			log.Error("Could not find global configuration, using default configuration.")
 			panic("Could not find global configuration, using default configuration.")
 		}
 		if globalConfig.Type != "global" {
-			log.Log.Error("Could not find global configuration, might missed the mongodb connection.")
+			log.Error("Could not find global configuration, might missed the mongodb connection.")
 			panic("Could not find global configuration, might missed the mongodb connection.")
 		}
 
@@ -85,21 +104,16 @@ func OpenConfig(configDirectory string, configuration *models.Configuration) {
 
 		var customConfig models.Config
 		deploymentName := os.Getenv("DEPLOYMENT_NAME")
-		res = collection.FindOne(context.Background(), bson.M{
+		err = readFactoryConfig(collection, bson.M{
 			"type": "config",
 			"name": deploymentName,
-		})
-		if res.Err() != nil {
-			log.Log.Error("Could not find configuration for " + deploymentName + ", using global configuration.")
-		}
-		err = res.Decode(&customConfig)
+		}, &customConfig)
 		if err != nil {
-			log.Log.Error("Could not find configuration for " + deploymentName + ", using global configuration.")
-		}
-
-		if customConfig.Type != "config" {
-			log.Log.Error("Could not find custom configuration, might missed the mongodb connection.")
-			panic("Could not find custom configuration, might missed the mongodb connection.")
+			log.Error("Could not find configuration for " + deploymentName + ", using global configuration.")
+			customConfig = models.Config{}
+		} else if customConfig.Type != "config" {
+			log.Error("Custom configuration has an invalid type, using global configuration.")
+			customConfig = models.Config{}
 		}
 		configuration.CustomConfig = customConfig
 
@@ -166,21 +180,29 @@ func OpenConfig(configDirectory string, configuration *models.Configuration) {
 
 		// Open device config
 		for {
-			jsonFile, err := os.Open(configDirectory + "/data/config/config.json")
+			configPath := configDirectory + "/data/config/config.json"
+			jsonFile, err := os.Open(configPath)
 			if err != nil {
-				log.Log.Error("Config file is not found " + configDirectory + "/data/config/config.json" + ", trying again in 5s.")
+				log.WithError(err).WithFields(log.Fields{
+					"component":      "config",
+					"event":          "configuration_open_failed",
+					"retry_delay_ms": 5000,
+				}).Warn("Configuration file unavailable; retrying")
 				time.Sleep(5 * time.Second)
 			} else {
-				log.Log.Info("Successfully Opened config.json from " + configuration.Name)
+				log.WithFields(log.Fields{
+					"component": "config",
+					"event":     "configuration_opened",
+				}).Info("Configuration file opened")
 				byteValue, _ := ioutil.ReadAll(jsonFile)
 				err = json.Unmarshal(byteValue, &configuration.Config)
 				jsonFile.Close()
 				if err != nil {
-					log.Log.Error("JSON file not valid: " + err.Error())
+					log.Error("JSON file not valid: " + err.Error())
 				} else {
 					err = json.Unmarshal(byteValue, &configuration.CustomConfig)
 					if err != nil {
-						log.Log.Error("JSON file not valid: " + err.Error())
+						log.Error("JSON file not valid: " + err.Error())
 					} else {
 						break
 					}
@@ -293,6 +315,15 @@ func initConfigPointers(config *models.Config) {
 func applyAgentEnvVars(configuration *models.Configuration, prefix string, applyDefaults bool) {
 	environmentVariables := os.Environ()
 
+	if prefix == "" {
+		if deploymentName := os.Getenv("DEPLOYMENT_NAME"); deploymentName != "" {
+			configuration.Config.Name = deploymentName
+			if configuration.Config.FriendlyName == "" {
+				configuration.Config.FriendlyName = deploymentName
+			}
+		}
+	}
+
 	// Initialize the configuration for some new fields.
 	if configuration.Config.KStorageSecondary == nil {
 		configuration.Config.KStorageSecondary = &models.KStorage{}
@@ -581,6 +612,9 @@ func applyAgentEnvVars(configuration *models.Configuration, prefix string, apply
 			case "AGENT_KERBEROSVAULT_DIRECTORY":
 				configuration.Config.KStorage.Directory = value
 				break
+			case "AGENT_KERBEROSVAULT_CUSTOM_HEADERS":
+				configuration.Config.KStorage.CustomHeaders = value
+				break
 
 			/* Retry policy and timeout */
 			case "AGENT_KERBEROSVAULT_MAX_RETRIES":
@@ -612,6 +646,9 @@ func applyAgentEnvVars(configuration *models.Configuration, prefix string, apply
 			case "AGENT_KERBEROSVAULT_SECONDARY_DIRECTORY":
 				configuration.Config.KStorageSecondary.Directory = value
 				break
+			case "AGENT_KERBEROSVAULT_SECONDARY_CUSTOM_HEADERS":
+				configuration.Config.KStorageSecondary.CustomHeaders = value
+				break
 
 			/* When storing in dropbox */
 			case "AGENT_DROPBOX_ACCESS_TOKEN":
@@ -687,12 +724,12 @@ func SaveConfig(configDirectory string, config models.Config, configuration *mod
 			return err
 		}
 
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			select {
 			case communication.HandleBootstrap <- "restart":
-				log.Log.Info("config.main.SaveConfig(): update config, restart agent.")
+				log.Info("config.main.SaveConfig(): update config, restart agent.")
 			case <-time.After(1 * time.Second):
-				log.Log.Info("config.main.SaveConfig(): update config, restart agent.")
+				log.Info("config.main.SaveConfig(): update config, restart agent.")
 			}
 		}
 

machinery/src/config/main_test.go

@@ -1,11 +1,64 @@
 package config
 
 import (
+	"context"
+	"errors"
+	"os"
 	"testing"
+	"time"
 
+	"github.com/kerberos-io/agent/machinery/src/database"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	"go.mongodb.org/mongo-driver/mongo"
 )
 
+func TestApplyAgentEnvVarsDeploymentName(t *testing.T) {
+	tests := []struct {
+		name           string
+		prefix         string
+		deploymentName string
+		agentName      string
+		initialName    string
+		initialDisplay string
+		wantName       string
+		wantDisplay    string
+	}{
+		{name: "deployment fallback", deploymentName: "camera-1", wantName: "camera-1", wantDisplay: "camera-1"},
+		{name: "explicit display name", deploymentName: "camera-1", agentName: "Front Door", wantName: "camera-1", wantDisplay: "Front Door"},
+		{name: "existing display name", deploymentName: "camera-1", initialDisplay: "Front Door", wantName: "camera-1", wantDisplay: "Front Door"},
+		{name: "deployment overrides bundled identity", deploymentName: "camera-1", initialName: "default", wantName: "camera-1", wantDisplay: "camera-1"},
+		{name: "no deployment", initialName: "existing", initialDisplay: "Existing Camera", wantName: "existing", wantDisplay: "Existing Camera"},
+		{name: "standalone display name", agentName: "Front Door", wantDisplay: "Front Door"},
+		{name: "global layer excludes deployment", prefix: "GLOBAL_", deploymentName: "camera-1"},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			t.Setenv("DEPLOYMENT_NAME", test.deploymentName)
+			t.Setenv(test.prefix+"AGENT_NAME", test.agentName)
+			if test.agentName == "" {
+				if err := os.Unsetenv(test.prefix + "AGENT_NAME"); err != nil {
+					t.Fatal(err)
+				}
+			}
+			configuration := &models.Configuration{Config: models.Config{
+				Name:         test.initialName,
+				FriendlyName: test.initialDisplay,
+			}}
+			initConfigPointers(&configuration.Config)
+
+			applyAgentEnvVars(configuration, test.prefix, false)
+
+			if got := configuration.Config.Name; got != test.wantName {
+				t.Errorf("Name = %q, want %q", got, test.wantName)
+			}
+			if got := configuration.Config.FriendlyName; got != test.wantDisplay {
+				t.Errorf("FriendlyName = %q, want %q", got, test.wantDisplay)
+			}
+		})
+	}
+}
+
 func TestApplyAgentEnvVarsPixelChangeThresholdDefault(t *testing.T) {
 	tests := []struct {
 		name      string
@@ -35,6 +88,55 @@ func TestApplyAgentEnvVarsPixelChangeThresholdDefault(t *testing.T) {
 	}
 }
 
+func TestApplyAgentEnvVarsVaultCustomHeaders(t *testing.T) {
+	t.Setenv("AGENT_KERBEROSVAULT_CUSTOM_HEADERS", `{"site_id":"site-1"}`)
+	t.Setenv("AGENT_KERBEROSVAULT_SECONDARY_CUSTOM_HEADERS", `{"site_id":"site-2"}`)
+	configuration := &models.Configuration{}
+	initConfigPointers(&configuration.Config)
+
+	applyAgentEnvVars(configuration, "", false)
+
+	if got := configuration.Config.KStorage.CustomHeaders; got != `{"site_id":"site-1"}` {
+		t.Fatalf("primary custom headers = %q", got)
+	}
+	if got := configuration.Config.KStorageSecondary.CustomHeaders; got != `{"site_id":"site-2"}` {
+		t.Fatalf("secondary custom headers = %q", got)
+	}
+}
+
 func intPointer(value int) *int {
 	return &value
 }
+
+func TestNewFactoryConfigReadContextUsesDatabaseTimeout(t *testing.T) {
+	ctx, cancel := newFactoryConfigReadContext()
+	defer cancel()
+
+	deadline, ok := ctx.Deadline()
+	if !ok {
+		t.Fatal("expected read context deadline")
+	}
+
+	remaining := time.Until(deadline)
+	if remaining <= 0 {
+		t.Fatalf("deadline already expired: %v", remaining)
+	}
+	if remaining > database.TIMEOUT {
+		t.Fatalf("remaining deadline = %v, want <= %v", remaining, database.TIMEOUT)
+	}
+	if remaining < database.TIMEOUT-time.Second {
+		t.Fatalf("remaining deadline = %v, want close to %v", remaining, database.TIMEOUT)
+	}
+}
+
+func TestFactoryConfigRetryableErrors(t *testing.T) {
+	if !isRetryableFactoryConfigReadError(context.DeadlineExceeded) {
+		t.Fatal("context deadline should be retryable")
+	}
+	if isRetryableFactoryConfigReadError(mongo.ErrNoDocuments) {
+		t.Fatal("missing configuration should not retry")
+	}
+	if isRetryableFactoryConfigReadError(errors.New("invalid BSON")) {
+		t.Fatal("decode errors should not retry")
+	}
+}

machinery/src/database/main.go

@@ -7,6 +7,7 @@ import (
 	"sync"
 	"time"
 
+	log "github.com/sirupsen/logrus"
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
 )
@@ -44,18 +45,34 @@ func New() *DB {
 		// We can also apply the complete URI
 		// e.g. "mongodb+srv://<username>:<password>@kerberos-hub.shhng.mongodb.net/?retryWrites=true&w=majority&appName=kerberos-hub"
 		if mongodbURI != "" {
+			log.WithFields(log.Fields{
+				"component":      "database",
+				"database":       DatabaseName,
+				"event":          "client_configuring",
+				"uri_configured": true,
+			}).Debug("Configuring MongoDB client")
 			serverAPI := options.ServerAPI(options.ServerAPIVersion1)
 			opts := options.Client().ApplyURI(mongodbURI).SetServerAPIOptions(serverAPI)
 
 			// Create a new client and connect to the server
 			client, err := mongo.Connect(ctx, opts)
 			if err != nil {
-				fmt.Printf("Error setting up mongodb connection: %+v\n", err)
-				os.Exit(1)
+				log.WithError(err).WithFields(log.Fields{
+					"component": "database",
+					"database":  DatabaseName,
+					"event":     "client_configuration_failed",
+				}).Fatal("Failed to configure MongoDB client")
 			}
 			_instance.Client = client
 
 		} else {
+			log.WithFields(log.Fields{
+				"component":             "database",
+				"database":              DatabaseName,
+				"event":                 "client_configuring",
+				"replicaset_configured": replicaset != "",
+				"uri_configured":        false,
+			}).Debug("Configuring MongoDB client")
 
 			// New MongoDB driver
 			mongodbURI := fmt.Sprintf("mongodb://%s:%s@%s", username, password, host)
@@ -69,8 +86,11 @@ func New() *DB {
 				Password:      password,
 			}))
 			if err != nil {
-				fmt.Printf("Error setting up mongodb connection: %+v\n", err)
-				os.Exit(1)
+				log.WithError(err).WithFields(log.Fields{
+					"component": "database",
+					"database":  DatabaseName,
+					"event":     "client_configuration_failed",
+				}).Fatal("Failed to configure MongoDB client")
 			}
 			_instance.Client = client
 		}

machinery/src/lifecycle/supervisor.go

@@ -0,0 +1,336 @@
+package lifecycle
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"runtime/debug"
+	"sort"
+	"sync"
+	"time"
+)
+
+var (
+	ErrSupervisorSealed  = errors.New("supervisor is sealed")
+	ErrSupervisorStopped = errors.New("supervisor is shutting down")
+	ErrTaskExists        = errors.New("task already exists")
+)
+
+type TaskFunc func(context.Context) error
+
+type TaskPolicy struct {
+	Required    bool
+	LongRunning bool
+}
+
+type TaskStatus string
+
+const (
+	TaskRunning   TaskStatus = "running"
+	TaskSucceeded TaskStatus = "succeeded"
+	TaskFailed    TaskStatus = "failed"
+	TaskPanicked  TaskStatus = "panicked"
+)
+
+type TaskSnapshot struct {
+	Name      string
+	Policy    TaskPolicy
+	Status    TaskStatus
+	StartedAt time.Time
+	EndedAt   time.Time
+	Error     string
+	Panic     string
+	Stack     []byte
+}
+
+type Failure struct {
+	Task  string
+	Cause error
+	Panic string
+	Stack []byte
+}
+
+func (f Failure) Error() string {
+	if f.Cause == nil {
+		return fmt.Sprintf("task %q failed", f.Task)
+	}
+	return f.Cause.Error()
+}
+
+type ShutdownReport struct {
+	Complete bool
+	Cause    error
+	Tasks    []TaskSnapshot
+	Running  []TaskSnapshot
+}
+
+type supervisorPhase uint8
+
+const (
+	phaseStarting supervisorPhase = iota
+	phaseRunning
+	phaseDraining
+)
+
+type taskState struct {
+	TaskSnapshot
+}
+
+type Supervisor struct {
+	ctx    context.Context
+	cancel context.CancelCauseFunc
+
+	mu        sync.Mutex
+	phase     supervisorPhase
+	sealed    bool
+	active    int
+	tasks     map[string]*taskState
+	allDone   chan struct{}
+	doneOnce  sync.Once
+	start     chan struct{}
+	startOnce sync.Once
+
+	failures    chan Failure
+	failureOnce sync.Once
+
+	stopParentCancel func() bool
+}
+
+func NewSupervisor(parent context.Context) *Supervisor {
+	if parent == nil {
+		parent = context.Background()
+	}
+
+	ctx, cancel := context.WithCancelCause(parent)
+	supervisor := &Supervisor{
+		ctx:      ctx,
+		cancel:   cancel,
+		phase:    phaseStarting,
+		tasks:    make(map[string]*taskState),
+		allDone:  make(chan struct{}),
+		start:    make(chan struct{}),
+		failures: make(chan Failure, 1),
+	}
+
+	stopParentCancel := context.AfterFunc(parent, func() {
+		supervisor.BeginShutdown(context.Cause(parent))
+	})
+	supervisor.mu.Lock()
+	if supervisor.sealed && supervisor.active == 0 {
+		supervisor.mu.Unlock()
+		stopParentCancel()
+	} else {
+		supervisor.stopParentCancel = stopParentCancel
+		supervisor.mu.Unlock()
+	}
+
+	return supervisor
+}
+
+func (s *Supervisor) Context() context.Context {
+	return s.ctx
+}
+
+func (s *Supervisor) Failures() <-chan Failure {
+	return s.failures
+}
+
+func (s *Supervisor) Go(name string, policy TaskPolicy, task TaskFunc) error {
+	if name == "" {
+		return errors.New("task name is required")
+	}
+	if task == nil {
+		return errors.New("task function is required")
+	}
+
+	s.mu.Lock()
+	if s.sealed {
+		s.mu.Unlock()
+		return fmt.Errorf("%w: %s", ErrSupervisorSealed, name)
+	}
+	if s.ctx.Err() != nil {
+		s.mu.Unlock()
+		return fmt.Errorf("%w: %s", ErrSupervisorStopped, name)
+	}
+	if _, exists := s.tasks[name]; exists {
+		s.mu.Unlock()
+		return fmt.Errorf("%w: %s", ErrTaskExists, name)
+	}
+
+	state := &taskState{TaskSnapshot: TaskSnapshot{
+		Name:      name,
+		Policy:    policy,
+		Status:    TaskRunning,
+		StartedAt: time.Now(),
+	}}
+	s.tasks[name] = state
+	s.active++
+	s.mu.Unlock()
+
+	go s.run(state, task)
+	return nil
+}
+
+func (s *Supervisor) run(state *taskState, task TaskFunc) {
+	var taskErr error
+	panicked := true
+	var panicValue string
+	var panicStack []byte
+
+	defer func() {
+		recovered := recover()
+		if panicked {
+			panicValue = fmt.Sprint(recovered)
+			panicStack = debug.Stack()
+			taskErr = fmt.Errorf("task %q panicked: %s", state.Name, panicValue)
+		}
+		s.finish(state, taskErr, panicked, panicValue, panicStack)
+	}()
+
+	select {
+	case <-s.start:
+	case <-s.ctx.Done():
+		panicked = false
+		return
+	}
+
+	taskErr = task(s.ctx)
+	panicked = false
+}
+
+func (s *Supervisor) finish(
+	state *taskState,
+	taskErr error,
+	panicked bool,
+	panicValue string,
+	panicStack []byte,
+) {
+	var failure *Failure
+
+	s.mu.Lock()
+	state.EndedAt = time.Now()
+	state.Error = ""
+	state.Panic = panicValue
+	state.Stack = append([]byte(nil), panicStack...)
+	switch {
+	case panicked:
+		state.Status = TaskPanicked
+		state.Error = taskErr.Error()
+	case taskErr != nil:
+		state.Status = TaskFailed
+		state.Error = taskErr.Error()
+	default:
+		state.Status = TaskSucceeded
+	}
+
+	shuttingDown := s.phase == phaseDraining || s.ctx.Err() != nil
+	unexpected := panicked || (!shuttingDown &&
+		(state.Policy.Required && (taskErr != nil || state.Policy.LongRunning)))
+	if unexpected {
+		cause := taskErr
+		if cause == nil {
+			cause = fmt.Errorf("required long-running task %q exited", state.Name)
+		}
+		failure = &Failure{
+			Task:  state.Name,
+			Cause: cause,
+			Panic: panicValue,
+			Stack: append([]byte(nil), panicStack...),
+		}
+		s.phase = phaseDraining
+		s.sealed = true
+		s.cancel(failure.Cause)
+		s.failureOnce.Do(func() {
+			s.failures <- *failure
+		})
+	}
+
+	s.active--
+	s.closeDoneIfReadyLocked()
+	s.mu.Unlock()
+}
+
+func (s *Supervisor) Seal() {
+	s.mu.Lock()
+	s.sealed = true
+	if s.phase == phaseStarting {
+		s.phase = phaseRunning
+	}
+	s.startOnce.Do(func() {
+		close(s.start)
+	})
+	s.closeDoneIfReadyLocked()
+	s.mu.Unlock()
+}
+
+func (s *Supervisor) BeginShutdown(cause error) {
+	if cause == nil {
+		cause = context.Canceled
+	}
+
+	s.mu.Lock()
+	s.phase = phaseDraining
+	s.sealed = true
+	s.cancel(cause)
+	s.closeDoneIfReadyLocked()
+	s.mu.Unlock()
+}
+
+func (s *Supervisor) Wait(ctx context.Context) ShutdownReport {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+
+	select {
+	case <-s.allDone:
+	case <-ctx.Done():
+	}
+
+	return s.report()
+}
+
+func (s *Supervisor) Snapshot() []TaskSnapshot {
+	return s.report().Tasks
+}
+
+func (s *Supervisor) closeDoneIfReadyLocked() {
+	if !s.sealed || s.active != 0 {
+		return
+	}
+	if s.stopParentCancel != nil {
+		s.stopParentCancel()
+		s.stopParentCancel = nil
+	}
+	s.doneOnce.Do(func() {
+		close(s.allDone)
+	})
+}
+
+func (s *Supervisor) report() ShutdownReport {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	tasks := make([]TaskSnapshot, 0, len(s.tasks))
+	running := make([]TaskSnapshot, 0)
+	for _, state := range s.tasks {
+		snapshot := state.TaskSnapshot
+		snapshot.Stack = append([]byte(nil), state.Stack...)
+		tasks = append(tasks, snapshot)
+		if snapshot.Status == TaskRunning {
+			running = append(running, snapshot)
+		}
+	}
+	sort.Slice(tasks, func(i, j int) bool {
+		return tasks[i].Name < tasks[j].Name
+	})
+	sort.Slice(running, func(i, j int) bool {
+		return running[i].Name < running[j].Name
+	})
+
+	return ShutdownReport{
+		Complete: s.sealed && s.active == 0,
+		Cause:    context.Cause(s.ctx),
+		Tasks:    tasks,
+		Running:  running,
+	}
+}

machinery/src/lifecycle/supervisor_test.go

@@ -0,0 +1,300 @@
+package lifecycle
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+func TestSupervisorWaitsForSeal(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	started := make(chan struct{})
+	if err := supervisor.Go("quick", TaskPolicy{}, func(context.Context) error {
+		close(started)
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	waitContext, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+	defer cancel()
+	if report := supervisor.Wait(waitContext); report.Complete {
+		t.Fatal("Wait() completed before Seal()")
+	}
+	select {
+	case <-started:
+		t.Fatal("task started before Seal()")
+	default:
+	}
+
+	supervisor.Seal()
+	<-started
+	if report := supervisor.Wait(context.Background()); !report.Complete {
+		t.Fatal("Wait() did not complete after Seal()")
+	}
+}
+
+func TestSupervisorSealsEmptyGroup(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	supervisor.Seal()
+
+	report := supervisor.Wait(context.Background())
+	if !report.Complete || len(report.Tasks) != 0 {
+		t.Fatalf("Wait() report = %+v, want complete empty report", report)
+	}
+}
+
+func TestSupervisorReportsRunningTasksOnTimeout(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	release := make(chan struct{})
+	if err := supervisor.Go("blocked", TaskPolicy{}, func(context.Context) error {
+		<-release
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+
+	waitContext, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+	defer cancel()
+	report := supervisor.Wait(waitContext)
+	if report.Complete {
+		t.Fatal("Wait() unexpectedly completed")
+	}
+	if len(report.Running) != 1 || report.Running[0].Name != "blocked" {
+		t.Fatalf("Wait() running tasks = %+v, want blocked", report.Running)
+	}
+
+	close(release)
+	if report = supervisor.Wait(context.Background()); !report.Complete {
+		t.Fatal("Wait() did not complete after releasing task")
+	}
+}
+
+func TestSupervisorFailsWhenRequiredLongRunningTaskExits(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	release := make(chan struct{})
+	if err := supervisor.Go("recorder", TaskPolicy{Required: true, LongRunning: true}, func(context.Context) error {
+		<-release
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+	close(release)
+
+	select {
+	case failure := <-supervisor.Failures():
+		if failure.Task != "recorder" || !strings.Contains(failure.Error(), "exited") {
+			t.Fatalf("failure = %+v", failure)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("required task exit did not emit a failure")
+	}
+}
+
+func TestSupervisorAllowsRequiredOneShotTaskToSucceed(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	if err := supervisor.Go("setup", TaskPolicy{Required: true}, func(context.Context) error {
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+	if report := supervisor.Wait(context.Background()); !report.Complete {
+		t.Fatal("Wait() did not complete")
+	}
+
+	select {
+	case failure := <-supervisor.Failures():
+		t.Fatalf("successful one-shot task emitted failure: %+v", failure)
+	default:
+	}
+}
+
+func TestSupervisorFailsWhenRequiredOneShotTaskReturnsError(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	wantErr := errors.New("setup failed")
+	if err := supervisor.Go("setup", TaskPolicy{Required: true}, func(context.Context) error {
+		return wantErr
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+
+	select {
+	case failure := <-supervisor.Failures():
+		if !errors.Is(failure.Cause, wantErr) {
+			t.Fatalf("failure cause = %v, want %v", failure.Cause, wantErr)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("required task error did not emit a failure")
+	}
+}
+
+func TestSupervisorRecoversPanicAndCapturesStack(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	if err := supervisor.Go("panic", TaskPolicy{}, func(context.Context) error {
+		panic("boom")
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+
+	select {
+	case failure := <-supervisor.Failures():
+		if failure.Task != "panic" || failure.Panic != "boom" || len(failure.Stack) == 0 {
+			t.Fatalf("failure = %+v, want captured panic and stack", failure)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("panic did not emit a failure")
+	}
+
+	report := supervisor.Wait(context.Background())
+	if !report.Complete || len(report.Tasks) != 1 || len(report.Tasks[0].Stack) == 0 {
+		t.Fatalf("report = %+v, want completed task with captured stack", report)
+	}
+}
+
+func TestSupervisorCapturesEmptyPanicDuringShutdown(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	started := make(chan struct{})
+	if err := supervisor.Go("panic", TaskPolicy{}, func(ctx context.Context) error {
+		close(started)
+		<-ctx.Done()
+		panic("")
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+	<-started
+	supervisor.BeginShutdown(errors.New("requested shutdown"))
+
+	select {
+	case failure := <-supervisor.Failures():
+		if failure.Task != "panic" || len(failure.Stack) == 0 {
+			t.Fatalf("failure = %+v, want captured panic and stack", failure)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("panic during shutdown did not emit a failure")
+	}
+	if report := supervisor.Wait(context.Background()); !report.Complete || report.Tasks[0].Status != TaskPanicked {
+		t.Fatalf("report = %+v, want completed panicked task", report)
+	}
+}
+
+func TestSupervisorPublishesFailureBeforeCompletion(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	wantErr := errors.New("setup failed")
+	if err := supervisor.Go("setup", TaskPolicy{Required: true}, func(context.Context) error {
+		return wantErr
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+
+	report := supervisor.Wait(context.Background())
+	if !report.Complete || !errors.Is(report.Cause, wantErr) {
+		t.Fatalf("report = %+v, want complete report caused by task failure", report)
+	}
+	select {
+	case failure := <-supervisor.Failures():
+		if !errors.Is(failure.Cause, wantErr) {
+			t.Fatalf("failure cause = %v, want %v", failure.Cause, wantErr)
+		}
+	default:
+		t.Fatal("failure was not published before completion")
+	}
+}
+
+func TestSupervisorCompletedWaitWinsOverCanceledContext(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	supervisor.Seal()
+
+	waitContext, cancel := context.WithCancel(context.Background())
+	cancel()
+	if report := supervisor.Wait(waitContext); !report.Complete {
+		t.Fatalf("report = %+v, want authoritative completion state", report)
+	}
+}
+
+func TestSupervisorRejectsTaskAfterParentCancellation(t *testing.T) {
+	parent, cancel := context.WithCancel(context.Background())
+	supervisor := NewSupervisor(parent)
+	cancel()
+
+	err := supervisor.Go("late", TaskPolicy{}, func(context.Context) error { return nil })
+	if !errors.Is(err, ErrSupervisorStopped) && !errors.Is(err, ErrSupervisorSealed) {
+		t.Fatalf("Go() error = %v, want stopped or sealed supervisor", err)
+	}
+	supervisor.BeginShutdown(context.Canceled)
+	if report := supervisor.Wait(context.Background()); !report.Complete {
+		t.Fatalf("report = %+v, want complete canceled supervisor", report)
+	}
+}
+
+func TestSupervisorShutdownBeforeSealDoesNotStartTasks(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	started := make(chan struct{})
+	if err := supervisor.Go("worker", TaskPolicy{}, func(context.Context) error {
+		close(started)
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	supervisor.BeginShutdown(errors.New("startup aborted"))
+	if report := supervisor.Wait(context.Background()); !report.Complete {
+		t.Fatalf("report = %+v, want completed aborted startup", report)
+	}
+	select {
+	case <-started:
+		t.Fatal("task started after startup was aborted")
+	default:
+	}
+}
+
+func TestSupervisorRejectsDuplicateAndLateTasks(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	release := make(chan struct{})
+	if err := supervisor.Go("worker", TaskPolicy{}, func(context.Context) error {
+		<-release
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := supervisor.Go("worker", TaskPolicy{}, func(context.Context) error { return nil }); !errors.Is(err, ErrTaskExists) {
+		t.Fatalf("duplicate Go() error = %v, want ErrTaskExists", err)
+	}
+
+	supervisor.Seal()
+	if err := supervisor.Go("late", TaskPolicy{}, func(context.Context) error { return nil }); !errors.Is(err, ErrSupervisorSealed) {
+		t.Fatalf("late Go() error = %v, want ErrSupervisorSealed", err)
+	}
+	close(release)
+}
+
+func TestSupervisorConcurrentStartAndSeal(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	var starters sync.WaitGroup
+	for index := 0; index < 100; index++ {
+		starters.Add(1)
+		go func(index int) {
+			defer starters.Done()
+			_ = supervisor.Go(
+				"worker-"+time.Unix(0, int64(index)).Format("150405.000000000"),
+				TaskPolicy{},
+				func(context.Context) error { return nil },
+			)
+		}(index)
+	}
+
+	supervisor.Seal()
+	starters.Wait()
+	if report := supervisor.Wait(context.Background()); !report.Complete {
+		t.Fatal("Wait() did not complete after concurrent Start and Seal")
+	}
+}

machinery/src/log/main.go

@@ -1,179 +0,0 @@
-package log
-
-import (
-	"os"
-	"time"
-
-	"github.com/op/go-logging"
-	"github.com/sirupsen/logrus"
-	"gopkg.in/natefinch/lumberjack.v2"
-)
-
-// The logging library being used everywhere.
-var Log = Logging{
-	Logger: "logrus",
-}
-
-// -----------------
-// This a gologging
-// -> github.com/op/go-logging
-
-var gologging = logging.MustGetLogger("gologger")
-
-func ConfigureGoLogging(configDirectory string, timezone *time.Location) {
-	// Logging
-	var format = logging.MustStringFormatter(
-		`%{color}%{time:15:04:05.000} %{shortfunc} ▶ %{level:.4s} %{id:03x}%{color:reset} %{message}`,
-	)
-	var format2 = logging.MustStringFormatter(
-		`%{time:15:04:05.000} %{shortfunc} ▶ %{level:.4s}  %{id:03x}%{color:reset} %{message}`,
-	)
-	stdBackend := logging.NewLogBackend(os.Stderr, "", 0)
-	stdBackendLeveled := logging.NewBackendFormatter(stdBackend, format)
-	fileBackend := logging.NewLogBackend(&lumberjack.Logger{
-		Filename: configDirectory + "/data/log/machinery.txt",
-		MaxSize:  2,    // megabytes
-		Compress: true, // disabled by default
-	}, "", 0)
-	fileBackendLeveled := logging.NewBackendFormatter(fileBackend, format2)
-	logging.SetBackend(stdBackendLeveled, fileBackendLeveled)
-	logging.SetLevel(logging.DEBUG, "")
-}
-
-// -----------------
-// This a logrus
-// -> github.com/sirupsen/logrus
-
-func ConfigureLogrus(level string, output string, timezone *time.Location) {
-
-	if output == "json" {
-		// Log as JSON instead of the default ASCII formatter.
-		logrus.SetFormatter(LocalTimeZoneFormatter{
-			Timezone:  timezone,
-			Formatter: &logrus.JSONFormatter{},
-		})
-	} else if output == "text" {
-		// Log as text with colors.
-		formatter := logrus.TextFormatter{
-			ForceColors:   true,
-			FullTimestamp: true,
-		}
-		logrus.SetFormatter(LocalTimeZoneFormatter{
-			Timezone:  timezone,
-			Formatter: &formatter,
-		})
-	}
-
-	// Use local timezone for providing datetime in logs!
-
-	// Output to stdout instead of the default stderr
-	// Can be any io.Writer, see below for File example
-	logrus.SetOutput(os.Stdout)
-
-	// Only log the warning severity or above.
-	logLevel := logrus.InfoLevel
-	if level == "error" {
-		logLevel = logrus.ErrorLevel
-	} else if level == "debug" {
-		logLevel = logrus.DebugLevel
-	} else if level == "fatal" {
-		logLevel = logrus.FatalLevel
-	} else if level == "warning" {
-		logLevel = logrus.WarnLevel
-	} // Add this line for logging filename and line number!
-	logrus.SetLevel(logLevel)
-}
-
-type LocalTimeZoneFormatter struct {
-	Timezone  *time.Location
-	Formatter logrus.Formatter
-}
-
-func (u LocalTimeZoneFormatter) Format(e *logrus.Entry) ([]byte, error) {
-	e.Time = e.Time.In(u.Timezone)
-	return u.Formatter.Format(e)
-}
-
-type Logging struct {
-	Logger string
-}
-
-func (self *Logging) Init(level string, logoutput string, configDirectory string, timezone *time.Location) {
-	switch self.Logger {
-	case "go-logging":
-		ConfigureGoLogging(configDirectory, timezone)
-	case "logrus":
-		ConfigureLogrus(level, logoutput, timezone)
-	default:
-	}
-}
-
-func (self *Logging) Info(sentence string) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Info(sentence)
-	case "logrus":
-		logrus.Info(sentence)
-	default:
-	}
-}
-
-func (self *Logging) Infof(format string, args ...interface{}) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Infof(format, args...)
-	case "logrus":
-		logrus.Infof(format, args...)
-	default:
-	}
-}
-
-func (self *Logging) Warning(sentence string) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Warning(sentence)
-	case "logrus":
-		logrus.Warn(sentence)
-	default:
-	}
-}
-
-func (self *Logging) Debug(sentence string) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Debug(sentence)
-	case "logrus":
-		logrus.Debug(sentence)
-	default:
-	}
-}
-
-func (self *Logging) Debugf(format string, args ...interface{}) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Debugf(format, args...)
-	case "logrus":
-		logrus.Debugf(format, args...)
-	default:
-	}
-}
-
-func (self *Logging) Error(sentence string) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Error(sentence)
-	case "logrus":
-		logrus.Error(sentence)
-	default:
-	}
-}
-
-func (self *Logging) Fatal(sentence string) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Fatal(sentence)
-	case "logrus":
-		logrus.Fatal(sentence)
-	default:
-	}
-}

machinery/src/models/agent_run.go

@@ -0,0 +1,477 @@
+package models
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/kerberos-io/agent/machinery/src/lifecycle"
+	"github.com/kerberos-io/agent/machinery/src/packets"
+	log "github.com/sirupsen/logrus"
+)
+
+var (
+	ErrAgentRunActive  = errors.New("another agent run is active")
+	ErrAgentRunStarted = errors.New("agent run already started")
+	ErrAgentRunStopped = errors.New("agent run is stopping")
+	nextAgentRunID     atomic.Uint64
+)
+
+type AgentRunClient interface {
+	Close(context.Context) error
+}
+
+type AgentRunResourceError struct {
+	Resource string
+	Err      error
+}
+
+func (e AgentRunResourceError) Error() string {
+	return fmt.Sprintf("close %s: %v", e.Resource, e.Err)
+}
+
+func (e AgentRunResourceError) Unwrap() error {
+	return e.Err
+}
+
+type AgentRunShutdownReport struct {
+	lifecycle.ShutdownReport
+	ResourceErrors      []AgentRunResourceError
+	UploadStopDelivered bool
+	StreamStopDelivered bool
+}
+
+type AgentRun struct {
+	id            uint64
+	ctx           context.Context
+	cancel        context.CancelCauseFunc
+	supervisor    *lifecycle.Supervisor
+	communication *Communication
+	stopUpload    bool
+
+	stateMu   sync.Mutex
+	activated bool
+	stopping  bool
+
+	resourcesMu       sync.RWMutex
+	mainClient        AgentRunClient
+	subClient         AgentRunClient
+	backchannelClient AgentRunClient
+	mainQueue         *packets.Queue
+	subQueue          *packets.Queue
+	releaseClients    func()
+
+	channelsMu       sync.RWMutex
+	channelsClosed   bool
+	liveHDHandshakes chan LiveHDHandshake
+	motionEvents     chan MotionDataPartial
+	onvifActions     chan OnvifAction
+
+	shutdownOnce   sync.Once
+	shutdownReport AgentRunShutdownReport
+}
+
+func NewAgentRun(parent context.Context, communication *Communication, stopUpload bool) *AgentRun {
+	if parent == nil {
+		parent = context.Background()
+	}
+	ctx, cancel := context.WithCancelCause(parent)
+	run := &AgentRun{
+		id:               nextAgentRunID.Add(1),
+		ctx:              ctx,
+		cancel:           cancel,
+		supervisor:       lifecycle.NewSupervisor(ctx),
+		communication:    communication,
+		stopUpload:       stopUpload,
+		liveHDHandshakes: make(chan LiveHDHandshake, 100),
+		motionEvents:     make(chan MotionDataPartial, 10),
+		onvifActions:     make(chan OnvifAction, 10),
+	}
+	log.WithFields(log.Fields{
+		"component":   "agent_run",
+		"event":       "run_created",
+		"run_id":      run.id,
+		"stop_upload": stopUpload,
+	}).Debug("Agent run created")
+	return run
+}
+
+func (r *AgentRun) ID() uint64 {
+	return r.id
+}
+
+func (r *AgentRun) Context() context.Context {
+	return r.ctx
+}
+
+func (r *AgentRun) Go(name string, policy lifecycle.TaskPolicy, task lifecycle.TaskFunc) error {
+	if task == nil {
+		return r.supervisor.Go(name, policy, task)
+	}
+
+	fields := log.Fields{
+		"component":    "agent_run",
+		"event":        "task_lifecycle",
+		"long_running": policy.LongRunning,
+		"required":     policy.Required,
+		"run_id":       r.id,
+		"task":         name,
+	}
+	wrappedTask := func(ctx context.Context) error {
+		startedAt := time.Now()
+		log.WithFields(fields).Debug("Agent task started")
+		err := task(ctx)
+		entry := log.WithFields(fields).
+			WithField("duration_ms", time.Since(startedAt).Milliseconds())
+		if err != nil {
+			entry = entry.WithError(err)
+			if ctx.Err() == nil {
+				entry.Warn("Agent task stopped with an error")
+			} else {
+				entry.Debug("Agent task stopped during shutdown")
+			}
+		} else {
+			entry.Debug("Agent task stopped")
+		}
+		return err
+	}
+
+	if err := r.supervisor.Go(name, policy, wrappedTask); err != nil {
+		log.WithError(err).WithFields(fields).Error("Failed to register Agent task")
+		return err
+	}
+	log.WithFields(fields).Debug("Agent task registered")
+	return nil
+}
+
+func (r *AgentRun) Seal() {
+	r.supervisor.Seal()
+	log.WithFields(log.Fields{
+		"component":  "agent_run",
+		"event":      "run_sealed",
+		"run_id":     r.id,
+		"task_count": len(r.supervisor.Snapshot()),
+	}).Debug("Agent run sealed")
+}
+
+func (r *AgentRun) Failures() <-chan lifecycle.Failure {
+	return r.supervisor.Failures()
+}
+
+func (r *AgentRun) Snapshot() []lifecycle.TaskSnapshot {
+	return r.supervisor.Snapshot()
+}
+
+func (r *AgentRun) Wait(ctx context.Context) lifecycle.ShutdownReport {
+	return r.supervisor.Wait(ctx)
+}
+
+func (r *AgentRun) Activate() error {
+	r.stateMu.Lock()
+	defer r.stateMu.Unlock()
+	if r.stopping {
+		return ErrAgentRunStopped
+	}
+	if r.ctx.Err() != nil {
+		return ErrAgentRunStopped
+	}
+	if r.activated {
+		return ErrAgentRunStarted
+	}
+	if r.communication == nil {
+		return errors.New("agent run communication is required")
+	}
+	if err := r.communication.attachRun(r); err != nil {
+		return err
+	}
+	r.activated = true
+	log.WithFields(log.Fields{
+		"component": "agent_run",
+		"event":     "run_activated",
+		"run_id":    r.id,
+	}).Info("Agent run activated")
+	return nil
+}
+
+func (r *AgentRun) SetMainClient(client AgentRunClient) {
+	r.resourcesMu.Lock()
+	r.mainClient = client
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) SetSubClient(client AgentRunClient) {
+	r.resourcesMu.Lock()
+	r.subClient = client
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) SetBackchannelClient(client AgentRunClient) {
+	r.resourcesMu.Lock()
+	r.backchannelClient = client
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) SetQueues(mainQueue, subQueue *packets.Queue) {
+	r.resourcesMu.Lock()
+	r.mainQueue = mainQueue
+	r.subQueue = subQueue
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) SetMainQueue(queue *packets.Queue) {
+	r.resourcesMu.Lock()
+	r.mainQueue = queue
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) SetSubQueue(queue *packets.Queue) {
+	r.resourcesMu.Lock()
+	r.subQueue = queue
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) SetClientRelease(release func()) {
+	r.resourcesMu.Lock()
+	r.releaseClients = release
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) MainQueue() *packets.Queue {
+	if r.isStopping() {
+		return nil
+	}
+	r.resourcesMu.RLock()
+	defer r.resourcesMu.RUnlock()
+	return r.mainQueue
+}
+
+func (r *AgentRun) SubQueue() *packets.Queue {
+	if r.isStopping() {
+		return nil
+	}
+	r.resourcesMu.RLock()
+	defer r.resourcesMu.RUnlock()
+	return r.subQueue
+}
+
+func (r *AgentRun) LiveHDHandshakes() <-chan LiveHDHandshake {
+	return r.liveHDHandshakes
+}
+
+func (r *AgentRun) MotionEvents() <-chan MotionDataPartial {
+	return r.motionEvents
+}
+
+func (r *AgentRun) ONVIFActions() <-chan OnvifAction {
+	return r.onvifActions
+}
+
+func (r *AgentRun) TrySendLiveHDHandshake(handshake LiveHDHandshake) bool {
+	if r.isStopping() {
+		return false
+	}
+	r.channelsMu.RLock()
+	defer r.channelsMu.RUnlock()
+	if r.channelsClosed {
+		return false
+	}
+	select {
+	case r.liveHDHandshakes <- handshake:
+		return true
+	default:
+		return false
+	}
+}
+
+func (r *AgentRun) PendingLiveHDHandshakes() int {
+	if r.isStopping() {
+		return 0
+	}
+	r.channelsMu.RLock()
+	defer r.channelsMu.RUnlock()
+	if r.channelsClosed {
+		return 0
+	}
+	return len(r.liveHDHandshakes)
+}
+
+func (r *AgentRun) TrySendMotion(motion MotionDataPartial) bool {
+	if r.isStopping() {
+		return false
+	}
+	r.channelsMu.RLock()
+	defer r.channelsMu.RUnlock()
+	if r.channelsClosed {
+		return false
+	}
+	select {
+	case r.motionEvents <- motion:
+		return true
+	default:
+		return false
+	}
+}
+
+func (r *AgentRun) TrySendONVIF(action OnvifAction) bool {
+	if r.isStopping() {
+		return false
+	}
+	r.channelsMu.RLock()
+	defer r.channelsMu.RUnlock()
+	if r.channelsClosed {
+		return false
+	}
+	select {
+	case r.onvifActions <- action:
+		return true
+	default:
+		return false
+	}
+}
+
+func (r *AgentRun) Shutdown(ctx context.Context, cause error) AgentRunShutdownReport {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	if cause == nil {
+		cause = context.Canceled
+	}
+
+	r.shutdownOnce.Do(func() {
+		startedAt := time.Now()
+		log.WithError(cause).WithFields(log.Fields{
+			"component": "agent_run",
+			"event":     "shutdown_started",
+			"run_id":    r.id,
+		}).Info("Agent run shutdown started")
+
+		r.stateMu.Lock()
+		r.stopping = true
+		activated := r.activated
+		r.stateMu.Unlock()
+
+		r.cancel(cause)
+		r.supervisor.BeginShutdown(cause)
+
+		if activated && r.stopUpload {
+			r.shutdownReport.UploadStopDelivered = sendRunStop(ctx, r.communication.HandleUpload)
+		}
+		if activated {
+			r.shutdownReport.StreamStopDelivered = sendRunStop(ctx, r.communication.HandleStream)
+		}
+
+		r.resourcesMu.RLock()
+		mainClient := r.mainClient
+		subClient := r.subClient
+		backchannelClient := r.backchannelClient
+		mainQueue := r.mainQueue
+		subQueue := r.subQueue
+		releaseClients := r.releaseClients
+		r.resourcesMu.RUnlock()
+
+		r.closeClient(ctx, "main RTSP client", mainClient)
+		if mainQueue != nil {
+			_ = mainQueue.Close()
+		}
+		r.closeClient(ctx, "sub RTSP client", subClient)
+		if subQueue != nil {
+			_ = subQueue.Close()
+		}
+		r.closeClient(ctx, "RTSP backchannel client", backchannelClient)
+		r.closeChannels()
+		if releaseClients != nil {
+			releaseClients()
+		}
+
+		r.shutdownReport.ShutdownReport = r.supervisor.Wait(ctx)
+		if r.shutdownReport.Complete && r.communication != nil {
+			r.communication.detachRun(r)
+		}
+
+		runningTasks := make([]string, 0, len(r.shutdownReport.Running))
+		for _, task := range r.shutdownReport.Running {
+			runningTasks = append(runningTasks, task.Name)
+		}
+		entry := log.WithFields(log.Fields{
+			"component":            "agent_run",
+			"duration_ms":          time.Since(startedAt).Milliseconds(),
+			"resource_error_count": len(r.shutdownReport.ResourceErrors),
+			"run_id":               r.id,
+			"running_tasks":        runningTasks,
+			"task_count":           len(r.shutdownReport.Tasks),
+			"upload_stop_sent":     r.shutdownReport.UploadStopDelivered,
+			"stream_stop_sent":     r.shutdownReport.StreamStopDelivered,
+		})
+		if r.shutdownReport.Complete {
+			entry.WithField("event", "shutdown_completed").Info("Agent run shutdown completed")
+		} else {
+			entry.WithField("event", "shutdown_timed_out").Error("Agent run shutdown timed out")
+		}
+	})
+
+	report := r.shutdownReport
+	report.ResourceErrors = append([]AgentRunResourceError(nil), report.ResourceErrors...)
+	return report
+}
+
+func (r *AgentRun) isStopping() bool {
+	r.stateMu.Lock()
+	defer r.stateMu.Unlock()
+	return r.stopping
+}
+
+func (r *AgentRun) closeClient(ctx context.Context, name string, client AgentRunClient) {
+	if client == nil {
+		return
+	}
+	if err := client.Close(ctx); err != nil {
+		r.shutdownReport.ResourceErrors = append(r.shutdownReport.ResourceErrors, AgentRunResourceError{
+			Resource: name,
+			Err:      err,
+		})
+		log.WithError(err).WithFields(log.Fields{
+			"component": "agent_run",
+			"event":     "resource_close_failed",
+			"resource":  name,
+			"run_id":    r.id,
+		}).Warn("Failed to close Agent run resource")
+		return
+	}
+	log.WithFields(log.Fields{
+		"component": "agent_run",
+		"event":     "resource_closed",
+		"resource":  name,
+		"run_id":    r.id,
+	}).Debug("Agent run resource closed")
+}
+
+func (r *AgentRun) closeChannels() {
+	r.channelsMu.Lock()
+	defer r.channelsMu.Unlock()
+	if r.channelsClosed {
+		return
+	}
+	r.channelsClosed = true
+	close(r.liveHDHandshakes)
+	close(r.motionEvents)
+	close(r.onvifActions)
+}
+
+func sendRunStop(ctx context.Context, channel chan<- string) bool {
+	if channel == nil {
+		return false
+	}
+	timer := time.NewTimer(time.Second)
+	defer timer.Stop()
+	select {
+	case channel <- "stop":
+		return true
+	case <-ctx.Done():
+		return false
+	case <-timer.C:
+		return false
+	}
+}

machinery/src/models/agent_run_test.go

@@ -0,0 +1,303 @@
+package models
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/kerberos-io/agent/machinery/src/lifecycle"
+	"github.com/kerberos-io/agent/machinery/src/packets"
+)
+
+type fakeAgentRunClient struct {
+	mu       sync.Mutex
+	name     string
+	order    *[]string
+	calls    int
+	closeErr error
+}
+
+func (c *fakeAgentRunClient) Close(context.Context) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.calls++
+	*c.order = append(*c.order, c.name)
+	return c.closeErr
+}
+
+func (c *fakeAgentRunClient) Calls() int {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.calls
+}
+
+func TestAgentRunOwnsAndShutsDownResources(t *testing.T) {
+	communication := &Communication{
+		HandleStream: make(chan string, 1),
+		HandleUpload: make(chan string, 1),
+	}
+	run := NewAgentRun(context.Background(), communication, true)
+	mainQueue := packets.NewQueue()
+	subQueue := packets.NewQueue()
+	run.SetQueues(mainQueue, subQueue)
+
+	var order []string
+	mainClient := &fakeAgentRunClient{name: "main", order: &order}
+	subClient := &fakeAgentRunClient{name: "sub", order: &order}
+	backchannelClient := &fakeAgentRunClient{name: "backchannel", order: &order}
+	run.SetMainClient(mainClient)
+	run.SetSubClient(subClient)
+	run.SetBackchannelClient(backchannelClient)
+	run.SetClientRelease(func() {
+		order = append(order, "release")
+	})
+
+	taskStarted := make(chan struct{})
+	if err := run.Go("worker", lifecycle.TaskPolicy{}, func(ctx context.Context) error {
+		close(taskStarted)
+		<-ctx.Done()
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := run.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	run.Seal()
+	<-taskStarted
+
+	report := run.Shutdown(context.Background(), errors.New("test shutdown"))
+	if !report.Complete {
+		t.Fatalf("shutdown report = %+v, want complete", report)
+	}
+	if communication.CurrentRun() != nil {
+		t.Fatal("shutdown left the run attached")
+	}
+	if got, want := order, []string{"main", "sub", "backchannel", "release"}; !equalStrings(got, want) {
+		t.Fatalf("resource order = %v, want %v", got, want)
+	}
+	if !report.UploadStopDelivered || !report.StreamStopDelivered {
+		t.Fatalf("stop delivery = upload:%t stream:%t, want both", report.UploadStopDelivered, report.StreamStopDelivered)
+	}
+	select {
+	case value := <-communication.HandleUpload:
+		if value != "stop" {
+			t.Fatalf("upload stop = %q, want stop", value)
+		}
+	default:
+		t.Fatal("upload stop was not delivered")
+	}
+	select {
+	case value := <-communication.HandleStream:
+		if value != "stop" {
+			t.Fatalf("stream stop = %q, want stop", value)
+		}
+	default:
+		t.Fatal("stream stop was not delivered")
+	}
+	if _, err := mainQueue.Latest().ReadPacket(); err == nil {
+		t.Fatal("main queue remained open")
+	}
+	if _, err := subQueue.Latest().ReadPacket(); err == nil {
+		t.Fatal("sub queue remained open")
+	}
+	if _, ok := <-run.LiveHDHandshakes(); ok {
+		t.Fatal("handshake channel remained open")
+	}
+	if _, ok := <-run.MotionEvents(); ok {
+		t.Fatal("motion channel remained open")
+	}
+	if _, ok := <-run.ONVIFActions(); ok {
+		t.Fatal("ONVIF channel remained open")
+	}
+}
+
+func TestAgentRunShutdownIsConcurrentAndIdempotent(t *testing.T) {
+	communication := &Communication{}
+	run := NewAgentRun(context.Background(), communication, false)
+	var order []string
+	client := &fakeAgentRunClient{name: "main", order: &order}
+	run.SetMainClient(client)
+	if err := run.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	run.Seal()
+
+	var callers sync.WaitGroup
+	reports := make(chan AgentRunShutdownReport, 20)
+	for index := 0; index < 20; index++ {
+		callers.Add(1)
+		go func() {
+			defer callers.Done()
+			reports <- run.Shutdown(context.Background(), errors.New("test shutdown"))
+		}()
+	}
+	callers.Wait()
+	close(reports)
+
+	for report := range reports {
+		if !report.Complete {
+			t.Fatalf("shutdown report = %+v, want complete", report)
+		}
+	}
+	if got := client.Calls(); got != 1 {
+		t.Fatalf("client Close() calls = %d, want 1", got)
+	}
+}
+
+func TestAgentRunStaleShutdownDoesNotDetachNewRun(t *testing.T) {
+	communication := &Communication{}
+	oldRun := NewAgentRun(context.Background(), communication, false)
+	if err := oldRun.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	if !communication.detachRun(oldRun) {
+		t.Fatal("failed to detach old run during test setup")
+	}
+
+	newRun := NewAgentRun(context.Background(), communication, false)
+	if err := newRun.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	newRun.Seal()
+	t.Cleanup(func() {
+		newRun.Shutdown(context.Background(), errors.New("test complete"))
+	})
+
+	oldRun.Shutdown(context.Background(), errors.New("stale shutdown"))
+	if got := communication.CurrentRun(); got != newRun {
+		t.Fatalf("current run = %p, want new run %p", got, newRun)
+	}
+}
+
+func TestAgentRunRejectsOverlap(t *testing.T) {
+	communication := &Communication{}
+	first := NewAgentRun(context.Background(), communication, false)
+	if err := first.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	first.Seal()
+	t.Cleanup(func() {
+		first.Shutdown(context.Background(), errors.New("test complete"))
+	})
+
+	second := NewAgentRun(context.Background(), communication, false)
+	if err := second.Activate(); !errors.Is(err, ErrAgentRunActive) {
+		t.Fatalf("Activate() error = %v, want ErrAgentRunActive", err)
+	}
+	second.Shutdown(context.Background(), errors.New("test complete"))
+}
+
+func TestAgentRunRetainsOwnershipUntilWorkersStop(t *testing.T) {
+	communication := &Communication{}
+	run := NewAgentRun(context.Background(), communication, false)
+	release := make(chan struct{})
+	started := make(chan struct{})
+	if err := run.Go("blocked", lifecycle.TaskPolicy{}, func(context.Context) error {
+		close(started)
+		<-release
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := run.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	run.Seal()
+	<-started
+
+	shutdownDone := make(chan AgentRunShutdownReport, 1)
+	go func() {
+		shutdownDone <- run.Shutdown(context.Background(), errors.New("test shutdown"))
+	}()
+
+	deadline := time.Now().Add(time.Second)
+	for run.Context().Err() == nil && time.Now().Before(deadline) {
+		time.Sleep(time.Millisecond)
+	}
+	if run.Context().Err() == nil {
+		t.Fatal("run context was not canceled")
+	}
+	if communication.CurrentRun() != run {
+		t.Fatal("run ownership was released before its worker stopped")
+	}
+	if communication.TrySendMotion(MotionDataPartial{}) {
+		t.Fatal("stopping run accepted new input")
+	}
+	if communication.MainQueue() != nil {
+		t.Fatal("stopping run exposed its queue")
+	}
+
+	replacement := NewAgentRun(context.Background(), communication, false)
+	if err := replacement.Activate(); !errors.Is(err, ErrAgentRunActive) {
+		t.Fatalf("replacement Activate() error = %v, want ErrAgentRunActive", err)
+	}
+	replacement.Shutdown(context.Background(), errors.New("test complete"))
+
+	close(release)
+	report := <-shutdownDone
+	if !report.Complete {
+		t.Fatalf("shutdown report = %+v, want complete", report)
+	}
+	if communication.CurrentRun() != nil {
+		t.Fatal("completed shutdown left the run attached")
+	}
+}
+
+func TestAgentRunShutdownReportsStuckTask(t *testing.T) {
+	communication := &Communication{}
+	run := NewAgentRun(context.Background(), communication, false)
+	release := make(chan struct{})
+	started := make(chan struct{})
+	if err := run.Go("blocked", lifecycle.TaskPolicy{}, func(context.Context) error {
+		close(started)
+		<-release
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := run.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	run.Seal()
+	<-started
+
+	waitContext, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+	defer cancel()
+	report := run.Shutdown(waitContext, errors.New("test shutdown"))
+	if report.Complete {
+		t.Fatal("shutdown unexpectedly completed")
+	}
+	if len(report.Running) != 1 || report.Running[0].Name != "blocked" {
+		t.Fatalf("running tasks = %+v, want blocked", report.Running)
+	}
+	if communication.CurrentRun() != run {
+		t.Fatal("timed-out shutdown released run ownership")
+	}
+	replacement := NewAgentRun(context.Background(), communication, false)
+	if err := replacement.Activate(); !errors.Is(err, ErrAgentRunActive) {
+		t.Fatalf("replacement Activate() error = %v, want ErrAgentRunActive", err)
+	}
+	replacement.Shutdown(context.Background(), errors.New("test complete"))
+	close(release)
+	waitContext, waitCancel := context.WithTimeout(context.Background(), time.Second)
+	defer waitCancel()
+	if followup := run.Wait(waitContext); !followup.Complete {
+		t.Fatalf("follow-up report = %+v, want complete", followup)
+	}
+	communication.detachRun(run)
+}
+
+func equalStrings(left, right []string) bool {
+	if len(left) != len(right) {
+		return false
+	}
+	for index := range left {
+		if left[index] != right[index] {
+			return false
+		}
+	}
+	return true
+}

machinery/src/models/communication.go

@@ -1,8 +1,10 @@
 package models
 
 import (
-	"context"
+	"math"
+	"sync"
 	"sync/atomic"
+	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/tevino/abool"
@@ -19,11 +21,84 @@ type LiveHDHandshake struct {
 	Signaling *LiveHDSignalingCallbacks
 }
 
+type MoQRecoveryTelemetry struct {
+	Reconnects          uint64 `json:"reconnects"`
+	LastFrameUnixMillis int64  `json:"lastFrameUnixMillis"`
+	LastWriteMillis     int64  `json:"lastWriteMillis"`
+}
+
+type RecoveryTelemetry struct {
+	MoQHigh                   MoQRecoveryTelemetry `json:"moqHigh"`
+	MoQLow                    MoQRecoveryTelemetry `json:"moqLow"`
+	MoQWriteTimeouts          uint64               `json:"moqWriteTimeouts"`
+	DroppedLiveHDHandshakes   uint64               `json:"droppedLiveHDHandshakes"`
+	DroppedMotionEvents       uint64               `json:"droppedMotionEvents"`
+	DroppedONVIFActions       uint64               `json:"droppedOnvifActions"`
+	WatchdogRestarts          uint64               `json:"watchdogRestarts"`
+	WatchdogCooldownSeconds   int64                `json:"watchdogCooldownSeconds"`
+	RunWorkerShutdownTimeouts uint64               `json:"runWorkerShutdownTimeouts"`
+}
+
+type StreamKind uint8
+
+const (
+	MainStream StreamKind = iota
+	SubStream
+)
+
+type StreamRuntimeTelemetry struct {
+	Configured        bool
+	PackagesProcessed uint64
+	FPS               float64
+	Width             int64
+	Height            int64
+	LastPacketAt      int64
+}
+
+type streamRuntimeTelemetry struct {
+	configured        atomic.Bool
+	packagesProcessed atomic.Uint64
+	fpsBits           atomic.Uint64
+	width             atomic.Int64
+	height            atomic.Int64
+	lastPacketAt      atomic.Int64
+}
+
+type HubRuntimeTelemetry struct {
+	Configured                bool
+	Connected                 bool
+	LastHeartbeatAttemptAt    int64
+	LastSuccessfulHeartbeatAt int64
+}
+
+type hubRuntimeTelemetry struct {
+	configured                atomic.Bool
+	connected                 atomic.Bool
+	lastHeartbeatAttemptAt    atomic.Int64
+	lastSuccessfulHeartbeatAt atomic.Int64
+}
+
+type recoveryTelemetry struct {
+	moqHighReconnects          atomic.Uint64
+	moqHighLastFrameUnixMillis atomic.Int64
+	moqHighLastWriteMillis     atomic.Int64
+	moqLowReconnects           atomic.Uint64
+	moqLowLastFrameUnixMillis  atomic.Int64
+	moqLowLastWriteMillis      atomic.Int64
+	moqWriteTimeouts           atomic.Uint64
+	droppedLiveHDHandshakes    atomic.Uint64
+	droppedMotionEvents        atomic.Uint64
+	droppedONVIFActions        atomic.Uint64
+	watchdogRestarts           atomic.Uint64
+	watchdogCooldownSeconds    atomic.Int64
+	runWorkerShutdownTimeouts  atomic.Uint64
+}
+
 // The communication struct that is managing
 // all the communication between the different goroutines.
 type Communication struct {
-	Context               *context.Context
-	CancelContext         *context.CancelFunc
+	currentRunMu          sync.RWMutex
+	currentRun            *AgentRun
 	PackageCounter        *atomic.Value
 	LastPacketTimer       *atomic.Value
 	PackageCounterSub     *atomic.Value
@@ -32,20 +107,17 @@ type Communication struct {
 	HandleBootstrap       chan string
 	HandleStream          chan string
 	HandleSubStream       chan string
-	HandleMotion          chan MotionDataPartial
 	HandleAudio           chan AudioDataPartial
 	HandleUpload          chan string
 	HandleHeartBeat       chan string
 	HandleLiveSD          chan int64
 	HandleLiveSDHTTP      chan int64
 	HandleLiveHDKeepalive chan string
-	HandleLiveHDHandshake chan LiveHDHandshake
 	HandleLiveHDPeers     chan string
 	// HandleLiveHLS is the live HLS viewer keepalive. It carries the requested
 	// quality tier ("auto"|"high"|"low"; empty => auto) so the producer can switch
 	// the live session between the main and sub stream on demand.
 	HandleLiveHLS chan string
-	HandleONVIF   chan OnvifAction
 	IsConfiguring *abool.AtomicBool
 	// IsRecordingManual is set while a viewer has requested a manual recording
 	// from the live view (the record button). While set, the motion-based
@@ -71,11 +143,224 @@ type Communication struct {
 	// viewer that starts a recording but never heartbeats (an older frontend)
 	// still records up to the max-duration cap instead of being cut off early.
 	RecordingManualHeartbeatSeen *abool.AtomicBool
-	Queue                        *packets.Queue
-	SubQueue                     *packets.Queue
 	Image                        string
-	CameraConnected              bool
-	MainStreamConnected          bool
-	SubStreamConnected           bool
-	HasBackChannel               bool
+	CameraConnected              atomic.Bool
+	MainStreamConnected          atomic.Bool
+	SubStreamConnected           atomic.Bool
+	HasBackChannel               atomic.Bool
+	mainStreamTelemetry          streamRuntimeTelemetry
+	subStreamTelemetry           streamRuntimeTelemetry
+	hubTelemetry                 hubRuntimeTelemetry
+	recovery                     recoveryTelemetry
+}
+
+func (c *Communication) streamTelemetry(stream StreamKind) *streamRuntimeTelemetry {
+	switch stream {
+	case MainStream:
+		return &c.mainStreamTelemetry
+	case SubStream:
+		return &c.subStreamTelemetry
+	default:
+		panic("unsupported stream kind")
+	}
+}
+
+func (c *Communication) SetStreamConfigured(stream StreamKind, configured bool) {
+	c.streamTelemetry(stream).configured.Store(configured)
+}
+
+func (c *Communication) RecordStreamPackage(stream StreamKind, fps float64, width, height int, at time.Time) {
+	telemetry := c.streamTelemetry(stream)
+	telemetry.packagesProcessed.Add(1)
+	if fps > 0 && !math.IsNaN(fps) && !math.IsInf(fps, 0) {
+		telemetry.fpsBits.Store(math.Float64bits(fps))
+	}
+	if width > 0 {
+		telemetry.width.Store(int64(width))
+	}
+	if height > 0 {
+		telemetry.height.Store(int64(height))
+	}
+	if !at.IsZero() {
+		telemetry.lastPacketAt.Store(at.Unix())
+	}
+}
+
+func (c *Communication) StreamRuntimeTelemetry(stream StreamKind) StreamRuntimeTelemetry {
+	telemetry := c.streamTelemetry(stream)
+	return StreamRuntimeTelemetry{
+		Configured:        telemetry.configured.Load(),
+		PackagesProcessed: telemetry.packagesProcessed.Load(),
+		FPS:               math.Float64frombits(telemetry.fpsBits.Load()),
+		Width:             telemetry.width.Load(),
+		Height:            telemetry.height.Load(),
+		LastPacketAt:      telemetry.lastPacketAt.Load(),
+	}
+}
+
+func (c *Communication) SetHubConfigured(configured bool) {
+	c.hubTelemetry.configured.Store(configured)
+	if !configured {
+		c.hubTelemetry.connected.Store(false)
+	}
+}
+
+func (c *Communication) RecordHubHeartbeatAttempt(at time.Time) {
+	if !at.IsZero() {
+		c.hubTelemetry.lastHeartbeatAttemptAt.Store(at.Unix())
+	}
+}
+
+func (c *Communication) RecordHubHeartbeatSuccess(at time.Time) {
+	if !at.IsZero() {
+		c.hubTelemetry.lastSuccessfulHeartbeatAt.Store(at.Unix())
+	}
+	c.hubTelemetry.connected.Store(true)
+}
+
+func (c *Communication) RecordHubHeartbeatFailure() {
+	c.hubTelemetry.connected.Store(false)
+}
+
+func (c *Communication) HubRuntimeTelemetry() HubRuntimeTelemetry {
+	return HubRuntimeTelemetry{
+		Configured:                c.hubTelemetry.configured.Load(),
+		Connected:                 c.hubTelemetry.connected.Load(),
+		LastHeartbeatAttemptAt:    c.hubTelemetry.lastHeartbeatAttemptAt.Load(),
+		LastSuccessfulHeartbeatAt: c.hubTelemetry.lastSuccessfulHeartbeatAt.Load(),
+	}
+}
+
+func (c *Communication) RecordMoQReconnect(quality string) {
+	if quality == StreamQualityLow {
+		c.recovery.moqLowReconnects.Add(1)
+		return
+	}
+	c.recovery.moqHighReconnects.Add(1)
+}
+
+func (c *Communication) RecordMoQWrite(quality string, duration time.Duration, at time.Time) {
+	if quality == StreamQualityLow {
+		c.recovery.moqLowLastWriteMillis.Store(duration.Milliseconds())
+		c.recovery.moqLowLastFrameUnixMillis.Store(at.UnixMilli())
+		return
+	}
+	c.recovery.moqHighLastWriteMillis.Store(duration.Milliseconds())
+	c.recovery.moqHighLastFrameUnixMillis.Store(at.UnixMilli())
+}
+
+func (c *Communication) RecordMoQWriteTimeout() {
+	c.recovery.moqWriteTimeouts.Add(1)
+}
+
+func (c *Communication) RecordWatchdogRestart(cooldown time.Duration) {
+	c.recovery.watchdogRestarts.Add(1)
+	c.SetWatchdogCooldown(cooldown)
+}
+
+func (c *Communication) SetWatchdogCooldown(cooldown time.Duration) {
+	c.recovery.watchdogCooldownSeconds.Store(int64(cooldown / time.Second))
+}
+
+func (c *Communication) RecordRunWorkerShutdownTimeout() {
+	c.recovery.runWorkerShutdownTimeouts.Add(1)
+}
+
+func (c *Communication) RecoveryTelemetry() RecoveryTelemetry {
+	return RecoveryTelemetry{
+		MoQHigh: MoQRecoveryTelemetry{
+			Reconnects:          c.recovery.moqHighReconnects.Load(),
+			LastFrameUnixMillis: c.recovery.moqHighLastFrameUnixMillis.Load(),
+			LastWriteMillis:     c.recovery.moqHighLastWriteMillis.Load(),
+		},
+		MoQLow: MoQRecoveryTelemetry{
+			Reconnects:          c.recovery.moqLowReconnects.Load(),
+			LastFrameUnixMillis: c.recovery.moqLowLastFrameUnixMillis.Load(),
+			LastWriteMillis:     c.recovery.moqLowLastWriteMillis.Load(),
+		},
+		MoQWriteTimeouts:          c.recovery.moqWriteTimeouts.Load(),
+		DroppedLiveHDHandshakes:   c.recovery.droppedLiveHDHandshakes.Load(),
+		DroppedMotionEvents:       c.recovery.droppedMotionEvents.Load(),
+		DroppedONVIFActions:       c.recovery.droppedONVIFActions.Load(),
+		WatchdogRestarts:          c.recovery.watchdogRestarts.Load(),
+		WatchdogCooldownSeconds:   c.recovery.watchdogCooldownSeconds.Load(),
+		RunWorkerShutdownTimeouts: c.recovery.runWorkerShutdownTimeouts.Load(),
+	}
+}
+
+func (c *Communication) attachRun(run *AgentRun) error {
+	c.currentRunMu.Lock()
+	defer c.currentRunMu.Unlock()
+	if c.currentRun != nil {
+		return ErrAgentRunActive
+	}
+	c.currentRun = run
+	return nil
+}
+
+func (c *Communication) detachRun(run *AgentRun) bool {
+	c.currentRunMu.Lock()
+	defer c.currentRunMu.Unlock()
+	if c.currentRun != run {
+		return false
+	}
+	c.currentRun = nil
+	return true
+}
+
+func (c *Communication) CurrentRun() *AgentRun {
+	c.currentRunMu.RLock()
+	defer c.currentRunMu.RUnlock()
+	return c.currentRun
+}
+
+func (c *Communication) MainQueue() *packets.Queue {
+	run := c.CurrentRun()
+	if run == nil {
+		return nil
+	}
+	return run.MainQueue()
+}
+
+func (c *Communication) SubQueue() *packets.Queue {
+	run := c.CurrentRun()
+	if run == nil {
+		return nil
+	}
+	return run.SubQueue()
+}
+
+func (c *Communication) TrySendLiveHDHandshake(handshake LiveHDHandshake) bool {
+	run := c.CurrentRun()
+	if run == nil || !run.TrySendLiveHDHandshake(handshake) {
+		c.recovery.droppedLiveHDHandshakes.Add(1)
+		return false
+	}
+	return true
+}
+
+func (c *Communication) PendingLiveHDHandshakes() int {
+	run := c.CurrentRun()
+	if run == nil {
+		return 0
+	}
+	return run.PendingLiveHDHandshakes()
+}
+
+func (c *Communication) TrySendMotion(motion MotionDataPartial) bool {
+	run := c.CurrentRun()
+	if run == nil || !run.TrySendMotion(motion) {
+		c.recovery.droppedMotionEvents.Add(1)
+		return false
+	}
+	return true
+}
+
+func (c *Communication) TrySendONVIF(action OnvifAction) bool {
+	run := c.CurrentRun()
+	if run == nil || !run.TrySendONVIF(action) {
+		c.recovery.droppedONVIFActions.Add(1)
+		return false
+	}
+	return true
 }

machinery/src/models/communication_test.go

@@ -0,0 +1,221 @@
+package models
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+	"time"
+)
+
+func TestCommunicationRunChannelDispatchAfterClose(t *testing.T) {
+	communication := &Communication{}
+	run := NewAgentRun(context.Background(), communication, false)
+	if err := run.Activate(); err != nil {
+		t.Fatal(err)
+	}
+
+	if !communication.TrySendLiveHDHandshake(LiveHDHandshake{}) {
+		t.Fatal("TrySendLiveHDHandshake() rejected an available channel")
+	}
+	if !communication.TrySendMotion(MotionDataPartial{}) {
+		t.Fatal("TrySendMotion() rejected an available channel")
+	}
+	if !communication.TrySendONVIF(OnvifAction{}) {
+		t.Fatal("TrySendONVIF() rejected an available channel")
+	}
+
+	report := run.Shutdown(context.Background(), errors.New("test complete"))
+	if !report.Complete {
+		t.Fatalf("shutdown report = %+v, want complete", report)
+	}
+	if communication.TrySendLiveHDHandshake(LiveHDHandshake{}) {
+		t.Fatal("TrySendLiveHDHandshake() accepted a closed run")
+	}
+	if communication.TrySendMotion(MotionDataPartial{}) {
+		t.Fatal("TrySendMotion() accepted a closed run")
+	}
+	if communication.TrySendONVIF(OnvifAction{}) {
+		t.Fatal("TrySendONVIF() accepted a closed run")
+	}
+}
+
+func TestCommunicationDispatchCanRaceRunChannelClose(t *testing.T) {
+	communication := &Communication{}
+	run := NewAgentRun(context.Background(), communication, false)
+	if err := run.Activate(); err != nil {
+		t.Fatal(err)
+	}
+
+	start := make(chan struct{})
+	var senders sync.WaitGroup
+	for sender := 0; sender < 20; sender++ {
+		senders.Add(1)
+		go func() {
+			defer senders.Done()
+			<-start
+			for attempt := 0; attempt < 100; attempt++ {
+				communication.TrySendLiveHDHandshake(LiveHDHandshake{})
+				communication.TrySendMotion(MotionDataPartial{})
+				communication.TrySendONVIF(OnvifAction{})
+			}
+		}()
+	}
+
+	close(start)
+	run.Shutdown(context.Background(), errors.New("test complete"))
+	senders.Wait()
+}
+
+func TestCommunicationRunChannelLifecycleSoak(t *testing.T) {
+	communication := &Communication{}
+	for cycle := 0; cycle < 100; cycle++ {
+		run := NewAgentRun(context.Background(), communication, false)
+		if err := run.Activate(); err != nil {
+			t.Fatalf("cycle %d Activate() error = %v", cycle, err)
+		}
+		handshakes := run.LiveHDHandshakes()
+		motion := run.MotionEvents()
+		onvif := run.ONVIFActions()
+
+		var consumers sync.WaitGroup
+		consumers.Add(3)
+		go func() {
+			defer consumers.Done()
+			for range handshakes {
+			}
+		}()
+		go func() {
+			defer consumers.Done()
+			for range motion {
+			}
+		}()
+		go func() {
+			defer consumers.Done()
+			for range onvif {
+			}
+		}()
+
+		var producers sync.WaitGroup
+		for producer := 0; producer < 4; producer++ {
+			producers.Add(1)
+			go func() {
+				defer producers.Done()
+				for attempt := 0; attempt < 50; attempt++ {
+					communication.TrySendLiveHDHandshake(LiveHDHandshake{})
+					communication.TrySendMotion(MotionDataPartial{})
+					communication.TrySendONVIF(OnvifAction{})
+				}
+			}()
+		}
+
+		run.Shutdown(context.Background(), errors.New("test complete"))
+		producers.Wait()
+
+		done := make(chan struct{})
+		go func() {
+			consumers.Wait()
+			close(done)
+		}()
+		select {
+		case <-done:
+		case <-time.After(time.Second):
+			t.Fatalf("cycle %d consumers did not stop after channel close", cycle)
+		}
+	}
+}
+
+func TestCommunicationRecoveryTelemetry(t *testing.T) {
+	communication := &Communication{}
+	communication.RecordMoQReconnect(StreamQualityHigh)
+	communication.RecordMoQReconnect(StreamQualityLow)
+	communication.RecordMoQWrite(StreamQualityHigh, 1250*time.Millisecond, time.UnixMilli(1234))
+	communication.RecordMoQWrite(StreamQualityLow, 2500*time.Millisecond, time.UnixMilli(5678))
+	communication.RecordMoQWriteTimeout()
+	communication.RecordWatchdogRestart(30 * time.Second)
+	communication.RecordRunWorkerShutdownTimeout()
+
+	if communication.TrySendLiveHDHandshake(LiveHDHandshake{}) {
+		t.Fatal("TrySendLiveHDHandshake() accepted an unavailable channel")
+	}
+	if communication.TrySendMotion(MotionDataPartial{}) {
+		t.Fatal("TrySendMotion() accepted an unavailable channel")
+	}
+	if communication.TrySendONVIF(OnvifAction{}) {
+		t.Fatal("TrySendONVIF() accepted an unavailable channel")
+	}
+
+	got := communication.RecoveryTelemetry()
+	if got.MoQHigh != (MoQRecoveryTelemetry{Reconnects: 1, LastFrameUnixMillis: 1234, LastWriteMillis: 1250}) {
+		t.Fatalf("high MoQ telemetry = %+v", got.MoQHigh)
+	}
+	if got.MoQLow != (MoQRecoveryTelemetry{Reconnects: 1, LastFrameUnixMillis: 5678, LastWriteMillis: 2500}) {
+		t.Fatalf("low MoQ telemetry = %+v", got.MoQLow)
+	}
+	if got.MoQWriteTimeouts != 1 || got.WatchdogRestarts != 1 || got.WatchdogCooldownSeconds != 30 || got.RunWorkerShutdownTimeouts != 1 {
+		t.Fatalf("recovery telemetry = %+v", got)
+	}
+	if got.DroppedLiveHDHandshakes != 1 || got.DroppedMotionEvents != 1 || got.DroppedONVIFActions != 1 {
+		t.Fatalf("drop telemetry = %+v", got)
+	}
+}
+
+func TestCommunicationOperationalTelemetry(t *testing.T) {
+	communication := &Communication{}
+	mainPacketAt := time.Unix(1_788_710_401, 0)
+	subPacketAt := time.Unix(1_788_710_402, 0)
+	hubAttemptAt := time.Unix(1_788_710_403, 0)
+	hubSuccessAt := time.Unix(1_788_710_404, 0)
+
+	communication.SetStreamConfigured(MainStream, true)
+	communication.SetStreamConfigured(SubStream, true)
+	communication.RecordStreamPackage(MainStream, 29.97, 1920, 1080, mainPacketAt)
+	communication.RecordStreamPackage(MainStream, 29.97, 1920, 1080, mainPacketAt)
+	communication.RecordStreamPackage(SubStream, 15, 640, 360, subPacketAt)
+	communication.SetHubConfigured(true)
+	communication.RecordHubHeartbeatAttempt(hubAttemptAt)
+	communication.RecordHubHeartbeatSuccess(hubSuccessAt)
+
+	main := communication.StreamRuntimeTelemetry(MainStream)
+	if main != (StreamRuntimeTelemetry{
+		Configured:        true,
+		PackagesProcessed: 2,
+		FPS:               29.97,
+		Width:             1920,
+		Height:            1080,
+		LastPacketAt:      mainPacketAt.Unix(),
+	}) {
+		t.Fatalf("main stream telemetry = %+v", main)
+	}
+
+	sub := communication.StreamRuntimeTelemetry(SubStream)
+	if sub != (StreamRuntimeTelemetry{
+		Configured:        true,
+		PackagesProcessed: 1,
+		FPS:               15,
+		Width:             640,
+		Height:            360,
+		LastPacketAt:      subPacketAt.Unix(),
+	}) {
+		t.Fatalf("sub stream telemetry = %+v", sub)
+	}
+
+	hub := communication.HubRuntimeTelemetry()
+	if hub != (HubRuntimeTelemetry{
+		Configured:                true,
+		Connected:                 true,
+		LastHeartbeatAttemptAt:    hubAttemptAt.Unix(),
+		LastSuccessfulHeartbeatAt: hubSuccessAt.Unix(),
+	}) {
+		t.Fatalf("Hub telemetry = %+v", hub)
+	}
+
+	communication.RecordHubHeartbeatFailure()
+	if communication.HubRuntimeTelemetry().Connected {
+		t.Fatal("Hub telemetry remained connected after a failed heartbeat")
+	}
+	communication.SetHubConfigured(false)
+	if communication.HubRuntimeTelemetry().Configured {
+		t.Fatal("Hub telemetry remained configured after it was disabled")
+	}
+}

machinery/src/models/config.go

@@ -187,6 +187,7 @@ type KStorage struct {
 	SecretAccessKey string `json:"secret_access_key,omitempty" bson:"secret_access_key,omitempty"`
 	Provider        string `json:"provider,omitempty" bson:"provider,omitempty"`
 	Directory       string `json:"directory,omitempty" bson:"directory,omitempty"`
+	CustomHeaders   string `json:"custom_headers,omitempty" bson:"custom_headers,omitempty"`
 	MaxRetries      int    `json:"max_retries,omitempty" bson:"max_retries,omitempty"`
 	Timeout         int    `json:"timeout,omitempty" bson:"timeout,omitempty"`
 }

machinery/src/models/mqtt.go

@@ -6,20 +6,96 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"encoding/pem"
-	"io"
-	"strings"
+	"errors"
+	"sync"
 	"time"
 
 	"github.com/gofrs/uuid"
 	"github.com/kerberos-io/agent/machinery/src/encryption"
-	"github.com/kerberos-io/agent/machinery/src/log"
+	log "github.com/sirupsen/logrus"
 )
 
+var (
+	errMQTTPrivateKeyPEMDecode = errors.New("error decoding PEM block containing private key")
+	errMQTTPrivateKeyNotRSA    = errors.New("private key is not RSA")
+
+	rsaPrivateKeyCache   = newRSAPrivateKeyCache()
+	parsePKCS8PrivateKey = x509.ParsePKCS8PrivateKey
+)
+
+type rsaPrivateKeyCacheState struct {
+	mu      sync.Mutex
+	cond    *sync.Cond
+	pem     string
+	key     *rsa.PrivateKey
+	err     error
+	parsing bool
+	ready   bool
+}
+
+func newRSAPrivateKeyCache() *rsaPrivateKeyCacheState {
+	cache := &rsaPrivateKeyCacheState{}
+	cache.cond = sync.NewCond(&cache.mu)
+	return cache
+}
+
+func (cache *rsaPrivateKeyCacheState) get(privateKey string) (*rsa.PrivateKey, error) {
+	cache.mu.Lock()
+	for {
+		if cache.ready && cache.pem == privateKey && !cache.parsing {
+			key, err := cache.key, cache.err
+			cache.mu.Unlock()
+			return key, err
+		}
+		if cache.parsing {
+			cache.cond.Wait()
+			continue
+		}
+
+		cache.pem = privateKey
+		cache.key = nil
+		cache.err = nil
+		cache.parsing = true
+		cache.ready = false
+		cache.mu.Unlock()
+
+		key, err := parseRSAPrivateKey(privateKey)
+
+		cache.mu.Lock()
+		cache.key = key
+		cache.err = err
+		cache.parsing = false
+		cache.ready = true
+		cache.cond.Broadcast()
+		cache.mu.Unlock()
+		return key, err
+	}
+}
+
+func parseRSAPrivateKey(privateKey string) (*rsa.PrivateKey, error) {
+	block, _ := pem.Decode([]byte(privateKey))
+	if block == nil {
+		return nil, errMQTTPrivateKeyPEMDecode
+	}
+
+	key, err := parsePKCS8PrivateKey(block.Bytes)
+	if err != nil {
+		return nil, err
+	}
+
+	rsaKey, ok := key.(*rsa.PrivateKey)
+	if !ok {
+		return nil, errMQTTPrivateKeyNotRSA
+	}
+
+	return rsaKey, nil
+}
+
 func PackageMQTTMessage(configuration *Configuration, msg Message) ([]byte, error) {
 	// Create a Version 4 UUID.
 	u2, err := uuid.NewV4()
 	if err != nil {
-		log.Log.Error("failed to generate UUID: " + err.Error())
+		log.Error("failed to generate UUID: " + err.Error())
 	}
 
 	// We'll generate an unique id, and encrypt / decrypt it using the private key if available.
@@ -45,27 +121,25 @@ func PackageMQTTMessage(configuration *Configuration, msg Message) ([]byte, erro
 		// Pload to base64
 		data, err := json.Marshal(pload)
 		if err != nil {
-			log.Log.Error("models.mqtt.PackageMQTTMessage(): failed to marshal payload: " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component": "models/mqtt",
+				"event":     "payload_marshal_failed",
+			}).Error("Failed to marshal MQTT payload")
 		}
 
 		// Encrypt the value
 		privateKey := configuration.Config.Encryption.PrivateKey
-		r := strings.NewReader(privateKey)
-		pemBytes, _ := io.ReadAll(r)
-		block, _ := pem.Decode(pemBytes)
-		if block == nil {
-			log.Log.Error("models.mqtt.PackageMQTTMessage(): error decoding PEM block containing private key")
-		} else {
-			// Parse private key
-			b := block.Bytes
-			key, err := x509.ParsePKCS8PrivateKey(b)
-			if err != nil {
-				log.Log.Error("models.mqtt.PackageMQTTMessage(): error parsing private key: " + err.Error())
+		rsaKey, err := rsaPrivateKeyCache.get(privateKey)
+		if err != nil {
+			if errors.Is(err, errMQTTPrivateKeyPEMDecode) {
+				log.Error("models.mqtt.PackageMQTTMessage(): error decoding PEM block containing private key")
+			} else {
+				log.WithError(err).WithFields(log.Fields{
+					"component": "models/mqtt",
+					"event":     "private_key_parse_failed",
+				}).Error("Failed to parse MQTT private key")
 			}
-
-			// Conver key to *rsa.PrivateKey
-			rsaKey, _ := key.(*rsa.PrivateKey)
-
+		} else {
 			// Create a 16bit key random
 			if config.Encryption != nil && config.Encryption.SymmetricKey != "" {
 				k := config.Encryption.SymmetricKey
@@ -250,3 +324,23 @@ type TriggerRelay struct {
 	DeviceId  string `json:"device_id"` // device id
 	Token     string `json:"token"`     // token
 }
+
+// RemoteSessionPayload controls an interactive shell or log stream over MQTT.
+// Data is base64 encoded so terminal control bytes remain valid JSON.
+type RemoteSessionPayload struct {
+	Timestamp int64  `json:"timestamp"`
+	SessionID string `json:"session_id"`
+	Kind      string `json:"kind,omitempty"`
+	Data      string `json:"data,omitempty"`
+	Rows      uint16 `json:"rows,omitempty"`
+	Columns   uint16 `json:"columns,omitempty"`
+	Tail      int    `json:"tail,omitempty"`
+}
+
+type RemoteSessionStatus struct {
+	Timestamp int64  `json:"timestamp"`
+	SessionID string `json:"session_id"`
+	Kind      string `json:"kind"`
+	State     string `json:"state"`
+	Error     string `json:"error,omitempty"`
+}

machinery/src/models/mqtt_test.go

@@ -0,0 +1,199 @@
+package models
+
+import (
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/json"
+	"encoding/pem"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestPackageMQTTMessageCachesCurrentSigningKey(t *testing.T) {
+	resetMQTTPrivateKeyCache(t)
+
+	privateKey := generatePKCS8RSAPrivateKeyPEM(t)
+
+	var parseCalls atomic.Int32
+	restore := hookPKCS8PrivateKeyParser(t, func(der []byte) (any, error) {
+		parseCalls.Add(1)
+		return x509.ParsePKCS8PrivateKey(der)
+	})
+	defer restore()
+
+	msg := Message{
+		Payload: Payload{
+			DeviceId: "device-1",
+			Value: map[string]interface{}{
+				"hello": "world",
+			},
+		},
+	}
+
+	first, err := PackageMQTTMessage(encryptedConfiguration(privateKey), msg)
+	if err != nil {
+		t.Fatalf("first PackageMQTTMessage() error = %v", err)
+	}
+	assertEncryptedMQTTMessage(t, first)
+
+	second, err := PackageMQTTMessage(encryptedConfiguration(privateKey), msg)
+	if err != nil {
+		t.Fatalf("second PackageMQTTMessage() error = %v", err)
+	}
+	assertEncryptedMQTTMessage(t, second)
+
+	if got := parseCalls.Load(); got != 1 {
+		t.Fatalf("cached parse calls = %d, want 1", got)
+	}
+}
+
+func TestPackageMQTTMessageReplacesCachedSigningKeyOnRotation(t *testing.T) {
+	resetMQTTPrivateKeyCache(t)
+
+	privateKey := generatePKCS8RSAPrivateKeyPEM(t)
+	rotatedPrivateKey := generatePKCS8RSAPrivateKeyPEM(t)
+
+	var parseCalls atomic.Int32
+	restore := hookPKCS8PrivateKeyParser(t, func(der []byte) (any, error) {
+		parseCalls.Add(1)
+		return x509.ParsePKCS8PrivateKey(der)
+	})
+	defer restore()
+
+	msg := Message{
+		Payload: Payload{
+			DeviceId: "device-1",
+			Value: map[string]interface{}{
+				"hello": "world",
+			},
+		},
+	}
+
+	for _, key := range []string{privateKey, rotatedPrivateKey, privateKey} {
+		payload, err := PackageMQTTMessage(encryptedConfiguration(key), msg)
+		if err != nil {
+			t.Fatalf("PackageMQTTMessage() error = %v", err)
+		}
+		assertEncryptedMQTTMessage(t, payload)
+	}
+
+	if got := parseCalls.Load(); got != 3 {
+		t.Fatalf("parse calls across old/new/old rotation = %d, want 3", got)
+	}
+}
+
+func TestPackageMQTTMessageConcurrentCallsReuseCachedSigningKey(t *testing.T) {
+	resetMQTTPrivateKeyCache(t)
+
+	privateKey := generatePKCS8RSAPrivateKeyPEM(t)
+
+	var parseCalls atomic.Int32
+	restore := hookPKCS8PrivateKeyParser(t, func(der []byte) (any, error) {
+		parseCalls.Add(1)
+		time.Sleep(10 * time.Millisecond)
+		return x509.ParsePKCS8PrivateKey(der)
+	})
+	defer restore()
+
+	msg := Message{
+		Payload: Payload{
+			DeviceId: "device-1",
+			Value: map[string]interface{}{
+				"hello": "world",
+			},
+		},
+	}
+
+	start := make(chan struct{})
+	var workers sync.WaitGroup
+	for i := 0; i < 16; i++ {
+		workers.Add(1)
+		go func() {
+			defer workers.Done()
+			<-start
+
+			payload, err := PackageMQTTMessage(encryptedConfiguration(privateKey), msg)
+			if err != nil {
+				t.Errorf("PackageMQTTMessage() error = %v", err)
+				return
+			}
+			assertEncryptedMQTTMessage(t, payload)
+		}()
+	}
+
+	close(start)
+	workers.Wait()
+
+	if got := parseCalls.Load(); got != 1 {
+		t.Fatalf("concurrent parse calls = %d, want 1", got)
+	}
+}
+
+func encryptedConfiguration(privateKey string) *Configuration {
+	return &Configuration{
+		Config: Config{
+			Encryption: &Encryption{
+				Enabled:      "true",
+				PrivateKey:   privateKey,
+				SymmetricKey: "secret",
+			},
+		},
+	}
+}
+
+func generatePKCS8RSAPrivateKeyPEM(t *testing.T) string {
+	t.Helper()
+
+	privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		t.Fatalf("rsa.GenerateKey() error = %v", err)
+	}
+
+	der, err := x509.MarshalPKCS8PrivateKey(privateKey)
+	if err != nil {
+		t.Fatalf("x509.MarshalPKCS8PrivateKey() error = %v", err)
+	}
+
+	return string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}))
+}
+
+func assertEncryptedMQTTMessage(t *testing.T, payload []byte) {
+	t.Helper()
+
+	var got Message
+	if err := json.Unmarshal(payload, &got); err != nil {
+		t.Fatalf("json.Unmarshal() error = %v", err)
+	}
+
+	if !got.Encrypted {
+		t.Fatal("message is not marked encrypted")
+	}
+	if got.Payload.EncryptedValue == "" {
+		t.Fatal("encrypted value is empty")
+	}
+	if got.Payload.Signature == "" {
+		t.Fatal("signature is empty")
+	}
+	if len(got.Payload.Value) != 0 {
+		t.Fatalf("payload value = %#v, want cleared map", got.Payload.Value)
+	}
+}
+
+func hookPKCS8PrivateKeyParser(t *testing.T, parser func([]byte) (any, error)) func() {
+	t.Helper()
+
+	original := parsePKCS8PrivateKey
+	parsePKCS8PrivateKey = parser
+
+	return func() {
+		parsePKCS8PrivateKey = original
+	}
+}
+
+func resetMQTTPrivateKeyCache(t *testing.T) {
+	t.Helper()
+	rsaPrivateKeyCache = newRSAPrivateKeyCache()
+}

machinery/src/models/recording_upload_metadata.go

@@ -16,6 +16,7 @@ type RecordingUploadMetadata struct {
 	Timestamp int64   `json:"timestamp"` // Unix milliseconds.
 	Duration  uint64  `json:"duration"`  // Milliseconds.
 	FPS       float64 `json:"fps,omitempty"`
+	Encrypted bool    `json:"encrypted,omitempty"`
 }
 
 // RecordingUploadMetadataFileName returns the queue marker name associated

machinery/src/onvif/discovery.go

@@ -13,8 +13,8 @@ import (
 	"time"
 
 	onvifc "github.com/cedricve/go-onvif"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 // scanPort describes a TCP port we probe while scanning the local network,
@@ -105,7 +105,7 @@ func DiscoverDevices(timeout time.Duration, subnets ...string) []models.Discover
 	// 1) ONVIF WS-Discovery. This is quick and reliable for ONVIF cameras.
 	onvifDevices, err := onvifc.StartDiscovery(timeout)
 	if err != nil {
-		log.Log.Error("onvif.DiscoverDevices(): WS-Discovery failed: " + err.Error())
+		log.Error("onvif.DiscoverDevices(): WS-Discovery failed: " + err.Error())
 	} else {
 		for _, onvifDevice := range onvifDevices {
 			ip := hostFromXAddr(onvifDevice.XAddr)
@@ -129,7 +129,7 @@ func DiscoverDevices(timeout time.Duration, subnets ...string) []models.Discover
 	} else {
 		targets = localScanTargets()
 	}
-	log.Log.Info("onvif.DiscoverDevices(): scanning " + strconv.Itoa(len(targets)) + " hosts on the local network(s)")
+	log.Info("onvif.DiscoverDevices(): scanning " + strconv.Itoa(len(targets)) + " hosts on the local network(s)")
 
 	// Bound the amount of concurrent dials so we do not exhaust file
 	// descriptors on constrained devices (e.g. Raspberry Pi).
@@ -319,17 +319,17 @@ func targetsFromSubnets(subnets []string) []string {
 					targets = append(targets, subnet)
 				}
 			} else {
-				log.Log.Error("onvif.targetsFromSubnets(): invalid address '" + subnet + "'")
+				log.Error("onvif.targetsFromSubnets(): invalid address '" + subnet + "'")
 			}
 			continue
 		}
 		_, ipNet, err := net.ParseCIDR(subnet)
 		if err != nil || ipNet.IP.To4() == nil {
-			log.Log.Error("onvif.targetsFromSubnets(): invalid CIDR '" + subnet + "'")
+			log.Error("onvif.targetsFromSubnets(): invalid CIDR '" + subnet + "'")
 			continue
 		}
 		if ones, bits := ipNet.Mask.Size(); bits != 32 || ones < 22 {
-			log.Log.Error("onvif.targetsFromSubnets(): range '" + subnet + "' is too large to scan (use /22 or smaller)")
+			log.Error("onvif.targetsFromSubnets(): range '" + subnet + "' is too large to scan (use /22 or smaller)")
 			continue
 		}
 		for _, host := range hostsInNetwork(ipNet) {
@@ -352,7 +352,7 @@ func localScanTargets() []string {
 
 	interfaces, err := net.Interfaces()
 	if err != nil {
-		log.Log.Error("onvif.localScanTargets(): " + err.Error())
+		log.Error("onvif.localScanTargets(): " + err.Error())
 		return targets
 	}
 

machinery/src/onvif/events.go

@@ -7,9 +7,9 @@ import (
 	"strings"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/onvif/event/stream"
+	log "github.com/sirupsen/logrus"
 )
 
 // The library handles in-stream reconnect; these guards cover the
@@ -29,14 +29,14 @@ const (
 // blip, credential reload) the goroutine retries with exponential
 // backoff. Exits when ctx is cancelled.
 func HandleONVIFEventStream(ctx context.Context, configuration *models.Configuration, communication *models.Communication) {
-	log.Log.Debug("onvif.HandleONVIFEventStream(): started")
-	defer log.Log.Debug("onvif.HandleONVIFEventStream(): finished")
+	log.Debug("onvif.HandleONVIFEventStream(): started")
+	defer log.Debug("onvif.HandleONVIFEventStream(): finished")
 
 	if !isONVIFMotionEnabled(configuration.Config.Capture.ONVIFMotion) {
 		return
 	}
 	if configuration.Config.Capture.IPCamera.ONVIFXAddr == "" {
-		log.Log.Warning("onvif.HandleONVIFEventStream(): ONVIFMotion enabled but ONVIFXAddr is empty; nothing to do")
+		log.Warn("onvif.HandleONVIFEventStream(): ONVIFMotion enabled but ONVIFXAddr is empty; nothing to do")
 		return
 	}
 
@@ -66,23 +66,26 @@ func runStreamOnce(ctx context.Context, configuration *models.Configuration, com
 
 	device, _, err := ConnectToOnvifDevice(&camera)
 	if err != nil {
-		log.Log.Error("onvif.HandleONVIFEventStream(): connect: " + err.Error())
+		log.Error("onvif.HandleONVIFEventStream(): connect: " + err.Error())
 		return true
 	}
 
 	deviceID := resolveDeviceID(configuration.Name, camera.ONVIFXAddr)
 	s, err := stream.NewStream(ctx, device, stream.Options{DeviceID: deviceID})
 	if err != nil {
-		log.Log.Error("onvif.HandleONVIFEventStream(): open stream: " + err.Error())
+		log.Error("onvif.HandleONVIFEventStream(): open stream: " + err.Error())
 		return true
 	}
 	defer func() {
 		if err := s.Close(); err != nil {
-			log.Log.Debug("onvif.HandleONVIFEventStream(): close: " + err.Error())
+			log.Debug("onvif.HandleONVIFEventStream(): close: " + err.Error())
 		}
 	}()
 
-	log.Log.Info("onvif.HandleONVIFEventStream(): consuming events for " + deviceID)
+	log.WithFields(log.Fields{
+		"component": "onvif",
+		"event":     "event_stream_started",
+	}).Info("Consuming ONVIF events")
 
 	// recovering = the first successful event after an error streak
 	// logs a recovery line so on-call operators see the clear-of-
@@ -97,7 +100,10 @@ func runStreamOnce(ctx context.Context, configuration *models.Configuration, com
 				return false
 			}
 			if recovering {
-				log.Log.Info("onvif.HandleONVIFEventStream(): event stream recovered for " + deviceID)
+				log.WithFields(log.Fields{
+					"component": "onvif",
+					"event":     "event_stream_recovered",
+				}).Info("ONVIF event stream recovered")
 				recovering = false
 			}
 			dispatchEvent(ctx, ev, configuration, communication)
@@ -119,14 +125,14 @@ func runStreamOnce(ctx context.Context, configuration *models.Configuration, com
 func dispatchEvent(ctx context.Context, ev stream.Event, configuration *models.Configuration, communication *models.Communication) {
 	topic := sanitiseTopic(ev.Topic)
 	if ev.Kind != stream.KindMotion {
-		log.Log.Debug("onvif.dispatchEvent(): non-motion event " + ev.Kind.String() + " topic=" + topic)
+		log.Debug("onvif.dispatchEvent(): non-motion event " + ev.Kind.String() + " topic=" + topic)
 		return
 	}
 	if ev.State != stream.StateActive {
 		return
 	}
 	if !isTransition(ev.Operation) {
-		log.Log.Debug("onvif.dispatchEvent(): " + ev.Operation.String() + " is not a transition, not a trigger: topic=" + topic)
+		log.Debug("onvif.dispatchEvent(): " + ev.Operation.String() + " is not a transition, not a trigger: topic=" + topic)
 		return
 	}
 	if configuration.Config.Capture.Recording == "false" {
@@ -140,14 +146,15 @@ func dispatchEvent(ctx context.Context, ev stream.Event, configuration *models.C
 		Timestamp:       time.Now().Unix(),
 		NumberOfChanges: 0, // ONVIF does not quantify motion area.
 	}
-	select {
-	case <-ctx.Done():
-	case communication.HandleMotion <- dataToPass:
+	if ctx.Err() != nil {
+		return
+	}
+	if communication.TrySendMotion(dataToPass) {
 		// Logged on the send, not before it: this line records that a
 		// recording started, so a dropped event must not leave one.
-		log.Log.Debug("onvif.dispatchEvent(): recording trigger " + ev.Kind.String() + " topic=" + topic)
-	default:
-		log.Log.Debug("onvif.dispatchEvent(): HandleMotion full, dropping ONVIF motion event")
+		log.Debug("onvif.dispatchEvent(): recording trigger " + ev.Kind.String() + " topic=" + topic)
+	} else {
+		log.Debug("onvif.dispatchEvent(): HandleMotion full, dropping ONVIF motion event")
 	}
 }
 
@@ -185,13 +192,13 @@ func logStreamError(e error) {
 	var renew stream.ErrRenewFailed
 	switch {
 	case errors.As(e, &recreate):
-		log.Log.Error("onvif.HandleONVIFEventStream(): subscription recreate failed (camera may be offline): " + recreate.Err.Error())
+		log.Error("onvif.HandleONVIFEventStream(): subscription recreate failed (camera may be offline): " + recreate.Err.Error())
 	case errors.As(e, &renew):
-		log.Log.Debug("onvif.HandleONVIFEventStream(): renew failed (will recover via pull/recreate): " + renew.Err.Error())
+		log.Debug("onvif.HandleONVIFEventStream(): renew failed (will recover via pull/recreate): " + renew.Err.Error())
 	case errors.As(e, &pull):
-		log.Log.Debug("onvif.HandleONVIFEventStream(): pull failed (will retry): " + pull.Err.Error())
+		log.Debug("onvif.HandleONVIFEventStream(): pull failed (will retry): " + pull.Err.Error())
 	default:
-		log.Log.Info("onvif.HandleONVIFEventStream(): stream error: " + e.Error())
+		log.Info("onvif.HandleONVIFEventStream(): stream error: " + e.Error())
 	}
 }
 

machinery/src/onvif/events_test.go

@@ -3,6 +3,7 @@ package onvif
 import (
 	"bytes"
 	"context"
+	"errors"
 	"strings"
 	"testing"
 	"time"
@@ -26,17 +27,31 @@ func makeConfig(recording, onvifMotion, name string) *models.Configuration {
 	}
 }
 
-func makeCommunication(buffer int) *models.Communication {
-	return &models.Communication{
-		HandleMotion: make(chan models.MotionDataPartial, buffer),
+func makeCommunication(t *testing.T) *models.Communication {
+	t.Helper()
+	communication := &models.Communication{}
+	run := models.NewAgentRun(context.Background(), communication, false)
+	require.NoError(t, run.Activate())
+	t.Cleanup(func() {
+		run.Shutdown(context.Background(), errors.New("test complete"))
+	})
+	return communication
+}
+
+func motionEvents(t *testing.T, communication *models.Communication) <-chan models.MotionDataPartial {
+	t.Helper()
+	run := communication.CurrentRun()
+	if run == nil {
+		t.Fatal("expected active agent run")
 	}
+	return run.MotionEvents()
 }
 
 // --- dispatchEvent ---------------------------------------------------
 
 func TestDispatchEvent_MotionActive_SendsToHandleMotion(t *testing.T) {
 	cfg := makeConfig("true", "true", "cam-1")
-	comm := makeCommunication(1)
+	comm := makeCommunication(t)
 	ev := stream.Event{Kind: stream.KindMotion, State: stream.StateActive}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -44,7 +59,7 @@ func TestDispatchEvent_MotionActive_SendsToHandleMotion(t *testing.T) {
 	dispatchEvent(ctx, ev, cfg, comm)
 
 	select {
-	case m := <-comm.HandleMotion:
+	case m := <-motionEvents(t, comm):
 		assert.NotZero(t, m.Timestamp)
 	case <-time.After(time.Second):
 		t.Fatal("expected motion data on HandleMotion")
@@ -53,7 +68,7 @@ func TestDispatchEvent_MotionActive_SendsToHandleMotion(t *testing.T) {
 
 func TestDispatchEvent_MotionInactive_DoesNotSend(t *testing.T) {
 	cfg := makeConfig("true", "true", "cam-1")
-	comm := makeCommunication(1)
+	comm := makeCommunication(t)
 	ev := stream.Event{Kind: stream.KindMotion, State: stream.StateInactive}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -61,7 +76,7 @@ func TestDispatchEvent_MotionInactive_DoesNotSend(t *testing.T) {
 	dispatchEvent(ctx, ev, cfg, comm)
 
 	select {
-	case <-comm.HandleMotion:
+	case <-motionEvents(t, comm):
 		t.Fatal("inactive motion must not reach HandleMotion (motion-stop is a follow-up)")
 	case <-time.After(100 * time.Millisecond):
 	}
@@ -69,7 +84,7 @@ func TestDispatchEvent_MotionInactive_DoesNotSend(t *testing.T) {
 
 func TestDispatchEvent_NonMotionKindIgnored(t *testing.T) {
 	cfg := makeConfig("true", "true", "cam-1")
-	comm := makeCommunication(1)
+	comm := makeCommunication(t)
 	ev := stream.Event{Kind: stream.KindDigitalInput, State: stream.StateActive}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -77,7 +92,7 @@ func TestDispatchEvent_NonMotionKindIgnored(t *testing.T) {
 	dispatchEvent(ctx, ev, cfg, comm)
 
 	select {
-	case <-comm.HandleMotion:
+	case <-motionEvents(t, comm):
 		t.Fatal("non-motion kinds must not reach HandleMotion")
 	case <-time.After(100 * time.Millisecond):
 	}
@@ -108,7 +123,7 @@ func TestDispatchEvent_LogsTheTriggeringTopic(t *testing.T) {
 	buf := captureDebugLog(t)
 
 	cfg := makeConfig("true", "true", "cam-1")
-	comm := makeCommunication(1)
+	comm := makeCommunication(t)
 	ev := stream.Event{
 		Kind:  stream.KindMotion,
 		State: stream.StateActive,
@@ -147,7 +162,7 @@ func TestDispatchEvent_PropertyOperation(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			cfg := makeConfig("true", "true", "cam-1")
-			comm := makeCommunication(1)
+			comm := makeCommunication(t)
 			ev := stream.Event{
 				Kind:      stream.KindMotion,
 				State:     stream.StateActive,
@@ -160,7 +175,7 @@ func TestDispatchEvent_PropertyOperation(t *testing.T) {
 			dispatchEvent(ctx, ev, cfg, comm)
 
 			select {
-			case <-comm.HandleMotion:
+			case <-motionEvents(t, comm):
 				if !tt.wantSend {
 					t.Fatalf("%v must not trigger a recording", tt.op)
 				}
@@ -175,7 +190,7 @@ func TestDispatchEvent_PropertyOperation(t *testing.T) {
 
 func TestDispatchEvent_RecordingDisabled_DoesNotSend(t *testing.T) {
 	cfg := makeConfig("false", "true", "cam-1")
-	comm := makeCommunication(1)
+	comm := makeCommunication(t)
 	ev := stream.Event{Kind: stream.KindMotion, State: stream.StateActive}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -183,7 +198,7 @@ func TestDispatchEvent_RecordingDisabled_DoesNotSend(t *testing.T) {
 	dispatchEvent(ctx, ev, cfg, comm)
 
 	select {
-	case <-comm.HandleMotion:
+	case <-motionEvents(t, comm):
 		t.Fatal("Recording=false must gate the send (matches computervision behaviour)")
 	case <-time.After(100 * time.Millisecond):
 	}
@@ -191,9 +206,10 @@ func TestDispatchEvent_RecordingDisabled_DoesNotSend(t *testing.T) {
 
 func TestDispatchEvent_HandleMotionFull_DropsRatherThanBlocks(t *testing.T) {
 	cfg := makeConfig("true", "true", "cam-1")
-	// Pre-fill the buffer so the next send would block.
-	comm := &models.Communication{HandleMotion: make(chan models.MotionDataPartial, 1)}
-	comm.HandleMotion <- models.MotionDataPartial{}
+	// Fill the buffer so the next send would be dropped.
+	comm := makeCommunication(t)
+	for comm.TrySendMotion(models.MotionDataPartial{}) {
+	}
 	ev := stream.Event{Kind: stream.KindMotion, State: stream.StateActive}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -219,8 +235,9 @@ func TestDispatchEvent_CtxCancelledAndHandleMotionClosed_DoesNotPanic(t *testing
 	// send would panic. The ctx pre-check must short-circuit before the
 	// send is attempted.
 	cfg := makeConfig("true", "true", "cam-1")
-	comm := &models.Communication{HandleMotion: make(chan models.MotionDataPartial, 1)}
-	close(comm.HandleMotion)
+	comm := makeCommunication(t)
+	run := comm.CurrentRun()
+	run.Shutdown(context.Background(), errors.New("test shutdown"))
 	ev := stream.Event{Kind: stream.KindMotion, State: stream.StateActive}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -299,7 +316,7 @@ func TestDispatchEvent_OnlyRealTransitionsTrigger(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.op.String(), func(t *testing.T) {
 			cfg := makeConfig("true", "true", "cam-1")
-			comm := makeCommunication(1)
+			comm := makeCommunication(t)
 			ev := stream.Event{Kind: stream.KindMotion, State: stream.StateActive, Operation: tt.op}
 
 			ctx, cancel := context.WithCancel(context.Background())
@@ -307,10 +324,10 @@ func TestDispatchEvent_OnlyRealTransitionsTrigger(t *testing.T) {
 			dispatchEvent(ctx, ev, cfg, comm)
 
 			if tt.wantSend {
-				require.Len(t, comm.HandleMotion, 1, "%v must trigger a recording", tt.op)
+				require.Len(t, motionEvents(t, comm), 1, "%v must trigger a recording", tt.op)
 				return
 			}
-			require.Empty(t, comm.HandleMotion, "%v must not trigger a recording", tt.op)
+			require.Empty(t, motionEvents(t, comm), "%v must not trigger a recording", tt.op)
 		})
 	}
 }
@@ -361,8 +378,9 @@ func TestDispatchEvent_LogsTriggerOnlyWhenSent(t *testing.T) {
 	buf := captureDebugLog(t)
 
 	cfg := makeConfig("true", "true", "cam-1")
-	comm := &models.Communication{HandleMotion: make(chan models.MotionDataPartial, 1)}
-	comm.HandleMotion <- models.MotionDataPartial{} // full
+	comm := makeCommunication(t)
+	for comm.TrySendMotion(models.MotionDataPartial{}) {
+	}
 	ev := stream.Event{Kind: stream.KindMotion, State: stream.StateActive, Topic: "tns1:VideoSource/MotionAlarm"}
 
 	ctx, cancel := context.WithCancel(context.Background())

machinery/src/onvif/main_test.go

@@ -0,0 +1,42 @@
+package onvif
+
+import (
+	"bytes"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func TestHandleONVIFResponseDoesNotLogBody(t *testing.T) {
+	logger := log.StandardLogger()
+	originalOutput := logger.Out
+	originalFormatter := logger.Formatter
+	originalLevel := logger.Level
+	defer func() {
+		logger.SetOutput(originalOutput)
+		logger.SetFormatter(originalFormatter)
+		logger.SetLevel(originalLevel)
+	}()
+
+	var output bytes.Buffer
+	logger.SetOutput(&output)
+	logger.SetFormatter(&log.JSONFormatter{})
+	logger.SetLevel(log.DebugLevel)
+
+	response := &http.Response{
+		StatusCode: http.StatusOK,
+		Body:       io.NopCloser(strings.NewReader(`{"credential":"do-not-log"}`)),
+	}
+	if err := handleONVIFResponse("test_operation", response, nil); err != nil {
+		t.Fatalf("handleONVIFResponse() error = %v", err)
+	}
+	if strings.Contains(output.String(), "do-not-log") {
+		t.Fatalf("ONVIF response log exposed response body: %q", output.String())
+	}
+	if !strings.Contains(output.String(), `"response_bytes":27`) {
+		t.Fatalf("ONVIF response log omitted response byte count: %q", output.String())
+	}
+}

machinery/src/onvif/move_test.go

@@ -0,0 +1,264 @@
+package onvif
+
+import (
+	"encoding/xml"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/kerberos-io/onvif/ptz"
+	xsdonvif "github.com/kerberos-io/onvif/xsd/onvif"
+)
+
+type fakePTZMoveClient struct {
+	current              ptzTarget
+	target               ptzTarget
+	lastVelocity         ptzVelocity
+	absoluteErr          error
+	continuousErr        error
+	stopErr              error
+	ignoreAbsolute       bool
+	stalled              bool
+	incompletePosition   bool
+	positionErr          error
+	positionErrAtCall    int
+	positionCalls        int
+	absoluteCalls        int
+	continuousVelocities []ptzVelocity
+	stopCalls            int
+	moving               bool
+}
+
+func (client *fakePTZMoveClient) GetPosition() (xsdonvif.PTZVector, error) {
+	client.positionCalls++
+	if client.positionErr != nil && client.positionCalls == client.positionErrAtCall {
+		return xsdonvif.PTZVector{}, client.positionErr
+	}
+	if client.moving && !client.stalled {
+		client.current.Pan = moveFakeAxis(client.current.Pan, client.target.Pan, client.lastVelocity.Pan)
+		client.current.Tilt = moveFakeAxis(client.current.Tilt, client.target.Tilt, client.lastVelocity.Tilt)
+		client.current.Zoom = moveFakeAxis(client.current.Zoom, client.target.Zoom, client.lastVelocity.Zoom)
+	}
+
+	panTilt := &xsdonvif.Vector2D{X: client.current.Pan, Y: client.current.Tilt}
+	if client.incompletePosition {
+		return xsdonvif.PTZVector{PanTilt: panTilt}, nil
+	}
+	return xsdonvif.PTZVector{
+		PanTilt: panTilt,
+		Zoom:    &xsdonvif.Vector1D{X: client.current.Zoom},
+	}, nil
+}
+
+func (client *fakePTZMoveClient) AbsoluteMove(target ptzTarget) error {
+	client.absoluteCalls++
+	client.target = target
+	if client.absoluteErr != nil {
+		return client.absoluteErr
+	}
+	if !client.ignoreAbsolute {
+		client.current = target
+	}
+	return nil
+}
+
+func (client *fakePTZMoveClient) ContinuousMove(velocity ptzVelocity) error {
+	if client.continuousErr != nil {
+		return client.continuousErr
+	}
+	client.lastVelocity = velocity
+	client.continuousVelocities = append(client.continuousVelocities, velocity)
+	client.moving = true
+	return nil
+}
+
+func (client *fakePTZMoveClient) Stop() error {
+	client.stopCalls++
+	client.moving = false
+	return client.stopErr
+}
+
+func moveFakeAxis(current float64, target float64, velocity float64) float64 {
+	step := velocity * 0.5
+	if step == 0 {
+		return current
+	}
+	next := current + step
+	if (velocity > 0 && next >= target) || (velocity < 0 && next <= target) {
+		return target
+	}
+	return next
+}
+
+func testPTZMoveOptions() ptzMoveOptions {
+	options := defaultPTZMoveOptions
+	options.NativeTimeout = 50 * time.Millisecond
+	options.ContinuousTimeout = 100 * time.Millisecond
+	options.NoProgressTimeout = 10 * time.Millisecond
+	options.PollInterval = time.Millisecond
+	options.ProgressEpsilon = 0.0001
+	options.VelocityChange = 0.01
+	return options
+}
+
+func TestMoveToPositionUsesVerifiedNativeMove(t *testing.T) {
+	target := ptzTarget{Pan: 0.5, Tilt: -0.25, Zoom: 0.4}
+	client := &fakePTZMoveClient{}
+
+	err := moveToPosition(client, ptzMoveSupport{Absolute: true, Continuous: true}, target, testPTZMoveOptions())
+	if err != nil {
+		t.Fatalf("moveToPosition() error = %v", err)
+	}
+	if client.absoluteCalls != 1 {
+		t.Fatalf("absolute calls = %d, want 1", client.absoluteCalls)
+	}
+	if len(client.continuousVelocities) != 0 {
+		t.Fatalf("continuous calls = %d, want 0", len(client.continuousVelocities))
+	}
+}
+
+func TestMoveToPositionFallsBackToSimultaneousContinuousMove(t *testing.T) {
+	target := ptzTarget{Pan: 0.5, Tilt: -0.25, Zoom: 0.4}
+	client := &fakePTZMoveClient{
+		target:      target,
+		absoluteErr: errors.New("absolute move rejected"),
+	}
+
+	err := moveToPosition(client, ptzMoveSupport{Absolute: true, Continuous: true}, target, testPTZMoveOptions())
+	if err != nil {
+		t.Fatalf("moveToPosition() error = %v", err)
+	}
+	if len(client.continuousVelocities) == 0 {
+		t.Fatal("continuous fallback was not used")
+	}
+	first := client.continuousVelocities[0]
+	if first.Pan == 0 || first.Tilt == 0 || first.Zoom == 0 {
+		t.Fatalf("first fallback velocity = %+v, want all axes moving", first)
+	}
+	if client.stopCalls != 2 {
+		t.Fatalf("stop calls = %d, want native cleanup and fallback cleanup", client.stopCalls)
+	}
+}
+
+func TestMoveToPositionFallsBackWhenNativeMoveMakesNoProgress(t *testing.T) {
+	target := ptzTarget{Pan: 0.3, Tilt: 0.2, Zoom: 0.1}
+	client := &fakePTZMoveClient{
+		target:         target,
+		ignoreAbsolute: true,
+	}
+	options := testPTZMoveOptions()
+	options.NoProgressTimeout = 3 * time.Millisecond
+
+	err := moveToPosition(client, ptzMoveSupport{Absolute: true, Continuous: true}, target, options)
+	if err != nil {
+		t.Fatalf("moveToPosition() error = %v", err)
+	}
+	if len(client.continuousVelocities) == 0 {
+		t.Fatal("continuous fallback was not used")
+	}
+}
+
+func TestMoveToPositionStopsAfterContinuousStatusError(t *testing.T) {
+	target := ptzTarget{Pan: 0.3, Tilt: 0.2, Zoom: 0.1}
+	client := &fakePTZMoveClient{
+		target:            target,
+		positionErr:       errors.New("status unavailable"),
+		positionErrAtCall: 3,
+	}
+
+	err := moveToPosition(client, ptzMoveSupport{Continuous: true}, target, testPTZMoveOptions())
+	if err == nil || !strings.Contains(err.Error(), "status unavailable") {
+		t.Fatalf("moveToPosition() error = %v, want status error", err)
+	}
+	if client.stopCalls != 1 {
+		t.Fatalf("stop calls = %d, want 1", client.stopCalls)
+	}
+}
+
+func TestMoveToPositionStopsWhenContinuousMoveIsRejected(t *testing.T) {
+	target := ptzTarget{Pan: 0.3, Tilt: 0.2, Zoom: 0.1}
+	client := &fakePTZMoveClient{
+		target:        target,
+		continuousErr: errors.New("continuous move rejected"),
+	}
+
+	err := moveToPosition(client, ptzMoveSupport{Continuous: true}, target, testPTZMoveOptions())
+	if err == nil || !strings.Contains(err.Error(), "continuous move rejected") {
+		t.Fatalf("moveToPosition() error = %v, want movement error", err)
+	}
+	if client.stopCalls != 1 {
+		t.Fatalf("stop calls = %d, want 1", client.stopCalls)
+	}
+}
+
+func TestMoveToPositionReturnsNoProgressErrorAndStops(t *testing.T) {
+	target := ptzTarget{Pan: 0.3, Tilt: 0.2, Zoom: 0.1}
+	client := &fakePTZMoveClient{
+		target:  target,
+		stalled: true,
+	}
+	options := testPTZMoveOptions()
+	options.NoProgressTimeout = 3 * time.Millisecond
+
+	err := moveToPosition(client, ptzMoveSupport{Continuous: true}, target, options)
+	if err == nil || !strings.Contains(err.Error(), "made no progress") {
+		t.Fatalf("moveToPosition() error = %v, want no-progress error", err)
+	}
+	if client.stopCalls != 1 {
+		t.Fatalf("stop calls = %d, want 1", client.stopCalls)
+	}
+}
+
+func TestMoveToPositionReturnsDeadlineErrorAndStops(t *testing.T) {
+	target := ptzTarget{Pan: 100, Tilt: 100, Zoom: 100}
+	client := &fakePTZMoveClient{target: target}
+	options := testPTZMoveOptions()
+	options.ContinuousTimeout = 3 * time.Millisecond
+	options.NoProgressTimeout = 50 * time.Millisecond
+
+	err := moveToPosition(client, ptzMoveSupport{Continuous: true}, target, options)
+	if err == nil || !strings.Contains(err.Error(), "target not reached within") {
+		t.Fatalf("moveToPosition() error = %v, want deadline error", err)
+	}
+	if client.stopCalls != 1 {
+		t.Fatalf("stop calls = %d, want 1", client.stopCalls)
+	}
+}
+
+func TestMoveToPositionRejectsIncompletePosition(t *testing.T) {
+	client := &fakePTZMoveClient{incompletePosition: true}
+
+	err := moveToPosition(client, ptzMoveSupport{Continuous: true}, ptzTarget{Pan: 0.1}, testPTZMoveOptions())
+	if err == nil || !strings.Contains(err.Error(), "incomplete PTZ position") {
+		t.Fatalf("moveToPosition() error = %v, want incomplete-position error", err)
+	}
+}
+
+func TestCombinedContinuousMoveVelocityIncludesEveryAxis(t *testing.T) {
+	request := ptz.ContinuousMove{
+		Velocity: ptz.Speed{
+			PanTilt: &xsdonvif.Vector2D{X: 0.5, Y: -0.25},
+			Zoom:    &xsdonvif.Vector1D{X: 0.2},
+		},
+	}
+
+	encoded, err := xml.Marshal(request)
+	if err != nil {
+		t.Fatalf("xml.Marshal() error = %v", err)
+	}
+	xmlBody := string(encoded)
+	if !strings.Contains(xmlBody, `PanTilt x="0.5" y="-0.25"`) {
+		t.Fatalf("ContinuousMove XML = %s, want pan and tilt", xmlBody)
+	}
+	if !strings.Contains(xmlBody, `Zoom x="0.2"`) {
+		t.Fatalf("ContinuousMove XML = %s, want zoom", xmlBody)
+	}
+}
+
+func TestGetPTZFunctionsHandlesMissingConfiguration(t *testing.T) {
+	functions, canZoom, canPanTilt := GetPTZFunctionsFromDevice(ptz.GetConfigurationsResponse{})
+	if len(functions) != 0 || canZoom || canPanTilt {
+		t.Fatalf("GetPTZFunctionsFromDevice() = %v, %t, %t; want empty capabilities", functions, canZoom, canPanTilt)
+	}
+}

machinery/src/outputs/main.go

@@ -1,8 +1,8 @@
 package outputs
 
 import (
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 type Output interface {
@@ -20,36 +20,36 @@ func Execute(message *models.OutputMessage) (err error) {
 			slack := &SlackOutput{}
 			err := slack.Trigger(message)
 			if err == nil {
-				log.Log.Debug("outputs.main.Execute(slack): message was processed by output.")
+				log.Debug("outputs.main.Execute(slack): message was processed by output.")
 			} else {
-				log.Log.Error("outputs.main.Execute(slack): " + err.Error())
+				log.Error("outputs.main.Execute(slack): " + err.Error())
 			}
 			break
 		case "webhook":
 			webhook := &WebhookOutput{}
 			err := webhook.Trigger(message)
 			if err == nil {
-				log.Log.Debug("outputs.main.Execute(webhook): message was processed by output.")
+				log.Debug("outputs.main.Execute(webhook): message was processed by output.")
 			} else {
-				log.Log.Error("outputs.main.Execute(webhook): " + err.Error())
+				log.Error("outputs.main.Execute(webhook): " + err.Error())
 			}
 			break
 		case "onvif_relay":
 			onvif := &OnvifRelayOutput{}
 			err := onvif.Trigger(message)
 			if err == nil {
-				log.Log.Debug("outputs.main.Execute(onvif): message was processed by output.")
+				log.Debug("outputs.main.Execute(onvif): message was processed by output.")
 			} else {
-				log.Log.Error("outputs.main.Execute(onvif): " + err.Error())
+				log.Error("outputs.main.Execute(onvif): " + err.Error())
 			}
 			break
 		case "script":
 			script := &ScriptOutput{}
 			err := script.Trigger(message)
 			if err == nil {
-				log.Log.Debug("outputs.main.Execute(script): message was processed by output.")
+				log.Debug("outputs.main.Execute(script): message was processed by output.")
 			} else {
-				log.Log.Error("outputs.main.Execute(script): " + err.Error())
+				log.Error("outputs.main.Execute(script): " + err.Error())
 			}
 			break
 		}

machinery/src/packets/queue.go

@@ -2,6 +2,7 @@
 package packets
 
 import (
+	"context"
 	"io"
 	"sync"
 )
@@ -201,7 +202,26 @@ func (self *QueueCursor) Streams() (streams []Stream, err error) {
 
 // ReadPacket will not consume packets in Queue, it's just a cursor.
 func (self *QueueCursor) ReadPacket() (pkt Packet, err error) {
+	return self.readPacket(context.Background(), false)
+}
+
+// ReadPacketContext waits for the next packet or returns when ctx is cancelled.
+func (self *QueueCursor) ReadPacketContext(ctx context.Context) (pkt Packet, err error) {
+	if err := ctx.Err(); err != nil {
+		return Packet{}, err
+	}
+	return self.readPacket(ctx, true)
+}
+
+func (self *QueueCursor) readPacket(ctx context.Context, cancellable bool) (pkt Packet, err error) {
 	self.que.cond.L.Lock()
+	var stop func() bool
+	defer func() {
+		self.que.cond.L.Unlock()
+		if stop != nil {
+			stop()
+		}
+	}()
 	buf := self.que.buf
 	if !self.gotpos {
 		self.pos = self.init(buf, self.que.videoidx)
@@ -222,8 +242,21 @@ func (self *QueueCursor) ReadPacket() (pkt Packet, err error) {
 			err = io.EOF
 			break
 		}
+		if cancellable {
+			if err = ctx.Err(); err != nil {
+				break
+			}
+			if stop == nil {
+				stop = context.AfterFunc(ctx, func() {
+					// The write lock prevents cancellation from being signalled
+					// between the context check above and cond.Wait.
+					self.que.lock.Lock()
+					self.que.cond.Broadcast()
+					self.que.lock.Unlock()
+				})
+			}
+		}
 		self.que.cond.Wait()
 	}
-	self.que.cond.L.Unlock()
 	return
 }

machinery/src/packets/queue_test.go

@@ -0,0 +1,109 @@
+package packets
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+type observedCancelContext struct {
+	done        chan struct{}
+	cancelled   atomic.Bool
+	errCalls    atomic.Int32
+	waiting     chan struct{}
+	waitingOnce sync.Once
+}
+
+func newObservedCancelContext() *observedCancelContext {
+	return &observedCancelContext{
+		done:    make(chan struct{}),
+		waiting: make(chan struct{}),
+	}
+}
+
+func (c *observedCancelContext) Deadline() (time.Time, bool) { return time.Time{}, false }
+func (c *observedCancelContext) Done() <-chan struct{}       { return c.done }
+func (c *observedCancelContext) Value(any) any               { return nil }
+
+func (c *observedCancelContext) Err() error {
+	if c.errCalls.Add(1) >= 2 {
+		c.waitingOnce.Do(func() { close(c.waiting) })
+	}
+	if c.cancelled.Load() {
+		return context.Canceled
+	}
+	return nil
+}
+
+func (c *observedCancelContext) Cancel() {
+	c.cancelled.Store(true)
+	close(c.done)
+}
+
+func TestQueueCursorReadPacketContextCancelsWhileQueueIsIdle(t *testing.T) {
+	queue := NewQueue()
+	cursor := queue.Latest()
+	ctx := newObservedCancelContext()
+	done := make(chan error, 1)
+
+	go func() {
+		_, err := cursor.ReadPacketContext(ctx)
+		done <- err
+	}()
+
+	select {
+	case <-ctx.waiting:
+	case <-time.After(time.Second):
+		t.Fatal("ReadPacketContext() did not reach the queue wait")
+	}
+	ctx.Cancel()
+	select {
+	case err := <-done:
+		if !errors.Is(err, context.Canceled) {
+			t.Fatalf("ReadPacketContext() error = %v, want %v", err, context.Canceled)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("ReadPacketContext() did not return after cancellation")
+	}
+}
+
+func TestQueueCursorReadPacketDoesNotAllocate(t *testing.T) {
+	queue := NewQueue()
+	for i := 0; i < 200; i++ {
+		if err := queue.WritePacket(Packet{Data: []byte{1}}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	cursor := queue.Oldest()
+
+	allocations := testing.AllocsPerRun(100, func() {
+		if _, err := cursor.ReadPacket(); err != nil {
+			t.Fatal(err)
+		}
+	})
+	if allocations != 0 {
+		t.Fatalf("ReadPacket() allocations = %.2f, want 0", allocations)
+	}
+}
+
+func TestQueueCursorReadPacketContextDoesNotAllocateWhenPacketIsAvailable(t *testing.T) {
+	queue := NewQueue()
+	for i := 0; i < 200; i++ {
+		if err := queue.WritePacket(Packet{Data: []byte{1}}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	cursor := queue.Oldest()
+
+	allocations := testing.AllocsPerRun(100, func() {
+		if _, err := cursor.ReadPacketContext(context.Background()); err != nil {
+			t.Fatal(err)
+		}
+	})
+	if allocations != 0 {
+		t.Fatalf("ReadPacketContext() allocations = %.2f, want 0", allocations)
+	}
+}

machinery/src/routers/http/health.go

@@ -0,0 +1,126 @@
+package http
+
+import (
+	"math"
+	stdhttp "net/http"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/utils"
+)
+
+const (
+	healthApplicationStatus = "get_success"
+	healthEntityStatus      = "healthy"
+	hubConnectionFreshness  = 3 * time.Minute
+)
+
+// StreamResolution is the most recently observed encoded video size in pixels.
+type StreamResolution struct {
+	Width  int64 `json:"width"`
+	Height int64 `json:"height"`
+}
+
+// StreamHealth describes the most recently observed state of one camera stream.
+type StreamHealth struct {
+	Configured        bool             `json:"configured"`
+	Connected         bool             `json:"connected"`
+	PackagesProcessed uint64           `json:"packagesProcessed"`
+	FPS               float64          `json:"fps"`
+	Resolution        StreamResolution `json:"resolution"`
+	LastPacketAt      int64            `json:"lastPacketAt"`
+}
+
+// HubHealth describes the Agent's periodic HTTP heartbeat connection to Hub.
+type HubHealth struct {
+	Configured                bool  `json:"configured"`
+	Connected                 bool  `json:"connected"`
+	LastHeartbeatAttemptAt    int64 `json:"lastHeartbeatAttemptAt"`
+	LastSuccessfulHeartbeatAt int64 `json:"lastSuccessfulHeartbeatAt"`
+}
+
+// Health describes the Agent process health exposed to API clients.
+type Health struct {
+	Description     string       `json:"description"`
+	CameraConnected bool         `json:"cameraConnected"`
+	MainStream      StreamHealth `json:"mainStream"`
+	SubStream       StreamHealth `json:"subStream"`
+	Hub             HubHealth    `json:"hub"`
+}
+
+// HealthResponseData contains the typed payload of a health response.
+type HealthResponseData struct {
+	Health Health `json:"health"`
+}
+
+// HealthResponse is the standard successful response returned by GET /health.
+type HealthResponse struct {
+	SuccessResponse
+	Data HealthResponseData `json:"data"`
+}
+
+// HealthCheck godoc
+// @Summary Check Agent health
+// @Description Confirms that the Agent HTTP process can serve requests and reports current camera stream and Hub heartbeat diagnostics. Operational dependency failures do not change the liveness HTTP status.
+// @Tags health
+// @Produce json
+// @Success 200 {object} HealthResponse
+// @Router /health [get]
+func HealthCheck(c *gin.Context, communication *models.Communication) {
+	c.Header("Cache-Control", "no-store")
+	c.JSON(stdhttp.StatusOK, HealthResponse{
+		SuccessResponse: NewSuccessResponse(
+			stdhttp.StatusOK,
+			healthApplicationStatus,
+			healthEntityStatus,
+			"Healthy",
+			ResponseMetadata{
+				ApplicationName:    "agent",
+				ApplicationVersion: utils.VERSION,
+				Path:               c.Request.URL.Path,
+			},
+		),
+		Data: HealthResponseData{
+			Health: buildHealth(communication, time.Now()),
+		},
+	})
+}
+
+func buildHealth(communication *models.Communication, now time.Time) Health {
+	mainTelemetry := communication.StreamRuntimeTelemetry(models.MainStream)
+	subTelemetry := communication.StreamRuntimeTelemetry(models.SubStream)
+	hubTelemetry := communication.HubRuntimeTelemetry()
+
+	hubConnected := hubTelemetry.Configured &&
+		hubTelemetry.Connected &&
+		hubTelemetry.LastSuccessfulHeartbeatAt > 0 &&
+		now.Unix()-hubTelemetry.LastSuccessfulHeartbeatAt <= int64(hubConnectionFreshness/time.Second)
+
+	return Health{
+		Description:     "Agent HTTP service is healthy",
+		CameraConnected: communication.CameraConnected.Load(),
+		MainStream:      newStreamHealth(mainTelemetry, communication.MainStreamConnected.Load()),
+		SubStream:       newStreamHealth(subTelemetry, communication.SubStreamConnected.Load()),
+		Hub: HubHealth{
+			Configured:                hubTelemetry.Configured,
+			Connected:                 hubConnected,
+			LastHeartbeatAttemptAt:    hubTelemetry.LastHeartbeatAttemptAt,
+			LastSuccessfulHeartbeatAt: hubTelemetry.LastSuccessfulHeartbeatAt,
+		},
+	}
+}
+
+func newStreamHealth(telemetry models.StreamRuntimeTelemetry, connected bool) StreamHealth {
+	return StreamHealth{
+		Configured:        telemetry.Configured,
+		Connected:         connected,
+		PackagesProcessed: telemetry.PackagesProcessed,
+		FPS:               math.Round(telemetry.FPS*100) / 100,
+		Resolution: StreamResolution{
+			Width:  telemetry.Width,
+			Height: telemetry.Height,
+		},
+		LastPacketAt: telemetry.LastPacketAt,
+	}
+}

machinery/src/routers/http/health_test.go

@@ -0,0 +1,117 @@
+package http
+
+import (
+	"encoding/json"
+	stdhttp "net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	jwt "github.com/appleboy/gin-jwt/v2"
+	"github.com/gin-gonic/gin"
+	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/utils"
+)
+
+func TestHealthCheckReturnsStandardPublicResponse(t *testing.T) {
+	gin.SetMode(gin.TestMode)
+
+	middlewareConfig := JWTMiddleWare()
+	authMiddleware, err := jwt.New(&middlewareConfig)
+	if err != nil {
+		t.Fatalf("initialize JWT middleware: %v", err)
+	}
+
+	now := time.Now()
+	communication := &models.Communication{}
+	communication.SetStreamConfigured(models.MainStream, true)
+	communication.SetStreamConfigured(models.SubStream, true)
+	communication.RecordStreamPackage(models.MainStream, 29.969, 1920, 1080, now.Add(-2*time.Second))
+	communication.RecordStreamPackage(models.MainStream, 29.969, 1920, 1080, now.Add(-time.Second))
+	communication.RecordStreamPackage(models.SubStream, 15, 640, 360, now.Add(-time.Second))
+	communication.CameraConnected.Store(true)
+	communication.MainStreamConnected.Store(true)
+	communication.SubStreamConnected.Store(true)
+	communication.SetHubConfigured(true)
+	communication.RecordHubHeartbeatAttempt(now.Add(-2 * time.Second))
+	communication.RecordHubHeartbeatSuccess(now.Add(-time.Second))
+
+	router := gin.New()
+	AddRoutes(router, authMiddleware, "", nil, communication, nil)
+
+	response := httptest.NewRecorder()
+	request := httptest.NewRequest(stdhttp.MethodGet, "/health", nil)
+	router.ServeHTTP(response, request)
+
+	if response.Code != stdhttp.StatusOK {
+		t.Fatalf("status = %d, want %d; body=%s", response.Code, stdhttp.StatusOK, response.Body.String())
+	}
+	if got := response.Header().Get("Cache-Control"); got != "no-store" {
+		t.Fatalf("Cache-Control = %q, want %q", got, "no-store")
+	}
+
+	var body HealthResponse
+	if err := json.Unmarshal(response.Body.Bytes(), &body); err != nil {
+		t.Fatalf("decode health response: %v; body=%s", err, response.Body.String())
+	}
+
+	if body.HTTPStatusCode != stdhttp.StatusOK {
+		t.Errorf("httpStatusCode = %d, want %d", body.HTTPStatusCode, stdhttp.StatusOK)
+	}
+	if body.ApplicationStatusCode != healthApplicationStatus {
+		t.Errorf("applicationStatusCode = %q, want %q", body.ApplicationStatusCode, healthApplicationStatus)
+	}
+	if body.EntityStatusCode != healthEntityStatus {
+		t.Errorf("entityStatusCode = %q, want %q", body.EntityStatusCode, healthEntityStatus)
+	}
+	if body.Message != "Healthy" {
+		t.Errorf("message = %q, want %q", body.Message, "Healthy")
+	}
+	if body.Metadata.ApplicationName != "agent" {
+		t.Errorf("metadata.applicationName = %q, want %q", body.Metadata.ApplicationName, "agent")
+	}
+	if body.Metadata.ApplicationVersion != utils.VERSION {
+		t.Errorf("metadata.applicationVersion = %q, want %q", body.Metadata.ApplicationVersion, utils.VERSION)
+	}
+	if body.Metadata.Timestamp <= 0 {
+		t.Errorf("metadata.timestamp = %d, want a positive Unix timestamp", body.Metadata.Timestamp)
+	}
+	if body.Metadata.Path != "/health" {
+		t.Errorf("metadata.path = %q, want %q", body.Metadata.Path, "/health")
+	}
+	health := body.Data.Health
+	if health.Description != "Agent HTTP service is healthy" {
+		t.Errorf("data.health.description = %q, want %q", health.Description, "Agent HTTP service is healthy")
+	}
+	if !health.CameraConnected {
+		t.Error("data.health.cameraConnected = false, want true")
+	}
+	if health.MainStream.PackagesProcessed != 2 || health.MainStream.FPS != 29.97 {
+		t.Errorf("data.health.mainStream = %+v, want 2 packages at 29.97 FPS", health.MainStream)
+	}
+	if health.MainStream.Resolution != (StreamResolution{Width: 1920, Height: 1080}) {
+		t.Errorf("data.health.mainStream.resolution = %+v", health.MainStream.Resolution)
+	}
+	if health.SubStream.PackagesProcessed != 1 || health.SubStream.FPS != 15 {
+		t.Errorf("data.health.subStream = %+v, want 1 package at 15 FPS", health.SubStream)
+	}
+	if health.SubStream.Resolution != (StreamResolution{Width: 640, Height: 360}) {
+		t.Errorf("data.health.subStream.resolution = %+v", health.SubStream.Resolution)
+	}
+	if !health.Hub.Configured || !health.Hub.Connected {
+		t.Errorf("data.health.hub = %+v, want configured and connected", health.Hub)
+	}
+}
+
+func TestBuildHealthMarksStaleHubHeartbeatDisconnected(t *testing.T) {
+	now := time.Unix(1_788_710_500, 0)
+	communication := &models.Communication{}
+	communication.SetHubConfigured(true)
+	communication.RecordHubHeartbeatAttempt(now.Add(-4 * time.Minute))
+	communication.RecordHubHeartbeatSuccess(now.Add(-4 * time.Minute))
+
+	health := buildHealth(communication, now)
+	if health.Hub.Connected {
+		t.Fatalf("Hub health = %+v, want stale heartbeat to be disconnected", health.Hub)
+	}
+}

machinery/src/routers/http/methods.go

@@ -6,9 +6,9 @@ import (
 	"time"
 
 	"github.com/gin-gonic/gin"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/onvif"
+	log "github.com/sirupsen/logrus"
 )
 
 // Login godoc
@@ -598,27 +598,27 @@ func DoTriggerRelayOutput(c *gin.Context) {
 			err := onvif.TriggerRelayOutput(device, output)
 			if err == nil {
 				msg := "relay output triggered: " + output
-				log.Log.Info("routers.http.methods.DoTriggerRelayOutput(): " + msg)
+				log.Info("routers.http.methods.DoTriggerRelayOutput(): " + msg)
 				c.JSON(200, gin.H{
 					"data": msg,
 				})
 			} else {
 				msg := "something went wrong: " + err.Error()
-				log.Log.Error("routers.http.methods.DoTriggerRelayOutput(): " + msg)
+				log.Error("routers.http.methods.DoTriggerRelayOutput(): " + msg)
 				c.JSON(400, gin.H{
 					"data": msg,
 				})
 			}
 		} else {
 			msg := "something went wrong: " + err.Error()
-			log.Log.Error("routers.http.methods.DoTriggerRelayOutput(): " + msg)
+			log.Error("routers.http.methods.DoTriggerRelayOutput(): " + msg)
 			c.JSON(400, gin.H{
 				"data": msg,
 			})
 		}
 	} else {
 		msg := "something went wrong: " + err.Error()
-		log.Log.Error("routers.http.methods.DoTriggerRelayOutput(): " + msg)
+		log.Error("routers.http.methods.DoTriggerRelayOutput(): " + msg)
 		c.JSON(400, gin.H{
 			"data": msg,
 		})

machinery/src/routers/http/response.go

@@ -0,0 +1,41 @@
+package http
+
+import "time"
+
+// ResponseMetadata contains request and application context shared by API responses.
+type ResponseMetadata struct {
+	ApplicationName    string `json:"applicationName,omitempty"`
+	ApplicationVersion string `json:"applicationVersion,omitempty"`
+	Timestamp          int64  `json:"timestamp,omitempty"`
+	Path               string `json:"path,omitempty"`
+	TraceID            string `json:"traceId,omitempty"`
+}
+
+// SuccessResponse is the common envelope for successful Agent API responses.
+type SuccessResponse struct {
+	HTTPStatusCode        int              `json:"httpStatusCode"`
+	ApplicationStatusCode string           `json:"applicationStatusCode"`
+	EntityStatusCode      string           `json:"entityStatusCode"`
+	Message               string           `json:"message"`
+	Metadata              ResponseMetadata `json:"metadata"`
+}
+
+// NewSuccessResponse creates the Hub-compatible success envelope used by new
+// Agent API endpoints.
+func NewSuccessResponse(
+	httpStatusCode int,
+	applicationStatusCode string,
+	entityStatusCode string,
+	message string,
+	metadata ResponseMetadata,
+) SuccessResponse {
+	metadata.Timestamp = time.Now().Unix()
+
+	return SuccessResponse{
+		HTTPStatusCode:        httpStatusCode,
+		ApplicationStatusCode: applicationStatusCode,
+		EntityStatusCode:      entityStatusCode,
+		Message:               message,
+		Metadata:              metadata,
+	}
+}

machinery/src/routers/http/routes.go

@@ -14,6 +14,10 @@ import (
 
 func AddRoutes(r *gin.Engine, authMiddleware *jwt.GinJWTMiddleware, configDirectory string, configuration *models.Configuration, communication *models.Communication, captureDevice *capture.Capture) *gin.RouterGroup {
 
+	r.GET("/health", func(c *gin.Context) {
+		HealthCheck(c, communication)
+	})
+
 	r.GET("/ws", func(c *gin.Context) {
 		websocket.WebsocketHandler(c, configuration, communication, captureDevice)
 	})

machinery/src/routers/http/server.go

@@ -1,9 +1,11 @@
 package http
 
 import (
+	"fmt"
 	"io"
 	"os"
 	"strconv"
+	"time"
 
 	jwt "github.com/appleboy/gin-jwt/v2"
 	"github.com/gin-contrib/pprof"
@@ -11,7 +13,7 @@ import (
 	"github.com/gin-gonic/gin"
 
 	//Swagger documentantion
-	"log"
+	log "github.com/sirupsen/logrus"
 
 	_ "github.com/kerberos-io/agent/machinery/docs"
 	"github.com/kerberos-io/agent/machinery/src/capture"
@@ -45,7 +47,17 @@ func StartServer(configDirectory string, configuration *models.Configuration, co
 	gin.SetMode(gin.ReleaseMode)
 
 	// Initialize REST API
-	r := gin.Default()
+	r := gin.New()
+	r.Use(requestLogger(), gin.CustomRecovery(func(c *gin.Context, recovered interface{}) {
+		log.WithFields(log.Fields{
+			"component": "http",
+			"event":     "request_panic",
+			"method":    c.Request.Method,
+			"path":      c.Request.URL.Path,
+			"panic":     fmt.Sprint(recovered),
+		}).Error("HTTP request panicked")
+		c.AbortWithStatus(500)
+	}))
 
 	// Profiler
 	pprof.Register(r)
@@ -60,7 +72,10 @@ func StartServer(configDirectory string, configuration *models.Configuration, co
 	middleWare := JWTMiddleWare()
 	authMiddleware, err := jwt.New(&middleWare)
 	if err != nil {
-		log.Fatal("JWT Error:" + err.Error())
+		log.WithError(err).WithFields(log.Fields{
+			"component": "http",
+			"event":     "jwt_initialization_failed",
+		}).Fatal("Failed to initialize JWT middleware")
 	}
 
 	// Add all routes
@@ -73,7 +88,10 @@ func StartServer(configDirectory string, configuration *models.Configuration, co
 		// Move demo environment variables to environment variables
 		err := os.Rename(demoEnvironmentVariables, environmentVariables)
 		if err != nil {
-			log.Fatal(err)
+			log.WithError(err).WithFields(log.Fields{
+				"component": "http",
+				"event":     "demo_environment_move_failed",
+			}).Fatal("Failed to activate demo UI environment")
 		}
 	}
 
@@ -88,9 +106,54 @@ func StartServer(configDirectory string, configuration *models.Configuration, co
 	})
 
 	// Run the api on port
+	log.WithFields(log.Fields{
+		"component": "http",
+		"event":     "server_starting",
+		"port":      configuration.Port,
+	}).Info("HTTP server starting")
 	err = r.Run(":" + configuration.Port)
 	if err != nil {
-		log.Fatal(err)
+		log.WithError(err).WithFields(log.Fields{
+			"component": "http",
+			"event":     "server_failed",
+			"port":      configuration.Port,
+		}).Fatal("HTTP server stopped")
+	}
+}
+
+func requestLogger() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		startedAt := time.Now()
+		c.Next()
+
+		fields := log.Fields{
+			"component":      "http",
+			"duration_ms":    time.Since(startedAt).Milliseconds(),
+			"event":          "request_completed",
+			"method":         c.Request.Method,
+			"path":           c.Request.URL.Path,
+			"response_bytes": c.Writer.Size(),
+			"status":         c.Writer.Status(),
+		}
+		if log.IsLevelEnabled(log.DebugLevel) {
+			fields["client_ip"] = c.ClientIP()
+			fields["user_agent"] = c.Request.UserAgent()
+		}
+		entry := log.WithFields(fields)
+		if len(c.Errors) > 0 {
+			entry = entry.WithField("request_errors", c.Errors.String())
+		}
+
+		switch status := c.Writer.Status(); {
+		case status >= 500:
+			entry.Error("HTTP request completed")
+		case status >= 400:
+			entry.Warn("HTTP request completed")
+		case c.Request.URL.Path == "/health":
+			entry.Debug("HTTP request completed")
+		default:
+			entry.Info("HTTP request completed")
+		}
 	}
 }
 

machinery/src/routers/http/server_test.go

@@ -0,0 +1,89 @@
+package http
+
+import (
+	"bytes"
+	"encoding/json"
+	stdhttp "net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/gin-gonic/gin"
+	log "github.com/sirupsen/logrus"
+)
+
+func TestRequestLoggerEmitsStructuredFields(t *testing.T) {
+	originalOutput := log.StandardLogger().Out
+	originalFormatter := log.StandardLogger().Formatter
+	originalLevel := log.GetLevel()
+	defer func() {
+		log.SetOutput(originalOutput)
+		log.SetFormatter(originalFormatter)
+		log.SetLevel(originalLevel)
+	}()
+
+	var output bytes.Buffer
+	log.SetOutput(&output)
+	log.SetFormatter(&log.JSONFormatter{})
+	log.SetLevel(log.InfoLevel)
+	gin.SetMode(gin.TestMode)
+
+	router := gin.New()
+	router.Use(requestLogger())
+	router.GET("/status", func(c *gin.Context) {
+		c.Status(stdhttp.StatusNoContent)
+	})
+
+	response := httptest.NewRecorder()
+	request := httptest.NewRequest(stdhttp.MethodGet, "/status?token=do-not-log", nil)
+	router.ServeHTTP(response, request)
+
+	var entry map[string]interface{}
+	if err := json.Unmarshal(output.Bytes(), &entry); err != nil {
+		t.Fatalf("decode request log: %v; output=%q", err, output.String())
+	}
+	for key, want := range map[string]interface{}{
+		"component": "http",
+		"event":     "request_completed",
+		"method":    stdhttp.MethodGet,
+		"path":      "/status",
+		"status":    float64(stdhttp.StatusNoContent),
+	} {
+		if got := entry[key]; got != want {
+			t.Fatalf("%s = %v, want %v", key, got, want)
+		}
+	}
+	if bytes.Contains(output.Bytes(), []byte("do-not-log")) {
+		t.Fatalf("request log exposed query parameters: %q", output.String())
+	}
+}
+
+func TestRequestLoggerKeepsSuccessfulHealthChecksBelowInfo(t *testing.T) {
+	originalOutput := log.StandardLogger().Out
+	originalFormatter := log.StandardLogger().Formatter
+	originalLevel := log.GetLevel()
+	defer func() {
+		log.SetOutput(originalOutput)
+		log.SetFormatter(originalFormatter)
+		log.SetLevel(originalLevel)
+	}()
+
+	var output bytes.Buffer
+	log.SetOutput(&output)
+	log.SetFormatter(&log.JSONFormatter{})
+	log.SetLevel(log.InfoLevel)
+	gin.SetMode(gin.TestMode)
+
+	router := gin.New()
+	router.Use(requestLogger())
+	router.GET("/health", func(c *gin.Context) {
+		c.Status(stdhttp.StatusNoContent)
+	})
+
+	response := httptest.NewRecorder()
+	request := httptest.NewRequest(stdhttp.MethodGet, "/health", nil)
+	router.ServeHTTP(response, request)
+
+	if output.Len() != 0 {
+		t.Fatalf("successful health check emitted an info log: %q", output.String())
+	}
+}

machinery/src/routers/mqtt/main.go

@@ -1,7 +1,9 @@
 package mqtt
 
 import (
+	"context"
 	"crypto/rsa"
+	"crypto/tls"
 	"crypto/x509"
 	"encoding/base64"
 	"encoding/json"
@@ -9,21 +11,23 @@ import (
 	"fmt"
 	"io/ioutil"
 	"math/rand"
+	"net"
+	"net/url"
+	"os"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
-	"context"
-
 	mqtt "github.com/eclipse/paho.mqtt.golang"
 	"github.com/kerberos-io/agent/machinery/src/capture"
 	configService "github.com/kerberos-io/agent/machinery/src/config"
 	"github.com/kerberos-io/agent/machinery/src/encryption"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/onvif"
 	"github.com/kerberos-io/agent/machinery/src/webrtc"
+	log "github.com/sirupsen/logrus"
+	"golang.org/x/net/proxy"
 )
 
 // We'll cache the MQTT settings to know if we need to reinitialize the MQTT client connection.
@@ -34,6 +38,177 @@ var PREV_MQTTPassword string
 var PREV_HubKey string
 var PREV_AgentKey string
 
+type pahoErrorLogger struct{}
+
+func (pahoErrorLogger) Println(values ...interface{}) {
+	log.WithFields(log.Fields{
+		"component": "routers/mqtt",
+		"event":     "paho_error",
+	}).Error(strings.TrimSpace(fmt.Sprintln(values...)))
+}
+
+func (pahoErrorLogger) Printf(format string, values ...interface{}) {
+	log.WithFields(log.Fields{
+		"component": "routers/mqtt",
+		"event":     "paho_error",
+	}).Errorf(strings.TrimSpace(format), values...)
+}
+
+func init() {
+	mqtt.ERROR = pahoErrorLogger{}
+}
+
+func enableMQTTConnectionDiagnostics(options *mqtt.ClientOptions, brokerURL string) {
+	if !strings.Contains(brokerURL, "://") {
+		options.SetCustomOpenConnectionFn(openMQTTConnection)
+		return
+	}
+
+	parsedURL, err := url.Parse(brokerURL)
+	if err != nil {
+		return
+	}
+
+	switch strings.ToLower(parsedURL.Scheme) {
+	case "", "mqtt", "tcp", "ssl", "tls", "mqtts", "mqtt+ssl", "tcps":
+		options.SetCustomOpenConnectionFn(openMQTTConnection)
+	}
+}
+
+func openMQTTConnection(uri *url.URL, options mqtt.ClientOptions) (net.Conn, error) {
+	host := uri.Hostname()
+	fields := log.Fields{
+		"component": "routers/mqtt",
+		"host":      host,
+		"port":      uri.Port(),
+		"scheme":    uri.Scheme,
+	}
+	logMQTTDNSResolution(host, options.ConnectTimeout, fields)
+
+	connectionStartedAt := time.Now()
+	dialer := options.Dialer
+	if dialer == nil {
+		dialer = &net.Dialer{Timeout: options.ConnectTimeout}
+	}
+
+	proxyConfigured := os.Getenv("all_proxy") != ""
+	proxyMode := "direct"
+	if proxyConfigured {
+		proxyMode = "socks"
+	}
+	fields["proxy_mode"] = proxyMode
+	log.WithFields(fields).Info("Opening MQTT TCP connection")
+
+	var (
+		connection net.Conn
+		err        error
+	)
+	if proxyConfigured {
+		connection, err = proxy.FromEnvironment().Dial("tcp", uri.Host)
+	} else {
+		connection, err = dialer.Dial("tcp", uri.Host)
+	}
+	fields["duration_ms"] = time.Since(connectionStartedAt).Milliseconds()
+	if err != nil {
+		logMQTTNetworkError("MQTT TCP connection failed", err, fields)
+		return nil, err
+	}
+
+	fields["local_address"] = connection.LocalAddr().String()
+	fields["remote_address"] = connection.RemoteAddr().String()
+	log.WithFields(fields).Info("MQTT TCP connection established")
+
+	if !isSecureMQTTScheme(uri.Scheme) {
+		return connection, nil
+	}
+
+	tlsConfig := options.TLSConfig
+	if tlsConfig == nil {
+		tlsConfig = &tls.Config{}
+	} else {
+		tlsConfig = tlsConfig.Clone()
+	}
+	if tlsConfig.ServerName == "" {
+		tlsConfig.ServerName = host
+	}
+
+	tlsConnection := tls.Client(connection, tlsConfig)
+	tlsStartedAt := time.Now()
+	if options.ConnectTimeout > 0 {
+		_ = tlsConnection.SetDeadline(connectionStartedAt.Add(options.ConnectTimeout))
+	}
+	if err = tlsConnection.Handshake(); err != nil {
+		_ = connection.Close()
+		fields["duration_ms"] = time.Since(tlsStartedAt).Milliseconds()
+		fields["server_name"] = tlsConfig.ServerName
+		logMQTTNetworkError("MQTT TLS handshake failed", err, fields)
+		return nil, err
+	}
+	_ = tlsConnection.SetDeadline(time.Time{})
+
+	state := tlsConnection.ConnectionState()
+	fields["cipher_suite"] = tls.CipherSuiteName(state.CipherSuite)
+	fields["duration_ms"] = time.Since(tlsStartedAt).Milliseconds()
+	fields["server_name"] = tlsConfig.ServerName
+	fields["tls_version"] = tls.VersionName(state.Version)
+	log.WithFields(fields).Info("MQTT TLS handshake established")
+	return tlsConnection, nil
+}
+
+func logMQTTDNSResolution(host string, timeout time.Duration, fields log.Fields) {
+	if host == "" || net.ParseIP(host) != nil {
+		return
+	}
+
+	lookupTimeout := timeout
+	if lookupTimeout <= 0 || lookupTimeout > 5*time.Second {
+		lookupTimeout = 5 * time.Second
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), lookupTimeout)
+	defer cancel()
+
+	startedAt := time.Now()
+	addresses, err := net.DefaultResolver.LookupHost(ctx, host)
+	dnsFields := cloneLogFields(fields)
+	dnsFields["duration_ms"] = time.Since(startedAt).Milliseconds()
+	if err != nil {
+		logMQTTNetworkError("MQTT broker DNS resolution failed", err, dnsFields)
+		return
+	}
+
+	dnsFields["resolved_addresses"] = addresses
+	log.WithFields(dnsFields).Info("MQTT broker DNS resolved")
+}
+
+func logMQTTNetworkError(message string, err error, fields log.Fields) {
+	errorFields := cloneLogFields(fields)
+	if networkError, ok := err.(net.Error); ok {
+		errorFields["network_timeout"] = networkError.Timeout()
+	}
+	if operationError, ok := err.(*net.OpError); ok {
+		errorFields["network"] = operationError.Net
+		errorFields["operation"] = operationError.Op
+	}
+	log.WithError(err).WithFields(errorFields).Error(message)
+}
+
+func cloneLogFields(fields log.Fields) log.Fields {
+	cloned := make(log.Fields, len(fields))
+	for key, value := range fields {
+		cloned[key] = value
+	}
+	return cloned
+}
+
+func isSecureMQTTScheme(scheme string) bool {
+	switch strings.ToLower(scheme) {
+	case "ssl", "tls", "mqtts", "mqtt+ssl", "tcps":
+		return true
+	default:
+		return false
+	}
+}
+
 func HasMQTTClientModified(configuration *models.Configuration) bool {
 	MTTURI := configuration.Config.MQTTURI
 	MTTUsername := configuration.Config.MQTTUsername
@@ -41,7 +216,7 @@ func HasMQTTClientModified(configuration *models.Configuration) bool {
 	HubKey := configuration.Config.HubKey
 	AgentKey := configuration.Config.Key
 	if PREV_MQTTURI != MTTURI || PREV_MQTTUsername != MTTUsername || PREV_MQTTPassword != MQTTPassword || PREV_HubKey != HubKey || PREV_AgentKey != AgentKey {
-		log.Log.Info("HasMQTTClientModified: MQTT settings have been modified, restarting MQTT client.")
+		log.Info("HasMQTTClientModified: MQTT settings have been modified, restarting MQTT client.")
 		return true
 	}
 	return false
@@ -59,6 +234,7 @@ func HasMQTTClientModified(configuration *models.Configuration) bool {
 // - kerberos/{hubkey}/device/{devicekey}/motion: a motion signal
 
 func ConfigureMQTT(configDirectory string, configuration *models.Configuration, communication *models.Communication) mqtt.Client {
+	installRemoteAccessHook()
 
 	config := configuration.Config
 
@@ -70,8 +246,25 @@ func ConfigureMQTT(configDirectory string, configuration *models.Configuration,
 	PREV_AgentKey = configuration.Config.Key
 
 	if config.Offline == "true" {
-		log.Log.Info("routers.mqtt.main.ConfigureMQTT(): not starting as running in Offline mode.")
+		log.Info("routers.mqtt.main.ConfigureMQTT(): not starting as running in Offline mode.")
 	} else {
+		hubKey := ""
+		if config.Cloud == "s3" && config.S3 != nil && config.S3.Publickey != "" {
+			hubKey = config.S3.Publickey
+		} else if config.Cloud == "kstorage" && config.KStorage != nil && config.KStorage.CloudKey != "" {
+			hubKey = config.KStorage.CloudKey
+		}
+		if config.HubKey != "" {
+			hubKey = config.HubKey
+		}
+		if hubKey == "" {
+			log.Warn("routers.mqtt.main.ConfigureMQTT(): not starting without a Hub key")
+			return nil
+		}
+		if config.Key == "" {
+			log.Warn("routers.mqtt.main.ConfigureMQTT(): not starting without an Agent key")
+			return nil
+		}
 
 		opts := mqtt.NewClientOptions()
 
@@ -79,7 +272,6 @@ func ConfigureMQTT(configDirectory string, configuration *models.Configuration,
 		// and share and receive messages to/from.
 		mqttURL := config.MQTTURI
 		opts.AddBroker(mqttURL)
-		log.Log.Debug("routers.mqtt.main.ConfigureMQTT(): Set broker uri " + mqttURL)
 
 		// Our MQTT broker can have username/password credentials
 		// to protect it from the outside.
@@ -88,13 +280,19 @@ func ConfigureMQTT(configDirectory string, configuration *models.Configuration,
 		if mqtt_username != "" || mqtt_password != "" {
 			opts.SetUsername(mqtt_username)
 			opts.SetPassword(mqtt_password)
-			log.Log.Debug("routers.mqtt.main.ConfigureMQTT(): Set username " + mqtt_username)
-			log.Log.Debug("routers.mqtt.main.ConfigureMQTT(): Set password " + mqtt_password)
 		}
+		log.WithFields(log.Fields{
+			"broker_configured":   mqttURL != "",
+			"component":           "routers/mqtt",
+			"event":               "credentials_configured",
+			"password_configured": mqtt_password != "",
+			"username_configured": mqtt_username != "",
+		}).Debug("MQTT connection configuration loaded")
 
 		// Some extra options to make sure the connection behaves
 		// properly. More information here: github.com/eclipse/paho.mqtt.golang.
 		//opts.SetCleanSession(true)
+		enableMQTTConnectionDiagnostics(opts, mqttURL)
 		opts.SetCleanSession(false)
 		opts.SetResumeSubs(true)
 		opts.SetStore(mqtt.NewMemoryStore())
@@ -108,65 +306,81 @@ func ConfigureMQTT(configDirectory string, configuration *models.Configuration,
 		opts.SetOrderMatters(false)
 		opts.SetConnectTimeout(30 * time.Second)
 		opts.SetConnectionLostHandler(func(client mqtt.Client, err error) {
+			entry := log.WithFields(log.Fields{
+				"component": "routers/mqtt",
+				"event":     "connection_lost",
+			})
 			if err != nil {
-				log.Log.Error("routers.mqtt.main.ConfigureMQTT(): MQTT connection lost: " + err.Error())
-			} else {
-				log.Log.Error("routers.mqtt.main.ConfigureMQTT(): MQTT connection lost")
+				entry = entry.WithError(err)
 			}
+			entry.Error("MQTT connection lost")
 		})
 		opts.SetReconnectingHandler(func(client mqtt.Client, options *mqtt.ClientOptions) {
-			log.Log.Warning("routers.mqtt.main.ConfigureMQTT(): reconnecting to MQTT broker")
+			log.WithFields(log.Fields{
+				"component": "routers/mqtt",
+				"event":     "reconnecting",
+			}).Warn("Reconnecting to MQTT broker")
 		})
 		opts.SetOnConnectHandler(func(c mqtt.Client) {
-			log.Log.Info("routers.mqtt.main.ConfigureMQTT(): MQTT session is online")
+			log.WithFields(log.Fields{
+				"component": "routers/mqtt",
+				"event":     "session_online",
+			}).Info("MQTT session is online")
 		})
 
-		hubKey := ""
-		// This is the old way ;)
-		if config.Cloud == "s3" && config.S3 != nil && config.S3.Publickey != "" {
-			hubKey = config.S3.Publickey
-		} else if config.Cloud == "kstorage" && config.KStorage != nil && config.KStorage.CloudKey != "" {
-			hubKey = config.KStorage.CloudKey
-		}
-		// This is the new way ;)
-		if config.HubKey != "" {
-			hubKey = config.HubKey
+		rand.Seed(time.Now().UnixNano())
+		random := rand.Intn(100)
+		mqttClientID := config.Key + strconv.Itoa(random) // this random int is to avoid conflicts.
+
+		// This is a worked-around.
+		// current S3 (Kerberos Hub SAAS) is using a secured MQTT, where the client id,
+		// should match the kerberos hub key.
+		if config.Cloud == "s3" {
+			mqttClientID = config.Key
 		}
 
-		if hubKey != "" {
+		opts.SetClientID(mqttClientID)
+		log.WithFields(log.Fields{
+			"component":            "routers/mqtt",
+			"event":                "client_configured",
+			"randomized_client_id": config.Cloud != "s3",
+		}).Debug("MQTT client configured")
+		rand.Seed(time.Now().UnixNano())
 
-			rand.Seed(time.Now().UnixNano())
-			random := rand.Intn(100)
-			mqttClientID := config.Key + strconv.Itoa(random) // this random int is to avoid conflicts.
+		opts.OnConnect = func(c mqtt.Client) {
+			// We managed to connect to the MQTT broker, hurray!
+			log.WithFields(log.Fields{
+				"component": "routers/mqtt",
+				"event":     "connected",
+			}).Info("MQTT broker connected")
+			log.WithFields(log.Fields{
+				"broker_configured": mqttURL != "",
+				"component":         "routers/mqtt",
+				"event":             "connection_details",
+			}).Debug("MQTT connection details")
 
-			// This is a worked-around.
-			// current S3 (Kerberos Hub SAAS) is using a secured MQTT, where the client id,
-			// should match the kerberos hub key.
-			if config.Cloud == "s3" {
-				mqttClientID = config.Key
-			}
-
-			opts.SetClientID(mqttClientID)
-			log.Log.Info("routers.mqtt.main.ConfigureMQTT(): Set ClientID " + mqttClientID)
-			rand.Seed(time.Now().UnixNano())
-
-			opts.OnConnect = func(c mqtt.Client) {
-				// We managed to connect to the MQTT broker, hurray!
-				log.Log.Info("routers.mqtt.main.ConfigureMQTT(): " + mqttClientID + " connected to " + mqttURL)
-
-				// Create a susbcription for listen and reply
-				MQTTListenerHandler(c, hubKey, configDirectory, configuration, communication)
-			}
+			// Create a susbcription for listen and reply
+			MQTTListenerHandler(c, hubKey, configDirectory, configuration, communication)
 		}
 		mqc := mqtt.NewClient(opts)
 		if token := mqc.Connect(); token.WaitTimeout(30 * time.Second) {
 			if token.Error() != nil {
-				log.Log.Error("routers.mqtt.main.ConfigureMQTT(): unable to establish mqtt broker connection, error was: " + token.Error().Error())
+				log.WithError(token.Error()).WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "initial_connection_failed",
+				}).Error("Failed to establish initial MQTT connection")
 			} else {
-				log.Log.Info("routers.mqtt.main.ConfigureMQTT(): initial MQTT connection established")
+				log.WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "initial_connection_established",
+				}).Info("Initial MQTT connection established")
 			}
 		} else {
-			log.Log.Error("routers.mqtt.main.ConfigureMQTT(): timed out while establishing mqtt broker connection")
+			log.WithFields(log.Fields{
+				"component":  "routers/mqtt",
+				"event":      "initial_connection_timeout",
+				"timeout_ms": (30 * time.Second).Milliseconds(),
+			}).Warn("Initial MQTT connection is still retrying")
 		}
 		return mqc
 	}
@@ -216,7 +430,7 @@ func markHDSessionSeen(sessionID string) bool {
 
 func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory string, configuration *models.Configuration, communication *models.Communication) {
 	if hubKey == "" {
-		log.Log.Info("routers.mqtt.main.MQTTListenerHandler(): no hub key provided, not subscribing to kerberos/hub/{hubkey}")
+		log.Info("routers.mqtt.main.MQTTListenerHandler(): no hub key provided, not subscribing to kerberos/hub/{hubkey}")
 	} else {
 		agentListener := fmt.Sprintf("kerberos/agent/%s", hubKey)
 		token := mqttClient.Subscribe(agentListener, 1, func(c mqtt.Client, msg mqtt.Message) {
@@ -236,6 +450,7 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 			// We will receive all messages from our hub, so we'll need to filter to the relevant device.
 			if message.Mid != "" && message.Timestamp != 0 && message.DeviceId == configuration.Config.Key {
 				var payload models.Payload
+				remoteAuthenticated := false
 
 				// Messages might be hidden, if so we'll need to decrypt them using the Kerberos Hub private key.
 				if message.Hidden && configuration.Config.HubEncryption == "true" {
@@ -249,13 +464,15 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 							}
 							visibleValue, err := encryption.AesDecrypt(data, privateKey)
 							if err != nil {
-								log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message: " + err.Error())
+								log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message: " + err.Error())
 								return
 							}
-							json.Unmarshal(visibleValue, &payload)
-							message.Payload = payload
+							if err := json.Unmarshal(visibleValue, &payload); err == nil {
+								message.Payload = payload
+								remoteAuthenticated = true
+							}
 						} else {
-							log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message, no private key provided.")
+							log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message, no private key provided.")
 						}
 					}
 				}
@@ -271,14 +488,14 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 						pemBytes, _ := ioutil.ReadAll(r)
 						block, _ := pem.Decode(pemBytes)
 						if block == nil {
-							log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error decoding PEM block containing private key")
+							log.Error("routers.mqtt.main.MQTTListenerHandler(): error decoding PEM block containing private key")
 							return
 						} else {
 							// Parse private key
 							b := block.Bytes
 							key, err := x509.ParsePKCS8PrivateKey(b)
 							if err != nil {
-								log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error parsing private key: " + err.Error())
+								log.Error("routers.mqtt.main.MQTTListenerHandler(): error parsing private key: " + err.Error())
 								return
 							} else {
 								// Conver key to *rsa.PrivateKey
@@ -298,16 +515,18 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 										}
 										decryptedValue, err := encryption.AesDecrypt(data, string(decryptedKey))
 										if err != nil {
-											log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message: " + err.Error())
+											log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message: " + err.Error())
 											return
 										}
-										json.Unmarshal(decryptedValue, &payload)
+										if err := json.Unmarshal(decryptedValue, &payload); err == nil {
+											remoteAuthenticated = true
+										}
 									} else {
-										log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message, assymetric keys do not match.")
+										log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message, assymetric keys do not match.")
 										return
 									}
 								} else if err != nil {
-									log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message: " + err.Error())
+									log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message: " + err.Error())
 									return
 								}
 							}
@@ -318,7 +537,11 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 				}
 
 				// We'll find out which message we received, and act accordingly.
-				log.Log.Info("routers.mqtt.main.MQTTListenerHandler(): received message with action: " + payload.Action)
+				log.WithFields(log.Fields{
+					"action":    payload.Action,
+					"component": "routers/mqtt",
+					"event":     "message_received",
+				}).Debug("MQTT message received")
 
 				// NOTE: We intentionally do NOT discard request-hd-stream /
 				// receive-hd-candidates messages based on a wall-clock age. The
@@ -355,6 +578,14 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 					go HandleReceiveHDCandidates(mqttClient, hubKey, payload, configuration, communication)
 				case "trigger-relay":
 					go HandleTriggerRelay(mqttClient, hubKey, payload, configuration, communication)
+				case "remote-session-open":
+					go HandleRemoteSessionOpen(mqttClient, hubKey, payload, remoteAuthenticated, configuration)
+				case "remote-session-input":
+					go HandleRemoteSessionInput(mqttClient, hubKey, payload, remoteAuthenticated, configuration)
+				case "remote-session-resize":
+					go HandleRemoteSessionResize(payload, remoteAuthenticated)
+				case "remote-session-close":
+					go HandleRemoteSessionClose(payload, remoteAuthenticated)
 				}
 
 			}
@@ -362,12 +593,15 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 
 		if token.WaitTimeout(10 * time.Second) {
 			if token.Error() != nil {
-				log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): failed to subscribe to " + agentListener + ": " + token.Error().Error())
+				log.WithError(token.Error()).WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "subscription_failed",
+				}).Error("Failed to subscribe to Agent MQTT topic")
 			} else {
-				log.Log.Info("routers.mqtt.main.MQTTListenerHandler(): subscribed to " + agentListener)
+				log.Info("routers.mqtt.main.MQTTListenerHandler(): subscribed to " + agentListener)
 			}
 		} else {
-			log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): timed out while subscribing to " + agentListener)
+			log.Error("routers.mqtt.main.MQTTListenerHandler(): timed out while subscribing to " + agentListener)
 		}
 	}
 }
@@ -396,9 +630,9 @@ func HandleRecording(mqttClient mqtt.Client, hubKey string, payload models.Paylo
 			if communication.IsRecordingManual.IsSet() {
 				communication.RecordingManualHeartbeat.Store(now)
 				communication.RecordingManualHeartbeatSeen.Set()
-				log.Log.Debug("routers.mqtt.main.HandleRecording(): manual recording heartbeat received.")
+				log.Debug("routers.mqtt.main.HandleRecording(): manual recording heartbeat received.")
 			} else {
-				log.Log.Debug("routers.mqtt.main.HandleRecording(): ignoring heartbeat, no active manual recording.")
+				log.Debug("routers.mqtt.main.HandleRecording(): ignoring heartbeat, no active manual recording.")
 			}
 		} else {
 			// Explicit start from the live view (record button). Start a manual
@@ -410,18 +644,16 @@ func HandleRecording(mqttClient mqtt.Client, hubKey string, payload models.Paylo
 			if communication.IsRecordingManual.SetToIf(false, true) {
 				communication.RecordingManualStart.Store(now)
 				communication.RecordingManualHeartbeatSeen.UnSet()
-				log.Log.Info("routers.mqtt.main.HandleRecording(): manual recording started.")
-				select {
-				case communication.HandleMotion <- models.MotionDataPartial{Timestamp: timestamp, NumberOfChanges: 100000000}:
-				default:
-					log.Log.Warning("routers.mqtt.main.HandleRecording(): motion channel full, manual recording start not queued.")
+				log.Info("routers.mqtt.main.HandleRecording(): manual recording started.")
+				if !communication.TrySendMotion(models.MotionDataPartial{Timestamp: timestamp, NumberOfChanges: 100000000}) {
+					log.Warn("routers.mqtt.main.HandleRecording(): motion channel full, manual recording start not queued.")
 				}
 			}
 		}
 	} else {
 		// Stop the manual recording; the motion recorder closes the clip once the
 		// post-recording window elapses. Clear the heartbeat/start markers too.
-		log.Log.Info("routers.mqtt.main.HandleRecording(): manual recording stopped.")
+		log.Info("routers.mqtt.main.HandleRecording(): manual recording stopped.")
 		communication.IsRecordingManual.UnSet()
 		communication.RecordingManualHeartbeat.Store(0)
 		communication.RecordingManualStart.Store(0)
@@ -443,7 +675,7 @@ func HandleAudio(mqttClient mqtt.Client, hubKey string, payload models.Payload,
 			Data:      audioPayload.Data,
 		}
 		if enqueueLatestAudio(communication.HandleAudio, audioDataPartial) {
-			log.Log.Debug("routers.mqtt.main.HandleAudio(): dropped stale audio because the backchannel queue was full")
+			log.Debug("routers.mqtt.main.HandleAudio(): dropped stale audio because the backchannel queue was full")
 		}
 	}
 }
@@ -479,7 +711,7 @@ func HandleGetPTZPosition(mqttClient mqtt.Client, hubKey string, payload models.
 		// Get Position from device
 		pos, err := onvif.GetPositionFromDevice(*configuration)
 		if err != nil {
-			log.Log.Error("routers.mqtt.main.HandlePTZPosition(): error getting position from device: " + err.Error())
+			log.Error("routers.mqtt.main.HandlePTZPosition(): error getting position from device: " + err.Error())
 		} else {
 			// Needs to wrapped!
 			posString := fmt.Sprintf("%f,%f,%f", pos.PanTilt.X, pos.PanTilt.Y, pos.Zoom.X)
@@ -497,7 +729,10 @@ func HandleGetPTZPosition(mqttClient mqtt.Client, hubKey string, payload models.
 			if err == nil {
 				mqttClient.Publish("kerberos/hub/"+hubKey, 2, false, payload)
 			} else {
-				log.Log.Info("routers.mqtt.main.HandlePTZPosition(): something went wrong while sending position to hub: " + string(payload))
+				log.WithError(err).WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "ptz_position_packaging_failed",
+				}).Error("Failed to package PTZ position message")
 			}
 		}
 	}
@@ -512,11 +747,14 @@ func HandleUpdatePTZPosition(mqttClient mqtt.Client, hubKey string, payload mode
 	json.Unmarshal(jsonData, &onvifAction)
 
 	if onvifAction.Action != "" {
-		if communication.CameraConnected {
-			communication.HandleONVIF <- onvifAction
-			log.Log.Info("routers.mqtt.main.MQTTListenerHandleONVIF(): Received an action - " + onvifAction.Action)
+		if communication.CameraConnected.Load() {
+			if communication.TrySendONVIF(onvifAction) {
+				log.Info("routers.mqtt.main.MQTTListenerHandleONVIF(): Received an action - " + onvifAction.Action)
+			} else {
+				log.Warn("routers.mqtt.main.MQTTListenerHandleONVIF(): action channel unavailable or full")
+			}
 		} else {
-			log.Log.Info("routers.mqtt.main.MQTTListenerHandleONVIF(): received action, but camera is not connected.")
+			log.Info("routers.mqtt.main.MQTTListenerHandleONVIF(): received action, but camera is not connected.")
 		}
 	}
 }
@@ -561,14 +799,17 @@ func HandleRequestConfig(mqttClient mqtt.Client, hubKey string, payload models.P
 			if err == nil {
 				mqttClient.Publish("kerberos/hub/"+hubKey, 2, false, payload)
 			} else {
-				log.Log.Info("routers.mqtt.main.HandleRequestConfig(): something went wrong while sending config to hub: " + string(payload))
+				log.WithError(err).WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "configuration_packaging_failed",
+				}).Error("Failed to package configuration message")
 			}
 
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleRequestConfig(): no config available")
+			log.Info("routers.mqtt.main.HandleRequestConfig(): no config available")
 		}
 
-		log.Log.Info("routers.mqtt.main.HandleRequestConfig(): Received a request for the config")
+		log.Info("routers.mqtt.main.HandleRequestConfig(): Received a request for the config")
 	}
 }
 
@@ -660,7 +901,10 @@ func HandleVerifyStream(mqttClient mqtt.Client, hubKey string, payload models.Pa
 	if err == nil {
 		mqttClient.Publish("kerberos/hub/"+hubKey, 2, false, packagedPayload)
 	} else {
-		log.Log.Info("routers.mqtt.main.HandleVerifyStream(): something went wrong while sending result to hub: " + string(packagedPayload))
+		log.WithError(err).WithFields(log.Fields{
+			"component": "routers/mqtt",
+			"event":     "stream_verification_packaging_failed",
+		}).Error("Failed to package stream verification result")
 	}
 }
 
@@ -681,7 +925,7 @@ func HandleUpdateConfig(mqttClient mqtt.Client, hubKey string, payload models.Pa
 
 		err := configService.SaveConfig(configDirectory, config, configuration, communication)
 		if err == nil {
-			log.Log.Info("routers.mqtt.main.HandleUpdateConfig(): Config updated")
+			log.Info("routers.mqtt.main.HandleUpdateConfig(): Config updated")
 			message := models.Message{
 				Payload: models.Payload{
 					Action:   "acknowledge-update-config",
@@ -692,10 +936,13 @@ func HandleUpdateConfig(mqttClient mqtt.Client, hubKey string, payload models.Pa
 			if err == nil {
 				mqttClient.Publish("kerberos/hub/"+hubKey, 2, false, payload)
 			} else {
-				log.Log.Info("routers.mqtt.main.HandleUpdateConfig(): something went wrong while sending acknowledge config to hub: " + string(payload))
+				log.WithError(err).WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "configuration_acknowledgement_packaging_failed",
+				}).Error("Failed to package configuration acknowledgement")
 			}
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleUpdateConfig(): Config update failed")
+			log.Info("routers.mqtt.main.HandleUpdateConfig(): Config update failed")
 		}
 	}
 }
@@ -708,7 +955,7 @@ func HandleRequestSDStream(mqttClient mqtt.Client, hubKey string, payload models
 	json.Unmarshal(jsonData, &requestSDStreamPayload)
 
 	if requestSDStreamPayload.Timestamp != 0 {
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			// A viewer that opted into the HTTP transport is signalled on a separate
 			// channel so the producer ships its frames to hub-api over HTTP instead of
 			// publishing them over MQTT. Any other (or absent) transport keeps the
@@ -724,9 +971,9 @@ func HandleRequestSDStream(mqttClient mqtt.Client, hubKey string, payload models
 				default:
 				}
 			}
-			log.Log.Info("routers.mqtt.main.HandleRequestSDStream(): received request to livestream.")
+			log.Info("routers.mqtt.main.HandleRequestSDStream(): received request to livestream.")
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleRequestSDStream(): received request to livestream, but camera is not connected.")
+			log.Info("routers.mqtt.main.HandleRequestSDStream(): received request to livestream, but camera is not connected.")
 		}
 	}
 }
@@ -743,7 +990,7 @@ func HandleRequestHLSStream(mqttClient mqtt.Client, hubKey string, payload model
 	json.Unmarshal(jsonData, &requestHLSStreamPayload)
 
 	if requestHLSStreamPayload.Timestamp != 0 {
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			// Forward the requested quality ("auto"|"high"|"low"; empty => auto) so
 			// the producer can switch the live session between the main and sub
 			// stream on demand. The send doubles as the viewer keepalive.
@@ -751,9 +998,9 @@ func HandleRequestHLSStream(mqttClient mqtt.Client, hubKey string, payload model
 			case communication.HandleLiveHLS <- requestHLSStreamPayload.Quality:
 			default:
 			}
-			log.Log.Info("routers.mqtt.main.HandleRequestHLSStream(): received request to livestream over HLS.")
+			log.Info("routers.mqtt.main.HandleRequestHLSStream(): received request to livestream over HLS.")
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleRequestHLSStream(): received request to livestream over HLS, but camera is not connected.")
+			log.Info("routers.mqtt.main.HandleRequestHLSStream(): received request to livestream over HLS, but camera is not connected.")
 		}
 	}
 }
@@ -766,29 +1013,30 @@ func HandleRequestHDStream(mqttClient mqtt.Client, hubKey string, payload models
 	json.Unmarshal(jsonData, &requestHDStreamPayload)
 
 	if requestHDStreamPayload.Timestamp != 0 {
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			// Dedupe by session_id: the viewer republishes its offer while
 			// waiting for an answer (and the broker may redeliver), and we
 			// don't want to spawn multiple peer connections for the same
 			// browser session.
 			if markHDSessionSeen(requestHDStreamPayload.SessionID) {
-				log.Log.Info("routers.mqtt.main.HandleRequestHDStream(): duplicate request for session " +
-					requestHDStreamPayload.SessionID + ", ignoring")
+				log.WithFields(log.Fields{
+					"component":  "routers/mqtt",
+					"event":      "duplicate_hd_stream_request",
+					"session_id": requestHDStreamPayload.SessionID,
+				}).Debug("Ignoring duplicate HD stream request")
 				return
 			}
 			// Set the Hub key, so we can send back the answer.
 			requestHDStreamPayload.HubKey = hubKey
-			if communication.HandleLiveHDHandshake == nil {
-				log.Log.Error("routers.mqtt.main.HandleRequestHDStream(): handshake channel is nil, dropping request")
+			if !communication.TrySendLiveHDHandshake(models.LiveHDHandshake{
+				Payload: requestHDStreamPayload,
+			}) {
+				log.Error("routers.mqtt.main.HandleRequestHDStream(): handshake channel unavailable or full, dropping request")
 				return
 			}
-
-			communication.HandleLiveHDHandshake <- models.LiveHDHandshake{
-				Payload: requestHDStreamPayload,
-			}
-			log.Log.Info("routers.mqtt.main.HandleRequestHDStream(): received request to setup webrtc.")
+			log.Info("routers.mqtt.main.HandleRequestHDStream(): received request to setup webrtc.")
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleRequestHDStream(): received request to setup webrtc, but camera is not connected.")
+			log.Info("routers.mqtt.main.HandleRequestHDStream(): received request to setup webrtc, but camera is not connected.")
 		}
 	}
 }
@@ -801,12 +1049,12 @@ func HandleReceiveHDCandidates(mqttClient mqtt.Client, hubKey string, payload mo
 	json.Unmarshal(jsonData, &receiveHDCandidatesPayload)
 
 	if receiveHDCandidatesPayload.Timestamp != 0 {
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			// Register candidate channel
 			key := configuration.Config.Key + "/" + receiveHDCandidatesPayload.SessionID
 			go webrtc.RegisterCandidates(key, receiveHDCandidatesPayload)
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleReceiveHDCandidates(): received candidate, but camera is not connected.")
+			log.Info("routers.mqtt.main.HandleReceiveHDCandidates(): received candidate, but camera is not connected.")
 		}
 	}
 }
@@ -818,14 +1066,17 @@ func HandleNavigatePTZ(mqttClient mqtt.Client, hubKey string, payload models.Pay
 	json.Unmarshal(jsonData, &navigatePTZPayload)
 
 	if navigatePTZPayload.Timestamp != 0 {
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			action := navigatePTZPayload.Action
 			var onvifAction models.OnvifAction
 			json.Unmarshal([]byte(action), &onvifAction)
-			communication.HandleONVIF <- onvifAction
-			log.Log.Info("routers.mqtt.main.HandleNavigatePTZ(): Received an action - " + onvifAction.Action)
+			if communication.TrySendONVIF(onvifAction) {
+				log.Info("routers.mqtt.main.HandleNavigatePTZ(): Received an action - " + onvifAction.Action)
+			} else {
+				log.Warn("routers.mqtt.main.HandleNavigatePTZ(): action channel unavailable or full")
+			}
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleNavigatePTZ(): received action, but camera is not connected.")
+			log.Info("routers.mqtt.main.HandleNavigatePTZ(): received action, but camera is not connected.")
 		}
 	}
 }
@@ -837,7 +1088,7 @@ func HandleTriggerRelay(mqttClient mqtt.Client, hubKey string, payload models.Pa
 	json.Unmarshal(jsonData, &triggerRelayPayload)
 
 	if triggerRelayPayload.Timestamp != 0 {
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			// Get token (name of relay)
 			token := triggerRelayPayload.Token
 			// Connect to Onvif device
@@ -847,16 +1098,25 @@ func HandleTriggerRelay(mqttClient mqtt.Client, hubKey string, payload models.Pa
 				// Trigger relay output
 				err := onvif.TriggerRelayOutput(device, token)
 				if err != nil {
-					log.Log.Error("routers.mqtt.main.HandleTriggerRelay(): error triggering relay: " + err.Error())
+					log.WithError(err).WithFields(log.Fields{
+						"component": "routers/mqtt",
+						"event":     "relay_output_failed",
+					}).Error("Failed to trigger relay output")
 				} else {
-					log.Log.Info("routers.mqtt.main.HandleTriggerRelay(): trigger (" + token + ") relay output.")
+					log.WithFields(log.Fields{
+						"component": "routers/mqtt",
+						"event":     "relay_output_triggered",
+					}).Info("Relay output triggered")
 				}
 			} else {
-				log.Log.Error("routers.mqtt.main.HandleTriggerRelay(): error connecting to device: " + err.Error())
+				log.WithError(err).WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "relay_device_connection_failed",
+				}).Error("Failed to connect to relay device")
 			}
 
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleTriggerRelay(): received trigger, but camera is not connected.")
+			log.Info("routers.mqtt.main.HandleTriggerRelay(): received trigger, but camera is not connected.")
 		}
 	}
 }
@@ -868,6 +1128,6 @@ func DisconnectMQTT(mqttClient mqtt.Client, config *models.Config) {
 		mqttClient.Unsubscribe("kerberos/agent/" + PREV_HubKey)
 		mqttClient.Disconnect(1000)
 		mqttClient = nil
-		log.Log.Info("routers.mqtt.main.DisconnectMQTT(): MQTT client disconnected.")
+		log.Info("routers.mqtt.main.DisconnectMQTT(): MQTT client disconnected.")
 	}
 }

machinery/src/routers/mqtt/main_test.go

@@ -1,12 +1,111 @@
 package mqtt
 
 import (
+	"crypto/tls"
+	"errors"
+	"net/http/httptest"
+	"net/url"
+	"os"
 	"testing"
 	"time"
 
+	mqtt "github.com/eclipse/paho.mqtt.golang"
 	"github.com/kerberos-io/agent/machinery/src/models"
 )
 
+func TestEnableMQTTConnectionDiagnostics(t *testing.T) {
+	tests := []struct {
+		name      string
+		brokerURL string
+		want      bool
+	}{
+		{name: "ActiveMQ TLS", brokerURL: "mqtt+ssl://broker.example:8883", want: true},
+		{name: "TCP", brokerURL: "tcp://broker.example:1883", want: true},
+		{name: "default TCP", brokerURL: "broker.example:1883", want: true},
+		{name: "WebSocket", brokerURL: "wss://broker.example/mqtt", want: false},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			options := mqtt.NewClientOptions()
+			enableMQTTConnectionDiagnostics(options, test.brokerURL)
+			if got := options.CustomOpenConnectionFn != nil; got != test.want {
+				t.Fatalf("CustomOpenConnectionFn configured = %t, want %t", got, test.want)
+			}
+		})
+	}
+}
+
+func TestOpenMQTTConnectionReturnsTCPError(t *testing.T) {
+	options := *mqtt.NewClientOptions().SetConnectTimeout(100 * time.Millisecond)
+	brokerURL, err := url.Parse("tcp://127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	connection, err := openMQTTConnection(brokerURL, options)
+	if connection != nil {
+		connection.Close()
+		t.Fatal("openMQTTConnection() returned a connection for an unavailable endpoint")
+	}
+	if err == nil {
+		t.Fatal("openMQTTConnection() returned no TCP error")
+	}
+}
+
+func TestOpenMQTTConnectionEstablishesActiveMQTLS(t *testing.T) {
+	server := httptest.NewTLSServer(nil)
+	defer server.Close()
+
+	serverURL, err := url.Parse(server.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	brokerURL, err := url.Parse("mqtt+ssl://" + serverURL.Host)
+	if err != nil {
+		t.Fatal(err)
+	}
+	options := *mqtt.NewClientOptions().
+		SetConnectTimeout(time.Second).
+		SetTLSConfig(&tls.Config{InsecureSkipVerify: true}) // #nosec G402 -- local test server
+
+	connection, err := openMQTTConnection(brokerURL, options)
+	if err != nil {
+		t.Fatalf("openMQTTConnection() error = %v", err)
+	}
+	defer connection.Close()
+	if _, ok := connection.(*tls.Conn); !ok {
+		t.Fatalf("openMQTTConnection() connection type = %T, want *tls.Conn", connection)
+	}
+}
+
+func TestIsSecureMQTTScheme(t *testing.T) {
+	for _, scheme := range []string{"ssl", "tls", "mqtts", "mqtt+ssl", "tcps"} {
+		if !isSecureMQTTScheme(scheme) {
+			t.Errorf("isSecureMQTTScheme(%q) = false, want true", scheme)
+		}
+	}
+	if isSecureMQTTScheme("tcp") {
+		t.Error("isSecureMQTTScheme(\"tcp\") = true, want false")
+	}
+}
+
+func TestConfigureMQTTRequiresHubKey(t *testing.T) {
+	configuration := &models.Configuration{Config: models.Config{Key: "agent-key"}}
+
+	if client := ConfigureMQTT("", configuration, &models.Communication{}); client != nil {
+		t.Fatal("ConfigureMQTT() returned a client without a Hub key")
+	}
+}
+
+func TestConfigureMQTTRequiresAgentKey(t *testing.T) {
+	configuration := &models.Configuration{Config: models.Config{HubKey: "hub-key"}}
+
+	if client := ConfigureMQTT("", configuration, &models.Communication{}); client != nil {
+		t.Fatal("ConfigureMQTT() returned a client without an Agent key")
+	}
+}
+
 func TestEnqueueLatestAudioReplacesOldestFrameWhenFull(t *testing.T) {
 	audioChannel := make(chan models.AudioDataPartial, 2)
 	audioChannel <- models.AudioDataPartial{Timestamp: 1}
@@ -37,3 +136,73 @@ func TestEnqueueLatestAudioDoesNotBlockNilChannel(t *testing.T) {
 		t.Fatal("enqueueLatestAudio() blocked on a nil channel")
 	}
 }
+
+func TestRemoteAccessRequiresExplicitOptIn(t *testing.T) {
+	previous, present := os.LookupEnv(remoteAccessEnvironment)
+	t.Cleanup(func() {
+		if present {
+			_ = os.Setenv(remoteAccessEnvironment, previous)
+		} else {
+			_ = os.Unsetenv(remoteAccessEnvironment)
+		}
+	})
+
+	_ = os.Unsetenv(remoteAccessEnvironment)
+	if remoteAccessEnabled() {
+		t.Fatal("remoteAccessEnabled() = true without opt-in")
+	}
+	_ = os.Setenv(remoteAccessEnvironment, "true")
+	if !remoteAccessEnabled() {
+		t.Fatal("remoteAccessEnabled() = false after opt-in")
+	}
+}
+
+func TestNormalizeTerminalSize(t *testing.T) {
+	rows, columns := normalizeTerminalSize(0, 0)
+	if rows != 24 || columns != 80 {
+		t.Fatalf("normalizeTerminalSize(0, 0) = (%d, %d), want (24, 80)", rows, columns)
+	}
+
+	rows, columns = normalizeTerminalSize(500, 500)
+	if rows != 200 || columns != 400 {
+		t.Fatalf("normalizeTerminalSize(500, 500) = (%d, %d), want (200, 400)", rows, columns)
+	}
+}
+
+func TestDecodeRemotePayloadRejectsMissingSession(t *testing.T) {
+	_, err := decodeRemotePayload(models.Payload{Value: map[string]interface{}{
+		"kind": "shell",
+	}})
+	if err == nil {
+		t.Fatal("decodeRemotePayload() accepted a missing session id")
+	}
+}
+
+func TestRemoteSessionOpenRejectsUnprovenEncryption(t *testing.T) {
+	previous, present := os.LookupEnv(remoteAccessEnvironment)
+	t.Cleanup(func() {
+		if present {
+			_ = os.Setenv(remoteAccessEnvironment, previous)
+		} else {
+			_ = os.Unsetenv(remoteAccessEnvironment)
+		}
+	})
+	_ = os.Setenv(remoteAccessEnvironment, "true")
+
+	// The listener passes false when an envelope merely claims to be hidden but
+	// no ciphertext was successfully decrypted. The remote handler must reject it.
+	if err := validateRemoteAccess(false); !errors.Is(err, errRemoteUnauthenticated) {
+		t.Fatalf("validateRemoteAccess(false) error = %v, want %v", err, errRemoteUnauthenticated)
+	}
+}
+
+func TestRemoteSessionReservationIsIdempotent(t *testing.T) {
+	manager := newRemoteAccessManager()
+	session := &remoteSession{id: "session-1", kind: "logs"}
+	if _, err := manager.reserve(session); err != nil {
+		t.Fatalf("first reserve() failed: %v", err)
+	}
+	if _, err := manager.reserve(session); !errors.Is(err, errRemoteSessionExists) {
+		t.Fatalf("duplicate reserve() error = %v, want %v", err, errRemoteSessionExists)
+	}
+}

machinery/src/routers/mqtt/remote_access.go

@@ -0,0 +1,397 @@
+package mqtt
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"io"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/creack/pty"
+	paho "github.com/eclipse/paho.mqtt.golang"
+	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	remoteAccessEnvironment = "AGENT_REMOTE_ACCESS_ENABLED"
+	remoteHistoryLimit      = 500
+	remoteSessionLimit      = 5
+	remoteOutputChunkSize   = 4096
+	remoteInputLimit        = 64 * 1024
+	remoteSessionLifetime   = time.Hour
+)
+
+type remoteSession struct {
+	id     string
+	kind   string
+	pty    *os.File
+	cancel context.CancelFunc
+	logs   chan string
+	done   chan struct{}
+	timer  *time.Timer
+}
+
+type remoteAccessManager struct {
+	mu       sync.Mutex
+	sessions map[string]*remoteSession
+	history  []string
+}
+
+var (
+	remoteAccess             = newRemoteAccessManager()
+	remoteHookOnce           sync.Once
+	errRemoteDisabled        = errors.New("remote access is disabled on this agent")
+	errRemoteUnauthenticated = errors.New("remote access requires encrypted MQTT")
+	errRemoteSessionExists   = errors.New("remote session already exists")
+	errRemoteSessionLimit    = errors.New("remote session limit reached")
+)
+
+func newRemoteAccessManager() *remoteAccessManager {
+	return &remoteAccessManager{sessions: make(map[string]*remoteSession)}
+}
+
+func installRemoteAccessHook() {
+	remoteHookOnce.Do(func() {
+		log.AddHook(remoteAccess)
+	})
+}
+
+func (manager *remoteAccessManager) Levels() []log.Level {
+	return log.AllLevels
+}
+
+func (manager *remoteAccessManager) Fire(entry *log.Entry) error {
+	line, err := json.Marshal(map[string]interface{}{
+		"timestamp": entry.Time.Format(time.RFC3339Nano),
+		"level":     entry.Level.String(),
+		"message":   entry.Message,
+		"fields":    entry.Data,
+	})
+	if err != nil {
+		return nil
+	}
+	encoded := base64.StdEncoding.EncodeToString(append(line, '\n'))
+
+	manager.mu.Lock()
+	manager.history = append(manager.history, encoded)
+	if len(manager.history) > remoteHistoryLimit {
+		manager.history = manager.history[len(manager.history)-remoteHistoryLimit:]
+	}
+	for _, session := range manager.sessions {
+		if session.kind != "logs" {
+			continue
+		}
+		select {
+		case session.logs <- encoded:
+		default:
+		}
+	}
+	manager.mu.Unlock()
+	return nil
+}
+
+func remoteAccessEnabled() bool {
+	enabled, err := strconv.ParseBool(strings.TrimSpace(os.Getenv(remoteAccessEnvironment)))
+	return err == nil && enabled
+}
+
+func validateRemoteAccess(authenticated bool) error {
+	if !authenticated {
+		return errRemoteUnauthenticated
+	}
+	if !remoteAccessEnabled() {
+		return errRemoteDisabled
+	}
+	return nil
+}
+
+func decodeRemotePayload(payload models.Payload) (models.RemoteSessionPayload, error) {
+	data, err := json.Marshal(payload.Value)
+	if err != nil {
+		return models.RemoteSessionPayload{}, err
+	}
+	var request models.RemoteSessionPayload
+	if err := json.Unmarshal(data, &request); err != nil {
+		return models.RemoteSessionPayload{}, err
+	}
+	if request.SessionID == "" || len(request.SessionID) > 128 {
+		return models.RemoteSessionPayload{}, errors.New("invalid remote session id")
+	}
+	return request, nil
+}
+
+func normalizeTerminalSize(rows uint16, columns uint16) (uint16, uint16) {
+	if rows < 5 {
+		rows = 24
+	}
+	if rows > 200 {
+		rows = 200
+	}
+	if columns < 20 {
+		columns = 80
+	}
+	if columns > 400 {
+		columns = 400
+	}
+	return rows, columns
+}
+
+func HandleRemoteSessionOpen(client paho.Client, hubKey string, payload models.Payload, authenticated bool, configuration *models.Configuration) {
+	request, err := decodeRemotePayload(payload)
+	if err != nil {
+		return
+	}
+	if accessErr := validateRemoteAccess(authenticated); accessErr != nil {
+		publishRemoteStatus(client, hubKey, configuration, request.SessionID, request.Kind, "error", accessErr.Error())
+		return
+	}
+
+	switch request.Kind {
+	case "logs":
+		err = remoteAccess.openLogs(client, hubKey, configuration, request)
+	case "shell":
+		err = remoteAccess.openShell(client, hubKey, configuration, request)
+	default:
+		err = errors.New("unsupported remote session kind")
+	}
+	if err != nil {
+		if errors.Is(err, errRemoteSessionExists) {
+			publishRemoteStatus(client, hubKey, configuration, request.SessionID, request.Kind, "opened", "")
+			return
+		}
+		publishRemoteStatus(client, hubKey, configuration, request.SessionID, request.Kind, "error", err.Error())
+	}
+}
+
+func HandleRemoteSessionInput(client paho.Client, hubKey string, payload models.Payload, authenticated bool, configuration *models.Configuration) {
+	if validateRemoteAccess(authenticated) != nil {
+		return
+	}
+	request, err := decodeRemotePayload(payload)
+	if err != nil || len(request.Data) > remoteInputLimit*2 {
+		return
+	}
+	data, err := base64.StdEncoding.DecodeString(request.Data)
+	if err != nil || len(data) > remoteInputLimit {
+		return
+	}
+	remoteAccess.mu.Lock()
+	session := remoteAccess.sessions[request.SessionID]
+	remoteAccess.mu.Unlock()
+	if session == nil || session.kind != "shell" || session.pty == nil {
+		publishRemoteStatus(client, hubKey, configuration, request.SessionID, "shell", "error", "remote session is not open")
+		return
+	}
+	if _, err := session.pty.Write(data); err != nil {
+		publishRemoteStatus(client, hubKey, configuration, request.SessionID, "shell", "error", "failed to write terminal input")
+	}
+}
+
+func HandleRemoteSessionResize(payload models.Payload, authenticated bool) {
+	if validateRemoteAccess(authenticated) != nil {
+		return
+	}
+	request, err := decodeRemotePayload(payload)
+	if err != nil {
+		return
+	}
+	rows, columns := normalizeTerminalSize(request.Rows, request.Columns)
+	remoteAccess.mu.Lock()
+	session := remoteAccess.sessions[request.SessionID]
+	remoteAccess.mu.Unlock()
+	if session != nil && session.kind == "shell" && session.pty != nil {
+		_ = pty.Setsize(session.pty, &pty.Winsize{Rows: rows, Cols: columns})
+	}
+}
+
+func HandleRemoteSessionClose(payload models.Payload, authenticated bool) {
+	if !authenticated {
+		return
+	}
+	request, err := decodeRemotePayload(payload)
+	if err == nil {
+		remoteAccess.close(request.SessionID)
+	}
+}
+
+func (manager *remoteAccessManager) reserve(session *remoteSession) ([]string, error) {
+	manager.mu.Lock()
+	defer manager.mu.Unlock()
+	if _, exists := manager.sessions[session.id]; exists {
+		return nil, errRemoteSessionExists
+	}
+	if len(manager.sessions) >= remoteSessionLimit {
+		return nil, errRemoteSessionLimit
+	}
+	manager.sessions[session.id] = session
+	history := append([]string(nil), manager.history...)
+	return history, nil
+}
+
+func (manager *remoteAccessManager) expire(client paho.Client, hubKey string, configuration *models.Configuration, session *remoteSession) {
+	session.timer = time.AfterFunc(remoteSessionLifetime, func() {
+		manager.close(session.id)
+		if session.kind == "logs" {
+			publishRemoteStatus(client, hubKey, configuration, session.id, session.kind, "closed", "session lifetime reached")
+		}
+	})
+}
+
+func (manager *remoteAccessManager) openLogs(client paho.Client, hubKey string, configuration *models.Configuration, request models.RemoteSessionPayload) error {
+	session := &remoteSession{
+		id:   request.SessionID,
+		kind: "logs",
+		logs: make(chan string, 256),
+		done: make(chan struct{}),
+	}
+	history, err := manager.reserve(session)
+	if err != nil {
+		return err
+	}
+	manager.expire(client, hubKey, configuration, session)
+	tail := request.Tail
+	if tail <= 0 || tail > remoteHistoryLimit {
+		tail = 200
+	}
+	if len(history) > tail {
+		history = history[len(history)-tail:]
+	}
+
+	publishRemoteStatus(client, hubKey, configuration, session.id, session.kind, "opened", "")
+	go func() {
+		for _, line := range history {
+			publishRemoteOutput(client, hubKey, configuration, session.id, session.kind, line)
+		}
+		for {
+			select {
+			case line := <-session.logs:
+				publishRemoteOutput(client, hubKey, configuration, session.id, session.kind, line)
+			case <-session.done:
+				return
+			}
+		}
+	}()
+	return nil
+}
+
+func (manager *remoteAccessManager) openShell(client paho.Client, hubKey string, configuration *models.Configuration, request models.RemoteSessionPayload) error {
+	rows, columns := normalizeTerminalSize(request.Rows, request.Columns)
+	ctx, cancel := context.WithCancel(context.Background())
+	session := &remoteSession{
+		id:     request.SessionID,
+		kind:   "shell",
+		cancel: cancel,
+		done:   make(chan struct{}),
+	}
+	if _, err := manager.reserve(session); err != nil {
+		cancel()
+		return err
+	}
+	manager.expire(client, hubKey, configuration, session)
+	command := exec.CommandContext(ctx, "/bin/sh")
+	command.Env = append(os.Environ(), "TERM=xterm-256color", "HISTFILE=/dev/null")
+	terminal, err := pty.StartWithSize(command, &pty.Winsize{Rows: rows, Cols: columns})
+	if err != nil {
+		manager.remove(session.id)
+		cancel()
+		return err
+	}
+	session.pty = terminal
+
+	publishRemoteStatus(client, hubKey, configuration, session.id, session.kind, "opened", "")
+	go manager.forwardShell(client, hubKey, configuration, session, command)
+	return nil
+}
+
+func (manager *remoteAccessManager) forwardShell(client paho.Client, hubKey string, configuration *models.Configuration, session *remoteSession, command *exec.Cmd) {
+	buffer := make([]byte, remoteOutputChunkSize)
+	for {
+		count, err := session.pty.Read(buffer)
+		if count > 0 {
+			publishRemoteOutput(client, hubKey, configuration, session.id, session.kind, base64.StdEncoding.EncodeToString(buffer[:count]))
+		}
+		if err != nil {
+			if !errors.Is(err, io.EOF) && !errors.Is(err, os.ErrClosed) {
+				publishRemoteStatus(client, hubKey, configuration, session.id, session.kind, "error", "terminal stream closed unexpectedly")
+			}
+			break
+		}
+	}
+	_ = command.Wait()
+	manager.remove(session.id)
+	publishRemoteStatus(client, hubKey, configuration, session.id, session.kind, "closed", "")
+}
+
+func (manager *remoteAccessManager) close(sessionID string) {
+	manager.mu.Lock()
+	session := manager.sessions[sessionID]
+	delete(manager.sessions, sessionID)
+	manager.mu.Unlock()
+	if session == nil {
+		return
+	}
+	if session.cancel != nil {
+		session.cancel()
+	}
+	if session.pty != nil {
+		_ = session.pty.Close()
+	}
+	if session.logs != nil {
+		close(session.done)
+	}
+	if session.timer != nil {
+		session.timer.Stop()
+	}
+}
+
+func (manager *remoteAccessManager) remove(sessionID string) {
+	manager.mu.Lock()
+	session := manager.sessions[sessionID]
+	delete(manager.sessions, sessionID)
+	manager.mu.Unlock()
+	if session != nil && session.timer != nil {
+		session.timer.Stop()
+	}
+}
+
+func publishRemoteStatus(client paho.Client, hubKey string, configuration *models.Configuration, sessionID string, kind string, state string, errorMessage string) {
+	status := models.RemoteSessionStatus{
+		Timestamp: time.Now().Unix(),
+		SessionID: sessionID,
+		Kind:      kind,
+		State:     state,
+		Error:     errorMessage,
+	}
+	value, _ := json.Marshal(status)
+	var statusValue map[string]interface{}
+	_ = json.Unmarshal(value, &statusValue)
+	publishRemote(client, hubKey, configuration, "remote-session-status", statusValue, 1)
+}
+
+func publishRemoteOutput(client paho.Client, hubKey string, configuration *models.Configuration, sessionID string, kind string, data string) {
+	publishRemote(client, hubKey, configuration, "remote-session-output", map[string]interface{}{
+		"timestamp":  time.Now().Unix(),
+		"session_id": sessionID,
+		"kind":       kind,
+		"data":       data,
+	}, 0)
+}
+
+func publishRemote(client paho.Client, hubKey string, configuration *models.Configuration, action string, value map[string]interface{}, qos byte) {
+	message := models.Message{Payload: models.Payload{
+		Action:   action,
+		DeviceId: configuration.Config.Key,
+		Value:    value,
+	}}
+	payload, err := models.PackageMQTTMessage(configuration, message)
+	if err == nil {
+		client.Publish("kerberos/hub/"+hubKey, qos, false, payload)
+	}
+}

machinery/src/routers/websocket/main.go

@@ -11,11 +11,11 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/gorilla/websocket"
 	"github.com/kerberos-io/agent/machinery/src/capture"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/kerberos-io/agent/machinery/src/utils"
 	"github.com/kerberos-io/agent/machinery/src/webrtc"
+	log "github.com/sirupsen/logrus"
 )
 
 type Message struct {
@@ -43,7 +43,7 @@ func writeWebRTCError(connection *Connection, clientID string, sessionID string,
 			"message":    errorMessage,
 		},
 	}); err != nil {
-		log.Log.Error("routers.websocket.main.writeWebRTCError(): " + err.Error())
+		log.Error("routers.websocket.main.writeWebRTCError(): " + err.Error())
 	}
 }
 
@@ -80,7 +80,7 @@ func WebsocketHandler(c *gin.Context, configuration *models.Configuration, commu
 		var message Message
 		err = conn.ReadJSON(&message)
 		if err != nil {
-			log.Log.Error("routers.websocket.main.WebsocketHandler(): " + err.Error())
+			log.Error("routers.websocket.main.WebsocketHandler(): " + err.Error())
 			return
 		}
 		clientID := message.ClientID
@@ -89,7 +89,7 @@ func WebsocketHandler(c *gin.Context, configuration *models.Configuration, commu
 			connection.Socket = conn
 			sockets[clientID] = connection
 			sockets[clientID].Cancels = make(map[string]context.CancelFunc)
-			log.Log.Info("routers.websocket.main.WebsocketHandler(): " + clientID + ": connected.")
+			log.Info("routers.websocket.main.WebsocketHandler(): " + clientID + ": connected.")
 		}
 
 		// Continuously read messages
@@ -111,14 +111,14 @@ func WebsocketHandler(c *gin.Context, configuration *models.Configuration, commu
 				if exists {
 					sockets[clientID].Cancels["stream-sd"]()
 				} else {
-					log.Log.Error("routers.websocket.main.WebsocketHandler(): streaming sd does not exists for " + clientID)
+					log.Error("routers.websocket.main.WebsocketHandler(): streaming sd does not exists for " + clientID)
 				}
 
 			case "stream-sd":
-				if communication.CameraConnected {
+				if communication.CameraConnected.Load() {
 					_, exists := sockets[clientID].Cancels["stream-sd"]
 					if exists {
-						log.Log.Debug("routers.websocket.main.WebsocketHandler(): already streaming sd for " + clientID)
+						log.Debug("routers.websocket.main.WebsocketHandler(): already streaming sd for " + clientID)
 					} else {
 						startStream := Message{
 							ClientID:    clientID,
@@ -144,16 +144,11 @@ func WebsocketHandler(c *gin.Context, configuration *models.Configuration, commu
 					break
 				}
 
-				if !communication.CameraConnected {
+				if !communication.CameraConnected.Load() {
 					writeWebRTCError(sockets[clientID], clientID, sessionID, "camera is not connected")
 					break
 				}
 
-				if communication.HandleLiveHDHandshake == nil {
-					writeWebRTCError(sockets[clientID], clientID, sessionID, "webrtc liveview is not available")
-					break
-				}
-
 				handshake := models.LiveHDHandshake{
 					Payload: models.RequestHDStreamPayload{
 						Timestamp:          time.Now().Unix(),
@@ -188,7 +183,9 @@ func WebsocketHandler(c *gin.Context, configuration *models.Configuration, commu
 					},
 				}
 
-				communication.HandleLiveHDHandshake <- handshake
+				if !communication.TrySendLiveHDHandshake(handshake) {
+					writeWebRTCError(sockets[clientID], clientID, handshake.Payload.SessionID, "camera is restarting or live-view queue is full")
+				}
 
 			case "webrtc-candidate":
 				sessionID := message.Message["session_id"]
@@ -199,7 +196,7 @@ func WebsocketHandler(c *gin.Context, configuration *models.Configuration, commu
 					break
 				}
 
-				if !communication.CameraConnected {
+				if !communication.CameraConnected.Load() {
 					writeWebRTCError(sockets[clientID], clientID, sessionID, "camera is not connected")
 					break
 				}
@@ -221,7 +218,7 @@ func WebsocketHandler(c *gin.Context, configuration *models.Configuration, commu
 		_, exists := sockets[clientID]
 		if exists {
 			delete(sockets, clientID)
-			log.Log.Info("routers.websocket.main.WebsocketHandler(): " + clientID + ": terminated and disconnected websocket connection.")
+			log.Info("routers.websocket.main.WebsocketHandler(): " + clientID + ": terminated and disconnected websocket connection.")
 		}
 	}
 }
@@ -232,13 +229,14 @@ func ForwardSDStream(ctx context.Context, clientID string, connection *Connectio
 	var cursor *packets.QueueCursor
 
 	// We'll pick the right client and decoder.
-	rtspClient := captureDevice.RTSPSubClient
+	rtspClient := captureDevice.SubClient()
 	if rtspClient != nil {
-		queue = communication.SubQueue
-		cursor = queue.Latest()
+		queue = communication.SubQueue()
 	} else {
-		rtspClient = captureDevice.RTSPClient
-		queue = communication.Queue
+		rtspClient = captureDevice.MainClient()
+		queue = communication.MainQueue()
+	}
+	if queue != nil {
 		cursor = queue.Latest()
 	}
 
@@ -263,7 +261,7 @@ logreader:
 					continue
 				}
 			} else {
-				log.Log.Error("routers.websocket.main.ForwardSDStream():" + err.Error())
+				log.Error("routers.websocket.main.ForwardSDStream():" + err.Error())
 				break logreader
 			}
 		}
@@ -277,7 +275,7 @@ logreader:
 		}
 		err := connection.WriteJson(startStrean)
 		if err != nil {
-			log.Log.Error("routers.websocket.main.ForwardSDStream():" + err.Error())
+			log.Error("routers.websocket.main.ForwardSDStream():" + err.Error())
 			break logreader
 		}
 		select {
@@ -292,9 +290,9 @@ logreader:
 	if exists {
 		delete(connection.Cancels, "stream-sd")
 	} else {
-		log.Log.Error("routers.websocket.main.ForwardSDStream(): streaming sd does not exists for " + clientID)
+		log.Error("routers.websocket.main.ForwardSDStream(): streaming sd does not exists for " + clientID)
 	}
 
 	// Send stop streaming message
-	log.Log.Info("routers.websocket.main.ForwardSDStream(): stop sending streaming over websocket")
+	log.Info("routers.websocket.main.ForwardSDStream(): stop sending streaming over websocket")
 }

machinery/src/utils/main.go

@@ -1,7 +1,6 @@
 package utils
 
 import (
-	"bufio"
 	"bytes"
 	"errors"
 	"fmt"
@@ -12,15 +11,14 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
-	"reflect"
 	"sort"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/encryption"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 
 	"github.com/nfnt/resize"
 )
@@ -140,11 +138,11 @@ func CheckDataDirectoryPermissions(configDirectory string) error {
 	}
 
 	if err != nil {
-		log.Log.Error("Checking data directory permissions: " + err.Error())
+		log.Error("Checking data directory permissions: " + err.Error())
 		return err
 	}
 
-	log.Log.Info("Checking data directory permissions: OK")
+	log.Info("Checking data directory permissions: OK")
 	return nil
 }
 
@@ -301,16 +299,16 @@ func CreateFragmentedMP4(fullName string, fragmentedDuration int64) {
 	// This timescale is crucial, as it should be the same as the one defined in JOY4.
 	cmd := exec.Command("mp4fragment", "--timescale", "10000000", "--fragment-duration", strconv.FormatInt(duration, 10), fullName, fullName+"f.mp4")
 	cmd.Dir = path
-	log.Log.Info(cmd.String())
+	log.Info(cmd.String())
 	var out bytes.Buffer
 	var stderr bytes.Buffer
 	cmd.Stdout = &out
 	cmd.Stderr = &stderr
 	err := cmd.Run()
 	if err != nil {
-		log.Log.Error(fmt.Sprint(err) + ": " + stderr.String())
+		log.Error(fmt.Sprint(err) + ": " + stderr.String())
 	} else {
-		log.Log.Info("Created Fragmented: " + out.String())
+		log.Info("Created Fragmented: " + out.String())
 	}
 
 	// We will swap the files.
@@ -319,42 +317,93 @@ func CreateFragmentedMP4(fullName string, fragmentedDuration int64) {
 }
 
 func PrintEnvironmentVariables() {
-	// Print environment variables that include "AGENT_" as a prefix.
-	environmentVariables := ""
-	for _, e := range os.Environ() {
-		if strings.Contains(e, "AGENT_") {
-			pair := strings.Split(e, "=")
-			environmentVariables = environmentVariables + pair[0] + "=" + pair[1] + " "
+	names := agentEnvironmentVariableNames(os.Environ())
+	log.WithFields(log.Fields{
+		"component":      "configuration",
+		"event":          "environment_loaded",
+		"variable_count": len(names),
+	}).Info("Agent environment loaded")
+	log.WithFields(log.Fields{
+		"component": "configuration",
+		"event":     "environment_details",
+		"variables": names,
+	}).Debug("Agent environment details")
+}
+
+func agentEnvironmentVariableNames(environment []string) []string {
+	names := make([]string, 0)
+	for _, entry := range environment {
+		name, _, found := strings.Cut(entry, "=")
+		if found && strings.HasPrefix(name, "AGENT_") {
+			names = append(names, name)
 		}
 	}
-	log.Log.Info("Printing out environmentVariables (AGENT_...): " + environmentVariables)
+	sort.Strings(names)
+	return names
 }
 
 func PrintConfiguration(configuration *models.Configuration) {
-	// We will print out the struct.
 	if configuration == nil {
-		log.Log.Info("Configuration is nil")
+		log.WithFields(log.Fields{
+			"component": "configuration",
+			"event":     "configuration_missing",
+		}).Warn("Agent configuration is nil")
 		return
 	}
-	config := configuration.Config
-	// Iterate over the struct and printout the values.
-	v := reflect.ValueOf(config)
-	typeOfS := v.Type()
-	configurationVariables := ""
-	for i := 0; i < v.NumField(); i++ {
-		key := typeOfS.Field(i).Name
-		value := v.Field(i).Interface()
-		// Convert to string.
-		configurationVariables = configurationVariables + key + ": " + fmt.Sprintf("%v", value) + " "
+
+	log.WithFields(configurationLogFields(configuration.Config)).
+		Info("Agent configuration loaded")
+	log.WithFields(configurationDebugLogFields(configuration.Config)).
+		Debug("Agent configuration details")
+}
+
+func configurationLogFields(config models.Config) log.Fields {
+	return log.Fields{
+		"agent_name":     config.Name,
+		"camera_type":    config.Capture.Name,
+		"cloud_provider": config.Cloud,
+		"component":      "configuration",
+		"event":          "configuration_loaded",
+		"friendly_name":  config.FriendlyName,
+		"liveview":       config.Capture.Liveview,
+		"offline":        config.Offline,
+		"recording":      config.Capture.Recording,
+		"timezone":       config.Timezone,
+	}
+}
+
+func configurationDebugLogFields(config models.Config) log.Fields {
+	return log.Fields{
+		"auto_clean":             config.AutoClean,
+		"component":              "configuration",
+		"continuous":             config.Capture.Continuous,
+		"event":                  "configuration_details",
+		"gop_size":               config.Capture.GopSize,
+		"main_fps":               config.Capture.IPCamera.FPS,
+		"main_height":            config.Capture.IPCamera.Height,
+		"main_stream_configured": config.Capture.IPCamera.RTSP != "",
+		"main_width":             config.Capture.IPCamera.Width,
+		"max_recording_seconds":  config.Capture.MaxLengthRecording,
+		"motion":                 config.Capture.Motion,
+		"onvif_configured":       config.Capture.IPCamera.ONVIFXAddr != "",
+		"onvif_motion":           config.Capture.ONVIFMotion,
+		"post_recording_seconds": config.Capture.PostRecording,
+		"pre_recording_seconds":  config.Capture.PreRecording,
+		"remove_after_upload":    config.RemoveAfterUpload,
+		"snapshots":              config.Capture.Snapshots,
+		"sub_fps":                config.Capture.IPCamera.SubFPS,
+		"sub_height":             config.Capture.IPCamera.SubHeight,
+		"sub_stream_configured":  config.Capture.IPCamera.SubRTSP != "",
+		"sub_width":              config.Capture.IPCamera.SubWidth,
+		"transcoding_webrtc":     config.Capture.TranscodingWebRTC,
 	}
-	log.Log.Info("Printing our configuration (config.json): " + configurationVariables)
 }
 
 func Decrypt(directoryOrFile string, symmetricKey []byte) {
 	// Check if file or directory
 	fileInfo, err := os.Stat(directoryOrFile)
 	if err != nil {
-		log.Log.Fatal(err.Error())
+		log.Fatal(err.Error())
 		return
 	}
 
@@ -363,12 +412,12 @@ func Decrypt(directoryOrFile string, symmetricKey []byte) {
 		// Create decrypted directory
 		err = os.MkdirAll(directoryOrFile+"/decrypted", 0755)
 		if err != nil {
-			log.Log.Fatal(err.Error())
+			log.Fatal(err.Error())
 			return
 		}
 		dir, err := os.ReadDir(directoryOrFile)
 		if err != nil {
-			log.Log.Fatal(err.Error())
+			log.Fatal(err.Error())
 			return
 		}
 		for _, file := range dir {
@@ -390,13 +439,13 @@ func Decrypt(directoryOrFile string, symmetricKey []byte) {
 		// Read file
 		content, err := os.ReadFile(file)
 		if err != nil {
-			log.Log.Fatal(err.Error())
+			log.Fatal(err.Error())
 			return
 		}
 		// Decrypt using AES key
 		decrypted, err := encryption.AesDecrypt(content, string(symmetricKey))
 		if err != nil {
-			log.Log.Fatal("Something went wrong while decrypting: " + err.Error())
+			log.Fatal("Something went wrong while decrypting: " + err.Error())
 			return
 		}
 
@@ -408,7 +457,7 @@ func Decrypt(directoryOrFile string, symmetricKey []byte) {
 
 		err = os.WriteFile(pathToFile+"/decrypted/"+fileName, []byte(decrypted), 0644)
 		if err != nil {
-			log.Log.Fatal(err.Error())
+			log.Fatal(err.Error())
 			return
 		}
 	}
@@ -416,9 +465,12 @@ func Decrypt(directoryOrFile string, symmetricKey []byte) {
 
 func ImageToBytes(img *image.Image) ([]byte, error) {
 	buffer := new(bytes.Buffer)
-	w := bufio.NewWriter(buffer)
-	err := jpeg.Encode(w, *img, &jpeg.Options{Quality: 35})
-	log.Log.Debug("ImageToBytes() - buffer size: " + strconv.Itoa(buffer.Len()))
+	err := jpeg.Encode(buffer, *img, &jpeg.Options{Quality: 35})
+	log.WithFields(log.Fields{
+		"bytes":     buffer.Len(),
+		"component": "image",
+		"event":     "jpeg_encoded",
+	}).Debug("Image encoded as JPEG")
 	return buffer.Bytes(), err
 }
 

machinery/src/utils/main_test.go

@@ -1,13 +1,112 @@
 package utils
 
 import (
+	"bytes"
+	"fmt"
+	"image"
+	"image/color"
+	"image/jpeg"
 	"os"
+	"reflect"
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/models"
 )
 
+func TestImageToBytesReturnsCompleteJPEG(t *testing.T) {
+	source := image.NewRGBA(image.Rect(0, 0, 16, 12))
+	source.Set(8, 6, color.RGBA{R: 255, A: 255})
+	var input image.Image = source
+
+	encoded, err := ImageToBytes(&input)
+	if err != nil {
+		t.Fatalf("ImageToBytes() error = %v", err)
+	}
+	if len(encoded) == 0 {
+		t.Fatal("ImageToBytes() returned an empty JPEG")
+	}
+
+	decoded, err := jpeg.Decode(bytes.NewReader(encoded))
+	if err != nil {
+		t.Fatalf("decoding ImageToBytes() output: %v", err)
+	}
+	if got := decoded.Bounds().Size(); got.X != 16 || got.Y != 12 {
+		t.Fatalf("decoded JPEG size = %dx%d, want 16x12", got.X, got.Y)
+	}
+}
+
+func TestAgentEnvironmentVariableNamesOmitsValues(t *testing.T) {
+	got := agentEnvironmentVariableNames([]string{
+		"AGENT_HUB_PRIVATE_KEY=do-not-log",
+		"PATH=/usr/bin",
+		"NOT_AGENT_SECRET=also-do-not-log",
+		"AGENT_CAPTURE_LIVEVIEW=true",
+	})
+	want := []string{"AGENT_CAPTURE_LIVEVIEW", "AGENT_HUB_PRIVATE_KEY"}
+
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("agentEnvironmentVariableNames() = %v, want %v", got, want)
+	}
+}
+
+func TestConfigurationLogFieldsOmitCredentialsAndEndpoints(t *testing.T) {
+	config := models.Config{
+		Name:          "Front Door",
+		FriendlyName:  "Entrance",
+		Cloud:         "kstorage",
+		HubKey:        "hub-key-secret",
+		HubPrivateKey: "hub-private-secret",
+		MQTTURI:       "mqtt://internal.example",
+		MQTTUsername:  "mqtt-user",
+		MQTTPassword:  "mqtt-password-secret",
+		Capture: models.Capture{
+			Liveview:           "true",
+			MaxLengthRecording: 20,
+			IPCamera: models.IPCamera{
+				RTSP:          "rtsp://camera-user:camera-password@10.0.30.11/live",
+				ONVIFUsername: "onvif-user",
+				ONVIFPassword: "onvif-password-secret",
+			},
+		},
+	}
+
+	fields := []map[string]interface{}{
+		configurationLogFields(config),
+		configurationDebugLogFields(config),
+	}
+	summary := fmt.Sprint(fields)
+	for _, secret := range []string{
+		config.HubKey,
+		config.HubPrivateKey,
+		config.MQTTURI,
+		config.MQTTUsername,
+		config.MQTTPassword,
+		config.Capture.IPCamera.RTSP,
+		config.Capture.IPCamera.ONVIFUsername,
+		config.Capture.IPCamera.ONVIFPassword,
+	} {
+		if strings.Contains(summary, secret) {
+			t.Fatalf("configuration log fields exposed sensitive value %q in %q", secret, summary)
+		}
+	}
+	infoFields := configurationLogFields(config)
+	if got := infoFields["agent_name"]; got != "Front Door" {
+		t.Fatalf("agent_name = %v, want Front Door", got)
+	}
+	if got := infoFields["cloud_provider"]; got != "kstorage" {
+		t.Fatalf("cloud_provider = %v, want kstorage", got)
+	}
+	debugFields := configurationDebugLogFields(config)
+	if got := debugFields["max_recording_seconds"]; got != int64(20) {
+		t.Fatalf("max_recording_seconds = %v, want 20", got)
+	}
+	if got := debugFields["main_stream_configured"]; got != true {
+		t.Fatalf("main_stream_configured = %v, want true", got)
+	}
+}
+
 type stubFileInfo struct {
 	name string
 }

machinery/src/video/livehls.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 
 	mp4ff "github.com/Eyevinn/mp4ff/mp4"
-	"github.com/kerberos-io/agent/machinery/src/log"
+	log "github.com/sirupsen/logrus"
 )
 
 // LiveSegmenter turns a live stream of Annex B video samples into HLS-ready
@@ -217,7 +217,7 @@ func (ls *LiveSegmenter) buildInit() error {
 		// fall back to a manually built avcC just like the recording muxer does so
 		// those cameras still produce a valid init segment.
 		if err := trak.SetAVCDescriptor("avc1", sps, pps, true); err != nil {
-			log.Log.Warning("livehls: SetAVCDescriptor failed, using manual avcC fallback: " + err.Error())
+			log.Warn("livehls: SetAVCDescriptor failed, using manual avcC fallback: " + err.Error())
 			if fbErr := addAVCDescriptorFallback(trak, sps, pps, ls.width, ls.height); fbErr != nil {
 				return fmt.Errorf("livehls: AVC descriptor fallback: %w", fbErr)
 			}
@@ -279,7 +279,7 @@ func (ls *LiveSegmenter) WriteSample(isKeyframe bool, annexB []byte, ptsMs uint6
 	// A session must open on a random-access point; otherwise the first segment
 	// would reference frames that never arrived.
 	if !ls.started && !isKeyframe {
-		log.Log.Debug("LiveSegmenter.WriteSample(): dropping leading non-keyframe before first IDR")
+		log.Debug("LiveSegmenter.WriteSample(): dropping leading non-keyframe before first IDR")
 		return nil
 	}
 
@@ -352,7 +352,7 @@ func (ls *LiveSegmenter) openSegment(startPTS uint64) {
 	ls.seg = mp4ff.NewMediaSegment() // includes a CMAF styp box by default
 	frag, err := mp4ff.CreateFragment(ls.seqNr, ls.videoTrackID)
 	if err != nil {
-		log.Log.Error("LiveSegmenter.openSegment(): CreateFragment failed: " + err.Error())
+		log.Error("LiveSegmenter.openSegment(): CreateFragment failed: " + err.Error())
 		return
 	}
 	ls.seg.AddFragment(frag)
@@ -543,7 +543,7 @@ func (ls *LiveSegmenter) openPartFragment() {
 	ls.fragSeq++
 	frag, err := mp4ff.CreateFragment(ls.fragSeq, ls.videoTrackID)
 	if err != nil {
-		log.Log.Error("LiveSegmenter.openPartFragment(): CreateFragment failed: " + err.Error())
+		log.Error("LiveSegmenter.openPartFragment(): CreateFragment failed: " + err.Error())
 		return
 	}
 	ls.partFrag = frag

machinery/src/video/mp4.go

@@ -16,9 +16,9 @@ import (
 	"github.com/Eyevinn/mp4ff/avc"
 	mp4ff "github.com/Eyevinn/mp4ff/mp4"
 	"github.com/kerberos-io/agent/machinery/src/encryption"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/utils"
+	log "github.com/sirupsen/logrus"
 )
 
 var LastPTS uint64 = 0 // Last PTS for the current segment
@@ -216,13 +216,13 @@ func (mp4 *MP4) updateVideoParameterSetsFromAnnexB(data []byte) {
 				if needSPS {
 					mp4.SPSNALUs = [][]byte{nalu}
 					needSPS = false
-					log.Log.Warning("mp4.updateVideoParameterSetsFromAnnexB(): SPS recovered from in-band NALU")
+					log.Warn("mp4.updateVideoParameterSetsFromAnnexB(): SPS recovered from in-band NALU")
 				}
 			case 8: // PPS
 				if needPPS {
 					mp4.PPSNALUs = [][]byte{nalu}
 					needPPS = false
-					log.Log.Warning("mp4.updateVideoParameterSetsFromAnnexB(): PPS recovered from in-band NALU")
+					log.Warn("mp4.updateVideoParameterSetsFromAnnexB(): PPS recovered from in-band NALU")
 				}
 			}
 		case "H265", "HVC1":
@@ -232,19 +232,19 @@ func (mp4 *MP4) updateVideoParameterSetsFromAnnexB(data []byte) {
 				if needVPS {
 					mp4.VPSNALUs = [][]byte{nalu}
 					needVPS = false
-					log.Log.Warning("mp4.updateVideoParameterSetsFromAnnexB(): VPS recovered from in-band NALU")
+					log.Warn("mp4.updateVideoParameterSetsFromAnnexB(): VPS recovered from in-band NALU")
 				}
 			case 33: // SPS
 				if needSPS {
 					mp4.SPSNALUs = [][]byte{nalu}
 					needSPS = false
-					log.Log.Warning("mp4.updateVideoParameterSetsFromAnnexB(): SPS recovered from in-band NALU")
+					log.Warn("mp4.updateVideoParameterSetsFromAnnexB(): SPS recovered from in-band NALU")
 				}
 			case 34: // PPS
 				if needPPS {
 					mp4.PPSNALUs = [][]byte{nalu}
 					needPPS = false
-					log.Log.Warning("mp4.updateVideoParameterSetsFromAnnexB(): PPS recovered from in-band NALU")
+					log.Warn("mp4.updateVideoParameterSetsFromAnnexB(): PPS recovered from in-band NALU")
 				}
 			}
 		}
@@ -266,7 +266,12 @@ func (mp4 *MP4) flushPendingVideoSample(nextPTS uint64) bool {
 	} else {
 		// No valid nextPTS (Close case) or PTS went backwards (jitter/discontinuity)
 		if nextPTS > 0 {
-			log.Log.Warning(fmt.Sprintf("mp4.flushPendingVideoSample(): video PTS went backwards or zero duration (nextPTS=%d, prevDTS=%d), using last known duration", nextPTS, mp4.VideoFullSample.DecodeTime))
+			log.WithFields(log.Fields{
+				"component":                 "video/mp4",
+				"event":                     "invalid_video_timestamp",
+				"next_presentation_time_ms": nextPTS,
+				"previous_decode_time_ms":   mp4.VideoFullSample.DecodeTime,
+			}).Warn("Video timestamp moved backwards or produced zero duration")
 		}
 		duration = mp4.LastVideoSampleDTS
 		if duration == 0 {
@@ -282,15 +287,23 @@ func (mp4 *MP4) flushPendingVideoSample(nextPTS uint64) bool {
 	isKF := mp4.PendingSampleIsKeyframe
 	err := mp4.MultiTrackFragment.AddFullSampleToTrack(*mp4.VideoFullSample, uint32(mp4.VideoTrack))
 	if err != nil {
-		log.Log.Error("mp4.flushPendingVideoSample(): error adding sample: " + err.Error())
+		log.Error("mp4.flushPendingVideoSample(): error adding sample: " + err.Error())
 	} else {
 		mp4.SampleCount++
 	}
 	if isKF {
 		mp4.TotalKeyframesWritten++
 		mp4.FragmentKeyframeCount++
-		log.Log.Debug(fmt.Sprintf("mp4.flushPendingVideoSample(): KEYFRAME WRITTEN to trun - totalWritten=%d, fragmentKF=%d, flags=0x%08x, dur=%d, DTS=%d",
-			mp4.TotalKeyframesWritten, mp4.FragmentKeyframeCount, mp4.VideoFullSample.Sample.Flags, duration, mp4.VideoFullSample.DecodeTime))
+		log.WithFields(log.Fields{
+			"component":               "video/mp4",
+			"decode_time_ms":          mp4.VideoFullSample.DecodeTime,
+			"duration_ms":             duration,
+			"event":                   "keyframe_written",
+			"fragment_keyframe_count": mp4.FragmentKeyframeCount,
+			"sample_flags":            mp4.VideoFullSample.Sample.Flags,
+			"total_keyframes_written": mp4.TotalKeyframesWritten,
+		}).Debug("MP4 keyframe written")
+
 	}
 
 	mp4.VideoFullSample = nil
@@ -378,7 +391,14 @@ func (mp4 *MP4) AddSampleToTrack(trackID uint32, isKeyframe bool, data []byte, p
 		truncatedTail := fullGopFrames > 0 && bufferedVideo*2 < fullGopFrames
 		if closeKeyframe && truncatedTail {
 			seam = true
-			log.Log.Warning(fmt.Sprintf("mp4.AddSampleToTrack(): dropping truncated GOP at premature keyframe (interval=%d ms, min interval=%d ms, buffered video frames=%d of ~%d) - likely upstream loop/restart discontinuity", gap, mp4.MinKeyframeGapMs, bufferedVideo, fullGopFrames))
+			log.WithFields(log.Fields{
+				"buffered_video_frames":   bufferedVideo,
+				"component":               "video/mp4",
+				"event":                   "truncated_gop_dropped",
+				"expected_gop_frames":     fullGopFrames,
+				"keyframe_interval_ms":    gap,
+				"minimum_keyframe_gap_ms": mp4.MinKeyframeGapMs,
+			}).Warn("Dropping truncated GOP at premature keyframe")
 		}
 		mp4.LastKeyframeGapMs = gap
 		if !seam && (mp4.MinKeyframeGapMs == 0 || gap < mp4.MinKeyframeGapMs) {
@@ -483,7 +503,7 @@ func (mp4 *MP4) commitBufferedGOP() {
 	mp4.gopBuffer = nil // detach so commitSampleToTrack never observes a half-cleared buffer
 	for _, s := range buffered {
 		if err := mp4.commitSampleToTrack(s.trackID, s.isKeyframe, s.data, s.pts, s.compositionOffset); err != nil {
-			log.Log.Error("mp4.commitBufferedGOP(): " + err.Error())
+			log.Error("mp4.commitBufferedGOP(): " + err.Error())
 		}
 	}
 }
@@ -499,8 +519,18 @@ func (mp4 *MP4) commitSampleToTrack(trackID uint32, isKeyframe bool, data []byte
 		if mp4.Start {
 			elapsedDbg = pts - mp4.FragmentStartRawPTS
 		}
-		log.Log.Debug(fmt.Sprintf("mp4.AddSampleToTrack(): KEYFRAME #%d received - PTS=%d, size=%d, elapsed=%dms, started=%t, segment=%d, fragKF=%d",
-			mp4.TotalKeyframesReceived, pts, len(data), elapsedDbg, mp4.Start, mp4.SegmentCount, mp4.FragmentKeyframeCount))
+		log.WithFields(log.Fields{
+			"component":                "video/mp4",
+			"elapsed_ms":               elapsedDbg,
+			"event":                    "keyframe_received",
+			"fragment_keyframe_count":  mp4.FragmentKeyframeCount,
+			"presentation_time_ms":     pts,
+			"recording_started":        mp4.Start,
+			"sample_bytes":             len(data),
+			"segment_count":            mp4.SegmentCount,
+			"total_keyframes_received": mp4.TotalKeyframesReceived,
+		}).Debug("MP4 keyframe received")
+
 	}
 
 	if isKeyframe {
@@ -524,8 +554,15 @@ func (mp4 *MP4) commitSampleToTrack(trackID uint32, isKeyframe bool, data []byte
 					mp4.flushPendingVideoSample(pts)
 				}
 
-				log.Log.Debug(fmt.Sprintf("mp4.AddSampleToTrack(): FLUSHING segment #%d - keyframes_in_fragment=%d, totalKF_received=%d, totalKF_written=%d",
-					mp4.SegmentCount, mp4.FragmentKeyframeCount, mp4.TotalKeyframesReceived, mp4.TotalKeyframesWritten))
+				log.WithFields(log.Fields{
+					"component":                "video/mp4",
+					"event":                    "segment_flush_started",
+					"fragment_keyframe_count":  mp4.FragmentKeyframeCount,
+					"segment_count":            mp4.SegmentCount,
+					"total_keyframes_received": mp4.TotalKeyframesReceived,
+					"total_keyframes_written":  mp4.TotalKeyframesWritten,
+				}).Debug("Flushing MP4 segment")
+
 				mp4.MoofBoxes = mp4.MoofBoxes + 1
 				mp4.MoofBoxSizes = append(mp4.MoofBoxSizes, int64(mp4.Segment.Size()))
 				// Track the segment's duration and base decode time for sidx.
@@ -536,7 +573,7 @@ func (mp4 *MP4) commitSampleToTrack(trackID uint32, isKeyframe bool, data []byte
 				mp4.SegmentBaseDecTimes = append(mp4.SegmentBaseDecTimes, mp4.FragmentStartDTS)
 				err := mp4.Segment.Encode(mp4.Writer)
 				if err != nil {
-					log.Log.Error("mp4.AddSampleToTrack(): error encoding segment: " + err.Error())
+					log.Error("mp4.AddSampleToTrack(): error encoding segment: " + err.Error())
 				}
 				mp4.Segments = append(mp4.Segments, mp4.Segment)
 			}
@@ -552,7 +589,7 @@ func (mp4 *MP4) commitSampleToTrack(trackID uint32, isKeyframe bool, data []byte
 			// Create a video fragment
 			multiTrackFragment, err := mp4ff.CreateMultiTrackFragment(uint32(mp4.SegmentCount), mp4.TrackIDs)
 			if err != nil {
-				log.Log.Error("mp4.AddSampleToTrack(): error creating multi track fragment: " + err.Error())
+				log.Error("mp4.AddSampleToTrack(): error creating multi track fragment: " + err.Error())
 			}
 			mp4.MultiTrackFragment = multiTrackFragment
 			seg.AddFragment(multiTrackFragment)
@@ -585,7 +622,14 @@ func (mp4 *MP4) commitSampleToTrack(trackID uint32, isKeyframe bool, data []byte
 			if err == nil {
 				// Flush previous pending sample before storing the new one
 				if mp4.VideoFullSample != nil {
-					log.Log.Debug("Adding sample to track " + fmt.Sprintf("%d, PTS: %d, size: %d, Keyframe: %t", trackID, pts, len(lengthPrefixed), isKeyframe))
+					log.WithFields(log.Fields{
+						"bytes":     len(lengthPrefixed),
+						"component": "mp4",
+						"event":     "video_sample_added",
+						"keyframe":  isKeyframe,
+						"pts":       pts,
+						"track_id":  trackID,
+					}).Trace("Adding MP4 video sample")
 					mp4.flushPendingVideoSample(pts)
 				}
 
@@ -628,7 +672,11 @@ func (mp4 *MP4) commitSampleToTrack(trackID uint32, isKeyframe bool, data []byte
 					sampleToAdd.Sample.Size = uint32(len(aac[7:]))
 					err := mp4.MultiTrackFragment.AddFullSampleToTrack(sampleToAdd, trackID)
 					if err != nil {
-						log.Log.Error("mp4.AddSampleToTrack(): error adding sample to track " + fmt.Sprintf("%d: %v", trackID, err))
+						log.WithError(err).WithFields(log.Fields{
+							"component": "video/mp4",
+							"event":     "audio_sample_write_failed",
+							"track_id":  trackID,
+						}).Error("Failed to add audio sample to MP4 track")
 					}
 				})
 			}
@@ -657,11 +705,17 @@ func (mp4 *MP4) Close(config *models.Config) {
 	// a recording is never a loop seam, so it must always be written out.
 	mp4.commitBufferedGOP()
 
-	log.Log.Info(fmt.Sprintf("mp4.Close(): KEYFRAME SUMMARY - totalReceived=%d, totalWritten=%d, segments=%d, lastFragmentKF=%d",
-		mp4.TotalKeyframesReceived, mp4.TotalKeyframesWritten, mp4.SegmentCount, mp4.FragmentKeyframeCount))
+	log.WithFields(log.Fields{
+		"component":                "video/mp4",
+		"event":                    "recording_keyframe_summary",
+		"fragment_keyframe_count":  mp4.FragmentKeyframeCount,
+		"segment_count":            mp4.SegmentCount,
+		"total_keyframes_received": mp4.TotalKeyframesReceived,
+		"total_keyframes_written":  mp4.TotalKeyframesWritten,
+	}).Info("MP4 recording keyframe summary")
 
 	if mp4.VideoTotalDuration == 0 && mp4.AudioTotalDuration == 0 {
-		log.Log.Error("mp4.Close(): no video or audio samples added, removing empty MP4 file")
+		log.Error("mp4.Close(): no video or audio samples added, removing empty MP4 file")
 		mp4.Writer.Flush()
 		_ = mp4.FileWriter.Sync()
 		_ = mp4.FileWriter.Close()
@@ -690,7 +744,7 @@ func (mp4 *MP4) Close(config *models.Config) {
 				sampleToAdd.Sample.Size = uint32(len(aac[7:]))
 				err := mp4.MultiTrackFragment.AddFullSampleToTrack(sampleToAdd, uint32(mp4.AudioTrack))
 				if err != nil {
-					log.Log.Error("mp4.Close(): error adding final audio sample: " + err.Error())
+					log.Error("mp4.Close(): error adding final audio sample: " + err.Error())
 				}
 			})
 			mp4.AudioFullSample = nil
@@ -712,14 +766,14 @@ func (mp4 *MP4) Close(config *models.Config) {
 
 		err := mp4.Segment.Encode(mp4.Writer)
 		if err != nil {
-			log.Log.Error("mp4.Close(): error encoding last segment: " + err.Error())
+			log.Error("mp4.Close(): error encoding last segment: " + err.Error())
 		}
 	}
 
 	mp4.Writer.Flush()
 	// Ensure all segment data is on disk before we overwrite the placeholder at offset 0.
 	if err := mp4.FileWriter.Sync(); err != nil {
-		log.Log.Error("mp4.Close(): error syncing file: " + err.Error())
+		log.Error("mp4.Close(): error syncing file: " + err.Error())
 	}
 
 	// Now we have all the moof and mdat boxes written to the file.
@@ -747,8 +801,14 @@ func (mp4 *MP4) Close(config *models.Config) {
 		actualVideoDuration += d
 	}
 	if actualVideoDuration != mp4.VideoTotalDuration {
-		log.Log.Warning(fmt.Sprintf("mp4.Close(): duration mismatch: accumulated VideoTotalDuration=%d, sum of segment durations=%d (diff=%d ms)",
-			mp4.VideoTotalDuration, actualVideoDuration, int64(mp4.VideoTotalDuration)-int64(actualVideoDuration)))
+		log.WithFields(log.Fields{
+			"accumulated_duration_ms": mp4.VideoTotalDuration,
+			"component":               "video/mp4",
+			"difference_ms":           int64(mp4.VideoTotalDuration) - int64(actualVideoDuration),
+			"event":                   "duration_mismatch",
+			"segment_duration_ms":     actualVideoDuration,
+		}).Warn("MP4 video duration does not match segment durations")
+
 	}
 
 	// Set the creation time and modification time for the moov box.
@@ -790,21 +850,40 @@ func (mp4 *MP4) Close(config *models.Config) {
 		init.AddEmptyTrack(videoTimescale, "video", "und")
 		includePS := true
 		spsNALUs, ppsNALUs := normalizeH264ParameterSets(mp4.SPSNALUs, mp4.PPSNALUs)
-		log.Log.Debug("mp4.Close(): AVC parameter sets: SPS=" + formatNaluDebug(spsNALUs) + ", PPS=" + formatNaluDebug(ppsNALUs))
+		log.WithFields(log.Fields{
+			"component": "video/mp4",
+			"event":     "avc_parameter_sets",
+			"pps":       formatNaluDebug(ppsNALUs),
+			"sps":       formatNaluDebug(spsNALUs),
+		}).Debug("AVC parameter sets normalized")
 		if len(spsNALUs) == 0 || len(ppsNALUs) == 0 {
 			// An avcC without both SPS and PPS is invalid: downstream FFmpeg-based
 			// pipelines decoding this file will report "non-existing PPS 0 referenced"
 			// and fail to extract any frame. Surface it loudly so the capture-side
 			// parameter-set handling can be diagnosed.
-			log.Log.Error(fmt.Sprintf("mp4.Close(): incomplete H264 parameter sets (SPS=%d, PPS=%d) - the avcC will be invalid and downstream decoders will report 'non-existing PPS 0 referenced'", len(spsNALUs), len(ppsNALUs)))
+			log.WithFields(log.Fields{
+				"component": "video/mp4",
+				"event":     "avc_parameter_sets_incomplete",
+				"pps_count": len(ppsNALUs),
+				"sps_count": len(spsNALUs),
+			}).Error("Incomplete H264 parameter sets will produce an invalid AVC descriptor")
 		}
 		err := init.Moov.Traks[0].SetAVCDescriptor("avc1", spsNALUs, ppsNALUs, includePS)
 		if err != nil {
-			log.Log.Error("mp4.Close(): error setting AVC descriptor: " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component": "video/mp4",
+				"event":     "avc_descriptor_failed",
+			}).Error("Failed to set AVC descriptor")
 			if fallbackErr := addAVCDescriptorFallback(init.Moov.Traks[0], spsNALUs, ppsNALUs, uint16(mp4.width), uint16(mp4.height)); fallbackErr != nil {
-				log.Log.Error("mp4.Close(): error setting AVC descriptor fallback: " + fallbackErr.Error())
+				log.WithError(fallbackErr).WithFields(log.Fields{
+					"component": "video/mp4",
+					"event":     "avc_descriptor_fallback_failed",
+				}).Error("Failed to set fallback AVC descriptor")
 			} else {
-				log.Log.Warning("mp4.Close(): AVC descriptor fallback used due to SPS parse error")
+				log.WithFields(log.Fields{
+					"component": "video/mp4",
+					"event":     "avc_descriptor_fallback_used",
+				}).Warn("Using fallback AVC descriptor")
 			}
 		}
 		init.Moov.Traks[0].Tkhd.Duration = actualVideoDuration
@@ -823,15 +902,30 @@ func (mp4 *MP4) Close(config *models.Config) {
 		init.AddEmptyTrack(videoTimescale, "video", "und")
 		includePS := true
 		vpsNALUs, spsNALUs, ppsNALUs := normalizeH265ParameterSets(mp4.VPSNALUs, mp4.SPSNALUs, mp4.PPSNALUs)
-		log.Log.Debug("mp4.Close(): HEVC parameter sets: VPS=" + formatNaluDebug(vpsNALUs) + ", SPS=" + formatNaluDebug(spsNALUs) + ", PPS=" + formatNaluDebug(ppsNALUs))
+		log.WithFields(log.Fields{
+			"component": "video/mp4",
+			"event":     "hevc_parameter_sets",
+			"pps":       formatNaluDebug(ppsNALUs),
+			"sps":       formatNaluDebug(spsNALUs),
+			"vps":       formatNaluDebug(vpsNALUs),
+		}).Debug("HEVC parameter sets normalized")
 		if len(vpsNALUs) == 0 || len(spsNALUs) == 0 || len(ppsNALUs) == 0 {
 			// An hvcC missing VPS/SPS/PPS is invalid and downstream FFmpeg-based
 			// pipelines will fail to decode the recording. Surface it loudly.
-			log.Log.Error(fmt.Sprintf("mp4.Close(): incomplete H265 parameter sets (VPS=%d, SPS=%d, PPS=%d) - the hvcC will be invalid and downstream decoders will fail to process the recording", len(vpsNALUs), len(spsNALUs), len(ppsNALUs)))
+			log.WithFields(log.Fields{
+				"component": "video/mp4",
+				"event":     "hevc_parameter_sets_incomplete",
+				"pps_count": len(ppsNALUs),
+				"sps_count": len(spsNALUs),
+				"vps_count": len(vpsNALUs),
+			}).Error("Incomplete H265 parameter sets will produce an invalid HEVC descriptor")
 		}
 		err := init.Moov.Traks[0].SetHEVCDescriptor("hvc1", vpsNALUs, spsNALUs, ppsNALUs, [][]byte{}, includePS)
 		if err != nil {
-			log.Log.Error("mp4.Close(): error setting HEVC descriptor: " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component": "video/mp4",
+				"event":     "hevc_descriptor_failed",
+			}).Error("Failed to set HEVC descriptor")
 		}
 		init.Moov.Traks[0].Tkhd.Duration = actualVideoDuration
 		init.Moov.Traks[0].Tkhd.Width = mp4ff.Fixed32(uint32(mp4.width) << 16)
@@ -876,7 +970,7 @@ func (mp4 *MP4) Close(config *models.Config) {
 		// Set the subtitle descriptor
 		err := init.Moov.Traks[2].SetWvttDescriptor("")
 		if err != nil {
-			//log.Log.Error("mp4.Close(): error setting VTT descriptor: " + err.Error())
+			// log.Error("mp4.Close(): error setting VTT descriptor: " + err.Error())
 			//return
 		}
 		init.Moov.Traks[2].Mdia.Hdlr.Name = "agent " + utils.VERSION
@@ -924,14 +1018,14 @@ func (mp4 *MP4) Close(config *models.Config) {
 	block, _ := pem.Decode(pemBytes)
 
 	if block == nil {
-		//log.Log.Error("mp4.Close(): error decoding PEM block containing private key")
+		// log.Error("mp4.Close(): error decoding PEM block containing private key")
 		//return
 	} else {
 		// Parse private key
 		b := block.Bytes
 		key, err := x509.ParsePKCS8PrivateKey(b)
 		if err != nil {
-			//log.Log.Error("mp4.Close(): error parsing private key: " + err.Error())
+			// log.Error("mp4.Close(): error parsing private key: " + err.Error())
 			//return
 		} else {
 			// Conver key to *rsa.PrivateKey
@@ -944,7 +1038,7 @@ func (mp4 *MP4) Close(config *models.Config) {
 				uuid.UnknownPayload = signature
 				init.Moov.AddChild(uuid)
 			} else {
-				//log.Log.Error("mp4.Close(): error signing fingerprint: " + err.Error())
+				// log.Error("mp4.Close(): error signing fingerprint: " + err.Error())
 			}
 		}
 	}
@@ -979,7 +1073,7 @@ func (mp4 *MP4) Close(config *models.Config) {
 	// and re-encode with the corrected value.
 	var initBuf bytes.Buffer
 	if err := init.Encode(&initBuf); err != nil {
-		log.Log.Error("mp4.Close(): error encoding init segment: " + err.Error())
+		log.Error("mp4.Close(): error encoding init segment: " + err.Error())
 	}
 
 	initSize := int64(initBuf.Len())
@@ -991,7 +1085,7 @@ func (mp4 *MP4) Close(config *models.Config) {
 	if len(mp4.SegmentDurations) > 0 {
 		if mp4.FreeBoxSize < initSize {
 			// Avoid computing a negative offset and wrapping it to uint64.
-			log.Log.Error("mp4.Close(): FreeBoxSize is smaller than initSize; skipping sidx FirstOffset adjustment")
+			log.Error("mp4.Close(): FreeBoxSize is smaller than initSize; skipping sidx FirstOffset adjustment")
 		} else {
 			firstOffset := uint64(mp4.FreeBoxSize - initSize)
 			// Find the sidx we added and update its FirstOffset
@@ -1004,19 +1098,24 @@ func (mp4 *MP4) Close(config *models.Config) {
 			// Re-encode with the corrected FirstOffset (same size, no layout change)
 			initBuf.Reset()
 			if err := init.Encode(&initBuf); err != nil {
-				log.Log.Error("mp4.Close(): error re-encoding init segment: " + err.Error())
+				log.Error("mp4.Close(): error re-encoding init segment: " + err.Error())
 			}
 			initSize = int64(initBuf.Len())
 		}
 	}
 
 	if initSize > mp4.FreeBoxSize {
-		log.Log.Error(fmt.Sprintf("mp4.Close(): init segment (%d bytes) exceeds reserved space (%d bytes), file may be corrupt", initSize, mp4.FreeBoxSize))
+		log.WithFields(log.Fields{
+			"component":      "video/mp4",
+			"event":          "init_segment_oversized",
+			"init_bytes":     initSize,
+			"reserved_bytes": mp4.FreeBoxSize,
+		}).Error("MP4 init segment exceeds reserved space")
 	}
 
 	// Write the init segment at the beginning of the file, overwriting the free box placeholder.
 	if _, err := mp4.FileWriter.WriteAt(initBuf.Bytes(), 0); err != nil {
-		log.Log.Error("mp4.Close(): error writing init segment: " + err.Error())
+		log.Error("mp4.Close(): error writing init segment: " + err.Error())
 	}
 
 	// Fill any remaining reserved space with a new (smaller) free box so
@@ -1026,15 +1125,15 @@ func (mp4 *MP4) Close(config *models.Config) {
 		newFree := mp4ff.NewFreeBox(make([]byte, remainingSize-8))
 		var freeBuf bytes.Buffer
 		if err := newFree.Encode(&freeBuf); err != nil {
-			log.Log.Error("mp4.Close(): error encoding free box: " + err.Error())
+			log.Error("mp4.Close(): error encoding free box: " + err.Error())
 		}
 		if _, err := mp4.FileWriter.WriteAt(freeBuf.Bytes(), initSize); err != nil {
-			log.Log.Error("mp4.Close(): error writing free box: " + err.Error())
+			log.Error("mp4.Close(): error writing free box: " + err.Error())
 		}
 	}
 
 	if err := mp4.FileWriter.Sync(); err != nil {
-		log.Log.Error("mp4.Close(): error syncing file: " + err.Error())
+		log.Error("mp4.Close(): error syncing file: " + err.Error())
 	}
 	mp4.FileWriter.Close()
 }

machinery/src/webrtc/aac_transcoder_ffmpeg.go

@@ -193,10 +193,9 @@ import "C"
 
 import (
 	"errors"
-	"fmt"
 	"unsafe"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
+	log "github.com/sirupsen/logrus"
 	"github.com/zaf/g711"
 )
 
@@ -216,7 +215,7 @@ func NewAACTranscoder() (*AACTranscoder, error) {
 	if h == nil {
 		return nil, errors.New("failed to create AAC transcoder (FFmpeg AAC decoder not available?)")
 	}
-	log.Log.Info("webrtc.aac_transcoder: AAC → G.711 µ-law transcoder initialised (FFmpeg)")
+	log.Info("webrtc.aac_transcoder: AAC → G.711 µ-law transcoder initialised (FFmpeg)")
 	return &AACTranscoder{handle: h}, nil
 }
 
@@ -250,9 +249,16 @@ func (t *AACTranscoder) Transcode(adtsData []byte) ([]byte, error) {
 
 	// Log resampler details once.
 	if t.handle.swr_initialized == 1 && t.handle.in_sample_rate != 0 {
-		log.Log.Info(fmt.Sprintf(
-			"webrtc.aac_transcoder: first output – resampling %d Hz / %d ch → 8000 Hz mono → µ-law",
-			int(t.handle.in_sample_rate), int(t.handle.in_channels)))
+		log.WithFields(log.Fields{
+			"component":        "webrtc",
+			"event":            "aac_resampler_initialized",
+			"input_channels":   int(t.handle.in_channels),
+			"input_sample_hz":  int(t.handle.in_sample_rate),
+			"output_channels":  1,
+			"output_codec":     "PCMU",
+			"output_sample_hz": 8000,
+		}).Info("AAC resampler initialized")
+
 		// Prevent repeated logging by zeroing the field we check.
 		t.handle.in_sample_rate = 0
 	}
@@ -265,6 +271,6 @@ func (t *AACTranscoder) Close() {
 	if t != nil && t.handle != nil {
 		C.aac_transcoder_destroy(t.handle)
 		t.handle = nil
-		log.Log.Info("webrtc.aac_transcoder: transcoder closed")
+		log.Info("webrtc.aac_transcoder: transcoder closed")
 	}
 }

machinery/src/webrtc/aac_transcoder_stub.go

@@ -15,7 +15,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
+	log "github.com/sirupsen/logrus"
 )
 
 // AACTranscodingAvailable reports whether AAC→PCMU transcoding
@@ -45,7 +45,7 @@ func NewAACTranscoder() (*AACTranscoder, error) {
 	if err != nil {
 		return nil, errors.New("AAC transcoding not available: ffmpeg binary not found in PATH")
 	}
-	log.Log.Info("webrtc.aac_transcoder: using ffmpeg binary at " + ffmpegPath)
+	log.Info("webrtc.aac_transcoder: using ffmpeg binary at " + ffmpegPath)
 
 	cmd := exec.Command(
 		ffmpegPath,
@@ -96,19 +96,19 @@ func NewAACTranscoder() (*AACTranscoder, error) {
 				buffered := t.outBuf.Len()
 				t.outMu.Unlock()
 				if buffered <= 8192 || buffered%16000 == 0 {
-					log.Log.Debug("webrtc.aac_transcoder: ffmpeg produced PCMU bytes, buffered=" + strconv.Itoa(buffered))
+					log.Debug("webrtc.aac_transcoder: ffmpeg produced PCMU bytes, buffered=" + strconv.Itoa(buffered))
 				}
 			}
 			if readErr != nil {
 				if readErr != io.EOF {
-					log.Log.Warning("webrtc.aac_transcoder: stdout reader stopped: " + readErr.Error())
+					log.Warn("webrtc.aac_transcoder: stdout reader stopped: " + readErr.Error())
 				}
 				return
 			}
 		}
 	}()
 
-	log.Log.Info("webrtc.aac_transcoder: AAC → PCMU transcoder initialised (ffmpeg process)")
+	log.Info("webrtc.aac_transcoder: AAC → PCMU transcoder initialised (ffmpeg process)")
 	return t, nil
 }
 
@@ -129,22 +129,22 @@ func (t *AACTranscoder) Transcode(adtsData []byte) ([]byte, error) {
 		return nil, err
 	}
 	if len(adtsData) <= 512 || len(adtsData)%1024 == 0 {
-		log.Log.Debug("webrtc.aac_transcoder: wrote AAC bytes to ffmpeg, input=" + strconv.Itoa(len(adtsData)))
+		log.Debug("webrtc.aac_transcoder: wrote AAC bytes to ffmpeg, input=" + strconv.Itoa(len(adtsData)))
 	}
 
 	deadline := time.Now().Add(75 * time.Millisecond)
 	for {
 		data := t.readAvailable()
 		if len(data) > 0 {
-			log.Log.Debug("webrtc.aac_transcoder: returning PCMU bytes=" + strconv.Itoa(len(data)))
+			log.Debug("webrtc.aac_transcoder: returning PCMU bytes=" + strconv.Itoa(len(data)))
 			return data, nil
 		}
 
 		if time.Now().After(deadline) {
 			if stderr := t.stderrString(); stderr != "" {
-				log.Log.Warning("webrtc.aac_transcoder: no output before deadline, ffmpeg stderr: " + stderr)
+				log.Warn("webrtc.aac_transcoder: no output before deadline, ffmpeg stderr: " + stderr)
 			} else {
-				log.Log.Debug("webrtc.aac_transcoder: no PCMU output before deadline")
+				log.Debug("webrtc.aac_transcoder: no PCMU output before deadline")
 			}
 			return nil, nil
 		}
@@ -198,7 +198,7 @@ func (t *AACTranscoder) Close() {
 			_ = t.cmd.Process.Kill()
 			_, _ = t.cmd.Process.Wait()
 			if stderr := t.stderrString(); stderr != "" {
-				log.Log.Info("webrtc.aac_transcoder: ffmpeg stderr on close: " + stderr)
+				log.Info("webrtc.aac_transcoder: ffmpeg stderr on close: " + stderr)
 			}
 		}
 	})

machinery/src/webrtc/broadcaster.go

@@ -4,9 +4,9 @@ import (
 	"io"
 	"sync"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	pionWebRTC "github.com/pion/webrtc/v4"
 	pionMedia "github.com/pion/webrtc/v4/pkg/media"
+	log "github.com/sirupsen/logrus"
 )
 
 const (
@@ -74,12 +74,20 @@ func (b *TrackBroadcaster) AddPeer(sessionKey string) (*pionWebRTC.TrackLocalSta
 				if err == io.ErrClosedPipe {
 					return
 				}
-				log.Log.Error("webrtc.broadcaster.peerWriter(): error writing sample for " + sessionKey + ": " + err.Error())
+				log.WithError(err).WithFields(log.Fields{
+					"component":  "webrtc",
+					"event":      "sample_write_failed",
+					"session_id": sessionKey,
+				}).Error("Failed to write WebRTC sample")
 			}
 		}
 	}()
 
-	log.Log.Info("webrtc.broadcaster.AddPeer(): added peer track for " + sessionKey)
+	log.WithFields(log.Fields{
+		"component":  "webrtc",
+		"event":      "peer_added",
+		"session_id": sessionKey,
+	}).Info("WebRTC peer added")
 	return track, nil
 }
 
@@ -95,7 +103,11 @@ func (b *TrackBroadcaster) RemovePeer(sessionKey string) {
 	if exists {
 		close(pt.samples)
 		<-pt.done // wait for writer goroutine to finish
-		log.Log.Info("webrtc.broadcaster.RemovePeer(): removed peer track for " + sessionKey)
+		log.WithFields(log.Fields{
+			"component":  "webrtc",
+			"event":      "peer_removed",
+			"session_id": sessionKey,
+		}).Info("WebRTC peer removed")
 	}
 }
 
@@ -110,7 +122,11 @@ func (b *TrackBroadcaster) WriteSample(sample pionMedia.Sample) {
 		select {
 		case pt.samples <- sample:
 		default:
-			log.Log.Warning("webrtc.broadcaster.WriteSample(): dropping sample for slow peer " + sessionKey)
+			log.WithFields(log.Fields{
+				"component":  "webrtc",
+				"event":      "slow_peer_sample_dropped",
+				"session_id": sessionKey,
+			}).Warn("Dropping WebRTC sample for slow peer")
 		}
 	}
 }

machinery/src/webrtc/main.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/base64"
 	"encoding/json"
-	"fmt"
 	"io"
 	"strconv"
 	"strings"
@@ -13,9 +12,9 @@ import (
 	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/capture"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
+	log "github.com/sirupsen/logrus"
 
 	mqtt "github.com/eclipse/paho.mqtt.golang"
 	"github.com/pion/interceptor"
@@ -180,7 +179,7 @@ func cleanupPeerConnection(sessionKey string, wrapper *peerConnectionWrapper) {
 	wrapper.closeOnce.Do(func() {
 		if wrapper.connected.Swap(false) {
 			count := globalConnectionManager.DecrementPeerCount()
-			log.Log.Info("webrtc.main.cleanupPeerConnection(): Peer disconnected. Active peers: " + strconv.FormatInt(count, 10))
+			log.Info("webrtc.main.cleanupPeerConnection(): Peer disconnected. Active peers: " + strconv.FormatInt(count, 10))
 		}
 
 		// Remove per-peer tracks from broadcasters so the fan-out stops
@@ -196,7 +195,7 @@ func cleanupPeerConnection(sessionKey string, wrapper *peerConnectionWrapper) {
 
 		if wrapper.conn != nil {
 			if err := wrapper.conn.Close(); err != nil {
-				log.Log.Error("webrtc.main.cleanupPeerConnection(): error closing peer connection: " + err.Error())
+				log.Error("webrtc.main.cleanupPeerConnection(): error closing peer connection: " + err.Error())
 			}
 		}
 
@@ -226,10 +225,35 @@ func CreateWebRTC(name string, stunServers []string, turnServers []string, turnS
 	}
 }
 
+func nonEmptyICEURLs(urls []string) []string {
+	nonEmpty := make([]string, 0, len(urls))
+	for _, uri := range urls {
+		if uri = strings.TrimSpace(uri); uri != "" {
+			nonEmpty = append(nonEmpty, uri)
+		}
+	}
+	return nonEmpty
+}
+
+func buildICEServers(w WebRTC) []pionWebRTC.ICEServer {
+	iceServers := make([]pionWebRTC.ICEServer, 0, 2)
+	if stunURLs := nonEmptyICEURLs(w.StunServers); len(stunURLs) > 0 {
+		iceServers = append(iceServers, pionWebRTC.ICEServer{URLs: stunURLs})
+	}
+	if turnURLs := nonEmptyICEURLs(w.TurnServers); len(turnURLs) > 0 {
+		iceServers = append(iceServers, pionWebRTC.ICEServer{
+			URLs:       turnURLs,
+			Username:   w.TurnServersUsername,
+			Credential: w.TurnServersCredential,
+		})
+	}
+	return iceServers
+}
+
 func (w WebRTC) DecodeSessionDescription(data string) ([]byte, error) {
 	sd, err := base64.StdEncoding.DecodeString(data)
 	if err != nil {
-		log.Log.Error("webrtc.main.DecodeSessionDescription(): " + err.Error())
+		log.Error("webrtc.main.DecodeSessionDescription(): " + err.Error())
 		return []byte{}, err
 	}
 	return sd, nil
@@ -244,9 +268,9 @@ func (w WebRTC) CreateOffer(sd []byte) pionWebRTC.SessionDescription {
 }
 
 func RegisterCandidates(key string, candidate models.ReceiveHDCandidatesPayload) {
-	log.Log.Info("webrtc.main.RegisterCandidates(): " + candidate.Candidate)
+	log.Info("webrtc.main.RegisterCandidates(): " + candidate.Candidate)
 	if !globalConnectionManager.QueueCandidate(key, candidate.Candidate) {
-		log.Log.Info("webrtc.main.RegisterCandidates(): channel is full, dropping candidate")
+		log.Info("webrtc.main.RegisterCandidates(): channel is full, dropping candidate")
 	}
 }
 
@@ -280,18 +304,18 @@ func RegisterDefaultInterceptors(mediaEngine *pionWebRTC.MediaEngine, intercepto
 
 func publishSignalingMessageAsync(mqttClient mqtt.Client, topic string, payload []byte, description string) {
 	if mqttClient == nil {
-		log.Log.Error("webrtc.main.publishSignalingMessageAsync(): mqtt client is nil for " + description)
+		log.Error("webrtc.main.publishSignalingMessageAsync(): mqtt client is nil for " + description)
 		return
 	}
 
 	token := mqttClient.Publish(topic, 2, false, payload)
 	go func() {
 		if !token.WaitTimeout(5 * time.Second) {
-			log.Log.Warning("webrtc.main.publishSignalingMessageAsync(): timed out publishing " + description)
+			log.Warn("webrtc.main.publishSignalingMessageAsync(): timed out publishing " + description)
 			return
 		}
 		if err := token.Error(); err != nil {
-			log.Log.Error("webrtc.main.publishSignalingMessageAsync(): failed publishing " + description + ": " + err.Error())
+			log.Error("webrtc.main.publishSignalingMessageAsync(): failed publishing " + description + ": " + err.Error())
 		}
 	}()
 }
@@ -299,7 +323,7 @@ func publishSignalingMessageAsync(mqttClient mqtt.Client, topic string, payload
 func sendCandidateSignal(configuration *models.Configuration, mqttClient mqtt.Client, hubKey string, handshake models.LiveHDHandshake, candidateJSON []byte) {
 	if handshake.Signaling != nil && handshake.Signaling.SendCandidate != nil {
 		if err := handshake.Signaling.SendCandidate(handshake.Payload.SessionID, string(candidateJSON)); err != nil {
-			log.Log.Error("webrtc.main.sendCandidateSignal(): " + err.Error())
+			log.Error("webrtc.main.sendCandidateSignal(): " + err.Error())
 		}
 		return
 	}
@@ -318,7 +342,7 @@ func sendCandidateSignal(configuration *models.Configuration, mqttClient mqtt.Cl
 	if err == nil {
 		publishSignalingMessageAsync(mqttClient, "kerberos/hub/"+hubKey, payload, "ICE candidate for session "+handshake.Payload.SessionID)
 	} else {
-		log.Log.Info("webrtc.main.sendCandidateSignal(): while packaging mqtt message: " + err.Error())
+		log.Info("webrtc.main.sendCandidateSignal(): while packaging mqtt message: " + err.Error())
 	}
 }
 
@@ -327,7 +351,7 @@ func sendAnswerSignal(configuration *models.Configuration, mqttClient mqtt.Clien
 
 	if handshake.Signaling != nil && handshake.Signaling.SendAnswer != nil {
 		if err := handshake.Signaling.SendAnswer(handshake.Payload.SessionID, encodedAnswer); err != nil {
-			log.Log.Error("webrtc.main.sendAnswerSignal(): " + err.Error())
+			log.Error("webrtc.main.sendAnswerSignal(): " + err.Error())
 		}
 		return
 	}
@@ -346,7 +370,7 @@ func sendAnswerSignal(configuration *models.Configuration, mqttClient mqtt.Clien
 	if err == nil {
 		publishSignalingMessageAsync(mqttClient, "kerberos/hub/"+hubKey, payload, "SDP answer for session "+handshake.Payload.SessionID)
 	} else {
-		log.Log.Info("webrtc.main.sendAnswerSignal(): while packaging mqtt message: " + err.Error())
+		log.Info("webrtc.main.sendAnswerSignal(): while packaging mqtt message: " + err.Error())
 	}
 }
 
@@ -368,7 +392,11 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 	// first so we start from a clean slate. Without this, the new request would
 	// race against a stale PC that still owns the per-peer broadcaster tracks.
 	if globalConnectionManager.CloseExistingPeerConnection(sessionKey) {
-		log.Log.Info("webrtc.main.InitializeWebRTCConnection(): closed stale peer connection for session " + handshakePayload.SessionID)
+		log.WithFields(log.Fields{
+			"component":  "webrtc",
+			"event":      "stale_peer_closed",
+			"session_id": handshakePayload.SessionID,
+		}).Debug("Closed stale WebRTC peer connection")
 	}
 	// Drain/reset the candidate channel too \u2014 leftover candidates from the
 	// prior session are not valid for the new ICE agent.
@@ -387,7 +415,7 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 
 		mediaEngine := &pionWebRTC.MediaEngine{}
 		if err := mediaEngine.RegisterDefaultCodecs(); err != nil {
-			log.Log.Error("webrtc.main.InitializeWebRTCConnection(): something went wrong registering codecs for media engine: " + err.Error())
+			log.Error("webrtc.main.InitializeWebRTCConnection(): something went wrong registering codecs for media engine: " + err.Error())
 		}
 
 		// Create a InterceptorRegistry. This is the user configurable RTP/RTCP Pipeline.
@@ -423,21 +451,29 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 
 		peerConnection, err := api.NewPeerConnection(
 			pionWebRTC.Configuration{
-				ICEServers: []pionWebRTC.ICEServer{
-					{
-						URLs: w.StunServers,
-					},
-					{
-						URLs:       w.TurnServers,
-						Username:   w.TurnServersUsername,
-						Credential: w.TurnServersCredential,
-					},
-				},
+				ICEServers:         buildICEServers(*w),
 				ICETransportPolicy: policy,
 			},
 		)
 
-		if err == nil && peerConnection != nil {
+		if err != nil {
+			globalConnectionManager.CloseCandidateChannel(sessionKey)
+			log.Error("webrtc.main.InitializeWebRTCConnection(): failed to create peer connection: " + err.Error() +
+				" (STUN configured: " + strconv.FormatBool(strings.TrimSpace(config.STUNURI) != "") +
+				", TURN configured: " + strconv.FormatBool(strings.TrimSpace(config.TURNURI) != "") +
+				", TURN username configured: " + strconv.FormatBool(config.TURNUsername != "") +
+				", TURN credential configured: " + strconv.FormatBool(config.TURNPassword != "") +
+				", ForceTurn: " + config.ForceTurn + ")")
+
+			return
+		}
+		if peerConnection == nil {
+			globalConnectionManager.CloseCandidateChannel(sessionKey)
+			log.Error("webrtc.main.InitializeWebRTCConnection(): failed to create peer connection: pion returned a nil connection")
+			return
+		}
+
+		{
 
 			// Create context for this connection
 			ctx, cancel := context.WithCancel(context.Background())
@@ -456,17 +492,17 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 			if videoBroadcaster != nil {
 				peerVideoTrack, trackErr := videoBroadcaster.AddPeer(sessionKey)
 				if trackErr != nil {
-					log.Log.Error("webrtc.main.InitializeWebRTCConnection(): error creating per-peer video track: " + trackErr.Error())
+					log.Error("webrtc.main.InitializeWebRTCConnection(): error creating per-peer video track: " + trackErr.Error())
 					cleanupPeerConnection(sessionKey, wrapper)
 					return
 				}
 				if videoSender, err = peerConnection.AddTrack(peerVideoTrack); err != nil {
-					log.Log.Error("webrtc.main.InitializeWebRTCConnection(): error adding video track: " + err.Error())
+					log.Error("webrtc.main.InitializeWebRTCConnection(): error adding video track: " + err.Error())
 					cleanupPeerConnection(sessionKey, wrapper)
 					return
 				}
 			} else {
-				log.Log.Info("webrtc.main.InitializeWebRTCConnection(): video track is nil, skipping video")
+				log.Info("webrtc.main.InitializeWebRTCConnection(): video track is nil, skipping video")
 			}
 
 			// Read incoming RTCP packets
@@ -475,7 +511,7 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 			if videoSender != nil {
 				go func() {
 					defer func() {
-						log.Log.Info("webrtc.main.InitializeWebRTCConnection(): video RTCP reader stopped")
+						log.Info("webrtc.main.InitializeWebRTCConnection(): video RTCP reader stopped")
 					}()
 					rtcpBuf := make([]byte, rtcpBufferSize)
 					for {
@@ -496,17 +532,17 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 			if audioBroadcaster != nil {
 				peerAudioTrack, trackErr := audioBroadcaster.AddPeer(sessionKey)
 				if trackErr != nil {
-					log.Log.Error("webrtc.main.InitializeWebRTCConnection(): error creating per-peer audio track: " + trackErr.Error())
+					log.Error("webrtc.main.InitializeWebRTCConnection(): error creating per-peer audio track: " + trackErr.Error())
 					cleanupPeerConnection(sessionKey, wrapper)
 					return
 				}
 				if audioSender, err = peerConnection.AddTrack(peerAudioTrack); err != nil {
-					log.Log.Error("webrtc.main.InitializeWebRTCConnection(): error adding audio track: " + err.Error())
+					log.Error("webrtc.main.InitializeWebRTCConnection(): error adding audio track: " + err.Error())
 					cleanupPeerConnection(sessionKey, wrapper)
 					return
 				}
 			} else {
-				log.Log.Info("webrtc.main.InitializeWebRTCConnection(): audio track is nil, skipping audio")
+				log.Info("webrtc.main.InitializeWebRTCConnection(): audio track is nil, skipping audio")
 			}
 
 			// Read incoming RTCP packets
@@ -515,7 +551,7 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 			if audioSender != nil {
 				go func() {
 					defer func() {
-						log.Log.Info("webrtc.main.InitializeWebRTCConnection(): audio RTCP reader stopped")
+						log.Info("webrtc.main.InitializeWebRTCConnection(): audio RTCP reader stopped")
 					}()
 					rtcpBuf := make([]byte, rtcpBufferSize)
 					for {
@@ -533,12 +569,13 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 
 			// Log ICE connection state changes for diagnostics
 			peerConnection.OnICEConnectionStateChange(func(iceState pionWebRTC.ICEConnectionState) {
-				log.Log.Info("webrtc.main.InitializeWebRTCConnection(): ICE connection state changed to: " + iceState.String() +
+				log.Info("webrtc.main.InitializeWebRTCConnection(): ICE connection state changed to: " + iceState.String() +
 					" (session: " + handshakePayload.SessionID + ")")
+
 			})
 
 			peerConnection.OnConnectionStateChange(func(connectionState pionWebRTC.PeerConnectionState) {
-				log.Log.Info("webrtc.main.InitializeWebRTCConnection(): connection state changed to: " + connectionState.String() +
+				log.Info("webrtc.main.InitializeWebRTCConnection(): connection state changed to: " + connectionState.String() +
 					" (session: " + handshakePayload.SessionID + ")")
 
 				switch connectionState {
@@ -547,10 +584,15 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 					// Start a grace period timer; if we don't recover, then cleanup.
 					wrapper.disconnectMu.Lock()
 					if wrapper.disconnectTimer == nil {
-						log.Log.Info("webrtc.main.InitializeWebRTCConnection(): peer disconnected, waiting " +
+						log.Info("webrtc.main.InitializeWebRTCConnection(): peer disconnected, waiting " +
 							disconnectGracePeriod.String() + " for recovery (session: " + handshakePayload.SessionID + ")")
+
 						wrapper.disconnectTimer = time.AfterFunc(disconnectGracePeriod, func() {
-							log.Log.Info("webrtc.main.InitializeWebRTCConnection(): disconnect grace period expired, closing connection (session: " + handshakePayload.SessionID + ")")
+							log.WithFields(log.Fields{
+								"component":  "webrtc",
+								"event":      "disconnect_grace_expired",
+								"session_id": handshakePayload.SessionID,
+							}).Info("WebRTC disconnect grace period expired")
 							cleanupPeerConnection(sessionKey, wrapper)
 						})
 					}
@@ -582,13 +624,17 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 					if wrapper.disconnectTimer != nil {
 						wrapper.disconnectTimer.Stop()
 						wrapper.disconnectTimer = nil
-						log.Log.Info("webrtc.main.InitializeWebRTCConnection(): connection recovered from disconnected state (session: " + handshakePayload.SessionID + ")")
+						log.WithFields(log.Fields{
+							"component":  "webrtc",
+							"event":      "connection_recovered",
+							"session_id": handshakePayload.SessionID,
+						}).Info("WebRTC connection recovered")
 					}
 					wrapper.disconnectMu.Unlock()
 
 					if wrapper.connected.CompareAndSwap(false, true) {
 						count := globalConnectionManager.IncrementPeerCount()
-						log.Log.Info("webrtc.main.InitializeWebRTCConnection(): Peer connected. Active peers: " + strconv.FormatInt(count, 10))
+						log.Info("webrtc.main.InitializeWebRTCConnection(): Peer connected. Active peers: " + strconv.FormatInt(count, 10))
 					}
 				}
 			})
@@ -600,10 +646,11 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 			peerConnection.OnICECandidate(func(candidate *pionWebRTC.ICECandidate) {
 
 				if candidate == nil {
-					log.Log.Info("webrtc.main.InitializeWebRTCConnection(): ICE gathering complete (candidate is nil)")
+					log.Info("webrtc.main.InitializeWebRTCConnection(): ICE gathering complete (candidate is nil)")
 					if !hasRelayCandidates {
-						log.Log.Error("webrtc.main.InitializeWebRTCConnection(): WARNING - No TURN (relay) candidates were gathered! TURN servers: " +
+						log.Error("webrtc.main.InitializeWebRTCConnection(): WARNING - No TURN (relay) candidates were gathered! TURN servers: " +
 							config.TURNURI + ", Username: " + config.TURNUsername + ", ForceTurn: " + config.ForceTurn)
+
 					}
 					return
 				}
@@ -632,7 +679,7 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 					hasRelayCandidates = true
 				}
 
-				log.Log.Info("webrtc.main.InitializeWebRTCConnection(): ICE candidate received - Type: " + candidateType +
+				log.Info("webrtc.main.InitializeWebRTCConnection(): ICE candidate received - Type: " + candidateType +
 					", Candidate: " + candidateStr)
 
 				//  Create a config map
@@ -641,9 +688,9 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 				if err == nil {
 					valueMap["candidate"] = string(candateBinary)
 					valueMap["session_id"] = handshakePayload.SessionID
-					log.Log.Info("webrtc.main.InitializeWebRTCConnection(): sending " + candidateType + " candidate to hub")
+					log.Info("webrtc.main.InitializeWebRTCConnection(): sending " + candidateType + " candidate to hub")
 				} else {
-					log.Log.Error("webrtc.main.InitializeWebRTCConnection(): failed to marshal candidate: " + err.Error())
+					log.Error("webrtc.main.InitializeWebRTCConnection(): failed to marshal candidate: " + err.Error())
 				}
 
 				sendCandidateSignal(configuration, mqttClient, hubKey, handshake, candateBinary)
@@ -651,14 +698,18 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 
 			offer := w.CreateOffer(sd)
 			if err = peerConnection.SetRemoteDescription(offer); err != nil {
-				log.Log.Error("webrtc.main.InitializeWebRTCConnection(): something went wrong while setting remote description: " + err.Error())
+				log.Error("webrtc.main.InitializeWebRTCConnection(): something went wrong while setting remote description: " + err.Error())
 				cleanupPeerConnection(sessionKey, wrapper)
 				return
 			}
 
 			go func() {
 				defer func() {
-					log.Log.Info("webrtc.main.InitializeWebRTCConnection(): candidate processor stopped for session: " + handshakePayload.SessionID)
+					log.WithFields(log.Fields{
+						"component":  "webrtc",
+						"event":      "candidate_processor_stopped",
+						"session_id": handshakePayload.SessionID,
+					}).Debug("WebRTC candidate processor stopped")
 				}()
 
 				// Process remote candidates only after the remote description is set.
@@ -672,14 +723,14 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 						if !ok {
 							return
 						}
-						log.Log.Info("webrtc.main.InitializeWebRTCConnection(): Received candidate from channel: " + candidate)
+						log.Info("webrtc.main.InitializeWebRTCConnection(): Received candidate from channel: " + candidate)
 						candidateInit, decodeErr := decodeICECandidate(candidate)
 						if decodeErr != nil {
-							log.Log.Error("webrtc.main.InitializeWebRTCConnection(): error decoding candidate: " + decodeErr.Error())
+							log.Error("webrtc.main.InitializeWebRTCConnection(): error decoding candidate: " + decodeErr.Error())
 							continue
 						}
 						if candidateErr := peerConnection.AddICECandidate(candidateInit); candidateErr != nil {
-							log.Log.Error("webrtc.main.InitializeWebRTCConnection(): error adding candidate: " + candidateErr.Error())
+							log.Error("webrtc.main.InitializeWebRTCConnection(): error adding candidate: " + candidateErr.Error())
 						}
 					}
 				}
@@ -687,11 +738,11 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 
 			answer, err := peerConnection.CreateAnswer(nil)
 			if err != nil {
-				log.Log.Error("webrtc.main.InitializeWebRTCConnection(): something went wrong while creating answer: " + err.Error())
+				log.Error("webrtc.main.InitializeWebRTCConnection(): something went wrong while creating answer: " + err.Error())
 				cleanupPeerConnection(sessionKey, wrapper)
 				return
 			} else if err = peerConnection.SetLocalDescription(answer); err != nil {
-				log.Log.Error("webrtc.main.InitializeWebRTCConnection(): something went wrong while setting local description: " + err.Error())
+				log.Error("webrtc.main.InitializeWebRTCConnection(): something went wrong while setting local description: " + err.Error())
 				cleanupPeerConnection(sessionKey, wrapper)
 				return
 			}
@@ -699,13 +750,13 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 			// Store peer connection in manager
 			globalConnectionManager.AddPeerConnection(sessionKey, wrapper)
 
-			log.Log.Info("webrtc.main.InitializeWebRTCConnection(): Send SDP answer")
+			log.Info("webrtc.main.InitializeWebRTCConnection(): Send SDP answer")
 
 			sendAnswerSignal(configuration, mqttClient, hubKey, handshake, answer)
 		}
 	} else {
 		globalConnectionManager.CloseCandidateChannel(sessionKey)
-		log.Log.Error("webrtc.main.InitializeWebRTCConnection(): failed to decode remote session description: " + err.Error())
+		log.Error("webrtc.main.InitializeWebRTCConnection(): failed to decode remote session description: " + err.Error())
 	}
 }
 
@@ -716,7 +767,7 @@ func NewVideoBroadcaster(streams []packets.Stream) *TrackBroadcaster {
 			return NewTrackBroadcaster(pionWebRTC.MimeTypeH264, "video", trackStreamID)
 		}
 	}
-	log.Log.Error("webrtc.main.NewVideoBroadcaster(): no H264 stream found")
+	log.Error("webrtc.main.NewVideoBroadcaster(): no H264 stream found")
 	return nil
 }
 
@@ -739,12 +790,16 @@ func NewAudioBroadcaster(streams []packets.Stream) *TrackBroadcaster {
 		}
 	}
 	if hasAAC {
-		log.Log.Info("webrtc.main.NewAudioBroadcaster(): AAC detected, creating PCMU audio track for transcoded output")
+		log.Info("webrtc.main.NewAudioBroadcaster(): AAC detected, creating PCMU audio track for transcoded output")
 		return NewTrackBroadcaster(pionWebRTC.MimeTypePCMU, "audio", trackStreamID)
 	} else if len(audioCodecNames) > 0 {
-		log.Log.Error(fmt.Sprintf("webrtc.main.NewAudioBroadcaster(): no supported audio codec found (detected: %s; supported: OPUS, PCM_MULAW, PCM_ALAW)", strings.Join(audioCodecNames, ", ")))
+		log.WithFields(log.Fields{
+			"component":       "webrtc",
+			"detected_codecs": audioCodecNames,
+			"event":           "unsupported_audio_codecs",
+		}).Error("No supported WebRTC audio codec found")
 	} else {
-		log.Log.Info("webrtc.main.NewAudioBroadcaster(): no audio stream found in camera feed")
+		log.Info("webrtc.main.NewAudioBroadcaster(): no audio stream found in camera feed")
 	}
 	return nil
 }
@@ -753,7 +808,7 @@ func NewVideoTrack(streams []packets.Stream) *pionWebRTC.TrackLocalStaticSample
 	mimeType := pionWebRTC.MimeTypeH264
 	outboundVideoTrack, err := pionWebRTC.NewTrackLocalStaticSample(pionWebRTC.RTPCodecCapability{MimeType: mimeType}, "video", trackStreamID)
 	if err != nil {
-		log.Log.Error("webrtc.main.NewVideoTrack(): error creating video track: " + err.Error())
+		log.Error("webrtc.main.NewVideoTrack(): error creating video track: " + err.Error())
 		return nil
 	}
 	return outboundVideoTrack
@@ -780,18 +835,22 @@ func NewAudioTrack(streams []packets.Stream) *pionWebRTC.TrackLocalStaticSample
 	if mimeType == "" {
 		if hasAAC {
 			mimeType = pionWebRTC.MimeTypePCMU
-			log.Log.Info("webrtc.main.NewAudioTrack(): AAC detected, creating PCMU audio track for transcoded output")
+			log.Info("webrtc.main.NewAudioTrack(): AAC detected, creating PCMU audio track for transcoded output")
 		} else if len(audioCodecNames) > 0 {
-			log.Log.Error(fmt.Sprintf("webrtc.main.NewAudioTrack(): no supported audio codec found (detected: %s; supported: OPUS, PCM_MULAW, PCM_ALAW)", strings.Join(audioCodecNames, ", ")))
+			log.WithFields(log.Fields{
+				"component":       "webrtc",
+				"detected_codecs": audioCodecNames,
+				"event":           "unsupported_audio_codecs",
+			}).Error("No supported WebRTC audio codec found")
 			return nil
 		} else {
-			log.Log.Info("webrtc.main.NewAudioTrack(): no audio stream found in camera feed")
+			log.Info("webrtc.main.NewAudioTrack(): no audio stream found in camera feed")
 			return nil
 		}
 	}
 	outboundAudioTrack, err := pionWebRTC.NewTrackLocalStaticSample(pionWebRTC.RTPCodecCapability{MimeType: mimeType}, "audio", trackStreamID)
 	if err != nil {
-		log.Log.Error("webrtc.main.NewAudioTrack(): error creating audio track: " + err.Error())
+		log.Error("webrtc.main.NewAudioTrack(): error creating audio track: " + err.Error())
 		return nil
 	}
 	return outboundAudioTrack
@@ -946,7 +1005,7 @@ func processVideoPacket(pkt packets.Packet, state *streamState, videoBroadcaster
 
 	if config.Capture.ForwardWebRTC == "true" {
 		// Remote forwarding not yet implemented
-		log.Log.Debug("webrtc.main.processVideoPacket(): remote forwarding not implemented")
+		log.Debug("webrtc.main.processVideoPacket(): remote forwarding not implemented")
 		return
 	}
 
@@ -975,25 +1034,49 @@ func processAudioPacket(pkt packets.Packet, state *streamState, audioBroadcaster
 		if transcoder == nil {
 			state.aacErrors++
 			if state.aacErrors <= 3 || state.aacErrors%100 == 0 {
-				log.Log.Warning(fmt.Sprintf("webrtc.main.processAudioPacket(): AAC packet dropped because transcoder is nil (aac_packets=%d, input_bytes=%d)", state.aacPacketsSeen, len(pkt.Data)))
+				log.WithFields(log.Fields{
+					"aac_packets": state.aacPacketsSeen,
+					"component":   "webrtc",
+					"event":       "aac_packet_dropped",
+					"input_bytes": len(pkt.Data),
+					"reason":      "transcoder_unavailable",
+				}).Warn("AAC packet dropped")
 			}
 			return // no transcoder – silently drop
 		}
 		pcmu, err := transcoder.Transcode(pkt.Data)
 		if err != nil {
 			state.aacErrors++
-			log.Log.Error("webrtc.main.processAudioPacket(): AAC transcode error: " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"aac_packets": state.aacPacketsSeen,
+				"component":   "webrtc",
+				"event":       "aac_transcode_failed",
+				"input_bytes": len(pkt.Data),
+			}).Error("Failed to transcode AAC packet")
 			return
 		}
 		if len(pcmu) == 0 {
 			state.aacNoOutput++
 			if state.aacNoOutput <= 5 || state.aacNoOutput%100 == 0 {
-				log.Log.Debug(fmt.Sprintf("webrtc.main.processAudioPacket(): AAC packet produced no PCMU output yet (aac_packets=%d, no_output=%d, input_bytes=%d)", state.aacPacketsSeen, state.aacNoOutput, len(pkt.Data)))
+				log.WithFields(log.Fields{
+					"aac_packets":     state.aacPacketsSeen,
+					"component":       "webrtc",
+					"event":           "aac_output_buffering",
+					"input_bytes":     len(pkt.Data),
+					"no_output_count": state.aacNoOutput,
+				}).Debug("AAC decoder produced no PCMU output yet")
 			}
 			return // decoder still buffering
 		}
 		if state.aacPacketsSeen <= 5 || state.aacPacketsSeen%100 == 0 {
-			log.Log.Info(fmt.Sprintf("webrtc.main.processAudioPacket(): AAC transcoded to PCMU (aac_packets=%d, input_bytes=%d, output_bytes=%d, peers=%d)", state.aacPacketsSeen, len(pkt.Data), len(pcmu), audioBroadcaster.PeerCount()))
+			log.WithFields(log.Fields{
+				"aac_packets":  state.aacPacketsSeen,
+				"component":    "webrtc",
+				"event":        "aac_packet_transcoded",
+				"input_bytes":  len(pkt.Data),
+				"output_bytes": len(pcmu),
+				"peer_count":   audioBroadcaster.PeerCount(),
+			}).Debug("AAC packet transcoded to PCMU")
 		}
 		audioData = pcmu
 	}
@@ -1004,7 +1087,15 @@ func processAudioPacket(pkt packets.Packet, state *streamState, audioBroadcaster
 		state.lastAudioSample.Duration = sampleDuration(pkt, state.lastAudioSample.PacketTimestamp, 20*time.Millisecond)
 		state.audioSamplesSent++
 		if state.audioSamplesSent <= 5 || state.audioSamplesSent%100 == 0 {
-			log.Log.Debug(fmt.Sprintf("webrtc.main.processAudioPacket(): queueing audio sample (samples=%d, codec=%s, bytes=%d, duration_ms=%d, peers=%d)", state.audioSamplesSent, pkt.Codec, len(state.lastAudioSample.Data), state.lastAudioSample.Duration.Milliseconds(), audioBroadcaster.PeerCount()))
+			log.WithFields(log.Fields{
+				"codec":          pkt.Codec,
+				"component":      "webrtc",
+				"duration_ms":    state.lastAudioSample.Duration.Milliseconds(),
+				"event":          "audio_sample_queued",
+				"peer_count":     audioBroadcaster.PeerCount(),
+				"sample_bytes":   len(state.lastAudioSample.Data),
+				"samples_queued": state.audioSamplesSent,
+			}).Debug("WebRTC audio sample queued")
 		}
 		audioBroadcaster.WriteSample(*state.lastAudioSample)
 	}
@@ -1027,7 +1118,7 @@ func WriteToTrack(livestreamCursor *packets.QueueCursor, configuration *models.C
 
 	// Check if at least one broadcaster is available
 	if videoBroadcaster == nil && audioBroadcaster == nil {
-		log.Log.Error("webrtc.main.WriteToTrack(): both video and audio broadcasters are nil, cannot proceed")
+		log.Error("webrtc.main.WriteToTrack(): both video and audio broadcasters are nil, cannot proceed")
 		return
 	}
 
@@ -1035,26 +1126,30 @@ func WriteToTrack(livestreamCursor *packets.QueueCursor, configuration *models.C
 	codecs := detectCodecs(rtspClient)
 
 	if !codecs.hasValidCodecs() {
-		log.Log.Error("webrtc.main.WriteToTrack(): no valid video or audio codec found")
+		log.Error("webrtc.main.WriteToTrack(): no valid video or audio codec found")
 		return
 	}
 
 	// Create AAC transcoder if needed (AAC → G.711 µ-law).
 	var aacTranscoder *AACTranscoder
 	if codecs.hasAAC && audioBroadcaster != nil {
-		log.Log.Info(fmt.Sprintf("webrtc.main.WriteToTrack(): AAC audio detected, creating transcoder (audio_peers=%d)", audioBroadcaster.PeerCount()))
+		log.WithFields(log.Fields{
+			"component":  "webrtc",
+			"event":      "aac_transcoder_starting",
+			"peer_count": audioBroadcaster.PeerCount(),
+		}).Info("Starting AAC transcoder")
 		t, err := NewAACTranscoder()
 		if err != nil {
-			log.Log.Error("webrtc.main.WriteToTrack(): failed to create AAC transcoder: " + err.Error())
+			log.Error("webrtc.main.WriteToTrack(): failed to create AAC transcoder: " + err.Error())
 		} else {
 			aacTranscoder = t
-			log.Log.Info("webrtc.main.WriteToTrack(): AAC transcoder created successfully")
+			log.Info("webrtc.main.WriteToTrack(): AAC transcoder created successfully")
 			defer aacTranscoder.Close()
 		}
 	}
 
 	if config.Capture.TranscodingWebRTC == "true" {
-		log.Log.Info("webrtc.main.WriteToTrack(): transcoding config enabled")
+		log.Info("webrtc.main.WriteToTrack(): transcoding config enabled")
 	}
 
 	// Initialize streaming state
@@ -1064,14 +1159,25 @@ func WriteToTrack(livestreamCursor *packets.QueueCursor, configuration *models.C
 	}
 
 	defer func() {
-		log.Log.Info(fmt.Sprintf("webrtc.main.WriteToTrack(): audio summary packets=%d aac_packets=%d sent=%d aac_no_output=%d aac_errors=%d peers=%d", state.audioPacketsSeen, state.aacPacketsSeen, state.audioSamplesSent, state.aacNoOutput, state.aacErrors, func() int {
+		peerCount := func() int {
 			if audioBroadcaster == nil {
 				return 0
 			}
 			return audioBroadcaster.PeerCount()
-		}()))
+		}()
+		log.WithFields(log.Fields{
+			"aac_errors":    state.aacErrors,
+			"aac_no_output": state.aacNoOutput,
+			"aac_packets":   state.aacPacketsSeen,
+			"audio_packets": state.audioPacketsSeen,
+			"component":     "webrtc",
+			"event":         "audio_stream_summary",
+			"peer_count":    peerCount,
+			"samples_sent":  state.audioSamplesSent,
+		}).Info("WebRTC audio stream summary")
+
 		writeFinalSamples(state, videoBroadcaster, audioBroadcaster)
-		log.Log.Info("webrtc.main.WriteToTrack(): stopped writing to track")
+		log.Info("webrtc.main.WriteToTrack(): stopped writing to track")
 	}()
 
 	var pkt packets.Packet
@@ -1105,7 +1211,7 @@ func WriteToTrack(livestreamCursor *packets.QueueCursor, configuration *models.C
 		// wait for a recent keyframe before resuming video.
 		if shouldDropPacketForLatency(pkt) {
 			if !state.catchingUp {
-				log.Log.Warning("webrtc.main.WriteToTrack(): stream is lagging behind, dropping old packets until the next recent keyframe")
+				log.Warn("webrtc.main.WriteToTrack(): stream is lagging behind, dropping old packets until the next recent keyframe")
 			}
 			state.catchingUp = true
 			state.start = false
@@ -1122,7 +1228,7 @@ func WriteToTrack(livestreamCursor *packets.QueueCursor, configuration *models.C
 			state.catchingUp = false
 			state.start = false
 			state.receivedKeyFrame = false
-			log.Log.Info("webrtc.main.WriteToTrack(): caught up with live stream at a recent keyframe")
+			log.Info("webrtc.main.WriteToTrack(): caught up with live stream at a recent keyframe")
 		}
 
 		// Wait for first keyframe before processing

machinery/src/webrtc/main_test.go

@@ -0,0 +1,38 @@
+package webrtc
+
+import "testing"
+
+func TestBuildICEServersOmitsEmptyURLs(t *testing.T) {
+	webRTC := CreateWebRTC("camera", []string{""}, []string{""}, "", "")
+
+	iceServers := buildICEServers(*webRTC)
+
+	if len(iceServers) != 0 {
+		t.Fatalf("buildICEServers() returned %d servers, want 0", len(iceServers))
+	}
+}
+
+func TestBuildICEServersIncludesConfiguredURLs(t *testing.T) {
+	webRTC := CreateWebRTC(
+		"camera",
+		[]string{"", " stun:turn-fra1.kerberos.io:3478 "},
+		[]string{" turn:turn-fra1.kerberos.io:3478 "},
+		"username",
+		"credential",
+	)
+
+	iceServers := buildICEServers(*webRTC)
+
+	if len(iceServers) != 2 {
+		t.Fatalf("buildICEServers() returned %d servers, want 2", len(iceServers))
+	}
+	if got := iceServers[0].URLs[0]; got != "stun:turn-fra1.kerberos.io:3478" {
+		t.Fatalf("STUN URL = %q, want trimmed URL", got)
+	}
+	if got := iceServers[1].URLs[0]; got != "turn:turn-fra1.kerberos.io:3478" {
+		t.Fatalf("TURN URL = %q, want trimmed URL", got)
+	}
+	if iceServers[1].Username != "username" || iceServers[1].Credential != "credential" {
+		t.Fatal("TURN credentials were not preserved")
+	}
+}

machinery/verify-moq-devcontainer.sh

@@ -17,7 +17,7 @@ binary="${TMPDIR:-/tmp}/agent-moq"
 trap 'rm -f "$binary"' EXIT
 
 echo "==> Linking the MoQ Agent"
-GOWORK=off go build -tags moq -o "$binary" ./main.go
+GOWORK=off go build -tags moq -o "$binary" .
 
 echo "==> Running the linked binary"
 "$binary" -action version

ui/public/locales/en/translation.json

@@ -223,6 +223,8 @@
       "kerberosvault_description_accesskey": "The access key of your Vault account.",
       "kerberosvault_secretkey": "Secret key",
       "kerberosvault_description_secretkey": "The secret key of your Vault account.",
+      "kerberosvault_customheaders": "Custom upload headers",
+      "kerberosvault_description_customheaders": "A JSON object of custom HTTP headers, for example {\"site_id\":\"site-1\",\"line_id\":\"line-2\"}. These values are included as metadata in Vault integration messages.",
       "kerberosvault_maxretries": "Max retries",
       "kerberosvault_description_maxretries": "The maximum number of retries to upload a recording.",
       "kerberosvault_timeout": "Timeout",