Fix legacy metadata and recording scan races

Backfill missing S3 checkpoint descriptions from validated legacy filenames while preserving checkpoint values, timestamp, and device key. Skip recordings deleted during directory scans while continuing to report other errors. Add regression coverage and document the metadata fallback.

.circleci/config.yml

@@ -1,32 +0,0 @@
-# Golang CircleCI 2.0 configuration file
-#
-# Check https://circleci.com/docs/2.0/language-go/ for more details
-version: 2
-jobs:
-  machinery:
-    docker:
-      - image: kerberos/base:0a50dc9
-    working_directory: /go/src/github.com/{{ORG_NAME}}/{{REPO_NAME}}
-    steps:
-      - checkout
-      - run: apt-get update && apt-get install -y --no-install-recommends libavcodec-dev libavformat-dev libswscale-dev libjpeg62-turbo-dev
-      - run: cd machinery && go mod download
-      - run: cd machinery && go test -v ./...
-      - run: cd machinery && go vet
-
-  ui:
-    docker:
-      - image: cimg/python:3.9.13-node
-    steps:
-      - checkout
-      - run: node --version
-      - run: cd ui && yarn
-      - run: cd ui && yarn test --passWithNoTests
-      - run: cd ui && yarn build
-
-workflows:
-  version: 2
-  build:
-    jobs:
-      - machinery
-      - ui

.devcontainer/devcontainer.json

@@ -18,7 +18,7 @@
 	},
 	"forwardPorts": [
 		3000,
-		8080
+		8082
 	],
 	"postCreateCommand": "cd ui && yarn install && yarn build && cd ../machinery && go mod download && bash ./verify-moq-devcontainer.sh"
 }

.github/workflows/go.yml

@@ -10,24 +10,27 @@ jobs:
   build:
     name: Build
     runs-on: ubuntu-latest
-    container:
-      image: kerberos/base:eb6b088
-
-    strategy:
-      matrix:
-        #No longer supported Go versions.
-        #go-version: ['1.17', '1.18', '1.19', '1.20', '1.21']
-        go-version: ["1.24"]
 
     steps:
-      - name: Set up Go ${{ matrix.go-version }}
-        uses: actions/setup-go@v2
-        with:
-          go-version: ${{ matrix.go-version }}
       - name: Check out code into the Go module directory
         uses: actions/checkout@v4
-      - name: Set up git ownershi
-        run: git config --system --add safe.directory /__w/agent/agent
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.25.x"
+          check-latest: true
+          cache: true
+          cache-dependency-path: machinery/go.sum
+      - name: Install native dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential \
+            libavcodec-dev \
+            libavutil-dev \
+            libswresample-dev \
+            libswscale-dev \
+            pkg-config
       - name: Get dependencies
         run: cd machinery && go mod download
       - name: Build

.github/workflows/nightly-build.yml

@@ -15,7 +15,7 @@ jobs:
         architecture: [amd64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
@@ -24,13 +24,14 @@ jobs:
         with:
           ref: master
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        id: buildx
+        uses: docker/setup-buildx-action@v3
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
       - name: Run Buildx
-        run: docker buildx build --platform linux/${{matrix.architecture}} -t kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7) --push .
+        run: docker buildx build --build-arg VERSION=$(echo $GITHUB_SHA | cut -c1-7) --platform linux/${{matrix.architecture}} -t kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7) --push .
       - name: Create new and append to manifest
         run: docker buildx imagetools create -t kerberos/agent-nightly:$(echo $GITHUB_SHA | cut -c1-7) kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7)
   nightly-build-other:
@@ -40,7 +41,7 @@ jobs:
         architecture: [arm64, arm/v7, arm/v6]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
@@ -49,12 +50,13 @@ jobs:
         with:
           ref: master
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        id: buildx
+        uses: docker/setup-buildx-action@v3
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
       - name: Run Buildx
-        run: docker buildx build --platform linux/${{matrix.architecture}} -t kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7) --push .
+        run: docker buildx build --build-arg VERSION=$(echo $GITHUB_SHA | cut -c1-7) --platform linux/${{matrix.architecture}} -t kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7) --push .
       - name: Create new and append to manifest
         run: docker buildx imagetools create --append -t kerberos/agent-nightly:$(echo $GITHUB_SHA | cut -c1-7) kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7)

.github/workflows/pr-build.yml

@@ -10,7 +10,7 @@ jobs:
   build:
     runs-on: ${{ matrix.runner }}
     permissions:
-      contents: write
+      contents: read
     strategy:
       matrix:
         include:
@@ -20,19 +20,10 @@ jobs:
             runner: ubuntu-24.04-arm
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      - uses: benjlevesque/short-sha@v2.1
-        id: short-sha
-        with:
-          length: 7
+        uses: actions/checkout@v4
       - name: Run Build
         run: |
-          docker build -t ${{ matrix.architecture }} .
+          docker build --build-arg VERSION=$(echo "${{ github.event.pull_request.head.sha }}" | cut -c1-7) -t ${{ matrix.architecture }} .
           CID=$(docker create ${{matrix.architecture}})
           docker cp ${CID}:/home/agent ./output-${{matrix.architecture}}
           docker rm ${CID}
@@ -43,4 +34,3 @@ jobs:
         with:
           name: agent-${{matrix.architecture}}.tar
           path: agent-${{matrix.architecture}}.tar
-

.github/workflows/react.yml

@@ -11,21 +11,22 @@ on:
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     strategy:
       matrix:
-        node-version: [14.x, 16.x]
+        node-version: [22.x]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v2
+      uses: actions/setup-node@v4
       with:
         node-version: ${{ matrix.node-version }}
+        cache: yarn
+        cache-dependency-path: ui/yarn.lock
     - name: Yarn install
-      run: cd ui && yarn
+      run: cd ui && yarn --frozen-lockfile
     - name: Yarn test
       run: cd ui && yarn test --passWithNoTests
     - name: Yarn build

.github/workflows/release-bump.yml

@@ -58,16 +58,12 @@ jobs:
         architecture: [amd64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Checkout
-        uses: actions/checkout@v3
-      - uses: benjlevesque/short-sha@v2.1
-        id: short-sha
-        with:
-          length: 7
+        uses: actions/checkout@v4
       - name: Run Build
         run: |
           docker build --provenance=false --build-arg VERSION=${{ needs.bump-release.outputs.tag }} -t ${{matrix.architecture}} .
@@ -96,16 +92,12 @@ jobs:
         architecture: [arm64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Checkout
-        uses: actions/checkout@v3
-      - uses: benjlevesque/short-sha@v2.1
-        id: short-sha
-        with:
-          length: 7
+        uses: actions/checkout@v4
       - name: Run Build
         run: |
           docker build --provenance=false --build-arg VERSION=${{ needs.bump-release.outputs.tag }} -t ${{matrix.architecture}} .
@@ -129,7 +121,7 @@ jobs:
     needs: [bump-release, build-amd64, build-arm64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/release-create.yml

@@ -22,16 +22,12 @@ jobs:
         architecture: [amd64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Checkout
-        uses: actions/checkout@v3
-      - uses: benjlevesque/short-sha@v2.1
-        id: short-sha
-        with:
-          length: 7
+        uses: actions/checkout@v4
       - name: Run Build
         run: |
           docker build --provenance=false --build-arg VERSION=${{github.event.inputs.tag || github.ref_name}} -t ${{matrix.architecture}} .
@@ -59,16 +55,12 @@ jobs:
         architecture: [arm64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Checkout
-        uses: actions/checkout@v3
-      - uses: benjlevesque/short-sha@v2.1
-        id: short-sha
-        with:
-          length: 7
+        uses: actions/checkout@v4
       - name: Run Build
         run: |
           docker build --provenance=false --build-arg VERSION=${{github.event.inputs.tag || github.ref_name}} -t ${{matrix.architecture}} .
@@ -92,7 +84,7 @@ jobs:
     needs: [build-amd64, build-arm64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}

.vscode/launch.json

@@ -14,10 +14,10 @@
                 "-action",
                 "run",
                 "-port",
-                "8080"
+                "8082"
             ],
             "envFile": "${workspaceFolder}/machinery/.env.local",
-            "buildFlags": "--tags dynamic",
+            "buildFlags": "--tags dynamic,moq",
             "env": {
                 "GOWORK": "off"
             },

Dockerfile

@@ -26,29 +26,27 @@ RUN apt-get update && apt-get install -y --fix-missing --no-install-recommends \
 	rm -rf /var/lib/apt/lists/*
 
 ##############################################################################
-# Copy all the relevant source code in the Docker image, so we can build this.
+# Copy dependency metadata first so module downloads can be cached separately.
 
 RUN mkdir -p /go/src/github.com/kerberos-io/agent
-COPY machinery /go/src/github.com/kerberos-io/agent/machinery
-RUN rm -rf /go/src/github.com/kerberos-io/agent/machinery/.env
+WORKDIR /go/src/github.com/kerberos-io/agent/machinery
+COPY machinery/go.mod machinery/go.sum ./
+RUN go mod download
 
-##################################################################
-# Get the latest commit hash, so we know which version we're running
-COPY .git /go/src/github.com/kerberos-io/agent/.git
-RUN cd /go/src/github.com/kerberos-io/agent/.git && git log --format="%H" -n 1 | head -c7 > /go/src/github.com/kerberos-io/agent/machinery/version
-RUN cat /go/src/github.com/kerberos-io/agent/machinery/version
+##############################################################################
+# Copy the rest of the source after dependencies are primed.
+
+COPY machinery ./
+RUN rm -rf .env
 
 ##################
 # Build Machinery
 
-RUN cd /go/src/github.com/kerberos-io/agent/machinery && \
-	go mod download && \
-	if [ -z "${VERSION}" ] || [ "${VERSION}" = "0.0.0" ]; then \
-		VERSION=$(cd /go/src/github.com/kerberos-io/agent && git describe --tags --always 2>/dev/null || echo "0.0.0"); \
-	fi && \
+RUN RESOLVED_VERSION="${VERSION:-0.0.0}" && \
+	printf '%s' "${RESOLVED_VERSION}" > version && \
 	BUILD_TAGS=timetzdata,netgo,osusergo && \
 	case "${TARGETARCH:-$(go env GOARCH)}" in amd64|arm64) BUILD_TAGS="moq,${BUILD_TAGS}" ;; esac && \
-	go build -tags "${BUILD_TAGS}" --ldflags "-s -w -X github.com/kerberos-io/agent/machinery/src/utils.VERSION=${VERSION}" main.go && \
+	go build -o main -tags "${BUILD_TAGS}" --ldflags "-s -w -X github.com/kerberos-io/agent/machinery/src/utils.VERSION=${RESOLVED_VERSION}" . && \
 	mkdir -p /agent && \
 	mv main /agent && \
 	mv version /agent && \
@@ -80,9 +78,11 @@ RUN apk update && apk upgrade --available && sync
 # Build Web (React app)
 
 RUN mkdir -p /go/src/github.com/kerberos-io/agent/machinery/www
-COPY ui /go/src/github.com/kerberos-io/agent/ui
-RUN cd /go/src/github.com/kerberos-io/agent/ui && rm -rf yarn.lock && yarn config set network-timeout 300000 && \
-	yarn && yarn build
+WORKDIR /go/src/github.com/kerberos-io/agent/ui
+COPY ui/package.json ui/yarn.lock ./
+RUN yarn config set network-timeout 300000 && yarn --frozen-lockfile
+COPY ui ./
+RUN yarn build
 
 ####################################
 # Let's create a /dist folder containing just the files necessary for runtime.
@@ -146,10 +146,10 @@ EXPOSE 80
 ######################################
 # Check if agent is still running
 
-HEALTHCHECK CMD curl --fail http://localhost:80 || exit 1   
+HEALTHCHECK CMD curl --fail "http://localhost:${AGENT_PORT:-80}/health" || exit 1
 
 ###################################################
 # Leeeeettttt'ssss goooooo!!!
 # Run the shizzle from the right working directory.
 WORKDIR /home/agent
-CMD ["./main", "-action", "run", "-port", "80"]
+CMD ["./main", "-action", "run"]

README-RTSPS-TLS.md

@@ -75,7 +75,7 @@ sequenceDiagram
   participant Trust as Go trust pool
   participant Camera as Camera RTSPS :9554
 
-  Agent->>Trust: Load trusted CAs from SSL_CERT_FILE and CA directories
+  Agent->>Trust: Load system roots and append AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE
   Agent->>Camera: Open TCP connection
   Agent->>Camera: Send TLS ClientHello
   Camera-->>Agent: Send TLS ServerHello and camera certificate
@@ -377,7 +377,7 @@ For a process running directly in the same environment:
 AGENT_CAPTURE_IPCAMERA_RTSP="rtsps://<user>:<password>@10.0.30.11:9554/?inst=1"
 AGENT_CAPTURE_IPCAMERA_SUB_RTSP="rtsps://<user>:<password>@10.0.30.11:9554/?inst=2"
 AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE=false
-SSL_CERT_FILE=/home/agent/data/config/uug-camera-trust-bundle.pem
+AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE=/home/agent/data/config/uug-camera-trust-bundle.pem
 ```
 
 For a container, mount the public bundle read-only at the exact path visible
@@ -387,20 +387,22 @@ includes Debian's `ca-certificates` package:
 ```bash
 docker run \
   -v /secure/config/uug-camera-trust-bundle.pem:/home/agent/data/config/uug-camera-trust-bundle.pem:ro \
-  -e SSL_CERT_FILE=/home/agent/data/config/uug-camera-trust-bundle.pem \
+  -e AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE=/home/agent/data/config/uug-camera-trust-bundle.pem \
   -e AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE=false \
   -e 'AGENT_CAPTURE_IPCAMERA_RTSP=rtsps://<user>:<password>@10.0.30.11:9554/?inst=1' \
   -e 'AGENT_CAPTURE_IPCAMERA_SUB_RTSP=rtsps://<user>:<password>@10.0.30.11:9554/?inst=2' \
   kerberos/agent:latest
 ```
 
-Restart the Agent after changing trust files. Go can cache the process system
-certificate pool after its first use, so editing a file does not guarantee that
-an already-running process reloads it.
+`AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE` starts with the operating system's roots
+and appends the camera bundle only to the gortsplib TLS configuration. Other
+clients, including MoQ, Hub, and Vault, retain the normal public CA chain.
+Restart the Agent after changing trust files.
 
-The default production mode retains the image's normal public roots in addition
-to the private camera CA. For a deliberately private-CA-only deployment, mount
-an empty directory and set `SSL_CERT_DIR` to its path:
+Do not set `SSL_CERT_FILE` or `SSL_CERT_DIR` in production solely for camera
+trust. They are process-wide and can prevent other clients from validating
+public services. For a deliberate process-wide isolation test, mount an empty
+directory and set `SSL_CERT_DIR` to its path:
 
 ```bash
 -v /secure/config/empty-ca-dir:/home/agent/data/config/empty-ca-dir:ro \
@@ -503,7 +505,8 @@ openssl s_client \
 On Unix, Go uses `SSL_CERT_FILE` instead of its default aggregate CA file, but it
 still scans default certificate directories such as `/etc/ssl/certs`. Setting
 `SSL_CERT_FILE` alone therefore does not remove CA certificates installed with
-`update-ca-certificates`.
+`update-ca-certificates`. `AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE` is appended
+after this system pool is loaded; it does not replace the system roots.
 
 Blank values do not select empty trust sources. Both `SSL_CERT_FILE=` and
 `SSL_CERT_DIR=` are treated as unset, so Go falls back to its default aggregate
@@ -515,6 +518,7 @@ directory path that contains no certificates:
 mkdir -p /tmp/empty-ca-dir
 SSL_CERT_FILE=/dev/null \
 SSL_CERT_DIR=/tmp/empty-ca-dir \
+AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE= \
 AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE=false \
 GOWORK=off \
 go run -tags moq . -action run -port 8080
@@ -523,9 +527,10 @@ go run -tags moq . -action run -port 8080
 That fresh process must fail with `x509: certificate signed by unknown
 authority`.
 
-Use exactly one trust-distribution approach when possible:
+Use exactly one camera trust-distribution approach when possible:
 
-1. Mount a private trust bundle and set `SSL_CERT_FILE`; or
+1. Mount a private trust bundle and set
+  `AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE`; or
 2. Install the CA certificates into the operating-system trust store.
 
 Using both is valid, but makes isolation tests less obvious.
@@ -550,8 +555,9 @@ openssl req \
   -keyout /tmp/unrelated-test-root.key \
   -out /tmp/unrelated-test-root.crt
 
-SSL_CERT_FILE=/tmp/unrelated-test-root.crt \
+SSL_CERT_FILE=/dev/null \
 SSL_CERT_DIR=/tmp/empty-ca-dir \
+AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE=/tmp/unrelated-test-root.crt \
 AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE=false \
 GOWORK=off \
 go run -tags moq . -action run -port 8080

README.md

@@ -3,7 +3,6 @@
 <a target="_blank" href="https://kerberos.io"><img src="https://img.shields.io/badge/kerberos-website-gray.svg?longCache=true&colorB=brightgreen" alt="Kerberos Agent"></a>
 <a target="_blank" href="https://doc.kerberos.io"><img src="https://img.shields.io/badge/kerberos-documentation-gray.svg?longCache=true&colorB=brightgreen" alt="Kerberos Agent"></a>
 
-<a target="_blank" href="https://circleci.com/gh/kerberos-io/agent"><img src="https://circleci.com/gh/kerberos-io/agent.svg?style=svg"/></a>
 <img src="https://github.com/kerberos-io/agent/workflows/Go/badge.svg"/>
 <img src="https://github.com/kerberos-io/agent/workflows/React/badge.svg"/>
 <img src="https://github.com/kerberos-io/agent/workflows/CodeQL/badge.svg"/>
@@ -77,6 +76,60 @@ There are a myriad of cameras out there (USB, IP and other cameras), and it migh
 1. [What is new?](#what-is-new)
 1. [Contributors](#contributors)
 
+### Direct S3 recording layout
+
+Direct S3 uploads store recordings as
+`<s3.username>/yyyy/mm/dd/<timestamp>-<deviceKey>.mp4`. The timestamp is the
+recording's Unix timestamp in milliseconds, and the date is its UTC date, not the upload
+date. Timestamp and device key come from the recording's `.metadata` upload
+checkpoint, which stores the timestamp in milliseconds. The uploader also
+preserves checkpoint duration, FPS (when available), and encryption status as S3
+object metadata. The device key must be non-empty and contain no path separators.
+Retried uploads use the same object key, even after midnight. Millisecond
+precision keeps recordings starting within the same second distinct.
+
+This replaces the previous `<s3.username>/<filename>.mp4` layout for new uploads.
+Existing objects are not moved; consumers of direct S3 uploads must support both
+layouts when reading historical recordings. Updated Kerberos Vault deployments
+use the same filename and dated layout under the authenticated account directory
+for both direct Vault uploads and Hub-proxied uploads.
+
+Only historical recordings with missing, empty, or FPS-only queue markers fall
+back to the legacy filename format and configured Agent key. A structured
+checkpoint takes precedence over both the filename and current configured key;
+an invalid or unreadable checkpoint is reported as an upload error, not silently
+replaced with filename-derived values.
+For older structured checkpoints, missing camera name, region, and change count
+are recovered from a validated legacy filename. Populated checkpoint fields
+take precedence, and the checkpoint timestamp and device key remain authoritative.
+
+The shorter filename cannot be decoded by legacy filename-only consumers.
+Integrations must preserve structured recording metadata, including the device
+key (`productid`) and recording timestamp (`event-timestamp`), as well as the full
+object key.
+
+### Local recording layout
+
+New local recordings are stored under
+`data/recordings/yyyy/mm/dd/<timestamp>-<deviceKey>.mp4`, using the recording's UTC
+start date and Unix-millisecond timestamp. For example:
+`data/recordings/2026/01/02/1767355200123-device-key.mp4`.
+The filename no longer embeds the camera name, region, change count, or duration,
+and it does not change when the recording is finalized.
+Those attributes, together with FPS and encryption status, are captured in the
+`.metadata` upload checkpoint and sent as structured HTTP/tus metadata.
+
+Existing flat recordings and dated recordings with older long filenames remain
+supported without a migration. Recording lists, day filters, playback, uploaders,
+and automatic cleanup handle both filename formats and layouts. Upload queue
+checkpoints remain flat in `data/cloud/`. Successful-upload removal also resolves
+both local layouts. Deploy the updated Vault alongside Agent for canonical
+`<account-directory>/yyyy/mm/dd/<timestamp>-<deviceKey>.mp4` storage keys and
+metadata-driven Hub integration events.
+Directory decryption also processes dated subdirectories and writes each output
+to a sibling `decrypted/` directory; those outputs are excluded from recording
+lists and automatic cleanup.
+
 ## Quickstart - Docker
 
 The easiest way to get your Kerberos Agent up and running is to use our public image on [Docker hub](https://hub.docker.com/r/kerberos/agent). Once you have selected a specific tag, run `docker` command below, which will open the web interface of your Kerberos agent on port `80`, and off you go. For a more configurable and persistent deployment have a look at [Running and automating a Kerberos Agent](#running-and-automating-a-kerberos-agent).
@@ -189,6 +242,103 @@ Next to attaching the configuration file, it is also possible to override the co
     -e AGENT_CAPTURE_CONTINUOUS=true \
     -d --restart=always kerberos/agent:latest
 
+Set `AGENT_PORT` to change the Agent's internal HTTP port. The environment
+variable takes precedence over `-port`. When using Docker, publish the same
+container port:
+
+```bash
+docker run -p 8082:8082 \
+  -e AGENT_PORT=8082 \
+  --name mycamera \
+  -d --restart=always kerberos/agent:latest
+```
+
+### Health checks and API responses
+
+`GET /health` is an unauthenticated liveness and diagnostics endpoint for
+container orchestrators and external monitors. `healthy` means that the Agent
+HTTP process can serve requests. Camera streams and Hub connectivity are
+reported independently in the response, so an external camera or Hub outage
+does not make Docker restart an otherwise functioning Agent process. Successful
+probes return `200 OK` and are logged at debug level to avoid periodic health
+checks filling production logs.
+
+```bash
+curl http://localhost:8082/health
+```
+
+```json
+{
+  "httpStatusCode": 200,
+  "applicationStatusCode": "get_success",
+  "entityStatusCode": "healthy",
+  "message": "Healthy",
+  "metadata": {
+    "applicationName": "agent",
+    "applicationVersion": "0.0.0",
+    "timestamp": 1788710400,
+    "path": "/health"
+  },
+  "data": {
+    "health": {
+      "description": "Agent HTTP service is healthy",
+      "cameraConnected": true,
+      "mainStream": {
+        "configured": true,
+        "connected": true,
+        "packagesProcessed": 183421,
+        "fps": 29.97,
+        "resolution": {
+          "width": 1920,
+          "height": 1080
+        },
+        "lastPacketAt": 1788710399
+      },
+      "subStream": {
+        "configured": true,
+        "connected": true,
+        "packagesProcessed": 91710,
+        "fps": 15,
+        "resolution": {
+          "width": 640,
+          "height": 360
+        },
+        "lastPacketAt": 1788710399
+      },
+      "hub": {
+        "configured": true,
+        "connected": true,
+        "lastHeartbeatAttemptAt": 1788710398,
+        "lastSuccessfulHeartbeatAt": 1788710398
+      }
+    }
+  }
+}
+```
+
+Stream `packagesProcessed` counts complete encoded video access units processed
+since the Agent process started; it does not count individual fragmented RTP
+packets. `fps` is the latest PTS-derived frame-rate estimate. Resolution is the
+most recently observed encoded width and height. `lastPacketAt`, Hub heartbeat
+timestamps, and response metadata timestamps are Unix seconds; `0` means that
+no value has been observed yet. A stream can be configured but temporarily
+disconnected.
+
+Hub `configured` means heartbeat delivery is enabled and has the required URI
+and key. Hub `connected` means the most recent heartbeat succeeded and the last
+success is no more than three minutes old. A failed heartbeat marks it
+disconnected immediately while retaining the last-success timestamp for
+diagnosis.
+
+New JSON endpoints should use this Hub-compatible response envelope rather than
+adding unrelated top-level response shapes. HTTP status describes the transport
+result, `applicationStatusCode` describes the operation, `entityStatusCode` and
+`message` describe the domain outcome, `metadata` carries request/application
+context, and endpoint-specific content belongs in a typed `data` object. Client
+responses must contain safe messages only; detailed internal errors belong in
+structured logs. Existing endpoints retain their legacy response shapes until
+they are migrated deliberately.
+
 ### Secure camera streams (RTSPS)
 
 The Agent accepts `rtsps://` camera URLs. Do not use `srtsp://`; RTSPS is RTSP over TLS. For a Bosch FLEXIDOME micro 3100i, enable **Secure RTSP** under **Network > Network Services** and use port `9554`:
@@ -200,7 +350,7 @@ AGENT_CAPTURE_IPCAMERA_SUB_RTSP='rtsps://username:password@camera.example:9554/?
 
 Certificate verification is enabled by default. The URL hostname or IP address must match the camera certificate SAN. On this Bosch firmware, RTSPS presents the certificate assigned to **HTTPS**; there is no separate SRTSP certificate usage. Leave **CBS client** assigned to the Bosch device certificate.
 
-For a private CA, mount a PEM trust bundle containing every CA certificate needed to build the camera certificate chain and set `SSL_CERT_FILE` to that file. This Bosch firmware presents only its leaf certificate, so include both the issuing intermediate and root certificates in the bundle. As a temporary fallback, `AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE=true` disables certificate verification for camera streams only.
+For a private CA, mount a PEM trust bundle containing every CA certificate needed to build the camera certificate chain and set `AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE` to its path inside the Agent. The bundle is appended to the system roots for camera RTSPS connections only. This Bosch firmware presents only its leaf certificate, so include both the issuing intermediate and root certificates in the bundle. As a temporary fallback, `AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE=true` disables certificate verification for camera streams only.
 
 See [RTSPS and TLS certificates](README-RTSPS-TLS.md) for the complete Bosch UI, private-CA, deployment, validation, and troubleshooting procedure.
 
@@ -208,6 +358,7 @@ See [RTSPS and TLS certificates](README-RTSPS-TLS.md) for the complete Bosch UI,
 | --------------------------------------- | ----------------------------------------------------------------------------------------------- | ------------------------------ |
 | `LOG_LEVEL`                                 | Level for logging, could be "info", "warning", "debug", "error" or "fatal".                     | "info"                         |
 | `LOG_OUTPUT`                                | Logging output format "json" or "text".                                                         | "text"                         |
+| `AGENT_PORT`                                | HTTP web server port. Overrides the `-port` command-line value and must be between 1 and 65535. | "80"                           |
 | `AGENT_MODE`                                | You can choose to run this in 'release' for production, and or 'demo' for showcasing.           | "release"                      |
 | `AGENT_TLS_INSECURE`                        | Specify if you want to use `InsecureSkipVerify` for the internal HTTP client.                   | "false"                        |
 | `AGENT_USERNAME`                            | The username used to authenticate against the Kerberos Agent login page.                        | "root"                         |
@@ -225,6 +376,7 @@ See [RTSPS and TLS certificates](README-RTSPS-TLS.md) for the complete Bosch UI,
 | `AGENT_REGION_POLYGON`                      | A single polygon set for motion detection: "x1,y1;x2,y2;x3,y3;...                               | ""                             |
 | `AGENT_CAPTURE_IPCAMERA_RTSP`               | Full-HD RTSP or RTSPS endpoint for the target camera.                                           | ""                             |
 | `AGENT_CAPTURE_IPCAMERA_SUB_RTSP`           | RTSP or RTSPS sub-stream endpoint used for livestreaming (WebRTC).                              | ""                             |
+| `AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE`      | PEM CA bundle appended to the system roots for RTSPS camera certificate verification.           | ""                             |
 | `AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE`     | Disable RTSPS camera certificate verification; use only when a trusted CA cannot be installed.  | "false"                        |
 | `AGENT_CAPTURE_IPCAMERA_BASE_WIDTH`         | Force a specific width resolution for live view processing.                                     | ""                             |
 | `AGENT_CAPTURE_IPCAMERA_BASE_HEIGHT`        | Force a specific height resolution for live view processing.                                    | ""                             |
@@ -243,9 +395,10 @@ See [RTSPS and TLS certificates](README-RTSPS-TLS.md) for the complete Bosch UI,
 | `AGENT_CAPTURE_PIXEL_CHANGE`                | If `CONTINUOUS` set to `false`, the number of pixel require to change before motion triggers.   | "150"                          |
 | `AGENT_CAPTURE_FRAGMENTED`                  | Set the format of the recorded MP4 to fragmented (suitable for HLS).                            | "false"                        |
 | `AGENT_CAPTURE_FRAGMENTED_DURATION`         | If `AGENT_CAPTURE_FRAGMENTED` set to `true`, define the duration (seconds) of a fragment.       | "8"                            |
-| `AGENT_MQTT_URI`                            | An MQTT broker endpoint that is used for bi-directional communication (live view, onvif, etc)   | "tcp://mqtt.kerberos.io:1883"  |
+| `AGENT_MQTT_URI`                            | MQTT broker endpoint for bi-directional communication. Accepts ActiveMQ `mqtt+ssl://` URLs.     | "tcp://mqtt.kerberos.io:1883"  |
 | `AGENT_MQTT_USERNAME`                       | Username of the MQTT broker.                                                                    | ""                             |
 | `AGENT_MQTT_PASSWORD`                       | Password of the MQTT broker.                                                                    | ""                             |
+| `AGENT_REMOTE_ACCESS_ENABLED`               | Allow encrypted Hub MQTT sessions to stream Agent logs and open an interactive shell. Enable only for trusted deployments. | "false"                        |
 | `AGENT_REALTIME_PROCESSING`                 | If `AGENT_REALTIME_PROCESSING` set to `true`, the agent will send key frames to the topic       | ""                             |
 | `AGENT_REALTIME_PROCESSING_TOPIC`           | The topic to which keyframes will be sent in base64 encoded format.                             | ""                             |
 | `AGENT_STUN_URI`                            | When using WebRTC, you'll need to provide a STUN server.                                        | "stun:turn-fra1.kerberos.io:3478"|
@@ -260,16 +413,19 @@ See [RTSPS and TLS certificates](README-RTSPS-TLS.md) for the complete Bosch UI,
 | `AGENT_HUB_PRIVATE_KEY`                     | The secret access key linked to your account in Kerberos Hub.                                   | ""                             |
 | `AGENT_HUB_REGION`                          | The Kerberos Hub region, to which you want to upload.                                           | ""                             |
 | `AGENT_HUB_SITE`                            | The site ID of a site you've created in your Kerberos Hub account.                              | ""                             |
+| `AGENT_TUS_CHUNK_SIZE_BYTES`                | Bytes sent in each resumable-upload PATCH. Set to `0` or a negative value to send all remaining bytes in one PATCH. | "8388608" (8 MiB)              |
 | `AGENT_KERBEROSVAULT_URI`                   | The Kerberos Vault API url.                                                                     | "https://vault.domain.com/api" |
 | `AGENT_KERBEROSVAULT_ACCESS_KEY`            | The access key of a Kerberos Vault account.                                                     | ""                             |
 | `AGENT_KERBEROSVAULT_SECRET_KEY`            | The secret key of a Kerberos Vault account.                                                     | ""                             |
 | `AGENT_KERBEROSVAULT_PROVIDER`              | A Kerberos Vault provider you have created (optional).                                          | ""                             |
 | `AGENT_KERBEROSVAULT_DIRECTORY`             | The directory, in the Kerberos vault, where the recordings will be stored.                      | ""                             |
+| `AGENT_KERBEROSVAULT_CUSTOM_HEADERS`        | JSON object of custom upload headers that Vault includes as integration metadata.               | `{"site_id":"site-1"}`         |
 | `AGENT_KERBEROSVAULT_SECONDARY_URI`         | The Kerberos Vault API url.                                                                     | "https://vault.domain.com/api" |
 | `AGENT_KERBEROSVAULT_SECONDARY_ACCESS_KEY`  | The access key of a secondary Kerberos Vault account.                                           | ""                             |
 | `AGENT_KERBEROSVAULT_SECONDARY_SECRET_KEY`  | The secret key of a secondary Kerberos Vault account.                                           | ""                             |
 | `AGENT_KERBEROSVAULT_SECONDARY_PROVIDER`    | A secondary Kerberos Vault provider you have created (optional).                                | ""                             |
 | `AGENT_KERBEROSVAULT_SECONDARY_DIRECTORY`   | The directory, in the secondary Kerberos vault, where the recordings will be stored.            | ""                             |
+| `AGENT_KERBEROSVAULT_SECONDARY_CUSTOM_HEADERS` | JSON object of custom upload headers for the secondary Vault.                                | `{"site_id":"site-1"}`         |
 | `AGENT_DROPBOX_ACCESS_TOKEN`                | The Access Token from your Dropbox app, that is used to leverage the Dropbox SDK.               | ""                             |
 | `AGENT_DROPBOX_DIRECTORY`                   | The directory, in Dropbox, where the recordings will be stored.                                 | ""                             |
 | `AGENT_ENCRYPTION`                          | Enable 'true' or disable 'false' end-to-end encryption for MQTT messages.                       | "false"                        |
@@ -280,6 +436,122 @@ See [RTSPS and TLS certificates](README-RTSPS-TLS.md) for the complete Bosch UI,
 | `AGENT_SIGNING`                             | Enable 'true' or disable 'false' for signing recordings.                                        | "true"                         |
 | `AGENT_SIGNING_PRIVATE_KEY`                 | The private key (RSA) to sign the recordings fingerprint to validate origin.                    | "" - uses default one if empty |
 
+Remote console access is disabled unless `AGENT_REMOTE_ACCESS_ENABLED=true`.
+The Agent also rejects remote session messages unless Hub encryption or
+end-to-end MQTT encryption is configured and used. A remote shell runs inside
+the Agent process environment as the Agent operating-system user; it is not an
+SSH server and does not expose a new network port. Keep the feature disabled on
+deployments where Hub owners should not have operating-system access.
+
+### Resumable upload chunk size
+
+Hub and Vault resumable uploads use `AGENT_TUS_CHUNK_SIZE_BYTES` as the maximum
+body size of each tus `PATCH` request. The value is a number of bytes, not a
+number of chunks. If the variable is unset or invalid, the Agent uses 8 MiB:
+
+```dotenv
+# 8 MiB (default)
+AGENT_TUS_CHUNK_SIZE_BYTES=8388608
+
+# 4 MiB: more frequent checkpoints on unstable connections
+AGENT_TUS_CHUNK_SIZE_BYTES=4194304
+
+# Disable chunking and send all remaining bytes in one PATCH
+AGENT_TUS_CHUNK_SIZE_BYTES=0
+```
+
+Smaller chunks provide more frequent resumable checkpoints but create more HTTP
+requests. Larger chunks reduce request overhead but require more data to be
+retransmitted when a request fails.
+
+### Custom metadata headers for Kerberos Vault
+
+Use `AGENT_KERBEROSVAULT_CUSTOM_HEADERS` to attach your own metadata (for
+example a site, line, or equipment identifier) to every recording uploaded to
+Kerberos Vault. The value is a JSON object whose keys are header names and whose
+values are strings. The Agent sends these as HTTP headers on every upload, and
+Vault adds them to the `metadata` object of its integration messages (Kafka,
+MQTT, SQS, RabbitMQ, etc.):
+
+```json
+{
+  "timestamp": "2026-09-24T15:06:45Z",
+  "key": "agent1/1790262405_3-425_agent1_0-0-0-0_-1_10000.mp4",
+  "device": "agent1",
+  "metadata": {
+    "site_id": "site-1",
+    "line_id": "line-2",
+    "equipment_id": "equipment-3"
+  }
+}
+```
+
+The value must be valid JSON on a single line, so wrap it in single quotes in a
+shell:
+
+```bash
+docker run -p 80:80 --name mycamera \
+  -e AGENT_CLOUD=kstorage \
+  -e AGENT_KERBEROSVAULT_URI=https://vault.domain.com/api \
+  -e AGENT_KERBEROSVAULT_ACCESS_KEY=xxx \
+  -e AGENT_KERBEROSVAULT_SECRET_KEY=xxx \
+  -e AGENT_KERBEROSVAULT_DIRECTORY=agent1 \
+  -e AGENT_KERBEROSVAULT_CUSTOM_HEADERS='{"site_id":"site-1","line_id":"line-2","equipment_id":"equipment-3"}' \
+  -d --restart=always kerberos/agent:latest
+```
+
+In `docker compose` or an `.env` file:
+
+```yaml
+services:
+  agent:
+    image: kerberos/agent:latest
+    environment:
+      AGENT_CLOUD: kstorage
+      AGENT_KERBEROSVAULT_CUSTOM_HEADERS: '{"site_id":"site-1","line_id":"line-2","equipment_id":"equipment-3"}'
+```
+
+```dotenv
+AGENT_KERBEROSVAULT_CUSTOM_HEADERS={"site_id":"site-1","line_id":"line-2","equipment_id":"equipment-3"}
+```
+
+In Kubernetes, quote the value so YAML keeps it as a string:
+
+```yaml
+env:
+  - name: AGENT_KERBEROSVAULT_CUSTOM_HEADERS
+    value: '{"site_id":"site-1","line_id":"line-2","equipment_id":"equipment-3"}'
+```
+
+Use `AGENT_KERBEROSVAULT_SECONDARY_CUSTOM_HEADERS` for the secondary Vault. In
+ConfigMap deployments you can set shared defaults with the `GLOBAL_` prefix
+(for example `GLOBAL_AGENT_KERBEROSVAULT_CUSTOM_HEADERS`) and override them per
+agent with `AGENT_KERBEROSVAULT_CUSTOM_HEADERS`. You can also set the same JSON
+under **Settings → Persistence → Custom upload headers** in the Agent UI, which
+stores it as `kstorage.custom_headers` in `config.json`.
+
+The rules for this value are:
+
+- It must be a JSON object with string values. Numbers, `null`, arrays, and
+  nested objects are rejected.
+- It can contain up to 32 headers, with names up to 128 characters and values
+  up to 4096 bytes.
+- Header names must be valid HTTP tokens (letters, digits, and ``!#$%&'*+-.^_`|~``).
+  Keys are matched case-insensitively, so `site_id` and `SITE_ID` cannot both
+  be used.
+- Reserved names are rejected: anything starting with `X-Kerberos-`,
+  `Authorization`, `Cookie`, `Host`, `Content-Type`, `Content-Length`,
+  `Trace-Id`, tus protocol headers (`Tus-Resumable`, `Upload-*`), and
+  hop-by-hop headers.
+- Values cannot contain line breaks, tabs, or other control characters.
+
+If the value is invalid, the Agent logs an error and does not upload to that
+Vault until the configuration is fixed. The recording stays queued, so it will
+upload once the value is corrected. Header names reach Vault unchanged, but
+proxies may normalise their casing. If Vault redirects an upload to another
+host, the Agent removes the custom headers and Vault credentials before
+following the redirect.
+
 
 ## Encryption
 
@@ -384,7 +656,10 @@ The `web` is a **React** project which is the main entry point for an end user t
     cd ui
     yarn start
 
-This will start a webserver and launches the web app on port `3000`.
+This starts the web app on port `3000`. In development, the UI connects to the
+Agent API on port `8082`. Override that port when needed:
+
+    REACT_APP_AGENT_API_PORT=9090 yarn start
 
 ![login-agent](./assets/img/agent-login.gif)
 
@@ -400,9 +675,14 @@ You can simply run the `machinery` using following commands.
 
     git clone https://github.com/kerberos-io/agent
     cd machinery
-    go run main.go -action run -port 80
+    go run main.go -action run -port 8082
 
-This will launch the Kerberos Agent and run a webserver on port `80`. You can change the port by your own preference. We strongly support the usage of [Goland](https://www.jetbrains.com/go/) or [Visual Studio Code](https://code.visualstudio.com/), as it comes with all the debugging and linting features built in.
+This will launch the Kerberos Agent and run a webserver on port `8082`, matching
+the UI development default. You can change the port together with
+`REACT_APP_AGENT_API_PORT`. We strongly support the usage of
+[Goland](https://www.jetbrains.com/go/) or
+[Visual Studio Code](https://code.visualstudio.com/), as it comes with all the
+debugging and linting features built in.
 
 ![VSCode desktop](./assets/img/vscode-desktop.png)
 
@@ -454,11 +734,26 @@ its own broadcast and switching quality is simply a resubscribe:
 | `high` | `devices/<agent-key>/live.hang`       | highest-resolution camera stream           |
 | `low`  | `devices/<agent-key>/live-low.hang`   | sub stream (main stream when none is set)  |
 
-Each tier only uploads while it has at least one subscriber, so the tier nobody
-watches costs virtually no bandwidth. `AGENT_LIVE_MOQ_QUALITY` accepts `high` or
-`low` to pin the Agent to a single tier; viewers requesting the other tier then
-find no broadcast. Any other value (including the default) publishes both. The
-initial implementation publishes H.264 video only.
+Each tier uploads every frame while it has subscribers. While idle, it continues
+to upload fresh keyframes only. This keeps the relay's latest cached GOP current,
+so a new viewer does not initially render the final frame from the previous
+viewer session, while avoiding the bandwidth cost of continuously sending every
+delta frame. `AGENT_LIVE_MOQ_QUALITY` accepts `high` or `low` to pin the Agent to
+a single tier; viewers requesting the other tier then find no broadcast. Any
+other value (including the default) publishes both. The initial implementation
+publishes H.264 video only.
+
+Two bounded duration settings tune recovery for unusual network conditions:
+
+| Variable | Default | Allowed range |
+| -------- | ------- | ------------- |
+| `AGENT_LIVE_MOQ_MAX_PACKET_AGE` | `1.5s` | `250ms` to `30s` |
+| `AGENT_LIVE_MOQ_WRITE_TIMEOUT` | `5s` | `1s` to `1m` |
+
+Values use Go duration syntax and are clamped to the documented range. The
+dashboard API's `recovery` object reports per-tier reconnects, last successful
+frame/write timing, native write timeouts, watchdog restarts/cooldown, dropped
+run-channel events, and run-worker shutdown timeouts.
 
 The `/anon` relay route is intended for interoperability testing. Production
 deployments must set `AGENT_LIVE_MOQ_URL` to a short-lived, device-scoped
@@ -471,8 +766,9 @@ monorepo devcontainer so it uses the Trixie base, then run the VS Code task
     cd machinery
     bash ./verify-moq-devcontainer.sh
 
-The check requires glibc 2.38 or newer, runs the tagged package tests, links the
-complete Agent with `-tags moq`, and executes the resulting binary's version
+The check requires glibc 2.38 or newer, runs the tagged package and in-process
+native QUIC lifecycle tests, links the complete Agent with `-tags moq`, and
+executes the resulting binary's version
 command. Both devcontainers also run this check during their post-create setup.
 
 ## What is new?

machinery/README.md

@@ -5,7 +5,6 @@
 <a target="_blank" href="https://twitter.com/kerberosio?ref_src=twsrc%5Etfw"><img src="https://img.shields.io/twitter/url.svg?label=Follow%20%40kerberosio&style=social&url=https%3A%2F%2Ftwitter.com%2Fkerberosio" alt="Twitter Widget"></a>
 <a target="_blank" href="https://join.slack.com/t/kerberosio/shared_invite/zt-1a5oj4pwm-O4qCAN9c5r2um0Ns0ge8ww"><img src="https://img.shields.io/badge/slack-@kerberosio-yellow.svg?logo=slack " alt="Kerberos.io"></a>
 
-<a target="_blank" href="https://circleci.com/gh/kerberos-io/agent"><img src="https://circleci.com/gh/kerberos-io/agent.svg?style=svg"/></a>
 <img src="https://github.com/kerberos-io/agent/workflows/Go/badge.svg"/>
 <img src="https://github.com/kerberos-io/agent/workflows/CodeQL/badge.svg"/>
 

machinery/data/tus/1786715908_3-737_encrypted_0-0-0-0_-1_25498.mp4.primary.json

@@ -0,0 +1 @@
+{"upload_url":"https://vault.kerberos.io/api/storage/tus/19e42fbc666a38064904caf8c46d182a","vault_uri":"https://vault.kerberos.io/api/storage/tus/","size":1591581}

machinery/data/tus/1788718145_3-694_encrypted_0-0-0-0_-1_25497.mp4.primary.json

@@ -0,0 +1 @@
+{"upload_url":"https://vault.kerberos.io/api/storage/tus/bd3be0f9ecd3873d381f57cb8b4a7e96","vault_uri":"https://vault.kerberos.io/api/storage/tus/","size":2560480}

machinery/docs/docs.go

@@ -24,6 +24,38 @@ const docTemplate = `{
     "host": "{{.Host}}",
     "basePath": "{{.BasePath}}",
     "paths": {
+        "/api/camera/discover": {
+            "get": {
+                "description": "Runs an advanced Fing/WiFiman-style scan (ONVIF WS-Discovery + TCP port scan + MAC/vendor lookup) and returns the devices found on the local network.",
+                "tags": [
+                    "onvif"
+                ],
+                "summary": "Discover cameras and other devices on the local network.",
+                "operationId": "camera-discover",
+                "parameters": [
+                    {
+                        "type": "integer",
+                        "description": "Discovery timeout in milliseconds (default 2000)",
+                        "name": "timeout",
+                        "in": "query"
+                    },
+                    {
+                        "type": "string",
+                        "description": "Optional subnet(s) to scan, e.g. '192.168.1.0/24' (comma-separated). Defaults to the local interfaces.",
+                        "name": "subnet",
+                        "in": "query"
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/models.APIResponse"
+                        }
+                    }
+                }
+            }
+        },
         "/api/camera/onvif/capabilities": {
             "post": {
                 "description": "Will return the ONVIF capabilities for the specific camera.",
@@ -623,6 +655,40 @@ const docTemplate = `{
                 }
             }
         },
+        "/api/persistence/secondary/verify": {
+            "post": {
+                "security": [
+                    {
+                        "Bearer": []
+                    }
+                ],
+                "description": "Will verify the secondary persistence.",
+                "tags": [
+                    "persistence"
+                ],
+                "summary": "Will verify the secondary persistence.",
+                "operationId": "verify-secondary-persistence",
+                "parameters": [
+                    {
+                        "description": "Config",
+                        "name": "config",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/models.Config"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/models.APIResponse"
+                        }
+                    }
+                }
+            }
+        },
         "/api/persistence/verify": {
             "post": {
                 "security": [
@@ -656,9 +722,151 @@ const docTemplate = `{
                     }
                 }
             }
+        },
+        "/health": {
+            "get": {
+                "description": "Confirms that the Agent HTTP process can serve requests and reports current camera stream and Hub heartbeat diagnostics. Operational dependency failures do not change the liveness HTTP status.",
+                "produces": [
+                    "application/json"
+                ],
+                "tags": [
+                    "health"
+                ],
+                "summary": "Check Agent health",
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/http.HealthResponse"
+                        }
+                    }
+                }
+            }
         }
     },
     "definitions": {
+        "http.Health": {
+            "type": "object",
+            "properties": {
+                "cameraConnected": {
+                    "type": "boolean"
+                },
+                "description": {
+                    "type": "string"
+                },
+                "hub": {
+                    "$ref": "#/definitions/http.HubHealth"
+                },
+                "mainStream": {
+                    "$ref": "#/definitions/http.StreamHealth"
+                },
+                "subStream": {
+                    "$ref": "#/definitions/http.StreamHealth"
+                }
+            }
+        },
+        "http.HealthResponse": {
+            "type": "object",
+            "properties": {
+                "applicationStatusCode": {
+                    "type": "string"
+                },
+                "data": {
+                    "$ref": "#/definitions/http.HealthResponseData"
+                },
+                "entityStatusCode": {
+                    "type": "string"
+                },
+                "httpStatusCode": {
+                    "type": "integer"
+                },
+                "message": {
+                    "type": "string"
+                },
+                "metadata": {
+                    "$ref": "#/definitions/http.ResponseMetadata"
+                }
+            }
+        },
+        "http.HealthResponseData": {
+            "type": "object",
+            "properties": {
+                "health": {
+                    "$ref": "#/definitions/http.Health"
+                }
+            }
+        },
+        "http.HubHealth": {
+            "type": "object",
+            "properties": {
+                "configured": {
+                    "type": "boolean"
+                },
+                "connected": {
+                    "type": "boolean"
+                },
+                "lastHeartbeatAttemptAt": {
+                    "type": "integer"
+                },
+                "lastSuccessfulHeartbeatAt": {
+                    "type": "integer"
+                }
+            }
+        },
+        "http.ResponseMetadata": {
+            "type": "object",
+            "properties": {
+                "applicationName": {
+                    "type": "string"
+                },
+                "applicationVersion": {
+                    "type": "string"
+                },
+                "path": {
+                    "type": "string"
+                },
+                "timestamp": {
+                    "type": "integer"
+                },
+                "traceId": {
+                    "type": "string"
+                }
+            }
+        },
+        "http.StreamHealth": {
+            "type": "object",
+            "properties": {
+                "configured": {
+                    "type": "boolean"
+                },
+                "connected": {
+                    "type": "boolean"
+                },
+                "fps": {
+                    "type": "number"
+                },
+                "lastPacketAt": {
+                    "type": "integer"
+                },
+                "packagesProcessed": {
+                    "type": "integer"
+                },
+                "resolution": {
+                    "$ref": "#/definitions/http.StreamResolution"
+                }
+            }
+        },
+        "http.StreamResolution": {
+            "type": "object",
+            "properties": {
+                "height": {
+                    "type": "integer"
+                },
+                "width": {
+                    "type": "integer"
+                }
+            }
+        },
         "models.APIResponse": {
             "type": "object",
             "properties": {
@@ -730,12 +938,19 @@ const docTemplate = `{
                 "fragmentedduration": {
                     "type": "integer"
                 },
+                "gopsize": {
+                    "description": "GOP size in seconds, used for pre-recording",
+                    "type": "integer"
+                },
                 "ipcamera": {
                     "$ref": "#/definitions/models.IPCamera"
                 },
                 "liveview": {
                     "type": "string"
                 },
+                "liveview_chunking": {
+                    "type": "string"
+                },
                 "maxlengthrecording": {
                     "type": "integer"
                 },
@@ -745,6 +960,10 @@ const docTemplate = `{
                 "name": {
                     "type": "string"
                 },
+                "onvif_motion": {
+                    "description": "ONVIFMotion routes the camera's ONVIF motion events into the\nagent's motion-triggered recording pipeline. When \"true\" the\nagent opens an event/stream against the configured ONVIF\nendpoint and forwards Motion+Active events to HandleMotion.\nRequires Capture.IPCamera.ONVIFXAddr / ONVIFUsername /\nONVIFPassword to be set. Default empty (disabled) keeps the\nexisting pixel-diff motion detection as the only source.",
+                    "type": "string"
+                },
                 "pixelChangeThreshold": {
                     "type": "integer"
                 },
@@ -823,9 +1042,15 @@ const docTemplate = `{
                 "kstorage": {
                     "$ref": "#/definitions/models.KStorage"
                 },
+                "kstorage_secondary": {
+                    "$ref": "#/definitions/models.KStorage"
+                },
                 "max_directory_size": {
                     "type": "integer"
                 },
+                "min_free_space": {
+                    "type": "integer"
+                },
                 "mqtt_password": {
                     "type": "string"
                 },
@@ -856,6 +1081,9 @@ const docTemplate = `{
                 "s3": {
                     "$ref": "#/definitions/models.S3"
                 },
+                "signing": {
+                    "$ref": "#/definitions/models.Signing"
+                },
                 "stunuri": {
                     "type": "string"
                 },
@@ -947,6 +1175,15 @@ const docTemplate = `{
         "models.IPCamera": {
             "type": "object",
             "properties": {
+                "base_height": {
+                    "type": "integer"
+                },
+                "base_width": {
+                    "type": "integer"
+                },
+                "channels": {
+                    "type": "integer"
+                },
                 "fps": {
                     "type": "string"
                 },
@@ -965,21 +1202,85 @@ const docTemplate = `{
                 "onvif_xaddr": {
                     "type": "string"
                 },
+                "pps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "rtsp": {
                     "type": "string"
                 },
+                "sample_rate": {
+                    "type": "integer"
+                },
+                "sps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_fps": {
                     "type": "string"
                 },
                 "sub_height": {
                     "type": "integer"
                 },
+                "sub_pps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_rtsp": {
                     "type": "string"
                 },
+                "sub_sps_nalus": {
+                    "description": "Sub stream parameter sets, captured separately from the main stream so the\nlive HLS muxer can build a correct init segment when a viewer switches the\nlive view to the sub (low-resolution) stream.",
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
+                "sub_vps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_width": {
                     "type": "integer"
                 },
+                "vps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "width": {
                     "type": "integer"
                 }
@@ -995,15 +1296,24 @@ const docTemplate = `{
                     "description": "old way, remove this",
                     "type": "string"
                 },
+                "custom_headers": {
+                    "type": "string"
+                },
                 "directory": {
                     "type": "string"
                 },
+                "max_retries": {
+                    "type": "integer"
+                },
                 "provider": {
                     "type": "string"
                 },
                 "secret_access_key": {
                     "type": "string"
                 },
+                "timeout": {
+                    "type": "integer"
+                },
                 "uri": {
                     "type": "string"
                 }
@@ -1141,6 +1451,17 @@ const docTemplate = `{
                 }
             }
         },
+        "models.Signing": {
+            "type": "object",
+            "properties": {
+                "enabled": {
+                    "type": "string"
+                },
+                "private_key": {
+                    "type": "string"
+                }
+            }
+        },
         "models.Timetable": {
             "type": "object",
             "properties": {

machinery/docs/swagger.json

@@ -17,6 +17,38 @@
     },
     "basePath": "/",
     "paths": {
+        "/api/camera/discover": {
+            "get": {
+                "description": "Runs an advanced Fing/WiFiman-style scan (ONVIF WS-Discovery + TCP port scan + MAC/vendor lookup) and returns the devices found on the local network.",
+                "tags": [
+                    "onvif"
+                ],
+                "summary": "Discover cameras and other devices on the local network.",
+                "operationId": "camera-discover",
+                "parameters": [
+                    {
+                        "type": "integer",
+                        "description": "Discovery timeout in milliseconds (default 2000)",
+                        "name": "timeout",
+                        "in": "query"
+                    },
+                    {
+                        "type": "string",
+                        "description": "Optional subnet(s) to scan, e.g. '192.168.1.0/24' (comma-separated). Defaults to the local interfaces.",
+                        "name": "subnet",
+                        "in": "query"
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/models.APIResponse"
+                        }
+                    }
+                }
+            }
+        },
         "/api/camera/onvif/capabilities": {
             "post": {
                 "description": "Will return the ONVIF capabilities for the specific camera.",
@@ -616,6 +648,40 @@
                 }
             }
         },
+        "/api/persistence/secondary/verify": {
+            "post": {
+                "security": [
+                    {
+                        "Bearer": []
+                    }
+                ],
+                "description": "Will verify the secondary persistence.",
+                "tags": [
+                    "persistence"
+                ],
+                "summary": "Will verify the secondary persistence.",
+                "operationId": "verify-secondary-persistence",
+                "parameters": [
+                    {
+                        "description": "Config",
+                        "name": "config",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/models.Config"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/models.APIResponse"
+                        }
+                    }
+                }
+            }
+        },
         "/api/persistence/verify": {
             "post": {
                 "security": [
@@ -649,9 +715,151 @@
                     }
                 }
             }
+        },
+        "/health": {
+            "get": {
+                "description": "Confirms that the Agent HTTP process can serve requests and reports current camera stream and Hub heartbeat diagnostics. Operational dependency failures do not change the liveness HTTP status.",
+                "produces": [
+                    "application/json"
+                ],
+                "tags": [
+                    "health"
+                ],
+                "summary": "Check Agent health",
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/http.HealthResponse"
+                        }
+                    }
+                }
+            }
         }
     },
     "definitions": {
+        "http.Health": {
+            "type": "object",
+            "properties": {
+                "cameraConnected": {
+                    "type": "boolean"
+                },
+                "description": {
+                    "type": "string"
+                },
+                "hub": {
+                    "$ref": "#/definitions/http.HubHealth"
+                },
+                "mainStream": {
+                    "$ref": "#/definitions/http.StreamHealth"
+                },
+                "subStream": {
+                    "$ref": "#/definitions/http.StreamHealth"
+                }
+            }
+        },
+        "http.HealthResponse": {
+            "type": "object",
+            "properties": {
+                "applicationStatusCode": {
+                    "type": "string"
+                },
+                "data": {
+                    "$ref": "#/definitions/http.HealthResponseData"
+                },
+                "entityStatusCode": {
+                    "type": "string"
+                },
+                "httpStatusCode": {
+                    "type": "integer"
+                },
+                "message": {
+                    "type": "string"
+                },
+                "metadata": {
+                    "$ref": "#/definitions/http.ResponseMetadata"
+                }
+            }
+        },
+        "http.HealthResponseData": {
+            "type": "object",
+            "properties": {
+                "health": {
+                    "$ref": "#/definitions/http.Health"
+                }
+            }
+        },
+        "http.HubHealth": {
+            "type": "object",
+            "properties": {
+                "configured": {
+                    "type": "boolean"
+                },
+                "connected": {
+                    "type": "boolean"
+                },
+                "lastHeartbeatAttemptAt": {
+                    "type": "integer"
+                },
+                "lastSuccessfulHeartbeatAt": {
+                    "type": "integer"
+                }
+            }
+        },
+        "http.ResponseMetadata": {
+            "type": "object",
+            "properties": {
+                "applicationName": {
+                    "type": "string"
+                },
+                "applicationVersion": {
+                    "type": "string"
+                },
+                "path": {
+                    "type": "string"
+                },
+                "timestamp": {
+                    "type": "integer"
+                },
+                "traceId": {
+                    "type": "string"
+                }
+            }
+        },
+        "http.StreamHealth": {
+            "type": "object",
+            "properties": {
+                "configured": {
+                    "type": "boolean"
+                },
+                "connected": {
+                    "type": "boolean"
+                },
+                "fps": {
+                    "type": "number"
+                },
+                "lastPacketAt": {
+                    "type": "integer"
+                },
+                "packagesProcessed": {
+                    "type": "integer"
+                },
+                "resolution": {
+                    "$ref": "#/definitions/http.StreamResolution"
+                }
+            }
+        },
+        "http.StreamResolution": {
+            "type": "object",
+            "properties": {
+                "height": {
+                    "type": "integer"
+                },
+                "width": {
+                    "type": "integer"
+                }
+            }
+        },
         "models.APIResponse": {
             "type": "object",
             "properties": {
@@ -723,12 +931,19 @@
                 "fragmentedduration": {
                     "type": "integer"
                 },
+                "gopsize": {
+                    "description": "GOP size in seconds, used for pre-recording",
+                    "type": "integer"
+                },
                 "ipcamera": {
                     "$ref": "#/definitions/models.IPCamera"
                 },
                 "liveview": {
                     "type": "string"
                 },
+                "liveview_chunking": {
+                    "type": "string"
+                },
                 "maxlengthrecording": {
                     "type": "integer"
                 },
@@ -738,6 +953,10 @@
                 "name": {
                     "type": "string"
                 },
+                "onvif_motion": {
+                    "description": "ONVIFMotion routes the camera's ONVIF motion events into the\nagent's motion-triggered recording pipeline. When \"true\" the\nagent opens an event/stream against the configured ONVIF\nendpoint and forwards Motion+Active events to HandleMotion.\nRequires Capture.IPCamera.ONVIFXAddr / ONVIFUsername /\nONVIFPassword to be set. Default empty (disabled) keeps the\nexisting pixel-diff motion detection as the only source.",
+                    "type": "string"
+                },
                 "pixelChangeThreshold": {
                     "type": "integer"
                 },
@@ -816,9 +1035,15 @@
                 "kstorage": {
                     "$ref": "#/definitions/models.KStorage"
                 },
+                "kstorage_secondary": {
+                    "$ref": "#/definitions/models.KStorage"
+                },
                 "max_directory_size": {
                     "type": "integer"
                 },
+                "min_free_space": {
+                    "type": "integer"
+                },
                 "mqtt_password": {
                     "type": "string"
                 },
@@ -849,6 +1074,9 @@
                 "s3": {
                     "$ref": "#/definitions/models.S3"
                 },
+                "signing": {
+                    "$ref": "#/definitions/models.Signing"
+                },
                 "stunuri": {
                     "type": "string"
                 },
@@ -940,6 +1168,15 @@
         "models.IPCamera": {
             "type": "object",
             "properties": {
+                "base_height": {
+                    "type": "integer"
+                },
+                "base_width": {
+                    "type": "integer"
+                },
+                "channels": {
+                    "type": "integer"
+                },
                 "fps": {
                     "type": "string"
                 },
@@ -958,21 +1195,85 @@
                 "onvif_xaddr": {
                     "type": "string"
                 },
+                "pps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "rtsp": {
                     "type": "string"
                 },
+                "sample_rate": {
+                    "type": "integer"
+                },
+                "sps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_fps": {
                     "type": "string"
                 },
                 "sub_height": {
                     "type": "integer"
                 },
+                "sub_pps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_rtsp": {
                     "type": "string"
                 },
+                "sub_sps_nalus": {
+                    "description": "Sub stream parameter sets, captured separately from the main stream so the\nlive HLS muxer can build a correct init segment when a viewer switches the\nlive view to the sub (low-resolution) stream.",
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
+                "sub_vps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_width": {
                     "type": "integer"
                 },
+                "vps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "width": {
                     "type": "integer"
                 }
@@ -988,15 +1289,24 @@
                     "description": "old way, remove this",
                     "type": "string"
                 },
+                "custom_headers": {
+                    "type": "string"
+                },
                 "directory": {
                     "type": "string"
                 },
+                "max_retries": {
+                    "type": "integer"
+                },
                 "provider": {
                     "type": "string"
                 },
                 "secret_access_key": {
                     "type": "string"
                 },
+                "timeout": {
+                    "type": "integer"
+                },
                 "uri": {
                     "type": "string"
                 }
@@ -1134,6 +1444,17 @@
                 }
             }
         },
+        "models.Signing": {
+            "type": "object",
+            "properties": {
+                "enabled": {
+                    "type": "string"
+                },
+                "private_key": {
+                    "type": "string"
+                }
+            }
+        },
         "models.Timetable": {
             "type": "object",
             "properties": {

machinery/docs/swagger.yaml

@@ -1,5 +1,84 @@
 basePath: /
 definitions:
+  http.Health:
+    properties:
+      cameraConnected:
+        type: boolean
+      description:
+        type: string
+      hub:
+        $ref: '#/definitions/http.HubHealth'
+      mainStream:
+        $ref: '#/definitions/http.StreamHealth'
+      subStream:
+        $ref: '#/definitions/http.StreamHealth'
+    type: object
+  http.HealthResponse:
+    properties:
+      applicationStatusCode:
+        type: string
+      data:
+        $ref: '#/definitions/http.HealthResponseData'
+      entityStatusCode:
+        type: string
+      httpStatusCode:
+        type: integer
+      message:
+        type: string
+      metadata:
+        $ref: '#/definitions/http.ResponseMetadata'
+    type: object
+  http.HealthResponseData:
+    properties:
+      health:
+        $ref: '#/definitions/http.Health'
+    type: object
+  http.HubHealth:
+    properties:
+      configured:
+        type: boolean
+      connected:
+        type: boolean
+      lastHeartbeatAttemptAt:
+        type: integer
+      lastSuccessfulHeartbeatAt:
+        type: integer
+    type: object
+  http.ResponseMetadata:
+    properties:
+      applicationName:
+        type: string
+      applicationVersion:
+        type: string
+      path:
+        type: string
+      timestamp:
+        type: integer
+      traceId:
+        type: string
+    type: object
+  http.StreamHealth:
+    properties:
+      configured:
+        type: boolean
+      connected:
+        type: boolean
+      fps:
+        type: number
+      lastPacketAt:
+        type: integer
+      packagesProcessed:
+        type: integer
+      resolution:
+        $ref: '#/definitions/http.StreamResolution'
+    type: object
+  http.StreamResolution:
+    properties:
+      height:
+        type: integer
+      width:
+        type: integer
+    type: object
   models.APIResponse:
     properties:
       can_pan_tilt:
@@ -47,16 +126,31 @@ definitions:
         type: string
       fragmentedduration:
         type: integer
+      gopsize:
+        description: GOP size in seconds, used for pre-recording
+        type: integer
       ipcamera:
         $ref: '#/definitions/models.IPCamera'
       liveview:
         type: string
+      liveview_chunking:
+        type: string
       maxlengthrecording:
         type: integer
       motion:
         type: string
       name:
         type: string
+      onvif_motion:
+        description: |-
+          ONVIFMotion routes the camera's ONVIF motion events into the
+          agent's motion-triggered recording pipeline. When "true" the
+          agent opens an event/stream against the configured ONVIF
+          endpoint and forwards Motion+Active events to HandleMotion.
+          Requires Capture.IPCamera.ONVIFXAddr / ONVIFUsername /
+          ONVIFPassword to be set. Default empty (disabled) keeps the
+          existing pixel-diff motion detection as the only source.
+        type: string
       pixelChangeThreshold:
         type: integer
       postrecording:
@@ -109,8 +203,12 @@ definitions:
         type: string
       kstorage:
         $ref: '#/definitions/models.KStorage'
+      kstorage_secondary:
+        $ref: '#/definitions/models.KStorage'
       max_directory_size:
         type: integer
+      min_free_space:
+        type: integer
       mqtt_password:
         type: string
       mqtt_username:
@@ -131,6 +229,8 @@ definitions:
         type: string
       s3:
         $ref: '#/definitions/models.S3'
+      signing:
+        $ref: '#/definitions/models.Signing'
       stunuri:
         type: string
       time:
@@ -190,6 +290,12 @@ definitions:
     type: object
   models.IPCamera:
     properties:
+      base_height:
+        type: integer
+      base_width:
+        type: integer
+      channels:
+        type: integer
       fps:
         type: string
       height:
@@ -202,16 +308,64 @@ definitions:
         type: string
       onvif_xaddr:
         type: string
+      pps_nalus:
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
       rtsp:
         type: string
+      sample_rate:
+        type: integer
+      sps_nalus:
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
       sub_fps:
         type: string
       sub_height:
         type: integer
+      sub_pps_nalus:
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
       sub_rtsp:
         type: string
+      sub_sps_nalus:
+        description: |-
+          Sub stream parameter sets, captured separately from the main stream so the
+          live HLS muxer can build a correct init segment when a viewer switches the
+          live view to the sub (low-resolution) stream.
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
+      sub_vps_nalus:
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
       sub_width:
         type: integer
+      vps_nalus:
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
       width:
         type: integer
     type: object
@@ -224,10 +378,14 @@ definitions:
         type: string
       directory:
         type: string
+      max_retries:
+        type: integer
       provider:
         type: string
       secret_access_key:
         type: string
+      timeout:
+        type: integer
       uri:
         type: string
     type: object
@@ -316,6 +474,13 @@ definitions:
       username:
         type: string
     type: object
+  models.Signing:
+    properties:
+      enabled:
+        type: string
+      private_key:
+        type: string
+    type: object
   models.Timetable:
     properties:
       end1:
@@ -345,6 +510,30 @@ info:
   title: Swagger Kerberos Agent API
   version: "1.0"
 paths:
+  /api/camera/discover:
+    get:
+      description: Runs an advanced Fing/WiFiman-style scan (ONVIF WS-Discovery +
+        TCP port scan + MAC/vendor lookup) and returns the devices found on the local
+        network.
+      operationId: camera-discover
+      parameters:
+      - description: Discovery timeout in milliseconds (default 2000)
+        in: query
+        name: timeout
+        type: integer
+      - description: Optional subnet(s) to scan, e.g. '192.168.1.0/24' (comma-separated).
+          Defaults to the local interfaces.
+        in: query
+        name: subnet
+        type: string
+      responses:
+        "200":
+          description: OK
+          schema:
+            $ref: '#/definitions/models.APIResponse'
+      summary: Discover cameras and other devices on the local network.
+      tags:
+      - onvif
   /api/camera/onvif/capabilities:
     post:
       description: Will return the ONVIF capabilities for the specific camera.
@@ -735,6 +924,27 @@ paths:
       summary: Get Authorization token.
       tags:
       - authentication
+  /api/persistence/secondary/verify:
+    post:
+      description: Will verify the secondary persistence.
+      operationId: verify-secondary-persistence
+      parameters:
+      - description: Config
+        in: body
+        name: config
+        required: true
+        schema:
+          $ref: '#/definitions/models.Config'
+      responses:
+        "200":
+          description: OK
+          schema:
+            $ref: '#/definitions/models.APIResponse'
+      security:
+      - Bearer: []
+      summary: Will verify the secondary persistence.
+      tags:
+      - persistence
   /api/persistence/verify:
     post:
       description: Will verify the persistence.
@@ -756,6 +966,21 @@ paths:
       summary: Will verify the persistence.
       tags:
       - persistence
+  /health:
+    get:
+      description: Confirms that the Agent HTTP process can serve requests and reports
+        current camera stream and Hub heartbeat diagnostics. Operational dependency
+        failures do not change the liveness HTTP status.
+      produces:
+      - application/json
+      responses:
+        "200":
+          description: OK
+          schema:
+            $ref: '#/definitions/http.HealthResponse'
+      summary: Check Agent health
+      tags:
+      - health
 securityDefinitions:
   Bearer:
     in: header

machinery/go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/bluenviron/gortsplib/v5 v5.6.3
 	github.com/bluenviron/mediacommon v1.14.0
 	github.com/cedricve/go-onvif v0.0.0-20200222191200-567e8ce298f6
+	github.com/creack/pty v1.1.24
 	github.com/dromara/carbon/v2 v2.6.8
 	github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5
 	github.com/eclipse/paho.mqtt.golang v1.5.0
@@ -24,11 +25,10 @@ require (
 	github.com/gorilla/websocket v1.5.3
 	github.com/kellydunn/golang-geo v0.7.0
 	github.com/kerberos-io/joy4 v1.0.64
-	github.com/kerberos-io/onvif v1.2.2
+	github.com/kerberos-io/onvif v1.2.3
 	github.com/minio/minio-go/v6 v6.0.57
 	github.com/moq-dev/moq-go v0.5.7
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
-	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/pion/interceptor v0.1.47
 	github.com/pion/rtp v1.10.5
 	github.com/pion/webrtc/v4 v4.2.18
@@ -45,7 +45,6 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.36.0
 	go.opentelemetry.io/otel/sdk v1.36.0
 	go.opentelemetry.io/otel/trace v1.36.0
-	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 )
 
 require (

machinery/go.sum

@@ -456,6 +456,8 @@ github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78/go.mod h1:W+zGtBO5Y1Ig
 github.com/cncf/xds/go v0.0.0-20241223141626-cff3c89139a3/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
 github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
+github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -774,8 +776,8 @@ github.com/kellydunn/golang-geo v0.7.0 h1:A5j0/BvNgGwY6Yb6inXQxzYwlPHc6WVZR+Mrar
 github.com/kellydunn/golang-geo v0.7.0/go.mod h1:YYlQPJ+DPEzrHx8kT3oPHC/NjyvCCXE+IuKGKdrjrcU=
 github.com/kerberos-io/joy4 v1.0.64 h1:gTUSotHSOhp9mNqEecgq88tQHvpj7TjmrvPUsPm0idg=
 github.com/kerberos-io/joy4 v1.0.64/go.mod h1:nZp4AjvKvTOXRrmDyAIOw+Da+JA5OcSo/JundGfOlFU=
-github.com/kerberos-io/onvif v1.2.2 h1:QnxITps7xvAVD2abWRsa3+p9QexjKESQQldUFMx/mYA=
-github.com/kerberos-io/onvif v1.2.2/go.mod h1:XSgEQXmEDjUQTbdXvsaRJt6Az8YPGj7L+j5iXKEGijU=
+github.com/kerberos-io/onvif v1.2.3 h1:RDnDvLBd92VCgFOrlbGavzCVN9lQA/qs9novdNFIAOU=
+github.com/kerberos-io/onvif v1.2.3/go.mod h1:XSgEQXmEDjUQTbdXvsaRJt6Az8YPGj7L+j5iXKEGijU=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
@@ -858,8 +860,6 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
 github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE=
 github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
-github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
-github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
 github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
 github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY=
@@ -2106,8 +2106,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk=
 gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
-gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

machinery/logging.go

@@ -0,0 +1,160 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	defaultLogLevel  = log.InfoLevel
+	defaultLogOutput = "text"
+)
+
+type localTimeFormatter struct {
+	timezone  *time.Location
+	formatter log.Formatter
+}
+
+func (f localTimeFormatter) Format(entry *log.Entry) ([]byte, error) {
+	entry.Time = entry.Time.In(f.timezone)
+	return f.formatter.Format(entry)
+}
+
+type componentHook struct{}
+
+func (componentHook) Levels() []log.Level {
+	return log.AllLevels
+}
+
+func (componentHook) Fire(entry *log.Entry) error {
+	if _, exists := entry.Data["component"]; exists {
+		return nil
+	}
+	entry.Data["component"] = componentFromCaller(entry.Caller)
+	return nil
+}
+
+func configureLogging(levelValue string, outputValue string, timezone *time.Location) {
+	configureLogger(log.StandardLogger(), levelValue, outputValue, timezone)
+}
+
+func configureLogger(logger *log.Logger, levelValue string, outputValue string, timezone *time.Location) {
+	if timezone == nil {
+		timezone = time.Local
+	}
+
+	level, levelErr := parseLogLevel(levelValue)
+	output, outputErr := parseLogOutput(outputValue)
+
+	logger.SetOutput(os.Stdout)
+	logger.SetLevel(level)
+	logger.SetReportCaller(true)
+	logger.SetFormatter(localTimeFormatter{
+		timezone:  timezone,
+		formatter: newLogFormatter(output),
+	})
+	installComponentHook(logger)
+
+	if levelErr != nil {
+		logger.WithFields(log.Fields{
+			"configured_level": levelValue,
+			"effective_level":  level.String(),
+		}).WithError(levelErr).Warn("invalid log level; using default")
+	}
+	if outputErr != nil {
+		logger.WithFields(log.Fields{
+			"configured_output": outputValue,
+			"effective_output":  output,
+		}).WithError(outputErr).Warn("invalid log output; using default")
+	}
+
+	logger.WithFields(log.Fields{
+		"event":         "logger_configured",
+		"log_level":     level.String(),
+		"output":        output,
+		"report_caller": logger.ReportCaller,
+		"timezone":      timezone.String(),
+	}).Debug("logging configured")
+}
+
+func installComponentHook(logger *log.Logger) {
+	for _, hooks := range logger.Hooks {
+		for _, hook := range hooks {
+			if _, ok := hook.(componentHook); ok {
+				return
+			}
+		}
+	}
+	logger.AddHook(componentHook{})
+}
+
+func componentFromCaller(frame *runtime.Frame) string {
+	if frame == nil {
+		return "unknown"
+	}
+
+	const sourceMarker = "/machinery/src/"
+	normalizedFile := filepath.ToSlash(frame.File)
+	if markerIndex := strings.Index(normalizedFile, sourceMarker); markerIndex >= 0 {
+		relativeFile := normalizedFile[markerIndex+len(sourceMarker):]
+		if directory := filepath.ToSlash(filepath.Dir(relativeFile)); directory != "." {
+			return directory
+		}
+	}
+	if strings.Contains(normalizedFile, "/machinery/") {
+		return "agent"
+	}
+	return "unknown"
+}
+
+func parseLogLevel(value string) (log.Level, error) {
+	normalized := strings.ToLower(strings.TrimSpace(value))
+	if normalized == "" {
+		return defaultLogLevel, nil
+	}
+	if normalized == "warning" {
+		normalized = "warn"
+	}
+
+	level, err := log.ParseLevel(normalized)
+	if err != nil {
+		return defaultLogLevel, fmt.Errorf("parse LOG_LEVEL: %w", err)
+	}
+	return level, nil
+}
+
+func parseLogOutput(value string) (string, error) {
+	normalized := strings.ToLower(strings.TrimSpace(value))
+	if normalized == "" {
+		return defaultLogOutput, nil
+	}
+	switch normalized {
+	case "json", "text":
+		return normalized, nil
+	default:
+		return defaultLogOutput, fmt.Errorf("unsupported LOG_OUTPUT %q", value)
+	}
+}
+
+func newLogFormatter(output string) log.Formatter {
+	callerPrettyfier := func(frame *runtime.Frame) (string, string) {
+		return filepath.Base(frame.Function), fmt.Sprintf("%s:%d", filepath.Base(frame.File), frame.Line)
+	}
+	if output == "json" {
+		return &log.JSONFormatter{
+			CallerPrettyfier: callerPrettyfier,
+			TimestampFormat:  time.RFC3339Nano,
+		}
+	}
+	return &log.TextFormatter{
+		CallerPrettyfier: callerPrettyfier,
+		FullTimestamp:    true,
+		TimestampFormat:  time.RFC3339Nano,
+	}
+}

machinery/logging_test.go

@@ -0,0 +1,165 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func TestConfigureLoggingDebugIncludesStructuredContext(t *testing.T) {
+	logger := log.New()
+	var output bytes.Buffer
+	configureLogger(logger, "debug", "json", time.UTC)
+	logger.SetOutput(&output)
+	logger.WithField("event", "test_event").Debug("structured debug test")
+
+	lines := strings.Split(strings.TrimSpace(output.String()), "\n")
+	var entry map[string]interface{}
+	if err := json.Unmarshal([]byte(lines[len(lines)-1]), &entry); err != nil {
+		t.Fatalf("decode debug log: %v; output=%q", err, output.String())
+	}
+	for key, want := range map[string]interface{}{
+		"component": "agent",
+		"event":     "test_event",
+		"level":     "debug",
+		"msg":       "structured debug test",
+	} {
+		if got := entry[key]; got != want {
+			t.Fatalf("%s = %v, want %v", key, got, want)
+		}
+	}
+	if entry["file"] == nil || entry["func"] == nil {
+		t.Fatalf("debug log is missing caller metadata: %v", entry)
+	}
+}
+
+func TestConfigureLoggerInstallsComponentHookOnce(t *testing.T) {
+	logger := log.New()
+	configureLogger(logger, "info", "text", time.UTC)
+	configureLogger(logger, "debug", "json", time.UTC)
+
+	var componentHooks int
+	for _, hooks := range logger.Hooks {
+		for _, hook := range hooks {
+			if _, ok := hook.(componentHook); ok {
+				componentHooks++
+			}
+		}
+	}
+	if componentHooks != len(log.AllLevels) {
+		t.Fatalf("component hook registrations = %d, want %d", componentHooks, len(log.AllLevels))
+	}
+}
+
+func TestComponentFromCaller(t *testing.T) {
+	tests := []struct {
+		name  string
+		frame *runtime.Frame
+		want  string
+	}{
+		{
+			name:  "nested runtime package",
+			frame: &runtime.Frame{File: "/workspace/agent/machinery/src/routers/mqtt/main.go"},
+			want:  "routers/mqtt",
+		},
+		{
+			name:  "top-level runtime package",
+			frame: &runtime.Frame{File: "/workspace/agent/machinery/src/capture/main.go"},
+			want:  "capture",
+		},
+		{
+			name:  "executable",
+			frame: &runtime.Frame{File: "/workspace/agent/machinery/main.go"},
+			want:  "agent",
+		},
+		{name: "missing caller", want: "unknown"},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if got := componentFromCaller(test.frame); got != test.want {
+				t.Fatalf("componentFromCaller() = %q, want %q", got, test.want)
+			}
+		})
+	}
+}
+
+func TestComponentHookAddsAndPreservesComponent(t *testing.T) {
+	hook := componentHook{}
+
+	entry := log.NewEntry(log.New())
+	entry.Caller = &runtime.Frame{File: "/workspace/agent/machinery/src/cloud/livehls/session.go"}
+	if err := hook.Fire(entry); err != nil {
+		t.Fatalf("componentHook.Fire() error = %v", err)
+	}
+	if got := entry.Data["component"]; got != "cloud/livehls" {
+		t.Fatalf("component = %v, want cloud/livehls", got)
+	}
+
+	entry.Data["component"] = "explicit"
+	if err := hook.Fire(entry); err != nil {
+		t.Fatalf("componentHook.Fire() preserving field error = %v", err)
+	}
+	if got := entry.Data["component"]; got != "explicit" {
+		t.Fatalf("component = %v, want explicit", got)
+	}
+}
+
+func TestParseLogLevel(t *testing.T) {
+	tests := []struct {
+		name    string
+		value   string
+		want    log.Level
+		wantErr bool
+	}{
+		{name: "default", want: log.InfoLevel},
+		{name: "info", value: "INFO", want: log.InfoLevel},
+		{name: "warning alias", value: "warning", want: log.WarnLevel},
+		{name: "debug", value: "debug", want: log.DebugLevel},
+		{name: "trace", value: "trace", want: log.TraceLevel},
+		{name: "invalid", value: "verbose", want: log.InfoLevel, wantErr: true},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got, err := parseLogLevel(test.value)
+			if (err != nil) != test.wantErr {
+				t.Fatalf("parseLogLevel(%q) error = %v, wantErr %t", test.value, err, test.wantErr)
+			}
+			if got != test.want {
+				t.Fatalf("parseLogLevel(%q) = %s, want %s", test.value, got, test.want)
+			}
+		})
+	}
+}
+
+func TestParseLogOutput(t *testing.T) {
+	tests := []struct {
+		name    string
+		value   string
+		want    string
+		wantErr bool
+	}{
+		{name: "default", want: "text"},
+		{name: "text", value: "TEXT", want: "text"},
+		{name: "json", value: "json", want: "json"},
+		{name: "invalid", value: "console", want: "text", wantErr: true},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got, err := parseLogOutput(test.value)
+			if (err != nil) != test.wantErr {
+				t.Fatalf("parseLogOutput(%q) error = %v, wantErr %t", test.value, err, test.wantErr)
+			}
+			if got != test.want {
+				t.Fatalf("parseLogOutput(%q) = %q, want %q", test.value, got, test.want)
+			}
+		})
+	}
+}

machinery/main.go

@@ -5,14 +5,15 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/capture"
 	"github.com/kerberos-io/agent/machinery/src/components"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/onvif"
+	log "github.com/sirupsen/logrus"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
@@ -28,6 +29,22 @@ import (
 
 var VERSION = utils.VERSION
 
+func resolveServerPort(flagValue, environmentValue string) (string, error) {
+	value := strings.TrimSpace(environmentValue)
+	if value == "" {
+		value = strings.TrimSpace(flagValue)
+	}
+	if value == "" {
+		value = "80"
+	}
+
+	port, err := strconv.Atoi(value)
+	if err != nil || port < 1 || port > 65535 {
+		return "", fmt.Errorf("port must be an integer between 1 and 65535, got %q", value)
+	}
+	return strconv.Itoa(port), nil
+}
+
 func startTracing(agentKey string, otelEndpoint string) (*trace.TracerProvider, error) {
 	serviceName := "agent-" + agentKey
 	headers := map[string]string{
@@ -99,20 +116,44 @@ func main() {
 	}
 	// Specify the timezone of the log: "UTC" or "Local".
 	timezone, _ := time.LoadLocation("CET")
-	log.Log.Init(logLevel, logOutput, configDirectory, timezone)
+	configureLogging(logLevel, logOutput, timezone)
+	if action == "run" {
+		resolvedPort, err := resolveServerPort(port, os.Getenv("AGENT_PORT"))
+		if err != nil {
+			log.WithError(err).WithFields(log.Fields{
+				"component": "http",
+				"event":     "server_port_invalid",
+			}).Fatal("Invalid HTTP server port")
+			return
+		}
+		port = resolvedPort
+	}
+	log.WithFields(log.Fields{
+		"action":           action,
+		"component":        "agent",
+		"config_directory": configDirectory,
+		"event":            "command_parsed",
+		"port":             port,
+		"version":          VERSION,
+	}).Debug("Agent command parsed")
 
 	switch action {
 
 	case "version":
 		{
-			log.Log.Info("main.Main(): You are currrently running Kerberos Agent " + VERSION)
+			log.WithFields(log.Fields{
+				"component": "agent",
+				"event":     "version",
+				"version":   VERSION,
+			}).Info("Kerberos Agent version")
 		}
 	case "discover":
 		{
 			// Convert duration to int
 			timeout, err := time.ParseDuration(timeout + "ms")
 			if err != nil {
-				log.Log.Fatal("main.Main(): could not parse timeout: " + err.Error())
+				log.WithError(err).WithField("component", "onvif").
+					Fatal("invalid ONVIF discovery timeout")
 				return
 			}
 			var subnets []string
@@ -125,15 +166,19 @@ func main() {
 		}
 	case "decrypt":
 		{
-			log.Log.Info("main.Main(): Decrypting: " + flag.Arg(0) + " with key: " + flag.Arg(1))
+			log.WithFields(log.Fields{
+				"component": "encryption",
+				"event":     "decrypt_started",
+				"path":      flag.Arg(0),
+			}).Info("Decrypting recording")
 			symmetricKey := []byte(flag.Arg(1))
 
 			if len(symmetricKey) == 0 {
-				log.Log.Fatal("main.Main(): symmetric key should not be empty")
+				log.Fatal("main.Main(): symmetric key should not be empty")
 				return
 			}
 			if len(symmetricKey) != 32 {
-				log.Log.Fatal("main.Main(): symmetric key should be 32 bytes")
+				log.Fatal("main.Main(): symmetric key should be 32 bytes")
 				return
 			}
 
@@ -163,19 +208,28 @@ func main() {
 
 			// Start OpenTelemetry tracing
 			if otelEndpoint := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"); otelEndpoint == "" {
-				log.Log.Info("main.Main(): No OpenTelemetry endpoint provided, skipping tracing")
+				log.WithFields(log.Fields{
+					"component": "tracing",
+					"event":     "tracing_disabled",
+				}).Debug("OpenTelemetry tracing disabled")
 			} else {
-				log.Log.Info("main.Main(): Starting OpenTelemetry tracing with endpoint: " + otelEndpoint)
+				log.WithFields(log.Fields{
+					"component": "tracing",
+					"event":     "tracing_starting",
+				}).Info("Starting OpenTelemetry tracing")
 				agentKey := configuration.Config.Key
 				traceProvider, err := startTracing(agentKey, otelEndpoint)
 				if err != nil {
-					log.Log.Error("traceprovider: " + err.Error())
+					log.WithError(err).WithField("component", "tracing").
+						Error("Failed to start OpenTelemetry tracing")
+				} else {
+					defer func() {
+						if err := traceProvider.Shutdown(context.Background()); err != nil {
+							log.WithError(err).WithField("component", "tracing").
+								Error("Failed to shut down OpenTelemetry tracing")
+						}
+					}()
 				}
-				defer func() {
-					if err := traceProvider.Shutdown(context.Background()); err != nil {
-						log.Log.Error("traceprovider: " + err.Error())
-					}
-				}()
 			}
 
 			// Printing final configuration
@@ -186,8 +240,13 @@ func main() {
 			utils.CheckDataDirectoryPermissions(configDirectory)
 
 			// Set timezone
-			timezone, _ := time.LoadLocation(configuration.Config.Timezone)
-			log.Log.Init(logLevel, logOutput, configDirectory, timezone)
+			timezone, err := time.LoadLocation(configuration.Config.Timezone)
+			if err != nil {
+				log.WithError(err).WithField("timezone", configuration.Config.Timezone).
+					Warn("invalid Agent timezone; using the host timezone for logs")
+				timezone = time.Local
+			}
+			configureLogging(logLevel, logOutput, timezone)
 
 			// Check if we have a device Key or not, if not
 			// we will generate one.
@@ -196,30 +255,37 @@ func main() {
 				configuration.Config.Key = key
 				err := configService.StoreConfig(configDirectory, configuration.Config)
 				if err == nil {
-					log.Log.Info("main.Main(): updated unique key for agent to: " + key)
+					log.WithFields(log.Fields{
+						"component": "configuration",
+						"event":     "agent_key_generated",
+					}).Info("Generated and stored a unique Agent key")
 				} else {
-					log.Log.Info("main.Main(): something went wrong while trying to store key: " + key)
+					log.WithError(err).WithFields(log.Fields{
+						"component": "configuration",
+						"event":     "agent_key_store_failed",
+					}).Error("Failed to store the generated Agent key")
 				}
 			}
 
 			// Create a cancelable context, which will be used to cancel and restart.
 			// This is used to restart the agent when the configuration is updated.
 			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
 
 			// We create a capture object, this will contain all the streaming clients.
 			// And allow us to extract media from within difference places in the agent.
-			capture := capture.Capture{
-				RTSPClient:    nil,
-				RTSPSubClient: nil,
-			}
+			capture := capture.Capture{}
 
 			// Bootstrapping the agent
 			communication := models.Communication{
-				Context:         &ctx,
-				CancelContext:   &cancel,
 				HandleBootstrap: make(chan string, 1),
 			}
 
+			log.WithFields(log.Fields{
+				"component": "agent",
+				"event":     "runtime_starting",
+				"port":      configuration.Port,
+			}).Info("Starting Agent runtime")
 			go components.Bootstrap(ctx, configDirectory, &configuration, &communication, &capture)
 
 			// Start the REST API.
@@ -227,7 +293,7 @@ func main() {
 		}
 	default:
 		{
-			log.Log.Error("main.Main(): Sorry I don't understand :(")
+			log.Error("main.Main(): Sorry I don't understand :(")
 		}
 	}
 }

machinery/main_test.go

@@ -0,0 +1,33 @@
+package main
+
+import "testing"
+
+func TestResolveServerPort(t *testing.T) {
+	tests := []struct {
+		name      string
+		flagValue string
+		envValue  string
+		want      string
+		wantError bool
+	}{
+		{name: "flag default", flagValue: "80", want: "80"},
+		{name: "environment overrides flag", flagValue: "80", envValue: "8082", want: "8082"},
+		{name: "trims environment", flagValue: "80", envValue: " 9090 ", want: "9090"},
+		{name: "empty values use default", want: "80"},
+		{name: "invalid text", flagValue: "80", envValue: "http", wantError: true},
+		{name: "zero", flagValue: "80", envValue: "0", wantError: true},
+		{name: "above maximum", flagValue: "80", envValue: "65536", wantError: true},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got, err := resolveServerPort(test.flagValue, test.envValue)
+			if (err != nil) != test.wantError {
+				t.Fatalf("resolveServerPort(%q, %q) error = %v, wantError %t", test.flagValue, test.envValue, err, test.wantError)
+			}
+			if got != test.want {
+				t.Fatalf("resolveServerPort(%q, %q) = %q, want %q", test.flagValue, test.envValue, got, test.want)
+			}
+		})
+	}
+}

machinery/src/capture/cleanup_test.go

@@ -10,11 +10,58 @@ import (
 	"github.com/kerberos-io/agent/machinery/src/models"
 )
 
+func TestPickRecordingToCleanupAcrossDatedDirectories(t *testing.T) {
+	for _, legacyMarker := range []bool{false, true} {
+		t.Run(map[bool]string{false: "metadata", true: "legacy"}[legacyMarker], func(t *testing.T) {
+			recordingsDir, cloudDir := newCleanupDirs(t)
+			pendingName := "2026/01/01/oldest_pending.mp4"
+			safeName := "2026/01/02/uploaded.mp4"
+			writeRecording(t, recordingsDir, pendingName, 30)
+			if legacyMarker {
+				if err := os.WriteFile(filepath.Join(cloudDir, filepath.Base(pendingName)), nil, 0o644); err != nil {
+					t.Fatal(err)
+				}
+			} else {
+				markPending(t, cloudDir, pendingName)
+			}
+			writeRecording(t, recordingsDir, safeName, 20)
+			writeRecording(t, recordingsDir, "flat_uploaded.mp4", 10)
+			name, pending, err := pickRecordingToCleanup(recordingsDir, cloudDir)
+			if err != nil || pending || name != safeName {
+				t.Fatalf("cleanup = (%q, %v, %v), want (%q, false, nil)", name, pending, err, safeName)
+			}
+		})
+	}
+}
+
+func TestCleanupDatedPendingRecordingRemovesFlatMarker(t *testing.T) {
+	recordingsDir, cloudDir := newCleanupDirs(t)
+	name := "2026/01/01/pending.mp4"
+	if err := os.MkdirAll(filepath.Dir(filepath.Join(recordingsDir, name)), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	writeSizedRecording(t, recordingsDir, name, 2*1000*1000)
+	markPending(t, cloudDir, name)
+	config := &models.Configuration{Config: models.Config{AutoClean: "true", MaxDirectorySize: 1}}
+	CleanupRecordingDirectory(filepath.Dir(filepath.Dir(recordingsDir)), config)
+	for _, filePath := range []string{
+		filepath.Join(recordingsDir, name),
+		filepath.Join(cloudDir, models.RecordingUploadMetadataFileName(name)),
+	} {
+		if _, err := os.Stat(filePath); !os.IsNotExist(err) {
+			t.Fatalf("expected %s to be removed, got %v", filePath, err)
+		}
+	}
+}
+
 // writeRecording creates a file under recordingsDir and sets its modtime so the
 // tests can control the "oldest" ordering deterministically.
 func writeRecording(t *testing.T, recordingsDir, name string, ageMinutes int) {
 	t.Helper()
 	full := filepath.Join(recordingsDir, name)
+	if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
+		t.Fatalf("mkdir recording directory: %v", err)
+	}
 	if err := os.WriteFile(full, []byte("data"), 0o644); err != nil {
 		t.Fatalf("write recording %s: %v", name, err)
 	}

machinery/src/capture/gortsplib_test.go

@@ -1,22 +1,249 @@
 package capture
 
-import "testing"
+import (
+	"bytes"
+	"context"
+	"encoding/pem"
+	"errors"
+	"math"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/bluenviron/mediacommon/pkg/codecs/h264"
+)
+
+func TestPreRecordingGOPCount(t *testing.T) {
+	maxInt := int64(^uint(0) >> 1)
+	tests := []struct {
+		name         string
+		preRecording int64
+		gopDuration  float64
+		want         int
+		wantOK       bool
+	}{
+		{name: "normal duration", preRecording: 10, gopDuration: 2.9, want: 6, wantOK: true},
+		{name: "duration longer than buffer", preRecording: 1, gopDuration: 2, want: 1, wantOK: true},
+		{name: "largest representable result", preRecording: maxInt - 1, gopDuration: 1, want: int(maxInt), wantOK: true},
+		{name: "result exceeds int", preRecording: maxInt, gopDuration: 1, wantOK: false},
+		{name: "non-positive pre-recording", preRecording: 0, gopDuration: 1, wantOK: false},
+		{name: "sub-second GOP", preRecording: 10, gopDuration: 0.9, wantOK: false},
+		{name: "NaN GOP", preRecording: 10, gopDuration: math.NaN(), wantOK: false},
+		{name: "infinite GOP", preRecording: 10, gopDuration: math.Inf(1), wantOK: false},
+		{name: "GOP exceeds int64", preRecording: 10, gopDuration: float64(math.MaxInt64), wantOK: false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, ok := preRecordingGOPCount(tt.preRecording, tt.gopDuration)
+			if ok != tt.wantOK || got != tt.want {
+				t.Fatalf("preRecordingGOPCount(%d, %v) = (%d, %t), want (%d, %t)",
+					tt.preRecording, tt.gopDuration, got, ok, tt.want, tt.wantOK)
+			}
+		})
+	}
+}
+
+func TestGolibrtspCloseBeforeClientStart(t *testing.T) {
+	client := &Golibrtsp{}
+
+	if err := client.Close(context.Background()); err != nil {
+		t.Fatalf("Close() error = %v", err)
+	}
+}
+
+func TestSanitizeRTSPErrorRemovesCredentialsAndQuery(t *testing.T) {
+	rawURL := "rtsp://camera-user:camera-password@10.0.20.15/live?access_token=secret"
+	got := sanitizeRTSPError(errors.New("describe "+rawURL+": bad status code"), rawURL)
+
+	for _, secret := range []string{"camera-user", "camera-password", "access_token", "secret"} {
+		if strings.Contains(got.Error(), secret) {
+			t.Fatalf("sanitizeRTSPError() exposed %q in %q", secret, got)
+		}
+	}
+	if !strings.Contains(got.Error(), "rtsp://10.0.20.15/live") {
+		t.Fatalf("sanitizeRTSPError() removed useful host/path context: %q", got)
+	}
+}
 
 func TestRTSPSTLSConfig(t *testing.T) {
 	t.Run("verifies certificates by default", func(t *testing.T) {
+		t.Setenv(rtspsCAFileEnv, "")
 		t.Setenv(rtspsInsecureEnv, "")
 
-		if got := rtspsTLSConfig(); got != nil {
+		got, err := rtspsTLSConfig()
+		if err != nil {
+			t.Fatalf("rtspsTLSConfig() error = %v", err)
+		}
+		if got != nil {
 			t.Fatalf("rtspsTLSConfig() = %#v, want nil", got)
 		}
 	})
 
 	t.Run("allows explicit insecure mode", func(t *testing.T) {
+		t.Setenv(rtspsCAFileEnv, "/missing/ignored-in-insecure-mode.pem")
 		t.Setenv(rtspsInsecureEnv, "true")
 
-		got := rtspsTLSConfig()
+		got, err := rtspsTLSConfig()
+		if err != nil {
+			t.Fatalf("rtspsTLSConfig() error = %v", err)
+		}
 		if got == nil || !got.InsecureSkipVerify {
 			t.Fatalf("rtspsTLSConfig() = %#v, want InsecureSkipVerify enabled", got)
 		}
 	})
+
+	t.Run("adds a camera CA to system roots", func(t *testing.T) {
+		t.Setenv(rtspsInsecureEnv, "")
+		server := httptest.NewTLSServer(nil)
+		defer server.Close()
+
+		certificate := server.Certificate()
+		caFile := filepath.Join(t.TempDir(), "camera-ca.pem")
+		caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certificate.Raw})
+		if err := os.WriteFile(caFile, caPEM, 0o600); err != nil {
+			t.Fatal(err)
+		}
+		t.Setenv(rtspsCAFileEnv, caFile)
+
+		got, err := rtspsTLSConfig()
+		if err != nil {
+			t.Fatalf("rtspsTLSConfig() error = %v", err)
+		}
+		if got == nil || got.RootCAs == nil {
+			t.Fatalf("rtspsTLSConfig() = %#v, want custom RootCAs", got)
+		}
+		for _, subject := range got.RootCAs.Subjects() {
+			if bytes.Equal(subject, certificate.RawSubject) {
+				return
+			}
+		}
+		t.Fatal("camera CA was not added to RootCAs")
+	})
+
+	t.Run("rejects an invalid camera CA file", func(t *testing.T) {
+		t.Setenv(rtspsInsecureEnv, "")
+		caFile := filepath.Join(t.TempDir(), "camera-ca.pem")
+		if err := os.WriteFile(caFile, []byte("not a certificate"), 0o600); err != nil {
+			t.Fatal(err)
+		}
+		t.Setenv(rtspsCAFileEnv, caFile)
+
+		if _, err := rtspsTLSConfig(); err == nil {
+			t.Fatal("rtspsTLSConfig() error = nil, want invalid CA error")
+		}
+	})
+}
+
+func TestVideoReferenceStateDropsDependentFramesUntilRandomAccess(t *testing.T) {
+	state := &videoReferenceState{}
+
+	accepted, recovered := state.accept(false)
+	if accepted || recovered != 0 || state.droppedAccessUnits != 1 {
+		t.Fatalf("initial dependent frame = (accepted=%t, recovered=%d, dropped=%d), want (false, 0, 1)",
+			accepted, recovered, state.droppedAccessUnits)
+	}
+
+	accepted, recovered = state.accept(true)
+	if !accepted || recovered != 1 || !state.valid {
+		t.Fatalf("random-access recovery = (accepted=%t, recovered=%d, valid=%t), want (true, 1, true)",
+			accepted, recovered, state.valid)
+	}
+
+	if expected, gap := state.observeSequence(100); gap || expected != 100 {
+		t.Fatalf("first sequence observation = (expected=%d, gap=%t), want (100, false)", expected, gap)
+	}
+	if expected, gap := state.observeSequence(102); !gap || expected != 101 || state.valid {
+		t.Fatalf("sequence gap = (expected=%d, gap=%t, valid=%t), want (101, true, false)",
+			expected, gap, state.valid)
+	}
+
+	accepted, recovered = state.accept(false)
+	if accepted || recovered != 0 || state.droppedAccessUnits != 1 {
+		t.Fatalf("post-loss dependent frame = (accepted=%t, recovered=%d, dropped=%d), want (false, 0, 1)",
+			accepted, recovered, state.droppedAccessUnits)
+	}
+
+	accepted, recovered = state.accept(true)
+	if !accepted || recovered != 1 || !state.valid {
+		t.Fatalf("post-loss recovery = (accepted=%t, recovered=%d, valid=%t), want (true, 1, true)",
+			accepted, recovered, state.valid)
+	}
+}
+
+func TestVideoReferenceStateAcceptsSequenceWraparound(t *testing.T) {
+	state := &videoReferenceState{valid: true}
+
+	if _, gap := state.observeSequence(^uint16(0)); gap {
+		t.Fatal("first sequence observation reported a gap")
+	}
+	if expected, gap := state.observeSequence(0); gap || expected != 0 || !state.valid {
+		t.Fatalf("sequence wraparound = (expected=%d, gap=%t, valid=%t), want (0, false, true)",
+			expected, gap, state.valid)
+	}
+}
+
+func TestH264FrameNumTrackerDetectsMissingIDR(t *testing.T) {
+	tracker := &h264FrameNumTracker{}
+	tracker.setSPS(&h264.SPS{
+		Log2MaxFrameNumMinus4: 1,
+		FrameMbsOnlyFlag:      true,
+	})
+
+	if _, _, gap, ok := tracker.observe([][]byte{testH264Slice(h264.NALUTypeIDR, 0)}); !ok || gap {
+		t.Fatalf("initial IDR observation = (ok=%t, gap=%t), want (true, false)", ok, gap)
+	}
+	for frameNum := uint32(1); frameNum <= 24; frameNum++ {
+		if expected, received, gap, ok := tracker.observe([][]byte{testH264Slice(h264.NALUTypeNonIDR, frameNum)}); !ok || gap {
+			t.Fatalf("frame_num %d observation = (expected=%d, received=%d, ok=%t, gap=%t), want no gap",
+				frameNum, expected, received, ok, gap)
+		}
+	}
+
+	expected, received, gap, ok := tracker.observe([][]byte{testH264Slice(h264.NALUTypeNonIDR, 1)})
+	if !ok || !gap || expected != 25 || received != 1 {
+		t.Fatalf("missing IDR transition = (expected=%d, received=%d, ok=%t, gap=%t), want (25, 1, true, true)",
+			expected, received, ok, gap)
+	}
+}
+
+func TestH264FrameNumTrackerResetsAtIDR(t *testing.T) {
+	tracker := &h264FrameNumTracker{}
+	tracker.setSPS(&h264.SPS{
+		Log2MaxFrameNumMinus4: 1,
+		FrameMbsOnlyFlag:      true,
+	})
+
+	tracker.observe([][]byte{testH264Slice(h264.NALUTypeIDR, 0)})
+	tracker.observe([][]byte{testH264Slice(h264.NALUTypeNonIDR, 1)})
+	if expected, received, gap, ok := tracker.observe([][]byte{testH264Slice(h264.NALUTypeIDR, 0)}); !ok || gap {
+		t.Fatalf("IDR reset = (expected=%d, received=%d, ok=%t, gap=%t), want no gap",
+			expected, received, ok, gap)
+	}
+	if expected, received, gap, ok := tracker.observe([][]byte{testH264Slice(h264.NALUTypeNonIDR, 1)}); !ok || gap || expected != 1 || received != 1 {
+		t.Fatalf("post-IDR frame = (expected=%d, received=%d, ok=%t, gap=%t), want (1, 1, true, false)",
+			expected, received, ok, gap)
+	}
+}
+
+func TestH264FrameNumTrackerHonorsPermittedGaps(t *testing.T) {
+	tracker := &h264FrameNumTracker{}
+	tracker.setSPS(&h264.SPS{
+		Log2MaxFrameNumMinus4:          1,
+		FrameMbsOnlyFlag:               true,
+		GapsInFrameNumValueAllowedFlag: true,
+	})
+
+	tracker.observe([][]byte{testH264Slice(h264.NALUTypeIDR, 0)})
+	expected, received, gap, ok := tracker.observe([][]byte{testH264Slice(h264.NALUTypeNonIDR, 2)})
+	if !ok || gap || expected != 1 || received != 2 {
+		t.Fatalf("permitted frame_num gap = (expected=%d, received=%d, ok=%t, gap=%t), want (1, 2, true, false)",
+			expected, received, ok, gap)
+	}
+}
+
+func testH264Slice(naluType h264.NALUType, frameNum uint32) []byte {
+	return []byte{0x60 | byte(naluType), 0xE0 | byte(frameNum&0x1F)}
 }

machinery/src/capture/main.go

@@ -16,11 +16,11 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/kerberos-io/agent/machinery/src/conditions"
 	"github.com/kerberos-io/agent/machinery/src/encryption"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/kerberos-io/agent/machinery/src/utils"
 	"github.com/kerberos-io/agent/machinery/src/video"
+	log "github.com/sirupsen/logrus"
 	"go.opentelemetry.io/otel/trace"
 )
 
@@ -49,16 +49,20 @@ func publishRecordingState(mqttClient mqtt.Client, hubKey string, configuration
 	if err == nil {
 		mqttClient.Publish("kerberos/hub/"+hubKey, 2, false, payload)
 	} else {
-		log.Log.Error("capture.main.publishRecordingState(): failed to package MQTT message: " + err.Error())
+		log.Error("capture.main.publishRecordingState(): failed to package MQTT message: " + err.Error())
 	}
 }
 
-func recordingUploadMetadata(name, deviceKey string, timestamp int64, mp4Video *video.MP4) models.RecordingUploadMetadata {
+func recordingUploadMetadata(name string, config models.Config, timestamp int64, mp4Video *video.MP4, encrypted bool, coordinates string, changes int) models.RecordingUploadMetadata {
 	metadata := models.RecordingUploadMetadata{
-		FileName:  filepath.Base(name),
-		DeviceKey: deviceKey,
-		Timestamp: timestamp,
-		Duration:  mp4Video.VideoTotalDuration,
+		FileName:          filepath.Base(name),
+		DeviceKey:         config.Key,
+		Timestamp:         timestamp,
+		Duration:          mp4Video.VideoTotalDuration,
+		Encrypted:         encrypted,
+		DeviceName:        config.Name,
+		RegionCoordinates: coordinates,
+		NumberOfChanges:   strconv.Itoa(changes),
 	}
 	value := mp4Video.AverageFPS()
 	if value > 0 && value <= 240 && !math.IsInf(value, 0) && !math.IsNaN(value) {
@@ -72,7 +76,7 @@ func recordingUploadMetadata(name, deviceKey string, timestamp int64, mp4Video *
 func queueRecordingForUpload(configDirectory string, metadata models.RecordingUploadMetadata) {
 	payload, err := json.Marshal(metadata)
 	if err != nil {
-		log.Log.Error("capture.main.queueRecordingForUpload(): " + err.Error())
+		log.Error("capture.main.queueRecordingForUpload(): " + err.Error())
 		return
 	}
 
@@ -95,7 +99,7 @@ func queueRecordingForUpload(configDirectory string, metadata models.RecordingUp
 		err = os.Rename(marker.Name(), filepath.Join(configDirectory, "data", "cloud", models.RecordingUploadMetadataFileName(metadata.FileName)))
 	}
 	if err != nil {
-		log.Log.Error("capture.main.queueRecordingForUpload(): " + err.Error())
+		log.Error("capture.main.queueRecordingForUpload(): " + err.Error())
 	}
 }
 
@@ -137,9 +141,9 @@ func manualRecordingExpired(communication *models.Communication, now int64) bool
 		return false
 	}
 	if heartbeatExpired {
-		log.Log.Info("capture.main.HandleRecordStream(motiondetection): auto-stopping manual recording, no viewer heartbeat within timeout.")
+		log.Info("capture.main.HandleRecordStream(motiondetection): auto-stopping manual recording, no viewer heartbeat within timeout.")
 	} else {
-		log.Log.Info("capture.main.HandleRecordStream(motiondetection): auto-stopping manual recording, maximum duration reached.")
+		log.Info("capture.main.HandleRecordStream(motiondetection): auto-stopping manual recording, maximum duration reached.")
 	}
 	communication.IsRecordingManual.UnSet()
 	communication.RecordingManualHeartbeat.Store(0)
@@ -151,7 +155,7 @@ func manualRecordingExpired(communication *models.Communication, now int64) bool
 func CleanupRecordingDirectory(configDirectory string, configuration *models.Configuration) {
 	autoClean := configuration.Config.AutoClean
 	if autoClean != "true" {
-		log.Log.Info("HandleRecordStream: Autoclean disabled, nothing to do here.")
+		log.Info("HandleRecordStream: Autoclean disabled, nothing to do here.")
 		return
 	}
 
@@ -164,7 +168,7 @@ func CleanupRecordingDirectory(configDirectory string, configuration *models.Con
 	// disk while keeping a free-space reserve.
 	needsCleanup, err := recordingsNeedCleanup(recordingsDirectory, configuration)
 	if err != nil {
-		log.Log.Info("HandleRecordStream: something went wrong, " + err.Error())
+		log.Info("HandleRecordStream: something went wrong, " + err.Error())
 		return
 	}
 	if !needsCleanup {
@@ -185,12 +189,12 @@ func CleanupRecordingDirectory(configDirectory string, configuration *models.Con
 	// (otherwise a long outage would fill the disk and stop new recordings).
 	name, pending, err := pickRecordingToCleanup(recordingsDirectory, cloudDirectory)
 	if err != nil {
-		log.Log.Info("HandleRecordStream: something went wrong, " + err.Error())
+		log.Info("HandleRecordStream: something went wrong, " + err.Error())
 		return
 	}
 
-	if err := os.Remove(recordingsDirectory + "/" + name); err != nil {
-		log.Log.Info("HandleRecordStream: something went wrong, " + err.Error())
+	if err := utils.RemoveRecording(recordingsDirectory, name); err != nil {
+		log.Info("HandleRecordStream: something went wrong, " + err.Error())
 		return
 	}
 
@@ -200,14 +204,14 @@ func CleanupRecordingDirectory(configDirectory string, configuration *models.Con
 		// that was never uploaded to keep recording new footage. Also remove the
 		// now-dangling upload marker so the upload loop doesn't keep trying to
 		// upload a file that no longer exists.
-		log.Log.Warning("HandleRecordStream: removed oldest recording as part of cleanup, but it was STILL PENDING UPLOAD (disk full of un-uploaded recordings) - " + recordingsDirectory + "/" + name)
+		log.Warn("HandleRecordStream: removed oldest recording as part of cleanup, but it was STILL PENDING UPLOAD (disk full of un-uploaded recordings) - " + recordingsDirectory + "/" + name)
 		for _, markerName := range uploadMarkerNames(name) {
 			if err := os.Remove(filepath.Join(cloudDirectory, markerName)); err != nil && !os.IsNotExist(err) {
-				log.Log.Info("HandleRecordStream: could not remove dangling upload marker " + markerName + ", " + err.Error())
+				log.Info("HandleRecordStream: could not remove dangling upload marker " + markerName + ", " + err.Error())
 			}
 		}
 	} else {
-		log.Log.Info("HandleRecordStream: removed oldest file as part of cleanup - " + recordingsDirectory + "/" + name)
+		log.Info("HandleRecordStream: removed oldest file as part of cleanup - " + recordingsDirectory + "/" + name)
 	}
 }
 
@@ -274,7 +278,7 @@ func defaultReserveMB(totalMB int64) int64 {
 // overall with pending=true, signalling the caller that it is about to drop an
 // un-uploaded recording as a last resort.
 func pickRecordingToCleanup(recordingsDirectory, cloudDirectory string) (string, bool, error) {
-	entries, err := os.ReadDir(recordingsDirectory)
+	entries, err := utils.ReadRecordingDirectory(recordingsDirectory)
 	if err != nil {
 		return "", false, err
 	}
@@ -283,14 +287,7 @@ func pickRecordingToCleanup(recordingsDirectory, cloudDirectory string) (string,
 	var oldestSafeTime, oldestAnyTime time.Time
 
 	for _, entry := range entries {
-		if entry.IsDir() {
-			continue
-		}
-		info, err := entry.Info()
-		if err != nil || !info.Mode().IsRegular() {
-			continue
-		}
-		modTime := info.ModTime()
+		modTime := entry.ModTime()
 
 		if oldestAnyName == "" || modTime.Before(oldestAnyTime) {
 			oldestAnyName = entry.Name()
@@ -331,7 +328,7 @@ func recordingPendingUpload(cloudDirectory, recordingName string) bool {
 	return false
 }
 
-func HandleRecordStream(queue *packets.Queue, configDirectory string, configuration *models.Configuration, communication *models.Communication, rtspClient RTSPClient, mqttClient mqtt.Client) {
+func HandleRecordStream(queue *packets.Queue, configDirectory string, configuration *models.Configuration, communication *models.Communication, rtspClient RTSPClient, mqttClient mqtt.Client, motionEvents <-chan models.MotionDataPartial) {
 
 	config := configuration.Config
 	hubKey := config.HubKey
@@ -345,9 +342,9 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 	communication.RecordingManualHeartbeatSeen.UnSet()
 
 	if config.Capture.Recording == "false" {
-		log.Log.Info("capture.main.HandleRecordStream(): disabled, we will not record anything.")
+		log.Info("capture.main.HandleRecordStream(): disabled, we will not record anything.")
 	} else {
-		log.Log.Debug("capture.main.HandleRecordStream(): started")
+		log.Debug("capture.main.HandleRecordStream(): started")
 
 		preRecording := config.Capture.PreRecording * 1000
 		postRecording := config.Capture.PostRecording * 1000           // number of seconds to record.
@@ -360,7 +357,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 		}
 
 		if maxRecordingPeriod < preRecording+postRecording {
-			log.Log.Error("capture.main.HandleRecordStream(): maxRecordingPeriod is less than preRecording + postRecording, this is not allowed. Setting maxRecordingPeriod to preRecording + postRecording.")
+			log.Error("capture.main.HandleRecordStream(): maxRecordingPeriod is less than preRecording + postRecording, this is not allowed. Setting maxRecordingPeriod to preRecording + postRecording.")
 			maxRecordingPeriod = preRecording + postRecording
 		}
 
@@ -394,7 +391,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 			var name string
 
 			// Do not do anything!
-			log.Log.Info("capture.main.HandleRecordStream(continuous): start recording")
+			log.Info("capture.main.HandleRecordStream(continuous): start recording")
 
 			start := false
 
@@ -440,47 +437,15 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					}
 					if (videoCodec == "H264" && (len(mp4Video.SPSNALUs) == 0 || len(mp4Video.PPSNALUs) == 0)) ||
 						(videoCodec == "H265" && (len(mp4Video.VPSNALUs) == 0 || len(mp4Video.SPSNALUs) == 0 || len(mp4Video.PPSNALUs) == 0)) {
-						log.Log.Warning("capture.main.HandleRecordStream(continuous): closing MP4 without full parameter sets, moov may be incomplete")
+						log.Warn("capture.main.HandleRecordStream(continuous): closing MP4 without full parameter sets, moov may be incomplete")
 					}
 					mp4Video.Close(&config)
-					log.Log.Info("capture.main.HandleRecordStream(continuous): recording finished: file save: " + name)
+					log.Info("capture.main.HandleRecordStream(continuous): recording finished: file save: " + name)
 
 					// Cleanup muxer
 					start = false
 
-					// Update the name of the recording with the duration.
-					// We will update the name of the recording with the duration in milliseconds.
-					if mp4Video.VideoTotalDuration > 0 {
-						duration := mp4Video.VideoTotalDuration
-						// Update the name with the duration in milliseconds.
-						startRecordingSeconds := startRecording / 1000      // convert to seconds
-						startRecordingMilliseconds := startRecording % 1000 // convert to milliseconds
-						s := strconv.FormatInt(startRecordingSeconds, 10) + "_" +
-							strconv.Itoa(len(strconv.FormatInt(startRecordingMilliseconds, 10))) + "-" +
-							strconv.FormatInt(startRecordingMilliseconds, 10) + "_" +
-							config.Name + "_" +
-							"0-0-0-0" + "_" + // region coordinates, we
-							"-1" + "_" + // token
-							strconv.FormatInt(int64(duration), 10) // + "_" + // duration of recording
-							//utils.VERSION // version of the agent
-
-						oldName := name
-						name = s + ".mp4"
-						fullName = configDirectory + "/data/recordings/" + name
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
-
-						// Rename the file to the new name.
-						err := os.Rename(
-							configDirectory+"/data/recordings/"+oldName,
-							configDirectory+"/data/recordings/"+s+".mp4")
-
-						if err != nil {
-							log.Log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
-						}
-					} else {
-						log.Log.Info("capture.main.HandleRecordStream(continuous): no video data recorded, not renaming file.")
-					}
-
+					encrypted := false
 					// Check if we need to encrypt the recording.
 					if config.Encryption != nil && config.Encryption.Enabled == "true" && config.Encryption.Recordings == "true" && config.Encryption.SymmetricKey != "" {
 						// reopen file into memory 'fullName'
@@ -492,17 +457,19 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 								// write back to file
 								err := os.WriteFile(fullName, []byte(encryptedContents), 0644)
 								if err != nil {
-									log.Log.Error("capture.main.HandleRecordStream(continuous): error writing file: " + err.Error())
+									log.Error("capture.main.HandleRecordStream(continuous): error writing file: " + err.Error())
+								} else {
+									encrypted = true
 								}
 							} else {
-								log.Log.Error("capture.main.HandleRecordStream(continuous): error encrypting file: " + err.Error())
+								log.Error("capture.main.HandleRecordStream(continuous): error encrypting file: " + err.Error())
 							}
 						} else {
-							log.Log.Error("capture.main.HandleRecordStream(continuous): error reading file: " + err.Error())
+							log.Error("capture.main.HandleRecordStream(continuous): error reading file: " + err.Error())
 						}
 					}
 
-					queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, startRecording, mp4Video))
+					queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config, startRecording, mp4Video, encrypted, "0-0-0-0", -1))
 
 					recordingStatus = "idle"
 
@@ -520,39 +487,27 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					// We'll validate those conditions and if not valid we'll not do anything.
 					valid, err := conditions.Validate(loc, configuration)
 					if !valid && err != nil {
-						log.Log.Debug("capture.main.HandleRecordStream(continuous): " + err.Error() + ".")
+						log.Debug("capture.main.HandleRecordStream(continuous): " + err.Error() + ".")
 						time.Sleep(5 * time.Second)
 						continue
 					}
 
 					start = true
 
-					// timestamp_microseconds_instanceName_regionCoordinates_numberOfChanges_token
-					// 1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4
-					// - Timestamp
-					// - Size + - + microseconds
-					// - device
-					// - Region
-					// - Number of changes
-					// - Token
-
 					startRecording = pkt.CurrentTime
-					startRecordingSeconds := startRecording / 1000            // convert to seconds
-					startRecordingMilliseconds := startRecording % 1000       // convert to milliseconds
-					s := strconv.FormatInt(startRecordingSeconds, 10) + "_" + // start timestamp in seconds
-						strconv.Itoa(len(strconv.FormatInt(startRecordingMilliseconds, 10))) + "-" + // length of milliseconds
-						strconv.FormatInt(startRecordingMilliseconds, 10) + "_" + // milliseconds
-						config.Name + "_" + // device name
-						"0-0-0-0" + "_" + // region coordinates, we will not use this for continuous recording
-						"0" + "_" + // token
-						"0" + "_" //+ // duration of recording in milliseconds
-					//utils.VERSION // version of the agent
-
-					name = s + ".mp4"
-					fullName = configDirectory + "/data/recordings/" + name
+					name, err = utils.RecordingFileName(startRecording, config.Key)
+					if err == nil {
+						fullName, err = utils.CreateRecordingPath(configDirectory+"/data/recordings", name, time.UnixMilli(startRecording))
+					}
+					if err != nil {
+						log.WithError(err).Error("capture: could not create continuous recording path")
+						start = false
+						pkt = nextPkt
+						continue
+					}
 
 					// Running...
-					log.Log.Info("capture.main.HandleRecordStream(continuous): recording started")
+					log.Info("capture.main.HandleRecordStream(continuous): recording started")
 
 					// Get width and height from the camera.
 					width := configuration.Config.Capture.IPCamera.Width
@@ -564,7 +519,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					vpsNALUS := configuration.Config.Capture.IPCamera.VPSNALUs
 
 					if len(spsNALUS) == 0 || len(ppsNALUS) == 0 {
-						log.Log.Warning("capture.main.HandleRecordStream(continuous): missing SPS/PPS at recording start")
+						log.Warn("capture.main.HandleRecordStream(continuous): missing SPS/PPS at recording start")
 					}
 					// Create a video file, and set the dimensions.
 					mp4Video = video.NewMP4(fullName, spsNALUS, ppsNALUS, vpsNALUS, configuration.Config.Capture.MaxLengthRecording)
@@ -579,7 +534,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					if audioCodec == "AAC" {
 						audioTrack = mp4Video.AddAudioTrack("AAC")
 					} else if audioCodec == "PCM_MULAW" {
-						log.Log.Debug("capture.main.HandleRecordStream(continuous): no AAC audio codec detected, skipping audio track.")
+						log.Debug("capture.main.HandleRecordStream(continuous): no AAC audio codec detected, skipping audio track.")
 					}
 
 					writeSampleToMP4(mp4Video, videoTrack, audioTrack, pkt)
@@ -600,44 +555,12 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 			if cursorError != nil {
 				if recordingStatus == "started" {
 
-					log.Log.Info("capture.main.HandleRecordStream(continuous): Recording finished: file save: " + name)
+					log.Info("capture.main.HandleRecordStream(continuous): Recording finished: file save: " + name)
 
 					// Cleanup muxer
 					start = false
 
-					// Update the name of the recording with the duration.
-					// We will update the name of the recording with the duration in milliseconds.
-					if mp4Video.VideoTotalDuration > 0 {
-						duration := mp4Video.VideoTotalDuration
-						// Update the name with the duration in milliseconds.
-						startRecordingSeconds := startRecording / 1000      // convert to seconds
-						startRecordingMilliseconds := startRecording % 1000 // convert to milliseconds
-						s := strconv.FormatInt(startRecordingSeconds, 10) + "_" +
-							strconv.Itoa(len(strconv.FormatInt(startRecordingMilliseconds, 10))) + "-" +
-							strconv.FormatInt(startRecordingMilliseconds, 10) + "_" +
-							config.Name + "_" +
-							"0-0-0-0" + "_" + // region coordinates, we
-							"-1" + "_" + // token
-							strconv.FormatInt(int64(duration), 10) // + "_" + // duration of recording
-							//utils.VERSION // version of the agent
-
-						oldName := name
-						name = s + ".mp4"
-						fullName = configDirectory + "/data/recordings/" + name
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
-
-						// Rename the file to the new name.
-						err := os.Rename(
-							configDirectory+"/data/recordings/"+oldName,
-							configDirectory+"/data/recordings/"+s+".mp4")
-
-						if err != nil {
-							log.Log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
-						}
-					} else {
-						log.Log.Info("capture.main.HandleRecordStream(continuous): no video data recorded, not renaming file.")
-					}
-
+					encrypted := false
 					// Check if we need to encrypt the recording.
 					if config.Encryption != nil && config.Encryption.Enabled == "true" && config.Encryption.Recordings == "true" && config.Encryption.SymmetricKey != "" {
 						// reopen file into memory 'fullName'
@@ -649,17 +572,19 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 								// write back to file
 								err := os.WriteFile(fullName, []byte(encryptedContents), 0644)
 								if err != nil {
-									log.Log.Error("capture.main.HandleRecordStream(motiondetection): error writing file: " + err.Error())
+									log.Error("capture.main.HandleRecordStream(motiondetection): error writing file: " + err.Error())
+								} else {
+									encrypted = true
 								}
 							} else {
-								log.Log.Error("capture.main.HandleRecordStream(motiondetection): error encrypting file: " + err.Error())
+								log.Error("capture.main.HandleRecordStream(motiondetection): error encrypting file: " + err.Error())
 							}
 						} else {
-							log.Log.Error("capture.main.HandleRecordStream(motiondetection): error reading file: " + err.Error())
+							log.Error("capture.main.HandleRecordStream(motiondetection): error reading file: " + err.Error())
 						}
 					}
 
-					queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, startRecording, mp4Video))
+					queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config, startRecording, mp4Video, encrypted, "0-0-0-0", -1))
 
 					recordingStatus = "idle"
 
@@ -672,7 +597,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 			}
 		} else {
 
-			log.Log.Info("capture.main.HandleRecordStream(motiondetection): Start motion based recording ")
+			log.Info("capture.main.HandleRecordStream(motiondetection): Start motion based recording ")
 
 			var lastRecordingTime int64 = 0 // last recording timestamp in milliseconds
 			var displayTime int64 = 0       // display time in milliseconds
@@ -680,7 +605,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 			var videoTrack uint32
 			var audioTrack uint32
 
-			for motion := range communication.HandleMotion {
+			for motion := range motionEvents {
 
 				// Get as much packets we need.
 				var cursorError error
@@ -710,21 +635,11 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 
 				// If startRecording is 0, we will continue as it might be we are in a state of restarting the agent.
 				if startRecording == 0 {
-					log.Log.Info("capture.main.HandleRecordStream(motiondetection): startRecording is 0, we will continue as it might be we are in a state of restarting the agent.")
+					log.Info("capture.main.HandleRecordStream(motiondetection): startRecording is 0, we will continue as it might be we are in a state of restarting the agent.")
 					continue
 				}
 
-				// timestamp_microseconds_instanceName_regionCoordinates_numberOfChanges_token
-				// 1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4
-				// - Timestamp
-				// - Size + - + microseconds
-				// - device
-				// - Region
-				// - Number of changes
-				// - Token
-
-				displayTimeSeconds := displayTime / 1000      // convert to seconds
-				displayTimeMilliseconds := displayTime % 1000 // convert to milliseconds
+				displayTimeSeconds := displayTime / 1000
 				motionRectangleString := "0-0-0-0"
 				if motion.Rectangle.X != 0 || motion.Rectangle.Y != 0 ||
 					motion.Rectangle.Width != 0 || motion.Rectangle.Height != 0 {
@@ -735,20 +650,18 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 				// Get the number of changes from the motion detection.
 				numberOfChanges := motion.NumberOfChanges
 
-				s := strconv.FormatInt(displayTimeSeconds, 10) + "_" + // start timestamp in seconds
-					strconv.Itoa(len(strconv.FormatInt(displayTimeMilliseconds, 10))) + "-" + // length of milliseconds
-					strconv.FormatInt(displayTimeMilliseconds, 10) + "_" + // milliseconds
-					config.Name + "_" + // device name
-					motionRectangleString + "_" + // region coordinates, we will not use this for continuous recording
-					strconv.Itoa(numberOfChanges) + "_" + // number of changes
-					"0" // + "_" + // duration of recording in milliseconds
-					//utils.VERSION // version of the agent
-
-				name := s + ".mp4"
-				fullName := configDirectory + "/data/recordings/" + name
+				name, err := utils.RecordingFileName(displayTime, config.Key)
+				var fullName string
+				if err == nil {
+					fullName, err = utils.CreateRecordingPath(configDirectory+"/data/recordings", name, time.UnixMilli(displayTime))
+				}
+				if err != nil {
+					log.WithError(err).Error("capture: could not create motion recording path")
+					continue
+				}
 
 				// Running...
-				log.Log.Info("capture.main.HandleRecordStream(motiondetection): recording started (" + name + ")" + " at " + strconv.FormatInt(displayTimeSeconds, 10) + " unix")
+				log.Info("capture.main.HandleRecordStream(motiondetection): recording started (" + name + ")" + " at " + strconv.FormatInt(displayTimeSeconds, 10) + " unix")
 
 				// Get width and height from the camera.
 				width := configuration.Config.Capture.IPCamera.Width
@@ -760,7 +673,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 				vpsNALUS := configuration.Config.Capture.IPCamera.VPSNALUs
 
 				if len(spsNALUS) == 0 || len(ppsNALUS) == 0 {
-					log.Log.Warning("capture.main.HandleRecordStream(motiondetection): missing SPS/PPS at recording start")
+					log.Warn("capture.main.HandleRecordStream(motiondetection): missing SPS/PPS at recording start")
 				}
 				// Create the MP4 only once the first keyframe arrives.
 				var mp4Video *video.MP4
@@ -769,16 +682,16 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 
 					nextPkt, cursorError = recordingCursor.ReadPacket()
 					if cursorError != nil {
-						log.Log.Error("capture.main.HandleRecordStream(motiondetection): " + cursorError.Error())
+						log.Error("capture.main.HandleRecordStream(motiondetection): " + cursorError.Error())
 					}
 
 					now = time.Now().UnixMilli()
 					select {
-					case motion := <-communication.HandleMotion:
+					case motion := <-motionEvents:
 						motionTimestamp = now
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): motion detected while recording. Expanding recording.")
+						log.Info("capture.main.HandleRecordStream(motiondetection): motion detected while recording. Expanding recording.")
 						numberOfChanges := motion.NumberOfChanges
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): Received message with recording data, detected changes to save: " + strconv.Itoa(numberOfChanges))
+						log.Info("capture.main.HandleRecordStream(motiondetection): Received message with recording data, detected changes to save: " + strconv.Itoa(numberOfChanges))
 					default:
 					}
 
@@ -794,16 +707,16 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					}
 
 					if start && (motionTimestamp+postRecording-now < 0 || now-startRecording > maxRecordingPeriod-500) && nextPkt.IsKeyFrame {
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): timestamp+postRecording-now < 0  - " + strconv.FormatInt(motionTimestamp+postRecording-now, 10) + " < 0")
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): now-startRecording > maxRecordingPeriod-500 - " + strconv.FormatInt(now-startRecording, 10) + " > " + strconv.FormatInt(maxRecordingPeriod-500, 10))
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): closing recording (timestamp: " + strconv.FormatInt(motionTimestamp, 10) + ", postRecording: " + strconv.FormatInt(postRecording, 10) + ", now: " + strconv.FormatInt(now, 10) + ", startRecording: " + strconv.FormatInt(startRecording, 10) + ", maxRecordingPeriod: " + strconv.FormatInt(maxRecordingPeriod, 10))
+						log.Info("capture.main.HandleRecordStream(motiondetection): timestamp+postRecording-now < 0  - " + strconv.FormatInt(motionTimestamp+postRecording-now, 10) + " < 0")
+						log.Info("capture.main.HandleRecordStream(motiondetection): now-startRecording > maxRecordingPeriod-500 - " + strconv.FormatInt(now-startRecording, 10) + " > " + strconv.FormatInt(maxRecordingPeriod-500, 10))
+						log.Info("capture.main.HandleRecordStream(motiondetection): closing recording (timestamp: " + strconv.FormatInt(motionTimestamp, 10) + ", postRecording: " + strconv.FormatInt(postRecording, 10) + ", now: " + strconv.FormatInt(now, 10) + ", startRecording: " + strconv.FormatInt(startRecording, 10) + ", maxRecordingPeriod: " + strconv.FormatInt(maxRecordingPeriod, 10))
 						break
 					}
 					if pkt.IsKeyFrame && !start && pkt.CurrentTime >= startRecording {
 						// We start the recording if we have a keyframe and the last duration is 0 or less than the current packet time.
 						// It could be start we start from the beginning of the recording.
-						log.Log.Debug("capture.main.HandleRecordStream(motiondetection): write frames")
-						log.Log.Debug("capture.main.HandleRecordStream(motiondetection): recording started on keyframe")
+						log.Debug("capture.main.HandleRecordStream(motiondetection): write frames")
+						log.Debug("capture.main.HandleRecordStream(motiondetection): recording started on keyframe")
 
 						// Align duration timers with the first keyframe.
 						startRecording = pkt.CurrentTime
@@ -821,7 +734,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 						if audioCodec == "AAC" {
 							audioTrack = mp4Video.AddAudioTrack("AAC")
 						} else if audioCodec == "PCM_MULAW" {
-							log.Log.Debug("capture.main.HandleRecordStream(continuous): no AAC audio codec detected, skipping audio track.")
+							log.Debug("capture.main.HandleRecordStream(continuous): no AAC audio codec detected, skipping audio track.")
 						}
 						start = true
 
@@ -840,7 +753,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 				lastRecordingTime = pkt.CurrentTime
 
 				if mp4Video == nil {
-					log.Log.Warning("capture.main.HandleRecordStream(motiondetection): recording closed without keyframe; no MP4 created")
+					log.Warn("capture.main.HandleRecordStream(motiondetection): recording closed without keyframe; no MP4 created")
 					continue
 				}
 
@@ -856,10 +769,10 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 				}
 				if (videoCodec == "H264" && (len(mp4Video.SPSNALUs) == 0 || len(mp4Video.PPSNALUs) == 0)) ||
 					(videoCodec == "H265" && (len(mp4Video.VPSNALUs) == 0 || len(mp4Video.SPSNALUs) == 0 || len(mp4Video.PPSNALUs) == 0)) {
-					log.Log.Warning("capture.main.HandleRecordStream(motiondetection): closing MP4 without full parameter sets, moov may be incomplete")
+					log.Warn("capture.main.HandleRecordStream(motiondetection): closing MP4 without full parameter sets, moov may be incomplete")
 				}
 				mp4Video.Close(&config)
-				log.Log.Info("capture.main.HandleRecordStream(motiondetection): file save: " + name)
+				log.Info("capture.main.HandleRecordStream(motiondetection): file save: " + name)
 
 				// Notify the hub / live-view UI that this camera stopped recording.
 				publishRecordingState(mqttClient, hubKey, configuration, false)
@@ -870,44 +783,10 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 				// restart when the recording has expired (heartbeat lapsed or max
 				// duration reached), so it ends here instead of recording forever.
 				if communication.IsRecordingManual.IsSet() && !manualRecordingExpired(communication, time.Now().UnixMilli()) {
-					select {
-					case communication.HandleMotion <- models.MotionDataPartial{Timestamp: time.Now().Unix(), NumberOfChanges: 100000000}:
-					default:
-					}
-				}
-
-				// Update the name of the recording with the duration.
-				// We will update the name of the recording with the duration in milliseconds.
-				if mp4Video.VideoTotalDuration > 0 {
-					duration := mp4Video.VideoTotalDuration
-
-					// Update the name with the duration in milliseconds.
-					s := strconv.FormatInt(displayTimeSeconds, 10) + "_" +
-						strconv.Itoa(len(strconv.FormatInt(displayTimeMilliseconds, 10))) + "-" +
-						strconv.FormatInt(displayTimeMilliseconds, 10) + "_" +
-						config.Name + "_" +
-						motionRectangleString + "_" +
-						strconv.Itoa(numberOfChanges) + "_" + // number of changes
-						strconv.FormatInt(int64(duration), 10) // + "_" + // duration of recording in milliseconds
-						//utils.VERSION // version of the agent
-
-					oldName := name
-					name = s + ".mp4"
-					fullName = configDirectory + "/data/recordings/" + name
-					log.Log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
-
-					// Rename the file to the new name.
-					err := os.Rename(
-						configDirectory+"/data/recordings/"+oldName,
-						configDirectory+"/data/recordings/"+s+".mp4")
-
-					if err != nil {
-						log.Log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
-					}
-				} else {
-					log.Log.Info("capture.main.HandleRecordStream(motiondetection): no video data recorded, not renaming file.")
+					communication.TrySendMotion(models.MotionDataPartial{Timestamp: time.Now().Unix(), NumberOfChanges: 100000000})
 				}
 
+				encrypted := false
 				// Check if we need to encrypt the recording.
 				if config.Encryption != nil && config.Encryption.Enabled == "true" && config.Encryption.Recordings == "true" && config.Encryption.SymmetricKey != "" {
 					// reopen file into memory 'fullName'
@@ -919,24 +798,26 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 							// write back to file
 							err := os.WriteFile(fullName, []byte(encryptedContents), 0644)
 							if err != nil {
-								log.Log.Error("capture.main.HandleRecordStream(motiondetection): error writing file: " + err.Error())
+								log.Error("capture.main.HandleRecordStream(motiondetection): error writing file: " + err.Error())
+							} else {
+								encrypted = true
 							}
 						} else {
-							log.Log.Error("capture.main.HandleRecordStream(motiondetection): error encrypting file: " + err.Error())
+							log.Error("capture.main.HandleRecordStream(motiondetection): error encrypting file: " + err.Error())
 						}
 					} else {
-						log.Log.Error("capture.main.HandleRecordStream(motiondetection): error reading file: " + err.Error())
+						log.Error("capture.main.HandleRecordStream(motiondetection): error reading file: " + err.Error())
 					}
 				}
 
-				queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, displayTime, mp4Video))
+				queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config, displayTime, mp4Video, encrypted, motionRectangleString, numberOfChanges))
 
 				// Clean up the recording directory if necessary.
 				CleanupRecordingDirectory(configDirectory, configuration)
 			}
 		}
 
-		log.Log.Debug("capture.main.HandleRecordStream(): finished")
+		log.Debug("capture.main.HandleRecordStream(): finished")
 	}
 }
 
@@ -1030,13 +911,14 @@ func Base64Image(captureDevice *Capture, communication *models.Communication, co
 	var cursor *packets.QueueCursor
 
 	// We'll pick the right client and decoder.
-	rtspClient := captureDevice.RTSPSubClient
+	rtspClient := captureDevice.SubClient()
 	if rtspClient != nil {
-		queue = communication.SubQueue
-		cursor = queue.Latest()
+		queue = communication.SubQueue()
 	} else {
-		rtspClient = captureDevice.RTSPClient
-		queue = communication.Queue
+		rtspClient = captureDevice.MainClient()
+		queue = communication.MainQueue()
+	}
+	if queue != nil {
 		cursor = queue.Latest()
 	}
 
@@ -1062,6 +944,10 @@ func Base64Image(captureDevice *Capture, communication *models.Communication, co
 					count++
 					continue
 				}
+			} else {
+				// A stopped run closes this queue permanently. Retrying its
+				// cursor would spin on EOF even after the HTTP client leaves.
+				break
 			}
 		} else {
 			break
@@ -1076,13 +962,14 @@ func JpegImage(captureDevice *Capture, communication *models.Communication) imag
 	var cursor *packets.QueueCursor
 
 	// We'll pick the right client and decoder.
-	rtspClient := captureDevice.RTSPSubClient
+	rtspClient := captureDevice.SubClient()
 	if rtspClient != nil {
-		queue = communication.SubQueue
-		cursor = queue.Latest()
+		queue = communication.SubQueue()
 	} else {
-		rtspClient = captureDevice.RTSPClient
-		queue = communication.Queue
+		rtspClient = captureDevice.MainClient()
+		queue = communication.MainQueue()
+	}
+	if queue != nil {
 		cursor = queue.Latest()
 	}
 
@@ -1104,6 +991,10 @@ func JpegImage(captureDevice *Capture, communication *models.Communication) imag
 				} else {
 					break
 				}
+			} else {
+				// A stopped run closes this queue permanently. Retrying its
+				// cursor would spin on EOF even after the HTTP client leaves.
+				break
 			}
 		} else {
 			break
@@ -1137,20 +1028,16 @@ func writeSampleToMP4(mp4Video *video.MP4, videoTrack, audioTrack uint32, pkt pa
 			dts = pts - uint64(compositionOffset)
 		}
 		if err := mp4Video.AddSampleToTrack(videoTrack, pkt.IsKeyFrame, pkt.Data, dts, compositionOffset); err != nil {
-			log.Log.Error("capture.main.writeSampleToMP4(): " + err.Error())
+			log.Error("capture.main.writeSampleToMP4(): " + err.Error())
 		}
 	} else if pkt.IsAudio {
 		if pkt.Codec == "AAC" {
 			if err := mp4Video.AddSampleToTrack(audioTrack, pkt.IsKeyFrame, pkt.Data, pts, 0); err != nil {
-				log.Log.Error("capture.main.writeSampleToMP4(): " + err.Error())
+				log.Error("capture.main.writeSampleToMP4(): " + err.Error())
 			}
 		} else if pkt.Codec == "PCM_MULAW" {
 			// TODO: transcode to AAC, some work to do..
-			log.Log.Debug("capture.main.writeSampleToMP4(): no AAC audio codec detected, skipping audio track.")
+			log.Debug("capture.main.writeSampleToMP4(): no AAC audio codec detected, skipping audio track.")
 		}
 	}
 }
-
-/*func convertPTS2(v int64) uint64 {
-	return uint64(v) / 100
-}*/

machinery/src/capture/main_test.go

@@ -42,10 +42,11 @@ func TestQueueRecordingForUploadStoresFinalizedMetadata(t *testing.T) {
 	}
 
 	mp4Video := &video.MP4{VideoTotalDuration: 20452, SampleCount: 613}
-	metadata := recordingUploadMetadata("recording.mp4", "device-key", 1785934709414, mp4Video)
+	config := models.Config{Key: "device-key", Name: "camera-name"}
+	metadata := recordingUploadMetadata("1785934709414-device-key.mp4", config, 1785934709414, mp4Video, true, "1-2-3-4", 57)
 	queueRecordingForUpload(configDirectory, metadata)
 
-	got, err := os.ReadFile(filepath.Join(configDirectory, "data", "cloud", "recording.metadata"))
+	got, err := os.ReadFile(filepath.Join(configDirectory, "data", "cloud", "1785934709414-device-key.metadata"))
 	if err != nil {
 		t.Fatalf("read upload marker: %v", err)
 	}
@@ -54,7 +55,8 @@ func TestQueueRecordingForUploadStoresFinalizedMetadata(t *testing.T) {
 		t.Fatalf("decode upload marker: %v", err)
 	}
 	expectedFPS := mp4Video.AverageFPS()
-	if stored.FileName != "recording.mp4" || stored.DeviceKey != "device-key" || stored.Timestamp != 1785934709414 || stored.Duration != 20452 || math.Abs(stored.FPS-expectedFPS) > 1e-9 {
+	if stored.FileName != "1785934709414-device-key.mp4" || stored.DeviceKey != "device-key" || stored.Timestamp != 1785934709414 || stored.Duration != 20452 || math.Abs(stored.FPS-expectedFPS) > 1e-9 || !stored.Encrypted ||
+		stored.DeviceName != "camera-name" || stored.RegionCoordinates != "1-2-3-4" || stored.NumberOfChanges != "57" {
 		t.Fatalf("upload marker = %+v", stored)
 	}
 	if stored.FPS == math.Floor(stored.FPS) {

machinery/src/capture/rtsp_client.go

@@ -3,36 +3,73 @@ package capture
 import (
 	"context"
 	"image"
+	"sync"
 
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 )
 
 type Capture struct {
-	RTSPClient            *Golibrtsp
-	RTSPSubClient         *Golibrtsp
-	RTSPBackChannelClient *Golibrtsp
+	clientsMu             sync.RWMutex
+	rtspClient            *Golibrtsp
+	rtspSubClient         *Golibrtsp
+	rtspBackChannelClient *Golibrtsp
 }
 
 func (c *Capture) SetMainClient(rtspUrl string) *Golibrtsp {
-	c.RTSPClient = &Golibrtsp{
+	client := &Golibrtsp{
 		Url: rtspUrl,
 	}
-	return c.RTSPClient
+	c.clientsMu.Lock()
+	c.rtspClient = client
+	c.clientsMu.Unlock()
+	return client
 }
 
 func (c *Capture) SetSubClient(rtspUrl string) *Golibrtsp {
-	c.RTSPSubClient = &Golibrtsp{
+	client := &Golibrtsp{
 		Url: rtspUrl,
 	}
-	return c.RTSPSubClient
+	c.clientsMu.Lock()
+	c.rtspSubClient = client
+	c.clientsMu.Unlock()
+	return client
 }
 
 func (c *Capture) SetBackChannelClient(rtspUrl string) *Golibrtsp {
-	c.RTSPBackChannelClient = &Golibrtsp{
+	client := &Golibrtsp{
 		Url: rtspUrl,
 	}
-	return c.RTSPBackChannelClient
+	c.clientsMu.Lock()
+	c.rtspBackChannelClient = client
+	c.clientsMu.Unlock()
+	return client
+}
+
+func (c *Capture) MainClient() *Golibrtsp {
+	c.clientsMu.RLock()
+	defer c.clientsMu.RUnlock()
+	return c.rtspClient
+}
+
+func (c *Capture) SubClient() *Golibrtsp {
+	c.clientsMu.RLock()
+	defer c.clientsMu.RUnlock()
+	return c.rtspSubClient
+}
+
+func (c *Capture) ClearClients(main, sub, backchannel *Golibrtsp) {
+	c.clientsMu.Lock()
+	defer c.clientsMu.Unlock()
+	if c.rtspClient == main {
+		c.rtspClient = nil
+	}
+	if c.rtspSubClient == sub {
+		c.rtspSubClient = nil
+	}
+	if c.rtspBackChannelClient == backchannel {
+		c.rtspBackChannelClient = nil
+	}
 }
 
 // RTSPClient is a interface that abstracts the RTSP client implementation.

machinery/src/capture/rtsp_client_test.go

@@ -0,0 +1,62 @@
+package capture
+
+import (
+	"strconv"
+	"sync"
+	"testing"
+)
+
+func TestCaptureClientAccessCanRaceReplacement(t *testing.T) {
+	captureDevice := &Capture{}
+	captureDevice.SetMainClient("rtsp://main/0")
+	captureDevice.SetSubClient("rtsp://sub/0")
+
+	var workers sync.WaitGroup
+	workers.Add(2)
+	go func() {
+		defer workers.Done()
+		for replacement := 1; replacement <= 1000; replacement++ {
+			suffix := strconv.Itoa(replacement)
+			captureDevice.SetMainClient("rtsp://main/" + suffix)
+			captureDevice.SetSubClient("rtsp://sub/" + suffix)
+		}
+	}()
+	go func() {
+		defer workers.Done()
+		for snapshot := 0; snapshot < 1000; snapshot++ {
+			if captureDevice.MainClient() == nil {
+				t.Error("MainClient() returned nil")
+				return
+			}
+			if captureDevice.SubClient() == nil {
+				t.Error("SubClient() returned nil")
+				return
+			}
+		}
+	}()
+	workers.Wait()
+}
+
+func TestCaptureClearClientsOnlyClearsMatchingRun(t *testing.T) {
+	captureDevice := &Capture{}
+	oldMain := captureDevice.SetMainClient("rtsp://main/old")
+	oldSub := captureDevice.SetSubClient("rtsp://sub/old")
+	oldBackchannel := captureDevice.SetBackChannelClient("rtsp://back/old")
+
+	newMain := captureDevice.SetMainClient("rtsp://main/new")
+	newSub := captureDevice.SetSubClient("rtsp://sub/new")
+	newBackchannel := captureDevice.SetBackChannelClient("rtsp://back/new")
+
+	captureDevice.ClearClients(oldMain, oldSub, oldBackchannel)
+	if captureDevice.MainClient() != newMain {
+		t.Fatal("stale cleanup cleared the new main client")
+	}
+	if captureDevice.SubClient() != newSub {
+		t.Fatal("stale cleanup cleared the new sub client")
+	}
+
+	captureDevice.ClearClients(newMain, newSub, newBackchannel)
+	if captureDevice.MainClient() != nil || captureDevice.SubClient() != nil {
+		t.Fatal("matching cleanup did not clear current clients")
+	}
+}

machinery/src/capture/snapshot_test.go

@@ -0,0 +1,68 @@
+package capture
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/packets"
+)
+
+func TestSnapshotReturnsAfterQueueClosure(t *testing.T) {
+	for _, stream := range []string{"main", "sub"} {
+		for _, format := range []string{"jpeg", "base64"} {
+			for _, alreadyClosed := range []bool{true, false} {
+				name := stream + "/" + format + "/closed_during_read"
+				if alreadyClosed {
+					name = stream + "/" + format + "/already_closed"
+				}
+				t.Run(name, func(t *testing.T) {
+					communication := &models.Communication{}
+					run := models.NewAgentRun(context.Background(), communication, false)
+					queue := packets.NewQueue()
+					captureDevice := &Capture{}
+					if stream == "main" {
+						captureDevice.SetMainClient("rtsp://unused.invalid/main")
+						run.SetMainQueue(queue)
+					} else {
+						captureDevice.SetSubClient("rtsp://unused.invalid/sub")
+						run.SetSubQueue(queue)
+					}
+					if err := run.Activate(); err != nil {
+						t.Fatal(err)
+					}
+					defer queue.Close()
+					if alreadyClosed {
+						queue.Close()
+					}
+					done := make(chan bool, 1)
+					go func() {
+						if format == "jpeg" {
+							image := JpegImage(captureDevice, communication)
+							done <- image.Rect.Empty()
+						} else {
+							done <- Base64Image(captureDevice, communication, &models.Configuration{}) == ""
+						}
+					}()
+					if !alreadyClosed {
+						select {
+						case <-done:
+							t.Fatal("snapshot returned before a frame or queue closure")
+						case <-time.After(10 * time.Millisecond):
+						}
+						queue.Close()
+					}
+					select {
+					case empty := <-done:
+						if !empty {
+							t.Fatal("closed empty queue produced a snapshot")
+						}
+					case <-time.After(time.Second):
+						t.Fatal("snapshot did not return after its packet queue closed")
+					}
+				})
+			}
+		}
+	}
+}

machinery/src/cloud/cloud_test.go

@@ -0,0 +1,849 @@
+package cloud
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/kerberos-io/agent/machinery/src/models"
+	agentonvif "github.com/kerberos-io/agent/machinery/src/onvif"
+	goonvif "github.com/kerberos-io/onvif"
+	goonvifdevice "github.com/kerberos-io/onvif/device"
+	goonvifptz "github.com/kerberos-io/onvif/ptz"
+	goonvifxsd "github.com/kerberos-io/onvif/xsd/onvif"
+)
+
+func TestVerifyPersistenceKerberosHubUsesAuthorizationHead(t *testing.T) {
+	var requestMethod string
+	var requestBody []byte
+	var requestHeaders http.Header
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requestMethod = r.Method
+		requestHeaders = r.Header.Clone()
+		requestBody, _ = io.ReadAll(r.Body)
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer server.Close()
+
+	config := models.Config{
+		Cloud:         "kerberoshub",
+		Key:           "camera-1",
+		Name:          "front-door",
+		HubURI:        server.URL,
+		HubKey:        "project-public",
+		HubPrivateKey: "project-private",
+		S3:            &models.S3{Region: "eu"},
+	}
+	payload, err := json.Marshal(config)
+	if err != nil {
+		t.Fatalf("json.Marshal() error = %v", err)
+	}
+
+	gin.SetMode(gin.TestMode)
+	recorder := httptest.NewRecorder()
+	ctx, _ := gin.CreateTestContext(recorder)
+	ctx.Request = httptest.NewRequest(http.MethodPost, "/api/persistence/verify", bytes.NewReader(payload))
+	ctx.Request.Header.Set("Content-Type", "application/json")
+
+	VerifyPersistence(ctx, t.TempDir(), server.URL)
+
+	if recorder.Code != http.StatusOK {
+		t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String())
+	}
+	if requestMethod != http.MethodHead {
+		t.Fatalf("Hub request method = %q, want HEAD", requestMethod)
+	}
+	if len(requestBody) != 0 {
+		t.Fatalf("Hub request body has %d bytes, want none", len(requestBody))
+	}
+	for header, want := range map[string]string{
+		"X-Kerberos-Storage-Device":  "camera-1",
+		"X-Kerberos-Hub-PublicKey":   "project-public",
+		"X-Kerberos-Hub-PrivateKey":  "project-private",
+		"X-Kerberos-Hub-Region":      "eu",
+		"X-Kerberos-Storage-Capture": "IPCamera",
+	} {
+		if got := requestHeaders.Get(header); got != want {
+			t.Errorf("%s = %q, want %q", header, got, want)
+		}
+	}
+}
+
+func TestVerifyPersistenceKerberosHubRejectsCrossHostRedirect(t *testing.T) {
+	var redirectedHeaders http.Header
+	target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		redirectedHeaders = r.Header.Clone()
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer target.Close()
+
+	var originalHeaders http.Header
+	hub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		originalHeaders = r.Header.Clone()
+		http.Redirect(w, r, target.URL+"/storage/upload", http.StatusTemporaryRedirect)
+	}))
+	defer hub.Close()
+
+	config := models.Config{
+		Cloud:         "kerberoshub",
+		Key:           "camera-1",
+		Name:          "front-door",
+		HubURI:        hub.URL,
+		HubKey:        "project-public",
+		HubPrivateKey: "project-private",
+		S3:            &models.S3{Region: "eu"},
+	}
+	payload, err := json.Marshal(config)
+	if err != nil {
+		t.Fatalf("json.Marshal() error = %v", err)
+	}
+
+	gin.SetMode(gin.TestMode)
+	recorder := httptest.NewRecorder()
+	ctx, _ := gin.CreateTestContext(recorder)
+	ctx.Request = httptest.NewRequest(http.MethodPost, "/api/persistence/verify", bytes.NewReader(payload))
+	ctx.Request.Header.Set("Content-Type", "application/json")
+
+	VerifyPersistence(ctx, t.TempDir(), hub.URL)
+
+	if recorder.Code != http.StatusBadRequest {
+		t.Fatalf("status = %d, want %d; body = %s", recorder.Code, http.StatusBadRequest, recorder.Body.String())
+	}
+	for _, header := range []string{"X-Kerberos-Hub-PublicKey", "X-Kerberos-Hub-PrivateKey"} {
+		if originalHeaders.Get(header) == "" {
+			t.Errorf("%s was not sent to the configured Hub", header)
+		}
+		if redirectedHeaders != nil {
+			t.Errorf("redirect target received a request with %s=%q", header, redirectedHeaders.Get(header))
+		}
+	}
+}
+
+func TestVerifyPersistenceKerberosHubRejectsUnsavedURI(t *testing.T) {
+	requestCount := 0
+	untrusted := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		requestCount++
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer untrusted.Close()
+
+	config := models.Config{
+		Cloud:         "kerberoshub",
+		Key:           "camera-1",
+		Name:          "front-door",
+		HubURI:        untrusted.URL,
+		HubKey:        "project-public",
+		HubPrivateKey: "project-private",
+		S3:            &models.S3{Region: "eu"},
+	}
+	payload, err := json.Marshal(config)
+	if err != nil {
+		t.Fatalf("json.Marshal() error = %v", err)
+	}
+
+	gin.SetMode(gin.TestMode)
+	recorder := httptest.NewRecorder()
+	ctx, _ := gin.CreateTestContext(recorder)
+	ctx.Request = httptest.NewRequest(http.MethodPost, "/api/persistence/verify", bytes.NewReader(payload))
+	ctx.Request.Header.Set("Content-Type", "application/json")
+
+	VerifyPersistence(ctx, t.TempDir(), "https://saved-hub.example.com")
+
+	if recorder.Code != http.StatusBadRequest {
+		t.Fatalf("status = %d, want %d; body = %s", recorder.Code, http.StatusBadRequest, recorder.Body.String())
+	}
+	if requestCount != 0 {
+		t.Fatalf("untrusted Hub received %d requests, want none", requestCount)
+	}
+}
+
+func TestVerifyHubRejectsUnsavedURI(t *testing.T) {
+	requestCount := 0
+	untrusted := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		requestCount++
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer untrusted.Close()
+
+	config := models.Config{
+		HubURI:        untrusted.URL,
+		HubKey:        "project-public",
+		HubPrivateKey: "project-private",
+	}
+	payload, err := json.Marshal(config)
+	if err != nil {
+		t.Fatalf("json.Marshal() error = %v", err)
+	}
+
+	gin.SetMode(gin.TestMode)
+	recorder := httptest.NewRecorder()
+	ctx, _ := gin.CreateTestContext(recorder)
+	ctx.Request = httptest.NewRequest(http.MethodPost, "/api/hub/verify", bytes.NewReader(payload))
+	ctx.Request.Header.Set("Content-Type", "application/json")
+
+	VerifyHub(ctx, "https://saved-hub.example.com")
+
+	if recorder.Code != http.StatusBadRequest {
+		t.Fatalf("status = %d, want %d; body = %s", recorder.Code, http.StatusBadRequest, recorder.Body.String())
+	}
+	if requestCount != 0 {
+		t.Fatalf("untrusted Hub received %d requests, want none", requestCount)
+	}
+}
+
+func TestHubPersistenceVerificationErrorExplainsStatus(t *testing.T) {
+	for _, test := range []struct {
+		status int
+		want   string
+	}{
+		{status: http.StatusBadRequest, want: "credentials or device/project assignment"},
+		{status: http.StatusUnauthorized, want: "no active subscription"},
+		{status: http.StatusForbidden, want: "device is muted"},
+		{status: http.StatusBadGateway, want: "HTTP 502"},
+	} {
+		t.Run(http.StatusText(test.status), func(t *testing.T) {
+			if got := hubPersistenceVerificationError(test.status); !strings.Contains(got, test.want) {
+				t.Fatalf("hubPersistenceVerificationError(%d) = %q, want it to contain %q", test.status, got, test.want)
+			}
+		})
+	}
+}
+
+func TestHeartbeatFailureLogOmitsHubResponseBody(t *testing.T) {
+	response := &http.Response{
+		StatusCode: http.StatusBadRequest,
+		Status:     "400 Bad Request",
+		Body:       io.NopCloser(strings.NewReader(`{"error":"invalid heartbeat"}`)),
+	}
+
+	responseBody, truncated, err := readHeartbeatResponseBody(response)
+	if err != nil {
+		t.Fatalf("readHeartbeatResponseBody() error = %v", err)
+	}
+	fields := heartbeatFailureLogFields(response, responseBody, truncated, 125*time.Millisecond)
+
+	for key, want := range map[string]interface{}{
+		"duration_ms":             int64(125),
+		"response_body_bytes":     len(responseBody),
+		"response_body_truncated": false,
+		"status_code":             http.StatusBadRequest,
+	} {
+		if got := fields[key]; got != want {
+			t.Errorf("%s = %v, want %v", key, got, want)
+		}
+	}
+	for key, value := range fields {
+		if strings.Contains(key, "response_body") && key != "response_body_bytes" && key != "response_body_truncated" {
+			t.Fatalf("unexpected response body field %q=%v", key, value)
+		}
+		if strings.Contains(fmt.Sprint(value), "invalid heartbeat") {
+			t.Fatalf("heartbeat log fields exposed response body in %q=%v", key, value)
+		}
+	}
+}
+
+func TestReadHeartbeatResponseBodyTruncatesLargeBody(t *testing.T) {
+	response := &http.Response{
+		Body: io.NopCloser(strings.NewReader(strings.Repeat("x", heartbeatResponseBodyLogLimit+1))),
+	}
+
+	body, truncated, err := readHeartbeatResponseBody(response)
+	if err != nil {
+		t.Fatalf("readHeartbeatResponseBody() error = %v", err)
+	}
+	if !truncated {
+		t.Fatal("readHeartbeatResponseBody() truncated = false, want true")
+	}
+	if len(body) != heartbeatResponseBodyLogLimit {
+		t.Fatalf("len(body) = %d, want %d", len(body), heartbeatResponseBodyLogLimit)
+	}
+}
+
+func TestHeartbeatONVIFPayloadCachesStaticAndReusesLoopSubscription(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	device := newTestONVIFDevice()
+	camera := models.IPCamera{
+		ONVIFXAddr:    "http://camera/onvif",
+		ONVIFUsername: "operator",
+		ONVIFPassword: "secret",
+	}
+	initialEvents := []agentonvif.ONVIFEvents{{Key: "input-1", Type: "input", Value: "true", Timestamp: 1}}
+	loopEvents := []agentonvif.ONVIFEvents{{Key: "output-1", Type: "output", Value: "false", Timestamp: 2}}
+	wantPresets := mustJSONMarshal(t, []models.OnvifActionPreset{{Name: "Lobby", Token: "1"}})
+	wantInitialEvents := mustJSONMarshal(t, initialEvents)
+	wantLoopEvents := mustJSONMarshal(t, loopEvents)
+
+	var connectCalls, ptzConfigCalls, ptzFunctionCalls, presetCalls, createCalls, eventCalls, unsubscribeCalls int
+
+	heartbeatConnectToONVIFDevice = func(*models.IPCamera) (*goonvif.Device, goonvifdevice.GetCapabilitiesResponse, error) {
+		connectCalls++
+		return device, goonvifdevice.GetCapabilitiesResponse{}, nil
+	}
+	heartbeatGetPTZConfigurationsFromDevice = func(*goonvif.Device) (goonvifptz.GetConfigurationsResponse, error) {
+		ptzConfigCalls++
+		return goonvifptz.GetConfigurationsResponse{}, nil
+	}
+	heartbeatGetPTZFunctionsFromDevice = func(goonvifptz.GetConfigurationsResponse) ([]string, bool, bool) {
+		ptzFunctionCalls++
+		return nil, true, true
+	}
+	heartbeatGetPresetsFromDevice = func(*goonvif.Device) ([]models.OnvifActionPreset, error) {
+		presetCalls++
+		return []models.OnvifActionPreset{{Name: "Lobby", Token: "1"}}, nil
+	}
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		createCalls++
+		switch createCalls {
+		case 1:
+			return "initial-1", nil
+		case 2:
+			return "loop", nil
+		case 3:
+			return "initial-2", nil
+		default:
+			t.Fatalf("unexpected create pull point call %d", createCalls)
+			return "", nil
+		}
+	}
+	heartbeatGetEventMessages = func(_ *goonvif.Device, pullPointAddress string) ([]agentonvif.ONVIFEvents, error) {
+		eventCalls++
+		switch pullPointAddress {
+		case "initial-1", "initial-2":
+			return initialEvents, nil
+		case "loop":
+			return loopEvents, nil
+		default:
+			t.Fatalf("unexpected pull point address %q", pullPointAddress)
+			return nil, nil
+		}
+	}
+	heartbeatUnsubscribePullPoint = func(_ *goonvif.Device, pullPointAddress string) error {
+		unsubscribeCalls++
+		if pullPointAddress != "initial-1" && pullPointAddress != "initial-2" {
+			t.Fatalf("unexpected unsubscribe pull point %q", pullPointAddress)
+		}
+		return nil
+	}
+
+	state := newHeartbeatONVIFState()
+
+	payload := getHeartbeatONVIFPayload(camera, state)
+	if payload.enabled != "true" || payload.zoom != "true" || payload.panTilt != "true" || payload.presets != "true" {
+		t.Fatalf("unexpected static payload: %+v", payload)
+	}
+	if string(payload.presetsList) != string(wantPresets) {
+		t.Fatalf("payload.presetsList = %s, want %s", payload.presetsList, wantPresets)
+	}
+	if string(payload.eventsList) != string(wantInitialEvents) {
+		t.Fatalf("payload.eventsList = %s, want %s", payload.eventsList, wantInitialEvents)
+	}
+	if connectCalls != 1 || ptzConfigCalls != 1 || ptzFunctionCalls != 1 || presetCalls != 1 {
+		t.Fatalf("unexpected static call counts after first cycle: connect=%d ptzConfig=%d ptzFunctions=%d presets=%d", connectCalls, ptzConfigCalls, ptzFunctionCalls, presetCalls)
+	}
+	if createCalls != 2 || eventCalls != 1 || unsubscribeCalls != 1 {
+		t.Fatalf("unexpected event call counts after first cycle: create=%d events=%d unsubscribe=%d", createCalls, eventCalls, unsubscribeCalls)
+	}
+
+	payload = getHeartbeatONVIFPayload(camera, state)
+	if string(payload.eventsList) != string(wantLoopEvents) {
+		t.Fatalf("second payload.eventsList = %s, want %s", payload.eventsList, wantLoopEvents)
+	}
+	if connectCalls != 1 {
+		t.Fatalf("connectCalls = %d, want 1", connectCalls)
+	}
+	if ptzConfigCalls != 1 || ptzFunctionCalls != 1 || presetCalls != 1 {
+		t.Fatalf("static calls were not cached: ptzConfig=%d ptzFunctions=%d presets=%d", ptzConfigCalls, ptzFunctionCalls, presetCalls)
+	}
+	if createCalls != 3 || eventCalls != 3 || unsubscribeCalls != 2 {
+		t.Fatalf("temporary state subscription was not refreshed or loop subscription was not reused: create=%d events=%d unsubscribe=%d", createCalls, eventCalls, unsubscribeCalls)
+	}
+}
+
+func TestHeartbeatONVIFPayloadRefreshesAfterCameraConfigChange(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	deviceA := newTestONVIFDevice()
+	deviceB := newTestONVIFDevice()
+	cameraA := models.IPCamera{ONVIFXAddr: "http://camera-a/onvif", ONVIFUsername: "user", ONVIFPassword: "secret-a"}
+	cameraB := models.IPCamera{ONVIFXAddr: "http://camera-b/onvif", ONVIFUsername: "user", ONVIFPassword: "secret-b"}
+	var connectCalls, ptzConfigCalls, presetCalls, createCalls int
+	var unsubscribed []string
+
+	heartbeatConnectToONVIFDevice = func(camera *models.IPCamera) (*goonvif.Device, goonvifdevice.GetCapabilitiesResponse, error) {
+		connectCalls++
+		switch camera.ONVIFXAddr {
+		case cameraA.ONVIFXAddr:
+			return deviceA, goonvifdevice.GetCapabilitiesResponse{}, nil
+		case cameraB.ONVIFXAddr:
+			return deviceB, goonvifdevice.GetCapabilitiesResponse{}, nil
+		default:
+			t.Fatalf("unexpected camera address %q", camera.ONVIFXAddr)
+			return nil, goonvifdevice.GetCapabilitiesResponse{}, nil
+		}
+	}
+	heartbeatGetPTZConfigurationsFromDevice = func(*goonvif.Device) (goonvifptz.GetConfigurationsResponse, error) {
+		ptzConfigCalls++
+		return goonvifptz.GetConfigurationsResponse{}, nil
+	}
+	heartbeatGetPTZFunctionsFromDevice = func(goonvifptz.GetConfigurationsResponse) ([]string, bool, bool) {
+		return nil, false, true
+	}
+	heartbeatGetPresetsFromDevice = func(*goonvif.Device) ([]models.OnvifActionPreset, error) {
+		presetCalls++
+		return nil, nil
+	}
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		createCalls++
+		switch createCalls {
+		case 1:
+			return "initial-a", nil
+		case 2:
+			return "loop-a", nil
+		case 3:
+			return "initial-b", nil
+		case 4:
+			return "loop-b", nil
+		default:
+			t.Fatalf("unexpected create pull point call %d", createCalls)
+			return "", nil
+		}
+	}
+	heartbeatGetEventMessages = func(_ *goonvif.Device, pullPointAddress string) ([]agentonvif.ONVIFEvents, error) {
+		switch pullPointAddress {
+		case "initial-a":
+			return []agentonvif.ONVIFEvents{{Key: "a", Type: "input", Value: "true", Timestamp: 1}}, nil
+		case "initial-b":
+			return []agentonvif.ONVIFEvents{{Key: "b", Type: "input", Value: "false", Timestamp: 2}}, nil
+		default:
+			t.Fatalf("unexpected pull point address %q", pullPointAddress)
+			return nil, nil
+		}
+	}
+	heartbeatUnsubscribePullPoint = func(_ *goonvif.Device, pullPointAddress string) error {
+		unsubscribed = append(unsubscribed, pullPointAddress)
+		return nil
+	}
+
+	state := newHeartbeatONVIFState()
+	_ = getHeartbeatONVIFPayload(cameraA, state)
+	payload := getHeartbeatONVIFPayload(cameraB, state)
+
+	if connectCalls != 2 {
+		t.Fatalf("connectCalls = %d, want 2", connectCalls)
+	}
+	if ptzConfigCalls != 2 || presetCalls != 2 {
+		t.Fatalf("camera config change did not refresh static state: ptzConfig=%d presets=%d", ptzConfigCalls, presetCalls)
+	}
+	if createCalls != 4 {
+		t.Fatalf("camera config change did not recreate subscriptions: create=%d", createCalls)
+	}
+	if countString(unsubscribed, "loop-a") != 1 {
+		t.Fatalf("unsubscribed loop-a %d times, want 1; unsubscribed=%v", countString(unsubscribed, "loop-a"), unsubscribed)
+	}
+	wantEvents := mustJSONMarshal(t, []agentonvif.ONVIFEvents{{Key: "b", Type: "input", Value: "false", Timestamp: 2}})
+	if string(payload.eventsList) != string(wantEvents) {
+		t.Fatalf("payload.eventsList = %s, want %s", payload.eventsList, wantEvents)
+	}
+}
+
+func TestHeartbeatONVIFPayloadRetriesStaticFetchFailuresWithoutReconnect(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	device := newTestONVIFDevice()
+	camera := models.IPCamera{
+		ONVIFXAddr:    "http://camera/onvif",
+		ONVIFUsername: "operator",
+		ONVIFPassword: "secret",
+	}
+	var connectCalls, ptzConfigCalls, presetCalls, createCalls, eventCalls int
+
+	heartbeatConnectToONVIFDevice = func(*models.IPCamera) (*goonvif.Device, goonvifdevice.GetCapabilitiesResponse, error) {
+		connectCalls++
+		return device, goonvifdevice.GetCapabilitiesResponse{}, nil
+	}
+	heartbeatGetPTZConfigurationsFromDevice = func(*goonvif.Device) (goonvifptz.GetConfigurationsResponse, error) {
+		ptzConfigCalls++
+		return goonvifptz.GetConfigurationsResponse{}, nil
+	}
+	heartbeatGetPTZFunctionsFromDevice = func(goonvifptz.GetConfigurationsResponse) ([]string, bool, bool) {
+		return nil, true, true
+	}
+	heartbeatGetPresetsFromDevice = func(*goonvif.Device) ([]models.OnvifActionPreset, error) {
+		presetCalls++
+		if presetCalls == 1 {
+			return nil, errors.New("temporary preset failure")
+		}
+		return []models.OnvifActionPreset{{Name: "Lobby", Token: "1"}}, nil
+	}
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		createCalls++
+		switch createCalls {
+		case 1:
+			return "initial-1", nil
+		case 2:
+			return "loop", nil
+		case 3:
+			return "initial-2", nil
+		default:
+			t.Fatalf("unexpected create pull point call %d", createCalls)
+			return "", nil
+		}
+	}
+	heartbeatGetEventMessages = func(_ *goonvif.Device, pullPointAddress string) ([]agentonvif.ONVIFEvents, error) {
+		eventCalls++
+		switch pullPointAddress {
+		case "initial-1", "initial-2":
+			return []agentonvif.ONVIFEvents{{Key: "one", Type: "input", Value: "true", Timestamp: 1}}, nil
+		case "loop":
+			return []agentonvif.ONVIFEvents{{Key: "two", Type: "output", Value: "false", Timestamp: 2}}, nil
+		default:
+			t.Fatalf("unexpected pull point address %q", pullPointAddress)
+			return nil, nil
+		}
+	}
+	heartbeatUnsubscribePullPoint = func(*goonvif.Device, string) error { return nil }
+
+	state := newHeartbeatONVIFState()
+	payload := getHeartbeatONVIFPayload(camera, state)
+	if payload.presets != "false" {
+		t.Fatalf("payload.presets = %q, want false on transient preset failure", payload.presets)
+	}
+	payload = getHeartbeatONVIFPayload(camera, state)
+	if payload.presets != "true" {
+		t.Fatalf("payload.presets = %q, want true after retry", payload.presets)
+	}
+	if connectCalls != 1 {
+		t.Fatalf("connectCalls = %d, want 1", connectCalls)
+	}
+	if ptzConfigCalls != 2 || presetCalls != 2 {
+		t.Fatalf("static failures were not retried on heartbeat cadence: ptzConfig=%d presets=%d", ptzConfigCalls, presetCalls)
+	}
+	if createCalls != 3 || eventCalls != 3 {
+		t.Fatalf("unexpected event behavior during retry: create=%d events=%d", createCalls, eventCalls)
+	}
+}
+
+func TestHeartbeatONVIFStateReleasesSubscriptionWhenDisabled(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	device := newTestONVIFDevice()
+	state := newHeartbeatONVIFState()
+	state.cameraConfiguration = models.IPCamera{ONVIFXAddr: "http://camera/onvif"}
+	state.cameraKey = heartbeatONVIFCameraKey(state.cameraConfiguration)
+	state.device = device
+	state.loopPullPoint = "loop"
+
+	var unsubscribed []string
+	heartbeatUnsubscribePullPoint = func(gotDevice *goonvif.Device, pullPointAddress string) error {
+		if gotDevice != device {
+			t.Fatal("unsubscribe used a different ONVIF device")
+		}
+		unsubscribed = append(unsubscribed, pullPointAddress)
+		return nil
+	}
+
+	state.prepare(models.IPCamera{})
+
+	if len(unsubscribed) != 1 || unsubscribed[0] != "loop" {
+		t.Fatalf("unsubscribed = %v, want [loop]", unsubscribed)
+	}
+	if state.device != nil || state.loopPullPoint != "" {
+		t.Fatalf("disabled state retained device or pull point: %+v", state)
+	}
+}
+
+func TestHeartbeatONVIFPayloadDoesNotCreateSubscriptionsWhenConnectFails(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	var createCalls int
+
+	heartbeatConnectToONVIFDevice = func(*models.IPCamera) (*goonvif.Device, goonvifdevice.GetCapabilitiesResponse, error) {
+		return nil, goonvifdevice.GetCapabilitiesResponse{}, errors.New("connect failed")
+	}
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		createCalls++
+		return "unexpected", nil
+	}
+
+	payload := getHeartbeatONVIFPayload(models.IPCamera{
+		ONVIFXAddr:    "http://camera/onvif",
+		ONVIFUsername: "operator",
+		ONVIFPassword: "secret",
+	}, newHeartbeatONVIFState())
+
+	if createCalls != 0 {
+		t.Fatalf("createCalls = %d, want 0", createCalls)
+	}
+	assertDefaultHeartbeatONVIFPayload(t, payload)
+}
+
+func TestHeartbeatONVIFPayloadReconnectFailureInvalidatesCache(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	device1 := newTestONVIFDevice()
+	device2 := newTestONVIFDevice()
+	camera := models.IPCamera{
+		ONVIFXAddr:    "http://camera/onvif",
+		ONVIFUsername: "operator",
+		ONVIFPassword: "secret",
+	}
+	var connectCalls, ptzConfigCalls, presetCalls, createCalls int
+	var unsubscribed []string
+
+	heartbeatConnectToONVIFDevice = func(*models.IPCamera) (*goonvif.Device, goonvifdevice.GetCapabilitiesResponse, error) {
+		connectCalls++
+		switch connectCalls {
+		case 1:
+			return device1, goonvifdevice.GetCapabilitiesResponse{}, nil
+		case 2:
+			return device2, goonvifdevice.GetCapabilitiesResponse{}, nil
+		default:
+			t.Fatalf("unexpected connect call %d", connectCalls)
+			return nil, goonvifdevice.GetCapabilitiesResponse{}, nil
+		}
+	}
+	heartbeatGetPTZConfigurationsFromDevice = func(*goonvif.Device) (goonvifptz.GetConfigurationsResponse, error) {
+		ptzConfigCalls++
+		return goonvifptz.GetConfigurationsResponse{}, nil
+	}
+	heartbeatGetPTZFunctionsFromDevice = func(goonvifptz.GetConfigurationsResponse) ([]string, bool, bool) {
+		return nil, true, false
+	}
+	heartbeatGetPresetsFromDevice = func(*goonvif.Device) ([]models.OnvifActionPreset, error) {
+		presetCalls++
+		return []models.OnvifActionPreset{{Name: "Preset", Token: "1"}}, nil
+	}
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		createCalls++
+		switch createCalls {
+		case 1:
+			return "initial-1", nil
+		case 2:
+			return "loop-1", nil
+		case 3:
+			return "initial-2", nil
+		case 4:
+			return "initial-3", nil
+		case 5:
+			return "loop-3", nil
+		default:
+			t.Fatalf("unexpected create pull point call %d", createCalls)
+			return "", nil
+		}
+	}
+	heartbeatGetEventMessages = func(_ *goonvif.Device, pullPointAddress string) ([]agentonvif.ONVIFEvents, error) {
+		switch pullPointAddress {
+		case "initial-1":
+			return []agentonvif.ONVIFEvents{{Key: "before", Type: "input", Value: "true", Timestamp: 1}}, nil
+		case "initial-2":
+			return []agentonvif.ONVIFEvents{{Key: "during", Type: "input", Value: "true", Timestamp: 2}}, nil
+		case "loop-1":
+			return nil, errors.New("pull failed")
+		case "initial-3":
+			return []agentonvif.ONVIFEvents{{Key: "after", Type: "input", Value: "false", Timestamp: 2}}, nil
+		default:
+			t.Fatalf("unexpected pull point address %q", pullPointAddress)
+			return nil, nil
+		}
+	}
+	heartbeatUnsubscribePullPoint = func(_ *goonvif.Device, pullPointAddress string) error {
+		unsubscribed = append(unsubscribed, pullPointAddress)
+		return nil
+	}
+
+	state := newHeartbeatONVIFState()
+	_ = getHeartbeatONVIFPayload(camera, state)
+	payload := getHeartbeatONVIFPayload(camera, state)
+	wantEventsDuringFailure := mustJSONMarshal(t, []agentonvif.ONVIFEvents{{Key: "during", Type: "input", Value: "true", Timestamp: 2}})
+	if string(payload.eventsList) != string(wantEventsDuringFailure) {
+		t.Fatalf("payload.eventsList after operation failure = %s, want %s", payload.eventsList, wantEventsDuringFailure)
+	}
+	payload = getHeartbeatONVIFPayload(camera, state)
+
+	if ptzConfigCalls != 2 || presetCalls != 2 {
+		t.Fatalf("reconnect did not refresh static state: ptzConfig=%d presets=%d", ptzConfigCalls, presetCalls)
+	}
+	if connectCalls != 2 {
+		t.Fatalf("connectCalls = %d, want 2 after reconnect", connectCalls)
+	}
+	if createCalls != 5 {
+		t.Fatalf("reconnect did not recreate subscriptions: create=%d", createCalls)
+	}
+	if countString(unsubscribed, "loop-1") != 1 {
+		t.Fatalf("operation failure cleanup mismatch, unsubscribed=%v", unsubscribed)
+	}
+	wantEvents := mustJSONMarshal(t, []agentonvif.ONVIFEvents{{Key: "after", Type: "input", Value: "false", Timestamp: 2}})
+	if string(payload.eventsList) != string(wantEvents) {
+		t.Fatalf("payload.eventsList = %s, want %s", payload.eventsList, wantEvents)
+	}
+}
+
+func TestHeartbeatONVIFPayloadKeepsCachedConnectionWhenInitialStateFetchFails(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	device := newTestONVIFDevice()
+	camera := models.IPCamera{
+		ONVIFXAddr:    "http://camera/onvif",
+		ONVIFUsername: "operator",
+		ONVIFPassword: "secret",
+	}
+	loopEvents := []agentonvif.ONVIFEvents{{Key: "input-1", Type: "input", Value: "true", Timestamp: 1}}
+
+	state := newHeartbeatONVIFState()
+	state.cameraConfiguration = camera
+	state.cameraKey = heartbeatONVIFCameraKey(camera)
+	state.device = device
+	state.loopPullPoint = "loop"
+	state.staticLoaded = true
+	state.staticPayload.enabled = "true"
+
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		return "", errors.New("temporary initial-state failure")
+	}
+	heartbeatGetEventMessages = func(_ *goonvif.Device, pullPointAddress string) ([]agentonvif.ONVIFEvents, error) {
+		if pullPointAddress != "loop" {
+			t.Fatalf("unexpected pull point address %q", pullPointAddress)
+		}
+		return loopEvents, nil
+	}
+
+	payload := getHeartbeatONVIFPayload(camera, state)
+
+	wantEvents := mustJSONMarshal(t, loopEvents)
+	if string(payload.eventsList) != string(wantEvents) {
+		t.Fatalf("payload.eventsList = %s, want %s", payload.eventsList, wantEvents)
+	}
+	if state.device != device || state.loopPullPoint != "loop" || !state.staticLoaded {
+		t.Fatalf("temporary failure invalidated healthy cached state: %+v", state)
+	}
+}
+
+func restoreHeartbeatONVIFStubs(t *testing.T) {
+	t.Helper()
+
+	originalConnect := heartbeatConnectToONVIFDevice
+	originalCreate := heartbeatCreatePullPointSubscription
+	originalDigitalInputs := heartbeatGetDigitalInputs
+	originalEvents := heartbeatGetEventMessages
+	originalPresets := heartbeatGetPresetsFromDevice
+	originalPTZConfigurations := heartbeatGetPTZConfigurationsFromDevice
+	originalPTZFunctions := heartbeatGetPTZFunctionsFromDevice
+	originalRelayOutputs := heartbeatGetRelayOutputs
+	originalUnsubscribe := heartbeatUnsubscribePullPoint
+
+	t.Cleanup(func() {
+		heartbeatConnectToONVIFDevice = originalConnect
+		heartbeatCreatePullPointSubscription = originalCreate
+		heartbeatGetDigitalInputs = originalDigitalInputs
+		heartbeatGetEventMessages = originalEvents
+		heartbeatGetPresetsFromDevice = originalPresets
+		heartbeatGetPTZConfigurationsFromDevice = originalPTZConfigurations
+		heartbeatGetPTZFunctionsFromDevice = originalPTZFunctions
+		heartbeatGetRelayOutputs = originalRelayOutputs
+		heartbeatUnsubscribePullPoint = originalUnsubscribe
+	})
+}
+
+func mustJSONMarshal(t *testing.T, v interface{}) []byte {
+	t.Helper()
+
+	b, err := json.Marshal(v)
+	if err != nil {
+		t.Fatalf("json.Marshal() error = %v", err)
+	}
+	return b
+}
+
+func assertDefaultHeartbeatONVIFPayload(t *testing.T, payload heartbeatONVIFPayload) {
+	t.Helper()
+
+	defaultPayload := defaultHeartbeatONVIFPayload()
+	if payload.enabled != defaultPayload.enabled ||
+		payload.zoom != defaultPayload.zoom ||
+		payload.panTilt != defaultPayload.panTilt ||
+		payload.presets != defaultPayload.presets ||
+		string(payload.presetsList) != string(defaultPayload.presetsList) ||
+		string(payload.eventsList) != string(defaultPayload.eventsList) {
+		t.Fatalf("payload = %+v, want default payload", payload)
+	}
+}
+
+func newTestONVIFDevice() *goonvif.Device {
+	return &goonvif.Device{}
+}
+
+func countString(values []string, want string) int {
+	count := 0
+	for _, value := range values {
+		if value == want {
+			count++
+		}
+	}
+	return count
+}
+
+// Some devices accept the subscription but reject the pulls, or never publish
+// the initial I/O state. The relay outputs and digital inputs must then be
+// queried directly instead of reporting no I/O at all.
+func TestHeartbeatONVIFPayloadFallsBackToDirectIOWhenSubscriptionsReturnNoEvents(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	device := newTestONVIFDevice()
+	camera := models.IPCamera{
+		ONVIFXAddr:    "http://camera/onvif",
+		ONVIFUsername: "operator",
+		ONVIFPassword: "secret",
+	}
+
+	state := newHeartbeatONVIFState()
+	state.cameraConfiguration = camera
+	state.cameraKey = heartbeatONVIFCameraKey(camera)
+	state.device = device
+	state.staticLoaded = true
+	state.staticPayload.enabled = "true"
+
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		return "pull-point", nil
+	}
+	heartbeatGetEventMessages = func(*goonvif.Device, string) ([]agentonvif.ONVIFEvents, error) {
+		return nil, errors.New("pull rejected")
+	}
+	heartbeatUnsubscribePullPoint = func(*goonvif.Device, string) error {
+		return nil
+	}
+	var relayCalls, inputCalls int
+	heartbeatGetRelayOutputs = func(*goonvif.Device) (goonvifdevice.GetRelayOutputsResponse, error) {
+		relayCalls++
+		return goonvifdevice.GetRelayOutputsResponse{RelayOutputs: []goonvifxsd.RelayOutput{{DeviceEntity: goonvifxsd.DeviceEntity{Token: "relay0"}}}}, nil
+	}
+	heartbeatGetDigitalInputs = func(*goonvif.Device) (goonvifdevice.GetDigitalInputsResponse, error) {
+		inputCalls++
+		return goonvifdevice.GetDigitalInputsResponse{DigitalInputs: []goonvifxsd.DigitalInput{{Token: "di0"}}}, nil
+	}
+
+	payload := getHeartbeatONVIFPayload(camera, state)
+
+	if relayCalls != 1 || inputCalls != 1 {
+		t.Fatalf("direct I/O queries: relay=%d inputs=%d, want 1 each", relayCalls, inputCalls)
+	}
+	var events []agentonvif.ONVIFEvents
+	if err := json.Unmarshal(payload.eventsList, &events); err != nil {
+		t.Fatalf("payload.eventsList is not valid JSON: %v", err)
+	}
+	got := map[string]string{}
+	for _, event := range events {
+		got[event.Key] = event.Type
+	}
+	if got["relay0"] != "output" || got["di0"] != "input" || len(got) != 2 {
+		t.Fatalf("payload.eventsList = %s, want relay0 output and di0 input", payload.eventsList)
+	}
+}

machinery/src/cloud/custom_headers.go

@@ -0,0 +1,195 @@
+package cloud
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"sort"
+	"strings"
+)
+
+const (
+	customHeadersManifestHeader = "X-Kerberos-Storage-Custom-Headers"
+	maxCustomHeaders            = 32
+	maxCustomHeaderNameLength   = 128
+	maxCustomHeaderValueLength  = 4096
+)
+
+var disallowedCustomHeaders = map[string]struct{}{
+	"authorization":       {},
+	"connection":          {},
+	"content-length":      {},
+	"content-type":        {},
+	"cookie":              {},
+	"expect":              {},
+	"host":                {},
+	"keep-alive":          {},
+	"proxy-authorization": {},
+	"proxy-connection":    {},
+	"te":                  {},
+	"trailer":             {},
+	"trace-id":            {},
+	"transfer-encoding":   {},
+	"tus-resumable":       {},
+	"upgrade":             {},
+}
+
+func parseCustomVaultHeaders(value string) (map[string]string, error) {
+	if strings.TrimSpace(value) == "" {
+		return nil, nil
+	}
+
+	// Decode into pointers so JSON null values can be told apart from strings.
+	var rawHeaders map[string]*string
+	if err := json.Unmarshal([]byte(value), &rawHeaders); err != nil {
+		return nil, fmt.Errorf("custom Vault headers must be a JSON object of string values: %w", err)
+	}
+	if rawHeaders == nil {
+		return nil, errors.New("custom Vault headers must be a JSON object of string values")
+	}
+	if len(rawHeaders) > maxCustomHeaders {
+		return nil, fmt.Errorf("custom Vault headers exceed the limit of %d", maxCustomHeaders)
+	}
+	headers := make(map[string]string, len(rawHeaders))
+	seen := make(map[string]struct{}, len(rawHeaders))
+	for name, rawValue := range rawHeaders {
+		if rawValue == nil {
+			return nil, fmt.Errorf("custom Vault header %q must be a string, not null", name)
+		}
+		value := *rawValue
+		headers[name] = value
+		if err := validateCustomVaultHeader(name, value); err != nil {
+			return nil, err
+		}
+		lowerName := strings.ToLower(name)
+		if _, duplicate := seen[lowerName]; duplicate {
+			return nil, fmt.Errorf("custom Vault header %q is duplicated with different casing", name)
+		}
+		seen[lowerName] = struct{}{}
+	}
+	return headers, nil
+}
+
+func validateCustomVaultHeader(name, value string) error {
+	if name == "" || len(name) > maxCustomHeaderNameLength || !isHTTPToken(name) {
+		return fmt.Errorf("invalid custom Vault header name %q", name)
+	}
+	lowerName := strings.ToLower(name)
+	if strings.HasPrefix(lowerName, "x-kerberos-") {
+		return fmt.Errorf("custom Vault header %q uses the reserved X-Kerberos namespace", name)
+	}
+	if strings.HasPrefix(lowerName, "upload-") {
+		return fmt.Errorf("custom Vault header %q uses the reserved tus Upload namespace", name)
+	}
+	if _, disallowed := disallowedCustomHeaders[lowerName]; disallowed {
+		return fmt.Errorf("custom Vault header %q is reserved", name)
+	}
+	if len(value) > maxCustomHeaderValueLength {
+		return fmt.Errorf("custom Vault header %q exceeds the %d byte value limit", name, maxCustomHeaderValueLength)
+	}
+	for _, character := range value {
+		if character < 0x20 || character == 0x7f {
+			return fmt.Errorf("custom Vault header %q contains an invalid value", name)
+		}
+	}
+	return nil
+}
+
+func isHTTPToken(value string) bool {
+	if value == "" {
+		return false
+	}
+	for _, character := range value {
+		if (character >= 'a' && character <= 'z') ||
+			(character >= 'A' && character <= 'Z') ||
+			(character >= '0' && character <= '9') ||
+			strings.ContainsRune("!#$%&'*+-.^_`|~", character) {
+			continue
+		}
+		return false
+	}
+	return true
+}
+
+func setCustomVaultHeaders(headers http.Header, customHeaders map[string]string) error {
+	if len(customHeaders) == 0 {
+		return nil
+	}
+
+	names := make([]string, 0, len(customHeaders))
+	for name, value := range customHeaders {
+		if err := validateCustomVaultHeader(name, value); err != nil {
+			return err
+		}
+		names = append(names, name)
+		// Assign directly so the configured name is not canonicalised by Set.
+		headers[name] = []string{value}
+	}
+	sort.Strings(names)
+	manifest, err := json.Marshal(names)
+	if err != nil {
+		return err
+	}
+	headers.Set(customHeadersManifestHeader, base64.RawURLEncoding.EncodeToString(manifest))
+	return nil
+}
+
+const maxVaultRedirects = 10
+
+// stripVaultHeadersOnCrossHostRedirect removes Vault credentials and the
+// customer-supplied metadata (custom headers, their manifest and the tus
+// Upload-Metadata that embeds them) before following a redirect to another host.
+func stripVaultHeadersOnCrossHostRedirect(req *http.Request, via []*http.Request) error {
+	if len(via) >= maxVaultRedirects {
+		return fmt.Errorf("stopped after %d redirects", maxVaultRedirects)
+	}
+	if len(via) == 0 || req.URL.Host == via[0].URL.Host {
+		return nil
+	}
+
+	customNames := customVaultHeaderNamesFromManifest(req.Header)
+	for name := range req.Header {
+		canonicalName := http.CanonicalHeaderKey(name)
+		_, custom := customNames[strings.ToLower(name)]
+		if custom || strings.HasPrefix(canonicalName, "X-Kerberos-") || canonicalName == "Upload-Metadata" {
+			delete(req.Header, name)
+		}
+	}
+	return nil
+}
+
+func customVaultHeaderNamesFromManifest(headers http.Header) map[string]struct{} {
+	encoded := headers.Get(customHeadersManifestHeader)
+	if encoded == "" {
+		return nil
+	}
+	manifest, err := base64.RawURLEncoding.DecodeString(encoded)
+	if err != nil {
+		return nil
+	}
+	var names []string
+	if err := json.Unmarshal(manifest, &names); err != nil {
+		return nil
+	}
+	lowerNames := make(map[string]struct{}, len(names))
+	for _, name := range names {
+		lowerNames[strings.ToLower(name)] = struct{}{}
+	}
+	return lowerNames
+}
+
+func encodeCustomVaultHeaders(customHeaders map[string]string) (string, error) {
+	if len(customHeaders) == 0 {
+		return "", nil
+	}
+	encoded, err := json.Marshal(customHeaders)
+	if err != nil {
+		return "", err
+	}
+	if len(encoded) == 0 {
+		return "", errors.New("custom Vault headers could not be encoded")
+	}
+	return string(encoded), nil
+}

machinery/src/cloud/custom_headers_test.go

@@ -0,0 +1,100 @@
+package cloud
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"net/http"
+	"reflect"
+	"testing"
+)
+
+func TestCustomVaultHeadersAreValidatedAndAdded(t *testing.T) {
+	customHeaders, err := parseCustomVaultHeaders(`{"site_id":"site-1","line_id":"line-2"}`)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	headers := http.Header{}
+	if err := setCustomVaultHeaders(headers, customHeaders); err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(headers["site_id"], []string{"site-1"}) || !reflect.DeepEqual(headers["line_id"], []string{"line-2"}) {
+		t.Fatalf("custom header names were not preserved: %#v", headers)
+	}
+	if _, canonicalised := headers["Site_id"]; canonicalised {
+		t.Fatalf("custom header name was canonicalised: %#v", headers)
+	}
+
+	encodedManifest, err := base64.RawURLEncoding.DecodeString(headers.Get(customHeadersManifestHeader))
+	if err != nil {
+		t.Fatal(err)
+	}
+	var names []string
+	if err := json.Unmarshal(encodedManifest, &names); err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(names, []string{"line_id", "site_id"}) {
+		t.Fatalf("manifest = %#v", names)
+	}
+}
+
+func TestCustomVaultHeadersRejectReservedAndInvalidValues(t *testing.T) {
+	for _, value := range []string{
+		`{"X-Kerberos-Storage-AccessKey":"secret"}`,
+		`{"Authorization":"secret"}`,
+		`{"site_id":"one\nInjected: value"}`,
+		`{"bad header":"value"}`,
+		`{"site_id":1}`,
+		`{"Content-Type":"text/plain"}`,
+		`{"site_id":"one","SITE_ID":"two"}`,
+		`{"site_id":null}`,
+		`{"site_id":"one\ttwo"}`,
+		`{"Upload-Checksum":"sha1 abc"}`,
+		`{"upload-concat":"final;/files/a"}`,
+		`{"Upload-Defer-Length":"1"}`,
+		`null`,
+	} {
+		if _, err := parseCustomVaultHeaders(value); err == nil {
+			t.Fatalf("parseCustomVaultHeaders(%q) succeeded", value)
+		}
+	}
+}
+
+func TestVaultRedirectStripsCustomHeadersOnHostChange(t *testing.T) {
+	newRedirect := func(t *testing.T, host string) (*http.Request, []*http.Request) {
+		t.Helper()
+		original, err := http.NewRequest(http.MethodPost, "https://vault.example.com/api/storage", nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		redirected, err := http.NewRequest(http.MethodPost, "https://"+host+"/api/storage", nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		redirected.Header.Set("X-Kerberos-Storage-AccessKey", "secret")
+		redirected.Header.Set("Upload-Metadata", "custom_headers abc")
+		redirected.Header.Set("Tus-Resumable", "1.0.0")
+		if err := setCustomVaultHeaders(redirected.Header, map[string]string{"site_id": "site-1", "Line-Id": "line-2"}); err != nil {
+			t.Fatal(err)
+		}
+		return redirected, []*http.Request{original}
+	}
+
+	crossHost, via := newRedirect(t, "other.example.com")
+	if err := stripVaultHeadersOnCrossHostRedirect(crossHost, via); err != nil {
+		t.Fatal(err)
+	}
+	if len(crossHost.Header) != 1 || crossHost.Header.Get("Tus-Resumable") != "1.0.0" {
+		t.Fatalf("cross-host redirect headers = %#v", crossHost.Header)
+	}
+
+	sameHost, via := newRedirect(t, "vault.example.com")
+	if err := stripVaultHeadersOnCrossHostRedirect(sameHost, via); err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(sameHost.Header["site_id"], []string{"site-1"}) ||
+		sameHost.Header.Get("X-Kerberos-Storage-AccessKey") != "secret" ||
+		sameHost.Header.Get(customHeadersManifestHeader) == "" {
+		t.Fatalf("same-host redirect headers = %#v", sameHost.Header)
+	}
+}

machinery/src/cloud/dropbox.go

@@ -5,14 +5,14 @@ package cloud
 import (
 	"bytes"
 	"errors"
-	"os"
 
 	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox"
 	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
 	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/users"
 	"github.com/gin-gonic/gin"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/utils"
+	log "github.com/sirupsen/logrus"
 )
 
 // UploadDropbox uploads the file to your Dropbox account using the access token and directory.
@@ -30,25 +30,23 @@ func UploadDropbox(configuration *models.Configuration, fileName string) (bool,
 
 	if token == "" {
 		err := "UploadDropbox: Dropbox not properly configured"
-		log.Log.Info(err)
+		log.Info(err)
 		return false, true, errors.New(err)
 	}
 
 	// Upload to Dropbox
-	log.Log.Info("UploadDropbox: Uploading to Dropbox")
-	log.Log.Info("UploadDropbox: Upload started for " + fileName)
-	fullname := "data/recordings/" + fileName
-
+	log.Info("UploadDropbox: Uploading to Dropbox")
+	log.Info("UploadDropbox: Upload started for " + fileName)
 	dConfig := dropbox.Config{
 		Token:    token,
 		LogLevel: dropbox.LogInfo, // if needed, set the desired logging level. Default is off
 	}
 
-	file, err := os.OpenFile(fullname, os.O_RDWR, 0755)
+	file, err := utils.OpenRecording("data/recordings", fileName)
 	if file != nil {
 		defer func() {
 			if cerr := file.Close(); cerr != nil {
-				log.Log.Error("UploadDropbox: Error closing file: " + cerr.Error())
+				log.Error("UploadDropbox: Error closing file: " + cerr.Error())
 			}
 		}()
 	}
@@ -68,15 +66,15 @@ func UploadDropbox(configuration *models.Configuration, fileName string) (bool,
 		}, file)
 
 		if err != nil {
-			log.Log.Error("UploadDropbox: Error uploading file: " + err.Error())
+			log.Error("UploadDropbox: Error uploading file: " + err.Error())
 			return false, false, err
 		}
 
-		log.Log.Info("UploadDropbox: File uploaded successfully, " + res.Name)
+		log.Info("UploadDropbox: File uploaded successfully, " + res.Name)
 		return true, true, nil
 	}
 
-	log.Log.Error("UploadDropbox: Error opening file: " + err.Error())
+	log.Error("UploadDropbox: Error opening file: " + err.Error())
 	return false, true, err
 }
 

machinery/src/cloud/kerberos_hub.go

@@ -3,12 +3,14 @@ package cloud
 import (
 	"crypto/tls"
 	"errors"
+	"fmt"
 	"io/ioutil"
 	"net/http"
 	"os"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/utils"
+	log "github.com/sirupsen/logrus"
 )
 
 func UploadKerberosHub(configuration *models.Configuration, fileName string) (bool, bool, error) {
@@ -19,7 +21,7 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 		config.HubPrivateKey == "" ||
 		config.S3.Region == "" {
 		err := "UploadKerberosHub: Kerberos Hub not properly configured."
-		log.Log.Info(err)
+		log.Info(err)
 		return false, false, errors.New(err)
 	}
 
@@ -32,8 +34,10 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 	// - Number of changes
 	// - Token
 
-	log.Log.Info("UploadKerberosHub: Uploading to Kerberos Hub (" + config.HubURI + ")")
-	log.Log.Info("UploadKerberosHub: Upload started for " + fileName)
+	log.WithFields(log.Fields{
+		"component": "kerberos_hub",
+		"event":     "upload_started",
+	}).Info("Uploading recording to Kerberos Hub")
 
 	// Prefer the resumable (tus) upload when enabled (the default). Kerberos Hub
 	// authenticates the agent with its Hub public/private key and proxies the
@@ -44,29 +48,45 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 		uploaded, _, supported, body, rerr := uploadHubResumable(&config, fileName, "UploadKerberosHub", "hub")
 		if supported {
 			if uploaded {
-				log.Log.Info("UploadKerberosHub: Upload Finished (resumable), " + body)
+				log.WithFields(log.Fields{
+					"component":      "kerberos_hub",
+					"event":          "upload_completed",
+					"response_bytes": len(body),
+					"transport":      "tus",
+				}).Info("Hub upload completed")
 				return true, true, nil
 			}
 			if rerr != nil {
-				log.Log.Info("UploadKerberosHub: resumable upload failed, " + rerr.Error())
+				log.WithError(rerr).WithFields(log.Fields{
+					"component": "kerberos_hub",
+					"event":     "upload_failed",
+					"transport": "tus",
+				}).Error("Hub upload failed")
 			} else {
-				log.Log.Info("UploadKerberosHub: resumable upload incomplete, " + body)
+				log.WithFields(log.Fields{
+					"component":      "kerberos_hub",
+					"event":          "upload_incomplete",
+					"response_bytes": len(body),
+					"transport":      "tus",
+				}).Warn("Hub upload incomplete")
 			}
 			return false, true, rerr
 		}
-		log.Log.Info("UploadKerberosHub: resumable (tus) endpoint not available, falling back to legacy upload")
+		log.WithFields(log.Fields{
+			"component": "kerberos_hub",
+			"event":     "upload_transport_fallback",
+			"transport": "http",
+		}).Info("Resumable Hub upload unavailable; using legacy upload")
 	}
 
-	fullname := "data/recordings/" + fileName
-
 	// Check if we still have the file otherwise we abort the request.
-	file, err := os.OpenFile(fullname, os.O_RDWR, 0755)
+	file, err := utils.OpenRecording("data/recordings", fileName)
 	if file != nil {
 		defer file.Close()
 	}
 	if err != nil {
 		err := "UploadKerberosHub: Upload Failed, file doesn't exists anymore."
-		log.Log.Info(err)
+		log.Info(err)
 		return false, false, errors.New(err)
 	}
 
@@ -74,9 +94,11 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 	// There might be different reasons like (muted, read-only..)
 	req, err := http.NewRequest("HEAD", config.HubURI+"/storage/upload", nil)
 	if err != nil {
-		errorMessage := "UploadKerberosHub: error reading HEAD request, " + config.HubURI + "/storage: " + err.Error()
-		log.Log.Error(errorMessage)
-		return false, true, errors.New(errorMessage)
+		log.WithError(err).WithFields(log.Fields{
+			"component": "kerberos_hub",
+			"event":     "authorization_request_creation_failed",
+		}).Error("Failed to create Hub upload authorization request")
+		return false, true, fmt.Errorf("create Hub upload authorization request: %w", err)
 	}
 
 	req.Header.Set("X-Kerberos-Storage-FileName", fileName)
@@ -102,25 +124,31 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 		defer resp.Body.Close()
 	}
 
-	if err == nil {
-		if resp != nil {
-			if err == nil {
-				if resp.StatusCode == 200 {
-					log.Log.Info("UploadKerberosHub: Upload allowed using the credentials provided (" + config.HubKey + ", " + config.HubPrivateKey + ")")
-				} else {
-					log.Log.Info("UploadKerberosHub: Upload NOT allowed using the credentials provided (" + config.HubKey + ", " + config.HubPrivateKey + ")")
-					return false, true, nil
-				}
-			}
+	if err == nil && resp != nil {
+		if resp.StatusCode == 200 {
+			log.WithFields(log.Fields{
+				"component":   "kerberos_hub",
+				"event":       "upload_authorized",
+				"status_code": resp.StatusCode,
+			}).Debug("Hub upload authorized")
+		} else {
+			log.WithFields(log.Fields{
+				"component":   "kerberos_hub",
+				"event":       "upload_rejected",
+				"status_code": resp.StatusCode,
+			}).Warn("Hub upload rejected")
+			return false, true, nil
 		}
 	}
 
 	// Now we know we are allowed to upload to the hub, we can start uploading.
 	req, err = http.NewRequest("POST", config.HubURI+"/storage/upload", file)
 	if err != nil {
-		errorMessage := "UploadKerberosHub: error reading POST request, " + config.KStorage.URI + "/storage/upload: " + err.Error()
-		log.Log.Error(errorMessage)
-		return false, true, errors.New(errorMessage)
+		log.WithError(err).WithFields(log.Fields{
+			"component": "kerberos_hub",
+			"event":     "upload_request_creation_failed",
+		}).Error("Failed to create Hub upload request")
+		return false, true, fmt.Errorf("create Hub upload request: %w", err)
 	}
 	req.Header.Set("Content-Type", "video/mp4")
 	req.Header.Set("X-Kerberos-Storage-FileName", fileName)
@@ -140,19 +168,36 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 			body, err := ioutil.ReadAll(resp.Body)
 			if err == nil {
 				if resp.StatusCode == 200 {
-					log.Log.Info("UploadKerberosHub: Upload Finished, " + resp.Status + ".")
+					log.WithFields(log.Fields{
+						"component":   "kerberos_hub",
+						"event":       "upload_completed",
+						"status_code": resp.StatusCode,
+						"transport":   "http",
+					}).Info("Hub upload completed")
 					return true, true, nil
 				} else {
-					log.Log.Info("UploadKerberosHub: Upload Failed, " + resp.Status + ", " + string(body))
+					log.WithFields(log.Fields{
+						"component":      "kerberos_hub",
+						"event":          "upload_rejected",
+						"response_bytes": len(body),
+						"status_code":    resp.StatusCode,
+						"transport":      "http",
+					}).Warn("Hub upload rejected")
 					return false, true, nil
 				}
 			}
 		}
 	}
 
-	errorMessage := "UploadKerberosHub: Upload Failed, " + err.Error()
-	log.Log.Info(errorMessage)
-	return false, true, errors.New(errorMessage)
+	if err == nil {
+		err = errors.New("Hub upload failed without a response")
+	}
+	log.WithError(err).WithFields(log.Fields{
+		"component": "kerberos_hub",
+		"event":     "upload_failed",
+		"transport": "http",
+	}).Error("Hub upload failed")
+	return false, true, fmt.Errorf("Hub upload failed: %w", err)
 }
 
 // stripHubCredentialsOnCrossHostRedirect removes the custom Kerberos Hub

machinery/src/cloud/kerberos_vault.go

@@ -9,8 +9,9 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/utils"
+	log "github.com/sirupsen/logrus"
 )
 
 // We will count the number of retries we have done.
@@ -27,7 +28,7 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string) (
 		config.KStorage.Directory == "" ||
 		config.KStorage.URI == "" {
 		err := "UploadKerberosVault: Kerberos Vault not properly configured"
-		log.Log.Info(err)
+		log.Info(err)
 		return false, false, errors.New(err)
 	}
 
@@ -35,8 +36,26 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string) (
 	// This can happen when the file was already removed (e.g. cleanup, or an
 	// earlier successful upload). Skip it so the watcher drops the marker
 	// instead of retrying indefinitely.
-	if _, err := os.Stat("data/recordings/" + fileName); err != nil {
-		log.Log.Info("UploadKerberosVault: skipping " + fileName + ", file doesn't exist anymore")
+	filePath, err := utils.ResolveRecordingPath("data/recordings", fileName)
+	if err != nil {
+		if os.IsNotExist(err) {
+			log.Info("UploadKerberosVault: skipping " + fileName + ", file doesn't exist anymore")
+			return false, false, nil
+		}
+		log.WithError(err).Error("UploadKerberosVault: could not resolve recording path")
+		return false, true, err
+	}
+	info, err := os.Stat(filePath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			log.Info("UploadKerberosVault: skipping " + fileName + ", file doesn't exist anymore")
+			return false, false, nil
+		}
+		log.WithError(err).Error("UploadKerberosVault: could not stat recording")
+		return false, true, err
+	}
+	if info.Size() == 0 {
+		log.Warn("UploadKerberosVault: skipping " + fileName + ", recording is empty")
 		return false, false, nil
 	}
 
@@ -49,8 +68,11 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string) (
 	// - Number of changes
 	// - Token
 	// KerberosCloud, this means storage is disabled and proxy enabled.
-	log.Log.Info("UploadKerberosVault: Uploading to Kerberos Vault (" + config.KStorage.URI + ")")
-	log.Log.Info("UploadKerberosVault: Upload started for " + fileName)
+	log.WithFields(log.Fields{
+		"component": "kerberos_vault",
+		"event":     "upload_started",
+		"storage":   "primary",
+	}).Info("Vault upload started")
 
 	publicKey := config.KStorage.CloudKey
 	if config.HubKey != "" {
@@ -62,14 +84,27 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string) (
 		uploaded, responded, body, err := sendToVault(*config.KStorage, publicKey, config.Key, fileName, "UploadKerberosVault", "primary")
 		if uploaded {
 			kstorageRetryCount = 0
-			log.Log.Info("UploadKerberosVault: Upload Finished, " + body)
+			log.WithFields(log.Fields{
+				"component": "kerberos_vault",
+				"event":     "upload_completed",
+				"storage":   "primary",
+			}).Info("Vault upload completed")
 			return true, true, nil
 		}
 
 		if err != nil {
-			log.Log.Info("UploadKerberosVault: Upload Failed, " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component": "kerberos_vault",
+				"event":     "upload_failed",
+				"storage":   "primary",
+			}).Error("Vault upload failed")
 		} else {
-			log.Log.Info("UploadKerberosVault: Upload Failed, " + body)
+			log.WithFields(log.Fields{
+				"component":      "kerberos_vault",
+				"event":          "upload_rejected",
+				"response_bytes": len(body),
+				"storage":        "primary",
+			}).Warn("Vault upload rejected")
 		}
 
 		// We only advance the retry policy when the vault gave a definitive
@@ -91,26 +126,43 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string) (
 		config.KStorageSecondary.SecretAccessKey == "" ||
 		config.KStorageSecondary.Directory == "" ||
 		config.KStorageSecondary.URI == "" {
-		log.Log.Info("UploadKerberosVault (Secondary): Secondary Kerberos Vault not properly configured.")
+		log.Info("UploadKerberosVault (Secondary): Secondary Kerberos Vault not properly configured.")
 	} else {
 
 		if kstorageRetryCount < config.KStorage.MaxRetries {
-			log.Log.Info("UploadKerberosVault (Secondary): Do not upload to secondary storage, we are still in retry policy.")
+			log.Info("UploadKerberosVault (Secondary): Do not upload to secondary storage, we are still in retry policy.")
 			return false, true, nil
 		}
 
-		log.Log.Info("UploadKerberosVault (Secondary): Uploading to Secondary Kerberos Vault (" + config.KStorageSecondary.URI + ")")
+		log.WithFields(log.Fields{
+			"component": "kerberos_vault",
+			"event":     "upload_started",
+			"storage":   "secondary",
+		}).Info("Vault upload started")
 
 		uploaded, _, body, err := sendToVault(*config.KStorageSecondary, publicKey, config.Key, fileName, "UploadKerberosVault (Secondary)", "secondary")
 		if uploaded {
-			log.Log.Info("UploadKerberosVault (Secondary): Upload Finished to secondary, " + body)
+			log.WithFields(log.Fields{
+				"component": "kerberos_vault",
+				"event":     "upload_completed",
+				"storage":   "secondary",
+			}).Info("Vault upload completed")
 			return true, true, nil
 		}
 
 		if err != nil {
-			log.Log.Info("UploadKerberosVault (Secondary): Upload Failed to secondary, " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component": "kerberos_vault",
+				"event":     "upload_failed",
+				"storage":   "secondary",
+			}).Error("Vault upload failed")
 		} else {
-			log.Log.Info("UploadKerberosVault (Secondary): Upload Failed to secondary, " + body)
+			log.WithFields(log.Fields{
+				"component":      "kerberos_vault",
+				"event":          "upload_rejected",
+				"response_bytes": len(body),
+				"storage":        "secondary",
+			}).Warn("Vault upload rejected")
 		}
 	}
 
@@ -131,7 +183,7 @@ func sendToVault(vault models.KStorage, publicKey, deviceKey, fileName, label, s
 		if supported {
 			return uploaded, responded, body, err
 		}
-		log.Log.Info(label + ": resumable (tus) endpoint not available, falling back to legacy upload")
+		log.Info(label + ": resumable (tus) endpoint not available, falling back to legacy upload")
 	}
 	return uploadVaultLegacy(vault, publicKey, deviceKey, fileName, label)
 }
@@ -140,15 +192,17 @@ func sendToVault(vault models.KStorage, publicKey, deviceKey, fileName, label, s
 // is sent as the body of a POST to {URI}/storage. Kept for backwards
 // compatibility with vault deployments that do not support resumable uploads.
 func uploadVaultLegacy(vault models.KStorage, publicKey, deviceKey, fileName, label string) (bool, bool, string, error) {
-	fullname := "data/recordings/" + fileName
-
-	file, err := os.Open(fullname)
+	customHeaders, err := parseCustomVaultHeaders(vault.CustomHeaders)
+	if err != nil {
+		return false, false, "", err
+	}
+	file, err := utils.OpenRecording("data/recordings", fileName)
 	if file != nil {
 		defer file.Close()
 	}
 	if err != nil {
 		msg := label + ": Upload Failed, file doesn't exists anymore"
-		log.Log.Info(msg)
+		log.Info(msg)
 		return false, false, "", errors.New(msg)
 	}
 
@@ -160,11 +214,14 @@ func uploadVaultLegacy(vault models.KStorage, publicKey, deviceKey, fileName, la
 	req, err := http.NewRequest("POST", uri+"/storage", file)
 	if err != nil {
 		errorMessage := label + ": error reading request, " + uri + "/storage: " + err.Error()
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, false, "", errors.New(errorMessage)
 	}
 	req.Header.Set("Content-Type", "video/mp4")
 	setVaultHeaders(req.Header, vault, publicKey, deviceKey, fileName)
+	if err := setCustomVaultHeaders(req.Header, customHeaders); err != nil {
+		return false, false, "", err
+	}
 	setQueuedRecordingMetadataHeaders(req.Header, fileName)
 
 	client := newVaultHTTPClient(0)
@@ -229,7 +286,7 @@ func newVaultHTTPClient(timeout time.Duration) *http.Client {
 		transport.TLSClientConfig.InsecureSkipVerify = true
 	}
 
-	client := &http.Client{Transport: transport}
+	client := &http.Client{Transport: transport, CheckRedirect: stripVaultHeadersOnCrossHostRedirect}
 	if timeout > 0 {
 		client.Timeout = timeout
 	}

machinery/src/cloud/livehls.go

@@ -7,9 +7,9 @@ import (
 	mqtt "github.com/eclipse/paho.mqtt.golang"
 
 	"github.com/kerberos-io/agent/machinery/src/cloud/livehls"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
+	log "github.com/sirupsen/logrus"
 )
 
 // hlsViewerTimeoutSeconds is how long the agent keeps shipping live HLS segments
@@ -45,22 +45,29 @@ const hlsReadyReannounceSeconds = 2
 // trades a little idle CPU for a near-instant "requesting stream", so viewers no
 // longer wait a full GOP for the first segment to be cut. Set
 // AGENT_LIVE_HLS_PREWARM=false to fall back to the lazy on-demand path above.
-func HandleLiveStreamHLS(configuration *models.Configuration, communication *models.Communication, mqttClient mqtt.Client, subStreamEnabled bool) {
+func HandleLiveStreamHLS(
+	configuration *models.Configuration,
+	communication *models.Communication,
+	mqttClient mqtt.Client,
+	subStreamEnabled bool,
+	mainQueue *packets.Queue,
+	subQueue *packets.Queue,
+) {
 
-	log.Log.Debug("cloud.HandleLiveStreamHLS(): started")
+	log.Debug("cloud.HandleLiveStreamHLS(): started")
 
 	config := configuration.Config
 
 	if config.Offline == "true" {
-		log.Log.Debug("cloud.HandleLiveStreamHLS(): stopping as Offline is enabled.")
+		log.Debug("cloud.HandleLiveStreamHLS(): stopping as Offline is enabled.")
 		return
 	}
 	if config.Capture.Liveview == "false" {
-		log.Log.Debug("cloud.HandleLiveStreamHLS(): stopping as Liveview is disabled.")
+		log.Debug("cloud.HandleLiveStreamHLS(): stopping as Liveview is disabled.")
 		return
 	}
 	if config.HubURI == "" || config.HubKey == "" {
-		log.Log.Debug("cloud.HandleLiveStreamHLS(): stopping as the Hub is not configured (HubURI/HubKey).")
+		log.Debug("cloud.HandleLiveStreamHLS(): stopping as the Hub is not configured (HubURI/HubKey).")
 		return
 	}
 
@@ -88,8 +95,8 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 	// mp4ff's strict parser rejects).
 	requestedQuality := models.StreamQualityAuto
 	useSub := models.SelectSubStreamForQuality(config, requestedQuality, subStreamEnabled)
-	source := buildHLSSource(config, communication, useSub)
-	log.Log.Info("cloud.HandleLiveStreamHLS(): serving live HLS from the " + source.label + " stream")
+	source := buildHLSSource(config, mainQueue, subQueue, useSub)
+	log.Info("cloud.HandleLiveStreamHLS(): serving live HLS from the " + source.label + " stream")
 
 	// prewarm keeps a single long-lived session muxing into an in-memory ring
 	// buffer while idle and flushes it the instant a viewer arrives, eliminating
@@ -97,9 +104,9 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 	// to fall back to the lazy on-demand path.
 	prewarm := os.Getenv("AGENT_LIVE_HLS_PREWARM") != "false"
 	if prewarm {
-		log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS prewarm ENABLED (set AGENT_LIVE_HLS_PREWARM=false to disable)")
+		log.Info("cloud.HandleLiveStreamHLS(): live HLS prewarm ENABLED (set AGENT_LIVE_HLS_PREWARM=false to disable)")
 	} else {
-		log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS prewarm DISABLED (AGENT_LIVE_HLS_PREWARM=false)")
+		log.Info("cloud.HandleLiveStreamHLS(): live HLS prewarm DISABLED (AGENT_LIVE_HLS_PREWARM=false)")
 	}
 
 	// lowLatency enables LL-HLS: each segment is sliced into CMAF parts shipped the
@@ -109,9 +116,9 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 	partTargetMs := uint64(0)
 	if os.Getenv("AGENT_LIVE_HLS_LOW_LATENCY") != "false" {
 		partTargetMs = livehls.DefaultPartTargetMs
-		log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS low-latency (LL-HLS) ENABLED (set AGENT_LIVE_HLS_LOW_LATENCY=false to disable)")
+		log.Info("cloud.HandleLiveStreamHLS(): live HLS low-latency (LL-HLS) ENABLED (set AGENT_LIVE_HLS_LOW_LATENCY=false to disable)")
 	} else {
-		log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS low-latency (LL-HLS) DISABLED (AGENT_LIVE_HLS_LOW_LATENCY=false)")
+		log.Info("cloud.HandleLiveStreamHLS(): live HLS low-latency (LL-HLS) DISABLED (AGENT_LIVE_HLS_LOW_LATENCY=false)")
 	}
 
 	var session *livehls.Session
@@ -123,6 +130,9 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 
 	for cursorError == nil {
 		pkt, cursorError = source.cursor.ReadPacket()
+		if cursorError != nil {
+			break
+		}
 
 		now := time.Now().Unix()
 		select {
@@ -156,9 +166,9 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 				_ = session.Close()
 				session = nil
 			}
-			source = buildHLSSource(config, communication, useSub)
+			source = buildHLSSource(config, mainQueue, subQueue, useSub)
 			lastReadyAnnounce = 0
-			log.Log.Info("cloud.HandleLiveStreamHLS(): switched live HLS to the " + source.label + " stream (quality=" + requestedQuality + ")")
+			log.Info("cloud.HandleLiveStreamHLS(): switched live HLS to the " + source.label + " stream (quality=" + requestedQuality + ")")
 			continue
 		}
 
@@ -184,11 +194,22 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 					StartBuffering: true,
 				})
 				session.SetOnReady(func(sessionID string) {
-					log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS session ready, announcing " + sessionID)
+					log.WithFields(log.Fields{
+						"component":  "cloud/livehls",
+						"event":      "session_ready",
+						"session_id": sessionID,
+					}).Info("Live HLS session ready")
 					publishHLSReady(configuration, mqttClient, hubKey, deviceId, sessionID)
 					lastReadyAnnounce = time.Now().Unix()
 				})
-				log.Log.Info("cloud.HandleLiveStreamHLS(): prewarming live HLS session " + session.SessionID())
+				session.SetOnFailure(func(sessionID, reason string) {
+					publishHLSFailure(configuration, mqttClient, hubKey, deviceId, sessionID, reason)
+				})
+				log.WithFields(log.Fields{
+					"component":  "cloud/livehls",
+					"event":      "session_prewarming",
+					"session_id": session.SessionID(),
+				}).Info("Prewarming live HLS session")
 			}
 
 			if viewerActive {
@@ -207,7 +228,7 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 
 			if len(pkt.Data) > 0 && pkt.IsVideo {
 				if err := session.WritePacket(pkt); err != nil {
-					log.Log.Error("cloud.HandleLiveStreamHLS(): " + err.Error())
+					log.Error("cloud.HandleLiveStreamHLS(): " + err.Error())
 				}
 			}
 			continue
@@ -217,7 +238,12 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 			// No viewer: stop and discard the session so we stop shipping segments.
 			if session != nil {
 				_ = session.Close()
-				log.Log.Info("cloud.HandleLiveStreamHLS(): no active viewers, stopped live HLS session " + session.SessionID())
+				log.WithFields(log.Fields{
+					"component":  "cloud/livehls",
+					"event":      "session_stopped",
+					"reason":     "no_active_viewers",
+					"session_id": session.SessionID(),
+				}).Info("Live HLS session stopped")
 				session = nil
 			}
 			continue
@@ -243,22 +269,33 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 				PartTargetMs: partTargetMs,
 			})
 			session.SetOnReady(func(sessionID string) {
-				log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS session ready, announcing " + sessionID)
+				log.WithFields(log.Fields{
+					"component":  "cloud/livehls",
+					"event":      "session_ready",
+					"session_id": sessionID,
+				}).Info("Live HLS session ready")
 				publishHLSReady(configuration, mqttClient, hubKey, deviceId, sessionID)
 				lastReadyAnnounce = time.Now().Unix()
 			})
-			log.Log.Info("cloud.HandleLiveStreamHLS(): started live HLS session " + session.SessionID())
+			session.SetOnFailure(func(sessionID, reason string) {
+				publishHLSFailure(configuration, mqttClient, hubKey, deviceId, sessionID, reason)
+			})
+			log.WithFields(log.Fields{
+				"component":  "cloud/livehls",
+				"event":      "session_started",
+				"session_id": session.SessionID(),
+			}).Info("Live HLS session started")
 		}
 
 		if err := session.WritePacket(pkt); err != nil {
-			log.Log.Error("cloud.HandleLiveStreamHLS(): " + err.Error())
+			log.Error("cloud.HandleLiveStreamHLS(): " + err.Error())
 		}
 	}
 
 	if session != nil {
 		_ = session.Close()
 	}
-	log.Log.Debug("cloud.HandleLiveStreamHLS(): finished")
+	log.Debug("cloud.HandleLiveStreamHLS(): finished")
 }
 
 // publishHLSReady announces, over MQTT, that a live HLS session is available so
@@ -278,9 +315,48 @@ func publishHLSReady(configuration *models.Configuration, mqttClient mqtt.Client
 	payload, err := models.PackageMQTTMessage(configuration, message)
 	if err == nil {
 		mqttClient.Publish("kerberos/hub/"+hubKey, 0, false, payload)
-		log.Log.Info("cloud.HandleLiveStreamHLS(): announced live HLS session " + sessionID)
+		log.WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "session_announced",
+			"session_id": sessionID,
+		}).Info("Live HLS session announced")
 	} else {
-		log.Log.Error("cloud.HandleLiveStreamHLS(): failed to package receive-hls-ready message: " + err.Error())
+		log.WithError(err).WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "session_announcement_failed",
+			"session_id": sessionID,
+		}).Error("Failed to package live HLS session announcement")
+	}
+}
+
+func publishHLSFailure(configuration *models.Configuration, mqttClient mqtt.Client, hubKey, deviceId, sessionID, reason string) {
+	valueMap := map[string]interface{}{
+		"session": sessionID,
+		"device":  deviceId,
+		"reason":  reason,
+	}
+	message := models.Message{
+		Payload: models.Payload{
+			Action:   "receive-hls-error",
+			DeviceId: deviceId,
+			Value:    valueMap,
+		},
+	}
+	payload, err := models.PackageMQTTMessage(configuration, message)
+	if err == nil {
+		mqttClient.Publish("kerberos/hub/"+hubKey, 0, false, payload)
+		log.WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "session_failure_announced",
+			"reason":     reason,
+			"session_id": sessionID,
+		}).Warn("Live HLS session failure announced")
+	} else {
+		log.WithError(err).WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "session_failure_announcement_failed",
+			"session_id": sessionID,
+		}).Error("Failed to package live HLS session failure")
 	}
 }
 
@@ -302,11 +378,11 @@ type hlsStreamSource struct {
 // is available; otherwise the main (high-resolution) stream is used. A fresh
 // Latest() cursor is created so muxing resumes from the live edge of the chosen
 // stream after a switch.
-func buildHLSSource(config models.Config, communication *models.Communication, useSub bool) hlsStreamSource {
+func buildHLSSource(config models.Config, mainQueue, subQueue *packets.Queue, useSub bool) hlsStreamSource {
 	cam := config.Capture.IPCamera
-	if useSub && communication.SubQueue != nil {
+	if useSub && subQueue != nil {
 		return hlsStreamSource{
-			cursor: communication.SubQueue.Latest(),
+			cursor: subQueue.Latest(),
 			sps:    cam.SubSPSNALUs,
 			pps:    cam.SubPPSNALUs,
 			vps:    cam.SubVPSNALUs,
@@ -316,7 +392,7 @@ func buildHLSSource(config models.Config, communication *models.Communication, u
 		}
 	}
 	return hlsStreamSource{
-		cursor: communication.Queue.Latest(),
+		cursor: mainQueue.Latest(),
 		sps:    cam.SPSNALUs,
 		pps:    cam.PPSNALUs,
 		vps:    cam.VPSNALUs,

machinery/src/cloud/livehls/publisher.go

@@ -24,13 +24,14 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"net/http"
 	"strconv"
 	"strings"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/video"
+	log "github.com/sirupsen/logrus"
 )
 
 const (
@@ -67,6 +68,7 @@ const (
 	// cannot be delivered within roughly its own duration is stale, so the upload
 	// is abandoned (dropped) rather than allowed to back up the pipeline.
 	defaultPublishTimeout = 4 * time.Second
+	maxErrorResponseBytes = 4 << 10
 )
 
 // PublisherConfig carries the hub endpoint and credentials needed to ship live
@@ -214,9 +216,15 @@ func (p *Publisher) post(ctx context.Context, params postParams) error {
 	defer resp.Body.Close()
 
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		return fmt.Errorf("livehls: upload %s rejected: %s", params.name, resp.Status)
+		body, _ := io.ReadAll(io.LimitReader(resp.Body, maxErrorResponseBytes))
+		return fmt.Errorf("livehls: upload %s rejected with status %d (%d response bytes)", params.name, resp.StatusCode, len(body))
 	}
-	log.Log.Debug("livehls.Publisher.post(): shipped " + params.name + " for session " + params.sessionID)
+	log.WithFields(log.Fields{
+		"component":  "cloud/livehls",
+		"event":      "object_uploaded",
+		"object":     params.name,
+		"session_id": params.sessionID,
+	}).Trace("Live HLS object uploaded")
 	return nil
 }
 

machinery/src/cloud/livehls/publisher_test.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"sync"
 	"testing"
 	"time"
@@ -144,12 +145,22 @@ func TestPublisherPublishSegmentSendsSequenceAndDuration(t *testing.T) {
 }
 
 func TestPublisherReturnsErrorOnNon2xx(t *testing.T) {
-	srv, _, _ := newCapturingServer(t, http.StatusInternalServerError)
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusBadRequest)
+		_, _ = w.Write([]byte(`{"error":true,"data":"No user found with this public and private key."}`))
+	}))
+	t.Cleanup(srv.Close)
 	p := testPublisher(srv.URL)
 
 	err := p.PublishSegment(context.Background(), "s", video.LiveSegment{SequenceNumber: 1, Data: []byte("x")})
 	if err == nil {
-		t.Fatal("expected an error on 500 response")
+		t.Fatal("expected an error on 400 response")
+	}
+	if !strings.Contains(err.Error(), "status 400") || !strings.Contains(err.Error(), "71 response bytes") {
+		t.Fatalf("error = %q, want status code and response size", err)
+	}
+	if strings.Contains(err.Error(), "No user found with this public and private key") {
+		t.Fatalf("error exposed Hub response body: %q", err)
 	}
 }
 
@@ -281,7 +292,13 @@ func TestSessionRetriesInitWhenFirstAttemptFails(t *testing.T) {
 	})
 
 	var ready int
+	var failures int
+	var failureReason string
 	sess.SetOnReady(func(string) { ready++ })
+	sess.SetOnFailure(func(_ string, reason string) {
+		failures++
+		failureReason = reason
+	})
 
 	for i := 0; i < 60; i++ {
 		isKey := i%25 == 0
@@ -304,6 +321,9 @@ func TestSessionRetriesInitWhenFirstAttemptFails(t *testing.T) {
 	if ready != 1 {
 		t.Errorf("OnReady fired %d times, want 1", ready)
 	}
+	if failures != 1 || failureReason != "init-upload-failed" {
+		t.Errorf("OnFailure = %d/%q, want 1/init-upload-failed", failures, failureReason)
+	}
 }
 
 // liveTestSPSForSession is the known-good baseline SPS reused across tests.

machinery/src/cloud/livehls/session.go

@@ -8,9 +8,9 @@ import (
 	"sync"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/kerberos-io/agent/machinery/src/video"
+	log "github.com/sirupsen/logrus"
 )
 
 // DefaultTargetSegmentMs is the nominal live segment length. ~2s keeps standard
@@ -46,9 +46,11 @@ type Session struct {
 	// lastInitAt is when the init segment was last (re)uploaded. The init is
 	// re-sent periodically so its short TTL in the hub live window never lapses
 	// mid-session; see refreshInitIfStale.
-	lastInitAt time.Time
-	readyFired bool
-	onReady    func(sessionID string)
+	lastInitAt   time.Time
+	readyFired   bool
+	onReady      func(sessionID string)
+	failureFired bool
+	onFailure    func(sessionID, reason string)
 
 	// uploadsActive gates whether the init and completed segments are shipped to
 	// hub-api. It is true for the default on-demand path. The prewarm path starts
@@ -142,14 +144,25 @@ func NewSession(publisher *Publisher, opts SessionOptions) *Session {
 			return nil
 		}
 		if !s.publishInitIfNeeded() {
-			log.Log.Warning("livehls.Session: dropping segment " +
-				fmt.Sprintf("%d", segment.SequenceNumber) + " because init has not been delivered yet")
+			log.WithFields(log.Fields{
+				"component":        "cloud/livehls",
+				"event":            "segment_dropped",
+				"reason":           "init_unavailable",
+				"segment_sequence": segment.SequenceNumber,
+				"session_id":       s.id,
+			}).Warn("Dropping live HLS segment")
 			return nil
 		}
 		ctx, cancel := s.newContext()
 		defer cancel()
 		if err := s.publisher.PublishSegment(ctx, s.id, segment); err != nil {
-			log.Log.Warning("livehls.Session: " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component":        "cloud/livehls",
+				"event":            "segment_upload_failed",
+				"segment_sequence": segment.SequenceNumber,
+				"session_id":       s.id,
+			}).Warn("Failed to upload live HLS segment")
+			s.fireFailureOnce("segment-upload-failed")
 			return nil
 		}
 		s.fireReadyOnce()
@@ -172,15 +185,27 @@ func NewSession(publisher *Publisher, opts SessionOptions) *Session {
 				return nil
 			}
 			if !s.publishInitIfNeeded() {
-				log.Log.Warning("livehls.Session: dropping part " +
-					fmt.Sprintf("%d.%d", part.SegmentSeq, part.PartIndex) +
-					" because init has not been delivered yet")
+				log.WithFields(log.Fields{
+					"component":        "cloud/livehls",
+					"event":            "part_dropped",
+					"part_index":       part.PartIndex,
+					"reason":           "init_unavailable",
+					"segment_sequence": part.SegmentSeq,
+					"session_id":       s.id,
+				}).Warn("Dropping live HLS part")
 				return nil
 			}
 			ctx, cancel := s.newContext()
 			defer cancel()
 			if err := s.publisher.PublishPart(ctx, s.id, part); err != nil {
-				log.Log.Warning("livehls.Session: " + err.Error())
+				log.WithError(err).WithFields(log.Fields{
+					"component":        "cloud/livehls",
+					"event":            "part_upload_failed",
+					"part_index":       part.PartIndex,
+					"segment_sequence": part.SegmentSeq,
+					"session_id":       s.id,
+				}).Warn("Failed to upload live HLS part")
+				s.fireFailureOnce("part-upload-failed")
 				return nil
 			}
 			s.fireReadyOnce()
@@ -216,6 +241,15 @@ func (s *Session) SetOnReady(fn func(sessionID string)) {
 	s.mu.Unlock()
 }
 
+// SetOnFailure registers a one-shot callback for startup upload failures. The
+// reason is a fixed code rather than an HTTP response body, so credentials or
+// server details cannot leak through MQTT diagnostics.
+func (s *Session) SetOnFailure(fn func(sessionID, reason string)) {
+	s.mu.Lock()
+	s.onFailure = fn
+	s.mu.Unlock()
+}
+
 // prewarmMaxBufferedSegments is how many of the most recent completed segments
 // the prewarm path keeps in memory while idle and flushes to a viewer on arrival.
 // One segment keeps startup instant (the viewer immediately gets a playable
@@ -268,7 +302,13 @@ func (s *Session) SetUploadsActive(active bool) bool {
 		}
 		ctx, cancel := s.newContext()
 		if err := s.publisher.PublishSegment(ctx, s.id, buffered[i]); err != nil {
-			log.Log.Warning("livehls.Session: prewarm flush: " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component":        "cloud/livehls",
+				"event":            "prewarm_segment_upload_failed",
+				"segment_sequence": buffered[i].SequenceNumber,
+				"session_id":       s.id,
+			}).Warn("Failed to upload buffered live HLS segment")
+			s.fireFailureOnce("segment-upload-failed")
 			cancel()
 			continue
 		}
@@ -284,7 +324,14 @@ func (s *Session) SetUploadsActive(active bool) bool {
 		}
 		ctx, cancel := s.newContext()
 		if err := s.publisher.PublishPart(ctx, s.id, bufferedParts[i]); err != nil {
-			log.Log.Warning("livehls.Session: prewarm flush (part): " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component":        "cloud/livehls",
+				"event":            "prewarm_part_upload_failed",
+				"part_index":       bufferedParts[i].PartIndex,
+				"segment_sequence": bufferedParts[i].SegmentSeq,
+				"session_id":       s.id,
+			}).Warn("Failed to upload buffered live HLS part")
+			s.fireFailureOnce("part-upload-failed")
 			cancel()
 			continue
 		}
@@ -381,7 +428,12 @@ func (s *Session) publishInitIfNeeded() bool {
 	ctx, cancel := s.newContext()
 	defer cancel()
 	if err := s.publisher.PublishInit(ctx, s.id, initBytes); err != nil {
-		log.Log.Warning("livehls.Session: init upload failed, will retry: " + err.Error())
+		log.WithError(err).WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "init_upload_failed",
+			"session_id": s.id,
+		}).Warn("Failed to upload live HLS init segment; retrying")
+		s.fireFailureOnce("init-upload-failed")
 		return false
 	}
 
@@ -389,7 +441,11 @@ func (s *Session) publishInitIfNeeded() bool {
 	s.initPublished = true
 	s.lastInitAt = time.Now()
 	s.mu.Unlock()
-	log.Log.Info("livehls.Session: init segment delivered for session " + s.id)
+	log.WithFields(log.Fields{
+		"component":  "cloud/livehls",
+		"event":      "init_uploaded",
+		"session_id": s.id,
+	}).Info("Live HLS init segment uploaded")
 	return true
 }
 
@@ -422,14 +478,22 @@ func (s *Session) refreshInitIfStale() {
 	ctx, cancel := s.newContext()
 	defer cancel()
 	if err := s.publisher.PublishInit(ctx, s.id, initBytes); err != nil {
-		log.Log.Warning("livehls.Session: init refresh failed, will retry: " + err.Error())
+		log.WithError(err).WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "init_refresh_failed",
+			"session_id": s.id,
+		}).Warn("Failed to refresh live HLS init segment; retrying")
 		return
 	}
 
 	s.mu.Lock()
 	s.lastInitAt = time.Now()
 	s.mu.Unlock()
-	log.Log.Debug("livehls.Session: refreshed init segment TTL for session " + s.id)
+	log.WithFields(log.Fields{
+		"component":  "cloud/livehls",
+		"event":      "init_refreshed",
+		"session_id": s.id,
+	}).Debug("Live HLS init segment refreshed")
 }
 
 // fireReadyOnce invokes the OnReady callback the first time it is called.
@@ -445,6 +509,18 @@ func (s *Session) fireReadyOnce() {
 	fn(s.id)
 }
 
+func (s *Session) fireFailureOnce(reason string) {
+	s.mu.Lock()
+	if s.failureFired || s.readyFired || s.onFailure == nil {
+		s.mu.Unlock()
+		return
+	}
+	s.failureFired = true
+	fn := s.onFailure
+	s.mu.Unlock()
+	fn(s.id, reason)
+}
+
 // newSessionID returns a short, unique, URL-safe session identifier of the form
 // <unix-seconds>-<random-hex>.
 func newSessionID() string {

machinery/src/cloud/livemoq/recovery.go

@@ -1,6 +1,9 @@
 package livemoq
 
-import "time"
+import (
+	"sync/atomic"
+	"time"
+)
 
 type FrameGateEvent uint8
 
@@ -17,9 +20,52 @@ type FrameGate struct {
 	recovering bool
 }
 
-// Reset closes the gate so publication resumes on the next keyframe. It is used
-// when the publisher stopped writing for a reason unrelated to the stream health
-// (no subscribers), so the next viewer never receives a partial GOP.
+// AudienceGate publishes every frame while watched and only keyframes while
+// idle. The idle keyframes keep the relay's latest cached group current without
+// paying the bandwidth cost of a full stream when nobody is watching.
+type AudienceGate struct {
+	idle bool
+}
+
+func (g *AudienceGate) Allow(hasSubscribers bool, isKeyFrame bool) (allowed bool, enteredIdle bool) {
+	if hasSubscribers {
+		g.idle = false
+		return true, false
+	}
+
+	enteredIdle = !g.idle
+	g.idle = true
+	return isKeyFrame, enteredIdle
+}
+
+// WriteWatchdog tracks the single synchronous frame write performed by a MoQ
+// publisher so another goroutine can interrupt a wedged native call.
+type WriteWatchdog struct {
+	startedAt atomic.Int64
+}
+
+func (w *WriteWatchdog) Begin(now time.Time) {
+	w.startedAt.Store(now.UnixNano())
+}
+
+func (w *WriteWatchdog) End() {
+	w.startedAt.Store(0)
+}
+
+func (w *WriteWatchdog) Elapsed(now time.Time) (time.Duration, bool) {
+	startedAt := w.startedAt.Load()
+	if startedAt == 0 {
+		return 0, false
+	}
+	elapsed := now.Sub(time.Unix(0, startedAt))
+	if elapsed < 0 {
+		elapsed = 0
+	}
+	return elapsed, true
+}
+
+// Reset closes the gate so publication resumes on the next keyframe. Entering
+// idle mode uses it to start the relay-cache refresh on a complete GOP.
 func (g *FrameGate) Reset() {
 	g.started = false
 	g.recovering = false

machinery/src/cloud/livemoq/recovery_test.go

@@ -47,3 +47,45 @@ func TestFrameGateAllowsMissingCaptureTime(t *testing.T) {
 		t.Fatalf("Allow() = (%t, %d), want (true, %d)", allowed, event, FrameGateEventStarted)
 	}
 }
+
+func TestAudienceGateRefreshesOnlyKeyframesWhileIdle(t *testing.T) {
+	gate := AudienceGate{}
+
+	if allowed, enteredIdle := gate.Allow(true, false); !allowed || enteredIdle {
+		t.Fatalf("active delta = (%t, %t), want (true, false)", allowed, enteredIdle)
+	}
+	if allowed, enteredIdle := gate.Allow(false, false); allowed || !enteredIdle {
+		t.Fatalf("first idle delta = (%t, %t), want (false, true)", allowed, enteredIdle)
+	}
+	if allowed, enteredIdle := gate.Allow(false, true); !allowed || enteredIdle {
+		t.Fatalf("idle keyframe = (%t, %t), want (true, false)", allowed, enteredIdle)
+	}
+	if allowed, enteredIdle := gate.Allow(false, false); allowed || enteredIdle {
+		t.Fatalf("later idle delta = (%t, %t), want (false, false)", allowed, enteredIdle)
+	}
+	if allowed, enteredIdle := gate.Allow(true, false); !allowed || enteredIdle {
+		t.Fatalf("resumed delta = (%t, %t), want (true, false)", allowed, enteredIdle)
+	}
+	if allowed, enteredIdle := gate.Allow(false, true); !allowed || !enteredIdle {
+		t.Fatalf("next idle keyframe = (%t, %t), want (true, true)", allowed, enteredIdle)
+	}
+}
+
+func TestWriteWatchdogTracksActiveWrite(t *testing.T) {
+	now := time.Unix(10, 0)
+	watchdog := WriteWatchdog{}
+
+	if elapsed, active := watchdog.Elapsed(now); active || elapsed != 0 {
+		t.Fatalf("Elapsed() before Begin() = (%s, %t), want (0, false)", elapsed, active)
+	}
+
+	watchdog.Begin(now)
+	if elapsed, active := watchdog.Elapsed(now.Add(5 * time.Second)); !active || elapsed != 5*time.Second {
+		t.Fatalf("Elapsed() during write = (%s, %t), want (5s, true)", elapsed, active)
+	}
+
+	watchdog.End()
+	if elapsed, active := watchdog.Elapsed(now.Add(6 * time.Second)); active || elapsed != 0 {
+		t.Fatalf("Elapsed() after End() = (%s, %t), want (0, false)", elapsed, active)
+	}
+}

machinery/src/cloud/livemoq_disabled.go

@@ -2,7 +2,20 @@
 
 package cloud
 
-import "github.com/kerberos-io/agent/machinery/src/models"
+import (
+	"context"
+
+	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/packets"
+)
 
 // StartLiveStreamMoQ is disabled in the standard Agent build.
-func StartLiveStreamMoQ(_ *models.Configuration, _ *models.Communication, _ bool) {}
+func StartLiveStreamMoQ(
+	_ context.Context,
+	_ *models.Configuration,
+	_ *models.Communication,
+	_ bool,
+	_ *packets.Queue,
+	_ *packets.Queue,
+) {
+}

machinery/src/cloud/livemoq_enabled.go

@@ -4,41 +4,120 @@ package cloud
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"net/url"
 	"os"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/cloud/livemoq"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/moq-dev/moq-go/moq"
+	log "github.com/sirupsen/logrus"
 )
 
 const (
 	defaultMoQRelayURL      = "https://relay.uug.ai/anon"
 	minMoQRetryDelay        = time.Second
 	maxMoQRetryDelay        = 30 * time.Second
-	maxMoQLivePacketAge     = 1500 * time.Millisecond
+	defaultMoQLivePacketAge = 1500 * time.Millisecond
+	minMoQLivePacketAge     = 250 * time.Millisecond
+	maxMoQLivePacketAge     = 30 * time.Second
+	defaultMoQWriteTimeout  = 5 * time.Second
+	minMoQWriteTimeout      = time.Second
+	maxMoQWriteTimeout      = time.Minute
+	moQWriteWatchInterval   = 250 * time.Millisecond
 	slowMoQWriteThreshold   = 100 * time.Millisecond
 	moQWriteWarningInterval = 10 * time.Second
 	duplicateKeyframeWindow = 500 * time.Millisecond
 )
 
 type liveMoQConfig struct {
-	relayURL    string
-	broadcast   string
-	quality     string
-	sourceLabel string
-	queue       *packets.Queue
+	relayURL      string
+	broadcast     string
+	quality       string
+	sourceLabel   string
+	queue         *packets.Queue
+	communication *models.Communication
+	maxPacketAge  time.Duration
+	writeTimeout  time.Duration
 }
 
-// label identifies the tier in log lines, since one Agent runs a publisher per
-// quality tier.
-func (c liveMoQConfig) label() string {
-	return c.quality + " (" + c.sourceLabel + " stream)"
+func (c liveMoQConfig) logEntry(event string) *log.Entry {
+	return log.WithFields(log.Fields{
+		"component": "moq",
+		"event":     event,
+		"quality":   c.quality,
+		"stream":    c.sourceLabel,
+	})
+}
+
+func moQRelayHost(relayURL string) string {
+	parsed, err := url.Parse(relayURL)
+	if err != nil {
+		return ""
+	}
+	return parsed.Host
+}
+
+func (c liveMoQConfig) packetAgeLimit() time.Duration {
+	if c.maxPacketAge > 0 {
+		return c.maxPacketAge
+	}
+	return defaultMoQLivePacketAge
+}
+
+func (c liveMoQConfig) writeTimeoutLimit() time.Duration {
+	if c.writeTimeout > 0 {
+		return c.writeTimeout
+	}
+	return defaultMoQWriteTimeout
+}
+
+func boundedMoQDuration(name string, fallback, minimum, maximum time.Duration) time.Duration {
+	raw := strings.TrimSpace(os.Getenv(name))
+	if raw == "" {
+		return fallback
+	}
+
+	value, err := time.ParseDuration(raw)
+	if err != nil {
+		log.WithError(err).WithFields(log.Fields{
+			"component":        "moq",
+			"configured_value": raw,
+			"effective_value":  fallback.String(),
+			"event":            "duration_invalid",
+			"variable":         name,
+		}).Warn("Invalid MoQ duration; using default")
+		return fallback
+	}
+	if value < minimum {
+		log.WithFields(log.Fields{
+			"component":        "moq",
+			"configured_value": value.String(),
+			"effective_value":  minimum.String(),
+			"event":            "duration_clamped",
+			"minimum":          minimum.String(),
+			"variable":         name,
+		}).Warn("MoQ duration is below the supported minimum")
+		return minimum
+	}
+	if value > maximum {
+		log.WithFields(log.Fields{
+			"component":        "moq",
+			"configured_value": value.String(),
+			"effective_value":  maximum.String(),
+			"event":            "duration_clamped",
+			"maximum":          maximum.String(),
+			"variable":         name,
+		}).Warn("MoQ duration exceeds the supported maximum")
+		return maximum
+	}
+	return value
 }
 
 // StartLiveStreamMoQ starts the publisher only in the dedicated MoQ build and
@@ -50,20 +129,37 @@ func (c liveMoQConfig) label() string {
 // publishing each tier as its OWN broadcast (see livemoq.BroadcastPath): the
 // high tier from the camera's highest-resolution stream and the low tier from
 // its sub stream, so switching quality in the frontend is a resubscribe to the
-// other path. Each tier only uploads while it actually has subscribers, so the
-// second broadcast is close to free when nobody watches it.
-func StartLiveStreamMoQ(configuration *models.Configuration, communication *models.Communication, subStreamEnabled bool) {
+// other path. Each tier uploads every frame while watched and keyframes only
+// while idle, keeping the relay cache current without continuously sending the
+// full unused stream.
+func StartLiveStreamMoQ(
+	ctx context.Context,
+	configuration *models.Configuration,
+	communication *models.Communication,
+	subStreamEnabled bool,
+	mainQueue *packets.Queue,
+	subQueue *packets.Queue,
+) {
 	if os.Getenv("AGENT_LIVE_MOQ_ENABLED") != "true" {
 		return
 	}
 
 	config := configuration.Config
 	if config.Offline == "true" || config.Capture.Liveview == "false" {
-		log.Log.Info("cloud.StartLiveStreamMoQ(): disabled by Agent live-view configuration")
+		log.WithFields(log.Fields{
+			"component": "moq",
+			"event":     "publisher_disabled",
+			"liveview":  config.Capture.Liveview,
+			"offline":   config.Offline,
+		}).Debug("MoQ publisher disabled by Agent configuration")
 		return
 	}
 	if config.Key == "" {
-		log.Log.Warning("cloud.StartLiveStreamMoQ(): AGENT_KEY is required")
+		log.WithFields(log.Fields{
+			"component": "moq",
+			"event":     "publisher_configuration_invalid",
+			"variable":  "AGENT_KEY",
+		}).Warn("MoQ publisher requires an Agent key")
 		return
 	}
 
@@ -83,38 +179,51 @@ func StartLiveStreamMoQ(configuration *models.Configuration, communication *mode
 		relayURL = defaultMoQRelayURL
 	}
 	broadcastPrefix := os.Getenv("AGENT_LIVE_MOQ_BROADCAST_PREFIX")
+	maxPacketAge := boundedMoQDuration("AGENT_LIVE_MOQ_MAX_PACKET_AGE", defaultMoQLivePacketAge, minMoQLivePacketAge, maxMoQLivePacketAge)
+	writeTimeout := boundedMoQDuration("AGENT_LIVE_MOQ_WRITE_TIMEOUT", defaultMoQWriteTimeout, minMoQWriteTimeout, maxMoQWriteTimeout)
 
-	ctx := context.Background()
-	if communication.Context != nil {
-		ctx = *communication.Context
-	}
-
+	var publishers sync.WaitGroup
 	for _, quality := range qualities {
-		queue := communication.Queue
+		queue := mainQueue
 		sourceLabel := "main"
-		if models.SelectSubStreamForQuality(config, quality, subStreamEnabled) && communication.SubQueue != nil {
-			queue = communication.SubQueue
+		if models.SelectSubStreamForQuality(config, quality, subStreamEnabled) && subQueue != nil {
+			queue = subQueue
 			sourceLabel = "sub"
 		}
 		if queue == nil {
-			log.Log.Warning("cloud.StartLiveStreamMoQ(): packet queue for the " + quality + " tier is unavailable")
+			log.WithFields(log.Fields{
+				"component": "moq",
+				"event":     "packet_queue_unavailable",
+				"quality":   quality,
+			}).Warn("MoQ packet queue is unavailable")
 			continue
 		}
-		go runLiveStreamMoQ(ctx, liveMoQConfig{
-			relayURL:    relayURL,
-			broadcast:   livemoq.BroadcastPath(broadcastPrefix, config.Key, quality),
-			quality:     quality,
-			sourceLabel: sourceLabel,
-			queue:       queue,
-		})
+		publisherConfig := liveMoQConfig{
+			relayURL:      relayURL,
+			broadcast:     livemoq.BroadcastPath(broadcastPrefix, config.Key, quality),
+			quality:       quality,
+			sourceLabel:   sourceLabel,
+			queue:         queue,
+			communication: communication,
+			maxPacketAge:  maxPacketAge,
+			writeTimeout:  writeTimeout,
+		}
+		publishers.Add(1)
+		go func() {
+			defer publishers.Done()
+			runLiveStreamMoQ(ctx, publisherConfig)
+		}()
 	}
+	publishers.Wait()
 }
 
 func runLiveStreamMoQ(ctx context.Context, config liveMoQConfig) {
-	log.Log.Info(fmt.Sprintf(
-		"cloud.runLiveStreamMoQ(): publishing %s stream (quality=%s) to %s/%s",
-		config.sourceLabel, config.quality, strings.TrimRight(config.relayURL, "/"), config.broadcast,
-	))
+	config.logEntry("publisher_started").Info("MoQ publisher started")
+	config.logEntry("publisher_configuration").WithFields(log.Fields{
+		"max_packet_age_ms": config.packetAgeLimit().Milliseconds(),
+		"relay_host":        moQRelayHost(config.relayURL),
+		"write_timeout_ms":  config.writeTimeoutLimit().Milliseconds(),
+	}).Debug("MoQ publisher configuration")
 
 	retryDelay := minMoQRetryDelay
 	for ctx.Err() == nil {
@@ -123,7 +232,13 @@ func runLiveStreamMoQ(ctx context.Context, config liveMoQConfig) {
 		if ctx.Err() != nil {
 			return
 		}
-		log.Log.Warning("cloud.runLiveStreamMoQ(): publisher stopped: " + err.Error())
+		if config.communication != nil {
+			config.communication.RecordMoQReconnect(config.quality)
+		}
+		config.logEntry("publisher_reconnecting").WithError(err).WithFields(log.Fields{
+			"connected_duration_ms": time.Since(connectedAt).Milliseconds(),
+			"retry_delay_ms":        retryDelay.Milliseconds(),
+		}).Warn("MoQ publisher stopped; reconnecting")
 		if time.Since(connectedAt) >= time.Minute {
 			retryDelay = minMoQRetryDelay
 		}
@@ -150,62 +265,125 @@ func publishLiveStreamMoQ(ctx context.Context, config liveMoQConfig) error {
 		return fmt.Errorf("connect to relay: %w", err)
 	}
 	defer client.Close()
+	config.logEntry("relay_connected").WithField("relay_host", moQRelayHost(config.relayURL)).
+		Info("MoQ relay connected")
+
+	publisherCtx, cancelPublisher := context.WithCancel(ctx)
+	defer cancelPublisher()
+	sessionClosed := make(chan error, 1)
+	sessionWatchDone := make(chan struct{})
+	go func() {
+		defer close(sessionWatchDone)
+		sessionClosed <- client.Session().Closed(publisherCtx)
+		cancelPublisher()
+	}()
 
 	broadcast, err := client.CreateBroadcast(config.broadcast)
 	if err != nil {
 		return fmt.Errorf("create broadcast: %w", err)
 	}
-	defer broadcast.Finish()
+	var finishBroadcastOnce sync.Once
+	var finishBroadcastErr error
+	finishBroadcast := func() error {
+		finishBroadcastOnce.Do(func() {
+			finishBroadcastErr = broadcast.Finish()
+		})
+		return finishBroadcastErr
+	}
+	defer finishBroadcast()
 
 	stream, err := broadcast.PublishMedia("avc3", nil)
 	if err != nil {
 		return fmt.Errorf("create H.264 media stream: %w", err)
 	}
-	defer stream.Finish()
+	var finishStreamOnce sync.Once
+	var finishStreamErr error
+	finishStream := func() error {
+		finishStreamOnce.Do(func() {
+			finishStreamErr = stream.Finish()
+		})
+		return finishStreamErr
+	}
+	defer finishStream()
 
-	// Only upload while this tier is actually being watched. `publishing` starts
-	// true so the track becomes discoverable on the relay even before the first
-	// subscriber ever arrives; from the moment a viewer has attached once, the
-	// subscriber watcher takes over and idles the tier again when everybody left.
-	watchCtx, cancelWatch := context.WithCancel(ctx)
-	defer cancelWatch()
+	// Publish the complete stream while watched and only keyframes while idle.
+	// `publishing` starts true so the track becomes discoverable on the relay
+	// before the first subscriber; after that, the subscriber watcher selects the
+	// full-rate or keyframe-refresh mode.
 	publishing := &atomic.Bool{}
 	publishing.Store(true)
-	go watchLiveStreamMoQSubscribers(watchCtx, stream, publishing, config)
+	subscriberWatchDone := make(chan struct{})
+	go func() {
+		defer close(subscriberWatchDone)
+		watchLiveStreamMoQSubscribers(publisherCtx, stream, publishing, config)
+	}()
+	writeWatchdog := &livemoq.WriteWatchdog{}
+	writeWatchDone := make(chan struct{})
+	closePublisher := func() error {
+		return closeLiveMoQPublisher(finishStream, finishBroadcast, client.Close)
+	}
+	go func() {
+		defer close(writeWatchDone)
+		watchLiveStreamMoQWrites(publisherCtx, closePublisher, writeWatchdog, config)
+	}()
+	defer func() {
+		cancelPublisher()
+		<-writeWatchDone
+		<-subscriberWatchDone
+		<-sessionWatchDone
+	}()
 
 	cursor := config.queue.Latest()
 	gate := livemoq.FrameGate{}
+	audienceGate := livemoq.AudienceGate{}
 	deduplicator := livemoq.KeyframeDeduplicator{}
 	var lastSlowWriteWarning time.Time
 	var lastDuplicateKeyframeWarning time.Time
-	idle := false
 	for {
-		packet, err := cursor.ReadPacket()
+		select {
+		case err := <-sessionClosed:
+			return fmt.Errorf("relay session closed: %w", err)
+		default:
+		}
+
+		packet, err := cursor.ReadPacketContext(publisherCtx)
 		if err != nil {
+			select {
+			case sessionErr := <-sessionClosed:
+				return fmt.Errorf("relay session closed: %w", sessionErr)
+			default:
+			}
+			if ctx.Err() != nil {
+				return ctx.Err()
+			}
 			return fmt.Errorf("read packet: %w", err)
 		}
-		if !publishing.Load() {
-			// Keep draining the cursor so we stay at the live edge, but publish
-			// nothing. The gate is closed so the next viewer resumes on a keyframe.
-			if !idle {
-				gate.Reset()
-				deduplicator.Reset()
-				idle = true
-			}
-			continue
-		}
-		idle = false
 		if !packet.IsVideo || len(packet.Data) == 0 || !strings.EqualFold(packet.Codec, "H264") {
 			continue
 		}
-		allowed, event := gate.Allow(packet.IsKeyFrame, packet.CurrentTime, time.Now(), maxMoQLivePacketAge)
+		allowedForAudience, enteredIdle := audienceGate.Allow(publishing.Load(), packet.IsKeyFrame)
+		if enteredIdle {
+			gate.Reset()
+			deduplicator.Reset()
+		}
+		if !allowedForAudience {
+			continue
+		}
+		allowed, event := gate.Allow(packet.IsKeyFrame, packet.CurrentTime, time.Now(), config.packetAgeLimit())
 		switch event {
 		case livemoq.FrameGateEventStarted:
-			log.Log.Info("cloud.publishLiveStreamMoQ(): first H.264 keyframe received; " + config.label() + " broadcast is live")
+			config.logEntry("broadcast_live").WithFields(log.Fields{
+				"codec":        packet.Codec,
+				"timestamp_ms": packet.Time,
+			}).Info("MoQ broadcast is live")
 		case livemoq.FrameGateEventLagging:
-			log.Log.Warning("cloud.publishLiveStreamMoQ(): " + config.label() + " stream is lagging; dropping packets until a recent keyframe")
+			config.logEntry("stream_lagging").WithFields(log.Fields{
+				"max_packet_age_ms": config.packetAgeLimit().Milliseconds(),
+				"timestamp_ms":      packet.Time,
+			}).Warn("MoQ stream is lagging; dropping packets until a recent keyframe")
 		case livemoq.FrameGateEventRecovered:
-			log.Log.Info("cloud.publishLiveStreamMoQ(): caught up with the " + config.label() + " live stream at a recent keyframe")
+			config.logEntry("stream_recovered").WithField("timestamp_ms", packet.Time).
+				Info("MoQ stream recovered at a recent keyframe")
 		}
 		if !allowed {
 			continue
@@ -215,18 +393,18 @@ func publishLiveStreamMoQ(ctx context.Context, config liveMoQConfig) error {
 			return fmt.Errorf("normalize H.264 access unit: %w", err)
 		}
 		if normalizationStats.DuplicateIDRNALUs > 0 && time.Since(lastDuplicateKeyframeWarning) >= moQWriteWarningInterval {
-			log.Log.Warning(fmt.Sprintf(
-				"cloud.publishLiveStreamMoQ(): %s removed %d duplicate IDR NALU(s) from H.264 keyframe (timestamp_ms=%d)",
-				config.label(), normalizationStats.DuplicateIDRNALUs, packet.Time,
-			))
+			config.logEntry("duplicate_idr_removed").WithFields(log.Fields{
+				"duplicate_idr_count": normalizationStats.DuplicateIDRNALUs,
+				"timestamp_ms":        packet.Time,
+			}).Warn("Removed duplicate IDR NAL units from MoQ keyframe")
+
 			lastDuplicateKeyframeWarning = time.Now()
 		}
 		if packet.IsKeyFrame && deduplicator.IsDuplicate(packet.Time, packet.CurrentTime, payload, time.Now(), duplicateKeyframeWindow) {
 			if time.Since(lastDuplicateKeyframeWarning) >= moQWriteWarningInterval {
-				log.Log.Warning(fmt.Sprintf(
-					"cloud.publishLiveStreamMoQ(): %s dropping duplicate H.264 keyframe (timestamp_ms=%d)",
-					config.label(), packet.Time,
-				))
+				config.logEntry("duplicate_keyframe_dropped").WithField("timestamp_ms", packet.Time).
+					Warn("Dropped duplicate MoQ keyframe")
+
 				lastDuplicateKeyframeWarning = time.Now()
 			}
 			continue
@@ -236,10 +414,16 @@ func publishLiveStreamMoQ(ctx context.Context, config liveMoQConfig) error {
 			TimestampUs: livemoq.TimestampUs(packet.Time),
 		}
 		writeStartedAt := time.Now()
-		if err := stream.WriteFrame(frame); err != nil {
+		writeWatchdog.Begin(writeStartedAt)
+		err = stream.WriteFrame(frame)
+		writeWatchdog.End()
+		if err != nil {
 			return fmt.Errorf("write H.264 access unit: %w", err)
 		}
 		writeDuration := time.Since(writeStartedAt)
+		if config.communication != nil {
+			config.communication.RecordMoQWrite(config.quality, writeDuration, time.Now())
+		}
 		if writeDuration >= slowMoQWriteThreshold && time.Since(lastSlowWriteWarning) >= moQWriteWarningInterval {
 			packetAge := time.Duration(0)
 			if packet.CurrentTime > 0 {
@@ -248,15 +432,58 @@ func publishLiveStreamMoQ(ctx context.Context, config liveMoQConfig) error {
 					packetAge = 0
 				}
 			}
-			log.Log.Warning(fmt.Sprintf(
-				"cloud.publishLiveStreamMoQ(): %s WriteFrame blocked for %s (packet_age=%s keyframe=%t)",
-				config.label(), writeDuration.Round(time.Millisecond), packetAge.Round(time.Millisecond), packet.IsKeyFrame,
-			))
+			config.logEntry("slow_frame_write").WithFields(log.Fields{
+				"duration_ms":   writeDuration.Milliseconds(),
+				"keyframe":      packet.IsKeyFrame,
+				"packet_age_ms": packetAge.Milliseconds(),
+			}).Warn("MoQ frame write was slow")
+
 			lastSlowWriteWarning = time.Now()
 		}
 	}
 }
 
+func closeLiveMoQPublisher(finishStream, finishBroadcast, closeClient func() error) error {
+	streamErr := finishStream()
+	broadcastErr := finishBroadcast()
+	clientErr := closeClient()
+	return errors.Join(streamErr, broadcastErr, clientErr)
+}
+
+func watchLiveStreamMoQWrites(ctx context.Context, closePublisher func() error, watchdog *livemoq.WriteWatchdog, config liveMoQConfig) {
+	ticker := time.NewTicker(moQWriteWatchInterval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			if err := closePublisher(); err != nil {
+				config.logEntry("publisher_close_failed").WithError(err).
+					Debug("MoQ publisher reported an error while closing")
+			}
+			return
+		case now := <-ticker.C:
+			writeDuration, active := watchdog.Elapsed(now)
+			if !active || writeDuration < config.writeTimeoutLimit() {
+				continue
+			}
+			config.logEntry("frame_write_timeout").WithFields(log.Fields{
+				"duration_ms": writeDuration.Milliseconds(),
+				"timeout_ms":  config.writeTimeoutLimit().Milliseconds(),
+			}).Warn("MoQ frame write timed out; reconnecting relay client")
+
+			if config.communication != nil {
+				config.communication.RecordMoQWriteTimeout()
+			}
+			if err := closePublisher(); err != nil {
+				config.logEntry("publisher_close_failed").WithError(err).
+					Warn("MoQ publisher reported an error while closing after a write timeout")
+			}
+			return
+		}
+	}
+}
+
 // watchLiveStreamMoQSubscribers flips the publisher between uploading and idling
 // as viewers subscribe to and leave this tier's broadcast. Used and Unused both
 // block, so they are followed from their own goroutine.
@@ -272,7 +499,7 @@ func watchLiveStreamMoQSubscribers(ctx context.Context, stream *moq.MediaProduce
 			return
 		}
 		if publishing.CompareAndSwap(false, true) {
-			log.Log.Info("cloud.watchLiveStreamMoQSubscribers(): viewer subscribed, resuming the " + config.label() + " broadcast")
+			config.logEntry("subscriber_joined").Info("MoQ subscriber joined; resuming broadcast")
 		}
 
 		if err := stream.Unused(ctx); err != nil {
@@ -280,6 +507,6 @@ func watchLiveStreamMoQSubscribers(ctx context.Context, stream *moq.MediaProduce
 			return
 		}
 		publishing.Store(false)
-		log.Log.Info("cloud.watchLiveStreamMoQSubscribers(): no viewers left, idling the " + config.label() + " broadcast")
+		config.logEntry("subscribers_idle").Info("MoQ broadcast idle; refreshing keyframes only")
 	}
 }

machinery/src/cloud/livemoq_enabled_test.go

@@ -0,0 +1,212 @@
+//go:build moq
+
+package cloud
+
+import (
+	"context"
+	"errors"
+	"reflect"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/kerberos-io/agent/machinery/src/cloud/livemoq"
+	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/moq-dev/moq-go/moq"
+)
+
+func TestCloseLiveMoQPublisherFinishesBeforeClosingClient(t *testing.T) {
+	var order []string
+	streamErr := errors.New("stream finish failed")
+	broadcastErr := errors.New("broadcast finish failed")
+	clientErr := errors.New("client close failed")
+
+	err := closeLiveMoQPublisher(
+		func() error {
+			order = append(order, "stream")
+			return streamErr
+		},
+		func() error {
+			order = append(order, "broadcast")
+			return broadcastErr
+		},
+		func() error {
+			order = append(order, "client")
+			return clientErr
+		},
+	)
+
+	if want := []string{"stream", "broadcast", "client"}; !reflect.DeepEqual(order, want) {
+		t.Fatalf("shutdown order = %v, want %v", order, want)
+	}
+	for _, want := range []error{streamErr, broadcastErr, clientErr} {
+		if !errors.Is(err, want) {
+			t.Errorf("shutdown error %v does not include %v", err, want)
+		}
+	}
+}
+
+func TestWatchLiveStreamMoQWritesClosesClientWhenContextEnds(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	closed := make(chan struct{})
+	var closeCalls atomic.Int32
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		watchLiveStreamMoQWrites(ctx, func() error {
+			if closeCalls.Add(1) == 1 {
+				close(closed)
+			}
+			return nil
+		}, &livemoq.WriteWatchdog{}, liveMoQConfig{quality: "low", sourceLabel: "sub"})
+	}()
+
+	cancel()
+	select {
+	case <-closed:
+	case <-time.After(time.Second):
+		t.Fatal("watchLiveStreamMoQWrites() did not close the client after cancellation")
+	}
+	<-done
+	if got := closeCalls.Load(); got != 1 {
+		t.Fatalf("close calls = %d, want 1", got)
+	}
+}
+
+func TestBoundedMoQDuration(t *testing.T) {
+	const name = "AGENT_LIVE_MOQ_TEST_DURATION"
+	tests := []struct {
+		name string
+		raw  string
+		want time.Duration
+	}{
+		{name: "unset", want: 5 * time.Second},
+		{name: "valid", raw: "12s", want: 12 * time.Second},
+		{name: "invalid", raw: "later", want: 5 * time.Second},
+		{name: "below minimum", raw: "100ms", want: time.Second},
+		{name: "above maximum", raw: "2m", want: time.Minute},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			t.Setenv(name, test.raw)
+			if got := boundedMoQDuration(name, 5*time.Second, time.Second, time.Minute); got != test.want {
+				t.Fatalf("boundedMoQDuration() = %s, want %s", got, test.want)
+			}
+		})
+	}
+}
+
+func TestWatchLiveStreamMoQWritesRecordsTimeout(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	closed := make(chan struct{})
+	communication := &models.Communication{}
+	watchdog := &livemoq.WriteWatchdog{}
+	watchdog.Begin(time.Now().Add(-time.Second))
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		watchLiveStreamMoQWrites(ctx, func() error {
+			close(closed)
+			return nil
+		}, watchdog, liveMoQConfig{
+			quality:       models.StreamQualityLow,
+			sourceLabel:   "sub",
+			communication: communication,
+			writeTimeout:  10 * time.Millisecond,
+		})
+	}()
+
+	select {
+	case <-closed:
+	case <-time.After(time.Second):
+		t.Fatal("watchLiveStreamMoQWrites() did not close the client after a write timeout")
+	}
+	<-done
+	if got := communication.RecoveryTelemetry().MoQWriteTimeouts; got != 1 {
+		t.Fatalf("MoQ write timeouts = %d, want 1", got)
+	}
+}
+
+func TestNativeMoQClientCloseStopsActiveWriter(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	server, err := moq.Listen(ctx, "127.0.0.1:0", moq.WithTLSGenerate("localhost"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	serveDone := make(chan error, 1)
+	go func() {
+		serveDone <- server.Serve(ctx)
+	}()
+
+	client, err := moq.Dial(ctx, "https://"+server.LocalAddr(), moq.WithTLSVerify(false))
+	if err != nil {
+		server.Close()
+		t.Fatal(err)
+	}
+	broadcast, err := client.CreateBroadcast("test/native-close")
+	if err != nil {
+		client.Close()
+		server.Close()
+		t.Fatal(err)
+	}
+	stream, err := broadcast.PublishMedia("avc3", nil)
+	if err != nil {
+		client.Close()
+		server.Close()
+		t.Fatal(err)
+	}
+
+	firstWrite := make(chan struct{})
+	writerDone := make(chan error, 1)
+	go func() {
+		payload := []byte{
+			0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x1e, 0xd9, 0x00, 0xa0, 0x47, 0xfe, 0xc8,
+			0x00, 0x00, 0x00, 0x01, 0x68, 0xce, 0x38, 0x80,
+			0x00, 0x00, 0x00, 0x01, 0x65, 0x88,
+		}
+		for timestamp := uint64(0); ; timestamp++ {
+			err := stream.WriteFrame(moq.Frame{Payload: payload, TimestampUs: timestamp})
+			if err != nil {
+				writerDone <- err
+				return
+			}
+			if timestamp == 0 {
+				close(firstWrite)
+			}
+		}
+	}()
+
+	select {
+	case <-firstWrite:
+	case <-ctx.Done():
+		t.Fatal("native MoQ writer did not start")
+	}
+	if err := stream.Finish(); err != nil {
+		t.Fatal(err)
+	}
+	if err := client.Close(); err != nil {
+		t.Fatal(err)
+	}
+	select {
+	case err := <-writerDone:
+		if err == nil {
+			t.Fatal("native MoQ writer stopped without an error after client close")
+		}
+	case <-ctx.Done():
+		t.Fatal("native MoQ client close did not stop the writer")
+	}
+
+	_ = broadcast.Finish()
+	_ = server.Close()
+	select {
+	case <-serveDone:
+	case <-ctx.Done():
+		t.Fatal("native MoQ server did not stop")
+	}
+}

machinery/src/cloud/livesnapshot/publisher.go

@@ -30,7 +30,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
+	log "github.com/sirupsen/logrus"
 )
 
 const (
@@ -131,7 +131,10 @@ func (p *Publisher) PublishSnapshot(ctx context.Context, jpeg []byte) error {
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
 		return fmt.Errorf("livesnapshot: upload snapshot rejected: %s", resp.Status)
 	}
-	log.Log.Debug("livesnapshot.Publisher.PublishSnapshot(): shipped preview frame for device " + p.cfg.DeviceKey)
+	log.WithFields(log.Fields{
+		"component": "livesnapshot",
+		"event":     "snapshot_published",
+	}).Debug("Live snapshot published")
 	return nil
 }
 

machinery/src/cloud/recording_metadata.go

@@ -2,6 +2,7 @@ package cloud
 
 import (
 	"encoding/json"
+	"fmt"
 	"math"
 	"net/http"
 	"os"
@@ -10,11 +11,16 @@ import (
 	"strings"
 
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 const recordingFPSHeader = "X-Kerberos-Storage-Fps"
 const recordingDurationHeader = "X-Kerberos-Storage-Duration"
 const recordingTimestampHeader = "X-Kerberos-Storage-Timestamp"
+const recordingEncryptedHeader = "X-Kerberos-Storage-Encrypted"
+const recordingDeviceNameHeader = "X-Kerberos-Storage-DeviceName"
+const recordingRegionHeader = "X-Kerberos-Storage-RegionCoordinates"
+const recordingChangesHeader = "X-Kerberos-Storage-NumberOfChanges"
 
 // queuedRecordingFPS reads the FPS snapshot written into the upload marker
 // when the recording was finalized. Historical empty markers intentionally
@@ -60,6 +66,14 @@ func decodeRecordingUploadMetadata(value []byte) (models.RecordingUploadMetadata
 }
 
 func readRecordingUploadMetadata(fileName string) ([]byte, bool) {
+	value, err := readRecordingUploadMetadataFile(fileName)
+	if err != nil && !os.IsNotExist(err) {
+		log.WithError(err).Error("Failed to read recording upload metadata")
+	}
+	return value, err == nil
+}
+
+func readRecordingUploadMetadataFile(fileName string) ([]byte, error) {
 	markerNames := []string{
 		models.RecordingUploadMetadataFileName(fileName),
 		filepath.Base(fileName),
@@ -67,10 +81,13 @@ func readRecordingUploadMetadata(fileName string) ([]byte, bool) {
 	for _, markerName := range markerNames {
 		value, err := os.ReadFile(filepath.Join("data", "cloud", markerName))
 		if err == nil {
-			return value, true
+			return value, nil
+		}
+		if !os.IsNotExist(err) {
+			return nil, fmt.Errorf("read recording upload metadata %s: %w", markerName, err)
 		}
 	}
-	return nil, false
+	return nil, os.ErrNotExist
 }
 
 func setQueuedRecordingMetadataHeaders(header http.Header, fileName string) {
@@ -78,11 +95,26 @@ func setQueuedRecordingMetadataHeaders(header http.Header, fileName string) {
 		header.Set(recordingFPSHeader, fps)
 	}
 	if metadata, ok := queuedRecordingMetadata(fileName); ok {
+		if metadata.DeviceKey != "" {
+			header.Set("X-Kerberos-Storage-Device", metadata.DeviceKey)
+		}
+		if metadata.DeviceName != "" {
+			header.Set(recordingDeviceNameHeader, metadata.DeviceName)
+		}
+		if metadata.RegionCoordinates != "" {
+			header.Set(recordingRegionHeader, metadata.RegionCoordinates)
+		}
+		if metadata.NumberOfChanges != "" {
+			header.Set(recordingChangesHeader, metadata.NumberOfChanges)
+		}
 		if metadata.Duration > 0 {
 			header.Set(recordingDurationHeader, strconv.FormatUint(metadata.Duration, 10))
 		}
 		if metadata.Timestamp > 0 {
 			header.Set(recordingTimestampHeader, strconv.FormatInt(metadata.Timestamp, 10))
 		}
+		if metadata.Encrypted {
+			header.Set(recordingEncryptedHeader, "true")
+		}
 	}
 }

machinery/src/cloud/s3.go

@@ -2,34 +2,31 @@ package cloud
 
 import (
 	"crypto/tls"
+	"encoding/json"
 	"errors"
+	"fmt"
+	"math"
 	"net/http"
 	"net/url"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
+	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/utils"
 	"github.com/minio/minio-go/v6"
+	log "github.com/sirupsen/logrus"
 )
 
 func UploadS3(configuration *models.Configuration, fileName string) (bool, bool, error) {
 
 	config := configuration.Config
 
-	// timestamp_microseconds_instanceName_regionCoordinates_numberOfChanges_token
-	// 1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4
-	// - Timestamp
-	// - Size + - + microseconds
-	// - device
-	// - Region
-	// - Number of changes
-	// - Token
-
 	if config.S3 == nil {
 		errorMessage := "UploadS3: Uploading Failed, as no settings found"
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, false, errors.New(errorMessage)
 	}
 
@@ -49,14 +46,14 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
 	// Check if we have some credentials otherwise we abort the request.
 	if aws_access_key_id == "" || aws_secret_access_key == "" {
 		errorMessage := "UploadS3: Uploading Failed, as no credentials found"
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, false, errors.New(errorMessage)
 	}
 
 	s3Client, err := minio.NewWithRegion("s3.amazonaws.com", aws_access_key_id, aws_secret_access_key, true, aws_region)
 	if err != nil {
 		errorMessage := "UploadS3: " + err.Error()
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, true, errors.New(errorMessage)
 	}
 
@@ -71,67 +68,189 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
 		s3Client.SetCustomTransport(transport)
 	}
 
-	fileParts := strings.Split(fileName, "_")
-	if len(fileParts) == 1 {
-		errorMessage := "UploadS3: " + fileName + " is not a valid name."
-		log.Log.Error(errorMessage)
-		return false, true, errors.New(errorMessage)
+	objectName, userMetadata, err := s3RecordingUploadDetails(config, fileName, aws_access_key_id)
+	if err != nil {
+		log.WithError(err).Error("UploadS3: invalid recording upload metadata")
+		return false, true, err
 	}
 
-	deviceKey := config.Key
-	startRecording, _ := strconv.ParseInt(fileParts[0], 10, 64)
-	devicename := fileParts[2]
-	coordinates := fileParts[3]
-	//numberOfChanges := fileParts[4]
-	token, _ := strconv.Atoi(fileParts[5])
-
-	log.Log.Info("UploadS3: Upload started for " + fileName)
-	fullname := "data/recordings/" + fileName
-
-	file, err := os.OpenFile(fullname, os.O_RDWR, 0755)
+	log.Info("UploadS3: Upload started for " + fileName)
+	file, err := utils.OpenRecording("data/recordings", fileName)
 	if file != nil {
 		defer file.Close()
 	}
 
 	if err != nil {
 		errorMessage := "UploadS3: " + err.Error()
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, true, errors.New(errorMessage)
 	}
 
 	fileInfo, err := file.Stat()
 	if err != nil {
 		errorMessage := "UploadS3: " + err.Error()
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, true, errors.New(errorMessage)
 	}
 
 	n, err := s3Client.PutObject(config.S3.Bucket,
-		config.S3.Username+"/"+fileName,
+		objectName,
 		file,
 		fileInfo.Size(),
 		minio.PutObjectOptions{
 			ContentType:  "video/mp4",
 			StorageClass: "ONEZONE_IA",
-			UserMetadata: map[string]string{
-				"event-timestamp":         strconv.FormatInt(startRecording, 10),
-				"event-microseconds":      deviceKey,
-				"event-instancename":      devicename,
-				"event-regioncoordinates": coordinates,
-				"event-numberofchanges":   deviceKey,
-				"event-token":             strconv.Itoa(token),
-				"productid":               deviceKey,
-				"publickey":               aws_access_key_id,
-				"uploadtime":              "now",
-			},
+			UserMetadata: userMetadata,
 		})
 
 	if err != nil {
 		errorMessage := "UploadS3: Uploading Failed, " + err.Error()
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, true, errors.New(errorMessage)
 	} else {
-		log.Log.Info("UploadS3: Upload Finished, file has been uploaded to bucket: " + strconv.FormatInt(n, 10))
+		log.Info("UploadS3: Upload Finished, file has been uploaded to bucket: " + strconv.FormatInt(n, 10))
 		return true, true, nil
 	}
 }
+
+func s3RecordingObjectName(username, deviceKey string, startRecording int64) string {
+	return username + "/" + time.UnixMilli(startRecording).UTC().Format("2006/01/02") +
+		"/" + strconv.FormatInt(startRecording, 10) + "-" + deviceKey + ".mp4"
+}
+
+func s3RecordingUploadDetails(config models.Config, fileName, publicKey string) (string, map[string]string, error) {
+	value, err := readRecordingUploadMetadataFile(fileName)
+	if err != nil && !os.IsNotExist(err) {
+		return "", nil, err
+	}
+
+	deviceKey := config.Key
+	var startRecording int64
+	userMetadata := map[string]string{
+		"publickey":  publicKey,
+		"uploadtime": "now",
+	}
+	marker := strings.TrimSpace(string(value))
+	_, fpsErr := strconv.ParseFloat(marker, 64)
+	if err == nil && marker != "" && fpsErr != nil {
+		var metadata models.RecordingUploadMetadata
+		if err := json.Unmarshal(value, &metadata); err != nil {
+			return "", nil, fmt.Errorf("decode recording upload metadata for %s: %w", fileName, err)
+		}
+		if metadata.FileName != "" && metadata.FileName != filepath.Base(fileName) {
+			return "", nil, fmt.Errorf("recording upload metadata filename does not match %s", fileName)
+		}
+		if metadata.Timestamp <= 0 {
+			return "", nil, fmt.Errorf("invalid recording timestamp in upload metadata for %s", fileName)
+		}
+		deviceKey = metadata.DeviceKey
+		startRecording = metadata.Timestamp
+		userMetadata["event-instancename"] = config.Name
+		if metadata.DeviceName == "" || metadata.RegionCoordinates == "" || metadata.NumberOfChanges == "" {
+			if description, ok := s3LegacyRecordingDescription(fileName); ok {
+				for key, value := range description {
+					userMetadata[key] = value
+				}
+			}
+		}
+		if metadata.DeviceName != "" {
+			userMetadata["event-instancename"] = metadata.DeviceName
+		}
+		if metadata.RegionCoordinates != "" {
+			userMetadata["event-regioncoordinates"] = metadata.RegionCoordinates
+		}
+		if metadata.NumberOfChanges != "" {
+			userMetadata["event-numberofchanges"] = metadata.NumberOfChanges
+		}
+		userMetadata["event-microseconds"] = strconv.FormatInt(metadata.Timestamp%1000*1000, 10)
+		userMetadata["duration"] = strconv.FormatUint(metadata.Duration, 10)
+		userMetadata["encrypted"] = strconv.FormatBool(metadata.Encrypted)
+		if metadata.FPS > 0 && metadata.FPS <= 240 {
+			userMetadata["fps"] = strconv.FormatFloat(metadata.FPS, 'f', -1, 64)
+		}
+	} else {
+		// Empty and FPS-only markers predate structured recording metadata.
+		fileParts := strings.Split(fileName, "_")
+		if len(fileParts) < 6 {
+			return "", nil, fmt.Errorf("%s is not a valid legacy recording name", fileName)
+		}
+		startRecording, err = strconv.ParseInt(fileParts[0], 10, 64)
+		if err != nil {
+			return "", nil, fmt.Errorf("invalid recording timestamp in %s: %w", fileName, err)
+		}
+		if startRecording <= 0 || startRecording > math.MaxInt64/1000 {
+			return "", nil, fmt.Errorf("invalid recording timestamp in %s", fileName)
+		}
+		startRecording *= 1000
+		_, precision, found := strings.Cut(fileParts[1], "-")
+		if found {
+			fraction, err := strconv.ParseInt(precision, 10, 64)
+			if err != nil || fraction < 0 || fraction >= 1000000 {
+				return "", nil, fmt.Errorf("invalid recording precision in %s", fileName)
+			}
+			if len(precision) > 3 {
+				fraction /= 1000
+			}
+			startRecording += fraction
+		}
+		token, _ := strconv.Atoi(fileParts[5])
+		userMetadata["event-microseconds"] = deviceKey
+		userMetadata["event-instancename"] = fileParts[2]
+		userMetadata["event-regioncoordinates"] = fileParts[3]
+		userMetadata["event-numberofchanges"] = deviceKey
+		userMetadata["event-token"] = strconv.Itoa(token)
+	}
+	if deviceKey == "" || strings.ContainsAny(deviceKey, "/\\") {
+		return "", nil, errors.New("missing or invalid recording device key")
+	}
+	if startRecording <= 0 {
+		return "", nil, fmt.Errorf("invalid recording timestamp for %s", fileName)
+	}
+	userMetadata["event-timestamp"] = strconv.FormatInt(startRecording/1000, 10)
+	userMetadata["productid"] = deviceKey
+	return s3RecordingObjectName(config.S3.Username, deviceKey, startRecording), userMetadata, nil
+}
+
+func s3LegacyRecordingDescription(fileName string) (map[string]string, bool) {
+	base := filepath.Base(fileName)
+	if filepath.Ext(base) != ".mp4" {
+		return nil, false
+	}
+	parts := strings.Split(strings.TrimSuffix(base, ".mp4"), "_")
+	if len(parts) != 6 || parts[2] == "" {
+		return nil, false
+	}
+	seconds, err := strconv.ParseInt(parts[0], 10, 64)
+	if err != nil || seconds <= 0 || seconds > math.MaxInt64/1000 {
+		return nil, false
+	}
+	digits, fraction, found := strings.Cut(parts[1], "-")
+	precision, err := strconv.Atoi(digits)
+	if !found || err != nil || precision < 1 || precision > 6 || len(fraction) != precision {
+		return nil, false
+	}
+	if _, err := strconv.ParseUint(fraction, 10, 64); err != nil {
+		return nil, false
+	}
+	coordinates := strings.Split(parts[3], "-")
+	if len(coordinates) != 4 {
+		return nil, false
+	}
+	for _, coordinate := range coordinates {
+		if _, err := strconv.ParseUint(coordinate, 10, 64); err != nil {
+			return nil, false
+		}
+	}
+	changes, err := strconv.ParseInt(parts[4], 10, 64)
+	if err != nil || changes < -1 {
+		return nil, false
+	}
+	if _, err := strconv.ParseUint(parts[5], 10, 64); err != nil {
+		return nil, false
+	}
+	return map[string]string{
+		"event-instancename":      parts[2],
+		"event-regioncoordinates": parts[3],
+		"event-numberofchanges":   parts[4],
+	}, true
+}

machinery/src/cloud/s3_test.go

@@ -0,0 +1,328 @@
+package cloud
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/kerberos-io/agent/machinery/src/models"
+)
+
+func TestS3RecordingUploadDetailsUsesCheckpoint(t *testing.T) {
+	for _, fileName := range []string{
+		"1767355200123-checkpoint-key.mp4",
+		"1564859471_6-474162_old-camera_0-0-0-0_1153_27.mp4",
+	} {
+		t.Run(fileName, func(t *testing.T) {
+			withRecording(t, fileName, []byte("recording"))
+			recorded := time.Date(2026, time.January, 2, 12, 0, 0, 123000000, time.UTC)
+			metadata := models.RecordingUploadMetadata{
+				FileName:          fileName,
+				DeviceKey:         "checkpoint-key",
+				Timestamp:         recorded.UnixMilli(),
+				Duration:          20452,
+				FPS:               29.97,
+				Encrypted:         true,
+				DeviceName:        "checkpoint-camera",
+				RegionCoordinates: "1-2-3-4",
+				NumberOfChanges:   "57",
+			}
+			payload, err := json.Marshal(metadata)
+			if err != nil {
+				t.Fatal(err)
+			}
+			withQueuedRecordingFPS(t, fileName, string(payload))
+			config := models.Config{
+				Key:  "changed-config-key",
+				Name: "camera-name",
+				S3:   &models.S3{Username: "user"},
+			}
+			objectName, userMetadata, err := s3RecordingUploadDetails(config, fileName, "public-key")
+			if err != nil {
+				t.Fatal(err)
+			}
+			wantObjectName := "user/2026/01/02/" + strconv.FormatInt(recorded.UnixMilli(), 10) + "-checkpoint-key.mp4"
+			if objectName != wantObjectName {
+				t.Fatalf("object name = %q, want %q", objectName, wantObjectName)
+			}
+			wantMetadata := map[string]string{
+				"publickey":               "public-key",
+				"uploadtime":              "now",
+				"event-instancename":      "checkpoint-camera",
+				"event-regioncoordinates": "1-2-3-4",
+				"event-numberofchanges":   "57",
+				"event-microseconds":      "123000",
+				"event-timestamp":         strconv.FormatInt(recorded.Unix(), 10),
+				"productid":               "checkpoint-key",
+				"duration":                "20452",
+				"fps":                     "29.97",
+				"encrypted":               "true",
+			}
+			if !reflect.DeepEqual(userMetadata, wantMetadata) {
+				t.Fatalf("metadata = %#v, want %#v", userMetadata, wantMetadata)
+			}
+		})
+	}
+}
+
+func TestS3RecordingUploadDetailsBackfillsLegacyCheckpointDescriptions(t *testing.T) {
+	const legacyName = "1564859471_6-474162_recorded-camera_1-2-3-4_1153_27.mp4"
+	tests := []struct {
+		name        string
+		fileName    string
+		camera      string
+		region      string
+		changes     string
+		wantCamera  string
+		wantRegion  string
+		wantChanges string
+	}{
+		{"pre-upgrade checkpoint", legacyName, "", "", "", "recorded-camera", "1-2-3-4", "1153"},
+		{"checkpoint camera", legacyName, "snapshot-camera", "", "", "snapshot-camera", "1-2-3-4", "1153"},
+		{"checkpoint region", legacyName, "", "5-6-7-8", "", "recorded-camera", "5-6-7-8", "1153"},
+		{"checkpoint zero changes", legacyName, "", "", "0", "recorded-camera", "1-2-3-4", "0"},
+		{"dated reference", "2019/08/03/" + legacyName, "", "", "", "recorded-camera", "1-2-3-4", "1153"},
+		{"continuous recording", "1564859471_3-474_recorded-camera_0-0-0-0_-1_27.mp4", "", "", "", "recorded-camera", "0-0-0-0", "-1"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			withRecording(t, filepath.Base(tt.fileName), []byte("recording"))
+			metadata := models.RecordingUploadMetadata{
+				FileName:          filepath.Base(tt.fileName),
+				DeviceKey:         "checkpoint-key",
+				Timestamp:         1767355200123,
+				Duration:          20452,
+				FPS:               29.97,
+				Encrypted:         true,
+				DeviceName:        tt.camera,
+				RegionCoordinates: tt.region,
+				NumberOfChanges:   tt.changes,
+			}
+			payload, err := json.Marshal(metadata)
+			if err != nil {
+				t.Fatal(err)
+			}
+			withQueuedRecordingFPS(t, tt.fileName, string(payload))
+			config := models.Config{Key: "current-key", Name: "current-camera", S3: &models.S3{Username: "user"}}
+			objectName, got, err := s3RecordingUploadDetails(config, tt.fileName, "public-key")
+			if err != nil {
+				t.Fatal(err)
+			}
+			if want := "user/2026/01/02/1767355200123-checkpoint-key.mp4"; objectName != want {
+				t.Fatalf("object name = %q, want %q", objectName, want)
+			}
+			want := map[string]string{
+				"publickey":               "public-key",
+				"uploadtime":              "now",
+				"event-instancename":      tt.wantCamera,
+				"event-regioncoordinates": tt.wantRegion,
+				"event-numberofchanges":   tt.wantChanges,
+				"event-microseconds":      "123000",
+				"event-timestamp":         "1767355200",
+				"productid":               "checkpoint-key",
+				"duration":                "20452",
+				"fps":                     "29.97",
+				"encrypted":               "true",
+			}
+			if !reflect.DeepEqual(got, want) {
+				t.Fatalf("metadata = %#v, want %#v", got, want)
+			}
+		})
+	}
+}
+
+func TestS3RecordingUploadDetailsDoesNotBackfillInvalidLegacyNames(t *testing.T) {
+	for _, fileName := range []string{
+		"1767355200123-checkpoint-key.mp4",
+		"invalid_6-474162_camera_0-0-0-0_1153_27.mp4",
+		"1564859471_6-invalid_camera_0-0-0-0_1153_27.mp4",
+		"1564859471_3-474162_camera_0-0-0-0_1153_27.mp4",
+		"1564859471_6-474162__0-0-0-0_1153_27.mp4",
+		"1564859471_6-474162_camera_invalid_1153_27.mp4",
+		"1564859471_6-474162_camera_0-0-0-0_invalid_27.mp4",
+		"1564859471_6-474162_camera_0-0-0-0_1153_invalid.mp4",
+		"1564859471_6-474162_camera_0-0-0-0_1153_27_extra.mp4",
+	} {
+		t.Run(fileName, func(t *testing.T) {
+			withRecording(t, fileName, []byte("recording"))
+			metadata := models.RecordingUploadMetadata{
+				FileName: fileName, DeviceKey: "checkpoint-key", Timestamp: 1767355200123, Duration: 20452,
+			}
+			payload, err := json.Marshal(metadata)
+			if err != nil {
+				t.Fatal(err)
+			}
+			withQueuedRecordingFPS(t, fileName, string(payload))
+			config := models.Config{Key: "current-key", Name: "current-camera", S3: &models.S3{Username: "user"}}
+			objectName, got, err := s3RecordingUploadDetails(config, fileName, "public-key")
+			if err != nil {
+				t.Fatal(err)
+			}
+			if objectName != "user/2026/01/02/1767355200123-checkpoint-key.mp4" ||
+				got["event-instancename"] != "current-camera" || got["event-regioncoordinates"] != "" ||
+				got["event-numberofchanges"] != "" || got["event-timestamp"] != "1767355200" ||
+				got["productid"] != "checkpoint-key" {
+				t.Fatalf("unexpected fallback for invalid legacy name: %q, %#v", objectName, got)
+			}
+		})
+	}
+}
+
+func TestS3RecordingUploadDetailsLegacyMarkers(t *testing.T) {
+	for _, marker := range []string{"missing", "empty", "fps-only", "old-marker-name"} {
+		t.Run(marker, func(t *testing.T) {
+			const fileName = "1564859471_6-474162_camera_0-0-0-0_1153_27.mp4"
+			withRecording(t, fileName, []byte("recording"))
+			switch marker {
+			case "empty":
+				withQueuedRecordingFPS(t, fileName, "")
+			case "fps-only", "old-marker-name":
+				withQueuedRecordingFPS(t, fileName, "29.97")
+				if marker == "old-marker-name" {
+					err := os.Rename(filepath.Join("data/cloud", models.RecordingUploadMetadataFileName(fileName)),
+						filepath.Join("data/cloud", fileName))
+					if err != nil {
+						t.Fatal(err)
+					}
+				}
+			}
+			config := models.Config{Key: "legacy-key", S3: &models.S3{Username: "user"}}
+			objectName, metadata, err := s3RecordingUploadDetails(config, fileName, "public-key")
+			if err != nil {
+				t.Fatal(err)
+			}
+			if want := "user/2019/08/03/1564859471474-legacy-key.mp4"; objectName != want {
+				t.Fatalf("object name = %q, want %q", objectName, want)
+			}
+			if metadata["productid"] != "legacy-key" || metadata["event-instancename"] != "camera" ||
+				metadata["event-regioncoordinates"] != "0-0-0-0" || metadata["event-timestamp"] != "1564859471" {
+				t.Fatalf("unexpected legacy metadata: %#v", metadata)
+			}
+		})
+	}
+}
+
+func TestS3RecordingUploadDetailsRejectsInvalidCheckpoint(t *testing.T) {
+	tests := []struct {
+		name   string
+		marker string
+		want   string
+	}{
+		{"malformed JSON", `{"timestamp":`, "decode"},
+		{"missing timestamp", `{"device_key":"camera"}`, "timestamp"},
+		{"missing device key", `{"timestamp":1787015373000}`, "device key"},
+		{"invalid device key", `{"timestamp":1787015373000,"device_key":"camera/key"}`, "device key"},
+		{"mismatched filename", `{"timestamp":1787015373000,"device_key":"camera","filename":"other.mp4"}`, "filename"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			const fileName = "1564859471_6-474162_camera_0-0-0-0_1153_27.mp4"
+			withRecording(t, fileName, []byte("recording"))
+			withQueuedRecordingFPS(t, fileName, tt.marker)
+			config := models.Config{Key: "legacy-key", S3: &models.S3{Username: "user"}}
+			_, _, err := s3RecordingUploadDetails(config, fileName, "public-key")
+			if err == nil || !strings.Contains(err.Error(), tt.want) {
+				t.Fatalf("error = %v, want error containing %q instead of legacy fallback", err, tt.want)
+			}
+		})
+	}
+}
+
+func TestS3RecordingUploadDetailsReportsCheckpointReadError(t *testing.T) {
+	const fileName = "1564859471_6-474162_camera_0-0-0-0_1153_27.mp4"
+	withRecording(t, fileName, []byte("recording"))
+	if err := os.MkdirAll(filepath.Join("data/cloud", models.RecordingUploadMetadataFileName(fileName)), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	config := models.Config{Key: "legacy-key", S3: &models.S3{Username: "user"}}
+	_, _, err := s3RecordingUploadDetails(config, fileName, "public-key")
+	if err == nil || !strings.Contains(err.Error(), "read recording upload metadata") {
+		t.Fatalf("error = %v, want checkpoint read error instead of legacy fallback", err)
+	}
+}
+
+func TestS3RecordingObjectName(t *testing.T) {
+	tests := []struct {
+		name      string
+		recorded  time.Time
+		directory string
+	}{
+		{
+			name:      "zero padded month and day",
+			recorded:  time.Date(2026, time.January, 2, 12, 0, 0, 0, time.UTC),
+			directory: "2026/01/02",
+		},
+		{
+			name:      "UTC date across year boundary",
+			recorded:  time.Date(2026, time.January, 1, 1, 0, 0, 0, time.FixedZone("UTC+2", 2*60*60)),
+			directory: "2025/12/31",
+		},
+		{
+			name:      "leap day",
+			recorded:  time.Date(2024, time.February, 29, 23, 59, 59, 0, time.UTC),
+			directory: "2024/02/29",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			const deviceKey = "camera-key-123"
+			want := "user/" + tt.directory + "/" + strconv.FormatInt(tt.recorded.UnixMilli(), 10) + "-" + deviceKey + ".mp4"
+			for attempt := 0; attempt < 2; attempt++ {
+				if got := s3RecordingObjectName("user", deviceKey, tt.recorded.UnixMilli()); got != want {
+					t.Fatalf("s3RecordingObjectName() = %q, want %q", got, want)
+				}
+			}
+		})
+	}
+}
+
+func TestUploadS3RejectsInvalidRecordingNames(t *testing.T) {
+	configuration := &models.Configuration{
+		Config: models.Config{
+			Key: "camera-key-123",
+			S3: &models.S3{
+				Publickey: "test-public-key",
+				Secretkey: "test-secret-key",
+				Region:    "eu-west-1",
+			},
+		},
+	}
+	for _, fileName := range []string{
+		"recording.mp4",
+		"1787015373_camera.mp4",
+		"invalid_6-474162_camera_0-0-0-0_1153_27.mp4",
+	} {
+		t.Run(fileName, func(t *testing.T) {
+			uploaded, configured, err := UploadS3(configuration, fileName)
+			if uploaded || !configured || err == nil {
+				t.Fatalf("UploadS3() = (%v, %v, %v), want (false, true, error)", uploaded, configured, err)
+			}
+		})
+	}
+}
+
+func TestUploadS3RejectsInvalidDeviceKeys(t *testing.T) {
+	for _, deviceKey := range []string{"", "camera/key", `camera\key`} {
+		t.Run(deviceKey, func(t *testing.T) {
+			configuration := &models.Configuration{
+				Config: models.Config{
+					Key: deviceKey,
+					S3: &models.S3{
+						Publickey: "test-public-key",
+						Secretkey: "test-secret-key",
+						Region:    "eu-west-1",
+					},
+				},
+			}
+			uploaded, configured, err := UploadS3(configuration, "1787015373_6-474162_camera_0-0-0-0_1153_27.mp4")
+			if uploaded || !configured || err == nil || !strings.Contains(err.Error(), "device key") {
+				t.Fatalf("UploadS3() = (%v, %v, %v), want (false, true, device key error)", uploaded, configured, err)
+			}
+		})
+	}
+}

machinery/src/cloud/tus_client.go

@@ -15,8 +15,9 @@ import (
 	"strings"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/utils"
+	log "github.com/sirupsen/logrus"
 )
 
 // tusResumableVersion is the tus protocol version implemented by this client.
@@ -90,7 +91,14 @@ func logTusUploadProgress(label string, offset, size int64, loggedBucket *int64)
 	if percent > 100 {
 		percent = 100
 	}
-	log.Log.Infof("%s: resumable upload progress %d%% (%d/%d bytes)", label, percent, offset, size)
+	log.WithFields(log.Fields{
+		"bytes_sent":  offset,
+		"bytes_total": size,
+		"component":   "tus",
+		"event":       "upload_progress",
+		"progress":    percent,
+		"target":      label,
+	}).Info("Resumable upload progress")
 }
 
 // tusHeaderFunc sets the authentication and routing headers required on every
@@ -99,7 +107,7 @@ func logTusUploadProgress(label string, offset, size int64, loggedBucket *int64)
 // creation request; it is empty on HEAD/PATCH/DELETE.
 type tusHeaderFunc func(h http.Header, fileName string)
 
-// runTusUpload performs a resumable (tus) upload of data/recordings/<fileName>
+// runTusUpload performs a resumable (tus) upload of a flat or dated recording.
 // to baseURL, sending target-specific authentication/routing headers via
 // setHeaders on every request. It encapsulates the create/resume/chunk/finalize
 // state machine shared by the Kerberos Vault (direct) and Kerberos Hub (proxied)
@@ -113,15 +121,13 @@ type tusHeaderFunc func(h http.Header, fileName string)
 //     should fall back to the legacy single-POST upload (older deployments).
 //   - body:      a short message for logging.
 func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tusHeaderFunc) (uploaded bool, responded bool, supported bool, body string, err error) {
-	fullname := "data/recordings/" + fileName
-
-	file, ferr := os.Open(fullname)
+	file, ferr := utils.OpenRecording("data/recordings", fileName)
 	if file != nil {
 		defer file.Close()
 	}
 	if ferr != nil {
 		msg := label + ": resumable upload failed, file doesn't exist anymore"
-		log.Log.Info(msg)
+		log.Info(msg)
 		// The file is gone, so the legacy path cannot help either. Report it as
 		// "supported" to avoid a pointless fallback attempt.
 		return false, false, true, "", errors.New(msg)
@@ -134,19 +140,6 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
 	size := info.Size()
 
 	client := newVaultHTTPClient(0)
-	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
-		if len(via) == 0 {
-			return nil
-		}
-		if req.URL.Host != via[0].URL.Host {
-			for k := range req.Header {
-				if strings.HasPrefix(http.CanonicalHeaderKey(k), "X-Kerberos-") {
-					req.Header.Del(k)
-				}
-			}
-		}
-		return nil
-	}
 
 	sidecar := tusSidecarPath(fileName, slot)
 	uploadURL := loadTusResumeState(sidecar, baseURL)
@@ -182,7 +175,7 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
 					// The vault does not implement tus; let the caller fall back.
 					return false, false, false, "", cerr
 				}
-				log.Log.Info(label + ": resumable create failed, " + cerr.Error())
+				log.Info(label + ": resumable create failed, " + cerr.Error())
 				tusBackoff(attempt)
 				continue
 			}
@@ -200,7 +193,7 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
 				uploadURL = ""
 				continue
 			}
-			log.Log.Info(label + ": resumable head failed, " + herr.Error())
+			log.Info(label + ": resumable head failed, " + herr.Error())
 			tusBackoff(attempt)
 			continue
 		}
@@ -253,9 +246,9 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
 				if status >= 400 {
 					// Definitive rejection (e.g. provider push failed during finalize).
 					// Re-evaluate via HEAD on the next iteration to decide retry/restart.
-					log.Log.Info(label + ": resumable patch rejected, " + perr.Error())
+					log.Info(label + ": resumable patch rejected, " + perr.Error())
 				} else {
-					log.Log.Info(label + ": resumable patch failed, " + perr.Error())
+					log.Info(label + ": resumable patch failed, " + perr.Error())
 				}
 				tusBackoff(attempt)
 				patchFailed = true
@@ -304,6 +297,10 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
 // X-Kerberos-Storage-* headers on every request and routing (directory/provider)
 // is additionally carried in the tus Upload-Metadata.
 func uploadVaultResumable(vault models.KStorage, publicKey, deviceKey, fileName, label, slot string) (bool, bool, bool, string, error) {
+	customHeaders, err := parseCustomVaultHeaders(vault.CustomHeaders)
+	if err != nil {
+		return false, false, true, "", err
+	}
 	baseURL := strings.TrimRight(vault.URI, "/") + tusUploadPath
 	metadataValues := map[string]string{
 		"filename":  fileName,
@@ -314,10 +311,22 @@ func uploadVaultResumable(vault models.KStorage, publicKey, deviceKey, fileName,
 		"cloudkey":  publicKey,
 		"fps":       queuedRecordingFPS(fileName),
 	}
+	if encodedHeaders, err := encodeCustomVaultHeaders(customHeaders); err != nil {
+		return false, false, true, "", err
+	} else if encodedHeaders != "" {
+		metadataValues["custom_headers"] = encodedHeaders
+	}
+	customRequestHeaders := http.Header{}
+	if err := setCustomVaultHeaders(customRequestHeaders, customHeaders); err != nil {
+		return false, false, true, "", err
+	}
 	addRecordingTusMetadata(metadataValues, fileName)
 	metadata := encodeTusMetadata(metadataValues)
 	setHeaders := func(h http.Header, fn string) {
 		setVaultTusHeaders(h, vault, publicKey, deviceKey, fn)
+		for name, values := range customRequestHeaders {
+			h[name] = append([]string(nil), values...)
+		}
 	}
 	return runTusUpload(baseURL, metadata, fileName, label, slot, setHeaders)
 }
@@ -348,12 +357,21 @@ func addRecordingTusMetadata(values map[string]string, fileName string) {
 	if !ok {
 		return
 	}
+	if metadata.DeviceKey != "" {
+		values["device"] = metadata.DeviceKey
+	}
+	values["device_name"] = metadata.DeviceName
+	values["region_coordinates"] = metadata.RegionCoordinates
+	values["number_of_changes"] = metadata.NumberOfChanges
 	if metadata.Duration > 0 {
 		values["duration"] = strconv.FormatUint(metadata.Duration, 10)
 	}
 	if metadata.Timestamp > 0 {
 		values["timestamp"] = strconv.FormatInt(metadata.Timestamp, 10)
 	}
+	if metadata.Encrypted {
+		values["encrypted"] = "true"
+	}
 }
 
 // tusCreate performs the tus "creation" request (POST). On success it returns
@@ -487,6 +505,7 @@ func setVaultTusHeaders(h http.Header, vault models.KStorage, publicKey, deviceK
 	h.Set("X-Kerberos-Storage-Capture", "IPCamera")
 	if fileName != "" {
 		h.Set("X-Kerberos-Storage-FileName", fileName)
+		setQueuedRecordingMetadataHeaders(h, fileName)
 	}
 }
 
@@ -503,6 +522,7 @@ func setHubTusHeaders(h http.Header, config *models.Config, fileName string) {
 	h.Set("X-Kerberos-Storage-Capture", "IPCamera")
 	if fileName != "" {
 		h.Set("X-Kerberos-Storage-FileName", fileName)
+		setQueuedRecordingMetadataHeaders(h, fileName)
 	}
 }
 

machinery/src/cloud/tus_client_test.go

@@ -3,6 +3,7 @@ package cloud
 import (
 	"bytes"
 	"encoding/base64"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net"
@@ -17,8 +18,86 @@ import (
 	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/utils"
 )
 
+func TestUploadKerberosVaultReadsDatedLocalRecordings(t *testing.T) {
+	previousCount, previousTimeout := kstorageRetryCount, kstorageRetryTimeout
+	t.Cleanup(func() { kstorageRetryCount, kstorageRetryTimeout = previousCount, previousTimeout })
+	for _, transport := range []string{"resumable", "legacy"} {
+		t.Run(transport, func(t *testing.T) {
+			const fileName = "1564859471123-device-key.mp4"
+			payload := bytes.Repeat([]byte("video"), 1024)
+			withRecording(t, fileName, payload)
+			datedPath, err := utils.CreateRecordingPath("data/recordings", fileName, time.Unix(1564859471, 0))
+			if err != nil {
+				t.Fatal(err)
+			}
+			if err := os.Rename(filepath.Join("data/recordings", fileName), datedPath); err != nil {
+				t.Fatal(err)
+			}
+			withQueuedRecordingFPS(t, fileName, `{"filename":"1564859471123-device-key.mp4","device_key":"device-key","timestamp":1564859471123,"duration":27,"fps":25,"device_name":"camera-name","region_coordinates":"1-2-3-4","number_of_changes":"57"}`)
+			fake := newFakeTus()
+			var legacyBody []byte
+			var legacyHeaders http.Header
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				if transport == "resumable" {
+					fake.ServeHTTP(w, r)
+					return
+				}
+				body, readErr := io.ReadAll(r.Body)
+				if readErr != nil {
+					t.Error(readErr)
+				}
+				legacyBody = body
+				legacyHeaders = r.Header.Clone()
+				w.WriteHeader(http.StatusOK)
+			}))
+			defer server.Close()
+			if transport == "legacy" {
+				t.Setenv("AGENT_DISABLE_RESUMABLE_UPLOAD", "true")
+			} else {
+				t.Setenv("AGENT_DISABLE_RESUMABLE_UPLOAD", "false")
+			}
+			kstorageRetryCount, kstorageRetryTimeout = 0, 0
+			vault := testVault(server.URL)
+			config := &models.Configuration{Config: models.Config{
+				Key:               "changed-config-key",
+				KStorage:          &vault,
+				KStorageSecondary: &models.KStorage{},
+			}}
+			uploaded, configured, err := UploadKerberosVault(config, fileName)
+			if err != nil || !uploaded || !configured {
+				t.Fatalf("UploadKerberosVault() = (%v, %v, %v), want success", uploaded, configured, err)
+			}
+			if transport == "resumable" {
+				if got := fake.totalBytes(); got != int64(len(payload)) {
+					t.Fatalf("uploaded %d bytes, want %d", got, len(payload))
+				}
+				posts := fake.requestsForMethod(http.MethodPost)
+				if len(posts) != 1 {
+					t.Fatalf("got %d creation requests, want 1", len(posts))
+				}
+				metadata := decodeTusMetadata(posts[0].header.Get("Upload-Metadata"))
+				if metadata["filename"] != fileName || metadata["directory"] != "dir" || metadata["timestamp"] != "1564859471123" ||
+					metadata["device"] != "device-key" || metadata["device_name"] != "camera-name" ||
+					metadata["region_coordinates"] != "1-2-3-4" || metadata["number_of_changes"] != "57" {
+					t.Fatalf("unexpected upload metadata: %#v", metadata)
+				}
+			} else if !bytes.Equal(legacyBody, payload) ||
+				legacyHeaders.Get("X-Kerberos-Storage-FileName") != fileName ||
+				legacyHeaders.Get("X-Kerberos-Storage-Directory") != "dir" ||
+				legacyHeaders.Get("X-Kerberos-Storage-Device") != "device-key" ||
+				legacyHeaders.Get(recordingDeviceNameHeader) != "camera-name" ||
+				legacyHeaders.Get(recordingRegionHeader) != "1-2-3-4" ||
+				legacyHeaders.Get(recordingChangesHeader) != "57" ||
+				legacyHeaders.Get(recordingTimestampHeader) != "1564859471123" {
+				t.Fatalf("unexpected legacy upload: %d bytes, headers %#v", len(legacyBody), legacyHeaders)
+			}
+		})
+	}
+}
+
 // fakeUpload tracks the state of a single resumable upload on the fake server.
 type fakeUpload struct {
 	size   int64
@@ -264,6 +343,36 @@ func testVault(uri string) models.KStorage {
 	}
 }
 
+func TestUploadKerberosVaultSkipsEmptyRecording(t *testing.T) {
+	fileName := "1787015373_3-654_office-camera17_0-0-0-0_-1_1960.mp4"
+	withRecording(t, fileName, nil)
+
+	requestCount := 0
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requestCount++
+		w.WriteHeader(http.StatusInternalServerError)
+	}))
+	defer server.Close()
+
+	vault := testVault(server.URL)
+	configuration := &models.Configuration{Config: models.Config{
+		Key:               "device-key",
+		KStorage:          &vault,
+		KStorageSecondary: &models.KStorage{},
+	}}
+
+	uploaded, configured, err := UploadKerberosVault(configuration, fileName)
+	if err != nil {
+		t.Fatalf("UploadKerberosVault() error = %v", err)
+	}
+	if uploaded || configured {
+		t.Fatalf("UploadKerberosVault() uploaded/configured = %v/%v, want false/false", uploaded, configured)
+	}
+	if requestCount != 0 {
+		t.Fatalf("Vault received %d requests, want 0", requestCount)
+	}
+}
+
 func TestUploadVaultResumable_HappyPath(t *testing.T) {
 	srv := newFakeTus()
 	ts := httptest.NewServer(srv)
@@ -274,7 +383,9 @@ func TestUploadVaultResumable_HappyPath(t *testing.T) {
 	withRecording(t, fileName, payload)
 	withQueuedRecordingFPS(t, fileName, `{"filename":"recording.mp4","device_key":"device-key","timestamp":1785934709414,"duration":20452,"fps":29.97}`)
 
-	uploaded, responded, supported, _, err := uploadVaultResumable(testVault(ts.URL), "pk", "dev", fileName, "test", "primary")
+	vault := testVault(ts.URL)
+	vault.CustomHeaders = `{"site_id":"site-1","equipment_id":"equipment-3"}`
+	uploaded, responded, supported, _, err := uploadVaultResumable(vault, "pk", "dev", fileName, "test", "primary")
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -298,6 +409,23 @@ func TestUploadVaultResumable_HappyPath(t *testing.T) {
 	if got := metadata["timestamp"]; got != "1785934709414" {
 		t.Fatalf("POST metadata timestamp = %q, want %q", got, "1785934709414")
 	}
+	var customHeaders map[string]string
+	if err := json.Unmarshal([]byte(metadata["custom_headers"]), &customHeaders); err != nil {
+		t.Fatal(err)
+	}
+	if customHeaders["site_id"] != "site-1" || customHeaders["equipment_id"] != "equipment-3" {
+		t.Fatalf("POST custom header metadata = %#v", customHeaders)
+	}
+	for _, method := range []string{http.MethodPost, http.MethodHead, http.MethodPatch} {
+		for _, request := range srv.requestsForMethod(method) {
+			if request.header.Get("site_id") != "site-1" || request.header.Get("equipment_id") != "equipment-3" {
+				t.Fatalf("%s custom headers = %#v", method, request.header)
+			}
+			if request.header.Get(customHeadersManifestHeader) == "" {
+				t.Fatalf("%s is missing the custom header manifest", method)
+			}
+		}
+	}
 }
 
 func TestQueuedRecordingFPSValidation(t *testing.T) {
@@ -356,7 +484,7 @@ func TestQueuedRecordingFPSAllowsMissingHistoricalMarker(t *testing.T) {
 func TestQueuedRecordingMetadataHeaders(t *testing.T) {
 	fileName := "recording.mp4"
 	withRecording(t, fileName, []byte("recording"))
-	withQueuedRecordingFPS(t, fileName, `{"filename":"recording.mp4","device_key":"device-key","timestamp":1785934709414,"duration":20452,"fps":25}`)
+	withQueuedRecordingFPS(t, fileName, `{"filename":"recording.mp4","device_key":"device-key","timestamp":1785934709414,"duration":20452,"fps":25,"encrypted":true}`)
 
 	header := make(http.Header)
 	setQueuedRecordingMetadataHeaders(header, fileName)
@@ -369,6 +497,14 @@ func TestQueuedRecordingMetadataHeaders(t *testing.T) {
 	if got := header.Get(recordingTimestampHeader); got != "1785934709414" {
 		t.Fatalf("timestamp header = %q", got)
 	}
+	if got := header.Get(recordingEncryptedHeader); got != "true" {
+		t.Fatalf("encrypted header = %q", got)
+	}
+	metadata := map[string]string{}
+	addRecordingTusMetadata(metadata, fileName)
+	if got := metadata["encrypted"]; got != "true" {
+		t.Fatalf("encrypted TUS metadata = %q", got)
+	}
 }
 
 func TestQueuedRecordingFPSAllowsLegacyMarkerFileName(t *testing.T) {

machinery/src/components/backchannel.go

@@ -3,17 +3,16 @@ package components
 import (
 	"bufio"
 	"context"
-	"fmt"
 	"math/rand"
 	"os"
 	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/capture"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/kerberos-io/joy4/av"
 	"github.com/pion/rtp"
+	log "github.com/sirupsen/logrus"
 	"github.com/zaf/g711"
 )
 
@@ -79,7 +78,7 @@ func GetBackChannelAudioCodec(streams []av.CodecData, communication *models.Comm
 			if stream.Type().String() == "PCM_MULAW" {
 				pcmuCodec := stream.(av.AudioCodecData)
 				if pcmuCodec.IsBackChannel() {
-					communication.HasBackChannel = true
+					communication.HasBackChannel.Store(true)
 					return pcmuCodec
 				}
 			}
@@ -88,22 +87,17 @@ func GetBackChannelAudioCodec(streams []av.CodecData, communication *models.Comm
 	return nil
 }
 
-func WriteAudioToBackchannel(communication *models.Communication, rtspClient capture.RTSPClient) {
-	ctx := context.Background()
-	if communication.Context != nil {
-		ctx = *communication.Context
-	}
-
+func WriteAudioToBackchannel(ctx context.Context, communication *models.Communication, rtspClient capture.RTSPClient) {
 	writeAudioToBackchannel(ctx, ctx, communication.HandleAudio, rtspClient)
 }
 
 func writeAudioToBackchannel(ctx context.Context, otelContext context.Context, audioChannel <-chan models.AudioDataPartial, rtspClient backchannelClient) {
-	log.Log.Info("Audio.WriteAudioToBackchannel(): writing to backchannel audio codec")
+	log.Info("Audio.WriteAudioToBackchannel(): writing to backchannel audio codec")
 
 	if err := rtspClient.StartBackChannel(ctx, otelContext); err != nil {
-		log.Log.Error("Audio.WriteAudioToBackchannel(): error starting backchannel: " + err.Error())
+		log.Error("Audio.WriteAudioToBackchannel(): error starting backchannel: " + err.Error())
 		if !reconnectBackchannel(ctx, otelContext, rtspClient) {
-			log.Log.Info("Audio.WriteAudioToBackchannel(): stopped while reconnecting")
+			log.Info("Audio.WriteAudioToBackchannel(): stopped while reconnecting")
 			return
 		}
 	}
@@ -112,11 +106,11 @@ func writeAudioToBackchannel(ctx context.Context, otelContext context.Context, a
 	for {
 		select {
 		case <-ctx.Done():
-			log.Log.Info("Audio.WriteAudioToBackchannel(): stopped")
+			log.Info("Audio.WriteAudioToBackchannel(): stopped")
 			return
 		case audio, ok := <-audioChannel:
 			if !ok {
-				log.Log.Info("Audio.WriteAudioToBackchannel(): finished")
+				log.Info("Audio.WriteAudioToBackchannel(): finished")
 				return
 			}
 
@@ -127,9 +121,9 @@ func writeAudioToBackchannel(ctx context.Context, otelContext context.Context, a
 
 			pkt := packetizer.packet(audio, time.Now())
 			if err := rtspClient.WritePacket(pkt); err != nil {
-				log.Log.Error("Audio.WriteAudioToBackchannel(): error writing packet to backchannel: " + err.Error())
+				log.Error("Audio.WriteAudioToBackchannel(): error writing packet to backchannel: " + err.Error())
 				if !reconnectBackchannel(ctx, otelContext, rtspClient) {
-					log.Log.Info("Audio.WriteAudioToBackchannel(): stopped while reconnecting")
+					log.Info("Audio.WriteAudioToBackchannel(): stopped while reconnecting")
 					return
 				}
 				packetizer = newBackchannelPacketizer()
@@ -137,7 +131,7 @@ func writeAudioToBackchannel(ctx context.Context, otelContext context.Context, a
 			}
 
 			if !waitForBackchannel(ctx, time.Duration(len(audio.Data))*time.Second/backchannelSampleRate) {
-				log.Log.Info("Audio.WriteAudioToBackchannel(): stopped")
+				log.Info("Audio.WriteAudioToBackchannel(): stopped")
 				return
 			}
 		}
@@ -162,7 +156,7 @@ func reconnectBackchannel(ctx context.Context, otelContext context.Context, rtsp
 	backoff := backchannelReconnectInitial
 	for {
 		if err := rtspClient.Close(otelContext); err != nil {
-			log.Log.Error("Audio.WriteAudioToBackchannel(): error closing failed backchannel: " + err.Error())
+			log.Error("Audio.WriteAudioToBackchannel(): error closing failed backchannel: " + err.Error())
 		}
 		if ctx.Err() != nil {
 			return false
@@ -173,11 +167,11 @@ func reconnectBackchannel(ctx context.Context, otelContext context.Context, rtsp
 			err = rtspClient.StartBackChannel(ctx, otelContext)
 		}
 		if err == nil {
-			log.Log.Info("Audio.WriteAudioToBackchannel(): reconnected backchannel")
+			log.Info("Audio.WriteAudioToBackchannel(): reconnected backchannel")
 			return true
 		}
 
-		log.Log.Error("Audio.WriteAudioToBackchannel(): error reconnecting backchannel: " + err.Error())
+		log.Error("Audio.WriteAudioToBackchannel(): error reconnecting backchannel: " + err.Error())
 		if !waitForBackchannel(ctx, backoff) {
 			return false
 		}
@@ -204,7 +198,12 @@ func WriteFileToBackChannel(infile av.DemuxCloser) {
 	// Do the warmup!
 	file, err := os.Open("./audiofile.bye")
 	if err != nil {
-		fmt.Println("WriteFileToBackChannel: error opening audiofile.bye file")
+		log.WithError(err).WithFields(log.Fields{
+			"component": "backchannel",
+			"event":     "audio_file_open_failed",
+			"path":      "./audiofile.bye",
+		}).Error("Failed to open backchannel audio file")
+		return
 	}
 	defer file.Close()
 

machinery/src/components/watchdog_test.go

@@ -0,0 +1,82 @@
+package components
+
+import (
+	"context"
+	"testing"
+	"time"
+)
+
+func TestWaitForRunRetryStopsOnCancellation(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	started := time.Now()
+	if waitForRunRetry(ctx) {
+		t.Fatal("waitForRunRetry() completed the retry delay after cancellation")
+	}
+	if elapsed := time.Since(started); elapsed > 100*time.Millisecond {
+		t.Fatalf("waitForRunRetry() took %s after cancellation", elapsed)
+	}
+}
+
+func TestStreamRestartWatchdogCoalescesStallsAndBacksOff(t *testing.T) {
+	now := time.Unix(1_000, 0)
+	watchdog := newStreamRestartWatchdog()
+	if _, restart := watchdog.Observe(now, 1, 1, true, false); restart {
+		t.Fatal("initial observation requested a restart")
+	}
+
+	for check := 1; check < streamWatchdogStallChecks; check++ {
+		now = now.Add(streamWatchdogInterval)
+		if _, restart := watchdog.Observe(now, 1, 1, true, false); restart {
+			t.Fatalf("check %d requested an early restart", check)
+		}
+	}
+	now = now.Add(streamWatchdogInterval)
+	reason, restart := watchdog.Observe(now, 1, 1, true, false)
+	if !restart || reason != "main and sub streams" {
+		t.Fatalf("Observe() = (%q, %t), want coalesced restart", reason, restart)
+	}
+
+	watchdog.MarkRestart(now)
+	if got := watchdog.Backoff(); got != 30*time.Second {
+		t.Fatalf("Backoff() = %s, want 30s", got)
+	}
+	for now = now.Add(streamWatchdogInterval); now.Before(watchdog.nextRestart); now = now.Add(streamWatchdogInterval) {
+		if _, restart := watchdog.Observe(now, 1, 1, true, false); restart {
+			t.Fatal("restart requested during cooldown")
+		}
+	}
+}
+
+func TestStreamRestartWatchdogResetsAfterHealthyMinute(t *testing.T) {
+	now := time.Unix(2_000, 0)
+	watchdog := newStreamRestartWatchdog()
+	watchdog.backoff = streamWatchdogMaxBackoff
+	watchdog.nextRestart = now.Add(streamWatchdogMaxBackoff)
+	watchdog.Observe(now, 1, 1, true, false)
+
+	for elapsed := streamWatchdogInterval; elapsed <= streamWatchdogHealthyReset+streamWatchdogInterval; elapsed += streamWatchdogInterval {
+		now = now.Add(streamWatchdogInterval)
+		watchdog.Observe(now, int64(elapsed), int64(elapsed), true, false)
+	}
+
+	if got := watchdog.Backoff(); got != streamWatchdogBaseBackoff {
+		t.Fatalf("Backoff() = %s, want %s", got, streamWatchdogBaseBackoff)
+	}
+	if !watchdog.nextRestart.IsZero() {
+		t.Fatalf("nextRestart = %s, want zero", watchdog.nextRestart)
+	}
+}
+
+func TestStreamRestartWatchdogPausesWhileConfiguring(t *testing.T) {
+	now := time.Unix(3_000, 0)
+	watchdog := newStreamRestartWatchdog()
+	watchdog.Observe(now, 1, 0, false, false)
+	for check := 0; check < streamWatchdogStallChecks+1; check++ {
+		now = now.Add(streamWatchdogInterval)
+		if _, restart := watchdog.Observe(now, 1, 0, false, true); restart {
+			t.Fatal("restart requested while configuring")
+		}
+	}
+}

machinery/src/computervision/main.go

@@ -8,14 +8,14 @@ import (
 	geo "github.com/kellydunn/golang-geo"
 	"github.com/kerberos-io/agent/machinery/src/capture"
 	"github.com/kerberos-io/agent/machinery/src/conditions"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
+	log "github.com/sirupsen/logrus"
 )
 
 func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Configuration, communication *models.Communication, mqttClient mqtt.Client, rtspClient capture.RTSPClient) {
 
-	log.Log.Debug("computervision.main.ProcessMotion(): start motion detection")
+	log.Debug("computervision.main.ProcessMotion(): start motion detection")
 	config := configuration.Config
 	loc, _ := time.LoadLocation(config.Timezone)
 
@@ -42,14 +42,14 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 
 	if continuousMode && !hasMotionRegion {
 
-		log.Log.Info("computervision.main.ProcessMotion(): continuous recording enabled and no motion region configured, so no motion detection required.")
+		log.Info("computervision.main.ProcessMotion(): continuous recording enabled and no motion region configured, so no motion detection required.")
 
 	} else {
 
 		if continuousMode {
-			log.Log.Info("computervision.main.ProcessMotion(): continuous recording enabled with a motion region, running motion detection for live-view visualisation only (no motion-triggered recording).")
+			log.Info("computervision.main.ProcessMotion(): continuous recording enabled with a motion region, running motion detection for live-view visualisation only (no motion-triggered recording).")
 		} else {
-			log.Log.Info("computervision.main.ProcessMotion(): motion detected is enabled, so starting the motion detection.")
+			log.Info("computervision.main.ProcessMotion(): motion detected is enabled, so starting the motion detection.")
 		}
 
 		hubKey := config.HubKey
@@ -185,7 +185,7 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 				// We'll validate those conditions and if not valid we'll not do anything.
 				detectMotion, err := conditions.Validate(loc, configuration)
 				if !detectMotion && err != nil {
-					log.Log.Debug("computervision.main.ProcessMotion(): " + err.Error() + ".")
+					log.Debug("computervision.main.ProcessMotion(): " + err.Error() + ".")
 				}
 
 				// Run detection when motion is enabled, OR when we're in continuous
@@ -235,7 +235,7 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 										if err == nil {
 											mqttClient.Publish("kerberos/hub/"+hubKey, 2, false, payload)
 										} else {
-											log.Log.Info("computervision.main.ProcessMotion(): failed to package MQTT message: " + err.Error())
+											log.Info("computervision.main.ProcessMotion(): failed to package MQTT message: " + err.Error())
 										}
 									} else {
 										mqttClient.Publish("kerberos/agent/"+deviceKey, 2, false, "motion")
@@ -254,7 +254,9 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 									NumberOfChanges: changesToReturn,
 									Rectangle:       motionRectangle,
 								}
-								communication.HandleMotion <- dataToPass //Save data to the channel
+								if !communication.TrySendMotion(dataToPass) {
+									log.Warn("computervision.main.ProcessMotion(): motion channel unavailable or full, dropping recording trigger")
+								}
 							}
 						}
 					}
@@ -271,7 +273,7 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 		}
 	}
 
-	log.Log.Debug("computervision.main.ProcessMotion(): stop the motion detection.")
+	log.Debug("computervision.main.ProcessMotion(): stop the motion detection.")
 }
 
 func FindMotion(imageArray [3]*image.Gray, coordinatesPerRegion [][]int, pixelChangeThreshold int) (thresholdReached bool, changesDetected int, motionRectangle models.MotionRectangle, motionRectangles []models.MotionRectangle) {
@@ -374,14 +376,28 @@ func AbsDiffBitwiseAndThreshold(img1 *image.Gray, img2 *image.Gray, img3 *image.
 				endY = pixel[1]
 			}
 		}
-		log.Log.Debugf("Rectangle of changes detected: startX: %d, startY: %d, endX: %d, endY: %d", startX, startY, endX, endY)
+		log.WithFields(log.Fields{
+			"component": "computer_vision",
+			"end_x":     endX,
+			"end_y":     endY,
+			"event":     "motion_bounds_detected",
+			"start_x":   startX,
+			"start_y":   startY,
+		}).Debug("Motion bounds detected")
 		motionRectangle = models.MotionRectangle{
 			X:      startX,
 			Y:      startY,
 			Width:  endX - startX,
 			Height: endY - startY,
 		}
-		log.Log.Debugf("Motion rectangle: %+v", motionRectangle)
+		log.WithFields(log.Fields{
+			"component": "computer_vision",
+			"event":     "motion_rectangle_created",
+			"height":    motionRectangle.Height,
+			"width":     motionRectangle.Width,
+			"x":         motionRectangle.X,
+			"y":         motionRectangle.Y,
+		}).Debug("Motion rectangle created")
 	}
 
 	// Cluster the changed pixels into separate bounding boxes so the live view can

machinery/src/conditions/timewindow.go

@@ -3,8 +3,8 @@ package conditions
 import (
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 func IsWithinTimeInterval(loc *time.Location, configuration *models.Configuration) (enabled bool) {
@@ -27,9 +27,9 @@ func IsWithinTimeInterval(loc *time.Location, configuration *models.Configuratio
 				currentTimeInSeconds := hour*60*60 + minute*60 + second
 				if (currentTimeInSeconds >= start1 && currentTimeInSeconds <= end1) ||
 					(currentTimeInSeconds >= start2 && currentTimeInSeconds <= end2) {
-					log.Log.Debug("conditions.timewindow.IsWithinTimeInterval(): time interval valid, enabling recording.")
+					log.Debug("conditions.timewindow.IsWithinTimeInterval(): time interval valid, enabling recording.")
 				} else {
-					log.Log.Info("conditions.timewindow.IsWithinTimeInterval(): time interval not valid, disabling recording.")
+					log.Info("conditions.timewindow.IsWithinTimeInterval(): time interval not valid, disabling recording.")
 					enabled = false
 				}
 			}

machinery/src/conditions/uri.go

@@ -3,13 +3,26 @@ package conditions
 import (
 	"bytes"
 	"crypto/tls"
-	"fmt"
+	"encoding/json"
+	"io"
 	"net/http"
 	"os"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
+)
+
+const conditionHTTPTimeout = 10 * time.Second
+
+var (
+	conditionHTTPClient         = &http.Client{Timeout: conditionHTTPTimeout}
+	conditionInsecureHTTPClient = &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // #nosec G402 -- explicit operator opt-in
+		},
+		Timeout: conditionHTTPTimeout,
+	}
 )
 
 func IsValidUriResponse(configuration *models.Configuration) (enabled bool) {
@@ -17,41 +30,66 @@ func IsValidUriResponse(configuration *models.Configuration) (enabled bool) {
 	conditionURI := config.ConditionURI
 	enabled = true
 	if conditionURI != "" {
-
-		// We will send a POST request to the conditionURI, and expect a 200 response.
-		// In the payload we will send some information, so the other end can decide
-		// if it should enable or disable recording.
-
-		var client *http.Client
+		client := conditionHTTPClient
 		if os.Getenv("AGENT_TLS_INSECURE") == "true" {
-			tr := &http.Transport{
-				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-			}
-			client = &http.Client{Transport: tr}
-		} else {
-			client = &http.Client{}
+			client = conditionInsecureHTTPClient
 		}
 
-		var object = fmt.Sprintf(`{
-			"camera_id" : "%s",
-			"camera_name" : "%s",
-			"site_id" : "%s",
-			"hub_key" : "%s",
-			"timestamp" : "%s",
-		}`, config.Key, config.FriendlyName, config.HubSite, config.HubKey, time.Now().Format("2006-01-02 15:04:05"))
+		payload := struct {
+			CameraID   string `json:"camera_id"`
+			CameraName string `json:"camera_name"`
+			SiteID     string `json:"site_id"`
+			HubKey     string `json:"hub_key"`
+			Timestamp  string `json:"timestamp"`
+		}{
+			CameraID:   config.Key,
+			CameraName: config.FriendlyName,
+			SiteID:     config.HubSite,
+			HubKey:     config.HubKey,
+			Timestamp:  time.Now().Format("2006-01-02 15:04:05"),
+		}
+		jsonBody, err := json.Marshal(payload)
+		if err != nil {
+			log.WithError(err).WithFields(log.Fields{
+				"component": "conditions/uri",
+				"event":     "request_encoding_failed",
+			}).Error("Failed to encode condition request")
+			return false
+		}
 
-		var jsonStr = []byte(object)
-		buffy := bytes.NewBuffer(jsonStr)
-		req, _ := http.NewRequest("POST", conditionURI, buffy)
+		req, err := http.NewRequest(http.MethodPost, conditionURI, bytes.NewReader(jsonBody))
+		if err != nil {
+			log.WithError(err).WithFields(log.Fields{
+				"component": "conditions/uri",
+				"event":     "request_creation_failed",
+			}).Error("Failed to create condition request")
+			return false
+		}
 		req.Header.Set("Content-Type", "application/json")
 		resp, err := client.Do(req)
 		if resp != nil {
+			_, _ = io.Copy(io.Discard, resp.Body)
 			resp.Body.Close()
 		}
-		if err == nil && resp.StatusCode == 200 {
-			log.Log.Info("conditions.uri.IsValidUriResponse(): response 200, enabling recording.")
+		if err == nil && resp != nil && resp.StatusCode == http.StatusOK {
+			log.WithFields(log.Fields{
+				"component":   "conditions/uri",
+				"event":       "recording_enabled",
+				"status_code": resp.StatusCode,
+			}).Info("Condition request enabled recording")
 		} else {
-			log.Log.Info("conditions.uri.IsValidUriResponse(): response not 200, disabling recording.")
+			if err != nil {
+				log.WithError(err).WithFields(log.Fields{
+					"component": "conditions/uri",
+					"event":     "request_failed",
+				}).Error("Condition request failed")
+			} else {
+				log.WithFields(log.Fields{
+					"component":   "conditions/uri",
+					"event":       "recording_disabled",
+					"status_code": resp.StatusCode,
+				}).Info("Condition request disabled recording")
+			}
 			enabled = false
 		}
 	}

machinery/src/conditions/uri_test.go

@@ -0,0 +1,63 @@
+package conditions
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+
+	"github.com/kerberos-io/agent/machinery/src/models"
+)
+
+func TestIsValidUriResponseReusesClientAndEncodesPayload(t *testing.T) {
+	var requests atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		requests.Add(1)
+		if request.Method != http.MethodPost {
+			t.Errorf("method = %s, want POST", request.Method)
+		}
+		if got := request.Header.Get("Content-Type"); got != "application/json" {
+			t.Errorf("Content-Type = %q, want application/json", got)
+		}
+		var payload struct {
+			CameraID   string `json:"camera_id"`
+			CameraName string `json:"camera_name"`
+		}
+		if err := json.NewDecoder(request.Body).Decode(&payload); err != nil {
+			t.Errorf("decode request: %v", err)
+		}
+		if payload.CameraID != "camera-1" || payload.CameraName != `Front "Door"` {
+			t.Errorf("payload = %+v", payload)
+		}
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer server.Close()
+
+	configuration := &models.Configuration{}
+	configuration.Config.ConditionURI = server.URL
+	configuration.Config.Key = "camera-1"
+	configuration.Config.FriendlyName = `Front "Door"`
+
+	for i := 0; i < 2; i++ {
+		if !IsValidUriResponse(configuration) {
+			t.Fatal("IsValidUriResponse() = false, want true")
+		}
+	}
+	if got := requests.Load(); got != 2 {
+		t.Fatalf("requests = %d, want 2", got)
+	}
+}
+
+func TestIsValidUriResponseRejectsNonOK(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		http.Error(w, "unavailable", http.StatusServiceUnavailable)
+	}))
+	defer server.Close()
+
+	configuration := &models.Configuration{}
+	configuration.Config.ConditionURI = server.URL
+	if IsValidUriResponse(configuration) {
+		t.Fatal("IsValidUriResponse() = true, want false")
+	}
+}

machinery/src/config/main.go

@@ -13,11 +13,36 @@ import (
 
 	"github.com/InVisionApp/conjungo"
 	"github.com/kerberos-io/agent/machinery/src/database"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
 )
 
+const factoryConfigRetryDelay = 5 * time.Second
+
+func newFactoryConfigReadContext() (context.Context, context.CancelFunc) {
+	return context.WithTimeout(context.Background(), database.TIMEOUT)
+}
+
+func isRetryableFactoryConfigReadError(err error) bool {
+	return errors.Is(err, context.DeadlineExceeded) || mongo.IsTimeout(err) || mongo.IsNetworkError(err)
+}
+
+func readFactoryConfig(collection *mongo.Collection, filter bson.M, destination *models.Config) error {
+	for {
+		ctx, cancel := newFactoryConfigReadContext()
+		err := collection.FindOne(ctx, filter).Decode(destination)
+		cancel()
+		if !isRetryableFactoryConfigReadError(err) {
+			return err
+		}
+
+		log.Warn("Factory configuration read timed out or lost its database connection; retrying.")
+		time.Sleep(factoryConfigRetryDelay)
+	}
+}
+
 // ReadUserConfig Reads the user configuration of the Kerberos Open Source instance.
 // This will return a models.User struct including the username, password,
 // selected language, and if the installation was completed or not.
@@ -25,14 +50,14 @@ func ReadUserConfig(configDirectory string) (userConfig models.User) {
 	for {
 		jsonFile, err := os.Open(configDirectory + "/data/config/user.json")
 		if err != nil {
-			log.Log.Error("Config file is not found " + configDirectory + "/data/config/user.json, trying again in 5s: " + err.Error())
+			log.Error("Config file is not found " + configDirectory + "/data/config/user.json, trying again in 5s: " + err.Error())
 			time.Sleep(5 * time.Second)
 		} else {
-			log.Log.Info("Successfully Opened user.json")
+			log.Info("Successfully Opened user.json")
 			byteValue, _ := ioutil.ReadAll(jsonFile)
 			err = json.Unmarshal(byteValue, &userConfig)
 			if err != nil {
-				log.Log.Error("JSON file not valid: " + err.Error())
+				log.Error("JSON file not valid: " + err.Error())
 			} else {
 				jsonFile.Close()
 				break
@@ -63,21 +88,15 @@ func OpenConfig(configDirectory string, configuration *models.Configuration) {
 		collection := db.Collection("configuration")
 
 		var globalConfig models.Config
-		res := collection.FindOne(context.Background(), bson.M{
+		err := readFactoryConfig(collection, bson.M{
 			"type": "global",
-		})
-
-		if res.Err() != nil {
-			log.Log.Error("Could not find global configuration, using default configuration.")
-			panic("Could not find global configuration, using default configuration.")
-		}
-		err := res.Decode(&globalConfig)
+		}, &globalConfig)
 		if err != nil {
-			log.Log.Error("Could not find global configuration, using default configuration.")
+			log.Error("Could not find global configuration, using default configuration.")
 			panic("Could not find global configuration, using default configuration.")
 		}
 		if globalConfig.Type != "global" {
-			log.Log.Error("Could not find global configuration, might missed the mongodb connection.")
+			log.Error("Could not find global configuration, might missed the mongodb connection.")
 			panic("Could not find global configuration, might missed the mongodb connection.")
 		}
 
@@ -85,21 +104,16 @@ func OpenConfig(configDirectory string, configuration *models.Configuration) {
 
 		var customConfig models.Config
 		deploymentName := os.Getenv("DEPLOYMENT_NAME")
-		res = collection.FindOne(context.Background(), bson.M{
+		err = readFactoryConfig(collection, bson.M{
 			"type": "config",
 			"name": deploymentName,
-		})
-		if res.Err() != nil {
-			log.Log.Error("Could not find configuration for " + deploymentName + ", using global configuration.")
-		}
-		err = res.Decode(&customConfig)
+		}, &customConfig)
 		if err != nil {
-			log.Log.Error("Could not find configuration for " + deploymentName + ", using global configuration.")
-		}
-
-		if customConfig.Type != "config" {
-			log.Log.Error("Could not find custom configuration, might missed the mongodb connection.")
-			panic("Could not find custom configuration, might missed the mongodb connection.")
+			log.Error("Could not find configuration for " + deploymentName + ", using global configuration.")
+			customConfig = models.Config{}
+		} else if customConfig.Type != "config" {
+			log.Error("Custom configuration has an invalid type, using global configuration.")
+			customConfig = models.Config{}
 		}
 		configuration.CustomConfig = customConfig
 
@@ -166,21 +180,29 @@ func OpenConfig(configDirectory string, configuration *models.Configuration) {
 
 		// Open device config
 		for {
-			jsonFile, err := os.Open(configDirectory + "/data/config/config.json")
+			configPath := configDirectory + "/data/config/config.json"
+			jsonFile, err := os.Open(configPath)
 			if err != nil {
-				log.Log.Error("Config file is not found " + configDirectory + "/data/config/config.json" + ", trying again in 5s.")
+				log.WithError(err).WithFields(log.Fields{
+					"component":      "config",
+					"event":          "configuration_open_failed",
+					"retry_delay_ms": 5000,
+				}).Warn("Configuration file unavailable; retrying")
 				time.Sleep(5 * time.Second)
 			} else {
-				log.Log.Info("Successfully Opened config.json from " + configuration.Name)
+				log.WithFields(log.Fields{
+					"component": "config",
+					"event":     "configuration_opened",
+				}).Info("Configuration file opened")
 				byteValue, _ := ioutil.ReadAll(jsonFile)
 				err = json.Unmarshal(byteValue, &configuration.Config)
 				jsonFile.Close()
 				if err != nil {
-					log.Log.Error("JSON file not valid: " + err.Error())
+					log.Error("JSON file not valid: " + err.Error())
 				} else {
 					err = json.Unmarshal(byteValue, &configuration.CustomConfig)
 					if err != nil {
-						log.Log.Error("JSON file not valid: " + err.Error())
+						log.Error("JSON file not valid: " + err.Error())
 					} else {
 						break
 					}
@@ -293,6 +315,15 @@ func initConfigPointers(config *models.Config) {
 func applyAgentEnvVars(configuration *models.Configuration, prefix string, applyDefaults bool) {
 	environmentVariables := os.Environ()
 
+	if prefix == "" {
+		if deploymentName := os.Getenv("DEPLOYMENT_NAME"); deploymentName != "" {
+			configuration.Config.Name = deploymentName
+			if configuration.Config.FriendlyName == "" {
+				configuration.Config.FriendlyName = deploymentName
+			}
+		}
+	}
+
 	// Initialize the configuration for some new fields.
 	if configuration.Config.KStorageSecondary == nil {
 		configuration.Config.KStorageSecondary = &models.KStorage{}
@@ -581,6 +612,9 @@ func applyAgentEnvVars(configuration *models.Configuration, prefix string, apply
 			case "AGENT_KERBEROSVAULT_DIRECTORY":
 				configuration.Config.KStorage.Directory = value
 				break
+			case "AGENT_KERBEROSVAULT_CUSTOM_HEADERS":
+				configuration.Config.KStorage.CustomHeaders = value
+				break
 
 			/* Retry policy and timeout */
 			case "AGENT_KERBEROSVAULT_MAX_RETRIES":
@@ -612,6 +646,9 @@ func applyAgentEnvVars(configuration *models.Configuration, prefix string, apply
 			case "AGENT_KERBEROSVAULT_SECONDARY_DIRECTORY":
 				configuration.Config.KStorageSecondary.Directory = value
 				break
+			case "AGENT_KERBEROSVAULT_SECONDARY_CUSTOM_HEADERS":
+				configuration.Config.KStorageSecondary.CustomHeaders = value
+				break
 
 			/* When storing in dropbox */
 			case "AGENT_DROPBOX_ACCESS_TOKEN":
@@ -687,12 +724,12 @@ func SaveConfig(configDirectory string, config models.Config, configuration *mod
 			return err
 		}
 
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			select {
 			case communication.HandleBootstrap <- "restart":
-				log.Log.Info("config.main.SaveConfig(): update config, restart agent.")
+				log.Info("config.main.SaveConfig(): update config, restart agent.")
 			case <-time.After(1 * time.Second):
-				log.Log.Info("config.main.SaveConfig(): update config, restart agent.")
+				log.Info("config.main.SaveConfig(): update config, restart agent.")
 			}
 		}
 

machinery/src/config/main_test.go

@@ -1,11 +1,64 @@
 package config
 
 import (
+	"context"
+	"errors"
+	"os"
 	"testing"
+	"time"
 
+	"github.com/kerberos-io/agent/machinery/src/database"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	"go.mongodb.org/mongo-driver/mongo"
 )
 
+func TestApplyAgentEnvVarsDeploymentName(t *testing.T) {
+	tests := []struct {
+		name           string
+		prefix         string
+		deploymentName string
+		agentName      string
+		initialName    string
+		initialDisplay string
+		wantName       string
+		wantDisplay    string
+	}{
+		{name: "deployment fallback", deploymentName: "camera-1", wantName: "camera-1", wantDisplay: "camera-1"},
+		{name: "explicit display name", deploymentName: "camera-1", agentName: "Front Door", wantName: "camera-1", wantDisplay: "Front Door"},
+		{name: "existing display name", deploymentName: "camera-1", initialDisplay: "Front Door", wantName: "camera-1", wantDisplay: "Front Door"},
+		{name: "deployment overrides bundled identity", deploymentName: "camera-1", initialName: "default", wantName: "camera-1", wantDisplay: "camera-1"},
+		{name: "no deployment", initialName: "existing", initialDisplay: "Existing Camera", wantName: "existing", wantDisplay: "Existing Camera"},
+		{name: "standalone display name", agentName: "Front Door", wantDisplay: "Front Door"},
+		{name: "global layer excludes deployment", prefix: "GLOBAL_", deploymentName: "camera-1"},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			t.Setenv("DEPLOYMENT_NAME", test.deploymentName)
+			t.Setenv(test.prefix+"AGENT_NAME", test.agentName)
+			if test.agentName == "" {
+				if err := os.Unsetenv(test.prefix + "AGENT_NAME"); err != nil {
+					t.Fatal(err)
+				}
+			}
+			configuration := &models.Configuration{Config: models.Config{
+				Name:         test.initialName,
+				FriendlyName: test.initialDisplay,
+			}}
+			initConfigPointers(&configuration.Config)
+
+			applyAgentEnvVars(configuration, test.prefix, false)
+
+			if got := configuration.Config.Name; got != test.wantName {
+				t.Errorf("Name = %q, want %q", got, test.wantName)
+			}
+			if got := configuration.Config.FriendlyName; got != test.wantDisplay {
+				t.Errorf("FriendlyName = %q, want %q", got, test.wantDisplay)
+			}
+		})
+	}
+}
+
 func TestApplyAgentEnvVarsPixelChangeThresholdDefault(t *testing.T) {
 	tests := []struct {
 		name      string
@@ -35,6 +88,55 @@ func TestApplyAgentEnvVarsPixelChangeThresholdDefault(t *testing.T) {
 	}
 }
 
+func TestApplyAgentEnvVarsVaultCustomHeaders(t *testing.T) {
+	t.Setenv("AGENT_KERBEROSVAULT_CUSTOM_HEADERS", `{"site_id":"site-1"}`)
+	t.Setenv("AGENT_KERBEROSVAULT_SECONDARY_CUSTOM_HEADERS", `{"site_id":"site-2"}`)
+	configuration := &models.Configuration{}
+	initConfigPointers(&configuration.Config)
+
+	applyAgentEnvVars(configuration, "", false)
+
+	if got := configuration.Config.KStorage.CustomHeaders; got != `{"site_id":"site-1"}` {
+		t.Fatalf("primary custom headers = %q", got)
+	}
+	if got := configuration.Config.KStorageSecondary.CustomHeaders; got != `{"site_id":"site-2"}` {
+		t.Fatalf("secondary custom headers = %q", got)
+	}
+}
+
 func intPointer(value int) *int {
 	return &value
 }
+
+func TestNewFactoryConfigReadContextUsesDatabaseTimeout(t *testing.T) {
+	ctx, cancel := newFactoryConfigReadContext()
+	defer cancel()
+
+	deadline, ok := ctx.Deadline()
+	if !ok {
+		t.Fatal("expected read context deadline")
+	}
+
+	remaining := time.Until(deadline)
+	if remaining <= 0 {
+		t.Fatalf("deadline already expired: %v", remaining)
+	}
+	if remaining > database.TIMEOUT {
+		t.Fatalf("remaining deadline = %v, want <= %v", remaining, database.TIMEOUT)
+	}
+	if remaining < database.TIMEOUT-time.Second {
+		t.Fatalf("remaining deadline = %v, want close to %v", remaining, database.TIMEOUT)
+	}
+}
+
+func TestFactoryConfigRetryableErrors(t *testing.T) {
+	if !isRetryableFactoryConfigReadError(context.DeadlineExceeded) {
+		t.Fatal("context deadline should be retryable")
+	}
+	if isRetryableFactoryConfigReadError(mongo.ErrNoDocuments) {
+		t.Fatal("missing configuration should not retry")
+	}
+	if isRetryableFactoryConfigReadError(errors.New("invalid BSON")) {
+		t.Fatal("decode errors should not retry")
+	}
+}

machinery/src/database/main.go

@@ -7,6 +7,7 @@ import (
 	"sync"
 	"time"
 
+	log "github.com/sirupsen/logrus"
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
 )
@@ -44,18 +45,34 @@ func New() *DB {
 		// We can also apply the complete URI
 		// e.g. "mongodb+srv://<username>:<password>@kerberos-hub.shhng.mongodb.net/?retryWrites=true&w=majority&appName=kerberos-hub"
 		if mongodbURI != "" {
+			log.WithFields(log.Fields{
+				"component":      "database",
+				"database":       DatabaseName,
+				"event":          "client_configuring",
+				"uri_configured": true,
+			}).Debug("Configuring MongoDB client")
 			serverAPI := options.ServerAPI(options.ServerAPIVersion1)
 			opts := options.Client().ApplyURI(mongodbURI).SetServerAPIOptions(serverAPI)
 
 			// Create a new client and connect to the server
 			client, err := mongo.Connect(ctx, opts)
 			if err != nil {
-				fmt.Printf("Error setting up mongodb connection: %+v\n", err)
-				os.Exit(1)
+				log.WithError(err).WithFields(log.Fields{
+					"component": "database",
+					"database":  DatabaseName,
+					"event":     "client_configuration_failed",
+				}).Fatal("Failed to configure MongoDB client")
 			}
 			_instance.Client = client
 
 		} else {
+			log.WithFields(log.Fields{
+				"component":             "database",
+				"database":              DatabaseName,
+				"event":                 "client_configuring",
+				"replicaset_configured": replicaset != "",
+				"uri_configured":        false,
+			}).Debug("Configuring MongoDB client")
 
 			// New MongoDB driver
 			mongodbURI := fmt.Sprintf("mongodb://%s:%s@%s", username, password, host)
@@ -69,8 +86,11 @@ func New() *DB {
 				Password:      password,
 			}))
 			if err != nil {
-				fmt.Printf("Error setting up mongodb connection: %+v\n", err)
-				os.Exit(1)
+				log.WithError(err).WithFields(log.Fields{
+					"component": "database",
+					"database":  DatabaseName,
+					"event":     "client_configuration_failed",
+				}).Fatal("Failed to configure MongoDB client")
 			}
 			_instance.Client = client
 		}

machinery/src/lifecycle/supervisor.go

@@ -0,0 +1,336 @@
+package lifecycle
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"runtime/debug"
+	"sort"
+	"sync"
+	"time"
+)
+
+var (
+	ErrSupervisorSealed  = errors.New("supervisor is sealed")
+	ErrSupervisorStopped = errors.New("supervisor is shutting down")
+	ErrTaskExists        = errors.New("task already exists")
+)
+
+type TaskFunc func(context.Context) error
+
+type TaskPolicy struct {
+	Required    bool
+	LongRunning bool
+}
+
+type TaskStatus string
+
+const (
+	TaskRunning   TaskStatus = "running"
+	TaskSucceeded TaskStatus = "succeeded"
+	TaskFailed    TaskStatus = "failed"
+	TaskPanicked  TaskStatus = "panicked"
+)
+
+type TaskSnapshot struct {
+	Name      string
+	Policy    TaskPolicy
+	Status    TaskStatus
+	StartedAt time.Time
+	EndedAt   time.Time
+	Error     string
+	Panic     string
+	Stack     []byte
+}
+
+type Failure struct {
+	Task  string
+	Cause error
+	Panic string
+	Stack []byte
+}
+
+func (f Failure) Error() string {
+	if f.Cause == nil {
+		return fmt.Sprintf("task %q failed", f.Task)
+	}
+	return f.Cause.Error()
+}
+
+type ShutdownReport struct {
+	Complete bool
+	Cause    error
+	Tasks    []TaskSnapshot
+	Running  []TaskSnapshot
+}
+
+type supervisorPhase uint8
+
+const (
+	phaseStarting supervisorPhase = iota
+	phaseRunning
+	phaseDraining
+)
+
+type taskState struct {
+	TaskSnapshot
+}
+
+type Supervisor struct {
+	ctx    context.Context
+	cancel context.CancelCauseFunc
+
+	mu        sync.Mutex
+	phase     supervisorPhase
+	sealed    bool
+	active    int
+	tasks     map[string]*taskState
+	allDone   chan struct{}
+	doneOnce  sync.Once
+	start     chan struct{}
+	startOnce sync.Once
+
+	failures    chan Failure
+	failureOnce sync.Once
+
+	stopParentCancel func() bool
+}
+
+func NewSupervisor(parent context.Context) *Supervisor {
+	if parent == nil {
+		parent = context.Background()
+	}
+
+	ctx, cancel := context.WithCancelCause(parent)
+	supervisor := &Supervisor{
+		ctx:      ctx,
+		cancel:   cancel,
+		phase:    phaseStarting,
+		tasks:    make(map[string]*taskState),
+		allDone:  make(chan struct{}),
+		start:    make(chan struct{}),
+		failures: make(chan Failure, 1),
+	}
+
+	stopParentCancel := context.AfterFunc(parent, func() {
+		supervisor.BeginShutdown(context.Cause(parent))
+	})
+	supervisor.mu.Lock()
+	if supervisor.sealed && supervisor.active == 0 {
+		supervisor.mu.Unlock()
+		stopParentCancel()
+	} else {
+		supervisor.stopParentCancel = stopParentCancel
+		supervisor.mu.Unlock()
+	}
+
+	return supervisor
+}
+
+func (s *Supervisor) Context() context.Context {
+	return s.ctx
+}
+
+func (s *Supervisor) Failures() <-chan Failure {
+	return s.failures
+}
+
+func (s *Supervisor) Go(name string, policy TaskPolicy, task TaskFunc) error {
+	if name == "" {
+		return errors.New("task name is required")
+	}
+	if task == nil {
+		return errors.New("task function is required")
+	}
+
+	s.mu.Lock()
+	if s.sealed {
+		s.mu.Unlock()
+		return fmt.Errorf("%w: %s", ErrSupervisorSealed, name)
+	}
+	if s.ctx.Err() != nil {
+		s.mu.Unlock()
+		return fmt.Errorf("%w: %s", ErrSupervisorStopped, name)
+	}
+	if _, exists := s.tasks[name]; exists {
+		s.mu.Unlock()
+		return fmt.Errorf("%w: %s", ErrTaskExists, name)
+	}
+
+	state := &taskState{TaskSnapshot: TaskSnapshot{
+		Name:      name,
+		Policy:    policy,
+		Status:    TaskRunning,
+		StartedAt: time.Now(),
+	}}
+	s.tasks[name] = state
+	s.active++
+	s.mu.Unlock()
+
+	go s.run(state, task)
+	return nil
+}
+
+func (s *Supervisor) run(state *taskState, task TaskFunc) {
+	var taskErr error
+	panicked := true
+	var panicValue string
+	var panicStack []byte
+
+	defer func() {
+		recovered := recover()
+		if panicked {
+			panicValue = fmt.Sprint(recovered)
+			panicStack = debug.Stack()
+			taskErr = fmt.Errorf("task %q panicked: %s", state.Name, panicValue)
+		}
+		s.finish(state, taskErr, panicked, panicValue, panicStack)
+	}()
+
+	select {
+	case <-s.start:
+	case <-s.ctx.Done():
+		panicked = false
+		return
+	}
+
+	taskErr = task(s.ctx)
+	panicked = false
+}
+
+func (s *Supervisor) finish(
+	state *taskState,
+	taskErr error,
+	panicked bool,
+	panicValue string,
+	panicStack []byte,
+) {
+	var failure *Failure
+
+	s.mu.Lock()
+	state.EndedAt = time.Now()
+	state.Error = ""
+	state.Panic = panicValue
+	state.Stack = append([]byte(nil), panicStack...)
+	switch {
+	case panicked:
+		state.Status = TaskPanicked
+		state.Error = taskErr.Error()
+	case taskErr != nil:
+		state.Status = TaskFailed
+		state.Error = taskErr.Error()
+	default:
+		state.Status = TaskSucceeded
+	}
+
+	shuttingDown := s.phase == phaseDraining || s.ctx.Err() != nil
+	unexpected := panicked || (!shuttingDown &&
+		(state.Policy.Required && (taskErr != nil || state.Policy.LongRunning)))
+	if unexpected {
+		cause := taskErr
+		if cause == nil {
+			cause = fmt.Errorf("required long-running task %q exited", state.Name)
+		}
+		failure = &Failure{
+			Task:  state.Name,
+			Cause: cause,
+			Panic: panicValue,
+			Stack: append([]byte(nil), panicStack...),
+		}
+		s.phase = phaseDraining
+		s.sealed = true
+		s.cancel(failure.Cause)
+		s.failureOnce.Do(func() {
+			s.failures <- *failure
+		})
+	}
+
+	s.active--
+	s.closeDoneIfReadyLocked()
+	s.mu.Unlock()
+}
+
+func (s *Supervisor) Seal() {
+	s.mu.Lock()
+	s.sealed = true
+	if s.phase == phaseStarting {
+		s.phase = phaseRunning
+	}
+	s.startOnce.Do(func() {
+		close(s.start)
+	})
+	s.closeDoneIfReadyLocked()
+	s.mu.Unlock()
+}
+
+func (s *Supervisor) BeginShutdown(cause error) {
+	if cause == nil {
+		cause = context.Canceled
+	}
+
+	s.mu.Lock()
+	s.phase = phaseDraining
+	s.sealed = true
+	s.cancel(cause)
+	s.closeDoneIfReadyLocked()
+	s.mu.Unlock()
+}
+
+func (s *Supervisor) Wait(ctx context.Context) ShutdownReport {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+
+	select {
+	case <-s.allDone:
+	case <-ctx.Done():
+	}
+
+	return s.report()
+}
+
+func (s *Supervisor) Snapshot() []TaskSnapshot {
+	return s.report().Tasks
+}
+
+func (s *Supervisor) closeDoneIfReadyLocked() {
+	if !s.sealed || s.active != 0 {
+		return
+	}
+	if s.stopParentCancel != nil {
+		s.stopParentCancel()
+		s.stopParentCancel = nil
+	}
+	s.doneOnce.Do(func() {
+		close(s.allDone)
+	})
+}
+
+func (s *Supervisor) report() ShutdownReport {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	tasks := make([]TaskSnapshot, 0, len(s.tasks))
+	running := make([]TaskSnapshot, 0)
+	for _, state := range s.tasks {
+		snapshot := state.TaskSnapshot
+		snapshot.Stack = append([]byte(nil), state.Stack...)
+		tasks = append(tasks, snapshot)
+		if snapshot.Status == TaskRunning {
+			running = append(running, snapshot)
+		}
+	}
+	sort.Slice(tasks, func(i, j int) bool {
+		return tasks[i].Name < tasks[j].Name
+	})
+	sort.Slice(running, func(i, j int) bool {
+		return running[i].Name < running[j].Name
+	})
+
+	return ShutdownReport{
+		Complete: s.sealed && s.active == 0,
+		Cause:    context.Cause(s.ctx),
+		Tasks:    tasks,
+		Running:  running,
+	}
+}

machinery/src/lifecycle/supervisor_test.go

@@ -0,0 +1,300 @@
+package lifecycle
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+func TestSupervisorWaitsForSeal(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	started := make(chan struct{})
+	if err := supervisor.Go("quick", TaskPolicy{}, func(context.Context) error {
+		close(started)
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	waitContext, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+	defer cancel()
+	if report := supervisor.Wait(waitContext); report.Complete {
+		t.Fatal("Wait() completed before Seal()")
+	}
+	select {
+	case <-started:
+		t.Fatal("task started before Seal()")
+	default:
+	}
+
+	supervisor.Seal()
+	<-started
+	if report := supervisor.Wait(context.Background()); !report.Complete {
+		t.Fatal("Wait() did not complete after Seal()")
+	}
+}
+
+func TestSupervisorSealsEmptyGroup(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	supervisor.Seal()
+
+	report := supervisor.Wait(context.Background())
+	if !report.Complete || len(report.Tasks) != 0 {
+		t.Fatalf("Wait() report = %+v, want complete empty report", report)
+	}
+}
+
+func TestSupervisorReportsRunningTasksOnTimeout(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	release := make(chan struct{})
+	if err := supervisor.Go("blocked", TaskPolicy{}, func(context.Context) error {
+		<-release
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+
+	waitContext, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+	defer cancel()
+	report := supervisor.Wait(waitContext)
+	if report.Complete {
+		t.Fatal("Wait() unexpectedly completed")
+	}
+	if len(report.Running) != 1 || report.Running[0].Name != "blocked" {
+		t.Fatalf("Wait() running tasks = %+v, want blocked", report.Running)
+	}
+
+	close(release)
+	if report = supervisor.Wait(context.Background()); !report.Complete {
+		t.Fatal("Wait() did not complete after releasing task")
+	}
+}
+
+func TestSupervisorFailsWhenRequiredLongRunningTaskExits(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	release := make(chan struct{})
+	if err := supervisor.Go("recorder", TaskPolicy{Required: true, LongRunning: true}, func(context.Context) error {
+		<-release
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+	close(release)
+
+	select {
+	case failure := <-supervisor.Failures():
+		if failure.Task != "recorder" || !strings.Contains(failure.Error(), "exited") {
+			t.Fatalf("failure = %+v", failure)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("required task exit did not emit a failure")
+	}
+}
+
+func TestSupervisorAllowsRequiredOneShotTaskToSucceed(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	if err := supervisor.Go("setup", TaskPolicy{Required: true}, func(context.Context) error {
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+	if report := supervisor.Wait(context.Background()); !report.Complete {
+		t.Fatal("Wait() did not complete")
+	}
+
+	select {
+	case failure := <-supervisor.Failures():
+		t.Fatalf("successful one-shot task emitted failure: %+v", failure)
+	default:
+	}
+}
+
+func TestSupervisorFailsWhenRequiredOneShotTaskReturnsError(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	wantErr := errors.New("setup failed")
+	if err := supervisor.Go("setup", TaskPolicy{Required: true}, func(context.Context) error {
+		return wantErr
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+
+	select {
+	case failure := <-supervisor.Failures():
+		if !errors.Is(failure.Cause, wantErr) {
+			t.Fatalf("failure cause = %v, want %v", failure.Cause, wantErr)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("required task error did not emit a failure")
+	}
+}
+
+func TestSupervisorRecoversPanicAndCapturesStack(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	if err := supervisor.Go("panic", TaskPolicy{}, func(context.Context) error {
+		panic("boom")
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+
+	select {
+	case failure := <-supervisor.Failures():
+		if failure.Task != "panic" || failure.Panic != "boom" || len(failure.Stack) == 0 {
+			t.Fatalf("failure = %+v, want captured panic and stack", failure)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("panic did not emit a failure")
+	}
+
+	report := supervisor.Wait(context.Background())
+	if !report.Complete || len(report.Tasks) != 1 || len(report.Tasks[0].Stack) == 0 {
+		t.Fatalf("report = %+v, want completed task with captured stack", report)
+	}
+}
+
+func TestSupervisorCapturesEmptyPanicDuringShutdown(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	started := make(chan struct{})
+	if err := supervisor.Go("panic", TaskPolicy{}, func(ctx context.Context) error {
+		close(started)
+		<-ctx.Done()
+		panic("")
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+	<-started
+	supervisor.BeginShutdown(errors.New("requested shutdown"))
+
+	select {
+	case failure := <-supervisor.Failures():
+		if failure.Task != "panic" || len(failure.Stack) == 0 {
+			t.Fatalf("failure = %+v, want captured panic and stack", failure)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("panic during shutdown did not emit a failure")
+	}
+	if report := supervisor.Wait(context.Background()); !report.Complete || report.Tasks[0].Status != TaskPanicked {
+		t.Fatalf("report = %+v, want completed panicked task", report)
+	}
+}
+
+func TestSupervisorPublishesFailureBeforeCompletion(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	wantErr := errors.New("setup failed")
+	if err := supervisor.Go("setup", TaskPolicy{Required: true}, func(context.Context) error {
+		return wantErr
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+
+	report := supervisor.Wait(context.Background())
+	if !report.Complete || !errors.Is(report.Cause, wantErr) {
+		t.Fatalf("report = %+v, want complete report caused by task failure", report)
+	}
+	select {
+	case failure := <-supervisor.Failures():
+		if !errors.Is(failure.Cause, wantErr) {
+			t.Fatalf("failure cause = %v, want %v", failure.Cause, wantErr)
+		}
+	default:
+		t.Fatal("failure was not published before completion")
+	}
+}
+
+func TestSupervisorCompletedWaitWinsOverCanceledContext(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	supervisor.Seal()
+
+	waitContext, cancel := context.WithCancel(context.Background())
+	cancel()
+	if report := supervisor.Wait(waitContext); !report.Complete {
+		t.Fatalf("report = %+v, want authoritative completion state", report)
+	}
+}
+
+func TestSupervisorRejectsTaskAfterParentCancellation(t *testing.T) {
+	parent, cancel := context.WithCancel(context.Background())
+	supervisor := NewSupervisor(parent)
+	cancel()
+
+	err := supervisor.Go("late", TaskPolicy{}, func(context.Context) error { return nil })
+	if !errors.Is(err, ErrSupervisorStopped) && !errors.Is(err, ErrSupervisorSealed) {
+		t.Fatalf("Go() error = %v, want stopped or sealed supervisor", err)
+	}
+	supervisor.BeginShutdown(context.Canceled)
+	if report := supervisor.Wait(context.Background()); !report.Complete {
+		t.Fatalf("report = %+v, want complete canceled supervisor", report)
+	}
+}
+
+func TestSupervisorShutdownBeforeSealDoesNotStartTasks(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	started := make(chan struct{})
+	if err := supervisor.Go("worker", TaskPolicy{}, func(context.Context) error {
+		close(started)
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	supervisor.BeginShutdown(errors.New("startup aborted"))
+	if report := supervisor.Wait(context.Background()); !report.Complete {
+		t.Fatalf("report = %+v, want completed aborted startup", report)
+	}
+	select {
+	case <-started:
+		t.Fatal("task started after startup was aborted")
+	default:
+	}
+}
+
+func TestSupervisorRejectsDuplicateAndLateTasks(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	release := make(chan struct{})
+	if err := supervisor.Go("worker", TaskPolicy{}, func(context.Context) error {
+		<-release
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := supervisor.Go("worker", TaskPolicy{}, func(context.Context) error { return nil }); !errors.Is(err, ErrTaskExists) {
+		t.Fatalf("duplicate Go() error = %v, want ErrTaskExists", err)
+	}
+
+	supervisor.Seal()
+	if err := supervisor.Go("late", TaskPolicy{}, func(context.Context) error { return nil }); !errors.Is(err, ErrSupervisorSealed) {
+		t.Fatalf("late Go() error = %v, want ErrSupervisorSealed", err)
+	}
+	close(release)
+}
+
+func TestSupervisorConcurrentStartAndSeal(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	var starters sync.WaitGroup
+	for index := 0; index < 100; index++ {
+		starters.Add(1)
+		go func(index int) {
+			defer starters.Done()
+			_ = supervisor.Go(
+				"worker-"+time.Unix(0, int64(index)).Format("150405.000000000"),
+				TaskPolicy{},
+				func(context.Context) error { return nil },
+			)
+		}(index)
+	}
+
+	supervisor.Seal()
+	starters.Wait()
+	if report := supervisor.Wait(context.Background()); !report.Complete {
+		t.Fatal("Wait() did not complete after concurrent Start and Seal")
+	}
+}

machinery/src/log/main.go

@@ -1,179 +0,0 @@
-package log
-
-import (
-	"os"
-	"time"
-
-	"github.com/op/go-logging"
-	"github.com/sirupsen/logrus"
-	"gopkg.in/natefinch/lumberjack.v2"
-)
-
-// The logging library being used everywhere.
-var Log = Logging{
-	Logger: "logrus",
-}
-
-// -----------------
-// This a gologging
-// -> github.com/op/go-logging
-
-var gologging = logging.MustGetLogger("gologger")
-
-func ConfigureGoLogging(configDirectory string, timezone *time.Location) {
-	// Logging
-	var format = logging.MustStringFormatter(
-		`%{color}%{time:15:04:05.000} %{shortfunc} ▶ %{level:.4s} %{id:03x}%{color:reset} %{message}`,
-	)
-	var format2 = logging.MustStringFormatter(
-		`%{time:15:04:05.000} %{shortfunc} ▶ %{level:.4s}  %{id:03x}%{color:reset} %{message}`,
-	)
-	stdBackend := logging.NewLogBackend(os.Stderr, "", 0)
-	stdBackendLeveled := logging.NewBackendFormatter(stdBackend, format)
-	fileBackend := logging.NewLogBackend(&lumberjack.Logger{
-		Filename: configDirectory + "/data/log/machinery.txt",
-		MaxSize:  2,    // megabytes
-		Compress: true, // disabled by default
-	}, "", 0)
-	fileBackendLeveled := logging.NewBackendFormatter(fileBackend, format2)
-	logging.SetBackend(stdBackendLeveled, fileBackendLeveled)
-	logging.SetLevel(logging.DEBUG, "")
-}
-
-// -----------------
-// This a logrus
-// -> github.com/sirupsen/logrus
-
-func ConfigureLogrus(level string, output string, timezone *time.Location) {
-
-	if output == "json" {
-		// Log as JSON instead of the default ASCII formatter.
-		logrus.SetFormatter(LocalTimeZoneFormatter{
-			Timezone:  timezone,
-			Formatter: &logrus.JSONFormatter{},
-		})
-	} else if output == "text" {
-		// Log as text with colors.
-		formatter := logrus.TextFormatter{
-			ForceColors:   true,
-			FullTimestamp: true,
-		}
-		logrus.SetFormatter(LocalTimeZoneFormatter{
-			Timezone:  timezone,
-			Formatter: &formatter,
-		})
-	}
-
-	// Use local timezone for providing datetime in logs!
-
-	// Output to stdout instead of the default stderr
-	// Can be any io.Writer, see below for File example
-	logrus.SetOutput(os.Stdout)
-
-	// Only log the warning severity or above.
-	logLevel := logrus.InfoLevel
-	if level == "error" {
-		logLevel = logrus.ErrorLevel
-	} else if level == "debug" {
-		logLevel = logrus.DebugLevel
-	} else if level == "fatal" {
-		logLevel = logrus.FatalLevel
-	} else if level == "warning" {
-		logLevel = logrus.WarnLevel
-	} // Add this line for logging filename and line number!
-	logrus.SetLevel(logLevel)
-}
-
-type LocalTimeZoneFormatter struct {
-	Timezone  *time.Location
-	Formatter logrus.Formatter
-}
-
-func (u LocalTimeZoneFormatter) Format(e *logrus.Entry) ([]byte, error) {
-	e.Time = e.Time.In(u.Timezone)
-	return u.Formatter.Format(e)
-}
-
-type Logging struct {
-	Logger string
-}
-
-func (self *Logging) Init(level string, logoutput string, configDirectory string, timezone *time.Location) {
-	switch self.Logger {
-	case "go-logging":
-		ConfigureGoLogging(configDirectory, timezone)
-	case "logrus":
-		ConfigureLogrus(level, logoutput, timezone)
-	default:
-	}
-}
-
-func (self *Logging) Info(sentence string) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Info(sentence)
-	case "logrus":
-		logrus.Info(sentence)
-	default:
-	}
-}
-
-func (self *Logging) Infof(format string, args ...interface{}) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Infof(format, args...)
-	case "logrus":
-		logrus.Infof(format, args...)
-	default:
-	}
-}
-
-func (self *Logging) Warning(sentence string) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Warning(sentence)
-	case "logrus":
-		logrus.Warn(sentence)
-	default:
-	}
-}
-
-func (self *Logging) Debug(sentence string) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Debug(sentence)
-	case "logrus":
-		logrus.Debug(sentence)
-	default:
-	}
-}
-
-func (self *Logging) Debugf(format string, args ...interface{}) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Debugf(format, args...)
-	case "logrus":
-		logrus.Debugf(format, args...)
-	default:
-	}
-}
-
-func (self *Logging) Error(sentence string) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Error(sentence)
-	case "logrus":
-		logrus.Error(sentence)
-	default:
-	}
-}
-
-func (self *Logging) Fatal(sentence string) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Fatal(sentence)
-	case "logrus":
-		logrus.Fatal(sentence)
-	default:
-	}
-}

machinery/src/models/agent_run.go

@@ -0,0 +1,477 @@
+package models
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/kerberos-io/agent/machinery/src/lifecycle"
+	"github.com/kerberos-io/agent/machinery/src/packets"
+	log "github.com/sirupsen/logrus"
+)
+
+var (
+	ErrAgentRunActive  = errors.New("another agent run is active")
+	ErrAgentRunStarted = errors.New("agent run already started")
+	ErrAgentRunStopped = errors.New("agent run is stopping")
+	nextAgentRunID     atomic.Uint64
+)
+
+type AgentRunClient interface {
+	Close(context.Context) error
+}
+
+type AgentRunResourceError struct {
+	Resource string
+	Err      error
+}
+
+func (e AgentRunResourceError) Error() string {
+	return fmt.Sprintf("close %s: %v", e.Resource, e.Err)
+}
+
+func (e AgentRunResourceError) Unwrap() error {
+	return e.Err
+}
+
+type AgentRunShutdownReport struct {
+	lifecycle.ShutdownReport
+	ResourceErrors      []AgentRunResourceError
+	UploadStopDelivered bool
+	StreamStopDelivered bool
+}
+
+type AgentRun struct {
+	id            uint64
+	ctx           context.Context
+	cancel        context.CancelCauseFunc
+	supervisor    *lifecycle.Supervisor
+	communication *Communication
+	stopUpload    bool
+
+	stateMu   sync.Mutex
+	activated bool
+	stopping  bool
+
+	resourcesMu       sync.RWMutex
+	mainClient        AgentRunClient
+	subClient         AgentRunClient
+	backchannelClient AgentRunClient
+	mainQueue         *packets.Queue
+	subQueue          *packets.Queue
+	releaseClients    func()
+
+	channelsMu       sync.RWMutex
+	channelsClosed   bool
+	liveHDHandshakes chan LiveHDHandshake
+	motionEvents     chan MotionDataPartial
+	onvifActions     chan OnvifAction
+
+	shutdownOnce   sync.Once
+	shutdownReport AgentRunShutdownReport
+}
+
+func NewAgentRun(parent context.Context, communication *Communication, stopUpload bool) *AgentRun {
+	if parent == nil {
+		parent = context.Background()
+	}
+	ctx, cancel := context.WithCancelCause(parent)
+	run := &AgentRun{
+		id:               nextAgentRunID.Add(1),
+		ctx:              ctx,
+		cancel:           cancel,
+		supervisor:       lifecycle.NewSupervisor(ctx),
+		communication:    communication,
+		stopUpload:       stopUpload,
+		liveHDHandshakes: make(chan LiveHDHandshake, 100),
+		motionEvents:     make(chan MotionDataPartial, 10),
+		onvifActions:     make(chan OnvifAction, 10),
+	}
+	log.WithFields(log.Fields{
+		"component":   "agent_run",
+		"event":       "run_created",
+		"run_id":      run.id,
+		"stop_upload": stopUpload,
+	}).Debug("Agent run created")
+	return run
+}
+
+func (r *AgentRun) ID() uint64 {
+	return r.id
+}
+
+func (r *AgentRun) Context() context.Context {
+	return r.ctx
+}
+
+func (r *AgentRun) Go(name string, policy lifecycle.TaskPolicy, task lifecycle.TaskFunc) error {
+	if task == nil {
+		return r.supervisor.Go(name, policy, task)
+	}
+
+	fields := log.Fields{
+		"component":    "agent_run",
+		"event":        "task_lifecycle",
+		"long_running": policy.LongRunning,
+		"required":     policy.Required,
+		"run_id":       r.id,
+		"task":         name,
+	}
+	wrappedTask := func(ctx context.Context) error {
+		startedAt := time.Now()
+		log.WithFields(fields).Debug("Agent task started")
+		err := task(ctx)
+		entry := log.WithFields(fields).
+			WithField("duration_ms", time.Since(startedAt).Milliseconds())
+		if err != nil {
+			entry = entry.WithError(err)
+			if ctx.Err() == nil {
+				entry.Warn("Agent task stopped with an error")
+			} else {
+				entry.Debug("Agent task stopped during shutdown")
+			}
+		} else {
+			entry.Debug("Agent task stopped")
+		}
+		return err
+	}
+
+	if err := r.supervisor.Go(name, policy, wrappedTask); err != nil {
+		log.WithError(err).WithFields(fields).Error("Failed to register Agent task")
+		return err
+	}
+	log.WithFields(fields).Debug("Agent task registered")
+	return nil
+}
+
+func (r *AgentRun) Seal() {
+	r.supervisor.Seal()
+	log.WithFields(log.Fields{
+		"component":  "agent_run",
+		"event":      "run_sealed",
+		"run_id":     r.id,
+		"task_count": len(r.supervisor.Snapshot()),
+	}).Debug("Agent run sealed")
+}
+
+func (r *AgentRun) Failures() <-chan lifecycle.Failure {
+	return r.supervisor.Failures()
+}
+
+func (r *AgentRun) Snapshot() []lifecycle.TaskSnapshot {
+	return r.supervisor.Snapshot()
+}
+
+func (r *AgentRun) Wait(ctx context.Context) lifecycle.ShutdownReport {
+	return r.supervisor.Wait(ctx)
+}
+
+func (r *AgentRun) Activate() error {
+	r.stateMu.Lock()
+	defer r.stateMu.Unlock()
+	if r.stopping {
+		return ErrAgentRunStopped
+	}
+	if r.ctx.Err() != nil {
+		return ErrAgentRunStopped
+	}
+	if r.activated {
+		return ErrAgentRunStarted
+	}
+	if r.communication == nil {
+		return errors.New("agent run communication is required")
+	}
+	if err := r.communication.attachRun(r); err != nil {
+		return err
+	}
+	r.activated = true
+	log.WithFields(log.Fields{
+		"component": "agent_run",
+		"event":     "run_activated",
+		"run_id":    r.id,
+	}).Info("Agent run activated")
+	return nil
+}
+
+func (r *AgentRun) SetMainClient(client AgentRunClient) {
+	r.resourcesMu.Lock()
+	r.mainClient = client
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) SetSubClient(client AgentRunClient) {
+	r.resourcesMu.Lock()
+	r.subClient = client
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) SetBackchannelClient(client AgentRunClient) {
+	r.resourcesMu.Lock()
+	r.backchannelClient = client
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) SetQueues(mainQueue, subQueue *packets.Queue) {
+	r.resourcesMu.Lock()
+	r.mainQueue = mainQueue
+	r.subQueue = subQueue
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) SetMainQueue(queue *packets.Queue) {
+	r.resourcesMu.Lock()
+	r.mainQueue = queue
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) SetSubQueue(queue *packets.Queue) {
+	r.resourcesMu.Lock()
+	r.subQueue = queue
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) SetClientRelease(release func()) {
+	r.resourcesMu.Lock()
+	r.releaseClients = release
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) MainQueue() *packets.Queue {
+	if r.isStopping() {
+		return nil
+	}
+	r.resourcesMu.RLock()
+	defer r.resourcesMu.RUnlock()
+	return r.mainQueue
+}
+
+func (r *AgentRun) SubQueue() *packets.Queue {
+	if r.isStopping() {
+		return nil
+	}
+	r.resourcesMu.RLock()
+	defer r.resourcesMu.RUnlock()
+	return r.subQueue
+}
+
+func (r *AgentRun) LiveHDHandshakes() <-chan LiveHDHandshake {
+	return r.liveHDHandshakes
+}
+
+func (r *AgentRun) MotionEvents() <-chan MotionDataPartial {
+	return r.motionEvents
+}
+
+func (r *AgentRun) ONVIFActions() <-chan OnvifAction {
+	return r.onvifActions
+}
+
+func (r *AgentRun) TrySendLiveHDHandshake(handshake LiveHDHandshake) bool {
+	if r.isStopping() {
+		return false
+	}
+	r.channelsMu.RLock()
+	defer r.channelsMu.RUnlock()
+	if r.channelsClosed {
+		return false
+	}
+	select {
+	case r.liveHDHandshakes <- handshake:
+		return true
+	default:
+		return false
+	}
+}
+
+func (r *AgentRun) PendingLiveHDHandshakes() int {
+	if r.isStopping() {
+		return 0
+	}
+	r.channelsMu.RLock()
+	defer r.channelsMu.RUnlock()
+	if r.channelsClosed {
+		return 0
+	}
+	return len(r.liveHDHandshakes)
+}
+
+func (r *AgentRun) TrySendMotion(motion MotionDataPartial) bool {
+	if r.isStopping() {
+		return false
+	}
+	r.channelsMu.RLock()
+	defer r.channelsMu.RUnlock()
+	if r.channelsClosed {
+		return false
+	}
+	select {
+	case r.motionEvents <- motion:
+		return true
+	default:
+		return false
+	}
+}
+
+func (r *AgentRun) TrySendONVIF(action OnvifAction) bool {
+	if r.isStopping() {
+		return false
+	}
+	r.channelsMu.RLock()
+	defer r.channelsMu.RUnlock()
+	if r.channelsClosed {
+		return false
+	}
+	select {
+	case r.onvifActions <- action:
+		return true
+	default:
+		return false
+	}
+}
+
+func (r *AgentRun) Shutdown(ctx context.Context, cause error) AgentRunShutdownReport {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	if cause == nil {
+		cause = context.Canceled
+	}
+
+	r.shutdownOnce.Do(func() {
+		startedAt := time.Now()
+		log.WithError(cause).WithFields(log.Fields{
+			"component": "agent_run",
+			"event":     "shutdown_started",
+			"run_id":    r.id,
+		}).Info("Agent run shutdown started")
+
+		r.stateMu.Lock()
+		r.stopping = true
+		activated := r.activated
+		r.stateMu.Unlock()
+
+		r.cancel(cause)
+		r.supervisor.BeginShutdown(cause)
+
+		if activated && r.stopUpload {
+			r.shutdownReport.UploadStopDelivered = sendRunStop(ctx, r.communication.HandleUpload)
+		}
+		if activated {
+			r.shutdownReport.StreamStopDelivered = sendRunStop(ctx, r.communication.HandleStream)
+		}
+
+		r.resourcesMu.RLock()
+		mainClient := r.mainClient
+		subClient := r.subClient
+		backchannelClient := r.backchannelClient
+		mainQueue := r.mainQueue
+		subQueue := r.subQueue
+		releaseClients := r.releaseClients
+		r.resourcesMu.RUnlock()
+
+		r.closeClient(ctx, "main RTSP client", mainClient)
+		if mainQueue != nil {
+			_ = mainQueue.Close()
+		}
+		r.closeClient(ctx, "sub RTSP client", subClient)
+		if subQueue != nil {
+			_ = subQueue.Close()
+		}
+		r.closeClient(ctx, "RTSP backchannel client", backchannelClient)
+		r.closeChannels()
+		if releaseClients != nil {
+			releaseClients()
+		}
+
+		r.shutdownReport.ShutdownReport = r.supervisor.Wait(ctx)
+		if r.shutdownReport.Complete && r.communication != nil {
+			r.communication.detachRun(r)
+		}
+
+		runningTasks := make([]string, 0, len(r.shutdownReport.Running))
+		for _, task := range r.shutdownReport.Running {
+			runningTasks = append(runningTasks, task.Name)
+		}
+		entry := log.WithFields(log.Fields{
+			"component":            "agent_run",
+			"duration_ms":          time.Since(startedAt).Milliseconds(),
+			"resource_error_count": len(r.shutdownReport.ResourceErrors),
+			"run_id":               r.id,
+			"running_tasks":        runningTasks,
+			"task_count":           len(r.shutdownReport.Tasks),
+			"upload_stop_sent":     r.shutdownReport.UploadStopDelivered,
+			"stream_stop_sent":     r.shutdownReport.StreamStopDelivered,
+		})
+		if r.shutdownReport.Complete {
+			entry.WithField("event", "shutdown_completed").Info("Agent run shutdown completed")
+		} else {
+			entry.WithField("event", "shutdown_timed_out").Error("Agent run shutdown timed out")
+		}
+	})
+
+	report := r.shutdownReport
+	report.ResourceErrors = append([]AgentRunResourceError(nil), report.ResourceErrors...)
+	return report
+}
+
+func (r *AgentRun) isStopping() bool {
+	r.stateMu.Lock()
+	defer r.stateMu.Unlock()
+	return r.stopping
+}
+
+func (r *AgentRun) closeClient(ctx context.Context, name string, client AgentRunClient) {
+	if client == nil {
+		return
+	}
+	if err := client.Close(ctx); err != nil {
+		r.shutdownReport.ResourceErrors = append(r.shutdownReport.ResourceErrors, AgentRunResourceError{
+			Resource: name,
+			Err:      err,
+		})
+		log.WithError(err).WithFields(log.Fields{
+			"component": "agent_run",
+			"event":     "resource_close_failed",
+			"resource":  name,
+			"run_id":    r.id,
+		}).Warn("Failed to close Agent run resource")
+		return
+	}
+	log.WithFields(log.Fields{
+		"component": "agent_run",
+		"event":     "resource_closed",
+		"resource":  name,
+		"run_id":    r.id,
+	}).Debug("Agent run resource closed")
+}
+
+func (r *AgentRun) closeChannels() {
+	r.channelsMu.Lock()
+	defer r.channelsMu.Unlock()
+	if r.channelsClosed {
+		return
+	}
+	r.channelsClosed = true
+	close(r.liveHDHandshakes)
+	close(r.motionEvents)
+	close(r.onvifActions)
+}
+
+func sendRunStop(ctx context.Context, channel chan<- string) bool {
+	if channel == nil {
+		return false
+	}
+	timer := time.NewTimer(time.Second)
+	defer timer.Stop()
+	select {
+	case channel <- "stop":
+		return true
+	case <-ctx.Done():
+		return false
+	case <-timer.C:
+		return false
+	}
+}

machinery/src/models/agent_run_test.go

@@ -0,0 +1,303 @@
+package models
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/kerberos-io/agent/machinery/src/lifecycle"
+	"github.com/kerberos-io/agent/machinery/src/packets"
+)
+
+type fakeAgentRunClient struct {
+	mu       sync.Mutex
+	name     string
+	order    *[]string
+	calls    int
+	closeErr error
+}
+
+func (c *fakeAgentRunClient) Close(context.Context) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.calls++
+	*c.order = append(*c.order, c.name)
+	return c.closeErr
+}
+
+func (c *fakeAgentRunClient) Calls() int {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.calls
+}
+
+func TestAgentRunOwnsAndShutsDownResources(t *testing.T) {
+	communication := &Communication{
+		HandleStream: make(chan string, 1),
+		HandleUpload: make(chan string, 1),
+	}
+	run := NewAgentRun(context.Background(), communication, true)
+	mainQueue := packets.NewQueue()
+	subQueue := packets.NewQueue()
+	run.SetQueues(mainQueue, subQueue)
+
+	var order []string
+	mainClient := &fakeAgentRunClient{name: "main", order: &order}
+	subClient := &fakeAgentRunClient{name: "sub", order: &order}
+	backchannelClient := &fakeAgentRunClient{name: "backchannel", order: &order}
+	run.SetMainClient(mainClient)
+	run.SetSubClient(subClient)
+	run.SetBackchannelClient(backchannelClient)
+	run.SetClientRelease(func() {
+		order = append(order, "release")
+	})
+
+	taskStarted := make(chan struct{})
+	if err := run.Go("worker", lifecycle.TaskPolicy{}, func(ctx context.Context) error {
+		close(taskStarted)
+		<-ctx.Done()
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := run.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	run.Seal()
+	<-taskStarted
+
+	report := run.Shutdown(context.Background(), errors.New("test shutdown"))
+	if !report.Complete {
+		t.Fatalf("shutdown report = %+v, want complete", report)
+	}
+	if communication.CurrentRun() != nil {
+		t.Fatal("shutdown left the run attached")
+	}
+	if got, want := order, []string{"main", "sub", "backchannel", "release"}; !equalStrings(got, want) {
+		t.Fatalf("resource order = %v, want %v", got, want)
+	}
+	if !report.UploadStopDelivered || !report.StreamStopDelivered {
+		t.Fatalf("stop delivery = upload:%t stream:%t, want both", report.UploadStopDelivered, report.StreamStopDelivered)
+	}
+	select {
+	case value := <-communication.HandleUpload:
+		if value != "stop" {
+			t.Fatalf("upload stop = %q, want stop", value)
+		}
+	default:
+		t.Fatal("upload stop was not delivered")
+	}
+	select {
+	case value := <-communication.HandleStream:
+		if value != "stop" {
+			t.Fatalf("stream stop = %q, want stop", value)
+		}
+	default:
+		t.Fatal("stream stop was not delivered")
+	}
+	if _, err := mainQueue.Latest().ReadPacket(); err == nil {
+		t.Fatal("main queue remained open")
+	}
+	if _, err := subQueue.Latest().ReadPacket(); err == nil {
+		t.Fatal("sub queue remained open")
+	}
+	if _, ok := <-run.LiveHDHandshakes(); ok {
+		t.Fatal("handshake channel remained open")
+	}
+	if _, ok := <-run.MotionEvents(); ok {
+		t.Fatal("motion channel remained open")
+	}
+	if _, ok := <-run.ONVIFActions(); ok {
+		t.Fatal("ONVIF channel remained open")
+	}
+}
+
+func TestAgentRunShutdownIsConcurrentAndIdempotent(t *testing.T) {
+	communication := &Communication{}
+	run := NewAgentRun(context.Background(), communication, false)
+	var order []string
+	client := &fakeAgentRunClient{name: "main", order: &order}
+	run.SetMainClient(client)
+	if err := run.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	run.Seal()
+
+	var callers sync.WaitGroup
+	reports := make(chan AgentRunShutdownReport, 20)
+	for index := 0; index < 20; index++ {
+		callers.Add(1)
+		go func() {
+			defer callers.Done()
+			reports <- run.Shutdown(context.Background(), errors.New("test shutdown"))
+		}()
+	}
+	callers.Wait()
+	close(reports)
+
+	for report := range reports {
+		if !report.Complete {
+			t.Fatalf("shutdown report = %+v, want complete", report)
+		}
+	}
+	if got := client.Calls(); got != 1 {
+		t.Fatalf("client Close() calls = %d, want 1", got)
+	}
+}
+
+func TestAgentRunStaleShutdownDoesNotDetachNewRun(t *testing.T) {
+	communication := &Communication{}
+	oldRun := NewAgentRun(context.Background(), communication, false)
+	if err := oldRun.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	if !communication.detachRun(oldRun) {
+		t.Fatal("failed to detach old run during test setup")
+	}
+
+	newRun := NewAgentRun(context.Background(), communication, false)
+	if err := newRun.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	newRun.Seal()
+	t.Cleanup(func() {
+		newRun.Shutdown(context.Background(), errors.New("test complete"))
+	})
+
+	oldRun.Shutdown(context.Background(), errors.New("stale shutdown"))
+	if got := communication.CurrentRun(); got != newRun {
+		t.Fatalf("current run = %p, want new run %p", got, newRun)
+	}
+}
+
+func TestAgentRunRejectsOverlap(t *testing.T) {
+	communication := &Communication{}
+	first := NewAgentRun(context.Background(), communication, false)
+	if err := first.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	first.Seal()
+	t.Cleanup(func() {
+		first.Shutdown(context.Background(), errors.New("test complete"))
+	})
+
+	second := NewAgentRun(context.Background(), communication, false)
+	if err := second.Activate(); !errors.Is(err, ErrAgentRunActive) {
+		t.Fatalf("Activate() error = %v, want ErrAgentRunActive", err)
+	}
+	second.Shutdown(context.Background(), errors.New("test complete"))
+}
+
+func TestAgentRunRetainsOwnershipUntilWorkersStop(t *testing.T) {
+	communication := &Communication{}
+	run := NewAgentRun(context.Background(), communication, false)
+	release := make(chan struct{})
+	started := make(chan struct{})
+	if err := run.Go("blocked", lifecycle.TaskPolicy{}, func(context.Context) error {
+		close(started)
+		<-release
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := run.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	run.Seal()
+	<-started
+
+	shutdownDone := make(chan AgentRunShutdownReport, 1)
+	go func() {
+		shutdownDone <- run.Shutdown(context.Background(), errors.New("test shutdown"))
+	}()
+
+	deadline := time.Now().Add(time.Second)
+	for run.Context().Err() == nil && time.Now().Before(deadline) {
+		time.Sleep(time.Millisecond)
+	}
+	if run.Context().Err() == nil {
+		t.Fatal("run context was not canceled")
+	}
+	if communication.CurrentRun() != run {
+		t.Fatal("run ownership was released before its worker stopped")
+	}
+	if communication.TrySendMotion(MotionDataPartial{}) {
+		t.Fatal("stopping run accepted new input")
+	}
+	if communication.MainQueue() != nil {
+		t.Fatal("stopping run exposed its queue")
+	}
+
+	replacement := NewAgentRun(context.Background(), communication, false)
+	if err := replacement.Activate(); !errors.Is(err, ErrAgentRunActive) {
+		t.Fatalf("replacement Activate() error = %v, want ErrAgentRunActive", err)
+	}
+	replacement.Shutdown(context.Background(), errors.New("test complete"))
+
+	close(release)
+	report := <-shutdownDone
+	if !report.Complete {
+		t.Fatalf("shutdown report = %+v, want complete", report)
+	}
+	if communication.CurrentRun() != nil {
+		t.Fatal("completed shutdown left the run attached")
+	}
+}
+
+func TestAgentRunShutdownReportsStuckTask(t *testing.T) {
+	communication := &Communication{}
+	run := NewAgentRun(context.Background(), communication, false)
+	release := make(chan struct{})
+	started := make(chan struct{})
+	if err := run.Go("blocked", lifecycle.TaskPolicy{}, func(context.Context) error {
+		close(started)
+		<-release
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := run.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	run.Seal()
+	<-started
+
+	waitContext, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+	defer cancel()
+	report := run.Shutdown(waitContext, errors.New("test shutdown"))
+	if report.Complete {
+		t.Fatal("shutdown unexpectedly completed")
+	}
+	if len(report.Running) != 1 || report.Running[0].Name != "blocked" {
+		t.Fatalf("running tasks = %+v, want blocked", report.Running)
+	}
+	if communication.CurrentRun() != run {
+		t.Fatal("timed-out shutdown released run ownership")
+	}
+	replacement := NewAgentRun(context.Background(), communication, false)
+	if err := replacement.Activate(); !errors.Is(err, ErrAgentRunActive) {
+		t.Fatalf("replacement Activate() error = %v, want ErrAgentRunActive", err)
+	}
+	replacement.Shutdown(context.Background(), errors.New("test complete"))
+	close(release)
+	waitContext, waitCancel := context.WithTimeout(context.Background(), time.Second)
+	defer waitCancel()
+	if followup := run.Wait(waitContext); !followup.Complete {
+		t.Fatalf("follow-up report = %+v, want complete", followup)
+	}
+	communication.detachRun(run)
+}
+
+func equalStrings(left, right []string) bool {
+	if len(left) != len(right) {
+		return false
+	}
+	for index := range left {
+		if left[index] != right[index] {
+			return false
+		}
+	}
+	return true
+}

machinery/src/models/communication.go

@@ -1,8 +1,10 @@
 package models
 
 import (
-	"context"
+	"math"
+	"sync"
 	"sync/atomic"
+	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/tevino/abool"
@@ -19,11 +21,84 @@ type LiveHDHandshake struct {
 	Signaling *LiveHDSignalingCallbacks
 }
 
+type MoQRecoveryTelemetry struct {
+	Reconnects          uint64 `json:"reconnects"`
+	LastFrameUnixMillis int64  `json:"lastFrameUnixMillis"`
+	LastWriteMillis     int64  `json:"lastWriteMillis"`
+}
+
+type RecoveryTelemetry struct {
+	MoQHigh                   MoQRecoveryTelemetry `json:"moqHigh"`
+	MoQLow                    MoQRecoveryTelemetry `json:"moqLow"`
+	MoQWriteTimeouts          uint64               `json:"moqWriteTimeouts"`
+	DroppedLiveHDHandshakes   uint64               `json:"droppedLiveHDHandshakes"`
+	DroppedMotionEvents       uint64               `json:"droppedMotionEvents"`
+	DroppedONVIFActions       uint64               `json:"droppedOnvifActions"`
+	WatchdogRestarts          uint64               `json:"watchdogRestarts"`
+	WatchdogCooldownSeconds   int64                `json:"watchdogCooldownSeconds"`
+	RunWorkerShutdownTimeouts uint64               `json:"runWorkerShutdownTimeouts"`
+}
+
+type StreamKind uint8
+
+const (
+	MainStream StreamKind = iota
+	SubStream
+)
+
+type StreamRuntimeTelemetry struct {
+	Configured        bool
+	PackagesProcessed uint64
+	FPS               float64
+	Width             int64
+	Height            int64
+	LastPacketAt      int64
+}
+
+type streamRuntimeTelemetry struct {
+	configured        atomic.Bool
+	packagesProcessed atomic.Uint64
+	fpsBits           atomic.Uint64
+	width             atomic.Int64
+	height            atomic.Int64
+	lastPacketAt      atomic.Int64
+}
+
+type HubRuntimeTelemetry struct {
+	Configured                bool
+	Connected                 bool
+	LastHeartbeatAttemptAt    int64
+	LastSuccessfulHeartbeatAt int64
+}
+
+type hubRuntimeTelemetry struct {
+	configured                atomic.Bool
+	connected                 atomic.Bool
+	lastHeartbeatAttemptAt    atomic.Int64
+	lastSuccessfulHeartbeatAt atomic.Int64
+}
+
+type recoveryTelemetry struct {
+	moqHighReconnects          atomic.Uint64
+	moqHighLastFrameUnixMillis atomic.Int64
+	moqHighLastWriteMillis     atomic.Int64
+	moqLowReconnects           atomic.Uint64
+	moqLowLastFrameUnixMillis  atomic.Int64
+	moqLowLastWriteMillis      atomic.Int64
+	moqWriteTimeouts           atomic.Uint64
+	droppedLiveHDHandshakes    atomic.Uint64
+	droppedMotionEvents        atomic.Uint64
+	droppedONVIFActions        atomic.Uint64
+	watchdogRestarts           atomic.Uint64
+	watchdogCooldownSeconds    atomic.Int64
+	runWorkerShutdownTimeouts  atomic.Uint64
+}
+
 // The communication struct that is managing
 // all the communication between the different goroutines.
 type Communication struct {
-	Context               *context.Context
-	CancelContext         *context.CancelFunc
+	currentRunMu          sync.RWMutex
+	currentRun            *AgentRun
 	PackageCounter        *atomic.Value
 	LastPacketTimer       *atomic.Value
 	PackageCounterSub     *atomic.Value
@@ -32,20 +107,17 @@ type Communication struct {
 	HandleBootstrap       chan string
 	HandleStream          chan string
 	HandleSubStream       chan string
-	HandleMotion          chan MotionDataPartial
 	HandleAudio           chan AudioDataPartial
 	HandleUpload          chan string
 	HandleHeartBeat       chan string
 	HandleLiveSD          chan int64
 	HandleLiveSDHTTP      chan int64
 	HandleLiveHDKeepalive chan string
-	HandleLiveHDHandshake chan LiveHDHandshake
 	HandleLiveHDPeers     chan string
 	// HandleLiveHLS is the live HLS viewer keepalive. It carries the requested
 	// quality tier ("auto"|"high"|"low"; empty => auto) so the producer can switch
 	// the live session between the main and sub stream on demand.
 	HandleLiveHLS chan string
-	HandleONVIF   chan OnvifAction
 	IsConfiguring *abool.AtomicBool
 	// IsRecordingManual is set while a viewer has requested a manual recording
 	// from the live view (the record button). While set, the motion-based
@@ -71,11 +143,224 @@ type Communication struct {
 	// viewer that starts a recording but never heartbeats (an older frontend)
 	// still records up to the max-duration cap instead of being cut off early.
 	RecordingManualHeartbeatSeen *abool.AtomicBool
-	Queue                        *packets.Queue
-	SubQueue                     *packets.Queue
 	Image                        string
-	CameraConnected              bool
-	MainStreamConnected          bool
-	SubStreamConnected           bool
-	HasBackChannel               bool
+	CameraConnected              atomic.Bool
+	MainStreamConnected          atomic.Bool
+	SubStreamConnected           atomic.Bool
+	HasBackChannel               atomic.Bool
+	mainStreamTelemetry          streamRuntimeTelemetry
+	subStreamTelemetry           streamRuntimeTelemetry
+	hubTelemetry                 hubRuntimeTelemetry
+	recovery                     recoveryTelemetry
+}
+
+func (c *Communication) streamTelemetry(stream StreamKind) *streamRuntimeTelemetry {
+	switch stream {
+	case MainStream:
+		return &c.mainStreamTelemetry
+	case SubStream:
+		return &c.subStreamTelemetry
+	default:
+		panic("unsupported stream kind")
+	}
+}
+
+func (c *Communication) SetStreamConfigured(stream StreamKind, configured bool) {
+	c.streamTelemetry(stream).configured.Store(configured)
+}
+
+func (c *Communication) RecordStreamPackage(stream StreamKind, fps float64, width, height int, at time.Time) {
+	telemetry := c.streamTelemetry(stream)
+	telemetry.packagesProcessed.Add(1)
+	if fps > 0 && !math.IsNaN(fps) && !math.IsInf(fps, 0) {
+		telemetry.fpsBits.Store(math.Float64bits(fps))
+	}
+	if width > 0 {
+		telemetry.width.Store(int64(width))
+	}
+	if height > 0 {
+		telemetry.height.Store(int64(height))
+	}
+	if !at.IsZero() {
+		telemetry.lastPacketAt.Store(at.Unix())
+	}
+}
+
+func (c *Communication) StreamRuntimeTelemetry(stream StreamKind) StreamRuntimeTelemetry {
+	telemetry := c.streamTelemetry(stream)
+	return StreamRuntimeTelemetry{
+		Configured:        telemetry.configured.Load(),
+		PackagesProcessed: telemetry.packagesProcessed.Load(),
+		FPS:               math.Float64frombits(telemetry.fpsBits.Load()),
+		Width:             telemetry.width.Load(),
+		Height:            telemetry.height.Load(),
+		LastPacketAt:      telemetry.lastPacketAt.Load(),
+	}
+}
+
+func (c *Communication) SetHubConfigured(configured bool) {
+	c.hubTelemetry.configured.Store(configured)
+	if !configured {
+		c.hubTelemetry.connected.Store(false)
+	}
+}
+
+func (c *Communication) RecordHubHeartbeatAttempt(at time.Time) {
+	if !at.IsZero() {
+		c.hubTelemetry.lastHeartbeatAttemptAt.Store(at.Unix())
+	}
+}
+
+func (c *Communication) RecordHubHeartbeatSuccess(at time.Time) {
+	if !at.IsZero() {
+		c.hubTelemetry.lastSuccessfulHeartbeatAt.Store(at.Unix())
+	}
+	c.hubTelemetry.connected.Store(true)
+}
+
+func (c *Communication) RecordHubHeartbeatFailure() {
+	c.hubTelemetry.connected.Store(false)
+}
+
+func (c *Communication) HubRuntimeTelemetry() HubRuntimeTelemetry {
+	return HubRuntimeTelemetry{
+		Configured:                c.hubTelemetry.configured.Load(),
+		Connected:                 c.hubTelemetry.connected.Load(),
+		LastHeartbeatAttemptAt:    c.hubTelemetry.lastHeartbeatAttemptAt.Load(),
+		LastSuccessfulHeartbeatAt: c.hubTelemetry.lastSuccessfulHeartbeatAt.Load(),
+	}
+}
+
+func (c *Communication) RecordMoQReconnect(quality string) {
+	if quality == StreamQualityLow {
+		c.recovery.moqLowReconnects.Add(1)
+		return
+	}
+	c.recovery.moqHighReconnects.Add(1)
+}
+
+func (c *Communication) RecordMoQWrite(quality string, duration time.Duration, at time.Time) {
+	if quality == StreamQualityLow {
+		c.recovery.moqLowLastWriteMillis.Store(duration.Milliseconds())
+		c.recovery.moqLowLastFrameUnixMillis.Store(at.UnixMilli())
+		return
+	}
+	c.recovery.moqHighLastWriteMillis.Store(duration.Milliseconds())
+	c.recovery.moqHighLastFrameUnixMillis.Store(at.UnixMilli())
+}
+
+func (c *Communication) RecordMoQWriteTimeout() {
+	c.recovery.moqWriteTimeouts.Add(1)
+}
+
+func (c *Communication) RecordWatchdogRestart(cooldown time.Duration) {
+	c.recovery.watchdogRestarts.Add(1)
+	c.SetWatchdogCooldown(cooldown)
+}
+
+func (c *Communication) SetWatchdogCooldown(cooldown time.Duration) {
+	c.recovery.watchdogCooldownSeconds.Store(int64(cooldown / time.Second))
+}
+
+func (c *Communication) RecordRunWorkerShutdownTimeout() {
+	c.recovery.runWorkerShutdownTimeouts.Add(1)
+}
+
+func (c *Communication) RecoveryTelemetry() RecoveryTelemetry {
+	return RecoveryTelemetry{
+		MoQHigh: MoQRecoveryTelemetry{
+			Reconnects:          c.recovery.moqHighReconnects.Load(),
+			LastFrameUnixMillis: c.recovery.moqHighLastFrameUnixMillis.Load(),
+			LastWriteMillis:     c.recovery.moqHighLastWriteMillis.Load(),
+		},
+		MoQLow: MoQRecoveryTelemetry{
+			Reconnects:          c.recovery.moqLowReconnects.Load(),
+			LastFrameUnixMillis: c.recovery.moqLowLastFrameUnixMillis.Load(),
+			LastWriteMillis:     c.recovery.moqLowLastWriteMillis.Load(),
+		},
+		MoQWriteTimeouts:          c.recovery.moqWriteTimeouts.Load(),
+		DroppedLiveHDHandshakes:   c.recovery.droppedLiveHDHandshakes.Load(),
+		DroppedMotionEvents:       c.recovery.droppedMotionEvents.Load(),
+		DroppedONVIFActions:       c.recovery.droppedONVIFActions.Load(),
+		WatchdogRestarts:          c.recovery.watchdogRestarts.Load(),
+		WatchdogCooldownSeconds:   c.recovery.watchdogCooldownSeconds.Load(),
+		RunWorkerShutdownTimeouts: c.recovery.runWorkerShutdownTimeouts.Load(),
+	}
+}
+
+func (c *Communication) attachRun(run *AgentRun) error {
+	c.currentRunMu.Lock()
+	defer c.currentRunMu.Unlock()
+	if c.currentRun != nil {
+		return ErrAgentRunActive
+	}
+	c.currentRun = run
+	return nil
+}
+
+func (c *Communication) detachRun(run *AgentRun) bool {
+	c.currentRunMu.Lock()
+	defer c.currentRunMu.Unlock()
+	if c.currentRun != run {
+		return false
+	}
+	c.currentRun = nil
+	return true
+}
+
+func (c *Communication) CurrentRun() *AgentRun {
+	c.currentRunMu.RLock()
+	defer c.currentRunMu.RUnlock()
+	return c.currentRun
+}
+
+func (c *Communication) MainQueue() *packets.Queue {
+	run := c.CurrentRun()
+	if run == nil {
+		return nil
+	}
+	return run.MainQueue()
+}
+
+func (c *Communication) SubQueue() *packets.Queue {
+	run := c.CurrentRun()
+	if run == nil {
+		return nil
+	}
+	return run.SubQueue()
+}
+
+func (c *Communication) TrySendLiveHDHandshake(handshake LiveHDHandshake) bool {
+	run := c.CurrentRun()
+	if run == nil || !run.TrySendLiveHDHandshake(handshake) {
+		c.recovery.droppedLiveHDHandshakes.Add(1)
+		return false
+	}
+	return true
+}
+
+func (c *Communication) PendingLiveHDHandshakes() int {
+	run := c.CurrentRun()
+	if run == nil {
+		return 0
+	}
+	return run.PendingLiveHDHandshakes()
+}
+
+func (c *Communication) TrySendMotion(motion MotionDataPartial) bool {
+	run := c.CurrentRun()
+	if run == nil || !run.TrySendMotion(motion) {
+		c.recovery.droppedMotionEvents.Add(1)
+		return false
+	}
+	return true
+}
+
+func (c *Communication) TrySendONVIF(action OnvifAction) bool {
+	run := c.CurrentRun()
+	if run == nil || !run.TrySendONVIF(action) {
+		c.recovery.droppedONVIFActions.Add(1)
+		return false
+	}
+	return true
 }

machinery/src/models/communication_test.go

@@ -0,0 +1,221 @@
+package models
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+	"time"
+)
+
+func TestCommunicationRunChannelDispatchAfterClose(t *testing.T) {
+	communication := &Communication{}
+	run := NewAgentRun(context.Background(), communication, false)
+	if err := run.Activate(); err != nil {
+		t.Fatal(err)
+	}
+
+	if !communication.TrySendLiveHDHandshake(LiveHDHandshake{}) {
+		t.Fatal("TrySendLiveHDHandshake() rejected an available channel")
+	}
+	if !communication.TrySendMotion(MotionDataPartial{}) {
+		t.Fatal("TrySendMotion() rejected an available channel")
+	}
+	if !communication.TrySendONVIF(OnvifAction{}) {
+		t.Fatal("TrySendONVIF() rejected an available channel")
+	}
+
+	report := run.Shutdown(context.Background(), errors.New("test complete"))
+	if !report.Complete {
+		t.Fatalf("shutdown report = %+v, want complete", report)
+	}
+	if communication.TrySendLiveHDHandshake(LiveHDHandshake{}) {
+		t.Fatal("TrySendLiveHDHandshake() accepted a closed run")
+	}
+	if communication.TrySendMotion(MotionDataPartial{}) {
+		t.Fatal("TrySendMotion() accepted a closed run")
+	}
+	if communication.TrySendONVIF(OnvifAction{}) {
+		t.Fatal("TrySendONVIF() accepted a closed run")
+	}
+}
+
+func TestCommunicationDispatchCanRaceRunChannelClose(t *testing.T) {
+	communication := &Communication{}
+	run := NewAgentRun(context.Background(), communication, false)
+	if err := run.Activate(); err != nil {
+		t.Fatal(err)
+	}
+
+	start := make(chan struct{})
+	var senders sync.WaitGroup
+	for sender := 0; sender < 20; sender++ {
+		senders.Add(1)
+		go func() {
+			defer senders.Done()
+			<-start
+			for attempt := 0; attempt < 100; attempt++ {
+				communication.TrySendLiveHDHandshake(LiveHDHandshake{})
+				communication.TrySendMotion(MotionDataPartial{})
+				communication.TrySendONVIF(OnvifAction{})
+			}
+		}()
+	}
+
+	close(start)
+	run.Shutdown(context.Background(), errors.New("test complete"))
+	senders.Wait()
+}
+
+func TestCommunicationRunChannelLifecycleSoak(t *testing.T) {
+	communication := &Communication{}
+	for cycle := 0; cycle < 100; cycle++ {
+		run := NewAgentRun(context.Background(), communication, false)
+		if err := run.Activate(); err != nil {
+			t.Fatalf("cycle %d Activate() error = %v", cycle, err)
+		}
+		handshakes := run.LiveHDHandshakes()
+		motion := run.MotionEvents()
+		onvif := run.ONVIFActions()
+
+		var consumers sync.WaitGroup
+		consumers.Add(3)
+		go func() {
+			defer consumers.Done()
+			for range handshakes {
+			}
+		}()
+		go func() {
+			defer consumers.Done()
+			for range motion {
+			}
+		}()
+		go func() {
+			defer consumers.Done()
+			for range onvif {
+			}
+		}()
+
+		var producers sync.WaitGroup
+		for producer := 0; producer < 4; producer++ {
+			producers.Add(1)
+			go func() {
+				defer producers.Done()
+				for attempt := 0; attempt < 50; attempt++ {
+					communication.TrySendLiveHDHandshake(LiveHDHandshake{})
+					communication.TrySendMotion(MotionDataPartial{})
+					communication.TrySendONVIF(OnvifAction{})
+				}
+			}()
+		}
+
+		run.Shutdown(context.Background(), errors.New("test complete"))
+		producers.Wait()
+
+		done := make(chan struct{})
+		go func() {
+			consumers.Wait()
+			close(done)
+		}()
+		select {
+		case <-done:
+		case <-time.After(time.Second):
+			t.Fatalf("cycle %d consumers did not stop after channel close", cycle)
+		}
+	}
+}
+
+func TestCommunicationRecoveryTelemetry(t *testing.T) {
+	communication := &Communication{}
+	communication.RecordMoQReconnect(StreamQualityHigh)
+	communication.RecordMoQReconnect(StreamQualityLow)
+	communication.RecordMoQWrite(StreamQualityHigh, 1250*time.Millisecond, time.UnixMilli(1234))
+	communication.RecordMoQWrite(StreamQualityLow, 2500*time.Millisecond, time.UnixMilli(5678))
+	communication.RecordMoQWriteTimeout()
+	communication.RecordWatchdogRestart(30 * time.Second)
+	communication.RecordRunWorkerShutdownTimeout()
+
+	if communication.TrySendLiveHDHandshake(LiveHDHandshake{}) {
+		t.Fatal("TrySendLiveHDHandshake() accepted an unavailable channel")
+	}
+	if communication.TrySendMotion(MotionDataPartial{}) {
+		t.Fatal("TrySendMotion() accepted an unavailable channel")
+	}
+	if communication.TrySendONVIF(OnvifAction{}) {
+		t.Fatal("TrySendONVIF() accepted an unavailable channel")
+	}
+
+	got := communication.RecoveryTelemetry()
+	if got.MoQHigh != (MoQRecoveryTelemetry{Reconnects: 1, LastFrameUnixMillis: 1234, LastWriteMillis: 1250}) {
+		t.Fatalf("high MoQ telemetry = %+v", got.MoQHigh)
+	}
+	if got.MoQLow != (MoQRecoveryTelemetry{Reconnects: 1, LastFrameUnixMillis: 5678, LastWriteMillis: 2500}) {
+		t.Fatalf("low MoQ telemetry = %+v", got.MoQLow)
+	}
+	if got.MoQWriteTimeouts != 1 || got.WatchdogRestarts != 1 || got.WatchdogCooldownSeconds != 30 || got.RunWorkerShutdownTimeouts != 1 {
+		t.Fatalf("recovery telemetry = %+v", got)
+	}
+	if got.DroppedLiveHDHandshakes != 1 || got.DroppedMotionEvents != 1 || got.DroppedONVIFActions != 1 {
+		t.Fatalf("drop telemetry = %+v", got)
+	}
+}
+
+func TestCommunicationOperationalTelemetry(t *testing.T) {
+	communication := &Communication{}
+	mainPacketAt := time.Unix(1_788_710_401, 0)
+	subPacketAt := time.Unix(1_788_710_402, 0)
+	hubAttemptAt := time.Unix(1_788_710_403, 0)
+	hubSuccessAt := time.Unix(1_788_710_404, 0)
+
+	communication.SetStreamConfigured(MainStream, true)
+	communication.SetStreamConfigured(SubStream, true)
+	communication.RecordStreamPackage(MainStream, 29.97, 1920, 1080, mainPacketAt)
+	communication.RecordStreamPackage(MainStream, 29.97, 1920, 1080, mainPacketAt)
+	communication.RecordStreamPackage(SubStream, 15, 640, 360, subPacketAt)
+	communication.SetHubConfigured(true)
+	communication.RecordHubHeartbeatAttempt(hubAttemptAt)
+	communication.RecordHubHeartbeatSuccess(hubSuccessAt)
+
+	main := communication.StreamRuntimeTelemetry(MainStream)
+	if main != (StreamRuntimeTelemetry{
+		Configured:        true,
+		PackagesProcessed: 2,
+		FPS:               29.97,
+		Width:             1920,
+		Height:            1080,
+		LastPacketAt:      mainPacketAt.Unix(),
+	}) {
+		t.Fatalf("main stream telemetry = %+v", main)
+	}
+
+	sub := communication.StreamRuntimeTelemetry(SubStream)
+	if sub != (StreamRuntimeTelemetry{
+		Configured:        true,
+		PackagesProcessed: 1,
+		FPS:               15,
+		Width:             640,
+		Height:            360,
+		LastPacketAt:      subPacketAt.Unix(),
+	}) {
+		t.Fatalf("sub stream telemetry = %+v", sub)
+	}
+
+	hub := communication.HubRuntimeTelemetry()
+	if hub != (HubRuntimeTelemetry{
+		Configured:                true,
+		Connected:                 true,
+		LastHeartbeatAttemptAt:    hubAttemptAt.Unix(),
+		LastSuccessfulHeartbeatAt: hubSuccessAt.Unix(),
+	}) {
+		t.Fatalf("Hub telemetry = %+v", hub)
+	}
+
+	communication.RecordHubHeartbeatFailure()
+	if communication.HubRuntimeTelemetry().Connected {
+		t.Fatal("Hub telemetry remained connected after a failed heartbeat")
+	}
+	communication.SetHubConfigured(false)
+	if communication.HubRuntimeTelemetry().Configured {
+		t.Fatal("Hub telemetry remained configured after it was disabled")
+	}
+}

machinery/src/models/config.go

@@ -187,6 +187,7 @@ type KStorage struct {
 	SecretAccessKey string `json:"secret_access_key,omitempty" bson:"secret_access_key,omitempty"`
 	Provider        string `json:"provider,omitempty" bson:"provider,omitempty"`
 	Directory       string `json:"directory,omitempty" bson:"directory,omitempty"`
+	CustomHeaders   string `json:"custom_headers,omitempty" bson:"custom_headers,omitempty"`
 	MaxRetries      int    `json:"max_retries,omitempty" bson:"max_retries,omitempty"`
 	Timeout         int    `json:"timeout,omitempty" bson:"timeout,omitempty"`
 }

machinery/src/models/mqtt.go

@@ -6,20 +6,96 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"encoding/pem"
-	"io"
-	"strings"
+	"errors"
+	"sync"
 	"time"
 
 	"github.com/gofrs/uuid"
 	"github.com/kerberos-io/agent/machinery/src/encryption"
-	"github.com/kerberos-io/agent/machinery/src/log"
+	log "github.com/sirupsen/logrus"
 )
 
+var (
+	errMQTTPrivateKeyPEMDecode = errors.New("error decoding PEM block containing private key")
+	errMQTTPrivateKeyNotRSA    = errors.New("private key is not RSA")
+
+	rsaPrivateKeyCache   = newRSAPrivateKeyCache()
+	parsePKCS8PrivateKey = x509.ParsePKCS8PrivateKey
+)
+
+type rsaPrivateKeyCacheState struct {
+	mu      sync.Mutex
+	cond    *sync.Cond
+	pem     string
+	key     *rsa.PrivateKey
+	err     error
+	parsing bool
+	ready   bool
+}
+
+func newRSAPrivateKeyCache() *rsaPrivateKeyCacheState {
+	cache := &rsaPrivateKeyCacheState{}
+	cache.cond = sync.NewCond(&cache.mu)
+	return cache
+}
+
+func (cache *rsaPrivateKeyCacheState) get(privateKey string) (*rsa.PrivateKey, error) {
+	cache.mu.Lock()
+	for {
+		if cache.ready && cache.pem == privateKey && !cache.parsing {
+			key, err := cache.key, cache.err
+			cache.mu.Unlock()
+			return key, err
+		}
+		if cache.parsing {
+			cache.cond.Wait()
+			continue
+		}
+
+		cache.pem = privateKey
+		cache.key = nil
+		cache.err = nil
+		cache.parsing = true
+		cache.ready = false
+		cache.mu.Unlock()
+
+		key, err := parseRSAPrivateKey(privateKey)
+
+		cache.mu.Lock()
+		cache.key = key
+		cache.err = err
+		cache.parsing = false
+		cache.ready = true
+		cache.cond.Broadcast()
+		cache.mu.Unlock()
+		return key, err
+	}
+}
+
+func parseRSAPrivateKey(privateKey string) (*rsa.PrivateKey, error) {
+	block, _ := pem.Decode([]byte(privateKey))
+	if block == nil {
+		return nil, errMQTTPrivateKeyPEMDecode
+	}
+
+	key, err := parsePKCS8PrivateKey(block.Bytes)
+	if err != nil {
+		return nil, err
+	}
+
+	rsaKey, ok := key.(*rsa.PrivateKey)
+	if !ok {
+		return nil, errMQTTPrivateKeyNotRSA
+	}
+
+	return rsaKey, nil
+}
+
 func PackageMQTTMessage(configuration *Configuration, msg Message) ([]byte, error) {
 	// Create a Version 4 UUID.
 	u2, err := uuid.NewV4()
 	if err != nil {
-		log.Log.Error("failed to generate UUID: " + err.Error())
+		log.Error("failed to generate UUID: " + err.Error())
 	}
 
 	// We'll generate an unique id, and encrypt / decrypt it using the private key if available.
@@ -45,27 +121,25 @@ func PackageMQTTMessage(configuration *Configuration, msg Message) ([]byte, erro
 		// Pload to base64
 		data, err := json.Marshal(pload)
 		if err != nil {
-			log.Log.Error("models.mqtt.PackageMQTTMessage(): failed to marshal payload: " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component": "models/mqtt",
+				"event":     "payload_marshal_failed",
+			}).Error("Failed to marshal MQTT payload")
 		}
 
 		// Encrypt the value
 		privateKey := configuration.Config.Encryption.PrivateKey
-		r := strings.NewReader(privateKey)
-		pemBytes, _ := io.ReadAll(r)
-		block, _ := pem.Decode(pemBytes)
-		if block == nil {
-			log.Log.Error("models.mqtt.PackageMQTTMessage(): error decoding PEM block containing private key")
-		} else {
-			// Parse private key
-			b := block.Bytes
-			key, err := x509.ParsePKCS8PrivateKey(b)
-			if err != nil {
-				log.Log.Error("models.mqtt.PackageMQTTMessage(): error parsing private key: " + err.Error())
+		rsaKey, err := rsaPrivateKeyCache.get(privateKey)
+		if err != nil {
+			if errors.Is(err, errMQTTPrivateKeyPEMDecode) {
+				log.Error("models.mqtt.PackageMQTTMessage(): error decoding PEM block containing private key")
+			} else {
+				log.WithError(err).WithFields(log.Fields{
+					"component": "models/mqtt",
+					"event":     "private_key_parse_failed",
+				}).Error("Failed to parse MQTT private key")
 			}
-
-			// Conver key to *rsa.PrivateKey
-			rsaKey, _ := key.(*rsa.PrivateKey)
-
+		} else {
 			// Create a 16bit key random
 			if config.Encryption != nil && config.Encryption.SymmetricKey != "" {
 				k := config.Encryption.SymmetricKey
@@ -250,3 +324,23 @@ type TriggerRelay struct {
 	DeviceId  string `json:"device_id"` // device id
 	Token     string `json:"token"`     // token
 }
+
+// RemoteSessionPayload controls an interactive shell or log stream over MQTT.
+// Data is base64 encoded so terminal control bytes remain valid JSON.
+type RemoteSessionPayload struct {
+	Timestamp int64  `json:"timestamp"`
+	SessionID string `json:"session_id"`
+	Kind      string `json:"kind,omitempty"`
+	Data      string `json:"data,omitempty"`
+	Rows      uint16 `json:"rows,omitempty"`
+	Columns   uint16 `json:"columns,omitempty"`
+	Tail      int    `json:"tail,omitempty"`
+}
+
+type RemoteSessionStatus struct {
+	Timestamp int64  `json:"timestamp"`
+	SessionID string `json:"session_id"`
+	Kind      string `json:"kind"`
+	State     string `json:"state"`
+	Error     string `json:"error,omitempty"`
+}

machinery/src/models/mqtt_test.go

@@ -0,0 +1,199 @@
+package models
+
+import (
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/json"
+	"encoding/pem"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestPackageMQTTMessageCachesCurrentSigningKey(t *testing.T) {
+	resetMQTTPrivateKeyCache(t)
+
+	privateKey := generatePKCS8RSAPrivateKeyPEM(t)
+
+	var parseCalls atomic.Int32
+	restore := hookPKCS8PrivateKeyParser(t, func(der []byte) (any, error) {
+		parseCalls.Add(1)
+		return x509.ParsePKCS8PrivateKey(der)
+	})
+	defer restore()
+
+	msg := Message{
+		Payload: Payload{
+			DeviceId: "device-1",
+			Value: map[string]interface{}{
+				"hello": "world",
+			},
+		},
+	}
+
+	first, err := PackageMQTTMessage(encryptedConfiguration(privateKey), msg)
+	if err != nil {
+		t.Fatalf("first PackageMQTTMessage() error = %v", err)
+	}
+	assertEncryptedMQTTMessage(t, first)
+
+	second, err := PackageMQTTMessage(encryptedConfiguration(privateKey), msg)
+	if err != nil {
+		t.Fatalf("second PackageMQTTMessage() error = %v", err)
+	}
+	assertEncryptedMQTTMessage(t, second)
+
+	if got := parseCalls.Load(); got != 1 {
+		t.Fatalf("cached parse calls = %d, want 1", got)
+	}
+}
+
+func TestPackageMQTTMessageReplacesCachedSigningKeyOnRotation(t *testing.T) {
+	resetMQTTPrivateKeyCache(t)
+
+	privateKey := generatePKCS8RSAPrivateKeyPEM(t)
+	rotatedPrivateKey := generatePKCS8RSAPrivateKeyPEM(t)
+
+	var parseCalls atomic.Int32
+	restore := hookPKCS8PrivateKeyParser(t, func(der []byte) (any, error) {
+		parseCalls.Add(1)
+		return x509.ParsePKCS8PrivateKey(der)
+	})
+	defer restore()
+
+	msg := Message{
+		Payload: Payload{
+			DeviceId: "device-1",
+			Value: map[string]interface{}{
+				"hello": "world",
+			},
+		},
+	}
+
+	for _, key := range []string{privateKey, rotatedPrivateKey, privateKey} {
+		payload, err := PackageMQTTMessage(encryptedConfiguration(key), msg)
+		if err != nil {
+			t.Fatalf("PackageMQTTMessage() error = %v", err)
+		}
+		assertEncryptedMQTTMessage(t, payload)
+	}
+
+	if got := parseCalls.Load(); got != 3 {
+		t.Fatalf("parse calls across old/new/old rotation = %d, want 3", got)
+	}
+}
+
+func TestPackageMQTTMessageConcurrentCallsReuseCachedSigningKey(t *testing.T) {
+	resetMQTTPrivateKeyCache(t)
+
+	privateKey := generatePKCS8RSAPrivateKeyPEM(t)
+
+	var parseCalls atomic.Int32
+	restore := hookPKCS8PrivateKeyParser(t, func(der []byte) (any, error) {
+		parseCalls.Add(1)
+		time.Sleep(10 * time.Millisecond)
+		return x509.ParsePKCS8PrivateKey(der)
+	})
+	defer restore()
+
+	msg := Message{
+		Payload: Payload{
+			DeviceId: "device-1",
+			Value: map[string]interface{}{
+				"hello": "world",
+			},
+		},
+	}
+
+	start := make(chan struct{})
+	var workers sync.WaitGroup
+	for i := 0; i < 16; i++ {
+		workers.Add(1)
+		go func() {
+			defer workers.Done()
+			<-start
+
+			payload, err := PackageMQTTMessage(encryptedConfiguration(privateKey), msg)
+			if err != nil {
+				t.Errorf("PackageMQTTMessage() error = %v", err)
+				return
+			}
+			assertEncryptedMQTTMessage(t, payload)
+		}()
+	}
+
+	close(start)
+	workers.Wait()
+
+	if got := parseCalls.Load(); got != 1 {
+		t.Fatalf("concurrent parse calls = %d, want 1", got)
+	}
+}
+
+func encryptedConfiguration(privateKey string) *Configuration {
+	return &Configuration{
+		Config: Config{
+			Encryption: &Encryption{
+				Enabled:      "true",
+				PrivateKey:   privateKey,
+				SymmetricKey: "secret",
+			},
+		},
+	}
+}
+
+func generatePKCS8RSAPrivateKeyPEM(t *testing.T) string {
+	t.Helper()
+
+	privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		t.Fatalf("rsa.GenerateKey() error = %v", err)
+	}
+
+	der, err := x509.MarshalPKCS8PrivateKey(privateKey)
+	if err != nil {
+		t.Fatalf("x509.MarshalPKCS8PrivateKey() error = %v", err)
+	}
+
+	return string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}))
+}
+
+func assertEncryptedMQTTMessage(t *testing.T, payload []byte) {
+	t.Helper()
+
+	var got Message
+	if err := json.Unmarshal(payload, &got); err != nil {
+		t.Fatalf("json.Unmarshal() error = %v", err)
+	}
+
+	if !got.Encrypted {
+		t.Fatal("message is not marked encrypted")
+	}
+	if got.Payload.EncryptedValue == "" {
+		t.Fatal("encrypted value is empty")
+	}
+	if got.Payload.Signature == "" {
+		t.Fatal("signature is empty")
+	}
+	if len(got.Payload.Value) != 0 {
+		t.Fatalf("payload value = %#v, want cleared map", got.Payload.Value)
+	}
+}
+
+func hookPKCS8PrivateKeyParser(t *testing.T, parser func([]byte) (any, error)) func() {
+	t.Helper()
+
+	original := parsePKCS8PrivateKey
+	parsePKCS8PrivateKey = parser
+
+	return func() {
+		parsePKCS8PrivateKey = original
+	}
+}
+
+func resetMQTTPrivateKeyCache(t *testing.T) {
+	t.Helper()
+	rsaPrivateKeyCache = newRSAPrivateKeyCache()
+}

machinery/src/models/recording_upload_metadata.go

@@ -11,11 +11,15 @@ const RecordingUploadMetadataExtension = ".metadata"
 // with a recording. New optional fields can be added without changing the queue
 // mechanism or breaking older agents.
 type RecordingUploadMetadata struct {
-	FileName  string  `json:"filename"`
-	DeviceKey string  `json:"device_key"`
-	Timestamp int64   `json:"timestamp"` // Unix milliseconds.
-	Duration  uint64  `json:"duration"`  // Milliseconds.
-	FPS       float64 `json:"fps,omitempty"`
+	FileName          string  `json:"filename"`
+	DeviceKey         string  `json:"device_key"`
+	Timestamp         int64   `json:"timestamp"` // Unix milliseconds.
+	Duration          uint64  `json:"duration"`  // Milliseconds.
+	FPS               float64 `json:"fps,omitempty"`
+	Encrypted         bool    `json:"encrypted,omitempty"`
+	DeviceName        string  `json:"device_name,omitempty"`
+	RegionCoordinates string  `json:"region_coordinates,omitempty"`
+	NumberOfChanges   string  `json:"number_of_changes,omitempty"`
 }
 
 // RecordingUploadMetadataFileName returns the queue marker name associated

machinery/src/onvif/discovery.go

@@ -13,8 +13,8 @@ import (
 	"time"
 
 	onvifc "github.com/cedricve/go-onvif"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 // scanPort describes a TCP port we probe while scanning the local network,
@@ -105,7 +105,7 @@ func DiscoverDevices(timeout time.Duration, subnets ...string) []models.Discover
 	// 1) ONVIF WS-Discovery. This is quick and reliable for ONVIF cameras.
 	onvifDevices, err := onvifc.StartDiscovery(timeout)
 	if err != nil {
-		log.Log.Error("onvif.DiscoverDevices(): WS-Discovery failed: " + err.Error())
+		log.Error("onvif.DiscoverDevices(): WS-Discovery failed: " + err.Error())
 	} else {
 		for _, onvifDevice := range onvifDevices {
 			ip := hostFromXAddr(onvifDevice.XAddr)
@@ -129,7 +129,7 @@ func DiscoverDevices(timeout time.Duration, subnets ...string) []models.Discover
 	} else {
 		targets = localScanTargets()
 	}
-	log.Log.Info("onvif.DiscoverDevices(): scanning " + strconv.Itoa(len(targets)) + " hosts on the local network(s)")
+	log.Info("onvif.DiscoverDevices(): scanning " + strconv.Itoa(len(targets)) + " hosts on the local network(s)")
 
 	// Bound the amount of concurrent dials so we do not exhaust file
 	// descriptors on constrained devices (e.g. Raspberry Pi).
@@ -319,17 +319,17 @@ func targetsFromSubnets(subnets []string) []string {
 					targets = append(targets, subnet)
 				}
 			} else {
-				log.Log.Error("onvif.targetsFromSubnets(): invalid address '" + subnet + "'")
+				log.Error("onvif.targetsFromSubnets(): invalid address '" + subnet + "'")
 			}
 			continue
 		}
 		_, ipNet, err := net.ParseCIDR(subnet)
 		if err != nil || ipNet.IP.To4() == nil {
-			log.Log.Error("onvif.targetsFromSubnets(): invalid CIDR '" + subnet + "'")
+			log.Error("onvif.targetsFromSubnets(): invalid CIDR '" + subnet + "'")
 			continue
 		}
 		if ones, bits := ipNet.Mask.Size(); bits != 32 || ones < 22 {
-			log.Log.Error("onvif.targetsFromSubnets(): range '" + subnet + "' is too large to scan (use /22 or smaller)")
+			log.Error("onvif.targetsFromSubnets(): range '" + subnet + "' is too large to scan (use /22 or smaller)")
 			continue
 		}
 		for _, host := range hostsInNetwork(ipNet) {
@@ -352,7 +352,7 @@ func localScanTargets() []string {
 
 	interfaces, err := net.Interfaces()
 	if err != nil {
-		log.Log.Error("onvif.localScanTargets(): " + err.Error())
+		log.Error("onvif.localScanTargets(): " + err.Error())
 		return targets
 	}
 

machinery/src/onvif/events.go

@@ -7,9 +7,9 @@ import (
 	"strings"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/onvif/event/stream"
+	log "github.com/sirupsen/logrus"
 )
 
 // The library handles in-stream reconnect; these guards cover the
@@ -29,14 +29,14 @@ const (
 // blip, credential reload) the goroutine retries with exponential
 // backoff. Exits when ctx is cancelled.
 func HandleONVIFEventStream(ctx context.Context, configuration *models.Configuration, communication *models.Communication) {
-	log.Log.Debug("onvif.HandleONVIFEventStream(): started")
-	defer log.Log.Debug("onvif.HandleONVIFEventStream(): finished")
+	log.Debug("onvif.HandleONVIFEventStream(): started")
+	defer log.Debug("onvif.HandleONVIFEventStream(): finished")
 
 	if !isONVIFMotionEnabled(configuration.Config.Capture.ONVIFMotion) {
 		return
 	}
 	if configuration.Config.Capture.IPCamera.ONVIFXAddr == "" {
-		log.Log.Warning("onvif.HandleONVIFEventStream(): ONVIFMotion enabled but ONVIFXAddr is empty; nothing to do")
+		log.Warn("onvif.HandleONVIFEventStream(): ONVIFMotion enabled but ONVIFXAddr is empty; nothing to do")
 		return
 	}
 
@@ -66,23 +66,26 @@ func runStreamOnce(ctx context.Context, configuration *models.Configuration, com
 
 	device, _, err := ConnectToOnvifDevice(&camera)
 	if err != nil {
-		log.Log.Error("onvif.HandleONVIFEventStream(): connect: " + err.Error())
+		log.Error("onvif.HandleONVIFEventStream(): connect: " + err.Error())
 		return true
 	}
 
 	deviceID := resolveDeviceID(configuration.Name, camera.ONVIFXAddr)
 	s, err := stream.NewStream(ctx, device, stream.Options{DeviceID: deviceID})
 	if err != nil {
-		log.Log.Error("onvif.HandleONVIFEventStream(): open stream: " + err.Error())
+		log.Error("onvif.HandleONVIFEventStream(): open stream: " + err.Error())
 		return true
 	}
 	defer func() {
 		if err := s.Close(); err != nil {
-			log.Log.Debug("onvif.HandleONVIFEventStream(): close: " + err.Error())
+			log.Debug("onvif.HandleONVIFEventStream(): close: " + err.Error())
 		}
 	}()
 
-	log.Log.Info("onvif.HandleONVIFEventStream(): consuming events for " + deviceID)
+	log.WithFields(log.Fields{
+		"component": "onvif",
+		"event":     "event_stream_started",
+	}).Info("Consuming ONVIF events")
 
 	// recovering = the first successful event after an error streak
 	// logs a recovery line so on-call operators see the clear-of-
@@ -97,7 +100,10 @@ func runStreamOnce(ctx context.Context, configuration *models.Configuration, com
 				return false
 			}
 			if recovering {
-				log.Log.Info("onvif.HandleONVIFEventStream(): event stream recovered for " + deviceID)
+				log.WithFields(log.Fields{
+					"component": "onvif",
+					"event":     "event_stream_recovered",
+				}).Info("ONVIF event stream recovered")
 				recovering = false
 			}
 			dispatchEvent(ctx, ev, configuration, communication)
@@ -119,14 +125,14 @@ func runStreamOnce(ctx context.Context, configuration *models.Configuration, com
 func dispatchEvent(ctx context.Context, ev stream.Event, configuration *models.Configuration, communication *models.Communication) {
 	topic := sanitiseTopic(ev.Topic)
 	if ev.Kind != stream.KindMotion {
-		log.Log.Debug("onvif.dispatchEvent(): non-motion event " + ev.Kind.String() + " topic=" + topic)
+		log.Debug("onvif.dispatchEvent(): non-motion event " + ev.Kind.String() + " topic=" + topic)
 		return
 	}
 	if ev.State != stream.StateActive {
 		return
 	}
 	if !isTransition(ev.Operation) {
-		log.Log.Debug("onvif.dispatchEvent(): " + ev.Operation.String() + " is not a transition, not a trigger: topic=" + topic)
+		log.Debug("onvif.dispatchEvent(): " + ev.Operation.String() + " is not a transition, not a trigger: topic=" + topic)
 		return
 	}
 	if configuration.Config.Capture.Recording == "false" {
@@ -140,14 +146,15 @@ func dispatchEvent(ctx context.Context, ev stream.Event, configuration *models.C
 		Timestamp:       time.Now().Unix(),
 		NumberOfChanges: 0, // ONVIF does not quantify motion area.
 	}
-	select {
-	case <-ctx.Done():
-	case communication.HandleMotion <- dataToPass:
+	if ctx.Err() != nil {
+		return
+	}
+	if communication.TrySendMotion(dataToPass) {
 		// Logged on the send, not before it: this line records that a
 		// recording started, so a dropped event must not leave one.
-		log.Log.Debug("onvif.dispatchEvent(): recording trigger " + ev.Kind.String() + " topic=" + topic)
-	default:
-		log.Log.Debug("onvif.dispatchEvent(): HandleMotion full, dropping ONVIF motion event")
+		log.Debug("onvif.dispatchEvent(): recording trigger " + ev.Kind.String() + " topic=" + topic)
+	} else {
+		log.Debug("onvif.dispatchEvent(): HandleMotion full, dropping ONVIF motion event")
 	}
 }
 
@@ -185,13 +192,13 @@ func logStreamError(e error) {
 	var renew stream.ErrRenewFailed
 	switch {
 	case errors.As(e, &recreate):
-		log.Log.Error("onvif.HandleONVIFEventStream(): subscription recreate failed (camera may be offline): " + recreate.Err.Error())
+		log.Error("onvif.HandleONVIFEventStream(): subscription recreate failed (camera may be offline): " + recreate.Err.Error())
 	case errors.As(e, &renew):
-		log.Log.Debug("onvif.HandleONVIFEventStream(): renew failed (will recover via pull/recreate): " + renew.Err.Error())
+		log.Debug("onvif.HandleONVIFEventStream(): renew failed (will recover via pull/recreate): " + renew.Err.Error())
 	case errors.As(e, &pull):
-		log.Log.Debug("onvif.HandleONVIFEventStream(): pull failed (will retry): " + pull.Err.Error())
+		log.Debug("onvif.HandleONVIFEventStream(): pull failed (will retry): " + pull.Err.Error())
 	default:
-		log.Log.Info("onvif.HandleONVIFEventStream(): stream error: " + e.Error())
+		log.Info("onvif.HandleONVIFEventStream(): stream error: " + e.Error())
 	}
 }
 

machinery/src/onvif/events_test.go

@@ -3,6 +3,7 @@ package onvif
 import (
 	"bytes"
 	"context"
+	"errors"
 	"strings"
 	"testing"
 	"time"
@@ -26,17 +27,31 @@ func makeConfig(recording, onvifMotion, name string) *models.Configuration {
 	}
 }
 
-func makeCommunication(buffer int) *models.Communication {
-	return &models.Communication{
-		HandleMotion: make(chan models.MotionDataPartial, buffer),
+func makeCommunication(t *testing.T) *models.Communication {
+	t.Helper()
+	communication := &models.Communication{}
+	run := models.NewAgentRun(context.Background(), communication, false)
+	require.NoError(t, run.Activate())
+	t.Cleanup(func() {
+		run.Shutdown(context.Background(), errors.New("test complete"))
+	})
+	return communication
+}
+
+func motionEvents(t *testing.T, communication *models.Communication) <-chan models.MotionDataPartial {
+	t.Helper()
+	run := communication.CurrentRun()
+	if run == nil {
+		t.Fatal("expected active agent run")
 	}
+	return run.MotionEvents()
 }
 
 // --- dispatchEvent ---------------------------------------------------
 
 func TestDispatchEvent_MotionActive_SendsToHandleMotion(t *testing.T) {
 	cfg := makeConfig("true", "true", "cam-1")
-	comm := makeCommunication(1)
+	comm := makeCommunication(t)
 	ev := stream.Event{Kind: stream.KindMotion, State: stream.StateActive}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -44,7 +59,7 @@ func TestDispatchEvent_MotionActive_SendsToHandleMotion(t *testing.T) {
 	dispatchEvent(ctx, ev, cfg, comm)
 
 	select {
-	case m := <-comm.HandleMotion:
+	case m := <-motionEvents(t, comm):
 		assert.NotZero(t, m.Timestamp)
 	case <-time.After(time.Second):
 		t.Fatal("expected motion data on HandleMotion")
@@ -53,7 +68,7 @@ func TestDispatchEvent_MotionActive_SendsToHandleMotion(t *testing.T) {
 
 func TestDispatchEvent_MotionInactive_DoesNotSend(t *testing.T) {
 	cfg := makeConfig("true", "true", "cam-1")
-	comm := makeCommunication(1)
+	comm := makeCommunication(t)
 	ev := stream.Event{Kind: stream.KindMotion, State: stream.StateInactive}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -61,7 +76,7 @@ func TestDispatchEvent_MotionInactive_DoesNotSend(t *testing.T) {
 	dispatchEvent(ctx, ev, cfg, comm)
 
 	select {
-	case <-comm.HandleMotion:
+	case <-motionEvents(t, comm):
 		t.Fatal("inactive motion must not reach HandleMotion (motion-stop is a follow-up)")
 	case <-time.After(100 * time.Millisecond):
 	}
@@ -69,7 +84,7 @@ func TestDispatchEvent_MotionInactive_DoesNotSend(t *testing.T) {
 
 func TestDispatchEvent_NonMotionKindIgnored(t *testing.T) {
 	cfg := makeConfig("true", "true", "cam-1")
-	comm := makeCommunication(1)
+	comm := makeCommunication(t)
 	ev := stream.Event{Kind: stream.KindDigitalInput, State: stream.StateActive}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -77,7 +92,7 @@ func TestDispatchEvent_NonMotionKindIgnored(t *testing.T) {
 	dispatchEvent(ctx, ev, cfg, comm)
 
 	select {
-	case <-comm.HandleMotion:
+	case <-motionEvents(t, comm):
 		t.Fatal("non-motion kinds must not reach HandleMotion")
 	case <-time.After(100 * time.Millisecond):
 	}
@@ -108,7 +123,7 @@ func TestDispatchEvent_LogsTheTriggeringTopic(t *testing.T) {
 	buf := captureDebugLog(t)
 
 	cfg := makeConfig("true", "true", "cam-1")
-	comm := makeCommunication(1)
+	comm := makeCommunication(t)
 	ev := stream.Event{
 		Kind:  stream.KindMotion,
 		State: stream.StateActive,
@@ -147,7 +162,7 @@ func TestDispatchEvent_PropertyOperation(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			cfg := makeConfig("true", "true", "cam-1")
-			comm := makeCommunication(1)
+			comm := makeCommunication(t)
 			ev := stream.Event{
 				Kind:      stream.KindMotion,
 				State:     stream.StateActive,
@@ -160,7 +175,7 @@ func TestDispatchEvent_PropertyOperation(t *testing.T) {
 			dispatchEvent(ctx, ev, cfg, comm)
 
 			select {
-			case <-comm.HandleMotion:
+			case <-motionEvents(t, comm):
 				if !tt.wantSend {
 					t.Fatalf("%v must not trigger a recording", tt.op)
 				}
@@ -175,7 +190,7 @@ func TestDispatchEvent_PropertyOperation(t *testing.T) {
 
 func TestDispatchEvent_RecordingDisabled_DoesNotSend(t *testing.T) {
 	cfg := makeConfig("false", "true", "cam-1")
-	comm := makeCommunication(1)
+	comm := makeCommunication(t)
 	ev := stream.Event{Kind: stream.KindMotion, State: stream.StateActive}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -183,7 +198,7 @@ func TestDispatchEvent_RecordingDisabled_DoesNotSend(t *testing.T) {
 	dispatchEvent(ctx, ev, cfg, comm)
 
 	select {
-	case <-comm.HandleMotion:
+	case <-motionEvents(t, comm):
 		t.Fatal("Recording=false must gate the send (matches computervision behaviour)")
 	case <-time.After(100 * time.Millisecond):
 	}
@@ -191,9 +206,10 @@ func TestDispatchEvent_RecordingDisabled_DoesNotSend(t *testing.T) {
 
 func TestDispatchEvent_HandleMotionFull_DropsRatherThanBlocks(t *testing.T) {
 	cfg := makeConfig("true", "true", "cam-1")
-	// Pre-fill the buffer so the next send would block.
-	comm := &models.Communication{HandleMotion: make(chan models.MotionDataPartial, 1)}
-	comm.HandleMotion <- models.MotionDataPartial{}
+	// Fill the buffer so the next send would be dropped.
+	comm := makeCommunication(t)
+	for comm.TrySendMotion(models.MotionDataPartial{}) {
+	}
 	ev := stream.Event{Kind: stream.KindMotion, State: stream.StateActive}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -219,8 +235,9 @@ func TestDispatchEvent_CtxCancelledAndHandleMotionClosed_DoesNotPanic(t *testing
 	// send would panic. The ctx pre-check must short-circuit before the
 	// send is attempted.
 	cfg := makeConfig("true", "true", "cam-1")
-	comm := &models.Communication{HandleMotion: make(chan models.MotionDataPartial, 1)}
-	close(comm.HandleMotion)
+	comm := makeCommunication(t)
+	run := comm.CurrentRun()
+	run.Shutdown(context.Background(), errors.New("test shutdown"))
 	ev := stream.Event{Kind: stream.KindMotion, State: stream.StateActive}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -299,7 +316,7 @@ func TestDispatchEvent_OnlyRealTransitionsTrigger(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.op.String(), func(t *testing.T) {
 			cfg := makeConfig("true", "true", "cam-1")
-			comm := makeCommunication(1)
+			comm := makeCommunication(t)
 			ev := stream.Event{Kind: stream.KindMotion, State: stream.StateActive, Operation: tt.op}
 
 			ctx, cancel := context.WithCancel(context.Background())
@@ -307,10 +324,10 @@ func TestDispatchEvent_OnlyRealTransitionsTrigger(t *testing.T) {
 			dispatchEvent(ctx, ev, cfg, comm)
 
 			if tt.wantSend {
-				require.Len(t, comm.HandleMotion, 1, "%v must trigger a recording", tt.op)
+				require.Len(t, motionEvents(t, comm), 1, "%v must trigger a recording", tt.op)
 				return
 			}
-			require.Empty(t, comm.HandleMotion, "%v must not trigger a recording", tt.op)
+			require.Empty(t, motionEvents(t, comm), "%v must not trigger a recording", tt.op)
 		})
 	}
 }
@@ -361,8 +378,9 @@ func TestDispatchEvent_LogsTriggerOnlyWhenSent(t *testing.T) {
 	buf := captureDebugLog(t)
 
 	cfg := makeConfig("true", "true", "cam-1")
-	comm := &models.Communication{HandleMotion: make(chan models.MotionDataPartial, 1)}
-	comm.HandleMotion <- models.MotionDataPartial{} // full
+	comm := makeCommunication(t)
+	for comm.TrySendMotion(models.MotionDataPartial{}) {
+	}
 	ev := stream.Event{Kind: stream.KindMotion, State: stream.StateActive, Topic: "tns1:VideoSource/MotionAlarm"}
 
 	ctx, cancel := context.WithCancel(context.Background())

machinery/src/onvif/main_test.go

@@ -0,0 +1,42 @@
+package onvif
+
+import (
+	"bytes"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func TestHandleONVIFResponseDoesNotLogBody(t *testing.T) {
+	logger := log.StandardLogger()
+	originalOutput := logger.Out
+	originalFormatter := logger.Formatter
+	originalLevel := logger.Level
+	defer func() {
+		logger.SetOutput(originalOutput)
+		logger.SetFormatter(originalFormatter)
+		logger.SetLevel(originalLevel)
+	}()
+
+	var output bytes.Buffer
+	logger.SetOutput(&output)
+	logger.SetFormatter(&log.JSONFormatter{})
+	logger.SetLevel(log.DebugLevel)
+
+	response := &http.Response{
+		StatusCode: http.StatusOK,
+		Body:       io.NopCloser(strings.NewReader(`{"credential":"do-not-log"}`)),
+	}
+	if err := handleONVIFResponse("test_operation", response, nil); err != nil {
+		t.Fatalf("handleONVIFResponse() error = %v", err)
+	}
+	if strings.Contains(output.String(), "do-not-log") {
+		t.Fatalf("ONVIF response log exposed response body: %q", output.String())
+	}
+	if !strings.Contains(output.String(), `"response_bytes":27`) {
+		t.Fatalf("ONVIF response log omitted response byte count: %q", output.String())
+	}
+}

machinery/src/onvif/move_test.go

@@ -0,0 +1,264 @@
+package onvif
+
+import (
+	"encoding/xml"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/kerberos-io/onvif/ptz"
+	xsdonvif "github.com/kerberos-io/onvif/xsd/onvif"
+)
+
+type fakePTZMoveClient struct {
+	current              ptzTarget
+	target               ptzTarget
+	lastVelocity         ptzVelocity
+	absoluteErr          error
+	continuousErr        error
+	stopErr              error
+	ignoreAbsolute       bool
+	stalled              bool
+	incompletePosition   bool
+	positionErr          error
+	positionErrAtCall    int
+	positionCalls        int
+	absoluteCalls        int
+	continuousVelocities []ptzVelocity
+	stopCalls            int
+	moving               bool
+}
+
+func (client *fakePTZMoveClient) GetPosition() (xsdonvif.PTZVector, error) {
+	client.positionCalls++
+	if client.positionErr != nil && client.positionCalls == client.positionErrAtCall {
+		return xsdonvif.PTZVector{}, client.positionErr
+	}
+	if client.moving && !client.stalled {
+		client.current.Pan = moveFakeAxis(client.current.Pan, client.target.Pan, client.lastVelocity.Pan)
+		client.current.Tilt = moveFakeAxis(client.current.Tilt, client.target.Tilt, client.lastVelocity.Tilt)
+		client.current.Zoom = moveFakeAxis(client.current.Zoom, client.target.Zoom, client.lastVelocity.Zoom)
+	}
+
+	panTilt := &xsdonvif.Vector2D{X: client.current.Pan, Y: client.current.Tilt}
+	if client.incompletePosition {
+		return xsdonvif.PTZVector{PanTilt: panTilt}, nil
+	}
+	return xsdonvif.PTZVector{
+		PanTilt: panTilt,
+		Zoom:    &xsdonvif.Vector1D{X: client.current.Zoom},
+	}, nil
+}
+
+func (client *fakePTZMoveClient) AbsoluteMove(target ptzTarget) error {
+	client.absoluteCalls++
+	client.target = target
+	if client.absoluteErr != nil {
+		return client.absoluteErr
+	}
+	if !client.ignoreAbsolute {
+		client.current = target
+	}
+	return nil
+}
+
+func (client *fakePTZMoveClient) ContinuousMove(velocity ptzVelocity) error {
+	if client.continuousErr != nil {
+		return client.continuousErr
+	}
+	client.lastVelocity = velocity
+	client.continuousVelocities = append(client.continuousVelocities, velocity)
+	client.moving = true
+	return nil
+}
+
+func (client *fakePTZMoveClient) Stop() error {
+	client.stopCalls++
+	client.moving = false
+	return client.stopErr
+}
+
+func moveFakeAxis(current float64, target float64, velocity float64) float64 {
+	step := velocity * 0.5
+	if step == 0 {
+		return current
+	}
+	next := current + step
+	if (velocity > 0 && next >= target) || (velocity < 0 && next <= target) {
+		return target
+	}
+	return next
+}
+
+func testPTZMoveOptions() ptzMoveOptions {
+	options := defaultPTZMoveOptions
+	options.NativeTimeout = 50 * time.Millisecond
+	options.ContinuousTimeout = 100 * time.Millisecond
+	options.NoProgressTimeout = 10 * time.Millisecond
+	options.PollInterval = time.Millisecond
+	options.ProgressEpsilon = 0.0001
+	options.VelocityChange = 0.01
+	return options
+}
+
+func TestMoveToPositionUsesVerifiedNativeMove(t *testing.T) {
+	target := ptzTarget{Pan: 0.5, Tilt: -0.25, Zoom: 0.4}
+	client := &fakePTZMoveClient{}
+
+	err := moveToPosition(client, ptzMoveSupport{Absolute: true, Continuous: true}, target, testPTZMoveOptions())
+	if err != nil {
+		t.Fatalf("moveToPosition() error = %v", err)
+	}
+	if client.absoluteCalls != 1 {
+		t.Fatalf("absolute calls = %d, want 1", client.absoluteCalls)
+	}
+	if len(client.continuousVelocities) != 0 {
+		t.Fatalf("continuous calls = %d, want 0", len(client.continuousVelocities))
+	}
+}
+
+func TestMoveToPositionFallsBackToSimultaneousContinuousMove(t *testing.T) {
+	target := ptzTarget{Pan: 0.5, Tilt: -0.25, Zoom: 0.4}
+	client := &fakePTZMoveClient{
+		target:      target,
+		absoluteErr: errors.New("absolute move rejected"),
+	}
+
+	err := moveToPosition(client, ptzMoveSupport{Absolute: true, Continuous: true}, target, testPTZMoveOptions())
+	if err != nil {
+		t.Fatalf("moveToPosition() error = %v", err)
+	}
+	if len(client.continuousVelocities) == 0 {
+		t.Fatal("continuous fallback was not used")
+	}
+	first := client.continuousVelocities[0]
+	if first.Pan == 0 || first.Tilt == 0 || first.Zoom == 0 {
+		t.Fatalf("first fallback velocity = %+v, want all axes moving", first)
+	}
+	if client.stopCalls != 2 {
+		t.Fatalf("stop calls = %d, want native cleanup and fallback cleanup", client.stopCalls)
+	}
+}
+
+func TestMoveToPositionFallsBackWhenNativeMoveMakesNoProgress(t *testing.T) {
+	target := ptzTarget{Pan: 0.3, Tilt: 0.2, Zoom: 0.1}
+	client := &fakePTZMoveClient{
+		target:         target,
+		ignoreAbsolute: true,
+	}
+	options := testPTZMoveOptions()
+	options.NoProgressTimeout = 3 * time.Millisecond
+
+	err := moveToPosition(client, ptzMoveSupport{Absolute: true, Continuous: true}, target, options)
+	if err != nil {
+		t.Fatalf("moveToPosition() error = %v", err)
+	}
+	if len(client.continuousVelocities) == 0 {
+		t.Fatal("continuous fallback was not used")
+	}
+}
+
+func TestMoveToPositionStopsAfterContinuousStatusError(t *testing.T) {
+	target := ptzTarget{Pan: 0.3, Tilt: 0.2, Zoom: 0.1}
+	client := &fakePTZMoveClient{
+		target:            target,
+		positionErr:       errors.New("status unavailable"),
+		positionErrAtCall: 3,
+	}
+
+	err := moveToPosition(client, ptzMoveSupport{Continuous: true}, target, testPTZMoveOptions())
+	if err == nil || !strings.Contains(err.Error(), "status unavailable") {
+		t.Fatalf("moveToPosition() error = %v, want status error", err)
+	}
+	if client.stopCalls != 1 {
+		t.Fatalf("stop calls = %d, want 1", client.stopCalls)
+	}
+}
+
+func TestMoveToPositionStopsWhenContinuousMoveIsRejected(t *testing.T) {
+	target := ptzTarget{Pan: 0.3, Tilt: 0.2, Zoom: 0.1}
+	client := &fakePTZMoveClient{
+		target:        target,
+		continuousErr: errors.New("continuous move rejected"),
+	}
+
+	err := moveToPosition(client, ptzMoveSupport{Continuous: true}, target, testPTZMoveOptions())
+	if err == nil || !strings.Contains(err.Error(), "continuous move rejected") {
+		t.Fatalf("moveToPosition() error = %v, want movement error", err)
+	}
+	if client.stopCalls != 1 {
+		t.Fatalf("stop calls = %d, want 1", client.stopCalls)
+	}
+}
+
+func TestMoveToPositionReturnsNoProgressErrorAndStops(t *testing.T) {
+	target := ptzTarget{Pan: 0.3, Tilt: 0.2, Zoom: 0.1}
+	client := &fakePTZMoveClient{
+		target:  target,
+		stalled: true,
+	}
+	options := testPTZMoveOptions()
+	options.NoProgressTimeout = 3 * time.Millisecond
+
+	err := moveToPosition(client, ptzMoveSupport{Continuous: true}, target, options)
+	if err == nil || !strings.Contains(err.Error(), "made no progress") {
+		t.Fatalf("moveToPosition() error = %v, want no-progress error", err)
+	}
+	if client.stopCalls != 1 {
+		t.Fatalf("stop calls = %d, want 1", client.stopCalls)
+	}
+}
+
+func TestMoveToPositionReturnsDeadlineErrorAndStops(t *testing.T) {
+	target := ptzTarget{Pan: 100, Tilt: 100, Zoom: 100}
+	client := &fakePTZMoveClient{target: target}
+	options := testPTZMoveOptions()
+	options.ContinuousTimeout = 3 * time.Millisecond
+	options.NoProgressTimeout = 50 * time.Millisecond
+
+	err := moveToPosition(client, ptzMoveSupport{Continuous: true}, target, options)
+	if err == nil || !strings.Contains(err.Error(), "target not reached within") {
+		t.Fatalf("moveToPosition() error = %v, want deadline error", err)
+	}
+	if client.stopCalls != 1 {
+		t.Fatalf("stop calls = %d, want 1", client.stopCalls)
+	}
+}
+
+func TestMoveToPositionRejectsIncompletePosition(t *testing.T) {
+	client := &fakePTZMoveClient{incompletePosition: true}
+
+	err := moveToPosition(client, ptzMoveSupport{Continuous: true}, ptzTarget{Pan: 0.1}, testPTZMoveOptions())
+	if err == nil || !strings.Contains(err.Error(), "incomplete PTZ position") {
+		t.Fatalf("moveToPosition() error = %v, want incomplete-position error", err)
+	}
+}
+
+func TestCombinedContinuousMoveVelocityIncludesEveryAxis(t *testing.T) {
+	request := ptz.ContinuousMove{
+		Velocity: ptz.Speed{
+			PanTilt: &xsdonvif.Vector2D{X: 0.5, Y: -0.25},
+			Zoom:    &xsdonvif.Vector1D{X: 0.2},
+		},
+	}
+
+	encoded, err := xml.Marshal(request)
+	if err != nil {
+		t.Fatalf("xml.Marshal() error = %v", err)
+	}
+	xmlBody := string(encoded)
+	if !strings.Contains(xmlBody, `PanTilt x="0.5" y="-0.25"`) {
+		t.Fatalf("ContinuousMove XML = %s, want pan and tilt", xmlBody)
+	}
+	if !strings.Contains(xmlBody, `Zoom x="0.2"`) {
+		t.Fatalf("ContinuousMove XML = %s, want zoom", xmlBody)
+	}
+}
+
+func TestGetPTZFunctionsHandlesMissingConfiguration(t *testing.T) {
+	functions, canZoom, canPanTilt := GetPTZFunctionsFromDevice(ptz.GetConfigurationsResponse{})
+	if len(functions) != 0 || canZoom || canPanTilt {
+		t.Fatalf("GetPTZFunctionsFromDevice() = %v, %t, %t; want empty capabilities", functions, canZoom, canPanTilt)
+	}
+}

machinery/src/onvif/subscription_test.go

@@ -0,0 +1,88 @@
+package onvif
+
+import (
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/kerberos-io/onvif"
+)
+
+// newSubscriptionTestDevice starts a fake ONVIF device and returns a connected
+// device plus the request bodies it received on the subscription endpoint.
+func newSubscriptionTestDevice(t *testing.T, response string) (*onvif.Device, string, func() []string) {
+	t.Helper()
+
+	var mu sync.Mutex
+	var subscriptionBodies []string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, _ := io.ReadAll(r.Body)
+		if r.URL.Path == "/onvif/subscription" {
+			mu.Lock()
+			subscriptionBodies = append(subscriptionBodies, string(body))
+			mu.Unlock()
+			_, _ = w.Write([]byte(response))
+			return
+		}
+		_, _ = w.Write([]byte(`<Envelope><Body/></Envelope>`))
+	}))
+	t.Cleanup(srv.Close)
+
+	dev, err := onvif.NewDevice(onvif.DeviceParams{Xaddr: strings.TrimPrefix(srv.URL, "http://")})
+	if err != nil {
+		t.Fatalf("NewDevice() error = %v", err)
+	}
+	subscriptionAddress := srv.URL + "/onvif/subscription?subscription=7&idx=2"
+	return dev, subscriptionAddress, func() []string {
+		mu.Lock()
+		defer mu.Unlock()
+		return append([]string(nil), subscriptionBodies...)
+	}
+}
+
+func assertSubscriptionAddressing(t *testing.T, body string, action string) {
+	t.Helper()
+
+	for _, want := range []string{
+		`<wsa:Action xmlns:wsa="http://www.w3.org/2005/08/addressing">` + action + `</wsa:Action>`,
+		`<wsa:To xmlns:wsa="http://www.w3.org/2005/08/addressing">`,
+		`/onvif/subscription?subscription=7&amp;idx=2</wsa:To>`,
+	} {
+		if !strings.Contains(body, want) {
+			t.Fatalf("request body does not contain %q:\n%s", want, body)
+		}
+	}
+}
+
+// Some devices identify a pull-point subscription by wsa:To and reject
+// PullMessages without it as NotAuthorized.
+func TestGetEventMessagesSendsSubscriptionAddressingHeaders(t *testing.T) {
+	dev, subscriptionAddress, bodies := newSubscriptionTestDevice(t, `<Envelope><Body><PullMessagesResponse/></Body></Envelope>`)
+
+	if _, err := GetEventMessages(dev, subscriptionAddress); err != nil {
+		t.Fatalf("GetEventMessages() error = %v", err)
+	}
+
+	received := bodies()
+	if len(received) != 1 {
+		t.Fatalf("subscription requests = %d, want 1", len(received))
+	}
+	assertSubscriptionAddressing(t, received[0], pullMessagesAction)
+}
+
+func TestUnsubscribePullPointSendsSubscriptionAddressingHeaders(t *testing.T) {
+	dev, subscriptionAddress, bodies := newSubscriptionTestDevice(t, `<Envelope><Body><UnsubscribeResponse/></Body></Envelope>`)
+
+	if err := UnsubscribePullPoint(dev, subscriptionAddress); err != nil {
+		t.Fatalf("UnsubscribePullPoint() error = %v", err)
+	}
+
+	received := bodies()
+	if len(received) != 1 {
+		t.Fatalf("subscription requests = %d, want 1", len(received))
+	}
+	assertSubscriptionAddressing(t, received[0], unsubscribeAction)
+}

machinery/src/outputs/main.go

@@ -1,8 +1,8 @@
 package outputs
 
 import (
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 type Output interface {
@@ -20,36 +20,36 @@ func Execute(message *models.OutputMessage) (err error) {
 			slack := &SlackOutput{}
 			err := slack.Trigger(message)
 			if err == nil {
-				log.Log.Debug("outputs.main.Execute(slack): message was processed by output.")
+				log.Debug("outputs.main.Execute(slack): message was processed by output.")
 			} else {
-				log.Log.Error("outputs.main.Execute(slack): " + err.Error())
+				log.Error("outputs.main.Execute(slack): " + err.Error())
 			}
 			break
 		case "webhook":
 			webhook := &WebhookOutput{}
 			err := webhook.Trigger(message)
 			if err == nil {
-				log.Log.Debug("outputs.main.Execute(webhook): message was processed by output.")
+				log.Debug("outputs.main.Execute(webhook): message was processed by output.")
 			} else {
-				log.Log.Error("outputs.main.Execute(webhook): " + err.Error())
+				log.Error("outputs.main.Execute(webhook): " + err.Error())
 			}
 			break
 		case "onvif_relay":
 			onvif := &OnvifRelayOutput{}
 			err := onvif.Trigger(message)
 			if err == nil {
-				log.Log.Debug("outputs.main.Execute(onvif): message was processed by output.")
+				log.Debug("outputs.main.Execute(onvif): message was processed by output.")
 			} else {
-				log.Log.Error("outputs.main.Execute(onvif): " + err.Error())
+				log.Error("outputs.main.Execute(onvif): " + err.Error())
 			}
 			break
 		case "script":
 			script := &ScriptOutput{}
 			err := script.Trigger(message)
 			if err == nil {
-				log.Log.Debug("outputs.main.Execute(script): message was processed by output.")
+				log.Debug("outputs.main.Execute(script): message was processed by output.")
 			} else {
-				log.Log.Error("outputs.main.Execute(script): " + err.Error())
+				log.Error("outputs.main.Execute(script): " + err.Error())
 			}
 			break
 		}

machinery/src/packets/queue.go

@@ -2,6 +2,7 @@
 package packets
 
 import (
+	"context"
 	"io"
 	"sync"
 )
@@ -39,6 +40,9 @@ func NewQueue() *Queue {
 }
 
 func (self *Queue) SetMaxGopCount(n int) {
+	if n < 1 {
+		n = 1
+	}
 	self.lock.Lock()
 	self.maxgopcount = n
 	self.lock.Unlock()
@@ -96,20 +100,31 @@ func (self *Queue) WritePacket(pkt Packet) (err error) {
 	self.lock.Lock()
 
 	self.buf.Push(pkt)
-	if pkt.Idx == int8(self.videoidx) && pkt.IsKeyFrame {
+	isVideoKeyframe := pkt.Idx == int8(self.videoidx) && pkt.IsKeyFrame
+	if isVideoKeyframe {
 		self.curgopcount++
-	}
 
-	for self.curgopcount >= self.maxgopcount && self.buf.Count > 1 {
-		pkt := self.buf.Pop()
-		if pkt.Idx == int8(self.videoidx) && pkt.IsKeyFrame {
-			self.curgopcount--
-		}
-		if self.curgopcount < self.maxgopcount {
-			break
+		for self.curgopcount > self.maxgopcount && self.buf.Count > 0 {
+			for self.buf.Count > 0 {
+				oldest := self.buf.Pop()
+				if oldest.Idx == int8(self.videoidx) && oldest.IsKeyFrame {
+					self.curgopcount--
+					break
+				}
+			}
+
+			// Discard the dependent frames from the evicted GOP as well. Keeping
+			// them would let an overrun cursor resume on a P-frame whose reference
+			// keyframe is no longer in the queue.
+			for self.buf.Count > 0 {
+				oldest := self.buf.Get(self.buf.Head)
+				if oldest.Idx == int8(self.videoidx) && oldest.IsKeyFrame {
+					break
+				}
+				self.buf.Pop()
+			}
 		}
 	}
-	//println("shrink", self.curgopcount, self.maxgopcount, self.buf.Head, self.buf.Tail, "count", self.buf.Count, "size", self.buf.Size)
 
 	self.cond.Broadcast()
 
@@ -201,7 +216,26 @@ func (self *QueueCursor) Streams() (streams []Stream, err error) {
 
 // ReadPacket will not consume packets in Queue, it's just a cursor.
 func (self *QueueCursor) ReadPacket() (pkt Packet, err error) {
+	return self.readPacket(context.Background(), false)
+}
+
+// ReadPacketContext waits for the next packet or returns when ctx is cancelled.
+func (self *QueueCursor) ReadPacketContext(ctx context.Context) (pkt Packet, err error) {
+	if err := ctx.Err(); err != nil {
+		return Packet{}, err
+	}
+	return self.readPacket(ctx, true)
+}
+
+func (self *QueueCursor) readPacket(ctx context.Context, cancellable bool) (pkt Packet, err error) {
 	self.que.cond.L.Lock()
+	var stop func() bool
+	defer func() {
+		self.que.cond.L.Unlock()
+		if stop != nil {
+			stop()
+		}
+	}()
 	buf := self.que.buf
 	if !self.gotpos {
 		self.pos = self.init(buf, self.que.videoidx)
@@ -222,8 +256,21 @@ func (self *QueueCursor) ReadPacket() (pkt Packet, err error) {
 			err = io.EOF
 			break
 		}
+		if cancellable {
+			if err = ctx.Err(); err != nil {
+				break
+			}
+			if stop == nil {
+				stop = context.AfterFunc(ctx, func() {
+					// The write lock prevents cancellation from being signalled
+					// between the context check above and cond.Wait.
+					self.que.lock.Lock()
+					self.que.cond.Broadcast()
+					self.que.lock.Unlock()
+				})
+			}
+		}
 		self.que.cond.Wait()
 	}
-	self.que.cond.L.Unlock()
 	return
 }

machinery/src/packets/queue_test.go

@@ -0,0 +1,208 @@
+package packets
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+type observedCancelContext struct {
+	done        chan struct{}
+	cancelled   atomic.Bool
+	errCalls    atomic.Int32
+	waiting     chan struct{}
+	waitingOnce sync.Once
+}
+
+func newObservedCancelContext() *observedCancelContext {
+	return &observedCancelContext{
+		done:    make(chan struct{}),
+		waiting: make(chan struct{}),
+	}
+}
+
+func (c *observedCancelContext) Deadline() (time.Time, bool) { return time.Time{}, false }
+func (c *observedCancelContext) Done() <-chan struct{}       { return c.done }
+func (c *observedCancelContext) Value(any) any               { return nil }
+
+func (c *observedCancelContext) Err() error {
+	if c.errCalls.Add(1) >= 2 {
+		c.waitingOnce.Do(func() { close(c.waiting) })
+	}
+	if c.cancelled.Load() {
+		return context.Canceled
+	}
+	return nil
+}
+
+func (c *observedCancelContext) Cancel() {
+	c.cancelled.Store(true)
+	close(c.done)
+}
+
+func TestQueueCursorReadPacketContextCancelsWhileQueueIsIdle(t *testing.T) {
+	queue := NewQueue()
+	cursor := queue.Latest()
+	ctx := newObservedCancelContext()
+	done := make(chan error, 1)
+
+	go func() {
+		_, err := cursor.ReadPacketContext(ctx)
+		done <- err
+	}()
+
+	select {
+	case <-ctx.waiting:
+	case <-time.After(time.Second):
+		t.Fatal("ReadPacketContext() did not reach the queue wait")
+	}
+	ctx.Cancel()
+	select {
+	case err := <-done:
+		if !errors.Is(err, context.Canceled) {
+			t.Fatalf("ReadPacketContext() error = %v, want %v", err, context.Canceled)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("ReadPacketContext() did not return after cancellation")
+	}
+}
+
+func TestQueueCursorReadPacketDoesNotAllocate(t *testing.T) {
+	queue := NewQueue()
+	for i := 0; i < 200; i++ {
+		if err := queue.WritePacket(Packet{Data: []byte{1}}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	cursor := queue.Oldest()
+
+	allocations := testing.AllocsPerRun(100, func() {
+		if _, err := cursor.ReadPacket(); err != nil {
+			t.Fatal(err)
+		}
+	})
+	if allocations != 0 {
+		t.Fatalf("ReadPacket() allocations = %.2f, want 0", allocations)
+	}
+}
+
+func TestQueueCursorReadPacketContextDoesNotAllocateWhenPacketIsAvailable(t *testing.T) {
+	queue := NewQueue()
+	for i := 0; i < 200; i++ {
+		if err := queue.WritePacket(Packet{Data: []byte{1}}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	cursor := queue.Oldest()
+
+	allocations := testing.AllocsPerRun(100, func() {
+		if _, err := cursor.ReadPacketContext(context.Background()); err != nil {
+			t.Fatal(err)
+		}
+	})
+	if allocations != 0 {
+		t.Fatalf("ReadPacketContext() allocations = %.2f, want 0", allocations)
+	}
+}
+
+func TestQueueEvictionKeepsKeyframeAtHead(t *testing.T) {
+	queue := NewQueue()
+	if err := queue.WriteHeader([]Stream{{Index: 0, IsVideo: true}}); err != nil {
+		t.Fatal(err)
+	}
+	queue.SetMaxGopCount(2)
+
+	for _, packet := range []Packet{
+		{Idx: 0, IsVideo: true, IsKeyFrame: true, Data: []byte("IDR-0")},
+		{Idx: 1, IsAudio: true, Data: []byte("audio-0")},
+		{Idx: 0, IsVideo: true, Data: []byte("P-0")},
+		{Idx: 0, IsVideo: true, IsKeyFrame: true, Data: []byte("IDR-1")},
+		{Idx: 0, IsVideo: true, Data: []byte("P-1")},
+		{Idx: 0, IsVideo: true, IsKeyFrame: true, Data: []byte("IDR-2")},
+	} {
+		if err := queue.WritePacket(packet); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	oldest, err := queue.Oldest().ReadPacket()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !oldest.IsKeyFrame || string(oldest.Data) != "IDR-1" {
+		t.Fatalf("oldest packet = %q (keyframe=%t), want IDR-1 keyframe", oldest.Data, oldest.IsKeyFrame)
+	}
+}
+
+func TestQueueWithSingleGOPRetainsCurrentKeyframe(t *testing.T) {
+	queue := NewQueue()
+	if err := queue.WriteHeader([]Stream{{Index: 0, IsVideo: true}}); err != nil {
+		t.Fatal(err)
+	}
+	queue.SetMaxGopCount(1)
+
+	for _, packet := range []Packet{
+		{Idx: 0, IsVideo: true, IsKeyFrame: true, Data: []byte("IDR-0")},
+		{Idx: 0, IsVideo: true, Data: []byte("P-0")},
+		{Idx: 0, IsVideo: true, IsKeyFrame: true, Data: []byte("IDR-1")},
+	} {
+		if err := queue.WritePacket(packet); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	oldest, err := queue.Oldest().ReadPacket()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !oldest.IsKeyFrame || string(oldest.Data) != "IDR-1" {
+		t.Fatalf("oldest packet = %q (keyframe=%t), want IDR-1 keyframe", oldest.Data, oldest.IsKeyFrame)
+	}
+}
+
+func TestLaggingQueueCursorResumesAtKeyframe(t *testing.T) {
+	queue := NewQueue()
+	if err := queue.WriteHeader([]Stream{{Index: 0, IsVideo: true}}); err != nil {
+		t.Fatal(err)
+	}
+	queue.SetMaxGopCount(2)
+
+	for _, packet := range []Packet{
+		{Idx: 0, IsVideo: true, IsKeyFrame: true, Data: []byte("IDR-0")},
+		{Idx: 0, IsVideo: true, Data: []byte("P-0")},
+		{Idx: 0, IsVideo: true, IsKeyFrame: true, Data: []byte("IDR-1")},
+	} {
+		if err := queue.WritePacket(packet); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	cursor := queue.Oldest()
+	first, err := cursor.ReadPacket()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(first.Data) != "IDR-0" {
+		t.Fatalf("first packet = %q, want IDR-0", first.Data)
+	}
+
+	for _, packet := range []Packet{
+		{Idx: 0, IsVideo: true, Data: []byte("P-1")},
+		{Idx: 0, IsVideo: true, IsKeyFrame: true, Data: []byte("IDR-2")},
+	} {
+		if err := queue.WritePacket(packet); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	resumed, err := cursor.ReadPacket()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !resumed.IsKeyFrame || string(resumed.Data) != "IDR-1" {
+		t.Fatalf("resumed packet = %q (keyframe=%t), want IDR-1 keyframe", resumed.Data, resumed.IsKeyFrame)
+	}
+}

machinery/src/routers/http/health.go

@@ -0,0 +1,126 @@
+package http
+
+import (
+	"math"
+	stdhttp "net/http"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/utils"
+)
+
+const (
+	healthApplicationStatus = "get_success"
+	healthEntityStatus      = "healthy"
+	hubConnectionFreshness  = 3 * time.Minute
+)
+
+// StreamResolution is the most recently observed encoded video size in pixels.
+type StreamResolution struct {
+	Width  int64 `json:"width"`
+	Height int64 `json:"height"`
+}
+
+// StreamHealth describes the most recently observed state of one camera stream.
+type StreamHealth struct {
+	Configured        bool             `json:"configured"`
+	Connected         bool             `json:"connected"`
+	PackagesProcessed uint64           `json:"packagesProcessed"`
+	FPS               float64          `json:"fps"`
+	Resolution        StreamResolution `json:"resolution"`
+	LastPacketAt      int64            `json:"lastPacketAt"`
+}
+
+// HubHealth describes the Agent's periodic HTTP heartbeat connection to Hub.
+type HubHealth struct {
+	Configured                bool  `json:"configured"`
+	Connected                 bool  `json:"connected"`
+	LastHeartbeatAttemptAt    int64 `json:"lastHeartbeatAttemptAt"`
+	LastSuccessfulHeartbeatAt int64 `json:"lastSuccessfulHeartbeatAt"`
+}
+
+// Health describes the Agent process health exposed to API clients.
+type Health struct {
+	Description     string       `json:"description"`
+	CameraConnected bool         `json:"cameraConnected"`
+	MainStream      StreamHealth `json:"mainStream"`
+	SubStream       StreamHealth `json:"subStream"`
+	Hub             HubHealth    `json:"hub"`
+}
+
+// HealthResponseData contains the typed payload of a health response.
+type HealthResponseData struct {
+	Health Health `json:"health"`
+}
+
+// HealthResponse is the standard successful response returned by GET /health.
+type HealthResponse struct {
+	SuccessResponse
+	Data HealthResponseData `json:"data"`
+}
+
+// HealthCheck godoc
+// @Summary Check Agent health
+// @Description Confirms that the Agent HTTP process can serve requests and reports current camera stream and Hub heartbeat diagnostics. Operational dependency failures do not change the liveness HTTP status.
+// @Tags health
+// @Produce json
+// @Success 200 {object} HealthResponse
+// @Router /health [get]
+func HealthCheck(c *gin.Context, communication *models.Communication) {
+	c.Header("Cache-Control", "no-store")
+	c.JSON(stdhttp.StatusOK, HealthResponse{
+		SuccessResponse: NewSuccessResponse(
+			stdhttp.StatusOK,
+			healthApplicationStatus,
+			healthEntityStatus,
+			"Healthy",
+			ResponseMetadata{
+				ApplicationName:    "agent",
+				ApplicationVersion: utils.VERSION,
+				Path:               c.Request.URL.Path,
+			},
+		),
+		Data: HealthResponseData{
+			Health: buildHealth(communication, time.Now()),
+		},
+	})
+}
+
+func buildHealth(communication *models.Communication, now time.Time) Health {
+	mainTelemetry := communication.StreamRuntimeTelemetry(models.MainStream)
+	subTelemetry := communication.StreamRuntimeTelemetry(models.SubStream)
+	hubTelemetry := communication.HubRuntimeTelemetry()
+
+	hubConnected := hubTelemetry.Configured &&
+		hubTelemetry.Connected &&
+		hubTelemetry.LastSuccessfulHeartbeatAt > 0 &&
+		now.Unix()-hubTelemetry.LastSuccessfulHeartbeatAt <= int64(hubConnectionFreshness/time.Second)
+
+	return Health{
+		Description:     "Agent HTTP service is healthy",
+		CameraConnected: communication.CameraConnected.Load(),
+		MainStream:      newStreamHealth(mainTelemetry, communication.MainStreamConnected.Load()),
+		SubStream:       newStreamHealth(subTelemetry, communication.SubStreamConnected.Load()),
+		Hub: HubHealth{
+			Configured:                hubTelemetry.Configured,
+			Connected:                 hubConnected,
+			LastHeartbeatAttemptAt:    hubTelemetry.LastHeartbeatAttemptAt,
+			LastSuccessfulHeartbeatAt: hubTelemetry.LastSuccessfulHeartbeatAt,
+		},
+	}
+}
+
+func newStreamHealth(telemetry models.StreamRuntimeTelemetry, connected bool) StreamHealth {
+	return StreamHealth{
+		Configured:        telemetry.Configured,
+		Connected:         connected,
+		PackagesProcessed: telemetry.PackagesProcessed,
+		FPS:               math.Round(telemetry.FPS*100) / 100,
+		Resolution: StreamResolution{
+			Width:  telemetry.Width,
+			Height: telemetry.Height,
+		},
+		LastPacketAt: telemetry.LastPacketAt,
+	}
+}

machinery/src/routers/http/health_test.go

@@ -0,0 +1,117 @@
+package http
+
+import (
+	"encoding/json"
+	stdhttp "net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	jwt "github.com/appleboy/gin-jwt/v2"
+	"github.com/gin-gonic/gin"
+	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/utils"
+)
+
+func TestHealthCheckReturnsStandardPublicResponse(t *testing.T) {
+	gin.SetMode(gin.TestMode)
+
+	middlewareConfig := JWTMiddleWare()
+	authMiddleware, err := jwt.New(&middlewareConfig)
+	if err != nil {
+		t.Fatalf("initialize JWT middleware: %v", err)
+	}
+
+	now := time.Now()
+	communication := &models.Communication{}
+	communication.SetStreamConfigured(models.MainStream, true)
+	communication.SetStreamConfigured(models.SubStream, true)
+	communication.RecordStreamPackage(models.MainStream, 29.969, 1920, 1080, now.Add(-2*time.Second))
+	communication.RecordStreamPackage(models.MainStream, 29.969, 1920, 1080, now.Add(-time.Second))
+	communication.RecordStreamPackage(models.SubStream, 15, 640, 360, now.Add(-time.Second))
+	communication.CameraConnected.Store(true)
+	communication.MainStreamConnected.Store(true)
+	communication.SubStreamConnected.Store(true)
+	communication.SetHubConfigured(true)
+	communication.RecordHubHeartbeatAttempt(now.Add(-2 * time.Second))
+	communication.RecordHubHeartbeatSuccess(now.Add(-time.Second))
+
+	router := gin.New()
+	AddRoutes(router, authMiddleware, "", nil, communication, nil)
+
+	response := httptest.NewRecorder()
+	request := httptest.NewRequest(stdhttp.MethodGet, "/health", nil)
+	router.ServeHTTP(response, request)
+
+	if response.Code != stdhttp.StatusOK {
+		t.Fatalf("status = %d, want %d; body=%s", response.Code, stdhttp.StatusOK, response.Body.String())
+	}
+	if got := response.Header().Get("Cache-Control"); got != "no-store" {
+		t.Fatalf("Cache-Control = %q, want %q", got, "no-store")
+	}
+
+	var body HealthResponse
+	if err := json.Unmarshal(response.Body.Bytes(), &body); err != nil {
+		t.Fatalf("decode health response: %v; body=%s", err, response.Body.String())
+	}
+
+	if body.HTTPStatusCode != stdhttp.StatusOK {
+		t.Errorf("httpStatusCode = %d, want %d", body.HTTPStatusCode, stdhttp.StatusOK)
+	}
+	if body.ApplicationStatusCode != healthApplicationStatus {
+		t.Errorf("applicationStatusCode = %q, want %q", body.ApplicationStatusCode, healthApplicationStatus)
+	}
+	if body.EntityStatusCode != healthEntityStatus {
+		t.Errorf("entityStatusCode = %q, want %q", body.EntityStatusCode, healthEntityStatus)
+	}
+	if body.Message != "Healthy" {
+		t.Errorf("message = %q, want %q", body.Message, "Healthy")
+	}
+	if body.Metadata.ApplicationName != "agent" {
+		t.Errorf("metadata.applicationName = %q, want %q", body.Metadata.ApplicationName, "agent")
+	}
+	if body.Metadata.ApplicationVersion != utils.VERSION {
+		t.Errorf("metadata.applicationVersion = %q, want %q", body.Metadata.ApplicationVersion, utils.VERSION)
+	}
+	if body.Metadata.Timestamp <= 0 {
+		t.Errorf("metadata.timestamp = %d, want a positive Unix timestamp", body.Metadata.Timestamp)
+	}
+	if body.Metadata.Path != "/health" {
+		t.Errorf("metadata.path = %q, want %q", body.Metadata.Path, "/health")
+	}
+	health := body.Data.Health
+	if health.Description != "Agent HTTP service is healthy" {
+		t.Errorf("data.health.description = %q, want %q", health.Description, "Agent HTTP service is healthy")
+	}
+	if !health.CameraConnected {
+		t.Error("data.health.cameraConnected = false, want true")
+	}
+	if health.MainStream.PackagesProcessed != 2 || health.MainStream.FPS != 29.97 {
+		t.Errorf("data.health.mainStream = %+v, want 2 packages at 29.97 FPS", health.MainStream)
+	}
+	if health.MainStream.Resolution != (StreamResolution{Width: 1920, Height: 1080}) {
+		t.Errorf("data.health.mainStream.resolution = %+v", health.MainStream.Resolution)
+	}
+	if health.SubStream.PackagesProcessed != 1 || health.SubStream.FPS != 15 {
+		t.Errorf("data.health.subStream = %+v, want 1 package at 15 FPS", health.SubStream)
+	}
+	if health.SubStream.Resolution != (StreamResolution{Width: 640, Height: 360}) {
+		t.Errorf("data.health.subStream.resolution = %+v", health.SubStream.Resolution)
+	}
+	if !health.Hub.Configured || !health.Hub.Connected {
+		t.Errorf("data.health.hub = %+v, want configured and connected", health.Hub)
+	}
+}
+
+func TestBuildHealthMarksStaleHubHeartbeatDisconnected(t *testing.T) {
+	now := time.Unix(1_788_710_500, 0)
+	communication := &models.Communication{}
+	communication.SetHubConfigured(true)
+	communication.RecordHubHeartbeatAttempt(now.Add(-4 * time.Minute))
+	communication.RecordHubHeartbeatSuccess(now.Add(-4 * time.Minute))
+
+	health := buildHealth(communication, now)
+	if health.Hub.Connected {
+		t.Fatalf("Hub health = %+v, want stale heartbeat to be disconnected", health.Hub)
+	}
+}

machinery/src/routers/http/methods.go

@@ -6,9 +6,9 @@ import (
 	"time"
 
 	"github.com/gin-gonic/gin"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/onvif"
+	log "github.com/sirupsen/logrus"
 )
 
 // Login godoc
@@ -598,27 +598,27 @@ func DoTriggerRelayOutput(c *gin.Context) {
 			err := onvif.TriggerRelayOutput(device, output)
 			if err == nil {
 				msg := "relay output triggered: " + output
-				log.Log.Info("routers.http.methods.DoTriggerRelayOutput(): " + msg)
+				log.Info("routers.http.methods.DoTriggerRelayOutput(): " + msg)
 				c.JSON(200, gin.H{
 					"data": msg,
 				})
 			} else {
 				msg := "something went wrong: " + err.Error()
-				log.Log.Error("routers.http.methods.DoTriggerRelayOutput(): " + msg)
+				log.Error("routers.http.methods.DoTriggerRelayOutput(): " + msg)
 				c.JSON(400, gin.H{
 					"data": msg,
 				})
 			}
 		} else {
 			msg := "something went wrong: " + err.Error()
-			log.Log.Error("routers.http.methods.DoTriggerRelayOutput(): " + msg)
+			log.Error("routers.http.methods.DoTriggerRelayOutput(): " + msg)
 			c.JSON(400, gin.H{
 				"data": msg,
 			})
 		}
 	} else {
 		msg := "something went wrong: " + err.Error()
-		log.Log.Error("routers.http.methods.DoTriggerRelayOutput(): " + msg)
+		log.Error("routers.http.methods.DoTriggerRelayOutput(): " + msg)
 		c.JSON(400, gin.H{
 			"data": msg,
 		})

machinery/src/routers/http/recordings_test.go

@@ -0,0 +1,81 @@
+package http
+
+import (
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/gin-gonic/gin"
+	"github.com/kerberos-io/agent/machinery/src/encryption"
+	"github.com/kerberos-io/agent/machinery/src/models"
+)
+
+func TestFilesServesFlatAndDatedRecordings(t *testing.T) {
+	gin.SetMode(gin.TestMode)
+	for _, layout := range []string{"flat", "dated"} {
+		for _, encrypted := range []bool{false, true} {
+			name := layout + "-plain"
+			if encrypted {
+				name = layout + "-encrypted"
+			}
+			t.Run(name, func(t *testing.T) {
+				const fileName = "1767355200123-device-key.mp4"
+				directory := t.TempDir()
+				relativePath := fileName
+				if layout == "dated" {
+					relativePath = "2026/01/02/" + fileName
+				}
+				filePath := filepath.Join(directory, "data/recordings", relativePath)
+				if err := os.MkdirAll(filepath.Dir(filePath), 0o755); err != nil {
+					t.Fatal(err)
+				}
+				config := &models.Configuration{Config: models.Config{Encryption: &models.Encryption{}}}
+				content := []byte("recording-payload")
+				if encrypted {
+					config.Config.Encryption.Recordings = "true"
+					config.Config.Encryption.SymmetricKey = "0123456789abcdef0123456789abcdef"
+					ciphertext, err := encryption.AesEncrypt(content, config.Config.Encryption.SymmetricKey)
+					if err != nil {
+						t.Fatal(err)
+					}
+					content = []byte(ciphertext)
+				}
+				if err := os.WriteFile(filePath, content, 0o644); err != nil {
+					t.Fatal(err)
+				}
+				router := gin.New()
+				router.GET("/file/*filepath", func(c *gin.Context) { Files(c, directory, config) })
+				for _, reference := range []string{relativePath, fileName} {
+					response := httptest.NewRecorder()
+					router.ServeHTTP(response, httptest.NewRequest("GET", "/file/"+reference, nil))
+					if response.Code != 200 || response.Body.String() != "recording-payload" {
+						t.Fatalf("GET %s = %d %q", reference, response.Code, response.Body.String())
+					}
+					if got := response.Header().Get("Content-Type"); got != "video/mp4" {
+						t.Fatalf("Content-Type = %q, want video/mp4", got)
+					}
+				}
+			})
+		}
+	}
+}
+
+func TestFilesRejectsMissingAndOutsideRecordings(t *testing.T) {
+	directory := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(directory, "data/recordings"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(directory, "data/outside.mp4"), []byte("not a recording"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	router := gin.New()
+	router.GET("/file/*filepath", func(c *gin.Context) { Files(c, directory, &models.Configuration{}) })
+	for _, reference := range []string{"missing.mp4", "../outside.mp4", "2026/01/02/missing.mp4"} {
+		response := httptest.NewRecorder()
+		router.ServeHTTP(response, httptest.NewRequest("GET", "/file/"+reference, nil))
+		if response.Code != 404 {
+			t.Fatalf("GET %s = %d, want 404", reference, response.Code)
+		}
+	}
+}

machinery/src/routers/http/response.go

@@ -0,0 +1,41 @@
+package http
+
+import "time"
+
+// ResponseMetadata contains request and application context shared by API responses.
+type ResponseMetadata struct {
+	ApplicationName    string `json:"applicationName,omitempty"`
+	ApplicationVersion string `json:"applicationVersion,omitempty"`
+	Timestamp          int64  `json:"timestamp,omitempty"`
+	Path               string `json:"path,omitempty"`
+	TraceID            string `json:"traceId,omitempty"`
+}
+
+// SuccessResponse is the common envelope for successful Agent API responses.
+type SuccessResponse struct {
+	HTTPStatusCode        int              `json:"httpStatusCode"`
+	ApplicationStatusCode string           `json:"applicationStatusCode"`
+	EntityStatusCode      string           `json:"entityStatusCode"`
+	Message               string           `json:"message"`
+	Metadata              ResponseMetadata `json:"metadata"`
+}
+
+// NewSuccessResponse creates the Hub-compatible success envelope used by new
+// Agent API endpoints.
+func NewSuccessResponse(
+	httpStatusCode int,
+	applicationStatusCode string,
+	entityStatusCode string,
+	message string,
+	metadata ResponseMetadata,
+) SuccessResponse {
+	metadata.Timestamp = time.Now().Unix()
+
+	return SuccessResponse{
+		HTTPStatusCode:        httpStatusCode,
+		ApplicationStatusCode: applicationStatusCode,
+		EntityStatusCode:      entityStatusCode,
+		Message:               message,
+		Metadata:              metadata,
+	}
+}

machinery/src/routers/http/routes.go

@@ -14,6 +14,10 @@ import (
 
 func AddRoutes(r *gin.Engine, authMiddleware *jwt.GinJWTMiddleware, configDirectory string, configuration *models.Configuration, communication *models.Communication, captureDevice *capture.Capture) *gin.RouterGroup {
 
+	r.GET("/health", func(c *gin.Context) {
+		HealthCheck(c, communication)
+	})
+
 	r.GET("/ws", func(c *gin.Context) {
 		websocket.WebsocketHandler(c, configuration, communication, captureDevice)
 	})
@@ -61,12 +65,12 @@ func AddRoutes(r *gin.Engine, authMiddleware *jwt.GinJWTMiddleware, configDirect
 
 			// Will verify the hub settings.
 			api.POST("/hub/verify", func(c *gin.Context) {
-				cloud.VerifyHub(c)
+				cloud.VerifyHub(c, configuration.Config.HubURI)
 			})
 
 			// Will verify the persistence settings.
 			api.POST("/persistence/verify", func(c *gin.Context) {
-				cloud.VerifyPersistence(c, configDirectory)
+				cloud.VerifyPersistence(c, configDirectory, configuration.Config.HubURI)
 			})
 
 			// Will verify the secondary persistence settings.

machinery/src/routers/http/server.go

@@ -1,9 +1,12 @@
 package http
 
 import (
+	"fmt"
 	"io"
 	"os"
 	"strconv"
+	"strings"
+	"time"
 
 	jwt "github.com/appleboy/gin-jwt/v2"
 	"github.com/gin-contrib/pprof"
@@ -11,12 +14,13 @@ import (
 	"github.com/gin-gonic/gin"
 
 	//Swagger documentantion
-	"log"
+	log "github.com/sirupsen/logrus"
 
 	_ "github.com/kerberos-io/agent/machinery/docs"
 	"github.com/kerberos-io/agent/machinery/src/capture"
 	"github.com/kerberos-io/agent/machinery/src/encryption"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/utils"
 	swaggerFiles "github.com/swaggo/files"
 	ginSwagger "github.com/swaggo/gin-swagger"
 )
@@ -45,7 +49,17 @@ func StartServer(configDirectory string, configuration *models.Configuration, co
 	gin.SetMode(gin.ReleaseMode)
 
 	// Initialize REST API
-	r := gin.Default()
+	r := gin.New()
+	r.Use(requestLogger(), gin.CustomRecovery(func(c *gin.Context, recovered interface{}) {
+		log.WithFields(log.Fields{
+			"component": "http",
+			"event":     "request_panic",
+			"method":    c.Request.Method,
+			"path":      c.Request.URL.Path,
+			"panic":     fmt.Sprint(recovered),
+		}).Error("HTTP request panicked")
+		c.AbortWithStatus(500)
+	}))
 
 	// Profiler
 	pprof.Register(r)
@@ -60,7 +74,10 @@ func StartServer(configDirectory string, configuration *models.Configuration, co
 	middleWare := JWTMiddleWare()
 	authMiddleware, err := jwt.New(&middleWare)
 	if err != nil {
-		log.Fatal("JWT Error:" + err.Error())
+		log.WithError(err).WithFields(log.Fields{
+			"component": "http",
+			"event":     "jwt_initialization_failed",
+		}).Fatal("Failed to initialize JWT middleware")
 	}
 
 	// Add all routes
@@ -73,7 +90,10 @@ func StartServer(configDirectory string, configuration *models.Configuration, co
 		// Move demo environment variables to environment variables
 		err := os.Rename(demoEnvironmentVariables, environmentVariables)
 		if err != nil {
-			log.Fatal(err)
+			log.WithError(err).WithFields(log.Fields{
+				"component": "http",
+				"event":     "demo_environment_move_failed",
+			}).Fatal("Failed to activate demo UI environment")
 		}
 	}
 
@@ -88,18 +108,63 @@ func StartServer(configDirectory string, configuration *models.Configuration, co
 	})
 
 	// Run the api on port
+	log.WithFields(log.Fields{
+		"component": "http",
+		"event":     "server_starting",
+		"port":      configuration.Port,
+	}).Info("HTTP server starting")
 	err = r.Run(":" + configuration.Port)
 	if err != nil {
-		log.Fatal(err)
+		log.WithError(err).WithFields(log.Fields{
+			"component": "http",
+			"event":     "server_failed",
+			"port":      configuration.Port,
+		}).Fatal("HTTP server stopped")
+	}
+}
+
+func requestLogger() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		startedAt := time.Now()
+		c.Next()
+
+		fields := log.Fields{
+			"component":      "http",
+			"duration_ms":    time.Since(startedAt).Milliseconds(),
+			"event":          "request_completed",
+			"method":         c.Request.Method,
+			"path":           c.Request.URL.Path,
+			"response_bytes": c.Writer.Size(),
+			"status":         c.Writer.Status(),
+		}
+		if log.IsLevelEnabled(log.DebugLevel) {
+			fields["client_ip"] = c.ClientIP()
+			fields["user_agent"] = c.Request.UserAgent()
+		}
+		entry := log.WithFields(fields)
+		if len(c.Errors) > 0 {
+			entry = entry.WithField("request_errors", c.Errors.String())
+		}
+
+		switch status := c.Writer.Status(); {
+		case status >= 500:
+			entry.Error("HTTP request completed")
+		case status >= 400:
+			entry.Warn("HTTP request completed")
+		case c.Request.URL.Path == "/health":
+			entry.Debug("HTTP request completed")
+		default:
+			entry.Info("HTTP request completed")
+		}
 	}
 }
 
 func Files(c *gin.Context, configDirectory string, configuration *models.Configuration) {
 
 	// Get File
-	filePath := configDirectory + "/data/recordings" + c.Param("filepath")
-	_, err := os.Open(filePath)
+	filePath, err := utils.ResolveRecordingPath(configDirectory+"/data/recordings", strings.TrimPrefix(c.Param("filepath"), "/"))
 	if err != nil {
+		log.WithError(err).Warn("Could not resolve recording for playback")
 		c.JSON(404, gin.H{"error": "File not found"})
 		return
 	}

machinery/src/routers/http/server_test.go

@@ -0,0 +1,89 @@
+package http
+
+import (
+	"bytes"
+	"encoding/json"
+	stdhttp "net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/gin-gonic/gin"
+	log "github.com/sirupsen/logrus"
+)
+
+func TestRequestLoggerEmitsStructuredFields(t *testing.T) {
+	originalOutput := log.StandardLogger().Out
+	originalFormatter := log.StandardLogger().Formatter
+	originalLevel := log.GetLevel()
+	defer func() {
+		log.SetOutput(originalOutput)
+		log.SetFormatter(originalFormatter)
+		log.SetLevel(originalLevel)
+	}()
+
+	var output bytes.Buffer
+	log.SetOutput(&output)
+	log.SetFormatter(&log.JSONFormatter{})
+	log.SetLevel(log.InfoLevel)
+	gin.SetMode(gin.TestMode)
+
+	router := gin.New()
+	router.Use(requestLogger())
+	router.GET("/status", func(c *gin.Context) {
+		c.Status(stdhttp.StatusNoContent)
+	})
+
+	response := httptest.NewRecorder()
+	request := httptest.NewRequest(stdhttp.MethodGet, "/status?token=do-not-log", nil)
+	router.ServeHTTP(response, request)
+
+	var entry map[string]interface{}
+	if err := json.Unmarshal(output.Bytes(), &entry); err != nil {
+		t.Fatalf("decode request log: %v; output=%q", err, output.String())
+	}
+	for key, want := range map[string]interface{}{
+		"component": "http",
+		"event":     "request_completed",
+		"method":    stdhttp.MethodGet,
+		"path":      "/status",
+		"status":    float64(stdhttp.StatusNoContent),
+	} {
+		if got := entry[key]; got != want {
+			t.Fatalf("%s = %v, want %v", key, got, want)
+		}
+	}
+	if bytes.Contains(output.Bytes(), []byte("do-not-log")) {
+		t.Fatalf("request log exposed query parameters: %q", output.String())
+	}
+}
+
+func TestRequestLoggerKeepsSuccessfulHealthChecksBelowInfo(t *testing.T) {
+	originalOutput := log.StandardLogger().Out
+	originalFormatter := log.StandardLogger().Formatter
+	originalLevel := log.GetLevel()
+	defer func() {
+		log.SetOutput(originalOutput)
+		log.SetFormatter(originalFormatter)
+		log.SetLevel(originalLevel)
+	}()
+
+	var output bytes.Buffer
+	log.SetOutput(&output)
+	log.SetFormatter(&log.JSONFormatter{})
+	log.SetLevel(log.InfoLevel)
+	gin.SetMode(gin.TestMode)
+
+	router := gin.New()
+	router.Use(requestLogger())
+	router.GET("/health", func(c *gin.Context) {
+		c.Status(stdhttp.StatusNoContent)
+	})
+
+	response := httptest.NewRecorder()
+	request := httptest.NewRequest(stdhttp.MethodGet, "/health", nil)
+	router.ServeHTTP(response, request)
+
+	if output.Len() != 0 {
+		t.Fatalf("successful health check emitted an info log: %q", output.String())
+	}
+}

machinery/src/routers/mqtt/main.go

@@ -1,7 +1,9 @@
 package mqtt
 
 import (
+	"context"
 	"crypto/rsa"
+	"crypto/tls"
 	"crypto/x509"
 	"encoding/base64"
 	"encoding/json"
@@ -9,21 +11,23 @@ import (
 	"fmt"
 	"io/ioutil"
 	"math/rand"
+	"net"
+	"net/url"
+	"os"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
-	"context"
-
 	mqtt "github.com/eclipse/paho.mqtt.golang"
 	"github.com/kerberos-io/agent/machinery/src/capture"
 	configService "github.com/kerberos-io/agent/machinery/src/config"
 	"github.com/kerberos-io/agent/machinery/src/encryption"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/onvif"
 	"github.com/kerberos-io/agent/machinery/src/webrtc"
+	log "github.com/sirupsen/logrus"
+	"golang.org/x/net/proxy"
 )
 
 // We'll cache the MQTT settings to know if we need to reinitialize the MQTT client connection.
@@ -34,6 +38,177 @@ var PREV_MQTTPassword string
 var PREV_HubKey string
 var PREV_AgentKey string
 
+type pahoErrorLogger struct{}
+
+func (pahoErrorLogger) Println(values ...interface{}) {
+	log.WithFields(log.Fields{
+		"component": "routers/mqtt",
+		"event":     "paho_error",
+	}).Error(strings.TrimSpace(fmt.Sprintln(values...)))
+}
+
+func (pahoErrorLogger) Printf(format string, values ...interface{}) {
+	log.WithFields(log.Fields{
+		"component": "routers/mqtt",
+		"event":     "paho_error",
+	}).Errorf(strings.TrimSpace(format), values...)
+}
+
+func init() {
+	mqtt.ERROR = pahoErrorLogger{}
+}
+
+func enableMQTTConnectionDiagnostics(options *mqtt.ClientOptions, brokerURL string) {
+	if !strings.Contains(brokerURL, "://") {
+		options.SetCustomOpenConnectionFn(openMQTTConnection)
+		return
+	}
+
+	parsedURL, err := url.Parse(brokerURL)
+	if err != nil {
+		return
+	}
+
+	switch strings.ToLower(parsedURL.Scheme) {
+	case "", "mqtt", "tcp", "ssl", "tls", "mqtts", "mqtt+ssl", "tcps":
+		options.SetCustomOpenConnectionFn(openMQTTConnection)
+	}
+}
+
+func openMQTTConnection(uri *url.URL, options mqtt.ClientOptions) (net.Conn, error) {
+	host := uri.Hostname()
+	fields := log.Fields{
+		"component": "routers/mqtt",
+		"host":      host,
+		"port":      uri.Port(),
+		"scheme":    uri.Scheme,
+	}
+	logMQTTDNSResolution(host, options.ConnectTimeout, fields)
+
+	connectionStartedAt := time.Now()
+	dialer := options.Dialer
+	if dialer == nil {
+		dialer = &net.Dialer{Timeout: options.ConnectTimeout}
+	}
+
+	proxyConfigured := os.Getenv("all_proxy") != ""
+	proxyMode := "direct"
+	if proxyConfigured {
+		proxyMode = "socks"
+	}
+	fields["proxy_mode"] = proxyMode
+	log.WithFields(fields).Info("Opening MQTT TCP connection")
+
+	var (
+		connection net.Conn
+		err        error
+	)
+	if proxyConfigured {
+		connection, err = proxy.FromEnvironment().Dial("tcp", uri.Host)
+	} else {
+		connection, err = dialer.Dial("tcp", uri.Host)
+	}
+	fields["duration_ms"] = time.Since(connectionStartedAt).Milliseconds()
+	if err != nil {
+		logMQTTNetworkError("MQTT TCP connection failed", err, fields)
+		return nil, err
+	}
+
+	fields["local_address"] = connection.LocalAddr().String()
+	fields["remote_address"] = connection.RemoteAddr().String()
+	log.WithFields(fields).Info("MQTT TCP connection established")
+
+	if !isSecureMQTTScheme(uri.Scheme) {
+		return connection, nil
+	}
+
+	tlsConfig := options.TLSConfig
+	if tlsConfig == nil {
+		tlsConfig = &tls.Config{}
+	} else {
+		tlsConfig = tlsConfig.Clone()
+	}
+	if tlsConfig.ServerName == "" {
+		tlsConfig.ServerName = host
+	}
+
+	tlsConnection := tls.Client(connection, tlsConfig)
+	tlsStartedAt := time.Now()
+	if options.ConnectTimeout > 0 {
+		_ = tlsConnection.SetDeadline(connectionStartedAt.Add(options.ConnectTimeout))
+	}
+	if err = tlsConnection.Handshake(); err != nil {
+		_ = connection.Close()
+		fields["duration_ms"] = time.Since(tlsStartedAt).Milliseconds()
+		fields["server_name"] = tlsConfig.ServerName
+		logMQTTNetworkError("MQTT TLS handshake failed", err, fields)
+		return nil, err
+	}
+	_ = tlsConnection.SetDeadline(time.Time{})
+
+	state := tlsConnection.ConnectionState()
+	fields["cipher_suite"] = tls.CipherSuiteName(state.CipherSuite)
+	fields["duration_ms"] = time.Since(tlsStartedAt).Milliseconds()
+	fields["server_name"] = tlsConfig.ServerName
+	fields["tls_version"] = tls.VersionName(state.Version)
+	log.WithFields(fields).Info("MQTT TLS handshake established")
+	return tlsConnection, nil
+}
+
+func logMQTTDNSResolution(host string, timeout time.Duration, fields log.Fields) {
+	if host == "" || net.ParseIP(host) != nil {
+		return
+	}
+
+	lookupTimeout := timeout
+	if lookupTimeout <= 0 || lookupTimeout > 5*time.Second {
+		lookupTimeout = 5 * time.Second
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), lookupTimeout)
+	defer cancel()
+
+	startedAt := time.Now()
+	addresses, err := net.DefaultResolver.LookupHost(ctx, host)
+	dnsFields := cloneLogFields(fields)
+	dnsFields["duration_ms"] = time.Since(startedAt).Milliseconds()
+	if err != nil {
+		logMQTTNetworkError("MQTT broker DNS resolution failed", err, dnsFields)
+		return
+	}
+
+	dnsFields["resolved_addresses"] = addresses
+	log.WithFields(dnsFields).Info("MQTT broker DNS resolved")
+}
+
+func logMQTTNetworkError(message string, err error, fields log.Fields) {
+	errorFields := cloneLogFields(fields)
+	if networkError, ok := err.(net.Error); ok {
+		errorFields["network_timeout"] = networkError.Timeout()
+	}
+	if operationError, ok := err.(*net.OpError); ok {
+		errorFields["network"] = operationError.Net
+		errorFields["operation"] = operationError.Op
+	}
+	log.WithError(err).WithFields(errorFields).Error(message)
+}
+
+func cloneLogFields(fields log.Fields) log.Fields {
+	cloned := make(log.Fields, len(fields))
+	for key, value := range fields {
+		cloned[key] = value
+	}
+	return cloned
+}
+
+func isSecureMQTTScheme(scheme string) bool {
+	switch strings.ToLower(scheme) {
+	case "ssl", "tls", "mqtts", "mqtt+ssl", "tcps":
+		return true
+	default:
+		return false
+	}
+}
+
 func HasMQTTClientModified(configuration *models.Configuration) bool {
 	MTTURI := configuration.Config.MQTTURI
 	MTTUsername := configuration.Config.MQTTUsername
@@ -41,7 +216,7 @@ func HasMQTTClientModified(configuration *models.Configuration) bool {
 	HubKey := configuration.Config.HubKey
 	AgentKey := configuration.Config.Key
 	if PREV_MQTTURI != MTTURI || PREV_MQTTUsername != MTTUsername || PREV_MQTTPassword != MQTTPassword || PREV_HubKey != HubKey || PREV_AgentKey != AgentKey {
-		log.Log.Info("HasMQTTClientModified: MQTT settings have been modified, restarting MQTT client.")
+		log.Info("HasMQTTClientModified: MQTT settings have been modified, restarting MQTT client.")
 		return true
 	}
 	return false
@@ -59,6 +234,7 @@ func HasMQTTClientModified(configuration *models.Configuration) bool {
 // - kerberos/{hubkey}/device/{devicekey}/motion: a motion signal
 
 func ConfigureMQTT(configDirectory string, configuration *models.Configuration, communication *models.Communication) mqtt.Client {
+	installRemoteAccessHook()
 
 	config := configuration.Config
 
@@ -70,8 +246,25 @@ func ConfigureMQTT(configDirectory string, configuration *models.Configuration,
 	PREV_AgentKey = configuration.Config.Key
 
 	if config.Offline == "true" {
-		log.Log.Info("routers.mqtt.main.ConfigureMQTT(): not starting as running in Offline mode.")
+		log.Info("routers.mqtt.main.ConfigureMQTT(): not starting as running in Offline mode.")
 	} else {
+		hubKey := ""
+		if config.Cloud == "s3" && config.S3 != nil && config.S3.Publickey != "" {
+			hubKey = config.S3.Publickey
+		} else if config.Cloud == "kstorage" && config.KStorage != nil && config.KStorage.CloudKey != "" {
+			hubKey = config.KStorage.CloudKey
+		}
+		if config.HubKey != "" {
+			hubKey = config.HubKey
+		}
+		if hubKey == "" {
+			log.Warn("routers.mqtt.main.ConfigureMQTT(): not starting without a Hub key")
+			return nil
+		}
+		if config.Key == "" {
+			log.Warn("routers.mqtt.main.ConfigureMQTT(): not starting without an Agent key")
+			return nil
+		}
 
 		opts := mqtt.NewClientOptions()
 
@@ -79,7 +272,6 @@ func ConfigureMQTT(configDirectory string, configuration *models.Configuration,
 		// and share and receive messages to/from.
 		mqttURL := config.MQTTURI
 		opts.AddBroker(mqttURL)
-		log.Log.Debug("routers.mqtt.main.ConfigureMQTT(): Set broker uri " + mqttURL)
 
 		// Our MQTT broker can have username/password credentials
 		// to protect it from the outside.
@@ -88,13 +280,19 @@ func ConfigureMQTT(configDirectory string, configuration *models.Configuration,
 		if mqtt_username != "" || mqtt_password != "" {
 			opts.SetUsername(mqtt_username)
 			opts.SetPassword(mqtt_password)
-			log.Log.Debug("routers.mqtt.main.ConfigureMQTT(): Set username " + mqtt_username)
-			log.Log.Debug("routers.mqtt.main.ConfigureMQTT(): Set password " + mqtt_password)
 		}
+		log.WithFields(log.Fields{
+			"broker_configured":   mqttURL != "",
+			"component":           "routers/mqtt",
+			"event":               "credentials_configured",
+			"password_configured": mqtt_password != "",
+			"username_configured": mqtt_username != "",
+		}).Debug("MQTT connection configuration loaded")
 
 		// Some extra options to make sure the connection behaves
 		// properly. More information here: github.com/eclipse/paho.mqtt.golang.
 		//opts.SetCleanSession(true)
+		enableMQTTConnectionDiagnostics(opts, mqttURL)
 		opts.SetCleanSession(false)
 		opts.SetResumeSubs(true)
 		opts.SetStore(mqtt.NewMemoryStore())
@@ -108,65 +306,81 @@ func ConfigureMQTT(configDirectory string, configuration *models.Configuration,
 		opts.SetOrderMatters(false)
 		opts.SetConnectTimeout(30 * time.Second)
 		opts.SetConnectionLostHandler(func(client mqtt.Client, err error) {
+			entry := log.WithFields(log.Fields{
+				"component": "routers/mqtt",
+				"event":     "connection_lost",
+			})
 			if err != nil {
-				log.Log.Error("routers.mqtt.main.ConfigureMQTT(): MQTT connection lost: " + err.Error())
-			} else {
-				log.Log.Error("routers.mqtt.main.ConfigureMQTT(): MQTT connection lost")
+				entry = entry.WithError(err)
 			}
+			entry.Error("MQTT connection lost")
 		})
 		opts.SetReconnectingHandler(func(client mqtt.Client, options *mqtt.ClientOptions) {
-			log.Log.Warning("routers.mqtt.main.ConfigureMQTT(): reconnecting to MQTT broker")
+			log.WithFields(log.Fields{
+				"component": "routers/mqtt",
+				"event":     "reconnecting",
+			}).Warn("Reconnecting to MQTT broker")
 		})
 		opts.SetOnConnectHandler(func(c mqtt.Client) {
-			log.Log.Info("routers.mqtt.main.ConfigureMQTT(): MQTT session is online")
+			log.WithFields(log.Fields{
+				"component": "routers/mqtt",
+				"event":     "session_online",
+			}).Info("MQTT session is online")
 		})
 
-		hubKey := ""
-		// This is the old way ;)
-		if config.Cloud == "s3" && config.S3 != nil && config.S3.Publickey != "" {
-			hubKey = config.S3.Publickey
-		} else if config.Cloud == "kstorage" && config.KStorage != nil && config.KStorage.CloudKey != "" {
-			hubKey = config.KStorage.CloudKey
-		}
-		// This is the new way ;)
-		if config.HubKey != "" {
-			hubKey = config.HubKey
+		rand.Seed(time.Now().UnixNano())
+		random := rand.Intn(100)
+		mqttClientID := config.Key + strconv.Itoa(random) // this random int is to avoid conflicts.
+
+		// This is a worked-around.
+		// current S3 (Kerberos Hub SAAS) is using a secured MQTT, where the client id,
+		// should match the kerberos hub key.
+		if config.Cloud == "s3" {
+			mqttClientID = config.Key
 		}
 
-		if hubKey != "" {
+		opts.SetClientID(mqttClientID)
+		log.WithFields(log.Fields{
+			"component":            "routers/mqtt",
+			"event":                "client_configured",
+			"randomized_client_id": config.Cloud != "s3",
+		}).Debug("MQTT client configured")
+		rand.Seed(time.Now().UnixNano())
 
-			rand.Seed(time.Now().UnixNano())
-			random := rand.Intn(100)
-			mqttClientID := config.Key + strconv.Itoa(random) // this random int is to avoid conflicts.
+		opts.OnConnect = func(c mqtt.Client) {
+			// We managed to connect to the MQTT broker, hurray!
+			log.WithFields(log.Fields{
+				"component": "routers/mqtt",
+				"event":     "connected",
+			}).Info("MQTT broker connected")
+			log.WithFields(log.Fields{
+				"broker_configured": mqttURL != "",
+				"component":         "routers/mqtt",
+				"event":             "connection_details",
+			}).Debug("MQTT connection details")
 
-			// This is a worked-around.
-			// current S3 (Kerberos Hub SAAS) is using a secured MQTT, where the client id,
-			// should match the kerberos hub key.
-			if config.Cloud == "s3" {
-				mqttClientID = config.Key
-			}
-
-			opts.SetClientID(mqttClientID)
-			log.Log.Info("routers.mqtt.main.ConfigureMQTT(): Set ClientID " + mqttClientID)
-			rand.Seed(time.Now().UnixNano())
-
-			opts.OnConnect = func(c mqtt.Client) {
-				// We managed to connect to the MQTT broker, hurray!
-				log.Log.Info("routers.mqtt.main.ConfigureMQTT(): " + mqttClientID + " connected to " + mqttURL)
-
-				// Create a susbcription for listen and reply
-				MQTTListenerHandler(c, hubKey, configDirectory, configuration, communication)
-			}
+			// Create a susbcription for listen and reply
+			MQTTListenerHandler(c, hubKey, configDirectory, configuration, communication)
 		}
 		mqc := mqtt.NewClient(opts)
 		if token := mqc.Connect(); token.WaitTimeout(30 * time.Second) {
 			if token.Error() != nil {
-				log.Log.Error("routers.mqtt.main.ConfigureMQTT(): unable to establish mqtt broker connection, error was: " + token.Error().Error())
+				log.WithError(token.Error()).WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "initial_connection_failed",
+				}).Error("Failed to establish initial MQTT connection")
 			} else {
-				log.Log.Info("routers.mqtt.main.ConfigureMQTT(): initial MQTT connection established")
+				log.WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "initial_connection_established",
+				}).Info("Initial MQTT connection established")
 			}
 		} else {
-			log.Log.Error("routers.mqtt.main.ConfigureMQTT(): timed out while establishing mqtt broker connection")
+			log.WithFields(log.Fields{
+				"component":  "routers/mqtt",
+				"event":      "initial_connection_timeout",
+				"timeout_ms": (30 * time.Second).Milliseconds(),
+			}).Warn("Initial MQTT connection is still retrying")
 		}
 		return mqc
 	}
@@ -216,7 +430,7 @@ func markHDSessionSeen(sessionID string) bool {
 
 func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory string, configuration *models.Configuration, communication *models.Communication) {
 	if hubKey == "" {
-		log.Log.Info("routers.mqtt.main.MQTTListenerHandler(): no hub key provided, not subscribing to kerberos/hub/{hubkey}")
+		log.Info("routers.mqtt.main.MQTTListenerHandler(): no hub key provided, not subscribing to kerberos/hub/{hubkey}")
 	} else {
 		agentListener := fmt.Sprintf("kerberos/agent/%s", hubKey)
 		token := mqttClient.Subscribe(agentListener, 1, func(c mqtt.Client, msg mqtt.Message) {
@@ -236,6 +450,7 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 			// We will receive all messages from our hub, so we'll need to filter to the relevant device.
 			if message.Mid != "" && message.Timestamp != 0 && message.DeviceId == configuration.Config.Key {
 				var payload models.Payload
+				remoteAuthenticated := false
 
 				// Messages might be hidden, if so we'll need to decrypt them using the Kerberos Hub private key.
 				if message.Hidden && configuration.Config.HubEncryption == "true" {
@@ -249,13 +464,15 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 							}
 							visibleValue, err := encryption.AesDecrypt(data, privateKey)
 							if err != nil {
-								log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message: " + err.Error())
+								log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message: " + err.Error())
 								return
 							}
-							json.Unmarshal(visibleValue, &payload)
-							message.Payload = payload
+							if err := json.Unmarshal(visibleValue, &payload); err == nil {
+								message.Payload = payload
+								remoteAuthenticated = true
+							}
 						} else {
-							log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message, no private key provided.")
+							log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message, no private key provided.")
 						}
 					}
 				}
@@ -271,14 +488,14 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 						pemBytes, _ := ioutil.ReadAll(r)
 						block, _ := pem.Decode(pemBytes)
 						if block == nil {
-							log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error decoding PEM block containing private key")
+							log.Error("routers.mqtt.main.MQTTListenerHandler(): error decoding PEM block containing private key")
 							return
 						} else {
 							// Parse private key
 							b := block.Bytes
 							key, err := x509.ParsePKCS8PrivateKey(b)
 							if err != nil {
-								log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error parsing private key: " + err.Error())
+								log.Error("routers.mqtt.main.MQTTListenerHandler(): error parsing private key: " + err.Error())
 								return
 							} else {
 								// Conver key to *rsa.PrivateKey
@@ -298,16 +515,18 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 										}
 										decryptedValue, err := encryption.AesDecrypt(data, string(decryptedKey))
 										if err != nil {
-											log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message: " + err.Error())
+											log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message: " + err.Error())
 											return
 										}
-										json.Unmarshal(decryptedValue, &payload)
+										if err := json.Unmarshal(decryptedValue, &payload); err == nil {
+											remoteAuthenticated = true
+										}
 									} else {
-										log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message, assymetric keys do not match.")
+										log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message, assymetric keys do not match.")
 										return
 									}
 								} else if err != nil {
-									log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message: " + err.Error())
+									log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message: " + err.Error())
 									return
 								}
 							}
@@ -318,7 +537,11 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 				}
 
 				// We'll find out which message we received, and act accordingly.
-				log.Log.Info("routers.mqtt.main.MQTTListenerHandler(): received message with action: " + payload.Action)
+				log.WithFields(log.Fields{
+					"action":    payload.Action,
+					"component": "routers/mqtt",
+					"event":     "message_received",
+				}).Debug("MQTT message received")
 
 				// NOTE: We intentionally do NOT discard request-hd-stream /
 				// receive-hd-candidates messages based on a wall-clock age. The
@@ -355,6 +578,14 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 					go HandleReceiveHDCandidates(mqttClient, hubKey, payload, configuration, communication)
 				case "trigger-relay":
 					go HandleTriggerRelay(mqttClient, hubKey, payload, configuration, communication)
+				case "remote-session-open":
+					go HandleRemoteSessionOpen(mqttClient, hubKey, payload, remoteAuthenticated, configuration)
+				case "remote-session-input":
+					go HandleRemoteSessionInput(mqttClient, hubKey, payload, remoteAuthenticated, configuration)
+				case "remote-session-resize":
+					go HandleRemoteSessionResize(payload, remoteAuthenticated)
+				case "remote-session-close":
+					go HandleRemoteSessionClose(payload, remoteAuthenticated)
 				}
 
 			}
@@ -362,12 +593,15 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 
 		if token.WaitTimeout(10 * time.Second) {
 			if token.Error() != nil {
-				log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): failed to subscribe to " + agentListener + ": " + token.Error().Error())
+				log.WithError(token.Error()).WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "subscription_failed",
+				}).Error("Failed to subscribe to Agent MQTT topic")
 			} else {
-				log.Log.Info("routers.mqtt.main.MQTTListenerHandler(): subscribed to " + agentListener)
+				log.Info("routers.mqtt.main.MQTTListenerHandler(): subscribed to " + agentListener)
 			}
 		} else {
-			log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): timed out while subscribing to " + agentListener)
+			log.Error("routers.mqtt.main.MQTTListenerHandler(): timed out while subscribing to " + agentListener)
 		}
 	}
 }
@@ -396,9 +630,9 @@ func HandleRecording(mqttClient mqtt.Client, hubKey string, payload models.Paylo
 			if communication.IsRecordingManual.IsSet() {
 				communication.RecordingManualHeartbeat.Store(now)
 				communication.RecordingManualHeartbeatSeen.Set()
-				log.Log.Debug("routers.mqtt.main.HandleRecording(): manual recording heartbeat received.")
+				log.Debug("routers.mqtt.main.HandleRecording(): manual recording heartbeat received.")
 			} else {
-				log.Log.Debug("routers.mqtt.main.HandleRecording(): ignoring heartbeat, no active manual recording.")
+				log.Debug("routers.mqtt.main.HandleRecording(): ignoring heartbeat, no active manual recording.")
 			}
 		} else {
 			// Explicit start from the live view (record button). Start a manual
@@ -410,18 +644,16 @@ func HandleRecording(mqttClient mqtt.Client, hubKey string, payload models.Paylo
 			if communication.IsRecordingManual.SetToIf(false, true) {
 				communication.RecordingManualStart.Store(now)
 				communication.RecordingManualHeartbeatSeen.UnSet()
-				log.Log.Info("routers.mqtt.main.HandleRecording(): manual recording started.")
-				select {
-				case communication.HandleMotion <- models.MotionDataPartial{Timestamp: timestamp, NumberOfChanges: 100000000}:
-				default:
-					log.Log.Warning("routers.mqtt.main.HandleRecording(): motion channel full, manual recording start not queued.")
+				log.Info("routers.mqtt.main.HandleRecording(): manual recording started.")
+				if !communication.TrySendMotion(models.MotionDataPartial{Timestamp: timestamp, NumberOfChanges: 100000000}) {
+					log.Warn("routers.mqtt.main.HandleRecording(): motion channel full, manual recording start not queued.")
 				}
 			}
 		}
 	} else {
 		// Stop the manual recording; the motion recorder closes the clip once the
 		// post-recording window elapses. Clear the heartbeat/start markers too.
-		log.Log.Info("routers.mqtt.main.HandleRecording(): manual recording stopped.")
+		log.Info("routers.mqtt.main.HandleRecording(): manual recording stopped.")
 		communication.IsRecordingManual.UnSet()
 		communication.RecordingManualHeartbeat.Store(0)
 		communication.RecordingManualStart.Store(0)
@@ -443,7 +675,7 @@ func HandleAudio(mqttClient mqtt.Client, hubKey string, payload models.Payload,
 			Data:      audioPayload.Data,
 		}
 		if enqueueLatestAudio(communication.HandleAudio, audioDataPartial) {
-			log.Log.Debug("routers.mqtt.main.HandleAudio(): dropped stale audio because the backchannel queue was full")
+			log.Debug("routers.mqtt.main.HandleAudio(): dropped stale audio because the backchannel queue was full")
 		}
 	}
 }
@@ -479,7 +711,7 @@ func HandleGetPTZPosition(mqttClient mqtt.Client, hubKey string, payload models.
 		// Get Position from device
 		pos, err := onvif.GetPositionFromDevice(*configuration)
 		if err != nil {
-			log.Log.Error("routers.mqtt.main.HandlePTZPosition(): error getting position from device: " + err.Error())
+			log.Error("routers.mqtt.main.HandlePTZPosition(): error getting position from device: " + err.Error())
 		} else {
 			// Needs to wrapped!
 			posString := fmt.Sprintf("%f,%f,%f", pos.PanTilt.X, pos.PanTilt.Y, pos.Zoom.X)
@@ -497,7 +729,10 @@ func HandleGetPTZPosition(mqttClient mqtt.Client, hubKey string, payload models.
 			if err == nil {
 				mqttClient.Publish("kerberos/hub/"+hubKey, 2, false, payload)
 			} else {
-				log.Log.Info("routers.mqtt.main.HandlePTZPosition(): something went wrong while sending position to hub: " + string(payload))
+				log.WithError(err).WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "ptz_position_packaging_failed",
+				}).Error("Failed to package PTZ position message")
 			}
 		}
 	}
@@ -512,11 +747,14 @@ func HandleUpdatePTZPosition(mqttClient mqtt.Client, hubKey string, payload mode
 	json.Unmarshal(jsonData, &onvifAction)
 
 	if onvifAction.Action != "" {
-		if communication.CameraConnected {
-			communication.HandleONVIF <- onvifAction
-			log.Log.Info("routers.mqtt.main.MQTTListenerHandleONVIF(): Received an action - " + onvifAction.Action)
+		if communication.CameraConnected.Load() {
+			if communication.TrySendONVIF(onvifAction) {
+				log.Info("routers.mqtt.main.MQTTListenerHandleONVIF(): Received an action - " + onvifAction.Action)
+			} else {
+				log.Warn("routers.mqtt.main.MQTTListenerHandleONVIF(): action channel unavailable or full")
+			}
 		} else {
-			log.Log.Info("routers.mqtt.main.MQTTListenerHandleONVIF(): received action, but camera is not connected.")
+			log.Info("routers.mqtt.main.MQTTListenerHandleONVIF(): received action, but camera is not connected.")
 		}
 	}
 }
@@ -561,14 +799,17 @@ func HandleRequestConfig(mqttClient mqtt.Client, hubKey string, payload models.P
 			if err == nil {
 				mqttClient.Publish("kerberos/hub/"+hubKey, 2, false, payload)
 			} else {
-				log.Log.Info("routers.mqtt.main.HandleRequestConfig(): something went wrong while sending config to hub: " + string(payload))
+				log.WithError(err).WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "configuration_packaging_failed",
+				}).Error("Failed to package configuration message")
 			}
 
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleRequestConfig(): no config available")
+			log.Info("routers.mqtt.main.HandleRequestConfig(): no config available")
 		}
 
-		log.Log.Info("routers.mqtt.main.HandleRequestConfig(): Received a request for the config")
+		log.Info("routers.mqtt.main.HandleRequestConfig(): Received a request for the config")
 	}
 }
 
@@ -660,7 +901,10 @@ func HandleVerifyStream(mqttClient mqtt.Client, hubKey string, payload models.Pa
 	if err == nil {
 		mqttClient.Publish("kerberos/hub/"+hubKey, 2, false, packagedPayload)
 	} else {
-		log.Log.Info("routers.mqtt.main.HandleVerifyStream(): something went wrong while sending result to hub: " + string(packagedPayload))
+		log.WithError(err).WithFields(log.Fields{
+			"component": "routers/mqtt",
+			"event":     "stream_verification_packaging_failed",
+		}).Error("Failed to package stream verification result")
 	}
 }
 
@@ -681,7 +925,7 @@ func HandleUpdateConfig(mqttClient mqtt.Client, hubKey string, payload models.Pa
 
 		err := configService.SaveConfig(configDirectory, config, configuration, communication)
 		if err == nil {
-			log.Log.Info("routers.mqtt.main.HandleUpdateConfig(): Config updated")
+			log.Info("routers.mqtt.main.HandleUpdateConfig(): Config updated")
 			message := models.Message{
 				Payload: models.Payload{
 					Action:   "acknowledge-update-config",
@@ -692,10 +936,13 @@ func HandleUpdateConfig(mqttClient mqtt.Client, hubKey string, payload models.Pa
 			if err == nil {
 				mqttClient.Publish("kerberos/hub/"+hubKey, 2, false, payload)
 			} else {
-				log.Log.Info("routers.mqtt.main.HandleUpdateConfig(): something went wrong while sending acknowledge config to hub: " + string(payload))
+				log.WithError(err).WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "configuration_acknowledgement_packaging_failed",
+				}).Error("Failed to package configuration acknowledgement")
 			}
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleUpdateConfig(): Config update failed")
+			log.Info("routers.mqtt.main.HandleUpdateConfig(): Config update failed")
 		}
 	}
 }
@@ -708,7 +955,7 @@ func HandleRequestSDStream(mqttClient mqtt.Client, hubKey string, payload models
 	json.Unmarshal(jsonData, &requestSDStreamPayload)
 
 	if requestSDStreamPayload.Timestamp != 0 {
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			// A viewer that opted into the HTTP transport is signalled on a separate
 			// channel so the producer ships its frames to hub-api over HTTP instead of
 			// publishing them over MQTT. Any other (or absent) transport keeps the
@@ -724,9 +971,9 @@ func HandleRequestSDStream(mqttClient mqtt.Client, hubKey string, payload models
 				default:
 				}
 			}
-			log.Log.Info("routers.mqtt.main.HandleRequestSDStream(): received request to livestream.")
+			log.Info("routers.mqtt.main.HandleRequestSDStream(): received request to livestream.")
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleRequestSDStream(): received request to livestream, but camera is not connected.")
+			log.Info("routers.mqtt.main.HandleRequestSDStream(): received request to livestream, but camera is not connected.")
 		}
 	}
 }
@@ -743,7 +990,7 @@ func HandleRequestHLSStream(mqttClient mqtt.Client, hubKey string, payload model
 	json.Unmarshal(jsonData, &requestHLSStreamPayload)
 
 	if requestHLSStreamPayload.Timestamp != 0 {
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			// Forward the requested quality ("auto"|"high"|"low"; empty => auto) so
 			// the producer can switch the live session between the main and sub
 			// stream on demand. The send doubles as the viewer keepalive.
@@ -751,9 +998,9 @@ func HandleRequestHLSStream(mqttClient mqtt.Client, hubKey string, payload model
 			case communication.HandleLiveHLS <- requestHLSStreamPayload.Quality:
 			default:
 			}
-			log.Log.Info("routers.mqtt.main.HandleRequestHLSStream(): received request to livestream over HLS.")
+			log.Info("routers.mqtt.main.HandleRequestHLSStream(): received request to livestream over HLS.")
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleRequestHLSStream(): received request to livestream over HLS, but camera is not connected.")
+			log.Info("routers.mqtt.main.HandleRequestHLSStream(): received request to livestream over HLS, but camera is not connected.")
 		}
 	}
 }
@@ -766,29 +1013,30 @@ func HandleRequestHDStream(mqttClient mqtt.Client, hubKey string, payload models
 	json.Unmarshal(jsonData, &requestHDStreamPayload)
 
 	if requestHDStreamPayload.Timestamp != 0 {
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			// Dedupe by session_id: the viewer republishes its offer while
 			// waiting for an answer (and the broker may redeliver), and we
 			// don't want to spawn multiple peer connections for the same
 			// browser session.
 			if markHDSessionSeen(requestHDStreamPayload.SessionID) {
-				log.Log.Info("routers.mqtt.main.HandleRequestHDStream(): duplicate request for session " +
-					requestHDStreamPayload.SessionID + ", ignoring")
+				log.WithFields(log.Fields{
+					"component":  "routers/mqtt",
+					"event":      "duplicate_hd_stream_request",
+					"session_id": requestHDStreamPayload.SessionID,
+				}).Debug("Ignoring duplicate HD stream request")
 				return
 			}
 			// Set the Hub key, so we can send back the answer.
 			requestHDStreamPayload.HubKey = hubKey
-			if communication.HandleLiveHDHandshake == nil {
-				log.Log.Error("routers.mqtt.main.HandleRequestHDStream(): handshake channel is nil, dropping request")
+			if !communication.TrySendLiveHDHandshake(models.LiveHDHandshake{
+				Payload: requestHDStreamPayload,
+			}) {
+				log.Error("routers.mqtt.main.HandleRequestHDStream(): handshake channel unavailable or full, dropping request")
 				return
 			}
-
-			communication.HandleLiveHDHandshake <- models.LiveHDHandshake{
-				Payload: requestHDStreamPayload,
-			}
-			log.Log.Info("routers.mqtt.main.HandleRequestHDStream(): received request to setup webrtc.")
+			log.Info("routers.mqtt.main.HandleRequestHDStream(): received request to setup webrtc.")
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleRequestHDStream(): received request to setup webrtc, but camera is not connected.")
+			log.Info("routers.mqtt.main.HandleRequestHDStream(): received request to setup webrtc, but camera is not connected.")
 		}
 	}
 }
@@ -801,12 +1049,12 @@ func HandleReceiveHDCandidates(mqttClient mqtt.Client, hubKey string, payload mo
 	json.Unmarshal(jsonData, &receiveHDCandidatesPayload)
 
 	if receiveHDCandidatesPayload.Timestamp != 0 {
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			// Register candidate channel
 			key := configuration.Config.Key + "/" + receiveHDCandidatesPayload.SessionID
 			go webrtc.RegisterCandidates(key, receiveHDCandidatesPayload)
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleReceiveHDCandidates(): received candidate, but camera is not connected.")
+			log.Info("routers.mqtt.main.HandleReceiveHDCandidates(): received candidate, but camera is not connected.")
 		}
 	}
 }
@@ -818,14 +1066,17 @@ func HandleNavigatePTZ(mqttClient mqtt.Client, hubKey string, payload models.Pay
 	json.Unmarshal(jsonData, &navigatePTZPayload)
 
 	if navigatePTZPayload.Timestamp != 0 {
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			action := navigatePTZPayload.Action
 			var onvifAction models.OnvifAction
 			json.Unmarshal([]byte(action), &onvifAction)
-			communication.HandleONVIF <- onvifAction
-			log.Log.Info("routers.mqtt.main.HandleNavigatePTZ(): Received an action - " + onvifAction.Action)
+			if communication.TrySendONVIF(onvifAction) {
+				log.Info("routers.mqtt.main.HandleNavigatePTZ(): Received an action - " + onvifAction.Action)
+			} else {
+				log.Warn("routers.mqtt.main.HandleNavigatePTZ(): action channel unavailable or full")
+			}
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleNavigatePTZ(): received action, but camera is not connected.")
+			log.Info("routers.mqtt.main.HandleNavigatePTZ(): received action, but camera is not connected.")
 		}
 	}
 }
@@ -837,7 +1088,7 @@ func HandleTriggerRelay(mqttClient mqtt.Client, hubKey string, payload models.Pa
 	json.Unmarshal(jsonData, &triggerRelayPayload)
 
 	if triggerRelayPayload.Timestamp != 0 {
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			// Get token (name of relay)
 			token := triggerRelayPayload.Token
 			// Connect to Onvif device
@@ -847,16 +1098,25 @@ func HandleTriggerRelay(mqttClient mqtt.Client, hubKey string, payload models.Pa
 				// Trigger relay output
 				err := onvif.TriggerRelayOutput(device, token)
 				if err != nil {
-					log.Log.Error("routers.mqtt.main.HandleTriggerRelay(): error triggering relay: " + err.Error())
+					log.WithError(err).WithFields(log.Fields{
+						"component": "routers/mqtt",
+						"event":     "relay_output_failed",
+					}).Error("Failed to trigger relay output")
 				} else {
-					log.Log.Info("routers.mqtt.main.HandleTriggerRelay(): trigger (" + token + ") relay output.")
+					log.WithFields(log.Fields{
+						"component": "routers/mqtt",
+						"event":     "relay_output_triggered",
+					}).Info("Relay output triggered")
 				}
 			} else {
-				log.Log.Error("routers.mqtt.main.HandleTriggerRelay(): error connecting to device: " + err.Error())
+				log.WithError(err).WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "relay_device_connection_failed",
+				}).Error("Failed to connect to relay device")
 			}
 
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleTriggerRelay(): received trigger, but camera is not connected.")
+			log.Info("routers.mqtt.main.HandleTriggerRelay(): received trigger, but camera is not connected.")
 		}
 	}
 }
@@ -868,6 +1128,6 @@ func DisconnectMQTT(mqttClient mqtt.Client, config *models.Config) {
 		mqttClient.Unsubscribe("kerberos/agent/" + PREV_HubKey)
 		mqttClient.Disconnect(1000)
 		mqttClient = nil
-		log.Log.Info("routers.mqtt.main.DisconnectMQTT(): MQTT client disconnected.")
+		log.Info("routers.mqtt.main.DisconnectMQTT(): MQTT client disconnected.")
 	}
 }

machinery/src/routers/mqtt/main_test.go

@@ -1,12 +1,111 @@
 package mqtt
 
 import (
+	"crypto/tls"
+	"errors"
+	"net/http/httptest"
+	"net/url"
+	"os"
 	"testing"
 	"time"
 
+	mqtt "github.com/eclipse/paho.mqtt.golang"
 	"github.com/kerberos-io/agent/machinery/src/models"
 )
 
+func TestEnableMQTTConnectionDiagnostics(t *testing.T) {
+	tests := []struct {
+		name      string
+		brokerURL string
+		want      bool
+	}{
+		{name: "ActiveMQ TLS", brokerURL: "mqtt+ssl://broker.example:8883", want: true},
+		{name: "TCP", brokerURL: "tcp://broker.example:1883", want: true},
+		{name: "default TCP", brokerURL: "broker.example:1883", want: true},
+		{name: "WebSocket", brokerURL: "wss://broker.example/mqtt", want: false},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			options := mqtt.NewClientOptions()
+			enableMQTTConnectionDiagnostics(options, test.brokerURL)
+			if got := options.CustomOpenConnectionFn != nil; got != test.want {
+				t.Fatalf("CustomOpenConnectionFn configured = %t, want %t", got, test.want)
+			}
+		})
+	}
+}
+
+func TestOpenMQTTConnectionReturnsTCPError(t *testing.T) {
+	options := *mqtt.NewClientOptions().SetConnectTimeout(100 * time.Millisecond)
+	brokerURL, err := url.Parse("tcp://127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	connection, err := openMQTTConnection(brokerURL, options)
+	if connection != nil {
+		connection.Close()
+		t.Fatal("openMQTTConnection() returned a connection for an unavailable endpoint")
+	}
+	if err == nil {
+		t.Fatal("openMQTTConnection() returned no TCP error")
+	}
+}
+
+func TestOpenMQTTConnectionEstablishesActiveMQTLS(t *testing.T) {
+	server := httptest.NewTLSServer(nil)
+	defer server.Close()
+
+	serverURL, err := url.Parse(server.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	brokerURL, err := url.Parse("mqtt+ssl://" + serverURL.Host)
+	if err != nil {
+		t.Fatal(err)
+	}
+	options := *mqtt.NewClientOptions().
+		SetConnectTimeout(time.Second).
+		SetTLSConfig(&tls.Config{InsecureSkipVerify: true}) // #nosec G402 -- local test server
+
+	connection, err := openMQTTConnection(brokerURL, options)
+	if err != nil {
+		t.Fatalf("openMQTTConnection() error = %v", err)
+	}
+	defer connection.Close()
+	if _, ok := connection.(*tls.Conn); !ok {
+		t.Fatalf("openMQTTConnection() connection type = %T, want *tls.Conn", connection)
+	}
+}
+
+func TestIsSecureMQTTScheme(t *testing.T) {
+	for _, scheme := range []string{"ssl", "tls", "mqtts", "mqtt+ssl", "tcps"} {
+		if !isSecureMQTTScheme(scheme) {
+			t.Errorf("isSecureMQTTScheme(%q) = false, want true", scheme)
+		}
+	}
+	if isSecureMQTTScheme("tcp") {
+		t.Error("isSecureMQTTScheme(\"tcp\") = true, want false")
+	}
+}
+
+func TestConfigureMQTTRequiresHubKey(t *testing.T) {
+	configuration := &models.Configuration{Config: models.Config{Key: "agent-key"}}
+
+	if client := ConfigureMQTT("", configuration, &models.Communication{}); client != nil {
+		t.Fatal("ConfigureMQTT() returned a client without a Hub key")
+	}
+}
+
+func TestConfigureMQTTRequiresAgentKey(t *testing.T) {
+	configuration := &models.Configuration{Config: models.Config{HubKey: "hub-key"}}
+
+	if client := ConfigureMQTT("", configuration, &models.Communication{}); client != nil {
+		t.Fatal("ConfigureMQTT() returned a client without an Agent key")
+	}
+}
+
 func TestEnqueueLatestAudioReplacesOldestFrameWhenFull(t *testing.T) {
 	audioChannel := make(chan models.AudioDataPartial, 2)
 	audioChannel <- models.AudioDataPartial{Timestamp: 1}
@@ -37,3 +136,73 @@ func TestEnqueueLatestAudioDoesNotBlockNilChannel(t *testing.T) {
 		t.Fatal("enqueueLatestAudio() blocked on a nil channel")
 	}
 }
+
+func TestRemoteAccessRequiresExplicitOptIn(t *testing.T) {
+	previous, present := os.LookupEnv(remoteAccessEnvironment)
+	t.Cleanup(func() {
+		if present {
+			_ = os.Setenv(remoteAccessEnvironment, previous)
+		} else {
+			_ = os.Unsetenv(remoteAccessEnvironment)
+		}
+	})
+
+	_ = os.Unsetenv(remoteAccessEnvironment)
+	if remoteAccessEnabled() {
+		t.Fatal("remoteAccessEnabled() = true without opt-in")
+	}
+	_ = os.Setenv(remoteAccessEnvironment, "true")
+	if !remoteAccessEnabled() {
+		t.Fatal("remoteAccessEnabled() = false after opt-in")
+	}
+}
+
+func TestNormalizeTerminalSize(t *testing.T) {
+	rows, columns := normalizeTerminalSize(0, 0)
+	if rows != 24 || columns != 80 {
+		t.Fatalf("normalizeTerminalSize(0, 0) = (%d, %d), want (24, 80)", rows, columns)
+	}
+
+	rows, columns = normalizeTerminalSize(500, 500)
+	if rows != 200 || columns != 400 {
+		t.Fatalf("normalizeTerminalSize(500, 500) = (%d, %d), want (200, 400)", rows, columns)
+	}
+}
+
+func TestDecodeRemotePayloadRejectsMissingSession(t *testing.T) {
+	_, err := decodeRemotePayload(models.Payload{Value: map[string]interface{}{
+		"kind": "shell",
+	}})
+	if err == nil {
+		t.Fatal("decodeRemotePayload() accepted a missing session id")
+	}
+}
+
+func TestRemoteSessionOpenRejectsUnprovenEncryption(t *testing.T) {
+	previous, present := os.LookupEnv(remoteAccessEnvironment)
+	t.Cleanup(func() {
+		if present {
+			_ = os.Setenv(remoteAccessEnvironment, previous)
+		} else {
+			_ = os.Unsetenv(remoteAccessEnvironment)
+		}
+	})
+	_ = os.Setenv(remoteAccessEnvironment, "true")
+
+	// The listener passes false when an envelope merely claims to be hidden but
+	// no ciphertext was successfully decrypted. The remote handler must reject it.
+	if err := validateRemoteAccess(false); !errors.Is(err, errRemoteUnauthenticated) {
+		t.Fatalf("validateRemoteAccess(false) error = %v, want %v", err, errRemoteUnauthenticated)
+	}
+}
+
+func TestRemoteSessionReservationIsIdempotent(t *testing.T) {
+	manager := newRemoteAccessManager()
+	session := &remoteSession{id: "session-1", kind: "logs"}
+	if _, err := manager.reserve(session); err != nil {
+		t.Fatalf("first reserve() failed: %v", err)
+	}
+	if _, err := manager.reserve(session); !errors.Is(err, errRemoteSessionExists) {
+		t.Fatalf("duplicate reserve() error = %v, want %v", err, errRemoteSessionExists)
+	}
+}

machinery/src/routers/mqtt/remote_access.go

@@ -0,0 +1,397 @@
+package mqtt
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"io"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/creack/pty"
+	paho "github.com/eclipse/paho.mqtt.golang"
+	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	remoteAccessEnvironment = "AGENT_REMOTE_ACCESS_ENABLED"
+	remoteHistoryLimit      = 500
+	remoteSessionLimit      = 5
+	remoteOutputChunkSize   = 4096
+	remoteInputLimit        = 64 * 1024
+	remoteSessionLifetime   = time.Hour
+)
+
+type remoteSession struct {
+	id     string
+	kind   string
+	pty    *os.File
+	cancel context.CancelFunc
+	logs   chan string
+	done   chan struct{}
+	timer  *time.Timer
+}
+
+type remoteAccessManager struct {
+	mu       sync.Mutex
+	sessions map[string]*remoteSession
+	history  []string
+}
+
+var (
+	remoteAccess             = newRemoteAccessManager()
+	remoteHookOnce           sync.Once
+	errRemoteDisabled        = errors.New("remote access is disabled on this agent")
+	errRemoteUnauthenticated = errors.New("remote access requires encrypted MQTT")
+	errRemoteSessionExists   = errors.New("remote session already exists")
+	errRemoteSessionLimit    = errors.New("remote session limit reached")
+)
+
+func newRemoteAccessManager() *remoteAccessManager {
+	return &remoteAccessManager{sessions: make(map[string]*remoteSession)}
+}
+
+func installRemoteAccessHook() {
+	remoteHookOnce.Do(func() {
+		log.AddHook(remoteAccess)
+	})
+}
+
+func (manager *remoteAccessManager) Levels() []log.Level {
+	return log.AllLevels
+}
+
+func (manager *remoteAccessManager) Fire(entry *log.Entry) error {
+	line, err := json.Marshal(map[string]interface{}{
+		"timestamp": entry.Time.Format(time.RFC3339Nano),
+		"level":     entry.Level.String(),
+		"message":   entry.Message,
+		"fields":    entry.Data,
+	})
+	if err != nil {
+		return nil
+	}
+	encoded := base64.StdEncoding.EncodeToString(append(line, '\n'))
+
+	manager.mu.Lock()
+	manager.history = append(manager.history, encoded)
+	if len(manager.history) > remoteHistoryLimit {
+		manager.history = manager.history[len(manager.history)-remoteHistoryLimit:]
+	}
+	for _, session := range manager.sessions {
+		if session.kind != "logs" {
+			continue
+		}
+		select {
+		case session.logs <- encoded:
+		default:
+		}
+	}
+	manager.mu.Unlock()
+	return nil
+}
+
+func remoteAccessEnabled() bool {
+	enabled, err := strconv.ParseBool(strings.TrimSpace(os.Getenv(remoteAccessEnvironment)))
+	return err == nil && enabled
+}
+
+func validateRemoteAccess(authenticated bool) error {
+	if !authenticated {
+		return errRemoteUnauthenticated
+	}
+	if !remoteAccessEnabled() {
+		return errRemoteDisabled
+	}
+	return nil
+}
+
+func decodeRemotePayload(payload models.Payload) (models.RemoteSessionPayload, error) {
+	data, err := json.Marshal(payload.Value)
+	if err != nil {
+		return models.RemoteSessionPayload{}, err
+	}
+	var request models.RemoteSessionPayload
+	if err := json.Unmarshal(data, &request); err != nil {
+		return models.RemoteSessionPayload{}, err
+	}
+	if request.SessionID == "" || len(request.SessionID) > 128 {
+		return models.RemoteSessionPayload{}, errors.New("invalid remote session id")
+	}
+	return request, nil
+}
+
+func normalizeTerminalSize(rows uint16, columns uint16) (uint16, uint16) {
+	if rows < 5 {
+		rows = 24
+	}
+	if rows > 200 {
+		rows = 200
+	}
+	if columns < 20 {
+		columns = 80
+	}
+	if columns > 400 {
+		columns = 400
+	}
+	return rows, columns
+}
+
+func HandleRemoteSessionOpen(client paho.Client, hubKey string, payload models.Payload, authenticated bool, configuration *models.Configuration) {
+	request, err := decodeRemotePayload(payload)
+	if err != nil {
+		return
+	}
+	if accessErr := validateRemoteAccess(authenticated); accessErr != nil {
+		publishRemoteStatus(client, hubKey, configuration, request.SessionID, request.Kind, "error", accessErr.Error())
+		return
+	}
+
+	switch request.Kind {
+	case "logs":
+		err = remoteAccess.openLogs(client, hubKey, configuration, request)
+	case "shell":
+		err = remoteAccess.openShell(client, hubKey, configuration, request)
+	default:
+		err = errors.New("unsupported remote session kind")
+	}
+	if err != nil {
+		if errors.Is(err, errRemoteSessionExists) {
+			publishRemoteStatus(client, hubKey, configuration, request.SessionID, request.Kind, "opened", "")
+			return
+		}
+		publishRemoteStatus(client, hubKey, configuration, request.SessionID, request.Kind, "error", err.Error())
+	}
+}
+
+func HandleRemoteSessionInput(client paho.Client, hubKey string, payload models.Payload, authenticated bool, configuration *models.Configuration) {
+	if validateRemoteAccess(authenticated) != nil {
+		return
+	}
+	request, err := decodeRemotePayload(payload)
+	if err != nil || len(request.Data) > remoteInputLimit*2 {
+		return
+	}
+	data, err := base64.StdEncoding.DecodeString(request.Data)
+	if err != nil || len(data) > remoteInputLimit {
+		return
+	}
+	remoteAccess.mu.Lock()
+	session := remoteAccess.sessions[request.SessionID]
+	remoteAccess.mu.Unlock()
+	if session == nil || session.kind != "shell" || session.pty == nil {
+		publishRemoteStatus(client, hubKey, configuration, request.SessionID, "shell", "error", "remote session is not open")
+		return
+	}
+	if _, err := session.pty.Write(data); err != nil {
+		publishRemoteStatus(client, hubKey, configuration, request.SessionID, "shell", "error", "failed to write terminal input")
+	}
+}
+
+func HandleRemoteSessionResize(payload models.Payload, authenticated bool) {
+	if validateRemoteAccess(authenticated) != nil {
+		return
+	}
+	request, err := decodeRemotePayload(payload)
+	if err != nil {
+		return
+	}
+	rows, columns := normalizeTerminalSize(request.Rows, request.Columns)
+	remoteAccess.mu.Lock()
+	session := remoteAccess.sessions[request.SessionID]
+	remoteAccess.mu.Unlock()
+	if session != nil && session.kind == "shell" && session.pty != nil {
+		_ = pty.Setsize(session.pty, &pty.Winsize{Rows: rows, Cols: columns})
+	}
+}
+
+func HandleRemoteSessionClose(payload models.Payload, authenticated bool) {
+	if !authenticated {
+		return
+	}
+	request, err := decodeRemotePayload(payload)
+	if err == nil {
+		remoteAccess.close(request.SessionID)
+	}
+}
+
+func (manager *remoteAccessManager) reserve(session *remoteSession) ([]string, error) {
+	manager.mu.Lock()
+	defer manager.mu.Unlock()
+	if _, exists := manager.sessions[session.id]; exists {
+		return nil, errRemoteSessionExists
+	}
+	if len(manager.sessions) >= remoteSessionLimit {
+		return nil, errRemoteSessionLimit
+	}
+	manager.sessions[session.id] = session
+	history := append([]string(nil), manager.history...)
+	return history, nil
+}
+
+func (manager *remoteAccessManager) expire(client paho.Client, hubKey string, configuration *models.Configuration, session *remoteSession) {
+	session.timer = time.AfterFunc(remoteSessionLifetime, func() {
+		manager.close(session.id)
+		if session.kind == "logs" {
+			publishRemoteStatus(client, hubKey, configuration, session.id, session.kind, "closed", "session lifetime reached")
+		}
+	})
+}
+
+func (manager *remoteAccessManager) openLogs(client paho.Client, hubKey string, configuration *models.Configuration, request models.RemoteSessionPayload) error {
+	session := &remoteSession{
+		id:   request.SessionID,
+		kind: "logs",
+		logs: make(chan string, 256),
+		done: make(chan struct{}),
+	}
+	history, err := manager.reserve(session)
+	if err != nil {
+		return err
+	}
+	manager.expire(client, hubKey, configuration, session)
+	tail := request.Tail
+	if tail <= 0 || tail > remoteHistoryLimit {
+		tail = 200
+	}
+	if len(history) > tail {
+		history = history[len(history)-tail:]
+	}
+
+	publishRemoteStatus(client, hubKey, configuration, session.id, session.kind, "opened", "")
+	go func() {
+		for _, line := range history {
+			publishRemoteOutput(client, hubKey, configuration, session.id, session.kind, line)
+		}
+		for {
+			select {
+			case line := <-session.logs:
+				publishRemoteOutput(client, hubKey, configuration, session.id, session.kind, line)
+			case <-session.done:
+				return
+			}
+		}
+	}()
+	return nil
+}
+
+func (manager *remoteAccessManager) openShell(client paho.Client, hubKey string, configuration *models.Configuration, request models.RemoteSessionPayload) error {
+	rows, columns := normalizeTerminalSize(request.Rows, request.Columns)
+	ctx, cancel := context.WithCancel(context.Background())
+	session := &remoteSession{
+		id:     request.SessionID,
+		kind:   "shell",
+		cancel: cancel,
+		done:   make(chan struct{}),
+	}
+	if _, err := manager.reserve(session); err != nil {
+		cancel()
+		return err
+	}
+	manager.expire(client, hubKey, configuration, session)
+	command := exec.CommandContext(ctx, "/bin/sh")
+	command.Env = append(os.Environ(), "TERM=xterm-256color", "HISTFILE=/dev/null")
+	terminal, err := pty.StartWithSize(command, &pty.Winsize{Rows: rows, Cols: columns})
+	if err != nil {
+		manager.remove(session.id)
+		cancel()
+		return err
+	}
+	session.pty = terminal
+
+	publishRemoteStatus(client, hubKey, configuration, session.id, session.kind, "opened", "")
+	go manager.forwardShell(client, hubKey, configuration, session, command)
+	return nil
+}
+
+func (manager *remoteAccessManager) forwardShell(client paho.Client, hubKey string, configuration *models.Configuration, session *remoteSession, command *exec.Cmd) {
+	buffer := make([]byte, remoteOutputChunkSize)
+	for {
+		count, err := session.pty.Read(buffer)
+		if count > 0 {
+			publishRemoteOutput(client, hubKey, configuration, session.id, session.kind, base64.StdEncoding.EncodeToString(buffer[:count]))
+		}
+		if err != nil {
+			if !errors.Is(err, io.EOF) && !errors.Is(err, os.ErrClosed) {
+				publishRemoteStatus(client, hubKey, configuration, session.id, session.kind, "error", "terminal stream closed unexpectedly")
+			}
+			break
+		}
+	}
+	_ = command.Wait()
+	manager.remove(session.id)
+	publishRemoteStatus(client, hubKey, configuration, session.id, session.kind, "closed", "")
+}
+
+func (manager *remoteAccessManager) close(sessionID string) {
+	manager.mu.Lock()
+	session := manager.sessions[sessionID]
+	delete(manager.sessions, sessionID)
+	manager.mu.Unlock()
+	if session == nil {
+		return
+	}
+	if session.cancel != nil {
+		session.cancel()
+	}
+	if session.pty != nil {
+		_ = session.pty.Close()
+	}
+	if session.logs != nil {
+		close(session.done)
+	}
+	if session.timer != nil {
+		session.timer.Stop()
+	}
+}
+
+func (manager *remoteAccessManager) remove(sessionID string) {
+	manager.mu.Lock()
+	session := manager.sessions[sessionID]
+	delete(manager.sessions, sessionID)
+	manager.mu.Unlock()
+	if session != nil && session.timer != nil {
+		session.timer.Stop()
+	}
+}
+
+func publishRemoteStatus(client paho.Client, hubKey string, configuration *models.Configuration, sessionID string, kind string, state string, errorMessage string) {
+	status := models.RemoteSessionStatus{
+		Timestamp: time.Now().Unix(),
+		SessionID: sessionID,
+		Kind:      kind,
+		State:     state,
+		Error:     errorMessage,
+	}
+	value, _ := json.Marshal(status)
+	var statusValue map[string]interface{}
+	_ = json.Unmarshal(value, &statusValue)
+	publishRemote(client, hubKey, configuration, "remote-session-status", statusValue, 1)
+}
+
+func publishRemoteOutput(client paho.Client, hubKey string, configuration *models.Configuration, sessionID string, kind string, data string) {
+	publishRemote(client, hubKey, configuration, "remote-session-output", map[string]interface{}{
+		"timestamp":  time.Now().Unix(),
+		"session_id": sessionID,
+		"kind":       kind,
+		"data":       data,
+	}, 0)
+}
+
+func publishRemote(client paho.Client, hubKey string, configuration *models.Configuration, action string, value map[string]interface{}, qos byte) {
+	message := models.Message{Payload: models.Payload{
+		Action:   action,
+		DeviceId: configuration.Config.Key,
+		Value:    value,
+	}}
+	payload, err := models.PackageMQTTMessage(configuration, message)
+	if err == nil {
+		client.Publish("kerberos/hub/"+hubKey, qos, false, payload)
+	}
+}

machinery/src/routers/websocket/main.go

@@ -11,11 +11,11 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/gorilla/websocket"
 	"github.com/kerberos-io/agent/machinery/src/capture"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/kerberos-io/agent/machinery/src/utils"
 	"github.com/kerberos-io/agent/machinery/src/webrtc"
+	log "github.com/sirupsen/logrus"
 )
 
 type Message struct {
@@ -43,7 +43,7 @@ func writeWebRTCError(connection *Connection, clientID string, sessionID string,
 			"message":    errorMessage,
 		},
 	}); err != nil {
-		log.Log.Error("routers.websocket.main.writeWebRTCError(): " + err.Error())
+		log.Error("routers.websocket.main.writeWebRTCError(): " + err.Error())
 	}
 }
 
@@ -80,7 +80,7 @@ func WebsocketHandler(c *gin.Context, configuration *models.Configuration, commu
 		var message Message
 		err = conn.ReadJSON(&message)
 		if err != nil {
-			log.Log.Error("routers.websocket.main.WebsocketHandler(): " + err.Error())
+			log.Error("routers.websocket.main.WebsocketHandler(): " + err.Error())
 			return
 		}
 		clientID := message.ClientID
@@ -89,7 +89,7 @@ func WebsocketHandler(c *gin.Context, configuration *models.Configuration, commu
 			connection.Socket = conn
 			sockets[clientID] = connection
 			sockets[clientID].Cancels = make(map[string]context.CancelFunc)
-			log.Log.Info("routers.websocket.main.WebsocketHandler(): " + clientID + ": connected.")
+			log.Info("routers.websocket.main.WebsocketHandler(): " + clientID + ": connected.")
 		}
 
 		// Continuously read messages
@@ -111,14 +111,14 @@ func WebsocketHandler(c *gin.Context, configuration *models.Configuration, commu
 				if exists {
 					sockets[clientID].Cancels["stream-sd"]()
 				} else {
-					log.Log.Error("routers.websocket.main.WebsocketHandler(): streaming sd does not exists for " + clientID)
+					log.Error("routers.websocket.main.WebsocketHandler(): streaming sd does not exists for " + clientID)
 				}
 
 			case "stream-sd":
-				if communication.CameraConnected {
+				if communication.CameraConnected.Load() {
 					_, exists := sockets[clientID].Cancels["stream-sd"]
 					if exists {
-						log.Log.Debug("routers.websocket.main.WebsocketHandler(): already streaming sd for " + clientID)
+						log.Debug("routers.websocket.main.WebsocketHandler(): already streaming sd for " + clientID)
 					} else {
 						startStream := Message{
 							ClientID:    clientID,
@@ -144,16 +144,11 @@ func WebsocketHandler(c *gin.Context, configuration *models.Configuration, commu
 					break
 				}
 
-				if !communication.CameraConnected {
+				if !communication.CameraConnected.Load() {
 					writeWebRTCError(sockets[clientID], clientID, sessionID, "camera is not connected")
 					break
 				}
 
-				if communication.HandleLiveHDHandshake == nil {
-					writeWebRTCError(sockets[clientID], clientID, sessionID, "webrtc liveview is not available")
-					break
-				}
-
 				handshake := models.LiveHDHandshake{
 					Payload: models.RequestHDStreamPayload{
 						Timestamp:          time.Now().Unix(),
@@ -188,7 +183,9 @@ func WebsocketHandler(c *gin.Context, configuration *models.Configuration, commu
 					},
 				}
 
-				communication.HandleLiveHDHandshake <- handshake
+				if !communication.TrySendLiveHDHandshake(handshake) {
+					writeWebRTCError(sockets[clientID], clientID, handshake.Payload.SessionID, "camera is restarting or live-view queue is full")
+				}
 
 			case "webrtc-candidate":
 				sessionID := message.Message["session_id"]
@@ -199,7 +196,7 @@ func WebsocketHandler(c *gin.Context, configuration *models.Configuration, commu
 					break
 				}
 
-				if !communication.CameraConnected {
+				if !communication.CameraConnected.Load() {
 					writeWebRTCError(sockets[clientID], clientID, sessionID, "camera is not connected")
 					break
 				}
@@ -221,7 +218,7 @@ func WebsocketHandler(c *gin.Context, configuration *models.Configuration, commu
 		_, exists := sockets[clientID]
 		if exists {
 			delete(sockets, clientID)
-			log.Log.Info("routers.websocket.main.WebsocketHandler(): " + clientID + ": terminated and disconnected websocket connection.")
+			log.Info("routers.websocket.main.WebsocketHandler(): " + clientID + ": terminated and disconnected websocket connection.")
 		}
 	}
 }
@@ -232,13 +229,14 @@ func ForwardSDStream(ctx context.Context, clientID string, connection *Connectio
 	var cursor *packets.QueueCursor
 
 	// We'll pick the right client and decoder.
-	rtspClient := captureDevice.RTSPSubClient
+	rtspClient := captureDevice.SubClient()
 	if rtspClient != nil {
-		queue = communication.SubQueue
-		cursor = queue.Latest()
+		queue = communication.SubQueue()
 	} else {
-		rtspClient = captureDevice.RTSPClient
-		queue = communication.Queue
+		rtspClient = captureDevice.MainClient()
+		queue = communication.MainQueue()
+	}
+	if queue != nil {
 		cursor = queue.Latest()
 	}
 
@@ -263,7 +261,7 @@ logreader:
 					continue
 				}
 			} else {
-				log.Log.Error("routers.websocket.main.ForwardSDStream():" + err.Error())
+				log.Error("routers.websocket.main.ForwardSDStream():" + err.Error())
 				break logreader
 			}
 		}
@@ -277,7 +275,7 @@ logreader:
 		}
 		err := connection.WriteJson(startStrean)
 		if err != nil {
-			log.Log.Error("routers.websocket.main.ForwardSDStream():" + err.Error())
+			log.Error("routers.websocket.main.ForwardSDStream():" + err.Error())
 			break logreader
 		}
 		select {
@@ -292,9 +290,9 @@ logreader:
 	if exists {
 		delete(connection.Cancels, "stream-sd")
 	} else {
-		log.Log.Error("routers.websocket.main.ForwardSDStream(): streaming sd does not exists for " + clientID)
+		log.Error("routers.websocket.main.ForwardSDStream(): streaming sd does not exists for " + clientID)
 	}
 
 	// Send stop streaming message
-	log.Log.Info("routers.websocket.main.ForwardSDStream(): stop sending streaming over websocket")
+	log.Info("routers.websocket.main.ForwardSDStream(): stop sending streaming over websocket")
 }

machinery/src/utils/main.go

@@ -1,7 +1,6 @@
 package utils
 
 import (
-	"bufio"
 	"bytes"
 	"errors"
 	"fmt"
@@ -12,15 +11,14 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
-	"reflect"
 	"sort"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/encryption"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 
 	"github.com/nfnt/resize"
 )
@@ -140,11 +138,11 @@ func CheckDataDirectoryPermissions(configDirectory string) error {
 	}
 
 	if err != nil {
-		log.Log.Error("Checking data directory permissions: " + err.Error())
+		log.Error("Checking data directory permissions: " + err.Error())
 		return err
 	}
 
-	log.Log.Info("Checking data directory permissions: OK")
+	log.Info("Checking data directory permissions: OK")
 	return nil
 }
 
@@ -193,51 +191,32 @@ func GetMediaFormatted(files []os.FileInfo, recordingDirectory string, configura
 	count := 0
 	for _, file := range files {
 		fileName := file.Name()
-		fileParts := strings.Split(fileName, "_")
-		if len(fileParts) == 6 {
-			timestamp := fileParts[0]
-			timestampInt, err := strconv.ParseInt(timestamp, 10, 64)
-			if err == nil {
-
-				if eventFilter.TimestampOffsetStart > 0 {
-					// TimestampOffsetStart represents the newest lower bound to include.
-					if timestampInt < eventFilter.TimestampOffsetStart {
-						continue
-					}
-				}
-
-				// If we have an offset we will check if we should skip or not
-				if eventFilter.TimestampOffsetEnd > 0 {
-					// Medias are sorted from new to older. TimestampOffsetEnd holds the oldest
-					// timestamp of the previous batch of events. By doing this check, we make sure
-					// to skip the previous batch.
-					if timestampInt >= eventFilter.TimestampOffsetEnd {
-						continue
-					}
-				}
-
-				loc, _ := time.LoadLocation(configuration.Config.Timezone)
-				time := time.Unix(timestampInt, 0).In(loc)
-				day := time.Format("02-01-2006")
-				timeString := time.Format("15:04:05")
-				shortDay := time.Format("Jan _2")
-
-				media := models.Media{
-					Key:        fileName,
-					Path:       recordingDirectory + "/" + fileName,
-					CameraName: configuration.Config.Name,
-					CameraKey:  configuration.Config.Key,
-					Day:        day,
-					ShortDay:   shortDay,
-					Time:       timeString,
-					Timestamp:  timestamp,
-				}
-				filePaths = append(filePaths, media)
-				count = count + 1
-				if eventFilter.NumberOfElements > 0 && count >= eventFilter.NumberOfElements {
-					break
-				}
-			}
+		recordedAt, err := RecordingStartTime(fileName)
+		if err != nil {
+			continue
+		}
+		timestamp := recordedAt.Unix()
+		if eventFilter.TimestampOffsetStart > 0 && timestamp < eventFilter.TimestampOffsetStart {
+			continue
+		}
+		if eventFilter.TimestampOffsetEnd > 0 && timestamp >= eventFilter.TimestampOffsetEnd {
+			continue
+		}
+		loc, _ := time.LoadLocation(configuration.Config.Timezone)
+		localTime := recordedAt.In(loc)
+		filePaths = append(filePaths, models.Media{
+			Key:        fileName,
+			Path:       recordingDirectory + "/" + fileName,
+			CameraName: configuration.Config.Name,
+			CameraKey:  configuration.Config.Key,
+			Day:        localTime.Format("02-01-2006"),
+			ShortDay:   localTime.Format("Jan _2"),
+			Time:       localTime.Format("15:04:05"),
+			Timestamp:  strconv.FormatInt(timestamp, 10),
+		})
+		count++
+		if eventFilter.NumberOfElements > 0 && count >= eventFilter.NumberOfElements {
+			break
 		}
 	}
 	return filePaths
@@ -246,18 +225,12 @@ func GetMediaFormatted(files []os.FileInfo, recordingDirectory string, configura
 func GetDays(files []os.FileInfo, recordingDirectory string, configuration *models.Configuration) []string {
 	days := []string{}
 	for _, file := range files {
-		fileName := file.Name()
-		fileParts := strings.Split(fileName, "_")
-		if len(fileParts) == 6 {
-			timestamp := fileParts[0]
-			timestampInt, err := strconv.ParseInt(timestamp, 10, 64)
-			if err == nil {
-				loc, _ := time.LoadLocation(configuration.Config.Timezone)
-				time := time.Unix(timestampInt, 0).In(loc)
-				day := time.Format("02-01-2006")
-				days = append(days, day)
-			}
+		recordedAt, err := RecordingStartTime(file.Name())
+		if err != nil {
+			continue
 		}
+		loc, _ := time.LoadLocation(configuration.Config.Timezone)
+		days = append(days, recordedAt.In(loc).Format("02-01-2006"))
 	}
 	uniqueDays := Unique(days)
 	return uniqueDays
@@ -280,10 +253,13 @@ func NumberOfFilesInDirectory(path string) int {
 	return len(files)
 }
 
-// NumberOfMP4sInDirectory returns the count of all files with mp4 extension in current directory
+// NumberOfMP4sInDirectory counts MP4 files, including dated subdirectories.
 func NumberOfMP4sInDirectory(path string) int {
-	pattern := filepath.Join(path, "*.mp4")
-	files, _ := filepath.Glob(pattern)
+	files, err := ReadRecordingDirectory(path)
+	if err != nil {
+		log.WithError(err).Error("Could not count recordings")
+		return 0
+	}
 	return len(files)
 }
 
@@ -301,16 +277,16 @@ func CreateFragmentedMP4(fullName string, fragmentedDuration int64) {
 	// This timescale is crucial, as it should be the same as the one defined in JOY4.
 	cmd := exec.Command("mp4fragment", "--timescale", "10000000", "--fragment-duration", strconv.FormatInt(duration, 10), fullName, fullName+"f.mp4")
 	cmd.Dir = path
-	log.Log.Info(cmd.String())
+	log.Info(cmd.String())
 	var out bytes.Buffer
 	var stderr bytes.Buffer
 	cmd.Stdout = &out
 	cmd.Stderr = &stderr
 	err := cmd.Run()
 	if err != nil {
-		log.Log.Error(fmt.Sprint(err) + ": " + stderr.String())
+		log.Error(fmt.Sprint(err) + ": " + stderr.String())
 	} else {
-		log.Log.Info("Created Fragmented: " + out.String())
+		log.Info("Created Fragmented: " + out.String())
 	}
 
 	// We will swap the files.
@@ -319,66 +295,105 @@ func CreateFragmentedMP4(fullName string, fragmentedDuration int64) {
 }
 
 func PrintEnvironmentVariables() {
-	// Print environment variables that include "AGENT_" as a prefix.
-	environmentVariables := ""
-	for _, e := range os.Environ() {
-		if strings.Contains(e, "AGENT_") {
-			pair := strings.Split(e, "=")
-			environmentVariables = environmentVariables + pair[0] + "=" + pair[1] + " "
+	names := agentEnvironmentVariableNames(os.Environ())
+	log.WithFields(log.Fields{
+		"component":      "configuration",
+		"event":          "environment_loaded",
+		"variable_count": len(names),
+	}).Info("Agent environment loaded")
+	log.WithFields(log.Fields{
+		"component": "configuration",
+		"event":     "environment_details",
+		"variables": names,
+	}).Debug("Agent environment details")
+}
+
+func agentEnvironmentVariableNames(environment []string) []string {
+	names := make([]string, 0)
+	for _, entry := range environment {
+		name, _, found := strings.Cut(entry, "=")
+		if found && strings.HasPrefix(name, "AGENT_") {
+			names = append(names, name)
 		}
 	}
-	log.Log.Info("Printing out environmentVariables (AGENT_...): " + environmentVariables)
+	sort.Strings(names)
+	return names
 }
 
 func PrintConfiguration(configuration *models.Configuration) {
-	// We will print out the struct.
 	if configuration == nil {
-		log.Log.Info("Configuration is nil")
+		log.WithFields(log.Fields{
+			"component": "configuration",
+			"event":     "configuration_missing",
+		}).Warn("Agent configuration is nil")
 		return
 	}
-	config := configuration.Config
-	// Iterate over the struct and printout the values.
-	v := reflect.ValueOf(config)
-	typeOfS := v.Type()
-	configurationVariables := ""
-	for i := 0; i < v.NumField(); i++ {
-		key := typeOfS.Field(i).Name
-		value := v.Field(i).Interface()
-		// Convert to string.
-		configurationVariables = configurationVariables + key + ": " + fmt.Sprintf("%v", value) + " "
+
+	log.WithFields(configurationLogFields(configuration.Config)).
+		Info("Agent configuration loaded")
+	log.WithFields(configurationDebugLogFields(configuration.Config)).
+		Debug("Agent configuration details")
+}
+
+func configurationLogFields(config models.Config) log.Fields {
+	return log.Fields{
+		"agent_name":     config.Name,
+		"camera_type":    config.Capture.Name,
+		"cloud_provider": config.Cloud,
+		"component":      "configuration",
+		"event":          "configuration_loaded",
+		"friendly_name":  config.FriendlyName,
+		"liveview":       config.Capture.Liveview,
+		"offline":        config.Offline,
+		"recording":      config.Capture.Recording,
+		"timezone":       config.Timezone,
+	}
+}
+
+func configurationDebugLogFields(config models.Config) log.Fields {
+	return log.Fields{
+		"auto_clean":             config.AutoClean,
+		"component":              "configuration",
+		"continuous":             config.Capture.Continuous,
+		"event":                  "configuration_details",
+		"gop_size":               config.Capture.GopSize,
+		"main_fps":               config.Capture.IPCamera.FPS,
+		"main_height":            config.Capture.IPCamera.Height,
+		"main_stream_configured": config.Capture.IPCamera.RTSP != "",
+		"main_width":             config.Capture.IPCamera.Width,
+		"max_recording_seconds":  config.Capture.MaxLengthRecording,
+		"motion":                 config.Capture.Motion,
+		"onvif_configured":       config.Capture.IPCamera.ONVIFXAddr != "",
+		"onvif_motion":           config.Capture.ONVIFMotion,
+		"post_recording_seconds": config.Capture.PostRecording,
+		"pre_recording_seconds":  config.Capture.PreRecording,
+		"remove_after_upload":    config.RemoveAfterUpload,
+		"snapshots":              config.Capture.Snapshots,
+		"sub_fps":                config.Capture.IPCamera.SubFPS,
+		"sub_height":             config.Capture.IPCamera.SubHeight,
+		"sub_stream_configured":  config.Capture.IPCamera.SubRTSP != "",
+		"sub_width":              config.Capture.IPCamera.SubWidth,
+		"transcoding_webrtc":     config.Capture.TranscodingWebRTC,
 	}
-	log.Log.Info("Printing our configuration (config.json): " + configurationVariables)
 }
 
 func Decrypt(directoryOrFile string, symmetricKey []byte) {
 	// Check if file or directory
 	fileInfo, err := os.Stat(directoryOrFile)
 	if err != nil {
-		log.Log.Fatal(err.Error())
+		log.Fatal(err.Error())
 		return
 	}
 
 	var files []string
 	if fileInfo.IsDir() {
-		// Create decrypted directory
-		err = os.MkdirAll(directoryOrFile+"/decrypted", 0755)
+		dir, err := ReadRecordingDirectory(directoryOrFile)
 		if err != nil {
-			log.Log.Fatal(err.Error())
-			return
-		}
-		dir, err := os.ReadDir(directoryOrFile)
-		if err != nil {
-			log.Log.Fatal(err.Error())
+			log.Fatal(err.Error())
 			return
 		}
 		for _, file := range dir {
-			// Check if file is not a directory
-			if !file.IsDir() {
-				// Check if an mp4 file
-				if strings.HasSuffix(file.Name(), ".mp4") {
-					files = append(files, directoryOrFile+"/"+file.Name())
-				}
-			}
+			files = append(files, filepath.Join(directoryOrFile, file.Name()))
 		}
 	} else {
 		files = append(files, directoryOrFile)
@@ -390,25 +405,24 @@ func Decrypt(directoryOrFile string, symmetricKey []byte) {
 		// Read file
 		content, err := os.ReadFile(file)
 		if err != nil {
-			log.Log.Fatal(err.Error())
+			log.Fatal(err.Error())
 			return
 		}
 		// Decrypt using AES key
 		decrypted, err := encryption.AesDecrypt(content, string(symmetricKey))
 		if err != nil {
-			log.Log.Fatal("Something went wrong while decrypting: " + err.Error())
+			log.Fatal("Something went wrong while decrypting: " + err.Error())
 			return
 		}
 
-		// Write decrypted content to file with appended .decrypted
-		// Get filename split by / and get last element.
-		fileParts := strings.Split(file, "/")
-		fileName := fileParts[len(fileParts)-1]
-		pathToFile := strings.Join(fileParts[:len(fileParts)-1], "/")
-
-		err = os.WriteFile(pathToFile+"/decrypted/"+fileName, []byte(decrypted), 0644)
+		outputDirectory := filepath.Join(filepath.Dir(file), "decrypted")
+		if err := os.MkdirAll(outputDirectory, 0o755); err != nil {
+			log.Fatal(err.Error())
+			return
+		}
+		err = os.WriteFile(filepath.Join(outputDirectory, filepath.Base(file)), []byte(decrypted), 0644)
 		if err != nil {
-			log.Log.Fatal(err.Error())
+			log.Fatal(err.Error())
 			return
 		}
 	}
@@ -416,9 +430,12 @@ func Decrypt(directoryOrFile string, symmetricKey []byte) {
 
 func ImageToBytes(img *image.Image) ([]byte, error) {
 	buffer := new(bytes.Buffer)
-	w := bufio.NewWriter(buffer)
-	err := jpeg.Encode(w, *img, &jpeg.Options{Quality: 35})
-	log.Log.Debug("ImageToBytes() - buffer size: " + strconv.Itoa(buffer.Len()))
+	err := jpeg.Encode(buffer, *img, &jpeg.Options{Quality: 35})
+	log.WithFields(log.Fields{
+		"bytes":     buffer.Len(),
+		"component": "image",
+		"event":     "jpeg_encoded",
+	}).Debug("Image encoded as JPEG")
 	return buffer.Bytes(), err
 }
 

machinery/src/utils/main_test.go

@@ -1,13 +1,112 @@
 package utils
 
 import (
+	"bytes"
+	"fmt"
+	"image"
+	"image/color"
+	"image/jpeg"
 	"os"
+	"reflect"
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/models"
 )
 
+func TestImageToBytesReturnsCompleteJPEG(t *testing.T) {
+	source := image.NewRGBA(image.Rect(0, 0, 16, 12))
+	source.Set(8, 6, color.RGBA{R: 255, A: 255})
+	var input image.Image = source
+
+	encoded, err := ImageToBytes(&input)
+	if err != nil {
+		t.Fatalf("ImageToBytes() error = %v", err)
+	}
+	if len(encoded) == 0 {
+		t.Fatal("ImageToBytes() returned an empty JPEG")
+	}
+
+	decoded, err := jpeg.Decode(bytes.NewReader(encoded))
+	if err != nil {
+		t.Fatalf("decoding ImageToBytes() output: %v", err)
+	}
+	if got := decoded.Bounds().Size(); got.X != 16 || got.Y != 12 {
+		t.Fatalf("decoded JPEG size = %dx%d, want 16x12", got.X, got.Y)
+	}
+}
+
+func TestAgentEnvironmentVariableNamesOmitsValues(t *testing.T) {
+	got := agentEnvironmentVariableNames([]string{
+		"AGENT_HUB_PRIVATE_KEY=do-not-log",
+		"PATH=/usr/bin",
+		"NOT_AGENT_SECRET=also-do-not-log",
+		"AGENT_CAPTURE_LIVEVIEW=true",
+	})
+	want := []string{"AGENT_CAPTURE_LIVEVIEW", "AGENT_HUB_PRIVATE_KEY"}
+
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("agentEnvironmentVariableNames() = %v, want %v", got, want)
+	}
+}
+
+func TestConfigurationLogFieldsOmitCredentialsAndEndpoints(t *testing.T) {
+	config := models.Config{
+		Name:          "Front Door",
+		FriendlyName:  "Entrance",
+		Cloud:         "kstorage",
+		HubKey:        "hub-key-secret",
+		HubPrivateKey: "hub-private-secret",
+		MQTTURI:       "mqtt://internal.example",
+		MQTTUsername:  "mqtt-user",
+		MQTTPassword:  "mqtt-password-secret",
+		Capture: models.Capture{
+			Liveview:           "true",
+			MaxLengthRecording: 20,
+			IPCamera: models.IPCamera{
+				RTSP:          "rtsp://camera-user:camera-password@10.0.30.11/live",
+				ONVIFUsername: "onvif-user",
+				ONVIFPassword: "onvif-password-secret",
+			},
+		},
+	}
+
+	fields := []map[string]interface{}{
+		configurationLogFields(config),
+		configurationDebugLogFields(config),
+	}
+	summary := fmt.Sprint(fields)
+	for _, secret := range []string{
+		config.HubKey,
+		config.HubPrivateKey,
+		config.MQTTURI,
+		config.MQTTUsername,
+		config.MQTTPassword,
+		config.Capture.IPCamera.RTSP,
+		config.Capture.IPCamera.ONVIFUsername,
+		config.Capture.IPCamera.ONVIFPassword,
+	} {
+		if strings.Contains(summary, secret) {
+			t.Fatalf("configuration log fields exposed sensitive value %q in %q", secret, summary)
+		}
+	}
+	infoFields := configurationLogFields(config)
+	if got := infoFields["agent_name"]; got != "Front Door" {
+		t.Fatalf("agent_name = %v, want Front Door", got)
+	}
+	if got := infoFields["cloud_provider"]; got != "kstorage" {
+		t.Fatalf("cloud_provider = %v, want kstorage", got)
+	}
+	debugFields := configurationDebugLogFields(config)
+	if got := debugFields["max_recording_seconds"]; got != int64(20) {
+		t.Fatalf("max_recording_seconds = %v, want 20", got)
+	}
+	if got := debugFields["main_stream_configured"]; got != true {
+		t.Fatalf("main_stream_configured = %v, want true", got)
+	}
+}
+
 type stubFileInfo struct {
 	name string
 }

machinery/src/utils/recordings.go

@@ -0,0 +1,146 @@
+package utils
+
+import (
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+)
+
+func RecordingFileName(timestamp int64, deviceKey string) (string, error) {
+	if timestamp <= 0 {
+		return "", fmt.Errorf("invalid recording timestamp %d", timestamp)
+	}
+	if deviceKey == "" || strings.ContainsAny(deviceKey, "/\\") {
+		return "", fmt.Errorf("missing or invalid recording device key")
+	}
+	return strconv.FormatInt(timestamp, 10) + "-" + deviceKey + ".mp4", nil
+}
+
+func RecordingStartTime(name string) (time.Time, error) {
+	base := filepath.Base(name)
+	if !strings.HasSuffix(base, ".mp4") {
+		return time.Time{}, fmt.Errorf("invalid recording filename %q", name)
+	}
+	timestamp, deviceKey, short := strings.Cut(strings.TrimSuffix(base, ".mp4"), "-")
+	if short && deviceKey != "" {
+		if millis, err := strconv.ParseInt(timestamp, 10, 64); err == nil && millis > 0 {
+			return time.UnixMilli(millis), nil
+		}
+	}
+	parts := strings.Split(base, "_")
+	if len(parts) == 6 {
+		seconds, err := strconv.ParseInt(parts[0], 10, 64)
+		if err == nil && seconds > 0 {
+			return time.Unix(seconds, 0), nil
+		}
+	}
+	return time.Time{}, fmt.Errorf("invalid recording filename %q", name)
+}
+
+func CreateRecordingPath(directory, fileName string, recordedAt time.Time) (string, error) {
+	if !filepath.IsLocal(fileName) || fileName == "." || filepath.Base(fileName) != fileName || strings.ContainsAny(fileName, "/\\") {
+		return "", fmt.Errorf("invalid recording filename %q", fileName)
+	}
+	if recordedAt.UnixMilli() <= 0 {
+		return "", fmt.Errorf("invalid recording start time %s", recordedAt)
+	}
+	directory = filepath.Join(directory, recordedAt.UTC().Format("2006/01/02"))
+	if err := os.MkdirAll(directory, 0o755); err != nil {
+		return "", fmt.Errorf("create recording directory: %w", err)
+	}
+	return filepath.Join(directory, fileName), nil
+}
+
+// ResolveRecordingPath accepts relative paths from listings and historical
+// basename-only references from upload markers and playback URLs.
+func ResolveRecordingPath(directory, name string) (string, error) {
+	if !filepath.IsLocal(name) {
+		return "", fmt.Errorf("invalid recording path %q", name)
+	}
+	filePath := filepath.Join(directory, name)
+	info, err := os.Stat(filePath)
+	if os.IsNotExist(err) && filepath.Base(name) == name {
+		recordedAt, parseErr := RecordingStartTime(name)
+		if parseErr == nil {
+			filePath = filepath.Join(directory, recordedAt.UTC().Format("2006/01/02"), name)
+			info, err = os.Stat(filePath)
+		}
+	}
+	if err != nil {
+		return "", err
+	}
+	if !info.Mode().IsRegular() {
+		return "", fmt.Errorf("recording %q is not a regular file", name)
+	}
+	return filePath, nil
+}
+
+func OpenRecording(directory, name string) (*os.File, error) {
+	filePath, err := ResolveRecordingPath(directory, name)
+	if err != nil {
+		return nil, err
+	}
+	return os.Open(filePath)
+}
+
+func RemoveRecording(directory, name string) error {
+	filePath, err := ResolveRecordingPath(directory, name)
+	if err != nil {
+		return err
+	}
+	return os.Remove(filePath)
+}
+
+type recordingFileInfo struct {
+	os.FileInfo
+	relativePath string
+}
+
+func (file recordingFileInfo) Name() string {
+	return file.relativePath
+}
+
+// ReadRecordingDirectory returns MP4s with root-relative names, preserving the
+// file information used by the existing sorting and media formatting helpers.
+func ReadRecordingDirectory(directory string) ([]os.FileInfo, error) {
+	return readRecordingDirectory(directory, filepath.WalkDir)
+}
+
+func readRecordingDirectory(directory string, walkDir func(string, fs.WalkDirFunc) error) ([]os.FileInfo, error) {
+	files := []os.FileInfo{}
+	err := walkDir(directory, func(filePath string, entry fs.DirEntry, err error) error {
+		if err != nil {
+			if os.IsNotExist(err) && filePath != directory {
+				return nil
+			}
+			return err
+		}
+		if entry.IsDir() && entry.Name() == "decrypted" && filePath != directory {
+			return filepath.SkipDir
+		}
+		if !entry.Type().IsRegular() || filepath.Ext(entry.Name()) != ".mp4" {
+			return nil
+		}
+		info, err := entry.Info()
+		if err != nil {
+			if os.IsNotExist(err) {
+				return nil
+			}
+			return err
+		}
+		name, err := filepath.Rel(directory, filePath)
+		if err != nil {
+			return err
+		}
+		files = append(files, recordingFileInfo{FileInfo: info, relativePath: filepath.ToSlash(name)})
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return files, nil
+}