feat: add frame processor reference service

feature/add-encrypted-metadata-field

.circleci/config.yml

@@ -1,32 +0,0 @@
-# Golang CircleCI 2.0 configuration file
-#
-# Check https://circleci.com/docs/2.0/language-go/ for more details
-version: 2
-jobs:
-  machinery:
-    docker:
-      - image: kerberos/base:0a50dc9
-    working_directory: /go/src/github.com/{{ORG_NAME}}/{{REPO_NAME}}
-    steps:
-      - checkout
-      - run: apt-get update && apt-get install -y --no-install-recommends libavcodec-dev libavformat-dev libswscale-dev libjpeg62-turbo-dev
-      - run: cd machinery && go mod download
-      - run: cd machinery && go test -v ./...
-      - run: cd machinery && go vet
-
-  ui:
-    docker:
-      - image: cimg/python:3.9.13-node
-    steps:
-      - checkout
-      - run: node --version
-      - run: cd ui && yarn
-      - run: cd ui && yarn test --passWithNoTests
-      - run: cd ui && yarn build
-
-workflows:
-  version: 2
-  build:
-    jobs:
-      - machinery
-      - ui

.github/workflows/go.yml

@@ -10,24 +10,29 @@ jobs:
   build:
     name: Build
     runs-on: ubuntu-latest
-    container:
-      image: kerberos/base:eb6b088
-
-    strategy:
-      matrix:
-        #No longer supported Go versions.
-        #go-version: ['1.17', '1.18', '1.19', '1.20', '1.21']
-        go-version: ["1.24"]
 
     steps:
-      - name: Set up Go ${{ matrix.go-version }}
-        uses: actions/setup-go@v2
-        with:
-          go-version: ${{ matrix.go-version }}
       - name: Check out code into the Go module directory
         uses: actions/checkout@v4
-      - name: Set up git ownershi
+      - name: Set up Go
-        run: git config --system --add safe.directory /__w/agent/agent
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.25.x"
+          check-latest: true
+          cache: true
+          cache-dependency-path: |
+            machinery/go.sum
+            examples/frame-processor/go.sum
+      - name: Install native dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential \
+            libavcodec-dev \
+            libavutil-dev \
+            libswresample-dev \
+            libswscale-dev \
+            pkg-config
       - name: Get dependencies
         run: cd machinery && go mod download
       - name: Build
@@ -36,3 +41,5 @@ jobs:
         run: cd machinery && go vet -v ./...
       - name: Test
         run: cd machinery && go test -v ./...
+      - name: Test frame processor example
+        run: cd examples/frame-processor && GOWORK=off go test -v ./...

.github/workflows/nightly-build.yml

@@ -15,7 +15,7 @@ jobs:
         architecture: [amd64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
@@ -24,13 +24,14 @@ jobs:
         with:
           ref: master
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        id: buildx
+        uses: docker/setup-buildx-action@v3
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
       - name: Run Buildx
-        run: docker buildx build --platform linux/${{matrix.architecture}} -t kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7) --push .
+        run: docker buildx build --build-arg VERSION=$(echo $GITHUB_SHA | cut -c1-7) --platform linux/${{matrix.architecture}} -t kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7) --push .
       - name: Create new and append to manifest
         run: docker buildx imagetools create -t kerberos/agent-nightly:$(echo $GITHUB_SHA | cut -c1-7) kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7)
   nightly-build-other:
@@ -40,7 +41,7 @@ jobs:
         architecture: [arm64, arm/v7, arm/v6]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
@@ -49,12 +50,13 @@ jobs:
         with:
           ref: master
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        id: buildx
+        uses: docker/setup-buildx-action@v3
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
       - name: Run Buildx
-        run: docker buildx build --platform linux/${{matrix.architecture}} -t kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7) --push .
+        run: docker buildx build --build-arg VERSION=$(echo $GITHUB_SHA | cut -c1-7) --platform linux/${{matrix.architecture}} -t kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7) --push .
       - name: Create new and append to manifest
         run: docker buildx imagetools create --append -t kerberos/agent-nightly:$(echo $GITHUB_SHA | cut -c1-7) kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7)

.github/workflows/pr-build.yml

@@ -20,19 +20,15 @@ jobs:
             runner: ubuntu-24.04-arm
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-      - uses: benjlevesque/short-sha@v2.1
-        id: short-sha
-        with:
-          length: 7
       - name: Run Build
         run: |
-          docker build -t ${{ matrix.architecture }} .
+          docker build --build-arg VERSION=$(echo "${{ github.event.pull_request.head.sha }}" | cut -c1-7) -t ${{ matrix.architecture }} .
           CID=$(docker create ${{matrix.architecture}})
           docker cp ${CID}:/home/agent ./output-${{matrix.architecture}}
           docker rm ${CID}
@@ -43,4 +39,3 @@ jobs:
         with:
           name: agent-${{matrix.architecture}}.tar
           path: agent-${{matrix.architecture}}.tar
-

.github/workflows/react.yml

@@ -11,21 +11,22 @@ on:
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     strategy:
       matrix:
-        node-version: [14.x, 16.x]
+        node-version: [22.x]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v2
+      uses: actions/setup-node@v4
       with:
         node-version: ${{ matrix.node-version }}
+        cache: yarn
+        cache-dependency-path: ui/yarn.lock
     - name: Yarn install
-      run: cd ui && yarn
+      run: cd ui && yarn --frozen-lockfile
     - name: Yarn test
       run: cd ui && yarn test --passWithNoTests
     - name: Yarn build

.github/workflows/release-bump.yml

@@ -58,16 +58,12 @@ jobs:
         architecture: [amd64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
-      - uses: benjlevesque/short-sha@v2.1
-        id: short-sha
-        with:
-          length: 7
       - name: Run Build
         run: |
           docker build --provenance=false --build-arg VERSION=${{ needs.bump-release.outputs.tag }} -t ${{matrix.architecture}} .
@@ -96,16 +92,12 @@ jobs:
         architecture: [arm64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
-      - uses: benjlevesque/short-sha@v2.1
-        id: short-sha
-        with:
-          length: 7
       - name: Run Build
         run: |
           docker build --provenance=false --build-arg VERSION=${{ needs.bump-release.outputs.tag }} -t ${{matrix.architecture}} .
@@ -129,7 +121,7 @@ jobs:
     needs: [bump-release, build-amd64, build-arm64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/release-create.yml

@@ -22,16 +22,12 @@ jobs:
         architecture: [amd64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
-      - uses: benjlevesque/short-sha@v2.1
-        id: short-sha
-        with:
-          length: 7
       - name: Run Build
         run: |
           docker build --provenance=false --build-arg VERSION=${{github.event.inputs.tag || github.ref_name}} -t ${{matrix.architecture}} .
@@ -59,16 +55,12 @@ jobs:
         architecture: [arm64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
-      - uses: benjlevesque/short-sha@v2.1
-        id: short-sha
-        with:
-          length: 7
       - name: Run Build
         run: |
           docker build --provenance=false --build-arg VERSION=${{github.event.inputs.tag || github.ref_name}} -t ${{matrix.architecture}} .
@@ -92,7 +84,7 @@ jobs:
     needs: [build-amd64, build-arm64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}

Dockerfile

@@ -26,29 +26,27 @@ RUN apt-get update && apt-get install -y --fix-missing --no-install-recommends \
 	rm -rf /var/lib/apt/lists/*
 
 ##############################################################################
-# Copy all the relevant source code in the Docker image, so we can build this.
+# Copy dependency metadata first so module downloads can be cached separately.
 
 RUN mkdir -p /go/src/github.com/kerberos-io/agent
-COPY machinery /go/src/github.com/kerberos-io/agent/machinery
+WORKDIR /go/src/github.com/kerberos-io/agent/machinery
-RUN rm -rf /go/src/github.com/kerberos-io/agent/machinery/.env
+COPY machinery/go.mod machinery/go.sum ./
+RUN go mod download
 
-##################################################################
+##############################################################################
-# Get the latest commit hash, so we know which version we're running
+# Copy the rest of the source after dependencies are primed.
-COPY .git /go/src/github.com/kerberos-io/agent/.git
+
-RUN cd /go/src/github.com/kerberos-io/agent/.git && git log --format="%H" -n 1 | head -c7 > /go/src/github.com/kerberos-io/agent/machinery/version
+COPY machinery ./
-RUN cat /go/src/github.com/kerberos-io/agent/machinery/version
+RUN rm -rf .env
 
 ##################
 # Build Machinery
 
-RUN cd /go/src/github.com/kerberos-io/agent/machinery && \
+RUN RESOLVED_VERSION="${VERSION:-0.0.0}" && \
-	go mod download && \
+	printf '%s' "${RESOLVED_VERSION}" > version && \
-	if [ -z "${VERSION}" ] || [ "${VERSION}" = "0.0.0" ]; then \
-		VERSION=$(cd /go/src/github.com/kerberos-io/agent && git describe --tags --always 2>/dev/null || echo "0.0.0"); \
-	fi && \
 	BUILD_TAGS=timetzdata,netgo,osusergo && \
 	case "${TARGETARCH:-$(go env GOARCH)}" in amd64|arm64) BUILD_TAGS="moq,${BUILD_TAGS}" ;; esac && \
-	go build -tags "${BUILD_TAGS}" --ldflags "-s -w -X github.com/kerberos-io/agent/machinery/src/utils.VERSION=${VERSION}" main.go && \
+	go build -o main -tags "${BUILD_TAGS}" --ldflags "-s -w -X github.com/kerberos-io/agent/machinery/src/utils.VERSION=${RESOLVED_VERSION}" . && \
 	mkdir -p /agent && \
 	mv main /agent && \
 	mv version /agent && \
@@ -80,9 +78,11 @@ RUN apk update && apk upgrade --available && sync
 # Build Web (React app)
 
 RUN mkdir -p /go/src/github.com/kerberos-io/agent/machinery/www
-COPY ui /go/src/github.com/kerberos-io/agent/ui
+WORKDIR /go/src/github.com/kerberos-io/agent/ui
-RUN cd /go/src/github.com/kerberos-io/agent/ui && rm -rf yarn.lock && yarn config set network-timeout 300000 && \
+COPY ui/package.json ui/yarn.lock ./
-	yarn && yarn build
+RUN yarn config set network-timeout 300000 && yarn --frozen-lockfile
+COPY ui ./
+RUN yarn build
 
 ####################################
 # Let's create a /dist folder containing just the files necessary for runtime.
@@ -146,10 +146,10 @@ EXPOSE 80
 ######################################
 # Check if agent is still running
 
-HEALTHCHECK CMD curl --fail http://localhost:80 || exit 1   
+HEALTHCHECK CMD curl --fail "http://localhost:${AGENT_PORT:-80}/health" || exit 1
 
 ###################################################
 # Leeeeettttt'ssss goooooo!!!
 # Run the shizzle from the right working directory.
 WORKDIR /home/agent
-CMD ["./main", "-action", "run", "-port", "80"]
+CMD ["./main", "-action", "run"]

README.md

@@ -3,7 +3,6 @@
 <a target="_blank" href="https://kerberos.io"><img src="https://img.shields.io/badge/kerberos-website-gray.svg?longCache=true&colorB=brightgreen" alt="Kerberos Agent"></a>
 <a target="_blank" href="https://doc.kerberos.io"><img src="https://img.shields.io/badge/kerberos-documentation-gray.svg?longCache=true&colorB=brightgreen" alt="Kerberos Agent"></a>
 
-<a target="_blank" href="https://circleci.com/gh/kerberos-io/agent"><img src="https://circleci.com/gh/kerberos-io/agent.svg?style=svg"/></a>
 <img src="https://github.com/kerberos-io/agent/workflows/Go/badge.svg"/>
 <img src="https://github.com/kerberos-io/agent/workflows/React/badge.svg"/>
 <img src="https://github.com/kerberos-io/agent/workflows/CodeQL/badge.svg"/>
@@ -189,6 +188,103 @@ Next to attaching the configuration file, it is also possible to override the co
     -e AGENT_CAPTURE_CONTINUOUS=true \
     -d --restart=always kerberos/agent:latest
 
+Set `AGENT_PORT` to change the Agent's internal HTTP port. The environment
+variable takes precedence over `-port`. When using Docker, publish the same
+container port:
+
+```bash
+docker run -p 8082:8082 \
+  -e AGENT_PORT=8082 \
+  --name mycamera \
+  -d --restart=always kerberos/agent:latest
+```
+
+### Health checks and API responses
+
+`GET /health` is an unauthenticated liveness and diagnostics endpoint for
+container orchestrators and external monitors. `healthy` means that the Agent
+HTTP process can serve requests. Camera streams and Hub connectivity are
+reported independently in the response, so an external camera or Hub outage
+does not make Docker restart an otherwise functioning Agent process. Successful
+probes return `200 OK` and are logged at debug level to avoid periodic health
+checks filling production logs.
+
+```bash
+curl http://localhost:8082/health
+```
+
+```json
+{
+  "httpStatusCode": 200,
+  "applicationStatusCode": "get_success",
+  "entityStatusCode": "healthy",
+  "message": "Healthy",
+  "metadata": {
+    "applicationName": "agent",
+    "applicationVersion": "0.0.0",
+    "timestamp": 1788710400,
+    "path": "/health"
+  },
+  "data": {
+    "health": {
+      "description": "Agent HTTP service is healthy",
+      "cameraConnected": true,
+      "mainStream": {
+        "configured": true,
+        "connected": true,
+        "packagesProcessed": 183421,
+        "fps": 29.97,
+        "resolution": {
+          "width": 1920,
+          "height": 1080
+        },
+        "lastPacketAt": 1788710399
+      },
+      "subStream": {
+        "configured": true,
+        "connected": true,
+        "packagesProcessed": 91710,
+        "fps": 15,
+        "resolution": {
+          "width": 640,
+          "height": 360
+        },
+        "lastPacketAt": 1788710399
+      },
+      "hub": {
+        "configured": true,
+        "connected": true,
+        "lastHeartbeatAttemptAt": 1788710398,
+        "lastSuccessfulHeartbeatAt": 1788710398
+      }
+    }
+  }
+}
+```
+
+Stream `packagesProcessed` counts complete encoded video access units processed
+since the Agent process started; it does not count individual fragmented RTP
+packets. `fps` is the latest PTS-derived frame-rate estimate. Resolution is the
+most recently observed encoded width and height. `lastPacketAt`, Hub heartbeat
+timestamps, and response metadata timestamps are Unix seconds; `0` means that
+no value has been observed yet. A stream can be configured but temporarily
+disconnected.
+
+Hub `configured` means heartbeat delivery is enabled and has the required URI
+and key. Hub `connected` means the most recent heartbeat succeeded and the last
+success is no more than three minutes old. A failed heartbeat marks it
+disconnected immediately while retaining the last-success timestamp for
+diagnosis.
+
+New JSON endpoints should use this Hub-compatible response envelope rather than
+adding unrelated top-level response shapes. HTTP status describes the transport
+result, `applicationStatusCode` describes the operation, `entityStatusCode` and
+`message` describe the domain outcome, `metadata` carries request/application
+context, and endpoint-specific content belongs in a typed `data` object. Client
+responses must contain safe messages only; detailed internal errors belong in
+structured logs. Existing endpoints retain their legacy response shapes until
+they are migrated deliberately.
+
 ### Secure camera streams (RTSPS)
 
 The Agent accepts `rtsps://` camera URLs. Do not use `srtsp://`; RTSPS is RTSP over TLS. For a Bosch FLEXIDOME micro 3100i, enable **Secure RTSP** under **Network > Network Services** and use port `9554`:
@@ -208,6 +304,7 @@ See [RTSPS and TLS certificates](README-RTSPS-TLS.md) for the complete Bosch UI,
 | --------------------------------------- | ----------------------------------------------------------------------------------------------- | ------------------------------ |
 | `LOG_LEVEL`                                 | Level for logging, could be "info", "warning", "debug", "error" or "fatal".                     | "info"                         |
 | `LOG_OUTPUT`                                | Logging output format "json" or "text".                                                         | "text"                         |
+| `AGENT_PORT`                                | HTTP web server port. Overrides the `-port` command-line value and must be between 1 and 65535. | "80"                           |
 | `AGENT_MODE`                                | You can choose to run this in 'release' for production, and or 'demo' for showcasing.           | "release"                      |
 | `AGENT_TLS_INSECURE`                        | Specify if you want to use `InsecureSkipVerify` for the internal HTTP client.                   | "false"                        |
 | `AGENT_USERNAME`                            | The username used to authenticate against the Kerberos Agent login page.                        | "root"                         |
@@ -244,9 +341,10 @@ See [RTSPS and TLS certificates](README-RTSPS-TLS.md) for the complete Bosch UI,
 | `AGENT_CAPTURE_PIXEL_CHANGE`                | If `CONTINUOUS` set to `false`, the number of pixel require to change before motion triggers.   | "150"                          |
 | `AGENT_CAPTURE_FRAGMENTED`                  | Set the format of the recorded MP4 to fragmented (suitable for HLS).                            | "false"                        |
 | `AGENT_CAPTURE_FRAGMENTED_DURATION`         | If `AGENT_CAPTURE_FRAGMENTED` set to `true`, define the duration (seconds) of a fragment.       | "8"                            |
-| `AGENT_MQTT_URI`                            | An MQTT broker endpoint that is used for bi-directional communication (live view, onvif, etc)   | "tcp://mqtt.kerberos.io:1883"  |
+| `AGENT_MQTT_URI`                            | MQTT broker endpoint for bi-directional communication. Accepts ActiveMQ `mqtt+ssl://` URLs.     | "tcp://mqtt.kerberos.io:1883"  |
 | `AGENT_MQTT_USERNAME`                       | Username of the MQTT broker.                                                                    | ""                             |
 | `AGENT_MQTT_PASSWORD`                       | Password of the MQTT broker.                                                                    | ""                             |
+| `AGENT_REMOTE_ACCESS_ENABLED`               | Allow encrypted Hub MQTT sessions to stream Agent logs and open an interactive shell. Enable only for trusted deployments. | "false"                        |
 | `AGENT_REALTIME_PROCESSING`                 | If `AGENT_REALTIME_PROCESSING` set to `true`, the agent will send key frames to the topic       | ""                             |
 | `AGENT_REALTIME_PROCESSING_TOPIC`           | The topic to which keyframes will be sent in base64 encoded format.                             | ""                             |
 | `AGENT_STUN_URI`                            | When using WebRTC, you'll need to provide a STUN server.                                        | "stun:turn-fra1.kerberos.io:3478"|
@@ -261,6 +359,7 @@ See [RTSPS and TLS certificates](README-RTSPS-TLS.md) for the complete Bosch UI,
 | `AGENT_HUB_PRIVATE_KEY`                     | The secret access key linked to your account in Kerberos Hub.                                   | ""                             |
 | `AGENT_HUB_REGION`                          | The Kerberos Hub region, to which you want to upload.                                           | ""                             |
 | `AGENT_HUB_SITE`                            | The site ID of a site you've created in your Kerberos Hub account.                              | ""                             |
+| `AGENT_TUS_CHUNK_SIZE_BYTES`                | Bytes sent in each resumable-upload PATCH. Set to `0` or a negative value to send all remaining bytes in one PATCH. | "8388608" (8 MiB)              |
 | `AGENT_KERBEROSVAULT_URI`                   | The Kerberos Vault API url.                                                                     | "https://vault.domain.com/api" |
 | `AGENT_KERBEROSVAULT_ACCESS_KEY`            | The access key of a Kerberos Vault account.                                                     | ""                             |
 | `AGENT_KERBEROSVAULT_SECRET_KEY`            | The secret key of a Kerberos Vault account.                                                     | ""                             |
@@ -281,6 +380,34 @@ See [RTSPS and TLS certificates](README-RTSPS-TLS.md) for the complete Bosch UI,
 | `AGENT_SIGNING`                             | Enable 'true' or disable 'false' for signing recordings.                                        | "true"                         |
 | `AGENT_SIGNING_PRIVATE_KEY`                 | The private key (RSA) to sign the recordings fingerprint to validate origin.                    | "" - uses default one if empty |
 
+Remote console access is disabled unless `AGENT_REMOTE_ACCESS_ENABLED=true`.
+The Agent also rejects remote session messages unless Hub encryption or
+end-to-end MQTT encryption is configured and used. A remote shell runs inside
+the Agent process environment as the Agent operating-system user; it is not an
+SSH server and does not expose a new network port. Keep the feature disabled on
+deployments where Hub owners should not have operating-system access.
+
+### Resumable upload chunk size
+
+Hub and Vault resumable uploads use `AGENT_TUS_CHUNK_SIZE_BYTES` as the maximum
+body size of each tus `PATCH` request. The value is a number of bytes, not a
+number of chunks. If the variable is unset or invalid, the Agent uses 8 MiB:
+
+```dotenv
+# 8 MiB (default)
+AGENT_TUS_CHUNK_SIZE_BYTES=8388608
+
+# 4 MiB: more frequent checkpoints on unstable connections
+AGENT_TUS_CHUNK_SIZE_BYTES=4194304
+
+# Disable chunking and send all remaining bytes in one PATCH
+AGENT_TUS_CHUNK_SIZE_BYTES=0
+```
+
+Smaller chunks provide more frequent resumable checkpoints but create more HTTP
+requests. Larger chunks reduce request overhead but require more data to be
+retransmitted when a request fails.
+
 
 ## Encryption
 
@@ -397,6 +524,11 @@ Once signed in you'll see the dashboard page. After successfull configuration of
 
 The `machinery` is a **Golang** project which delivers two functions: it acts as the Kerberos Agent which is doing all the heavy lifting with camera processing and other kinds of logic and on the other hand it acts as a webserver (Rest API) that allows communication from the web (React) or any other custom application. The API is documented using `swagger`.
 
+An executable reference for the Agent frame-processing HTTP and MQTT contracts is
+available in [`examples/frame-processor`](examples/frame-processor). It provides
+deterministic processing profiles for integration testing without requiring a
+machine-learning runtime.
+
 You can simply run the `machinery` using following commands.
 
     git clone https://github.com/kerberos-io/agent
@@ -455,11 +587,26 @@ its own broadcast and switching quality is simply a resubscribe:
 | `high` | `devices/<agent-key>/live.hang`       | highest-resolution camera stream           |
 | `low`  | `devices/<agent-key>/live-low.hang`   | sub stream (main stream when none is set)  |
 
-Each tier only uploads while it has at least one subscriber, so the tier nobody
+Each tier uploads every frame while it has subscribers. While idle, it continues
-watches costs virtually no bandwidth. `AGENT_LIVE_MOQ_QUALITY` accepts `high` or
+to upload fresh keyframes only. This keeps the relay's latest cached GOP current,
-`low` to pin the Agent to a single tier; viewers requesting the other tier then
+so a new viewer does not initially render the final frame from the previous
-find no broadcast. Any other value (including the default) publishes both. The
+viewer session, while avoiding the bandwidth cost of continuously sending every
-initial implementation publishes H.264 video only.
+delta frame. `AGENT_LIVE_MOQ_QUALITY` accepts `high` or `low` to pin the Agent to
+a single tier; viewers requesting the other tier then find no broadcast. Any
+other value (including the default) publishes both. The initial implementation
+publishes H.264 video only.
+
+Two bounded duration settings tune recovery for unusual network conditions:
+
+| Variable | Default | Allowed range |
+| -------- | ------- | ------------- |
+| `AGENT_LIVE_MOQ_MAX_PACKET_AGE` | `1.5s` | `250ms` to `30s` |
+| `AGENT_LIVE_MOQ_WRITE_TIMEOUT` | `5s` | `1s` to `1m` |
+
+Values use Go duration syntax and are clamped to the documented range. The
+dashboard API's `recovery` object reports per-tier reconnects, last successful
+frame/write timing, native write timeouts, watchdog restarts/cooldown, dropped
+run-channel events, and run-worker shutdown timeouts.
 
 The `/anon` relay route is intended for interoperability testing. Production
 deployments must set `AGENT_LIVE_MOQ_URL` to a short-lived, device-scoped
@@ -472,8 +619,9 @@ monorepo devcontainer so it uses the Trixie base, then run the VS Code task
     cd machinery
     bash ./verify-moq-devcontainer.sh
 
-The check requires glibc 2.38 or newer, runs the tagged package tests, links the
+The check requires glibc 2.38 or newer, runs the tagged package and in-process
-complete Agent with `-tags moq`, and executes the resulting binary's version
+native QUIC lifecycle tests, links the complete Agent with `-tags moq`, and
+executes the resulting binary's version
 command. Both devcontainers also run this check during their post-create setup.
 
 ## What is new?

examples/frame-processor/Dockerfile

@@ -0,0 +1,12 @@
+FROM golang:1.24-bookworm AS build
+
+WORKDIR /src
+COPY go.mod go.sum ./
+RUN go mod download
+COPY . .
+RUN CGO_ENABLED=0 go build -trimpath -o /frame-processor .
+
+FROM gcr.io/distroless/static-debian12:nonroot
+COPY --from=build /frame-processor /frame-processor
+EXPOSE 8080
+ENTRYPOINT ["/frame-processor"]

examples/frame-processor/MQTT.md

@@ -0,0 +1,88 @@
+# MQTT control contract
+
+The Frame Processor publishes control messages to
+`kerberos/agent/<hubKey>`. Each message targets one Agent through the envelope's
+`device_id`. It subscribes to `kerberos/hub/<hubKey>` for correlated status
+events.
+
+MQTT transports control data only. JPEG frames use `POST /v1/frames` and full
+recordings use the Agent's existing Vault upload path.
+
+## Envelope
+
+```json
+{
+  "mid": "0c556fc7-637b-4b2a-8a90-2d1cf8450956",
+  "device_id": "camera-1",
+  "timestamp": 1789380000,
+  "encrypted": false,
+  "hidden": false,
+  "public_key": "",
+  "fingerprint": "",
+  "payload": {
+    "version": "1.0",
+    "action": "capture-frame",
+    "device_id": "camera-1",
+    "signature": "",
+    "encrypted_value": "",
+    "hidden_value": "",
+    "value": {}
+  }
+}
+```
+
+The reference service publishes plaintext envelopes for local contract testing.
+Production deployment must use a trusted broker and should adopt the Agent's
+encrypted-message packaging before commands cross an untrusted broker.
+
+## `capture-frame`
+
+```json
+{
+  "schemaVersion": "1.0",
+  "requestId": "request-1",
+  "processingProfile": "always-trigger",
+  "expiresAt": 1789380030000,
+  "traceId": "optional-trace-id"
+}
+```
+
+## `request-recording-window`
+
+```json
+{
+  "schemaVersion": "1.0",
+  "requestId": "request-1",
+  "frameId": "frame-1",
+  "capturedAt": 1789380000123,
+  "preRollSeconds": 10,
+  "eventClipSeconds": 30,
+  "expiresAt": 1789380030000,
+  "processingProfile": "always-trigger",
+  "traceId": "optional-trace-id"
+}
+```
+
+`capturedAt` is generated by the Agent and must be echoed unchanged. The Agent
+uses `requestId` for command idempotency and selects the local recording that
+contains `capturedAt`.
+
+## `frame-processing-status`
+
+```json
+{
+  "schemaVersion": "1.0",
+  "requestId": "request-1",
+  "frameId": "frame-1",
+  "deviceId": "camera-1",
+  "status": "queued",
+  "occurredAt": 1789380001000,
+  "retryable": false,
+  "message": "",
+  "traceId": "optional-trace-id"
+}
+```
+
+Expected statuses are `accepted`, `captured`, `submitted`, `no-event`, `event`,
+`pending-finalisation`, `queued`, `uploaded`, `expired`, `not-found`, `rejected`,
+and `failed`.

examples/frame-processor/README.md

@@ -0,0 +1,74 @@
+# Example Frame Processor
+
+This reference service defines and exercises the Kerberos Agent frame-processing
+contract. It accepts Agent JPEGs over HTTP, makes a deterministic decision, and
+publishes Agent control commands over MQTT. It has no RabbitMQ or machine-learning
+runtime dependency.
+
+## Endpoints
+
+- `GET /health`
+- `POST /v1/frames` with multipart `metadata` JSON and `frame` JPEG parts
+- `POST /v1/frame-requests` with JSON to request capture from one or more Agents
+
+See [openapi.yaml](openapi.yaml) and [MQTT.md](MQTT.md) for the versioned wire
+contracts.
+
+## Run
+
+```bash
+export FRAME_PROCESSOR_API_TOKEN=development-token
+export FRAME_PROCESSOR_MQTT_URI=tcp://localhost:1883
+export FRAME_PROCESSOR_HUB_KEY=development-hub
+export FRAME_PROCESSOR_PROFILE=never-trigger
+go run .
+```
+
+The broker credentials are optional when the local broker permits anonymous
+connections:
+
+```bash
+export FRAME_PROCESSOR_MQTT_USERNAME=...
+export FRAME_PROCESSOR_MQTT_PASSWORD=...
+```
+
+Request a frame from an Agent:
+
+```bash
+curl --fail-with-body \
+  -H 'Authorization: Bearer development-token' \
+  -H 'Content-Type: application/json' \
+  --data @testdata/frame-request.json \
+  http://localhost:8080/v1/frame-requests
+```
+
+## Profiles
+
+- `never-trigger`
+- `always-trigger`
+- `every-nth-frame`
+- `brightness-threshold`
+
+Use `FRAME_PROCESSOR_EVERY_N` and `FRAME_PROCESSOR_BRIGHTNESS_THRESHOLD` to tune
+the last two profiles. `FRAME_PROCESSOR_DELAY_MILLISECONDS` and
+`FRAME_PROCESSOR_FORCE_ERROR` provide deterministic latency and failure
+simulation.
+
+Recording commands default to a 30-second event clip with 10 seconds of pre-roll.
+Configure them with `FRAME_PROCESSOR_EVENT_CLIP_SECONDS` and
+`FRAME_PROCESSOR_PRE_ROLL_SECONDS`.
+
+The reference MQTT publisher emits plaintext Agent envelopes for local contract
+testing. Use a trusted broker. Production support for untrusted brokers requires
+the same encrypted-message packaging used by Hub and Agent.
+
+Frame idempotency is guaranteed until the submitted frame's `expiresAt`. The
+service rejects frame TTLs longer than `FRAME_PROCESSOR_MAX_FRAME_TTL_SECONDS`
+(five minutes by default), then evicts the cached result at expiry.
+
+## Verify
+
+```bash
+GOWORK=off go test ./...
+GOWORK=off go vet ./...
+```

examples/frame-processor/contract/contract.go

@@ -0,0 +1,162 @@
+package contract
+
+import (
+	"errors"
+	"fmt"
+)
+
+const (
+	SchemaVersion     = "1.0"
+	MaxImageDimension = 8192
+
+	ActionCaptureFrame           = "capture-frame"
+	ActionRequestRecordingWindow = "request-recording-window"
+	ActionFrameStatus            = "frame-processing-status"
+)
+
+type FrameMetadata struct {
+	SchemaVersion     string `json:"schemaVersion"`
+	RequestID         string `json:"requestId"`
+	FrameID           string `json:"frameId"`
+	DeviceID          string `json:"deviceId"`
+	CapturedAt        int64  `json:"capturedAt"`
+	ExpiresAt         int64  `json:"expiresAt"`
+	ProcessingProfile string `json:"processingProfile"`
+	SourceStream      string `json:"sourceStream"`
+	Width             int    `json:"width"`
+	Height            int    `json:"height"`
+	TraceID           string `json:"traceId,omitempty"`
+}
+
+func (m FrameMetadata) Validate(nowMillis int64) error {
+	if m.SchemaVersion != SchemaVersion {
+		return fmt.Errorf("unsupported schemaVersion %q", m.SchemaVersion)
+	}
+	if m.RequestID == "" || m.FrameID == "" || m.DeviceID == "" {
+		return errors.New("requestId, frameId, and deviceId are required")
+	}
+	if m.CapturedAt <= 0 {
+		return errors.New("capturedAt must be a positive Unix millisecond timestamp")
+	}
+	if m.ExpiresAt <= m.CapturedAt {
+		return errors.New("expiresAt must be later than capturedAt")
+	}
+	if nowMillis > 0 && m.ExpiresAt <= nowMillis {
+		return errors.New("frame has expired")
+	}
+	if m.ProcessingProfile == "" {
+		return errors.New("processingProfile is required")
+	}
+	if m.SourceStream != "main" && m.SourceStream != "sub" {
+		return errors.New("sourceStream must be main or sub")
+	}
+	if m.Width <= 0 || m.Height <= 0 || m.Width > MaxImageDimension || m.Height > MaxImageDimension {
+		return fmt.Errorf("width and height must be between 1 and %d", MaxImageDimension)
+	}
+	return nil
+}
+
+type FrameRequest struct {
+	SchemaVersion     string   `json:"schemaVersion"`
+	RequestID         string   `json:"requestId,omitempty"`
+	DeviceIDs         []string `json:"deviceIds"`
+	ProcessingProfile string   `json:"processingProfile"`
+	ExpiresAt         int64    `json:"expiresAt"`
+	TraceID           string   `json:"traceId,omitempty"`
+}
+
+func (r FrameRequest) Validate(nowMillis int64) error {
+	if r.SchemaVersion != SchemaVersion {
+		return fmt.Errorf("unsupported schemaVersion %q", r.SchemaVersion)
+	}
+	if len(r.DeviceIDs) == 0 {
+		return errors.New("at least one deviceId is required")
+	}
+	for _, deviceID := range r.DeviceIDs {
+		if deviceID == "" {
+			return errors.New("deviceIds cannot contain empty values")
+		}
+	}
+	if r.ProcessingProfile == "" {
+		return errors.New("processingProfile is required")
+	}
+	if r.ExpiresAt <= nowMillis {
+		return errors.New("expiresAt must be in the future")
+	}
+	return nil
+}
+
+type CaptureFrameCommand struct {
+	SchemaVersion     string `json:"schemaVersion"`
+	RequestID         string `json:"requestId"`
+	ProcessingProfile string `json:"processingProfile"`
+	ExpiresAt         int64  `json:"expiresAt"`
+	TraceID           string `json:"traceId,omitempty"`
+}
+
+type RecordingWindowCommand struct {
+	SchemaVersion     string `json:"schemaVersion"`
+	RequestID         string `json:"requestId"`
+	FrameID           string `json:"frameId"`
+	CapturedAt        int64  `json:"capturedAt"`
+	PreRollSeconds    int64  `json:"preRollSeconds"`
+	EventClipSeconds  int64  `json:"eventClipSeconds"`
+	ExpiresAt         int64  `json:"expiresAt"`
+	ProcessingProfile string `json:"processingProfile"`
+	TraceID           string `json:"traceId,omitempty"`
+}
+
+func (c RecordingWindowCommand) Validate(nowMillis int64) error {
+	if c.SchemaVersion != SchemaVersion {
+		return fmt.Errorf("unsupported schemaVersion %q", c.SchemaVersion)
+	}
+	if c.RequestID == "" || c.FrameID == "" {
+		return errors.New("requestId and frameId are required")
+	}
+	if c.CapturedAt <= 0 {
+		return errors.New("capturedAt must be a positive Unix millisecond timestamp")
+	}
+	if c.EventClipSeconds <= 0 {
+		return errors.New("eventClipSeconds must be positive")
+	}
+	if c.PreRollSeconds < 0 || c.PreRollSeconds > c.EventClipSeconds {
+		return errors.New("preRollSeconds must be between zero and eventClipSeconds")
+	}
+	if c.ExpiresAt <= nowMillis {
+		return errors.New("expiresAt must be in the future")
+	}
+	return nil
+}
+
+type StatusEvent struct {
+	SchemaVersion string `json:"schemaVersion"`
+	RequestID     string `json:"requestId"`
+	FrameID       string `json:"frameId,omitempty"`
+	DeviceID      string `json:"deviceId"`
+	Status        string `json:"status"`
+	OccurredAt    int64  `json:"occurredAt"`
+	Retryable     bool   `json:"retryable,omitempty"`
+	Message       string `json:"message,omitempty"`
+	TraceID       string `json:"traceId,omitempty"`
+}
+
+type MQTTMessage struct {
+	MID         string      `json:"mid"`
+	DeviceID    string      `json:"device_id"`
+	Timestamp   int64       `json:"timestamp"`
+	Encrypted   bool        `json:"encrypted"`
+	Hidden      bool        `json:"hidden"`
+	PublicKey   string      `json:"public_key"`
+	Fingerprint string      `json:"fingerprint"`
+	Payload     MQTTPayload `json:"payload"`
+}
+
+type MQTTPayload struct {
+	Version        string `json:"version"`
+	Action         string `json:"action"`
+	DeviceID       string `json:"device_id"`
+	Signature      string `json:"signature"`
+	EncryptedValue string `json:"encrypted_value"`
+	HiddenValue    string `json:"hidden_value"`
+	Value          any    `json:"value"`
+}

examples/frame-processor/contract/contract_test.go

@@ -0,0 +1,73 @@
+package contract
+
+import (
+	"encoding/json"
+	"os"
+	"testing"
+)
+
+func TestFrameMetadataValidate(t *testing.T) {
+	now := int64(1_000)
+	metadata := FrameMetadata{
+		SchemaVersion:     SchemaVersion,
+		RequestID:         "request-1",
+		FrameID:           "frame-1",
+		DeviceID:          "device-1",
+		CapturedAt:        900,
+		ExpiresAt:         1_100,
+		ProcessingProfile: "always-trigger",
+		SourceStream:      "sub",
+		Width:             640,
+		Height:            480,
+	}
+
+	if err := metadata.Validate(now); err != nil {
+		t.Fatalf("Validate() error = %v", err)
+	}
+
+	metadata.ExpiresAt = now
+	if err := metadata.Validate(now); err == nil {
+		t.Fatal("Validate() accepted an expired frame")
+	}
+}
+
+func TestRecordingWindowCommandValidate(t *testing.T) {
+	command := RecordingWindowCommand{
+		SchemaVersion:    SchemaVersion,
+		RequestID:        "request-1",
+		FrameID:          "frame-1",
+		CapturedAt:       900,
+		PreRollSeconds:   10,
+		EventClipSeconds: 30,
+		ExpiresAt:        2_000,
+	}
+
+	if err := command.Validate(1_000); err != nil {
+		t.Fatalf("Validate() error = %v", err)
+	}
+
+	command.PreRollSeconds = 31
+	if err := command.Validate(1_000); err == nil {
+		t.Fatal("Validate() accepted pre-roll longer than the event clip")
+	}
+}
+
+func TestContractFixturesDecode(t *testing.T) {
+	tests := []struct {
+		path   string
+		target any
+	}{
+		{"../testdata/frame-request.json", &FrameRequest{}},
+		{"../testdata/capture-frame.json", &CaptureFrameCommand{}},
+		{"../testdata/request-recording-window.json", &RecordingWindowCommand{}},
+	}
+	for _, test := range tests {
+		value, err := os.ReadFile(test.path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if err := json.Unmarshal(value, test.target); err != nil {
+			t.Fatalf("decode %s: %v", test.path, err)
+		}
+	}
+}

examples/frame-processor/go.mod

@@ -0,0 +1,11 @@
+module github.com/kerberos-io/agent/examples/frame-processor
+
+go 1.24.2
+
+require github.com/eclipse/paho.mqtt.golang v1.5.0
+
+require (
+	github.com/gorilla/websocket v1.5.3 // indirect
+	golang.org/x/net v0.27.0 // indirect
+	golang.org/x/sync v0.7.0 // indirect
+)

examples/frame-processor/go.sum

@@ -0,0 +1,8 @@
+github.com/eclipse/paho.mqtt.golang v1.5.0 h1:EH+bUVJNgttidWFkLLVKaQPGmkTUfQQqjOsyvMGvD6o=
+github.com/eclipse/paho.mqtt.golang v1.5.0/go.mod h1:du/2qNQVqJf/Sqs4MEL77kR8QTqANF7XU7Fk0aOTAgk=
+github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
+github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
+golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=

examples/frame-processor/main.go

@@ -0,0 +1,184 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"os"
+	"os/signal"
+	"strconv"
+	"syscall"
+	"time"
+
+	"github.com/kerberos-io/agent/examples/frame-processor/contract"
+	"github.com/kerberos-io/agent/examples/frame-processor/mqttpublisher"
+	"github.com/kerberos-io/agent/examples/frame-processor/processor"
+	"github.com/kerberos-io/agent/examples/frame-processor/service"
+)
+
+func main() {
+	config, err := loadConfig()
+	if err != nil {
+		slog.Error("invalid configuration", "error", err)
+		os.Exit(1)
+	}
+
+	publisher, err := mqttpublisher.New(config.mqtt, func(status contract.StatusEvent) {
+		slog.Info("Agent frame-processing status",
+			"deviceId", status.DeviceID,
+			"requestId", status.RequestID,
+			"frameId", status.FrameID,
+			"status", status.Status,
+		)
+	})
+	if err != nil {
+		slog.Error("failed to initialize MQTT", "error", err)
+		os.Exit(1)
+	}
+	defer publisher.Close()
+
+	engine := processor.New(config.processor)
+	application := service.New(config.service, engine, publisher, nil)
+	server := &http.Server{
+		Addr:              config.address,
+		Handler:           application.Handler(),
+		ReadHeaderTimeout: 5 * time.Second,
+		IdleTimeout:       60 * time.Second,
+	}
+
+	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer stop()
+	go func() {
+		<-ctx.Done()
+		shutdownContext, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		_ = server.Shutdown(shutdownContext)
+	}()
+
+	slog.Info("Frame Processor listening", "address", config.address, "profile", config.processor.DefaultProfile)
+	if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+		slog.Error("Frame Processor stopped", "error", err)
+		os.Exit(1)
+	}
+}
+
+type applicationConfig struct {
+	address   string
+	mqtt      mqttpublisher.Config
+	processor processor.Config
+	service   service.Config
+}
+
+func loadConfig() (applicationConfig, error) {
+	brightnessThreshold := envInt("FRAME_PROCESSOR_BRIGHTNESS_THRESHOLD", 200)
+	config := applicationConfig{
+		address: envString("FRAME_PROCESSOR_ADDRESS", ":8080"),
+		mqtt: mqttpublisher.Config{
+			BrokerURI: os.Getenv("FRAME_PROCESSOR_MQTT_URI"),
+			Username:  os.Getenv("FRAME_PROCESSOR_MQTT_USERNAME"),
+			Password:  os.Getenv("FRAME_PROCESSOR_MQTT_PASSWORD"),
+			HubKey:    os.Getenv("FRAME_PROCESSOR_HUB_KEY"),
+			ClientID:  os.Getenv("FRAME_PROCESSOR_MQTT_CLIENT_ID"),
+			Timeout:   envDurationSeconds("FRAME_PROCESSOR_MQTT_TIMEOUT_SECONDS", 10),
+		},
+		processor: processor.Config{
+			DefaultProfile:      envString("FRAME_PROCESSOR_PROFILE", processor.ProfileNeverTrigger),
+			EveryN:              envInt("FRAME_PROCESSOR_EVERY_N", 2),
+			BrightnessThreshold: uint8(brightnessThreshold),
+			Delay:               envDurationMillis("FRAME_PROCESSOR_DELAY_MILLISECONDS", 0),
+			ForceError:          envBool("FRAME_PROCESSOR_FORCE_ERROR", false),
+		},
+		service: service.Config{
+			APIToken:         os.Getenv("FRAME_PROCESSOR_API_TOKEN"),
+			MaxFrameBytes:    int64(envInt("FRAME_PROCESSOR_MAX_FRAME_BYTES", 4<<20)),
+			MaxFrameTTL:      envDurationSeconds("FRAME_PROCESSOR_MAX_FRAME_TTL_SECONDS", 300),
+			CommandTTL:       envDurationSeconds("FRAME_PROCESSOR_COMMAND_TTL_SECONDS", 30),
+			PreRollSeconds:   int64(envInt("FRAME_PROCESSOR_PRE_ROLL_SECONDS", 10)),
+			EventClipSeconds: int64(envInt("FRAME_PROCESSOR_EVENT_CLIP_SECONDS", 30)),
+		},
+	}
+	if config.service.APIToken == "" {
+		return applicationConfig{}, errors.New("FRAME_PROCESSOR_API_TOKEN is required")
+	}
+	if config.mqtt.BrokerURI == "" || config.mqtt.HubKey == "" {
+		return applicationConfig{}, errors.New("FRAME_PROCESSOR_MQTT_URI and FRAME_PROCESSOR_HUB_KEY are required")
+	}
+	if config.processor.EveryN <= 0 {
+		return applicationConfig{}, errors.New("FRAME_PROCESSOR_EVERY_N must be positive")
+	}
+	if !processor.IsProfileSupported(config.processor.DefaultProfile) {
+		return applicationConfig{}, fmt.Errorf("unsupported FRAME_PROCESSOR_PROFILE %q", config.processor.DefaultProfile)
+	}
+	if brightnessThreshold < 0 || brightnessThreshold > 255 {
+		return applicationConfig{}, errors.New("FRAME_PROCESSOR_BRIGHTNESS_THRESHOLD must be between 0 and 255")
+	}
+	if config.processor.Delay < 0 || config.processor.Delay > time.Minute {
+		return applicationConfig{}, errors.New("FRAME_PROCESSOR_DELAY_MILLISECONDS must be between 0 and 60000")
+	}
+	if config.service.MaxFrameBytes <= 0 || config.service.MaxFrameBytes > 100<<20 {
+		return applicationConfig{}, errors.New("FRAME_PROCESSOR_MAX_FRAME_BYTES must be between 1 and 104857600")
+	}
+	if config.service.MaxFrameTTL <= 0 || config.service.MaxFrameTTL > time.Hour {
+		return applicationConfig{}, errors.New("FRAME_PROCESSOR_MAX_FRAME_TTL_SECONDS must be between 1 and 3600")
+	}
+	if config.service.PreRollSeconds < 0 || config.service.PreRollSeconds > config.service.EventClipSeconds {
+		return applicationConfig{}, errors.New("FRAME_PROCESSOR_PRE_ROLL_SECONDS must be between zero and FRAME_PROCESSOR_EVENT_CLIP_SECONDS")
+	}
+	if config.service.CommandTTL <= 0 || config.service.CommandTTL > time.Hour {
+		return applicationConfig{}, errors.New("FRAME_PROCESSOR_COMMAND_TTL_SECONDS must be between 1 and 3600")
+	}
+	if config.service.EventClipSeconds <= 0 || config.service.EventClipSeconds > 24*60*60 {
+		return applicationConfig{}, errors.New("FRAME_PROCESSOR_EVENT_CLIP_SECONDS must be between 1 and 86400")
+	}
+	if config.mqtt.Timeout <= 0 || config.mqtt.Timeout > time.Minute {
+		return applicationConfig{}, errors.New("FRAME_PROCESSOR_MQTT_TIMEOUT_SECONDS must be between 1 and 60")
+	}
+	return config, nil
+}
+
+func envString(name, fallback string) string {
+	if value := os.Getenv(name); value != "" {
+		return value
+	}
+	return fallback
+}
+
+func envInt(name string, fallback int) int {
+	value := os.Getenv(name)
+	if value == "" {
+		return fallback
+	}
+	parsed, err := strconv.Atoi(value)
+	if err != nil {
+		slog.Warn("invalid integer environment value, using default", "name", name)
+		return fallback
+	}
+	return parsed
+}
+
+func envBool(name string, fallback bool) bool {
+	value := os.Getenv(name)
+	if value == "" {
+		return fallback
+	}
+	parsed, err := strconv.ParseBool(value)
+	if err != nil {
+		slog.Warn("invalid boolean environment value, using default", "name", name)
+		return fallback
+	}
+	return parsed
+}
+
+func envDurationSeconds(name string, fallback int) time.Duration {
+	return time.Duration(envInt(name, fallback)) * time.Second
+}
+
+func envDurationMillis(name string, fallback int) time.Duration {
+	return time.Duration(envInt(name, fallback)) * time.Millisecond
+}
+
+func (c applicationConfig) String() string {
+	return fmt.Sprintf("address=%s profile=%s", c.address, c.processor.DefaultProfile)
+}

examples/frame-processor/main_test.go

@@ -0,0 +1,25 @@
+package main
+
+import "testing"
+
+func TestLoadConfigRejectsInvalidBounds(t *testing.T) {
+	t.Setenv("FRAME_PROCESSOR_API_TOKEN", "secret")
+	t.Setenv("FRAME_PROCESSOR_MQTT_URI", "tcp://localhost:1883")
+	t.Setenv("FRAME_PROCESSOR_HUB_KEY", "hub")
+	t.Setenv("FRAME_PROCESSOR_BRIGHTNESS_THRESHOLD", "256")
+
+	if _, err := loadConfig(); err == nil {
+		t.Fatal("loadConfig() accepted an invalid brightness threshold")
+	}
+}
+
+func TestLoadConfigRejectsUnknownProfile(t *testing.T) {
+	t.Setenv("FRAME_PROCESSOR_API_TOKEN", "secret")
+	t.Setenv("FRAME_PROCESSOR_MQTT_URI", "tcp://localhost:1883")
+	t.Setenv("FRAME_PROCESSOR_HUB_KEY", "hub")
+	t.Setenv("FRAME_PROCESSOR_PROFILE", "unknown")
+
+	if _, err := loadConfig(); err == nil {
+		t.Fatal("loadConfig() accepted an unknown profile")
+	}
+}

examples/frame-processor/mqttpublisher/publisher.go

@@ -0,0 +1,171 @@
+package mqttpublisher
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log/slog"
+	"strings"
+	"time"
+
+	mqtt "github.com/eclipse/paho.mqtt.golang"
+	"github.com/kerberos-io/agent/examples/frame-processor/contract"
+)
+
+const (
+	commandQoS = byte(1)
+	statusQoS  = byte(1)
+)
+
+type Config struct {
+	BrokerURI string
+	Username  string
+	Password  string
+	HubKey    string
+	ClientID  string
+	Timeout   time.Duration
+}
+
+type StatusHandler func(contract.StatusEvent)
+
+type Publisher struct {
+	client       mqtt.Client
+	commandTopic string
+	timeout      time.Duration
+}
+
+func New(config Config, handler StatusHandler) (*Publisher, error) {
+	if config.BrokerURI == "" || config.HubKey == "" {
+		return nil, errors.New("MQTT broker URI and Hub key are required")
+	}
+	if config.Timeout <= 0 {
+		config.Timeout = 10 * time.Second
+	}
+	if config.ClientID == "" {
+		config.ClientID = "frame-processor-" + randomID()
+	}
+
+	statusTopic := "kerberos/hub/" + config.HubKey
+	options := mqtt.NewClientOptions().
+		AddBroker(config.BrokerURI).
+		SetClientID(config.ClientID).
+		SetUsername(config.Username).
+		SetPassword(config.Password).
+		SetCleanSession(false).
+		SetResumeSubs(true).
+		SetAutoReconnect(true).
+		SetConnectRetry(true).
+		SetConnectRetryInterval(5 * time.Second).
+		SetMaxReconnectInterval(time.Minute).
+		SetKeepAlive(30 * time.Second).
+		SetPingTimeout(10 * time.Second)
+	if handler != nil {
+		options.SetOnConnectHandler(func(client mqtt.Client) {
+			token := client.Subscribe(statusTopic, statusQoS, statusMessageHandler(handler))
+			if !token.WaitTimeout(config.Timeout) || token.Error() != nil {
+				slog.Error("failed to subscribe to Agent status events", "topic", statusTopic, "error", token.Error())
+			}
+		})
+	}
+
+	client := mqtt.NewClient(options)
+	token := client.Connect()
+	if !token.WaitTimeout(config.Timeout) {
+		return nil, errors.New("MQTT connection timed out")
+	}
+	if err := token.Error(); err != nil {
+		return nil, fmt.Errorf("connect MQTT: %w", err)
+	}
+	return &Publisher{
+		client: client, commandTopic: "kerberos/agent/" + config.HubKey,
+		timeout: config.Timeout,
+	}, nil
+}
+
+func (p *Publisher) Close() {
+	if p != nil && p.client != nil && p.client.IsConnected() {
+		p.client.Disconnect(250)
+	}
+}
+
+func (p *Publisher) PublishCaptureFrame(ctx context.Context, deviceID string, command contract.CaptureFrameCommand) error {
+	return p.publish(ctx, deviceID, contract.ActionCaptureFrame, command)
+}
+
+func (p *Publisher) PublishRecordingWindow(ctx context.Context, deviceID string, command contract.RecordingWindowCommand) error {
+	return p.publish(ctx, deviceID, contract.ActionRequestRecordingWindow, command)
+}
+
+func (p *Publisher) publish(ctx context.Context, deviceID, action string, value any) error {
+	message := newMessage(deviceID, action, value, time.Now())
+	payload, err := json.Marshal(message)
+	if err != nil {
+		return fmt.Errorf("marshal MQTT command: %w", err)
+	}
+	token := p.client.Publish(p.commandTopic, commandQoS, false, payload)
+	timer := time.NewTimer(p.timeout)
+	defer timer.Stop()
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-timer.C:
+		return errors.New("MQTT publish timed out")
+	case <-token.Done():
+		if err := token.Error(); err != nil {
+			return fmt.Errorf("publish MQTT command: %w", err)
+		}
+		return nil
+	}
+}
+
+func newMessage(deviceID, action string, value any, now time.Time) contract.MQTTMessage {
+	return contract.MQTTMessage{
+		MID:       randomID(),
+		DeviceID:  deviceID,
+		Timestamp: now.Unix(),
+		Payload: contract.MQTTPayload{
+			Version:  contract.SchemaVersion,
+			Action:   action,
+			DeviceID: deviceID,
+			Value:    value,
+		},
+	}
+}
+
+func randomID() string {
+	value := make([]byte, 16)
+	if _, err := rand.Read(value); err != nil {
+		return fmt.Sprintf("%d", time.Now().UnixNano())
+	}
+	value[6] = (value[6] & 0x0f) | 0x40
+	value[8] = (value[8] & 0x3f) | 0x80
+	encoded := hex.EncodeToString(value)
+	return strings.Join([]string{encoded[0:8], encoded[8:12], encoded[12:16], encoded[16:20], encoded[20:32]}, "-")
+}
+
+func statusMessageHandler(handler StatusHandler) mqtt.MessageHandler {
+	return func(_ mqtt.Client, message mqtt.Message) {
+		var envelope contract.MQTTMessage
+		if err := json.Unmarshal(message.Payload(), &envelope); err != nil {
+			slog.Warn("discarding malformed Agent status envelope", "error", err)
+			return
+		}
+		if envelope.Payload.Action != contract.ActionFrameStatus {
+			return
+		}
+		value, err := json.Marshal(envelope.Payload.Value)
+		if err != nil {
+			slog.Warn("discarding unencodable Agent status value", "error", err)
+			return
+		}
+		var status contract.StatusEvent
+		if err := json.Unmarshal(value, &status); err != nil {
+			slog.Warn("discarding malformed Agent status value", "error", err)
+			return
+		}
+		handler(status)
+	}
+}

examples/frame-processor/mqttpublisher/publisher_test.go

@@ -0,0 +1,54 @@
+package mqttpublisher
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/kerberos-io/agent/examples/frame-processor/contract"
+)
+
+func TestNewMessageMatchesAgentEnvelope(t *testing.T) {
+	command := contract.CaptureFrameCommand{
+		SchemaVersion:     contract.SchemaVersion,
+		RequestID:         "request-1",
+		ProcessingProfile: "always-trigger",
+		ExpiresAt:         2_000,
+	}
+	message := newMessage("device-1", contract.ActionCaptureFrame, command, time.Unix(1_000, 0))
+	payload, err := json.Marshal(message)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var decoded map[string]any
+	if err := json.Unmarshal(payload, &decoded); err != nil {
+		t.Fatal(err)
+	}
+	if decoded["device_id"] != "device-1" || decoded["timestamp"] != float64(1_000) {
+		t.Fatalf("envelope = %s", payload)
+	}
+	inner := decoded["payload"].(map[string]any)
+	if inner["action"] != contract.ActionCaptureFrame || inner["device_id"] != "device-1" {
+		t.Fatalf("payload = %#v", inner)
+	}
+}
+
+func TestStatusMessageHandlerIgnoresOtherActions(t *testing.T) {
+	called := false
+	handler := statusMessageHandler(func(contract.StatusEvent) { called = true })
+	handler(nil, fakeMessage(`{"payload":{"action":"motion","value":{}}}`))
+	if called {
+		t.Fatal("handler accepted an unrelated action")
+	}
+}
+
+type fakeMessage string
+
+func (m fakeMessage) Duplicate() bool   { return false }
+func (m fakeMessage) Qos() byte         { return 1 }
+func (m fakeMessage) Retained() bool    { return false }
+func (m fakeMessage) Topic() string     { return "test" }
+func (m fakeMessage) MessageID() uint16 { return 1 }
+func (m fakeMessage) Payload() []byte   { return []byte(m) }
+func (m fakeMessage) Ack()              {}

examples/frame-processor/openapi.yaml

@@ -0,0 +1,216 @@
+openapi: 3.0.3
+info:
+  title: Kerberos Frame Processing API
+  version: 1.0.0
+  description: Reference contract for Agent frame submission and externally requested capture.
+servers:
+  - url: http://localhost:8080
+security:
+  - bearerAuth: []
+paths:
+  /health:
+    get:
+      security: []
+      summary: Check service liveness
+      responses:
+        "200":
+          description: Service is healthy
+          content:
+            application/json:
+              schema:
+                type: object
+                required: [status]
+                properties:
+                  status:
+                    type: string
+                    enum: [healthy]
+  /v1/frames:
+    post:
+      summary: Process one Agent frame
+      requestBody:
+        required: true
+        content:
+          multipart/form-data:
+            schema:
+              type: object
+              required: [metadata, frame]
+              properties:
+                metadata:
+                  $ref: "#/components/schemas/FrameMetadata"
+                frame:
+                  type: string
+                  format: binary
+            encoding:
+              metadata:
+                contentType: application/json
+              frame:
+                contentType: image/jpeg
+      responses:
+        "200":
+          description: Frame processed
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/FrameResponse"
+        "400":
+          $ref: "#/components/responses/BadRequest"
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "413":
+          description: Frame exceeds the configured request limit
+        "415":
+          description: Frame is not a valid JPEG
+        "422":
+          description: Metadata is invalid or expired
+        "502":
+          description: Processing or MQTT command publication failed
+  /v1/frame-requests:
+    post:
+      summary: Request a new frame from one or more Agents
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/FrameRequest"
+      responses:
+        "202":
+          description: Capture commands accepted for publication
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/FrameRequestResponse"
+        "400":
+          $ref: "#/components/responses/BadRequest"
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "422":
+          description: Request is invalid or expired
+        "502":
+          description: MQTT command publication failed
+components:
+  securitySchemes:
+    bearerAuth:
+      type: http
+      scheme: bearer
+  responses:
+    BadRequest:
+      description: Malformed request
+      content:
+        application/json:
+          schema:
+            $ref: "#/components/schemas/Error"
+    Unauthorized:
+      description: Missing or invalid bearer token
+      content:
+        application/json:
+          schema:
+            $ref: "#/components/schemas/Error"
+  schemas:
+    FrameMetadata:
+      type: object
+      required:
+        - schemaVersion
+        - requestId
+        - frameId
+        - deviceId
+        - capturedAt
+        - expiresAt
+        - processingProfile
+        - sourceStream
+        - width
+        - height
+      properties:
+        schemaVersion:
+          type: string
+          enum: ["1.0"]
+        requestId:
+          type: string
+        frameId:
+          type: string
+        deviceId:
+          type: string
+        capturedAt:
+          type: integer
+          format: int64
+          description: Agent wall-clock capture time in Unix milliseconds.
+        expiresAt:
+          type: integer
+          format: int64
+          description: Unix milliseconds after which the frame must not be processed.
+        processingProfile:
+          type: string
+        sourceStream:
+          type: string
+          enum: [main, sub]
+        width:
+          type: integer
+          minimum: 1
+        height:
+          type: integer
+          minimum: 1
+        traceId:
+          type: string
+    FrameRequest:
+      type: object
+      required: [schemaVersion, deviceIds, processingProfile, expiresAt]
+      properties:
+        schemaVersion:
+          type: string
+          enum: ["1.0"]
+        requestId:
+          type: string
+          description: Generated by the service when omitted.
+        deviceIds:
+          type: array
+          minItems: 1
+          items:
+            type: string
+        processingProfile:
+          type: string
+        expiresAt:
+          type: integer
+          format: int64
+        traceId:
+          type: string
+    FrameResponse:
+      type: object
+      required: [schemaVersion, requestId, frameId, decision, reason]
+      properties:
+        schemaVersion:
+          type: string
+          enum: ["1.0"]
+        requestId:
+          type: string
+        frameId:
+          type: string
+        decision:
+          type: string
+          enum: [no-event, event]
+        reason:
+          type: string
+    FrameRequestResponse:
+      type: object
+      required: [schemaVersion, requestId, deviceIds, status]
+      properties:
+        schemaVersion:
+          type: string
+          enum: ["1.0"]
+        requestId:
+          type: string
+        deviceIds:
+          type: array
+          items:
+            type: string
+        status:
+          type: string
+          enum: [accepted]
+    Error:
+      type: object
+      required: [schemaVersion, error]
+      properties:
+        schemaVersion:
+          type: string
+          enum: ["1.0"]
+        error:
+          type: string

examples/frame-processor/processor/processor.go

@@ -0,0 +1,117 @@
+package processor
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"image/jpeg"
+	"sync"
+	"time"
+
+	"github.com/kerberos-io/agent/examples/frame-processor/contract"
+)
+
+const (
+	ProfileNeverTrigger        = "never-trigger"
+	ProfileAlwaysTrigger       = "always-trigger"
+	ProfileEveryNthFrame       = "every-nth-frame"
+	ProfileBrightnessThreshold = "brightness-threshold"
+)
+
+type Decision struct {
+	Triggered bool   `json:"triggered"`
+	Reason    string `json:"reason"`
+}
+
+type Config struct {
+	DefaultProfile      string
+	EveryN              int
+	BrightnessThreshold uint8
+	Delay               time.Duration
+	ForceError          bool
+}
+
+type Engine struct {
+	config Config
+	mu     sync.Mutex
+	counts map[string]int
+}
+
+func IsProfileSupported(profile string) bool {
+	switch profile {
+	case ProfileNeverTrigger, ProfileAlwaysTrigger, ProfileEveryNthFrame, ProfileBrightnessThreshold:
+		return true
+	default:
+		return false
+	}
+}
+
+func New(config Config) *Engine {
+	if config.DefaultProfile == "" {
+		config.DefaultProfile = ProfileNeverTrigger
+	}
+	if config.EveryN <= 0 {
+		config.EveryN = 2
+	}
+	return &Engine{config: config, counts: make(map[string]int)}
+}
+
+func (e *Engine) Process(ctx context.Context, metadata contract.FrameMetadata, frame []byte) (Decision, error) {
+	if e.config.Delay > 0 {
+		timer := time.NewTimer(e.config.Delay)
+		defer timer.Stop()
+		select {
+		case <-ctx.Done():
+			return Decision{}, ctx.Err()
+		case <-timer.C:
+		}
+	}
+	if e.config.ForceError {
+		return Decision{}, errors.New("configured processing failure")
+	}
+
+	profile := metadata.ProcessingProfile
+	if profile == "" {
+		profile = e.config.DefaultProfile
+	}
+	switch profile {
+	case ProfileNeverTrigger:
+		return Decision{Reason: ProfileNeverTrigger}, nil
+	case ProfileAlwaysTrigger:
+		return Decision{Triggered: true, Reason: ProfileAlwaysTrigger}, nil
+	case ProfileEveryNthFrame:
+		e.mu.Lock()
+		e.counts[metadata.DeviceID]++
+		count := e.counts[metadata.DeviceID]
+		e.mu.Unlock()
+		return Decision{
+			Triggered: count%e.config.EveryN == 0,
+			Reason:    fmt.Sprintf("frame %d of every %d", count, e.config.EveryN),
+		}, nil
+	case ProfileBrightnessThreshold:
+		image, err := jpeg.Decode(bytes.NewReader(frame))
+		if err != nil {
+			return Decision{}, fmt.Errorf("decode JPEG: %w", err)
+		}
+		bounds := image.Bounds()
+		var total uint64
+		for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
+			for x := bounds.Min.X; x < bounds.Max.X; x++ {
+				gray, _, _, _ := image.At(x, y).RGBA()
+				total += uint64(gray >> 8)
+			}
+		}
+		pixels := uint64(bounds.Dx() * bounds.Dy())
+		if pixels == 0 {
+			return Decision{}, errors.New("JPEG has no pixels")
+		}
+		average := uint8(total / pixels)
+		return Decision{
+			Triggered: average >= e.config.BrightnessThreshold,
+			Reason:    fmt.Sprintf("average brightness %d, threshold %d", average, e.config.BrightnessThreshold),
+		}, nil
+	default:
+		return Decision{}, fmt.Errorf("unknown processing profile %q", profile)
+	}
+}

examples/frame-processor/processor/processor_test.go

@@ -0,0 +1,41 @@
+package processor
+
+import (
+	"context"
+	"testing"
+
+	"github.com/kerberos-io/agent/examples/frame-processor/contract"
+)
+
+func TestEveryNthFrameIsTrackedPerDevice(t *testing.T) {
+	engine := New(Config{EveryN: 2})
+	metadata := contract.FrameMetadata{ProcessingProfile: ProfileEveryNthFrame, DeviceID: "device-1"}
+
+	first, err := engine.Process(context.Background(), metadata, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	second, err := engine.Process(context.Background(), metadata, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	metadata.DeviceID = "device-2"
+	otherDevice, err := engine.Process(context.Background(), metadata, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if first.Triggered || !second.Triggered || otherDevice.Triggered {
+		t.Fatalf("decisions = first:%t second:%t other:%t", first.Triggered, second.Triggered, otherDevice.Triggered)
+	}
+}
+
+func TestUnknownProfileFails(t *testing.T) {
+	engine := New(Config{})
+	_, err := engine.Process(context.Background(), contract.FrameMetadata{
+		ProcessingProfile: "missing-profile",
+	}, nil)
+	if err == nil {
+		t.Fatal("Process() accepted an unknown profile")
+	}
+}

examples/frame-processor/service/service.go

@@ -0,0 +1,358 @@
+package service
+
+import (
+	"bytes"
+	"context"
+	"crypto/subtle"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"image"
+	_ "image/jpeg"
+	"io"
+	"log/slog"
+	"mime/multipart"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/kerberos-io/agent/examples/frame-processor/contract"
+	"github.com/kerberos-io/agent/examples/frame-processor/processor"
+)
+
+const maxMetadataBytes = 64 << 10
+
+var errRequestTooLarge = errors.New("request exceeds maximum size")
+
+type Publisher interface {
+	PublishCaptureFrame(context.Context, string, contract.CaptureFrameCommand) error
+	PublishRecordingWindow(context.Context, string, contract.RecordingWindowCommand) error
+}
+
+type Processor interface {
+	Process(context.Context, contract.FrameMetadata, []byte) (processor.Decision, error)
+}
+
+type Config struct {
+	APIToken         string
+	MaxFrameBytes    int64
+	MaxFrameTTL      time.Duration
+	CommandTTL       time.Duration
+	PreRollSeconds   int64
+	EventClipSeconds int64
+}
+
+type Service struct {
+	config    Config
+	processor Processor
+	publisher Publisher
+	now       func() time.Time
+	newID     func() string
+
+	resultsMu sync.Mutex
+	results   map[string]*frameResult
+}
+
+type frameResult struct {
+	done      chan struct{}
+	expiresAt int64
+	response  FrameResponse
+	err       error
+}
+
+type FrameResponse struct {
+	SchemaVersion string `json:"schemaVersion"`
+	RequestID     string `json:"requestId"`
+	FrameID       string `json:"frameId"`
+	Decision      string `json:"decision"`
+	Reason        string `json:"reason"`
+}
+
+type FrameRequestResponse struct {
+	SchemaVersion string   `json:"schemaVersion"`
+	RequestID     string   `json:"requestId"`
+	DeviceIDs     []string `json:"deviceIds"`
+	Status        string   `json:"status"`
+}
+
+func New(config Config, frameProcessor Processor, publisher Publisher, newID func() string) *Service {
+	if config.MaxFrameBytes <= 0 {
+		config.MaxFrameBytes = 4 << 20
+	}
+	if config.CommandTTL <= 0 {
+		config.CommandTTL = 30 * time.Second
+	}
+	if config.MaxFrameTTL <= 0 {
+		config.MaxFrameTTL = 5 * time.Minute
+	}
+	if config.EventClipSeconds <= 0 {
+		config.EventClipSeconds = 30
+	}
+	if newID == nil {
+		newID = func() string { return fmt.Sprintf("request-%d", time.Now().UnixNano()) }
+	}
+	return &Service{
+		config: config, processor: frameProcessor, publisher: publisher,
+		now: time.Now, newID: newID, results: make(map[string]*frameResult),
+	}
+}
+
+func (s *Service) Handler() http.Handler {
+	mux := http.NewServeMux()
+	mux.HandleFunc("GET /health", s.handleHealth)
+	mux.HandleFunc("POST /v1/frames", s.authorize(s.handleFrame))
+	mux.HandleFunc("POST /v1/frame-requests", s.authorize(s.handleFrameRequest))
+	return mux
+}
+
+func (s *Service) authorize(next http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if s.config.APIToken == "" {
+			writeError(w, http.StatusServiceUnavailable, "service authentication is not configured")
+			return
+		}
+		provided := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
+		if subtle.ConstantTimeCompare([]byte(provided), []byte(s.config.APIToken)) != 1 {
+			writeError(w, http.StatusUnauthorized, "unauthorized")
+			return
+		}
+		next(w, r)
+	}
+}
+
+func (s *Service) handleHealth(w http.ResponseWriter, _ *http.Request) {
+	writeJSON(w, http.StatusOK, map[string]string{"status": "healthy"})
+}
+
+func (s *Service) handleFrame(w http.ResponseWriter, r *http.Request) {
+	metadata, frame, err := readFrame(w, r, s.config.MaxFrameBytes)
+	if err != nil {
+		if errors.Is(err, errRequestTooLarge) {
+			writeError(w, http.StatusRequestEntityTooLarge, err.Error())
+			return
+		}
+		writeError(w, http.StatusBadRequest, err.Error())
+		return
+	}
+	nowMillis := s.now().UnixMilli()
+	if err := metadata.Validate(nowMillis); err != nil {
+		writeError(w, http.StatusUnprocessableEntity, err.Error())
+		return
+	}
+	if time.Duration(metadata.ExpiresAt-nowMillis)*time.Millisecond > s.config.MaxFrameTTL {
+		writeError(w, http.StatusUnprocessableEntity, "expiresAt exceeds maximum frame TTL")
+		return
+	}
+	imageConfig, _, err := image.DecodeConfig(bytes.NewReader(frame))
+	if err != nil {
+		writeError(w, http.StatusUnsupportedMediaType, "frame must be a valid JPEG")
+		return
+	}
+	if imageConfig.Width != metadata.Width || imageConfig.Height != metadata.Height {
+		writeError(w, http.StatusUnprocessableEntity, "frame dimensions do not match metadata")
+		return
+	}
+
+	result, owner := s.beginFrame(metadata.FrameID, metadata.ExpiresAt, nowMillis)
+	if !owner {
+		select {
+		case <-r.Context().Done():
+			writeError(w, http.StatusRequestTimeout, "request cancelled")
+			return
+		case <-result.done:
+		}
+		if result.err != nil {
+			writeError(w, http.StatusBadGateway, result.err.Error())
+			return
+		}
+		writeJSON(w, http.StatusOK, result.response)
+		return
+	}
+
+	response, processErr := s.processFrame(r.Context(), metadata, frame)
+	s.finishFrame(result, response, processErr)
+	if processErr != nil {
+		writeError(w, http.StatusBadGateway, processErr.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, response)
+}
+
+func (s *Service) processFrame(ctx context.Context, metadata contract.FrameMetadata, frame []byte) (FrameResponse, error) {
+	decision, err := s.processor.Process(ctx, metadata, frame)
+	if err != nil {
+		return FrameResponse{}, fmt.Errorf("process frame: %w", err)
+	}
+	response := FrameResponse{
+		SchemaVersion: contract.SchemaVersion,
+		RequestID:     metadata.RequestID,
+		FrameID:       metadata.FrameID,
+		Decision:      "no-event",
+		Reason:        decision.Reason,
+	}
+	if !decision.Triggered {
+		return response, nil
+	}
+	command := contract.RecordingWindowCommand{
+		SchemaVersion:     contract.SchemaVersion,
+		RequestID:         metadata.RequestID,
+		FrameID:           metadata.FrameID,
+		CapturedAt:        metadata.CapturedAt,
+		PreRollSeconds:    s.config.PreRollSeconds,
+		EventClipSeconds:  s.config.EventClipSeconds,
+		ExpiresAt:         s.now().Add(s.config.CommandTTL).UnixMilli(),
+		ProcessingProfile: metadata.ProcessingProfile,
+		TraceID:           metadata.TraceID,
+	}
+	if err := command.Validate(s.now().UnixMilli()); err != nil {
+		return FrameResponse{}, fmt.Errorf("build recording command: %w", err)
+	}
+	if err := s.publisher.PublishRecordingWindow(ctx, metadata.DeviceID, command); err != nil {
+		return FrameResponse{}, fmt.Errorf("publish recording command: %w", err)
+	}
+	response.Decision = "event"
+	return response, nil
+}
+
+func (s *Service) handleFrameRequest(w http.ResponseWriter, r *http.Request) {
+	var request contract.FrameRequest
+	decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxMetadataBytes))
+	decoder.DisallowUnknownFields()
+	if err := decoder.Decode(&request); err != nil {
+		writeError(w, http.StatusBadRequest, "invalid JSON request")
+		return
+	}
+	if request.RequestID == "" {
+		request.RequestID = s.newID()
+	}
+	nowMillis := s.now().UnixMilli()
+	if err := request.Validate(nowMillis); err != nil {
+		writeError(w, http.StatusUnprocessableEntity, err.Error())
+		return
+	}
+	if time.Duration(request.ExpiresAt-nowMillis)*time.Millisecond > s.config.MaxFrameTTL {
+		writeError(w, http.StatusUnprocessableEntity, "expiresAt exceeds maximum frame TTL")
+		return
+	}
+	for _, deviceID := range request.DeviceIDs {
+		command := contract.CaptureFrameCommand{
+			SchemaVersion:     contract.SchemaVersion,
+			RequestID:         request.RequestID,
+			ProcessingProfile: request.ProcessingProfile,
+			ExpiresAt:         request.ExpiresAt,
+			TraceID:           request.TraceID,
+		}
+		if err := s.publisher.PublishCaptureFrame(r.Context(), deviceID, command); err != nil {
+			writeError(w, http.StatusBadGateway, "failed to publish capture command")
+			return
+		}
+	}
+	writeJSON(w, http.StatusAccepted, FrameRequestResponse{
+		SchemaVersion: contract.SchemaVersion,
+		RequestID:     request.RequestID,
+		DeviceIDs:     request.DeviceIDs,
+		Status:        "accepted",
+	})
+}
+
+func (s *Service) beginFrame(frameID string, expiresAt, nowMillis int64) (*frameResult, bool) {
+	s.resultsMu.Lock()
+	defer s.resultsMu.Unlock()
+	for id, result := range s.results {
+		if result.expiresAt <= nowMillis {
+			delete(s.results, id)
+		}
+	}
+	if result, ok := s.results[frameID]; ok {
+		return result, false
+	}
+	result := &frameResult{done: make(chan struct{}), expiresAt: expiresAt}
+	s.results[frameID] = result
+	delay := time.Duration(expiresAt-nowMillis) * time.Millisecond
+	time.AfterFunc(delay, func() {
+		s.resultsMu.Lock()
+		if s.results[frameID] == result {
+			delete(s.results, frameID)
+		}
+		s.resultsMu.Unlock()
+	})
+	return result, true
+}
+
+func (s *Service) finishFrame(result *frameResult, response FrameResponse, err error) {
+	s.resultsMu.Lock()
+	result.response = response
+	result.err = err
+	close(result.done)
+	s.resultsMu.Unlock()
+}
+
+func readFrame(w http.ResponseWriter, r *http.Request, maxFrameBytes int64) (contract.FrameMetadata, []byte, error) {
+	r.Body = http.MaxBytesReader(w, r.Body, maxFrameBytes+maxMetadataBytes)
+	reader, err := r.MultipartReader()
+	if err != nil {
+		return contract.FrameMetadata{}, nil, errors.New("content type must be multipart/form-data")
+	}
+	var metadata contract.FrameMetadata
+	var frame []byte
+	for {
+		part, err := reader.NextPart()
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			var tooLarge *http.MaxBytesError
+			if errors.As(err, &tooLarge) {
+				return contract.FrameMetadata{}, nil, errRequestTooLarge
+			}
+			return contract.FrameMetadata{}, nil, errors.New("invalid multipart body")
+		}
+		switch part.FormName() {
+		case "metadata":
+			if err := decodeMetadataPart(part, &metadata); err != nil {
+				return contract.FrameMetadata{}, nil, err
+			}
+		case "frame":
+			if part.Header.Get("Content-Type") != "image/jpeg" {
+				return contract.FrameMetadata{}, nil, errors.New("frame content type must be image/jpeg")
+			}
+			frame, err = io.ReadAll(io.LimitReader(part, maxFrameBytes+1))
+			if err != nil || int64(len(frame)) > maxFrameBytes {
+				return contract.FrameMetadata{}, nil, errRequestTooLarge
+			}
+		}
+	}
+	if metadata.FrameID == "" || len(frame) == 0 {
+		return contract.FrameMetadata{}, nil, errors.New("metadata and frame parts are required")
+	}
+	return metadata, frame, nil
+}
+
+func decodeMetadataPart(part *multipart.Part, target *contract.FrameMetadata) error {
+	value, err := io.ReadAll(io.LimitReader(part, maxMetadataBytes+1))
+	if err != nil || len(value) > maxMetadataBytes {
+		return errRequestTooLarge
+	}
+	decoder := json.NewDecoder(bytes.NewReader(value))
+	decoder.DisallowUnknownFields()
+	if err := decoder.Decode(target); err != nil {
+		return errors.New("invalid metadata JSON")
+	}
+	return nil
+}
+
+func writeError(w http.ResponseWriter, status int, message string) {
+	writeJSON(w, status, map[string]any{
+		"schemaVersion": contract.SchemaVersion,
+		"error":         message,
+	})
+}
+
+func writeJSON(w http.ResponseWriter, status int, value any) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	if err := json.NewEncoder(w).Encode(value); err != nil {
+		slog.Error("failed to encode HTTP response", "error", err)
+	}
+}

examples/frame-processor/service/service_test.go

@@ -0,0 +1,267 @@
+package service
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"image"
+	"image/color"
+	"image/jpeg"
+	"mime/multipart"
+	"net/http"
+	"net/http/httptest"
+	"net/textproto"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/kerberos-io/agent/examples/frame-processor/contract"
+	"github.com/kerberos-io/agent/examples/frame-processor/processor"
+)
+
+type recordingPublish struct {
+	deviceID string
+	command  contract.RecordingWindowCommand
+}
+
+type fakePublisher struct {
+	mu         sync.Mutex
+	captures   []contract.CaptureFrameCommand
+	recordings []recordingPublish
+}
+
+type blockingProcessor struct {
+	started chan struct{}
+	release chan struct{}
+	mu      sync.Mutex
+	calls   int
+}
+
+func (p *blockingProcessor) Process(ctx context.Context, _ contract.FrameMetadata, _ []byte) (processor.Decision, error) {
+	p.mu.Lock()
+	p.calls++
+	if p.calls == 1 {
+		close(p.started)
+	}
+	p.mu.Unlock()
+	select {
+	case <-ctx.Done():
+		return processor.Decision{}, ctx.Err()
+	case <-p.release:
+		return processor.Decision{Triggered: true, Reason: "test"}, nil
+	}
+}
+
+func (p *fakePublisher) PublishCaptureFrame(_ context.Context, _ string, command contract.CaptureFrameCommand) error {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.captures = append(p.captures, command)
+	return nil
+}
+
+func (p *fakePublisher) PublishRecordingWindow(_ context.Context, deviceID string, command contract.RecordingWindowCommand) error {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.recordings = append(p.recordings, recordingPublish{deviceID: deviceID, command: command})
+	return nil
+}
+
+func TestFrameAlwaysTriggerPublishesOneIdempotentRecordingCommand(t *testing.T) {
+	publisher := &fakePublisher{}
+	service := New(Config{
+		APIToken: "secret", CommandTTL: time.Minute,
+		PreRollSeconds: 10, EventClipSeconds: 30,
+	}, processor.New(processor.Config{}), publisher, nil)
+	service.now = func() time.Time { return time.UnixMilli(1_000) }
+	server := httptest.NewServer(service.Handler())
+	defer server.Close()
+
+	metadata := validMetadata()
+	for range 2 {
+		response := postFrame(t, server.URL, "secret", metadata, jpegFrame(t, 2, 2, 255))
+		if response.StatusCode != http.StatusOK {
+			t.Fatalf("POST /v1/frames status = %d", response.StatusCode)
+		}
+		response.Body.Close()
+	}
+
+	if got := len(publisher.recordings); got != 1 {
+		t.Fatalf("recording commands = %d, want 1", got)
+	}
+	published := publisher.recordings[0]
+	if published.deviceID != metadata.DeviceID || published.command.CapturedAt != metadata.CapturedAt {
+		t.Fatalf("published command = %#v", published)
+	}
+}
+
+func TestFrameRequestPublishesCaptureCommand(t *testing.T) {
+	publisher := &fakePublisher{}
+	service := New(Config{APIToken: "secret"}, processor.New(processor.Config{}), publisher, func() string { return "generated-request" })
+	service.now = func() time.Time { return time.UnixMilli(1_000) }
+	server := httptest.NewServer(service.Handler())
+	defer server.Close()
+
+	body := `{"schemaVersion":"1.0","deviceIds":["device-1"],"processingProfile":"always-trigger","expiresAt":2000}`
+	request, err := http.NewRequest(http.MethodPost, server.URL+"/v1/frame-requests", strings.NewReader(body))
+	if err != nil {
+		t.Fatal(err)
+	}
+	request.Header.Set("Authorization", "Bearer secret")
+	request.Header.Set("Content-Type", "application/json")
+	response, err := http.DefaultClient.Do(request)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusAccepted {
+		t.Fatalf("POST /v1/frame-requests status = %d", response.StatusCode)
+	}
+	if got := len(publisher.captures); got != 1 || publisher.captures[0].RequestID != "generated-request" {
+		t.Fatalf("capture commands = %#v", publisher.captures)
+	}
+}
+
+func TestFrameRejectsUnauthorizedRequest(t *testing.T) {
+	service := New(Config{APIToken: "secret"}, processor.New(processor.Config{}), &fakePublisher{}, nil)
+	server := httptest.NewServer(service.Handler())
+	defer server.Close()
+
+	response := postFrame(t, server.URL, "wrong", validMetadata(), jpegFrame(t, 2, 2, 255))
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusUnauthorized {
+		t.Fatalf("POST /v1/frames status = %d", response.StatusCode)
+	}
+}
+
+func TestFrameFailsClosedWithoutConfiguredToken(t *testing.T) {
+	application := New(Config{}, processor.New(processor.Config{}), &fakePublisher{}, nil)
+	server := httptest.NewServer(application.Handler())
+	defer server.Close()
+
+	response := postFrame(t, server.URL, "", validMetadata(), jpegFrame(t, 2, 2, 255))
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusServiceUnavailable {
+		t.Fatalf("POST /v1/frames status = %d", response.StatusCode)
+	}
+}
+
+func TestConcurrentDuplicateFramesPublishOneRecordingCommand(t *testing.T) {
+	publisher := &fakePublisher{}
+	frameProcessor := &blockingProcessor{started: make(chan struct{}), release: make(chan struct{})}
+	application := New(Config{
+		APIToken: "secret", CommandTTL: time.Minute,
+		PreRollSeconds: 10, EventClipSeconds: 30,
+	}, frameProcessor, publisher, nil)
+	application.now = func() time.Time { return time.UnixMilli(1_000) }
+	server := httptest.NewServer(application.Handler())
+	defer server.Close()
+
+	metadata := validMetadata()
+	statuses := make(chan int, 2)
+	go func() {
+		response := postFrame(t, server.URL, "secret", metadata, jpegFrame(t, 2, 2, 255))
+		defer response.Body.Close()
+		statuses <- response.StatusCode
+	}()
+	<-frameProcessor.started
+	go func() {
+		response := postFrame(t, server.URL, "secret", metadata, jpegFrame(t, 2, 2, 255))
+		defer response.Body.Close()
+		statuses <- response.StatusCode
+	}()
+	close(frameProcessor.release)
+
+	for range 2 {
+		if status := <-statuses; status != http.StatusOK {
+			t.Fatalf("POST /v1/frames status = %d", status)
+		}
+	}
+	if got := len(publisher.recordings); got != 1 {
+		t.Fatalf("recording commands = %d, want 1", got)
+	}
+	frameProcessor.mu.Lock()
+	defer frameProcessor.mu.Unlock()
+	if frameProcessor.calls != 1 {
+		t.Fatalf("processor calls = %d, want 1", frameProcessor.calls)
+	}
+}
+
+func TestFrameRejectsTTLAboveConfiguredMaximum(t *testing.T) {
+	application := New(Config{APIToken: "secret", MaxFrameTTL: time.Second}, processor.New(processor.Config{}), &fakePublisher{}, nil)
+	application.now = func() time.Time { return time.UnixMilli(1_000) }
+	server := httptest.NewServer(application.Handler())
+	defer server.Close()
+
+	metadata := validMetadata()
+	metadata.ExpiresAt = 2_001
+	response := postFrame(t, server.URL, "secret", metadata, jpegFrame(t, 2, 2, 255))
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusUnprocessableEntity {
+		t.Fatalf("POST /v1/frames status = %d", response.StatusCode)
+	}
+}
+
+func validMetadata() contract.FrameMetadata {
+	return contract.FrameMetadata{
+		SchemaVersion: contract.SchemaVersion,
+		RequestID:     "request-1", FrameID: "frame-1", DeviceID: "device-1",
+		CapturedAt: 900, ExpiresAt: 2_000, ProcessingProfile: processor.ProfileAlwaysTrigger,
+		SourceStream: "sub", Width: 2, Height: 2,
+	}
+}
+
+func jpegFrame(t *testing.T, width, height int, brightness uint8) []byte {
+	t.Helper()
+	frame := image.NewGray(image.Rect(0, 0, width, height))
+	for index := range frame.Pix {
+		frame.Pix[index] = brightness
+	}
+	frame.SetGray(0, 0, color.Gray{Y: brightness})
+	var output bytes.Buffer
+	if err := jpeg.Encode(&output, frame, nil); err != nil {
+		t.Fatal(err)
+	}
+	return output.Bytes()
+}
+
+func postFrame(t *testing.T, baseURL, token string, metadata contract.FrameMetadata, frame []byte) *http.Response {
+	t.Helper()
+	var body bytes.Buffer
+	writer := multipart.NewWriter(&body)
+	metadataHeader := make(textproto.MIMEHeader)
+	metadataHeader.Set("Content-Disposition", `form-data; name="metadata"`)
+	metadataHeader.Set("Content-Type", "application/json")
+	part, err := writer.CreatePart(metadataHeader)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := json.NewEncoder(part).Encode(metadata); err != nil {
+		t.Fatal(err)
+	}
+	frameHeader := make(textproto.MIMEHeader)
+	frameHeader.Set("Content-Disposition", `form-data; name="frame"; filename="frame.jpg"`)
+	frameHeader.Set("Content-Type", "image/jpeg")
+	part, err = writer.CreatePart(frameHeader)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := part.Write(frame); err != nil {
+		t.Fatal(err)
+	}
+	if err := writer.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	request, err := http.NewRequest(http.MethodPost, baseURL+"/v1/frames", &body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	request.Header.Set("Authorization", "Bearer "+token)
+	request.Header.Set("Content-Type", writer.FormDataContentType())
+	response, err := http.DefaultClient.Do(request)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return response
+}

examples/frame-processor/testdata/capture-frame.json

@@ -0,0 +1,7 @@
+{
+  "schemaVersion": "1.0",
+  "requestId": "request-example-1",
+  "processingProfile": "always-trigger",
+  "expiresAt": 4102444800000,
+  "traceId": "trace-example-1"
+}

examples/frame-processor/testdata/frame-request.json

@@ -0,0 +1,8 @@
+{
+  "schemaVersion": "1.0",
+  "requestId": "request-example-1",
+  "deviceIds": ["camera-1"],
+  "processingProfile": "always-trigger",
+  "expiresAt": 4102444800000,
+  "traceId": "trace-example-1"
+}

examples/frame-processor/testdata/request-recording-window.json

@@ -0,0 +1,11 @@
+{
+  "schemaVersion": "1.0",
+  "requestId": "request-example-1",
+  "frameId": "frame-example-1",
+  "capturedAt": 1789380000123,
+  "preRollSeconds": 10,
+  "eventClipSeconds": 30,
+  "expiresAt": 4102444800000,
+  "processingProfile": "always-trigger",
+  "traceId": "trace-example-1"
+}

machinery/README.md

@@ -5,7 +5,6 @@
 <a target="_blank" href="https://twitter.com/kerberosio?ref_src=twsrc%5Etfw"><img src="https://img.shields.io/twitter/url.svg?label=Follow%20%40kerberosio&style=social&url=https%3A%2F%2Ftwitter.com%2Fkerberosio" alt="Twitter Widget"></a>
 <a target="_blank" href="https://join.slack.com/t/kerberosio/shared_invite/zt-1a5oj4pwm-O4qCAN9c5r2um0Ns0ge8ww"><img src="https://img.shields.io/badge/slack-@kerberosio-yellow.svg?logo=slack " alt="Kerberos.io"></a>
 
-<a target="_blank" href="https://circleci.com/gh/kerberos-io/agent"><img src="https://circleci.com/gh/kerberos-io/agent.svg?style=svg"/></a>
 <img src="https://github.com/kerberos-io/agent/workflows/Go/badge.svg"/>
 <img src="https://github.com/kerberos-io/agent/workflows/CodeQL/badge.svg"/>
 

machinery/data/tus/1788718145_3-694_encrypted_0-0-0-0_-1_25497.mp4.primary.json

@@ -0,0 +1 @@
+{"upload_url":"https://vault.kerberos.io/api/storage/tus/bd3be0f9ecd3873d381f57cb8b4a7e96","vault_uri":"https://vault.kerberos.io/api/storage/tus/","size":2560480}

machinery/docs/docs.go

@@ -24,6 +24,38 @@ const docTemplate = `{
     "host": "{{.Host}}",
     "basePath": "{{.BasePath}}",
     "paths": {
+        "/api/camera/discover": {
+            "get": {
+                "description": "Runs an advanced Fing/WiFiman-style scan (ONVIF WS-Discovery + TCP port scan + MAC/vendor lookup) and returns the devices found on the local network.",
+                "tags": [
+                    "onvif"
+                ],
+                "summary": "Discover cameras and other devices on the local network.",
+                "operationId": "camera-discover",
+                "parameters": [
+                    {
+                        "type": "integer",
+                        "description": "Discovery timeout in milliseconds (default 2000)",
+                        "name": "timeout",
+                        "in": "query"
+                    },
+                    {
+                        "type": "string",
+                        "description": "Optional subnet(s) to scan, e.g. '192.168.1.0/24' (comma-separated). Defaults to the local interfaces.",
+                        "name": "subnet",
+                        "in": "query"
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/models.APIResponse"
+                        }
+                    }
+                }
+            }
+        },
         "/api/camera/onvif/capabilities": {
             "post": {
                 "description": "Will return the ONVIF capabilities for the specific camera.",
@@ -623,6 +655,40 @@ const docTemplate = `{
                 }
             }
         },
+        "/api/persistence/secondary/verify": {
+            "post": {
+                "security": [
+                    {
+                        "Bearer": []
+                    }
+                ],
+                "description": "Will verify the secondary persistence.",
+                "tags": [
+                    "persistence"
+                ],
+                "summary": "Will verify the secondary persistence.",
+                "operationId": "verify-secondary-persistence",
+                "parameters": [
+                    {
+                        "description": "Config",
+                        "name": "config",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/models.Config"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/models.APIResponse"
+                        }
+                    }
+                }
+            }
+        },
         "/api/persistence/verify": {
             "post": {
                 "security": [
@@ -656,9 +722,151 @@ const docTemplate = `{
                     }
                 }
             }
+        },
+        "/health": {
+            "get": {
+                "description": "Confirms that the Agent HTTP process can serve requests and reports current camera stream and Hub heartbeat diagnostics. Operational dependency failures do not change the liveness HTTP status.",
+                "produces": [
+                    "application/json"
+                ],
+                "tags": [
+                    "health"
+                ],
+                "summary": "Check Agent health",
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/http.HealthResponse"
+                        }
+                    }
+                }
+            }
         }
     },
     "definitions": {
+        "http.Health": {
+            "type": "object",
+            "properties": {
+                "cameraConnected": {
+                    "type": "boolean"
+                },
+                "description": {
+                    "type": "string"
+                },
+                "hub": {
+                    "$ref": "#/definitions/http.HubHealth"
+                },
+                "mainStream": {
+                    "$ref": "#/definitions/http.StreamHealth"
+                },
+                "subStream": {
+                    "$ref": "#/definitions/http.StreamHealth"
+                }
+            }
+        },
+        "http.HealthResponse": {
+            "type": "object",
+            "properties": {
+                "applicationStatusCode": {
+                    "type": "string"
+                },
+                "data": {
+                    "$ref": "#/definitions/http.HealthResponseData"
+                },
+                "entityStatusCode": {
+                    "type": "string"
+                },
+                "httpStatusCode": {
+                    "type": "integer"
+                },
+                "message": {
+                    "type": "string"
+                },
+                "metadata": {
+                    "$ref": "#/definitions/http.ResponseMetadata"
+                }
+            }
+        },
+        "http.HealthResponseData": {
+            "type": "object",
+            "properties": {
+                "health": {
+                    "$ref": "#/definitions/http.Health"
+                }
+            }
+        },
+        "http.HubHealth": {
+            "type": "object",
+            "properties": {
+                "configured": {
+                    "type": "boolean"
+                },
+                "connected": {
+                    "type": "boolean"
+                },
+                "lastHeartbeatAttemptAt": {
+                    "type": "integer"
+                },
+                "lastSuccessfulHeartbeatAt": {
+                    "type": "integer"
+                }
+            }
+        },
+        "http.ResponseMetadata": {
+            "type": "object",
+            "properties": {
+                "applicationName": {
+                    "type": "string"
+                },
+                "applicationVersion": {
+                    "type": "string"
+                },
+                "path": {
+                    "type": "string"
+                },
+                "timestamp": {
+                    "type": "integer"
+                },
+                "traceId": {
+                    "type": "string"
+                }
+            }
+        },
+        "http.StreamHealth": {
+            "type": "object",
+            "properties": {
+                "configured": {
+                    "type": "boolean"
+                },
+                "connected": {
+                    "type": "boolean"
+                },
+                "fps": {
+                    "type": "number"
+                },
+                "lastPacketAt": {
+                    "type": "integer"
+                },
+                "packagesProcessed": {
+                    "type": "integer"
+                },
+                "resolution": {
+                    "$ref": "#/definitions/http.StreamResolution"
+                }
+            }
+        },
+        "http.StreamResolution": {
+            "type": "object",
+            "properties": {
+                "height": {
+                    "type": "integer"
+                },
+                "width": {
+                    "type": "integer"
+                }
+            }
+        },
         "models.APIResponse": {
             "type": "object",
             "properties": {
@@ -730,12 +938,19 @@ const docTemplate = `{
                 "fragmentedduration": {
                     "type": "integer"
                 },
+                "gopsize": {
+                    "description": "GOP size in seconds, used for pre-recording",
+                    "type": "integer"
+                },
                 "ipcamera": {
                     "$ref": "#/definitions/models.IPCamera"
                 },
                 "liveview": {
                     "type": "string"
                 },
+                "liveview_chunking": {
+                    "type": "string"
+                },
                 "maxlengthrecording": {
                     "type": "integer"
                 },
@@ -745,6 +960,10 @@ const docTemplate = `{
                 "name": {
                     "type": "string"
                 },
+                "onvif_motion": {
+                    "description": "ONVIFMotion routes the camera's ONVIF motion events into the\nagent's motion-triggered recording pipeline. When \"true\" the\nagent opens an event/stream against the configured ONVIF\nendpoint and forwards Motion+Active events to HandleMotion.\nRequires Capture.IPCamera.ONVIFXAddr / ONVIFUsername /\nONVIFPassword to be set. Default empty (disabled) keeps the\nexisting pixel-diff motion detection as the only source.",
+                    "type": "string"
+                },
                 "pixelChangeThreshold": {
                     "type": "integer"
                 },
@@ -823,9 +1042,15 @@ const docTemplate = `{
                 "kstorage": {
                     "$ref": "#/definitions/models.KStorage"
                 },
+                "kstorage_secondary": {
+                    "$ref": "#/definitions/models.KStorage"
+                },
                 "max_directory_size": {
                     "type": "integer"
                 },
+                "min_free_space": {
+                    "type": "integer"
+                },
                 "mqtt_password": {
                     "type": "string"
                 },
@@ -856,6 +1081,9 @@ const docTemplate = `{
                 "s3": {
                     "$ref": "#/definitions/models.S3"
                 },
+                "signing": {
+                    "$ref": "#/definitions/models.Signing"
+                },
                 "stunuri": {
                     "type": "string"
                 },
@@ -947,6 +1175,15 @@ const docTemplate = `{
         "models.IPCamera": {
             "type": "object",
             "properties": {
+                "base_height": {
+                    "type": "integer"
+                },
+                "base_width": {
+                    "type": "integer"
+                },
+                "channels": {
+                    "type": "integer"
+                },
                 "fps": {
                     "type": "string"
                 },
@@ -965,21 +1202,85 @@ const docTemplate = `{
                 "onvif_xaddr": {
                     "type": "string"
                 },
+                "pps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "rtsp": {
                     "type": "string"
                 },
+                "sample_rate": {
+                    "type": "integer"
+                },
+                "sps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_fps": {
                     "type": "string"
                 },
                 "sub_height": {
                     "type": "integer"
                 },
+                "sub_pps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_rtsp": {
                     "type": "string"
                 },
+                "sub_sps_nalus": {
+                    "description": "Sub stream parameter sets, captured separately from the main stream so the\nlive HLS muxer can build a correct init segment when a viewer switches the\nlive view to the sub (low-resolution) stream.",
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
+                "sub_vps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_width": {
                     "type": "integer"
                 },
+                "vps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "width": {
                     "type": "integer"
                 }
@@ -998,12 +1299,18 @@ const docTemplate = `{
                 "directory": {
                     "type": "string"
                 },
+                "max_retries": {
+                    "type": "integer"
+                },
                 "provider": {
                     "type": "string"
                 },
                 "secret_access_key": {
                     "type": "string"
                 },
+                "timeout": {
+                    "type": "integer"
+                },
                 "uri": {
                     "type": "string"
                 }
@@ -1141,6 +1448,17 @@ const docTemplate = `{
                 }
             }
         },
+        "models.Signing": {
+            "type": "object",
+            "properties": {
+                "enabled": {
+                    "type": "string"
+                },
+                "private_key": {
+                    "type": "string"
+                }
+            }
+        },
         "models.Timetable": {
             "type": "object",
             "properties": {

machinery/docs/swagger.json

@@ -17,6 +17,38 @@
     },
     "basePath": "/",
     "paths": {
+        "/api/camera/discover": {
+            "get": {
+                "description": "Runs an advanced Fing/WiFiman-style scan (ONVIF WS-Discovery + TCP port scan + MAC/vendor lookup) and returns the devices found on the local network.",
+                "tags": [
+                    "onvif"
+                ],
+                "summary": "Discover cameras and other devices on the local network.",
+                "operationId": "camera-discover",
+                "parameters": [
+                    {
+                        "type": "integer",
+                        "description": "Discovery timeout in milliseconds (default 2000)",
+                        "name": "timeout",
+                        "in": "query"
+                    },
+                    {
+                        "type": "string",
+                        "description": "Optional subnet(s) to scan, e.g. '192.168.1.0/24' (comma-separated). Defaults to the local interfaces.",
+                        "name": "subnet",
+                        "in": "query"
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/models.APIResponse"
+                        }
+                    }
+                }
+            }
+        },
         "/api/camera/onvif/capabilities": {
             "post": {
                 "description": "Will return the ONVIF capabilities for the specific camera.",
@@ -616,6 +648,40 @@
                 }
             }
         },
+        "/api/persistence/secondary/verify": {
+            "post": {
+                "security": [
+                    {
+                        "Bearer": []
+                    }
+                ],
+                "description": "Will verify the secondary persistence.",
+                "tags": [
+                    "persistence"
+                ],
+                "summary": "Will verify the secondary persistence.",
+                "operationId": "verify-secondary-persistence",
+                "parameters": [
+                    {
+                        "description": "Config",
+                        "name": "config",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/models.Config"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/models.APIResponse"
+                        }
+                    }
+                }
+            }
+        },
         "/api/persistence/verify": {
             "post": {
                 "security": [
@@ -649,9 +715,151 @@
                     }
                 }
             }
+        },
+        "/health": {
+            "get": {
+                "description": "Confirms that the Agent HTTP process can serve requests and reports current camera stream and Hub heartbeat diagnostics. Operational dependency failures do not change the liveness HTTP status.",
+                "produces": [
+                    "application/json"
+                ],
+                "tags": [
+                    "health"
+                ],
+                "summary": "Check Agent health",
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/http.HealthResponse"
+                        }
+                    }
+                }
+            }
         }
     },
     "definitions": {
+        "http.Health": {
+            "type": "object",
+            "properties": {
+                "cameraConnected": {
+                    "type": "boolean"
+                },
+                "description": {
+                    "type": "string"
+                },
+                "hub": {
+                    "$ref": "#/definitions/http.HubHealth"
+                },
+                "mainStream": {
+                    "$ref": "#/definitions/http.StreamHealth"
+                },
+                "subStream": {
+                    "$ref": "#/definitions/http.StreamHealth"
+                }
+            }
+        },
+        "http.HealthResponse": {
+            "type": "object",
+            "properties": {
+                "applicationStatusCode": {
+                    "type": "string"
+                },
+                "data": {
+                    "$ref": "#/definitions/http.HealthResponseData"
+                },
+                "entityStatusCode": {
+                    "type": "string"
+                },
+                "httpStatusCode": {
+                    "type": "integer"
+                },
+                "message": {
+                    "type": "string"
+                },
+                "metadata": {
+                    "$ref": "#/definitions/http.ResponseMetadata"
+                }
+            }
+        },
+        "http.HealthResponseData": {
+            "type": "object",
+            "properties": {
+                "health": {
+                    "$ref": "#/definitions/http.Health"
+                }
+            }
+        },
+        "http.HubHealth": {
+            "type": "object",
+            "properties": {
+                "configured": {
+                    "type": "boolean"
+                },
+                "connected": {
+                    "type": "boolean"
+                },
+                "lastHeartbeatAttemptAt": {
+                    "type": "integer"
+                },
+                "lastSuccessfulHeartbeatAt": {
+                    "type": "integer"
+                }
+            }
+        },
+        "http.ResponseMetadata": {
+            "type": "object",
+            "properties": {
+                "applicationName": {
+                    "type": "string"
+                },
+                "applicationVersion": {
+                    "type": "string"
+                },
+                "path": {
+                    "type": "string"
+                },
+                "timestamp": {
+                    "type": "integer"
+                },
+                "traceId": {
+                    "type": "string"
+                }
+            }
+        },
+        "http.StreamHealth": {
+            "type": "object",
+            "properties": {
+                "configured": {
+                    "type": "boolean"
+                },
+                "connected": {
+                    "type": "boolean"
+                },
+                "fps": {
+                    "type": "number"
+                },
+                "lastPacketAt": {
+                    "type": "integer"
+                },
+                "packagesProcessed": {
+                    "type": "integer"
+                },
+                "resolution": {
+                    "$ref": "#/definitions/http.StreamResolution"
+                }
+            }
+        },
+        "http.StreamResolution": {
+            "type": "object",
+            "properties": {
+                "height": {
+                    "type": "integer"
+                },
+                "width": {
+                    "type": "integer"
+                }
+            }
+        },
         "models.APIResponse": {
             "type": "object",
             "properties": {
@@ -723,12 +931,19 @@
                 "fragmentedduration": {
                     "type": "integer"
                 },
+                "gopsize": {
+                    "description": "GOP size in seconds, used for pre-recording",
+                    "type": "integer"
+                },
                 "ipcamera": {
                     "$ref": "#/definitions/models.IPCamera"
                 },
                 "liveview": {
                     "type": "string"
                 },
+                "liveview_chunking": {
+                    "type": "string"
+                },
                 "maxlengthrecording": {
                     "type": "integer"
                 },
@@ -738,6 +953,10 @@
                 "name": {
                     "type": "string"
                 },
+                "onvif_motion": {
+                    "description": "ONVIFMotion routes the camera's ONVIF motion events into the\nagent's motion-triggered recording pipeline. When \"true\" the\nagent opens an event/stream against the configured ONVIF\nendpoint and forwards Motion+Active events to HandleMotion.\nRequires Capture.IPCamera.ONVIFXAddr / ONVIFUsername /\nONVIFPassword to be set. Default empty (disabled) keeps the\nexisting pixel-diff motion detection as the only source.",
+                    "type": "string"
+                },
                 "pixelChangeThreshold": {
                     "type": "integer"
                 },
@@ -816,9 +1035,15 @@
                 "kstorage": {
                     "$ref": "#/definitions/models.KStorage"
                 },
+                "kstorage_secondary": {
+                    "$ref": "#/definitions/models.KStorage"
+                },
                 "max_directory_size": {
                     "type": "integer"
                 },
+                "min_free_space": {
+                    "type": "integer"
+                },
                 "mqtt_password": {
                     "type": "string"
                 },
@@ -849,6 +1074,9 @@
                 "s3": {
                     "$ref": "#/definitions/models.S3"
                 },
+                "signing": {
+                    "$ref": "#/definitions/models.Signing"
+                },
                 "stunuri": {
                     "type": "string"
                 },
@@ -940,6 +1168,15 @@
         "models.IPCamera": {
             "type": "object",
             "properties": {
+                "base_height": {
+                    "type": "integer"
+                },
+                "base_width": {
+                    "type": "integer"
+                },
+                "channels": {
+                    "type": "integer"
+                },
                 "fps": {
                     "type": "string"
                 },
@@ -958,21 +1195,85 @@
                 "onvif_xaddr": {
                     "type": "string"
                 },
+                "pps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "rtsp": {
                     "type": "string"
                 },
+                "sample_rate": {
+                    "type": "integer"
+                },
+                "sps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_fps": {
                     "type": "string"
                 },
                 "sub_height": {
                     "type": "integer"
                 },
+                "sub_pps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_rtsp": {
                     "type": "string"
                 },
+                "sub_sps_nalus": {
+                    "description": "Sub stream parameter sets, captured separately from the main stream so the\nlive HLS muxer can build a correct init segment when a viewer switches the\nlive view to the sub (low-resolution) stream.",
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
+                "sub_vps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_width": {
                     "type": "integer"
                 },
+                "vps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "width": {
                     "type": "integer"
                 }
@@ -991,12 +1292,18 @@
                 "directory": {
                     "type": "string"
                 },
+                "max_retries": {
+                    "type": "integer"
+                },
                 "provider": {
                     "type": "string"
                 },
                 "secret_access_key": {
                     "type": "string"
                 },
+                "timeout": {
+                    "type": "integer"
+                },
                 "uri": {
                     "type": "string"
                 }
@@ -1134,6 +1441,17 @@
                 }
             }
         },
+        "models.Signing": {
+            "type": "object",
+            "properties": {
+                "enabled": {
+                    "type": "string"
+                },
+                "private_key": {
+                    "type": "string"
+                }
+            }
+        },
         "models.Timetable": {
             "type": "object",
             "properties": {

machinery/docs/swagger.yaml

@@ -1,5 +1,84 @@
 basePath: /
 definitions:
+  http.Health:
+    properties:
+      cameraConnected:
+        type: boolean
+      description:
+        type: string
+      hub:
+        $ref: '#/definitions/http.HubHealth'
+      mainStream:
+        $ref: '#/definitions/http.StreamHealth'
+      subStream:
+        $ref: '#/definitions/http.StreamHealth'
+    type: object
+  http.HealthResponse:
+    properties:
+      applicationStatusCode:
+        type: string
+      data:
+        $ref: '#/definitions/http.HealthResponseData'
+      entityStatusCode:
+        type: string
+      httpStatusCode:
+        type: integer
+      message:
+        type: string
+      metadata:
+        $ref: '#/definitions/http.ResponseMetadata'
+    type: object
+  http.HealthResponseData:
+    properties:
+      health:
+        $ref: '#/definitions/http.Health'
+    type: object
+  http.HubHealth:
+    properties:
+      configured:
+        type: boolean
+      connected:
+        type: boolean
+      lastHeartbeatAttemptAt:
+        type: integer
+      lastSuccessfulHeartbeatAt:
+        type: integer
+    type: object
+  http.ResponseMetadata:
+    properties:
+      applicationName:
+        type: string
+      applicationVersion:
+        type: string
+      path:
+        type: string
+      timestamp:
+        type: integer
+      traceId:
+        type: string
+    type: object
+  http.StreamHealth:
+    properties:
+      configured:
+        type: boolean
+      connected:
+        type: boolean
+      fps:
+        type: number
+      lastPacketAt:
+        type: integer
+      packagesProcessed:
+        type: integer
+      resolution:
+        $ref: '#/definitions/http.StreamResolution'
+    type: object
+  http.StreamResolution:
+    properties:
+      height:
+        type: integer
+      width:
+        type: integer
+    type: object
   models.APIResponse:
     properties:
       can_pan_tilt:
@@ -47,16 +126,31 @@ definitions:
         type: string
       fragmentedduration:
         type: integer
+      gopsize:
+        description: GOP size in seconds, used for pre-recording
+        type: integer
       ipcamera:
         $ref: '#/definitions/models.IPCamera'
       liveview:
         type: string
+      liveview_chunking:
+        type: string
       maxlengthrecording:
         type: integer
       motion:
         type: string
       name:
         type: string
+      onvif_motion:
+        description: |-
+          ONVIFMotion routes the camera's ONVIF motion events into the
+          agent's motion-triggered recording pipeline. When "true" the
+          agent opens an event/stream against the configured ONVIF
+          endpoint and forwards Motion+Active events to HandleMotion.
+          Requires Capture.IPCamera.ONVIFXAddr / ONVIFUsername /
+          ONVIFPassword to be set. Default empty (disabled) keeps the
+          existing pixel-diff motion detection as the only source.
+        type: string
       pixelChangeThreshold:
         type: integer
       postrecording:
@@ -109,8 +203,12 @@ definitions:
         type: string
       kstorage:
         $ref: '#/definitions/models.KStorage'
+      kstorage_secondary:
+        $ref: '#/definitions/models.KStorage'
       max_directory_size:
         type: integer
+      min_free_space:
+        type: integer
       mqtt_password:
         type: string
       mqtt_username:
@@ -131,6 +229,8 @@ definitions:
         type: string
       s3:
         $ref: '#/definitions/models.S3'
+      signing:
+        $ref: '#/definitions/models.Signing'
       stunuri:
         type: string
       time:
@@ -190,6 +290,12 @@ definitions:
     type: object
   models.IPCamera:
     properties:
+      base_height:
+        type: integer
+      base_width:
+        type: integer
+      channels:
+        type: integer
       fps:
         type: string
       height:
@@ -202,16 +308,64 @@ definitions:
         type: string
       onvif_xaddr:
         type: string
+      pps_nalus:
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
       rtsp:
         type: string
+      sample_rate:
+        type: integer
+      sps_nalus:
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
       sub_fps:
         type: string
       sub_height:
         type: integer
+      sub_pps_nalus:
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
       sub_rtsp:
         type: string
+      sub_sps_nalus:
+        description: |-
+          Sub stream parameter sets, captured separately from the main stream so the
+          live HLS muxer can build a correct init segment when a viewer switches the
+          live view to the sub (low-resolution) stream.
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
+      sub_vps_nalus:
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
       sub_width:
         type: integer
+      vps_nalus:
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
       width:
         type: integer
     type: object
@@ -224,10 +378,14 @@ definitions:
         type: string
       directory:
         type: string
+      max_retries:
+        type: integer
       provider:
         type: string
       secret_access_key:
         type: string
+      timeout:
+        type: integer
       uri:
         type: string
     type: object
@@ -316,6 +474,13 @@ definitions:
       username:
         type: string
     type: object
+  models.Signing:
+    properties:
+      enabled:
+        type: string
+      private_key:
+        type: string
+    type: object
   models.Timetable:
     properties:
       end1:
@@ -345,6 +510,30 @@ info:
   title: Swagger Kerberos Agent API
   version: "1.0"
 paths:
+  /api/camera/discover:
+    get:
+      description: Runs an advanced Fing/WiFiman-style scan (ONVIF WS-Discovery +
+        TCP port scan + MAC/vendor lookup) and returns the devices found on the local
+        network.
+      operationId: camera-discover
+      parameters:
+      - description: Discovery timeout in milliseconds (default 2000)
+        in: query
+        name: timeout
+        type: integer
+      - description: Optional subnet(s) to scan, e.g. '192.168.1.0/24' (comma-separated).
+          Defaults to the local interfaces.
+        in: query
+        name: subnet
+        type: string
+      responses:
+        "200":
+          description: OK
+          schema:
+            $ref: '#/definitions/models.APIResponse'
+      summary: Discover cameras and other devices on the local network.
+      tags:
+      - onvif
   /api/camera/onvif/capabilities:
     post:
       description: Will return the ONVIF capabilities for the specific camera.
@@ -735,6 +924,27 @@ paths:
       summary: Get Authorization token.
       tags:
       - authentication
+  /api/persistence/secondary/verify:
+    post:
+      description: Will verify the secondary persistence.
+      operationId: verify-secondary-persistence
+      parameters:
+      - description: Config
+        in: body
+        name: config
+        required: true
+        schema:
+          $ref: '#/definitions/models.Config'
+      responses:
+        "200":
+          description: OK
+          schema:
+            $ref: '#/definitions/models.APIResponse'
+      security:
+      - Bearer: []
+      summary: Will verify the secondary persistence.
+      tags:
+      - persistence
   /api/persistence/verify:
     post:
       description: Will verify the persistence.
@@ -756,6 +966,21 @@ paths:
       summary: Will verify the persistence.
       tags:
       - persistence
+  /health:
+    get:
+      description: Confirms that the Agent HTTP process can serve requests and reports
+        current camera stream and Hub heartbeat diagnostics. Operational dependency
+        failures do not change the liveness HTTP status.
+      produces:
+      - application/json
+      responses:
+        "200":
+          description: OK
+          schema:
+            $ref: '#/definitions/http.HealthResponse'
+      summary: Check Agent health
+      tags:
+      - health
 securityDefinitions:
   Bearer:
     in: header

machinery/go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/bluenviron/gortsplib/v5 v5.6.3
 	github.com/bluenviron/mediacommon v1.14.0
 	github.com/cedricve/go-onvif v0.0.0-20200222191200-567e8ce298f6
+	github.com/creack/pty v1.1.24
 	github.com/dromara/carbon/v2 v2.6.8
 	github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5
 	github.com/eclipse/paho.mqtt.golang v1.5.0
@@ -28,7 +29,6 @@ require (
 	github.com/minio/minio-go/v6 v6.0.57
 	github.com/moq-dev/moq-go v0.5.7
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
-	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/pion/interceptor v0.1.47
 	github.com/pion/rtp v1.10.5
 	github.com/pion/webrtc/v4 v4.2.18
@@ -45,7 +45,6 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.36.0
 	go.opentelemetry.io/otel/sdk v1.36.0
 	go.opentelemetry.io/otel/trace v1.36.0
-	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 )
 
 require (

machinery/go.sum

@@ -456,6 +456,8 @@ github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78/go.mod h1:W+zGtBO5Y1Ig
 github.com/cncf/xds/go v0.0.0-20241223141626-cff3c89139a3/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
 github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
+github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -858,8 +860,6 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
 github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE=
 github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
-github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
-github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
 github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
 github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY=
@@ -2106,8 +2106,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk=
 gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
-gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

machinery/logging.go

@@ -0,0 +1,160 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	defaultLogLevel  = log.InfoLevel
+	defaultLogOutput = "text"
+)
+
+type localTimeFormatter struct {
+	timezone  *time.Location
+	formatter log.Formatter
+}
+
+func (f localTimeFormatter) Format(entry *log.Entry) ([]byte, error) {
+	entry.Time = entry.Time.In(f.timezone)
+	return f.formatter.Format(entry)
+}
+
+type componentHook struct{}
+
+func (componentHook) Levels() []log.Level {
+	return log.AllLevels
+}
+
+func (componentHook) Fire(entry *log.Entry) error {
+	if _, exists := entry.Data["component"]; exists {
+		return nil
+	}
+	entry.Data["component"] = componentFromCaller(entry.Caller)
+	return nil
+}
+
+func configureLogging(levelValue string, outputValue string, timezone *time.Location) {
+	configureLogger(log.StandardLogger(), levelValue, outputValue, timezone)
+}
+
+func configureLogger(logger *log.Logger, levelValue string, outputValue string, timezone *time.Location) {
+	if timezone == nil {
+		timezone = time.Local
+	}
+
+	level, levelErr := parseLogLevel(levelValue)
+	output, outputErr := parseLogOutput(outputValue)
+
+	logger.SetOutput(os.Stdout)
+	logger.SetLevel(level)
+	logger.SetReportCaller(true)
+	logger.SetFormatter(localTimeFormatter{
+		timezone:  timezone,
+		formatter: newLogFormatter(output),
+	})
+	installComponentHook(logger)
+
+	if levelErr != nil {
+		logger.WithFields(log.Fields{
+			"configured_level": levelValue,
+			"effective_level":  level.String(),
+		}).WithError(levelErr).Warn("invalid log level; using default")
+	}
+	if outputErr != nil {
+		logger.WithFields(log.Fields{
+			"configured_output": outputValue,
+			"effective_output":  output,
+		}).WithError(outputErr).Warn("invalid log output; using default")
+	}
+
+	logger.WithFields(log.Fields{
+		"event":         "logger_configured",
+		"log_level":     level.String(),
+		"output":        output,
+		"report_caller": logger.ReportCaller,
+		"timezone":      timezone.String(),
+	}).Debug("logging configured")
+}
+
+func installComponentHook(logger *log.Logger) {
+	for _, hooks := range logger.Hooks {
+		for _, hook := range hooks {
+			if _, ok := hook.(componentHook); ok {
+				return
+			}
+		}
+	}
+	logger.AddHook(componentHook{})
+}
+
+func componentFromCaller(frame *runtime.Frame) string {
+	if frame == nil {
+		return "unknown"
+	}
+
+	const sourceMarker = "/machinery/src/"
+	normalizedFile := filepath.ToSlash(frame.File)
+	if markerIndex := strings.Index(normalizedFile, sourceMarker); markerIndex >= 0 {
+		relativeFile := normalizedFile[markerIndex+len(sourceMarker):]
+		if directory := filepath.ToSlash(filepath.Dir(relativeFile)); directory != "." {
+			return directory
+		}
+	}
+	if strings.Contains(normalizedFile, "/machinery/") {
+		return "agent"
+	}
+	return "unknown"
+}
+
+func parseLogLevel(value string) (log.Level, error) {
+	normalized := strings.ToLower(strings.TrimSpace(value))
+	if normalized == "" {
+		return defaultLogLevel, nil
+	}
+	if normalized == "warning" {
+		normalized = "warn"
+	}
+
+	level, err := log.ParseLevel(normalized)
+	if err != nil {
+		return defaultLogLevel, fmt.Errorf("parse LOG_LEVEL: %w", err)
+	}
+	return level, nil
+}
+
+func parseLogOutput(value string) (string, error) {
+	normalized := strings.ToLower(strings.TrimSpace(value))
+	if normalized == "" {
+		return defaultLogOutput, nil
+	}
+	switch normalized {
+	case "json", "text":
+		return normalized, nil
+	default:
+		return defaultLogOutput, fmt.Errorf("unsupported LOG_OUTPUT %q", value)
+	}
+}
+
+func newLogFormatter(output string) log.Formatter {
+	callerPrettyfier := func(frame *runtime.Frame) (string, string) {
+		return filepath.Base(frame.Function), fmt.Sprintf("%s:%d", filepath.Base(frame.File), frame.Line)
+	}
+	if output == "json" {
+		return &log.JSONFormatter{
+			CallerPrettyfier: callerPrettyfier,
+			TimestampFormat:  time.RFC3339Nano,
+		}
+	}
+	return &log.TextFormatter{
+		CallerPrettyfier: callerPrettyfier,
+		FullTimestamp:    true,
+		TimestampFormat:  time.RFC3339Nano,
+	}
+}

machinery/logging_test.go

@@ -0,0 +1,165 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func TestConfigureLoggingDebugIncludesStructuredContext(t *testing.T) {
+	logger := log.New()
+	var output bytes.Buffer
+	configureLogger(logger, "debug", "json", time.UTC)
+	logger.SetOutput(&output)
+	logger.WithField("event", "test_event").Debug("structured debug test")
+
+	lines := strings.Split(strings.TrimSpace(output.String()), "\n")
+	var entry map[string]interface{}
+	if err := json.Unmarshal([]byte(lines[len(lines)-1]), &entry); err != nil {
+		t.Fatalf("decode debug log: %v; output=%q", err, output.String())
+	}
+	for key, want := range map[string]interface{}{
+		"component": "agent",
+		"event":     "test_event",
+		"level":     "debug",
+		"msg":       "structured debug test",
+	} {
+		if got := entry[key]; got != want {
+			t.Fatalf("%s = %v, want %v", key, got, want)
+		}
+	}
+	if entry["file"] == nil || entry["func"] == nil {
+		t.Fatalf("debug log is missing caller metadata: %v", entry)
+	}
+}
+
+func TestConfigureLoggerInstallsComponentHookOnce(t *testing.T) {
+	logger := log.New()
+	configureLogger(logger, "info", "text", time.UTC)
+	configureLogger(logger, "debug", "json", time.UTC)
+
+	var componentHooks int
+	for _, hooks := range logger.Hooks {
+		for _, hook := range hooks {
+			if _, ok := hook.(componentHook); ok {
+				componentHooks++
+			}
+		}
+	}
+	if componentHooks != len(log.AllLevels) {
+		t.Fatalf("component hook registrations = %d, want %d", componentHooks, len(log.AllLevels))
+	}
+}
+
+func TestComponentFromCaller(t *testing.T) {
+	tests := []struct {
+		name  string
+		frame *runtime.Frame
+		want  string
+	}{
+		{
+			name:  "nested runtime package",
+			frame: &runtime.Frame{File: "/workspace/agent/machinery/src/routers/mqtt/main.go"},
+			want:  "routers/mqtt",
+		},
+		{
+			name:  "top-level runtime package",
+			frame: &runtime.Frame{File: "/workspace/agent/machinery/src/capture/main.go"},
+			want:  "capture",
+		},
+		{
+			name:  "executable",
+			frame: &runtime.Frame{File: "/workspace/agent/machinery/main.go"},
+			want:  "agent",
+		},
+		{name: "missing caller", want: "unknown"},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if got := componentFromCaller(test.frame); got != test.want {
+				t.Fatalf("componentFromCaller() = %q, want %q", got, test.want)
+			}
+		})
+	}
+}
+
+func TestComponentHookAddsAndPreservesComponent(t *testing.T) {
+	hook := componentHook{}
+
+	entry := log.NewEntry(log.New())
+	entry.Caller = &runtime.Frame{File: "/workspace/agent/machinery/src/cloud/livehls/session.go"}
+	if err := hook.Fire(entry); err != nil {
+		t.Fatalf("componentHook.Fire() error = %v", err)
+	}
+	if got := entry.Data["component"]; got != "cloud/livehls" {
+		t.Fatalf("component = %v, want cloud/livehls", got)
+	}
+
+	entry.Data["component"] = "explicit"
+	if err := hook.Fire(entry); err != nil {
+		t.Fatalf("componentHook.Fire() preserving field error = %v", err)
+	}
+	if got := entry.Data["component"]; got != "explicit" {
+		t.Fatalf("component = %v, want explicit", got)
+	}
+}
+
+func TestParseLogLevel(t *testing.T) {
+	tests := []struct {
+		name    string
+		value   string
+		want    log.Level
+		wantErr bool
+	}{
+		{name: "default", want: log.InfoLevel},
+		{name: "info", value: "INFO", want: log.InfoLevel},
+		{name: "warning alias", value: "warning", want: log.WarnLevel},
+		{name: "debug", value: "debug", want: log.DebugLevel},
+		{name: "trace", value: "trace", want: log.TraceLevel},
+		{name: "invalid", value: "verbose", want: log.InfoLevel, wantErr: true},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got, err := parseLogLevel(test.value)
+			if (err != nil) != test.wantErr {
+				t.Fatalf("parseLogLevel(%q) error = %v, wantErr %t", test.value, err, test.wantErr)
+			}
+			if got != test.want {
+				t.Fatalf("parseLogLevel(%q) = %s, want %s", test.value, got, test.want)
+			}
+		})
+	}
+}
+
+func TestParseLogOutput(t *testing.T) {
+	tests := []struct {
+		name    string
+		value   string
+		want    string
+		wantErr bool
+	}{
+		{name: "default", want: "text"},
+		{name: "text", value: "TEXT", want: "text"},
+		{name: "json", value: "json", want: "json"},
+		{name: "invalid", value: "console", want: "text", wantErr: true},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got, err := parseLogOutput(test.value)
+			if (err != nil) != test.wantErr {
+				t.Fatalf("parseLogOutput(%q) error = %v, wantErr %t", test.value, err, test.wantErr)
+			}
+			if got != test.want {
+				t.Fatalf("parseLogOutput(%q) = %q, want %q", test.value, got, test.want)
+			}
+		})
+	}
+}

machinery/main.go

@@ -5,14 +5,15 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/capture"
 	"github.com/kerberos-io/agent/machinery/src/components"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/onvif"
+	log "github.com/sirupsen/logrus"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
@@ -28,6 +29,22 @@ import (
 
 var VERSION = utils.VERSION
 
+func resolveServerPort(flagValue, environmentValue string) (string, error) {
+	value := strings.TrimSpace(environmentValue)
+	if value == "" {
+		value = strings.TrimSpace(flagValue)
+	}
+	if value == "" {
+		value = "80"
+	}
+
+	port, err := strconv.Atoi(value)
+	if err != nil || port < 1 || port > 65535 {
+		return "", fmt.Errorf("port must be an integer between 1 and 65535, got %q", value)
+	}
+	return strconv.Itoa(port), nil
+}
+
 func startTracing(agentKey string, otelEndpoint string) (*trace.TracerProvider, error) {
 	serviceName := "agent-" + agentKey
 	headers := map[string]string{
@@ -99,20 +116,44 @@ func main() {
 	}
 	// Specify the timezone of the log: "UTC" or "Local".
 	timezone, _ := time.LoadLocation("CET")
-	log.Log.Init(logLevel, logOutput, configDirectory, timezone)
+	configureLogging(logLevel, logOutput, timezone)
+	if action == "run" {
+		resolvedPort, err := resolveServerPort(port, os.Getenv("AGENT_PORT"))
+		if err != nil {
+			log.WithError(err).WithFields(log.Fields{
+				"component": "http",
+				"event":     "server_port_invalid",
+			}).Fatal("Invalid HTTP server port")
+			return
+		}
+		port = resolvedPort
+	}
+	log.WithFields(log.Fields{
+		"action":           action,
+		"component":        "agent",
+		"config_directory": configDirectory,
+		"event":            "command_parsed",
+		"port":             port,
+		"version":          VERSION,
+	}).Debug("Agent command parsed")
 
 	switch action {
 
 	case "version":
 		{
-			log.Log.Info("main.Main(): You are currrently running Kerberos Agent " + VERSION)
+			log.WithFields(log.Fields{
+				"component": "agent",
+				"event":     "version",
+				"version":   VERSION,
+			}).Info("Kerberos Agent version")
 		}
 	case "discover":
 		{
 			// Convert duration to int
 			timeout, err := time.ParseDuration(timeout + "ms")
 			if err != nil {
-				log.Log.Fatal("main.Main(): could not parse timeout: " + err.Error())
+				log.WithError(err).WithField("component", "onvif").
+					Fatal("invalid ONVIF discovery timeout")
 				return
 			}
 			var subnets []string
@@ -125,15 +166,19 @@ func main() {
 		}
 	case "decrypt":
 		{
-			log.Log.Info("main.Main(): Decrypting: " + flag.Arg(0) + " with key: " + flag.Arg(1))
+			log.WithFields(log.Fields{
+				"component": "encryption",
+				"event":     "decrypt_started",
+				"path":      flag.Arg(0),
+			}).Info("Decrypting recording")
 			symmetricKey := []byte(flag.Arg(1))
 
 			if len(symmetricKey) == 0 {
-				log.Log.Fatal("main.Main(): symmetric key should not be empty")
+				log.Fatal("main.Main(): symmetric key should not be empty")
 				return
 			}
 			if len(symmetricKey) != 32 {
-				log.Log.Fatal("main.Main(): symmetric key should be 32 bytes")
+				log.Fatal("main.Main(): symmetric key should be 32 bytes")
 				return
 			}
 
@@ -163,19 +208,28 @@ func main() {
 
 			// Start OpenTelemetry tracing
 			if otelEndpoint := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"); otelEndpoint == "" {
-				log.Log.Info("main.Main(): No OpenTelemetry endpoint provided, skipping tracing")
+				log.WithFields(log.Fields{
+					"component": "tracing",
+					"event":     "tracing_disabled",
+				}).Debug("OpenTelemetry tracing disabled")
 			} else {
-				log.Log.Info("main.Main(): Starting OpenTelemetry tracing with endpoint: " + otelEndpoint)
+				log.WithFields(log.Fields{
+					"component": "tracing",
+					"event":     "tracing_starting",
+				}).Info("Starting OpenTelemetry tracing")
 				agentKey := configuration.Config.Key
 				traceProvider, err := startTracing(agentKey, otelEndpoint)
 				if err != nil {
-					log.Log.Error("traceprovider: " + err.Error())
+					log.WithError(err).WithField("component", "tracing").
+						Error("Failed to start OpenTelemetry tracing")
+				} else {
+					defer func() {
+						if err := traceProvider.Shutdown(context.Background()); err != nil {
+							log.WithError(err).WithField("component", "tracing").
+								Error("Failed to shut down OpenTelemetry tracing")
+						}
+					}()
 				}
-				defer func() {
-					if err := traceProvider.Shutdown(context.Background()); err != nil {
-						log.Log.Error("traceprovider: " + err.Error())
-					}
-				}()
 			}
 
 			// Printing final configuration
@@ -186,8 +240,13 @@ func main() {
 			utils.CheckDataDirectoryPermissions(configDirectory)
 
 			// Set timezone
-			timezone, _ := time.LoadLocation(configuration.Config.Timezone)
+			timezone, err := time.LoadLocation(configuration.Config.Timezone)
-			log.Log.Init(logLevel, logOutput, configDirectory, timezone)
+			if err != nil {
+				log.WithError(err).WithField("timezone", configuration.Config.Timezone).
+					Warn("invalid Agent timezone; using the host timezone for logs")
+				timezone = time.Local
+			}
+			configureLogging(logLevel, logOutput, timezone)
 
 			// Check if we have a device Key or not, if not
 			// we will generate one.
@@ -196,30 +255,37 @@ func main() {
 				configuration.Config.Key = key
 				err := configService.StoreConfig(configDirectory, configuration.Config)
 				if err == nil {
-					log.Log.Info("main.Main(): updated unique key for agent to: " + key)
+					log.WithFields(log.Fields{
+						"component": "configuration",
+						"event":     "agent_key_generated",
+					}).Info("Generated and stored a unique Agent key")
 				} else {
-					log.Log.Info("main.Main(): something went wrong while trying to store key: " + key)
+					log.WithError(err).WithFields(log.Fields{
+						"component": "configuration",
+						"event":     "agent_key_store_failed",
+					}).Error("Failed to store the generated Agent key")
 				}
 			}
 
 			// Create a cancelable context, which will be used to cancel and restart.
 			// This is used to restart the agent when the configuration is updated.
 			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
 
 			// We create a capture object, this will contain all the streaming clients.
 			// And allow us to extract media from within difference places in the agent.
-			capture := capture.Capture{
+			capture := capture.Capture{}
-				RTSPClient:    nil,
-				RTSPSubClient: nil,
-			}
 
 			// Bootstrapping the agent
 			communication := models.Communication{
-				Context:         &ctx,
-				CancelContext:   &cancel,
 				HandleBootstrap: make(chan string, 1),
 			}
 
+			log.WithFields(log.Fields{
+				"component": "agent",
+				"event":     "runtime_starting",
+				"port":      configuration.Port,
+			}).Info("Starting Agent runtime")
 			go components.Bootstrap(ctx, configDirectory, &configuration, &communication, &capture)
 
 			// Start the REST API.
@@ -227,7 +293,7 @@ func main() {
 		}
 	default:
 		{
-			log.Log.Error("main.Main(): Sorry I don't understand :(")
+			log.Error("main.Main(): Sorry I don't understand :(")
 		}
 	}
 }

machinery/main_test.go

@@ -0,0 +1,33 @@
+package main
+
+import "testing"
+
+func TestResolveServerPort(t *testing.T) {
+	tests := []struct {
+		name      string
+		flagValue string
+		envValue  string
+		want      string
+		wantError bool
+	}{
+		{name: "flag default", flagValue: "80", want: "80"},
+		{name: "environment overrides flag", flagValue: "80", envValue: "8082", want: "8082"},
+		{name: "trims environment", flagValue: "80", envValue: " 9090 ", want: "9090"},
+		{name: "empty values use default", want: "80"},
+		{name: "invalid text", flagValue: "80", envValue: "http", wantError: true},
+		{name: "zero", flagValue: "80", envValue: "0", wantError: true},
+		{name: "above maximum", flagValue: "80", envValue: "65536", wantError: true},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got, err := resolveServerPort(test.flagValue, test.envValue)
+			if (err != nil) != test.wantError {
+				t.Fatalf("resolveServerPort(%q, %q) error = %v, wantError %t", test.flagValue, test.envValue, err, test.wantError)
+			}
+			if got != test.want {
+				t.Fatalf("resolveServerPort(%q, %q) = %q, want %q", test.flagValue, test.envValue, got, test.want)
+			}
+		})
+	}
+}

machinery/src/capture/gortsplib.go

@@ -13,9 +13,12 @@ import (
 	"errors"
 	"fmt"
 	"image"
+	"math"
+	"net/url"
 	"os"
 	"reflect"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 	"unsafe"
@@ -32,10 +35,10 @@ import (
 	"github.com/bluenviron/mediacommon/pkg/codecs/h264"
 	"github.com/bluenviron/mediacommon/pkg/codecs/h265"
 	"github.com/bluenviron/mediacommon/pkg/codecs/mpeg4audio"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/pion/rtp"
+	log "github.com/sirupsen/logrus"
 	"go.opentelemetry.io/otel"
 )
 
@@ -71,12 +74,31 @@ func rtspsTLSConfig() (*tls.Config, error) {
 	return &tls.Config{RootCAs: rootCAs}, nil
 }
 
+func sanitizeRTSPError(err error, rawURL string) error {
+	if err == nil {
+		return nil
+	}
+	if rawURL == "" || !strings.Contains(err.Error(), rawURL) {
+		return err
+	}
+
+	replacement := "<redacted-rtsp-url>"
+	if parsed, parseErr := url.Parse(rawURL); parseErr == nil {
+		parsed.User = nil
+		parsed.RawQuery = ""
+		parsed.Fragment = ""
+		replacement = parsed.String()
+	}
+	return errors.New(strings.ReplaceAll(err.Error(), rawURL, replacement))
+}
+
 // Implements the RTSPClient interface.
 type Golibrtsp struct {
 	RTSPClient
 	Url string
 
 	Client            gortsplib.Client
+	clientStarted     bool
 	VideoDecoderMutex *sync.Mutex
 
 	VideoH264Index        int8
@@ -197,14 +219,20 @@ func (h *streamHealth) observePacket(streamType string, writeDur time.Duration)
 		h.gapMax = gap
 	}
 	if writeDur >= streamHealthWriteWarn {
-		log.Log.Warning(fmt.Sprintf(
+		log.WithFields(log.Fields{
-			"capture.golibrtsp.health(%s): WritePacket blocked %dms — downstream back-pressure / CPU starvation",
+			"component":   "capture",
-			streamType, writeDur.Milliseconds()))
+			"duration_ms": writeDur.Milliseconds(),
+			"event":       "packet_queue_write_slow",
+			"stream":      streamType,
+		}).Warn("Packet queue write was slow")
 	}
 	if gap >= streamHealthGapWarn {
-		log.Log.Warning(fmt.Sprintf(
+		log.WithFields(log.Fields{
-			"capture.golibrtsp.health(%s): %dms since previous frame — upstream network / camera stall",
+			"component":    "capture",
-			streamType, gap.Milliseconds()))
+			"event":        "frame_gap_detected",
+			"frame_gap_ms": gap.Milliseconds(),
+			"stream":       streamType,
+		}).Warn("Large gap detected between camera frames")
 	}
 	if now.Sub(h.windowStart) >= streamHealthWindow {
 		elapsed := now.Sub(h.windowStart).Seconds()
@@ -212,10 +240,20 @@ func (h *streamHealth) observePacket(streamType string, writeDur time.Duration)
 		if h.frames > 0 {
 			avgWriteMs = float64(h.writeSum.Milliseconds()) / float64(h.frames)
 		}
-		log.Log.Info(fmt.Sprintf(
+		log.WithFields(log.Fields{
-			"capture.golibrtsp.health(%s): %.0fs window — frames=%d (%.1f/s) writeAvg=%.1fms writeMax=%dms gapMax=%dms lost=%d decodeErrs=%d",
+			"average_write_ms": avgWriteMs,
-			streamType, elapsed, h.frames, float64(h.frames)/elapsed, avgWriteMs,
+			"component":        "capture",
-			h.writeMax.Milliseconds(), h.gapMax.Milliseconds(), h.lost, h.decodeErrs))
+			"decode_errors":    h.decodeErrs,
+			"event":            "stream_health_sampled",
+			"frame_rate":       float64(h.frames) / elapsed,
+			"frames":           h.frames,
+			"lost_packets":     h.lost,
+			"max_frame_gap_ms": h.gapMax.Milliseconds(),
+			"max_write_ms":     h.writeMax.Milliseconds(),
+			"stream":           streamType,
+			"window_seconds":   elapsed,
+		}).Debug("RTSP stream health sampled")
+
 		h.windowStart = now
 		h.frames = 0
 		h.writeSum = 0
@@ -236,9 +274,12 @@ func (h *streamHealth) observeLost(streamType string, lost uint64) {
 	h.mu.Lock()
 	h.lost += lost
 	h.mu.Unlock()
-	log.Log.Warning(fmt.Sprintf(
+	log.WithFields(log.Fields{
-		"capture.golibrtsp.health(%s): %d RTP packet(s) lost — sender-side gap (receiver not draining TCP fast enough)",
+		"component":    "capture",
-		streamType, lost))
+		"event":        "rtp_packets_lost",
+		"lost_packets": lost,
+		"stream":       streamType,
+	}).Warn("RTSP sender reported lost RTP packets")
 }
 
 // observeDecodeError is invoked by gortsplib on incomplete/invalid access units,
@@ -250,7 +291,11 @@ func (h *streamHealth) observeDecodeError(streamType string, err error) {
 	h.mu.Lock()
 	h.decodeErrs++
 	h.mu.Unlock()
-	log.Log.Debug(fmt.Sprintf("capture.golibrtsp.health(%s): decode error: %s", streamType, err.Error()))
+	log.WithError(err).WithFields(log.Fields{
+		"component": "capture",
+		"event":     "access_unit_decode_failed",
+		"stream":    streamType,
+	}).Debug("Failed to decode RTSP access unit")
 }
 
 // fpsTracker holds per-stream state for PTS-based FPS calculation.
@@ -334,13 +379,13 @@ func init() {
 	// setup H264 -> raw frames decoder
 	H264FrameDecoder, err = newDecoder("H264")
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.init(): " + err.Error())
+		log.Error("capture.golibrtsp.init(): " + err.Error())
 	}
 
 	// setup H265 -> raw frames decoder
 	H265FrameDecoder, err = newDecoder("H265")
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.init(): " + err.Error())
+		log.Error("capture.golibrtsp.init(): " + err.Error())
 	}
 }
 
@@ -376,8 +421,7 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 	// parse URL
 	u, err := base.ParseURL(g.Url)
 	if err != nil {
-		log.Log.Debug("capture.golibrtsp.Connect(ParseURL): " + err.Error())
+		return sanitizeRTSPError(err, g.Url)
-		return
 	}
 
 	// connect to the server
@@ -385,14 +429,14 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 	g.Client.Host = u.Host
 	err = g.Client.Start()
 	if err != nil {
-		log.Log.Debug("capture.golibrtsp.Connect(Start): " + err.Error())
+		return sanitizeRTSPError(err, g.Url)
 	}
+	g.clientStarted = true
 
 	// find published medias
 	desc, _, err := g.Client.Describe(u)
 	if err != nil {
-		log.Log.Debug("capture.golibrtsp.Connect(Describe): " + err.Error())
+		return sanitizeRTSPError(err, g.Url)
-		return
 	}
 
 	// Initialize the mutex and FPS calculation.
@@ -405,13 +449,13 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 	g.VideoH264Media = mediH264
 	g.VideoH264Forma = formaH264
 	if mediH264 == nil {
-		log.Log.Debug("capture.golibrtsp.Connect(H264): " + "video media not found")
+		log.Debug("capture.golibrtsp.Connect(H264): " + "video media not found")
 	} else {
 		// setup a video media
 		_, err = g.Client.Setup(desc.BaseURL, mediH264, 0, 0)
 		if err != nil {
 			// Something went wrong .. Do something
-			log.Log.Error("capture.golibrtsp.Connect(H264): " + err.Error())
+			log.Error("capture.golibrtsp.Connect(H264): " + err.Error())
 		} else {
 			// Get SPS and PPS from the SDP
 			// Calculate the width and height of the video
@@ -420,7 +464,7 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 			// It might be that the SPS is not available yet, so we'll proceed,
 			// but try to fetch it later on.
 			if errSPS != nil {
-				log.Log.Debug("capture.golibrtsp.Connect(H264): " + errSPS.Error())
+				log.Debug("capture.golibrtsp.Connect(H264): " + errSPS.Error())
 				streamIndex := len(g.Streams)
 				g.Streams = append(g.Streams, packets.Stream{
 					Index:         streamIndex,
@@ -456,7 +500,7 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 			// setup RTP/H264 -> H264 decoder
 			rtpDec, err := formaH264.CreateDecoder()
 			if err != nil {
-				log.Log.Error("capture.golibrtsp.Connect(H264): " + err.Error())
+				log.Error("capture.golibrtsp.Connect(H264): " + err.Error())
 			}
 			g.VideoH264Decoder = rtpDec
 			g.VideoH264FrameDecoder = H264FrameDecoder
@@ -469,20 +513,20 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 	g.VideoH265Media = mediH265
 	g.VideoH265Forma = formaH265
 	if mediH265 == nil {
-		log.Log.Debug("capture.golibrtsp.Connect(H265): " + "video media not found")
+		log.Debug("capture.golibrtsp.Connect(H265): " + "video media not found")
 	} else {
 		// setup a video media
 		_, err = g.Client.Setup(desc.BaseURL, mediH265, 0, 0)
 		if err != nil {
 			// Something went wrong .. Do something
-			log.Log.Error("capture.golibrtsp.Connect(H265): " + err.Error())
+			log.Error("capture.golibrtsp.Connect(H265): " + err.Error())
 		} else {
 			// Get SPS from the SDP
 			// Calculate the width and height of the video
 			var sps h265.SPS
 			err = sps.Unmarshal(formaH265.SPS)
 			if err != nil {
-				log.Log.Info("capture.golibrtsp.Connect(H265): " + err.Error())
+				log.Info("capture.golibrtsp.Connect(H265): " + err.Error())
 				return
 			}
 			streamIndex := len(g.Streams)
@@ -506,7 +550,7 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 			// setup RTP/H265 -> H265 decoder
 			rtpDec, err := formaH265.CreateDecoder()
 			if err != nil {
-				log.Log.Error("capture.golibrtsp.Connect(H265): " + err.Error())
+				log.Error("capture.golibrtsp.Connect(H265): " + err.Error())
 			}
 			g.VideoH265Decoder = rtpDec
 
@@ -520,19 +564,19 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 	g.AudioG711Media = audioMedi
 	g.AudioG711Forma = audioForma
 	if audioMedi == nil {
-		log.Log.Debug("capture.golibrtsp.Connect(G711): " + "audio media not found")
+		log.Debug("capture.golibrtsp.Connect(G711): " + "audio media not found")
 	} else {
 		// setup a audio media
 		_, err = g.Client.Setup(desc.BaseURL, audioMedi, 0, 0)
 		if err != nil {
 			// Something went wrong .. Do something
-			log.Log.Error("capture.golibrtsp.Connect(G711): " + err.Error())
+			log.Error("capture.golibrtsp.Connect(G711): " + err.Error())
 		} else {
 			// create decoder
 			audiortpDec, err := audioForma.CreateDecoder()
 			if err != nil {
 				// Something went wrong .. Do something
-				log.Log.Error("capture.golibrtsp.Connect(G711): " + err.Error())
+				log.Error("capture.golibrtsp.Connect(G711): " + err.Error())
 			} else {
 				g.AudioG711Decoder = audiortpDec
 				streamIndex := len(g.Streams)
@@ -556,19 +600,19 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 	g.AudioOpusMedia = audioMediOpus
 	g.AudioOpusForma = audioFormaOpus
 	if audioMediOpus == nil {
-		log.Log.Debug("capture.golibrtsp.Connect(Opus): " + "audio media not found")
+		log.Debug("capture.golibrtsp.Connect(Opus): " + "audio media not found")
 	} else {
 		// setup a audio media
 		_, err = g.Client.Setup(desc.BaseURL, audioMediOpus, 0, 0)
 		if err != nil {
 			// Something went wrong .. Do something
-			log.Log.Error("capture.golibrtsp.Connect(Opus): " + err.Error())
+			log.Error("capture.golibrtsp.Connect(Opus): " + err.Error())
 		} else {
 			// create decoder
 			audiortpDec, err := audioFormaOpus.CreateDecoder()
 			if err != nil {
 				// Something went wrong .. Do something
-				log.Log.Error("capture.golibrtsp.Connect(Opus): " + err.Error())
+				log.Error("capture.golibrtsp.Connect(Opus): " + err.Error())
 			} else {
 				g.AudioOpusDecoder = audiortpDec
 				streamIndex := len(g.Streams)
@@ -592,13 +636,13 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 	g.AudioMPEG4Media = audioMediMPEG4
 	g.AudioMPEG4Forma = audioFormaMPEG4
 	if audioMediMPEG4 == nil {
-		log.Log.Debug("capture.golibrtsp.Connect(MPEG4): " + "audio media not found")
+		log.Debug("capture.golibrtsp.Connect(MPEG4): " + "audio media not found")
 	} else {
 		// setup a audio media
 		_, err = g.Client.Setup(desc.BaseURL, audioMediMPEG4, 0, 0)
 		if err != nil {
 			// Something went wrong .. Do something
-			log.Log.Error("capture.golibrtsp.Connect(MPEG4): " + err.Error())
+			log.Error("capture.golibrtsp.Connect(MPEG4): " + err.Error())
 		} else {
 			streamIndex := len(g.Streams)
 			g.Streams = append(g.Streams, packets.Stream{
@@ -618,7 +662,7 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 			audiortpDec, err := audioFormaMPEG4.CreateDecoder()
 			if err != nil {
 				// Something went wrong .. Do something
-				log.Log.Error("capture.golibrtsp.Connect(MPEG4): " + err.Error())
+				log.Error("capture.golibrtsp.Connect(MPEG4): " + err.Error())
 			}
 			g.AudioMPEG4Decoder = audiortpDec
 
@@ -648,8 +692,7 @@ func (g *Golibrtsp) ConnectBackChannel(ctx context.Context, ctxRunAgent context.
 	// parse URL
 	u, err := base.ParseURL(g.Url)
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.ConnectBackChannel(): " + err.Error())
+		return sanitizeRTSPError(err, g.Url)
-		return
 	}
 
 	// connect to the server
@@ -657,14 +700,14 @@ func (g *Golibrtsp) ConnectBackChannel(ctx context.Context, ctxRunAgent context.
 	g.Client.Host = u.Host
 	err = g.Client.Start()
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.ConnectBackChannel(): " + err.Error())
+		return sanitizeRTSPError(err, g.Url)
 	}
+	g.clientStarted = true
 
 	// find published medias
 	desc, _, err := g.Client.Describe(u)
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.ConnectBackChannel(): " + err.Error())
+		return sanitizeRTSPError(err, g.Url)
-		return
 	}
 
 	// Look for audio back channel.
@@ -674,15 +717,17 @@ func (g *Golibrtsp) ConnectBackChannel(ctx context.Context, ctxRunAgent context.
 	g.AudioG711MediaBackChannel = audioMediBackChannel
 	g.AudioG711FormaBackChannel = audioFormaBackChannel
 	if audioMediBackChannel == nil {
-		log.Log.Error("capture.golibrtsp.ConnectBackChannel(): audio backchannel not found, not a real error, however you might expect a backchannel. One of the reasons might be that the device already has an active client connected to the backchannel.")
+		log.WithFields(log.Fields{
+			"component": "capture",
+			"event":     "backchannel_unavailable",
+		}).Debug("Optional camera audio backchannel unavailable")
 		err = errors.New("no audio backchannel found")
 	} else {
 		// setup a audio media
 		_, err = g.Client.Setup(desc.BaseURL, audioMediBackChannel, 0, 0)
 		if err != nil {
-			// Something went wrong .. Do something
-			log.Log.Error("capture.golibrtsp.ConnectBackChannel(): " + err.Error())
 			g.HasBackChannel = false
+			return sanitizeRTSPError(err, g.Url)
 		} else {
 			g.HasBackChannel = true
 			streamIndex := len(g.Streams)
@@ -735,9 +780,26 @@ func ptsToDuration(pts int64, clockRate int) time.Duration {
 		time.Duration(pts%rate)*time.Second/time.Duration(rate)
 }
 
+func preRecordingGOPCount(preRecording int64, gopDuration float64) (int, bool) {
+	if preRecording <= 0 ||
+		gopDuration < 1 ||
+		math.IsNaN(gopDuration) ||
+		math.IsInf(gopDuration, 0) ||
+		gopDuration >= float64(math.MaxInt64) {
+		return 0, false
+	}
+
+	count := preRecording / int64(gopDuration)
+	maxInt := int64(^uint(0) >> 1)
+	if count >= maxInt {
+		return 0, false
+	}
+	return int(count) + 1, true
+}
+
 // Start the RTSP client, and start reading packets.
 func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets.Queue, configuration *models.Configuration, communication *models.Communication) (err error) {
-	log.Log.Debug("capture.golibrtsp.Start(): started")
+	log.Debug("capture.golibrtsp.Start(): started")
 
 	// Label this client's loss/decode/health logging with the stream type.
 	g.streamLabel = streamType
@@ -751,7 +813,7 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 			// decode timestamp
 			pts2, ok := g.Client.PacketPTS(g.AudioG711Media, rtppkt)
 			if !ok {
-				log.Log.Debug("capture.golibrtsp.Start(): " + "unable to get PTS")
+				log.Debug("capture.golibrtsp.Start(): " + "unable to get PTS")
 				return
 			}
 			pts := ptsToDuration(pts2, g.AudioG711Forma.ClockRate())
@@ -759,7 +821,7 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 			// extract LPCM samples from RTP packets
 			op, err := g.AudioG711Decoder.Decode(rtppkt)
 			if err != nil {
-				log.Log.Error("capture.golibrtsp.Start(): " + err.Error())
+				log.Error("capture.golibrtsp.Start(): " + err.Error())
 				return
 			}
 
@@ -786,7 +848,7 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 			// decode timestamp
 			pts2, ok := g.Client.PacketPTS(g.AudioMPEG4Media, rtppkt)
 			if !ok {
-				log.Log.Error("capture.golibrtsp.Start(): " + "unable to get PTS")
+				log.Error("capture.golibrtsp.Start(): " + "unable to get PTS")
 				return
 			}
 			pts := ptsToDuration(pts2, g.AudioMPEG4Forma.ClockRate())
@@ -795,13 +857,13 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 			// extract access units from RTP packets
 			aus, err := g.AudioMPEG4Decoder.Decode(rtppkt)
 			if err != nil {
-				log.Log.Error("capture.golibrtsp.Start(): " + err.Error())
+				log.Error("capture.golibrtsp.Start(): " + err.Error())
 				return
 			}
 
 			enc, err := WriteMPEG4Audio(g.AudioMPEG4Forma, aus)
 			if err != nil {
-				log.Log.Error("capture.golibrtsp.Start(): " + err.Error())
+				log.Error("capture.golibrtsp.Start(): " + err.Error())
 				return
 			}
 
@@ -845,7 +907,7 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 				// decode timestamps — validate each call separately
 				pts2, okPTS2 := g.Client.PacketPTS(g.VideoH264Media, rtppkt)
 				if !okPTS2 {
-					log.Log.Debug("capture.golibrtsp.Start(): unable to get PTS")
+					log.Debug("capture.golibrtsp.Start(): unable to get PTS")
 					return
 				}
 				pts := ptsToDuration(pts2, g.VideoH264Forma.ClockRate())
@@ -856,7 +918,7 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 				au, errDecode := g.VideoH264Decoder.Decode(rtppkt)
 				if errDecode != nil {
 					if errDecode != rtph264.ErrNonStartingPacketAndNoPrevious && errDecode != rtph264.ErrMorePacketsNeeded {
-						log.Log.Error("capture.golibrtsp.Start(): " + errDecode.Error())
+						log.Error("capture.golibrtsp.Start(): " + errDecode.Error())
 					}
 					return
 				}
@@ -918,7 +980,12 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 							fps := g.getEnhancedFPS(&sps, g.VideoH264Index)
 							g.Streams[g.VideoH264Index].FPS = fps
 							g.persistStreamFPS(configuration, streamType, fps)
-							log.Log.Debug(fmt.Sprintf("capture.golibrtsp.Start(%s): Final FPS=%.2f", streamType, fps))
+							log.WithFields(log.Fields{
+								"component": "capture",
+								"event":     "frame_rate_detected",
+								"fps":       fps,
+								"stream":    streamType,
+							}).Debug("RTSP frame rate detected")
 							g.VideoH264Forma.SPS = nalu
 							if streamType == "main" && len(nalu) > 0 {
 								// Fallback: store SPS from in-band NALUs when SDP was missing it.
@@ -940,14 +1007,14 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 					// Ensure config has parameter sets before recordings start.
 					if len(configuration.Config.Capture.IPCamera.SPSNALUs) == 0 && len(g.VideoH264Forma.SPS) > 0 {
 						configuration.Config.Capture.IPCamera.SPSNALUs = [][]byte{g.VideoH264Forma.SPS}
-						log.Log.Warning("capture.golibrtsp.Start(main): fallback SPS set from keyframe")
+						log.Warn("capture.golibrtsp.Start(main): fallback SPS set from keyframe")
 					}
 					if len(configuration.Config.Capture.IPCamera.PPSNALUs) == 0 && len(g.VideoH264Forma.PPS) > 0 {
 						configuration.Config.Capture.IPCamera.PPSNALUs = [][]byte{g.VideoH264Forma.PPS}
-						log.Log.Warning("capture.golibrtsp.Start(main): fallback PPS set from keyframe")
+						log.Warn("capture.golibrtsp.Start(main): fallback PPS set from keyframe")
 					}
 					if len(configuration.Config.Capture.IPCamera.SPSNALUs) == 0 || len(configuration.Config.Capture.IPCamera.PPSNALUs) == 0 {
-						log.Log.Warning("capture.golibrtsp.Start(main): SPS/PPS still missing after IDR keyframe")
+						log.Warn("capture.golibrtsp.Start(main): SPS/PPS still missing after IDR keyframe")
 					}
 				}
 
@@ -956,14 +1023,18 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 				}
 
 				if idrPresent {
-					log.Log.Debug(fmt.Sprintf("capture.golibrtsp.Start(%s): IDR frame NALUs: [%s]",
+					log.WithFields(log.Fields{
-						streamType, fmt.Sprintf("%v", naluTypes)))
+						"component":  "capture",
+						"event":      "idr_frame_received",
+						"nalu_types": naluTypes,
+						"stream":     streamType,
+					}).Debug("RTSP IDR frame received")
 				}
 
 				// Convert to packet.
 				enc, err := h264.AnnexBMarshal(filteredAU)
 				if err != nil {
-					log.Log.Error("capture.golibrtsp.Start(): " + err.Error())
+					log.Error("capture.golibrtsp.Start(): " + err.Error())
 					return
 				}
 
@@ -997,11 +1068,19 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 					gopDuration := float64(keyframeInterval) / fps
 					gopSize := int(avgInterval) // Store GOP size in a separate variable
 					g.Streams[g.VideoH264Index].GopSize = gopSize
-					log.Log.Debug(fmt.Sprintf("capture.golibrtsp.Start(%s): Keyframe interval=%d packets, Avg=%.1f, GOP=%.1fs, GOPSize=%d",
+					log.WithFields(log.Fields{
-						streamType, keyframeInterval, avgInterval, gopDuration, gopSize))
+						"average_keyframe_interval_packets": avgInterval,
+						"component":                         "capture",
+						"event":                             "keyframe_interval_observed",
+						"gop_duration_seconds":              gopDuration,
+						"gop_size_packets":                  gopSize,
+						"keyframe_interval_packets":         keyframeInterval,
+						"stream":                            streamType,
+					}).Debug("RTSP keyframe interval observed")
+
 					preRecording := configuration.Config.Capture.PreRecording
-					if preRecording > 0 && int(gopDuration) > 0 {
+					if maxGOPCount, ok := preRecordingGOPCount(preRecording, gopDuration); ok {
-						queue.SetMaxGopCount(int(preRecording)/int(gopDuration) + 1)
+						queue.SetMaxGopCount(maxGOPCount)
 					}
 				}
 
@@ -1048,20 +1127,36 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 				default:
 				}
 
-				if idrPresent {
+				// Count every complete video access unit. Keyframe-only counters make
-					// Increment packets, so we know the device
+				// healthy cameras with GOPs longer than the watchdog window look stalled.
-					// is not blocking.
+				if streamType == "main" {
-					if streamType == "main" {
+					observedAt := time.Now()
-						r := communication.PackageCounter.Load().(int64)
+					r := communication.PackageCounter.Load().(int64)
-						log.Log.Debug("capture.golibrtsp.Start(): packet size " + strconv.Itoa(len(pkt.Data)))
+					log.WithFields(log.Fields{
-						communication.PackageCounter.Store((r + 1) % 1000)
+						"bytes":     len(pkt.Data),
-						communication.LastPacketTimer.Store(time.Now().Unix())
+						"codec":     "H264",
-					} else if streamType == "sub" {
+						"component": "capture",
-						r := communication.PackageCounterSub.Load().(int64)
+						"event":     "access_unit_received",
-						log.Log.Debug("capture.golibrtsp.Start(): packet size " + strconv.Itoa(len(pkt.Data)))
+						"keyframe":  pkt.IsKeyFrame,
-						communication.PackageCounterSub.Store((r + 1) % 1000)
+						"stream":    streamType,
-						communication.LastPacketTimerSub.Store(time.Now().Unix())
+					}).Trace("RTSP access unit received")
-					}
+					communication.RecordStreamPackage(models.MainStream, g.Streams[g.VideoH264Index].FPS, g.Streams[g.VideoH264Index].Width, g.Streams[g.VideoH264Index].Height, observedAt)
+					communication.PackageCounter.Store((r + 1) % 1000)
+					communication.LastPacketTimer.Store(observedAt.Unix())
+				} else if streamType == "sub" {
+					observedAt := time.Now()
+					r := communication.PackageCounterSub.Load().(int64)
+					log.WithFields(log.Fields{
+						"bytes":     len(pkt.Data),
+						"codec":     "H264",
+						"component": "capture",
+						"event":     "access_unit_received",
+						"keyframe":  pkt.IsKeyFrame,
+						"stream":    streamType,
+					}).Trace("RTSP access unit received")
+					communication.RecordStreamPackage(models.SubStream, g.Streams[g.VideoH264Index].FPS, g.Streams[g.VideoH264Index].Width, g.Streams[g.VideoH264Index].Height, observedAt)
+					communication.PackageCounterSub.Store((r + 1) % 1000)
+					communication.LastPacketTimerSub.Store(observedAt.Unix())
 				}
 			}
 
@@ -1090,7 +1185,7 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 				// decode timestamps — validate each call separately
 				pts2, okPTS2 := g.Client.PacketPTS(g.VideoH265Media, rtppkt)
 				if !okPTS2 {
-					log.Log.Debug("capture.golibrtsp.Start(): unable to get PTS")
+					log.Debug("capture.golibrtsp.Start(): unable to get PTS")
 					return
 				}
 				pts := ptsToDuration(pts2, g.VideoH265Forma.ClockRate())
@@ -1101,7 +1196,7 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 				au, errDecode := g.VideoH265Decoder.Decode(rtppkt)
 				if errDecode != nil {
 					if errDecode != rtph265.ErrNonStartingPacketAndNoPrevious && errDecode != rtph265.ErrMorePacketsNeeded {
-						log.Log.Error("capture.golibrtsp.Start(): " + errDecode.Error())
+						log.Error("capture.golibrtsp.Start(): " + errDecode.Error())
 					}
 					return
 				}
@@ -1159,7 +1254,7 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 
 				enc, err := h264.AnnexBMarshal(au)
 				if err != nil {
-					log.Log.Error("capture.golibrtsp.Start(): " + err.Error())
+					log.Error("capture.golibrtsp.Start(): " + err.Error())
 					return
 				}
 
@@ -1191,11 +1286,19 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 					gopDuration := float64(keyframeInterval) / fps
 					gopSize := int(avgInterval) // Store GOP size in a separate variable
 					g.Streams[g.VideoH265Index].GopSize = gopSize
-					log.Log.Debug(fmt.Sprintf("capture.golibrtsp.Start(%s): Keyframe interval=%d packets, Avg=%.1f, GOP=%.1fs, GOPSize=%d",
+					log.WithFields(log.Fields{
-						streamType, keyframeInterval, avgInterval, gopDuration, gopSize))
+						"average_keyframe_interval_packets": avgInterval,
+						"component":                         "capture",
+						"event":                             "keyframe_interval_observed",
+						"gop_duration_seconds":              gopDuration,
+						"gop_size_packets":                  gopSize,
+						"keyframe_interval_packets":         keyframeInterval,
+						"stream":                            streamType,
+					}).Debug("RTSP keyframe interval observed")
+
 					preRecording := configuration.Config.Capture.PreRecording
-					if preRecording > 0 && int(gopDuration) > 0 {
+					if maxGOPCount, ok := preRecordingGOPCount(preRecording, gopDuration); ok {
-						queue.SetMaxGopCount(int(preRecording)/int(gopDuration) + 1)
+						queue.SetMaxGopCount(maxGOPCount)
 					}
 				}
 
@@ -1213,20 +1316,36 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 				default:
 				}
 
-				if isRandomAccess {
+				// Count every complete video access unit; random-access frames remain
-					// Increment packets, so we know the device
+				// responsible only for GOP tracking above.
-					// is not blocking.
+				if streamType == "main" {
-					if streamType == "main" {
+					observedAt := time.Now()
-						r := communication.PackageCounter.Load().(int64)
+					r := communication.PackageCounter.Load().(int64)
-						log.Log.Debug("capture.golibrtsp.Start(): packet size " + strconv.Itoa(len(pkt.Data)))
+					log.WithFields(log.Fields{
-						communication.PackageCounter.Store((r + 1) % 1000)
+						"bytes":     len(pkt.Data),
-						communication.LastPacketTimer.Store(time.Now().Unix())
+						"codec":     "H265",
-					} else if streamType == "sub" {
+						"component": "capture",
-						r := communication.PackageCounterSub.Load().(int64)
+						"event":     "access_unit_received",
-						log.Log.Debug("capture.golibrtsp.Start(): packet size " + strconv.Itoa(len(pkt.Data)))
+						"keyframe":  pkt.IsKeyFrame,
-						communication.PackageCounterSub.Store((r + 1) % 1000)
+						"stream":    streamType,
-						communication.LastPacketTimerSub.Store(time.Now().Unix())
+					}).Trace("RTSP access unit received")
-					}
+					communication.RecordStreamPackage(models.MainStream, g.Streams[g.VideoH265Index].FPS, g.Streams[g.VideoH265Index].Width, g.Streams[g.VideoH265Index].Height, observedAt)
+					communication.PackageCounter.Store((r + 1) % 1000)
+					communication.LastPacketTimer.Store(observedAt.Unix())
+				} else if streamType == "sub" {
+					observedAt := time.Now()
+					r := communication.PackageCounterSub.Load().(int64)
+					log.WithFields(log.Fields{
+						"bytes":     len(pkt.Data),
+						"codec":     "H265",
+						"component": "capture",
+						"event":     "access_unit_received",
+						"keyframe":  pkt.IsKeyFrame,
+						"stream":    streamType,
+					}).Trace("RTSP access unit received")
+					communication.RecordStreamPackage(models.SubStream, g.Streams[g.VideoH265Index].FPS, g.Streams[g.VideoH265Index].Width, g.Streams[g.VideoH265Index].Height, observedAt)
+					communication.PackageCounterSub.Store((r + 1) % 1000)
+					communication.LastPacketTimerSub.Store(observedAt.Unix())
 				}
 			}
 
@@ -1238,7 +1357,7 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 	// Play the stream.
 	_, err = g.Client.Play(nil)
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.Start(): " + err.Error())
+		log.Error("capture.golibrtsp.Start(): " + err.Error())
 	}
 
 	return
@@ -1246,13 +1365,13 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 
 // Start the RTSP client, and start reading packets.
 func (g *Golibrtsp) StartBackChannel(ctx context.Context, ctxRunAgent context.Context) (err error) {
-	log.Log.Info("capture.golibrtsp.StartBackChannel(): started")
+	log.Info("capture.golibrtsp.StartBackChannel(): started")
 	// Wait for a second, so we can be sure the stream is playing.
 	time.Sleep(1 * time.Second)
 	// Play the stream.
 	_, err = g.Client.Play(nil)
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.StartBackChannel(): " + err.Error())
+		log.Error("capture.golibrtsp.StartBackChannel(): " + err.Error())
 	}
 	return
 }
@@ -1261,7 +1380,7 @@ func (g *Golibrtsp) WritePacket(pkt packets.Packet) error {
 	if g.HasBackChannel && g.AudioG711MediaBackChannel != nil {
 		err := g.Client.WritePacketRTP(g.AudioG711MediaBackChannel, pkt.Packet)
 		if err != nil {
-			log.Log.Debug("capture.golibrtsp.WritePacket(): " + err.Error())
+			log.Debug("capture.golibrtsp.WritePacket(): " + err.Error())
 			return err
 		}
 	}
@@ -1284,11 +1403,11 @@ func (g *Golibrtsp) DecodePacket(pkt packets.Packet) (image.YCbCr, error) {
 	}
 	g.VideoDecoderMutex.Unlock()
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.DecodePacket(): " + err.Error())
+		log.Error("capture.golibrtsp.DecodePacket(): " + err.Error())
 		return image.YCbCr{}, err
 	}
 	if img.Bounds().Empty() {
-		log.Log.Debug("capture.golibrtsp.DecodePacket(): empty frame")
+		log.Debug("capture.golibrtsp.DecodePacket(): empty frame")
 		return image.YCbCr{}, errors.New("Empty image")
 	}
 	return img, nil
@@ -1310,11 +1429,11 @@ func (g *Golibrtsp) DecodePacketRaw(pkt packets.Packet) (image.Gray, error) {
 	}
 	g.VideoDecoderMutex.Unlock()
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.DecodePacketRaw(): " + err.Error())
+		log.Error("capture.golibrtsp.DecodePacketRaw(): " + err.Error())
 		return image.Gray{}, err
 	}
 	if img.Bounds().Empty() {
-		log.Log.Debug("capture.golibrtsp.DecodePacketRaw(): empty image")
+		log.Debug("capture.golibrtsp.DecodePacketRaw(): empty image")
 		return image.Gray{}, errors.New("Empty image")
 	}
 
@@ -1359,7 +1478,11 @@ func (g *Golibrtsp) Close(ctxOtel context.Context) error {
 	_, span := tracer.Start(ctxOtel, "Close")
 	defer span.End()
 
-	// Close the demuxer.
+	if !g.clientStarted {
+		return nil
+	}
+
+	g.clientStarted = false
 	g.Client.Close()
 
 	// We will have created the decoders globally, so we don't need to close them here.
@@ -1404,13 +1527,13 @@ func newDecoder(codecName string) (*Decoder, error) {
 
 	res := C.avcodec_open2(codecCtx, codec, nil)
 	if res < 0 {
-		C.avcodec_close(codecCtx)
+		C.avcodec_free_context(&codecCtx)
 		return nil, fmt.Errorf("avcodec_open2() failed")
 	}
 
 	srcFrame := C.av_frame_alloc()
 	if srcFrame == nil {
-		C.avcodec_close(codecCtx)
+		C.avcodec_free_context(&codecCtx)
 		return nil, fmt.Errorf("av_frame_alloc() failed")
 	}
 
@@ -1426,7 +1549,7 @@ func (d *Decoder) Close() {
 		C.av_frame_free(&d.srcFrame)
 	}
 	C.av_frame_free(&d.srcFrame)
-	C.avcodec_close(d.codecCtx)
+	C.avcodec_free_context(&d.codecCtx)
 }
 
 func (d *Decoder) decode(nalu []byte) (image.YCbCr, error) {
@@ -1613,7 +1736,12 @@ func (g *Golibrtsp) getEnhancedFPS(sps *h264.SPS, streamIndex int8) float64 {
 
 	// Check if SPS FPS is reasonable (between 1 and 120 fps)
 	if spsFPS > 0 && spsFPS <= 120 {
-		log.Log.Debug(fmt.Sprintf("capture.golibrtsp.getEnhancedFPS(): SPS FPS: %.2f", spsFPS))
+		log.WithFields(log.Fields{
+			"component": "capture",
+			"event":     "frame_rate_selected",
+			"fps":       spsFPS,
+			"source":    "sps",
+		}).Debug("RTSP frame rate selected")
 		return spsFPS
 	}
 
@@ -1621,7 +1749,12 @@ func (g *Golibrtsp) getEnhancedFPS(sps *h264.SPS, streamIndex int8) float64 {
 	if ft := g.fpsTrackers[streamIndex]; ft != nil {
 		ptsFPS := ft.fps()
 		if ptsFPS > 0 && ptsFPS <= 120 {
-			log.Log.Debug(fmt.Sprintf("capture.golibrtsp.getEnhancedFPS(): PTS FPS: %.2f", ptsFPS))
+			log.WithFields(log.Fields{
+				"component": "capture",
+				"event":     "frame_rate_selected",
+				"fps":       ptsFPS,
+				"source":    "presentation_timestamp",
+			}).Debug("RTSP frame rate selected")
 			return ptsFPS
 		}
 	}
@@ -1723,13 +1856,15 @@ func (g *Golibrtsp) getSPSTimingInfo(sps *h264.SPS) (hasVUI bool, timeScale uint
 func (g *Golibrtsp) debugSPSInfo(sps *h264.SPS, streamType string) {
 	hasVUI, timeScale, numUnitsInTick, fps := g.getSPSTimingInfo(sps)
 
-	log.Log.Debug(fmt.Sprintf("capture.golibrtsp.debugSPSInfo(%s): Width=%d, Height=%d",
+	log.WithFields(log.Fields{
-		streamType, sps.Width(), sps.Height()))
+		"component":         "capture",
-	log.Log.Debug(fmt.Sprintf("capture.golibrtsp.debugSPSInfo(%s): HasVUI=%t, FPS=%.2f",
+		"event":             "sps_inspected",
-		streamType, hasVUI, fps))
+		"fps":               fps,
-
+		"has_vui":           hasVUI,
-	if hasVUI {
+		"height_pixels":     sps.Height(),
-		log.Log.Debug(fmt.Sprintf("capture.golibrtsp.debugSPSInfo(%s): TimeScale=%d, NumUnitsInTick=%d",
+		"num_units_in_tick": numUnitsInTick,
-			streamType, timeScale, numUnitsInTick))
+		"stream":            streamType,
-	}
+		"time_scale":        timeScale,
+		"width_pixels":      sps.Width(),
+	}).Debug("RTSP sequence parameter set inspected")
 }

machinery/src/capture/gortsplib_test.go

@@ -2,13 +2,70 @@ package capture
 
 import (
 	"bytes"
+	"context"
 	"encoding/pem"
+	"errors"
+	"math"
 	"net/http/httptest"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 )
 
+func TestPreRecordingGOPCount(t *testing.T) {
+	maxInt := int64(^uint(0) >> 1)
+	tests := []struct {
+		name         string
+		preRecording int64
+		gopDuration  float64
+		want         int
+		wantOK       bool
+	}{
+		{name: "normal duration", preRecording: 10, gopDuration: 2.9, want: 6, wantOK: true},
+		{name: "duration longer than buffer", preRecording: 1, gopDuration: 2, want: 1, wantOK: true},
+		{name: "largest representable result", preRecording: maxInt - 1, gopDuration: 1, want: int(maxInt), wantOK: true},
+		{name: "result exceeds int", preRecording: maxInt, gopDuration: 1, wantOK: false},
+		{name: "non-positive pre-recording", preRecording: 0, gopDuration: 1, wantOK: false},
+		{name: "sub-second GOP", preRecording: 10, gopDuration: 0.9, wantOK: false},
+		{name: "NaN GOP", preRecording: 10, gopDuration: math.NaN(), wantOK: false},
+		{name: "infinite GOP", preRecording: 10, gopDuration: math.Inf(1), wantOK: false},
+		{name: "GOP exceeds int64", preRecording: 10, gopDuration: float64(math.MaxInt64), wantOK: false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, ok := preRecordingGOPCount(tt.preRecording, tt.gopDuration)
+			if ok != tt.wantOK || got != tt.want {
+				t.Fatalf("preRecordingGOPCount(%d, %v) = (%d, %t), want (%d, %t)",
+					tt.preRecording, tt.gopDuration, got, ok, tt.want, tt.wantOK)
+			}
+		})
+	}
+}
+
+func TestGolibrtspCloseBeforeClientStart(t *testing.T) {
+	client := &Golibrtsp{}
+
+	if err := client.Close(context.Background()); err != nil {
+		t.Fatalf("Close() error = %v", err)
+	}
+}
+
+func TestSanitizeRTSPErrorRemovesCredentialsAndQuery(t *testing.T) {
+	rawURL := "rtsp://camera-user:camera-password@10.0.20.15/live?access_token=secret"
+	got := sanitizeRTSPError(errors.New("describe "+rawURL+": bad status code"), rawURL)
+
+	for _, secret := range []string{"camera-user", "camera-password", "access_token", "secret"} {
+		if strings.Contains(got.Error(), secret) {
+			t.Fatalf("sanitizeRTSPError() exposed %q in %q", secret, got)
+		}
+	}
+	if !strings.Contains(got.Error(), "rtsp://10.0.20.15/live") {
+		t.Fatalf("sanitizeRTSPError() removed useful host/path context: %q", got)
+	}
+}
+
 func TestRTSPSTLSConfig(t *testing.T) {
 	t.Run("verifies certificates by default", func(t *testing.T) {
 		t.Setenv(rtspsCAFileEnv, "")

machinery/src/capture/main.go

@@ -16,11 +16,11 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/kerberos-io/agent/machinery/src/conditions"
 	"github.com/kerberos-io/agent/machinery/src/encryption"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/kerberos-io/agent/machinery/src/utils"
 	"github.com/kerberos-io/agent/machinery/src/video"
+	log "github.com/sirupsen/logrus"
 	"go.opentelemetry.io/otel/trace"
 )
 
@@ -49,16 +49,17 @@ func publishRecordingState(mqttClient mqtt.Client, hubKey string, configuration
 	if err == nil {
 		mqttClient.Publish("kerberos/hub/"+hubKey, 2, false, payload)
 	} else {
-		log.Log.Error("capture.main.publishRecordingState(): failed to package MQTT message: " + err.Error())
+		log.Error("capture.main.publishRecordingState(): failed to package MQTT message: " + err.Error())
 	}
 }
 
-func recordingUploadMetadata(name, deviceKey string, timestamp int64, mp4Video *video.MP4) models.RecordingUploadMetadata {
+func recordingUploadMetadata(name, deviceKey string, timestamp int64, mp4Video *video.MP4, encrypted bool) models.RecordingUploadMetadata {
 	metadata := models.RecordingUploadMetadata{
 		FileName:  filepath.Base(name),
 		DeviceKey: deviceKey,
 		Timestamp: timestamp,
 		Duration:  mp4Video.VideoTotalDuration,
+		Encrypted: encrypted,
 	}
 	value := mp4Video.AverageFPS()
 	if value > 0 && value <= 240 && !math.IsInf(value, 0) && !math.IsNaN(value) {
@@ -72,7 +73,7 @@ func recordingUploadMetadata(name, deviceKey string, timestamp int64, mp4Video *
 func queueRecordingForUpload(configDirectory string, metadata models.RecordingUploadMetadata) {
 	payload, err := json.Marshal(metadata)
 	if err != nil {
-		log.Log.Error("capture.main.queueRecordingForUpload(): " + err.Error())
+		log.Error("capture.main.queueRecordingForUpload(): " + err.Error())
 		return
 	}
 
@@ -95,7 +96,7 @@ func queueRecordingForUpload(configDirectory string, metadata models.RecordingUp
 		err = os.Rename(marker.Name(), filepath.Join(configDirectory, "data", "cloud", models.RecordingUploadMetadataFileName(metadata.FileName)))
 	}
 	if err != nil {
-		log.Log.Error("capture.main.queueRecordingForUpload(): " + err.Error())
+		log.Error("capture.main.queueRecordingForUpload(): " + err.Error())
 	}
 }
 
@@ -137,9 +138,9 @@ func manualRecordingExpired(communication *models.Communication, now int64) bool
 		return false
 	}
 	if heartbeatExpired {
-		log.Log.Info("capture.main.HandleRecordStream(motiondetection): auto-stopping manual recording, no viewer heartbeat within timeout.")
+		log.Info("capture.main.HandleRecordStream(motiondetection): auto-stopping manual recording, no viewer heartbeat within timeout.")
 	} else {
-		log.Log.Info("capture.main.HandleRecordStream(motiondetection): auto-stopping manual recording, maximum duration reached.")
+		log.Info("capture.main.HandleRecordStream(motiondetection): auto-stopping manual recording, maximum duration reached.")
 	}
 	communication.IsRecordingManual.UnSet()
 	communication.RecordingManualHeartbeat.Store(0)
@@ -151,7 +152,7 @@ func manualRecordingExpired(communication *models.Communication, now int64) bool
 func CleanupRecordingDirectory(configDirectory string, configuration *models.Configuration) {
 	autoClean := configuration.Config.AutoClean
 	if autoClean != "true" {
-		log.Log.Info("HandleRecordStream: Autoclean disabled, nothing to do here.")
+		log.Info("HandleRecordStream: Autoclean disabled, nothing to do here.")
 		return
 	}
 
@@ -164,7 +165,7 @@ func CleanupRecordingDirectory(configDirectory string, configuration *models.Con
 	// disk while keeping a free-space reserve.
 	needsCleanup, err := recordingsNeedCleanup(recordingsDirectory, configuration)
 	if err != nil {
-		log.Log.Info("HandleRecordStream: something went wrong, " + err.Error())
+		log.Info("HandleRecordStream: something went wrong, " + err.Error())
 		return
 	}
 	if !needsCleanup {
@@ -185,12 +186,12 @@ func CleanupRecordingDirectory(configDirectory string, configuration *models.Con
 	// (otherwise a long outage would fill the disk and stop new recordings).
 	name, pending, err := pickRecordingToCleanup(recordingsDirectory, cloudDirectory)
 	if err != nil {
-		log.Log.Info("HandleRecordStream: something went wrong, " + err.Error())
+		log.Info("HandleRecordStream: something went wrong, " + err.Error())
 		return
 	}
 
 	if err := os.Remove(recordingsDirectory + "/" + name); err != nil {
-		log.Log.Info("HandleRecordStream: something went wrong, " + err.Error())
+		log.Info("HandleRecordStream: something went wrong, " + err.Error())
 		return
 	}
 
@@ -200,14 +201,14 @@ func CleanupRecordingDirectory(configDirectory string, configuration *models.Con
 		// that was never uploaded to keep recording new footage. Also remove the
 		// now-dangling upload marker so the upload loop doesn't keep trying to
 		// upload a file that no longer exists.
-		log.Log.Warning("HandleRecordStream: removed oldest recording as part of cleanup, but it was STILL PENDING UPLOAD (disk full of un-uploaded recordings) - " + recordingsDirectory + "/" + name)
+		log.Warn("HandleRecordStream: removed oldest recording as part of cleanup, but it was STILL PENDING UPLOAD (disk full of un-uploaded recordings) - " + recordingsDirectory + "/" + name)
 		for _, markerName := range uploadMarkerNames(name) {
 			if err := os.Remove(filepath.Join(cloudDirectory, markerName)); err != nil && !os.IsNotExist(err) {
-				log.Log.Info("HandleRecordStream: could not remove dangling upload marker " + markerName + ", " + err.Error())
+				log.Info("HandleRecordStream: could not remove dangling upload marker " + markerName + ", " + err.Error())
 			}
 		}
 	} else {
-		log.Log.Info("HandleRecordStream: removed oldest file as part of cleanup - " + recordingsDirectory + "/" + name)
+		log.Info("HandleRecordStream: removed oldest file as part of cleanup - " + recordingsDirectory + "/" + name)
 	}
 }
 
@@ -331,7 +332,7 @@ func recordingPendingUpload(cloudDirectory, recordingName string) bool {
 	return false
 }
 
-func HandleRecordStream(queue *packets.Queue, configDirectory string, configuration *models.Configuration, communication *models.Communication, rtspClient RTSPClient, mqttClient mqtt.Client) {
+func HandleRecordStream(queue *packets.Queue, configDirectory string, configuration *models.Configuration, communication *models.Communication, rtspClient RTSPClient, mqttClient mqtt.Client, motionEvents <-chan models.MotionDataPartial) {
 
 	config := configuration.Config
 	hubKey := config.HubKey
@@ -345,9 +346,9 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 	communication.RecordingManualHeartbeatSeen.UnSet()
 
 	if config.Capture.Recording == "false" {
-		log.Log.Info("capture.main.HandleRecordStream(): disabled, we will not record anything.")
+		log.Info("capture.main.HandleRecordStream(): disabled, we will not record anything.")
 	} else {
-		log.Log.Debug("capture.main.HandleRecordStream(): started")
+		log.Debug("capture.main.HandleRecordStream(): started")
 
 		preRecording := config.Capture.PreRecording * 1000
 		postRecording := config.Capture.PostRecording * 1000           // number of seconds to record.
@@ -360,7 +361,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 		}
 
 		if maxRecordingPeriod < preRecording+postRecording {
-			log.Log.Error("capture.main.HandleRecordStream(): maxRecordingPeriod is less than preRecording + postRecording, this is not allowed. Setting maxRecordingPeriod to preRecording + postRecording.")
+			log.Error("capture.main.HandleRecordStream(): maxRecordingPeriod is less than preRecording + postRecording, this is not allowed. Setting maxRecordingPeriod to preRecording + postRecording.")
 			maxRecordingPeriod = preRecording + postRecording
 		}
 
@@ -394,7 +395,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 			var name string
 
 			// Do not do anything!
-			log.Log.Info("capture.main.HandleRecordStream(continuous): start recording")
+			log.Info("capture.main.HandleRecordStream(continuous): start recording")
 
 			start := false
 
@@ -440,10 +441,10 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					}
 					if (videoCodec == "H264" && (len(mp4Video.SPSNALUs) == 0 || len(mp4Video.PPSNALUs) == 0)) ||
 						(videoCodec == "H265" && (len(mp4Video.VPSNALUs) == 0 || len(mp4Video.SPSNALUs) == 0 || len(mp4Video.PPSNALUs) == 0)) {
-						log.Log.Warning("capture.main.HandleRecordStream(continuous): closing MP4 without full parameter sets, moov may be incomplete")
+						log.Warn("capture.main.HandleRecordStream(continuous): closing MP4 without full parameter sets, moov may be incomplete")
 					}
 					mp4Video.Close(&config)
-					log.Log.Info("capture.main.HandleRecordStream(continuous): recording finished: file save: " + name)
+					log.Info("capture.main.HandleRecordStream(continuous): recording finished: file save: " + name)
 
 					// Cleanup muxer
 					start = false
@@ -467,7 +468,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 						oldName := name
 						name = s + ".mp4"
 						fullName = configDirectory + "/data/recordings/" + name
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
+						log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
 
 						// Rename the file to the new name.
 						err := os.Rename(
@@ -475,12 +476,13 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 							configDirectory+"/data/recordings/"+s+".mp4")
 
 						if err != nil {
-							log.Log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
+							log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
 						}
 					} else {
-						log.Log.Info("capture.main.HandleRecordStream(continuous): no video data recorded, not renaming file.")
+						log.Info("capture.main.HandleRecordStream(continuous): no video data recorded, not renaming file.")
 					}
 
+					encrypted := false
 					// Check if we need to encrypt the recording.
 					if config.Encryption != nil && config.Encryption.Enabled == "true" && config.Encryption.Recordings == "true" && config.Encryption.SymmetricKey != "" {
 						// reopen file into memory 'fullName'
@@ -492,17 +494,19 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 								// write back to file
 								err := os.WriteFile(fullName, []byte(encryptedContents), 0644)
 								if err != nil {
-									log.Log.Error("capture.main.HandleRecordStream(continuous): error writing file: " + err.Error())
+									log.Error("capture.main.HandleRecordStream(continuous): error writing file: " + err.Error())
+								} else {
+									encrypted = true
 								}
 							} else {
-								log.Log.Error("capture.main.HandleRecordStream(continuous): error encrypting file: " + err.Error())
+								log.Error("capture.main.HandleRecordStream(continuous): error encrypting file: " + err.Error())
 							}
 						} else {
-							log.Log.Error("capture.main.HandleRecordStream(continuous): error reading file: " + err.Error())
+							log.Error("capture.main.HandleRecordStream(continuous): error reading file: " + err.Error())
 						}
 					}
 
-					queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, startRecording, mp4Video))
+					queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, startRecording, mp4Video, encrypted))
 
 					recordingStatus = "idle"
 
@@ -520,7 +524,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					// We'll validate those conditions and if not valid we'll not do anything.
 					valid, err := conditions.Validate(loc, configuration)
 					if !valid && err != nil {
-						log.Log.Debug("capture.main.HandleRecordStream(continuous): " + err.Error() + ".")
+						log.Debug("capture.main.HandleRecordStream(continuous): " + err.Error() + ".")
 						time.Sleep(5 * time.Second)
 						continue
 					}
@@ -552,7 +556,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					fullName = configDirectory + "/data/recordings/" + name
 
 					// Running...
-					log.Log.Info("capture.main.HandleRecordStream(continuous): recording started")
+					log.Info("capture.main.HandleRecordStream(continuous): recording started")
 
 					// Get width and height from the camera.
 					width := configuration.Config.Capture.IPCamera.Width
@@ -564,7 +568,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					vpsNALUS := configuration.Config.Capture.IPCamera.VPSNALUs
 
 					if len(spsNALUS) == 0 || len(ppsNALUS) == 0 {
-						log.Log.Warning("capture.main.HandleRecordStream(continuous): missing SPS/PPS at recording start")
+						log.Warn("capture.main.HandleRecordStream(continuous): missing SPS/PPS at recording start")
 					}
 					// Create a video file, and set the dimensions.
 					mp4Video = video.NewMP4(fullName, spsNALUS, ppsNALUS, vpsNALUS, configuration.Config.Capture.MaxLengthRecording)
@@ -579,7 +583,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					if audioCodec == "AAC" {
 						audioTrack = mp4Video.AddAudioTrack("AAC")
 					} else if audioCodec == "PCM_MULAW" {
-						log.Log.Debug("capture.main.HandleRecordStream(continuous): no AAC audio codec detected, skipping audio track.")
+						log.Debug("capture.main.HandleRecordStream(continuous): no AAC audio codec detected, skipping audio track.")
 					}
 
 					writeSampleToMP4(mp4Video, videoTrack, audioTrack, pkt)
@@ -600,7 +604,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 			if cursorError != nil {
 				if recordingStatus == "started" {
 
-					log.Log.Info("capture.main.HandleRecordStream(continuous): Recording finished: file save: " + name)
+					log.Info("capture.main.HandleRecordStream(continuous): Recording finished: file save: " + name)
 
 					// Cleanup muxer
 					start = false
@@ -624,7 +628,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 						oldName := name
 						name = s + ".mp4"
 						fullName = configDirectory + "/data/recordings/" + name
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
+						log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
 
 						// Rename the file to the new name.
 						err := os.Rename(
@@ -632,12 +636,13 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 							configDirectory+"/data/recordings/"+s+".mp4")
 
 						if err != nil {
-							log.Log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
+							log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
 						}
 					} else {
-						log.Log.Info("capture.main.HandleRecordStream(continuous): no video data recorded, not renaming file.")
+						log.Info("capture.main.HandleRecordStream(continuous): no video data recorded, not renaming file.")
 					}
 
+					encrypted := false
 					// Check if we need to encrypt the recording.
 					if config.Encryption != nil && config.Encryption.Enabled == "true" && config.Encryption.Recordings == "true" && config.Encryption.SymmetricKey != "" {
 						// reopen file into memory 'fullName'
@@ -649,17 +654,19 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 								// write back to file
 								err := os.WriteFile(fullName, []byte(encryptedContents), 0644)
 								if err != nil {
-									log.Log.Error("capture.main.HandleRecordStream(motiondetection): error writing file: " + err.Error())
+									log.Error("capture.main.HandleRecordStream(motiondetection): error writing file: " + err.Error())
+								} else {
+									encrypted = true
 								}
 							} else {
-								log.Log.Error("capture.main.HandleRecordStream(motiondetection): error encrypting file: " + err.Error())
+								log.Error("capture.main.HandleRecordStream(motiondetection): error encrypting file: " + err.Error())
 							}
 						} else {
-							log.Log.Error("capture.main.HandleRecordStream(motiondetection): error reading file: " + err.Error())
+							log.Error("capture.main.HandleRecordStream(motiondetection): error reading file: " + err.Error())
 						}
 					}
 
-					queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, startRecording, mp4Video))
+					queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, startRecording, mp4Video, encrypted))
 
 					recordingStatus = "idle"
 
@@ -672,7 +679,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 			}
 		} else {
 
-			log.Log.Info("capture.main.HandleRecordStream(motiondetection): Start motion based recording ")
+			log.Info("capture.main.HandleRecordStream(motiondetection): Start motion based recording ")
 
 			var lastRecordingTime int64 = 0 // last recording timestamp in milliseconds
 			var displayTime int64 = 0       // display time in milliseconds
@@ -680,7 +687,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 			var videoTrack uint32
 			var audioTrack uint32
 
-			for motion := range communication.HandleMotion {
+			for motion := range motionEvents {
 
 				// Get as much packets we need.
 				var cursorError error
@@ -710,7 +717,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 
 				// If startRecording is 0, we will continue as it might be we are in a state of restarting the agent.
 				if startRecording == 0 {
-					log.Log.Info("capture.main.HandleRecordStream(motiondetection): startRecording is 0, we will continue as it might be we are in a state of restarting the agent.")
+					log.Info("capture.main.HandleRecordStream(motiondetection): startRecording is 0, we will continue as it might be we are in a state of restarting the agent.")
 					continue
 				}
 
@@ -748,7 +755,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 				fullName := configDirectory + "/data/recordings/" + name
 
 				// Running...
-				log.Log.Info("capture.main.HandleRecordStream(motiondetection): recording started (" + name + ")" + " at " + strconv.FormatInt(displayTimeSeconds, 10) + " unix")
+				log.Info("capture.main.HandleRecordStream(motiondetection): recording started (" + name + ")" + " at " + strconv.FormatInt(displayTimeSeconds, 10) + " unix")
 
 				// Get width and height from the camera.
 				width := configuration.Config.Capture.IPCamera.Width
@@ -760,7 +767,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 				vpsNALUS := configuration.Config.Capture.IPCamera.VPSNALUs
 
 				if len(spsNALUS) == 0 || len(ppsNALUS) == 0 {
-					log.Log.Warning("capture.main.HandleRecordStream(motiondetection): missing SPS/PPS at recording start")
+					log.Warn("capture.main.HandleRecordStream(motiondetection): missing SPS/PPS at recording start")
 				}
 				// Create the MP4 only once the first keyframe arrives.
 				var mp4Video *video.MP4
@@ -769,16 +776,16 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 
 					nextPkt, cursorError = recordingCursor.ReadPacket()
 					if cursorError != nil {
-						log.Log.Error("capture.main.HandleRecordStream(motiondetection): " + cursorError.Error())
+						log.Error("capture.main.HandleRecordStream(motiondetection): " + cursorError.Error())
 					}
 
 					now = time.Now().UnixMilli()
 					select {
-					case motion := <-communication.HandleMotion:
+					case motion := <-motionEvents:
 						motionTimestamp = now
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): motion detected while recording. Expanding recording.")
+						log.Info("capture.main.HandleRecordStream(motiondetection): motion detected while recording. Expanding recording.")
 						numberOfChanges := motion.NumberOfChanges
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): Received message with recording data, detected changes to save: " + strconv.Itoa(numberOfChanges))
+						log.Info("capture.main.HandleRecordStream(motiondetection): Received message with recording data, detected changes to save: " + strconv.Itoa(numberOfChanges))
 					default:
 					}
 
@@ -794,16 +801,16 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					}
 
 					if start && (motionTimestamp+postRecording-now < 0 || now-startRecording > maxRecordingPeriod-500) && nextPkt.IsKeyFrame {
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): timestamp+postRecording-now < 0  - " + strconv.FormatInt(motionTimestamp+postRecording-now, 10) + " < 0")
+						log.Info("capture.main.HandleRecordStream(motiondetection): timestamp+postRecording-now < 0  - " + strconv.FormatInt(motionTimestamp+postRecording-now, 10) + " < 0")
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): now-startRecording > maxRecordingPeriod-500 - " + strconv.FormatInt(now-startRecording, 10) + " > " + strconv.FormatInt(maxRecordingPeriod-500, 10))
+						log.Info("capture.main.HandleRecordStream(motiondetection): now-startRecording > maxRecordingPeriod-500 - " + strconv.FormatInt(now-startRecording, 10) + " > " + strconv.FormatInt(maxRecordingPeriod-500, 10))
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): closing recording (timestamp: " + strconv.FormatInt(motionTimestamp, 10) + ", postRecording: " + strconv.FormatInt(postRecording, 10) + ", now: " + strconv.FormatInt(now, 10) + ", startRecording: " + strconv.FormatInt(startRecording, 10) + ", maxRecordingPeriod: " + strconv.FormatInt(maxRecordingPeriod, 10))
+						log.Info("capture.main.HandleRecordStream(motiondetection): closing recording (timestamp: " + strconv.FormatInt(motionTimestamp, 10) + ", postRecording: " + strconv.FormatInt(postRecording, 10) + ", now: " + strconv.FormatInt(now, 10) + ", startRecording: " + strconv.FormatInt(startRecording, 10) + ", maxRecordingPeriod: " + strconv.FormatInt(maxRecordingPeriod, 10))
 						break
 					}
 					if pkt.IsKeyFrame && !start && pkt.CurrentTime >= startRecording {
 						// We start the recording if we have a keyframe and the last duration is 0 or less than the current packet time.
 						// It could be start we start from the beginning of the recording.
-						log.Log.Debug("capture.main.HandleRecordStream(motiondetection): write frames")
+						log.Debug("capture.main.HandleRecordStream(motiondetection): write frames")
-						log.Log.Debug("capture.main.HandleRecordStream(motiondetection): recording started on keyframe")
+						log.Debug("capture.main.HandleRecordStream(motiondetection): recording started on keyframe")
 
 						// Align duration timers with the first keyframe.
 						startRecording = pkt.CurrentTime
@@ -821,7 +828,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 						if audioCodec == "AAC" {
 							audioTrack = mp4Video.AddAudioTrack("AAC")
 						} else if audioCodec == "PCM_MULAW" {
-							log.Log.Debug("capture.main.HandleRecordStream(continuous): no AAC audio codec detected, skipping audio track.")
+							log.Debug("capture.main.HandleRecordStream(continuous): no AAC audio codec detected, skipping audio track.")
 						}
 						start = true
 
@@ -840,7 +847,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 				lastRecordingTime = pkt.CurrentTime
 
 				if mp4Video == nil {
-					log.Log.Warning("capture.main.HandleRecordStream(motiondetection): recording closed without keyframe; no MP4 created")
+					log.Warn("capture.main.HandleRecordStream(motiondetection): recording closed without keyframe; no MP4 created")
 					continue
 				}
 
@@ -856,10 +863,10 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 				}
 				if (videoCodec == "H264" && (len(mp4Video.SPSNALUs) == 0 || len(mp4Video.PPSNALUs) == 0)) ||
 					(videoCodec == "H265" && (len(mp4Video.VPSNALUs) == 0 || len(mp4Video.SPSNALUs) == 0 || len(mp4Video.PPSNALUs) == 0)) {
-					log.Log.Warning("capture.main.HandleRecordStream(motiondetection): closing MP4 without full parameter sets, moov may be incomplete")
+					log.Warn("capture.main.HandleRecordStream(motiondetection): closing MP4 without full parameter sets, moov may be incomplete")
 				}
 				mp4Video.Close(&config)
-				log.Log.Info("capture.main.HandleRecordStream(motiondetection): file save: " + name)
+				log.Info("capture.main.HandleRecordStream(motiondetection): file save: " + name)
 
 				// Notify the hub / live-view UI that this camera stopped recording.
 				publishRecordingState(mqttClient, hubKey, configuration, false)
@@ -870,10 +877,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 				// restart when the recording has expired (heartbeat lapsed or max
 				// duration reached), so it ends here instead of recording forever.
 				if communication.IsRecordingManual.IsSet() && !manualRecordingExpired(communication, time.Now().UnixMilli()) {
-					select {
+					communication.TrySendMotion(models.MotionDataPartial{Timestamp: time.Now().Unix(), NumberOfChanges: 100000000})
-					case communication.HandleMotion <- models.MotionDataPartial{Timestamp: time.Now().Unix(), NumberOfChanges: 100000000}:
-					default:
-					}
 				}
 
 				// Update the name of the recording with the duration.
@@ -894,7 +898,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					oldName := name
 					name = s + ".mp4"
 					fullName = configDirectory + "/data/recordings/" + name
-					log.Log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
+					log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
 
 					// Rename the file to the new name.
 					err := os.Rename(
@@ -902,12 +906,13 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 						configDirectory+"/data/recordings/"+s+".mp4")
 
 					if err != nil {
-						log.Log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
+						log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
 					}
 				} else {
-					log.Log.Info("capture.main.HandleRecordStream(motiondetection): no video data recorded, not renaming file.")
+					log.Info("capture.main.HandleRecordStream(motiondetection): no video data recorded, not renaming file.")
 				}
 
+				encrypted := false
 				// Check if we need to encrypt the recording.
 				if config.Encryption != nil && config.Encryption.Enabled == "true" && config.Encryption.Recordings == "true" && config.Encryption.SymmetricKey != "" {
 					// reopen file into memory 'fullName'
@@ -919,24 +924,26 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 							// write back to file
 							err := os.WriteFile(fullName, []byte(encryptedContents), 0644)
 							if err != nil {
-								log.Log.Error("capture.main.HandleRecordStream(motiondetection): error writing file: " + err.Error())
+								log.Error("capture.main.HandleRecordStream(motiondetection): error writing file: " + err.Error())
+							} else {
+								encrypted = true
 							}
 						} else {
-							log.Log.Error("capture.main.HandleRecordStream(motiondetection): error encrypting file: " + err.Error())
+							log.Error("capture.main.HandleRecordStream(motiondetection): error encrypting file: " + err.Error())
 						}
 					} else {
-						log.Log.Error("capture.main.HandleRecordStream(motiondetection): error reading file: " + err.Error())
+						log.Error("capture.main.HandleRecordStream(motiondetection): error reading file: " + err.Error())
 					}
 				}
 
-				queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, displayTime, mp4Video))
+				queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, displayTime, mp4Video, encrypted))
 
 				// Clean up the recording directory if necessary.
 				CleanupRecordingDirectory(configDirectory, configuration)
 			}
 		}
 
-		log.Log.Debug("capture.main.HandleRecordStream(): finished")
+		log.Debug("capture.main.HandleRecordStream(): finished")
 	}
 }
 
@@ -1030,13 +1037,14 @@ func Base64Image(captureDevice *Capture, communication *models.Communication, co
 	var cursor *packets.QueueCursor
 
 	// We'll pick the right client and decoder.
-	rtspClient := captureDevice.RTSPSubClient
+	rtspClient := captureDevice.SubClient()
 	if rtspClient != nil {
-		queue = communication.SubQueue
+		queue = communication.SubQueue()
-		cursor = queue.Latest()
 	} else {
-		rtspClient = captureDevice.RTSPClient
+		rtspClient = captureDevice.MainClient()
-		queue = communication.Queue
+		queue = communication.MainQueue()
+	}
+	if queue != nil {
 		cursor = queue.Latest()
 	}
 
@@ -1076,13 +1084,14 @@ func JpegImage(captureDevice *Capture, communication *models.Communication) imag
 	var cursor *packets.QueueCursor
 
 	// We'll pick the right client and decoder.
-	rtspClient := captureDevice.RTSPSubClient
+	rtspClient := captureDevice.SubClient()
 	if rtspClient != nil {
-		queue = communication.SubQueue
+		queue = communication.SubQueue()
-		cursor = queue.Latest()
 	} else {
-		rtspClient = captureDevice.RTSPClient
+		rtspClient = captureDevice.MainClient()
-		queue = communication.Queue
+		queue = communication.MainQueue()
+	}
+	if queue != nil {
 		cursor = queue.Latest()
 	}
 
@@ -1137,20 +1146,16 @@ func writeSampleToMP4(mp4Video *video.MP4, videoTrack, audioTrack uint32, pkt pa
 			dts = pts - uint64(compositionOffset)
 		}
 		if err := mp4Video.AddSampleToTrack(videoTrack, pkt.IsKeyFrame, pkt.Data, dts, compositionOffset); err != nil {
-			log.Log.Error("capture.main.writeSampleToMP4(): " + err.Error())
+			log.Error("capture.main.writeSampleToMP4(): " + err.Error())
 		}
 	} else if pkt.IsAudio {
 		if pkt.Codec == "AAC" {
 			if err := mp4Video.AddSampleToTrack(audioTrack, pkt.IsKeyFrame, pkt.Data, pts, 0); err != nil {
-				log.Log.Error("capture.main.writeSampleToMP4(): " + err.Error())
+				log.Error("capture.main.writeSampleToMP4(): " + err.Error())
 			}
 		} else if pkt.Codec == "PCM_MULAW" {
 			// TODO: transcode to AAC, some work to do..
-			log.Log.Debug("capture.main.writeSampleToMP4(): no AAC audio codec detected, skipping audio track.")
+			log.Debug("capture.main.writeSampleToMP4(): no AAC audio codec detected, skipping audio track.")
 		}
 	}
 }
-
-/*func convertPTS2(v int64) uint64 {
-	return uint64(v) / 100
-}*/

machinery/src/capture/main_test.go

@@ -42,7 +42,7 @@ func TestQueueRecordingForUploadStoresFinalizedMetadata(t *testing.T) {
 	}
 
 	mp4Video := &video.MP4{VideoTotalDuration: 20452, SampleCount: 613}
-	metadata := recordingUploadMetadata("recording.mp4", "device-key", 1785934709414, mp4Video)
+	metadata := recordingUploadMetadata("recording.mp4", "device-key", 1785934709414, mp4Video, true)
 	queueRecordingForUpload(configDirectory, metadata)
 
 	got, err := os.ReadFile(filepath.Join(configDirectory, "data", "cloud", "recording.metadata"))
@@ -54,7 +54,7 @@ func TestQueueRecordingForUploadStoresFinalizedMetadata(t *testing.T) {
 		t.Fatalf("decode upload marker: %v", err)
 	}
 	expectedFPS := mp4Video.AverageFPS()
-	if stored.FileName != "recording.mp4" || stored.DeviceKey != "device-key" || stored.Timestamp != 1785934709414 || stored.Duration != 20452 || math.Abs(stored.FPS-expectedFPS) > 1e-9 {
+	if stored.FileName != "recording.mp4" || stored.DeviceKey != "device-key" || stored.Timestamp != 1785934709414 || stored.Duration != 20452 || math.Abs(stored.FPS-expectedFPS) > 1e-9 || !stored.Encrypted {
 		t.Fatalf("upload marker = %+v", stored)
 	}
 	if stored.FPS == math.Floor(stored.FPS) {

machinery/src/capture/rtsp_client.go

@@ -3,36 +3,73 @@ package capture
 import (
 	"context"
 	"image"
+	"sync"
 
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 )
 
 type Capture struct {
-	RTSPClient            *Golibrtsp
+	clientsMu             sync.RWMutex
-	RTSPSubClient         *Golibrtsp
+	rtspClient            *Golibrtsp
-	RTSPBackChannelClient *Golibrtsp
+	rtspSubClient         *Golibrtsp
+	rtspBackChannelClient *Golibrtsp
 }
 
 func (c *Capture) SetMainClient(rtspUrl string) *Golibrtsp {
-	c.RTSPClient = &Golibrtsp{
+	client := &Golibrtsp{
 		Url: rtspUrl,
 	}
-	return c.RTSPClient
+	c.clientsMu.Lock()
+	c.rtspClient = client
+	c.clientsMu.Unlock()
+	return client
 }
 
 func (c *Capture) SetSubClient(rtspUrl string) *Golibrtsp {
-	c.RTSPSubClient = &Golibrtsp{
+	client := &Golibrtsp{
 		Url: rtspUrl,
 	}
-	return c.RTSPSubClient
+	c.clientsMu.Lock()
+	c.rtspSubClient = client
+	c.clientsMu.Unlock()
+	return client
 }
 
 func (c *Capture) SetBackChannelClient(rtspUrl string) *Golibrtsp {
-	c.RTSPBackChannelClient = &Golibrtsp{
+	client := &Golibrtsp{
 		Url: rtspUrl,
 	}
-	return c.RTSPBackChannelClient
+	c.clientsMu.Lock()
+	c.rtspBackChannelClient = client
+	c.clientsMu.Unlock()
+	return client
+}
+
+func (c *Capture) MainClient() *Golibrtsp {
+	c.clientsMu.RLock()
+	defer c.clientsMu.RUnlock()
+	return c.rtspClient
+}
+
+func (c *Capture) SubClient() *Golibrtsp {
+	c.clientsMu.RLock()
+	defer c.clientsMu.RUnlock()
+	return c.rtspSubClient
+}
+
+func (c *Capture) ClearClients(main, sub, backchannel *Golibrtsp) {
+	c.clientsMu.Lock()
+	defer c.clientsMu.Unlock()
+	if c.rtspClient == main {
+		c.rtspClient = nil
+	}
+	if c.rtspSubClient == sub {
+		c.rtspSubClient = nil
+	}
+	if c.rtspBackChannelClient == backchannel {
+		c.rtspBackChannelClient = nil
+	}
 }
 
 // RTSPClient is a interface that abstracts the RTSP client implementation.

machinery/src/capture/rtsp_client_test.go

@@ -0,0 +1,62 @@
+package capture
+
+import (
+	"strconv"
+	"sync"
+	"testing"
+)
+
+func TestCaptureClientAccessCanRaceReplacement(t *testing.T) {
+	captureDevice := &Capture{}
+	captureDevice.SetMainClient("rtsp://main/0")
+	captureDevice.SetSubClient("rtsp://sub/0")
+
+	var workers sync.WaitGroup
+	workers.Add(2)
+	go func() {
+		defer workers.Done()
+		for replacement := 1; replacement <= 1000; replacement++ {
+			suffix := strconv.Itoa(replacement)
+			captureDevice.SetMainClient("rtsp://main/" + suffix)
+			captureDevice.SetSubClient("rtsp://sub/" + suffix)
+		}
+	}()
+	go func() {
+		defer workers.Done()
+		for snapshot := 0; snapshot < 1000; snapshot++ {
+			if captureDevice.MainClient() == nil {
+				t.Error("MainClient() returned nil")
+				return
+			}
+			if captureDevice.SubClient() == nil {
+				t.Error("SubClient() returned nil")
+				return
+			}
+		}
+	}()
+	workers.Wait()
+}
+
+func TestCaptureClearClientsOnlyClearsMatchingRun(t *testing.T) {
+	captureDevice := &Capture{}
+	oldMain := captureDevice.SetMainClient("rtsp://main/old")
+	oldSub := captureDevice.SetSubClient("rtsp://sub/old")
+	oldBackchannel := captureDevice.SetBackChannelClient("rtsp://back/old")
+
+	newMain := captureDevice.SetMainClient("rtsp://main/new")
+	newSub := captureDevice.SetSubClient("rtsp://sub/new")
+	newBackchannel := captureDevice.SetBackChannelClient("rtsp://back/new")
+
+	captureDevice.ClearClients(oldMain, oldSub, oldBackchannel)
+	if captureDevice.MainClient() != newMain {
+		t.Fatal("stale cleanup cleared the new main client")
+	}
+	if captureDevice.SubClient() != newSub {
+		t.Fatal("stale cleanup cleared the new sub client")
+	}
+
+	captureDevice.ClearClients(newMain, newSub, newBackchannel)
+	if captureDevice.MainClient() != nil || captureDevice.SubClient() != nil {
+		t.Fatal("matching cleanup did not clear current clients")
+	}
+}

machinery/src/cloud/cloud_test.go

@@ -1,14 +1,23 @@
 package cloud
 
 import (
+	"encoding/json"
+	"errors"
+	"fmt"
 	"io"
 	"net/http"
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/kerberos-io/agent/machinery/src/models"
+	agentonvif "github.com/kerberos-io/agent/machinery/src/onvif"
+	goonvif "github.com/kerberos-io/onvif"
+	goonvifdevice "github.com/kerberos-io/onvif/device"
+	goonvifptz "github.com/kerberos-io/onvif/ptz"
 )
 
-func TestHeartbeatFailureLogIncludesHubResponse(t *testing.T) {
+func TestHeartbeatFailureLogOmitsHubResponseBody(t *testing.T) {
 	response := &http.Response{
 		StatusCode: http.StatusBadRequest,
 		Status:     "400 Bad Request",
@@ -19,16 +28,24 @@ func TestHeartbeatFailureLogIncludesHubResponse(t *testing.T) {
 	if err != nil {
 		t.Fatalf("readHeartbeatResponseBody() error = %v", err)
 	}
-	message := formatHeartbeatFailureLog(response, nil, responseBody, truncated, nil, 125*time.Millisecond)
+	fields := heartbeatFailureLogFields(response, responseBody, truncated, 125*time.Millisecond)
 
-	for _, expected := range []string{
+	for key, want := range map[string]interface{}{
-		"status_code=400",
+		"duration_ms":             int64(125),
-		`status="400 Bad Request"`,
+		"response_body_bytes":     len(responseBody),
-		"duration=125ms",
+		"response_body_truncated": false,
-		`response_body="{\"error\":\"invalid heartbeat\"}"`,
+		"status_code":             http.StatusBadRequest,
 	} {
-		if !strings.Contains(message, expected) {
+		if got := fields[key]; got != want {
-			t.Errorf("formatHeartbeatFailureLog() = %q, want it to contain %q", message, expected)
+			t.Errorf("%s = %v, want %v", key, got, want)
+		}
+	}
+	for key, value := range fields {
+		if strings.Contains(key, "response_body") && key != "response_body_bytes" && key != "response_body_truncated" {
+			t.Fatalf("unexpected response body field %q=%v", key, value)
+		}
+		if strings.Contains(fmt.Sprint(value), "invalid heartbeat") {
+			t.Fatalf("heartbeat log fields exposed response body in %q=%v", key, value)
 		}
 	}
 }
@@ -49,3 +66,527 @@ func TestReadHeartbeatResponseBodyTruncatesLargeBody(t *testing.T) {
 		t.Fatalf("len(body) = %d, want %d", len(body), heartbeatResponseBodyLogLimit)
 	}
 }
+
+func TestHeartbeatONVIFPayloadCachesStaticAndReusesLoopSubscription(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	device := newTestONVIFDevice()
+	camera := models.IPCamera{
+		ONVIFXAddr:    "http://camera/onvif",
+		ONVIFUsername: "operator",
+		ONVIFPassword: "secret",
+	}
+	initialEvents := []agentonvif.ONVIFEvents{{Key: "input-1", Type: "input", Value: "true", Timestamp: 1}}
+	loopEvents := []agentonvif.ONVIFEvents{{Key: "output-1", Type: "output", Value: "false", Timestamp: 2}}
+	wantPresets := mustJSONMarshal(t, []models.OnvifActionPreset{{Name: "Lobby", Token: "1"}})
+	wantInitialEvents := mustJSONMarshal(t, initialEvents)
+	wantLoopEvents := mustJSONMarshal(t, loopEvents)
+
+	var connectCalls, ptzConfigCalls, ptzFunctionCalls, presetCalls, createCalls, eventCalls, unsubscribeCalls int
+
+	heartbeatConnectToONVIFDevice = func(*models.IPCamera) (*goonvif.Device, goonvifdevice.GetCapabilitiesResponse, error) {
+		connectCalls++
+		return device, goonvifdevice.GetCapabilitiesResponse{}, nil
+	}
+	heartbeatGetPTZConfigurationsFromDevice = func(*goonvif.Device) (goonvifptz.GetConfigurationsResponse, error) {
+		ptzConfigCalls++
+		return goonvifptz.GetConfigurationsResponse{}, nil
+	}
+	heartbeatGetPTZFunctionsFromDevice = func(goonvifptz.GetConfigurationsResponse) ([]string, bool, bool) {
+		ptzFunctionCalls++
+		return nil, true, true
+	}
+	heartbeatGetPresetsFromDevice = func(*goonvif.Device) ([]models.OnvifActionPreset, error) {
+		presetCalls++
+		return []models.OnvifActionPreset{{Name: "Lobby", Token: "1"}}, nil
+	}
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		createCalls++
+		switch createCalls {
+		case 1:
+			return "initial-1", nil
+		case 2:
+			return "loop", nil
+		case 3:
+			return "initial-2", nil
+		default:
+			t.Fatalf("unexpected create pull point call %d", createCalls)
+			return "", nil
+		}
+	}
+	heartbeatGetEventMessages = func(_ *goonvif.Device, pullPointAddress string) ([]agentonvif.ONVIFEvents, error) {
+		eventCalls++
+		switch pullPointAddress {
+		case "initial-1", "initial-2":
+			return initialEvents, nil
+		case "loop":
+			return loopEvents, nil
+		default:
+			t.Fatalf("unexpected pull point address %q", pullPointAddress)
+			return nil, nil
+		}
+	}
+	heartbeatUnsubscribePullPoint = func(_ *goonvif.Device, pullPointAddress string) error {
+		unsubscribeCalls++
+		if pullPointAddress != "initial-1" && pullPointAddress != "initial-2" {
+			t.Fatalf("unexpected unsubscribe pull point %q", pullPointAddress)
+		}
+		return nil
+	}
+
+	state := newHeartbeatONVIFState()
+
+	payload := getHeartbeatONVIFPayload(camera, state)
+	if payload.enabled != "true" || payload.zoom != "true" || payload.panTilt != "true" || payload.presets != "true" {
+		t.Fatalf("unexpected static payload: %+v", payload)
+	}
+	if string(payload.presetsList) != string(wantPresets) {
+		t.Fatalf("payload.presetsList = %s, want %s", payload.presetsList, wantPresets)
+	}
+	if string(payload.eventsList) != string(wantInitialEvents) {
+		t.Fatalf("payload.eventsList = %s, want %s", payload.eventsList, wantInitialEvents)
+	}
+	if connectCalls != 1 || ptzConfigCalls != 1 || ptzFunctionCalls != 1 || presetCalls != 1 {
+		t.Fatalf("unexpected static call counts after first cycle: connect=%d ptzConfig=%d ptzFunctions=%d presets=%d", connectCalls, ptzConfigCalls, ptzFunctionCalls, presetCalls)
+	}
+	if createCalls != 2 || eventCalls != 1 || unsubscribeCalls != 1 {
+		t.Fatalf("unexpected event call counts after first cycle: create=%d events=%d unsubscribe=%d", createCalls, eventCalls, unsubscribeCalls)
+	}
+
+	payload = getHeartbeatONVIFPayload(camera, state)
+	if string(payload.eventsList) != string(wantLoopEvents) {
+		t.Fatalf("second payload.eventsList = %s, want %s", payload.eventsList, wantLoopEvents)
+	}
+	if connectCalls != 1 {
+		t.Fatalf("connectCalls = %d, want 1", connectCalls)
+	}
+	if ptzConfigCalls != 1 || ptzFunctionCalls != 1 || presetCalls != 1 {
+		t.Fatalf("static calls were not cached: ptzConfig=%d ptzFunctions=%d presets=%d", ptzConfigCalls, ptzFunctionCalls, presetCalls)
+	}
+	if createCalls != 3 || eventCalls != 3 || unsubscribeCalls != 2 {
+		t.Fatalf("temporary state subscription was not refreshed or loop subscription was not reused: create=%d events=%d unsubscribe=%d", createCalls, eventCalls, unsubscribeCalls)
+	}
+}
+
+func TestHeartbeatONVIFPayloadRefreshesAfterCameraConfigChange(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	deviceA := newTestONVIFDevice()
+	deviceB := newTestONVIFDevice()
+	cameraA := models.IPCamera{ONVIFXAddr: "http://camera-a/onvif", ONVIFUsername: "user", ONVIFPassword: "secret-a"}
+	cameraB := models.IPCamera{ONVIFXAddr: "http://camera-b/onvif", ONVIFUsername: "user", ONVIFPassword: "secret-b"}
+	var connectCalls, ptzConfigCalls, presetCalls, createCalls int
+	var unsubscribed []string
+
+	heartbeatConnectToONVIFDevice = func(camera *models.IPCamera) (*goonvif.Device, goonvifdevice.GetCapabilitiesResponse, error) {
+		connectCalls++
+		switch camera.ONVIFXAddr {
+		case cameraA.ONVIFXAddr:
+			return deviceA, goonvifdevice.GetCapabilitiesResponse{}, nil
+		case cameraB.ONVIFXAddr:
+			return deviceB, goonvifdevice.GetCapabilitiesResponse{}, nil
+		default:
+			t.Fatalf("unexpected camera address %q", camera.ONVIFXAddr)
+			return nil, goonvifdevice.GetCapabilitiesResponse{}, nil
+		}
+	}
+	heartbeatGetPTZConfigurationsFromDevice = func(*goonvif.Device) (goonvifptz.GetConfigurationsResponse, error) {
+		ptzConfigCalls++
+		return goonvifptz.GetConfigurationsResponse{}, nil
+	}
+	heartbeatGetPTZFunctionsFromDevice = func(goonvifptz.GetConfigurationsResponse) ([]string, bool, bool) {
+		return nil, false, true
+	}
+	heartbeatGetPresetsFromDevice = func(*goonvif.Device) ([]models.OnvifActionPreset, error) {
+		presetCalls++
+		return nil, nil
+	}
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		createCalls++
+		switch createCalls {
+		case 1:
+			return "initial-a", nil
+		case 2:
+			return "loop-a", nil
+		case 3:
+			return "initial-b", nil
+		case 4:
+			return "loop-b", nil
+		default:
+			t.Fatalf("unexpected create pull point call %d", createCalls)
+			return "", nil
+		}
+	}
+	heartbeatGetEventMessages = func(_ *goonvif.Device, pullPointAddress string) ([]agentonvif.ONVIFEvents, error) {
+		switch pullPointAddress {
+		case "initial-a":
+			return []agentonvif.ONVIFEvents{{Key: "a", Type: "input", Value: "true", Timestamp: 1}}, nil
+		case "initial-b":
+			return []agentonvif.ONVIFEvents{{Key: "b", Type: "input", Value: "false", Timestamp: 2}}, nil
+		default:
+			t.Fatalf("unexpected pull point address %q", pullPointAddress)
+			return nil, nil
+		}
+	}
+	heartbeatUnsubscribePullPoint = func(_ *goonvif.Device, pullPointAddress string) error {
+		unsubscribed = append(unsubscribed, pullPointAddress)
+		return nil
+	}
+
+	state := newHeartbeatONVIFState()
+	_ = getHeartbeatONVIFPayload(cameraA, state)
+	payload := getHeartbeatONVIFPayload(cameraB, state)
+
+	if connectCalls != 2 {
+		t.Fatalf("connectCalls = %d, want 2", connectCalls)
+	}
+	if ptzConfigCalls != 2 || presetCalls != 2 {
+		t.Fatalf("camera config change did not refresh static state: ptzConfig=%d presets=%d", ptzConfigCalls, presetCalls)
+	}
+	if createCalls != 4 {
+		t.Fatalf("camera config change did not recreate subscriptions: create=%d", createCalls)
+	}
+	if countString(unsubscribed, "loop-a") != 1 {
+		t.Fatalf("unsubscribed loop-a %d times, want 1; unsubscribed=%v", countString(unsubscribed, "loop-a"), unsubscribed)
+	}
+	wantEvents := mustJSONMarshal(t, []agentonvif.ONVIFEvents{{Key: "b", Type: "input", Value: "false", Timestamp: 2}})
+	if string(payload.eventsList) != string(wantEvents) {
+		t.Fatalf("payload.eventsList = %s, want %s", payload.eventsList, wantEvents)
+	}
+}
+
+func TestHeartbeatONVIFPayloadRetriesStaticFetchFailuresWithoutReconnect(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	device := newTestONVIFDevice()
+	camera := models.IPCamera{
+		ONVIFXAddr:    "http://camera/onvif",
+		ONVIFUsername: "operator",
+		ONVIFPassword: "secret",
+	}
+	var connectCalls, ptzConfigCalls, presetCalls, createCalls, eventCalls int
+
+	heartbeatConnectToONVIFDevice = func(*models.IPCamera) (*goonvif.Device, goonvifdevice.GetCapabilitiesResponse, error) {
+		connectCalls++
+		return device, goonvifdevice.GetCapabilitiesResponse{}, nil
+	}
+	heartbeatGetPTZConfigurationsFromDevice = func(*goonvif.Device) (goonvifptz.GetConfigurationsResponse, error) {
+		ptzConfigCalls++
+		return goonvifptz.GetConfigurationsResponse{}, nil
+	}
+	heartbeatGetPTZFunctionsFromDevice = func(goonvifptz.GetConfigurationsResponse) ([]string, bool, bool) {
+		return nil, true, true
+	}
+	heartbeatGetPresetsFromDevice = func(*goonvif.Device) ([]models.OnvifActionPreset, error) {
+		presetCalls++
+		if presetCalls == 1 {
+			return nil, errors.New("temporary preset failure")
+		}
+		return []models.OnvifActionPreset{{Name: "Lobby", Token: "1"}}, nil
+	}
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		createCalls++
+		switch createCalls {
+		case 1:
+			return "initial-1", nil
+		case 2:
+			return "loop", nil
+		case 3:
+			return "initial-2", nil
+		default:
+			t.Fatalf("unexpected create pull point call %d", createCalls)
+			return "", nil
+		}
+	}
+	heartbeatGetEventMessages = func(_ *goonvif.Device, pullPointAddress string) ([]agentonvif.ONVIFEvents, error) {
+		eventCalls++
+		switch pullPointAddress {
+		case "initial-1", "initial-2":
+			return []agentonvif.ONVIFEvents{{Key: "one", Type: "input", Value: "true", Timestamp: 1}}, nil
+		case "loop":
+			return []agentonvif.ONVIFEvents{{Key: "two", Type: "output", Value: "false", Timestamp: 2}}, nil
+		default:
+			t.Fatalf("unexpected pull point address %q", pullPointAddress)
+			return nil, nil
+		}
+	}
+	heartbeatUnsubscribePullPoint = func(*goonvif.Device, string) error { return nil }
+
+	state := newHeartbeatONVIFState()
+	payload := getHeartbeatONVIFPayload(camera, state)
+	if payload.presets != "false" {
+		t.Fatalf("payload.presets = %q, want false on transient preset failure", payload.presets)
+	}
+	payload = getHeartbeatONVIFPayload(camera, state)
+	if payload.presets != "true" {
+		t.Fatalf("payload.presets = %q, want true after retry", payload.presets)
+	}
+	if connectCalls != 1 {
+		t.Fatalf("connectCalls = %d, want 1", connectCalls)
+	}
+	if ptzConfigCalls != 2 || presetCalls != 2 {
+		t.Fatalf("static failures were not retried on heartbeat cadence: ptzConfig=%d presets=%d", ptzConfigCalls, presetCalls)
+	}
+	if createCalls != 3 || eventCalls != 3 {
+		t.Fatalf("unexpected event behavior during retry: create=%d events=%d", createCalls, eventCalls)
+	}
+}
+
+func TestHeartbeatONVIFStateReleasesSubscriptionWhenDisabled(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	device := newTestONVIFDevice()
+	state := newHeartbeatONVIFState()
+	state.cameraConfiguration = models.IPCamera{ONVIFXAddr: "http://camera/onvif"}
+	state.cameraKey = heartbeatONVIFCameraKey(state.cameraConfiguration)
+	state.device = device
+	state.loopPullPoint = "loop"
+
+	var unsubscribed []string
+	heartbeatUnsubscribePullPoint = func(gotDevice *goonvif.Device, pullPointAddress string) error {
+		if gotDevice != device {
+			t.Fatal("unsubscribe used a different ONVIF device")
+		}
+		unsubscribed = append(unsubscribed, pullPointAddress)
+		return nil
+	}
+
+	state.prepare(models.IPCamera{})
+
+	if len(unsubscribed) != 1 || unsubscribed[0] != "loop" {
+		t.Fatalf("unsubscribed = %v, want [loop]", unsubscribed)
+	}
+	if state.device != nil || state.loopPullPoint != "" {
+		t.Fatalf("disabled state retained device or pull point: %+v", state)
+	}
+}
+
+func TestHeartbeatONVIFPayloadDoesNotCreateSubscriptionsWhenConnectFails(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	var createCalls int
+
+	heartbeatConnectToONVIFDevice = func(*models.IPCamera) (*goonvif.Device, goonvifdevice.GetCapabilitiesResponse, error) {
+		return nil, goonvifdevice.GetCapabilitiesResponse{}, errors.New("connect failed")
+	}
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		createCalls++
+		return "unexpected", nil
+	}
+
+	payload := getHeartbeatONVIFPayload(models.IPCamera{
+		ONVIFXAddr:    "http://camera/onvif",
+		ONVIFUsername: "operator",
+		ONVIFPassword: "secret",
+	}, newHeartbeatONVIFState())
+
+	if createCalls != 0 {
+		t.Fatalf("createCalls = %d, want 0", createCalls)
+	}
+	assertDefaultHeartbeatONVIFPayload(t, payload)
+}
+
+func TestHeartbeatONVIFPayloadReconnectFailureInvalidatesCache(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	device1 := newTestONVIFDevice()
+	device2 := newTestONVIFDevice()
+	camera := models.IPCamera{
+		ONVIFXAddr:    "http://camera/onvif",
+		ONVIFUsername: "operator",
+		ONVIFPassword: "secret",
+	}
+	var connectCalls, ptzConfigCalls, presetCalls, createCalls int
+	var unsubscribed []string
+
+	heartbeatConnectToONVIFDevice = func(*models.IPCamera) (*goonvif.Device, goonvifdevice.GetCapabilitiesResponse, error) {
+		connectCalls++
+		switch connectCalls {
+		case 1:
+			return device1, goonvifdevice.GetCapabilitiesResponse{}, nil
+		case 2:
+			return device2, goonvifdevice.GetCapabilitiesResponse{}, nil
+		default:
+			t.Fatalf("unexpected connect call %d", connectCalls)
+			return nil, goonvifdevice.GetCapabilitiesResponse{}, nil
+		}
+	}
+	heartbeatGetPTZConfigurationsFromDevice = func(*goonvif.Device) (goonvifptz.GetConfigurationsResponse, error) {
+		ptzConfigCalls++
+		return goonvifptz.GetConfigurationsResponse{}, nil
+	}
+	heartbeatGetPTZFunctionsFromDevice = func(goonvifptz.GetConfigurationsResponse) ([]string, bool, bool) {
+		return nil, true, false
+	}
+	heartbeatGetPresetsFromDevice = func(*goonvif.Device) ([]models.OnvifActionPreset, error) {
+		presetCalls++
+		return []models.OnvifActionPreset{{Name: "Preset", Token: "1"}}, nil
+	}
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		createCalls++
+		switch createCalls {
+		case 1:
+			return "initial-1", nil
+		case 2:
+			return "loop-1", nil
+		case 3:
+			return "initial-2", nil
+		case 4:
+			return "initial-3", nil
+		case 5:
+			return "loop-3", nil
+		default:
+			t.Fatalf("unexpected create pull point call %d", createCalls)
+			return "", nil
+		}
+	}
+	heartbeatGetEventMessages = func(_ *goonvif.Device, pullPointAddress string) ([]agentonvif.ONVIFEvents, error) {
+		switch pullPointAddress {
+		case "initial-1":
+			return []agentonvif.ONVIFEvents{{Key: "before", Type: "input", Value: "true", Timestamp: 1}}, nil
+		case "initial-2":
+			return []agentonvif.ONVIFEvents{{Key: "during", Type: "input", Value: "true", Timestamp: 2}}, nil
+		case "loop-1":
+			return nil, errors.New("pull failed")
+		case "initial-3":
+			return []agentonvif.ONVIFEvents{{Key: "after", Type: "input", Value: "false", Timestamp: 2}}, nil
+		default:
+			t.Fatalf("unexpected pull point address %q", pullPointAddress)
+			return nil, nil
+		}
+	}
+	heartbeatUnsubscribePullPoint = func(_ *goonvif.Device, pullPointAddress string) error {
+		unsubscribed = append(unsubscribed, pullPointAddress)
+		return nil
+	}
+
+	state := newHeartbeatONVIFState()
+	_ = getHeartbeatONVIFPayload(camera, state)
+	payload := getHeartbeatONVIFPayload(camera, state)
+	wantEventsDuringFailure := mustJSONMarshal(t, []agentonvif.ONVIFEvents{{Key: "during", Type: "input", Value: "true", Timestamp: 2}})
+	if string(payload.eventsList) != string(wantEventsDuringFailure) {
+		t.Fatalf("payload.eventsList after operation failure = %s, want %s", payload.eventsList, wantEventsDuringFailure)
+	}
+	payload = getHeartbeatONVIFPayload(camera, state)
+
+	if ptzConfigCalls != 2 || presetCalls != 2 {
+		t.Fatalf("reconnect did not refresh static state: ptzConfig=%d presets=%d", ptzConfigCalls, presetCalls)
+	}
+	if connectCalls != 2 {
+		t.Fatalf("connectCalls = %d, want 2 after reconnect", connectCalls)
+	}
+	if createCalls != 5 {
+		t.Fatalf("reconnect did not recreate subscriptions: create=%d", createCalls)
+	}
+	if countString(unsubscribed, "loop-1") != 1 {
+		t.Fatalf("operation failure cleanup mismatch, unsubscribed=%v", unsubscribed)
+	}
+	wantEvents := mustJSONMarshal(t, []agentonvif.ONVIFEvents{{Key: "after", Type: "input", Value: "false", Timestamp: 2}})
+	if string(payload.eventsList) != string(wantEvents) {
+		t.Fatalf("payload.eventsList = %s, want %s", payload.eventsList, wantEvents)
+	}
+}
+
+func TestHeartbeatONVIFPayloadKeepsCachedConnectionWhenInitialStateFetchFails(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	device := newTestONVIFDevice()
+	camera := models.IPCamera{
+		ONVIFXAddr:    "http://camera/onvif",
+		ONVIFUsername: "operator",
+		ONVIFPassword: "secret",
+	}
+	loopEvents := []agentonvif.ONVIFEvents{{Key: "input-1", Type: "input", Value: "true", Timestamp: 1}}
+
+	state := newHeartbeatONVIFState()
+	state.cameraConfiguration = camera
+	state.cameraKey = heartbeatONVIFCameraKey(camera)
+	state.device = device
+	state.loopPullPoint = "loop"
+	state.staticLoaded = true
+	state.staticPayload.enabled = "true"
+
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		return "", errors.New("temporary initial-state failure")
+	}
+	heartbeatGetEventMessages = func(_ *goonvif.Device, pullPointAddress string) ([]agentonvif.ONVIFEvents, error) {
+		if pullPointAddress != "loop" {
+			t.Fatalf("unexpected pull point address %q", pullPointAddress)
+		}
+		return loopEvents, nil
+	}
+
+	payload := getHeartbeatONVIFPayload(camera, state)
+
+	wantEvents := mustJSONMarshal(t, loopEvents)
+	if string(payload.eventsList) != string(wantEvents) {
+		t.Fatalf("payload.eventsList = %s, want %s", payload.eventsList, wantEvents)
+	}
+	if state.device != device || state.loopPullPoint != "loop" || !state.staticLoaded {
+		t.Fatalf("temporary failure invalidated healthy cached state: %+v", state)
+	}
+}
+
+func restoreHeartbeatONVIFStubs(t *testing.T) {
+	t.Helper()
+
+	originalConnect := heartbeatConnectToONVIFDevice
+	originalCreate := heartbeatCreatePullPointSubscription
+	originalDigitalInputs := heartbeatGetDigitalInputs
+	originalEvents := heartbeatGetEventMessages
+	originalPresets := heartbeatGetPresetsFromDevice
+	originalPTZConfigurations := heartbeatGetPTZConfigurationsFromDevice
+	originalPTZFunctions := heartbeatGetPTZFunctionsFromDevice
+	originalRelayOutputs := heartbeatGetRelayOutputs
+	originalUnsubscribe := heartbeatUnsubscribePullPoint
+
+	t.Cleanup(func() {
+		heartbeatConnectToONVIFDevice = originalConnect
+		heartbeatCreatePullPointSubscription = originalCreate
+		heartbeatGetDigitalInputs = originalDigitalInputs
+		heartbeatGetEventMessages = originalEvents
+		heartbeatGetPresetsFromDevice = originalPresets
+		heartbeatGetPTZConfigurationsFromDevice = originalPTZConfigurations
+		heartbeatGetPTZFunctionsFromDevice = originalPTZFunctions
+		heartbeatGetRelayOutputs = originalRelayOutputs
+		heartbeatUnsubscribePullPoint = originalUnsubscribe
+	})
+}
+
+func mustJSONMarshal(t *testing.T, v interface{}) []byte {
+	t.Helper()
+
+	b, err := json.Marshal(v)
+	if err != nil {
+		t.Fatalf("json.Marshal() error = %v", err)
+	}
+	return b
+}
+
+func assertDefaultHeartbeatONVIFPayload(t *testing.T, payload heartbeatONVIFPayload) {
+	t.Helper()
+
+	defaultPayload := defaultHeartbeatONVIFPayload()
+	if payload.enabled != defaultPayload.enabled ||
+		payload.zoom != defaultPayload.zoom ||
+		payload.panTilt != defaultPayload.panTilt ||
+		payload.presets != defaultPayload.presets ||
+		string(payload.presetsList) != string(defaultPayload.presetsList) ||
+		string(payload.eventsList) != string(defaultPayload.eventsList) {
+		t.Fatalf("payload = %+v, want default payload", payload)
+	}
+}
+
+func newTestONVIFDevice() *goonvif.Device {
+	return &goonvif.Device{}
+}
+
+func countString(values []string, want string) int {
+	count := 0
+	for _, value := range values {
+		if value == want {
+			count++
+		}
+	}
+	return count
+}

machinery/src/cloud/dropbox.go

@@ -11,8 +11,8 @@ import (
 	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
 	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/users"
 	"github.com/gin-gonic/gin"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 // UploadDropbox uploads the file to your Dropbox account using the access token and directory.
@@ -30,13 +30,13 @@ func UploadDropbox(configuration *models.Configuration, fileName string) (bool,
 
 	if token == "" {
 		err := "UploadDropbox: Dropbox not properly configured"
-		log.Log.Info(err)
+		log.Info(err)
 		return false, true, errors.New(err)
 	}
 
 	// Upload to Dropbox
-	log.Log.Info("UploadDropbox: Uploading to Dropbox")
+	log.Info("UploadDropbox: Uploading to Dropbox")
-	log.Log.Info("UploadDropbox: Upload started for " + fileName)
+	log.Info("UploadDropbox: Upload started for " + fileName)
 	fullname := "data/recordings/" + fileName
 
 	dConfig := dropbox.Config{
@@ -48,7 +48,7 @@ func UploadDropbox(configuration *models.Configuration, fileName string) (bool,
 	if file != nil {
 		defer func() {
 			if cerr := file.Close(); cerr != nil {
-				log.Log.Error("UploadDropbox: Error closing file: " + cerr.Error())
+				log.Error("UploadDropbox: Error closing file: " + cerr.Error())
 			}
 		}()
 	}
@@ -68,15 +68,15 @@ func UploadDropbox(configuration *models.Configuration, fileName string) (bool,
 		}, file)
 
 		if err != nil {
-			log.Log.Error("UploadDropbox: Error uploading file: " + err.Error())
+			log.Error("UploadDropbox: Error uploading file: " + err.Error())
 			return false, false, err
 		}
 
-		log.Log.Info("UploadDropbox: File uploaded successfully, " + res.Name)
+		log.Info("UploadDropbox: File uploaded successfully, " + res.Name)
 		return true, true, nil
 	}
 
-	log.Log.Error("UploadDropbox: Error opening file: " + err.Error())
+	log.Error("UploadDropbox: Error opening file: " + err.Error())
 	return false, true, err
 }
 

machinery/src/cloud/kerberos_hub.go

@@ -3,12 +3,13 @@ package cloud
 import (
 	"crypto/tls"
 	"errors"
+	"fmt"
 	"io/ioutil"
 	"net/http"
 	"os"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 func UploadKerberosHub(configuration *models.Configuration, fileName string) (bool, bool, error) {
@@ -19,7 +20,7 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 		config.HubPrivateKey == "" ||
 		config.S3.Region == "" {
 		err := "UploadKerberosHub: Kerberos Hub not properly configured."
-		log.Log.Info(err)
+		log.Info(err)
 		return false, false, errors.New(err)
 	}
 
@@ -32,8 +33,10 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 	// - Number of changes
 	// - Token
 
-	log.Log.Info("UploadKerberosHub: Uploading to Kerberos Hub (" + config.HubURI + ")")
+	log.WithFields(log.Fields{
-	log.Log.Info("UploadKerberosHub: Upload started for " + fileName)
+		"component": "kerberos_hub",
+		"event":     "upload_started",
+	}).Info("Uploading recording to Kerberos Hub")
 
 	// Prefer the resumable (tus) upload when enabled (the default). Kerberos Hub
 	// authenticates the agent with its Hub public/private key and proxies the
@@ -44,17 +47,35 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 		uploaded, _, supported, body, rerr := uploadHubResumable(&config, fileName, "UploadKerberosHub", "hub")
 		if supported {
 			if uploaded {
-				log.Log.Info("UploadKerberosHub: Upload Finished (resumable), " + body)
+				log.WithFields(log.Fields{
+					"component":      "kerberos_hub",
+					"event":          "upload_completed",
+					"response_bytes": len(body),
+					"transport":      "tus",
+				}).Info("Hub upload completed")
 				return true, true, nil
 			}
 			if rerr != nil {
-				log.Log.Info("UploadKerberosHub: resumable upload failed, " + rerr.Error())
+				log.WithError(rerr).WithFields(log.Fields{
+					"component": "kerberos_hub",
+					"event":     "upload_failed",
+					"transport": "tus",
+				}).Error("Hub upload failed")
 			} else {
-				log.Log.Info("UploadKerberosHub: resumable upload incomplete, " + body)
+				log.WithFields(log.Fields{
+					"component":      "kerberos_hub",
+					"event":          "upload_incomplete",
+					"response_bytes": len(body),
+					"transport":      "tus",
+				}).Warn("Hub upload incomplete")
 			}
 			return false, true, rerr
 		}
-		log.Log.Info("UploadKerberosHub: resumable (tus) endpoint not available, falling back to legacy upload")
+		log.WithFields(log.Fields{
+			"component": "kerberos_hub",
+			"event":     "upload_transport_fallback",
+			"transport": "http",
+		}).Info("Resumable Hub upload unavailable; using legacy upload")
 	}
 
 	fullname := "data/recordings/" + fileName
@@ -66,7 +87,7 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 	}
 	if err != nil {
 		err := "UploadKerberosHub: Upload Failed, file doesn't exists anymore."
-		log.Log.Info(err)
+		log.Info(err)
 		return false, false, errors.New(err)
 	}
 
@@ -74,9 +95,11 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 	// There might be different reasons like (muted, read-only..)
 	req, err := http.NewRequest("HEAD", config.HubURI+"/storage/upload", nil)
 	if err != nil {
-		errorMessage := "UploadKerberosHub: error reading HEAD request, " + config.HubURI + "/storage: " + err.Error()
+		log.WithError(err).WithFields(log.Fields{
-		log.Log.Error(errorMessage)
+			"component": "kerberos_hub",
-		return false, true, errors.New(errorMessage)
+			"event":     "authorization_request_creation_failed",
+		}).Error("Failed to create Hub upload authorization request")
+		return false, true, fmt.Errorf("create Hub upload authorization request: %w", err)
 	}
 
 	req.Header.Set("X-Kerberos-Storage-FileName", fileName)
@@ -102,25 +125,31 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 		defer resp.Body.Close()
 	}
 
-	if err == nil {
+	if err == nil && resp != nil {
-		if resp != nil {
+		if resp.StatusCode == 200 {
-			if err == nil {
+			log.WithFields(log.Fields{
-				if resp.StatusCode == 200 {
+				"component":   "kerberos_hub",
-					log.Log.Info("UploadKerberosHub: Upload allowed using the credentials provided (" + config.HubKey + ", " + config.HubPrivateKey + ")")
+				"event":       "upload_authorized",
-				} else {
+				"status_code": resp.StatusCode,
-					log.Log.Info("UploadKerberosHub: Upload NOT allowed using the credentials provided (" + config.HubKey + ", " + config.HubPrivateKey + ")")
+			}).Debug("Hub upload authorized")
-					return false, true, nil
+		} else {
-				}
+			log.WithFields(log.Fields{
-			}
+				"component":   "kerberos_hub",
+				"event":       "upload_rejected",
+				"status_code": resp.StatusCode,
+			}).Warn("Hub upload rejected")
+			return false, true, nil
 		}
 	}
 
 	// Now we know we are allowed to upload to the hub, we can start uploading.
 	req, err = http.NewRequest("POST", config.HubURI+"/storage/upload", file)
 	if err != nil {
-		errorMessage := "UploadKerberosHub: error reading POST request, " + config.KStorage.URI + "/storage/upload: " + err.Error()
+		log.WithError(err).WithFields(log.Fields{
-		log.Log.Error(errorMessage)
+			"component": "kerberos_hub",
-		return false, true, errors.New(errorMessage)
+			"event":     "upload_request_creation_failed",
+		}).Error("Failed to create Hub upload request")
+		return false, true, fmt.Errorf("create Hub upload request: %w", err)
 	}
 	req.Header.Set("Content-Type", "video/mp4")
 	req.Header.Set("X-Kerberos-Storage-FileName", fileName)
@@ -140,19 +169,36 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 			body, err := ioutil.ReadAll(resp.Body)
 			if err == nil {
 				if resp.StatusCode == 200 {
-					log.Log.Info("UploadKerberosHub: Upload Finished, " + resp.Status + ".")
+					log.WithFields(log.Fields{
+						"component":   "kerberos_hub",
+						"event":       "upload_completed",
+						"status_code": resp.StatusCode,
+						"transport":   "http",
+					}).Info("Hub upload completed")
 					return true, true, nil
 				} else {
-					log.Log.Info("UploadKerberosHub: Upload Failed, " + resp.Status + ", " + string(body))
+					log.WithFields(log.Fields{
+						"component":      "kerberos_hub",
+						"event":          "upload_rejected",
+						"response_bytes": len(body),
+						"status_code":    resp.StatusCode,
+						"transport":      "http",
+					}).Warn("Hub upload rejected")
 					return false, true, nil
 				}
 			}
 		}
 	}
 
-	errorMessage := "UploadKerberosHub: Upload Failed, " + err.Error()
+	if err == nil {
-	log.Log.Info(errorMessage)
+		err = errors.New("Hub upload failed without a response")
-	return false, true, errors.New(errorMessage)
+	}
+	log.WithError(err).WithFields(log.Fields{
+		"component": "kerberos_hub",
+		"event":     "upload_failed",
+		"transport": "http",
+	}).Error("Hub upload failed")
+	return false, true, fmt.Errorf("Hub upload failed: %w", err)
 }
 
 // stripHubCredentialsOnCrossHostRedirect removes the custom Kerberos Hub

machinery/src/cloud/kerberos_vault.go

@@ -9,8 +9,8 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 // We will count the number of retries we have done.
@@ -27,7 +27,7 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string) (
 		config.KStorage.Directory == "" ||
 		config.KStorage.URI == "" {
 		err := "UploadKerberosVault: Kerberos Vault not properly configured"
-		log.Log.Info(err)
+		log.Info(err)
 		return false, false, errors.New(err)
 	}
 
@@ -35,8 +35,13 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string) (
 	// This can happen when the file was already removed (e.g. cleanup, or an
 	// earlier successful upload). Skip it so the watcher drops the marker
 	// instead of retrying indefinitely.
-	if _, err := os.Stat("data/recordings/" + fileName); err != nil {
+	info, err := os.Stat("data/recordings/" + fileName)
-		log.Log.Info("UploadKerberosVault: skipping " + fileName + ", file doesn't exist anymore")
+	if err != nil {
+		log.Info("UploadKerberosVault: skipping " + fileName + ", file doesn't exist anymore")
+		return false, false, nil
+	}
+	if info.Size() == 0 {
+		log.Warn("UploadKerberosVault: skipping " + fileName + ", recording is empty")
 		return false, false, nil
 	}
 
@@ -49,8 +54,11 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string) (
 	// - Number of changes
 	// - Token
 	// KerberosCloud, this means storage is disabled and proxy enabled.
-	log.Log.Info("UploadKerberosVault: Uploading to Kerberos Vault (" + config.KStorage.URI + ")")
+	log.WithFields(log.Fields{
-	log.Log.Info("UploadKerberosVault: Upload started for " + fileName)
+		"component": "kerberos_vault",
+		"event":     "upload_started",
+		"storage":   "primary",
+	}).Info("Vault upload started")
 
 	publicKey := config.KStorage.CloudKey
 	if config.HubKey != "" {
@@ -62,14 +70,27 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string) (
 		uploaded, responded, body, err := sendToVault(*config.KStorage, publicKey, config.Key, fileName, "UploadKerberosVault", "primary")
 		if uploaded {
 			kstorageRetryCount = 0
-			log.Log.Info("UploadKerberosVault: Upload Finished, " + body)
+			log.WithFields(log.Fields{
+				"component": "kerberos_vault",
+				"event":     "upload_completed",
+				"storage":   "primary",
+			}).Info("Vault upload completed")
 			return true, true, nil
 		}
 
 		if err != nil {
-			log.Log.Info("UploadKerberosVault: Upload Failed, " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component": "kerberos_vault",
+				"event":     "upload_failed",
+				"storage":   "primary",
+			}).Error("Vault upload failed")
 		} else {
-			log.Log.Info("UploadKerberosVault: Upload Failed, " + body)
+			log.WithFields(log.Fields{
+				"component":      "kerberos_vault",
+				"event":          "upload_rejected",
+				"response_bytes": len(body),
+				"storage":        "primary",
+			}).Warn("Vault upload rejected")
 		}
 
 		// We only advance the retry policy when the vault gave a definitive
@@ -91,26 +112,43 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string) (
 		config.KStorageSecondary.SecretAccessKey == "" ||
 		config.KStorageSecondary.Directory == "" ||
 		config.KStorageSecondary.URI == "" {
-		log.Log.Info("UploadKerberosVault (Secondary): Secondary Kerberos Vault not properly configured.")
+		log.Info("UploadKerberosVault (Secondary): Secondary Kerberos Vault not properly configured.")
 	} else {
 
 		if kstorageRetryCount < config.KStorage.MaxRetries {
-			log.Log.Info("UploadKerberosVault (Secondary): Do not upload to secondary storage, we are still in retry policy.")
+			log.Info("UploadKerberosVault (Secondary): Do not upload to secondary storage, we are still in retry policy.")
 			return false, true, nil
 		}
 
-		log.Log.Info("UploadKerberosVault (Secondary): Uploading to Secondary Kerberos Vault (" + config.KStorageSecondary.URI + ")")
+		log.WithFields(log.Fields{
+			"component": "kerberos_vault",
+			"event":     "upload_started",
+			"storage":   "secondary",
+		}).Info("Vault upload started")
 
 		uploaded, _, body, err := sendToVault(*config.KStorageSecondary, publicKey, config.Key, fileName, "UploadKerberosVault (Secondary)", "secondary")
 		if uploaded {
-			log.Log.Info("UploadKerberosVault (Secondary): Upload Finished to secondary, " + body)
+			log.WithFields(log.Fields{
+				"component": "kerberos_vault",
+				"event":     "upload_completed",
+				"storage":   "secondary",
+			}).Info("Vault upload completed")
 			return true, true, nil
 		}
 
 		if err != nil {
-			log.Log.Info("UploadKerberosVault (Secondary): Upload Failed to secondary, " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component": "kerberos_vault",
+				"event":     "upload_failed",
+				"storage":   "secondary",
+			}).Error("Vault upload failed")
 		} else {
-			log.Log.Info("UploadKerberosVault (Secondary): Upload Failed to secondary, " + body)
+			log.WithFields(log.Fields{
+				"component":      "kerberos_vault",
+				"event":          "upload_rejected",
+				"response_bytes": len(body),
+				"storage":        "secondary",
+			}).Warn("Vault upload rejected")
 		}
 	}
 
@@ -131,7 +169,7 @@ func sendToVault(vault models.KStorage, publicKey, deviceKey, fileName, label, s
 		if supported {
 			return uploaded, responded, body, err
 		}
-		log.Log.Info(label + ": resumable (tus) endpoint not available, falling back to legacy upload")
+		log.Info(label + ": resumable (tus) endpoint not available, falling back to legacy upload")
 	}
 	return uploadVaultLegacy(vault, publicKey, deviceKey, fileName, label)
 }
@@ -148,7 +186,7 @@ func uploadVaultLegacy(vault models.KStorage, publicKey, deviceKey, fileName, la
 	}
 	if err != nil {
 		msg := label + ": Upload Failed, file doesn't exists anymore"
-		log.Log.Info(msg)
+		log.Info(msg)
 		return false, false, "", errors.New(msg)
 	}
 
@@ -160,7 +198,7 @@ func uploadVaultLegacy(vault models.KStorage, publicKey, deviceKey, fileName, la
 	req, err := http.NewRequest("POST", uri+"/storage", file)
 	if err != nil {
 		errorMessage := label + ": error reading request, " + uri + "/storage: " + err.Error()
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, false, "", errors.New(errorMessage)
 	}
 	req.Header.Set("Content-Type", "video/mp4")

machinery/src/cloud/livehls.go

@@ -7,9 +7,9 @@ import (
 	mqtt "github.com/eclipse/paho.mqtt.golang"
 
 	"github.com/kerberos-io/agent/machinery/src/cloud/livehls"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
+	log "github.com/sirupsen/logrus"
 )
 
 // hlsViewerTimeoutSeconds is how long the agent keeps shipping live HLS segments
@@ -45,22 +45,29 @@ const hlsReadyReannounceSeconds = 2
 // trades a little idle CPU for a near-instant "requesting stream", so viewers no
 // longer wait a full GOP for the first segment to be cut. Set
 // AGENT_LIVE_HLS_PREWARM=false to fall back to the lazy on-demand path above.
-func HandleLiveStreamHLS(configuration *models.Configuration, communication *models.Communication, mqttClient mqtt.Client, subStreamEnabled bool) {
+func HandleLiveStreamHLS(
+	configuration *models.Configuration,
+	communication *models.Communication,
+	mqttClient mqtt.Client,
+	subStreamEnabled bool,
+	mainQueue *packets.Queue,
+	subQueue *packets.Queue,
+) {
 
-	log.Log.Debug("cloud.HandleLiveStreamHLS(): started")
+	log.Debug("cloud.HandleLiveStreamHLS(): started")
 
 	config := configuration.Config
 
 	if config.Offline == "true" {
-		log.Log.Debug("cloud.HandleLiveStreamHLS(): stopping as Offline is enabled.")
+		log.Debug("cloud.HandleLiveStreamHLS(): stopping as Offline is enabled.")
 		return
 	}
 	if config.Capture.Liveview == "false" {
-		log.Log.Debug("cloud.HandleLiveStreamHLS(): stopping as Liveview is disabled.")
+		log.Debug("cloud.HandleLiveStreamHLS(): stopping as Liveview is disabled.")
 		return
 	}
 	if config.HubURI == "" || config.HubKey == "" {
-		log.Log.Debug("cloud.HandleLiveStreamHLS(): stopping as the Hub is not configured (HubURI/HubKey).")
+		log.Debug("cloud.HandleLiveStreamHLS(): stopping as the Hub is not configured (HubURI/HubKey).")
 		return
 	}
 
@@ -88,8 +95,8 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 	// mp4ff's strict parser rejects).
 	requestedQuality := models.StreamQualityAuto
 	useSub := models.SelectSubStreamForQuality(config, requestedQuality, subStreamEnabled)
-	source := buildHLSSource(config, communication, useSub)
+	source := buildHLSSource(config, mainQueue, subQueue, useSub)
-	log.Log.Info("cloud.HandleLiveStreamHLS(): serving live HLS from the " + source.label + " stream")
+	log.Info("cloud.HandleLiveStreamHLS(): serving live HLS from the " + source.label + " stream")
 
 	// prewarm keeps a single long-lived session muxing into an in-memory ring
 	// buffer while idle and flushes it the instant a viewer arrives, eliminating
@@ -97,9 +104,9 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 	// to fall back to the lazy on-demand path.
 	prewarm := os.Getenv("AGENT_LIVE_HLS_PREWARM") != "false"
 	if prewarm {
-		log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS prewarm ENABLED (set AGENT_LIVE_HLS_PREWARM=false to disable)")
+		log.Info("cloud.HandleLiveStreamHLS(): live HLS prewarm ENABLED (set AGENT_LIVE_HLS_PREWARM=false to disable)")
 	} else {
-		log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS prewarm DISABLED (AGENT_LIVE_HLS_PREWARM=false)")
+		log.Info("cloud.HandleLiveStreamHLS(): live HLS prewarm DISABLED (AGENT_LIVE_HLS_PREWARM=false)")
 	}
 
 	// lowLatency enables LL-HLS: each segment is sliced into CMAF parts shipped the
@@ -109,9 +116,9 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 	partTargetMs := uint64(0)
 	if os.Getenv("AGENT_LIVE_HLS_LOW_LATENCY") != "false" {
 		partTargetMs = livehls.DefaultPartTargetMs
-		log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS low-latency (LL-HLS) ENABLED (set AGENT_LIVE_HLS_LOW_LATENCY=false to disable)")
+		log.Info("cloud.HandleLiveStreamHLS(): live HLS low-latency (LL-HLS) ENABLED (set AGENT_LIVE_HLS_LOW_LATENCY=false to disable)")
 	} else {
-		log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS low-latency (LL-HLS) DISABLED (AGENT_LIVE_HLS_LOW_LATENCY=false)")
+		log.Info("cloud.HandleLiveStreamHLS(): live HLS low-latency (LL-HLS) DISABLED (AGENT_LIVE_HLS_LOW_LATENCY=false)")
 	}
 
 	var session *livehls.Session
@@ -123,6 +130,9 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 
 	for cursorError == nil {
 		pkt, cursorError = source.cursor.ReadPacket()
+		if cursorError != nil {
+			break
+		}
 
 		now := time.Now().Unix()
 		select {
@@ -156,9 +166,9 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 				_ = session.Close()
 				session = nil
 			}
-			source = buildHLSSource(config, communication, useSub)
+			source = buildHLSSource(config, mainQueue, subQueue, useSub)
 			lastReadyAnnounce = 0
-			log.Log.Info("cloud.HandleLiveStreamHLS(): switched live HLS to the " + source.label + " stream (quality=" + requestedQuality + ")")
+			log.Info("cloud.HandleLiveStreamHLS(): switched live HLS to the " + source.label + " stream (quality=" + requestedQuality + ")")
 			continue
 		}
 
@@ -184,11 +194,22 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 					StartBuffering: true,
 				})
 				session.SetOnReady(func(sessionID string) {
-					log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS session ready, announcing " + sessionID)
+					log.WithFields(log.Fields{
+						"component":  "cloud/livehls",
+						"event":      "session_ready",
+						"session_id": sessionID,
+					}).Info("Live HLS session ready")
 					publishHLSReady(configuration, mqttClient, hubKey, deviceId, sessionID)
 					lastReadyAnnounce = time.Now().Unix()
 				})
-				log.Log.Info("cloud.HandleLiveStreamHLS(): prewarming live HLS session " + session.SessionID())
+				session.SetOnFailure(func(sessionID, reason string) {
+					publishHLSFailure(configuration, mqttClient, hubKey, deviceId, sessionID, reason)
+				})
+				log.WithFields(log.Fields{
+					"component":  "cloud/livehls",
+					"event":      "session_prewarming",
+					"session_id": session.SessionID(),
+				}).Info("Prewarming live HLS session")
 			}
 
 			if viewerActive {
@@ -207,7 +228,7 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 
 			if len(pkt.Data) > 0 && pkt.IsVideo {
 				if err := session.WritePacket(pkt); err != nil {
-					log.Log.Error("cloud.HandleLiveStreamHLS(): " + err.Error())
+					log.Error("cloud.HandleLiveStreamHLS(): " + err.Error())
 				}
 			}
 			continue
@@ -217,7 +238,12 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 			// No viewer: stop and discard the session so we stop shipping segments.
 			if session != nil {
 				_ = session.Close()
-				log.Log.Info("cloud.HandleLiveStreamHLS(): no active viewers, stopped live HLS session " + session.SessionID())
+				log.WithFields(log.Fields{
+					"component":  "cloud/livehls",
+					"event":      "session_stopped",
+					"reason":     "no_active_viewers",
+					"session_id": session.SessionID(),
+				}).Info("Live HLS session stopped")
 				session = nil
 			}
 			continue
@@ -243,22 +269,33 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 				PartTargetMs: partTargetMs,
 			})
 			session.SetOnReady(func(sessionID string) {
-				log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS session ready, announcing " + sessionID)
+				log.WithFields(log.Fields{
+					"component":  "cloud/livehls",
+					"event":      "session_ready",
+					"session_id": sessionID,
+				}).Info("Live HLS session ready")
 				publishHLSReady(configuration, mqttClient, hubKey, deviceId, sessionID)
 				lastReadyAnnounce = time.Now().Unix()
 			})
-			log.Log.Info("cloud.HandleLiveStreamHLS(): started live HLS session " + session.SessionID())
+			session.SetOnFailure(func(sessionID, reason string) {
+				publishHLSFailure(configuration, mqttClient, hubKey, deviceId, sessionID, reason)
+			})
+			log.WithFields(log.Fields{
+				"component":  "cloud/livehls",
+				"event":      "session_started",
+				"session_id": session.SessionID(),
+			}).Info("Live HLS session started")
 		}
 
 		if err := session.WritePacket(pkt); err != nil {
-			log.Log.Error("cloud.HandleLiveStreamHLS(): " + err.Error())
+			log.Error("cloud.HandleLiveStreamHLS(): " + err.Error())
 		}
 	}
 
 	if session != nil {
 		_ = session.Close()
 	}
-	log.Log.Debug("cloud.HandleLiveStreamHLS(): finished")
+	log.Debug("cloud.HandleLiveStreamHLS(): finished")
 }
 
 // publishHLSReady announces, over MQTT, that a live HLS session is available so
@@ -278,9 +315,48 @@ func publishHLSReady(configuration *models.Configuration, mqttClient mqtt.Client
 	payload, err := models.PackageMQTTMessage(configuration, message)
 	if err == nil {
 		mqttClient.Publish("kerberos/hub/"+hubKey, 0, false, payload)
-		log.Log.Info("cloud.HandleLiveStreamHLS(): announced live HLS session " + sessionID)
+		log.WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "session_announced",
+			"session_id": sessionID,
+		}).Info("Live HLS session announced")
 	} else {
-		log.Log.Error("cloud.HandleLiveStreamHLS(): failed to package receive-hls-ready message: " + err.Error())
+		log.WithError(err).WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "session_announcement_failed",
+			"session_id": sessionID,
+		}).Error("Failed to package live HLS session announcement")
+	}
+}
+
+func publishHLSFailure(configuration *models.Configuration, mqttClient mqtt.Client, hubKey, deviceId, sessionID, reason string) {
+	valueMap := map[string]interface{}{
+		"session": sessionID,
+		"device":  deviceId,
+		"reason":  reason,
+	}
+	message := models.Message{
+		Payload: models.Payload{
+			Action:   "receive-hls-error",
+			DeviceId: deviceId,
+			Value:    valueMap,
+		},
+	}
+	payload, err := models.PackageMQTTMessage(configuration, message)
+	if err == nil {
+		mqttClient.Publish("kerberos/hub/"+hubKey, 0, false, payload)
+		log.WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "session_failure_announced",
+			"reason":     reason,
+			"session_id": sessionID,
+		}).Warn("Live HLS session failure announced")
+	} else {
+		log.WithError(err).WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "session_failure_announcement_failed",
+			"session_id": sessionID,
+		}).Error("Failed to package live HLS session failure")
 	}
 }
 
@@ -302,11 +378,11 @@ type hlsStreamSource struct {
 // is available; otherwise the main (high-resolution) stream is used. A fresh
 // Latest() cursor is created so muxing resumes from the live edge of the chosen
 // stream after a switch.
-func buildHLSSource(config models.Config, communication *models.Communication, useSub bool) hlsStreamSource {
+func buildHLSSource(config models.Config, mainQueue, subQueue *packets.Queue, useSub bool) hlsStreamSource {
 	cam := config.Capture.IPCamera
-	if useSub && communication.SubQueue != nil {
+	if useSub && subQueue != nil {
 		return hlsStreamSource{
-			cursor: communication.SubQueue.Latest(),
+			cursor: subQueue.Latest(),
 			sps:    cam.SubSPSNALUs,
 			pps:    cam.SubPPSNALUs,
 			vps:    cam.SubVPSNALUs,
@@ -316,7 +392,7 @@ func buildHLSSource(config models.Config, communication *models.Communication, u
 		}
 	}
 	return hlsStreamSource{
-		cursor: communication.Queue.Latest(),
+		cursor: mainQueue.Latest(),
 		sps:    cam.SPSNALUs,
 		pps:    cam.PPSNALUs,
 		vps:    cam.VPSNALUs,

machinery/src/cloud/livehls/publisher.go

@@ -24,13 +24,14 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"net/http"
 	"strconv"
 	"strings"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/video"
+	log "github.com/sirupsen/logrus"
 )
 
 const (
@@ -67,6 +68,7 @@ const (
 	// cannot be delivered within roughly its own duration is stale, so the upload
 	// is abandoned (dropped) rather than allowed to back up the pipeline.
 	defaultPublishTimeout = 4 * time.Second
+	maxErrorResponseBytes = 4 << 10
 )
 
 // PublisherConfig carries the hub endpoint and credentials needed to ship live
@@ -214,9 +216,15 @@ func (p *Publisher) post(ctx context.Context, params postParams) error {
 	defer resp.Body.Close()
 
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		return fmt.Errorf("livehls: upload %s rejected: %s", params.name, resp.Status)
+		body, _ := io.ReadAll(io.LimitReader(resp.Body, maxErrorResponseBytes))
+		return fmt.Errorf("livehls: upload %s rejected with status %d (%d response bytes)", params.name, resp.StatusCode, len(body))
 	}
-	log.Log.Debug("livehls.Publisher.post(): shipped " + params.name + " for session " + params.sessionID)
+	log.WithFields(log.Fields{
+		"component":  "cloud/livehls",
+		"event":      "object_uploaded",
+		"object":     params.name,
+		"session_id": params.sessionID,
+	}).Trace("Live HLS object uploaded")
 	return nil
 }
 

machinery/src/cloud/livehls/publisher_test.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"sync"
 	"testing"
 	"time"
@@ -144,12 +145,22 @@ func TestPublisherPublishSegmentSendsSequenceAndDuration(t *testing.T) {
 }
 
 func TestPublisherReturnsErrorOnNon2xx(t *testing.T) {
-	srv, _, _ := newCapturingServer(t, http.StatusInternalServerError)
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusBadRequest)
+		_, _ = w.Write([]byte(`{"error":true,"data":"No user found with this public and private key."}`))
+	}))
+	t.Cleanup(srv.Close)
 	p := testPublisher(srv.URL)
 
 	err := p.PublishSegment(context.Background(), "s", video.LiveSegment{SequenceNumber: 1, Data: []byte("x")})
 	if err == nil {
-		t.Fatal("expected an error on 500 response")
+		t.Fatal("expected an error on 400 response")
+	}
+	if !strings.Contains(err.Error(), "status 400") || !strings.Contains(err.Error(), "71 response bytes") {
+		t.Fatalf("error = %q, want status code and response size", err)
+	}
+	if strings.Contains(err.Error(), "No user found with this public and private key") {
+		t.Fatalf("error exposed Hub response body: %q", err)
 	}
 }
 
@@ -281,7 +292,13 @@ func TestSessionRetriesInitWhenFirstAttemptFails(t *testing.T) {
 	})
 
 	var ready int
+	var failures int
+	var failureReason string
 	sess.SetOnReady(func(string) { ready++ })
+	sess.SetOnFailure(func(_ string, reason string) {
+		failures++
+		failureReason = reason
+	})
 
 	for i := 0; i < 60; i++ {
 		isKey := i%25 == 0
@@ -304,6 +321,9 @@ func TestSessionRetriesInitWhenFirstAttemptFails(t *testing.T) {
 	if ready != 1 {
 		t.Errorf("OnReady fired %d times, want 1", ready)
 	}
+	if failures != 1 || failureReason != "init-upload-failed" {
+		t.Errorf("OnFailure = %d/%q, want 1/init-upload-failed", failures, failureReason)
+	}
 }
 
 // liveTestSPSForSession is the known-good baseline SPS reused across tests.

machinery/src/cloud/livehls/session.go

@@ -8,9 +8,9 @@ import (
 	"sync"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/kerberos-io/agent/machinery/src/video"
+	log "github.com/sirupsen/logrus"
 )
 
 // DefaultTargetSegmentMs is the nominal live segment length. ~2s keeps standard
@@ -46,9 +46,11 @@ type Session struct {
 	// lastInitAt is when the init segment was last (re)uploaded. The init is
 	// re-sent periodically so its short TTL in the hub live window never lapses
 	// mid-session; see refreshInitIfStale.
-	lastInitAt time.Time
+	lastInitAt   time.Time
-	readyFired bool
+	readyFired   bool
-	onReady    func(sessionID string)
+	onReady      func(sessionID string)
+	failureFired bool
+	onFailure    func(sessionID, reason string)
 
 	// uploadsActive gates whether the init and completed segments are shipped to
 	// hub-api. It is true for the default on-demand path. The prewarm path starts
@@ -142,14 +144,25 @@ func NewSession(publisher *Publisher, opts SessionOptions) *Session {
 			return nil
 		}
 		if !s.publishInitIfNeeded() {
-			log.Log.Warning("livehls.Session: dropping segment " +
+			log.WithFields(log.Fields{
-				fmt.Sprintf("%d", segment.SequenceNumber) + " because init has not been delivered yet")
+				"component":        "cloud/livehls",
+				"event":            "segment_dropped",
+				"reason":           "init_unavailable",
+				"segment_sequence": segment.SequenceNumber,
+				"session_id":       s.id,
+			}).Warn("Dropping live HLS segment")
 			return nil
 		}
 		ctx, cancel := s.newContext()
 		defer cancel()
 		if err := s.publisher.PublishSegment(ctx, s.id, segment); err != nil {
-			log.Log.Warning("livehls.Session: " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component":        "cloud/livehls",
+				"event":            "segment_upload_failed",
+				"segment_sequence": segment.SequenceNumber,
+				"session_id":       s.id,
+			}).Warn("Failed to upload live HLS segment")
+			s.fireFailureOnce("segment-upload-failed")
 			return nil
 		}
 		s.fireReadyOnce()
@@ -172,15 +185,27 @@ func NewSession(publisher *Publisher, opts SessionOptions) *Session {
 				return nil
 			}
 			if !s.publishInitIfNeeded() {
-				log.Log.Warning("livehls.Session: dropping part " +
+				log.WithFields(log.Fields{
-					fmt.Sprintf("%d.%d", part.SegmentSeq, part.PartIndex) +
+					"component":        "cloud/livehls",
-					" because init has not been delivered yet")
+					"event":            "part_dropped",
+					"part_index":       part.PartIndex,
+					"reason":           "init_unavailable",
+					"segment_sequence": part.SegmentSeq,
+					"session_id":       s.id,
+				}).Warn("Dropping live HLS part")
 				return nil
 			}
 			ctx, cancel := s.newContext()
 			defer cancel()
 			if err := s.publisher.PublishPart(ctx, s.id, part); err != nil {
-				log.Log.Warning("livehls.Session: " + err.Error())
+				log.WithError(err).WithFields(log.Fields{
+					"component":        "cloud/livehls",
+					"event":            "part_upload_failed",
+					"part_index":       part.PartIndex,
+					"segment_sequence": part.SegmentSeq,
+					"session_id":       s.id,
+				}).Warn("Failed to upload live HLS part")
+				s.fireFailureOnce("part-upload-failed")
 				return nil
 			}
 			s.fireReadyOnce()
@@ -216,6 +241,15 @@ func (s *Session) SetOnReady(fn func(sessionID string)) {
 	s.mu.Unlock()
 }
 
+// SetOnFailure registers a one-shot callback for startup upload failures. The
+// reason is a fixed code rather than an HTTP response body, so credentials or
+// server details cannot leak through MQTT diagnostics.
+func (s *Session) SetOnFailure(fn func(sessionID, reason string)) {
+	s.mu.Lock()
+	s.onFailure = fn
+	s.mu.Unlock()
+}
+
 // prewarmMaxBufferedSegments is how many of the most recent completed segments
 // the prewarm path keeps in memory while idle and flushes to a viewer on arrival.
 // One segment keeps startup instant (the viewer immediately gets a playable
@@ -268,7 +302,13 @@ func (s *Session) SetUploadsActive(active bool) bool {
 		}
 		ctx, cancel := s.newContext()
 		if err := s.publisher.PublishSegment(ctx, s.id, buffered[i]); err != nil {
-			log.Log.Warning("livehls.Session: prewarm flush: " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component":        "cloud/livehls",
+				"event":            "prewarm_segment_upload_failed",
+				"segment_sequence": buffered[i].SequenceNumber,
+				"session_id":       s.id,
+			}).Warn("Failed to upload buffered live HLS segment")
+			s.fireFailureOnce("segment-upload-failed")
 			cancel()
 			continue
 		}
@@ -284,7 +324,14 @@ func (s *Session) SetUploadsActive(active bool) bool {
 		}
 		ctx, cancel := s.newContext()
 		if err := s.publisher.PublishPart(ctx, s.id, bufferedParts[i]); err != nil {
-			log.Log.Warning("livehls.Session: prewarm flush (part): " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component":        "cloud/livehls",
+				"event":            "prewarm_part_upload_failed",
+				"part_index":       bufferedParts[i].PartIndex,
+				"segment_sequence": bufferedParts[i].SegmentSeq,
+				"session_id":       s.id,
+			}).Warn("Failed to upload buffered live HLS part")
+			s.fireFailureOnce("part-upload-failed")
 			cancel()
 			continue
 		}
@@ -381,7 +428,12 @@ func (s *Session) publishInitIfNeeded() bool {
 	ctx, cancel := s.newContext()
 	defer cancel()
 	if err := s.publisher.PublishInit(ctx, s.id, initBytes); err != nil {
-		log.Log.Warning("livehls.Session: init upload failed, will retry: " + err.Error())
+		log.WithError(err).WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "init_upload_failed",
+			"session_id": s.id,
+		}).Warn("Failed to upload live HLS init segment; retrying")
+		s.fireFailureOnce("init-upload-failed")
 		return false
 	}
 
@@ -389,7 +441,11 @@ func (s *Session) publishInitIfNeeded() bool {
 	s.initPublished = true
 	s.lastInitAt = time.Now()
 	s.mu.Unlock()
-	log.Log.Info("livehls.Session: init segment delivered for session " + s.id)
+	log.WithFields(log.Fields{
+		"component":  "cloud/livehls",
+		"event":      "init_uploaded",
+		"session_id": s.id,
+	}).Info("Live HLS init segment uploaded")
 	return true
 }
 
@@ -422,14 +478,22 @@ func (s *Session) refreshInitIfStale() {
 	ctx, cancel := s.newContext()
 	defer cancel()
 	if err := s.publisher.PublishInit(ctx, s.id, initBytes); err != nil {
-		log.Log.Warning("livehls.Session: init refresh failed, will retry: " + err.Error())
+		log.WithError(err).WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "init_refresh_failed",
+			"session_id": s.id,
+		}).Warn("Failed to refresh live HLS init segment; retrying")
 		return
 	}
 
 	s.mu.Lock()
 	s.lastInitAt = time.Now()
 	s.mu.Unlock()
-	log.Log.Debug("livehls.Session: refreshed init segment TTL for session " + s.id)
+	log.WithFields(log.Fields{
+		"component":  "cloud/livehls",
+		"event":      "init_refreshed",
+		"session_id": s.id,
+	}).Debug("Live HLS init segment refreshed")
 }
 
 // fireReadyOnce invokes the OnReady callback the first time it is called.
@@ -445,6 +509,18 @@ func (s *Session) fireReadyOnce() {
 	fn(s.id)
 }
 
+func (s *Session) fireFailureOnce(reason string) {
+	s.mu.Lock()
+	if s.failureFired || s.readyFired || s.onFailure == nil {
+		s.mu.Unlock()
+		return
+	}
+	s.failureFired = true
+	fn := s.onFailure
+	s.mu.Unlock()
+	fn(s.id, reason)
+}
+
 // newSessionID returns a short, unique, URL-safe session identifier of the form
 // <unix-seconds>-<random-hex>.
 func newSessionID() string {

machinery/src/cloud/livemoq/recovery.go

@@ -1,6 +1,9 @@
 package livemoq
 
-import "time"
+import (
+	"sync/atomic"
+	"time"
+)
 
 type FrameGateEvent uint8
 
@@ -17,9 +20,52 @@ type FrameGate struct {
 	recovering bool
 }
 
-// Reset closes the gate so publication resumes on the next keyframe. It is used
+// AudienceGate publishes every frame while watched and only keyframes while
-// when the publisher stopped writing for a reason unrelated to the stream health
+// idle. The idle keyframes keep the relay's latest cached group current without
-// (no subscribers), so the next viewer never receives a partial GOP.
+// paying the bandwidth cost of a full stream when nobody is watching.
+type AudienceGate struct {
+	idle bool
+}
+
+func (g *AudienceGate) Allow(hasSubscribers bool, isKeyFrame bool) (allowed bool, enteredIdle bool) {
+	if hasSubscribers {
+		g.idle = false
+		return true, false
+	}
+
+	enteredIdle = !g.idle
+	g.idle = true
+	return isKeyFrame, enteredIdle
+}
+
+// WriteWatchdog tracks the single synchronous frame write performed by a MoQ
+// publisher so another goroutine can interrupt a wedged native call.
+type WriteWatchdog struct {
+	startedAt atomic.Int64
+}
+
+func (w *WriteWatchdog) Begin(now time.Time) {
+	w.startedAt.Store(now.UnixNano())
+}
+
+func (w *WriteWatchdog) End() {
+	w.startedAt.Store(0)
+}
+
+func (w *WriteWatchdog) Elapsed(now time.Time) (time.Duration, bool) {
+	startedAt := w.startedAt.Load()
+	if startedAt == 0 {
+		return 0, false
+	}
+	elapsed := now.Sub(time.Unix(0, startedAt))
+	if elapsed < 0 {
+		elapsed = 0
+	}
+	return elapsed, true
+}
+
+// Reset closes the gate so publication resumes on the next keyframe. Entering
+// idle mode uses it to start the relay-cache refresh on a complete GOP.
 func (g *FrameGate) Reset() {
 	g.started = false
 	g.recovering = false

machinery/src/cloud/livemoq/recovery_test.go

@@ -47,3 +47,45 @@ func TestFrameGateAllowsMissingCaptureTime(t *testing.T) {
 		t.Fatalf("Allow() = (%t, %d), want (true, %d)", allowed, event, FrameGateEventStarted)
 	}
 }
+
+func TestAudienceGateRefreshesOnlyKeyframesWhileIdle(t *testing.T) {
+	gate := AudienceGate{}
+
+	if allowed, enteredIdle := gate.Allow(true, false); !allowed || enteredIdle {
+		t.Fatalf("active delta = (%t, %t), want (true, false)", allowed, enteredIdle)
+	}
+	if allowed, enteredIdle := gate.Allow(false, false); allowed || !enteredIdle {
+		t.Fatalf("first idle delta = (%t, %t), want (false, true)", allowed, enteredIdle)
+	}
+	if allowed, enteredIdle := gate.Allow(false, true); !allowed || enteredIdle {
+		t.Fatalf("idle keyframe = (%t, %t), want (true, false)", allowed, enteredIdle)
+	}
+	if allowed, enteredIdle := gate.Allow(false, false); allowed || enteredIdle {
+		t.Fatalf("later idle delta = (%t, %t), want (false, false)", allowed, enteredIdle)
+	}
+	if allowed, enteredIdle := gate.Allow(true, false); !allowed || enteredIdle {
+		t.Fatalf("resumed delta = (%t, %t), want (true, false)", allowed, enteredIdle)
+	}
+	if allowed, enteredIdle := gate.Allow(false, true); !allowed || !enteredIdle {
+		t.Fatalf("next idle keyframe = (%t, %t), want (true, true)", allowed, enteredIdle)
+	}
+}
+
+func TestWriteWatchdogTracksActiveWrite(t *testing.T) {
+	now := time.Unix(10, 0)
+	watchdog := WriteWatchdog{}
+
+	if elapsed, active := watchdog.Elapsed(now); active || elapsed != 0 {
+		t.Fatalf("Elapsed() before Begin() = (%s, %t), want (0, false)", elapsed, active)
+	}
+
+	watchdog.Begin(now)
+	if elapsed, active := watchdog.Elapsed(now.Add(5 * time.Second)); !active || elapsed != 5*time.Second {
+		t.Fatalf("Elapsed() during write = (%s, %t), want (5s, true)", elapsed, active)
+	}
+
+	watchdog.End()
+	if elapsed, active := watchdog.Elapsed(now.Add(6 * time.Second)); active || elapsed != 0 {
+		t.Fatalf("Elapsed() after End() = (%s, %t), want (0, false)", elapsed, active)
+	}
+}

machinery/src/cloud/livemoq_disabled.go

@@ -2,7 +2,20 @@
 
 package cloud
 
-import "github.com/kerberos-io/agent/machinery/src/models"
+import (
+	"context"
+
+	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/packets"
+)
 
 // StartLiveStreamMoQ is disabled in the standard Agent build.
-func StartLiveStreamMoQ(_ *models.Configuration, _ *models.Communication, _ bool) {}
+func StartLiveStreamMoQ(
+	_ context.Context,
+	_ *models.Configuration,
+	_ *models.Communication,
+	_ bool,
+	_ *packets.Queue,
+	_ *packets.Queue,
+) {
+}

machinery/src/cloud/livemoq_enabled.go

@@ -4,41 +4,120 @@ package cloud
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"net/url"
 	"os"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/cloud/livemoq"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/moq-dev/moq-go/moq"
+	log "github.com/sirupsen/logrus"
 )
 
 const (
 	defaultMoQRelayURL      = "https://relay.uug.ai/anon"
 	minMoQRetryDelay        = time.Second
 	maxMoQRetryDelay        = 30 * time.Second
-	maxMoQLivePacketAge     = 1500 * time.Millisecond
+	defaultMoQLivePacketAge = 1500 * time.Millisecond
+	minMoQLivePacketAge     = 250 * time.Millisecond
+	maxMoQLivePacketAge     = 30 * time.Second
+	defaultMoQWriteTimeout  = 5 * time.Second
+	minMoQWriteTimeout      = time.Second
+	maxMoQWriteTimeout      = time.Minute
+	moQWriteWatchInterval   = 250 * time.Millisecond
 	slowMoQWriteThreshold   = 100 * time.Millisecond
 	moQWriteWarningInterval = 10 * time.Second
 	duplicateKeyframeWindow = 500 * time.Millisecond
 )
 
 type liveMoQConfig struct {
-	relayURL    string
+	relayURL      string
-	broadcast   string
+	broadcast     string
-	quality     string
+	quality       string
-	sourceLabel string
+	sourceLabel   string
-	queue       *packets.Queue
+	queue         *packets.Queue
+	communication *models.Communication
+	maxPacketAge  time.Duration
+	writeTimeout  time.Duration
 }
 
-// label identifies the tier in log lines, since one Agent runs a publisher per
+func (c liveMoQConfig) logEntry(event string) *log.Entry {
-// quality tier.
+	return log.WithFields(log.Fields{
-func (c liveMoQConfig) label() string {
+		"component": "moq",
-	return c.quality + " (" + c.sourceLabel + " stream)"
+		"event":     event,
+		"quality":   c.quality,
+		"stream":    c.sourceLabel,
+	})
+}
+
+func moQRelayHost(relayURL string) string {
+	parsed, err := url.Parse(relayURL)
+	if err != nil {
+		return ""
+	}
+	return parsed.Host
+}
+
+func (c liveMoQConfig) packetAgeLimit() time.Duration {
+	if c.maxPacketAge > 0 {
+		return c.maxPacketAge
+	}
+	return defaultMoQLivePacketAge
+}
+
+func (c liveMoQConfig) writeTimeoutLimit() time.Duration {
+	if c.writeTimeout > 0 {
+		return c.writeTimeout
+	}
+	return defaultMoQWriteTimeout
+}
+
+func boundedMoQDuration(name string, fallback, minimum, maximum time.Duration) time.Duration {
+	raw := strings.TrimSpace(os.Getenv(name))
+	if raw == "" {
+		return fallback
+	}
+
+	value, err := time.ParseDuration(raw)
+	if err != nil {
+		log.WithError(err).WithFields(log.Fields{
+			"component":        "moq",
+			"configured_value": raw,
+			"effective_value":  fallback.String(),
+			"event":            "duration_invalid",
+			"variable":         name,
+		}).Warn("Invalid MoQ duration; using default")
+		return fallback
+	}
+	if value < minimum {
+		log.WithFields(log.Fields{
+			"component":        "moq",
+			"configured_value": value.String(),
+			"effective_value":  minimum.String(),
+			"event":            "duration_clamped",
+			"minimum":          minimum.String(),
+			"variable":         name,
+		}).Warn("MoQ duration is below the supported minimum")
+		return minimum
+	}
+	if value > maximum {
+		log.WithFields(log.Fields{
+			"component":        "moq",
+			"configured_value": value.String(),
+			"effective_value":  maximum.String(),
+			"event":            "duration_clamped",
+			"maximum":          maximum.String(),
+			"variable":         name,
+		}).Warn("MoQ duration exceeds the supported maximum")
+		return maximum
+	}
+	return value
 }
 
 // StartLiveStreamMoQ starts the publisher only in the dedicated MoQ build and
@@ -50,20 +129,37 @@ func (c liveMoQConfig) label() string {
 // publishing each tier as its OWN broadcast (see livemoq.BroadcastPath): the
 // high tier from the camera's highest-resolution stream and the low tier from
 // its sub stream, so switching quality in the frontend is a resubscribe to the
-// other path. Each tier only uploads while it actually has subscribers, so the
+// other path. Each tier uploads every frame while watched and keyframes only
-// second broadcast is close to free when nobody watches it.
+// while idle, keeping the relay cache current without continuously sending the
-func StartLiveStreamMoQ(configuration *models.Configuration, communication *models.Communication, subStreamEnabled bool) {
+// full unused stream.
+func StartLiveStreamMoQ(
+	ctx context.Context,
+	configuration *models.Configuration,
+	communication *models.Communication,
+	subStreamEnabled bool,
+	mainQueue *packets.Queue,
+	subQueue *packets.Queue,
+) {
 	if os.Getenv("AGENT_LIVE_MOQ_ENABLED") != "true" {
 		return
 	}
 
 	config := configuration.Config
 	if config.Offline == "true" || config.Capture.Liveview == "false" {
-		log.Log.Info("cloud.StartLiveStreamMoQ(): disabled by Agent live-view configuration")
+		log.WithFields(log.Fields{
+			"component": "moq",
+			"event":     "publisher_disabled",
+			"liveview":  config.Capture.Liveview,
+			"offline":   config.Offline,
+		}).Debug("MoQ publisher disabled by Agent configuration")
 		return
 	}
 	if config.Key == "" {
-		log.Log.Warning("cloud.StartLiveStreamMoQ(): AGENT_KEY is required")
+		log.WithFields(log.Fields{
+			"component": "moq",
+			"event":     "publisher_configuration_invalid",
+			"variable":  "AGENT_KEY",
+		}).Warn("MoQ publisher requires an Agent key")
 		return
 	}
 
@@ -83,38 +179,51 @@ func StartLiveStreamMoQ(configuration *models.Configuration, communication *mode
 		relayURL = defaultMoQRelayURL
 	}
 	broadcastPrefix := os.Getenv("AGENT_LIVE_MOQ_BROADCAST_PREFIX")
+	maxPacketAge := boundedMoQDuration("AGENT_LIVE_MOQ_MAX_PACKET_AGE", defaultMoQLivePacketAge, minMoQLivePacketAge, maxMoQLivePacketAge)
+	writeTimeout := boundedMoQDuration("AGENT_LIVE_MOQ_WRITE_TIMEOUT", defaultMoQWriteTimeout, minMoQWriteTimeout, maxMoQWriteTimeout)
 
-	ctx := context.Background()
+	var publishers sync.WaitGroup
-	if communication.Context != nil {
-		ctx = *communication.Context
-	}
-
 	for _, quality := range qualities {
-		queue := communication.Queue
+		queue := mainQueue
 		sourceLabel := "main"
-		if models.SelectSubStreamForQuality(config, quality, subStreamEnabled) && communication.SubQueue != nil {
+		if models.SelectSubStreamForQuality(config, quality, subStreamEnabled) && subQueue != nil {
-			queue = communication.SubQueue
+			queue = subQueue
 			sourceLabel = "sub"
 		}
 		if queue == nil {
-			log.Log.Warning("cloud.StartLiveStreamMoQ(): packet queue for the " + quality + " tier is unavailable")
+			log.WithFields(log.Fields{
+				"component": "moq",
+				"event":     "packet_queue_unavailable",
+				"quality":   quality,
+			}).Warn("MoQ packet queue is unavailable")
 			continue
 		}
-		go runLiveStreamMoQ(ctx, liveMoQConfig{
+		publisherConfig := liveMoQConfig{
-			relayURL:    relayURL,
+			relayURL:      relayURL,
-			broadcast:   livemoq.BroadcastPath(broadcastPrefix, config.Key, quality),
+			broadcast:     livemoq.BroadcastPath(broadcastPrefix, config.Key, quality),
-			quality:     quality,
+			quality:       quality,
-			sourceLabel: sourceLabel,
+			sourceLabel:   sourceLabel,
-			queue:       queue,
+			queue:         queue,
-		})
+			communication: communication,
+			maxPacketAge:  maxPacketAge,
+			writeTimeout:  writeTimeout,
+		}
+		publishers.Add(1)
+		go func() {
+			defer publishers.Done()
+			runLiveStreamMoQ(ctx, publisherConfig)
+		}()
 	}
+	publishers.Wait()
 }
 
 func runLiveStreamMoQ(ctx context.Context, config liveMoQConfig) {
-	log.Log.Info(fmt.Sprintf(
+	config.logEntry("publisher_started").Info("MoQ publisher started")
-		"cloud.runLiveStreamMoQ(): publishing %s stream (quality=%s) to %s/%s",
+	config.logEntry("publisher_configuration").WithFields(log.Fields{
-		config.sourceLabel, config.quality, strings.TrimRight(config.relayURL, "/"), config.broadcast,
+		"max_packet_age_ms": config.packetAgeLimit().Milliseconds(),
-	))
+		"relay_host":        moQRelayHost(config.relayURL),
+		"write_timeout_ms":  config.writeTimeoutLimit().Milliseconds(),
+	}).Debug("MoQ publisher configuration")
 
 	retryDelay := minMoQRetryDelay
 	for ctx.Err() == nil {
@@ -123,7 +232,13 @@ func runLiveStreamMoQ(ctx context.Context, config liveMoQConfig) {
 		if ctx.Err() != nil {
 			return
 		}
-		log.Log.Warning("cloud.runLiveStreamMoQ(): publisher stopped: " + err.Error())
+		if config.communication != nil {
+			config.communication.RecordMoQReconnect(config.quality)
+		}
+		config.logEntry("publisher_reconnecting").WithError(err).WithFields(log.Fields{
+			"connected_duration_ms": time.Since(connectedAt).Milliseconds(),
+			"retry_delay_ms":        retryDelay.Milliseconds(),
+		}).Warn("MoQ publisher stopped; reconnecting")
 		if time.Since(connectedAt) >= time.Minute {
 			retryDelay = minMoQRetryDelay
 		}
@@ -150,42 +265,80 @@ func publishLiveStreamMoQ(ctx context.Context, config liveMoQConfig) error {
 		return fmt.Errorf("connect to relay: %w", err)
 	}
 	defer client.Close()
+	config.logEntry("relay_connected").WithField("relay_host", moQRelayHost(config.relayURL)).
+		Info("MoQ relay connected")
 
-	sessionCtx, cancelSessionWatch := context.WithCancel(ctx)
+	publisherCtx, cancelPublisher := context.WithCancel(ctx)
-	defer cancelSessionWatch()
+	defer cancelPublisher()
 	sessionClosed := make(chan error, 1)
+	sessionWatchDone := make(chan struct{})
 	go func() {
-		sessionClosed <- client.Session().Closed(sessionCtx)
+		defer close(sessionWatchDone)
+		sessionClosed <- client.Session().Closed(publisherCtx)
+		cancelPublisher()
 	}()
 
 	broadcast, err := client.CreateBroadcast(config.broadcast)
 	if err != nil {
 		return fmt.Errorf("create broadcast: %w", err)
 	}
-	defer broadcast.Finish()
+	var finishBroadcastOnce sync.Once
+	var finishBroadcastErr error
+	finishBroadcast := func() error {
+		finishBroadcastOnce.Do(func() {
+			finishBroadcastErr = broadcast.Finish()
+		})
+		return finishBroadcastErr
+	}
+	defer finishBroadcast()
 
 	stream, err := broadcast.PublishMedia("avc3", nil)
 	if err != nil {
 		return fmt.Errorf("create H.264 media stream: %w", err)
 	}
-	defer stream.Finish()
+	var finishStreamOnce sync.Once
+	var finishStreamErr error
+	finishStream := func() error {
+		finishStreamOnce.Do(func() {
+			finishStreamErr = stream.Finish()
+		})
+		return finishStreamErr
+	}
+	defer finishStream()
 
-	// Only upload while this tier is actually being watched. `publishing` starts
+	// Publish the complete stream while watched and only keyframes while idle.
-	// true so the track becomes discoverable on the relay even before the first
+	// `publishing` starts true so the track becomes discoverable on the relay
-	// subscriber ever arrives; from the moment a viewer has attached once, the
+	// before the first subscriber; after that, the subscriber watcher selects the
-	// subscriber watcher takes over and idles the tier again when everybody left.
+	// full-rate or keyframe-refresh mode.
-	watchCtx, cancelWatch := context.WithCancel(ctx)
-	defer cancelWatch()
 	publishing := &atomic.Bool{}
 	publishing.Store(true)
-	go watchLiveStreamMoQSubscribers(watchCtx, stream, publishing, config)
+	subscriberWatchDone := make(chan struct{})
+	go func() {
+		defer close(subscriberWatchDone)
+		watchLiveStreamMoQSubscribers(publisherCtx, stream, publishing, config)
+	}()
+	writeWatchdog := &livemoq.WriteWatchdog{}
+	writeWatchDone := make(chan struct{})
+	closePublisher := func() error {
+		return closeLiveMoQPublisher(finishStream, finishBroadcast, client.Close)
+	}
+	go func() {
+		defer close(writeWatchDone)
+		watchLiveStreamMoQWrites(publisherCtx, closePublisher, writeWatchdog, config)
+	}()
+	defer func() {
+		cancelPublisher()
+		<-writeWatchDone
+		<-subscriberWatchDone
+		<-sessionWatchDone
+	}()
 
 	cursor := config.queue.Latest()
 	gate := livemoq.FrameGate{}
+	audienceGate := livemoq.AudienceGate{}
 	deduplicator := livemoq.KeyframeDeduplicator{}
 	var lastSlowWriteWarning time.Time
 	var lastDuplicateKeyframeWarning time.Time
-	idle := false
 	for {
 		select {
 		case err := <-sessionClosed:
@@ -193,32 +346,44 @@ func publishLiveStreamMoQ(ctx context.Context, config liveMoQConfig) error {
 		default:
 		}
 
-		packet, err := cursor.ReadPacket()
+		packet, err := cursor.ReadPacketContext(publisherCtx)
 		if err != nil {
+			select {
+			case sessionErr := <-sessionClosed:
+				return fmt.Errorf("relay session closed: %w", sessionErr)
+			default:
+			}
+			if ctx.Err() != nil {
+				return ctx.Err()
+			}
 			return fmt.Errorf("read packet: %w", err)
 		}
-		if !publishing.Load() {
-			// Keep draining the cursor so we stay at the live edge, but publish
-			// nothing. The gate is closed so the next viewer resumes on a keyframe.
-			if !idle {
-				gate.Reset()
-				deduplicator.Reset()
-				idle = true
-			}
-			continue
-		}
-		idle = false
 		if !packet.IsVideo || len(packet.Data) == 0 || !strings.EqualFold(packet.Codec, "H264") {
 			continue
 		}
-		allowed, event := gate.Allow(packet.IsKeyFrame, packet.CurrentTime, time.Now(), maxMoQLivePacketAge)
+		allowedForAudience, enteredIdle := audienceGate.Allow(publishing.Load(), packet.IsKeyFrame)
+		if enteredIdle {
+			gate.Reset()
+			deduplicator.Reset()
+		}
+		if !allowedForAudience {
+			continue
+		}
+		allowed, event := gate.Allow(packet.IsKeyFrame, packet.CurrentTime, time.Now(), config.packetAgeLimit())
 		switch event {
 		case livemoq.FrameGateEventStarted:
-			log.Log.Info("cloud.publishLiveStreamMoQ(): first H.264 keyframe received; " + config.label() + " broadcast is live")
+			config.logEntry("broadcast_live").WithFields(log.Fields{
+				"codec":        packet.Codec,
+				"timestamp_ms": packet.Time,
+			}).Info("MoQ broadcast is live")
 		case livemoq.FrameGateEventLagging:
-			log.Log.Warning("cloud.publishLiveStreamMoQ(): " + config.label() + " stream is lagging; dropping packets until a recent keyframe")
+			config.logEntry("stream_lagging").WithFields(log.Fields{
+				"max_packet_age_ms": config.packetAgeLimit().Milliseconds(),
+				"timestamp_ms":      packet.Time,
+			}).Warn("MoQ stream is lagging; dropping packets until a recent keyframe")
 		case livemoq.FrameGateEventRecovered:
-			log.Log.Info("cloud.publishLiveStreamMoQ(): caught up with the " + config.label() + " live stream at a recent keyframe")
+			config.logEntry("stream_recovered").WithField("timestamp_ms", packet.Time).
+				Info("MoQ stream recovered at a recent keyframe")
 		}
 		if !allowed {
 			continue
@@ -228,18 +393,18 @@ func publishLiveStreamMoQ(ctx context.Context, config liveMoQConfig) error {
 			return fmt.Errorf("normalize H.264 access unit: %w", err)
 		}
 		if normalizationStats.DuplicateIDRNALUs > 0 && time.Since(lastDuplicateKeyframeWarning) >= moQWriteWarningInterval {
-			log.Log.Warning(fmt.Sprintf(
+			config.logEntry("duplicate_idr_removed").WithFields(log.Fields{
-				"cloud.publishLiveStreamMoQ(): %s removed %d duplicate IDR NALU(s) from H.264 keyframe (timestamp_ms=%d)",
+				"duplicate_idr_count": normalizationStats.DuplicateIDRNALUs,
-				config.label(), normalizationStats.DuplicateIDRNALUs, packet.Time,
+				"timestamp_ms":        packet.Time,
-			))
+			}).Warn("Removed duplicate IDR NAL units from MoQ keyframe")
+
 			lastDuplicateKeyframeWarning = time.Now()
 		}
 		if packet.IsKeyFrame && deduplicator.IsDuplicate(packet.Time, packet.CurrentTime, payload, time.Now(), duplicateKeyframeWindow) {
 			if time.Since(lastDuplicateKeyframeWarning) >= moQWriteWarningInterval {
-				log.Log.Warning(fmt.Sprintf(
+				config.logEntry("duplicate_keyframe_dropped").WithField("timestamp_ms", packet.Time).
-					"cloud.publishLiveStreamMoQ(): %s dropping duplicate H.264 keyframe (timestamp_ms=%d)",
+					Warn("Dropped duplicate MoQ keyframe")
-					config.label(), packet.Time,
+
-				))
 				lastDuplicateKeyframeWarning = time.Now()
 			}
 			continue
@@ -249,10 +414,16 @@ func publishLiveStreamMoQ(ctx context.Context, config liveMoQConfig) error {
 			TimestampUs: livemoq.TimestampUs(packet.Time),
 		}
 		writeStartedAt := time.Now()
-		if err := stream.WriteFrame(frame); err != nil {
+		writeWatchdog.Begin(writeStartedAt)
+		err = stream.WriteFrame(frame)
+		writeWatchdog.End()
+		if err != nil {
 			return fmt.Errorf("write H.264 access unit: %w", err)
 		}
 		writeDuration := time.Since(writeStartedAt)
+		if config.communication != nil {
+			config.communication.RecordMoQWrite(config.quality, writeDuration, time.Now())
+		}
 		if writeDuration >= slowMoQWriteThreshold && time.Since(lastSlowWriteWarning) >= moQWriteWarningInterval {
 			packetAge := time.Duration(0)
 			if packet.CurrentTime > 0 {
@@ -261,15 +432,58 @@ func publishLiveStreamMoQ(ctx context.Context, config liveMoQConfig) error {
 					packetAge = 0
 				}
 			}
-			log.Log.Warning(fmt.Sprintf(
+			config.logEntry("slow_frame_write").WithFields(log.Fields{
-				"cloud.publishLiveStreamMoQ(): %s WriteFrame blocked for %s (packet_age=%s keyframe=%t)",
+				"duration_ms":   writeDuration.Milliseconds(),
-				config.label(), writeDuration.Round(time.Millisecond), packetAge.Round(time.Millisecond), packet.IsKeyFrame,
+				"keyframe":      packet.IsKeyFrame,
-			))
+				"packet_age_ms": packetAge.Milliseconds(),
+			}).Warn("MoQ frame write was slow")
+
 			lastSlowWriteWarning = time.Now()
 		}
 	}
 }
 
+func closeLiveMoQPublisher(finishStream, finishBroadcast, closeClient func() error) error {
+	streamErr := finishStream()
+	broadcastErr := finishBroadcast()
+	clientErr := closeClient()
+	return errors.Join(streamErr, broadcastErr, clientErr)
+}
+
+func watchLiveStreamMoQWrites(ctx context.Context, closePublisher func() error, watchdog *livemoq.WriteWatchdog, config liveMoQConfig) {
+	ticker := time.NewTicker(moQWriteWatchInterval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			if err := closePublisher(); err != nil {
+				config.logEntry("publisher_close_failed").WithError(err).
+					Debug("MoQ publisher reported an error while closing")
+			}
+			return
+		case now := <-ticker.C:
+			writeDuration, active := watchdog.Elapsed(now)
+			if !active || writeDuration < config.writeTimeoutLimit() {
+				continue
+			}
+			config.logEntry("frame_write_timeout").WithFields(log.Fields{
+				"duration_ms": writeDuration.Milliseconds(),
+				"timeout_ms":  config.writeTimeoutLimit().Milliseconds(),
+			}).Warn("MoQ frame write timed out; reconnecting relay client")
+
+			if config.communication != nil {
+				config.communication.RecordMoQWriteTimeout()
+			}
+			if err := closePublisher(); err != nil {
+				config.logEntry("publisher_close_failed").WithError(err).
+					Warn("MoQ publisher reported an error while closing after a write timeout")
+			}
+			return
+		}
+	}
+}
+
 // watchLiveStreamMoQSubscribers flips the publisher between uploading and idling
 // as viewers subscribe to and leave this tier's broadcast. Used and Unused both
 // block, so they are followed from their own goroutine.
@@ -285,7 +499,7 @@ func watchLiveStreamMoQSubscribers(ctx context.Context, stream *moq.MediaProduce
 			return
 		}
 		if publishing.CompareAndSwap(false, true) {
-			log.Log.Info("cloud.watchLiveStreamMoQSubscribers(): viewer subscribed, resuming the " + config.label() + " broadcast")
+			config.logEntry("subscriber_joined").Info("MoQ subscriber joined; resuming broadcast")
 		}
 
 		if err := stream.Unused(ctx); err != nil {
@@ -293,6 +507,6 @@ func watchLiveStreamMoQSubscribers(ctx context.Context, stream *moq.MediaProduce
 			return
 		}
 		publishing.Store(false)
-		log.Log.Info("cloud.watchLiveStreamMoQSubscribers(): no viewers left, idling the " + config.label() + " broadcast")
+		config.logEntry("subscribers_idle").Info("MoQ broadcast idle; refreshing keyframes only")
 	}
 }

machinery/src/cloud/livemoq_enabled_test.go

@@ -0,0 +1,212 @@
+//go:build moq
+
+package cloud
+
+import (
+	"context"
+	"errors"
+	"reflect"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/kerberos-io/agent/machinery/src/cloud/livemoq"
+	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/moq-dev/moq-go/moq"
+)
+
+func TestCloseLiveMoQPublisherFinishesBeforeClosingClient(t *testing.T) {
+	var order []string
+	streamErr := errors.New("stream finish failed")
+	broadcastErr := errors.New("broadcast finish failed")
+	clientErr := errors.New("client close failed")
+
+	err := closeLiveMoQPublisher(
+		func() error {
+			order = append(order, "stream")
+			return streamErr
+		},
+		func() error {
+			order = append(order, "broadcast")
+			return broadcastErr
+		},
+		func() error {
+			order = append(order, "client")
+			return clientErr
+		},
+	)
+
+	if want := []string{"stream", "broadcast", "client"}; !reflect.DeepEqual(order, want) {
+		t.Fatalf("shutdown order = %v, want %v", order, want)
+	}
+	for _, want := range []error{streamErr, broadcastErr, clientErr} {
+		if !errors.Is(err, want) {
+			t.Errorf("shutdown error %v does not include %v", err, want)
+		}
+	}
+}
+
+func TestWatchLiveStreamMoQWritesClosesClientWhenContextEnds(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	closed := make(chan struct{})
+	var closeCalls atomic.Int32
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		watchLiveStreamMoQWrites(ctx, func() error {
+			if closeCalls.Add(1) == 1 {
+				close(closed)
+			}
+			return nil
+		}, &livemoq.WriteWatchdog{}, liveMoQConfig{quality: "low", sourceLabel: "sub"})
+	}()
+
+	cancel()
+	select {
+	case <-closed:
+	case <-time.After(time.Second):
+		t.Fatal("watchLiveStreamMoQWrites() did not close the client after cancellation")
+	}
+	<-done
+	if got := closeCalls.Load(); got != 1 {
+		t.Fatalf("close calls = %d, want 1", got)
+	}
+}
+
+func TestBoundedMoQDuration(t *testing.T) {
+	const name = "AGENT_LIVE_MOQ_TEST_DURATION"
+	tests := []struct {
+		name string
+		raw  string
+		want time.Duration
+	}{
+		{name: "unset", want: 5 * time.Second},
+		{name: "valid", raw: "12s", want: 12 * time.Second},
+		{name: "invalid", raw: "later", want: 5 * time.Second},
+		{name: "below minimum", raw: "100ms", want: time.Second},
+		{name: "above maximum", raw: "2m", want: time.Minute},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			t.Setenv(name, test.raw)
+			if got := boundedMoQDuration(name, 5*time.Second, time.Second, time.Minute); got != test.want {
+				t.Fatalf("boundedMoQDuration() = %s, want %s", got, test.want)
+			}
+		})
+	}
+}
+
+func TestWatchLiveStreamMoQWritesRecordsTimeout(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	closed := make(chan struct{})
+	communication := &models.Communication{}
+	watchdog := &livemoq.WriteWatchdog{}
+	watchdog.Begin(time.Now().Add(-time.Second))
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		watchLiveStreamMoQWrites(ctx, func() error {
+			close(closed)
+			return nil
+		}, watchdog, liveMoQConfig{
+			quality:       models.StreamQualityLow,
+			sourceLabel:   "sub",
+			communication: communication,
+			writeTimeout:  10 * time.Millisecond,
+		})
+	}()
+
+	select {
+	case <-closed:
+	case <-time.After(time.Second):
+		t.Fatal("watchLiveStreamMoQWrites() did not close the client after a write timeout")
+	}
+	<-done
+	if got := communication.RecoveryTelemetry().MoQWriteTimeouts; got != 1 {
+		t.Fatalf("MoQ write timeouts = %d, want 1", got)
+	}
+}
+
+func TestNativeMoQClientCloseStopsActiveWriter(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	server, err := moq.Listen(ctx, "127.0.0.1:0", moq.WithTLSGenerate("localhost"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	serveDone := make(chan error, 1)
+	go func() {
+		serveDone <- server.Serve(ctx)
+	}()
+
+	client, err := moq.Dial(ctx, "https://"+server.LocalAddr(), moq.WithTLSVerify(false))
+	if err != nil {
+		server.Close()
+		t.Fatal(err)
+	}
+	broadcast, err := client.CreateBroadcast("test/native-close")
+	if err != nil {
+		client.Close()
+		server.Close()
+		t.Fatal(err)
+	}
+	stream, err := broadcast.PublishMedia("avc3", nil)
+	if err != nil {
+		client.Close()
+		server.Close()
+		t.Fatal(err)
+	}
+
+	firstWrite := make(chan struct{})
+	writerDone := make(chan error, 1)
+	go func() {
+		payload := []byte{
+			0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x1e, 0xd9, 0x00, 0xa0, 0x47, 0xfe, 0xc8,
+			0x00, 0x00, 0x00, 0x01, 0x68, 0xce, 0x38, 0x80,
+			0x00, 0x00, 0x00, 0x01, 0x65, 0x88,
+		}
+		for timestamp := uint64(0); ; timestamp++ {
+			err := stream.WriteFrame(moq.Frame{Payload: payload, TimestampUs: timestamp})
+			if err != nil {
+				writerDone <- err
+				return
+			}
+			if timestamp == 0 {
+				close(firstWrite)
+			}
+		}
+	}()
+
+	select {
+	case <-firstWrite:
+	case <-ctx.Done():
+		t.Fatal("native MoQ writer did not start")
+	}
+	if err := stream.Finish(); err != nil {
+		t.Fatal(err)
+	}
+	if err := client.Close(); err != nil {
+		t.Fatal(err)
+	}
+	select {
+	case err := <-writerDone:
+		if err == nil {
+			t.Fatal("native MoQ writer stopped without an error after client close")
+		}
+	case <-ctx.Done():
+		t.Fatal("native MoQ client close did not stop the writer")
+	}
+
+	_ = broadcast.Finish()
+	_ = server.Close()
+	select {
+	case <-serveDone:
+	case <-ctx.Done():
+		t.Fatal("native MoQ server did not stop")
+	}
+}

machinery/src/cloud/livesnapshot/publisher.go

@@ -30,7 +30,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
+	log "github.com/sirupsen/logrus"
 )
 
 const (
@@ -131,7 +131,10 @@ func (p *Publisher) PublishSnapshot(ctx context.Context, jpeg []byte) error {
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
 		return fmt.Errorf("livesnapshot: upload snapshot rejected: %s", resp.Status)
 	}
-	log.Log.Debug("livesnapshot.Publisher.PublishSnapshot(): shipped preview frame for device " + p.cfg.DeviceKey)
+	log.WithFields(log.Fields{
+		"component": "livesnapshot",
+		"event":     "snapshot_published",
+	}).Debug("Live snapshot published")
 	return nil
 }
 

machinery/src/cloud/recording_metadata.go

@@ -15,6 +15,7 @@ import (
 const recordingFPSHeader = "X-Kerberos-Storage-Fps"
 const recordingDurationHeader = "X-Kerberos-Storage-Duration"
 const recordingTimestampHeader = "X-Kerberos-Storage-Timestamp"
+const recordingEncryptedHeader = "X-Kerberos-Storage-Encrypted"
 
 // queuedRecordingFPS reads the FPS snapshot written into the upload marker
 // when the recording was finalized. Historical empty markers intentionally
@@ -84,5 +85,8 @@ func setQueuedRecordingMetadataHeaders(header http.Header, fileName string) {
 		if metadata.Timestamp > 0 {
 			header.Set(recordingTimestampHeader, strconv.FormatInt(metadata.Timestamp, 10))
 		}
+		if metadata.Encrypted {
+			header.Set(recordingEncryptedHeader, "true")
+		}
 	}
 }

machinery/src/cloud/s3.go

@@ -9,9 +9,9 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/minio/minio-go/v6"
+	log "github.com/sirupsen/logrus"
 )
 
 func UploadS3(configuration *models.Configuration, fileName string) (bool, bool, error) {
@@ -29,7 +29,7 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
 
 	if config.S3 == nil {
 		errorMessage := "UploadS3: Uploading Failed, as no settings found"
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, false, errors.New(errorMessage)
 	}
 
@@ -49,14 +49,14 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
 	// Check if we have some credentials otherwise we abort the request.
 	if aws_access_key_id == "" || aws_secret_access_key == "" {
 		errorMessage := "UploadS3: Uploading Failed, as no credentials found"
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, false, errors.New(errorMessage)
 	}
 
 	s3Client, err := minio.NewWithRegion("s3.amazonaws.com", aws_access_key_id, aws_secret_access_key, true, aws_region)
 	if err != nil {
 		errorMessage := "UploadS3: " + err.Error()
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, true, errors.New(errorMessage)
 	}
 
@@ -74,7 +74,7 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
 	fileParts := strings.Split(fileName, "_")
 	if len(fileParts) == 1 {
 		errorMessage := "UploadS3: " + fileName + " is not a valid name."
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, true, errors.New(errorMessage)
 	}
 
@@ -85,7 +85,7 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
 	//numberOfChanges := fileParts[4]
 	token, _ := strconv.Atoi(fileParts[5])
 
-	log.Log.Info("UploadS3: Upload started for " + fileName)
+	log.Info("UploadS3: Upload started for " + fileName)
 	fullname := "data/recordings/" + fileName
 
 	file, err := os.OpenFile(fullname, os.O_RDWR, 0755)
@@ -95,14 +95,14 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
 
 	if err != nil {
 		errorMessage := "UploadS3: " + err.Error()
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, true, errors.New(errorMessage)
 	}
 
 	fileInfo, err := file.Stat()
 	if err != nil {
 		errorMessage := "UploadS3: " + err.Error()
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, true, errors.New(errorMessage)
 	}
 
@@ -128,10 +128,10 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
 
 	if err != nil {
 		errorMessage := "UploadS3: Uploading Failed, " + err.Error()
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, true, errors.New(errorMessage)
 	} else {
-		log.Log.Info("UploadS3: Upload Finished, file has been uploaded to bucket: " + strconv.FormatInt(n, 10))
+		log.Info("UploadS3: Upload Finished, file has been uploaded to bucket: " + strconv.FormatInt(n, 10))
 		return true, true, nil
 	}
 }

machinery/src/cloud/tus_client.go

@@ -15,8 +15,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 // tusResumableVersion is the tus protocol version implemented by this client.
@@ -90,7 +90,14 @@ func logTusUploadProgress(label string, offset, size int64, loggedBucket *int64)
 	if percent > 100 {
 		percent = 100
 	}
-	log.Log.Infof("%s: resumable upload progress %d%% (%d/%d bytes)", label, percent, offset, size)
+	log.WithFields(log.Fields{
+		"bytes_sent":  offset,
+		"bytes_total": size,
+		"component":   "tus",
+		"event":       "upload_progress",
+		"progress":    percent,
+		"target":      label,
+	}).Info("Resumable upload progress")
 }
 
 // tusHeaderFunc sets the authentication and routing headers required on every
@@ -121,7 +128,7 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
 	}
 	if ferr != nil {
 		msg := label + ": resumable upload failed, file doesn't exist anymore"
-		log.Log.Info(msg)
+		log.Info(msg)
 		// The file is gone, so the legacy path cannot help either. Report it as
 		// "supported" to avoid a pointless fallback attempt.
 		return false, false, true, "", errors.New(msg)
@@ -182,7 +189,7 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
 					// The vault does not implement tus; let the caller fall back.
 					return false, false, false, "", cerr
 				}
-				log.Log.Info(label + ": resumable create failed, " + cerr.Error())
+				log.Info(label + ": resumable create failed, " + cerr.Error())
 				tusBackoff(attempt)
 				continue
 			}
@@ -200,7 +207,7 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
 				uploadURL = ""
 				continue
 			}
-			log.Log.Info(label + ": resumable head failed, " + herr.Error())
+			log.Info(label + ": resumable head failed, " + herr.Error())
 			tusBackoff(attempt)
 			continue
 		}
@@ -253,9 +260,9 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
 				if status >= 400 {
 					// Definitive rejection (e.g. provider push failed during finalize).
 					// Re-evaluate via HEAD on the next iteration to decide retry/restart.
-					log.Log.Info(label + ": resumable patch rejected, " + perr.Error())
+					log.Info(label + ": resumable patch rejected, " + perr.Error())
 				} else {
-					log.Log.Info(label + ": resumable patch failed, " + perr.Error())
+					log.Info(label + ": resumable patch failed, " + perr.Error())
 				}
 				tusBackoff(attempt)
 				patchFailed = true
@@ -354,6 +361,9 @@ func addRecordingTusMetadata(values map[string]string, fileName string) {
 	if metadata.Timestamp > 0 {
 		values["timestamp"] = strconv.FormatInt(metadata.Timestamp, 10)
 	}
+	if metadata.Encrypted {
+		values["encrypted"] = "true"
+	}
 }
 
 // tusCreate performs the tus "creation" request (POST). On success it returns

machinery/src/cloud/tus_client_test.go

@@ -264,6 +264,36 @@ func testVault(uri string) models.KStorage {
 	}
 }
 
+func TestUploadKerberosVaultSkipsEmptyRecording(t *testing.T) {
+	fileName := "1787015373_3-654_office-camera17_0-0-0-0_-1_1960.mp4"
+	withRecording(t, fileName, nil)
+
+	requestCount := 0
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requestCount++
+		w.WriteHeader(http.StatusInternalServerError)
+	}))
+	defer server.Close()
+
+	vault := testVault(server.URL)
+	configuration := &models.Configuration{Config: models.Config{
+		Key:               "device-key",
+		KStorage:          &vault,
+		KStorageSecondary: &models.KStorage{},
+	}}
+
+	uploaded, configured, err := UploadKerberosVault(configuration, fileName)
+	if err != nil {
+		t.Fatalf("UploadKerberosVault() error = %v", err)
+	}
+	if uploaded || configured {
+		t.Fatalf("UploadKerberosVault() uploaded/configured = %v/%v, want false/false", uploaded, configured)
+	}
+	if requestCount != 0 {
+		t.Fatalf("Vault received %d requests, want 0", requestCount)
+	}
+}
+
 func TestUploadVaultResumable_HappyPath(t *testing.T) {
 	srv := newFakeTus()
 	ts := httptest.NewServer(srv)
@@ -356,7 +386,7 @@ func TestQueuedRecordingFPSAllowsMissingHistoricalMarker(t *testing.T) {
 func TestQueuedRecordingMetadataHeaders(t *testing.T) {
 	fileName := "recording.mp4"
 	withRecording(t, fileName, []byte("recording"))
-	withQueuedRecordingFPS(t, fileName, `{"filename":"recording.mp4","device_key":"device-key","timestamp":1785934709414,"duration":20452,"fps":25}`)
+	withQueuedRecordingFPS(t, fileName, `{"filename":"recording.mp4","device_key":"device-key","timestamp":1785934709414,"duration":20452,"fps":25,"encrypted":true}`)
 
 	header := make(http.Header)
 	setQueuedRecordingMetadataHeaders(header, fileName)
@@ -369,6 +399,14 @@ func TestQueuedRecordingMetadataHeaders(t *testing.T) {
 	if got := header.Get(recordingTimestampHeader); got != "1785934709414" {
 		t.Fatalf("timestamp header = %q", got)
 	}
+	if got := header.Get(recordingEncryptedHeader); got != "true" {
+		t.Fatalf("encrypted header = %q", got)
+	}
+	metadata := map[string]string{}
+	addRecordingTusMetadata(metadata, fileName)
+	if got := metadata["encrypted"]; got != "true" {
+		t.Fatalf("encrypted TUS metadata = %q", got)
+	}
 }
 
 func TestQueuedRecordingFPSAllowsLegacyMarkerFileName(t *testing.T) {

machinery/src/components/backchannel.go

@@ -3,17 +3,16 @@ package components
 import (
 	"bufio"
 	"context"
-	"fmt"
 	"math/rand"
 	"os"
 	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/capture"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/kerberos-io/joy4/av"
 	"github.com/pion/rtp"
+	log "github.com/sirupsen/logrus"
 	"github.com/zaf/g711"
 )
 
@@ -79,7 +78,7 @@ func GetBackChannelAudioCodec(streams []av.CodecData, communication *models.Comm
 			if stream.Type().String() == "PCM_MULAW" {
 				pcmuCodec := stream.(av.AudioCodecData)
 				if pcmuCodec.IsBackChannel() {
-					communication.HasBackChannel = true
+					communication.HasBackChannel.Store(true)
 					return pcmuCodec
 				}
 			}
@@ -88,22 +87,17 @@ func GetBackChannelAudioCodec(streams []av.CodecData, communication *models.Comm
 	return nil
 }
 
-func WriteAudioToBackchannel(communication *models.Communication, rtspClient capture.RTSPClient) {
+func WriteAudioToBackchannel(ctx context.Context, communication *models.Communication, rtspClient capture.RTSPClient) {
-	ctx := context.Background()
-	if communication.Context != nil {
-		ctx = *communication.Context
-	}
-
 	writeAudioToBackchannel(ctx, ctx, communication.HandleAudio, rtspClient)
 }
 
 func writeAudioToBackchannel(ctx context.Context, otelContext context.Context, audioChannel <-chan models.AudioDataPartial, rtspClient backchannelClient) {
-	log.Log.Info("Audio.WriteAudioToBackchannel(): writing to backchannel audio codec")
+	log.Info("Audio.WriteAudioToBackchannel(): writing to backchannel audio codec")
 
 	if err := rtspClient.StartBackChannel(ctx, otelContext); err != nil {
-		log.Log.Error("Audio.WriteAudioToBackchannel(): error starting backchannel: " + err.Error())
+		log.Error("Audio.WriteAudioToBackchannel(): error starting backchannel: " + err.Error())
 		if !reconnectBackchannel(ctx, otelContext, rtspClient) {
-			log.Log.Info("Audio.WriteAudioToBackchannel(): stopped while reconnecting")
+			log.Info("Audio.WriteAudioToBackchannel(): stopped while reconnecting")
 			return
 		}
 	}
@@ -112,11 +106,11 @@ func writeAudioToBackchannel(ctx context.Context, otelContext context.Context, a
 	for {
 		select {
 		case <-ctx.Done():
-			log.Log.Info("Audio.WriteAudioToBackchannel(): stopped")
+			log.Info("Audio.WriteAudioToBackchannel(): stopped")
 			return
 		case audio, ok := <-audioChannel:
 			if !ok {
-				log.Log.Info("Audio.WriteAudioToBackchannel(): finished")
+				log.Info("Audio.WriteAudioToBackchannel(): finished")
 				return
 			}
 
@@ -127,9 +121,9 @@ func writeAudioToBackchannel(ctx context.Context, otelContext context.Context, a
 
 			pkt := packetizer.packet(audio, time.Now())
 			if err := rtspClient.WritePacket(pkt); err != nil {
-				log.Log.Error("Audio.WriteAudioToBackchannel(): error writing packet to backchannel: " + err.Error())
+				log.Error("Audio.WriteAudioToBackchannel(): error writing packet to backchannel: " + err.Error())
 				if !reconnectBackchannel(ctx, otelContext, rtspClient) {
-					log.Log.Info("Audio.WriteAudioToBackchannel(): stopped while reconnecting")
+					log.Info("Audio.WriteAudioToBackchannel(): stopped while reconnecting")
 					return
 				}
 				packetizer = newBackchannelPacketizer()
@@ -137,7 +131,7 @@ func writeAudioToBackchannel(ctx context.Context, otelContext context.Context, a
 			}
 
 			if !waitForBackchannel(ctx, time.Duration(len(audio.Data))*time.Second/backchannelSampleRate) {
-				log.Log.Info("Audio.WriteAudioToBackchannel(): stopped")
+				log.Info("Audio.WriteAudioToBackchannel(): stopped")
 				return
 			}
 		}
@@ -162,7 +156,7 @@ func reconnectBackchannel(ctx context.Context, otelContext context.Context, rtsp
 	backoff := backchannelReconnectInitial
 	for {
 		if err := rtspClient.Close(otelContext); err != nil {
-			log.Log.Error("Audio.WriteAudioToBackchannel(): error closing failed backchannel: " + err.Error())
+			log.Error("Audio.WriteAudioToBackchannel(): error closing failed backchannel: " + err.Error())
 		}
 		if ctx.Err() != nil {
 			return false
@@ -173,11 +167,11 @@ func reconnectBackchannel(ctx context.Context, otelContext context.Context, rtsp
 			err = rtspClient.StartBackChannel(ctx, otelContext)
 		}
 		if err == nil {
-			log.Log.Info("Audio.WriteAudioToBackchannel(): reconnected backchannel")
+			log.Info("Audio.WriteAudioToBackchannel(): reconnected backchannel")
 			return true
 		}
 
-		log.Log.Error("Audio.WriteAudioToBackchannel(): error reconnecting backchannel: " + err.Error())
+		log.Error("Audio.WriteAudioToBackchannel(): error reconnecting backchannel: " + err.Error())
 		if !waitForBackchannel(ctx, backoff) {
 			return false
 		}
@@ -204,7 +198,12 @@ func WriteFileToBackChannel(infile av.DemuxCloser) {
 	// Do the warmup!
 	file, err := os.Open("./audiofile.bye")
 	if err != nil {
-		fmt.Println("WriteFileToBackChannel: error opening audiofile.bye file")
+		log.WithError(err).WithFields(log.Fields{
+			"component": "backchannel",
+			"event":     "audio_file_open_failed",
+			"path":      "./audiofile.bye",
+		}).Error("Failed to open backchannel audio file")
+		return
 	}
 	defer file.Close()
 

machinery/src/components/watchdog_test.go

@@ -0,0 +1,82 @@
+package components
+
+import (
+	"context"
+	"testing"
+	"time"
+)
+
+func TestWaitForRunRetryStopsOnCancellation(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	started := time.Now()
+	if waitForRunRetry(ctx) {
+		t.Fatal("waitForRunRetry() completed the retry delay after cancellation")
+	}
+	if elapsed := time.Since(started); elapsed > 100*time.Millisecond {
+		t.Fatalf("waitForRunRetry() took %s after cancellation", elapsed)
+	}
+}
+
+func TestStreamRestartWatchdogCoalescesStallsAndBacksOff(t *testing.T) {
+	now := time.Unix(1_000, 0)
+	watchdog := newStreamRestartWatchdog()
+	if _, restart := watchdog.Observe(now, 1, 1, true, false); restart {
+		t.Fatal("initial observation requested a restart")
+	}
+
+	for check := 1; check < streamWatchdogStallChecks; check++ {
+		now = now.Add(streamWatchdogInterval)
+		if _, restart := watchdog.Observe(now, 1, 1, true, false); restart {
+			t.Fatalf("check %d requested an early restart", check)
+		}
+	}
+	now = now.Add(streamWatchdogInterval)
+	reason, restart := watchdog.Observe(now, 1, 1, true, false)
+	if !restart || reason != "main and sub streams" {
+		t.Fatalf("Observe() = (%q, %t), want coalesced restart", reason, restart)
+	}
+
+	watchdog.MarkRestart(now)
+	if got := watchdog.Backoff(); got != 30*time.Second {
+		t.Fatalf("Backoff() = %s, want 30s", got)
+	}
+	for now = now.Add(streamWatchdogInterval); now.Before(watchdog.nextRestart); now = now.Add(streamWatchdogInterval) {
+		if _, restart := watchdog.Observe(now, 1, 1, true, false); restart {
+			t.Fatal("restart requested during cooldown")
+		}
+	}
+}
+
+func TestStreamRestartWatchdogResetsAfterHealthyMinute(t *testing.T) {
+	now := time.Unix(2_000, 0)
+	watchdog := newStreamRestartWatchdog()
+	watchdog.backoff = streamWatchdogMaxBackoff
+	watchdog.nextRestart = now.Add(streamWatchdogMaxBackoff)
+	watchdog.Observe(now, 1, 1, true, false)
+
+	for elapsed := streamWatchdogInterval; elapsed <= streamWatchdogHealthyReset+streamWatchdogInterval; elapsed += streamWatchdogInterval {
+		now = now.Add(streamWatchdogInterval)
+		watchdog.Observe(now, int64(elapsed), int64(elapsed), true, false)
+	}
+
+	if got := watchdog.Backoff(); got != streamWatchdogBaseBackoff {
+		t.Fatalf("Backoff() = %s, want %s", got, streamWatchdogBaseBackoff)
+	}
+	if !watchdog.nextRestart.IsZero() {
+		t.Fatalf("nextRestart = %s, want zero", watchdog.nextRestart)
+	}
+}
+
+func TestStreamRestartWatchdogPausesWhileConfiguring(t *testing.T) {
+	now := time.Unix(3_000, 0)
+	watchdog := newStreamRestartWatchdog()
+	watchdog.Observe(now, 1, 0, false, false)
+	for check := 0; check < streamWatchdogStallChecks+1; check++ {
+		now = now.Add(streamWatchdogInterval)
+		if _, restart := watchdog.Observe(now, 1, 0, false, true); restart {
+			t.Fatal("restart requested while configuring")
+		}
+	}
+}

machinery/src/computervision/main.go

@@ -8,14 +8,14 @@ import (
 	geo "github.com/kellydunn/golang-geo"
 	"github.com/kerberos-io/agent/machinery/src/capture"
 	"github.com/kerberos-io/agent/machinery/src/conditions"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
+	log "github.com/sirupsen/logrus"
 )
 
 func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Configuration, communication *models.Communication, mqttClient mqtt.Client, rtspClient capture.RTSPClient) {
 
-	log.Log.Debug("computervision.main.ProcessMotion(): start motion detection")
+	log.Debug("computervision.main.ProcessMotion(): start motion detection")
 	config := configuration.Config
 	loc, _ := time.LoadLocation(config.Timezone)
 
@@ -42,14 +42,14 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 
 	if continuousMode && !hasMotionRegion {
 
-		log.Log.Info("computervision.main.ProcessMotion(): continuous recording enabled and no motion region configured, so no motion detection required.")
+		log.Info("computervision.main.ProcessMotion(): continuous recording enabled and no motion region configured, so no motion detection required.")
 
 	} else {
 
 		if continuousMode {
-			log.Log.Info("computervision.main.ProcessMotion(): continuous recording enabled with a motion region, running motion detection for live-view visualisation only (no motion-triggered recording).")
+			log.Info("computervision.main.ProcessMotion(): continuous recording enabled with a motion region, running motion detection for live-view visualisation only (no motion-triggered recording).")
 		} else {
-			log.Log.Info("computervision.main.ProcessMotion(): motion detected is enabled, so starting the motion detection.")
+			log.Info("computervision.main.ProcessMotion(): motion detected is enabled, so starting the motion detection.")
 		}
 
 		hubKey := config.HubKey
@@ -185,7 +185,7 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 				// We'll validate those conditions and if not valid we'll not do anything.
 				detectMotion, err := conditions.Validate(loc, configuration)
 				if !detectMotion && err != nil {
-					log.Log.Debug("computervision.main.ProcessMotion(): " + err.Error() + ".")
+					log.Debug("computervision.main.ProcessMotion(): " + err.Error() + ".")
 				}
 
 				// Run detection when motion is enabled, OR when we're in continuous
@@ -235,7 +235,7 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 										if err == nil {
 											mqttClient.Publish("kerberos/hub/"+hubKey, 2, false, payload)
 										} else {
-											log.Log.Info("computervision.main.ProcessMotion(): failed to package MQTT message: " + err.Error())
+											log.Info("computervision.main.ProcessMotion(): failed to package MQTT message: " + err.Error())
 										}
 									} else {
 										mqttClient.Publish("kerberos/agent/"+deviceKey, 2, false, "motion")
@@ -254,7 +254,9 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 									NumberOfChanges: changesToReturn,
 									Rectangle:       motionRectangle,
 								}
-								communication.HandleMotion <- dataToPass //Save data to the channel
+								if !communication.TrySendMotion(dataToPass) {
+									log.Warn("computervision.main.ProcessMotion(): motion channel unavailable or full, dropping recording trigger")
+								}
 							}
 						}
 					}
@@ -271,7 +273,7 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 		}
 	}
 
-	log.Log.Debug("computervision.main.ProcessMotion(): stop the motion detection.")
+	log.Debug("computervision.main.ProcessMotion(): stop the motion detection.")
 }
 
 func FindMotion(imageArray [3]*image.Gray, coordinatesPerRegion [][]int, pixelChangeThreshold int) (thresholdReached bool, changesDetected int, motionRectangle models.MotionRectangle, motionRectangles []models.MotionRectangle) {
@@ -374,14 +376,28 @@ func AbsDiffBitwiseAndThreshold(img1 *image.Gray, img2 *image.Gray, img3 *image.
 				endY = pixel[1]
 			}
 		}
-		log.Log.Debugf("Rectangle of changes detected: startX: %d, startY: %d, endX: %d, endY: %d", startX, startY, endX, endY)
+		log.WithFields(log.Fields{
+			"component": "computer_vision",
+			"end_x":     endX,
+			"end_y":     endY,
+			"event":     "motion_bounds_detected",
+			"start_x":   startX,
+			"start_y":   startY,
+		}).Debug("Motion bounds detected")
 		motionRectangle = models.MotionRectangle{
 			X:      startX,
 			Y:      startY,
 			Width:  endX - startX,
 			Height: endY - startY,
 		}
-		log.Log.Debugf("Motion rectangle: %+v", motionRectangle)
+		log.WithFields(log.Fields{
+			"component": "computer_vision",
+			"event":     "motion_rectangle_created",
+			"height":    motionRectangle.Height,
+			"width":     motionRectangle.Width,
+			"x":         motionRectangle.X,
+			"y":         motionRectangle.Y,
+		}).Debug("Motion rectangle created")
 	}
 
 	// Cluster the changed pixels into separate bounding boxes so the live view can

machinery/src/conditions/timewindow.go

@@ -3,8 +3,8 @@ package conditions
 import (
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 func IsWithinTimeInterval(loc *time.Location, configuration *models.Configuration) (enabled bool) {
@@ -27,9 +27,9 @@ func IsWithinTimeInterval(loc *time.Location, configuration *models.Configuratio
 				currentTimeInSeconds := hour*60*60 + minute*60 + second
 				if (currentTimeInSeconds >= start1 && currentTimeInSeconds <= end1) ||
 					(currentTimeInSeconds >= start2 && currentTimeInSeconds <= end2) {
-					log.Log.Debug("conditions.timewindow.IsWithinTimeInterval(): time interval valid, enabling recording.")
+					log.Debug("conditions.timewindow.IsWithinTimeInterval(): time interval valid, enabling recording.")
 				} else {
-					log.Log.Info("conditions.timewindow.IsWithinTimeInterval(): time interval not valid, disabling recording.")
+					log.Info("conditions.timewindow.IsWithinTimeInterval(): time interval not valid, disabling recording.")
 					enabled = false
 				}
 			}

machinery/src/conditions/uri.go

@@ -3,13 +3,26 @@ package conditions
 import (
 	"bytes"
 	"crypto/tls"
-	"fmt"
+	"encoding/json"
+	"io"
 	"net/http"
 	"os"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
+)
+
+const conditionHTTPTimeout = 10 * time.Second
+
+var (
+	conditionHTTPClient         = &http.Client{Timeout: conditionHTTPTimeout}
+	conditionInsecureHTTPClient = &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // #nosec G402 -- explicit operator opt-in
+		},
+		Timeout: conditionHTTPTimeout,
+	}
 )
 
 func IsValidUriResponse(configuration *models.Configuration) (enabled bool) {
@@ -17,41 +30,66 @@ func IsValidUriResponse(configuration *models.Configuration) (enabled bool) {
 	conditionURI := config.ConditionURI
 	enabled = true
 	if conditionURI != "" {
-
+		client := conditionHTTPClient
-		// We will send a POST request to the conditionURI, and expect a 200 response.
-		// In the payload we will send some information, so the other end can decide
-		// if it should enable or disable recording.
-
-		var client *http.Client
 		if os.Getenv("AGENT_TLS_INSECURE") == "true" {
-			tr := &http.Transport{
+			client = conditionInsecureHTTPClient
-				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-			}
-			client = &http.Client{Transport: tr}
-		} else {
-			client = &http.Client{}
 		}
 
-		var object = fmt.Sprintf(`{
+		payload := struct {
-			"camera_id" : "%s",
+			CameraID   string `json:"camera_id"`
-			"camera_name" : "%s",
+			CameraName string `json:"camera_name"`
-			"site_id" : "%s",
+			SiteID     string `json:"site_id"`
-			"hub_key" : "%s",
+			HubKey     string `json:"hub_key"`
-			"timestamp" : "%s",
+			Timestamp  string `json:"timestamp"`
-		}`, config.Key, config.FriendlyName, config.HubSite, config.HubKey, time.Now().Format("2006-01-02 15:04:05"))
+		}{
+			CameraID:   config.Key,
+			CameraName: config.FriendlyName,
+			SiteID:     config.HubSite,
+			HubKey:     config.HubKey,
+			Timestamp:  time.Now().Format("2006-01-02 15:04:05"),
+		}
+		jsonBody, err := json.Marshal(payload)
+		if err != nil {
+			log.WithError(err).WithFields(log.Fields{
+				"component": "conditions/uri",
+				"event":     "request_encoding_failed",
+			}).Error("Failed to encode condition request")
+			return false
+		}
 
-		var jsonStr = []byte(object)
+		req, err := http.NewRequest(http.MethodPost, conditionURI, bytes.NewReader(jsonBody))
-		buffy := bytes.NewBuffer(jsonStr)
+		if err != nil {
-		req, _ := http.NewRequest("POST", conditionURI, buffy)
+			log.WithError(err).WithFields(log.Fields{
+				"component": "conditions/uri",
+				"event":     "request_creation_failed",
+			}).Error("Failed to create condition request")
+			return false
+		}
 		req.Header.Set("Content-Type", "application/json")
 		resp, err := client.Do(req)
 		if resp != nil {
+			_, _ = io.Copy(io.Discard, resp.Body)
 			resp.Body.Close()
 		}
-		if err == nil && resp.StatusCode == 200 {
+		if err == nil && resp != nil && resp.StatusCode == http.StatusOK {
-			log.Log.Info("conditions.uri.IsValidUriResponse(): response 200, enabling recording.")
+			log.WithFields(log.Fields{
+				"component":   "conditions/uri",
+				"event":       "recording_enabled",
+				"status_code": resp.StatusCode,
+			}).Info("Condition request enabled recording")
 		} else {
-			log.Log.Info("conditions.uri.IsValidUriResponse(): response not 200, disabling recording.")
+			if err != nil {
+				log.WithError(err).WithFields(log.Fields{
+					"component": "conditions/uri",
+					"event":     "request_failed",
+				}).Error("Condition request failed")
+			} else {
+				log.WithFields(log.Fields{
+					"component":   "conditions/uri",
+					"event":       "recording_disabled",
+					"status_code": resp.StatusCode,
+				}).Info("Condition request disabled recording")
+			}
 			enabled = false
 		}
 	}

machinery/src/conditions/uri_test.go

@@ -0,0 +1,63 @@
+package conditions
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+
+	"github.com/kerberos-io/agent/machinery/src/models"
+)
+
+func TestIsValidUriResponseReusesClientAndEncodesPayload(t *testing.T) {
+	var requests atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		requests.Add(1)
+		if request.Method != http.MethodPost {
+			t.Errorf("method = %s, want POST", request.Method)
+		}
+		if got := request.Header.Get("Content-Type"); got != "application/json" {
+			t.Errorf("Content-Type = %q, want application/json", got)
+		}
+		var payload struct {
+			CameraID   string `json:"camera_id"`
+			CameraName string `json:"camera_name"`
+		}
+		if err := json.NewDecoder(request.Body).Decode(&payload); err != nil {
+			t.Errorf("decode request: %v", err)
+		}
+		if payload.CameraID != "camera-1" || payload.CameraName != `Front "Door"` {
+			t.Errorf("payload = %+v", payload)
+		}
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer server.Close()
+
+	configuration := &models.Configuration{}
+	configuration.Config.ConditionURI = server.URL
+	configuration.Config.Key = "camera-1"
+	configuration.Config.FriendlyName = `Front "Door"`
+
+	for i := 0; i < 2; i++ {
+		if !IsValidUriResponse(configuration) {
+			t.Fatal("IsValidUriResponse() = false, want true")
+		}
+	}
+	if got := requests.Load(); got != 2 {
+		t.Fatalf("requests = %d, want 2", got)
+	}
+}
+
+func TestIsValidUriResponseRejectsNonOK(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		http.Error(w, "unavailable", http.StatusServiceUnavailable)
+	}))
+	defer server.Close()
+
+	configuration := &models.Configuration{}
+	configuration.Config.ConditionURI = server.URL
+	if IsValidUriResponse(configuration) {
+		t.Fatal("IsValidUriResponse() = true, want false")
+	}
+}

machinery/src/config/main.go

@@ -13,11 +13,36 @@ import (
 
 	"github.com/InVisionApp/conjungo"
 	"github.com/kerberos-io/agent/machinery/src/database"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
 )
 
+const factoryConfigRetryDelay = 5 * time.Second
+
+func newFactoryConfigReadContext() (context.Context, context.CancelFunc) {
+	return context.WithTimeout(context.Background(), database.TIMEOUT)
+}
+
+func isRetryableFactoryConfigReadError(err error) bool {
+	return errors.Is(err, context.DeadlineExceeded) || mongo.IsTimeout(err) || mongo.IsNetworkError(err)
+}
+
+func readFactoryConfig(collection *mongo.Collection, filter bson.M, destination *models.Config) error {
+	for {
+		ctx, cancel := newFactoryConfigReadContext()
+		err := collection.FindOne(ctx, filter).Decode(destination)
+		cancel()
+		if !isRetryableFactoryConfigReadError(err) {
+			return err
+		}
+
+		log.Warn("Factory configuration read timed out or lost its database connection; retrying.")
+		time.Sleep(factoryConfigRetryDelay)
+	}
+}
+
 // ReadUserConfig Reads the user configuration of the Kerberos Open Source instance.
 // This will return a models.User struct including the username, password,
 // selected language, and if the installation was completed or not.
@@ -25,14 +50,14 @@ func ReadUserConfig(configDirectory string) (userConfig models.User) {
 	for {
 		jsonFile, err := os.Open(configDirectory + "/data/config/user.json")
 		if err != nil {
-			log.Log.Error("Config file is not found " + configDirectory + "/data/config/user.json, trying again in 5s: " + err.Error())
+			log.Error("Config file is not found " + configDirectory + "/data/config/user.json, trying again in 5s: " + err.Error())
 			time.Sleep(5 * time.Second)
 		} else {
-			log.Log.Info("Successfully Opened user.json")
+			log.Info("Successfully Opened user.json")
 			byteValue, _ := ioutil.ReadAll(jsonFile)
 			err = json.Unmarshal(byteValue, &userConfig)
 			if err != nil {
-				log.Log.Error("JSON file not valid: " + err.Error())
+				log.Error("JSON file not valid: " + err.Error())
 			} else {
 				jsonFile.Close()
 				break
@@ -63,21 +88,15 @@ func OpenConfig(configDirectory string, configuration *models.Configuration) {
 		collection := db.Collection("configuration")
 
 		var globalConfig models.Config
-		res := collection.FindOne(context.Background(), bson.M{
+		err := readFactoryConfig(collection, bson.M{
 			"type": "global",
-		})
+		}, &globalConfig)
-
-		if res.Err() != nil {
-			log.Log.Error("Could not find global configuration, using default configuration.")
-			panic("Could not find global configuration, using default configuration.")
-		}
-		err := res.Decode(&globalConfig)
 		if err != nil {
-			log.Log.Error("Could not find global configuration, using default configuration.")
+			log.Error("Could not find global configuration, using default configuration.")
 			panic("Could not find global configuration, using default configuration.")
 		}
 		if globalConfig.Type != "global" {
-			log.Log.Error("Could not find global configuration, might missed the mongodb connection.")
+			log.Error("Could not find global configuration, might missed the mongodb connection.")
 			panic("Could not find global configuration, might missed the mongodb connection.")
 		}
 
@@ -85,21 +104,16 @@ func OpenConfig(configDirectory string, configuration *models.Configuration) {
 
 		var customConfig models.Config
 		deploymentName := os.Getenv("DEPLOYMENT_NAME")
-		res = collection.FindOne(context.Background(), bson.M{
+		err = readFactoryConfig(collection, bson.M{
 			"type": "config",
 			"name": deploymentName,
-		})
+		}, &customConfig)
-		if res.Err() != nil {
-			log.Log.Error("Could not find configuration for " + deploymentName + ", using global configuration.")
-		}
-		err = res.Decode(&customConfig)
 		if err != nil {
-			log.Log.Error("Could not find configuration for " + deploymentName + ", using global configuration.")
+			log.Error("Could not find configuration for " + deploymentName + ", using global configuration.")
-		}
+			customConfig = models.Config{}
-
+		} else if customConfig.Type != "config" {
-		if customConfig.Type != "config" {
+			log.Error("Custom configuration has an invalid type, using global configuration.")
-			log.Log.Error("Could not find custom configuration, might missed the mongodb connection.")
+			customConfig = models.Config{}
-			panic("Could not find custom configuration, might missed the mongodb connection.")
 		}
 		configuration.CustomConfig = customConfig
 
@@ -166,21 +180,29 @@ func OpenConfig(configDirectory string, configuration *models.Configuration) {
 
 		// Open device config
 		for {
-			jsonFile, err := os.Open(configDirectory + "/data/config/config.json")
+			configPath := configDirectory + "/data/config/config.json"
+			jsonFile, err := os.Open(configPath)
 			if err != nil {
-				log.Log.Error("Config file is not found " + configDirectory + "/data/config/config.json" + ", trying again in 5s.")
+				log.WithError(err).WithFields(log.Fields{
+					"component":      "config",
+					"event":          "configuration_open_failed",
+					"retry_delay_ms": 5000,
+				}).Warn("Configuration file unavailable; retrying")
 				time.Sleep(5 * time.Second)
 			} else {
-				log.Log.Info("Successfully Opened config.json from " + configuration.Name)
+				log.WithFields(log.Fields{
+					"component": "config",
+					"event":     "configuration_opened",
+				}).Info("Configuration file opened")
 				byteValue, _ := ioutil.ReadAll(jsonFile)
 				err = json.Unmarshal(byteValue, &configuration.Config)
 				jsonFile.Close()
 				if err != nil {
-					log.Log.Error("JSON file not valid: " + err.Error())
+					log.Error("JSON file not valid: " + err.Error())
 				} else {
 					err = json.Unmarshal(byteValue, &configuration.CustomConfig)
 					if err != nil {
-						log.Log.Error("JSON file not valid: " + err.Error())
+						log.Error("JSON file not valid: " + err.Error())
 					} else {
 						break
 					}
@@ -687,12 +709,12 @@ func SaveConfig(configDirectory string, config models.Config, configuration *mod
 			return err
 		}
 
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			select {
 			case communication.HandleBootstrap <- "restart":
-				log.Log.Info("config.main.SaveConfig(): update config, restart agent.")
+				log.Info("config.main.SaveConfig(): update config, restart agent.")
 			case <-time.After(1 * time.Second):
-				log.Log.Info("config.main.SaveConfig(): update config, restart agent.")
+				log.Info("config.main.SaveConfig(): update config, restart agent.")
 			}
 		}
 

machinery/src/config/main_test.go

@@ -1,9 +1,14 @@
 package config
 
 import (
+	"context"
+	"errors"
 	"testing"
+	"time"
 
+	"github.com/kerberos-io/agent/machinery/src/database"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	"go.mongodb.org/mongo-driver/mongo"
 )
 
 func TestApplyAgentEnvVarsPixelChangeThresholdDefault(t *testing.T) {
@@ -38,3 +43,36 @@ func TestApplyAgentEnvVarsPixelChangeThresholdDefault(t *testing.T) {
 func intPointer(value int) *int {
 	return &value
 }
+
+func TestNewFactoryConfigReadContextUsesDatabaseTimeout(t *testing.T) {
+	ctx, cancel := newFactoryConfigReadContext()
+	defer cancel()
+
+	deadline, ok := ctx.Deadline()
+	if !ok {
+		t.Fatal("expected read context deadline")
+	}
+
+	remaining := time.Until(deadline)
+	if remaining <= 0 {
+		t.Fatalf("deadline already expired: %v", remaining)
+	}
+	if remaining > database.TIMEOUT {
+		t.Fatalf("remaining deadline = %v, want <= %v", remaining, database.TIMEOUT)
+	}
+	if remaining < database.TIMEOUT-time.Second {
+		t.Fatalf("remaining deadline = %v, want close to %v", remaining, database.TIMEOUT)
+	}
+}
+
+func TestFactoryConfigRetryableErrors(t *testing.T) {
+	if !isRetryableFactoryConfigReadError(context.DeadlineExceeded) {
+		t.Fatal("context deadline should be retryable")
+	}
+	if isRetryableFactoryConfigReadError(mongo.ErrNoDocuments) {
+		t.Fatal("missing configuration should not retry")
+	}
+	if isRetryableFactoryConfigReadError(errors.New("invalid BSON")) {
+		t.Fatal("decode errors should not retry")
+	}
+}

machinery/src/database/main.go

@@ -7,6 +7,7 @@ import (
 	"sync"
 	"time"
 
+	log "github.com/sirupsen/logrus"
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
 )
@@ -44,18 +45,34 @@ func New() *DB {
 		// We can also apply the complete URI
 		// e.g. "mongodb+srv://<username>:<password>@kerberos-hub.shhng.mongodb.net/?retryWrites=true&w=majority&appName=kerberos-hub"
 		if mongodbURI != "" {
+			log.WithFields(log.Fields{
+				"component":      "database",
+				"database":       DatabaseName,
+				"event":          "client_configuring",
+				"uri_configured": true,
+			}).Debug("Configuring MongoDB client")
 			serverAPI := options.ServerAPI(options.ServerAPIVersion1)
 			opts := options.Client().ApplyURI(mongodbURI).SetServerAPIOptions(serverAPI)
 
 			// Create a new client and connect to the server
 			client, err := mongo.Connect(ctx, opts)
 			if err != nil {
-				fmt.Printf("Error setting up mongodb connection: %+v\n", err)
+				log.WithError(err).WithFields(log.Fields{
-				os.Exit(1)
+					"component": "database",
+					"database":  DatabaseName,
+					"event":     "client_configuration_failed",
+				}).Fatal("Failed to configure MongoDB client")
 			}
 			_instance.Client = client
 
 		} else {
+			log.WithFields(log.Fields{
+				"component":             "database",
+				"database":              DatabaseName,
+				"event":                 "client_configuring",
+				"replicaset_configured": replicaset != "",
+				"uri_configured":        false,
+			}).Debug("Configuring MongoDB client")
 
 			// New MongoDB driver
 			mongodbURI := fmt.Sprintf("mongodb://%s:%s@%s", username, password, host)
@@ -69,8 +86,11 @@ func New() *DB {
 				Password:      password,
 			}))
 			if err != nil {
-				fmt.Printf("Error setting up mongodb connection: %+v\n", err)
+				log.WithError(err).WithFields(log.Fields{
-				os.Exit(1)
+					"component": "database",
+					"database":  DatabaseName,
+					"event":     "client_configuration_failed",
+				}).Fatal("Failed to configure MongoDB client")
 			}
 			_instance.Client = client
 		}

machinery/src/lifecycle/supervisor.go

@@ -0,0 +1,336 @@
+package lifecycle
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"runtime/debug"
+	"sort"
+	"sync"
+	"time"
+)
+
+var (
+	ErrSupervisorSealed  = errors.New("supervisor is sealed")
+	ErrSupervisorStopped = errors.New("supervisor is shutting down")
+	ErrTaskExists        = errors.New("task already exists")
+)
+
+type TaskFunc func(context.Context) error
+
+type TaskPolicy struct {
+	Required    bool
+	LongRunning bool
+}
+
+type TaskStatus string
+
+const (
+	TaskRunning   TaskStatus = "running"
+	TaskSucceeded TaskStatus = "succeeded"
+	TaskFailed    TaskStatus = "failed"
+	TaskPanicked  TaskStatus = "panicked"
+)
+
+type TaskSnapshot struct {
+	Name      string
+	Policy    TaskPolicy
+	Status    TaskStatus
+	StartedAt time.Time
+	EndedAt   time.Time
+	Error     string
+	Panic     string
+	Stack     []byte
+}
+
+type Failure struct {
+	Task  string
+	Cause error
+	Panic string
+	Stack []byte
+}
+
+func (f Failure) Error() string {
+	if f.Cause == nil {
+		return fmt.Sprintf("task %q failed", f.Task)
+	}
+	return f.Cause.Error()
+}
+
+type ShutdownReport struct {
+	Complete bool
+	Cause    error
+	Tasks    []TaskSnapshot
+	Running  []TaskSnapshot
+}
+
+type supervisorPhase uint8
+
+const (
+	phaseStarting supervisorPhase = iota
+	phaseRunning
+	phaseDraining
+)
+
+type taskState struct {
+	TaskSnapshot
+}
+
+type Supervisor struct {
+	ctx    context.Context
+	cancel context.CancelCauseFunc
+
+	mu        sync.Mutex
+	phase     supervisorPhase
+	sealed    bool
+	active    int
+	tasks     map[string]*taskState
+	allDone   chan struct{}
+	doneOnce  sync.Once
+	start     chan struct{}
+	startOnce sync.Once
+
+	failures    chan Failure
+	failureOnce sync.Once
+
+	stopParentCancel func() bool
+}
+
+func NewSupervisor(parent context.Context) *Supervisor {
+	if parent == nil {
+		parent = context.Background()
+	}
+
+	ctx, cancel := context.WithCancelCause(parent)
+	supervisor := &Supervisor{
+		ctx:      ctx,
+		cancel:   cancel,
+		phase:    phaseStarting,
+		tasks:    make(map[string]*taskState),
+		allDone:  make(chan struct{}),
+		start:    make(chan struct{}),
+		failures: make(chan Failure, 1),
+	}
+
+	stopParentCancel := context.AfterFunc(parent, func() {
+		supervisor.BeginShutdown(context.Cause(parent))
+	})
+	supervisor.mu.Lock()
+	if supervisor.sealed && supervisor.active == 0 {
+		supervisor.mu.Unlock()
+		stopParentCancel()
+	} else {
+		supervisor.stopParentCancel = stopParentCancel
+		supervisor.mu.Unlock()
+	}
+
+	return supervisor
+}
+
+func (s *Supervisor) Context() context.Context {
+	return s.ctx
+}
+
+func (s *Supervisor) Failures() <-chan Failure {
+	return s.failures
+}
+
+func (s *Supervisor) Go(name string, policy TaskPolicy, task TaskFunc) error {
+	if name == "" {
+		return errors.New("task name is required")
+	}
+	if task == nil {
+		return errors.New("task function is required")
+	}
+
+	s.mu.Lock()
+	if s.sealed {
+		s.mu.Unlock()
+		return fmt.Errorf("%w: %s", ErrSupervisorSealed, name)
+	}
+	if s.ctx.Err() != nil {
+		s.mu.Unlock()
+		return fmt.Errorf("%w: %s", ErrSupervisorStopped, name)
+	}
+	if _, exists := s.tasks[name]; exists {
+		s.mu.Unlock()
+		return fmt.Errorf("%w: %s", ErrTaskExists, name)
+	}
+
+	state := &taskState{TaskSnapshot: TaskSnapshot{
+		Name:      name,
+		Policy:    policy,
+		Status:    TaskRunning,
+		StartedAt: time.Now(),
+	}}
+	s.tasks[name] = state
+	s.active++
+	s.mu.Unlock()
+
+	go s.run(state, task)
+	return nil
+}
+
+func (s *Supervisor) run(state *taskState, task TaskFunc) {
+	var taskErr error
+	panicked := true
+	var panicValue string
+	var panicStack []byte
+
+	defer func() {
+		recovered := recover()
+		if panicked {
+			panicValue = fmt.Sprint(recovered)
+			panicStack = debug.Stack()
+			taskErr = fmt.Errorf("task %q panicked: %s", state.Name, panicValue)
+		}
+		s.finish(state, taskErr, panicked, panicValue, panicStack)
+	}()
+
+	select {
+	case <-s.start:
+	case <-s.ctx.Done():
+		panicked = false
+		return
+	}
+
+	taskErr = task(s.ctx)
+	panicked = false
+}
+
+func (s *Supervisor) finish(
+	state *taskState,
+	taskErr error,
+	panicked bool,
+	panicValue string,
+	panicStack []byte,
+) {
+	var failure *Failure
+
+	s.mu.Lock()
+	state.EndedAt = time.Now()
+	state.Error = ""
+	state.Panic = panicValue
+	state.Stack = append([]byte(nil), panicStack...)
+	switch {
+	case panicked:
+		state.Status = TaskPanicked
+		state.Error = taskErr.Error()
+	case taskErr != nil:
+		state.Status = TaskFailed
+		state.Error = taskErr.Error()
+	default:
+		state.Status = TaskSucceeded
+	}
+
+	shuttingDown := s.phase == phaseDraining || s.ctx.Err() != nil
+	unexpected := panicked || (!shuttingDown &&
+		(state.Policy.Required && (taskErr != nil || state.Policy.LongRunning)))
+	if unexpected {
+		cause := taskErr
+		if cause == nil {
+			cause = fmt.Errorf("required long-running task %q exited", state.Name)
+		}
+		failure = &Failure{
+			Task:  state.Name,
+			Cause: cause,
+			Panic: panicValue,
+			Stack: append([]byte(nil), panicStack...),
+		}
+		s.phase = phaseDraining
+		s.sealed = true
+		s.cancel(failure.Cause)
+		s.failureOnce.Do(func() {
+			s.failures <- *failure
+		})
+	}
+
+	s.active--
+	s.closeDoneIfReadyLocked()
+	s.mu.Unlock()
+}
+
+func (s *Supervisor) Seal() {
+	s.mu.Lock()
+	s.sealed = true
+	if s.phase == phaseStarting {
+		s.phase = phaseRunning
+	}
+	s.startOnce.Do(func() {
+		close(s.start)
+	})
+	s.closeDoneIfReadyLocked()
+	s.mu.Unlock()
+}
+
+func (s *Supervisor) BeginShutdown(cause error) {
+	if cause == nil {
+		cause = context.Canceled
+	}
+
+	s.mu.Lock()
+	s.phase = phaseDraining
+	s.sealed = true
+	s.cancel(cause)
+	s.closeDoneIfReadyLocked()
+	s.mu.Unlock()
+}
+
+func (s *Supervisor) Wait(ctx context.Context) ShutdownReport {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+
+	select {
+	case <-s.allDone:
+	case <-ctx.Done():
+	}
+
+	return s.report()
+}
+
+func (s *Supervisor) Snapshot() []TaskSnapshot {
+	return s.report().Tasks
+}
+
+func (s *Supervisor) closeDoneIfReadyLocked() {
+	if !s.sealed || s.active != 0 {
+		return
+	}
+	if s.stopParentCancel != nil {
+		s.stopParentCancel()
+		s.stopParentCancel = nil
+	}
+	s.doneOnce.Do(func() {
+		close(s.allDone)
+	})
+}
+
+func (s *Supervisor) report() ShutdownReport {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	tasks := make([]TaskSnapshot, 0, len(s.tasks))
+	running := make([]TaskSnapshot, 0)
+	for _, state := range s.tasks {
+		snapshot := state.TaskSnapshot
+		snapshot.Stack = append([]byte(nil), state.Stack...)
+		tasks = append(tasks, snapshot)
+		if snapshot.Status == TaskRunning {
+			running = append(running, snapshot)
+		}
+	}
+	sort.Slice(tasks, func(i, j int) bool {
+		return tasks[i].Name < tasks[j].Name
+	})
+	sort.Slice(running, func(i, j int) bool {
+		return running[i].Name < running[j].Name
+	})
+
+	return ShutdownReport{
+		Complete: s.sealed && s.active == 0,
+		Cause:    context.Cause(s.ctx),
+		Tasks:    tasks,
+		Running:  running,
+	}
+}

machinery/src/lifecycle/supervisor_test.go

@@ -0,0 +1,300 @@
+package lifecycle
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+func TestSupervisorWaitsForSeal(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	started := make(chan struct{})
+	if err := supervisor.Go("quick", TaskPolicy{}, func(context.Context) error {
+		close(started)
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	waitContext, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+	defer cancel()
+	if report := supervisor.Wait(waitContext); report.Complete {
+		t.Fatal("Wait() completed before Seal()")
+	}
+	select {
+	case <-started:
+		t.Fatal("task started before Seal()")
+	default:
+	}
+
+	supervisor.Seal()
+	<-started
+	if report := supervisor.Wait(context.Background()); !report.Complete {
+		t.Fatal("Wait() did not complete after Seal()")
+	}
+}
+
+func TestSupervisorSealsEmptyGroup(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	supervisor.Seal()
+
+	report := supervisor.Wait(context.Background())
+	if !report.Complete || len(report.Tasks) != 0 {
+		t.Fatalf("Wait() report = %+v, want complete empty report", report)
+	}
+}
+
+func TestSupervisorReportsRunningTasksOnTimeout(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	release := make(chan struct{})
+	if err := supervisor.Go("blocked", TaskPolicy{}, func(context.Context) error {
+		<-release
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+
+	waitContext, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+	defer cancel()
+	report := supervisor.Wait(waitContext)
+	if report.Complete {
+		t.Fatal("Wait() unexpectedly completed")
+	}
+	if len(report.Running) != 1 || report.Running[0].Name != "blocked" {
+		t.Fatalf("Wait() running tasks = %+v, want blocked", report.Running)
+	}
+
+	close(release)
+	if report = supervisor.Wait(context.Background()); !report.Complete {
+		t.Fatal("Wait() did not complete after releasing task")
+	}
+}
+
+func TestSupervisorFailsWhenRequiredLongRunningTaskExits(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	release := make(chan struct{})
+	if err := supervisor.Go("recorder", TaskPolicy{Required: true, LongRunning: true}, func(context.Context) error {
+		<-release
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+	close(release)
+
+	select {
+	case failure := <-supervisor.Failures():
+		if failure.Task != "recorder" || !strings.Contains(failure.Error(), "exited") {
+			t.Fatalf("failure = %+v", failure)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("required task exit did not emit a failure")
+	}
+}
+
+func TestSupervisorAllowsRequiredOneShotTaskToSucceed(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	if err := supervisor.Go("setup", TaskPolicy{Required: true}, func(context.Context) error {
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+	if report := supervisor.Wait(context.Background()); !report.Complete {
+		t.Fatal("Wait() did not complete")
+	}
+
+	select {
+	case failure := <-supervisor.Failures():
+		t.Fatalf("successful one-shot task emitted failure: %+v", failure)
+	default:
+	}
+}
+
+func TestSupervisorFailsWhenRequiredOneShotTaskReturnsError(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	wantErr := errors.New("setup failed")
+	if err := supervisor.Go("setup", TaskPolicy{Required: true}, func(context.Context) error {
+		return wantErr
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+
+	select {
+	case failure := <-supervisor.Failures():
+		if !errors.Is(failure.Cause, wantErr) {
+			t.Fatalf("failure cause = %v, want %v", failure.Cause, wantErr)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("required task error did not emit a failure")
+	}
+}
+
+func TestSupervisorRecoversPanicAndCapturesStack(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	if err := supervisor.Go("panic", TaskPolicy{}, func(context.Context) error {
+		panic("boom")
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+
+	select {
+	case failure := <-supervisor.Failures():
+		if failure.Task != "panic" || failure.Panic != "boom" || len(failure.Stack) == 0 {
+			t.Fatalf("failure = %+v, want captured panic and stack", failure)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("panic did not emit a failure")
+	}
+
+	report := supervisor.Wait(context.Background())
+	if !report.Complete || len(report.Tasks) != 1 || len(report.Tasks[0].Stack) == 0 {
+		t.Fatalf("report = %+v, want completed task with captured stack", report)
+	}
+}
+
+func TestSupervisorCapturesEmptyPanicDuringShutdown(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	started := make(chan struct{})
+	if err := supervisor.Go("panic", TaskPolicy{}, func(ctx context.Context) error {
+		close(started)
+		<-ctx.Done()
+		panic("")
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+	<-started
+	supervisor.BeginShutdown(errors.New("requested shutdown"))
+
+	select {
+	case failure := <-supervisor.Failures():
+		if failure.Task != "panic" || len(failure.Stack) == 0 {
+			t.Fatalf("failure = %+v, want captured panic and stack", failure)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("panic during shutdown did not emit a failure")
+	}
+	if report := supervisor.Wait(context.Background()); !report.Complete || report.Tasks[0].Status != TaskPanicked {
+		t.Fatalf("report = %+v, want completed panicked task", report)
+	}
+}
+
+func TestSupervisorPublishesFailureBeforeCompletion(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	wantErr := errors.New("setup failed")
+	if err := supervisor.Go("setup", TaskPolicy{Required: true}, func(context.Context) error {
+		return wantErr
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+
+	report := supervisor.Wait(context.Background())
+	if !report.Complete || !errors.Is(report.Cause, wantErr) {
+		t.Fatalf("report = %+v, want complete report caused by task failure", report)
+	}
+	select {
+	case failure := <-supervisor.Failures():
+		if !errors.Is(failure.Cause, wantErr) {
+			t.Fatalf("failure cause = %v, want %v", failure.Cause, wantErr)
+		}
+	default:
+		t.Fatal("failure was not published before completion")
+	}
+}
+
+func TestSupervisorCompletedWaitWinsOverCanceledContext(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	supervisor.Seal()
+
+	waitContext, cancel := context.WithCancel(context.Background())
+	cancel()
+	if report := supervisor.Wait(waitContext); !report.Complete {
+		t.Fatalf("report = %+v, want authoritative completion state", report)
+	}
+}
+
+func TestSupervisorRejectsTaskAfterParentCancellation(t *testing.T) {
+	parent, cancel := context.WithCancel(context.Background())
+	supervisor := NewSupervisor(parent)
+	cancel()
+
+	err := supervisor.Go("late", TaskPolicy{}, func(context.Context) error { return nil })
+	if !errors.Is(err, ErrSupervisorStopped) && !errors.Is(err, ErrSupervisorSealed) {
+		t.Fatalf("Go() error = %v, want stopped or sealed supervisor", err)
+	}
+	supervisor.BeginShutdown(context.Canceled)
+	if report := supervisor.Wait(context.Background()); !report.Complete {
+		t.Fatalf("report = %+v, want complete canceled supervisor", report)
+	}
+}
+
+func TestSupervisorShutdownBeforeSealDoesNotStartTasks(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	started := make(chan struct{})
+	if err := supervisor.Go("worker", TaskPolicy{}, func(context.Context) error {
+		close(started)
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	supervisor.BeginShutdown(errors.New("startup aborted"))
+	if report := supervisor.Wait(context.Background()); !report.Complete {
+		t.Fatalf("report = %+v, want completed aborted startup", report)
+	}
+	select {
+	case <-started:
+		t.Fatal("task started after startup was aborted")
+	default:
+	}
+}
+
+func TestSupervisorRejectsDuplicateAndLateTasks(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	release := make(chan struct{})
+	if err := supervisor.Go("worker", TaskPolicy{}, func(context.Context) error {
+		<-release
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := supervisor.Go("worker", TaskPolicy{}, func(context.Context) error { return nil }); !errors.Is(err, ErrTaskExists) {
+		t.Fatalf("duplicate Go() error = %v, want ErrTaskExists", err)
+	}
+
+	supervisor.Seal()
+	if err := supervisor.Go("late", TaskPolicy{}, func(context.Context) error { return nil }); !errors.Is(err, ErrSupervisorSealed) {
+		t.Fatalf("late Go() error = %v, want ErrSupervisorSealed", err)
+	}
+	close(release)
+}
+
+func TestSupervisorConcurrentStartAndSeal(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	var starters sync.WaitGroup
+	for index := 0; index < 100; index++ {
+		starters.Add(1)
+		go func(index int) {
+			defer starters.Done()
+			_ = supervisor.Go(
+				"worker-"+time.Unix(0, int64(index)).Format("150405.000000000"),
+				TaskPolicy{},
+				func(context.Context) error { return nil },
+			)
+		}(index)
+	}
+
+	supervisor.Seal()
+	starters.Wait()
+	if report := supervisor.Wait(context.Background()); !report.Complete {
+		t.Fatal("Wait() did not complete after concurrent Start and Seal")
+	}
+}

machinery/src/log/main.go

@@ -1,179 +0,0 @@
-package log
-
-import (
-	"os"
-	"time"
-
-	"github.com/op/go-logging"
-	"github.com/sirupsen/logrus"
-	"gopkg.in/natefinch/lumberjack.v2"
-)
-
-// The logging library being used everywhere.
-var Log = Logging{
-	Logger: "logrus",
-}
-
-// -----------------
-// This a gologging
-// -> github.com/op/go-logging
-
-var gologging = logging.MustGetLogger("gologger")
-
-func ConfigureGoLogging(configDirectory string, timezone *time.Location) {
-	// Logging
-	var format = logging.MustStringFormatter(
-		`%{color}%{time:15:04:05.000} %{shortfunc} ▶ %{level:.4s} %{id:03x}%{color:reset} %{message}`,
-	)
-	var format2 = logging.MustStringFormatter(
-		`%{time:15:04:05.000} %{shortfunc} ▶ %{level:.4s}  %{id:03x}%{color:reset} %{message}`,
-	)
-	stdBackend := logging.NewLogBackend(os.Stderr, "", 0)
-	stdBackendLeveled := logging.NewBackendFormatter(stdBackend, format)
-	fileBackend := logging.NewLogBackend(&lumberjack.Logger{
-		Filename: configDirectory + "/data/log/machinery.txt",
-		MaxSize:  2,    // megabytes
-		Compress: true, // disabled by default
-	}, "", 0)
-	fileBackendLeveled := logging.NewBackendFormatter(fileBackend, format2)
-	logging.SetBackend(stdBackendLeveled, fileBackendLeveled)
-	logging.SetLevel(logging.DEBUG, "")
-}
-
-// -----------------
-// This a logrus
-// -> github.com/sirupsen/logrus
-
-func ConfigureLogrus(level string, output string, timezone *time.Location) {
-
-	if output == "json" {
-		// Log as JSON instead of the default ASCII formatter.
-		logrus.SetFormatter(LocalTimeZoneFormatter{
-			Timezone:  timezone,
-			Formatter: &logrus.JSONFormatter{},
-		})
-	} else if output == "text" {
-		// Log as text with colors.
-		formatter := logrus.TextFormatter{
-			ForceColors:   true,
-			FullTimestamp: true,
-		}
-		logrus.SetFormatter(LocalTimeZoneFormatter{
-			Timezone:  timezone,
-			Formatter: &formatter,
-		})
-	}
-
-	// Use local timezone for providing datetime in logs!
-
-	// Output to stdout instead of the default stderr
-	// Can be any io.Writer, see below for File example
-	logrus.SetOutput(os.Stdout)
-
-	// Only log the warning severity or above.
-	logLevel := logrus.InfoLevel
-	if level == "error" {
-		logLevel = logrus.ErrorLevel
-	} else if level == "debug" {
-		logLevel = logrus.DebugLevel
-	} else if level == "fatal" {
-		logLevel = logrus.FatalLevel
-	} else if level == "warning" {
-		logLevel = logrus.WarnLevel
-	} // Add this line for logging filename and line number!
-	logrus.SetLevel(logLevel)
-}
-
-type LocalTimeZoneFormatter struct {
-	Timezone  *time.Location
-	Formatter logrus.Formatter
-}
-
-func (u LocalTimeZoneFormatter) Format(e *logrus.Entry) ([]byte, error) {
-	e.Time = e.Time.In(u.Timezone)
-	return u.Formatter.Format(e)
-}
-
-type Logging struct {
-	Logger string
-}
-
-func (self *Logging) Init(level string, logoutput string, configDirectory string, timezone *time.Location) {
-	switch self.Logger {
-	case "go-logging":
-		ConfigureGoLogging(configDirectory, timezone)
-	case "logrus":
-		ConfigureLogrus(level, logoutput, timezone)
-	default:
-	}
-}
-
-func (self *Logging) Info(sentence string) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Info(sentence)
-	case "logrus":
-		logrus.Info(sentence)
-	default:
-	}
-}
-
-func (self *Logging) Infof(format string, args ...interface{}) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Infof(format, args...)
-	case "logrus":
-		logrus.Infof(format, args...)
-	default:
-	}
-}
-
-func (self *Logging) Warning(sentence string) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Warning(sentence)
-	case "logrus":
-		logrus.Warn(sentence)
-	default:
-	}
-}
-
-func (self *Logging) Debug(sentence string) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Debug(sentence)
-	case "logrus":
-		logrus.Debug(sentence)
-	default:
-	}
-}
-
-func (self *Logging) Debugf(format string, args ...interface{}) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Debugf(format, args...)
-	case "logrus":
-		logrus.Debugf(format, args...)
-	default:
-	}
-}
-
-func (self *Logging) Error(sentence string) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Error(sentence)
-	case "logrus":
-		logrus.Error(sentence)
-	default:
-	}
-}
-
-func (self *Logging) Fatal(sentence string) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Fatal(sentence)
-	case "logrus":
-		logrus.Fatal(sentence)
-	default:
-	}
-}

machinery/src/models/agent_run.go

@@ -0,0 +1,477 @@
+package models
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/kerberos-io/agent/machinery/src/lifecycle"
+	"github.com/kerberos-io/agent/machinery/src/packets"
+	log "github.com/sirupsen/logrus"
+)
+
+var (
+	ErrAgentRunActive  = errors.New("another agent run is active")
+	ErrAgentRunStarted = errors.New("agent run already started")
+	ErrAgentRunStopped = errors.New("agent run is stopping")
+	nextAgentRunID     atomic.Uint64
+)
+
+type AgentRunClient interface {
+	Close(context.Context) error
+}
+
+type AgentRunResourceError struct {
+	Resource string
+	Err      error
+}
+
+func (e AgentRunResourceError) Error() string {
+	return fmt.Sprintf("close %s: %v", e.Resource, e.Err)
+}
+
+func (e AgentRunResourceError) Unwrap() error {
+	return e.Err
+}
+
+type AgentRunShutdownReport struct {
+	lifecycle.ShutdownReport
+	ResourceErrors      []AgentRunResourceError
+	UploadStopDelivered bool
+	StreamStopDelivered bool
+}
+
+type AgentRun struct {
+	id            uint64
+	ctx           context.Context
+	cancel        context.CancelCauseFunc
+	supervisor    *lifecycle.Supervisor
+	communication *Communication
+	stopUpload    bool
+
+	stateMu   sync.Mutex
+	activated bool
+	stopping  bool
+
+	resourcesMu       sync.RWMutex
+	mainClient        AgentRunClient
+	subClient         AgentRunClient
+	backchannelClient AgentRunClient
+	mainQueue         *packets.Queue
+	subQueue          *packets.Queue
+	releaseClients    func()
+
+	channelsMu       sync.RWMutex
+	channelsClosed   bool
+	liveHDHandshakes chan LiveHDHandshake
+	motionEvents     chan MotionDataPartial
+	onvifActions     chan OnvifAction
+
+	shutdownOnce   sync.Once
+	shutdownReport AgentRunShutdownReport
+}
+
+func NewAgentRun(parent context.Context, communication *Communication, stopUpload bool) *AgentRun {
+	if parent == nil {
+		parent = context.Background()
+	}
+	ctx, cancel := context.WithCancelCause(parent)
+	run := &AgentRun{
+		id:               nextAgentRunID.Add(1),
+		ctx:              ctx,
+		cancel:           cancel,
+		supervisor:       lifecycle.NewSupervisor(ctx),
+		communication:    communication,
+		stopUpload:       stopUpload,
+		liveHDHandshakes: make(chan LiveHDHandshake, 100),
+		motionEvents:     make(chan MotionDataPartial, 10),
+		onvifActions:     make(chan OnvifAction, 10),
+	}
+	log.WithFields(log.Fields{
+		"component":   "agent_run",
+		"event":       "run_created",
+		"run_id":      run.id,
+		"stop_upload": stopUpload,
+	}).Debug("Agent run created")
+	return run
+}
+
+func (r *AgentRun) ID() uint64 {
+	return r.id
+}
+
+func (r *AgentRun) Context() context.Context {
+	return r.ctx
+}
+
+func (r *AgentRun) Go(name string, policy lifecycle.TaskPolicy, task lifecycle.TaskFunc) error {
+	if task == nil {
+		return r.supervisor.Go(name, policy, task)
+	}
+
+	fields := log.Fields{
+		"component":    "agent_run",
+		"event":        "task_lifecycle",
+		"long_running": policy.LongRunning,
+		"required":     policy.Required,
+		"run_id":       r.id,
+		"task":         name,
+	}
+	wrappedTask := func(ctx context.Context) error {
+		startedAt := time.Now()
+		log.WithFields(fields).Debug("Agent task started")
+		err := task(ctx)
+		entry := log.WithFields(fields).
+			WithField("duration_ms", time.Since(startedAt).Milliseconds())
+		if err != nil {
+			entry = entry.WithError(err)
+			if ctx.Err() == nil {
+				entry.Warn("Agent task stopped with an error")
+			} else {
+				entry.Debug("Agent task stopped during shutdown")
+			}
+		} else {
+			entry.Debug("Agent task stopped")
+		}
+		return err
+	}
+
+	if err := r.supervisor.Go(name, policy, wrappedTask); err != nil {
+		log.WithError(err).WithFields(fields).Error("Failed to register Agent task")
+		return err
+	}
+	log.WithFields(fields).Debug("Agent task registered")
+	return nil
+}
+
+func (r *AgentRun) Seal() {
+	r.supervisor.Seal()
+	log.WithFields(log.Fields{
+		"component":  "agent_run",
+		"event":      "run_sealed",
+		"run_id":     r.id,
+		"task_count": len(r.supervisor.Snapshot()),
+	}).Debug("Agent run sealed")
+}
+
+func (r *AgentRun) Failures() <-chan lifecycle.Failure {
+	return r.supervisor.Failures()
+}
+
+func (r *AgentRun) Snapshot() []lifecycle.TaskSnapshot {
+	return r.supervisor.Snapshot()
+}
+
+func (r *AgentRun) Wait(ctx context.Context) lifecycle.ShutdownReport {
+	return r.supervisor.Wait(ctx)
+}
+
+func (r *AgentRun) Activate() error {
+	r.stateMu.Lock()
+	defer r.stateMu.Unlock()
+	if r.stopping {
+		return ErrAgentRunStopped
+	}
+	if r.ctx.Err() != nil {
+		return ErrAgentRunStopped
+	}
+	if r.activated {
+		return ErrAgentRunStarted
+	}
+	if r.communication == nil {
+		return errors.New("agent run communication is required")
+	}
+	if err := r.communication.attachRun(r); err != nil {
+		return err
+	}
+	r.activated = true
+	log.WithFields(log.Fields{
+		"component": "agent_run",
+		"event":     "run_activated",
+		"run_id":    r.id,
+	}).Info("Agent run activated")
+	return nil
+}
+
+func (r *AgentRun) SetMainClient(client AgentRunClient) {
+	r.resourcesMu.Lock()
+	r.mainClient = client
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) SetSubClient(client AgentRunClient) {
+	r.resourcesMu.Lock()
+	r.subClient = client
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) SetBackchannelClient(client AgentRunClient) {
+	r.resourcesMu.Lock()
+	r.backchannelClient = client
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) SetQueues(mainQueue, subQueue *packets.Queue) {
+	r.resourcesMu.Lock()
+	r.mainQueue = mainQueue
+	r.subQueue = subQueue
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) SetMainQueue(queue *packets.Queue) {
+	r.resourcesMu.Lock()
+	r.mainQueue = queue
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) SetSubQueue(queue *packets.Queue) {
+	r.resourcesMu.Lock()
+	r.subQueue = queue
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) SetClientRelease(release func()) {
+	r.resourcesMu.Lock()
+	r.releaseClients = release
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) MainQueue() *packets.Queue {
+	if r.isStopping() {
+		return nil
+	}
+	r.resourcesMu.RLock()
+	defer r.resourcesMu.RUnlock()
+	return r.mainQueue
+}
+
+func (r *AgentRun) SubQueue() *packets.Queue {
+	if r.isStopping() {
+		return nil
+	}
+	r.resourcesMu.RLock()
+	defer r.resourcesMu.RUnlock()
+	return r.subQueue
+}
+
+func (r *AgentRun) LiveHDHandshakes() <-chan LiveHDHandshake {
+	return r.liveHDHandshakes
+}
+
+func (r *AgentRun) MotionEvents() <-chan MotionDataPartial {
+	return r.motionEvents
+}
+
+func (r *AgentRun) ONVIFActions() <-chan OnvifAction {
+	return r.onvifActions
+}
+
+func (r *AgentRun) TrySendLiveHDHandshake(handshake LiveHDHandshake) bool {
+	if r.isStopping() {
+		return false
+	}
+	r.channelsMu.RLock()
+	defer r.channelsMu.RUnlock()
+	if r.channelsClosed {
+		return false
+	}
+	select {
+	case r.liveHDHandshakes <- handshake:
+		return true
+	default:
+		return false
+	}
+}
+
+func (r *AgentRun) PendingLiveHDHandshakes() int {
+	if r.isStopping() {
+		return 0
+	}
+	r.channelsMu.RLock()
+	defer r.channelsMu.RUnlock()
+	if r.channelsClosed {
+		return 0
+	}
+	return len(r.liveHDHandshakes)
+}
+
+func (r *AgentRun) TrySendMotion(motion MotionDataPartial) bool {
+	if r.isStopping() {
+		return false
+	}
+	r.channelsMu.RLock()
+	defer r.channelsMu.RUnlock()
+	if r.channelsClosed {
+		return false
+	}
+	select {
+	case r.motionEvents <- motion:
+		return true
+	default:
+		return false
+	}
+}
+
+func (r *AgentRun) TrySendONVIF(action OnvifAction) bool {
+	if r.isStopping() {
+		return false
+	}
+	r.channelsMu.RLock()
+	defer r.channelsMu.RUnlock()
+	if r.channelsClosed {
+		return false
+	}
+	select {
+	case r.onvifActions <- action:
+		return true
+	default:
+		return false
+	}
+}
+
+func (r *AgentRun) Shutdown(ctx context.Context, cause error) AgentRunShutdownReport {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	if cause == nil {
+		cause = context.Canceled
+	}
+
+	r.shutdownOnce.Do(func() {
+		startedAt := time.Now()
+		log.WithError(cause).WithFields(log.Fields{
+			"component": "agent_run",
+			"event":     "shutdown_started",
+			"run_id":    r.id,
+		}).Info("Agent run shutdown started")
+
+		r.stateMu.Lock()
+		r.stopping = true
+		activated := r.activated
+		r.stateMu.Unlock()
+
+		r.cancel(cause)
+		r.supervisor.BeginShutdown(cause)
+
+		if activated && r.stopUpload {
+			r.shutdownReport.UploadStopDelivered = sendRunStop(ctx, r.communication.HandleUpload)
+		}
+		if activated {
+			r.shutdownReport.StreamStopDelivered = sendRunStop(ctx, r.communication.HandleStream)
+		}
+
+		r.resourcesMu.RLock()
+		mainClient := r.mainClient
+		subClient := r.subClient
+		backchannelClient := r.backchannelClient
+		mainQueue := r.mainQueue
+		subQueue := r.subQueue
+		releaseClients := r.releaseClients
+		r.resourcesMu.RUnlock()
+
+		r.closeClient(ctx, "main RTSP client", mainClient)
+		if mainQueue != nil {
+			_ = mainQueue.Close()
+		}
+		r.closeClient(ctx, "sub RTSP client", subClient)
+		if subQueue != nil {
+			_ = subQueue.Close()
+		}
+		r.closeClient(ctx, "RTSP backchannel client", backchannelClient)
+		r.closeChannels()
+		if releaseClients != nil {
+			releaseClients()
+		}
+
+		r.shutdownReport.ShutdownReport = r.supervisor.Wait(ctx)
+		if r.shutdownReport.Complete && r.communication != nil {
+			r.communication.detachRun(r)
+		}
+
+		runningTasks := make([]string, 0, len(r.shutdownReport.Running))
+		for _, task := range r.shutdownReport.Running {
+			runningTasks = append(runningTasks, task.Name)
+		}
+		entry := log.WithFields(log.Fields{
+			"component":            "agent_run",
+			"duration_ms":          time.Since(startedAt).Milliseconds(),
+			"resource_error_count": len(r.shutdownReport.ResourceErrors),
+			"run_id":               r.id,
+			"running_tasks":        runningTasks,
+			"task_count":           len(r.shutdownReport.Tasks),
+			"upload_stop_sent":     r.shutdownReport.UploadStopDelivered,
+			"stream_stop_sent":     r.shutdownReport.StreamStopDelivered,
+		})
+		if r.shutdownReport.Complete {
+			entry.WithField("event", "shutdown_completed").Info("Agent run shutdown completed")
+		} else {
+			entry.WithField("event", "shutdown_timed_out").Error("Agent run shutdown timed out")
+		}
+	})
+
+	report := r.shutdownReport
+	report.ResourceErrors = append([]AgentRunResourceError(nil), report.ResourceErrors...)
+	return report
+}
+
+func (r *AgentRun) isStopping() bool {
+	r.stateMu.Lock()
+	defer r.stateMu.Unlock()
+	return r.stopping
+}
+
+func (r *AgentRun) closeClient(ctx context.Context, name string, client AgentRunClient) {
+	if client == nil {
+		return
+	}
+	if err := client.Close(ctx); err != nil {
+		r.shutdownReport.ResourceErrors = append(r.shutdownReport.ResourceErrors, AgentRunResourceError{
+			Resource: name,
+			Err:      err,
+		})
+		log.WithError(err).WithFields(log.Fields{
+			"component": "agent_run",
+			"event":     "resource_close_failed",
+			"resource":  name,
+			"run_id":    r.id,
+		}).Warn("Failed to close Agent run resource")
+		return
+	}
+	log.WithFields(log.Fields{
+		"component": "agent_run",
+		"event":     "resource_closed",
+		"resource":  name,
+		"run_id":    r.id,
+	}).Debug("Agent run resource closed")
+}
+
+func (r *AgentRun) closeChannels() {
+	r.channelsMu.Lock()
+	defer r.channelsMu.Unlock()
+	if r.channelsClosed {
+		return
+	}
+	r.channelsClosed = true
+	close(r.liveHDHandshakes)
+	close(r.motionEvents)
+	close(r.onvifActions)
+}
+
+func sendRunStop(ctx context.Context, channel chan<- string) bool {
+	if channel == nil {
+		return false
+	}
+	timer := time.NewTimer(time.Second)
+	defer timer.Stop()
+	select {
+	case channel <- "stop":
+		return true
+	case <-ctx.Done():
+		return false
+	case <-timer.C:
+		return false
+	}
+}

machinery/src/models/agent_run_test.go

@@ -0,0 +1,303 @@
+package models
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/kerberos-io/agent/machinery/src/lifecycle"
+	"github.com/kerberos-io/agent/machinery/src/packets"
+)
+
+type fakeAgentRunClient struct {
+	mu       sync.Mutex
+	name     string
+	order    *[]string
+	calls    int
+	closeErr error
+}
+
+func (c *fakeAgentRunClient) Close(context.Context) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.calls++
+	*c.order = append(*c.order, c.name)
+	return c.closeErr
+}
+
+func (c *fakeAgentRunClient) Calls() int {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.calls
+}
+
+func TestAgentRunOwnsAndShutsDownResources(t *testing.T) {
+	communication := &Communication{
+		HandleStream: make(chan string, 1),
+		HandleUpload: make(chan string, 1),
+	}
+	run := NewAgentRun(context.Background(), communication, true)
+	mainQueue := packets.NewQueue()
+	subQueue := packets.NewQueue()
+	run.SetQueues(mainQueue, subQueue)
+
+	var order []string
+	mainClient := &fakeAgentRunClient{name: "main", order: &order}
+	subClient := &fakeAgentRunClient{name: "sub", order: &order}
+	backchannelClient := &fakeAgentRunClient{name: "backchannel", order: &order}
+	run.SetMainClient(mainClient)
+	run.SetSubClient(subClient)
+	run.SetBackchannelClient(backchannelClient)
+	run.SetClientRelease(func() {
+		order = append(order, "release")
+	})
+
+	taskStarted := make(chan struct{})
+	if err := run.Go("worker", lifecycle.TaskPolicy{}, func(ctx context.Context) error {
+		close(taskStarted)
+		<-ctx.Done()
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := run.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	run.Seal()
+	<-taskStarted
+
+	report := run.Shutdown(context.Background(), errors.New("test shutdown"))
+	if !report.Complete {
+		t.Fatalf("shutdown report = %+v, want complete", report)
+	}
+	if communication.CurrentRun() != nil {
+		t.Fatal("shutdown left the run attached")
+	}
+	if got, want := order, []string{"main", "sub", "backchannel", "release"}; !equalStrings(got, want) {
+		t.Fatalf("resource order = %v, want %v", got, want)
+	}
+	if !report.UploadStopDelivered || !report.StreamStopDelivered {
+		t.Fatalf("stop delivery = upload:%t stream:%t, want both", report.UploadStopDelivered, report.StreamStopDelivered)
+	}
+	select {
+	case value := <-communication.HandleUpload:
+		if value != "stop" {
+			t.Fatalf("upload stop = %q, want stop", value)
+		}
+	default:
+		t.Fatal("upload stop was not delivered")
+	}
+	select {
+	case value := <-communication.HandleStream:
+		if value != "stop" {
+			t.Fatalf("stream stop = %q, want stop", value)
+		}
+	default:
+		t.Fatal("stream stop was not delivered")
+	}
+	if _, err := mainQueue.Latest().ReadPacket(); err == nil {
+		t.Fatal("main queue remained open")
+	}
+	if _, err := subQueue.Latest().ReadPacket(); err == nil {
+		t.Fatal("sub queue remained open")
+	}
+	if _, ok := <-run.LiveHDHandshakes(); ok {
+		t.Fatal("handshake channel remained open")
+	}
+	if _, ok := <-run.MotionEvents(); ok {
+		t.Fatal("motion channel remained open")
+	}
+	if _, ok := <-run.ONVIFActions(); ok {
+		t.Fatal("ONVIF channel remained open")
+	}
+}
+
+func TestAgentRunShutdownIsConcurrentAndIdempotent(t *testing.T) {
+	communication := &Communication{}
+	run := NewAgentRun(context.Background(), communication, false)
+	var order []string
+	client := &fakeAgentRunClient{name: "main", order: &order}
+	run.SetMainClient(client)
+	if err := run.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	run.Seal()
+
+	var callers sync.WaitGroup
+	reports := make(chan AgentRunShutdownReport, 20)
+	for index := 0; index < 20; index++ {
+		callers.Add(1)
+		go func() {
+			defer callers.Done()
+			reports <- run.Shutdown(context.Background(), errors.New("test shutdown"))
+		}()
+	}
+	callers.Wait()
+	close(reports)
+
+	for report := range reports {
+		if !report.Complete {
+			t.Fatalf("shutdown report = %+v, want complete", report)
+		}
+	}
+	if got := client.Calls(); got != 1 {
+		t.Fatalf("client Close() calls = %d, want 1", got)
+	}
+}
+
+func TestAgentRunStaleShutdownDoesNotDetachNewRun(t *testing.T) {
+	communication := &Communication{}
+	oldRun := NewAgentRun(context.Background(), communication, false)
+	if err := oldRun.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	if !communication.detachRun(oldRun) {
+		t.Fatal("failed to detach old run during test setup")
+	}
+
+	newRun := NewAgentRun(context.Background(), communication, false)
+	if err := newRun.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	newRun.Seal()
+	t.Cleanup(func() {
+		newRun.Shutdown(context.Background(), errors.New("test complete"))
+	})
+
+	oldRun.Shutdown(context.Background(), errors.New("stale shutdown"))
+	if got := communication.CurrentRun(); got != newRun {
+		t.Fatalf("current run = %p, want new run %p", got, newRun)
+	}
+}
+
+func TestAgentRunRejectsOverlap(t *testing.T) {
+	communication := &Communication{}
+	first := NewAgentRun(context.Background(), communication, false)
+	if err := first.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	first.Seal()
+	t.Cleanup(func() {
+		first.Shutdown(context.Background(), errors.New("test complete"))
+	})
+
+	second := NewAgentRun(context.Background(), communication, false)
+	if err := second.Activate(); !errors.Is(err, ErrAgentRunActive) {
+		t.Fatalf("Activate() error = %v, want ErrAgentRunActive", err)
+	}
+	second.Shutdown(context.Background(), errors.New("test complete"))
+}
+
+func TestAgentRunRetainsOwnershipUntilWorkersStop(t *testing.T) {
+	communication := &Communication{}
+	run := NewAgentRun(context.Background(), communication, false)
+	release := make(chan struct{})
+	started := make(chan struct{})
+	if err := run.Go("blocked", lifecycle.TaskPolicy{}, func(context.Context) error {
+		close(started)
+		<-release
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := run.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	run.Seal()
+	<-started
+
+	shutdownDone := make(chan AgentRunShutdownReport, 1)
+	go func() {
+		shutdownDone <- run.Shutdown(context.Background(), errors.New("test shutdown"))
+	}()
+
+	deadline := time.Now().Add(time.Second)
+	for run.Context().Err() == nil && time.Now().Before(deadline) {
+		time.Sleep(time.Millisecond)
+	}
+	if run.Context().Err() == nil {
+		t.Fatal("run context was not canceled")
+	}
+	if communication.CurrentRun() != run {
+		t.Fatal("run ownership was released before its worker stopped")
+	}
+	if communication.TrySendMotion(MotionDataPartial{}) {
+		t.Fatal("stopping run accepted new input")
+	}
+	if communication.MainQueue() != nil {
+		t.Fatal("stopping run exposed its queue")
+	}
+
+	replacement := NewAgentRun(context.Background(), communication, false)
+	if err := replacement.Activate(); !errors.Is(err, ErrAgentRunActive) {
+		t.Fatalf("replacement Activate() error = %v, want ErrAgentRunActive", err)
+	}
+	replacement.Shutdown(context.Background(), errors.New("test complete"))
+
+	close(release)
+	report := <-shutdownDone
+	if !report.Complete {
+		t.Fatalf("shutdown report = %+v, want complete", report)
+	}
+	if communication.CurrentRun() != nil {
+		t.Fatal("completed shutdown left the run attached")
+	}
+}
+
+func TestAgentRunShutdownReportsStuckTask(t *testing.T) {
+	communication := &Communication{}
+	run := NewAgentRun(context.Background(), communication, false)
+	release := make(chan struct{})
+	started := make(chan struct{})
+	if err := run.Go("blocked", lifecycle.TaskPolicy{}, func(context.Context) error {
+		close(started)
+		<-release
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := run.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	run.Seal()
+	<-started
+
+	waitContext, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+	defer cancel()
+	report := run.Shutdown(waitContext, errors.New("test shutdown"))
+	if report.Complete {
+		t.Fatal("shutdown unexpectedly completed")
+	}
+	if len(report.Running) != 1 || report.Running[0].Name != "blocked" {
+		t.Fatalf("running tasks = %+v, want blocked", report.Running)
+	}
+	if communication.CurrentRun() != run {
+		t.Fatal("timed-out shutdown released run ownership")
+	}
+	replacement := NewAgentRun(context.Background(), communication, false)
+	if err := replacement.Activate(); !errors.Is(err, ErrAgentRunActive) {
+		t.Fatalf("replacement Activate() error = %v, want ErrAgentRunActive", err)
+	}
+	replacement.Shutdown(context.Background(), errors.New("test complete"))
+	close(release)
+	waitContext, waitCancel := context.WithTimeout(context.Background(), time.Second)
+	defer waitCancel()
+	if followup := run.Wait(waitContext); !followup.Complete {
+		t.Fatalf("follow-up report = %+v, want complete", followup)
+	}
+	communication.detachRun(run)
+}
+
+func equalStrings(left, right []string) bool {
+	if len(left) != len(right) {
+		return false
+	}
+	for index := range left {
+		if left[index] != right[index] {
+			return false
+		}
+	}
+	return true
+}

machinery/src/models/communication.go

@@ -1,8 +1,10 @@
 package models
 
 import (
-	"context"
+	"math"
+	"sync"
 	"sync/atomic"
+	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/tevino/abool"
@@ -19,11 +21,84 @@ type LiveHDHandshake struct {
 	Signaling *LiveHDSignalingCallbacks
 }
 
+type MoQRecoveryTelemetry struct {
+	Reconnects          uint64 `json:"reconnects"`
+	LastFrameUnixMillis int64  `json:"lastFrameUnixMillis"`
+	LastWriteMillis     int64  `json:"lastWriteMillis"`
+}
+
+type RecoveryTelemetry struct {
+	MoQHigh                   MoQRecoveryTelemetry `json:"moqHigh"`
+	MoQLow                    MoQRecoveryTelemetry `json:"moqLow"`
+	MoQWriteTimeouts          uint64               `json:"moqWriteTimeouts"`
+	DroppedLiveHDHandshakes   uint64               `json:"droppedLiveHDHandshakes"`
+	DroppedMotionEvents       uint64               `json:"droppedMotionEvents"`
+	DroppedONVIFActions       uint64               `json:"droppedOnvifActions"`
+	WatchdogRestarts          uint64               `json:"watchdogRestarts"`
+	WatchdogCooldownSeconds   int64                `json:"watchdogCooldownSeconds"`
+	RunWorkerShutdownTimeouts uint64               `json:"runWorkerShutdownTimeouts"`
+}
+
+type StreamKind uint8
+
+const (
+	MainStream StreamKind = iota
+	SubStream
+)
+
+type StreamRuntimeTelemetry struct {
+	Configured        bool
+	PackagesProcessed uint64
+	FPS               float64
+	Width             int64
+	Height            int64
+	LastPacketAt      int64
+}
+
+type streamRuntimeTelemetry struct {
+	configured        atomic.Bool
+	packagesProcessed atomic.Uint64
+	fpsBits           atomic.Uint64
+	width             atomic.Int64
+	height            atomic.Int64
+	lastPacketAt      atomic.Int64
+}
+
+type HubRuntimeTelemetry struct {
+	Configured                bool
+	Connected                 bool
+	LastHeartbeatAttemptAt    int64
+	LastSuccessfulHeartbeatAt int64
+}
+
+type hubRuntimeTelemetry struct {
+	configured                atomic.Bool
+	connected                 atomic.Bool
+	lastHeartbeatAttemptAt    atomic.Int64
+	lastSuccessfulHeartbeatAt atomic.Int64
+}
+
+type recoveryTelemetry struct {
+	moqHighReconnects          atomic.Uint64
+	moqHighLastFrameUnixMillis atomic.Int64
+	moqHighLastWriteMillis     atomic.Int64
+	moqLowReconnects           atomic.Uint64
+	moqLowLastFrameUnixMillis  atomic.Int64
+	moqLowLastWriteMillis      atomic.Int64
+	moqWriteTimeouts           atomic.Uint64
+	droppedLiveHDHandshakes    atomic.Uint64
+	droppedMotionEvents        atomic.Uint64
+	droppedONVIFActions        atomic.Uint64
+	watchdogRestarts           atomic.Uint64
+	watchdogCooldownSeconds    atomic.Int64
+	runWorkerShutdownTimeouts  atomic.Uint64
+}
+
 // The communication struct that is managing
 // all the communication between the different goroutines.
 type Communication struct {
-	Context               *context.Context
+	currentRunMu          sync.RWMutex
-	CancelContext         *context.CancelFunc
+	currentRun            *AgentRun
 	PackageCounter        *atomic.Value
 	LastPacketTimer       *atomic.Value
 	PackageCounterSub     *atomic.Value
@@ -32,20 +107,17 @@ type Communication struct {
 	HandleBootstrap       chan string
 	HandleStream          chan string
 	HandleSubStream       chan string
-	HandleMotion          chan MotionDataPartial
 	HandleAudio           chan AudioDataPartial
 	HandleUpload          chan string
 	HandleHeartBeat       chan string
 	HandleLiveSD          chan int64
 	HandleLiveSDHTTP      chan int64
 	HandleLiveHDKeepalive chan string
-	HandleLiveHDHandshake chan LiveHDHandshake
 	HandleLiveHDPeers     chan string
 	// HandleLiveHLS is the live HLS viewer keepalive. It carries the requested
 	// quality tier ("auto"|"high"|"low"; empty => auto) so the producer can switch
 	// the live session between the main and sub stream on demand.
 	HandleLiveHLS chan string
-	HandleONVIF   chan OnvifAction
 	IsConfiguring *abool.AtomicBool
 	// IsRecordingManual is set while a viewer has requested a manual recording
 	// from the live view (the record button). While set, the motion-based
@@ -71,11 +143,224 @@ type Communication struct {
 	// viewer that starts a recording but never heartbeats (an older frontend)
 	// still records up to the max-duration cap instead of being cut off early.
 	RecordingManualHeartbeatSeen *abool.AtomicBool
-	Queue                        *packets.Queue
-	SubQueue                     *packets.Queue
 	Image                        string
-	CameraConnected              bool
+	CameraConnected              atomic.Bool
-	MainStreamConnected          bool
+	MainStreamConnected          atomic.Bool
-	SubStreamConnected           bool
+	SubStreamConnected           atomic.Bool
-	HasBackChannel               bool
+	HasBackChannel               atomic.Bool
+	mainStreamTelemetry          streamRuntimeTelemetry
+	subStreamTelemetry           streamRuntimeTelemetry
+	hubTelemetry                 hubRuntimeTelemetry
+	recovery                     recoveryTelemetry
+}
+
+func (c *Communication) streamTelemetry(stream StreamKind) *streamRuntimeTelemetry {
+	switch stream {
+	case MainStream:
+		return &c.mainStreamTelemetry
+	case SubStream:
+		return &c.subStreamTelemetry
+	default:
+		panic("unsupported stream kind")
+	}
+}
+
+func (c *Communication) SetStreamConfigured(stream StreamKind, configured bool) {
+	c.streamTelemetry(stream).configured.Store(configured)
+}
+
+func (c *Communication) RecordStreamPackage(stream StreamKind, fps float64, width, height int, at time.Time) {
+	telemetry := c.streamTelemetry(stream)
+	telemetry.packagesProcessed.Add(1)
+	if fps > 0 && !math.IsNaN(fps) && !math.IsInf(fps, 0) {
+		telemetry.fpsBits.Store(math.Float64bits(fps))
+	}
+	if width > 0 {
+		telemetry.width.Store(int64(width))
+	}
+	if height > 0 {
+		telemetry.height.Store(int64(height))
+	}
+	if !at.IsZero() {
+		telemetry.lastPacketAt.Store(at.Unix())
+	}
+}
+
+func (c *Communication) StreamRuntimeTelemetry(stream StreamKind) StreamRuntimeTelemetry {
+	telemetry := c.streamTelemetry(stream)
+	return StreamRuntimeTelemetry{
+		Configured:        telemetry.configured.Load(),
+		PackagesProcessed: telemetry.packagesProcessed.Load(),
+		FPS:               math.Float64frombits(telemetry.fpsBits.Load()),
+		Width:             telemetry.width.Load(),
+		Height:            telemetry.height.Load(),
+		LastPacketAt:      telemetry.lastPacketAt.Load(),
+	}
+}
+
+func (c *Communication) SetHubConfigured(configured bool) {
+	c.hubTelemetry.configured.Store(configured)
+	if !configured {
+		c.hubTelemetry.connected.Store(false)
+	}
+}
+
+func (c *Communication) RecordHubHeartbeatAttempt(at time.Time) {
+	if !at.IsZero() {
+		c.hubTelemetry.lastHeartbeatAttemptAt.Store(at.Unix())
+	}
+}
+
+func (c *Communication) RecordHubHeartbeatSuccess(at time.Time) {
+	if !at.IsZero() {
+		c.hubTelemetry.lastSuccessfulHeartbeatAt.Store(at.Unix())
+	}
+	c.hubTelemetry.connected.Store(true)
+}
+
+func (c *Communication) RecordHubHeartbeatFailure() {
+	c.hubTelemetry.connected.Store(false)
+}
+
+func (c *Communication) HubRuntimeTelemetry() HubRuntimeTelemetry {
+	return HubRuntimeTelemetry{
+		Configured:                c.hubTelemetry.configured.Load(),
+		Connected:                 c.hubTelemetry.connected.Load(),
+		LastHeartbeatAttemptAt:    c.hubTelemetry.lastHeartbeatAttemptAt.Load(),
+		LastSuccessfulHeartbeatAt: c.hubTelemetry.lastSuccessfulHeartbeatAt.Load(),
+	}
+}
+
+func (c *Communication) RecordMoQReconnect(quality string) {
+	if quality == StreamQualityLow {
+		c.recovery.moqLowReconnects.Add(1)
+		return
+	}
+	c.recovery.moqHighReconnects.Add(1)
+}
+
+func (c *Communication) RecordMoQWrite(quality string, duration time.Duration, at time.Time) {
+	if quality == StreamQualityLow {
+		c.recovery.moqLowLastWriteMillis.Store(duration.Milliseconds())
+		c.recovery.moqLowLastFrameUnixMillis.Store(at.UnixMilli())
+		return
+	}
+	c.recovery.moqHighLastWriteMillis.Store(duration.Milliseconds())
+	c.recovery.moqHighLastFrameUnixMillis.Store(at.UnixMilli())
+}
+
+func (c *Communication) RecordMoQWriteTimeout() {
+	c.recovery.moqWriteTimeouts.Add(1)
+}
+
+func (c *Communication) RecordWatchdogRestart(cooldown time.Duration) {
+	c.recovery.watchdogRestarts.Add(1)
+	c.SetWatchdogCooldown(cooldown)
+}
+
+func (c *Communication) SetWatchdogCooldown(cooldown time.Duration) {
+	c.recovery.watchdogCooldownSeconds.Store(int64(cooldown / time.Second))
+}
+
+func (c *Communication) RecordRunWorkerShutdownTimeout() {
+	c.recovery.runWorkerShutdownTimeouts.Add(1)
+}
+
+func (c *Communication) RecoveryTelemetry() RecoveryTelemetry {
+	return RecoveryTelemetry{
+		MoQHigh: MoQRecoveryTelemetry{
+			Reconnects:          c.recovery.moqHighReconnects.Load(),
+			LastFrameUnixMillis: c.recovery.moqHighLastFrameUnixMillis.Load(),
+			LastWriteMillis:     c.recovery.moqHighLastWriteMillis.Load(),
+		},
+		MoQLow: MoQRecoveryTelemetry{
+			Reconnects:          c.recovery.moqLowReconnects.Load(),
+			LastFrameUnixMillis: c.recovery.moqLowLastFrameUnixMillis.Load(),
+			LastWriteMillis:     c.recovery.moqLowLastWriteMillis.Load(),
+		},
+		MoQWriteTimeouts:          c.recovery.moqWriteTimeouts.Load(),
+		DroppedLiveHDHandshakes:   c.recovery.droppedLiveHDHandshakes.Load(),
+		DroppedMotionEvents:       c.recovery.droppedMotionEvents.Load(),
+		DroppedONVIFActions:       c.recovery.droppedONVIFActions.Load(),
+		WatchdogRestarts:          c.recovery.watchdogRestarts.Load(),
+		WatchdogCooldownSeconds:   c.recovery.watchdogCooldownSeconds.Load(),
+		RunWorkerShutdownTimeouts: c.recovery.runWorkerShutdownTimeouts.Load(),
+	}
+}
+
+func (c *Communication) attachRun(run *AgentRun) error {
+	c.currentRunMu.Lock()
+	defer c.currentRunMu.Unlock()
+	if c.currentRun != nil {
+		return ErrAgentRunActive
+	}
+	c.currentRun = run
+	return nil
+}
+
+func (c *Communication) detachRun(run *AgentRun) bool {
+	c.currentRunMu.Lock()
+	defer c.currentRunMu.Unlock()
+	if c.currentRun != run {
+		return false
+	}
+	c.currentRun = nil
+	return true
+}
+
+func (c *Communication) CurrentRun() *AgentRun {
+	c.currentRunMu.RLock()
+	defer c.currentRunMu.RUnlock()
+	return c.currentRun
+}
+
+func (c *Communication) MainQueue() *packets.Queue {
+	run := c.CurrentRun()
+	if run == nil {
+		return nil
+	}
+	return run.MainQueue()
+}
+
+func (c *Communication) SubQueue() *packets.Queue {
+	run := c.CurrentRun()
+	if run == nil {
+		return nil
+	}
+	return run.SubQueue()
+}
+
+func (c *Communication) TrySendLiveHDHandshake(handshake LiveHDHandshake) bool {
+	run := c.CurrentRun()
+	if run == nil || !run.TrySendLiveHDHandshake(handshake) {
+		c.recovery.droppedLiveHDHandshakes.Add(1)
+		return false
+	}
+	return true
+}
+
+func (c *Communication) PendingLiveHDHandshakes() int {
+	run := c.CurrentRun()
+	if run == nil {
+		return 0
+	}
+	return run.PendingLiveHDHandshakes()
+}
+
+func (c *Communication) TrySendMotion(motion MotionDataPartial) bool {
+	run := c.CurrentRun()
+	if run == nil || !run.TrySendMotion(motion) {
+		c.recovery.droppedMotionEvents.Add(1)
+		return false
+	}
+	return true
+}
+
+func (c *Communication) TrySendONVIF(action OnvifAction) bool {
+	run := c.CurrentRun()
+	if run == nil || !run.TrySendONVIF(action) {
+		c.recovery.droppedONVIFActions.Add(1)
+		return false
+	}
+	return true
 }

machinery/src/models/communication_test.go

@@ -0,0 +1,221 @@
+package models
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+	"time"
+)
+
+func TestCommunicationRunChannelDispatchAfterClose(t *testing.T) {
+	communication := &Communication{}
+	run := NewAgentRun(context.Background(), communication, false)
+	if err := run.Activate(); err != nil {
+		t.Fatal(err)
+	}
+
+	if !communication.TrySendLiveHDHandshake(LiveHDHandshake{}) {
+		t.Fatal("TrySendLiveHDHandshake() rejected an available channel")
+	}
+	if !communication.TrySendMotion(MotionDataPartial{}) {
+		t.Fatal("TrySendMotion() rejected an available channel")
+	}
+	if !communication.TrySendONVIF(OnvifAction{}) {
+		t.Fatal("TrySendONVIF() rejected an available channel")
+	}
+
+	report := run.Shutdown(context.Background(), errors.New("test complete"))
+	if !report.Complete {
+		t.Fatalf("shutdown report = %+v, want complete", report)
+	}
+	if communication.TrySendLiveHDHandshake(LiveHDHandshake{}) {
+		t.Fatal("TrySendLiveHDHandshake() accepted a closed run")
+	}
+	if communication.TrySendMotion(MotionDataPartial{}) {
+		t.Fatal("TrySendMotion() accepted a closed run")
+	}
+	if communication.TrySendONVIF(OnvifAction{}) {
+		t.Fatal("TrySendONVIF() accepted a closed run")
+	}
+}
+
+func TestCommunicationDispatchCanRaceRunChannelClose(t *testing.T) {
+	communication := &Communication{}
+	run := NewAgentRun(context.Background(), communication, false)
+	if err := run.Activate(); err != nil {
+		t.Fatal(err)
+	}
+
+	start := make(chan struct{})
+	var senders sync.WaitGroup
+	for sender := 0; sender < 20; sender++ {
+		senders.Add(1)
+		go func() {
+			defer senders.Done()
+			<-start
+			for attempt := 0; attempt < 100; attempt++ {
+				communication.TrySendLiveHDHandshake(LiveHDHandshake{})
+				communication.TrySendMotion(MotionDataPartial{})
+				communication.TrySendONVIF(OnvifAction{})
+			}
+		}()
+	}
+
+	close(start)
+	run.Shutdown(context.Background(), errors.New("test complete"))
+	senders.Wait()
+}
+
+func TestCommunicationRunChannelLifecycleSoak(t *testing.T) {
+	communication := &Communication{}
+	for cycle := 0; cycle < 100; cycle++ {
+		run := NewAgentRun(context.Background(), communication, false)
+		if err := run.Activate(); err != nil {
+			t.Fatalf("cycle %d Activate() error = %v", cycle, err)
+		}
+		handshakes := run.LiveHDHandshakes()
+		motion := run.MotionEvents()
+		onvif := run.ONVIFActions()
+
+		var consumers sync.WaitGroup
+		consumers.Add(3)
+		go func() {
+			defer consumers.Done()
+			for range handshakes {
+			}
+		}()
+		go func() {
+			defer consumers.Done()
+			for range motion {
+			}
+		}()
+		go func() {
+			defer consumers.Done()
+			for range onvif {
+			}
+		}()
+
+		var producers sync.WaitGroup
+		for producer := 0; producer < 4; producer++ {
+			producers.Add(1)
+			go func() {
+				defer producers.Done()
+				for attempt := 0; attempt < 50; attempt++ {
+					communication.TrySendLiveHDHandshake(LiveHDHandshake{})
+					communication.TrySendMotion(MotionDataPartial{})
+					communication.TrySendONVIF(OnvifAction{})
+				}
+			}()
+		}
+
+		run.Shutdown(context.Background(), errors.New("test complete"))
+		producers.Wait()
+
+		done := make(chan struct{})
+		go func() {
+			consumers.Wait()
+			close(done)
+		}()
+		select {
+		case <-done:
+		case <-time.After(time.Second):
+			t.Fatalf("cycle %d consumers did not stop after channel close", cycle)
+		}
+	}
+}
+
+func TestCommunicationRecoveryTelemetry(t *testing.T) {
+	communication := &Communication{}
+	communication.RecordMoQReconnect(StreamQualityHigh)
+	communication.RecordMoQReconnect(StreamQualityLow)
+	communication.RecordMoQWrite(StreamQualityHigh, 1250*time.Millisecond, time.UnixMilli(1234))
+	communication.RecordMoQWrite(StreamQualityLow, 2500*time.Millisecond, time.UnixMilli(5678))
+	communication.RecordMoQWriteTimeout()
+	communication.RecordWatchdogRestart(30 * time.Second)
+	communication.RecordRunWorkerShutdownTimeout()
+
+	if communication.TrySendLiveHDHandshake(LiveHDHandshake{}) {
+		t.Fatal("TrySendLiveHDHandshake() accepted an unavailable channel")
+	}
+	if communication.TrySendMotion(MotionDataPartial{}) {
+		t.Fatal("TrySendMotion() accepted an unavailable channel")
+	}
+	if communication.TrySendONVIF(OnvifAction{}) {
+		t.Fatal("TrySendONVIF() accepted an unavailable channel")
+	}
+
+	got := communication.RecoveryTelemetry()
+	if got.MoQHigh != (MoQRecoveryTelemetry{Reconnects: 1, LastFrameUnixMillis: 1234, LastWriteMillis: 1250}) {
+		t.Fatalf("high MoQ telemetry = %+v", got.MoQHigh)
+	}
+	if got.MoQLow != (MoQRecoveryTelemetry{Reconnects: 1, LastFrameUnixMillis: 5678, LastWriteMillis: 2500}) {
+		t.Fatalf("low MoQ telemetry = %+v", got.MoQLow)
+	}
+	if got.MoQWriteTimeouts != 1 || got.WatchdogRestarts != 1 || got.WatchdogCooldownSeconds != 30 || got.RunWorkerShutdownTimeouts != 1 {
+		t.Fatalf("recovery telemetry = %+v", got)
+	}
+	if got.DroppedLiveHDHandshakes != 1 || got.DroppedMotionEvents != 1 || got.DroppedONVIFActions != 1 {
+		t.Fatalf("drop telemetry = %+v", got)
+	}
+}
+
+func TestCommunicationOperationalTelemetry(t *testing.T) {
+	communication := &Communication{}
+	mainPacketAt := time.Unix(1_788_710_401, 0)
+	subPacketAt := time.Unix(1_788_710_402, 0)
+	hubAttemptAt := time.Unix(1_788_710_403, 0)
+	hubSuccessAt := time.Unix(1_788_710_404, 0)
+
+	communication.SetStreamConfigured(MainStream, true)
+	communication.SetStreamConfigured(SubStream, true)
+	communication.RecordStreamPackage(MainStream, 29.97, 1920, 1080, mainPacketAt)
+	communication.RecordStreamPackage(MainStream, 29.97, 1920, 1080, mainPacketAt)
+	communication.RecordStreamPackage(SubStream, 15, 640, 360, subPacketAt)
+	communication.SetHubConfigured(true)
+	communication.RecordHubHeartbeatAttempt(hubAttemptAt)
+	communication.RecordHubHeartbeatSuccess(hubSuccessAt)
+
+	main := communication.StreamRuntimeTelemetry(MainStream)
+	if main != (StreamRuntimeTelemetry{
+		Configured:        true,
+		PackagesProcessed: 2,
+		FPS:               29.97,
+		Width:             1920,
+		Height:            1080,
+		LastPacketAt:      mainPacketAt.Unix(),
+	}) {
+		t.Fatalf("main stream telemetry = %+v", main)
+	}
+
+	sub := communication.StreamRuntimeTelemetry(SubStream)
+	if sub != (StreamRuntimeTelemetry{
+		Configured:        true,
+		PackagesProcessed: 1,
+		FPS:               15,
+		Width:             640,
+		Height:            360,
+		LastPacketAt:      subPacketAt.Unix(),
+	}) {
+		t.Fatalf("sub stream telemetry = %+v", sub)
+	}
+
+	hub := communication.HubRuntimeTelemetry()
+	if hub != (HubRuntimeTelemetry{
+		Configured:                true,
+		Connected:                 true,
+		LastHeartbeatAttemptAt:    hubAttemptAt.Unix(),
+		LastSuccessfulHeartbeatAt: hubSuccessAt.Unix(),
+	}) {
+		t.Fatalf("Hub telemetry = %+v", hub)
+	}
+
+	communication.RecordHubHeartbeatFailure()
+	if communication.HubRuntimeTelemetry().Connected {
+		t.Fatal("Hub telemetry remained connected after a failed heartbeat")
+	}
+	communication.SetHubConfigured(false)
+	if communication.HubRuntimeTelemetry().Configured {
+		t.Fatal("Hub telemetry remained configured after it was disabled")
+	}
+}

machinery/src/models/mqtt.go

@@ -6,20 +6,96 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"encoding/pem"
-	"io"
+	"errors"
-	"strings"
+	"sync"
 	"time"
 
 	"github.com/gofrs/uuid"
 	"github.com/kerberos-io/agent/machinery/src/encryption"
-	"github.com/kerberos-io/agent/machinery/src/log"
+	log "github.com/sirupsen/logrus"
 )
 
+var (
+	errMQTTPrivateKeyPEMDecode = errors.New("error decoding PEM block containing private key")
+	errMQTTPrivateKeyNotRSA    = errors.New("private key is not RSA")
+
+	rsaPrivateKeyCache   = newRSAPrivateKeyCache()
+	parsePKCS8PrivateKey = x509.ParsePKCS8PrivateKey
+)
+
+type rsaPrivateKeyCacheState struct {
+	mu      sync.Mutex
+	cond    *sync.Cond
+	pem     string
+	key     *rsa.PrivateKey
+	err     error
+	parsing bool
+	ready   bool
+}
+
+func newRSAPrivateKeyCache() *rsaPrivateKeyCacheState {
+	cache := &rsaPrivateKeyCacheState{}
+	cache.cond = sync.NewCond(&cache.mu)
+	return cache
+}
+
+func (cache *rsaPrivateKeyCacheState) get(privateKey string) (*rsa.PrivateKey, error) {
+	cache.mu.Lock()
+	for {
+		if cache.ready && cache.pem == privateKey && !cache.parsing {
+			key, err := cache.key, cache.err
+			cache.mu.Unlock()
+			return key, err
+		}
+		if cache.parsing {
+			cache.cond.Wait()
+			continue
+		}
+
+		cache.pem = privateKey
+		cache.key = nil
+		cache.err = nil
+		cache.parsing = true
+		cache.ready = false
+		cache.mu.Unlock()
+
+		key, err := parseRSAPrivateKey(privateKey)
+
+		cache.mu.Lock()
+		cache.key = key
+		cache.err = err
+		cache.parsing = false
+		cache.ready = true
+		cache.cond.Broadcast()
+		cache.mu.Unlock()
+		return key, err
+	}
+}
+
+func parseRSAPrivateKey(privateKey string) (*rsa.PrivateKey, error) {
+	block, _ := pem.Decode([]byte(privateKey))
+	if block == nil {
+		return nil, errMQTTPrivateKeyPEMDecode
+	}
+
+	key, err := parsePKCS8PrivateKey(block.Bytes)
+	if err != nil {
+		return nil, err
+	}
+
+	rsaKey, ok := key.(*rsa.PrivateKey)
+	if !ok {
+		return nil, errMQTTPrivateKeyNotRSA
+	}
+
+	return rsaKey, nil
+}
+
 func PackageMQTTMessage(configuration *Configuration, msg Message) ([]byte, error) {
 	// Create a Version 4 UUID.
 	u2, err := uuid.NewV4()
 	if err != nil {
-		log.Log.Error("failed to generate UUID: " + err.Error())
+		log.Error("failed to generate UUID: " + err.Error())
 	}
 
 	// We'll generate an unique id, and encrypt / decrypt it using the private key if available.
@@ -45,27 +121,25 @@ func PackageMQTTMessage(configuration *Configuration, msg Message) ([]byte, erro
 		// Pload to base64
 		data, err := json.Marshal(pload)
 		if err != nil {
-			log.Log.Error("models.mqtt.PackageMQTTMessage(): failed to marshal payload: " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component": "models/mqtt",
+				"event":     "payload_marshal_failed",
+			}).Error("Failed to marshal MQTT payload")
 		}
 
 		// Encrypt the value
 		privateKey := configuration.Config.Encryption.PrivateKey
-		r := strings.NewReader(privateKey)
+		rsaKey, err := rsaPrivateKeyCache.get(privateKey)
-		pemBytes, _ := io.ReadAll(r)
+		if err != nil {
-		block, _ := pem.Decode(pemBytes)
+			if errors.Is(err, errMQTTPrivateKeyPEMDecode) {
-		if block == nil {
+				log.Error("models.mqtt.PackageMQTTMessage(): error decoding PEM block containing private key")
-			log.Log.Error("models.mqtt.PackageMQTTMessage(): error decoding PEM block containing private key")
+			} else {
-		} else {
+				log.WithError(err).WithFields(log.Fields{
-			// Parse private key
+					"component": "models/mqtt",
-			b := block.Bytes
+					"event":     "private_key_parse_failed",
-			key, err := x509.ParsePKCS8PrivateKey(b)
+				}).Error("Failed to parse MQTT private key")
-			if err != nil {
-				log.Log.Error("models.mqtt.PackageMQTTMessage(): error parsing private key: " + err.Error())
 			}
-
+		} else {
-			// Conver key to *rsa.PrivateKey
-			rsaKey, _ := key.(*rsa.PrivateKey)
-
 			// Create a 16bit key random
 			if config.Encryption != nil && config.Encryption.SymmetricKey != "" {
 				k := config.Encryption.SymmetricKey
@@ -250,3 +324,23 @@ type TriggerRelay struct {
 	DeviceId  string `json:"device_id"` // device id
 	Token     string `json:"token"`     // token
 }
+
+// RemoteSessionPayload controls an interactive shell or log stream over MQTT.
+// Data is base64 encoded so terminal control bytes remain valid JSON.
+type RemoteSessionPayload struct {
+	Timestamp int64  `json:"timestamp"`
+	SessionID string `json:"session_id"`
+	Kind      string `json:"kind,omitempty"`
+	Data      string `json:"data,omitempty"`
+	Rows      uint16 `json:"rows,omitempty"`
+	Columns   uint16 `json:"columns,omitempty"`
+	Tail      int    `json:"tail,omitempty"`
+}
+
+type RemoteSessionStatus struct {
+	Timestamp int64  `json:"timestamp"`
+	SessionID string `json:"session_id"`
+	Kind      string `json:"kind"`
+	State     string `json:"state"`
+	Error     string `json:"error,omitempty"`
+}

machinery/src/models/mqtt_test.go

@@ -0,0 +1,199 @@
+package models
+
+import (
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/json"
+	"encoding/pem"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestPackageMQTTMessageCachesCurrentSigningKey(t *testing.T) {
+	resetMQTTPrivateKeyCache(t)
+
+	privateKey := generatePKCS8RSAPrivateKeyPEM(t)
+
+	var parseCalls atomic.Int32
+	restore := hookPKCS8PrivateKeyParser(t, func(der []byte) (any, error) {
+		parseCalls.Add(1)
+		return x509.ParsePKCS8PrivateKey(der)
+	})
+	defer restore()
+
+	msg := Message{
+		Payload: Payload{
+			DeviceId: "device-1",
+			Value: map[string]interface{}{
+				"hello": "world",
+			},
+		},
+	}
+
+	first, err := PackageMQTTMessage(encryptedConfiguration(privateKey), msg)
+	if err != nil {
+		t.Fatalf("first PackageMQTTMessage() error = %v", err)
+	}
+	assertEncryptedMQTTMessage(t, first)
+
+	second, err := PackageMQTTMessage(encryptedConfiguration(privateKey), msg)
+	if err != nil {
+		t.Fatalf("second PackageMQTTMessage() error = %v", err)
+	}
+	assertEncryptedMQTTMessage(t, second)
+
+	if got := parseCalls.Load(); got != 1 {
+		t.Fatalf("cached parse calls = %d, want 1", got)
+	}
+}
+
+func TestPackageMQTTMessageReplacesCachedSigningKeyOnRotation(t *testing.T) {
+	resetMQTTPrivateKeyCache(t)
+
+	privateKey := generatePKCS8RSAPrivateKeyPEM(t)
+	rotatedPrivateKey := generatePKCS8RSAPrivateKeyPEM(t)
+
+	var parseCalls atomic.Int32
+	restore := hookPKCS8PrivateKeyParser(t, func(der []byte) (any, error) {
+		parseCalls.Add(1)
+		return x509.ParsePKCS8PrivateKey(der)
+	})
+	defer restore()
+
+	msg := Message{
+		Payload: Payload{
+			DeviceId: "device-1",
+			Value: map[string]interface{}{
+				"hello": "world",
+			},
+		},
+	}
+
+	for _, key := range []string{privateKey, rotatedPrivateKey, privateKey} {
+		payload, err := PackageMQTTMessage(encryptedConfiguration(key), msg)
+		if err != nil {
+			t.Fatalf("PackageMQTTMessage() error = %v", err)
+		}
+		assertEncryptedMQTTMessage(t, payload)
+	}
+
+	if got := parseCalls.Load(); got != 3 {
+		t.Fatalf("parse calls across old/new/old rotation = %d, want 3", got)
+	}
+}
+
+func TestPackageMQTTMessageConcurrentCallsReuseCachedSigningKey(t *testing.T) {
+	resetMQTTPrivateKeyCache(t)
+
+	privateKey := generatePKCS8RSAPrivateKeyPEM(t)
+
+	var parseCalls atomic.Int32
+	restore := hookPKCS8PrivateKeyParser(t, func(der []byte) (any, error) {
+		parseCalls.Add(1)
+		time.Sleep(10 * time.Millisecond)
+		return x509.ParsePKCS8PrivateKey(der)
+	})
+	defer restore()
+
+	msg := Message{
+		Payload: Payload{
+			DeviceId: "device-1",
+			Value: map[string]interface{}{
+				"hello": "world",
+			},
+		},
+	}
+
+	start := make(chan struct{})
+	var workers sync.WaitGroup
+	for i := 0; i < 16; i++ {
+		workers.Add(1)
+		go func() {
+			defer workers.Done()
+			<-start
+
+			payload, err := PackageMQTTMessage(encryptedConfiguration(privateKey), msg)
+			if err != nil {
+				t.Errorf("PackageMQTTMessage() error = %v", err)
+				return
+			}
+			assertEncryptedMQTTMessage(t, payload)
+		}()
+	}
+
+	close(start)
+	workers.Wait()
+
+	if got := parseCalls.Load(); got != 1 {
+		t.Fatalf("concurrent parse calls = %d, want 1", got)
+	}
+}
+
+func encryptedConfiguration(privateKey string) *Configuration {
+	return &Configuration{
+		Config: Config{
+			Encryption: &Encryption{
+				Enabled:      "true",
+				PrivateKey:   privateKey,
+				SymmetricKey: "secret",
+			},
+		},
+	}
+}
+
+func generatePKCS8RSAPrivateKeyPEM(t *testing.T) string {
+	t.Helper()
+
+	privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		t.Fatalf("rsa.GenerateKey() error = %v", err)
+	}
+
+	der, err := x509.MarshalPKCS8PrivateKey(privateKey)
+	if err != nil {
+		t.Fatalf("x509.MarshalPKCS8PrivateKey() error = %v", err)
+	}
+
+	return string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}))
+}
+
+func assertEncryptedMQTTMessage(t *testing.T, payload []byte) {
+	t.Helper()
+
+	var got Message
+	if err := json.Unmarshal(payload, &got); err != nil {
+		t.Fatalf("json.Unmarshal() error = %v", err)
+	}
+
+	if !got.Encrypted {
+		t.Fatal("message is not marked encrypted")
+	}
+	if got.Payload.EncryptedValue == "" {
+		t.Fatal("encrypted value is empty")
+	}
+	if got.Payload.Signature == "" {
+		t.Fatal("signature is empty")
+	}
+	if len(got.Payload.Value) != 0 {
+		t.Fatalf("payload value = %#v, want cleared map", got.Payload.Value)
+	}
+}
+
+func hookPKCS8PrivateKeyParser(t *testing.T, parser func([]byte) (any, error)) func() {
+	t.Helper()
+
+	original := parsePKCS8PrivateKey
+	parsePKCS8PrivateKey = parser
+
+	return func() {
+		parsePKCS8PrivateKey = original
+	}
+}
+
+func resetMQTTPrivateKeyCache(t *testing.T) {
+	t.Helper()
+	rsaPrivateKeyCache = newRSAPrivateKeyCache()
+}

machinery/src/models/recording_upload_metadata.go

@@ -16,6 +16,7 @@ type RecordingUploadMetadata struct {
 	Timestamp int64   `json:"timestamp"` // Unix milliseconds.
 	Duration  uint64  `json:"duration"`  // Milliseconds.
 	FPS       float64 `json:"fps,omitempty"`
+	Encrypted bool    `json:"encrypted,omitempty"`
 }
 
 // RecordingUploadMetadataFileName returns the queue marker name associated

machinery/src/onvif/discovery.go

@@ -13,8 +13,8 @@ import (
 	"time"
 
 	onvifc "github.com/cedricve/go-onvif"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 // scanPort describes a TCP port we probe while scanning the local network,
@@ -105,7 +105,7 @@ func DiscoverDevices(timeout time.Duration, subnets ...string) []models.Discover
 	// 1) ONVIF WS-Discovery. This is quick and reliable for ONVIF cameras.
 	onvifDevices, err := onvifc.StartDiscovery(timeout)
 	if err != nil {
-		log.Log.Error("onvif.DiscoverDevices(): WS-Discovery failed: " + err.Error())
+		log.Error("onvif.DiscoverDevices(): WS-Discovery failed: " + err.Error())
 	} else {
 		for _, onvifDevice := range onvifDevices {
 			ip := hostFromXAddr(onvifDevice.XAddr)
@@ -129,7 +129,7 @@ func DiscoverDevices(timeout time.Duration, subnets ...string) []models.Discover
 	} else {
 		targets = localScanTargets()
 	}
-	log.Log.Info("onvif.DiscoverDevices(): scanning " + strconv.Itoa(len(targets)) + " hosts on the local network(s)")
+	log.Info("onvif.DiscoverDevices(): scanning " + strconv.Itoa(len(targets)) + " hosts on the local network(s)")
 
 	// Bound the amount of concurrent dials so we do not exhaust file
 	// descriptors on constrained devices (e.g. Raspberry Pi).
@@ -319,17 +319,17 @@ func targetsFromSubnets(subnets []string) []string {
 					targets = append(targets, subnet)
 				}
 			} else {
-				log.Log.Error("onvif.targetsFromSubnets(): invalid address '" + subnet + "'")
+				log.Error("onvif.targetsFromSubnets(): invalid address '" + subnet + "'")
 			}
 			continue
 		}
 		_, ipNet, err := net.ParseCIDR(subnet)
 		if err != nil || ipNet.IP.To4() == nil {
-			log.Log.Error("onvif.targetsFromSubnets(): invalid CIDR '" + subnet + "'")
+			log.Error("onvif.targetsFromSubnets(): invalid CIDR '" + subnet + "'")
 			continue
 		}
 		if ones, bits := ipNet.Mask.Size(); bits != 32 || ones < 22 {
-			log.Log.Error("onvif.targetsFromSubnets(): range '" + subnet + "' is too large to scan (use /22 or smaller)")
+			log.Error("onvif.targetsFromSubnets(): range '" + subnet + "' is too large to scan (use /22 or smaller)")
 			continue
 		}
 		for _, host := range hostsInNetwork(ipNet) {
@@ -352,7 +352,7 @@ func localScanTargets() []string {
 
 	interfaces, err := net.Interfaces()
 	if err != nil {
-		log.Log.Error("onvif.localScanTargets(): " + err.Error())
+		log.Error("onvif.localScanTargets(): " + err.Error())
 		return targets
 	}
 

machinery/src/onvif/events.go

@@ -7,9 +7,9 @@ import (
 	"strings"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/onvif/event/stream"
+	log "github.com/sirupsen/logrus"
 )
 
 // The library handles in-stream reconnect; these guards cover the
@@ -29,14 +29,14 @@ const (
 // blip, credential reload) the goroutine retries with exponential
 // backoff. Exits when ctx is cancelled.
 func HandleONVIFEventStream(ctx context.Context, configuration *models.Configuration, communication *models.Communication) {
-	log.Log.Debug("onvif.HandleONVIFEventStream(): started")
+	log.Debug("onvif.HandleONVIFEventStream(): started")
-	defer log.Log.Debug("onvif.HandleONVIFEventStream(): finished")
+	defer log.Debug("onvif.HandleONVIFEventStream(): finished")
 
 	if !isONVIFMotionEnabled(configuration.Config.Capture.ONVIFMotion) {
 		return
 	}
 	if configuration.Config.Capture.IPCamera.ONVIFXAddr == "" {
-		log.Log.Warning("onvif.HandleONVIFEventStream(): ONVIFMotion enabled but ONVIFXAddr is empty; nothing to do")
+		log.Warn("onvif.HandleONVIFEventStream(): ONVIFMotion enabled but ONVIFXAddr is empty; nothing to do")
 		return
 	}
 
@@ -66,23 +66,26 @@ func runStreamOnce(ctx context.Context, configuration *models.Configuration, com
 
 	device, _, err := ConnectToOnvifDevice(&camera)
 	if err != nil {
-		log.Log.Error("onvif.HandleONVIFEventStream(): connect: " + err.Error())
+		log.Error("onvif.HandleONVIFEventStream(): connect: " + err.Error())
 		return true
 	}
 
 	deviceID := resolveDeviceID(configuration.Name, camera.ONVIFXAddr)
 	s, err := stream.NewStream(ctx, device, stream.Options{DeviceID: deviceID})
 	if err != nil {
-		log.Log.Error("onvif.HandleONVIFEventStream(): open stream: " + err.Error())
+		log.Error("onvif.HandleONVIFEventStream(): open stream: " + err.Error())
 		return true
 	}
 	defer func() {
 		if err := s.Close(); err != nil {
-			log.Log.Debug("onvif.HandleONVIFEventStream(): close: " + err.Error())
+			log.Debug("onvif.HandleONVIFEventStream(): close: " + err.Error())
 		}
 	}()
 
-	log.Log.Info("onvif.HandleONVIFEventStream(): consuming events for " + deviceID)
+	log.WithFields(log.Fields{
+		"component": "onvif",
+		"event":     "event_stream_started",
+	}).Info("Consuming ONVIF events")
 
 	// recovering = the first successful event after an error streak
 	// logs a recovery line so on-call operators see the clear-of-
@@ -97,7 +100,10 @@ func runStreamOnce(ctx context.Context, configuration *models.Configuration, com
 				return false
 			}
 			if recovering {
-				log.Log.Info("onvif.HandleONVIFEventStream(): event stream recovered for " + deviceID)
+				log.WithFields(log.Fields{
+					"component": "onvif",
+					"event":     "event_stream_recovered",
+				}).Info("ONVIF event stream recovered")
 				recovering = false
 			}
 			dispatchEvent(ctx, ev, configuration, communication)
@@ -119,14 +125,14 @@ func runStreamOnce(ctx context.Context, configuration *models.Configuration, com
 func dispatchEvent(ctx context.Context, ev stream.Event, configuration *models.Configuration, communication *models.Communication) {
 	topic := sanitiseTopic(ev.Topic)
 	if ev.Kind != stream.KindMotion {
-		log.Log.Debug("onvif.dispatchEvent(): non-motion event " + ev.Kind.String() + " topic=" + topic)
+		log.Debug("onvif.dispatchEvent(): non-motion event " + ev.Kind.String() + " topic=" + topic)
 		return
 	}
 	if ev.State != stream.StateActive {
 		return
 	}
 	if !isTransition(ev.Operation) {
-		log.Log.Debug("onvif.dispatchEvent(): " + ev.Operation.String() + " is not a transition, not a trigger: topic=" + topic)
+		log.Debug("onvif.dispatchEvent(): " + ev.Operation.String() + " is not a transition, not a trigger: topic=" + topic)
 		return
 	}
 	if configuration.Config.Capture.Recording == "false" {
@@ -140,14 +146,15 @@ func dispatchEvent(ctx context.Context, ev stream.Event, configuration *models.C
 		Timestamp:       time.Now().Unix(),
 		NumberOfChanges: 0, // ONVIF does not quantify motion area.
 	}
-	select {
+	if ctx.Err() != nil {
-	case <-ctx.Done():
+		return
-	case communication.HandleMotion <- dataToPass:
+	}
+	if communication.TrySendMotion(dataToPass) {
 		// Logged on the send, not before it: this line records that a
 		// recording started, so a dropped event must not leave one.
-		log.Log.Debug("onvif.dispatchEvent(): recording trigger " + ev.Kind.String() + " topic=" + topic)
+		log.Debug("onvif.dispatchEvent(): recording trigger " + ev.Kind.String() + " topic=" + topic)
-	default:
+	} else {
-		log.Log.Debug("onvif.dispatchEvent(): HandleMotion full, dropping ONVIF motion event")
+		log.Debug("onvif.dispatchEvent(): HandleMotion full, dropping ONVIF motion event")
 	}
 }
 
@@ -185,13 +192,13 @@ func logStreamError(e error) {
 	var renew stream.ErrRenewFailed
 	switch {
 	case errors.As(e, &recreate):
-		log.Log.Error("onvif.HandleONVIFEventStream(): subscription recreate failed (camera may be offline): " + recreate.Err.Error())
+		log.Error("onvif.HandleONVIFEventStream(): subscription recreate failed (camera may be offline): " + recreate.Err.Error())
 	case errors.As(e, &renew):
-		log.Log.Debug("onvif.HandleONVIFEventStream(): renew failed (will recover via pull/recreate): " + renew.Err.Error())
+		log.Debug("onvif.HandleONVIFEventStream(): renew failed (will recover via pull/recreate): " + renew.Err.Error())
 	case errors.As(e, &pull):
-		log.Log.Debug("onvif.HandleONVIFEventStream(): pull failed (will retry): " + pull.Err.Error())
+		log.Debug("onvif.HandleONVIFEventStream(): pull failed (will retry): " + pull.Err.Error())
 	default:
-		log.Log.Info("onvif.HandleONVIFEventStream(): stream error: " + e.Error())
+		log.Info("onvif.HandleONVIFEventStream(): stream error: " + e.Error())
 	}
 }
 

machinery/src/onvif/events_test.go

@@ -3,6 +3,7 @@ package onvif
 import (
 	"bytes"
 	"context"
+	"errors"
 	"strings"
 	"testing"
 	"time"
@@ -26,17 +27,31 @@ func makeConfig(recording, onvifMotion, name string) *models.Configuration {
 	}
 }
 
-func makeCommunication(buffer int) *models.Communication {
+func makeCommunication(t *testing.T) *models.Communication {
-	return &models.Communication{
+	t.Helper()
-		HandleMotion: make(chan models.MotionDataPartial, buffer),
+	communication := &models.Communication{}
+	run := models.NewAgentRun(context.Background(), communication, false)
+	require.NoError(t, run.Activate())
+	t.Cleanup(func() {
+		run.Shutdown(context.Background(), errors.New("test complete"))
+	})
+	return communication
+}
+
+func motionEvents(t *testing.T, communication *models.Communication) <-chan models.MotionDataPartial {
+	t.Helper()
+	run := communication.CurrentRun()
+	if run == nil {
+		t.Fatal("expected active agent run")
 	}
+	return run.MotionEvents()
 }
 
 // --- dispatchEvent ---------------------------------------------------
 
 func TestDispatchEvent_MotionActive_SendsToHandleMotion(t *testing.T) {
 	cfg := makeConfig("true", "true", "cam-1")
-	comm := makeCommunication(1)
+	comm := makeCommunication(t)
 	ev := stream.Event{Kind: stream.KindMotion, State: stream.StateActive}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -44,7 +59,7 @@ func TestDispatchEvent_MotionActive_SendsToHandleMotion(t *testing.T) {
 	dispatchEvent(ctx, ev, cfg, comm)
 
 	select {
-	case m := <-comm.HandleMotion:
+	case m := <-motionEvents(t, comm):
 		assert.NotZero(t, m.Timestamp)
 	case <-time.After(time.Second):
 		t.Fatal("expected motion data on HandleMotion")
@@ -53,7 +68,7 @@ func TestDispatchEvent_MotionActive_SendsToHandleMotion(t *testing.T) {
 
 func TestDispatchEvent_MotionInactive_DoesNotSend(t *testing.T) {
 	cfg := makeConfig("true", "true", "cam-1")
-	comm := makeCommunication(1)
+	comm := makeCommunication(t)
 	ev := stream.Event{Kind: stream.KindMotion, State: stream.StateInactive}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -61,7 +76,7 @@ func TestDispatchEvent_MotionInactive_DoesNotSend(t *testing.T) {
 	dispatchEvent(ctx, ev, cfg, comm)
 
 	select {
-	case <-comm.HandleMotion:
+	case <-motionEvents(t, comm):
 		t.Fatal("inactive motion must not reach HandleMotion (motion-stop is a follow-up)")
 	case <-time.After(100 * time.Millisecond):
 	}
@@ -69,7 +84,7 @@ func TestDispatchEvent_MotionInactive_DoesNotSend(t *testing.T) {
 
 func TestDispatchEvent_NonMotionKindIgnored(t *testing.T) {
 	cfg := makeConfig("true", "true", "cam-1")
-	comm := makeCommunication(1)
+	comm := makeCommunication(t)
 	ev := stream.Event{Kind: stream.KindDigitalInput, State: stream.StateActive}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -77,7 +92,7 @@ func TestDispatchEvent_NonMotionKindIgnored(t *testing.T) {
 	dispatchEvent(ctx, ev, cfg, comm)
 
 	select {
-	case <-comm.HandleMotion:
+	case <-motionEvents(t, comm):
 		t.Fatal("non-motion kinds must not reach HandleMotion")
 	case <-time.After(100 * time.Millisecond):
 	}
@@ -108,7 +123,7 @@ func TestDispatchEvent_LogsTheTriggeringTopic(t *testing.T) {
 	buf := captureDebugLog(t)
 
 	cfg := makeConfig("true", "true", "cam-1")
-	comm := makeCommunication(1)
+	comm := makeCommunication(t)
 	ev := stream.Event{
 		Kind:  stream.KindMotion,
 		State: stream.StateActive,
@@ -147,7 +162,7 @@ func TestDispatchEvent_PropertyOperation(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			cfg := makeConfig("true", "true", "cam-1")
-			comm := makeCommunication(1)
+			comm := makeCommunication(t)
 			ev := stream.Event{
 				Kind:      stream.KindMotion,
 				State:     stream.StateActive,
@@ -160,7 +175,7 @@ func TestDispatchEvent_PropertyOperation(t *testing.T) {
 			dispatchEvent(ctx, ev, cfg, comm)
 
 			select {
-			case <-comm.HandleMotion:
+			case <-motionEvents(t, comm):
 				if !tt.wantSend {
 					t.Fatalf("%v must not trigger a recording", tt.op)
 				}
@@ -175,7 +190,7 @@ func TestDispatchEvent_PropertyOperation(t *testing.T) {
 
 func TestDispatchEvent_RecordingDisabled_DoesNotSend(t *testing.T) {
 	cfg := makeConfig("false", "true", "cam-1")
-	comm := makeCommunication(1)
+	comm := makeCommunication(t)
 	ev := stream.Event{Kind: stream.KindMotion, State: stream.StateActive}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -183,7 +198,7 @@ func TestDispatchEvent_RecordingDisabled_DoesNotSend(t *testing.T) {
 	dispatchEvent(ctx, ev, cfg, comm)
 
 	select {
-	case <-comm.HandleMotion:
+	case <-motionEvents(t, comm):
 		t.Fatal("Recording=false must gate the send (matches computervision behaviour)")
 	case <-time.After(100 * time.Millisecond):
 	}
@@ -191,9 +206,10 @@ func TestDispatchEvent_RecordingDisabled_DoesNotSend(t *testing.T) {
 
 func TestDispatchEvent_HandleMotionFull_DropsRatherThanBlocks(t *testing.T) {
 	cfg := makeConfig("true", "true", "cam-1")
-	// Pre-fill the buffer so the next send would block.
+	// Fill the buffer so the next send would be dropped.
-	comm := &models.Communication{HandleMotion: make(chan models.MotionDataPartial, 1)}
+	comm := makeCommunication(t)
-	comm.HandleMotion <- models.MotionDataPartial{}
+	for comm.TrySendMotion(models.MotionDataPartial{}) {
+	}
 	ev := stream.Event{Kind: stream.KindMotion, State: stream.StateActive}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -219,8 +235,9 @@ func TestDispatchEvent_CtxCancelledAndHandleMotionClosed_DoesNotPanic(t *testing
 	// send would panic. The ctx pre-check must short-circuit before the
 	// send is attempted.
 	cfg := makeConfig("true", "true", "cam-1")
-	comm := &models.Communication{HandleMotion: make(chan models.MotionDataPartial, 1)}
+	comm := makeCommunication(t)
-	close(comm.HandleMotion)
+	run := comm.CurrentRun()
+	run.Shutdown(context.Background(), errors.New("test shutdown"))
 	ev := stream.Event{Kind: stream.KindMotion, State: stream.StateActive}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -299,7 +316,7 @@ func TestDispatchEvent_OnlyRealTransitionsTrigger(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.op.String(), func(t *testing.T) {
 			cfg := makeConfig("true", "true", "cam-1")
-			comm := makeCommunication(1)
+			comm := makeCommunication(t)
 			ev := stream.Event{Kind: stream.KindMotion, State: stream.StateActive, Operation: tt.op}
 
 			ctx, cancel := context.WithCancel(context.Background())
@@ -307,10 +324,10 @@ func TestDispatchEvent_OnlyRealTransitionsTrigger(t *testing.T) {
 			dispatchEvent(ctx, ev, cfg, comm)
 
 			if tt.wantSend {
-				require.Len(t, comm.HandleMotion, 1, "%v must trigger a recording", tt.op)
+				require.Len(t, motionEvents(t, comm), 1, "%v must trigger a recording", tt.op)
 				return
 			}
-			require.Empty(t, comm.HandleMotion, "%v must not trigger a recording", tt.op)
+			require.Empty(t, motionEvents(t, comm), "%v must not trigger a recording", tt.op)
 		})
 	}
 }
@@ -361,8 +378,9 @@ func TestDispatchEvent_LogsTriggerOnlyWhenSent(t *testing.T) {
 	buf := captureDebugLog(t)
 
 	cfg := makeConfig("true", "true", "cam-1")
-	comm := &models.Communication{HandleMotion: make(chan models.MotionDataPartial, 1)}
+	comm := makeCommunication(t)
-	comm.HandleMotion <- models.MotionDataPartial{} // full
+	for comm.TrySendMotion(models.MotionDataPartial{}) {
+	}
 	ev := stream.Event{Kind: stream.KindMotion, State: stream.StateActive, Topic: "tns1:VideoSource/MotionAlarm"}
 
 	ctx, cancel := context.WithCancel(context.Background())

machinery/src/onvif/main.go

@@ -6,12 +6,12 @@ import (
 	"encoding/xml"
 	"errors"
 	"io"
+	"net/http"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/gin-gonic/gin"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/onvif"
 	"github.com/kerberos-io/onvif/device"
@@ -21,6 +21,7 @@ import (
 	"github.com/kerberos-io/onvif/ptz"
 	xsd "github.com/kerberos-io/onvif/xsd"
 	xsdonvif "github.com/kerberos-io/onvif/xsd/onvif"
+	log "github.com/sirupsen/logrus"
 )
 
 // Discover performs an advanced Fing/WiFiman-style scan of the local network
@@ -29,12 +30,12 @@ import (
 // `-action discover` CLI command. Optional subnets (CIDR, e.g.
 // "192.168.1.0/24") override the auto-detected local subnets.
 func Discover(timeout time.Duration, subnets ...string) {
-	log.Log.Info("onvif.Discover(): starting advanced network discovery")
+	log.Info("onvif.Discover(): starting advanced network discovery")
-	log.Log.Info("onvif.Discover(): this may take up to " + timeout.String() + " for the ONVIF probe plus the port scan")
+	log.Info("onvif.Discover(): this may take up to " + timeout.String() + " for the ONVIF probe plus the port scan")
 
 	devices := DiscoverDevices(timeout, subnets...)
 	if len(devices) == 0 {
-		log.Log.Info("onvif.Discover(): no devices discovered on the local network")
+		log.Info("onvif.Discover(): no devices discovered on the local network")
 		return
 	}
 
@@ -44,7 +45,7 @@ func Discover(timeout time.Duration, subnets ...string) {
 			cameraCount++
 		}
 	}
-	log.Log.Info("onvif.Discover(): found " + strconv.Itoa(len(devices)) + " device(s), " + strconv.Itoa(cameraCount) + " likely camera(s)")
+	log.Info("onvif.Discover(): found " + strconv.Itoa(len(devices)) + " device(s), " + strconv.Itoa(cameraCount) + " likely camera(s)")
 
 	for _, device := range devices {
 		label := "device"
@@ -84,7 +85,7 @@ func Discover(timeout time.Duration, subnets ...string) {
 		if device.RTSPURL != "" {
 			summary += " rtsp=" + device.RTSPURL
 		}
-		log.Log.Info(summary)
+		log.Info(summary)
 
 		// Detail the guessed RTSP stream URLs from the brand -> RTSP mapping.
 		for _, stream := range device.RTSPStreams {
@@ -97,15 +98,15 @@ func Discover(timeout time.Duration, subnets ...string) {
 				line += " (auth required)"
 			}
 			line += ": " + stream.URL
-			log.Log.Info(line)
+			log.Info(line)
 		}
 	}
 }
 
-func HandleONVIFActions(configuration *models.Configuration, communication *models.Communication) {
+func HandleONVIFActions(configuration *models.Configuration, communication *models.Communication, actions <-chan models.OnvifAction) {
-	log.Log.Debug("onvif.HandleONVIFActions(): started")
+	log.Debug("onvif.HandleONVIFActions(): started")
 
-	for onvifAction := range communication.HandleONVIF {
+	for onvifAction := range actions {
 
 		// First we'll get the desired PTZ action from the payload
 		// We need to know if we need to move left, right, up, down, zoom in, zoom out, center.
@@ -138,7 +139,7 @@ func HandleONVIFActions(configuration *models.Configuration, communication *mode
 						functions, _, _ := GetPTZFunctionsFromDevice(configurations)
 
 						// Log functions
-						log.Log.Debug("onvif.HandleONVIFActions(): functions: " + strings.Join(functions, ", "))
+						log.Debug("onvif.HandleONVIFActions(): functions: " + strings.Join(functions, ", "))
 
 						// Check if we need to use absolute or continuous move
 						/*canAbsoluteMove := false
@@ -159,20 +160,20 @@ func HandleONVIFActions(configuration *models.Configuration, communication *mode
 						// on the ContinuousPanTiltMove function which is more compatible with more cameras.
 						err = AbsolutePanTiltMoveFake(device, configurations, token, x, y, z)
 						if err != nil {
-							log.Log.Debug("onvif.HandleONVIFActions() - AbsolutePanTitleMoveFake: " + err.Error())
+							log.Debug("onvif.HandleONVIFActions() - AbsolutePanTitleMoveFake: " + err.Error())
 						} else {
-							log.Log.Info("onvif.HandleONVIFActions() - AbsolutePanTitleMoveFake: successfully moved camera.")
+							log.Info("onvif.HandleONVIFActions() - AbsolutePanTitleMoveFake: successfully moved camera.")
 						}
 
 						/*if canAbsoluteMove {
 							err = AbsolutePanTiltMove(device, configurations, token, x, y, z)
 							if err != nil {
-								log.Log.Error("HandleONVIFActions (AbsolutePanTitleMove): " + err.Error())
+								log.Error("HandleONVIFActions (AbsolutePanTitleMove): " + err.Error())
 							}
 						} else if canContinuousMove {
 							err = AbsolutePanTiltMoveFake(device, configurations, token, x, y, z)
 							if err != nil {
-								log.Log.Error("HandleONVIFActions (AbsolutePanTitleMoveFake): " + err.Error())
+								log.Error("HandleONVIFActions (AbsolutePanTitleMoveFake): " + err.Error())
 							}
 						}*/
 
@@ -182,9 +183,9 @@ func HandleONVIFActions(configuration *models.Configuration, communication *mode
 						preset := ptzAction.Preset
 						err := GoToPresetFromDevice(device, preset)
 						if err != nil {
-							log.Log.Debug("onvif.HandleONVIFActions() - GotoPreset: " + err.Error())
+							log.Debug("onvif.HandleONVIFActions() - GotoPreset: " + err.Error())
 						} else {
-							log.Log.Info("onvif.HandleONVIFActions() - GotoPreset: successfully moved camera")
+							log.Info("onvif.HandleONVIFActions() - GotoPreset: successfully moved camera")
 						}
 
 					} else if onvifAction.Action == "ptz" {
@@ -196,9 +197,9 @@ func HandleONVIFActions(configuration *models.Configuration, communication *mode
 								// We will move the camera to zero position.
 								err := AbsolutePanTiltMove(device, configurations, token, 0, 0, 0)
 								if err != nil {
-									log.Log.Debug("onvif.HandleONVIFActions() - AbsolutePanTitleMove: " + err.Error())
+									log.Debug("onvif.HandleONVIFActions() - AbsolutePanTitleMove: " + err.Error())
 								} else {
-									log.Log.Info("onvif.HandleONVIFActions() - AbsolutePanTitleMove: successfully centered camera")
+									log.Info("onvif.HandleONVIFActions() - AbsolutePanTitleMove: successfully centered camera")
 								}
 
 							} else {
@@ -225,9 +226,9 @@ func HandleONVIFActions(configuration *models.Configuration, communication *mode
 
 								err := ContinuousPanTilt(device, configurations, token, x, y)
 								if err != nil {
-									log.Log.Debug("onvif.HandleONVIFActions() - ContinuousPanTilt: " + err.Error())
+									log.Debug("onvif.HandleONVIFActions() - ContinuousPanTilt: " + err.Error())
 								} else {
-									log.Log.Info("onvif.HandleONVIFActions() - ContinuousPanTilt: successfully pan tilted camera")
+									log.Info("onvif.HandleONVIFActions() - ContinuousPanTilt: successfully pan tilted camera")
 								}
 							}
 						}
@@ -237,9 +238,9 @@ func HandleONVIFActions(configuration *models.Configuration, communication *mode
 							zoom := ptzAction.Zoom
 							err := ContinuousZoom(device, configurations, token, zoom)
 							if err != nil {
-								log.Log.Debug("onvif.HandleONVIFActions() - ContinuousZoom: " + err.Error())
+								log.Debug("onvif.HandleONVIFActions() - ContinuousZoom: " + err.Error())
 							} else {
-								log.Log.Info("onvif.HandleONVIFActions() - ContinuousZoom: successfully zoomed camera")
+								log.Info("onvif.HandleONVIFActions() - ContinuousZoom: successfully zoomed camera")
 							}
 						}
 					}
@@ -247,11 +248,11 @@ func HandleONVIFActions(configuration *models.Configuration, communication *mode
 			}
 		}
 	}
-	log.Log.Debug("onvif.HandleONVIFActions(): finished")
+	log.Debug("onvif.HandleONVIFActions(): finished")
 }
 
 func ConnectToOnvifDevice(cameraConfiguration *models.IPCamera) (*onvif.Device, device.GetCapabilitiesResponse, error) {
-	log.Log.Debug("onvif.ConnectToOnvifDevice(): started")
+	log.Debug("onvif.ConnectToOnvifDevice(): started")
 	dev, err := onvif.NewDevice(onvif.DeviceParams{
 		Xaddr:    cameraConfiguration.ONVIFXAddr,
 		Username: cameraConfiguration.ONVIFUsername,
@@ -269,7 +270,7 @@ func ConnectToOnvifDevice(cameraConfiguration *models.IPCamera) (*onvif.Device,
 			AuthMode: "digest",
 		})
 		if err != nil {
-			log.Log.Debug("onvif.ConnectToOnvifDevice(): " + err.Error())
+			log.Debug("onvif.ConnectToOnvifDevice(): " + err.Error())
 		}
 	}
 
@@ -277,7 +278,7 @@ func ConnectToOnvifDevice(cameraConfiguration *models.IPCamera) (*onvif.Device,
 		getCapabilities := device.GetCapabilities{Category: []xsdonvif.CapabilityCategory{"All"}}
 		resp, err := dev.CallMethod(getCapabilities)
 		if err != nil {
-			log.Log.Error("onvif.ConnectToOnvifDevice(): " + err.Error())
+			log.Error("onvif.ConnectToOnvifDevice(): " + err.Error())
 		}
 
 		var b []byte
@@ -285,24 +286,24 @@ func ConnectToOnvifDevice(cameraConfiguration *models.IPCamera) (*onvif.Device,
 			b, err = io.ReadAll(resp.Body)
 			resp.Body.Close() // Ensure the response body is closed
 			if err != nil {
-				log.Log.Error("onvif.ConnectToOnvifDevice(): " + err.Error())
+				log.Error("onvif.ConnectToOnvifDevice(): " + err.Error())
 			}
 		}
 		stringBody := string(b)
 		decodedXML, et, err := getXMLNode(stringBody, "GetCapabilitiesResponse")
 		if err != nil {
-			log.Log.Error("onvif.ConnectToOnvifDevice(): " + err.Error())
+			log.Error("onvif.ConnectToOnvifDevice(): " + err.Error())
 		} else {
 			if err := decodedXML.DecodeElement(&capabilities, et); err != nil {
-				log.Log.Error("onvif.ConnectToOnvifDevice(): " + err.Error())
+				log.Error("onvif.ConnectToOnvifDevice(): " + err.Error())
 			} else {
-				log.Log.Debug("onvif.ConnectToOnvifDevice(): capabilities.")
+				log.Debug("onvif.ConnectToOnvifDevice(): capabilities.")
 			}
 		}
 
-		log.Log.Info("onvif.ConnectToOnvifDevice(): successfully connected to device")
+		log.Info("onvif.ConnectToOnvifDevice(): successfully connected to device")
 	}
-	log.Log.Debug("onvif.ConnectToOnvifDevice(): finished")
+	log.Debug("onvif.ConnectToOnvifDevice(): finished")
 	return dev, capabilities, err
 }
 
@@ -319,13 +320,13 @@ func GetTokenFromProfile(device *onvif.Device, profileId int) (xsdonvif.Referenc
 			resp.Body.Close() // Ensure the response body is closed
 			decodedXML, et, err := getXMLNode(stringBody, "GetProfilesResponse")
 			if err != nil {
-				log.Log.Debug("onvif.GetTokenFromProfile(): " + err.Error())
+				log.Debug("onvif.GetTokenFromProfile(): " + err.Error())
 				return profileToken, err
 			} else {
 				// Decode the profiles from the server
 				var mProfilesResp media.GetProfilesResponse
 				if err := decodedXML.DecodeElement(&mProfilesResp, et); err != nil {
-					log.Log.Debug("onvif.GetTokenFromProfile(): " + err.Error())
+					log.Debug("onvif.GetTokenFromProfile(): " + err.Error())
 				}
 
 				// We'll try to get the token from a preferred profile
@@ -356,11 +357,11 @@ func GetPTZConfigurationsFromDevice(device *onvif.Device) (ptz.GetConfigurations
 		stringBody := string(b)
 		decodedXML, et, err := getXMLNode(stringBody, "GetConfigurationsResponse")
 		if err != nil {
-			log.Log.Debug("onvif.GetPTZConfigurationsFromDevice(): " + err.Error())
+			log.Debug("onvif.GetPTZConfigurationsFromDevice(): " + err.Error())
 			return configurations, err
 		} else {
 			if err := decodedXML.DecodeElement(&configurations, et); err != nil {
-				log.Log.Debug("onvif.GetPTZConfigurationsFromDevice(): " + err.Error())
+				log.Debug("onvif.GetPTZConfigurationsFromDevice(): " + err.Error())
 				return configurations, err
 			}
 		}
@@ -386,22 +387,22 @@ func GetPositionFromDevice(configuration models.Configuration) (xsdonvif.PTZVect
 					x := strconv.FormatFloat(position.PanTilt.X, 'f', 6, 64)
 					y := strconv.FormatFloat(position.PanTilt.Y, 'f', 6, 64)
 					z := strconv.FormatFloat(position.Zoom.X, 'f', 6, 64)
-					log.Log.Info("onvif.GetPositionFromDevice(): successfully got position (" + x + ", " + y + ", " + z + ")")
+					log.Info("onvif.GetPositionFromDevice(): successfully got position (" + x + ", " + y + ", " + z + ")")
 					return position, err
 				} else {
-					log.Log.Debug("onvif.GetPositionFromDevice(): position is nil")
+					log.Debug("onvif.GetPositionFromDevice(): position is nil")
 					return position, errors.New("position is nil")
 				}
 			} else {
-				log.Log.Debug("onvif.GetPositionFromDevice(): " + err.Error())
+				log.Debug("onvif.GetPositionFromDevice(): " + err.Error())
 				return position, err
 			}
 		} else {
-			log.Log.Debug("onvif.GetPositionFromDevice(): " + err.Error())
+			log.Debug("onvif.GetPositionFromDevice(): " + err.Error())
 			return position, err
 		}
 	} else {
-		log.Log.Debug("onvif.GetPositionFromDevice(): " + err.Error())
+		log.Debug("onvif.GetPositionFromDevice(): " + err.Error())
 		return position, err
 	}
 }
@@ -426,11 +427,11 @@ func GetPosition(device *onvif.Device, token xsdonvif.ReferenceToken) (xsdonvif.
 		stringBody := string(b)
 		decodedXML, et, err := getXMLNode(stringBody, "GetStatusResponse")
 		if err != nil {
-			log.Log.Error("GetPositionFromDevice: " + err.Error())
+			log.Error("GetPositionFromDevice: " + err.Error())
 			return position, err
 		} else {
 			if err := decodedXML.DecodeElement(&status, et); err != nil {
-				log.Log.Error("GetPositionFromDevice: " + err.Error())
+				log.Error("GetPositionFromDevice: " + err.Error())
 				return position, err
 			}
 		}
@@ -439,6 +440,36 @@ func GetPosition(device *onvif.Device, token xsdonvif.ReferenceToken) (xsdonvif.
 	return position, err
 }
 
+func handleONVIFResponse(operation string, response *http.Response, requestErr error) error {
+	fields := log.Fields{
+		"component": "onvif",
+		"operation": operation,
+	}
+	var responseErr error
+	if response != nil {
+		fields["status_code"] = response.StatusCode
+		fields["response_bytes"], responseErr = io.Copy(io.Discard, response.Body)
+		response.Body.Close()
+	}
+
+	err := errors.Join(requestErr, responseErr)
+	if response == nil && err == nil {
+		err = errors.New("ONVIF operation returned no response")
+	}
+	if err == nil && (response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices) {
+		err = errors.New("ONVIF operation was rejected")
+	}
+	if err != nil {
+		fields["event"] = "operation_failed"
+		log.WithError(err).WithFields(fields).Error("ONVIF operation failed")
+		return err
+	}
+
+	fields["event"] = "operation_completed"
+	log.WithFields(fields).Debug("ONVIF operation completed")
+	return nil
+}
+
 func AbsolutePanTiltMove(device *onvif.Device, configuration ptz.GetConfigurationsResponse, token xsdonvif.ReferenceToken, pan float64, tilt float64, zoom float64) error {
 
 	absolutePantiltVector := xsdonvif.Vector2D{
@@ -459,18 +490,7 @@ func AbsolutePanTiltMove(device *onvif.Device, configuration ptz.GetConfiguratio
 			Zoom:    &absoluteZoomVector,
 		},
 	})
-
+	return handleONVIFResponse("absolute_move", resp, err)
-	var b []byte
-	if resp != nil {
-		b, err = io.ReadAll(resp.Body)
-		resp.Body.Close()
-	}
-	if err != nil {
-		log.Log.Error("AbsoluteMove: " + err.Error())
-	}
-	log.Log.Info("AbsoluteMove: " + string(b))
-
-	return err
 }
 
 // This function will simulate the AbsolutePanTiltMove function.
@@ -479,7 +499,7 @@ func AbsolutePanTiltMove(device *onvif.Device, configuration ptz.GetConfiguratio
 func AbsolutePanTiltMoveFake(device *onvif.Device, configuration ptz.GetConfigurationsResponse, token xsdonvif.ReferenceToken, pan float64, tilt float64, zoom float64) error {
 	position, err := GetPosition(device, token)
 	if position.PanTilt.X >= pan-0.01 && position.PanTilt.X <= pan+0.01 && position.PanTilt.Y >= tilt-0.01 && position.PanTilt.Y <= tilt+0.01 && position.Zoom.X >= zoom-0.01 && position.Zoom.X <= zoom+0.01 {
-		log.Log.Debug("AbsolutePanTiltMoveFake: already at position")
+		log.Debug("AbsolutePanTiltMoveFake: already at position")
 	} else {
 
 		// The speed of panning, the higher the faster we'll pan the camera
@@ -518,7 +538,7 @@ func ZoomOutCompletely(device *onvif.Device, configuration ptz.GetConfigurations
 		},
 	})
 	if err != nil {
-		log.Log.Error("ZoomOutCompletely: " + err.Error())
+		log.Error("ZoomOutCompletely: " + err.Error())
 	}
 
 	for {
@@ -534,7 +554,7 @@ func ZoomOutCompletely(device *onvif.Device, configuration ptz.GetConfigurations
 		Zoom:         true,
 	})
 	if err != nil {
-		log.Log.Error("ZoomOutCompletely: " + err.Error())
+		log.Error("ZoomOutCompletely: " + err.Error())
 	}
 	return err
 }
@@ -564,16 +584,7 @@ func PanUntilPosition(device *onvif.Device, configuration ptz.GetConfigurationsR
 				PanTilt: panTiltVector,
 			},
 		})
-
+		err = handleONVIFResponse("continuous_pan", resp, err)
-		var b []byte
-		if resp != nil {
-			b, err = io.ReadAll(resp.Body)
-			resp.Body.Close()
-		}
-		if err != nil {
-			log.Log.Error("ContinuousPanTiltMove (Pan): " + err.Error())
-		}
-		log.Log.Debug("ContinuousPanTiltMove (Pan): " + string(b))
 
 		// While moving we'll check if we reached the desired position.
 		// or if we overshot the desired position.
@@ -598,7 +609,7 @@ func PanUntilPosition(device *onvif.Device, configuration ptz.GetConfigurationsR
 		})
 
 		if err != nil {
-			log.Log.Error("ContinuousPanTiltMove (Pan): " + err.Error())
+			log.Error("ContinuousPanTiltMove (Pan): " + err.Error())
 		}
 	}
 	return err
@@ -632,17 +643,7 @@ func TiltUntilPosition(device *onvif.Device, configuration ptz.GetConfigurations
 			ProfileToken: &token,
 			Velocity:     velocity,
 		})
-
+		err = handleONVIFResponse("continuous_tilt", resp, err)
-		var b []byte
-		if resp != nil {
-			b, err = io.ReadAll(resp.Body)
-			resp.Body.Close()
-		}
-
-		if err != nil {
-			log.Log.Error("ContinuousPanTiltMove (Tilt): " + err.Error())
-		}
-		log.Log.Debug("ContinuousPanTiltMove (Tilt) " + string(b))
 
 		// While moving we'll check if we reached the desired position.
 		// or if we overshot the desired position.
@@ -667,7 +668,7 @@ func TiltUntilPosition(device *onvif.Device, configuration ptz.GetConfigurations
 		})
 
 		if err != nil {
-			log.Log.Error("ContinuousPanTiltMove (Tilt): " + err.Error())
+			log.Error("ContinuousPanTiltMove (Tilt): " + err.Error())
 		}
 	}
 	return err
@@ -697,17 +698,7 @@ func ZoomUntilPosition(device *onvif.Device, configuration ptz.GetConfigurations
 				Zoom: zoomVector,
 			},
 		})
-
+		err = handleONVIFResponse("continuous_zoom_to_position", resp, err)
-		var b []byte
-		if resp != nil {
-			b, err = io.ReadAll(resp.Body)
-			resp.Body.Close()
-		}
-		if err != nil {
-			log.Log.Error("ContinuousPanTiltMove (Zoom): " + err.Error())
-		}
-
-		log.Log.Debug("ContinuousPanTiltMove (Zoom) " + string(b))
 
 		// While moving we'll check if we reached the desired position.
 		// or if we overshot the desired position.
@@ -732,7 +723,7 @@ func ZoomUntilPosition(device *onvif.Device, configuration ptz.GetConfigurations
 		})
 
 		if err != nil {
-			log.Log.Error("ContinuousPanTiltMove (Zoom): " + err.Error())
+			log.Error("ContinuousPanTiltMove (Zoom): " + err.Error())
 		}
 	}
 	return err
@@ -752,17 +743,7 @@ func ContinuousPanTilt(device *onvif.Device, configuration ptz.GetConfigurations
 			PanTilt: panTiltVector,
 		},
 	})
-
+	moveErr := handleONVIFResponse("continuous_pan_tilt", resp, err)
-	var b []byte
-	if resp != nil {
-		b, err = io.ReadAll(resp.Body)
-		resp.Body.Close()
-	}
-	if err != nil {
-		log.Log.Error("ContinuousPanTiltMove: " + err.Error())
-	}
-
-	log.Log.Debug("ContinuousPanTiltMove: " + string(b))
 
 	time.Sleep(200 * time.Millisecond)
 
@@ -770,18 +751,8 @@ func ContinuousPanTilt(device *onvif.Device, configuration ptz.GetConfigurations
 		ProfileToken: token,
 		PanTilt:      true,
 	})
-
+	stopErr := handleONVIFResponse("continuous_pan_tilt_stop", resp, err)
-	b = []byte{}
+	return errors.Join(moveErr, stopErr)
-	if resp != nil {
-		b, err = io.ReadAll(resp.Body)
-		resp.Body.Close()
-	}
-
-	if err != nil {
-		log.Log.Error("ContinuousPanTiltMove: " + err.Error())
-	}
-
-	return err
 }
 
 func ContinuousZoom(device *onvif.Device, configuration ptz.GetConfigurationsResponse, token xsdonvif.ReferenceToken, zoom float64) error {
@@ -799,36 +770,26 @@ func ContinuousZoom(device *onvif.Device, configuration ptz.GetConfigurationsRes
 		ProfileToken: &token,
 		Velocity:     &velocity,
 	})
-
+	moveErr := handleONVIFResponse("continuous_zoom", resp, err)
-	var b []byte
-	if resp != nil {
-		b, err = io.ReadAll(resp.Body)
-		resp.Body.Close()
-	}
-	if err != nil {
-		log.Log.Error("onvif.main.ContinuousZoom(): " + err.Error())
-	}
-
-	log.Log.Debug("onvif.main.ContinuousZoom(): " + string(b))
 	time.Sleep(500 * time.Millisecond)
 
-	_, err = device.CallMethod(ptz.Stop{
+	resp, err = device.CallMethod(ptz.Stop{
 		ProfileToken: token,
 		Zoom:         true,
 	})
-
+	stopErr := handleONVIFResponse("continuous_zoom_stop", resp, err)
-	if err != nil {
+	return errors.Join(moveErr, stopErr)
-		log.Log.Error("onvif.main.ContinuousZoom(): " + err.Error())
-	}
-
-	return err
 }
 
 func GetCapabilitiesFromDevice(dev *onvif.Device) []string {
 	var capabilities []string
 	services := dev.GetServices()
 	for key, _ := range services {
-		log.Log.Debug("onvif.main.GetCapabilitiesFromDevice(): has key: " + key)
+		log.WithFields(log.Fields{
+			"capability": key,
+			"component":  "onvif",
+			"event":      "capability_detected",
+		}).Debug("ONVIF capability detected")
 		if key != "" {
 			keyParts := strings.Split(key, "/")
 			if len(keyParts) > 0 {
@@ -859,11 +820,11 @@ func GetPresetsFromDevice(device *onvif.Device) ([]models.OnvifActionPreset, err
 			stringBody := string(b)
 			decodedXML, et, err := getXMLNode(stringBody, "GetPresetsResponse")
 			if err != nil {
-				log.Log.Error("onvif.main.GetPresetsFromDevice(): " + err.Error())
+				log.Error("onvif.main.GetPresetsFromDevice(): " + err.Error())
 				return presets, err
 			} else {
 				if err := decodedXML.DecodeElement(&presetsResponse, et); err != nil {
-					log.Log.Error("onvif.main.GetPresetsFromDevice(): " + err.Error())
+					log.Error("onvif.main.GetPresetsFromDevice(): " + err.Error())
 					return presets, err
 				}
 
@@ -876,15 +837,15 @@ func GetPresetsFromDevice(device *onvif.Device) ([]models.OnvifActionPreset, err
 					presetsList += string(preset.Name) + " (" + string(preset.Token) + "), "
 					presets = append(presets, p)
 				}
-				log.Log.Debug("onvif.main.GetPresetsFromDevice(): " + presetsList)
+				log.Debug("onvif.main.GetPresetsFromDevice(): " + presetsList)
 
 				return presets, err
 			}
 		} else {
-			log.Log.Error("onvif.main.GetPresetsFromDevice(): " + err.Error())
+			log.Error("onvif.main.GetPresetsFromDevice(): " + err.Error())
 		}
 	} else {
-		log.Log.Error("onvif.main.GetPresetsFromDevice(): " + err.Error())
+		log.Error("onvif.main.GetPresetsFromDevice(): " + err.Error())
 	}
 
 	return presets, err
@@ -910,20 +871,20 @@ func GoToPresetFromDevice(device *onvif.Device, presetName string) error {
 			stringBody := string(b)
 			decodedXML, et, err := getXMLNode(stringBody, "GotoPresetResponses")
 			if err != nil {
-				log.Log.Error("onvif.main.GoToPresetFromDevice(): " + err.Error())
+				log.Error("onvif.main.GoToPresetFromDevice(): " + err.Error())
 				return err
 			} else {
 				if err := decodedXML.DecodeElement(&goToPresetResponse, et); err != nil {
-					log.Log.Error("onvif.main.GoToPresetFromDevice(): " + err.Error())
+					log.Error("onvif.main.GoToPresetFromDevice(): " + err.Error())
 					return err
 				}
 				return err
 			}
 		} else {
-			log.Log.Error("onvif.main.GoToPresetFromDevice(): " + err.Error())
+			log.Error("onvif.main.GoToPresetFromDevice(): " + err.Error())
 		}
 	} else {
-		log.Log.Error("onvif.main.GoToPresetFromDevice(): " + err.Error())
+		log.Error("onvif.main.GoToPresetFromDevice(): " + err.Error())
 	}
 
 	return err
@@ -1068,10 +1029,10 @@ func CreatePullPointSubscription(dev *onvif.Device) (string, error) {
 			stringBody := string(b2)
 			decodedXML, et, err := getXMLNode(stringBody, "CreatePullPointSubscriptionResponse")
 			if err != nil {
-				log.Log.Debug("onvif.main.CreatePullPointSubscription(): " + err.Error())
+				log.Debug("onvif.main.CreatePullPointSubscription(): " + err.Error())
 			} else {
 				if err := decodedXML.DecodeElement(&createPullPointSubscriptionResponse, et); err != nil {
-					log.Log.Error("onvif.main.CreatePullPointSubscription(): " + err.Error())
+					log.Error("onvif.main.CreatePullPointSubscription(): " + err.Error())
 				} else {
 					pullPointAdress = string(createPullPointSubscriptionResponse.SubscriptionReference.Address)
 				}
@@ -1087,25 +1048,40 @@ func UnsubscribePullPoint(dev *onvif.Device, pullPointAddress string) error {
 	unsubscribe := event.Unsubscribe{}
 	requestBody, err := xml.Marshal(unsubscribe)
 	if err != nil {
-		log.Log.Error("onvif.main.UnsubscribePullPoint(): " + err.Error())
+		log.WithError(err).WithFields(log.Fields{
+			"component": "onvif",
+			"event":     "unsubscribe_request_encoding_failed",
+		}).Error("Failed to encode ONVIF unsubscribe request")
+		return err
 	}
 
 	res, err := dev.SendSoap(pullPointAddress, string(requestBody))
 	if err != nil {
-		log.Log.Error("onvif.main.UnsubscribePullPoint(): " + err.Error())
+		log.WithError(err).WithFields(log.Fields{
+			"component": "onvif",
+			"event":     "unsubscribe_request_failed",
+		}).Error("Failed to send ONVIF unsubscribe request")
+		return err
 	}
-	if res != nil {
+	if res == nil {
-		b, err := io.ReadAll(res.Body)
+		return errors.New("ONVIF unsubscribe returned no response")
-		res.Body.Close() // Ensure the response body is closed
-		if err == nil {
-			stringBody := string(b)
-			log.Log.Debug("onvif.main.UnsubscribePullPoint(): " + stringBody)
-		}
-		if err != nil {
-			log.Log.Error("onvif.main.UnsubscribePullPoint(): " + err.Error())
-		}
 	}
-	return err
+	defer res.Body.Close()
+	responseBytes, err := io.Copy(io.Discard, res.Body)
+	if err != nil {
+		log.WithError(err).WithFields(log.Fields{
+			"component": "onvif",
+			"event":     "unsubscribe_response_read_failed",
+		}).Error("Failed to read ONVIF unsubscribe response")
+		return err
+	}
+	log.WithFields(log.Fields{
+		"component":      "onvif",
+		"event":          "unsubscribe_completed",
+		"response_bytes": responseBytes,
+		"status_code":    res.StatusCode,
+	}).Debug("ONVIF unsubscribe completed")
+	return nil
 }
 
 // Look for Source of input and output
@@ -1126,7 +1102,13 @@ func GetInputOutputs() ([]ONVIFEvents, error) {
 		eventsArray = append(eventsArray, *value)
 	}
 	for _, value := range eventsArray {
-		log.Log.Debug("onvif.main.GetInputOutputs(): " + value.Key + " - " + value.Value + " (" + strconv.FormatInt(value.Timestamp, 10) + ")")
+		log.WithFields(log.Fields{
+			"component":      "onvif",
+			"event":          "input_output_state",
+			"input_output":   value.Key,
+			"state":          value.Value,
+			"timestamp_unix": value.Timestamp,
+		}).Debug("ONVIF input/output state")
 	}
 	return eventsArray, nil
 }
@@ -1143,7 +1125,7 @@ func GetEventMessages(dev *onvif.Device, pullPointAddress string) ([]ONVIFEvents
 		// We were able to create a subscription to the device. Now pull some messages from the subscription.
 		subscriptionURI := pullPointAddress
 		if subscriptionURI == "" {
-			log.Log.Error("onvif.main.GetEventMessages(): subscriptionURI is empty")
+			log.Error("onvif.main.GetEventMessages(): subscriptionURI is empty")
 		} else {
 			// Pull message
 			pullMessage := event.PullMessages{
@@ -1152,12 +1134,12 @@ func GetEventMessages(dev *onvif.Device, pullPointAddress string) ([]ONVIFEvents
 			}
 			requestBody, err := xml.Marshal(pullMessage)
 			if err != nil {
-				log.Log.Error("onvif.main.GetEventMessages(pullMessages): " + err.Error())
+				log.Error("onvif.main.GetEventMessages(pullMessages): " + err.Error())
 				return eventsArray, err
 			}
 			res, err := dev.SendSoap(string(subscriptionURI), string(requestBody))
 			if err != nil {
-				log.Log.Error("onvif.main.GetEventMessages(pullMessages): " + err.Error())
+				log.Error("onvif.main.GetEventMessages(pullMessages): " + err.Error())
 				return eventsArray, err
 			}
 
@@ -1169,11 +1151,11 @@ func GetEventMessages(dev *onvif.Device, pullPointAddress string) ([]ONVIFEvents
 					stringBody := string(bs)
 					decodedXML, et, err := getXMLNode(stringBody, "PullMessagesResponse")
 					if err != nil {
-						log.Log.Error("onvif.main.GetEventMessages(pullMessages): " + err.Error())
+						log.Error("onvif.main.GetEventMessages(pullMessages): " + err.Error())
 						return eventsArray, err
 					} else {
 						if err := decodedXML.DecodeElement(&pullMessagesResponse, et); err != nil {
-							log.Log.Error("onvif.main.GetEventMessages(pullMessages): " + err.Error())
+							log.Error("onvif.main.GetEventMessages(pullMessages): " + err.Error())
 							return eventsArray, err
 						}
 					}
@@ -1181,9 +1163,9 @@ func GetEventMessages(dev *onvif.Device, pullPointAddress string) ([]ONVIFEvents
 			}
 
 			for _, message := range pullMessagesResponse.NotificationMessage {
-				log.Log.Debug("onvif.main.GetEventMessages(pullMessages): " + string(message.Topic.TopicKinds))
+				log.Debug("onvif.main.GetEventMessages(pullMessages): " + string(message.Topic.TopicKinds))
 				//if len(message.Message.Message.Data.SimpleItem) > 0 {
-				//	log.Log.Debug("onvif.main.GetEventMessages(pullMessages): " + string(message.Message.Message.Data.SimpleItem[0].Name) + " " + string(message.Message.Message.Data.SimpleItem[0].Value))
+				// log.Debug("onvif.main.GetEventMessages(pullMessages): " + string(message.Message.Message.Data.SimpleItem[0].Name) + " " + string(message.Message.Message.Data.SimpleItem[0].Value))
 				//}
 				if message.Topic.TopicKinds == "tns1:Device/Trigger/Relay" ||
 					message.Topic.TopicKinds == "tns1:Device/tns1:Trigger/tns1:Relay" { // This is for avigilon cameras
@@ -1193,7 +1175,13 @@ func GetEventMessages(dev *onvif.Device, pullPointAddress string) ([]ONVIFEvents
 							key := string(message.Message.Message.Source.SimpleItem[0].Value)
 							value := string(message.Message.Message.Data.SimpleItem[0].Value)
 							propertyOperation := string(message.Message.Message.PropertyOperation)
-							log.Log.Debug("onvif.main.GetEventMessages(pullMessages) output: " + key + " " + value + " (" + propertyOperation + ")")
+							log.WithFields(log.Fields{
+								"component":          "onvif",
+								"event":              "event_property_output",
+								"property":           key,
+								"property_operation": propertyOperation,
+								"value":              value,
+							}).Debug("ONVIF event property output")
 
 							// Depending on the onvif library they might use different values for active and inactive.
 							if value == "active" || value == "1" {
@@ -1227,7 +1215,13 @@ func GetEventMessages(dev *onvif.Device, pullPointAddress string) ([]ONVIFEvents
 							key := string(message.Message.Message.Source.SimpleItem[0].Value)
 							value := string(message.Message.Message.Data.SimpleItem[0].Value)
 							propertyOperation := string(message.Message.Message.PropertyOperation)
-							log.Log.Debug("onvif.main.GetEventMessages(pullMessages) input: " + key + " " + value + " (" + propertyOperation + ")")
+							log.WithFields(log.Fields{
+								"component":          "onvif",
+								"event":              "event_property_input",
+								"property":           key,
+								"property_operation": propertyOperation,
+								"value":              value,
+							}).Debug("ONVIF event property input")
 
 							// Depending on the onvif library they might use different values for active and inactive.
 							if value == "active" || value == "1" {
@@ -1281,11 +1275,11 @@ func GetDigitalInputs(dev *onvif.Device) (device.GetDigitalInputsResponse, error
 			stringBody := string(b)
 			decodedXML, et, err := getXMLNode(stringBody, "GetDigitalInputsResponse")
 			if err != nil {
-				log.Log.Error("onvif.main.GetDigitalInputs(): " + err.Error())
+				log.Error("onvif.main.GetDigitalInputs(): " + err.Error())
 				return digitalinputs, err
 			} else {
 				if err := decodedXML.DecodeElement(&digitalinputs, et); err != nil {
-					log.Log.Debug("onvif.main.GetDigitalInputs(): " + err.Error())
+					log.Debug("onvif.main.GetDigitalInputs(): " + err.Error())
 					return digitalinputs, err
 				}
 			}
@@ -1312,11 +1306,11 @@ func GetRelayOutputs(dev *onvif.Device) (device.GetRelayOutputsResponse, error)
 		stringBody := string(b)
 		decodedXML, et, err := getXMLNode(stringBody, "GetRelayOutputsResponse")
 		if err != nil {
-			log.Log.Error("onvif.main.GetRelayOutputs(): " + err.Error())
+			log.Error("onvif.main.GetRelayOutputs(): " + err.Error())
 			return relayoutputs, err
 		} else {
 			if err := decodedXML.DecodeElement(&relayoutputs, et); err != nil {
-				log.Log.Debug("onvif.main.GetRelayOutputs(): " + err.Error())
+				log.Debug("onvif.main.GetRelayOutputs(): " + err.Error())
 				return relayoutputs, err
 			}
 		}
@@ -1325,42 +1319,79 @@ func GetRelayOutputs(dev *onvif.Device) (device.GetRelayOutputsResponse, error)
 	return relayoutputs, err
 }
 
-func TriggerRelayOutput(dev *onvif.Device, output string) (err error) {
+func TriggerRelayOutput(dev *onvif.Device, output string) error {
-	err = nil
-
 	// Get all outputs
-	relayoutputs, err := GetRelayOutputs(dev)
+	relayOutputs, err := GetRelayOutputs(dev)
-
+	if err != nil {
-	// For the moment we expect a single output
+		log.WithError(err).WithFields(log.Fields{
-	// However in theory there might be multiple outputs. We might need to change
+			"component": "onvif",
-	// this in the future "kerberos-io/onvif" library.
+			"event":     "relay_outputs_query_failed",
-	if err == nil {
+		}).Error("Failed to query ONVIF relay outputs")
-		token := relayoutputs.RelayOutputs[0].Token
+		return err
-		if output == string(token+"-output") {
-			outputState := device.SetRelayOutputState{
-				RelayOutputToken: token,
-				LogicalState:     "active",
-			}
-
-			resp, errResp := dev.CallMethod(outputState)
-			var b []byte
-			if errResp != nil {
-				b, err = io.ReadAll(resp.Body)
-				resp.Body.Close() // Ensure the response body is closed
-			}
-			stringBody := string(b)
-			if err == nil && resp.StatusCode == 200 {
-				log.Log.Info("onvif.main.TriggerRelayOutput(): triggered relay output (" + string(token) + ")")
-			} else {
-				log.Log.Error("onvif.main.TriggerRelayOutput(): " + stringBody)
-			}
-		} else {
-			log.Log.Error("onvif.main.TriggerRelayOutput(): could not find relay output (" + output + ")")
-		}
-	} else {
-		log.Log.Error("onvif.main.TriggerRelayOutput(): something went wrong while getting the relay outputs " + err.Error())
 	}
-	return
+	if len(relayOutputs.RelayOutputs) == 0 {
+		err := errors.New("ONVIF device has no relay outputs")
+		log.WithError(err).WithFields(log.Fields{
+			"component": "onvif",
+			"event":     "relay_output_unavailable",
+		}).Error("ONVIF relay output unavailable")
+		return err
+	}
+
+	// For the moment we expect a single output. Supporting multiple outputs
+	// requires matching the requested output against every returned token.
+	token := relayOutputs.RelayOutputs[0].Token
+	if output != string(token+"-output") {
+		err := errors.New("requested ONVIF relay output was not found")
+		log.WithError(err).WithFields(log.Fields{
+			"component": "onvif",
+			"event":     "relay_output_unavailable",
+		}).Error("ONVIF relay output unavailable")
+		return err
+	}
+
+	resp, err := dev.CallMethod(device.SetRelayOutputState{
+		RelayOutputToken: token,
+		LogicalState:     "active",
+	})
+	if err != nil {
+		log.WithError(err).WithFields(log.Fields{
+			"component": "onvif",
+			"event":     "relay_output_request_failed",
+		}).Error("Failed to trigger ONVIF relay output")
+		return err
+	}
+	if resp == nil {
+		err := errors.New("ONVIF relay request returned no response")
+		log.WithError(err).WithFields(log.Fields{
+			"component": "onvif",
+			"event":     "relay_output_request_failed",
+		}).Error("Failed to trigger ONVIF relay output")
+		return err
+	}
+	defer resp.Body.Close()
+	if _, err := io.Copy(io.Discard, resp.Body); err != nil {
+		log.WithError(err).WithFields(log.Fields{
+			"component": "onvif",
+			"event":     "relay_output_response_read_failed",
+		}).Warn("Failed to read ONVIF relay response")
+		return err
+	}
+	if resp.StatusCode != 200 {
+		err := errors.New("ONVIF relay request was rejected")
+		log.WithError(err).WithFields(log.Fields{
+			"component":   "onvif",
+			"event":       "relay_output_rejected",
+			"status_code": resp.StatusCode,
+		}).Error("ONVIF relay request rejected")
+		return err
+	}
+
+	log.WithFields(log.Fields{
+		"component": "onvif",
+		"event":     "relay_output_triggered",
+	}).Info("ONVIF relay output triggered")
+	return nil
 }
 
 func getXMLNode(xmlBody string, nodeName string) (*xml.Decoder, *xml.StartElement, error) {

machinery/src/onvif/main_test.go

@@ -0,0 +1,42 @@
+package onvif
+
+import (
+	"bytes"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func TestHandleONVIFResponseDoesNotLogBody(t *testing.T) {
+	logger := log.StandardLogger()
+	originalOutput := logger.Out
+	originalFormatter := logger.Formatter
+	originalLevel := logger.Level
+	defer func() {
+		logger.SetOutput(originalOutput)
+		logger.SetFormatter(originalFormatter)
+		logger.SetLevel(originalLevel)
+	}()
+
+	var output bytes.Buffer
+	logger.SetOutput(&output)
+	logger.SetFormatter(&log.JSONFormatter{})
+	logger.SetLevel(log.DebugLevel)
+
+	response := &http.Response{
+		StatusCode: http.StatusOK,
+		Body:       io.NopCloser(strings.NewReader(`{"credential":"do-not-log"}`)),
+	}
+	if err := handleONVIFResponse("test_operation", response, nil); err != nil {
+		t.Fatalf("handleONVIFResponse() error = %v", err)
+	}
+	if strings.Contains(output.String(), "do-not-log") {
+		t.Fatalf("ONVIF response log exposed response body: %q", output.String())
+	}
+	if !strings.Contains(output.String(), `"response_bytes":27`) {
+		t.Fatalf("ONVIF response log omitted response byte count: %q", output.String())
+	}
+}

machinery/src/outputs/main.go

@@ -1,8 +1,8 @@
 package outputs
 
 import (
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 type Output interface {
@@ -20,36 +20,36 @@ func Execute(message *models.OutputMessage) (err error) {
 			slack := &SlackOutput{}
 			err := slack.Trigger(message)
 			if err == nil {
-				log.Log.Debug("outputs.main.Execute(slack): message was processed by output.")
+				log.Debug("outputs.main.Execute(slack): message was processed by output.")
 			} else {
-				log.Log.Error("outputs.main.Execute(slack): " + err.Error())
+				log.Error("outputs.main.Execute(slack): " + err.Error())
 			}
 			break
 		case "webhook":
 			webhook := &WebhookOutput{}
 			err := webhook.Trigger(message)
 			if err == nil {
-				log.Log.Debug("outputs.main.Execute(webhook): message was processed by output.")
+				log.Debug("outputs.main.Execute(webhook): message was processed by output.")
 			} else {
-				log.Log.Error("outputs.main.Execute(webhook): " + err.Error())
+				log.Error("outputs.main.Execute(webhook): " + err.Error())
 			}
 			break
 		case "onvif_relay":
 			onvif := &OnvifRelayOutput{}
 			err := onvif.Trigger(message)
 			if err == nil {
-				log.Log.Debug("outputs.main.Execute(onvif): message was processed by output.")
+				log.Debug("outputs.main.Execute(onvif): message was processed by output.")
 			} else {
-				log.Log.Error("outputs.main.Execute(onvif): " + err.Error())
+				log.Error("outputs.main.Execute(onvif): " + err.Error())
 			}
 			break
 		case "script":
 			script := &ScriptOutput{}
 			err := script.Trigger(message)
 			if err == nil {
-				log.Log.Debug("outputs.main.Execute(script): message was processed by output.")
+				log.Debug("outputs.main.Execute(script): message was processed by output.")
 			} else {
-				log.Log.Error("outputs.main.Execute(script): " + err.Error())
+				log.Error("outputs.main.Execute(script): " + err.Error())
 			}
 			break
 		}

machinery/src/packets/queue.go

@@ -2,6 +2,7 @@
 package packets
 
 import (
+	"context"
 	"io"
 	"sync"
 )
@@ -201,7 +202,26 @@ func (self *QueueCursor) Streams() (streams []Stream, err error) {
 
 // ReadPacket will not consume packets in Queue, it's just a cursor.
 func (self *QueueCursor) ReadPacket() (pkt Packet, err error) {
+	return self.readPacket(context.Background(), false)
+}
+
+// ReadPacketContext waits for the next packet or returns when ctx is cancelled.
+func (self *QueueCursor) ReadPacketContext(ctx context.Context) (pkt Packet, err error) {
+	if err := ctx.Err(); err != nil {
+		return Packet{}, err
+	}
+	return self.readPacket(ctx, true)
+}
+
+func (self *QueueCursor) readPacket(ctx context.Context, cancellable bool) (pkt Packet, err error) {
 	self.que.cond.L.Lock()
+	var stop func() bool
+	defer func() {
+		self.que.cond.L.Unlock()
+		if stop != nil {
+			stop()
+		}
+	}()
 	buf := self.que.buf
 	if !self.gotpos {
 		self.pos = self.init(buf, self.que.videoidx)
@@ -222,8 +242,21 @@ func (self *QueueCursor) ReadPacket() (pkt Packet, err error) {
 			err = io.EOF
 			break
 		}
+		if cancellable {
+			if err = ctx.Err(); err != nil {
+				break
+			}
+			if stop == nil {
+				stop = context.AfterFunc(ctx, func() {
+					// The write lock prevents cancellation from being signalled
+					// between the context check above and cond.Wait.
+					self.que.lock.Lock()
+					self.que.cond.Broadcast()
+					self.que.lock.Unlock()
+				})
+			}
+		}
 		self.que.cond.Wait()
 	}
-	self.que.cond.L.Unlock()
 	return
 }

machinery/src/packets/queue_test.go

@@ -0,0 +1,109 @@
+package packets
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+type observedCancelContext struct {
+	done        chan struct{}
+	cancelled   atomic.Bool
+	errCalls    atomic.Int32
+	waiting     chan struct{}
+	waitingOnce sync.Once
+}
+
+func newObservedCancelContext() *observedCancelContext {
+	return &observedCancelContext{
+		done:    make(chan struct{}),
+		waiting: make(chan struct{}),
+	}
+}
+
+func (c *observedCancelContext) Deadline() (time.Time, bool) { return time.Time{}, false }
+func (c *observedCancelContext) Done() <-chan struct{}       { return c.done }
+func (c *observedCancelContext) Value(any) any               { return nil }
+
+func (c *observedCancelContext) Err() error {
+	if c.errCalls.Add(1) >= 2 {
+		c.waitingOnce.Do(func() { close(c.waiting) })
+	}
+	if c.cancelled.Load() {
+		return context.Canceled
+	}
+	return nil
+}
+
+func (c *observedCancelContext) Cancel() {
+	c.cancelled.Store(true)
+	close(c.done)
+}
+
+func TestQueueCursorReadPacketContextCancelsWhileQueueIsIdle(t *testing.T) {
+	queue := NewQueue()
+	cursor := queue.Latest()
+	ctx := newObservedCancelContext()
+	done := make(chan error, 1)
+
+	go func() {
+		_, err := cursor.ReadPacketContext(ctx)
+		done <- err
+	}()
+
+	select {
+	case <-ctx.waiting:
+	case <-time.After(time.Second):
+		t.Fatal("ReadPacketContext() did not reach the queue wait")
+	}
+	ctx.Cancel()
+	select {
+	case err := <-done:
+		if !errors.Is(err, context.Canceled) {
+			t.Fatalf("ReadPacketContext() error = %v, want %v", err, context.Canceled)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("ReadPacketContext() did not return after cancellation")
+	}
+}
+
+func TestQueueCursorReadPacketDoesNotAllocate(t *testing.T) {
+	queue := NewQueue()
+	for i := 0; i < 200; i++ {
+		if err := queue.WritePacket(Packet{Data: []byte{1}}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	cursor := queue.Oldest()
+
+	allocations := testing.AllocsPerRun(100, func() {
+		if _, err := cursor.ReadPacket(); err != nil {
+			t.Fatal(err)
+		}
+	})
+	if allocations != 0 {
+		t.Fatalf("ReadPacket() allocations = %.2f, want 0", allocations)
+	}
+}
+
+func TestQueueCursorReadPacketContextDoesNotAllocateWhenPacketIsAvailable(t *testing.T) {
+	queue := NewQueue()
+	for i := 0; i < 200; i++ {
+		if err := queue.WritePacket(Packet{Data: []byte{1}}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	cursor := queue.Oldest()
+
+	allocations := testing.AllocsPerRun(100, func() {
+		if _, err := cursor.ReadPacketContext(context.Background()); err != nil {
+			t.Fatal(err)
+		}
+	})
+	if allocations != 0 {
+		t.Fatalf("ReadPacketContext() allocations = %.2f, want 0", allocations)
+	}
+}

machinery/src/routers/http/health.go

@@ -0,0 +1,126 @@
+package http
+
+import (
+	"math"
+	stdhttp "net/http"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/utils"
+)
+
+const (
+	healthApplicationStatus = "get_success"
+	healthEntityStatus      = "healthy"
+	hubConnectionFreshness  = 3 * time.Minute
+)
+
+// StreamResolution is the most recently observed encoded video size in pixels.
+type StreamResolution struct {
+	Width  int64 `json:"width"`
+	Height int64 `json:"height"`
+}
+
+// StreamHealth describes the most recently observed state of one camera stream.
+type StreamHealth struct {
+	Configured        bool             `json:"configured"`
+	Connected         bool             `json:"connected"`
+	PackagesProcessed uint64           `json:"packagesProcessed"`
+	FPS               float64          `json:"fps"`
+	Resolution        StreamResolution `json:"resolution"`
+	LastPacketAt      int64            `json:"lastPacketAt"`
+}
+
+// HubHealth describes the Agent's periodic HTTP heartbeat connection to Hub.
+type HubHealth struct {
+	Configured                bool  `json:"configured"`
+	Connected                 bool  `json:"connected"`
+	LastHeartbeatAttemptAt    int64 `json:"lastHeartbeatAttemptAt"`
+	LastSuccessfulHeartbeatAt int64 `json:"lastSuccessfulHeartbeatAt"`
+}
+
+// Health describes the Agent process health exposed to API clients.
+type Health struct {
+	Description     string       `json:"description"`
+	CameraConnected bool         `json:"cameraConnected"`
+	MainStream      StreamHealth `json:"mainStream"`
+	SubStream       StreamHealth `json:"subStream"`
+	Hub             HubHealth    `json:"hub"`
+}
+
+// HealthResponseData contains the typed payload of a health response.
+type HealthResponseData struct {
+	Health Health `json:"health"`
+}
+
+// HealthResponse is the standard successful response returned by GET /health.
+type HealthResponse struct {
+	SuccessResponse
+	Data HealthResponseData `json:"data"`
+}
+
+// HealthCheck godoc
+// @Summary Check Agent health
+// @Description Confirms that the Agent HTTP process can serve requests and reports current camera stream and Hub heartbeat diagnostics. Operational dependency failures do not change the liveness HTTP status.
+// @Tags health
+// @Produce json
+// @Success 200 {object} HealthResponse
+// @Router /health [get]
+func HealthCheck(c *gin.Context, communication *models.Communication) {
+	c.Header("Cache-Control", "no-store")
+	c.JSON(stdhttp.StatusOK, HealthResponse{
+		SuccessResponse: NewSuccessResponse(
+			stdhttp.StatusOK,
+			healthApplicationStatus,
+			healthEntityStatus,
+			"Healthy",
+			ResponseMetadata{
+				ApplicationName:    "agent",
+				ApplicationVersion: utils.VERSION,
+				Path:               c.Request.URL.Path,
+			},
+		),
+		Data: HealthResponseData{
+			Health: buildHealth(communication, time.Now()),
+		},
+	})
+}
+
+func buildHealth(communication *models.Communication, now time.Time) Health {
+	mainTelemetry := communication.StreamRuntimeTelemetry(models.MainStream)
+	subTelemetry := communication.StreamRuntimeTelemetry(models.SubStream)
+	hubTelemetry := communication.HubRuntimeTelemetry()
+
+	hubConnected := hubTelemetry.Configured &&
+		hubTelemetry.Connected &&
+		hubTelemetry.LastSuccessfulHeartbeatAt > 0 &&
+		now.Unix()-hubTelemetry.LastSuccessfulHeartbeatAt <= int64(hubConnectionFreshness/time.Second)
+
+	return Health{
+		Description:     "Agent HTTP service is healthy",
+		CameraConnected: communication.CameraConnected.Load(),
+		MainStream:      newStreamHealth(mainTelemetry, communication.MainStreamConnected.Load()),
+		SubStream:       newStreamHealth(subTelemetry, communication.SubStreamConnected.Load()),
+		Hub: HubHealth{
+			Configured:                hubTelemetry.Configured,
+			Connected:                 hubConnected,
+			LastHeartbeatAttemptAt:    hubTelemetry.LastHeartbeatAttemptAt,
+			LastSuccessfulHeartbeatAt: hubTelemetry.LastSuccessfulHeartbeatAt,
+		},
+	}
+}
+
+func newStreamHealth(telemetry models.StreamRuntimeTelemetry, connected bool) StreamHealth {
+	return StreamHealth{
+		Configured:        telemetry.Configured,
+		Connected:         connected,
+		PackagesProcessed: telemetry.PackagesProcessed,
+		FPS:               math.Round(telemetry.FPS*100) / 100,
+		Resolution: StreamResolution{
+			Width:  telemetry.Width,
+			Height: telemetry.Height,
+		},
+		LastPacketAt: telemetry.LastPacketAt,
+	}
+}

machinery/src/routers/http/health_test.go

@@ -0,0 +1,117 @@
+package http
+
+import (
+	"encoding/json"
+	stdhttp "net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	jwt "github.com/appleboy/gin-jwt/v2"
+	"github.com/gin-gonic/gin"
+	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/utils"
+)
+
+func TestHealthCheckReturnsStandardPublicResponse(t *testing.T) {
+	gin.SetMode(gin.TestMode)
+
+	middlewareConfig := JWTMiddleWare()
+	authMiddleware, err := jwt.New(&middlewareConfig)
+	if err != nil {
+		t.Fatalf("initialize JWT middleware: %v", err)
+	}
+
+	now := time.Now()
+	communication := &models.Communication{}
+	communication.SetStreamConfigured(models.MainStream, true)
+	communication.SetStreamConfigured(models.SubStream, true)
+	communication.RecordStreamPackage(models.MainStream, 29.969, 1920, 1080, now.Add(-2*time.Second))
+	communication.RecordStreamPackage(models.MainStream, 29.969, 1920, 1080, now.Add(-time.Second))
+	communication.RecordStreamPackage(models.SubStream, 15, 640, 360, now.Add(-time.Second))
+	communication.CameraConnected.Store(true)
+	communication.MainStreamConnected.Store(true)
+	communication.SubStreamConnected.Store(true)
+	communication.SetHubConfigured(true)
+	communication.RecordHubHeartbeatAttempt(now.Add(-2 * time.Second))
+	communication.RecordHubHeartbeatSuccess(now.Add(-time.Second))
+
+	router := gin.New()
+	AddRoutes(router, authMiddleware, "", nil, communication, nil)
+
+	response := httptest.NewRecorder()
+	request := httptest.NewRequest(stdhttp.MethodGet, "/health", nil)
+	router.ServeHTTP(response, request)
+
+	if response.Code != stdhttp.StatusOK {
+		t.Fatalf("status = %d, want %d; body=%s", response.Code, stdhttp.StatusOK, response.Body.String())
+	}
+	if got := response.Header().Get("Cache-Control"); got != "no-store" {
+		t.Fatalf("Cache-Control = %q, want %q", got, "no-store")
+	}
+
+	var body HealthResponse
+	if err := json.Unmarshal(response.Body.Bytes(), &body); err != nil {
+		t.Fatalf("decode health response: %v; body=%s", err, response.Body.String())
+	}
+
+	if body.HTTPStatusCode != stdhttp.StatusOK {
+		t.Errorf("httpStatusCode = %d, want %d", body.HTTPStatusCode, stdhttp.StatusOK)
+	}
+	if body.ApplicationStatusCode != healthApplicationStatus {
+		t.Errorf("applicationStatusCode = %q, want %q", body.ApplicationStatusCode, healthApplicationStatus)
+	}
+	if body.EntityStatusCode != healthEntityStatus {
+		t.Errorf("entityStatusCode = %q, want %q", body.EntityStatusCode, healthEntityStatus)
+	}
+	if body.Message != "Healthy" {
+		t.Errorf("message = %q, want %q", body.Message, "Healthy")
+	}
+	if body.Metadata.ApplicationName != "agent" {
+		t.Errorf("metadata.applicationName = %q, want %q", body.Metadata.ApplicationName, "agent")
+	}
+	if body.Metadata.ApplicationVersion != utils.VERSION {
+		t.Errorf("metadata.applicationVersion = %q, want %q", body.Metadata.ApplicationVersion, utils.VERSION)
+	}
+	if body.Metadata.Timestamp <= 0 {
+		t.Errorf("metadata.timestamp = %d, want a positive Unix timestamp", body.Metadata.Timestamp)
+	}
+	if body.Metadata.Path != "/health" {
+		t.Errorf("metadata.path = %q, want %q", body.Metadata.Path, "/health")
+	}
+	health := body.Data.Health
+	if health.Description != "Agent HTTP service is healthy" {
+		t.Errorf("data.health.description = %q, want %q", health.Description, "Agent HTTP service is healthy")
+	}
+	if !health.CameraConnected {
+		t.Error("data.health.cameraConnected = false, want true")
+	}
+	if health.MainStream.PackagesProcessed != 2 || health.MainStream.FPS != 29.97 {
+		t.Errorf("data.health.mainStream = %+v, want 2 packages at 29.97 FPS", health.MainStream)
+	}
+	if health.MainStream.Resolution != (StreamResolution{Width: 1920, Height: 1080}) {
+		t.Errorf("data.health.mainStream.resolution = %+v", health.MainStream.Resolution)
+	}
+	if health.SubStream.PackagesProcessed != 1 || health.SubStream.FPS != 15 {
+		t.Errorf("data.health.subStream = %+v, want 1 package at 15 FPS", health.SubStream)
+	}
+	if health.SubStream.Resolution != (StreamResolution{Width: 640, Height: 360}) {
+		t.Errorf("data.health.subStream.resolution = %+v", health.SubStream.Resolution)
+	}
+	if !health.Hub.Configured || !health.Hub.Connected {
+		t.Errorf("data.health.hub = %+v, want configured and connected", health.Hub)
+	}
+}
+
+func TestBuildHealthMarksStaleHubHeartbeatDisconnected(t *testing.T) {
+	now := time.Unix(1_788_710_500, 0)
+	communication := &models.Communication{}
+	communication.SetHubConfigured(true)
+	communication.RecordHubHeartbeatAttempt(now.Add(-4 * time.Minute))
+	communication.RecordHubHeartbeatSuccess(now.Add(-4 * time.Minute))
+
+	health := buildHealth(communication, now)
+	if health.Hub.Connected {
+		t.Fatalf("Hub health = %+v, want stale heartbeat to be disconnected", health.Hub)
+	}
+}

machinery/src/routers/http/methods.go

@@ -6,9 +6,9 @@ import (
 	"time"
 
 	"github.com/gin-gonic/gin"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/onvif"
+	log "github.com/sirupsen/logrus"
 )
 
 // Login godoc
@@ -598,27 +598,27 @@ func DoTriggerRelayOutput(c *gin.Context) {
 			err := onvif.TriggerRelayOutput(device, output)
 			if err == nil {
 				msg := "relay output triggered: " + output
-				log.Log.Info("routers.http.methods.DoTriggerRelayOutput(): " + msg)
+				log.Info("routers.http.methods.DoTriggerRelayOutput(): " + msg)
 				c.JSON(200, gin.H{
 					"data": msg,
 				})
 			} else {
 				msg := "something went wrong: " + err.Error()
-				log.Log.Error("routers.http.methods.DoTriggerRelayOutput(): " + msg)
+				log.Error("routers.http.methods.DoTriggerRelayOutput(): " + msg)
 				c.JSON(400, gin.H{
 					"data": msg,
 				})
 			}
 		} else {
 			msg := "something went wrong: " + err.Error()
-			log.Log.Error("routers.http.methods.DoTriggerRelayOutput(): " + msg)
+			log.Error("routers.http.methods.DoTriggerRelayOutput(): " + msg)
 			c.JSON(400, gin.H{
 				"data": msg,
 			})
 		}
 	} else {
 		msg := "something went wrong: " + err.Error()
-		log.Log.Error("routers.http.methods.DoTriggerRelayOutput(): " + msg)
+		log.Error("routers.http.methods.DoTriggerRelayOutput(): " + msg)
 		c.JSON(400, gin.H{
 			"data": msg,
 		})

machinery/src/routers/http/response.go

@@ -0,0 +1,41 @@
+package http
+
+import "time"
+
+// ResponseMetadata contains request and application context shared by API responses.
+type ResponseMetadata struct {
+	ApplicationName    string `json:"applicationName,omitempty"`
+	ApplicationVersion string `json:"applicationVersion,omitempty"`
+	Timestamp          int64  `json:"timestamp,omitempty"`
+	Path               string `json:"path,omitempty"`
+	TraceID            string `json:"traceId,omitempty"`
+}
+
+// SuccessResponse is the common envelope for successful Agent API responses.
+type SuccessResponse struct {
+	HTTPStatusCode        int              `json:"httpStatusCode"`
+	ApplicationStatusCode string           `json:"applicationStatusCode"`
+	EntityStatusCode      string           `json:"entityStatusCode"`
+	Message               string           `json:"message"`
+	Metadata              ResponseMetadata `json:"metadata"`
+}
+
+// NewSuccessResponse creates the Hub-compatible success envelope used by new
+// Agent API endpoints.
+func NewSuccessResponse(
+	httpStatusCode int,
+	applicationStatusCode string,
+	entityStatusCode string,
+	message string,
+	metadata ResponseMetadata,
+) SuccessResponse {
+	metadata.Timestamp = time.Now().Unix()
+
+	return SuccessResponse{
+		HTTPStatusCode:        httpStatusCode,
+		ApplicationStatusCode: applicationStatusCode,
+		EntityStatusCode:      entityStatusCode,
+		Message:               message,
+		Metadata:              metadata,
+	}
+}

machinery/src/routers/http/routes.go

@@ -14,6 +14,10 @@ import (
 
 func AddRoutes(r *gin.Engine, authMiddleware *jwt.GinJWTMiddleware, configDirectory string, configuration *models.Configuration, communication *models.Communication, captureDevice *capture.Capture) *gin.RouterGroup {
 
+	r.GET("/health", func(c *gin.Context) {
+		HealthCheck(c, communication)
+	})
+
 	r.GET("/ws", func(c *gin.Context) {
 		websocket.WebsocketHandler(c, configuration, communication, captureDevice)
 	})

machinery/src/routers/http/server.go

@@ -1,9 +1,11 @@
 package http
 
 import (
+	"fmt"
 	"io"
 	"os"
 	"strconv"
+	"time"
 
 	jwt "github.com/appleboy/gin-jwt/v2"
 	"github.com/gin-contrib/pprof"
@@ -11,7 +13,7 @@ import (
 	"github.com/gin-gonic/gin"
 
 	//Swagger documentantion
-	"log"
+	log "github.com/sirupsen/logrus"
 
 	_ "github.com/kerberos-io/agent/machinery/docs"
 	"github.com/kerberos-io/agent/machinery/src/capture"
@@ -45,7 +47,17 @@ func StartServer(configDirectory string, configuration *models.Configuration, co
 	gin.SetMode(gin.ReleaseMode)
 
 	// Initialize REST API
-	r := gin.Default()
+	r := gin.New()
+	r.Use(requestLogger(), gin.CustomRecovery(func(c *gin.Context, recovered interface{}) {
+		log.WithFields(log.Fields{
+			"component": "http",
+			"event":     "request_panic",
+			"method":    c.Request.Method,
+			"path":      c.Request.URL.Path,
+			"panic":     fmt.Sprint(recovered),
+		}).Error("HTTP request panicked")
+		c.AbortWithStatus(500)
+	}))
 
 	// Profiler
 	pprof.Register(r)
@@ -60,7 +72,10 @@ func StartServer(configDirectory string, configuration *models.Configuration, co
 	middleWare := JWTMiddleWare()
 	authMiddleware, err := jwt.New(&middleWare)
 	if err != nil {
-		log.Fatal("JWT Error:" + err.Error())
+		log.WithError(err).WithFields(log.Fields{
+			"component": "http",
+			"event":     "jwt_initialization_failed",
+		}).Fatal("Failed to initialize JWT middleware")
 	}
 
 	// Add all routes
@@ -73,7 +88,10 @@ func StartServer(configDirectory string, configuration *models.Configuration, co
 		// Move demo environment variables to environment variables
 		err := os.Rename(demoEnvironmentVariables, environmentVariables)
 		if err != nil {
-			log.Fatal(err)
+			log.WithError(err).WithFields(log.Fields{
+				"component": "http",
+				"event":     "demo_environment_move_failed",
+			}).Fatal("Failed to activate demo UI environment")
 		}
 	}
 
@@ -88,9 +106,54 @@ func StartServer(configDirectory string, configuration *models.Configuration, co
 	})
 
 	// Run the api on port
+	log.WithFields(log.Fields{
+		"component": "http",
+		"event":     "server_starting",
+		"port":      configuration.Port,
+	}).Info("HTTP server starting")
 	err = r.Run(":" + configuration.Port)
 	if err != nil {
-		log.Fatal(err)
+		log.WithError(err).WithFields(log.Fields{
+			"component": "http",
+			"event":     "server_failed",
+			"port":      configuration.Port,
+		}).Fatal("HTTP server stopped")
+	}
+}
+
+func requestLogger() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		startedAt := time.Now()
+		c.Next()
+
+		fields := log.Fields{
+			"component":      "http",
+			"duration_ms":    time.Since(startedAt).Milliseconds(),
+			"event":          "request_completed",
+			"method":         c.Request.Method,
+			"path":           c.Request.URL.Path,
+			"response_bytes": c.Writer.Size(),
+			"status":         c.Writer.Status(),
+		}
+		if log.IsLevelEnabled(log.DebugLevel) {
+			fields["client_ip"] = c.ClientIP()
+			fields["user_agent"] = c.Request.UserAgent()
+		}
+		entry := log.WithFields(fields)
+		if len(c.Errors) > 0 {
+			entry = entry.WithField("request_errors", c.Errors.String())
+		}
+
+		switch status := c.Writer.Status(); {
+		case status >= 500:
+			entry.Error("HTTP request completed")
+		case status >= 400:
+			entry.Warn("HTTP request completed")
+		case c.Request.URL.Path == "/health":
+			entry.Debug("HTTP request completed")
+		default:
+			entry.Info("HTTP request completed")
+		}
 	}
 }
 

machinery/src/routers/http/server_test.go

@@ -0,0 +1,89 @@
+package http
+
+import (
+	"bytes"
+	"encoding/json"
+	stdhttp "net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/gin-gonic/gin"
+	log "github.com/sirupsen/logrus"
+)
+
+func TestRequestLoggerEmitsStructuredFields(t *testing.T) {
+	originalOutput := log.StandardLogger().Out
+	originalFormatter := log.StandardLogger().Formatter
+	originalLevel := log.GetLevel()
+	defer func() {
+		log.SetOutput(originalOutput)
+		log.SetFormatter(originalFormatter)
+		log.SetLevel(originalLevel)
+	}()
+
+	var output bytes.Buffer
+	log.SetOutput(&output)
+	log.SetFormatter(&log.JSONFormatter{})
+	log.SetLevel(log.InfoLevel)
+	gin.SetMode(gin.TestMode)
+
+	router := gin.New()
+	router.Use(requestLogger())
+	router.GET("/status", func(c *gin.Context) {
+		c.Status(stdhttp.StatusNoContent)
+	})
+
+	response := httptest.NewRecorder()
+	request := httptest.NewRequest(stdhttp.MethodGet, "/status?token=do-not-log", nil)
+	router.ServeHTTP(response, request)
+
+	var entry map[string]interface{}
+	if err := json.Unmarshal(output.Bytes(), &entry); err != nil {
+		t.Fatalf("decode request log: %v; output=%q", err, output.String())
+	}
+	for key, want := range map[string]interface{}{
+		"component": "http",
+		"event":     "request_completed",
+		"method":    stdhttp.MethodGet,
+		"path":      "/status",
+		"status":    float64(stdhttp.StatusNoContent),
+	} {
+		if got := entry[key]; got != want {
+			t.Fatalf("%s = %v, want %v", key, got, want)
+		}
+	}
+	if bytes.Contains(output.Bytes(), []byte("do-not-log")) {
+		t.Fatalf("request log exposed query parameters: %q", output.String())
+	}
+}
+
+func TestRequestLoggerKeepsSuccessfulHealthChecksBelowInfo(t *testing.T) {
+	originalOutput := log.StandardLogger().Out
+	originalFormatter := log.StandardLogger().Formatter
+	originalLevel := log.GetLevel()
+	defer func() {
+		log.SetOutput(originalOutput)
+		log.SetFormatter(originalFormatter)
+		log.SetLevel(originalLevel)
+	}()
+
+	var output bytes.Buffer
+	log.SetOutput(&output)
+	log.SetFormatter(&log.JSONFormatter{})
+	log.SetLevel(log.InfoLevel)
+	gin.SetMode(gin.TestMode)
+
+	router := gin.New()
+	router.Use(requestLogger())
+	router.GET("/health", func(c *gin.Context) {
+		c.Status(stdhttp.StatusNoContent)
+	})
+
+	response := httptest.NewRecorder()
+	request := httptest.NewRequest(stdhttp.MethodGet, "/health", nil)
+	router.ServeHTTP(response, request)
+
+	if output.Len() != 0 {
+		t.Fatalf("successful health check emitted an info log: %q", output.String())
+	}
+}

machinery/src/routers/mqtt/main.go

@@ -1,7 +1,9 @@
 package mqtt
 
 import (
+	"context"
 	"crypto/rsa"
+	"crypto/tls"
 	"crypto/x509"
 	"encoding/base64"
 	"encoding/json"
@@ -9,21 +11,23 @@ import (
 	"fmt"
 	"io/ioutil"
 	"math/rand"
+	"net"
+	"net/url"
+	"os"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
-	"context"
-
 	mqtt "github.com/eclipse/paho.mqtt.golang"
 	"github.com/kerberos-io/agent/machinery/src/capture"
 	configService "github.com/kerberos-io/agent/machinery/src/config"
 	"github.com/kerberos-io/agent/machinery/src/encryption"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/onvif"
 	"github.com/kerberos-io/agent/machinery/src/webrtc"
+	log "github.com/sirupsen/logrus"
+	"golang.org/x/net/proxy"
 )
 
 // We'll cache the MQTT settings to know if we need to reinitialize the MQTT client connection.
@@ -34,6 +38,177 @@ var PREV_MQTTPassword string
 var PREV_HubKey string
 var PREV_AgentKey string
 
+type pahoErrorLogger struct{}
+
+func (pahoErrorLogger) Println(values ...interface{}) {
+	log.WithFields(log.Fields{
+		"component": "routers/mqtt",
+		"event":     "paho_error",
+	}).Error(strings.TrimSpace(fmt.Sprintln(values...)))
+}
+
+func (pahoErrorLogger) Printf(format string, values ...interface{}) {
+	log.WithFields(log.Fields{
+		"component": "routers/mqtt",
+		"event":     "paho_error",
+	}).Errorf(strings.TrimSpace(format), values...)
+}
+
+func init() {
+	mqtt.ERROR = pahoErrorLogger{}
+}
+
+func enableMQTTConnectionDiagnostics(options *mqtt.ClientOptions, brokerURL string) {
+	if !strings.Contains(brokerURL, "://") {
+		options.SetCustomOpenConnectionFn(openMQTTConnection)
+		return
+	}
+
+	parsedURL, err := url.Parse(brokerURL)
+	if err != nil {
+		return
+	}
+
+	switch strings.ToLower(parsedURL.Scheme) {
+	case "", "mqtt", "tcp", "ssl", "tls", "mqtts", "mqtt+ssl", "tcps":
+		options.SetCustomOpenConnectionFn(openMQTTConnection)
+	}
+}
+
+func openMQTTConnection(uri *url.URL, options mqtt.ClientOptions) (net.Conn, error) {
+	host := uri.Hostname()
+	fields := log.Fields{
+		"component": "routers/mqtt",
+		"host":      host,
+		"port":      uri.Port(),
+		"scheme":    uri.Scheme,
+	}
+	logMQTTDNSResolution(host, options.ConnectTimeout, fields)
+
+	connectionStartedAt := time.Now()
+	dialer := options.Dialer
+	if dialer == nil {
+		dialer = &net.Dialer{Timeout: options.ConnectTimeout}
+	}
+
+	proxyConfigured := os.Getenv("all_proxy") != ""
+	proxyMode := "direct"
+	if proxyConfigured {
+		proxyMode = "socks"
+	}
+	fields["proxy_mode"] = proxyMode
+	log.WithFields(fields).Info("Opening MQTT TCP connection")
+
+	var (
+		connection net.Conn
+		err        error
+	)
+	if proxyConfigured {
+		connection, err = proxy.FromEnvironment().Dial("tcp", uri.Host)
+	} else {
+		connection, err = dialer.Dial("tcp", uri.Host)
+	}
+	fields["duration_ms"] = time.Since(connectionStartedAt).Milliseconds()
+	if err != nil {
+		logMQTTNetworkError("MQTT TCP connection failed", err, fields)
+		return nil, err
+	}
+
+	fields["local_address"] = connection.LocalAddr().String()
+	fields["remote_address"] = connection.RemoteAddr().String()
+	log.WithFields(fields).Info("MQTT TCP connection established")
+
+	if !isSecureMQTTScheme(uri.Scheme) {
+		return connection, nil
+	}
+
+	tlsConfig := options.TLSConfig
+	if tlsConfig == nil {
+		tlsConfig = &tls.Config{}
+	} else {
+		tlsConfig = tlsConfig.Clone()
+	}
+	if tlsConfig.ServerName == "" {
+		tlsConfig.ServerName = host
+	}
+
+	tlsConnection := tls.Client(connection, tlsConfig)
+	tlsStartedAt := time.Now()
+	if options.ConnectTimeout > 0 {
+		_ = tlsConnection.SetDeadline(connectionStartedAt.Add(options.ConnectTimeout))
+	}
+	if err = tlsConnection.Handshake(); err != nil {
+		_ = connection.Close()
+		fields["duration_ms"] = time.Since(tlsStartedAt).Milliseconds()
+		fields["server_name"] = tlsConfig.ServerName
+		logMQTTNetworkError("MQTT TLS handshake failed", err, fields)
+		return nil, err
+	}
+	_ = tlsConnection.SetDeadline(time.Time{})
+
+	state := tlsConnection.ConnectionState()
+	fields["cipher_suite"] = tls.CipherSuiteName(state.CipherSuite)
+	fields["duration_ms"] = time.Since(tlsStartedAt).Milliseconds()
+	fields["server_name"] = tlsConfig.ServerName
+	fields["tls_version"] = tls.VersionName(state.Version)
+	log.WithFields(fields).Info("MQTT TLS handshake established")
+	return tlsConnection, nil
+}
+
+func logMQTTDNSResolution(host string, timeout time.Duration, fields log.Fields) {
+	if host == "" || net.ParseIP(host) != nil {
+		return
+	}
+
+	lookupTimeout := timeout
+	if lookupTimeout <= 0 || lookupTimeout > 5*time.Second {
+		lookupTimeout = 5 * time.Second
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), lookupTimeout)
+	defer cancel()
+
+	startedAt := time.Now()
+	addresses, err := net.DefaultResolver.LookupHost(ctx, host)
+	dnsFields := cloneLogFields(fields)
+	dnsFields["duration_ms"] = time.Since(startedAt).Milliseconds()
+	if err != nil {
+		logMQTTNetworkError("MQTT broker DNS resolution failed", err, dnsFields)
+		return
+	}
+
+	dnsFields["resolved_addresses"] = addresses
+	log.WithFields(dnsFields).Info("MQTT broker DNS resolved")
+}
+
+func logMQTTNetworkError(message string, err error, fields log.Fields) {
+	errorFields := cloneLogFields(fields)
+	if networkError, ok := err.(net.Error); ok {
+		errorFields["network_timeout"] = networkError.Timeout()
+	}
+	if operationError, ok := err.(*net.OpError); ok {
+		errorFields["network"] = operationError.Net
+		errorFields["operation"] = operationError.Op
+	}
+	log.WithError(err).WithFields(errorFields).Error(message)
+}
+
+func cloneLogFields(fields log.Fields) log.Fields {
+	cloned := make(log.Fields, len(fields))
+	for key, value := range fields {
+		cloned[key] = value
+	}
+	return cloned
+}
+
+func isSecureMQTTScheme(scheme string) bool {
+	switch strings.ToLower(scheme) {
+	case "ssl", "tls", "mqtts", "mqtt+ssl", "tcps":
+		return true
+	default:
+		return false
+	}
+}
+
 func HasMQTTClientModified(configuration *models.Configuration) bool {
 	MTTURI := configuration.Config.MQTTURI
 	MTTUsername := configuration.Config.MQTTUsername
@@ -41,7 +216,7 @@ func HasMQTTClientModified(configuration *models.Configuration) bool {
 	HubKey := configuration.Config.HubKey
 	AgentKey := configuration.Config.Key
 	if PREV_MQTTURI != MTTURI || PREV_MQTTUsername != MTTUsername || PREV_MQTTPassword != MQTTPassword || PREV_HubKey != HubKey || PREV_AgentKey != AgentKey {
-		log.Log.Info("HasMQTTClientModified: MQTT settings have been modified, restarting MQTT client.")
+		log.Info("HasMQTTClientModified: MQTT settings have been modified, restarting MQTT client.")
 		return true
 	}
 	return false
@@ -59,6 +234,7 @@ func HasMQTTClientModified(configuration *models.Configuration) bool {
 // - kerberos/{hubkey}/device/{devicekey}/motion: a motion signal
 
 func ConfigureMQTT(configDirectory string, configuration *models.Configuration, communication *models.Communication) mqtt.Client {
+	installRemoteAccessHook()
 
 	config := configuration.Config
 
@@ -70,8 +246,25 @@ func ConfigureMQTT(configDirectory string, configuration *models.Configuration,
 	PREV_AgentKey = configuration.Config.Key
 
 	if config.Offline == "true" {
-		log.Log.Info("routers.mqtt.main.ConfigureMQTT(): not starting as running in Offline mode.")
+		log.Info("routers.mqtt.main.ConfigureMQTT(): not starting as running in Offline mode.")
 	} else {
+		hubKey := ""
+		if config.Cloud == "s3" && config.S3 != nil && config.S3.Publickey != "" {
+			hubKey = config.S3.Publickey
+		} else if config.Cloud == "kstorage" && config.KStorage != nil && config.KStorage.CloudKey != "" {
+			hubKey = config.KStorage.CloudKey
+		}
+		if config.HubKey != "" {
+			hubKey = config.HubKey
+		}
+		if hubKey == "" {
+			log.Warn("routers.mqtt.main.ConfigureMQTT(): not starting without a Hub key")
+			return nil
+		}
+		if config.Key == "" {
+			log.Warn("routers.mqtt.main.ConfigureMQTT(): not starting without an Agent key")
+			return nil
+		}
 
 		opts := mqtt.NewClientOptions()
 
@@ -79,7 +272,6 @@ func ConfigureMQTT(configDirectory string, configuration *models.Configuration,
 		// and share and receive messages to/from.
 		mqttURL := config.MQTTURI
 		opts.AddBroker(mqttURL)
-		log.Log.Debug("routers.mqtt.main.ConfigureMQTT(): Set broker uri " + mqttURL)
 
 		// Our MQTT broker can have username/password credentials
 		// to protect it from the outside.
@@ -88,13 +280,19 @@ func ConfigureMQTT(configDirectory string, configuration *models.Configuration,
 		if mqtt_username != "" || mqtt_password != "" {
 			opts.SetUsername(mqtt_username)
 			opts.SetPassword(mqtt_password)
-			log.Log.Debug("routers.mqtt.main.ConfigureMQTT(): Set username " + mqtt_username)
-			log.Log.Debug("routers.mqtt.main.ConfigureMQTT(): Set password " + mqtt_password)
 		}
+		log.WithFields(log.Fields{
+			"broker_configured":   mqttURL != "",
+			"component":           "routers/mqtt",
+			"event":               "credentials_configured",
+			"password_configured": mqtt_password != "",
+			"username_configured": mqtt_username != "",
+		}).Debug("MQTT connection configuration loaded")
 
 		// Some extra options to make sure the connection behaves
 		// properly. More information here: github.com/eclipse/paho.mqtt.golang.
 		//opts.SetCleanSession(true)
+		enableMQTTConnectionDiagnostics(opts, mqttURL)
 		opts.SetCleanSession(false)
 		opts.SetResumeSubs(true)
 		opts.SetStore(mqtt.NewMemoryStore())
@@ -108,65 +306,81 @@ func ConfigureMQTT(configDirectory string, configuration *models.Configuration,
 		opts.SetOrderMatters(false)
 		opts.SetConnectTimeout(30 * time.Second)
 		opts.SetConnectionLostHandler(func(client mqtt.Client, err error) {
+			entry := log.WithFields(log.Fields{
+				"component": "routers/mqtt",
+				"event":     "connection_lost",
+			})
 			if err != nil {
-				log.Log.Error("routers.mqtt.main.ConfigureMQTT(): MQTT connection lost: " + err.Error())
+				entry = entry.WithError(err)
-			} else {
-				log.Log.Error("routers.mqtt.main.ConfigureMQTT(): MQTT connection lost")
 			}
+			entry.Error("MQTT connection lost")
 		})
 		opts.SetReconnectingHandler(func(client mqtt.Client, options *mqtt.ClientOptions) {
-			log.Log.Warning("routers.mqtt.main.ConfigureMQTT(): reconnecting to MQTT broker")
+			log.WithFields(log.Fields{
+				"component": "routers/mqtt",
+				"event":     "reconnecting",
+			}).Warn("Reconnecting to MQTT broker")
 		})
 		opts.SetOnConnectHandler(func(c mqtt.Client) {
-			log.Log.Info("routers.mqtt.main.ConfigureMQTT(): MQTT session is online")
+			log.WithFields(log.Fields{
+				"component": "routers/mqtt",
+				"event":     "session_online",
+			}).Info("MQTT session is online")
 		})
 
-		hubKey := ""
+		rand.Seed(time.Now().UnixNano())
-		// This is the old way ;)
+		random := rand.Intn(100)
-		if config.Cloud == "s3" && config.S3 != nil && config.S3.Publickey != "" {
+		mqttClientID := config.Key + strconv.Itoa(random) // this random int is to avoid conflicts.
-			hubKey = config.S3.Publickey
+
-		} else if config.Cloud == "kstorage" && config.KStorage != nil && config.KStorage.CloudKey != "" {
+		// This is a worked-around.
-			hubKey = config.KStorage.CloudKey
+		// current S3 (Kerberos Hub SAAS) is using a secured MQTT, where the client id,
-		}
+		// should match the kerberos hub key.
-		// This is the new way ;)
+		if config.Cloud == "s3" {
-		if config.HubKey != "" {
+			mqttClientID = config.Key
-			hubKey = config.HubKey
 		}
 
-		if hubKey != "" {
+		opts.SetClientID(mqttClientID)
+		log.WithFields(log.Fields{
+			"component":            "routers/mqtt",
+			"event":                "client_configured",
+			"randomized_client_id": config.Cloud != "s3",
+		}).Debug("MQTT client configured")
+		rand.Seed(time.Now().UnixNano())
 
-			rand.Seed(time.Now().UnixNano())
+		opts.OnConnect = func(c mqtt.Client) {
-			random := rand.Intn(100)
+			// We managed to connect to the MQTT broker, hurray!
-			mqttClientID := config.Key + strconv.Itoa(random) // this random int is to avoid conflicts.
+			log.WithFields(log.Fields{
+				"component": "routers/mqtt",
+				"event":     "connected",
+			}).Info("MQTT broker connected")
+			log.WithFields(log.Fields{
+				"broker_configured": mqttURL != "",
+				"component":         "routers/mqtt",
+				"event":             "connection_details",
+			}).Debug("MQTT connection details")
 
-			// This is a worked-around.
+			// Create a susbcription for listen and reply
-			// current S3 (Kerberos Hub SAAS) is using a secured MQTT, where the client id,
+			MQTTListenerHandler(c, hubKey, configDirectory, configuration, communication)
-			// should match the kerberos hub key.
-			if config.Cloud == "s3" {
-				mqttClientID = config.Key
-			}
-
-			opts.SetClientID(mqttClientID)
-			log.Log.Info("routers.mqtt.main.ConfigureMQTT(): Set ClientID " + mqttClientID)
-			rand.Seed(time.Now().UnixNano())
-
-			opts.OnConnect = func(c mqtt.Client) {
-				// We managed to connect to the MQTT broker, hurray!
-				log.Log.Info("routers.mqtt.main.ConfigureMQTT(): " + mqttClientID + " connected to " + mqttURL)
-
-				// Create a susbcription for listen and reply
-				MQTTListenerHandler(c, hubKey, configDirectory, configuration, communication)
-			}
 		}
 		mqc := mqtt.NewClient(opts)
 		if token := mqc.Connect(); token.WaitTimeout(30 * time.Second) {
 			if token.Error() != nil {
-				log.Log.Error("routers.mqtt.main.ConfigureMQTT(): unable to establish mqtt broker connection, error was: " + token.Error().Error())
+				log.WithError(token.Error()).WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "initial_connection_failed",
+				}).Error("Failed to establish initial MQTT connection")
 			} else {
-				log.Log.Info("routers.mqtt.main.ConfigureMQTT(): initial MQTT connection established")
+				log.WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "initial_connection_established",
+				}).Info("Initial MQTT connection established")
 			}
 		} else {
-			log.Log.Error("routers.mqtt.main.ConfigureMQTT(): timed out while establishing mqtt broker connection")
+			log.WithFields(log.Fields{
+				"component":  "routers/mqtt",
+				"event":      "initial_connection_timeout",
+				"timeout_ms": (30 * time.Second).Milliseconds(),
+			}).Warn("Initial MQTT connection is still retrying")
 		}
 		return mqc
 	}
@@ -216,7 +430,7 @@ func markHDSessionSeen(sessionID string) bool {
 
 func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory string, configuration *models.Configuration, communication *models.Communication) {
 	if hubKey == "" {
-		log.Log.Info("routers.mqtt.main.MQTTListenerHandler(): no hub key provided, not subscribing to kerberos/hub/{hubkey}")
+		log.Info("routers.mqtt.main.MQTTListenerHandler(): no hub key provided, not subscribing to kerberos/hub/{hubkey}")
 	} else {
 		agentListener := fmt.Sprintf("kerberos/agent/%s", hubKey)
 		token := mqttClient.Subscribe(agentListener, 1, func(c mqtt.Client, msg mqtt.Message) {
@@ -236,6 +450,7 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 			// We will receive all messages from our hub, so we'll need to filter to the relevant device.
 			if message.Mid != "" && message.Timestamp != 0 && message.DeviceId == configuration.Config.Key {
 				var payload models.Payload
+				remoteAuthenticated := false
 
 				// Messages might be hidden, if so we'll need to decrypt them using the Kerberos Hub private key.
 				if message.Hidden && configuration.Config.HubEncryption == "true" {
@@ -249,13 +464,15 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 							}
 							visibleValue, err := encryption.AesDecrypt(data, privateKey)
 							if err != nil {
-								log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message: " + err.Error())
+								log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message: " + err.Error())
 								return
 							}
-							json.Unmarshal(visibleValue, &payload)
+							if err := json.Unmarshal(visibleValue, &payload); err == nil {
-							message.Payload = payload
+								message.Payload = payload
+								remoteAuthenticated = true
+							}
 						} else {
-							log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message, no private key provided.")
+							log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message, no private key provided.")
 						}
 					}
 				}
@@ -271,14 +488,14 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 						pemBytes, _ := ioutil.ReadAll(r)
 						block, _ := pem.Decode(pemBytes)
 						if block == nil {
-							log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error decoding PEM block containing private key")
+							log.Error("routers.mqtt.main.MQTTListenerHandler(): error decoding PEM block containing private key")
 							return
 						} else {
 							// Parse private key
 							b := block.Bytes
 							key, err := x509.ParsePKCS8PrivateKey(b)
 							if err != nil {
-								log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error parsing private key: " + err.Error())
+								log.Error("routers.mqtt.main.MQTTListenerHandler(): error parsing private key: " + err.Error())
 								return
 							} else {
 								// Conver key to *rsa.PrivateKey
@@ -298,16 +515,18 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 										}
 										decryptedValue, err := encryption.AesDecrypt(data, string(decryptedKey))
 										if err != nil {
-											log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message: " + err.Error())
+											log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message: " + err.Error())
 											return
 										}
-										json.Unmarshal(decryptedValue, &payload)
+										if err := json.Unmarshal(decryptedValue, &payload); err == nil {
+											remoteAuthenticated = true
+										}
 									} else {
-										log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message, assymetric keys do not match.")
+										log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message, assymetric keys do not match.")
 										return
 									}
 								} else if err != nil {
-									log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message: " + err.Error())
+									log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message: " + err.Error())
 									return
 								}
 							}
@@ -318,7 +537,11 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 				}
 
 				// We'll find out which message we received, and act accordingly.
-				log.Log.Info("routers.mqtt.main.MQTTListenerHandler(): received message with action: " + payload.Action)
+				log.WithFields(log.Fields{
+					"action":    payload.Action,
+					"component": "routers/mqtt",
+					"event":     "message_received",
+				}).Debug("MQTT message received")
 
 				// NOTE: We intentionally do NOT discard request-hd-stream /
 				// receive-hd-candidates messages based on a wall-clock age. The
@@ -355,6 +578,14 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 					go HandleReceiveHDCandidates(mqttClient, hubKey, payload, configuration, communication)
 				case "trigger-relay":
 					go HandleTriggerRelay(mqttClient, hubKey, payload, configuration, communication)
+				case "remote-session-open":
+					go HandleRemoteSessionOpen(mqttClient, hubKey, payload, remoteAuthenticated, configuration)
+				case "remote-session-input":
+					go HandleRemoteSessionInput(mqttClient, hubKey, payload, remoteAuthenticated, configuration)
+				case "remote-session-resize":
+					go HandleRemoteSessionResize(payload, remoteAuthenticated)
+				case "remote-session-close":
+					go HandleRemoteSessionClose(payload, remoteAuthenticated)
 				}
 
 			}
@@ -362,12 +593,15 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 
 		if token.WaitTimeout(10 * time.Second) {
 			if token.Error() != nil {
-				log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): failed to subscribe to " + agentListener + ": " + token.Error().Error())
+				log.WithError(token.Error()).WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "subscription_failed",
+				}).Error("Failed to subscribe to Agent MQTT topic")
 			} else {
-				log.Log.Info("routers.mqtt.main.MQTTListenerHandler(): subscribed to " + agentListener)
+				log.Info("routers.mqtt.main.MQTTListenerHandler(): subscribed to " + agentListener)
 			}
 		} else {
-			log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): timed out while subscribing to " + agentListener)
+			log.Error("routers.mqtt.main.MQTTListenerHandler(): timed out while subscribing to " + agentListener)
 		}
 	}
 }
@@ -396,9 +630,9 @@ func HandleRecording(mqttClient mqtt.Client, hubKey string, payload models.Paylo
 			if communication.IsRecordingManual.IsSet() {
 				communication.RecordingManualHeartbeat.Store(now)
 				communication.RecordingManualHeartbeatSeen.Set()
-				log.Log.Debug("routers.mqtt.main.HandleRecording(): manual recording heartbeat received.")
+				log.Debug("routers.mqtt.main.HandleRecording(): manual recording heartbeat received.")
 			} else {
-				log.Log.Debug("routers.mqtt.main.HandleRecording(): ignoring heartbeat, no active manual recording.")
+				log.Debug("routers.mqtt.main.HandleRecording(): ignoring heartbeat, no active manual recording.")
 			}
 		} else {
 			// Explicit start from the live view (record button). Start a manual
@@ -410,18 +644,16 @@ func HandleRecording(mqttClient mqtt.Client, hubKey string, payload models.Paylo
 			if communication.IsRecordingManual.SetToIf(false, true) {
 				communication.RecordingManualStart.Store(now)
 				communication.RecordingManualHeartbeatSeen.UnSet()
-				log.Log.Info("routers.mqtt.main.HandleRecording(): manual recording started.")
+				log.Info("routers.mqtt.main.HandleRecording(): manual recording started.")
-				select {
+				if !communication.TrySendMotion(models.MotionDataPartial{Timestamp: timestamp, NumberOfChanges: 100000000}) {
-				case communication.HandleMotion <- models.MotionDataPartial{Timestamp: timestamp, NumberOfChanges: 100000000}:
+					log.Warn("routers.mqtt.main.HandleRecording(): motion channel full, manual recording start not queued.")
-				default:
-					log.Log.Warning("routers.mqtt.main.HandleRecording(): motion channel full, manual recording start not queued.")
 				}
 			}
 		}
 	} else {
 		// Stop the manual recording; the motion recorder closes the clip once the
 		// post-recording window elapses. Clear the heartbeat/start markers too.
-		log.Log.Info("routers.mqtt.main.HandleRecording(): manual recording stopped.")
+		log.Info("routers.mqtt.main.HandleRecording(): manual recording stopped.")
 		communication.IsRecordingManual.UnSet()
 		communication.RecordingManualHeartbeat.Store(0)
 		communication.RecordingManualStart.Store(0)
@@ -443,7 +675,7 @@ func HandleAudio(mqttClient mqtt.Client, hubKey string, payload models.Payload,
 			Data:      audioPayload.Data,
 		}
 		if enqueueLatestAudio(communication.HandleAudio, audioDataPartial) {
-			log.Log.Debug("routers.mqtt.main.HandleAudio(): dropped stale audio because the backchannel queue was full")
+			log.Debug("routers.mqtt.main.HandleAudio(): dropped stale audio because the backchannel queue was full")
 		}
 	}
 }
@@ -479,7 +711,7 @@ func HandleGetPTZPosition(mqttClient mqtt.Client, hubKey string, payload models.
 		// Get Position from device
 		pos, err := onvif.GetPositionFromDevice(*configuration)
 		if err != nil {
-			log.Log.Error("routers.mqtt.main.HandlePTZPosition(): error getting position from device: " + err.Error())
+			log.Error("routers.mqtt.main.HandlePTZPosition(): error getting position from device: " + err.Error())
 		} else {
 			// Needs to wrapped!
 			posString := fmt.Sprintf("%f,%f,%f", pos.PanTilt.X, pos.PanTilt.Y, pos.Zoom.X)
@@ -497,7 +729,10 @@ func HandleGetPTZPosition(mqttClient mqtt.Client, hubKey string, payload models.
 			if err == nil {
 				mqttClient.Publish("kerberos/hub/"+hubKey, 2, false, payload)
 			} else {
-				log.Log.Info("routers.mqtt.main.HandlePTZPosition(): something went wrong while sending position to hub: " + string(payload))
+				log.WithError(err).WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "ptz_position_packaging_failed",
+				}).Error("Failed to package PTZ position message")
 			}
 		}
 	}
@@ -512,11 +747,14 @@ func HandleUpdatePTZPosition(mqttClient mqtt.Client, hubKey string, payload mode
 	json.Unmarshal(jsonData, &onvifAction)
 
 	if onvifAction.Action != "" {
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
-			communication.HandleONVIF <- onvifAction
+			if communication.TrySendONVIF(onvifAction) {
-			log.Log.Info("routers.mqtt.main.MQTTListenerHandleONVIF(): Received an action - " + onvifAction.Action)
+				log.Info("routers.mqtt.main.MQTTListenerHandleONVIF(): Received an action - " + onvifAction.Action)
+			} else {
+				log.Warn("routers.mqtt.main.MQTTListenerHandleONVIF(): action channel unavailable or full")
+			}
 		} else {
-			log.Log.Info("routers.mqtt.main.MQTTListenerHandleONVIF(): received action, but camera is not connected.")
+			log.Info("routers.mqtt.main.MQTTListenerHandleONVIF(): received action, but camera is not connected.")
 		}
 	}
 }
@@ -561,14 +799,17 @@ func HandleRequestConfig(mqttClient mqtt.Client, hubKey string, payload models.P
 			if err == nil {
 				mqttClient.Publish("kerberos/hub/"+hubKey, 2, false, payload)
 			} else {
-				log.Log.Info("routers.mqtt.main.HandleRequestConfig(): something went wrong while sending config to hub: " + string(payload))
+				log.WithError(err).WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "configuration_packaging_failed",
+				}).Error("Failed to package configuration message")
 			}
 
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleRequestConfig(): no config available")
+			log.Info("routers.mqtt.main.HandleRequestConfig(): no config available")
 		}
 
-		log.Log.Info("routers.mqtt.main.HandleRequestConfig(): Received a request for the config")
+		log.Info("routers.mqtt.main.HandleRequestConfig(): Received a request for the config")
 	}
 }
 
@@ -660,7 +901,10 @@ func HandleVerifyStream(mqttClient mqtt.Client, hubKey string, payload models.Pa
 	if err == nil {
 		mqttClient.Publish("kerberos/hub/"+hubKey, 2, false, packagedPayload)
 	} else {
-		log.Log.Info("routers.mqtt.main.HandleVerifyStream(): something went wrong while sending result to hub: " + string(packagedPayload))
+		log.WithError(err).WithFields(log.Fields{
+			"component": "routers/mqtt",
+			"event":     "stream_verification_packaging_failed",
+		}).Error("Failed to package stream verification result")
 	}
 }
 
@@ -681,7 +925,7 @@ func HandleUpdateConfig(mqttClient mqtt.Client, hubKey string, payload models.Pa
 
 		err := configService.SaveConfig(configDirectory, config, configuration, communication)
 		if err == nil {
-			log.Log.Info("routers.mqtt.main.HandleUpdateConfig(): Config updated")
+			log.Info("routers.mqtt.main.HandleUpdateConfig(): Config updated")
 			message := models.Message{
 				Payload: models.Payload{
 					Action:   "acknowledge-update-config",
@@ -692,10 +936,13 @@ func HandleUpdateConfig(mqttClient mqtt.Client, hubKey string, payload models.Pa
 			if err == nil {
 				mqttClient.Publish("kerberos/hub/"+hubKey, 2, false, payload)
 			} else {
-				log.Log.Info("routers.mqtt.main.HandleUpdateConfig(): something went wrong while sending acknowledge config to hub: " + string(payload))
+				log.WithError(err).WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "configuration_acknowledgement_packaging_failed",
+				}).Error("Failed to package configuration acknowledgement")
 			}
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleUpdateConfig(): Config update failed")
+			log.Info("routers.mqtt.main.HandleUpdateConfig(): Config update failed")
 		}
 	}
 }
@@ -708,7 +955,7 @@ func HandleRequestSDStream(mqttClient mqtt.Client, hubKey string, payload models
 	json.Unmarshal(jsonData, &requestSDStreamPayload)
 
 	if requestSDStreamPayload.Timestamp != 0 {
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			// A viewer that opted into the HTTP transport is signalled on a separate
 			// channel so the producer ships its frames to hub-api over HTTP instead of
 			// publishing them over MQTT. Any other (or absent) transport keeps the
@@ -724,9 +971,9 @@ func HandleRequestSDStream(mqttClient mqtt.Client, hubKey string, payload models
 				default:
 				}
 			}
-			log.Log.Info("routers.mqtt.main.HandleRequestSDStream(): received request to livestream.")
+			log.Info("routers.mqtt.main.HandleRequestSDStream(): received request to livestream.")
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleRequestSDStream(): received request to livestream, but camera is not connected.")
+			log.Info("routers.mqtt.main.HandleRequestSDStream(): received request to livestream, but camera is not connected.")
 		}
 	}
 }
@@ -743,7 +990,7 @@ func HandleRequestHLSStream(mqttClient mqtt.Client, hubKey string, payload model
 	json.Unmarshal(jsonData, &requestHLSStreamPayload)
 
 	if requestHLSStreamPayload.Timestamp != 0 {
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			// Forward the requested quality ("auto"|"high"|"low"; empty => auto) so
 			// the producer can switch the live session between the main and sub
 			// stream on demand. The send doubles as the viewer keepalive.
@@ -751,9 +998,9 @@ func HandleRequestHLSStream(mqttClient mqtt.Client, hubKey string, payload model
 			case communication.HandleLiveHLS <- requestHLSStreamPayload.Quality:
 			default:
 			}
-			log.Log.Info("routers.mqtt.main.HandleRequestHLSStream(): received request to livestream over HLS.")
+			log.Info("routers.mqtt.main.HandleRequestHLSStream(): received request to livestream over HLS.")
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleRequestHLSStream(): received request to livestream over HLS, but camera is not connected.")
+			log.Info("routers.mqtt.main.HandleRequestHLSStream(): received request to livestream over HLS, but camera is not connected.")
 		}
 	}
 }
@@ -766,29 +1013,30 @@ func HandleRequestHDStream(mqttClient mqtt.Client, hubKey string, payload models
 	json.Unmarshal(jsonData, &requestHDStreamPayload)
 
 	if requestHDStreamPayload.Timestamp != 0 {
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			// Dedupe by session_id: the viewer republishes its offer while
 			// waiting for an answer (and the broker may redeliver), and we
 			// don't want to spawn multiple peer connections for the same
 			// browser session.
 			if markHDSessionSeen(requestHDStreamPayload.SessionID) {
-				log.Log.Info("routers.mqtt.main.HandleRequestHDStream(): duplicate request for session " +
+				log.WithFields(log.Fields{
-					requestHDStreamPayload.SessionID + ", ignoring")
+					"component":  "routers/mqtt",
+					"event":      "duplicate_hd_stream_request",
+					"session_id": requestHDStreamPayload.SessionID,
+				}).Debug("Ignoring duplicate HD stream request")
 				return
 			}
 			// Set the Hub key, so we can send back the answer.
 			requestHDStreamPayload.HubKey = hubKey
-			if communication.HandleLiveHDHandshake == nil {
+			if !communication.TrySendLiveHDHandshake(models.LiveHDHandshake{
-				log.Log.Error("routers.mqtt.main.HandleRequestHDStream(): handshake channel is nil, dropping request")
+				Payload: requestHDStreamPayload,
+			}) {
+				log.Error("routers.mqtt.main.HandleRequestHDStream(): handshake channel unavailable or full, dropping request")
 				return
 			}
-
+			log.Info("routers.mqtt.main.HandleRequestHDStream(): received request to setup webrtc.")
-			communication.HandleLiveHDHandshake <- models.LiveHDHandshake{
-				Payload: requestHDStreamPayload,
-			}
-			log.Log.Info("routers.mqtt.main.HandleRequestHDStream(): received request to setup webrtc.")
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleRequestHDStream(): received request to setup webrtc, but camera is not connected.")
+			log.Info("routers.mqtt.main.HandleRequestHDStream(): received request to setup webrtc, but camera is not connected.")
 		}
 	}
 }
@@ -801,12 +1049,12 @@ func HandleReceiveHDCandidates(mqttClient mqtt.Client, hubKey string, payload mo
 	json.Unmarshal(jsonData, &receiveHDCandidatesPayload)
 
 	if receiveHDCandidatesPayload.Timestamp != 0 {
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			// Register candidate channel
 			key := configuration.Config.Key + "/" + receiveHDCandidatesPayload.SessionID
 			go webrtc.RegisterCandidates(key, receiveHDCandidatesPayload)
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleReceiveHDCandidates(): received candidate, but camera is not connected.")
+			log.Info("routers.mqtt.main.HandleReceiveHDCandidates(): received candidate, but camera is not connected.")
 		}
 	}
 }
@@ -818,14 +1066,17 @@ func HandleNavigatePTZ(mqttClient mqtt.Client, hubKey string, payload models.Pay
 	json.Unmarshal(jsonData, &navigatePTZPayload)
 
 	if navigatePTZPayload.Timestamp != 0 {
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			action := navigatePTZPayload.Action
 			var onvifAction models.OnvifAction
 			json.Unmarshal([]byte(action), &onvifAction)
-			communication.HandleONVIF <- onvifAction
+			if communication.TrySendONVIF(onvifAction) {
-			log.Log.Info("routers.mqtt.main.HandleNavigatePTZ(): Received an action - " + onvifAction.Action)
+				log.Info("routers.mqtt.main.HandleNavigatePTZ(): Received an action - " + onvifAction.Action)
+			} else {
+				log.Warn("routers.mqtt.main.HandleNavigatePTZ(): action channel unavailable or full")
+			}
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleNavigatePTZ(): received action, but camera is not connected.")
+			log.Info("routers.mqtt.main.HandleNavigatePTZ(): received action, but camera is not connected.")
 		}
 	}
 }
@@ -837,7 +1088,7 @@ func HandleTriggerRelay(mqttClient mqtt.Client, hubKey string, payload models.Pa
 	json.Unmarshal(jsonData, &triggerRelayPayload)
 
 	if triggerRelayPayload.Timestamp != 0 {
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			// Get token (name of relay)
 			token := triggerRelayPayload.Token
 			// Connect to Onvif device
@@ -847,16 +1098,25 @@ func HandleTriggerRelay(mqttClient mqtt.Client, hubKey string, payload models.Pa
 				// Trigger relay output
 				err := onvif.TriggerRelayOutput(device, token)
 				if err != nil {
-					log.Log.Error("routers.mqtt.main.HandleTriggerRelay(): error triggering relay: " + err.Error())
+					log.WithError(err).WithFields(log.Fields{
+						"component": "routers/mqtt",
+						"event":     "relay_output_failed",
+					}).Error("Failed to trigger relay output")
 				} else {
-					log.Log.Info("routers.mqtt.main.HandleTriggerRelay(): trigger (" + token + ") relay output.")
+					log.WithFields(log.Fields{
+						"component": "routers/mqtt",
+						"event":     "relay_output_triggered",
+					}).Info("Relay output triggered")
 				}
 			} else {
-				log.Log.Error("routers.mqtt.main.HandleTriggerRelay(): error connecting to device: " + err.Error())
+				log.WithError(err).WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "relay_device_connection_failed",
+				}).Error("Failed to connect to relay device")
 			}
 
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleTriggerRelay(): received trigger, but camera is not connected.")
+			log.Info("routers.mqtt.main.HandleTriggerRelay(): received trigger, but camera is not connected.")
 		}
 	}
 }
@@ -868,6 +1128,6 @@ func DisconnectMQTT(mqttClient mqtt.Client, config *models.Config) {
 		mqttClient.Unsubscribe("kerberos/agent/" + PREV_HubKey)
 		mqttClient.Disconnect(1000)
 		mqttClient = nil
-		log.Log.Info("routers.mqtt.main.DisconnectMQTT(): MQTT client disconnected.")
+		log.Info("routers.mqtt.main.DisconnectMQTT(): MQTT client disconnected.")
 	}
 }