feat: enhance recording metadata structure and handling for uploads

fix ffmpeg 8.1 2: Update codec context and frame deallocation methods

.circleci/config.yml

@@ -1,32 +0,0 @@
-# Golang CircleCI 2.0 configuration file
-#
-# Check https://circleci.com/docs/2.0/language-go/ for more details
-version: 2
-jobs:
-  machinery:
-    docker:
-      - image: kerberos/base:0a50dc9
-    working_directory: /go/src/github.com/{{ORG_NAME}}/{{REPO_NAME}}
-    steps:
-      - checkout
-      - run: apt-get update && apt-get install -y --no-install-recommends libavcodec-dev libavformat-dev libswscale-dev libjpeg62-turbo-dev
-      - run: cd machinery && go mod download
-      - run: cd machinery && go test -v ./...
-      - run: cd machinery && go vet
-
-  ui:
-    docker:
-      - image: cimg/python:3.9.13-node
-    steps:
-      - checkout
-      - run: node --version
-      - run: cd ui && yarn
-      - run: cd ui && yarn test --passWithNoTests
-      - run: cd ui && yarn build
-
-workflows:
-  version: 2
-  build:
-    jobs:
-      - machinery
-      - ui

.github/workflows/go.yml

@@ -10,24 +10,27 @@ jobs:
   build:
     name: Build
     runs-on: ubuntu-latest
-    container:
-      image: kerberos/base:eb6b088
-
-    strategy:
-      matrix:
-        #No longer supported Go versions.
-        #go-version: ['1.17', '1.18', '1.19', '1.20', '1.21']
-        go-version: ["1.24"]
 
     steps:
-      - name: Set up Go ${{ matrix.go-version }}
-        uses: actions/setup-go@v2
-        with:
-          go-version: ${{ matrix.go-version }}
       - name: Check out code into the Go module directory
         uses: actions/checkout@v4
-      - name: Set up git ownershi
-        run: git config --system --add safe.directory /__w/agent/agent
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.25.x"
+          check-latest: true
+          cache: true
+          cache-dependency-path: machinery/go.sum
+      - name: Install native dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential \
+            libavcodec-dev \
+            libavutil-dev \
+            libswresample-dev \
+            libswscale-dev \
+            pkg-config
       - name: Get dependencies
         run: cd machinery && go mod download
       - name: Build

.github/workflows/nightly-build.yml

@@ -15,7 +15,7 @@ jobs:
         architecture: [amd64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
@@ -24,13 +24,14 @@ jobs:
         with:
           ref: master
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        id: buildx
+        uses: docker/setup-buildx-action@v3
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
       - name: Run Buildx
-        run: docker buildx build --platform linux/${{matrix.architecture}} -t kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7) --push .
+        run: docker buildx build --build-arg VERSION=$(echo $GITHUB_SHA | cut -c1-7) --platform linux/${{matrix.architecture}} -t kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7) --push .
       - name: Create new and append to manifest
         run: docker buildx imagetools create -t kerberos/agent-nightly:$(echo $GITHUB_SHA | cut -c1-7) kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7)
   nightly-build-other:
@@ -40,7 +41,7 @@ jobs:
         architecture: [arm64, arm/v7, arm/v6]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
@@ -49,12 +50,13 @@ jobs:
         with:
           ref: master
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        id: buildx
+        uses: docker/setup-buildx-action@v3
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
       - name: Run Buildx
-        run: docker buildx build --platform linux/${{matrix.architecture}} -t kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7) --push .
+        run: docker buildx build --build-arg VERSION=$(echo $GITHUB_SHA | cut -c1-7) --platform linux/${{matrix.architecture}} -t kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7) --push .
       - name: Create new and append to manifest
         run: docker buildx imagetools create --append -t kerberos/agent-nightly:$(echo $GITHUB_SHA | cut -c1-7) kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7)

.github/workflows/pr-build.yml

@@ -20,19 +20,15 @@ jobs:
             runner: ubuntu-24.04-arm
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-      - uses: benjlevesque/short-sha@v2.1
-        id: short-sha
-        with:
-          length: 7
       - name: Run Build
         run: |
-          docker build -t ${{ matrix.architecture }} .
+          docker build --build-arg VERSION=$(echo "${{ github.event.pull_request.head.sha }}" | cut -c1-7) -t ${{ matrix.architecture }} .
           CID=$(docker create ${{matrix.architecture}})
           docker cp ${CID}:/home/agent ./output-${{matrix.architecture}}
           docker rm ${CID}
@@ -43,4 +39,3 @@ jobs:
         with:
           name: agent-${{matrix.architecture}}.tar
           path: agent-${{matrix.architecture}}.tar
-

.github/workflows/react.yml

@@ -11,21 +11,22 @@ on:
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     strategy:
       matrix:
-        node-version: [14.x, 16.x]
+        node-version: [22.x]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v2
+      uses: actions/setup-node@v4
       with:
         node-version: ${{ matrix.node-version }}
+        cache: yarn
+        cache-dependency-path: ui/yarn.lock
     - name: Yarn install
-      run: cd ui && yarn
+      run: cd ui && yarn --frozen-lockfile
     - name: Yarn test
       run: cd ui && yarn test --passWithNoTests
     - name: Yarn build

.github/workflows/release-bump.yml

@@ -58,16 +58,12 @@ jobs:
         architecture: [amd64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Checkout
-        uses: actions/checkout@v3
-      - uses: benjlevesque/short-sha@v2.1
-        id: short-sha
-        with:
-          length: 7
+        uses: actions/checkout@v4
       - name: Run Build
         run: |
           docker build --provenance=false --build-arg VERSION=${{ needs.bump-release.outputs.tag }} -t ${{matrix.architecture}} .
@@ -96,16 +92,12 @@ jobs:
         architecture: [arm64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Checkout
-        uses: actions/checkout@v3
-      - uses: benjlevesque/short-sha@v2.1
-        id: short-sha
-        with:
-          length: 7
+        uses: actions/checkout@v4
       - name: Run Build
         run: |
           docker build --provenance=false --build-arg VERSION=${{ needs.bump-release.outputs.tag }} -t ${{matrix.architecture}} .
@@ -129,7 +121,7 @@ jobs:
     needs: [bump-release, build-amd64, build-arm64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/release-create.yml

@@ -22,16 +22,12 @@ jobs:
         architecture: [amd64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Checkout
-        uses: actions/checkout@v3
-      - uses: benjlevesque/short-sha@v2.1
-        id: short-sha
-        with:
-          length: 7
+        uses: actions/checkout@v4
       - name: Run Build
         run: |
           docker build --provenance=false --build-arg VERSION=${{github.event.inputs.tag || github.ref_name}} -t ${{matrix.architecture}} .
@@ -59,16 +55,12 @@ jobs:
         architecture: [arm64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Checkout
-        uses: actions/checkout@v3
-      - uses: benjlevesque/short-sha@v2.1
-        id: short-sha
-        with:
-          length: 7
+        uses: actions/checkout@v4
       - name: Run Build
         run: |
           docker build --provenance=false --build-arg VERSION=${{github.event.inputs.tag || github.ref_name}} -t ${{matrix.architecture}} .
@@ -92,7 +84,7 @@ jobs:
     needs: [build-amd64, build-arm64]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}

.vscode/launch.json

@@ -17,7 +17,7 @@
                 "8080"
             ],
             "envFile": "${workspaceFolder}/machinery/.env.local",
-            "buildFlags": "--tags dynamic",
+            "buildFlags": "--tags dynamic,moq",
             "env": {
                 "GOWORK": "off"
             },

Dockerfile

@@ -26,29 +26,27 @@ RUN apt-get update && apt-get install -y --fix-missing --no-install-recommends \
 	rm -rf /var/lib/apt/lists/*
 
 ##############################################################################
-# Copy all the relevant source code in the Docker image, so we can build this.
+# Copy dependency metadata first so module downloads can be cached separately.
 
 RUN mkdir -p /go/src/github.com/kerberos-io/agent
-COPY machinery /go/src/github.com/kerberos-io/agent/machinery
-RUN rm -rf /go/src/github.com/kerberos-io/agent/machinery/.env
+WORKDIR /go/src/github.com/kerberos-io/agent/machinery
+COPY machinery/go.mod machinery/go.sum ./
+RUN go mod download
 
-##################################################################
-# Get the latest commit hash, so we know which version we're running
-COPY .git /go/src/github.com/kerberos-io/agent/.git
-RUN cd /go/src/github.com/kerberos-io/agent/.git && git log --format="%H" -n 1 | head -c7 > /go/src/github.com/kerberos-io/agent/machinery/version
-RUN cat /go/src/github.com/kerberos-io/agent/machinery/version
+##############################################################################
+# Copy the rest of the source after dependencies are primed.
+
+COPY machinery ./
+RUN rm -rf .env
 
 ##################
 # Build Machinery
 
-RUN cd /go/src/github.com/kerberos-io/agent/machinery && \
-	go mod download && \
-	if [ -z "${VERSION}" ] || [ "${VERSION}" = "0.0.0" ]; then \
-		VERSION=$(cd /go/src/github.com/kerberos-io/agent && git describe --tags --always 2>/dev/null || echo "0.0.0"); \
-	fi && \
+RUN RESOLVED_VERSION="${VERSION:-0.0.0}" && \
+	printf '%s' "${RESOLVED_VERSION}" > version && \
 	BUILD_TAGS=timetzdata,netgo,osusergo && \
 	case "${TARGETARCH:-$(go env GOARCH)}" in amd64|arm64) BUILD_TAGS="moq,${BUILD_TAGS}" ;; esac && \
-	go build -tags "${BUILD_TAGS}" --ldflags "-s -w -X github.com/kerberos-io/agent/machinery/src/utils.VERSION=${VERSION}" main.go && \
+	go build -o main -tags "${BUILD_TAGS}" --ldflags "-s -w -X github.com/kerberos-io/agent/machinery/src/utils.VERSION=${RESOLVED_VERSION}" . && \
 	mkdir -p /agent && \
 	mv main /agent && \
 	mv version /agent && \
@@ -80,9 +78,11 @@ RUN apk update && apk upgrade --available && sync
 # Build Web (React app)
 
 RUN mkdir -p /go/src/github.com/kerberos-io/agent/machinery/www
-COPY ui /go/src/github.com/kerberos-io/agent/ui
-RUN cd /go/src/github.com/kerberos-io/agent/ui && rm -rf yarn.lock && yarn config set network-timeout 300000 && \
-	yarn && yarn build
+WORKDIR /go/src/github.com/kerberos-io/agent/ui
+COPY ui/package.json ui/yarn.lock ./
+RUN yarn config set network-timeout 300000 && yarn --frozen-lockfile
+COPY ui ./
+RUN yarn build
 
 ####################################
 # Let's create a /dist folder containing just the files necessary for runtime.
@@ -103,7 +103,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 	ca-certificates curl ffmpeg libatomic1 libcap2-bin libstdc++6 && \
 	rm -rf /var/lib/apt/lists/* && \
 	groupadd --system kerberosio && \
-	useradd --system --gid kerberosio --groups video --create-home agent
+	useradd --system --gid kerberosio --groups video --create-home agent && \
+	chmod 0755 /home/agent
 
 #################################
 # Copy files from previous images
@@ -145,10 +146,10 @@ EXPOSE 80
 ######################################
 # Check if agent is still running
 
-HEALTHCHECK CMD curl --fail http://localhost:80 || exit 1   
+HEALTHCHECK CMD curl --fail "http://localhost:${AGENT_PORT:-80}/health" || exit 1
 
 ###################################################
 # Leeeeettttt'ssss goooooo!!!
 # Run the shizzle from the right working directory.
 WORKDIR /home/agent
-CMD ["./main", "-action", "run", "-port", "80"]
+CMD ["./main", "-action", "run"]

README-RTSPS-TLS.md

@@ -75,7 +75,7 @@ sequenceDiagram
   participant Trust as Go trust pool
   participant Camera as Camera RTSPS :9554
 
-  Agent->>Trust: Load trusted CAs from SSL_CERT_FILE and CA directories
+  Agent->>Trust: Load system roots and append AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE
   Agent->>Camera: Open TCP connection
   Agent->>Camera: Send TLS ClientHello
   Camera-->>Agent: Send TLS ServerHello and camera certificate
@@ -377,7 +377,7 @@ For a process running directly in the same environment:
 AGENT_CAPTURE_IPCAMERA_RTSP="rtsps://<user>:<password>@10.0.30.11:9554/?inst=1"
 AGENT_CAPTURE_IPCAMERA_SUB_RTSP="rtsps://<user>:<password>@10.0.30.11:9554/?inst=2"
 AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE=false
-SSL_CERT_FILE=/home/agent/data/config/uug-camera-trust-bundle.pem
+AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE=/home/agent/data/config/uug-camera-trust-bundle.pem
 ```
 
 For a container, mount the public bundle read-only at the exact path visible
@@ -387,20 +387,22 @@ includes Debian's `ca-certificates` package:
 ```bash
 docker run \
   -v /secure/config/uug-camera-trust-bundle.pem:/home/agent/data/config/uug-camera-trust-bundle.pem:ro \
-  -e SSL_CERT_FILE=/home/agent/data/config/uug-camera-trust-bundle.pem \
+  -e AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE=/home/agent/data/config/uug-camera-trust-bundle.pem \
   -e AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE=false \
   -e 'AGENT_CAPTURE_IPCAMERA_RTSP=rtsps://<user>:<password>@10.0.30.11:9554/?inst=1' \
   -e 'AGENT_CAPTURE_IPCAMERA_SUB_RTSP=rtsps://<user>:<password>@10.0.30.11:9554/?inst=2' \
   kerberos/agent:latest
 ```
 
-Restart the Agent after changing trust files. Go can cache the process system
-certificate pool after its first use, so editing a file does not guarantee that
-an already-running process reloads it.
+`AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE` starts with the operating system's roots
+and appends the camera bundle only to the gortsplib TLS configuration. Other
+clients, including MoQ, Hub, and Vault, retain the normal public CA chain.
+Restart the Agent after changing trust files.
 
-The default production mode retains the image's normal public roots in addition
-to the private camera CA. For a deliberately private-CA-only deployment, mount
-an empty directory and set `SSL_CERT_DIR` to its path:
+Do not set `SSL_CERT_FILE` or `SSL_CERT_DIR` in production solely for camera
+trust. They are process-wide and can prevent other clients from validating
+public services. For a deliberate process-wide isolation test, mount an empty
+directory and set `SSL_CERT_DIR` to its path:
 
 ```bash
 -v /secure/config/empty-ca-dir:/home/agent/data/config/empty-ca-dir:ro \
@@ -503,7 +505,8 @@ openssl s_client \
 On Unix, Go uses `SSL_CERT_FILE` instead of its default aggregate CA file, but it
 still scans default certificate directories such as `/etc/ssl/certs`. Setting
 `SSL_CERT_FILE` alone therefore does not remove CA certificates installed with
-`update-ca-certificates`.
+`update-ca-certificates`. `AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE` is appended
+after this system pool is loaded; it does not replace the system roots.
 
 Blank values do not select empty trust sources. Both `SSL_CERT_FILE=` and
 `SSL_CERT_DIR=` are treated as unset, so Go falls back to its default aggregate
@@ -515,6 +518,7 @@ directory path that contains no certificates:
 mkdir -p /tmp/empty-ca-dir
 SSL_CERT_FILE=/dev/null \
 SSL_CERT_DIR=/tmp/empty-ca-dir \
+AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE= \
 AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE=false \
 GOWORK=off \
 go run -tags moq . -action run -port 8080
@@ -523,9 +527,10 @@ go run -tags moq . -action run -port 8080
 That fresh process must fail with `x509: certificate signed by unknown
 authority`.
 
-Use exactly one trust-distribution approach when possible:
+Use exactly one camera trust-distribution approach when possible:
 
-1. Mount a private trust bundle and set `SSL_CERT_FILE`; or
+1. Mount a private trust bundle and set
+  `AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE`; or
 2. Install the CA certificates into the operating-system trust store.
 
 Using both is valid, but makes isolation tests less obvious.
@@ -550,8 +555,9 @@ openssl req \
   -keyout /tmp/unrelated-test-root.key \
   -out /tmp/unrelated-test-root.crt
 
-SSL_CERT_FILE=/tmp/unrelated-test-root.crt \
+SSL_CERT_FILE=/dev/null \
 SSL_CERT_DIR=/tmp/empty-ca-dir \
+AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE=/tmp/unrelated-test-root.crt \
 AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE=false \
 GOWORK=off \
 go run -tags moq . -action run -port 8080

README.md

@@ -3,7 +3,6 @@
 <a target="_blank" href="https://kerberos.io"><img src="https://img.shields.io/badge/kerberos-website-gray.svg?longCache=true&colorB=brightgreen" alt="Kerberos Agent"></a>
 <a target="_blank" href="https://doc.kerberos.io"><img src="https://img.shields.io/badge/kerberos-documentation-gray.svg?longCache=true&colorB=brightgreen" alt="Kerberos Agent"></a>
 
-<a target="_blank" href="https://circleci.com/gh/kerberos-io/agent"><img src="https://circleci.com/gh/kerberos-io/agent.svg?style=svg"/></a>
 <img src="https://github.com/kerberos-io/agent/workflows/Go/badge.svg"/>
 <img src="https://github.com/kerberos-io/agent/workflows/React/badge.svg"/>
 <img src="https://github.com/kerberos-io/agent/workflows/CodeQL/badge.svg"/>
@@ -189,6 +188,103 @@ Next to attaching the configuration file, it is also possible to override the co
     -e AGENT_CAPTURE_CONTINUOUS=true \
     -d --restart=always kerberos/agent:latest
 
+Set `AGENT_PORT` to change the Agent's internal HTTP port. The environment
+variable takes precedence over `-port`. When using Docker, publish the same
+container port:
+
+```bash
+docker run -p 8082:8082 \
+  -e AGENT_PORT=8082 \
+  --name mycamera \
+  -d --restart=always kerberos/agent:latest
+```
+
+### Health checks and API responses
+
+`GET /health` is an unauthenticated liveness and diagnostics endpoint for
+container orchestrators and external monitors. `healthy` means that the Agent
+HTTP process can serve requests. Camera streams and Hub connectivity are
+reported independently in the response, so an external camera or Hub outage
+does not make Docker restart an otherwise functioning Agent process. Successful
+probes return `200 OK` and are logged at debug level to avoid periodic health
+checks filling production logs.
+
+```bash
+curl http://localhost:8082/health
+```
+
+```json
+{
+  "httpStatusCode": 200,
+  "applicationStatusCode": "get_success",
+  "entityStatusCode": "healthy",
+  "message": "Healthy",
+  "metadata": {
+    "applicationName": "agent",
+    "applicationVersion": "0.0.0",
+    "timestamp": 1788710400,
+    "path": "/health"
+  },
+  "data": {
+    "health": {
+      "description": "Agent HTTP service is healthy",
+      "cameraConnected": true,
+      "mainStream": {
+        "configured": true,
+        "connected": true,
+        "packagesProcessed": 183421,
+        "fps": 29.97,
+        "resolution": {
+          "width": 1920,
+          "height": 1080
+        },
+        "lastPacketAt": 1788710399
+      },
+      "subStream": {
+        "configured": true,
+        "connected": true,
+        "packagesProcessed": 91710,
+        "fps": 15,
+        "resolution": {
+          "width": 640,
+          "height": 360
+        },
+        "lastPacketAt": 1788710399
+      },
+      "hub": {
+        "configured": true,
+        "connected": true,
+        "lastHeartbeatAttemptAt": 1788710398,
+        "lastSuccessfulHeartbeatAt": 1788710398
+      }
+    }
+  }
+}
+```
+
+Stream `packagesProcessed` counts complete encoded video access units processed
+since the Agent process started; it does not count individual fragmented RTP
+packets. `fps` is the latest PTS-derived frame-rate estimate. Resolution is the
+most recently observed encoded width and height. `lastPacketAt`, Hub heartbeat
+timestamps, and response metadata timestamps are Unix seconds; `0` means that
+no value has been observed yet. A stream can be configured but temporarily
+disconnected.
+
+Hub `configured` means heartbeat delivery is enabled and has the required URI
+and key. Hub `connected` means the most recent heartbeat succeeded and the last
+success is no more than three minutes old. A failed heartbeat marks it
+disconnected immediately while retaining the last-success timestamp for
+diagnosis.
+
+New JSON endpoints should use this Hub-compatible response envelope rather than
+adding unrelated top-level response shapes. HTTP status describes the transport
+result, `applicationStatusCode` describes the operation, `entityStatusCode` and
+`message` describe the domain outcome, `metadata` carries request/application
+context, and endpoint-specific content belongs in a typed `data` object. Client
+responses must contain safe messages only; detailed internal errors belong in
+structured logs. Existing endpoints retain their legacy response shapes until
+they are migrated deliberately.
+
 ### Secure camera streams (RTSPS)
 
 The Agent accepts `rtsps://` camera URLs. Do not use `srtsp://`; RTSPS is RTSP over TLS. For a Bosch FLEXIDOME micro 3100i, enable **Secure RTSP** under **Network > Network Services** and use port `9554`:
@@ -200,7 +296,7 @@ AGENT_CAPTURE_IPCAMERA_SUB_RTSP='rtsps://username:password@camera.example:9554/?
 
 Certificate verification is enabled by default. The URL hostname or IP address must match the camera certificate SAN. On this Bosch firmware, RTSPS presents the certificate assigned to **HTTPS**; there is no separate SRTSP certificate usage. Leave **CBS client** assigned to the Bosch device certificate.
 
-For a private CA, mount a PEM trust bundle containing every CA certificate needed to build the camera certificate chain and set `SSL_CERT_FILE` to that file. This Bosch firmware presents only its leaf certificate, so include both the issuing intermediate and root certificates in the bundle. As a temporary fallback, `AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE=true` disables certificate verification for camera streams only.
+For a private CA, mount a PEM trust bundle containing every CA certificate needed to build the camera certificate chain and set `AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE` to its path inside the Agent. The bundle is appended to the system roots for camera RTSPS connections only. This Bosch firmware presents only its leaf certificate, so include both the issuing intermediate and root certificates in the bundle. As a temporary fallback, `AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE=true` disables certificate verification for camera streams only.
 
 See [RTSPS and TLS certificates](README-RTSPS-TLS.md) for the complete Bosch UI, private-CA, deployment, validation, and troubleshooting procedure.
 
@@ -208,6 +304,7 @@ See [RTSPS and TLS certificates](README-RTSPS-TLS.md) for the complete Bosch UI,
 | --------------------------------------- | ----------------------------------------------------------------------------------------------- | ------------------------------ |
 | `LOG_LEVEL`                                 | Level for logging, could be "info", "warning", "debug", "error" or "fatal".                     | "info"                         |
 | `LOG_OUTPUT`                                | Logging output format "json" or "text".                                                         | "text"                         |
+| `AGENT_PORT`                                | HTTP web server port. Overrides the `-port` command-line value and must be between 1 and 65535. | "80"                           |
 | `AGENT_MODE`                                | You can choose to run this in 'release' for production, and or 'demo' for showcasing.           | "release"                      |
 | `AGENT_TLS_INSECURE`                        | Specify if you want to use `InsecureSkipVerify` for the internal HTTP client.                   | "false"                        |
 | `AGENT_USERNAME`                            | The username used to authenticate against the Kerberos Agent login page.                        | "root"                         |
@@ -225,6 +322,7 @@ See [RTSPS and TLS certificates](README-RTSPS-TLS.md) for the complete Bosch UI,
 | `AGENT_REGION_POLYGON`                      | A single polygon set for motion detection: "x1,y1;x2,y2;x3,y3;...                               | ""                             |
 | `AGENT_CAPTURE_IPCAMERA_RTSP`               | Full-HD RTSP or RTSPS endpoint for the target camera.                                           | ""                             |
 | `AGENT_CAPTURE_IPCAMERA_SUB_RTSP`           | RTSP or RTSPS sub-stream endpoint used for livestreaming (WebRTC).                              | ""                             |
+| `AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE`      | PEM CA bundle appended to the system roots for RTSPS camera certificate verification.           | ""                             |
 | `AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE`     | Disable RTSPS camera certificate verification; use only when a trusted CA cannot be installed.  | "false"                        |
 | `AGENT_CAPTURE_IPCAMERA_BASE_WIDTH`         | Force a specific width resolution for live view processing.                                     | ""                             |
 | `AGENT_CAPTURE_IPCAMERA_BASE_HEIGHT`        | Force a specific height resolution for live view processing.                                    | ""                             |
@@ -260,6 +358,7 @@ See [RTSPS and TLS certificates](README-RTSPS-TLS.md) for the complete Bosch UI,
 | `AGENT_HUB_PRIVATE_KEY`                     | The secret access key linked to your account in Kerberos Hub.                                   | ""                             |
 | `AGENT_HUB_REGION`                          | The Kerberos Hub region, to which you want to upload.                                           | ""                             |
 | `AGENT_HUB_SITE`                            | The site ID of a site you've created in your Kerberos Hub account.                              | ""                             |
+| `AGENT_TUS_CHUNK_SIZE_BYTES`                | Bytes sent in each resumable-upload PATCH. Set to `0` or a negative value to send all remaining bytes in one PATCH. | "8388608" (8 MiB)              |
 | `AGENT_KERBEROSVAULT_URI`                   | The Kerberos Vault API url.                                                                     | "https://vault.domain.com/api" |
 | `AGENT_KERBEROSVAULT_ACCESS_KEY`            | The access key of a Kerberos Vault account.                                                     | ""                             |
 | `AGENT_KERBEROSVAULT_SECRET_KEY`            | The secret key of a Kerberos Vault account.                                                     | ""                             |
@@ -280,6 +379,27 @@ See [RTSPS and TLS certificates](README-RTSPS-TLS.md) for the complete Bosch UI,
 | `AGENT_SIGNING`                             | Enable 'true' or disable 'false' for signing recordings.                                        | "true"                         |
 | `AGENT_SIGNING_PRIVATE_KEY`                 | The private key (RSA) to sign the recordings fingerprint to validate origin.                    | "" - uses default one if empty |
 
+### Resumable upload chunk size
+
+Hub and Vault resumable uploads use `AGENT_TUS_CHUNK_SIZE_BYTES` as the maximum
+body size of each tus `PATCH` request. The value is a number of bytes, not a
+number of chunks. If the variable is unset or invalid, the Agent uses 8 MiB:
+
+```dotenv
+# 8 MiB (default)
+AGENT_TUS_CHUNK_SIZE_BYTES=8388608
+
+# 4 MiB: more frequent checkpoints on unstable connections
+AGENT_TUS_CHUNK_SIZE_BYTES=4194304
+
+# Disable chunking and send all remaining bytes in one PATCH
+AGENT_TUS_CHUNK_SIZE_BYTES=0
+```
+
+Smaller chunks provide more frequent resumable checkpoints but create more HTTP
+requests. Larger chunks reduce request overhead but require more data to be
+retransmitted when a request fails.
+
 
 ## Encryption
 
@@ -454,11 +574,26 @@ its own broadcast and switching quality is simply a resubscribe:
 | `high` | `devices/<agent-key>/live.hang`       | highest-resolution camera stream           |
 | `low`  | `devices/<agent-key>/live-low.hang`   | sub stream (main stream when none is set)  |
 
-Each tier only uploads while it has at least one subscriber, so the tier nobody
-watches costs virtually no bandwidth. `AGENT_LIVE_MOQ_QUALITY` accepts `high` or
-`low` to pin the Agent to a single tier; viewers requesting the other tier then
-find no broadcast. Any other value (including the default) publishes both. The
-initial implementation publishes H.264 video only.
+Each tier uploads every frame while it has subscribers. While idle, it continues
+to upload fresh keyframes only. This keeps the relay's latest cached GOP current,
+so a new viewer does not initially render the final frame from the previous
+viewer session, while avoiding the bandwidth cost of continuously sending every
+delta frame. `AGENT_LIVE_MOQ_QUALITY` accepts `high` or `low` to pin the Agent to
+a single tier; viewers requesting the other tier then find no broadcast. Any
+other value (including the default) publishes both. The initial implementation
+publishes H.264 video only.
+
+Two bounded duration settings tune recovery for unusual network conditions:
+
+| Variable | Default | Allowed range |
+| -------- | ------- | ------------- |
+| `AGENT_LIVE_MOQ_MAX_PACKET_AGE` | `1.5s` | `250ms` to `30s` |
+| `AGENT_LIVE_MOQ_WRITE_TIMEOUT` | `5s` | `1s` to `1m` |
+
+Values use Go duration syntax and are clamped to the documented range. The
+dashboard API's `recovery` object reports per-tier reconnects, last successful
+frame/write timing, native write timeouts, watchdog restarts/cooldown, dropped
+run-channel events, and run-worker shutdown timeouts.
 
 The `/anon` relay route is intended for interoperability testing. Production
 deployments must set `AGENT_LIVE_MOQ_URL` to a short-lived, device-scoped
@@ -471,8 +606,9 @@ monorepo devcontainer so it uses the Trixie base, then run the VS Code task
     cd machinery
     bash ./verify-moq-devcontainer.sh
 
-The check requires glibc 2.38 or newer, runs the tagged package tests, links the
-complete Agent with `-tags moq`, and executes the resulting binary's version
+The check requires glibc 2.38 or newer, runs the tagged package and in-process
+native QUIC lifecycle tests, links the complete Agent with `-tags moq`, and
+executes the resulting binary's version
 command. Both devcontainers also run this check during their post-create setup.
 
 ## What is new?

machinery/README.md

@@ -5,7 +5,6 @@
 <a target="_blank" href="https://twitter.com/kerberosio?ref_src=twsrc%5Etfw"><img src="https://img.shields.io/twitter/url.svg?label=Follow%20%40kerberosio&style=social&url=https%3A%2F%2Ftwitter.com%2Fkerberosio" alt="Twitter Widget"></a>
 <a target="_blank" href="https://join.slack.com/t/kerberosio/shared_invite/zt-1a5oj4pwm-O4qCAN9c5r2um0Ns0ge8ww"><img src="https://img.shields.io/badge/slack-@kerberosio-yellow.svg?logo=slack " alt="Kerberos.io"></a>
 
-<a target="_blank" href="https://circleci.com/gh/kerberos-io/agent"><img src="https://circleci.com/gh/kerberos-io/agent.svg?style=svg"/></a>
 <img src="https://github.com/kerberos-io/agent/workflows/Go/badge.svg"/>
 <img src="https://github.com/kerberos-io/agent/workflows/CodeQL/badge.svg"/>
 

machinery/data/tus/1786715908_3-737_encrypted_0-0-0-0_-1_25498.mp4.primary.json

@@ -0,0 +1 @@
+{"upload_url":"https://vault.kerberos.io/api/storage/tus/19e42fbc666a38064904caf8c46d182a","vault_uri":"https://vault.kerberos.io/api/storage/tus/","size":1591581}

machinery/data/tus/1788718145_3-694_encrypted_0-0-0-0_-1_25497.mp4.primary.json

@@ -0,0 +1 @@
+{"upload_url":"https://vault.kerberos.io/api/storage/tus/bd3be0f9ecd3873d381f57cb8b4a7e96","vault_uri":"https://vault.kerberos.io/api/storage/tus/","size":2560480}

machinery/docs/docs.go

@@ -24,6 +24,38 @@ const docTemplate = `{
     "host": "{{.Host}}",
     "basePath": "{{.BasePath}}",
     "paths": {
+        "/api/camera/discover": {
+            "get": {
+                "description": "Runs an advanced Fing/WiFiman-style scan (ONVIF WS-Discovery + TCP port scan + MAC/vendor lookup) and returns the devices found on the local network.",
+                "tags": [
+                    "onvif"
+                ],
+                "summary": "Discover cameras and other devices on the local network.",
+                "operationId": "camera-discover",
+                "parameters": [
+                    {
+                        "type": "integer",
+                        "description": "Discovery timeout in milliseconds (default 2000)",
+                        "name": "timeout",
+                        "in": "query"
+                    },
+                    {
+                        "type": "string",
+                        "description": "Optional subnet(s) to scan, e.g. '192.168.1.0/24' (comma-separated). Defaults to the local interfaces.",
+                        "name": "subnet",
+                        "in": "query"
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/models.APIResponse"
+                        }
+                    }
+                }
+            }
+        },
         "/api/camera/onvif/capabilities": {
             "post": {
                 "description": "Will return the ONVIF capabilities for the specific camera.",
@@ -623,6 +655,40 @@ const docTemplate = `{
                 }
             }
         },
+        "/api/persistence/secondary/verify": {
+            "post": {
+                "security": [
+                    {
+                        "Bearer": []
+                    }
+                ],
+                "description": "Will verify the secondary persistence.",
+                "tags": [
+                    "persistence"
+                ],
+                "summary": "Will verify the secondary persistence.",
+                "operationId": "verify-secondary-persistence",
+                "parameters": [
+                    {
+                        "description": "Config",
+                        "name": "config",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/models.Config"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/models.APIResponse"
+                        }
+                    }
+                }
+            }
+        },
         "/api/persistence/verify": {
             "post": {
                 "security": [
@@ -656,9 +722,151 @@ const docTemplate = `{
                     }
                 }
             }
+        },
+        "/health": {
+            "get": {
+                "description": "Confirms that the Agent HTTP process can serve requests and reports current camera stream and Hub heartbeat diagnostics. Operational dependency failures do not change the liveness HTTP status.",
+                "produces": [
+                    "application/json"
+                ],
+                "tags": [
+                    "health"
+                ],
+                "summary": "Check Agent health",
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/http.HealthResponse"
+                        }
+                    }
+                }
+            }
         }
     },
     "definitions": {
+        "http.Health": {
+            "type": "object",
+            "properties": {
+                "cameraConnected": {
+                    "type": "boolean"
+                },
+                "description": {
+                    "type": "string"
+                },
+                "hub": {
+                    "$ref": "#/definitions/http.HubHealth"
+                },
+                "mainStream": {
+                    "$ref": "#/definitions/http.StreamHealth"
+                },
+                "subStream": {
+                    "$ref": "#/definitions/http.StreamHealth"
+                }
+            }
+        },
+        "http.HealthResponse": {
+            "type": "object",
+            "properties": {
+                "applicationStatusCode": {
+                    "type": "string"
+                },
+                "data": {
+                    "$ref": "#/definitions/http.HealthResponseData"
+                },
+                "entityStatusCode": {
+                    "type": "string"
+                },
+                "httpStatusCode": {
+                    "type": "integer"
+                },
+                "message": {
+                    "type": "string"
+                },
+                "metadata": {
+                    "$ref": "#/definitions/http.ResponseMetadata"
+                }
+            }
+        },
+        "http.HealthResponseData": {
+            "type": "object",
+            "properties": {
+                "health": {
+                    "$ref": "#/definitions/http.Health"
+                }
+            }
+        },
+        "http.HubHealth": {
+            "type": "object",
+            "properties": {
+                "configured": {
+                    "type": "boolean"
+                },
+                "connected": {
+                    "type": "boolean"
+                },
+                "lastHeartbeatAttemptAt": {
+                    "type": "integer"
+                },
+                "lastSuccessfulHeartbeatAt": {
+                    "type": "integer"
+                }
+            }
+        },
+        "http.ResponseMetadata": {
+            "type": "object",
+            "properties": {
+                "applicationName": {
+                    "type": "string"
+                },
+                "applicationVersion": {
+                    "type": "string"
+                },
+                "path": {
+                    "type": "string"
+                },
+                "timestamp": {
+                    "type": "integer"
+                },
+                "traceId": {
+                    "type": "string"
+                }
+            }
+        },
+        "http.StreamHealth": {
+            "type": "object",
+            "properties": {
+                "configured": {
+                    "type": "boolean"
+                },
+                "connected": {
+                    "type": "boolean"
+                },
+                "fps": {
+                    "type": "number"
+                },
+                "lastPacketAt": {
+                    "type": "integer"
+                },
+                "packagesProcessed": {
+                    "type": "integer"
+                },
+                "resolution": {
+                    "$ref": "#/definitions/http.StreamResolution"
+                }
+            }
+        },
+        "http.StreamResolution": {
+            "type": "object",
+            "properties": {
+                "height": {
+                    "type": "integer"
+                },
+                "width": {
+                    "type": "integer"
+                }
+            }
+        },
         "models.APIResponse": {
             "type": "object",
             "properties": {
@@ -730,12 +938,19 @@ const docTemplate = `{
                 "fragmentedduration": {
                     "type": "integer"
                 },
+                "gopsize": {
+                    "description": "GOP size in seconds, used for pre-recording",
+                    "type": "integer"
+                },
                 "ipcamera": {
                     "$ref": "#/definitions/models.IPCamera"
                 },
                 "liveview": {
                     "type": "string"
                 },
+                "liveview_chunking": {
+                    "type": "string"
+                },
                 "maxlengthrecording": {
                     "type": "integer"
                 },
@@ -745,6 +960,10 @@ const docTemplate = `{
                 "name": {
                     "type": "string"
                 },
+                "onvif_motion": {
+                    "description": "ONVIFMotion routes the camera's ONVIF motion events into the\nagent's motion-triggered recording pipeline. When \"true\" the\nagent opens an event/stream against the configured ONVIF\nendpoint and forwards Motion+Active events to HandleMotion.\nRequires Capture.IPCamera.ONVIFXAddr / ONVIFUsername /\nONVIFPassword to be set. Default empty (disabled) keeps the\nexisting pixel-diff motion detection as the only source.",
+                    "type": "string"
+                },
                 "pixelChangeThreshold": {
                     "type": "integer"
                 },
@@ -823,9 +1042,15 @@ const docTemplate = `{
                 "kstorage": {
                     "$ref": "#/definitions/models.KStorage"
                 },
+                "kstorage_secondary": {
+                    "$ref": "#/definitions/models.KStorage"
+                },
                 "max_directory_size": {
                     "type": "integer"
                 },
+                "min_free_space": {
+                    "type": "integer"
+                },
                 "mqtt_password": {
                     "type": "string"
                 },
@@ -856,6 +1081,9 @@ const docTemplate = `{
                 "s3": {
                     "$ref": "#/definitions/models.S3"
                 },
+                "signing": {
+                    "$ref": "#/definitions/models.Signing"
+                },
                 "stunuri": {
                     "type": "string"
                 },
@@ -947,6 +1175,15 @@ const docTemplate = `{
         "models.IPCamera": {
             "type": "object",
             "properties": {
+                "base_height": {
+                    "type": "integer"
+                },
+                "base_width": {
+                    "type": "integer"
+                },
+                "channels": {
+                    "type": "integer"
+                },
                 "fps": {
                     "type": "string"
                 },
@@ -965,21 +1202,85 @@ const docTemplate = `{
                 "onvif_xaddr": {
                     "type": "string"
                 },
+                "pps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "rtsp": {
                     "type": "string"
                 },
+                "sample_rate": {
+                    "type": "integer"
+                },
+                "sps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_fps": {
                     "type": "string"
                 },
                 "sub_height": {
                     "type": "integer"
                 },
+                "sub_pps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_rtsp": {
                     "type": "string"
                 },
+                "sub_sps_nalus": {
+                    "description": "Sub stream parameter sets, captured separately from the main stream so the\nlive HLS muxer can build a correct init segment when a viewer switches the\nlive view to the sub (low-resolution) stream.",
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
+                "sub_vps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_width": {
                     "type": "integer"
                 },
+                "vps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "width": {
                     "type": "integer"
                 }
@@ -998,12 +1299,18 @@ const docTemplate = `{
                 "directory": {
                     "type": "string"
                 },
+                "max_retries": {
+                    "type": "integer"
+                },
                 "provider": {
                     "type": "string"
                 },
                 "secret_access_key": {
                     "type": "string"
                 },
+                "timeout": {
+                    "type": "integer"
+                },
                 "uri": {
                     "type": "string"
                 }
@@ -1141,6 +1448,17 @@ const docTemplate = `{
                 }
             }
         },
+        "models.Signing": {
+            "type": "object",
+            "properties": {
+                "enabled": {
+                    "type": "string"
+                },
+                "private_key": {
+                    "type": "string"
+                }
+            }
+        },
         "models.Timetable": {
             "type": "object",
             "properties": {

machinery/docs/swagger.json

@@ -17,6 +17,38 @@
     },
     "basePath": "/",
     "paths": {
+        "/api/camera/discover": {
+            "get": {
+                "description": "Runs an advanced Fing/WiFiman-style scan (ONVIF WS-Discovery + TCP port scan + MAC/vendor lookup) and returns the devices found on the local network.",
+                "tags": [
+                    "onvif"
+                ],
+                "summary": "Discover cameras and other devices on the local network.",
+                "operationId": "camera-discover",
+                "parameters": [
+                    {
+                        "type": "integer",
+                        "description": "Discovery timeout in milliseconds (default 2000)",
+                        "name": "timeout",
+                        "in": "query"
+                    },
+                    {
+                        "type": "string",
+                        "description": "Optional subnet(s) to scan, e.g. '192.168.1.0/24' (comma-separated). Defaults to the local interfaces.",
+                        "name": "subnet",
+                        "in": "query"
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/models.APIResponse"
+                        }
+                    }
+                }
+            }
+        },
         "/api/camera/onvif/capabilities": {
             "post": {
                 "description": "Will return the ONVIF capabilities for the specific camera.",
@@ -616,6 +648,40 @@
                 }
             }
         },
+        "/api/persistence/secondary/verify": {
+            "post": {
+                "security": [
+                    {
+                        "Bearer": []
+                    }
+                ],
+                "description": "Will verify the secondary persistence.",
+                "tags": [
+                    "persistence"
+                ],
+                "summary": "Will verify the secondary persistence.",
+                "operationId": "verify-secondary-persistence",
+                "parameters": [
+                    {
+                        "description": "Config",
+                        "name": "config",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/models.Config"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/models.APIResponse"
+                        }
+                    }
+                }
+            }
+        },
         "/api/persistence/verify": {
             "post": {
                 "security": [
@@ -649,9 +715,151 @@
                     }
                 }
             }
+        },
+        "/health": {
+            "get": {
+                "description": "Confirms that the Agent HTTP process can serve requests and reports current camera stream and Hub heartbeat diagnostics. Operational dependency failures do not change the liveness HTTP status.",
+                "produces": [
+                    "application/json"
+                ],
+                "tags": [
+                    "health"
+                ],
+                "summary": "Check Agent health",
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/http.HealthResponse"
+                        }
+                    }
+                }
+            }
         }
     },
     "definitions": {
+        "http.Health": {
+            "type": "object",
+            "properties": {
+                "cameraConnected": {
+                    "type": "boolean"
+                },
+                "description": {
+                    "type": "string"
+                },
+                "hub": {
+                    "$ref": "#/definitions/http.HubHealth"
+                },
+                "mainStream": {
+                    "$ref": "#/definitions/http.StreamHealth"
+                },
+                "subStream": {
+                    "$ref": "#/definitions/http.StreamHealth"
+                }
+            }
+        },
+        "http.HealthResponse": {
+            "type": "object",
+            "properties": {
+                "applicationStatusCode": {
+                    "type": "string"
+                },
+                "data": {
+                    "$ref": "#/definitions/http.HealthResponseData"
+                },
+                "entityStatusCode": {
+                    "type": "string"
+                },
+                "httpStatusCode": {
+                    "type": "integer"
+                },
+                "message": {
+                    "type": "string"
+                },
+                "metadata": {
+                    "$ref": "#/definitions/http.ResponseMetadata"
+                }
+            }
+        },
+        "http.HealthResponseData": {
+            "type": "object",
+            "properties": {
+                "health": {
+                    "$ref": "#/definitions/http.Health"
+                }
+            }
+        },
+        "http.HubHealth": {
+            "type": "object",
+            "properties": {
+                "configured": {
+                    "type": "boolean"
+                },
+                "connected": {
+                    "type": "boolean"
+                },
+                "lastHeartbeatAttemptAt": {
+                    "type": "integer"
+                },
+                "lastSuccessfulHeartbeatAt": {
+                    "type": "integer"
+                }
+            }
+        },
+        "http.ResponseMetadata": {
+            "type": "object",
+            "properties": {
+                "applicationName": {
+                    "type": "string"
+                },
+                "applicationVersion": {
+                    "type": "string"
+                },
+                "path": {
+                    "type": "string"
+                },
+                "timestamp": {
+                    "type": "integer"
+                },
+                "traceId": {
+                    "type": "string"
+                }
+            }
+        },
+        "http.StreamHealth": {
+            "type": "object",
+            "properties": {
+                "configured": {
+                    "type": "boolean"
+                },
+                "connected": {
+                    "type": "boolean"
+                },
+                "fps": {
+                    "type": "number"
+                },
+                "lastPacketAt": {
+                    "type": "integer"
+                },
+                "packagesProcessed": {
+                    "type": "integer"
+                },
+                "resolution": {
+                    "$ref": "#/definitions/http.StreamResolution"
+                }
+            }
+        },
+        "http.StreamResolution": {
+            "type": "object",
+            "properties": {
+                "height": {
+                    "type": "integer"
+                },
+                "width": {
+                    "type": "integer"
+                }
+            }
+        },
         "models.APIResponse": {
             "type": "object",
             "properties": {
@@ -723,12 +931,19 @@
                 "fragmentedduration": {
                     "type": "integer"
                 },
+                "gopsize": {
+                    "description": "GOP size in seconds, used for pre-recording",
+                    "type": "integer"
+                },
                 "ipcamera": {
                     "$ref": "#/definitions/models.IPCamera"
                 },
                 "liveview": {
                     "type": "string"
                 },
+                "liveview_chunking": {
+                    "type": "string"
+                },
                 "maxlengthrecording": {
                     "type": "integer"
                 },
@@ -738,6 +953,10 @@
                 "name": {
                     "type": "string"
                 },
+                "onvif_motion": {
+                    "description": "ONVIFMotion routes the camera's ONVIF motion events into the\nagent's motion-triggered recording pipeline. When \"true\" the\nagent opens an event/stream against the configured ONVIF\nendpoint and forwards Motion+Active events to HandleMotion.\nRequires Capture.IPCamera.ONVIFXAddr / ONVIFUsername /\nONVIFPassword to be set. Default empty (disabled) keeps the\nexisting pixel-diff motion detection as the only source.",
+                    "type": "string"
+                },
                 "pixelChangeThreshold": {
                     "type": "integer"
                 },
@@ -816,9 +1035,15 @@
                 "kstorage": {
                     "$ref": "#/definitions/models.KStorage"
                 },
+                "kstorage_secondary": {
+                    "$ref": "#/definitions/models.KStorage"
+                },
                 "max_directory_size": {
                     "type": "integer"
                 },
+                "min_free_space": {
+                    "type": "integer"
+                },
                 "mqtt_password": {
                     "type": "string"
                 },
@@ -849,6 +1074,9 @@
                 "s3": {
                     "$ref": "#/definitions/models.S3"
                 },
+                "signing": {
+                    "$ref": "#/definitions/models.Signing"
+                },
                 "stunuri": {
                     "type": "string"
                 },
@@ -940,6 +1168,15 @@
         "models.IPCamera": {
             "type": "object",
             "properties": {
+                "base_height": {
+                    "type": "integer"
+                },
+                "base_width": {
+                    "type": "integer"
+                },
+                "channels": {
+                    "type": "integer"
+                },
                 "fps": {
                     "type": "string"
                 },
@@ -958,21 +1195,85 @@
                 "onvif_xaddr": {
                     "type": "string"
                 },
+                "pps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "rtsp": {
                     "type": "string"
                 },
+                "sample_rate": {
+                    "type": "integer"
+                },
+                "sps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_fps": {
                     "type": "string"
                 },
                 "sub_height": {
                     "type": "integer"
                 },
+                "sub_pps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_rtsp": {
                     "type": "string"
                 },
+                "sub_sps_nalus": {
+                    "description": "Sub stream parameter sets, captured separately from the main stream so the\nlive HLS muxer can build a correct init segment when a viewer switches the\nlive view to the sub (low-resolution) stream.",
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
+                "sub_vps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_width": {
                     "type": "integer"
                 },
+                "vps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "width": {
                     "type": "integer"
                 }
@@ -991,12 +1292,18 @@
                 "directory": {
                     "type": "string"
                 },
+                "max_retries": {
+                    "type": "integer"
+                },
                 "provider": {
                     "type": "string"
                 },
                 "secret_access_key": {
                     "type": "string"
                 },
+                "timeout": {
+                    "type": "integer"
+                },
                 "uri": {
                     "type": "string"
                 }
@@ -1134,6 +1441,17 @@
                 }
             }
         },
+        "models.Signing": {
+            "type": "object",
+            "properties": {
+                "enabled": {
+                    "type": "string"
+                },
+                "private_key": {
+                    "type": "string"
+                }
+            }
+        },
         "models.Timetable": {
             "type": "object",
             "properties": {

machinery/docs/swagger.yaml

@@ -1,5 +1,84 @@
 basePath: /
 definitions:
+  http.Health:
+    properties:
+      cameraConnected:
+        type: boolean
+      description:
+        type: string
+      hub:
+        $ref: '#/definitions/http.HubHealth'
+      mainStream:
+        $ref: '#/definitions/http.StreamHealth'
+      subStream:
+        $ref: '#/definitions/http.StreamHealth'
+    type: object
+  http.HealthResponse:
+    properties:
+      applicationStatusCode:
+        type: string
+      data:
+        $ref: '#/definitions/http.HealthResponseData'
+      entityStatusCode:
+        type: string
+      httpStatusCode:
+        type: integer
+      message:
+        type: string
+      metadata:
+        $ref: '#/definitions/http.ResponseMetadata'
+    type: object
+  http.HealthResponseData:
+    properties:
+      health:
+        $ref: '#/definitions/http.Health'
+    type: object
+  http.HubHealth:
+    properties:
+      configured:
+        type: boolean
+      connected:
+        type: boolean
+      lastHeartbeatAttemptAt:
+        type: integer
+      lastSuccessfulHeartbeatAt:
+        type: integer
+    type: object
+  http.ResponseMetadata:
+    properties:
+      applicationName:
+        type: string
+      applicationVersion:
+        type: string
+      path:
+        type: string
+      timestamp:
+        type: integer
+      traceId:
+        type: string
+    type: object
+  http.StreamHealth:
+    properties:
+      configured:
+        type: boolean
+      connected:
+        type: boolean
+      fps:
+        type: number
+      lastPacketAt:
+        type: integer
+      packagesProcessed:
+        type: integer
+      resolution:
+        $ref: '#/definitions/http.StreamResolution'
+    type: object
+  http.StreamResolution:
+    properties:
+      height:
+        type: integer
+      width:
+        type: integer
+    type: object
   models.APIResponse:
     properties:
       can_pan_tilt:
@@ -47,16 +126,31 @@ definitions:
         type: string
       fragmentedduration:
         type: integer
+      gopsize:
+        description: GOP size in seconds, used for pre-recording
+        type: integer
       ipcamera:
         $ref: '#/definitions/models.IPCamera'
       liveview:
         type: string
+      liveview_chunking:
+        type: string
       maxlengthrecording:
         type: integer
       motion:
         type: string
       name:
         type: string
+      onvif_motion:
+        description: |-
+          ONVIFMotion routes the camera's ONVIF motion events into the
+          agent's motion-triggered recording pipeline. When "true" the
+          agent opens an event/stream against the configured ONVIF
+          endpoint and forwards Motion+Active events to HandleMotion.
+          Requires Capture.IPCamera.ONVIFXAddr / ONVIFUsername /
+          ONVIFPassword to be set. Default empty (disabled) keeps the
+          existing pixel-diff motion detection as the only source.
+        type: string
       pixelChangeThreshold:
         type: integer
       postrecording:
@@ -109,8 +203,12 @@ definitions:
         type: string
       kstorage:
         $ref: '#/definitions/models.KStorage'
+      kstorage_secondary:
+        $ref: '#/definitions/models.KStorage'
       max_directory_size:
         type: integer
+      min_free_space:
+        type: integer
       mqtt_password:
         type: string
       mqtt_username:
@@ -131,6 +229,8 @@ definitions:
         type: string
       s3:
         $ref: '#/definitions/models.S3'
+      signing:
+        $ref: '#/definitions/models.Signing'
       stunuri:
         type: string
       time:
@@ -190,6 +290,12 @@ definitions:
     type: object
   models.IPCamera:
     properties:
+      base_height:
+        type: integer
+      base_width:
+        type: integer
+      channels:
+        type: integer
       fps:
         type: string
       height:
@@ -202,16 +308,64 @@ definitions:
         type: string
       onvif_xaddr:
         type: string
+      pps_nalus:
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
       rtsp:
         type: string
+      sample_rate:
+        type: integer
+      sps_nalus:
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
       sub_fps:
         type: string
       sub_height:
         type: integer
+      sub_pps_nalus:
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
       sub_rtsp:
         type: string
+      sub_sps_nalus:
+        description: |-
+          Sub stream parameter sets, captured separately from the main stream so the
+          live HLS muxer can build a correct init segment when a viewer switches the
+          live view to the sub (low-resolution) stream.
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
+      sub_vps_nalus:
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
       sub_width:
         type: integer
+      vps_nalus:
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
       width:
         type: integer
     type: object
@@ -224,10 +378,14 @@ definitions:
         type: string
       directory:
         type: string
+      max_retries:
+        type: integer
       provider:
         type: string
       secret_access_key:
         type: string
+      timeout:
+        type: integer
       uri:
         type: string
     type: object
@@ -316,6 +474,13 @@ definitions:
       username:
         type: string
     type: object
+  models.Signing:
+    properties:
+      enabled:
+        type: string
+      private_key:
+        type: string
+    type: object
   models.Timetable:
     properties:
       end1:
@@ -345,6 +510,30 @@ info:
   title: Swagger Kerberos Agent API
   version: "1.0"
 paths:
+  /api/camera/discover:
+    get:
+      description: Runs an advanced Fing/WiFiman-style scan (ONVIF WS-Discovery +
+        TCP port scan + MAC/vendor lookup) and returns the devices found on the local
+        network.
+      operationId: camera-discover
+      parameters:
+      - description: Discovery timeout in milliseconds (default 2000)
+        in: query
+        name: timeout
+        type: integer
+      - description: Optional subnet(s) to scan, e.g. '192.168.1.0/24' (comma-separated).
+          Defaults to the local interfaces.
+        in: query
+        name: subnet
+        type: string
+      responses:
+        "200":
+          description: OK
+          schema:
+            $ref: '#/definitions/models.APIResponse'
+      summary: Discover cameras and other devices on the local network.
+      tags:
+      - onvif
   /api/camera/onvif/capabilities:
     post:
       description: Will return the ONVIF capabilities for the specific camera.
@@ -735,6 +924,27 @@ paths:
       summary: Get Authorization token.
       tags:
       - authentication
+  /api/persistence/secondary/verify:
+    post:
+      description: Will verify the secondary persistence.
+      operationId: verify-secondary-persistence
+      parameters:
+      - description: Config
+        in: body
+        name: config
+        required: true
+        schema:
+          $ref: '#/definitions/models.Config'
+      responses:
+        "200":
+          description: OK
+          schema:
+            $ref: '#/definitions/models.APIResponse'
+      security:
+      - Bearer: []
+      summary: Will verify the secondary persistence.
+      tags:
+      - persistence
   /api/persistence/verify:
     post:
       description: Will verify the persistence.
@@ -756,6 +966,21 @@ paths:
       summary: Will verify the persistence.
       tags:
       - persistence
+  /health:
+    get:
+      description: Confirms that the Agent HTTP process can serve requests and reports
+        current camera stream and Hub heartbeat diagnostics. Operational dependency
+        failures do not change the liveness HTTP status.
+      produces:
+      - application/json
+      responses:
+        "200":
+          description: OK
+          schema:
+            $ref: '#/definitions/http.HealthResponse'
+      summary: Check Agent health
+      tags:
+      - health
 securityDefinitions:
   Bearer:
     in: header

machinery/go.mod

@@ -28,7 +28,6 @@ require (
 	github.com/minio/minio-go/v6 v6.0.57
 	github.com/moq-dev/moq-go v0.5.7
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
-	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/pion/interceptor v0.1.47
 	github.com/pion/rtp v1.10.5
 	github.com/pion/webrtc/v4 v4.2.18
@@ -45,7 +44,6 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.36.0
 	go.opentelemetry.io/otel/sdk v1.36.0
 	go.opentelemetry.io/otel/trace v1.36.0
-	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 )
 
 require (

machinery/go.sum

@@ -858,8 +858,6 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
 github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE=
 github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
-github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
-github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
 github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
 github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY=
@@ -2106,8 +2104,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk=
 gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
-gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

machinery/logging.go

@@ -0,0 +1,160 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	defaultLogLevel  = log.InfoLevel
+	defaultLogOutput = "text"
+)
+
+type localTimeFormatter struct {
+	timezone  *time.Location
+	formatter log.Formatter
+}
+
+func (f localTimeFormatter) Format(entry *log.Entry) ([]byte, error) {
+	entry.Time = entry.Time.In(f.timezone)
+	return f.formatter.Format(entry)
+}
+
+type componentHook struct{}
+
+func (componentHook) Levels() []log.Level {
+	return log.AllLevels
+}
+
+func (componentHook) Fire(entry *log.Entry) error {
+	if _, exists := entry.Data["component"]; exists {
+		return nil
+	}
+	entry.Data["component"] = componentFromCaller(entry.Caller)
+	return nil
+}
+
+func configureLogging(levelValue string, outputValue string, timezone *time.Location) {
+	configureLogger(log.StandardLogger(), levelValue, outputValue, timezone)
+}
+
+func configureLogger(logger *log.Logger, levelValue string, outputValue string, timezone *time.Location) {
+	if timezone == nil {
+		timezone = time.Local
+	}
+
+	level, levelErr := parseLogLevel(levelValue)
+	output, outputErr := parseLogOutput(outputValue)
+
+	logger.SetOutput(os.Stdout)
+	logger.SetLevel(level)
+	logger.SetReportCaller(true)
+	logger.SetFormatter(localTimeFormatter{
+		timezone:  timezone,
+		formatter: newLogFormatter(output),
+	})
+	installComponentHook(logger)
+
+	if levelErr != nil {
+		logger.WithFields(log.Fields{
+			"configured_level": levelValue,
+			"effective_level":  level.String(),
+		}).WithError(levelErr).Warn("invalid log level; using default")
+	}
+	if outputErr != nil {
+		logger.WithFields(log.Fields{
+			"configured_output": outputValue,
+			"effective_output":  output,
+		}).WithError(outputErr).Warn("invalid log output; using default")
+	}
+
+	logger.WithFields(log.Fields{
+		"event":         "logger_configured",
+		"log_level":     level.String(),
+		"output":        output,
+		"report_caller": logger.ReportCaller,
+		"timezone":      timezone.String(),
+	}).Debug("logging configured")
+}
+
+func installComponentHook(logger *log.Logger) {
+	for _, hooks := range logger.Hooks {
+		for _, hook := range hooks {
+			if _, ok := hook.(componentHook); ok {
+				return
+			}
+		}
+	}
+	logger.AddHook(componentHook{})
+}
+
+func componentFromCaller(frame *runtime.Frame) string {
+	if frame == nil {
+		return "unknown"
+	}
+
+	const sourceMarker = "/machinery/src/"
+	normalizedFile := filepath.ToSlash(frame.File)
+	if markerIndex := strings.Index(normalizedFile, sourceMarker); markerIndex >= 0 {
+		relativeFile := normalizedFile[markerIndex+len(sourceMarker):]
+		if directory := filepath.ToSlash(filepath.Dir(relativeFile)); directory != "." {
+			return directory
+		}
+	}
+	if strings.Contains(normalizedFile, "/machinery/") {
+		return "agent"
+	}
+	return "unknown"
+}
+
+func parseLogLevel(value string) (log.Level, error) {
+	normalized := strings.ToLower(strings.TrimSpace(value))
+	if normalized == "" {
+		return defaultLogLevel, nil
+	}
+	if normalized == "warning" {
+		normalized = "warn"
+	}
+
+	level, err := log.ParseLevel(normalized)
+	if err != nil {
+		return defaultLogLevel, fmt.Errorf("parse LOG_LEVEL: %w", err)
+	}
+	return level, nil
+}
+
+func parseLogOutput(value string) (string, error) {
+	normalized := strings.ToLower(strings.TrimSpace(value))
+	if normalized == "" {
+		return defaultLogOutput, nil
+	}
+	switch normalized {
+	case "json", "text":
+		return normalized, nil
+	default:
+		return defaultLogOutput, fmt.Errorf("unsupported LOG_OUTPUT %q", value)
+	}
+}
+
+func newLogFormatter(output string) log.Formatter {
+	callerPrettyfier := func(frame *runtime.Frame) (string, string) {
+		return filepath.Base(frame.Function), fmt.Sprintf("%s:%d", filepath.Base(frame.File), frame.Line)
+	}
+	if output == "json" {
+		return &log.JSONFormatter{
+			CallerPrettyfier: callerPrettyfier,
+			TimestampFormat:  time.RFC3339Nano,
+		}
+	}
+	return &log.TextFormatter{
+		CallerPrettyfier: callerPrettyfier,
+		FullTimestamp:    true,
+		TimestampFormat:  time.RFC3339Nano,
+	}
+}

machinery/logging_test.go

@@ -0,0 +1,165 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func TestConfigureLoggingDebugIncludesStructuredContext(t *testing.T) {
+	logger := log.New()
+	var output bytes.Buffer
+	configureLogger(logger, "debug", "json", time.UTC)
+	logger.SetOutput(&output)
+	logger.WithField("event", "test_event").Debug("structured debug test")
+
+	lines := strings.Split(strings.TrimSpace(output.String()), "\n")
+	var entry map[string]interface{}
+	if err := json.Unmarshal([]byte(lines[len(lines)-1]), &entry); err != nil {
+		t.Fatalf("decode debug log: %v; output=%q", err, output.String())
+	}
+	for key, want := range map[string]interface{}{
+		"component": "agent",
+		"event":     "test_event",
+		"level":     "debug",
+		"msg":       "structured debug test",
+	} {
+		if got := entry[key]; got != want {
+			t.Fatalf("%s = %v, want %v", key, got, want)
+		}
+	}
+	if entry["file"] == nil || entry["func"] == nil {
+		t.Fatalf("debug log is missing caller metadata: %v", entry)
+	}
+}
+
+func TestConfigureLoggerInstallsComponentHookOnce(t *testing.T) {
+	logger := log.New()
+	configureLogger(logger, "info", "text", time.UTC)
+	configureLogger(logger, "debug", "json", time.UTC)
+
+	var componentHooks int
+	for _, hooks := range logger.Hooks {
+		for _, hook := range hooks {
+			if _, ok := hook.(componentHook); ok {
+				componentHooks++
+			}
+		}
+	}
+	if componentHooks != len(log.AllLevels) {
+		t.Fatalf("component hook registrations = %d, want %d", componentHooks, len(log.AllLevels))
+	}
+}
+
+func TestComponentFromCaller(t *testing.T) {
+	tests := []struct {
+		name  string
+		frame *runtime.Frame
+		want  string
+	}{
+		{
+			name:  "nested runtime package",
+			frame: &runtime.Frame{File: "/workspace/agent/machinery/src/routers/mqtt/main.go"},
+			want:  "routers/mqtt",
+		},
+		{
+			name:  "top-level runtime package",
+			frame: &runtime.Frame{File: "/workspace/agent/machinery/src/capture/main.go"},
+			want:  "capture",
+		},
+		{
+			name:  "executable",
+			frame: &runtime.Frame{File: "/workspace/agent/machinery/main.go"},
+			want:  "agent",
+		},
+		{name: "missing caller", want: "unknown"},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if got := componentFromCaller(test.frame); got != test.want {
+				t.Fatalf("componentFromCaller() = %q, want %q", got, test.want)
+			}
+		})
+	}
+}
+
+func TestComponentHookAddsAndPreservesComponent(t *testing.T) {
+	hook := componentHook{}
+
+	entry := log.NewEntry(log.New())
+	entry.Caller = &runtime.Frame{File: "/workspace/agent/machinery/src/cloud/livehls/session.go"}
+	if err := hook.Fire(entry); err != nil {
+		t.Fatalf("componentHook.Fire() error = %v", err)
+	}
+	if got := entry.Data["component"]; got != "cloud/livehls" {
+		t.Fatalf("component = %v, want cloud/livehls", got)
+	}
+
+	entry.Data["component"] = "explicit"
+	if err := hook.Fire(entry); err != nil {
+		t.Fatalf("componentHook.Fire() preserving field error = %v", err)
+	}
+	if got := entry.Data["component"]; got != "explicit" {
+		t.Fatalf("component = %v, want explicit", got)
+	}
+}
+
+func TestParseLogLevel(t *testing.T) {
+	tests := []struct {
+		name    string
+		value   string
+		want    log.Level
+		wantErr bool
+	}{
+		{name: "default", want: log.InfoLevel},
+		{name: "info", value: "INFO", want: log.InfoLevel},
+		{name: "warning alias", value: "warning", want: log.WarnLevel},
+		{name: "debug", value: "debug", want: log.DebugLevel},
+		{name: "trace", value: "trace", want: log.TraceLevel},
+		{name: "invalid", value: "verbose", want: log.InfoLevel, wantErr: true},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got, err := parseLogLevel(test.value)
+			if (err != nil) != test.wantErr {
+				t.Fatalf("parseLogLevel(%q) error = %v, wantErr %t", test.value, err, test.wantErr)
+			}
+			if got != test.want {
+				t.Fatalf("parseLogLevel(%q) = %s, want %s", test.value, got, test.want)
+			}
+		})
+	}
+}
+
+func TestParseLogOutput(t *testing.T) {
+	tests := []struct {
+		name    string
+		value   string
+		want    string
+		wantErr bool
+	}{
+		{name: "default", want: "text"},
+		{name: "text", value: "TEXT", want: "text"},
+		{name: "json", value: "json", want: "json"},
+		{name: "invalid", value: "console", want: "text", wantErr: true},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got, err := parseLogOutput(test.value)
+			if (err != nil) != test.wantErr {
+				t.Fatalf("parseLogOutput(%q) error = %v, wantErr %t", test.value, err, test.wantErr)
+			}
+			if got != test.want {
+				t.Fatalf("parseLogOutput(%q) = %q, want %q", test.value, got, test.want)
+			}
+		})
+	}
+}

machinery/main.go

@@ -5,14 +5,15 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/capture"
 	"github.com/kerberos-io/agent/machinery/src/components"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/onvif"
+	log "github.com/sirupsen/logrus"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
@@ -28,6 +29,22 @@ import (
 
 var VERSION = utils.VERSION
 
+func resolveServerPort(flagValue, environmentValue string) (string, error) {
+	value := strings.TrimSpace(environmentValue)
+	if value == "" {
+		value = strings.TrimSpace(flagValue)
+	}
+	if value == "" {
+		value = "80"
+	}
+
+	port, err := strconv.Atoi(value)
+	if err != nil || port < 1 || port > 65535 {
+		return "", fmt.Errorf("port must be an integer between 1 and 65535, got %q", value)
+	}
+	return strconv.Itoa(port), nil
+}
+
 func startTracing(agentKey string, otelEndpoint string) (*trace.TracerProvider, error) {
 	serviceName := "agent-" + agentKey
 	headers := map[string]string{
@@ -99,20 +116,44 @@ func main() {
 	}
 	// Specify the timezone of the log: "UTC" or "Local".
 	timezone, _ := time.LoadLocation("CET")
-	log.Log.Init(logLevel, logOutput, configDirectory, timezone)
+	configureLogging(logLevel, logOutput, timezone)
+	if action == "run" {
+		resolvedPort, err := resolveServerPort(port, os.Getenv("AGENT_PORT"))
+		if err != nil {
+			log.WithError(err).WithFields(log.Fields{
+				"component": "http",
+				"event":     "server_port_invalid",
+			}).Fatal("Invalid HTTP server port")
+			return
+		}
+		port = resolvedPort
+	}
+	log.WithFields(log.Fields{
+		"action":           action,
+		"component":        "agent",
+		"config_directory": configDirectory,
+		"event":            "command_parsed",
+		"port":             port,
+		"version":          VERSION,
+	}).Debug("Agent command parsed")
 
 	switch action {
 
 	case "version":
 		{
-			log.Log.Info("main.Main(): You are currrently running Kerberos Agent " + VERSION)
+			log.WithFields(log.Fields{
+				"component": "agent",
+				"event":     "version",
+				"version":   VERSION,
+			}).Info("Kerberos Agent version")
 		}
 	case "discover":
 		{
 			// Convert duration to int
 			timeout, err := time.ParseDuration(timeout + "ms")
 			if err != nil {
-				log.Log.Fatal("main.Main(): could not parse timeout: " + err.Error())
+				log.WithError(err).WithField("component", "onvif").
+					Fatal("invalid ONVIF discovery timeout")
 				return
 			}
 			var subnets []string
@@ -125,15 +166,19 @@ func main() {
 		}
 	case "decrypt":
 		{
-			log.Log.Info("main.Main(): Decrypting: " + flag.Arg(0) + " with key: " + flag.Arg(1))
+			log.WithFields(log.Fields{
+				"component": "encryption",
+				"event":     "decrypt_started",
+				"path":      flag.Arg(0),
+			}).Info("Decrypting recording")
 			symmetricKey := []byte(flag.Arg(1))
 
 			if len(symmetricKey) == 0 {
-				log.Log.Fatal("main.Main(): symmetric key should not be empty")
+				log.Fatal("main.Main(): symmetric key should not be empty")
 				return
 			}
 			if len(symmetricKey) != 32 {
-				log.Log.Fatal("main.Main(): symmetric key should be 32 bytes")
+				log.Fatal("main.Main(): symmetric key should be 32 bytes")
 				return
 			}
 
@@ -163,19 +208,28 @@ func main() {
 
 			// Start OpenTelemetry tracing
 			if otelEndpoint := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"); otelEndpoint == "" {
-				log.Log.Info("main.Main(): No OpenTelemetry endpoint provided, skipping tracing")
+				log.WithFields(log.Fields{
+					"component": "tracing",
+					"event":     "tracing_disabled",
+				}).Debug("OpenTelemetry tracing disabled")
 			} else {
-				log.Log.Info("main.Main(): Starting OpenTelemetry tracing with endpoint: " + otelEndpoint)
+				log.WithFields(log.Fields{
+					"component": "tracing",
+					"event":     "tracing_starting",
+				}).Info("Starting OpenTelemetry tracing")
 				agentKey := configuration.Config.Key
 				traceProvider, err := startTracing(agentKey, otelEndpoint)
 				if err != nil {
-					log.Log.Error("traceprovider: " + err.Error())
+					log.WithError(err).WithField("component", "tracing").
+						Error("Failed to start OpenTelemetry tracing")
+				} else {
+					defer func() {
+						if err := traceProvider.Shutdown(context.Background()); err != nil {
+							log.WithError(err).WithField("component", "tracing").
+								Error("Failed to shut down OpenTelemetry tracing")
+						}
+					}()
 				}
-				defer func() {
-					if err := traceProvider.Shutdown(context.Background()); err != nil {
-						log.Log.Error("traceprovider: " + err.Error())
-					}
-				}()
 			}
 
 			// Printing final configuration
@@ -186,8 +240,13 @@ func main() {
 			utils.CheckDataDirectoryPermissions(configDirectory)
 
 			// Set timezone
-			timezone, _ := time.LoadLocation(configuration.Config.Timezone)
-			log.Log.Init(logLevel, logOutput, configDirectory, timezone)
+			timezone, err := time.LoadLocation(configuration.Config.Timezone)
+			if err != nil {
+				log.WithError(err).WithField("timezone", configuration.Config.Timezone).
+					Warn("invalid Agent timezone; using the host timezone for logs")
+				timezone = time.Local
+			}
+			configureLogging(logLevel, logOutput, timezone)
 
 			// Check if we have a device Key or not, if not
 			// we will generate one.
@@ -196,30 +255,37 @@ func main() {
 				configuration.Config.Key = key
 				err := configService.StoreConfig(configDirectory, configuration.Config)
 				if err == nil {
-					log.Log.Info("main.Main(): updated unique key for agent to: " + key)
+					log.WithFields(log.Fields{
+						"component": "configuration",
+						"event":     "agent_key_generated",
+					}).Info("Generated and stored a unique Agent key")
 				} else {
-					log.Log.Info("main.Main(): something went wrong while trying to store key: " + key)
+					log.WithError(err).WithFields(log.Fields{
+						"component": "configuration",
+						"event":     "agent_key_store_failed",
+					}).Error("Failed to store the generated Agent key")
 				}
 			}
 
 			// Create a cancelable context, which will be used to cancel and restart.
 			// This is used to restart the agent when the configuration is updated.
 			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
 
 			// We create a capture object, this will contain all the streaming clients.
 			// And allow us to extract media from within difference places in the agent.
-			capture := capture.Capture{
-				RTSPClient:    nil,
-				RTSPSubClient: nil,
-			}
+			capture := capture.Capture{}
 
 			// Bootstrapping the agent
 			communication := models.Communication{
-				Context:         &ctx,
-				CancelContext:   &cancel,
 				HandleBootstrap: make(chan string, 1),
 			}
 
+			log.WithFields(log.Fields{
+				"component": "agent",
+				"event":     "runtime_starting",
+				"port":      configuration.Port,
+			}).Info("Starting Agent runtime")
 			go components.Bootstrap(ctx, configDirectory, &configuration, &communication, &capture)
 
 			// Start the REST API.
@@ -227,7 +293,7 @@ func main() {
 		}
 	default:
 		{
-			log.Log.Error("main.Main(): Sorry I don't understand :(")
+			log.Error("main.Main(): Sorry I don't understand :(")
 		}
 	}
 }

machinery/main_test.go

@@ -0,0 +1,33 @@
+package main
+
+import "testing"
+
+func TestResolveServerPort(t *testing.T) {
+	tests := []struct {
+		name      string
+		flagValue string
+		envValue  string
+		want      string
+		wantError bool
+	}{
+		{name: "flag default", flagValue: "80", want: "80"},
+		{name: "environment overrides flag", flagValue: "80", envValue: "8082", want: "8082"},
+		{name: "trims environment", flagValue: "80", envValue: " 9090 ", want: "9090"},
+		{name: "empty values use default", want: "80"},
+		{name: "invalid text", flagValue: "80", envValue: "http", wantError: true},
+		{name: "zero", flagValue: "80", envValue: "0", wantError: true},
+		{name: "above maximum", flagValue: "80", envValue: "65536", wantError: true},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got, err := resolveServerPort(test.flagValue, test.envValue)
+			if (err != nil) != test.wantError {
+				t.Fatalf("resolveServerPort(%q, %q) error = %v, wantError %t", test.flagValue, test.envValue, err, test.wantError)
+			}
+			if got != test.want {
+				t.Fatalf("resolveServerPort(%q, %q) = %q, want %q", test.flagValue, test.envValue, got, test.want)
+			}
+		})
+	}
+}

machinery/src/capture/gortsplib.go

@@ -9,12 +9,16 @@ import "C"
 import (
 	"context"
 	"crypto/tls"
+	"crypto/x509"
 	"errors"
 	"fmt"
 	"image"
+	"math"
+	"net/url"
 	"os"
 	"reflect"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 	"unsafe"
@@ -31,23 +35,61 @@ import (
 	"github.com/bluenviron/mediacommon/pkg/codecs/h264"
 	"github.com/bluenviron/mediacommon/pkg/codecs/h265"
 	"github.com/bluenviron/mediacommon/pkg/codecs/mpeg4audio"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/pion/rtp"
+	log "github.com/sirupsen/logrus"
 	"go.opentelemetry.io/otel"
 )
 
 var tracer = otel.Tracer("github.com/kerberos-io/agent/machinery/src/capture")
 
-const rtspsInsecureEnv = "AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE"
+const (
+	rtspsCAFileEnv   = "AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE"
+	rtspsInsecureEnv = "AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE"
+)
 
-func rtspsTLSConfig() *tls.Config {
-	if os.Getenv(rtspsInsecureEnv) != "true" {
-		return nil
+func rtspsTLSConfig() (*tls.Config, error) {
+	if os.Getenv(rtspsInsecureEnv) == "true" {
+		return &tls.Config{InsecureSkipVerify: true}, nil // #nosec G402 -- explicit opt-in for cameras with self-signed certificates
 	}
 
-	return &tls.Config{InsecureSkipVerify: true} // #nosec G402 -- explicit opt-in for cameras with self-signed certificates
+	caFile := os.Getenv(rtspsCAFileEnv)
+	if caFile == "" {
+		return nil, nil
+	}
+
+	rootCAs, err := x509.SystemCertPool()
+	if err != nil {
+		return nil, fmt.Errorf("load system CA pool: %w", err)
+	}
+	caPEM, err := os.ReadFile(caFile)
+	if err != nil {
+		return nil, fmt.Errorf("read RTSPS CA file %q: %w", caFile, err)
+	}
+	if !rootCAs.AppendCertsFromPEM(caPEM) {
+		return nil, fmt.Errorf("RTSPS CA file %q contains no certificates", caFile)
+	}
+
+	return &tls.Config{RootCAs: rootCAs}, nil
+}
+
+func sanitizeRTSPError(err error, rawURL string) error {
+	if err == nil {
+		return nil
+	}
+	if rawURL == "" || !strings.Contains(err.Error(), rawURL) {
+		return err
+	}
+
+	replacement := "<redacted-rtsp-url>"
+	if parsed, parseErr := url.Parse(rawURL); parseErr == nil {
+		parsed.User = nil
+		parsed.RawQuery = ""
+		parsed.Fragment = ""
+		replacement = parsed.String()
+	}
+	return errors.New(strings.ReplaceAll(err.Error(), rawURL, replacement))
 }
 
 // Implements the RTSPClient interface.
@@ -56,6 +98,7 @@ type Golibrtsp struct {
 	Url string
 
 	Client            gortsplib.Client
+	clientStarted     bool
 	VideoDecoderMutex *sync.Mutex
 
 	VideoH264Index        int8
@@ -176,14 +219,20 @@ func (h *streamHealth) observePacket(streamType string, writeDur time.Duration)
 		h.gapMax = gap
 	}
 	if writeDur >= streamHealthWriteWarn {
-		log.Log.Warning(fmt.Sprintf(
-			"capture.golibrtsp.health(%s): WritePacket blocked %dms — downstream back-pressure / CPU starvation",
-			streamType, writeDur.Milliseconds()))
+		log.WithFields(log.Fields{
+			"component":   "capture",
+			"duration_ms": writeDur.Milliseconds(),
+			"event":       "packet_queue_write_slow",
+			"stream":      streamType,
+		}).Warn("Packet queue write was slow")
 	}
 	if gap >= streamHealthGapWarn {
-		log.Log.Warning(fmt.Sprintf(
-			"capture.golibrtsp.health(%s): %dms since previous frame — upstream network / camera stall",
-			streamType, gap.Milliseconds()))
+		log.WithFields(log.Fields{
+			"component":    "capture",
+			"event":        "frame_gap_detected",
+			"frame_gap_ms": gap.Milliseconds(),
+			"stream":       streamType,
+		}).Warn("Large gap detected between camera frames")
 	}
 	if now.Sub(h.windowStart) >= streamHealthWindow {
 		elapsed := now.Sub(h.windowStart).Seconds()
@@ -191,10 +240,20 @@ func (h *streamHealth) observePacket(streamType string, writeDur time.Duration)
 		if h.frames > 0 {
 			avgWriteMs = float64(h.writeSum.Milliseconds()) / float64(h.frames)
 		}
-		log.Log.Info(fmt.Sprintf(
-			"capture.golibrtsp.health(%s): %.0fs window — frames=%d (%.1f/s) writeAvg=%.1fms writeMax=%dms gapMax=%dms lost=%d decodeErrs=%d",
-			streamType, elapsed, h.frames, float64(h.frames)/elapsed, avgWriteMs,
-			h.writeMax.Milliseconds(), h.gapMax.Milliseconds(), h.lost, h.decodeErrs))
+		log.WithFields(log.Fields{
+			"average_write_ms": avgWriteMs,
+			"component":        "capture",
+			"decode_errors":    h.decodeErrs,
+			"event":            "stream_health_sampled",
+			"frame_rate":       float64(h.frames) / elapsed,
+			"frames":           h.frames,
+			"lost_packets":     h.lost,
+			"max_frame_gap_ms": h.gapMax.Milliseconds(),
+			"max_write_ms":     h.writeMax.Milliseconds(),
+			"stream":           streamType,
+			"window_seconds":   elapsed,
+		}).Debug("RTSP stream health sampled")
+
 		h.windowStart = now
 		h.frames = 0
 		h.writeSum = 0
@@ -215,9 +274,12 @@ func (h *streamHealth) observeLost(streamType string, lost uint64) {
 	h.mu.Lock()
 	h.lost += lost
 	h.mu.Unlock()
-	log.Log.Warning(fmt.Sprintf(
-		"capture.golibrtsp.health(%s): %d RTP packet(s) lost — sender-side gap (receiver not draining TCP fast enough)",
-		streamType, lost))
+	log.WithFields(log.Fields{
+		"component":    "capture",
+		"event":        "rtp_packets_lost",
+		"lost_packets": lost,
+		"stream":       streamType,
+	}).Warn("RTSP sender reported lost RTP packets")
 }
 
 // observeDecodeError is invoked by gortsplib on incomplete/invalid access units,
@@ -229,7 +291,11 @@ func (h *streamHealth) observeDecodeError(streamType string, err error) {
 	h.mu.Lock()
 	h.decodeErrs++
 	h.mu.Unlock()
-	log.Log.Debug(fmt.Sprintf("capture.golibrtsp.health(%s): decode error: %s", streamType, err.Error()))
+	log.WithError(err).WithFields(log.Fields{
+		"component": "capture",
+		"event":     "access_unit_decode_failed",
+		"stream":    streamType,
+	}).Debug("Failed to decode RTSP access unit")
 }
 
 // fpsTracker holds per-stream state for PTS-based FPS calculation.
@@ -313,13 +379,13 @@ func init() {
 	// setup H264 -> raw frames decoder
 	H264FrameDecoder, err = newDecoder("H264")
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.init(): " + err.Error())
+		log.Error("capture.golibrtsp.init(): " + err.Error())
 	}
 
 	// setup H265 -> raw frames decoder
 	H265FrameDecoder, err = newDecoder("H265")
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.init(): " + err.Error())
+		log.Error("capture.golibrtsp.init(): " + err.Error())
 	}
 }
 
@@ -329,12 +395,17 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 	_, span := tracer.Start(ctxOtel, "Connect")
 	defer span.End()
 
+	tlsConfig, err := rtspsTLSConfig()
+	if err != nil {
+		return fmt.Errorf("configure RTSPS TLS: %w", err)
+	}
+
 	protocol := gortsplib.ProtocolTCP
 	g.health = newStreamHealth()
 	g.Client = gortsplib.Client{
 		RequestBackChannels: false,
 		Protocol:            &protocol,
-		TLSConfig:           rtspsTLSConfig(),
+		TLSConfig:           tlsConfig,
 		// Route gortsplib's packet-loss / decode-error reporting through our
 		// structured logger with stream context (replaces its plain stdout
 		// logging). These hooks are what let us tell whether the camera is
@@ -350,8 +421,7 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 	// parse URL
 	u, err := base.ParseURL(g.Url)
 	if err != nil {
-		log.Log.Debug("capture.golibrtsp.Connect(ParseURL): " + err.Error())
-		return
+		return sanitizeRTSPError(err, g.Url)
 	}
 
 	// connect to the server
@@ -359,14 +429,14 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 	g.Client.Host = u.Host
 	err = g.Client.Start()
 	if err != nil {
-		log.Log.Debug("capture.golibrtsp.Connect(Start): " + err.Error())
+		return sanitizeRTSPError(err, g.Url)
 	}
+	g.clientStarted = true
 
 	// find published medias
 	desc, _, err := g.Client.Describe(u)
 	if err != nil {
-		log.Log.Debug("capture.golibrtsp.Connect(Describe): " + err.Error())
-		return
+		return sanitizeRTSPError(err, g.Url)
 	}
 
 	// Initialize the mutex and FPS calculation.
@@ -379,13 +449,13 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 	g.VideoH264Media = mediH264
 	g.VideoH264Forma = formaH264
 	if mediH264 == nil {
-		log.Log.Debug("capture.golibrtsp.Connect(H264): " + "video media not found")
+		log.Debug("capture.golibrtsp.Connect(H264): " + "video media not found")
 	} else {
 		// setup a video media
 		_, err = g.Client.Setup(desc.BaseURL, mediH264, 0, 0)
 		if err != nil {
 			// Something went wrong .. Do something
-			log.Log.Error("capture.golibrtsp.Connect(H264): " + err.Error())
+			log.Error("capture.golibrtsp.Connect(H264): " + err.Error())
 		} else {
 			// Get SPS and PPS from the SDP
 			// Calculate the width and height of the video
@@ -394,7 +464,7 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 			// It might be that the SPS is not available yet, so we'll proceed,
 			// but try to fetch it later on.
 			if errSPS != nil {
-				log.Log.Debug("capture.golibrtsp.Connect(H264): " + errSPS.Error())
+				log.Debug("capture.golibrtsp.Connect(H264): " + errSPS.Error())
 				streamIndex := len(g.Streams)
 				g.Streams = append(g.Streams, packets.Stream{
 					Index:         streamIndex,
@@ -430,7 +500,7 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 			// setup RTP/H264 -> H264 decoder
 			rtpDec, err := formaH264.CreateDecoder()
 			if err != nil {
-				log.Log.Error("capture.golibrtsp.Connect(H264): " + err.Error())
+				log.Error("capture.golibrtsp.Connect(H264): " + err.Error())
 			}
 			g.VideoH264Decoder = rtpDec
 			g.VideoH264FrameDecoder = H264FrameDecoder
@@ -443,20 +513,20 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 	g.VideoH265Media = mediH265
 	g.VideoH265Forma = formaH265
 	if mediH265 == nil {
-		log.Log.Debug("capture.golibrtsp.Connect(H265): " + "video media not found")
+		log.Debug("capture.golibrtsp.Connect(H265): " + "video media not found")
 	} else {
 		// setup a video media
 		_, err = g.Client.Setup(desc.BaseURL, mediH265, 0, 0)
 		if err != nil {
 			// Something went wrong .. Do something
-			log.Log.Error("capture.golibrtsp.Connect(H265): " + err.Error())
+			log.Error("capture.golibrtsp.Connect(H265): " + err.Error())
 		} else {
 			// Get SPS from the SDP
 			// Calculate the width and height of the video
 			var sps h265.SPS
 			err = sps.Unmarshal(formaH265.SPS)
 			if err != nil {
-				log.Log.Info("capture.golibrtsp.Connect(H265): " + err.Error())
+				log.Info("capture.golibrtsp.Connect(H265): " + err.Error())
 				return
 			}
 			streamIndex := len(g.Streams)
@@ -480,7 +550,7 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 			// setup RTP/H265 -> H265 decoder
 			rtpDec, err := formaH265.CreateDecoder()
 			if err != nil {
-				log.Log.Error("capture.golibrtsp.Connect(H265): " + err.Error())
+				log.Error("capture.golibrtsp.Connect(H265): " + err.Error())
 			}
 			g.VideoH265Decoder = rtpDec
 
@@ -494,19 +564,19 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 	g.AudioG711Media = audioMedi
 	g.AudioG711Forma = audioForma
 	if audioMedi == nil {
-		log.Log.Debug("capture.golibrtsp.Connect(G711): " + "audio media not found")
+		log.Debug("capture.golibrtsp.Connect(G711): " + "audio media not found")
 	} else {
 		// setup a audio media
 		_, err = g.Client.Setup(desc.BaseURL, audioMedi, 0, 0)
 		if err != nil {
 			// Something went wrong .. Do something
-			log.Log.Error("capture.golibrtsp.Connect(G711): " + err.Error())
+			log.Error("capture.golibrtsp.Connect(G711): " + err.Error())
 		} else {
 			// create decoder
 			audiortpDec, err := audioForma.CreateDecoder()
 			if err != nil {
 				// Something went wrong .. Do something
-				log.Log.Error("capture.golibrtsp.Connect(G711): " + err.Error())
+				log.Error("capture.golibrtsp.Connect(G711): " + err.Error())
 			} else {
 				g.AudioG711Decoder = audiortpDec
 				streamIndex := len(g.Streams)
@@ -530,19 +600,19 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 	g.AudioOpusMedia = audioMediOpus
 	g.AudioOpusForma = audioFormaOpus
 	if audioMediOpus == nil {
-		log.Log.Debug("capture.golibrtsp.Connect(Opus): " + "audio media not found")
+		log.Debug("capture.golibrtsp.Connect(Opus): " + "audio media not found")
 	} else {
 		// setup a audio media
 		_, err = g.Client.Setup(desc.BaseURL, audioMediOpus, 0, 0)
 		if err != nil {
 			// Something went wrong .. Do something
-			log.Log.Error("capture.golibrtsp.Connect(Opus): " + err.Error())
+			log.Error("capture.golibrtsp.Connect(Opus): " + err.Error())
 		} else {
 			// create decoder
 			audiortpDec, err := audioFormaOpus.CreateDecoder()
 			if err != nil {
 				// Something went wrong .. Do something
-				log.Log.Error("capture.golibrtsp.Connect(Opus): " + err.Error())
+				log.Error("capture.golibrtsp.Connect(Opus): " + err.Error())
 			} else {
 				g.AudioOpusDecoder = audiortpDec
 				streamIndex := len(g.Streams)
@@ -566,13 +636,13 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 	g.AudioMPEG4Media = audioMediMPEG4
 	g.AudioMPEG4Forma = audioFormaMPEG4
 	if audioMediMPEG4 == nil {
-		log.Log.Debug("capture.golibrtsp.Connect(MPEG4): " + "audio media not found")
+		log.Debug("capture.golibrtsp.Connect(MPEG4): " + "audio media not found")
 	} else {
 		// setup a audio media
 		_, err = g.Client.Setup(desc.BaseURL, audioMediMPEG4, 0, 0)
 		if err != nil {
 			// Something went wrong .. Do something
-			log.Log.Error("capture.golibrtsp.Connect(MPEG4): " + err.Error())
+			log.Error("capture.golibrtsp.Connect(MPEG4): " + err.Error())
 		} else {
 			streamIndex := len(g.Streams)
 			g.Streams = append(g.Streams, packets.Stream{
@@ -592,7 +662,7 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 			audiortpDec, err := audioFormaMPEG4.CreateDecoder()
 			if err != nil {
 				// Something went wrong .. Do something
-				log.Log.Error("capture.golibrtsp.Connect(MPEG4): " + err.Error())
+				log.Error("capture.golibrtsp.Connect(MPEG4): " + err.Error())
 			}
 			g.AudioMPEG4Decoder = audiortpDec
 
@@ -607,18 +677,22 @@ func (g *Golibrtsp) ConnectBackChannel(ctx context.Context, ctxRunAgent context.
 	_, span := tracer.Start(ctxRunAgent, "ConnectBackChannel")
 	defer span.End()
 
+	tlsConfig, err := rtspsTLSConfig()
+	if err != nil {
+		return fmt.Errorf("configure RTSPS TLS: %w", err)
+	}
+
 	// Transport TCP
 	protocol := gortsplib.ProtocolTCP
 	g.Client = gortsplib.Client{
 		RequestBackChannels: true,
 		Protocol:            &protocol,
-		TLSConfig:           rtspsTLSConfig(),
+		TLSConfig:           tlsConfig,
 	}
 	// parse URL
 	u, err := base.ParseURL(g.Url)
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.ConnectBackChannel(): " + err.Error())
-		return
+		return sanitizeRTSPError(err, g.Url)
 	}
 
 	// connect to the server
@@ -626,14 +700,14 @@ func (g *Golibrtsp) ConnectBackChannel(ctx context.Context, ctxRunAgent context.
 	g.Client.Host = u.Host
 	err = g.Client.Start()
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.ConnectBackChannel(): " + err.Error())
+		return sanitizeRTSPError(err, g.Url)
 	}
+	g.clientStarted = true
 
 	// find published medias
 	desc, _, err := g.Client.Describe(u)
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.ConnectBackChannel(): " + err.Error())
-		return
+		return sanitizeRTSPError(err, g.Url)
 	}
 
 	// Look for audio back channel.
@@ -643,15 +717,17 @@ func (g *Golibrtsp) ConnectBackChannel(ctx context.Context, ctxRunAgent context.
 	g.AudioG711MediaBackChannel = audioMediBackChannel
 	g.AudioG711FormaBackChannel = audioFormaBackChannel
 	if audioMediBackChannel == nil {
-		log.Log.Error("capture.golibrtsp.ConnectBackChannel(): audio backchannel not found, not a real error, however you might expect a backchannel. One of the reasons might be that the device already has an active client connected to the backchannel.")
+		log.WithFields(log.Fields{
+			"component": "capture",
+			"event":     "backchannel_unavailable",
+		}).Debug("Optional camera audio backchannel unavailable")
 		err = errors.New("no audio backchannel found")
 	} else {
 		// setup a audio media
 		_, err = g.Client.Setup(desc.BaseURL, audioMediBackChannel, 0, 0)
 		if err != nil {
-			// Something went wrong .. Do something
-			log.Log.Error("capture.golibrtsp.ConnectBackChannel(): " + err.Error())
 			g.HasBackChannel = false
+			return sanitizeRTSPError(err, g.Url)
 		} else {
 			g.HasBackChannel = true
 			streamIndex := len(g.Streams)
@@ -704,9 +780,26 @@ func ptsToDuration(pts int64, clockRate int) time.Duration {
 		time.Duration(pts%rate)*time.Second/time.Duration(rate)
 }
 
+func preRecordingGOPCount(preRecording int64, gopDuration float64) (int, bool) {
+	if preRecording <= 0 ||
+		gopDuration < 1 ||
+		math.IsNaN(gopDuration) ||
+		math.IsInf(gopDuration, 0) ||
+		gopDuration >= float64(math.MaxInt64) {
+		return 0, false
+	}
+
+	count := preRecording / int64(gopDuration)
+	maxInt := int64(^uint(0) >> 1)
+	if count >= maxInt {
+		return 0, false
+	}
+	return int(count) + 1, true
+}
+
 // Start the RTSP client, and start reading packets.
 func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets.Queue, configuration *models.Configuration, communication *models.Communication) (err error) {
-	log.Log.Debug("capture.golibrtsp.Start(): started")
+	log.Debug("capture.golibrtsp.Start(): started")
 
 	// Label this client's loss/decode/health logging with the stream type.
 	g.streamLabel = streamType
@@ -720,7 +813,7 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 			// decode timestamp
 			pts2, ok := g.Client.PacketPTS(g.AudioG711Media, rtppkt)
 			if !ok {
-				log.Log.Debug("capture.golibrtsp.Start(): " + "unable to get PTS")
+				log.Debug("capture.golibrtsp.Start(): " + "unable to get PTS")
 				return
 			}
 			pts := ptsToDuration(pts2, g.AudioG711Forma.ClockRate())
@@ -728,7 +821,7 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 			// extract LPCM samples from RTP packets
 			op, err := g.AudioG711Decoder.Decode(rtppkt)
 			if err != nil {
-				log.Log.Error("capture.golibrtsp.Start(): " + err.Error())
+				log.Error("capture.golibrtsp.Start(): " + err.Error())
 				return
 			}
 
@@ -755,7 +848,7 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 			// decode timestamp
 			pts2, ok := g.Client.PacketPTS(g.AudioMPEG4Media, rtppkt)
 			if !ok {
-				log.Log.Error("capture.golibrtsp.Start(): " + "unable to get PTS")
+				log.Error("capture.golibrtsp.Start(): " + "unable to get PTS")
 				return
 			}
 			pts := ptsToDuration(pts2, g.AudioMPEG4Forma.ClockRate())
@@ -764,13 +857,13 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 			// extract access units from RTP packets
 			aus, err := g.AudioMPEG4Decoder.Decode(rtppkt)
 			if err != nil {
-				log.Log.Error("capture.golibrtsp.Start(): " + err.Error())
+				log.Error("capture.golibrtsp.Start(): " + err.Error())
 				return
 			}
 
 			enc, err := WriteMPEG4Audio(g.AudioMPEG4Forma, aus)
 			if err != nil {
-				log.Log.Error("capture.golibrtsp.Start(): " + err.Error())
+				log.Error("capture.golibrtsp.Start(): " + err.Error())
 				return
 			}
 
@@ -814,7 +907,7 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 				// decode timestamps — validate each call separately
 				pts2, okPTS2 := g.Client.PacketPTS(g.VideoH264Media, rtppkt)
 				if !okPTS2 {
-					log.Log.Debug("capture.golibrtsp.Start(): unable to get PTS")
+					log.Debug("capture.golibrtsp.Start(): unable to get PTS")
 					return
 				}
 				pts := ptsToDuration(pts2, g.VideoH264Forma.ClockRate())
@@ -825,7 +918,7 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 				au, errDecode := g.VideoH264Decoder.Decode(rtppkt)
 				if errDecode != nil {
 					if errDecode != rtph264.ErrNonStartingPacketAndNoPrevious && errDecode != rtph264.ErrMorePacketsNeeded {
-						log.Log.Error("capture.golibrtsp.Start(): " + errDecode.Error())
+						log.Error("capture.golibrtsp.Start(): " + errDecode.Error())
 					}
 					return
 				}
@@ -887,7 +980,12 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 							fps := g.getEnhancedFPS(&sps, g.VideoH264Index)
 							g.Streams[g.VideoH264Index].FPS = fps
 							g.persistStreamFPS(configuration, streamType, fps)
-							log.Log.Debug(fmt.Sprintf("capture.golibrtsp.Start(%s): Final FPS=%.2f", streamType, fps))
+							log.WithFields(log.Fields{
+								"component": "capture",
+								"event":     "frame_rate_detected",
+								"fps":       fps,
+								"stream":    streamType,
+							}).Debug("RTSP frame rate detected")
 							g.VideoH264Forma.SPS = nalu
 							if streamType == "main" && len(nalu) > 0 {
 								// Fallback: store SPS from in-band NALUs when SDP was missing it.
@@ -909,14 +1007,14 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 					// Ensure config has parameter sets before recordings start.
 					if len(configuration.Config.Capture.IPCamera.SPSNALUs) == 0 && len(g.VideoH264Forma.SPS) > 0 {
 						configuration.Config.Capture.IPCamera.SPSNALUs = [][]byte{g.VideoH264Forma.SPS}
-						log.Log.Warning("capture.golibrtsp.Start(main): fallback SPS set from keyframe")
+						log.Warn("capture.golibrtsp.Start(main): fallback SPS set from keyframe")
 					}
 					if len(configuration.Config.Capture.IPCamera.PPSNALUs) == 0 && len(g.VideoH264Forma.PPS) > 0 {
 						configuration.Config.Capture.IPCamera.PPSNALUs = [][]byte{g.VideoH264Forma.PPS}
-						log.Log.Warning("capture.golibrtsp.Start(main): fallback PPS set from keyframe")
+						log.Warn("capture.golibrtsp.Start(main): fallback PPS set from keyframe")
 					}
 					if len(configuration.Config.Capture.IPCamera.SPSNALUs) == 0 || len(configuration.Config.Capture.IPCamera.PPSNALUs) == 0 {
-						log.Log.Warning("capture.golibrtsp.Start(main): SPS/PPS still missing after IDR keyframe")
+						log.Warn("capture.golibrtsp.Start(main): SPS/PPS still missing after IDR keyframe")
 					}
 				}
 
@@ -925,14 +1023,18 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 				}
 
 				if idrPresent {
-					log.Log.Debug(fmt.Sprintf("capture.golibrtsp.Start(%s): IDR frame NALUs: [%s]",
-						streamType, fmt.Sprintf("%v", naluTypes)))
+					log.WithFields(log.Fields{
+						"component":  "capture",
+						"event":      "idr_frame_received",
+						"nalu_types": naluTypes,
+						"stream":     streamType,
+					}).Debug("RTSP IDR frame received")
 				}
 
 				// Convert to packet.
 				enc, err := h264.AnnexBMarshal(filteredAU)
 				if err != nil {
-					log.Log.Error("capture.golibrtsp.Start(): " + err.Error())
+					log.Error("capture.golibrtsp.Start(): " + err.Error())
 					return
 				}
 
@@ -966,11 +1068,19 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 					gopDuration := float64(keyframeInterval) / fps
 					gopSize := int(avgInterval) // Store GOP size in a separate variable
 					g.Streams[g.VideoH264Index].GopSize = gopSize
-					log.Log.Debug(fmt.Sprintf("capture.golibrtsp.Start(%s): Keyframe interval=%d packets, Avg=%.1f, GOP=%.1fs, GOPSize=%d",
-						streamType, keyframeInterval, avgInterval, gopDuration, gopSize))
+					log.WithFields(log.Fields{
+						"average_keyframe_interval_packets": avgInterval,
+						"component":                         "capture",
+						"event":                             "keyframe_interval_observed",
+						"gop_duration_seconds":              gopDuration,
+						"gop_size_packets":                  gopSize,
+						"keyframe_interval_packets":         keyframeInterval,
+						"stream":                            streamType,
+					}).Debug("RTSP keyframe interval observed")
+
 					preRecording := configuration.Config.Capture.PreRecording
-					if preRecording > 0 && int(gopDuration) > 0 {
-						queue.SetMaxGopCount(int(preRecording)/int(gopDuration) + 1)
+					if maxGOPCount, ok := preRecordingGOPCount(preRecording, gopDuration); ok {
+						queue.SetMaxGopCount(maxGOPCount)
 					}
 				}
 
@@ -1017,20 +1127,36 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 				default:
 				}
 
-				if idrPresent {
-					// Increment packets, so we know the device
-					// is not blocking.
-					if streamType == "main" {
-						r := communication.PackageCounter.Load().(int64)
-						log.Log.Debug("capture.golibrtsp.Start(): packet size " + strconv.Itoa(len(pkt.Data)))
-						communication.PackageCounter.Store((r + 1) % 1000)
-						communication.LastPacketTimer.Store(time.Now().Unix())
-					} else if streamType == "sub" {
-						r := communication.PackageCounterSub.Load().(int64)
-						log.Log.Debug("capture.golibrtsp.Start(): packet size " + strconv.Itoa(len(pkt.Data)))
-						communication.PackageCounterSub.Store((r + 1) % 1000)
-						communication.LastPacketTimerSub.Store(time.Now().Unix())
-					}
+				// Count every complete video access unit. Keyframe-only counters make
+				// healthy cameras with GOPs longer than the watchdog window look stalled.
+				if streamType == "main" {
+					observedAt := time.Now()
+					r := communication.PackageCounter.Load().(int64)
+					log.WithFields(log.Fields{
+						"bytes":     len(pkt.Data),
+						"codec":     "H264",
+						"component": "capture",
+						"event":     "access_unit_received",
+						"keyframe":  pkt.IsKeyFrame,
+						"stream":    streamType,
+					}).Trace("RTSP access unit received")
+					communication.RecordStreamPackage(models.MainStream, g.Streams[g.VideoH264Index].FPS, g.Streams[g.VideoH264Index].Width, g.Streams[g.VideoH264Index].Height, observedAt)
+					communication.PackageCounter.Store((r + 1) % 1000)
+					communication.LastPacketTimer.Store(observedAt.Unix())
+				} else if streamType == "sub" {
+					observedAt := time.Now()
+					r := communication.PackageCounterSub.Load().(int64)
+					log.WithFields(log.Fields{
+						"bytes":     len(pkt.Data),
+						"codec":     "H264",
+						"component": "capture",
+						"event":     "access_unit_received",
+						"keyframe":  pkt.IsKeyFrame,
+						"stream":    streamType,
+					}).Trace("RTSP access unit received")
+					communication.RecordStreamPackage(models.SubStream, g.Streams[g.VideoH264Index].FPS, g.Streams[g.VideoH264Index].Width, g.Streams[g.VideoH264Index].Height, observedAt)
+					communication.PackageCounterSub.Store((r + 1) % 1000)
+					communication.LastPacketTimerSub.Store(observedAt.Unix())
 				}
 			}
 
@@ -1059,7 +1185,7 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 				// decode timestamps — validate each call separately
 				pts2, okPTS2 := g.Client.PacketPTS(g.VideoH265Media, rtppkt)
 				if !okPTS2 {
-					log.Log.Debug("capture.golibrtsp.Start(): unable to get PTS")
+					log.Debug("capture.golibrtsp.Start(): unable to get PTS")
 					return
 				}
 				pts := ptsToDuration(pts2, g.VideoH265Forma.ClockRate())
@@ -1070,7 +1196,7 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 				au, errDecode := g.VideoH265Decoder.Decode(rtppkt)
 				if errDecode != nil {
 					if errDecode != rtph265.ErrNonStartingPacketAndNoPrevious && errDecode != rtph265.ErrMorePacketsNeeded {
-						log.Log.Error("capture.golibrtsp.Start(): " + errDecode.Error())
+						log.Error("capture.golibrtsp.Start(): " + errDecode.Error())
 					}
 					return
 				}
@@ -1128,7 +1254,7 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 
 				enc, err := h264.AnnexBMarshal(au)
 				if err != nil {
-					log.Log.Error("capture.golibrtsp.Start(): " + err.Error())
+					log.Error("capture.golibrtsp.Start(): " + err.Error())
 					return
 				}
 
@@ -1160,11 +1286,19 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 					gopDuration := float64(keyframeInterval) / fps
 					gopSize := int(avgInterval) // Store GOP size in a separate variable
 					g.Streams[g.VideoH265Index].GopSize = gopSize
-					log.Log.Debug(fmt.Sprintf("capture.golibrtsp.Start(%s): Keyframe interval=%d packets, Avg=%.1f, GOP=%.1fs, GOPSize=%d",
-						streamType, keyframeInterval, avgInterval, gopDuration, gopSize))
+					log.WithFields(log.Fields{
+						"average_keyframe_interval_packets": avgInterval,
+						"component":                         "capture",
+						"event":                             "keyframe_interval_observed",
+						"gop_duration_seconds":              gopDuration,
+						"gop_size_packets":                  gopSize,
+						"keyframe_interval_packets":         keyframeInterval,
+						"stream":                            streamType,
+					}).Debug("RTSP keyframe interval observed")
+
 					preRecording := configuration.Config.Capture.PreRecording
-					if preRecording > 0 && int(gopDuration) > 0 {
-						queue.SetMaxGopCount(int(preRecording)/int(gopDuration) + 1)
+					if maxGOPCount, ok := preRecordingGOPCount(preRecording, gopDuration); ok {
+						queue.SetMaxGopCount(maxGOPCount)
 					}
 				}
 
@@ -1182,20 +1316,36 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 				default:
 				}
 
-				if isRandomAccess {
-					// Increment packets, so we know the device
-					// is not blocking.
-					if streamType == "main" {
-						r := communication.PackageCounter.Load().(int64)
-						log.Log.Debug("capture.golibrtsp.Start(): packet size " + strconv.Itoa(len(pkt.Data)))
-						communication.PackageCounter.Store((r + 1) % 1000)
-						communication.LastPacketTimer.Store(time.Now().Unix())
-					} else if streamType == "sub" {
-						r := communication.PackageCounterSub.Load().(int64)
-						log.Log.Debug("capture.golibrtsp.Start(): packet size " + strconv.Itoa(len(pkt.Data)))
-						communication.PackageCounterSub.Store((r + 1) % 1000)
-						communication.LastPacketTimerSub.Store(time.Now().Unix())
-					}
+				// Count every complete video access unit; random-access frames remain
+				// responsible only for GOP tracking above.
+				if streamType == "main" {
+					observedAt := time.Now()
+					r := communication.PackageCounter.Load().(int64)
+					log.WithFields(log.Fields{
+						"bytes":     len(pkt.Data),
+						"codec":     "H265",
+						"component": "capture",
+						"event":     "access_unit_received",
+						"keyframe":  pkt.IsKeyFrame,
+						"stream":    streamType,
+					}).Trace("RTSP access unit received")
+					communication.RecordStreamPackage(models.MainStream, g.Streams[g.VideoH265Index].FPS, g.Streams[g.VideoH265Index].Width, g.Streams[g.VideoH265Index].Height, observedAt)
+					communication.PackageCounter.Store((r + 1) % 1000)
+					communication.LastPacketTimer.Store(observedAt.Unix())
+				} else if streamType == "sub" {
+					observedAt := time.Now()
+					r := communication.PackageCounterSub.Load().(int64)
+					log.WithFields(log.Fields{
+						"bytes":     len(pkt.Data),
+						"codec":     "H265",
+						"component": "capture",
+						"event":     "access_unit_received",
+						"keyframe":  pkt.IsKeyFrame,
+						"stream":    streamType,
+					}).Trace("RTSP access unit received")
+					communication.RecordStreamPackage(models.SubStream, g.Streams[g.VideoH265Index].FPS, g.Streams[g.VideoH265Index].Width, g.Streams[g.VideoH265Index].Height, observedAt)
+					communication.PackageCounterSub.Store((r + 1) % 1000)
+					communication.LastPacketTimerSub.Store(observedAt.Unix())
 				}
 			}
 
@@ -1207,7 +1357,7 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 	// Play the stream.
 	_, err = g.Client.Play(nil)
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.Start(): " + err.Error())
+		log.Error("capture.golibrtsp.Start(): " + err.Error())
 	}
 
 	return
@@ -1215,13 +1365,13 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 
 // Start the RTSP client, and start reading packets.
 func (g *Golibrtsp) StartBackChannel(ctx context.Context, ctxRunAgent context.Context) (err error) {
-	log.Log.Info("capture.golibrtsp.StartBackChannel(): started")
+	log.Info("capture.golibrtsp.StartBackChannel(): started")
 	// Wait for a second, so we can be sure the stream is playing.
 	time.Sleep(1 * time.Second)
 	// Play the stream.
 	_, err = g.Client.Play(nil)
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.StartBackChannel(): " + err.Error())
+		log.Error("capture.golibrtsp.StartBackChannel(): " + err.Error())
 	}
 	return
 }
@@ -1230,7 +1380,7 @@ func (g *Golibrtsp) WritePacket(pkt packets.Packet) error {
 	if g.HasBackChannel && g.AudioG711MediaBackChannel != nil {
 		err := g.Client.WritePacketRTP(g.AudioG711MediaBackChannel, pkt.Packet)
 		if err != nil {
-			log.Log.Debug("capture.golibrtsp.WritePacket(): " + err.Error())
+			log.Debug("capture.golibrtsp.WritePacket(): " + err.Error())
 			return err
 		}
 	}
@@ -1253,11 +1403,11 @@ func (g *Golibrtsp) DecodePacket(pkt packets.Packet) (image.YCbCr, error) {
 	}
 	g.VideoDecoderMutex.Unlock()
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.DecodePacket(): " + err.Error())
+		log.Error("capture.golibrtsp.DecodePacket(): " + err.Error())
 		return image.YCbCr{}, err
 	}
 	if img.Bounds().Empty() {
-		log.Log.Debug("capture.golibrtsp.DecodePacket(): empty frame")
+		log.Debug("capture.golibrtsp.DecodePacket(): empty frame")
 		return image.YCbCr{}, errors.New("Empty image")
 	}
 	return img, nil
@@ -1279,11 +1429,11 @@ func (g *Golibrtsp) DecodePacketRaw(pkt packets.Packet) (image.Gray, error) {
 	}
 	g.VideoDecoderMutex.Unlock()
 	if err != nil {
-		log.Log.Error("capture.golibrtsp.DecodePacketRaw(): " + err.Error())
+		log.Error("capture.golibrtsp.DecodePacketRaw(): " + err.Error())
 		return image.Gray{}, err
 	}
 	if img.Bounds().Empty() {
-		log.Log.Debug("capture.golibrtsp.DecodePacketRaw(): empty image")
+		log.Debug("capture.golibrtsp.DecodePacketRaw(): empty image")
 		return image.Gray{}, errors.New("Empty image")
 	}
 
@@ -1328,7 +1478,11 @@ func (g *Golibrtsp) Close(ctxOtel context.Context) error {
 	_, span := tracer.Start(ctxOtel, "Close")
 	defer span.End()
 
-	// Close the demuxer.
+	if !g.clientStarted {
+		return nil
+	}
+
+	g.clientStarted = false
 	g.Client.Close()
 
 	// We will have created the decoders globally, so we don't need to close them here.
@@ -1373,13 +1527,13 @@ func newDecoder(codecName string) (*Decoder, error) {
 
 	res := C.avcodec_open2(codecCtx, codec, nil)
 	if res < 0 {
-		C.avcodec_close(codecCtx)
+		C.avcodec_free_context(&codecCtx)
 		return nil, fmt.Errorf("avcodec_open2() failed")
 	}
 
 	srcFrame := C.av_frame_alloc()
 	if srcFrame == nil {
-		C.avcodec_close(codecCtx)
+		C.avcodec_free_context(&codecCtx)
 		return nil, fmt.Errorf("av_frame_alloc() failed")
 	}
 
@@ -1395,7 +1549,7 @@ func (d *Decoder) Close() {
 		C.av_frame_free(&d.srcFrame)
 	}
 	C.av_frame_free(&d.srcFrame)
-	C.avcodec_close(d.codecCtx)
+	C.avcodec_free_context(&d.codecCtx)
 }
 
 func (d *Decoder) decode(nalu []byte) (image.YCbCr, error) {
@@ -1582,7 +1736,12 @@ func (g *Golibrtsp) getEnhancedFPS(sps *h264.SPS, streamIndex int8) float64 {
 
 	// Check if SPS FPS is reasonable (between 1 and 120 fps)
 	if spsFPS > 0 && spsFPS <= 120 {
-		log.Log.Debug(fmt.Sprintf("capture.golibrtsp.getEnhancedFPS(): SPS FPS: %.2f", spsFPS))
+		log.WithFields(log.Fields{
+			"component": "capture",
+			"event":     "frame_rate_selected",
+			"fps":       spsFPS,
+			"source":    "sps",
+		}).Debug("RTSP frame rate selected")
 		return spsFPS
 	}
 
@@ -1590,7 +1749,12 @@ func (g *Golibrtsp) getEnhancedFPS(sps *h264.SPS, streamIndex int8) float64 {
 	if ft := g.fpsTrackers[streamIndex]; ft != nil {
 		ptsFPS := ft.fps()
 		if ptsFPS > 0 && ptsFPS <= 120 {
-			log.Log.Debug(fmt.Sprintf("capture.golibrtsp.getEnhancedFPS(): PTS FPS: %.2f", ptsFPS))
+			log.WithFields(log.Fields{
+				"component": "capture",
+				"event":     "frame_rate_selected",
+				"fps":       ptsFPS,
+				"source":    "presentation_timestamp",
+			}).Debug("RTSP frame rate selected")
 			return ptsFPS
 		}
 	}
@@ -1692,13 +1856,15 @@ func (g *Golibrtsp) getSPSTimingInfo(sps *h264.SPS) (hasVUI bool, timeScale uint
 func (g *Golibrtsp) debugSPSInfo(sps *h264.SPS, streamType string) {
 	hasVUI, timeScale, numUnitsInTick, fps := g.getSPSTimingInfo(sps)
 
-	log.Log.Debug(fmt.Sprintf("capture.golibrtsp.debugSPSInfo(%s): Width=%d, Height=%d",
-		streamType, sps.Width(), sps.Height()))
-	log.Log.Debug(fmt.Sprintf("capture.golibrtsp.debugSPSInfo(%s): HasVUI=%t, FPS=%.2f",
-		streamType, hasVUI, fps))
-
-	if hasVUI {
-		log.Log.Debug(fmt.Sprintf("capture.golibrtsp.debugSPSInfo(%s): TimeScale=%d, NumUnitsInTick=%d",
-			streamType, timeScale, numUnitsInTick))
-	}
+	log.WithFields(log.Fields{
+		"component":         "capture",
+		"event":             "sps_inspected",
+		"fps":               fps,
+		"has_vui":           hasVUI,
+		"height_pixels":     sps.Height(),
+		"num_units_in_tick": numUnitsInTick,
+		"stream":            streamType,
+		"time_scale":        timeScale,
+		"width_pixels":      sps.Width(),
+	}).Debug("RTSP sequence parameter set inspected")
 }

machinery/src/capture/gortsplib_test.go

@@ -1,22 +1,136 @@
 package capture
 
-import "testing"
+import (
+	"bytes"
+	"context"
+	"encoding/pem"
+	"errors"
+	"math"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestPreRecordingGOPCount(t *testing.T) {
+	maxInt := int64(^uint(0) >> 1)
+	tests := []struct {
+		name         string
+		preRecording int64
+		gopDuration  float64
+		want         int
+		wantOK       bool
+	}{
+		{name: "normal duration", preRecording: 10, gopDuration: 2.9, want: 6, wantOK: true},
+		{name: "duration longer than buffer", preRecording: 1, gopDuration: 2, want: 1, wantOK: true},
+		{name: "largest representable result", preRecording: maxInt - 1, gopDuration: 1, want: int(maxInt), wantOK: true},
+		{name: "result exceeds int", preRecording: maxInt, gopDuration: 1, wantOK: false},
+		{name: "non-positive pre-recording", preRecording: 0, gopDuration: 1, wantOK: false},
+		{name: "sub-second GOP", preRecording: 10, gopDuration: 0.9, wantOK: false},
+		{name: "NaN GOP", preRecording: 10, gopDuration: math.NaN(), wantOK: false},
+		{name: "infinite GOP", preRecording: 10, gopDuration: math.Inf(1), wantOK: false},
+		{name: "GOP exceeds int64", preRecording: 10, gopDuration: float64(math.MaxInt64), wantOK: false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, ok := preRecordingGOPCount(tt.preRecording, tt.gopDuration)
+			if ok != tt.wantOK || got != tt.want {
+				t.Fatalf("preRecordingGOPCount(%d, %v) = (%d, %t), want (%d, %t)",
+					tt.preRecording, tt.gopDuration, got, ok, tt.want, tt.wantOK)
+			}
+		})
+	}
+}
+
+func TestGolibrtspCloseBeforeClientStart(t *testing.T) {
+	client := &Golibrtsp{}
+
+	if err := client.Close(context.Background()); err != nil {
+		t.Fatalf("Close() error = %v", err)
+	}
+}
+
+func TestSanitizeRTSPErrorRemovesCredentialsAndQuery(t *testing.T) {
+	rawURL := "rtsp://camera-user:camera-password@10.0.20.15/live?access_token=secret"
+	got := sanitizeRTSPError(errors.New("describe "+rawURL+": bad status code"), rawURL)
+
+	for _, secret := range []string{"camera-user", "camera-password", "access_token", "secret"} {
+		if strings.Contains(got.Error(), secret) {
+			t.Fatalf("sanitizeRTSPError() exposed %q in %q", secret, got)
+		}
+	}
+	if !strings.Contains(got.Error(), "rtsp://10.0.20.15/live") {
+		t.Fatalf("sanitizeRTSPError() removed useful host/path context: %q", got)
+	}
+}
 
 func TestRTSPSTLSConfig(t *testing.T) {
 	t.Run("verifies certificates by default", func(t *testing.T) {
+		t.Setenv(rtspsCAFileEnv, "")
 		t.Setenv(rtspsInsecureEnv, "")
 
-		if got := rtspsTLSConfig(); got != nil {
+		got, err := rtspsTLSConfig()
+		if err != nil {
+			t.Fatalf("rtspsTLSConfig() error = %v", err)
+		}
+		if got != nil {
 			t.Fatalf("rtspsTLSConfig() = %#v, want nil", got)
 		}
 	})
 
 	t.Run("allows explicit insecure mode", func(t *testing.T) {
+		t.Setenv(rtspsCAFileEnv, "/missing/ignored-in-insecure-mode.pem")
 		t.Setenv(rtspsInsecureEnv, "true")
 
-		got := rtspsTLSConfig()
+		got, err := rtspsTLSConfig()
+		if err != nil {
+			t.Fatalf("rtspsTLSConfig() error = %v", err)
+		}
 		if got == nil || !got.InsecureSkipVerify {
 			t.Fatalf("rtspsTLSConfig() = %#v, want InsecureSkipVerify enabled", got)
 		}
 	})
+
+	t.Run("adds a camera CA to system roots", func(t *testing.T) {
+		t.Setenv(rtspsInsecureEnv, "")
+		server := httptest.NewTLSServer(nil)
+		defer server.Close()
+
+		certificate := server.Certificate()
+		caFile := filepath.Join(t.TempDir(), "camera-ca.pem")
+		caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certificate.Raw})
+		if err := os.WriteFile(caFile, caPEM, 0o600); err != nil {
+			t.Fatal(err)
+		}
+		t.Setenv(rtspsCAFileEnv, caFile)
+
+		got, err := rtspsTLSConfig()
+		if err != nil {
+			t.Fatalf("rtspsTLSConfig() error = %v", err)
+		}
+		if got == nil || got.RootCAs == nil {
+			t.Fatalf("rtspsTLSConfig() = %#v, want custom RootCAs", got)
+		}
+		for _, subject := range got.RootCAs.Subjects() {
+			if bytes.Equal(subject, certificate.RawSubject) {
+				return
+			}
+		}
+		t.Fatal("camera CA was not added to RootCAs")
+	})
+
+	t.Run("rejects an invalid camera CA file", func(t *testing.T) {
+		t.Setenv(rtspsInsecureEnv, "")
+		caFile := filepath.Join(t.TempDir(), "camera-ca.pem")
+		if err := os.WriteFile(caFile, []byte("not a certificate"), 0o600); err != nil {
+			t.Fatal(err)
+		}
+		t.Setenv(rtspsCAFileEnv, caFile)
+
+		if _, err := rtspsTLSConfig(); err == nil {
+			t.Fatal("rtspsTLSConfig() error = nil, want invalid CA error")
+		}
+	})
 }

machinery/src/capture/main.go

@@ -16,11 +16,11 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/kerberos-io/agent/machinery/src/conditions"
 	"github.com/kerberos-io/agent/machinery/src/encryption"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/kerberos-io/agent/machinery/src/utils"
 	"github.com/kerberos-io/agent/machinery/src/video"
+	log "github.com/sirupsen/logrus"
 	"go.opentelemetry.io/otel/trace"
 )
 
@@ -49,16 +49,19 @@ func publishRecordingState(mqttClient mqtt.Client, hubKey string, configuration
 	if err == nil {
 		mqttClient.Publish("kerberos/hub/"+hubKey, 2, false, payload)
 	} else {
-		log.Log.Error("capture.main.publishRecordingState(): failed to package MQTT message: " + err.Error())
+		log.Error("capture.main.publishRecordingState(): failed to package MQTT message: " + err.Error())
 	}
 }
 
-func recordingUploadMetadata(name, deviceKey string, timestamp int64, mp4Video *video.MP4) models.RecordingUploadMetadata {
+func recordingUploadMetadata(name, deviceKey, deviceName, regionCoordinates, numberOfChanges string, timestamp int64, mp4Video *video.MP4) models.RecordingUploadMetadata {
 	metadata := models.RecordingUploadMetadata{
-		FileName:  filepath.Base(name),
-		DeviceKey: deviceKey,
-		Timestamp: timestamp,
-		Duration:  mp4Video.VideoTotalDuration,
+		FileName:          filepath.Base(name),
+		DeviceKey:         deviceKey,
+		DeviceName:        deviceName,
+		Timestamp:         timestamp,
+		Duration:          mp4Video.VideoTotalDuration,
+		RegionCoordinates: regionCoordinates,
+		NumberOfChanges:   numberOfChanges,
 	}
 	value := mp4Video.AverageFPS()
 	if value > 0 && value <= 240 && !math.IsInf(value, 0) && !math.IsNaN(value) {
@@ -72,7 +75,7 @@ func recordingUploadMetadata(name, deviceKey string, timestamp int64, mp4Video *
 func queueRecordingForUpload(configDirectory string, metadata models.RecordingUploadMetadata) {
 	payload, err := json.Marshal(metadata)
 	if err != nil {
-		log.Log.Error("capture.main.queueRecordingForUpload(): " + err.Error())
+		log.Error("capture.main.queueRecordingForUpload(): " + err.Error())
 		return
 	}
 
@@ -95,7 +98,7 @@ func queueRecordingForUpload(configDirectory string, metadata models.RecordingUp
 		err = os.Rename(marker.Name(), filepath.Join(configDirectory, "data", "cloud", models.RecordingUploadMetadataFileName(metadata.FileName)))
 	}
 	if err != nil {
-		log.Log.Error("capture.main.queueRecordingForUpload(): " + err.Error())
+		log.Error("capture.main.queueRecordingForUpload(): " + err.Error())
 	}
 }
 
@@ -137,9 +140,9 @@ func manualRecordingExpired(communication *models.Communication, now int64) bool
 		return false
 	}
 	if heartbeatExpired {
-		log.Log.Info("capture.main.HandleRecordStream(motiondetection): auto-stopping manual recording, no viewer heartbeat within timeout.")
+		log.Info("capture.main.HandleRecordStream(motiondetection): auto-stopping manual recording, no viewer heartbeat within timeout.")
 	} else {
-		log.Log.Info("capture.main.HandleRecordStream(motiondetection): auto-stopping manual recording, maximum duration reached.")
+		log.Info("capture.main.HandleRecordStream(motiondetection): auto-stopping manual recording, maximum duration reached.")
 	}
 	communication.IsRecordingManual.UnSet()
 	communication.RecordingManualHeartbeat.Store(0)
@@ -151,7 +154,7 @@ func manualRecordingExpired(communication *models.Communication, now int64) bool
 func CleanupRecordingDirectory(configDirectory string, configuration *models.Configuration) {
 	autoClean := configuration.Config.AutoClean
 	if autoClean != "true" {
-		log.Log.Info("HandleRecordStream: Autoclean disabled, nothing to do here.")
+		log.Info("HandleRecordStream: Autoclean disabled, nothing to do here.")
 		return
 	}
 
@@ -164,7 +167,7 @@ func CleanupRecordingDirectory(configDirectory string, configuration *models.Con
 	// disk while keeping a free-space reserve.
 	needsCleanup, err := recordingsNeedCleanup(recordingsDirectory, configuration)
 	if err != nil {
-		log.Log.Info("HandleRecordStream: something went wrong, " + err.Error())
+		log.Info("HandleRecordStream: something went wrong, " + err.Error())
 		return
 	}
 	if !needsCleanup {
@@ -185,12 +188,12 @@ func CleanupRecordingDirectory(configDirectory string, configuration *models.Con
 	// (otherwise a long outage would fill the disk and stop new recordings).
 	name, pending, err := pickRecordingToCleanup(recordingsDirectory, cloudDirectory)
 	if err != nil {
-		log.Log.Info("HandleRecordStream: something went wrong, " + err.Error())
+		log.Info("HandleRecordStream: something went wrong, " + err.Error())
 		return
 	}
 
 	if err := os.Remove(recordingsDirectory + "/" + name); err != nil {
-		log.Log.Info("HandleRecordStream: something went wrong, " + err.Error())
+		log.Info("HandleRecordStream: something went wrong, " + err.Error())
 		return
 	}
 
@@ -200,14 +203,14 @@ func CleanupRecordingDirectory(configDirectory string, configuration *models.Con
 		// that was never uploaded to keep recording new footage. Also remove the
 		// now-dangling upload marker so the upload loop doesn't keep trying to
 		// upload a file that no longer exists.
-		log.Log.Warning("HandleRecordStream: removed oldest recording as part of cleanup, but it was STILL PENDING UPLOAD (disk full of un-uploaded recordings) - " + recordingsDirectory + "/" + name)
+		log.Warn("HandleRecordStream: removed oldest recording as part of cleanup, but it was STILL PENDING UPLOAD (disk full of un-uploaded recordings) - " + recordingsDirectory + "/" + name)
 		for _, markerName := range uploadMarkerNames(name) {
 			if err := os.Remove(filepath.Join(cloudDirectory, markerName)); err != nil && !os.IsNotExist(err) {
-				log.Log.Info("HandleRecordStream: could not remove dangling upload marker " + markerName + ", " + err.Error())
+				log.Info("HandleRecordStream: could not remove dangling upload marker " + markerName + ", " + err.Error())
 			}
 		}
 	} else {
-		log.Log.Info("HandleRecordStream: removed oldest file as part of cleanup - " + recordingsDirectory + "/" + name)
+		log.Info("HandleRecordStream: removed oldest file as part of cleanup - " + recordingsDirectory + "/" + name)
 	}
 }
 
@@ -331,7 +334,7 @@ func recordingPendingUpload(cloudDirectory, recordingName string) bool {
 	return false
 }
 
-func HandleRecordStream(queue *packets.Queue, configDirectory string, configuration *models.Configuration, communication *models.Communication, rtspClient RTSPClient, mqttClient mqtt.Client) {
+func HandleRecordStream(queue *packets.Queue, configDirectory string, configuration *models.Configuration, communication *models.Communication, rtspClient RTSPClient, mqttClient mqtt.Client, motionEvents <-chan models.MotionDataPartial) {
 
 	config := configuration.Config
 	hubKey := config.HubKey
@@ -345,9 +348,9 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 	communication.RecordingManualHeartbeatSeen.UnSet()
 
 	if config.Capture.Recording == "false" {
-		log.Log.Info("capture.main.HandleRecordStream(): disabled, we will not record anything.")
+		log.Info("capture.main.HandleRecordStream(): disabled, we will not record anything.")
 	} else {
-		log.Log.Debug("capture.main.HandleRecordStream(): started")
+		log.Debug("capture.main.HandleRecordStream(): started")
 
 		preRecording := config.Capture.PreRecording * 1000
 		postRecording := config.Capture.PostRecording * 1000           // number of seconds to record.
@@ -360,7 +363,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 		}
 
 		if maxRecordingPeriod < preRecording+postRecording {
-			log.Log.Error("capture.main.HandleRecordStream(): maxRecordingPeriod is less than preRecording + postRecording, this is not allowed. Setting maxRecordingPeriod to preRecording + postRecording.")
+			log.Error("capture.main.HandleRecordStream(): maxRecordingPeriod is less than preRecording + postRecording, this is not allowed. Setting maxRecordingPeriod to preRecording + postRecording.")
 			maxRecordingPeriod = preRecording + postRecording
 		}
 
@@ -394,7 +397,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 			var name string
 
 			// Do not do anything!
-			log.Log.Info("capture.main.HandleRecordStream(continuous): start recording")
+			log.Info("capture.main.HandleRecordStream(continuous): start recording")
 
 			start := false
 
@@ -440,10 +443,10 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					}
 					if (videoCodec == "H264" && (len(mp4Video.SPSNALUs) == 0 || len(mp4Video.PPSNALUs) == 0)) ||
 						(videoCodec == "H265" && (len(mp4Video.VPSNALUs) == 0 || len(mp4Video.SPSNALUs) == 0 || len(mp4Video.PPSNALUs) == 0)) {
-						log.Log.Warning("capture.main.HandleRecordStream(continuous): closing MP4 without full parameter sets, moov may be incomplete")
+						log.Warn("capture.main.HandleRecordStream(continuous): closing MP4 without full parameter sets, moov may be incomplete")
 					}
 					mp4Video.Close(&config)
-					log.Log.Info("capture.main.HandleRecordStream(continuous): recording finished: file save: " + name)
+					log.Info("capture.main.HandleRecordStream(continuous): recording finished: file save: " + name)
 
 					// Cleanup muxer
 					start = false
@@ -467,7 +470,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 						oldName := name
 						name = s + ".mp4"
 						fullName = configDirectory + "/data/recordings/" + name
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
+						log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
 
 						// Rename the file to the new name.
 						err := os.Rename(
@@ -475,10 +478,10 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 							configDirectory+"/data/recordings/"+s+".mp4")
 
 						if err != nil {
-							log.Log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
+							log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
 						}
 					} else {
-						log.Log.Info("capture.main.HandleRecordStream(continuous): no video data recorded, not renaming file.")
+						log.Info("capture.main.HandleRecordStream(continuous): no video data recorded, not renaming file.")
 					}
 
 					// Check if we need to encrypt the recording.
@@ -492,17 +495,17 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 								// write back to file
 								err := os.WriteFile(fullName, []byte(encryptedContents), 0644)
 								if err != nil {
-									log.Log.Error("capture.main.HandleRecordStream(continuous): error writing file: " + err.Error())
+									log.Error("capture.main.HandleRecordStream(continuous): error writing file: " + err.Error())
 								}
 							} else {
-								log.Log.Error("capture.main.HandleRecordStream(continuous): error encrypting file: " + err.Error())
+								log.Error("capture.main.HandleRecordStream(continuous): error encrypting file: " + err.Error())
 							}
 						} else {
-							log.Log.Error("capture.main.HandleRecordStream(continuous): error reading file: " + err.Error())
+							log.Error("capture.main.HandleRecordStream(continuous): error reading file: " + err.Error())
 						}
 					}
 
-					queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, startRecording, mp4Video))
+					queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, config.Name, "0-0-0-0", "-1", startRecording, mp4Video))
 
 					recordingStatus = "idle"
 
@@ -520,7 +523,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					// We'll validate those conditions and if not valid we'll not do anything.
 					valid, err := conditions.Validate(loc, configuration)
 					if !valid && err != nil {
-						log.Log.Debug("capture.main.HandleRecordStream(continuous): " + err.Error() + ".")
+						log.Debug("capture.main.HandleRecordStream(continuous): " + err.Error() + ".")
 						time.Sleep(5 * time.Second)
 						continue
 					}
@@ -552,7 +555,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					fullName = configDirectory + "/data/recordings/" + name
 
 					// Running...
-					log.Log.Info("capture.main.HandleRecordStream(continuous): recording started")
+					log.Info("capture.main.HandleRecordStream(continuous): recording started")
 
 					// Get width and height from the camera.
 					width := configuration.Config.Capture.IPCamera.Width
@@ -564,7 +567,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					vpsNALUS := configuration.Config.Capture.IPCamera.VPSNALUs
 
 					if len(spsNALUS) == 0 || len(ppsNALUS) == 0 {
-						log.Log.Warning("capture.main.HandleRecordStream(continuous): missing SPS/PPS at recording start")
+						log.Warn("capture.main.HandleRecordStream(continuous): missing SPS/PPS at recording start")
 					}
 					// Create a video file, and set the dimensions.
 					mp4Video = video.NewMP4(fullName, spsNALUS, ppsNALUS, vpsNALUS, configuration.Config.Capture.MaxLengthRecording)
@@ -579,7 +582,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					if audioCodec == "AAC" {
 						audioTrack = mp4Video.AddAudioTrack("AAC")
 					} else if audioCodec == "PCM_MULAW" {
-						log.Log.Debug("capture.main.HandleRecordStream(continuous): no AAC audio codec detected, skipping audio track.")
+						log.Debug("capture.main.HandleRecordStream(continuous): no AAC audio codec detected, skipping audio track.")
 					}
 
 					writeSampleToMP4(mp4Video, videoTrack, audioTrack, pkt)
@@ -600,7 +603,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 			if cursorError != nil {
 				if recordingStatus == "started" {
 
-					log.Log.Info("capture.main.HandleRecordStream(continuous): Recording finished: file save: " + name)
+					log.Info("capture.main.HandleRecordStream(continuous): Recording finished: file save: " + name)
 
 					// Cleanup muxer
 					start = false
@@ -624,7 +627,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 						oldName := name
 						name = s + ".mp4"
 						fullName = configDirectory + "/data/recordings/" + name
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
+						log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
 
 						// Rename the file to the new name.
 						err := os.Rename(
@@ -632,10 +635,10 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 							configDirectory+"/data/recordings/"+s+".mp4")
 
 						if err != nil {
-							log.Log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
+							log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
 						}
 					} else {
-						log.Log.Info("capture.main.HandleRecordStream(continuous): no video data recorded, not renaming file.")
+						log.Info("capture.main.HandleRecordStream(continuous): no video data recorded, not renaming file.")
 					}
 
 					// Check if we need to encrypt the recording.
@@ -649,17 +652,17 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 								// write back to file
 								err := os.WriteFile(fullName, []byte(encryptedContents), 0644)
 								if err != nil {
-									log.Log.Error("capture.main.HandleRecordStream(motiondetection): error writing file: " + err.Error())
+									log.Error("capture.main.HandleRecordStream(motiondetection): error writing file: " + err.Error())
 								}
 							} else {
-								log.Log.Error("capture.main.HandleRecordStream(motiondetection): error encrypting file: " + err.Error())
+								log.Error("capture.main.HandleRecordStream(motiondetection): error encrypting file: " + err.Error())
 							}
 						} else {
-							log.Log.Error("capture.main.HandleRecordStream(motiondetection): error reading file: " + err.Error())
+							log.Error("capture.main.HandleRecordStream(motiondetection): error reading file: " + err.Error())
 						}
 					}
 
-					queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, startRecording, mp4Video))
+					queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, config.Name, "0-0-0-0", "-1", startRecording, mp4Video))
 
 					recordingStatus = "idle"
 
@@ -672,7 +675,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 			}
 		} else {
 
-			log.Log.Info("capture.main.HandleRecordStream(motiondetection): Start motion based recording ")
+			log.Info("capture.main.HandleRecordStream(motiondetection): Start motion based recording ")
 
 			var lastRecordingTime int64 = 0 // last recording timestamp in milliseconds
 			var displayTime int64 = 0       // display time in milliseconds
@@ -680,7 +683,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 			var videoTrack uint32
 			var audioTrack uint32
 
-			for motion := range communication.HandleMotion {
+			for motion := range motionEvents {
 
 				// Get as much packets we need.
 				var cursorError error
@@ -710,7 +713,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 
 				// If startRecording is 0, we will continue as it might be we are in a state of restarting the agent.
 				if startRecording == 0 {
-					log.Log.Info("capture.main.HandleRecordStream(motiondetection): startRecording is 0, we will continue as it might be we are in a state of restarting the agent.")
+					log.Info("capture.main.HandleRecordStream(motiondetection): startRecording is 0, we will continue as it might be we are in a state of restarting the agent.")
 					continue
 				}
 
@@ -748,7 +751,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 				fullName := configDirectory + "/data/recordings/" + name
 
 				// Running...
-				log.Log.Info("capture.main.HandleRecordStream(motiondetection): recording started (" + name + ")" + " at " + strconv.FormatInt(displayTimeSeconds, 10) + " unix")
+				log.Info("capture.main.HandleRecordStream(motiondetection): recording started (" + name + ")" + " at " + strconv.FormatInt(displayTimeSeconds, 10) + " unix")
 
 				// Get width and height from the camera.
 				width := configuration.Config.Capture.IPCamera.Width
@@ -760,7 +763,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 				vpsNALUS := configuration.Config.Capture.IPCamera.VPSNALUs
 
 				if len(spsNALUS) == 0 || len(ppsNALUS) == 0 {
-					log.Log.Warning("capture.main.HandleRecordStream(motiondetection): missing SPS/PPS at recording start")
+					log.Warn("capture.main.HandleRecordStream(motiondetection): missing SPS/PPS at recording start")
 				}
 				// Create the MP4 only once the first keyframe arrives.
 				var mp4Video *video.MP4
@@ -769,16 +772,16 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 
 					nextPkt, cursorError = recordingCursor.ReadPacket()
 					if cursorError != nil {
-						log.Log.Error("capture.main.HandleRecordStream(motiondetection): " + cursorError.Error())
+						log.Error("capture.main.HandleRecordStream(motiondetection): " + cursorError.Error())
 					}
 
 					now = time.Now().UnixMilli()
 					select {
-					case motion := <-communication.HandleMotion:
+					case motion := <-motionEvents:
 						motionTimestamp = now
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): motion detected while recording. Expanding recording.")
+						log.Info("capture.main.HandleRecordStream(motiondetection): motion detected while recording. Expanding recording.")
 						numberOfChanges := motion.NumberOfChanges
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): Received message with recording data, detected changes to save: " + strconv.Itoa(numberOfChanges))
+						log.Info("capture.main.HandleRecordStream(motiondetection): Received message with recording data, detected changes to save: " + strconv.Itoa(numberOfChanges))
 					default:
 					}
 
@@ -794,16 +797,16 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					}
 
 					if start && (motionTimestamp+postRecording-now < 0 || now-startRecording > maxRecordingPeriod-500) && nextPkt.IsKeyFrame {
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): timestamp+postRecording-now < 0  - " + strconv.FormatInt(motionTimestamp+postRecording-now, 10) + " < 0")
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): now-startRecording > maxRecordingPeriod-500 - " + strconv.FormatInt(now-startRecording, 10) + " > " + strconv.FormatInt(maxRecordingPeriod-500, 10))
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): closing recording (timestamp: " + strconv.FormatInt(motionTimestamp, 10) + ", postRecording: " + strconv.FormatInt(postRecording, 10) + ", now: " + strconv.FormatInt(now, 10) + ", startRecording: " + strconv.FormatInt(startRecording, 10) + ", maxRecordingPeriod: " + strconv.FormatInt(maxRecordingPeriod, 10))
+						log.Info("capture.main.HandleRecordStream(motiondetection): timestamp+postRecording-now < 0  - " + strconv.FormatInt(motionTimestamp+postRecording-now, 10) + " < 0")
+						log.Info("capture.main.HandleRecordStream(motiondetection): now-startRecording > maxRecordingPeriod-500 - " + strconv.FormatInt(now-startRecording, 10) + " > " + strconv.FormatInt(maxRecordingPeriod-500, 10))
+						log.Info("capture.main.HandleRecordStream(motiondetection): closing recording (timestamp: " + strconv.FormatInt(motionTimestamp, 10) + ", postRecording: " + strconv.FormatInt(postRecording, 10) + ", now: " + strconv.FormatInt(now, 10) + ", startRecording: " + strconv.FormatInt(startRecording, 10) + ", maxRecordingPeriod: " + strconv.FormatInt(maxRecordingPeriod, 10))
 						break
 					}
 					if pkt.IsKeyFrame && !start && pkt.CurrentTime >= startRecording {
 						// We start the recording if we have a keyframe and the last duration is 0 or less than the current packet time.
 						// It could be start we start from the beginning of the recording.
-						log.Log.Debug("capture.main.HandleRecordStream(motiondetection): write frames")
-						log.Log.Debug("capture.main.HandleRecordStream(motiondetection): recording started on keyframe")
+						log.Debug("capture.main.HandleRecordStream(motiondetection): write frames")
+						log.Debug("capture.main.HandleRecordStream(motiondetection): recording started on keyframe")
 
 						// Align duration timers with the first keyframe.
 						startRecording = pkt.CurrentTime
@@ -821,7 +824,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 						if audioCodec == "AAC" {
 							audioTrack = mp4Video.AddAudioTrack("AAC")
 						} else if audioCodec == "PCM_MULAW" {
-							log.Log.Debug("capture.main.HandleRecordStream(continuous): no AAC audio codec detected, skipping audio track.")
+							log.Debug("capture.main.HandleRecordStream(continuous): no AAC audio codec detected, skipping audio track.")
 						}
 						start = true
 
@@ -840,7 +843,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 				lastRecordingTime = pkt.CurrentTime
 
 				if mp4Video == nil {
-					log.Log.Warning("capture.main.HandleRecordStream(motiondetection): recording closed without keyframe; no MP4 created")
+					log.Warn("capture.main.HandleRecordStream(motiondetection): recording closed without keyframe; no MP4 created")
 					continue
 				}
 
@@ -856,10 +859,10 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 				}
 				if (videoCodec == "H264" && (len(mp4Video.SPSNALUs) == 0 || len(mp4Video.PPSNALUs) == 0)) ||
 					(videoCodec == "H265" && (len(mp4Video.VPSNALUs) == 0 || len(mp4Video.SPSNALUs) == 0 || len(mp4Video.PPSNALUs) == 0)) {
-					log.Log.Warning("capture.main.HandleRecordStream(motiondetection): closing MP4 without full parameter sets, moov may be incomplete")
+					log.Warn("capture.main.HandleRecordStream(motiondetection): closing MP4 without full parameter sets, moov may be incomplete")
 				}
 				mp4Video.Close(&config)
-				log.Log.Info("capture.main.HandleRecordStream(motiondetection): file save: " + name)
+				log.Info("capture.main.HandleRecordStream(motiondetection): file save: " + name)
 
 				// Notify the hub / live-view UI that this camera stopped recording.
 				publishRecordingState(mqttClient, hubKey, configuration, false)
@@ -870,10 +873,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 				// restart when the recording has expired (heartbeat lapsed or max
 				// duration reached), so it ends here instead of recording forever.
 				if communication.IsRecordingManual.IsSet() && !manualRecordingExpired(communication, time.Now().UnixMilli()) {
-					select {
-					case communication.HandleMotion <- models.MotionDataPartial{Timestamp: time.Now().Unix(), NumberOfChanges: 100000000}:
-					default:
-					}
+					communication.TrySendMotion(models.MotionDataPartial{Timestamp: time.Now().Unix(), NumberOfChanges: 100000000})
 				}
 
 				// Update the name of the recording with the duration.
@@ -894,7 +894,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					oldName := name
 					name = s + ".mp4"
 					fullName = configDirectory + "/data/recordings/" + name
-					log.Log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
+					log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
 
 					// Rename the file to the new name.
 					err := os.Rename(
@@ -902,10 +902,10 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 						configDirectory+"/data/recordings/"+s+".mp4")
 
 					if err != nil {
-						log.Log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
+						log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
 					}
 				} else {
-					log.Log.Info("capture.main.HandleRecordStream(motiondetection): no video data recorded, not renaming file.")
+					log.Info("capture.main.HandleRecordStream(motiondetection): no video data recorded, not renaming file.")
 				}
 
 				// Check if we need to encrypt the recording.
@@ -919,24 +919,24 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 							// write back to file
 							err := os.WriteFile(fullName, []byte(encryptedContents), 0644)
 							if err != nil {
-								log.Log.Error("capture.main.HandleRecordStream(motiondetection): error writing file: " + err.Error())
+								log.Error("capture.main.HandleRecordStream(motiondetection): error writing file: " + err.Error())
 							}
 						} else {
-							log.Log.Error("capture.main.HandleRecordStream(motiondetection): error encrypting file: " + err.Error())
+							log.Error("capture.main.HandleRecordStream(motiondetection): error encrypting file: " + err.Error())
 						}
 					} else {
-						log.Log.Error("capture.main.HandleRecordStream(motiondetection): error reading file: " + err.Error())
+						log.Error("capture.main.HandleRecordStream(motiondetection): error reading file: " + err.Error())
 					}
 				}
 
-				queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, displayTime, mp4Video))
+				queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, config.Name, motionRectangleString, strconv.Itoa(numberOfChanges), displayTime, mp4Video))
 
 				// Clean up the recording directory if necessary.
 				CleanupRecordingDirectory(configDirectory, configuration)
 			}
 		}
 
-		log.Log.Debug("capture.main.HandleRecordStream(): finished")
+		log.Debug("capture.main.HandleRecordStream(): finished")
 	}
 }
 
@@ -1030,13 +1030,14 @@ func Base64Image(captureDevice *Capture, communication *models.Communication, co
 	var cursor *packets.QueueCursor
 
 	// We'll pick the right client and decoder.
-	rtspClient := captureDevice.RTSPSubClient
+	rtspClient := captureDevice.SubClient()
 	if rtspClient != nil {
-		queue = communication.SubQueue
-		cursor = queue.Latest()
+		queue = communication.SubQueue()
 	} else {
-		rtspClient = captureDevice.RTSPClient
-		queue = communication.Queue
+		rtspClient = captureDevice.MainClient()
+		queue = communication.MainQueue()
+	}
+	if queue != nil {
 		cursor = queue.Latest()
 	}
 
@@ -1076,13 +1077,14 @@ func JpegImage(captureDevice *Capture, communication *models.Communication) imag
 	var cursor *packets.QueueCursor
 
 	// We'll pick the right client and decoder.
-	rtspClient := captureDevice.RTSPSubClient
+	rtspClient := captureDevice.SubClient()
 	if rtspClient != nil {
-		queue = communication.SubQueue
-		cursor = queue.Latest()
+		queue = communication.SubQueue()
 	} else {
-		rtspClient = captureDevice.RTSPClient
-		queue = communication.Queue
+		rtspClient = captureDevice.MainClient()
+		queue = communication.MainQueue()
+	}
+	if queue != nil {
 		cursor = queue.Latest()
 	}
 
@@ -1137,20 +1139,16 @@ func writeSampleToMP4(mp4Video *video.MP4, videoTrack, audioTrack uint32, pkt pa
 			dts = pts - uint64(compositionOffset)
 		}
 		if err := mp4Video.AddSampleToTrack(videoTrack, pkt.IsKeyFrame, pkt.Data, dts, compositionOffset); err != nil {
-			log.Log.Error("capture.main.writeSampleToMP4(): " + err.Error())
+			log.Error("capture.main.writeSampleToMP4(): " + err.Error())
 		}
 	} else if pkt.IsAudio {
 		if pkt.Codec == "AAC" {
 			if err := mp4Video.AddSampleToTrack(audioTrack, pkt.IsKeyFrame, pkt.Data, pts, 0); err != nil {
-				log.Log.Error("capture.main.writeSampleToMP4(): " + err.Error())
+				log.Error("capture.main.writeSampleToMP4(): " + err.Error())
 			}
 		} else if pkt.Codec == "PCM_MULAW" {
 			// TODO: transcode to AAC, some work to do..
-			log.Log.Debug("capture.main.writeSampleToMP4(): no AAC audio codec detected, skipping audio track.")
+			log.Debug("capture.main.writeSampleToMP4(): no AAC audio codec detected, skipping audio track.")
 		}
 	}
 }
-
-/*func convertPTS2(v int64) uint64 {
-	return uint64(v) / 100
-}*/

machinery/src/capture/main_test.go

@@ -42,7 +42,7 @@ func TestQueueRecordingForUploadStoresFinalizedMetadata(t *testing.T) {
 	}
 
 	mp4Video := &video.MP4{VideoTotalDuration: 20452, SampleCount: 613}
-	metadata := recordingUploadMetadata("recording.mp4", "device-key", 1785934709414, mp4Video)
+	metadata := recordingUploadMetadata("recording.mp4", "device-key", "camera-name", "1-2-3-4", "57", 1785934709414, mp4Video)
 	queueRecordingForUpload(configDirectory, metadata)
 
 	got, err := os.ReadFile(filepath.Join(configDirectory, "data", "cloud", "recording.metadata"))
@@ -54,7 +54,7 @@ func TestQueueRecordingForUploadStoresFinalizedMetadata(t *testing.T) {
 		t.Fatalf("decode upload marker: %v", err)
 	}
 	expectedFPS := mp4Video.AverageFPS()
-	if stored.FileName != "recording.mp4" || stored.DeviceKey != "device-key" || stored.Timestamp != 1785934709414 || stored.Duration != 20452 || math.Abs(stored.FPS-expectedFPS) > 1e-9 {
+	if stored.FileName != "recording.mp4" || stored.DeviceKey != "device-key" || stored.DeviceName != "camera-name" || stored.Timestamp != 1785934709414 || stored.Duration != 20452 || stored.RegionCoordinates != "1-2-3-4" || stored.NumberOfChanges != "57" || math.Abs(stored.FPS-expectedFPS) > 1e-9 {
 		t.Fatalf("upload marker = %+v", stored)
 	}
 	if stored.FPS == math.Floor(stored.FPS) {

machinery/src/capture/rtsp_client.go

@@ -3,36 +3,73 @@ package capture
 import (
 	"context"
 	"image"
+	"sync"
 
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 )
 
 type Capture struct {
-	RTSPClient            *Golibrtsp
-	RTSPSubClient         *Golibrtsp
-	RTSPBackChannelClient *Golibrtsp
+	clientsMu             sync.RWMutex
+	rtspClient            *Golibrtsp
+	rtspSubClient         *Golibrtsp
+	rtspBackChannelClient *Golibrtsp
 }
 
 func (c *Capture) SetMainClient(rtspUrl string) *Golibrtsp {
-	c.RTSPClient = &Golibrtsp{
+	client := &Golibrtsp{
 		Url: rtspUrl,
 	}
-	return c.RTSPClient
+	c.clientsMu.Lock()
+	c.rtspClient = client
+	c.clientsMu.Unlock()
+	return client
 }
 
 func (c *Capture) SetSubClient(rtspUrl string) *Golibrtsp {
-	c.RTSPSubClient = &Golibrtsp{
+	client := &Golibrtsp{
 		Url: rtspUrl,
 	}
-	return c.RTSPSubClient
+	c.clientsMu.Lock()
+	c.rtspSubClient = client
+	c.clientsMu.Unlock()
+	return client
 }
 
 func (c *Capture) SetBackChannelClient(rtspUrl string) *Golibrtsp {
-	c.RTSPBackChannelClient = &Golibrtsp{
+	client := &Golibrtsp{
 		Url: rtspUrl,
 	}
-	return c.RTSPBackChannelClient
+	c.clientsMu.Lock()
+	c.rtspBackChannelClient = client
+	c.clientsMu.Unlock()
+	return client
+}
+
+func (c *Capture) MainClient() *Golibrtsp {
+	c.clientsMu.RLock()
+	defer c.clientsMu.RUnlock()
+	return c.rtspClient
+}
+
+func (c *Capture) SubClient() *Golibrtsp {
+	c.clientsMu.RLock()
+	defer c.clientsMu.RUnlock()
+	return c.rtspSubClient
+}
+
+func (c *Capture) ClearClients(main, sub, backchannel *Golibrtsp) {
+	c.clientsMu.Lock()
+	defer c.clientsMu.Unlock()
+	if c.rtspClient == main {
+		c.rtspClient = nil
+	}
+	if c.rtspSubClient == sub {
+		c.rtspSubClient = nil
+	}
+	if c.rtspBackChannelClient == backchannel {
+		c.rtspBackChannelClient = nil
+	}
 }
 
 // RTSPClient is a interface that abstracts the RTSP client implementation.

machinery/src/capture/rtsp_client_test.go

@@ -0,0 +1,62 @@
+package capture
+
+import (
+	"strconv"
+	"sync"
+	"testing"
+)
+
+func TestCaptureClientAccessCanRaceReplacement(t *testing.T) {
+	captureDevice := &Capture{}
+	captureDevice.SetMainClient("rtsp://main/0")
+	captureDevice.SetSubClient("rtsp://sub/0")
+
+	var workers sync.WaitGroup
+	workers.Add(2)
+	go func() {
+		defer workers.Done()
+		for replacement := 1; replacement <= 1000; replacement++ {
+			suffix := strconv.Itoa(replacement)
+			captureDevice.SetMainClient("rtsp://main/" + suffix)
+			captureDevice.SetSubClient("rtsp://sub/" + suffix)
+		}
+	}()
+	go func() {
+		defer workers.Done()
+		for snapshot := 0; snapshot < 1000; snapshot++ {
+			if captureDevice.MainClient() == nil {
+				t.Error("MainClient() returned nil")
+				return
+			}
+			if captureDevice.SubClient() == nil {
+				t.Error("SubClient() returned nil")
+				return
+			}
+		}
+	}()
+	workers.Wait()
+}
+
+func TestCaptureClearClientsOnlyClearsMatchingRun(t *testing.T) {
+	captureDevice := &Capture{}
+	oldMain := captureDevice.SetMainClient("rtsp://main/old")
+	oldSub := captureDevice.SetSubClient("rtsp://sub/old")
+	oldBackchannel := captureDevice.SetBackChannelClient("rtsp://back/old")
+
+	newMain := captureDevice.SetMainClient("rtsp://main/new")
+	newSub := captureDevice.SetSubClient("rtsp://sub/new")
+	newBackchannel := captureDevice.SetBackChannelClient("rtsp://back/new")
+
+	captureDevice.ClearClients(oldMain, oldSub, oldBackchannel)
+	if captureDevice.MainClient() != newMain {
+		t.Fatal("stale cleanup cleared the new main client")
+	}
+	if captureDevice.SubClient() != newSub {
+		t.Fatal("stale cleanup cleared the new sub client")
+	}
+
+	captureDevice.ClearClients(newMain, newSub, newBackchannel)
+	if captureDevice.MainClient() != nil || captureDevice.SubClient() != nil {
+		t.Fatal("matching cleanup did not clear current clients")
+	}
+}

machinery/src/cloud/cloud_test.go

@@ -0,0 +1,592 @@
+package cloud
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/kerberos-io/agent/machinery/src/models"
+	agentonvif "github.com/kerberos-io/agent/machinery/src/onvif"
+	goonvif "github.com/kerberos-io/onvif"
+	goonvifdevice "github.com/kerberos-io/onvif/device"
+	goonvifptz "github.com/kerberos-io/onvif/ptz"
+)
+
+func TestHeartbeatFailureLogOmitsHubResponseBody(t *testing.T) {
+	response := &http.Response{
+		StatusCode: http.StatusBadRequest,
+		Status:     "400 Bad Request",
+		Body:       io.NopCloser(strings.NewReader(`{"error":"invalid heartbeat"}`)),
+	}
+
+	responseBody, truncated, err := readHeartbeatResponseBody(response)
+	if err != nil {
+		t.Fatalf("readHeartbeatResponseBody() error = %v", err)
+	}
+	fields := heartbeatFailureLogFields(response, responseBody, truncated, 125*time.Millisecond)
+
+	for key, want := range map[string]interface{}{
+		"duration_ms":             int64(125),
+		"response_body_bytes":     len(responseBody),
+		"response_body_truncated": false,
+		"status_code":             http.StatusBadRequest,
+	} {
+		if got := fields[key]; got != want {
+			t.Errorf("%s = %v, want %v", key, got, want)
+		}
+	}
+	for key, value := range fields {
+		if strings.Contains(key, "response_body") && key != "response_body_bytes" && key != "response_body_truncated" {
+			t.Fatalf("unexpected response body field %q=%v", key, value)
+		}
+		if strings.Contains(fmt.Sprint(value), "invalid heartbeat") {
+			t.Fatalf("heartbeat log fields exposed response body in %q=%v", key, value)
+		}
+	}
+}
+
+func TestReadHeartbeatResponseBodyTruncatesLargeBody(t *testing.T) {
+	response := &http.Response{
+		Body: io.NopCloser(strings.NewReader(strings.Repeat("x", heartbeatResponseBodyLogLimit+1))),
+	}
+
+	body, truncated, err := readHeartbeatResponseBody(response)
+	if err != nil {
+		t.Fatalf("readHeartbeatResponseBody() error = %v", err)
+	}
+	if !truncated {
+		t.Fatal("readHeartbeatResponseBody() truncated = false, want true")
+	}
+	if len(body) != heartbeatResponseBodyLogLimit {
+		t.Fatalf("len(body) = %d, want %d", len(body), heartbeatResponseBodyLogLimit)
+	}
+}
+
+func TestHeartbeatONVIFPayloadCachesStaticAndReusesLoopSubscription(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	device := newTestONVIFDevice()
+	camera := models.IPCamera{
+		ONVIFXAddr:    "http://camera/onvif",
+		ONVIFUsername: "operator",
+		ONVIFPassword: "secret",
+	}
+	initialEvents := []agentonvif.ONVIFEvents{{Key: "input-1", Type: "input", Value: "true", Timestamp: 1}}
+	loopEvents := []agentonvif.ONVIFEvents{{Key: "output-1", Type: "output", Value: "false", Timestamp: 2}}
+	wantPresets := mustJSONMarshal(t, []models.OnvifActionPreset{{Name: "Lobby", Token: "1"}})
+	wantInitialEvents := mustJSONMarshal(t, initialEvents)
+	wantLoopEvents := mustJSONMarshal(t, loopEvents)
+
+	var connectCalls, ptzConfigCalls, ptzFunctionCalls, presetCalls, createCalls, eventCalls, unsubscribeCalls int
+
+	heartbeatConnectToONVIFDevice = func(*models.IPCamera) (*goonvif.Device, goonvifdevice.GetCapabilitiesResponse, error) {
+		connectCalls++
+		return device, goonvifdevice.GetCapabilitiesResponse{}, nil
+	}
+	heartbeatGetPTZConfigurationsFromDevice = func(*goonvif.Device) (goonvifptz.GetConfigurationsResponse, error) {
+		ptzConfigCalls++
+		return goonvifptz.GetConfigurationsResponse{}, nil
+	}
+	heartbeatGetPTZFunctionsFromDevice = func(goonvifptz.GetConfigurationsResponse) ([]string, bool, bool) {
+		ptzFunctionCalls++
+		return nil, true, true
+	}
+	heartbeatGetPresetsFromDevice = func(*goonvif.Device) ([]models.OnvifActionPreset, error) {
+		presetCalls++
+		return []models.OnvifActionPreset{{Name: "Lobby", Token: "1"}}, nil
+	}
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		createCalls++
+		switch createCalls {
+		case 1:
+			return "initial-1", nil
+		case 2:
+			return "loop", nil
+		case 3:
+			return "initial-2", nil
+		default:
+			t.Fatalf("unexpected create pull point call %d", createCalls)
+			return "", nil
+		}
+	}
+	heartbeatGetEventMessages = func(_ *goonvif.Device, pullPointAddress string) ([]agentonvif.ONVIFEvents, error) {
+		eventCalls++
+		switch pullPointAddress {
+		case "initial-1", "initial-2":
+			return initialEvents, nil
+		case "loop":
+			return loopEvents, nil
+		default:
+			t.Fatalf("unexpected pull point address %q", pullPointAddress)
+			return nil, nil
+		}
+	}
+	heartbeatUnsubscribePullPoint = func(_ *goonvif.Device, pullPointAddress string) error {
+		unsubscribeCalls++
+		if pullPointAddress != "initial-1" && pullPointAddress != "initial-2" {
+			t.Fatalf("unexpected unsubscribe pull point %q", pullPointAddress)
+		}
+		return nil
+	}
+
+	state := newHeartbeatONVIFState()
+
+	payload := getHeartbeatONVIFPayload(camera, state)
+	if payload.enabled != "true" || payload.zoom != "true" || payload.panTilt != "true" || payload.presets != "true" {
+		t.Fatalf("unexpected static payload: %+v", payload)
+	}
+	if string(payload.presetsList) != string(wantPresets) {
+		t.Fatalf("payload.presetsList = %s, want %s", payload.presetsList, wantPresets)
+	}
+	if string(payload.eventsList) != string(wantInitialEvents) {
+		t.Fatalf("payload.eventsList = %s, want %s", payload.eventsList, wantInitialEvents)
+	}
+	if connectCalls != 1 || ptzConfigCalls != 1 || ptzFunctionCalls != 1 || presetCalls != 1 {
+		t.Fatalf("unexpected static call counts after first cycle: connect=%d ptzConfig=%d ptzFunctions=%d presets=%d", connectCalls, ptzConfigCalls, ptzFunctionCalls, presetCalls)
+	}
+	if createCalls != 2 || eventCalls != 1 || unsubscribeCalls != 1 {
+		t.Fatalf("unexpected event call counts after first cycle: create=%d events=%d unsubscribe=%d", createCalls, eventCalls, unsubscribeCalls)
+	}
+
+	payload = getHeartbeatONVIFPayload(camera, state)
+	if string(payload.eventsList) != string(wantLoopEvents) {
+		t.Fatalf("second payload.eventsList = %s, want %s", payload.eventsList, wantLoopEvents)
+	}
+	if connectCalls != 1 {
+		t.Fatalf("connectCalls = %d, want 1", connectCalls)
+	}
+	if ptzConfigCalls != 1 || ptzFunctionCalls != 1 || presetCalls != 1 {
+		t.Fatalf("static calls were not cached: ptzConfig=%d ptzFunctions=%d presets=%d", ptzConfigCalls, ptzFunctionCalls, presetCalls)
+	}
+	if createCalls != 3 || eventCalls != 3 || unsubscribeCalls != 2 {
+		t.Fatalf("temporary state subscription was not refreshed or loop subscription was not reused: create=%d events=%d unsubscribe=%d", createCalls, eventCalls, unsubscribeCalls)
+	}
+}
+
+func TestHeartbeatONVIFPayloadRefreshesAfterCameraConfigChange(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	deviceA := newTestONVIFDevice()
+	deviceB := newTestONVIFDevice()
+	cameraA := models.IPCamera{ONVIFXAddr: "http://camera-a/onvif", ONVIFUsername: "user", ONVIFPassword: "secret-a"}
+	cameraB := models.IPCamera{ONVIFXAddr: "http://camera-b/onvif", ONVIFUsername: "user", ONVIFPassword: "secret-b"}
+	var connectCalls, ptzConfigCalls, presetCalls, createCalls int
+	var unsubscribed []string
+
+	heartbeatConnectToONVIFDevice = func(camera *models.IPCamera) (*goonvif.Device, goonvifdevice.GetCapabilitiesResponse, error) {
+		connectCalls++
+		switch camera.ONVIFXAddr {
+		case cameraA.ONVIFXAddr:
+			return deviceA, goonvifdevice.GetCapabilitiesResponse{}, nil
+		case cameraB.ONVIFXAddr:
+			return deviceB, goonvifdevice.GetCapabilitiesResponse{}, nil
+		default:
+			t.Fatalf("unexpected camera address %q", camera.ONVIFXAddr)
+			return nil, goonvifdevice.GetCapabilitiesResponse{}, nil
+		}
+	}
+	heartbeatGetPTZConfigurationsFromDevice = func(*goonvif.Device) (goonvifptz.GetConfigurationsResponse, error) {
+		ptzConfigCalls++
+		return goonvifptz.GetConfigurationsResponse{}, nil
+	}
+	heartbeatGetPTZFunctionsFromDevice = func(goonvifptz.GetConfigurationsResponse) ([]string, bool, bool) {
+		return nil, false, true
+	}
+	heartbeatGetPresetsFromDevice = func(*goonvif.Device) ([]models.OnvifActionPreset, error) {
+		presetCalls++
+		return nil, nil
+	}
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		createCalls++
+		switch createCalls {
+		case 1:
+			return "initial-a", nil
+		case 2:
+			return "loop-a", nil
+		case 3:
+			return "initial-b", nil
+		case 4:
+			return "loop-b", nil
+		default:
+			t.Fatalf("unexpected create pull point call %d", createCalls)
+			return "", nil
+		}
+	}
+	heartbeatGetEventMessages = func(_ *goonvif.Device, pullPointAddress string) ([]agentonvif.ONVIFEvents, error) {
+		switch pullPointAddress {
+		case "initial-a":
+			return []agentonvif.ONVIFEvents{{Key: "a", Type: "input", Value: "true", Timestamp: 1}}, nil
+		case "initial-b":
+			return []agentonvif.ONVIFEvents{{Key: "b", Type: "input", Value: "false", Timestamp: 2}}, nil
+		default:
+			t.Fatalf("unexpected pull point address %q", pullPointAddress)
+			return nil, nil
+		}
+	}
+	heartbeatUnsubscribePullPoint = func(_ *goonvif.Device, pullPointAddress string) error {
+		unsubscribed = append(unsubscribed, pullPointAddress)
+		return nil
+	}
+
+	state := newHeartbeatONVIFState()
+	_ = getHeartbeatONVIFPayload(cameraA, state)
+	payload := getHeartbeatONVIFPayload(cameraB, state)
+
+	if connectCalls != 2 {
+		t.Fatalf("connectCalls = %d, want 2", connectCalls)
+	}
+	if ptzConfigCalls != 2 || presetCalls != 2 {
+		t.Fatalf("camera config change did not refresh static state: ptzConfig=%d presets=%d", ptzConfigCalls, presetCalls)
+	}
+	if createCalls != 4 {
+		t.Fatalf("camera config change did not recreate subscriptions: create=%d", createCalls)
+	}
+	if countString(unsubscribed, "loop-a") != 1 {
+		t.Fatalf("unsubscribed loop-a %d times, want 1; unsubscribed=%v", countString(unsubscribed, "loop-a"), unsubscribed)
+	}
+	wantEvents := mustJSONMarshal(t, []agentonvif.ONVIFEvents{{Key: "b", Type: "input", Value: "false", Timestamp: 2}})
+	if string(payload.eventsList) != string(wantEvents) {
+		t.Fatalf("payload.eventsList = %s, want %s", payload.eventsList, wantEvents)
+	}
+}
+
+func TestHeartbeatONVIFPayloadRetriesStaticFetchFailuresWithoutReconnect(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	device := newTestONVIFDevice()
+	camera := models.IPCamera{
+		ONVIFXAddr:    "http://camera/onvif",
+		ONVIFUsername: "operator",
+		ONVIFPassword: "secret",
+	}
+	var connectCalls, ptzConfigCalls, presetCalls, createCalls, eventCalls int
+
+	heartbeatConnectToONVIFDevice = func(*models.IPCamera) (*goonvif.Device, goonvifdevice.GetCapabilitiesResponse, error) {
+		connectCalls++
+		return device, goonvifdevice.GetCapabilitiesResponse{}, nil
+	}
+	heartbeatGetPTZConfigurationsFromDevice = func(*goonvif.Device) (goonvifptz.GetConfigurationsResponse, error) {
+		ptzConfigCalls++
+		return goonvifptz.GetConfigurationsResponse{}, nil
+	}
+	heartbeatGetPTZFunctionsFromDevice = func(goonvifptz.GetConfigurationsResponse) ([]string, bool, bool) {
+		return nil, true, true
+	}
+	heartbeatGetPresetsFromDevice = func(*goonvif.Device) ([]models.OnvifActionPreset, error) {
+		presetCalls++
+		if presetCalls == 1 {
+			return nil, errors.New("temporary preset failure")
+		}
+		return []models.OnvifActionPreset{{Name: "Lobby", Token: "1"}}, nil
+	}
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		createCalls++
+		switch createCalls {
+		case 1:
+			return "initial-1", nil
+		case 2:
+			return "loop", nil
+		case 3:
+			return "initial-2", nil
+		default:
+			t.Fatalf("unexpected create pull point call %d", createCalls)
+			return "", nil
+		}
+	}
+	heartbeatGetEventMessages = func(_ *goonvif.Device, pullPointAddress string) ([]agentonvif.ONVIFEvents, error) {
+		eventCalls++
+		switch pullPointAddress {
+		case "initial-1", "initial-2":
+			return []agentonvif.ONVIFEvents{{Key: "one", Type: "input", Value: "true", Timestamp: 1}}, nil
+		case "loop":
+			return []agentonvif.ONVIFEvents{{Key: "two", Type: "output", Value: "false", Timestamp: 2}}, nil
+		default:
+			t.Fatalf("unexpected pull point address %q", pullPointAddress)
+			return nil, nil
+		}
+	}
+	heartbeatUnsubscribePullPoint = func(*goonvif.Device, string) error { return nil }
+
+	state := newHeartbeatONVIFState()
+	payload := getHeartbeatONVIFPayload(camera, state)
+	if payload.presets != "false" {
+		t.Fatalf("payload.presets = %q, want false on transient preset failure", payload.presets)
+	}
+	payload = getHeartbeatONVIFPayload(camera, state)
+	if payload.presets != "true" {
+		t.Fatalf("payload.presets = %q, want true after retry", payload.presets)
+	}
+	if connectCalls != 1 {
+		t.Fatalf("connectCalls = %d, want 1", connectCalls)
+	}
+	if ptzConfigCalls != 2 || presetCalls != 2 {
+		t.Fatalf("static failures were not retried on heartbeat cadence: ptzConfig=%d presets=%d", ptzConfigCalls, presetCalls)
+	}
+	if createCalls != 3 || eventCalls != 3 {
+		t.Fatalf("unexpected event behavior during retry: create=%d events=%d", createCalls, eventCalls)
+	}
+}
+
+func TestHeartbeatONVIFStateReleasesSubscriptionWhenDisabled(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	device := newTestONVIFDevice()
+	state := newHeartbeatONVIFState()
+	state.cameraConfiguration = models.IPCamera{ONVIFXAddr: "http://camera/onvif"}
+	state.cameraKey = heartbeatONVIFCameraKey(state.cameraConfiguration)
+	state.device = device
+	state.loopPullPoint = "loop"
+
+	var unsubscribed []string
+	heartbeatUnsubscribePullPoint = func(gotDevice *goonvif.Device, pullPointAddress string) error {
+		if gotDevice != device {
+			t.Fatal("unsubscribe used a different ONVIF device")
+		}
+		unsubscribed = append(unsubscribed, pullPointAddress)
+		return nil
+	}
+
+	state.prepare(models.IPCamera{})
+
+	if len(unsubscribed) != 1 || unsubscribed[0] != "loop" {
+		t.Fatalf("unsubscribed = %v, want [loop]", unsubscribed)
+	}
+	if state.device != nil || state.loopPullPoint != "" {
+		t.Fatalf("disabled state retained device or pull point: %+v", state)
+	}
+}
+
+func TestHeartbeatONVIFPayloadDoesNotCreateSubscriptionsWhenConnectFails(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	var createCalls int
+
+	heartbeatConnectToONVIFDevice = func(*models.IPCamera) (*goonvif.Device, goonvifdevice.GetCapabilitiesResponse, error) {
+		return nil, goonvifdevice.GetCapabilitiesResponse{}, errors.New("connect failed")
+	}
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		createCalls++
+		return "unexpected", nil
+	}
+
+	payload := getHeartbeatONVIFPayload(models.IPCamera{
+		ONVIFXAddr:    "http://camera/onvif",
+		ONVIFUsername: "operator",
+		ONVIFPassword: "secret",
+	}, newHeartbeatONVIFState())
+
+	if createCalls != 0 {
+		t.Fatalf("createCalls = %d, want 0", createCalls)
+	}
+	assertDefaultHeartbeatONVIFPayload(t, payload)
+}
+
+func TestHeartbeatONVIFPayloadReconnectFailureInvalidatesCache(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	device1 := newTestONVIFDevice()
+	device2 := newTestONVIFDevice()
+	camera := models.IPCamera{
+		ONVIFXAddr:    "http://camera/onvif",
+		ONVIFUsername: "operator",
+		ONVIFPassword: "secret",
+	}
+	var connectCalls, ptzConfigCalls, presetCalls, createCalls int
+	var unsubscribed []string
+
+	heartbeatConnectToONVIFDevice = func(*models.IPCamera) (*goonvif.Device, goonvifdevice.GetCapabilitiesResponse, error) {
+		connectCalls++
+		switch connectCalls {
+		case 1:
+			return device1, goonvifdevice.GetCapabilitiesResponse{}, nil
+		case 2:
+			return device2, goonvifdevice.GetCapabilitiesResponse{}, nil
+		default:
+			t.Fatalf("unexpected connect call %d", connectCalls)
+			return nil, goonvifdevice.GetCapabilitiesResponse{}, nil
+		}
+	}
+	heartbeatGetPTZConfigurationsFromDevice = func(*goonvif.Device) (goonvifptz.GetConfigurationsResponse, error) {
+		ptzConfigCalls++
+		return goonvifptz.GetConfigurationsResponse{}, nil
+	}
+	heartbeatGetPTZFunctionsFromDevice = func(goonvifptz.GetConfigurationsResponse) ([]string, bool, bool) {
+		return nil, true, false
+	}
+	heartbeatGetPresetsFromDevice = func(*goonvif.Device) ([]models.OnvifActionPreset, error) {
+		presetCalls++
+		return []models.OnvifActionPreset{{Name: "Preset", Token: "1"}}, nil
+	}
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		createCalls++
+		switch createCalls {
+		case 1:
+			return "initial-1", nil
+		case 2:
+			return "loop-1", nil
+		case 3:
+			return "initial-2", nil
+		case 4:
+			return "initial-3", nil
+		case 5:
+			return "loop-3", nil
+		default:
+			t.Fatalf("unexpected create pull point call %d", createCalls)
+			return "", nil
+		}
+	}
+	heartbeatGetEventMessages = func(_ *goonvif.Device, pullPointAddress string) ([]agentonvif.ONVIFEvents, error) {
+		switch pullPointAddress {
+		case "initial-1":
+			return []agentonvif.ONVIFEvents{{Key: "before", Type: "input", Value: "true", Timestamp: 1}}, nil
+		case "initial-2":
+			return []agentonvif.ONVIFEvents{{Key: "during", Type: "input", Value: "true", Timestamp: 2}}, nil
+		case "loop-1":
+			return nil, errors.New("pull failed")
+		case "initial-3":
+			return []agentonvif.ONVIFEvents{{Key: "after", Type: "input", Value: "false", Timestamp: 2}}, nil
+		default:
+			t.Fatalf("unexpected pull point address %q", pullPointAddress)
+			return nil, nil
+		}
+	}
+	heartbeatUnsubscribePullPoint = func(_ *goonvif.Device, pullPointAddress string) error {
+		unsubscribed = append(unsubscribed, pullPointAddress)
+		return nil
+	}
+
+	state := newHeartbeatONVIFState()
+	_ = getHeartbeatONVIFPayload(camera, state)
+	payload := getHeartbeatONVIFPayload(camera, state)
+	wantEventsDuringFailure := mustJSONMarshal(t, []agentonvif.ONVIFEvents{{Key: "during", Type: "input", Value: "true", Timestamp: 2}})
+	if string(payload.eventsList) != string(wantEventsDuringFailure) {
+		t.Fatalf("payload.eventsList after operation failure = %s, want %s", payload.eventsList, wantEventsDuringFailure)
+	}
+	payload = getHeartbeatONVIFPayload(camera, state)
+
+	if ptzConfigCalls != 2 || presetCalls != 2 {
+		t.Fatalf("reconnect did not refresh static state: ptzConfig=%d presets=%d", ptzConfigCalls, presetCalls)
+	}
+	if connectCalls != 2 {
+		t.Fatalf("connectCalls = %d, want 2 after reconnect", connectCalls)
+	}
+	if createCalls != 5 {
+		t.Fatalf("reconnect did not recreate subscriptions: create=%d", createCalls)
+	}
+	if countString(unsubscribed, "loop-1") != 1 {
+		t.Fatalf("operation failure cleanup mismatch, unsubscribed=%v", unsubscribed)
+	}
+	wantEvents := mustJSONMarshal(t, []agentonvif.ONVIFEvents{{Key: "after", Type: "input", Value: "false", Timestamp: 2}})
+	if string(payload.eventsList) != string(wantEvents) {
+		t.Fatalf("payload.eventsList = %s, want %s", payload.eventsList, wantEvents)
+	}
+}
+
+func TestHeartbeatONVIFPayloadKeepsCachedConnectionWhenInitialStateFetchFails(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	device := newTestONVIFDevice()
+	camera := models.IPCamera{
+		ONVIFXAddr:    "http://camera/onvif",
+		ONVIFUsername: "operator",
+		ONVIFPassword: "secret",
+	}
+	loopEvents := []agentonvif.ONVIFEvents{{Key: "input-1", Type: "input", Value: "true", Timestamp: 1}}
+
+	state := newHeartbeatONVIFState()
+	state.cameraConfiguration = camera
+	state.cameraKey = heartbeatONVIFCameraKey(camera)
+	state.device = device
+	state.loopPullPoint = "loop"
+	state.staticLoaded = true
+	state.staticPayload.enabled = "true"
+
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		return "", errors.New("temporary initial-state failure")
+	}
+	heartbeatGetEventMessages = func(_ *goonvif.Device, pullPointAddress string) ([]agentonvif.ONVIFEvents, error) {
+		if pullPointAddress != "loop" {
+			t.Fatalf("unexpected pull point address %q", pullPointAddress)
+		}
+		return loopEvents, nil
+	}
+
+	payload := getHeartbeatONVIFPayload(camera, state)
+
+	wantEvents := mustJSONMarshal(t, loopEvents)
+	if string(payload.eventsList) != string(wantEvents) {
+		t.Fatalf("payload.eventsList = %s, want %s", payload.eventsList, wantEvents)
+	}
+	if state.device != device || state.loopPullPoint != "loop" || !state.staticLoaded {
+		t.Fatalf("temporary failure invalidated healthy cached state: %+v", state)
+	}
+}
+
+func restoreHeartbeatONVIFStubs(t *testing.T) {
+	t.Helper()
+
+	originalConnect := heartbeatConnectToONVIFDevice
+	originalCreate := heartbeatCreatePullPointSubscription
+	originalDigitalInputs := heartbeatGetDigitalInputs
+	originalEvents := heartbeatGetEventMessages
+	originalPresets := heartbeatGetPresetsFromDevice
+	originalPTZConfigurations := heartbeatGetPTZConfigurationsFromDevice
+	originalPTZFunctions := heartbeatGetPTZFunctionsFromDevice
+	originalRelayOutputs := heartbeatGetRelayOutputs
+	originalUnsubscribe := heartbeatUnsubscribePullPoint
+
+	t.Cleanup(func() {
+		heartbeatConnectToONVIFDevice = originalConnect
+		heartbeatCreatePullPointSubscription = originalCreate
+		heartbeatGetDigitalInputs = originalDigitalInputs
+		heartbeatGetEventMessages = originalEvents
+		heartbeatGetPresetsFromDevice = originalPresets
+		heartbeatGetPTZConfigurationsFromDevice = originalPTZConfigurations
+		heartbeatGetPTZFunctionsFromDevice = originalPTZFunctions
+		heartbeatGetRelayOutputs = originalRelayOutputs
+		heartbeatUnsubscribePullPoint = originalUnsubscribe
+	})
+}
+
+func mustJSONMarshal(t *testing.T, v interface{}) []byte {
+	t.Helper()
+
+	b, err := json.Marshal(v)
+	if err != nil {
+		t.Fatalf("json.Marshal() error = %v", err)
+	}
+	return b
+}
+
+func assertDefaultHeartbeatONVIFPayload(t *testing.T, payload heartbeatONVIFPayload) {
+	t.Helper()
+
+	defaultPayload := defaultHeartbeatONVIFPayload()
+	if payload.enabled != defaultPayload.enabled ||
+		payload.zoom != defaultPayload.zoom ||
+		payload.panTilt != defaultPayload.panTilt ||
+		payload.presets != defaultPayload.presets ||
+		string(payload.presetsList) != string(defaultPayload.presetsList) ||
+		string(payload.eventsList) != string(defaultPayload.eventsList) {
+		t.Fatalf("payload = %+v, want default payload", payload)
+	}
+}
+
+func newTestONVIFDevice() *goonvif.Device {
+	return &goonvif.Device{}
+}
+
+func countString(values []string, want string) int {
+	count := 0
+	for _, value := range values {
+		if value == want {
+			count++
+		}
+	}
+	return count
+}

machinery/src/cloud/dropbox.go

@@ -11,8 +11,8 @@ import (
 	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
 	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/users"
 	"github.com/gin-gonic/gin"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 // UploadDropbox uploads the file to your Dropbox account using the access token and directory.
@@ -30,13 +30,13 @@ func UploadDropbox(configuration *models.Configuration, fileName string) (bool,
 
 	if token == "" {
 		err := "UploadDropbox: Dropbox not properly configured"
-		log.Log.Info(err)
+		log.Info(err)
 		return false, true, errors.New(err)
 	}
 
 	// Upload to Dropbox
-	log.Log.Info("UploadDropbox: Uploading to Dropbox")
-	log.Log.Info("UploadDropbox: Upload started for " + fileName)
+	log.Info("UploadDropbox: Uploading to Dropbox")
+	log.Info("UploadDropbox: Upload started for " + fileName)
 	fullname := "data/recordings/" + fileName
 
 	dConfig := dropbox.Config{
@@ -48,7 +48,7 @@ func UploadDropbox(configuration *models.Configuration, fileName string) (bool,
 	if file != nil {
 		defer func() {
 			if cerr := file.Close(); cerr != nil {
-				log.Log.Error("UploadDropbox: Error closing file: " + cerr.Error())
+				log.Error("UploadDropbox: Error closing file: " + cerr.Error())
 			}
 		}()
 	}
@@ -68,15 +68,15 @@ func UploadDropbox(configuration *models.Configuration, fileName string) (bool,
 		}, file)
 
 		if err != nil {
-			log.Log.Error("UploadDropbox: Error uploading file: " + err.Error())
+			log.Error("UploadDropbox: Error uploading file: " + err.Error())
 			return false, false, err
 		}
 
-		log.Log.Info("UploadDropbox: File uploaded successfully, " + res.Name)
+		log.Info("UploadDropbox: File uploaded successfully, " + res.Name)
 		return true, true, nil
 	}
 
-	log.Log.Error("UploadDropbox: Error opening file: " + err.Error())
+	log.Error("UploadDropbox: Error opening file: " + err.Error())
 	return false, true, err
 }
 

machinery/src/cloud/kerberos_hub.go

@@ -3,12 +3,13 @@ package cloud
 import (
 	"crypto/tls"
 	"errors"
+	"fmt"
 	"io/ioutil"
 	"net/http"
 	"os"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 func UploadKerberosHub(configuration *models.Configuration, fileName string) (bool, bool, error) {
@@ -19,7 +20,7 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 		config.HubPrivateKey == "" ||
 		config.S3.Region == "" {
 		err := "UploadKerberosHub: Kerberos Hub not properly configured."
-		log.Log.Info(err)
+		log.Info(err)
 		return false, false, errors.New(err)
 	}
 
@@ -32,8 +33,10 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 	// - Number of changes
 	// - Token
 
-	log.Log.Info("UploadKerberosHub: Uploading to Kerberos Hub (" + config.HubURI + ")")
-	log.Log.Info("UploadKerberosHub: Upload started for " + fileName)
+	log.WithFields(log.Fields{
+		"component": "kerberos_hub",
+		"event":     "upload_started",
+	}).Info("Uploading recording to Kerberos Hub")
 
 	// Prefer the resumable (tus) upload when enabled (the default). Kerberos Hub
 	// authenticates the agent with its Hub public/private key and proxies the
@@ -44,17 +47,35 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 		uploaded, _, supported, body, rerr := uploadHubResumable(&config, fileName, "UploadKerberosHub", "hub")
 		if supported {
 			if uploaded {
-				log.Log.Info("UploadKerberosHub: Upload Finished (resumable), " + body)
+				log.WithFields(log.Fields{
+					"component":      "kerberos_hub",
+					"event":          "upload_completed",
+					"response_bytes": len(body),
+					"transport":      "tus",
+				}).Info("Hub upload completed")
 				return true, true, nil
 			}
 			if rerr != nil {
-				log.Log.Info("UploadKerberosHub: resumable upload failed, " + rerr.Error())
+				log.WithError(rerr).WithFields(log.Fields{
+					"component": "kerberos_hub",
+					"event":     "upload_failed",
+					"transport": "tus",
+				}).Error("Hub upload failed")
 			} else {
-				log.Log.Info("UploadKerberosHub: resumable upload incomplete, " + body)
+				log.WithFields(log.Fields{
+					"component":      "kerberos_hub",
+					"event":          "upload_incomplete",
+					"response_bytes": len(body),
+					"transport":      "tus",
+				}).Warn("Hub upload incomplete")
 			}
 			return false, true, rerr
 		}
-		log.Log.Info("UploadKerberosHub: resumable (tus) endpoint not available, falling back to legacy upload")
+		log.WithFields(log.Fields{
+			"component": "kerberos_hub",
+			"event":     "upload_transport_fallback",
+			"transport": "http",
+		}).Info("Resumable Hub upload unavailable; using legacy upload")
 	}
 
 	fullname := "data/recordings/" + fileName
@@ -66,7 +87,7 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 	}
 	if err != nil {
 		err := "UploadKerberosHub: Upload Failed, file doesn't exists anymore."
-		log.Log.Info(err)
+		log.Info(err)
 		return false, false, errors.New(err)
 	}
 
@@ -74,9 +95,11 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 	// There might be different reasons like (muted, read-only..)
 	req, err := http.NewRequest("HEAD", config.HubURI+"/storage/upload", nil)
 	if err != nil {
-		errorMessage := "UploadKerberosHub: error reading HEAD request, " + config.HubURI + "/storage: " + err.Error()
-		log.Log.Error(errorMessage)
-		return false, true, errors.New(errorMessage)
+		log.WithError(err).WithFields(log.Fields{
+			"component": "kerberos_hub",
+			"event":     "authorization_request_creation_failed",
+		}).Error("Failed to create Hub upload authorization request")
+		return false, true, fmt.Errorf("create Hub upload authorization request: %w", err)
 	}
 
 	req.Header.Set("X-Kerberos-Storage-FileName", fileName)
@@ -102,25 +125,31 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 		defer resp.Body.Close()
 	}
 
-	if err == nil {
-		if resp != nil {
-			if err == nil {
-				if resp.StatusCode == 200 {
-					log.Log.Info("UploadKerberosHub: Upload allowed using the credentials provided (" + config.HubKey + ", " + config.HubPrivateKey + ")")
-				} else {
-					log.Log.Info("UploadKerberosHub: Upload NOT allowed using the credentials provided (" + config.HubKey + ", " + config.HubPrivateKey + ")")
-					return false, true, nil
-				}
-			}
+	if err == nil && resp != nil {
+		if resp.StatusCode == 200 {
+			log.WithFields(log.Fields{
+				"component":   "kerberos_hub",
+				"event":       "upload_authorized",
+				"status_code": resp.StatusCode,
+			}).Debug("Hub upload authorized")
+		} else {
+			log.WithFields(log.Fields{
+				"component":   "kerberos_hub",
+				"event":       "upload_rejected",
+				"status_code": resp.StatusCode,
+			}).Warn("Hub upload rejected")
+			return false, true, nil
 		}
 	}
 
 	// Now we know we are allowed to upload to the hub, we can start uploading.
 	req, err = http.NewRequest("POST", config.HubURI+"/storage/upload", file)
 	if err != nil {
-		errorMessage := "UploadKerberosHub: error reading POST request, " + config.KStorage.URI + "/storage/upload: " + err.Error()
-		log.Log.Error(errorMessage)
-		return false, true, errors.New(errorMessage)
+		log.WithError(err).WithFields(log.Fields{
+			"component": "kerberos_hub",
+			"event":     "upload_request_creation_failed",
+		}).Error("Failed to create Hub upload request")
+		return false, true, fmt.Errorf("create Hub upload request: %w", err)
 	}
 	req.Header.Set("Content-Type", "video/mp4")
 	req.Header.Set("X-Kerberos-Storage-FileName", fileName)
@@ -140,19 +169,36 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 			body, err := ioutil.ReadAll(resp.Body)
 			if err == nil {
 				if resp.StatusCode == 200 {
-					log.Log.Info("UploadKerberosHub: Upload Finished, " + resp.Status + ".")
+					log.WithFields(log.Fields{
+						"component":   "kerberos_hub",
+						"event":       "upload_completed",
+						"status_code": resp.StatusCode,
+						"transport":   "http",
+					}).Info("Hub upload completed")
 					return true, true, nil
 				} else {
-					log.Log.Info("UploadKerberosHub: Upload Failed, " + resp.Status + ", " + string(body))
+					log.WithFields(log.Fields{
+						"component":      "kerberos_hub",
+						"event":          "upload_rejected",
+						"response_bytes": len(body),
+						"status_code":    resp.StatusCode,
+						"transport":      "http",
+					}).Warn("Hub upload rejected")
 					return false, true, nil
 				}
 			}
 		}
 	}
 
-	errorMessage := "UploadKerberosHub: Upload Failed, " + err.Error()
-	log.Log.Info(errorMessage)
-	return false, true, errors.New(errorMessage)
+	if err == nil {
+		err = errors.New("Hub upload failed without a response")
+	}
+	log.WithError(err).WithFields(log.Fields{
+		"component": "kerberos_hub",
+		"event":     "upload_failed",
+		"transport": "http",
+	}).Error("Hub upload failed")
+	return false, true, fmt.Errorf("Hub upload failed: %w", err)
 }
 
 // stripHubCredentialsOnCrossHostRedirect removes the custom Kerberos Hub

machinery/src/cloud/kerberos_vault.go

@@ -9,8 +9,8 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 // We will count the number of retries we have done.
@@ -27,7 +27,7 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string) (
 		config.KStorage.Directory == "" ||
 		config.KStorage.URI == "" {
 		err := "UploadKerberosVault: Kerberos Vault not properly configured"
-		log.Log.Info(err)
+		log.Info(err)
 		return false, false, errors.New(err)
 	}
 
@@ -35,8 +35,13 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string) (
 	// This can happen when the file was already removed (e.g. cleanup, or an
 	// earlier successful upload). Skip it so the watcher drops the marker
 	// instead of retrying indefinitely.
-	if _, err := os.Stat("data/recordings/" + fileName); err != nil {
-		log.Log.Info("UploadKerberosVault: skipping " + fileName + ", file doesn't exist anymore")
+	info, err := os.Stat("data/recordings/" + fileName)
+	if err != nil {
+		log.Info("UploadKerberosVault: skipping " + fileName + ", file doesn't exist anymore")
+		return false, false, nil
+	}
+	if info.Size() == 0 {
+		log.Warn("UploadKerberosVault: skipping " + fileName + ", recording is empty")
 		return false, false, nil
 	}
 
@@ -49,8 +54,11 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string) (
 	// - Number of changes
 	// - Token
 	// KerberosCloud, this means storage is disabled and proxy enabled.
-	log.Log.Info("UploadKerberosVault: Uploading to Kerberos Vault (" + config.KStorage.URI + ")")
-	log.Log.Info("UploadKerberosVault: Upload started for " + fileName)
+	log.WithFields(log.Fields{
+		"component": "kerberos_vault",
+		"event":     "upload_started",
+		"storage":   "primary",
+	}).Info("Vault upload started")
 
 	publicKey := config.KStorage.CloudKey
 	if config.HubKey != "" {
@@ -62,14 +70,27 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string) (
 		uploaded, responded, body, err := sendToVault(*config.KStorage, publicKey, config.Key, fileName, "UploadKerberosVault", "primary")
 		if uploaded {
 			kstorageRetryCount = 0
-			log.Log.Info("UploadKerberosVault: Upload Finished, " + body)
+			log.WithFields(log.Fields{
+				"component": "kerberos_vault",
+				"event":     "upload_completed",
+				"storage":   "primary",
+			}).Info("Vault upload completed")
 			return true, true, nil
 		}
 
 		if err != nil {
-			log.Log.Info("UploadKerberosVault: Upload Failed, " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component": "kerberos_vault",
+				"event":     "upload_failed",
+				"storage":   "primary",
+			}).Error("Vault upload failed")
 		} else {
-			log.Log.Info("UploadKerberosVault: Upload Failed, " + body)
+			log.WithFields(log.Fields{
+				"component":      "kerberos_vault",
+				"event":          "upload_rejected",
+				"response_bytes": len(body),
+				"storage":        "primary",
+			}).Warn("Vault upload rejected")
 		}
 
 		// We only advance the retry policy when the vault gave a definitive
@@ -91,26 +112,43 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string) (
 		config.KStorageSecondary.SecretAccessKey == "" ||
 		config.KStorageSecondary.Directory == "" ||
 		config.KStorageSecondary.URI == "" {
-		log.Log.Info("UploadKerberosVault (Secondary): Secondary Kerberos Vault not properly configured.")
+		log.Info("UploadKerberosVault (Secondary): Secondary Kerberos Vault not properly configured.")
 	} else {
 
 		if kstorageRetryCount < config.KStorage.MaxRetries {
-			log.Log.Info("UploadKerberosVault (Secondary): Do not upload to secondary storage, we are still in retry policy.")
+			log.Info("UploadKerberosVault (Secondary): Do not upload to secondary storage, we are still in retry policy.")
 			return false, true, nil
 		}
 
-		log.Log.Info("UploadKerberosVault (Secondary): Uploading to Secondary Kerberos Vault (" + config.KStorageSecondary.URI + ")")
+		log.WithFields(log.Fields{
+			"component": "kerberos_vault",
+			"event":     "upload_started",
+			"storage":   "secondary",
+		}).Info("Vault upload started")
 
 		uploaded, _, body, err := sendToVault(*config.KStorageSecondary, publicKey, config.Key, fileName, "UploadKerberosVault (Secondary)", "secondary")
 		if uploaded {
-			log.Log.Info("UploadKerberosVault (Secondary): Upload Finished to secondary, " + body)
+			log.WithFields(log.Fields{
+				"component": "kerberos_vault",
+				"event":     "upload_completed",
+				"storage":   "secondary",
+			}).Info("Vault upload completed")
 			return true, true, nil
 		}
 
 		if err != nil {
-			log.Log.Info("UploadKerberosVault (Secondary): Upload Failed to secondary, " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component": "kerberos_vault",
+				"event":     "upload_failed",
+				"storage":   "secondary",
+			}).Error("Vault upload failed")
 		} else {
-			log.Log.Info("UploadKerberosVault (Secondary): Upload Failed to secondary, " + body)
+			log.WithFields(log.Fields{
+				"component":      "kerberos_vault",
+				"event":          "upload_rejected",
+				"response_bytes": len(body),
+				"storage":        "secondary",
+			}).Warn("Vault upload rejected")
 		}
 	}
 
@@ -131,7 +169,7 @@ func sendToVault(vault models.KStorage, publicKey, deviceKey, fileName, label, s
 		if supported {
 			return uploaded, responded, body, err
 		}
-		log.Log.Info(label + ": resumable (tus) endpoint not available, falling back to legacy upload")
+		log.Info(label + ": resumable (tus) endpoint not available, falling back to legacy upload")
 	}
 	return uploadVaultLegacy(vault, publicKey, deviceKey, fileName, label)
 }
@@ -148,7 +186,7 @@ func uploadVaultLegacy(vault models.KStorage, publicKey, deviceKey, fileName, la
 	}
 	if err != nil {
 		msg := label + ": Upload Failed, file doesn't exists anymore"
-		log.Log.Info(msg)
+		log.Info(msg)
 		return false, false, "", errors.New(msg)
 	}
 
@@ -160,7 +198,7 @@ func uploadVaultLegacy(vault models.KStorage, publicKey, deviceKey, fileName, la
 	req, err := http.NewRequest("POST", uri+"/storage", file)
 	if err != nil {
 		errorMessage := label + ": error reading request, " + uri + "/storage: " + err.Error()
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, false, "", errors.New(errorMessage)
 	}
 	req.Header.Set("Content-Type", "video/mp4")

machinery/src/cloud/livehls.go

@@ -7,9 +7,9 @@ import (
 	mqtt "github.com/eclipse/paho.mqtt.golang"
 
 	"github.com/kerberos-io/agent/machinery/src/cloud/livehls"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
+	log "github.com/sirupsen/logrus"
 )
 
 // hlsViewerTimeoutSeconds is how long the agent keeps shipping live HLS segments
@@ -45,22 +45,29 @@ const hlsReadyReannounceSeconds = 2
 // trades a little idle CPU for a near-instant "requesting stream", so viewers no
 // longer wait a full GOP for the first segment to be cut. Set
 // AGENT_LIVE_HLS_PREWARM=false to fall back to the lazy on-demand path above.
-func HandleLiveStreamHLS(configuration *models.Configuration, communication *models.Communication, mqttClient mqtt.Client, subStreamEnabled bool) {
+func HandleLiveStreamHLS(
+	configuration *models.Configuration,
+	communication *models.Communication,
+	mqttClient mqtt.Client,
+	subStreamEnabled bool,
+	mainQueue *packets.Queue,
+	subQueue *packets.Queue,
+) {
 
-	log.Log.Debug("cloud.HandleLiveStreamHLS(): started")
+	log.Debug("cloud.HandleLiveStreamHLS(): started")
 
 	config := configuration.Config
 
 	if config.Offline == "true" {
-		log.Log.Debug("cloud.HandleLiveStreamHLS(): stopping as Offline is enabled.")
+		log.Debug("cloud.HandleLiveStreamHLS(): stopping as Offline is enabled.")
 		return
 	}
 	if config.Capture.Liveview == "false" {
-		log.Log.Debug("cloud.HandleLiveStreamHLS(): stopping as Liveview is disabled.")
+		log.Debug("cloud.HandleLiveStreamHLS(): stopping as Liveview is disabled.")
 		return
 	}
 	if config.HubURI == "" || config.HubKey == "" {
-		log.Log.Debug("cloud.HandleLiveStreamHLS(): stopping as the Hub is not configured (HubURI/HubKey).")
+		log.Debug("cloud.HandleLiveStreamHLS(): stopping as the Hub is not configured (HubURI/HubKey).")
 		return
 	}
 
@@ -88,8 +95,8 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 	// mp4ff's strict parser rejects).
 	requestedQuality := models.StreamQualityAuto
 	useSub := models.SelectSubStreamForQuality(config, requestedQuality, subStreamEnabled)
-	source := buildHLSSource(config, communication, useSub)
-	log.Log.Info("cloud.HandleLiveStreamHLS(): serving live HLS from the " + source.label + " stream")
+	source := buildHLSSource(config, mainQueue, subQueue, useSub)
+	log.Info("cloud.HandleLiveStreamHLS(): serving live HLS from the " + source.label + " stream")
 
 	// prewarm keeps a single long-lived session muxing into an in-memory ring
 	// buffer while idle and flushes it the instant a viewer arrives, eliminating
@@ -97,9 +104,9 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 	// to fall back to the lazy on-demand path.
 	prewarm := os.Getenv("AGENT_LIVE_HLS_PREWARM") != "false"
 	if prewarm {
-		log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS prewarm ENABLED (set AGENT_LIVE_HLS_PREWARM=false to disable)")
+		log.Info("cloud.HandleLiveStreamHLS(): live HLS prewarm ENABLED (set AGENT_LIVE_HLS_PREWARM=false to disable)")
 	} else {
-		log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS prewarm DISABLED (AGENT_LIVE_HLS_PREWARM=false)")
+		log.Info("cloud.HandleLiveStreamHLS(): live HLS prewarm DISABLED (AGENT_LIVE_HLS_PREWARM=false)")
 	}
 
 	// lowLatency enables LL-HLS: each segment is sliced into CMAF parts shipped the
@@ -109,9 +116,9 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 	partTargetMs := uint64(0)
 	if os.Getenv("AGENT_LIVE_HLS_LOW_LATENCY") != "false" {
 		partTargetMs = livehls.DefaultPartTargetMs
-		log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS low-latency (LL-HLS) ENABLED (set AGENT_LIVE_HLS_LOW_LATENCY=false to disable)")
+		log.Info("cloud.HandleLiveStreamHLS(): live HLS low-latency (LL-HLS) ENABLED (set AGENT_LIVE_HLS_LOW_LATENCY=false to disable)")
 	} else {
-		log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS low-latency (LL-HLS) DISABLED (AGENT_LIVE_HLS_LOW_LATENCY=false)")
+		log.Info("cloud.HandleLiveStreamHLS(): live HLS low-latency (LL-HLS) DISABLED (AGENT_LIVE_HLS_LOW_LATENCY=false)")
 	}
 
 	var session *livehls.Session
@@ -123,6 +130,9 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 
 	for cursorError == nil {
 		pkt, cursorError = source.cursor.ReadPacket()
+		if cursorError != nil {
+			break
+		}
 
 		now := time.Now().Unix()
 		select {
@@ -156,9 +166,9 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 				_ = session.Close()
 				session = nil
 			}
-			source = buildHLSSource(config, communication, useSub)
+			source = buildHLSSource(config, mainQueue, subQueue, useSub)
 			lastReadyAnnounce = 0
-			log.Log.Info("cloud.HandleLiveStreamHLS(): switched live HLS to the " + source.label + " stream (quality=" + requestedQuality + ")")
+			log.Info("cloud.HandleLiveStreamHLS(): switched live HLS to the " + source.label + " stream (quality=" + requestedQuality + ")")
 			continue
 		}
 
@@ -184,11 +194,22 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 					StartBuffering: true,
 				})
 				session.SetOnReady(func(sessionID string) {
-					log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS session ready, announcing " + sessionID)
+					log.WithFields(log.Fields{
+						"component":  "cloud/livehls",
+						"event":      "session_ready",
+						"session_id": sessionID,
+					}).Info("Live HLS session ready")
 					publishHLSReady(configuration, mqttClient, hubKey, deviceId, sessionID)
 					lastReadyAnnounce = time.Now().Unix()
 				})
-				log.Log.Info("cloud.HandleLiveStreamHLS(): prewarming live HLS session " + session.SessionID())
+				session.SetOnFailure(func(sessionID, reason string) {
+					publishHLSFailure(configuration, mqttClient, hubKey, deviceId, sessionID, reason)
+				})
+				log.WithFields(log.Fields{
+					"component":  "cloud/livehls",
+					"event":      "session_prewarming",
+					"session_id": session.SessionID(),
+				}).Info("Prewarming live HLS session")
 			}
 
 			if viewerActive {
@@ -207,7 +228,7 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 
 			if len(pkt.Data) > 0 && pkt.IsVideo {
 				if err := session.WritePacket(pkt); err != nil {
-					log.Log.Error("cloud.HandleLiveStreamHLS(): " + err.Error())
+					log.Error("cloud.HandleLiveStreamHLS(): " + err.Error())
 				}
 			}
 			continue
@@ -217,7 +238,12 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 			// No viewer: stop and discard the session so we stop shipping segments.
 			if session != nil {
 				_ = session.Close()
-				log.Log.Info("cloud.HandleLiveStreamHLS(): no active viewers, stopped live HLS session " + session.SessionID())
+				log.WithFields(log.Fields{
+					"component":  "cloud/livehls",
+					"event":      "session_stopped",
+					"reason":     "no_active_viewers",
+					"session_id": session.SessionID(),
+				}).Info("Live HLS session stopped")
 				session = nil
 			}
 			continue
@@ -243,22 +269,33 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 				PartTargetMs: partTargetMs,
 			})
 			session.SetOnReady(func(sessionID string) {
-				log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS session ready, announcing " + sessionID)
+				log.WithFields(log.Fields{
+					"component":  "cloud/livehls",
+					"event":      "session_ready",
+					"session_id": sessionID,
+				}).Info("Live HLS session ready")
 				publishHLSReady(configuration, mqttClient, hubKey, deviceId, sessionID)
 				lastReadyAnnounce = time.Now().Unix()
 			})
-			log.Log.Info("cloud.HandleLiveStreamHLS(): started live HLS session " + session.SessionID())
+			session.SetOnFailure(func(sessionID, reason string) {
+				publishHLSFailure(configuration, mqttClient, hubKey, deviceId, sessionID, reason)
+			})
+			log.WithFields(log.Fields{
+				"component":  "cloud/livehls",
+				"event":      "session_started",
+				"session_id": session.SessionID(),
+			}).Info("Live HLS session started")
 		}
 
 		if err := session.WritePacket(pkt); err != nil {
-			log.Log.Error("cloud.HandleLiveStreamHLS(): " + err.Error())
+			log.Error("cloud.HandleLiveStreamHLS(): " + err.Error())
 		}
 	}
 
 	if session != nil {
 		_ = session.Close()
 	}
-	log.Log.Debug("cloud.HandleLiveStreamHLS(): finished")
+	log.Debug("cloud.HandleLiveStreamHLS(): finished")
 }
 
 // publishHLSReady announces, over MQTT, that a live HLS session is available so
@@ -278,9 +315,48 @@ func publishHLSReady(configuration *models.Configuration, mqttClient mqtt.Client
 	payload, err := models.PackageMQTTMessage(configuration, message)
 	if err == nil {
 		mqttClient.Publish("kerberos/hub/"+hubKey, 0, false, payload)
-		log.Log.Info("cloud.HandleLiveStreamHLS(): announced live HLS session " + sessionID)
+		log.WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "session_announced",
+			"session_id": sessionID,
+		}).Info("Live HLS session announced")
 	} else {
-		log.Log.Error("cloud.HandleLiveStreamHLS(): failed to package receive-hls-ready message: " + err.Error())
+		log.WithError(err).WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "session_announcement_failed",
+			"session_id": sessionID,
+		}).Error("Failed to package live HLS session announcement")
+	}
+}
+
+func publishHLSFailure(configuration *models.Configuration, mqttClient mqtt.Client, hubKey, deviceId, sessionID, reason string) {
+	valueMap := map[string]interface{}{
+		"session": sessionID,
+		"device":  deviceId,
+		"reason":  reason,
+	}
+	message := models.Message{
+		Payload: models.Payload{
+			Action:   "receive-hls-error",
+			DeviceId: deviceId,
+			Value:    valueMap,
+		},
+	}
+	payload, err := models.PackageMQTTMessage(configuration, message)
+	if err == nil {
+		mqttClient.Publish("kerberos/hub/"+hubKey, 0, false, payload)
+		log.WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "session_failure_announced",
+			"reason":     reason,
+			"session_id": sessionID,
+		}).Warn("Live HLS session failure announced")
+	} else {
+		log.WithError(err).WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "session_failure_announcement_failed",
+			"session_id": sessionID,
+		}).Error("Failed to package live HLS session failure")
 	}
 }
 
@@ -302,11 +378,11 @@ type hlsStreamSource struct {
 // is available; otherwise the main (high-resolution) stream is used. A fresh
 // Latest() cursor is created so muxing resumes from the live edge of the chosen
 // stream after a switch.
-func buildHLSSource(config models.Config, communication *models.Communication, useSub bool) hlsStreamSource {
+func buildHLSSource(config models.Config, mainQueue, subQueue *packets.Queue, useSub bool) hlsStreamSource {
 	cam := config.Capture.IPCamera
-	if useSub && communication.SubQueue != nil {
+	if useSub && subQueue != nil {
 		return hlsStreamSource{
-			cursor: communication.SubQueue.Latest(),
+			cursor: subQueue.Latest(),
 			sps:    cam.SubSPSNALUs,
 			pps:    cam.SubPPSNALUs,
 			vps:    cam.SubVPSNALUs,
@@ -316,7 +392,7 @@ func buildHLSSource(config models.Config, communication *models.Communication, u
 		}
 	}
 	return hlsStreamSource{
-		cursor: communication.Queue.Latest(),
+		cursor: mainQueue.Latest(),
 		sps:    cam.SPSNALUs,
 		pps:    cam.PPSNALUs,
 		vps:    cam.VPSNALUs,

machinery/src/cloud/livehls/publisher.go

@@ -24,13 +24,14 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"net/http"
 	"strconv"
 	"strings"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/video"
+	log "github.com/sirupsen/logrus"
 )
 
 const (
@@ -67,6 +68,7 @@ const (
 	// cannot be delivered within roughly its own duration is stale, so the upload
 	// is abandoned (dropped) rather than allowed to back up the pipeline.
 	defaultPublishTimeout = 4 * time.Second
+	maxErrorResponseBytes = 4 << 10
 )
 
 // PublisherConfig carries the hub endpoint and credentials needed to ship live
@@ -214,9 +216,15 @@ func (p *Publisher) post(ctx context.Context, params postParams) error {
 	defer resp.Body.Close()
 
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		return fmt.Errorf("livehls: upload %s rejected: %s", params.name, resp.Status)
+		body, _ := io.ReadAll(io.LimitReader(resp.Body, maxErrorResponseBytes))
+		return fmt.Errorf("livehls: upload %s rejected with status %d (%d response bytes)", params.name, resp.StatusCode, len(body))
 	}
-	log.Log.Debug("livehls.Publisher.post(): shipped " + params.name + " for session " + params.sessionID)
+	log.WithFields(log.Fields{
+		"component":  "cloud/livehls",
+		"event":      "object_uploaded",
+		"object":     params.name,
+		"session_id": params.sessionID,
+	}).Trace("Live HLS object uploaded")
 	return nil
 }
 

machinery/src/cloud/livehls/publisher_test.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"sync"
 	"testing"
 	"time"
@@ -144,12 +145,22 @@ func TestPublisherPublishSegmentSendsSequenceAndDuration(t *testing.T) {
 }
 
 func TestPublisherReturnsErrorOnNon2xx(t *testing.T) {
-	srv, _, _ := newCapturingServer(t, http.StatusInternalServerError)
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusBadRequest)
+		_, _ = w.Write([]byte(`{"error":true,"data":"No user found with this public and private key."}`))
+	}))
+	t.Cleanup(srv.Close)
 	p := testPublisher(srv.URL)
 
 	err := p.PublishSegment(context.Background(), "s", video.LiveSegment{SequenceNumber: 1, Data: []byte("x")})
 	if err == nil {
-		t.Fatal("expected an error on 500 response")
+		t.Fatal("expected an error on 400 response")
+	}
+	if !strings.Contains(err.Error(), "status 400") || !strings.Contains(err.Error(), "71 response bytes") {
+		t.Fatalf("error = %q, want status code and response size", err)
+	}
+	if strings.Contains(err.Error(), "No user found with this public and private key") {
+		t.Fatalf("error exposed Hub response body: %q", err)
 	}
 }
 
@@ -281,7 +292,13 @@ func TestSessionRetriesInitWhenFirstAttemptFails(t *testing.T) {
 	})
 
 	var ready int
+	var failures int
+	var failureReason string
 	sess.SetOnReady(func(string) { ready++ })
+	sess.SetOnFailure(func(_ string, reason string) {
+		failures++
+		failureReason = reason
+	})
 
 	for i := 0; i < 60; i++ {
 		isKey := i%25 == 0
@@ -304,6 +321,9 @@ func TestSessionRetriesInitWhenFirstAttemptFails(t *testing.T) {
 	if ready != 1 {
 		t.Errorf("OnReady fired %d times, want 1", ready)
 	}
+	if failures != 1 || failureReason != "init-upload-failed" {
+		t.Errorf("OnFailure = %d/%q, want 1/init-upload-failed", failures, failureReason)
+	}
 }
 
 // liveTestSPSForSession is the known-good baseline SPS reused across tests.

machinery/src/cloud/livehls/session.go

@@ -8,9 +8,9 @@ import (
 	"sync"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/kerberos-io/agent/machinery/src/video"
+	log "github.com/sirupsen/logrus"
 )
 
 // DefaultTargetSegmentMs is the nominal live segment length. ~2s keeps standard
@@ -46,9 +46,11 @@ type Session struct {
 	// lastInitAt is when the init segment was last (re)uploaded. The init is
 	// re-sent periodically so its short TTL in the hub live window never lapses
 	// mid-session; see refreshInitIfStale.
-	lastInitAt time.Time
-	readyFired bool
-	onReady    func(sessionID string)
+	lastInitAt   time.Time
+	readyFired   bool
+	onReady      func(sessionID string)
+	failureFired bool
+	onFailure    func(sessionID, reason string)
 
 	// uploadsActive gates whether the init and completed segments are shipped to
 	// hub-api. It is true for the default on-demand path. The prewarm path starts
@@ -142,14 +144,25 @@ func NewSession(publisher *Publisher, opts SessionOptions) *Session {
 			return nil
 		}
 		if !s.publishInitIfNeeded() {
-			log.Log.Warning("livehls.Session: dropping segment " +
-				fmt.Sprintf("%d", segment.SequenceNumber) + " because init has not been delivered yet")
+			log.WithFields(log.Fields{
+				"component":        "cloud/livehls",
+				"event":            "segment_dropped",
+				"reason":           "init_unavailable",
+				"segment_sequence": segment.SequenceNumber,
+				"session_id":       s.id,
+			}).Warn("Dropping live HLS segment")
 			return nil
 		}
 		ctx, cancel := s.newContext()
 		defer cancel()
 		if err := s.publisher.PublishSegment(ctx, s.id, segment); err != nil {
-			log.Log.Warning("livehls.Session: " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component":        "cloud/livehls",
+				"event":            "segment_upload_failed",
+				"segment_sequence": segment.SequenceNumber,
+				"session_id":       s.id,
+			}).Warn("Failed to upload live HLS segment")
+			s.fireFailureOnce("segment-upload-failed")
 			return nil
 		}
 		s.fireReadyOnce()
@@ -172,15 +185,27 @@ func NewSession(publisher *Publisher, opts SessionOptions) *Session {
 				return nil
 			}
 			if !s.publishInitIfNeeded() {
-				log.Log.Warning("livehls.Session: dropping part " +
-					fmt.Sprintf("%d.%d", part.SegmentSeq, part.PartIndex) +
-					" because init has not been delivered yet")
+				log.WithFields(log.Fields{
+					"component":        "cloud/livehls",
+					"event":            "part_dropped",
+					"part_index":       part.PartIndex,
+					"reason":           "init_unavailable",
+					"segment_sequence": part.SegmentSeq,
+					"session_id":       s.id,
+				}).Warn("Dropping live HLS part")
 				return nil
 			}
 			ctx, cancel := s.newContext()
 			defer cancel()
 			if err := s.publisher.PublishPart(ctx, s.id, part); err != nil {
-				log.Log.Warning("livehls.Session: " + err.Error())
+				log.WithError(err).WithFields(log.Fields{
+					"component":        "cloud/livehls",
+					"event":            "part_upload_failed",
+					"part_index":       part.PartIndex,
+					"segment_sequence": part.SegmentSeq,
+					"session_id":       s.id,
+				}).Warn("Failed to upload live HLS part")
+				s.fireFailureOnce("part-upload-failed")
 				return nil
 			}
 			s.fireReadyOnce()
@@ -216,6 +241,15 @@ func (s *Session) SetOnReady(fn func(sessionID string)) {
 	s.mu.Unlock()
 }
 
+// SetOnFailure registers a one-shot callback for startup upload failures. The
+// reason is a fixed code rather than an HTTP response body, so credentials or
+// server details cannot leak through MQTT diagnostics.
+func (s *Session) SetOnFailure(fn func(sessionID, reason string)) {
+	s.mu.Lock()
+	s.onFailure = fn
+	s.mu.Unlock()
+}
+
 // prewarmMaxBufferedSegments is how many of the most recent completed segments
 // the prewarm path keeps in memory while idle and flushes to a viewer on arrival.
 // One segment keeps startup instant (the viewer immediately gets a playable
@@ -268,7 +302,13 @@ func (s *Session) SetUploadsActive(active bool) bool {
 		}
 		ctx, cancel := s.newContext()
 		if err := s.publisher.PublishSegment(ctx, s.id, buffered[i]); err != nil {
-			log.Log.Warning("livehls.Session: prewarm flush: " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component":        "cloud/livehls",
+				"event":            "prewarm_segment_upload_failed",
+				"segment_sequence": buffered[i].SequenceNumber,
+				"session_id":       s.id,
+			}).Warn("Failed to upload buffered live HLS segment")
+			s.fireFailureOnce("segment-upload-failed")
 			cancel()
 			continue
 		}
@@ -284,7 +324,14 @@ func (s *Session) SetUploadsActive(active bool) bool {
 		}
 		ctx, cancel := s.newContext()
 		if err := s.publisher.PublishPart(ctx, s.id, bufferedParts[i]); err != nil {
-			log.Log.Warning("livehls.Session: prewarm flush (part): " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component":        "cloud/livehls",
+				"event":            "prewarm_part_upload_failed",
+				"part_index":       bufferedParts[i].PartIndex,
+				"segment_sequence": bufferedParts[i].SegmentSeq,
+				"session_id":       s.id,
+			}).Warn("Failed to upload buffered live HLS part")
+			s.fireFailureOnce("part-upload-failed")
 			cancel()
 			continue
 		}
@@ -381,7 +428,12 @@ func (s *Session) publishInitIfNeeded() bool {
 	ctx, cancel := s.newContext()
 	defer cancel()
 	if err := s.publisher.PublishInit(ctx, s.id, initBytes); err != nil {
-		log.Log.Warning("livehls.Session: init upload failed, will retry: " + err.Error())
+		log.WithError(err).WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "init_upload_failed",
+			"session_id": s.id,
+		}).Warn("Failed to upload live HLS init segment; retrying")
+		s.fireFailureOnce("init-upload-failed")
 		return false
 	}
 
@@ -389,7 +441,11 @@ func (s *Session) publishInitIfNeeded() bool {
 	s.initPublished = true
 	s.lastInitAt = time.Now()
 	s.mu.Unlock()
-	log.Log.Info("livehls.Session: init segment delivered for session " + s.id)
+	log.WithFields(log.Fields{
+		"component":  "cloud/livehls",
+		"event":      "init_uploaded",
+		"session_id": s.id,
+	}).Info("Live HLS init segment uploaded")
 	return true
 }
 
@@ -422,14 +478,22 @@ func (s *Session) refreshInitIfStale() {
 	ctx, cancel := s.newContext()
 	defer cancel()
 	if err := s.publisher.PublishInit(ctx, s.id, initBytes); err != nil {
-		log.Log.Warning("livehls.Session: init refresh failed, will retry: " + err.Error())
+		log.WithError(err).WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "init_refresh_failed",
+			"session_id": s.id,
+		}).Warn("Failed to refresh live HLS init segment; retrying")
 		return
 	}
 
 	s.mu.Lock()
 	s.lastInitAt = time.Now()
 	s.mu.Unlock()
-	log.Log.Debug("livehls.Session: refreshed init segment TTL for session " + s.id)
+	log.WithFields(log.Fields{
+		"component":  "cloud/livehls",
+		"event":      "init_refreshed",
+		"session_id": s.id,
+	}).Debug("Live HLS init segment refreshed")
 }
 
 // fireReadyOnce invokes the OnReady callback the first time it is called.
@@ -445,6 +509,18 @@ func (s *Session) fireReadyOnce() {
 	fn(s.id)
 }
 
+func (s *Session) fireFailureOnce(reason string) {
+	s.mu.Lock()
+	if s.failureFired || s.readyFired || s.onFailure == nil {
+		s.mu.Unlock()
+		return
+	}
+	s.failureFired = true
+	fn := s.onFailure
+	s.mu.Unlock()
+	fn(s.id, reason)
+}
+
 // newSessionID returns a short, unique, URL-safe session identifier of the form
 // <unix-seconds>-<random-hex>.
 func newSessionID() string {

machinery/src/cloud/livemoq/recovery.go

@@ -1,6 +1,9 @@
 package livemoq
 
-import "time"
+import (
+	"sync/atomic"
+	"time"
+)
 
 type FrameGateEvent uint8
 
@@ -17,9 +20,52 @@ type FrameGate struct {
 	recovering bool
 }
 
-// Reset closes the gate so publication resumes on the next keyframe. It is used
-// when the publisher stopped writing for a reason unrelated to the stream health
-// (no subscribers), so the next viewer never receives a partial GOP.
+// AudienceGate publishes every frame while watched and only keyframes while
+// idle. The idle keyframes keep the relay's latest cached group current without
+// paying the bandwidth cost of a full stream when nobody is watching.
+type AudienceGate struct {
+	idle bool
+}
+
+func (g *AudienceGate) Allow(hasSubscribers bool, isKeyFrame bool) (allowed bool, enteredIdle bool) {
+	if hasSubscribers {
+		g.idle = false
+		return true, false
+	}
+
+	enteredIdle = !g.idle
+	g.idle = true
+	return isKeyFrame, enteredIdle
+}
+
+// WriteWatchdog tracks the single synchronous frame write performed by a MoQ
+// publisher so another goroutine can interrupt a wedged native call.
+type WriteWatchdog struct {
+	startedAt atomic.Int64
+}
+
+func (w *WriteWatchdog) Begin(now time.Time) {
+	w.startedAt.Store(now.UnixNano())
+}
+
+func (w *WriteWatchdog) End() {
+	w.startedAt.Store(0)
+}
+
+func (w *WriteWatchdog) Elapsed(now time.Time) (time.Duration, bool) {
+	startedAt := w.startedAt.Load()
+	if startedAt == 0 {
+		return 0, false
+	}
+	elapsed := now.Sub(time.Unix(0, startedAt))
+	if elapsed < 0 {
+		elapsed = 0
+	}
+	return elapsed, true
+}
+
+// Reset closes the gate so publication resumes on the next keyframe. Entering
+// idle mode uses it to start the relay-cache refresh on a complete GOP.
 func (g *FrameGate) Reset() {
 	g.started = false
 	g.recovering = false

machinery/src/cloud/livemoq/recovery_test.go

@@ -47,3 +47,45 @@ func TestFrameGateAllowsMissingCaptureTime(t *testing.T) {
 		t.Fatalf("Allow() = (%t, %d), want (true, %d)", allowed, event, FrameGateEventStarted)
 	}
 }
+
+func TestAudienceGateRefreshesOnlyKeyframesWhileIdle(t *testing.T) {
+	gate := AudienceGate{}
+
+	if allowed, enteredIdle := gate.Allow(true, false); !allowed || enteredIdle {
+		t.Fatalf("active delta = (%t, %t), want (true, false)", allowed, enteredIdle)
+	}
+	if allowed, enteredIdle := gate.Allow(false, false); allowed || !enteredIdle {
+		t.Fatalf("first idle delta = (%t, %t), want (false, true)", allowed, enteredIdle)
+	}
+	if allowed, enteredIdle := gate.Allow(false, true); !allowed || enteredIdle {
+		t.Fatalf("idle keyframe = (%t, %t), want (true, false)", allowed, enteredIdle)
+	}
+	if allowed, enteredIdle := gate.Allow(false, false); allowed || enteredIdle {
+		t.Fatalf("later idle delta = (%t, %t), want (false, false)", allowed, enteredIdle)
+	}
+	if allowed, enteredIdle := gate.Allow(true, false); !allowed || enteredIdle {
+		t.Fatalf("resumed delta = (%t, %t), want (true, false)", allowed, enteredIdle)
+	}
+	if allowed, enteredIdle := gate.Allow(false, true); !allowed || !enteredIdle {
+		t.Fatalf("next idle keyframe = (%t, %t), want (true, true)", allowed, enteredIdle)
+	}
+}
+
+func TestWriteWatchdogTracksActiveWrite(t *testing.T) {
+	now := time.Unix(10, 0)
+	watchdog := WriteWatchdog{}
+
+	if elapsed, active := watchdog.Elapsed(now); active || elapsed != 0 {
+		t.Fatalf("Elapsed() before Begin() = (%s, %t), want (0, false)", elapsed, active)
+	}
+
+	watchdog.Begin(now)
+	if elapsed, active := watchdog.Elapsed(now.Add(5 * time.Second)); !active || elapsed != 5*time.Second {
+		t.Fatalf("Elapsed() during write = (%s, %t), want (5s, true)", elapsed, active)
+	}
+
+	watchdog.End()
+	if elapsed, active := watchdog.Elapsed(now.Add(6 * time.Second)); active || elapsed != 0 {
+		t.Fatalf("Elapsed() after End() = (%s, %t), want (0, false)", elapsed, active)
+	}
+}

machinery/src/cloud/livemoq_disabled.go

@@ -2,7 +2,20 @@
 
 package cloud
 
-import "github.com/kerberos-io/agent/machinery/src/models"
+import (
+	"context"
+
+	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/packets"
+)
 
 // StartLiveStreamMoQ is disabled in the standard Agent build.
-func StartLiveStreamMoQ(_ *models.Configuration, _ *models.Communication, _ bool) {}
+func StartLiveStreamMoQ(
+	_ context.Context,
+	_ *models.Configuration,
+	_ *models.Communication,
+	_ bool,
+	_ *packets.Queue,
+	_ *packets.Queue,
+) {
+}

machinery/src/cloud/livemoq_enabled.go

@@ -4,41 +4,120 @@ package cloud
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"net/url"
 	"os"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/cloud/livemoq"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/moq-dev/moq-go/moq"
+	log "github.com/sirupsen/logrus"
 )
 
 const (
 	defaultMoQRelayURL      = "https://relay.uug.ai/anon"
 	minMoQRetryDelay        = time.Second
 	maxMoQRetryDelay        = 30 * time.Second
-	maxMoQLivePacketAge     = 1500 * time.Millisecond
+	defaultMoQLivePacketAge = 1500 * time.Millisecond
+	minMoQLivePacketAge     = 250 * time.Millisecond
+	maxMoQLivePacketAge     = 30 * time.Second
+	defaultMoQWriteTimeout  = 5 * time.Second
+	minMoQWriteTimeout      = time.Second
+	maxMoQWriteTimeout      = time.Minute
+	moQWriteWatchInterval   = 250 * time.Millisecond
 	slowMoQWriteThreshold   = 100 * time.Millisecond
 	moQWriteWarningInterval = 10 * time.Second
 	duplicateKeyframeWindow = 500 * time.Millisecond
 )
 
 type liveMoQConfig struct {
-	relayURL    string
-	broadcast   string
-	quality     string
-	sourceLabel string
-	queue       *packets.Queue
+	relayURL      string
+	broadcast     string
+	quality       string
+	sourceLabel   string
+	queue         *packets.Queue
+	communication *models.Communication
+	maxPacketAge  time.Duration
+	writeTimeout  time.Duration
 }
 
-// label identifies the tier in log lines, since one Agent runs a publisher per
-// quality tier.
-func (c liveMoQConfig) label() string {
-	return c.quality + " (" + c.sourceLabel + " stream)"
+func (c liveMoQConfig) logEntry(event string) *log.Entry {
+	return log.WithFields(log.Fields{
+		"component": "moq",
+		"event":     event,
+		"quality":   c.quality,
+		"stream":    c.sourceLabel,
+	})
+}
+
+func moQRelayHost(relayURL string) string {
+	parsed, err := url.Parse(relayURL)
+	if err != nil {
+		return ""
+	}
+	return parsed.Host
+}
+
+func (c liveMoQConfig) packetAgeLimit() time.Duration {
+	if c.maxPacketAge > 0 {
+		return c.maxPacketAge
+	}
+	return defaultMoQLivePacketAge
+}
+
+func (c liveMoQConfig) writeTimeoutLimit() time.Duration {
+	if c.writeTimeout > 0 {
+		return c.writeTimeout
+	}
+	return defaultMoQWriteTimeout
+}
+
+func boundedMoQDuration(name string, fallback, minimum, maximum time.Duration) time.Duration {
+	raw := strings.TrimSpace(os.Getenv(name))
+	if raw == "" {
+		return fallback
+	}
+
+	value, err := time.ParseDuration(raw)
+	if err != nil {
+		log.WithError(err).WithFields(log.Fields{
+			"component":        "moq",
+			"configured_value": raw,
+			"effective_value":  fallback.String(),
+			"event":            "duration_invalid",
+			"variable":         name,
+		}).Warn("Invalid MoQ duration; using default")
+		return fallback
+	}
+	if value < minimum {
+		log.WithFields(log.Fields{
+			"component":        "moq",
+			"configured_value": value.String(),
+			"effective_value":  minimum.String(),
+			"event":            "duration_clamped",
+			"minimum":          minimum.String(),
+			"variable":         name,
+		}).Warn("MoQ duration is below the supported minimum")
+		return minimum
+	}
+	if value > maximum {
+		log.WithFields(log.Fields{
+			"component":        "moq",
+			"configured_value": value.String(),
+			"effective_value":  maximum.String(),
+			"event":            "duration_clamped",
+			"maximum":          maximum.String(),
+			"variable":         name,
+		}).Warn("MoQ duration exceeds the supported maximum")
+		return maximum
+	}
+	return value
 }
 
 // StartLiveStreamMoQ starts the publisher only in the dedicated MoQ build and
@@ -50,20 +129,37 @@ func (c liveMoQConfig) label() string {
 // publishing each tier as its OWN broadcast (see livemoq.BroadcastPath): the
 // high tier from the camera's highest-resolution stream and the low tier from
 // its sub stream, so switching quality in the frontend is a resubscribe to the
-// other path. Each tier only uploads while it actually has subscribers, so the
-// second broadcast is close to free when nobody watches it.
-func StartLiveStreamMoQ(configuration *models.Configuration, communication *models.Communication, subStreamEnabled bool) {
+// other path. Each tier uploads every frame while watched and keyframes only
+// while idle, keeping the relay cache current without continuously sending the
+// full unused stream.
+func StartLiveStreamMoQ(
+	ctx context.Context,
+	configuration *models.Configuration,
+	communication *models.Communication,
+	subStreamEnabled bool,
+	mainQueue *packets.Queue,
+	subQueue *packets.Queue,
+) {
 	if os.Getenv("AGENT_LIVE_MOQ_ENABLED") != "true" {
 		return
 	}
 
 	config := configuration.Config
 	if config.Offline == "true" || config.Capture.Liveview == "false" {
-		log.Log.Info("cloud.StartLiveStreamMoQ(): disabled by Agent live-view configuration")
+		log.WithFields(log.Fields{
+			"component": "moq",
+			"event":     "publisher_disabled",
+			"liveview":  config.Capture.Liveview,
+			"offline":   config.Offline,
+		}).Debug("MoQ publisher disabled by Agent configuration")
 		return
 	}
 	if config.Key == "" {
-		log.Log.Warning("cloud.StartLiveStreamMoQ(): AGENT_KEY is required")
+		log.WithFields(log.Fields{
+			"component": "moq",
+			"event":     "publisher_configuration_invalid",
+			"variable":  "AGENT_KEY",
+		}).Warn("MoQ publisher requires an Agent key")
 		return
 	}
 
@@ -83,38 +179,51 @@ func StartLiveStreamMoQ(configuration *models.Configuration, communication *mode
 		relayURL = defaultMoQRelayURL
 	}
 	broadcastPrefix := os.Getenv("AGENT_LIVE_MOQ_BROADCAST_PREFIX")
+	maxPacketAge := boundedMoQDuration("AGENT_LIVE_MOQ_MAX_PACKET_AGE", defaultMoQLivePacketAge, minMoQLivePacketAge, maxMoQLivePacketAge)
+	writeTimeout := boundedMoQDuration("AGENT_LIVE_MOQ_WRITE_TIMEOUT", defaultMoQWriteTimeout, minMoQWriteTimeout, maxMoQWriteTimeout)
 
-	ctx := context.Background()
-	if communication.Context != nil {
-		ctx = *communication.Context
-	}
-
+	var publishers sync.WaitGroup
 	for _, quality := range qualities {
-		queue := communication.Queue
+		queue := mainQueue
 		sourceLabel := "main"
-		if models.SelectSubStreamForQuality(config, quality, subStreamEnabled) && communication.SubQueue != nil {
-			queue = communication.SubQueue
+		if models.SelectSubStreamForQuality(config, quality, subStreamEnabled) && subQueue != nil {
+			queue = subQueue
 			sourceLabel = "sub"
 		}
 		if queue == nil {
-			log.Log.Warning("cloud.StartLiveStreamMoQ(): packet queue for the " + quality + " tier is unavailable")
+			log.WithFields(log.Fields{
+				"component": "moq",
+				"event":     "packet_queue_unavailable",
+				"quality":   quality,
+			}).Warn("MoQ packet queue is unavailable")
 			continue
 		}
-		go runLiveStreamMoQ(ctx, liveMoQConfig{
-			relayURL:    relayURL,
-			broadcast:   livemoq.BroadcastPath(broadcastPrefix, config.Key, quality),
-			quality:     quality,
-			sourceLabel: sourceLabel,
-			queue:       queue,
-		})
+		publisherConfig := liveMoQConfig{
+			relayURL:      relayURL,
+			broadcast:     livemoq.BroadcastPath(broadcastPrefix, config.Key, quality),
+			quality:       quality,
+			sourceLabel:   sourceLabel,
+			queue:         queue,
+			communication: communication,
+			maxPacketAge:  maxPacketAge,
+			writeTimeout:  writeTimeout,
+		}
+		publishers.Add(1)
+		go func() {
+			defer publishers.Done()
+			runLiveStreamMoQ(ctx, publisherConfig)
+		}()
 	}
+	publishers.Wait()
 }
 
 func runLiveStreamMoQ(ctx context.Context, config liveMoQConfig) {
-	log.Log.Info(fmt.Sprintf(
-		"cloud.runLiveStreamMoQ(): publishing %s stream (quality=%s) to %s/%s",
-		config.sourceLabel, config.quality, strings.TrimRight(config.relayURL, "/"), config.broadcast,
-	))
+	config.logEntry("publisher_started").Info("MoQ publisher started")
+	config.logEntry("publisher_configuration").WithFields(log.Fields{
+		"max_packet_age_ms": config.packetAgeLimit().Milliseconds(),
+		"relay_host":        moQRelayHost(config.relayURL),
+		"write_timeout_ms":  config.writeTimeoutLimit().Milliseconds(),
+	}).Debug("MoQ publisher configuration")
 
 	retryDelay := minMoQRetryDelay
 	for ctx.Err() == nil {
@@ -123,7 +232,13 @@ func runLiveStreamMoQ(ctx context.Context, config liveMoQConfig) {
 		if ctx.Err() != nil {
 			return
 		}
-		log.Log.Warning("cloud.runLiveStreamMoQ(): publisher stopped: " + err.Error())
+		if config.communication != nil {
+			config.communication.RecordMoQReconnect(config.quality)
+		}
+		config.logEntry("publisher_reconnecting").WithError(err).WithFields(log.Fields{
+			"connected_duration_ms": time.Since(connectedAt).Milliseconds(),
+			"retry_delay_ms":        retryDelay.Milliseconds(),
+		}).Warn("MoQ publisher stopped; reconnecting")
 		if time.Since(connectedAt) >= time.Minute {
 			retryDelay = minMoQRetryDelay
 		}
@@ -150,62 +265,125 @@ func publishLiveStreamMoQ(ctx context.Context, config liveMoQConfig) error {
 		return fmt.Errorf("connect to relay: %w", err)
 	}
 	defer client.Close()
+	config.logEntry("relay_connected").WithField("relay_host", moQRelayHost(config.relayURL)).
+		Info("MoQ relay connected")
+
+	publisherCtx, cancelPublisher := context.WithCancel(ctx)
+	defer cancelPublisher()
+	sessionClosed := make(chan error, 1)
+	sessionWatchDone := make(chan struct{})
+	go func() {
+		defer close(sessionWatchDone)
+		sessionClosed <- client.Session().Closed(publisherCtx)
+		cancelPublisher()
+	}()
 
 	broadcast, err := client.CreateBroadcast(config.broadcast)
 	if err != nil {
 		return fmt.Errorf("create broadcast: %w", err)
 	}
-	defer broadcast.Finish()
+	var finishBroadcastOnce sync.Once
+	var finishBroadcastErr error
+	finishBroadcast := func() error {
+		finishBroadcastOnce.Do(func() {
+			finishBroadcastErr = broadcast.Finish()
+		})
+		return finishBroadcastErr
+	}
+	defer finishBroadcast()
 
 	stream, err := broadcast.PublishMedia("avc3", nil)
 	if err != nil {
 		return fmt.Errorf("create H.264 media stream: %w", err)
 	}
-	defer stream.Finish()
+	var finishStreamOnce sync.Once
+	var finishStreamErr error
+	finishStream := func() error {
+		finishStreamOnce.Do(func() {
+			finishStreamErr = stream.Finish()
+		})
+		return finishStreamErr
+	}
+	defer finishStream()
 
-	// Only upload while this tier is actually being watched. `publishing` starts
-	// true so the track becomes discoverable on the relay even before the first
-	// subscriber ever arrives; from the moment a viewer has attached once, the
-	// subscriber watcher takes over and idles the tier again when everybody left.
-	watchCtx, cancelWatch := context.WithCancel(ctx)
-	defer cancelWatch()
+	// Publish the complete stream while watched and only keyframes while idle.
+	// `publishing` starts true so the track becomes discoverable on the relay
+	// before the first subscriber; after that, the subscriber watcher selects the
+	// full-rate or keyframe-refresh mode.
 	publishing := &atomic.Bool{}
 	publishing.Store(true)
-	go watchLiveStreamMoQSubscribers(watchCtx, stream, publishing, config)
+	subscriberWatchDone := make(chan struct{})
+	go func() {
+		defer close(subscriberWatchDone)
+		watchLiveStreamMoQSubscribers(publisherCtx, stream, publishing, config)
+	}()
+	writeWatchdog := &livemoq.WriteWatchdog{}
+	writeWatchDone := make(chan struct{})
+	closePublisher := func() error {
+		return closeLiveMoQPublisher(finishStream, finishBroadcast, client.Close)
+	}
+	go func() {
+		defer close(writeWatchDone)
+		watchLiveStreamMoQWrites(publisherCtx, closePublisher, writeWatchdog, config)
+	}()
+	defer func() {
+		cancelPublisher()
+		<-writeWatchDone
+		<-subscriberWatchDone
+		<-sessionWatchDone
+	}()
 
 	cursor := config.queue.Latest()
 	gate := livemoq.FrameGate{}
+	audienceGate := livemoq.AudienceGate{}
 	deduplicator := livemoq.KeyframeDeduplicator{}
 	var lastSlowWriteWarning time.Time
 	var lastDuplicateKeyframeWarning time.Time
-	idle := false
 	for {
-		packet, err := cursor.ReadPacket()
+		select {
+		case err := <-sessionClosed:
+			return fmt.Errorf("relay session closed: %w", err)
+		default:
+		}
+
+		packet, err := cursor.ReadPacketContext(publisherCtx)
 		if err != nil {
+			select {
+			case sessionErr := <-sessionClosed:
+				return fmt.Errorf("relay session closed: %w", sessionErr)
+			default:
+			}
+			if ctx.Err() != nil {
+				return ctx.Err()
+			}
 			return fmt.Errorf("read packet: %w", err)
 		}
-		if !publishing.Load() {
-			// Keep draining the cursor so we stay at the live edge, but publish
-			// nothing. The gate is closed so the next viewer resumes on a keyframe.
-			if !idle {
-				gate.Reset()
-				deduplicator.Reset()
-				idle = true
-			}
-			continue
-		}
-		idle = false
 		if !packet.IsVideo || len(packet.Data) == 0 || !strings.EqualFold(packet.Codec, "H264") {
 			continue
 		}
-		allowed, event := gate.Allow(packet.IsKeyFrame, packet.CurrentTime, time.Now(), maxMoQLivePacketAge)
+		allowedForAudience, enteredIdle := audienceGate.Allow(publishing.Load(), packet.IsKeyFrame)
+		if enteredIdle {
+			gate.Reset()
+			deduplicator.Reset()
+		}
+		if !allowedForAudience {
+			continue
+		}
+		allowed, event := gate.Allow(packet.IsKeyFrame, packet.CurrentTime, time.Now(), config.packetAgeLimit())
 		switch event {
 		case livemoq.FrameGateEventStarted:
-			log.Log.Info("cloud.publishLiveStreamMoQ(): first H.264 keyframe received; " + config.label() + " broadcast is live")
+			config.logEntry("broadcast_live").WithFields(log.Fields{
+				"codec":        packet.Codec,
+				"timestamp_ms": packet.Time,
+			}).Info("MoQ broadcast is live")
 		case livemoq.FrameGateEventLagging:
-			log.Log.Warning("cloud.publishLiveStreamMoQ(): " + config.label() + " stream is lagging; dropping packets until a recent keyframe")
+			config.logEntry("stream_lagging").WithFields(log.Fields{
+				"max_packet_age_ms": config.packetAgeLimit().Milliseconds(),
+				"timestamp_ms":      packet.Time,
+			}).Warn("MoQ stream is lagging; dropping packets until a recent keyframe")
 		case livemoq.FrameGateEventRecovered:
-			log.Log.Info("cloud.publishLiveStreamMoQ(): caught up with the " + config.label() + " live stream at a recent keyframe")
+			config.logEntry("stream_recovered").WithField("timestamp_ms", packet.Time).
+				Info("MoQ stream recovered at a recent keyframe")
 		}
 		if !allowed {
 			continue
@@ -215,18 +393,18 @@ func publishLiveStreamMoQ(ctx context.Context, config liveMoQConfig) error {
 			return fmt.Errorf("normalize H.264 access unit: %w", err)
 		}
 		if normalizationStats.DuplicateIDRNALUs > 0 && time.Since(lastDuplicateKeyframeWarning) >= moQWriteWarningInterval {
-			log.Log.Warning(fmt.Sprintf(
-				"cloud.publishLiveStreamMoQ(): %s removed %d duplicate IDR NALU(s) from H.264 keyframe (timestamp_ms=%d)",
-				config.label(), normalizationStats.DuplicateIDRNALUs, packet.Time,
-			))
+			config.logEntry("duplicate_idr_removed").WithFields(log.Fields{
+				"duplicate_idr_count": normalizationStats.DuplicateIDRNALUs,
+				"timestamp_ms":        packet.Time,
+			}).Warn("Removed duplicate IDR NAL units from MoQ keyframe")
+
 			lastDuplicateKeyframeWarning = time.Now()
 		}
 		if packet.IsKeyFrame && deduplicator.IsDuplicate(packet.Time, packet.CurrentTime, payload, time.Now(), duplicateKeyframeWindow) {
 			if time.Since(lastDuplicateKeyframeWarning) >= moQWriteWarningInterval {
-				log.Log.Warning(fmt.Sprintf(
-					"cloud.publishLiveStreamMoQ(): %s dropping duplicate H.264 keyframe (timestamp_ms=%d)",
-					config.label(), packet.Time,
-				))
+				config.logEntry("duplicate_keyframe_dropped").WithField("timestamp_ms", packet.Time).
+					Warn("Dropped duplicate MoQ keyframe")
+
 				lastDuplicateKeyframeWarning = time.Now()
 			}
 			continue
@@ -236,10 +414,16 @@ func publishLiveStreamMoQ(ctx context.Context, config liveMoQConfig) error {
 			TimestampUs: livemoq.TimestampUs(packet.Time),
 		}
 		writeStartedAt := time.Now()
-		if err := stream.WriteFrame(frame); err != nil {
+		writeWatchdog.Begin(writeStartedAt)
+		err = stream.WriteFrame(frame)
+		writeWatchdog.End()
+		if err != nil {
 			return fmt.Errorf("write H.264 access unit: %w", err)
 		}
 		writeDuration := time.Since(writeStartedAt)
+		if config.communication != nil {
+			config.communication.RecordMoQWrite(config.quality, writeDuration, time.Now())
+		}
 		if writeDuration >= slowMoQWriteThreshold && time.Since(lastSlowWriteWarning) >= moQWriteWarningInterval {
 			packetAge := time.Duration(0)
 			if packet.CurrentTime > 0 {
@@ -248,15 +432,58 @@ func publishLiveStreamMoQ(ctx context.Context, config liveMoQConfig) error {
 					packetAge = 0
 				}
 			}
-			log.Log.Warning(fmt.Sprintf(
-				"cloud.publishLiveStreamMoQ(): %s WriteFrame blocked for %s (packet_age=%s keyframe=%t)",
-				config.label(), writeDuration.Round(time.Millisecond), packetAge.Round(time.Millisecond), packet.IsKeyFrame,
-			))
+			config.logEntry("slow_frame_write").WithFields(log.Fields{
+				"duration_ms":   writeDuration.Milliseconds(),
+				"keyframe":      packet.IsKeyFrame,
+				"packet_age_ms": packetAge.Milliseconds(),
+			}).Warn("MoQ frame write was slow")
+
 			lastSlowWriteWarning = time.Now()
 		}
 	}
 }
 
+func closeLiveMoQPublisher(finishStream, finishBroadcast, closeClient func() error) error {
+	streamErr := finishStream()
+	broadcastErr := finishBroadcast()
+	clientErr := closeClient()
+	return errors.Join(streamErr, broadcastErr, clientErr)
+}
+
+func watchLiveStreamMoQWrites(ctx context.Context, closePublisher func() error, watchdog *livemoq.WriteWatchdog, config liveMoQConfig) {
+	ticker := time.NewTicker(moQWriteWatchInterval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			if err := closePublisher(); err != nil {
+				config.logEntry("publisher_close_failed").WithError(err).
+					Debug("MoQ publisher reported an error while closing")
+			}
+			return
+		case now := <-ticker.C:
+			writeDuration, active := watchdog.Elapsed(now)
+			if !active || writeDuration < config.writeTimeoutLimit() {
+				continue
+			}
+			config.logEntry("frame_write_timeout").WithFields(log.Fields{
+				"duration_ms": writeDuration.Milliseconds(),
+				"timeout_ms":  config.writeTimeoutLimit().Milliseconds(),
+			}).Warn("MoQ frame write timed out; reconnecting relay client")
+
+			if config.communication != nil {
+				config.communication.RecordMoQWriteTimeout()
+			}
+			if err := closePublisher(); err != nil {
+				config.logEntry("publisher_close_failed").WithError(err).
+					Warn("MoQ publisher reported an error while closing after a write timeout")
+			}
+			return
+		}
+	}
+}
+
 // watchLiveStreamMoQSubscribers flips the publisher between uploading and idling
 // as viewers subscribe to and leave this tier's broadcast. Used and Unused both
 // block, so they are followed from their own goroutine.
@@ -272,7 +499,7 @@ func watchLiveStreamMoQSubscribers(ctx context.Context, stream *moq.MediaProduce
 			return
 		}
 		if publishing.CompareAndSwap(false, true) {
-			log.Log.Info("cloud.watchLiveStreamMoQSubscribers(): viewer subscribed, resuming the " + config.label() + " broadcast")
+			config.logEntry("subscriber_joined").Info("MoQ subscriber joined; resuming broadcast")
 		}
 
 		if err := stream.Unused(ctx); err != nil {
@@ -280,6 +507,6 @@ func watchLiveStreamMoQSubscribers(ctx context.Context, stream *moq.MediaProduce
 			return
 		}
 		publishing.Store(false)
-		log.Log.Info("cloud.watchLiveStreamMoQSubscribers(): no viewers left, idling the " + config.label() + " broadcast")
+		config.logEntry("subscribers_idle").Info("MoQ broadcast idle; refreshing keyframes only")
 	}
 }

machinery/src/cloud/livemoq_enabled_test.go

@@ -0,0 +1,212 @@
+//go:build moq
+
+package cloud
+
+import (
+	"context"
+	"errors"
+	"reflect"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/kerberos-io/agent/machinery/src/cloud/livemoq"
+	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/moq-dev/moq-go/moq"
+)
+
+func TestCloseLiveMoQPublisherFinishesBeforeClosingClient(t *testing.T) {
+	var order []string
+	streamErr := errors.New("stream finish failed")
+	broadcastErr := errors.New("broadcast finish failed")
+	clientErr := errors.New("client close failed")
+
+	err := closeLiveMoQPublisher(
+		func() error {
+			order = append(order, "stream")
+			return streamErr
+		},
+		func() error {
+			order = append(order, "broadcast")
+			return broadcastErr
+		},
+		func() error {
+			order = append(order, "client")
+			return clientErr
+		},
+	)
+
+	if want := []string{"stream", "broadcast", "client"}; !reflect.DeepEqual(order, want) {
+		t.Fatalf("shutdown order = %v, want %v", order, want)
+	}
+	for _, want := range []error{streamErr, broadcastErr, clientErr} {
+		if !errors.Is(err, want) {
+			t.Errorf("shutdown error %v does not include %v", err, want)
+		}
+	}
+}
+
+func TestWatchLiveStreamMoQWritesClosesClientWhenContextEnds(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	closed := make(chan struct{})
+	var closeCalls atomic.Int32
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		watchLiveStreamMoQWrites(ctx, func() error {
+			if closeCalls.Add(1) == 1 {
+				close(closed)
+			}
+			return nil
+		}, &livemoq.WriteWatchdog{}, liveMoQConfig{quality: "low", sourceLabel: "sub"})
+	}()
+
+	cancel()
+	select {
+	case <-closed:
+	case <-time.After(time.Second):
+		t.Fatal("watchLiveStreamMoQWrites() did not close the client after cancellation")
+	}
+	<-done
+	if got := closeCalls.Load(); got != 1 {
+		t.Fatalf("close calls = %d, want 1", got)
+	}
+}
+
+func TestBoundedMoQDuration(t *testing.T) {
+	const name = "AGENT_LIVE_MOQ_TEST_DURATION"
+	tests := []struct {
+		name string
+		raw  string
+		want time.Duration
+	}{
+		{name: "unset", want: 5 * time.Second},
+		{name: "valid", raw: "12s", want: 12 * time.Second},
+		{name: "invalid", raw: "later", want: 5 * time.Second},
+		{name: "below minimum", raw: "100ms", want: time.Second},
+		{name: "above maximum", raw: "2m", want: time.Minute},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			t.Setenv(name, test.raw)
+			if got := boundedMoQDuration(name, 5*time.Second, time.Second, time.Minute); got != test.want {
+				t.Fatalf("boundedMoQDuration() = %s, want %s", got, test.want)
+			}
+		})
+	}
+}
+
+func TestWatchLiveStreamMoQWritesRecordsTimeout(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	closed := make(chan struct{})
+	communication := &models.Communication{}
+	watchdog := &livemoq.WriteWatchdog{}
+	watchdog.Begin(time.Now().Add(-time.Second))
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		watchLiveStreamMoQWrites(ctx, func() error {
+			close(closed)
+			return nil
+		}, watchdog, liveMoQConfig{
+			quality:       models.StreamQualityLow,
+			sourceLabel:   "sub",
+			communication: communication,
+			writeTimeout:  10 * time.Millisecond,
+		})
+	}()
+
+	select {
+	case <-closed:
+	case <-time.After(time.Second):
+		t.Fatal("watchLiveStreamMoQWrites() did not close the client after a write timeout")
+	}
+	<-done
+	if got := communication.RecoveryTelemetry().MoQWriteTimeouts; got != 1 {
+		t.Fatalf("MoQ write timeouts = %d, want 1", got)
+	}
+}
+
+func TestNativeMoQClientCloseStopsActiveWriter(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	server, err := moq.Listen(ctx, "127.0.0.1:0", moq.WithTLSGenerate("localhost"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	serveDone := make(chan error, 1)
+	go func() {
+		serveDone <- server.Serve(ctx)
+	}()
+
+	client, err := moq.Dial(ctx, "https://"+server.LocalAddr(), moq.WithTLSVerify(false))
+	if err != nil {
+		server.Close()
+		t.Fatal(err)
+	}
+	broadcast, err := client.CreateBroadcast("test/native-close")
+	if err != nil {
+		client.Close()
+		server.Close()
+		t.Fatal(err)
+	}
+	stream, err := broadcast.PublishMedia("avc3", nil)
+	if err != nil {
+		client.Close()
+		server.Close()
+		t.Fatal(err)
+	}
+
+	firstWrite := make(chan struct{})
+	writerDone := make(chan error, 1)
+	go func() {
+		payload := []byte{
+			0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x1e, 0xd9, 0x00, 0xa0, 0x47, 0xfe, 0xc8,
+			0x00, 0x00, 0x00, 0x01, 0x68, 0xce, 0x38, 0x80,
+			0x00, 0x00, 0x00, 0x01, 0x65, 0x88,
+		}
+		for timestamp := uint64(0); ; timestamp++ {
+			err := stream.WriteFrame(moq.Frame{Payload: payload, TimestampUs: timestamp})
+			if err != nil {
+				writerDone <- err
+				return
+			}
+			if timestamp == 0 {
+				close(firstWrite)
+			}
+		}
+	}()
+
+	select {
+	case <-firstWrite:
+	case <-ctx.Done():
+		t.Fatal("native MoQ writer did not start")
+	}
+	if err := stream.Finish(); err != nil {
+		t.Fatal(err)
+	}
+	if err := client.Close(); err != nil {
+		t.Fatal(err)
+	}
+	select {
+	case err := <-writerDone:
+		if err == nil {
+			t.Fatal("native MoQ writer stopped without an error after client close")
+		}
+	case <-ctx.Done():
+		t.Fatal("native MoQ client close did not stop the writer")
+	}
+
+	_ = broadcast.Finish()
+	_ = server.Close()
+	select {
+	case <-serveDone:
+	case <-ctx.Done():
+		t.Fatal("native MoQ server did not stop")
+	}
+}

machinery/src/cloud/livesnapshot/publisher.go

@@ -30,7 +30,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
+	log "github.com/sirupsen/logrus"
 )
 
 const (
@@ -131,7 +131,10 @@ func (p *Publisher) PublishSnapshot(ctx context.Context, jpeg []byte) error {
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
 		return fmt.Errorf("livesnapshot: upload snapshot rejected: %s", resp.Status)
 	}
-	log.Log.Debug("livesnapshot.Publisher.PublishSnapshot(): shipped preview frame for device " + p.cfg.DeviceKey)
+	log.WithFields(log.Fields{
+		"component": "livesnapshot",
+		"event":     "snapshot_published",
+	}).Debug("Live snapshot published")
 	return nil
 }
 

machinery/src/cloud/recording_metadata.go

@@ -15,6 +15,9 @@ import (
 const recordingFPSHeader = "X-Kerberos-Storage-Fps"
 const recordingDurationHeader = "X-Kerberos-Storage-Duration"
 const recordingTimestampHeader = "X-Kerberos-Storage-Timestamp"
+const recordingDeviceNameHeader = "X-Kerberos-Storage-DeviceName"
+const recordingRegionCoordinatesHeader = "X-Kerberos-Storage-RegionCoordinates"
+const recordingNumberOfChangesHeader = "X-Kerberos-Storage-NumberOfChanges"
 
 // queuedRecordingFPS reads the FPS snapshot written into the upload marker
 // when the recording was finalized. Historical empty markers intentionally
@@ -84,5 +87,14 @@ func setQueuedRecordingMetadataHeaders(header http.Header, fileName string) {
 		if metadata.Timestamp > 0 {
 			header.Set(recordingTimestampHeader, strconv.FormatInt(metadata.Timestamp, 10))
 		}
+		if metadata.DeviceName != "" {
+			header.Set(recordingDeviceNameHeader, metadata.DeviceName)
+		}
+		if metadata.RegionCoordinates != "" {
+			header.Set(recordingRegionCoordinatesHeader, metadata.RegionCoordinates)
+		}
+		if metadata.NumberOfChanges != "" {
+			header.Set(recordingNumberOfChangesHeader, metadata.NumberOfChanges)
+		}
 	}
 }

machinery/src/cloud/s3.go

@@ -6,12 +6,13 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/minio/minio-go/v6"
+	log "github.com/sirupsen/logrus"
 )
 
 func UploadS3(configuration *models.Configuration, fileName string) (bool, bool, error) {
@@ -29,7 +30,7 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
 
 	if config.S3 == nil {
 		errorMessage := "UploadS3: Uploading Failed, as no settings found"
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, false, errors.New(errorMessage)
 	}
 
@@ -49,14 +50,14 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
 	// Check if we have some credentials otherwise we abort the request.
 	if aws_access_key_id == "" || aws_secret_access_key == "" {
 		errorMessage := "UploadS3: Uploading Failed, as no credentials found"
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, false, errors.New(errorMessage)
 	}
 
 	s3Client, err := minio.NewWithRegion("s3.amazonaws.com", aws_access_key_id, aws_secret_access_key, true, aws_region)
 	if err != nil {
 		errorMessage := "UploadS3: " + err.Error()
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, true, errors.New(errorMessage)
 	}
 
@@ -71,21 +72,17 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
 		s3Client.SetCustomTransport(transport)
 	}
 
-	fileParts := strings.Split(fileName, "_")
-	if len(fileParts) == 1 {
+	recordingMetadata, ok := queuedRecordingMetadata(fileName)
+	if !ok {
+		recordingMetadata, ok = legacyS3RecordingMetadata(fileName, config.Key)
+	}
+	if !ok {
 		errorMessage := "UploadS3: " + fileName + " is not a valid name."
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, true, errors.New(errorMessage)
 	}
 
-	deviceKey := config.Key
-	startRecording, _ := strconv.ParseInt(fileParts[0], 10, 64)
-	devicename := fileParts[2]
-	coordinates := fileParts[3]
-	//numberOfChanges := fileParts[4]
-	token, _ := strconv.Atoi(fileParts[5])
-
-	log.Log.Info("UploadS3: Upload started for " + fileName)
+	log.Info("UploadS3: Upload started for " + fileName)
 	fullname := "data/recordings/" + fileName
 
 	file, err := os.OpenFile(fullname, os.O_RDWR, 0755)
@@ -95,14 +92,14 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
 
 	if err != nil {
 		errorMessage := "UploadS3: " + err.Error()
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, true, errors.New(errorMessage)
 	}
 
 	fileInfo, err := file.Stat()
 	if err != nil {
 		errorMessage := "UploadS3: " + err.Error()
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, true, errors.New(errorMessage)
 	}
 
@@ -113,25 +110,56 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
 		minio.PutObjectOptions{
 			ContentType:  "video/mp4",
 			StorageClass: "ONEZONE_IA",
-			UserMetadata: map[string]string{
-				"event-timestamp":         strconv.FormatInt(startRecording, 10),
-				"event-microseconds":      deviceKey,
-				"event-instancename":      devicename,
-				"event-regioncoordinates": coordinates,
-				"event-numberofchanges":   deviceKey,
-				"event-token":             strconv.Itoa(token),
-				"productid":               deviceKey,
-				"publickey":               aws_access_key_id,
-				"uploadtime":              "now",
-			},
+			UserMetadata: s3ObjectMetadata(recordingMetadata, config.Key, aws_access_key_id),
 		})
 
 	if err != nil {
 		errorMessage := "UploadS3: Uploading Failed, " + err.Error()
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, true, errors.New(errorMessage)
 	} else {
-		log.Log.Info("UploadS3: Upload Finished, file has been uploaded to bucket: " + strconv.FormatInt(n, 10))
+		log.Info("UploadS3: Upload Finished, file has been uploaded to bucket: " + strconv.FormatInt(n, 10))
 		return true, true, nil
 	}
 }
+
+func s3ObjectMetadata(metadata models.RecordingUploadMetadata, deviceKey string, publicKey string) map[string]string {
+	return map[string]string{
+		"event-timestamp":         strconv.FormatInt(metadata.Timestamp/1000, 10),
+		"event-microseconds":      strconv.FormatInt(metadata.Timestamp%1000, 10),
+		"event-instancename":      metadata.DeviceName,
+		"event-regioncoordinates": metadata.RegionCoordinates,
+		"event-numberofchanges":   metadata.NumberOfChanges,
+		"event-duration":          strconv.FormatUint(metadata.Duration, 10),
+		"event-token":             strconv.FormatUint(metadata.Duration, 10),
+		"productid":               deviceKey,
+		"publickey":               publicKey,
+		"uploadtime":              "now",
+	}
+}
+
+func legacyS3RecordingMetadata(fileName string, deviceKey string) (models.RecordingUploadMetadata, bool) {
+	fileParts := strings.Split(fileName, "_")
+	if len(fileParts) < 6 {
+		return models.RecordingUploadMetadata{}, false
+	}
+	seconds, secondsErr := strconv.ParseInt(fileParts[0], 10, 64)
+	milliseconds := int64(0)
+	precisionParts := strings.SplitN(fileParts[1], "-", 2)
+	if len(precisionParts) == 2 {
+		milliseconds, _ = strconv.ParseInt(precisionParts[1], 10, 64)
+	}
+	duration, durationErr := strconv.ParseUint(strings.TrimSuffix(fileParts[5], filepath.Ext(fileParts[5])), 10, 64)
+	if secondsErr != nil || durationErr != nil || milliseconds < 0 || milliseconds >= 1000 {
+		return models.RecordingUploadMetadata{}, false
+	}
+	return models.RecordingUploadMetadata{
+		FileName:          fileName,
+		DeviceKey:         deviceKey,
+		DeviceName:        fileParts[2],
+		Timestamp:         seconds*1000 + milliseconds,
+		Duration:          duration,
+		RegionCoordinates: fileParts[3],
+		NumberOfChanges:   fileParts[4],
+	}, true
+}

machinery/src/cloud/s3_test.go

@@ -0,0 +1,40 @@
+package cloud
+
+import (
+	"testing"
+
+	"github.com/kerberos-io/agent/machinery/src/models"
+)
+
+func TestS3ObjectMetadataUsesStructuredRecordingMetadata(t *testing.T) {
+	metadata := s3ObjectMetadata(models.RecordingUploadMetadata{
+		Timestamp:         1785934709414,
+		Duration:          20452,
+		DeviceName:        "camera-name",
+		RegionCoordinates: "1-2-3-4",
+		NumberOfChanges:   "57",
+	}, "device-key", "public-key")
+
+	for key, want := range map[string]string{
+		"event-timestamp":         "1785934709",
+		"event-microseconds":      "414",
+		"event-instancename":      "camera-name",
+		"event-regioncoordinates": "1-2-3-4",
+		"event-numberofchanges":   "57",
+		"event-duration":          "20452",
+		"event-token":             "20452",
+		"productid":               "device-key",
+		"publickey":               "public-key",
+	} {
+		if metadata[key] != want {
+			t.Errorf("metadata[%q] = %q, want %q", key, metadata[key], want)
+		}
+	}
+}
+
+func TestLegacyS3RecordingMetadata(t *testing.T) {
+	metadata, ok := legacyS3RecordingMetadata("1785934709_3-414_camera_1-2-3-4_57_20452.mp4", "device-key")
+	if !ok || metadata.Timestamp != 1785934709414 || metadata.Duration != 20452 || metadata.DeviceName != "camera" || metadata.RegionCoordinates != "1-2-3-4" || metadata.NumberOfChanges != "57" {
+		t.Fatalf("legacyS3RecordingMetadata() = %#v/%v", metadata, ok)
+	}
+}

machinery/src/cloud/tus_client.go

@@ -15,8 +15,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 // tusResumableVersion is the tus protocol version implemented by this client.
@@ -90,7 +90,14 @@ func logTusUploadProgress(label string, offset, size int64, loggedBucket *int64)
 	if percent > 100 {
 		percent = 100
 	}
-	log.Log.Infof("%s: resumable upload progress %d%% (%d/%d bytes)", label, percent, offset, size)
+	log.WithFields(log.Fields{
+		"bytes_sent":  offset,
+		"bytes_total": size,
+		"component":   "tus",
+		"event":       "upload_progress",
+		"progress":    percent,
+		"target":      label,
+	}).Info("Resumable upload progress")
 }
 
 // tusHeaderFunc sets the authentication and routing headers required on every
@@ -121,7 +128,7 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
 	}
 	if ferr != nil {
 		msg := label + ": resumable upload failed, file doesn't exist anymore"
-		log.Log.Info(msg)
+		log.Info(msg)
 		// The file is gone, so the legacy path cannot help either. Report it as
 		// "supported" to avoid a pointless fallback attempt.
 		return false, false, true, "", errors.New(msg)
@@ -182,7 +189,7 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
 					// The vault does not implement tus; let the caller fall back.
 					return false, false, false, "", cerr
 				}
-				log.Log.Info(label + ": resumable create failed, " + cerr.Error())
+				log.Info(label + ": resumable create failed, " + cerr.Error())
 				tusBackoff(attempt)
 				continue
 			}
@@ -200,7 +207,7 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
 				uploadURL = ""
 				continue
 			}
-			log.Log.Info(label + ": resumable head failed, " + herr.Error())
+			log.Info(label + ": resumable head failed, " + herr.Error())
 			tusBackoff(attempt)
 			continue
 		}
@@ -253,9 +260,9 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
 				if status >= 400 {
 					// Definitive rejection (e.g. provider push failed during finalize).
 					// Re-evaluate via HEAD on the next iteration to decide retry/restart.
-					log.Log.Info(label + ": resumable patch rejected, " + perr.Error())
+					log.Info(label + ": resumable patch rejected, " + perr.Error())
 				} else {
-					log.Log.Info(label + ": resumable patch failed, " + perr.Error())
+					log.Info(label + ": resumable patch failed, " + perr.Error())
 				}
 				tusBackoff(attempt)
 				patchFailed = true
@@ -354,6 +361,9 @@ func addRecordingTusMetadata(values map[string]string, fileName string) {
 	if metadata.Timestamp > 0 {
 		values["timestamp"] = strconv.FormatInt(metadata.Timestamp, 10)
 	}
+	values["device_name"] = metadata.DeviceName
+	values["region_coordinates"] = metadata.RegionCoordinates
+	values["number_of_changes"] = metadata.NumberOfChanges
 }
 
 // tusCreate performs the tus "creation" request (POST). On success it returns

machinery/src/cloud/tus_client_test.go

@@ -264,6 +264,36 @@ func testVault(uri string) models.KStorage {
 	}
 }
 
+func TestUploadKerberosVaultSkipsEmptyRecording(t *testing.T) {
+	fileName := "1787015373_3-654_office-camera17_0-0-0-0_-1_1960.mp4"
+	withRecording(t, fileName, nil)
+
+	requestCount := 0
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requestCount++
+		w.WriteHeader(http.StatusInternalServerError)
+	}))
+	defer server.Close()
+
+	vault := testVault(server.URL)
+	configuration := &models.Configuration{Config: models.Config{
+		Key:               "device-key",
+		KStorage:          &vault,
+		KStorageSecondary: &models.KStorage{},
+	}}
+
+	uploaded, configured, err := UploadKerberosVault(configuration, fileName)
+	if err != nil {
+		t.Fatalf("UploadKerberosVault() error = %v", err)
+	}
+	if uploaded || configured {
+		t.Fatalf("UploadKerberosVault() uploaded/configured = %v/%v, want false/false", uploaded, configured)
+	}
+	if requestCount != 0 {
+		t.Fatalf("Vault received %d requests, want 0", requestCount)
+	}
+}
+
 func TestUploadVaultResumable_HappyPath(t *testing.T) {
 	srv := newFakeTus()
 	ts := httptest.NewServer(srv)
@@ -272,7 +302,7 @@ func TestUploadVaultResumable_HappyPath(t *testing.T) {
 	fileName := "1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4"
 	payload := bytes.Repeat([]byte("x"), 4096)
 	withRecording(t, fileName, payload)
-	withQueuedRecordingFPS(t, fileName, `{"filename":"recording.mp4","device_key":"device-key","timestamp":1785934709414,"duration":20452,"fps":29.97}`)
+	withQueuedRecordingFPS(t, fileName, `{"filename":"recording.mp4","device_key":"device-key","device_name":"camera-name","timestamp":1785934709414,"duration":20452,"fps":29.97,"region_coordinates":"1-2-3-4","number_of_changes":"57"}`)
 
 	uploaded, responded, supported, _, err := uploadVaultResumable(testVault(ts.URL), "pk", "dev", fileName, "test", "primary")
 	if err != nil {
@@ -298,6 +328,15 @@ func TestUploadVaultResumable_HappyPath(t *testing.T) {
 	if got := metadata["timestamp"]; got != "1785934709414" {
 		t.Fatalf("POST metadata timestamp = %q, want %q", got, "1785934709414")
 	}
+	if got := metadata["device_name"]; got != "camera-name" {
+		t.Fatalf("POST metadata device_name = %q, want camera-name", got)
+	}
+	if got := metadata["region_coordinates"]; got != "1-2-3-4" {
+		t.Fatalf("POST metadata region_coordinates = %q, want 1-2-3-4", got)
+	}
+	if got := metadata["number_of_changes"]; got != "57" {
+		t.Fatalf("POST metadata number_of_changes = %q, want 57", got)
+	}
 }
 
 func TestQueuedRecordingFPSValidation(t *testing.T) {
@@ -356,7 +395,7 @@ func TestQueuedRecordingFPSAllowsMissingHistoricalMarker(t *testing.T) {
 func TestQueuedRecordingMetadataHeaders(t *testing.T) {
 	fileName := "recording.mp4"
 	withRecording(t, fileName, []byte("recording"))
-	withQueuedRecordingFPS(t, fileName, `{"filename":"recording.mp4","device_key":"device-key","timestamp":1785934709414,"duration":20452,"fps":25}`)
+	withQueuedRecordingFPS(t, fileName, `{"filename":"recording.mp4","device_key":"device-key","device_name":"camera-name","timestamp":1785934709414,"duration":20452,"fps":25,"region_coordinates":"1-2-3-4","number_of_changes":"57"}`)
 
 	header := make(http.Header)
 	setQueuedRecordingMetadataHeaders(header, fileName)
@@ -369,6 +408,15 @@ func TestQueuedRecordingMetadataHeaders(t *testing.T) {
 	if got := header.Get(recordingTimestampHeader); got != "1785934709414" {
 		t.Fatalf("timestamp header = %q", got)
 	}
+	if got := header.Get(recordingDeviceNameHeader); got != "camera-name" {
+		t.Fatalf("device name header = %q", got)
+	}
+	if got := header.Get(recordingRegionCoordinatesHeader); got != "1-2-3-4" {
+		t.Fatalf("region coordinates header = %q", got)
+	}
+	if got := header.Get(recordingNumberOfChangesHeader); got != "57" {
+		t.Fatalf("number of changes header = %q", got)
+	}
 }
 
 func TestQueuedRecordingFPSAllowsLegacyMarkerFileName(t *testing.T) {

machinery/src/components/backchannel.go

@@ -3,17 +3,16 @@ package components
 import (
 	"bufio"
 	"context"
-	"fmt"
 	"math/rand"
 	"os"
 	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/capture"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/kerberos-io/joy4/av"
 	"github.com/pion/rtp"
+	log "github.com/sirupsen/logrus"
 	"github.com/zaf/g711"
 )
 
@@ -79,7 +78,7 @@ func GetBackChannelAudioCodec(streams []av.CodecData, communication *models.Comm
 			if stream.Type().String() == "PCM_MULAW" {
 				pcmuCodec := stream.(av.AudioCodecData)
 				if pcmuCodec.IsBackChannel() {
-					communication.HasBackChannel = true
+					communication.HasBackChannel.Store(true)
 					return pcmuCodec
 				}
 			}
@@ -88,22 +87,17 @@ func GetBackChannelAudioCodec(streams []av.CodecData, communication *models.Comm
 	return nil
 }
 
-func WriteAudioToBackchannel(communication *models.Communication, rtspClient capture.RTSPClient) {
-	ctx := context.Background()
-	if communication.Context != nil {
-		ctx = *communication.Context
-	}
-
+func WriteAudioToBackchannel(ctx context.Context, communication *models.Communication, rtspClient capture.RTSPClient) {
 	writeAudioToBackchannel(ctx, ctx, communication.HandleAudio, rtspClient)
 }
 
 func writeAudioToBackchannel(ctx context.Context, otelContext context.Context, audioChannel <-chan models.AudioDataPartial, rtspClient backchannelClient) {
-	log.Log.Info("Audio.WriteAudioToBackchannel(): writing to backchannel audio codec")
+	log.Info("Audio.WriteAudioToBackchannel(): writing to backchannel audio codec")
 
 	if err := rtspClient.StartBackChannel(ctx, otelContext); err != nil {
-		log.Log.Error("Audio.WriteAudioToBackchannel(): error starting backchannel: " + err.Error())
+		log.Error("Audio.WriteAudioToBackchannel(): error starting backchannel: " + err.Error())
 		if !reconnectBackchannel(ctx, otelContext, rtspClient) {
-			log.Log.Info("Audio.WriteAudioToBackchannel(): stopped while reconnecting")
+			log.Info("Audio.WriteAudioToBackchannel(): stopped while reconnecting")
 			return
 		}
 	}
@@ -112,11 +106,11 @@ func writeAudioToBackchannel(ctx context.Context, otelContext context.Context, a
 	for {
 		select {
 		case <-ctx.Done():
-			log.Log.Info("Audio.WriteAudioToBackchannel(): stopped")
+			log.Info("Audio.WriteAudioToBackchannel(): stopped")
 			return
 		case audio, ok := <-audioChannel:
 			if !ok {
-				log.Log.Info("Audio.WriteAudioToBackchannel(): finished")
+				log.Info("Audio.WriteAudioToBackchannel(): finished")
 				return
 			}
 
@@ -127,9 +121,9 @@ func writeAudioToBackchannel(ctx context.Context, otelContext context.Context, a
 
 			pkt := packetizer.packet(audio, time.Now())
 			if err := rtspClient.WritePacket(pkt); err != nil {
-				log.Log.Error("Audio.WriteAudioToBackchannel(): error writing packet to backchannel: " + err.Error())
+				log.Error("Audio.WriteAudioToBackchannel(): error writing packet to backchannel: " + err.Error())
 				if !reconnectBackchannel(ctx, otelContext, rtspClient) {
-					log.Log.Info("Audio.WriteAudioToBackchannel(): stopped while reconnecting")
+					log.Info("Audio.WriteAudioToBackchannel(): stopped while reconnecting")
 					return
 				}
 				packetizer = newBackchannelPacketizer()
@@ -137,7 +131,7 @@ func writeAudioToBackchannel(ctx context.Context, otelContext context.Context, a
 			}
 
 			if !waitForBackchannel(ctx, time.Duration(len(audio.Data))*time.Second/backchannelSampleRate) {
-				log.Log.Info("Audio.WriteAudioToBackchannel(): stopped")
+				log.Info("Audio.WriteAudioToBackchannel(): stopped")
 				return
 			}
 		}
@@ -162,7 +156,7 @@ func reconnectBackchannel(ctx context.Context, otelContext context.Context, rtsp
 	backoff := backchannelReconnectInitial
 	for {
 		if err := rtspClient.Close(otelContext); err != nil {
-			log.Log.Error("Audio.WriteAudioToBackchannel(): error closing failed backchannel: " + err.Error())
+			log.Error("Audio.WriteAudioToBackchannel(): error closing failed backchannel: " + err.Error())
 		}
 		if ctx.Err() != nil {
 			return false
@@ -173,11 +167,11 @@ func reconnectBackchannel(ctx context.Context, otelContext context.Context, rtsp
 			err = rtspClient.StartBackChannel(ctx, otelContext)
 		}
 		if err == nil {
-			log.Log.Info("Audio.WriteAudioToBackchannel(): reconnected backchannel")
+			log.Info("Audio.WriteAudioToBackchannel(): reconnected backchannel")
 			return true
 		}
 
-		log.Log.Error("Audio.WriteAudioToBackchannel(): error reconnecting backchannel: " + err.Error())
+		log.Error("Audio.WriteAudioToBackchannel(): error reconnecting backchannel: " + err.Error())
 		if !waitForBackchannel(ctx, backoff) {
 			return false
 		}
@@ -204,7 +198,12 @@ func WriteFileToBackChannel(infile av.DemuxCloser) {
 	// Do the warmup!
 	file, err := os.Open("./audiofile.bye")
 	if err != nil {
-		fmt.Println("WriteFileToBackChannel: error opening audiofile.bye file")
+		log.WithError(err).WithFields(log.Fields{
+			"component": "backchannel",
+			"event":     "audio_file_open_failed",
+			"path":      "./audiofile.bye",
+		}).Error("Failed to open backchannel audio file")
+		return
 	}
 	defer file.Close()
 

machinery/src/components/watchdog_test.go

@@ -0,0 +1,82 @@
+package components
+
+import (
+	"context"
+	"testing"
+	"time"
+)
+
+func TestWaitForRunRetryStopsOnCancellation(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	started := time.Now()
+	if waitForRunRetry(ctx) {
+		t.Fatal("waitForRunRetry() completed the retry delay after cancellation")
+	}
+	if elapsed := time.Since(started); elapsed > 100*time.Millisecond {
+		t.Fatalf("waitForRunRetry() took %s after cancellation", elapsed)
+	}
+}
+
+func TestStreamRestartWatchdogCoalescesStallsAndBacksOff(t *testing.T) {
+	now := time.Unix(1_000, 0)
+	watchdog := newStreamRestartWatchdog()
+	if _, restart := watchdog.Observe(now, 1, 1, true, false); restart {
+		t.Fatal("initial observation requested a restart")
+	}
+
+	for check := 1; check < streamWatchdogStallChecks; check++ {
+		now = now.Add(streamWatchdogInterval)
+		if _, restart := watchdog.Observe(now, 1, 1, true, false); restart {
+			t.Fatalf("check %d requested an early restart", check)
+		}
+	}
+	now = now.Add(streamWatchdogInterval)
+	reason, restart := watchdog.Observe(now, 1, 1, true, false)
+	if !restart || reason != "main and sub streams" {
+		t.Fatalf("Observe() = (%q, %t), want coalesced restart", reason, restart)
+	}
+
+	watchdog.MarkRestart(now)
+	if got := watchdog.Backoff(); got != 30*time.Second {
+		t.Fatalf("Backoff() = %s, want 30s", got)
+	}
+	for now = now.Add(streamWatchdogInterval); now.Before(watchdog.nextRestart); now = now.Add(streamWatchdogInterval) {
+		if _, restart := watchdog.Observe(now, 1, 1, true, false); restart {
+			t.Fatal("restart requested during cooldown")
+		}
+	}
+}
+
+func TestStreamRestartWatchdogResetsAfterHealthyMinute(t *testing.T) {
+	now := time.Unix(2_000, 0)
+	watchdog := newStreamRestartWatchdog()
+	watchdog.backoff = streamWatchdogMaxBackoff
+	watchdog.nextRestart = now.Add(streamWatchdogMaxBackoff)
+	watchdog.Observe(now, 1, 1, true, false)
+
+	for elapsed := streamWatchdogInterval; elapsed <= streamWatchdogHealthyReset+streamWatchdogInterval; elapsed += streamWatchdogInterval {
+		now = now.Add(streamWatchdogInterval)
+		watchdog.Observe(now, int64(elapsed), int64(elapsed), true, false)
+	}
+
+	if got := watchdog.Backoff(); got != streamWatchdogBaseBackoff {
+		t.Fatalf("Backoff() = %s, want %s", got, streamWatchdogBaseBackoff)
+	}
+	if !watchdog.nextRestart.IsZero() {
+		t.Fatalf("nextRestart = %s, want zero", watchdog.nextRestart)
+	}
+}
+
+func TestStreamRestartWatchdogPausesWhileConfiguring(t *testing.T) {
+	now := time.Unix(3_000, 0)
+	watchdog := newStreamRestartWatchdog()
+	watchdog.Observe(now, 1, 0, false, false)
+	for check := 0; check < streamWatchdogStallChecks+1; check++ {
+		now = now.Add(streamWatchdogInterval)
+		if _, restart := watchdog.Observe(now, 1, 0, false, true); restart {
+			t.Fatal("restart requested while configuring")
+		}
+	}
+}

machinery/src/computervision/main.go

@@ -8,14 +8,14 @@ import (
 	geo "github.com/kellydunn/golang-geo"
 	"github.com/kerberos-io/agent/machinery/src/capture"
 	"github.com/kerberos-io/agent/machinery/src/conditions"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
+	log "github.com/sirupsen/logrus"
 )
 
 func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Configuration, communication *models.Communication, mqttClient mqtt.Client, rtspClient capture.RTSPClient) {
 
-	log.Log.Debug("computervision.main.ProcessMotion(): start motion detection")
+	log.Debug("computervision.main.ProcessMotion(): start motion detection")
 	config := configuration.Config
 	loc, _ := time.LoadLocation(config.Timezone)
 
@@ -42,14 +42,14 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 
 	if continuousMode && !hasMotionRegion {
 
-		log.Log.Info("computervision.main.ProcessMotion(): continuous recording enabled and no motion region configured, so no motion detection required.")
+		log.Info("computervision.main.ProcessMotion(): continuous recording enabled and no motion region configured, so no motion detection required.")
 
 	} else {
 
 		if continuousMode {
-			log.Log.Info("computervision.main.ProcessMotion(): continuous recording enabled with a motion region, running motion detection for live-view visualisation only (no motion-triggered recording).")
+			log.Info("computervision.main.ProcessMotion(): continuous recording enabled with a motion region, running motion detection for live-view visualisation only (no motion-triggered recording).")
 		} else {
-			log.Log.Info("computervision.main.ProcessMotion(): motion detected is enabled, so starting the motion detection.")
+			log.Info("computervision.main.ProcessMotion(): motion detected is enabled, so starting the motion detection.")
 		}
 
 		hubKey := config.HubKey
@@ -185,7 +185,7 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 				// We'll validate those conditions and if not valid we'll not do anything.
 				detectMotion, err := conditions.Validate(loc, configuration)
 				if !detectMotion && err != nil {
-					log.Log.Debug("computervision.main.ProcessMotion(): " + err.Error() + ".")
+					log.Debug("computervision.main.ProcessMotion(): " + err.Error() + ".")
 				}
 
 				// Run detection when motion is enabled, OR when we're in continuous
@@ -235,7 +235,7 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 										if err == nil {
 											mqttClient.Publish("kerberos/hub/"+hubKey, 2, false, payload)
 										} else {
-											log.Log.Info("computervision.main.ProcessMotion(): failed to package MQTT message: " + err.Error())
+											log.Info("computervision.main.ProcessMotion(): failed to package MQTT message: " + err.Error())
 										}
 									} else {
 										mqttClient.Publish("kerberos/agent/"+deviceKey, 2, false, "motion")
@@ -254,7 +254,9 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 									NumberOfChanges: changesToReturn,
 									Rectangle:       motionRectangle,
 								}
-								communication.HandleMotion <- dataToPass //Save data to the channel
+								if !communication.TrySendMotion(dataToPass) {
+									log.Warn("computervision.main.ProcessMotion(): motion channel unavailable or full, dropping recording trigger")
+								}
 							}
 						}
 					}
@@ -271,7 +273,7 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 		}
 	}
 
-	log.Log.Debug("computervision.main.ProcessMotion(): stop the motion detection.")
+	log.Debug("computervision.main.ProcessMotion(): stop the motion detection.")
 }
 
 func FindMotion(imageArray [3]*image.Gray, coordinatesPerRegion [][]int, pixelChangeThreshold int) (thresholdReached bool, changesDetected int, motionRectangle models.MotionRectangle, motionRectangles []models.MotionRectangle) {
@@ -374,14 +376,28 @@ func AbsDiffBitwiseAndThreshold(img1 *image.Gray, img2 *image.Gray, img3 *image.
 				endY = pixel[1]
 			}
 		}
-		log.Log.Debugf("Rectangle of changes detected: startX: %d, startY: %d, endX: %d, endY: %d", startX, startY, endX, endY)
+		log.WithFields(log.Fields{
+			"component": "computer_vision",
+			"end_x":     endX,
+			"end_y":     endY,
+			"event":     "motion_bounds_detected",
+			"start_x":   startX,
+			"start_y":   startY,
+		}).Debug("Motion bounds detected")
 		motionRectangle = models.MotionRectangle{
 			X:      startX,
 			Y:      startY,
 			Width:  endX - startX,
 			Height: endY - startY,
 		}
-		log.Log.Debugf("Motion rectangle: %+v", motionRectangle)
+		log.WithFields(log.Fields{
+			"component": "computer_vision",
+			"event":     "motion_rectangle_created",
+			"height":    motionRectangle.Height,
+			"width":     motionRectangle.Width,
+			"x":         motionRectangle.X,
+			"y":         motionRectangle.Y,
+		}).Debug("Motion rectangle created")
 	}
 
 	// Cluster the changed pixels into separate bounding boxes so the live view can

machinery/src/conditions/timewindow.go

@@ -3,8 +3,8 @@ package conditions
 import (
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 func IsWithinTimeInterval(loc *time.Location, configuration *models.Configuration) (enabled bool) {
@@ -27,9 +27,9 @@ func IsWithinTimeInterval(loc *time.Location, configuration *models.Configuratio
 				currentTimeInSeconds := hour*60*60 + minute*60 + second
 				if (currentTimeInSeconds >= start1 && currentTimeInSeconds <= end1) ||
 					(currentTimeInSeconds >= start2 && currentTimeInSeconds <= end2) {
-					log.Log.Debug("conditions.timewindow.IsWithinTimeInterval(): time interval valid, enabling recording.")
+					log.Debug("conditions.timewindow.IsWithinTimeInterval(): time interval valid, enabling recording.")
 				} else {
-					log.Log.Info("conditions.timewindow.IsWithinTimeInterval(): time interval not valid, disabling recording.")
+					log.Info("conditions.timewindow.IsWithinTimeInterval(): time interval not valid, disabling recording.")
 					enabled = false
 				}
 			}

machinery/src/conditions/uri.go

@@ -3,13 +3,26 @@ package conditions
 import (
 	"bytes"
 	"crypto/tls"
-	"fmt"
+	"encoding/json"
+	"io"
 	"net/http"
 	"os"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
+)
+
+const conditionHTTPTimeout = 10 * time.Second
+
+var (
+	conditionHTTPClient         = &http.Client{Timeout: conditionHTTPTimeout}
+	conditionInsecureHTTPClient = &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // #nosec G402 -- explicit operator opt-in
+		},
+		Timeout: conditionHTTPTimeout,
+	}
 )
 
 func IsValidUriResponse(configuration *models.Configuration) (enabled bool) {
@@ -17,41 +30,66 @@ func IsValidUriResponse(configuration *models.Configuration) (enabled bool) {
 	conditionURI := config.ConditionURI
 	enabled = true
 	if conditionURI != "" {
-
-		// We will send a POST request to the conditionURI, and expect a 200 response.
-		// In the payload we will send some information, so the other end can decide
-		// if it should enable or disable recording.
-
-		var client *http.Client
+		client := conditionHTTPClient
 		if os.Getenv("AGENT_TLS_INSECURE") == "true" {
-			tr := &http.Transport{
-				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-			}
-			client = &http.Client{Transport: tr}
-		} else {
-			client = &http.Client{}
+			client = conditionInsecureHTTPClient
 		}
 
-		var object = fmt.Sprintf(`{
-			"camera_id" : "%s",
-			"camera_name" : "%s",
-			"site_id" : "%s",
-			"hub_key" : "%s",
-			"timestamp" : "%s",
-		}`, config.Key, config.FriendlyName, config.HubSite, config.HubKey, time.Now().Format("2006-01-02 15:04:05"))
+		payload := struct {
+			CameraID   string `json:"camera_id"`
+			CameraName string `json:"camera_name"`
+			SiteID     string `json:"site_id"`
+			HubKey     string `json:"hub_key"`
+			Timestamp  string `json:"timestamp"`
+		}{
+			CameraID:   config.Key,
+			CameraName: config.FriendlyName,
+			SiteID:     config.HubSite,
+			HubKey:     config.HubKey,
+			Timestamp:  time.Now().Format("2006-01-02 15:04:05"),
+		}
+		jsonBody, err := json.Marshal(payload)
+		if err != nil {
+			log.WithError(err).WithFields(log.Fields{
+				"component": "conditions/uri",
+				"event":     "request_encoding_failed",
+			}).Error("Failed to encode condition request")
+			return false
+		}
 
-		var jsonStr = []byte(object)
-		buffy := bytes.NewBuffer(jsonStr)
-		req, _ := http.NewRequest("POST", conditionURI, buffy)
+		req, err := http.NewRequest(http.MethodPost, conditionURI, bytes.NewReader(jsonBody))
+		if err != nil {
+			log.WithError(err).WithFields(log.Fields{
+				"component": "conditions/uri",
+				"event":     "request_creation_failed",
+			}).Error("Failed to create condition request")
+			return false
+		}
 		req.Header.Set("Content-Type", "application/json")
 		resp, err := client.Do(req)
 		if resp != nil {
+			_, _ = io.Copy(io.Discard, resp.Body)
 			resp.Body.Close()
 		}
-		if err == nil && resp.StatusCode == 200 {
-			log.Log.Info("conditions.uri.IsValidUriResponse(): response 200, enabling recording.")
+		if err == nil && resp != nil && resp.StatusCode == http.StatusOK {
+			log.WithFields(log.Fields{
+				"component":   "conditions/uri",
+				"event":       "recording_enabled",
+				"status_code": resp.StatusCode,
+			}).Info("Condition request enabled recording")
 		} else {
-			log.Log.Info("conditions.uri.IsValidUriResponse(): response not 200, disabling recording.")
+			if err != nil {
+				log.WithError(err).WithFields(log.Fields{
+					"component": "conditions/uri",
+					"event":     "request_failed",
+				}).Error("Condition request failed")
+			} else {
+				log.WithFields(log.Fields{
+					"component":   "conditions/uri",
+					"event":       "recording_disabled",
+					"status_code": resp.StatusCode,
+				}).Info("Condition request disabled recording")
+			}
 			enabled = false
 		}
 	}

machinery/src/conditions/uri_test.go

@@ -0,0 +1,63 @@
+package conditions
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+
+	"github.com/kerberos-io/agent/machinery/src/models"
+)
+
+func TestIsValidUriResponseReusesClientAndEncodesPayload(t *testing.T) {
+	var requests atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		requests.Add(1)
+		if request.Method != http.MethodPost {
+			t.Errorf("method = %s, want POST", request.Method)
+		}
+		if got := request.Header.Get("Content-Type"); got != "application/json" {
+			t.Errorf("Content-Type = %q, want application/json", got)
+		}
+		var payload struct {
+			CameraID   string `json:"camera_id"`
+			CameraName string `json:"camera_name"`
+		}
+		if err := json.NewDecoder(request.Body).Decode(&payload); err != nil {
+			t.Errorf("decode request: %v", err)
+		}
+		if payload.CameraID != "camera-1" || payload.CameraName != `Front "Door"` {
+			t.Errorf("payload = %+v", payload)
+		}
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer server.Close()
+
+	configuration := &models.Configuration{}
+	configuration.Config.ConditionURI = server.URL
+	configuration.Config.Key = "camera-1"
+	configuration.Config.FriendlyName = `Front "Door"`
+
+	for i := 0; i < 2; i++ {
+		if !IsValidUriResponse(configuration) {
+			t.Fatal("IsValidUriResponse() = false, want true")
+		}
+	}
+	if got := requests.Load(); got != 2 {
+		t.Fatalf("requests = %d, want 2", got)
+	}
+}
+
+func TestIsValidUriResponseRejectsNonOK(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		http.Error(w, "unavailable", http.StatusServiceUnavailable)
+	}))
+	defer server.Close()
+
+	configuration := &models.Configuration{}
+	configuration.Config.ConditionURI = server.URL
+	if IsValidUriResponse(configuration) {
+		t.Fatal("IsValidUriResponse() = true, want false")
+	}
+}

machinery/src/config/main.go

@@ -13,11 +13,36 @@ import (
 
 	"github.com/InVisionApp/conjungo"
 	"github.com/kerberos-io/agent/machinery/src/database"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
 )
 
+const factoryConfigRetryDelay = 5 * time.Second
+
+func newFactoryConfigReadContext() (context.Context, context.CancelFunc) {
+	return context.WithTimeout(context.Background(), database.TIMEOUT)
+}
+
+func isRetryableFactoryConfigReadError(err error) bool {
+	return errors.Is(err, context.DeadlineExceeded) || mongo.IsTimeout(err) || mongo.IsNetworkError(err)
+}
+
+func readFactoryConfig(collection *mongo.Collection, filter bson.M, destination *models.Config) error {
+	for {
+		ctx, cancel := newFactoryConfigReadContext()
+		err := collection.FindOne(ctx, filter).Decode(destination)
+		cancel()
+		if !isRetryableFactoryConfigReadError(err) {
+			return err
+		}
+
+		log.Warn("Factory configuration read timed out or lost its database connection; retrying.")
+		time.Sleep(factoryConfigRetryDelay)
+	}
+}
+
 // ReadUserConfig Reads the user configuration of the Kerberos Open Source instance.
 // This will return a models.User struct including the username, password,
 // selected language, and if the installation was completed or not.
@@ -25,14 +50,14 @@ func ReadUserConfig(configDirectory string) (userConfig models.User) {
 	for {
 		jsonFile, err := os.Open(configDirectory + "/data/config/user.json")
 		if err != nil {
-			log.Log.Error("Config file is not found " + configDirectory + "/data/config/user.json, trying again in 5s: " + err.Error())
+			log.Error("Config file is not found " + configDirectory + "/data/config/user.json, trying again in 5s: " + err.Error())
 			time.Sleep(5 * time.Second)
 		} else {
-			log.Log.Info("Successfully Opened user.json")
+			log.Info("Successfully Opened user.json")
 			byteValue, _ := ioutil.ReadAll(jsonFile)
 			err = json.Unmarshal(byteValue, &userConfig)
 			if err != nil {
-				log.Log.Error("JSON file not valid: " + err.Error())
+				log.Error("JSON file not valid: " + err.Error())
 			} else {
 				jsonFile.Close()
 				break
@@ -63,21 +88,15 @@ func OpenConfig(configDirectory string, configuration *models.Configuration) {
 		collection := db.Collection("configuration")
 
 		var globalConfig models.Config
-		res := collection.FindOne(context.Background(), bson.M{
+		err := readFactoryConfig(collection, bson.M{
 			"type": "global",
-		})
-
-		if res.Err() != nil {
-			log.Log.Error("Could not find global configuration, using default configuration.")
-			panic("Could not find global configuration, using default configuration.")
-		}
-		err := res.Decode(&globalConfig)
+		}, &globalConfig)
 		if err != nil {
-			log.Log.Error("Could not find global configuration, using default configuration.")
+			log.Error("Could not find global configuration, using default configuration.")
 			panic("Could not find global configuration, using default configuration.")
 		}
 		if globalConfig.Type != "global" {
-			log.Log.Error("Could not find global configuration, might missed the mongodb connection.")
+			log.Error("Could not find global configuration, might missed the mongodb connection.")
 			panic("Could not find global configuration, might missed the mongodb connection.")
 		}
 
@@ -85,21 +104,16 @@ func OpenConfig(configDirectory string, configuration *models.Configuration) {
 
 		var customConfig models.Config
 		deploymentName := os.Getenv("DEPLOYMENT_NAME")
-		res = collection.FindOne(context.Background(), bson.M{
+		err = readFactoryConfig(collection, bson.M{
 			"type": "config",
 			"name": deploymentName,
-		})
-		if res.Err() != nil {
-			log.Log.Error("Could not find configuration for " + deploymentName + ", using global configuration.")
-		}
-		err = res.Decode(&customConfig)
+		}, &customConfig)
 		if err != nil {
-			log.Log.Error("Could not find configuration for " + deploymentName + ", using global configuration.")
-		}
-
-		if customConfig.Type != "config" {
-			log.Log.Error("Could not find custom configuration, might missed the mongodb connection.")
-			panic("Could not find custom configuration, might missed the mongodb connection.")
+			log.Error("Could not find configuration for " + deploymentName + ", using global configuration.")
+			customConfig = models.Config{}
+		} else if customConfig.Type != "config" {
+			log.Error("Custom configuration has an invalid type, using global configuration.")
+			customConfig = models.Config{}
 		}
 		configuration.CustomConfig = customConfig
 
@@ -166,21 +180,29 @@ func OpenConfig(configDirectory string, configuration *models.Configuration) {
 
 		// Open device config
 		for {
-			jsonFile, err := os.Open(configDirectory + "/data/config/config.json")
+			configPath := configDirectory + "/data/config/config.json"
+			jsonFile, err := os.Open(configPath)
 			if err != nil {
-				log.Log.Error("Config file is not found " + configDirectory + "/data/config/config.json" + ", trying again in 5s.")
+				log.WithError(err).WithFields(log.Fields{
+					"component":      "config",
+					"event":          "configuration_open_failed",
+					"retry_delay_ms": 5000,
+				}).Warn("Configuration file unavailable; retrying")
 				time.Sleep(5 * time.Second)
 			} else {
-				log.Log.Info("Successfully Opened config.json from " + configuration.Name)
+				log.WithFields(log.Fields{
+					"component": "config",
+					"event":     "configuration_opened",
+				}).Info("Configuration file opened")
 				byteValue, _ := ioutil.ReadAll(jsonFile)
 				err = json.Unmarshal(byteValue, &configuration.Config)
 				jsonFile.Close()
 				if err != nil {
-					log.Log.Error("JSON file not valid: " + err.Error())
+					log.Error("JSON file not valid: " + err.Error())
 				} else {
 					err = json.Unmarshal(byteValue, &configuration.CustomConfig)
 					if err != nil {
-						log.Log.Error("JSON file not valid: " + err.Error())
+						log.Error("JSON file not valid: " + err.Error())
 					} else {
 						break
 					}
@@ -687,12 +709,12 @@ func SaveConfig(configDirectory string, config models.Config, configuration *mod
 			return err
 		}
 
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			select {
 			case communication.HandleBootstrap <- "restart":
-				log.Log.Info("config.main.SaveConfig(): update config, restart agent.")
+				log.Info("config.main.SaveConfig(): update config, restart agent.")
 			case <-time.After(1 * time.Second):
-				log.Log.Info("config.main.SaveConfig(): update config, restart agent.")
+				log.Info("config.main.SaveConfig(): update config, restart agent.")
 			}
 		}
 

machinery/src/config/main_test.go

@@ -1,9 +1,14 @@
 package config
 
 import (
+	"context"
+	"errors"
 	"testing"
+	"time"
 
+	"github.com/kerberos-io/agent/machinery/src/database"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	"go.mongodb.org/mongo-driver/mongo"
 )
 
 func TestApplyAgentEnvVarsPixelChangeThresholdDefault(t *testing.T) {
@@ -38,3 +43,36 @@ func TestApplyAgentEnvVarsPixelChangeThresholdDefault(t *testing.T) {
 func intPointer(value int) *int {
 	return &value
 }
+
+func TestNewFactoryConfigReadContextUsesDatabaseTimeout(t *testing.T) {
+	ctx, cancel := newFactoryConfigReadContext()
+	defer cancel()
+
+	deadline, ok := ctx.Deadline()
+	if !ok {
+		t.Fatal("expected read context deadline")
+	}
+
+	remaining := time.Until(deadline)
+	if remaining <= 0 {
+		t.Fatalf("deadline already expired: %v", remaining)
+	}
+	if remaining > database.TIMEOUT {
+		t.Fatalf("remaining deadline = %v, want <= %v", remaining, database.TIMEOUT)
+	}
+	if remaining < database.TIMEOUT-time.Second {
+		t.Fatalf("remaining deadline = %v, want close to %v", remaining, database.TIMEOUT)
+	}
+}
+
+func TestFactoryConfigRetryableErrors(t *testing.T) {
+	if !isRetryableFactoryConfigReadError(context.DeadlineExceeded) {
+		t.Fatal("context deadline should be retryable")
+	}
+	if isRetryableFactoryConfigReadError(mongo.ErrNoDocuments) {
+		t.Fatal("missing configuration should not retry")
+	}
+	if isRetryableFactoryConfigReadError(errors.New("invalid BSON")) {
+		t.Fatal("decode errors should not retry")
+	}
+}

machinery/src/database/main.go

@@ -7,6 +7,7 @@ import (
 	"sync"
 	"time"
 
+	log "github.com/sirupsen/logrus"
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
 )
@@ -44,18 +45,34 @@ func New() *DB {
 		// We can also apply the complete URI
 		// e.g. "mongodb+srv://<username>:<password>@kerberos-hub.shhng.mongodb.net/?retryWrites=true&w=majority&appName=kerberos-hub"
 		if mongodbURI != "" {
+			log.WithFields(log.Fields{
+				"component":      "database",
+				"database":       DatabaseName,
+				"event":          "client_configuring",
+				"uri_configured": true,
+			}).Debug("Configuring MongoDB client")
 			serverAPI := options.ServerAPI(options.ServerAPIVersion1)
 			opts := options.Client().ApplyURI(mongodbURI).SetServerAPIOptions(serverAPI)
 
 			// Create a new client and connect to the server
 			client, err := mongo.Connect(ctx, opts)
 			if err != nil {
-				fmt.Printf("Error setting up mongodb connection: %+v\n", err)
-				os.Exit(1)
+				log.WithError(err).WithFields(log.Fields{
+					"component": "database",
+					"database":  DatabaseName,
+					"event":     "client_configuration_failed",
+				}).Fatal("Failed to configure MongoDB client")
 			}
 			_instance.Client = client
 
 		} else {
+			log.WithFields(log.Fields{
+				"component":             "database",
+				"database":              DatabaseName,
+				"event":                 "client_configuring",
+				"replicaset_configured": replicaset != "",
+				"uri_configured":        false,
+			}).Debug("Configuring MongoDB client")
 
 			// New MongoDB driver
 			mongodbURI := fmt.Sprintf("mongodb://%s:%s@%s", username, password, host)
@@ -69,8 +86,11 @@ func New() *DB {
 				Password:      password,
 			}))
 			if err != nil {
-				fmt.Printf("Error setting up mongodb connection: %+v\n", err)
-				os.Exit(1)
+				log.WithError(err).WithFields(log.Fields{
+					"component": "database",
+					"database":  DatabaseName,
+					"event":     "client_configuration_failed",
+				}).Fatal("Failed to configure MongoDB client")
 			}
 			_instance.Client = client
 		}

machinery/src/lifecycle/supervisor.go

@@ -0,0 +1,336 @@
+package lifecycle
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"runtime/debug"
+	"sort"
+	"sync"
+	"time"
+)
+
+var (
+	ErrSupervisorSealed  = errors.New("supervisor is sealed")
+	ErrSupervisorStopped = errors.New("supervisor is shutting down")
+	ErrTaskExists        = errors.New("task already exists")
+)
+
+type TaskFunc func(context.Context) error
+
+type TaskPolicy struct {
+	Required    bool
+	LongRunning bool
+}
+
+type TaskStatus string
+
+const (
+	TaskRunning   TaskStatus = "running"
+	TaskSucceeded TaskStatus = "succeeded"
+	TaskFailed    TaskStatus = "failed"
+	TaskPanicked  TaskStatus = "panicked"
+)
+
+type TaskSnapshot struct {
+	Name      string
+	Policy    TaskPolicy
+	Status    TaskStatus
+	StartedAt time.Time
+	EndedAt   time.Time
+	Error     string
+	Panic     string
+	Stack     []byte
+}
+
+type Failure struct {
+	Task  string
+	Cause error
+	Panic string
+	Stack []byte
+}
+
+func (f Failure) Error() string {
+	if f.Cause == nil {
+		return fmt.Sprintf("task %q failed", f.Task)
+	}
+	return f.Cause.Error()
+}
+
+type ShutdownReport struct {
+	Complete bool
+	Cause    error
+	Tasks    []TaskSnapshot
+	Running  []TaskSnapshot
+}
+
+type supervisorPhase uint8
+
+const (
+	phaseStarting supervisorPhase = iota
+	phaseRunning
+	phaseDraining
+)
+
+type taskState struct {
+	TaskSnapshot
+}
+
+type Supervisor struct {
+	ctx    context.Context
+	cancel context.CancelCauseFunc
+
+	mu        sync.Mutex
+	phase     supervisorPhase
+	sealed    bool
+	active    int
+	tasks     map[string]*taskState
+	allDone   chan struct{}
+	doneOnce  sync.Once
+	start     chan struct{}
+	startOnce sync.Once
+
+	failures    chan Failure
+	failureOnce sync.Once
+
+	stopParentCancel func() bool
+}
+
+func NewSupervisor(parent context.Context) *Supervisor {
+	if parent == nil {
+		parent = context.Background()
+	}
+
+	ctx, cancel := context.WithCancelCause(parent)
+	supervisor := &Supervisor{
+		ctx:      ctx,
+		cancel:   cancel,
+		phase:    phaseStarting,
+		tasks:    make(map[string]*taskState),
+		allDone:  make(chan struct{}),
+		start:    make(chan struct{}),
+		failures: make(chan Failure, 1),
+	}
+
+	stopParentCancel := context.AfterFunc(parent, func() {
+		supervisor.BeginShutdown(context.Cause(parent))
+	})
+	supervisor.mu.Lock()
+	if supervisor.sealed && supervisor.active == 0 {
+		supervisor.mu.Unlock()
+		stopParentCancel()
+	} else {
+		supervisor.stopParentCancel = stopParentCancel
+		supervisor.mu.Unlock()
+	}
+
+	return supervisor
+}
+
+func (s *Supervisor) Context() context.Context {
+	return s.ctx
+}
+
+func (s *Supervisor) Failures() <-chan Failure {
+	return s.failures
+}
+
+func (s *Supervisor) Go(name string, policy TaskPolicy, task TaskFunc) error {
+	if name == "" {
+		return errors.New("task name is required")
+	}
+	if task == nil {
+		return errors.New("task function is required")
+	}
+
+	s.mu.Lock()
+	if s.sealed {
+		s.mu.Unlock()
+		return fmt.Errorf("%w: %s", ErrSupervisorSealed, name)
+	}
+	if s.ctx.Err() != nil {
+		s.mu.Unlock()
+		return fmt.Errorf("%w: %s", ErrSupervisorStopped, name)
+	}
+	if _, exists := s.tasks[name]; exists {
+		s.mu.Unlock()
+		return fmt.Errorf("%w: %s", ErrTaskExists, name)
+	}
+
+	state := &taskState{TaskSnapshot: TaskSnapshot{
+		Name:      name,
+		Policy:    policy,
+		Status:    TaskRunning,
+		StartedAt: time.Now(),
+	}}
+	s.tasks[name] = state
+	s.active++
+	s.mu.Unlock()
+
+	go s.run(state, task)
+	return nil
+}
+
+func (s *Supervisor) run(state *taskState, task TaskFunc) {
+	var taskErr error
+	panicked := true
+	var panicValue string
+	var panicStack []byte
+
+	defer func() {
+		recovered := recover()
+		if panicked {
+			panicValue = fmt.Sprint(recovered)
+			panicStack = debug.Stack()
+			taskErr = fmt.Errorf("task %q panicked: %s", state.Name, panicValue)
+		}
+		s.finish(state, taskErr, panicked, panicValue, panicStack)
+	}()
+
+	select {
+	case <-s.start:
+	case <-s.ctx.Done():
+		panicked = false
+		return
+	}
+
+	taskErr = task(s.ctx)
+	panicked = false
+}
+
+func (s *Supervisor) finish(
+	state *taskState,
+	taskErr error,
+	panicked bool,
+	panicValue string,
+	panicStack []byte,
+) {
+	var failure *Failure
+
+	s.mu.Lock()
+	state.EndedAt = time.Now()
+	state.Error = ""
+	state.Panic = panicValue
+	state.Stack = append([]byte(nil), panicStack...)
+	switch {
+	case panicked:
+		state.Status = TaskPanicked
+		state.Error = taskErr.Error()
+	case taskErr != nil:
+		state.Status = TaskFailed
+		state.Error = taskErr.Error()
+	default:
+		state.Status = TaskSucceeded
+	}
+
+	shuttingDown := s.phase == phaseDraining || s.ctx.Err() != nil
+	unexpected := panicked || (!shuttingDown &&
+		(state.Policy.Required && (taskErr != nil || state.Policy.LongRunning)))
+	if unexpected {
+		cause := taskErr
+		if cause == nil {
+			cause = fmt.Errorf("required long-running task %q exited", state.Name)
+		}
+		failure = &Failure{
+			Task:  state.Name,
+			Cause: cause,
+			Panic: panicValue,
+			Stack: append([]byte(nil), panicStack...),
+		}
+		s.phase = phaseDraining
+		s.sealed = true
+		s.cancel(failure.Cause)
+		s.failureOnce.Do(func() {
+			s.failures <- *failure
+		})
+	}
+
+	s.active--
+	s.closeDoneIfReadyLocked()
+	s.mu.Unlock()
+}
+
+func (s *Supervisor) Seal() {
+	s.mu.Lock()
+	s.sealed = true
+	if s.phase == phaseStarting {
+		s.phase = phaseRunning
+	}
+	s.startOnce.Do(func() {
+		close(s.start)
+	})
+	s.closeDoneIfReadyLocked()
+	s.mu.Unlock()
+}
+
+func (s *Supervisor) BeginShutdown(cause error) {
+	if cause == nil {
+		cause = context.Canceled
+	}
+
+	s.mu.Lock()
+	s.phase = phaseDraining
+	s.sealed = true
+	s.cancel(cause)
+	s.closeDoneIfReadyLocked()
+	s.mu.Unlock()
+}
+
+func (s *Supervisor) Wait(ctx context.Context) ShutdownReport {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+
+	select {
+	case <-s.allDone:
+	case <-ctx.Done():
+	}
+
+	return s.report()
+}
+
+func (s *Supervisor) Snapshot() []TaskSnapshot {
+	return s.report().Tasks
+}
+
+func (s *Supervisor) closeDoneIfReadyLocked() {
+	if !s.sealed || s.active != 0 {
+		return
+	}
+	if s.stopParentCancel != nil {
+		s.stopParentCancel()
+		s.stopParentCancel = nil
+	}
+	s.doneOnce.Do(func() {
+		close(s.allDone)
+	})
+}
+
+func (s *Supervisor) report() ShutdownReport {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	tasks := make([]TaskSnapshot, 0, len(s.tasks))
+	running := make([]TaskSnapshot, 0)
+	for _, state := range s.tasks {
+		snapshot := state.TaskSnapshot
+		snapshot.Stack = append([]byte(nil), state.Stack...)
+		tasks = append(tasks, snapshot)
+		if snapshot.Status == TaskRunning {
+			running = append(running, snapshot)
+		}
+	}
+	sort.Slice(tasks, func(i, j int) bool {
+		return tasks[i].Name < tasks[j].Name
+	})
+	sort.Slice(running, func(i, j int) bool {
+		return running[i].Name < running[j].Name
+	})
+
+	return ShutdownReport{
+		Complete: s.sealed && s.active == 0,
+		Cause:    context.Cause(s.ctx),
+		Tasks:    tasks,
+		Running:  running,
+	}
+}

machinery/src/lifecycle/supervisor_test.go

@@ -0,0 +1,300 @@
+package lifecycle
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+func TestSupervisorWaitsForSeal(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	started := make(chan struct{})
+	if err := supervisor.Go("quick", TaskPolicy{}, func(context.Context) error {
+		close(started)
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	waitContext, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+	defer cancel()
+	if report := supervisor.Wait(waitContext); report.Complete {
+		t.Fatal("Wait() completed before Seal()")
+	}
+	select {
+	case <-started:
+		t.Fatal("task started before Seal()")
+	default:
+	}
+
+	supervisor.Seal()
+	<-started
+	if report := supervisor.Wait(context.Background()); !report.Complete {
+		t.Fatal("Wait() did not complete after Seal()")
+	}
+}
+
+func TestSupervisorSealsEmptyGroup(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	supervisor.Seal()
+
+	report := supervisor.Wait(context.Background())
+	if !report.Complete || len(report.Tasks) != 0 {
+		t.Fatalf("Wait() report = %+v, want complete empty report", report)
+	}
+}
+
+func TestSupervisorReportsRunningTasksOnTimeout(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	release := make(chan struct{})
+	if err := supervisor.Go("blocked", TaskPolicy{}, func(context.Context) error {
+		<-release
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+
+	waitContext, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+	defer cancel()
+	report := supervisor.Wait(waitContext)
+	if report.Complete {
+		t.Fatal("Wait() unexpectedly completed")
+	}
+	if len(report.Running) != 1 || report.Running[0].Name != "blocked" {
+		t.Fatalf("Wait() running tasks = %+v, want blocked", report.Running)
+	}
+
+	close(release)
+	if report = supervisor.Wait(context.Background()); !report.Complete {
+		t.Fatal("Wait() did not complete after releasing task")
+	}
+}
+
+func TestSupervisorFailsWhenRequiredLongRunningTaskExits(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	release := make(chan struct{})
+	if err := supervisor.Go("recorder", TaskPolicy{Required: true, LongRunning: true}, func(context.Context) error {
+		<-release
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+	close(release)
+
+	select {
+	case failure := <-supervisor.Failures():
+		if failure.Task != "recorder" || !strings.Contains(failure.Error(), "exited") {
+			t.Fatalf("failure = %+v", failure)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("required task exit did not emit a failure")
+	}
+}
+
+func TestSupervisorAllowsRequiredOneShotTaskToSucceed(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	if err := supervisor.Go("setup", TaskPolicy{Required: true}, func(context.Context) error {
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+	if report := supervisor.Wait(context.Background()); !report.Complete {
+		t.Fatal("Wait() did not complete")
+	}
+
+	select {
+	case failure := <-supervisor.Failures():
+		t.Fatalf("successful one-shot task emitted failure: %+v", failure)
+	default:
+	}
+}
+
+func TestSupervisorFailsWhenRequiredOneShotTaskReturnsError(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	wantErr := errors.New("setup failed")
+	if err := supervisor.Go("setup", TaskPolicy{Required: true}, func(context.Context) error {
+		return wantErr
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+
+	select {
+	case failure := <-supervisor.Failures():
+		if !errors.Is(failure.Cause, wantErr) {
+			t.Fatalf("failure cause = %v, want %v", failure.Cause, wantErr)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("required task error did not emit a failure")
+	}
+}
+
+func TestSupervisorRecoversPanicAndCapturesStack(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	if err := supervisor.Go("panic", TaskPolicy{}, func(context.Context) error {
+		panic("boom")
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+
+	select {
+	case failure := <-supervisor.Failures():
+		if failure.Task != "panic" || failure.Panic != "boom" || len(failure.Stack) == 0 {
+			t.Fatalf("failure = %+v, want captured panic and stack", failure)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("panic did not emit a failure")
+	}
+
+	report := supervisor.Wait(context.Background())
+	if !report.Complete || len(report.Tasks) != 1 || len(report.Tasks[0].Stack) == 0 {
+		t.Fatalf("report = %+v, want completed task with captured stack", report)
+	}
+}
+
+func TestSupervisorCapturesEmptyPanicDuringShutdown(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	started := make(chan struct{})
+	if err := supervisor.Go("panic", TaskPolicy{}, func(ctx context.Context) error {
+		close(started)
+		<-ctx.Done()
+		panic("")
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+	<-started
+	supervisor.BeginShutdown(errors.New("requested shutdown"))
+
+	select {
+	case failure := <-supervisor.Failures():
+		if failure.Task != "panic" || len(failure.Stack) == 0 {
+			t.Fatalf("failure = %+v, want captured panic and stack", failure)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("panic during shutdown did not emit a failure")
+	}
+	if report := supervisor.Wait(context.Background()); !report.Complete || report.Tasks[0].Status != TaskPanicked {
+		t.Fatalf("report = %+v, want completed panicked task", report)
+	}
+}
+
+func TestSupervisorPublishesFailureBeforeCompletion(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	wantErr := errors.New("setup failed")
+	if err := supervisor.Go("setup", TaskPolicy{Required: true}, func(context.Context) error {
+		return wantErr
+	}); err != nil {
+		t.Fatal(err)
+	}
+	supervisor.Seal()
+
+	report := supervisor.Wait(context.Background())
+	if !report.Complete || !errors.Is(report.Cause, wantErr) {
+		t.Fatalf("report = %+v, want complete report caused by task failure", report)
+	}
+	select {
+	case failure := <-supervisor.Failures():
+		if !errors.Is(failure.Cause, wantErr) {
+			t.Fatalf("failure cause = %v, want %v", failure.Cause, wantErr)
+		}
+	default:
+		t.Fatal("failure was not published before completion")
+	}
+}
+
+func TestSupervisorCompletedWaitWinsOverCanceledContext(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	supervisor.Seal()
+
+	waitContext, cancel := context.WithCancel(context.Background())
+	cancel()
+	if report := supervisor.Wait(waitContext); !report.Complete {
+		t.Fatalf("report = %+v, want authoritative completion state", report)
+	}
+}
+
+func TestSupervisorRejectsTaskAfterParentCancellation(t *testing.T) {
+	parent, cancel := context.WithCancel(context.Background())
+	supervisor := NewSupervisor(parent)
+	cancel()
+
+	err := supervisor.Go("late", TaskPolicy{}, func(context.Context) error { return nil })
+	if !errors.Is(err, ErrSupervisorStopped) && !errors.Is(err, ErrSupervisorSealed) {
+		t.Fatalf("Go() error = %v, want stopped or sealed supervisor", err)
+	}
+	supervisor.BeginShutdown(context.Canceled)
+	if report := supervisor.Wait(context.Background()); !report.Complete {
+		t.Fatalf("report = %+v, want complete canceled supervisor", report)
+	}
+}
+
+func TestSupervisorShutdownBeforeSealDoesNotStartTasks(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	started := make(chan struct{})
+	if err := supervisor.Go("worker", TaskPolicy{}, func(context.Context) error {
+		close(started)
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	supervisor.BeginShutdown(errors.New("startup aborted"))
+	if report := supervisor.Wait(context.Background()); !report.Complete {
+		t.Fatalf("report = %+v, want completed aborted startup", report)
+	}
+	select {
+	case <-started:
+		t.Fatal("task started after startup was aborted")
+	default:
+	}
+}
+
+func TestSupervisorRejectsDuplicateAndLateTasks(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	release := make(chan struct{})
+	if err := supervisor.Go("worker", TaskPolicy{}, func(context.Context) error {
+		<-release
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := supervisor.Go("worker", TaskPolicy{}, func(context.Context) error { return nil }); !errors.Is(err, ErrTaskExists) {
+		t.Fatalf("duplicate Go() error = %v, want ErrTaskExists", err)
+	}
+
+	supervisor.Seal()
+	if err := supervisor.Go("late", TaskPolicy{}, func(context.Context) error { return nil }); !errors.Is(err, ErrSupervisorSealed) {
+		t.Fatalf("late Go() error = %v, want ErrSupervisorSealed", err)
+	}
+	close(release)
+}
+
+func TestSupervisorConcurrentStartAndSeal(t *testing.T) {
+	supervisor := NewSupervisor(context.Background())
+	var starters sync.WaitGroup
+	for index := 0; index < 100; index++ {
+		starters.Add(1)
+		go func(index int) {
+			defer starters.Done()
+			_ = supervisor.Go(
+				"worker-"+time.Unix(0, int64(index)).Format("150405.000000000"),
+				TaskPolicy{},
+				func(context.Context) error { return nil },
+			)
+		}(index)
+	}
+
+	supervisor.Seal()
+	starters.Wait()
+	if report := supervisor.Wait(context.Background()); !report.Complete {
+		t.Fatal("Wait() did not complete after concurrent Start and Seal")
+	}
+}

machinery/src/log/main.go

@@ -1,179 +0,0 @@
-package log
-
-import (
-	"os"
-	"time"
-
-	"github.com/op/go-logging"
-	"github.com/sirupsen/logrus"
-	"gopkg.in/natefinch/lumberjack.v2"
-)
-
-// The logging library being used everywhere.
-var Log = Logging{
-	Logger: "logrus",
-}
-
-// -----------------
-// This a gologging
-// -> github.com/op/go-logging
-
-var gologging = logging.MustGetLogger("gologger")
-
-func ConfigureGoLogging(configDirectory string, timezone *time.Location) {
-	// Logging
-	var format = logging.MustStringFormatter(
-		`%{color}%{time:15:04:05.000} %{shortfunc} ▶ %{level:.4s} %{id:03x}%{color:reset} %{message}`,
-	)
-	var format2 = logging.MustStringFormatter(
-		`%{time:15:04:05.000} %{shortfunc} ▶ %{level:.4s}  %{id:03x}%{color:reset} %{message}`,
-	)
-	stdBackend := logging.NewLogBackend(os.Stderr, "", 0)
-	stdBackendLeveled := logging.NewBackendFormatter(stdBackend, format)
-	fileBackend := logging.NewLogBackend(&lumberjack.Logger{
-		Filename: configDirectory + "/data/log/machinery.txt",
-		MaxSize:  2,    // megabytes
-		Compress: true, // disabled by default
-	}, "", 0)
-	fileBackendLeveled := logging.NewBackendFormatter(fileBackend, format2)
-	logging.SetBackend(stdBackendLeveled, fileBackendLeveled)
-	logging.SetLevel(logging.DEBUG, "")
-}
-
-// -----------------
-// This a logrus
-// -> github.com/sirupsen/logrus
-
-func ConfigureLogrus(level string, output string, timezone *time.Location) {
-
-	if output == "json" {
-		// Log as JSON instead of the default ASCII formatter.
-		logrus.SetFormatter(LocalTimeZoneFormatter{
-			Timezone:  timezone,
-			Formatter: &logrus.JSONFormatter{},
-		})
-	} else if output == "text" {
-		// Log as text with colors.
-		formatter := logrus.TextFormatter{
-			ForceColors:   true,
-			FullTimestamp: true,
-		}
-		logrus.SetFormatter(LocalTimeZoneFormatter{
-			Timezone:  timezone,
-			Formatter: &formatter,
-		})
-	}
-
-	// Use local timezone for providing datetime in logs!
-
-	// Output to stdout instead of the default stderr
-	// Can be any io.Writer, see below for File example
-	logrus.SetOutput(os.Stdout)
-
-	// Only log the warning severity or above.
-	logLevel := logrus.InfoLevel
-	if level == "error" {
-		logLevel = logrus.ErrorLevel
-	} else if level == "debug" {
-		logLevel = logrus.DebugLevel
-	} else if level == "fatal" {
-		logLevel = logrus.FatalLevel
-	} else if level == "warning" {
-		logLevel = logrus.WarnLevel
-	} // Add this line for logging filename and line number!
-	logrus.SetLevel(logLevel)
-}
-
-type LocalTimeZoneFormatter struct {
-	Timezone  *time.Location
-	Formatter logrus.Formatter
-}
-
-func (u LocalTimeZoneFormatter) Format(e *logrus.Entry) ([]byte, error) {
-	e.Time = e.Time.In(u.Timezone)
-	return u.Formatter.Format(e)
-}
-
-type Logging struct {
-	Logger string
-}
-
-func (self *Logging) Init(level string, logoutput string, configDirectory string, timezone *time.Location) {
-	switch self.Logger {
-	case "go-logging":
-		ConfigureGoLogging(configDirectory, timezone)
-	case "logrus":
-		ConfigureLogrus(level, logoutput, timezone)
-	default:
-	}
-}
-
-func (self *Logging) Info(sentence string) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Info(sentence)
-	case "logrus":
-		logrus.Info(sentence)
-	default:
-	}
-}
-
-func (self *Logging) Infof(format string, args ...interface{}) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Infof(format, args...)
-	case "logrus":
-		logrus.Infof(format, args...)
-	default:
-	}
-}
-
-func (self *Logging) Warning(sentence string) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Warning(sentence)
-	case "logrus":
-		logrus.Warn(sentence)
-	default:
-	}
-}
-
-func (self *Logging) Debug(sentence string) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Debug(sentence)
-	case "logrus":
-		logrus.Debug(sentence)
-	default:
-	}
-}
-
-func (self *Logging) Debugf(format string, args ...interface{}) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Debugf(format, args...)
-	case "logrus":
-		logrus.Debugf(format, args...)
-	default:
-	}
-}
-
-func (self *Logging) Error(sentence string) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Error(sentence)
-	case "logrus":
-		logrus.Error(sentence)
-	default:
-	}
-}
-
-func (self *Logging) Fatal(sentence string) {
-	switch self.Logger {
-	case "go-logging":
-		gologging.Fatal(sentence)
-	case "logrus":
-		logrus.Fatal(sentence)
-	default:
-	}
-}

machinery/src/models/agent_run.go

@@ -0,0 +1,477 @@
+package models
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/kerberos-io/agent/machinery/src/lifecycle"
+	"github.com/kerberos-io/agent/machinery/src/packets"
+	log "github.com/sirupsen/logrus"
+)
+
+var (
+	ErrAgentRunActive  = errors.New("another agent run is active")
+	ErrAgentRunStarted = errors.New("agent run already started")
+	ErrAgentRunStopped = errors.New("agent run is stopping")
+	nextAgentRunID     atomic.Uint64
+)
+
+type AgentRunClient interface {
+	Close(context.Context) error
+}
+
+type AgentRunResourceError struct {
+	Resource string
+	Err      error
+}
+
+func (e AgentRunResourceError) Error() string {
+	return fmt.Sprintf("close %s: %v", e.Resource, e.Err)
+}
+
+func (e AgentRunResourceError) Unwrap() error {
+	return e.Err
+}
+
+type AgentRunShutdownReport struct {
+	lifecycle.ShutdownReport
+	ResourceErrors      []AgentRunResourceError
+	UploadStopDelivered bool
+	StreamStopDelivered bool
+}
+
+type AgentRun struct {
+	id            uint64
+	ctx           context.Context
+	cancel        context.CancelCauseFunc
+	supervisor    *lifecycle.Supervisor
+	communication *Communication
+	stopUpload    bool
+
+	stateMu   sync.Mutex
+	activated bool
+	stopping  bool
+
+	resourcesMu       sync.RWMutex
+	mainClient        AgentRunClient
+	subClient         AgentRunClient
+	backchannelClient AgentRunClient
+	mainQueue         *packets.Queue
+	subQueue          *packets.Queue
+	releaseClients    func()
+
+	channelsMu       sync.RWMutex
+	channelsClosed   bool
+	liveHDHandshakes chan LiveHDHandshake
+	motionEvents     chan MotionDataPartial
+	onvifActions     chan OnvifAction
+
+	shutdownOnce   sync.Once
+	shutdownReport AgentRunShutdownReport
+}
+
+func NewAgentRun(parent context.Context, communication *Communication, stopUpload bool) *AgentRun {
+	if parent == nil {
+		parent = context.Background()
+	}
+	ctx, cancel := context.WithCancelCause(parent)
+	run := &AgentRun{
+		id:               nextAgentRunID.Add(1),
+		ctx:              ctx,
+		cancel:           cancel,
+		supervisor:       lifecycle.NewSupervisor(ctx),
+		communication:    communication,
+		stopUpload:       stopUpload,
+		liveHDHandshakes: make(chan LiveHDHandshake, 100),
+		motionEvents:     make(chan MotionDataPartial, 10),
+		onvifActions:     make(chan OnvifAction, 10),
+	}
+	log.WithFields(log.Fields{
+		"component":   "agent_run",
+		"event":       "run_created",
+		"run_id":      run.id,
+		"stop_upload": stopUpload,
+	}).Debug("Agent run created")
+	return run
+}
+
+func (r *AgentRun) ID() uint64 {
+	return r.id
+}
+
+func (r *AgentRun) Context() context.Context {
+	return r.ctx
+}
+
+func (r *AgentRun) Go(name string, policy lifecycle.TaskPolicy, task lifecycle.TaskFunc) error {
+	if task == nil {
+		return r.supervisor.Go(name, policy, task)
+	}
+
+	fields := log.Fields{
+		"component":    "agent_run",
+		"event":        "task_lifecycle",
+		"long_running": policy.LongRunning,
+		"required":     policy.Required,
+		"run_id":       r.id,
+		"task":         name,
+	}
+	wrappedTask := func(ctx context.Context) error {
+		startedAt := time.Now()
+		log.WithFields(fields).Debug("Agent task started")
+		err := task(ctx)
+		entry := log.WithFields(fields).
+			WithField("duration_ms", time.Since(startedAt).Milliseconds())
+		if err != nil {
+			entry = entry.WithError(err)
+			if ctx.Err() == nil {
+				entry.Warn("Agent task stopped with an error")
+			} else {
+				entry.Debug("Agent task stopped during shutdown")
+			}
+		} else {
+			entry.Debug("Agent task stopped")
+		}
+		return err
+	}
+
+	if err := r.supervisor.Go(name, policy, wrappedTask); err != nil {
+		log.WithError(err).WithFields(fields).Error("Failed to register Agent task")
+		return err
+	}
+	log.WithFields(fields).Debug("Agent task registered")
+	return nil
+}
+
+func (r *AgentRun) Seal() {
+	r.supervisor.Seal()
+	log.WithFields(log.Fields{
+		"component":  "agent_run",
+		"event":      "run_sealed",
+		"run_id":     r.id,
+		"task_count": len(r.supervisor.Snapshot()),
+	}).Debug("Agent run sealed")
+}
+
+func (r *AgentRun) Failures() <-chan lifecycle.Failure {
+	return r.supervisor.Failures()
+}
+
+func (r *AgentRun) Snapshot() []lifecycle.TaskSnapshot {
+	return r.supervisor.Snapshot()
+}
+
+func (r *AgentRun) Wait(ctx context.Context) lifecycle.ShutdownReport {
+	return r.supervisor.Wait(ctx)
+}
+
+func (r *AgentRun) Activate() error {
+	r.stateMu.Lock()
+	defer r.stateMu.Unlock()
+	if r.stopping {
+		return ErrAgentRunStopped
+	}
+	if r.ctx.Err() != nil {
+		return ErrAgentRunStopped
+	}
+	if r.activated {
+		return ErrAgentRunStarted
+	}
+	if r.communication == nil {
+		return errors.New("agent run communication is required")
+	}
+	if err := r.communication.attachRun(r); err != nil {
+		return err
+	}
+	r.activated = true
+	log.WithFields(log.Fields{
+		"component": "agent_run",
+		"event":     "run_activated",
+		"run_id":    r.id,
+	}).Info("Agent run activated")
+	return nil
+}
+
+func (r *AgentRun) SetMainClient(client AgentRunClient) {
+	r.resourcesMu.Lock()
+	r.mainClient = client
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) SetSubClient(client AgentRunClient) {
+	r.resourcesMu.Lock()
+	r.subClient = client
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) SetBackchannelClient(client AgentRunClient) {
+	r.resourcesMu.Lock()
+	r.backchannelClient = client
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) SetQueues(mainQueue, subQueue *packets.Queue) {
+	r.resourcesMu.Lock()
+	r.mainQueue = mainQueue
+	r.subQueue = subQueue
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) SetMainQueue(queue *packets.Queue) {
+	r.resourcesMu.Lock()
+	r.mainQueue = queue
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) SetSubQueue(queue *packets.Queue) {
+	r.resourcesMu.Lock()
+	r.subQueue = queue
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) SetClientRelease(release func()) {
+	r.resourcesMu.Lock()
+	r.releaseClients = release
+	r.resourcesMu.Unlock()
+}
+
+func (r *AgentRun) MainQueue() *packets.Queue {
+	if r.isStopping() {
+		return nil
+	}
+	r.resourcesMu.RLock()
+	defer r.resourcesMu.RUnlock()
+	return r.mainQueue
+}
+
+func (r *AgentRun) SubQueue() *packets.Queue {
+	if r.isStopping() {
+		return nil
+	}
+	r.resourcesMu.RLock()
+	defer r.resourcesMu.RUnlock()
+	return r.subQueue
+}
+
+func (r *AgentRun) LiveHDHandshakes() <-chan LiveHDHandshake {
+	return r.liveHDHandshakes
+}
+
+func (r *AgentRun) MotionEvents() <-chan MotionDataPartial {
+	return r.motionEvents
+}
+
+func (r *AgentRun) ONVIFActions() <-chan OnvifAction {
+	return r.onvifActions
+}
+
+func (r *AgentRun) TrySendLiveHDHandshake(handshake LiveHDHandshake) bool {
+	if r.isStopping() {
+		return false
+	}
+	r.channelsMu.RLock()
+	defer r.channelsMu.RUnlock()
+	if r.channelsClosed {
+		return false
+	}
+	select {
+	case r.liveHDHandshakes <- handshake:
+		return true
+	default:
+		return false
+	}
+}
+
+func (r *AgentRun) PendingLiveHDHandshakes() int {
+	if r.isStopping() {
+		return 0
+	}
+	r.channelsMu.RLock()
+	defer r.channelsMu.RUnlock()
+	if r.channelsClosed {
+		return 0
+	}
+	return len(r.liveHDHandshakes)
+}
+
+func (r *AgentRun) TrySendMotion(motion MotionDataPartial) bool {
+	if r.isStopping() {
+		return false
+	}
+	r.channelsMu.RLock()
+	defer r.channelsMu.RUnlock()
+	if r.channelsClosed {
+		return false
+	}
+	select {
+	case r.motionEvents <- motion:
+		return true
+	default:
+		return false
+	}
+}
+
+func (r *AgentRun) TrySendONVIF(action OnvifAction) bool {
+	if r.isStopping() {
+		return false
+	}
+	r.channelsMu.RLock()
+	defer r.channelsMu.RUnlock()
+	if r.channelsClosed {
+		return false
+	}
+	select {
+	case r.onvifActions <- action:
+		return true
+	default:
+		return false
+	}
+}
+
+func (r *AgentRun) Shutdown(ctx context.Context, cause error) AgentRunShutdownReport {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	if cause == nil {
+		cause = context.Canceled
+	}
+
+	r.shutdownOnce.Do(func() {
+		startedAt := time.Now()
+		log.WithError(cause).WithFields(log.Fields{
+			"component": "agent_run",
+			"event":     "shutdown_started",
+			"run_id":    r.id,
+		}).Info("Agent run shutdown started")
+
+		r.stateMu.Lock()
+		r.stopping = true
+		activated := r.activated
+		r.stateMu.Unlock()
+
+		r.cancel(cause)
+		r.supervisor.BeginShutdown(cause)
+
+		if activated && r.stopUpload {
+			r.shutdownReport.UploadStopDelivered = sendRunStop(ctx, r.communication.HandleUpload)
+		}
+		if activated {
+			r.shutdownReport.StreamStopDelivered = sendRunStop(ctx, r.communication.HandleStream)
+		}
+
+		r.resourcesMu.RLock()
+		mainClient := r.mainClient
+		subClient := r.subClient
+		backchannelClient := r.backchannelClient
+		mainQueue := r.mainQueue
+		subQueue := r.subQueue
+		releaseClients := r.releaseClients
+		r.resourcesMu.RUnlock()
+
+		r.closeClient(ctx, "main RTSP client", mainClient)
+		if mainQueue != nil {
+			_ = mainQueue.Close()
+		}
+		r.closeClient(ctx, "sub RTSP client", subClient)
+		if subQueue != nil {
+			_ = subQueue.Close()
+		}
+		r.closeClient(ctx, "RTSP backchannel client", backchannelClient)
+		r.closeChannels()
+		if releaseClients != nil {
+			releaseClients()
+		}
+
+		r.shutdownReport.ShutdownReport = r.supervisor.Wait(ctx)
+		if r.shutdownReport.Complete && r.communication != nil {
+			r.communication.detachRun(r)
+		}
+
+		runningTasks := make([]string, 0, len(r.shutdownReport.Running))
+		for _, task := range r.shutdownReport.Running {
+			runningTasks = append(runningTasks, task.Name)
+		}
+		entry := log.WithFields(log.Fields{
+			"component":            "agent_run",
+			"duration_ms":          time.Since(startedAt).Milliseconds(),
+			"resource_error_count": len(r.shutdownReport.ResourceErrors),
+			"run_id":               r.id,
+			"running_tasks":        runningTasks,
+			"task_count":           len(r.shutdownReport.Tasks),
+			"upload_stop_sent":     r.shutdownReport.UploadStopDelivered,
+			"stream_stop_sent":     r.shutdownReport.StreamStopDelivered,
+		})
+		if r.shutdownReport.Complete {
+			entry.WithField("event", "shutdown_completed").Info("Agent run shutdown completed")
+		} else {
+			entry.WithField("event", "shutdown_timed_out").Error("Agent run shutdown timed out")
+		}
+	})
+
+	report := r.shutdownReport
+	report.ResourceErrors = append([]AgentRunResourceError(nil), report.ResourceErrors...)
+	return report
+}
+
+func (r *AgentRun) isStopping() bool {
+	r.stateMu.Lock()
+	defer r.stateMu.Unlock()
+	return r.stopping
+}
+
+func (r *AgentRun) closeClient(ctx context.Context, name string, client AgentRunClient) {
+	if client == nil {
+		return
+	}
+	if err := client.Close(ctx); err != nil {
+		r.shutdownReport.ResourceErrors = append(r.shutdownReport.ResourceErrors, AgentRunResourceError{
+			Resource: name,
+			Err:      err,
+		})
+		log.WithError(err).WithFields(log.Fields{
+			"component": "agent_run",
+			"event":     "resource_close_failed",
+			"resource":  name,
+			"run_id":    r.id,
+		}).Warn("Failed to close Agent run resource")
+		return
+	}
+	log.WithFields(log.Fields{
+		"component": "agent_run",
+		"event":     "resource_closed",
+		"resource":  name,
+		"run_id":    r.id,
+	}).Debug("Agent run resource closed")
+}
+
+func (r *AgentRun) closeChannels() {
+	r.channelsMu.Lock()
+	defer r.channelsMu.Unlock()
+	if r.channelsClosed {
+		return
+	}
+	r.channelsClosed = true
+	close(r.liveHDHandshakes)
+	close(r.motionEvents)
+	close(r.onvifActions)
+}
+
+func sendRunStop(ctx context.Context, channel chan<- string) bool {
+	if channel == nil {
+		return false
+	}
+	timer := time.NewTimer(time.Second)
+	defer timer.Stop()
+	select {
+	case channel <- "stop":
+		return true
+	case <-ctx.Done():
+		return false
+	case <-timer.C:
+		return false
+	}
+}

machinery/src/models/agent_run_test.go

@@ -0,0 +1,303 @@
+package models
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/kerberos-io/agent/machinery/src/lifecycle"
+	"github.com/kerberos-io/agent/machinery/src/packets"
+)
+
+type fakeAgentRunClient struct {
+	mu       sync.Mutex
+	name     string
+	order    *[]string
+	calls    int
+	closeErr error
+}
+
+func (c *fakeAgentRunClient) Close(context.Context) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.calls++
+	*c.order = append(*c.order, c.name)
+	return c.closeErr
+}
+
+func (c *fakeAgentRunClient) Calls() int {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.calls
+}
+
+func TestAgentRunOwnsAndShutsDownResources(t *testing.T) {
+	communication := &Communication{
+		HandleStream: make(chan string, 1),
+		HandleUpload: make(chan string, 1),
+	}
+	run := NewAgentRun(context.Background(), communication, true)
+	mainQueue := packets.NewQueue()
+	subQueue := packets.NewQueue()
+	run.SetQueues(mainQueue, subQueue)
+
+	var order []string
+	mainClient := &fakeAgentRunClient{name: "main", order: &order}
+	subClient := &fakeAgentRunClient{name: "sub", order: &order}
+	backchannelClient := &fakeAgentRunClient{name: "backchannel", order: &order}
+	run.SetMainClient(mainClient)
+	run.SetSubClient(subClient)
+	run.SetBackchannelClient(backchannelClient)
+	run.SetClientRelease(func() {
+		order = append(order, "release")
+	})
+
+	taskStarted := make(chan struct{})
+	if err := run.Go("worker", lifecycle.TaskPolicy{}, func(ctx context.Context) error {
+		close(taskStarted)
+		<-ctx.Done()
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := run.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	run.Seal()
+	<-taskStarted
+
+	report := run.Shutdown(context.Background(), errors.New("test shutdown"))
+	if !report.Complete {
+		t.Fatalf("shutdown report = %+v, want complete", report)
+	}
+	if communication.CurrentRun() != nil {
+		t.Fatal("shutdown left the run attached")
+	}
+	if got, want := order, []string{"main", "sub", "backchannel", "release"}; !equalStrings(got, want) {
+		t.Fatalf("resource order = %v, want %v", got, want)
+	}
+	if !report.UploadStopDelivered || !report.StreamStopDelivered {
+		t.Fatalf("stop delivery = upload:%t stream:%t, want both", report.UploadStopDelivered, report.StreamStopDelivered)
+	}
+	select {
+	case value := <-communication.HandleUpload:
+		if value != "stop" {
+			t.Fatalf("upload stop = %q, want stop", value)
+		}
+	default:
+		t.Fatal("upload stop was not delivered")
+	}
+	select {
+	case value := <-communication.HandleStream:
+		if value != "stop" {
+			t.Fatalf("stream stop = %q, want stop", value)
+		}
+	default:
+		t.Fatal("stream stop was not delivered")
+	}
+	if _, err := mainQueue.Latest().ReadPacket(); err == nil {
+		t.Fatal("main queue remained open")
+	}
+	if _, err := subQueue.Latest().ReadPacket(); err == nil {
+		t.Fatal("sub queue remained open")
+	}
+	if _, ok := <-run.LiveHDHandshakes(); ok {
+		t.Fatal("handshake channel remained open")
+	}
+	if _, ok := <-run.MotionEvents(); ok {
+		t.Fatal("motion channel remained open")
+	}
+	if _, ok := <-run.ONVIFActions(); ok {
+		t.Fatal("ONVIF channel remained open")
+	}
+}
+
+func TestAgentRunShutdownIsConcurrentAndIdempotent(t *testing.T) {
+	communication := &Communication{}
+	run := NewAgentRun(context.Background(), communication, false)
+	var order []string
+	client := &fakeAgentRunClient{name: "main", order: &order}
+	run.SetMainClient(client)
+	if err := run.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	run.Seal()
+
+	var callers sync.WaitGroup
+	reports := make(chan AgentRunShutdownReport, 20)
+	for index := 0; index < 20; index++ {
+		callers.Add(1)
+		go func() {
+			defer callers.Done()
+			reports <- run.Shutdown(context.Background(), errors.New("test shutdown"))
+		}()
+	}
+	callers.Wait()
+	close(reports)
+
+	for report := range reports {
+		if !report.Complete {
+			t.Fatalf("shutdown report = %+v, want complete", report)
+		}
+	}
+	if got := client.Calls(); got != 1 {
+		t.Fatalf("client Close() calls = %d, want 1", got)
+	}
+}
+
+func TestAgentRunStaleShutdownDoesNotDetachNewRun(t *testing.T) {
+	communication := &Communication{}
+	oldRun := NewAgentRun(context.Background(), communication, false)
+	if err := oldRun.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	if !communication.detachRun(oldRun) {
+		t.Fatal("failed to detach old run during test setup")
+	}
+
+	newRun := NewAgentRun(context.Background(), communication, false)
+	if err := newRun.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	newRun.Seal()
+	t.Cleanup(func() {
+		newRun.Shutdown(context.Background(), errors.New("test complete"))
+	})
+
+	oldRun.Shutdown(context.Background(), errors.New("stale shutdown"))
+	if got := communication.CurrentRun(); got != newRun {
+		t.Fatalf("current run = %p, want new run %p", got, newRun)
+	}
+}
+
+func TestAgentRunRejectsOverlap(t *testing.T) {
+	communication := &Communication{}
+	first := NewAgentRun(context.Background(), communication, false)
+	if err := first.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	first.Seal()
+	t.Cleanup(func() {
+		first.Shutdown(context.Background(), errors.New("test complete"))
+	})
+
+	second := NewAgentRun(context.Background(), communication, false)
+	if err := second.Activate(); !errors.Is(err, ErrAgentRunActive) {
+		t.Fatalf("Activate() error = %v, want ErrAgentRunActive", err)
+	}
+	second.Shutdown(context.Background(), errors.New("test complete"))
+}
+
+func TestAgentRunRetainsOwnershipUntilWorkersStop(t *testing.T) {
+	communication := &Communication{}
+	run := NewAgentRun(context.Background(), communication, false)
+	release := make(chan struct{})
+	started := make(chan struct{})
+	if err := run.Go("blocked", lifecycle.TaskPolicy{}, func(context.Context) error {
+		close(started)
+		<-release
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := run.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	run.Seal()
+	<-started
+
+	shutdownDone := make(chan AgentRunShutdownReport, 1)
+	go func() {
+		shutdownDone <- run.Shutdown(context.Background(), errors.New("test shutdown"))
+	}()
+
+	deadline := time.Now().Add(time.Second)
+	for run.Context().Err() == nil && time.Now().Before(deadline) {
+		time.Sleep(time.Millisecond)
+	}
+	if run.Context().Err() == nil {
+		t.Fatal("run context was not canceled")
+	}
+	if communication.CurrentRun() != run {
+		t.Fatal("run ownership was released before its worker stopped")
+	}
+	if communication.TrySendMotion(MotionDataPartial{}) {
+		t.Fatal("stopping run accepted new input")
+	}
+	if communication.MainQueue() != nil {
+		t.Fatal("stopping run exposed its queue")
+	}
+
+	replacement := NewAgentRun(context.Background(), communication, false)
+	if err := replacement.Activate(); !errors.Is(err, ErrAgentRunActive) {
+		t.Fatalf("replacement Activate() error = %v, want ErrAgentRunActive", err)
+	}
+	replacement.Shutdown(context.Background(), errors.New("test complete"))
+
+	close(release)
+	report := <-shutdownDone
+	if !report.Complete {
+		t.Fatalf("shutdown report = %+v, want complete", report)
+	}
+	if communication.CurrentRun() != nil {
+		t.Fatal("completed shutdown left the run attached")
+	}
+}
+
+func TestAgentRunShutdownReportsStuckTask(t *testing.T) {
+	communication := &Communication{}
+	run := NewAgentRun(context.Background(), communication, false)
+	release := make(chan struct{})
+	started := make(chan struct{})
+	if err := run.Go("blocked", lifecycle.TaskPolicy{}, func(context.Context) error {
+		close(started)
+		<-release
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := run.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	run.Seal()
+	<-started
+
+	waitContext, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+	defer cancel()
+	report := run.Shutdown(waitContext, errors.New("test shutdown"))
+	if report.Complete {
+		t.Fatal("shutdown unexpectedly completed")
+	}
+	if len(report.Running) != 1 || report.Running[0].Name != "blocked" {
+		t.Fatalf("running tasks = %+v, want blocked", report.Running)
+	}
+	if communication.CurrentRun() != run {
+		t.Fatal("timed-out shutdown released run ownership")
+	}
+	replacement := NewAgentRun(context.Background(), communication, false)
+	if err := replacement.Activate(); !errors.Is(err, ErrAgentRunActive) {
+		t.Fatalf("replacement Activate() error = %v, want ErrAgentRunActive", err)
+	}
+	replacement.Shutdown(context.Background(), errors.New("test complete"))
+	close(release)
+	waitContext, waitCancel := context.WithTimeout(context.Background(), time.Second)
+	defer waitCancel()
+	if followup := run.Wait(waitContext); !followup.Complete {
+		t.Fatalf("follow-up report = %+v, want complete", followup)
+	}
+	communication.detachRun(run)
+}
+
+func equalStrings(left, right []string) bool {
+	if len(left) != len(right) {
+		return false
+	}
+	for index := range left {
+		if left[index] != right[index] {
+			return false
+		}
+	}
+	return true
+}

machinery/src/models/communication.go

@@ -1,8 +1,10 @@
 package models
 
 import (
-	"context"
+	"math"
+	"sync"
 	"sync/atomic"
+	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/tevino/abool"
@@ -19,11 +21,84 @@ type LiveHDHandshake struct {
 	Signaling *LiveHDSignalingCallbacks
 }
 
+type MoQRecoveryTelemetry struct {
+	Reconnects          uint64 `json:"reconnects"`
+	LastFrameUnixMillis int64  `json:"lastFrameUnixMillis"`
+	LastWriteMillis     int64  `json:"lastWriteMillis"`
+}
+
+type RecoveryTelemetry struct {
+	MoQHigh                   MoQRecoveryTelemetry `json:"moqHigh"`
+	MoQLow                    MoQRecoveryTelemetry `json:"moqLow"`
+	MoQWriteTimeouts          uint64               `json:"moqWriteTimeouts"`
+	DroppedLiveHDHandshakes   uint64               `json:"droppedLiveHDHandshakes"`
+	DroppedMotionEvents       uint64               `json:"droppedMotionEvents"`
+	DroppedONVIFActions       uint64               `json:"droppedOnvifActions"`
+	WatchdogRestarts          uint64               `json:"watchdogRestarts"`
+	WatchdogCooldownSeconds   int64                `json:"watchdogCooldownSeconds"`
+	RunWorkerShutdownTimeouts uint64               `json:"runWorkerShutdownTimeouts"`
+}
+
+type StreamKind uint8
+
+const (
+	MainStream StreamKind = iota
+	SubStream
+)
+
+type StreamRuntimeTelemetry struct {
+	Configured        bool
+	PackagesProcessed uint64
+	FPS               float64
+	Width             int64
+	Height            int64
+	LastPacketAt      int64
+}
+
+type streamRuntimeTelemetry struct {
+	configured        atomic.Bool
+	packagesProcessed atomic.Uint64
+	fpsBits           atomic.Uint64
+	width             atomic.Int64
+	height            atomic.Int64
+	lastPacketAt      atomic.Int64
+}
+
+type HubRuntimeTelemetry struct {
+	Configured                bool
+	Connected                 bool
+	LastHeartbeatAttemptAt    int64
+	LastSuccessfulHeartbeatAt int64
+}
+
+type hubRuntimeTelemetry struct {
+	configured                atomic.Bool
+	connected                 atomic.Bool
+	lastHeartbeatAttemptAt    atomic.Int64
+	lastSuccessfulHeartbeatAt atomic.Int64
+}
+
+type recoveryTelemetry struct {
+	moqHighReconnects          atomic.Uint64
+	moqHighLastFrameUnixMillis atomic.Int64
+	moqHighLastWriteMillis     atomic.Int64
+	moqLowReconnects           atomic.Uint64
+	moqLowLastFrameUnixMillis  atomic.Int64
+	moqLowLastWriteMillis      atomic.Int64
+	moqWriteTimeouts           atomic.Uint64
+	droppedLiveHDHandshakes    atomic.Uint64
+	droppedMotionEvents        atomic.Uint64
+	droppedONVIFActions        atomic.Uint64
+	watchdogRestarts           atomic.Uint64
+	watchdogCooldownSeconds    atomic.Int64
+	runWorkerShutdownTimeouts  atomic.Uint64
+}
+
 // The communication struct that is managing
 // all the communication between the different goroutines.
 type Communication struct {
-	Context               *context.Context
-	CancelContext         *context.CancelFunc
+	currentRunMu          sync.RWMutex
+	currentRun            *AgentRun
 	PackageCounter        *atomic.Value
 	LastPacketTimer       *atomic.Value
 	PackageCounterSub     *atomic.Value
@@ -32,20 +107,17 @@ type Communication struct {
 	HandleBootstrap       chan string
 	HandleStream          chan string
 	HandleSubStream       chan string
-	HandleMotion          chan MotionDataPartial
 	HandleAudio           chan AudioDataPartial
 	HandleUpload          chan string
 	HandleHeartBeat       chan string
 	HandleLiveSD          chan int64
 	HandleLiveSDHTTP      chan int64
 	HandleLiveHDKeepalive chan string
-	HandleLiveHDHandshake chan LiveHDHandshake
 	HandleLiveHDPeers     chan string
 	// HandleLiveHLS is the live HLS viewer keepalive. It carries the requested
 	// quality tier ("auto"|"high"|"low"; empty => auto) so the producer can switch
 	// the live session between the main and sub stream on demand.
 	HandleLiveHLS chan string
-	HandleONVIF   chan OnvifAction
 	IsConfiguring *abool.AtomicBool
 	// IsRecordingManual is set while a viewer has requested a manual recording
 	// from the live view (the record button). While set, the motion-based
@@ -71,11 +143,224 @@ type Communication struct {
 	// viewer that starts a recording but never heartbeats (an older frontend)
 	// still records up to the max-duration cap instead of being cut off early.
 	RecordingManualHeartbeatSeen *abool.AtomicBool
-	Queue                        *packets.Queue
-	SubQueue                     *packets.Queue
 	Image                        string
-	CameraConnected              bool
-	MainStreamConnected          bool
-	SubStreamConnected           bool
-	HasBackChannel               bool
+	CameraConnected              atomic.Bool
+	MainStreamConnected          atomic.Bool
+	SubStreamConnected           atomic.Bool
+	HasBackChannel               atomic.Bool
+	mainStreamTelemetry          streamRuntimeTelemetry
+	subStreamTelemetry           streamRuntimeTelemetry
+	hubTelemetry                 hubRuntimeTelemetry
+	recovery                     recoveryTelemetry
+}
+
+func (c *Communication) streamTelemetry(stream StreamKind) *streamRuntimeTelemetry {
+	switch stream {
+	case MainStream:
+		return &c.mainStreamTelemetry
+	case SubStream:
+		return &c.subStreamTelemetry
+	default:
+		panic("unsupported stream kind")
+	}
+}
+
+func (c *Communication) SetStreamConfigured(stream StreamKind, configured bool) {
+	c.streamTelemetry(stream).configured.Store(configured)
+}
+
+func (c *Communication) RecordStreamPackage(stream StreamKind, fps float64, width, height int, at time.Time) {
+	telemetry := c.streamTelemetry(stream)
+	telemetry.packagesProcessed.Add(1)
+	if fps > 0 && !math.IsNaN(fps) && !math.IsInf(fps, 0) {
+		telemetry.fpsBits.Store(math.Float64bits(fps))
+	}
+	if width > 0 {
+		telemetry.width.Store(int64(width))
+	}
+	if height > 0 {
+		telemetry.height.Store(int64(height))
+	}
+	if !at.IsZero() {
+		telemetry.lastPacketAt.Store(at.Unix())
+	}
+}
+
+func (c *Communication) StreamRuntimeTelemetry(stream StreamKind) StreamRuntimeTelemetry {
+	telemetry := c.streamTelemetry(stream)
+	return StreamRuntimeTelemetry{
+		Configured:        telemetry.configured.Load(),
+		PackagesProcessed: telemetry.packagesProcessed.Load(),
+		FPS:               math.Float64frombits(telemetry.fpsBits.Load()),
+		Width:             telemetry.width.Load(),
+		Height:            telemetry.height.Load(),
+		LastPacketAt:      telemetry.lastPacketAt.Load(),
+	}
+}
+
+func (c *Communication) SetHubConfigured(configured bool) {
+	c.hubTelemetry.configured.Store(configured)
+	if !configured {
+		c.hubTelemetry.connected.Store(false)
+	}
+}
+
+func (c *Communication) RecordHubHeartbeatAttempt(at time.Time) {
+	if !at.IsZero() {
+		c.hubTelemetry.lastHeartbeatAttemptAt.Store(at.Unix())
+	}
+}
+
+func (c *Communication) RecordHubHeartbeatSuccess(at time.Time) {
+	if !at.IsZero() {
+		c.hubTelemetry.lastSuccessfulHeartbeatAt.Store(at.Unix())
+	}
+	c.hubTelemetry.connected.Store(true)
+}
+
+func (c *Communication) RecordHubHeartbeatFailure() {
+	c.hubTelemetry.connected.Store(false)
+}
+
+func (c *Communication) HubRuntimeTelemetry() HubRuntimeTelemetry {
+	return HubRuntimeTelemetry{
+		Configured:                c.hubTelemetry.configured.Load(),
+		Connected:                 c.hubTelemetry.connected.Load(),
+		LastHeartbeatAttemptAt:    c.hubTelemetry.lastHeartbeatAttemptAt.Load(),
+		LastSuccessfulHeartbeatAt: c.hubTelemetry.lastSuccessfulHeartbeatAt.Load(),
+	}
+}
+
+func (c *Communication) RecordMoQReconnect(quality string) {
+	if quality == StreamQualityLow {
+		c.recovery.moqLowReconnects.Add(1)
+		return
+	}
+	c.recovery.moqHighReconnects.Add(1)
+}
+
+func (c *Communication) RecordMoQWrite(quality string, duration time.Duration, at time.Time) {
+	if quality == StreamQualityLow {
+		c.recovery.moqLowLastWriteMillis.Store(duration.Milliseconds())
+		c.recovery.moqLowLastFrameUnixMillis.Store(at.UnixMilli())
+		return
+	}
+	c.recovery.moqHighLastWriteMillis.Store(duration.Milliseconds())
+	c.recovery.moqHighLastFrameUnixMillis.Store(at.UnixMilli())
+}
+
+func (c *Communication) RecordMoQWriteTimeout() {
+	c.recovery.moqWriteTimeouts.Add(1)
+}
+
+func (c *Communication) RecordWatchdogRestart(cooldown time.Duration) {
+	c.recovery.watchdogRestarts.Add(1)
+	c.SetWatchdogCooldown(cooldown)
+}
+
+func (c *Communication) SetWatchdogCooldown(cooldown time.Duration) {
+	c.recovery.watchdogCooldownSeconds.Store(int64(cooldown / time.Second))
+}
+
+func (c *Communication) RecordRunWorkerShutdownTimeout() {
+	c.recovery.runWorkerShutdownTimeouts.Add(1)
+}
+
+func (c *Communication) RecoveryTelemetry() RecoveryTelemetry {
+	return RecoveryTelemetry{
+		MoQHigh: MoQRecoveryTelemetry{
+			Reconnects:          c.recovery.moqHighReconnects.Load(),
+			LastFrameUnixMillis: c.recovery.moqHighLastFrameUnixMillis.Load(),
+			LastWriteMillis:     c.recovery.moqHighLastWriteMillis.Load(),
+		},
+		MoQLow: MoQRecoveryTelemetry{
+			Reconnects:          c.recovery.moqLowReconnects.Load(),
+			LastFrameUnixMillis: c.recovery.moqLowLastFrameUnixMillis.Load(),
+			LastWriteMillis:     c.recovery.moqLowLastWriteMillis.Load(),
+		},
+		MoQWriteTimeouts:          c.recovery.moqWriteTimeouts.Load(),
+		DroppedLiveHDHandshakes:   c.recovery.droppedLiveHDHandshakes.Load(),
+		DroppedMotionEvents:       c.recovery.droppedMotionEvents.Load(),
+		DroppedONVIFActions:       c.recovery.droppedONVIFActions.Load(),
+		WatchdogRestarts:          c.recovery.watchdogRestarts.Load(),
+		WatchdogCooldownSeconds:   c.recovery.watchdogCooldownSeconds.Load(),
+		RunWorkerShutdownTimeouts: c.recovery.runWorkerShutdownTimeouts.Load(),
+	}
+}
+
+func (c *Communication) attachRun(run *AgentRun) error {
+	c.currentRunMu.Lock()
+	defer c.currentRunMu.Unlock()
+	if c.currentRun != nil {
+		return ErrAgentRunActive
+	}
+	c.currentRun = run
+	return nil
+}
+
+func (c *Communication) detachRun(run *AgentRun) bool {
+	c.currentRunMu.Lock()
+	defer c.currentRunMu.Unlock()
+	if c.currentRun != run {
+		return false
+	}
+	c.currentRun = nil
+	return true
+}
+
+func (c *Communication) CurrentRun() *AgentRun {
+	c.currentRunMu.RLock()
+	defer c.currentRunMu.RUnlock()
+	return c.currentRun
+}
+
+func (c *Communication) MainQueue() *packets.Queue {
+	run := c.CurrentRun()
+	if run == nil {
+		return nil
+	}
+	return run.MainQueue()
+}
+
+func (c *Communication) SubQueue() *packets.Queue {
+	run := c.CurrentRun()
+	if run == nil {
+		return nil
+	}
+	return run.SubQueue()
+}
+
+func (c *Communication) TrySendLiveHDHandshake(handshake LiveHDHandshake) bool {
+	run := c.CurrentRun()
+	if run == nil || !run.TrySendLiveHDHandshake(handshake) {
+		c.recovery.droppedLiveHDHandshakes.Add(1)
+		return false
+	}
+	return true
+}
+
+func (c *Communication) PendingLiveHDHandshakes() int {
+	run := c.CurrentRun()
+	if run == nil {
+		return 0
+	}
+	return run.PendingLiveHDHandshakes()
+}
+
+func (c *Communication) TrySendMotion(motion MotionDataPartial) bool {
+	run := c.CurrentRun()
+	if run == nil || !run.TrySendMotion(motion) {
+		c.recovery.droppedMotionEvents.Add(1)
+		return false
+	}
+	return true
+}
+
+func (c *Communication) TrySendONVIF(action OnvifAction) bool {
+	run := c.CurrentRun()
+	if run == nil || !run.TrySendONVIF(action) {
+		c.recovery.droppedONVIFActions.Add(1)
+		return false
+	}
+	return true
 }

machinery/src/models/communication_test.go

@@ -0,0 +1,221 @@
+package models
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+	"time"
+)
+
+func TestCommunicationRunChannelDispatchAfterClose(t *testing.T) {
+	communication := &Communication{}
+	run := NewAgentRun(context.Background(), communication, false)
+	if err := run.Activate(); err != nil {
+		t.Fatal(err)
+	}
+
+	if !communication.TrySendLiveHDHandshake(LiveHDHandshake{}) {
+		t.Fatal("TrySendLiveHDHandshake() rejected an available channel")
+	}
+	if !communication.TrySendMotion(MotionDataPartial{}) {
+		t.Fatal("TrySendMotion() rejected an available channel")
+	}
+	if !communication.TrySendONVIF(OnvifAction{}) {
+		t.Fatal("TrySendONVIF() rejected an available channel")
+	}
+
+	report := run.Shutdown(context.Background(), errors.New("test complete"))
+	if !report.Complete {
+		t.Fatalf("shutdown report = %+v, want complete", report)
+	}
+	if communication.TrySendLiveHDHandshake(LiveHDHandshake{}) {
+		t.Fatal("TrySendLiveHDHandshake() accepted a closed run")
+	}
+	if communication.TrySendMotion(MotionDataPartial{}) {
+		t.Fatal("TrySendMotion() accepted a closed run")
+	}
+	if communication.TrySendONVIF(OnvifAction{}) {
+		t.Fatal("TrySendONVIF() accepted a closed run")
+	}
+}
+
+func TestCommunicationDispatchCanRaceRunChannelClose(t *testing.T) {
+	communication := &Communication{}
+	run := NewAgentRun(context.Background(), communication, false)
+	if err := run.Activate(); err != nil {
+		t.Fatal(err)
+	}
+
+	start := make(chan struct{})
+	var senders sync.WaitGroup
+	for sender := 0; sender < 20; sender++ {
+		senders.Add(1)
+		go func() {
+			defer senders.Done()
+			<-start
+			for attempt := 0; attempt < 100; attempt++ {
+				communication.TrySendLiveHDHandshake(LiveHDHandshake{})
+				communication.TrySendMotion(MotionDataPartial{})
+				communication.TrySendONVIF(OnvifAction{})
+			}
+		}()
+	}
+
+	close(start)
+	run.Shutdown(context.Background(), errors.New("test complete"))
+	senders.Wait()
+}
+
+func TestCommunicationRunChannelLifecycleSoak(t *testing.T) {
+	communication := &Communication{}
+	for cycle := 0; cycle < 100; cycle++ {
+		run := NewAgentRun(context.Background(), communication, false)
+		if err := run.Activate(); err != nil {
+			t.Fatalf("cycle %d Activate() error = %v", cycle, err)
+		}
+		handshakes := run.LiveHDHandshakes()
+		motion := run.MotionEvents()
+		onvif := run.ONVIFActions()
+
+		var consumers sync.WaitGroup
+		consumers.Add(3)
+		go func() {
+			defer consumers.Done()
+			for range handshakes {
+			}
+		}()
+		go func() {
+			defer consumers.Done()
+			for range motion {
+			}
+		}()
+		go func() {
+			defer consumers.Done()
+			for range onvif {
+			}
+		}()
+
+		var producers sync.WaitGroup
+		for producer := 0; producer < 4; producer++ {
+			producers.Add(1)
+			go func() {
+				defer producers.Done()
+				for attempt := 0; attempt < 50; attempt++ {
+					communication.TrySendLiveHDHandshake(LiveHDHandshake{})
+					communication.TrySendMotion(MotionDataPartial{})
+					communication.TrySendONVIF(OnvifAction{})
+				}
+			}()
+		}
+
+		run.Shutdown(context.Background(), errors.New("test complete"))
+		producers.Wait()
+
+		done := make(chan struct{})
+		go func() {
+			consumers.Wait()
+			close(done)
+		}()
+		select {
+		case <-done:
+		case <-time.After(time.Second):
+			t.Fatalf("cycle %d consumers did not stop after channel close", cycle)
+		}
+	}
+}
+
+func TestCommunicationRecoveryTelemetry(t *testing.T) {
+	communication := &Communication{}
+	communication.RecordMoQReconnect(StreamQualityHigh)
+	communication.RecordMoQReconnect(StreamQualityLow)
+	communication.RecordMoQWrite(StreamQualityHigh, 1250*time.Millisecond, time.UnixMilli(1234))
+	communication.RecordMoQWrite(StreamQualityLow, 2500*time.Millisecond, time.UnixMilli(5678))
+	communication.RecordMoQWriteTimeout()
+	communication.RecordWatchdogRestart(30 * time.Second)
+	communication.RecordRunWorkerShutdownTimeout()
+
+	if communication.TrySendLiveHDHandshake(LiveHDHandshake{}) {
+		t.Fatal("TrySendLiveHDHandshake() accepted an unavailable channel")
+	}
+	if communication.TrySendMotion(MotionDataPartial{}) {
+		t.Fatal("TrySendMotion() accepted an unavailable channel")
+	}
+	if communication.TrySendONVIF(OnvifAction{}) {
+		t.Fatal("TrySendONVIF() accepted an unavailable channel")
+	}
+
+	got := communication.RecoveryTelemetry()
+	if got.MoQHigh != (MoQRecoveryTelemetry{Reconnects: 1, LastFrameUnixMillis: 1234, LastWriteMillis: 1250}) {
+		t.Fatalf("high MoQ telemetry = %+v", got.MoQHigh)
+	}
+	if got.MoQLow != (MoQRecoveryTelemetry{Reconnects: 1, LastFrameUnixMillis: 5678, LastWriteMillis: 2500}) {
+		t.Fatalf("low MoQ telemetry = %+v", got.MoQLow)
+	}
+	if got.MoQWriteTimeouts != 1 || got.WatchdogRestarts != 1 || got.WatchdogCooldownSeconds != 30 || got.RunWorkerShutdownTimeouts != 1 {
+		t.Fatalf("recovery telemetry = %+v", got)
+	}
+	if got.DroppedLiveHDHandshakes != 1 || got.DroppedMotionEvents != 1 || got.DroppedONVIFActions != 1 {
+		t.Fatalf("drop telemetry = %+v", got)
+	}
+}
+
+func TestCommunicationOperationalTelemetry(t *testing.T) {
+	communication := &Communication{}
+	mainPacketAt := time.Unix(1_788_710_401, 0)
+	subPacketAt := time.Unix(1_788_710_402, 0)
+	hubAttemptAt := time.Unix(1_788_710_403, 0)
+	hubSuccessAt := time.Unix(1_788_710_404, 0)
+
+	communication.SetStreamConfigured(MainStream, true)
+	communication.SetStreamConfigured(SubStream, true)
+	communication.RecordStreamPackage(MainStream, 29.97, 1920, 1080, mainPacketAt)
+	communication.RecordStreamPackage(MainStream, 29.97, 1920, 1080, mainPacketAt)
+	communication.RecordStreamPackage(SubStream, 15, 640, 360, subPacketAt)
+	communication.SetHubConfigured(true)
+	communication.RecordHubHeartbeatAttempt(hubAttemptAt)
+	communication.RecordHubHeartbeatSuccess(hubSuccessAt)
+
+	main := communication.StreamRuntimeTelemetry(MainStream)
+	if main != (StreamRuntimeTelemetry{
+		Configured:        true,
+		PackagesProcessed: 2,
+		FPS:               29.97,
+		Width:             1920,
+		Height:            1080,
+		LastPacketAt:      mainPacketAt.Unix(),
+	}) {
+		t.Fatalf("main stream telemetry = %+v", main)
+	}
+
+	sub := communication.StreamRuntimeTelemetry(SubStream)
+	if sub != (StreamRuntimeTelemetry{
+		Configured:        true,
+		PackagesProcessed: 1,
+		FPS:               15,
+		Width:             640,
+		Height:            360,
+		LastPacketAt:      subPacketAt.Unix(),
+	}) {
+		t.Fatalf("sub stream telemetry = %+v", sub)
+	}
+
+	hub := communication.HubRuntimeTelemetry()
+	if hub != (HubRuntimeTelemetry{
+		Configured:                true,
+		Connected:                 true,
+		LastHeartbeatAttemptAt:    hubAttemptAt.Unix(),
+		LastSuccessfulHeartbeatAt: hubSuccessAt.Unix(),
+	}) {
+		t.Fatalf("Hub telemetry = %+v", hub)
+	}
+
+	communication.RecordHubHeartbeatFailure()
+	if communication.HubRuntimeTelemetry().Connected {
+		t.Fatal("Hub telemetry remained connected after a failed heartbeat")
+	}
+	communication.SetHubConfigured(false)
+	if communication.HubRuntimeTelemetry().Configured {
+		t.Fatal("Hub telemetry remained configured after it was disabled")
+	}
+}

machinery/src/models/mqtt.go

@@ -6,20 +6,96 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"encoding/pem"
-	"io"
-	"strings"
+	"errors"
+	"sync"
 	"time"
 
 	"github.com/gofrs/uuid"
 	"github.com/kerberos-io/agent/machinery/src/encryption"
-	"github.com/kerberos-io/agent/machinery/src/log"
+	log "github.com/sirupsen/logrus"
 )
 
+var (
+	errMQTTPrivateKeyPEMDecode = errors.New("error decoding PEM block containing private key")
+	errMQTTPrivateKeyNotRSA    = errors.New("private key is not RSA")
+
+	rsaPrivateKeyCache   = newRSAPrivateKeyCache()
+	parsePKCS8PrivateKey = x509.ParsePKCS8PrivateKey
+)
+
+type rsaPrivateKeyCacheState struct {
+	mu      sync.Mutex
+	cond    *sync.Cond
+	pem     string
+	key     *rsa.PrivateKey
+	err     error
+	parsing bool
+	ready   bool
+}
+
+func newRSAPrivateKeyCache() *rsaPrivateKeyCacheState {
+	cache := &rsaPrivateKeyCacheState{}
+	cache.cond = sync.NewCond(&cache.mu)
+	return cache
+}
+
+func (cache *rsaPrivateKeyCacheState) get(privateKey string) (*rsa.PrivateKey, error) {
+	cache.mu.Lock()
+	for {
+		if cache.ready && cache.pem == privateKey && !cache.parsing {
+			key, err := cache.key, cache.err
+			cache.mu.Unlock()
+			return key, err
+		}
+		if cache.parsing {
+			cache.cond.Wait()
+			continue
+		}
+
+		cache.pem = privateKey
+		cache.key = nil
+		cache.err = nil
+		cache.parsing = true
+		cache.ready = false
+		cache.mu.Unlock()
+
+		key, err := parseRSAPrivateKey(privateKey)
+
+		cache.mu.Lock()
+		cache.key = key
+		cache.err = err
+		cache.parsing = false
+		cache.ready = true
+		cache.cond.Broadcast()
+		cache.mu.Unlock()
+		return key, err
+	}
+}
+
+func parseRSAPrivateKey(privateKey string) (*rsa.PrivateKey, error) {
+	block, _ := pem.Decode([]byte(privateKey))
+	if block == nil {
+		return nil, errMQTTPrivateKeyPEMDecode
+	}
+
+	key, err := parsePKCS8PrivateKey(block.Bytes)
+	if err != nil {
+		return nil, err
+	}
+
+	rsaKey, ok := key.(*rsa.PrivateKey)
+	if !ok {
+		return nil, errMQTTPrivateKeyNotRSA
+	}
+
+	return rsaKey, nil
+}
+
 func PackageMQTTMessage(configuration *Configuration, msg Message) ([]byte, error) {
 	// Create a Version 4 UUID.
 	u2, err := uuid.NewV4()
 	if err != nil {
-		log.Log.Error("failed to generate UUID: " + err.Error())
+		log.Error("failed to generate UUID: " + err.Error())
 	}
 
 	// We'll generate an unique id, and encrypt / decrypt it using the private key if available.
@@ -45,27 +121,25 @@ func PackageMQTTMessage(configuration *Configuration, msg Message) ([]byte, erro
 		// Pload to base64
 		data, err := json.Marshal(pload)
 		if err != nil {
-			log.Log.Error("models.mqtt.PackageMQTTMessage(): failed to marshal payload: " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component": "models/mqtt",
+				"event":     "payload_marshal_failed",
+			}).Error("Failed to marshal MQTT payload")
 		}
 
 		// Encrypt the value
 		privateKey := configuration.Config.Encryption.PrivateKey
-		r := strings.NewReader(privateKey)
-		pemBytes, _ := io.ReadAll(r)
-		block, _ := pem.Decode(pemBytes)
-		if block == nil {
-			log.Log.Error("models.mqtt.PackageMQTTMessage(): error decoding PEM block containing private key")
-		} else {
-			// Parse private key
-			b := block.Bytes
-			key, err := x509.ParsePKCS8PrivateKey(b)
-			if err != nil {
-				log.Log.Error("models.mqtt.PackageMQTTMessage(): error parsing private key: " + err.Error())
+		rsaKey, err := rsaPrivateKeyCache.get(privateKey)
+		if err != nil {
+			if errors.Is(err, errMQTTPrivateKeyPEMDecode) {
+				log.Error("models.mqtt.PackageMQTTMessage(): error decoding PEM block containing private key")
+			} else {
+				log.WithError(err).WithFields(log.Fields{
+					"component": "models/mqtt",
+					"event":     "private_key_parse_failed",
+				}).Error("Failed to parse MQTT private key")
 			}
-
-			// Conver key to *rsa.PrivateKey
-			rsaKey, _ := key.(*rsa.PrivateKey)
-
+		} else {
 			// Create a 16bit key random
 			if config.Encryption != nil && config.Encryption.SymmetricKey != "" {
 				k := config.Encryption.SymmetricKey

machinery/src/models/mqtt_test.go

@@ -0,0 +1,199 @@
+package models
+
+import (
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/json"
+	"encoding/pem"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestPackageMQTTMessageCachesCurrentSigningKey(t *testing.T) {
+	resetMQTTPrivateKeyCache(t)
+
+	privateKey := generatePKCS8RSAPrivateKeyPEM(t)
+
+	var parseCalls atomic.Int32
+	restore := hookPKCS8PrivateKeyParser(t, func(der []byte) (any, error) {
+		parseCalls.Add(1)
+		return x509.ParsePKCS8PrivateKey(der)
+	})
+	defer restore()
+
+	msg := Message{
+		Payload: Payload{
+			DeviceId: "device-1",
+			Value: map[string]interface{}{
+				"hello": "world",
+			},
+		},
+	}
+
+	first, err := PackageMQTTMessage(encryptedConfiguration(privateKey), msg)
+	if err != nil {
+		t.Fatalf("first PackageMQTTMessage() error = %v", err)
+	}
+	assertEncryptedMQTTMessage(t, first)
+
+	second, err := PackageMQTTMessage(encryptedConfiguration(privateKey), msg)
+	if err != nil {
+		t.Fatalf("second PackageMQTTMessage() error = %v", err)
+	}
+	assertEncryptedMQTTMessage(t, second)
+
+	if got := parseCalls.Load(); got != 1 {
+		t.Fatalf("cached parse calls = %d, want 1", got)
+	}
+}
+
+func TestPackageMQTTMessageReplacesCachedSigningKeyOnRotation(t *testing.T) {
+	resetMQTTPrivateKeyCache(t)
+
+	privateKey := generatePKCS8RSAPrivateKeyPEM(t)
+	rotatedPrivateKey := generatePKCS8RSAPrivateKeyPEM(t)
+
+	var parseCalls atomic.Int32
+	restore := hookPKCS8PrivateKeyParser(t, func(der []byte) (any, error) {
+		parseCalls.Add(1)
+		return x509.ParsePKCS8PrivateKey(der)
+	})
+	defer restore()
+
+	msg := Message{
+		Payload: Payload{
+			DeviceId: "device-1",
+			Value: map[string]interface{}{
+				"hello": "world",
+			},
+		},
+	}
+
+	for _, key := range []string{privateKey, rotatedPrivateKey, privateKey} {
+		payload, err := PackageMQTTMessage(encryptedConfiguration(key), msg)
+		if err != nil {
+			t.Fatalf("PackageMQTTMessage() error = %v", err)
+		}
+		assertEncryptedMQTTMessage(t, payload)
+	}
+
+	if got := parseCalls.Load(); got != 3 {
+		t.Fatalf("parse calls across old/new/old rotation = %d, want 3", got)
+	}
+}
+
+func TestPackageMQTTMessageConcurrentCallsReuseCachedSigningKey(t *testing.T) {
+	resetMQTTPrivateKeyCache(t)
+
+	privateKey := generatePKCS8RSAPrivateKeyPEM(t)
+
+	var parseCalls atomic.Int32
+	restore := hookPKCS8PrivateKeyParser(t, func(der []byte) (any, error) {
+		parseCalls.Add(1)
+		time.Sleep(10 * time.Millisecond)
+		return x509.ParsePKCS8PrivateKey(der)
+	})
+	defer restore()
+
+	msg := Message{
+		Payload: Payload{
+			DeviceId: "device-1",
+			Value: map[string]interface{}{
+				"hello": "world",
+			},
+		},
+	}
+
+	start := make(chan struct{})
+	var workers sync.WaitGroup
+	for i := 0; i < 16; i++ {
+		workers.Add(1)
+		go func() {
+			defer workers.Done()
+			<-start
+
+			payload, err := PackageMQTTMessage(encryptedConfiguration(privateKey), msg)
+			if err != nil {
+				t.Errorf("PackageMQTTMessage() error = %v", err)
+				return
+			}
+			assertEncryptedMQTTMessage(t, payload)
+		}()
+	}
+
+	close(start)
+	workers.Wait()
+
+	if got := parseCalls.Load(); got != 1 {
+		t.Fatalf("concurrent parse calls = %d, want 1", got)
+	}
+}
+
+func encryptedConfiguration(privateKey string) *Configuration {
+	return &Configuration{
+		Config: Config{
+			Encryption: &Encryption{
+				Enabled:      "true",
+				PrivateKey:   privateKey,
+				SymmetricKey: "secret",
+			},
+		},
+	}
+}
+
+func generatePKCS8RSAPrivateKeyPEM(t *testing.T) string {
+	t.Helper()
+
+	privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		t.Fatalf("rsa.GenerateKey() error = %v", err)
+	}
+
+	der, err := x509.MarshalPKCS8PrivateKey(privateKey)
+	if err != nil {
+		t.Fatalf("x509.MarshalPKCS8PrivateKey() error = %v", err)
+	}
+
+	return string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}))
+}
+
+func assertEncryptedMQTTMessage(t *testing.T, payload []byte) {
+	t.Helper()
+
+	var got Message
+	if err := json.Unmarshal(payload, &got); err != nil {
+		t.Fatalf("json.Unmarshal() error = %v", err)
+	}
+
+	if !got.Encrypted {
+		t.Fatal("message is not marked encrypted")
+	}
+	if got.Payload.EncryptedValue == "" {
+		t.Fatal("encrypted value is empty")
+	}
+	if got.Payload.Signature == "" {
+		t.Fatal("signature is empty")
+	}
+	if len(got.Payload.Value) != 0 {
+		t.Fatalf("payload value = %#v, want cleared map", got.Payload.Value)
+	}
+}
+
+func hookPKCS8PrivateKeyParser(t *testing.T, parser func([]byte) (any, error)) func() {
+	t.Helper()
+
+	original := parsePKCS8PrivateKey
+	parsePKCS8PrivateKey = parser
+
+	return func() {
+		parsePKCS8PrivateKey = original
+	}
+}
+
+func resetMQTTPrivateKeyCache(t *testing.T) {
+	t.Helper()
+	rsaPrivateKeyCache = newRSAPrivateKeyCache()
+}

machinery/src/models/recording_upload_metadata.go

@@ -11,11 +11,14 @@ const RecordingUploadMetadataExtension = ".metadata"
 // with a recording. New optional fields can be added without changing the queue
 // mechanism or breaking older agents.
 type RecordingUploadMetadata struct {
-	FileName  string  `json:"filename"`
-	DeviceKey string  `json:"device_key"`
-	Timestamp int64   `json:"timestamp"` // Unix milliseconds.
-	Duration  uint64  `json:"duration"`  // Milliseconds.
-	FPS       float64 `json:"fps,omitempty"`
+	FileName          string  `json:"filename"`
+	DeviceKey         string  `json:"device_key"`
+	DeviceName        string  `json:"device_name,omitempty"`
+	Timestamp         int64   `json:"timestamp"` // Unix milliseconds.
+	Duration          uint64  `json:"duration"`  // Milliseconds.
+	FPS               float64 `json:"fps,omitempty"`
+	RegionCoordinates string  `json:"region_coordinates,omitempty"`
+	NumberOfChanges   string  `json:"number_of_changes,omitempty"`
 }
 
 // RecordingUploadMetadataFileName returns the queue marker name associated

machinery/src/onvif/discovery.go

@@ -13,8 +13,8 @@ import (
 	"time"
 
 	onvifc "github.com/cedricve/go-onvif"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 // scanPort describes a TCP port we probe while scanning the local network,
@@ -105,7 +105,7 @@ func DiscoverDevices(timeout time.Duration, subnets ...string) []models.Discover
 	// 1) ONVIF WS-Discovery. This is quick and reliable for ONVIF cameras.
 	onvifDevices, err := onvifc.StartDiscovery(timeout)
 	if err != nil {
-		log.Log.Error("onvif.DiscoverDevices(): WS-Discovery failed: " + err.Error())
+		log.Error("onvif.DiscoverDevices(): WS-Discovery failed: " + err.Error())
 	} else {
 		for _, onvifDevice := range onvifDevices {
 			ip := hostFromXAddr(onvifDevice.XAddr)
@@ -129,7 +129,7 @@ func DiscoverDevices(timeout time.Duration, subnets ...string) []models.Discover
 	} else {
 		targets = localScanTargets()
 	}
-	log.Log.Info("onvif.DiscoverDevices(): scanning " + strconv.Itoa(len(targets)) + " hosts on the local network(s)")
+	log.Info("onvif.DiscoverDevices(): scanning " + strconv.Itoa(len(targets)) + " hosts on the local network(s)")
 
 	// Bound the amount of concurrent dials so we do not exhaust file
 	// descriptors on constrained devices (e.g. Raspberry Pi).
@@ -319,17 +319,17 @@ func targetsFromSubnets(subnets []string) []string {
 					targets = append(targets, subnet)
 				}
 			} else {
-				log.Log.Error("onvif.targetsFromSubnets(): invalid address '" + subnet + "'")
+				log.Error("onvif.targetsFromSubnets(): invalid address '" + subnet + "'")
 			}
 			continue
 		}
 		_, ipNet, err := net.ParseCIDR(subnet)
 		if err != nil || ipNet.IP.To4() == nil {
-			log.Log.Error("onvif.targetsFromSubnets(): invalid CIDR '" + subnet + "'")
+			log.Error("onvif.targetsFromSubnets(): invalid CIDR '" + subnet + "'")
 			continue
 		}
 		if ones, bits := ipNet.Mask.Size(); bits != 32 || ones < 22 {
-			log.Log.Error("onvif.targetsFromSubnets(): range '" + subnet + "' is too large to scan (use /22 or smaller)")
+			log.Error("onvif.targetsFromSubnets(): range '" + subnet + "' is too large to scan (use /22 or smaller)")
 			continue
 		}
 		for _, host := range hostsInNetwork(ipNet) {
@@ -352,7 +352,7 @@ func localScanTargets() []string {
 
 	interfaces, err := net.Interfaces()
 	if err != nil {
-		log.Log.Error("onvif.localScanTargets(): " + err.Error())
+		log.Error("onvif.localScanTargets(): " + err.Error())
 		return targets
 	}
 

machinery/src/onvif/events.go

@@ -7,9 +7,9 @@ import (
 	"strings"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/onvif/event/stream"
+	log "github.com/sirupsen/logrus"
 )
 
 // The library handles in-stream reconnect; these guards cover the
@@ -29,14 +29,14 @@ const (
 // blip, credential reload) the goroutine retries with exponential
 // backoff. Exits when ctx is cancelled.
 func HandleONVIFEventStream(ctx context.Context, configuration *models.Configuration, communication *models.Communication) {
-	log.Log.Debug("onvif.HandleONVIFEventStream(): started")
-	defer log.Log.Debug("onvif.HandleONVIFEventStream(): finished")
+	log.Debug("onvif.HandleONVIFEventStream(): started")
+	defer log.Debug("onvif.HandleONVIFEventStream(): finished")
 
 	if !isONVIFMotionEnabled(configuration.Config.Capture.ONVIFMotion) {
 		return
 	}
 	if configuration.Config.Capture.IPCamera.ONVIFXAddr == "" {
-		log.Log.Warning("onvif.HandleONVIFEventStream(): ONVIFMotion enabled but ONVIFXAddr is empty; nothing to do")
+		log.Warn("onvif.HandleONVIFEventStream(): ONVIFMotion enabled but ONVIFXAddr is empty; nothing to do")
 		return
 	}
 
@@ -66,23 +66,26 @@ func runStreamOnce(ctx context.Context, configuration *models.Configuration, com
 
 	device, _, err := ConnectToOnvifDevice(&camera)
 	if err != nil {
-		log.Log.Error("onvif.HandleONVIFEventStream(): connect: " + err.Error())
+		log.Error("onvif.HandleONVIFEventStream(): connect: " + err.Error())
 		return true
 	}
 
 	deviceID := resolveDeviceID(configuration.Name, camera.ONVIFXAddr)
 	s, err := stream.NewStream(ctx, device, stream.Options{DeviceID: deviceID})
 	if err != nil {
-		log.Log.Error("onvif.HandleONVIFEventStream(): open stream: " + err.Error())
+		log.Error("onvif.HandleONVIFEventStream(): open stream: " + err.Error())
 		return true
 	}
 	defer func() {
 		if err := s.Close(); err != nil {
-			log.Log.Debug("onvif.HandleONVIFEventStream(): close: " + err.Error())
+			log.Debug("onvif.HandleONVIFEventStream(): close: " + err.Error())
 		}
 	}()
 
-	log.Log.Info("onvif.HandleONVIFEventStream(): consuming events for " + deviceID)
+	log.WithFields(log.Fields{
+		"component": "onvif",
+		"event":     "event_stream_started",
+	}).Info("Consuming ONVIF events")
 
 	// recovering = the first successful event after an error streak
 	// logs a recovery line so on-call operators see the clear-of-
@@ -97,7 +100,10 @@ func runStreamOnce(ctx context.Context, configuration *models.Configuration, com
 				return false
 			}
 			if recovering {
-				log.Log.Info("onvif.HandleONVIFEventStream(): event stream recovered for " + deviceID)
+				log.WithFields(log.Fields{
+					"component": "onvif",
+					"event":     "event_stream_recovered",
+				}).Info("ONVIF event stream recovered")
 				recovering = false
 			}
 			dispatchEvent(ctx, ev, configuration, communication)
@@ -119,14 +125,14 @@ func runStreamOnce(ctx context.Context, configuration *models.Configuration, com
 func dispatchEvent(ctx context.Context, ev stream.Event, configuration *models.Configuration, communication *models.Communication) {
 	topic := sanitiseTopic(ev.Topic)
 	if ev.Kind != stream.KindMotion {
-		log.Log.Debug("onvif.dispatchEvent(): non-motion event " + ev.Kind.String() + " topic=" + topic)
+		log.Debug("onvif.dispatchEvent(): non-motion event " + ev.Kind.String() + " topic=" + topic)
 		return
 	}
 	if ev.State != stream.StateActive {
 		return
 	}
 	if !isTransition(ev.Operation) {
-		log.Log.Debug("onvif.dispatchEvent(): " + ev.Operation.String() + " is not a transition, not a trigger: topic=" + topic)
+		log.Debug("onvif.dispatchEvent(): " + ev.Operation.String() + " is not a transition, not a trigger: topic=" + topic)
 		return
 	}
 	if configuration.Config.Capture.Recording == "false" {
@@ -140,14 +146,15 @@ func dispatchEvent(ctx context.Context, ev stream.Event, configuration *models.C
 		Timestamp:       time.Now().Unix(),
 		NumberOfChanges: 0, // ONVIF does not quantify motion area.
 	}
-	select {
-	case <-ctx.Done():
-	case communication.HandleMotion <- dataToPass:
+	if ctx.Err() != nil {
+		return
+	}
+	if communication.TrySendMotion(dataToPass) {
 		// Logged on the send, not before it: this line records that a
 		// recording started, so a dropped event must not leave one.
-		log.Log.Debug("onvif.dispatchEvent(): recording trigger " + ev.Kind.String() + " topic=" + topic)
-	default:
-		log.Log.Debug("onvif.dispatchEvent(): HandleMotion full, dropping ONVIF motion event")
+		log.Debug("onvif.dispatchEvent(): recording trigger " + ev.Kind.String() + " topic=" + topic)
+	} else {
+		log.Debug("onvif.dispatchEvent(): HandleMotion full, dropping ONVIF motion event")
 	}
 }
 
@@ -185,13 +192,13 @@ func logStreamError(e error) {
 	var renew stream.ErrRenewFailed
 	switch {
 	case errors.As(e, &recreate):
-		log.Log.Error("onvif.HandleONVIFEventStream(): subscription recreate failed (camera may be offline): " + recreate.Err.Error())
+		log.Error("onvif.HandleONVIFEventStream(): subscription recreate failed (camera may be offline): " + recreate.Err.Error())
 	case errors.As(e, &renew):
-		log.Log.Debug("onvif.HandleONVIFEventStream(): renew failed (will recover via pull/recreate): " + renew.Err.Error())
+		log.Debug("onvif.HandleONVIFEventStream(): renew failed (will recover via pull/recreate): " + renew.Err.Error())
 	case errors.As(e, &pull):
-		log.Log.Debug("onvif.HandleONVIFEventStream(): pull failed (will retry): " + pull.Err.Error())
+		log.Debug("onvif.HandleONVIFEventStream(): pull failed (will retry): " + pull.Err.Error())
 	default:
-		log.Log.Info("onvif.HandleONVIFEventStream(): stream error: " + e.Error())
+		log.Info("onvif.HandleONVIFEventStream(): stream error: " + e.Error())
 	}
 }
 

machinery/src/onvif/events_test.go

@@ -3,6 +3,7 @@ package onvif
 import (
 	"bytes"
 	"context"
+	"errors"
 	"strings"
 	"testing"
 	"time"
@@ -26,17 +27,31 @@ func makeConfig(recording, onvifMotion, name string) *models.Configuration {
 	}
 }
 
-func makeCommunication(buffer int) *models.Communication {
-	return &models.Communication{
-		HandleMotion: make(chan models.MotionDataPartial, buffer),
+func makeCommunication(t *testing.T) *models.Communication {
+	t.Helper()
+	communication := &models.Communication{}
+	run := models.NewAgentRun(context.Background(), communication, false)
+	require.NoError(t, run.Activate())
+	t.Cleanup(func() {
+		run.Shutdown(context.Background(), errors.New("test complete"))
+	})
+	return communication
+}
+
+func motionEvents(t *testing.T, communication *models.Communication) <-chan models.MotionDataPartial {
+	t.Helper()
+	run := communication.CurrentRun()
+	if run == nil {
+		t.Fatal("expected active agent run")
 	}
+	return run.MotionEvents()
 }
 
 // --- dispatchEvent ---------------------------------------------------
 
 func TestDispatchEvent_MotionActive_SendsToHandleMotion(t *testing.T) {
 	cfg := makeConfig("true", "true", "cam-1")
-	comm := makeCommunication(1)
+	comm := makeCommunication(t)
 	ev := stream.Event{Kind: stream.KindMotion, State: stream.StateActive}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -44,7 +59,7 @@ func TestDispatchEvent_MotionActive_SendsToHandleMotion(t *testing.T) {
 	dispatchEvent(ctx, ev, cfg, comm)
 
 	select {
-	case m := <-comm.HandleMotion:
+	case m := <-motionEvents(t, comm):
 		assert.NotZero(t, m.Timestamp)
 	case <-time.After(time.Second):
 		t.Fatal("expected motion data on HandleMotion")
@@ -53,7 +68,7 @@ func TestDispatchEvent_MotionActive_SendsToHandleMotion(t *testing.T) {
 
 func TestDispatchEvent_MotionInactive_DoesNotSend(t *testing.T) {
 	cfg := makeConfig("true", "true", "cam-1")
-	comm := makeCommunication(1)
+	comm := makeCommunication(t)
 	ev := stream.Event{Kind: stream.KindMotion, State: stream.StateInactive}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -61,7 +76,7 @@ func TestDispatchEvent_MotionInactive_DoesNotSend(t *testing.T) {
 	dispatchEvent(ctx, ev, cfg, comm)
 
 	select {
-	case <-comm.HandleMotion:
+	case <-motionEvents(t, comm):
 		t.Fatal("inactive motion must not reach HandleMotion (motion-stop is a follow-up)")
 	case <-time.After(100 * time.Millisecond):
 	}
@@ -69,7 +84,7 @@ func TestDispatchEvent_MotionInactive_DoesNotSend(t *testing.T) {
 
 func TestDispatchEvent_NonMotionKindIgnored(t *testing.T) {
 	cfg := makeConfig("true", "true", "cam-1")
-	comm := makeCommunication(1)
+	comm := makeCommunication(t)
 	ev := stream.Event{Kind: stream.KindDigitalInput, State: stream.StateActive}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -77,7 +92,7 @@ func TestDispatchEvent_NonMotionKindIgnored(t *testing.T) {
 	dispatchEvent(ctx, ev, cfg, comm)
 
 	select {
-	case <-comm.HandleMotion:
+	case <-motionEvents(t, comm):
 		t.Fatal("non-motion kinds must not reach HandleMotion")
 	case <-time.After(100 * time.Millisecond):
 	}
@@ -108,7 +123,7 @@ func TestDispatchEvent_LogsTheTriggeringTopic(t *testing.T) {
 	buf := captureDebugLog(t)
 
 	cfg := makeConfig("true", "true", "cam-1")
-	comm := makeCommunication(1)
+	comm := makeCommunication(t)
 	ev := stream.Event{
 		Kind:  stream.KindMotion,
 		State: stream.StateActive,
@@ -147,7 +162,7 @@ func TestDispatchEvent_PropertyOperation(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			cfg := makeConfig("true", "true", "cam-1")
-			comm := makeCommunication(1)
+			comm := makeCommunication(t)
 			ev := stream.Event{
 				Kind:      stream.KindMotion,
 				State:     stream.StateActive,
@@ -160,7 +175,7 @@ func TestDispatchEvent_PropertyOperation(t *testing.T) {
 			dispatchEvent(ctx, ev, cfg, comm)
 
 			select {
-			case <-comm.HandleMotion:
+			case <-motionEvents(t, comm):
 				if !tt.wantSend {
 					t.Fatalf("%v must not trigger a recording", tt.op)
 				}
@@ -175,7 +190,7 @@ func TestDispatchEvent_PropertyOperation(t *testing.T) {
 
 func TestDispatchEvent_RecordingDisabled_DoesNotSend(t *testing.T) {
 	cfg := makeConfig("false", "true", "cam-1")
-	comm := makeCommunication(1)
+	comm := makeCommunication(t)
 	ev := stream.Event{Kind: stream.KindMotion, State: stream.StateActive}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -183,7 +198,7 @@ func TestDispatchEvent_RecordingDisabled_DoesNotSend(t *testing.T) {
 	dispatchEvent(ctx, ev, cfg, comm)
 
 	select {
-	case <-comm.HandleMotion:
+	case <-motionEvents(t, comm):
 		t.Fatal("Recording=false must gate the send (matches computervision behaviour)")
 	case <-time.After(100 * time.Millisecond):
 	}
@@ -191,9 +206,10 @@ func TestDispatchEvent_RecordingDisabled_DoesNotSend(t *testing.T) {
 
 func TestDispatchEvent_HandleMotionFull_DropsRatherThanBlocks(t *testing.T) {
 	cfg := makeConfig("true", "true", "cam-1")
-	// Pre-fill the buffer so the next send would block.
-	comm := &models.Communication{HandleMotion: make(chan models.MotionDataPartial, 1)}
-	comm.HandleMotion <- models.MotionDataPartial{}
+	// Fill the buffer so the next send would be dropped.
+	comm := makeCommunication(t)
+	for comm.TrySendMotion(models.MotionDataPartial{}) {
+	}
 	ev := stream.Event{Kind: stream.KindMotion, State: stream.StateActive}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -219,8 +235,9 @@ func TestDispatchEvent_CtxCancelledAndHandleMotionClosed_DoesNotPanic(t *testing
 	// send would panic. The ctx pre-check must short-circuit before the
 	// send is attempted.
 	cfg := makeConfig("true", "true", "cam-1")
-	comm := &models.Communication{HandleMotion: make(chan models.MotionDataPartial, 1)}
-	close(comm.HandleMotion)
+	comm := makeCommunication(t)
+	run := comm.CurrentRun()
+	run.Shutdown(context.Background(), errors.New("test shutdown"))
 	ev := stream.Event{Kind: stream.KindMotion, State: stream.StateActive}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -299,7 +316,7 @@ func TestDispatchEvent_OnlyRealTransitionsTrigger(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.op.String(), func(t *testing.T) {
 			cfg := makeConfig("true", "true", "cam-1")
-			comm := makeCommunication(1)
+			comm := makeCommunication(t)
 			ev := stream.Event{Kind: stream.KindMotion, State: stream.StateActive, Operation: tt.op}
 
 			ctx, cancel := context.WithCancel(context.Background())
@@ -307,10 +324,10 @@ func TestDispatchEvent_OnlyRealTransitionsTrigger(t *testing.T) {
 			dispatchEvent(ctx, ev, cfg, comm)
 
 			if tt.wantSend {
-				require.Len(t, comm.HandleMotion, 1, "%v must trigger a recording", tt.op)
+				require.Len(t, motionEvents(t, comm), 1, "%v must trigger a recording", tt.op)
 				return
 			}
-			require.Empty(t, comm.HandleMotion, "%v must not trigger a recording", tt.op)
+			require.Empty(t, motionEvents(t, comm), "%v must not trigger a recording", tt.op)
 		})
 	}
 }
@@ -361,8 +378,9 @@ func TestDispatchEvent_LogsTriggerOnlyWhenSent(t *testing.T) {
 	buf := captureDebugLog(t)
 
 	cfg := makeConfig("true", "true", "cam-1")
-	comm := &models.Communication{HandleMotion: make(chan models.MotionDataPartial, 1)}
-	comm.HandleMotion <- models.MotionDataPartial{} // full
+	comm := makeCommunication(t)
+	for comm.TrySendMotion(models.MotionDataPartial{}) {
+	}
 	ev := stream.Event{Kind: stream.KindMotion, State: stream.StateActive, Topic: "tns1:VideoSource/MotionAlarm"}
 
 	ctx, cancel := context.WithCancel(context.Background())

machinery/src/onvif/main.go

@@ -6,12 +6,12 @@ import (
 	"encoding/xml"
 	"errors"
 	"io"
+	"net/http"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/gin-gonic/gin"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/onvif"
 	"github.com/kerberos-io/onvif/device"
@@ -21,6 +21,7 @@ import (
 	"github.com/kerberos-io/onvif/ptz"
 	xsd "github.com/kerberos-io/onvif/xsd"
 	xsdonvif "github.com/kerberos-io/onvif/xsd/onvif"
+	log "github.com/sirupsen/logrus"
 )
 
 // Discover performs an advanced Fing/WiFiman-style scan of the local network
@@ -29,12 +30,12 @@ import (
 // `-action discover` CLI command. Optional subnets (CIDR, e.g.
 // "192.168.1.0/24") override the auto-detected local subnets.
 func Discover(timeout time.Duration, subnets ...string) {
-	log.Log.Info("onvif.Discover(): starting advanced network discovery")
-	log.Log.Info("onvif.Discover(): this may take up to " + timeout.String() + " for the ONVIF probe plus the port scan")
+	log.Info("onvif.Discover(): starting advanced network discovery")
+	log.Info("onvif.Discover(): this may take up to " + timeout.String() + " for the ONVIF probe plus the port scan")
 
 	devices := DiscoverDevices(timeout, subnets...)
 	if len(devices) == 0 {
-		log.Log.Info("onvif.Discover(): no devices discovered on the local network")
+		log.Info("onvif.Discover(): no devices discovered on the local network")
 		return
 	}
 
@@ -44,7 +45,7 @@ func Discover(timeout time.Duration, subnets ...string) {
 			cameraCount++
 		}
 	}
-	log.Log.Info("onvif.Discover(): found " + strconv.Itoa(len(devices)) + " device(s), " + strconv.Itoa(cameraCount) + " likely camera(s)")
+	log.Info("onvif.Discover(): found " + strconv.Itoa(len(devices)) + " device(s), " + strconv.Itoa(cameraCount) + " likely camera(s)")
 
 	for _, device := range devices {
 		label := "device"
@@ -84,7 +85,7 @@ func Discover(timeout time.Duration, subnets ...string) {
 		if device.RTSPURL != "" {
 			summary += " rtsp=" + device.RTSPURL
 		}
-		log.Log.Info(summary)
+		log.Info(summary)
 
 		// Detail the guessed RTSP stream URLs from the brand -> RTSP mapping.
 		for _, stream := range device.RTSPStreams {
@@ -97,15 +98,15 @@ func Discover(timeout time.Duration, subnets ...string) {
 				line += " (auth required)"
 			}
 			line += ": " + stream.URL
-			log.Log.Info(line)
+			log.Info(line)
 		}
 	}
 }
 
-func HandleONVIFActions(configuration *models.Configuration, communication *models.Communication) {
-	log.Log.Debug("onvif.HandleONVIFActions(): started")
+func HandleONVIFActions(configuration *models.Configuration, communication *models.Communication, actions <-chan models.OnvifAction) {
+	log.Debug("onvif.HandleONVIFActions(): started")
 
-	for onvifAction := range communication.HandleONVIF {
+	for onvifAction := range actions {
 
 		// First we'll get the desired PTZ action from the payload
 		// We need to know if we need to move left, right, up, down, zoom in, zoom out, center.
@@ -138,7 +139,7 @@ func HandleONVIFActions(configuration *models.Configuration, communication *mode
 						functions, _, _ := GetPTZFunctionsFromDevice(configurations)
 
 						// Log functions
-						log.Log.Debug("onvif.HandleONVIFActions(): functions: " + strings.Join(functions, ", "))
+						log.Debug("onvif.HandleONVIFActions(): functions: " + strings.Join(functions, ", "))
 
 						// Check if we need to use absolute or continuous move
 						/*canAbsoluteMove := false
@@ -159,20 +160,20 @@ func HandleONVIFActions(configuration *models.Configuration, communication *mode
 						// on the ContinuousPanTiltMove function which is more compatible with more cameras.
 						err = AbsolutePanTiltMoveFake(device, configurations, token, x, y, z)
 						if err != nil {
-							log.Log.Debug("onvif.HandleONVIFActions() - AbsolutePanTitleMoveFake: " + err.Error())
+							log.Debug("onvif.HandleONVIFActions() - AbsolutePanTitleMoveFake: " + err.Error())
 						} else {
-							log.Log.Info("onvif.HandleONVIFActions() - AbsolutePanTitleMoveFake: successfully moved camera.")
+							log.Info("onvif.HandleONVIFActions() - AbsolutePanTitleMoveFake: successfully moved camera.")
 						}
 
 						/*if canAbsoluteMove {
 							err = AbsolutePanTiltMove(device, configurations, token, x, y, z)
 							if err != nil {
-								log.Log.Error("HandleONVIFActions (AbsolutePanTitleMove): " + err.Error())
+								log.Error("HandleONVIFActions (AbsolutePanTitleMove): " + err.Error())
 							}
 						} else if canContinuousMove {
 							err = AbsolutePanTiltMoveFake(device, configurations, token, x, y, z)
 							if err != nil {
-								log.Log.Error("HandleONVIFActions (AbsolutePanTitleMoveFake): " + err.Error())
+								log.Error("HandleONVIFActions (AbsolutePanTitleMoveFake): " + err.Error())
 							}
 						}*/
 
@@ -182,9 +183,9 @@ func HandleONVIFActions(configuration *models.Configuration, communication *mode
 						preset := ptzAction.Preset
 						err := GoToPresetFromDevice(device, preset)
 						if err != nil {
-							log.Log.Debug("onvif.HandleONVIFActions() - GotoPreset: " + err.Error())
+							log.Debug("onvif.HandleONVIFActions() - GotoPreset: " + err.Error())
 						} else {
-							log.Log.Info("onvif.HandleONVIFActions() - GotoPreset: successfully moved camera")
+							log.Info("onvif.HandleONVIFActions() - GotoPreset: successfully moved camera")
 						}
 
 					} else if onvifAction.Action == "ptz" {
@@ -196,9 +197,9 @@ func HandleONVIFActions(configuration *models.Configuration, communication *mode
 								// We will move the camera to zero position.
 								err := AbsolutePanTiltMove(device, configurations, token, 0, 0, 0)
 								if err != nil {
-									log.Log.Debug("onvif.HandleONVIFActions() - AbsolutePanTitleMove: " + err.Error())
+									log.Debug("onvif.HandleONVIFActions() - AbsolutePanTitleMove: " + err.Error())
 								} else {
-									log.Log.Info("onvif.HandleONVIFActions() - AbsolutePanTitleMove: successfully centered camera")
+									log.Info("onvif.HandleONVIFActions() - AbsolutePanTitleMove: successfully centered camera")
 								}
 
 							} else {
@@ -225,9 +226,9 @@ func HandleONVIFActions(configuration *models.Configuration, communication *mode
 
 								err := ContinuousPanTilt(device, configurations, token, x, y)
 								if err != nil {
-									log.Log.Debug("onvif.HandleONVIFActions() - ContinuousPanTilt: " + err.Error())
+									log.Debug("onvif.HandleONVIFActions() - ContinuousPanTilt: " + err.Error())
 								} else {
-									log.Log.Info("onvif.HandleONVIFActions() - ContinuousPanTilt: successfully pan tilted camera")
+									log.Info("onvif.HandleONVIFActions() - ContinuousPanTilt: successfully pan tilted camera")
 								}
 							}
 						}
@@ -237,9 +238,9 @@ func HandleONVIFActions(configuration *models.Configuration, communication *mode
 							zoom := ptzAction.Zoom
 							err := ContinuousZoom(device, configurations, token, zoom)
 							if err != nil {
-								log.Log.Debug("onvif.HandleONVIFActions() - ContinuousZoom: " + err.Error())
+								log.Debug("onvif.HandleONVIFActions() - ContinuousZoom: " + err.Error())
 							} else {
-								log.Log.Info("onvif.HandleONVIFActions() - ContinuousZoom: successfully zoomed camera")
+								log.Info("onvif.HandleONVIFActions() - ContinuousZoom: successfully zoomed camera")
 							}
 						}
 					}
@@ -247,11 +248,11 @@ func HandleONVIFActions(configuration *models.Configuration, communication *mode
 			}
 		}
 	}
-	log.Log.Debug("onvif.HandleONVIFActions(): finished")
+	log.Debug("onvif.HandleONVIFActions(): finished")
 }
 
 func ConnectToOnvifDevice(cameraConfiguration *models.IPCamera) (*onvif.Device, device.GetCapabilitiesResponse, error) {
-	log.Log.Debug("onvif.ConnectToOnvifDevice(): started")
+	log.Debug("onvif.ConnectToOnvifDevice(): started")
 	dev, err := onvif.NewDevice(onvif.DeviceParams{
 		Xaddr:    cameraConfiguration.ONVIFXAddr,
 		Username: cameraConfiguration.ONVIFUsername,
@@ -269,7 +270,7 @@ func ConnectToOnvifDevice(cameraConfiguration *models.IPCamera) (*onvif.Device,
 			AuthMode: "digest",
 		})
 		if err != nil {
-			log.Log.Debug("onvif.ConnectToOnvifDevice(): " + err.Error())
+			log.Debug("onvif.ConnectToOnvifDevice(): " + err.Error())
 		}
 	}
 
@@ -277,7 +278,7 @@ func ConnectToOnvifDevice(cameraConfiguration *models.IPCamera) (*onvif.Device,
 		getCapabilities := device.GetCapabilities{Category: []xsdonvif.CapabilityCategory{"All"}}
 		resp, err := dev.CallMethod(getCapabilities)
 		if err != nil {
-			log.Log.Error("onvif.ConnectToOnvifDevice(): " + err.Error())
+			log.Error("onvif.ConnectToOnvifDevice(): " + err.Error())
 		}
 
 		var b []byte
@@ -285,24 +286,24 @@ func ConnectToOnvifDevice(cameraConfiguration *models.IPCamera) (*onvif.Device,
 			b, err = io.ReadAll(resp.Body)
 			resp.Body.Close() // Ensure the response body is closed
 			if err != nil {
-				log.Log.Error("onvif.ConnectToOnvifDevice(): " + err.Error())
+				log.Error("onvif.ConnectToOnvifDevice(): " + err.Error())
 			}
 		}
 		stringBody := string(b)
 		decodedXML, et, err := getXMLNode(stringBody, "GetCapabilitiesResponse")
 		if err != nil {
-			log.Log.Error("onvif.ConnectToOnvifDevice(): " + err.Error())
+			log.Error("onvif.ConnectToOnvifDevice(): " + err.Error())
 		} else {
 			if err := decodedXML.DecodeElement(&capabilities, et); err != nil {
-				log.Log.Error("onvif.ConnectToOnvifDevice(): " + err.Error())
+				log.Error("onvif.ConnectToOnvifDevice(): " + err.Error())
 			} else {
-				log.Log.Debug("onvif.ConnectToOnvifDevice(): capabilities.")
+				log.Debug("onvif.ConnectToOnvifDevice(): capabilities.")
 			}
 		}
 
-		log.Log.Info("onvif.ConnectToOnvifDevice(): successfully connected to device")
+		log.Info("onvif.ConnectToOnvifDevice(): successfully connected to device")
 	}
-	log.Log.Debug("onvif.ConnectToOnvifDevice(): finished")
+	log.Debug("onvif.ConnectToOnvifDevice(): finished")
 	return dev, capabilities, err
 }
 
@@ -319,13 +320,13 @@ func GetTokenFromProfile(device *onvif.Device, profileId int) (xsdonvif.Referenc
 			resp.Body.Close() // Ensure the response body is closed
 			decodedXML, et, err := getXMLNode(stringBody, "GetProfilesResponse")
 			if err != nil {
-				log.Log.Debug("onvif.GetTokenFromProfile(): " + err.Error())
+				log.Debug("onvif.GetTokenFromProfile(): " + err.Error())
 				return profileToken, err
 			} else {
 				// Decode the profiles from the server
 				var mProfilesResp media.GetProfilesResponse
 				if err := decodedXML.DecodeElement(&mProfilesResp, et); err != nil {
-					log.Log.Debug("onvif.GetTokenFromProfile(): " + err.Error())
+					log.Debug("onvif.GetTokenFromProfile(): " + err.Error())
 				}
 
 				// We'll try to get the token from a preferred profile
@@ -356,11 +357,11 @@ func GetPTZConfigurationsFromDevice(device *onvif.Device) (ptz.GetConfigurations
 		stringBody := string(b)
 		decodedXML, et, err := getXMLNode(stringBody, "GetConfigurationsResponse")
 		if err != nil {
-			log.Log.Debug("onvif.GetPTZConfigurationsFromDevice(): " + err.Error())
+			log.Debug("onvif.GetPTZConfigurationsFromDevice(): " + err.Error())
 			return configurations, err
 		} else {
 			if err := decodedXML.DecodeElement(&configurations, et); err != nil {
-				log.Log.Debug("onvif.GetPTZConfigurationsFromDevice(): " + err.Error())
+				log.Debug("onvif.GetPTZConfigurationsFromDevice(): " + err.Error())
 				return configurations, err
 			}
 		}
@@ -386,22 +387,22 @@ func GetPositionFromDevice(configuration models.Configuration) (xsdonvif.PTZVect
 					x := strconv.FormatFloat(position.PanTilt.X, 'f', 6, 64)
 					y := strconv.FormatFloat(position.PanTilt.Y, 'f', 6, 64)
 					z := strconv.FormatFloat(position.Zoom.X, 'f', 6, 64)
-					log.Log.Info("onvif.GetPositionFromDevice(): successfully got position (" + x + ", " + y + ", " + z + ")")
+					log.Info("onvif.GetPositionFromDevice(): successfully got position (" + x + ", " + y + ", " + z + ")")
 					return position, err
 				} else {
-					log.Log.Debug("onvif.GetPositionFromDevice(): position is nil")
+					log.Debug("onvif.GetPositionFromDevice(): position is nil")
 					return position, errors.New("position is nil")
 				}
 			} else {
-				log.Log.Debug("onvif.GetPositionFromDevice(): " + err.Error())
+				log.Debug("onvif.GetPositionFromDevice(): " + err.Error())
 				return position, err
 			}
 		} else {
-			log.Log.Debug("onvif.GetPositionFromDevice(): " + err.Error())
+			log.Debug("onvif.GetPositionFromDevice(): " + err.Error())
 			return position, err
 		}
 	} else {
-		log.Log.Debug("onvif.GetPositionFromDevice(): " + err.Error())
+		log.Debug("onvif.GetPositionFromDevice(): " + err.Error())
 		return position, err
 	}
 }
@@ -426,11 +427,11 @@ func GetPosition(device *onvif.Device, token xsdonvif.ReferenceToken) (xsdonvif.
 		stringBody := string(b)
 		decodedXML, et, err := getXMLNode(stringBody, "GetStatusResponse")
 		if err != nil {
-			log.Log.Error("GetPositionFromDevice: " + err.Error())
+			log.Error("GetPositionFromDevice: " + err.Error())
 			return position, err
 		} else {
 			if err := decodedXML.DecodeElement(&status, et); err != nil {
-				log.Log.Error("GetPositionFromDevice: " + err.Error())
+				log.Error("GetPositionFromDevice: " + err.Error())
 				return position, err
 			}
 		}
@@ -439,6 +440,36 @@ func GetPosition(device *onvif.Device, token xsdonvif.ReferenceToken) (xsdonvif.
 	return position, err
 }
 
+func handleONVIFResponse(operation string, response *http.Response, requestErr error) error {
+	fields := log.Fields{
+		"component": "onvif",
+		"operation": operation,
+	}
+	var responseErr error
+	if response != nil {
+		fields["status_code"] = response.StatusCode
+		fields["response_bytes"], responseErr = io.Copy(io.Discard, response.Body)
+		response.Body.Close()
+	}
+
+	err := errors.Join(requestErr, responseErr)
+	if response == nil && err == nil {
+		err = errors.New("ONVIF operation returned no response")
+	}
+	if err == nil && (response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices) {
+		err = errors.New("ONVIF operation was rejected")
+	}
+	if err != nil {
+		fields["event"] = "operation_failed"
+		log.WithError(err).WithFields(fields).Error("ONVIF operation failed")
+		return err
+	}
+
+	fields["event"] = "operation_completed"
+	log.WithFields(fields).Debug("ONVIF operation completed")
+	return nil
+}
+
 func AbsolutePanTiltMove(device *onvif.Device, configuration ptz.GetConfigurationsResponse, token xsdonvif.ReferenceToken, pan float64, tilt float64, zoom float64) error {
 
 	absolutePantiltVector := xsdonvif.Vector2D{
@@ -459,18 +490,7 @@ func AbsolutePanTiltMove(device *onvif.Device, configuration ptz.GetConfiguratio
 			Zoom:    &absoluteZoomVector,
 		},
 	})
-
-	var b []byte
-	if resp != nil {
-		b, err = io.ReadAll(resp.Body)
-		resp.Body.Close()
-	}
-	if err != nil {
-		log.Log.Error("AbsoluteMove: " + err.Error())
-	}
-	log.Log.Info("AbsoluteMove: " + string(b))
-
-	return err
+	return handleONVIFResponse("absolute_move", resp, err)
 }
 
 // This function will simulate the AbsolutePanTiltMove function.
@@ -479,7 +499,7 @@ func AbsolutePanTiltMove(device *onvif.Device, configuration ptz.GetConfiguratio
 func AbsolutePanTiltMoveFake(device *onvif.Device, configuration ptz.GetConfigurationsResponse, token xsdonvif.ReferenceToken, pan float64, tilt float64, zoom float64) error {
 	position, err := GetPosition(device, token)
 	if position.PanTilt.X >= pan-0.01 && position.PanTilt.X <= pan+0.01 && position.PanTilt.Y >= tilt-0.01 && position.PanTilt.Y <= tilt+0.01 && position.Zoom.X >= zoom-0.01 && position.Zoom.X <= zoom+0.01 {
-		log.Log.Debug("AbsolutePanTiltMoveFake: already at position")
+		log.Debug("AbsolutePanTiltMoveFake: already at position")
 	} else {
 
 		// The speed of panning, the higher the faster we'll pan the camera
@@ -518,7 +538,7 @@ func ZoomOutCompletely(device *onvif.Device, configuration ptz.GetConfigurations
 		},
 	})
 	if err != nil {
-		log.Log.Error("ZoomOutCompletely: " + err.Error())
+		log.Error("ZoomOutCompletely: " + err.Error())
 	}
 
 	for {
@@ -534,7 +554,7 @@ func ZoomOutCompletely(device *onvif.Device, configuration ptz.GetConfigurations
 		Zoom:         true,
 	})
 	if err != nil {
-		log.Log.Error("ZoomOutCompletely: " + err.Error())
+		log.Error("ZoomOutCompletely: " + err.Error())
 	}
 	return err
 }
@@ -564,16 +584,7 @@ func PanUntilPosition(device *onvif.Device, configuration ptz.GetConfigurationsR
 				PanTilt: panTiltVector,
 			},
 		})
-
-		var b []byte
-		if resp != nil {
-			b, err = io.ReadAll(resp.Body)
-			resp.Body.Close()
-		}
-		if err != nil {
-			log.Log.Error("ContinuousPanTiltMove (Pan): " + err.Error())
-		}
-		log.Log.Debug("ContinuousPanTiltMove (Pan): " + string(b))
+		err = handleONVIFResponse("continuous_pan", resp, err)
 
 		// While moving we'll check if we reached the desired position.
 		// or if we overshot the desired position.
@@ -598,7 +609,7 @@ func PanUntilPosition(device *onvif.Device, configuration ptz.GetConfigurationsR
 		})
 
 		if err != nil {
-			log.Log.Error("ContinuousPanTiltMove (Pan): " + err.Error())
+			log.Error("ContinuousPanTiltMove (Pan): " + err.Error())
 		}
 	}
 	return err
@@ -632,17 +643,7 @@ func TiltUntilPosition(device *onvif.Device, configuration ptz.GetConfigurations
 			ProfileToken: &token,
 			Velocity:     velocity,
 		})
-
-		var b []byte
-		if resp != nil {
-			b, err = io.ReadAll(resp.Body)
-			resp.Body.Close()
-		}
-
-		if err != nil {
-			log.Log.Error("ContinuousPanTiltMove (Tilt): " + err.Error())
-		}
-		log.Log.Debug("ContinuousPanTiltMove (Tilt) " + string(b))
+		err = handleONVIFResponse("continuous_tilt", resp, err)
 
 		// While moving we'll check if we reached the desired position.
 		// or if we overshot the desired position.
@@ -667,7 +668,7 @@ func TiltUntilPosition(device *onvif.Device, configuration ptz.GetConfigurations
 		})
 
 		if err != nil {
-			log.Log.Error("ContinuousPanTiltMove (Tilt): " + err.Error())
+			log.Error("ContinuousPanTiltMove (Tilt): " + err.Error())
 		}
 	}
 	return err
@@ -697,17 +698,7 @@ func ZoomUntilPosition(device *onvif.Device, configuration ptz.GetConfigurations
 				Zoom: zoomVector,
 			},
 		})
-
-		var b []byte
-		if resp != nil {
-			b, err = io.ReadAll(resp.Body)
-			resp.Body.Close()
-		}
-		if err != nil {
-			log.Log.Error("ContinuousPanTiltMove (Zoom): " + err.Error())
-		}
-
-		log.Log.Debug("ContinuousPanTiltMove (Zoom) " + string(b))
+		err = handleONVIFResponse("continuous_zoom_to_position", resp, err)
 
 		// While moving we'll check if we reached the desired position.
 		// or if we overshot the desired position.
@@ -732,7 +723,7 @@ func ZoomUntilPosition(device *onvif.Device, configuration ptz.GetConfigurations
 		})
 
 		if err != nil {
-			log.Log.Error("ContinuousPanTiltMove (Zoom): " + err.Error())
+			log.Error("ContinuousPanTiltMove (Zoom): " + err.Error())
 		}
 	}
 	return err
@@ -752,17 +743,7 @@ func ContinuousPanTilt(device *onvif.Device, configuration ptz.GetConfigurations
 			PanTilt: panTiltVector,
 		},
 	})
-
-	var b []byte
-	if resp != nil {
-		b, err = io.ReadAll(resp.Body)
-		resp.Body.Close()
-	}
-	if err != nil {
-		log.Log.Error("ContinuousPanTiltMove: " + err.Error())
-	}
-
-	log.Log.Debug("ContinuousPanTiltMove: " + string(b))
+	moveErr := handleONVIFResponse("continuous_pan_tilt", resp, err)
 
 	time.Sleep(200 * time.Millisecond)
 
@@ -770,18 +751,8 @@ func ContinuousPanTilt(device *onvif.Device, configuration ptz.GetConfigurations
 		ProfileToken: token,
 		PanTilt:      true,
 	})
-
-	b = []byte{}
-	if resp != nil {
-		b, err = io.ReadAll(resp.Body)
-		resp.Body.Close()
-	}
-
-	if err != nil {
-		log.Log.Error("ContinuousPanTiltMove: " + err.Error())
-	}
-
-	return err
+	stopErr := handleONVIFResponse("continuous_pan_tilt_stop", resp, err)
+	return errors.Join(moveErr, stopErr)
 }
 
 func ContinuousZoom(device *onvif.Device, configuration ptz.GetConfigurationsResponse, token xsdonvif.ReferenceToken, zoom float64) error {
@@ -799,36 +770,26 @@ func ContinuousZoom(device *onvif.Device, configuration ptz.GetConfigurationsRes
 		ProfileToken: &token,
 		Velocity:     &velocity,
 	})
-
-	var b []byte
-	if resp != nil {
-		b, err = io.ReadAll(resp.Body)
-		resp.Body.Close()
-	}
-	if err != nil {
-		log.Log.Error("onvif.main.ContinuousZoom(): " + err.Error())
-	}
-
-	log.Log.Debug("onvif.main.ContinuousZoom(): " + string(b))
+	moveErr := handleONVIFResponse("continuous_zoom", resp, err)
 	time.Sleep(500 * time.Millisecond)
 
-	_, err = device.CallMethod(ptz.Stop{
+	resp, err = device.CallMethod(ptz.Stop{
 		ProfileToken: token,
 		Zoom:         true,
 	})
-
-	if err != nil {
-		log.Log.Error("onvif.main.ContinuousZoom(): " + err.Error())
-	}
-
-	return err
+	stopErr := handleONVIFResponse("continuous_zoom_stop", resp, err)
+	return errors.Join(moveErr, stopErr)
 }
 
 func GetCapabilitiesFromDevice(dev *onvif.Device) []string {
 	var capabilities []string
 	services := dev.GetServices()
 	for key, _ := range services {
-		log.Log.Debug("onvif.main.GetCapabilitiesFromDevice(): has key: " + key)
+		log.WithFields(log.Fields{
+			"capability": key,
+			"component":  "onvif",
+			"event":      "capability_detected",
+		}).Debug("ONVIF capability detected")
 		if key != "" {
 			keyParts := strings.Split(key, "/")
 			if len(keyParts) > 0 {
@@ -859,11 +820,11 @@ func GetPresetsFromDevice(device *onvif.Device) ([]models.OnvifActionPreset, err
 			stringBody := string(b)
 			decodedXML, et, err := getXMLNode(stringBody, "GetPresetsResponse")
 			if err != nil {
-				log.Log.Error("onvif.main.GetPresetsFromDevice(): " + err.Error())
+				log.Error("onvif.main.GetPresetsFromDevice(): " + err.Error())
 				return presets, err
 			} else {
 				if err := decodedXML.DecodeElement(&presetsResponse, et); err != nil {
-					log.Log.Error("onvif.main.GetPresetsFromDevice(): " + err.Error())
+					log.Error("onvif.main.GetPresetsFromDevice(): " + err.Error())
 					return presets, err
 				}
 
@@ -876,15 +837,15 @@ func GetPresetsFromDevice(device *onvif.Device) ([]models.OnvifActionPreset, err
 					presetsList += string(preset.Name) + " (" + string(preset.Token) + "), "
 					presets = append(presets, p)
 				}
-				log.Log.Debug("onvif.main.GetPresetsFromDevice(): " + presetsList)
+				log.Debug("onvif.main.GetPresetsFromDevice(): " + presetsList)
 
 				return presets, err
 			}
 		} else {
-			log.Log.Error("onvif.main.GetPresetsFromDevice(): " + err.Error())
+			log.Error("onvif.main.GetPresetsFromDevice(): " + err.Error())
 		}
 	} else {
-		log.Log.Error("onvif.main.GetPresetsFromDevice(): " + err.Error())
+		log.Error("onvif.main.GetPresetsFromDevice(): " + err.Error())
 	}
 
 	return presets, err
@@ -910,20 +871,20 @@ func GoToPresetFromDevice(device *onvif.Device, presetName string) error {
 			stringBody := string(b)
 			decodedXML, et, err := getXMLNode(stringBody, "GotoPresetResponses")
 			if err != nil {
-				log.Log.Error("onvif.main.GoToPresetFromDevice(): " + err.Error())
+				log.Error("onvif.main.GoToPresetFromDevice(): " + err.Error())
 				return err
 			} else {
 				if err := decodedXML.DecodeElement(&goToPresetResponse, et); err != nil {
-					log.Log.Error("onvif.main.GoToPresetFromDevice(): " + err.Error())
+					log.Error("onvif.main.GoToPresetFromDevice(): " + err.Error())
 					return err
 				}
 				return err
 			}
 		} else {
-			log.Log.Error("onvif.main.GoToPresetFromDevice(): " + err.Error())
+			log.Error("onvif.main.GoToPresetFromDevice(): " + err.Error())
 		}
 	} else {
-		log.Log.Error("onvif.main.GoToPresetFromDevice(): " + err.Error())
+		log.Error("onvif.main.GoToPresetFromDevice(): " + err.Error())
 	}
 
 	return err
@@ -1068,10 +1029,10 @@ func CreatePullPointSubscription(dev *onvif.Device) (string, error) {
 			stringBody := string(b2)
 			decodedXML, et, err := getXMLNode(stringBody, "CreatePullPointSubscriptionResponse")
 			if err != nil {
-				log.Log.Debug("onvif.main.CreatePullPointSubscription(): " + err.Error())
+				log.Debug("onvif.main.CreatePullPointSubscription(): " + err.Error())
 			} else {
 				if err := decodedXML.DecodeElement(&createPullPointSubscriptionResponse, et); err != nil {
-					log.Log.Error("onvif.main.CreatePullPointSubscription(): " + err.Error())
+					log.Error("onvif.main.CreatePullPointSubscription(): " + err.Error())
 				} else {
 					pullPointAdress = string(createPullPointSubscriptionResponse.SubscriptionReference.Address)
 				}
@@ -1087,25 +1048,40 @@ func UnsubscribePullPoint(dev *onvif.Device, pullPointAddress string) error {
 	unsubscribe := event.Unsubscribe{}
 	requestBody, err := xml.Marshal(unsubscribe)
 	if err != nil {
-		log.Log.Error("onvif.main.UnsubscribePullPoint(): " + err.Error())
+		log.WithError(err).WithFields(log.Fields{
+			"component": "onvif",
+			"event":     "unsubscribe_request_encoding_failed",
+		}).Error("Failed to encode ONVIF unsubscribe request")
+		return err
 	}
 
 	res, err := dev.SendSoap(pullPointAddress, string(requestBody))
 	if err != nil {
-		log.Log.Error("onvif.main.UnsubscribePullPoint(): " + err.Error())
+		log.WithError(err).WithFields(log.Fields{
+			"component": "onvif",
+			"event":     "unsubscribe_request_failed",
+		}).Error("Failed to send ONVIF unsubscribe request")
+		return err
 	}
-	if res != nil {
-		b, err := io.ReadAll(res.Body)
-		res.Body.Close() // Ensure the response body is closed
-		if err == nil {
-			stringBody := string(b)
-			log.Log.Debug("onvif.main.UnsubscribePullPoint(): " + stringBody)
-		}
-		if err != nil {
-			log.Log.Error("onvif.main.UnsubscribePullPoint(): " + err.Error())
-		}
+	if res == nil {
+		return errors.New("ONVIF unsubscribe returned no response")
 	}
-	return err
+	defer res.Body.Close()
+	responseBytes, err := io.Copy(io.Discard, res.Body)
+	if err != nil {
+		log.WithError(err).WithFields(log.Fields{
+			"component": "onvif",
+			"event":     "unsubscribe_response_read_failed",
+		}).Error("Failed to read ONVIF unsubscribe response")
+		return err
+	}
+	log.WithFields(log.Fields{
+		"component":      "onvif",
+		"event":          "unsubscribe_completed",
+		"response_bytes": responseBytes,
+		"status_code":    res.StatusCode,
+	}).Debug("ONVIF unsubscribe completed")
+	return nil
 }
 
 // Look for Source of input and output
@@ -1126,7 +1102,13 @@ func GetInputOutputs() ([]ONVIFEvents, error) {
 		eventsArray = append(eventsArray, *value)
 	}
 	for _, value := range eventsArray {
-		log.Log.Debug("onvif.main.GetInputOutputs(): " + value.Key + " - " + value.Value + " (" + strconv.FormatInt(value.Timestamp, 10) + ")")
+		log.WithFields(log.Fields{
+			"component":      "onvif",
+			"event":          "input_output_state",
+			"input_output":   value.Key,
+			"state":          value.Value,
+			"timestamp_unix": value.Timestamp,
+		}).Debug("ONVIF input/output state")
 	}
 	return eventsArray, nil
 }
@@ -1143,7 +1125,7 @@ func GetEventMessages(dev *onvif.Device, pullPointAddress string) ([]ONVIFEvents
 		// We were able to create a subscription to the device. Now pull some messages from the subscription.
 		subscriptionURI := pullPointAddress
 		if subscriptionURI == "" {
-			log.Log.Error("onvif.main.GetEventMessages(): subscriptionURI is empty")
+			log.Error("onvif.main.GetEventMessages(): subscriptionURI is empty")
 		} else {
 			// Pull message
 			pullMessage := event.PullMessages{
@@ -1152,12 +1134,12 @@ func GetEventMessages(dev *onvif.Device, pullPointAddress string) ([]ONVIFEvents
 			}
 			requestBody, err := xml.Marshal(pullMessage)
 			if err != nil {
-				log.Log.Error("onvif.main.GetEventMessages(pullMessages): " + err.Error())
+				log.Error("onvif.main.GetEventMessages(pullMessages): " + err.Error())
 				return eventsArray, err
 			}
 			res, err := dev.SendSoap(string(subscriptionURI), string(requestBody))
 			if err != nil {
-				log.Log.Error("onvif.main.GetEventMessages(pullMessages): " + err.Error())
+				log.Error("onvif.main.GetEventMessages(pullMessages): " + err.Error())
 				return eventsArray, err
 			}
 
@@ -1169,11 +1151,11 @@ func GetEventMessages(dev *onvif.Device, pullPointAddress string) ([]ONVIFEvents
 					stringBody := string(bs)
 					decodedXML, et, err := getXMLNode(stringBody, "PullMessagesResponse")
 					if err != nil {
-						log.Log.Error("onvif.main.GetEventMessages(pullMessages): " + err.Error())
+						log.Error("onvif.main.GetEventMessages(pullMessages): " + err.Error())
 						return eventsArray, err
 					} else {
 						if err := decodedXML.DecodeElement(&pullMessagesResponse, et); err != nil {
-							log.Log.Error("onvif.main.GetEventMessages(pullMessages): " + err.Error())
+							log.Error("onvif.main.GetEventMessages(pullMessages): " + err.Error())
 							return eventsArray, err
 						}
 					}
@@ -1181,9 +1163,9 @@ func GetEventMessages(dev *onvif.Device, pullPointAddress string) ([]ONVIFEvents
 			}
 
 			for _, message := range pullMessagesResponse.NotificationMessage {
-				log.Log.Debug("onvif.main.GetEventMessages(pullMessages): " + string(message.Topic.TopicKinds))
+				log.Debug("onvif.main.GetEventMessages(pullMessages): " + string(message.Topic.TopicKinds))
 				//if len(message.Message.Message.Data.SimpleItem) > 0 {
-				//	log.Log.Debug("onvif.main.GetEventMessages(pullMessages): " + string(message.Message.Message.Data.SimpleItem[0].Name) + " " + string(message.Message.Message.Data.SimpleItem[0].Value))
+				// log.Debug("onvif.main.GetEventMessages(pullMessages): " + string(message.Message.Message.Data.SimpleItem[0].Name) + " " + string(message.Message.Message.Data.SimpleItem[0].Value))
 				//}
 				if message.Topic.TopicKinds == "tns1:Device/Trigger/Relay" ||
 					message.Topic.TopicKinds == "tns1:Device/tns1:Trigger/tns1:Relay" { // This is for avigilon cameras
@@ -1193,7 +1175,13 @@ func GetEventMessages(dev *onvif.Device, pullPointAddress string) ([]ONVIFEvents
 							key := string(message.Message.Message.Source.SimpleItem[0].Value)
 							value := string(message.Message.Message.Data.SimpleItem[0].Value)
 							propertyOperation := string(message.Message.Message.PropertyOperation)
-							log.Log.Debug("onvif.main.GetEventMessages(pullMessages) output: " + key + " " + value + " (" + propertyOperation + ")")
+							log.WithFields(log.Fields{
+								"component":          "onvif",
+								"event":              "event_property_output",
+								"property":           key,
+								"property_operation": propertyOperation,
+								"value":              value,
+							}).Debug("ONVIF event property output")
 
 							// Depending on the onvif library they might use different values for active and inactive.
 							if value == "active" || value == "1" {
@@ -1227,7 +1215,13 @@ func GetEventMessages(dev *onvif.Device, pullPointAddress string) ([]ONVIFEvents
 							key := string(message.Message.Message.Source.SimpleItem[0].Value)
 							value := string(message.Message.Message.Data.SimpleItem[0].Value)
 							propertyOperation := string(message.Message.Message.PropertyOperation)
-							log.Log.Debug("onvif.main.GetEventMessages(pullMessages) input: " + key + " " + value + " (" + propertyOperation + ")")
+							log.WithFields(log.Fields{
+								"component":          "onvif",
+								"event":              "event_property_input",
+								"property":           key,
+								"property_operation": propertyOperation,
+								"value":              value,
+							}).Debug("ONVIF event property input")
 
 							// Depending on the onvif library they might use different values for active and inactive.
 							if value == "active" || value == "1" {
@@ -1281,11 +1275,11 @@ func GetDigitalInputs(dev *onvif.Device) (device.GetDigitalInputsResponse, error
 			stringBody := string(b)
 			decodedXML, et, err := getXMLNode(stringBody, "GetDigitalInputsResponse")
 			if err != nil {
-				log.Log.Error("onvif.main.GetDigitalInputs(): " + err.Error())
+				log.Error("onvif.main.GetDigitalInputs(): " + err.Error())
 				return digitalinputs, err
 			} else {
 				if err := decodedXML.DecodeElement(&digitalinputs, et); err != nil {
-					log.Log.Debug("onvif.main.GetDigitalInputs(): " + err.Error())
+					log.Debug("onvif.main.GetDigitalInputs(): " + err.Error())
 					return digitalinputs, err
 				}
 			}
@@ -1312,11 +1306,11 @@ func GetRelayOutputs(dev *onvif.Device) (device.GetRelayOutputsResponse, error)
 		stringBody := string(b)
 		decodedXML, et, err := getXMLNode(stringBody, "GetRelayOutputsResponse")
 		if err != nil {
-			log.Log.Error("onvif.main.GetRelayOutputs(): " + err.Error())
+			log.Error("onvif.main.GetRelayOutputs(): " + err.Error())
 			return relayoutputs, err
 		} else {
 			if err := decodedXML.DecodeElement(&relayoutputs, et); err != nil {
-				log.Log.Debug("onvif.main.GetRelayOutputs(): " + err.Error())
+				log.Debug("onvif.main.GetRelayOutputs(): " + err.Error())
 				return relayoutputs, err
 			}
 		}
@@ -1325,42 +1319,79 @@ func GetRelayOutputs(dev *onvif.Device) (device.GetRelayOutputsResponse, error)
 	return relayoutputs, err
 }
 
-func TriggerRelayOutput(dev *onvif.Device, output string) (err error) {
-	err = nil
-
+func TriggerRelayOutput(dev *onvif.Device, output string) error {
 	// Get all outputs
-	relayoutputs, err := GetRelayOutputs(dev)
-
-	// For the moment we expect a single output
-	// However in theory there might be multiple outputs. We might need to change
-	// this in the future "kerberos-io/onvif" library.
-	if err == nil {
-		token := relayoutputs.RelayOutputs[0].Token
-		if output == string(token+"-output") {
-			outputState := device.SetRelayOutputState{
-				RelayOutputToken: token,
-				LogicalState:     "active",
-			}
-
-			resp, errResp := dev.CallMethod(outputState)
-			var b []byte
-			if errResp != nil {
-				b, err = io.ReadAll(resp.Body)
-				resp.Body.Close() // Ensure the response body is closed
-			}
-			stringBody := string(b)
-			if err == nil && resp.StatusCode == 200 {
-				log.Log.Info("onvif.main.TriggerRelayOutput(): triggered relay output (" + string(token) + ")")
-			} else {
-				log.Log.Error("onvif.main.TriggerRelayOutput(): " + stringBody)
-			}
-		} else {
-			log.Log.Error("onvif.main.TriggerRelayOutput(): could not find relay output (" + output + ")")
-		}
-	} else {
-		log.Log.Error("onvif.main.TriggerRelayOutput(): something went wrong while getting the relay outputs " + err.Error())
+	relayOutputs, err := GetRelayOutputs(dev)
+	if err != nil {
+		log.WithError(err).WithFields(log.Fields{
+			"component": "onvif",
+			"event":     "relay_outputs_query_failed",
+		}).Error("Failed to query ONVIF relay outputs")
+		return err
 	}
-	return
+	if len(relayOutputs.RelayOutputs) == 0 {
+		err := errors.New("ONVIF device has no relay outputs")
+		log.WithError(err).WithFields(log.Fields{
+			"component": "onvif",
+			"event":     "relay_output_unavailable",
+		}).Error("ONVIF relay output unavailable")
+		return err
+	}
+
+	// For the moment we expect a single output. Supporting multiple outputs
+	// requires matching the requested output against every returned token.
+	token := relayOutputs.RelayOutputs[0].Token
+	if output != string(token+"-output") {
+		err := errors.New("requested ONVIF relay output was not found")
+		log.WithError(err).WithFields(log.Fields{
+			"component": "onvif",
+			"event":     "relay_output_unavailable",
+		}).Error("ONVIF relay output unavailable")
+		return err
+	}
+
+	resp, err := dev.CallMethod(device.SetRelayOutputState{
+		RelayOutputToken: token,
+		LogicalState:     "active",
+	})
+	if err != nil {
+		log.WithError(err).WithFields(log.Fields{
+			"component": "onvif",
+			"event":     "relay_output_request_failed",
+		}).Error("Failed to trigger ONVIF relay output")
+		return err
+	}
+	if resp == nil {
+		err := errors.New("ONVIF relay request returned no response")
+		log.WithError(err).WithFields(log.Fields{
+			"component": "onvif",
+			"event":     "relay_output_request_failed",
+		}).Error("Failed to trigger ONVIF relay output")
+		return err
+	}
+	defer resp.Body.Close()
+	if _, err := io.Copy(io.Discard, resp.Body); err != nil {
+		log.WithError(err).WithFields(log.Fields{
+			"component": "onvif",
+			"event":     "relay_output_response_read_failed",
+		}).Warn("Failed to read ONVIF relay response")
+		return err
+	}
+	if resp.StatusCode != 200 {
+		err := errors.New("ONVIF relay request was rejected")
+		log.WithError(err).WithFields(log.Fields{
+			"component":   "onvif",
+			"event":       "relay_output_rejected",
+			"status_code": resp.StatusCode,
+		}).Error("ONVIF relay request rejected")
+		return err
+	}
+
+	log.WithFields(log.Fields{
+		"component": "onvif",
+		"event":     "relay_output_triggered",
+	}).Info("ONVIF relay output triggered")
+	return nil
 }
 
 func getXMLNode(xmlBody string, nodeName string) (*xml.Decoder, *xml.StartElement, error) {

machinery/src/onvif/main_test.go

@@ -0,0 +1,42 @@
+package onvif
+
+import (
+	"bytes"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func TestHandleONVIFResponseDoesNotLogBody(t *testing.T) {
+	logger := log.StandardLogger()
+	originalOutput := logger.Out
+	originalFormatter := logger.Formatter
+	originalLevel := logger.Level
+	defer func() {
+		logger.SetOutput(originalOutput)
+		logger.SetFormatter(originalFormatter)
+		logger.SetLevel(originalLevel)
+	}()
+
+	var output bytes.Buffer
+	logger.SetOutput(&output)
+	logger.SetFormatter(&log.JSONFormatter{})
+	logger.SetLevel(log.DebugLevel)
+
+	response := &http.Response{
+		StatusCode: http.StatusOK,
+		Body:       io.NopCloser(strings.NewReader(`{"credential":"do-not-log"}`)),
+	}
+	if err := handleONVIFResponse("test_operation", response, nil); err != nil {
+		t.Fatalf("handleONVIFResponse() error = %v", err)
+	}
+	if strings.Contains(output.String(), "do-not-log") {
+		t.Fatalf("ONVIF response log exposed response body: %q", output.String())
+	}
+	if !strings.Contains(output.String(), `"response_bytes":27`) {
+		t.Fatalf("ONVIF response log omitted response byte count: %q", output.String())
+	}
+}

machinery/src/outputs/main.go

@@ -1,8 +1,8 @@
 package outputs
 
 import (
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 type Output interface {
@@ -20,36 +20,36 @@ func Execute(message *models.OutputMessage) (err error) {
 			slack := &SlackOutput{}
 			err := slack.Trigger(message)
 			if err == nil {
-				log.Log.Debug("outputs.main.Execute(slack): message was processed by output.")
+				log.Debug("outputs.main.Execute(slack): message was processed by output.")
 			} else {
-				log.Log.Error("outputs.main.Execute(slack): " + err.Error())
+				log.Error("outputs.main.Execute(slack): " + err.Error())
 			}
 			break
 		case "webhook":
 			webhook := &WebhookOutput{}
 			err := webhook.Trigger(message)
 			if err == nil {
-				log.Log.Debug("outputs.main.Execute(webhook): message was processed by output.")
+				log.Debug("outputs.main.Execute(webhook): message was processed by output.")
 			} else {
-				log.Log.Error("outputs.main.Execute(webhook): " + err.Error())
+				log.Error("outputs.main.Execute(webhook): " + err.Error())
 			}
 			break
 		case "onvif_relay":
 			onvif := &OnvifRelayOutput{}
 			err := onvif.Trigger(message)
 			if err == nil {
-				log.Log.Debug("outputs.main.Execute(onvif): message was processed by output.")
+				log.Debug("outputs.main.Execute(onvif): message was processed by output.")
 			} else {
-				log.Log.Error("outputs.main.Execute(onvif): " + err.Error())
+				log.Error("outputs.main.Execute(onvif): " + err.Error())
 			}
 			break
 		case "script":
 			script := &ScriptOutput{}
 			err := script.Trigger(message)
 			if err == nil {
-				log.Log.Debug("outputs.main.Execute(script): message was processed by output.")
+				log.Debug("outputs.main.Execute(script): message was processed by output.")
 			} else {
-				log.Log.Error("outputs.main.Execute(script): " + err.Error())
+				log.Error("outputs.main.Execute(script): " + err.Error())
 			}
 			break
 		}

machinery/src/packets/queue.go

@@ -2,6 +2,7 @@
 package packets
 
 import (
+	"context"
 	"io"
 	"sync"
 )
@@ -201,7 +202,26 @@ func (self *QueueCursor) Streams() (streams []Stream, err error) {
 
 // ReadPacket will not consume packets in Queue, it's just a cursor.
 func (self *QueueCursor) ReadPacket() (pkt Packet, err error) {
+	return self.readPacket(context.Background(), false)
+}
+
+// ReadPacketContext waits for the next packet or returns when ctx is cancelled.
+func (self *QueueCursor) ReadPacketContext(ctx context.Context) (pkt Packet, err error) {
+	if err := ctx.Err(); err != nil {
+		return Packet{}, err
+	}
+	return self.readPacket(ctx, true)
+}
+
+func (self *QueueCursor) readPacket(ctx context.Context, cancellable bool) (pkt Packet, err error) {
 	self.que.cond.L.Lock()
+	var stop func() bool
+	defer func() {
+		self.que.cond.L.Unlock()
+		if stop != nil {
+			stop()
+		}
+	}()
 	buf := self.que.buf
 	if !self.gotpos {
 		self.pos = self.init(buf, self.que.videoidx)
@@ -222,8 +242,21 @@ func (self *QueueCursor) ReadPacket() (pkt Packet, err error) {
 			err = io.EOF
 			break
 		}
+		if cancellable {
+			if err = ctx.Err(); err != nil {
+				break
+			}
+			if stop == nil {
+				stop = context.AfterFunc(ctx, func() {
+					// The write lock prevents cancellation from being signalled
+					// between the context check above and cond.Wait.
+					self.que.lock.Lock()
+					self.que.cond.Broadcast()
+					self.que.lock.Unlock()
+				})
+			}
+		}
 		self.que.cond.Wait()
 	}
-	self.que.cond.L.Unlock()
 	return
 }

machinery/src/packets/queue_test.go

@@ -0,0 +1,109 @@
+package packets
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+type observedCancelContext struct {
+	done        chan struct{}
+	cancelled   atomic.Bool
+	errCalls    atomic.Int32
+	waiting     chan struct{}
+	waitingOnce sync.Once
+}
+
+func newObservedCancelContext() *observedCancelContext {
+	return &observedCancelContext{
+		done:    make(chan struct{}),
+		waiting: make(chan struct{}),
+	}
+}
+
+func (c *observedCancelContext) Deadline() (time.Time, bool) { return time.Time{}, false }
+func (c *observedCancelContext) Done() <-chan struct{}       { return c.done }
+func (c *observedCancelContext) Value(any) any               { return nil }
+
+func (c *observedCancelContext) Err() error {
+	if c.errCalls.Add(1) >= 2 {
+		c.waitingOnce.Do(func() { close(c.waiting) })
+	}
+	if c.cancelled.Load() {
+		return context.Canceled
+	}
+	return nil
+}
+
+func (c *observedCancelContext) Cancel() {
+	c.cancelled.Store(true)
+	close(c.done)
+}
+
+func TestQueueCursorReadPacketContextCancelsWhileQueueIsIdle(t *testing.T) {
+	queue := NewQueue()
+	cursor := queue.Latest()
+	ctx := newObservedCancelContext()
+	done := make(chan error, 1)
+
+	go func() {
+		_, err := cursor.ReadPacketContext(ctx)
+		done <- err
+	}()
+
+	select {
+	case <-ctx.waiting:
+	case <-time.After(time.Second):
+		t.Fatal("ReadPacketContext() did not reach the queue wait")
+	}
+	ctx.Cancel()
+	select {
+	case err := <-done:
+		if !errors.Is(err, context.Canceled) {
+			t.Fatalf("ReadPacketContext() error = %v, want %v", err, context.Canceled)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("ReadPacketContext() did not return after cancellation")
+	}
+}
+
+func TestQueueCursorReadPacketDoesNotAllocate(t *testing.T) {
+	queue := NewQueue()
+	for i := 0; i < 200; i++ {
+		if err := queue.WritePacket(Packet{Data: []byte{1}}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	cursor := queue.Oldest()
+
+	allocations := testing.AllocsPerRun(100, func() {
+		if _, err := cursor.ReadPacket(); err != nil {
+			t.Fatal(err)
+		}
+	})
+	if allocations != 0 {
+		t.Fatalf("ReadPacket() allocations = %.2f, want 0", allocations)
+	}
+}
+
+func TestQueueCursorReadPacketContextDoesNotAllocateWhenPacketIsAvailable(t *testing.T) {
+	queue := NewQueue()
+	for i := 0; i < 200; i++ {
+		if err := queue.WritePacket(Packet{Data: []byte{1}}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	cursor := queue.Oldest()
+
+	allocations := testing.AllocsPerRun(100, func() {
+		if _, err := cursor.ReadPacketContext(context.Background()); err != nil {
+			t.Fatal(err)
+		}
+	})
+	if allocations != 0 {
+		t.Fatalf("ReadPacketContext() allocations = %.2f, want 0", allocations)
+	}
+}

machinery/src/routers/http/health.go

@@ -0,0 +1,126 @@
+package http
+
+import (
+	"math"
+	stdhttp "net/http"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/utils"
+)
+
+const (
+	healthApplicationStatus = "get_success"
+	healthEntityStatus      = "healthy"
+	hubConnectionFreshness  = 3 * time.Minute
+)
+
+// StreamResolution is the most recently observed encoded video size in pixels.
+type StreamResolution struct {
+	Width  int64 `json:"width"`
+	Height int64 `json:"height"`
+}
+
+// StreamHealth describes the most recently observed state of one camera stream.
+type StreamHealth struct {
+	Configured        bool             `json:"configured"`
+	Connected         bool             `json:"connected"`
+	PackagesProcessed uint64           `json:"packagesProcessed"`
+	FPS               float64          `json:"fps"`
+	Resolution        StreamResolution `json:"resolution"`
+	LastPacketAt      int64            `json:"lastPacketAt"`
+}
+
+// HubHealth describes the Agent's periodic HTTP heartbeat connection to Hub.
+type HubHealth struct {
+	Configured                bool  `json:"configured"`
+	Connected                 bool  `json:"connected"`
+	LastHeartbeatAttemptAt    int64 `json:"lastHeartbeatAttemptAt"`
+	LastSuccessfulHeartbeatAt int64 `json:"lastSuccessfulHeartbeatAt"`
+}
+
+// Health describes the Agent process health exposed to API clients.
+type Health struct {
+	Description     string       `json:"description"`
+	CameraConnected bool         `json:"cameraConnected"`
+	MainStream      StreamHealth `json:"mainStream"`
+	SubStream       StreamHealth `json:"subStream"`
+	Hub             HubHealth    `json:"hub"`
+}
+
+// HealthResponseData contains the typed payload of a health response.
+type HealthResponseData struct {
+	Health Health `json:"health"`
+}
+
+// HealthResponse is the standard successful response returned by GET /health.
+type HealthResponse struct {
+	SuccessResponse
+	Data HealthResponseData `json:"data"`
+}
+
+// HealthCheck godoc
+// @Summary Check Agent health
+// @Description Confirms that the Agent HTTP process can serve requests and reports current camera stream and Hub heartbeat diagnostics. Operational dependency failures do not change the liveness HTTP status.
+// @Tags health
+// @Produce json
+// @Success 200 {object} HealthResponse
+// @Router /health [get]
+func HealthCheck(c *gin.Context, communication *models.Communication) {
+	c.Header("Cache-Control", "no-store")
+	c.JSON(stdhttp.StatusOK, HealthResponse{
+		SuccessResponse: NewSuccessResponse(
+			stdhttp.StatusOK,
+			healthApplicationStatus,
+			healthEntityStatus,
+			"Healthy",
+			ResponseMetadata{
+				ApplicationName:    "agent",
+				ApplicationVersion: utils.VERSION,
+				Path:               c.Request.URL.Path,
+			},
+		),
+		Data: HealthResponseData{
+			Health: buildHealth(communication, time.Now()),
+		},
+	})
+}
+
+func buildHealth(communication *models.Communication, now time.Time) Health {
+	mainTelemetry := communication.StreamRuntimeTelemetry(models.MainStream)
+	subTelemetry := communication.StreamRuntimeTelemetry(models.SubStream)
+	hubTelemetry := communication.HubRuntimeTelemetry()
+
+	hubConnected := hubTelemetry.Configured &&
+		hubTelemetry.Connected &&
+		hubTelemetry.LastSuccessfulHeartbeatAt > 0 &&
+		now.Unix()-hubTelemetry.LastSuccessfulHeartbeatAt <= int64(hubConnectionFreshness/time.Second)
+
+	return Health{
+		Description:     "Agent HTTP service is healthy",
+		CameraConnected: communication.CameraConnected.Load(),
+		MainStream:      newStreamHealth(mainTelemetry, communication.MainStreamConnected.Load()),
+		SubStream:       newStreamHealth(subTelemetry, communication.SubStreamConnected.Load()),
+		Hub: HubHealth{
+			Configured:                hubTelemetry.Configured,
+			Connected:                 hubConnected,
+			LastHeartbeatAttemptAt:    hubTelemetry.LastHeartbeatAttemptAt,
+			LastSuccessfulHeartbeatAt: hubTelemetry.LastSuccessfulHeartbeatAt,
+		},
+	}
+}
+
+func newStreamHealth(telemetry models.StreamRuntimeTelemetry, connected bool) StreamHealth {
+	return StreamHealth{
+		Configured:        telemetry.Configured,
+		Connected:         connected,
+		PackagesProcessed: telemetry.PackagesProcessed,
+		FPS:               math.Round(telemetry.FPS*100) / 100,
+		Resolution: StreamResolution{
+			Width:  telemetry.Width,
+			Height: telemetry.Height,
+		},
+		LastPacketAt: telemetry.LastPacketAt,
+	}
+}

machinery/src/routers/http/health_test.go

@@ -0,0 +1,117 @@
+package http
+
+import (
+	"encoding/json"
+	stdhttp "net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	jwt "github.com/appleboy/gin-jwt/v2"
+	"github.com/gin-gonic/gin"
+	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/utils"
+)
+
+func TestHealthCheckReturnsStandardPublicResponse(t *testing.T) {
+	gin.SetMode(gin.TestMode)
+
+	middlewareConfig := JWTMiddleWare()
+	authMiddleware, err := jwt.New(&middlewareConfig)
+	if err != nil {
+		t.Fatalf("initialize JWT middleware: %v", err)
+	}
+
+	now := time.Now()
+	communication := &models.Communication{}
+	communication.SetStreamConfigured(models.MainStream, true)
+	communication.SetStreamConfigured(models.SubStream, true)
+	communication.RecordStreamPackage(models.MainStream, 29.969, 1920, 1080, now.Add(-2*time.Second))
+	communication.RecordStreamPackage(models.MainStream, 29.969, 1920, 1080, now.Add(-time.Second))
+	communication.RecordStreamPackage(models.SubStream, 15, 640, 360, now.Add(-time.Second))
+	communication.CameraConnected.Store(true)
+	communication.MainStreamConnected.Store(true)
+	communication.SubStreamConnected.Store(true)
+	communication.SetHubConfigured(true)
+	communication.RecordHubHeartbeatAttempt(now.Add(-2 * time.Second))
+	communication.RecordHubHeartbeatSuccess(now.Add(-time.Second))
+
+	router := gin.New()
+	AddRoutes(router, authMiddleware, "", nil, communication, nil)
+
+	response := httptest.NewRecorder()
+	request := httptest.NewRequest(stdhttp.MethodGet, "/health", nil)
+	router.ServeHTTP(response, request)
+
+	if response.Code != stdhttp.StatusOK {
+		t.Fatalf("status = %d, want %d; body=%s", response.Code, stdhttp.StatusOK, response.Body.String())
+	}
+	if got := response.Header().Get("Cache-Control"); got != "no-store" {
+		t.Fatalf("Cache-Control = %q, want %q", got, "no-store")
+	}
+
+	var body HealthResponse
+	if err := json.Unmarshal(response.Body.Bytes(), &body); err != nil {
+		t.Fatalf("decode health response: %v; body=%s", err, response.Body.String())
+	}
+
+	if body.HTTPStatusCode != stdhttp.StatusOK {
+		t.Errorf("httpStatusCode = %d, want %d", body.HTTPStatusCode, stdhttp.StatusOK)
+	}
+	if body.ApplicationStatusCode != healthApplicationStatus {
+		t.Errorf("applicationStatusCode = %q, want %q", body.ApplicationStatusCode, healthApplicationStatus)
+	}
+	if body.EntityStatusCode != healthEntityStatus {
+		t.Errorf("entityStatusCode = %q, want %q", body.EntityStatusCode, healthEntityStatus)
+	}
+	if body.Message != "Healthy" {
+		t.Errorf("message = %q, want %q", body.Message, "Healthy")
+	}
+	if body.Metadata.ApplicationName != "agent" {
+		t.Errorf("metadata.applicationName = %q, want %q", body.Metadata.ApplicationName, "agent")
+	}
+	if body.Metadata.ApplicationVersion != utils.VERSION {
+		t.Errorf("metadata.applicationVersion = %q, want %q", body.Metadata.ApplicationVersion, utils.VERSION)
+	}
+	if body.Metadata.Timestamp <= 0 {
+		t.Errorf("metadata.timestamp = %d, want a positive Unix timestamp", body.Metadata.Timestamp)
+	}
+	if body.Metadata.Path != "/health" {
+		t.Errorf("metadata.path = %q, want %q", body.Metadata.Path, "/health")
+	}
+	health := body.Data.Health
+	if health.Description != "Agent HTTP service is healthy" {
+		t.Errorf("data.health.description = %q, want %q", health.Description, "Agent HTTP service is healthy")
+	}
+	if !health.CameraConnected {
+		t.Error("data.health.cameraConnected = false, want true")
+	}
+	if health.MainStream.PackagesProcessed != 2 || health.MainStream.FPS != 29.97 {
+		t.Errorf("data.health.mainStream = %+v, want 2 packages at 29.97 FPS", health.MainStream)
+	}
+	if health.MainStream.Resolution != (StreamResolution{Width: 1920, Height: 1080}) {
+		t.Errorf("data.health.mainStream.resolution = %+v", health.MainStream.Resolution)
+	}
+	if health.SubStream.PackagesProcessed != 1 || health.SubStream.FPS != 15 {
+		t.Errorf("data.health.subStream = %+v, want 1 package at 15 FPS", health.SubStream)
+	}
+	if health.SubStream.Resolution != (StreamResolution{Width: 640, Height: 360}) {
+		t.Errorf("data.health.subStream.resolution = %+v", health.SubStream.Resolution)
+	}
+	if !health.Hub.Configured || !health.Hub.Connected {
+		t.Errorf("data.health.hub = %+v, want configured and connected", health.Hub)
+	}
+}
+
+func TestBuildHealthMarksStaleHubHeartbeatDisconnected(t *testing.T) {
+	now := time.Unix(1_788_710_500, 0)
+	communication := &models.Communication{}
+	communication.SetHubConfigured(true)
+	communication.RecordHubHeartbeatAttempt(now.Add(-4 * time.Minute))
+	communication.RecordHubHeartbeatSuccess(now.Add(-4 * time.Minute))
+
+	health := buildHealth(communication, now)
+	if health.Hub.Connected {
+		t.Fatalf("Hub health = %+v, want stale heartbeat to be disconnected", health.Hub)
+	}
+}

machinery/src/routers/http/methods.go

@@ -6,9 +6,9 @@ import (
 	"time"
 
 	"github.com/gin-gonic/gin"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/onvif"
+	log "github.com/sirupsen/logrus"
 )
 
 // Login godoc
@@ -598,27 +598,27 @@ func DoTriggerRelayOutput(c *gin.Context) {
 			err := onvif.TriggerRelayOutput(device, output)
 			if err == nil {
 				msg := "relay output triggered: " + output
-				log.Log.Info("routers.http.methods.DoTriggerRelayOutput(): " + msg)
+				log.Info("routers.http.methods.DoTriggerRelayOutput(): " + msg)
 				c.JSON(200, gin.H{
 					"data": msg,
 				})
 			} else {
 				msg := "something went wrong: " + err.Error()
-				log.Log.Error("routers.http.methods.DoTriggerRelayOutput(): " + msg)
+				log.Error("routers.http.methods.DoTriggerRelayOutput(): " + msg)
 				c.JSON(400, gin.H{
 					"data": msg,
 				})
 			}
 		} else {
 			msg := "something went wrong: " + err.Error()
-			log.Log.Error("routers.http.methods.DoTriggerRelayOutput(): " + msg)
+			log.Error("routers.http.methods.DoTriggerRelayOutput(): " + msg)
 			c.JSON(400, gin.H{
 				"data": msg,
 			})
 		}
 	} else {
 		msg := "something went wrong: " + err.Error()
-		log.Log.Error("routers.http.methods.DoTriggerRelayOutput(): " + msg)
+		log.Error("routers.http.methods.DoTriggerRelayOutput(): " + msg)
 		c.JSON(400, gin.H{
 			"data": msg,
 		})

machinery/src/routers/http/response.go

@@ -0,0 +1,41 @@
+package http
+
+import "time"
+
+// ResponseMetadata contains request and application context shared by API responses.
+type ResponseMetadata struct {
+	ApplicationName    string `json:"applicationName,omitempty"`
+	ApplicationVersion string `json:"applicationVersion,omitempty"`
+	Timestamp          int64  `json:"timestamp,omitempty"`
+	Path               string `json:"path,omitempty"`
+	TraceID            string `json:"traceId,omitempty"`
+}
+
+// SuccessResponse is the common envelope for successful Agent API responses.
+type SuccessResponse struct {
+	HTTPStatusCode        int              `json:"httpStatusCode"`
+	ApplicationStatusCode string           `json:"applicationStatusCode"`
+	EntityStatusCode      string           `json:"entityStatusCode"`
+	Message               string           `json:"message"`
+	Metadata              ResponseMetadata `json:"metadata"`
+}
+
+// NewSuccessResponse creates the Hub-compatible success envelope used by new
+// Agent API endpoints.
+func NewSuccessResponse(
+	httpStatusCode int,
+	applicationStatusCode string,
+	entityStatusCode string,
+	message string,
+	metadata ResponseMetadata,
+) SuccessResponse {
+	metadata.Timestamp = time.Now().Unix()
+
+	return SuccessResponse{
+		HTTPStatusCode:        httpStatusCode,
+		ApplicationStatusCode: applicationStatusCode,
+		EntityStatusCode:      entityStatusCode,
+		Message:               message,
+		Metadata:              metadata,
+	}
+}

machinery/src/routers/http/routes.go

@@ -14,6 +14,10 @@ import (
 
 func AddRoutes(r *gin.Engine, authMiddleware *jwt.GinJWTMiddleware, configDirectory string, configuration *models.Configuration, communication *models.Communication, captureDevice *capture.Capture) *gin.RouterGroup {
 
+	r.GET("/health", func(c *gin.Context) {
+		HealthCheck(c, communication)
+	})
+
 	r.GET("/ws", func(c *gin.Context) {
 		websocket.WebsocketHandler(c, configuration, communication, captureDevice)
 	})

machinery/src/routers/http/server.go

@@ -1,9 +1,11 @@
 package http
 
 import (
+	"fmt"
 	"io"
 	"os"
 	"strconv"
+	"time"
 
 	jwt "github.com/appleboy/gin-jwt/v2"
 	"github.com/gin-contrib/pprof"
@@ -11,7 +13,7 @@ import (
 	"github.com/gin-gonic/gin"
 
 	//Swagger documentantion
-	"log"
+	log "github.com/sirupsen/logrus"
 
 	_ "github.com/kerberos-io/agent/machinery/docs"
 	"github.com/kerberos-io/agent/machinery/src/capture"
@@ -45,7 +47,17 @@ func StartServer(configDirectory string, configuration *models.Configuration, co
 	gin.SetMode(gin.ReleaseMode)
 
 	// Initialize REST API
-	r := gin.Default()
+	r := gin.New()
+	r.Use(requestLogger(), gin.CustomRecovery(func(c *gin.Context, recovered interface{}) {
+		log.WithFields(log.Fields{
+			"component": "http",
+			"event":     "request_panic",
+			"method":    c.Request.Method,
+			"path":      c.Request.URL.Path,
+			"panic":     fmt.Sprint(recovered),
+		}).Error("HTTP request panicked")
+		c.AbortWithStatus(500)
+	}))
 
 	// Profiler
 	pprof.Register(r)
@@ -60,7 +72,10 @@ func StartServer(configDirectory string, configuration *models.Configuration, co
 	middleWare := JWTMiddleWare()
 	authMiddleware, err := jwt.New(&middleWare)
 	if err != nil {
-		log.Fatal("JWT Error:" + err.Error())
+		log.WithError(err).WithFields(log.Fields{
+			"component": "http",
+			"event":     "jwt_initialization_failed",
+		}).Fatal("Failed to initialize JWT middleware")
 	}
 
 	// Add all routes
@@ -73,7 +88,10 @@ func StartServer(configDirectory string, configuration *models.Configuration, co
 		// Move demo environment variables to environment variables
 		err := os.Rename(demoEnvironmentVariables, environmentVariables)
 		if err != nil {
-			log.Fatal(err)
+			log.WithError(err).WithFields(log.Fields{
+				"component": "http",
+				"event":     "demo_environment_move_failed",
+			}).Fatal("Failed to activate demo UI environment")
 		}
 	}
 
@@ -88,9 +106,54 @@ func StartServer(configDirectory string, configuration *models.Configuration, co
 	})
 
 	// Run the api on port
+	log.WithFields(log.Fields{
+		"component": "http",
+		"event":     "server_starting",
+		"port":      configuration.Port,
+	}).Info("HTTP server starting")
 	err = r.Run(":" + configuration.Port)
 	if err != nil {
-		log.Fatal(err)
+		log.WithError(err).WithFields(log.Fields{
+			"component": "http",
+			"event":     "server_failed",
+			"port":      configuration.Port,
+		}).Fatal("HTTP server stopped")
+	}
+}
+
+func requestLogger() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		startedAt := time.Now()
+		c.Next()
+
+		fields := log.Fields{
+			"component":      "http",
+			"duration_ms":    time.Since(startedAt).Milliseconds(),
+			"event":          "request_completed",
+			"method":         c.Request.Method,
+			"path":           c.Request.URL.Path,
+			"response_bytes": c.Writer.Size(),
+			"status":         c.Writer.Status(),
+		}
+		if log.IsLevelEnabled(log.DebugLevel) {
+			fields["client_ip"] = c.ClientIP()
+			fields["user_agent"] = c.Request.UserAgent()
+		}
+		entry := log.WithFields(fields)
+		if len(c.Errors) > 0 {
+			entry = entry.WithField("request_errors", c.Errors.String())
+		}
+
+		switch status := c.Writer.Status(); {
+		case status >= 500:
+			entry.Error("HTTP request completed")
+		case status >= 400:
+			entry.Warn("HTTP request completed")
+		case c.Request.URL.Path == "/health":
+			entry.Debug("HTTP request completed")
+		default:
+			entry.Info("HTTP request completed")
+		}
 	}
 }
 

machinery/src/routers/http/server_test.go

@@ -0,0 +1,89 @@
+package http
+
+import (
+	"bytes"
+	"encoding/json"
+	stdhttp "net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/gin-gonic/gin"
+	log "github.com/sirupsen/logrus"
+)
+
+func TestRequestLoggerEmitsStructuredFields(t *testing.T) {
+	originalOutput := log.StandardLogger().Out
+	originalFormatter := log.StandardLogger().Formatter
+	originalLevel := log.GetLevel()
+	defer func() {
+		log.SetOutput(originalOutput)
+		log.SetFormatter(originalFormatter)
+		log.SetLevel(originalLevel)
+	}()
+
+	var output bytes.Buffer
+	log.SetOutput(&output)
+	log.SetFormatter(&log.JSONFormatter{})
+	log.SetLevel(log.InfoLevel)
+	gin.SetMode(gin.TestMode)
+
+	router := gin.New()
+	router.Use(requestLogger())
+	router.GET("/status", func(c *gin.Context) {
+		c.Status(stdhttp.StatusNoContent)
+	})
+
+	response := httptest.NewRecorder()
+	request := httptest.NewRequest(stdhttp.MethodGet, "/status?token=do-not-log", nil)
+	router.ServeHTTP(response, request)
+
+	var entry map[string]interface{}
+	if err := json.Unmarshal(output.Bytes(), &entry); err != nil {
+		t.Fatalf("decode request log: %v; output=%q", err, output.String())
+	}
+	for key, want := range map[string]interface{}{
+		"component": "http",
+		"event":     "request_completed",
+		"method":    stdhttp.MethodGet,
+		"path":      "/status",
+		"status":    float64(stdhttp.StatusNoContent),
+	} {
+		if got := entry[key]; got != want {
+			t.Fatalf("%s = %v, want %v", key, got, want)
+		}
+	}
+	if bytes.Contains(output.Bytes(), []byte("do-not-log")) {
+		t.Fatalf("request log exposed query parameters: %q", output.String())
+	}
+}
+
+func TestRequestLoggerKeepsSuccessfulHealthChecksBelowInfo(t *testing.T) {
+	originalOutput := log.StandardLogger().Out
+	originalFormatter := log.StandardLogger().Formatter
+	originalLevel := log.GetLevel()
+	defer func() {
+		log.SetOutput(originalOutput)
+		log.SetFormatter(originalFormatter)
+		log.SetLevel(originalLevel)
+	}()
+
+	var output bytes.Buffer
+	log.SetOutput(&output)
+	log.SetFormatter(&log.JSONFormatter{})
+	log.SetLevel(log.InfoLevel)
+	gin.SetMode(gin.TestMode)
+
+	router := gin.New()
+	router.Use(requestLogger())
+	router.GET("/health", func(c *gin.Context) {
+		c.Status(stdhttp.StatusNoContent)
+	})
+
+	response := httptest.NewRecorder()
+	request := httptest.NewRequest(stdhttp.MethodGet, "/health", nil)
+	router.ServeHTTP(response, request)
+
+	if output.Len() != 0 {
+		t.Fatalf("successful health check emitted an info log: %q", output.String())
+	}
+}

machinery/src/routers/mqtt/main.go

@@ -20,10 +20,10 @@ import (
 	"github.com/kerberos-io/agent/machinery/src/capture"
 	configService "github.com/kerberos-io/agent/machinery/src/config"
 	"github.com/kerberos-io/agent/machinery/src/encryption"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/onvif"
 	"github.com/kerberos-io/agent/machinery/src/webrtc"
+	log "github.com/sirupsen/logrus"
 )
 
 // We'll cache the MQTT settings to know if we need to reinitialize the MQTT client connection.
@@ -41,7 +41,7 @@ func HasMQTTClientModified(configuration *models.Configuration) bool {
 	HubKey := configuration.Config.HubKey
 	AgentKey := configuration.Config.Key
 	if PREV_MQTTURI != MTTURI || PREV_MQTTUsername != MTTUsername || PREV_MQTTPassword != MQTTPassword || PREV_HubKey != HubKey || PREV_AgentKey != AgentKey {
-		log.Log.Info("HasMQTTClientModified: MQTT settings have been modified, restarting MQTT client.")
+		log.Info("HasMQTTClientModified: MQTT settings have been modified, restarting MQTT client.")
 		return true
 	}
 	return false
@@ -70,8 +70,25 @@ func ConfigureMQTT(configDirectory string, configuration *models.Configuration,
 	PREV_AgentKey = configuration.Config.Key
 
 	if config.Offline == "true" {
-		log.Log.Info("routers.mqtt.main.ConfigureMQTT(): not starting as running in Offline mode.")
+		log.Info("routers.mqtt.main.ConfigureMQTT(): not starting as running in Offline mode.")
 	} else {
+		hubKey := ""
+		if config.Cloud == "s3" && config.S3 != nil && config.S3.Publickey != "" {
+			hubKey = config.S3.Publickey
+		} else if config.Cloud == "kstorage" && config.KStorage != nil && config.KStorage.CloudKey != "" {
+			hubKey = config.KStorage.CloudKey
+		}
+		if config.HubKey != "" {
+			hubKey = config.HubKey
+		}
+		if hubKey == "" {
+			log.Warn("routers.mqtt.main.ConfigureMQTT(): not starting without a Hub key")
+			return nil
+		}
+		if config.Key == "" {
+			log.Warn("routers.mqtt.main.ConfigureMQTT(): not starting without an Agent key")
+			return nil
+		}
 
 		opts := mqtt.NewClientOptions()
 
@@ -79,7 +96,6 @@ func ConfigureMQTT(configDirectory string, configuration *models.Configuration,
 		// and share and receive messages to/from.
 		mqttURL := config.MQTTURI
 		opts.AddBroker(mqttURL)
-		log.Log.Debug("routers.mqtt.main.ConfigureMQTT(): Set broker uri " + mqttURL)
 
 		// Our MQTT broker can have username/password credentials
 		// to protect it from the outside.
@@ -88,9 +104,14 @@ func ConfigureMQTT(configDirectory string, configuration *models.Configuration,
 		if mqtt_username != "" || mqtt_password != "" {
 			opts.SetUsername(mqtt_username)
 			opts.SetPassword(mqtt_password)
-			log.Log.Debug("routers.mqtt.main.ConfigureMQTT(): Set username " + mqtt_username)
-			log.Log.Debug("routers.mqtt.main.ConfigureMQTT(): Set password " + mqtt_password)
 		}
+		log.WithFields(log.Fields{
+			"broker_configured":   mqttURL != "",
+			"component":           "routers/mqtt",
+			"event":               "credentials_configured",
+			"password_configured": mqtt_password != "",
+			"username_configured": mqtt_username != "",
+		}).Debug("MQTT connection configuration loaded")
 
 		// Some extra options to make sure the connection behaves
 		// properly. More information here: github.com/eclipse/paho.mqtt.golang.
@@ -108,65 +129,81 @@ func ConfigureMQTT(configDirectory string, configuration *models.Configuration,
 		opts.SetOrderMatters(false)
 		opts.SetConnectTimeout(30 * time.Second)
 		opts.SetConnectionLostHandler(func(client mqtt.Client, err error) {
+			entry := log.WithFields(log.Fields{
+				"component": "routers/mqtt",
+				"event":     "connection_lost",
+			})
 			if err != nil {
-				log.Log.Error("routers.mqtt.main.ConfigureMQTT(): MQTT connection lost: " + err.Error())
-			} else {
-				log.Log.Error("routers.mqtt.main.ConfigureMQTT(): MQTT connection lost")
+				entry = entry.WithError(err)
 			}
+			entry.Error("MQTT connection lost")
 		})
 		opts.SetReconnectingHandler(func(client mqtt.Client, options *mqtt.ClientOptions) {
-			log.Log.Warning("routers.mqtt.main.ConfigureMQTT(): reconnecting to MQTT broker")
+			log.WithFields(log.Fields{
+				"component": "routers/mqtt",
+				"event":     "reconnecting",
+			}).Warn("Reconnecting to MQTT broker")
 		})
 		opts.SetOnConnectHandler(func(c mqtt.Client) {
-			log.Log.Info("routers.mqtt.main.ConfigureMQTT(): MQTT session is online")
+			log.WithFields(log.Fields{
+				"component": "routers/mqtt",
+				"event":     "session_online",
+			}).Info("MQTT session is online")
 		})
 
-		hubKey := ""
-		// This is the old way ;)
-		if config.Cloud == "s3" && config.S3 != nil && config.S3.Publickey != "" {
-			hubKey = config.S3.Publickey
-		} else if config.Cloud == "kstorage" && config.KStorage != nil && config.KStorage.CloudKey != "" {
-			hubKey = config.KStorage.CloudKey
-		}
-		// This is the new way ;)
-		if config.HubKey != "" {
-			hubKey = config.HubKey
+		rand.Seed(time.Now().UnixNano())
+		random := rand.Intn(100)
+		mqttClientID := config.Key + strconv.Itoa(random) // this random int is to avoid conflicts.
+
+		// This is a worked-around.
+		// current S3 (Kerberos Hub SAAS) is using a secured MQTT, where the client id,
+		// should match the kerberos hub key.
+		if config.Cloud == "s3" {
+			mqttClientID = config.Key
 		}
 
-		if hubKey != "" {
+		opts.SetClientID(mqttClientID)
+		log.WithFields(log.Fields{
+			"component":            "routers/mqtt",
+			"event":                "client_configured",
+			"randomized_client_id": config.Cloud != "s3",
+		}).Debug("MQTT client configured")
+		rand.Seed(time.Now().UnixNano())
 
-			rand.Seed(time.Now().UnixNano())
-			random := rand.Intn(100)
-			mqttClientID := config.Key + strconv.Itoa(random) // this random int is to avoid conflicts.
+		opts.OnConnect = func(c mqtt.Client) {
+			// We managed to connect to the MQTT broker, hurray!
+			log.WithFields(log.Fields{
+				"component": "routers/mqtt",
+				"event":     "connected",
+			}).Info("MQTT broker connected")
+			log.WithFields(log.Fields{
+				"broker_configured": mqttURL != "",
+				"component":         "routers/mqtt",
+				"event":             "connection_details",
+			}).Debug("MQTT connection details")
 
-			// This is a worked-around.
-			// current S3 (Kerberos Hub SAAS) is using a secured MQTT, where the client id,
-			// should match the kerberos hub key.
-			if config.Cloud == "s3" {
-				mqttClientID = config.Key
-			}
-
-			opts.SetClientID(mqttClientID)
-			log.Log.Info("routers.mqtt.main.ConfigureMQTT(): Set ClientID " + mqttClientID)
-			rand.Seed(time.Now().UnixNano())
-
-			opts.OnConnect = func(c mqtt.Client) {
-				// We managed to connect to the MQTT broker, hurray!
-				log.Log.Info("routers.mqtt.main.ConfigureMQTT(): " + mqttClientID + " connected to " + mqttURL)
-
-				// Create a susbcription for listen and reply
-				MQTTListenerHandler(c, hubKey, configDirectory, configuration, communication)
-			}
+			// Create a susbcription for listen and reply
+			MQTTListenerHandler(c, hubKey, configDirectory, configuration, communication)
 		}
 		mqc := mqtt.NewClient(opts)
 		if token := mqc.Connect(); token.WaitTimeout(30 * time.Second) {
 			if token.Error() != nil {
-				log.Log.Error("routers.mqtt.main.ConfigureMQTT(): unable to establish mqtt broker connection, error was: " + token.Error().Error())
+				log.WithError(token.Error()).WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "initial_connection_failed",
+				}).Error("Failed to establish initial MQTT connection")
 			} else {
-				log.Log.Info("routers.mqtt.main.ConfigureMQTT(): initial MQTT connection established")
+				log.WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "initial_connection_established",
+				}).Info("Initial MQTT connection established")
 			}
 		} else {
-			log.Log.Error("routers.mqtt.main.ConfigureMQTT(): timed out while establishing mqtt broker connection")
+			log.WithFields(log.Fields{
+				"component":  "routers/mqtt",
+				"event":      "initial_connection_timeout",
+				"timeout_ms": (30 * time.Second).Milliseconds(),
+			}).Error("Timed out establishing initial MQTT connection")
 		}
 		return mqc
 	}
@@ -216,7 +253,7 @@ func markHDSessionSeen(sessionID string) bool {
 
 func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory string, configuration *models.Configuration, communication *models.Communication) {
 	if hubKey == "" {
-		log.Log.Info("routers.mqtt.main.MQTTListenerHandler(): no hub key provided, not subscribing to kerberos/hub/{hubkey}")
+		log.Info("routers.mqtt.main.MQTTListenerHandler(): no hub key provided, not subscribing to kerberos/hub/{hubkey}")
 	} else {
 		agentListener := fmt.Sprintf("kerberos/agent/%s", hubKey)
 		token := mqttClient.Subscribe(agentListener, 1, func(c mqtt.Client, msg mqtt.Message) {
@@ -249,13 +286,13 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 							}
 							visibleValue, err := encryption.AesDecrypt(data, privateKey)
 							if err != nil {
-								log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message: " + err.Error())
+								log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message: " + err.Error())
 								return
 							}
 							json.Unmarshal(visibleValue, &payload)
 							message.Payload = payload
 						} else {
-							log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message, no private key provided.")
+							log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message, no private key provided.")
 						}
 					}
 				}
@@ -271,14 +308,14 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 						pemBytes, _ := ioutil.ReadAll(r)
 						block, _ := pem.Decode(pemBytes)
 						if block == nil {
-							log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error decoding PEM block containing private key")
+							log.Error("routers.mqtt.main.MQTTListenerHandler(): error decoding PEM block containing private key")
 							return
 						} else {
 							// Parse private key
 							b := block.Bytes
 							key, err := x509.ParsePKCS8PrivateKey(b)
 							if err != nil {
-								log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error parsing private key: " + err.Error())
+								log.Error("routers.mqtt.main.MQTTListenerHandler(): error parsing private key: " + err.Error())
 								return
 							} else {
 								// Conver key to *rsa.PrivateKey
@@ -298,16 +335,16 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 										}
 										decryptedValue, err := encryption.AesDecrypt(data, string(decryptedKey))
 										if err != nil {
-											log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message: " + err.Error())
+											log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message: " + err.Error())
 											return
 										}
 										json.Unmarshal(decryptedValue, &payload)
 									} else {
-										log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message, assymetric keys do not match.")
+										log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message, assymetric keys do not match.")
 										return
 									}
 								} else if err != nil {
-									log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message: " + err.Error())
+									log.Error("routers.mqtt.main.MQTTListenerHandler(): error decrypting message: " + err.Error())
 									return
 								}
 							}
@@ -318,7 +355,11 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 				}
 
 				// We'll find out which message we received, and act accordingly.
-				log.Log.Info("routers.mqtt.main.MQTTListenerHandler(): received message with action: " + payload.Action)
+				log.WithFields(log.Fields{
+					"action":    payload.Action,
+					"component": "routers/mqtt",
+					"event":     "message_received",
+				}).Debug("MQTT message received")
 
 				// NOTE: We intentionally do NOT discard request-hd-stream /
 				// receive-hd-candidates messages based on a wall-clock age. The
@@ -362,12 +403,15 @@ func MQTTListenerHandler(mqttClient mqtt.Client, hubKey string, configDirectory
 
 		if token.WaitTimeout(10 * time.Second) {
 			if token.Error() != nil {
-				log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): failed to subscribe to " + agentListener + ": " + token.Error().Error())
+				log.WithError(token.Error()).WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "subscription_failed",
+				}).Error("Failed to subscribe to Agent MQTT topic")
 			} else {
-				log.Log.Info("routers.mqtt.main.MQTTListenerHandler(): subscribed to " + agentListener)
+				log.Info("routers.mqtt.main.MQTTListenerHandler(): subscribed to " + agentListener)
 			}
 		} else {
-			log.Log.Error("routers.mqtt.main.MQTTListenerHandler(): timed out while subscribing to " + agentListener)
+			log.Error("routers.mqtt.main.MQTTListenerHandler(): timed out while subscribing to " + agentListener)
 		}
 	}
 }
@@ -396,9 +440,9 @@ func HandleRecording(mqttClient mqtt.Client, hubKey string, payload models.Paylo
 			if communication.IsRecordingManual.IsSet() {
 				communication.RecordingManualHeartbeat.Store(now)
 				communication.RecordingManualHeartbeatSeen.Set()
-				log.Log.Debug("routers.mqtt.main.HandleRecording(): manual recording heartbeat received.")
+				log.Debug("routers.mqtt.main.HandleRecording(): manual recording heartbeat received.")
 			} else {
-				log.Log.Debug("routers.mqtt.main.HandleRecording(): ignoring heartbeat, no active manual recording.")
+				log.Debug("routers.mqtt.main.HandleRecording(): ignoring heartbeat, no active manual recording.")
 			}
 		} else {
 			// Explicit start from the live view (record button). Start a manual
@@ -410,18 +454,16 @@ func HandleRecording(mqttClient mqtt.Client, hubKey string, payload models.Paylo
 			if communication.IsRecordingManual.SetToIf(false, true) {
 				communication.RecordingManualStart.Store(now)
 				communication.RecordingManualHeartbeatSeen.UnSet()
-				log.Log.Info("routers.mqtt.main.HandleRecording(): manual recording started.")
-				select {
-				case communication.HandleMotion <- models.MotionDataPartial{Timestamp: timestamp, NumberOfChanges: 100000000}:
-				default:
-					log.Log.Warning("routers.mqtt.main.HandleRecording(): motion channel full, manual recording start not queued.")
+				log.Info("routers.mqtt.main.HandleRecording(): manual recording started.")
+				if !communication.TrySendMotion(models.MotionDataPartial{Timestamp: timestamp, NumberOfChanges: 100000000}) {
+					log.Warn("routers.mqtt.main.HandleRecording(): motion channel full, manual recording start not queued.")
 				}
 			}
 		}
 	} else {
 		// Stop the manual recording; the motion recorder closes the clip once the
 		// post-recording window elapses. Clear the heartbeat/start markers too.
-		log.Log.Info("routers.mqtt.main.HandleRecording(): manual recording stopped.")
+		log.Info("routers.mqtt.main.HandleRecording(): manual recording stopped.")
 		communication.IsRecordingManual.UnSet()
 		communication.RecordingManualHeartbeat.Store(0)
 		communication.RecordingManualStart.Store(0)
@@ -443,7 +485,7 @@ func HandleAudio(mqttClient mqtt.Client, hubKey string, payload models.Payload,
 			Data:      audioPayload.Data,
 		}
 		if enqueueLatestAudio(communication.HandleAudio, audioDataPartial) {
-			log.Log.Debug("routers.mqtt.main.HandleAudio(): dropped stale audio because the backchannel queue was full")
+			log.Debug("routers.mqtt.main.HandleAudio(): dropped stale audio because the backchannel queue was full")
 		}
 	}
 }
@@ -479,7 +521,7 @@ func HandleGetPTZPosition(mqttClient mqtt.Client, hubKey string, payload models.
 		// Get Position from device
 		pos, err := onvif.GetPositionFromDevice(*configuration)
 		if err != nil {
-			log.Log.Error("routers.mqtt.main.HandlePTZPosition(): error getting position from device: " + err.Error())
+			log.Error("routers.mqtt.main.HandlePTZPosition(): error getting position from device: " + err.Error())
 		} else {
 			// Needs to wrapped!
 			posString := fmt.Sprintf("%f,%f,%f", pos.PanTilt.X, pos.PanTilt.Y, pos.Zoom.X)
@@ -497,7 +539,10 @@ func HandleGetPTZPosition(mqttClient mqtt.Client, hubKey string, payload models.
 			if err == nil {
 				mqttClient.Publish("kerberos/hub/"+hubKey, 2, false, payload)
 			} else {
-				log.Log.Info("routers.mqtt.main.HandlePTZPosition(): something went wrong while sending position to hub: " + string(payload))
+				log.WithError(err).WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "ptz_position_packaging_failed",
+				}).Error("Failed to package PTZ position message")
 			}
 		}
 	}
@@ -512,11 +557,14 @@ func HandleUpdatePTZPosition(mqttClient mqtt.Client, hubKey string, payload mode
 	json.Unmarshal(jsonData, &onvifAction)
 
 	if onvifAction.Action != "" {
-		if communication.CameraConnected {
-			communication.HandleONVIF <- onvifAction
-			log.Log.Info("routers.mqtt.main.MQTTListenerHandleONVIF(): Received an action - " + onvifAction.Action)
+		if communication.CameraConnected.Load() {
+			if communication.TrySendONVIF(onvifAction) {
+				log.Info("routers.mqtt.main.MQTTListenerHandleONVIF(): Received an action - " + onvifAction.Action)
+			} else {
+				log.Warn("routers.mqtt.main.MQTTListenerHandleONVIF(): action channel unavailable or full")
+			}
 		} else {
-			log.Log.Info("routers.mqtt.main.MQTTListenerHandleONVIF(): received action, but camera is not connected.")
+			log.Info("routers.mqtt.main.MQTTListenerHandleONVIF(): received action, but camera is not connected.")
 		}
 	}
 }
@@ -561,14 +609,17 @@ func HandleRequestConfig(mqttClient mqtt.Client, hubKey string, payload models.P
 			if err == nil {
 				mqttClient.Publish("kerberos/hub/"+hubKey, 2, false, payload)
 			} else {
-				log.Log.Info("routers.mqtt.main.HandleRequestConfig(): something went wrong while sending config to hub: " + string(payload))
+				log.WithError(err).WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "configuration_packaging_failed",
+				}).Error("Failed to package configuration message")
 			}
 
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleRequestConfig(): no config available")
+			log.Info("routers.mqtt.main.HandleRequestConfig(): no config available")
 		}
 
-		log.Log.Info("routers.mqtt.main.HandleRequestConfig(): Received a request for the config")
+		log.Info("routers.mqtt.main.HandleRequestConfig(): Received a request for the config")
 	}
 }
 
@@ -660,7 +711,10 @@ func HandleVerifyStream(mqttClient mqtt.Client, hubKey string, payload models.Pa
 	if err == nil {
 		mqttClient.Publish("kerberos/hub/"+hubKey, 2, false, packagedPayload)
 	} else {
-		log.Log.Info("routers.mqtt.main.HandleVerifyStream(): something went wrong while sending result to hub: " + string(packagedPayload))
+		log.WithError(err).WithFields(log.Fields{
+			"component": "routers/mqtt",
+			"event":     "stream_verification_packaging_failed",
+		}).Error("Failed to package stream verification result")
 	}
 }
 
@@ -681,7 +735,7 @@ func HandleUpdateConfig(mqttClient mqtt.Client, hubKey string, payload models.Pa
 
 		err := configService.SaveConfig(configDirectory, config, configuration, communication)
 		if err == nil {
-			log.Log.Info("routers.mqtt.main.HandleUpdateConfig(): Config updated")
+			log.Info("routers.mqtt.main.HandleUpdateConfig(): Config updated")
 			message := models.Message{
 				Payload: models.Payload{
 					Action:   "acknowledge-update-config",
@@ -692,10 +746,13 @@ func HandleUpdateConfig(mqttClient mqtt.Client, hubKey string, payload models.Pa
 			if err == nil {
 				mqttClient.Publish("kerberos/hub/"+hubKey, 2, false, payload)
 			} else {
-				log.Log.Info("routers.mqtt.main.HandleUpdateConfig(): something went wrong while sending acknowledge config to hub: " + string(payload))
+				log.WithError(err).WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "configuration_acknowledgement_packaging_failed",
+				}).Error("Failed to package configuration acknowledgement")
 			}
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleUpdateConfig(): Config update failed")
+			log.Info("routers.mqtt.main.HandleUpdateConfig(): Config update failed")
 		}
 	}
 }
@@ -708,7 +765,7 @@ func HandleRequestSDStream(mqttClient mqtt.Client, hubKey string, payload models
 	json.Unmarshal(jsonData, &requestSDStreamPayload)
 
 	if requestSDStreamPayload.Timestamp != 0 {
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			// A viewer that opted into the HTTP transport is signalled on a separate
 			// channel so the producer ships its frames to hub-api over HTTP instead of
 			// publishing them over MQTT. Any other (or absent) transport keeps the
@@ -724,9 +781,9 @@ func HandleRequestSDStream(mqttClient mqtt.Client, hubKey string, payload models
 				default:
 				}
 			}
-			log.Log.Info("routers.mqtt.main.HandleRequestSDStream(): received request to livestream.")
+			log.Info("routers.mqtt.main.HandleRequestSDStream(): received request to livestream.")
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleRequestSDStream(): received request to livestream, but camera is not connected.")
+			log.Info("routers.mqtt.main.HandleRequestSDStream(): received request to livestream, but camera is not connected.")
 		}
 	}
 }
@@ -743,7 +800,7 @@ func HandleRequestHLSStream(mqttClient mqtt.Client, hubKey string, payload model
 	json.Unmarshal(jsonData, &requestHLSStreamPayload)
 
 	if requestHLSStreamPayload.Timestamp != 0 {
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			// Forward the requested quality ("auto"|"high"|"low"; empty => auto) so
 			// the producer can switch the live session between the main and sub
 			// stream on demand. The send doubles as the viewer keepalive.
@@ -751,9 +808,9 @@ func HandleRequestHLSStream(mqttClient mqtt.Client, hubKey string, payload model
 			case communication.HandleLiveHLS <- requestHLSStreamPayload.Quality:
 			default:
 			}
-			log.Log.Info("routers.mqtt.main.HandleRequestHLSStream(): received request to livestream over HLS.")
+			log.Info("routers.mqtt.main.HandleRequestHLSStream(): received request to livestream over HLS.")
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleRequestHLSStream(): received request to livestream over HLS, but camera is not connected.")
+			log.Info("routers.mqtt.main.HandleRequestHLSStream(): received request to livestream over HLS, but camera is not connected.")
 		}
 	}
 }
@@ -766,29 +823,30 @@ func HandleRequestHDStream(mqttClient mqtt.Client, hubKey string, payload models
 	json.Unmarshal(jsonData, &requestHDStreamPayload)
 
 	if requestHDStreamPayload.Timestamp != 0 {
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			// Dedupe by session_id: the viewer republishes its offer while
 			// waiting for an answer (and the broker may redeliver), and we
 			// don't want to spawn multiple peer connections for the same
 			// browser session.
 			if markHDSessionSeen(requestHDStreamPayload.SessionID) {
-				log.Log.Info("routers.mqtt.main.HandleRequestHDStream(): duplicate request for session " +
-					requestHDStreamPayload.SessionID + ", ignoring")
+				log.WithFields(log.Fields{
+					"component":  "routers/mqtt",
+					"event":      "duplicate_hd_stream_request",
+					"session_id": requestHDStreamPayload.SessionID,
+				}).Debug("Ignoring duplicate HD stream request")
 				return
 			}
 			// Set the Hub key, so we can send back the answer.
 			requestHDStreamPayload.HubKey = hubKey
-			if communication.HandleLiveHDHandshake == nil {
-				log.Log.Error("routers.mqtt.main.HandleRequestHDStream(): handshake channel is nil, dropping request")
+			if !communication.TrySendLiveHDHandshake(models.LiveHDHandshake{
+				Payload: requestHDStreamPayload,
+			}) {
+				log.Error("routers.mqtt.main.HandleRequestHDStream(): handshake channel unavailable or full, dropping request")
 				return
 			}
-
-			communication.HandleLiveHDHandshake <- models.LiveHDHandshake{
-				Payload: requestHDStreamPayload,
-			}
-			log.Log.Info("routers.mqtt.main.HandleRequestHDStream(): received request to setup webrtc.")
+			log.Info("routers.mqtt.main.HandleRequestHDStream(): received request to setup webrtc.")
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleRequestHDStream(): received request to setup webrtc, but camera is not connected.")
+			log.Info("routers.mqtt.main.HandleRequestHDStream(): received request to setup webrtc, but camera is not connected.")
 		}
 	}
 }
@@ -801,12 +859,12 @@ func HandleReceiveHDCandidates(mqttClient mqtt.Client, hubKey string, payload mo
 	json.Unmarshal(jsonData, &receiveHDCandidatesPayload)
 
 	if receiveHDCandidatesPayload.Timestamp != 0 {
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			// Register candidate channel
 			key := configuration.Config.Key + "/" + receiveHDCandidatesPayload.SessionID
 			go webrtc.RegisterCandidates(key, receiveHDCandidatesPayload)
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleReceiveHDCandidates(): received candidate, but camera is not connected.")
+			log.Info("routers.mqtt.main.HandleReceiveHDCandidates(): received candidate, but camera is not connected.")
 		}
 	}
 }
@@ -818,14 +876,17 @@ func HandleNavigatePTZ(mqttClient mqtt.Client, hubKey string, payload models.Pay
 	json.Unmarshal(jsonData, &navigatePTZPayload)
 
 	if navigatePTZPayload.Timestamp != 0 {
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			action := navigatePTZPayload.Action
 			var onvifAction models.OnvifAction
 			json.Unmarshal([]byte(action), &onvifAction)
-			communication.HandleONVIF <- onvifAction
-			log.Log.Info("routers.mqtt.main.HandleNavigatePTZ(): Received an action - " + onvifAction.Action)
+			if communication.TrySendONVIF(onvifAction) {
+				log.Info("routers.mqtt.main.HandleNavigatePTZ(): Received an action - " + onvifAction.Action)
+			} else {
+				log.Warn("routers.mqtt.main.HandleNavigatePTZ(): action channel unavailable or full")
+			}
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleNavigatePTZ(): received action, but camera is not connected.")
+			log.Info("routers.mqtt.main.HandleNavigatePTZ(): received action, but camera is not connected.")
 		}
 	}
 }
@@ -837,7 +898,7 @@ func HandleTriggerRelay(mqttClient mqtt.Client, hubKey string, payload models.Pa
 	json.Unmarshal(jsonData, &triggerRelayPayload)
 
 	if triggerRelayPayload.Timestamp != 0 {
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			// Get token (name of relay)
 			token := triggerRelayPayload.Token
 			// Connect to Onvif device
@@ -847,16 +908,25 @@ func HandleTriggerRelay(mqttClient mqtt.Client, hubKey string, payload models.Pa
 				// Trigger relay output
 				err := onvif.TriggerRelayOutput(device, token)
 				if err != nil {
-					log.Log.Error("routers.mqtt.main.HandleTriggerRelay(): error triggering relay: " + err.Error())
+					log.WithError(err).WithFields(log.Fields{
+						"component": "routers/mqtt",
+						"event":     "relay_output_failed",
+					}).Error("Failed to trigger relay output")
 				} else {
-					log.Log.Info("routers.mqtt.main.HandleTriggerRelay(): trigger (" + token + ") relay output.")
+					log.WithFields(log.Fields{
+						"component": "routers/mqtt",
+						"event":     "relay_output_triggered",
+					}).Info("Relay output triggered")
 				}
 			} else {
-				log.Log.Error("routers.mqtt.main.HandleTriggerRelay(): error connecting to device: " + err.Error())
+				log.WithError(err).WithFields(log.Fields{
+					"component": "routers/mqtt",
+					"event":     "relay_device_connection_failed",
+				}).Error("Failed to connect to relay device")
 			}
 
 		} else {
-			log.Log.Info("routers.mqtt.main.HandleTriggerRelay(): received trigger, but camera is not connected.")
+			log.Info("routers.mqtt.main.HandleTriggerRelay(): received trigger, but camera is not connected.")
 		}
 	}
 }
@@ -868,6 +938,6 @@ func DisconnectMQTT(mqttClient mqtt.Client, config *models.Config) {
 		mqttClient.Unsubscribe("kerberos/agent/" + PREV_HubKey)
 		mqttClient.Disconnect(1000)
 		mqttClient = nil
-		log.Log.Info("routers.mqtt.main.DisconnectMQTT(): MQTT client disconnected.")
+		log.Info("routers.mqtt.main.DisconnectMQTT(): MQTT client disconnected.")
 	}
 }

machinery/src/routers/mqtt/main_test.go

@@ -7,6 +7,22 @@ import (
 	"github.com/kerberos-io/agent/machinery/src/models"
 )
 
+func TestConfigureMQTTRequiresHubKey(t *testing.T) {
+	configuration := &models.Configuration{Config: models.Config{Key: "agent-key"}}
+
+	if client := ConfigureMQTT("", configuration, &models.Communication{}); client != nil {
+		t.Fatal("ConfigureMQTT() returned a client without a Hub key")
+	}
+}
+
+func TestConfigureMQTTRequiresAgentKey(t *testing.T) {
+	configuration := &models.Configuration{Config: models.Config{HubKey: "hub-key"}}
+
+	if client := ConfigureMQTT("", configuration, &models.Communication{}); client != nil {
+		t.Fatal("ConfigureMQTT() returned a client without an Agent key")
+	}
+}
+
 func TestEnqueueLatestAudioReplacesOldestFrameWhenFull(t *testing.T) {
 	audioChannel := make(chan models.AudioDataPartial, 2)
 	audioChannel <- models.AudioDataPartial{Timestamp: 1}

machinery/src/routers/websocket/main.go

@@ -11,11 +11,11 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/gorilla/websocket"
 	"github.com/kerberos-io/agent/machinery/src/capture"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/kerberos-io/agent/machinery/src/utils"
 	"github.com/kerberos-io/agent/machinery/src/webrtc"
+	log "github.com/sirupsen/logrus"
 )
 
 type Message struct {
@@ -43,7 +43,7 @@ func writeWebRTCError(connection *Connection, clientID string, sessionID string,
 			"message":    errorMessage,
 		},
 	}); err != nil {
-		log.Log.Error("routers.websocket.main.writeWebRTCError(): " + err.Error())
+		log.Error("routers.websocket.main.writeWebRTCError(): " + err.Error())
 	}
 }
 
@@ -80,7 +80,7 @@ func WebsocketHandler(c *gin.Context, configuration *models.Configuration, commu
 		var message Message
 		err = conn.ReadJSON(&message)
 		if err != nil {
-			log.Log.Error("routers.websocket.main.WebsocketHandler(): " + err.Error())
+			log.Error("routers.websocket.main.WebsocketHandler(): " + err.Error())
 			return
 		}
 		clientID := message.ClientID
@@ -89,7 +89,7 @@ func WebsocketHandler(c *gin.Context, configuration *models.Configuration, commu
 			connection.Socket = conn
 			sockets[clientID] = connection
 			sockets[clientID].Cancels = make(map[string]context.CancelFunc)
-			log.Log.Info("routers.websocket.main.WebsocketHandler(): " + clientID + ": connected.")
+			log.Info("routers.websocket.main.WebsocketHandler(): " + clientID + ": connected.")
 		}
 
 		// Continuously read messages
@@ -111,14 +111,14 @@ func WebsocketHandler(c *gin.Context, configuration *models.Configuration, commu
 				if exists {
 					sockets[clientID].Cancels["stream-sd"]()
 				} else {
-					log.Log.Error("routers.websocket.main.WebsocketHandler(): streaming sd does not exists for " + clientID)
+					log.Error("routers.websocket.main.WebsocketHandler(): streaming sd does not exists for " + clientID)
 				}
 
 			case "stream-sd":
-				if communication.CameraConnected {
+				if communication.CameraConnected.Load() {
 					_, exists := sockets[clientID].Cancels["stream-sd"]
 					if exists {
-						log.Log.Debug("routers.websocket.main.WebsocketHandler(): already streaming sd for " + clientID)
+						log.Debug("routers.websocket.main.WebsocketHandler(): already streaming sd for " + clientID)
 					} else {
 						startStream := Message{
 							ClientID:    clientID,
@@ -144,16 +144,11 @@ func WebsocketHandler(c *gin.Context, configuration *models.Configuration, commu
 					break
 				}
 
-				if !communication.CameraConnected {
+				if !communication.CameraConnected.Load() {
 					writeWebRTCError(sockets[clientID], clientID, sessionID, "camera is not connected")
 					break
 				}
 
-				if communication.HandleLiveHDHandshake == nil {
-					writeWebRTCError(sockets[clientID], clientID, sessionID, "webrtc liveview is not available")
-					break
-				}
-
 				handshake := models.LiveHDHandshake{
 					Payload: models.RequestHDStreamPayload{
 						Timestamp:          time.Now().Unix(),
@@ -188,7 +183,9 @@ func WebsocketHandler(c *gin.Context, configuration *models.Configuration, commu
 					},
 				}
 
-				communication.HandleLiveHDHandshake <- handshake
+				if !communication.TrySendLiveHDHandshake(handshake) {
+					writeWebRTCError(sockets[clientID], clientID, handshake.Payload.SessionID, "camera is restarting or live-view queue is full")
+				}
 
 			case "webrtc-candidate":
 				sessionID := message.Message["session_id"]
@@ -199,7 +196,7 @@ func WebsocketHandler(c *gin.Context, configuration *models.Configuration, commu
 					break
 				}
 
-				if !communication.CameraConnected {
+				if !communication.CameraConnected.Load() {
 					writeWebRTCError(sockets[clientID], clientID, sessionID, "camera is not connected")
 					break
 				}
@@ -221,7 +218,7 @@ func WebsocketHandler(c *gin.Context, configuration *models.Configuration, commu
 		_, exists := sockets[clientID]
 		if exists {
 			delete(sockets, clientID)
-			log.Log.Info("routers.websocket.main.WebsocketHandler(): " + clientID + ": terminated and disconnected websocket connection.")
+			log.Info("routers.websocket.main.WebsocketHandler(): " + clientID + ": terminated and disconnected websocket connection.")
 		}
 	}
 }
@@ -232,13 +229,14 @@ func ForwardSDStream(ctx context.Context, clientID string, connection *Connectio
 	var cursor *packets.QueueCursor
 
 	// We'll pick the right client and decoder.
-	rtspClient := captureDevice.RTSPSubClient
+	rtspClient := captureDevice.SubClient()
 	if rtspClient != nil {
-		queue = communication.SubQueue
-		cursor = queue.Latest()
+		queue = communication.SubQueue()
 	} else {
-		rtspClient = captureDevice.RTSPClient
-		queue = communication.Queue
+		rtspClient = captureDevice.MainClient()
+		queue = communication.MainQueue()
+	}
+	if queue != nil {
 		cursor = queue.Latest()
 	}
 
@@ -263,7 +261,7 @@ logreader:
 					continue
 				}
 			} else {
-				log.Log.Error("routers.websocket.main.ForwardSDStream():" + err.Error())
+				log.Error("routers.websocket.main.ForwardSDStream():" + err.Error())
 				break logreader
 			}
 		}
@@ -277,7 +275,7 @@ logreader:
 		}
 		err := connection.WriteJson(startStrean)
 		if err != nil {
-			log.Log.Error("routers.websocket.main.ForwardSDStream():" + err.Error())
+			log.Error("routers.websocket.main.ForwardSDStream():" + err.Error())
 			break logreader
 		}
 		select {
@@ -292,9 +290,9 @@ logreader:
 	if exists {
 		delete(connection.Cancels, "stream-sd")
 	} else {
-		log.Log.Error("routers.websocket.main.ForwardSDStream(): streaming sd does not exists for " + clientID)
+		log.Error("routers.websocket.main.ForwardSDStream(): streaming sd does not exists for " + clientID)
 	}
 
 	// Send stop streaming message
-	log.Log.Info("routers.websocket.main.ForwardSDStream(): stop sending streaming over websocket")
+	log.Info("routers.websocket.main.ForwardSDStream(): stop sending streaming over websocket")
 }

machinery/src/utils/main.go

@@ -1,8 +1,8 @@
 package utils
 
 import (
-	"bufio"
 	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"image"
@@ -12,15 +12,14 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
-	"reflect"
 	"sort"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/encryption"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 
 	"github.com/nfnt/resize"
 )
@@ -140,11 +139,11 @@ func CheckDataDirectoryPermissions(configDirectory string) error {
 	}
 
 	if err != nil {
-		log.Log.Error("Checking data directory permissions: " + err.Error())
+		log.Error("Checking data directory permissions: " + err.Error())
 		return err
 	}
 
-	log.Log.Info("Checking data directory permissions: OK")
+	log.Info("Checking data directory permissions: OK")
 	return nil
 }
 
@@ -193,50 +192,47 @@ func GetMediaFormatted(files []os.FileInfo, recordingDirectory string, configura
 	count := 0
 	for _, file := range files {
 		fileName := file.Name()
-		fileParts := strings.Split(fileName, "_")
-		if len(fileParts) == 6 {
-			timestamp := fileParts[0]
-			timestampInt, err := strconv.ParseInt(timestamp, 10, 64)
-			if err == nil {
+		timestampInt, ok := recordingTimestamp(file, recordingDirectory)
+		if ok {
+			timestamp := strconv.FormatInt(timestampInt, 10)
 
-				if eventFilter.TimestampOffsetStart > 0 {
-					// TimestampOffsetStart represents the newest lower bound to include.
-					if timestampInt < eventFilter.TimestampOffsetStart {
-						continue
-					}
+			if eventFilter.TimestampOffsetStart > 0 {
+				// TimestampOffsetStart represents the newest lower bound to include.
+				if timestampInt < eventFilter.TimestampOffsetStart {
+					continue
 				}
+			}
 
-				// If we have an offset we will check if we should skip or not
-				if eventFilter.TimestampOffsetEnd > 0 {
-					// Medias are sorted from new to older. TimestampOffsetEnd holds the oldest
-					// timestamp of the previous batch of events. By doing this check, we make sure
-					// to skip the previous batch.
-					if timestampInt >= eventFilter.TimestampOffsetEnd {
-						continue
-					}
+			// If we have an offset we will check if we should skip or not
+			if eventFilter.TimestampOffsetEnd > 0 {
+				// Medias are sorted from new to older. TimestampOffsetEnd holds the oldest
+				// timestamp of the previous batch of events. By doing this check, we make sure
+				// to skip the previous batch.
+				if timestampInt >= eventFilter.TimestampOffsetEnd {
+					continue
 				}
+			}
 
-				loc, _ := time.LoadLocation(configuration.Config.Timezone)
-				time := time.Unix(timestampInt, 0).In(loc)
-				day := time.Format("02-01-2006")
-				timeString := time.Format("15:04:05")
-				shortDay := time.Format("Jan _2")
+			loc, _ := time.LoadLocation(configuration.Config.Timezone)
+			mediaTime := time.Unix(timestampInt, 0).In(loc)
+			day := mediaTime.Format("02-01-2006")
+			timeString := mediaTime.Format("15:04:05")
+			shortDay := mediaTime.Format("Jan _2")
 
-				media := models.Media{
-					Key:        fileName,
-					Path:       recordingDirectory + "/" + fileName,
-					CameraName: configuration.Config.Name,
-					CameraKey:  configuration.Config.Key,
-					Day:        day,
-					ShortDay:   shortDay,
-					Time:       timeString,
-					Timestamp:  timestamp,
-				}
-				filePaths = append(filePaths, media)
-				count = count + 1
-				if eventFilter.NumberOfElements > 0 && count >= eventFilter.NumberOfElements {
-					break
-				}
+			media := models.Media{
+				Key:        fileName,
+				Path:       recordingDirectory + "/" + fileName,
+				CameraName: configuration.Config.Name,
+				CameraKey:  configuration.Config.Key,
+				Day:        day,
+				ShortDay:   shortDay,
+				Time:       timeString,
+				Timestamp:  timestamp,
+			}
+			filePaths = append(filePaths, media)
+			count = count + 1
+			if eventFilter.NumberOfElements > 0 && count >= eventFilter.NumberOfElements {
+				break
 			}
 		}
 	}
@@ -246,23 +242,44 @@ func GetMediaFormatted(files []os.FileInfo, recordingDirectory string, configura
 func GetDays(files []os.FileInfo, recordingDirectory string, configuration *models.Configuration) []string {
 	days := []string{}
 	for _, file := range files {
-		fileName := file.Name()
-		fileParts := strings.Split(fileName, "_")
-		if len(fileParts) == 6 {
-			timestamp := fileParts[0]
-			timestampInt, err := strconv.ParseInt(timestamp, 10, 64)
-			if err == nil {
-				loc, _ := time.LoadLocation(configuration.Config.Timezone)
-				time := time.Unix(timestampInt, 0).In(loc)
-				day := time.Format("02-01-2006")
-				days = append(days, day)
-			}
+		if timestamp, ok := recordingTimestamp(file, recordingDirectory); ok {
+			loc, _ := time.LoadLocation(configuration.Config.Timezone)
+			mediaTime := time.Unix(timestamp, 0).In(loc)
+			days = append(days, mediaTime.Format("02-01-2006"))
 		}
 	}
 	uniqueDays := Unique(days)
 	return uniqueDays
 }
 
+func recordingTimestamp(file os.FileInfo, recordingDirectory string) (int64, bool) {
+	markerPath := filepath.Join(filepath.Dir(recordingDirectory), "cloud", models.RecordingUploadMetadataFileName(file.Name()))
+	if value, err := os.ReadFile(markerPath); err == nil {
+		var metadata models.RecordingUploadMetadata
+		if json.Unmarshal(value, &metadata) == nil && metadata.Timestamp > 0 {
+			return metadata.Timestamp / 1000, true
+		}
+	}
+
+	if timestamp, ok := legacyRecordingTimestampFromFilename(file.Name()); ok {
+		return timestamp, true
+	}
+
+	if timestamp := file.ModTime().Unix(); timestamp > 0 {
+		return timestamp, true
+	}
+	return 0, false
+}
+
+func legacyRecordingTimestampFromFilename(fileName string) (int64, bool) {
+	fileParts := strings.Split(fileName, "_")
+	if len(fileParts) != 6 {
+		return 0, false
+	}
+	timestamp, err := strconv.ParseInt(fileParts[0], 10, 64)
+	return timestamp, err == nil
+}
+
 func Unique(intSlice []string) []string {
 	keys := make(map[string]bool)
 	list := []string{}
@@ -301,16 +318,16 @@ func CreateFragmentedMP4(fullName string, fragmentedDuration int64) {
 	// This timescale is crucial, as it should be the same as the one defined in JOY4.
 	cmd := exec.Command("mp4fragment", "--timescale", "10000000", "--fragment-duration", strconv.FormatInt(duration, 10), fullName, fullName+"f.mp4")
 	cmd.Dir = path
-	log.Log.Info(cmd.String())
+	log.Info(cmd.String())
 	var out bytes.Buffer
 	var stderr bytes.Buffer
 	cmd.Stdout = &out
 	cmd.Stderr = &stderr
 	err := cmd.Run()
 	if err != nil {
-		log.Log.Error(fmt.Sprint(err) + ": " + stderr.String())
+		log.Error(fmt.Sprint(err) + ": " + stderr.String())
 	} else {
-		log.Log.Info("Created Fragmented: " + out.String())
+		log.Info("Created Fragmented: " + out.String())
 	}
 
 	// We will swap the files.
@@ -319,42 +336,93 @@ func CreateFragmentedMP4(fullName string, fragmentedDuration int64) {
 }
 
 func PrintEnvironmentVariables() {
-	// Print environment variables that include "AGENT_" as a prefix.
-	environmentVariables := ""
-	for _, e := range os.Environ() {
-		if strings.Contains(e, "AGENT_") {
-			pair := strings.Split(e, "=")
-			environmentVariables = environmentVariables + pair[0] + "=" + pair[1] + " "
+	names := agentEnvironmentVariableNames(os.Environ())
+	log.WithFields(log.Fields{
+		"component":      "configuration",
+		"event":          "environment_loaded",
+		"variable_count": len(names),
+	}).Info("Agent environment loaded")
+	log.WithFields(log.Fields{
+		"component": "configuration",
+		"event":     "environment_details",
+		"variables": names,
+	}).Debug("Agent environment details")
+}
+
+func agentEnvironmentVariableNames(environment []string) []string {
+	names := make([]string, 0)
+	for _, entry := range environment {
+		name, _, found := strings.Cut(entry, "=")
+		if found && strings.HasPrefix(name, "AGENT_") {
+			names = append(names, name)
 		}
 	}
-	log.Log.Info("Printing out environmentVariables (AGENT_...): " + environmentVariables)
+	sort.Strings(names)
+	return names
 }
 
 func PrintConfiguration(configuration *models.Configuration) {
-	// We will print out the struct.
 	if configuration == nil {
-		log.Log.Info("Configuration is nil")
+		log.WithFields(log.Fields{
+			"component": "configuration",
+			"event":     "configuration_missing",
+		}).Warn("Agent configuration is nil")
 		return
 	}
-	config := configuration.Config
-	// Iterate over the struct and printout the values.
-	v := reflect.ValueOf(config)
-	typeOfS := v.Type()
-	configurationVariables := ""
-	for i := 0; i < v.NumField(); i++ {
-		key := typeOfS.Field(i).Name
-		value := v.Field(i).Interface()
-		// Convert to string.
-		configurationVariables = configurationVariables + key + ": " + fmt.Sprintf("%v", value) + " "
+
+	log.WithFields(configurationLogFields(configuration.Config)).
+		Info("Agent configuration loaded")
+	log.WithFields(configurationDebugLogFields(configuration.Config)).
+		Debug("Agent configuration details")
+}
+
+func configurationLogFields(config models.Config) log.Fields {
+	return log.Fields{
+		"agent_name":     config.Name,
+		"camera_type":    config.Capture.Name,
+		"cloud_provider": config.Cloud,
+		"component":      "configuration",
+		"event":          "configuration_loaded",
+		"friendly_name":  config.FriendlyName,
+		"liveview":       config.Capture.Liveview,
+		"offline":        config.Offline,
+		"recording":      config.Capture.Recording,
+		"timezone":       config.Timezone,
+	}
+}
+
+func configurationDebugLogFields(config models.Config) log.Fields {
+	return log.Fields{
+		"auto_clean":             config.AutoClean,
+		"component":              "configuration",
+		"continuous":             config.Capture.Continuous,
+		"event":                  "configuration_details",
+		"gop_size":               config.Capture.GopSize,
+		"main_fps":               config.Capture.IPCamera.FPS,
+		"main_height":            config.Capture.IPCamera.Height,
+		"main_stream_configured": config.Capture.IPCamera.RTSP != "",
+		"main_width":             config.Capture.IPCamera.Width,
+		"max_recording_seconds":  config.Capture.MaxLengthRecording,
+		"motion":                 config.Capture.Motion,
+		"onvif_configured":       config.Capture.IPCamera.ONVIFXAddr != "",
+		"onvif_motion":           config.Capture.ONVIFMotion,
+		"post_recording_seconds": config.Capture.PostRecording,
+		"pre_recording_seconds":  config.Capture.PreRecording,
+		"remove_after_upload":    config.RemoveAfterUpload,
+		"snapshots":              config.Capture.Snapshots,
+		"sub_fps":                config.Capture.IPCamera.SubFPS,
+		"sub_height":             config.Capture.IPCamera.SubHeight,
+		"sub_stream_configured":  config.Capture.IPCamera.SubRTSP != "",
+		"sub_width":              config.Capture.IPCamera.SubWidth,
+		"transcoding_webrtc":     config.Capture.TranscodingWebRTC,
 	}
-	log.Log.Info("Printing our configuration (config.json): " + configurationVariables)
 }
 
 func Decrypt(directoryOrFile string, symmetricKey []byte) {
 	// Check if file or directory
 	fileInfo, err := os.Stat(directoryOrFile)
 	if err != nil {
-		log.Log.Fatal(err.Error())
+		log.Fatal(err.Error())
 		return
 	}
 
@@ -363,12 +431,12 @@ func Decrypt(directoryOrFile string, symmetricKey []byte) {
 		// Create decrypted directory
 		err = os.MkdirAll(directoryOrFile+"/decrypted", 0755)
 		if err != nil {
-			log.Log.Fatal(err.Error())
+			log.Fatal(err.Error())
 			return
 		}
 		dir, err := os.ReadDir(directoryOrFile)
 		if err != nil {
-			log.Log.Fatal(err.Error())
+			log.Fatal(err.Error())
 			return
 		}
 		for _, file := range dir {
@@ -390,13 +458,13 @@ func Decrypt(directoryOrFile string, symmetricKey []byte) {
 		// Read file
 		content, err := os.ReadFile(file)
 		if err != nil {
-			log.Log.Fatal(err.Error())
+			log.Fatal(err.Error())
 			return
 		}
 		// Decrypt using AES key
 		decrypted, err := encryption.AesDecrypt(content, string(symmetricKey))
 		if err != nil {
-			log.Log.Fatal("Something went wrong while decrypting: " + err.Error())
+			log.Fatal("Something went wrong while decrypting: " + err.Error())
 			return
 		}
 
@@ -408,7 +476,7 @@ func Decrypt(directoryOrFile string, symmetricKey []byte) {
 
 		err = os.WriteFile(pathToFile+"/decrypted/"+fileName, []byte(decrypted), 0644)
 		if err != nil {
-			log.Log.Fatal(err.Error())
+			log.Fatal(err.Error())
 			return
 		}
 	}
@@ -416,9 +484,12 @@ func Decrypt(directoryOrFile string, symmetricKey []byte) {
 
 func ImageToBytes(img *image.Image) ([]byte, error) {
 	buffer := new(bytes.Buffer)
-	w := bufio.NewWriter(buffer)
-	err := jpeg.Encode(w, *img, &jpeg.Options{Quality: 35})
-	log.Log.Debug("ImageToBytes() - buffer size: " + strconv.Itoa(buffer.Len()))
+	err := jpeg.Encode(buffer, *img, &jpeg.Options{Quality: 35})
+	log.WithFields(log.Fields{
+		"bytes":     buffer.Len(),
+		"component": "image",
+		"event":     "jpeg_encoded",
+	}).Debug("Image encoded as JPEG")
 	return buffer.Bytes(), err
 }
 

machinery/src/utils/main_test.go

@@ -1,21 +1,122 @@
 package utils
 
 import (
+	"bytes"
+	"fmt"
+	"image"
+	"image/color"
+	"image/jpeg"
 	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/models"
 )
 
+func TestImageToBytesReturnsCompleteJPEG(t *testing.T) {
+	source := image.NewRGBA(image.Rect(0, 0, 16, 12))
+	source.Set(8, 6, color.RGBA{R: 255, A: 255})
+	var input image.Image = source
+
+	encoded, err := ImageToBytes(&input)
+	if err != nil {
+		t.Fatalf("ImageToBytes() error = %v", err)
+	}
+	if len(encoded) == 0 {
+		t.Fatal("ImageToBytes() returned an empty JPEG")
+	}
+
+	decoded, err := jpeg.Decode(bytes.NewReader(encoded))
+	if err != nil {
+		t.Fatalf("decoding ImageToBytes() output: %v", err)
+	}
+	if got := decoded.Bounds().Size(); got.X != 16 || got.Y != 12 {
+		t.Fatalf("decoded JPEG size = %dx%d, want 16x12", got.X, got.Y)
+	}
+}
+
+func TestAgentEnvironmentVariableNamesOmitsValues(t *testing.T) {
+	got := agentEnvironmentVariableNames([]string{
+		"AGENT_HUB_PRIVATE_KEY=do-not-log",
+		"PATH=/usr/bin",
+		"NOT_AGENT_SECRET=also-do-not-log",
+		"AGENT_CAPTURE_LIVEVIEW=true",
+	})
+	want := []string{"AGENT_CAPTURE_LIVEVIEW", "AGENT_HUB_PRIVATE_KEY"}
+
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("agentEnvironmentVariableNames() = %v, want %v", got, want)
+	}
+}
+
+func TestConfigurationLogFieldsOmitCredentialsAndEndpoints(t *testing.T) {
+	config := models.Config{
+		Name:          "Front Door",
+		FriendlyName:  "Entrance",
+		Cloud:         "kstorage",
+		HubKey:        "hub-key-secret",
+		HubPrivateKey: "hub-private-secret",
+		MQTTURI:       "mqtt://internal.example",
+		MQTTUsername:  "mqtt-user",
+		MQTTPassword:  "mqtt-password-secret",
+		Capture: models.Capture{
+			Liveview:           "true",
+			MaxLengthRecording: 20,
+			IPCamera: models.IPCamera{
+				RTSP:          "rtsp://camera-user:camera-password@10.0.30.11/live",
+				ONVIFUsername: "onvif-user",
+				ONVIFPassword: "onvif-password-secret",
+			},
+		},
+	}
+
+	fields := []map[string]interface{}{
+		configurationLogFields(config),
+		configurationDebugLogFields(config),
+	}
+	summary := fmt.Sprint(fields)
+	for _, secret := range []string{
+		config.HubKey,
+		config.HubPrivateKey,
+		config.MQTTURI,
+		config.MQTTUsername,
+		config.MQTTPassword,
+		config.Capture.IPCamera.RTSP,
+		config.Capture.IPCamera.ONVIFUsername,
+		config.Capture.IPCamera.ONVIFPassword,
+	} {
+		if strings.Contains(summary, secret) {
+			t.Fatalf("configuration log fields exposed sensitive value %q in %q", secret, summary)
+		}
+	}
+	infoFields := configurationLogFields(config)
+	if got := infoFields["agent_name"]; got != "Front Door" {
+		t.Fatalf("agent_name = %v, want Front Door", got)
+	}
+	if got := infoFields["cloud_provider"]; got != "kstorage" {
+		t.Fatalf("cloud_provider = %v, want kstorage", got)
+	}
+	debugFields := configurationDebugLogFields(config)
+	if got := debugFields["max_recording_seconds"]; got != int64(20) {
+		t.Fatalf("max_recording_seconds = %v, want 20", got)
+	}
+	if got := debugFields["main_stream_configured"]; got != true {
+		t.Fatalf("main_stream_configured = %v, want true", got)
+	}
+}
+
 type stubFileInfo struct {
-	name string
+	name    string
+	modTime time.Time
 }
 
 func (s stubFileInfo) Name() string       { return s.name }
 func (s stubFileInfo) Size() int64        { return 0 }
 func (s stubFileInfo) Mode() os.FileMode  { return 0 }
-func (s stubFileInfo) ModTime() time.Time { return time.Unix(0, 0) }
+func (s stubFileInfo) ModTime() time.Time { return s.modTime }
 func (s stubFileInfo) IsDir() bool        { return false }
 func (s stubFileInfo) Sys() interface{}   { return nil }
 
@@ -52,3 +153,32 @@ func TestGetMediaFormattedHonorsTimestampRange(t *testing.T) {
 		t.Fatalf("expected camera key to be preserved, got %s", media[0].CameraKey)
 	}
 }
+
+func TestGetMediaFormattedSupportsOpaqueFileNames(t *testing.T) {
+	root := t.TempDir()
+	recordingDirectory := filepath.Join(root, "recordings")
+	cloudDirectory := filepath.Join(root, "cloud")
+	if err := os.MkdirAll(cloudDirectory, 0755); err != nil {
+		t.Fatal(err)
+	}
+	fileName := "opaque recording.mp4"
+	marker := []byte(`{"filename":"opaque recording.mp4","timestamp":1785934709414}`)
+	if err := os.WriteFile(filepath.Join(cloudDirectory, models.RecordingUploadMetadataFileName(fileName)), marker, 0644); err != nil {
+		t.Fatal(err)
+	}
+	configuration := &models.Configuration{}
+	configuration.Config.Timezone = "UTC"
+
+	media := GetMediaFormatted([]os.FileInfo{stubFileInfo{name: fileName}}, recordingDirectory, configuration, models.EventFilter{})
+	if len(media) != 1 || media[0].Timestamp != "1785934709" {
+		t.Fatalf("GetMediaFormatted() = %#v", media)
+	}
+}
+
+func TestRecordingTimestampFallsBackToModificationTime(t *testing.T) {
+	want := time.Unix(1785934709, 0)
+	timestamp, ok := recordingTimestamp(stubFileInfo{name: "opaque.mp4", modTime: want}, "/tmp/recordings")
+	if !ok || timestamp != want.Unix() {
+		t.Fatalf("recordingTimestamp() = %d/%v, want %d/true", timestamp, ok, want.Unix())
+	}
+}

machinery/src/video/livehls.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 
 	mp4ff "github.com/Eyevinn/mp4ff/mp4"
-	"github.com/kerberos-io/agent/machinery/src/log"
+	log "github.com/sirupsen/logrus"
 )
 
 // LiveSegmenter turns a live stream of Annex B video samples into HLS-ready
@@ -217,7 +217,7 @@ func (ls *LiveSegmenter) buildInit() error {
 		// fall back to a manually built avcC just like the recording muxer does so
 		// those cameras still produce a valid init segment.
 		if err := trak.SetAVCDescriptor("avc1", sps, pps, true); err != nil {
-			log.Log.Warning("livehls: SetAVCDescriptor failed, using manual avcC fallback: " + err.Error())
+			log.Warn("livehls: SetAVCDescriptor failed, using manual avcC fallback: " + err.Error())
 			if fbErr := addAVCDescriptorFallback(trak, sps, pps, ls.width, ls.height); fbErr != nil {
 				return fmt.Errorf("livehls: AVC descriptor fallback: %w", fbErr)
 			}
@@ -279,7 +279,7 @@ func (ls *LiveSegmenter) WriteSample(isKeyframe bool, annexB []byte, ptsMs uint6
 	// A session must open on a random-access point; otherwise the first segment
 	// would reference frames that never arrived.
 	if !ls.started && !isKeyframe {
-		log.Log.Debug("LiveSegmenter.WriteSample(): dropping leading non-keyframe before first IDR")
+		log.Debug("LiveSegmenter.WriteSample(): dropping leading non-keyframe before first IDR")
 		return nil
 	}
 
@@ -352,7 +352,7 @@ func (ls *LiveSegmenter) openSegment(startPTS uint64) {
 	ls.seg = mp4ff.NewMediaSegment() // includes a CMAF styp box by default
 	frag, err := mp4ff.CreateFragment(ls.seqNr, ls.videoTrackID)
 	if err != nil {
-		log.Log.Error("LiveSegmenter.openSegment(): CreateFragment failed: " + err.Error())
+		log.Error("LiveSegmenter.openSegment(): CreateFragment failed: " + err.Error())
 		return
 	}
 	ls.seg.AddFragment(frag)
@@ -543,7 +543,7 @@ func (ls *LiveSegmenter) openPartFragment() {
 	ls.fragSeq++
 	frag, err := mp4ff.CreateFragment(ls.fragSeq, ls.videoTrackID)
 	if err != nil {
-		log.Log.Error("LiveSegmenter.openPartFragment(): CreateFragment failed: " + err.Error())
+		log.Error("LiveSegmenter.openPartFragment(): CreateFragment failed: " + err.Error())
 		return
 	}
 	ls.partFrag = frag

machinery/src/video/mp4.go

@@ -16,9 +16,9 @@ import (
 	"github.com/Eyevinn/mp4ff/avc"
 	mp4ff "github.com/Eyevinn/mp4ff/mp4"
 	"github.com/kerberos-io/agent/machinery/src/encryption"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/utils"
+	log "github.com/sirupsen/logrus"
 )
 
 var LastPTS uint64 = 0 // Last PTS for the current segment
@@ -216,13 +216,13 @@ func (mp4 *MP4) updateVideoParameterSetsFromAnnexB(data []byte) {
 				if needSPS {
 					mp4.SPSNALUs = [][]byte{nalu}
 					needSPS = false
-					log.Log.Warning("mp4.updateVideoParameterSetsFromAnnexB(): SPS recovered from in-band NALU")
+					log.Warn("mp4.updateVideoParameterSetsFromAnnexB(): SPS recovered from in-band NALU")
 				}
 			case 8: // PPS
 				if needPPS {
 					mp4.PPSNALUs = [][]byte{nalu}
 					needPPS = false
-					log.Log.Warning("mp4.updateVideoParameterSetsFromAnnexB(): PPS recovered from in-band NALU")
+					log.Warn("mp4.updateVideoParameterSetsFromAnnexB(): PPS recovered from in-band NALU")
 				}
 			}
 		case "H265", "HVC1":
@@ -232,19 +232,19 @@ func (mp4 *MP4) updateVideoParameterSetsFromAnnexB(data []byte) {
 				if needVPS {
 					mp4.VPSNALUs = [][]byte{nalu}
 					needVPS = false
-					log.Log.Warning("mp4.updateVideoParameterSetsFromAnnexB(): VPS recovered from in-band NALU")
+					log.Warn("mp4.updateVideoParameterSetsFromAnnexB(): VPS recovered from in-band NALU")
 				}
 			case 33: // SPS
 				if needSPS {
 					mp4.SPSNALUs = [][]byte{nalu}
 					needSPS = false
-					log.Log.Warning("mp4.updateVideoParameterSetsFromAnnexB(): SPS recovered from in-band NALU")
+					log.Warn("mp4.updateVideoParameterSetsFromAnnexB(): SPS recovered from in-band NALU")
 				}
 			case 34: // PPS
 				if needPPS {
 					mp4.PPSNALUs = [][]byte{nalu}
 					needPPS = false
-					log.Log.Warning("mp4.updateVideoParameterSetsFromAnnexB(): PPS recovered from in-band NALU")
+					log.Warn("mp4.updateVideoParameterSetsFromAnnexB(): PPS recovered from in-band NALU")
 				}
 			}
 		}
@@ -266,7 +266,12 @@ func (mp4 *MP4) flushPendingVideoSample(nextPTS uint64) bool {
 	} else {
 		// No valid nextPTS (Close case) or PTS went backwards (jitter/discontinuity)
 		if nextPTS > 0 {
-			log.Log.Warning(fmt.Sprintf("mp4.flushPendingVideoSample(): video PTS went backwards or zero duration (nextPTS=%d, prevDTS=%d), using last known duration", nextPTS, mp4.VideoFullSample.DecodeTime))
+			log.WithFields(log.Fields{
+				"component":                 "video/mp4",
+				"event":                     "invalid_video_timestamp",
+				"next_presentation_time_ms": nextPTS,
+				"previous_decode_time_ms":   mp4.VideoFullSample.DecodeTime,
+			}).Warn("Video timestamp moved backwards or produced zero duration")
 		}
 		duration = mp4.LastVideoSampleDTS
 		if duration == 0 {
@@ -282,15 +287,23 @@ func (mp4 *MP4) flushPendingVideoSample(nextPTS uint64) bool {
 	isKF := mp4.PendingSampleIsKeyframe
 	err := mp4.MultiTrackFragment.AddFullSampleToTrack(*mp4.VideoFullSample, uint32(mp4.VideoTrack))
 	if err != nil {
-		log.Log.Error("mp4.flushPendingVideoSample(): error adding sample: " + err.Error())
+		log.Error("mp4.flushPendingVideoSample(): error adding sample: " + err.Error())
 	} else {
 		mp4.SampleCount++
 	}
 	if isKF {
 		mp4.TotalKeyframesWritten++
 		mp4.FragmentKeyframeCount++
-		log.Log.Debug(fmt.Sprintf("mp4.flushPendingVideoSample(): KEYFRAME WRITTEN to trun - totalWritten=%d, fragmentKF=%d, flags=0x%08x, dur=%d, DTS=%d",
-			mp4.TotalKeyframesWritten, mp4.FragmentKeyframeCount, mp4.VideoFullSample.Sample.Flags, duration, mp4.VideoFullSample.DecodeTime))
+		log.WithFields(log.Fields{
+			"component":               "video/mp4",
+			"decode_time_ms":          mp4.VideoFullSample.DecodeTime,
+			"duration_ms":             duration,
+			"event":                   "keyframe_written",
+			"fragment_keyframe_count": mp4.FragmentKeyframeCount,
+			"sample_flags":            mp4.VideoFullSample.Sample.Flags,
+			"total_keyframes_written": mp4.TotalKeyframesWritten,
+		}).Debug("MP4 keyframe written")
+
 	}
 
 	mp4.VideoFullSample = nil
@@ -378,7 +391,14 @@ func (mp4 *MP4) AddSampleToTrack(trackID uint32, isKeyframe bool, data []byte, p
 		truncatedTail := fullGopFrames > 0 && bufferedVideo*2 < fullGopFrames
 		if closeKeyframe && truncatedTail {
 			seam = true
-			log.Log.Warning(fmt.Sprintf("mp4.AddSampleToTrack(): dropping truncated GOP at premature keyframe (interval=%d ms, min interval=%d ms, buffered video frames=%d of ~%d) - likely upstream loop/restart discontinuity", gap, mp4.MinKeyframeGapMs, bufferedVideo, fullGopFrames))
+			log.WithFields(log.Fields{
+				"buffered_video_frames":   bufferedVideo,
+				"component":               "video/mp4",
+				"event":                   "truncated_gop_dropped",
+				"expected_gop_frames":     fullGopFrames,
+				"keyframe_interval_ms":    gap,
+				"minimum_keyframe_gap_ms": mp4.MinKeyframeGapMs,
+			}).Warn("Dropping truncated GOP at premature keyframe")
 		}
 		mp4.LastKeyframeGapMs = gap
 		if !seam && (mp4.MinKeyframeGapMs == 0 || gap < mp4.MinKeyframeGapMs) {
@@ -483,7 +503,7 @@ func (mp4 *MP4) commitBufferedGOP() {
 	mp4.gopBuffer = nil // detach so commitSampleToTrack never observes a half-cleared buffer
 	for _, s := range buffered {
 		if err := mp4.commitSampleToTrack(s.trackID, s.isKeyframe, s.data, s.pts, s.compositionOffset); err != nil {
-			log.Log.Error("mp4.commitBufferedGOP(): " + err.Error())
+			log.Error("mp4.commitBufferedGOP(): " + err.Error())
 		}
 	}
 }
@@ -499,8 +519,18 @@ func (mp4 *MP4) commitSampleToTrack(trackID uint32, isKeyframe bool, data []byte
 		if mp4.Start {
 			elapsedDbg = pts - mp4.FragmentStartRawPTS
 		}
-		log.Log.Debug(fmt.Sprintf("mp4.AddSampleToTrack(): KEYFRAME #%d received - PTS=%d, size=%d, elapsed=%dms, started=%t, segment=%d, fragKF=%d",
-			mp4.TotalKeyframesReceived, pts, len(data), elapsedDbg, mp4.Start, mp4.SegmentCount, mp4.FragmentKeyframeCount))
+		log.WithFields(log.Fields{
+			"component":                "video/mp4",
+			"elapsed_ms":               elapsedDbg,
+			"event":                    "keyframe_received",
+			"fragment_keyframe_count":  mp4.FragmentKeyframeCount,
+			"presentation_time_ms":     pts,
+			"recording_started":        mp4.Start,
+			"sample_bytes":             len(data),
+			"segment_count":            mp4.SegmentCount,
+			"total_keyframes_received": mp4.TotalKeyframesReceived,
+		}).Debug("MP4 keyframe received")
+
 	}
 
 	if isKeyframe {
@@ -524,8 +554,15 @@ func (mp4 *MP4) commitSampleToTrack(trackID uint32, isKeyframe bool, data []byte
 					mp4.flushPendingVideoSample(pts)
 				}
 
-				log.Log.Debug(fmt.Sprintf("mp4.AddSampleToTrack(): FLUSHING segment #%d - keyframes_in_fragment=%d, totalKF_received=%d, totalKF_written=%d",
-					mp4.SegmentCount, mp4.FragmentKeyframeCount, mp4.TotalKeyframesReceived, mp4.TotalKeyframesWritten))
+				log.WithFields(log.Fields{
+					"component":                "video/mp4",
+					"event":                    "segment_flush_started",
+					"fragment_keyframe_count":  mp4.FragmentKeyframeCount,
+					"segment_count":            mp4.SegmentCount,
+					"total_keyframes_received": mp4.TotalKeyframesReceived,
+					"total_keyframes_written":  mp4.TotalKeyframesWritten,
+				}).Debug("Flushing MP4 segment")
+
 				mp4.MoofBoxes = mp4.MoofBoxes + 1
 				mp4.MoofBoxSizes = append(mp4.MoofBoxSizes, int64(mp4.Segment.Size()))
 				// Track the segment's duration and base decode time for sidx.
@@ -536,7 +573,7 @@ func (mp4 *MP4) commitSampleToTrack(trackID uint32, isKeyframe bool, data []byte
 				mp4.SegmentBaseDecTimes = append(mp4.SegmentBaseDecTimes, mp4.FragmentStartDTS)
 				err := mp4.Segment.Encode(mp4.Writer)
 				if err != nil {
-					log.Log.Error("mp4.AddSampleToTrack(): error encoding segment: " + err.Error())
+					log.Error("mp4.AddSampleToTrack(): error encoding segment: " + err.Error())
 				}
 				mp4.Segments = append(mp4.Segments, mp4.Segment)
 			}
@@ -552,7 +589,7 @@ func (mp4 *MP4) commitSampleToTrack(trackID uint32, isKeyframe bool, data []byte
 			// Create a video fragment
 			multiTrackFragment, err := mp4ff.CreateMultiTrackFragment(uint32(mp4.SegmentCount), mp4.TrackIDs)
 			if err != nil {
-				log.Log.Error("mp4.AddSampleToTrack(): error creating multi track fragment: " + err.Error())
+				log.Error("mp4.AddSampleToTrack(): error creating multi track fragment: " + err.Error())
 			}
 			mp4.MultiTrackFragment = multiTrackFragment
 			seg.AddFragment(multiTrackFragment)
@@ -585,7 +622,14 @@ func (mp4 *MP4) commitSampleToTrack(trackID uint32, isKeyframe bool, data []byte
 			if err == nil {
 				// Flush previous pending sample before storing the new one
 				if mp4.VideoFullSample != nil {
-					log.Log.Debug("Adding sample to track " + fmt.Sprintf("%d, PTS: %d, size: %d, Keyframe: %t", trackID, pts, len(lengthPrefixed), isKeyframe))
+					log.WithFields(log.Fields{
+						"bytes":     len(lengthPrefixed),
+						"component": "mp4",
+						"event":     "video_sample_added",
+						"keyframe":  isKeyframe,
+						"pts":       pts,
+						"track_id":  trackID,
+					}).Trace("Adding MP4 video sample")
 					mp4.flushPendingVideoSample(pts)
 				}
 
@@ -628,7 +672,11 @@ func (mp4 *MP4) commitSampleToTrack(trackID uint32, isKeyframe bool, data []byte
 					sampleToAdd.Sample.Size = uint32(len(aac[7:]))
 					err := mp4.MultiTrackFragment.AddFullSampleToTrack(sampleToAdd, trackID)
 					if err != nil {
-						log.Log.Error("mp4.AddSampleToTrack(): error adding sample to track " + fmt.Sprintf("%d: %v", trackID, err))
+						log.WithError(err).WithFields(log.Fields{
+							"component": "video/mp4",
+							"event":     "audio_sample_write_failed",
+							"track_id":  trackID,
+						}).Error("Failed to add audio sample to MP4 track")
 					}
 				})
 			}
@@ -657,11 +705,17 @@ func (mp4 *MP4) Close(config *models.Config) {
 	// a recording is never a loop seam, so it must always be written out.
 	mp4.commitBufferedGOP()
 
-	log.Log.Info(fmt.Sprintf("mp4.Close(): KEYFRAME SUMMARY - totalReceived=%d, totalWritten=%d, segments=%d, lastFragmentKF=%d",
-		mp4.TotalKeyframesReceived, mp4.TotalKeyframesWritten, mp4.SegmentCount, mp4.FragmentKeyframeCount))
+	log.WithFields(log.Fields{
+		"component":                "video/mp4",
+		"event":                    "recording_keyframe_summary",
+		"fragment_keyframe_count":  mp4.FragmentKeyframeCount,
+		"segment_count":            mp4.SegmentCount,
+		"total_keyframes_received": mp4.TotalKeyframesReceived,
+		"total_keyframes_written":  mp4.TotalKeyframesWritten,
+	}).Info("MP4 recording keyframe summary")
 
 	if mp4.VideoTotalDuration == 0 && mp4.AudioTotalDuration == 0 {
-		log.Log.Error("mp4.Close(): no video or audio samples added, removing empty MP4 file")
+		log.Error("mp4.Close(): no video or audio samples added, removing empty MP4 file")
 		mp4.Writer.Flush()
 		_ = mp4.FileWriter.Sync()
 		_ = mp4.FileWriter.Close()
@@ -690,7 +744,7 @@ func (mp4 *MP4) Close(config *models.Config) {
 				sampleToAdd.Sample.Size = uint32(len(aac[7:]))
 				err := mp4.MultiTrackFragment.AddFullSampleToTrack(sampleToAdd, uint32(mp4.AudioTrack))
 				if err != nil {
-					log.Log.Error("mp4.Close(): error adding final audio sample: " + err.Error())
+					log.Error("mp4.Close(): error adding final audio sample: " + err.Error())
 				}
 			})
 			mp4.AudioFullSample = nil
@@ -712,14 +766,14 @@ func (mp4 *MP4) Close(config *models.Config) {
 
 		err := mp4.Segment.Encode(mp4.Writer)
 		if err != nil {
-			log.Log.Error("mp4.Close(): error encoding last segment: " + err.Error())
+			log.Error("mp4.Close(): error encoding last segment: " + err.Error())
 		}
 	}
 
 	mp4.Writer.Flush()
 	// Ensure all segment data is on disk before we overwrite the placeholder at offset 0.
 	if err := mp4.FileWriter.Sync(); err != nil {
-		log.Log.Error("mp4.Close(): error syncing file: " + err.Error())
+		log.Error("mp4.Close(): error syncing file: " + err.Error())
 	}
 
 	// Now we have all the moof and mdat boxes written to the file.
@@ -747,8 +801,14 @@ func (mp4 *MP4) Close(config *models.Config) {
 		actualVideoDuration += d
 	}
 	if actualVideoDuration != mp4.VideoTotalDuration {
-		log.Log.Warning(fmt.Sprintf("mp4.Close(): duration mismatch: accumulated VideoTotalDuration=%d, sum of segment durations=%d (diff=%d ms)",
-			mp4.VideoTotalDuration, actualVideoDuration, int64(mp4.VideoTotalDuration)-int64(actualVideoDuration)))
+		log.WithFields(log.Fields{
+			"accumulated_duration_ms": mp4.VideoTotalDuration,
+			"component":               "video/mp4",
+			"difference_ms":           int64(mp4.VideoTotalDuration) - int64(actualVideoDuration),
+			"event":                   "duration_mismatch",
+			"segment_duration_ms":     actualVideoDuration,
+		}).Warn("MP4 video duration does not match segment durations")
+
 	}
 
 	// Set the creation time and modification time for the moov box.
@@ -790,21 +850,40 @@ func (mp4 *MP4) Close(config *models.Config) {
 		init.AddEmptyTrack(videoTimescale, "video", "und")
 		includePS := true
 		spsNALUs, ppsNALUs := normalizeH264ParameterSets(mp4.SPSNALUs, mp4.PPSNALUs)
-		log.Log.Debug("mp4.Close(): AVC parameter sets: SPS=" + formatNaluDebug(spsNALUs) + ", PPS=" + formatNaluDebug(ppsNALUs))
+		log.WithFields(log.Fields{
+			"component": "video/mp4",
+			"event":     "avc_parameter_sets",
+			"pps":       formatNaluDebug(ppsNALUs),
+			"sps":       formatNaluDebug(spsNALUs),
+		}).Debug("AVC parameter sets normalized")
 		if len(spsNALUs) == 0 || len(ppsNALUs) == 0 {
 			// An avcC without both SPS and PPS is invalid: downstream FFmpeg-based
 			// pipelines decoding this file will report "non-existing PPS 0 referenced"
 			// and fail to extract any frame. Surface it loudly so the capture-side
 			// parameter-set handling can be diagnosed.
-			log.Log.Error(fmt.Sprintf("mp4.Close(): incomplete H264 parameter sets (SPS=%d, PPS=%d) - the avcC will be invalid and downstream decoders will report 'non-existing PPS 0 referenced'", len(spsNALUs), len(ppsNALUs)))
+			log.WithFields(log.Fields{
+				"component": "video/mp4",
+				"event":     "avc_parameter_sets_incomplete",
+				"pps_count": len(ppsNALUs),
+				"sps_count": len(spsNALUs),
+			}).Error("Incomplete H264 parameter sets will produce an invalid AVC descriptor")
 		}
 		err := init.Moov.Traks[0].SetAVCDescriptor("avc1", spsNALUs, ppsNALUs, includePS)
 		if err != nil {
-			log.Log.Error("mp4.Close(): error setting AVC descriptor: " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component": "video/mp4",
+				"event":     "avc_descriptor_failed",
+			}).Error("Failed to set AVC descriptor")
 			if fallbackErr := addAVCDescriptorFallback(init.Moov.Traks[0], spsNALUs, ppsNALUs, uint16(mp4.width), uint16(mp4.height)); fallbackErr != nil {
-				log.Log.Error("mp4.Close(): error setting AVC descriptor fallback: " + fallbackErr.Error())
+				log.WithError(fallbackErr).WithFields(log.Fields{
+					"component": "video/mp4",
+					"event":     "avc_descriptor_fallback_failed",
+				}).Error("Failed to set fallback AVC descriptor")
 			} else {
-				log.Log.Warning("mp4.Close(): AVC descriptor fallback used due to SPS parse error")
+				log.WithFields(log.Fields{
+					"component": "video/mp4",
+					"event":     "avc_descriptor_fallback_used",
+				}).Warn("Using fallback AVC descriptor")
 			}
 		}
 		init.Moov.Traks[0].Tkhd.Duration = actualVideoDuration
@@ -823,15 +902,30 @@ func (mp4 *MP4) Close(config *models.Config) {
 		init.AddEmptyTrack(videoTimescale, "video", "und")
 		includePS := true
 		vpsNALUs, spsNALUs, ppsNALUs := normalizeH265ParameterSets(mp4.VPSNALUs, mp4.SPSNALUs, mp4.PPSNALUs)
-		log.Log.Debug("mp4.Close(): HEVC parameter sets: VPS=" + formatNaluDebug(vpsNALUs) + ", SPS=" + formatNaluDebug(spsNALUs) + ", PPS=" + formatNaluDebug(ppsNALUs))
+		log.WithFields(log.Fields{
+			"component": "video/mp4",
+			"event":     "hevc_parameter_sets",
+			"pps":       formatNaluDebug(ppsNALUs),
+			"sps":       formatNaluDebug(spsNALUs),
+			"vps":       formatNaluDebug(vpsNALUs),
+		}).Debug("HEVC parameter sets normalized")
 		if len(vpsNALUs) == 0 || len(spsNALUs) == 0 || len(ppsNALUs) == 0 {
 			// An hvcC missing VPS/SPS/PPS is invalid and downstream FFmpeg-based
 			// pipelines will fail to decode the recording. Surface it loudly.
-			log.Log.Error(fmt.Sprintf("mp4.Close(): incomplete H265 parameter sets (VPS=%d, SPS=%d, PPS=%d) - the hvcC will be invalid and downstream decoders will fail to process the recording", len(vpsNALUs), len(spsNALUs), len(ppsNALUs)))
+			log.WithFields(log.Fields{
+				"component": "video/mp4",
+				"event":     "hevc_parameter_sets_incomplete",
+				"pps_count": len(ppsNALUs),
+				"sps_count": len(spsNALUs),
+				"vps_count": len(vpsNALUs),
+			}).Error("Incomplete H265 parameter sets will produce an invalid HEVC descriptor")
 		}
 		err := init.Moov.Traks[0].SetHEVCDescriptor("hvc1", vpsNALUs, spsNALUs, ppsNALUs, [][]byte{}, includePS)
 		if err != nil {
-			log.Log.Error("mp4.Close(): error setting HEVC descriptor: " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component": "video/mp4",
+				"event":     "hevc_descriptor_failed",
+			}).Error("Failed to set HEVC descriptor")
 		}
 		init.Moov.Traks[0].Tkhd.Duration = actualVideoDuration
 		init.Moov.Traks[0].Tkhd.Width = mp4ff.Fixed32(uint32(mp4.width) << 16)
@@ -876,7 +970,7 @@ func (mp4 *MP4) Close(config *models.Config) {
 		// Set the subtitle descriptor
 		err := init.Moov.Traks[2].SetWvttDescriptor("")
 		if err != nil {
-			//log.Log.Error("mp4.Close(): error setting VTT descriptor: " + err.Error())
+			// log.Error("mp4.Close(): error setting VTT descriptor: " + err.Error())
 			//return
 		}
 		init.Moov.Traks[2].Mdia.Hdlr.Name = "agent " + utils.VERSION
@@ -924,14 +1018,14 @@ func (mp4 *MP4) Close(config *models.Config) {
 	block, _ := pem.Decode(pemBytes)
 
 	if block == nil {
-		//log.Log.Error("mp4.Close(): error decoding PEM block containing private key")
+		// log.Error("mp4.Close(): error decoding PEM block containing private key")
 		//return
 	} else {
 		// Parse private key
 		b := block.Bytes
 		key, err := x509.ParsePKCS8PrivateKey(b)
 		if err != nil {
-			//log.Log.Error("mp4.Close(): error parsing private key: " + err.Error())
+			// log.Error("mp4.Close(): error parsing private key: " + err.Error())
 			//return
 		} else {
 			// Conver key to *rsa.PrivateKey
@@ -944,7 +1038,7 @@ func (mp4 *MP4) Close(config *models.Config) {
 				uuid.UnknownPayload = signature
 				init.Moov.AddChild(uuid)
 			} else {
-				//log.Log.Error("mp4.Close(): error signing fingerprint: " + err.Error())
+				// log.Error("mp4.Close(): error signing fingerprint: " + err.Error())
 			}
 		}
 	}
@@ -979,7 +1073,7 @@ func (mp4 *MP4) Close(config *models.Config) {
 	// and re-encode with the corrected value.
 	var initBuf bytes.Buffer
 	if err := init.Encode(&initBuf); err != nil {
-		log.Log.Error("mp4.Close(): error encoding init segment: " + err.Error())
+		log.Error("mp4.Close(): error encoding init segment: " + err.Error())
 	}
 
 	initSize := int64(initBuf.Len())
@@ -991,7 +1085,7 @@ func (mp4 *MP4) Close(config *models.Config) {
 	if len(mp4.SegmentDurations) > 0 {
 		if mp4.FreeBoxSize < initSize {
 			// Avoid computing a negative offset and wrapping it to uint64.
-			log.Log.Error("mp4.Close(): FreeBoxSize is smaller than initSize; skipping sidx FirstOffset adjustment")
+			log.Error("mp4.Close(): FreeBoxSize is smaller than initSize; skipping sidx FirstOffset adjustment")
 		} else {
 			firstOffset := uint64(mp4.FreeBoxSize - initSize)
 			// Find the sidx we added and update its FirstOffset
@@ -1004,19 +1098,24 @@ func (mp4 *MP4) Close(config *models.Config) {
 			// Re-encode with the corrected FirstOffset (same size, no layout change)
 			initBuf.Reset()
 			if err := init.Encode(&initBuf); err != nil {
-				log.Log.Error("mp4.Close(): error re-encoding init segment: " + err.Error())
+				log.Error("mp4.Close(): error re-encoding init segment: " + err.Error())
 			}
 			initSize = int64(initBuf.Len())
 		}
 	}
 
 	if initSize > mp4.FreeBoxSize {
-		log.Log.Error(fmt.Sprintf("mp4.Close(): init segment (%d bytes) exceeds reserved space (%d bytes), file may be corrupt", initSize, mp4.FreeBoxSize))
+		log.WithFields(log.Fields{
+			"component":      "video/mp4",
+			"event":          "init_segment_oversized",
+			"init_bytes":     initSize,
+			"reserved_bytes": mp4.FreeBoxSize,
+		}).Error("MP4 init segment exceeds reserved space")
 	}
 
 	// Write the init segment at the beginning of the file, overwriting the free box placeholder.
 	if _, err := mp4.FileWriter.WriteAt(initBuf.Bytes(), 0); err != nil {
-		log.Log.Error("mp4.Close(): error writing init segment: " + err.Error())
+		log.Error("mp4.Close(): error writing init segment: " + err.Error())
 	}
 
 	// Fill any remaining reserved space with a new (smaller) free box so
@@ -1026,15 +1125,15 @@ func (mp4 *MP4) Close(config *models.Config) {
 		newFree := mp4ff.NewFreeBox(make([]byte, remainingSize-8))
 		var freeBuf bytes.Buffer
 		if err := newFree.Encode(&freeBuf); err != nil {
-			log.Log.Error("mp4.Close(): error encoding free box: " + err.Error())
+			log.Error("mp4.Close(): error encoding free box: " + err.Error())
 		}
 		if _, err := mp4.FileWriter.WriteAt(freeBuf.Bytes(), initSize); err != nil {
-			log.Log.Error("mp4.Close(): error writing free box: " + err.Error())
+			log.Error("mp4.Close(): error writing free box: " + err.Error())
 		}
 	}
 
 	if err := mp4.FileWriter.Sync(); err != nil {
-		log.Log.Error("mp4.Close(): error syncing file: " + err.Error())
+		log.Error("mp4.Close(): error syncing file: " + err.Error())
 	}
 	mp4.FileWriter.Close()
 }

machinery/src/webrtc/aac_transcoder_ffmpeg.go

@@ -193,10 +193,9 @@ import "C"
 
 import (
 	"errors"
-	"fmt"
 	"unsafe"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
+	log "github.com/sirupsen/logrus"
 	"github.com/zaf/g711"
 )
 
@@ -216,7 +215,7 @@ func NewAACTranscoder() (*AACTranscoder, error) {
 	if h == nil {
 		return nil, errors.New("failed to create AAC transcoder (FFmpeg AAC decoder not available?)")
 	}
-	log.Log.Info("webrtc.aac_transcoder: AAC → G.711 µ-law transcoder initialised (FFmpeg)")
+	log.Info("webrtc.aac_transcoder: AAC → G.711 µ-law transcoder initialised (FFmpeg)")
 	return &AACTranscoder{handle: h}, nil
 }
 
@@ -250,9 +249,16 @@ func (t *AACTranscoder) Transcode(adtsData []byte) ([]byte, error) {
 
 	// Log resampler details once.
 	if t.handle.swr_initialized == 1 && t.handle.in_sample_rate != 0 {
-		log.Log.Info(fmt.Sprintf(
-			"webrtc.aac_transcoder: first output – resampling %d Hz / %d ch → 8000 Hz mono → µ-law",
-			int(t.handle.in_sample_rate), int(t.handle.in_channels)))
+		log.WithFields(log.Fields{
+			"component":        "webrtc",
+			"event":            "aac_resampler_initialized",
+			"input_channels":   int(t.handle.in_channels),
+			"input_sample_hz":  int(t.handle.in_sample_rate),
+			"output_channels":  1,
+			"output_codec":     "PCMU",
+			"output_sample_hz": 8000,
+		}).Info("AAC resampler initialized")
+
 		// Prevent repeated logging by zeroing the field we check.
 		t.handle.in_sample_rate = 0
 	}
@@ -265,6 +271,6 @@ func (t *AACTranscoder) Close() {
 	if t != nil && t.handle != nil {
 		C.aac_transcoder_destroy(t.handle)
 		t.handle = nil
-		log.Log.Info("webrtc.aac_transcoder: transcoder closed")
+		log.Info("webrtc.aac_transcoder: transcoder closed")
 	}
 }

machinery/src/webrtc/aac_transcoder_stub.go

@@ -15,7 +15,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
+	log "github.com/sirupsen/logrus"
 )
 
 // AACTranscodingAvailable reports whether AAC→PCMU transcoding
@@ -45,7 +45,7 @@ func NewAACTranscoder() (*AACTranscoder, error) {
 	if err != nil {
 		return nil, errors.New("AAC transcoding not available: ffmpeg binary not found in PATH")
 	}
-	log.Log.Info("webrtc.aac_transcoder: using ffmpeg binary at " + ffmpegPath)
+	log.Info("webrtc.aac_transcoder: using ffmpeg binary at " + ffmpegPath)
 
 	cmd := exec.Command(
 		ffmpegPath,
@@ -96,19 +96,19 @@ func NewAACTranscoder() (*AACTranscoder, error) {
 				buffered := t.outBuf.Len()
 				t.outMu.Unlock()
 				if buffered <= 8192 || buffered%16000 == 0 {
-					log.Log.Debug("webrtc.aac_transcoder: ffmpeg produced PCMU bytes, buffered=" + strconv.Itoa(buffered))
+					log.Debug("webrtc.aac_transcoder: ffmpeg produced PCMU bytes, buffered=" + strconv.Itoa(buffered))
 				}
 			}
 			if readErr != nil {
 				if readErr != io.EOF {
-					log.Log.Warning("webrtc.aac_transcoder: stdout reader stopped: " + readErr.Error())
+					log.Warn("webrtc.aac_transcoder: stdout reader stopped: " + readErr.Error())
 				}
 				return
 			}
 		}
 	}()
 
-	log.Log.Info("webrtc.aac_transcoder: AAC → PCMU transcoder initialised (ffmpeg process)")
+	log.Info("webrtc.aac_transcoder: AAC → PCMU transcoder initialised (ffmpeg process)")
 	return t, nil
 }
 
@@ -129,22 +129,22 @@ func (t *AACTranscoder) Transcode(adtsData []byte) ([]byte, error) {
 		return nil, err
 	}
 	if len(adtsData) <= 512 || len(adtsData)%1024 == 0 {
-		log.Log.Debug("webrtc.aac_transcoder: wrote AAC bytes to ffmpeg, input=" + strconv.Itoa(len(adtsData)))
+		log.Debug("webrtc.aac_transcoder: wrote AAC bytes to ffmpeg, input=" + strconv.Itoa(len(adtsData)))
 	}
 
 	deadline := time.Now().Add(75 * time.Millisecond)
 	for {
 		data := t.readAvailable()
 		if len(data) > 0 {
-			log.Log.Debug("webrtc.aac_transcoder: returning PCMU bytes=" + strconv.Itoa(len(data)))
+			log.Debug("webrtc.aac_transcoder: returning PCMU bytes=" + strconv.Itoa(len(data)))
 			return data, nil
 		}
 
 		if time.Now().After(deadline) {
 			if stderr := t.stderrString(); stderr != "" {
-				log.Log.Warning("webrtc.aac_transcoder: no output before deadline, ffmpeg stderr: " + stderr)
+				log.Warn("webrtc.aac_transcoder: no output before deadline, ffmpeg stderr: " + stderr)
 			} else {
-				log.Log.Debug("webrtc.aac_transcoder: no PCMU output before deadline")
+				log.Debug("webrtc.aac_transcoder: no PCMU output before deadline")
 			}
 			return nil, nil
 		}
@@ -198,7 +198,7 @@ func (t *AACTranscoder) Close() {
 			_ = t.cmd.Process.Kill()
 			_, _ = t.cmd.Process.Wait()
 			if stderr := t.stderrString(); stderr != "" {
-				log.Log.Info("webrtc.aac_transcoder: ffmpeg stderr on close: " + stderr)
+				log.Info("webrtc.aac_transcoder: ffmpeg stderr on close: " + stderr)
 			}
 		}
 	})

machinery/src/webrtc/broadcaster.go

@@ -4,9 +4,9 @@ import (
 	"io"
 	"sync"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	pionWebRTC "github.com/pion/webrtc/v4"
 	pionMedia "github.com/pion/webrtc/v4/pkg/media"
+	log "github.com/sirupsen/logrus"
 )
 
 const (
@@ -74,12 +74,20 @@ func (b *TrackBroadcaster) AddPeer(sessionKey string) (*pionWebRTC.TrackLocalSta
 				if err == io.ErrClosedPipe {
 					return
 				}
-				log.Log.Error("webrtc.broadcaster.peerWriter(): error writing sample for " + sessionKey + ": " + err.Error())
+				log.WithError(err).WithFields(log.Fields{
+					"component":  "webrtc",
+					"event":      "sample_write_failed",
+					"session_id": sessionKey,
+				}).Error("Failed to write WebRTC sample")
 			}
 		}
 	}()
 
-	log.Log.Info("webrtc.broadcaster.AddPeer(): added peer track for " + sessionKey)
+	log.WithFields(log.Fields{
+		"component":  "webrtc",
+		"event":      "peer_added",
+		"session_id": sessionKey,
+	}).Info("WebRTC peer added")
 	return track, nil
 }
 
@@ -95,7 +103,11 @@ func (b *TrackBroadcaster) RemovePeer(sessionKey string) {
 	if exists {
 		close(pt.samples)
 		<-pt.done // wait for writer goroutine to finish
-		log.Log.Info("webrtc.broadcaster.RemovePeer(): removed peer track for " + sessionKey)
+		log.WithFields(log.Fields{
+			"component":  "webrtc",
+			"event":      "peer_removed",
+			"session_id": sessionKey,
+		}).Info("WebRTC peer removed")
 	}
 }
 
@@ -110,7 +122,11 @@ func (b *TrackBroadcaster) WriteSample(sample pionMedia.Sample) {
 		select {
 		case pt.samples <- sample:
 		default:
-			log.Log.Warning("webrtc.broadcaster.WriteSample(): dropping sample for slow peer " + sessionKey)
+			log.WithFields(log.Fields{
+				"component":  "webrtc",
+				"event":      "slow_peer_sample_dropped",
+				"session_id": sessionKey,
+			}).Warn("Dropping WebRTC sample for slow peer")
 		}
 	}
 }

machinery/src/webrtc/main.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/base64"
 	"encoding/json"
-	"fmt"
 	"io"
 	"strconv"
 	"strings"
@@ -13,9 +12,9 @@ import (
 	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/capture"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
+	log "github.com/sirupsen/logrus"
 
 	mqtt "github.com/eclipse/paho.mqtt.golang"
 	"github.com/pion/interceptor"
@@ -180,7 +179,7 @@ func cleanupPeerConnection(sessionKey string, wrapper *peerConnectionWrapper) {
 	wrapper.closeOnce.Do(func() {
 		if wrapper.connected.Swap(false) {
 			count := globalConnectionManager.DecrementPeerCount()
-			log.Log.Info("webrtc.main.cleanupPeerConnection(): Peer disconnected. Active peers: " + strconv.FormatInt(count, 10))
+			log.Info("webrtc.main.cleanupPeerConnection(): Peer disconnected. Active peers: " + strconv.FormatInt(count, 10))
 		}
 
 		// Remove per-peer tracks from broadcasters so the fan-out stops
@@ -196,7 +195,7 @@ func cleanupPeerConnection(sessionKey string, wrapper *peerConnectionWrapper) {
 
 		if wrapper.conn != nil {
 			if err := wrapper.conn.Close(); err != nil {
-				log.Log.Error("webrtc.main.cleanupPeerConnection(): error closing peer connection: " + err.Error())
+				log.Error("webrtc.main.cleanupPeerConnection(): error closing peer connection: " + err.Error())
 			}
 		}
 
@@ -226,10 +225,35 @@ func CreateWebRTC(name string, stunServers []string, turnServers []string, turnS
 	}
 }
 
+func nonEmptyICEURLs(urls []string) []string {
+	nonEmpty := make([]string, 0, len(urls))
+	for _, uri := range urls {
+		if uri = strings.TrimSpace(uri); uri != "" {
+			nonEmpty = append(nonEmpty, uri)
+		}
+	}
+	return nonEmpty
+}
+
+func buildICEServers(w WebRTC) []pionWebRTC.ICEServer {
+	iceServers := make([]pionWebRTC.ICEServer, 0, 2)
+	if stunURLs := nonEmptyICEURLs(w.StunServers); len(stunURLs) > 0 {
+		iceServers = append(iceServers, pionWebRTC.ICEServer{URLs: stunURLs})
+	}
+	if turnURLs := nonEmptyICEURLs(w.TurnServers); len(turnURLs) > 0 {
+		iceServers = append(iceServers, pionWebRTC.ICEServer{
+			URLs:       turnURLs,
+			Username:   w.TurnServersUsername,
+			Credential: w.TurnServersCredential,
+		})
+	}
+	return iceServers
+}
+
 func (w WebRTC) DecodeSessionDescription(data string) ([]byte, error) {
 	sd, err := base64.StdEncoding.DecodeString(data)
 	if err != nil {
-		log.Log.Error("webrtc.main.DecodeSessionDescription(): " + err.Error())
+		log.Error("webrtc.main.DecodeSessionDescription(): " + err.Error())
 		return []byte{}, err
 	}
 	return sd, nil
@@ -244,9 +268,9 @@ func (w WebRTC) CreateOffer(sd []byte) pionWebRTC.SessionDescription {
 }
 
 func RegisterCandidates(key string, candidate models.ReceiveHDCandidatesPayload) {
-	log.Log.Info("webrtc.main.RegisterCandidates(): " + candidate.Candidate)
+	log.Info("webrtc.main.RegisterCandidates(): " + candidate.Candidate)
 	if !globalConnectionManager.QueueCandidate(key, candidate.Candidate) {
-		log.Log.Info("webrtc.main.RegisterCandidates(): channel is full, dropping candidate")
+		log.Info("webrtc.main.RegisterCandidates(): channel is full, dropping candidate")
 	}
 }
 
@@ -280,18 +304,18 @@ func RegisterDefaultInterceptors(mediaEngine *pionWebRTC.MediaEngine, intercepto
 
 func publishSignalingMessageAsync(mqttClient mqtt.Client, topic string, payload []byte, description string) {
 	if mqttClient == nil {
-		log.Log.Error("webrtc.main.publishSignalingMessageAsync(): mqtt client is nil for " + description)
+		log.Error("webrtc.main.publishSignalingMessageAsync(): mqtt client is nil for " + description)
 		return
 	}
 
 	token := mqttClient.Publish(topic, 2, false, payload)
 	go func() {
 		if !token.WaitTimeout(5 * time.Second) {
-			log.Log.Warning("webrtc.main.publishSignalingMessageAsync(): timed out publishing " + description)
+			log.Warn("webrtc.main.publishSignalingMessageAsync(): timed out publishing " + description)
 			return
 		}
 		if err := token.Error(); err != nil {
-			log.Log.Error("webrtc.main.publishSignalingMessageAsync(): failed publishing " + description + ": " + err.Error())
+			log.Error("webrtc.main.publishSignalingMessageAsync(): failed publishing " + description + ": " + err.Error())
 		}
 	}()
 }
@@ -299,7 +323,7 @@ func publishSignalingMessageAsync(mqttClient mqtt.Client, topic string, payload
 func sendCandidateSignal(configuration *models.Configuration, mqttClient mqtt.Client, hubKey string, handshake models.LiveHDHandshake, candidateJSON []byte) {
 	if handshake.Signaling != nil && handshake.Signaling.SendCandidate != nil {
 		if err := handshake.Signaling.SendCandidate(handshake.Payload.SessionID, string(candidateJSON)); err != nil {
-			log.Log.Error("webrtc.main.sendCandidateSignal(): " + err.Error())
+			log.Error("webrtc.main.sendCandidateSignal(): " + err.Error())
 		}
 		return
 	}
@@ -318,7 +342,7 @@ func sendCandidateSignal(configuration *models.Configuration, mqttClient mqtt.Cl
 	if err == nil {
 		publishSignalingMessageAsync(mqttClient, "kerberos/hub/"+hubKey, payload, "ICE candidate for session "+handshake.Payload.SessionID)
 	} else {
-		log.Log.Info("webrtc.main.sendCandidateSignal(): while packaging mqtt message: " + err.Error())
+		log.Info("webrtc.main.sendCandidateSignal(): while packaging mqtt message: " + err.Error())
 	}
 }
 
@@ -327,7 +351,7 @@ func sendAnswerSignal(configuration *models.Configuration, mqttClient mqtt.Clien
 
 	if handshake.Signaling != nil && handshake.Signaling.SendAnswer != nil {
 		if err := handshake.Signaling.SendAnswer(handshake.Payload.SessionID, encodedAnswer); err != nil {
-			log.Log.Error("webrtc.main.sendAnswerSignal(): " + err.Error())
+			log.Error("webrtc.main.sendAnswerSignal(): " + err.Error())
 		}
 		return
 	}
@@ -346,7 +370,7 @@ func sendAnswerSignal(configuration *models.Configuration, mqttClient mqtt.Clien
 	if err == nil {
 		publishSignalingMessageAsync(mqttClient, "kerberos/hub/"+hubKey, payload, "SDP answer for session "+handshake.Payload.SessionID)
 	} else {
-		log.Log.Info("webrtc.main.sendAnswerSignal(): while packaging mqtt message: " + err.Error())
+		log.Info("webrtc.main.sendAnswerSignal(): while packaging mqtt message: " + err.Error())
 	}
 }
 
@@ -368,7 +392,11 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 	// first so we start from a clean slate. Without this, the new request would
 	// race against a stale PC that still owns the per-peer broadcaster tracks.
 	if globalConnectionManager.CloseExistingPeerConnection(sessionKey) {
-		log.Log.Info("webrtc.main.InitializeWebRTCConnection(): closed stale peer connection for session " + handshakePayload.SessionID)
+		log.WithFields(log.Fields{
+			"component":  "webrtc",
+			"event":      "stale_peer_closed",
+			"session_id": handshakePayload.SessionID,
+		}).Debug("Closed stale WebRTC peer connection")
 	}
 	// Drain/reset the candidate channel too \u2014 leftover candidates from the
 	// prior session are not valid for the new ICE agent.
@@ -387,7 +415,7 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 
 		mediaEngine := &pionWebRTC.MediaEngine{}
 		if err := mediaEngine.RegisterDefaultCodecs(); err != nil {
-			log.Log.Error("webrtc.main.InitializeWebRTCConnection(): something went wrong registering codecs for media engine: " + err.Error())
+			log.Error("webrtc.main.InitializeWebRTCConnection(): something went wrong registering codecs for media engine: " + err.Error())
 		}
 
 		// Create a InterceptorRegistry. This is the user configurable RTP/RTCP Pipeline.
@@ -423,21 +451,29 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 
 		peerConnection, err := api.NewPeerConnection(
 			pionWebRTC.Configuration{
-				ICEServers: []pionWebRTC.ICEServer{
-					{
-						URLs: w.StunServers,
-					},
-					{
-						URLs:       w.TurnServers,
-						Username:   w.TurnServersUsername,
-						Credential: w.TurnServersCredential,
-					},
-				},
+				ICEServers:         buildICEServers(*w),
 				ICETransportPolicy: policy,
 			},
 		)
 
-		if err == nil && peerConnection != nil {
+		if err != nil {
+			globalConnectionManager.CloseCandidateChannel(sessionKey)
+			log.Error("webrtc.main.InitializeWebRTCConnection(): failed to create peer connection: " + err.Error() +
+				" (STUN configured: " + strconv.FormatBool(strings.TrimSpace(config.STUNURI) != "") +
+				", TURN configured: " + strconv.FormatBool(strings.TrimSpace(config.TURNURI) != "") +
+				", TURN username configured: " + strconv.FormatBool(config.TURNUsername != "") +
+				", TURN credential configured: " + strconv.FormatBool(config.TURNPassword != "") +
+				", ForceTurn: " + config.ForceTurn + ")")
+
+			return
+		}
+		if peerConnection == nil {
+			globalConnectionManager.CloseCandidateChannel(sessionKey)
+			log.Error("webrtc.main.InitializeWebRTCConnection(): failed to create peer connection: pion returned a nil connection")
+			return
+		}
+
+		{
 
 			// Create context for this connection
 			ctx, cancel := context.WithCancel(context.Background())
@@ -456,17 +492,17 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 			if videoBroadcaster != nil {
 				peerVideoTrack, trackErr := videoBroadcaster.AddPeer(sessionKey)
 				if trackErr != nil {
-					log.Log.Error("webrtc.main.InitializeWebRTCConnection(): error creating per-peer video track: " + trackErr.Error())
+					log.Error("webrtc.main.InitializeWebRTCConnection(): error creating per-peer video track: " + trackErr.Error())
 					cleanupPeerConnection(sessionKey, wrapper)
 					return
 				}
 				if videoSender, err = peerConnection.AddTrack(peerVideoTrack); err != nil {
-					log.Log.Error("webrtc.main.InitializeWebRTCConnection(): error adding video track: " + err.Error())
+					log.Error("webrtc.main.InitializeWebRTCConnection(): error adding video track: " + err.Error())
 					cleanupPeerConnection(sessionKey, wrapper)
 					return
 				}
 			} else {
-				log.Log.Info("webrtc.main.InitializeWebRTCConnection(): video track is nil, skipping video")
+				log.Info("webrtc.main.InitializeWebRTCConnection(): video track is nil, skipping video")
 			}
 
 			// Read incoming RTCP packets
@@ -475,7 +511,7 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 			if videoSender != nil {
 				go func() {
 					defer func() {
-						log.Log.Info("webrtc.main.InitializeWebRTCConnection(): video RTCP reader stopped")
+						log.Info("webrtc.main.InitializeWebRTCConnection(): video RTCP reader stopped")
 					}()
 					rtcpBuf := make([]byte, rtcpBufferSize)
 					for {
@@ -496,17 +532,17 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 			if audioBroadcaster != nil {
 				peerAudioTrack, trackErr := audioBroadcaster.AddPeer(sessionKey)
 				if trackErr != nil {
-					log.Log.Error("webrtc.main.InitializeWebRTCConnection(): error creating per-peer audio track: " + trackErr.Error())
+					log.Error("webrtc.main.InitializeWebRTCConnection(): error creating per-peer audio track: " + trackErr.Error())
 					cleanupPeerConnection(sessionKey, wrapper)
 					return
 				}
 				if audioSender, err = peerConnection.AddTrack(peerAudioTrack); err != nil {
-					log.Log.Error("webrtc.main.InitializeWebRTCConnection(): error adding audio track: " + err.Error())
+					log.Error("webrtc.main.InitializeWebRTCConnection(): error adding audio track: " + err.Error())
 					cleanupPeerConnection(sessionKey, wrapper)
 					return
 				}
 			} else {
-				log.Log.Info("webrtc.main.InitializeWebRTCConnection(): audio track is nil, skipping audio")
+				log.Info("webrtc.main.InitializeWebRTCConnection(): audio track is nil, skipping audio")
 			}
 
 			// Read incoming RTCP packets
@@ -515,7 +551,7 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 			if audioSender != nil {
 				go func() {
 					defer func() {
-						log.Log.Info("webrtc.main.InitializeWebRTCConnection(): audio RTCP reader stopped")
+						log.Info("webrtc.main.InitializeWebRTCConnection(): audio RTCP reader stopped")
 					}()
 					rtcpBuf := make([]byte, rtcpBufferSize)
 					for {
@@ -533,12 +569,13 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 
 			// Log ICE connection state changes for diagnostics
 			peerConnection.OnICEConnectionStateChange(func(iceState pionWebRTC.ICEConnectionState) {
-				log.Log.Info("webrtc.main.InitializeWebRTCConnection(): ICE connection state changed to: " + iceState.String() +
+				log.Info("webrtc.main.InitializeWebRTCConnection(): ICE connection state changed to: " + iceState.String() +
 					" (session: " + handshakePayload.SessionID + ")")
+
 			})
 
 			peerConnection.OnConnectionStateChange(func(connectionState pionWebRTC.PeerConnectionState) {
-				log.Log.Info("webrtc.main.InitializeWebRTCConnection(): connection state changed to: " + connectionState.String() +
+				log.Info("webrtc.main.InitializeWebRTCConnection(): connection state changed to: " + connectionState.String() +
 					" (session: " + handshakePayload.SessionID + ")")
 
 				switch connectionState {
@@ -547,10 +584,15 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 					// Start a grace period timer; if we don't recover, then cleanup.
 					wrapper.disconnectMu.Lock()
 					if wrapper.disconnectTimer == nil {
-						log.Log.Info("webrtc.main.InitializeWebRTCConnection(): peer disconnected, waiting " +
+						log.Info("webrtc.main.InitializeWebRTCConnection(): peer disconnected, waiting " +
 							disconnectGracePeriod.String() + " for recovery (session: " + handshakePayload.SessionID + ")")
+
 						wrapper.disconnectTimer = time.AfterFunc(disconnectGracePeriod, func() {
-							log.Log.Info("webrtc.main.InitializeWebRTCConnection(): disconnect grace period expired, closing connection (session: " + handshakePayload.SessionID + ")")
+							log.WithFields(log.Fields{
+								"component":  "webrtc",
+								"event":      "disconnect_grace_expired",
+								"session_id": handshakePayload.SessionID,
+							}).Info("WebRTC disconnect grace period expired")
 							cleanupPeerConnection(sessionKey, wrapper)
 						})
 					}
@@ -582,13 +624,17 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 					if wrapper.disconnectTimer != nil {
 						wrapper.disconnectTimer.Stop()
 						wrapper.disconnectTimer = nil
-						log.Log.Info("webrtc.main.InitializeWebRTCConnection(): connection recovered from disconnected state (session: " + handshakePayload.SessionID + ")")
+						log.WithFields(log.Fields{
+							"component":  "webrtc",
+							"event":      "connection_recovered",
+							"session_id": handshakePayload.SessionID,
+						}).Info("WebRTC connection recovered")
 					}
 					wrapper.disconnectMu.Unlock()
 
 					if wrapper.connected.CompareAndSwap(false, true) {
 						count := globalConnectionManager.IncrementPeerCount()
-						log.Log.Info("webrtc.main.InitializeWebRTCConnection(): Peer connected. Active peers: " + strconv.FormatInt(count, 10))
+						log.Info("webrtc.main.InitializeWebRTCConnection(): Peer connected. Active peers: " + strconv.FormatInt(count, 10))
 					}
 				}
 			})
@@ -600,10 +646,11 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 			peerConnection.OnICECandidate(func(candidate *pionWebRTC.ICECandidate) {
 
 				if candidate == nil {
-					log.Log.Info("webrtc.main.InitializeWebRTCConnection(): ICE gathering complete (candidate is nil)")
+					log.Info("webrtc.main.InitializeWebRTCConnection(): ICE gathering complete (candidate is nil)")
 					if !hasRelayCandidates {
-						log.Log.Error("webrtc.main.InitializeWebRTCConnection(): WARNING - No TURN (relay) candidates were gathered! TURN servers: " +
+						log.Error("webrtc.main.InitializeWebRTCConnection(): WARNING - No TURN (relay) candidates were gathered! TURN servers: " +
 							config.TURNURI + ", Username: " + config.TURNUsername + ", ForceTurn: " + config.ForceTurn)
+
 					}
 					return
 				}
@@ -632,7 +679,7 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 					hasRelayCandidates = true
 				}
 
-				log.Log.Info("webrtc.main.InitializeWebRTCConnection(): ICE candidate received - Type: " + candidateType +
+				log.Info("webrtc.main.InitializeWebRTCConnection(): ICE candidate received - Type: " + candidateType +
 					", Candidate: " + candidateStr)
 
 				//  Create a config map
@@ -641,9 +688,9 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 				if err == nil {
 					valueMap["candidate"] = string(candateBinary)
 					valueMap["session_id"] = handshakePayload.SessionID
-					log.Log.Info("webrtc.main.InitializeWebRTCConnection(): sending " + candidateType + " candidate to hub")
+					log.Info("webrtc.main.InitializeWebRTCConnection(): sending " + candidateType + " candidate to hub")
 				} else {
-					log.Log.Error("webrtc.main.InitializeWebRTCConnection(): failed to marshal candidate: " + err.Error())
+					log.Error("webrtc.main.InitializeWebRTCConnection(): failed to marshal candidate: " + err.Error())
 				}
 
 				sendCandidateSignal(configuration, mqttClient, hubKey, handshake, candateBinary)
@@ -651,14 +698,18 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 
 			offer := w.CreateOffer(sd)
 			if err = peerConnection.SetRemoteDescription(offer); err != nil {
-				log.Log.Error("webrtc.main.InitializeWebRTCConnection(): something went wrong while setting remote description: " + err.Error())
+				log.Error("webrtc.main.InitializeWebRTCConnection(): something went wrong while setting remote description: " + err.Error())
 				cleanupPeerConnection(sessionKey, wrapper)
 				return
 			}
 
 			go func() {
 				defer func() {
-					log.Log.Info("webrtc.main.InitializeWebRTCConnection(): candidate processor stopped for session: " + handshakePayload.SessionID)
+					log.WithFields(log.Fields{
+						"component":  "webrtc",
+						"event":      "candidate_processor_stopped",
+						"session_id": handshakePayload.SessionID,
+					}).Debug("WebRTC candidate processor stopped")
 				}()
 
 				// Process remote candidates only after the remote description is set.
@@ -672,14 +723,14 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 						if !ok {
 							return
 						}
-						log.Log.Info("webrtc.main.InitializeWebRTCConnection(): Received candidate from channel: " + candidate)
+						log.Info("webrtc.main.InitializeWebRTCConnection(): Received candidate from channel: " + candidate)
 						candidateInit, decodeErr := decodeICECandidate(candidate)
 						if decodeErr != nil {
-							log.Log.Error("webrtc.main.InitializeWebRTCConnection(): error decoding candidate: " + decodeErr.Error())
+							log.Error("webrtc.main.InitializeWebRTCConnection(): error decoding candidate: " + decodeErr.Error())
 							continue
 						}
 						if candidateErr := peerConnection.AddICECandidate(candidateInit); candidateErr != nil {
-							log.Log.Error("webrtc.main.InitializeWebRTCConnection(): error adding candidate: " + candidateErr.Error())
+							log.Error("webrtc.main.InitializeWebRTCConnection(): error adding candidate: " + candidateErr.Error())
 						}
 					}
 				}
@@ -687,11 +738,11 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 
 			answer, err := peerConnection.CreateAnswer(nil)
 			if err != nil {
-				log.Log.Error("webrtc.main.InitializeWebRTCConnection(): something went wrong while creating answer: " + err.Error())
+				log.Error("webrtc.main.InitializeWebRTCConnection(): something went wrong while creating answer: " + err.Error())
 				cleanupPeerConnection(sessionKey, wrapper)
 				return
 			} else if err = peerConnection.SetLocalDescription(answer); err != nil {
-				log.Log.Error("webrtc.main.InitializeWebRTCConnection(): something went wrong while setting local description: " + err.Error())
+				log.Error("webrtc.main.InitializeWebRTCConnection(): something went wrong while setting local description: " + err.Error())
 				cleanupPeerConnection(sessionKey, wrapper)
 				return
 			}
@@ -699,13 +750,13 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 			// Store peer connection in manager
 			globalConnectionManager.AddPeerConnection(sessionKey, wrapper)
 
-			log.Log.Info("webrtc.main.InitializeWebRTCConnection(): Send SDP answer")
+			log.Info("webrtc.main.InitializeWebRTCConnection(): Send SDP answer")
 
 			sendAnswerSignal(configuration, mqttClient, hubKey, handshake, answer)
 		}
 	} else {
 		globalConnectionManager.CloseCandidateChannel(sessionKey)
-		log.Log.Error("webrtc.main.InitializeWebRTCConnection(): failed to decode remote session description: " + err.Error())
+		log.Error("webrtc.main.InitializeWebRTCConnection(): failed to decode remote session description: " + err.Error())
 	}
 }
 
@@ -716,7 +767,7 @@ func NewVideoBroadcaster(streams []packets.Stream) *TrackBroadcaster {
 			return NewTrackBroadcaster(pionWebRTC.MimeTypeH264, "video", trackStreamID)
 		}
 	}
-	log.Log.Error("webrtc.main.NewVideoBroadcaster(): no H264 stream found")
+	log.Error("webrtc.main.NewVideoBroadcaster(): no H264 stream found")
 	return nil
 }
 
@@ -739,12 +790,16 @@ func NewAudioBroadcaster(streams []packets.Stream) *TrackBroadcaster {
 		}
 	}
 	if hasAAC {
-		log.Log.Info("webrtc.main.NewAudioBroadcaster(): AAC detected, creating PCMU audio track for transcoded output")
+		log.Info("webrtc.main.NewAudioBroadcaster(): AAC detected, creating PCMU audio track for transcoded output")
 		return NewTrackBroadcaster(pionWebRTC.MimeTypePCMU, "audio", trackStreamID)
 	} else if len(audioCodecNames) > 0 {
-		log.Log.Error(fmt.Sprintf("webrtc.main.NewAudioBroadcaster(): no supported audio codec found (detected: %s; supported: OPUS, PCM_MULAW, PCM_ALAW)", strings.Join(audioCodecNames, ", ")))
+		log.WithFields(log.Fields{
+			"component":       "webrtc",
+			"detected_codecs": audioCodecNames,
+			"event":           "unsupported_audio_codecs",
+		}).Error("No supported WebRTC audio codec found")
 	} else {
-		log.Log.Info("webrtc.main.NewAudioBroadcaster(): no audio stream found in camera feed")
+		log.Info("webrtc.main.NewAudioBroadcaster(): no audio stream found in camera feed")
 	}
 	return nil
 }
@@ -753,7 +808,7 @@ func NewVideoTrack(streams []packets.Stream) *pionWebRTC.TrackLocalStaticSample
 	mimeType := pionWebRTC.MimeTypeH264
 	outboundVideoTrack, err := pionWebRTC.NewTrackLocalStaticSample(pionWebRTC.RTPCodecCapability{MimeType: mimeType}, "video", trackStreamID)
 	if err != nil {
-		log.Log.Error("webrtc.main.NewVideoTrack(): error creating video track: " + err.Error())
+		log.Error("webrtc.main.NewVideoTrack(): error creating video track: " + err.Error())
 		return nil
 	}
 	return outboundVideoTrack
@@ -780,18 +835,22 @@ func NewAudioTrack(streams []packets.Stream) *pionWebRTC.TrackLocalStaticSample
 	if mimeType == "" {
 		if hasAAC {
 			mimeType = pionWebRTC.MimeTypePCMU
-			log.Log.Info("webrtc.main.NewAudioTrack(): AAC detected, creating PCMU audio track for transcoded output")
+			log.Info("webrtc.main.NewAudioTrack(): AAC detected, creating PCMU audio track for transcoded output")
 		} else if len(audioCodecNames) > 0 {
-			log.Log.Error(fmt.Sprintf("webrtc.main.NewAudioTrack(): no supported audio codec found (detected: %s; supported: OPUS, PCM_MULAW, PCM_ALAW)", strings.Join(audioCodecNames, ", ")))
+			log.WithFields(log.Fields{
+				"component":       "webrtc",
+				"detected_codecs": audioCodecNames,
+				"event":           "unsupported_audio_codecs",
+			}).Error("No supported WebRTC audio codec found")
 			return nil
 		} else {
-			log.Log.Info("webrtc.main.NewAudioTrack(): no audio stream found in camera feed")
+			log.Info("webrtc.main.NewAudioTrack(): no audio stream found in camera feed")
 			return nil
 		}
 	}
 	outboundAudioTrack, err := pionWebRTC.NewTrackLocalStaticSample(pionWebRTC.RTPCodecCapability{MimeType: mimeType}, "audio", trackStreamID)
 	if err != nil {
-		log.Log.Error("webrtc.main.NewAudioTrack(): error creating audio track: " + err.Error())
+		log.Error("webrtc.main.NewAudioTrack(): error creating audio track: " + err.Error())
 		return nil
 	}
 	return outboundAudioTrack
@@ -946,7 +1005,7 @@ func processVideoPacket(pkt packets.Packet, state *streamState, videoBroadcaster
 
 	if config.Capture.ForwardWebRTC == "true" {
 		// Remote forwarding not yet implemented
-		log.Log.Debug("webrtc.main.processVideoPacket(): remote forwarding not implemented")
+		log.Debug("webrtc.main.processVideoPacket(): remote forwarding not implemented")
 		return
 	}
 
@@ -975,25 +1034,49 @@ func processAudioPacket(pkt packets.Packet, state *streamState, audioBroadcaster
 		if transcoder == nil {
 			state.aacErrors++
 			if state.aacErrors <= 3 || state.aacErrors%100 == 0 {
-				log.Log.Warning(fmt.Sprintf("webrtc.main.processAudioPacket(): AAC packet dropped because transcoder is nil (aac_packets=%d, input_bytes=%d)", state.aacPacketsSeen, len(pkt.Data)))
+				log.WithFields(log.Fields{
+					"aac_packets": state.aacPacketsSeen,
+					"component":   "webrtc",
+					"event":       "aac_packet_dropped",
+					"input_bytes": len(pkt.Data),
+					"reason":      "transcoder_unavailable",
+				}).Warn("AAC packet dropped")
 			}
 			return // no transcoder – silently drop
 		}
 		pcmu, err := transcoder.Transcode(pkt.Data)
 		if err != nil {
 			state.aacErrors++
-			log.Log.Error("webrtc.main.processAudioPacket(): AAC transcode error: " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"aac_packets": state.aacPacketsSeen,
+				"component":   "webrtc",
+				"event":       "aac_transcode_failed",
+				"input_bytes": len(pkt.Data),
+			}).Error("Failed to transcode AAC packet")
 			return
 		}
 		if len(pcmu) == 0 {
 			state.aacNoOutput++
 			if state.aacNoOutput <= 5 || state.aacNoOutput%100 == 0 {
-				log.Log.Debug(fmt.Sprintf("webrtc.main.processAudioPacket(): AAC packet produced no PCMU output yet (aac_packets=%d, no_output=%d, input_bytes=%d)", state.aacPacketsSeen, state.aacNoOutput, len(pkt.Data)))
+				log.WithFields(log.Fields{
+					"aac_packets":     state.aacPacketsSeen,
+					"component":       "webrtc",
+					"event":           "aac_output_buffering",
+					"input_bytes":     len(pkt.Data),
+					"no_output_count": state.aacNoOutput,
+				}).Debug("AAC decoder produced no PCMU output yet")
 			}
 			return // decoder still buffering
 		}
 		if state.aacPacketsSeen <= 5 || state.aacPacketsSeen%100 == 0 {
-			log.Log.Info(fmt.Sprintf("webrtc.main.processAudioPacket(): AAC transcoded to PCMU (aac_packets=%d, input_bytes=%d, output_bytes=%d, peers=%d)", state.aacPacketsSeen, len(pkt.Data), len(pcmu), audioBroadcaster.PeerCount()))
+			log.WithFields(log.Fields{
+				"aac_packets":  state.aacPacketsSeen,
+				"component":    "webrtc",
+				"event":        "aac_packet_transcoded",
+				"input_bytes":  len(pkt.Data),
+				"output_bytes": len(pcmu),
+				"peer_count":   audioBroadcaster.PeerCount(),
+			}).Debug("AAC packet transcoded to PCMU")
 		}
 		audioData = pcmu
 	}
@@ -1004,7 +1087,15 @@ func processAudioPacket(pkt packets.Packet, state *streamState, audioBroadcaster
 		state.lastAudioSample.Duration = sampleDuration(pkt, state.lastAudioSample.PacketTimestamp, 20*time.Millisecond)
 		state.audioSamplesSent++
 		if state.audioSamplesSent <= 5 || state.audioSamplesSent%100 == 0 {
-			log.Log.Debug(fmt.Sprintf("webrtc.main.processAudioPacket(): queueing audio sample (samples=%d, codec=%s, bytes=%d, duration_ms=%d, peers=%d)", state.audioSamplesSent, pkt.Codec, len(state.lastAudioSample.Data), state.lastAudioSample.Duration.Milliseconds(), audioBroadcaster.PeerCount()))
+			log.WithFields(log.Fields{
+				"codec":          pkt.Codec,
+				"component":      "webrtc",
+				"duration_ms":    state.lastAudioSample.Duration.Milliseconds(),
+				"event":          "audio_sample_queued",
+				"peer_count":     audioBroadcaster.PeerCount(),
+				"sample_bytes":   len(state.lastAudioSample.Data),
+				"samples_queued": state.audioSamplesSent,
+			}).Debug("WebRTC audio sample queued")
 		}
 		audioBroadcaster.WriteSample(*state.lastAudioSample)
 	}
@@ -1027,7 +1118,7 @@ func WriteToTrack(livestreamCursor *packets.QueueCursor, configuration *models.C
 
 	// Check if at least one broadcaster is available
 	if videoBroadcaster == nil && audioBroadcaster == nil {
-		log.Log.Error("webrtc.main.WriteToTrack(): both video and audio broadcasters are nil, cannot proceed")
+		log.Error("webrtc.main.WriteToTrack(): both video and audio broadcasters are nil, cannot proceed")
 		return
 	}
 
@@ -1035,26 +1126,30 @@ func WriteToTrack(livestreamCursor *packets.QueueCursor, configuration *models.C
 	codecs := detectCodecs(rtspClient)
 
 	if !codecs.hasValidCodecs() {
-		log.Log.Error("webrtc.main.WriteToTrack(): no valid video or audio codec found")
+		log.Error("webrtc.main.WriteToTrack(): no valid video or audio codec found")
 		return
 	}
 
 	// Create AAC transcoder if needed (AAC → G.711 µ-law).
 	var aacTranscoder *AACTranscoder
 	if codecs.hasAAC && audioBroadcaster != nil {
-		log.Log.Info(fmt.Sprintf("webrtc.main.WriteToTrack(): AAC audio detected, creating transcoder (audio_peers=%d)", audioBroadcaster.PeerCount()))
+		log.WithFields(log.Fields{
+			"component":  "webrtc",
+			"event":      "aac_transcoder_starting",
+			"peer_count": audioBroadcaster.PeerCount(),
+		}).Info("Starting AAC transcoder")
 		t, err := NewAACTranscoder()
 		if err != nil {
-			log.Log.Error("webrtc.main.WriteToTrack(): failed to create AAC transcoder: " + err.Error())
+			log.Error("webrtc.main.WriteToTrack(): failed to create AAC transcoder: " + err.Error())
 		} else {
 			aacTranscoder = t
-			log.Log.Info("webrtc.main.WriteToTrack(): AAC transcoder created successfully")
+			log.Info("webrtc.main.WriteToTrack(): AAC transcoder created successfully")
 			defer aacTranscoder.Close()
 		}
 	}
 
 	if config.Capture.TranscodingWebRTC == "true" {
-		log.Log.Info("webrtc.main.WriteToTrack(): transcoding config enabled")
+		log.Info("webrtc.main.WriteToTrack(): transcoding config enabled")
 	}
 
 	// Initialize streaming state
@@ -1064,14 +1159,25 @@ func WriteToTrack(livestreamCursor *packets.QueueCursor, configuration *models.C
 	}
 
 	defer func() {
-		log.Log.Info(fmt.Sprintf("webrtc.main.WriteToTrack(): audio summary packets=%d aac_packets=%d sent=%d aac_no_output=%d aac_errors=%d peers=%d", state.audioPacketsSeen, state.aacPacketsSeen, state.audioSamplesSent, state.aacNoOutput, state.aacErrors, func() int {
+		peerCount := func() int {
 			if audioBroadcaster == nil {
 				return 0
 			}
 			return audioBroadcaster.PeerCount()
-		}()))
+		}()
+		log.WithFields(log.Fields{
+			"aac_errors":    state.aacErrors,
+			"aac_no_output": state.aacNoOutput,
+			"aac_packets":   state.aacPacketsSeen,
+			"audio_packets": state.audioPacketsSeen,
+			"component":     "webrtc",
+			"event":         "audio_stream_summary",
+			"peer_count":    peerCount,
+			"samples_sent":  state.audioSamplesSent,
+		}).Info("WebRTC audio stream summary")
+
 		writeFinalSamples(state, videoBroadcaster, audioBroadcaster)
-		log.Log.Info("webrtc.main.WriteToTrack(): stopped writing to track")
+		log.Info("webrtc.main.WriteToTrack(): stopped writing to track")
 	}()
 
 	var pkt packets.Packet
@@ -1105,7 +1211,7 @@ func WriteToTrack(livestreamCursor *packets.QueueCursor, configuration *models.C
 		// wait for a recent keyframe before resuming video.
 		if shouldDropPacketForLatency(pkt) {
 			if !state.catchingUp {
-				log.Log.Warning("webrtc.main.WriteToTrack(): stream is lagging behind, dropping old packets until the next recent keyframe")
+				log.Warn("webrtc.main.WriteToTrack(): stream is lagging behind, dropping old packets until the next recent keyframe")
 			}
 			state.catchingUp = true
 			state.start = false
@@ -1122,7 +1228,7 @@ func WriteToTrack(livestreamCursor *packets.QueueCursor, configuration *models.C
 			state.catchingUp = false
 			state.start = false
 			state.receivedKeyFrame = false
-			log.Log.Info("webrtc.main.WriteToTrack(): caught up with live stream at a recent keyframe")
+			log.Info("webrtc.main.WriteToTrack(): caught up with live stream at a recent keyframe")
 		}
 
 		// Wait for first keyframe before processing

machinery/src/webrtc/main_test.go

@@ -0,0 +1,38 @@
+package webrtc
+
+import "testing"
+
+func TestBuildICEServersOmitsEmptyURLs(t *testing.T) {
+	webRTC := CreateWebRTC("camera", []string{""}, []string{""}, "", "")
+
+	iceServers := buildICEServers(*webRTC)
+
+	if len(iceServers) != 0 {
+		t.Fatalf("buildICEServers() returned %d servers, want 0", len(iceServers))
+	}
+}
+
+func TestBuildICEServersIncludesConfiguredURLs(t *testing.T) {
+	webRTC := CreateWebRTC(
+		"camera",
+		[]string{"", " stun:turn-fra1.kerberos.io:3478 "},
+		[]string{" turn:turn-fra1.kerberos.io:3478 "},
+		"username",
+		"credential",
+	)
+
+	iceServers := buildICEServers(*webRTC)
+
+	if len(iceServers) != 2 {
+		t.Fatalf("buildICEServers() returned %d servers, want 2", len(iceServers))
+	}
+	if got := iceServers[0].URLs[0]; got != "stun:turn-fra1.kerberos.io:3478" {
+		t.Fatalf("STUN URL = %q, want trimmed URL", got)
+	}
+	if got := iceServers[1].URLs[0]; got != "turn:turn-fra1.kerberos.io:3478" {
+		t.Fatalf("TURN URL = %q, want trimmed URL", got)
+	}
+	if iceServers[1].Username != "username" || iceServers[1].Credential != "credential" {
+		t.Fatal("TURN credentials were not preserved")
+	}
+}

machinery/verify-moq-devcontainer.sh

@@ -17,7 +17,7 @@ binary="${TMPDIR:-/tmp}/agent-moq"
 trap 'rm -f "$binary"' EXIT
 
 echo "==> Linking the MoQ Agent"
-GOWORK=off go build -tags moq -o "$binary" ./main.go
+GOWORK=off go build -tags moq -o "$binary" .
 
 echo "==> Running the linked binary"
 "$binary" -action version

ui/src/App.jsx

@@ -1,7 +1,6 @@
 import React from 'react';
 import PropTypes from 'prop-types';
 import { withTranslation } from 'react-i18next';
-import uuid from 'uuidv4';
 import {
   connect as connectWS,
   disconnect as disconnectWS,
@@ -27,67 +26,106 @@ import { logout } from './actions';
 import config from './config';
 import { getDashboardInformation } from './actions/agent';
 import LanguageSelect from './components/LanguageSelect/LanguageSelect';
+import websocketClientId from './websocket';
 import logo from './header-minimal-logo-36x36.svg';
 import '@kerberos-io/ui/lib/index.css';
 import './App.scss';
 
 // eslint-disable-next-line react/prefer-stateless-function
-class App extends React.Component {
+export class App extends React.Component {
   componentDidMount() {
-    const { dispatchGetDashboardInformation, dispatchConnect } = this.props;
-    dispatchGetDashboardInformation();
+    const { dispatchConnect } = this.props;
+    this.dashboardRequestInFlight = false;
+    this.handleVisibilityChange = this.handleVisibilityChange.bind(this);
     dispatchConnect();
+    this.refreshDashboardInformation(true);
 
-    const connectInterval = interval(1000);
-    this.connectionSubscription = connectInterval.subscribe(() => {
-      const { connected } = this.props;
-      if (connected) {
-        // Already connected
-      } else {
-        dispatchConnect();
-      }
-    });
+    document.addEventListener(
+      'visibilitychange',
+      this.handleVisibilityChange,
+      false
+    );
 
     const interval$ = interval(5000);
     this.subscription = interval$.subscribe(() => {
-      dispatchGetDashboardInformation();
+      this.refreshDashboardInformation();
     });
   }
 
   componentDidUpdate(prevProps) {
     // We are connected again, lets fire the initial events.
-    const { connected, dispatchSend, dispatchConnect } = this.props;
+    const { connected, dispatchSend } = this.props;
     const { connected: connectedPrev } = prevProps;
     if (connectedPrev === false && connected === true) {
       const message = {
-        client_id: uuid(),
+        client_id: websocketClientId,
         message_type: 'hello',
       };
       dispatchSend(message);
     }
-
-    // We disconnected, let's try to connect again
-    if (connectedPrev === true && connected === false) {
-      dispatchConnect();
-    }
   }
 
   componentWillUnmount() {
-    this.subscription.unsubscribe();
-    this.connectionSubscription.unsubscribe();
+    this.dashboardRequestInFlight = false;
+    document.removeEventListener(
+      'visibilitychange',
+      this.handleVisibilityChange,
+      false
+    );
+
+    if (this.subscription) {
+      this.subscription.unsubscribe();
+      this.subscription = null;
+    }
+
     const message = {
-      client_id: uuid(),
+      client_id: websocketClientId,
       message_type: 'goodbye',
     };
-    const { dispatchSend, dispatchDisconnect } = this.props;
-    dispatchSend(message);
+    const { connected, dispatchSend, dispatchDisconnect } = this.props;
+    if (connected) {
+      dispatchSend(message);
+    }
     dispatchDisconnect();
   }
 
+  handleVisibilityChange() {
+    if (!this.isDocumentHidden()) {
+      this.refreshDashboardInformation();
+    }
+  }
+
   getCurrentTimestamp() {
     return Math.round(Date.now() / 1000);
   }
 
+  isDocumentHidden() {
+    return typeof document !== 'undefined' && document.hidden;
+  }
+
+  refreshDashboardInformation(force = false) {
+    if (this.dashboardRequestInFlight) {
+      return;
+    }
+
+    if (!force && this.isDocumentHidden()) {
+      return;
+    }
+
+    this.dashboardRequestInFlight = true;
+    const { dispatchGetDashboardInformation } = this.props;
+    dispatchGetDashboardInformation(
+      () => {
+        this.dashboardRequestInFlight = false;
+      },
+      (error) => {
+        this.dashboardRequestInFlight = false;
+        // eslint-disable-next-line no-console
+        console.warn('Unable to refresh dashboard information.', error);
+      }
+    );
+  }
+
   render() {
     const { t, connected } = this.props;
     const { children, username, dashboard, dispatchLogout } = this.props;
@@ -226,8 +264,8 @@ const mapDispatchToProps = (dispatch) => ({
   },
   dispatchDisconnect: () => dispatch(disconnectWS()),
   dispatchSend: (message) => dispatch(send(message)),
-  dispatchGetDashboardInformation: (dashboard, success, error) =>
-    dispatch(getDashboardInformation(dashboard, success, error)),
+  dispatchGetDashboardInformation: (onSuccess, onError) =>
+    dispatch(getDashboardInformation(onSuccess, onError)),
 });
 
 App.propTypes = {

ui/src/App.test.jsx

@@ -0,0 +1,159 @@
+/* eslint-env jest */
+/* eslint-disable react/prop-types */
+import React from 'react';
+import { interval } from 'rxjs';
+import { App } from './App';
+
+const subscriptions = [];
+
+jest.mock('rxjs', () => ({
+  interval: jest.fn(),
+}));
+
+jest.mock('@kerberos-io/ui', () => {
+  const Mock = ({ children }) => <div>{children}</div>;
+  return {
+    Badge: Mock,
+    Main: Mock,
+    MainBody: Mock,
+    Gradient: Mock,
+    Sidebar: Mock,
+    Navigation: Mock,
+    NavigationSection: Mock,
+    NavigationItem: Mock,
+    NavigationGroup: Mock,
+    Profilebar: Mock,
+    Icon: Mock,
+  };
+});
+
+jest.mock('./components/LanguageSelect/LanguageSelect', () => () => null);
+jest.mock('./actions', () => ({ logout: jest.fn() }));
+jest.mock('./actions/agent', () => ({
+  getDashboardInformation: jest.fn(),
+}));
+jest.mock('./config', () => ({
+  MODE: 'release',
+  URL: 'http://localhost',
+  WS_URL: 'ws://localhost/ws',
+}));
+jest.mock('react-redux', () => ({
+  connect: () => (Component) => Component,
+}));
+jest.mock('react-i18next', () => ({
+  withTranslation: () => (Component) => Component,
+}));
+jest.mock('react-router-dom', () => ({
+  Link: ({ children }) => <>{children}</>,
+}));
+jest.mock('@giantmachines/redux-websocket', () => ({
+  connect: jest.fn(),
+  disconnect: jest.fn(),
+  send: jest.fn(),
+}));
+jest.mock('uuidv4', () => ({
+  __esModule: true,
+  default: jest.fn(() => 'uuid-1'),
+}));
+
+describe('App lifecycle refresh behaviour', () => {
+  const baseProps = () => ({
+    t: (value) => value,
+    dispatchLogout: jest.fn(),
+    dispatchConnect: jest.fn(),
+    dispatchDisconnect: jest.fn(),
+    dispatchSend: jest.fn(),
+    dispatchGetDashboardInformation: jest.fn((onSuccess) => {
+      if (onSuccess) {
+        onSuccess();
+      }
+    }),
+    children: [],
+    username: 'admin',
+    connected: false,
+    dashboard: {
+      cloudOnline: 0,
+      offlineMode: 'false',
+    },
+  });
+
+  beforeEach(() => {
+    subscriptions.length = 0;
+    jest.clearAllMocks();
+    interval.mockImplementation(() => ({
+      subscribe: jest.fn((callback) => {
+        const subscription = {
+          callback,
+          unsubscribe: jest.fn(),
+        };
+        subscriptions.push(subscription);
+        return subscription;
+      }),
+    }));
+    Object.defineProperty(document, 'hidden', {
+      configurable: true,
+      writable: true,
+      value: false,
+    });
+  });
+
+  it('uses a single dashboard interval and skips hidden or in-flight refreshes', () => {
+    const props = baseProps();
+    const app = new App(props);
+    app.props = props;
+
+    app.componentDidMount();
+
+    expect(props.dispatchConnect).toHaveBeenCalledTimes(1);
+    expect(props.dispatchGetDashboardInformation).toHaveBeenCalledTimes(1);
+    expect(interval).toHaveBeenCalledWith(5000);
+    expect(subscriptions).toHaveLength(1);
+
+    document.hidden = true;
+    subscriptions[0].callback();
+    expect(props.dispatchGetDashboardInformation).toHaveBeenCalledTimes(1);
+
+    document.hidden = false;
+    app.dashboardRequestInFlight = true;
+    subscriptions[0].callback();
+    expect(props.dispatchGetDashboardInformation).toHaveBeenCalledTimes(1);
+
+    app.dashboardRequestInFlight = false;
+    subscriptions[0].callback();
+    expect(props.dispatchGetDashboardInformation).toHaveBeenCalledTimes(2);
+  });
+
+  it('sends hello after reconnect without issuing a second manual reconnect', () => {
+    const props = {
+      ...baseProps(),
+      connected: true,
+    };
+    const app = new App(props);
+    app.props = props;
+
+    app.componentDidUpdate({
+      ...props,
+      connected: false,
+    });
+    expect(props.dispatchSend).toHaveBeenCalledWith(
+      expect.objectContaining({
+        client_id: 'uuid-1',
+        message_type: 'hello',
+      })
+    );
+
+    props.dispatchSend.mockClear();
+    props.dispatchConnect.mockClear();
+    app.props = {
+      ...props,
+      connected: false,
+    };
+    app.componentDidUpdate({
+      ...props,
+      connected: true,
+    });
+
+    expect(props.dispatchSend).not.toHaveBeenCalled();
+    expect(props.dispatchConnect).not.toHaveBeenCalled();
+  });
+});

ui/src/api/agent.js

@@ -158,7 +158,9 @@ export function doVerifyCamera(streamType, config, onSuccess, onError) {
 }
 
 export function doGetDashboardInformation(onSuccess, onError) {
-  const endpoint = API.get(`dashboard`);
+  const endpoint = API.get(`dashboard`, {
+    timeout: 15000,
+  });
   endpoint
     .then((res) => {
       if (res.status !== 200) {