Record approved expiring exception for exact shared signing key

Preserve the HIGH finding and auditable approval; fail closed on expiry, file or finding changes, and keep all other security gates enforced.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

.circleci/config.yml

@@ -1,32 +0,0 @@
-# Golang CircleCI 2.0 configuration file
-#
-# Check https://circleci.com/docs/2.0/language-go/ for more details
-version: 2
-jobs:
-  machinery:
-    docker:
-      - image: kerberos/base:0a50dc9
-    working_directory: /go/src/github.com/{{ORG_NAME}}/{{REPO_NAME}}
-    steps:
-      - checkout
-      - run: apt-get update && apt-get install -y --no-install-recommends libavcodec-dev libavformat-dev libswscale-dev libjpeg62-turbo-dev
-      - run: cd machinery && go mod download
-      - run: cd machinery && go test -v ./...
-      - run: cd machinery && go vet
-
-  ui:
-    docker:
-      - image: cimg/python:3.9.13-node
-    steps:
-      - checkout
-      - run: node --version
-      - run: cd ui && yarn
-      - run: cd ui && yarn test --passWithNoTests
-      - run: cd ui && yarn build
-
-workflows:
-  version: 2
-  build:
-    jobs:
-      - machinery
-      - ui

.devcontainer/Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/devcontainers/go:1.24-bookworm
+FROM mcr.microsoft.com/devcontainers/go:1.25-trixie
 
 # Install node environment
 RUN apt-get update && \

.devcontainer/devcontainer.json

@@ -1,7 +1,7 @@
 // For format details, see https://aka.ms/devcontainer.json. For config options, see the
 // README at: https://github.com/devcontainers/templates/tree/main/src/python
 {
-	"name": "go:1.24-bookworm",
+	"name": "go:1.24-trixie",
 	"runArgs": [
 		"--name=agent",
 		"--network=host"
@@ -18,7 +18,7 @@
 	},
 	"forwardPorts": [
 		3000,
-		8080
+		8082
 	],
-	"postCreateCommand": "cd ui && yarn install && yarn build && cd ../machinery && go mod download"
+	"postCreateCommand": "cd ui && yarn install && yarn build && cd ../machinery && go mod download && bash ./verify-moq-devcontainer.sh"
 }

.dockerignore

@@ -0,0 +1,42 @@
+# Only application sources are needed by the production build.
+**
+!Dockerfile
+!machinery/
+!machinery/**
+!ui/
+!ui/**
+!deployments/
+deployments/*
+!deployments/docker/
+deployments/docker/*
+!deployments/docker/assemble-rootfs.sh
+
+**/.git
+**/.env
+**/.env.*
+**/.git-credentials
+**/.netrc
+**/.npmrc
+**/.yarnrc.yml
+**/.aws
+**/.ssh
+**/.kube
+**/node_modules
+**/.DS_Store
+**/*.test
+**/*.pprof
+**/coverage.out
+machinery/main
+machinery/vendor
+machinery/go.work
+machinery/go.work.sum
+machinery/www
+machinery/data/**
+!machinery/data/config/
+!machinery/data/config/config.json
+!machinery/data/config/user.json
+!machinery/data/test-480p.mp4
+ui/build
+ui/coverage
+ui/.npmrc
+ui/.yarnrc

.github/scripts/image-evidence.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+"""Validate BuildKit provenance and retain resolved base-image materials."""
+
+import json
+from pathlib import Path
+import re
+import sys
+
+
+def materials(value):
+    if isinstance(value, dict):
+        if "uri" in value and value["uri"].startswith(("pkg:docker/", "docker-image://")):
+            digest = value.get("digest", {}).get("sha256", "")
+            if not re.fullmatch(r"[a-f0-9]{64}", digest):
+                raise ValueError(f"Missing base-image digest: {value['uri']}")
+            yield value
+        for child in value.values():
+            yield from materials(child)
+    elif isinstance(value, list):
+        for child in value:
+            yield from materials(child)
+
+
+if __name__ == "__main__":
+    directory = Path(sys.argv[1])
+    provenance = json.loads((directory / "provenance.json").read_text())
+    bases = list(materials(provenance))
+    if not bases:
+        raise SystemExit("Missing provenance/base-image materials")
+    (directory / "base-image-digests.json").write_text(json.dumps(bases, indent=2) + "\n")

.github/scripts/image-policy.py

@@ -0,0 +1,242 @@
+#!/usr/bin/env python3
+"""Fail-closed source/application and image scans with redacted evidence."""
+
+import argparse
+from datetime import datetime, timezone
+import hashlib
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+
+
+def secret_identity(target, finding):
+    return (target, finding.get("RuleID"), finding.get("Category"),
+            finding.get("Severity"), finding.get("StartLine"), finding.get("EndLine"))
+
+
+def approve_shared_key(report, source_root, policy, now=None):
+    """Bind the one reviewed exception to an unchanged file and exact finding."""
+    now = now or datetime.now(timezone.utc)
+    records = policy.get("exceptions")
+    if not isinstance(records, list) or len(records) != 1:
+        raise ValueError("Exactly one shared-key exception record is required")
+    record = records[0]
+    if not isinstance(record, dict) or record.get("id") != "shared-recording-signing-key":
+        raise ValueError("Unexpected scan exception")
+    if record.get("status") != "approved" or not all(
+        isinstance(record.get(key), str) and record[key].strip()
+        for key in ("owner", "approved_by", "reason", "approval_url", "expires")
+    ):
+        raise ValueError("Shared-key exception lacks approval/owner/reason/evidence/expiry")
+    if record["approval_url"] != "https://github.com/kerberos-io/agent/pull/350":
+        raise ValueError("Shared-key exception must reference its reviewed PR record")
+    expires = datetime.fromisoformat(record["expires"].replace("Z", "+00:00"))
+    if expires.tzinfo is None or expires <= now:
+        raise ValueError("Shared-key exception expired or lacks timezone")
+    if (record.get("target") != "machinery/src/config/main.go"
+            or record.get("rule_id") != "private-key"
+            or record.get("category") != "AsymmetricPrivateKey"
+            or record.get("severity") != "HIGH"
+            or type(record.get("start_line")) is not int
+            or record["start_line"] < 1
+            or record.get("end_line") != record["start_line"]):
+        raise ValueError("Shared-key exception has an unexpected finding scope")
+    root = Path(source_root).resolve()
+    path = root / record["target"]
+    if path.resolve() != path or hashlib.sha256(path.read_bytes()).hexdigest() != record.get("file_sha256"):
+        raise ValueError("Shared-key source file changed or is a symlink; renewed review required")
+    expected = (record["target"], record["rule_id"], record["category"],
+                record["severity"], record["start_line"], record["end_line"])
+    matches = [
+        finding for result in report["Results"]
+        for finding in result.get("Secrets", [])
+        if secret_identity(result.get("Target"), finding) == expected
+    ]
+    if len(matches) != 1:
+        raise ValueError("Shared-key exception must match exactly one existing source finding")
+    return [dict(record)]
+
+
+def check_report(report, image=False, source=False, approved_exceptions=()):
+    if report.get("SchemaVersion") != 2 or not isinstance(report.get("Results"), list):
+        raise ValueError("Missing or unsupported Trivy report")
+    failures = []
+    approved_secrets = {
+        (record["target"], record["rule_id"], record["category"], record["severity"],
+         record["start_line"], record["end_line"])
+        for record in approved_exceptions
+    } if source and not image else set()
+    for result in report["Results"]:
+        if result.get("Errors"):
+            failures.append(f"Scan errors in {result.get('Target')}: {result['Errors']}")
+        if result.get("MisconfSummary", {}).get("Exceptions", 0):
+            failures.append(f"Unapproved misconfiguration exceptions in {result.get('Target')}")
+        for kind in ("Vulnerabilities", "Secrets", "Misconfigurations"):
+            for finding in result.get(kind, []):
+                if kind == "Misconfigurations" and finding.get("Status") == "PASS":
+                    continue
+                if kind == "Misconfigurations" and finding.get("Status") != "FAIL":
+                    failures.append(f"Unexpected/suppressed misconfiguration status: {finding.get('Status')}")
+                severity = finding.get("Severity", "UNKNOWN")
+                if kind == "Secrets" and secret_identity(result.get("Target"), finding) in approved_secrets:
+                    continue
+                if severity not in ("LOW", "MEDIUM"):
+                    failures.append(f"{result.get('Target')}: {kind}: "
+                                    f"{finding.get('VulnerabilityID', finding.get('ID', finding.get('RuleID')))} "
+                                    f"{severity}")
+    if image:
+        # A scratch image without package metadata must not yield a false clean OS scan.
+        if not report.get("Metadata", {}).get("OS", {}).get("Family"):
+            failures.append("No OS detected: preserve runtime OS/package metadata")
+        if not any(r.get("Class") == "os-pkgs" and r.get("Packages")
+                   for r in report["Results"]):
+            failures.append("No OS package inventory detected")
+        if not any(r.get("Type") == "gobinary" and r.get("Packages")
+                   for r in report["Results"]):
+            failures.append("No Go application package inventory detected")
+    if source and not any(
+        r.get("Type") == "yarn" and r.get("Target", "").replace("\\", "/").endswith("ui/yarn.lock")
+        and r.get("Packages") for r in report["Results"]
+    ):
+        failures.append("Missing production UI yarn.lock package inventory")
+    return failures
+
+
+def redact_report(value):
+    """Keep findings actionable without persisting matches, excerpts or image env/history."""
+    if isinstance(value, list):
+        return [redact_report(item) for item in value]
+    if not isinstance(value, dict):
+        return value
+    result = {}
+    for key, child in value.items():
+        if key.lower() in {"imageconfig", "buildinfo"}:
+            continue
+        if key.lower() in {
+            "match", "code", "lines", "source", "sourcecode", "snippet", "context",
+            "content",
+        }:
+            result[key] = "[REDACTED]"
+        elif key == "Errors":
+            result[key] = ["[REDACTED scanner error]"] if child else []
+        elif key in ("Secrets", "Misconfigurations"):
+            allowed = {
+                "ID", "AVDID", "RuleID", "Type", "Category", "Title", "Severity",
+                "Status", "StartLine", "EndLine", "References",
+            }
+            result[key] = [
+                {field: finding[field] for field in allowed if field in finding}
+                | {"SourceExcerpt": "[REDACTED]"}
+                for finding in child
+            ]
+        else:
+            result[key] = redact_report(child)
+    return result
+
+
+def scan(target, output, platform=None):
+    output.mkdir(parents=True, exist_ok=True)
+    config = output / "scanner.yaml"
+    ignore = output / "empty.ignore"
+    secret_config = output / "secret.yaml"
+    config.write_text("{}\n")
+    secret_config.write_text("{}\n")
+    ignore.write_text("")
+    # Do not consume repository trivy.yaml/.trivyignore or TRIVY_* overrides.
+    env = {k: v for k, v in os.environ.items() if not k.startswith("TRIVY_")}
+    command = [
+        "trivy", "image" if platform else "fs", "--config", str(config),
+        "--cache-dir", ".ci-tools/trivy-cache",
+        "--ignorefile", str(ignore), "--secret-config", str(secret_config),
+        "--format", "json", "--exit-code", "0",
+        "--scanners", "vuln,secret,misconfig",
+        "--severity", "UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL", "--timeout", "20m",
+        "--list-all-pkgs", "--detection-priority", "comprehensive",
+    ]
+    if platform:
+        command += ["--platform", platform, "--pkg-types", "os,library",
+                    "--image-config-scanners", "secret,misconfig"]
+    else:
+        # Include development/build dependencies too: none are silently exempted.
+        # Trivy marks Dev packages using the adjacent package.json and yarn.lock.
+        command += ["--pkg-types", "library", "--include-dev-deps"]
+    command.append(target)
+    # Never write raw scanner output, even on scanner failure/interruption. stderr
+    # may contain source excerpts too; retain only a safe exit-status diagnostic.
+    completed = subprocess.run(command, env=env, stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE, text=True, check=False)
+    (output / "scanner.log").write_text(
+        f"Trivy exit status: {completed.returncode}\n"
+        "Raw scanner stdout/stderr omitted; scan.json is redacted before persistence.\n")
+    if completed.returncode:
+        raise ValueError(f"Trivy failed with exit status {completed.returncode}; raw diagnostics withheld")
+    try:
+        report = redact_report(json.loads(completed.stdout))
+    except ValueError:
+        raise ValueError("Invalid scanner JSON; raw output withheld") from None
+    (output / "scan.json").write_text(json.dumps(report, indent=2) + "\n")
+    approved = []
+    exception_failures = []
+    if not platform:
+        try:
+            policy = json.loads(Path(__file__).with_name("image-scan-exceptions.json").read_text())
+            (output / "exception-policy.json").write_text(json.dumps(policy, indent=2) + "\n")
+            approved = approve_shared_key(report, target, policy)
+        except (OSError, ValueError, TypeError, KeyError) as error:
+            exception_failures.append(f"Shared-key exception invalid: {error}")
+    (output / "exception-decisions.json").write_text(json.dumps({
+        "approved_exceptions": approved, "failures": exception_failures,
+    }, indent=2) + "\n")
+    failures = exception_failures + check_report(
+        report, image=bool(platform), source=not platform, approved_exceptions=approved)
+    if not platform:
+        inventory = [{
+            "target": result.get("Target"), "type": result.get("Type"),
+            "packages": [{"name": pkg.get("Name"), "version": pkg.get("Version"),
+                          "scope": "development" if pkg.get("Dev") else "production-or-unclassified"}
+                         for pkg in result.get("Packages", [])],
+        } for result in report["Results"] if result.get("Packages")]
+        (output / "application-inventory.json").write_text(json.dumps({
+            "policy": "All production and development/build dependencies are gated; "
+                      "only the reviewed, exact shared-key secret exception may apply.",
+            "inventories": inventory,
+        }, indent=2) + "\n")
+    converted = subprocess.run([
+        "trivy", "convert", "--cache-dir", ".ci-tools/trivy-cache",
+        "--format", "spdx-json", "--output", str(output / "sbom.spdx.json"),
+        str(output / "scan.json"),
+    ], env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
+    if converted.returncode:
+        raise ValueError(f"SPDX conversion failed with exit status {converted.returncode}")
+    sbom = json.loads((output / "sbom.spdx.json").read_text())
+    if not sbom.get("spdxVersion") or not sbom.get("packages"):
+        failures.append("Missing/empty SPDX SBOM")
+    return failures
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("target", help="Repository root, or final image reference with --platform")
+    parser.add_argument("output", type=Path, help="Directory for redacted reports and SPDX SBOM")
+    parser.add_argument("--platform", help="Image platform, e.g. linux/amd64; omit for source/UI scanning")
+    args = parser.parse_args()
+    args.output.mkdir(parents=True, exist_ok=True)
+    approved = []
+    try:
+        failures = scan(args.target, args.output, args.platform)
+        approved = json.loads((args.output / "exception-decisions.json").read_text())["approved_exceptions"]
+    except (OSError, ValueError, subprocess.CalledProcessError) as error:
+        failures = [f"Scanner/evidence error: {error}"]
+    (args.output / "policy.json").write_text(json.dumps({
+        "target": args.target, "platform": args.platform, "failures": failures,
+        "approved_exceptions": approved, "passed": not failures,
+    }, indent=2) + "\n")
+    for failure in failures:
+        print(failure, file=sys.stderr)
+    return int(bool(failures))
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/scripts/image-release-policy.json

@@ -0,0 +1,13 @@
+{
+  "required_exceptions": [
+    {
+      "id": "remote-terminal-pty",
+      "status": "pending",
+      "reason": "The product retains functional remote terminal/PTY support, contrary to the no-shell baseline.",
+      "owner": null,
+      "expires": null,
+      "approved_by": null,
+      "approval_url": null
+    }
+  ]
+}

.github/scripts/image-release-policy.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+"""Release-only policy gate. Approval records require a reviewed repository change."""
+
+from datetime import datetime, timezone
+import json
+from pathlib import Path
+import sys
+
+
+def check_policy(policy, now=None):
+    now = now or datetime.now(timezone.utc)
+    records = policy.get("required_exceptions")
+    if not isinstance(records, list) or not records:
+        return ["Required exception records are missing"]
+    if {record.get("id") for record in records} != {"remote-terminal-pty"}:
+        return ["Required remote-terminal-pty exception record is missing or unexpected"]
+    failures = []
+    for record in records:
+        name = record["id"]
+        if record.get("status") != "approved":
+            failures.append(f"{name}: required exception is PENDING, not approved")
+            continue
+        if not all(isinstance(record.get(key), str) and record[key].strip()
+                   for key in ("owner", "approved_by", "reason", "approval_url", "expires")):
+            failures.append(f"{name}: missing owner/approver/reason/approval evidence/expiry")
+            continue
+        if not record["approval_url"].startswith("https://github.com/kerberos-io/agent/"):
+            failures.append(f"{name}: approval must reference this repository's reviewed record")
+        try:
+            expires = datetime.fromisoformat(record["expires"].replace("Z", "+00:00"))
+            if expires.tzinfo is None or expires <= now:
+                failures.append(f"{name}: approval expired or lacks timezone")
+        except ValueError:
+            failures.append(f"{name}: invalid approval expiry")
+    return failures
+
+
+if __name__ == "__main__":
+    try:
+        failures = check_policy(json.loads(Path(sys.argv[1]).read_text()))
+    except (OSError, ValueError, TypeError, KeyError) as error:
+        failures = [f"Invalid release policy: {error}"]
+    for failure in failures:
+        print(failure, file=sys.stderr)
+    sys.exit(bool(failures))

.github/scripts/image-scan-exceptions.json

@@ -0,0 +1,20 @@
+{
+  "exceptions": [
+    {
+      "id": "shared-recording-signing-key",
+      "status": "approved",
+      "owner": "@cedricve",
+      "approved_by": "@cedricve",
+      "approval_url": "https://github.com/kerberos-io/agent/pull/350",
+      "expires": "2026-12-05T23:59:59Z",
+      "reason": "User approved temporary retention of the existing shared recording-signing key to preserve recording verification trust and startup behavior. Anyone possessing this shared private key can forge signatures under that identity.",
+      "target": "machinery/src/config/main.go",
+      "file_sha256": "e28a0bf6ae892559709e8ec534cfc7b7cb55180e33415461d966c1afe53ca25b",
+      "rule_id": "private-key",
+      "category": "AsymmetricPrivateKey",
+      "severity": "HIGH",
+      "start_line": 710,
+      "end_line": 710
+    }
+  ]
+}

.github/scripts/install-trivy.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+set -euo pipefail
+# Public OCI index pins the platform-specific executable without GitHub API access.
+version=0.76.1
+image=ghcr.io/aquasecurity/trivy:0.76.1@sha256:790fd66d1697395d8693d1acaa1ea702406a20f109b6fc7bac8db623eca6c06c
+evidence_dir="${1:-evidence/scanner-install}"
+mkdir -p "$evidence_dir" .ci-tools
+stage=host-platform
+platform=
+cid=
+finish() {
+  result=$?
+  trap - EXIT
+  if [[ -n "$cid" ]]; then
+    docker rm -f "$cid" >/dev/null 2>&1 || true
+  fi
+  rm -f .ci-tools/trivy.new
+  python3 - "$evidence_dir/install.json" "$result" "$stage" "$image" "$platform" "$version" <<'PY'
+import hashlib
+import json
+from pathlib import Path
+import sys
+path, code, stage, image, platform, version = sys.argv[1:]
+record = {
+    "status": "installed" if code == "0" else "failed",
+    "exit_code": int(code), "stage": stage, "image": image, "platform": platform,
+    "binary_source": "/usr/local/bin/trivy", "expected_version": version,
+}
+if code == "0":
+    record["binary_sha256"] = hashlib.sha256(Path(".ci-tools/trivy").read_bytes()).hexdigest()
+Path(path).write_text(json.dumps(record, indent=2) + "\n")
+PY
+  exit "$result"
+}
+trap finish EXIT
+case "$(uname -m)" in
+  x86_64) platform=linux/amd64 ;;
+  aarch64|arm64) platform=linux/arm64 ;;
+  *) echo "Unsupported scanner host architecture" >&2; exit 1 ;;
+esac
+stage=pull
+docker pull --platform "$platform" "$image"
+stage=create
+cid=$(docker create --platform "$platform" "$image")
+stage=extract
+docker cp "$cid:/usr/local/bin/trivy" .ci-tools/trivy.new
+chmod 0755 .ci-tools/trivy.new
+stage=verify-version
+.ci-tools/trivy.new --version | grep -Fx "Version: $version"
+mv .ci-tools/trivy.new .ci-tools/trivy
+stage=publish-path
+if [[ -n "${GITHUB_PATH:-}" ]]; then
+  echo "$PWD/.ci-tools" >> "$GITHUB_PATH"
+fi
+stage=complete

.github/scripts/test-image-policy.py

@@ -0,0 +1,288 @@
+#!/usr/bin/env python3
+"""Focused release guard tests; no registry, signing, or scanner network access."""
+
+from datetime import datetime, timezone
+import copy
+import hashlib
+import importlib.util
+import json
+from pathlib import Path
+import unittest
+from unittest.mock import patch
+import subprocess
+import tempfile
+
+
+ROOT = Path(__file__).parent
+
+
+def load(name):
+    spec = importlib.util.spec_from_file_location(name, ROOT / f"{name}.py")
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+scanner = load("image-policy")
+release = load("image-release-policy")
+evidence = load("image-evidence")
+
+
+class ScanPolicyTests(unittest.TestCase):
+    def report(self, severity=None, kind="Vulnerabilities"):
+        result = {"Class": "os-pkgs", "Packages": [{"Name": "libc6"}]}
+        if severity:
+            result[kind] = [{"Severity": severity, "Status": "FAIL", "ID": "fixture"}]
+        return {"SchemaVersion": 2, "Metadata": {"OS": {"Family": "debian"}},
+                "Results": [result, {"Class": "lang-pkgs", "Type": "gobinary",
+                                     "Packages": [{"Name": "stdlib"}]}]}
+
+    def test_low_and_medium_are_not_blocking(self):
+        for severity in ("LOW", "MEDIUM"):
+            self.assertEqual(scanner.check_report(self.report(severity), image=True), [])
+
+    def test_high_critical_unknown_and_unrecognized_block(self):
+        for kind in ("Vulnerabilities", "Secrets", "Misconfigurations"):
+            for severity in ("HIGH", "CRITICAL", "UNKNOWN", "invalid"):
+                with self.subTest(kind=kind, severity=severity):
+                    self.assertTrue(scanner.check_report(self.report(severity, kind), image=True))
+
+    def test_missing_inventory_blocks_false_clean_scratch_scan(self):
+        report = self.report()
+        report["Metadata"] = {}
+        report["Results"] = []
+        self.assertEqual(len(scanner.check_report(report, image=True)), 3)
+
+    def test_empty_os_inventory_blocks(self):
+        report = self.report()
+        report["Results"][0]["Packages"] = []
+        self.assertTrue(scanner.check_report(report, image=True))
+
+    def test_report_errors_and_suppression_summary_block(self):
+        report = self.report()
+        report["Results"][0]["Errors"] = ["fixture scanner error"]
+        report["Results"][0]["MisconfSummary"] = {"Exceptions": 1}
+        self.assertEqual(len(scanner.check_report(report)), 2)
+
+    def test_missing_or_unsupported_report_blocks(self):
+        for report in ({}, {"SchemaVersion": 1, "Results": []}):
+            with self.assertRaises(ValueError):
+                scanner.check_report(report)
+
+    def test_suppressed_misconfiguration_blocks(self):
+        report = self.report("LOW", "Misconfigurations")
+        report["Results"][0]["Misconfigurations"][0]["Status"] = "EXCEPTION"
+        self.assertTrue(scanner.check_report(report))
+
+    def test_scanner_errors_are_not_ignored(self):
+        directory = unittest.mock.MagicMock()
+        with patch.object(scanner.subprocess, "run",
+                          side_effect=subprocess.CalledProcessError(1, ["trivy"])):
+            with self.assertRaises(subprocess.CalledProcessError):
+                scanner.scan("fixture", directory, "linux/amd64")
+
+    def test_source_requires_ui_lockfile_inventory(self):
+        report = self.report()
+        self.assertTrue(scanner.check_report(report, source=True))
+        report["Results"].append({"Type": "yarn", "Target": "ui/yarn.lock",
+                                   "Packages": [{"Name": "react", "Version": "17.0.2"}]})
+        self.assertEqual(scanner.check_report(report, source=True), [])
+        report["Results"][-1]["Vulnerabilities"] = [{"Severity": "HIGH", "VulnerabilityID": "fixture"}]
+        self.assertTrue(scanner.check_report(report, source=True))
+
+    def test_secret_and_source_excerpts_are_removed_without_waiving_findings(self):
+        marker = "sensitive-fixture-text-not-a-real-credential"
+        report = self.report("HIGH", "Secrets")
+        report["Metadata"]["ImageConfig"] = {"config": {"Env": [marker]}}
+        report["Metadata"]["BuildInfo"] = {"attrs": {"secret": marker}}
+        report["Results"][0]["Secrets"][0].update(
+            Match=marker, Code={"Lines": [{"Content": marker}]}, Message=marker,
+            UnexpectedFutureExcerptField=marker, RuleID="private-key-fixture")
+        report["Results"][0]["Misconfigurations"] = [{
+            "ID": "fixture", "Severity": "HIGH", "Status": "FAIL",
+            "Message": marker, "CauseMetadata": {"Code": {"Lines": [{"Content": marker}]}}
+        }]
+        report["Results"][0]["Errors"] = [marker]
+        redacted = scanner.redact_report(report)
+        self.assertNotIn(marker, json.dumps(redacted))
+        self.assertNotIn("ImageConfig", redacted["Metadata"])
+        self.assertTrue(scanner.check_report(redacted, image=True))
+        self.assertEqual(redacted["Results"][0]["Secrets"][0]["RuleID"], "private-key-fixture")
+
+    def test_source_scan_captures_output_in_memory_and_generates_sbom(self):
+        report = self.report()
+        report["Results"].append({"Type": "yarn", "Target": "ui/yarn.lock",
+                                   "Packages": [{"Name": "fixture", "Version": "1.0", "Dev": True}]})
+        directory = unittest.mock.MagicMock()
+        directory.__truediv__.return_value.read_text.return_value = json.dumps({
+            "spdxVersion": "SPDX-2.3", "packages": [{"name": "fixture"}]})
+        completed = subprocess.CompletedProcess(["trivy"], 0, stdout=json.dumps(report), stderr="")
+        with patch.object(scanner.subprocess, "run", return_value=completed) as run, \
+                patch.object(scanner, "approve_shared_key", return_value=[]):
+            self.assertEqual(scanner.scan("fixture", directory), [])
+        command = run.call_args_list[0].args[0]
+        self.assertIn("--include-dev-deps", command)
+        self.assertIn("vuln,secret,misconfig", command)
+        self.assertNotIn("--output", command)
+        self.assertEqual(run.call_args_list[0].kwargs["stdout"], subprocess.PIPE)
+        self.assertEqual(run.call_args_list[1].args[0][1], "convert")
+
+    def test_raw_scanner_error_diagnostics_are_not_persisted(self):
+        directory = unittest.mock.MagicMock()
+        marker = "sensitive-fixture-text-not-a-real-credential"
+        completed = subprocess.CompletedProcess(["trivy"], 1, stdout=marker, stderr=marker)
+        with patch.object(scanner.subprocess, "run", return_value=completed):
+            with self.assertRaisesRegex(ValueError, "exit status 1"):
+                scanner.scan("fixture", directory)
+        self.assertNotIn(marker, str(directory.mock_calls))
+
+
+class SharedKeyExceptionTests(unittest.TestCase):
+    NOW = datetime(2026, 11, 5, tzinfo=timezone.utc)
+
+    def setUp(self):
+        self.directory = tempfile.TemporaryDirectory()
+        self.addCleanup(self.directory.cleanup)
+        self.root = Path(self.directory.name)
+        self.path = self.root / "machinery/src/config/main.go"
+        self.path.parent.mkdir(parents=True)
+        self.path.write_text("non-secret source fixture\n")
+        self.policy = json.loads((ROOT / "image-scan-exceptions.json").read_text())
+        self.record = self.policy["exceptions"][0]
+        self.record["file_sha256"] = hashlib.sha256(self.path.read_bytes()).hexdigest()
+        self.finding = {
+            "RuleID": self.record["rule_id"], "Category": self.record["category"],
+            "Severity": "HIGH", "StartLine": 710, "EndLine": 710,
+        }
+        self.report = {"SchemaVersion": 2, "Results": [
+            {"Target": self.record["target"], "Secrets": [self.finding]},
+            {"Target": "ui/yarn.lock", "Type": "yarn", "Packages": [{"Name": "react"}]},
+        ]}
+
+    def approved(self):
+        return scanner.approve_shared_key(self.report, self.root, self.policy, self.NOW)
+
+    def test_exact_key_is_approved_but_finding_is_preserved(self):
+        original = copy.deepcopy(self.report)
+        approved = self.approved()
+        self.assertTrue(scanner.check_report(self.report, source=True))
+        self.assertEqual(scanner.check_report(
+            self.report, source=True, approved_exceptions=approved), [])
+        self.assertEqual(self.report, original)
+        self.assertEqual(approved[0]["id"], "shared-recording-signing-key")
+
+    def test_other_secrets_and_vulnerabilities_still_block(self):
+        approved = self.approved()
+        result = self.report["Results"][0]
+        result["Secrets"].append(dict(self.finding, StartLine=711, EndLine=711))
+        result["Vulnerabilities"] = [{"VulnerabilityID": "fixture", "Severity": "CRITICAL"}]
+        self.assertEqual(len(scanner.check_report(
+            self.report, source=True, approved_exceptions=approved)), 2)
+
+    def test_exception_never_applies_to_image_or_non_source_scans(self):
+        approved = self.approved()
+        self.assertTrue(scanner.check_report(self.report, approved_exceptions=approved))
+        self.assertTrue(scanner.check_report(
+            self.report, image=True, source=True, approved_exceptions=approved))
+
+    def test_expiry_is_enforced_at_boundary(self):
+        expiry = datetime(2026, 12, 5, 23, 59, 59, tzinfo=timezone.utc)
+        with self.assertRaisesRegex(ValueError, "expired"):
+            scanner.approve_shared_key(self.report, self.root, self.policy, expiry)
+        self.record["expires"] = "2026-12-05"
+        with self.assertRaisesRegex(ValueError, "timezone"):
+            self.approved()
+
+    def test_incomplete_or_unapproved_metadata_blocks(self):
+        for field in ("owner", "approved_by", "reason", "approval_url", "expires", "status"):
+            with self.subTest(field=field):
+                original = self.record[field]
+                self.record[field] = ""
+                with self.assertRaises(ValueError):
+                    self.approved()
+                self.record[field] = original
+        self.record["approval_url"] = "https://example.com/unreviewed"
+        with self.assertRaises(ValueError):
+            self.approved()
+
+    def test_changed_source_or_symlink_requires_review(self):
+        self.path.write_text("changed source fixture\n")
+        with self.assertRaisesRegex(ValueError, "changed"):
+            self.approved()
+        target = self.root / "other.go"
+        self.path.rename(target)
+        self.path.symlink_to(target)
+        with self.assertRaisesRegex(ValueError, "symlink"):
+            self.approved()
+
+    def test_changed_finding_does_not_match(self):
+        for field, value in (("RuleID", "other"), ("Category", "other"),
+                             ("Severity", "CRITICAL"), ("StartLine", 711), ("EndLine", 711)):
+            with self.subTest(field=field):
+                original = self.finding[field]
+                self.finding[field] = value
+                with self.assertRaisesRegex(ValueError, "exactly one"):
+                    self.approved()
+                self.finding[field] = original
+
+    def test_duplicate_missing_or_broadened_records_block(self):
+        self.policy["exceptions"].append(dict(self.record))
+        with self.assertRaises(ValueError):
+            self.approved()
+        self.policy["exceptions"].pop()
+        self.report["Results"][0]["Secrets"].append(dict(self.finding))
+        with self.assertRaisesRegex(ValueError, "exactly one"):
+            self.approved()
+        self.report["Results"][0]["Secrets"] = []
+        with self.assertRaisesRegex(ValueError, "exactly one"):
+            self.approved()
+        self.record["target"] = "../other.go"
+        with self.assertRaisesRegex(ValueError, "scope"):
+            self.approved()
+
+    def test_checked_in_record_matches_unchanged_source(self):
+        report = copy.deepcopy(self.report)
+        policy = json.loads((ROOT / "image-scan-exceptions.json").read_text())
+        self.assertEqual(len(scanner.approve_shared_key(
+            report, ROOT.parent.parent, policy, self.NOW)), 1)
+
+
+class ReleasePolicyTests(unittest.TestCase):
+    NOW = datetime(2026, 11, 4, tzinfo=timezone.utc)
+
+    def test_checked_in_state_blocks_publication(self):
+        state = json.loads((ROOT / "image-release-policy.json").read_text())
+        self.assertTrue(release.check_policy(state, self.NOW))
+
+    def test_missing_records_block(self):
+        for state in ({}, {"required_exceptions": []},
+                      {"required_exceptions": [{"id": "unrelated"}]}):
+            self.assertTrue(release.check_policy(state, self.NOW))
+
+    def test_expired_or_incomplete_approval_blocks(self):
+        state = json.loads((ROOT / "image-release-policy.json").read_text())
+        record = state["required_exceptions"][0]
+        record["status"] = "approved"
+        self.assertTrue(release.check_policy(state, self.NOW))
+        record.update(owner="test-fixture", approved_by="test-fixture",
+                      approval_url="https://github.com/kerberos-io/agent/issues/1",
+                      expires="2020-01-01T00:00:00Z")
+        self.assertTrue(release.check_policy(state, self.NOW))
+        record["expires"] = "invalid"
+        self.assertTrue(release.check_policy(state, self.NOW))
+        record["expires"] = "2027-01-01"
+        self.assertTrue(release.check_policy(state, self.NOW))
+
+
+class ProvenanceTests(unittest.TestCase):
+    def test_base_digest_retained(self):
+        material = {"uri": "pkg:docker/debian@trixie", "digest": {"sha256": "a" * 64}}
+        self.assertEqual(list(evidence.materials({"SLSA": {"materials": [material]}})), [material])
+
+    def test_missing_digest_rejected(self):
+        with self.assertRaises(ValueError):
+            list(evidence.materials({"uri": "pkg:docker/debian@trixie"}))
+
+
+if __name__ == "__main__":
+    unittest.main()

.github/workflows/go.yml

@@ -10,28 +10,31 @@ jobs:
   build:
     name: Build
     runs-on: ubuntu-latest
-    container:
-      image: kerberos/base:eb6b088
-
-    strategy:
-      matrix:
-        #No longer supported Go versions.
-        #go-version: ['1.17', '1.18', '1.19', '1.20', '1.21']
-        go-version: ["1.24"]
 
     steps:
-      - name: Set up Go ${{ matrix.go-version }}
-        uses: actions/setup-go@v2
-        with:
-          go-version: ${{ matrix.go-version }}
       - name: Check out code into the Go module directory
         uses: actions/checkout@v4
-      - name: Set up git ownershi
-        run: git config --system --add safe.directory /__w/agent/agent
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.26.10"
+          check-latest: true
+          cache: true
+          cache-dependency-path: machinery/go.sum
+      - name: Install native dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential \
+            libavcodec-dev \
+            libavutil-dev \
+            libswresample-dev \
+            libswscale-dev \
+            pkg-config
       - name: Get dependencies
-        run: cd machinery && go mod download
+        run: cd machinery && go mod download && go mod verify
       - name: Build
-        run: cd machinery && go build -v ./...
+        run: cd machinery && go build -mod=readonly -trimpath -buildvcs=false -v ./...
       - name: Vet
         run: cd machinery && go vet -v ./...
       - name: Test

.github/workflows/image-release.yml

@@ -0,0 +1,324 @@
+name: Verified image publication
+on:
+  workflow_call:
+    inputs:
+      tag:
+        type: string
+        default: ""
+      source_ref:
+        type: string
+        required: true
+      nightly:
+        type: boolean
+        default: false
+      publish_latest:
+        type: boolean
+        default: false
+    secrets:
+      DOCKER_USERNAME:
+        required: true
+      DOCKER_PASSWORD:
+        required: true
+
+permissions:
+  contents: read
+
+# No parallel run can overwrite a customer tag while another run verifies it.
+concurrency:
+  group: image-publication-${{ inputs.nightly && 'nightly' || 'release' }}
+  cancel-in-progress: false
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  resolve:
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: write
+    outputs:
+      sha: ${{ steps.source.outputs.sha }}
+      tag: ${{ steps.source.outputs.tag }}
+      created: ${{ steps.source.outputs.created }}
+      matrix: ${{ steps.source.outputs.matrix }}
+      repository: ${{ steps.source.outputs.repository }}
+      staging_repository: ${{ steps.source.outputs.staging_repository }}
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          persist-credentials: false
+      - name: Validate requested tag
+        env:
+          TAG: ${{ inputs.tag }}
+          NIGHTLY: ${{ inputs.nightly }}
+        run: |
+          if [[ "$NIGHTLY" != true && ! "$TAG" =~ ^[a-zA-Z0-9_][a-zA-Z0-9_.-]{0,127}$ ]]; then
+            echo "Invalid Docker tag" >&2
+            exit 1
+          fi
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          ref: ${{ inputs.source_ref }}
+          path: source
+          persist-credentials: false
+      - name: Resolve once for every architecture
+        id: source
+        env:
+          TAG: ${{ inputs.tag }}
+          NIGHTLY: ${{ inputs.nightly }}
+        run: |
+          sha=$(git -C source rev-parse HEAD)
+          [[ "$sha" =~ ^[a-f0-9]{40}$ ]]
+          created=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+          repository=kerberos/agent
+          staging_repository=kerberos/agent-arch
+          matrix='{"include":[{"architecture":"amd64","runner":"ubuntu-24.04"},{"architecture":"arm64","runner":"ubuntu-24.04-arm"}]}'
+          if [[ "$NIGHTLY" == true ]]; then
+            TAG="${sha:0:7}"
+            repository=kerberos/agent-nightly
+            staging_repository="$repository"
+            # Keep arm/v6: the pinned Go trixie base currently lacks this platform.
+            # Its build must fail rather than publishing an incomplete nightly index.
+            matrix='{"include":[{"architecture":"amd64","runner":"ubuntu-24.04"},{"architecture":"arm64","runner":"ubuntu-24.04-arm"},{"architecture":"arm/v7","runner":"ubuntu-24.04"},{"architecture":"arm/v6","runner":"ubuntu-24.04"}]}'
+          fi
+          {
+            echo "sha=$sha"
+            echo "tag=$TAG"
+            echo "created=$created"
+            echo "repository=$repository"
+            echo "staging_repository=$staging_repository"
+            echo "matrix=$matrix"
+          } >> "$GITHUB_OUTPUT"
+          mkdir -p evidence
+          printf '%s\n' "$sha" > evidence/source-revision.txt
+          printf '%s\n' "$created" > evidence/created.txt
+          printf '%s\n' "$TAG" > evidence/version.txt
+          cp .github/scripts/image-release-policy.json evidence/runtime-policy.json
+      - name: Validate fail-closed policy helpers
+        run: python3 .github/scripts/test-image-policy.py
+      - name: Keep existing GitHub release unpublished until gates pass
+        if: ${{ !inputs.nightly }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TAG: ${{ steps.source.outputs.tag }}
+        run: |
+          # A release:created event may refer to an already public release.
+          # Return it to draft immediately; do not imply this run has verified it.
+          gh api --paginate "repos/$GITHUB_REPOSITORY/releases" --jq '.[]' | jq -s . > evidence/releases.json
+          release_id=$(jq -r --arg tag "$TAG" '.[] | select(.tag_name == $tag) | .id' evidence/releases.json)
+          if [[ -n "$release_id" ]]; then
+            gh api --method PATCH "repos/$GITHUB_REPOSITORY/releases/$release_id" -F draft=true > evidence/draft-release.json
+          fi
+      - name: Install pinned public OCI scanner
+        run: bash .github/scripts/install-trivy.sh
+      - name: Source/UI dependency vulnerability, secret and misconfiguration gate with SPDX SBOM
+        run: python3 .github/scripts/image-policy.py source evidence/source
+      - name: Retain source policy evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: image-source-evidence
+          path: evidence/
+          retention-days: 90
+          if-no-files-found: error
+
+  build:
+    needs: resolve
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.resolve.outputs.matrix) }}
+    env:
+      ARCH: ${{ matrix.architecture }}
+      VERSION: ${{ needs.resolve.outputs.tag }}
+      REVISION: ${{ needs.resolve.outputs.sha }}
+      CREATED: ${{ needs.resolve.outputs.created }}
+      STAGING_REPOSITORY: ${{ needs.resolve.outputs.staging_repository }}
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          persist-credentials: false
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          ref: ${{ needs.resolve.outputs.sha }}
+          path: source
+          persist-credentials: false
+      - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
+        if: ${{ startsWith(matrix.architecture, 'arm/') }}
+      - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
+      - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Build immutable staging image with maximal provenance
+        id: build
+        run: |
+          mkdir -p evidence
+          arch="${ARCH//\//-}"
+          echo "arch=$arch" >> "$GITHUB_OUTPUT"
+          staging="$STAGING_REPOSITORY:ci-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-$arch"
+          printf '%s\n' "$REVISION" > evidence/source-revision.txt
+          docker buildx build --platform "linux/$ARCH" --provenance=mode=max \
+            --build-arg "VERSION=$VERSION" --build-arg "REVISION=$REVISION" --build-arg "CREATED=$CREATED" \
+            --label "org.opencontainers.image.version=$VERSION" \
+            --label "org.opencontainers.image.revision=$REVISION" \
+            --label "org.opencontainers.image.created=$CREATED" \
+            --label "org.opencontainers.image.source=https://github.com/$GITHUB_REPOSITORY" \
+            --metadata-file evidence/build-metadata.json --tag "$staging" --push source
+          digest=$(jq -er '."containerimage.digest"' evidence/build-metadata.json)
+          [[ "$digest" =~ ^sha256:[a-f0-9]{64}$ ]]
+          image="$STAGING_REPOSITORY@$digest"
+          printf '%s\n' "$image" > evidence/image.txt
+          docker buildx imagetools inspect "$image" --format '{{json .Provenance}}' > evidence/provenance.json
+          python3 .github/scripts/image-evidence.py evidence
+      - name: Extract release payload and smoke test the exact staged image
+        run: |
+          image=$(cat evidence/image.txt)
+          docker pull --platform "linux/$ARCH" "$image"
+          cid=$(docker create --platform "linux/$ARCH" "$image")
+          trap 'docker rm -f "$cid" >/dev/null' EXIT
+          docker cp "$cid:/home/agent" output
+          # Run the existing smoke interface, including emulated nightly architectures.
+          DOCKER_DEFAULT_PLATFORM="linux/$ARCH" sh source/deployments/docker/test-image.sh "$image"
+          arch="${ARCH//\//-}"
+          tar -cf "agent-$arch.tar" -C output .
+          rm -rf output
+      - name: Retain build evidence even on failure
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: image-build-${{ steps.build.outputs.arch || strategy.job-index }}
+          path: evidence/
+          retention-days: 90
+          if-no-files-found: error
+      - name: Retain existing binary artifacts
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: agent-${{ steps.build.outputs.arch }}.tar
+          path: agent-*.tar
+          if-no-files-found: error
+
+  verify-publish:
+    needs: [resolve, build]
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: write
+      id-token: write
+    env:
+      REPOSITORY: ${{ needs.resolve.outputs.repository }}
+      STAGING_REPOSITORY: ${{ needs.resolve.outputs.staging_repository }}
+      VERSION: ${{ needs.resolve.outputs.tag }}
+      REVISION: ${{ needs.resolve.outputs.sha }}
+      PUBLISH_LATEST: ${{ inputs.publish_latest }}
+      MATRIX: ${{ needs.resolve.outputs.matrix }}
+      GH_TOKEN: ${{ github.token }}
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          persist-credentials: false
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          path: artifacts
+      - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
+      - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Install scanner
+        run: bash .github/scripts/install-trivy.sh
+      - uses: sigstore/cosign-installer@faadad0cce49287aee09b3a48701e75088a2c6ad # v4.0.0
+        with:
+          cosign-release: v3.1.4
+      - name: Assemble candidate index without publishing version/latest
+        run: |
+          mkdir -p evidence
+          images=()
+          while IFS= read -r arch; do
+            directory="artifacts/image-build-${arch//\//-}"
+            [[ "$(cat "$directory/source-revision.txt")" == "$REVISION" ]]
+            image=$(cat "$directory/image.txt")
+            [[ "$image" =~ ^${STAGING_REPOSITORY}@sha256:[a-f0-9]{64}$ ]]
+            images+=("$image")
+          done < <(jq -r '.include[].architecture' <<< "$MATRIX")
+          candidate="$REPOSITORY:ci-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"
+          docker buildx imagetools create --tag "$candidate" "${images[@]}"
+          digest=$(docker buildx imagetools inspect "$candidate" --format '{{.Manifest.Digest}}')
+          [[ "$digest" =~ ^sha256:[a-f0-9]{64}$ ]]
+          image="$REPOSITORY@$digest"
+          printf '%s\n' "$image" > evidence/final-image.txt
+          printf '%s\n' "$REVISION" > evidence/source-revision.txt
+          docker buildx imagetools inspect "$image" --raw > evidence/index.json
+          docker buildx imagetools inspect "$image" --format '{{json .Provenance}}' > evidence/provenance.json
+          python3 .github/scripts/image-evidence.py evidence
+          # Every requested runnable platform must be present, with no silent omissions.
+          jq -r '.manifests[] | select(.platform.os == "linux") |
+            .platform | .architecture + (if .variant and .architecture == "arm" then "/" + .variant else "" end)' \
+            evidence/index.json | sort > evidence/actual-platforms.txt
+          jq -r '.include[].architecture' <<< "$MATRIX" | sort > evidence/expected-platforms.txt
+          diff -u evidence/expected-platforms.txt evidence/actual-platforms.txt
+      - name: Scan every final-index platform and enforce findings/errors
+        run: |
+          image=$(cat evidence/final-image.txt)
+          failed=0
+          while IFS= read -r arch; do
+            # Generation uses exit-code 0 only for findings; the helper explicitly
+            # blocks HIGH/CRITICAL/UNKNOWN and any scanner/inventory/SBOM error.
+            if ! python3 .github/scripts/image-policy.py "$image" "evidence/${arch//\//-}" --platform "linux/$arch"; then
+              failed=1
+            fi
+          done < <(jq -r '.include[].architecture' <<< "$MATRIX")
+          exit "$failed"
+      - name: Required release policy exception gate
+        run: |
+          # The product deliberately retains remote terminal/PTY support.
+          # No approval exists. Keep this release-only gate separate from scans;
+          # there is no dispatch input/environment variable that can waive it.
+          cp .github/scripts/image-release-policy.json evidence/runtime-policy.json
+          python3 .github/scripts/image-release-policy.py evidence/runtime-policy.json
+      - name: Sign and verify the FINAL multi-platform digest using GitHub OIDC
+        run: |
+          image=$(cat evidence/final-image.txt)
+          # Fulcio's SAN is job_workflow_ref for this local reusable workflow,
+          # not the caller's filename. Bind the exact repository and executing ref.
+          # For example: https://github.com/kerberos-io/agent/.github/workflows/image-release.yml@refs/heads/master
+          identity="https://github.com/$GITHUB_REPOSITORY/.github/workflows/image-release.yml@$GITHUB_REF"
+          printf '%s\n' "$identity" > evidence/signing-identity.txt
+          cosign sign --yes --bundle evidence/signature.sigstore.json "$image"
+          cosign verify --certificate-identity "$identity" \
+            --certificate-oidc-issuer https://token.actions.githubusercontent.com \
+            "$image" > evidence/signature-verification.json
+      - name: Publish only the already verified and signed digest
+        run: |
+          image=$(cat evidence/final-image.txt)
+          while IFS= read -r arch; do
+            arch="${arch//\//-}"
+            staged=$(cat "artifacts/image-build-$arch/image.txt")
+            # Preserve the existing architecture tag names, after all gates.
+            docker buildx imagetools create --tag "$STAGING_REPOSITORY:arch-$arch-$VERSION" "$staged"
+          done < <(jq -r '.include[].architecture' <<< "$MATRIX")
+          docker buildx imagetools create --tag "$REPOSITORY:$VERSION" "$image"
+          [[ "$(docker buildx imagetools inspect "$REPOSITORY:$VERSION" --format '{{.Manifest.Digest}}')" == "${image##*@}" ]]
+          if [[ "$PUBLISH_LATEST" == true ]]; then
+            docker buildx imagetools create --tag "$REPOSITORY:latest" "$image"
+            [[ "$(docker buildx imagetools inspect "$REPOSITORY:latest" --format '{{.Manifest.Digest}}')" == "${image##*@}" ]]
+          fi
+      - name: Retain final reports/SBOM/provenance/signature even on failure
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: image-final-evidence
+          path: evidence/
+          retention-days: 90
+          if-no-files-found: error
+      - name: Attach binaries and all evidence, then publish GitHub release
+        if: ${{ !inputs.nightly }}
+        run: |
+          tar -czf image-security-evidence.tar.gz evidence artifacts/image-*
+          # Create a draft only if absent. API errors must not masquerade as absence.
+          gh api --paginate "repos/$GITHUB_REPOSITORY/releases" --jq '.[]' | jq -s . > releases.json
+          if ! jq -e --arg tag "$VERSION" '.[] | select(.tag_name == $tag)' releases.json >/dev/null; then
+            gh release create "$VERSION" --target "$REVISION" --title "$VERSION" --notes "" --draft
+          fi
+          gh release upload "$VERSION" artifacts/agent-*.tar/agent-*.tar image-security-evidence.tar.gz --clobber
+          gh release edit "$VERSION" --draft=false --latest --title "$VERSION"

.github/workflows/nightly-build.yml

@@ -1,60 +1,19 @@
 name: Nightly build
-
 on:
-  # Triggers the workflow every day at 9PM (CET).
   schedule:
     - cron: "0 22 * * *"
-  # Allows manual triggering from the Actions tab.
   workflow_dispatch:
 
+permissions:
+  contents: write
+
 jobs:
-  nightly-build-amd64:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        architecture: [amd64]
-    steps:
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          ref: master
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      - name: Available platforms
-        run: echo ${{ steps.buildx.outputs.platforms }}
-      - name: Run Buildx
-        run: docker buildx build --platform linux/${{matrix.architecture}} -t kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7) --push .
-      - name: Create new and append to manifest
-        run: docker buildx imagetools create -t kerberos/agent-nightly:$(echo $GITHUB_SHA | cut -c1-7) kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7)
-  nightly-build-other:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        architecture: [arm64, arm/v7, arm/v6]
-    steps:
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          ref: master
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      - name: Available platforms
-        run: echo ${{ steps.buildx.outputs.platforms }}
-      - name: Run Buildx
-        run: docker buildx build --platform linux/${{matrix.architecture}} -t kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7) --push .
-      - name: Create new and append to manifest
-        run: docker buildx imagetools create --append -t kerberos/agent-nightly:$(echo $GITHUB_SHA | cut -c1-7) kerberos/agent-nightly:arch-$(echo ${{matrix.architecture}} | tr / -)-$(echo $GITHUB_SHA | cut -c1-7)
+  nightly:
+    permissions:
+      contents: read
+      id-token: write
+    uses: ./.github/workflows/image-release.yml
+    with:
+      source_ref: master
+      nightly: true
+    secrets: inherit

.github/workflows/pr-build.yml

@@ -1,48 +1,119 @@
 name: Build pull request
 on:
   pull_request:
-    types: [opened, synchronize]
+    types: [opened, synchronize, reopened]
 
-env:
-  REPO: kerberos/agent
+permissions:
+  contents: read
+
+defaults:
+  run:
+    shell: bash
 
 jobs:
   build:
     runs-on: ${{ matrix.runner }}
-    permissions:
-      contents: write
     strategy:
+      fail-fast: false
       matrix:
         include:
           - architecture: amd64
             runner: ubuntu-24.04
-            dockerfile: Dockerfile
           - architecture: arm64
             runner: ubuntu-24.04-arm
-            dockerfile: Dockerfile.arm64
+    env:
+      ARCH: ${{ matrix.architecture }}
     steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      - uses: benjlevesque/short-sha@v2.1
-        id: short-sha
-        with:
-          length: 7
-      - name: Run Build
+          persist-credentials: false
+      - name: Initialize retained runtime/scanner state
         run: |
-          docker build -t ${{ matrix.architecture }} -f ${{ matrix.dockerfile }} .
-          CID=$(docker create ${{matrix.architecture}})
-          docker cp ${CID}:/home/agent ./output-${{matrix.architecture}}
-          docker rm ${CID}
-      - name: Strip binary
-        run: tar -cf agent-${{matrix.architecture}}.tar -C output-${{matrix.architecture}} . && rm -rf output-${{matrix.architecture}}
-      - name: Upload artifact
-        uses: actions/upload-artifact@v4
+          mkdir -p evidence/scanner-install
+          printf '%s\n' '{"status":"not_started","reason":"waiting_for_native_build_and_smoke"}' \
+            > evidence/scanner-install/install.json
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
-          name: agent-${{matrix.architecture}}.tar
-          path: agent-${{matrix.architecture}}.tar
-
+          ref: ${{ github.sha }}
+          path: source
+          persist-credentials: false
+      - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
+      - name: Validate fail-closed policy helpers
+        run: python3 .github/scripts/test-image-policy.py
+      - name: Build once, populating OCI metadata from CI
+        env:
+          BUILDX_METADATA_PROVENANCE: max
+        run: |
+          mkdir -p evidence/build
+          revision=$(git -C source rev-parse HEAD)
+          version="${revision:0:7}"
+          created=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+          printf '%s\n' "$revision" > evidence/build/source-revision.txt
+          # Classic Docker cannot load an attested index. Disable attestations
+          # only for this local export; BUILDX_METADATA_PROVENANCE=max retains
+          # full build provenance/materials separately, verified below.
+          docker buildx build --platform "linux/$ARCH" --provenance=false \
+            --build-arg "VERSION=$version" --build-arg "REVISION=$revision" --build-arg "CREATED=$created" \
+            --label "org.opencontainers.image.version=$version" \
+            --label "org.opencontainers.image.revision=$revision" \
+            --label "org.opencontainers.image.created=$created" \
+            --label "org.opencontainers.image.source=https://github.com/$GITHUB_REPOSITORY" \
+            --metadata-file evidence/build/build-metadata.json --tag "$ARCH" \
+            --output type=docker source
+          jq -e '."buildx.build.provenance" |
+            select(type == "object" and length > 0) |
+            select(.buildConfig | type == "object" and length > 0)' \
+            evidence/build/build-metadata.json > evidence/build/provenance.json
+          python3 .github/scripts/image-evidence.py evidence/build
+      - name: Extract release payload
+        run: |
+          cid=$(docker create "$ARCH")
+          trap 'docker rm -f "$cid" >/dev/null' EXIT
+          docker cp "$cid:/home/agent" "output-$ARCH"
+      - name: Test production image
+        id: smoke
+        run: sh source/deployments/docker/test-image.sh "$ARCH"
+      - name: Strip binary
+        run: tar -cf "agent-$ARCH.tar" -C "output-$ARCH" . && rm -rf "output-$ARCH"
+      - name: Install pinned public OCI scanner after runtime tests
+        run: bash .github/scripts/install-trivy.sh
+      - name: Source/UI dependency and final image gates with SPDX SBOMs
+        run: |
+          failed=0
+          if ! python3 .github/scripts/image-policy.py source evidence/source; then
+            failed=1
+          fi
+          if ! python3 .github/scripts/image-policy.py "$ARCH" "evidence/$ARCH" --platform "linux/$ARCH"; then
+            failed=1
+          fi
+          exit "$failed"
+      - name: Upload artifact only after secret and vulnerability gates pass
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: agent-${{ matrix.architecture }}.tar
+          path: agent-${{ matrix.architecture }}.tar
+          if-no-files-found: error
+      - name: Record runtime result independently of scanner availability
+        if: always()
+        env:
+          SMOKE_OUTCOME: ${{ steps.smoke.outcome }}
+        run: |
+          mkdir -p evidence
+          python3 - <<'PY'
+          import json
+          import os
+          from pathlib import Path
+          Path("evidence/runtime-smoke.json").write_text(json.dumps({
+              "architecture": os.environ["ARCH"],
+              "outcome": os.environ["SMOKE_OUTCOME"],
+              "workflow_revision": os.environ["GITHUB_SHA"],
+          }, indent=2) + "\n")
+          PY
+      - name: Retain reports on failure
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: image-pr-evidence-${{ matrix.architecture }}
+          path: evidence/
+          retention-days: 90
+          if-no-files-found: error

.github/workflows/react.yml

@@ -11,21 +11,22 @@ on:
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     strategy:
       matrix:
-        node-version: [14.x, 16.x]
+        node-version: [22.x]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v2
+      uses: actions/setup-node@v4
       with:
         node-version: ${{ matrix.node-version }}
+        cache: yarn
+        cache-dependency-path: ui/yarn.lock
     - name: Yarn install
-      run: cd ui && yarn
+      run: cd ui && yarn --frozen-lockfile
     - name: Yarn test
       run: cd ui && yarn test --passWithNoTests
     - name: Yarn build

.github/workflows/release-bump.yml

@@ -1,168 +1,36 @@
 name: Bump release
-
 on:
   workflow_dispatch:
     inputs:
       bump:
-        description: "Which part of the version to bump"
+        description: Which part of the version to bump
         required: true
         default: patch
         type: choice
-        options:
-          - major
-          - minor
-          - patch
+        options: [major, minor, patch]
 
 permissions:
-  contents: write
-
-env:
-  REPO: kerberos/agent
+  contents: read
 
 jobs:
-  # Determine the next version, create the GitHub release and expose the tag.
   bump-release:
-    uses: uug-ai/workflows/.github/workflows/release-bump.yml@main
+    permissions:
+      contents: write
+    # Verified revision supports reserving a tag with an unpublished draft.
+    uses: uug-ai/workflows/.github/workflows/release-bump.yml@f31d4260a833e82228ca0e9b9163231a970feeb8
     with:
-      bump: ${{ github.event.inputs.bump }}
+      bump: ${{ inputs.bump }}
+      draft: true
     secrets: inherit
 
-  # Publish the platform image to the uug-ai GitHub Container Registry
-  # (ghcr.io/uug-ai/agent-platform).
-  #release:
-  #  needs: bump-release
-  #  uses: uug-ai/workflows/.github/workflows/release-create.yml@main
-  #  with:
-  #    organization: uug-ai
-  #    project: ${{ github.event.repository.name }}
-  #    tag: ${{ needs.bump-release.outputs.tag }}
-  #    docker_context: "."
-  #    create_gitops_pr: false
-  #    runner_matrix: >-
-  #       [
-  #         {"architecture":"amd64","runner":"ubuntu-24.04"},
-  #         {"architecture":"arm64","runner":"ubuntu-24.04-arm"}
-  #      ]
-  #  secrets: inherit
-
-  # Everything below mirrors the agent's own release-create.yml pipeline and
-  # publishes the multi-arch image to the kerberos/agent Docker Hub repo, driven
-  # by the freshly bumped tag instead of a `release: created` event.
-  build-amd64:
+  release:
     needs: bump-release
-    runs-on: ubuntu-24.04
     permissions:
       contents: write
-    strategy:
-      matrix:
-        architecture: [amd64]
-    steps:
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      - name: Checkout
-        uses: actions/checkout@v3
-      - uses: benjlevesque/short-sha@v2.1
-        id: short-sha
-        with:
-          length: 7
-      - name: Run Build
-        run: |
-          docker build --provenance=false --build-arg VERSION=${{ needs.bump-release.outputs.tag }} -t ${{matrix.architecture}} .
-          CID=$(docker create ${{matrix.architecture}})
-          docker cp ${CID}:/home/agent ./output-${{matrix.architecture}}
-          docker rm ${CID}
-      - name: Strip binary
-        run: tar -cf agent-${{matrix.architecture}}.tar -C output-${{matrix.architecture}} . && rm -rf output-${{matrix.architecture}}
-      - name: Build and push Docker image
-        run: |
-          docker tag ${{matrix.architecture}} $REPO-arch:arch-${{matrix.architecture}}-${{ needs.bump-release.outputs.tag }}
-          docker push $REPO-arch:arch-${{matrix.architecture}}-${{ needs.bump-release.outputs.tag }}
-      - name: Upload artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: agent-${{matrix.architecture}}.tar
-          path: agent-${{matrix.architecture}}.tar
-
-  build-arm64:
-    needs: bump-release
-    runs-on: ubuntu-24.04-arm
-    permissions:
-      contents: write
-    strategy:
-      matrix:
-        architecture: [arm64]
-    steps:
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      - name: Checkout
-        uses: actions/checkout@v3
-      - uses: benjlevesque/short-sha@v2.1
-        id: short-sha
-        with:
-          length: 7
-      - name: Run Build
-        run: |
-          docker build --provenance=false --build-arg VERSION=${{ needs.bump-release.outputs.tag }} -t ${{matrix.architecture}} -f Dockerfile.arm64 .
-          CID=$(docker create ${{matrix.architecture}})
-          docker cp ${CID}:/home/agent ./output-${{matrix.architecture}}
-          docker rm ${CID}
-      - name: Strip binary
-        run: tar -cf agent-${{matrix.architecture}}.tar -C output-${{matrix.architecture}} . && rm -rf output-${{matrix.architecture}}
-      - name: Build and push Docker image
-        run: |
-          docker tag ${{matrix.architecture}} $REPO-arch:arch-${{matrix.architecture}}-${{ needs.bump-release.outputs.tag }}
-          docker push $REPO-arch:arch-${{matrix.architecture}}-${{ needs.bump-release.outputs.tag }}
-      - name: Upload artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: agent-${{matrix.architecture}}.tar
-          path: agent-${{matrix.architecture}}.tar
-
-  create-manifest:
-    runs-on: ubuntu-24.04
-    needs: [bump-release, build-amd64, build-arm64]
-    steps:
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      - name: Create and push multi-arch manifest
-        run: |
-          docker manifest create $REPO:${{ needs.bump-release.outputs.tag }} \
-            $REPO-arch:arch-amd64-${{ needs.bump-release.outputs.tag }} \
-            $REPO-arch:arch-arm64-${{ needs.bump-release.outputs.tag }}
-          docker manifest push $REPO:${{ needs.bump-release.outputs.tag }}
-      - name: Create and push latest manifest
-        run: |
-          docker manifest create $REPO:latest \
-            $REPO-arch:arch-amd64-${{ needs.bump-release.outputs.tag }} \
-            $REPO-arch:arch-arm64-${{ needs.bump-release.outputs.tag }}
-          docker manifest push $REPO:latest
-
-  create-release:
-    runs-on: ubuntu-24.04
-    needs: [bump-release, build-amd64, build-arm64]
-    permissions:
-      contents: write
-    steps:
-      - name: Download all artifacts
-        uses: actions/download-artifact@v4
-      - name: Create a release
-        uses: ncipollo/release-action@v1
-        with:
-          latest: true
-          allowUpdates: true
-          name: ${{ needs.bump-release.outputs.tag }}
-          tag: ${{ needs.bump-release.outputs.tag }}
-          generateReleaseNotes: false
-          omitBodyDuringUpdate: true
-          artifacts: "agent-*.tar/agent-*.tar"
-
-  
+      id-token: write
+    uses: ./.github/workflows/image-release.yml
+    with:
+      tag: ${{ needs.bump-release.outputs.tag }}
+      source_ref: ${{ format('refs/tags/{0}', needs.bump-release.outputs.tag) }}
+      publish_latest: true
+    secrets: inherit

.github/workflows/release-create.yml

@@ -5,126 +5,21 @@ on:
   workflow_dispatch:
     inputs:
       tag:
-        description: "Tag for the Docker image"
+        description: Tag for the Docker image
         required: true
-        default: "test"
+        default: test
 
-env:
-  REPO: kerberos/agent
+permissions:
+  contents: read
 
 jobs:
-  build-amd64:
-    runs-on: ubuntu-24.04
+  release:
     permissions:
       contents: write
-    strategy:
-      matrix:
-        architecture: [amd64]
-    steps:
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      - name: Checkout
-        uses: actions/checkout@v3
-      - uses: benjlevesque/short-sha@v2.1
-        id: short-sha
-        with:
-          length: 7
-      - name: Run Build
-        run: |
-          docker build --provenance=false --build-arg VERSION=${{github.event.inputs.tag || github.ref_name}} -t ${{matrix.architecture}} .
-          CID=$(docker create ${{matrix.architecture}})
-          docker cp ${CID}:/home/agent ./output-${{matrix.architecture}}
-          docker rm ${CID}
-      - name: Strip binary
-        run: tar -cf agent-${{matrix.architecture}}.tar -C output-${{matrix.architecture}} . && rm -rf output-${{matrix.architecture}}
-      - name: Build and push Docker image
-        run: |
-          docker tag ${{matrix.architecture}} $REPO-arch:arch-${{matrix.architecture}}-${{github.event.inputs.tag || github.ref_name}}
-          docker push $REPO-arch:arch-${{matrix.architecture}}-${{github.event.inputs.tag || github.ref_name}}
-      - name: Upload artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: agent-${{matrix.architecture}}.tar
-          path: agent-${{matrix.architecture}}.tar
-          
-  build-arm64:
-    runs-on: ubuntu-24.04-arm 
-    permissions:
-      contents: write
-    strategy:
-      matrix:
-        architecture: [arm64]
-    steps:
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      - name: Checkout
-        uses: actions/checkout@v3
-      - uses: benjlevesque/short-sha@v2.1
-        id: short-sha
-        with:
-          length: 7
-      - name: Run Build
-        run: |
-          docker build --provenance=false --build-arg VERSION=${{github.event.inputs.tag || github.ref_name}} -t ${{matrix.architecture}} -f Dockerfile.arm64 .
-          CID=$(docker create ${{matrix.architecture}})
-          docker cp ${CID}:/home/agent ./output-${{matrix.architecture}}
-          docker rm ${CID}
-      - name: Strip binary
-        run: tar -cf agent-${{matrix.architecture}}.tar -C output-${{matrix.architecture}} . && rm -rf output-${{matrix.architecture}}
-      - name: Build and push Docker image
-        run: |
-          docker tag ${{matrix.architecture}} $REPO-arch:arch-${{matrix.architecture}}-${{github.event.inputs.tag || github.ref_name}}
-          docker push $REPO-arch:arch-${{matrix.architecture}}-${{github.event.inputs.tag || github.ref_name}}
-      - name: Upload artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: agent-${{matrix.architecture}}.tar
-          path: agent-${{matrix.architecture}}.tar
-
-  create-manifest:
-    runs-on: ubuntu-24.04
-    needs: [build-amd64, build-arm64]
-    steps:
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      - name: Create and push multi-arch manifest
-        run: |
-          docker manifest create $REPO:${{ github.event.inputs.tag || github.ref_name }} \
-            $REPO-arch:arch-amd64-${{github.event.inputs.tag || github.ref_name}} \
-            $REPO-arch:arch-arm64-${{github.event.inputs.tag || github.ref_name}}
-          docker manifest push $REPO:${{ github.event.inputs.tag || github.ref_name }}
-      - name: Create and push latest manifest
-        run: |
-          docker manifest create $REPO:latest \
-            $REPO-arch:arch-amd64-${{github.event.inputs.tag || github.ref_name}} \
-            $REPO-arch:arch-arm64-${{github.event.inputs.tag || github.ref_name}}
-          docker manifest push $REPO:latest
-        if: github.event.inputs.tag == 'test'
-
-  create-release:
-    runs-on: ubuntu-24.04
-    needs: [build-amd64, build-arm64]
-    permissions:
-      contents: write
-    steps:
-      - name: Download all artifacts
-        uses: actions/download-artifact@v4
-      - name: Create a release
-        uses: ncipollo/release-action@v1
-        with:
-          latest: true
-          allowUpdates: true
-          name: ${{ github.event.inputs.tag || github.ref_name }}
-          tag: ${{ github.event.inputs.tag || github.ref_name }}
-          generateReleaseNotes: false
-          omitBodyDuringUpdate: true
-          artifacts: "agent-*.tar/agent-*.tar"
+      id-token: write
+    uses: ./.github/workflows/image-release.yml
+    with:
+      tag: ${{ inputs.tag || github.event.release.tag_name }}
+      source_ref: ${{ github.event_name == 'release' && format('refs/tags/{0}', github.event.release.tag_name) || github.sha }}
+      publish_latest: ${{ github.event_name == 'workflow_dispatch' && inputs.tag == 'test' }}
+    secrets: inherit

.vscode/launch.json

@@ -14,10 +14,10 @@
                 "-action",
                 "run",
                 "-port",
-                "8080"
+                "8082"
             ],
             "envFile": "${workspaceFolder}/machinery/.env.local",
-            "buildFlags": "--tags dynamic",
+            "buildFlags": "--tags dynamic,moq",
             "env": {
                 "GOWORK": "off"
             },

.vscode/tasks.json

@@ -0,0 +1,14 @@
+{
+  "version": "2.0.0",
+  "tasks": [
+    {
+      "label": "agent: moq verify",
+      "type": "shell",
+      "command": "bash ./verify-moq-devcontainer.sh",
+      "options": {
+        "cwd": "${workspaceFolder}/machinery"
+      },
+      "problemMatcher": []
+    }
+  ]
+}

Dockerfile

@@ -1,54 +1,58 @@
 
-ARG BASE_IMAGE_VERSION=amd64-ddbe40e
+ARG GO_IMAGE=golang:1.26.10-trixie@sha256:a050dcceee6eb9bf110151662dfac7662f3bec40927c7dee48731e3d92083d42
+ARG RUNTIME_IMAGE=debian:trixie-slim@sha256:f6b08b09a05725f9431502d3b5714f5c4576e82c1a84fa1debe0c24a7e0c3ec4
 ARG VERSION=0.0.0
-FROM kerberos/base:${BASE_IMAGE_VERSION} AS build-machinery
+FROM ${GO_IMAGE} AS build-machinery
 LABEL AUTHOR=uug.ai
 
 # Re-declare VERSION inside this stage so the value passed via
 # `--build-arg VERSION=...` (e.g. the release tag) is available below.
 # ARGs declared before the first FROM are not visible inside build stages.
 ARG VERSION
+ARG TARGETARCH
 
 ENV GOROOT=/usr/local/go
 ENV GOPATH=/go
 ENV PATH=$GOPATH/bin:$GOROOT/bin:/usr/local/lib:$PATH
-ENV GOSUMDB=off
+ENV GOTOOLCHAIN=local
+ENV CGO_ENABLED=1
 
 ##########################################
 # Installing some additional dependencies.
 
-RUN apt-get upgrade -y && apt-get update && apt-get install -y --fix-missing --no-install-recommends \
+RUN apt-get update && apt-get install -y --no-install-recommends \
 	git build-essential cmake pkg-config unzip libgtk2.0-dev \
-	curl ca-certificates libcurl4-openssl-dev libssl-dev libjpeg62-turbo-dev && \
+	curl ca-certificates libavcodec-dev libavutil-dev libcurl4-openssl-dev \
+	libssl-dev libjpeg62-turbo-dev libswscale-dev && \
 	rm -rf /var/lib/apt/lists/*
 
 ##############################################################################
-# Copy all the relevant source code in the Docker image, so we can build this.
+# Copy dependency metadata first so module downloads can be cached separately.
 
 RUN mkdir -p /go/src/github.com/kerberos-io/agent
-COPY machinery /go/src/github.com/kerberos-io/agent/machinery
-RUN rm -rf /go/src/github.com/kerberos-io/agent/machinery/.env
+WORKDIR /go/src/github.com/kerberos-io/agent/machinery
+COPY machinery/go.mod machinery/go.sum ./
+RUN go version && go mod download && go mod verify
 
-##################################################################
-# Get the latest commit hash, so we know which version we're running
-COPY .git /go/src/github.com/kerberos-io/agent/.git
-RUN cd /go/src/github.com/kerberos-io/agent/.git && git log --format="%H" -n 1 | head -c7 > /go/src/github.com/kerberos-io/agent/machinery/version
-RUN cat /go/src/github.com/kerberos-io/agent/machinery/version
+##############################################################################
+# Copy the rest of the source after dependencies are primed.
+
+COPY machinery ./
 
 ##################
 # Build Machinery
 
-RUN cd /go/src/github.com/kerberos-io/agent/machinery && \
-	go mod download && \
-	if [ -z "${VERSION}" ] || [ "${VERSION}" = "0.0.0" ]; then \
-		VERSION=$(cd /go/src/github.com/kerberos-io/agent && git describe --tags --always 2>/dev/null || echo "0.0.0"); \
-	fi && \
-	go build -tags timetzdata,netgo,osusergo --ldflags "-s -w -X github.com/kerberos-io/agent/machinery/src/utils.VERSION=${VERSION} -extldflags '-static -latomic'" main.go && \
+RUN RESOLVED_VERSION="${VERSION:-0.0.0}" && \
+	printf '%s' "${RESOLVED_VERSION}" > version && \
+	BUILD_TAGS=timetzdata,netgo,osusergo && \
+	case "${TARGETARCH:-$(go env GOARCH)}" in amd64|arm64) BUILD_TAGS="moq,${BUILD_TAGS}" ;; esac && \
+	go build -mod=readonly -trimpath -buildvcs=false -o main -tags "${BUILD_TAGS}" --ldflags "-s -w -X github.com/kerberos-io/agent/machinery/src/utils.VERSION=${RESOLVED_VERSION}" . && \
 	mkdir -p /agent && \
 	mv main /agent && \
 	mv version /agent && \
 	mv data /agent && \
 	mkdir -p /agent/data/cloud && \
+	mkdir -p /agent/data/tus && \
 	mkdir -p /agent/data/snapshots && \
 	mkdir -p /agent/data/log && \
 	mkdir -p /agent/data/recordings && \
@@ -67,7 +71,7 @@ RUN cp -r /agent ./
 
 RUN /dist/agent/main version
 
-FROM node:22-alpine AS build-ui
+FROM node:22-alpine@sha256:fe4f6d74e20370acb04ad889077d14d0f64a9c8ea6a42bc60cddf1e844699914 AS build-ui
 
 RUN apk update && apk upgrade --available && sync
 
@@ -75,9 +79,11 @@ RUN apk update && apk upgrade --available && sync
 # Build Web (React app)
 
 RUN mkdir -p /go/src/github.com/kerberos-io/agent/machinery/www
-COPY ui /go/src/github.com/kerberos-io/agent/ui
-RUN cd /go/src/github.com/kerberos-io/agent/ui && rm -rf yarn.lock && yarn config set network-timeout 300000 && \
-	yarn && yarn build
+WORKDIR /go/src/github.com/kerberos-io/agent/ui
+COPY ui/package.json ui/yarn.lock ./
+RUN yarn config set network-timeout 300000 && yarn --frozen-lockfile
+COPY ui ./
+RUN yarn build
 
 ####################################
 # Let's create a /dist folder containing just the files necessary for runtime.
@@ -89,59 +95,64 @@ RUN mkdir -p ./agent && cp -r /go/src/github.com/kerberos-io/agent/machinery/www
 ############################################
 # Publish main binary to GitHub release
 
-FROM alpine:latest
+FROM ${RUNTIME_IMAGE} AS runtime-rootfs
 
-############################
-# Protect by non-root user.
-
-RUN addgroup -S kerberosio && adduser -S agent -G kerberosio && addgroup agent video
+RUN apt-get update && apt-get install -y --no-install-recommends \
+	ca-certificates ffmpeg libatomic1 libstdc++6 \
+	dash coreutils findutils grep sed tzdata ncurses-base && \
+	groupadd --system --gid 999 kerberosio && \
+	useradd --system --uid 999 --gid kerberosio --groups video \
+		--home-dir /home/agent --no-create-home agent
 
 #################################
 # Copy files from previous images
 
-COPY --chown=0:0 --from=build-machinery /dist /
-COPY --chown=0:0 --from=build-ui /dist /
+COPY --chown=0:0 --from=build-machinery /dist/agent/ /runtime/home/agent/
+COPY --chown=0:0 --from=build-ui /dist/agent/ /runtime/home/agent/
+COPY deployments/docker/assemble-rootfs.sh /usr/local/bin/assemble-rootfs
 
-RUN apk update && apk add ca-certificates curl ffmpeg libstdc++ libc6-compat --no-cache && rm -rf /var/cache/apk/*
+RUN chmod 0755 /runtime/home/agent /runtime/home/agent/main && \
+	cp /runtime/home/agent/data/config/config.json /runtime/home/agent/data/config.template.json && \
+	chown -R agent:kerberosio /runtime/home/agent/data && \
+	sh /usr/local/bin/assemble-rootfs
 
-##################
-# Try running agent
+FROM scratch
 
-RUN mv /agent/* /home/agent/
-RUN /home/agent/main version
+COPY --from=runtime-rootfs /runtime/ /
 
-#######################
-# Make template config
+ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+ENV HOME=/home/agent
+ENV XDG_CACHE_HOME=/tmp/agent-cache
+ENV AGENT_PORT=8080
 
-RUN cp /home/agent/data/config/config.json /home/agent/data/config.template.json
-
-###########################
-# Set permissions correctly
-
-RUN chown -R agent:kerberosio /home/agent/data
-RUN chown -R agent:kerberosio /home/agent/www
-
-###########################
-# Grant the necessary root capabilities to the process trying to bind to the privileged port
-RUN apk add libcap && setcap 'cap_net_bind_service=+ep' /home/agent/main
+ARG VERSION
+ARG REVISION=unknown
+ARG CREATED
+LABEL org.opencontainers.image.title="Kerberos Agent" \
+	org.opencontainers.image.description="Camera and media agent with a minimal dynamic scratch runtime" \
+	org.opencontainers.image.source="https://github.com/kerberos-io/agent" \
+	org.opencontainers.image.version="${VERSION}" \
+	org.opencontainers.image.revision="${REVISION}" \
+	org.opencontainers.image.created="${CREATED}" \
+	org.opencontainers.image.vendor="UUG.AI"
 
 ###################
 # Run non-root user
 
-USER agent
+USER 999:999
 
 ######################################
-# By default the app runs on port 80
+# The image uses an unprivileged HTTP port.
 
-EXPOSE 80
+EXPOSE 8080/tcp
 
 ######################################
 # Check if agent is still running
 
-HEALTHCHECK CMD curl --fail http://localhost:80 || exit 1   
+HEALTHCHECK --timeout=10s CMD ["/home/agent/main", "-action", "healthcheck"]
 
 ###################################################
 # Leeeeettttt'ssss goooooo!!!
 # Run the shizzle from the right working directory.
 WORKDIR /home/agent
-CMD ["./main", "-action", "run", "-port", "80"]
+CMD ["./main", "-action", "run"]

Dockerfile.arm64

@@ -1,147 +0,0 @@
-
-ARG BASE_IMAGE_VERSION=arm64-ddbe40e
-ARG VERSION=0.0.0
-FROM kerberos/base:${BASE_IMAGE_VERSION} AS build-machinery
-LABEL AUTHOR=uug.ai
-
-# Re-declare VERSION inside this stage so the value passed via
-# `--build-arg VERSION=...` (e.g. the release tag) is available below.
-# ARGs declared before the first FROM are not visible inside build stages.
-ARG VERSION
-
-ENV GOROOT=/usr/local/go
-ENV GOPATH=/go
-ENV PATH=$GOPATH/bin:$GOROOT/bin:/usr/local/lib:$PATH
-ENV GOSUMDB=off
-
-##########################################
-# Installing some additional dependencies.
-
-RUN apt-get upgrade -y && apt-get update && apt-get install -y --fix-missing --no-install-recommends \
-	git build-essential cmake pkg-config unzip libgtk2.0-dev \
-	curl ca-certificates libcurl4-openssl-dev libssl-dev libjpeg62-turbo-dev && \
-	rm -rf /var/lib/apt/lists/*
-
-##############################################################################
-# Copy all the relevant source code in the Docker image, so we can build this.
-
-RUN mkdir -p /go/src/github.com/kerberos-io/agent
-COPY machinery /go/src/github.com/kerberos-io/agent/machinery
-RUN rm -rf /go/src/github.com/kerberos-io/agent/machinery/.env
-
-##################################################################
-# Get the latest commit hash, so we know which version we're running
-COPY .git /go/src/github.com/kerberos-io/agent/.git
-RUN cd /go/src/github.com/kerberos-io/agent/.git && git log --format="%H" -n 1 | head -c7 > /go/src/github.com/kerberos-io/agent/machinery/version
-RUN cat /go/src/github.com/kerberos-io/agent/machinery/version
-
-##################
-# Build Machinery
-
-RUN cd /go/src/github.com/kerberos-io/agent/machinery && \
-	go mod download && \
-	if [ -z "${VERSION}" ] || [ "${VERSION}" = "0.0.0" ]; then \
-		VERSION=$(cd /go/src/github.com/kerberos-io/agent && git describe --tags --always 2>/dev/null || echo "0.0.0"); \
-	fi && \
-	go build -tags timetzdata,netgo,osusergo --ldflags "-s -w -X github.com/kerberos-io/agent/machinery/src/utils.VERSION=${VERSION} -extldflags '-static -latomic'" main.go && \
-	mkdir -p /agent && \
-	mv main /agent && \
-	mv version /agent && \
-	mv data /agent && \
-	mkdir -p /agent/data/cloud && \
-	mkdir -p /agent/data/snapshots && \
-	mkdir -p /agent/data/log && \
-	mkdir -p /agent/data/recordings && \
-	mkdir -p /agent/data/capture-test && \
-	mkdir -p /agent/data/config
-
-####################################
-# Let's create a /dist folder containing just the files necessary for runtime.
-# Later, it will be copied as the / (root) of the output image.
-
-WORKDIR /dist
-RUN cp -r /agent ./
-
-####################################################################################
-# This will collect dependent libraries so they're later copied to the final image.
-
-RUN /dist/agent/main version
-
-FROM node:22-alpine AS build-ui
-
-RUN apk update && apk upgrade --available && sync
-
-########################
-# Build Web (React app)
-
-RUN mkdir -p /go/src/github.com/kerberos-io/agent/machinery/www
-COPY ui /go/src/github.com/kerberos-io/agent/ui
-RUN cd /go/src/github.com/kerberos-io/agent/ui && rm -rf yarn.lock && yarn config set network-timeout 300000 && \
-	yarn && yarn build
-
-####################################
-# Let's create a /dist folder containing just the files necessary for runtime.
-# Later, it will be copied as the / (root) of the output image.
-
-WORKDIR /dist
-RUN mkdir -p ./agent && cp -r /go/src/github.com/kerberos-io/agent/machinery/www ./agent/
-
-############################################
-# Publish main binary to GitHub release
-
-FROM alpine:latest
-
-############################
-# Protect by non-root user.
-
-RUN addgroup -S kerberosio && adduser -S agent -G kerberosio && addgroup agent video
-
-#################################
-# Copy files from previous images
-
-COPY --chown=0:0 --from=build-machinery /dist /
-COPY --chown=0:0 --from=build-ui /dist /
-
-RUN apk update && apk add ca-certificates curl ffmpeg libstdc++ libc6-compat --no-cache && rm -rf /var/cache/apk/*
-
-##################
-# Try running agent
-
-RUN mv /agent/* /home/agent/
-RUN /home/agent/main version
-
-#######################
-# Make template config
-
-RUN cp /home/agent/data/config/config.json /home/agent/data/config.template.json
-
-###########################
-# Set permissions correctly
-
-RUN chown -R agent:kerberosio /home/agent/data
-RUN chown -R agent:kerberosio /home/agent/www
-
-###########################
-# Grant the necessary root capabilities to the process trying to bind to the privileged port
-RUN apk add libcap && setcap 'cap_net_bind_service=+ep' /home/agent/main
-
-###################
-# Run non-root user
-
-USER agent
-
-######################################
-# By default the app runs on port 80
-
-EXPOSE 80
-
-######################################
-# Check if agent is still running
-
-HEALTHCHECK CMD curl --fail http://localhost:80 || exit 1   
-
-###################################################
-# Leeeeettttt'ssss goooooo!!!
-# Run the shizzle from the right working directory.
-WORKDIR /home/agent
-CMD ["./main", "-action", "run", "-port", "80"]

README-RTSPS-TLS.md

@@ -0,0 +1,645 @@
+# RTSPS and TLS certificates
+
+This guide explains how Kerberos Agent connects to an IP camera over RTSPS,
+how to issue a camera certificate with a private CA, and how to validate the
+complete trust path. It also explains why some apparently corrupted trust
+bundles can still allow a connection.
+
+The camera-specific steps were verified with a Bosch FLEXIDOME micro 3100i.
+Other Bosch firmware versions may use different labels or ports.
+
+The commands were tested with Smallstep CLI `0.30.6` and OpenSSL `3.5.6` on
+Debian. Check `step certificate sign --help` when using an older Smallstep CLI.
+The OpenSSL isolation flags `-no-CApath` and `-no-CAstore` require a version that
+lists them in `openssl s_client -help`.
+
+## Tested configuration
+
+| Setting | Value |
+| --- | --- |
+| Camera | Bosch FLEXIDOME micro 3100i |
+| Example camera address | `10.0.30.11` |
+| RTSPS port | `9554` |
+| Main stream | `rtsps://<user>:<password>@10.0.30.11:9554/?inst=1` |
+| Sub stream | `rtsps://<user>:<password>@10.0.30.11:9554/?inst=2` |
+| Certificate SAN | `IP Address:10.0.30.11` |
+| Bosch certificate usage | `HTTPS` |
+| Agent trust input | Issuing intermediate plus root CA |
+
+Replace the example address and certificate names throughout this guide. Keep
+camera credentials out of source control and percent-encode reserved URL
+characters in usernames and passwords.
+
+## Mental model
+
+### RTSPS, SRTSP, TLS, and SRTP
+
+- The standard URL scheme is `rtsps://`. Do not use `srtsp://`.
+- Bosch interfaces and documentation may use SRTSP or Secure RTSP as product
+  terminology.
+- RTSPS carries the RTSP control connection over TLS. With gortsplib, media is
+  normally interleaved over the same TCP/TLS connection for this camera.
+- SRTP is a separate media protection mechanism and is negotiated only when the
+  camera advertises an appropriate secure RTP profile.
+
+Encryption alone does not prove which camera the Agent reached. Verified TLS
+also checks that:
+
+1. The camera certificate is signed by a trusted authority.
+2. The certificate is valid at the current time.
+3. The URL host matches a certificate Subject Alternative Name (SAN).
+
+Modern Go verification uses SANs for identity. A Common Name alone is not
+sufficient. Connecting to `10.0.30.11` requires an IP SAN with that exact value,
+not `DNS:10.0.30.11` and not only a device-name DNS SAN.
+
+### Agent behavior
+
+Kerberos Agent uses gortsplib for RTSP and RTSPS. With the normal configuration,
+gortsplib receives a nil custom TLS configuration and Go performs standard
+certificate and hostname verification with the process trust pool.
+
+`AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE=true` is an explicit escape hatch that
+sets `InsecureSkipVerify` for camera clients. It should be false in a verified
+deployment.
+
+## Communication and certificate flow
+
+The certificate is used during the TLS handshake, before the first RTSP command
+is exchanged. It is not attached to `DESCRIBE`, `SETUP`, or `PLAY`, and the CA
+trust bundle is never sent to the camera.
+
+```mermaid
+sequenceDiagram
+  participant Agent as Kerberos Agent
+  participant Trust as Go trust pool
+  participant Camera as Camera RTSPS :9554
+
+  Agent->>Trust: Load system roots and append AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE
+  Agent->>Camera: Open TCP connection
+  Agent->>Camera: Send TLS ClientHello
+  Camera-->>Agent: Send TLS ServerHello and camera certificate
+  Agent->>Trust: Verify chain, validity, serverAuth, and URL host against SAN
+  Trust-->>Agent: Accept or reject the camera identity
+  Agent->>Camera: Complete TLS handshake
+  Note over Agent,Camera: All following traffic is encrypted by TLS
+  Agent->>Camera: DESCRIBE with RTSP authentication
+  Camera-->>Agent: Return SDP and available media tracks
+  Agent->>Camera: SETUP selected video and audio tracks over TCP
+  Agent->>Camera: PLAY
+  Camera-->>Agent: Send interleaved RTP and RTCP media over TLS
+```
+
+The files and keys have distinct roles:
+
+| Material | Location | Purpose | Sent over the connection |
+| --- | --- | --- | --- |
+| Camera leaf certificate | Camera | Identifies the camera and binds its public key to its SAN | Yes, by the camera during the TLS handshake |
+| Camera private key | Camera | Proves that the camera owns the presented certificate | No |
+| Intermediate and root CA PEM bundle | Agent | Lets Go build and trust the camera certificate chain | No |
+| RTSP username and password | Agent configuration or URL | Authenticates the Agent to the RTSP service after TLS succeeds | An authentication response is sent inside TLS; its form depends on the RTSP authentication method |
+
+For an `rtsps://` URL, the Agent parses the URL and gives gortsplib the host and
+TLS settings. gortsplib opens the TCP connection and starts TLS. Go compares the
+certificate presented by the camera with the local trust pool, checks its
+validity period and server usage, and matches the URL hostname or IP address to
+the certificate SAN. Only a successful handshake creates the encrypted channel
+needed for the RTSP exchange.
+
+The Agent then sends `DESCRIBE`, selects the advertised video and audio tracks,
+sends `SETUP`, and starts delivery with `PLAY`. For the tested camera, gortsplib
+uses interleaved TCP, so the RTSP control messages and RTP/RTCP media remain
+inside the same encrypted TLS connection. Main stream, sub stream, and enabled
+audio backchannel clients each establish and verify their own connection.
+
+If certificate verification fails, the TLS handshake does not complete and no
+usable RTSP session is established. Setting
+`AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE=true` keeps traffic encrypted but skips
+certificate-chain and hostname verification, so an attacker could impersonate
+the camera. It is not equivalent to trusting the camera certificate.
+
+## Decide the certificate identity first
+
+Choose the stable name used in every Agent URL before creating the certificate:
+
+- For an IP URL, add that address as an IP SAN.
+- For a DNS URL, add the exact hostname as a DNS SAN.
+- Add both when clients legitimately use both forms.
+
+A certificate stops matching if the camera IP changes. Use a static address,
+DHCP reservation, or stable DNS name.
+
+## Configure RTSPS in the Bosch UI
+
+1. Sign in to the camera as an administrator.
+2. Open **Configuration**.
+3. Open **Network > Network Services**.
+4. Enable **RTSPS**.
+5. Confirm port `9554`, or record the configured alternative.
+6. Click **Set**.
+
+RTSP on port `554` and RTSPS on port `9554` are separate services. Enabling
+RTSPS does not make an `rtsp://` URL secure.
+
+## Generate the private key and CSR on the camera
+
+Keeping the TLS private key on the camera avoids exporting it to an operator
+workstation or deployment system.
+
+1. Open **Service > Certificates**.
+2. Click **Add**.
+3. Select **Generate signing request**.
+4. Select `RSA 2048bit` or the stronger option supported by all clients.
+5. Enter a unique file name, such as `agent-rtsps`.
+6. Enter a descriptive Common Name and any required organization fields.
+7. Click **Generate**.
+8. Download the resulting CSR from the certificate table.
+
+On the tested firmware, this form contains no SAN field. The downloaded CSR
+therefore has no IP SAN. The CA must add the SAN while signing.
+
+Inspect the CSR before signing:
+
+```bash
+openssl req -in camera.csr.pem -noout -verify -subject
+openssl req -in camera.csr.pem -noout -text
+```
+
+The first command must report `Certificate request self-signature verify OK`.
+An absent `Subject Alternative Name` section is expected for this firmware.
+
+## Prepare Smallstep
+
+Use an existing organizational CA when one is available. Creating a new CA
+creates a new long-lived trust domain that must be distributed, protected,
+backed up, and eventually rotated.
+
+### Install the CLI on Debian amd64
+
+```bash
+curl -fsSL \
+  https://dl.smallstep.com/cli/docs-ca-install/latest/step-cli_amd64.deb \
+  -o /tmp/step-cli_amd64.deb
+sudo dpkg -i /tmp/step-cli_amd64.deb
+rm /tmp/step-cli_amd64.deb
+step version
+```
+
+Use the official package matching the host architecture on other systems.
+
+### Create a dedicated offline CA
+
+Skip this section when using an existing CA.
+
+```bash
+umask 077
+mkdir -p "$HOME/.step/secrets" "$HOME/.step/camera"
+
+openssl rand -base64 48 > "$HOME/.step/secrets/camera_ca_password"
+chmod 600 "$HOME/.step/secrets/camera_ca_password"
+
+step ca init \
+  --pki \
+  --name "UUG Camera CA" \
+  --password-file "$HOME/.step/secrets/camera_ca_password"
+```
+
+This produces:
+
+```text
+$HOME/.step/certs/root_ca.crt
+$HOME/.step/certs/intermediate_ca.crt
+$HOME/.step/secrets/root_ca_key
+$HOME/.step/secrets/intermediate_ca_key
+$HOME/.step/secrets/camera_ca_password
+```
+
+The files under `secrets/` are sensitive. Keep them mode `600`, never commit
+them, and back them up to encrypted persistent storage. A devcontainer can be
+rebuilt or deleted; it is not sufficient as the only CA backup.
+
+## Add the SAN while signing
+
+Copy the camera CSR into a protected working directory:
+
+```bash
+cp camera.csr.pem "$HOME/.step/camera/camera.csr.pem"
+```
+
+Create `$HOME/.step/camera/bosch-rtsps.tpl`:
+
+```json
+{
+  "subject": {
+    "commonName": {{ toJson .Insecure.CR.Subject.CommonName }}
+  },
+  "ipAddresses": ["10.0.30.11"],
+  "keyUsage": ["keyEncipherment", "digitalSignature"],
+  "extKeyUsage": ["serverAuth", "clientAuth"]
+}
+```
+
+The template preserves the camera CSR public key, sets the IP identity, and
+creates a TLS leaf rather than a CA certificate.
+
+Sign it with a validity period that ends before the intermediate CA expires.
+A one-year leaf is preferable to a ten-year leaf when automated renewal is
+available:
+
+```bash
+step certificate sign \
+  --template "$HOME/.step/camera/bosch-rtsps.tpl" \
+  --bundle \
+  --not-after 8760h \
+  --password-file "$HOME/.step/secrets/camera_ca_password" \
+  "$HOME/.step/camera/camera.csr.pem" \
+  "$HOME/.step/certs/intermediate_ca.crt" \
+  "$HOME/.step/secrets/intermediate_ca_key" \
+  > "$HOME/.step/camera/bosch-rtsps-chain.pem"
+```
+
+For an online `step-ca`, do not assume `step ca sign` accepts a `--san` flag. It
+does not. Authorize SANs in the one-time token or configure a provisioner
+template that produces the required SANs.
+
+## Validate before upload
+
+Inspect the leaf certificate, which is the first PEM block in the chain file:
+
+```bash
+openssl x509 \
+  -in "$HOME/.step/camera/bosch-rtsps-chain.pem" \
+  -noout -subject -issuer -dates -ext subjectAltName -ext extendedKeyUsage
+```
+
+Confirm the SAN separately because some OpenSSL versions display only the last
+requested extension:
+
+```bash
+openssl x509 \
+  -in "$HOME/.step/camera/bosch-rtsps-chain.pem" \
+  -noout -ext subjectAltName
+```
+
+Verify the path and IP identity:
+
+```bash
+openssl verify \
+  -CAfile "$HOME/.step/certs/root_ca.crt" \
+  -untrusted "$HOME/.step/certs/intermediate_ca.crt" \
+  -verify_ip 10.0.30.11 \
+  "$HOME/.step/camera/bosch-rtsps-chain.pem"
+```
+
+Confirm that the signed leaf uses the exact public key from the camera CSR:
+
+```bash
+csr_key=$(
+  openssl req -in "$HOME/.step/camera/camera.csr.pem" -pubkey -noout |
+    openssl pkey -pubin -outform DER 2>/dev/null |
+    sha256sum | cut -d' ' -f1
+)
+
+cert_key=$(
+  openssl x509 -in "$HOME/.step/camera/bosch-rtsps-chain.pem" -pubkey -noout |
+    openssl pkey -pubin -outform DER 2>/dev/null |
+    sha256sum | cut -d' ' -f1
+)
+
+test "$csr_key" = "$cert_key"
+```
+
+Do not upload a certificate when any of these checks fail.
+
+## Upload and assign the certificate
+
+1. Return to **Service > Certificates**.
+2. Click **Add > Upload certificate**.
+3. Select the leaf-plus-intermediate PEM chain.
+4. Click **Upload** and wait for `100%`.
+5. Confirm that the former CSR row is now a `Certificate`.
+6. Confirm that the key icon is present. It proves that the camera associated
+   the certificate with its retained private key.
+7. Open the new certificate's **Usage** selector.
+8. Select only **HTTPS**.
+9. Leave **CBS client** assigned to the original Bosch `DeviceCertificate`.
+10. Click **Set** and wait for the table to reload.
+
+On the tested firmware, there is no separate SRTSP usage. RTSPS presents the
+certificate assigned to HTTPS. Reassigning HTTPS therefore changes both the
+web interface and RTSPS certificate.
+
+After saving, the expected split is:
+
+| Certificate | Usage |
+| --- | --- |
+| Private-CA camera certificate | `HTTPS` |
+| Bosch `DeviceCertificate` | `CBS client` |
+
+The browser may warn about the new HTTPS certificate until the private root CA
+is trusted by the workstation.
+
+## Account for the Bosch chain behavior
+
+The tested firmware served only the leaf certificate on ports `443` and `9554`,
+even when the uploaded file contained the leaf and intermediate. Uploading the
+intermediate separately as a trusted camera certificate did not change the
+served chain.
+
+Confirm the behavior:
+
+```bash
+openssl s_client \
+  -connect 10.0.30.11:9554 \
+  -showcerts </dev/null 2>/dev/null |
+  grep -c '^-----BEGIN CERTIFICATE-----$'
+```
+
+A result of `1` means the client must already have the issuing intermediate.
+Create a portable trust bundle containing the intermediate and root:
+
+```bash
+step certificate bundle \
+  "$HOME/.step/certs/intermediate_ca.crt" \
+  "$HOME/.step/certs/root_ca.crt" \
+  "$HOME/.step/camera/uug-camera-trust-bundle.pem"
+
+chmod 644 "$HOME/.step/camera/uug-camera-trust-bundle.pem"
+```
+
+The trust bundle is public material. The CA private keys and password are not.
+
+## Configure Kerberos Agent
+
+For a process running directly in the same environment:
+
+```dotenv
+AGENT_CAPTURE_IPCAMERA_RTSP="rtsps://<user>:<password>@10.0.30.11:9554/?inst=1"
+AGENT_CAPTURE_IPCAMERA_SUB_RTSP="rtsps://<user>:<password>@10.0.30.11:9554/?inst=2"
+AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE=false
+AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE=/home/agent/data/config/uug-camera-trust-bundle.pem
+```
+
+For a container, mount the public bundle read-only at the exact path visible
+inside the container. The Agent image creates `/home/agent/data/config` and
+includes Debian's `ca-certificates` package:
+
+```bash
+docker run \
+  -v /secure/config/uug-camera-trust-bundle.pem:/home/agent/data/config/uug-camera-trust-bundle.pem:ro \
+  -e AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE=/home/agent/data/config/uug-camera-trust-bundle.pem \
+  -e AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE=false \
+  -e 'AGENT_CAPTURE_IPCAMERA_RTSP=rtsps://<user>:<password>@10.0.30.11:9554/?inst=1' \
+  -e 'AGENT_CAPTURE_IPCAMERA_SUB_RTSP=rtsps://<user>:<password>@10.0.30.11:9554/?inst=2' \
+  kerberos/agent:latest
+```
+
+`AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE` starts with the operating system's roots
+and appends the camera bundle only to the gortsplib TLS configuration. Other
+clients, including MoQ, Hub, and Vault, retain the normal public CA chain.
+Restart the Agent after changing trust files.
+
+Do not set `SSL_CERT_FILE` or `SSL_CERT_DIR` in production solely for camera
+trust. They are process-wide and can prevent other clients from validating
+public services. For a deliberate process-wide isolation test, mount an empty
+directory and set `SSL_CERT_DIR` to its path:
+
+```bash
+-v /secure/config/empty-ca-dir:/home/agent/data/config/empty-ca-dir:ro \
+-e SSL_CERT_DIR=/home/agent/data/config/empty-ca-dir
+```
+
+Do not use `SSL_CERT_DIR=`. Go treats an empty value as unset and scans its
+default certificate directories.
+
+Only use that mode when the Agent does not need public roots for other TLS
+connections.
+
+## Validate the live endpoints
+
+### Strict TLS and identity check
+
+Use only the specified bundle, without OpenSSL's default CA locations:
+
+```bash
+openssl s_client \
+  -brief \
+  -connect 10.0.30.11:9554 \
+  -verify_ip 10.0.30.11 \
+  -verify_return_error \
+  -CAfile "$HOME/.step/camera/uug-camera-trust-bundle.pem" \
+  -no-CApath \
+  -no-CAstore \
+  </dev/null
+```
+
+Repeat with port `443`. Both must report `Verification: OK`.
+
+Confirm that identity checking is active by repeating the command with a wrong
+address, such as `-verify_ip 10.0.30.12`. It must fail with an IP address
+mismatch.
+
+### Confirm the live leaf is the generated leaf
+
+```bash
+live_fingerprint=$(
+  openssl s_client -connect 10.0.30.11:9554 -showcerts </dev/null 2>/dev/null |
+    openssl x509 -noout -fingerprint -sha256 |
+    cut -d= -f2
+)
+
+local_fingerprint=$(
+  openssl x509 \
+    -in "$HOME/.step/camera/bosch-rtsps-chain.pem" \
+    -noout -fingerprint -sha256 |
+    cut -d= -f2
+)
+
+test "$live_fingerprint" = "$local_fingerprint"
+```
+
+### Validate the media path
+
+A successful TLS handshake does not prove that RTSP authentication, DESCRIBE,
+SETUP, PLAY, and RTP delivery work. Start a fresh Agent with verified TLS and
+confirm that it connects without an x509 error and receives frames. During the
+verified setup described here, a gortsplib probe completed all RTSP operations
+and received an RTP packet over TCP.
+
+## Why a tampered bundle may still connect
+
+Editing PEM text is not always a useful negative TLS test.
+
+### A certificate can still parse after a byte change
+
+Base64 can remain syntactically valid when one character changes. OpenSSL may
+still list the certificate subject and issuer even though a signature is now
+invalid. Parsing and signature verification are different operations.
+
+### Trust anchors are not validated through a parent
+
+Every certificate loaded into Go's root pool is a trust anchor, including a
+non-self-signed intermediate CA. Verification can terminate at that certificate.
+
+If tampering changes only the intermediate's signature from its parent root,
+but does not change its public key, that intermediate can still validate the
+camera leaf when it is trusted directly. Its now-invalid parent signature is
+not consulted at the trust boundary.
+
+This is equivalent to OpenSSL's partial-chain behavior:
+
+```bash
+openssl s_client \
+  -connect 10.0.30.11:9554 \
+  -verify_ip 10.0.30.11 \
+  -verify_return_error \
+  -partial_chain \
+  -CAfile tampered-bundle.pem \
+  -no-CApath \
+  -no-CAstore \
+  </dev/null
+```
+
+### `SSL_CERT_FILE` does not isolate Go from CA directories
+
+On Unix, Go uses `SSL_CERT_FILE` instead of its default aggregate CA file, but it
+still scans default certificate directories such as `/etc/ssl/certs`. Setting
+`SSL_CERT_FILE` alone therefore does not remove CA certificates installed with
+`update-ca-certificates`. `AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE` is appended
+after this system pool is loaded; it does not replace the system roots.
+
+Blank values do not select empty trust sources. Both `SSL_CERT_FILE=` and
+`SSL_CERT_DIR=` are treated as unset, so Go falls back to its default aggregate
+CA file and certificate directories. To test with no trusted certificates on
+Linux, use a non-empty file path that contains no certificates and a non-empty
+directory path that contains no certificates:
+
+```bash
+mkdir -p /tmp/empty-ca-dir
+SSL_CERT_FILE=/dev/null \
+SSL_CERT_DIR=/tmp/empty-ca-dir \
+AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE= \
+AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE=false \
+GOWORK=off \
+go run -tags moq . -action run -port 8080
+```
+
+That fresh process must fail with `x509: certificate signed by unknown
+authority`.
+
+Use exactly one camera trust-distribution approach when possible:
+
+1. Mount a private trust bundle and set
+  `AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE`; or
+2. Install the CA certificates into the operating-system trust store.
+
+Using both is valid, but makes isolation tests less obvious.
+
+### Running processes can retain old roots
+
+A long-running Go process may already have loaded and cached the trust pool.
+Always start a new process after changing trust configuration during a negative
+test.
+
+## Perform a meaningful negative test
+
+Do not corrupt only the root or intermediate signature. Instead, give a fresh
+Agent process a completely unrelated CA and hide the default CA directories.
+
+```bash
+mkdir -p /tmp/empty-ca-dir
+
+openssl req \
+  -x509 -newkey rsa:2048 -nodes -days 1 \
+  -subj '/CN=Unrelated Test Root' \
+  -keyout /tmp/unrelated-test-root.key \
+  -out /tmp/unrelated-test-root.crt
+
+SSL_CERT_FILE=/dev/null \
+SSL_CERT_DIR=/tmp/empty-ca-dir \
+AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE=/tmp/unrelated-test-root.crt \
+AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE=false \
+GOWORK=off \
+go run -tags moq . -action run -port 8080
+```
+
+The connection must fail with an unknown-authority or chain-building error.
+Delete the temporary test key and certificate afterward.
+
+To test bundle integrity rather than client distrust, validate the intermediate
+against the root explicitly:
+
+```bash
+openssl verify \
+  -CAfile "$HOME/.step/certs/root_ca.crt" \
+  -no-CApath \
+  -no-CAstore \
+  "$HOME/.step/certs/intermediate_ca.crt"
+```
+
+Store and compare approved SHA-256 fingerprints when detecting unauthorized
+certificate-file changes is a requirement.
+
+Restore an accidentally edited bundle from the protected CA certificates, then
+restart the Agent:
+
+```bash
+step certificate bundle -f \
+  "$HOME/.step/certs/intermediate_ca.crt" \
+  "$HOME/.step/certs/root_ca.crt" \
+  "$HOME/.step/camera/uug-camera-trust-bundle.pem"
+
+openssl verify \
+  -CAfile "$HOME/.step/certs/root_ca.crt" \
+  -no-CApath \
+  -no-CAstore \
+  "$HOME/.step/certs/intermediate_ca.crt"
+```
+
+## Optional system trust installation
+
+On Debian, install both public CA certificates when every process in the system
+should trust this camera PKI:
+
+```bash
+sudo install -m 0644 \
+  "$HOME/.step/certs/root_ca.crt" \
+  /usr/local/share/ca-certificates/uug-camera-ca.crt
+
+sudo install -m 0644 \
+  "$HOME/.step/certs/intermediate_ca.crt" \
+  /usr/local/share/ca-certificates/uug-camera-intermediate-ca.crt
+
+sudo update-ca-certificates
+```
+
+This creates links below `/etc/ssl/certs`. Remove those files and rerun
+`update-ca-certificates` before attempting an isolated trust-bundle test.
+
+## Renewal and recovery
+
+- Renew before the leaf or intermediate expires.
+- Generate a new camera CSR if the firmware cannot renew the existing key.
+- Sign the new CSR with all required SANs.
+- Upload and validate the new certificate before deleting the old one.
+- Preserve an alternate administrative access path while changing HTTPS usage.
+- Back up the CA certificates, encrypted CA keys, and password separately.
+- If the CA private keys are lost, create a new CA and redistribute its trust
+  before replacing camera certificates.
+
+## Production checklist
+
+- [ ] The Agent URL uses `rtsps://`, not `rtsp://` or `srtsp://`.
+- [ ] RTSPS is enabled on the camera and the configured port is reachable.
+- [ ] The certificate SAN exactly matches the Agent URL host.
+- [ ] The leaf public key matches the camera-generated CSR.
+- [ ] The certificate has `serverAuth` extended key usage.
+- [ ] The certificate expires before its issuer.
+- [ ] HTTPS is assigned to the private-CA certificate.
+- [ ] CBS client remains assigned to the Bosch device certificate.
+- [ ] The Agent has the intermediate and root CA certificates it needs.
+- [ ] `AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE=false`.
+- [ ] The Agent was restarted after trust changes.
+- [ ] A strict TLS check reports `Verification: OK`.
+- [ ] A real Agent connection receives RTP packets.
+- [ ] CA private keys and passwords are backed up outside the devcontainer.

README.md

@@ -3,7 +3,6 @@
 <a target="_blank" href="https://kerberos.io"><img src="https://img.shields.io/badge/kerberos-website-gray.svg?longCache=true&colorB=brightgreen" alt="Kerberos Agent"></a>
 <a target="_blank" href="https://doc.kerberos.io"><img src="https://img.shields.io/badge/kerberos-documentation-gray.svg?longCache=true&colorB=brightgreen" alt="Kerberos Agent"></a>
 
-<a target="_blank" href="https://circleci.com/gh/kerberos-io/agent"><img src="https://circleci.com/gh/kerberos-io/agent.svg?style=svg"/></a>
 <img src="https://github.com/kerberos-io/agent/workflows/Go/badge.svg"/>
 <img src="https://github.com/kerberos-io/agent/workflows/React/badge.svg"/>
 <img src="https://github.com/kerberos-io/agent/workflows/CodeQL/badge.svg"/>
@@ -77,16 +76,109 @@ There are a myriad of cameras out there (USB, IP and other cameras), and it migh
 1. [What is new?](#what-is-new)
 1. [Contributors](#contributors)
 
+### Direct S3 recording layout
+
+Direct S3 uploads store recordings as
+`<s3.username>/yyyy/mm/dd/<timestamp>-<deviceKey>.mp4`. The timestamp is the
+recording's Unix timestamp in milliseconds, and the date is its UTC date, not the upload
+date. Timestamp and device key come from the recording's `.metadata` upload
+checkpoint, which stores the timestamp in milliseconds. The uploader also
+preserves checkpoint duration, FPS (when available), and encryption status as S3
+object metadata. The device key must be non-empty and contain no path separators.
+Retried uploads use the same object key, even after midnight. Millisecond
+precision keeps recordings starting within the same second distinct.
+
+This replaces the previous `<s3.username>/<filename>.mp4` layout for new uploads.
+Existing objects are not moved; consumers of direct S3 uploads must support both
+layouts when reading historical recordings. Updated Kerberos Vault deployments
+use the same filename and dated layout under the authenticated account directory
+for both direct Vault uploads and Hub-proxied uploads.
+
+Only historical recordings with missing, empty, or FPS-only queue markers fall
+back to the legacy filename format and configured Agent key. A structured
+checkpoint takes precedence over both the filename and current configured key;
+an invalid or unreadable checkpoint is reported as an upload error, not silently
+replaced with filename-derived values.
+For older structured checkpoints, missing camera name, region, and change count
+are recovered from a validated legacy filename. Populated checkpoint fields
+take precedence, and the checkpoint timestamp and device key remain authoritative.
+
+The shorter filename cannot be decoded by legacy filename-only consumers.
+Integrations must preserve structured recording metadata, including the device
+key (`productid`) and recording timestamp (`event-timestamp`), as well as the full
+object key.
+
+### Local recording layout
+
+New local recordings are stored under
+`data/recordings/yyyy/mm/dd/<timestamp>-<deviceKey>.mp4`, using the recording's UTC
+start date and Unix-millisecond timestamp. For example:
+`data/recordings/2026/01/02/1767355200123-device-key.mp4`.
+The filename no longer embeds the camera name, region, change count, or duration,
+and it does not change when the recording is finalized.
+Those attributes, together with FPS and encryption status, are captured in the
+`.metadata` upload checkpoint and sent as structured HTTP/tus metadata.
+
+Existing flat recordings and dated recordings with older long filenames remain
+supported without a migration. Recording lists, day filters, playback, uploaders,
+and automatic cleanup handle both filename formats and layouts. Upload queue
+checkpoints remain flat in `data/cloud/`. Successful-upload removal also resolves
+both local layouts. Deploy the updated Vault alongside Agent for canonical
+`<account-directory>/yyyy/mm/dd/<timestamp>-<deviceKey>.mp4` storage keys and
+metadata-driven Hub integration events.
+Directory decryption also processes dated subdirectories and writes each output
+to a sibling `decrypted/` directory; those outputs are excluded from recording
+lists and automatic cleanup.
+
 ## Quickstart - Docker
 
 The easiest way to get your Kerberos Agent up and running is to use our public image on [Docker hub](https://hub.docker.com/r/kerberos/agent). Once you have selected a specific tag, run `docker` command below, which will open the web interface of your Kerberos agent on port `80`, and off you go. For a more configurable and persistent deployment have a look at [Running and automating a Kerberos Agent](#running-and-automating-a-kerberos-agent).
 
-    docker run -p 80:80 --name mycamera -d --restart=always kerberos/agent:latest
+**Forthcoming hardened image** (use this mapping only after upgrading to a
+release containing these changes):
+
+    docker run -p 80:8080 --name mycamera -d --restart=always kerberos/agent:latest
+
+The hardened image listens on **8080 inside the container**, not 80. The mapping
+`-p 80:8080` keeps the browser URL `http://localhost/` unchanged. When upgrading,
+change the container side of existing `80:80` or `8082:80` mappings to
+`80:8080` or `8082:8080`. Kubernetes Services must use `targetPort: 8080`.
+Older published images still use their original port: these examples require a
+release containing the hardening changes; pin its published digest in production.
+For the currently published `v3.14.1`, retain the old mapping:
+
+    docker run -p 80:80 --name mycamera -d --restart=always kerberos/agent:v3.14.1
+
+The image uses a Go 1.26 scratch runtime with only required dynamic/media
+dependencies, numeric non-root UID/GID `999:999`, no file capabilities or
+setuid/setgid programs, and an application healthcheck (no curl-based probe).
+UI assets are immutable; demo mode selects its environment without rewriting them.
+Source builds require the patched Go 1.26.10 toolchain or newer.
+It supports dropping all capabilities and preventing privilege escalation:
+
+```sh
+docker run --cap-drop=ALL --security-opt=no-new-privileges:true \
+  -p 80:8080 --name mycamera -d kerberos/agent:latest
+```
+
+See the [image contract](deployments/docker/README.md#image-contract) for writable
+paths, health checks, supported architectures, signature/evidence requirements,
+and release-policy gaps. Functional remote-shell support remains a **pending
+policy exception**, so this is not a claim of full image-hardening baseline or
+CIS compliance.
+Release promotion is also blocked by remaining native-library scan findings;
+this PR is not release-qualified. The existing recording-signing key and trust
+behavior are retained under the user's narrowly scoped exception through
+December 5, 2026 (23:59:59 UTC), recorded for review in this PR. The finding
+remains visible in scan reports; other keys and findings are not exempted.
 
 If you want to connect to a USB or Raspberry Pi camera, [you'll need to run our side car container](https://github.com/kerberos-io/camera-to-rtsp) which proxies the camera to an RTSP stream. In that case you'll want to configure the Kerberos Agent container to run in the host network, so it can connect directly to the RTSP sidecar.
 
     docker run --network=host --name mycamera -d --restart=always kerberos/agent:latest
 
+With host networking, no port mapping applies: the hardened Agent is reachable
+on host port `8080` by default.
+
 ## Quickstart - Balena
 
 Run Kerberos Agent with [Balena Cloud](https://www.balena.io/) super powers. Monitor your Kerberos Agent with seamless remote access, over the air updates, an encrypted public `https` endpoint and much more. Checkout our application `video-surveillance` on [Balena Hub](https://hub.balena.io/apps/2064752/video-surveillance), and create your first or fleet of Kerberos Agent(s).
@@ -115,7 +207,7 @@ This repository contains everything you'll need to know about our core product,
 
 - Low memory and CPU usage.
 - Simplified and modern user interface.
-- Multi architecture (ARMv6, ARMv7, ARM64, AMD64)
+- Multi architecture (AMD64/ARM64 release images; [legacy ARM build limitations](deployments/docker/README.md#building-and-checking-the-image)).
 - Multi stream, for example recording in H265, live streaming and motion detection in H264.
 - Multi camera support: IP Cameras (H264 and H265), USB cameras and Raspberry Pi Cameras [through a RTSP proxy](https://github.com/kerberos-io/camera-to-rtsp).
 - Single camera per instance (e.g. one container per camera).
@@ -168,31 +260,149 @@ An example of how to mount a host directory is shown below using `docker`, but i
 
 You attach a volume to your container by leveraging the `-v` option. To mount your own configuration file and recordings folder, run the following commands:
 
-    docker run -p 80:80 --name mycamera \
+    docker run -p 80:8080 --name mycamera \
     -v $(pwd)/agent/config:/home/agent/data/config \
     -v $(pwd)/agent/recordings:/home/agent/data/recordings \
     -d --restart=always kerberos/agent:latest
 
 More examples for each deployment and automation tool [can be found in the deployment section](https://github.com/kerberos-io/agent/tree/master/deployments). Be sure to verify the permissions of the directory/volume you are attaching. More information in [this issue](https://github.com/kerberos-io/agent/issues/80).
 
-    chmod -R 755 kerberos-agent/
-    chown 100:101 kerberos-agent/ -R
+The current image runs as UID `999`, GID `999`. Give that user ownership of
+the mounted directories (older image tags may use different IDs):
+
+    sudo chown -R 999:999 agent/config agent/recordings
+
+See [container permissions and hardening](deployments/docker/README.md#container-permissions-and-hardening)
+for reduced-capability deployment and migration guidance.
 
 ## Configure with environment variables
 
 Next to attaching the configuration file, it is also possible to override the configuration with environment variables. This makes deploying with `docker compose` or `kubernetes` much easier and more scalable. Using this approach, we simplify automation through `ansible` and `terraform`.
 
-    docker run -p 80:80 --name mycamera \
+    docker run -p 80:8080 --name mycamera \
     -e AGENT_NAME=mycamera \
     -e AGENT_TIMEZONE=Europe/Brussels \
     -e AGENT_CAPTURE_IPCAMERA_RTSP=rtsp://fake.kerberos.io/stream \
     -e AGENT_CAPTURE_CONTINUOUS=true \
     -d --restart=always kerberos/agent:latest
 
+Set `AGENT_PORT` to change the Agent's internal HTTP port. The environment
+variable takes precedence over `-port`. When using Docker, publish the same
+container port:
+
+```bash
+docker run -p 8082:8082 \
+  -e AGENT_PORT=8082 \
+  --name mycamera \
+  -d --restart=always kerberos/agent:latest
+```
+
+### Health checks and API responses
+
+`GET /health` is an unauthenticated liveness and diagnostics endpoint for
+container orchestrators and external monitors. `healthy` means that the Agent
+HTTP process can serve requests. Camera streams and Hub connectivity are
+reported independently in the response, so an external camera or Hub outage
+does not make Docker restart an otherwise functioning Agent process. Successful
+probes return `200 OK` and are logged at debug level to avoid periodic health
+checks filling production logs.
+
+```bash
+curl http://localhost:8082/health
+```
+
+```json
+{
+  "httpStatusCode": 200,
+  "applicationStatusCode": "get_success",
+  "entityStatusCode": "healthy",
+  "message": "Healthy",
+  "metadata": {
+    "applicationName": "agent",
+    "applicationVersion": "0.0.0",
+    "timestamp": 1788710400,
+    "path": "/health"
+  },
+  "data": {
+    "health": {
+      "description": "Agent HTTP service is healthy",
+      "cameraConnected": true,
+      "mainStream": {
+        "configured": true,
+        "connected": true,
+        "packagesProcessed": 183421,
+        "fps": 29.97,
+        "resolution": {
+          "width": 1920,
+          "height": 1080
+        },
+        "lastPacketAt": 1788710399
+      },
+      "subStream": {
+        "configured": true,
+        "connected": true,
+        "packagesProcessed": 91710,
+        "fps": 15,
+        "resolution": {
+          "width": 640,
+          "height": 360
+        },
+        "lastPacketAt": 1788710399
+      },
+      "hub": {
+        "configured": true,
+        "connected": true,
+        "lastHeartbeatAttemptAt": 1788710398,
+        "lastSuccessfulHeartbeatAt": 1788710398
+      }
+    }
+  }
+}
+```
+
+Stream `packagesProcessed` counts complete encoded video access units processed
+since the Agent process started; it does not count individual fragmented RTP
+packets. `fps` is the latest PTS-derived frame-rate estimate. Resolution is the
+most recently observed encoded width and height. `lastPacketAt`, Hub heartbeat
+timestamps, and response metadata timestamps are Unix seconds; `0` means that
+no value has been observed yet. A stream can be configured but temporarily
+disconnected.
+
+Hub `configured` means heartbeat delivery is enabled and has the required URI
+and key. Hub `connected` means the most recent heartbeat succeeded and the last
+success is no more than three minutes old. A failed heartbeat marks it
+disconnected immediately while retaining the last-success timestamp for
+diagnosis.
+
+New JSON endpoints should use this Hub-compatible response envelope rather than
+adding unrelated top-level response shapes. HTTP status describes the transport
+result, `applicationStatusCode` describes the operation, `entityStatusCode` and
+`message` describe the domain outcome, `metadata` carries request/application
+context, and endpoint-specific content belongs in a typed `data` object. Client
+responses must contain safe messages only; detailed internal errors belong in
+structured logs. Existing endpoints retain their legacy response shapes until
+they are migrated deliberately.
+
+### Secure camera streams (RTSPS)
+
+The Agent accepts `rtsps://` camera URLs. Do not use `srtsp://`; RTSPS is RTSP over TLS. For a Bosch FLEXIDOME micro 3100i, enable **Secure RTSP** under **Network > Network Services** and use port `9554`:
+
+```bash
+AGENT_CAPTURE_IPCAMERA_RTSP='rtsps://username:password@camera.example:9554/?inst=1'
+AGENT_CAPTURE_IPCAMERA_SUB_RTSP='rtsps://username:password@camera.example:9554/?inst=2'
+```
+
+Certificate verification is enabled by default. The URL hostname or IP address must match the camera certificate SAN. On this Bosch firmware, RTSPS presents the certificate assigned to **HTTPS**; there is no separate SRTSP certificate usage. Leave **CBS client** assigned to the Bosch device certificate.
+
+For a private CA, mount a PEM trust bundle containing every CA certificate needed to build the camera certificate chain and set `AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE` to its path inside the Agent. The bundle is appended to the system roots for camera RTSPS connections only. This Bosch firmware presents only its leaf certificate, so include both the issuing intermediate and root certificates in the bundle. As a temporary fallback, `AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE=true` disables certificate verification for camera streams only.
+
+See [RTSPS and TLS certificates](README-RTSPS-TLS.md) for the complete Bosch UI, private-CA, deployment, validation, and troubleshooting procedure.
+
 | Name                                    | Description                                                                                     | Default Value                  |
 | --------------------------------------- | ----------------------------------------------------------------------------------------------- | ------------------------------ |
 | `LOG_LEVEL`                                 | Level for logging, could be "info", "warning", "debug", "error" or "fatal".                     | "info"                         |
 | `LOG_OUTPUT`                                | Logging output format "json" or "text".                                                         | "text"                         |
+| `AGENT_PORT`                                | HTTP web server port. Overrides `-port`; use a port above 1024 in the hardened image. | "8080" hardened Docker; "80" legacy images/binaries |
 | `AGENT_MODE`                                | You can choose to run this in 'release' for production, and or 'demo' for showcasing.           | "release"                      |
 | `AGENT_TLS_INSECURE`                        | Specify if you want to use `InsecureSkipVerify` for the internal HTTP client.                   | "false"                        |
 | `AGENT_USERNAME`                            | The username used to authenticate against the Kerberos Agent login page.                        | "root"                         |
@@ -203,18 +413,22 @@ Next to attaching the configuration file, it is also possible to override the co
 | `AGENT_REMOVE_AFTER_UPLOAD`                 | When enabled, recordings uploaded successfully to a storage will be removed from disk.          | "true"                         |
 | `AGENT_OFFLINE`                             | Makes sure no external connection is made.                                                      | "false"                        |
 | `AGENT_AUTO_CLEAN`                          | Cleans up the recordings directory.                                                             | "true"                         |
-| `AGENT_AUTO_CLEAN_MAX_SIZE`                 | If `AUTO_CLEAN` enabled, set the max size of the recordings directory (in MB).                  | "100"                          |
+| `AGENT_AUTO_CLEAN_MAX_SIZE`                 | If `AUTO_CLEAN` enabled, cap the recordings directory at this size (in MB). When unset/0, recordings may use the whole disk instead (see `AGENT_AUTO_CLEAN_MIN_FREE_SPACE`). | "100"                          |
+| `AGENT_AUTO_CLEAN_MIN_FREE_SPACE`           | When `AUTO_CLEAN` is enabled and no `MAX_SIZE` is set, keep at least this much free space (in MB) on the recordings disk before deleting the oldest (already-uploaded first) recordings. Defaults to 5% of the disk. | ""                             |
 | `AGENT_TIME`                                | Enable the timetable for Kerberos Agent                                                         | "false"                        |
 | `AGENT_TIMETABLE`                           | A (weekly) time table to specify when to make recordings "start1,end1,start2,end2;start1..      | ""                             |
 | `AGENT_REGION_POLYGON`                      | A single polygon set for motion detection: "x1,y1;x2,y2;x3,y3;...                               | ""                             |
-| `AGENT_CAPTURE_IPCAMERA_RTSP`               | Full-HD RTSP endpoint to the camera you're targetting.                                          | ""                             |
-| `AGENT_CAPTURE_IPCAMERA_SUB_RTSP`           | Sub-stream RTSP endpoint used for livestreaming (WebRTC).                                       | ""                             |
+| `AGENT_CAPTURE_IPCAMERA_RTSP`               | Full-HD RTSP or RTSPS endpoint for the target camera.                                           | ""                             |
+| `AGENT_CAPTURE_IPCAMERA_SUB_RTSP`           | RTSP or RTSPS sub-stream endpoint used for livestreaming (WebRTC).                              | ""                             |
+| `AGENT_CAPTURE_IPCAMERA_RTSPS_CA_FILE`      | PEM CA bundle appended to the system roots for RTSPS camera certificate verification.           | ""                             |
+| `AGENT_CAPTURE_IPCAMERA_RTSPS_INSECURE`     | Disable RTSPS camera certificate verification; use only when a trusted CA cannot be installed.  | "false"                        |
 | `AGENT_CAPTURE_IPCAMERA_BASE_WIDTH`         | Force a specific width resolution for live view processing.                                     | ""                             |
 | `AGENT_CAPTURE_IPCAMERA_BASE_HEIGHT`        | Force a specific height resolution for live view processing.                                    | ""                             |
 | `AGENT_CAPTURE_IPCAMERA_ONVIF`              | Mark as a compliant ONVIF device.                                                               | ""                             |
 | `AGENT_CAPTURE_IPCAMERA_ONVIF_XADDR`        | ONVIF endpoint/address running on the camera.                                                   | ""                             |
 | `AGENT_CAPTURE_IPCAMERA_ONVIF_USERNAME`     | ONVIF username to authenticate against.                                                         | ""                             |
 | `AGENT_CAPTURE_IPCAMERA_ONVIF_PASSWORD`     | ONVIF password to authenticate against.                                                         | ""                             |
+| `AGENT_CAPTURE_IPCAMERA_ONVIF_CHANNEL`      | 1-based channel on multi-channel ONVIF devices (encoders, NVRs) used for PTZ, presets and focus. | "0" (first profile)            |
 | `AGENT_CAPTURE_MOTION`                      | Toggle for enabling or disabling motion.                                                        | "true"                         |
 | `AGENT_CAPTURE_LIVEVIEW`                    | Toggle for enabling or disabling liveview.                                                      | "true"                         |
 | `AGENT_CAPTURE_SNAPSHOTS`                   | Toggle for enabling or disabling snapshot generation.                                           | "true"                         |
@@ -226,9 +440,10 @@ Next to attaching the configuration file, it is also possible to override the co
 | `AGENT_CAPTURE_PIXEL_CHANGE`                | If `CONTINUOUS` set to `false`, the number of pixel require to change before motion triggers.   | "150"                          |
 | `AGENT_CAPTURE_FRAGMENTED`                  | Set the format of the recorded MP4 to fragmented (suitable for HLS).                            | "false"                        |
 | `AGENT_CAPTURE_FRAGMENTED_DURATION`         | If `AGENT_CAPTURE_FRAGMENTED` set to `true`, define the duration (seconds) of a fragment.       | "8"                            |
-| `AGENT_MQTT_URI`                            | An MQTT broker endpoint that is used for bi-directional communication (live view, onvif, etc)   | "tcp://mqtt.kerberos.io:1883"  |
+| `AGENT_MQTT_URI`                            | MQTT broker endpoint for bi-directional communication. Accepts ActiveMQ `mqtt+ssl://` URLs.     | "tcp://mqtt.kerberos.io:1883"  |
 | `AGENT_MQTT_USERNAME`                       | Username of the MQTT broker.                                                                    | ""                             |
 | `AGENT_MQTT_PASSWORD`                       | Password of the MQTT broker.                                                                    | ""                             |
+| `AGENT_REMOTE_ACCESS_ENABLED`               | Allow encrypted Hub MQTT sessions to stream Agent logs and open an interactive shell. Enable only for trusted deployments. | "false"                        |
 | `AGENT_REALTIME_PROCESSING`                 | If `AGENT_REALTIME_PROCESSING` set to `true`, the agent will send key frames to the topic       | ""                             |
 | `AGENT_REALTIME_PROCESSING_TOPIC`           | The topic to which keyframes will be sent in base64 encoded format.                             | ""                             |
 | `AGENT_STUN_URI`                            | When using WebRTC, you'll need to provide a STUN server.                                        | "stun:turn-fra1.kerberos.io:3478"|
@@ -243,16 +458,19 @@ Next to attaching the configuration file, it is also possible to override the co
 | `AGENT_HUB_PRIVATE_KEY`                     | The secret access key linked to your account in Kerberos Hub.                                   | ""                             |
 | `AGENT_HUB_REGION`                          | The Kerberos Hub region, to which you want to upload.                                           | ""                             |
 | `AGENT_HUB_SITE`                            | The site ID of a site you've created in your Kerberos Hub account.                              | ""                             |
+| `AGENT_TUS_CHUNK_SIZE_BYTES`                | Bytes sent in each resumable-upload PATCH. Set to `0` or a negative value to send all remaining bytes in one PATCH. | "8388608" (8 MiB)              |
 | `AGENT_KERBEROSVAULT_URI`                   | The Kerberos Vault API url.                                                                     | "https://vault.domain.com/api" |
 | `AGENT_KERBEROSVAULT_ACCESS_KEY`            | The access key of a Kerberos Vault account.                                                     | ""                             |
 | `AGENT_KERBEROSVAULT_SECRET_KEY`            | The secret key of a Kerberos Vault account.                                                     | ""                             |
 | `AGENT_KERBEROSVAULT_PROVIDER`              | A Kerberos Vault provider you have created (optional).                                          | ""                             |
 | `AGENT_KERBEROSVAULT_DIRECTORY`             | The directory, in the Kerberos vault, where the recordings will be stored.                      | ""                             |
+| `AGENT_KERBEROSVAULT_CUSTOM_HEADERS`        | JSON object of custom upload headers that Vault includes as integration metadata.               | `{"site_id":"site-1"}`         |
 | `AGENT_KERBEROSVAULT_SECONDARY_URI`         | The Kerberos Vault API url.                                                                     | "https://vault.domain.com/api" |
 | `AGENT_KERBEROSVAULT_SECONDARY_ACCESS_KEY`  | The access key of a secondary Kerberos Vault account.                                           | ""                             |
 | `AGENT_KERBEROSVAULT_SECONDARY_SECRET_KEY`  | The secret key of a secondary Kerberos Vault account.                                           | ""                             |
 | `AGENT_KERBEROSVAULT_SECONDARY_PROVIDER`    | A secondary Kerberos Vault provider you have created (optional).                                | ""                             |
 | `AGENT_KERBEROSVAULT_SECONDARY_DIRECTORY`   | The directory, in the secondary Kerberos vault, where the recordings will be stored.            | ""                             |
+| `AGENT_KERBEROSVAULT_SECONDARY_CUSTOM_HEADERS` | JSON object of custom upload headers for the secondary Vault.                                | `{"site_id":"site-1"}`         |
 | `AGENT_DROPBOX_ACCESS_TOKEN`                | The Access Token from your Dropbox app, that is used to leverage the Dropbox SDK.               | ""                             |
 | `AGENT_DROPBOX_DIRECTORY`                   | The directory, in Dropbox, where the recordings will be stored.                                 | ""                             |
 | `AGENT_ENCRYPTION`                          | Enable 'true' or disable 'false' end-to-end encryption for MQTT messages.                       | "false"                        |
@@ -263,6 +481,133 @@ Next to attaching the configuration file, it is also possible to override the co
 | `AGENT_SIGNING`                             | Enable 'true' or disable 'false' for signing recordings.                                        | "true"                         |
 | `AGENT_SIGNING_PRIVATE_KEY`                 | The private key (RSA) to sign the recordings fingerprint to validate origin.                    | "" - uses default one if empty |
 
+Remote console access is disabled unless `AGENT_REMOTE_ACCESS_ENABLED=true`.
+The Agent also rejects remote session messages unless Hub encryption or
+end-to-end MQTT encryption is configured and used. A remote shell runs inside
+the Agent process environment as the Agent operating-system user; it is not an
+SSH server and does not expose a new network port. Keep the feature disabled on
+deployments where Hub owners should not have operating-system access.
+
+### Resumable upload chunk size
+
+Hub and Vault resumable uploads use `AGENT_TUS_CHUNK_SIZE_BYTES` as the maximum
+body size of each tus `PATCH` request. The value is a number of bytes, not a
+number of chunks. If the variable is unset or invalid, the Agent uses 8 MiB:
+
+```dotenv
+# 8 MiB (default)
+AGENT_TUS_CHUNK_SIZE_BYTES=8388608
+
+# 4 MiB: more frequent checkpoints on unstable connections
+AGENT_TUS_CHUNK_SIZE_BYTES=4194304
+
+# Disable chunking and send all remaining bytes in one PATCH
+AGENT_TUS_CHUNK_SIZE_BYTES=0
+```
+
+Smaller chunks provide more frequent resumable checkpoints but create more HTTP
+requests. Larger chunks reduce request overhead but require more data to be
+retransmitted when a request fails.
+
+An HTTP 500 during resumable creation is a server/provider failure, not proof
+that the recording filename is invalid. Creation errors include up to 4 KiB of
+the server response, and the final retry error retains the latest failure cause.
+Created upload sessions remain usable if only the optional creation response
+body is interrupted; the successful response's `Location` is retained.
+Check the corresponding Hub/Vault logs and storage-provider configuration.
+As a temporary diagnostic workaround, set `AGENT_DISABLE_RESUMABLE_UPLOAD=true`
+and restart Agent to use ordinary uploads. This does not change the recording
+filename or remove its structured metadata, but interrupted uploads restart
+from the beginning. Remove the override after fixing the provider issue.
+
+### Custom metadata headers for Kerberos Vault
+
+Use `AGENT_KERBEROSVAULT_CUSTOM_HEADERS` to attach your own metadata (for
+example a site, line, or equipment identifier) to every recording uploaded to
+Kerberos Vault. The value is a JSON object whose keys are header names and whose
+values are strings. The Agent sends these as HTTP headers on every upload, and
+Vault adds them to the `metadata` object of its integration messages (Kafka,
+MQTT, SQS, RabbitMQ, etc.):
+
+```json
+{
+  "timestamp": "2026-09-24T15:06:45Z",
+  "key": "agent1/1790262405_3-425_agent1_0-0-0-0_-1_10000.mp4",
+  "device": "agent1",
+  "metadata": {
+    "site_id": "site-1",
+    "line_id": "line-2",
+    "equipment_id": "equipment-3"
+  }
+}
+```
+
+The value must be valid JSON on a single line, so wrap it in single quotes in a
+shell:
+
+```bash
+docker run -p 80:8080 --name mycamera \
+  -e AGENT_CLOUD=kstorage \
+  -e AGENT_KERBEROSVAULT_URI=https://vault.domain.com/api \
+  -e AGENT_KERBEROSVAULT_ACCESS_KEY=xxx \
+  -e AGENT_KERBEROSVAULT_SECRET_KEY=xxx \
+  -e AGENT_KERBEROSVAULT_DIRECTORY=agent1 \
+  -e AGENT_KERBEROSVAULT_CUSTOM_HEADERS='{"site_id":"site-1","line_id":"line-2","equipment_id":"equipment-3"}' \
+  -d --restart=always kerberos/agent:latest
+```
+
+In `docker compose` or an `.env` file:
+
+```yaml
+services:
+  agent:
+    image: kerberos/agent:latest
+    environment:
+      AGENT_CLOUD: kstorage
+      AGENT_KERBEROSVAULT_CUSTOM_HEADERS: '{"site_id":"site-1","line_id":"line-2","equipment_id":"equipment-3"}'
+```
+
+```dotenv
+AGENT_KERBEROSVAULT_CUSTOM_HEADERS={"site_id":"site-1","line_id":"line-2","equipment_id":"equipment-3"}
+```
+
+In Kubernetes, quote the value so YAML keeps it as a string:
+
+```yaml
+env:
+  - name: AGENT_KERBEROSVAULT_CUSTOM_HEADERS
+    value: '{"site_id":"site-1","line_id":"line-2","equipment_id":"equipment-3"}'
+```
+
+Use `AGENT_KERBEROSVAULT_SECONDARY_CUSTOM_HEADERS` for the secondary Vault. In
+ConfigMap deployments you can set shared defaults with the `GLOBAL_` prefix
+(for example `GLOBAL_AGENT_KERBEROSVAULT_CUSTOM_HEADERS`) and override them per
+agent with `AGENT_KERBEROSVAULT_CUSTOM_HEADERS`. You can also set the same JSON
+under **Settings → Persistence → Custom upload headers** in the Agent UI, which
+stores it as `kstorage.custom_headers` in `config.json`.
+
+The rules for this value are:
+
+- It must be a JSON object with string values. Numbers, `null`, arrays, and
+  nested objects are rejected.
+- It can contain up to 32 headers, with names up to 128 characters and values
+  up to 4096 bytes.
+- Header names must be valid HTTP tokens (letters, digits, and ``!#$%&'*+-.^_`|~``).
+  Keys are matched case-insensitively, so `site_id` and `SITE_ID` cannot both
+  be used.
+- Reserved names are rejected: anything starting with `X-Kerberos-`,
+  `Authorization`, `Cookie`, `Host`, `Content-Type`, `Content-Length`,
+  `Trace-Id`, tus protocol headers (`Tus-Resumable`, `Upload-*`), and
+  hop-by-hop headers.
+- Values cannot contain line breaks, tabs, or other control characters.
+
+If the value is invalid, the Agent logs an error and does not upload to that
+Vault until the configuration is fixed. The recording stays queued, so it will
+upload once the value is corrected. Header names reach Vault unchanged, but
+proxies may normalise their casing. If Vault redirects an upload to another
+host, the Agent removes the custom headers and Vault credentials before
+following the redirect.
+
 
 ## Encryption
 
@@ -342,7 +687,7 @@ Go and open two terminals: one for the `ui` project and one for the `machinery`
 1.  Terminal A:
 
         cd machinery/
-        go run main.go -action run -port 80
+        go run . -action run -port 80
 
 2.  Terminal B:
 
@@ -367,7 +712,10 @@ The `web` is a **React** project which is the main entry point for an end user t
     cd ui
     yarn start
 
-This will start a webserver and launches the web app on port `3000`.
+This starts the web app on port `3000`. In development, the UI connects to the
+Agent API on port `8082`. Override that port when needed:
+
+    REACT_APP_AGENT_API_PORT=9090 yarn start
 
 ![login-agent](./assets/img/agent-login.gif)
 
@@ -383,9 +731,14 @@ You can simply run the `machinery` using following commands.
 
     git clone https://github.com/kerberos-io/agent
     cd machinery
-    go run main.go -action run -port 80
+    go run . -action run -port 8082
 
-This will launch the Kerberos Agent and run a webserver on port `80`. You can change the port by your own preference. We strongly support the usage of [Goland](https://www.jetbrains.com/go/) or [Visual Studio Code](https://code.visualstudio.com/), as it comes with all the debugging and linting features built in.
+This will launch the Kerberos Agent and run a webserver on port `8082`, matching
+the UI development default. You can change the port together with
+`REACT_APP_AGENT_API_PORT`. We strongly support the usage of
+[Goland](https://www.jetbrains.com/go/) or
+[Visual Studio Code](https://code.visualstudio.com/), as it comes with all the
+debugging and linting features built in.
 
 ![VSCode desktop](./assets/img/vscode-desktop.png)
 
@@ -411,13 +764,69 @@ Remember the build step of the `web` part, during build time we move the build d
 
 ## Building for Docker
 
-Inside the root of this `agent` repository, you will find a `Dockerfile`. This file contains the instructions for building and shipping a **Kerberos Agent**. Important to note is that you start from a prebuilt base image, `kerberos/base:xxx`.
-This base image already contains a couple of tools, such as Golang, FFmpeg and OpenCV. We do this for faster compilation times.
+Inside the root of this `agent` repository, you will find a `Dockerfile`. This file contains the instructions for building and shipping a **Kerberos Agent**. It uses Debian Trixie to support the native dependencies used by the Agent, including Media over QUIC.
 
 By running the `docker build` command, you will create the Kerberos Agent Docker image. After building you can simply run the image as a Docker container.
 
     docker build -t kerberos/agent .
 
+### Media over QUIC
+
+The standard AMD64 and ARM64 images include the optional MoQ publisher. Its Rust
+FFI archive requires CGO and glibc 2.38 or newer, which is why the standard image
+uses Debian Trixie. The publisher is disabled unless explicitly enabled at runtime:
+
+    docker run --rm -p 80:8080 \
+      -e AGENT_LIVE_MOQ_ENABLED=true \
+      -e AGENT_LIVE_MOQ_URL=https://relay.uug.ai/anon \
+      kerberos/agent
+
+`AGENT_LIVE_MOQ_BROADCAST_PREFIX` defaults to `devices`. MoQ viewers subscribe to
+a relay and never negotiate with the Agent, so every quality tier is published as
+its own broadcast and switching quality is simply a resubscribe:
+
+| Tier   | Broadcast                             | Source                                     |
+| ------ | ------------------------------------- | ------------------------------------------ |
+| `high` | `devices/<agent-key>/live.hang`       | highest-resolution camera stream           |
+| `low`  | `devices/<agent-key>/live-low.hang`   | sub stream (main stream when none is set)  |
+
+Each tier uploads every frame while it has subscribers. While idle, it continues
+to upload fresh keyframes only. This keeps the relay's latest cached GOP current,
+so a new viewer does not initially render the final frame from the previous
+viewer session, while avoiding the bandwidth cost of continuously sending every
+delta frame. `AGENT_LIVE_MOQ_QUALITY` accepts `high` or `low` to pin the Agent to
+a single tier; viewers requesting the other tier then find no broadcast. Any
+other value (including the default) publishes both. The initial implementation
+publishes H.264 video only.
+
+Two bounded duration settings tune recovery for unusual network conditions:
+
+| Variable | Default | Allowed range |
+| -------- | ------- | ------------- |
+| `AGENT_LIVE_MOQ_MAX_PACKET_AGE` | `1.5s` | `250ms` to `30s` |
+| `AGENT_LIVE_MOQ_WRITE_TIMEOUT` | `5s` | `1s` to `1m` |
+
+Values use Go duration syntax and are clamped to the documented range. The
+dashboard API's `recovery` object reports per-tier reconnects, last successful
+frame/write timing, native write timeouts, watchdog restarts/cooldown, dropped
+run-channel events, and run-worker shutdown timeouts.
+
+The `/anon` relay route is intended for interoperability testing. Production
+deployments must set `AGENT_LIVE_MOQ_URL` to a short-lived, device-scoped
+publisher URL issued by Hub API.
+
+To verify the native SDK in a development container, rebuild the Agent or shared
+monorepo devcontainer so it uses the Trixie base, then run the VS Code task
+`agent: moq verify`. The same check is available from a terminal:
+
+    cd machinery
+    bash ./verify-moq-devcontainer.sh
+
+The check requires glibc 2.38 or newer, runs the tagged package and in-process
+native QUIC lifecycle tests, links the complete Agent with `-tags moq`, and
+executes the resulting binary's version
+command. Both devcontainers also run this check during their post-create setup.
+
 ## What is new?
 
 This repository contains the next generation of Kerberos.io, **Kerberos Agent (v3)**, and is the successor of the machinery and web repositories. A switch in technologies and architecture has been made. This version is still under active development and can be followed on the [develop branch](https://github.com/kerberos-io/agent/tree/develop) and [project overview](https://github.com/kerberos-io/agent/projects/1).

deployments/ansible-openshift/README.md

@@ -2,6 +2,17 @@
 
 Kubernetes is great, but you might love OpenShift even more. In this directory you'll find some resources to deploy your Kerberos Agent in an OpenShift cluster using Ansible playbook. We'll review the different tasks of the Ansible playbook step by step; find the complete `playbook.yaml` here.
 
+The playbook's port mapping targets the forthcoming hardened image:
+`containerPort: 8080` and `targetPort: 8080`, with external Service port `80`
+unchanged. Previously published images, including `v3.14.1`, still need the old
+container port 80. Upgrade the image and mapping together and pin its published
+digest. See the [image contract](../docker/README.md#image-contract) for numeric
+UID/GID `999:999`, writable data/cache paths and release blockers. OpenShift SCCs
+may override the UID; provision matching volume permissions and verify your
+cluster's policy rather than granting privileged access. This legacy playbook's
+credential-debugging/certificate-validation settings also require production
+review; the image/port update does not certify the cluster deployment.
+
 ## Variabeles
 
 We'll have a few `variables` in our `playbook.yml` that will help us to setup secure connection with the OpenShift cluster. We need the `cluster_url` and the `username` and `password` of the OpenShift cluster. If you don't know where to find this, you can find this in the OpenShift web ui.

deployments/ansible-openshift/playbook.yml

@@ -111,7 +111,7 @@
                         mountPath: /home/agent/data/cloud
                         subPath: cloud
                     ports:
-                      - containerPort: 80
+                      - containerPort: 8080
                         protocol: TCP
 
     - name: Expose Kerberos Agent
@@ -136,5 +136,5 @@
             ports:
               - name: http
                 port: 80
-                targetPort: 80
+                targetPort: 8080
                 protocol: TCP

deployments/docker/README.md

@@ -4,17 +4,258 @@ The easiest, and let's say most natural, deployment is done [by utilising `docke
 
 ## 1. Running a single container
 
-We are creating Docker images as part of our CI/CD process. You'll find our Docker images on [Docker hub](https://hub.docker.com/r/kerberos/agent). Pick a specific tag of choice, or use latest. Once done run below command, this will open the web interface of your Kerberos agent on port 80.
+We publish production images on [Docker Hub](https://hub.docker.com/r/kerberos/agent).
+The following examples target the forthcoming hardened image, which listens on
+**8080 inside the container**. They preserve host port 80 and the browser URL
+`http://localhost/`:
 
-    docker run -p 80:80 --name mycamera -d kerberos/agent:latest
+    docker run -p 80:8080 --name mycamera -d kerberos/agent:latest
 
-Or for a develop build:
+**Migration:** releases published before this change, including `v3.14.1`, still
+listen on container port 80 and require `-p 80:80`. Do not change their mappings
+until upgrading the image too. The separate `kerberos/agent-dev` image is not
+migrated by this production-image change:
 
     docker run -p 80:80 --name mycamera -d kerberos/agent-dev:latest
 
 Feel free to use another port if your host system already has a workload running on `80`. For example `8082`.
 
-    docker run -p 8082:80 --name mycamera -d kerberos/agent:latest
+    docker run -p 8082:8080 --name mycamera -d kerberos/agent:latest
+
+## Image contract
+
+This contract describes the hardened production Dockerfile, not an already
+published release or a claim of full
+[image-hardening baseline](https://learn.uug.ai/guides/security/image-hardening/)
+compliance. Use a release's actual published multi-platform digest, for example
+`kerberos/agent:<version>@sha256:<digest>`, rather than treating mutable `latest`
+as immutable. No new release digest or signature is asserted by this PR.
+
+| Property | Contract |
+|---|---|
+| Identity | Fixed numeric `USER 999:999`, account `agent`, primary group `kerberosio`; not root and no privilege-changing startup. |
+| Startup | Working directory `/home/agent`; exec-form default command `./main -action run`. Standalone binary CLI defaults remain unchanged. |
+| Inbound HTTP | TCP `8080`, configurable with `AGENT_PORT` (overrides `-port`). Use an unprivileged port. Map host `80:8080`, or Service port `80` to `targetPort: 8080`. Terminate inbound TLS at a trusted proxy; the default listener is HTTP. |
+| Privileges | No bundled setuid/setgid programs or file capabilities. Run with all capabilities dropped and `no-new-privileges`; no `NET_BIND_SERVICE` is required. |
+| Writable application paths | Only `/home/agent/data/` (configuration, recordings, snapshots, cloud, tus checkpoints, capture tests and logs). Binary, UI assets and application home remain root-owned/read-only to UID 999. Demo mode serves its environment without rewriting UI assets. |
+| Temporary state | `/tmp`, `/var/tmp`; user font/media caches under `XDG_CACHE_HOME=/tmp/agent-cache`, owned by `999:999` with mode `0700`. Recreate that directory with these permissions when mounting a fresh tmpfs over `/tmp`. |
+| Health | Exec `/home/agent/main -action healthcheck`; five-second bounded localhost `/health` probe on the resolved port, no proxy/redirects, validates the Agent JSON envelope, exits nonzero on failure. Docker timeout is ten seconds. |
+| Readiness and metrics | `/health` is HTTP liveness, not proof of camera, MQTT or Hub readiness. Camera/Hub details are diagnostic. This image does not add a Prometheus metrics listener. |
+| Certificates/time | Debian CA bundle, OpenSSL configuration and timezone data retained. Mount a prepared CA bundle read-only and use `SSL_CERT_FILE` where appropriate for Go clients; native FFmpeg/OpenSSL clients may require their own CA configuration. Do not disable TLS verification as a substitute. |
+| Secrets | Inject application credentials/keys at runtime using the orchestrator's existing environment/configuration mechanisms. Change default credentials before exposing HTTP. Never build production config, `.env`, private keys or registry credentials into an image. No new generic `*_FILE` interface is provided. |
+| Outbound traffic | Configured RTSP/RTSPS/ONVIF camera endpoints, MQTT, Hub/Vault/storage HTTPS endpoints, DNS and optional telemetry; WebRTC may require configured STUN/TURN and negotiated UDP/TCP connectivity. Apply an allowlist suited to the actual enabled integrations. |
+| Architecture | Native AMD64 and ARM64 build/runtime checks; ARMv7 remains a legacy build target needing its own evidence. ARMv6 is blocked by the default builder manifest; see below. |
+
+### Container permissions and hardening
+
+The final image uses `FROM scratch` with an explicitly assembled Debian/glibc
+runtime rather than the whole Debian filesystem. FFmpeg and its required native
+libraries/plugins remain for camera/media functionality, including AAC-to-PCMU.
+Apt/dpkg executables, account/capability-management tools, curl, and bundled
+setuid/setgid permissions are absent. An application subcommand, not shell/curl,
+implements the healthcheck.
+
+```sh
+docker run --cap-drop=ALL --security-opt=no-new-privileges:true \
+  -p 80:8080 --name mycamera -d kerberos/agent:latest
+```
+
+Do not use `--privileged` for ordinary IP cameras. Local cameras still use the
+separate RTSP sidecar; any device access belongs to that sidecar. Host networking
+remains supported, but without a mapping the Agent is reachable on host port
+`8080`, not 80.
+
+The numeric `USER 999:999` does **not** automatically inherit supplementary video
+membership from `/etc/group`. If a deployment deliberately passes a video device
+to Agent, explicitly add the device's host GID: `--group-add 44` for GID 44,
+Compose `group_add: ["44"]`, or Kubernetes `supplementalGroups: [44]`. The runtime
+retains the `video` group record. Match the actual device permissions; this does
+not grant arbitrary device access or replace the separate local-camera sidecar.
+
+**Pending policy exception:** functional remote terminal sessions require
+`/bin/sh`, PTY devices, terminal data and basic GNU utilities. These remain for
+that feature, not healthchecks or debugging convenience. The user chose to
+preserve the feature, but no policy approver, exception owner or expiry has been
+provided. This is not an approved waiver or full baseline/CIS compliance.
+Baseline-qualified release promotion must remain blocked until the exception is
+formally resolved; do not silently remove the feature or manufacture approval.
+
+Bind mounts must be writable by the image's UID/GID; the container does not
+change host ownership at startup. Before migrating an older volume, check the
+user of the exact old and new image with `docker run --rm --entrypoint id IMAGE`
+and adjust only the mounted directories as necessary. For example:
+
+```sh
+sudo chown -R 999:999 agent/config agent/recordings
+```
+
+A read-only root filesystem is not enabled by default: `data/` needs initialized
+writable storage (and FFmpeg or shell sessions may need `/tmp` and `/var/tmp`).
+Empty bind mounts hide packaged configuration/media files; initialize
+them before use. Docker named volumes populate packaged contents on first use.
+Customers enabling read-only root must supply those writable mounts and cache
+ownership, keep the runtime's device/PTY mounts, and validate their enabled media
+and remote-access features. Apply CPU/memory/PID limits, seccomp/AppArmor and
+network policies appropriate to the deployment.
+
+### Building and checking the image
+
+Build from a clean checkout: the default configuration files under
+`machinery/data/config/` are included in the image, so do not replace them with
+production credentials before building. `.dockerignore` excludes local `.env`
+files, credential directories/configuration, test/profiling outputs, recordings,
+upload checkpoints, dependency directories, and unrelated
+repository files from the build context. The builder uses Go `1.26.10` with
+`GOTOOLCHAIN=local`, checksum database verification, and `go mod verify`. Builds
+use `-mod=readonly` to reject implicit module changes, `-trimpath` to omit local
+source paths, and `-buildvcs=false` to omit checkout metadata. CGO remains enabled
+for the native camera/media dependencies.
+
+Source builds require at least Go 1.26.10, matching the pinned security patch floor.
+The Go, Node, and Debian bases are pinned to verified multi-platform manifest
+digests, including AMD64 and ARM64 (and ARMv7 where previously supported).
+The pinned Debian image is used only to assemble the scratch runtime with
+glibc-compatible FFmpeg, CGO, and MoQ dependencies. Its native package dependency
+metadata selects the architecture's dynamic loader, shared libraries, media
+plugins/drivers, and associated data files. This is intentionally broader than
+an `ldd`-only copy, which misses dynamically loaded plugins. Certificates,
+OpenSSL configuration, time zones, font/MIME/pixbuf caches, account records, and
+standard writable `/tmp` and device mount points are retained. The runtime
+uses `XDG_CACHE_HOME=/tmp/agent-cache` for disposable user caches without making
+the application home writable. It contains package provenance in
+`/var/lib/dpkg/status` and license notices, but
+not the package manager. Update it by rebuilding and replacing the image rather
+than installing packages in a running container.
+
+```sh
+docker build --pull -t kerberos/agent:local .
+sh deployments/docker/test-image.sh kerberos/agent:local
+```
+
+The smoke check uses temporary containers and a Docker volume, cleans them up,
+and checks permissions, FFmpeg audio transcoding, H264/sample-video decoding,
+H265 encoding/decoding, font rendering, shell/PTY devices, volume
+initialization/writes, default and custom HTTP ports, health probes, and
+demo-mode startup with a read-only root and immutable UI assets. It enforces
+privileged low ports while running on 8080 with no capabilities and
+`no-new-privileges`, and also checks an overridden port `8082`. PR builds
+run it on native AMD64 and ARM64 runners. It does not replace testing against a
+real camera, hardware codecs/devices, an MQTT-mediated remote shell session, or
+all supported ARM variants. Review and refresh the
+base-image digests regularly for security updates; `--pull` alone does not update
+a pinned digest. `GO_IMAGE` and `RUNTIME_IMAGE` remain overrideable build arguments
+for compatible alternatives. The package repositories are not snapshot-pinned,
+so base digest pins do not make the entire build bit-for-bit reproducible.
+
+Architecture selections are unchanged by this hardening.
+The pinned `golang:1.26.10-trixie` manifest, like the previous Go 1.25 trixie base,
+has no `linux/arm/v6` variant, so the existing ARMv6 nightly target cannot be
+validated with the default base.
+ARMv6 needs a separately reviewed compatible builder/runtime combination; do
+not assume an ARMv7 image runs on ARMv6 hardware.
+
+### Release evidence, vulnerabilities and lifecycle
+
+The hardened release path must build/test the supported release architectures,
+block HIGH/CRITICAL vulnerability/secret/misconfiguration findings, retain scan
+reports and an SBOM, publish build provenance, and sign the **final
+multi-platform index digest**, not just one child manifest. A pending policy
+exception is not an approved scan exception. There are no approved
+vulnerability exceptions in this change; findings require remediation or a
+reviewed owner/expiry/approval record. Scanner/database failures must fail
+closed. Workflow definitions are not evidence that registry publication,
+signing, or scans have already succeeded.
+
+Source scans cover UI production **and build/development** dependencies and retain
+their application inventory/SBOM, because compiled browser assets hide lockfile
+metadata from final-rootfs scanners. Final-image evidence separately covers native
+packages and the Go binary. Current native-library findings remain release
+blockers. The existing embedded recording-signing key and trust behavior are
+retained under the user's time-limited source-secret exception, recorded for
+review in this PR in
+[image-scan-exceptions.json](../../.github/scripts/image-scan-exceptions.json).
+Owner and approver: `@cedricve`; expiry: **December 5, 2026, 23:59:59 UTC**.
+Anyone possessing this shared private key can forge signatures under its
+identity; the exception accepts that risk temporarily, it does not fix it.
+
+The exception matches only that source file's SHA-256, exact scanner rule,
+category, HIGH severity and line location, and requires exactly one match.
+Changes, missing/incomplete approval, expiry or duplicate findings fail closed.
+It cannot exempt an image finding, another secret, a vulnerability or a
+misconfiguration. The policy is loaded from the executing workflow's checkout,
+not the source directory being scanned. Retained `scan.json` still contains the
+redacted HIGH finding; `exception-policy.json`, `exception-decisions.json` and
+`policy.json` record the approval/application, without key material. Review the
+record in PR #350 before merging; repository owners must enforce required reviews.
+This exception does not approve the separate pending remote-terminal policy.
+
+The build jobs run compilation and production-image smoke tests before the
+security gates. A job that fails at **Source/UI dependency and final image gates
+with SPDX SBOMs** is not a compiler or startup failure; inspect its retained
+reports rather than disabling the gate. Patched Go dependencies are checked
+with `govulncheck` as well as the image/source scanner. The pinned
+`golang.org/x/crypto` revision is the official reviewed OpenPGP module split
+for [GO-2026-5932](https://pkg.go.dev/vuln/GO-2026-5932); Agent does not import the
+retired OpenPGP packages.
+
+Debian 13 stable, security and backports currently provide no patched FFmpeg
+7.1.x or libxml2 packages for the blocking findings on either AMD64 or ARM64.
+For example, Debian defers [CVE-2026-58049](https://security-tracker.debian.org/tracker/CVE-2026-58049)
+until an upstream 7.1.x fix; a fix exists in FFmpeg 9. A Wolfi FFmpeg 7 prototype
+had no vendor-scanner findings but still used the same affected upstream
+7.1.5 code and lacked the existing hardware accelerators. It was **not**
+adopted to obtain a misleading clean scan. A newer media stack needs matched
+CGO headers/libraries and validation of codecs, hardware acceleration and
+supported architectures, not an unreviewed mix of testing-distribution packages.
+The user has chosen to keep the default recording-signing key; no trust
+migration or startup change is implemented. Its source-secret finding becomes
+release-blocking again if the scoped exception expires or no longer matches.
+Any future removal would require a separately
+authorized migration: per-installation keys need verifier/public-key provisioning,
+while requiring an injected key changes startup for existing keyless installations.
+
+Before deploying, obtain the release's digest, source revision, supported
+platforms, per-platform SBOM/scan reports, provenance, signature certificate
+identity and policy approvals from its retained release evidence. Verify the
+signature against the **exact trusted workflow identity** and GitHub OIDC issuer,
+not an unrestricted identity regexp:
+
+Workflow artifacts `image-source-evidence` and `image-final-evidence` retain
+sanitized reports, source/application and per-platform SBOMs, provenance, base
+digests and the source revision for 90 days. Successful releases also attach
+`image-security-evidence.tar.gz`. After successful signing,
+`signing-identity.txt` records the exact
+`https://github.com/kerberos-io/agent/.github/workflows/image-release.yml@...`
+identity; the suffix is the executing workflow ref, not necessarily the source
+ref. Require the expected trusted ref, not merely any identity found in an
+artifact. Secret matches/source excerpts are redacted; findings are not waived.
+
+```sh
+# Set these from the trusted release's real evidence, not from this example.
+cosign verify --certificate-identity="$COSIGN_IDENTITY" \
+  --certificate-oidc-issuer=https://token.actions.githubusercontent.com \
+  "$AGENT_IMAGE"
+```
+
+`AGENT_IMAGE` must contain the immutable registry digest. Reject missing,
+mismatched, expired or unapproved evidence; neither a successful local build nor
+`FROM scratch` certifies the image is vulnerability-free. Registry credentials,
+OIDC permissions and repository release protections must be configured by the
+repository owner; this PR does not invent them or certify those external controls.
+
+Review pinned digests and native/package/module findings for every release and
+regularly between releases. Debian 13 has
+[full support until August 9, 2028 and LTS until June 30, 2030](https://www.debian.org/releases/trixie/),
+with reduced LTS architecture/package coverage. The Go 1.26 line is supported
+[until two newer major Go releases exist](https://go.dev/doc/devel/release#policy);
+update its patch pin and replace it before that boundary. The build-only Node 22
+line has [scheduled EOL April 30, 2027](https://github.com/nodejs/Release/blob/main/schedule.json).
+The pinned UI builder is Node `22.23.4` on Alpine `3.24.3`;
+[Alpine 3.24 main support ends June 1, 2028](https://alpinelinux.org/releases/),
+while community support ends at the next stable release. Track both when
+refreshing that build-only base. Rebuilding does not refresh immutable
+base pins automatically, and APT repositories are not snapshot-pinned. Retain the
+actual installed-package/SBOM evidence per architecture and release.
 
 ### Attach a volume
 
@@ -34,7 +275,7 @@ You attach a volume to your container by leveraging the `-v` option. To mount yo
 
 3.  Run the docker command as following to attach your config directory and recording directory.
 
-        docker run -p 80:80 --name mycamera \
+        docker run -p 80:8080 --name mycamera \
         -v $(pwd)/agent/config:/home/agent/data/config \
         -v $(pwd)/agent/recordings:/home/agent/data/recordings \
         -d --restart=always kerberos/agent:latest
@@ -52,7 +293,7 @@ You'll find [an example `docker-compose.yaml` file here](https://github.com/kerb
     kerberos-agent2:
         image: "kerberos/agent:latest"
         ports:
-        - "8082:80"
+        - "8082:8080"
         environment:
         - AGENT_NAME=agent2
         - AGENT_CAPTURE_IPCAMERA_RTSP=rtsp://x.x.x.x:554/Streaming/Channels/101

deployments/docker/assemble-rootfs.sh

@@ -0,0 +1,107 @@
+#!/bin/sh
+set -eu
+
+root=/runtime
+mkdir -p "$root/usr/bin" "$root/usr/sbin" "$root/usr/lib" "$root/etc" \
+	"$root/var/lib/dpkg" "$root/dev/pts" "$root/proc" "$root/sys" "$root/run"
+ln -s usr/bin "$root/bin"
+ln -s usr/sbin "$root/sbin"
+ln -s usr/lib "$root/lib"
+if [ -L /lib64 ]; then
+	mkdir -p "$root/usr/lib64"
+	ln -s usr/lib64 "$root/lib64"
+fi
+
+# Package dependencies include dlopen-loaded codecs/drivers and their data, which
+# ldd alone cannot discover. Debian resolves the native architecture's loader too.
+apt-cache depends --recurse --installed --no-recommends --no-suggests \
+	--no-conflicts --no-breaks --no-replaces --no-enhances \
+	ffmpeg libatomic1 libstdc++6 dash coreutils findutils grep sed tzdata ncurses-base \
+	> /tmp/runtime-dependencies
+sed -n '/^[a-z0-9][a-z0-9+.:_-]*$/p' /tmp/runtime-dependencies \
+	| grep -vx debconf | sort -u > /tmp/runtime-candidates
+dpkg-query -W -f='${Package} ${db:Status-Status}\n' > /tmp/package-status
+awk '$2 == "installed" { print $1 }' /tmp/package-status \
+	| sort -u > /tmp/installed-packages
+comm -12 /tmp/runtime-candidates /tmp/installed-packages > /tmp/runtime-packages
+
+set --
+while IFS= read -r package; do
+	set -- "$@" "$package"
+done < /tmp/runtime-packages
+dpkg-query -L "$@" > /tmp/runtime-package-paths
+sort -u /tmp/runtime-package-paths > /tmp/runtime-paths
+while IFS= read -r path; do
+	case "$path" in
+		"") continue ;;
+		/usr/share/doc/*/copyright|/usr/share/common-licenses/*) ;;
+		/usr/share/doc/*|/usr/share/man/*|/usr/share/info/*|/usr/share/locale/*|\
+		/usr/share/lintian/*|/usr/share/bug/*|/usr/share/debconf/*|\
+		/usr/share/bash-completion/*|/usr/share/fish/*|\
+		/usr/lib/systemd/*|/lib/systemd/*|/etc/init.d/*|/etc/cron.*/*) continue ;;
+	esac
+	if [ -d "$path" ] && [ ! -L "$path" ]; then
+		continue
+	fi
+	if [ ! -f "$path" ] && [ ! -L "$path" ]; then
+		printf 'Missing selected runtime file: %s\n' "$path" >&2
+		exit 1
+	fi
+	printf '%s\n' "$path"
+done < /tmp/runtime-paths > /tmp/runtime-files
+tar --create --file /tmp/runtime.tar --no-recursion --files-from /tmp/runtime-files
+tar --extract --file /tmp/runtime.tar --directory "$root"
+
+# Preserve generated trust stores, font configuration/caches and pixbuf loaders.
+cp -a --parents /etc/ssl /usr/lib/ssl /usr/share/ca-certificates /etc/fonts \
+	/var/cache/fontconfig /usr/share/mime /usr/share/common-licenses "$root"
+find /usr/lib -path '*/gdk-pixbuf-2.0/*/loaders.cache' \
+	-exec cp -a --parents {} "$root" \;
+cp -a --parents /etc/nsswitch.conf /etc/gai.conf /etc/localtime \
+	/etc/ld.so.conf /etc/ld.so.conf.d /etc/os-release /etc/debian_version \
+	/usr/lib/os-release /usr/lib/locale "$root"
+
+# Some public SONAME links (BLAS/LAPACK, for example) are generated by Debian
+# alternatives rather than owned by a package. Keep only links to selected files.
+find /usr/lib -type l > /tmp/runtime-links
+if [ -d /usr/lib64 ]; then
+	find /usr/lib64 -type l >> /tmp/runtime-links
+fi
+while IFS= read -r link; do
+	target=$(readlink -f "$link")
+	if [ -e "$root$target" ]; then
+		while [ -L "$link" ]; do
+			cp -a --parents "$link" "$root"
+			target=$(readlink "$link")
+			case "$target" in
+				/*) link=$target ;;
+				*) link=$(dirname "$link")/$target ;;
+			esac
+		done
+	fi
+done < /tmp/runtime-links
+
+getent passwd root agent nobody > "$root/etc/passwd"
+getent group root kerberosio video nogroup > "$root/etc/group"
+mkdir -p "$root/tmp" "$root/var/tmp"
+chmod 1777 "$root/tmp" "$root/var/tmp"
+install -d -m 0700 -o 999 -g 999 "$root/tmp/agent-cache"
+
+# Retain package provenance for scanners without shipping apt/dpkg executables.
+dpkg-query -s "$@" ca-certificates base-files > "$root/var/lib/dpkg/status"
+mkdir -p "$root/usr/share/doc/ca-certificates"
+cp /usr/share/doc/ca-certificates/copyright "$root/usr/share/doc/ca-certificates/"
+ldconfig -r "$root"
+chroot "$root" /usr/bin/fc-cache --force
+find "$root" -type f -perm /6000 -exec chmod a-s {} +
+
+# Execute against the assembled loader/libraries, not the assembly-stage rootfs.
+chroot --userspec=999:999 "$root" /bin/sh -ec '
+	/home/agent/main -action version
+	ffmpeg -version
+	ffprobe -version
+	test -s /etc/ssl/certs/ca-certificates.crt
+	test ! -e /usr/bin/curl
+	test ! -e /usr/bin/apt
+	test ! -e /usr/bin/dpkg
+'

deployments/docker/docker-compose-with-volumes.yaml

@@ -3,7 +3,7 @@ services:
   kerberos-agent1:
     image: "kerberos/agent:latest"
     ports:
-      - "8081:80"
+      - "8081:8080"
     environment:
       - AGENT_NAME=agent1
       # You can still override the configuration with environment variables, but might not makes sense if you are attaching a host config.
@@ -14,14 +14,14 @@ services:
   kerberos-agent2:
     image: "kerberos/agent:latest"
     ports:
-      - "8082:80"
+      - "8082:8080"
     volumes:
       - ./agent2/config:/home/agent/data/config
       - ./agent2/recordings:/home/agent/data/recordings
   kerberos-agent3:
     image: "kerberos/agent:latest"
     ports:
-      - "8083:80"
+      - "8083:8080"
     volumes:
       - ./agent3/config:/home/agent/data/config
       - ./agent3/recordings:/home/agent/data/recordings

deployments/docker/docker-compose.yaml

@@ -8,7 +8,7 @@ services:
   kerberos-agent1:
     image: "kerberos/agent:latest"
     ports:
-      - "8081:80"
+      - "8081:8080"
     environment:
       <<: *common-variables
       AGENT_NAME: agent1
@@ -17,7 +17,7 @@ services:
   kerberos-agent2:
     image: "kerberos/agent:latest"
     ports:
-      - "8082:80"
+      - "8082:8080"
     environment:
       <<: *common-variables
       AGENT_NAME: agent2
@@ -26,7 +26,7 @@ services:
   kerberos-agent3:
     image: "kerberos/agent:latest"
     ports:
-      - "8083:80"
+      - "8083:8080"
     environment:
       <<: *common-variables
       AGENT_NAME: agent3

deployments/docker/test-image.sh

@@ -0,0 +1,128 @@
+#!/bin/sh
+set -eu
+
+image=${1:?Usage: sh deployments/docker/test-image.sh IMAGE}
+container=
+volume=
+temporary_volume=
+cleanup() {
+	if [ -n "$container" ]; then docker rm -f "$container" >/dev/null; fi
+	if [ -n "$volume" ]; then docker volume rm "$volume" >/dev/null; fi
+	if [ -n "$temporary_volume" ]; then docker volume rm "$temporary_volume" >/dev/null; fi
+}
+trap cleanup EXIT
+trap 'exit 1' HUP INT TERM
+
+docker run --rm --entrypoint /bin/sh "$image" -ec '
+	test "$(id -u)" = 999
+	test "$(id -g)" = 999
+	test ! -w /home/agent
+	test ! -w /home/agent/main
+	test -w /home/agent/data/config
+	test -w /home/agent/data/recordings
+	test -w /home/agent/data/tus
+	test ! -w /home/agent/www
+	test ! -w /home/agent/www/env.js
+	test -z "$(find / -xdev -type f -perm /6000 -print 2>/dev/null)"
+	! command -v setcap
+	! command -v capsh
+	! command -v apt
+	! command -v apt-get
+	! command -v dpkg
+	! command -v curl
+	test -s /var/lib/dpkg/status
+	test -r /etc/os-release
+	test -r /usr/share/terminfo/x/xterm-256color
+	test -s /etc/ssl/certs/ca-certificates.crt
+	test "$(id -Gn)" = "kerberosio"
+	test "$(stat -c %a /tmp)" = 1777
+	test -w "$XDG_CACHE_HOME"
+	test ! -e /home/agent/.env
+	test -z "$(find /home/agent/data/tus -type f -print)"
+	./main -action version
+	ffmpeg -nostdin -v error -f lavfi -i sine=frequency=1000:sample_rate=8000 \
+		-t 0.1 -c:a aac -f adts /tmp/test.aac
+	ffmpeg -nostdin -v error -i /tmp/test.aac -ar 8000 -ac 1 \
+		-c:a pcm_mulaw -f mulaw /tmp/test.pcm
+	test -s /tmp/test.pcm
+	ffprobe -v error -show_entries stream=codec_name data/test-480p.mp4
+	ffmpeg -nostdin -v error -i data/test-480p.mp4 -frames:v 1 -f null -
+	ffmpeg -nostdin -v error -i data/test-480p.mp4 -frames:v 1 \
+		-c:v libx265 -x265-params pools=none:frame-threads=1:log-level=error \
+		-f hevc /tmp/test.hevc
+	ffmpeg -nostdin -v error -i /tmp/test.hevc -frames:v 1 -f null -
+	ffmpeg -nostdin -v error -f lavfi -i color=black:s=64x64 \
+		-vf drawtext=text=scratch:fontcolor=white -frames:v 1 -f null -
+'
+
+# Exercise the same shell and PTY devices used by remote-access sessions.
+docker run --rm -t --entrypoint /bin/sh "$image" -ec '
+	test -t 0
+	test -t 1
+	test -c /dev/ptmx
+	test -d /dev/pts
+	stty size
+	printf "Shell and PTY ready\n"
+'
+docker run --rm --group-add 44 --entrypoint /bin/sh "$image" -ec '
+	test "$(id -Gn)" = "kerberosio video"
+'
+wait_healthy() {
+	attempt=0
+	while [ "$attempt" -lt 60 ]; do
+		status=$(docker inspect --format '{{.State.Health.Status}}' "$container")
+		if [ "$status" = healthy ]; then return; fi
+		if [ "$(docker inspect --format '{{.State.Running}}' "$container")" != true ]; then
+			break
+		fi
+		attempt=$((attempt + 1))
+		sleep 1
+	done
+	docker logs "$container"
+	docker inspect --format '{{json .State.Health}}' "$container"
+	echo "Agent did not become healthy" >&2
+	return 1
+}
+
+# Exercise the unprivileged default port without any capabilities.
+volume=$(docker volume create)
+container=$(docker run -d --cap-drop ALL --security-opt no-new-privileges=true \
+	--sysctl net.ipv4.ip_unprivileged_port_start=1024 \
+	--health-interval=1s --health-start-period=0s \
+	--mount "type=volume,source=$volume,target=/home/agent/data" "$image")
+wait_healthy
+docker exec "$container" sh -ec '
+	test "$AGENT_PORT" = 8080
+	./main -action healthcheck
+	test -r data/config/config.json
+	touch data/config/.permission-test data/recordings/.permission-test data/tus/.permission-test
+	rm data/config/.permission-test data/recordings/.permission-test data/tus/.permission-test
+'
+if docker exec -e AGENT_PORT=65534 "$container" ./main -action healthcheck; then
+	echo "Healthcheck incorrectly succeeded on a port with no Agent" >&2
+	exit 1
+fi
+docker stop --time 10 "$container" >/dev/null
+docker rm "$container" >/dev/null
+container=
+
+# Demo UI selection must work with immutable assets and a read-only root.
+temporary_volume=$(docker volume create)
+container=$(docker run -d --read-only --cap-drop ALL \
+	--security-opt no-new-privileges=true \
+	--sysctl net.ipv4.ip_unprivileged_port_start=1024 \
+	--health-interval=1s --health-start-period=0s \
+	--mount "type=volume,source=$volume,target=/home/agent/data" \
+	--mount "type=volume,source=$temporary_volume,target=/tmp" \
+	-e AGENT_PORT=8082 -e AGENT_MODE=demo \
+	-e http_proxy=http://127.0.0.1:9 -e HTTP_PROXY=http://127.0.0.1:9 \
+	-e no_proxy= -e NO_PROXY= "$image")
+wait_healthy
+docker exec "$container" sh -ec '
+	test -f www/env.js
+	test -f www/env.demo.js
+	test ! -w www
+	ffmpeg -nostdin -v error -f lavfi -i color=black:s=64x64 \
+		-vf drawtext=text=readonly:fontcolor=white -frames:v 1 -f null -
+'
+echo "Scratch image permissions, media, shell/PTY, volume writes, default/custom ports, and demo startup passed."

deployments/kubernetes/README.md

@@ -6,6 +6,27 @@ Kubernetes can help you to build a scalable, flexible and resilient deployment.
 
 We've provided an example deployment `deployment-agent.yml` in this directory, which show case you have to create a deployment (and under the hood a pod), to run a Kerberos Agent workload.
 
+These manifests target the forthcoming hardened image: **containerPort and
+Service targetPort are 8080**, while external Service/Ingress port 80 stays
+unchanged. Pin a published hardened release digest before production use.
+Previously published port-80 images (including `v3.14.1`) require the old
+container/target ports; upgrade the image and mapping together. Host-network
+deployments use host port 8080 without Service port translation.
+
+The examples use UID/GID `999:999`, `fsGroup: 999`, a read-only root filesystem,
+no privilege escalation, all capabilities dropped and runtime-default seccomp.
+A non-root init container copies missing packaged data to the writable volume
+without overwriting existing files and initializes the user cache. Temporary
+storage is ephemeral. The PVC example preserves the existing `config/`,
+`recordings/`, `snapshots/` and `cloud/` directories under the same PVC root,
+mounting that root at `/home/agent/data` to cover all writable data paths.
+Ensure the storage driver honors fsGroup or pre-provision UID/GID permissions.
+If deliberately passing video devices, add their actual GID with
+`supplementalGroups`; ordinary IP cameras do not need that group or devices.
+See the [image contract](../docker/README.md#image-contract), including the pending
+remote-terminal policy exception and release blockers. These examples do not
+assert a running cluster or security-policy approval.
+
 ## Create a Kerberos Agent deployment
 
 It's always a best practices to isolate and structure your workloads in Kubernetes. To achieve this we are utilising the concept of namespaces. For this example we will create a new namespace `demo`.
@@ -61,8 +82,7 @@ Depending on where and how you are hosting the Kubernetes cluster, you may need
           image: kerberos/agent:latest
           volumeMounts:
             - name: kerberos-data
-              mountPath: /home/agent/data/config
-              subPath: config
+              mountPath: /home/agent/data
         ...
 
 ## Expose with Ingress
@@ -76,6 +96,7 @@ In the first example `deployment-agent.yml` we are using a `LoadBalancer` to exp
     type: LoadBalancer
     ports:
         - port: 80
+          targetPort: 8080
     ...
 
 An alternative to `LoadBalancer` is `Ingress`. By leveraging an ingress such as `ingress-nginx` or `traefik` you setup a gateway (single point of contact), through which all communication to your apps (services) will flow.

deployments/kubernetes/deployment-agent-volume.yml

@@ -14,41 +14,56 @@ spec:
       labels:
         app: agent
     spec:
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 999
+        runAsGroup: 999
+        fsGroup: 999
+        seccompProfile:
+          type: RuntimeDefault
       volumes:
         - name: kerberos-data
           persistentVolumeClaim:
             claimName: kerberos-data
+        - name: temporary
+          emptyDir: {}
 
       initContainers:
         - name: download-config
           image: kerberos/agent:latest
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
           volumeMounts:
             - name: kerberos-data
-              mountPath: /home/agent/data/config
-              subPath: config
-          command:
-            [
-              "cp",
-              "/home/agent/data/config.template.json",
-              "/home/agent/data/config/config.json",
-            ]
+              mountPath: /runtime-data
+            - name: temporary
+              mountPath: /runtime-tmp
+          command: ["/bin/sh", "-ec", "cp -R --update=none /home/agent/data/. /runtime-data/; install -d -m 0700 /runtime-tmp/agent-cache"]
 
       containers:
         - name: agent
           image: kerberos/agent:latest
+          livenessProbe:
+            exec:
+              command: ["/home/agent/main", "-action", "healthcheck"]
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 10
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
           volumeMounts:
             - name: kerberos-data
-              mountPath: /home/agent/data/config
-              subPath: config
-            - name: kerberos-data
-              mountPath: /home/agent/data/recordings
-              subPath: recordings
-            - name: kerberos-data
-              mountPath: /home/agent/data/snapshots
-              subPath: snapshots
-            - name: kerberos-data
-              mountPath: /home/agent/data/cloud
-              subPath: cloud
+              mountPath: /home/agent/data
+            - name: temporary
+              mountPath: /tmp
+            - name: temporary
+              mountPath: /var/tmp
           ports:
-            - containerPort: 80
+            - containerPort: 8080
               protocol: TCP

deployments/kubernetes/deployment-agent-with-ingress.yml

@@ -14,11 +14,55 @@ spec:
       labels:
         app: agent
     spec:
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 999
+        runAsGroup: 999
+        fsGroup: 999
+        seccompProfile:
+          type: RuntimeDefault
+      volumes:
+        - name: data
+          emptyDir: {}
+        - name: temporary
+          emptyDir: {}
+      initContainers:
+        - name: initialize-data
+          image: kerberos/agent:latest
+          command: ["/bin/sh", "-ec", "cp -R --update=none /home/agent/data/. /runtime-data/; install -d -m 0700 /runtime-tmp/agent-cache"]
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: data
+              mountPath: /runtime-data
+            - name: temporary
+              mountPath: /runtime-tmp
       containers:
         - name: agent
           image: kerberos/agent:latest
+          livenessProbe:
+            exec:
+              command: ["/home/agent/main", "-action", "healthcheck"]
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 10
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: data
+              mountPath: /home/agent/data
+            - name: temporary
+              mountPath: /tmp
+            - name: temporary
+              mountPath: /var/tmp
           ports:
-            - containerPort: 80
+            - containerPort: 8080
               protocol: TCP
           resources:
             limits:
@@ -48,7 +92,7 @@ spec:
   #type: LoadBalancer
   ports:
     - port: 80
-      targetPort: 80
+      targetPort: 8080
   selector:
     app: agent
 ---

deployments/kubernetes/deployment-agent.yml

@@ -14,11 +14,55 @@ spec:
       labels:
         app: agent
     spec:
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 999
+        runAsGroup: 999
+        fsGroup: 999
+        seccompProfile:
+          type: RuntimeDefault
+      volumes:
+        - name: data
+          emptyDir: {}
+        - name: temporary
+          emptyDir: {}
+      initContainers:
+        - name: initialize-data
+          image: kerberos/agent:latest
+          command: ["/bin/sh", "-ec", "cp -R --update=none /home/agent/data/. /runtime-data/; install -d -m 0700 /runtime-tmp/agent-cache"]
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: data
+              mountPath: /runtime-data
+            - name: temporary
+              mountPath: /runtime-tmp
       containers:
         - name: agent
-          image: kerberos/agent:3.2.3
+          image: kerberos/agent:latest
+          livenessProbe:
+            exec:
+              command: ["/home/agent/main", "-action", "healthcheck"]
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 10
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: data
+              mountPath: /home/agent/data
+            - name: temporary
+              mountPath: /tmp
+            - name: temporary
+              mountPath: /var/tmp
           ports:
-            - containerPort: 80
+            - containerPort: 8080
               protocol: TCP
           resources:
             limits:
@@ -48,6 +92,6 @@ spec:
   type: LoadBalancer
   ports:
     - port: 80
-      targetPort: 80
+      targetPort: 8080
   selector:
     app: agent

deployments/terraform/README.md

@@ -4,6 +4,13 @@ If you are using Terraform as part of your DevOps stack, you might utilise it to
 
 For this example we will install Kerberos Agent using `docker` on a remote `linux` machine. Therefore we'll make sure we have the `TelkomIndonesia/linux` provider initialised.
 
+The example targets the forthcoming hardened image and maps the configured host
+port (80 by default) to **container port 8080**. Pin a published hardened release
+digest before deployment. Older releases, including `v3.14.1`, still require
+container port 80; change the image and mapping together. See the
+[image contract](../docker/README.md#image-contract) for runtime permissions,
+writable paths and release-policy limitations.
+
      terraform init
 
 Once initialised you should see similar output:

deployments/terraform/docker.tf

@@ -34,9 +34,9 @@ resource "linux_script" "install_docker" {
 
 resource "linux_script" "install_docker_kerberos_agent" {
     lifecycle_commands {
-        create = "docker pull $IMAGE:$VERSION && docker run -d -p $PORT:80 --name agent $IMAGE:$VERSION"
+        create = "docker pull $IMAGE:$VERSION && docker run -d -p $PORT:8080 --name agent $IMAGE:$VERSION"
         read = "docker inspect agent"
-        update = "docker pull $IMAGE:$VERSION && docker rm agent --force && docker run -d -p $PORT:80 --name agent $IMAGE:$VERSION"
+        update = "docker pull $IMAGE:$VERSION && docker rm agent --force && docker run -d -p $PORT:8080 --name agent $IMAGE:$VERSION"
         delete = "docker rm agent --force"
     }
     environment = {

machinery/README.md

@@ -5,7 +5,6 @@
 <a target="_blank" href="https://twitter.com/kerberosio?ref_src=twsrc%5Etfw"><img src="https://img.shields.io/twitter/url.svg?label=Follow%20%40kerberosio&style=social&url=https%3A%2F%2Ftwitter.com%2Fkerberosio" alt="Twitter Widget"></a>
 <a target="_blank" href="https://join.slack.com/t/kerberosio/shared_invite/zt-1a5oj4pwm-O4qCAN9c5r2um0Ns0ge8ww"><img src="https://img.shields.io/badge/slack-@kerberosio-yellow.svg?logo=slack " alt="Kerberos.io"></a>
 
-<a target="_blank" href="https://circleci.com/gh/kerberos-io/agent"><img src="https://circleci.com/gh/kerberos-io/agent.svg?style=svg"/></a>
 <img src="https://github.com/kerberos-io/agent/workflows/Go/badge.svg"/>
 <img src="https://github.com/kerberos-io/agent/workflows/CodeQL/badge.svg"/>
 

machinery/data/config/config.json

@@ -7,7 +7,7 @@
 	"offline": "false",
 	"auto_clean": "true",
 	"remove_after_upload": "true",
-	"max_directory_size": 100,
+	"max_directory_size": 0,
 	"timezone": "Africa/Ceuta",
 	"capture": {
 		"name": "",
@@ -122,4 +122,4 @@
 	"signing": {},
 	"realtimeprocessing": "false",
 	"realtimeprocessing_topic": ""
-}
+}

machinery/data/tus/1786715908_3-737_encrypted_0-0-0-0_-1_25498.mp4.primary.json

@@ -0,0 +1 @@
+{"upload_url":"https://vault.kerberos.io/api/storage/tus/19e42fbc666a38064904caf8c46d182a","vault_uri":"https://vault.kerberos.io/api/storage/tus/","size":1591581}

machinery/data/tus/1788718145_3-694_encrypted_0-0-0-0_-1_25497.mp4.primary.json

@@ -0,0 +1 @@
+{"upload_url":"https://vault.kerberos.io/api/storage/tus/bd3be0f9ecd3873d381f57cb8b4a7e96","vault_uri":"https://vault.kerberos.io/api/storage/tus/","size":2560480}

machinery/docs/docs.go

@@ -24,6 +24,38 @@ const docTemplate = `{
     "host": "{{.Host}}",
     "basePath": "{{.BasePath}}",
     "paths": {
+        "/api/camera/discover": {
+            "get": {
+                "description": "Runs an advanced Fing/WiFiman-style scan (ONVIF WS-Discovery + TCP port scan + MAC/vendor lookup) and returns the devices found on the local network.",
+                "tags": [
+                    "onvif"
+                ],
+                "summary": "Discover cameras and other devices on the local network.",
+                "operationId": "camera-discover",
+                "parameters": [
+                    {
+                        "type": "integer",
+                        "description": "Discovery timeout in milliseconds (default 2000)",
+                        "name": "timeout",
+                        "in": "query"
+                    },
+                    {
+                        "type": "string",
+                        "description": "Optional subnet(s) to scan, e.g. '192.168.1.0/24' (comma-separated). Defaults to the local interfaces.",
+                        "name": "subnet",
+                        "in": "query"
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/models.APIResponse"
+                        }
+                    }
+                }
+            }
+        },
         "/api/camera/onvif/capabilities": {
             "post": {
                 "description": "Will return the ONVIF capabilities for the specific camera.",
@@ -623,6 +655,40 @@ const docTemplate = `{
                 }
             }
         },
+        "/api/persistence/secondary/verify": {
+            "post": {
+                "security": [
+                    {
+                        "Bearer": []
+                    }
+                ],
+                "description": "Will verify the secondary persistence.",
+                "tags": [
+                    "persistence"
+                ],
+                "summary": "Will verify the secondary persistence.",
+                "operationId": "verify-secondary-persistence",
+                "parameters": [
+                    {
+                        "description": "Config",
+                        "name": "config",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/models.Config"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/models.APIResponse"
+                        }
+                    }
+                }
+            }
+        },
         "/api/persistence/verify": {
             "post": {
                 "security": [
@@ -656,9 +722,151 @@ const docTemplate = `{
                     }
                 }
             }
+        },
+        "/health": {
+            "get": {
+                "description": "Confirms that the Agent HTTP process can serve requests and reports current camera stream and Hub heartbeat diagnostics. Operational dependency failures do not change the liveness HTTP status.",
+                "produces": [
+                    "application/json"
+                ],
+                "tags": [
+                    "health"
+                ],
+                "summary": "Check Agent health",
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/http.HealthResponse"
+                        }
+                    }
+                }
+            }
         }
     },
     "definitions": {
+        "http.Health": {
+            "type": "object",
+            "properties": {
+                "cameraConnected": {
+                    "type": "boolean"
+                },
+                "description": {
+                    "type": "string"
+                },
+                "hub": {
+                    "$ref": "#/definitions/http.HubHealth"
+                },
+                "mainStream": {
+                    "$ref": "#/definitions/http.StreamHealth"
+                },
+                "subStream": {
+                    "$ref": "#/definitions/http.StreamHealth"
+                }
+            }
+        },
+        "http.HealthResponse": {
+            "type": "object",
+            "properties": {
+                "applicationStatusCode": {
+                    "type": "string"
+                },
+                "data": {
+                    "$ref": "#/definitions/http.HealthResponseData"
+                },
+                "entityStatusCode": {
+                    "type": "string"
+                },
+                "httpStatusCode": {
+                    "type": "integer"
+                },
+                "message": {
+                    "type": "string"
+                },
+                "metadata": {
+                    "$ref": "#/definitions/http.ResponseMetadata"
+                }
+            }
+        },
+        "http.HealthResponseData": {
+            "type": "object",
+            "properties": {
+                "health": {
+                    "$ref": "#/definitions/http.Health"
+                }
+            }
+        },
+        "http.HubHealth": {
+            "type": "object",
+            "properties": {
+                "configured": {
+                    "type": "boolean"
+                },
+                "connected": {
+                    "type": "boolean"
+                },
+                "lastHeartbeatAttemptAt": {
+                    "type": "integer"
+                },
+                "lastSuccessfulHeartbeatAt": {
+                    "type": "integer"
+                }
+            }
+        },
+        "http.ResponseMetadata": {
+            "type": "object",
+            "properties": {
+                "applicationName": {
+                    "type": "string"
+                },
+                "applicationVersion": {
+                    "type": "string"
+                },
+                "path": {
+                    "type": "string"
+                },
+                "timestamp": {
+                    "type": "integer"
+                },
+                "traceId": {
+                    "type": "string"
+                }
+            }
+        },
+        "http.StreamHealth": {
+            "type": "object",
+            "properties": {
+                "configured": {
+                    "type": "boolean"
+                },
+                "connected": {
+                    "type": "boolean"
+                },
+                "fps": {
+                    "type": "number"
+                },
+                "lastPacketAt": {
+                    "type": "integer"
+                },
+                "packagesProcessed": {
+                    "type": "integer"
+                },
+                "resolution": {
+                    "$ref": "#/definitions/http.StreamResolution"
+                }
+            }
+        },
+        "http.StreamResolution": {
+            "type": "object",
+            "properties": {
+                "height": {
+                    "type": "integer"
+                },
+                "width": {
+                    "type": "integer"
+                }
+            }
+        },
         "models.APIResponse": {
             "type": "object",
             "properties": {
@@ -730,12 +938,19 @@ const docTemplate = `{
                 "fragmentedduration": {
                     "type": "integer"
                 },
+                "gopsize": {
+                    "description": "GOP size in seconds, used for pre-recording",
+                    "type": "integer"
+                },
                 "ipcamera": {
                     "$ref": "#/definitions/models.IPCamera"
                 },
                 "liveview": {
                     "type": "string"
                 },
+                "liveview_chunking": {
+                    "type": "string"
+                },
                 "maxlengthrecording": {
                     "type": "integer"
                 },
@@ -745,6 +960,10 @@ const docTemplate = `{
                 "name": {
                     "type": "string"
                 },
+                "onvif_motion": {
+                    "description": "ONVIFMotion routes the camera's ONVIF motion events into the\nagent's motion-triggered recording pipeline. When \"true\" the\nagent opens an event/stream against the configured ONVIF\nendpoint and forwards Motion+Active events to HandleMotion.\nRequires Capture.IPCamera.ONVIFXAddr / ONVIFUsername /\nONVIFPassword to be set. Default empty (disabled) keeps the\nexisting pixel-diff motion detection as the only source.",
+                    "type": "string"
+                },
                 "pixelChangeThreshold": {
                     "type": "integer"
                 },
@@ -823,9 +1042,15 @@ const docTemplate = `{
                 "kstorage": {
                     "$ref": "#/definitions/models.KStorage"
                 },
+                "kstorage_secondary": {
+                    "$ref": "#/definitions/models.KStorage"
+                },
                 "max_directory_size": {
                     "type": "integer"
                 },
+                "min_free_space": {
+                    "type": "integer"
+                },
                 "mqtt_password": {
                     "type": "string"
                 },
@@ -856,6 +1081,9 @@ const docTemplate = `{
                 "s3": {
                     "$ref": "#/definitions/models.S3"
                 },
+                "signing": {
+                    "$ref": "#/definitions/models.Signing"
+                },
                 "stunuri": {
                     "type": "string"
                 },
@@ -947,6 +1175,15 @@ const docTemplate = `{
         "models.IPCamera": {
             "type": "object",
             "properties": {
+                "base_height": {
+                    "type": "integer"
+                },
+                "base_width": {
+                    "type": "integer"
+                },
+                "channels": {
+                    "type": "integer"
+                },
                 "fps": {
                     "type": "string"
                 },
@@ -965,21 +1202,85 @@ const docTemplate = `{
                 "onvif_xaddr": {
                     "type": "string"
                 },
+                "pps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "rtsp": {
                     "type": "string"
                 },
+                "sample_rate": {
+                    "type": "integer"
+                },
+                "sps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_fps": {
                     "type": "string"
                 },
                 "sub_height": {
                     "type": "integer"
                 },
+                "sub_pps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_rtsp": {
                     "type": "string"
                 },
+                "sub_sps_nalus": {
+                    "description": "Sub stream parameter sets, captured separately from the main stream so the\nlive HLS muxer can build a correct init segment when a viewer switches the\nlive view to the sub (low-resolution) stream.",
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
+                "sub_vps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_width": {
                     "type": "integer"
                 },
+                "vps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "width": {
                     "type": "integer"
                 }
@@ -995,15 +1296,24 @@ const docTemplate = `{
                     "description": "old way, remove this",
                     "type": "string"
                 },
+                "custom_headers": {
+                    "type": "string"
+                },
                 "directory": {
                     "type": "string"
                 },
+                "max_retries": {
+                    "type": "integer"
+                },
                 "provider": {
                     "type": "string"
                 },
                 "secret_access_key": {
                     "type": "string"
                 },
+                "timeout": {
+                    "type": "integer"
+                },
                 "uri": {
                     "type": "string"
                 }
@@ -1141,6 +1451,17 @@ const docTemplate = `{
                 }
             }
         },
+        "models.Signing": {
+            "type": "object",
+            "properties": {
+                "enabled": {
+                    "type": "string"
+                },
+                "private_key": {
+                    "type": "string"
+                }
+            }
+        },
         "models.Timetable": {
             "type": "object",
             "properties": {

machinery/docs/swagger.json

@@ -17,6 +17,38 @@
     },
     "basePath": "/",
     "paths": {
+        "/api/camera/discover": {
+            "get": {
+                "description": "Runs an advanced Fing/WiFiman-style scan (ONVIF WS-Discovery + TCP port scan + MAC/vendor lookup) and returns the devices found on the local network.",
+                "tags": [
+                    "onvif"
+                ],
+                "summary": "Discover cameras and other devices on the local network.",
+                "operationId": "camera-discover",
+                "parameters": [
+                    {
+                        "type": "integer",
+                        "description": "Discovery timeout in milliseconds (default 2000)",
+                        "name": "timeout",
+                        "in": "query"
+                    },
+                    {
+                        "type": "string",
+                        "description": "Optional subnet(s) to scan, e.g. '192.168.1.0/24' (comma-separated). Defaults to the local interfaces.",
+                        "name": "subnet",
+                        "in": "query"
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/models.APIResponse"
+                        }
+                    }
+                }
+            }
+        },
         "/api/camera/onvif/capabilities": {
             "post": {
                 "description": "Will return the ONVIF capabilities for the specific camera.",
@@ -616,6 +648,40 @@
                 }
             }
         },
+        "/api/persistence/secondary/verify": {
+            "post": {
+                "security": [
+                    {
+                        "Bearer": []
+                    }
+                ],
+                "description": "Will verify the secondary persistence.",
+                "tags": [
+                    "persistence"
+                ],
+                "summary": "Will verify the secondary persistence.",
+                "operationId": "verify-secondary-persistence",
+                "parameters": [
+                    {
+                        "description": "Config",
+                        "name": "config",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/models.Config"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/models.APIResponse"
+                        }
+                    }
+                }
+            }
+        },
         "/api/persistence/verify": {
             "post": {
                 "security": [
@@ -649,9 +715,151 @@
                     }
                 }
             }
+        },
+        "/health": {
+            "get": {
+                "description": "Confirms that the Agent HTTP process can serve requests and reports current camera stream and Hub heartbeat diagnostics. Operational dependency failures do not change the liveness HTTP status.",
+                "produces": [
+                    "application/json"
+                ],
+                "tags": [
+                    "health"
+                ],
+                "summary": "Check Agent health",
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/http.HealthResponse"
+                        }
+                    }
+                }
+            }
         }
     },
     "definitions": {
+        "http.Health": {
+            "type": "object",
+            "properties": {
+                "cameraConnected": {
+                    "type": "boolean"
+                },
+                "description": {
+                    "type": "string"
+                },
+                "hub": {
+                    "$ref": "#/definitions/http.HubHealth"
+                },
+                "mainStream": {
+                    "$ref": "#/definitions/http.StreamHealth"
+                },
+                "subStream": {
+                    "$ref": "#/definitions/http.StreamHealth"
+                }
+            }
+        },
+        "http.HealthResponse": {
+            "type": "object",
+            "properties": {
+                "applicationStatusCode": {
+                    "type": "string"
+                },
+                "data": {
+                    "$ref": "#/definitions/http.HealthResponseData"
+                },
+                "entityStatusCode": {
+                    "type": "string"
+                },
+                "httpStatusCode": {
+                    "type": "integer"
+                },
+                "message": {
+                    "type": "string"
+                },
+                "metadata": {
+                    "$ref": "#/definitions/http.ResponseMetadata"
+                }
+            }
+        },
+        "http.HealthResponseData": {
+            "type": "object",
+            "properties": {
+                "health": {
+                    "$ref": "#/definitions/http.Health"
+                }
+            }
+        },
+        "http.HubHealth": {
+            "type": "object",
+            "properties": {
+                "configured": {
+                    "type": "boolean"
+                },
+                "connected": {
+                    "type": "boolean"
+                },
+                "lastHeartbeatAttemptAt": {
+                    "type": "integer"
+                },
+                "lastSuccessfulHeartbeatAt": {
+                    "type": "integer"
+                }
+            }
+        },
+        "http.ResponseMetadata": {
+            "type": "object",
+            "properties": {
+                "applicationName": {
+                    "type": "string"
+                },
+                "applicationVersion": {
+                    "type": "string"
+                },
+                "path": {
+                    "type": "string"
+                },
+                "timestamp": {
+                    "type": "integer"
+                },
+                "traceId": {
+                    "type": "string"
+                }
+            }
+        },
+        "http.StreamHealth": {
+            "type": "object",
+            "properties": {
+                "configured": {
+                    "type": "boolean"
+                },
+                "connected": {
+                    "type": "boolean"
+                },
+                "fps": {
+                    "type": "number"
+                },
+                "lastPacketAt": {
+                    "type": "integer"
+                },
+                "packagesProcessed": {
+                    "type": "integer"
+                },
+                "resolution": {
+                    "$ref": "#/definitions/http.StreamResolution"
+                }
+            }
+        },
+        "http.StreamResolution": {
+            "type": "object",
+            "properties": {
+                "height": {
+                    "type": "integer"
+                },
+                "width": {
+                    "type": "integer"
+                }
+            }
+        },
         "models.APIResponse": {
             "type": "object",
             "properties": {
@@ -723,12 +931,19 @@
                 "fragmentedduration": {
                     "type": "integer"
                 },
+                "gopsize": {
+                    "description": "GOP size in seconds, used for pre-recording",
+                    "type": "integer"
+                },
                 "ipcamera": {
                     "$ref": "#/definitions/models.IPCamera"
                 },
                 "liveview": {
                     "type": "string"
                 },
+                "liveview_chunking": {
+                    "type": "string"
+                },
                 "maxlengthrecording": {
                     "type": "integer"
                 },
@@ -738,6 +953,10 @@
                 "name": {
                     "type": "string"
                 },
+                "onvif_motion": {
+                    "description": "ONVIFMotion routes the camera's ONVIF motion events into the\nagent's motion-triggered recording pipeline. When \"true\" the\nagent opens an event/stream against the configured ONVIF\nendpoint and forwards Motion+Active events to HandleMotion.\nRequires Capture.IPCamera.ONVIFXAddr / ONVIFUsername /\nONVIFPassword to be set. Default empty (disabled) keeps the\nexisting pixel-diff motion detection as the only source.",
+                    "type": "string"
+                },
                 "pixelChangeThreshold": {
                     "type": "integer"
                 },
@@ -816,9 +1035,15 @@
                 "kstorage": {
                     "$ref": "#/definitions/models.KStorage"
                 },
+                "kstorage_secondary": {
+                    "$ref": "#/definitions/models.KStorage"
+                },
                 "max_directory_size": {
                     "type": "integer"
                 },
+                "min_free_space": {
+                    "type": "integer"
+                },
                 "mqtt_password": {
                     "type": "string"
                 },
@@ -849,6 +1074,9 @@
                 "s3": {
                     "$ref": "#/definitions/models.S3"
                 },
+                "signing": {
+                    "$ref": "#/definitions/models.Signing"
+                },
                 "stunuri": {
                     "type": "string"
                 },
@@ -940,6 +1168,15 @@
         "models.IPCamera": {
             "type": "object",
             "properties": {
+                "base_height": {
+                    "type": "integer"
+                },
+                "base_width": {
+                    "type": "integer"
+                },
+                "channels": {
+                    "type": "integer"
+                },
                 "fps": {
                     "type": "string"
                 },
@@ -958,21 +1195,85 @@
                 "onvif_xaddr": {
                     "type": "string"
                 },
+                "pps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "rtsp": {
                     "type": "string"
                 },
+                "sample_rate": {
+                    "type": "integer"
+                },
+                "sps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_fps": {
                     "type": "string"
                 },
                 "sub_height": {
                     "type": "integer"
                 },
+                "sub_pps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_rtsp": {
                     "type": "string"
                 },
+                "sub_sps_nalus": {
+                    "description": "Sub stream parameter sets, captured separately from the main stream so the\nlive HLS muxer can build a correct init segment when a viewer switches the\nlive view to the sub (low-resolution) stream.",
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
+                "sub_vps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "sub_width": {
                     "type": "integer"
                 },
+                "vps_nalus": {
+                    "type": "array",
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "integer",
+                            "format": "int32"
+                        }
+                    }
+                },
                 "width": {
                     "type": "integer"
                 }
@@ -988,15 +1289,24 @@
                     "description": "old way, remove this",
                     "type": "string"
                 },
+                "custom_headers": {
+                    "type": "string"
+                },
                 "directory": {
                     "type": "string"
                 },
+                "max_retries": {
+                    "type": "integer"
+                },
                 "provider": {
                     "type": "string"
                 },
                 "secret_access_key": {
                     "type": "string"
                 },
+                "timeout": {
+                    "type": "integer"
+                },
                 "uri": {
                     "type": "string"
                 }
@@ -1134,6 +1444,17 @@
                 }
             }
         },
+        "models.Signing": {
+            "type": "object",
+            "properties": {
+                "enabled": {
+                    "type": "string"
+                },
+                "private_key": {
+                    "type": "string"
+                }
+            }
+        },
         "models.Timetable": {
             "type": "object",
             "properties": {

machinery/docs/swagger.yaml

@@ -1,5 +1,84 @@
 basePath: /
 definitions:
+  http.Health:
+    properties:
+      cameraConnected:
+        type: boolean
+      description:
+        type: string
+      hub:
+        $ref: '#/definitions/http.HubHealth'
+      mainStream:
+        $ref: '#/definitions/http.StreamHealth'
+      subStream:
+        $ref: '#/definitions/http.StreamHealth'
+    type: object
+  http.HealthResponse:
+    properties:
+      applicationStatusCode:
+        type: string
+      data:
+        $ref: '#/definitions/http.HealthResponseData'
+      entityStatusCode:
+        type: string
+      httpStatusCode:
+        type: integer
+      message:
+        type: string
+      metadata:
+        $ref: '#/definitions/http.ResponseMetadata'
+    type: object
+  http.HealthResponseData:
+    properties:
+      health:
+        $ref: '#/definitions/http.Health'
+    type: object
+  http.HubHealth:
+    properties:
+      configured:
+        type: boolean
+      connected:
+        type: boolean
+      lastHeartbeatAttemptAt:
+        type: integer
+      lastSuccessfulHeartbeatAt:
+        type: integer
+    type: object
+  http.ResponseMetadata:
+    properties:
+      applicationName:
+        type: string
+      applicationVersion:
+        type: string
+      path:
+        type: string
+      timestamp:
+        type: integer
+      traceId:
+        type: string
+    type: object
+  http.StreamHealth:
+    properties:
+      configured:
+        type: boolean
+      connected:
+        type: boolean
+      fps:
+        type: number
+      lastPacketAt:
+        type: integer
+      packagesProcessed:
+        type: integer
+      resolution:
+        $ref: '#/definitions/http.StreamResolution'
+    type: object
+  http.StreamResolution:
+    properties:
+      height:
+        type: integer
+      width:
+        type: integer
+    type: object
   models.APIResponse:
     properties:
       can_pan_tilt:
@@ -47,16 +126,31 @@ definitions:
         type: string
       fragmentedduration:
         type: integer
+      gopsize:
+        description: GOP size in seconds, used for pre-recording
+        type: integer
       ipcamera:
         $ref: '#/definitions/models.IPCamera'
       liveview:
         type: string
+      liveview_chunking:
+        type: string
       maxlengthrecording:
         type: integer
       motion:
         type: string
       name:
         type: string
+      onvif_motion:
+        description: |-
+          ONVIFMotion routes the camera's ONVIF motion events into the
+          agent's motion-triggered recording pipeline. When "true" the
+          agent opens an event/stream against the configured ONVIF
+          endpoint and forwards Motion+Active events to HandleMotion.
+          Requires Capture.IPCamera.ONVIFXAddr / ONVIFUsername /
+          ONVIFPassword to be set. Default empty (disabled) keeps the
+          existing pixel-diff motion detection as the only source.
+        type: string
       pixelChangeThreshold:
         type: integer
       postrecording:
@@ -109,8 +203,12 @@ definitions:
         type: string
       kstorage:
         $ref: '#/definitions/models.KStorage'
+      kstorage_secondary:
+        $ref: '#/definitions/models.KStorage'
       max_directory_size:
         type: integer
+      min_free_space:
+        type: integer
       mqtt_password:
         type: string
       mqtt_username:
@@ -131,6 +229,8 @@ definitions:
         type: string
       s3:
         $ref: '#/definitions/models.S3'
+      signing:
+        $ref: '#/definitions/models.Signing'
       stunuri:
         type: string
       time:
@@ -190,6 +290,12 @@ definitions:
     type: object
   models.IPCamera:
     properties:
+      base_height:
+        type: integer
+      base_width:
+        type: integer
+      channels:
+        type: integer
       fps:
         type: string
       height:
@@ -202,16 +308,64 @@ definitions:
         type: string
       onvif_xaddr:
         type: string
+      pps_nalus:
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
       rtsp:
         type: string
+      sample_rate:
+        type: integer
+      sps_nalus:
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
       sub_fps:
         type: string
       sub_height:
         type: integer
+      sub_pps_nalus:
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
       sub_rtsp:
         type: string
+      sub_sps_nalus:
+        description: |-
+          Sub stream parameter sets, captured separately from the main stream so the
+          live HLS muxer can build a correct init segment when a viewer switches the
+          live view to the sub (low-resolution) stream.
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
+      sub_vps_nalus:
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
       sub_width:
         type: integer
+      vps_nalus:
+        items:
+          items:
+            format: int32
+            type: integer
+          type: array
+        type: array
       width:
         type: integer
     type: object
@@ -224,10 +378,14 @@ definitions:
         type: string
       directory:
         type: string
+      max_retries:
+        type: integer
       provider:
         type: string
       secret_access_key:
         type: string
+      timeout:
+        type: integer
       uri:
         type: string
     type: object
@@ -316,6 +474,13 @@ definitions:
       username:
         type: string
     type: object
+  models.Signing:
+    properties:
+      enabled:
+        type: string
+      private_key:
+        type: string
+    type: object
   models.Timetable:
     properties:
       end1:
@@ -345,6 +510,30 @@ info:
   title: Swagger Kerberos Agent API
   version: "1.0"
 paths:
+  /api/camera/discover:
+    get:
+      description: Runs an advanced Fing/WiFiman-style scan (ONVIF WS-Discovery +
+        TCP port scan + MAC/vendor lookup) and returns the devices found on the local
+        network.
+      operationId: camera-discover
+      parameters:
+      - description: Discovery timeout in milliseconds (default 2000)
+        in: query
+        name: timeout
+        type: integer
+      - description: Optional subnet(s) to scan, e.g. '192.168.1.0/24' (comma-separated).
+          Defaults to the local interfaces.
+        in: query
+        name: subnet
+        type: string
+      responses:
+        "200":
+          description: OK
+          schema:
+            $ref: '#/definitions/models.APIResponse'
+      summary: Discover cameras and other devices on the local network.
+      tags:
+      - onvif
   /api/camera/onvif/capabilities:
     post:
       description: Will return the ONVIF capabilities for the specific camera.
@@ -735,6 +924,27 @@ paths:
       summary: Get Authorization token.
       tags:
       - authentication
+  /api/persistence/secondary/verify:
+    post:
+      description: Will verify the secondary persistence.
+      operationId: verify-secondary-persistence
+      parameters:
+      - description: Config
+        in: body
+        name: config
+        required: true
+        schema:
+          $ref: '#/definitions/models.Config'
+      responses:
+        "200":
+          description: OK
+          schema:
+            $ref: '#/definitions/models.APIResponse'
+      security:
+      - Bearer: []
+      summary: Will verify the secondary persistence.
+      tags:
+      - persistence
   /api/persistence/verify:
     post:
       description: Will verify the persistence.
@@ -756,6 +966,21 @@ paths:
       summary: Will verify the persistence.
       tags:
       - persistence
+  /health:
+    get:
+      description: Confirms that the Agent HTTP process can serve requests and reports
+        current camera stream and Hub heartbeat diagnostics. Operational dependency
+        failures do not change the liveness HTTP status.
+      produces:
+      - application/json
+      responses:
+        "200":
+          description: OK
+          schema:
+            $ref: '#/definitions/http.HealthResponse'
+      summary: Check Agent health
+      tags:
+      - health
 securityDefinitions:
   Bearer:
     in: header

machinery/go.mod

@@ -1,6 +1,6 @@
 module github.com/kerberos-io/agent/machinery
 
-go 1.24.2
+go 1.26.10
 
 replace google.golang.org/genproto => google.golang.org/genproto v0.0.0-20250519155744-55703ea1f237
 
@@ -8,12 +8,13 @@ require (
 	github.com/Eyevinn/mp4ff v0.48.0
 	github.com/InVisionApp/conjungo v1.1.0
 	github.com/appleboy/gin-jwt/v2 v2.10.3
-	github.com/bluenviron/gortsplib/v4 v4.14.1
+	github.com/bluenviron/gortsplib/v5 v5.6.3
 	github.com/bluenviron/mediacommon v1.14.0
 	github.com/cedricve/go-onvif v0.0.0-20200222191200-567e8ce298f6
+	github.com/creack/pty v1.1.24
 	github.com/dromara/carbon/v2 v2.6.8
 	github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5
-	github.com/eclipse/paho.mqtt.golang v1.5.0
+	github.com/eclipse/paho.mqtt.golang v1.5.1
 	github.com/elastic/go-sysinfo v1.15.3
 	github.com/gin-contrib/cors v1.7.5
 	github.com/gin-contrib/pprof v1.5.3
@@ -24,26 +25,27 @@ require (
 	github.com/gorilla/websocket v1.5.3
 	github.com/kellydunn/golang-geo v0.7.0
 	github.com/kerberos-io/joy4 v1.0.64
-	github.com/kerberos-io/onvif v1.0.0
+	github.com/kerberos-io/onvif v1.2.3
 	github.com/minio/minio-go/v6 v6.0.57
+	github.com/moq-dev/moq-go v0.5.7
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
-	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
-	github.com/pion/interceptor v0.1.40
-	github.com/pion/rtp v1.8.19
-	github.com/pion/webrtc/v4 v4.1.2
+	github.com/pion/interceptor v0.1.47
+	github.com/pion/rtp v1.10.5
+	github.com/pion/webrtc/v4 v4.2.18
 	github.com/sirupsen/logrus v1.9.3
+	github.com/stretchr/testify v1.11.1
 	github.com/swaggo/files v1.0.1
 	github.com/swaggo/gin-swagger v1.6.0
 	github.com/swaggo/swag v1.16.4
 	github.com/tevino/abool v1.2.0
 	github.com/zaf/g711 v1.4.0
-	go.mongodb.org/mongo-driver v1.17.3
-	go.opentelemetry.io/otel v1.36.0
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.36.0
-	go.opentelemetry.io/otel/sdk v1.36.0
-	go.opentelemetry.io/otel/trace v1.36.0
-	gopkg.in/natefinch/lumberjack.v2 v2.2.1
+	go.mongodb.org/mongo-driver v1.17.7
+	go.opentelemetry.io/otel v1.45.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0
+	go.opentelemetry.io/otel/sdk v1.45.0
+	go.opentelemetry.io/otel/trace v1.45.0
+	golang.org/x/net v0.62.0
 )
 
 require (
@@ -51,19 +53,21 @@ require (
 	github.com/PuerkitoBio/purell v1.1.1 // indirect
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/beevik/etree v1.2.0 // indirect
-	github.com/bluenviron/mediacommon/v2 v2.2.0 // indirect
+	github.com/bluenviron/mediacommon/v2 v2.9.2 // indirect
 	github.com/bytedance/sonic v1.13.2 // indirect
 	github.com/bytedance/sonic/loader v0.2.4 // indirect
-	github.com/cenkalti/backoff/v5 v5.0.2 // indirect
+	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/clbanning/mxj v1.8.4 // indirect
 	github.com/clbanning/mxj/v2 v2.7.0 // indirect
 	github.com/cloudwego/base64x v0.1.5 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/elastic/go-windows v1.0.2 // indirect
 	github.com/elgs/gostrgen v0.0.0-20161222160715-9d61ae07eeae // indirect
 	github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.8 // indirect
 	github.com/gin-contrib/sse v1.0.0 // indirect
-	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/logr v1.4.4 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.19.6 // indirect
@@ -75,12 +79,11 @@ require (
 	github.com/goccy/go-json v0.10.5 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/uuid v1.6.0 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect
-	github.com/icholy/digest v0.1.23 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/juju/errors v1.0.0 // indirect
-	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/klauspost/compress v1.18.7 // indirect
 	github.com/klauspost/cpuid v1.2.3 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.10 // indirect
 	github.com/kylelemons/go-gypsy v1.0.0 // indirect
@@ -94,21 +97,23 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/montanaflynn/stats v0.7.1 // indirect
+	github.com/moq-dev/moq-go-ffi v0.3.7 // indirect
 	github.com/nxadm/tail v1.4.11 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
-	github.com/pion/datachannel v1.5.10 // indirect
-	github.com/pion/dtls/v3 v3.0.6 // indirect
-	github.com/pion/ice/v4 v4.0.10 // indirect
-	github.com/pion/logging v0.2.3 // indirect
-	github.com/pion/mdns/v2 v2.0.7 // indirect
+	github.com/pion/datachannel v1.6.2 // indirect
+	github.com/pion/dtls/v3 v3.1.5 // indirect
+	github.com/pion/ice/v4 v4.4.0 // indirect
+	github.com/pion/logging v0.2.4 // indirect
+	github.com/pion/mdns/v2 v2.1.0 // indirect
 	github.com/pion/randutil v0.1.0 // indirect
-	github.com/pion/rtcp v1.2.15 // indirect
-	github.com/pion/sctp v1.8.39 // indirect
-	github.com/pion/sdp/v3 v3.0.13 // indirect
-	github.com/pion/srtp/v3 v3.0.5 // indirect
-	github.com/pion/stun/v3 v3.0.0 // indirect
-	github.com/pion/transport/v3 v3.0.7 // indirect
-	github.com/pion/turn/v4 v4.0.0 // indirect
+	github.com/pion/rtcp v1.2.17 // indirect
+	github.com/pion/sctp v1.11.1 // indirect
+	github.com/pion/sdp/v3 v3.0.19 // indirect
+	github.com/pion/srtp/v3 v3.0.12 // indirect
+	github.com/pion/stun/v3 v3.1.6 // indirect
+	github.com/pion/transport/v4 v4.0.2 // indirect
+	github.com/pion/turn/v5 v5.0.12 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.12 // indirect
@@ -118,21 +123,21 @@ require (
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
 	github.com/ziutek/mymysql v1.5.4 // indirect
-	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
-	go.opentelemetry.io/otel/metric v1.36.0 // indirect
-	go.opentelemetry.io/proto/otlp v1.6.0 // indirect
+	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+	go.opentelemetry.io/otel/metric v1.45.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.11.0 // indirect
 	golang.org/x/arch v0.16.0 // indirect
-	golang.org/x/crypto v0.38.0 // indirect
-	golang.org/x/net v0.40.0 // indirect
-	golang.org/x/oauth2 v0.30.0 // indirect
-	golang.org/x/sync v0.14.0 // indirect
-	golang.org/x/sys v0.33.0 // indirect
-	golang.org/x/text v0.25.0 // indirect
-	golang.org/x/tools v0.30.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20250519155744-55703ea1f237 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237 // indirect
-	google.golang.org/grpc v1.72.1 // indirect
-	google.golang.org/protobuf v1.36.6 // indirect
+	golang.org/x/crypto v0.58.1-0.20261009181718-2f1f834330b2 // indirect
+	golang.org/x/oauth2 v0.36.0 // indirect
+	golang.org/x/sync v0.24.0 // indirect
+	golang.org/x/sys v0.49.0 // indirect
+	golang.org/x/text v0.43.0 // indirect
+	golang.org/x/time v0.14.0 // indirect
+	golang.org/x/tools v0.51.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect
+	google.golang.org/grpc v1.83.2 // indirect
+	google.golang.org/protobuf v1.36.11 // indirect
 	gopkg.in/ini.v1 v1.42.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

machinery/go.sum

@@ -393,12 +393,12 @@ github.com/appleboy/gofight/v2 v2.1.2/go.mod h1:frW+U1QZEdDgixycTj4CygQ48yLTUhpl
 github.com/bazelbuild/rules_go v0.49.0/go.mod h1:Dhcz716Kqg1RHNWos+N6MlXNkjNP2EwZQ0LukRKJfMs=
 github.com/beevik/etree v1.2.0 h1:l7WETslUG/T+xOPs47dtd6jov2Ii/8/OjCldk5fYfQw=
 github.com/beevik/etree v1.2.0/go.mod h1:aiPf89g/1k3AShMVAzriilpcE4R/Vuor90y83zVZWFc=
-github.com/bluenviron/gortsplib/v4 v4.14.1 h1:v99NmXeeJFfbrO+ipPzPxYGibQaR5ZOUESOA9UQZhsI=
-github.com/bluenviron/gortsplib/v4 v4.14.1/go.mod h1:3LaEcg0d47+kfXju5KSlsSxCiZ3IKBI/sqIrBPcsS64=
+github.com/bluenviron/gortsplib/v5 v5.6.3 h1:OXvHthQZ9fZbLh6r3Go2wuF4XQ4/QW4WTIM2f4bv/W4=
+github.com/bluenviron/gortsplib/v5 v5.6.3/go.mod h1:kzHgUtvl8NWNsQ5Vsez6Vuugk6ItFT4ByCPm8J/kbSQ=
 github.com/bluenviron/mediacommon v1.14.0 h1:lWCwOBKNKgqmspRpwpvvg3CidYm+XOc2+z/Jw7LM5dQ=
 github.com/bluenviron/mediacommon v1.14.0/go.mod h1:z5LP9Tm1ZNfQV5Co54PyOzaIhGMusDfRKmh42nQSnyo=
-github.com/bluenviron/mediacommon/v2 v2.2.0 h1:fGXEX0OEvv5VhGHOv3Q2ABzOtSkIpl9UbwOHrnKWNTk=
-github.com/bluenviron/mediacommon/v2 v2.2.0/go.mod h1:a6MbPmXtYda9mKibKVMZlW20GYLLrX2R7ZkUE+1pwV0=
+github.com/bluenviron/mediacommon/v2 v2.9.2 h1:jvYeBjvhHKFOBRMTMm4hvrSjyOlCelkOkx6708DidQM=
+github.com/bluenviron/mediacommon/v2 v2.9.2/go.mod h1:jMf/OJDaJl02xRgkLM2zbidUHnDYqLnO1dMMveCmyyU=
 github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
 github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
 github.com/bytedance/sonic v1.13.2 h1:8/H1FempDZqC4VqjptGo14QQlJx8VdZJegxs6wwfqpQ=
@@ -410,12 +410,15 @@ github.com/cedricve/go-onvif v0.0.0-20200222191200-567e8ce298f6 h1:bzFZYgZD5vf4P
 github.com/cedricve/go-onvif v0.0.0-20200222191200-567e8ce298f6/go.mod h1:nBrjN2nMHendp0Cvb/6GaJ1v92Qv/kzqxWtNBnKJEK0=
 github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7Zss8=
 github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
+github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
+github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY=
@@ -456,6 +459,8 @@ github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78/go.mod h1:W+zGtBO5Y1Ig
 github.com/cncf/xds/go v0.0.0-20241223141626-cff3c89139a3/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
 github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
+github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -468,6 +473,8 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/eclipse/paho.mqtt.golang v1.5.0 h1:EH+bUVJNgttidWFkLLVKaQPGmkTUfQQqjOsyvMGvD6o=
 github.com/eclipse/paho.mqtt.golang v1.5.0/go.mod h1:du/2qNQVqJf/Sqs4MEL77kR8QTqANF7XU7Fk0aOTAgk=
+github.com/eclipse/paho.mqtt.golang v1.5.1 h1:/VSOv3oDLlpqR2Epjn1Q7b2bSTplJIeV2ISgCl2W7nE=
+github.com/eclipse/paho.mqtt.golang v1.5.1/go.mod h1:1/yJCneuyOoCOzKSsOTUc0AJfpsItBGWvYpBLimhArU=
 github.com/elastic/go-sysinfo v1.15.3 h1:W+RnmhKFkqPTCRoFq2VCTmsT4p/fwpo+3gKNQsn1XU0=
 github.com/elastic/go-sysinfo v1.15.3/go.mod h1:K/cNrqYTDrSoMh2oDkYEMS2+a72GRxMvNP+GC+vRIlo=
 github.com/elastic/go-windows v1.0.2 h1:yoLLsAsV5cfg9FLhZ9EXZ2n2sQFKeDYrHenkcivY4vI=
@@ -546,8 +553,11 @@ github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpx
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
-github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
 github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
+github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
+github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
@@ -746,6 +756,8 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs=
 github.com/hamba/avro/v2 v2.17.2/go.mod h1:Q9YK+qxAhtVrNqOhwlZTATLgLA8qxG2vtvkhK8fJ7Jo=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -755,8 +767,6 @@ github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
-github.com/icholy/digest v0.1.23 h1:4hX2pIloP0aDx7RJW0JewhPPy3R8kU+vWKdxPsCCGtY=
-github.com/icholy/digest v0.1.23/go.mod h1:QNrsSGQ5v7v9cReDI0+eyjsXGUoRSUZQHeQ5C4XLa0Y=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
@@ -776,13 +786,14 @@ github.com/kellydunn/golang-geo v0.7.0 h1:A5j0/BvNgGwY6Yb6inXQxzYwlPHc6WVZR+Mrar
 github.com/kellydunn/golang-geo v0.7.0/go.mod h1:YYlQPJ+DPEzrHx8kT3oPHC/NjyvCCXE+IuKGKdrjrcU=
 github.com/kerberos-io/joy4 v1.0.64 h1:gTUSotHSOhp9mNqEecgq88tQHvpj7TjmrvPUsPm0idg=
 github.com/kerberos-io/joy4 v1.0.64/go.mod h1:nZp4AjvKvTOXRrmDyAIOw+Da+JA5OcSo/JundGfOlFU=
-github.com/kerberos-io/onvif v1.0.0 h1:pLJrK6skPkK+5Bj4XfqHUkQ2I+p5pwELnp+kQTJWXiQ=
-github.com/kerberos-io/onvif v1.0.0/go.mod h1:P1kUcCfeotJSlL1jwGseH6NSnCwWiuJLl3gAzafnLbA=
+github.com/kerberos-io/onvif v1.2.3 h1:RDnDvLBd92VCgFOrlbGavzCVN9lQA/qs9novdNFIAOU=
+github.com/kerberos-io/onvif v1.2.3/go.mod h1:XSgEQXmEDjUQTbdXvsaRJt6Az8YPGj7L+j5iXKEGijU=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
-github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
 github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw=
+github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
 github.com/klauspost/cpuid v1.2.3 h1:CCtW0xUnWGVINKvE/WWOYKdsPV6mawAtvQuSl8guwQs=
 github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
@@ -847,6 +858,10 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
 github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
+github.com/moq-dev/moq-go v0.5.7 h1:LfFpgAU8FRMcnU85L5Lb03HzxhYbQ2+BhT/pJQdjy5U=
+github.com/moq-dev/moq-go v0.5.7/go.mod h1:5K8zjKKjWe5lzfCtlfxoAfdCr3KX6EBuTOZsz0WBnzw=
+github.com/moq-dev/moq-go-ffi v0.3.7 h1:+xwPOzTJHvB0tuTnW6znbvvOgT4yLnN7sMqXXZYh02M=
+github.com/moq-dev/moq-go-ffi v0.3.7/go.mod h1:zxpOlUetvaoxWBnbXTdILtLLUytaoVuLvO36lftwWO0=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
@@ -856,46 +871,46 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
 github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE=
 github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
-github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
-github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
 github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
 github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY=
 github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI=
 github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI=
 github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
-github.com/pion/datachannel v1.5.10 h1:ly0Q26K1i6ZkGf42W7D4hQYR90pZwzFOjTq5AuCKk4o=
-github.com/pion/datachannel v1.5.10/go.mod h1:p/jJfC9arb29W7WrxyKbepTU20CFgyx5oLo8Rs4Py/M=
-github.com/pion/dtls/v3 v3.0.6 h1:7Hkd8WhAJNbRgq9RgdNh1aaWlZlGpYTzdqjy9x9sK2E=
-github.com/pion/dtls/v3 v3.0.6/go.mod h1:iJxNQ3Uhn1NZWOMWlLxEEHAN5yX7GyPvvKw04v9bzYU=
-github.com/pion/ice/v4 v4.0.10 h1:P59w1iauC/wPk9PdY8Vjl4fOFL5B+USq1+xbDcN6gT4=
-github.com/pion/ice/v4 v4.0.10/go.mod h1:y3M18aPhIxLlcO/4dn9X8LzLLSma84cx6emMSu14FGw=
-github.com/pion/interceptor v0.1.40 h1:e0BjnPcGpr2CFQgKhrQisBU7V3GXK6wrfYrGYaU6Jq4=
-github.com/pion/interceptor v0.1.40/go.mod h1:Z6kqH7M/FYirg3frjGJ21VLSRJGBXB/KqaTIrdqnOic=
-github.com/pion/logging v0.2.3 h1:gHuf0zpoh1GW67Nr6Gj4cv5Z9ZscU7g/EaoC/Ke/igI=
-github.com/pion/logging v0.2.3/go.mod h1:z8YfknkquMe1csOrxK5kc+5/ZPAzMxbKLX5aXpbpC90=
-github.com/pion/mdns/v2 v2.0.7 h1:c9kM8ewCgjslaAmicYMFQIde2H9/lrZpjBkN8VwoVtM=
-github.com/pion/mdns/v2 v2.0.7/go.mod h1:vAdSYNAT0Jy3Ru0zl2YiW3Rm/fJCwIeM0nToenfOJKA=
+github.com/pion/datachannel v1.6.2 h1:7EXQ8TH3vTouBUdRWYbcX2edSx9Yj6k5zl5P+qyxEPc=
+github.com/pion/datachannel v1.6.2/go.mod h1:pzbdAZvyGtXbcHM1hBbsFaOTf40lZizU/dNlvVOak6E=
+github.com/pion/dtls/v3 v3.1.5 h1:9xJtVsHwMYeSjPp5Hh1FTis4DchnQWtnOa5o+6ygqfc=
+github.com/pion/dtls/v3 v3.1.5/go.mod h1:gz1K4jg6c+fq86oQMH4pilpCEOEPwmEr2jY+VcF/mkU=
+github.com/pion/ice/v4 v4.4.0 h1:wvHDDqimaC38Y7MVpD46Y63p246ChvXd87VKoLYS5b4=
+github.com/pion/ice/v4 v4.4.0/go.mod h1:obAyD+J+Hzs7QA7Y8YXHp5uIn6gb7z87pKedXZkrcFU=
+github.com/pion/interceptor v0.1.47 h1:yw8t5pJ2f8t78NgU+8EmxhaqYLXS7uFCC/tAGOaSDBo=
+github.com/pion/interceptor v0.1.47/go.mod h1:7yoRBzaIDETPC6cIN8Zj9EyGqHv1ImOpcTFPha6MuOM=
+github.com/pion/logging v0.2.4 h1:tTew+7cmQ+Mc1pTBLKH2puKsOvhm32dROumOZ655zB8=
+github.com/pion/logging v0.2.4/go.mod h1:DffhXTKYdNZU+KtJ5pyQDjvOAh/GsNSyv1lbkFbe3so=
+github.com/pion/mdns/v2 v2.1.0 h1:3IJ9+Xio6tWYjhN6WwuY142P/1jA0D5ERaIqawg/fOY=
+github.com/pion/mdns/v2 v2.1.0/go.mod h1:pcez23GdynwcfRU1977qKU0mDxSeucttSHbCSfFOd9A=
 github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA=
 github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8=
-github.com/pion/rtcp v1.2.15 h1:LZQi2JbdipLOj4eBjK4wlVoQWfrZbh3Q6eHtWtJBZBo=
-github.com/pion/rtcp v1.2.15/go.mod h1:jlGuAjHMEXwMUHK78RgX0UmEJFV4zUKOFHR7OP+D3D0=
-github.com/pion/rtp v1.8.19 h1:jhdO/3XhL/aKm/wARFVmvTfq0lC/CvN1xwYKmduly3c=
-github.com/pion/rtp v1.8.19/go.mod h1:bAu2UFKScgzyFqvUKmbvzSdPr+NGbZtv6UB2hesqXBk=
-github.com/pion/sctp v1.8.39 h1:PJma40vRHa3UTO3C4MyeJDQ+KIobVYRZQZ0Nt7SjQnE=
-github.com/pion/sctp v1.8.39/go.mod h1:cNiLdchXra8fHQwmIoqw0MbLLMs+f7uQ+dGMG2gWebE=
-github.com/pion/sdp/v3 v3.0.13 h1:uN3SS2b+QDZnWXgdr69SM8KB4EbcnPnPf2Laxhty/l4=
-github.com/pion/sdp/v3 v3.0.13/go.mod h1:88GMahN5xnScv1hIMTqLdu/cOcUkj6a9ytbncwMCq2E=
-github.com/pion/srtp/v3 v3.0.5 h1:8XLB6Dt3QXkMkRFpoqC3314BemkpMQK2mZeJc4pUKqo=
-github.com/pion/srtp/v3 v3.0.5/go.mod h1:r1G7y5r1scZRLe2QJI/is+/O83W2d+JoEsuIexpw+uM=
-github.com/pion/stun/v3 v3.0.0 h1:4h1gwhWLWuZWOJIJR9s2ferRO+W3zA/b6ijOI6mKzUw=
-github.com/pion/stun/v3 v3.0.0/go.mod h1:HvCN8txt8mwi4FBvS3EmDghW6aQJ24T+y+1TKjB5jyU=
-github.com/pion/transport/v3 v3.0.7 h1:iRbMH05BzSNwhILHoBoAPxoB9xQgOaJk+591KC9P1o0=
-github.com/pion/transport/v3 v3.0.7/go.mod h1:YleKiTZ4vqNxVwh77Z0zytYi7rXHl7j6uPLGhhz9rwo=
-github.com/pion/turn/v4 v4.0.0 h1:qxplo3Rxa9Yg1xXDxxH8xaqcyGUtbHYw4QSCvmFWvhM=
-github.com/pion/turn/v4 v4.0.0/go.mod h1:MuPDkm15nYSklKpN8vWJ9W2M0PlyQZqYt1McGuxG7mA=
-github.com/pion/webrtc/v4 v4.1.2 h1:mpuUo/EJ1zMNKGE79fAdYNFZBX790KE7kQQpLMjjR54=
-github.com/pion/webrtc/v4 v4.1.2/go.mod h1:xsCXiNAmMEjIdFxAYU0MbB3RwRieJsegSB2JZsGN+8U=
+github.com/pion/rtcp v1.2.17 h1:PxiT6L79yPZKtXIsXdG1eakBl6dtBj4x+4oVEL0DlSw=
+github.com/pion/rtcp v1.2.17/go.mod h1:7kBpuBJaWwax4hzc/pgexY8vkOpvh8atgYDbaKZq0iU=
+github.com/pion/rtp v1.10.5 h1:ip0HhO/wYZqQ4bKS+R99KnZh/GRCmIT0jDXikub7vlE=
+github.com/pion/rtp v1.10.5/go.mod h1:Au8fc6cEByy8RLTwKTQTEeQqDB/SJDxwL4mZuxYA5Pk=
+github.com/pion/sctp v1.11.1 h1:O4dIFyURw1KTST7w+gtD4gLeYXkhPa0xXLHMMoe/OSA=
+github.com/pion/sctp v1.11.1/go.mod h1:7KFmTwLcoYgJs/Z+99nJvsWL0qDpuyloSI0RbAqlrz0=
+github.com/pion/sdp/v3 v3.0.19 h1:1VMKs3gIkTQV5M3hNKfTAPrDXSNrYtOlmOD8+mSZUGQ=
+github.com/pion/sdp/v3 v3.0.19/go.mod h1:dE5WOSlzXrtiE/iuZqe9n+AcEbOjtAd3k5m5NtlV/qU=
+github.com/pion/srtp/v3 v3.0.12 h1:U7V17bckl7sI4mb3sepiojByDuBY0wNCqQE+6IlQBbc=
+github.com/pion/srtp/v3 v3.0.12/go.mod h1:EeZOi/sd6glM1EXapg051gdNWO9yWT1YSsgQ4SlJkns=
+github.com/pion/stun/v3 v3.1.6 h1:WnhsD0eHCiwCfKNkVx0VJJwr2Y3eV4Ueih3KJ+dfZy8=
+github.com/pion/stun/v3 v3.1.6/go.mod h1:zRUghXSQU32Lx5orJsz3uYMkIihweXb3mu5gIns02fs=
+github.com/pion/transport/v3 v3.1.1 h1:Tr684+fnnKlhPceU+ICdrw6KKkTms+5qHMgw6bIkYOM=
+github.com/pion/transport/v3 v3.1.1/go.mod h1:+c2eewC5WJQHiAA46fkMMzoYZSuGzA/7E2FPrOYHctQ=
+github.com/pion/transport/v4 v4.0.2 h1:ifYlPqNwsy6aKQ9y8yzxXlHae5431ZrH2avkD/Rn6Tk=
+github.com/pion/transport/v4 v4.0.2/go.mod h1:06hFI+jCFcok2X2MekVufNZ/uzNZXivGBPfviSVcjgM=
+github.com/pion/turn/v5 v5.0.12 h1:6+b69ivQQXSlyfkp2AKripqD2k3W32qXK8QzCzpJWPI=
+github.com/pion/turn/v5 v5.0.12/go.mod h1:CQACsRDJtjQ+6RSrGHrS2PCIerLwbW3uqXRqOvtjAFg=
+github.com/pion/webrtc/v4 v4.2.18 h1:smA/3g6Gy4RohM0VIZ5KKY/12TQbxv3XFgpUMyb2EUI=
+github.com/pion/webrtc/v4 v4.2.18/go.mod h1:vmzi6s+rvhoIuT94DPqivB+0xJXs9rG4QRD+4MgBtlY=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -920,8 +935,9 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
-github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
 github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
+github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
+github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
 github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w=
 github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk=
 github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo=
@@ -955,8 +971,9 @@ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o
 github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 github.com/substrait-io/substrait-go v0.4.2/go.mod h1:qhpnLmrcvAnlZsUyPXZRqldiHapPTXC3t7xFgDi3aQg=
 github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE=
 github.com/swaggo/files v1.0.1/go.mod h1:0qXmMNH6sXNf+73t65aKeB+ApmgxdnkQzVTAj2uaMUg=
@@ -1006,6 +1023,8 @@ go.einride.tech/aip v0.67.1/go.mod h1:ZGX4/zKw8dcgzdLsrvpOOGxfxI2QSk12SlP7d6c0/X
 go.einride.tech/aip v0.68.1/go.mod h1:XaFtaj4HuA3Zwk9xoBtTWgNubZ0ZZXv9BZJCkuKuWbg=
 go.mongodb.org/mongo-driver v1.17.3 h1:TQyXhnsWfWtgAhMtOgtYHMTkZIfBTpMTsMnd9ZBeHxQ=
 go.mongodb.org/mongo-driver v1.17.3/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ=
+go.mongodb.org/mongo-driver v1.17.7 h1:a9w+U3Vt67eYzcfq3k/OAv284/uUUkL0uP75VE5rCOU=
+go.mongodb.org/mongo-driver v1.17.7/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -1014,8 +1033,9 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
-go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
 go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
+go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
+go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
 go.opentelemetry.io/contrib/detectors/gcp v1.28.0/go.mod h1:9BIqH22qyHWAiZxQh0whuJygro59z+nbMVuc7ciiGug=
 go.opentelemetry.io/contrib/detectors/gcp v1.29.0/go.mod h1:GW2aWZNwR2ZxDLdv8OyC2G8zkRoQBuURgV7RPQgcPoU=
 go.opentelemetry.io/contrib/detectors/gcp v1.31.0/go.mod h1:tzQL6E1l+iV44YFTkcAeNQqzXUiekSYP9jjJjXwEd00=
@@ -1054,12 +1074,18 @@ go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGi
 go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I=
 go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI=
 go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y=
-go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg=
-go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E=
+go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
+go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
+go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU=
+go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 h1:dNzwXjZKpMpE2JhmO+9HsPl42NIXFIFSUSSs0fiqra0=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0/go.mod h1:90PoxvaEB5n6AOdZvi+yWJQoE95U8Dhhw2bSyRqnTD0=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 h1:QRefszxJmfPdjXUUm3j6iDzY03mTPXMjqErFqQ67vUg=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0/go.mod h1:Tiz03lTBVBrm7eWZBOidzEaYaJa8tjwGUGv6d8mlTyk=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.36.0 h1:nRVXXvf78e00EwY6Wp0YII8ww2JVWshZ20HfTlE11AM=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.36.0/go.mod h1:r49hO7CgrxY9Voaj3Xe8pANWtr0Oq916d0XAmOoCZAQ=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0 h1:QBajQ2SrwQijzHyZbQlPsuIzpl/ll8DY6wPWsajeGcI=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0/go.mod h1:08ZQLjrPLQ6R4kAXvuOvODEer5Yh4CoFvll5qB2BCI8=
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0/go.mod h1:BLbf7zbNIONBLPwvFnwNHGj4zge8uTCM/UPIVW1Mq2I=
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0/go.mod h1:U2R3XyVPzn0WX7wOIypPuptulsMcPDPs/oiSVOMVnHY=
 go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM=
@@ -1075,8 +1101,10 @@ go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzau
 go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M=
 go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE=
 go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE=
-go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE=
-go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs=
+go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
+go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
+go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M=
+go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s=
 go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E=
 go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc=
 go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg=
@@ -1088,16 +1116,20 @@ go.opentelemetry.io/otel/sdk v1.32.0/go.mod h1:LqgegDBjKMmb2GC6/PrTnteJG39I8/vJC
 go.opentelemetry.io/otel/sdk v1.33.0/go.mod h1:A1Q5oi7/9XaMlIWzPSxLRWOI8nG3FnzHJNbiENQuihM=
 go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU=
 go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg=
-go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs=
-go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY=
+go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58=
+go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=
+go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw=
+go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA=
 go.opentelemetry.io/otel/sdk/metric v1.28.0/go.mod h1:cWPjykihLAPvXKi4iZc1dpER3Jdq2Z0YLse3moQUCpg=
 go.opentelemetry.io/otel/sdk/metric v1.29.0/go.mod h1:6zZLdCl2fkauYoZIOn/soQIDSWFmNSRcICarHfuhNJQ=
 go.opentelemetry.io/otel/sdk/metric v1.30.0/go.mod h1:waS6P3YqFNzeP01kuo/MBBYqaoBJl7efRQHOaydhy1Y=
 go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8=
 go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ=
 go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w=
-go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o=
 go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w=
+go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI=
+go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=
+go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o=
 go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ=
 go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo=
 go.opentelemetry.io/otel/trace v1.23.0/go.mod h1:GSGTbIClEsuZrGIzoEHqsVfxgn5UkggkflQwDScNUsk=
@@ -1111,14 +1143,18 @@ go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L
 go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck=
 go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE=
 go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc=
-go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w=
-go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA=
+go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
+go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
+go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag=
+go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
 go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
 go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
 go.opentelemetry.io/proto/otlp v1.6.0 h1:jQjP+AQyTf+Fe7OKj/MfkDrmK4MNVtw2NpXsf9fefDI=
 go.opentelemetry.io/proto/otlp v1.6.0/go.mod h1:cicgGehlFuNdgZkcALOCh3VE6K/u2tAjzlRhDwmVpZc=
+go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
+go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 golang.org/x/arch v0.16.0 h1:foMtLTdyOmIniqWCHjY6+JxuC54XP1fDwx4N0ASyW+U=
@@ -1164,8 +1200,10 @@ golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5
 golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ=
 golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=
 golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
-golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8=
-golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw=
+golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
+golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
+golang.org/x/crypto v0.58.1-0.20261009181718-2f1f834330b2 h1:zGDaB+/Mus2mScKvZ23k+W9NAUbP56sNM9SptgvWI5A=
+golang.org/x/crypto v0.58.1-0.20261009181718-2f1f834330b2/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -1234,8 +1272,9 @@ golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM=
 golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
+golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
+golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1323,8 +1362,8 @@ golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
 golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk=
 golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
 golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E=
-golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY=
-golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds=
+golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY=
+golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1374,8 +1413,9 @@ golang.org/x/oauth2 v0.26.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbht
 golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8=
 golang.org/x/oauth2 v0.28.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8=
 golang.org/x/oauth2 v0.29.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8=
-golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=
 golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU=
+golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
+golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -1403,8 +1443,9 @@ golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
-golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ=
 golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=
+golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1513,8 +1554,8 @@ golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
-golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
-golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
+golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
 golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
 golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -1581,8 +1622,8 @@ golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
 golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
 golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
 golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
-golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
-golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
+golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc=
+golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1594,6 +1635,8 @@ golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
+golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
+golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -1667,8 +1710,9 @@ golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk
 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
 golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c=
 golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI=
-golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY=
 golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY=
+golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI=
+golang.org/x/tools v0.51.0/go.mod h1:9eEncMayCV6zRMGhR5eZEC2iBx98qWcF1HZ9Z7wJOoA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1683,6 +1727,8 @@ gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJ
 gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0=
 gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0=
 gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY=
+gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
+gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
 gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
 gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc=
 gonum.org/v1/plot v0.9.0/go.mod h1:3Pcqqmp6RHvJI72kgb8fThyUnav364FOsdDo2aGW5lY=
@@ -1856,8 +1902,10 @@ google.golang.org/genproto/googleapis/api v0.0.0-20250425173222-7b384671a197/go.
 google.golang.org/genproto/googleapis/api v0.0.0-20250428153025-10db94c68c34/go.mod h1:0awUlEkap+Pb1UMeJwJQQAdJQrt3moU7J2moTy69irI=
 google.golang.org/genproto/googleapis/api v0.0.0-20250505200425-f936aa4a68b2/go.mod h1:pKLAc5OolXC3ViWGI62vvC0n10CpwAtRcTNCFwTKBEw=
 google.golang.org/genproto/googleapis/api v0.0.0-20250512202823-5a2f75b736a9/go.mod h1:W3S/3np0/dPWsWLi1h/UymYctGXaGBM2StwzD0y140U=
-google.golang.org/genproto/googleapis/api v0.0.0-20250519155744-55703ea1f237 h1:Kog3KlB4xevJlAcbbbzPfRG0+X9fdoGM+UBRKVz6Wr0=
-google.golang.org/genproto/googleapis/api v0.0.0-20250519155744-55703ea1f237/go.mod h1:ezi0AVyMKDWy5xAncvjLWH7UcLBB5n7y2fQ8MzjJcto=
+google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8=
+google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY=
+google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d h1:FarXi840EJWSHYTN3ERkADbPWjl307+FGrA22KAVjjc=
+google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d/go.mod h1:K/+WGbmBY7aNW1HDw1fJnKYo10i0DkAX6pows00dLig=
 google.golang.org/genproto/googleapis/bytestream v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:ylj+BE99M198VPbBh6A8d9n3w8fChvyLK3wwBOjXBFA=
 google.golang.org/genproto/googleapis/bytestream v0.0.0-20231012201019-e917dd12ba7a/go.mod h1:+34luvCflYKiKylNwGJfn9cFBbcL/WrkciMmDmsTQ/A=
 google.golang.org/genproto/googleapis/bytestream v0.0.0-20231030173426-d783a09b4405/go.mod h1:GRUCuLdzVqZte8+Dl/D4N25yLzcGqqWaYkeVOwulFqw=
@@ -1976,8 +2024,10 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20250425173222-7b384671a197/go.
 google.golang.org/genproto/googleapis/rpc v0.0.0-20250428153025-10db94c68c34/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20250505200425-f936aa4a68b2/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20250512202823-5a2f75b736a9/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237 h1:cJfm9zPbe1e873mHJzmQ1nwVEeRDU/T1wXDK2kUSU34=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -2052,8 +2102,8 @@ google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40Rmc
 google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec=
 google.golang.org/grpc v1.71.1/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec=
 google.golang.org/grpc v1.72.0/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM=
-google.golang.org/grpc v1.72.1 h1:HR03wO6eyZ7lknl75XlxABNVLLFc2PAb6mHlYh756mA=
-google.golang.org/grpc v1.72.1/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM=
+google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU=
+google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0/go.mod h1:Dk1tviKTvMCz5tvh7t+fh94dhmQVHuCt2OzJB3CTW9Y=
 google.golang.org/grpc/examples v0.0.0-20230224211313-3775f633ce20/go.mod h1:Nr5H8+MlGWr5+xX/STzdoEqJrO+YteqFbMyCsrb6mH0=
@@ -2086,8 +2136,9 @@ google.golang.org/protobuf v1.36.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojt
 google.golang.org/protobuf v1.36.3/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
 google.golang.org/protobuf v1.36.4/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
 google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
-google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
 google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
+google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
+google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -2096,8 +2147,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk=
 gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
-gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
@@ -2109,7 +2158,6 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU=
 gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

machinery/healthcheck.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+)
+
+func checkAgentHealth(ctx context.Context, port string) error {
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://127.0.0.1:"+port+"/health", nil)
+	if err != nil {
+		return fmt.Errorf("creating health request: %w", err)
+	}
+	client := &http.Client{
+		Timeout:   5 * time.Second,
+		Transport: &http.Transport{DisableKeepAlives: true},
+		CheckRedirect: func(*http.Request, []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
+	defer client.CloseIdleConnections()
+	response, err := client.Do(request)
+	if err != nil {
+		return fmt.Errorf("requesting Agent health: %w", err)
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		return fmt.Errorf("Agent health returned HTTP %d", response.StatusCode)
+	}
+	var health struct {
+		HTTPStatusCode   int    `json:"httpStatusCode"`
+		EntityStatusCode string `json:"entityStatusCode"`
+		Metadata         struct {
+			ApplicationName string `json:"applicationName"`
+		} `json:"metadata"`
+	}
+	if err := json.NewDecoder(io.LimitReader(response.Body, 64*1024)).Decode(&health); err != nil {
+		return fmt.Errorf("decoding Agent health: %w", err)
+	}
+	if health.HTTPStatusCode != http.StatusOK || health.EntityStatusCode != "healthy" || health.Metadata.ApplicationName != "agent" {
+		return fmt.Errorf("Agent health returned an unexpected response")
+	}
+	return nil
+}

machinery/healthcheck_test.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+)
+
+func TestCheckAgentHealth(t *testing.T) {
+	tests := []struct {
+		name   string
+		status int
+		body   string
+		wantOK bool
+	}{
+		{"healthy", 200, `{"httpStatusCode":200,"entityStatusCode":"healthy","metadata":{"applicationName":"agent"}}`, true},
+		{"unhealthy status", 503, `{}`, false},
+		{"redirect", 302, `{}`, false},
+		{"invalid JSON", 200, `not JSON`, false},
+		{"wrong service", 200, `{"httpStatusCode":200,"entityStatusCode":"healthy","metadata":{"applicationName":"other"}}`, false},
+		{"unhealthy body", 200, `{"httpStatusCode":200,"entityStatusCode":"unhealthy","metadata":{"applicationName":"agent"}}`, false},
+	}
+	t.Setenv("HTTP_PROXY", "http://127.0.0.1:1")
+	t.Setenv("NO_PROXY", "")
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				if r.URL.Path != "/health" {
+					t.Errorf("path = %q, want /health", r.URL.Path)
+				}
+				w.Header().Set("Location", "/unexpected-redirect-target")
+				w.WriteHeader(test.status)
+				_, _ = w.Write([]byte(test.body))
+			}))
+			defer server.Close()
+			_, port, err := net.SplitHostPort(server.Listener.Addr().String())
+			if err != nil {
+				t.Fatal(err)
+			}
+			err = checkAgentHealth(context.Background(), port)
+			if (err == nil) != test.wantOK {
+				t.Fatalf("checkAgentHealth() = %v, wantOK %t", err, test.wantOK)
+			}
+		})
+	}
+}
+
+func TestCheckAgentHealthTimesOut(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		<-r.Context().Done()
+	}))
+	defer server.Close()
+	_, port, err := net.SplitHostPort(server.Listener.Addr().String())
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 25*time.Millisecond)
+	defer cancel()
+	if err := checkAgentHealth(ctx, port); !errors.Is(err, context.DeadlineExceeded) {
+		t.Fatalf("healthcheck error = %v, want context deadline exceeded", err)
+	}
+}

machinery/logging.go

@@ -0,0 +1,160 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	defaultLogLevel  = log.InfoLevel
+	defaultLogOutput = "text"
+)
+
+type localTimeFormatter struct {
+	timezone  *time.Location
+	formatter log.Formatter
+}
+
+func (f localTimeFormatter) Format(entry *log.Entry) ([]byte, error) {
+	entry.Time = entry.Time.In(f.timezone)
+	return f.formatter.Format(entry)
+}
+
+type componentHook struct{}
+
+func (componentHook) Levels() []log.Level {
+	return log.AllLevels
+}
+
+func (componentHook) Fire(entry *log.Entry) error {
+	if _, exists := entry.Data["component"]; exists {
+		return nil
+	}
+	entry.Data["component"] = componentFromCaller(entry.Caller)
+	return nil
+}
+
+func configureLogging(levelValue string, outputValue string, timezone *time.Location) {
+	configureLogger(log.StandardLogger(), levelValue, outputValue, timezone)
+}
+
+func configureLogger(logger *log.Logger, levelValue string, outputValue string, timezone *time.Location) {
+	if timezone == nil {
+		timezone = time.Local
+	}
+
+	level, levelErr := parseLogLevel(levelValue)
+	output, outputErr := parseLogOutput(outputValue)
+
+	logger.SetOutput(os.Stdout)
+	logger.SetLevel(level)
+	logger.SetReportCaller(true)
+	logger.SetFormatter(localTimeFormatter{
+		timezone:  timezone,
+		formatter: newLogFormatter(output),
+	})
+	installComponentHook(logger)
+
+	if levelErr != nil {
+		logger.WithFields(log.Fields{
+			"configured_level": levelValue,
+			"effective_level":  level.String(),
+		}).WithError(levelErr).Warn("invalid log level; using default")
+	}
+	if outputErr != nil {
+		logger.WithFields(log.Fields{
+			"configured_output": outputValue,
+			"effective_output":  output,
+		}).WithError(outputErr).Warn("invalid log output; using default")
+	}
+
+	logger.WithFields(log.Fields{
+		"event":         "logger_configured",
+		"log_level":     level.String(),
+		"output":        output,
+		"report_caller": logger.ReportCaller,
+		"timezone":      timezone.String(),
+	}).Debug("logging configured")
+}
+
+func installComponentHook(logger *log.Logger) {
+	for _, hooks := range logger.Hooks {
+		for _, hook := range hooks {
+			if _, ok := hook.(componentHook); ok {
+				return
+			}
+		}
+	}
+	logger.AddHook(componentHook{})
+}
+
+func componentFromCaller(frame *runtime.Frame) string {
+	if frame == nil {
+		return "unknown"
+	}
+
+	const sourceMarker = "/machinery/src/"
+	normalizedFile := filepath.ToSlash(frame.File)
+	if markerIndex := strings.Index(normalizedFile, sourceMarker); markerIndex >= 0 {
+		relativeFile := normalizedFile[markerIndex+len(sourceMarker):]
+		if directory := filepath.ToSlash(filepath.Dir(relativeFile)); directory != "." {
+			return directory
+		}
+	}
+	if strings.Contains(normalizedFile, "/machinery/") {
+		return "agent"
+	}
+	return "unknown"
+}
+
+func parseLogLevel(value string) (log.Level, error) {
+	normalized := strings.ToLower(strings.TrimSpace(value))
+	if normalized == "" {
+		return defaultLogLevel, nil
+	}
+	if normalized == "warning" {
+		normalized = "warn"
+	}
+
+	level, err := log.ParseLevel(normalized)
+	if err != nil {
+		return defaultLogLevel, fmt.Errorf("parse LOG_LEVEL: %w", err)
+	}
+	return level, nil
+}
+
+func parseLogOutput(value string) (string, error) {
+	normalized := strings.ToLower(strings.TrimSpace(value))
+	if normalized == "" {
+		return defaultLogOutput, nil
+	}
+	switch normalized {
+	case "json", "text":
+		return normalized, nil
+	default:
+		return defaultLogOutput, fmt.Errorf("unsupported LOG_OUTPUT %q", value)
+	}
+}
+
+func newLogFormatter(output string) log.Formatter {
+	callerPrettyfier := func(frame *runtime.Frame) (string, string) {
+		return filepath.Base(frame.Function), fmt.Sprintf("%s:%d", filepath.Base(frame.File), frame.Line)
+	}
+	if output == "json" {
+		return &log.JSONFormatter{
+			CallerPrettyfier: callerPrettyfier,
+			TimestampFormat:  time.RFC3339Nano,
+		}
+	}
+	return &log.TextFormatter{
+		CallerPrettyfier: callerPrettyfier,
+		FullTimestamp:    true,
+		TimestampFormat:  time.RFC3339Nano,
+	}
+}

machinery/logging_test.go

@@ -0,0 +1,165 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func TestConfigureLoggingDebugIncludesStructuredContext(t *testing.T) {
+	logger := log.New()
+	var output bytes.Buffer
+	configureLogger(logger, "debug", "json", time.UTC)
+	logger.SetOutput(&output)
+	logger.WithField("event", "test_event").Debug("structured debug test")
+
+	lines := strings.Split(strings.TrimSpace(output.String()), "\n")
+	var entry map[string]interface{}
+	if err := json.Unmarshal([]byte(lines[len(lines)-1]), &entry); err != nil {
+		t.Fatalf("decode debug log: %v; output=%q", err, output.String())
+	}
+	for key, want := range map[string]interface{}{
+		"component": "agent",
+		"event":     "test_event",
+		"level":     "debug",
+		"msg":       "structured debug test",
+	} {
+		if got := entry[key]; got != want {
+			t.Fatalf("%s = %v, want %v", key, got, want)
+		}
+	}
+	if entry["file"] == nil || entry["func"] == nil {
+		t.Fatalf("debug log is missing caller metadata: %v", entry)
+	}
+}
+
+func TestConfigureLoggerInstallsComponentHookOnce(t *testing.T) {
+	logger := log.New()
+	configureLogger(logger, "info", "text", time.UTC)
+	configureLogger(logger, "debug", "json", time.UTC)
+
+	var componentHooks int
+	for _, hooks := range logger.Hooks {
+		for _, hook := range hooks {
+			if _, ok := hook.(componentHook); ok {
+				componentHooks++
+			}
+		}
+	}
+	if componentHooks != len(log.AllLevels) {
+		t.Fatalf("component hook registrations = %d, want %d", componentHooks, len(log.AllLevels))
+	}
+}
+
+func TestComponentFromCaller(t *testing.T) {
+	tests := []struct {
+		name  string
+		frame *runtime.Frame
+		want  string
+	}{
+		{
+			name:  "nested runtime package",
+			frame: &runtime.Frame{File: "/workspace/agent/machinery/src/routers/mqtt/main.go"},
+			want:  "routers/mqtt",
+		},
+		{
+			name:  "top-level runtime package",
+			frame: &runtime.Frame{File: "/workspace/agent/machinery/src/capture/main.go"},
+			want:  "capture",
+		},
+		{
+			name:  "executable",
+			frame: &runtime.Frame{File: "/workspace/agent/machinery/main.go"},
+			want:  "agent",
+		},
+		{name: "missing caller", want: "unknown"},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if got := componentFromCaller(test.frame); got != test.want {
+				t.Fatalf("componentFromCaller() = %q, want %q", got, test.want)
+			}
+		})
+	}
+}
+
+func TestComponentHookAddsAndPreservesComponent(t *testing.T) {
+	hook := componentHook{}
+
+	entry := log.NewEntry(log.New())
+	entry.Caller = &runtime.Frame{File: "/workspace/agent/machinery/src/cloud/livehls/session.go"}
+	if err := hook.Fire(entry); err != nil {
+		t.Fatalf("componentHook.Fire() error = %v", err)
+	}
+	if got := entry.Data["component"]; got != "cloud/livehls" {
+		t.Fatalf("component = %v, want cloud/livehls", got)
+	}
+
+	entry.Data["component"] = "explicit"
+	if err := hook.Fire(entry); err != nil {
+		t.Fatalf("componentHook.Fire() preserving field error = %v", err)
+	}
+	if got := entry.Data["component"]; got != "explicit" {
+		t.Fatalf("component = %v, want explicit", got)
+	}
+}
+
+func TestParseLogLevel(t *testing.T) {
+	tests := []struct {
+		name    string
+		value   string
+		want    log.Level
+		wantErr bool
+	}{
+		{name: "default", want: log.InfoLevel},
+		{name: "info", value: "INFO", want: log.InfoLevel},
+		{name: "warning alias", value: "warning", want: log.WarnLevel},
+		{name: "debug", value: "debug", want: log.DebugLevel},
+		{name: "trace", value: "trace", want: log.TraceLevel},
+		{name: "invalid", value: "verbose", want: log.InfoLevel, wantErr: true},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got, err := parseLogLevel(test.value)
+			if (err != nil) != test.wantErr {
+				t.Fatalf("parseLogLevel(%q) error = %v, wantErr %t", test.value, err, test.wantErr)
+			}
+			if got != test.want {
+				t.Fatalf("parseLogLevel(%q) = %s, want %s", test.value, got, test.want)
+			}
+		})
+	}
+}
+
+func TestParseLogOutput(t *testing.T) {
+	tests := []struct {
+		name    string
+		value   string
+		want    string
+		wantErr bool
+	}{
+		{name: "default", want: "text"},
+		{name: "text", value: "TEXT", want: "text"},
+		{name: "json", value: "json", want: "json"},
+		{name: "invalid", value: "console", want: "text", wantErr: true},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got, err := parseLogOutput(test.value)
+			if (err != nil) != test.wantErr {
+				t.Fatalf("parseLogOutput(%q) error = %v, wantErr %t", test.value, err, test.wantErr)
+			}
+			if got != test.want {
+				t.Fatalf("parseLogOutput(%q) = %q, want %q", test.value, got, test.want)
+			}
+		})
+	}
+}

machinery/main.go

@@ -5,13 +5,15 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"strconv"
+	"strings"
 	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/capture"
 	"github.com/kerberos-io/agent/machinery/src/components"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/onvif"
+	log "github.com/sirupsen/logrus"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
@@ -27,6 +29,22 @@ import (
 
 var VERSION = utils.VERSION
 
+func resolveServerPort(flagValue, environmentValue string) (string, error) {
+	value := strings.TrimSpace(environmentValue)
+	if value == "" {
+		value = strings.TrimSpace(flagValue)
+	}
+	if value == "" {
+		value = "80"
+	}
+
+	port, err := strconv.Atoi(value)
+	if err != nil || port < 1 || port > 65535 {
+		return "", fmt.Errorf("port must be an integer between 1 and 65535, got %q", value)
+	}
+	return strconv.Itoa(port), nil
+}
+
 func startTracing(agentKey string, otelEndpoint string) (*trace.TracerProvider, error) {
 	serviceName := "agent-" + agentKey
 	headers := map[string]string{
@@ -76,12 +94,14 @@ func main() {
 	var name string
 	var port string
 	var timeout string
+	var subnet string
 
-	flag.StringVar(&action, "action", "version", "Tell us what you want do 'run' or 'version'")
+	flag.StringVar(&action, "action", "version", "Action: run, version, healthcheck, discover, or decrypt")
 	flag.StringVar(&configDirectory, "config", ".", "Where is the configuration stored")
 	flag.StringVar(&name, "name", "agent", "Provide a name for the agent")
 	flag.StringVar(&port, "port", "80", "On which port should the agent run")
 	flag.StringVar(&timeout, "timeout", "2000", "Number of milliseconds to wait for the ONVIF discovery to complete")
+	flag.StringVar(&subnet, "subnet", "", "Optional subnet(s) to scan for discovery, e.g. '192.168.1.0/24' (comma-separated). Defaults to the local interfaces.")
 	flag.Parse()
 
 	// Specify the level of loggin: "info", "warning", "debug", "error" or "fatal."
@@ -96,35 +116,76 @@ func main() {
 	}
 	// Specify the timezone of the log: "UTC" or "Local".
 	timezone, _ := time.LoadLocation("CET")
-	log.Log.Init(logLevel, logOutput, configDirectory, timezone)
+	configureLogging(logLevel, logOutput, timezone)
+	if action == "run" || action == "healthcheck" {
+		resolvedPort, err := resolveServerPort(port, os.Getenv("AGENT_PORT"))
+		if err != nil {
+			log.WithError(err).WithFields(log.Fields{
+				"component": "http",
+				"event":     "server_port_invalid",
+			}).Fatal("Invalid HTTP server port")
+			return
+		}
+		port = resolvedPort
+	}
+	log.WithFields(log.Fields{
+		"action":           action,
+		"component":        "agent",
+		"config_directory": configDirectory,
+		"event":            "command_parsed",
+		"port":             port,
+		"version":          VERSION,
+	}).Debug("Agent command parsed")
 
 	switch action {
 
+	case "healthcheck":
+		if err := checkAgentHealth(context.Background(), port); err != nil {
+			log.WithError(err).WithFields(log.Fields{
+				"component": "healthcheck",
+				"event":     "healthcheck_failed",
+			}).Fatal("Agent healthcheck failed")
+		}
 	case "version":
 		{
-			log.Log.Info("main.Main(): You are currrently running Kerberos Agent " + VERSION)
+			log.WithFields(log.Fields{
+				"component": "agent",
+				"event":     "version",
+				"version":   VERSION,
+			}).Info("Kerberos Agent version")
 		}
 	case "discover":
 		{
 			// Convert duration to int
 			timeout, err := time.ParseDuration(timeout + "ms")
 			if err != nil {
-				log.Log.Fatal("main.Main(): could not parse timeout: " + err.Error())
+				log.WithError(err).WithField("component", "onvif").
+					Fatal("invalid ONVIF discovery timeout")
 				return
 			}
-			onvif.Discover(timeout)
+			var subnets []string
+			for _, part := range strings.Split(subnet, ",") {
+				if trimmed := strings.TrimSpace(part); trimmed != "" {
+					subnets = append(subnets, trimmed)
+				}
+			}
+			onvif.Discover(timeout, subnets...)
 		}
 	case "decrypt":
 		{
-			log.Log.Info("main.Main(): Decrypting: " + flag.Arg(0) + " with key: " + flag.Arg(1))
+			log.WithFields(log.Fields{
+				"component": "encryption",
+				"event":     "decrypt_started",
+				"path":      flag.Arg(0),
+			}).Info("Decrypting recording")
 			symmetricKey := []byte(flag.Arg(1))
 
 			if len(symmetricKey) == 0 {
-				log.Log.Fatal("main.Main(): symmetric key should not be empty")
+				log.Fatal("main.Main(): symmetric key should not be empty")
 				return
 			}
 			if len(symmetricKey) != 32 {
-				log.Log.Fatal("main.Main(): symmetric key should be 32 bytes")
+				log.Fatal("main.Main(): symmetric key should be 32 bytes")
 				return
 			}
 
@@ -154,19 +215,28 @@ func main() {
 
 			// Start OpenTelemetry tracing
 			if otelEndpoint := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"); otelEndpoint == "" {
-				log.Log.Info("main.Main(): No OpenTelemetry endpoint provided, skipping tracing")
+				log.WithFields(log.Fields{
+					"component": "tracing",
+					"event":     "tracing_disabled",
+				}).Debug("OpenTelemetry tracing disabled")
 			} else {
-				log.Log.Info("main.Main(): Starting OpenTelemetry tracing with endpoint: " + otelEndpoint)
+				log.WithFields(log.Fields{
+					"component": "tracing",
+					"event":     "tracing_starting",
+				}).Info("Starting OpenTelemetry tracing")
 				agentKey := configuration.Config.Key
 				traceProvider, err := startTracing(agentKey, otelEndpoint)
 				if err != nil {
-					log.Log.Error("traceprovider: " + err.Error())
+					log.WithError(err).WithField("component", "tracing").
+						Error("Failed to start OpenTelemetry tracing")
+				} else {
+					defer func() {
+						if err := traceProvider.Shutdown(context.Background()); err != nil {
+							log.WithError(err).WithField("component", "tracing").
+								Error("Failed to shut down OpenTelemetry tracing")
+						}
+					}()
 				}
-				defer func() {
-					if err := traceProvider.Shutdown(context.Background()); err != nil {
-						log.Log.Error("traceprovider: " + err.Error())
-					}
-				}()
 			}
 
 			// Printing final configuration
@@ -177,8 +247,13 @@ func main() {
 			utils.CheckDataDirectoryPermissions(configDirectory)
 
 			// Set timezone
-			timezone, _ := time.LoadLocation(configuration.Config.Timezone)
-			log.Log.Init(logLevel, logOutput, configDirectory, timezone)
+			timezone, err := time.LoadLocation(configuration.Config.Timezone)
+			if err != nil {
+				log.WithError(err).WithField("timezone", configuration.Config.Timezone).
+					Warn("invalid Agent timezone; using the host timezone for logs")
+				timezone = time.Local
+			}
+			configureLogging(logLevel, logOutput, timezone)
 
 			// Check if we have a device Key or not, if not
 			// we will generate one.
@@ -187,30 +262,37 @@ func main() {
 				configuration.Config.Key = key
 				err := configService.StoreConfig(configDirectory, configuration.Config)
 				if err == nil {
-					log.Log.Info("main.Main(): updated unique key for agent to: " + key)
+					log.WithFields(log.Fields{
+						"component": "configuration",
+						"event":     "agent_key_generated",
+					}).Info("Generated and stored a unique Agent key")
 				} else {
-					log.Log.Info("main.Main(): something went wrong while trying to store key: " + key)
+					log.WithError(err).WithFields(log.Fields{
+						"component": "configuration",
+						"event":     "agent_key_store_failed",
+					}).Error("Failed to store the generated Agent key")
 				}
 			}
 
 			// Create a cancelable context, which will be used to cancel and restart.
 			// This is used to restart the agent when the configuration is updated.
 			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
 
 			// We create a capture object, this will contain all the streaming clients.
 			// And allow us to extract media from within difference places in the agent.
-			capture := capture.Capture{
-				RTSPClient:    nil,
-				RTSPSubClient: nil,
-			}
+			capture := capture.Capture{}
 
 			// Bootstrapping the agent
 			communication := models.Communication{
-				Context:         &ctx,
-				CancelContext:   &cancel,
 				HandleBootstrap: make(chan string, 1),
 			}
 
+			log.WithFields(log.Fields{
+				"component": "agent",
+				"event":     "runtime_starting",
+				"port":      configuration.Port,
+			}).Info("Starting Agent runtime")
 			go components.Bootstrap(ctx, configDirectory, &configuration, &communication, &capture)
 
 			// Start the REST API.
@@ -218,7 +300,7 @@ func main() {
 		}
 	default:
 		{
-			log.Log.Error("main.Main(): Sorry I don't understand :(")
+			log.Error("main.Main(): Sorry I don't understand :(")
 		}
 	}
 }

machinery/main_test.go

@@ -0,0 +1,33 @@
+package main
+
+import "testing"
+
+func TestResolveServerPort(t *testing.T) {
+	tests := []struct {
+		name      string
+		flagValue string
+		envValue  string
+		want      string
+		wantError bool
+	}{
+		{name: "flag default", flagValue: "80", want: "80"},
+		{name: "environment overrides flag", flagValue: "80", envValue: "8082", want: "8082"},
+		{name: "trims environment", flagValue: "80", envValue: " 9090 ", want: "9090"},
+		{name: "empty values use default", want: "80"},
+		{name: "invalid text", flagValue: "80", envValue: "http", wantError: true},
+		{name: "zero", flagValue: "80", envValue: "0", wantError: true},
+		{name: "above maximum", flagValue: "80", envValue: "65536", wantError: true},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got, err := resolveServerPort(test.flagValue, test.envValue)
+			if (err != nil) != test.wantError {
+				t.Fatalf("resolveServerPort(%q, %q) error = %v, wantError %t", test.flagValue, test.envValue, err, test.wantError)
+			}
+			if got != test.want {
+				t.Fatalf("resolveServerPort(%q, %q) = %q, want %q", test.flagValue, test.envValue, got, test.want)
+			}
+		})
+	}
+}

machinery/src/capture/cleanup_test.go

@@ -0,0 +1,285 @@
+package capture
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+	"time"
+
+	"github.com/kerberos-io/agent/machinery/src/models"
+)
+
+func TestPickRecordingToCleanupAcrossDatedDirectories(t *testing.T) {
+	for _, legacyMarker := range []bool{false, true} {
+		t.Run(map[bool]string{false: "metadata", true: "legacy"}[legacyMarker], func(t *testing.T) {
+			recordingsDir, cloudDir := newCleanupDirs(t)
+			pendingName := "2026/01/01/oldest_pending.mp4"
+			safeName := "2026/01/02/uploaded.mp4"
+			writeRecording(t, recordingsDir, pendingName, 30)
+			if legacyMarker {
+				if err := os.WriteFile(filepath.Join(cloudDir, filepath.Base(pendingName)), nil, 0o644); err != nil {
+					t.Fatal(err)
+				}
+			} else {
+				markPending(t, cloudDir, pendingName)
+			}
+			writeRecording(t, recordingsDir, safeName, 20)
+			writeRecording(t, recordingsDir, "flat_uploaded.mp4", 10)
+			name, pending, err := pickRecordingToCleanup(recordingsDir, cloudDir)
+			if err != nil || pending || name != safeName {
+				t.Fatalf("cleanup = (%q, %v, %v), want (%q, false, nil)", name, pending, err, safeName)
+			}
+		})
+	}
+}
+
+func TestCleanupDatedPendingRecordingRemovesFlatMarker(t *testing.T) {
+	recordingsDir, cloudDir := newCleanupDirs(t)
+	name := "2026/01/01/pending.mp4"
+	if err := os.MkdirAll(filepath.Dir(filepath.Join(recordingsDir, name)), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	writeSizedRecording(t, recordingsDir, name, 2*1000*1000)
+	markPending(t, cloudDir, name)
+	config := &models.Configuration{Config: models.Config{AutoClean: "true", MaxDirectorySize: 1}}
+	CleanupRecordingDirectory(filepath.Dir(filepath.Dir(recordingsDir)), config)
+	for _, filePath := range []string{
+		filepath.Join(recordingsDir, name),
+		filepath.Join(cloudDir, models.RecordingUploadMetadataFileName(name)),
+	} {
+		if _, err := os.Stat(filePath); !os.IsNotExist(err) {
+			t.Fatalf("expected %s to be removed, got %v", filePath, err)
+		}
+	}
+}
+
+// writeRecording creates a file under recordingsDir and sets its modtime so the
+// tests can control the "oldest" ordering deterministically.
+func writeRecording(t *testing.T, recordingsDir, name string, ageMinutes int) {
+	t.Helper()
+	full := filepath.Join(recordingsDir, name)
+	if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
+		t.Fatalf("mkdir recording directory: %v", err)
+	}
+	if err := os.WriteFile(full, []byte("data"), 0o644); err != nil {
+		t.Fatalf("write recording %s: %v", name, err)
+	}
+	mod := time.Now().Add(-time.Duration(ageMinutes) * time.Minute)
+	if err := os.Chtimes(full, mod, mod); err != nil {
+		t.Fatalf("chtimes %s: %v", name, err)
+	}
+}
+
+// markPending creates the upload marker in cloudDir for the given recording,
+// marking it as still queued for upload.
+func markPending(t *testing.T, cloudDir, name string) {
+	t.Helper()
+	markerName := models.RecordingUploadMetadataFileName(name)
+	if err := os.WriteFile(filepath.Join(cloudDir, markerName), nil, 0o644); err != nil {
+		t.Fatalf("write marker %s: %v", name, err)
+	}
+}
+
+func newCleanupDirs(t *testing.T) (string, string) {
+	t.Helper()
+	base := t.TempDir()
+	recordingsDir := filepath.Join(base, "data", "recordings")
+	cloudDir := filepath.Join(base, "data", "cloud")
+	if err := os.MkdirAll(recordingsDir, 0o755); err != nil {
+		t.Fatalf("mkdir recordings: %v", err)
+	}
+	if err := os.MkdirAll(cloudDir, 0o755); err != nil {
+		t.Fatalf("mkdir cloud: %v", err)
+	}
+	return recordingsDir, cloudDir
+}
+
+// The core regression: when the oldest recording is still pending upload but a
+// newer one has already been uploaded, cleanup must delete the uploaded (safe)
+// one and leave the pending recording on disk so it can still be uploaded.
+func TestPickRecordingToCleanup_PrefersUploaded(t *testing.T) {
+	recordingsDir, cloudDir := newCleanupDirs(t)
+
+	// oldest is still pending upload (marker present).
+	writeRecording(t, recordingsDir, "oldest_pending.mp4", 30)
+	markPending(t, cloudDir, "oldest_pending.mp4")
+	// newer one has already been uploaded (no marker).
+	writeRecording(t, recordingsDir, "newer_uploaded.mp4", 10)
+
+	name, pending, err := pickRecordingToCleanup(recordingsDir, cloudDir)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if pending {
+		t.Fatalf("expected a safe (already-uploaded) deletion, got pending=true")
+	}
+	if name != "newer_uploaded.mp4" {
+		t.Fatalf("cleanup picked %q, want the uploaded recording newer_uploaded.mp4", name)
+	}
+}
+
+func TestPickRecordingToCleanup_RecognizesLegacyMarkerName(t *testing.T) {
+	recordingsDir, cloudDir := newCleanupDirs(t)
+
+	writeRecording(t, recordingsDir, "legacy_pending.mp4", 30)
+	if err := os.WriteFile(filepath.Join(cloudDir, "legacy_pending.mp4"), nil, 0o644); err != nil {
+		t.Fatalf("write legacy marker: %v", err)
+	}
+	writeRecording(t, recordingsDir, "uploaded.mp4", 10)
+
+	name, pending, err := pickRecordingToCleanup(recordingsDir, cloudDir)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if pending || name != "uploaded.mp4" {
+		t.Fatalf("cleanup picked name=%q pending=%v, want uploaded.mp4 pending=false", name, pending)
+	}
+}
+
+// Among several already-uploaded recordings, the oldest uploaded one is chosen.
+func TestPickRecordingToCleanup_OldestUploadedFirst(t *testing.T) {
+	recordingsDir, cloudDir := newCleanupDirs(t)
+
+	writeRecording(t, recordingsDir, "old_uploaded.mp4", 40)
+	writeRecording(t, recordingsDir, "mid_uploaded.mp4", 20)
+	// pending one must be ignored even though it is not the oldest.
+	writeRecording(t, recordingsDir, "pending.mp4", 30)
+	markPending(t, cloudDir, "pending.mp4")
+
+	name, pending, err := pickRecordingToCleanup(recordingsDir, cloudDir)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if pending {
+		t.Fatalf("expected pending=false, got true")
+	}
+	if name != "old_uploaded.mp4" {
+		t.Fatalf("cleanup picked %q, want old_uploaded.mp4", name)
+	}
+}
+
+// Last resort: when every recording is still pending upload, cleanup returns the
+// oldest one with pending=true so the caller can drop it (and its marker) to keep
+// the disk bounded.
+func TestPickRecordingToCleanup_AllPendingFallsBackToOldest(t *testing.T) {
+	recordingsDir, cloudDir := newCleanupDirs(t)
+
+	writeRecording(t, recordingsDir, "a_old.mp4", 50)
+	markPending(t, cloudDir, "a_old.mp4")
+	writeRecording(t, recordingsDir, "b_new.mp4", 5)
+	markPending(t, cloudDir, "b_new.mp4")
+
+	name, pending, err := pickRecordingToCleanup(recordingsDir, cloudDir)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if !pending {
+		t.Fatalf("expected pending=true when every recording is queued for upload")
+	}
+	if name != "a_old.mp4" {
+		t.Fatalf("cleanup picked %q, want the oldest pending a_old.mp4", name)
+	}
+}
+
+// An empty recordings directory yields os.ErrNotExist so the caller does nothing.
+func TestPickRecordingToCleanup_Empty(t *testing.T) {
+	recordingsDir, cloudDir := newCleanupDirs(t)
+
+	if _, _, err := pickRecordingToCleanup(recordingsDir, cloudDir); err != os.ErrNotExist {
+		t.Fatalf("expected os.ErrNotExist for an empty directory, got %v", err)
+	}
+}
+
+// writeSizedRecording writes a recording of an exact byte size so tests can
+// exercise the megabyte-based directory-cap threshold.
+func writeSizedRecording(t *testing.T, dir, name string, size int) {
+	t.Helper()
+	if err := os.WriteFile(filepath.Join(dir, name), make([]byte, size), 0o644); err != nil {
+		t.Fatalf("write sized recording %s: %v", name, err)
+	}
+}
+
+// When AGENT_AUTO_CLEAN_MAX_SIZE (MaxDirectorySize) is set, cleanup triggers once
+// the recordings directory grows past that many megabytes.
+func TestRecordingsNeedCleanup_FixedCap(t *testing.T) {
+	recordingsDir, _ := newCleanupDirs(t)
+	// ~2 MB of recordings on disk.
+	writeSizedRecording(t, recordingsDir, "big.mp4", 2*1000*1000)
+
+	over := &models.Configuration{Config: models.Config{MaxDirectorySize: 1}}
+	need, err := recordingsNeedCleanup(recordingsDir, over)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if !need {
+		t.Fatalf("expected cleanup when 2MB of recordings exceed the 1MB cap")
+	}
+
+	under := &models.Configuration{Config: models.Config{MaxDirectorySize: 100}}
+	need, err = recordingsNeedCleanup(recordingsDir, under)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if need {
+		t.Fatalf("expected no cleanup when 2MB of recordings stay under the 100MB cap")
+	}
+}
+
+// With no fixed cap (the default), cleanup is driven by the free space left on
+// the recordings filesystem versus the reserve.
+func TestRecordingsNeedCleanup_DefaultDiskReserve(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip("disk usage stats are only implemented on linux")
+	}
+	recordingsDir, _ := newCleanupDirs(t)
+
+	totalMB, availableMB, err := diskUsageMB(recordingsDir)
+	if err != nil {
+		t.Fatalf("diskUsageMB: %v", err)
+	}
+	if totalMB <= 0 || availableMB <= 0 {
+		t.Skipf("unexpected disk stats total=%dMB available=%dMB", totalMB, availableMB)
+	}
+
+	// A reserve larger than the whole disk means free space is always below it.
+	over := &models.Configuration{Config: models.Config{MinFreeSpace: totalMB + availableMB}}
+	need, err := recordingsNeedCleanup(recordingsDir, over)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if !need {
+		t.Fatalf("expected cleanup when free space (%dMB) is below the reserve", availableMB)
+	}
+
+	// A 1 MB reserve leaves plenty of free space, so nothing should be cleaned.
+	under := &models.Configuration{Config: models.Config{MinFreeSpace: 1}}
+	need, err = recordingsNeedCleanup(recordingsDir, under)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if need {
+		t.Fatalf("expected no cleanup when free space (%dMB) exceeds the 1MB reserve", availableMB)
+	}
+}
+
+// The default 5% reserve must never truncate to 0MB on small disks, otherwise
+// cleanup would only trigger once the disk is completely full.
+func TestDefaultReserveMB(t *testing.T) {
+	cases := []struct {
+		totalMB int64
+		want    int64
+	}{
+		{totalMB: 0, want: 1},   // no/unknown disk size still reserves 1MB
+		{totalMB: 10, want: 1},  // 5% = 0MB -> floored to 1MB
+		{totalMB: 19, want: 1},  // 5% = 0MB -> floored to 1MB
+		{totalMB: 20, want: 1},  // 5% = exactly 1MB
+		{totalMB: 100, want: 5}, // 5% = 5MB
+		{totalMB: 1000, want: 50},
+	}
+	for _, c := range cases {
+		if got := defaultReserveMB(c.totalMB); got != c.want {
+			t.Errorf("defaultReserveMB(%d) = %d, want %d", c.totalMB, got, c.want)
+		}
+	}
+}

machinery/src/capture/disk_linux.go

@@ -0,0 +1,23 @@
+//go:build linux
+
+package capture
+
+import "syscall"
+
+// diskUsageMB returns the total capacity and the currently available space (both
+// in megabytes, decimal) of the filesystem that contains path. Auto-clean uses
+// it to default its cleanup threshold to the real disk capacity instead of a
+// fixed size, so recordings can grow to fill the disk while keeping a reserve
+// free. Linux is the agent's deployment target (amd64/arm64 containers).
+func diskUsageMB(path string) (totalMB int64, availableMB int64, err error) {
+	var stat syscall.Statfs_t
+	if err = syscall.Statfs(path, &stat); err != nil {
+		return 0, 0, err
+	}
+	blockSize := int64(stat.Bsize)
+	totalMB = int64(stat.Blocks) * blockSize / 1000 / 1000
+	// Bavail is the free space available to unprivileged users, which is the
+	// space we can actually keep writing recordings into.
+	availableMB = int64(stat.Bavail) * blockSize / 1000 / 1000
+	return totalMB, availableMB, nil
+}

machinery/src/capture/disk_other.go

@@ -0,0 +1,13 @@
+//go:build !linux
+
+package capture
+
+import "errors"
+
+// diskUsageMB is only implemented on Linux (the agent's deployment target). On
+// other platforms (e.g. local macOS/Windows dev builds) auto-clean falls back to
+// its historical fixed-size directory cap, so this reports the capability as
+// unavailable.
+func diskUsageMB(path string) (totalMB int64, availableMB int64, err error) {
+	return 0, 0, errors.New("disk usage stats are not supported on this platform")
+}

machinery/src/capture/gortsplib_test.go

@@ -0,0 +1,249 @@
+package capture
+
+import (
+	"bytes"
+	"context"
+	"encoding/pem"
+	"errors"
+	"math"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/bluenviron/mediacommon/pkg/codecs/h264"
+)
+
+func TestPreRecordingGOPCount(t *testing.T) {
+	maxInt := int64(^uint(0) >> 1)
+	tests := []struct {
+		name         string
+		preRecording int64
+		gopDuration  float64
+		want         int
+		wantOK       bool
+	}{
+		{name: "normal duration", preRecording: 10, gopDuration: 2.9, want: 6, wantOK: true},
+		{name: "duration longer than buffer", preRecording: 1, gopDuration: 2, want: 1, wantOK: true},
+		{name: "largest representable result", preRecording: maxInt - 1, gopDuration: 1, want: int(maxInt), wantOK: true},
+		{name: "result exceeds int", preRecording: maxInt, gopDuration: 1, wantOK: false},
+		{name: "non-positive pre-recording", preRecording: 0, gopDuration: 1, wantOK: false},
+		{name: "sub-second GOP", preRecording: 10, gopDuration: 0.9, wantOK: false},
+		{name: "NaN GOP", preRecording: 10, gopDuration: math.NaN(), wantOK: false},
+		{name: "infinite GOP", preRecording: 10, gopDuration: math.Inf(1), wantOK: false},
+		{name: "GOP exceeds int64", preRecording: 10, gopDuration: float64(math.MaxInt64), wantOK: false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, ok := preRecordingGOPCount(tt.preRecording, tt.gopDuration)
+			if ok != tt.wantOK || got != tt.want {
+				t.Fatalf("preRecordingGOPCount(%d, %v) = (%d, %t), want (%d, %t)",
+					tt.preRecording, tt.gopDuration, got, ok, tt.want, tt.wantOK)
+			}
+		})
+	}
+}
+
+func TestGolibrtspCloseBeforeClientStart(t *testing.T) {
+	client := &Golibrtsp{}
+
+	if err := client.Close(context.Background()); err != nil {
+		t.Fatalf("Close() error = %v", err)
+	}
+}
+
+func TestSanitizeRTSPErrorRemovesCredentialsAndQuery(t *testing.T) {
+	rawURL := "rtsp://camera-user:camera-password@10.0.20.15/live?access_token=secret"
+	got := sanitizeRTSPError(errors.New("describe "+rawURL+": bad status code"), rawURL)
+
+	for _, secret := range []string{"camera-user", "camera-password", "access_token", "secret"} {
+		if strings.Contains(got.Error(), secret) {
+			t.Fatalf("sanitizeRTSPError() exposed %q in %q", secret, got)
+		}
+	}
+	if !strings.Contains(got.Error(), "rtsp://10.0.20.15/live") {
+		t.Fatalf("sanitizeRTSPError() removed useful host/path context: %q", got)
+	}
+}
+
+func TestRTSPSTLSConfig(t *testing.T) {
+	t.Run("verifies certificates by default", func(t *testing.T) {
+		t.Setenv(rtspsCAFileEnv, "")
+		t.Setenv(rtspsInsecureEnv, "")
+
+		got, err := rtspsTLSConfig()
+		if err != nil {
+			t.Fatalf("rtspsTLSConfig() error = %v", err)
+		}
+		if got != nil {
+			t.Fatalf("rtspsTLSConfig() = %#v, want nil", got)
+		}
+	})
+
+	t.Run("allows explicit insecure mode", func(t *testing.T) {
+		t.Setenv(rtspsCAFileEnv, "/missing/ignored-in-insecure-mode.pem")
+		t.Setenv(rtspsInsecureEnv, "true")
+
+		got, err := rtspsTLSConfig()
+		if err != nil {
+			t.Fatalf("rtspsTLSConfig() error = %v", err)
+		}
+		if got == nil || !got.InsecureSkipVerify {
+			t.Fatalf("rtspsTLSConfig() = %#v, want InsecureSkipVerify enabled", got)
+		}
+	})
+
+	t.Run("adds a camera CA to system roots", func(t *testing.T) {
+		t.Setenv(rtspsInsecureEnv, "")
+		server := httptest.NewTLSServer(nil)
+		defer server.Close()
+
+		certificate := server.Certificate()
+		caFile := filepath.Join(t.TempDir(), "camera-ca.pem")
+		caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certificate.Raw})
+		if err := os.WriteFile(caFile, caPEM, 0o600); err != nil {
+			t.Fatal(err)
+		}
+		t.Setenv(rtspsCAFileEnv, caFile)
+
+		got, err := rtspsTLSConfig()
+		if err != nil {
+			t.Fatalf("rtspsTLSConfig() error = %v", err)
+		}
+		if got == nil || got.RootCAs == nil {
+			t.Fatalf("rtspsTLSConfig() = %#v, want custom RootCAs", got)
+		}
+		for _, subject := range got.RootCAs.Subjects() {
+			if bytes.Equal(subject, certificate.RawSubject) {
+				return
+			}
+		}
+		t.Fatal("camera CA was not added to RootCAs")
+	})
+
+	t.Run("rejects an invalid camera CA file", func(t *testing.T) {
+		t.Setenv(rtspsInsecureEnv, "")
+		caFile := filepath.Join(t.TempDir(), "camera-ca.pem")
+		if err := os.WriteFile(caFile, []byte("not a certificate"), 0o600); err != nil {
+			t.Fatal(err)
+		}
+		t.Setenv(rtspsCAFileEnv, caFile)
+
+		if _, err := rtspsTLSConfig(); err == nil {
+			t.Fatal("rtspsTLSConfig() error = nil, want invalid CA error")
+		}
+	})
+}
+
+func TestVideoReferenceStateDropsDependentFramesUntilRandomAccess(t *testing.T) {
+	state := &videoReferenceState{}
+
+	accepted, recovered := state.accept(false)
+	if accepted || recovered != 0 || state.droppedAccessUnits != 1 {
+		t.Fatalf("initial dependent frame = (accepted=%t, recovered=%d, dropped=%d), want (false, 0, 1)",
+			accepted, recovered, state.droppedAccessUnits)
+	}
+
+	accepted, recovered = state.accept(true)
+	if !accepted || recovered != 1 || !state.valid {
+		t.Fatalf("random-access recovery = (accepted=%t, recovered=%d, valid=%t), want (true, 1, true)",
+			accepted, recovered, state.valid)
+	}
+
+	if expected, gap := state.observeSequence(100); gap || expected != 100 {
+		t.Fatalf("first sequence observation = (expected=%d, gap=%t), want (100, false)", expected, gap)
+	}
+	if expected, gap := state.observeSequence(102); !gap || expected != 101 || state.valid {
+		t.Fatalf("sequence gap = (expected=%d, gap=%t, valid=%t), want (101, true, false)",
+			expected, gap, state.valid)
+	}
+
+	accepted, recovered = state.accept(false)
+	if accepted || recovered != 0 || state.droppedAccessUnits != 1 {
+		t.Fatalf("post-loss dependent frame = (accepted=%t, recovered=%d, dropped=%d), want (false, 0, 1)",
+			accepted, recovered, state.droppedAccessUnits)
+	}
+
+	accepted, recovered = state.accept(true)
+	if !accepted || recovered != 1 || !state.valid {
+		t.Fatalf("post-loss recovery = (accepted=%t, recovered=%d, valid=%t), want (true, 1, true)",
+			accepted, recovered, state.valid)
+	}
+}
+
+func TestVideoReferenceStateAcceptsSequenceWraparound(t *testing.T) {
+	state := &videoReferenceState{valid: true}
+
+	if _, gap := state.observeSequence(^uint16(0)); gap {
+		t.Fatal("first sequence observation reported a gap")
+	}
+	if expected, gap := state.observeSequence(0); gap || expected != 0 || !state.valid {
+		t.Fatalf("sequence wraparound = (expected=%d, gap=%t, valid=%t), want (0, false, true)",
+			expected, gap, state.valid)
+	}
+}
+
+func TestH264FrameNumTrackerDetectsMissingIDR(t *testing.T) {
+	tracker := &h264FrameNumTracker{}
+	tracker.setSPS(&h264.SPS{
+		Log2MaxFrameNumMinus4: 1,
+		FrameMbsOnlyFlag:      true,
+	})
+
+	if _, _, gap, ok := tracker.observe([][]byte{testH264Slice(h264.NALUTypeIDR, 0)}); !ok || gap {
+		t.Fatalf("initial IDR observation = (ok=%t, gap=%t), want (true, false)", ok, gap)
+	}
+	for frameNum := uint32(1); frameNum <= 24; frameNum++ {
+		if expected, received, gap, ok := tracker.observe([][]byte{testH264Slice(h264.NALUTypeNonIDR, frameNum)}); !ok || gap {
+			t.Fatalf("frame_num %d observation = (expected=%d, received=%d, ok=%t, gap=%t), want no gap",
+				frameNum, expected, received, ok, gap)
+		}
+	}
+
+	expected, received, gap, ok := tracker.observe([][]byte{testH264Slice(h264.NALUTypeNonIDR, 1)})
+	if !ok || !gap || expected != 25 || received != 1 {
+		t.Fatalf("missing IDR transition = (expected=%d, received=%d, ok=%t, gap=%t), want (25, 1, true, true)",
+			expected, received, ok, gap)
+	}
+}
+
+func TestH264FrameNumTrackerResetsAtIDR(t *testing.T) {
+	tracker := &h264FrameNumTracker{}
+	tracker.setSPS(&h264.SPS{
+		Log2MaxFrameNumMinus4: 1,
+		FrameMbsOnlyFlag:      true,
+	})
+
+	tracker.observe([][]byte{testH264Slice(h264.NALUTypeIDR, 0)})
+	tracker.observe([][]byte{testH264Slice(h264.NALUTypeNonIDR, 1)})
+	if expected, received, gap, ok := tracker.observe([][]byte{testH264Slice(h264.NALUTypeIDR, 0)}); !ok || gap {
+		t.Fatalf("IDR reset = (expected=%d, received=%d, ok=%t, gap=%t), want no gap",
+			expected, received, ok, gap)
+	}
+	if expected, received, gap, ok := tracker.observe([][]byte{testH264Slice(h264.NALUTypeNonIDR, 1)}); !ok || gap || expected != 1 || received != 1 {
+		t.Fatalf("post-IDR frame = (expected=%d, received=%d, ok=%t, gap=%t), want (1, 1, true, false)",
+			expected, received, ok, gap)
+	}
+}
+
+func TestH264FrameNumTrackerHonorsPermittedGaps(t *testing.T) {
+	tracker := &h264FrameNumTracker{}
+	tracker.setSPS(&h264.SPS{
+		Log2MaxFrameNumMinus4:          1,
+		FrameMbsOnlyFlag:               true,
+		GapsInFrameNumValueAllowedFlag: true,
+	})
+
+	tracker.observe([][]byte{testH264Slice(h264.NALUTypeIDR, 0)})
+	expected, received, gap, ok := tracker.observe([][]byte{testH264Slice(h264.NALUTypeNonIDR, 2)})
+	if !ok || gap || expected != 1 || received != 2 {
+		t.Fatalf("permitted frame_num gap = (expected=%d, received=%d, ok=%t, gap=%t), want (1, 2, true, false)",
+			expected, received, ok, gap)
+	}
+}
+
+func testH264Slice(naluType h264.NALUType, frameNum uint32) []byte {
+	return []byte{0x60 | byte(naluType), 0xE0 | byte(frameNum&0x1F)}
+}

machinery/src/capture/main.go

@@ -4,65 +4,347 @@ package capture
 import (
 	"context"
 	"encoding/base64"
+	"encoding/json"
 	"image"
+	"math"
 	"os"
+	"path/filepath"
 	"strconv"
 	"time"
 
+	mqtt "github.com/eclipse/paho.mqtt.golang"
 	"github.com/gin-gonic/gin"
 	"github.com/kerberos-io/agent/machinery/src/conditions"
 	"github.com/kerberos-io/agent/machinery/src/encryption"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/kerberos-io/agent/machinery/src/utils"
 	"github.com/kerberos-io/agent/machinery/src/video"
+	log "github.com/sirupsen/logrus"
 	"go.opentelemetry.io/otel/trace"
 )
 
-func CleanupRecordingDirectory(configDirectory string, configuration *models.Configuration) {
-	autoClean := configuration.Config.AutoClean
-	if autoClean == "true" {
-		maxSize := configuration.Config.MaxDirectorySize
-		if maxSize == 0 {
-			maxSize = 300
-		}
-		// Total size of the recording directory.
-		recordingsDirectory := configDirectory + "/data/recordings"
-		size, err := utils.DirSize(recordingsDirectory)
-		if err == nil {
-			sizeInMB := size / 1000 / 1000
-			if sizeInMB >= maxSize {
-				// Remove the oldest recording
-				oldestFile, err := utils.FindOldestFile(recordingsDirectory)
-				if err == nil {
-					err := os.Remove(recordingsDirectory + "/" + oldestFile.Name())
-					log.Log.Info("HandleRecordStream: removed oldest file as part of cleanup - " + recordingsDirectory + "/" + oldestFile.Name())
-					if err != nil {
-						log.Log.Info("HandleRecordStream: something went wrong, " + err.Error())
-					}
-				} else {
-					log.Log.Info("HandleRecordStream: something went wrong, " + err.Error())
-				}
-			}
-		} else {
-			log.Log.Info("HandleRecordStream: something went wrong, " + err.Error())
-		}
-
+// publishRecordingState notifies the hub (and ultimately the live-view UI) that
+// this camera started ("recording": true) or stopped ("recording": false)
+// recording, so the frontend can show a "recording" indicator while the agent
+// is recording (e.g. a motion clip triggered manually from the live view or by
+// motion detection). It is a best-effort broadcast: when no hub/MQTT is
+// configured (or the agent is offline) it is a no-op, and a missed message is
+// self-healed by the frontend's safety timeout.
+func publishRecordingState(mqttClient mqtt.Client, hubKey string, configuration *models.Configuration, recording bool) {
+	if mqttClient == nil || hubKey == "" || configuration.Config.Offline == "true" {
+		return
+	}
+	message := models.Message{
+		Payload: models.Payload{
+			Action:   "recording",
+			DeviceId: configuration.Config.Key,
+			Value: map[string]interface{}{
+				"timestamp": time.Now().Unix(),
+				"recording": recording,
+			},
+		},
+	}
+	payload, err := models.PackageMQTTMessage(configuration, message)
+	if err == nil {
+		mqttClient.Publish("kerberos/hub/"+hubKey, 2, false, payload)
 	} else {
-		log.Log.Info("HandleRecordStream: Autoclean disabled, nothing to do here.")
+		log.Error("capture.main.publishRecordingState(): failed to package MQTT message: " + err.Error())
 	}
 }
 
-func HandleRecordStream(queue *packets.Queue, configDirectory string, configuration *models.Configuration, communication *models.Communication, rtspClient RTSPClient) {
+func recordingUploadMetadata(name string, config models.Config, timestamp int64, mp4Video *video.MP4, encrypted bool, coordinates string, changes int) models.RecordingUploadMetadata {
+	metadata := models.RecordingUploadMetadata{
+		FileName:          filepath.Base(name),
+		DeviceKey:         config.Key,
+		Timestamp:         timestamp,
+		Duration:          mp4Video.VideoTotalDuration,
+		Encrypted:         encrypted,
+		DeviceName:        config.Name,
+		RegionCoordinates: coordinates,
+		NumberOfChanges:   strconv.Itoa(changes),
+	}
+	value := mp4Video.AverageFPS()
+	if value > 0 && value <= 240 && !math.IsInf(value, 0) && !math.IsNaN(value) {
+		metadata.FPS = value
+	}
+	return metadata
+}
+
+// queueRecordingForUpload creates the marker consumed by the upload worker and
+// stores metadata captured from the finalized recording.
+func queueRecordingForUpload(configDirectory string, metadata models.RecordingUploadMetadata) {
+	payload, err := json.Marshal(metadata)
+	if err != nil {
+		log.Error("capture.main.queueRecordingForUpload(): " + err.Error())
+		return
+	}
+
+	// Publish the marker with a same-filesystem rename. Writing directly to the
+	// watched directory would briefly expose an empty file to the upload poller.
+	marker, err := os.CreateTemp(filepath.Join(configDirectory, "data"), ".upload-marker-*")
+	if err == nil {
+		_, err = marker.Write(payload)
+	}
+	if err == nil {
+		err = marker.Chmod(0644)
+	}
+	if marker != nil {
+		if closeErr := marker.Close(); err == nil {
+			err = closeErr
+		}
+		defer os.Remove(marker.Name())
+	}
+	if err == nil {
+		err = os.Rename(marker.Name(), filepath.Join(configDirectory, "data", "cloud", models.RecordingUploadMetadataFileName(metadata.FileName)))
+	}
+	if err != nil {
+		log.Error("capture.main.queueRecordingForUpload(): " + err.Error())
+	}
+}
+
+const (
+	// manualRecordingHeartbeatTimeout is how long the agent keeps a manual
+	// (live-view / remote) recording alive after the LAST viewer heartbeat. The
+	// frontend re-sends the record command every ~15s while the user stays on the
+	// page; if several heartbeats are missed (the viewer closed the tab, went idle
+	// or lost connectivity) the recorder auto-stops the recording so the camera
+	// doesn't record forever when the "stop" message never arrives.
+	manualRecordingHeartbeatTimeout = 45 * time.Second
+	// manualRecordingMaxDuration caps a single manual recording so a forgotten
+	// record button can't record indefinitely even while the viewer keeps sending
+	// heartbeats. After this the recording auto-stops and the viewer must press
+	// record again to continue.
+	manualRecordingMaxDuration = 5 * time.Minute
+)
+
+// manualRecordingExpired reports whether an active manual (live-view) recording
+// has outlived its viewer heartbeat window or the maximum duration cap. When it
+// has, it clears the manual-recording state (so the motion recorder lets the
+// current clip close normally and broadcasts recording:false) and returns true.
+// It is a no-op returning false when no manual recording is active.
+func manualRecordingExpired(communication *models.Communication, now int64) bool {
+	if communication.IsRecordingManual.IsNotSet() {
+		return false
+	}
+	manualStart := communication.RecordingManualStart.Load()
+	maxDurationReached := manualStart > 0 && now-manualStart > manualRecordingMaxDuration.Milliseconds()
+	// The heartbeat timeout only applies once the viewer has proven it supports
+	// heartbeats (an older frontend that starts a recording but never heartbeats
+	// still records up to the max-duration cap instead of being cut off early).
+	heartbeatExpired := false
+	if communication.RecordingManualHeartbeatSeen.IsSet() {
+		lastHeartbeat := communication.RecordingManualHeartbeat.Load()
+		heartbeatExpired = lastHeartbeat > 0 && now-lastHeartbeat > manualRecordingHeartbeatTimeout.Milliseconds()
+	}
+	if !heartbeatExpired && !maxDurationReached {
+		return false
+	}
+	if heartbeatExpired {
+		log.Info("capture.main.HandleRecordStream(motiondetection): auto-stopping manual recording, no viewer heartbeat within timeout.")
+	} else {
+		log.Info("capture.main.HandleRecordStream(motiondetection): auto-stopping manual recording, maximum duration reached.")
+	}
+	communication.IsRecordingManual.UnSet()
+	communication.RecordingManualHeartbeat.Store(0)
+	communication.RecordingManualStart.Store(0)
+	communication.RecordingManualHeartbeatSeen.UnSet()
+	return true
+}
+
+func CleanupRecordingDirectory(configDirectory string, configuration *models.Configuration) {
+	autoClean := configuration.Config.AutoClean
+	if autoClean != "true" {
+		log.Info("HandleRecordStream: Autoclean disabled, nothing to do here.")
+		return
+	}
+
+	recordingsDirectory := configDirectory + "/data/recordings"
+	cloudDirectory := configDirectory + "/data/cloud"
+
+	// Decide whether we still need to free up space. See recordingsNeedCleanup
+	// for the two modes: an explicit fixed directory cap
+	// (AGENT_AUTO_CLEAN_MAX_SIZE) or, by default, letting recordings use the whole
+	// disk while keeping a free-space reserve.
+	needsCleanup, err := recordingsNeedCleanup(recordingsDirectory, configuration)
+	if err != nil {
+		log.Info("HandleRecordStream: something went wrong, " + err.Error())
+		return
+	}
+	if !needsCleanup {
+		return
+	}
+
+	// Remove the oldest recording, but PREFER recordings that have already been
+	// uploaded (i.e. no longer have a pending marker in data/cloud). This stops
+	// auto-clean from deleting recordings that are still queued for upload. That
+	// previously caused silent data loss: during a network outage the upload
+	// backlog grows, cleanup deletes the oldest (still un-uploaded) recording to
+	// stay under MaxDirectorySize, and when connectivity returns the upload loop
+	// finds the marker but the file is gone -> the recording is dropped and never
+	// reaches the vault.
+	//
+	// Only when EVERY recording on disk is still pending upload do we fall back to
+	// deleting the oldest pending one, as a last resort to keep the disk bounded
+	// (otherwise a long outage would fill the disk and stop new recordings).
+	name, pending, err := pickRecordingToCleanup(recordingsDirectory, cloudDirectory)
+	if err != nil {
+		log.Info("HandleRecordStream: something went wrong, " + err.Error())
+		return
+	}
+
+	if err := utils.RemoveRecording(recordingsDirectory, name); err != nil {
+		log.Info("HandleRecordStream: something went wrong, " + err.Error())
+		return
+	}
+
+	if pending {
+		// Data-loss event: the whole recordings directory is an un-uploaded
+		// backlog (e.g. a prolonged network outage), so we had to drop a recording
+		// that was never uploaded to keep recording new footage. Also remove the
+		// now-dangling upload marker so the upload loop doesn't keep trying to
+		// upload a file that no longer exists.
+		log.Warn("HandleRecordStream: removed oldest recording as part of cleanup, but it was STILL PENDING UPLOAD (disk full of un-uploaded recordings) - " + recordingsDirectory + "/" + name)
+		for _, markerName := range uploadMarkerNames(name) {
+			if err := os.Remove(filepath.Join(cloudDirectory, markerName)); err != nil && !os.IsNotExist(err) {
+				log.Info("HandleRecordStream: could not remove dangling upload marker " + markerName + ", " + err.Error())
+			}
+		}
+	} else {
+		log.Info("HandleRecordStream: removed oldest file as part of cleanup - " + recordingsDirectory + "/" + name)
+	}
+}
+
+// recordingsNeedCleanup reports whether auto-clean should free up space in the
+// recordings directory. There are two modes:
+//
+//   - AGENT_AUTO_CLEAN_MAX_SIZE (MaxDirectorySize, MB) set: cap the size of the
+//     recordings directory itself (the historical behaviour).
+//   - MaxDirectorySize == 0 (the default): recordings may use the WHOLE disk.
+//     Cleanup only triggers once the free space on the recordings filesystem
+//     drops to/below a reserve. The reserve is AGENT_AUTO_CLEAN_MIN_FREE_SPACE
+//     (MinFreeSpace, MB) when set, otherwise 5% of the disk's total capacity.
+//
+// If disk stats can't be read (e.g. non-Linux dev builds) it falls back to the
+// historical fixed 300 MB directory cap so behaviour stays bounded.
+func recordingsNeedCleanup(recordingsDirectory string, configuration *models.Configuration) (bool, error) {
+	maxSize := configuration.Config.MaxDirectorySize
+
+	// Explicit fixed cap on the recordings directory size.
+	if maxSize > 0 {
+		size, err := utils.DirSize(recordingsDirectory)
+		if err != nil {
+			return false, err
+		}
+		return size/1000/1000 >= maxSize, nil
+	}
+
+	// Default: allow recordings to use the full disk, keeping a reserve free.
+	totalMB, availableMB, err := diskUsageMB(recordingsDirectory)
+	if err != nil {
+		// Disk stats unavailable: fall back to the historical 300 MB cap.
+		size, derr := utils.DirSize(recordingsDirectory)
+		if derr != nil {
+			return false, derr
+		}
+		return size/1000/1000 >= 300, nil
+	}
+
+	reserveMB := configuration.Config.MinFreeSpace
+	if reserveMB <= 0 {
+		reserveMB = defaultReserveMB(totalMB)
+	}
+
+	return availableMB <= reserveMB, nil
+}
+
+// defaultReserveMB returns the free-space reserve (MB) to keep on the recordings
+// disk when AGENT_AUTO_CLEAN_MIN_FREE_SPACE is not set: 5% of the disk total,
+// but never below 1MB. On very small disks 5% truncates to 0MB, which would
+// disable the reserve entirely (cleanup only once availableMB <= 0), so we floor
+// it at 1MB to preserve the intended "keep some space free" behaviour.
+func defaultReserveMB(totalMB int64) int64 {
+	reserveMB := totalMB * 5 / 100
+	if reserveMB < 1 {
+		reserveMB = 1
+	}
+	return reserveMB
+}
+
+// pickRecordingToCleanup chooses which recording to delete to free space in the
+// recordings directory. It returns the oldest recording that has already been
+// uploaded (no pending marker with the same name in cloudDirectory). Only when
+// every recording is still pending upload does it return the oldest recording
+// overall with pending=true, signalling the caller that it is about to drop an
+// un-uploaded recording as a last resort.
+func pickRecordingToCleanup(recordingsDirectory, cloudDirectory string) (string, bool, error) {
+	entries, err := utils.ReadRecordingDirectory(recordingsDirectory)
+	if err != nil {
+		return "", false, err
+	}
+
+	var oldestSafeName, oldestAnyName string
+	var oldestSafeTime, oldestAnyTime time.Time
+
+	for _, entry := range entries {
+		modTime := entry.ModTime()
+
+		if oldestAnyName == "" || modTime.Before(oldestAnyTime) {
+			oldestAnyName = entry.Name()
+			oldestAnyTime = modTime
+		}
+
+		// A recording is still pending upload if either its current .metadata
+		// marker or a marker created by an older agent exists.
+		if recordingPendingUpload(cloudDirectory, entry.Name()) {
+			continue
+		}
+
+		if oldestSafeName == "" || modTime.Before(oldestSafeTime) {
+			oldestSafeName = entry.Name()
+			oldestSafeTime = modTime
+		}
+	}
+
+	if oldestSafeName != "" {
+		return oldestSafeName, false, nil
+	}
+	if oldestAnyName != "" {
+		return oldestAnyName, true, nil
+	}
+	return "", false, os.ErrNotExist
+}
+
+func uploadMarkerNames(recordingName string) []string {
+	return []string{models.RecordingUploadMetadataFileName(recordingName), filepath.Base(recordingName)}
+}
+
+func recordingPendingUpload(cloudDirectory, recordingName string) bool {
+	for _, markerName := range uploadMarkerNames(recordingName) {
+		if _, err := os.Stat(filepath.Join(cloudDirectory, markerName)); err == nil {
+			return true
+		}
+	}
+	return false
+}
+
+func HandleRecordStream(queue *packets.Queue, configDirectory string, configuration *models.Configuration, communication *models.Communication, rtspClient RTSPClient, mqttClient mqtt.Client, motionEvents <-chan models.MotionDataPartial) {
 
 	config := configuration.Config
+	hubKey := config.HubKey
 	loc, _ := time.LoadLocation(config.Timezone)
 
+	// Start each capture session with manual recording off, so a leftover
+	// request from before a restart/reconnect doesn't silently persist.
+	communication.IsRecordingManual.UnSet()
+	communication.RecordingManualHeartbeat.Store(0)
+	communication.RecordingManualStart.Store(0)
+	communication.RecordingManualHeartbeatSeen.UnSet()
+
 	if config.Capture.Recording == "false" {
-		log.Log.Info("capture.main.HandleRecordStream(): disabled, we will not record anything.")
+		log.Info("capture.main.HandleRecordStream(): disabled, we will not record anything.")
 	} else {
-		log.Log.Debug("capture.main.HandleRecordStream(): started")
+		log.Debug("capture.main.HandleRecordStream(): started")
 
 		preRecording := config.Capture.PreRecording * 1000
 		postRecording := config.Capture.PostRecording * 1000           // number of seconds to record.
@@ -75,7 +357,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 		}
 
 		if maxRecordingPeriod < preRecording+postRecording {
-			log.Log.Error("capture.main.HandleRecordStream(): maxRecordingPeriod is less than preRecording + postRecording, this is not allowed. Setting maxRecordingPeriod to preRecording + postRecording.")
+			log.Error("capture.main.HandleRecordStream(): maxRecordingPeriod is less than preRecording + postRecording, this is not allowed. Setting maxRecordingPeriod to preRecording + postRecording.")
 			maxRecordingPeriod = preRecording + postRecording
 		}
 
@@ -109,7 +391,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 			var name string
 
 			// Do not do anything!
-			log.Log.Info("capture.main.HandleRecordStream(continuous): start recording")
+			log.Info("capture.main.HandleRecordStream(continuous): start recording")
 
 			start := false
 
@@ -155,47 +437,15 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					}
 					if (videoCodec == "H264" && (len(mp4Video.SPSNALUs) == 0 || len(mp4Video.PPSNALUs) == 0)) ||
 						(videoCodec == "H265" && (len(mp4Video.VPSNALUs) == 0 || len(mp4Video.SPSNALUs) == 0 || len(mp4Video.PPSNALUs) == 0)) {
-						log.Log.Warning("capture.main.HandleRecordStream(continuous): closing MP4 without full parameter sets, moov may be incomplete")
+						log.Warn("capture.main.HandleRecordStream(continuous): closing MP4 without full parameter sets, moov may be incomplete")
 					}
 					mp4Video.Close(&config)
-					log.Log.Info("capture.main.HandleRecordStream(continuous): recording finished: file save: " + name)
+					log.Info("capture.main.HandleRecordStream(continuous): recording finished: file save: " + name)
 
 					// Cleanup muxer
 					start = false
 
-					// Update the name of the recording with the duration.
-					// We will update the name of the recording with the duration in milliseconds.
-					if mp4Video.VideoTotalDuration > 0 {
-						duration := mp4Video.VideoTotalDuration
-						// Update the name with the duration in milliseconds.
-						startRecordingSeconds := startRecording / 1000      // convert to seconds
-						startRecordingMilliseconds := startRecording % 1000 // convert to milliseconds
-						s := strconv.FormatInt(startRecordingSeconds, 10) + "_" +
-							strconv.Itoa(len(strconv.FormatInt(startRecordingMilliseconds, 10))) + "-" +
-							strconv.FormatInt(startRecordingMilliseconds, 10) + "_" +
-							config.Name + "_" +
-							"0-0-0-0" + "_" + // region coordinates, we
-							"-1" + "_" + // token
-							strconv.FormatInt(int64(duration), 10) // + "_" + // duration of recording
-							//utils.VERSION // version of the agent
-
-						oldName := name
-						name = s + ".mp4"
-						fullName = configDirectory + "/data/recordings/" + name
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
-
-						// Rename the file to the new name.
-						err := os.Rename(
-							configDirectory+"/data/recordings/"+oldName,
-							configDirectory+"/data/recordings/"+s+".mp4")
-
-						if err != nil {
-							log.Log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
-						}
-					} else {
-						log.Log.Info("capture.main.HandleRecordStream(continuous): no video data recorded, not renaming file.")
-					}
-
+					encrypted := false
 					// Check if we need to encrypt the recording.
 					if config.Encryption != nil && config.Encryption.Enabled == "true" && config.Encryption.Recordings == "true" && config.Encryption.SymmetricKey != "" {
 						// reopen file into memory 'fullName'
@@ -207,22 +457,25 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 								// write back to file
 								err := os.WriteFile(fullName, []byte(encryptedContents), 0644)
 								if err != nil {
-									log.Log.Error("capture.main.HandleRecordStream(continuous): error writing file: " + err.Error())
+									log.Error("capture.main.HandleRecordStream(continuous): error writing file: " + err.Error())
+								} else {
+									encrypted = true
 								}
 							} else {
-								log.Log.Error("capture.main.HandleRecordStream(continuous): error encrypting file: " + err.Error())
+								log.Error("capture.main.HandleRecordStream(continuous): error encrypting file: " + err.Error())
 							}
 						} else {
-							log.Log.Error("capture.main.HandleRecordStream(continuous): error reading file: " + err.Error())
+							log.Error("capture.main.HandleRecordStream(continuous): error reading file: " + err.Error())
 						}
 					}
 
-					// Create a symbol link.
-					fc, _ := os.Create(configDirectory + "/data/cloud/" + name)
-					fc.Close()
+					queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config, startRecording, mp4Video, encrypted, "0-0-0-0", -1))
 
 					recordingStatus = "idle"
 
+					// Notify the hub / live-view UI that this camera stopped recording.
+					publishRecordingState(mqttClient, hubKey, configuration, false)
+
 					// Clean up the recording directory if necessary.
 					CleanupRecordingDirectory(configDirectory, configuration)
 				}
@@ -234,39 +487,27 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					// We'll validate those conditions and if not valid we'll not do anything.
 					valid, err := conditions.Validate(loc, configuration)
 					if !valid && err != nil {
-						log.Log.Debug("capture.main.HandleRecordStream(continuous): " + err.Error() + ".")
+						log.Debug("capture.main.HandleRecordStream(continuous): " + err.Error() + ".")
 						time.Sleep(5 * time.Second)
 						continue
 					}
 
 					start = true
 
-					// timestamp_microseconds_instanceName_regionCoordinates_numberOfChanges_token
-					// 1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4
-					// - Timestamp
-					// - Size + - + microseconds
-					// - device
-					// - Region
-					// - Number of changes
-					// - Token
-
 					startRecording = pkt.CurrentTime
-					startRecordingSeconds := startRecording / 1000            // convert to seconds
-					startRecordingMilliseconds := startRecording % 1000       // convert to milliseconds
-					s := strconv.FormatInt(startRecordingSeconds, 10) + "_" + // start timestamp in seconds
-						strconv.Itoa(len(strconv.FormatInt(startRecordingMilliseconds, 10))) + "-" + // length of milliseconds
-						strconv.FormatInt(startRecordingMilliseconds, 10) + "_" + // milliseconds
-						config.Name + "_" + // device name
-						"0-0-0-0" + "_" + // region coordinates, we will not use this for continuous recording
-						"0" + "_" + // token
-						"0" + "_" //+ // duration of recording in milliseconds
-					//utils.VERSION // version of the agent
-
-					name = s + ".mp4"
-					fullName = configDirectory + "/data/recordings/" + name
+					name, err = utils.RecordingFileName(startRecording, config.Key)
+					if err == nil {
+						fullName, err = utils.CreateRecordingPath(configDirectory+"/data/recordings", name, time.UnixMilli(startRecording))
+					}
+					if err != nil {
+						log.WithError(err).Error("capture: could not create continuous recording path")
+						start = false
+						pkt = nextPkt
+						continue
+					}
 
 					// Running...
-					log.Log.Info("capture.main.HandleRecordStream(continuous): recording started")
+					log.Info("capture.main.HandleRecordStream(continuous): recording started")
 
 					// Get width and height from the camera.
 					width := configuration.Config.Capture.IPCamera.Width
@@ -278,7 +519,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					vpsNALUS := configuration.Config.Capture.IPCamera.VPSNALUs
 
 					if len(spsNALUS) == 0 || len(ppsNALUS) == 0 {
-						log.Log.Warning("capture.main.HandleRecordStream(continuous): missing SPS/PPS at recording start")
+						log.Warn("capture.main.HandleRecordStream(continuous): missing SPS/PPS at recording start")
 					}
 					// Create a video file, and set the dimensions.
 					mp4Video = video.NewMP4(fullName, spsNALUS, ppsNALUS, vpsNALUS, configuration.Config.Capture.MaxLengthRecording)
@@ -293,12 +534,15 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					if audioCodec == "AAC" {
 						audioTrack = mp4Video.AddAudioTrack("AAC")
 					} else if audioCodec == "PCM_MULAW" {
-						log.Log.Debug("capture.main.HandleRecordStream(continuous): no AAC audio codec detected, skipping audio track.")
+						log.Debug("capture.main.HandleRecordStream(continuous): no AAC audio codec detected, skipping audio track.")
 					}
 
 					writeSampleToMP4(mp4Video, videoTrack, audioTrack, pkt)
 					recordingStatus = "started"
 
+					// Notify the hub / live-view UI that this camera started recording.
+					publishRecordingState(mqttClient, hubKey, configuration, true)
+
 				} else if start {
 
 					writeSampleToMP4(mp4Video, videoTrack, audioTrack, pkt)
@@ -311,44 +555,12 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 			if cursorError != nil {
 				if recordingStatus == "started" {
 
-					log.Log.Info("capture.main.HandleRecordStream(continuous): Recording finished: file save: " + name)
+					log.Info("capture.main.HandleRecordStream(continuous): Recording finished: file save: " + name)
 
 					// Cleanup muxer
 					start = false
 
-					// Update the name of the recording with the duration.
-					// We will update the name of the recording with the duration in milliseconds.
-					if mp4Video.VideoTotalDuration > 0 {
-						duration := mp4Video.VideoTotalDuration
-						// Update the name with the duration in milliseconds.
-						startRecordingSeconds := startRecording / 1000      // convert to seconds
-						startRecordingMilliseconds := startRecording % 1000 // convert to milliseconds
-						s := strconv.FormatInt(startRecordingSeconds, 10) + "_" +
-							strconv.Itoa(len(strconv.FormatInt(startRecordingMilliseconds, 10))) + "-" +
-							strconv.FormatInt(startRecordingMilliseconds, 10) + "_" +
-							config.Name + "_" +
-							"0-0-0-0" + "_" + // region coordinates, we
-							"-1" + "_" + // token
-							strconv.FormatInt(int64(duration), 10) // + "_" + // duration of recording
-							//utils.VERSION // version of the agent
-
-						oldName := name
-						name = s + ".mp4"
-						fullName = configDirectory + "/data/recordings/" + name
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
-
-						// Rename the file to the new name.
-						err := os.Rename(
-							configDirectory+"/data/recordings/"+oldName,
-							configDirectory+"/data/recordings/"+s+".mp4")
-
-						if err != nil {
-							log.Log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
-						}
-					} else {
-						log.Log.Info("capture.main.HandleRecordStream(continuous): no video data recorded, not renaming file.")
-					}
-
+					encrypted := false
 					// Check if we need to encrypt the recording.
 					if config.Encryption != nil && config.Encryption.Enabled == "true" && config.Encryption.Recordings == "true" && config.Encryption.SymmetricKey != "" {
 						// reopen file into memory 'fullName'
@@ -360,29 +572,32 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 								// write back to file
 								err := os.WriteFile(fullName, []byte(encryptedContents), 0644)
 								if err != nil {
-									log.Log.Error("capture.main.HandleRecordStream(motiondetection): error writing file: " + err.Error())
+									log.Error("capture.main.HandleRecordStream(motiondetection): error writing file: " + err.Error())
+								} else {
+									encrypted = true
 								}
 							} else {
-								log.Log.Error("capture.main.HandleRecordStream(motiondetection): error encrypting file: " + err.Error())
+								log.Error("capture.main.HandleRecordStream(motiondetection): error encrypting file: " + err.Error())
 							}
 						} else {
-							log.Log.Error("capture.main.HandleRecordStream(motiondetection): error reading file: " + err.Error())
+							log.Error("capture.main.HandleRecordStream(motiondetection): error reading file: " + err.Error())
 						}
 					}
 
-					// Create a symbol link.
-					fc, _ := os.Create(configDirectory + "/data/cloud/" + name)
-					fc.Close()
+					queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config, startRecording, mp4Video, encrypted, "0-0-0-0", -1))
 
 					recordingStatus = "idle"
 
+					// Notify the hub / live-view UI that this camera stopped recording.
+					publishRecordingState(mqttClient, hubKey, configuration, false)
+
 					// Clean up the recording directory if necessary.
 					CleanupRecordingDirectory(configDirectory, configuration)
 				}
 			}
 		} else {
 
-			log.Log.Info("capture.main.HandleRecordStream(motiondetection): Start motion based recording ")
+			log.Info("capture.main.HandleRecordStream(motiondetection): Start motion based recording ")
 
 			var lastRecordingTime int64 = 0 // last recording timestamp in milliseconds
 			var displayTime int64 = 0       // display time in milliseconds
@@ -390,7 +605,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 			var videoTrack uint32
 			var audioTrack uint32
 
-			for motion := range communication.HandleMotion {
+			for motion := range motionEvents {
 
 				// Get as much packets we need.
 				var cursorError error
@@ -420,21 +635,11 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 
 				// If startRecording is 0, we will continue as it might be we are in a state of restarting the agent.
 				if startRecording == 0 {
-					log.Log.Info("capture.main.HandleRecordStream(motiondetection): startRecording is 0, we will continue as it might be we are in a state of restarting the agent.")
+					log.Info("capture.main.HandleRecordStream(motiondetection): startRecording is 0, we will continue as it might be we are in a state of restarting the agent.")
 					continue
 				}
 
-				// timestamp_microseconds_instanceName_regionCoordinates_numberOfChanges_token
-				// 1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4
-				// - Timestamp
-				// - Size + - + microseconds
-				// - device
-				// - Region
-				// - Number of changes
-				// - Token
-
-				displayTimeSeconds := displayTime / 1000      // convert to seconds
-				displayTimeMilliseconds := displayTime % 1000 // convert to milliseconds
+				displayTimeSeconds := displayTime / 1000
 				motionRectangleString := "0-0-0-0"
 				if motion.Rectangle.X != 0 || motion.Rectangle.Y != 0 ||
 					motion.Rectangle.Width != 0 || motion.Rectangle.Height != 0 {
@@ -445,20 +650,18 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 				// Get the number of changes from the motion detection.
 				numberOfChanges := motion.NumberOfChanges
 
-				s := strconv.FormatInt(displayTimeSeconds, 10) + "_" + // start timestamp in seconds
-					strconv.Itoa(len(strconv.FormatInt(displayTimeMilliseconds, 10))) + "-" + // length of milliseconds
-					strconv.FormatInt(displayTimeMilliseconds, 10) + "_" + // milliseconds
-					config.Name + "_" + // device name
-					motionRectangleString + "_" + // region coordinates, we will not use this for continuous recording
-					strconv.Itoa(numberOfChanges) + "_" + // number of changes
-					"0" // + "_" + // duration of recording in milliseconds
-					//utils.VERSION // version of the agent
-
-				name := s + ".mp4"
-				fullName := configDirectory + "/data/recordings/" + name
+				name, err := utils.RecordingFileName(displayTime, config.Key)
+				var fullName string
+				if err == nil {
+					fullName, err = utils.CreateRecordingPath(configDirectory+"/data/recordings", name, time.UnixMilli(displayTime))
+				}
+				if err != nil {
+					log.WithError(err).Error("capture: could not create motion recording path")
+					continue
+				}
 
 				// Running...
-				log.Log.Info("capture.main.HandleRecordStream(motiondetection): recording started (" + name + ")" + " at " + strconv.FormatInt(displayTimeSeconds, 10) + " unix")
+				log.Info("capture.main.HandleRecordStream(motiondetection): recording started (" + name + ")" + " at " + strconv.FormatInt(displayTimeSeconds, 10) + " unix")
 
 				// Get width and height from the camera.
 				width := configuration.Config.Capture.IPCamera.Width
@@ -470,7 +673,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 				vpsNALUS := configuration.Config.Capture.IPCamera.VPSNALUs
 
 				if len(spsNALUS) == 0 || len(ppsNALUS) == 0 {
-					log.Log.Warning("capture.main.HandleRecordStream(motiondetection): missing SPS/PPS at recording start")
+					log.Warn("capture.main.HandleRecordStream(motiondetection): missing SPS/PPS at recording start")
 				}
 				// Create the MP4 only once the first keyframe arrives.
 				var mp4Video *video.MP4
@@ -479,30 +682,41 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 
 					nextPkt, cursorError = recordingCursor.ReadPacket()
 					if cursorError != nil {
-						log.Log.Error("capture.main.HandleRecordStream(motiondetection): " + cursorError.Error())
+						log.Error("capture.main.HandleRecordStream(motiondetection): " + cursorError.Error())
 					}
 
 					now = time.Now().UnixMilli()
 					select {
-					case motion := <-communication.HandleMotion:
+					case motion := <-motionEvents:
 						motionTimestamp = now
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): motion detected while recording. Expanding recording.")
+						log.Info("capture.main.HandleRecordStream(motiondetection): motion detected while recording. Expanding recording.")
 						numberOfChanges := motion.NumberOfChanges
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): Received message with recording data, detected changes to save: " + strconv.Itoa(numberOfChanges))
+						log.Info("capture.main.HandleRecordStream(motiondetection): Received message with recording data, detected changes to save: " + strconv.Itoa(numberOfChanges))
 					default:
 					}
 
+					// While a manual recording is active, keep it alive: refresh the
+					// motion timestamp every iteration so the post-recording timeout
+					// never fires. The clip still rolls over at maxRecordingPeriod and
+					// is restarted below, until the viewer stops the manual recording.
+					// It also auto-stops when the viewer's heartbeat lapses (closed page
+					// or idle) or the max remote-recording duration is reached, so a
+					// missed "stop" message can't keep the camera recording forever.
+					if communication.IsRecordingManual.IsSet() && !manualRecordingExpired(communication, now) {
+						motionTimestamp = now
+					}
+
 					if start && (motionTimestamp+postRecording-now < 0 || now-startRecording > maxRecordingPeriod-500) && nextPkt.IsKeyFrame {
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): timestamp+postRecording-now < 0  - " + strconv.FormatInt(motionTimestamp+postRecording-now, 10) + " < 0")
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): now-startRecording > maxRecordingPeriod-500 - " + strconv.FormatInt(now-startRecording, 10) + " > " + strconv.FormatInt(maxRecordingPeriod-500, 10))
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): closing recording (timestamp: " + strconv.FormatInt(motionTimestamp, 10) + ", postRecording: " + strconv.FormatInt(postRecording, 10) + ", now: " + strconv.FormatInt(now, 10) + ", startRecording: " + strconv.FormatInt(startRecording, 10) + ", maxRecordingPeriod: " + strconv.FormatInt(maxRecordingPeriod, 10))
+						log.Info("capture.main.HandleRecordStream(motiondetection): timestamp+postRecording-now < 0  - " + strconv.FormatInt(motionTimestamp+postRecording-now, 10) + " < 0")
+						log.Info("capture.main.HandleRecordStream(motiondetection): now-startRecording > maxRecordingPeriod-500 - " + strconv.FormatInt(now-startRecording, 10) + " > " + strconv.FormatInt(maxRecordingPeriod-500, 10))
+						log.Info("capture.main.HandleRecordStream(motiondetection): closing recording (timestamp: " + strconv.FormatInt(motionTimestamp, 10) + ", postRecording: " + strconv.FormatInt(postRecording, 10) + ", now: " + strconv.FormatInt(now, 10) + ", startRecording: " + strconv.FormatInt(startRecording, 10) + ", maxRecordingPeriod: " + strconv.FormatInt(maxRecordingPeriod, 10))
 						break
 					}
 					if pkt.IsKeyFrame && !start && pkt.CurrentTime >= startRecording {
 						// We start the recording if we have a keyframe and the last duration is 0 or less than the current packet time.
 						// It could be start we start from the beginning of the recording.
-						log.Log.Debug("capture.main.HandleRecordStream(motiondetection): write frames")
-						log.Log.Debug("capture.main.HandleRecordStream(motiondetection): recording started on keyframe")
+						log.Debug("capture.main.HandleRecordStream(motiondetection): write frames")
+						log.Debug("capture.main.HandleRecordStream(motiondetection): recording started on keyframe")
 
 						// Align duration timers with the first keyframe.
 						startRecording = pkt.CurrentTime
@@ -520,9 +734,12 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 						if audioCodec == "AAC" {
 							audioTrack = mp4Video.AddAudioTrack("AAC")
 						} else if audioCodec == "PCM_MULAW" {
-							log.Log.Debug("capture.main.HandleRecordStream(continuous): no AAC audio codec detected, skipping audio track.")
+							log.Debug("capture.main.HandleRecordStream(continuous): no AAC audio codec detected, skipping audio track.")
 						}
 						start = true
+
+						// Notify the hub / live-view UI that this camera started recording.
+						publishRecordingState(mqttClient, hubKey, configuration, true)
 					}
 					if start {
 						writeSampleToMP4(mp4Video, videoTrack, audioTrack, pkt)
@@ -536,7 +753,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 				lastRecordingTime = pkt.CurrentTime
 
 				if mp4Video == nil {
-					log.Log.Warning("capture.main.HandleRecordStream(motiondetection): recording closed without keyframe; no MP4 created")
+					log.Warn("capture.main.HandleRecordStream(motiondetection): recording closed without keyframe; no MP4 created")
 					continue
 				}
 
@@ -552,43 +769,24 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 				}
 				if (videoCodec == "H264" && (len(mp4Video.SPSNALUs) == 0 || len(mp4Video.PPSNALUs) == 0)) ||
 					(videoCodec == "H265" && (len(mp4Video.VPSNALUs) == 0 || len(mp4Video.SPSNALUs) == 0 || len(mp4Video.PPSNALUs) == 0)) {
-					log.Log.Warning("capture.main.HandleRecordStream(motiondetection): closing MP4 without full parameter sets, moov may be incomplete")
+					log.Warn("capture.main.HandleRecordStream(motiondetection): closing MP4 without full parameter sets, moov may be incomplete")
 				}
 				mp4Video.Close(&config)
-				log.Log.Info("capture.main.HandleRecordStream(motiondetection): file save: " + name)
+				log.Info("capture.main.HandleRecordStream(motiondetection): file save: " + name)
 
-				// Update the name of the recording with the duration.
-				// We will update the name of the recording with the duration in milliseconds.
-				if mp4Video.VideoTotalDuration > 0 {
-					duration := mp4Video.VideoTotalDuration
+				// Notify the hub / live-view UI that this camera stopped recording.
+				publishRecordingState(mqttClient, hubKey, configuration, false)
 
-					// Update the name with the duration in milliseconds.
-					s := strconv.FormatInt(displayTimeSeconds, 10) + "_" +
-						strconv.Itoa(len(strconv.FormatInt(displayTimeMilliseconds, 10))) + "-" +
-						strconv.FormatInt(displayTimeMilliseconds, 10) + "_" +
-						config.Name + "_" +
-						motionRectangleString + "_" +
-						strconv.Itoa(numberOfChanges) + "_" + // number of changes
-						strconv.FormatInt(int64(duration), 10) // + "_" + // duration of recording in milliseconds
-						//utils.VERSION // version of the agent
-
-					oldName := name
-					name = s + ".mp4"
-					fullName = configDirectory + "/data/recordings/" + name
-					log.Log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
-
-					// Rename the file to the new name.
-					err := os.Rename(
-						configDirectory+"/data/recordings/"+oldName,
-						configDirectory+"/data/recordings/"+s+".mp4")
-
-					if err != nil {
-						log.Log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
-					}
-				} else {
-					log.Log.Info("capture.main.HandleRecordStream(motiondetection): no video data recorded, not renaming file.")
+				// If the viewer still has a manual recording running, this clip just
+				// rolled over at the max length — immediately kick off the next
+				// segment so recording stays continuous until they stop it. Skip the
+				// restart when the recording has expired (heartbeat lapsed or max
+				// duration reached), so it ends here instead of recording forever.
+				if communication.IsRecordingManual.IsSet() && !manualRecordingExpired(communication, time.Now().UnixMilli()) {
+					communication.TrySendMotion(models.MotionDataPartial{Timestamp: time.Now().Unix(), NumberOfChanges: 100000000})
 				}
 
+				encrypted := false
 				// Check if we need to encrypt the recording.
 				if config.Encryption != nil && config.Encryption.Enabled == "true" && config.Encryption.Recordings == "true" && config.Encryption.SymmetricKey != "" {
 					// reopen file into memory 'fullName'
@@ -600,26 +798,26 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 							// write back to file
 							err := os.WriteFile(fullName, []byte(encryptedContents), 0644)
 							if err != nil {
-								log.Log.Error("capture.main.HandleRecordStream(motiondetection): error writing file: " + err.Error())
+								log.Error("capture.main.HandleRecordStream(motiondetection): error writing file: " + err.Error())
+							} else {
+								encrypted = true
 							}
 						} else {
-							log.Log.Error("capture.main.HandleRecordStream(motiondetection): error encrypting file: " + err.Error())
+							log.Error("capture.main.HandleRecordStream(motiondetection): error encrypting file: " + err.Error())
 						}
 					} else {
-						log.Log.Error("capture.main.HandleRecordStream(motiondetection): error reading file: " + err.Error())
+						log.Error("capture.main.HandleRecordStream(motiondetection): error reading file: " + err.Error())
 					}
 				}
 
-				// Create a symbol linc.
-				fc, _ := os.Create(configDirectory + "/data/cloud/" + name)
-				fc.Close()
+				queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config, displayTime, mp4Video, encrypted, motionRectangleString, numberOfChanges))
 
 				// Clean up the recording directory if necessary.
 				CleanupRecordingDirectory(configDirectory, configuration)
 			}
 		}
 
-		log.Log.Debug("capture.main.HandleRecordStream(): finished")
+		log.Debug("capture.main.HandleRecordStream(): finished")
 	}
 }
 
@@ -713,13 +911,14 @@ func Base64Image(captureDevice *Capture, communication *models.Communication, co
 	var cursor *packets.QueueCursor
 
 	// We'll pick the right client and decoder.
-	rtspClient := captureDevice.RTSPSubClient
+	rtspClient := captureDevice.SubClient()
 	if rtspClient != nil {
-		queue = communication.SubQueue
-		cursor = queue.Latest()
+		queue = communication.SubQueue()
 	} else {
-		rtspClient = captureDevice.RTSPClient
-		queue = communication.Queue
+		rtspClient = captureDevice.MainClient()
+		queue = communication.MainQueue()
+	}
+	if queue != nil {
 		cursor = queue.Latest()
 	}
 
@@ -745,6 +944,10 @@ func Base64Image(captureDevice *Capture, communication *models.Communication, co
 					count++
 					continue
 				}
+			} else {
+				// A stopped run closes this queue permanently. Retrying its
+				// cursor would spin on EOF even after the HTTP client leaves.
+				break
 			}
 		} else {
 			break
@@ -759,13 +962,14 @@ func JpegImage(captureDevice *Capture, communication *models.Communication) imag
 	var cursor *packets.QueueCursor
 
 	// We'll pick the right client and decoder.
-	rtspClient := captureDevice.RTSPSubClient
+	rtspClient := captureDevice.SubClient()
 	if rtspClient != nil {
-		queue = communication.SubQueue
-		cursor = queue.Latest()
+		queue = communication.SubQueue()
 	} else {
-		rtspClient = captureDevice.RTSPClient
-		queue = communication.Queue
+		rtspClient = captureDevice.MainClient()
+		queue = communication.MainQueue()
+	}
+	if queue != nil {
 		cursor = queue.Latest()
 	}
 
@@ -787,6 +991,10 @@ func JpegImage(captureDevice *Capture, communication *models.Communication) imag
 				} else {
 					break
 				}
+			} else {
+				// A stopped run closes this queue permanently. Retrying its
+				// cursor would spin on EOF even after the HTTP client leaves.
+				break
 			}
 		} else {
 			break
@@ -820,20 +1028,16 @@ func writeSampleToMP4(mp4Video *video.MP4, videoTrack, audioTrack uint32, pkt pa
 			dts = pts - uint64(compositionOffset)
 		}
 		if err := mp4Video.AddSampleToTrack(videoTrack, pkt.IsKeyFrame, pkt.Data, dts, compositionOffset); err != nil {
-			log.Log.Error("capture.main.writeSampleToMP4(): " + err.Error())
+			log.Error("capture.main.writeSampleToMP4(): " + err.Error())
 		}
 	} else if pkt.IsAudio {
 		if pkt.Codec == "AAC" {
 			if err := mp4Video.AddSampleToTrack(audioTrack, pkt.IsKeyFrame, pkt.Data, pts, 0); err != nil {
-				log.Log.Error("capture.main.writeSampleToMP4(): " + err.Error())
+				log.Error("capture.main.writeSampleToMP4(): " + err.Error())
 			}
 		} else if pkt.Codec == "PCM_MULAW" {
 			// TODO: transcode to AAC, some work to do..
-			log.Log.Debug("capture.main.writeSampleToMP4(): no AAC audio codec detected, skipping audio track.")
+			log.Debug("capture.main.writeSampleToMP4(): no AAC audio codec detected, skipping audio track.")
 		}
 	}
 }
-
-/*func convertPTS2(v int64) uint64 {
-	return uint64(v) / 100
-}*/

machinery/src/capture/main_test.go

@@ -0,0 +1,90 @@
+package capture
+
+import (
+	"encoding/json"
+	"math"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/video"
+)
+
+func TestPTSToDuration(t *testing.T) {
+	tests := []struct {
+		name      string
+		pts       int64
+		clockRate int
+		want      time.Duration
+	}{
+		{name: "one video second", pts: 90_000, clockRate: 90_000, want: time.Second},
+		{name: "one audio frame", pts: 1_024, clockRate: 8_000, want: 128 * time.Millisecond},
+		{name: "fractional millisecond", pts: 45_045, clockRate: 90_000, want: 500*time.Millisecond + 500*time.Microsecond},
+		{name: "negative timestamp", pts: -45_045, clockRate: 90_000, want: -500*time.Millisecond - 500*time.Microsecond},
+		{name: "large timestamp", pts: 90_000 * 60 * 60 * 24, clockRate: 90_000, want: 24 * time.Hour},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if got := ptsToDuration(test.pts, test.clockRate); got != test.want {
+				t.Fatalf("ptsToDuration(%d, %d) = %s, want %s", test.pts, test.clockRate, got, test.want)
+			}
+		})
+	}
+}
+
+func TestQueueRecordingForUploadStoresFinalizedMetadata(t *testing.T) {
+	configDirectory := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(configDirectory, "data", "cloud"), 0o755); err != nil {
+		t.Fatalf("mkdir cloud queue: %v", err)
+	}
+
+	mp4Video := &video.MP4{VideoTotalDuration: 20452, SampleCount: 613}
+	config := models.Config{Key: "device-key", Name: "camera-name"}
+	metadata := recordingUploadMetadata("1785934709414-device-key.mp4", config, 1785934709414, mp4Video, true, "1-2-3-4", 57)
+	queueRecordingForUpload(configDirectory, metadata)
+
+	got, err := os.ReadFile(filepath.Join(configDirectory, "data", "cloud", "1785934709414-device-key.metadata"))
+	if err != nil {
+		t.Fatalf("read upload marker: %v", err)
+	}
+	var stored models.RecordingUploadMetadata
+	if err := json.Unmarshal(got, &stored); err != nil {
+		t.Fatalf("decode upload marker: %v", err)
+	}
+	expectedFPS := mp4Video.AverageFPS()
+	if stored.FileName != "1785934709414-device-key.mp4" || stored.DeviceKey != "device-key" || stored.Timestamp != 1785934709414 || stored.Duration != 20452 || math.Abs(stored.FPS-expectedFPS) > 1e-9 || !stored.Encrypted ||
+		stored.DeviceName != "camera-name" || stored.RegionCoordinates != "1-2-3-4" || stored.NumberOfChanges != "57" {
+		t.Fatalf("upload marker = %+v", stored)
+	}
+	if stored.FPS == math.Floor(stored.FPS) {
+		t.Fatalf("upload marker FPS = %v, want fractional precision", stored.FPS)
+	}
+}
+
+func TestQueueRecordingForUploadKeepsUnknownFPSCompatible(t *testing.T) {
+	for _, fps := range []float64{0, 0.99, -1, math.NaN(), math.Inf(1), 241} {
+		t.Run("invalid FPS", func(t *testing.T) {
+			configDirectory := t.TempDir()
+			if err := os.MkdirAll(filepath.Join(configDirectory, "data", "cloud"), 0o755); err != nil {
+				t.Fatalf("mkdir cloud queue: %v", err)
+			}
+
+			metadata := models.RecordingUploadMetadata{FileName: "recording.mp4"}
+			if fps >= 1 && fps <= 240 && !math.IsNaN(fps) && !math.IsInf(fps, 0) {
+				metadata.FPS = fps
+			}
+			queueRecordingForUpload(configDirectory, metadata)
+
+			got, err := os.ReadFile(filepath.Join(configDirectory, "data", "cloud", "recording.metadata"))
+			if err != nil {
+				t.Fatalf("read upload marker: %v", err)
+			}
+			if string(got) != `{"filename":"recording.mp4","device_key":"","timestamp":0,"duration":0}` {
+				t.Fatalf("upload marker = %q, want metadata without FPS", got)
+			}
+		})
+	}
+}

machinery/src/capture/rtsp_client.go

@@ -3,36 +3,73 @@ package capture
 import (
 	"context"
 	"image"
+	"sync"
 
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 )
 
 type Capture struct {
-	RTSPClient            *Golibrtsp
-	RTSPSubClient         *Golibrtsp
-	RTSPBackChannelClient *Golibrtsp
+	clientsMu             sync.RWMutex
+	rtspClient            *Golibrtsp
+	rtspSubClient         *Golibrtsp
+	rtspBackChannelClient *Golibrtsp
 }
 
 func (c *Capture) SetMainClient(rtspUrl string) *Golibrtsp {
-	c.RTSPClient = &Golibrtsp{
+	client := &Golibrtsp{
 		Url: rtspUrl,
 	}
-	return c.RTSPClient
+	c.clientsMu.Lock()
+	c.rtspClient = client
+	c.clientsMu.Unlock()
+	return client
 }
 
 func (c *Capture) SetSubClient(rtspUrl string) *Golibrtsp {
-	c.RTSPSubClient = &Golibrtsp{
+	client := &Golibrtsp{
 		Url: rtspUrl,
 	}
-	return c.RTSPSubClient
+	c.clientsMu.Lock()
+	c.rtspSubClient = client
+	c.clientsMu.Unlock()
+	return client
 }
 
 func (c *Capture) SetBackChannelClient(rtspUrl string) *Golibrtsp {
-	c.RTSPBackChannelClient = &Golibrtsp{
+	client := &Golibrtsp{
 		Url: rtspUrl,
 	}
-	return c.RTSPBackChannelClient
+	c.clientsMu.Lock()
+	c.rtspBackChannelClient = client
+	c.clientsMu.Unlock()
+	return client
+}
+
+func (c *Capture) MainClient() *Golibrtsp {
+	c.clientsMu.RLock()
+	defer c.clientsMu.RUnlock()
+	return c.rtspClient
+}
+
+func (c *Capture) SubClient() *Golibrtsp {
+	c.clientsMu.RLock()
+	defer c.clientsMu.RUnlock()
+	return c.rtspSubClient
+}
+
+func (c *Capture) ClearClients(main, sub, backchannel *Golibrtsp) {
+	c.clientsMu.Lock()
+	defer c.clientsMu.Unlock()
+	if c.rtspClient == main {
+		c.rtspClient = nil
+	}
+	if c.rtspSubClient == sub {
+		c.rtspSubClient = nil
+	}
+	if c.rtspBackChannelClient == backchannel {
+		c.rtspBackChannelClient = nil
+	}
 }
 
 // RTSPClient is a interface that abstracts the RTSP client implementation.

machinery/src/capture/rtsp_client_test.go

@@ -0,0 +1,62 @@
+package capture
+
+import (
+	"strconv"
+	"sync"
+	"testing"
+)
+
+func TestCaptureClientAccessCanRaceReplacement(t *testing.T) {
+	captureDevice := &Capture{}
+	captureDevice.SetMainClient("rtsp://main/0")
+	captureDevice.SetSubClient("rtsp://sub/0")
+
+	var workers sync.WaitGroup
+	workers.Add(2)
+	go func() {
+		defer workers.Done()
+		for replacement := 1; replacement <= 1000; replacement++ {
+			suffix := strconv.Itoa(replacement)
+			captureDevice.SetMainClient("rtsp://main/" + suffix)
+			captureDevice.SetSubClient("rtsp://sub/" + suffix)
+		}
+	}()
+	go func() {
+		defer workers.Done()
+		for snapshot := 0; snapshot < 1000; snapshot++ {
+			if captureDevice.MainClient() == nil {
+				t.Error("MainClient() returned nil")
+				return
+			}
+			if captureDevice.SubClient() == nil {
+				t.Error("SubClient() returned nil")
+				return
+			}
+		}
+	}()
+	workers.Wait()
+}
+
+func TestCaptureClearClientsOnlyClearsMatchingRun(t *testing.T) {
+	captureDevice := &Capture{}
+	oldMain := captureDevice.SetMainClient("rtsp://main/old")
+	oldSub := captureDevice.SetSubClient("rtsp://sub/old")
+	oldBackchannel := captureDevice.SetBackChannelClient("rtsp://back/old")
+
+	newMain := captureDevice.SetMainClient("rtsp://main/new")
+	newSub := captureDevice.SetSubClient("rtsp://sub/new")
+	newBackchannel := captureDevice.SetBackChannelClient("rtsp://back/new")
+
+	captureDevice.ClearClients(oldMain, oldSub, oldBackchannel)
+	if captureDevice.MainClient() != newMain {
+		t.Fatal("stale cleanup cleared the new main client")
+	}
+	if captureDevice.SubClient() != newSub {
+		t.Fatal("stale cleanup cleared the new sub client")
+	}
+
+	captureDevice.ClearClients(newMain, newSub, newBackchannel)
+	if captureDevice.MainClient() != nil || captureDevice.SubClient() != nil {
+		t.Fatal("matching cleanup did not clear current clients")
+	}
+}

machinery/src/capture/snapshot_test.go

@@ -0,0 +1,68 @@
+package capture
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/packets"
+)
+
+func TestSnapshotReturnsAfterQueueClosure(t *testing.T) {
+	for _, stream := range []string{"main", "sub"} {
+		for _, format := range []string{"jpeg", "base64"} {
+			for _, alreadyClosed := range []bool{true, false} {
+				name := stream + "/" + format + "/closed_during_read"
+				if alreadyClosed {
+					name = stream + "/" + format + "/already_closed"
+				}
+				t.Run(name, func(t *testing.T) {
+					communication := &models.Communication{}
+					run := models.NewAgentRun(context.Background(), communication, false)
+					queue := packets.NewQueue()
+					captureDevice := &Capture{}
+					if stream == "main" {
+						captureDevice.SetMainClient("rtsp://unused.invalid/main")
+						run.SetMainQueue(queue)
+					} else {
+						captureDevice.SetSubClient("rtsp://unused.invalid/sub")
+						run.SetSubQueue(queue)
+					}
+					if err := run.Activate(); err != nil {
+						t.Fatal(err)
+					}
+					defer queue.Close()
+					if alreadyClosed {
+						queue.Close()
+					}
+					done := make(chan bool, 1)
+					go func() {
+						if format == "jpeg" {
+							image := JpegImage(captureDevice, communication)
+							done <- image.Rect.Empty()
+						} else {
+							done <- Base64Image(captureDevice, communication, &models.Configuration{}) == ""
+						}
+					}()
+					if !alreadyClosed {
+						select {
+						case <-done:
+							t.Fatal("snapshot returned before a frame or queue closure")
+						case <-time.After(10 * time.Millisecond):
+						}
+						queue.Close()
+					}
+					select {
+					case empty := <-done:
+						if !empty {
+							t.Fatal("closed empty queue produced a snapshot")
+						}
+					case <-time.After(time.Second):
+						t.Fatal("snapshot did not return after its packet queue closed")
+					}
+				})
+			}
+		}
+	}
+}

machinery/src/cloud/cloud_test.go

@@ -0,0 +1,849 @@
+package cloud
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/kerberos-io/agent/machinery/src/models"
+	agentonvif "github.com/kerberos-io/agent/machinery/src/onvif"
+	goonvif "github.com/kerberos-io/onvif"
+	goonvifdevice "github.com/kerberos-io/onvif/device"
+	goonvifptz "github.com/kerberos-io/onvif/ptz"
+	goonvifxsd "github.com/kerberos-io/onvif/xsd/onvif"
+)
+
+func TestVerifyPersistenceKerberosHubUsesAuthorizationHead(t *testing.T) {
+	var requestMethod string
+	var requestBody []byte
+	var requestHeaders http.Header
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requestMethod = r.Method
+		requestHeaders = r.Header.Clone()
+		requestBody, _ = io.ReadAll(r.Body)
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer server.Close()
+
+	config := models.Config{
+		Cloud:         "kerberoshub",
+		Key:           "camera-1",
+		Name:          "front-door",
+		HubURI:        server.URL,
+		HubKey:        "project-public",
+		HubPrivateKey: "project-private",
+		S3:            &models.S3{Region: "eu"},
+	}
+	payload, err := json.Marshal(config)
+	if err != nil {
+		t.Fatalf("json.Marshal() error = %v", err)
+	}
+
+	gin.SetMode(gin.TestMode)
+	recorder := httptest.NewRecorder()
+	ctx, _ := gin.CreateTestContext(recorder)
+	ctx.Request = httptest.NewRequest(http.MethodPost, "/api/persistence/verify", bytes.NewReader(payload))
+	ctx.Request.Header.Set("Content-Type", "application/json")
+
+	VerifyPersistence(ctx, t.TempDir(), server.URL)
+
+	if recorder.Code != http.StatusOK {
+		t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String())
+	}
+	if requestMethod != http.MethodHead {
+		t.Fatalf("Hub request method = %q, want HEAD", requestMethod)
+	}
+	if len(requestBody) != 0 {
+		t.Fatalf("Hub request body has %d bytes, want none", len(requestBody))
+	}
+	for header, want := range map[string]string{
+		"X-Kerberos-Storage-Device":  "camera-1",
+		"X-Kerberos-Hub-PublicKey":   "project-public",
+		"X-Kerberos-Hub-PrivateKey":  "project-private",
+		"X-Kerberos-Hub-Region":      "eu",
+		"X-Kerberos-Storage-Capture": "IPCamera",
+	} {
+		if got := requestHeaders.Get(header); got != want {
+			t.Errorf("%s = %q, want %q", header, got, want)
+		}
+	}
+}
+
+func TestVerifyPersistenceKerberosHubRejectsCrossHostRedirect(t *testing.T) {
+	var redirectedHeaders http.Header
+	target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		redirectedHeaders = r.Header.Clone()
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer target.Close()
+
+	var originalHeaders http.Header
+	hub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		originalHeaders = r.Header.Clone()
+		http.Redirect(w, r, target.URL+"/storage/upload", http.StatusTemporaryRedirect)
+	}))
+	defer hub.Close()
+
+	config := models.Config{
+		Cloud:         "kerberoshub",
+		Key:           "camera-1",
+		Name:          "front-door",
+		HubURI:        hub.URL,
+		HubKey:        "project-public",
+		HubPrivateKey: "project-private",
+		S3:            &models.S3{Region: "eu"},
+	}
+	payload, err := json.Marshal(config)
+	if err != nil {
+		t.Fatalf("json.Marshal() error = %v", err)
+	}
+
+	gin.SetMode(gin.TestMode)
+	recorder := httptest.NewRecorder()
+	ctx, _ := gin.CreateTestContext(recorder)
+	ctx.Request = httptest.NewRequest(http.MethodPost, "/api/persistence/verify", bytes.NewReader(payload))
+	ctx.Request.Header.Set("Content-Type", "application/json")
+
+	VerifyPersistence(ctx, t.TempDir(), hub.URL)
+
+	if recorder.Code != http.StatusBadRequest {
+		t.Fatalf("status = %d, want %d; body = %s", recorder.Code, http.StatusBadRequest, recorder.Body.String())
+	}
+	for _, header := range []string{"X-Kerberos-Hub-PublicKey", "X-Kerberos-Hub-PrivateKey"} {
+		if originalHeaders.Get(header) == "" {
+			t.Errorf("%s was not sent to the configured Hub", header)
+		}
+		if redirectedHeaders != nil {
+			t.Errorf("redirect target received a request with %s=%q", header, redirectedHeaders.Get(header))
+		}
+	}
+}
+
+func TestVerifyPersistenceKerberosHubRejectsUnsavedURI(t *testing.T) {
+	requestCount := 0
+	untrusted := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		requestCount++
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer untrusted.Close()
+
+	config := models.Config{
+		Cloud:         "kerberoshub",
+		Key:           "camera-1",
+		Name:          "front-door",
+		HubURI:        untrusted.URL,
+		HubKey:        "project-public",
+		HubPrivateKey: "project-private",
+		S3:            &models.S3{Region: "eu"},
+	}
+	payload, err := json.Marshal(config)
+	if err != nil {
+		t.Fatalf("json.Marshal() error = %v", err)
+	}
+
+	gin.SetMode(gin.TestMode)
+	recorder := httptest.NewRecorder()
+	ctx, _ := gin.CreateTestContext(recorder)
+	ctx.Request = httptest.NewRequest(http.MethodPost, "/api/persistence/verify", bytes.NewReader(payload))
+	ctx.Request.Header.Set("Content-Type", "application/json")
+
+	VerifyPersistence(ctx, t.TempDir(), "https://saved-hub.example.com")
+
+	if recorder.Code != http.StatusBadRequest {
+		t.Fatalf("status = %d, want %d; body = %s", recorder.Code, http.StatusBadRequest, recorder.Body.String())
+	}
+	if requestCount != 0 {
+		t.Fatalf("untrusted Hub received %d requests, want none", requestCount)
+	}
+}
+
+func TestVerifyHubRejectsUnsavedURI(t *testing.T) {
+	requestCount := 0
+	untrusted := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		requestCount++
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer untrusted.Close()
+
+	config := models.Config{
+		HubURI:        untrusted.URL,
+		HubKey:        "project-public",
+		HubPrivateKey: "project-private",
+	}
+	payload, err := json.Marshal(config)
+	if err != nil {
+		t.Fatalf("json.Marshal() error = %v", err)
+	}
+
+	gin.SetMode(gin.TestMode)
+	recorder := httptest.NewRecorder()
+	ctx, _ := gin.CreateTestContext(recorder)
+	ctx.Request = httptest.NewRequest(http.MethodPost, "/api/hub/verify", bytes.NewReader(payload))
+	ctx.Request.Header.Set("Content-Type", "application/json")
+
+	VerifyHub(ctx, "https://saved-hub.example.com")
+
+	if recorder.Code != http.StatusBadRequest {
+		t.Fatalf("status = %d, want %d; body = %s", recorder.Code, http.StatusBadRequest, recorder.Body.String())
+	}
+	if requestCount != 0 {
+		t.Fatalf("untrusted Hub received %d requests, want none", requestCount)
+	}
+}
+
+func TestHubPersistenceVerificationErrorExplainsStatus(t *testing.T) {
+	for _, test := range []struct {
+		status int
+		want   string
+	}{
+		{status: http.StatusBadRequest, want: "credentials or device/project assignment"},
+		{status: http.StatusUnauthorized, want: "no active subscription"},
+		{status: http.StatusForbidden, want: "device is muted"},
+		{status: http.StatusBadGateway, want: "HTTP 502"},
+	} {
+		t.Run(http.StatusText(test.status), func(t *testing.T) {
+			if got := hubPersistenceVerificationError(test.status); !strings.Contains(got, test.want) {
+				t.Fatalf("hubPersistenceVerificationError(%d) = %q, want it to contain %q", test.status, got, test.want)
+			}
+		})
+	}
+}
+
+func TestHeartbeatFailureLogOmitsHubResponseBody(t *testing.T) {
+	response := &http.Response{
+		StatusCode: http.StatusBadRequest,
+		Status:     "400 Bad Request",
+		Body:       io.NopCloser(strings.NewReader(`{"error":"invalid heartbeat"}`)),
+	}
+
+	responseBody, truncated, err := readHeartbeatResponseBody(response)
+	if err != nil {
+		t.Fatalf("readHeartbeatResponseBody() error = %v", err)
+	}
+	fields := heartbeatFailureLogFields(response, responseBody, truncated, 125*time.Millisecond)
+
+	for key, want := range map[string]interface{}{
+		"duration_ms":             int64(125),
+		"response_body_bytes":     len(responseBody),
+		"response_body_truncated": false,
+		"status_code":             http.StatusBadRequest,
+	} {
+		if got := fields[key]; got != want {
+			t.Errorf("%s = %v, want %v", key, got, want)
+		}
+	}
+	for key, value := range fields {
+		if strings.Contains(key, "response_body") && key != "response_body_bytes" && key != "response_body_truncated" {
+			t.Fatalf("unexpected response body field %q=%v", key, value)
+		}
+		if strings.Contains(fmt.Sprint(value), "invalid heartbeat") {
+			t.Fatalf("heartbeat log fields exposed response body in %q=%v", key, value)
+		}
+	}
+}
+
+func TestReadHeartbeatResponseBodyTruncatesLargeBody(t *testing.T) {
+	response := &http.Response{
+		Body: io.NopCloser(strings.NewReader(strings.Repeat("x", heartbeatResponseBodyLogLimit+1))),
+	}
+
+	body, truncated, err := readHeartbeatResponseBody(response)
+	if err != nil {
+		t.Fatalf("readHeartbeatResponseBody() error = %v", err)
+	}
+	if !truncated {
+		t.Fatal("readHeartbeatResponseBody() truncated = false, want true")
+	}
+	if len(body) != heartbeatResponseBodyLogLimit {
+		t.Fatalf("len(body) = %d, want %d", len(body), heartbeatResponseBodyLogLimit)
+	}
+}
+
+func TestHeartbeatONVIFPayloadCachesStaticAndReusesLoopSubscription(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	device := newTestONVIFDevice()
+	camera := models.IPCamera{
+		ONVIFXAddr:    "http://camera/onvif",
+		ONVIFUsername: "operator",
+		ONVIFPassword: "secret",
+	}
+	initialEvents := []agentonvif.ONVIFEvents{{Key: "input-1", Type: "input", Value: "true", Timestamp: 1}}
+	loopEvents := []agentonvif.ONVIFEvents{{Key: "output-1", Type: "output", Value: "false", Timestamp: 2}}
+	wantPresets := mustJSONMarshal(t, []models.OnvifActionPreset{{Name: "Lobby", Token: "1"}})
+	wantInitialEvents := mustJSONMarshal(t, initialEvents)
+	wantLoopEvents := mustJSONMarshal(t, loopEvents)
+
+	var connectCalls, ptzConfigCalls, ptzFunctionCalls, presetCalls, createCalls, eventCalls, unsubscribeCalls int
+
+	heartbeatConnectToONVIFDevice = func(*models.IPCamera) (*goonvif.Device, goonvifdevice.GetCapabilitiesResponse, error) {
+		connectCalls++
+		return device, goonvifdevice.GetCapabilitiesResponse{}, nil
+	}
+	heartbeatGetPTZConfigurationsFromDevice = func(*goonvif.Device) (goonvifptz.GetConfigurationsResponse, error) {
+		ptzConfigCalls++
+		return goonvifptz.GetConfigurationsResponse{}, nil
+	}
+	heartbeatGetPTZFunctionsFromDevice = func(goonvifptz.GetConfigurationsResponse) ([]string, bool, bool) {
+		ptzFunctionCalls++
+		return nil, true, true
+	}
+	heartbeatGetPresetsFromDevice = func(*goonvif.Device, int) ([]models.OnvifActionPreset, error) {
+		presetCalls++
+		return []models.OnvifActionPreset{{Name: "Lobby", Token: "1"}}, nil
+	}
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		createCalls++
+		switch createCalls {
+		case 1:
+			return "initial-1", nil
+		case 2:
+			return "loop", nil
+		case 3:
+			return "initial-2", nil
+		default:
+			t.Fatalf("unexpected create pull point call %d", createCalls)
+			return "", nil
+		}
+	}
+	heartbeatGetEventMessages = func(_ *goonvif.Device, pullPointAddress string) ([]agentonvif.ONVIFEvents, error) {
+		eventCalls++
+		switch pullPointAddress {
+		case "initial-1", "initial-2":
+			return initialEvents, nil
+		case "loop":
+			return loopEvents, nil
+		default:
+			t.Fatalf("unexpected pull point address %q", pullPointAddress)
+			return nil, nil
+		}
+	}
+	heartbeatUnsubscribePullPoint = func(_ *goonvif.Device, pullPointAddress string) error {
+		unsubscribeCalls++
+		if pullPointAddress != "initial-1" && pullPointAddress != "initial-2" {
+			t.Fatalf("unexpected unsubscribe pull point %q", pullPointAddress)
+		}
+		return nil
+	}
+
+	state := newHeartbeatONVIFState()
+
+	payload := getHeartbeatONVIFPayload(camera, state)
+	if payload.enabled != "true" || payload.zoom != "true" || payload.panTilt != "true" || payload.presets != "true" {
+		t.Fatalf("unexpected static payload: %+v", payload)
+	}
+	if string(payload.presetsList) != string(wantPresets) {
+		t.Fatalf("payload.presetsList = %s, want %s", payload.presetsList, wantPresets)
+	}
+	if string(payload.eventsList) != string(wantInitialEvents) {
+		t.Fatalf("payload.eventsList = %s, want %s", payload.eventsList, wantInitialEvents)
+	}
+	if connectCalls != 1 || ptzConfigCalls != 1 || ptzFunctionCalls != 1 || presetCalls != 1 {
+		t.Fatalf("unexpected static call counts after first cycle: connect=%d ptzConfig=%d ptzFunctions=%d presets=%d", connectCalls, ptzConfigCalls, ptzFunctionCalls, presetCalls)
+	}
+	if createCalls != 2 || eventCalls != 1 || unsubscribeCalls != 1 {
+		t.Fatalf("unexpected event call counts after first cycle: create=%d events=%d unsubscribe=%d", createCalls, eventCalls, unsubscribeCalls)
+	}
+
+	payload = getHeartbeatONVIFPayload(camera, state)
+	if string(payload.eventsList) != string(wantLoopEvents) {
+		t.Fatalf("second payload.eventsList = %s, want %s", payload.eventsList, wantLoopEvents)
+	}
+	if connectCalls != 1 {
+		t.Fatalf("connectCalls = %d, want 1", connectCalls)
+	}
+	if ptzConfigCalls != 1 || ptzFunctionCalls != 1 || presetCalls != 1 {
+		t.Fatalf("static calls were not cached: ptzConfig=%d ptzFunctions=%d presets=%d", ptzConfigCalls, ptzFunctionCalls, presetCalls)
+	}
+	if createCalls != 3 || eventCalls != 3 || unsubscribeCalls != 2 {
+		t.Fatalf("temporary state subscription was not refreshed or loop subscription was not reused: create=%d events=%d unsubscribe=%d", createCalls, eventCalls, unsubscribeCalls)
+	}
+}
+
+func TestHeartbeatONVIFPayloadRefreshesAfterCameraConfigChange(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	deviceA := newTestONVIFDevice()
+	deviceB := newTestONVIFDevice()
+	cameraA := models.IPCamera{ONVIFXAddr: "http://camera-a/onvif", ONVIFUsername: "user", ONVIFPassword: "secret-a"}
+	cameraB := models.IPCamera{ONVIFXAddr: "http://camera-b/onvif", ONVIFUsername: "user", ONVIFPassword: "secret-b"}
+	var connectCalls, ptzConfigCalls, presetCalls, createCalls int
+	var unsubscribed []string
+
+	heartbeatConnectToONVIFDevice = func(camera *models.IPCamera) (*goonvif.Device, goonvifdevice.GetCapabilitiesResponse, error) {
+		connectCalls++
+		switch camera.ONVIFXAddr {
+		case cameraA.ONVIFXAddr:
+			return deviceA, goonvifdevice.GetCapabilitiesResponse{}, nil
+		case cameraB.ONVIFXAddr:
+			return deviceB, goonvifdevice.GetCapabilitiesResponse{}, nil
+		default:
+			t.Fatalf("unexpected camera address %q", camera.ONVIFXAddr)
+			return nil, goonvifdevice.GetCapabilitiesResponse{}, nil
+		}
+	}
+	heartbeatGetPTZConfigurationsFromDevice = func(*goonvif.Device) (goonvifptz.GetConfigurationsResponse, error) {
+		ptzConfigCalls++
+		return goonvifptz.GetConfigurationsResponse{}, nil
+	}
+	heartbeatGetPTZFunctionsFromDevice = func(goonvifptz.GetConfigurationsResponse) ([]string, bool, bool) {
+		return nil, false, true
+	}
+	heartbeatGetPresetsFromDevice = func(*goonvif.Device, int) ([]models.OnvifActionPreset, error) {
+		presetCalls++
+		return nil, nil
+	}
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		createCalls++
+		switch createCalls {
+		case 1:
+			return "initial-a", nil
+		case 2:
+			return "loop-a", nil
+		case 3:
+			return "initial-b", nil
+		case 4:
+			return "loop-b", nil
+		default:
+			t.Fatalf("unexpected create pull point call %d", createCalls)
+			return "", nil
+		}
+	}
+	heartbeatGetEventMessages = func(_ *goonvif.Device, pullPointAddress string) ([]agentonvif.ONVIFEvents, error) {
+		switch pullPointAddress {
+		case "initial-a":
+			return []agentonvif.ONVIFEvents{{Key: "a", Type: "input", Value: "true", Timestamp: 1}}, nil
+		case "initial-b":
+			return []agentonvif.ONVIFEvents{{Key: "b", Type: "input", Value: "false", Timestamp: 2}}, nil
+		default:
+			t.Fatalf("unexpected pull point address %q", pullPointAddress)
+			return nil, nil
+		}
+	}
+	heartbeatUnsubscribePullPoint = func(_ *goonvif.Device, pullPointAddress string) error {
+		unsubscribed = append(unsubscribed, pullPointAddress)
+		return nil
+	}
+
+	state := newHeartbeatONVIFState()
+	_ = getHeartbeatONVIFPayload(cameraA, state)
+	payload := getHeartbeatONVIFPayload(cameraB, state)
+
+	if connectCalls != 2 {
+		t.Fatalf("connectCalls = %d, want 2", connectCalls)
+	}
+	if ptzConfigCalls != 2 || presetCalls != 2 {
+		t.Fatalf("camera config change did not refresh static state: ptzConfig=%d presets=%d", ptzConfigCalls, presetCalls)
+	}
+	if createCalls != 4 {
+		t.Fatalf("camera config change did not recreate subscriptions: create=%d", createCalls)
+	}
+	if countString(unsubscribed, "loop-a") != 1 {
+		t.Fatalf("unsubscribed loop-a %d times, want 1; unsubscribed=%v", countString(unsubscribed, "loop-a"), unsubscribed)
+	}
+	wantEvents := mustJSONMarshal(t, []agentonvif.ONVIFEvents{{Key: "b", Type: "input", Value: "false", Timestamp: 2}})
+	if string(payload.eventsList) != string(wantEvents) {
+		t.Fatalf("payload.eventsList = %s, want %s", payload.eventsList, wantEvents)
+	}
+}
+
+func TestHeartbeatONVIFPayloadRetriesStaticFetchFailuresWithoutReconnect(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	device := newTestONVIFDevice()
+	camera := models.IPCamera{
+		ONVIFXAddr:    "http://camera/onvif",
+		ONVIFUsername: "operator",
+		ONVIFPassword: "secret",
+	}
+	var connectCalls, ptzConfigCalls, presetCalls, createCalls, eventCalls int
+
+	heartbeatConnectToONVIFDevice = func(*models.IPCamera) (*goonvif.Device, goonvifdevice.GetCapabilitiesResponse, error) {
+		connectCalls++
+		return device, goonvifdevice.GetCapabilitiesResponse{}, nil
+	}
+	heartbeatGetPTZConfigurationsFromDevice = func(*goonvif.Device) (goonvifptz.GetConfigurationsResponse, error) {
+		ptzConfigCalls++
+		return goonvifptz.GetConfigurationsResponse{}, nil
+	}
+	heartbeatGetPTZFunctionsFromDevice = func(goonvifptz.GetConfigurationsResponse) ([]string, bool, bool) {
+		return nil, true, true
+	}
+	heartbeatGetPresetsFromDevice = func(*goonvif.Device, int) ([]models.OnvifActionPreset, error) {
+		presetCalls++
+		if presetCalls == 1 {
+			return nil, errors.New("temporary preset failure")
+		}
+		return []models.OnvifActionPreset{{Name: "Lobby", Token: "1"}}, nil
+	}
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		createCalls++
+		switch createCalls {
+		case 1:
+			return "initial-1", nil
+		case 2:
+			return "loop", nil
+		case 3:
+			return "initial-2", nil
+		default:
+			t.Fatalf("unexpected create pull point call %d", createCalls)
+			return "", nil
+		}
+	}
+	heartbeatGetEventMessages = func(_ *goonvif.Device, pullPointAddress string) ([]agentonvif.ONVIFEvents, error) {
+		eventCalls++
+		switch pullPointAddress {
+		case "initial-1", "initial-2":
+			return []agentonvif.ONVIFEvents{{Key: "one", Type: "input", Value: "true", Timestamp: 1}}, nil
+		case "loop":
+			return []agentonvif.ONVIFEvents{{Key: "two", Type: "output", Value: "false", Timestamp: 2}}, nil
+		default:
+			t.Fatalf("unexpected pull point address %q", pullPointAddress)
+			return nil, nil
+		}
+	}
+	heartbeatUnsubscribePullPoint = func(*goonvif.Device, string) error { return nil }
+
+	state := newHeartbeatONVIFState()
+	payload := getHeartbeatONVIFPayload(camera, state)
+	if payload.presets != "false" {
+		t.Fatalf("payload.presets = %q, want false on transient preset failure", payload.presets)
+	}
+	payload = getHeartbeatONVIFPayload(camera, state)
+	if payload.presets != "true" {
+		t.Fatalf("payload.presets = %q, want true after retry", payload.presets)
+	}
+	if connectCalls != 1 {
+		t.Fatalf("connectCalls = %d, want 1", connectCalls)
+	}
+	if ptzConfigCalls != 2 || presetCalls != 2 {
+		t.Fatalf("static failures were not retried on heartbeat cadence: ptzConfig=%d presets=%d", ptzConfigCalls, presetCalls)
+	}
+	if createCalls != 3 || eventCalls != 3 {
+		t.Fatalf("unexpected event behavior during retry: create=%d events=%d", createCalls, eventCalls)
+	}
+}
+
+func TestHeartbeatONVIFStateReleasesSubscriptionWhenDisabled(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	device := newTestONVIFDevice()
+	state := newHeartbeatONVIFState()
+	state.cameraConfiguration = models.IPCamera{ONVIFXAddr: "http://camera/onvif"}
+	state.cameraKey = heartbeatONVIFCameraKey(state.cameraConfiguration)
+	state.device = device
+	state.loopPullPoint = "loop"
+
+	var unsubscribed []string
+	heartbeatUnsubscribePullPoint = func(gotDevice *goonvif.Device, pullPointAddress string) error {
+		if gotDevice != device {
+			t.Fatal("unsubscribe used a different ONVIF device")
+		}
+		unsubscribed = append(unsubscribed, pullPointAddress)
+		return nil
+	}
+
+	state.prepare(models.IPCamera{})
+
+	if len(unsubscribed) != 1 || unsubscribed[0] != "loop" {
+		t.Fatalf("unsubscribed = %v, want [loop]", unsubscribed)
+	}
+	if state.device != nil || state.loopPullPoint != "" {
+		t.Fatalf("disabled state retained device or pull point: %+v", state)
+	}
+}
+
+func TestHeartbeatONVIFPayloadDoesNotCreateSubscriptionsWhenConnectFails(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	var createCalls int
+
+	heartbeatConnectToONVIFDevice = func(*models.IPCamera) (*goonvif.Device, goonvifdevice.GetCapabilitiesResponse, error) {
+		return nil, goonvifdevice.GetCapabilitiesResponse{}, errors.New("connect failed")
+	}
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		createCalls++
+		return "unexpected", nil
+	}
+
+	payload := getHeartbeatONVIFPayload(models.IPCamera{
+		ONVIFXAddr:    "http://camera/onvif",
+		ONVIFUsername: "operator",
+		ONVIFPassword: "secret",
+	}, newHeartbeatONVIFState())
+
+	if createCalls != 0 {
+		t.Fatalf("createCalls = %d, want 0", createCalls)
+	}
+	assertDefaultHeartbeatONVIFPayload(t, payload)
+}
+
+func TestHeartbeatONVIFPayloadReconnectFailureInvalidatesCache(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	device1 := newTestONVIFDevice()
+	device2 := newTestONVIFDevice()
+	camera := models.IPCamera{
+		ONVIFXAddr:    "http://camera/onvif",
+		ONVIFUsername: "operator",
+		ONVIFPassword: "secret",
+	}
+	var connectCalls, ptzConfigCalls, presetCalls, createCalls int
+	var unsubscribed []string
+
+	heartbeatConnectToONVIFDevice = func(*models.IPCamera) (*goonvif.Device, goonvifdevice.GetCapabilitiesResponse, error) {
+		connectCalls++
+		switch connectCalls {
+		case 1:
+			return device1, goonvifdevice.GetCapabilitiesResponse{}, nil
+		case 2:
+			return device2, goonvifdevice.GetCapabilitiesResponse{}, nil
+		default:
+			t.Fatalf("unexpected connect call %d", connectCalls)
+			return nil, goonvifdevice.GetCapabilitiesResponse{}, nil
+		}
+	}
+	heartbeatGetPTZConfigurationsFromDevice = func(*goonvif.Device) (goonvifptz.GetConfigurationsResponse, error) {
+		ptzConfigCalls++
+		return goonvifptz.GetConfigurationsResponse{}, nil
+	}
+	heartbeatGetPTZFunctionsFromDevice = func(goonvifptz.GetConfigurationsResponse) ([]string, bool, bool) {
+		return nil, true, false
+	}
+	heartbeatGetPresetsFromDevice = func(*goonvif.Device, int) ([]models.OnvifActionPreset, error) {
+		presetCalls++
+		return []models.OnvifActionPreset{{Name: "Preset", Token: "1"}}, nil
+	}
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		createCalls++
+		switch createCalls {
+		case 1:
+			return "initial-1", nil
+		case 2:
+			return "loop-1", nil
+		case 3:
+			return "initial-2", nil
+		case 4:
+			return "initial-3", nil
+		case 5:
+			return "loop-3", nil
+		default:
+			t.Fatalf("unexpected create pull point call %d", createCalls)
+			return "", nil
+		}
+	}
+	heartbeatGetEventMessages = func(_ *goonvif.Device, pullPointAddress string) ([]agentonvif.ONVIFEvents, error) {
+		switch pullPointAddress {
+		case "initial-1":
+			return []agentonvif.ONVIFEvents{{Key: "before", Type: "input", Value: "true", Timestamp: 1}}, nil
+		case "initial-2":
+			return []agentonvif.ONVIFEvents{{Key: "during", Type: "input", Value: "true", Timestamp: 2}}, nil
+		case "loop-1":
+			return nil, errors.New("pull failed")
+		case "initial-3":
+			return []agentonvif.ONVIFEvents{{Key: "after", Type: "input", Value: "false", Timestamp: 2}}, nil
+		default:
+			t.Fatalf("unexpected pull point address %q", pullPointAddress)
+			return nil, nil
+		}
+	}
+	heartbeatUnsubscribePullPoint = func(_ *goonvif.Device, pullPointAddress string) error {
+		unsubscribed = append(unsubscribed, pullPointAddress)
+		return nil
+	}
+
+	state := newHeartbeatONVIFState()
+	_ = getHeartbeatONVIFPayload(camera, state)
+	payload := getHeartbeatONVIFPayload(camera, state)
+	wantEventsDuringFailure := mustJSONMarshal(t, []agentonvif.ONVIFEvents{{Key: "during", Type: "input", Value: "true", Timestamp: 2}})
+	if string(payload.eventsList) != string(wantEventsDuringFailure) {
+		t.Fatalf("payload.eventsList after operation failure = %s, want %s", payload.eventsList, wantEventsDuringFailure)
+	}
+	payload = getHeartbeatONVIFPayload(camera, state)
+
+	if ptzConfigCalls != 2 || presetCalls != 2 {
+		t.Fatalf("reconnect did not refresh static state: ptzConfig=%d presets=%d", ptzConfigCalls, presetCalls)
+	}
+	if connectCalls != 2 {
+		t.Fatalf("connectCalls = %d, want 2 after reconnect", connectCalls)
+	}
+	if createCalls != 5 {
+		t.Fatalf("reconnect did not recreate subscriptions: create=%d", createCalls)
+	}
+	if countString(unsubscribed, "loop-1") != 1 {
+		t.Fatalf("operation failure cleanup mismatch, unsubscribed=%v", unsubscribed)
+	}
+	wantEvents := mustJSONMarshal(t, []agentonvif.ONVIFEvents{{Key: "after", Type: "input", Value: "false", Timestamp: 2}})
+	if string(payload.eventsList) != string(wantEvents) {
+		t.Fatalf("payload.eventsList = %s, want %s", payload.eventsList, wantEvents)
+	}
+}
+
+func TestHeartbeatONVIFPayloadKeepsCachedConnectionWhenInitialStateFetchFails(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	device := newTestONVIFDevice()
+	camera := models.IPCamera{
+		ONVIFXAddr:    "http://camera/onvif",
+		ONVIFUsername: "operator",
+		ONVIFPassword: "secret",
+	}
+	loopEvents := []agentonvif.ONVIFEvents{{Key: "input-1", Type: "input", Value: "true", Timestamp: 1}}
+
+	state := newHeartbeatONVIFState()
+	state.cameraConfiguration = camera
+	state.cameraKey = heartbeatONVIFCameraKey(camera)
+	state.device = device
+	state.loopPullPoint = "loop"
+	state.staticLoaded = true
+	state.staticPayload.enabled = "true"
+
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		return "", errors.New("temporary initial-state failure")
+	}
+	heartbeatGetEventMessages = func(_ *goonvif.Device, pullPointAddress string) ([]agentonvif.ONVIFEvents, error) {
+		if pullPointAddress != "loop" {
+			t.Fatalf("unexpected pull point address %q", pullPointAddress)
+		}
+		return loopEvents, nil
+	}
+
+	payload := getHeartbeatONVIFPayload(camera, state)
+
+	wantEvents := mustJSONMarshal(t, loopEvents)
+	if string(payload.eventsList) != string(wantEvents) {
+		t.Fatalf("payload.eventsList = %s, want %s", payload.eventsList, wantEvents)
+	}
+	if state.device != device || state.loopPullPoint != "loop" || !state.staticLoaded {
+		t.Fatalf("temporary failure invalidated healthy cached state: %+v", state)
+	}
+}
+
+func restoreHeartbeatONVIFStubs(t *testing.T) {
+	t.Helper()
+
+	originalConnect := heartbeatConnectToONVIFDevice
+	originalCreate := heartbeatCreatePullPointSubscription
+	originalDigitalInputs := heartbeatGetDigitalInputs
+	originalEvents := heartbeatGetEventMessages
+	originalPresets := heartbeatGetPresetsFromDevice
+	originalPTZConfigurations := heartbeatGetPTZConfigurationsFromDevice
+	originalPTZFunctions := heartbeatGetPTZFunctionsFromDevice
+	originalRelayOutputs := heartbeatGetRelayOutputs
+	originalUnsubscribe := heartbeatUnsubscribePullPoint
+
+	t.Cleanup(func() {
+		heartbeatConnectToONVIFDevice = originalConnect
+		heartbeatCreatePullPointSubscription = originalCreate
+		heartbeatGetDigitalInputs = originalDigitalInputs
+		heartbeatGetEventMessages = originalEvents
+		heartbeatGetPresetsFromDevice = originalPresets
+		heartbeatGetPTZConfigurationsFromDevice = originalPTZConfigurations
+		heartbeatGetPTZFunctionsFromDevice = originalPTZFunctions
+		heartbeatGetRelayOutputs = originalRelayOutputs
+		heartbeatUnsubscribePullPoint = originalUnsubscribe
+	})
+}
+
+func mustJSONMarshal(t *testing.T, v interface{}) []byte {
+	t.Helper()
+
+	b, err := json.Marshal(v)
+	if err != nil {
+		t.Fatalf("json.Marshal() error = %v", err)
+	}
+	return b
+}
+
+func assertDefaultHeartbeatONVIFPayload(t *testing.T, payload heartbeatONVIFPayload) {
+	t.Helper()
+
+	defaultPayload := defaultHeartbeatONVIFPayload()
+	if payload.enabled != defaultPayload.enabled ||
+		payload.zoom != defaultPayload.zoom ||
+		payload.panTilt != defaultPayload.panTilt ||
+		payload.presets != defaultPayload.presets ||
+		string(payload.presetsList) != string(defaultPayload.presetsList) ||
+		string(payload.eventsList) != string(defaultPayload.eventsList) {
+		t.Fatalf("payload = %+v, want default payload", payload)
+	}
+}
+
+func newTestONVIFDevice() *goonvif.Device {
+	return &goonvif.Device{}
+}
+
+func countString(values []string, want string) int {
+	count := 0
+	for _, value := range values {
+		if value == want {
+			count++
+		}
+	}
+	return count
+}
+
+// Some devices accept the subscription but reject the pulls, or never publish
+// the initial I/O state. The relay outputs and digital inputs must then be
+// queried directly instead of reporting no I/O at all.
+func TestHeartbeatONVIFPayloadFallsBackToDirectIOWhenSubscriptionsReturnNoEvents(t *testing.T) {
+	restoreHeartbeatONVIFStubs(t)
+
+	device := newTestONVIFDevice()
+	camera := models.IPCamera{
+		ONVIFXAddr:    "http://camera/onvif",
+		ONVIFUsername: "operator",
+		ONVIFPassword: "secret",
+	}
+
+	state := newHeartbeatONVIFState()
+	state.cameraConfiguration = camera
+	state.cameraKey = heartbeatONVIFCameraKey(camera)
+	state.device = device
+	state.staticLoaded = true
+	state.staticPayload.enabled = "true"
+
+	heartbeatCreatePullPointSubscription = func(*goonvif.Device) (string, error) {
+		return "pull-point", nil
+	}
+	heartbeatGetEventMessages = func(*goonvif.Device, string) ([]agentonvif.ONVIFEvents, error) {
+		return nil, errors.New("pull rejected")
+	}
+	heartbeatUnsubscribePullPoint = func(*goonvif.Device, string) error {
+		return nil
+	}
+	var relayCalls, inputCalls int
+	heartbeatGetRelayOutputs = func(*goonvif.Device) (goonvifdevice.GetRelayOutputsResponse, error) {
+		relayCalls++
+		return goonvifdevice.GetRelayOutputsResponse{RelayOutputs: []goonvifxsd.RelayOutput{{DeviceEntity: goonvifxsd.DeviceEntity{Token: "relay0"}}}}, nil
+	}
+	heartbeatGetDigitalInputs = func(*goonvif.Device) (goonvifdevice.GetDigitalInputsResponse, error) {
+		inputCalls++
+		return goonvifdevice.GetDigitalInputsResponse{DigitalInputs: []goonvifxsd.DigitalInput{{Token: "di0"}}}, nil
+	}
+
+	payload := getHeartbeatONVIFPayload(camera, state)
+
+	if relayCalls != 1 || inputCalls != 1 {
+		t.Fatalf("direct I/O queries: relay=%d inputs=%d, want 1 each", relayCalls, inputCalls)
+	}
+	var events []agentonvif.ONVIFEvents
+	if err := json.Unmarshal(payload.eventsList, &events); err != nil {
+		t.Fatalf("payload.eventsList is not valid JSON: %v", err)
+	}
+	got := map[string]string{}
+	for _, event := range events {
+		got[event.Key] = event.Type
+	}
+	if got["relay0"] != "output" || got["di0"] != "input" || len(got) != 2 {
+		t.Fatalf("payload.eventsList = %s, want relay0 output and di0 input", payload.eventsList)
+	}
+}

machinery/src/cloud/custom_headers.go

@@ -0,0 +1,195 @@
+package cloud
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"sort"
+	"strings"
+)
+
+const (
+	customHeadersManifestHeader = "X-Kerberos-Storage-Custom-Headers"
+	maxCustomHeaders            = 32
+	maxCustomHeaderNameLength   = 128
+	maxCustomHeaderValueLength  = 4096
+)
+
+var disallowedCustomHeaders = map[string]struct{}{
+	"authorization":       {},
+	"connection":          {},
+	"content-length":      {},
+	"content-type":        {},
+	"cookie":              {},
+	"expect":              {},
+	"host":                {},
+	"keep-alive":          {},
+	"proxy-authorization": {},
+	"proxy-connection":    {},
+	"te":                  {},
+	"trailer":             {},
+	"trace-id":            {},
+	"transfer-encoding":   {},
+	"tus-resumable":       {},
+	"upgrade":             {},
+}
+
+func parseCustomVaultHeaders(value string) (map[string]string, error) {
+	if strings.TrimSpace(value) == "" {
+		return nil, nil
+	}
+
+	// Decode into pointers so JSON null values can be told apart from strings.
+	var rawHeaders map[string]*string
+	if err := json.Unmarshal([]byte(value), &rawHeaders); err != nil {
+		return nil, fmt.Errorf("custom Vault headers must be a JSON object of string values: %w", err)
+	}
+	if rawHeaders == nil {
+		return nil, errors.New("custom Vault headers must be a JSON object of string values")
+	}
+	if len(rawHeaders) > maxCustomHeaders {
+		return nil, fmt.Errorf("custom Vault headers exceed the limit of %d", maxCustomHeaders)
+	}
+	headers := make(map[string]string, len(rawHeaders))
+	seen := make(map[string]struct{}, len(rawHeaders))
+	for name, rawValue := range rawHeaders {
+		if rawValue == nil {
+			return nil, fmt.Errorf("custom Vault header %q must be a string, not null", name)
+		}
+		value := *rawValue
+		headers[name] = value
+		if err := validateCustomVaultHeader(name, value); err != nil {
+			return nil, err
+		}
+		lowerName := strings.ToLower(name)
+		if _, duplicate := seen[lowerName]; duplicate {
+			return nil, fmt.Errorf("custom Vault header %q is duplicated with different casing", name)
+		}
+		seen[lowerName] = struct{}{}
+	}
+	return headers, nil
+}
+
+func validateCustomVaultHeader(name, value string) error {
+	if name == "" || len(name) > maxCustomHeaderNameLength || !isHTTPToken(name) {
+		return fmt.Errorf("invalid custom Vault header name %q", name)
+	}
+	lowerName := strings.ToLower(name)
+	if strings.HasPrefix(lowerName, "x-kerberos-") {
+		return fmt.Errorf("custom Vault header %q uses the reserved X-Kerberos namespace", name)
+	}
+	if strings.HasPrefix(lowerName, "upload-") {
+		return fmt.Errorf("custom Vault header %q uses the reserved tus Upload namespace", name)
+	}
+	if _, disallowed := disallowedCustomHeaders[lowerName]; disallowed {
+		return fmt.Errorf("custom Vault header %q is reserved", name)
+	}
+	if len(value) > maxCustomHeaderValueLength {
+		return fmt.Errorf("custom Vault header %q exceeds the %d byte value limit", name, maxCustomHeaderValueLength)
+	}
+	for _, character := range value {
+		if character < 0x20 || character == 0x7f {
+			return fmt.Errorf("custom Vault header %q contains an invalid value", name)
+		}
+	}
+	return nil
+}
+
+func isHTTPToken(value string) bool {
+	if value == "" {
+		return false
+	}
+	for _, character := range value {
+		if (character >= 'a' && character <= 'z') ||
+			(character >= 'A' && character <= 'Z') ||
+			(character >= '0' && character <= '9') ||
+			strings.ContainsRune("!#$%&'*+-.^_`|~", character) {
+			continue
+		}
+		return false
+	}
+	return true
+}
+
+func setCustomVaultHeaders(headers http.Header, customHeaders map[string]string) error {
+	if len(customHeaders) == 0 {
+		return nil
+	}
+
+	names := make([]string, 0, len(customHeaders))
+	for name, value := range customHeaders {
+		if err := validateCustomVaultHeader(name, value); err != nil {
+			return err
+		}
+		names = append(names, name)
+		// Assign directly so the configured name is not canonicalised by Set.
+		headers[name] = []string{value}
+	}
+	sort.Strings(names)
+	manifest, err := json.Marshal(names)
+	if err != nil {
+		return err
+	}
+	headers.Set(customHeadersManifestHeader, base64.RawURLEncoding.EncodeToString(manifest))
+	return nil
+}
+
+const maxVaultRedirects = 10
+
+// stripVaultHeadersOnCrossHostRedirect removes Vault credentials and the
+// customer-supplied metadata (custom headers, their manifest and the tus
+// Upload-Metadata that embeds them) before following a redirect to another host.
+func stripVaultHeadersOnCrossHostRedirect(req *http.Request, via []*http.Request) error {
+	if len(via) >= maxVaultRedirects {
+		return fmt.Errorf("stopped after %d redirects", maxVaultRedirects)
+	}
+	if len(via) == 0 || req.URL.Host == via[0].URL.Host {
+		return nil
+	}
+
+	customNames := customVaultHeaderNamesFromManifest(req.Header)
+	for name := range req.Header {
+		canonicalName := http.CanonicalHeaderKey(name)
+		_, custom := customNames[strings.ToLower(name)]
+		if custom || strings.HasPrefix(canonicalName, "X-Kerberos-") || canonicalName == "Upload-Metadata" {
+			delete(req.Header, name)
+		}
+	}
+	return nil
+}
+
+func customVaultHeaderNamesFromManifest(headers http.Header) map[string]struct{} {
+	encoded := headers.Get(customHeadersManifestHeader)
+	if encoded == "" {
+		return nil
+	}
+	manifest, err := base64.RawURLEncoding.DecodeString(encoded)
+	if err != nil {
+		return nil
+	}
+	var names []string
+	if err := json.Unmarshal(manifest, &names); err != nil {
+		return nil
+	}
+	lowerNames := make(map[string]struct{}, len(names))
+	for _, name := range names {
+		lowerNames[strings.ToLower(name)] = struct{}{}
+	}
+	return lowerNames
+}
+
+func encodeCustomVaultHeaders(customHeaders map[string]string) (string, error) {
+	if len(customHeaders) == 0 {
+		return "", nil
+	}
+	encoded, err := json.Marshal(customHeaders)
+	if err != nil {
+		return "", err
+	}
+	if len(encoded) == 0 {
+		return "", errors.New("custom Vault headers could not be encoded")
+	}
+	return string(encoded), nil
+}

machinery/src/cloud/custom_headers_test.go

@@ -0,0 +1,100 @@
+package cloud
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"net/http"
+	"reflect"
+	"testing"
+)
+
+func TestCustomVaultHeadersAreValidatedAndAdded(t *testing.T) {
+	customHeaders, err := parseCustomVaultHeaders(`{"site_id":"site-1","line_id":"line-2"}`)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	headers := http.Header{}
+	if err := setCustomVaultHeaders(headers, customHeaders); err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(headers["site_id"], []string{"site-1"}) || !reflect.DeepEqual(headers["line_id"], []string{"line-2"}) {
+		t.Fatalf("custom header names were not preserved: %#v", headers)
+	}
+	if _, canonicalised := headers["Site_id"]; canonicalised {
+		t.Fatalf("custom header name was canonicalised: %#v", headers)
+	}
+
+	encodedManifest, err := base64.RawURLEncoding.DecodeString(headers.Get(customHeadersManifestHeader))
+	if err != nil {
+		t.Fatal(err)
+	}
+	var names []string
+	if err := json.Unmarshal(encodedManifest, &names); err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(names, []string{"line_id", "site_id"}) {
+		t.Fatalf("manifest = %#v", names)
+	}
+}
+
+func TestCustomVaultHeadersRejectReservedAndInvalidValues(t *testing.T) {
+	for _, value := range []string{
+		`{"X-Kerberos-Storage-AccessKey":"secret"}`,
+		`{"Authorization":"secret"}`,
+		`{"site_id":"one\nInjected: value"}`,
+		`{"bad header":"value"}`,
+		`{"site_id":1}`,
+		`{"Content-Type":"text/plain"}`,
+		`{"site_id":"one","SITE_ID":"two"}`,
+		`{"site_id":null}`,
+		`{"site_id":"one\ttwo"}`,
+		`{"Upload-Checksum":"sha1 abc"}`,
+		`{"upload-concat":"final;/files/a"}`,
+		`{"Upload-Defer-Length":"1"}`,
+		`null`,
+	} {
+		if _, err := parseCustomVaultHeaders(value); err == nil {
+			t.Fatalf("parseCustomVaultHeaders(%q) succeeded", value)
+		}
+	}
+}
+
+func TestVaultRedirectStripsCustomHeadersOnHostChange(t *testing.T) {
+	newRedirect := func(t *testing.T, host string) (*http.Request, []*http.Request) {
+		t.Helper()
+		original, err := http.NewRequest(http.MethodPost, "https://vault.example.com/api/storage", nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		redirected, err := http.NewRequest(http.MethodPost, "https://"+host+"/api/storage", nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		redirected.Header.Set("X-Kerberos-Storage-AccessKey", "secret")
+		redirected.Header.Set("Upload-Metadata", "custom_headers abc")
+		redirected.Header.Set("Tus-Resumable", "1.0.0")
+		if err := setCustomVaultHeaders(redirected.Header, map[string]string{"site_id": "site-1", "Line-Id": "line-2"}); err != nil {
+			t.Fatal(err)
+		}
+		return redirected, []*http.Request{original}
+	}
+
+	crossHost, via := newRedirect(t, "other.example.com")
+	if err := stripVaultHeadersOnCrossHostRedirect(crossHost, via); err != nil {
+		t.Fatal(err)
+	}
+	if len(crossHost.Header) != 1 || crossHost.Header.Get("Tus-Resumable") != "1.0.0" {
+		t.Fatalf("cross-host redirect headers = %#v", crossHost.Header)
+	}
+
+	sameHost, via := newRedirect(t, "vault.example.com")
+	if err := stripVaultHeadersOnCrossHostRedirect(sameHost, via); err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(sameHost.Header["site_id"], []string{"site-1"}) ||
+		sameHost.Header.Get("X-Kerberos-Storage-AccessKey") != "secret" ||
+		sameHost.Header.Get(customHeadersManifestHeader) == "" {
+		t.Fatalf("same-host redirect headers = %#v", sameHost.Header)
+	}
+}

machinery/src/cloud/dropbox.go

@@ -5,14 +5,14 @@ package cloud
 import (
 	"bytes"
 	"errors"
-	"os"
 
 	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox"
 	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
 	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/users"
 	"github.com/gin-gonic/gin"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/utils"
+	log "github.com/sirupsen/logrus"
 )
 
 // UploadDropbox uploads the file to your Dropbox account using the access token and directory.
@@ -30,25 +30,23 @@ func UploadDropbox(configuration *models.Configuration, fileName string) (bool,
 
 	if token == "" {
 		err := "UploadDropbox: Dropbox not properly configured"
-		log.Log.Info(err)
+		log.Info(err)
 		return false, true, errors.New(err)
 	}
 
 	// Upload to Dropbox
-	log.Log.Info("UploadDropbox: Uploading to Dropbox")
-	log.Log.Info("UploadDropbox: Upload started for " + fileName)
-	fullname := "data/recordings/" + fileName
-
+	log.Info("UploadDropbox: Uploading to Dropbox")
+	log.Info("UploadDropbox: Upload started for " + fileName)
 	dConfig := dropbox.Config{
 		Token:    token,
 		LogLevel: dropbox.LogInfo, // if needed, set the desired logging level. Default is off
 	}
 
-	file, err := os.OpenFile(fullname, os.O_RDWR, 0755)
+	file, err := utils.OpenRecording("data/recordings", fileName)
 	if file != nil {
 		defer func() {
 			if cerr := file.Close(); cerr != nil {
-				log.Log.Error("UploadDropbox: Error closing file: " + cerr.Error())
+				log.Error("UploadDropbox: Error closing file: " + cerr.Error())
 			}
 		}()
 	}
@@ -68,15 +66,15 @@ func UploadDropbox(configuration *models.Configuration, fileName string) (bool,
 		}, file)
 
 		if err != nil {
-			log.Log.Error("UploadDropbox: Error uploading file: " + err.Error())
+			log.Error("UploadDropbox: Error uploading file: " + err.Error())
 			return false, false, err
 		}
 
-		log.Log.Info("UploadDropbox: File uploaded successfully, " + res.Name)
+		log.Info("UploadDropbox: File uploaded successfully, " + res.Name)
 		return true, true, nil
 	}
 
-	log.Log.Error("UploadDropbox: Error opening file: " + err.Error())
+	log.Error("UploadDropbox: Error opening file: " + err.Error())
 	return false, true, err
 }
 

machinery/src/cloud/kerberos_hub.go

@@ -3,12 +3,14 @@ package cloud
 import (
 	"crypto/tls"
 	"errors"
+	"fmt"
 	"io/ioutil"
 	"net/http"
 	"os"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/utils"
+	log "github.com/sirupsen/logrus"
 )
 
 func UploadKerberosHub(configuration *models.Configuration, fileName string) (bool, bool, error) {
@@ -19,7 +21,7 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 		config.HubPrivateKey == "" ||
 		config.S3.Region == "" {
 		err := "UploadKerberosHub: Kerberos Hub not properly configured."
-		log.Log.Info(err)
+		log.Info(err)
 		return false, false, errors.New(err)
 	}
 
@@ -32,8 +34,10 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 	// - Number of changes
 	// - Token
 
-	log.Log.Info("UploadKerberosHub: Uploading to Kerberos Hub (" + config.HubURI + ")")
-	log.Log.Info("UploadKerberosHub: Upload started for " + fileName)
+	log.WithFields(log.Fields{
+		"component": "kerberos_hub",
+		"event":     "upload_started",
+	}).Info("Uploading recording to Kerberos Hub")
 
 	// Prefer the resumable (tus) upload when enabled (the default). Kerberos Hub
 	// authenticates the agent with its Hub public/private key and proxies the
@@ -44,29 +48,45 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 		uploaded, _, supported, body, rerr := uploadHubResumable(&config, fileName, "UploadKerberosHub", "hub")
 		if supported {
 			if uploaded {
-				log.Log.Info("UploadKerberosHub: Upload Finished (resumable), " + body)
+				log.WithFields(log.Fields{
+					"component":      "kerberos_hub",
+					"event":          "upload_completed",
+					"response_bytes": len(body),
+					"transport":      "tus",
+				}).Info("Hub upload completed")
 				return true, true, nil
 			}
 			if rerr != nil {
-				log.Log.Info("UploadKerberosHub: resumable upload failed, " + rerr.Error())
+				log.WithError(rerr).WithFields(log.Fields{
+					"component": "kerberos_hub",
+					"event":     "upload_failed",
+					"transport": "tus",
+				}).Error("Hub upload failed")
 			} else {
-				log.Log.Info("UploadKerberosHub: resumable upload incomplete, " + body)
+				log.WithFields(log.Fields{
+					"component":      "kerberos_hub",
+					"event":          "upload_incomplete",
+					"response_bytes": len(body),
+					"transport":      "tus",
+				}).Warn("Hub upload incomplete")
 			}
 			return false, true, rerr
 		}
-		log.Log.Info("UploadKerberosHub: resumable (tus) endpoint not available, falling back to legacy upload")
+		log.WithFields(log.Fields{
+			"component": "kerberos_hub",
+			"event":     "upload_transport_fallback",
+			"transport": "http",
+		}).Info("Resumable Hub upload unavailable; using legacy upload")
 	}
 
-	fullname := "data/recordings/" + fileName
-
 	// Check if we still have the file otherwise we abort the request.
-	file, err := os.OpenFile(fullname, os.O_RDWR, 0755)
+	file, err := utils.OpenRecording("data/recordings", fileName)
 	if file != nil {
 		defer file.Close()
 	}
 	if err != nil {
 		err := "UploadKerberosHub: Upload Failed, file doesn't exists anymore."
-		log.Log.Info(err)
+		log.Info(err)
 		return false, false, errors.New(err)
 	}
 
@@ -74,9 +94,11 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 	// There might be different reasons like (muted, read-only..)
 	req, err := http.NewRequest("HEAD", config.HubURI+"/storage/upload", nil)
 	if err != nil {
-		errorMessage := "UploadKerberosHub: error reading HEAD request, " + config.HubURI + "/storage: " + err.Error()
-		log.Log.Error(errorMessage)
-		return false, true, errors.New(errorMessage)
+		log.WithError(err).WithFields(log.Fields{
+			"component": "kerberos_hub",
+			"event":     "authorization_request_creation_failed",
+		}).Error("Failed to create Hub upload authorization request")
+		return false, true, fmt.Errorf("create Hub upload authorization request: %w", err)
 	}
 
 	req.Header.Set("X-Kerberos-Storage-FileName", fileName)
@@ -85,6 +107,7 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 	req.Header.Set("X-Kerberos-Hub-PublicKey", config.HubKey)
 	req.Header.Set("X-Kerberos-Hub-PrivateKey", config.HubPrivateKey)
 	req.Header.Set("X-Kerberos-Hub-Region", config.S3.Region)
+	setQueuedRecordingMetadataHeaders(req.Header, fileName)
 
 	var client *http.Client
 	if os.Getenv("AGENT_TLS_INSECURE") == "true" {
@@ -101,25 +124,31 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 		defer resp.Body.Close()
 	}
 
-	if err == nil {
-		if resp != nil {
-			if err == nil {
-				if resp.StatusCode == 200 {
-					log.Log.Info("UploadKerberosHub: Upload allowed using the credentials provided (" + config.HubKey + ", " + config.HubPrivateKey + ")")
-				} else {
-					log.Log.Info("UploadKerberosHub: Upload NOT allowed using the credentials provided (" + config.HubKey + ", " + config.HubPrivateKey + ")")
-					return false, true, nil
-				}
-			}
+	if err == nil && resp != nil {
+		if resp.StatusCode == 200 {
+			log.WithFields(log.Fields{
+				"component":   "kerberos_hub",
+				"event":       "upload_authorized",
+				"status_code": resp.StatusCode,
+			}).Debug("Hub upload authorized")
+		} else {
+			log.WithFields(log.Fields{
+				"component":   "kerberos_hub",
+				"event":       "upload_rejected",
+				"status_code": resp.StatusCode,
+			}).Warn("Hub upload rejected")
+			return false, true, nil
 		}
 	}
 
 	// Now we know we are allowed to upload to the hub, we can start uploading.
 	req, err = http.NewRequest("POST", config.HubURI+"/storage/upload", file)
 	if err != nil {
-		errorMessage := "UploadKerberosHub: error reading POST request, " + config.KStorage.URI + "/storage/upload: " + err.Error()
-		log.Log.Error(errorMessage)
-		return false, true, errors.New(errorMessage)
+		log.WithError(err).WithFields(log.Fields{
+			"component": "kerberos_hub",
+			"event":     "upload_request_creation_failed",
+		}).Error("Failed to create Hub upload request")
+		return false, true, fmt.Errorf("create Hub upload request: %w", err)
 	}
 	req.Header.Set("Content-Type", "video/mp4")
 	req.Header.Set("X-Kerberos-Storage-FileName", fileName)
@@ -128,6 +157,7 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 	req.Header.Set("X-Kerberos-Hub-PublicKey", config.HubKey)
 	req.Header.Set("X-Kerberos-Hub-PrivateKey", config.HubPrivateKey)
 	req.Header.Set("X-Kerberos-Hub-Region", config.S3.Region)
+	setQueuedRecordingMetadataHeaders(req.Header, fileName)
 	resp, err = client.Do(req)
 	if resp != nil {
 		defer resp.Body.Close()
@@ -138,19 +168,36 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
 			body, err := ioutil.ReadAll(resp.Body)
 			if err == nil {
 				if resp.StatusCode == 200 {
-					log.Log.Info("UploadKerberosHub: Upload Finished, " + resp.Status + ".")
+					log.WithFields(log.Fields{
+						"component":   "kerberos_hub",
+						"event":       "upload_completed",
+						"status_code": resp.StatusCode,
+						"transport":   "http",
+					}).Info("Hub upload completed")
 					return true, true, nil
 				} else {
-					log.Log.Info("UploadKerberosHub: Upload Failed, " + resp.Status + ", " + string(body))
+					log.WithFields(log.Fields{
+						"component":      "kerberos_hub",
+						"event":          "upload_rejected",
+						"response_bytes": len(body),
+						"status_code":    resp.StatusCode,
+						"transport":      "http",
+					}).Warn("Hub upload rejected")
 					return false, true, nil
 				}
 			}
 		}
 	}
 
-	errorMessage := "UploadKerberosHub: Upload Failed, " + err.Error()
-	log.Log.Info(errorMessage)
-	return false, true, errors.New(errorMessage)
+	if err == nil {
+		err = errors.New("Hub upload failed without a response")
+	}
+	log.WithError(err).WithFields(log.Fields{
+		"component": "kerberos_hub",
+		"event":     "upload_failed",
+		"transport": "http",
+	}).Error("Hub upload failed")
+	return false, true, fmt.Errorf("Hub upload failed: %w", err)
 }
 
 // stripHubCredentialsOnCrossHostRedirect removes the custom Kerberos Hub

machinery/src/cloud/kerberos_vault.go

@@ -6,10 +6,12 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"strconv"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/utils"
+	log "github.com/sirupsen/logrus"
 )
 
 // We will count the number of retries we have done.
@@ -26,7 +28,7 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string) (
 		config.KStorage.Directory == "" ||
 		config.KStorage.URI == "" {
 		err := "UploadKerberosVault: Kerberos Vault not properly configured"
-		log.Log.Info(err)
+		log.Info(err)
 		return false, false, errors.New(err)
 	}
 
@@ -34,8 +36,26 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string) (
 	// This can happen when the file was already removed (e.g. cleanup, or an
 	// earlier successful upload). Skip it so the watcher drops the marker
 	// instead of retrying indefinitely.
-	if _, err := os.Stat("data/recordings/" + fileName); err != nil {
-		log.Log.Info("UploadKerberosVault: skipping " + fileName + ", file doesn't exist anymore")
+	filePath, err := utils.ResolveRecordingPath("data/recordings", fileName)
+	if err != nil {
+		if os.IsNotExist(err) {
+			log.Info("UploadKerberosVault: skipping " + fileName + ", file doesn't exist anymore")
+			return false, false, nil
+		}
+		log.WithError(err).Error("UploadKerberosVault: could not resolve recording path")
+		return false, true, err
+	}
+	info, err := os.Stat(filePath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			log.Info("UploadKerberosVault: skipping " + fileName + ", file doesn't exist anymore")
+			return false, false, nil
+		}
+		log.WithError(err).Error("UploadKerberosVault: could not stat recording")
+		return false, true, err
+	}
+	if info.Size() == 0 {
+		log.Warn("UploadKerberosVault: skipping " + fileName + ", recording is empty")
 		return false, false, nil
 	}
 
@@ -48,8 +68,11 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string) (
 	// - Number of changes
 	// - Token
 	// KerberosCloud, this means storage is disabled and proxy enabled.
-	log.Log.Info("UploadKerberosVault: Uploading to Kerberos Vault (" + config.KStorage.URI + ")")
-	log.Log.Info("UploadKerberosVault: Upload started for " + fileName)
+	log.WithFields(log.Fields{
+		"component": "kerberos_vault",
+		"event":     "upload_started",
+		"storage":   "primary",
+	}).Info("Vault upload started")
 
 	publicKey := config.KStorage.CloudKey
 	if config.HubKey != "" {
@@ -61,14 +84,27 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string) (
 		uploaded, responded, body, err := sendToVault(*config.KStorage, publicKey, config.Key, fileName, "UploadKerberosVault", "primary")
 		if uploaded {
 			kstorageRetryCount = 0
-			log.Log.Info("UploadKerberosVault: Upload Finished, " + body)
+			log.WithFields(log.Fields{
+				"component": "kerberos_vault",
+				"event":     "upload_completed",
+				"storage":   "primary",
+			}).Info("Vault upload completed")
 			return true, true, nil
 		}
 
 		if err != nil {
-			log.Log.Info("UploadKerberosVault: Upload Failed, " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component": "kerberos_vault",
+				"event":     "upload_failed",
+				"storage":   "primary",
+			}).Error("Vault upload failed")
 		} else {
-			log.Log.Info("UploadKerberosVault: Upload Failed, " + body)
+			log.WithFields(log.Fields{
+				"component":      "kerberos_vault",
+				"event":          "upload_rejected",
+				"response_bytes": len(body),
+				"storage":        "primary",
+			}).Warn("Vault upload rejected")
 		}
 
 		// We only advance the retry policy when the vault gave a definitive
@@ -90,26 +126,43 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string) (
 		config.KStorageSecondary.SecretAccessKey == "" ||
 		config.KStorageSecondary.Directory == "" ||
 		config.KStorageSecondary.URI == "" {
-		log.Log.Info("UploadKerberosVault (Secondary): Secondary Kerberos Vault not properly configured.")
+		log.Info("UploadKerberosVault (Secondary): Secondary Kerberos Vault not properly configured.")
 	} else {
 
 		if kstorageRetryCount < config.KStorage.MaxRetries {
-			log.Log.Info("UploadKerberosVault (Secondary): Do not upload to secondary storage, we are still in retry policy.")
+			log.Info("UploadKerberosVault (Secondary): Do not upload to secondary storage, we are still in retry policy.")
 			return false, true, nil
 		}
 
-		log.Log.Info("UploadKerberosVault (Secondary): Uploading to Secondary Kerberos Vault (" + config.KStorageSecondary.URI + ")")
+		log.WithFields(log.Fields{
+			"component": "kerberos_vault",
+			"event":     "upload_started",
+			"storage":   "secondary",
+		}).Info("Vault upload started")
 
 		uploaded, _, body, err := sendToVault(*config.KStorageSecondary, publicKey, config.Key, fileName, "UploadKerberosVault (Secondary)", "secondary")
 		if uploaded {
-			log.Log.Info("UploadKerberosVault (Secondary): Upload Finished to secondary, " + body)
+			log.WithFields(log.Fields{
+				"component": "kerberos_vault",
+				"event":     "upload_completed",
+				"storage":   "secondary",
+			}).Info("Vault upload completed")
 			return true, true, nil
 		}
 
 		if err != nil {
-			log.Log.Info("UploadKerberosVault (Secondary): Upload Failed to secondary, " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component": "kerberos_vault",
+				"event":     "upload_failed",
+				"storage":   "secondary",
+			}).Error("Vault upload failed")
 		} else {
-			log.Log.Info("UploadKerberosVault (Secondary): Upload Failed to secondary, " + body)
+			log.WithFields(log.Fields{
+				"component":      "kerberos_vault",
+				"event":          "upload_rejected",
+				"response_bytes": len(body),
+				"storage":        "secondary",
+			}).Warn("Vault upload rejected")
 		}
 	}
 
@@ -130,7 +183,7 @@ func sendToVault(vault models.KStorage, publicKey, deviceKey, fileName, label, s
 		if supported {
 			return uploaded, responded, body, err
 		}
-		log.Log.Info(label + ": resumable (tus) endpoint not available, falling back to legacy upload")
+		log.Info(label + ": resumable (tus) endpoint not available, falling back to legacy upload")
 	}
 	return uploadVaultLegacy(vault, publicKey, deviceKey, fileName, label)
 }
@@ -139,15 +192,17 @@ func sendToVault(vault models.KStorage, publicKey, deviceKey, fileName, label, s
 // is sent as the body of a POST to {URI}/storage. Kept for backwards
 // compatibility with vault deployments that do not support resumable uploads.
 func uploadVaultLegacy(vault models.KStorage, publicKey, deviceKey, fileName, label string) (bool, bool, string, error) {
-	fullname := "data/recordings/" + fileName
-
-	file, err := os.Open(fullname)
+	customHeaders, err := parseCustomVaultHeaders(vault.CustomHeaders)
+	if err != nil {
+		return false, false, "", err
+	}
+	file, err := utils.OpenRecording("data/recordings", fileName)
 	if file != nil {
 		defer file.Close()
 	}
 	if err != nil {
 		msg := label + ": Upload Failed, file doesn't exists anymore"
-		log.Log.Info(msg)
+		log.Info(msg)
 		return false, false, "", errors.New(msg)
 	}
 
@@ -159,11 +214,15 @@ func uploadVaultLegacy(vault models.KStorage, publicKey, deviceKey, fileName, la
 	req, err := http.NewRequest("POST", uri+"/storage", file)
 	if err != nil {
 		errorMessage := label + ": error reading request, " + uri + "/storage: " + err.Error()
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, false, "", errors.New(errorMessage)
 	}
 	req.Header.Set("Content-Type", "video/mp4")
 	setVaultHeaders(req.Header, vault, publicKey, deviceKey, fileName)
+	if err := setCustomVaultHeaders(req.Header, customHeaders); err != nil {
+		return false, false, "", err
+	}
+	setQueuedRecordingMetadataHeaders(req.Header, fileName)
 
 	client := newVaultHTTPClient(0)
 	resp, err := client.Do(req)
@@ -199,17 +258,59 @@ func setVaultHeaders(h http.Header, vault models.KStorage, publicKey, deviceKey,
 }
 
 // newVaultHTTPClient builds an HTTP client honouring the AGENT_TLS_INSECURE
-// escape hatch. A timeout of 0 disables the client-level timeout, which is
-// required for streaming large upload bodies.
+// escape hatch. A timeout of 0 disables the *overall* client timeout, which is
+// required for streaming large upload bodies without capping the total transfer
+// time. Transport-level timeouts are still applied so that a lost network
+// connection (for example the internet being disconnected) fails reasonably
+// fast and the upload is retried, instead of the request hanging until the OS
+// TCP timeout (which can be many minutes) and blocking the whole upload loop.
 func newVaultHTTPClient(timeout time.Duration) *http.Client {
-	client := &http.Client{}
+	// Start from a clone of the default transport so we keep its sane dial and
+	// TLS-handshake timeouts, connection pooling and HTTP/2 support even when the
+	// AGENT_TLS_INSECURE escape hatch is enabled (a bare http.Transport would have
+	// no dial/handshake timeouts at all).
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+
+	// ResponseHeaderTimeout bounds how long we wait for the vault's response
+	// headers *after* the request body has been fully written. It does not limit
+	// the time spent streaming the (potentially large) upload body, so big
+	// recordings still upload fine, but a vault/network that disappears while we
+	// wait for the acknowledgement is detected and the upload is retried instead
+	// of hanging indefinitely.
+	transport.ResponseHeaderTimeout = vaultResponseHeaderTimeout()
+
 	if os.Getenv("AGENT_TLS_INSECURE") == "true" {
-		client.Transport = &http.Transport{
-			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		if transport.TLSClientConfig == nil {
+			transport.TLSClientConfig = &tls.Config{}
 		}
+		transport.TLSClientConfig.InsecureSkipVerify = true
 	}
+
+	client := &http.Client{Transport: transport, CheckRedirect: stripVaultHeadersOnCrossHostRedirect}
 	if timeout > 0 {
 		client.Timeout = timeout
 	}
 	return client
 }
+
+// vaultResponseHeaderTimeout returns the maximum time to wait for a vault's
+// response headers after the request body has been written. It defaults to 5
+// minutes — generous enough for the vault to persist/finalize a chunk or a full
+// recording to its storage provider — and can be tuned with the
+// AGENT_VAULT_RESPONSE_HEADER_TIMEOUT_SECONDS environment variable. A value of 0
+// (or a negative/invalid value) disables the timeout.
+func vaultResponseHeaderTimeout() time.Duration {
+	const def = 5 * time.Minute
+	v := os.Getenv("AGENT_VAULT_RESPONSE_HEADER_TIMEOUT_SECONDS")
+	if v == "" {
+		return def
+	}
+	n, err := strconv.Atoi(v)
+	if err != nil {
+		return def
+	}
+	if n <= 0 {
+		return 0
+	}
+	return time.Duration(n) * time.Second
+}

machinery/src/cloud/livehls.go

@@ -7,9 +7,9 @@ import (
 	mqtt "github.com/eclipse/paho.mqtt.golang"
 
 	"github.com/kerberos-io/agent/machinery/src/cloud/livehls"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
+	log "github.com/sirupsen/logrus"
 )
 
 // hlsViewerTimeoutSeconds is how long the agent keeps shipping live HLS segments
@@ -45,22 +45,29 @@ const hlsReadyReannounceSeconds = 2
 // trades a little idle CPU for a near-instant "requesting stream", so viewers no
 // longer wait a full GOP for the first segment to be cut. Set
 // AGENT_LIVE_HLS_PREWARM=false to fall back to the lazy on-demand path above.
-func HandleLiveStreamHLS(configuration *models.Configuration, communication *models.Communication, mqttClient mqtt.Client, subStreamEnabled bool) {
+func HandleLiveStreamHLS(
+	configuration *models.Configuration,
+	communication *models.Communication,
+	mqttClient mqtt.Client,
+	subStreamEnabled bool,
+	mainQueue *packets.Queue,
+	subQueue *packets.Queue,
+) {
 
-	log.Log.Debug("cloud.HandleLiveStreamHLS(): started")
+	log.Debug("cloud.HandleLiveStreamHLS(): started")
 
 	config := configuration.Config
 
 	if config.Offline == "true" {
-		log.Log.Debug("cloud.HandleLiveStreamHLS(): stopping as Offline is enabled.")
+		log.Debug("cloud.HandleLiveStreamHLS(): stopping as Offline is enabled.")
 		return
 	}
 	if config.Capture.Liveview == "false" {
-		log.Log.Debug("cloud.HandleLiveStreamHLS(): stopping as Liveview is disabled.")
+		log.Debug("cloud.HandleLiveStreamHLS(): stopping as Liveview is disabled.")
 		return
 	}
 	if config.HubURI == "" || config.HubKey == "" {
-		log.Log.Debug("cloud.HandleLiveStreamHLS(): stopping as the Hub is not configured (HubURI/HubKey).")
+		log.Debug("cloud.HandleLiveStreamHLS(): stopping as the Hub is not configured (HubURI/HubKey).")
 		return
 	}
 
@@ -88,8 +95,8 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 	// mp4ff's strict parser rejects).
 	requestedQuality := models.StreamQualityAuto
 	useSub := models.SelectSubStreamForQuality(config, requestedQuality, subStreamEnabled)
-	source := buildHLSSource(config, communication, useSub)
-	log.Log.Info("cloud.HandleLiveStreamHLS(): serving live HLS from the " + source.label + " stream")
+	source := buildHLSSource(config, mainQueue, subQueue, useSub)
+	log.Info("cloud.HandleLiveStreamHLS(): serving live HLS from the " + source.label + " stream")
 
 	// prewarm keeps a single long-lived session muxing into an in-memory ring
 	// buffer while idle and flushes it the instant a viewer arrives, eliminating
@@ -97,9 +104,9 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 	// to fall back to the lazy on-demand path.
 	prewarm := os.Getenv("AGENT_LIVE_HLS_PREWARM") != "false"
 	if prewarm {
-		log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS prewarm ENABLED (set AGENT_LIVE_HLS_PREWARM=false to disable)")
+		log.Info("cloud.HandleLiveStreamHLS(): live HLS prewarm ENABLED (set AGENT_LIVE_HLS_PREWARM=false to disable)")
 	} else {
-		log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS prewarm DISABLED (AGENT_LIVE_HLS_PREWARM=false)")
+		log.Info("cloud.HandleLiveStreamHLS(): live HLS prewarm DISABLED (AGENT_LIVE_HLS_PREWARM=false)")
 	}
 
 	// lowLatency enables LL-HLS: each segment is sliced into CMAF parts shipped the
@@ -109,9 +116,9 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 	partTargetMs := uint64(0)
 	if os.Getenv("AGENT_LIVE_HLS_LOW_LATENCY") != "false" {
 		partTargetMs = livehls.DefaultPartTargetMs
-		log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS low-latency (LL-HLS) ENABLED (set AGENT_LIVE_HLS_LOW_LATENCY=false to disable)")
+		log.Info("cloud.HandleLiveStreamHLS(): live HLS low-latency (LL-HLS) ENABLED (set AGENT_LIVE_HLS_LOW_LATENCY=false to disable)")
 	} else {
-		log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS low-latency (LL-HLS) DISABLED (AGENT_LIVE_HLS_LOW_LATENCY=false)")
+		log.Info("cloud.HandleLiveStreamHLS(): live HLS low-latency (LL-HLS) DISABLED (AGENT_LIVE_HLS_LOW_LATENCY=false)")
 	}
 
 	var session *livehls.Session
@@ -123,6 +130,9 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 
 	for cursorError == nil {
 		pkt, cursorError = source.cursor.ReadPacket()
+		if cursorError != nil {
+			break
+		}
 
 		now := time.Now().Unix()
 		select {
@@ -156,9 +166,9 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 				_ = session.Close()
 				session = nil
 			}
-			source = buildHLSSource(config, communication, useSub)
+			source = buildHLSSource(config, mainQueue, subQueue, useSub)
 			lastReadyAnnounce = 0
-			log.Log.Info("cloud.HandleLiveStreamHLS(): switched live HLS to the " + source.label + " stream (quality=" + requestedQuality + ")")
+			log.Info("cloud.HandleLiveStreamHLS(): switched live HLS to the " + source.label + " stream (quality=" + requestedQuality + ")")
 			continue
 		}
 
@@ -184,11 +194,22 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 					StartBuffering: true,
 				})
 				session.SetOnReady(func(sessionID string) {
-					log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS session ready, announcing " + sessionID)
+					log.WithFields(log.Fields{
+						"component":  "cloud/livehls",
+						"event":      "session_ready",
+						"session_id": sessionID,
+					}).Info("Live HLS session ready")
 					publishHLSReady(configuration, mqttClient, hubKey, deviceId, sessionID)
 					lastReadyAnnounce = time.Now().Unix()
 				})
-				log.Log.Info("cloud.HandleLiveStreamHLS(): prewarming live HLS session " + session.SessionID())
+				session.SetOnFailure(func(sessionID, reason string) {
+					publishHLSFailure(configuration, mqttClient, hubKey, deviceId, sessionID, reason)
+				})
+				log.WithFields(log.Fields{
+					"component":  "cloud/livehls",
+					"event":      "session_prewarming",
+					"session_id": session.SessionID(),
+				}).Info("Prewarming live HLS session")
 			}
 
 			if viewerActive {
@@ -207,7 +228,7 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 
 			if len(pkt.Data) > 0 && pkt.IsVideo {
 				if err := session.WritePacket(pkt); err != nil {
-					log.Log.Error("cloud.HandleLiveStreamHLS(): " + err.Error())
+					log.Error("cloud.HandleLiveStreamHLS(): " + err.Error())
 				}
 			}
 			continue
@@ -217,7 +238,12 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 			// No viewer: stop and discard the session so we stop shipping segments.
 			if session != nil {
 				_ = session.Close()
-				log.Log.Info("cloud.HandleLiveStreamHLS(): no active viewers, stopped live HLS session " + session.SessionID())
+				log.WithFields(log.Fields{
+					"component":  "cloud/livehls",
+					"event":      "session_stopped",
+					"reason":     "no_active_viewers",
+					"session_id": session.SessionID(),
+				}).Info("Live HLS session stopped")
 				session = nil
 			}
 			continue
@@ -243,22 +269,33 @@ func HandleLiveStreamHLS(configuration *models.Configuration, communication *mod
 				PartTargetMs: partTargetMs,
 			})
 			session.SetOnReady(func(sessionID string) {
-				log.Log.Info("cloud.HandleLiveStreamHLS(): live HLS session ready, announcing " + sessionID)
+				log.WithFields(log.Fields{
+					"component":  "cloud/livehls",
+					"event":      "session_ready",
+					"session_id": sessionID,
+				}).Info("Live HLS session ready")
 				publishHLSReady(configuration, mqttClient, hubKey, deviceId, sessionID)
 				lastReadyAnnounce = time.Now().Unix()
 			})
-			log.Log.Info("cloud.HandleLiveStreamHLS(): started live HLS session " + session.SessionID())
+			session.SetOnFailure(func(sessionID, reason string) {
+				publishHLSFailure(configuration, mqttClient, hubKey, deviceId, sessionID, reason)
+			})
+			log.WithFields(log.Fields{
+				"component":  "cloud/livehls",
+				"event":      "session_started",
+				"session_id": session.SessionID(),
+			}).Info("Live HLS session started")
 		}
 
 		if err := session.WritePacket(pkt); err != nil {
-			log.Log.Error("cloud.HandleLiveStreamHLS(): " + err.Error())
+			log.Error("cloud.HandleLiveStreamHLS(): " + err.Error())
 		}
 	}
 
 	if session != nil {
 		_ = session.Close()
 	}
-	log.Log.Debug("cloud.HandleLiveStreamHLS(): finished")
+	log.Debug("cloud.HandleLiveStreamHLS(): finished")
 }
 
 // publishHLSReady announces, over MQTT, that a live HLS session is available so
@@ -278,9 +315,48 @@ func publishHLSReady(configuration *models.Configuration, mqttClient mqtt.Client
 	payload, err := models.PackageMQTTMessage(configuration, message)
 	if err == nil {
 		mqttClient.Publish("kerberos/hub/"+hubKey, 0, false, payload)
-		log.Log.Info("cloud.HandleLiveStreamHLS(): announced live HLS session " + sessionID)
+		log.WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "session_announced",
+			"session_id": sessionID,
+		}).Info("Live HLS session announced")
 	} else {
-		log.Log.Error("cloud.HandleLiveStreamHLS(): failed to package receive-hls-ready message: " + err.Error())
+		log.WithError(err).WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "session_announcement_failed",
+			"session_id": sessionID,
+		}).Error("Failed to package live HLS session announcement")
+	}
+}
+
+func publishHLSFailure(configuration *models.Configuration, mqttClient mqtt.Client, hubKey, deviceId, sessionID, reason string) {
+	valueMap := map[string]interface{}{
+		"session": sessionID,
+		"device":  deviceId,
+		"reason":  reason,
+	}
+	message := models.Message{
+		Payload: models.Payload{
+			Action:   "receive-hls-error",
+			DeviceId: deviceId,
+			Value:    valueMap,
+		},
+	}
+	payload, err := models.PackageMQTTMessage(configuration, message)
+	if err == nil {
+		mqttClient.Publish("kerberos/hub/"+hubKey, 0, false, payload)
+		log.WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "session_failure_announced",
+			"reason":     reason,
+			"session_id": sessionID,
+		}).Warn("Live HLS session failure announced")
+	} else {
+		log.WithError(err).WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "session_failure_announcement_failed",
+			"session_id": sessionID,
+		}).Error("Failed to package live HLS session failure")
 	}
 }
 
@@ -302,11 +378,11 @@ type hlsStreamSource struct {
 // is available; otherwise the main (high-resolution) stream is used. A fresh
 // Latest() cursor is created so muxing resumes from the live edge of the chosen
 // stream after a switch.
-func buildHLSSource(config models.Config, communication *models.Communication, useSub bool) hlsStreamSource {
+func buildHLSSource(config models.Config, mainQueue, subQueue *packets.Queue, useSub bool) hlsStreamSource {
 	cam := config.Capture.IPCamera
-	if useSub && communication.SubQueue != nil {
+	if useSub && subQueue != nil {
 		return hlsStreamSource{
-			cursor: communication.SubQueue.Latest(),
+			cursor: subQueue.Latest(),
 			sps:    cam.SubSPSNALUs,
 			pps:    cam.SubPPSNALUs,
 			vps:    cam.SubVPSNALUs,
@@ -316,7 +392,7 @@ func buildHLSSource(config models.Config, communication *models.Communication, u
 		}
 	}
 	return hlsStreamSource{
-		cursor: communication.Queue.Latest(),
+		cursor: mainQueue.Latest(),
 		sps:    cam.SPSNALUs,
 		pps:    cam.PPSNALUs,
 		vps:    cam.VPSNALUs,

machinery/src/cloud/livehls/publisher.go

@@ -24,13 +24,14 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"net/http"
 	"strconv"
 	"strings"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/video"
+	log "github.com/sirupsen/logrus"
 )
 
 const (
@@ -67,6 +68,7 @@ const (
 	// cannot be delivered within roughly its own duration is stale, so the upload
 	// is abandoned (dropped) rather than allowed to back up the pipeline.
 	defaultPublishTimeout = 4 * time.Second
+	maxErrorResponseBytes = 4 << 10
 )
 
 // PublisherConfig carries the hub endpoint and credentials needed to ship live
@@ -214,9 +216,15 @@ func (p *Publisher) post(ctx context.Context, params postParams) error {
 	defer resp.Body.Close()
 
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		return fmt.Errorf("livehls: upload %s rejected: %s", params.name, resp.Status)
+		body, _ := io.ReadAll(io.LimitReader(resp.Body, maxErrorResponseBytes))
+		return fmt.Errorf("livehls: upload %s rejected with status %d (%d response bytes)", params.name, resp.StatusCode, len(body))
 	}
-	log.Log.Debug("livehls.Publisher.post(): shipped " + params.name + " for session " + params.sessionID)
+	log.WithFields(log.Fields{
+		"component":  "cloud/livehls",
+		"event":      "object_uploaded",
+		"object":     params.name,
+		"session_id": params.sessionID,
+	}).Trace("Live HLS object uploaded")
 	return nil
 }
 

machinery/src/cloud/livehls/publisher_test.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"sync"
 	"testing"
 	"time"
@@ -144,12 +145,22 @@ func TestPublisherPublishSegmentSendsSequenceAndDuration(t *testing.T) {
 }
 
 func TestPublisherReturnsErrorOnNon2xx(t *testing.T) {
-	srv, _, _ := newCapturingServer(t, http.StatusInternalServerError)
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusBadRequest)
+		_, _ = w.Write([]byte(`{"error":true,"data":"No user found with this public and private key."}`))
+	}))
+	t.Cleanup(srv.Close)
 	p := testPublisher(srv.URL)
 
 	err := p.PublishSegment(context.Background(), "s", video.LiveSegment{SequenceNumber: 1, Data: []byte("x")})
 	if err == nil {
-		t.Fatal("expected an error on 500 response")
+		t.Fatal("expected an error on 400 response")
+	}
+	if !strings.Contains(err.Error(), "status 400") || !strings.Contains(err.Error(), "71 response bytes") {
+		t.Fatalf("error = %q, want status code and response size", err)
+	}
+	if strings.Contains(err.Error(), "No user found with this public and private key") {
+		t.Fatalf("error exposed Hub response body: %q", err)
 	}
 }
 
@@ -281,7 +292,13 @@ func TestSessionRetriesInitWhenFirstAttemptFails(t *testing.T) {
 	})
 
 	var ready int
+	var failures int
+	var failureReason string
 	sess.SetOnReady(func(string) { ready++ })
+	sess.SetOnFailure(func(_ string, reason string) {
+		failures++
+		failureReason = reason
+	})
 
 	for i := 0; i < 60; i++ {
 		isKey := i%25 == 0
@@ -304,6 +321,9 @@ func TestSessionRetriesInitWhenFirstAttemptFails(t *testing.T) {
 	if ready != 1 {
 		t.Errorf("OnReady fired %d times, want 1", ready)
 	}
+	if failures != 1 || failureReason != "init-upload-failed" {
+		t.Errorf("OnFailure = %d/%q, want 1/init-upload-failed", failures, failureReason)
+	}
 }
 
 // liveTestSPSForSession is the known-good baseline SPS reused across tests.

machinery/src/cloud/livehls/session.go

@@ -8,9 +8,9 @@ import (
 	"sync"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/kerberos-io/agent/machinery/src/video"
+	log "github.com/sirupsen/logrus"
 )
 
 // DefaultTargetSegmentMs is the nominal live segment length. ~2s keeps standard
@@ -46,9 +46,11 @@ type Session struct {
 	// lastInitAt is when the init segment was last (re)uploaded. The init is
 	// re-sent periodically so its short TTL in the hub live window never lapses
 	// mid-session; see refreshInitIfStale.
-	lastInitAt time.Time
-	readyFired bool
-	onReady    func(sessionID string)
+	lastInitAt   time.Time
+	readyFired   bool
+	onReady      func(sessionID string)
+	failureFired bool
+	onFailure    func(sessionID, reason string)
 
 	// uploadsActive gates whether the init and completed segments are shipped to
 	// hub-api. It is true for the default on-demand path. The prewarm path starts
@@ -142,14 +144,25 @@ func NewSession(publisher *Publisher, opts SessionOptions) *Session {
 			return nil
 		}
 		if !s.publishInitIfNeeded() {
-			log.Log.Warning("livehls.Session: dropping segment " +
-				fmt.Sprintf("%d", segment.SequenceNumber) + " because init has not been delivered yet")
+			log.WithFields(log.Fields{
+				"component":        "cloud/livehls",
+				"event":            "segment_dropped",
+				"reason":           "init_unavailable",
+				"segment_sequence": segment.SequenceNumber,
+				"session_id":       s.id,
+			}).Warn("Dropping live HLS segment")
 			return nil
 		}
 		ctx, cancel := s.newContext()
 		defer cancel()
 		if err := s.publisher.PublishSegment(ctx, s.id, segment); err != nil {
-			log.Log.Warning("livehls.Session: " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component":        "cloud/livehls",
+				"event":            "segment_upload_failed",
+				"segment_sequence": segment.SequenceNumber,
+				"session_id":       s.id,
+			}).Warn("Failed to upload live HLS segment")
+			s.fireFailureOnce("segment-upload-failed")
 			return nil
 		}
 		s.fireReadyOnce()
@@ -172,15 +185,27 @@ func NewSession(publisher *Publisher, opts SessionOptions) *Session {
 				return nil
 			}
 			if !s.publishInitIfNeeded() {
-				log.Log.Warning("livehls.Session: dropping part " +
-					fmt.Sprintf("%d.%d", part.SegmentSeq, part.PartIndex) +
-					" because init has not been delivered yet")
+				log.WithFields(log.Fields{
+					"component":        "cloud/livehls",
+					"event":            "part_dropped",
+					"part_index":       part.PartIndex,
+					"reason":           "init_unavailable",
+					"segment_sequence": part.SegmentSeq,
+					"session_id":       s.id,
+				}).Warn("Dropping live HLS part")
 				return nil
 			}
 			ctx, cancel := s.newContext()
 			defer cancel()
 			if err := s.publisher.PublishPart(ctx, s.id, part); err != nil {
-				log.Log.Warning("livehls.Session: " + err.Error())
+				log.WithError(err).WithFields(log.Fields{
+					"component":        "cloud/livehls",
+					"event":            "part_upload_failed",
+					"part_index":       part.PartIndex,
+					"segment_sequence": part.SegmentSeq,
+					"session_id":       s.id,
+				}).Warn("Failed to upload live HLS part")
+				s.fireFailureOnce("part-upload-failed")
 				return nil
 			}
 			s.fireReadyOnce()
@@ -216,6 +241,15 @@ func (s *Session) SetOnReady(fn func(sessionID string)) {
 	s.mu.Unlock()
 }
 
+// SetOnFailure registers a one-shot callback for startup upload failures. The
+// reason is a fixed code rather than an HTTP response body, so credentials or
+// server details cannot leak through MQTT diagnostics.
+func (s *Session) SetOnFailure(fn func(sessionID, reason string)) {
+	s.mu.Lock()
+	s.onFailure = fn
+	s.mu.Unlock()
+}
+
 // prewarmMaxBufferedSegments is how many of the most recent completed segments
 // the prewarm path keeps in memory while idle and flushes to a viewer on arrival.
 // One segment keeps startup instant (the viewer immediately gets a playable
@@ -268,7 +302,13 @@ func (s *Session) SetUploadsActive(active bool) bool {
 		}
 		ctx, cancel := s.newContext()
 		if err := s.publisher.PublishSegment(ctx, s.id, buffered[i]); err != nil {
-			log.Log.Warning("livehls.Session: prewarm flush: " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component":        "cloud/livehls",
+				"event":            "prewarm_segment_upload_failed",
+				"segment_sequence": buffered[i].SequenceNumber,
+				"session_id":       s.id,
+			}).Warn("Failed to upload buffered live HLS segment")
+			s.fireFailureOnce("segment-upload-failed")
 			cancel()
 			continue
 		}
@@ -284,7 +324,14 @@ func (s *Session) SetUploadsActive(active bool) bool {
 		}
 		ctx, cancel := s.newContext()
 		if err := s.publisher.PublishPart(ctx, s.id, bufferedParts[i]); err != nil {
-			log.Log.Warning("livehls.Session: prewarm flush (part): " + err.Error())
+			log.WithError(err).WithFields(log.Fields{
+				"component":        "cloud/livehls",
+				"event":            "prewarm_part_upload_failed",
+				"part_index":       bufferedParts[i].PartIndex,
+				"segment_sequence": bufferedParts[i].SegmentSeq,
+				"session_id":       s.id,
+			}).Warn("Failed to upload buffered live HLS part")
+			s.fireFailureOnce("part-upload-failed")
 			cancel()
 			continue
 		}
@@ -381,7 +428,12 @@ func (s *Session) publishInitIfNeeded() bool {
 	ctx, cancel := s.newContext()
 	defer cancel()
 	if err := s.publisher.PublishInit(ctx, s.id, initBytes); err != nil {
-		log.Log.Warning("livehls.Session: init upload failed, will retry: " + err.Error())
+		log.WithError(err).WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "init_upload_failed",
+			"session_id": s.id,
+		}).Warn("Failed to upload live HLS init segment; retrying")
+		s.fireFailureOnce("init-upload-failed")
 		return false
 	}
 
@@ -389,7 +441,11 @@ func (s *Session) publishInitIfNeeded() bool {
 	s.initPublished = true
 	s.lastInitAt = time.Now()
 	s.mu.Unlock()
-	log.Log.Info("livehls.Session: init segment delivered for session " + s.id)
+	log.WithFields(log.Fields{
+		"component":  "cloud/livehls",
+		"event":      "init_uploaded",
+		"session_id": s.id,
+	}).Info("Live HLS init segment uploaded")
 	return true
 }
 
@@ -422,14 +478,22 @@ func (s *Session) refreshInitIfStale() {
 	ctx, cancel := s.newContext()
 	defer cancel()
 	if err := s.publisher.PublishInit(ctx, s.id, initBytes); err != nil {
-		log.Log.Warning("livehls.Session: init refresh failed, will retry: " + err.Error())
+		log.WithError(err).WithFields(log.Fields{
+			"component":  "cloud/livehls",
+			"event":      "init_refresh_failed",
+			"session_id": s.id,
+		}).Warn("Failed to refresh live HLS init segment; retrying")
 		return
 	}
 
 	s.mu.Lock()
 	s.lastInitAt = time.Now()
 	s.mu.Unlock()
-	log.Log.Debug("livehls.Session: refreshed init segment TTL for session " + s.id)
+	log.WithFields(log.Fields{
+		"component":  "cloud/livehls",
+		"event":      "init_refreshed",
+		"session_id": s.id,
+	}).Debug("Live HLS init segment refreshed")
 }
 
 // fireReadyOnce invokes the OnReady callback the first time it is called.
@@ -445,6 +509,18 @@ func (s *Session) fireReadyOnce() {
 	fn(s.id)
 }
 
+func (s *Session) fireFailureOnce(reason string) {
+	s.mu.Lock()
+	if s.failureFired || s.readyFired || s.onFailure == nil {
+		s.mu.Unlock()
+		return
+	}
+	s.failureFired = true
+	fn := s.onFailure
+	s.mu.Unlock()
+	fn(s.id, reason)
+}
+
 // newSessionID returns a short, unique, URL-safe session identifier of the form
 // <unix-seconds>-<random-hex>.
 func newSessionID() string {

machinery/src/cloud/livemoq/annexb.go

@@ -0,0 +1,103 @@
+package livemoq
+
+import (
+	"bytes"
+	"strings"
+
+	"github.com/bluenviron/mediacommon/pkg/codecs/h264"
+	"github.com/kerberos-io/agent/machinery/src/models"
+)
+
+var annexBStartCode = []byte{0x00, 0x00, 0x00, 0x01}
+
+// H264NormalizationStats reports malformed duplication removed from an access unit.
+type H264NormalizationStats struct {
+	DuplicateIDRNALUs int
+}
+
+// EnsureAnnexB restores the start code stripped by the Agent capture queue.
+func EnsureAnnexB(payload []byte) []byte {
+	if hasAnnexBStartCode(payload) {
+		return payload
+	}
+
+	framed := make([]byte, 0, len(annexBStartCode)+len(payload))
+	framed = append(framed, annexBStartCode...)
+	return append(framed, payload...)
+}
+
+// NormalizeH264AccessUnit removes delimiters and exact duplicate parameter-set
+// or IDR NALUs that can confuse older MoQ splitters and decoders.
+func NormalizeH264AccessUnit(payload []byte) ([]byte, error) {
+	normalized, _, err := NormalizeH264AccessUnitWithStats(payload)
+	return normalized, err
+}
+
+// NormalizeH264AccessUnitWithStats also reports exact duplicate IDR NALUs.
+func NormalizeH264AccessUnitWithStats(payload []byte) ([]byte, H264NormalizationStats, error) {
+	nalus, err := h264.AnnexBUnmarshal(EnsureAnnexB(payload))
+	if err != nil {
+		return nil, H264NormalizationStats{}, err
+	}
+
+	stats := H264NormalizationStats{}
+	normalized := make([][]byte, 0, len(nalus))
+	for _, nalu := range nalus {
+		if len(nalu) == 0 || nalu[0]&0x1f == 9 {
+			continue
+		}
+		naluType := nalu[0] & 0x1f
+		if naluType == 7 || naluType == 8 || naluType == 5 {
+			duplicate := false
+			for _, existing := range normalized {
+				if bytes.Equal(existing, nalu) {
+					duplicate = true
+					break
+				}
+			}
+			if duplicate {
+				if naluType == 5 {
+					stats.DuplicateIDRNALUs++
+				}
+				continue
+			}
+		}
+		normalized = append(normalized, nalu)
+	}
+
+	result, err := h264.AnnexBMarshal(normalized)
+	return result, stats, err
+}
+
+// BroadcastPath returns the relay path a quality tier is published on. Every
+// tier gets its own broadcast so a viewer switches between the camera's main and
+// sub stream by resubscribing to another path, without any control channel back
+// to the Agent. The high tier keeps the historical ".../live.hang" path so
+// existing viewers keep working; the low tier lives next to it on
+// ".../live-low.hang".
+func BroadcastPath(prefix string, deviceKey string, quality string) string {
+	prefix = strings.Trim(prefix, "/")
+	if prefix == "" {
+		prefix = "devices"
+	}
+	name := "live.hang"
+	if quality == models.StreamQualityLow {
+		name = "live-low.hang"
+	}
+	return prefix + "/" + strings.Trim(deviceKey, "/") + "/" + name
+}
+
+// TimestampUs converts the capture presentation timestamp from milliseconds.
+// CompositionTime must not be added: it is already represented in the PTS and
+// is only used by muxers to derive DTS for streams containing B-frames.
+func TimestampUs(presentationTimeMs int64) uint64 {
+	if presentationTimeMs < 0 {
+		return 0
+	}
+	return uint64(presentationTimeMs) * 1000
+}
+
+func hasAnnexBStartCode(payload []byte) bool {
+	return len(payload) >= 4 && payload[0] == 0 && payload[1] == 0 &&
+		((payload[2] == 0 && payload[3] == 1) || payload[2] == 1)
+}

machinery/src/cloud/livemoq/annexb_test.go

@@ -0,0 +1,116 @@
+package livemoq
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/kerberos-io/agent/machinery/src/models"
+)
+
+func TestEnsureAnnexB(t *testing.T) {
+	tests := []struct {
+		name    string
+		payload []byte
+		want    []byte
+	}{
+		{
+			name:    "missing start code",
+			payload: []byte{0x41, 0x01},
+			want:    []byte{0x00, 0x00, 0x00, 0x01, 0x41, 0x01},
+		},
+		{
+			name:    "four byte start code",
+			payload: []byte{0x00, 0x00, 0x00, 0x01, 0x65},
+			want:    []byte{0x00, 0x00, 0x00, 0x01, 0x65},
+		},
+		{
+			name:    "three byte start code",
+			payload: []byte{0x00, 0x00, 0x01, 0x41},
+			want:    []byte{0x00, 0x00, 0x01, 0x41},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if got := EnsureAnnexB(test.payload); !bytes.Equal(got, test.want) {
+				t.Fatalf("EnsureAnnexB() = %x, want %x", got, test.want)
+			}
+		})
+	}
+}
+
+func TestNormalizeH264AccessUnit(t *testing.T) {
+	startCode := []byte{0x00, 0x00, 0x00, 0x01}
+	sps := []byte{0x67, 0x42, 0x00, 0x1f}
+	pps := []byte{0x68, 0xce, 0x06, 0xe2}
+	aud := []byte{0x09, 0xf0}
+	idr := []byte{0x65, 0x88, 0x84}
+
+	payload := make([]byte, 0)
+	for _, nalu := range [][]byte{sps, pps, aud, sps, pps, idr} {
+		payload = append(payload, startCode...)
+		payload = append(payload, nalu...)
+	}
+
+	got, err := NormalizeH264AccessUnit(payload)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := make([]byte, 0)
+	for _, nalu := range [][]byte{sps, pps, idr} {
+		want = append(want, startCode...)
+		want = append(want, nalu...)
+	}
+	if !bytes.Equal(got, want) {
+		t.Fatalf("NormalizeH264AccessUnit() = %x, want %x", got, want)
+	}
+}
+
+func TestNormalizeH264AccessUnitRemovesOnlyExactDuplicateIDRSlices(t *testing.T) {
+	startCode := []byte{0x00, 0x00, 0x00, 0x01}
+	idrSlice1 := []byte{0x65, 0x88, 0x84}
+	idrSlice2 := []byte{0x65, 0x44, 0x22}
+
+	payload := make([]byte, 0)
+	for _, nalu := range [][]byte{idrSlice1, idrSlice1, idrSlice2} {
+		payload = append(payload, startCode...)
+		payload = append(payload, nalu...)
+	}
+
+	got, stats, err := NormalizeH264AccessUnitWithStats(payload)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := make([]byte, 0)
+	for _, nalu := range [][]byte{idrSlice1, idrSlice2} {
+		want = append(want, startCode...)
+		want = append(want, nalu...)
+	}
+	if !bytes.Equal(got, want) {
+		t.Fatalf("NormalizeH264AccessUnitWithStats() = %x, want %x", got, want)
+	}
+	if stats.DuplicateIDRNALUs != 1 {
+		t.Fatalf("DuplicateIDRNALUs = %d, want 1", stats.DuplicateIDRNALUs)
+	}
+}
+
+func TestBroadcastPath(t *testing.T) {
+	if got := BroadcastPath("/devices/", "/camera-1/", models.StreamQualityHigh); got != "devices/camera-1/live.hang" {
+		t.Fatalf("BroadcastPath() high = %q", got)
+	}
+	if got := BroadcastPath("", "camera-1", models.StreamQualityHigh); got != "devices/camera-1/live.hang" {
+		t.Fatalf("BroadcastPath() default = %q", got)
+	}
+	if got := BroadcastPath("", "camera-1", models.StreamQualityLow); got != "devices/camera-1/live-low.hang" {
+		t.Fatalf("BroadcastPath() low = %q", got)
+	}
+}
+
+func TestTimestampUs(t *testing.T) {
+	if got := TimestampUs(1234); got != 1_234_000 {
+		t.Fatalf("TimestampUs() = %d, want 1234000", got)
+	}
+	if got := TimestampUs(-1); got != 0 {
+		t.Fatalf("TimestampUs() negative = %d, want 0", got)
+	}
+}

machinery/src/cloud/livemoq/dedup.go

@@ -0,0 +1,41 @@
+package livemoq
+
+import (
+	"crypto/sha256"
+	"time"
+)
+
+type KeyframeDeduplicator struct {
+	hasPrevious  bool
+	timestampMs  int64
+	capturedAtMs int64
+	observedAt   time.Time
+	digest       [sha256.Size]byte
+}
+
+func (d *KeyframeDeduplicator) Reset() {
+	*d = KeyframeDeduplicator{}
+}
+
+// IsDuplicate reports exact repeated keyframe access units observed close
+// together. Distinct IDR slices within one access unit remain untouched.
+func (d *KeyframeDeduplicator) IsDuplicate(timestampMs int64, capturedAtMs int64, payload []byte, observedAt time.Time, window time.Duration) bool {
+	digest := sha256.Sum256(payload)
+	duplicate := d.hasPrevious && d.timestampMs == timestampMs && d.digest == digest
+	if duplicate {
+		if capturedAtMs > 0 && d.capturedAtMs > 0 {
+			gap := time.Duration(capturedAtMs-d.capturedAtMs) * time.Millisecond
+			duplicate = gap >= 0 && gap <= window
+		} else {
+			gap := observedAt.Sub(d.observedAt)
+			duplicate = gap >= 0 && gap <= window
+		}
+	}
+
+	d.hasPrevious = true
+	d.timestampMs = timestampMs
+	d.capturedAtMs = capturedAtMs
+	d.observedAt = observedAt
+	d.digest = digest
+	return duplicate
+}

machinery/src/cloud/livemoq/dedup_test.go

@@ -0,0 +1,60 @@
+package livemoq
+
+import (
+	"testing"
+	"time"
+)
+
+func TestKeyframeDeduplicator(t *testing.T) {
+	now := time.UnixMilli(10_000)
+	window := 500 * time.Millisecond
+	payload := []byte{0x00, 0x00, 0x00, 0x01, 0x65, 0x88}
+	deduplicator := KeyframeDeduplicator{}
+
+	if deduplicator.IsDuplicate(1_000, 10_000, payload, now, window) {
+		t.Fatal("first keyframe reported as duplicate")
+	}
+	if !deduplicator.IsDuplicate(1_000, 10_020, payload, now.Add(20*time.Millisecond), window) {
+		t.Fatal("exact repeated keyframe was not reported as duplicate")
+	}
+	if deduplicator.IsDuplicate(2_000, 11_000, payload, now.Add(time.Second), window) {
+		t.Fatal("same payload with a new timestamp reported as duplicate")
+	}
+	if deduplicator.IsDuplicate(2_000, 11_020, append(payload, 0x01), now.Add(1020*time.Millisecond), window) {
+		t.Fatal("different payload with the same timestamp reported as duplicate")
+	}
+}
+
+func TestKeyframeDeduplicatorAllowsTimestampReuseOutsideWindow(t *testing.T) {
+	now := time.UnixMilli(10_000)
+	payload := []byte{0x00, 0x00, 0x00, 0x01, 0x65, 0x88}
+	deduplicator := KeyframeDeduplicator{}
+
+	deduplicator.IsDuplicate(1_000, 10_000, payload, now, 500*time.Millisecond)
+	if deduplicator.IsDuplicate(1_000, 20_000, payload, now.Add(10*time.Second), 500*time.Millisecond) {
+		t.Fatal("later keyframe after timestamp reset reported as duplicate")
+	}
+}
+
+func TestKeyframeDeduplicatorFallsBackToObservationTime(t *testing.T) {
+	now := time.UnixMilli(10_000)
+	payload := []byte{0x65, 0x88}
+	deduplicator := KeyframeDeduplicator{}
+
+	deduplicator.IsDuplicate(1_000, 0, payload, now, 500*time.Millisecond)
+	if !deduplicator.IsDuplicate(1_000, 0, payload, now.Add(20*time.Millisecond), 500*time.Millisecond) {
+		t.Fatal("duplicate without capture time was not reported")
+	}
+}
+
+func TestKeyframeDeduplicatorReset(t *testing.T) {
+	now := time.UnixMilli(10_000)
+	payload := []byte{0x65, 0x88}
+	deduplicator := KeyframeDeduplicator{}
+
+	deduplicator.IsDuplicate(1_000, 10_000, payload, now, 500*time.Millisecond)
+	deduplicator.Reset()
+	if deduplicator.IsDuplicate(1_000, 10_020, payload, now.Add(20*time.Millisecond), 500*time.Millisecond) {
+		t.Fatal("first keyframe after reset reported as duplicate")
+	}
+}

machinery/src/cloud/livemoq/recovery.go

@@ -0,0 +1,101 @@
+package livemoq
+
+import (
+	"sync/atomic"
+	"time"
+)
+
+type FrameGateEvent uint8
+
+const (
+	FrameGateEventNone FrameGateEvent = iota
+	FrameGateEventStarted
+	FrameGateEventLagging
+	FrameGateEventRecovered
+)
+
+// FrameGate keeps publication on a decodable, recent GOP.
+type FrameGate struct {
+	started    bool
+	recovering bool
+}
+
+// AudienceGate publishes every frame while watched and only keyframes while
+// idle. The idle keyframes keep the relay's latest cached group current without
+// paying the bandwidth cost of a full stream when nobody is watching.
+type AudienceGate struct {
+	idle bool
+}
+
+func (g *AudienceGate) Allow(hasSubscribers bool, isKeyFrame bool) (allowed bool, enteredIdle bool) {
+	if hasSubscribers {
+		g.idle = false
+		return true, false
+	}
+
+	enteredIdle = !g.idle
+	g.idle = true
+	return isKeyFrame, enteredIdle
+}
+
+// WriteWatchdog tracks the single synchronous frame write performed by a MoQ
+// publisher so another goroutine can interrupt a wedged native call.
+type WriteWatchdog struct {
+	startedAt atomic.Int64
+}
+
+func (w *WriteWatchdog) Begin(now time.Time) {
+	w.startedAt.Store(now.UnixNano())
+}
+
+func (w *WriteWatchdog) End() {
+	w.startedAt.Store(0)
+}
+
+func (w *WriteWatchdog) Elapsed(now time.Time) (time.Duration, bool) {
+	startedAt := w.startedAt.Load()
+	if startedAt == 0 {
+		return 0, false
+	}
+	elapsed := now.Sub(time.Unix(0, startedAt))
+	if elapsed < 0 {
+		elapsed = 0
+	}
+	return elapsed, true
+}
+
+// Reset closes the gate so publication resumes on the next keyframe. Entering
+// idle mode uses it to start the relay-cache refresh on a complete GOP.
+func (g *FrameGate) Reset() {
+	g.started = false
+	g.recovering = false
+}
+
+// Allow rejects stale frames and waits for a fresh keyframe before reopening.
+func (g *FrameGate) Allow(isKeyFrame bool, capturedAtMs int64, now time.Time, maxAge time.Duration) (bool, FrameGateEvent) {
+	if capturedAtMs > 0 && now.Sub(time.UnixMilli(capturedAtMs)) > maxAge {
+		event := FrameGateEventNone
+		if g.started {
+			if !g.recovering {
+				event = FrameGateEventLagging
+			}
+			g.started = false
+			g.recovering = true
+		}
+		return false, event
+	}
+
+	if !g.started {
+		if !isKeyFrame {
+			return false, FrameGateEventNone
+		}
+		g.started = true
+		if g.recovering {
+			g.recovering = false
+			return true, FrameGateEventRecovered
+		}
+		return true, FrameGateEventStarted
+	}
+
+	return true, FrameGateEventNone
+}

machinery/src/cloud/livemoq/recovery_test.go

@@ -0,0 +1,91 @@
+package livemoq
+
+import (
+	"testing"
+	"time"
+)
+
+func TestFrameGateRecoversAtFreshKeyframe(t *testing.T) {
+	now := time.UnixMilli(10_000)
+	maxAge := 1500 * time.Millisecond
+	gate := FrameGate{}
+
+	tests := []struct {
+		name         string
+		isKeyFrame   bool
+		capturedAtMs int64
+		wantAllowed  bool
+		wantEvent    FrameGateEvent
+	}{
+		{name: "waits for initial keyframe", capturedAtMs: 10_000},
+		{name: "starts at initial keyframe", isKeyFrame: true, capturedAtMs: 10_000, wantAllowed: true, wantEvent: FrameGateEventStarted},
+		{name: "publishes fresh delta", capturedAtMs: 10_020, wantAllowed: true},
+		{name: "detects stale packet", capturedAtMs: 8_000, wantEvent: FrameGateEventLagging},
+		{name: "rejects fresh delta while recovering", capturedAtMs: 10_040},
+		{name: "rejects stale keyframe without duplicate event", isKeyFrame: true, capturedAtMs: 8_000},
+		{name: "recovers at fresh keyframe", isKeyFrame: true, capturedAtMs: 10_060, wantAllowed: true, wantEvent: FrameGateEventRecovered},
+		{name: "publishes delta after recovery", capturedAtMs: 10_080, wantAllowed: true},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			allowed, event := gate.Allow(test.isKeyFrame, test.capturedAtMs, now, maxAge)
+			if allowed != test.wantAllowed {
+				t.Fatalf("Allow() allowed = %t, want %t", allowed, test.wantAllowed)
+			}
+			if event != test.wantEvent {
+				t.Fatalf("Allow() event = %d, want %d", event, test.wantEvent)
+			}
+		})
+	}
+}
+
+func TestFrameGateAllowsMissingCaptureTime(t *testing.T) {
+	gate := FrameGate{}
+	allowed, event := gate.Allow(true, 0, time.Now(), time.Second)
+	if !allowed || event != FrameGateEventStarted {
+		t.Fatalf("Allow() = (%t, %d), want (true, %d)", allowed, event, FrameGateEventStarted)
+	}
+}
+
+func TestAudienceGateRefreshesOnlyKeyframesWhileIdle(t *testing.T) {
+	gate := AudienceGate{}
+
+	if allowed, enteredIdle := gate.Allow(true, false); !allowed || enteredIdle {
+		t.Fatalf("active delta = (%t, %t), want (true, false)", allowed, enteredIdle)
+	}
+	if allowed, enteredIdle := gate.Allow(false, false); allowed || !enteredIdle {
+		t.Fatalf("first idle delta = (%t, %t), want (false, true)", allowed, enteredIdle)
+	}
+	if allowed, enteredIdle := gate.Allow(false, true); !allowed || enteredIdle {
+		t.Fatalf("idle keyframe = (%t, %t), want (true, false)", allowed, enteredIdle)
+	}
+	if allowed, enteredIdle := gate.Allow(false, false); allowed || enteredIdle {
+		t.Fatalf("later idle delta = (%t, %t), want (false, false)", allowed, enteredIdle)
+	}
+	if allowed, enteredIdle := gate.Allow(true, false); !allowed || enteredIdle {
+		t.Fatalf("resumed delta = (%t, %t), want (true, false)", allowed, enteredIdle)
+	}
+	if allowed, enteredIdle := gate.Allow(false, true); !allowed || !enteredIdle {
+		t.Fatalf("next idle keyframe = (%t, %t), want (true, true)", allowed, enteredIdle)
+	}
+}
+
+func TestWriteWatchdogTracksActiveWrite(t *testing.T) {
+	now := time.Unix(10, 0)
+	watchdog := WriteWatchdog{}
+
+	if elapsed, active := watchdog.Elapsed(now); active || elapsed != 0 {
+		t.Fatalf("Elapsed() before Begin() = (%s, %t), want (0, false)", elapsed, active)
+	}
+
+	watchdog.Begin(now)
+	if elapsed, active := watchdog.Elapsed(now.Add(5 * time.Second)); !active || elapsed != 5*time.Second {
+		t.Fatalf("Elapsed() during write = (%s, %t), want (5s, true)", elapsed, active)
+	}
+
+	watchdog.End()
+	if elapsed, active := watchdog.Elapsed(now.Add(6 * time.Second)); active || elapsed != 0 {
+		t.Fatalf("Elapsed() after End() = (%s, %t), want (0, false)", elapsed, active)
+	}
+}

machinery/src/cloud/livemoq_disabled.go

@@ -0,0 +1,21 @@
+//go:build !moq
+
+package cloud
+
+import (
+	"context"
+
+	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/packets"
+)
+
+// StartLiveStreamMoQ is disabled in the standard Agent build.
+func StartLiveStreamMoQ(
+	_ context.Context,
+	_ *models.Configuration,
+	_ *models.Communication,
+	_ bool,
+	_ *packets.Queue,
+	_ *packets.Queue,
+) {
+}

machinery/src/cloud/livemoq_enabled.go

@@ -0,0 +1,512 @@
+//go:build moq
+
+package cloud
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/url"
+	"os"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/kerberos-io/agent/machinery/src/cloud/livemoq"
+	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/packets"
+	"github.com/moq-dev/moq-go/moq"
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	defaultMoQRelayURL      = "https://relay.uug.ai/anon"
+	minMoQRetryDelay        = time.Second
+	maxMoQRetryDelay        = 30 * time.Second
+	defaultMoQLivePacketAge = 1500 * time.Millisecond
+	minMoQLivePacketAge     = 250 * time.Millisecond
+	maxMoQLivePacketAge     = 30 * time.Second
+	defaultMoQWriteTimeout  = 5 * time.Second
+	minMoQWriteTimeout      = time.Second
+	maxMoQWriteTimeout      = time.Minute
+	moQWriteWatchInterval   = 250 * time.Millisecond
+	slowMoQWriteThreshold   = 100 * time.Millisecond
+	moQWriteWarningInterval = 10 * time.Second
+	duplicateKeyframeWindow = 500 * time.Millisecond
+)
+
+type liveMoQConfig struct {
+	relayURL      string
+	broadcast     string
+	quality       string
+	sourceLabel   string
+	queue         *packets.Queue
+	communication *models.Communication
+	maxPacketAge  time.Duration
+	writeTimeout  time.Duration
+}
+
+func (c liveMoQConfig) logEntry(event string) *log.Entry {
+	return log.WithFields(log.Fields{
+		"component": "moq",
+		"event":     event,
+		"quality":   c.quality,
+		"stream":    c.sourceLabel,
+	})
+}
+
+func moQRelayHost(relayURL string) string {
+	parsed, err := url.Parse(relayURL)
+	if err != nil {
+		return ""
+	}
+	return parsed.Host
+}
+
+func (c liveMoQConfig) packetAgeLimit() time.Duration {
+	if c.maxPacketAge > 0 {
+		return c.maxPacketAge
+	}
+	return defaultMoQLivePacketAge
+}
+
+func (c liveMoQConfig) writeTimeoutLimit() time.Duration {
+	if c.writeTimeout > 0 {
+		return c.writeTimeout
+	}
+	return defaultMoQWriteTimeout
+}
+
+func boundedMoQDuration(name string, fallback, minimum, maximum time.Duration) time.Duration {
+	raw := strings.TrimSpace(os.Getenv(name))
+	if raw == "" {
+		return fallback
+	}
+
+	value, err := time.ParseDuration(raw)
+	if err != nil {
+		log.WithError(err).WithFields(log.Fields{
+			"component":        "moq",
+			"configured_value": raw,
+			"effective_value":  fallback.String(),
+			"event":            "duration_invalid",
+			"variable":         name,
+		}).Warn("Invalid MoQ duration; using default")
+		return fallback
+	}
+	if value < minimum {
+		log.WithFields(log.Fields{
+			"component":        "moq",
+			"configured_value": value.String(),
+			"effective_value":  minimum.String(),
+			"event":            "duration_clamped",
+			"minimum":          minimum.String(),
+			"variable":         name,
+		}).Warn("MoQ duration is below the supported minimum")
+		return minimum
+	}
+	if value > maximum {
+		log.WithFields(log.Fields{
+			"component":        "moq",
+			"configured_value": value.String(),
+			"effective_value":  maximum.String(),
+			"event":            "duration_clamped",
+			"maximum":          maximum.String(),
+			"variable":         name,
+		}).Warn("MoQ duration exceeds the supported maximum")
+		return maximum
+	}
+	return value
+}
+
+// StartLiveStreamMoQ starts the publisher only in the dedicated MoQ build and
+// only when explicitly enabled by the deployment.
+//
+// Unlike WebRTC and HLS — where a viewer negotiates a session with the Agent and
+// can therefore ask for another quality on the fly — MoQ viewers subscribe to a
+// relay and never talk to the Agent. The quality selector is honoured by
+// publishing each tier as its OWN broadcast (see livemoq.BroadcastPath): the
+// high tier from the camera's highest-resolution stream and the low tier from
+// its sub stream, so switching quality in the frontend is a resubscribe to the
+// other path. Each tier uploads every frame while watched and keyframes only
+// while idle, keeping the relay cache current without continuously sending the
+// full unused stream.
+func StartLiveStreamMoQ(
+	ctx context.Context,
+	configuration *models.Configuration,
+	communication *models.Communication,
+	subStreamEnabled bool,
+	mainQueue *packets.Queue,
+	subQueue *packets.Queue,
+) {
+	if os.Getenv("AGENT_LIVE_MOQ_ENABLED") != "true" {
+		return
+	}
+
+	config := configuration.Config
+	if config.Offline == "true" || config.Capture.Liveview == "false" {
+		log.WithFields(log.Fields{
+			"component": "moq",
+			"event":     "publisher_disabled",
+			"liveview":  config.Capture.Liveview,
+			"offline":   config.Offline,
+		}).Debug("MoQ publisher disabled by Agent configuration")
+		return
+	}
+	if config.Key == "" {
+		log.WithFields(log.Fields{
+			"component": "moq",
+			"event":     "publisher_configuration_invalid",
+			"variable":  "AGENT_KEY",
+		}).Warn("MoQ publisher requires an Agent key")
+		return
+	}
+
+	// Both tiers are published by default. AGENT_LIVE_MOQ_QUALITY pins the Agent
+	// to a single tier for deployments that must never publish the other one
+	// (viewers asking for the pinned-away tier then find no broadcast).
+	qualities := []string{models.StreamQualityHigh, models.StreamQualityLow}
+	switch strings.ToLower(strings.TrimSpace(os.Getenv("AGENT_LIVE_MOQ_QUALITY"))) {
+	case models.StreamQualityHigh:
+		qualities = []string{models.StreamQualityHigh}
+	case models.StreamQualityLow:
+		qualities = []string{models.StreamQualityLow}
+	}
+
+	relayURL := os.Getenv("AGENT_LIVE_MOQ_URL")
+	if relayURL == "" {
+		relayURL = defaultMoQRelayURL
+	}
+	broadcastPrefix := os.Getenv("AGENT_LIVE_MOQ_BROADCAST_PREFIX")
+	maxPacketAge := boundedMoQDuration("AGENT_LIVE_MOQ_MAX_PACKET_AGE", defaultMoQLivePacketAge, minMoQLivePacketAge, maxMoQLivePacketAge)
+	writeTimeout := boundedMoQDuration("AGENT_LIVE_MOQ_WRITE_TIMEOUT", defaultMoQWriteTimeout, minMoQWriteTimeout, maxMoQWriteTimeout)
+
+	var publishers sync.WaitGroup
+	for _, quality := range qualities {
+		queue := mainQueue
+		sourceLabel := "main"
+		if models.SelectSubStreamForQuality(config, quality, subStreamEnabled) && subQueue != nil {
+			queue = subQueue
+			sourceLabel = "sub"
+		}
+		if queue == nil {
+			log.WithFields(log.Fields{
+				"component": "moq",
+				"event":     "packet_queue_unavailable",
+				"quality":   quality,
+			}).Warn("MoQ packet queue is unavailable")
+			continue
+		}
+		publisherConfig := liveMoQConfig{
+			relayURL:      relayURL,
+			broadcast:     livemoq.BroadcastPath(broadcastPrefix, config.Key, quality),
+			quality:       quality,
+			sourceLabel:   sourceLabel,
+			queue:         queue,
+			communication: communication,
+			maxPacketAge:  maxPacketAge,
+			writeTimeout:  writeTimeout,
+		}
+		publishers.Add(1)
+		go func() {
+			defer publishers.Done()
+			runLiveStreamMoQ(ctx, publisherConfig)
+		}()
+	}
+	publishers.Wait()
+}
+
+func runLiveStreamMoQ(ctx context.Context, config liveMoQConfig) {
+	config.logEntry("publisher_started").Info("MoQ publisher started")
+	config.logEntry("publisher_configuration").WithFields(log.Fields{
+		"max_packet_age_ms": config.packetAgeLimit().Milliseconds(),
+		"relay_host":        moQRelayHost(config.relayURL),
+		"write_timeout_ms":  config.writeTimeoutLimit().Milliseconds(),
+	}).Debug("MoQ publisher configuration")
+
+	retryDelay := minMoQRetryDelay
+	for ctx.Err() == nil {
+		connectedAt := time.Now()
+		err := publishLiveStreamMoQ(ctx, config)
+		if ctx.Err() != nil {
+			return
+		}
+		if config.communication != nil {
+			config.communication.RecordMoQReconnect(config.quality)
+		}
+		config.logEntry("publisher_reconnecting").WithError(err).WithFields(log.Fields{
+			"connected_duration_ms": time.Since(connectedAt).Milliseconds(),
+			"retry_delay_ms":        retryDelay.Milliseconds(),
+		}).Warn("MoQ publisher stopped; reconnecting")
+		if time.Since(connectedAt) >= time.Minute {
+			retryDelay = minMoQRetryDelay
+		}
+
+		timer := time.NewTimer(retryDelay)
+		select {
+		case <-ctx.Done():
+			timer.Stop()
+			return
+		case <-timer.C:
+		}
+		if retryDelay < maxMoQRetryDelay {
+			retryDelay *= 2
+			if retryDelay > maxMoQRetryDelay {
+				retryDelay = maxMoQRetryDelay
+			}
+		}
+	}
+}
+
+func publishLiveStreamMoQ(ctx context.Context, config liveMoQConfig) error {
+	client, err := moq.Dial(ctx, config.relayURL)
+	if err != nil {
+		return fmt.Errorf("connect to relay: %w", err)
+	}
+	defer client.Close()
+	config.logEntry("relay_connected").WithField("relay_host", moQRelayHost(config.relayURL)).
+		Info("MoQ relay connected")
+
+	publisherCtx, cancelPublisher := context.WithCancel(ctx)
+	defer cancelPublisher()
+	sessionClosed := make(chan error, 1)
+	sessionWatchDone := make(chan struct{})
+	go func() {
+		defer close(sessionWatchDone)
+		sessionClosed <- client.Session().Closed(publisherCtx)
+		cancelPublisher()
+	}()
+
+	broadcast, err := client.CreateBroadcast(config.broadcast)
+	if err != nil {
+		return fmt.Errorf("create broadcast: %w", err)
+	}
+	var finishBroadcastOnce sync.Once
+	var finishBroadcastErr error
+	finishBroadcast := func() error {
+		finishBroadcastOnce.Do(func() {
+			finishBroadcastErr = broadcast.Finish()
+		})
+		return finishBroadcastErr
+	}
+	defer finishBroadcast()
+
+	stream, err := broadcast.PublishMedia("avc3", nil)
+	if err != nil {
+		return fmt.Errorf("create H.264 media stream: %w", err)
+	}
+	var finishStreamOnce sync.Once
+	var finishStreamErr error
+	finishStream := func() error {
+		finishStreamOnce.Do(func() {
+			finishStreamErr = stream.Finish()
+		})
+		return finishStreamErr
+	}
+	defer finishStream()
+
+	// Publish the complete stream while watched and only keyframes while idle.
+	// `publishing` starts true so the track becomes discoverable on the relay
+	// before the first subscriber; after that, the subscriber watcher selects the
+	// full-rate or keyframe-refresh mode.
+	publishing := &atomic.Bool{}
+	publishing.Store(true)
+	subscriberWatchDone := make(chan struct{})
+	go func() {
+		defer close(subscriberWatchDone)
+		watchLiveStreamMoQSubscribers(publisherCtx, stream, publishing, config)
+	}()
+	writeWatchdog := &livemoq.WriteWatchdog{}
+	writeWatchDone := make(chan struct{})
+	closePublisher := func() error {
+		return closeLiveMoQPublisher(finishStream, finishBroadcast, client.Close)
+	}
+	go func() {
+		defer close(writeWatchDone)
+		watchLiveStreamMoQWrites(publisherCtx, closePublisher, writeWatchdog, config)
+	}()
+	defer func() {
+		cancelPublisher()
+		<-writeWatchDone
+		<-subscriberWatchDone
+		<-sessionWatchDone
+	}()
+
+	cursor := config.queue.Latest()
+	gate := livemoq.FrameGate{}
+	audienceGate := livemoq.AudienceGate{}
+	deduplicator := livemoq.KeyframeDeduplicator{}
+	var lastSlowWriteWarning time.Time
+	var lastDuplicateKeyframeWarning time.Time
+	for {
+		select {
+		case err := <-sessionClosed:
+			return fmt.Errorf("relay session closed: %w", err)
+		default:
+		}
+
+		packet, err := cursor.ReadPacketContext(publisherCtx)
+		if err != nil {
+			select {
+			case sessionErr := <-sessionClosed:
+				return fmt.Errorf("relay session closed: %w", sessionErr)
+			default:
+			}
+			if ctx.Err() != nil {
+				return ctx.Err()
+			}
+			return fmt.Errorf("read packet: %w", err)
+		}
+		if !packet.IsVideo || len(packet.Data) == 0 || !strings.EqualFold(packet.Codec, "H264") {
+			continue
+		}
+		allowedForAudience, enteredIdle := audienceGate.Allow(publishing.Load(), packet.IsKeyFrame)
+		if enteredIdle {
+			gate.Reset()
+			deduplicator.Reset()
+		}
+		if !allowedForAudience {
+			continue
+		}
+		allowed, event := gate.Allow(packet.IsKeyFrame, packet.CurrentTime, time.Now(), config.packetAgeLimit())
+		switch event {
+		case livemoq.FrameGateEventStarted:
+			config.logEntry("broadcast_live").WithFields(log.Fields{
+				"codec":        packet.Codec,
+				"timestamp_ms": packet.Time,
+			}).Info("MoQ broadcast is live")
+		case livemoq.FrameGateEventLagging:
+			config.logEntry("stream_lagging").WithFields(log.Fields{
+				"max_packet_age_ms": config.packetAgeLimit().Milliseconds(),
+				"timestamp_ms":      packet.Time,
+			}).Warn("MoQ stream is lagging; dropping packets until a recent keyframe")
+		case livemoq.FrameGateEventRecovered:
+			config.logEntry("stream_recovered").WithField("timestamp_ms", packet.Time).
+				Info("MoQ stream recovered at a recent keyframe")
+		}
+		if !allowed {
+			continue
+		}
+		payload, normalizationStats, err := livemoq.NormalizeH264AccessUnitWithStats(packet.Data)
+		if err != nil {
+			return fmt.Errorf("normalize H.264 access unit: %w", err)
+		}
+		if normalizationStats.DuplicateIDRNALUs > 0 && time.Since(lastDuplicateKeyframeWarning) >= moQWriteWarningInterval {
+			config.logEntry("duplicate_idr_removed").WithFields(log.Fields{
+				"duplicate_idr_count": normalizationStats.DuplicateIDRNALUs,
+				"timestamp_ms":        packet.Time,
+			}).Warn("Removed duplicate IDR NAL units from MoQ keyframe")
+
+			lastDuplicateKeyframeWarning = time.Now()
+		}
+		if packet.IsKeyFrame && deduplicator.IsDuplicate(packet.Time, packet.CurrentTime, payload, time.Now(), duplicateKeyframeWindow) {
+			if time.Since(lastDuplicateKeyframeWarning) >= moQWriteWarningInterval {
+				config.logEntry("duplicate_keyframe_dropped").WithField("timestamp_ms", packet.Time).
+					Warn("Dropped duplicate MoQ keyframe")
+
+				lastDuplicateKeyframeWarning = time.Now()
+			}
+			continue
+		}
+		frame := moq.Frame{
+			Payload:     payload,
+			TimestampUs: livemoq.TimestampUs(packet.Time),
+		}
+		writeStartedAt := time.Now()
+		writeWatchdog.Begin(writeStartedAt)
+		err = stream.WriteFrame(frame)
+		writeWatchdog.End()
+		if err != nil {
+			return fmt.Errorf("write H.264 access unit: %w", err)
+		}
+		writeDuration := time.Since(writeStartedAt)
+		if config.communication != nil {
+			config.communication.RecordMoQWrite(config.quality, writeDuration, time.Now())
+		}
+		if writeDuration >= slowMoQWriteThreshold && time.Since(lastSlowWriteWarning) >= moQWriteWarningInterval {
+			packetAge := time.Duration(0)
+			if packet.CurrentTime > 0 {
+				packetAge = time.Since(time.UnixMilli(packet.CurrentTime))
+				if packetAge < 0 {
+					packetAge = 0
+				}
+			}
+			config.logEntry("slow_frame_write").WithFields(log.Fields{
+				"duration_ms":   writeDuration.Milliseconds(),
+				"keyframe":      packet.IsKeyFrame,
+				"packet_age_ms": packetAge.Milliseconds(),
+			}).Warn("MoQ frame write was slow")
+
+			lastSlowWriteWarning = time.Now()
+		}
+	}
+}
+
+func closeLiveMoQPublisher(finishStream, finishBroadcast, closeClient func() error) error {
+	streamErr := finishStream()
+	broadcastErr := finishBroadcast()
+	clientErr := closeClient()
+	return errors.Join(streamErr, broadcastErr, clientErr)
+}
+
+func watchLiveStreamMoQWrites(ctx context.Context, closePublisher func() error, watchdog *livemoq.WriteWatchdog, config liveMoQConfig) {
+	ticker := time.NewTicker(moQWriteWatchInterval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			if err := closePublisher(); err != nil {
+				config.logEntry("publisher_close_failed").WithError(err).
+					Debug("MoQ publisher reported an error while closing")
+			}
+			return
+		case now := <-ticker.C:
+			writeDuration, active := watchdog.Elapsed(now)
+			if !active || writeDuration < config.writeTimeoutLimit() {
+				continue
+			}
+			config.logEntry("frame_write_timeout").WithFields(log.Fields{
+				"duration_ms": writeDuration.Milliseconds(),
+				"timeout_ms":  config.writeTimeoutLimit().Milliseconds(),
+			}).Warn("MoQ frame write timed out; reconnecting relay client")
+
+			if config.communication != nil {
+				config.communication.RecordMoQWriteTimeout()
+			}
+			if err := closePublisher(); err != nil {
+				config.logEntry("publisher_close_failed").WithError(err).
+					Warn("MoQ publisher reported an error while closing after a write timeout")
+			}
+			return
+		}
+	}
+}
+
+// watchLiveStreamMoQSubscribers flips the publisher between uploading and idling
+// as viewers subscribe to and leave this tier's broadcast. Used and Unused both
+// block, so they are followed from their own goroutine.
+//
+// It deliberately never turns publishing off before the first subscriber has
+// been observed: the relay catalog is only complete once media has flowed, so
+// going idle up front could keep the tier undiscoverable. On any error it fails
+// open (keeps publishing) — a stalled watcher must never take the live view down.
+func watchLiveStreamMoQSubscribers(ctx context.Context, stream *moq.MediaProducer, publishing *atomic.Bool, config liveMoQConfig) {
+	for ctx.Err() == nil {
+		if err := stream.Used(ctx); err != nil {
+			publishing.Store(true)
+			return
+		}
+		if publishing.CompareAndSwap(false, true) {
+			config.logEntry("subscriber_joined").Info("MoQ subscriber joined; resuming broadcast")
+		}
+
+		if err := stream.Unused(ctx); err != nil {
+			publishing.Store(true)
+			return
+		}
+		publishing.Store(false)
+		config.logEntry("subscribers_idle").Info("MoQ broadcast idle; refreshing keyframes only")
+	}
+}

machinery/src/cloud/livemoq_enabled_test.go

@@ -0,0 +1,212 @@
+//go:build moq
+
+package cloud
+
+import (
+	"context"
+	"errors"
+	"reflect"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/kerberos-io/agent/machinery/src/cloud/livemoq"
+	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/moq-dev/moq-go/moq"
+)
+
+func TestCloseLiveMoQPublisherFinishesBeforeClosingClient(t *testing.T) {
+	var order []string
+	streamErr := errors.New("stream finish failed")
+	broadcastErr := errors.New("broadcast finish failed")
+	clientErr := errors.New("client close failed")
+
+	err := closeLiveMoQPublisher(
+		func() error {
+			order = append(order, "stream")
+			return streamErr
+		},
+		func() error {
+			order = append(order, "broadcast")
+			return broadcastErr
+		},
+		func() error {
+			order = append(order, "client")
+			return clientErr
+		},
+	)
+
+	if want := []string{"stream", "broadcast", "client"}; !reflect.DeepEqual(order, want) {
+		t.Fatalf("shutdown order = %v, want %v", order, want)
+	}
+	for _, want := range []error{streamErr, broadcastErr, clientErr} {
+		if !errors.Is(err, want) {
+			t.Errorf("shutdown error %v does not include %v", err, want)
+		}
+	}
+}
+
+func TestWatchLiveStreamMoQWritesClosesClientWhenContextEnds(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	closed := make(chan struct{})
+	var closeCalls atomic.Int32
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		watchLiveStreamMoQWrites(ctx, func() error {
+			if closeCalls.Add(1) == 1 {
+				close(closed)
+			}
+			return nil
+		}, &livemoq.WriteWatchdog{}, liveMoQConfig{quality: "low", sourceLabel: "sub"})
+	}()
+
+	cancel()
+	select {
+	case <-closed:
+	case <-time.After(time.Second):
+		t.Fatal("watchLiveStreamMoQWrites() did not close the client after cancellation")
+	}
+	<-done
+	if got := closeCalls.Load(); got != 1 {
+		t.Fatalf("close calls = %d, want 1", got)
+	}
+}
+
+func TestBoundedMoQDuration(t *testing.T) {
+	const name = "AGENT_LIVE_MOQ_TEST_DURATION"
+	tests := []struct {
+		name string
+		raw  string
+		want time.Duration
+	}{
+		{name: "unset", want: 5 * time.Second},
+		{name: "valid", raw: "12s", want: 12 * time.Second},
+		{name: "invalid", raw: "later", want: 5 * time.Second},
+		{name: "below minimum", raw: "100ms", want: time.Second},
+		{name: "above maximum", raw: "2m", want: time.Minute},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			t.Setenv(name, test.raw)
+			if got := boundedMoQDuration(name, 5*time.Second, time.Second, time.Minute); got != test.want {
+				t.Fatalf("boundedMoQDuration() = %s, want %s", got, test.want)
+			}
+		})
+	}
+}
+
+func TestWatchLiveStreamMoQWritesRecordsTimeout(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	closed := make(chan struct{})
+	communication := &models.Communication{}
+	watchdog := &livemoq.WriteWatchdog{}
+	watchdog.Begin(time.Now().Add(-time.Second))
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		watchLiveStreamMoQWrites(ctx, func() error {
+			close(closed)
+			return nil
+		}, watchdog, liveMoQConfig{
+			quality:       models.StreamQualityLow,
+			sourceLabel:   "sub",
+			communication: communication,
+			writeTimeout:  10 * time.Millisecond,
+		})
+	}()
+
+	select {
+	case <-closed:
+	case <-time.After(time.Second):
+		t.Fatal("watchLiveStreamMoQWrites() did not close the client after a write timeout")
+	}
+	<-done
+	if got := communication.RecoveryTelemetry().MoQWriteTimeouts; got != 1 {
+		t.Fatalf("MoQ write timeouts = %d, want 1", got)
+	}
+}
+
+func TestNativeMoQClientCloseStopsActiveWriter(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	server, err := moq.Listen(ctx, "127.0.0.1:0", moq.WithTLSGenerate("localhost"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	serveDone := make(chan error, 1)
+	go func() {
+		serveDone <- server.Serve(ctx)
+	}()
+
+	client, err := moq.Dial(ctx, "https://"+server.LocalAddr(), moq.WithTLSVerify(false))
+	if err != nil {
+		server.Close()
+		t.Fatal(err)
+	}
+	broadcast, err := client.CreateBroadcast("test/native-close")
+	if err != nil {
+		client.Close()
+		server.Close()
+		t.Fatal(err)
+	}
+	stream, err := broadcast.PublishMedia("avc3", nil)
+	if err != nil {
+		client.Close()
+		server.Close()
+		t.Fatal(err)
+	}
+
+	firstWrite := make(chan struct{})
+	writerDone := make(chan error, 1)
+	go func() {
+		payload := []byte{
+			0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x1e, 0xd9, 0x00, 0xa0, 0x47, 0xfe, 0xc8,
+			0x00, 0x00, 0x00, 0x01, 0x68, 0xce, 0x38, 0x80,
+			0x00, 0x00, 0x00, 0x01, 0x65, 0x88,
+		}
+		for timestamp := uint64(0); ; timestamp++ {
+			err := stream.WriteFrame(moq.Frame{Payload: payload, TimestampUs: timestamp})
+			if err != nil {
+				writerDone <- err
+				return
+			}
+			if timestamp == 0 {
+				close(firstWrite)
+			}
+		}
+	}()
+
+	select {
+	case <-firstWrite:
+	case <-ctx.Done():
+		t.Fatal("native MoQ writer did not start")
+	}
+	if err := stream.Finish(); err != nil {
+		t.Fatal(err)
+	}
+	if err := client.Close(); err != nil {
+		t.Fatal(err)
+	}
+	select {
+	case err := <-writerDone:
+		if err == nil {
+			t.Fatal("native MoQ writer stopped without an error after client close")
+		}
+	case <-ctx.Done():
+		t.Fatal("native MoQ client close did not stop the writer")
+	}
+
+	_ = broadcast.Finish()
+	_ = server.Close()
+	select {
+	case <-serveDone:
+	case <-ctx.Done():
+		t.Fatal("native MoQ server did not stop")
+	}
+}

machinery/src/cloud/livesnapshot/publisher.go

@@ -30,7 +30,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
+	log "github.com/sirupsen/logrus"
 )
 
 const (
@@ -131,7 +131,10 @@ func (p *Publisher) PublishSnapshot(ctx context.Context, jpeg []byte) error {
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
 		return fmt.Errorf("livesnapshot: upload snapshot rejected: %s", resp.Status)
 	}
-	log.Log.Debug("livesnapshot.Publisher.PublishSnapshot(): shipped preview frame for device " + p.cfg.DeviceKey)
+	log.WithFields(log.Fields{
+		"component": "livesnapshot",
+		"event":     "snapshot_published",
+	}).Debug("Live snapshot published")
 	return nil
 }
 

machinery/src/cloud/realtime_processing_test.go

@@ -0,0 +1,58 @@
+package cloud
+
+import (
+	"testing"
+
+	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
+	"github.com/sirupsen/logrus/hooks/test"
+)
+
+func TestHandleRealtimeProcessingWithoutMQTTClient(t *testing.T) {
+	tests := []struct {
+		name        string
+		config      models.Config
+		wantWarning bool
+	}{
+		{
+			name:        "enabled without Hub credentials",
+			config:      models.Config{RealtimeProcessing: "true"},
+			wantWarning: true,
+		},
+		{
+			name:   "disabled",
+			config: models.Config{RealtimeProcessing: "false"},
+		},
+		{
+			name:   "offline",
+			config: models.Config{RealtimeProcessing: "true", Offline: "true"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			hook := test.NewGlobal()
+			defer hook.Reset()
+
+			configuration := &models.Configuration{Config: tt.config}
+			// No cursor or decoder should be accessed without an MQTT client.
+			HandleRealtimeProcessing(nil, configuration, nil, nil, nil)
+
+			warnings := 0
+			for _, entry := range hook.AllEntries() {
+				if entry.Level == log.WarnLevel {
+					warnings++
+					if entry.Message != "cloud.RealtimeProcessing(): stopping as MQTT client is not configured." {
+						t.Errorf("unexpected warning: %s", entry.Message)
+					}
+				}
+			}
+			wantWarnings := 0
+			if tt.wantWarning {
+				wantWarnings = 1
+			}
+			if warnings != wantWarnings {
+				t.Fatalf("warnings = %d, want %d", warnings, wantWarnings)
+			}
+		})
+	}
+}

machinery/src/cloud/recording_metadata.go

@@ -0,0 +1,120 @@
+package cloud
+
+import (
+	"encoding/json"
+	"fmt"
+	"math"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
+)
+
+const recordingFPSHeader = "X-Kerberos-Storage-Fps"
+const recordingDurationHeader = "X-Kerberos-Storage-Duration"
+const recordingTimestampHeader = "X-Kerberos-Storage-Timestamp"
+const recordingEncryptedHeader = "X-Kerberos-Storage-Encrypted"
+const recordingDeviceNameHeader = "X-Kerberos-Storage-DeviceName"
+const recordingRegionHeader = "X-Kerberos-Storage-RegionCoordinates"
+const recordingChangesHeader = "X-Kerberos-Storage-NumberOfChanges"
+
+// queuedRecordingFPS reads the FPS snapshot written into the upload marker
+// when the recording was finalized. Historical empty markers intentionally
+// return no value so receivers can retain their existing MP4-derived fallback.
+func queuedRecordingFPS(fileName string) string {
+	value, ok := readRecordingUploadMetadata(fileName)
+	if !ok {
+		return ""
+	}
+
+	marker := strings.TrimSpace(string(value))
+	if strings.HasPrefix(marker, "{") {
+		metadata, ok := decodeRecordingUploadMetadata(value)
+		if !ok || metadata.FPS <= 0 || metadata.FPS > 240 || math.IsInf(metadata.FPS, 0) || math.IsNaN(metadata.FPS) {
+			return ""
+		}
+		return strconv.FormatFloat(metadata.FPS, 'f', -1, 64)
+	}
+
+	// Compatibility with markers created before upload metadata used JSON.
+	fps := marker
+	parsed, err := strconv.ParseFloat(fps, 64)
+	if err != nil || parsed <= 0 || parsed > 240 || math.IsInf(parsed, 0) || math.IsNaN(parsed) {
+		return ""
+	}
+	return fps
+}
+
+func queuedRecordingMetadata(fileName string) (models.RecordingUploadMetadata, bool) {
+	value, ok := readRecordingUploadMetadata(fileName)
+	if !ok || !strings.HasPrefix(strings.TrimSpace(string(value)), "{") {
+		return models.RecordingUploadMetadata{}, false
+	}
+	return decodeRecordingUploadMetadata(value)
+}
+
+func decodeRecordingUploadMetadata(value []byte) (models.RecordingUploadMetadata, bool) {
+	var metadata models.RecordingUploadMetadata
+	if err := json.Unmarshal(value, &metadata); err != nil {
+		return models.RecordingUploadMetadata{}, false
+	}
+	return metadata, true
+}
+
+func readRecordingUploadMetadata(fileName string) ([]byte, bool) {
+	value, err := readRecordingUploadMetadataFile(fileName)
+	if err != nil && !os.IsNotExist(err) {
+		log.WithError(err).Error("Failed to read recording upload metadata")
+	}
+	return value, err == nil
+}
+
+func readRecordingUploadMetadataFile(fileName string) ([]byte, error) {
+	markerNames := []string{
+		models.RecordingUploadMetadataFileName(fileName),
+		filepath.Base(fileName),
+	}
+	for _, markerName := range markerNames {
+		value, err := os.ReadFile(filepath.Join("data", "cloud", markerName))
+		if err == nil {
+			return value, nil
+		}
+		if !os.IsNotExist(err) {
+			return nil, fmt.Errorf("read recording upload metadata %s: %w", markerName, err)
+		}
+	}
+	return nil, os.ErrNotExist
+}
+
+func setQueuedRecordingMetadataHeaders(header http.Header, fileName string) {
+	if fps := queuedRecordingFPS(fileName); fps != "" {
+		header.Set(recordingFPSHeader, fps)
+	}
+	if metadata, ok := queuedRecordingMetadata(fileName); ok {
+		if metadata.DeviceKey != "" {
+			header.Set("X-Kerberos-Storage-Device", metadata.DeviceKey)
+		}
+		if metadata.DeviceName != "" {
+			header.Set(recordingDeviceNameHeader, metadata.DeviceName)
+		}
+		if metadata.RegionCoordinates != "" {
+			header.Set(recordingRegionHeader, metadata.RegionCoordinates)
+		}
+		if metadata.NumberOfChanges != "" {
+			header.Set(recordingChangesHeader, metadata.NumberOfChanges)
+		}
+		if metadata.Duration > 0 {
+			header.Set(recordingDurationHeader, strconv.FormatUint(metadata.Duration, 10))
+		}
+		if metadata.Timestamp > 0 {
+			header.Set(recordingTimestampHeader, strconv.FormatInt(metadata.Timestamp, 10))
+		}
+		if metadata.Encrypted {
+			header.Set(recordingEncryptedHeader, "true")
+		}
+	}
+}

machinery/src/cloud/s3.go

@@ -2,34 +2,31 @@ package cloud
 
 import (
 	"crypto/tls"
+	"encoding/json"
 	"errors"
+	"fmt"
+	"math"
 	"net/http"
 	"net/url"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
+	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/utils"
 	"github.com/minio/minio-go/v6"
+	log "github.com/sirupsen/logrus"
 )
 
 func UploadS3(configuration *models.Configuration, fileName string) (bool, bool, error) {
 
 	config := configuration.Config
 
-	// timestamp_microseconds_instanceName_regionCoordinates_numberOfChanges_token
-	// 1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4
-	// - Timestamp
-	// - Size + - + microseconds
-	// - device
-	// - Region
-	// - Number of changes
-	// - Token
-
 	if config.S3 == nil {
 		errorMessage := "UploadS3: Uploading Failed, as no settings found"
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, false, errors.New(errorMessage)
 	}
 
@@ -49,14 +46,14 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
 	// Check if we have some credentials otherwise we abort the request.
 	if aws_access_key_id == "" || aws_secret_access_key == "" {
 		errorMessage := "UploadS3: Uploading Failed, as no credentials found"
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, false, errors.New(errorMessage)
 	}
 
 	s3Client, err := minio.NewWithRegion("s3.amazonaws.com", aws_access_key_id, aws_secret_access_key, true, aws_region)
 	if err != nil {
 		errorMessage := "UploadS3: " + err.Error()
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, true, errors.New(errorMessage)
 	}
 
@@ -71,67 +68,189 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
 		s3Client.SetCustomTransport(transport)
 	}
 
-	fileParts := strings.Split(fileName, "_")
-	if len(fileParts) == 1 {
-		errorMessage := "UploadS3: " + fileName + " is not a valid name."
-		log.Log.Error(errorMessage)
-		return false, true, errors.New(errorMessage)
+	objectName, userMetadata, err := s3RecordingUploadDetails(config, fileName, aws_access_key_id)
+	if err != nil {
+		log.WithError(err).Error("UploadS3: invalid recording upload metadata")
+		return false, true, err
 	}
 
-	deviceKey := config.Key
-	startRecording, _ := strconv.ParseInt(fileParts[0], 10, 64)
-	devicename := fileParts[2]
-	coordinates := fileParts[3]
-	//numberOfChanges := fileParts[4]
-	token, _ := strconv.Atoi(fileParts[5])
-
-	log.Log.Info("UploadS3: Upload started for " + fileName)
-	fullname := "data/recordings/" + fileName
-
-	file, err := os.OpenFile(fullname, os.O_RDWR, 0755)
+	log.Info("UploadS3: Upload started for " + fileName)
+	file, err := utils.OpenRecording("data/recordings", fileName)
 	if file != nil {
 		defer file.Close()
 	}
 
 	if err != nil {
 		errorMessage := "UploadS3: " + err.Error()
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, true, errors.New(errorMessage)
 	}
 
 	fileInfo, err := file.Stat()
 	if err != nil {
 		errorMessage := "UploadS3: " + err.Error()
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, true, errors.New(errorMessage)
 	}
 
 	n, err := s3Client.PutObject(config.S3.Bucket,
-		config.S3.Username+"/"+fileName,
+		objectName,
 		file,
 		fileInfo.Size(),
 		minio.PutObjectOptions{
 			ContentType:  "video/mp4",
 			StorageClass: "ONEZONE_IA",
-			UserMetadata: map[string]string{
-				"event-timestamp":         strconv.FormatInt(startRecording, 10),
-				"event-microseconds":      deviceKey,
-				"event-instancename":      devicename,
-				"event-regioncoordinates": coordinates,
-				"event-numberofchanges":   deviceKey,
-				"event-token":             strconv.Itoa(token),
-				"productid":               deviceKey,
-				"publickey":               aws_access_key_id,
-				"uploadtime":              "now",
-			},
+			UserMetadata: userMetadata,
 		})
 
 	if err != nil {
 		errorMessage := "UploadS3: Uploading Failed, " + err.Error()
-		log.Log.Error(errorMessage)
+		log.Error(errorMessage)
 		return false, true, errors.New(errorMessage)
 	} else {
-		log.Log.Info("UploadS3: Upload Finished, file has been uploaded to bucket: " + strconv.FormatInt(n, 10))
+		log.Info("UploadS3: Upload Finished, file has been uploaded to bucket: " + strconv.FormatInt(n, 10))
 		return true, true, nil
 	}
 }
+
+func s3RecordingObjectName(username, deviceKey string, startRecording int64) string {
+	return username + "/" + time.UnixMilli(startRecording).UTC().Format("2006/01/02") +
+		"/" + strconv.FormatInt(startRecording, 10) + "-" + deviceKey + ".mp4"
+}
+
+func s3RecordingUploadDetails(config models.Config, fileName, publicKey string) (string, map[string]string, error) {
+	value, err := readRecordingUploadMetadataFile(fileName)
+	if err != nil && !os.IsNotExist(err) {
+		return "", nil, err
+	}
+
+	deviceKey := config.Key
+	var startRecording int64
+	userMetadata := map[string]string{
+		"publickey":  publicKey,
+		"uploadtime": "now",
+	}
+	marker := strings.TrimSpace(string(value))
+	_, fpsErr := strconv.ParseFloat(marker, 64)
+	if err == nil && marker != "" && fpsErr != nil {
+		var metadata models.RecordingUploadMetadata
+		if err := json.Unmarshal(value, &metadata); err != nil {
+			return "", nil, fmt.Errorf("decode recording upload metadata for %s: %w", fileName, err)
+		}
+		if metadata.FileName != "" && metadata.FileName != filepath.Base(fileName) {
+			return "", nil, fmt.Errorf("recording upload metadata filename does not match %s", fileName)
+		}
+		if metadata.Timestamp <= 0 {
+			return "", nil, fmt.Errorf("invalid recording timestamp in upload metadata for %s", fileName)
+		}
+		deviceKey = metadata.DeviceKey
+		startRecording = metadata.Timestamp
+		userMetadata["event-instancename"] = config.Name
+		if metadata.DeviceName == "" || metadata.RegionCoordinates == "" || metadata.NumberOfChanges == "" {
+			if description, ok := s3LegacyRecordingDescription(fileName); ok {
+				for key, value := range description {
+					userMetadata[key] = value
+				}
+			}
+		}
+		if metadata.DeviceName != "" {
+			userMetadata["event-instancename"] = metadata.DeviceName
+		}
+		if metadata.RegionCoordinates != "" {
+			userMetadata["event-regioncoordinates"] = metadata.RegionCoordinates
+		}
+		if metadata.NumberOfChanges != "" {
+			userMetadata["event-numberofchanges"] = metadata.NumberOfChanges
+		}
+		userMetadata["event-microseconds"] = strconv.FormatInt(metadata.Timestamp%1000*1000, 10)
+		userMetadata["duration"] = strconv.FormatUint(metadata.Duration, 10)
+		userMetadata["encrypted"] = strconv.FormatBool(metadata.Encrypted)
+		if metadata.FPS > 0 && metadata.FPS <= 240 {
+			userMetadata["fps"] = strconv.FormatFloat(metadata.FPS, 'f', -1, 64)
+		}
+	} else {
+		// Empty and FPS-only markers predate structured recording metadata.
+		fileParts := strings.Split(fileName, "_")
+		if len(fileParts) < 6 {
+			return "", nil, fmt.Errorf("%s is not a valid legacy recording name", fileName)
+		}
+		startRecording, err = strconv.ParseInt(fileParts[0], 10, 64)
+		if err != nil {
+			return "", nil, fmt.Errorf("invalid recording timestamp in %s: %w", fileName, err)
+		}
+		if startRecording <= 0 || startRecording > math.MaxInt64/1000 {
+			return "", nil, fmt.Errorf("invalid recording timestamp in %s", fileName)
+		}
+		startRecording *= 1000
+		_, precision, found := strings.Cut(fileParts[1], "-")
+		if found {
+			fraction, err := strconv.ParseInt(precision, 10, 64)
+			if err != nil || fraction < 0 || fraction >= 1000000 {
+				return "", nil, fmt.Errorf("invalid recording precision in %s", fileName)
+			}
+			if len(precision) > 3 {
+				fraction /= 1000
+			}
+			startRecording += fraction
+		}
+		token, _ := strconv.Atoi(fileParts[5])
+		userMetadata["event-microseconds"] = deviceKey
+		userMetadata["event-instancename"] = fileParts[2]
+		userMetadata["event-regioncoordinates"] = fileParts[3]
+		userMetadata["event-numberofchanges"] = deviceKey
+		userMetadata["event-token"] = strconv.Itoa(token)
+	}
+	if deviceKey == "" || strings.ContainsAny(deviceKey, "/\\") {
+		return "", nil, errors.New("missing or invalid recording device key")
+	}
+	if startRecording <= 0 {
+		return "", nil, fmt.Errorf("invalid recording timestamp for %s", fileName)
+	}
+	userMetadata["event-timestamp"] = strconv.FormatInt(startRecording/1000, 10)
+	userMetadata["productid"] = deviceKey
+	return s3RecordingObjectName(config.S3.Username, deviceKey, startRecording), userMetadata, nil
+}
+
+func s3LegacyRecordingDescription(fileName string) (map[string]string, bool) {
+	base := filepath.Base(fileName)
+	if filepath.Ext(base) != ".mp4" {
+		return nil, false
+	}
+	parts := strings.Split(strings.TrimSuffix(base, ".mp4"), "_")
+	if len(parts) != 6 || parts[2] == "" {
+		return nil, false
+	}
+	seconds, err := strconv.ParseInt(parts[0], 10, 64)
+	if err != nil || seconds <= 0 || seconds > math.MaxInt64/1000 {
+		return nil, false
+	}
+	digits, fraction, found := strings.Cut(parts[1], "-")
+	precision, err := strconv.Atoi(digits)
+	if !found || err != nil || precision < 1 || precision > 6 || len(fraction) != precision {
+		return nil, false
+	}
+	if _, err := strconv.ParseUint(fraction, 10, 64); err != nil {
+		return nil, false
+	}
+	coordinates := strings.Split(parts[3], "-")
+	if len(coordinates) != 4 {
+		return nil, false
+	}
+	for _, coordinate := range coordinates {
+		if _, err := strconv.ParseUint(coordinate, 10, 64); err != nil {
+			return nil, false
+		}
+	}
+	changes, err := strconv.ParseInt(parts[4], 10, 64)
+	if err != nil || changes < -1 {
+		return nil, false
+	}
+	if _, err := strconv.ParseUint(parts[5], 10, 64); err != nil {
+		return nil, false
+	}
+	return map[string]string{
+		"event-instancename":      parts[2],
+		"event-regioncoordinates": parts[3],
+		"event-numberofchanges":   parts[4],
+	}, true
+}

machinery/src/cloud/s3_test.go

@@ -0,0 +1,328 @@
+package cloud
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/kerberos-io/agent/machinery/src/models"
+)
+
+func TestS3RecordingUploadDetailsUsesCheckpoint(t *testing.T) {
+	for _, fileName := range []string{
+		"1767355200123-checkpoint-key.mp4",
+		"1564859471_6-474162_old-camera_0-0-0-0_1153_27.mp4",
+	} {
+		t.Run(fileName, func(t *testing.T) {
+			withRecording(t, fileName, []byte("recording"))
+			recorded := time.Date(2026, time.January, 2, 12, 0, 0, 123000000, time.UTC)
+			metadata := models.RecordingUploadMetadata{
+				FileName:          fileName,
+				DeviceKey:         "checkpoint-key",
+				Timestamp:         recorded.UnixMilli(),
+				Duration:          20452,
+				FPS:               29.97,
+				Encrypted:         true,
+				DeviceName:        "checkpoint-camera",
+				RegionCoordinates: "1-2-3-4",
+				NumberOfChanges:   "57",
+			}
+			payload, err := json.Marshal(metadata)
+			if err != nil {
+				t.Fatal(err)
+			}
+			withQueuedRecordingFPS(t, fileName, string(payload))
+			config := models.Config{
+				Key:  "changed-config-key",
+				Name: "camera-name",
+				S3:   &models.S3{Username: "user"},
+			}
+			objectName, userMetadata, err := s3RecordingUploadDetails(config, fileName, "public-key")
+			if err != nil {
+				t.Fatal(err)
+			}
+			wantObjectName := "user/2026/01/02/" + strconv.FormatInt(recorded.UnixMilli(), 10) + "-checkpoint-key.mp4"
+			if objectName != wantObjectName {
+				t.Fatalf("object name = %q, want %q", objectName, wantObjectName)
+			}
+			wantMetadata := map[string]string{
+				"publickey":               "public-key",
+				"uploadtime":              "now",
+				"event-instancename":      "checkpoint-camera",
+				"event-regioncoordinates": "1-2-3-4",
+				"event-numberofchanges":   "57",
+				"event-microseconds":      "123000",
+				"event-timestamp":         strconv.FormatInt(recorded.Unix(), 10),
+				"productid":               "checkpoint-key",
+				"duration":                "20452",
+				"fps":                     "29.97",
+				"encrypted":               "true",
+			}
+			if !reflect.DeepEqual(userMetadata, wantMetadata) {
+				t.Fatalf("metadata = %#v, want %#v", userMetadata, wantMetadata)
+			}
+		})
+	}
+}
+
+func TestS3RecordingUploadDetailsBackfillsLegacyCheckpointDescriptions(t *testing.T) {
+	const legacyName = "1564859471_6-474162_recorded-camera_1-2-3-4_1153_27.mp4"
+	tests := []struct {
+		name        string
+		fileName    string
+		camera      string
+		region      string
+		changes     string
+		wantCamera  string
+		wantRegion  string
+		wantChanges string
+	}{
+		{"pre-upgrade checkpoint", legacyName, "", "", "", "recorded-camera", "1-2-3-4", "1153"},
+		{"checkpoint camera", legacyName, "snapshot-camera", "", "", "snapshot-camera", "1-2-3-4", "1153"},
+		{"checkpoint region", legacyName, "", "5-6-7-8", "", "recorded-camera", "5-6-7-8", "1153"},
+		{"checkpoint zero changes", legacyName, "", "", "0", "recorded-camera", "1-2-3-4", "0"},
+		{"dated reference", "2019/08/03/" + legacyName, "", "", "", "recorded-camera", "1-2-3-4", "1153"},
+		{"continuous recording", "1564859471_3-474_recorded-camera_0-0-0-0_-1_27.mp4", "", "", "", "recorded-camera", "0-0-0-0", "-1"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			withRecording(t, filepath.Base(tt.fileName), []byte("recording"))
+			metadata := models.RecordingUploadMetadata{
+				FileName:          filepath.Base(tt.fileName),
+				DeviceKey:         "checkpoint-key",
+				Timestamp:         1767355200123,
+				Duration:          20452,
+				FPS:               29.97,
+				Encrypted:         true,
+				DeviceName:        tt.camera,
+				RegionCoordinates: tt.region,
+				NumberOfChanges:   tt.changes,
+			}
+			payload, err := json.Marshal(metadata)
+			if err != nil {
+				t.Fatal(err)
+			}
+			withQueuedRecordingFPS(t, tt.fileName, string(payload))
+			config := models.Config{Key: "current-key", Name: "current-camera", S3: &models.S3{Username: "user"}}
+			objectName, got, err := s3RecordingUploadDetails(config, tt.fileName, "public-key")
+			if err != nil {
+				t.Fatal(err)
+			}
+			if want := "user/2026/01/02/1767355200123-checkpoint-key.mp4"; objectName != want {
+				t.Fatalf("object name = %q, want %q", objectName, want)
+			}
+			want := map[string]string{
+				"publickey":               "public-key",
+				"uploadtime":              "now",
+				"event-instancename":      tt.wantCamera,
+				"event-regioncoordinates": tt.wantRegion,
+				"event-numberofchanges":   tt.wantChanges,
+				"event-microseconds":      "123000",
+				"event-timestamp":         "1767355200",
+				"productid":               "checkpoint-key",
+				"duration":                "20452",
+				"fps":                     "29.97",
+				"encrypted":               "true",
+			}
+			if !reflect.DeepEqual(got, want) {
+				t.Fatalf("metadata = %#v, want %#v", got, want)
+			}
+		})
+	}
+}
+
+func TestS3RecordingUploadDetailsDoesNotBackfillInvalidLegacyNames(t *testing.T) {
+	for _, fileName := range []string{
+		"1767355200123-checkpoint-key.mp4",
+		"invalid_6-474162_camera_0-0-0-0_1153_27.mp4",
+		"1564859471_6-invalid_camera_0-0-0-0_1153_27.mp4",
+		"1564859471_3-474162_camera_0-0-0-0_1153_27.mp4",
+		"1564859471_6-474162__0-0-0-0_1153_27.mp4",
+		"1564859471_6-474162_camera_invalid_1153_27.mp4",
+		"1564859471_6-474162_camera_0-0-0-0_invalid_27.mp4",
+		"1564859471_6-474162_camera_0-0-0-0_1153_invalid.mp4",
+		"1564859471_6-474162_camera_0-0-0-0_1153_27_extra.mp4",
+	} {
+		t.Run(fileName, func(t *testing.T) {
+			withRecording(t, fileName, []byte("recording"))
+			metadata := models.RecordingUploadMetadata{
+				FileName: fileName, DeviceKey: "checkpoint-key", Timestamp: 1767355200123, Duration: 20452,
+			}
+			payload, err := json.Marshal(metadata)
+			if err != nil {
+				t.Fatal(err)
+			}
+			withQueuedRecordingFPS(t, fileName, string(payload))
+			config := models.Config{Key: "current-key", Name: "current-camera", S3: &models.S3{Username: "user"}}
+			objectName, got, err := s3RecordingUploadDetails(config, fileName, "public-key")
+			if err != nil {
+				t.Fatal(err)
+			}
+			if objectName != "user/2026/01/02/1767355200123-checkpoint-key.mp4" ||
+				got["event-instancename"] != "current-camera" || got["event-regioncoordinates"] != "" ||
+				got["event-numberofchanges"] != "" || got["event-timestamp"] != "1767355200" ||
+				got["productid"] != "checkpoint-key" {
+				t.Fatalf("unexpected fallback for invalid legacy name: %q, %#v", objectName, got)
+			}
+		})
+	}
+}
+
+func TestS3RecordingUploadDetailsLegacyMarkers(t *testing.T) {
+	for _, marker := range []string{"missing", "empty", "fps-only", "old-marker-name"} {
+		t.Run(marker, func(t *testing.T) {
+			const fileName = "1564859471_6-474162_camera_0-0-0-0_1153_27.mp4"
+			withRecording(t, fileName, []byte("recording"))
+			switch marker {
+			case "empty":
+				withQueuedRecordingFPS(t, fileName, "")
+			case "fps-only", "old-marker-name":
+				withQueuedRecordingFPS(t, fileName, "29.97")
+				if marker == "old-marker-name" {
+					err := os.Rename(filepath.Join("data/cloud", models.RecordingUploadMetadataFileName(fileName)),
+						filepath.Join("data/cloud", fileName))
+					if err != nil {
+						t.Fatal(err)
+					}
+				}
+			}
+			config := models.Config{Key: "legacy-key", S3: &models.S3{Username: "user"}}
+			objectName, metadata, err := s3RecordingUploadDetails(config, fileName, "public-key")
+			if err != nil {
+				t.Fatal(err)
+			}
+			if want := "user/2019/08/03/1564859471474-legacy-key.mp4"; objectName != want {
+				t.Fatalf("object name = %q, want %q", objectName, want)
+			}
+			if metadata["productid"] != "legacy-key" || metadata["event-instancename"] != "camera" ||
+				metadata["event-regioncoordinates"] != "0-0-0-0" || metadata["event-timestamp"] != "1564859471" {
+				t.Fatalf("unexpected legacy metadata: %#v", metadata)
+			}
+		})
+	}
+}
+
+func TestS3RecordingUploadDetailsRejectsInvalidCheckpoint(t *testing.T) {
+	tests := []struct {
+		name   string
+		marker string
+		want   string
+	}{
+		{"malformed JSON", `{"timestamp":`, "decode"},
+		{"missing timestamp", `{"device_key":"camera"}`, "timestamp"},
+		{"missing device key", `{"timestamp":1787015373000}`, "device key"},
+		{"invalid device key", `{"timestamp":1787015373000,"device_key":"camera/key"}`, "device key"},
+		{"mismatched filename", `{"timestamp":1787015373000,"device_key":"camera","filename":"other.mp4"}`, "filename"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			const fileName = "1564859471_6-474162_camera_0-0-0-0_1153_27.mp4"
+			withRecording(t, fileName, []byte("recording"))
+			withQueuedRecordingFPS(t, fileName, tt.marker)
+			config := models.Config{Key: "legacy-key", S3: &models.S3{Username: "user"}}
+			_, _, err := s3RecordingUploadDetails(config, fileName, "public-key")
+			if err == nil || !strings.Contains(err.Error(), tt.want) {
+				t.Fatalf("error = %v, want error containing %q instead of legacy fallback", err, tt.want)
+			}
+		})
+	}
+}
+
+func TestS3RecordingUploadDetailsReportsCheckpointReadError(t *testing.T) {
+	const fileName = "1564859471_6-474162_camera_0-0-0-0_1153_27.mp4"
+	withRecording(t, fileName, []byte("recording"))
+	if err := os.MkdirAll(filepath.Join("data/cloud", models.RecordingUploadMetadataFileName(fileName)), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	config := models.Config{Key: "legacy-key", S3: &models.S3{Username: "user"}}
+	_, _, err := s3RecordingUploadDetails(config, fileName, "public-key")
+	if err == nil || !strings.Contains(err.Error(), "read recording upload metadata") {
+		t.Fatalf("error = %v, want checkpoint read error instead of legacy fallback", err)
+	}
+}
+
+func TestS3RecordingObjectName(t *testing.T) {
+	tests := []struct {
+		name      string
+		recorded  time.Time
+		directory string
+	}{
+		{
+			name:      "zero padded month and day",
+			recorded:  time.Date(2026, time.January, 2, 12, 0, 0, 0, time.UTC),
+			directory: "2026/01/02",
+		},
+		{
+			name:      "UTC date across year boundary",
+			recorded:  time.Date(2026, time.January, 1, 1, 0, 0, 0, time.FixedZone("UTC+2", 2*60*60)),
+			directory: "2025/12/31",
+		},
+		{
+			name:      "leap day",
+			recorded:  time.Date(2024, time.February, 29, 23, 59, 59, 0, time.UTC),
+			directory: "2024/02/29",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			const deviceKey = "camera-key-123"
+			want := "user/" + tt.directory + "/" + strconv.FormatInt(tt.recorded.UnixMilli(), 10) + "-" + deviceKey + ".mp4"
+			for attempt := 0; attempt < 2; attempt++ {
+				if got := s3RecordingObjectName("user", deviceKey, tt.recorded.UnixMilli()); got != want {
+					t.Fatalf("s3RecordingObjectName() = %q, want %q", got, want)
+				}
+			}
+		})
+	}
+}
+
+func TestUploadS3RejectsInvalidRecordingNames(t *testing.T) {
+	configuration := &models.Configuration{
+		Config: models.Config{
+			Key: "camera-key-123",
+			S3: &models.S3{
+				Publickey: "test-public-key",
+				Secretkey: "test-secret-key",
+				Region:    "eu-west-1",
+			},
+		},
+	}
+	for _, fileName := range []string{
+		"recording.mp4",
+		"1787015373_camera.mp4",
+		"invalid_6-474162_camera_0-0-0-0_1153_27.mp4",
+	} {
+		t.Run(fileName, func(t *testing.T) {
+			uploaded, configured, err := UploadS3(configuration, fileName)
+			if uploaded || !configured || err == nil {
+				t.Fatalf("UploadS3() = (%v, %v, %v), want (false, true, error)", uploaded, configured, err)
+			}
+		})
+	}
+}
+
+func TestUploadS3RejectsInvalidDeviceKeys(t *testing.T) {
+	for _, deviceKey := range []string{"", "camera/key", `camera\key`} {
+		t.Run(deviceKey, func(t *testing.T) {
+			configuration := &models.Configuration{
+				Config: models.Config{
+					Key: deviceKey,
+					S3: &models.S3{
+						Publickey: "test-public-key",
+						Secretkey: "test-secret-key",
+						Region:    "eu-west-1",
+					},
+				},
+			}
+			uploaded, configured, err := UploadS3(configuration, "1787015373_6-474162_camera_0-0-0-0_1153_27.mp4")
+			if uploaded || !configured || err == nil || !strings.Contains(err.Error(), "device key") {
+				t.Fatalf("UploadS3() = (%v, %v, %v), want (false, true, device key error)", uploaded, configured, err)
+			}
+		})
+	}
+}

machinery/src/cloud/tus_client.go

@@ -15,8 +15,9 @@ import (
 	"strings"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/utils"
+	log "github.com/sirupsen/logrus"
 )
 
 // tusResumableVersion is the tus protocol version implemented by this client.
@@ -90,7 +91,14 @@ func logTusUploadProgress(label string, offset, size int64, loggedBucket *int64)
 	if percent > 100 {
 		percent = 100
 	}
-	log.Log.Infof("%s: resumable upload progress %d%% (%d/%d bytes)", label, percent, offset, size)
+	log.WithFields(log.Fields{
+		"bytes_sent":  offset,
+		"bytes_total": size,
+		"component":   "tus",
+		"event":       "upload_progress",
+		"progress":    percent,
+		"target":      label,
+	}).Info("Resumable upload progress")
 }
 
 // tusHeaderFunc sets the authentication and routing headers required on every
@@ -99,7 +107,7 @@ func logTusUploadProgress(label string, offset, size int64, loggedBucket *int64)
 // creation request; it is empty on HEAD/PATCH/DELETE.
 type tusHeaderFunc func(h http.Header, fileName string)
 
-// runTusUpload performs a resumable (tus) upload of data/recordings/<fileName>
+// runTusUpload performs a resumable (tus) upload of a flat or dated recording.
 // to baseURL, sending target-specific authentication/routing headers via
 // setHeaders on every request. It encapsulates the create/resume/chunk/finalize
 // state machine shared by the Kerberos Vault (direct) and Kerberos Hub (proxied)
@@ -113,15 +121,13 @@ type tusHeaderFunc func(h http.Header, fileName string)
 //     should fall back to the legacy single-POST upload (older deployments).
 //   - body:      a short message for logging.
 func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tusHeaderFunc) (uploaded bool, responded bool, supported bool, body string, err error) {
-	fullname := "data/recordings/" + fileName
-
-	file, ferr := os.Open(fullname)
+	file, ferr := utils.OpenRecording("data/recordings", fileName)
 	if file != nil {
 		defer file.Close()
 	}
 	if ferr != nil {
 		msg := label + ": resumable upload failed, file doesn't exist anymore"
-		log.Log.Info(msg)
+		log.Info(msg)
 		// The file is gone, so the legacy path cannot help either. Report it as
 		// "supported" to avoid a pointless fallback attempt.
 		return false, false, true, "", errors.New(msg)
@@ -134,19 +140,6 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
 	size := info.Size()
 
 	client := newVaultHTTPClient(0)
-	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
-		if len(via) == 0 {
-			return nil
-		}
-		if req.URL.Host != via[0].URL.Host {
-			for k := range req.Header {
-				if strings.HasPrefix(http.CanonicalHeaderKey(k), "X-Kerberos-") {
-					req.Header.Del(k)
-				}
-			}
-		}
-		return nil
-	}
 
 	sidecar := tusSidecarPath(fileName, slot)
 	uploadURL := loadTusResumeState(sidecar, baseURL)
@@ -154,16 +147,37 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
 	const maxAttempts = 4
 	restartedAfterComplete := false
 
+	// lastStatus holds the HTTP status code of the most recent tus request. A
+	// value of 0 means the request failed at the transport level (no HTTP
+	// response at all, e.g. the vault was unreachable or the connection dropped
+	// because the internet went down). It lets the final "gave up" return report
+	// whether the vault actually answered, so the caller only advances its
+	// retry/back-off policy on a definitive response and transient network errors
+	// never consume the retry budget (matching the legacy single-POST behaviour).
+	lastStatus := 0
+
+	// highWaterOffset is the furthest server-acknowledged offset observed across
+	// all attempts (via HEAD or PATCH). It lets the retry budget be refreshed only
+	// on GENUINE net forward progress. Without it, a server that keeps resetting the
+	// offset — e.g. a persistent 409 ERR_MISMATCHED_OFFSET where HEAD reports 0 again
+	// while the first chunk still "succeeds" — would refresh the budget every attempt
+	// and loop forever, wedging the upload worker on one recording and saturating the
+	// uplink.
+	highWaterOffset := int64(0)
+	var lastErr error
+
 	for attempt := 0; attempt < maxAttempts; attempt++ {
 		// (1) Ensure we have an active upload URL, creating one if needed.
 		if uploadURL == "" {
 			created, status, cerr := tusCreate(client, baseURL, size, metadata, setHeaders, fileName)
+			lastStatus = status
 			if cerr != nil {
+				lastErr = cerr
 				if status == http.StatusNotFound || status == http.StatusMethodNotAllowed || status == http.StatusNotImplemented {
 					// The vault does not implement tus; let the caller fall back.
 					return false, false, false, "", cerr
 				}
-				log.Log.Info(label + ": resumable create failed, " + cerr.Error())
+				log.Info(label + ": resumable create failed, " + cerr.Error())
 				tusBackoff(attempt)
 				continue
 			}
@@ -173,18 +187,29 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
 
 		// (2) Query the current server-side offset.
 		offset, status, herr := tusHead(client, uploadURL, setHeaders)
+		lastStatus = status
 		if herr != nil {
+			lastErr = herr
 			if status == http.StatusNotFound || status == http.StatusGone {
 				// The upload expired/was removed server-side; start over.
 				removeTusResumeState(sidecar)
 				uploadURL = ""
 				continue
 			}
-			log.Log.Info(label + ": resumable head failed, " + herr.Error())
+			log.Info(label + ": resumable head failed, " + herr.Error())
 			tusBackoff(attempt)
 			continue
 		}
 
+		// The furthest offset any previous attempt reached. If this attempt pushes
+		// past it (via HEAD showing server-side progress or a successful PATCH) we made
+		// genuine net progress and may refresh the retry budget; if not, a repeated
+		// failure at the same spot must count against maxAttempts.
+		startHighWater := highWaterOffset
+		if offset > highWaterOffset {
+			highWaterOffset = offset
+		}
+
 		// (3) All bytes are present but the upload was not finalized (e.g. the
 		// completion hook failed). A completed tus upload cannot be re-finalized
 		// with another PATCH, so delete it and re-upload to force a clean finalize.
@@ -205,7 +230,6 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
 		// checkpointing the offset after each one so an interruption resumes from the
 		// last completed chunk instead of re-uploading everything.
 		chunkSize := tusChunkSize()
-		progressed := false
 		patchFailed := false
 		var lastBody string
 		loggedProgressBucket := tusProgressBucket(offset, size)
@@ -220,22 +244,24 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
 				patchLen = chunkSize
 			}
 			newOffset, status, respBody, perr := tusPatch(client, uploadURL, offset, patchLen, file, setHeaders)
+			lastStatus = status
 			if perr != nil {
+				lastErr = perr
 				if status >= 400 {
 					// Definitive rejection (e.g. provider push failed during finalize).
 					// Re-evaluate via HEAD on the next iteration to decide retry/restart.
-					log.Log.Info(label + ": resumable patch rejected, " + perr.Error())
+					log.Info(label + ": resumable patch rejected, " + perr.Error())
 				} else {
-					log.Log.Info(label + ": resumable patch failed, " + perr.Error())
+					log.Info(label + ": resumable patch failed, " + perr.Error())
 				}
 				tusBackoff(attempt)
 				patchFailed = true
 				break
 			}
-			if newOffset > offset {
-				progressed = true
-			}
 			offset = newOffset
+			if offset > highWaterOffset {
+				highWaterOffset = offset
+			}
 			lastBody = respBody
 			logTusUploadProgress(label, offset, size, &loggedProgressBucket)
 			if offset < size {
@@ -244,10 +270,13 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
 			}
 		}
 		if patchFailed {
-			if progressed {
-				// Forward progress refreshes the retry budget: maxAttempts bounds the
-				// number of consecutive failures, not the number of chunks needed for
-				// a large recording.
+			if highWaterOffset > startHighWater {
+				// Genuine net progress (we advanced past the furthest point any previous
+				// attempt reached) refreshes the retry budget: maxAttempts bounds the
+				// number of consecutive *non-progressing* failures, not the number of
+				// chunks needed for a large recording. A server that keeps rejecting the
+				// same offset (no net progress, e.g. a persistent ERR_MISMATCHED_OFFSET)
+				// therefore gives up after maxAttempts instead of retrying forever.
 				attempt = -1
 			}
 			continue
@@ -258,7 +287,18 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
 		return true, true, true, lastBody, nil
 	}
 
-	return false, true, true, "resumable upload did not complete after retries", errors.New(label + ": resumable upload did not complete after retries")
+	// Every attempt failed. Only report responded=true when the vault actually
+	// answered on the last attempt (lastStatus > 0). If every attempt failed at
+	// the transport level (lastStatus == 0, e.g. the internet was disconnected),
+	// report responded=false so the caller keeps the recording queued and retries
+	// later instead of consuming its retry budget and entering the long back-off
+	// timeout.
+	message := label + ": resumable upload did not complete after retries"
+	err = errors.New(message)
+	if lastErr != nil {
+		err = fmt.Errorf("%s: %w", message, lastErr)
+	}
+	return false, lastStatus > 0, true, "resumable upload did not complete after retries", err
 }
 
 // uploadVaultResumable uploads a recording directly to a Kerberos Vault using
@@ -266,17 +306,36 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
 // X-Kerberos-Storage-* headers on every request and routing (directory/provider)
 // is additionally carried in the tus Upload-Metadata.
 func uploadVaultResumable(vault models.KStorage, publicKey, deviceKey, fileName, label, slot string) (bool, bool, bool, string, error) {
+	customHeaders, err := parseCustomVaultHeaders(vault.CustomHeaders)
+	if err != nil {
+		return false, false, true, "", err
+	}
 	baseURL := strings.TrimRight(vault.URI, "/") + tusUploadPath
-	metadata := encodeTusMetadata(map[string]string{
+	metadataValues := map[string]string{
 		"filename":  fileName,
 		"device":    deviceKey,
 		"directory": vault.Directory,
 		"provider":  vault.Provider,
 		"capture":   "IPCamera",
 		"cloudkey":  publicKey,
-	})
+		"fps":       queuedRecordingFPS(fileName),
+	}
+	if encodedHeaders, err := encodeCustomVaultHeaders(customHeaders); err != nil {
+		return false, false, true, "", err
+	} else if encodedHeaders != "" {
+		metadataValues["custom_headers"] = encodedHeaders
+	}
+	customRequestHeaders := http.Header{}
+	if err := setCustomVaultHeaders(customRequestHeaders, customHeaders); err != nil {
+		return false, false, true, "", err
+	}
+	addRecordingTusMetadata(metadataValues, fileName)
+	metadata := encodeTusMetadata(metadataValues)
 	setHeaders := func(h http.Header, fn string) {
 		setVaultTusHeaders(h, vault, publicKey, deviceKey, fn)
+		for name, values := range customRequestHeaders {
+			h[name] = append([]string(nil), values...)
+		}
 	}
 	return runTusUpload(baseURL, metadata, fileName, label, slot, setHeaders)
 }
@@ -288,17 +347,42 @@ func uploadVaultResumable(vault models.KStorage, publicKey, deviceKey, fileName,
 // intentionally omitted from the metadata here.
 func uploadHubResumable(config *models.Config, fileName, label, slot string) (bool, bool, bool, string, error) {
 	baseURL := strings.TrimRight(config.HubURI, "/") + tusUploadPath
-	metadata := encodeTusMetadata(map[string]string{
+	metadataValues := map[string]string{
 		"filename": fileName,
 		"device":   config.Key,
 		"capture":  "IPCamera",
-	})
+		"fps":      queuedRecordingFPS(fileName),
+	}
+	addRecordingTusMetadata(metadataValues, fileName)
+	metadata := encodeTusMetadata(metadataValues)
 	setHeaders := func(h http.Header, fn string) {
 		setHubTusHeaders(h, config, fn)
 	}
 	return runTusUpload(baseURL, metadata, fileName, label, slot, setHeaders)
 }
 
+func addRecordingTusMetadata(values map[string]string, fileName string) {
+	metadata, ok := queuedRecordingMetadata(fileName)
+	if !ok {
+		return
+	}
+	if metadata.DeviceKey != "" {
+		values["device"] = metadata.DeviceKey
+	}
+	values["device_name"] = metadata.DeviceName
+	values["region_coordinates"] = metadata.RegionCoordinates
+	values["number_of_changes"] = metadata.NumberOfChanges
+	if metadata.Duration > 0 {
+		values["duration"] = strconv.FormatUint(metadata.Duration, 10)
+	}
+	if metadata.Timestamp > 0 {
+		values["timestamp"] = strconv.FormatInt(metadata.Timestamp, 10)
+	}
+	if metadata.Encrypted {
+		values["encrypted"] = "true"
+	}
+}
+
 // tusCreate performs the tus "creation" request (POST). On success it returns
 // the resolved upload URL the agent should use for subsequent HEAD/PATCH calls.
 func tusCreate(client *http.Client, baseURL string, size int64, metadata string, setHeaders tusHeaderFunc, fileName string) (string, int, error) {
@@ -320,11 +404,29 @@ func tusCreate(client *http.Client, baseURL string, size int64, metadata string,
 	if err != nil {
 		return "", 0, err
 	}
-	io.Copy(io.Discard, resp.Body)
-
 	if resp.StatusCode != http.StatusCreated {
+		const maxErrorBodyBytes = 4096
+		body, readErr := io.ReadAll(io.LimitReader(resp.Body, maxErrorBodyBytes+1))
+		if readErr != nil {
+			return "", resp.StatusCode, fmt.Errorf("unexpected status creating upload: %s; read error response: %w", resp.Status, readErr)
+		}
+		truncated := len(body) > maxErrorBodyBytes
+		if truncated {
+			body = body[:maxErrorBodyBytes]
+		}
+		message := strings.TrimSpace(string(body))
+		if truncated {
+			message += " [truncated]"
+		}
+		if message != "" {
+			return "", resp.StatusCode, fmt.Errorf("unexpected status creating upload: %s: %q", resp.Status, message)
+		}
 		return "", resp.StatusCode, fmt.Errorf("unexpected status creating upload: %s", resp.Status)
 	}
+	// Location identifies the created upload; its response body is optional.
+	if _, err := io.Copy(io.Discard, resp.Body); err != nil {
+		log.WithError(err).WithField("component", "tus").Debug("Could not drain optional upload creation response body")
+	}
 	location := resp.Header.Get("Location")
 	if location == "" {
 		return "", resp.StatusCode, errors.New("missing Location header in create response")
@@ -430,6 +532,7 @@ func setVaultTusHeaders(h http.Header, vault models.KStorage, publicKey, deviceK
 	h.Set("X-Kerberos-Storage-Capture", "IPCamera")
 	if fileName != "" {
 		h.Set("X-Kerberos-Storage-FileName", fileName)
+		setQueuedRecordingMetadataHeaders(h, fileName)
 	}
 }
 
@@ -446,6 +549,7 @@ func setHubTusHeaders(h http.Header, config *models.Config, fileName string) {
 	h.Set("X-Kerberos-Storage-Capture", "IPCamera")
 	if fileName != "" {
 		h.Set("X-Kerberos-Storage-FileName", fileName)
+		setQueuedRecordingMetadataHeaders(h, fileName)
 	}
 }
 
@@ -535,10 +639,15 @@ func removeTusResumeState(path string) {
 	_ = os.Remove(path)
 }
 
+// tusBackoffBaseDelay is the base delay used by tusBackoff for the exponential
+// back-off between resume attempts. It is a package variable (rather than a
+// constant) so tests can shrink it to keep them fast.
+var tusBackoffBaseDelay = 500 * time.Millisecond
+
 // tusBackoff sleeps for an exponentially increasing duration (capped) between
 // resume attempts to avoid hammering a temporarily unavailable vault.
 func tusBackoff(attempt int) {
-	delay := time.Duration(500*(1<<uint(attempt))) * time.Millisecond
+	delay := tusBackoffBaseDelay * time.Duration(1<<uint(attempt))
 	if delay > 3*time.Second {
 		delay = 3 * time.Second
 	}

machinery/src/cloud/tus_client_test.go

@@ -3,8 +3,10 @@ package cloud
 import (
 	"bytes"
 	"encoding/base64"
+	"encoding/json"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -12,11 +14,231 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"testing"
+	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/utils"
 )
 
+func TestResumableEOFRestartPreservesLatestFailure(t *testing.T) {
+	previousDelay := tusBackoffBaseDelay
+	tusBackoffBaseDelay = 0
+	t.Cleanup(func() { tusBackoffBaseDelay = previousDelay })
+	const fileName = "1767355200123-device-key.mp4"
+	withRecording(t, fileName, []byte("recording"))
+	fake := newFakeTus()
+	fake.failFinalize = 1
+	var heads atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method == http.MethodHead && heads.Add(1) <= 2 {
+			w.WriteHeader(http.StatusServiceUnavailable)
+			return
+		}
+		fake.ServeHTTP(w, r)
+	}))
+	defer server.Close()
+
+	uploaded, responded, supported, _, err := uploadVaultResumable(testVault(server.URL), "public", "device-key", fileName, "test", "primary")
+	if uploaded || !responded || !supported || err == nil ||
+		!strings.Contains(err.Error(), "unexpected status on PATCH: 502 Bad Gateway") ||
+		strings.Contains(err.Error(), "503 Service Unavailable") {
+		t.Fatalf("upload = %t/%t/%t, %v; want the latest finalization failure after EOF restart", uploaded, responded, supported, err)
+	}
+	patches, _ := fake.patchCounts()
+	if heads.Load() != 4 || fake.createCount() != 1 || patches != 1 ||
+		len(fake.requestsForMethod(http.MethodDelete)) != 1 {
+		t.Fatalf("unexpected retry sequence: heads=%d, creates=%d, patches=%d", heads.Load(), fake.createCount(), patches)
+	}
+}
+
+func TestResumableCreationKeepsLocationAfterBodyReadError(t *testing.T) {
+	previousDelay := tusBackoffBaseDelay
+	tusBackoffBaseDelay = 0
+	t.Cleanup(func() { tusBackoffBaseDelay = previousDelay })
+	const fileName = "1767355200123-device-key.mp4"
+	payload := []byte("recording")
+	withRecording(t, fileName, payload)
+	fake := newFakeTus()
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method == http.MethodPost {
+			w.Header().Set("Content-Length", "100")
+			fake.ServeHTTP(w, r)
+			_, _ = io.WriteString(w, "short")
+			return
+		}
+		fake.ServeHTTP(w, r)
+	}))
+	defer server.Close()
+
+	uploaded, responded, supported, _, err := uploadVaultResumable(testVault(server.URL), "public", "device-key", fileName, "test", "primary")
+	if !uploaded || !responded || !supported || err != nil {
+		t.Fatalf("upload = %t/%t/%t, %v; want success despite the interrupted optional creation body", uploaded, responded, supported, err)
+	}
+	if fake.createCount() != 1 || fake.totalBytes() != int64(len(payload)) {
+		t.Fatalf("created %d slots and uploaded %d bytes; want one reused slot and %d bytes", fake.createCount(), fake.totalBytes(), len(payload))
+	}
+}
+
+func TestTusCreateStillRequiresLocationAfterBodyReadError(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Length", "100")
+		w.WriteHeader(http.StatusCreated)
+		_, _ = io.WriteString(w, "short")
+	}))
+	defer server.Close()
+	location, status, err := tusCreate(server.Client(), server.URL, 8, "",
+		func(http.Header, string) {}, "1767355200123-device-key.mp4")
+	if location != "" || status != http.StatusCreated || err == nil ||
+		!strings.Contains(err.Error(), "missing Location") {
+		t.Fatalf("creation = %q, %d, %v; want missing Location error", location, status, err)
+	}
+}
+
+func TestTusCreateReportsBoundedServerErrors(t *testing.T) {
+	for _, tc := range []struct {
+		name string
+		body string
+		want string
+	}{
+		{"server error", "ERR_INTERNAL_SERVER_ERROR: A region must be set when sending requests to S3.\n", "A region must be set"},
+		{"empty error", "", "500 Internal Server Error"},
+		{"large error", strings.Repeat("x", 10000), "[truncated]"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.WriteHeader(http.StatusInternalServerError)
+				_, _ = io.WriteString(w, tc.body)
+			}))
+			defer server.Close()
+			location, status, err := tusCreate(server.Client(), server.URL, 8, "",
+				func(http.Header, string) {}, "1767355200123-device-key.mp4")
+			if location != "" || status != http.StatusInternalServerError || err == nil ||
+				!strings.Contains(err.Error(), tc.want) || len(err.Error()) > 4300 {
+				t.Fatalf("creation = %q, %d, %v; want bounded server error", location, status, err)
+			}
+		})
+	}
+}
+
+func TestResumableCreateFailureRetainsServerCause(t *testing.T) {
+	previousDelay := tusBackoffBaseDelay
+	tusBackoffBaseDelay = 0
+	t.Cleanup(func() { tusBackoffBaseDelay = previousDelay })
+	for _, integration := range []string{"vault", "hub"} {
+		t.Run(integration, func(t *testing.T) {
+			const fileName = "1767355200123-device-key.mp4"
+			withRecording(t, fileName, []byte("recording"))
+			var requests atomic.Int32
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				requests.Add(1)
+				if r.Method != http.MethodPost || r.URL.Path != tusUploadPath {
+					t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
+				}
+				w.WriteHeader(http.StatusInternalServerError)
+				_, _ = io.WriteString(w, "ERR_INTERNAL_SERVER_ERROR: A region must be set when sending requests to S3.")
+			}))
+			defer server.Close()
+			var uploaded, responded, supported bool
+			var err error
+			if integration == "vault" {
+				uploaded, responded, supported, _, err = uploadVaultResumable(testVault(server.URL), "public", "device-key", fileName, "test", "primary")
+			} else {
+				uploaded, responded, supported, _, err = uploadHubResumable(testHubConfig(server.URL), fileName, "test", "hub")
+			}
+			if uploaded || !responded || !supported || err == nil ||
+				!strings.Contains(err.Error(), "resumable upload did not complete after retries") ||
+				!strings.Contains(err.Error(), "A region must be set") {
+				t.Fatalf("upload = %t/%t/%t, %v; want original server cause after retries", uploaded, responded, supported, err)
+			}
+			if requests.Load() != 4 {
+				t.Fatalf("creation attempts = %d, want 4 without legacy fallback", requests.Load())
+			}
+			if _, err := os.Stat(filepath.Join("data/recordings", fileName)); err != nil {
+				t.Fatalf("failed upload must retain the local recording: %v", err)
+			}
+		})
+	}
+}
+
+func TestUploadKerberosVaultReadsDatedLocalRecordings(t *testing.T) {
+	previousCount, previousTimeout := kstorageRetryCount, kstorageRetryTimeout
+	t.Cleanup(func() { kstorageRetryCount, kstorageRetryTimeout = previousCount, previousTimeout })
+	for _, transport := range []string{"resumable", "legacy"} {
+		t.Run(transport, func(t *testing.T) {
+			const fileName = "1564859471123-device-key.mp4"
+			payload := bytes.Repeat([]byte("video"), 1024)
+			withRecording(t, fileName, payload)
+			datedPath, err := utils.CreateRecordingPath("data/recordings", fileName, time.Unix(1564859471, 0))
+			if err != nil {
+				t.Fatal(err)
+			}
+			if err := os.Rename(filepath.Join("data/recordings", fileName), datedPath); err != nil {
+				t.Fatal(err)
+			}
+			withQueuedRecordingFPS(t, fileName, `{"filename":"1564859471123-device-key.mp4","device_key":"device-key","timestamp":1564859471123,"duration":27,"fps":25,"device_name":"camera-name","region_coordinates":"1-2-3-4","number_of_changes":"57"}`)
+			fake := newFakeTus()
+			var legacyBody []byte
+			var legacyHeaders http.Header
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				if transport == "resumable" {
+					fake.ServeHTTP(w, r)
+					return
+				}
+				body, readErr := io.ReadAll(r.Body)
+				if readErr != nil {
+					t.Error(readErr)
+				}
+				legacyBody = body
+				legacyHeaders = r.Header.Clone()
+				w.WriteHeader(http.StatusOK)
+			}))
+			defer server.Close()
+			if transport == "legacy" {
+				t.Setenv("AGENT_DISABLE_RESUMABLE_UPLOAD", "true")
+			} else {
+				t.Setenv("AGENT_DISABLE_RESUMABLE_UPLOAD", "false")
+			}
+			kstorageRetryCount, kstorageRetryTimeout = 0, 0
+			vault := testVault(server.URL)
+			config := &models.Configuration{Config: models.Config{
+				Key:               "changed-config-key",
+				KStorage:          &vault,
+				KStorageSecondary: &models.KStorage{},
+			}}
+			uploaded, configured, err := UploadKerberosVault(config, fileName)
+			if err != nil || !uploaded || !configured {
+				t.Fatalf("UploadKerberosVault() = (%v, %v, %v), want success", uploaded, configured, err)
+			}
+			if transport == "resumable" {
+				if got := fake.totalBytes(); got != int64(len(payload)) {
+					t.Fatalf("uploaded %d bytes, want %d", got, len(payload))
+				}
+				posts := fake.requestsForMethod(http.MethodPost)
+				if len(posts) != 1 {
+					t.Fatalf("got %d creation requests, want 1", len(posts))
+				}
+				metadata := decodeTusMetadata(posts[0].header.Get("Upload-Metadata"))
+				if metadata["filename"] != fileName || metadata["directory"] != "dir" || metadata["timestamp"] != "1564859471123" ||
+					metadata["device"] != "device-key" || metadata["device_name"] != "camera-name" ||
+					metadata["region_coordinates"] != "1-2-3-4" || metadata["number_of_changes"] != "57" {
+					t.Fatalf("unexpected upload metadata: %#v", metadata)
+				}
+			} else if !bytes.Equal(legacyBody, payload) ||
+				legacyHeaders.Get("X-Kerberos-Storage-FileName") != fileName ||
+				legacyHeaders.Get("X-Kerberos-Storage-Directory") != "dir" ||
+				legacyHeaders.Get("X-Kerberos-Storage-Device") != "device-key" ||
+				legacyHeaders.Get(recordingDeviceNameHeader) != "camera-name" ||
+				legacyHeaders.Get(recordingRegionHeader) != "1-2-3-4" ||
+				legacyHeaders.Get(recordingChangesHeader) != "57" ||
+				legacyHeaders.Get(recordingTimestampHeader) != "1564859471123" {
+				t.Fatalf("unexpected legacy upload: %d bytes, headers %#v", len(legacyBody), legacyHeaders)
+			}
+		})
+	}
+}
+
 // fakeUpload tracks the state of a single resumable upload on the fake server.
 type fakeUpload struct {
 	size   int64
@@ -47,6 +269,14 @@ type fakeTus struct {
 	// after storing the bytes, simulating a failed completion hook.
 	failFinalize int
 
+	// loseProgress simulates a vault that never durably retains the in-progress
+	// upload: every PATCH is acknowledged (the response advertises the advanced
+	// offset) but the stored offset is immediately reset to 0. HEAD therefore
+	// keeps reporting 0 and the next chunk — sent at the advanced offset — is
+	// rejected with 409, reproducing the cross-replica ERR_MISMATCHED_OFFSET
+	// loop that previously wedged the agent's upload worker forever.
+	loseProgress bool
+
 	// requests records the headers of every received request (in order) so
 	// tests can assert which auth/routing headers the client sent per method.
 	requests []recordedRequest
@@ -153,6 +383,31 @@ func (s *fakeTus) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			w.WriteHeader(http.StatusNotFound)
 			return
 		}
+		if s.loseProgress {
+			reqOffset, _ := strconv.ParseInt(r.Header.Get("Upload-Offset"), 10, 64)
+			s.mu.Lock()
+			cur := u.offset
+			if reqOffset != cur {
+				// The offset the client resumes from no longer matches what this
+				// "replica" retained, so reject like a vault returning
+				// ERR_MISMATCHED_OFFSET.
+				s.mu.Unlock()
+				w.Header().Set("Upload-Offset", strconv.FormatInt(cur, 10))
+				w.WriteHeader(http.StatusConflict)
+				return
+			}
+			n, _ := io.Copy(io.Discard, r.Body)
+			s.lastPatchBytes = n
+			s.patchSizes = append(s.patchSizes, n)
+			// Advertise progress to the client, then immediately forget it so the
+			// next chunk (sent at the advanced offset) mismatches again.
+			reported := cur + n
+			u.offset = 0
+			s.mu.Unlock()
+			w.Header().Set("Upload-Offset", strconv.FormatInt(reported, 10))
+			w.WriteHeader(http.StatusNoContent)
+			return
+		}
 		n, _ := io.Copy(io.Discard, r.Body)
 		s.mu.Lock()
 		u.offset += n
@@ -208,6 +463,17 @@ func withRecording(t *testing.T, fileName string, payload []byte) {
 	}
 }
 
+func withQueuedRecordingFPS(t *testing.T, fileName, fps string) {
+	t.Helper()
+	if err := os.MkdirAll("data/cloud", 0o755); err != nil {
+		t.Fatalf("mkdir cloud queue: %v", err)
+	}
+	markerName := models.RecordingUploadMetadataFileName(fileName)
+	if err := os.WriteFile(filepath.Join("data/cloud", markerName), []byte(fps), 0o644); err != nil {
+		t.Fatalf("write cloud queue marker: %v", err)
+	}
+}
+
 func testVault(uri string) models.KStorage {
 	return models.KStorage{
 		URI:             uri,
@@ -218,6 +484,36 @@ func testVault(uri string) models.KStorage {
 	}
 }
 
+func TestUploadKerberosVaultSkipsEmptyRecording(t *testing.T) {
+	fileName := "1787015373_3-654_office-camera17_0-0-0-0_-1_1960.mp4"
+	withRecording(t, fileName, nil)
+
+	requestCount := 0
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requestCount++
+		w.WriteHeader(http.StatusInternalServerError)
+	}))
+	defer server.Close()
+
+	vault := testVault(server.URL)
+	configuration := &models.Configuration{Config: models.Config{
+		Key:               "device-key",
+		KStorage:          &vault,
+		KStorageSecondary: &models.KStorage{},
+	}}
+
+	uploaded, configured, err := UploadKerberosVault(configuration, fileName)
+	if err != nil {
+		t.Fatalf("UploadKerberosVault() error = %v", err)
+	}
+	if uploaded || configured {
+		t.Fatalf("UploadKerberosVault() uploaded/configured = %v/%v, want false/false", uploaded, configured)
+	}
+	if requestCount != 0 {
+		t.Fatalf("Vault received %d requests, want 0", requestCount)
+	}
+}
+
 func TestUploadVaultResumable_HappyPath(t *testing.T) {
 	srv := newFakeTus()
 	ts := httptest.NewServer(srv)
@@ -226,8 +522,11 @@ func TestUploadVaultResumable_HappyPath(t *testing.T) {
 	fileName := "1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4"
 	payload := bytes.Repeat([]byte("x"), 4096)
 	withRecording(t, fileName, payload)
+	withQueuedRecordingFPS(t, fileName, `{"filename":"recording.mp4","device_key":"device-key","timestamp":1785934709414,"duration":20452,"fps":29.97}`)
 
-	uploaded, responded, supported, _, err := uploadVaultResumable(testVault(ts.URL), "pk", "dev", fileName, "test", "primary")
+	vault := testVault(ts.URL)
+	vault.CustomHeaders = `{"site_id":"site-1","equipment_id":"equipment-3"}`
+	uploaded, responded, supported, _, err := uploadVaultResumable(vault, "pk", "dev", fileName, "test", "primary")
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -240,6 +539,128 @@ func TestUploadVaultResumable_HappyPath(t *testing.T) {
 	if _, err := os.Stat(tusSidecarPath(fileName, "primary")); !os.IsNotExist(err) {
 		t.Fatalf("expected sidecar to be removed after success, stat err = %v", err)
 	}
+	posts := srv.requestsForMethod(http.MethodPost)
+	metadata := decodeTusMetadata(posts[0].header.Get("Upload-Metadata"))
+	if got := metadata["fps"]; got != "29.97" {
+		t.Fatalf("POST metadata fps = %q, want %q", got, "29.97")
+	}
+	if got := metadata["duration"]; got != "20452" {
+		t.Fatalf("POST metadata duration = %q, want %q", got, "20452")
+	}
+	if got := metadata["timestamp"]; got != "1785934709414" {
+		t.Fatalf("POST metadata timestamp = %q, want %q", got, "1785934709414")
+	}
+	var customHeaders map[string]string
+	if err := json.Unmarshal([]byte(metadata["custom_headers"]), &customHeaders); err != nil {
+		t.Fatal(err)
+	}
+	if customHeaders["site_id"] != "site-1" || customHeaders["equipment_id"] != "equipment-3" {
+		t.Fatalf("POST custom header metadata = %#v", customHeaders)
+	}
+	for _, method := range []string{http.MethodPost, http.MethodHead, http.MethodPatch} {
+		for _, request := range srv.requestsForMethod(method) {
+			if request.header.Get("site_id") != "site-1" || request.header.Get("equipment_id") != "equipment-3" {
+				t.Fatalf("%s custom headers = %#v", method, request.header)
+			}
+			if request.header.Get(customHeadersManifestHeader) == "" {
+				t.Fatalf("%s is missing the custom header manifest", method)
+			}
+		}
+	}
+}
+
+func TestQueuedRecordingFPSValidation(t *testing.T) {
+	for _, test := range []struct {
+		name string
+		fps  string
+		want string
+	}{
+		{name: "json", fps: `{"fps":29}`, want: "29"},
+		{name: "json fractional", fps: `{"fps":17.35}`, want: "17.35"},
+		{name: "json with future field", fps: `{"fps":29,"codec":"h264"}`, want: "29"},
+		{name: "json without fps", fps: `{}`},
+		{name: "json invalid fps", fps: `{"fps":241}`},
+		{name: "legacy fractional", fps: "29.97", want: "29.97"},
+		{name: "legacy trimmed", fps: " 25 \n", want: "25"},
+		{name: "empty"},
+		{name: "invalid", fps: "invalid"},
+		{name: "zero", fps: "0"},
+		{name: "negative", fps: "-1"},
+		{name: "nan", fps: "NaN"},
+		{name: "infinite", fps: "+Inf"},
+		{name: "unreasonable", fps: "241"},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			fileName := "recording.mp4"
+			withRecording(t, fileName, []byte("recording"))
+			withQueuedRecordingFPS(t, fileName, test.fps)
+
+			if got := queuedRecordingFPS(fileName); got != test.want {
+				t.Fatalf("queuedRecordingFPS() = %q, want %q", got, test.want)
+			}
+
+			header := make(http.Header)
+			setQueuedRecordingMetadataHeaders(header, fileName)
+			if got := header.Get(recordingFPSHeader); got != test.want {
+				t.Fatalf("legacy FPS header = %q, want %q", got, test.want)
+			}
+		})
+	}
+}
+
+func TestQueuedRecordingFPSAllowsMissingHistoricalMarker(t *testing.T) {
+	fileName := "recording.mp4"
+	withRecording(t, fileName, []byte("recording"))
+
+	if got := queuedRecordingFPS(fileName); got != "" {
+		t.Fatalf("queuedRecordingFPS() = %q, want empty for missing marker", got)
+	}
+	header := make(http.Header)
+	setQueuedRecordingMetadataHeaders(header, fileName)
+	if got := header.Get(recordingFPSHeader); got != "" {
+		t.Fatalf("legacy FPS header = %q, want empty for missing marker", got)
+	}
+}
+
+func TestQueuedRecordingMetadataHeaders(t *testing.T) {
+	fileName := "recording.mp4"
+	withRecording(t, fileName, []byte("recording"))
+	withQueuedRecordingFPS(t, fileName, `{"filename":"recording.mp4","device_key":"device-key","timestamp":1785934709414,"duration":20452,"fps":25,"encrypted":true}`)
+
+	header := make(http.Header)
+	setQueuedRecordingMetadataHeaders(header, fileName)
+	if got := header.Get(recordingFPSHeader); got != "25" {
+		t.Fatalf("FPS header = %q", got)
+	}
+	if got := header.Get(recordingDurationHeader); got != "20452" {
+		t.Fatalf("duration header = %q", got)
+	}
+	if got := header.Get(recordingTimestampHeader); got != "1785934709414" {
+		t.Fatalf("timestamp header = %q", got)
+	}
+	if got := header.Get(recordingEncryptedHeader); got != "true" {
+		t.Fatalf("encrypted header = %q", got)
+	}
+	metadata := map[string]string{}
+	addRecordingTusMetadata(metadata, fileName)
+	if got := metadata["encrypted"]; got != "true" {
+		t.Fatalf("encrypted TUS metadata = %q", got)
+	}
+}
+
+func TestQueuedRecordingFPSAllowsLegacyMarkerFileName(t *testing.T) {
+	fileName := "recording.mp4"
+	withRecording(t, fileName, []byte("recording"))
+	if err := os.MkdirAll("data/cloud", 0o755); err != nil {
+		t.Fatalf("mkdir cloud queue: %v", err)
+	}
+	if err := os.WriteFile(filepath.Join("data/cloud", fileName), []byte("25"), 0o644); err != nil {
+		t.Fatalf("write legacy cloud queue marker: %v", err)
+	}
+
+	if got := queuedRecordingFPS(fileName); got != "25" {
+		t.Fatalf("queuedRecordingFPS() = %q, want legacy marker FPS", got)
+	}
 }
 
 func TestUploadVaultResumable_Chunked(t *testing.T) {
@@ -350,6 +771,102 @@ func TestUploadVaultResumable_Unsupported(t *testing.T) {
 	}
 }
 
+// TestUploadVaultResumable_NetworkErrorKeepsRetryBudget verifies that when the
+// vault is unreachable (mimicking the internet being disconnected) the resumable
+// upload reports responded=false. That is what stops the caller
+// (UploadKerberosVault) from consuming its retry budget and entering the long
+// back-off timeout on a transient network outage, so the recording keeps being
+// retried until connectivity returns.
+func TestUploadVaultResumable_NetworkErrorKeepsRetryBudget(t *testing.T) {
+	// Bind then immediately release a loopback port so every connection to it is
+	// refused, producing a transport-level error (no HTTP response).
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	addr := ln.Addr().String()
+	if cerr := ln.Close(); cerr != nil {
+		t.Fatalf("close listener: %v", cerr)
+	}
+
+	// Keep the between-attempt back-off tiny so the test stays fast.
+	oldDelay := tusBackoffBaseDelay
+	tusBackoffBaseDelay = time.Millisecond
+	defer func() { tusBackoffBaseDelay = oldDelay }()
+
+	fileName := "1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4"
+	withRecording(t, fileName, bytes.Repeat([]byte("n"), 2048))
+
+	uploaded, responded, supported, _, err := uploadVaultResumable(testVault("http://"+addr), "pk", "dev", fileName, "test", "primary")
+	if uploaded {
+		t.Fatal("expected uploaded=false when the vault is unreachable")
+	}
+	if !supported {
+		t.Fatal("a transport error is not a missing tus endpoint; expected supported=true")
+	}
+	if responded {
+		t.Fatal("expected responded=false for a pure network error so the retry budget is preserved")
+	}
+	if err == nil {
+		t.Fatal("expected an error when the vault is unreachable")
+	}
+}
+
+func TestUploadVaultResumable_MismatchedOffsetGivesUp(t *testing.T) {
+	// A vault that never durably retains the in-progress upload (offset resets to
+	// 0 between chunks) makes every resume "progress" by one chunk and then fail
+	// the next chunk with 409. Before the high-water gating fix this refreshed the
+	// retry budget every attempt and looped forever, wedging the upload worker and
+	// saturating the uplink (which starved heartbeats and reported the camera
+	// offline). The loop must now be bounded: give up after a fixed number of
+	// non-progressing attempts and report responded=true so the caller re-queues.
+	srv := newFakeTus()
+	srv.loseProgress = true
+	ts := httptest.NewServer(srv)
+	defer ts.Close()
+
+	// Keep the between-attempt back-off tiny so the test stays fast.
+	oldDelay := tusBackoffBaseDelay
+	tusBackoffBaseDelay = time.Millisecond
+	defer func() { tusBackoffBaseDelay = oldDelay }()
+
+	// Force multiple chunks so there is always a second chunk to be rejected.
+	t.Setenv("AGENT_TUS_CHUNK_SIZE_BYTES", "4096")
+
+	fileName := "1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4"
+	withRecording(t, fileName, bytes.Repeat([]byte("m"), 12288))
+
+	done := make(chan struct{})
+	var uploaded, responded bool
+	var upErr error
+	go func() {
+		uploaded, responded, _, _, upErr = uploadVaultResumable(testVault(ts.URL), "pk", "dev", fileName, "test", "primary")
+		close(done)
+	}()
+
+	select {
+	case <-done:
+	case <-time.After(30 * time.Second):
+		t.Fatal("resumable upload did not terminate: the retry loop is unbounded on a persistent mismatched offset")
+	}
+
+	if uploaded {
+		t.Fatal("expected uploaded=false when the vault never retains the offset")
+	}
+	if !responded {
+		t.Fatal("expected responded=true (the vault answered) so the caller re-queues the recording")
+	}
+	if upErr == nil {
+		t.Fatal("expected an error when the upload cannot complete")
+	}
+
+	// The bounded retry budget must cap the number of PATCH requests. Two PATCHes
+	// per attempt across a handful of attempts stays comfortably below this.
+	if count, _ := srv.patchCounts(); count > 50 {
+		t.Fatalf("expected a bounded number of PATCH requests, got %d (retry loop not bounded)", count)
+	}
+}
+
 func TestUploadVaultResumable_FinalizeRetry(t *testing.T) {
 	srv := newFakeTus()
 	srv.failFinalize = 1
@@ -448,6 +965,7 @@ func TestUploadHubResumable_HappyPath(t *testing.T) {
 	fileName := "1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4"
 	payload := bytes.Repeat([]byte("h"), 4096)
 	withRecording(t, fileName, payload)
+	withQueuedRecordingFPS(t, fileName, "29.97")
 
 	uploaded, _, supported, _, err := uploadHubResumable(testHubConfig(ts.URL), fileName, "test", "hub")
 	if err != nil {
@@ -518,6 +1036,9 @@ func TestUploadHubResumable_HappyPath(t *testing.T) {
 	if meta["capture"] != "IPCamera" {
 		t.Errorf("hub metadata capture = %q, want %q", meta["capture"], "IPCamera")
 	}
+	if meta["fps"] != "29.97" {
+		t.Errorf("hub metadata fps = %q, want %q", meta["fps"], "29.97")
+	}
 }
 
 func TestUploadHubResumable_Unsupported(t *testing.T) {

machinery/src/components/backchannel.go

@@ -2,26 +2,83 @@ package components
 
 import (
 	"bufio"
-	"fmt"
+	"context"
+	"math/rand"
 	"os"
 	"time"
 
 	"github.com/kerberos-io/agent/machinery/src/capture"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
 	"github.com/kerberos-io/joy4/av"
 	"github.com/pion/rtp"
+	log "github.com/sirupsen/logrus"
 	"github.com/zaf/g711"
 )
 
+const (
+	backchannelSampleRate       = 8000
+	backchannelTalkspurtGap     = 500 * time.Millisecond
+	backchannelReconnectInitial = time.Second
+	backchannelReconnectMax     = 30 * time.Second
+)
+
+type backchannelClient interface {
+	ConnectBackChannel(ctx context.Context, otelContext context.Context) error
+	StartBackChannel(ctx context.Context, otelContext context.Context) error
+	WritePacket(pkt packets.Packet) error
+	Close(otelContext context.Context) error
+}
+
+type backchannelPacketizer struct {
+	sequenceNumber uint16
+	timestamp      uint32
+	ssrc           uint32
+	lastPacketAt   time.Time
+}
+
+func newBackchannelPacketizer() backchannelPacketizer {
+	return backchannelPacketizer{
+		sequenceNumber: uint16(rand.Uint32()),
+		timestamp:      rand.Uint32(),
+		ssrc:           rand.Uint32(),
+	}
+}
+
+func (p *backchannelPacketizer) packet(audio models.AudioDataPartial, now time.Time) packets.Packet {
+	bufferUlaw := make([]byte, len(audio.Data))
+	for index, sample := range audio.Data {
+		bufferUlaw[index] = g711.EncodeUlawFrame(sample)
+	}
+
+	pkt := packets.Packet{
+		Packet: &rtp.Packet{
+			Header: rtp.Header{
+				Version:        2,
+				Marker:         p.lastPacketAt.IsZero() || now.Sub(p.lastPacketAt) >= backchannelTalkspurtGap,
+				PayloadType:    0,
+				SequenceNumber: p.sequenceNumber,
+				Timestamp:      p.timestamp,
+				SSRC:           p.ssrc,
+			},
+			Payload: bufferUlaw,
+		},
+	}
+
+	p.timestamp += uint32(len(bufferUlaw))
+	p.sequenceNumber++
+	p.lastPacketAt = now
+
+	return pkt
+}
+
 func GetBackChannelAudioCodec(streams []av.CodecData, communication *models.Communication) av.AudioCodecData {
 	for _, stream := range streams {
 		if stream.Type().IsAudio() {
 			if stream.Type().String() == "PCM_MULAW" {
 				pcmuCodec := stream.(av.AudioCodecData)
 				if pcmuCodec.IsBackChannel() {
-					communication.HasBackChannel = true
+					communication.HasBackChannel.Store(true)
 					return pcmuCodec
 				}
 			}
@@ -30,49 +87,123 @@ func GetBackChannelAudioCodec(streams []av.CodecData, communication *models.Comm
 	return nil
 }
 
-func WriteAudioToBackchannel(communication *models.Communication, rtspClient capture.RTSPClient) {
-	log.Log.Info("Audio.WriteAudioToBackchannel(): writing to backchannel audio codec")
-	length := uint32(0)
-	sequenceNumber := uint16(0)
-	for audio := range communication.HandleAudio {
-		// Encode PCM to MULAW
-		var bufferUlaw []byte
-		for _, v := range audio.Data {
-			b := g711.EncodeUlawFrame(v)
-			bufferUlaw = append(bufferUlaw, b)
-		}
+func WriteAudioToBackchannel(ctx context.Context, communication *models.Communication, rtspClient capture.RTSPClient) {
+	writeAudioToBackchannel(ctx, ctx, communication.HandleAudio, rtspClient)
+}
 
-		pkt := packets.Packet{
-			Packet: &rtp.Packet{
-				Header: rtp.Header{
-					Version:        2,
-					Marker:         true, // should be true
-					PayloadType:    0,    //packet.PayloadType, // will be owerwriten
-					SequenceNumber: sequenceNumber,
-					Timestamp:      uint32(length),
-					SSRC:           1293847657,
-				},
-				Payload: bufferUlaw,
-			},
-		}
-		err := rtspClient.WritePacket(pkt)
-		if err != nil {
-			log.Log.Error("Audio.WriteAudioToBackchannel(): error writing packet to backchannel")
-		}
+func writeAudioToBackchannel(ctx context.Context, otelContext context.Context, audioChannel <-chan models.AudioDataPartial, rtspClient backchannelClient) {
+	log.Info("Audio.WriteAudioToBackchannel(): writing to backchannel audio codec")
 
-		length = (length + uint32(len(bufferUlaw))) % 65536
-		sequenceNumber = (sequenceNumber + 1) % 65535
-		time.Sleep(128 * time.Millisecond)
+	if err := rtspClient.StartBackChannel(ctx, otelContext); err != nil {
+		log.Error("Audio.WriteAudioToBackchannel(): error starting backchannel: " + err.Error())
+		if !reconnectBackchannel(ctx, otelContext, rtspClient) {
+			log.Info("Audio.WriteAudioToBackchannel(): stopped while reconnecting")
+			return
+		}
 	}
-	log.Log.Info("Audio.WriteAudioToBackchannel(): finished")
 
+	packetizer := newBackchannelPacketizer()
+	for {
+		select {
+		case <-ctx.Done():
+			log.Info("Audio.WriteAudioToBackchannel(): stopped")
+			return
+		case audio, ok := <-audioChannel:
+			if !ok {
+				log.Info("Audio.WriteAudioToBackchannel(): finished")
+				return
+			}
+
+			audio = latestBackchannelAudio(audio, audioChannel)
+			if len(audio.Data) == 0 {
+				continue
+			}
+
+			pkt := packetizer.packet(audio, time.Now())
+			if err := rtspClient.WritePacket(pkt); err != nil {
+				log.Error("Audio.WriteAudioToBackchannel(): error writing packet to backchannel: " + err.Error())
+				if !reconnectBackchannel(ctx, otelContext, rtspClient) {
+					log.Info("Audio.WriteAudioToBackchannel(): stopped while reconnecting")
+					return
+				}
+				packetizer = newBackchannelPacketizer()
+				continue
+			}
+
+			if !waitForBackchannel(ctx, time.Duration(len(audio.Data))*time.Second/backchannelSampleRate) {
+				log.Info("Audio.WriteAudioToBackchannel(): stopped")
+				return
+			}
+		}
+	}
+}
+
+func latestBackchannelAudio(audio models.AudioDataPartial, audioChannel <-chan models.AudioDataPartial) models.AudioDataPartial {
+	for {
+		select {
+		case next, ok := <-audioChannel:
+			if !ok {
+				return audio
+			}
+			audio = next
+		default:
+			return audio
+		}
+	}
+}
+
+func reconnectBackchannel(ctx context.Context, otelContext context.Context, rtspClient backchannelClient) bool {
+	backoff := backchannelReconnectInitial
+	for {
+		if err := rtspClient.Close(otelContext); err != nil {
+			log.Error("Audio.WriteAudioToBackchannel(): error closing failed backchannel: " + err.Error())
+		}
+		if ctx.Err() != nil {
+			return false
+		}
+
+		err := rtspClient.ConnectBackChannel(ctx, otelContext)
+		if err == nil {
+			err = rtspClient.StartBackChannel(ctx, otelContext)
+		}
+		if err == nil {
+			log.Info("Audio.WriteAudioToBackchannel(): reconnected backchannel")
+			return true
+		}
+
+		log.Error("Audio.WriteAudioToBackchannel(): error reconnecting backchannel: " + err.Error())
+		if !waitForBackchannel(ctx, backoff) {
+			return false
+		}
+		backoff *= 2
+		if backoff > backchannelReconnectMax {
+			backoff = backchannelReconnectMax
+		}
+	}
+}
+
+func waitForBackchannel(ctx context.Context, duration time.Duration) bool {
+	timer := time.NewTimer(duration)
+	defer timer.Stop()
+
+	select {
+	case <-ctx.Done():
+		return false
+	case <-timer.C:
+		return true
+	}
 }
 
 func WriteFileToBackChannel(infile av.DemuxCloser) {
 	// Do the warmup!
 	file, err := os.Open("./audiofile.bye")
 	if err != nil {
-		fmt.Println("WriteFileToBackChannel: error opening audiofile.bye file")
+		log.WithError(err).WithFields(log.Fields{
+			"component": "backchannel",
+			"event":     "audio_file_open_failed",
+			"path":      "./audiofile.bye",
+		}).Error("Failed to open backchannel audio file")
+		return
 	}
 	defer file.Close()
 

machinery/src/components/backchannel_test.go

@@ -0,0 +1,209 @@
+package components
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/kerberos-io/agent/machinery/src/models"
+	"github.com/kerberos-io/agent/machinery/src/packets"
+)
+
+type fakeBackchannelClient struct {
+	mutex           sync.Mutex
+	startErrors     []error
+	connectError    error
+	writeErrors     []error
+	startCalls      int
+	connectCalls    int
+	closeCalls      int
+	writeCalls      int
+	connectAttempt  chan struct{}
+	successfulWrite chan packets.Packet
+}
+
+func (f *fakeBackchannelClient) ConnectBackChannel(context.Context, context.Context) error {
+	f.mutex.Lock()
+	f.connectCalls++
+	err := f.connectError
+	f.mutex.Unlock()
+
+	select {
+	case f.connectAttempt <- struct{}{}:
+	default:
+	}
+	return err
+}
+
+func (f *fakeBackchannelClient) StartBackChannel(context.Context, context.Context) error {
+	f.mutex.Lock()
+	defer f.mutex.Unlock()
+	f.startCalls++
+	if len(f.startErrors) == 0 {
+		return nil
+	}
+	err := f.startErrors[0]
+	f.startErrors = f.startErrors[1:]
+	return err
+}
+
+func (f *fakeBackchannelClient) WritePacket(pkt packets.Packet) error {
+	f.mutex.Lock()
+	f.writeCalls++
+	var err error
+	if len(f.writeErrors) != 0 {
+		err = f.writeErrors[0]
+		f.writeErrors = f.writeErrors[1:]
+	}
+	f.mutex.Unlock()
+
+	if err == nil {
+		select {
+		case f.successfulWrite <- pkt:
+		default:
+		}
+	}
+	return err
+}
+
+func (f *fakeBackchannelClient) Close(context.Context) error {
+	f.mutex.Lock()
+	f.closeCalls++
+	f.mutex.Unlock()
+	return nil
+}
+
+func (f *fakeBackchannelClient) callCounts() (start, connect, close, write int) {
+	f.mutex.Lock()
+	defer f.mutex.Unlock()
+	return f.startCalls, f.connectCalls, f.closeCalls, f.writeCalls
+}
+
+func TestBackchannelPacketizerUsesFullRTPClock(t *testing.T) {
+	packetizer := backchannelPacketizer{ssrc: 1}
+	audio := models.AudioDataPartial{Data: make([]int16, 1024)}
+	startedAt := time.Unix(1, 0)
+
+	var timestamp uint32
+	for index := 0; index <= 64; index++ {
+		pkt := packetizer.packet(audio, startedAt.Add(time.Duration(index)*128*time.Millisecond))
+		timestamp = pkt.Packet.Timestamp
+	}
+
+	if timestamp != 65536 {
+		t.Fatalf("timestamp after 64 frames = %d, want 65536", timestamp)
+	}
+}
+
+func TestBackchannelPacketizerUsesNaturalSequenceRollover(t *testing.T) {
+	packetizer := backchannelPacketizer{sequenceNumber: ^uint16(0), ssrc: 1}
+	audio := models.AudioDataPartial{Data: []int16{0}}
+	startedAt := time.Unix(1, 0)
+
+	last := packetizer.packet(audio, startedAt)
+	firstAfterRollover := packetizer.packet(audio, startedAt.Add(time.Millisecond))
+
+	if last.Packet.SequenceNumber != ^uint16(0) {
+		t.Fatalf("last sequence number = %d, want %d", last.Packet.SequenceNumber, ^uint16(0))
+	}
+	if firstAfterRollover.Packet.SequenceNumber != 0 {
+		t.Fatalf("first sequence number after rollover = %d, want 0", firstAfterRollover.Packet.SequenceNumber)
+	}
+}
+
+func TestBackchannelPacketizerMarksTalkspurtStart(t *testing.T) {
+	packetizer := backchannelPacketizer{ssrc: 1}
+	audio := models.AudioDataPartial{Data: []int16{0}}
+	startedAt := time.Unix(1, 0)
+
+	first := packetizer.packet(audio, startedAt)
+	continuous := packetizer.packet(audio, startedAt.Add(128*time.Millisecond))
+	afterGap := packetizer.packet(audio, startedAt.Add(backchannelTalkspurtGap+128*time.Millisecond))
+
+	if !first.Packet.Marker {
+		t.Fatal("first packet must mark the start of a talkspurt")
+	}
+	if continuous.Packet.Marker {
+		t.Fatal("continuous packet must not carry the marker bit")
+	}
+	if !afterGap.Packet.Marker {
+		t.Fatal("packet after an audio gap must mark a new talkspurt")
+	}
+}
+
+func TestWriteAudioToBackchannelReconnectsAfterWriteFailure(t *testing.T) {
+	writeFailure := errors.New("EOF")
+	client := &fakeBackchannelClient{
+		writeErrors:     []error{writeFailure, nil},
+		connectAttempt:  make(chan struct{}, 1),
+		successfulWrite: make(chan packets.Packet, 1),
+	}
+	audioChannel := make(chan models.AudioDataPartial, 2)
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() {
+		writeAudioToBackchannel(ctx, ctx, audioChannel, client)
+		close(done)
+	}()
+
+	audioChannel <- models.AudioDataPartial{Data: make([]int16, 1024)}
+	select {
+	case <-client.connectAttempt:
+	case <-time.After(time.Second):
+		t.Fatal("backchannel was not reconnected after the write failure")
+	}
+
+	audioChannel <- models.AudioDataPartial{Data: make([]int16, 1024)}
+	select {
+	case pkt := <-client.successfulWrite:
+		if !pkt.Packet.Marker {
+			t.Fatal("first packet after reconnect must mark a new talkspurt")
+		}
+	case <-time.After(time.Second):
+		t.Fatal("fresh audio was not written after reconnect")
+	}
+
+	cancel()
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		t.Fatal("backchannel writer did not stop after cancellation")
+	}
+
+	startCalls, connectCalls, closeCalls, writeCalls := client.callCounts()
+	if startCalls != 2 || connectCalls != 1 || closeCalls != 1 || writeCalls != 2 {
+		t.Fatalf("calls (start, connect, close, write) = (%d, %d, %d, %d), want (2, 1, 1, 2)", startCalls, connectCalls, closeCalls, writeCalls)
+	}
+}
+
+func TestWriteAudioToBackchannelCancellationStopsReconnect(t *testing.T) {
+	client := &fakeBackchannelClient{
+		connectError:    errors.New("camera unavailable"),
+		writeErrors:     []error{errors.New("EOF")},
+		connectAttempt:  make(chan struct{}, 1),
+		successfulWrite: make(chan packets.Packet, 1),
+	}
+	audioChannel := make(chan models.AudioDataPartial, 1)
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() {
+		writeAudioToBackchannel(ctx, ctx, audioChannel, client)
+		close(done)
+	}()
+
+	audioChannel <- models.AudioDataPartial{Data: make([]int16, 1024)}
+	select {
+	case <-client.connectAttempt:
+	case <-time.After(time.Second):
+		t.Fatal("expected a reconnect attempt")
+	}
+	cancel()
+
+	select {
+	case <-done:
+	case <-time.After(250 * time.Millisecond):
+		t.Fatal("cancellation did not interrupt reconnect backoff")
+	}
+}

machinery/src/components/watchdog_test.go

@@ -0,0 +1,82 @@
+package components
+
+import (
+	"context"
+	"testing"
+	"time"
+)
+
+func TestWaitForRunRetryStopsOnCancellation(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	started := time.Now()
+	if waitForRunRetry(ctx) {
+		t.Fatal("waitForRunRetry() completed the retry delay after cancellation")
+	}
+	if elapsed := time.Since(started); elapsed > 100*time.Millisecond {
+		t.Fatalf("waitForRunRetry() took %s after cancellation", elapsed)
+	}
+}
+
+func TestStreamRestartWatchdogCoalescesStallsAndBacksOff(t *testing.T) {
+	now := time.Unix(1_000, 0)
+	watchdog := newStreamRestartWatchdog()
+	if _, restart := watchdog.Observe(now, 1, 1, true, false); restart {
+		t.Fatal("initial observation requested a restart")
+	}
+
+	for check := 1; check < streamWatchdogStallChecks; check++ {
+		now = now.Add(streamWatchdogInterval)
+		if _, restart := watchdog.Observe(now, 1, 1, true, false); restart {
+			t.Fatalf("check %d requested an early restart", check)
+		}
+	}
+	now = now.Add(streamWatchdogInterval)
+	reason, restart := watchdog.Observe(now, 1, 1, true, false)
+	if !restart || reason != "main and sub streams" {
+		t.Fatalf("Observe() = (%q, %t), want coalesced restart", reason, restart)
+	}
+
+	watchdog.MarkRestart(now)
+	if got := watchdog.Backoff(); got != 30*time.Second {
+		t.Fatalf("Backoff() = %s, want 30s", got)
+	}
+	for now = now.Add(streamWatchdogInterval); now.Before(watchdog.nextRestart); now = now.Add(streamWatchdogInterval) {
+		if _, restart := watchdog.Observe(now, 1, 1, true, false); restart {
+			t.Fatal("restart requested during cooldown")
+		}
+	}
+}
+
+func TestStreamRestartWatchdogResetsAfterHealthyMinute(t *testing.T) {
+	now := time.Unix(2_000, 0)
+	watchdog := newStreamRestartWatchdog()
+	watchdog.backoff = streamWatchdogMaxBackoff
+	watchdog.nextRestart = now.Add(streamWatchdogMaxBackoff)
+	watchdog.Observe(now, 1, 1, true, false)
+
+	for elapsed := streamWatchdogInterval; elapsed <= streamWatchdogHealthyReset+streamWatchdogInterval; elapsed += streamWatchdogInterval {
+		now = now.Add(streamWatchdogInterval)
+		watchdog.Observe(now, int64(elapsed), int64(elapsed), true, false)
+	}
+
+	if got := watchdog.Backoff(); got != streamWatchdogBaseBackoff {
+		t.Fatalf("Backoff() = %s, want %s", got, streamWatchdogBaseBackoff)
+	}
+	if !watchdog.nextRestart.IsZero() {
+		t.Fatalf("nextRestart = %s, want zero", watchdog.nextRestart)
+	}
+}
+
+func TestStreamRestartWatchdogPausesWhileConfiguring(t *testing.T) {
+	now := time.Unix(3_000, 0)
+	watchdog := newStreamRestartWatchdog()
+	watchdog.Observe(now, 1, 0, false, false)
+	for check := 0; check < streamWatchdogStallChecks+1; check++ {
+		now = now.Add(streamWatchdogInterval)
+		if _, restart := watchdog.Observe(now, 1, 0, false, true); restart {
+			t.Fatal("restart requested while configuring")
+		}
+	}
+}

machinery/src/computervision/main.go

@@ -8,34 +8,49 @@ import (
 	geo "github.com/kellydunn/golang-geo"
 	"github.com/kerberos-io/agent/machinery/src/capture"
 	"github.com/kerberos-io/agent/machinery/src/conditions"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
 	"github.com/kerberos-io/agent/machinery/src/packets"
+	log "github.com/sirupsen/logrus"
 )
 
 func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Configuration, communication *models.Communication, mqttClient mqtt.Client, rtspClient capture.RTSPClient) {
 
-	log.Log.Debug("computervision.main.ProcessMotion(): start motion detection")
+	log.Debug("computervision.main.ProcessMotion(): start motion detection")
 	config := configuration.Config
 	loc, _ := time.LoadLocation(config.Timezone)
 
 	var isPixelChangeThresholdReached = false
 	var changesToReturn = 0
 	var motionRectangle models.MotionRectangle
+	var motionRectangles []models.MotionRectangle
 
-	pixelThreshold := config.Capture.PixelChangeThreshold
-	// Might not be set in the config file, so set it to 150
-	if pixelThreshold == 0 {
-		pixelThreshold = 150
+	// Resolve the motion sensitivity (pixel-change threshold). Nil, zero, and
+	// negative values use the historical default so older configurations keep
+	// recording after an upgrade.
+	pixelThreshold := 150
+	if config.Capture.PixelChangeThreshold != nil && *config.Capture.PixelChangeThreshold > 0 {
+		pixelThreshold = *config.Capture.PixelChangeThreshold
 	}
+	// In motion mode we always run detection. In CONTINUOUS mode recording is
+	// 24/7 so motion detection is normally skipped, BUT if a motion region is
+	// configured we still run it so the live view can visualise the motion boxes
+	// + region. In that case we only emit the motion EVENT — no motion-triggered
+	// recording (continuous already records, and the recorder's motion branch
+	// isn't draining HandleMotion in continuous mode).
+	continuousMode := config.Capture.Continuous == "true"
+	hasMotionRegion := config.Region != nil && len(config.Region.Polygon) > 0
 
-	if config.Capture.Continuous == "true" {
+	if continuousMode && !hasMotionRegion {
 
-		log.Log.Info("computervision.main.ProcessMotion(): you've enabled continuous recording, so no motion detection required.")
+		log.Info("computervision.main.ProcessMotion(): continuous recording enabled and no motion region configured, so no motion detection required.")
 
 	} else {
 
-		log.Log.Info("computervision.main.ProcessMotion(): motion detected is enabled, so starting the motion detection.")
+		if continuousMode {
+			log.Info("computervision.main.ProcessMotion(): continuous recording enabled with a motion region, running motion detection for live-view visualisation only (no motion-triggered recording).")
+		} else {
+			log.Info("computervision.main.ProcessMotion(): motion detected is enabled, so starting the motion detection.")
+		}
 
 		hubKey := config.HubKey
 		deviceKey := config.Key
@@ -100,20 +115,47 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 			}
 		}
 
+		// Frame dimensions + the motion region polygon(s) in image space, shipped
+		// with each motion event so the live view can draw a motion-debug overlay
+		// (the boxes below + the detection region).
+		var imageCols, imageRows int
+		var regionPolygons [][]map[string]int
+		if config.Region != nil {
+			for _, polygon := range config.Region.Polygon {
+				var pts []map[string]int
+				for _, c := range polygon.Coordinates {
+					pts = append(pts, map[string]int{
+						"x": int(c.X * baseWidthRatio),
+						"y": int(c.Y * baseHeightRatio),
+					})
+				}
+				if len(pts) > 0 {
+					regionPolygons = append(regionPolygons, pts)
+				}
+			}
+		}
+
 		img := imageArray[0]
-		var coordinatesToCheck []int
+		var coordinatesPerRegion [][]int
+		totalCoordinates := 0
 		if img != nil {
 			bounds := img.Bounds()
 			rows := bounds.Dy()
 			cols := bounds.Dx()
+			imageCols = cols
+			imageRows = rows
 
-			// Make fixed size array of uinty8
+			// Build a SEPARATE coordinate list per region. Motion is evaluated
+			// independently per region: pixels are NOT shared between regions, so
+			// the threshold must be exceeded within a single region to trigger.
+			coordinatesPerRegion = make([][]int, len(polyObjects))
 			for y := 0; y < rows; y++ {
 				for x := 0; x < cols; x++ {
-					for _, poly := range polyObjects {
-						point := geo.NewPoint(float64(x), float64(y))
+					point := geo.NewPoint(float64(x), float64(y))
+					for idx, poly := range polyObjects {
 						if poly.Contains(point) {
-							coordinatesToCheck = append(coordinatesToCheck, y*cols+x)
+							coordinatesPerRegion[idx] = append(coordinatesPerRegion[idx], y*cols+x)
+							totalCoordinates++
 						}
 					}
 				}
@@ -121,7 +163,7 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 		}
 
 		// If no region is set, we'll skip the motion detection
-		if len(coordinatesToCheck) > 0 {
+		if totalCoordinates > 0 {
 
 			// Start the motion detection
 			i := 0
@@ -143,15 +185,19 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 				// We'll validate those conditions and if not valid we'll not do anything.
 				detectMotion, err := conditions.Validate(loc, configuration)
 				if !detectMotion && err != nil {
-					log.Log.Debug("computervision.main.ProcessMotion(): " + err.Error() + ".")
+					log.Debug("computervision.main.ProcessMotion(): " + err.Error() + ".")
 				}
 
-				if config.Capture.Motion != "false" {
+				// Run detection when motion is enabled, OR when we're in continuous
+				// mode with a region: there config.Capture.Motion (the motion-RECORDING
+				// switch) is irrelevant, so the configured region alone is enough to
+				// emit motion events for the live-view overlay.
+				if config.Capture.Motion != "false" || continuousMode {
 
 					if detectMotion {
 
 						// Remember additional information about the result of findmotion
-						isPixelChangeThresholdReached, changesToReturn, motionRectangle = FindMotion(imageArray, coordinatesToCheck, pixelThreshold)
+						isPixelChangeThresholdReached, changesToReturn, motionRectangle, motionRectangles = FindMotion(imageArray, coordinatesPerRegion, pixelThreshold)
 						if isPixelChangeThresholdReached {
 
 							// If offline mode is disabled, send a message to the hub
@@ -164,6 +210,24 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 												DeviceId: configuration.Config.Key,
 												Value: map[string]interface{}{
 													"timestamp": time.Now().Unix(),
+													// Live-view motion-debug overlay data. The boxes/region
+													// are in the MOTION frame's pixel space (width/height =
+													// the stream motion ran on, i.e. the sub stream when
+													// set). mainWidth/mainHeight are the MAIN stream's
+													// dimensions so the live view can extrapolate the
+													// boxes/region onto the high-res main view it shows —
+													// we know both, so no guessing from the <video> element.
+													"width":      imageCols,
+													"height":     imageRows,
+													"mainWidth":  configuration.Config.Capture.IPCamera.Width,
+													"mainHeight": configuration.Config.Capture.IPCamera.Height,
+													"regions":    motionRectangles,
+													"polygon":    regionPolygons, // Motion sensitivity = the pixel-change threshold that must
+													// be exceeded before motion triggers. The live view renders
+													// a reference square of sqrt(threshold) px (in this MOTION
+													// frame's pixel space) so the user can visually gauge how
+													// large a moving object must be before it is detected.
+													"pixelChangeThreshold": pixelThreshold,
 												},
 											},
 										}
@@ -171,7 +235,7 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 										if err == nil {
 											mqttClient.Publish("kerberos/hub/"+hubKey, 2, false, payload)
 										} else {
-											log.Log.Info("computervision.main.ProcessMotion(): failed to package MQTT message: " + err.Error())
+											log.Info("computervision.main.ProcessMotion(): failed to package MQTT message: " + err.Error())
 										}
 									} else {
 										mqttClient.Publish("kerberos/agent/"+deviceKey, 2, false, "motion")
@@ -179,13 +243,20 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 								}
 							}
 
-							if config.Capture.Recording != "false" {
+							// Trigger motion-based recording — but NOT in continuous mode:
+							// there the recorder runs the continuous branch and does not
+							// drain HandleMotion, so a (blocking) send would hang the motion
+							// loop. In continuous mode we only publish the motion event above
+							// for the live-view overlay.
+							if config.Capture.Recording != "false" && !continuousMode {
 								dataToPass := models.MotionDataPartial{
 									Timestamp:       time.Now().Unix(),
 									NumberOfChanges: changesToReturn,
 									Rectangle:       motionRectangle,
 								}
-								communication.HandleMotion <- dataToPass //Save data to the channel
+								if !communication.TrySendMotion(dataToPass) {
+									log.Warn("computervision.main.ProcessMotion(): motion channel unavailable or full, dropping recording trigger")
+								}
 							}
 						}
 					}
@@ -202,20 +273,76 @@ func ProcessMotion(motionCursor *packets.QueueCursor, configuration *models.Conf
 		}
 	}
 
-	log.Log.Debug("computervision.main.ProcessMotion(): stop the motion detection.")
+	log.Debug("computervision.main.ProcessMotion(): stop the motion detection.")
 }
 
-func FindMotion(imageArray [3]*image.Gray, coordinatesToCheck []int, pixelChangeThreshold int) (thresholdReached bool, changesDetected int, motionRectangle models.MotionRectangle) {
+func FindMotion(imageArray [3]*image.Gray, coordinatesPerRegion [][]int, pixelChangeThreshold int) (thresholdReached bool, changesDetected int, motionRectangle models.MotionRectangle, motionRectangles []models.MotionRectangle) {
 	image1 := imageArray[0]
 	image2 := imageArray[1]
 	image3 := imageArray[2]
 	threshold := 60
-	changes, motionRectangle := AbsDiffBitwiseAndThreshold(image1, image2, image3, threshold, coordinatesToCheck)
-	return changes > pixelChangeThreshold, changes, motionRectangle
+
+	// Evaluate each region INDEPENDENTLY — pixels are not shared between regions,
+	// so the threshold must be exceeded within a single region to trigger. The
+	// overall rectangle (recording metadata) and the per-cluster rectangles
+	// (live-view overlay) are aggregated across all regions.
+	var combinedRectangles []models.MotionRectangle
+	var overall models.MotionRectangle
+	haveOverall := false
+	totalChanges := 0
+	for _, coordinatesToCheck := range coordinatesPerRegion {
+		if len(coordinatesToCheck) == 0 {
+			continue
+		}
+		changes, rect, rects := AbsDiffBitwiseAndThreshold(image1, image2, image3, threshold, coordinatesToCheck)
+		totalChanges += changes
+		if changes > pixelChangeThreshold {
+			thresholdReached = true
+		}
+		combinedRectangles = append(combinedRectangles, rects...)
+		if changes > 0 {
+			if !haveOverall {
+				overall = rect
+				haveOverall = true
+			} else {
+				overall = unionMotionRectangle(overall, rect)
+			}
+		}
+	}
+
+	return thresholdReached, totalChanges, overall, combinedRectangles
 }
 
-func AbsDiffBitwiseAndThreshold(img1 *image.Gray, img2 *image.Gray, img3 *image.Gray, threshold int, coordinatesToCheck []int) (int, models.MotionRectangle) {
+// unionMotionRectangle returns the smallest rectangle that contains both a and b.
+func unionMotionRectangle(a, b models.MotionRectangle) models.MotionRectangle {
+	minX := a.X
+	if b.X < minX {
+		minX = b.X
+	}
+	minY := a.Y
+	if b.Y < minY {
+		minY = b.Y
+	}
+	maxX := a.X + a.Width
+	if b.X+b.Width > maxX {
+		maxX = b.X + b.Width
+	}
+	maxY := a.Y + a.Height
+	if b.Y+b.Height > maxY {
+		maxY = b.Y + b.Height
+	}
+	return models.MotionRectangle{
+		X:      minX,
+		Y:      minY,
+		Width:  maxX - minX,
+		Height: maxY - minY,
+	}
+}
+
+func AbsDiffBitwiseAndThreshold(img1 *image.Gray, img2 *image.Gray, img3 *image.Gray, threshold int, coordinatesToCheck []int) (int, models.MotionRectangle, []models.MotionRectangle) {
 	changes := 0
+	cols := img1.Bounds().Dx()
+	rows := img1.Bounds().Dy()
 	var pixelList [][]int
 	for i := 0; i < len(coordinatesToCheck); i++ {
 		pixel := coordinatesToCheck[i]
@@ -224,7 +351,7 @@ func AbsDiffBitwiseAndThreshold(img1 *image.Gray, img2 *image.Gray, img3 *image.
 		if (diff > threshold || diff < -threshold) && (diff2 > threshold || diff2 < -threshold) {
 			changes++
 			// Store the pixel coordinates where the change is detected
-			pixelList = append(pixelList, []int{pixel % img1.Bounds().Dx(), pixel / img1.Bounds().Dx()})
+			pixelList = append(pixelList, []int{pixel % cols, pixel / cols})
 		}
 	}
 
@@ -249,14 +376,141 @@ func AbsDiffBitwiseAndThreshold(img1 *image.Gray, img2 *image.Gray, img3 *image.
 				endY = pixel[1]
 			}
 		}
-		log.Log.Debugf("Rectangle of changes detected: startX: %d, startY: %d, endX: %d, endY: %d", startX, startY, endX, endY)
+		log.WithFields(log.Fields{
+			"component": "computer_vision",
+			"end_x":     endX,
+			"end_y":     endY,
+			"event":     "motion_bounds_detected",
+			"start_x":   startX,
+			"start_y":   startY,
+		}).Debug("Motion bounds detected")
 		motionRectangle = models.MotionRectangle{
 			X:      startX,
 			Y:      startY,
 			Width:  endX - startX,
 			Height: endY - startY,
 		}
-		log.Log.Debugf("Motion rectangle: %+v", motionRectangle)
+		log.WithFields(log.Fields{
+			"component": "computer_vision",
+			"event":     "motion_rectangle_created",
+			"height":    motionRectangle.Height,
+			"width":     motionRectangle.Width,
+			"x":         motionRectangle.X,
+			"y":         motionRectangle.Y,
+		}).Debug("Motion rectangle created")
 	}
-	return changes, motionRectangle
+
+	// Cluster the changed pixels into separate bounding boxes so the live view can
+	// visualise WHERE motion happened (a single overall rectangle is useless when
+	// two objects move in opposite corners). Cheap grid-based connected components.
+	motionRectangles := clusterMotionRectangles(pixelList, cols, rows)
+
+	return changes, motionRectangle, motionRectangles
+}
+
+// clusterMotionRectangles groups the changed-pixel coordinates into a handful of
+// bounding boxes using connected-components on a coarse grid (8-connectivity).
+// It is intentionally lightweight — it runs only when the motion threshold is
+// reached and the boxes are meant for a debug overlay, not precise detection.
+func clusterMotionRectangles(pixelList [][]int, cols, rows int) []models.MotionRectangle {
+	if len(pixelList) == 0 || cols <= 0 || rows <= 0 {
+		return nil
+	}
+
+	// ~40 cells across the longest side keeps the grid small (cheap to cluster)
+	// while still separating distinct motion blobs.
+	const gridDim = 40
+	cellW := cols / gridDim
+	if cellW < 1 {
+		cellW = 1
+	}
+	cellH := rows / gridDim
+	if cellH < 1 {
+		cellH = 1
+	}
+	gCols := (cols + cellW - 1) / cellW
+	gRows := (rows + cellH - 1) / cellH
+
+	grid := make([]bool, gCols*gRows)
+	for _, p := range pixelList {
+		cx := p[0] / cellW
+		cy := p[1] / cellH
+		if cx >= 0 && cx < gCols && cy >= 0 && cy < gRows {
+			grid[cy*gCols+cx] = true
+		}
+	}
+
+	visited := make([]bool, gCols*gRows)
+	var rectangles []models.MotionRectangle
+	const maxBoxes = 12
+	stack := make([][2]int, 0, 64)
+
+	for cy := 0; cy < gRows; cy++ {
+		for cx := 0; cx < gCols; cx++ {
+			idx := cy*gCols + cx
+			if !grid[idx] || visited[idx] {
+				continue
+			}
+
+			// Flood-fill this component (8-connectivity) and track its extent.
+			minX, minY, maxX, maxY := cx, cy, cx, cy
+			cellCount := 0
+			stack = stack[:0]
+			stack = append(stack, [2]int{cx, cy})
+			visited[idx] = true
+			for len(stack) > 0 {
+				cur := stack[len(stack)-1]
+				stack = stack[:len(stack)-1]
+				ccx, ccy := cur[0], cur[1]
+				cellCount++
+				if ccx < minX {
+					minX = ccx
+				}
+				if ccy < minY {
+					minY = ccy
+				}
+				if ccx > maxX {
+					maxX = ccx
+				}
+				if ccy > maxY {
+					maxY = ccy
+				}
+				for dy := -1; dy <= 1; dy++ {
+					for dx := -1; dx <= 1; dx++ {
+						nx, ny := ccx+dx, ccy+dy
+						if nx < 0 || ny < 0 || nx >= gCols || ny >= gRows {
+							continue
+						}
+						nIdx := ny*gCols + nx
+						if grid[nIdx] && !visited[nIdx] {
+							visited[nIdx] = true
+							stack = append(stack, [2]int{nx, ny})
+						}
+					}
+				}
+			}
+
+			// Skip single-cell specks (sensor noise) unless it's the only motion.
+			if cellCount < 2 && len(pixelList) > 4 {
+				continue
+			}
+
+			x := minX * cellW
+			y := minY * cellH
+			w := (maxX - minX + 1) * cellW
+			h := (maxY - minY + 1) * cellH
+			if x+w > cols {
+				w = cols - x
+			}
+			if y+h > rows {
+				h = rows - y
+			}
+			rectangles = append(rectangles, models.MotionRectangle{X: x, Y: y, Width: w, Height: h})
+			if len(rectangles) >= maxBoxes {
+				return rectangles
+			}
+		}
+	}
+
+	return rectangles
 }

machinery/src/conditions/timewindow.go

@@ -3,8 +3,8 @@ package conditions
 import (
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 )
 
 func IsWithinTimeInterval(loc *time.Location, configuration *models.Configuration) (enabled bool) {
@@ -27,9 +27,9 @@ func IsWithinTimeInterval(loc *time.Location, configuration *models.Configuratio
 				currentTimeInSeconds := hour*60*60 + minute*60 + second
 				if (currentTimeInSeconds >= start1 && currentTimeInSeconds <= end1) ||
 					(currentTimeInSeconds >= start2 && currentTimeInSeconds <= end2) {
-					log.Log.Debug("conditions.timewindow.IsWithinTimeInterval(): time interval valid, enabling recording.")
+					log.Debug("conditions.timewindow.IsWithinTimeInterval(): time interval valid, enabling recording.")
 				} else {
-					log.Log.Info("conditions.timewindow.IsWithinTimeInterval(): time interval not valid, disabling recording.")
+					log.Info("conditions.timewindow.IsWithinTimeInterval(): time interval not valid, disabling recording.")
 					enabled = false
 				}
 			}

machinery/src/conditions/uri.go

@@ -3,13 +3,26 @@ package conditions
 import (
 	"bytes"
 	"crypto/tls"
-	"fmt"
+	"encoding/json"
+	"io"
 	"net/http"
 	"os"
 	"time"
 
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
+)
+
+const conditionHTTPTimeout = 10 * time.Second
+
+var (
+	conditionHTTPClient         = &http.Client{Timeout: conditionHTTPTimeout}
+	conditionInsecureHTTPClient = &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // #nosec G402 -- explicit operator opt-in
+		},
+		Timeout: conditionHTTPTimeout,
+	}
 )
 
 func IsValidUriResponse(configuration *models.Configuration) (enabled bool) {
@@ -17,41 +30,66 @@ func IsValidUriResponse(configuration *models.Configuration) (enabled bool) {
 	conditionURI := config.ConditionURI
 	enabled = true
 	if conditionURI != "" {
-
-		// We will send a POST request to the conditionURI, and expect a 200 response.
-		// In the payload we will send some information, so the other end can decide
-		// if it should enable or disable recording.
-
-		var client *http.Client
+		client := conditionHTTPClient
 		if os.Getenv("AGENT_TLS_INSECURE") == "true" {
-			tr := &http.Transport{
-				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-			}
-			client = &http.Client{Transport: tr}
-		} else {
-			client = &http.Client{}
+			client = conditionInsecureHTTPClient
 		}
 
-		var object = fmt.Sprintf(`{
-			"camera_id" : "%s",
-			"camera_name" : "%s",
-			"site_id" : "%s",
-			"hub_key" : "%s",
-			"timestamp" : "%s",
-		}`, config.Key, config.FriendlyName, config.HubSite, config.HubKey, time.Now().Format("2006-01-02 15:04:05"))
+		payload := struct {
+			CameraID   string `json:"camera_id"`
+			CameraName string `json:"camera_name"`
+			SiteID     string `json:"site_id"`
+			HubKey     string `json:"hub_key"`
+			Timestamp  string `json:"timestamp"`
+		}{
+			CameraID:   config.Key,
+			CameraName: config.FriendlyName,
+			SiteID:     config.HubSite,
+			HubKey:     config.HubKey,
+			Timestamp:  time.Now().Format("2006-01-02 15:04:05"),
+		}
+		jsonBody, err := json.Marshal(payload)
+		if err != nil {
+			log.WithError(err).WithFields(log.Fields{
+				"component": "conditions/uri",
+				"event":     "request_encoding_failed",
+			}).Error("Failed to encode condition request")
+			return false
+		}
 
-		var jsonStr = []byte(object)
-		buffy := bytes.NewBuffer(jsonStr)
-		req, _ := http.NewRequest("POST", conditionURI, buffy)
+		req, err := http.NewRequest(http.MethodPost, conditionURI, bytes.NewReader(jsonBody))
+		if err != nil {
+			log.WithError(err).WithFields(log.Fields{
+				"component": "conditions/uri",
+				"event":     "request_creation_failed",
+			}).Error("Failed to create condition request")
+			return false
+		}
 		req.Header.Set("Content-Type", "application/json")
 		resp, err := client.Do(req)
 		if resp != nil {
+			_, _ = io.Copy(io.Discard, resp.Body)
 			resp.Body.Close()
 		}
-		if err == nil && resp.StatusCode == 200 {
-			log.Log.Info("conditions.uri.IsValidUriResponse(): response 200, enabling recording.")
+		if err == nil && resp != nil && resp.StatusCode == http.StatusOK {
+			log.WithFields(log.Fields{
+				"component":   "conditions/uri",
+				"event":       "recording_enabled",
+				"status_code": resp.StatusCode,
+			}).Info("Condition request enabled recording")
 		} else {
-			log.Log.Info("conditions.uri.IsValidUriResponse(): response not 200, disabling recording.")
+			if err != nil {
+				log.WithError(err).WithFields(log.Fields{
+					"component": "conditions/uri",
+					"event":     "request_failed",
+				}).Error("Condition request failed")
+			} else {
+				log.WithFields(log.Fields{
+					"component":   "conditions/uri",
+					"event":       "recording_disabled",
+					"status_code": resp.StatusCode,
+				}).Info("Condition request disabled recording")
+			}
 			enabled = false
 		}
 	}

machinery/src/conditions/uri_test.go

@@ -0,0 +1,63 @@
+package conditions
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+
+	"github.com/kerberos-io/agent/machinery/src/models"
+)
+
+func TestIsValidUriResponseReusesClientAndEncodesPayload(t *testing.T) {
+	var requests atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		requests.Add(1)
+		if request.Method != http.MethodPost {
+			t.Errorf("method = %s, want POST", request.Method)
+		}
+		if got := request.Header.Get("Content-Type"); got != "application/json" {
+			t.Errorf("Content-Type = %q, want application/json", got)
+		}
+		var payload struct {
+			CameraID   string `json:"camera_id"`
+			CameraName string `json:"camera_name"`
+		}
+		if err := json.NewDecoder(request.Body).Decode(&payload); err != nil {
+			t.Errorf("decode request: %v", err)
+		}
+		if payload.CameraID != "camera-1" || payload.CameraName != `Front "Door"` {
+			t.Errorf("payload = %+v", payload)
+		}
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer server.Close()
+
+	configuration := &models.Configuration{}
+	configuration.Config.ConditionURI = server.URL
+	configuration.Config.Key = "camera-1"
+	configuration.Config.FriendlyName = `Front "Door"`
+
+	for i := 0; i < 2; i++ {
+		if !IsValidUriResponse(configuration) {
+			t.Fatal("IsValidUriResponse() = false, want true")
+		}
+	}
+	if got := requests.Load(); got != 2 {
+		t.Fatalf("requests = %d, want 2", got)
+	}
+}
+
+func TestIsValidUriResponseRejectsNonOK(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		http.Error(w, "unavailable", http.StatusServiceUnavailable)
+	}))
+	defer server.Close()
+
+	configuration := &models.Configuration{}
+	configuration.Config.ConditionURI = server.URL
+	if IsValidUriResponse(configuration) {
+		t.Fatal("IsValidUriResponse() = true, want false")
+	}
+}

machinery/src/config/main.go

@@ -13,11 +13,36 @@ import (
 
 	"github.com/InVisionApp/conjungo"
 	"github.com/kerberos-io/agent/machinery/src/database"
-	"github.com/kerberos-io/agent/machinery/src/log"
 	"github.com/kerberos-io/agent/machinery/src/models"
+	log "github.com/sirupsen/logrus"
 	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
 )
 
+const factoryConfigRetryDelay = 5 * time.Second
+
+func newFactoryConfigReadContext() (context.Context, context.CancelFunc) {
+	return context.WithTimeout(context.Background(), database.TIMEOUT)
+}
+
+func isRetryableFactoryConfigReadError(err error) bool {
+	return errors.Is(err, context.DeadlineExceeded) || mongo.IsTimeout(err) || mongo.IsNetworkError(err)
+}
+
+func readFactoryConfig(collection *mongo.Collection, filter bson.M, destination *models.Config) error {
+	for {
+		ctx, cancel := newFactoryConfigReadContext()
+		err := collection.FindOne(ctx, filter).Decode(destination)
+		cancel()
+		if !isRetryableFactoryConfigReadError(err) {
+			return err
+		}
+
+		log.Warn("Factory configuration read timed out or lost its database connection; retrying.")
+		time.Sleep(factoryConfigRetryDelay)
+	}
+}
+
 // ReadUserConfig Reads the user configuration of the Kerberos Open Source instance.
 // This will return a models.User struct including the username, password,
 // selected language, and if the installation was completed or not.
@@ -25,14 +50,14 @@ func ReadUserConfig(configDirectory string) (userConfig models.User) {
 	for {
 		jsonFile, err := os.Open(configDirectory + "/data/config/user.json")
 		if err != nil {
-			log.Log.Error("Config file is not found " + configDirectory + "/data/config/user.json, trying again in 5s: " + err.Error())
+			log.Error("Config file is not found " + configDirectory + "/data/config/user.json, trying again in 5s: " + err.Error())
 			time.Sleep(5 * time.Second)
 		} else {
-			log.Log.Info("Successfully Opened user.json")
+			log.Info("Successfully Opened user.json")
 			byteValue, _ := ioutil.ReadAll(jsonFile)
 			err = json.Unmarshal(byteValue, &userConfig)
 			if err != nil {
-				log.Log.Error("JSON file not valid: " + err.Error())
+				log.Error("JSON file not valid: " + err.Error())
 			} else {
 				jsonFile.Close()
 				break
@@ -63,21 +88,15 @@ func OpenConfig(configDirectory string, configuration *models.Configuration) {
 		collection := db.Collection("configuration")
 
 		var globalConfig models.Config
-		res := collection.FindOne(context.Background(), bson.M{
+		err := readFactoryConfig(collection, bson.M{
 			"type": "global",
-		})
-
-		if res.Err() != nil {
-			log.Log.Error("Could not find global configuration, using default configuration.")
-			panic("Could not find global configuration, using default configuration.")
-		}
-		err := res.Decode(&globalConfig)
+		}, &globalConfig)
 		if err != nil {
-			log.Log.Error("Could not find global configuration, using default configuration.")
+			log.Error("Could not find global configuration, using default configuration.")
 			panic("Could not find global configuration, using default configuration.")
 		}
 		if globalConfig.Type != "global" {
-			log.Log.Error("Could not find global configuration, might missed the mongodb connection.")
+			log.Error("Could not find global configuration, might missed the mongodb connection.")
 			panic("Could not find global configuration, might missed the mongodb connection.")
 		}
 
@@ -85,21 +104,16 @@ func OpenConfig(configDirectory string, configuration *models.Configuration) {
 
 		var customConfig models.Config
 		deploymentName := os.Getenv("DEPLOYMENT_NAME")
-		res = collection.FindOne(context.Background(), bson.M{
+		err = readFactoryConfig(collection, bson.M{
 			"type": "config",
 			"name": deploymentName,
-		})
-		if res.Err() != nil {
-			log.Log.Error("Could not find configuration for " + deploymentName + ", using global configuration.")
-		}
-		err = res.Decode(&customConfig)
+		}, &customConfig)
 		if err != nil {
-			log.Log.Error("Could not find configuration for " + deploymentName + ", using global configuration.")
-		}
-
-		if customConfig.Type != "config" {
-			log.Log.Error("Could not find custom configuration, might missed the mongodb connection.")
-			panic("Could not find custom configuration, might missed the mongodb connection.")
+			log.Error("Could not find configuration for " + deploymentName + ", using global configuration.")
+			customConfig = models.Config{}
+		} else if customConfig.Type != "config" {
+			log.Error("Custom configuration has an invalid type, using global configuration.")
+			customConfig = models.Config{}
 		}
 		configuration.CustomConfig = customConfig
 
@@ -166,21 +180,29 @@ func OpenConfig(configDirectory string, configuration *models.Configuration) {
 
 		// Open device config
 		for {
-			jsonFile, err := os.Open(configDirectory + "/data/config/config.json")
+			configPath := configDirectory + "/data/config/config.json"
+			jsonFile, err := os.Open(configPath)
 			if err != nil {
-				log.Log.Error("Config file is not found " + configDirectory + "/data/config/config.json" + ", trying again in 5s.")
+				log.WithError(err).WithFields(log.Fields{
+					"component":      "config",
+					"event":          "configuration_open_failed",
+					"retry_delay_ms": 5000,
+				}).Warn("Configuration file unavailable; retrying")
 				time.Sleep(5 * time.Second)
 			} else {
-				log.Log.Info("Successfully Opened config.json from " + configuration.Name)
+				log.WithFields(log.Fields{
+					"component": "config",
+					"event":     "configuration_opened",
+				}).Info("Configuration file opened")
 				byteValue, _ := ioutil.ReadAll(jsonFile)
 				err = json.Unmarshal(byteValue, &configuration.Config)
 				jsonFile.Close()
 				if err != nil {
-					log.Log.Error("JSON file not valid: " + err.Error())
+					log.Error("JSON file not valid: " + err.Error())
 				} else {
 					err = json.Unmarshal(byteValue, &configuration.CustomConfig)
 					if err != nil {
-						log.Log.Error("JSON file not valid: " + err.Error())
+						log.Error("JSON file not valid: " + err.Error())
 					} else {
 						break
 					}
@@ -293,6 +315,15 @@ func initConfigPointers(config *models.Config) {
 func applyAgentEnvVars(configuration *models.Configuration, prefix string, applyDefaults bool) {
 	environmentVariables := os.Environ()
 
+	if prefix == "" {
+		if deploymentName := os.Getenv("DEPLOYMENT_NAME"); deploymentName != "" {
+			configuration.Config.Name = deploymentName
+			if configuration.Config.FriendlyName == "" {
+				configuration.Config.FriendlyName = deploymentName
+			}
+		}
+	}
+
 	// Initialize the configuration for some new fields.
 	if configuration.Config.KStorageSecondary == nil {
 		configuration.Config.KStorageSecondary = &models.KStorage{}
@@ -327,6 +358,12 @@ func applyAgentEnvVars(configuration *models.Configuration, prefix string, apply
 					configuration.Config.MaxDirectorySize = size
 				}
 				break
+			case "AGENT_AUTO_CLEAN_MIN_FREE_SPACE":
+				size, err := strconv.ParseInt(value, 10, 64)
+				if err == nil {
+					configuration.Config.MinFreeSpace = size
+				}
+				break
 
 			/* Camera configuration */
 			case "AGENT_CAPTURE_IPCAMERA_RTSP":
@@ -357,6 +394,9 @@ func applyAgentEnvVars(configuration *models.Configuration, prefix string, apply
 			case "AGENT_CAPTURE_IPCAMERA_ONVIF_PASSWORD":
 				configuration.Config.Capture.IPCamera.ONVIFPassword = value
 				break
+			case "AGENT_CAPTURE_IPCAMERA_ONVIF_CHANNEL":
+				configuration.Config.Capture.IPCamera.ONVIFChannel, _ = strconv.Atoi(value)
+				break
 
 			/* Recording mode */
 			case "AGENT_CAPTURE_RECORDING":
@@ -371,6 +411,9 @@ func applyAgentEnvVars(configuration *models.Configuration, prefix string, apply
 			case "AGENT_CAPTURE_MOTION":
 				configuration.Config.Capture.Motion = value
 				break
+			case "AGENT_CAPTURE_ONVIF_MOTION":
+				configuration.Config.Capture.ONVIFMotion = value
+				break
 			case "AGENT_CAPTURE_SNAPSHOTS":
 				configuration.Config.Capture.Snapshots = value
 				break
@@ -395,7 +438,7 @@ func applyAgentEnvVars(configuration *models.Configuration, prefix string, apply
 			case "AGENT_CAPTURE_PIXEL_CHANGE":
 				count, err := strconv.Atoi(value)
 				if err == nil {
-					configuration.Config.Capture.PixelChangeThreshold = count
+					configuration.Config.Capture.PixelChangeThreshold = &count
 				}
 				break
 			case "AGENT_CAPTURE_FRAGMENTED":
@@ -572,6 +615,9 @@ func applyAgentEnvVars(configuration *models.Configuration, prefix string, apply
 			case "AGENT_KERBEROSVAULT_DIRECTORY":
 				configuration.Config.KStorage.Directory = value
 				break
+			case "AGENT_KERBEROSVAULT_CUSTOM_HEADERS":
+				configuration.Config.KStorage.CustomHeaders = value
+				break
 
 			/* Retry policy and timeout */
 			case "AGENT_KERBEROSVAULT_MAX_RETRIES":
@@ -603,6 +649,9 @@ func applyAgentEnvVars(configuration *models.Configuration, prefix string, apply
 			case "AGENT_KERBEROSVAULT_SECONDARY_DIRECTORY":
 				configuration.Config.KStorageSecondary.Directory = value
 				break
+			case "AGENT_KERBEROSVAULT_SECONDARY_CUSTOM_HEADERS":
+				configuration.Config.KStorageSecondary.CustomHeaders = value
+				break
 
 			/* When storing in dropbox */
 			case "AGENT_DROPBOX_ACCESS_TOKEN":
@@ -642,6 +691,16 @@ func applyAgentEnvVars(configuration *models.Configuration, prefix string, apply
 		}
 	}
 
+	// Motion sensitivity historically used 0 to mean "use the default". Preserve
+	// that behaviour for configurations created before this field became a
+	// pointer, and also recover invalid negative values. Only apply this to the
+	// effective configuration so missing values can still be inherited between
+	// the separate global and custom layers.
+	if applyDefaults && (configuration.Config.Capture.PixelChangeThreshold == nil || *configuration.Config.Capture.PixelChangeThreshold <= 0) {
+		defaultPixelChangeThreshold := 150
+		configuration.Config.Capture.PixelChangeThreshold = &defaultPixelChangeThreshold
+	}
+
 	// Signing is a new feature, so if empty we set default values. Only applied
 	// for the effective configuration (applyDefaults), not for the separate
 	// global/custom views.
@@ -668,12 +727,12 @@ func SaveConfig(configDirectory string, config models.Config, configuration *mod
 			return err
 		}
 
-		if communication.CameraConnected {
+		if communication.CameraConnected.Load() {
 			select {
 			case communication.HandleBootstrap <- "restart":
-				log.Log.Info("config.main.SaveConfig(): update config, restart agent.")
+				log.Info("config.main.SaveConfig(): update config, restart agent.")
 			case <-time.After(1 * time.Second):
-				log.Log.Info("config.main.SaveConfig(): update config, restart agent.")
+				log.Info("config.main.SaveConfig(): update config, restart agent.")
 			}
 		}
 

machinery/src/config/main_test.go

@@ -0,0 +1,142 @@
+package config
+
+import (
+	"context"
+	"errors"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/kerberos-io/agent/machinery/src/database"
+	"github.com/kerberos-io/agent/machinery/src/models"
+	"go.mongodb.org/mongo-driver/mongo"
+)
+
+func TestApplyAgentEnvVarsDeploymentName(t *testing.T) {
+	tests := []struct {
+		name           string
+		prefix         string
+		deploymentName string
+		agentName      string
+		initialName    string
+		initialDisplay string
+		wantName       string
+		wantDisplay    string
+	}{
+		{name: "deployment fallback", deploymentName: "camera-1", wantName: "camera-1", wantDisplay: "camera-1"},
+		{name: "explicit display name", deploymentName: "camera-1", agentName: "Front Door", wantName: "camera-1", wantDisplay: "Front Door"},
+		{name: "existing display name", deploymentName: "camera-1", initialDisplay: "Front Door", wantName: "camera-1", wantDisplay: "Front Door"},
+		{name: "deployment overrides bundled identity", deploymentName: "camera-1", initialName: "default", wantName: "camera-1", wantDisplay: "camera-1"},
+		{name: "no deployment", initialName: "existing", initialDisplay: "Existing Camera", wantName: "existing", wantDisplay: "Existing Camera"},
+		{name: "standalone display name", agentName: "Front Door", wantDisplay: "Front Door"},
+		{name: "global layer excludes deployment", prefix: "GLOBAL_", deploymentName: "camera-1"},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			t.Setenv("DEPLOYMENT_NAME", test.deploymentName)
+			t.Setenv(test.prefix+"AGENT_NAME", test.agentName)
+			if test.agentName == "" {
+				if err := os.Unsetenv(test.prefix + "AGENT_NAME"); err != nil {
+					t.Fatal(err)
+				}
+			}
+			configuration := &models.Configuration{Config: models.Config{
+				Name:         test.initialName,
+				FriendlyName: test.initialDisplay,
+			}}
+			initConfigPointers(&configuration.Config)
+
+			applyAgentEnvVars(configuration, test.prefix, false)
+
+			if got := configuration.Config.Name; got != test.wantName {
+				t.Errorf("Name = %q, want %q", got, test.wantName)
+			}
+			if got := configuration.Config.FriendlyName; got != test.wantDisplay {
+				t.Errorf("FriendlyName = %q, want %q", got, test.wantDisplay)
+			}
+		})
+	}
+}
+
+func TestApplyAgentEnvVarsPixelChangeThresholdDefault(t *testing.T) {
+	tests := []struct {
+		name      string
+		threshold *int
+		want      int
+	}{
+		{name: "missing", want: 150},
+		{name: "legacy zero", threshold: intPointer(0), want: 150},
+		{name: "negative", threshold: intPointer(-1), want: 150},
+		{name: "positive", threshold: intPointer(275), want: 275},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			configuration := &models.Configuration{}
+			configuration.Config.Capture.PixelChangeThreshold = test.threshold
+
+			applyAgentEnvVars(configuration, "TEST_", true)
+
+			if configuration.Config.Capture.PixelChangeThreshold == nil {
+				t.Fatal("PixelChangeThreshold is nil after applying defaults")
+			}
+			if got := *configuration.Config.Capture.PixelChangeThreshold; got != test.want {
+				t.Fatalf("PixelChangeThreshold = %d, want %d", got, test.want)
+			}
+		})
+	}
+}
+
+func TestApplyAgentEnvVarsVaultCustomHeaders(t *testing.T) {
+	t.Setenv("AGENT_KERBEROSVAULT_CUSTOM_HEADERS", `{"site_id":"site-1"}`)
+	t.Setenv("AGENT_KERBEROSVAULT_SECONDARY_CUSTOM_HEADERS", `{"site_id":"site-2"}`)
+	configuration := &models.Configuration{}
+	initConfigPointers(&configuration.Config)
+
+	applyAgentEnvVars(configuration, "", false)
+
+	if got := configuration.Config.KStorage.CustomHeaders; got != `{"site_id":"site-1"}` {
+		t.Fatalf("primary custom headers = %q", got)
+	}
+	if got := configuration.Config.KStorageSecondary.CustomHeaders; got != `{"site_id":"site-2"}` {
+		t.Fatalf("secondary custom headers = %q", got)
+	}
+}
+
+func intPointer(value int) *int {
+	return &value
+}
+
+func TestNewFactoryConfigReadContextUsesDatabaseTimeout(t *testing.T) {
+	ctx, cancel := newFactoryConfigReadContext()
+	defer cancel()
+
+	deadline, ok := ctx.Deadline()
+	if !ok {
+		t.Fatal("expected read context deadline")
+	}
+
+	remaining := time.Until(deadline)
+	if remaining <= 0 {
+		t.Fatalf("deadline already expired: %v", remaining)
+	}
+	if remaining > database.TIMEOUT {
+		t.Fatalf("remaining deadline = %v, want <= %v", remaining, database.TIMEOUT)
+	}
+	if remaining < database.TIMEOUT-time.Second {
+		t.Fatalf("remaining deadline = %v, want close to %v", remaining, database.TIMEOUT)
+	}
+}
+
+func TestFactoryConfigRetryableErrors(t *testing.T) {
+	if !isRetryableFactoryConfigReadError(context.DeadlineExceeded) {
+		t.Fatal("context deadline should be retryable")
+	}
+	if isRetryableFactoryConfigReadError(mongo.ErrNoDocuments) {
+		t.Fatal("missing configuration should not retry")
+	}
+	if isRetryableFactoryConfigReadError(errors.New("invalid BSON")) {
+		t.Fatal("decode errors should not retry")
+	}
+}

machinery/src/database/main.go

@@ -7,6 +7,7 @@ import (
 	"sync"
 	"time"
 
+	log "github.com/sirupsen/logrus"
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
 )
@@ -44,18 +45,34 @@ func New() *DB {
 		// We can also apply the complete URI
 		// e.g. "mongodb+srv://<username>:<password>@kerberos-hub.shhng.mongodb.net/?retryWrites=true&w=majority&appName=kerberos-hub"
 		if mongodbURI != "" {
+			log.WithFields(log.Fields{
+				"component":      "database",
+				"database":       DatabaseName,
+				"event":          "client_configuring",
+				"uri_configured": true,
+			}).Debug("Configuring MongoDB client")
 			serverAPI := options.ServerAPI(options.ServerAPIVersion1)
 			opts := options.Client().ApplyURI(mongodbURI).SetServerAPIOptions(serverAPI)
 
 			// Create a new client and connect to the server
 			client, err := mongo.Connect(ctx, opts)
 			if err != nil {
-				fmt.Printf("Error setting up mongodb connection: %+v\n", err)
-				os.Exit(1)
+				log.WithError(err).WithFields(log.Fields{
+					"component": "database",
+					"database":  DatabaseName,
+					"event":     "client_configuration_failed",
+				}).Fatal("Failed to configure MongoDB client")
 			}
 			_instance.Client = client
 
 		} else {
+			log.WithFields(log.Fields{
+				"component":             "database",
+				"database":              DatabaseName,
+				"event":                 "client_configuring",
+				"replicaset_configured": replicaset != "",
+				"uri_configured":        false,
+			}).Debug("Configuring MongoDB client")
 
 			// New MongoDB driver
 			mongodbURI := fmt.Sprintf("mongodb://%s:%s@%s", username, password, host)
@@ -69,8 +86,11 @@ func New() *DB {
 				Password:      password,
 			}))
 			if err != nil {
-				fmt.Printf("Error setting up mongodb connection: %+v\n", err)
-				os.Exit(1)
+				log.WithError(err).WithFields(log.Fields{
+					"component": "database",
+					"database":  DatabaseName,
+					"event":     "client_configuration_failed",
+				}).Fatal("Failed to configure MongoDB client")
 			}
 			_instance.Client = client
 		}