Check enabled Hub deployments rather than optional component names

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

.github/workflows/k3d.yaml

@@ -8,6 +8,7 @@ jobs:
   deploy:
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu-22.04, ubuntu-24.04]
         #k3d: [v5.5.2, v5.6.0, v5.7.5]
@@ -29,8 +30,6 @@ jobs:
           kubectl get no
           kubectl get pods -A -o wide
           kubectl get sc
-      - name: Checkout repository
-        uses: actions/checkout@v2
       - name: Install OpenEBS
         run: |
           kubectl apply -f https://openebs.github.io/charts/openebs-operator.yaml
@@ -50,16 +49,11 @@ jobs:
           kubectl wait --for=condition=available deployment/prometheus-operator -n default --timeout=300s
           kubectl get crd
           kubectl get po -A -o wide
-      - name: Install Minio
-        id: install-minio
+      - name: Install Garage S3
+        id: install-garage
         run: |
-          git clone --depth 1 --branch v6.0.1 https://github.com/minio/operator.git && kubectl apply -k operator/
-          kubectl apply -f ./base/minio/minio-tenant-base.yaml
-          echo "Waiting for resources to be created..."
-          sleep 10
-          kubectl wait --for=jsonpath='{.status.readyReplicas}'=1 statefulset/myminio-pool-0 -n minio-tenant --timeout=300s
-          kubectl get po -A -o wide
-          kubectl get po -A -o wide | grep myminio-pool-0-0 | awk '{print $3}' | grep -q '2/2' && echo "myminio-pool-0-0 pod is running with status 2/2" || (echo "myminio-pool-0-0 pod is not running with status 2/2" && exit 1)
+          bash scripts/install-garage.sh
+          bash scripts/check-garage.sh
       - name: Install MongoDB
         id: install-mongodb
         run: |
@@ -109,34 +103,20 @@ jobs:
           kubectl create namespace kerberos-hub
           helm install hub kerberos/hub --values ./base/hub/kerberos-hub-values.yaml -n kerberos-hub --create-namespace
           sleep 10
-          kubectl wait --for=condition=available deployment/admin -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/hub-api -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/hub-cleanup -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/hub-frontend -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/hub-frontend-demo -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/hub-monitor-device -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/hub-reactivate-subscription -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-analysis -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-counting -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-dominantcolor -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-event -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-export -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-monitor -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-notify -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-notify-test -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-sequence -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-sprite -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-throttler -n kerberos-hub --timeout=300s
+          kubectl wait --for=condition=available deployment --all -n kerberos-hub --timeout=300s
           kubectl get pods -A -o wide
+      - name: Collect deployment diagnostics
+        if: failure()
+        run: bash scripts/diagnose-deployment.sh
       - name: Send Slack Notification on Success
         if: success()
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
         run: |
-          curl -X POST -H 'Content-type: application/json' --data "{\"text\": \"✅ K3d deployment successful on ${{ matrix.os }} (${{ matrix.k3d }}) - triggered by ${{ github.actor }}\"}" $SLACK_WEBHOOK_URL
+          bash scripts/notify-slack.sh "✅ K3d deployment successful on ${{ matrix.os }} (${{ matrix.k3d }}) - triggered by ${{ github.actor }}"
       - name: Send Slack Notification on Failure
         if: failure()
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
         run: |
-          curl -X POST -H 'Content-type: application/json' --data "{\"text\": \"❌ K3d deployment failed on ${{ matrix.os }} (${{ matrix.k3d }}) - <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>\"}" $SLACK_WEBHOOK_URL
+          bash scripts/notify-slack.sh "❌ K3d deployment failed on ${{ matrix.os }} (${{ matrix.k3d }}) - <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>"

.github/workflows/kind.yaml

@@ -8,6 +8,7 @@ jobs:
   deploy:
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu-22.04, ubuntu-24.04]
         #kind: [v0.21.0, v0.22.0, v0.23.0, v0.24.0, v0.25.0]
@@ -26,7 +27,11 @@ jobs:
           kubectl get pods -A -o wide
           kubectl get sc
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
+      - name: Check promoted Hub runtime
+        run: |
+          helm repo add kerberos https://charts.kerberos.io
+          bash scripts/check-hub-runtime.sh
       - name: Install OpenEBS
         run: |
           kubectl apply -f https://openebs.github.io/charts/openebs-operator.yaml
@@ -46,16 +51,11 @@ jobs:
           kubectl wait --for=condition=available deployment/prometheus-operator -n default --timeout=300s
           kubectl get crd
           kubectl get po -A -o wide
-      - name: Install Minio
-        id: install-minio
+      - name: Install Garage S3
+        id: install-garage
         run: |
-          git clone --depth 1 --branch v6.0.1 https://github.com/minio/operator.git && kubectl apply -k operator/
-          kubectl apply -f ./base/minio/minio-tenant-base.yaml
-          echo "Waiting for resources to be created..."
-          sleep 10
-          kubectl wait --for=jsonpath='{.status.readyReplicas}'=1 statefulset/myminio-pool-0 -n minio-tenant --timeout=300s
-          kubectl get po -A -o wide
-          kubectl get po -A -o wide | grep myminio-pool-0-0 | awk '{print $3}' | grep -q '2/2' && echo "myminio-pool-0-0 pod is running with status 2/2" || (echo "myminio-pool-0-0 pod is not running with status 2/2" && exit 1)
+          bash scripts/install-garage.sh
+          bash scripts/check-garage.sh
       - name: Install MongoDB
         id: install-mongodb
         run: |
@@ -105,34 +105,20 @@ jobs:
           kubectl create namespace kerberos-hub
           helm install hub kerberos/hub --values ./base/hub/kerberos-hub-values.yaml -n kerberos-hub --create-namespace
           sleep 10
-          kubectl wait --for=condition=available deployment/admin -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/hub-api -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/hub-cleanup -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/hub-frontend -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/hub-frontend-demo -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/hub-monitor-device -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/hub-reactivate-subscription -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-analysis -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-counting -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-dominantcolor -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-event -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-export -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-monitor -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-notify -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-notify-test -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-sequence -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-sprite -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-throttler -n kerberos-hub --timeout=300s
+          kubectl wait --for=condition=available deployment --all -n kerberos-hub --timeout=300s
           kubectl get pods -A -o wide
+      - name: Collect deployment diagnostics
+        if: failure()
+        run: bash scripts/diagnose-deployment.sh
       - name: Send Slack Notification on Success
         if: success()
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
         run: |
-          curl -X POST -H 'Content-type: application/json' --data "{\"text\": \"✅ Kind deployment successful on ${{ matrix.os }} (${{ matrix.kind }}) - triggered by ${{ github.actor }}\"}" $SLACK_WEBHOOK_URL
+          bash scripts/notify-slack.sh "✅ Kind deployment successful on ${{ matrix.os }} (${{ matrix.kind }}) - triggered by ${{ github.actor }}"
       - name: Send Slack Notification on Failure
         if: failure()
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
         run: |
-          curl -X POST -H 'Content-type: application/json' --data "{\"text\": \"❌ Kind deployment failed on ${{ matrix.os }} (${{ matrix.kind }}) - <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>\"}" $SLACK_WEBHOOK_URL
+          bash scripts/notify-slack.sh "❌ Kind deployment failed on ${{ matrix.os }} (${{ matrix.kind }}) - <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>"

.github/workflows/kustomize.yaml

@@ -1,4 +1,4 @@
-name: Deploy on kind using Kustmize
+name: Validate Kustomize and Garage on kind
 
 on:
   workflow_dispatch:
@@ -10,6 +10,7 @@ jobs:
   deploy:
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu-22.04, ubuntu-24.04]
         #kind: [v0.21.0, v0.22.0, v0.23.0, v0.24.0, v0.25.0]
@@ -28,16 +29,34 @@ jobs:
           kubectl get pods -A -o wide
           kubectl get sc
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
+      - name: Render base Kustomize manifests
+        run: kubectl kustomize base > "$RUNNER_TEMP/base.yaml"
+      - name: Use Kind storage provisioner
+        run: |
+          kubectl get storageclass standard -o json |
+            jq '{apiVersion,kind,metadata:{name:"ssd-hostpath"},provisioner,parameters,volumeBindingMode,reclaimPolicy}' |
+            kubectl apply -f -
+      - name: Install Garage and test persistent S3 access
+        run: |
+          bash scripts/install-garage.sh
+          bash scripts/check-garage.sh
+          credential_hash="$(kubectl -n garage get secret garage-credentials -o jsonpath='{.data}' | sha256sum)"
+          bash scripts/install-garage.sh
+          test "$credential_hash" = "$(kubectl -n garage get secret garage-credentials -o jsonpath='{.data}' | sha256sum)"
+          bash scripts/check-garage.sh --restart
+      - name: Collect deployment diagnostics
+        if: failure()
+        run: bash scripts/diagnose-deployment.sh
       - name: Send Slack Notification on Success
         if: success()
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
         run: |
-          curl -X POST -H 'Content-type: application/json' --data "{\"text\": \"✅ Kustomize deployment successful on ${{ matrix.os }} (${{ matrix.kind }}) - triggered by ${{ github.actor }}\"}" $SLACK_WEBHOOK_URL
+          bash scripts/notify-slack.sh "✅ Kustomize deployment successful on ${{ matrix.os }} (${{ matrix.kind }}) - triggered by ${{ github.actor }}"
       - name: Send Slack Notification on Failure
         if: failure()
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
         run: |
-          curl -X POST -H 'Content-type: application/json' --data "{\"text\": \"❌ Kustomize deployment failed on ${{ matrix.os }} (${{ matrix.kind }}) - <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>\"}" $SLACK_WEBHOOK_URL
+          bash scripts/notify-slack.sh "❌ Kustomize deployment failed on ${{ matrix.os }} (${{ matrix.kind }}) - <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>"

.github/workflows/microk8s.yaml

@@ -8,6 +8,7 @@ jobs:
   deploy:
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu-22.04, ubuntu-24.04]
         microk8s: [1.32/stable]
@@ -25,7 +26,7 @@ jobs:
           kubectl get no
           kubectl get pods -A -o wide
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Create storage class
         id: create-storage-class
         run: |
@@ -39,16 +40,11 @@ jobs:
           kubectl wait --for=condition=available deployment/prometheus-operator -n default --timeout=300s
           kubectl get crd
           kubectl get po -A -o wide
-      - name: Install Minio
-        id: install-minio
+      - name: Install Garage S3
+        id: install-garage
         run: |
-          git clone --depth 1 --branch v6.0.1 https://github.com/minio/operator.git && kubectl apply -k operator/
-          kubectl apply -f ./base/minio/minio-tenant-base.yaml
-          echo "Waiting for resources to be created..."
-          sleep 10
-          kubectl wait --for=jsonpath='{.status.readyReplicas}'=1 statefulset/myminio-pool-0 -n minio-tenant --timeout=300s
-          kubectl get po -A -o wide
-          kubectl get po -A -o wide | grep myminio-pool-0-0 | awk '{print $3}' | grep -q '2/2' && echo "myminio-pool-0-0 pod is running with status 2/2" || (echo "myminio-pool-0-0 pod is not running with status 2/2" && exit 1)
+          bash scripts/install-garage.sh
+          bash scripts/check-garage.sh
       - name: Install MongoDB
         id: install-mongodb
         run: |
@@ -98,35 +94,21 @@ jobs:
           kubectl create namespace kerberos-hub
           helm install hub kerberos/hub --values ./base/hub/kerberos-hub-values.yaml -n kerberos-hub --create-namespace
           sleep 10
-          kubectl wait --for=condition=available deployment/admin -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/hub-api -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/hub-cleanup -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/hub-frontend -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/hub-frontend-demo -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/hub-monitor-device -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/hub-reactivate-subscription -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-analysis -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-counting -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-dominantcolor -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-event -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-export -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-monitor -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-notify -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-notify-test -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-sequence -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-sprite -n kerberos-hub --timeout=300s
-          kubectl wait --for=condition=available deployment/pipe-throttler -n kerberos-hub --timeout=300s
+          kubectl wait --for=condition=available deployment --all -n kerberos-hub --timeout=300s
           kubectl get pods -A -o wide
+      - name: Collect deployment diagnostics
+        if: failure()
+        run: bash scripts/diagnose-deployment.sh
       - name: Send Slack Notification on Success
         if: success()
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
         run: |
-          curl -X POST -H 'Content-type: application/json' --data "{\"text\": \"✅ MicroK8s deployment successful on ${{ matrix.os }} (${{ matrix.microk8s }}) - triggered by ${{ github.actor }}\"}" $SLACK_WEBHOOK_URL
+          bash scripts/notify-slack.sh "✅ MicroK8s deployment successful on ${{ matrix.os }} (${{ matrix.microk8s }}) - triggered by ${{ github.actor }}"
       - name: Send Slack Notification on Failure
         if: failure()
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
         run: |
-          curl -X POST -H 'Content-type: application/json' --data "{\"text\": \"❌ MicroK8s deployment failed on ${{ matrix.os }} (${{ matrix.microk8s }}) - <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>\"}" $SLACK_WEBHOOK_URL
+          bash scripts/notify-slack.sh "❌ MicroK8s deployment failed on ${{ matrix.os }} (${{ matrix.microk8s }}) - <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>"
           

.github/workflows/pr-description.yml

@@ -1 +0,0 @@
-

README.configure.md

@@ -24,16 +24,17 @@ With the Vault installed, we can proceed to configure the various components. Cu
 
 ![Configure Vault](./assets/images/configure-vault.gif)
 
-- Navigate to the `Storage Providers` menu and select the (+ Add Storage Provider) button. A modal will appear where you can input the required details. After entering the information, click the "Verify" button to ensure the configuration is valid. Once you receive a "Configuration is valid and working" message, click the "Add Storage Provider" button to complete the process. **_(!You are advised to generate more complex access and secret key for Minio, this is just for demo purposes, do not use this in production)_**
+- Navigate to the `Storage Providers` menu and select the (+ Add Storage Provider) button. Verify and add the provider using the [Garage setup guide](./base/garage/README.md). These settings apply to the default Kubernetes installation; retain the matching provider settings for an existing MinIO or Docker installation.
 
-  - Minio
+  - MinIO/S3-compatible provider type (Garage backend)
     - Enabled: true
-    - Provider name: minio
+    - Provider name: garage
     - Bucket name: mybucket
-    - Region: na
-    - Hostname: myminio-hl.minio-tenant:9000
-    - Access key: minio
-    - Secret key: minio123
+    - Region: garage
+    - Hostname: garage.garage:3900
+    - SSL: false
+    - Access key: `access-key` from Secret `garage/garage-credentials`
+    - Secret key: `secret-key` from the same Secret
 
 - Navigate to the `Integrations` menu and select the (+ Add Integration) button. A modal will appear where you can input the required details. After entering the information, click the "Verify" button to ensure the configuration is valid. Once you receive a "Configuration is valid and working" message, click the "Add Integration" button to complete the process. **_(!You are advised to generate more complex username and password for RabbitMQ, this is just for demo purposes, do not use this in production)_**
 
@@ -50,7 +51,7 @@ With the Vault installed, we can proceed to configure the various components. Cu
 
   - Enabled: true
   - Account name: myaccount
-  - Main provider: minio
+  - Main provider: garage
   - Day limit: 30
   - Integration: rabbitmq
   - Directory: \*

README.k8s.md

@@ -116,7 +116,7 @@ When installing the Kerberos.io stack, several dependencies are required for sto
 
 ### Clone repository
 
-Next, we will clone this repository to our local environment. This will allow us to execute the necessary configuration files for installing the Minio operator, MongoDB Helm chart, and other required components.
+Next, clone this repository for the Garage S3 manifests, MongoDB Helm chart values, and other required components.
 
 ```bash
 git clone https://github.com/kerberos-io/deployment
@@ -127,7 +127,21 @@ cd deployment
 
 When you create a Kubernetes cluster using `kubeadm` on a bare metal machine
 
-### Object storage: MinIO
+### Object storage: Garage (default)
+
+After creating the `ssd-hostpath` StorageClass for your provisioner, run:
+
+```sh
+bash scripts/install-garage.sh
+bash scripts/check-garage.sh
+```
+
+See the [Garage guide](./base/garage/README.md) for generated credentials,
+Vault provider settings, persistence and existing MinIO compatibility. Garage
+uses S3 port `3900` internally and NodePort `30090`; it needs no operator.
+Skip the following legacy section when using the default installation.
+
+### Optional legacy object storage: MinIO
 
 MinIO is a high-performance, distributed object storage system that is compatible with Amazon S3 cloud storage service. It is designed to handle large-scale data storage and retrieval, making it an ideal choice for modern cloud-native applications.
 
@@ -318,14 +332,19 @@ With the Kerberos Vault installed, we can proceed to configure the various compo
 
 - Navigate to the `Storage Providers` menu and select the (+ Add Storage Provider) button. A modal will appear where you can input the required details. After entering the information, click the "Verify" button to ensure the configuration is valid. Once you receive a "Configuration is valid and working" message, click the "Add Storage Provider" button to complete the process.
 
-  - Minio
+  - MinIO/S3-compatible provider type (Garage backend)
     - Enabled: true
-    - Provider name: minio
+    - Provider name: garage
     - Bucket name: mybucket
-    - Region: na
-    - Hostname: myminio-hl.minio-tenant:9000
-    - Access key: minio
-    - Secret key: minio123
+    - Region: garage
+    - Hostname: garage.garage:3900
+    - SSL: false
+    - Access key: `access-key` from Secret `garage/garage-credentials`
+    - Secret key: `secret-key` from the same Secret
+
+For the optional legacy MinIO installation, retain that backend's hostname,
+credentials and provider name instead. See the [Garage guide](./base/garage/README.md)
+before switching an existing installation.
 
 - Navigate to the `Integrations` menu and select the (+ Add Integration) button. A modal will appear where you can input the required details. After entering the information, click the "Verify" button to ensure the configuration is valid. Once you receive a "Configuration is valid and working" message, click the "Add Integration" button to complete the process.
 
@@ -342,7 +361,7 @@ With the Kerberos Vault installed, we can proceed to configure the various compo
 
   - Enabled: true
   - Account name: myaccount
-  - Main provider: minio
+  - Main provider: garage (or the retained provider for an existing installation)
   - Day limit: 30
   - Integration: rabbitmq
   - Directory: \*

README.kustomize.md

@@ -86,7 +86,6 @@ microk8s enable dns
 microk8s enable dashboard
 microk8s enable nvidia
 microk8s enable hostpath-storage
-microk8s enable minio
 ```
 
 You can verify the status of the enabled addons by running the following command:
@@ -103,7 +102,7 @@ kubectl get po -w -A
 
 ### Clone repository
 
-Next, we will clone this repository to our local environment. This will allow us to execute the necessary configuration files for installing the Minio operator, MongoDB Helm chart, and other required components.
+Next, clone this repository for the Garage S3 manifests, MongoDB Helm chart values, and other required components.
 
 ```bash
 git clone https://github.com/kerberos-io/deployment
@@ -112,6 +111,13 @@ cd deployment
 
 ## Kustomize
 
+The default base uses [Garage S3](./base/garage/README.md). Before a native
+Kustomize apply, run `bash scripts/install-garage.sh --prepare-only` to create
+its namespace and persistent credentials. `configure.sh apply` does this
+automatically. Do not enable the MinIO addon for this default stack. Existing
+MinIO installations must keep their backend/provider settings until their data
+is explicitly migrated; do not prune their storage resources.
+
 In contrast to the detailed installation instructions, as mentioned here, an easier option to install is to use our Kustomize configure. This will allow you to specify and create your own overlays to install all the different components through a single command line.
 
 Kustomize uses the concept of `bases` and `overlays`, allowing you to customize the base installation with different settings (an overlay). Before executing the `kustomize` command below, navigate to the `overlays/microk8s/kustomization.yaml` file and modify the `inlineValues` of the Hub to match the IP address of your node. Please note that when using Multipass, WSL, or any other type of virtualization, your IP address will differ from the IP address of your host machine. Verify your IP address using the `ifconfig` command.
@@ -185,7 +191,7 @@ namespace/kerberos-agent unchanged
 namespace/kerberos-factory unchanged
 namespace/kerberos-hub unchanged
 namespace/kerberos-vault unchanged
-namespace/minio-tenant unchanged
+namespace/garage unchanged
 namespace/mongodb unchanged
 namespace/rabbitmq unchanged
 namespace/vernemq unchanged

README.md

@@ -17,6 +17,34 @@ The Kerberos.io stack offers flexible installation options, supporting deploymen
 
 This repository continuously tests the Kerberos.io deployment using GitHub Actions across various platforms. Refer to the badges for an overview of the operating systems and Kubernetes distributions we are deploying to. We welcome suggestions for additional automations to consider.
 
+Kubernetes examples and CI now use **Garage S3** for local object storage.
+See the [Garage setup guide](./base/garage/README.md) for generated credentials,
+Vault configuration, persistence tests and existing MinIO compatibility.
+This is a single-node example, not a replicated production storage deployment.
+The [MinIO manifests](./base/minio/) remain available; no existing data is
+migrated or removed automatically. The Docker guide retains its separate
+storage setup.
+
+Slack notifications are optional. If `SLACK_WEBHOOK_URL` is unavailable (for
+example on fork PRs), CI emits a warning and skips delivery without masking
+deployment failures. Configured webhook delivery errors still fail the step.
+Failed deployment jobs collect pod events and bounded logs for diagnosis.
+
+### Hardened Hub frontend
+
+The promoted frontend image requires Hub chart **0.170.0 or newer**, which
+provides port `8080`, non-root/read-only runtime settings and a writable `/tmp`
+mount. Both primary and demo frontends use these settings; Admin remains on its
+legacy image and port `80`. Keep `global.imageRegistry` empty for the supplied
+fully qualified `ghcr.io` image names. Service ports remain `80` and the frontend
+NodePort remains `32080`, targeting the container port named `http`.
+
+Run `bash scripts/check-hub-runtime.sh` to check the published chart with these
+values before installation, or pass a local chart path as the first argument.
+When adding custom branding mounts, preserve the chart's `frontend-tmp` volume
+and `/tmp` mount. Existing deployments may require the explicit
+[HTTP port migration](https://github.com/kerberos-io/helm-charts/blob/main/charts/hub/README.md#recovering-a-duplicate-http-port-during-upgrades).
+
 ## Architecture
 
 As previously indicated, several integral components, including Object Storage, Database, and Message Brokers, are deployed and configured to ensure the proper functionality of Kerberos.io. The architecture diagrams below aims to provide a visual representation of these components and their interactions with one another depending the deployment; self-hosted or managed cloud deployment.
@@ -31,7 +59,7 @@ Edge (self-hosted) deployments are typically used for camera processing and edge
 
 ![Edge - self-hosted deployment](./assets/images/deployment-self-hosted.svg)
 
-A key aspect of this deployment is the scaling of Agents. Each camera is assigned a dedicated Agent container, which is linked to a central Kerberos Vault. The Kerberos Vault stores metadata in MongoDB, recordings in Minio (or another S3-compliant object storage), and can generate events in a message broker like RabbitMQ. This approach enables you to develop custom applications/logic that can respond to the creation of new recordings.
+A key aspect of this deployment is the scaling of Agents. Each camera is assigned a dedicated Agent container, which is linked to a central Kerberos Vault. The Kerberos Vault stores metadata in MongoDB, recordings in Garage (or another S3-compliant object storage), and can generate events in a message broker like RabbitMQ. This approach enables you to develop custom applications/logic that can respond to the creation of new recordings.
 
 Based on your technology experience and preferences, you can choose from the following deployment guides:
 

README.microk8s.md

@@ -96,14 +96,29 @@ kubectl get po -w -A
 
 ### Clone repository
 
-Next, we will clone this repository to our local environment. This will allow us to execute the necessary configuration files for installing the Minio operator, MongoDB Helm chart, and other required components.
+Next, clone this repository for the Garage S3 manifests, MongoDB Helm chart values, and other required components.
 
 ```bash
 git clone https://github.com/kerberos-io/deployment
 cd deployment
 ```
 
-### Object storage: MinIO
+### Object storage: Garage (default)
+
+Create the `ssd-hostpath` StorageClass backed by MicroK8s hostpath storage:
+
+```sh
+kubectl apply -f base/ssd-hostpath-storageclass.yaml
+bash scripts/install-garage.sh
+bash scripts/check-garage.sh
+```
+
+See the [Garage guide](./base/garage/README.md) for generated credentials,
+Vault settings and persistence. Garage uses S3 port `3900` internally and
+NodePort `30090`; no MinIO operator or addon is required. Skip the following
+legacy section when using the default installation.
+
+### Optional legacy object storage: MinIO
 
 MinIO is a high-performance, distributed object storage system that is compatible with Amazon S3 cloud storage service. It is designed to handle large-scale data storage and retrieval, making it an ideal choice for modern cloud-native applications.
 
@@ -299,16 +314,20 @@ With the Vault installed, we can proceed to configure the various components. Cu
 
 ![Configure Vault](./assets/images/configure-vault.gif)
 
-- Navigate to the `Storage Providers` menu and select the (+ Add Storage Provider) button. A modal will appear where you can input the required details. After entering the information, click the "Verify" button to ensure the configuration is valid. Once you receive a "Configuration is valid and working" message, click the "Add Storage Provider" button to complete the process. **_(!You are advised to generate more complex access and secret key for Minio, this is just for demo purposes, do not use this in production)_**
+- Navigate to the `Storage Providers` menu and select the (+ Add Storage Provider) button. Verify and add the provider using the [Garage setup guide](./base/garage/README.md).
 
-  - Minio
+  - MinIO/S3-compatible provider type (Garage backend)
     - Enabled: true
-    - Provider name: minio
+    - Provider name: garage
     - Bucket name: mybucket
-    - Region: na
-    - Hostname: myminio-hl.minio-tenant:9000
-    - Access key: minio
-    - Secret key: minio123
+    - Region: garage
+    - Hostname: garage.garage:3900
+    - SSL: false
+    - Access key: `access-key` from Secret `garage/garage-credentials`
+    - Secret key: `secret-key` from the same Secret
+
+For the optional legacy MinIO installation, retain that backend's hostname,
+credentials and provider name instead.
 
 - Navigate to the `Integrations` menu and select the (+ Add Integration) button. A modal will appear where you can input the required details. After entering the information, click the "Verify" button to ensure the configuration is valid. Once you receive a "Configuration is valid and working" message, click the "Add Integration" button to complete the process. **_(!You are advised to generate more complex username and password for RabbitMQ, this is just for demo purposes, do not use this in production)_**
 
@@ -325,7 +344,7 @@ With the Vault installed, we can proceed to configure the various components. Cu
 
   - Enabled: true
   - Account name: myaccount
-  - Main provider: minio
+  - Main provider: garage (or the retained provider for an existing installation)
   - Day limit: 30
   - Integration: rabbitmq
   - Directory: \*

base/garage/README.md

@@ -0,0 +1,105 @@
+# Garage S3 for local Kubernetes deployments
+
+The default Kubernetes stack uses [Garage](https://garagehq.deuxfleurs.fr/),
+not the MinIO operator. The pinned multi-architecture `dxflrs/garage:v2.5.0`
+image runs as a single non-root node with SQLite metadata and object data on
+the same 10Gi `ssd-hostpath` PVC. This is a local/edge/CI example with **no
+replication or high availability**, not a production multi-node layout.
+Use Garage's [production deployment guide](https://garagehq.deuxfleurs.fr/documentation/cookbook/real-world/)
+for replicated storage. Choose capacity and a backup policy before storing
+recordings; resizing a bound PVC depends on your storage provisioner.
+
+## Install
+
+First install your storage provisioner and create the `ssd-hostpath`
+StorageClass (the same prerequisite used by MongoDB and RabbitMQ). Run commands
+from the deployment repository root:
+
+```sh
+bash scripts/install-garage.sh
+bash scripts/check-garage.sh
+```
+
+The installer generates an RPC secret and S3 credentials in the Kubernetes
+Secret `garage/garage-credentials`, and reuses them on subsequent runs. No
+credentials are committed to the repository. Garage's `--single-node` and
+`--default-bucket` startup flags initialize the layout and `mybucket`.
+The installer refuses to generate new credentials if the PVC already exists
+but its Secret is missing: restore the original Secret instead.
+
+The installer requires `kubectl`, `jq` and `openssl`. The smoke test also
+requires AWS CLI and `curl` (provided by the GitHub-hosted CI runners).
+
+For a full Kustomize installation, prepare credentials before applying the
+overlay; the default base already includes Garage:
+
+```sh
+bash scripts/install-garage.sh --prepare-only
+kubectl kustomize overlays/microk8s --enable-helm --load-restrictor LoadRestrictionsNone | kubectl apply -f -
+kubectl -n garage rollout status statefulset/garage --timeout=300s
+```
+
+The `configure.sh apply` helper also prepares the Secret before applying its
+overlay. Back up the Secret and the PVC together. Do not delete the namespace,
+Secret or PVC as an upgrade/recovery step.
+
+## Configure Vault
+
+Add a storage provider using Vault's existing **MinIO/S3-compatible provider
+type**. The provider *name* identifies the instance and is distinct from its
+implementation type:
+
+| Setting | Value |
+| --- | --- |
+| Provider type | MinIO (`minio`) |
+| Provider name | `garage` |
+| Bucket | `mybucket` |
+| Region | `garage` |
+| Internal hostname | `garage.garage:3900` (no URL scheme) |
+| SSL | Disabled for this local HTTP example |
+| Access key | Secret field `access-key` |
+| Secret key | Secret field `secret-key` |
+
+Retrieve credentials locally when configuring Vault; do not paste them into
+logs, issues, or committed values:
+
+```sh
+kubectl -n garage get secret garage-credentials -o jsonpath='{.data.access-key}' | base64 --decode
+kubectl -n garage get secret garage-credentials -o jsonpath='{.data.secret-key}' | base64 --decode
+```
+
+The S3 endpoint is exposed at `http://NODE_IP:30090`. If browsers or remote
+clients consume provider-native presigned URLs, configure Vault's external
+hostname as `NODE_IP:30090` with external SSL disabled (or a real TLS endpoint).
+Cluster-only DNS names cannot be resolved by external clients. Protect exposed
+ports with firewall rules and use TLS outside a trusted local network. Garage's
+RPC port is bound to loopback; no admin API or web console is exposed.
+
+Set the Vault account's main provider to `garage`. The supplied Hub values
+also use that name for recording, archive, thumbnail and sprite access.
+**Vault account credentials are separate from Garage S3 credentials**: keep
+the existing Agent/Hub account credentials aligned with the Vault account,
+not with the generated S3 key.
+
+## Verification
+
+`scripts/check-garage.sh` uses AWS CLI for an authenticated Signature V4 PUT/GET
+through the Kubernetes Service and compares object bytes. It retains a small,
+uniquely named object under `mybucket/ci-smoke/`; it never deletes existing
+objects. The optional `--restart` flag restarts the Garage pod between upload
+and download to verify persistence; use it only in CI or a maintenance window.
+CI also verifies that reinstalling Garage preserves its credential Secret.
+
+## Existing MinIO installations
+
+The [MinIO manifests](../minio/) remain available as an explicit legacy option.
+This change does **not** copy recordings, modify Vault's stored provider
+records, or delete MinIO resources. Do not use prune/apply cleanup to remove
+the old backend while it contains recordings.
+
+When upgrading an existing deployment, keep its existing Vault provider names
+in the Hub values (`kerberosvault.provider`, `archive.provider`,
+`thumbnail.provider`, `sprite.provider`) until recordings have been explicitly
+migrated and reads verified. Add Garage as a separate provider/account as
+needed. Treat object migration and retirement of MinIO as a separate,
+backed-up operation; this installer only provisions the new backend.

base/garage/garage.yaml

@@ -0,0 +1,128 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: garage
+data:
+  garage.toml: |
+    metadata_dir = "/var/lib/garage/meta"
+    data_dir = "/var/lib/garage/data"
+    db_engine = "sqlite"
+    replication_factor = 1
+    rpc_bind_addr = "127.0.0.1:3901"
+    rpc_public_addr = "127.0.0.1:3901"
+
+    [s3_api]
+    s3_region = "garage"
+    api_bind_addr = "[::]:3900"
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: garage
+spec:
+  type: NodePort
+  selector:
+    app: garage
+  ports:
+    - name: s3
+      port: 3900
+      targetPort: s3
+      nodePort: 30090
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: garage
+spec:
+  serviceName: garage
+  replicas: 1
+  selector:
+    matchLabels:
+      app: garage
+  template:
+    metadata:
+      labels:
+        app: garage
+    spec:
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
+        fsGroupChangePolicy: OnRootMismatch
+      containers:
+        - name: garage
+          image: dxflrs/garage:v2.5.0@sha256:677e5b758e4425a325b811c425c026bf35a4bf42f7ddbaa773de3f4fd779a5a5
+          command: ["/garage"]
+          args: ["server", "--single-node", "--default-bucket"]
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+            seccompProfile:
+              type: RuntimeDefault
+          env:
+            - name: GARAGE_RPC_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: garage-credentials
+                  key: rpc-secret
+            - name: GARAGE_DEFAULT_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: garage-credentials
+                  key: access-key
+            - name: GARAGE_DEFAULT_SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: garage-credentials
+                  key: secret-key
+            - name: GARAGE_DEFAULT_BUCKET
+              value: mybucket
+          ports:
+            - name: s3
+              containerPort: 3900
+          startupProbe:
+            tcpSocket:
+              port: s3
+            periodSeconds: 2
+            failureThreshold: 60
+          readinessProbe:
+            exec:
+              command: ["/garage", "health"]
+          livenessProbe:
+            tcpSocket:
+              port: s3
+          resources:
+            requests:
+              cpu: 100m
+              memory: 128Mi
+            limits:
+              memory: 512Mi
+          volumeMounts:
+            - name: config
+              mountPath: /etc/garage.toml
+              subPath: garage.toml
+              readOnly: true
+            - name: data
+              mountPath: /var/lib/garage
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: config
+          configMap:
+            name: garage
+        - name: tmp
+          emptyDir:
+            sizeLimit: 16Mi
+  volumeClaimTemplates:
+    - metadata:
+        name: data
+      spec:
+        accessModes: ["ReadWriteOnce"]
+        storageClassName: ssd-hostpath
+        resources:
+          requests:
+            storage: 10Gi

base/garage/kustomization.yaml

@@ -0,0 +1,4 @@
+namespace: garage
+resources:
+  - namespace.yaml
+  - garage.yaml

base/garage/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: garage

base/hub/kerberos-hub-nodeports.yaml

@@ -9,7 +9,7 @@ spec:
   type: NodePort
   ports:
     - port: 80
-      targetPort: 80
+      targetPort: http
       nodePort: 32080 # You can specify a port in the range 30000-32767 or let Kubernetes assign one automatically
       name: frontend
       protocol: TCP

base/hub/kerberos-hub-values.yaml

@@ -19,7 +19,7 @@ licenseServer:
 
 # When using a private registry, you can specify the registry here.
 global:
-  imageRegistry: "docker.io/"
+  imageRegistry: ""
 # Environment: set to 'production', 'develop', 'demo', 'staging' or 'acceptance'.
 # Set to 'true' if this is a private deployment.
 environment: "production"
@@ -92,22 +92,22 @@ openai:
 kerberosvault:
   #uri: "http://vault-lb.kerberos-vault" # this needs to be a public accessible DNS name.
   uri: http://vault-nodeport.kerberos-vault:80/api
-  provider: "minio"
+  provider: "garage"
   accesskey: "XJoi2@bgSOvOYBy#"
   secretkey: "OGGqat4lXRpL@9XBYc8FUaId@5"
   # Archiving is used when creating a task. The underlying recording of the task will be copied from its
   # existing provider to the below archived provider. Seperate credentials are used, as it makes possible to
   # specify another retention period.
   archive:
-    provider: "minio"
+    provider: "garage"
     accesskey: "XJoi2@bgSOvOYBy#"
     secretkey: "OGGqat4lXRpL@9XBYc8FUaId@5"
   thumbnail:
-    provider: "minio"
+    provider: "garage"
     accessKey: "XJoi2@bgSOvOYBy#"
     secretKey: "OGGqat4lXRpL@9XBYc8FUaId@5"
   sprite:
-    provider: "minio"
+    provider: "garage"
     accessKey: "XJoi2@bgSOvOYBy#"
     secretKey: "OGGqat4lXRpL@9XBYc8FUaId@5"
 email:
@@ -241,6 +241,8 @@ kerberoshub:
     repository: ghcr.io/uug-ai/hub-frontend
     pullPolicy: IfNotPresent
     tag: "v1.19.28"
+    # Requires Hub chart 0.170.0+ for the matching hardened runtime defaults.
+    containerPort: 8080
     replicas: 1
     schema: "http"
     url: "yourdomain.com"

base/kustomization.yaml

@@ -6,4 +6,4 @@ resources:
   - factory
   - vault
   - hub
-  - minio
+  - garage

base/vault/vault-runtime-configuration-secret.yaml

@@ -12,8 +12,8 @@ stringData:
   MQTT_USERNAME: ""
   MQTT_PASSWORD: ""
   VAULT_STORAGE_PROVIDERS: >-
-    [{"name":"archive","provider":"minio","host":"minio.minio.svc:9000","region":"us-east-1","bucket":"recordings","access_key":"example-key","secret":"example-secret","use_ssl":"false"}]
+    [{"name":"garage","provider":"minio","host":"garage.garage:3900","region":"garage","bucket":"mybucket","access_key":"replace-with-garage-access-key","secret":"replace-with-garage-secret-key","use_ssl":"false"}]
   VAULT_INTEGRATIONS: >-
     [{"name":"events","queue":"rabbitmq","broker":"amqp://rabbitmq.rabbitmq.svc:5672","topic":"recordings","username":"example-user","password":"example-password"}]
   VAULT_ACCOUNTS: >-
-    [{"account":"camera-fleet","directory":"fleet","provider":"archive","access_key":"example-account-key","secret_access_key":"example-account-secret","limit":"30","queues":["events"]}]
+    [{"account":"camera-fleet","directory":"fleet","provider":"garage","access_key":"example-account-key","secret_access_key":"example-account-secret","limit":"30","queues":["events"]}]

configure.sh

@@ -39,6 +39,8 @@ if [ "$command" == "apply" ]; then
   # Adjust the base path reference
   sed -i "s|../../base|./base|g" ./kustomization.yaml
 
+  bash scripts/install-garage.sh --prepare-only || exit 1
+
   # Apply kustomize installation
   kubectl kustomize ./ --enable-helm --load-restrictor LoadRestrictionsNone | kubectl apply -f -
 

overlays/microk8s/kustomization.yaml

@@ -26,7 +26,7 @@ helmCharts:
     releaseName: hub
     namespace: kerberos-hub
     repo: https://charts.kerberos.io
-    version: 0.84.0
+    version: 0.170.0
     valuesFile: ../../base/hub/kerberos-hub-values.yaml
     valuesInline:
         mqtt:

overlays/standalone/kustomization.yaml

@@ -8,7 +8,7 @@ helmCharts:
     releaseName: hub
     namespace: kerberos-hub
     repo: https://charts.kerberos.io
-    version: 0.84.0
+    version: 0.170.0
     valuesFile: ../../base/hub/kerberos-hub-values.yaml
     valuesInline:
         kerberoshub:

scripts/check-garage.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $# -gt 1 || ( $# -eq 1 && "$1" != --restart ) ]]; then
+  echo "Usage: bash scripts/check-garage.sh [--restart]" >&2
+  exit 1
+fi
+if ! command -v aws > /dev/null; then
+  echo "AWS CLI is required for the Garage S3 smoke test." >&2
+  exit 1
+fi
+temporary="$(mktemp -d)"
+pid=""
+cleanup() {
+  if [[ -n "$pid" ]]; then kill "$pid"; fi
+  rm -f "$temporary/upload" "$temporary/download" "$temporary/port-forward.log" "$temporary/aws-config"
+  rmdir "$temporary"
+}
+trap cleanup EXIT
+
+access_key="$(kubectl -n garage get secret garage-credentials -o jsonpath='{.data.access-key}' | base64 --decode)"
+secret_key="$(kubectl -n garage get secret garage-credentials -o jsonpath='{.data.secret-key}' | base64 --decode)"
+if [[ "${GITHUB_ACTIONS:-false}" == true ]]; then
+  echo "::add-mask::$access_key"
+  echo "::add-mask::$secret_key"
+fi
+forward() {
+  kubectl -n garage port-forward service/garage 13900:3900 > "$temporary/port-forward.log" 2>&1 &
+  pid=$!
+  for attempt in {1..30}; do
+    if curl --silent --output /dev/null http://127.0.0.1:13900/; then return; fi
+    if ! kill -0 "$pid"; then break; fi
+    sleep 1
+  done
+  cat "$temporary/port-forward.log" >&2
+  echo "Garage port-forward did not become ready." >&2
+  exit 1
+}
+forward
+
+export AWS_ACCESS_KEY_ID="$access_key" AWS_SECRET_ACCESS_KEY="$secret_key"
+export AWS_DEFAULT_REGION=garage AWS_EC2_METADATA_DISABLED=true
+export AWS_PAGER=""
+unset AWS_SESSION_TOKEN AWS_PROFILE AWS_DEFAULT_PROFILE
+export AWS_REQUEST_CHECKSUM_CALCULATION=when_required AWS_RESPONSE_CHECKSUM_VALIDATION=when_required
+export AWS_CONFIG_FILE="$temporary/aws-config"
+printf '[default]\ns3 =\n    addressing_style = path\n' > "$AWS_CONFIG_FILE"
+object="ci-smoke/$(openssl rand -hex 16)"
+openssl rand -hex 128 > "$temporary/upload"
+endpoint="http://127.0.0.1:13900"
+aws --endpoint-url "$endpoint" s3api put-object \
+  --bucket mybucket --key "$object" --body "$temporary/upload"
+if [[ $# -eq 1 ]]; then
+  kill "$pid"
+  wait "$pid" || [[ "$?" == 143 ]]
+  pid=""
+  kubectl -n garage rollout restart statefulset/garage
+  kubectl -n garage rollout status statefulset/garage --timeout=300s
+  forward
+fi
+aws --endpoint-url "$endpoint" s3api get-object \
+  --bucket mybucket --key "$object" "$temporary/download"
+cmp "$temporary/upload" "$temporary/download"
+echo "Garage S3 authenticated PUT/GET passed through the Kubernetes Service (object mybucket/$object retained)."

scripts/check-hub-runtime.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+CHART="${1:-kerberos/hub}"
+
+assert_contains() {
+  if ! grep -Eq -- "$2" <<< "$1"; then
+    printf 'Missing expected deployment runtime setting: %s\n' "$2" >&2
+    exit 1
+  fi
+}
+
+nodeport="$(awk '
+  /^---/ { selected=0 }
+  /name: hub-frontend-nodeport$/ { selected=1 }
+  selected { print }
+' "$ROOT/base/hub/kerberos-hub-nodeports.yaml")"
+assert_contains "$nodeport" 'targetPort: http$'
+assert_contains "$nodeport" 'nodePort: 32080'
+
+rendered="$(helm template hub "$CHART" --values "$ROOT/base/hub/kerberos-hub-values.yaml" \
+  --set mode=ui --set kerberoshub.frontend.demoEnabled=true)"
+if grep -q 'image: "docker.io/ghcr.io/' <<< "$rendered"; then
+  echo "Fully qualified GHCR image was prefixed with docker.io" >&2
+  exit 1
+fi
+for name in hub-frontend hub-frontend-demo; do
+  deployment="$(awk -v name="$name" '
+    function emit() { if (deployment && matched) printf "%s", block }
+    /^---$/ { emit(); block=""; deployment=0; matched=0 }
+    { block=block $0 "\n" }
+    /^kind: Deployment$/ { deployment=1 }
+    $0 == "  name: " name { matched=1 }
+    END { emit() }
+  ' <<< "$rendered")"
+  for setting in 'containerPort: 8080' 'runAsNonRoot: true' 'runAsUser: 65532' \
+    'readOnlyRootFilesystem: true' 'mountPath: /tmp' 'name: frontend-tmp'; do
+    assert_contains "$deployment" "^[[:space:]]*(- )?$setting$"
+  done
+  assert_contains "$deployment" 'image: "ghcr.io/uug-ai/hub-frontend:v1.19.28"$'
+  echo "OK: $name uses the promoted image and matching hardened runtime"
+done
+echo "Deployment Hub runtime preflight passed."

scripts/diagnose-deployment.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -uo pipefail
+
+kubectl get pods,pvc -A -o wide
+kubectl get events -A --sort-by=.lastTimestamp
+for namespace in garage mongodb rabbitmq vernemq kerberos-vault kerberos-agent kerberos-hub; do
+  if ! pods="$(kubectl -n "$namespace" get pods -o name)"; then
+    echo "::warning::Could not list pods in $namespace."
+    continue
+  fi
+  for pod in $pods; do
+    if ! kubectl -n "$namespace" describe "$pod"; then
+      echo "::warning::Could not describe $namespace/$pod."
+    fi
+    if ! kubectl -n "$namespace" logs "$pod" --all-containers=true --tail=80 --prefix; then
+      echo "::warning::Logs unavailable for $namespace/$pod."
+    fi
+  done
+done

scripts/install-garage.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+if [[ $# -gt 1 || ( $# -eq 1 && "$1" != --prepare-only ) ]]; then
+  echo "Usage: bash scripts/install-garage.sh [--prepare-only]" >&2
+  exit 1
+fi
+kubectl apply -f "$ROOT/base/garage/namespace.yaml"
+
+secret="$(kubectl -n garage get secret garage-credentials --ignore-not-found -o name)"
+if [[ -z "$secret" ]]; then
+  pvc="$(kubectl -n garage get pvc data-garage-0 --ignore-not-found -o name)"
+  if [[ -n "$pvc" ]]; then
+    echo "Garage data already exists without its credentials. Restore garage-credentials from backup; refusing to generate replacement keys." >&2
+    exit 1
+  fi
+  kubectl -n garage create secret generic garage-credentials \
+    --from-literal="rpc-secret=$(openssl rand -hex 32)" \
+    --from-literal="access-key=GK$(openssl rand -hex 16)" \
+    --from-literal="secret-key=$(openssl rand -hex 32)"
+fi
+
+if ! kubectl -n garage get secret garage-credentials -o json |
+  jq -e '.data |
+    (."rpc-secret" | @base64d | test("^[0-9a-f]{64}$")) and
+    (."access-key" | @base64d | test("^GK[0-9a-f]{32}$")) and
+    (."secret-key" | @base64d | test("^[0-9a-f]{64}$"))' > /dev/null; then
+  echo "Garage credentials are missing or invalid; restore a valid garage-credentials Secret." >&2
+  exit 1
+fi
+if [[ $# -eq 1 ]]; then
+  echo "Garage credentials are ready; apply your Kustomize overlay next."
+  exit 0
+fi
+kubectl apply -k "$ROOT/base/garage"
+kubectl -n garage rollout status statefulset/garage --timeout=300s
+kubectl -n garage exec garage-0 -- /garage bucket info mybucket > /dev/null
+echo "Garage is available at garage.garage:3900 (region garage, bucket mybucket). Credentials are in Secret garage/garage-credentials."

scripts/notify-slack.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $# != 1 ]]; then
+  echo "Usage: bash scripts/notify-slack.sh MESSAGE" >&2
+  exit 1
+fi
+
+if [[ -z "${SLACK_WEBHOOK_URL:-}" ]]; then
+  echo "::warning::SLACK_WEBHOOK_URL is not configured; skipping optional Slack notification."
+  exit 0
+fi
+
+jq -n --arg text "$1" '{text: $text}' |
+  curl --fail-with-body --silent --show-error --retry 3 \
+    -H 'Content-Type: application/json' --data-binary @- "$SLACK_WEBHOOK_URL"