Use persistent Garage S3 in local deployments and CI

Replace default MinIO operator installation with pinned non-root Garage, generated persistent credentials and authenticated S3 tests. Keep MinIO manifests intact for existing installations, add failure diagnostics and optional Slack handling, and align Kustomize chart pins.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

.github/workflows/k3d.yaml

@@ -8,6 +8,7 @@ jobs:
   deploy:
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu-22.04, ubuntu-24.04]
         #k3d: [v5.5.2, v5.6.0, v5.7.5]
@@ -29,8 +30,6 @@ jobs:
           kubectl get no
           kubectl get pods -A -o wide
           kubectl get sc
-      - name: Checkout repository
-        uses: actions/checkout@v2
       - name: Install OpenEBS
         run: |
           kubectl apply -f https://openebs.github.io/charts/openebs-operator.yaml
@@ -50,16 +49,11 @@ jobs:
           kubectl wait --for=condition=available deployment/prometheus-operator -n default --timeout=300s
           kubectl get crd
           kubectl get po -A -o wide
-      - name: Install Minio
-        id: install-minio
+      - name: Install Garage S3
+        id: install-garage
         run: |
-          git clone --depth 1 --branch v6.0.1 https://github.com/minio/operator.git && kubectl apply -k operator/
-          kubectl apply -f ./base/minio/minio-tenant-base.yaml
-          echo "Waiting for resources to be created..."
-          sleep 10
-          kubectl wait --for=jsonpath='{.status.readyReplicas}'=1 statefulset/myminio-pool-0 -n minio-tenant --timeout=300s
-          kubectl get po -A -o wide
-          kubectl get po -A -o wide | grep myminio-pool-0-0 | awk '{print $3}' | grep -q '2/2' && echo "myminio-pool-0-0 pod is running with status 2/2" || (echo "myminio-pool-0-0 pod is not running with status 2/2" && exit 1)
+          bash scripts/install-garage.sh
+          bash scripts/check-garage.sh
       - name: Install MongoDB
         id: install-mongodb
         run: |
@@ -128,15 +122,18 @@ jobs:
           kubectl wait --for=condition=available deployment/pipe-sprite -n kerberos-hub --timeout=300s
           kubectl wait --for=condition=available deployment/pipe-throttler -n kerberos-hub --timeout=300s
           kubectl get pods -A -o wide
+      - name: Collect deployment diagnostics
+        if: failure()
+        run: bash scripts/diagnose-deployment.sh
       - name: Send Slack Notification on Success
         if: success()
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
         run: |
-          curl -X POST -H 'Content-type: application/json' --data "{\"text\": \"✅ K3d deployment successful on ${{ matrix.os }} (${{ matrix.k3d }}) - triggered by ${{ github.actor }}\"}" $SLACK_WEBHOOK_URL
+          bash scripts/notify-slack.sh "✅ K3d deployment successful on ${{ matrix.os }} (${{ matrix.k3d }}) - triggered by ${{ github.actor }}"
       - name: Send Slack Notification on Failure
         if: failure()
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
         run: |
-          curl -X POST -H 'Content-type: application/json' --data "{\"text\": \"❌ K3d deployment failed on ${{ matrix.os }} (${{ matrix.k3d }}) - <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>\"}" $SLACK_WEBHOOK_URL
+          bash scripts/notify-slack.sh "❌ K3d deployment failed on ${{ matrix.os }} (${{ matrix.k3d }}) - <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>"

.github/workflows/kind.yaml

@@ -27,7 +27,7 @@ jobs:
           kubectl get pods -A -o wide
           kubectl get sc
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Check promoted Hub runtime
         run: |
           helm repo add kerberos https://charts.kerberos.io
@@ -51,16 +51,11 @@ jobs:
           kubectl wait --for=condition=available deployment/prometheus-operator -n default --timeout=300s
           kubectl get crd
           kubectl get po -A -o wide
-      - name: Install Minio
-        id: install-minio
+      - name: Install Garage S3
+        id: install-garage
         run: |
-          git clone --depth 1 --branch v6.0.1 https://github.com/minio/operator.git && kubectl apply -k operator/
-          kubectl apply -f ./base/minio/minio-tenant-base.yaml
-          echo "Waiting for resources to be created..."
-          sleep 10
-          kubectl wait --for=jsonpath='{.status.readyReplicas}'=1 statefulset/myminio-pool-0 -n minio-tenant --timeout=300s
-          kubectl get po -A -o wide
-          kubectl get po -A -o wide | grep myminio-pool-0-0 | awk '{print $3}' | grep -q '2/2' && echo "myminio-pool-0-0 pod is running with status 2/2" || (echo "myminio-pool-0-0 pod is not running with status 2/2" && exit 1)
+          bash scripts/install-garage.sh
+          bash scripts/check-garage.sh
       - name: Install MongoDB
         id: install-mongodb
         run: |
@@ -131,21 +126,16 @@ jobs:
           kubectl get pods -A -o wide
       - name: Collect deployment diagnostics
         if: failure()
-        run: |
-          kubectl get pods,pvc -A -o wide
-          kubectl get events -A --sort-by=.lastTimestamp
-          kubectl -n minio-tenant describe pods
-          kubectl -n minio-operator logs -l name=minio-operator --all-containers=true --tail=100
-          kubectl -n minio-tenant logs myminio-pool-0-0 --all-containers=true --tail=100
+        run: bash scripts/diagnose-deployment.sh
       - name: Send Slack Notification on Success
         if: success()
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
         run: |
-          curl -X POST -H 'Content-type: application/json' --data "{\"text\": \"✅ Kind deployment successful on ${{ matrix.os }} (${{ matrix.kind }}) - triggered by ${{ github.actor }}\"}" $SLACK_WEBHOOK_URL
+          bash scripts/notify-slack.sh "✅ Kind deployment successful on ${{ matrix.os }} (${{ matrix.kind }}) - triggered by ${{ github.actor }}"
       - name: Send Slack Notification on Failure
         if: failure()
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
         run: |
-          curl -X POST -H 'Content-type: application/json' --data "{\"text\": \"❌ Kind deployment failed on ${{ matrix.os }} (${{ matrix.kind }}) - <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>\"}" $SLACK_WEBHOOK_URL
+          bash scripts/notify-slack.sh "❌ Kind deployment failed on ${{ matrix.os }} (${{ matrix.kind }}) - <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>"

.github/workflows/kustomize.yaml

@@ -1,4 +1,4 @@
-name: Deploy on kind using Kustmize
+name: Validate Kustomize and Garage on kind
 
 on:
   workflow_dispatch:
@@ -28,16 +28,34 @@ jobs:
           kubectl get pods -A -o wide
           kubectl get sc
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
+      - name: Render base Kustomize manifests
+        run: kubectl kustomize base > "$RUNNER_TEMP/base.yaml"
+      - name: Use Kind storage provisioner
+        run: |
+          kubectl get storageclass standard -o json |
+            jq '{apiVersion,kind,metadata:{name:"ssd-hostpath"},provisioner,parameters,volumeBindingMode,reclaimPolicy}' |
+            kubectl apply -f -
+      - name: Install Garage and test persistent S3 access
+        run: |
+          bash scripts/install-garage.sh
+          bash scripts/check-garage.sh
+          credential_hash="$(kubectl -n garage get secret garage-credentials -o jsonpath='{.data}' | sha256sum)"
+          bash scripts/install-garage.sh
+          test "$credential_hash" = "$(kubectl -n garage get secret garage-credentials -o jsonpath='{.data}' | sha256sum)"
+          bash scripts/check-garage.sh --restart
+      - name: Collect deployment diagnostics
+        if: failure()
+        run: bash scripts/diagnose-deployment.sh
       - name: Send Slack Notification on Success
         if: success()
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
         run: |
-          curl -X POST -H 'Content-type: application/json' --data "{\"text\": \"✅ Kustomize deployment successful on ${{ matrix.os }} (${{ matrix.kind }}) - triggered by ${{ github.actor }}\"}" $SLACK_WEBHOOK_URL
+          bash scripts/notify-slack.sh "✅ Kustomize deployment successful on ${{ matrix.os }} (${{ matrix.kind }}) - triggered by ${{ github.actor }}"
       - name: Send Slack Notification on Failure
         if: failure()
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
         run: |
-          curl -X POST -H 'Content-type: application/json' --data "{\"text\": \"❌ Kustomize deployment failed on ${{ matrix.os }} (${{ matrix.kind }}) - <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>\"}" $SLACK_WEBHOOK_URL
+          bash scripts/notify-slack.sh "❌ Kustomize deployment failed on ${{ matrix.os }} (${{ matrix.kind }}) - <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>"

.github/workflows/microk8s.yaml

@@ -8,6 +8,7 @@ jobs:
   deploy:
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu-22.04, ubuntu-24.04]
         microk8s: [1.32/stable]
@@ -25,7 +26,7 @@ jobs:
           kubectl get no
           kubectl get pods -A -o wide
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Create storage class
         id: create-storage-class
         run: |
@@ -39,16 +40,11 @@ jobs:
           kubectl wait --for=condition=available deployment/prometheus-operator -n default --timeout=300s
           kubectl get crd
           kubectl get po -A -o wide
-      - name: Install Minio
-        id: install-minio
+      - name: Install Garage S3
+        id: install-garage
         run: |
-          git clone --depth 1 --branch v6.0.1 https://github.com/minio/operator.git && kubectl apply -k operator/
-          kubectl apply -f ./base/minio/minio-tenant-base.yaml
-          echo "Waiting for resources to be created..."
-          sleep 10
-          kubectl wait --for=jsonpath='{.status.readyReplicas}'=1 statefulset/myminio-pool-0 -n minio-tenant --timeout=300s
-          kubectl get po -A -o wide
-          kubectl get po -A -o wide | grep myminio-pool-0-0 | awk '{print $3}' | grep -q '2/2' && echo "myminio-pool-0-0 pod is running with status 2/2" || (echo "myminio-pool-0-0 pod is not running with status 2/2" && exit 1)
+          bash scripts/install-garage.sh
+          bash scripts/check-garage.sh
       - name: Install MongoDB
         id: install-mongodb
         run: |
@@ -117,16 +113,19 @@ jobs:
           kubectl wait --for=condition=available deployment/pipe-sprite -n kerberos-hub --timeout=300s
           kubectl wait --for=condition=available deployment/pipe-throttler -n kerberos-hub --timeout=300s
           kubectl get pods -A -o wide
+      - name: Collect deployment diagnostics
+        if: failure()
+        run: bash scripts/diagnose-deployment.sh
       - name: Send Slack Notification on Success
         if: success()
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
         run: |
-          curl -X POST -H 'Content-type: application/json' --data "{\"text\": \"✅ MicroK8s deployment successful on ${{ matrix.os }} (${{ matrix.microk8s }}) - triggered by ${{ github.actor }}\"}" $SLACK_WEBHOOK_URL
+          bash scripts/notify-slack.sh "✅ MicroK8s deployment successful on ${{ matrix.os }} (${{ matrix.microk8s }}) - triggered by ${{ github.actor }}"
       - name: Send Slack Notification on Failure
         if: failure()
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
         run: |
-          curl -X POST -H 'Content-type: application/json' --data "{\"text\": \"❌ MicroK8s deployment failed on ${{ matrix.os }} (${{ matrix.microk8s }}) - <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>\"}" $SLACK_WEBHOOK_URL
+          bash scripts/notify-slack.sh "❌ MicroK8s deployment failed on ${{ matrix.os }} (${{ matrix.microk8s }}) - <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>"
           

base/garage/garage.yaml

@@ -0,0 +1,134 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: garage
+data:
+  garage.toml: |
+    metadata_dir = "/var/lib/garage/meta"
+    data_dir = "/var/lib/garage/data"
+    db_engine = "sqlite"
+    replication_factor = 1
+    rpc_bind_addr = "127.0.0.1:3901"
+    rpc_public_addr = "127.0.0.1:3901"
+    rpc_secret_file = "/secrets/rpc-secret"
+
+    [s3_api]
+    s3_region = "garage"
+    api_bind_addr = "[::]:3900"
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: garage
+spec:
+  type: NodePort
+  selector:
+    app: garage
+  ports:
+    - name: s3
+      port: 3900
+      targetPort: s3
+      nodePort: 30090
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: garage
+spec:
+  serviceName: garage
+  replicas: 1
+  selector:
+    matchLabels:
+      app: garage
+  template:
+    metadata:
+      labels:
+        app: garage
+    spec:
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
+        fsGroupChangePolicy: OnRootMismatch
+      containers:
+        - name: garage
+          image: dxflrs/garage:v2.5.0@sha256:677e5b758e4425a325b811c425c026bf35a4bf42f7ddbaa773de3f4fd779a5a5
+          command: ["/garage"]
+          args: ["server", "--single-node", "--default-bucket"]
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+            seccompProfile:
+              type: RuntimeDefault
+          env:
+            - name: GARAGE_DEFAULT_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: garage-credentials
+                  key: access-key
+            - name: GARAGE_DEFAULT_SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: garage-credentials
+                  key: secret-key
+            - name: GARAGE_DEFAULT_BUCKET
+              value: mybucket
+          ports:
+            - name: s3
+              containerPort: 3900
+          startupProbe:
+            tcpSocket:
+              port: s3
+            periodSeconds: 2
+            failureThreshold: 60
+          readinessProbe:
+            tcpSocket:
+              port: s3
+          livenessProbe:
+            tcpSocket:
+              port: s3
+          resources:
+            requests:
+              cpu: 100m
+              memory: 128Mi
+            limits:
+              memory: 512Mi
+          volumeMounts:
+            - name: config
+              mountPath: /etc/garage.toml
+              subPath: garage.toml
+              readOnly: true
+            - name: credentials
+              mountPath: /secrets
+              readOnly: true
+            - name: data
+              mountPath: /var/lib/garage
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: config
+          configMap:
+            name: garage
+        - name: credentials
+          secret:
+            secretName: garage-credentials
+            defaultMode: 0440
+            items:
+              - key: rpc-secret
+                path: rpc-secret
+        - name: tmp
+          emptyDir:
+            sizeLimit: 16Mi
+  volumeClaimTemplates:
+    - metadata:
+        name: data
+      spec:
+        accessModes: ["ReadWriteOnce"]
+        storageClassName: ssd-hostpath
+        resources:
+          requests:
+            storage: 10Gi

base/garage/kustomization.yaml

@@ -0,0 +1,4 @@
+namespace: garage
+resources:
+  - namespace.yaml
+  - garage.yaml

base/garage/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: garage

base/hub/kerberos-hub-values.yaml

@@ -92,22 +92,22 @@ openai:
 kerberosvault:
   #uri: "http://vault-lb.kerberos-vault" # this needs to be a public accessible DNS name.
   uri: http://vault-nodeport.kerberos-vault:80/api
-  provider: "minio"
+  provider: "garage"
   accesskey: "XJoi2@bgSOvOYBy#"
   secretkey: "OGGqat4lXRpL@9XBYc8FUaId@5"
   # Archiving is used when creating a task. The underlying recording of the task will be copied from its
   # existing provider to the below archived provider. Seperate credentials are used, as it makes possible to
   # specify another retention period.
   archive:
-    provider: "minio"
+    provider: "garage"
     accesskey: "XJoi2@bgSOvOYBy#"
     secretkey: "OGGqat4lXRpL@9XBYc8FUaId@5"
   thumbnail:
-    provider: "minio"
+    provider: "garage"
     accessKey: "XJoi2@bgSOvOYBy#"
     secretKey: "OGGqat4lXRpL@9XBYc8FUaId@5"
   sprite:
-    provider: "minio"
+    provider: "garage"
     accessKey: "XJoi2@bgSOvOYBy#"
     secretKey: "OGGqat4lXRpL@9XBYc8FUaId@5"
 email:

base/kustomization.yaml

@@ -6,4 +6,4 @@ resources:
   - factory
   - vault
   - hub
-  - minio
+  - garage

configure.sh

@@ -39,6 +39,8 @@ if [ "$command" == "apply" ]; then
   # Adjust the base path reference
   sed -i "s|../../base|./base|g" ./kustomization.yaml
 
+  bash scripts/install-garage.sh --prepare-only || exit 1
+
   # Apply kustomize installation
   kubectl kustomize ./ --enable-helm --load-restrictor LoadRestrictionsNone | kubectl apply -f -
 

overlays/microk8s/kustomization.yaml

@@ -26,7 +26,7 @@ helmCharts:
     releaseName: hub
     namespace: kerberos-hub
     repo: https://charts.kerberos.io
-    version: 0.84.0
+    version: 0.170.0
     valuesFile: ../../base/hub/kerberos-hub-values.yaml
     valuesInline:
         mqtt:

overlays/standalone/kustomization.yaml

@@ -8,7 +8,7 @@ helmCharts:
     releaseName: hub
     namespace: kerberos-hub
     repo: https://charts.kerberos.io
-    version: 0.84.0
+    version: 0.170.0
     valuesFile: ../../base/hub/kerberos-hub-values.yaml
     valuesInline:
         kerberoshub:

scripts/check-garage.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $# -gt 1 || ( $# -eq 1 && "$1" != --restart ) ]]; then
+  echo "Usage: bash scripts/check-garage.sh [--restart]" >&2
+  exit 1
+fi
+temporary="$(mktemp -d)"
+pid=""
+cleanup() {
+  if [[ -n "$pid" ]]; then kill "$pid"; fi
+  rm -f "$temporary/upload" "$temporary/download" "$temporary/port-forward.log"
+  rmdir "$temporary"
+}
+trap cleanup EXIT
+
+access_key="$(kubectl -n garage get secret garage-credentials -o jsonpath='{.data.access-key}' | base64 --decode)"
+secret_key="$(kubectl -n garage get secret garage-credentials -o jsonpath='{.data.secret-key}' | base64 --decode)"
+if [[ "${GITHUB_ACTIONS:-false}" == true ]]; then
+  echo "::add-mask::$access_key"
+  echo "::add-mask::$secret_key"
+fi
+forward() {
+  kubectl -n garage port-forward service/garage 13900:3900 > "$temporary/port-forward.log" 2>&1 &
+  pid=$!
+  for attempt in {1..30}; do
+    if curl --silent --output /dev/null http://127.0.0.1:13900/; then return; fi
+    if ! kill -0 "$pid"; then break; fi
+    sleep 1
+  done
+  cat "$temporary/port-forward.log" >&2
+  echo "Garage port-forward did not become ready." >&2
+  exit 1
+}
+forward
+
+object="ci-smoke/$(openssl rand -hex 16)"
+openssl rand -hex 128 > "$temporary/upload"
+endpoint="http://127.0.0.1:13900/mybucket/$object"
+curl --fail-with-body --silent --show-error --aws-sigv4 "aws:amz:garage:s3" \
+  --user "$access_key:$secret_key" --upload-file "$temporary/upload" "$endpoint"
+if [[ $# -eq 1 ]]; then
+  kill "$pid"
+  wait "$pid" || [[ "$?" == 143 ]]
+  pid=""
+  kubectl -n garage rollout restart statefulset/garage
+  kubectl -n garage rollout status statefulset/garage --timeout=300s
+  forward
+fi
+curl --fail-with-body --silent --show-error --aws-sigv4 "aws:amz:garage:s3" \
+  --user "$access_key:$secret_key" "$endpoint" --output "$temporary/download"
+cmp "$temporary/upload" "$temporary/download"
+echo "Garage S3 authenticated PUT/GET passed through the Kubernetes Service (object mybucket/$object retained)."

scripts/diagnose-deployment.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -uo pipefail
+
+kubectl get pods,pvc -A -o wide
+kubectl get events -A --sort-by=.lastTimestamp
+for namespace in garage mongodb rabbitmq vernemq kerberos-vault kerberos-agent kerberos-hub; do
+  if ! pods="$(kubectl -n "$namespace" get pods -o name)"; then
+    echo "::warning::Could not list pods in $namespace."
+    continue
+  fi
+  for pod in $pods; do
+    if ! kubectl -n "$namespace" describe "$pod"; then
+      echo "::warning::Could not describe $namespace/$pod."
+    fi
+    if ! kubectl -n "$namespace" logs "$pod" --all-containers=true --tail=80 --prefix; then
+      echo "::warning::Logs unavailable for $namespace/$pod."
+    fi
+  done
+done

scripts/install-garage.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+if [[ $# -gt 1 || ( $# -eq 1 && "$1" != --prepare-only ) ]]; then
+  echo "Usage: bash scripts/install-garage.sh [--prepare-only]" >&2
+  exit 1
+fi
+kubectl apply -f "$ROOT/base/garage/namespace.yaml"
+
+secret="$(kubectl -n garage get secret garage-credentials --ignore-not-found -o name)"
+if [[ -z "$secret" ]]; then
+  pvc="$(kubectl -n garage get pvc data-garage-0 --ignore-not-found -o name)"
+  if [[ -n "$pvc" ]]; then
+    echo "Garage data already exists without its credentials. Restore garage-credentials from backup; refusing to generate replacement keys." >&2
+    exit 1
+  fi
+  kubectl -n garage create secret generic garage-credentials \
+    --from-literal="rpc-secret=$(openssl rand -hex 32)" \
+    --from-literal="access-key=GK$(openssl rand -hex 16)" \
+    --from-literal="secret-key=$(openssl rand -hex 32)"
+fi
+
+kubectl -n garage get secret garage-credentials -o json |
+  jq -e '.data | all(."rpc-secret", ."access-key", ."secret-key"; type == "string" and length > 0)' > /dev/null
+if [[ $# -eq 1 ]]; then
+  echo "Garage credentials are ready; apply your Kustomize overlay next."
+  exit 0
+fi
+kubectl apply -k "$ROOT/base/garage"
+kubectl -n garage rollout status statefulset/garage --timeout=300s
+kubectl -n garage exec garage-0 -- /garage bucket info mybucket
+echo "Garage is available at garage.garage:3900 (region garage, bucket mybucket). Credentials are in Secret garage/garage-credentials."

scripts/notify-slack.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $# != 1 ]]; then
+  echo "Usage: bash scripts/notify-slack.sh MESSAGE" >&2
+  exit 1
+fi
+
+if [[ -z "${SLACK_WEBHOOK_URL:-}" ]]; then
+  echo "::warning::SLACK_WEBHOOK_URL is not configured; skipping optional Slack notification."
+  exit 0
+fi
+
+jq -n --arg text "$1" '{text: $text}' |
+  curl --fail-with-body --silent --show-error --retry 3 \
+    -H 'Content-Type: application/json' --data-binary @- "$SLACK_WEBHOOK_URL"