Add safe recovery for hardened HTTP port upgrades

Reproduce duplicate named ports under strategic merge, add guarded JSON patch migration for frontend/demo/Admin, and validate recovery and rollback in Kubernetes CI without forcing workload replacement.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

.github/workflows/http-port-migration.yaml

@@ -0,0 +1,22 @@
+name: HTTP port migration
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  migration:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: azure/setup-helm@v4.3.1
+      - uses: helm/kind-action@v1
+      - name: Validate migration against Kubernetes
+        env:
+          SERVER_VALIDATION: "true"
+        run: bash scripts/check-http-port-migration.sh
+      - name: Validate chart runtime compatibility
+        run: |
+          helm lint charts/hub
+          bash scripts/check-admin-runtime.sh
+          bash scripts/check-hardened-workers.sh

charts/hub/README.md

@@ -128,6 +128,53 @@ security context, and disable `/healthz` probes together. Run
 `bash scripts/check-admin-runtime.sh` from the repository root to verify
 legacy defaults, hardened runtime/probes, and unchanged OAuth2 routing.
 
+### Recovering a duplicate HTTP port during upgrades
+
+An upgrade changing a container's HTTP port from `80` to `8080` can fail with
+`spec.template.spec.containers[0].ports[1].name: Duplicate value: "http"`.
+Kubernetes strategic merges identify container ports by number, not name:
+depending on the live state and apply mechanism, the old `80` entry can survive
+beside the new `8080` entry. Fresh chart renders contain only one HTTP port.
+This is not a reason to rename `http`: Services and probes reference that name.
+
+Inspect the failing Deployment and confirm the image's listening port first:
+
+```sh
+kubectl -n kerberos-hub get deployment hub-frontend -o yaml
+```
+
+In a checkout of this repository, use the targeted migration helper to replace
+the named HTTP entry (not strategically merge it). It supports `admin`,
+`hub-frontend` and `hub-frontend-demo`, preserves unrelated ports and containers,
+and rejects unfamiliar HTTP configurations. It requires `kubectl` and `jq` and
+uses the current kubeconfig context. The default is a **server dry run**:
+
+```sh
+bash scripts/migrate-http-port.sh kerberos-hub hub-frontend 8080
+# Apply only after inspecting the dry run and confirming the target/context.
+bash scripts/migrate-http-port.sh kerberos-hub hub-frontend 8080 --apply
+```
+
+Run this only for affected Deployments, using the port selected in your upgrade
+values. Then retry the Helm upgrade or GitOps sync with the coordinated image,
+port, security context and scratch-volume settings above. Pause GitOps auto-sync
+during manual recovery so it cannot race or undo the patch, and resume it after
+the desired state has synced. The helper uses resource-version/name tests to
+reject concurrent changes; re-inspect and retry if those tests fail.
+
+Changing the pod template triggers a rollout, and setting `8080` while the old
+image still listens on `80` can temporarily interrupt Service traffic. Schedule
+the recovery in a maintenance window and complete the matching image upgrade
+immediately. The helper does not change images, mount scratch volumes, or repair
+unrelated readiness failures. For a legacy-image rollback use port `80` instead,
+restore the matching runtime settings, and retry the rollback. Do not use blanket
+`helm upgrade --force` or resource replacement for unrelated workloads.
+
+Run `bash scripts/check-http-port-migration.sh` to reproduce the strategic-merge
+failure and test targeted replacement, idempotency, and reverse migration for all
+three Deployments. CI also checks API rejection of duplicate names and acceptance
+of the repaired spec against a disposable Kind cluster.
+
 ### Hardened Hub workers (opt-in, chart 0.167.3)
 
 The worker templates accept **per-component** `securityContext` (container) and

scripts/check-http-port-migration.sh

@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+REAL_KUBECTL="$(command -v kubectl)"
+export REAL_KUBECTL
+
+# Substitute only transport: exercise the migration against Kubernetes' real patch engine.
+kubectl() {
+  if [[ "$*" == *" get deployment "* ]]; then
+    printf '%s\n' "$LIVE"
+  else
+    local patch="" previous="" argument
+    for argument in "$@"; do
+      if [[ "$previous" == -p ]]; then patch="$argument"; fi
+      previous="$argument"
+    done
+    if [[ "${EXPECT_APPLY:-false}" == true ]]; then
+      [[ "$*" != *"--dry-run=server"* ]] || return 1
+    else
+      [[ "$*" == *"--dry-run=server"* ]] || return 1
+    fi
+    local current="$LIVE"
+    if [[ "${SIMULATE_RACE:-false}" == true ]]; then
+      current="$(jq '.metadata.resourceVersion = "2"' <<< "$LIVE")"
+    fi
+    "$REAL_KUBECTL" patch --local -f <(printf '%s\n' "$current") \
+      --type=json -p "$patch" -o json
+  fi
+}
+export -f kubectl
+
+for deployment in admin hub-frontend hub-frontend-demo; do
+  for old in 80 8080; do
+    target=8080
+    if [[ "$old" == 8080 ]]; then target=80; fi
+    LIVE="$(jq -nc --arg name "$deployment" --argjson port "$old" '{
+      apiVersion: "apps/v1", kind: "Deployment",
+      metadata: {name: $name, resourceVersion: "1"},
+      spec: {selector: {matchLabels: {app: $name}}, template: {
+        metadata: {labels: {app: $name}}, spec: {containers: [
+          {name: "sidecar", image: "example/sidecar:unchanged"},
+          {name: $name, image: "example/frontend:unchanged", ports: [
+            {name: "http", containerPort: $port},
+            {name: "metrics", containerPort: 2112}
+          ]}
+        ]}
+      }}
+    }')"
+    export LIVE
+    merge="$(jq -nc --arg name "$deployment" --argjson port "$target" \
+      '{spec:{template:{spec:{containers:[{name:$name,ports:[{name:"http",containerPort:$port}]}]}}}}')"
+    merged="$("$REAL_KUBECTL" patch --local -f <(printf '%s\n' "$LIVE") \
+      --type=strategic -p "$merge" -o json)"
+    jq -e '[.spec.template.spec.containers[] | .ports[]? |
+      select(.name == "http")] | length == 2' <<< "$merged" > /dev/null
+
+    fixed="$(bash "$ROOT/scripts/migrate-http-port.sh" test "$deployment" "$target")"
+    jq -e --argjson port "$target" '
+      [.spec.template.spec.containers[] | .ports[]? | select(.name == "http")] |
+      length == 1 and .[0].containerPort == $port
+    ' <<< "$fixed" > /dev/null
+    [[ "$(jq -Sc '.spec.template.spec.containers[0]' <<< "$fixed")" == \
+       "$(jq -Sc '.spec.template.spec.containers[0]' <<< "$LIVE")" ]]
+    [[ "$(jq -Sc '.spec.template.spec.containers[1] | del(.ports)' <<< "$fixed")" == \
+       "$(jq -Sc '.spec.template.spec.containers[1] | del(.ports)' <<< "$LIVE")" ]]
+    jq -e '.spec.template.spec.containers[1].ports |
+      any(.name == "metrics" and .containerPort == 2112)' <<< "$fixed" > /dev/null
+
+    LIVE="$fixed"
+    again="$(EXPECT_APPLY=true bash "$ROOT/scripts/migrate-http-port.sh" test "$deployment" "$target" --apply)"
+    [[ "$(jq -Sc . <<< "$again")" == "$(jq -Sc . <<< "$fixed")" ]]
+    LIVE="$merged"
+    repaired="$(EXPECT_APPLY=true bash "$ROOT/scripts/migrate-http-port.sh" test "$deployment" "$target" --apply)"
+    jq -e '[.spec.template.spec.containers[] | .ports[]? |
+      select(.name == "http")] | length == 1' <<< "$repaired" > /dev/null
+
+    if [[ "${SERVER_VALIDATION:-false}" == true ]]; then
+      if "$REAL_KUBECTL" create --dry-run=server \
+        -f <(jq 'del(.metadata.resourceVersion)' <<< "$merged") > /dev/null 2>&1; then
+        echo "Expected API rejection of duplicate http ports" >&2
+        exit 1
+      fi
+      "$REAL_KUBECTL" create --dry-run=server \
+        -f <(jq 'del(.metadata.resourceVersion)' <<< "$repaired") > /dev/null
+    fi
+    echo "OK: $deployment $old -> $target; duplicate reproduced, replacement and rollback safe"
+  done
+done
+
+if SIMULATE_RACE=true bash "$ROOT/scripts/migrate-http-port.sh" test hub-frontend-demo 8080 > /dev/null 2>&1; then
+  echo "Expected rejection of concurrent Deployment changes" >&2
+  exit 1
+fi
+LIVE="$(jq '.spec.template.spec.containers[1].ports = [{name:"http",containerPort:9000}]' <<< "$LIVE")"
+if bash "$ROOT/scripts/migrate-http-port.sh" test hub-frontend-demo 8080 > /dev/null 2>&1; then
+  echo "Expected rejection of an unknown HTTP port" >&2
+  exit 1
+fi
+echo "HTTP port migration checks passed."

scripts/migrate-http-port.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $# -lt 3 || $# -gt 4 ]]; then
+  echo "Usage: bash scripts/migrate-http-port.sh NAMESPACE DEPLOYMENT PORT [--apply]" >&2
+  exit 1
+fi
+namespace="$1" deployment="$2" port="$3"
+case "$deployment" in
+  admin|hub-frontend|hub-frontend-demo) ;;
+  *) echo "Unsupported Deployment: $deployment" >&2; exit 1 ;;
+esac
+case "$port" in
+  80|8080) ;;
+  *) echo "Expected HTTP port 80 or 8080, got: $port" >&2; exit 1 ;;
+esac
+dry_run=(--dry-run=server)
+if [[ $# -eq 4 ]]; then
+  if [[ "$4" != --apply ]]; then
+    echo "Unknown option: $4" >&2
+    exit 1
+  fi
+  dry_run=()
+fi
+
+live="$(kubectl -n "$namespace" get deployment "$deployment" -o json)"
+patch="$(jq -ce --arg container "$deployment" --argjson port "$port" '
+  . as $deployment |
+  [.spec.template.spec.containers | to_entries[] |
+    select(.value.name == $container)] |
+  if length != 1 then error("Expected exactly one matching container") else .[0] end |
+  .key as $index | .value.ports as $ports |
+  if ($ports | type) != "array" then error("Container ports must be an array") else . end |
+  [$ports[] | select(.name == "http")] as $http |
+  if ($http | length) == 0 then error("No named http port to migrate") else . end |
+  if any($http[]; (.containerPort != 80 and .containerPort != 8080) or
+    ((.protocol // "TCP") != "TCP") or has("hostPort") or has("hostIP"))
+  then error("Unsupported http port configuration; migrate manually") else . end |
+  if any($ports[]; .name != "http" and .containerPort == $port and
+    ((.protocol // "TCP") == "TCP"))
+  then error("Target port is already used by another named port") else . end |
+  ([$ports[] | select(.name != "http")] +
+    [($http[0] + {containerPort: $port})]) as $replacement |
+  [
+    {op: "test", path: "/metadata/resourceVersion", value: $deployment.metadata.resourceVersion},
+    {op: "test", path: ("/spec/template/spec/containers/" + ($index | tostring) + "/name"),
+      value: $container},
+    {op: "replace", path: ("/spec/template/spec/containers/" + ($index | tostring) + "/ports"),
+      value: $replacement}
+  ]
+' <<< "$live")"
+
+kubectl -n "$namespace" patch deployment "$deployment" --type=json \
+  -p "$patch" "${dry_run[@]}" -o yaml