From f20d156d86000df2f52c26ecf591e87431e8dfff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Verstraeten?= Date: Thu, 19 Nov 2026 11:36:48 +0000 Subject: [PATCH] Stop on 429 only for Kerberos Hub tus uploads runTusUpload also serves direct Kerberos Vault uploads. A 429 from a vault now keeps the existing in-cycle retries instead of ending the attempt, so it does not advance the vault's retry budget or secondary-storage failover early, and it never sets the Hub upload pause. Hub uploads still stop at once and pause until Retry-After (review feedback). Co-Authored-By: Claude Opus 5.5 --- machinery/src/cloud/tus_client.go | 18 ++++++++---- machinery/src/cloud/upload_pause_test.go | 37 ++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/machinery/src/cloud/tus_client.go b/machinery/src/cloud/tus_client.go index 8994b3f..585e010 100644 --- a/machinery/src/cloud/tus_client.go +++ b/machinery/src/cloud/tus_client.go @@ -120,7 +120,13 @@ type tusHeaderFunc func(h http.Header, fileName string) // - supported: the server exposes a tus endpoint. When false, the caller // should fall back to the legacy single-POST upload (older deployments). // - body: a short message for logging. -func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tusHeaderFunc) (uploaded bool, responded bool, supported bool, body string, err error) { +// +// stopOnRateLimit makes a 429 end the attempt at once (Kerberos Hub: the +// organisation reached its upload limit and the caller pauses uploads until +// Retry-After). Direct Vault uploads pass false, so a 429 from a vault keeps the +// existing in-cycle retries and does not advance the vault's retry budget or +// secondary-storage failover early. +func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tusHeaderFunc, stopOnRateLimit bool) (uploaded bool, responded bool, supported bool, body string, err error) { file, ferr := utils.OpenRecording("data/recordings", fileName) if file != nil { defer file.Close() @@ -177,7 +183,7 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu // The vault does not implement tus; let the caller fall back. return false, false, false, "", cerr } - if status == http.StatusTooManyRequests { + if stopOnRateLimit && status == http.StatusTooManyRequests { // Upload limit reached: retrying now cannot help. The caller // pauses uploads until Retry-After. return false, true, true, "", cerr @@ -201,7 +207,7 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu uploadURL = "" continue } - if status == http.StatusTooManyRequests { + if stopOnRateLimit && status == http.StatusTooManyRequests { // Keep the resume state so the upload continues after the pause. return false, true, true, "", herr } @@ -256,7 +262,7 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu lastStatus = status if perr != nil { lastErr = perr - if status == http.StatusTooManyRequests { + if stopOnRateLimit && status == http.StatusTooManyRequests { // The resume state saved at creation stays, so the upload // continues from the server offset after the pause. return false, true, true, "", perr @@ -351,7 +357,7 @@ func uploadVaultResumable(vault models.KStorage, publicKey, deviceKey, fileName, h[name] = append([]string(nil), values...) } } - return runTusUpload(baseURL, metadata, fileName, label, slot, setHeaders) + return runTusUpload(baseURL, metadata, fileName, label, slot, setHeaders, false) } // uploadHubResumable uploads a recording to Kerberos Hub's tus endpoint, which @@ -372,7 +378,7 @@ func uploadHubResumable(config *models.Config, fileName, label, slot string) (bo setHeaders := func(h http.Header, fn string) { setHubTusHeaders(h, config, fn) } - return runTusUpload(baseURL, metadata, fileName, label, slot, setHeaders) + return runTusUpload(baseURL, metadata, fileName, label, slot, setHeaders, true) } func addRecordingTusMetadata(values map[string]string, fileName string) { diff --git a/machinery/src/cloud/upload_pause_test.go b/machinery/src/cloud/upload_pause_test.go index 7a898c6..0af543a 100644 --- a/machinery/src/cloud/upload_pause_test.go +++ b/machinery/src/cloud/upload_pause_test.go @@ -348,3 +348,40 @@ func TestUploadKerberosHubOtherStatusesDoNotPause(t *testing.T) { t.Fatalf("pause after a non-429 answer = %v, want 0", remaining) } } + +// A direct Vault upload is not a Hub upload: a 429 from a vault keeps the +// existing in-cycle retries and never sets the Hub upload pause. +func TestVaultTusCreate429KeepsInCycleRetries(t *testing.T) { + withHubUploadPause(t) + previousDelay := tusBackoffBaseDelay + tusBackoffBaseDelay = 0 + t.Cleanup(func() { tusBackoffBaseDelay = previousDelay }) + + var posts int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + atomic.AddInt32(&posts, 1) + w.Header().Set("Retry-After", "3600") + http.Error(w, "slow down", http.StatusTooManyRequests) + return + } + t.Errorf("unexpected request %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusInternalServerError) + })) + defer server.Close() + + fileName := "1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4" + withRecording(t, fileName, bytes.Repeat([]byte("r"), 1024)) + + setHeaders := func(h http.Header, fn string) {} + uploaded, _, supported, _, _ := runTusUpload(server.URL+tusUploadPath, "", fileName, "vault-test", "primary", setHeaders, false) + if uploaded || !supported { + t.Fatalf("runTusUpload = uploaded %v, supported %v; want false, true", uploaded, supported) + } + if got := atomic.LoadInt32(&posts); got < 2 { + t.Fatalf("tus create requests = %d, want the in-cycle retries (> 1) for a vault", got) + } + if remaining := hubUploadPause.Remaining(); remaining != 0 { + t.Fatalf("hub upload pause = %v, want none for a vault upload", remaining) + } +}