Stop on 429 only for Kerberos Hub tus uploads

runTusUpload also serves direct Kerberos Vault uploads. A 429 from a vault
now keeps the existing in-cycle retries instead of ending the attempt, so it
does not advance the vault's retry budget or secondary-storage failover
early, and it never sets the Hub upload pause. Hub uploads still stop at
once and pause until Retry-After (review feedback).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Cédric Verstraeten
2026-11-19 11:36:48 +00:00
co-authored by Claude Opus 5.5
parent 98f1826fe1
commit f20d156d86
2 changed files with 49 additions and 6 deletions
+12 -6
View File
@@ -120,7 +120,13 @@ type tusHeaderFunc func(h http.Header, fileName string)
// - supported: the server exposes a tus endpoint. When false, the caller
// should fall back to the legacy single-POST upload (older deployments).
// - body: a short message for logging.
func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tusHeaderFunc) (uploaded bool, responded bool, supported bool, body string, err error) {
//
// stopOnRateLimit makes a 429 end the attempt at once (Kerberos Hub: the
// organisation reached its upload limit and the caller pauses uploads until
// Retry-After). Direct Vault uploads pass false, so a 429 from a vault keeps the
// existing in-cycle retries and does not advance the vault's retry budget or
// secondary-storage failover early.
func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tusHeaderFunc, stopOnRateLimit bool) (uploaded bool, responded bool, supported bool, body string, err error) {
file, ferr := utils.OpenRecording("data/recordings", fileName)
if file != nil {
defer file.Close()
@@ -177,7 +183,7 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
// The vault does not implement tus; let the caller fall back.
return false, false, false, "", cerr
}
if status == http.StatusTooManyRequests {
if stopOnRateLimit && status == http.StatusTooManyRequests {
// Upload limit reached: retrying now cannot help. The caller
// pauses uploads until Retry-After.
return false, true, true, "", cerr
@@ -201,7 +207,7 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
uploadURL = ""
continue
}
if status == http.StatusTooManyRequests {
if stopOnRateLimit && status == http.StatusTooManyRequests {
// Keep the resume state so the upload continues after the pause.
return false, true, true, "", herr
}
@@ -256,7 +262,7 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
lastStatus = status
if perr != nil {
lastErr = perr
if status == http.StatusTooManyRequests {
if stopOnRateLimit && status == http.StatusTooManyRequests {
// The resume state saved at creation stays, so the upload
// continues from the server offset after the pause.
return false, true, true, "", perr
@@ -351,7 +357,7 @@ func uploadVaultResumable(vault models.KStorage, publicKey, deviceKey, fileName,
h[name] = append([]string(nil), values...)
}
}
return runTusUpload(baseURL, metadata, fileName, label, slot, setHeaders)
return runTusUpload(baseURL, metadata, fileName, label, slot, setHeaders, false)
}
// uploadHubResumable uploads a recording to Kerberos Hub's tus endpoint, which
@@ -372,7 +378,7 @@ func uploadHubResumable(config *models.Config, fileName, label, slot string) (bo
setHeaders := func(h http.Header, fn string) {
setHubTusHeaders(h, config, fn)
}
return runTusUpload(baseURL, metadata, fileName, label, slot, setHeaders)
return runTusUpload(baseURL, metadata, fileName, label, slot, setHeaders, true)
}
func addRecordingTusMetadata(values map[string]string, fileName string) {
+37
View File
@@ -348,3 +348,40 @@ func TestUploadKerberosHubOtherStatusesDoNotPause(t *testing.T) {
t.Fatalf("pause after a non-429 answer = %v, want 0", remaining)
}
}
// A direct Vault upload is not a Hub upload: a 429 from a vault keeps the
// existing in-cycle retries and never sets the Hub upload pause.
func TestVaultTusCreate429KeepsInCycleRetries(t *testing.T) {
withHubUploadPause(t)
previousDelay := tusBackoffBaseDelay
tusBackoffBaseDelay = 0
t.Cleanup(func() { tusBackoffBaseDelay = previousDelay })
var posts int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodPost {
atomic.AddInt32(&posts, 1)
w.Header().Set("Retry-After", "3600")
http.Error(w, "slow down", http.StatusTooManyRequests)
return
}
t.Errorf("unexpected request %s %s", r.Method, r.URL.Path)
w.WriteHeader(http.StatusInternalServerError)
}))
defer server.Close()
fileName := "1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4"
withRecording(t, fileName, bytes.Repeat([]byte("r"), 1024))
setHeaders := func(h http.Header, fn string) {}
uploaded, _, supported, _, _ := runTusUpload(server.URL+tusUploadPath, "", fileName, "vault-test", "primary", setHeaders, false)
if uploaded || !supported {
t.Fatalf("runTusUpload = uploaded %v, supported %v; want false, true", uploaded, supported)
}
if got := atomic.LoadInt32(&posts); got < 2 {
t.Fatalf("tus create requests = %d, want the in-cycle retries (> 1) for a vault", got)
}
if remaining := hubUploadPause.Remaining(); remaining != 0 {
t.Fatalf("hub upload pause = %v, want none for a vault upload", remaining)
}
}