mirror of
https://github.com/kerberos-io/agent.git
synced 2026-11-19 22:30:45 +00:00
Stop on 429 only for Kerberos Hub tus uploads
runTusUpload also serves direct Kerberos Vault uploads. A 429 from a vault now keeps the existing in-cycle retries instead of ending the attempt, so it does not advance the vault's retry budget or secondary-storage failover early, and it never sets the Hub upload pause. Hub uploads still stop at once and pause until Retry-After (review feedback). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
98f1826fe1
commit
f20d156d86
@@ -120,7 +120,13 @@ type tusHeaderFunc func(h http.Header, fileName string)
|
||||
// - supported: the server exposes a tus endpoint. When false, the caller
|
||||
// should fall back to the legacy single-POST upload (older deployments).
|
||||
// - body: a short message for logging.
|
||||
func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tusHeaderFunc) (uploaded bool, responded bool, supported bool, body string, err error) {
|
||||
//
|
||||
// stopOnRateLimit makes a 429 end the attempt at once (Kerberos Hub: the
|
||||
// organisation reached its upload limit and the caller pauses uploads until
|
||||
// Retry-After). Direct Vault uploads pass false, so a 429 from a vault keeps the
|
||||
// existing in-cycle retries and does not advance the vault's retry budget or
|
||||
// secondary-storage failover early.
|
||||
func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tusHeaderFunc, stopOnRateLimit bool) (uploaded bool, responded bool, supported bool, body string, err error) {
|
||||
file, ferr := utils.OpenRecording("data/recordings", fileName)
|
||||
if file != nil {
|
||||
defer file.Close()
|
||||
@@ -177,7 +183,7 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
|
||||
// The vault does not implement tus; let the caller fall back.
|
||||
return false, false, false, "", cerr
|
||||
}
|
||||
if status == http.StatusTooManyRequests {
|
||||
if stopOnRateLimit && status == http.StatusTooManyRequests {
|
||||
// Upload limit reached: retrying now cannot help. The caller
|
||||
// pauses uploads until Retry-After.
|
||||
return false, true, true, "", cerr
|
||||
@@ -201,7 +207,7 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
|
||||
uploadURL = ""
|
||||
continue
|
||||
}
|
||||
if status == http.StatusTooManyRequests {
|
||||
if stopOnRateLimit && status == http.StatusTooManyRequests {
|
||||
// Keep the resume state so the upload continues after the pause.
|
||||
return false, true, true, "", herr
|
||||
}
|
||||
@@ -256,7 +262,7 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
|
||||
lastStatus = status
|
||||
if perr != nil {
|
||||
lastErr = perr
|
||||
if status == http.StatusTooManyRequests {
|
||||
if stopOnRateLimit && status == http.StatusTooManyRequests {
|
||||
// The resume state saved at creation stays, so the upload
|
||||
// continues from the server offset after the pause.
|
||||
return false, true, true, "", perr
|
||||
@@ -351,7 +357,7 @@ func uploadVaultResumable(vault models.KStorage, publicKey, deviceKey, fileName,
|
||||
h[name] = append([]string(nil), values...)
|
||||
}
|
||||
}
|
||||
return runTusUpload(baseURL, metadata, fileName, label, slot, setHeaders)
|
||||
return runTusUpload(baseURL, metadata, fileName, label, slot, setHeaders, false)
|
||||
}
|
||||
|
||||
// uploadHubResumable uploads a recording to Kerberos Hub's tus endpoint, which
|
||||
@@ -372,7 +378,7 @@ func uploadHubResumable(config *models.Config, fileName, label, slot string) (bo
|
||||
setHeaders := func(h http.Header, fn string) {
|
||||
setHubTusHeaders(h, config, fn)
|
||||
}
|
||||
return runTusUpload(baseURL, metadata, fileName, label, slot, setHeaders)
|
||||
return runTusUpload(baseURL, metadata, fileName, label, slot, setHeaders, true)
|
||||
}
|
||||
|
||||
func addRecordingTusMetadata(values map[string]string, fileName string) {
|
||||
|
||||
@@ -348,3 +348,40 @@ func TestUploadKerberosHubOtherStatusesDoNotPause(t *testing.T) {
|
||||
t.Fatalf("pause after a non-429 answer = %v, want 0", remaining)
|
||||
}
|
||||
}
|
||||
|
||||
// A direct Vault upload is not a Hub upload: a 429 from a vault keeps the
|
||||
// existing in-cycle retries and never sets the Hub upload pause.
|
||||
func TestVaultTusCreate429KeepsInCycleRetries(t *testing.T) {
|
||||
withHubUploadPause(t)
|
||||
previousDelay := tusBackoffBaseDelay
|
||||
tusBackoffBaseDelay = 0
|
||||
t.Cleanup(func() { tusBackoffBaseDelay = previousDelay })
|
||||
|
||||
var posts int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodPost {
|
||||
atomic.AddInt32(&posts, 1)
|
||||
w.Header().Set("Retry-After", "3600")
|
||||
http.Error(w, "slow down", http.StatusTooManyRequests)
|
||||
return
|
||||
}
|
||||
t.Errorf("unexpected request %s %s", r.Method, r.URL.Path)
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
fileName := "1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4"
|
||||
withRecording(t, fileName, bytes.Repeat([]byte("r"), 1024))
|
||||
|
||||
setHeaders := func(h http.Header, fn string) {}
|
||||
uploaded, _, supported, _, _ := runTusUpload(server.URL+tusUploadPath, "", fileName, "vault-test", "primary", setHeaders, false)
|
||||
if uploaded || !supported {
|
||||
t.Fatalf("runTusUpload = uploaded %v, supported %v; want false, true", uploaded, supported)
|
||||
}
|
||||
if got := atomic.LoadInt32(&posts); got < 2 {
|
||||
t.Fatalf("tus create requests = %d, want the in-cycle retries (> 1) for a vault", got)
|
||||
}
|
||||
if remaining := hubUploadPause.Remaining(); remaining != 0 {
|
||||
t.Fatalf("hub upload pause = %v, want none for a vault upload", remaining)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user