diff --git a/machinery/src/cloud/cloud.go b/machinery/src/cloud/cloud.go index c3d00e8..2c2fbdc 100644 --- a/machinery/src/cloud/cloud.go +++ b/machinery/src/cloud/cloud.go @@ -57,14 +57,27 @@ func HandleUpload(configDirectory string, configuration *models.Configuration, c // Half a second delay between two uploads delay := 500 * time.Millisecond + // Kerberos Hub answers 429 + Retry-After when the organisation reached + // its upload limit; uploads to Hub then pause until that time. A + // (re)started loop, e.g. after a reconfiguration, tries again at once. + uploadsToHub := config.Cloud == "s3" || config.Cloud == "kerberoshub" + hubUploadPause.Reset() + loop: for { + wait := 2 * time.Second + if uploadsToHub { + if remaining := hubUploadPause.Remaining(); remaining > wait { + wait = remaining + } + } + // This will check if we need to stop the thread, // because of a reconfiguration. select { case <-communication.HandleUpload: break loop - case <-time.After(2 * time.Second): + case <-time.After(wait): } ff, err := utils.ReadDirectory(watchDirectory) @@ -142,6 +155,12 @@ func HandleUpload(configDirectory string, configuration *models.Configuration, c } } + if uploadsToHub && hubUploadPause.Remaining() > 0 { + // Paused: leave this and the following recordings queued + // on disk; the outer loop waits until the pause ends. + break + } + time.Sleep(delay) } } @@ -1813,6 +1832,8 @@ func hubPersistenceVerificationError(statusCode int) string { reason += ": the account has no active subscription or the subscription expired" case http.StatusForbidden: reason += ": the device is muted" + case http.StatusTooManyRequests: + reason += ": the organisation reached its upload limit for this period" } return "cloud.VerifyPersistence(kerberoshub): " + reason + " (HTTP " + strconv.Itoa(statusCode) + ")" } diff --git a/machinery/src/cloud/kerberos_hub.go b/machinery/src/cloud/kerberos_hub.go index 6f7749a..22e1b5c 100644 --- a/machinery/src/cloud/kerberos_hub.go +++ b/machinery/src/cloud/kerberos_hub.go @@ -45,8 +45,17 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo // endpoint (older deployments) we transparently fall back to the legacy // single-POST upload below. if resumableUploadsEnabled() { - uploaded, _, supported, body, rerr := uploadHubResumable(&config, fileName, "UploadKerberosHub", "hub") + uploaded, responded, supported, body, rerr := uploadHubResumable(&config, fileName, "UploadKerberosHub", "hub") if supported { + var throttled *uploadThrottledError + if errors.As(rerr, &throttled) { + // Upload limit reached: keep the recording and pause uploads. + hubUploadPause.Throttled(throttled.retryAfter, throttled.ok) + return false, true, nil + } + if uploaded || responded { + hubUploadPause.Clear() + } if uploaded { log.WithFields(log.Fields{ "component": "kerberos_hub", @@ -125,6 +134,11 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo } if err == nil && resp != nil { + if resp.StatusCode == http.StatusTooManyRequests { + // Upload limit reached: keep the recording and pause uploads. + hubUploadPause.Throttled(retryAfterFromResponse(resp)) + return false, true, nil + } if resp.StatusCode == 200 { log.WithFields(log.Fields{ "component": "kerberos_hub", @@ -132,6 +146,7 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo "status_code": resp.StatusCode, }).Debug("Hub upload authorized") } else { + hubUploadPause.Clear() log.WithFields(log.Fields{ "component": "kerberos_hub", "event": "upload_rejected", @@ -165,6 +180,12 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo if err == nil { if resp != nil { + if resp.StatusCode == http.StatusTooManyRequests { + // Upload limit reached: keep the recording and pause uploads. + hubUploadPause.Throttled(retryAfterFromResponse(resp)) + return false, true, nil + } + hubUploadPause.Clear() body, err := ioutil.ReadAll(resp.Body) if err == nil { if resp.StatusCode == 200 { diff --git a/machinery/src/cloud/tus_client.go b/machinery/src/cloud/tus_client.go index bd788b9..8994b3f 100644 --- a/machinery/src/cloud/tus_client.go +++ b/machinery/src/cloud/tus_client.go @@ -177,6 +177,11 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu // The vault does not implement tus; let the caller fall back. return false, false, false, "", cerr } + if status == http.StatusTooManyRequests { + // Upload limit reached: retrying now cannot help. The caller + // pauses uploads until Retry-After. + return false, true, true, "", cerr + } log.Info(label + ": resumable create failed, " + cerr.Error()) tusBackoff(attempt) continue @@ -196,6 +201,10 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu uploadURL = "" continue } + if status == http.StatusTooManyRequests { + // Keep the resume state so the upload continues after the pause. + return false, true, true, "", herr + } log.Info(label + ": resumable head failed, " + herr.Error()) tusBackoff(attempt) continue @@ -247,6 +256,11 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu lastStatus = status if perr != nil { lastErr = perr + if status == http.StatusTooManyRequests { + // The resume state saved at creation stays, so the upload + // continues from the server offset after the pause. + return false, true, true, "", perr + } if status >= 400 { // Definitive rejection (e.g. provider push failed during finalize). // Re-evaluate via HEAD on the next iteration to decide retry/restart. @@ -408,7 +422,7 @@ func tusCreate(client *http.Client, baseURL string, size int64, metadata string, const maxErrorBodyBytes = 4096 body, readErr := io.ReadAll(io.LimitReader(resp.Body, maxErrorBodyBytes+1)) if readErr != nil { - return "", resp.StatusCode, fmt.Errorf("unexpected status creating upload: %s; read error response: %w", resp.Status, readErr) + return "", resp.StatusCode, throttledError(resp, fmt.Errorf("unexpected status creating upload: %s; read error response: %w", resp.Status, readErr)) } truncated := len(body) > maxErrorBodyBytes if truncated { @@ -419,9 +433,9 @@ func tusCreate(client *http.Client, baseURL string, size int64, metadata string, message += " [truncated]" } if message != "" { - return "", resp.StatusCode, fmt.Errorf("unexpected status creating upload: %s: %q", resp.Status, message) + return "", resp.StatusCode, throttledError(resp, fmt.Errorf("unexpected status creating upload: %s: %q", resp.Status, message)) } - return "", resp.StatusCode, fmt.Errorf("unexpected status creating upload: %s", resp.Status) + return "", resp.StatusCode, throttledError(resp, fmt.Errorf("unexpected status creating upload: %s", resp.Status)) } // Location identifies the created upload; its response body is optional. if _, err := io.Copy(io.Discard, resp.Body); err != nil { @@ -454,7 +468,7 @@ func tusHead(client *http.Client, uploadURL string, setHeaders tusHeaderFunc) (i io.Copy(io.Discard, resp.Body) if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { - return 0, resp.StatusCode, fmt.Errorf("unexpected status on HEAD: %s", resp.Status) + return 0, resp.StatusCode, throttledError(resp, fmt.Errorf("unexpected status on HEAD: %s", resp.Status)) } offsetStr := resp.Header.Get("Upload-Offset") offset, perr := strconv.ParseInt(offsetStr, 10, 64) @@ -489,7 +503,7 @@ func tusPatch(client *http.Client, uploadURL string, offset, length int64, file respBody := string(bodyBytes) if resp.StatusCode != http.StatusNoContent { - return offset, resp.StatusCode, respBody, fmt.Errorf("unexpected status on PATCH: %s, %s", resp.Status, respBody) + return offset, resp.StatusCode, respBody, throttledError(resp, fmt.Errorf("unexpected status on PATCH: %s, %s", resp.Status, respBody)) } newOffsetStr := resp.Header.Get("Upload-Offset") newOffset, perr := strconv.ParseInt(newOffsetStr, 10, 64) diff --git a/machinery/src/cloud/upload_pause.go b/machinery/src/cloud/upload_pause.go new file mode 100644 index 0000000..67ca09b --- /dev/null +++ b/machinery/src/cloud/upload_pause.go @@ -0,0 +1,209 @@ +package cloud + +import ( + "fmt" + "net/http" + "strconv" + "strings" + "sync" + "time" + + log "github.com/sirupsen/logrus" +) + +// Kerberos Hub answers a recording upload with 429 Too Many Requests when the +// organisation is over its daily or monthly upload limit, and sets Retry-After +// to the start of the next period. The agent then pauses uploads to Hub until +// that moment instead of retrying every few seconds. Recordings stay queued on +// disk and are uploaded, in the usual order, once the pause is over. +const ( + // uploadPauseMinBackoff is the first back-off when a 429 carries no usable + // Retry-After. It equals the existing retry interval after a refused upload. + uploadPauseMinBackoff = 5 * time.Second + // uploadPauseMaxBackoff caps the exponential back-off without Retry-After. + uploadPauseMaxBackoff = 15 * time.Minute + // uploadPauseMaxRetryAfter caps an explicit Retry-After. A monthly limit + // never needs more than 31 days; anything longer is treated as bogus. + uploadPauseMaxRetryAfter = 32 * 24 * time.Hour +) + +// hubUploadPause is shared by every uploader that talks to Kerberos Hub. +var hubUploadPause = newUploadPause(time.Now) + +// uploadPause holds the deadline until which uploads to one target are paused. +// It is safe for concurrent use. +type uploadPause struct { + mu sync.Mutex + now func() time.Time + until time.Time + backoff time.Duration // next back-off when Retry-After is missing + active bool // a 429 episode is in progress (for start/end logs) +} + +func newUploadPause(now func() time.Time) *uploadPause { + return &uploadPause{now: now} +} + +// Remaining returns how long uploads stay paused; zero when not paused. +func (p *uploadPause) Remaining() time.Duration { + p.mu.Lock() + defer p.mu.Unlock() + if !p.active { + return 0 + } + if remaining := p.until.Sub(p.now()); remaining > 0 { + return remaining + } + return 0 +} + +// Throttled records a 429 answer. retryAfter is the delay parsed from the +// Retry-After header; ok is false when the header was missing or invalid, in +// which case an exponential back-off is used. It returns the pause duration. +func (p *uploadPause) Throttled(retryAfter time.Duration, ok bool) time.Duration { + p.mu.Lock() + defer p.mu.Unlock() + + var wait time.Duration + if ok { + wait = clampRetryAfter(retryAfter) + } else { + if p.backoff < uploadPauseMinBackoff { + p.backoff = uploadPauseMinBackoff + } + wait = p.backoff + p.backoff *= 2 + if p.backoff > uploadPauseMaxBackoff { + p.backoff = uploadPauseMaxBackoff + } + } + + now := p.now() + p.until = now.Add(wait) + fields := log.Fields{ + "component": "upload", + "target": "kerberos_hub", + "status_code": http.StatusTooManyRequests, + "paused_until": now.Add(wait).Format(time.RFC3339), + "pause": wait.Round(time.Second).String(), + "retry_after_present": ok, + } + if !p.active { + p.active = true + log.WithFields(fields).WithField("event", "upload_paused"). + Warn("Kerberos Hub refused the upload (upload limit reached); pausing uploads, recordings stay queued on disk") + } else { + log.WithFields(fields).WithField("event", "upload_pause_extended"). + Debug("Kerberos Hub still refuses uploads; pause extended") + } + return wait +} + +// Clear ends a pause after Hub answered an upload with anything but 429. +func (p *uploadPause) Clear() { + p.mu.Lock() + defer p.mu.Unlock() + if p.active { + log.WithFields(log.Fields{ + "component": "upload", + "target": "kerberos_hub", + "event": "upload_resumed", + }).Info("Kerberos Hub accepts uploads again; upload pause ended") + } + p.active = false + p.until = time.Time{} + p.backoff = 0 +} + +// Reset drops any pause without logging, for example when the upload loop is +// restarted after a reconfiguration. +func (p *uploadPause) Reset() { + p.mu.Lock() + defer p.mu.Unlock() + p.active = false + p.until = time.Time{} + p.backoff = 0 +} + +func clampRetryAfter(d time.Duration) time.Duration { + if d < uploadPauseMinBackoff { + // Zero or a date in the past: wait as long as a normal refused upload. + return uploadPauseMinBackoff + } + if d > uploadPauseMaxRetryAfter { + return uploadPauseMaxRetryAfter + } + return d +} + +// parseRetryAfter parses a Retry-After value (RFC 9110, section 10.2.3): +// either delay-seconds or an HTTP-date. now is the reference time for an +// HTTP-date; callers pass the response Date header when present, so a wrong +// clock on the agent does not shift the deadline. A negative number or any +// other value is invalid. The result can be negative for a date in the past; +// Throttled clamps it. +func parseRetryAfter(value string, now time.Time) (time.Duration, bool) { + value = strings.TrimSpace(value) + if value == "" { + return 0, false + } + if allDigits(value) { + seconds, err := strconv.ParseInt(value, 10, 64) + if err != nil || seconds > int64(uploadPauseMaxRetryAfter/time.Second) { + // Too large to be meaningful (or to fit a Duration): cap it. + return uploadPauseMaxRetryAfter, true + } + return time.Duration(seconds) * time.Second, true + } + at, err := http.ParseTime(value) + if err != nil { + return 0, false + } + return at.Sub(now), true +} + +func allDigits(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] < '0' || s[i] > '9' { + return false + } + } + return s != "" +} + +// retryAfterFromResponse reads Retry-After from a response, using the +// response Date header (when valid) as the reference for an HTTP-date. +func retryAfterFromResponse(resp *http.Response) (time.Duration, bool) { + now := time.Now() + if date, err := http.ParseTime(resp.Header.Get("Date")); err == nil { + now = date + } + return parseRetryAfter(resp.Header.Get("Retry-After"), now) +} + +// uploadThrottledError marks an upload refused with 429 Too Many Requests and +// carries the parsed Retry-After. +type uploadThrottledError struct { + retryAfter time.Duration + ok bool + err error +} + +func (e *uploadThrottledError) Error() string { + if e.err != nil { + return e.err.Error() + } + return fmt.Sprintf("upload refused: %d %s", http.StatusTooManyRequests, http.StatusText(http.StatusTooManyRequests)) +} + +func (e *uploadThrottledError) Unwrap() error { return e.err } + +// throttledError wraps err in an uploadThrottledError when resp is a 429; +// otherwise it returns err unchanged. +func throttledError(resp *http.Response, err error) error { + if resp == nil || resp.StatusCode != http.StatusTooManyRequests { + return err + } + retryAfter, ok := retryAfterFromResponse(resp) + return &uploadThrottledError{retryAfter: retryAfter, ok: ok, err: err} +} diff --git a/machinery/src/cloud/upload_pause_test.go b/machinery/src/cloud/upload_pause_test.go new file mode 100644 index 0000000..7a898c6 --- /dev/null +++ b/machinery/src/cloud/upload_pause_test.go @@ -0,0 +1,350 @@ +package cloud + +import ( + "bytes" + "errors" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/kerberos-io/agent/machinery/src/models" +) + +func TestParseRetryAfter(t *testing.T) { + now := time.Date(2026, 11, 16, 22, 0, 0, 0, time.UTC) + tests := []struct { + name string + value string + want time.Duration + wantOK bool + }{ + {"delay seconds", "120", 120 * time.Second, true}, + {"delay seconds with spaces", " 3600 ", time.Hour, true}, + {"zero", "0", 0, true}, + {"IMF-fixdate", "Mon, 16 Nov 2026 23:00:00 GMT", time.Hour, true}, + {"RFC 850 date", "Monday, 16-Nov-26 23:30:00 GMT", 90 * time.Minute, true}, + {"asctime date", "Mon Nov 16 22:00:30 2026", 30 * time.Second, true}, + {"date in the past", "Mon, 16 Nov 2026 21:00:00 GMT", -time.Hour, true}, + {"more than the cap", "4000000", uploadPauseMaxRetryAfter, true}, + {"overflowing number", "99999999999999999999999999", uploadPauseMaxRetryAfter, true}, + {"missing", "", 0, false}, + {"negative", "-5", 0, false}, + {"fraction", "1.5", 0, false}, + {"unit suffix", "10s", 0, false}, + {"garbage", "tomorrow", 0, false}, + {"ISO date", "2026-11-17T00:00:00Z", 0, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, ok := parseRetryAfter(tt.value, now) + if ok != tt.wantOK || got != tt.want { + t.Fatalf("parseRetryAfter(%q) = %v, %v; want %v, %v", tt.value, got, ok, tt.want, tt.wantOK) + } + }) + } +} + +func TestClampRetryAfter(t *testing.T) { + tests := []struct { + in, want time.Duration + }{ + {-time.Hour, uploadPauseMinBackoff}, + {0, uploadPauseMinBackoff}, + {time.Second, uploadPauseMinBackoff}, + {2 * time.Hour, 2 * time.Hour}, + {uploadPauseMaxRetryAfter, uploadPauseMaxRetryAfter}, + {40 * 24 * time.Hour, uploadPauseMaxRetryAfter}, + } + for _, tt := range tests { + if got := clampRetryAfter(tt.in); got != tt.want { + t.Errorf("clampRetryAfter(%v) = %v, want %v", tt.in, got, tt.want) + } + } +} + +func TestRetryAfterFromResponseUsesDateHeader(t *testing.T) { + // The agent clock may be wrong; an HTTP-date is measured against the + // response Date so the pause still ends at the start of Hub's next period. + resp := &http.Response{Header: http.Header{}} + resp.Header.Set("Date", "Mon, 16 Nov 2020 22:00:00 GMT") + resp.Header.Set("Retry-After", "Tue, 17 Nov 2020 00:00:00 GMT") + got, ok := retryAfterFromResponse(resp) + if !ok || got != 2*time.Hour { + t.Fatalf("retryAfterFromResponse = %v, %v; want 2h, true", got, ok) + } +} + +type fakeClock struct { + mu sync.Mutex + now time.Time +} + +func (c *fakeClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.now +} + +func (c *fakeClock) Advance(d time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.now = c.now.Add(d) +} + +func TestUploadPauseHonoursRetryAfter(t *testing.T) { + clock := &fakeClock{now: time.Date(2026, 11, 16, 22, 0, 0, 0, time.UTC)} + p := newUploadPause(clock.Now) + + if got := p.Remaining(); got != 0 { + t.Fatalf("Remaining before any 429 = %v, want 0", got) + } + if got := p.Throttled(2*time.Hour, true); got != 2*time.Hour { + t.Fatalf("Throttled = %v, want 2h", got) + } + if got := p.Remaining(); got != 2*time.Hour { + t.Fatalf("Remaining = %v, want 2h", got) + } + clock.Advance(90 * time.Minute) + if got := p.Remaining(); got != 30*time.Minute { + t.Fatalf("Remaining after 90m = %v, want 30m", got) + } + clock.Advance(30 * time.Minute) + if got := p.Remaining(); got != 0 { + t.Fatalf("Remaining at the deadline = %v, want 0", got) + } + + // Absurd values are clamped. + if got := p.Throttled(-time.Hour, true); got != uploadPauseMinBackoff { + t.Fatalf("Throttled(past) = %v, want %v", got, uploadPauseMinBackoff) + } + if got := p.Throttled(365*24*time.Hour, true); got != uploadPauseMaxRetryAfter { + t.Fatalf("Throttled(1 year) = %v, want %v", got, uploadPauseMaxRetryAfter) + } + + p.Clear() + if got := p.Remaining(); got != 0 { + t.Fatalf("Remaining after Clear = %v, want 0", got) + } +} + +func TestUploadPauseBacksOffWithoutRetryAfter(t *testing.T) { + clock := &fakeClock{now: time.Date(2026, 11, 16, 22, 0, 0, 0, time.UTC)} + p := newUploadPause(clock.Now) + + want := []time.Duration{ + 5 * time.Second, 10 * time.Second, 20 * time.Second, 40 * time.Second, + 80 * time.Second, 160 * time.Second, 320 * time.Second, 640 * time.Second, + 15 * time.Minute, 15 * time.Minute, + } + for i, w := range want { + if got := p.Throttled(0, false); got != w { + t.Fatalf("attempt %d: Throttled = %v, want %v", i, got, w) + } + if got := p.Remaining(); got != w { + t.Fatalf("attempt %d: Remaining = %v, want %v", i, got, w) + } + clock.Advance(w) + } + + // A valid Retry-After does not disturb the back-off sequence. + if got := p.Throttled(time.Hour, true); got != time.Hour { + t.Fatalf("Throttled with Retry-After = %v, want 1h", got) + } + if got := p.Throttled(0, false); got != 15*time.Minute { + t.Fatalf("back-off after Retry-After = %v, want 15m", got) + } + + // Clear (Hub answered something other than 429) restarts the back-off. + p.Clear() + if got := p.Throttled(0, false); got != 5*time.Second { + t.Fatalf("back-off after Clear = %v, want 5s", got) + } + p.Reset() + if got := p.Remaining(); got != 0 { + t.Fatalf("Remaining after Reset = %v, want 0", got) + } +} + +func TestUploadPauseConcurrentUse(t *testing.T) { + p := newUploadPause(time.Now) + var wg sync.WaitGroup + for i := 0; i < 16; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + for j := 0; j < 100; j++ { + switch (i + j) % 4 { + case 0: + p.Throttled(time.Minute, true) + case 1: + p.Throttled(0, false) + case 2: + p.Remaining() + case 3: + p.Clear() + } + } + }(i) + } + wg.Wait() +} + +func TestThrottledErrorOnlyFor429(t *testing.T) { + base := errors.New("boom") + resp := &http.Response{StatusCode: http.StatusForbidden, Header: http.Header{}} + if got := throttledError(resp, base); got != base { + t.Fatalf("throttledError(403) = %v, want the original error", got) + } + resp = &http.Response{StatusCode: http.StatusTooManyRequests, Header: http.Header{}} + resp.Header.Set("Retry-After", "60") + var throttled *uploadThrottledError + if err := throttledError(resp, base); !errors.As(err, &throttled) || !errors.Is(err, base) { + t.Fatalf("throttledError(429) = %v, want an uploadThrottledError wrapping the original", err) + } + if !throttled.ok || throttled.retryAfter != time.Minute { + t.Fatalf("retryAfter = %v, %v; want 1m, true", throttled.retryAfter, throttled.ok) + } +} + +// withHubUploadPause gives a test a fresh shared pause and restores it after. +func withHubUploadPause(t *testing.T) { + t.Helper() + previous := hubUploadPause + hubUploadPause = newUploadPause(time.Now) + t.Cleanup(func() { hubUploadPause = previous }) +} + +func testHubConfiguration(hubURI string) *models.Configuration { + return &models.Configuration{Config: *testHubConfig(hubURI)} +} + +func TestUploadKerberosHubTusCreate429PausesWithoutRetrying(t *testing.T) { + withHubUploadPause(t) + previousDelay := tusBackoffBaseDelay + tusBackoffBaseDelay = 0 + t.Cleanup(func() { tusBackoffBaseDelay = previousDelay }) + + var posts int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost && strings.HasPrefix(r.URL.Path, tusUploadPath) { + atomic.AddInt32(&posts, 1) + w.Header().Set("Retry-After", time.Now().Add(3*time.Hour).UTC().Format(http.TimeFormat)) + http.Error(w, `{"error":"plan_limit_reached"}`, http.StatusTooManyRequests) + return + } + t.Errorf("unexpected request %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusInternalServerError) + })) + defer server.Close() + + fileName := "1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4" + withRecording(t, fileName, bytes.Repeat([]byte("r"), 1024)) + + uploaded, configured, err := UploadKerberosHub(testHubConfiguration(server.URL), fileName) + if uploaded || !configured || err != nil { + t.Fatalf("UploadKerberosHub = %v, %v, %v; want false, true, nil", uploaded, configured, err) + } + if got := atomic.LoadInt32(&posts); got != 1 { + t.Fatalf("tus create requests = %d, want 1 (no in-cycle retries on 429)", got) + } + if remaining := hubUploadPause.Remaining(); remaining < 2*time.Hour+50*time.Minute || remaining > 3*time.Hour { + t.Fatalf("pause = %v, want about 3h", remaining) + } + if _, err := os.Stat("data/recordings/" + fileName); err != nil { + t.Fatalf("recording must stay on disk: %v", err) + } +} + +func TestUploadKerberosHubTusPatch429KeepsResumeState(t *testing.T) { + withHubUploadPause(t) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Tus-Resumable", tusResumableVersion) + switch r.Method { + case http.MethodPost: + w.Header().Set("Location", tusUploadPath+"upload-1") + w.WriteHeader(http.StatusCreated) + case http.MethodHead: + w.Header().Set("Upload-Offset", "0") + w.WriteHeader(http.StatusOK) + case http.MethodPatch: + w.WriteHeader(http.StatusTooManyRequests) // no Retry-After + } + })) + defer server.Close() + + fileName := "1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4" + withRecording(t, fileName, bytes.Repeat([]byte("r"), 1024)) + + uploaded, configured, err := UploadKerberosHub(testHubConfiguration(server.URL), fileName) + if uploaded || !configured || err != nil { + t.Fatalf("UploadKerberosHub = %v, %v, %v; want false, true, nil", uploaded, configured, err) + } + if remaining := hubUploadPause.Remaining(); remaining <= 0 || remaining > uploadPauseMinBackoff { + t.Fatalf("pause without Retry-After = %v, want up to %v", remaining, uploadPauseMinBackoff) + } + if _, err := os.Stat(tusSidecarPath(fileName, "hub")); err != nil { + t.Fatalf("resume state must be kept during the pause: %v", err) + } +} + +func TestUploadKerberosHubLegacyHead429Pauses(t *testing.T) { + withHubUploadPause(t) + t.Setenv("AGENT_DISABLE_RESUMABLE_UPLOAD", "true") + + var uploads int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/storage/upload" { + t.Errorf("unexpected request %s %s", r.Method, r.URL.Path) + return + } + if r.Method == http.MethodPost { + atomic.AddInt32(&uploads, 1) + } + w.Header().Set("Retry-After", "7200") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + fileName := "1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4" + withRecording(t, fileName, bytes.Repeat([]byte("r"), 1024)) + + uploaded, configured, err := UploadKerberosHub(testHubConfiguration(server.URL), fileName) + if uploaded || !configured || err != nil { + t.Fatalf("UploadKerberosHub = %v, %v, %v; want false, true, nil", uploaded, configured, err) + } + if got := atomic.LoadInt32(&uploads); got != 0 { + t.Fatalf("POST after a 429 HEAD = %d, want 0", got) + } + if remaining := hubUploadPause.Remaining(); remaining < 119*time.Minute || remaining > 2*time.Hour { + t.Fatalf("pause = %v, want about 2h", remaining) + } +} + +func TestUploadKerberosHubOtherStatusesDoNotPause(t *testing.T) { + // Older Hub versions never send 429; a refusal keeps today's behaviour + // (recording kept, retried by the upload loop) and does not pause. + withHubUploadPause(t) + t.Setenv("AGENT_DISABLE_RESUMABLE_UPLOAD", "true") + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + })) + defer server.Close() + + fileName := "1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4" + withRecording(t, fileName, bytes.Repeat([]byte("r"), 1024)) + + hubUploadPause.Throttled(0, false) // an earlier 429 episode + uploaded, configured, err := UploadKerberosHub(testHubConfiguration(server.URL), fileName) + if uploaded || !configured || err != nil { + t.Fatalf("UploadKerberosHub = %v, %v, %v; want false, true, nil", uploaded, configured, err) + } + if remaining := hubUploadPause.Remaining(); remaining != 0 { + t.Fatalf("pause after a non-429 answer = %v, want 0", remaining) + } +}