Compare commits

..
Author SHA1 Message Date
Cédric VerstraetenandClaude Opus 5.5 f20d156d86 Stop on 429 only for Kerberos Hub tus uploads
runTusUpload also serves direct Kerberos Vault uploads. A 429 from a vault
now keeps the existing in-cycle retries instead of ending the attempt, so it
does not advance the vault's retry budget or secondary-storage failover
early, and it never sets the Hub upload pause. Hub uploads still stop at
once and pause until Retry-After (review feedback).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-19 11:36:48 +00:00
Cédric VerstraetenandClaude Opus 5.5 98f1826fe1 Honour 429 + Retry-After on uploads to Kerberos Hub
Kerberos Hub will answer a recording upload with 429 Too Many Requests and
Retry-After (start of the next day or month in the organisation's
timezone) when the organisation is over its upload limit. Until now the
agent kept the recording and retried every 5 seconds all night.

- A shared, mutex-guarded upload pause (upload_pause.go) holds the
  deadline. Retry-After is parsed in both RFC 9110 forms (delay-seconds
  and HTTP-date, the latter measured against the response Date header).
  Values are clamped to [5 s, 32 days]; a 429 without a usable
  Retry-After backs off exponentially from 5 s up to 15 minutes.
- The tus path (create, HEAD, PATCH) and the legacy HEAD/POST on
  /storage/upload report 429; the tus loop stops retrying on 429 and
  keeps its resume state.
- HandleUpload waits for the pause before touching the queue again.
  Recordings and markers stay on disk; the order is unchanged.
- One log line when the pause starts and one when Hub accepts uploads
  again. Heartbeats, live view and MQTT are untouched; other status codes
  and older Hub versions behave as before.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-19 07:15:53 +00:00
5 changed files with 668 additions and 10 deletions
+22 -1
View File
@@ -57,14 +57,27 @@ func HandleUpload(configDirectory string, configuration *models.Configuration, c
// Half a second delay between two uploads
delay := 500 * time.Millisecond
// Kerberos Hub answers 429 + Retry-After when the organisation reached
// its upload limit; uploads to Hub then pause until that time. A
// (re)started loop, e.g. after a reconfiguration, tries again at once.
uploadsToHub := config.Cloud == "s3" || config.Cloud == "kerberoshub"
hubUploadPause.Reset()
loop:
for {
wait := 2 * time.Second
if uploadsToHub {
if remaining := hubUploadPause.Remaining(); remaining > wait {
wait = remaining
}
}
// This will check if we need to stop the thread,
// because of a reconfiguration.
select {
case <-communication.HandleUpload:
break loop
case <-time.After(2 * time.Second):
case <-time.After(wait):
}
ff, err := utils.ReadDirectory(watchDirectory)
@@ -142,6 +155,12 @@ func HandleUpload(configDirectory string, configuration *models.Configuration, c
}
}
if uploadsToHub && hubUploadPause.Remaining() > 0 {
// Paused: leave this and the following recordings queued
// on disk; the outer loop waits until the pause ends.
break
}
time.Sleep(delay)
}
}
@@ -1813,6 +1832,8 @@ func hubPersistenceVerificationError(statusCode int) string {
reason += ": the account has no active subscription or the subscription expired"
case http.StatusForbidden:
reason += ": the device is muted"
case http.StatusTooManyRequests:
reason += ": the organisation reached its upload limit for this period"
}
return "cloud.VerifyPersistence(kerberoshub): " + reason + " (HTTP " + strconv.Itoa(statusCode) + ")"
}
+22 -1
View File
@@ -45,8 +45,17 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
// endpoint (older deployments) we transparently fall back to the legacy
// single-POST upload below.
if resumableUploadsEnabled() {
uploaded, _, supported, body, rerr := uploadHubResumable(&config, fileName, "UploadKerberosHub", "hub")
uploaded, responded, supported, body, rerr := uploadHubResumable(&config, fileName, "UploadKerberosHub", "hub")
if supported {
var throttled *uploadThrottledError
if errors.As(rerr, &throttled) {
// Upload limit reached: keep the recording and pause uploads.
hubUploadPause.Throttled(throttled.retryAfter, throttled.ok)
return false, true, nil
}
if uploaded || responded {
hubUploadPause.Clear()
}
if uploaded {
log.WithFields(log.Fields{
"component": "kerberos_hub",
@@ -125,6 +134,11 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
}
if err == nil && resp != nil {
if resp.StatusCode == http.StatusTooManyRequests {
// Upload limit reached: keep the recording and pause uploads.
hubUploadPause.Throttled(retryAfterFromResponse(resp))
return false, true, nil
}
if resp.StatusCode == 200 {
log.WithFields(log.Fields{
"component": "kerberos_hub",
@@ -132,6 +146,7 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
"status_code": resp.StatusCode,
}).Debug("Hub upload authorized")
} else {
hubUploadPause.Clear()
log.WithFields(log.Fields{
"component": "kerberos_hub",
"event": "upload_rejected",
@@ -165,6 +180,12 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
if err == nil {
if resp != nil {
if resp.StatusCode == http.StatusTooManyRequests {
// Upload limit reached: keep the recording and pause uploads.
hubUploadPause.Throttled(retryAfterFromResponse(resp))
return false, true, nil
}
hubUploadPause.Clear()
body, err := ioutil.ReadAll(resp.Body)
if err == nil {
if resp.StatusCode == 200 {
+28 -8
View File
@@ -120,7 +120,13 @@ type tusHeaderFunc func(h http.Header, fileName string)
// - supported: the server exposes a tus endpoint. When false, the caller
// should fall back to the legacy single-POST upload (older deployments).
// - body: a short message for logging.
func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tusHeaderFunc) (uploaded bool, responded bool, supported bool, body string, err error) {
//
// stopOnRateLimit makes a 429 end the attempt at once (Kerberos Hub: the
// organisation reached its upload limit and the caller pauses uploads until
// Retry-After). Direct Vault uploads pass false, so a 429 from a vault keeps the
// existing in-cycle retries and does not advance the vault's retry budget or
// secondary-storage failover early.
func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tusHeaderFunc, stopOnRateLimit bool) (uploaded bool, responded bool, supported bool, body string, err error) {
file, ferr := utils.OpenRecording("data/recordings", fileName)
if file != nil {
defer file.Close()
@@ -177,6 +183,11 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
// The vault does not implement tus; let the caller fall back.
return false, false, false, "", cerr
}
if stopOnRateLimit && status == http.StatusTooManyRequests {
// Upload limit reached: retrying now cannot help. The caller
// pauses uploads until Retry-After.
return false, true, true, "", cerr
}
log.Info(label + ": resumable create failed, " + cerr.Error())
tusBackoff(attempt)
continue
@@ -196,6 +207,10 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
uploadURL = ""
continue
}
if stopOnRateLimit && status == http.StatusTooManyRequests {
// Keep the resume state so the upload continues after the pause.
return false, true, true, "", herr
}
log.Info(label + ": resumable head failed, " + herr.Error())
tusBackoff(attempt)
continue
@@ -247,6 +262,11 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
lastStatus = status
if perr != nil {
lastErr = perr
if stopOnRateLimit && status == http.StatusTooManyRequests {
// The resume state saved at creation stays, so the upload
// continues from the server offset after the pause.
return false, true, true, "", perr
}
if status >= 400 {
// Definitive rejection (e.g. provider push failed during finalize).
// Re-evaluate via HEAD on the next iteration to decide retry/restart.
@@ -337,7 +357,7 @@ func uploadVaultResumable(vault models.KStorage, publicKey, deviceKey, fileName,
h[name] = append([]string(nil), values...)
}
}
return runTusUpload(baseURL, metadata, fileName, label, slot, setHeaders)
return runTusUpload(baseURL, metadata, fileName, label, slot, setHeaders, false)
}
// uploadHubResumable uploads a recording to Kerberos Hub's tus endpoint, which
@@ -358,7 +378,7 @@ func uploadHubResumable(config *models.Config, fileName, label, slot string) (bo
setHeaders := func(h http.Header, fn string) {
setHubTusHeaders(h, config, fn)
}
return runTusUpload(baseURL, metadata, fileName, label, slot, setHeaders)
return runTusUpload(baseURL, metadata, fileName, label, slot, setHeaders, true)
}
func addRecordingTusMetadata(values map[string]string, fileName string) {
@@ -408,7 +428,7 @@ func tusCreate(client *http.Client, baseURL string, size int64, metadata string,
const maxErrorBodyBytes = 4096
body, readErr := io.ReadAll(io.LimitReader(resp.Body, maxErrorBodyBytes+1))
if readErr != nil {
return "", resp.StatusCode, fmt.Errorf("unexpected status creating upload: %s; read error response: %w", resp.Status, readErr)
return "", resp.StatusCode, throttledError(resp, fmt.Errorf("unexpected status creating upload: %s; read error response: %w", resp.Status, readErr))
}
truncated := len(body) > maxErrorBodyBytes
if truncated {
@@ -419,9 +439,9 @@ func tusCreate(client *http.Client, baseURL string, size int64, metadata string,
message += " [truncated]"
}
if message != "" {
return "", resp.StatusCode, fmt.Errorf("unexpected status creating upload: %s: %q", resp.Status, message)
return "", resp.StatusCode, throttledError(resp, fmt.Errorf("unexpected status creating upload: %s: %q", resp.Status, message))
}
return "", resp.StatusCode, fmt.Errorf("unexpected status creating upload: %s", resp.Status)
return "", resp.StatusCode, throttledError(resp, fmt.Errorf("unexpected status creating upload: %s", resp.Status))
}
// Location identifies the created upload; its response body is optional.
if _, err := io.Copy(io.Discard, resp.Body); err != nil {
@@ -454,7 +474,7 @@ func tusHead(client *http.Client, uploadURL string, setHeaders tusHeaderFunc) (i
io.Copy(io.Discard, resp.Body)
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent {
return 0, resp.StatusCode, fmt.Errorf("unexpected status on HEAD: %s", resp.Status)
return 0, resp.StatusCode, throttledError(resp, fmt.Errorf("unexpected status on HEAD: %s", resp.Status))
}
offsetStr := resp.Header.Get("Upload-Offset")
offset, perr := strconv.ParseInt(offsetStr, 10, 64)
@@ -489,7 +509,7 @@ func tusPatch(client *http.Client, uploadURL string, offset, length int64, file
respBody := string(bodyBytes)
if resp.StatusCode != http.StatusNoContent {
return offset, resp.StatusCode, respBody, fmt.Errorf("unexpected status on PATCH: %s, %s", resp.Status, respBody)
return offset, resp.StatusCode, respBody, throttledError(resp, fmt.Errorf("unexpected status on PATCH: %s, %s", resp.Status, respBody))
}
newOffsetStr := resp.Header.Get("Upload-Offset")
newOffset, perr := strconv.ParseInt(newOffsetStr, 10, 64)
+209
View File
@@ -0,0 +1,209 @@
package cloud
import (
"fmt"
"net/http"
"strconv"
"strings"
"sync"
"time"
log "github.com/sirupsen/logrus"
)
// Kerberos Hub answers a recording upload with 429 Too Many Requests when the
// organisation is over its daily or monthly upload limit, and sets Retry-After
// to the start of the next period. The agent then pauses uploads to Hub until
// that moment instead of retrying every few seconds. Recordings stay queued on
// disk and are uploaded, in the usual order, once the pause is over.
const (
// uploadPauseMinBackoff is the first back-off when a 429 carries no usable
// Retry-After. It equals the existing retry interval after a refused upload.
uploadPauseMinBackoff = 5 * time.Second
// uploadPauseMaxBackoff caps the exponential back-off without Retry-After.
uploadPauseMaxBackoff = 15 * time.Minute
// uploadPauseMaxRetryAfter caps an explicit Retry-After. A monthly limit
// never needs more than 31 days; anything longer is treated as bogus.
uploadPauseMaxRetryAfter = 32 * 24 * time.Hour
)
// hubUploadPause is shared by every uploader that talks to Kerberos Hub.
var hubUploadPause = newUploadPause(time.Now)
// uploadPause holds the deadline until which uploads to one target are paused.
// It is safe for concurrent use.
type uploadPause struct {
mu sync.Mutex
now func() time.Time
until time.Time
backoff time.Duration // next back-off when Retry-After is missing
active bool // a 429 episode is in progress (for start/end logs)
}
func newUploadPause(now func() time.Time) *uploadPause {
return &uploadPause{now: now}
}
// Remaining returns how long uploads stay paused; zero when not paused.
func (p *uploadPause) Remaining() time.Duration {
p.mu.Lock()
defer p.mu.Unlock()
if !p.active {
return 0
}
if remaining := p.until.Sub(p.now()); remaining > 0 {
return remaining
}
return 0
}
// Throttled records a 429 answer. retryAfter is the delay parsed from the
// Retry-After header; ok is false when the header was missing or invalid, in
// which case an exponential back-off is used. It returns the pause duration.
func (p *uploadPause) Throttled(retryAfter time.Duration, ok bool) time.Duration {
p.mu.Lock()
defer p.mu.Unlock()
var wait time.Duration
if ok {
wait = clampRetryAfter(retryAfter)
} else {
if p.backoff < uploadPauseMinBackoff {
p.backoff = uploadPauseMinBackoff
}
wait = p.backoff
p.backoff *= 2
if p.backoff > uploadPauseMaxBackoff {
p.backoff = uploadPauseMaxBackoff
}
}
now := p.now()
p.until = now.Add(wait)
fields := log.Fields{
"component": "upload",
"target": "kerberos_hub",
"status_code": http.StatusTooManyRequests,
"paused_until": now.Add(wait).Format(time.RFC3339),
"pause": wait.Round(time.Second).String(),
"retry_after_present": ok,
}
if !p.active {
p.active = true
log.WithFields(fields).WithField("event", "upload_paused").
Warn("Kerberos Hub refused the upload (upload limit reached); pausing uploads, recordings stay queued on disk")
} else {
log.WithFields(fields).WithField("event", "upload_pause_extended").
Debug("Kerberos Hub still refuses uploads; pause extended")
}
return wait
}
// Clear ends a pause after Hub answered an upload with anything but 429.
func (p *uploadPause) Clear() {
p.mu.Lock()
defer p.mu.Unlock()
if p.active {
log.WithFields(log.Fields{
"component": "upload",
"target": "kerberos_hub",
"event": "upload_resumed",
}).Info("Kerberos Hub accepts uploads again; upload pause ended")
}
p.active = false
p.until = time.Time{}
p.backoff = 0
}
// Reset drops any pause without logging, for example when the upload loop is
// restarted after a reconfiguration.
func (p *uploadPause) Reset() {
p.mu.Lock()
defer p.mu.Unlock()
p.active = false
p.until = time.Time{}
p.backoff = 0
}
func clampRetryAfter(d time.Duration) time.Duration {
if d < uploadPauseMinBackoff {
// Zero or a date in the past: wait as long as a normal refused upload.
return uploadPauseMinBackoff
}
if d > uploadPauseMaxRetryAfter {
return uploadPauseMaxRetryAfter
}
return d
}
// parseRetryAfter parses a Retry-After value (RFC 9110, section 10.2.3):
// either delay-seconds or an HTTP-date. now is the reference time for an
// HTTP-date; callers pass the response Date header when present, so a wrong
// clock on the agent does not shift the deadline. A negative number or any
// other value is invalid. The result can be negative for a date in the past;
// Throttled clamps it.
func parseRetryAfter(value string, now time.Time) (time.Duration, bool) {
value = strings.TrimSpace(value)
if value == "" {
return 0, false
}
if allDigits(value) {
seconds, err := strconv.ParseInt(value, 10, 64)
if err != nil || seconds > int64(uploadPauseMaxRetryAfter/time.Second) {
// Too large to be meaningful (or to fit a Duration): cap it.
return uploadPauseMaxRetryAfter, true
}
return time.Duration(seconds) * time.Second, true
}
at, err := http.ParseTime(value)
if err != nil {
return 0, false
}
return at.Sub(now), true
}
func allDigits(s string) bool {
for i := 0; i < len(s); i++ {
if s[i] < '0' || s[i] > '9' {
return false
}
}
return s != ""
}
// retryAfterFromResponse reads Retry-After from a response, using the
// response Date header (when valid) as the reference for an HTTP-date.
func retryAfterFromResponse(resp *http.Response) (time.Duration, bool) {
now := time.Now()
if date, err := http.ParseTime(resp.Header.Get("Date")); err == nil {
now = date
}
return parseRetryAfter(resp.Header.Get("Retry-After"), now)
}
// uploadThrottledError marks an upload refused with 429 Too Many Requests and
// carries the parsed Retry-After.
type uploadThrottledError struct {
retryAfter time.Duration
ok bool
err error
}
func (e *uploadThrottledError) Error() string {
if e.err != nil {
return e.err.Error()
}
return fmt.Sprintf("upload refused: %d %s", http.StatusTooManyRequests, http.StatusText(http.StatusTooManyRequests))
}
func (e *uploadThrottledError) Unwrap() error { return e.err }
// throttledError wraps err in an uploadThrottledError when resp is a 429;
// otherwise it returns err unchanged.
func throttledError(resp *http.Response, err error) error {
if resp == nil || resp.StatusCode != http.StatusTooManyRequests {
return err
}
retryAfter, ok := retryAfterFromResponse(resp)
return &uploadThrottledError{retryAfter: retryAfter, ok: ok, err: err}
}
+387
View File
@@ -0,0 +1,387 @@
package cloud
import (
"bytes"
"errors"
"net/http"
"net/http/httptest"
"os"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/kerberos-io/agent/machinery/src/models"
)
func TestParseRetryAfter(t *testing.T) {
now := time.Date(2026, 11, 16, 22, 0, 0, 0, time.UTC)
tests := []struct {
name string
value string
want time.Duration
wantOK bool
}{
{"delay seconds", "120", 120 * time.Second, true},
{"delay seconds with spaces", " 3600 ", time.Hour, true},
{"zero", "0", 0, true},
{"IMF-fixdate", "Mon, 16 Nov 2026 23:00:00 GMT", time.Hour, true},
{"RFC 850 date", "Monday, 16-Nov-26 23:30:00 GMT", 90 * time.Minute, true},
{"asctime date", "Mon Nov 16 22:00:30 2026", 30 * time.Second, true},
{"date in the past", "Mon, 16 Nov 2026 21:00:00 GMT", -time.Hour, true},
{"more than the cap", "4000000", uploadPauseMaxRetryAfter, true},
{"overflowing number", "99999999999999999999999999", uploadPauseMaxRetryAfter, true},
{"missing", "", 0, false},
{"negative", "-5", 0, false},
{"fraction", "1.5", 0, false},
{"unit suffix", "10s", 0, false},
{"garbage", "tomorrow", 0, false},
{"ISO date", "2026-11-17T00:00:00Z", 0, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, ok := parseRetryAfter(tt.value, now)
if ok != tt.wantOK || got != tt.want {
t.Fatalf("parseRetryAfter(%q) = %v, %v; want %v, %v", tt.value, got, ok, tt.want, tt.wantOK)
}
})
}
}
func TestClampRetryAfter(t *testing.T) {
tests := []struct {
in, want time.Duration
}{
{-time.Hour, uploadPauseMinBackoff},
{0, uploadPauseMinBackoff},
{time.Second, uploadPauseMinBackoff},
{2 * time.Hour, 2 * time.Hour},
{uploadPauseMaxRetryAfter, uploadPauseMaxRetryAfter},
{40 * 24 * time.Hour, uploadPauseMaxRetryAfter},
}
for _, tt := range tests {
if got := clampRetryAfter(tt.in); got != tt.want {
t.Errorf("clampRetryAfter(%v) = %v, want %v", tt.in, got, tt.want)
}
}
}
func TestRetryAfterFromResponseUsesDateHeader(t *testing.T) {
// The agent clock may be wrong; an HTTP-date is measured against the
// response Date so the pause still ends at the start of Hub's next period.
resp := &http.Response{Header: http.Header{}}
resp.Header.Set("Date", "Mon, 16 Nov 2020 22:00:00 GMT")
resp.Header.Set("Retry-After", "Tue, 17 Nov 2020 00:00:00 GMT")
got, ok := retryAfterFromResponse(resp)
if !ok || got != 2*time.Hour {
t.Fatalf("retryAfterFromResponse = %v, %v; want 2h, true", got, ok)
}
}
type fakeClock struct {
mu sync.Mutex
now time.Time
}
func (c *fakeClock) Now() time.Time {
c.mu.Lock()
defer c.mu.Unlock()
return c.now
}
func (c *fakeClock) Advance(d time.Duration) {
c.mu.Lock()
defer c.mu.Unlock()
c.now = c.now.Add(d)
}
func TestUploadPauseHonoursRetryAfter(t *testing.T) {
clock := &fakeClock{now: time.Date(2026, 11, 16, 22, 0, 0, 0, time.UTC)}
p := newUploadPause(clock.Now)
if got := p.Remaining(); got != 0 {
t.Fatalf("Remaining before any 429 = %v, want 0", got)
}
if got := p.Throttled(2*time.Hour, true); got != 2*time.Hour {
t.Fatalf("Throttled = %v, want 2h", got)
}
if got := p.Remaining(); got != 2*time.Hour {
t.Fatalf("Remaining = %v, want 2h", got)
}
clock.Advance(90 * time.Minute)
if got := p.Remaining(); got != 30*time.Minute {
t.Fatalf("Remaining after 90m = %v, want 30m", got)
}
clock.Advance(30 * time.Minute)
if got := p.Remaining(); got != 0 {
t.Fatalf("Remaining at the deadline = %v, want 0", got)
}
// Absurd values are clamped.
if got := p.Throttled(-time.Hour, true); got != uploadPauseMinBackoff {
t.Fatalf("Throttled(past) = %v, want %v", got, uploadPauseMinBackoff)
}
if got := p.Throttled(365*24*time.Hour, true); got != uploadPauseMaxRetryAfter {
t.Fatalf("Throttled(1 year) = %v, want %v", got, uploadPauseMaxRetryAfter)
}
p.Clear()
if got := p.Remaining(); got != 0 {
t.Fatalf("Remaining after Clear = %v, want 0", got)
}
}
func TestUploadPauseBacksOffWithoutRetryAfter(t *testing.T) {
clock := &fakeClock{now: time.Date(2026, 11, 16, 22, 0, 0, 0, time.UTC)}
p := newUploadPause(clock.Now)
want := []time.Duration{
5 * time.Second, 10 * time.Second, 20 * time.Second, 40 * time.Second,
80 * time.Second, 160 * time.Second, 320 * time.Second, 640 * time.Second,
15 * time.Minute, 15 * time.Minute,
}
for i, w := range want {
if got := p.Throttled(0, false); got != w {
t.Fatalf("attempt %d: Throttled = %v, want %v", i, got, w)
}
if got := p.Remaining(); got != w {
t.Fatalf("attempt %d: Remaining = %v, want %v", i, got, w)
}
clock.Advance(w)
}
// A valid Retry-After does not disturb the back-off sequence.
if got := p.Throttled(time.Hour, true); got != time.Hour {
t.Fatalf("Throttled with Retry-After = %v, want 1h", got)
}
if got := p.Throttled(0, false); got != 15*time.Minute {
t.Fatalf("back-off after Retry-After = %v, want 15m", got)
}
// Clear (Hub answered something other than 429) restarts the back-off.
p.Clear()
if got := p.Throttled(0, false); got != 5*time.Second {
t.Fatalf("back-off after Clear = %v, want 5s", got)
}
p.Reset()
if got := p.Remaining(); got != 0 {
t.Fatalf("Remaining after Reset = %v, want 0", got)
}
}
func TestUploadPauseConcurrentUse(t *testing.T) {
p := newUploadPause(time.Now)
var wg sync.WaitGroup
for i := 0; i < 16; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
for j := 0; j < 100; j++ {
switch (i + j) % 4 {
case 0:
p.Throttled(time.Minute, true)
case 1:
p.Throttled(0, false)
case 2:
p.Remaining()
case 3:
p.Clear()
}
}
}(i)
}
wg.Wait()
}
func TestThrottledErrorOnlyFor429(t *testing.T) {
base := errors.New("boom")
resp := &http.Response{StatusCode: http.StatusForbidden, Header: http.Header{}}
if got := throttledError(resp, base); got != base {
t.Fatalf("throttledError(403) = %v, want the original error", got)
}
resp = &http.Response{StatusCode: http.StatusTooManyRequests, Header: http.Header{}}
resp.Header.Set("Retry-After", "60")
var throttled *uploadThrottledError
if err := throttledError(resp, base); !errors.As(err, &throttled) || !errors.Is(err, base) {
t.Fatalf("throttledError(429) = %v, want an uploadThrottledError wrapping the original", err)
}
if !throttled.ok || throttled.retryAfter != time.Minute {
t.Fatalf("retryAfter = %v, %v; want 1m, true", throttled.retryAfter, throttled.ok)
}
}
// withHubUploadPause gives a test a fresh shared pause and restores it after.
func withHubUploadPause(t *testing.T) {
t.Helper()
previous := hubUploadPause
hubUploadPause = newUploadPause(time.Now)
t.Cleanup(func() { hubUploadPause = previous })
}
func testHubConfiguration(hubURI string) *models.Configuration {
return &models.Configuration{Config: *testHubConfig(hubURI)}
}
func TestUploadKerberosHubTusCreate429PausesWithoutRetrying(t *testing.T) {
withHubUploadPause(t)
previousDelay := tusBackoffBaseDelay
tusBackoffBaseDelay = 0
t.Cleanup(func() { tusBackoffBaseDelay = previousDelay })
var posts int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodPost && strings.HasPrefix(r.URL.Path, tusUploadPath) {
atomic.AddInt32(&posts, 1)
w.Header().Set("Retry-After", time.Now().Add(3*time.Hour).UTC().Format(http.TimeFormat))
http.Error(w, `{"error":"plan_limit_reached"}`, http.StatusTooManyRequests)
return
}
t.Errorf("unexpected request %s %s", r.Method, r.URL.Path)
w.WriteHeader(http.StatusInternalServerError)
}))
defer server.Close()
fileName := "1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4"
withRecording(t, fileName, bytes.Repeat([]byte("r"), 1024))
uploaded, configured, err := UploadKerberosHub(testHubConfiguration(server.URL), fileName)
if uploaded || !configured || err != nil {
t.Fatalf("UploadKerberosHub = %v, %v, %v; want false, true, nil", uploaded, configured, err)
}
if got := atomic.LoadInt32(&posts); got != 1 {
t.Fatalf("tus create requests = %d, want 1 (no in-cycle retries on 429)", got)
}
if remaining := hubUploadPause.Remaining(); remaining < 2*time.Hour+50*time.Minute || remaining > 3*time.Hour {
t.Fatalf("pause = %v, want about 3h", remaining)
}
if _, err := os.Stat("data/recordings/" + fileName); err != nil {
t.Fatalf("recording must stay on disk: %v", err)
}
}
func TestUploadKerberosHubTusPatch429KeepsResumeState(t *testing.T) {
withHubUploadPause(t)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Tus-Resumable", tusResumableVersion)
switch r.Method {
case http.MethodPost:
w.Header().Set("Location", tusUploadPath+"upload-1")
w.WriteHeader(http.StatusCreated)
case http.MethodHead:
w.Header().Set("Upload-Offset", "0")
w.WriteHeader(http.StatusOK)
case http.MethodPatch:
w.WriteHeader(http.StatusTooManyRequests) // no Retry-After
}
}))
defer server.Close()
fileName := "1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4"
withRecording(t, fileName, bytes.Repeat([]byte("r"), 1024))
uploaded, configured, err := UploadKerberosHub(testHubConfiguration(server.URL), fileName)
if uploaded || !configured || err != nil {
t.Fatalf("UploadKerberosHub = %v, %v, %v; want false, true, nil", uploaded, configured, err)
}
if remaining := hubUploadPause.Remaining(); remaining <= 0 || remaining > uploadPauseMinBackoff {
t.Fatalf("pause without Retry-After = %v, want up to %v", remaining, uploadPauseMinBackoff)
}
if _, err := os.Stat(tusSidecarPath(fileName, "hub")); err != nil {
t.Fatalf("resume state must be kept during the pause: %v", err)
}
}
func TestUploadKerberosHubLegacyHead429Pauses(t *testing.T) {
withHubUploadPause(t)
t.Setenv("AGENT_DISABLE_RESUMABLE_UPLOAD", "true")
var uploads int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/storage/upload" {
t.Errorf("unexpected request %s %s", r.Method, r.URL.Path)
return
}
if r.Method == http.MethodPost {
atomic.AddInt32(&uploads, 1)
}
w.Header().Set("Retry-After", "7200")
w.WriteHeader(http.StatusTooManyRequests)
}))
defer server.Close()
fileName := "1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4"
withRecording(t, fileName, bytes.Repeat([]byte("r"), 1024))
uploaded, configured, err := UploadKerberosHub(testHubConfiguration(server.URL), fileName)
if uploaded || !configured || err != nil {
t.Fatalf("UploadKerberosHub = %v, %v, %v; want false, true, nil", uploaded, configured, err)
}
if got := atomic.LoadInt32(&uploads); got != 0 {
t.Fatalf("POST after a 429 HEAD = %d, want 0", got)
}
if remaining := hubUploadPause.Remaining(); remaining < 119*time.Minute || remaining > 2*time.Hour {
t.Fatalf("pause = %v, want about 2h", remaining)
}
}
func TestUploadKerberosHubOtherStatusesDoNotPause(t *testing.T) {
// Older Hub versions never send 429; a refusal keeps today's behaviour
// (recording kept, retried by the upload loop) and does not pause.
withHubUploadPause(t)
t.Setenv("AGENT_DISABLE_RESUMABLE_UPLOAD", "true")
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusBadRequest)
}))
defer server.Close()
fileName := "1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4"
withRecording(t, fileName, bytes.Repeat([]byte("r"), 1024))
hubUploadPause.Throttled(0, false) // an earlier 429 episode
uploaded, configured, err := UploadKerberosHub(testHubConfiguration(server.URL), fileName)
if uploaded || !configured || err != nil {
t.Fatalf("UploadKerberosHub = %v, %v, %v; want false, true, nil", uploaded, configured, err)
}
if remaining := hubUploadPause.Remaining(); remaining != 0 {
t.Fatalf("pause after a non-429 answer = %v, want 0", remaining)
}
}
// A direct Vault upload is not a Hub upload: a 429 from a vault keeps the
// existing in-cycle retries and never sets the Hub upload pause.
func TestVaultTusCreate429KeepsInCycleRetries(t *testing.T) {
withHubUploadPause(t)
previousDelay := tusBackoffBaseDelay
tusBackoffBaseDelay = 0
t.Cleanup(func() { tusBackoffBaseDelay = previousDelay })
var posts int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodPost {
atomic.AddInt32(&posts, 1)
w.Header().Set("Retry-After", "3600")
http.Error(w, "slow down", http.StatusTooManyRequests)
return
}
t.Errorf("unexpected request %s %s", r.Method, r.URL.Path)
w.WriteHeader(http.StatusInternalServerError)
}))
defer server.Close()
fileName := "1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4"
withRecording(t, fileName, bytes.Repeat([]byte("r"), 1024))
setHeaders := func(h http.Header, fn string) {}
uploaded, _, supported, _, _ := runTusUpload(server.URL+tusUploadPath, "", fileName, "vault-test", "primary", setHeaders, false)
if uploaded || !supported {
t.Fatalf("runTusUpload = uploaded %v, supported %v; want false, true", uploaded, supported)
}
if got := atomic.LoadInt32(&posts); got < 2 {
t.Fatalf("tus create requests = %d, want the in-cycle retries (> 1) for a vault", got)
}
if remaining := hubUploadPause.Remaining(); remaining != 0 {
t.Fatalf("hub upload pause = %v, want none for a vault upload", remaining)
}
}