mirror of
https://github.com/kerberos-io/agent.git
synced 2026-11-19 22:30:45 +00:00
Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
33e0a26f1c | ||
|
|
f20d156d86 | ||
|
|
98f1826fe1 | ||
|
|
bc8bc84cae |
@@ -57,14 +57,27 @@ func HandleUpload(configDirectory string, configuration *models.Configuration, c
|
||||
// Half a second delay between two uploads
|
||||
delay := 500 * time.Millisecond
|
||||
|
||||
// Kerberos Hub answers 429 + Retry-After when the organisation reached
|
||||
// its upload limit; uploads to Hub then pause until that time. A
|
||||
// (re)started loop, e.g. after a reconfiguration, tries again at once.
|
||||
uploadsToHub := config.Cloud == "s3" || config.Cloud == "kerberoshub"
|
||||
hubUploadPause.Reset()
|
||||
|
||||
loop:
|
||||
for {
|
||||
wait := 2 * time.Second
|
||||
if uploadsToHub {
|
||||
if remaining := hubUploadPause.Remaining(); remaining > wait {
|
||||
wait = remaining
|
||||
}
|
||||
}
|
||||
|
||||
// This will check if we need to stop the thread,
|
||||
// because of a reconfiguration.
|
||||
select {
|
||||
case <-communication.HandleUpload:
|
||||
break loop
|
||||
case <-time.After(2 * time.Second):
|
||||
case <-time.After(wait):
|
||||
}
|
||||
|
||||
ff, err := utils.ReadDirectory(watchDirectory)
|
||||
@@ -142,6 +155,12 @@ func HandleUpload(configDirectory string, configuration *models.Configuration, c
|
||||
}
|
||||
}
|
||||
|
||||
if uploadsToHub && hubUploadPause.Remaining() > 0 {
|
||||
// Paused: leave this and the following recordings queued
|
||||
// on disk; the outer loop waits until the pause ends.
|
||||
break
|
||||
}
|
||||
|
||||
time.Sleep(delay)
|
||||
}
|
||||
}
|
||||
@@ -1813,6 +1832,8 @@ func hubPersistenceVerificationError(statusCode int) string {
|
||||
reason += ": the account has no active subscription or the subscription expired"
|
||||
case http.StatusForbidden:
|
||||
reason += ": the device is muted"
|
||||
case http.StatusTooManyRequests:
|
||||
reason += ": the organisation reached its upload limit for this period"
|
||||
}
|
||||
return "cloud.VerifyPersistence(kerberoshub): " + reason + " (HTTP " + strconv.Itoa(statusCode) + ")"
|
||||
}
|
||||
|
||||
@@ -45,8 +45,17 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
|
||||
// endpoint (older deployments) we transparently fall back to the legacy
|
||||
// single-POST upload below.
|
||||
if resumableUploadsEnabled() {
|
||||
uploaded, _, supported, body, rerr := uploadHubResumable(&config, fileName, "UploadKerberosHub", "hub")
|
||||
uploaded, responded, supported, body, rerr := uploadHubResumable(&config, fileName, "UploadKerberosHub", "hub")
|
||||
if supported {
|
||||
var throttled *uploadThrottledError
|
||||
if errors.As(rerr, &throttled) {
|
||||
// Upload limit reached: keep the recording and pause uploads.
|
||||
hubUploadPause.Throttled(throttled.retryAfter, throttled.ok)
|
||||
return false, true, nil
|
||||
}
|
||||
if uploaded || responded {
|
||||
hubUploadPause.Clear()
|
||||
}
|
||||
if uploaded {
|
||||
log.WithFields(log.Fields{
|
||||
"component": "kerberos_hub",
|
||||
@@ -125,6 +134,11 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
|
||||
}
|
||||
|
||||
if err == nil && resp != nil {
|
||||
if resp.StatusCode == http.StatusTooManyRequests {
|
||||
// Upload limit reached: keep the recording and pause uploads.
|
||||
hubUploadPause.Throttled(retryAfterFromResponse(resp))
|
||||
return false, true, nil
|
||||
}
|
||||
if resp.StatusCode == 200 {
|
||||
log.WithFields(log.Fields{
|
||||
"component": "kerberos_hub",
|
||||
@@ -132,6 +146,7 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
|
||||
"status_code": resp.StatusCode,
|
||||
}).Debug("Hub upload authorized")
|
||||
} else {
|
||||
hubUploadPause.Clear()
|
||||
log.WithFields(log.Fields{
|
||||
"component": "kerberos_hub",
|
||||
"event": "upload_rejected",
|
||||
@@ -165,6 +180,12 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
|
||||
|
||||
if err == nil {
|
||||
if resp != nil {
|
||||
if resp.StatusCode == http.StatusTooManyRequests {
|
||||
// Upload limit reached: keep the recording and pause uploads.
|
||||
hubUploadPause.Throttled(retryAfterFromResponse(resp))
|
||||
return false, true, nil
|
||||
}
|
||||
hubUploadPause.Clear()
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err == nil {
|
||||
if resp.StatusCode == 200 {
|
||||
|
||||
@@ -120,7 +120,13 @@ type tusHeaderFunc func(h http.Header, fileName string)
|
||||
// - supported: the server exposes a tus endpoint. When false, the caller
|
||||
// should fall back to the legacy single-POST upload (older deployments).
|
||||
// - body: a short message for logging.
|
||||
func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tusHeaderFunc) (uploaded bool, responded bool, supported bool, body string, err error) {
|
||||
//
|
||||
// stopOnRateLimit makes a 429 end the attempt at once (Kerberos Hub: the
|
||||
// organisation reached its upload limit and the caller pauses uploads until
|
||||
// Retry-After). Direct Vault uploads pass false, so a 429 from a vault keeps the
|
||||
// existing in-cycle retries and does not advance the vault's retry budget or
|
||||
// secondary-storage failover early.
|
||||
func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tusHeaderFunc, stopOnRateLimit bool) (uploaded bool, responded bool, supported bool, body string, err error) {
|
||||
file, ferr := utils.OpenRecording("data/recordings", fileName)
|
||||
if file != nil {
|
||||
defer file.Close()
|
||||
@@ -177,6 +183,11 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
|
||||
// The vault does not implement tus; let the caller fall back.
|
||||
return false, false, false, "", cerr
|
||||
}
|
||||
if stopOnRateLimit && status == http.StatusTooManyRequests {
|
||||
// Upload limit reached: retrying now cannot help. The caller
|
||||
// pauses uploads until Retry-After.
|
||||
return false, true, true, "", cerr
|
||||
}
|
||||
log.Info(label + ": resumable create failed, " + cerr.Error())
|
||||
tusBackoff(attempt)
|
||||
continue
|
||||
@@ -196,6 +207,10 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
|
||||
uploadURL = ""
|
||||
continue
|
||||
}
|
||||
if stopOnRateLimit && status == http.StatusTooManyRequests {
|
||||
// Keep the resume state so the upload continues after the pause.
|
||||
return false, true, true, "", herr
|
||||
}
|
||||
log.Info(label + ": resumable head failed, " + herr.Error())
|
||||
tusBackoff(attempt)
|
||||
continue
|
||||
@@ -247,6 +262,11 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
|
||||
lastStatus = status
|
||||
if perr != nil {
|
||||
lastErr = perr
|
||||
if stopOnRateLimit && status == http.StatusTooManyRequests {
|
||||
// The resume state saved at creation stays, so the upload
|
||||
// continues from the server offset after the pause.
|
||||
return false, true, true, "", perr
|
||||
}
|
||||
if status >= 400 {
|
||||
// Definitive rejection (e.g. provider push failed during finalize).
|
||||
// Re-evaluate via HEAD on the next iteration to decide retry/restart.
|
||||
@@ -337,7 +357,7 @@ func uploadVaultResumable(vault models.KStorage, publicKey, deviceKey, fileName,
|
||||
h[name] = append([]string(nil), values...)
|
||||
}
|
||||
}
|
||||
return runTusUpload(baseURL, metadata, fileName, label, slot, setHeaders)
|
||||
return runTusUpload(baseURL, metadata, fileName, label, slot, setHeaders, false)
|
||||
}
|
||||
|
||||
// uploadHubResumable uploads a recording to Kerberos Hub's tus endpoint, which
|
||||
@@ -358,7 +378,7 @@ func uploadHubResumable(config *models.Config, fileName, label, slot string) (bo
|
||||
setHeaders := func(h http.Header, fn string) {
|
||||
setHubTusHeaders(h, config, fn)
|
||||
}
|
||||
return runTusUpload(baseURL, metadata, fileName, label, slot, setHeaders)
|
||||
return runTusUpload(baseURL, metadata, fileName, label, slot, setHeaders, true)
|
||||
}
|
||||
|
||||
func addRecordingTusMetadata(values map[string]string, fileName string) {
|
||||
@@ -408,7 +428,7 @@ func tusCreate(client *http.Client, baseURL string, size int64, metadata string,
|
||||
const maxErrorBodyBytes = 4096
|
||||
body, readErr := io.ReadAll(io.LimitReader(resp.Body, maxErrorBodyBytes+1))
|
||||
if readErr != nil {
|
||||
return "", resp.StatusCode, fmt.Errorf("unexpected status creating upload: %s; read error response: %w", resp.Status, readErr)
|
||||
return "", resp.StatusCode, throttledError(resp, fmt.Errorf("unexpected status creating upload: %s; read error response: %w", resp.Status, readErr))
|
||||
}
|
||||
truncated := len(body) > maxErrorBodyBytes
|
||||
if truncated {
|
||||
@@ -419,9 +439,9 @@ func tusCreate(client *http.Client, baseURL string, size int64, metadata string,
|
||||
message += " [truncated]"
|
||||
}
|
||||
if message != "" {
|
||||
return "", resp.StatusCode, fmt.Errorf("unexpected status creating upload: %s: %q", resp.Status, message)
|
||||
return "", resp.StatusCode, throttledError(resp, fmt.Errorf("unexpected status creating upload: %s: %q", resp.Status, message))
|
||||
}
|
||||
return "", resp.StatusCode, fmt.Errorf("unexpected status creating upload: %s", resp.Status)
|
||||
return "", resp.StatusCode, throttledError(resp, fmt.Errorf("unexpected status creating upload: %s", resp.Status))
|
||||
}
|
||||
// Location identifies the created upload; its response body is optional.
|
||||
if _, err := io.Copy(io.Discard, resp.Body); err != nil {
|
||||
@@ -454,7 +474,7 @@ func tusHead(client *http.Client, uploadURL string, setHeaders tusHeaderFunc) (i
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent {
|
||||
return 0, resp.StatusCode, fmt.Errorf("unexpected status on HEAD: %s", resp.Status)
|
||||
return 0, resp.StatusCode, throttledError(resp, fmt.Errorf("unexpected status on HEAD: %s", resp.Status))
|
||||
}
|
||||
offsetStr := resp.Header.Get("Upload-Offset")
|
||||
offset, perr := strconv.ParseInt(offsetStr, 10, 64)
|
||||
@@ -489,7 +509,7 @@ func tusPatch(client *http.Client, uploadURL string, offset, length int64, file
|
||||
respBody := string(bodyBytes)
|
||||
|
||||
if resp.StatusCode != http.StatusNoContent {
|
||||
return offset, resp.StatusCode, respBody, fmt.Errorf("unexpected status on PATCH: %s, %s", resp.Status, respBody)
|
||||
return offset, resp.StatusCode, respBody, throttledError(resp, fmt.Errorf("unexpected status on PATCH: %s, %s", resp.Status, respBody))
|
||||
}
|
||||
newOffsetStr := resp.Header.Get("Upload-Offset")
|
||||
newOffset, perr := strconv.ParseInt(newOffsetStr, 10, 64)
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
package cloud
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// Kerberos Hub answers a recording upload with 429 Too Many Requests when the
|
||||
// organisation is over its daily or monthly upload limit, and sets Retry-After
|
||||
// to the start of the next period. The agent then pauses uploads to Hub until
|
||||
// that moment instead of retrying every few seconds. Recordings stay queued on
|
||||
// disk and are uploaded, in the usual order, once the pause is over.
|
||||
const (
|
||||
// uploadPauseMinBackoff is the first back-off when a 429 carries no usable
|
||||
// Retry-After. It equals the existing retry interval after a refused upload.
|
||||
uploadPauseMinBackoff = 5 * time.Second
|
||||
// uploadPauseMaxBackoff caps the exponential back-off without Retry-After.
|
||||
uploadPauseMaxBackoff = 15 * time.Minute
|
||||
// uploadPauseMaxRetryAfter caps an explicit Retry-After. A monthly limit
|
||||
// never needs more than 31 days; anything longer is treated as bogus.
|
||||
uploadPauseMaxRetryAfter = 32 * 24 * time.Hour
|
||||
)
|
||||
|
||||
// hubUploadPause is shared by every uploader that talks to Kerberos Hub.
|
||||
var hubUploadPause = newUploadPause(time.Now)
|
||||
|
||||
// uploadPause holds the deadline until which uploads to one target are paused.
|
||||
// It is safe for concurrent use.
|
||||
type uploadPause struct {
|
||||
mu sync.Mutex
|
||||
now func() time.Time
|
||||
until time.Time
|
||||
backoff time.Duration // next back-off when Retry-After is missing
|
||||
active bool // a 429 episode is in progress (for start/end logs)
|
||||
}
|
||||
|
||||
func newUploadPause(now func() time.Time) *uploadPause {
|
||||
return &uploadPause{now: now}
|
||||
}
|
||||
|
||||
// Remaining returns how long uploads stay paused; zero when not paused.
|
||||
func (p *uploadPause) Remaining() time.Duration {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if !p.active {
|
||||
return 0
|
||||
}
|
||||
if remaining := p.until.Sub(p.now()); remaining > 0 {
|
||||
return remaining
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// Throttled records a 429 answer. retryAfter is the delay parsed from the
|
||||
// Retry-After header; ok is false when the header was missing or invalid, in
|
||||
// which case an exponential back-off is used. It returns the pause duration.
|
||||
func (p *uploadPause) Throttled(retryAfter time.Duration, ok bool) time.Duration {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
var wait time.Duration
|
||||
if ok {
|
||||
wait = clampRetryAfter(retryAfter)
|
||||
} else {
|
||||
if p.backoff < uploadPauseMinBackoff {
|
||||
p.backoff = uploadPauseMinBackoff
|
||||
}
|
||||
wait = p.backoff
|
||||
p.backoff *= 2
|
||||
if p.backoff > uploadPauseMaxBackoff {
|
||||
p.backoff = uploadPauseMaxBackoff
|
||||
}
|
||||
}
|
||||
|
||||
now := p.now()
|
||||
p.until = now.Add(wait)
|
||||
fields := log.Fields{
|
||||
"component": "upload",
|
||||
"target": "kerberos_hub",
|
||||
"status_code": http.StatusTooManyRequests,
|
||||
"paused_until": now.Add(wait).Format(time.RFC3339),
|
||||
"pause": wait.Round(time.Second).String(),
|
||||
"retry_after_present": ok,
|
||||
}
|
||||
if !p.active {
|
||||
p.active = true
|
||||
log.WithFields(fields).WithField("event", "upload_paused").
|
||||
Warn("Kerberos Hub refused the upload (upload limit reached); pausing uploads, recordings stay queued on disk")
|
||||
} else {
|
||||
log.WithFields(fields).WithField("event", "upload_pause_extended").
|
||||
Debug("Kerberos Hub still refuses uploads; pause extended")
|
||||
}
|
||||
return wait
|
||||
}
|
||||
|
||||
// Clear ends a pause after Hub answered an upload with anything but 429.
|
||||
func (p *uploadPause) Clear() {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.active {
|
||||
log.WithFields(log.Fields{
|
||||
"component": "upload",
|
||||
"target": "kerberos_hub",
|
||||
"event": "upload_resumed",
|
||||
}).Info("Kerberos Hub accepts uploads again; upload pause ended")
|
||||
}
|
||||
p.active = false
|
||||
p.until = time.Time{}
|
||||
p.backoff = 0
|
||||
}
|
||||
|
||||
// Reset drops any pause without logging, for example when the upload loop is
|
||||
// restarted after a reconfiguration.
|
||||
func (p *uploadPause) Reset() {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.active = false
|
||||
p.until = time.Time{}
|
||||
p.backoff = 0
|
||||
}
|
||||
|
||||
func clampRetryAfter(d time.Duration) time.Duration {
|
||||
if d < uploadPauseMinBackoff {
|
||||
// Zero or a date in the past: wait as long as a normal refused upload.
|
||||
return uploadPauseMinBackoff
|
||||
}
|
||||
if d > uploadPauseMaxRetryAfter {
|
||||
return uploadPauseMaxRetryAfter
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// parseRetryAfter parses a Retry-After value (RFC 9110, section 10.2.3):
|
||||
// either delay-seconds or an HTTP-date. now is the reference time for an
|
||||
// HTTP-date; callers pass the response Date header when present, so a wrong
|
||||
// clock on the agent does not shift the deadline. A negative number or any
|
||||
// other value is invalid. The result can be negative for a date in the past;
|
||||
// Throttled clamps it.
|
||||
func parseRetryAfter(value string, now time.Time) (time.Duration, bool) {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return 0, false
|
||||
}
|
||||
if allDigits(value) {
|
||||
seconds, err := strconv.ParseInt(value, 10, 64)
|
||||
if err != nil || seconds > int64(uploadPauseMaxRetryAfter/time.Second) {
|
||||
// Too large to be meaningful (or to fit a Duration): cap it.
|
||||
return uploadPauseMaxRetryAfter, true
|
||||
}
|
||||
return time.Duration(seconds) * time.Second, true
|
||||
}
|
||||
at, err := http.ParseTime(value)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
return at.Sub(now), true
|
||||
}
|
||||
|
||||
func allDigits(s string) bool {
|
||||
for i := 0; i < len(s); i++ {
|
||||
if s[i] < '0' || s[i] > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return s != ""
|
||||
}
|
||||
|
||||
// retryAfterFromResponse reads Retry-After from a response, using the
|
||||
// response Date header (when valid) as the reference for an HTTP-date.
|
||||
func retryAfterFromResponse(resp *http.Response) (time.Duration, bool) {
|
||||
now := time.Now()
|
||||
if date, err := http.ParseTime(resp.Header.Get("Date")); err == nil {
|
||||
now = date
|
||||
}
|
||||
return parseRetryAfter(resp.Header.Get("Retry-After"), now)
|
||||
}
|
||||
|
||||
// uploadThrottledError marks an upload refused with 429 Too Many Requests and
|
||||
// carries the parsed Retry-After.
|
||||
type uploadThrottledError struct {
|
||||
retryAfter time.Duration
|
||||
ok bool
|
||||
err error
|
||||
}
|
||||
|
||||
func (e *uploadThrottledError) Error() string {
|
||||
if e.err != nil {
|
||||
return e.err.Error()
|
||||
}
|
||||
return fmt.Sprintf("upload refused: %d %s", http.StatusTooManyRequests, http.StatusText(http.StatusTooManyRequests))
|
||||
}
|
||||
|
||||
func (e *uploadThrottledError) Unwrap() error { return e.err }
|
||||
|
||||
// throttledError wraps err in an uploadThrottledError when resp is a 429;
|
||||
// otherwise it returns err unchanged.
|
||||
func throttledError(resp *http.Response, err error) error {
|
||||
if resp == nil || resp.StatusCode != http.StatusTooManyRequests {
|
||||
return err
|
||||
}
|
||||
retryAfter, ok := retryAfterFromResponse(resp)
|
||||
return &uploadThrottledError{retryAfter: retryAfter, ok: ok, err: err}
|
||||
}
|
||||
@@ -0,0 +1,387 @@
|
||||
package cloud
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kerberos-io/agent/machinery/src/models"
|
||||
)
|
||||
|
||||
func TestParseRetryAfter(t *testing.T) {
|
||||
now := time.Date(2026, 11, 16, 22, 0, 0, 0, time.UTC)
|
||||
tests := []struct {
|
||||
name string
|
||||
value string
|
||||
want time.Duration
|
||||
wantOK bool
|
||||
}{
|
||||
{"delay seconds", "120", 120 * time.Second, true},
|
||||
{"delay seconds with spaces", " 3600 ", time.Hour, true},
|
||||
{"zero", "0", 0, true},
|
||||
{"IMF-fixdate", "Mon, 16 Nov 2026 23:00:00 GMT", time.Hour, true},
|
||||
{"RFC 850 date", "Monday, 16-Nov-26 23:30:00 GMT", 90 * time.Minute, true},
|
||||
{"asctime date", "Mon Nov 16 22:00:30 2026", 30 * time.Second, true},
|
||||
{"date in the past", "Mon, 16 Nov 2026 21:00:00 GMT", -time.Hour, true},
|
||||
{"more than the cap", "4000000", uploadPauseMaxRetryAfter, true},
|
||||
{"overflowing number", "99999999999999999999999999", uploadPauseMaxRetryAfter, true},
|
||||
{"missing", "", 0, false},
|
||||
{"negative", "-5", 0, false},
|
||||
{"fraction", "1.5", 0, false},
|
||||
{"unit suffix", "10s", 0, false},
|
||||
{"garbage", "tomorrow", 0, false},
|
||||
{"ISO date", "2026-11-17T00:00:00Z", 0, false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, ok := parseRetryAfter(tt.value, now)
|
||||
if ok != tt.wantOK || got != tt.want {
|
||||
t.Fatalf("parseRetryAfter(%q) = %v, %v; want %v, %v", tt.value, got, ok, tt.want, tt.wantOK)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestClampRetryAfter(t *testing.T) {
|
||||
tests := []struct {
|
||||
in, want time.Duration
|
||||
}{
|
||||
{-time.Hour, uploadPauseMinBackoff},
|
||||
{0, uploadPauseMinBackoff},
|
||||
{time.Second, uploadPauseMinBackoff},
|
||||
{2 * time.Hour, 2 * time.Hour},
|
||||
{uploadPauseMaxRetryAfter, uploadPauseMaxRetryAfter},
|
||||
{40 * 24 * time.Hour, uploadPauseMaxRetryAfter},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := clampRetryAfter(tt.in); got != tt.want {
|
||||
t.Errorf("clampRetryAfter(%v) = %v, want %v", tt.in, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryAfterFromResponseUsesDateHeader(t *testing.T) {
|
||||
// The agent clock may be wrong; an HTTP-date is measured against the
|
||||
// response Date so the pause still ends at the start of Hub's next period.
|
||||
resp := &http.Response{Header: http.Header{}}
|
||||
resp.Header.Set("Date", "Mon, 16 Nov 2020 22:00:00 GMT")
|
||||
resp.Header.Set("Retry-After", "Tue, 17 Nov 2020 00:00:00 GMT")
|
||||
got, ok := retryAfterFromResponse(resp)
|
||||
if !ok || got != 2*time.Hour {
|
||||
t.Fatalf("retryAfterFromResponse = %v, %v; want 2h, true", got, ok)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeClock struct {
|
||||
mu sync.Mutex
|
||||
now time.Time
|
||||
}
|
||||
|
||||
func (c *fakeClock) Now() time.Time {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.now
|
||||
}
|
||||
|
||||
func (c *fakeClock) Advance(d time.Duration) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.now = c.now.Add(d)
|
||||
}
|
||||
|
||||
func TestUploadPauseHonoursRetryAfter(t *testing.T) {
|
||||
clock := &fakeClock{now: time.Date(2026, 11, 16, 22, 0, 0, 0, time.UTC)}
|
||||
p := newUploadPause(clock.Now)
|
||||
|
||||
if got := p.Remaining(); got != 0 {
|
||||
t.Fatalf("Remaining before any 429 = %v, want 0", got)
|
||||
}
|
||||
if got := p.Throttled(2*time.Hour, true); got != 2*time.Hour {
|
||||
t.Fatalf("Throttled = %v, want 2h", got)
|
||||
}
|
||||
if got := p.Remaining(); got != 2*time.Hour {
|
||||
t.Fatalf("Remaining = %v, want 2h", got)
|
||||
}
|
||||
clock.Advance(90 * time.Minute)
|
||||
if got := p.Remaining(); got != 30*time.Minute {
|
||||
t.Fatalf("Remaining after 90m = %v, want 30m", got)
|
||||
}
|
||||
clock.Advance(30 * time.Minute)
|
||||
if got := p.Remaining(); got != 0 {
|
||||
t.Fatalf("Remaining at the deadline = %v, want 0", got)
|
||||
}
|
||||
|
||||
// Absurd values are clamped.
|
||||
if got := p.Throttled(-time.Hour, true); got != uploadPauseMinBackoff {
|
||||
t.Fatalf("Throttled(past) = %v, want %v", got, uploadPauseMinBackoff)
|
||||
}
|
||||
if got := p.Throttled(365*24*time.Hour, true); got != uploadPauseMaxRetryAfter {
|
||||
t.Fatalf("Throttled(1 year) = %v, want %v", got, uploadPauseMaxRetryAfter)
|
||||
}
|
||||
|
||||
p.Clear()
|
||||
if got := p.Remaining(); got != 0 {
|
||||
t.Fatalf("Remaining after Clear = %v, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadPauseBacksOffWithoutRetryAfter(t *testing.T) {
|
||||
clock := &fakeClock{now: time.Date(2026, 11, 16, 22, 0, 0, 0, time.UTC)}
|
||||
p := newUploadPause(clock.Now)
|
||||
|
||||
want := []time.Duration{
|
||||
5 * time.Second, 10 * time.Second, 20 * time.Second, 40 * time.Second,
|
||||
80 * time.Second, 160 * time.Second, 320 * time.Second, 640 * time.Second,
|
||||
15 * time.Minute, 15 * time.Minute,
|
||||
}
|
||||
for i, w := range want {
|
||||
if got := p.Throttled(0, false); got != w {
|
||||
t.Fatalf("attempt %d: Throttled = %v, want %v", i, got, w)
|
||||
}
|
||||
if got := p.Remaining(); got != w {
|
||||
t.Fatalf("attempt %d: Remaining = %v, want %v", i, got, w)
|
||||
}
|
||||
clock.Advance(w)
|
||||
}
|
||||
|
||||
// A valid Retry-After does not disturb the back-off sequence.
|
||||
if got := p.Throttled(time.Hour, true); got != time.Hour {
|
||||
t.Fatalf("Throttled with Retry-After = %v, want 1h", got)
|
||||
}
|
||||
if got := p.Throttled(0, false); got != 15*time.Minute {
|
||||
t.Fatalf("back-off after Retry-After = %v, want 15m", got)
|
||||
}
|
||||
|
||||
// Clear (Hub answered something other than 429) restarts the back-off.
|
||||
p.Clear()
|
||||
if got := p.Throttled(0, false); got != 5*time.Second {
|
||||
t.Fatalf("back-off after Clear = %v, want 5s", got)
|
||||
}
|
||||
p.Reset()
|
||||
if got := p.Remaining(); got != 0 {
|
||||
t.Fatalf("Remaining after Reset = %v, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadPauseConcurrentUse(t *testing.T) {
|
||||
p := newUploadPause(time.Now)
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 16; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
for j := 0; j < 100; j++ {
|
||||
switch (i + j) % 4 {
|
||||
case 0:
|
||||
p.Throttled(time.Minute, true)
|
||||
case 1:
|
||||
p.Throttled(0, false)
|
||||
case 2:
|
||||
p.Remaining()
|
||||
case 3:
|
||||
p.Clear()
|
||||
}
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestThrottledErrorOnlyFor429(t *testing.T) {
|
||||
base := errors.New("boom")
|
||||
resp := &http.Response{StatusCode: http.StatusForbidden, Header: http.Header{}}
|
||||
if got := throttledError(resp, base); got != base {
|
||||
t.Fatalf("throttledError(403) = %v, want the original error", got)
|
||||
}
|
||||
resp = &http.Response{StatusCode: http.StatusTooManyRequests, Header: http.Header{}}
|
||||
resp.Header.Set("Retry-After", "60")
|
||||
var throttled *uploadThrottledError
|
||||
if err := throttledError(resp, base); !errors.As(err, &throttled) || !errors.Is(err, base) {
|
||||
t.Fatalf("throttledError(429) = %v, want an uploadThrottledError wrapping the original", err)
|
||||
}
|
||||
if !throttled.ok || throttled.retryAfter != time.Minute {
|
||||
t.Fatalf("retryAfter = %v, %v; want 1m, true", throttled.retryAfter, throttled.ok)
|
||||
}
|
||||
}
|
||||
|
||||
// withHubUploadPause gives a test a fresh shared pause and restores it after.
|
||||
func withHubUploadPause(t *testing.T) {
|
||||
t.Helper()
|
||||
previous := hubUploadPause
|
||||
hubUploadPause = newUploadPause(time.Now)
|
||||
t.Cleanup(func() { hubUploadPause = previous })
|
||||
}
|
||||
|
||||
func testHubConfiguration(hubURI string) *models.Configuration {
|
||||
return &models.Configuration{Config: *testHubConfig(hubURI)}
|
||||
}
|
||||
|
||||
func TestUploadKerberosHubTusCreate429PausesWithoutRetrying(t *testing.T) {
|
||||
withHubUploadPause(t)
|
||||
previousDelay := tusBackoffBaseDelay
|
||||
tusBackoffBaseDelay = 0
|
||||
t.Cleanup(func() { tusBackoffBaseDelay = previousDelay })
|
||||
|
||||
var posts int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodPost && strings.HasPrefix(r.URL.Path, tusUploadPath) {
|
||||
atomic.AddInt32(&posts, 1)
|
||||
w.Header().Set("Retry-After", time.Now().Add(3*time.Hour).UTC().Format(http.TimeFormat))
|
||||
http.Error(w, `{"error":"plan_limit_reached"}`, http.StatusTooManyRequests)
|
||||
return
|
||||
}
|
||||
t.Errorf("unexpected request %s %s", r.Method, r.URL.Path)
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
fileName := "1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4"
|
||||
withRecording(t, fileName, bytes.Repeat([]byte("r"), 1024))
|
||||
|
||||
uploaded, configured, err := UploadKerberosHub(testHubConfiguration(server.URL), fileName)
|
||||
if uploaded || !configured || err != nil {
|
||||
t.Fatalf("UploadKerberosHub = %v, %v, %v; want false, true, nil", uploaded, configured, err)
|
||||
}
|
||||
if got := atomic.LoadInt32(&posts); got != 1 {
|
||||
t.Fatalf("tus create requests = %d, want 1 (no in-cycle retries on 429)", got)
|
||||
}
|
||||
if remaining := hubUploadPause.Remaining(); remaining < 2*time.Hour+50*time.Minute || remaining > 3*time.Hour {
|
||||
t.Fatalf("pause = %v, want about 3h", remaining)
|
||||
}
|
||||
if _, err := os.Stat("data/recordings/" + fileName); err != nil {
|
||||
t.Fatalf("recording must stay on disk: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadKerberosHubTusPatch429KeepsResumeState(t *testing.T) {
|
||||
withHubUploadPause(t)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Tus-Resumable", tusResumableVersion)
|
||||
switch r.Method {
|
||||
case http.MethodPost:
|
||||
w.Header().Set("Location", tusUploadPath+"upload-1")
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
case http.MethodHead:
|
||||
w.Header().Set("Upload-Offset", "0")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case http.MethodPatch:
|
||||
w.WriteHeader(http.StatusTooManyRequests) // no Retry-After
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
fileName := "1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4"
|
||||
withRecording(t, fileName, bytes.Repeat([]byte("r"), 1024))
|
||||
|
||||
uploaded, configured, err := UploadKerberosHub(testHubConfiguration(server.URL), fileName)
|
||||
if uploaded || !configured || err != nil {
|
||||
t.Fatalf("UploadKerberosHub = %v, %v, %v; want false, true, nil", uploaded, configured, err)
|
||||
}
|
||||
if remaining := hubUploadPause.Remaining(); remaining <= 0 || remaining > uploadPauseMinBackoff {
|
||||
t.Fatalf("pause without Retry-After = %v, want up to %v", remaining, uploadPauseMinBackoff)
|
||||
}
|
||||
if _, err := os.Stat(tusSidecarPath(fileName, "hub")); err != nil {
|
||||
t.Fatalf("resume state must be kept during the pause: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadKerberosHubLegacyHead429Pauses(t *testing.T) {
|
||||
withHubUploadPause(t)
|
||||
t.Setenv("AGENT_DISABLE_RESUMABLE_UPLOAD", "true")
|
||||
|
||||
var uploads int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/storage/upload" {
|
||||
t.Errorf("unexpected request %s %s", r.Method, r.URL.Path)
|
||||
return
|
||||
}
|
||||
if r.Method == http.MethodPost {
|
||||
atomic.AddInt32(&uploads, 1)
|
||||
}
|
||||
w.Header().Set("Retry-After", "7200")
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
fileName := "1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4"
|
||||
withRecording(t, fileName, bytes.Repeat([]byte("r"), 1024))
|
||||
|
||||
uploaded, configured, err := UploadKerberosHub(testHubConfiguration(server.URL), fileName)
|
||||
if uploaded || !configured || err != nil {
|
||||
t.Fatalf("UploadKerberosHub = %v, %v, %v; want false, true, nil", uploaded, configured, err)
|
||||
}
|
||||
if got := atomic.LoadInt32(&uploads); got != 0 {
|
||||
t.Fatalf("POST after a 429 HEAD = %d, want 0", got)
|
||||
}
|
||||
if remaining := hubUploadPause.Remaining(); remaining < 119*time.Minute || remaining > 2*time.Hour {
|
||||
t.Fatalf("pause = %v, want about 2h", remaining)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadKerberosHubOtherStatusesDoNotPause(t *testing.T) {
|
||||
// Older Hub versions never send 429; a refusal keeps today's behaviour
|
||||
// (recording kept, retried by the upload loop) and does not pause.
|
||||
withHubUploadPause(t)
|
||||
t.Setenv("AGENT_DISABLE_RESUMABLE_UPLOAD", "true")
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
fileName := "1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4"
|
||||
withRecording(t, fileName, bytes.Repeat([]byte("r"), 1024))
|
||||
|
||||
hubUploadPause.Throttled(0, false) // an earlier 429 episode
|
||||
uploaded, configured, err := UploadKerberosHub(testHubConfiguration(server.URL), fileName)
|
||||
if uploaded || !configured || err != nil {
|
||||
t.Fatalf("UploadKerberosHub = %v, %v, %v; want false, true, nil", uploaded, configured, err)
|
||||
}
|
||||
if remaining := hubUploadPause.Remaining(); remaining != 0 {
|
||||
t.Fatalf("pause after a non-429 answer = %v, want 0", remaining)
|
||||
}
|
||||
}
|
||||
|
||||
// A direct Vault upload is not a Hub upload: a 429 from a vault keeps the
|
||||
// existing in-cycle retries and never sets the Hub upload pause.
|
||||
func TestVaultTusCreate429KeepsInCycleRetries(t *testing.T) {
|
||||
withHubUploadPause(t)
|
||||
previousDelay := tusBackoffBaseDelay
|
||||
tusBackoffBaseDelay = 0
|
||||
t.Cleanup(func() { tusBackoffBaseDelay = previousDelay })
|
||||
|
||||
var posts int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodPost {
|
||||
atomic.AddInt32(&posts, 1)
|
||||
w.Header().Set("Retry-After", "3600")
|
||||
http.Error(w, "slow down", http.StatusTooManyRequests)
|
||||
return
|
||||
}
|
||||
t.Errorf("unexpected request %s %s", r.Method, r.URL.Path)
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
fileName := "1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4"
|
||||
withRecording(t, fileName, bytes.Repeat([]byte("r"), 1024))
|
||||
|
||||
setHeaders := func(h http.Header, fn string) {}
|
||||
uploaded, _, supported, _, _ := runTusUpload(server.URL+tusUploadPath, "", fileName, "vault-test", "primary", setHeaders, false)
|
||||
if uploaded || !supported {
|
||||
t.Fatalf("runTusUpload = uploaded %v, supported %v; want false, true", uploaded, supported)
|
||||
}
|
||||
if got := atomic.LoadInt32(&posts); got < 2 {
|
||||
t.Fatalf("tus create requests = %d, want the in-cycle retries (> 1) for a vault", got)
|
||||
}
|
||||
if remaining := hubUploadPause.Remaining(); remaining != 0 {
|
||||
t.Fatalf("hub upload pause = %v, want none for a vault upload", remaining)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user