mirror of
https://github.com/kerberos-io/agent.git
synced 2026-11-19 22:30:45 +00:00
Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f20d156d86 | ||
|
|
98f1826fe1 | ||
|
|
bc8bc84cae | ||
|
|
8c21638ff2 | ||
|
|
ef81ce5b32 | ||
|
|
8016b3ba93 | ||
|
|
4dc6fe7e18 |
@@ -26,3 +26,28 @@ Kerberos Open source (v3) is a cutting edge video surveillance management system
|
||||
## Security reporting
|
||||
|
||||
For sensitive vulnerabilities, use private disclosure channels documented in [../SECURITY.md](../SECURITY.md).
|
||||
|
||||
## WebRTC startup and keyframe recovery
|
||||
|
||||
Each H264 viewer waits until its WebRTC transport is connected and a fresh IDR
|
||||
arrives before receiving video. Reconnects, viewer PLI/FIR feedback, and a full
|
||||
per-viewer sample buffer reset that viewer's keyframe gate without stalling
|
||||
other viewers. Audio continues to use the existing independent sample queues.
|
||||
Each viewer's skipped and discarded video durations advance its RTP clock before
|
||||
recovery, without sending filler packets, so an IDR wait does not compress the
|
||||
video timeline relative to audio.
|
||||
|
||||
The main and sub streams request keyframes from their own RTSP sources. Viewer
|
||||
PLI and FIR feedback is translated into an RTSP PLI using the camera session's
|
||||
SSRCs, never forwarded with the browser's SSRCs. Requests are coalesced and
|
||||
limited to one per second per broadcaster, retrying while connected viewers
|
||||
still need an IDR. Requests stop once all waiting viewers receive an IDR or
|
||||
the broadcaster closes.
|
||||
|
||||
Camera support for RTCP PLI varies. Unsupported sources, cameras that ignore
|
||||
feedback, and failed requests fall back to the next periodic IDR; failures are
|
||||
logged as `keyframe_request_failed`. No encoder settings are changed. A camera
|
||||
with a long GOP can therefore still start slowly; configure a shorter IDR
|
||||
interval (for example 1-2 seconds) when camera feedback is unavailable, accounting
|
||||
for the bandwidth tradeoff. An isolated cached IDR must not be replayed followed
|
||||
by arbitrary live P-frames: the dependent reference chain would be incomplete.
|
||||
|
||||
+1
-1
@@ -30,6 +30,7 @@ require (
|
||||
github.com/moq-dev/moq-go v0.5.7
|
||||
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
|
||||
github.com/pion/interceptor v0.1.47
|
||||
github.com/pion/rtcp v1.2.17
|
||||
github.com/pion/rtp v1.10.5
|
||||
github.com/pion/webrtc/v4 v4.2.18
|
||||
github.com/sirupsen/logrus v1.9.3
|
||||
@@ -104,7 +105,6 @@ require (
|
||||
github.com/pion/logging v0.2.4 // indirect
|
||||
github.com/pion/mdns/v2 v2.1.0 // indirect
|
||||
github.com/pion/randutil v0.1.0 // indirect
|
||||
github.com/pion/rtcp v1.2.17 // indirect
|
||||
github.com/pion/sctp v1.11.1 // indirect
|
||||
github.com/pion/sdp/v3 v3.0.19 // indirect
|
||||
github.com/pion/srtp/v3 v3.0.12 // indirect
|
||||
|
||||
@@ -101,6 +101,8 @@ type Golibrtsp struct {
|
||||
Client gortsplib.Client
|
||||
clientStarted bool
|
||||
VideoDecoderMutex *sync.Mutex
|
||||
keyframeMu sync.Mutex
|
||||
keyframeReady bool
|
||||
|
||||
VideoH264Index int8
|
||||
VideoH264Media *description.Media
|
||||
@@ -813,6 +815,10 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
|
||||
}
|
||||
}
|
||||
|
||||
g.keyframeMu.Lock()
|
||||
g.keyframeReady = err == nil && g.VideoH264Media != nil && g.VideoH264Decoder != nil
|
||||
g.keyframeMu.Unlock()
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1713,6 +1719,10 @@ func (g *Golibrtsp) Close(ctxOtel context.Context) error {
|
||||
_, span := tracer.Start(ctxOtel, "Close")
|
||||
defer span.End()
|
||||
|
||||
g.keyframeMu.Lock()
|
||||
g.keyframeReady = false
|
||||
g.keyframeMu.Unlock()
|
||||
|
||||
if !g.clientStarted {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
package capture
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/pion/rtcp"
|
||||
)
|
||||
|
||||
func (g *Golibrtsp) RequestKeyframe() error {
|
||||
g.keyframeMu.Lock()
|
||||
defer g.keyframeMu.Unlock()
|
||||
|
||||
if !g.keyframeReady {
|
||||
return errors.New("RTSP H264 keyframe requests are unavailable")
|
||||
}
|
||||
|
||||
media, ok := g.Client.Stats().Session.Medias[g.VideoH264Media]
|
||||
if !ok {
|
||||
return errors.New("RTSP video media is not set up")
|
||||
}
|
||||
video, ok := media.Formats[g.VideoH264Forma]
|
||||
if !ok || video.InboundRTPPackets == 0 {
|
||||
return errors.New("RTSP video RTP has not arrived yet")
|
||||
}
|
||||
|
||||
// Browser and camera SSRCs differ: originate feedback on the RTSP session.
|
||||
if err := g.Client.WritePacketRTCP(g.VideoH264Media, &rtcp.PictureLossIndication{
|
||||
SenderSSRC: video.LocalSSRC,
|
||||
MediaSSRC: video.RemoteSSRC,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("request camera keyframe: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
package capture
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/bluenviron/gortsplib/v5"
|
||||
"github.com/bluenviron/gortsplib/v5/pkg/base"
|
||||
"github.com/bluenviron/gortsplib/v5/pkg/description"
|
||||
"github.com/bluenviron/gortsplib/v5/pkg/format"
|
||||
"github.com/pion/rtcp"
|
||||
"github.com/pion/rtp"
|
||||
)
|
||||
|
||||
type keyframeCamera struct {
|
||||
stream *gortsplib.ServerStream
|
||||
feedback chan rtcp.Packet
|
||||
}
|
||||
|
||||
func (c *keyframeCamera) OnDescribe(*gortsplib.ServerHandlerOnDescribeCtx) (*base.Response, *gortsplib.ServerStream, error) {
|
||||
return &base.Response{StatusCode: base.StatusOK}, c.stream, nil
|
||||
}
|
||||
|
||||
func (c *keyframeCamera) OnSetup(*gortsplib.ServerHandlerOnSetupCtx) (*base.Response, *gortsplib.ServerStream, error) {
|
||||
return &base.Response{StatusCode: base.StatusOK}, c.stream, nil
|
||||
}
|
||||
|
||||
func (c *keyframeCamera) OnPlay(ctx *gortsplib.ServerHandlerOnPlayCtx) (*base.Response, error) {
|
||||
ctx.Session.OnPacketRTCPAny(func(_ *description.Media, pkt rtcp.Packet) {
|
||||
c.feedback <- pkt
|
||||
})
|
||||
return &base.Response{StatusCode: base.StatusOK}, nil
|
||||
}
|
||||
|
||||
func connectedKeyframeCamera(t *testing.T) (*Golibrtsp, *keyframeCamera) {
|
||||
t.Helper()
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { listener.Close() })
|
||||
camera := &keyframeCamera{feedback: make(chan rtcp.Packet, 32)}
|
||||
server := &gortsplib.Server{
|
||||
RTSPAddress: "127.0.0.1:0",
|
||||
Handler: camera,
|
||||
Listen: func(string, string) (net.Listener, error) {
|
||||
return listener, nil
|
||||
},
|
||||
}
|
||||
if err := server.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(server.Close)
|
||||
media := &description.Media{
|
||||
Type: description.MediaTypeVideo,
|
||||
Formats: []format.Format{&format.H264{
|
||||
PayloadTyp: 96,
|
||||
PacketizationMode: 1,
|
||||
}},
|
||||
}
|
||||
camera.stream = &gortsplib.ServerStream{
|
||||
Server: server,
|
||||
Desc: &description.Session{Medias: []*description.Media{media}},
|
||||
}
|
||||
if err := camera.stream.Initialize(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(camera.stream.Close)
|
||||
|
||||
client := &Golibrtsp{Url: "rtsp://" + listener.Addr().String() + "/camera"}
|
||||
t.Cleanup(func() {
|
||||
if err := client.Close(context.Background()); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
})
|
||||
if err := client.Connect(context.Background(), context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := client.RequestKeyframe(); err == nil {
|
||||
t.Fatal("requested a keyframe before the camera SSRC was known")
|
||||
}
|
||||
received := make(chan struct{}, 1)
|
||||
client.Client.OnPacketRTP(client.VideoH264Media, client.VideoH264Forma, func(*rtp.Packet) {
|
||||
received <- struct{}{}
|
||||
})
|
||||
if _, err := client.Client.Play(nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := camera.stream.WritePacketRTP(media, &rtp.Packet{
|
||||
Header: rtp.Header{
|
||||
Version: 2,
|
||||
PayloadType: 96,
|
||||
SSRC: 0x12345678,
|
||||
SequenceNumber: 1,
|
||||
Timestamp: 90000,
|
||||
Marker: true,
|
||||
},
|
||||
Payload: []byte{0x65, 0x88, 0x84},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-received:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("test camera RTP did not arrive")
|
||||
}
|
||||
return client, camera
|
||||
}
|
||||
|
||||
func TestRTSPKeyframeRequestUsesCameraSessionSSRCs(t *testing.T) {
|
||||
client, camera := connectedKeyframeCamera(t)
|
||||
want := client.Client.Stats().Session.Medias[client.VideoH264Media].Formats[client.VideoH264Forma]
|
||||
if err := client.RequestKeyframe(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case pkt := <-camera.feedback:
|
||||
pli, ok := pkt.(*rtcp.PictureLossIndication)
|
||||
if !ok {
|
||||
t.Fatalf("camera received %T, want PLI", pkt)
|
||||
}
|
||||
if pli.MediaSSRC != want.RemoteSSRC || pli.SenderSSRC != want.LocalSSRC {
|
||||
t.Fatalf("camera PLI SSRCs = sender %x media %x, want sender %x media %x",
|
||||
pli.SenderSSRC, pli.MediaSSRC, want.LocalSSRC, want.RemoteSSRC)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("camera did not receive RTCP PLI")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRTSPKeyframeRequestUnavailableBeforeConnectAndAfterClose(t *testing.T) {
|
||||
if err := (&Golibrtsp{}).RequestKeyframe(); err == nil {
|
||||
t.Fatal("uninitialized client accepted a keyframe request")
|
||||
}
|
||||
client, _ := connectedKeyframeCamera(t)
|
||||
if err := client.Close(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := client.RequestKeyframe(); err == nil {
|
||||
t.Fatal("closed client accepted a keyframe request")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRTSPKeyframeRequestConcurrentWithClose(t *testing.T) {
|
||||
client, _ := connectedKeyframeCamera(t)
|
||||
var wg sync.WaitGroup
|
||||
for range 4 {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for range 5 {
|
||||
// Requests may race with Close and legitimately become unavailable.
|
||||
_ = client.RequestKeyframe()
|
||||
}
|
||||
}()
|
||||
}
|
||||
if err := client.Close(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
@@ -72,6 +72,10 @@ func (c *Capture) ClearClients(main, sub, backchannel *Golibrtsp) {
|
||||
}
|
||||
}
|
||||
|
||||
type KeyframeRequester interface {
|
||||
RequestKeyframe() error
|
||||
}
|
||||
|
||||
// RTSPClient is a interface that abstracts the RTSP client implementation.
|
||||
type RTSPClient interface {
|
||||
// Connect to the RTSP server.
|
||||
|
||||
@@ -57,14 +57,27 @@ func HandleUpload(configDirectory string, configuration *models.Configuration, c
|
||||
// Half a second delay between two uploads
|
||||
delay := 500 * time.Millisecond
|
||||
|
||||
// Kerberos Hub answers 429 + Retry-After when the organisation reached
|
||||
// its upload limit; uploads to Hub then pause until that time. A
|
||||
// (re)started loop, e.g. after a reconfiguration, tries again at once.
|
||||
uploadsToHub := config.Cloud == "s3" || config.Cloud == "kerberoshub"
|
||||
hubUploadPause.Reset()
|
||||
|
||||
loop:
|
||||
for {
|
||||
wait := 2 * time.Second
|
||||
if uploadsToHub {
|
||||
if remaining := hubUploadPause.Remaining(); remaining > wait {
|
||||
wait = remaining
|
||||
}
|
||||
}
|
||||
|
||||
// This will check if we need to stop the thread,
|
||||
// because of a reconfiguration.
|
||||
select {
|
||||
case <-communication.HandleUpload:
|
||||
break loop
|
||||
case <-time.After(2 * time.Second):
|
||||
case <-time.After(wait):
|
||||
}
|
||||
|
||||
ff, err := utils.ReadDirectory(watchDirectory)
|
||||
@@ -142,6 +155,12 @@ func HandleUpload(configDirectory string, configuration *models.Configuration, c
|
||||
}
|
||||
}
|
||||
|
||||
if uploadsToHub && hubUploadPause.Remaining() > 0 {
|
||||
// Paused: leave this and the following recordings queued
|
||||
// on disk; the outer loop waits until the pause ends.
|
||||
break
|
||||
}
|
||||
|
||||
time.Sleep(delay)
|
||||
}
|
||||
}
|
||||
@@ -1285,7 +1304,7 @@ func HandleLiveStreamHD(
|
||||
// viewer can pick the resolution it needs per peer connection without
|
||||
// the agent re-negotiating the RTSP source.
|
||||
mainStreams, _ := rtspClient.GetStreams()
|
||||
mainVideoBroadcaster := webrtc.NewVideoBroadcaster(mainStreams)
|
||||
mainVideoBroadcaster := webrtc.NewVideoBroadcaster(mainStreams, rtspClient)
|
||||
mainAudioBroadcaster := webrtc.NewAudioBroadcaster(mainStreams)
|
||||
|
||||
if mainVideoBroadcaster == nil && mainAudioBroadcaster == nil {
|
||||
@@ -1316,7 +1335,7 @@ func HandleLiveStreamHD(
|
||||
var subAudioBroadcaster *webrtc.TrackBroadcaster
|
||||
if subStreamEnabled && rtspSubClient != nil && subQueue != nil {
|
||||
subStreams, _ := rtspSubClient.GetStreams()
|
||||
subVideoBroadcaster = webrtc.NewVideoBroadcaster(subStreams)
|
||||
subVideoBroadcaster = webrtc.NewVideoBroadcaster(subStreams, rtspSubClient)
|
||||
subAudioBroadcaster = webrtc.NewAudioBroadcaster(subStreams)
|
||||
defer func() {
|
||||
if subVideoBroadcaster != nil {
|
||||
@@ -1380,6 +1399,10 @@ func HandleRealtimeProcessing(processingCursor *packets.QueueCursor, configurati
|
||||
|
||||
// Check if we need to enable the realtime processing
|
||||
if config.RealtimeProcessing == "true" {
|
||||
if mqttClient == nil {
|
||||
log.Warn("cloud.RealtimeProcessing(): stopping as MQTT client is not configured.")
|
||||
return
|
||||
}
|
||||
|
||||
hubKey := ""
|
||||
if config.Cloud == "s3" && config.S3 != nil && config.S3.Publickey != "" {
|
||||
@@ -1809,6 +1832,8 @@ func hubPersistenceVerificationError(statusCode int) string {
|
||||
reason += ": the account has no active subscription or the subscription expired"
|
||||
case http.StatusForbidden:
|
||||
reason += ": the device is muted"
|
||||
case http.StatusTooManyRequests:
|
||||
reason += ": the organisation reached its upload limit for this period"
|
||||
}
|
||||
return "cloud.VerifyPersistence(kerberoshub): " + reason + " (HTTP " + strconv.Itoa(statusCode) + ")"
|
||||
}
|
||||
|
||||
@@ -45,8 +45,17 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
|
||||
// endpoint (older deployments) we transparently fall back to the legacy
|
||||
// single-POST upload below.
|
||||
if resumableUploadsEnabled() {
|
||||
uploaded, _, supported, body, rerr := uploadHubResumable(&config, fileName, "UploadKerberosHub", "hub")
|
||||
uploaded, responded, supported, body, rerr := uploadHubResumable(&config, fileName, "UploadKerberosHub", "hub")
|
||||
if supported {
|
||||
var throttled *uploadThrottledError
|
||||
if errors.As(rerr, &throttled) {
|
||||
// Upload limit reached: keep the recording and pause uploads.
|
||||
hubUploadPause.Throttled(throttled.retryAfter, throttled.ok)
|
||||
return false, true, nil
|
||||
}
|
||||
if uploaded || responded {
|
||||
hubUploadPause.Clear()
|
||||
}
|
||||
if uploaded {
|
||||
log.WithFields(log.Fields{
|
||||
"component": "kerberos_hub",
|
||||
@@ -125,6 +134,11 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
|
||||
}
|
||||
|
||||
if err == nil && resp != nil {
|
||||
if resp.StatusCode == http.StatusTooManyRequests {
|
||||
// Upload limit reached: keep the recording and pause uploads.
|
||||
hubUploadPause.Throttled(retryAfterFromResponse(resp))
|
||||
return false, true, nil
|
||||
}
|
||||
if resp.StatusCode == 200 {
|
||||
log.WithFields(log.Fields{
|
||||
"component": "kerberos_hub",
|
||||
@@ -132,6 +146,7 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
|
||||
"status_code": resp.StatusCode,
|
||||
}).Debug("Hub upload authorized")
|
||||
} else {
|
||||
hubUploadPause.Clear()
|
||||
log.WithFields(log.Fields{
|
||||
"component": "kerberos_hub",
|
||||
"event": "upload_rejected",
|
||||
@@ -165,6 +180,12 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
|
||||
|
||||
if err == nil {
|
||||
if resp != nil {
|
||||
if resp.StatusCode == http.StatusTooManyRequests {
|
||||
// Upload limit reached: keep the recording and pause uploads.
|
||||
hubUploadPause.Throttled(retryAfterFromResponse(resp))
|
||||
return false, true, nil
|
||||
}
|
||||
hubUploadPause.Clear()
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err == nil {
|
||||
if resp.StatusCode == 200 {
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
package cloud
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/kerberos-io/agent/machinery/src/models"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/sirupsen/logrus/hooks/test"
|
||||
)
|
||||
|
||||
func TestHandleRealtimeProcessingWithoutMQTTClient(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
config models.Config
|
||||
wantWarning bool
|
||||
}{
|
||||
{
|
||||
name: "enabled without Hub credentials",
|
||||
config: models.Config{RealtimeProcessing: "true"},
|
||||
wantWarning: true,
|
||||
},
|
||||
{
|
||||
name: "disabled",
|
||||
config: models.Config{RealtimeProcessing: "false"},
|
||||
},
|
||||
{
|
||||
name: "offline",
|
||||
config: models.Config{RealtimeProcessing: "true", Offline: "true"},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
hook := test.NewGlobal()
|
||||
defer hook.Reset()
|
||||
|
||||
configuration := &models.Configuration{Config: tt.config}
|
||||
// No cursor or decoder should be accessed without an MQTT client.
|
||||
HandleRealtimeProcessing(nil, configuration, nil, nil, nil)
|
||||
|
||||
warnings := 0
|
||||
for _, entry := range hook.AllEntries() {
|
||||
if entry.Level == log.WarnLevel {
|
||||
warnings++
|
||||
if entry.Message != "cloud.RealtimeProcessing(): stopping as MQTT client is not configured." {
|
||||
t.Errorf("unexpected warning: %s", entry.Message)
|
||||
}
|
||||
}
|
||||
}
|
||||
wantWarnings := 0
|
||||
if tt.wantWarning {
|
||||
wantWarnings = 1
|
||||
}
|
||||
if warnings != wantWarnings {
|
||||
t.Fatalf("warnings = %d, want %d", warnings, wantWarnings)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -120,7 +120,13 @@ type tusHeaderFunc func(h http.Header, fileName string)
|
||||
// - supported: the server exposes a tus endpoint. When false, the caller
|
||||
// should fall back to the legacy single-POST upload (older deployments).
|
||||
// - body: a short message for logging.
|
||||
func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tusHeaderFunc) (uploaded bool, responded bool, supported bool, body string, err error) {
|
||||
//
|
||||
// stopOnRateLimit makes a 429 end the attempt at once (Kerberos Hub: the
|
||||
// organisation reached its upload limit and the caller pauses uploads until
|
||||
// Retry-After). Direct Vault uploads pass false, so a 429 from a vault keeps the
|
||||
// existing in-cycle retries and does not advance the vault's retry budget or
|
||||
// secondary-storage failover early.
|
||||
func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tusHeaderFunc, stopOnRateLimit bool) (uploaded bool, responded bool, supported bool, body string, err error) {
|
||||
file, ferr := utils.OpenRecording("data/recordings", fileName)
|
||||
if file != nil {
|
||||
defer file.Close()
|
||||
@@ -177,6 +183,11 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
|
||||
// The vault does not implement tus; let the caller fall back.
|
||||
return false, false, false, "", cerr
|
||||
}
|
||||
if stopOnRateLimit && status == http.StatusTooManyRequests {
|
||||
// Upload limit reached: retrying now cannot help. The caller
|
||||
// pauses uploads until Retry-After.
|
||||
return false, true, true, "", cerr
|
||||
}
|
||||
log.Info(label + ": resumable create failed, " + cerr.Error())
|
||||
tusBackoff(attempt)
|
||||
continue
|
||||
@@ -196,6 +207,10 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
|
||||
uploadURL = ""
|
||||
continue
|
||||
}
|
||||
if stopOnRateLimit && status == http.StatusTooManyRequests {
|
||||
// Keep the resume state so the upload continues after the pause.
|
||||
return false, true, true, "", herr
|
||||
}
|
||||
log.Info(label + ": resumable head failed, " + herr.Error())
|
||||
tusBackoff(attempt)
|
||||
continue
|
||||
@@ -247,6 +262,11 @@ func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tu
|
||||
lastStatus = status
|
||||
if perr != nil {
|
||||
lastErr = perr
|
||||
if stopOnRateLimit && status == http.StatusTooManyRequests {
|
||||
// The resume state saved at creation stays, so the upload
|
||||
// continues from the server offset after the pause.
|
||||
return false, true, true, "", perr
|
||||
}
|
||||
if status >= 400 {
|
||||
// Definitive rejection (e.g. provider push failed during finalize).
|
||||
// Re-evaluate via HEAD on the next iteration to decide retry/restart.
|
||||
@@ -337,7 +357,7 @@ func uploadVaultResumable(vault models.KStorage, publicKey, deviceKey, fileName,
|
||||
h[name] = append([]string(nil), values...)
|
||||
}
|
||||
}
|
||||
return runTusUpload(baseURL, metadata, fileName, label, slot, setHeaders)
|
||||
return runTusUpload(baseURL, metadata, fileName, label, slot, setHeaders, false)
|
||||
}
|
||||
|
||||
// uploadHubResumable uploads a recording to Kerberos Hub's tus endpoint, which
|
||||
@@ -358,7 +378,7 @@ func uploadHubResumable(config *models.Config, fileName, label, slot string) (bo
|
||||
setHeaders := func(h http.Header, fn string) {
|
||||
setHubTusHeaders(h, config, fn)
|
||||
}
|
||||
return runTusUpload(baseURL, metadata, fileName, label, slot, setHeaders)
|
||||
return runTusUpload(baseURL, metadata, fileName, label, slot, setHeaders, true)
|
||||
}
|
||||
|
||||
func addRecordingTusMetadata(values map[string]string, fileName string) {
|
||||
@@ -408,7 +428,7 @@ func tusCreate(client *http.Client, baseURL string, size int64, metadata string,
|
||||
const maxErrorBodyBytes = 4096
|
||||
body, readErr := io.ReadAll(io.LimitReader(resp.Body, maxErrorBodyBytes+1))
|
||||
if readErr != nil {
|
||||
return "", resp.StatusCode, fmt.Errorf("unexpected status creating upload: %s; read error response: %w", resp.Status, readErr)
|
||||
return "", resp.StatusCode, throttledError(resp, fmt.Errorf("unexpected status creating upload: %s; read error response: %w", resp.Status, readErr))
|
||||
}
|
||||
truncated := len(body) > maxErrorBodyBytes
|
||||
if truncated {
|
||||
@@ -419,9 +439,9 @@ func tusCreate(client *http.Client, baseURL string, size int64, metadata string,
|
||||
message += " [truncated]"
|
||||
}
|
||||
if message != "" {
|
||||
return "", resp.StatusCode, fmt.Errorf("unexpected status creating upload: %s: %q", resp.Status, message)
|
||||
return "", resp.StatusCode, throttledError(resp, fmt.Errorf("unexpected status creating upload: %s: %q", resp.Status, message))
|
||||
}
|
||||
return "", resp.StatusCode, fmt.Errorf("unexpected status creating upload: %s", resp.Status)
|
||||
return "", resp.StatusCode, throttledError(resp, fmt.Errorf("unexpected status creating upload: %s", resp.Status))
|
||||
}
|
||||
// Location identifies the created upload; its response body is optional.
|
||||
if _, err := io.Copy(io.Discard, resp.Body); err != nil {
|
||||
@@ -454,7 +474,7 @@ func tusHead(client *http.Client, uploadURL string, setHeaders tusHeaderFunc) (i
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent {
|
||||
return 0, resp.StatusCode, fmt.Errorf("unexpected status on HEAD: %s", resp.Status)
|
||||
return 0, resp.StatusCode, throttledError(resp, fmt.Errorf("unexpected status on HEAD: %s", resp.Status))
|
||||
}
|
||||
offsetStr := resp.Header.Get("Upload-Offset")
|
||||
offset, perr := strconv.ParseInt(offsetStr, 10, 64)
|
||||
@@ -489,7 +509,7 @@ func tusPatch(client *http.Client, uploadURL string, offset, length int64, file
|
||||
respBody := string(bodyBytes)
|
||||
|
||||
if resp.StatusCode != http.StatusNoContent {
|
||||
return offset, resp.StatusCode, respBody, fmt.Errorf("unexpected status on PATCH: %s, %s", resp.Status, respBody)
|
||||
return offset, resp.StatusCode, respBody, throttledError(resp, fmt.Errorf("unexpected status on PATCH: %s, %s", resp.Status, respBody))
|
||||
}
|
||||
newOffsetStr := resp.Header.Get("Upload-Offset")
|
||||
newOffset, perr := strconv.ParseInt(newOffsetStr, 10, 64)
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
package cloud
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// Kerberos Hub answers a recording upload with 429 Too Many Requests when the
|
||||
// organisation is over its daily or monthly upload limit, and sets Retry-After
|
||||
// to the start of the next period. The agent then pauses uploads to Hub until
|
||||
// that moment instead of retrying every few seconds. Recordings stay queued on
|
||||
// disk and are uploaded, in the usual order, once the pause is over.
|
||||
const (
|
||||
// uploadPauseMinBackoff is the first back-off when a 429 carries no usable
|
||||
// Retry-After. It equals the existing retry interval after a refused upload.
|
||||
uploadPauseMinBackoff = 5 * time.Second
|
||||
// uploadPauseMaxBackoff caps the exponential back-off without Retry-After.
|
||||
uploadPauseMaxBackoff = 15 * time.Minute
|
||||
// uploadPauseMaxRetryAfter caps an explicit Retry-After. A monthly limit
|
||||
// never needs more than 31 days; anything longer is treated as bogus.
|
||||
uploadPauseMaxRetryAfter = 32 * 24 * time.Hour
|
||||
)
|
||||
|
||||
// hubUploadPause is shared by every uploader that talks to Kerberos Hub.
|
||||
var hubUploadPause = newUploadPause(time.Now)
|
||||
|
||||
// uploadPause holds the deadline until which uploads to one target are paused.
|
||||
// It is safe for concurrent use.
|
||||
type uploadPause struct {
|
||||
mu sync.Mutex
|
||||
now func() time.Time
|
||||
until time.Time
|
||||
backoff time.Duration // next back-off when Retry-After is missing
|
||||
active bool // a 429 episode is in progress (for start/end logs)
|
||||
}
|
||||
|
||||
func newUploadPause(now func() time.Time) *uploadPause {
|
||||
return &uploadPause{now: now}
|
||||
}
|
||||
|
||||
// Remaining returns how long uploads stay paused; zero when not paused.
|
||||
func (p *uploadPause) Remaining() time.Duration {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if !p.active {
|
||||
return 0
|
||||
}
|
||||
if remaining := p.until.Sub(p.now()); remaining > 0 {
|
||||
return remaining
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// Throttled records a 429 answer. retryAfter is the delay parsed from the
|
||||
// Retry-After header; ok is false when the header was missing or invalid, in
|
||||
// which case an exponential back-off is used. It returns the pause duration.
|
||||
func (p *uploadPause) Throttled(retryAfter time.Duration, ok bool) time.Duration {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
var wait time.Duration
|
||||
if ok {
|
||||
wait = clampRetryAfter(retryAfter)
|
||||
} else {
|
||||
if p.backoff < uploadPauseMinBackoff {
|
||||
p.backoff = uploadPauseMinBackoff
|
||||
}
|
||||
wait = p.backoff
|
||||
p.backoff *= 2
|
||||
if p.backoff > uploadPauseMaxBackoff {
|
||||
p.backoff = uploadPauseMaxBackoff
|
||||
}
|
||||
}
|
||||
|
||||
now := p.now()
|
||||
p.until = now.Add(wait)
|
||||
fields := log.Fields{
|
||||
"component": "upload",
|
||||
"target": "kerberos_hub",
|
||||
"status_code": http.StatusTooManyRequests,
|
||||
"paused_until": now.Add(wait).Format(time.RFC3339),
|
||||
"pause": wait.Round(time.Second).String(),
|
||||
"retry_after_present": ok,
|
||||
}
|
||||
if !p.active {
|
||||
p.active = true
|
||||
log.WithFields(fields).WithField("event", "upload_paused").
|
||||
Warn("Kerberos Hub refused the upload (upload limit reached); pausing uploads, recordings stay queued on disk")
|
||||
} else {
|
||||
log.WithFields(fields).WithField("event", "upload_pause_extended").
|
||||
Debug("Kerberos Hub still refuses uploads; pause extended")
|
||||
}
|
||||
return wait
|
||||
}
|
||||
|
||||
// Clear ends a pause after Hub answered an upload with anything but 429.
|
||||
func (p *uploadPause) Clear() {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.active {
|
||||
log.WithFields(log.Fields{
|
||||
"component": "upload",
|
||||
"target": "kerberos_hub",
|
||||
"event": "upload_resumed",
|
||||
}).Info("Kerberos Hub accepts uploads again; upload pause ended")
|
||||
}
|
||||
p.active = false
|
||||
p.until = time.Time{}
|
||||
p.backoff = 0
|
||||
}
|
||||
|
||||
// Reset drops any pause without logging, for example when the upload loop is
|
||||
// restarted after a reconfiguration.
|
||||
func (p *uploadPause) Reset() {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.active = false
|
||||
p.until = time.Time{}
|
||||
p.backoff = 0
|
||||
}
|
||||
|
||||
func clampRetryAfter(d time.Duration) time.Duration {
|
||||
if d < uploadPauseMinBackoff {
|
||||
// Zero or a date in the past: wait as long as a normal refused upload.
|
||||
return uploadPauseMinBackoff
|
||||
}
|
||||
if d > uploadPauseMaxRetryAfter {
|
||||
return uploadPauseMaxRetryAfter
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// parseRetryAfter parses a Retry-After value (RFC 9110, section 10.2.3):
|
||||
// either delay-seconds or an HTTP-date. now is the reference time for an
|
||||
// HTTP-date; callers pass the response Date header when present, so a wrong
|
||||
// clock on the agent does not shift the deadline. A negative number or any
|
||||
// other value is invalid. The result can be negative for a date in the past;
|
||||
// Throttled clamps it.
|
||||
func parseRetryAfter(value string, now time.Time) (time.Duration, bool) {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return 0, false
|
||||
}
|
||||
if allDigits(value) {
|
||||
seconds, err := strconv.ParseInt(value, 10, 64)
|
||||
if err != nil || seconds > int64(uploadPauseMaxRetryAfter/time.Second) {
|
||||
// Too large to be meaningful (or to fit a Duration): cap it.
|
||||
return uploadPauseMaxRetryAfter, true
|
||||
}
|
||||
return time.Duration(seconds) * time.Second, true
|
||||
}
|
||||
at, err := http.ParseTime(value)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
return at.Sub(now), true
|
||||
}
|
||||
|
||||
func allDigits(s string) bool {
|
||||
for i := 0; i < len(s); i++ {
|
||||
if s[i] < '0' || s[i] > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return s != ""
|
||||
}
|
||||
|
||||
// retryAfterFromResponse reads Retry-After from a response, using the
|
||||
// response Date header (when valid) as the reference for an HTTP-date.
|
||||
func retryAfterFromResponse(resp *http.Response) (time.Duration, bool) {
|
||||
now := time.Now()
|
||||
if date, err := http.ParseTime(resp.Header.Get("Date")); err == nil {
|
||||
now = date
|
||||
}
|
||||
return parseRetryAfter(resp.Header.Get("Retry-After"), now)
|
||||
}
|
||||
|
||||
// uploadThrottledError marks an upload refused with 429 Too Many Requests and
|
||||
// carries the parsed Retry-After.
|
||||
type uploadThrottledError struct {
|
||||
retryAfter time.Duration
|
||||
ok bool
|
||||
err error
|
||||
}
|
||||
|
||||
func (e *uploadThrottledError) Error() string {
|
||||
if e.err != nil {
|
||||
return e.err.Error()
|
||||
}
|
||||
return fmt.Sprintf("upload refused: %d %s", http.StatusTooManyRequests, http.StatusText(http.StatusTooManyRequests))
|
||||
}
|
||||
|
||||
func (e *uploadThrottledError) Unwrap() error { return e.err }
|
||||
|
||||
// throttledError wraps err in an uploadThrottledError when resp is a 429;
|
||||
// otherwise it returns err unchanged.
|
||||
func throttledError(resp *http.Response, err error) error {
|
||||
if resp == nil || resp.StatusCode != http.StatusTooManyRequests {
|
||||
return err
|
||||
}
|
||||
retryAfter, ok := retryAfterFromResponse(resp)
|
||||
return &uploadThrottledError{retryAfter: retryAfter, ok: ok, err: err}
|
||||
}
|
||||
@@ -0,0 +1,387 @@
|
||||
package cloud
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kerberos-io/agent/machinery/src/models"
|
||||
)
|
||||
|
||||
func TestParseRetryAfter(t *testing.T) {
|
||||
now := time.Date(2026, 11, 16, 22, 0, 0, 0, time.UTC)
|
||||
tests := []struct {
|
||||
name string
|
||||
value string
|
||||
want time.Duration
|
||||
wantOK bool
|
||||
}{
|
||||
{"delay seconds", "120", 120 * time.Second, true},
|
||||
{"delay seconds with spaces", " 3600 ", time.Hour, true},
|
||||
{"zero", "0", 0, true},
|
||||
{"IMF-fixdate", "Mon, 16 Nov 2026 23:00:00 GMT", time.Hour, true},
|
||||
{"RFC 850 date", "Monday, 16-Nov-26 23:30:00 GMT", 90 * time.Minute, true},
|
||||
{"asctime date", "Mon Nov 16 22:00:30 2026", 30 * time.Second, true},
|
||||
{"date in the past", "Mon, 16 Nov 2026 21:00:00 GMT", -time.Hour, true},
|
||||
{"more than the cap", "4000000", uploadPauseMaxRetryAfter, true},
|
||||
{"overflowing number", "99999999999999999999999999", uploadPauseMaxRetryAfter, true},
|
||||
{"missing", "", 0, false},
|
||||
{"negative", "-5", 0, false},
|
||||
{"fraction", "1.5", 0, false},
|
||||
{"unit suffix", "10s", 0, false},
|
||||
{"garbage", "tomorrow", 0, false},
|
||||
{"ISO date", "2026-11-17T00:00:00Z", 0, false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, ok := parseRetryAfter(tt.value, now)
|
||||
if ok != tt.wantOK || got != tt.want {
|
||||
t.Fatalf("parseRetryAfter(%q) = %v, %v; want %v, %v", tt.value, got, ok, tt.want, tt.wantOK)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestClampRetryAfter(t *testing.T) {
|
||||
tests := []struct {
|
||||
in, want time.Duration
|
||||
}{
|
||||
{-time.Hour, uploadPauseMinBackoff},
|
||||
{0, uploadPauseMinBackoff},
|
||||
{time.Second, uploadPauseMinBackoff},
|
||||
{2 * time.Hour, 2 * time.Hour},
|
||||
{uploadPauseMaxRetryAfter, uploadPauseMaxRetryAfter},
|
||||
{40 * 24 * time.Hour, uploadPauseMaxRetryAfter},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := clampRetryAfter(tt.in); got != tt.want {
|
||||
t.Errorf("clampRetryAfter(%v) = %v, want %v", tt.in, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryAfterFromResponseUsesDateHeader(t *testing.T) {
|
||||
// The agent clock may be wrong; an HTTP-date is measured against the
|
||||
// response Date so the pause still ends at the start of Hub's next period.
|
||||
resp := &http.Response{Header: http.Header{}}
|
||||
resp.Header.Set("Date", "Mon, 16 Nov 2020 22:00:00 GMT")
|
||||
resp.Header.Set("Retry-After", "Tue, 17 Nov 2020 00:00:00 GMT")
|
||||
got, ok := retryAfterFromResponse(resp)
|
||||
if !ok || got != 2*time.Hour {
|
||||
t.Fatalf("retryAfterFromResponse = %v, %v; want 2h, true", got, ok)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeClock struct {
|
||||
mu sync.Mutex
|
||||
now time.Time
|
||||
}
|
||||
|
||||
func (c *fakeClock) Now() time.Time {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.now
|
||||
}
|
||||
|
||||
func (c *fakeClock) Advance(d time.Duration) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.now = c.now.Add(d)
|
||||
}
|
||||
|
||||
func TestUploadPauseHonoursRetryAfter(t *testing.T) {
|
||||
clock := &fakeClock{now: time.Date(2026, 11, 16, 22, 0, 0, 0, time.UTC)}
|
||||
p := newUploadPause(clock.Now)
|
||||
|
||||
if got := p.Remaining(); got != 0 {
|
||||
t.Fatalf("Remaining before any 429 = %v, want 0", got)
|
||||
}
|
||||
if got := p.Throttled(2*time.Hour, true); got != 2*time.Hour {
|
||||
t.Fatalf("Throttled = %v, want 2h", got)
|
||||
}
|
||||
if got := p.Remaining(); got != 2*time.Hour {
|
||||
t.Fatalf("Remaining = %v, want 2h", got)
|
||||
}
|
||||
clock.Advance(90 * time.Minute)
|
||||
if got := p.Remaining(); got != 30*time.Minute {
|
||||
t.Fatalf("Remaining after 90m = %v, want 30m", got)
|
||||
}
|
||||
clock.Advance(30 * time.Minute)
|
||||
if got := p.Remaining(); got != 0 {
|
||||
t.Fatalf("Remaining at the deadline = %v, want 0", got)
|
||||
}
|
||||
|
||||
// Absurd values are clamped.
|
||||
if got := p.Throttled(-time.Hour, true); got != uploadPauseMinBackoff {
|
||||
t.Fatalf("Throttled(past) = %v, want %v", got, uploadPauseMinBackoff)
|
||||
}
|
||||
if got := p.Throttled(365*24*time.Hour, true); got != uploadPauseMaxRetryAfter {
|
||||
t.Fatalf("Throttled(1 year) = %v, want %v", got, uploadPauseMaxRetryAfter)
|
||||
}
|
||||
|
||||
p.Clear()
|
||||
if got := p.Remaining(); got != 0 {
|
||||
t.Fatalf("Remaining after Clear = %v, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadPauseBacksOffWithoutRetryAfter(t *testing.T) {
|
||||
clock := &fakeClock{now: time.Date(2026, 11, 16, 22, 0, 0, 0, time.UTC)}
|
||||
p := newUploadPause(clock.Now)
|
||||
|
||||
want := []time.Duration{
|
||||
5 * time.Second, 10 * time.Second, 20 * time.Second, 40 * time.Second,
|
||||
80 * time.Second, 160 * time.Second, 320 * time.Second, 640 * time.Second,
|
||||
15 * time.Minute, 15 * time.Minute,
|
||||
}
|
||||
for i, w := range want {
|
||||
if got := p.Throttled(0, false); got != w {
|
||||
t.Fatalf("attempt %d: Throttled = %v, want %v", i, got, w)
|
||||
}
|
||||
if got := p.Remaining(); got != w {
|
||||
t.Fatalf("attempt %d: Remaining = %v, want %v", i, got, w)
|
||||
}
|
||||
clock.Advance(w)
|
||||
}
|
||||
|
||||
// A valid Retry-After does not disturb the back-off sequence.
|
||||
if got := p.Throttled(time.Hour, true); got != time.Hour {
|
||||
t.Fatalf("Throttled with Retry-After = %v, want 1h", got)
|
||||
}
|
||||
if got := p.Throttled(0, false); got != 15*time.Minute {
|
||||
t.Fatalf("back-off after Retry-After = %v, want 15m", got)
|
||||
}
|
||||
|
||||
// Clear (Hub answered something other than 429) restarts the back-off.
|
||||
p.Clear()
|
||||
if got := p.Throttled(0, false); got != 5*time.Second {
|
||||
t.Fatalf("back-off after Clear = %v, want 5s", got)
|
||||
}
|
||||
p.Reset()
|
||||
if got := p.Remaining(); got != 0 {
|
||||
t.Fatalf("Remaining after Reset = %v, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadPauseConcurrentUse(t *testing.T) {
|
||||
p := newUploadPause(time.Now)
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 16; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
for j := 0; j < 100; j++ {
|
||||
switch (i + j) % 4 {
|
||||
case 0:
|
||||
p.Throttled(time.Minute, true)
|
||||
case 1:
|
||||
p.Throttled(0, false)
|
||||
case 2:
|
||||
p.Remaining()
|
||||
case 3:
|
||||
p.Clear()
|
||||
}
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestThrottledErrorOnlyFor429(t *testing.T) {
|
||||
base := errors.New("boom")
|
||||
resp := &http.Response{StatusCode: http.StatusForbidden, Header: http.Header{}}
|
||||
if got := throttledError(resp, base); got != base {
|
||||
t.Fatalf("throttledError(403) = %v, want the original error", got)
|
||||
}
|
||||
resp = &http.Response{StatusCode: http.StatusTooManyRequests, Header: http.Header{}}
|
||||
resp.Header.Set("Retry-After", "60")
|
||||
var throttled *uploadThrottledError
|
||||
if err := throttledError(resp, base); !errors.As(err, &throttled) || !errors.Is(err, base) {
|
||||
t.Fatalf("throttledError(429) = %v, want an uploadThrottledError wrapping the original", err)
|
||||
}
|
||||
if !throttled.ok || throttled.retryAfter != time.Minute {
|
||||
t.Fatalf("retryAfter = %v, %v; want 1m, true", throttled.retryAfter, throttled.ok)
|
||||
}
|
||||
}
|
||||
|
||||
// withHubUploadPause gives a test a fresh shared pause and restores it after.
|
||||
func withHubUploadPause(t *testing.T) {
|
||||
t.Helper()
|
||||
previous := hubUploadPause
|
||||
hubUploadPause = newUploadPause(time.Now)
|
||||
t.Cleanup(func() { hubUploadPause = previous })
|
||||
}
|
||||
|
||||
func testHubConfiguration(hubURI string) *models.Configuration {
|
||||
return &models.Configuration{Config: *testHubConfig(hubURI)}
|
||||
}
|
||||
|
||||
func TestUploadKerberosHubTusCreate429PausesWithoutRetrying(t *testing.T) {
|
||||
withHubUploadPause(t)
|
||||
previousDelay := tusBackoffBaseDelay
|
||||
tusBackoffBaseDelay = 0
|
||||
t.Cleanup(func() { tusBackoffBaseDelay = previousDelay })
|
||||
|
||||
var posts int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodPost && strings.HasPrefix(r.URL.Path, tusUploadPath) {
|
||||
atomic.AddInt32(&posts, 1)
|
||||
w.Header().Set("Retry-After", time.Now().Add(3*time.Hour).UTC().Format(http.TimeFormat))
|
||||
http.Error(w, `{"error":"plan_limit_reached"}`, http.StatusTooManyRequests)
|
||||
return
|
||||
}
|
||||
t.Errorf("unexpected request %s %s", r.Method, r.URL.Path)
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
fileName := "1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4"
|
||||
withRecording(t, fileName, bytes.Repeat([]byte("r"), 1024))
|
||||
|
||||
uploaded, configured, err := UploadKerberosHub(testHubConfiguration(server.URL), fileName)
|
||||
if uploaded || !configured || err != nil {
|
||||
t.Fatalf("UploadKerberosHub = %v, %v, %v; want false, true, nil", uploaded, configured, err)
|
||||
}
|
||||
if got := atomic.LoadInt32(&posts); got != 1 {
|
||||
t.Fatalf("tus create requests = %d, want 1 (no in-cycle retries on 429)", got)
|
||||
}
|
||||
if remaining := hubUploadPause.Remaining(); remaining < 2*time.Hour+50*time.Minute || remaining > 3*time.Hour {
|
||||
t.Fatalf("pause = %v, want about 3h", remaining)
|
||||
}
|
||||
if _, err := os.Stat("data/recordings/" + fileName); err != nil {
|
||||
t.Fatalf("recording must stay on disk: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadKerberosHubTusPatch429KeepsResumeState(t *testing.T) {
|
||||
withHubUploadPause(t)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Tus-Resumable", tusResumableVersion)
|
||||
switch r.Method {
|
||||
case http.MethodPost:
|
||||
w.Header().Set("Location", tusUploadPath+"upload-1")
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
case http.MethodHead:
|
||||
w.Header().Set("Upload-Offset", "0")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case http.MethodPatch:
|
||||
w.WriteHeader(http.StatusTooManyRequests) // no Retry-After
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
fileName := "1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4"
|
||||
withRecording(t, fileName, bytes.Repeat([]byte("r"), 1024))
|
||||
|
||||
uploaded, configured, err := UploadKerberosHub(testHubConfiguration(server.URL), fileName)
|
||||
if uploaded || !configured || err != nil {
|
||||
t.Fatalf("UploadKerberosHub = %v, %v, %v; want false, true, nil", uploaded, configured, err)
|
||||
}
|
||||
if remaining := hubUploadPause.Remaining(); remaining <= 0 || remaining > uploadPauseMinBackoff {
|
||||
t.Fatalf("pause without Retry-After = %v, want up to %v", remaining, uploadPauseMinBackoff)
|
||||
}
|
||||
if _, err := os.Stat(tusSidecarPath(fileName, "hub")); err != nil {
|
||||
t.Fatalf("resume state must be kept during the pause: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadKerberosHubLegacyHead429Pauses(t *testing.T) {
|
||||
withHubUploadPause(t)
|
||||
t.Setenv("AGENT_DISABLE_RESUMABLE_UPLOAD", "true")
|
||||
|
||||
var uploads int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/storage/upload" {
|
||||
t.Errorf("unexpected request %s %s", r.Method, r.URL.Path)
|
||||
return
|
||||
}
|
||||
if r.Method == http.MethodPost {
|
||||
atomic.AddInt32(&uploads, 1)
|
||||
}
|
||||
w.Header().Set("Retry-After", "7200")
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
fileName := "1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4"
|
||||
withRecording(t, fileName, bytes.Repeat([]byte("r"), 1024))
|
||||
|
||||
uploaded, configured, err := UploadKerberosHub(testHubConfiguration(server.URL), fileName)
|
||||
if uploaded || !configured || err != nil {
|
||||
t.Fatalf("UploadKerberosHub = %v, %v, %v; want false, true, nil", uploaded, configured, err)
|
||||
}
|
||||
if got := atomic.LoadInt32(&uploads); got != 0 {
|
||||
t.Fatalf("POST after a 429 HEAD = %d, want 0", got)
|
||||
}
|
||||
if remaining := hubUploadPause.Remaining(); remaining < 119*time.Minute || remaining > 2*time.Hour {
|
||||
t.Fatalf("pause = %v, want about 2h", remaining)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadKerberosHubOtherStatusesDoNotPause(t *testing.T) {
|
||||
// Older Hub versions never send 429; a refusal keeps today's behaviour
|
||||
// (recording kept, retried by the upload loop) and does not pause.
|
||||
withHubUploadPause(t)
|
||||
t.Setenv("AGENT_DISABLE_RESUMABLE_UPLOAD", "true")
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
fileName := "1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4"
|
||||
withRecording(t, fileName, bytes.Repeat([]byte("r"), 1024))
|
||||
|
||||
hubUploadPause.Throttled(0, false) // an earlier 429 episode
|
||||
uploaded, configured, err := UploadKerberosHub(testHubConfiguration(server.URL), fileName)
|
||||
if uploaded || !configured || err != nil {
|
||||
t.Fatalf("UploadKerberosHub = %v, %v, %v; want false, true, nil", uploaded, configured, err)
|
||||
}
|
||||
if remaining := hubUploadPause.Remaining(); remaining != 0 {
|
||||
t.Fatalf("pause after a non-429 answer = %v, want 0", remaining)
|
||||
}
|
||||
}
|
||||
|
||||
// A direct Vault upload is not a Hub upload: a 429 from a vault keeps the
|
||||
// existing in-cycle retries and never sets the Hub upload pause.
|
||||
func TestVaultTusCreate429KeepsInCycleRetries(t *testing.T) {
|
||||
withHubUploadPause(t)
|
||||
previousDelay := tusBackoffBaseDelay
|
||||
tusBackoffBaseDelay = 0
|
||||
t.Cleanup(func() { tusBackoffBaseDelay = previousDelay })
|
||||
|
||||
var posts int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodPost {
|
||||
atomic.AddInt32(&posts, 1)
|
||||
w.Header().Set("Retry-After", "3600")
|
||||
http.Error(w, "slow down", http.StatusTooManyRequests)
|
||||
return
|
||||
}
|
||||
t.Errorf("unexpected request %s %s", r.Method, r.URL.Path)
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
fileName := "1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4"
|
||||
withRecording(t, fileName, bytes.Repeat([]byte("r"), 1024))
|
||||
|
||||
setHeaders := func(h http.Header, fn string) {}
|
||||
uploaded, _, supported, _, _ := runTusUpload(server.URL+tusUploadPath, "", fileName, "vault-test", "primary", setHeaders, false)
|
||||
if uploaded || !supported {
|
||||
t.Fatalf("runTusUpload = uploaded %v, supported %v; want false, true", uploaded, supported)
|
||||
}
|
||||
if got := atomic.LoadInt32(&posts); got < 2 {
|
||||
t.Fatalf("tus create requests = %d, want the in-cycle retries (> 1) for a vault", got)
|
||||
}
|
||||
if remaining := hubUploadPause.Remaining(); remaining != 0 {
|
||||
t.Fatalf("hub upload pause = %v, want none for a vault upload", remaining)
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,9 @@ package webrtc
|
||||
import (
|
||||
"io"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/pion/rtcp"
|
||||
pionWebRTC "github.com/pion/webrtc/v4"
|
||||
pionMedia "github.com/pion/webrtc/v4/pkg/media"
|
||||
log "github.com/sirupsen/logrus"
|
||||
@@ -12,14 +14,23 @@ import (
|
||||
const (
|
||||
// peerSampleBuffer controls how many samples can be buffered per peer before
|
||||
// dropping. Keeps slow peers from blocking the broadcaster.
|
||||
peerSampleBuffer = 60
|
||||
peerSampleBuffer = 60
|
||||
keyframeRequestInterval = time.Second
|
||||
)
|
||||
|
||||
type peerSample struct {
|
||||
pionMedia.Sample
|
||||
skippedDuration time.Duration
|
||||
}
|
||||
|
||||
// peerTrack is a per-peer track with its own non-blocking sample channel.
|
||||
type peerTrack struct {
|
||||
track *pionWebRTC.TrackLocalStaticSample
|
||||
samples chan pionMedia.Sample
|
||||
done chan struct{}
|
||||
track *pionWebRTC.TrackLocalStaticSample
|
||||
samples chan peerSample
|
||||
done chan struct{}
|
||||
connected bool
|
||||
needsKeyframe bool
|
||||
skippedDuration time.Duration
|
||||
}
|
||||
|
||||
// TrackBroadcaster fans out media samples to multiple peer-specific tracks
|
||||
@@ -27,11 +38,17 @@ type peerTrack struct {
|
||||
// goroutine that drains samples independently, so a slow/congested peer
|
||||
// cannot stall the others.
|
||||
type TrackBroadcaster struct {
|
||||
mu sync.RWMutex
|
||||
peers map[string]*peerTrack
|
||||
mimeType string
|
||||
id string
|
||||
streamID string
|
||||
mu sync.RWMutex
|
||||
peers map[string]*peerTrack
|
||||
mimeType string
|
||||
id string
|
||||
streamID string
|
||||
requestKeyframe func() error
|
||||
requestTimer *time.Timer
|
||||
requestInFlight bool
|
||||
lastRequest time.Time
|
||||
requests sync.WaitGroup
|
||||
closed bool
|
||||
}
|
||||
|
||||
// NewTrackBroadcaster creates a new broadcaster for either video or audio.
|
||||
@@ -57,12 +74,18 @@ func (b *TrackBroadcaster) AddPeer(sessionKey string) (*pionWebRTC.TrackLocalSta
|
||||
}
|
||||
|
||||
pt := &peerTrack{
|
||||
track: track,
|
||||
samples: make(chan pionMedia.Sample, peerSampleBuffer),
|
||||
done: make(chan struct{}),
|
||||
track: track,
|
||||
samples: make(chan peerSample, peerSampleBuffer),
|
||||
done: make(chan struct{}),
|
||||
connected: b.mimeType != pionWebRTC.MimeTypeH264,
|
||||
needsKeyframe: b.mimeType == pionWebRTC.MimeTypeH264,
|
||||
}
|
||||
|
||||
b.mu.Lock()
|
||||
if b.closed {
|
||||
b.mu.Unlock()
|
||||
return nil, io.ErrClosedPipe
|
||||
}
|
||||
b.peers[sessionKey] = pt
|
||||
b.mu.Unlock()
|
||||
|
||||
@@ -70,7 +93,7 @@ func (b *TrackBroadcaster) AddPeer(sessionKey string) (*pionWebRTC.TrackLocalSta
|
||||
go func() {
|
||||
defer close(pt.done)
|
||||
for sample := range pt.samples {
|
||||
if err := pt.track.WriteSample(sample); err != nil {
|
||||
if err := writePeerSample(pt.track, sample); err != nil {
|
||||
if err == io.ErrClosedPipe {
|
||||
return
|
||||
}
|
||||
@@ -91,6 +114,16 @@ func (b *TrackBroadcaster) AddPeer(sessionKey string) (*pionWebRTC.TrackLocalSta
|
||||
return track, nil
|
||||
}
|
||||
|
||||
func writePeerSample(track *pionWebRTC.TrackLocalStaticSample, sample peerSample) error {
|
||||
if sample.skippedDuration > 0 {
|
||||
// An empty sample advances Pion's RTP clock without sending a packet.
|
||||
if err := track.WriteSample(pionMedia.Sample{Duration: sample.skippedDuration}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return track.WriteSample(sample.Sample)
|
||||
}
|
||||
|
||||
// RemovePeer stops the writer goroutine and removes the peer.
|
||||
func (b *TrackBroadcaster) RemovePeer(sessionKey string) {
|
||||
b.mu.Lock()
|
||||
@@ -114,14 +147,37 @@ func (b *TrackBroadcaster) RemovePeer(sessionKey string) {
|
||||
// WriteSample fans out a sample to all connected peers without blocking.
|
||||
// If a peer's buffer is full (slow consumer), the sample is dropped for
|
||||
// that peer only — other peers are unaffected.
|
||||
func (b *TrackBroadcaster) WriteSample(sample pionMedia.Sample) {
|
||||
b.mu.RLock()
|
||||
defer b.mu.RUnlock()
|
||||
func (b *TrackBroadcaster) WriteSample(sample pionMedia.Sample, isKeyframe bool) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.closed {
|
||||
return
|
||||
}
|
||||
|
||||
for sessionKey, pt := range b.peers {
|
||||
video := b.mimeType == pionWebRTC.MimeTypeH264
|
||||
if video && (!pt.connected || (pt.needsKeyframe && !isKeyframe)) {
|
||||
pt.skippedDuration += sample.Duration
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case pt.samples <- sample:
|
||||
case pt.samples <- peerSample{Sample: sample, skippedDuration: pt.skippedDuration}:
|
||||
pt.skippedDuration = 0
|
||||
if video && isKeyframe {
|
||||
pt.needsKeyframe = false
|
||||
}
|
||||
default:
|
||||
if video {
|
||||
pt.needsKeyframe = true
|
||||
drainPeerSamples(pt)
|
||||
if isKeyframe {
|
||||
pt.samples <- peerSample{Sample: sample, skippedDuration: pt.skippedDuration}
|
||||
pt.skippedDuration = 0
|
||||
pt.needsKeyframe = false
|
||||
} else {
|
||||
pt.skippedDuration += sample.Duration
|
||||
}
|
||||
}
|
||||
log.WithFields(log.Fields{
|
||||
"component": "webrtc",
|
||||
"event": "slow_peer_sample_dropped",
|
||||
@@ -129,6 +185,106 @@ func (b *TrackBroadcaster) WriteSample(sample pionMedia.Sample) {
|
||||
}).Warn("Dropping WebRTC sample for slow peer")
|
||||
}
|
||||
}
|
||||
b.scheduleKeyframeRequestLocked()
|
||||
}
|
||||
|
||||
func drainPeerSamples(pt *peerTrack) {
|
||||
for {
|
||||
select {
|
||||
case sample := <-pt.samples:
|
||||
pt.skippedDuration += sample.skippedDuration + sample.Duration
|
||||
default:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (b *TrackBroadcaster) SetPeerConnected(sessionKey string, connected bool) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
|
||||
pt, ok := b.peers[sessionKey]
|
||||
if !ok || b.mimeType != pionWebRTC.MimeTypeH264 || pt.connected == connected {
|
||||
return
|
||||
}
|
||||
pt.connected = connected
|
||||
pt.needsKeyframe = true
|
||||
drainPeerSamples(pt)
|
||||
b.scheduleKeyframeRequestLocked()
|
||||
}
|
||||
|
||||
func (b *TrackBroadcaster) HandleRTCP(sessionKey string, data []byte) error {
|
||||
feedback, err := rtcp.Unmarshal(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, packet := range feedback {
|
||||
switch packet.(type) {
|
||||
case *rtcp.PictureLossIndication, *rtcp.FullIntraRequest:
|
||||
b.mu.Lock()
|
||||
if pt, ok := b.peers[sessionKey]; ok && pt.connected && b.mimeType == pionWebRTC.MimeTypeH264 {
|
||||
pt.needsKeyframe = true
|
||||
drainPeerSamples(pt)
|
||||
b.scheduleKeyframeRequestLocked()
|
||||
}
|
||||
b.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *TrackBroadcaster) waitingForKeyframeLocked() bool {
|
||||
for _, pt := range b.peers {
|
||||
if pt.connected && pt.needsKeyframe {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (b *TrackBroadcaster) scheduleKeyframeRequestLocked() {
|
||||
if b.closed || b.requestKeyframe == nil || b.requestInFlight || !b.waitingForKeyframeLocked() {
|
||||
return
|
||||
}
|
||||
if b.requestTimer == nil {
|
||||
delay := time.Until(b.lastRequest.Add(keyframeRequestInterval))
|
||||
if delay < 0 {
|
||||
delay = 0
|
||||
}
|
||||
b.requestTimer = time.AfterFunc(delay, b.sendKeyframeRequest)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *TrackBroadcaster) sendKeyframeRequest() {
|
||||
b.mu.Lock()
|
||||
b.requestTimer = nil
|
||||
if b.closed || !b.waitingForKeyframeLocked() {
|
||||
b.mu.Unlock()
|
||||
return
|
||||
}
|
||||
b.lastRequest = time.Now()
|
||||
b.requestInFlight = true
|
||||
b.requests.Add(1)
|
||||
b.mu.Unlock()
|
||||
defer b.requests.Done()
|
||||
|
||||
if err := b.requestKeyframe(); err != nil {
|
||||
log.WithError(err).WithFields(log.Fields{
|
||||
"component": "webrtc",
|
||||
"event": "keyframe_request_failed",
|
||||
}).Warn("Camera keyframe request failed; waiting for a periodic IDR")
|
||||
} else {
|
||||
log.WithFields(log.Fields{
|
||||
"component": "webrtc",
|
||||
"event": "keyframe_requested",
|
||||
}).Debug("Requested camera keyframe")
|
||||
}
|
||||
|
||||
b.mu.Lock()
|
||||
b.requestInFlight = false
|
||||
b.scheduleKeyframeRequestLocked()
|
||||
b.mu.Unlock()
|
||||
}
|
||||
|
||||
// PeerCount returns the current number of connected peers.
|
||||
@@ -141,11 +297,17 @@ func (b *TrackBroadcaster) PeerCount() int {
|
||||
// Close removes all peers and stops all writer goroutines.
|
||||
func (b *TrackBroadcaster) Close() {
|
||||
b.mu.Lock()
|
||||
b.closed = true
|
||||
if b.requestTimer != nil {
|
||||
b.requestTimer.Stop()
|
||||
b.requestTimer = nil
|
||||
}
|
||||
keys := make([]string, 0, len(b.peers))
|
||||
for k := range b.peers {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
b.mu.Unlock()
|
||||
b.requests.Wait()
|
||||
|
||||
for _, key := range keys {
|
||||
b.RemovePeer(key)
|
||||
|
||||
@@ -0,0 +1,360 @@
|
||||
package webrtc
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kerberos-io/agent/machinery/src/capture"
|
||||
"github.com/kerberos-io/agent/machinery/src/models"
|
||||
"github.com/kerberos-io/agent/machinery/src/packets"
|
||||
"github.com/pion/rtcp"
|
||||
pionWebRTC "github.com/pion/webrtc/v4"
|
||||
pionMedia "github.com/pion/webrtc/v4/pkg/media"
|
||||
)
|
||||
|
||||
func bufferedTestPeer(capacity int) *peerTrack {
|
||||
done := make(chan struct{})
|
||||
close(done)
|
||||
return &peerTrack{
|
||||
samples: make(chan peerSample, capacity),
|
||||
done: done,
|
||||
needsKeyframe: true,
|
||||
}
|
||||
}
|
||||
|
||||
func testVideoBroadcaster(t *testing.T) (*TrackBroadcaster, *peerTrack) {
|
||||
t.Helper()
|
||||
b := NewTrackBroadcaster(pionWebRTC.MimeTypeH264, "video", "test")
|
||||
pt := bufferedTestPeer(peerSampleBuffer)
|
||||
b.peers["viewer"] = pt
|
||||
t.Cleanup(b.Close)
|
||||
return b, pt
|
||||
}
|
||||
|
||||
func TestVideoPeerStartsAndReconnectsAtKeyframe(t *testing.T) {
|
||||
b, pt := testVideoBroadcaster(t)
|
||||
sample := pionMedia.Sample{Data: []byte{1}}
|
||||
b.WriteSample(sample, true)
|
||||
if len(pt.samples) != 0 {
|
||||
t.Fatal("sent a keyframe before the transport connected")
|
||||
}
|
||||
|
||||
b.SetPeerConnected("viewer", true)
|
||||
b.WriteSample(sample, false)
|
||||
if len(pt.samples) != 0 {
|
||||
t.Fatal("sent a dependent frame before the first IDR")
|
||||
}
|
||||
b.WriteSample(sample, true)
|
||||
b.WriteSample(sample, false)
|
||||
if len(pt.samples) != 2 || pt.needsKeyframe {
|
||||
t.Fatal("did not release video after the first IDR")
|
||||
}
|
||||
|
||||
b.SetPeerConnected("viewer", false)
|
||||
b.WriteSample(sample, true)
|
||||
if len(pt.samples) != 0 {
|
||||
t.Fatal("retained or sent video while disconnected")
|
||||
}
|
||||
b.SetPeerConnected("viewer", true)
|
||||
b.WriteSample(sample, false)
|
||||
if len(pt.samples) != 0 {
|
||||
t.Fatal("reconnected viewer resumed on a dependent frame")
|
||||
}
|
||||
b.WriteSample(sample, true)
|
||||
if len(pt.samples) != 1 {
|
||||
t.Fatal("reconnected viewer did not resume at an IDR")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLateViewerDoesNotResetExistingViewer(t *testing.T) {
|
||||
b, first := testVideoBroadcaster(t)
|
||||
b.SetPeerConnected("viewer", true)
|
||||
b.WriteSample(pionMedia.Sample{}, true)
|
||||
<-first.samples
|
||||
|
||||
late := bufferedTestPeer(peerSampleBuffer)
|
||||
b.peers["late"] = late
|
||||
b.SetPeerConnected("late", true)
|
||||
b.WriteSample(pionMedia.Sample{}, false)
|
||||
if len(first.samples) != 1 || len(late.samples) != 0 {
|
||||
t.Fatal("late viewer either interrupted an existing viewer or received a dependent frame")
|
||||
}
|
||||
b.WriteSample(pionMedia.Sample{}, true)
|
||||
if len(late.samples) != 1 {
|
||||
t.Fatal("late viewer did not receive the next IDR")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVideoFeedbackRequestsFreshKeyframe(t *testing.T) {
|
||||
for _, feedback := range []rtcp.Packet{
|
||||
&rtcp.PictureLossIndication{MediaSSRC: 123},
|
||||
&rtcp.FullIntraRequest{FIR: []rtcp.FIREntry{{SSRC: 123, SequenceNumber: 1}}},
|
||||
} {
|
||||
t.Run(fmt.Sprintf("%T", feedback), func(t *testing.T) {
|
||||
b, pt := testVideoBroadcaster(t)
|
||||
b.SetPeerConnected("viewer", true)
|
||||
b.WriteSample(pionMedia.Sample{}, true)
|
||||
b.WriteSample(pionMedia.Sample{}, false)
|
||||
|
||||
rr, err := rtcp.Marshal([]rtcp.Packet{&rtcp.ReceiverReport{}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := b.HandleRTCP("viewer", rr); err != nil || len(pt.samples) != 2 {
|
||||
t.Fatal("receiver report reset the video gate", err)
|
||||
}
|
||||
data, err := rtcp.Marshal([]rtcp.Packet{&rtcp.ReceiverReport{}, feedback})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := b.HandleRTCP("viewer", data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.WriteSample(pionMedia.Sample{}, false)
|
||||
if len(pt.samples) != 0 || !pt.needsKeyframe {
|
||||
t.Fatal("PLI/FIR did not discard stale queued frames and wait for an IDR")
|
||||
}
|
||||
b.WriteSample(pionMedia.Sample{}, true)
|
||||
if len(pt.samples) != 1 {
|
||||
t.Fatal("PLI/FIR recovery did not resume at an IDR")
|
||||
}
|
||||
if err := b.HandleRTCP("viewer", []byte{0}); err == nil {
|
||||
t.Fatal("malformed RTCP was accepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSlowVideoPeerRecoversWithoutBlockingOthers(t *testing.T) {
|
||||
for _, overflowIsKeyframe := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "dependent-frame-dropped", true: "new-IDR-replaces-backlog"}[overflowIsKeyframe], func(t *testing.T) {
|
||||
b, fast := testVideoBroadcaster(t)
|
||||
slow := bufferedTestPeer(2)
|
||||
b.peers["slow"] = slow
|
||||
b.SetPeerConnected("viewer", true)
|
||||
b.SetPeerConnected("slow", true)
|
||||
b.WriteSample(pionMedia.Sample{}, true)
|
||||
b.WriteSample(pionMedia.Sample{}, false)
|
||||
b.WriteSample(pionMedia.Sample{}, overflowIsKeyframe)
|
||||
if len(fast.samples) != 3 {
|
||||
t.Fatal("slow viewer interrupted another viewer")
|
||||
}
|
||||
if overflowIsKeyframe {
|
||||
if len(slow.samples) != 1 || slow.needsKeyframe {
|
||||
t.Fatal("fresh IDR did not replace the stale backlog")
|
||||
}
|
||||
} else {
|
||||
b.WriteSample(pionMedia.Sample{}, false)
|
||||
if len(slow.samples) != 0 || !slow.needsKeyframe {
|
||||
t.Fatal("slow viewer forwarded dependent frames after a drop")
|
||||
}
|
||||
b.WriteSample(pionMedia.Sample{}, true)
|
||||
if len(slow.samples) != 1 {
|
||||
t.Fatal("slow viewer did not recover at the next IDR")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAudioDoesNotWaitForVideoKeyframe(t *testing.T) {
|
||||
b := NewTrackBroadcaster(pionWebRTC.MimeTypePCMU, "audio", "test")
|
||||
pt := bufferedTestPeer(2)
|
||||
pt.connected = true
|
||||
pt.needsKeyframe = false
|
||||
b.peers["viewer"] = pt
|
||||
t.Cleanup(b.Close)
|
||||
|
||||
b.SetPeerConnected("viewer", false)
|
||||
for range 3 {
|
||||
b.WriteSample(pionMedia.Sample{}, false)
|
||||
}
|
||||
if len(pt.samples) != 2 {
|
||||
t.Fatal("audio queue behavior changed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVideoSampleBufferPreservesKeyframeFlag(t *testing.T) {
|
||||
b, pt := testVideoBroadcaster(t)
|
||||
b.SetPeerConnected("viewer", true)
|
||||
state := &streamState{}
|
||||
first := packets.Packet{Data: []byte{0x65}, IsKeyFrame: true, Time: 1}
|
||||
processVideoPacket(first, state, b, models.Config{})
|
||||
processVideoPacket(packets.Packet{Data: []byte{0x41}, Time: 34}, state, b, models.Config{})
|
||||
if len(pt.samples) != 1 || !bytes.Equal((<-pt.samples).Data, first.Data) {
|
||||
t.Fatal("buffered IDR lost its keyframe flag")
|
||||
}
|
||||
|
||||
b.SetPeerConnected("viewer", false)
|
||||
b.SetPeerConnected("viewer", true)
|
||||
writeFinalSamples(state, b, nil)
|
||||
if len(pt.samples) != 0 {
|
||||
t.Fatal("final dependent frame was mislabeled as an IDR")
|
||||
}
|
||||
state.lastVideoSample = &pionMedia.Sample{Data: first.Data}
|
||||
state.lastVideoKeyframe = true
|
||||
writeFinalSamples(state, b, nil)
|
||||
if len(pt.samples) != 1 {
|
||||
t.Fatal("final IDR lost its keyframe flag")
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeyframeRequestsCoalesceAndRetryUntilIDR(t *testing.T) {
|
||||
b, _ := testVideoBroadcaster(t)
|
||||
calls := make(chan time.Time, 10)
|
||||
var count atomic.Int32
|
||||
b.requestKeyframe = func() error {
|
||||
calls <- time.Now()
|
||||
if count.Add(1) == 1 {
|
||||
return errors.New("camera is not ready yet")
|
||||
}
|
||||
b.WriteSample(pionMedia.Sample{}, true)
|
||||
return nil
|
||||
}
|
||||
for _, id := range []string{"viewer", "second", "third"} {
|
||||
if id != "viewer" {
|
||||
b.mu.Lock()
|
||||
b.peers[id] = bufferedTestPeer(peerSampleBuffer)
|
||||
b.mu.Unlock()
|
||||
}
|
||||
b.SetPeerConnected(id, true)
|
||||
}
|
||||
var first time.Time
|
||||
select {
|
||||
case first = <-calls:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("connected viewer did not request a keyframe")
|
||||
}
|
||||
feedback, err := rtcp.Marshal([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: 123}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for range 100 {
|
||||
if err := b.HandleRTCP("viewer", feedback); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
select {
|
||||
case second := <-calls:
|
||||
if second.Sub(first) < keyframeRequestInterval-5*time.Millisecond {
|
||||
t.Fatal("feedback bypassed the per-stream request rate limit")
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("failed camera request was not retried")
|
||||
}
|
||||
select {
|
||||
case <-calls:
|
||||
t.Fatal("requests continued after all viewers received an IDR")
|
||||
case <-time.After(keyframeRequestInterval + 100*time.Millisecond):
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeyframeRequestsStopOnDisconnectAndClose(t *testing.T) {
|
||||
b, _ := testVideoBroadcaster(t)
|
||||
var calls atomic.Int32
|
||||
b.requestKeyframe = func() error { calls.Add(1); return nil }
|
||||
b.lastRequest = time.Now()
|
||||
b.SetPeerConnected("viewer", true)
|
||||
b.SetPeerConnected("viewer", false)
|
||||
time.Sleep(keyframeRequestInterval + 50*time.Millisecond)
|
||||
if calls.Load() != 0 {
|
||||
t.Fatal("disconnected viewer requested a keyframe")
|
||||
}
|
||||
b.SetPeerConnected("viewer", true)
|
||||
b.Close()
|
||||
if _, err := b.AddPeer("closed"); !errors.Is(err, io.ErrClosedPipe) {
|
||||
t.Fatalf("AddPeer() after Close() = %v, want closed pipe", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBroadcasterCloseWaitsForKeyframeRequest(t *testing.T) {
|
||||
b, _ := testVideoBroadcaster(t)
|
||||
started, release, closed := make(chan struct{}), make(chan struct{}), make(chan struct{})
|
||||
unblock := sync.OnceFunc(func() { close(release) })
|
||||
defer unblock()
|
||||
b.requestKeyframe = func() error {
|
||||
close(started)
|
||||
<-release
|
||||
return nil
|
||||
}
|
||||
b.SetPeerConnected("viewer", true)
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("request did not start")
|
||||
}
|
||||
go func() { b.Close(); close(closed) }()
|
||||
select {
|
||||
case <-closed:
|
||||
t.Fatal("Close returned before the in-flight camera request completed")
|
||||
case <-time.After(30 * time.Millisecond):
|
||||
}
|
||||
unblock()
|
||||
select {
|
||||
case <-closed:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("Close did not finish")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConcurrentVideoFeedbackAndPeerLifecycle(t *testing.T) {
|
||||
b := NewTrackBroadcaster(pionWebRTC.MimeTypeH264, "video", "test")
|
||||
t.Cleanup(b.Close)
|
||||
data, err := rtcp.Marshal([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: 123}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
for _, id := range []string{"first", "second", "third"} {
|
||||
if _, err := b.AddPeer(id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for j := range 200 {
|
||||
b.SetPeerConnected(id, j%5 != 0)
|
||||
b.WriteSample(pionMedia.Sample{Data: []byte{0, 0, 0, 1, 0x65}, Duration: time.Millisecond}, j%10 == 0)
|
||||
if err := b.HandleRTCP(id, data); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
b.RemovePeer(id)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
type keyframeTestSource struct {
|
||||
capture.RTSPClient
|
||||
request func() error
|
||||
}
|
||||
|
||||
func (s keyframeTestSource) RequestKeyframe() error { return s.request() }
|
||||
|
||||
func TestVideoBroadcastersRequestTheirOwnSource(t *testing.T) {
|
||||
streams := []packets.Stream{{Name: "H264", IsVideo: true}}
|
||||
var mainCalls, subCalls atomic.Int32
|
||||
main := NewVideoBroadcaster(streams, keyframeTestSource{request: func() error { mainCalls.Add(1); return nil }})
|
||||
sub := NewVideoBroadcaster(streams, keyframeTestSource{request: func() error { subCalls.Add(1); return nil }})
|
||||
t.Cleanup(main.Close)
|
||||
t.Cleanup(sub.Close)
|
||||
if err := main.requestKeyframe(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if mainCalls.Load() != 1 || subCalls.Load() != 0 {
|
||||
t.Fatal("main feedback reached the wrong source")
|
||||
}
|
||||
if err := sub.requestKeyframe(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if mainCalls.Load() != 1 || subCalls.Load() != 1 {
|
||||
t.Fatal("sub feedback reached the wrong source")
|
||||
}
|
||||
}
|
||||
@@ -18,7 +18,6 @@ import (
|
||||
|
||||
mqtt "github.com/eclipse/paho.mqtt.golang"
|
||||
"github.com/pion/interceptor"
|
||||
"github.com/pion/interceptor/pkg/intervalpli"
|
||||
pionWebRTC "github.com/pion/webrtc/v4"
|
||||
pionMedia "github.com/pion/webrtc/v4/pkg/media"
|
||||
)
|
||||
@@ -429,16 +428,6 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// Register a intervalpli factory
|
||||
// This interceptor sends a PLI every 3 seconds. A PLI causes a video keyframe to be generated by the sender.
|
||||
// This makes our video seekable and more error resilent, but at a cost of lower picture quality and higher bitrates
|
||||
// A real world application should process incoming RTCP packets from viewers and forward them to senders
|
||||
intervalPliFactory, err := intervalpli.NewReceiverInterceptor()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
interceptorRegistry.Add(intervalPliFactory)
|
||||
|
||||
api := pionWebRTC.NewAPI(
|
||||
pionWebRTC.WithMediaEngine(mediaEngine),
|
||||
pionWebRTC.WithInterceptorRegistry(interceptorRegistry),
|
||||
@@ -519,9 +508,16 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
if _, _, rtcpErr := videoSender.Read(rtcpBuf); rtcpErr != nil {
|
||||
n, _, rtcpErr := videoSender.Read(rtcpBuf)
|
||||
if rtcpErr != nil {
|
||||
if ctx.Err() == nil && rtcpErr != io.EOF && rtcpErr != io.ErrClosedPipe {
|
||||
log.WithError(rtcpErr).Warn("WebRTC video RTCP reader failed")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err := videoBroadcaster.HandleRTCP(sessionKey, rtcpBuf[:n]); err != nil {
|
||||
log.WithError(err).Warn("Ignoring malformed WebRTC video RTCP feedback")
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
@@ -580,6 +576,9 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
|
||||
|
||||
switch connectionState {
|
||||
case pionWebRTC.PeerConnectionStateDisconnected:
|
||||
if videoBroadcaster != nil {
|
||||
videoBroadcaster.SetPeerConnected(sessionKey, false)
|
||||
}
|
||||
// Disconnected is a transient state that can recover.
|
||||
// Start a grace period timer; if we don't recover, then cleanup.
|
||||
wrapper.disconnectMu.Lock()
|
||||
@@ -619,6 +618,9 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
|
||||
cleanupPeerConnection(sessionKey, wrapper)
|
||||
|
||||
case pionWebRTC.PeerConnectionStateConnected:
|
||||
if videoBroadcaster != nil {
|
||||
videoBroadcaster.SetPeerConnected(sessionKey, true)
|
||||
}
|
||||
// Cancel any pending disconnect timer — connection recovered
|
||||
wrapper.disconnectMu.Lock()
|
||||
if wrapper.disconnectTimer != nil {
|
||||
@@ -760,11 +762,17 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
|
||||
}
|
||||
}
|
||||
|
||||
func NewVideoBroadcaster(streams []packets.Stream) *TrackBroadcaster {
|
||||
func NewVideoBroadcaster(streams []packets.Stream, rtspClient capture.RTSPClient) *TrackBroadcaster {
|
||||
// Verify H264 is available (same check as NewVideoTrack)
|
||||
for _, s := range streams {
|
||||
if s.Name == "H264" {
|
||||
return NewTrackBroadcaster(pionWebRTC.MimeTypeH264, "video", trackStreamID)
|
||||
b := NewTrackBroadcaster(pionWebRTC.MimeTypeH264, "video", trackStreamID)
|
||||
if requester, ok := rtspClient.(capture.KeyframeRequester); ok {
|
||||
b.requestKeyframe = requester.RequestKeyframe
|
||||
} else {
|
||||
log.Warn("RTSP source does not support keyframe requests; waiting for periodic IDRs")
|
||||
}
|
||||
return b
|
||||
}
|
||||
}
|
||||
log.Error("webrtc.main.NewVideoBroadcaster(): no H264 stream found")
|
||||
@@ -858,18 +866,19 @@ func NewAudioTrack(streams []packets.Stream) *pionWebRTC.TrackLocalStaticSample
|
||||
|
||||
// streamState holds state information for the streaming process
|
||||
type streamState struct {
|
||||
lastKeepAlive int64
|
||||
peerCount int64
|
||||
start bool
|
||||
catchingUp bool
|
||||
receivedKeyFrame bool
|
||||
lastAudioSample *pionMedia.Sample
|
||||
lastVideoSample *pionMedia.Sample
|
||||
audioPacketsSeen int64
|
||||
aacPacketsSeen int64
|
||||
audioSamplesSent int64
|
||||
aacNoOutput int64
|
||||
aacErrors int64
|
||||
lastKeepAlive int64
|
||||
peerCount int64
|
||||
start bool
|
||||
catchingUp bool
|
||||
receivedKeyFrame bool
|
||||
lastAudioSample *pionMedia.Sample
|
||||
lastVideoSample *pionMedia.Sample
|
||||
lastVideoKeyframe bool
|
||||
audioPacketsSeen int64
|
||||
aacPacketsSeen int64
|
||||
audioSamplesSent int64
|
||||
aacNoOutput int64
|
||||
aacErrors int64
|
||||
}
|
||||
|
||||
// codecSupport tracks which codecs are available in the stream
|
||||
@@ -943,11 +952,11 @@ func updateStreamState(communication *models.Communication, state *streamState)
|
||||
// writeFinalSamples writes any remaining buffered samples
|
||||
func writeFinalSamples(state *streamState, videoBroadcaster, audioBroadcaster *TrackBroadcaster) {
|
||||
if state.lastVideoSample != nil && videoBroadcaster != nil {
|
||||
videoBroadcaster.WriteSample(*state.lastVideoSample)
|
||||
videoBroadcaster.WriteSample(*state.lastVideoSample, state.lastVideoKeyframe)
|
||||
}
|
||||
|
||||
if state.lastAudioSample != nil && audioBroadcaster != nil {
|
||||
audioBroadcaster.WriteSample(*state.lastAudioSample)
|
||||
audioBroadcaster.WriteSample(*state.lastAudioSample, false)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1011,10 +1020,11 @@ func processVideoPacket(pkt packets.Packet, state *streamState, videoBroadcaster
|
||||
|
||||
if state.lastVideoSample != nil {
|
||||
state.lastVideoSample.Duration = sampleDuration(pkt, state.lastVideoSample.PacketTimestamp, 33*time.Millisecond)
|
||||
videoBroadcaster.WriteSample(*state.lastVideoSample)
|
||||
videoBroadcaster.WriteSample(*state.lastVideoSample, state.lastVideoKeyframe)
|
||||
}
|
||||
|
||||
state.lastVideoSample = &sample
|
||||
state.lastVideoKeyframe = pkt.IsKeyFrame
|
||||
}
|
||||
|
||||
// processAudioPacket processes an audio packet and writes samples to the broadcaster.
|
||||
@@ -1097,7 +1107,7 @@ func processAudioPacket(pkt packets.Packet, state *streamState, audioBroadcaster
|
||||
"samples_queued": state.audioSamplesSent,
|
||||
}).Debug("WebRTC audio sample queued")
|
||||
}
|
||||
audioBroadcaster.WriteSample(*state.lastAudioSample)
|
||||
audioBroadcaster.WriteSample(*state.lastAudioSample, false)
|
||||
}
|
||||
|
||||
state.lastAudioSample = &sample
|
||||
|
||||
@@ -1,6 +1,18 @@
|
||||
package webrtc
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kerberos-io/agent/machinery/src/models"
|
||||
"github.com/kerberos-io/agent/machinery/src/packets"
|
||||
"github.com/pion/rtcp"
|
||||
pionWebRTC "github.com/pion/webrtc/v4"
|
||||
pionMedia "github.com/pion/webrtc/v4/pkg/media"
|
||||
)
|
||||
|
||||
func TestBuildICEServersOmitsEmptyURLs(t *testing.T) {
|
||||
webRTC := CreateWebRTC("camera", []string{""}, []string{""}, "", "")
|
||||
@@ -26,6 +38,7 @@ func TestBuildICEServersIncludesConfiguredURLs(t *testing.T) {
|
||||
if len(iceServers) != 2 {
|
||||
t.Fatalf("buildICEServers() returned %d servers, want 2", len(iceServers))
|
||||
}
|
||||
|
||||
if got := iceServers[0].URLs[0]; got != "stun:turn-fra1.kerberos.io:3478" {
|
||||
t.Fatalf("STUN URL = %q, want trimmed URL", got)
|
||||
}
|
||||
@@ -36,3 +49,138 @@ func TestBuildICEServersIncludesConfiguredURLs(t *testing.T) {
|
||||
t.Fatal("TURN credentials were not preserved")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebRTCConnectionRequestsStartupAndRecoveryKeyframes(t *testing.T) {
|
||||
viewer, err := pionWebRTC.NewPeerConnection(pionWebRTC.Configuration{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := viewer.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
})
|
||||
if _, err := viewer.AddTransceiverFromKind(pionWebRTC.RTPCodecTypeVideo, pionWebRTC.RTPTransceiverInit{
|
||||
Direction: pionWebRTC.RTPTransceiverDirectionRecvonly,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tracks := make(chan *pionWebRTC.TrackRemote, 1)
|
||||
viewer.OnTrack(func(track *pionWebRTC.TrackRemote, _ *pionWebRTC.RTPReceiver) {
|
||||
tracks <- track
|
||||
})
|
||||
offer, err := viewer.CreateOffer(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gathered := pionWebRTC.GatheringCompletePromise(viewer)
|
||||
if err := viewer.SetLocalDescription(offer); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-gathered:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("viewer ICE gathering timed out")
|
||||
}
|
||||
|
||||
requests := make(chan time.Time, 10)
|
||||
b := NewVideoBroadcaster([]packets.Stream{{Name: "H264", IsVideo: true}}, keyframeTestSource{
|
||||
request: func() error { requests <- time.Now(); return nil },
|
||||
})
|
||||
t.Cleanup(b.Close)
|
||||
const sessionKey = "unit-camera/startup-recovery"
|
||||
t.Cleanup(func() {
|
||||
globalConnectionManager.mu.RLock()
|
||||
wrapper := globalConnectionManager.peerConnections[sessionKey]
|
||||
globalConnectionManager.mu.RUnlock()
|
||||
if wrapper != nil {
|
||||
cleanupPeerConnection(sessionKey, wrapper)
|
||||
}
|
||||
})
|
||||
var signalingMu sync.Mutex
|
||||
var pending []pionWebRTC.ICECandidateInit
|
||||
signaling := &models.LiveHDSignalingCallbacks{
|
||||
SendCandidate: func(_ string, data string) error {
|
||||
var candidate pionWebRTC.ICECandidateInit
|
||||
if err := json.Unmarshal([]byte(data), &candidate); err != nil {
|
||||
return err
|
||||
}
|
||||
signalingMu.Lock()
|
||||
defer signalingMu.Unlock()
|
||||
if viewer.RemoteDescription() == nil {
|
||||
pending = append(pending, candidate)
|
||||
return nil
|
||||
}
|
||||
return viewer.AddICECandidate(candidate)
|
||||
},
|
||||
SendAnswer: func(_ string, data string) error {
|
||||
answer, err := base64.StdEncoding.DecodeString(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
signalingMu.Lock()
|
||||
defer signalingMu.Unlock()
|
||||
if err := viewer.SetRemoteDescription(pionWebRTC.SessionDescription{
|
||||
Type: pionWebRTC.SDPTypeAnswer, SDP: string(answer),
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, candidate := range pending {
|
||||
if err := viewer.AddICECandidate(candidate); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
pending = nil
|
||||
return nil
|
||||
},
|
||||
}
|
||||
InitializeWebRTCConnection(
|
||||
&models.Configuration{Config: models.Config{Key: "unit-camera"}},
|
||||
&models.Communication{}, nil, b, nil,
|
||||
models.LiveHDHandshake{
|
||||
Signaling: signaling,
|
||||
Payload: models.RequestHDStreamPayload{
|
||||
SessionID: "startup-recovery",
|
||||
SessionDescription: base64.StdEncoding.EncodeToString([]byte(viewer.LocalDescription().SDP)),
|
||||
},
|
||||
},
|
||||
)
|
||||
var firstRequest time.Time
|
||||
select {
|
||||
case firstRequest = <-requests:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("connected transport did not request a startup keyframe")
|
||||
}
|
||||
b.WriteSample(pionMedia.Sample{Data: []byte{0, 0, 0, 1, 0x41, 0x80}, Duration: time.Second / 30}, false)
|
||||
select {
|
||||
case <-tracks:
|
||||
t.Fatal("viewer received dependent video before its startup IDR")
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
b.WriteSample(pionMedia.Sample{Data: []byte{0, 0, 0, 1, 0x65, 0x88, 0x84}, Duration: time.Second / 30}, true)
|
||||
var track *pionWebRTC.TrackRemote
|
||||
select {
|
||||
case track = <-tracks:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("viewer did not receive the requested startup IDR")
|
||||
}
|
||||
|
||||
for _, feedback := range []rtcp.Packet{
|
||||
&rtcp.PictureLossIndication{SenderSSRC: 42, MediaSSRC: uint32(track.SSRC())},
|
||||
&rtcp.FullIntraRequest{SenderSSRC: 42, FIR: []rtcp.FIREntry{{SSRC: uint32(track.SSRC()), SequenceNumber: 1}}},
|
||||
} {
|
||||
if err := viewer.WriteRTCP([]rtcp.Packet{feedback}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case at := <-requests:
|
||||
if at.Sub(firstRequest) < keyframeRequestInterval-5*time.Millisecond {
|
||||
t.Fatal("real viewer feedback bypassed keyframe request throttling")
|
||||
}
|
||||
firstRequest = at
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatalf("real %T feedback did not request a recovery keyframe", feedback)
|
||||
}
|
||||
b.WriteSample(pionMedia.Sample{Data: []byte{0, 0, 0, 1, 0x65, 0x88, 0x84}, Duration: time.Second / 30}, true)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
package webrtc
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/pion/interceptor"
|
||||
"github.com/pion/rtcp"
|
||||
"github.com/pion/rtp"
|
||||
pionWebRTC "github.com/pion/webrtc/v4"
|
||||
pionMedia "github.com/pion/webrtc/v4/pkg/media"
|
||||
)
|
||||
|
||||
type clockTestWriter struct {
|
||||
timestamps []uint32
|
||||
}
|
||||
|
||||
func (w *clockTestWriter) WriteRTP(header *rtp.Header, payload []byte) (int, error) {
|
||||
w.timestamps = append(w.timestamps, header.Timestamp)
|
||||
return len(payload), nil
|
||||
}
|
||||
|
||||
func (*clockTestWriter) Write([]byte) (int, error) {
|
||||
return 0, errors.New("unexpected raw RTP write")
|
||||
}
|
||||
|
||||
type clockTestContext struct {
|
||||
writer *clockTestWriter
|
||||
}
|
||||
|
||||
func (clockTestContext) ID() string { return "clock-test" }
|
||||
func (clockTestContext) SSRC() pionWebRTC.SSRC { return 123 }
|
||||
func (clockTestContext) SSRCRetransmission() pionWebRTC.SSRC { return 0 }
|
||||
func (clockTestContext) SSRCForwardErrorCorrection() pionWebRTC.SSRC { return 0 }
|
||||
func (clockTestContext) RTCPReader() interceptor.RTCPReader { return nil }
|
||||
func (clockTestContext) HeaderExtensions() []pionWebRTC.RTPHeaderExtensionParameter {
|
||||
return nil
|
||||
}
|
||||
func (c clockTestContext) WriteStream() pionWebRTC.TrackLocalWriter { return c.writer }
|
||||
func (clockTestContext) CodecParameters() []pionWebRTC.RTPCodecParameters {
|
||||
return []pionWebRTC.RTPCodecParameters{{
|
||||
RTPCodecCapability: pionWebRTC.RTPCodecCapability{MimeType: pionWebRTC.MimeTypeH264, ClockRate: 90000},
|
||||
PayloadType: 96,
|
||||
}}
|
||||
}
|
||||
|
||||
func clockTestPeer(t *testing.T) (*TrackBroadcaster, *peerTrack, *clockTestWriter) {
|
||||
t.Helper()
|
||||
b, pt := testVideoBroadcaster(t)
|
||||
pt.samples = make(chan peerSample, 2)
|
||||
track, err := pionWebRTC.NewTrackLocalStaticSample(
|
||||
pionWebRTC.RTPCodecCapability{MimeType: pionWebRTC.MimeTypeH264},
|
||||
"video", "clock-test", pionWebRTC.WithRTPTimestamp(1000),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writer := &clockTestWriter{}
|
||||
if _, err := track.Bind(clockTestContext{writer: writer}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pt.track = track
|
||||
b.SetPeerConnected("viewer", true)
|
||||
return b, pt, writer
|
||||
}
|
||||
|
||||
func clockVideoSample(duration time.Duration) pionMedia.Sample {
|
||||
return pionMedia.Sample{
|
||||
Data: []byte{0, 0, 0, 1, 0x65, 0x88, 0x84},
|
||||
Duration: duration,
|
||||
}
|
||||
}
|
||||
|
||||
func writeClockSample(t *testing.T, pt *peerTrack) {
|
||||
t.Helper()
|
||||
select {
|
||||
case sample := <-pt.samples:
|
||||
if err := writePeerSample(pt.track, sample); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
default:
|
||||
t.Fatal("expected queued video sample")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVideoRecoveryPreservesRTPClock(t *testing.T) {
|
||||
for _, scenario := range []string{"PLI", "FIR", "reconnect", "overflow"} {
|
||||
t.Run(scenario, func(t *testing.T) {
|
||||
b, pt, writer := clockTestPeer(t)
|
||||
b.WriteSample(clockVideoSample(40*time.Millisecond), true)
|
||||
writeClockSample(t, pt)
|
||||
b.WriteSample(clockVideoSample(70*time.Millisecond), false)
|
||||
|
||||
switch scenario {
|
||||
case "PLI", "FIR":
|
||||
var feedback rtcp.Packet = &rtcp.PictureLossIndication{MediaSSRC: 123}
|
||||
if scenario == "FIR" {
|
||||
feedback = &rtcp.FullIntraRequest{FIR: []rtcp.FIREntry{{SSRC: 123, SequenceNumber: 1}}}
|
||||
}
|
||||
data, err := rtcp.Marshal([]rtcp.Packet{feedback})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := b.HandleRTCP("viewer", data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.WriteSample(clockVideoSample(30*time.Millisecond), false)
|
||||
case "reconnect":
|
||||
b.SetPeerConnected("viewer", false)
|
||||
b.WriteSample(clockVideoSample(30*time.Millisecond), false)
|
||||
b.SetPeerConnected("viewer", true)
|
||||
case "overflow":
|
||||
b.WriteSample(clockVideoSample(30*time.Millisecond), false)
|
||||
}
|
||||
b.WriteSample(clockVideoSample(1900*time.Millisecond), false)
|
||||
b.WriteSample(clockVideoSample(40*time.Millisecond), true)
|
||||
writeClockSample(t, pt)
|
||||
|
||||
if len(writer.timestamps) != 2 {
|
||||
t.Fatalf("sent %d RTP packets, want only the two IDRs", len(writer.timestamps))
|
||||
}
|
||||
// The initial 40 ms frame plus 2000 ms of discarded video at 90 kHz.
|
||||
if got := writer.timestamps[1] - writer.timestamps[0]; got != 183600 {
|
||||
t.Fatalf("recovery RTP timestamp delta = %d, want 183600", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOverflowIDRPreservesDiscardedRTPDuration(t *testing.T) {
|
||||
b, pt, writer := clockTestPeer(t)
|
||||
b.WriteSample(clockVideoSample(40*time.Millisecond), true)
|
||||
writeClockSample(t, pt)
|
||||
b.WriteSample(clockVideoSample(70*time.Millisecond), false)
|
||||
b.WriteSample(clockVideoSample(30*time.Millisecond), false)
|
||||
b.WriteSample(clockVideoSample(40*time.Millisecond), true)
|
||||
writeClockSample(t, pt)
|
||||
if len(writer.timestamps) != 2 || writer.timestamps[1]-writer.timestamps[0] != 12600 {
|
||||
t.Fatalf("overflow IDR compressed the RTP clock: %v", writer.timestamps)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepeatedFeedbackPreservesAlreadyQueuedSkippedTime(t *testing.T) {
|
||||
b, pt, writer := clockTestPeer(t)
|
||||
b.WriteSample(clockVideoSample(40*time.Millisecond), true)
|
||||
writeClockSample(t, pt)
|
||||
data, err := rtcp.Marshal([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: 123}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for range 2 {
|
||||
if err := b.HandleRTCP("viewer", data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.WriteSample(clockVideoSample(time.Second), false)
|
||||
b.WriteSample(clockVideoSample(40*time.Millisecond), true)
|
||||
}
|
||||
writeClockSample(t, pt)
|
||||
if len(writer.timestamps) != 2 || writer.timestamps[1]-writer.timestamps[0] != 187200 {
|
||||
t.Fatalf("repeated feedback lost queued skipped time: %v", writer.timestamps)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user