From ef81ce5b3275d136a98129276545644cf2e49bcb Mon Sep 17 00:00:00 2001 From: cedricve Date: Wed, 18 Nov 2026 07:04:26 +0000 Subject: [PATCH 1/2] fix: request camera keyframes for WebRTC startup and recovery Gate each video viewer on a fresh IDR after transport connection or recovery, translate viewer PLI/FIR into rate-limited RTSP PLI requests, and preserve keyframe metadata through sample buffering. Add transport, lifecycle, concurrency, and source-routing regressions. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- machinery/README.md | 22 ++ machinery/go.mod | 2 +- machinery/src/capture/gortsplib.go | 10 + machinery/src/capture/keyframe.go | 35 +++ machinery/src/capture/keyframe_test.go | 164 +++++++++++ machinery/src/capture/rtsp_client.go | 4 + machinery/src/cloud/cloud.go | 4 +- machinery/src/webrtc/broadcaster.go | 170 ++++++++++- machinery/src/webrtc/broadcaster_test.go | 360 +++++++++++++++++++++++ machinery/src/webrtc/main.go | 70 +++-- machinery/src/webrtc/main_test.go | 150 +++++++++- 11 files changed, 942 insertions(+), 49 deletions(-) create mode 100644 machinery/src/capture/keyframe.go create mode 100644 machinery/src/capture/keyframe_test.go create mode 100644 machinery/src/webrtc/broadcaster_test.go diff --git a/machinery/README.md b/machinery/README.md index 1f3a31f..64c41c4 100644 --- a/machinery/README.md +++ b/machinery/README.md @@ -26,3 +26,25 @@ Kerberos Open source (v3) is a cutting edge video surveillance management system ## Security reporting For sensitive vulnerabilities, use private disclosure channels documented in [../SECURITY.md](../SECURITY.md). + +## WebRTC startup and keyframe recovery + +Each H264 viewer waits until its WebRTC transport is connected and a fresh IDR +arrives before receiving video. Reconnects, viewer PLI/FIR feedback, and a full +per-viewer sample buffer reset that viewer's keyframe gate without stalling +other viewers. Audio continues to use the existing independent sample queues. + +The main and sub streams request keyframes from their own RTSP sources. Viewer +PLI and FIR feedback is translated into an RTSP PLI using the camera session's +SSRCs, never forwarded with the browser's SSRCs. Requests are coalesced and +limited to one per second per broadcaster, retrying while connected viewers +still need an IDR. Requests stop once all waiting viewers receive an IDR or +the broadcaster closes. + +Camera support for RTCP PLI varies. Unsupported sources, cameras that ignore +feedback, and failed requests fall back to the next periodic IDR; failures are +logged as `keyframe_request_failed`. No encoder settings are changed. A camera +with a long GOP can therefore still start slowly; configure a shorter IDR +interval (for example 1-2 seconds) when camera feedback is unavailable, accounting +for the bandwidth tradeoff. An isolated cached IDR must not be replayed followed +by arbitrary live P-frames: the dependent reference chain would be incomplete. diff --git a/machinery/go.mod b/machinery/go.mod index dffbea8..32d8b04 100644 --- a/machinery/go.mod +++ b/machinery/go.mod @@ -30,6 +30,7 @@ require ( github.com/moq-dev/moq-go v0.5.7 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 github.com/pion/interceptor v0.1.47 + github.com/pion/rtcp v1.2.17 github.com/pion/rtp v1.10.5 github.com/pion/webrtc/v4 v4.2.18 github.com/sirupsen/logrus v1.9.3 @@ -104,7 +105,6 @@ require ( github.com/pion/logging v0.2.4 // indirect github.com/pion/mdns/v2 v2.1.0 // indirect github.com/pion/randutil v0.1.0 // indirect - github.com/pion/rtcp v1.2.17 // indirect github.com/pion/sctp v1.11.1 // indirect github.com/pion/sdp/v3 v3.0.19 // indirect github.com/pion/srtp/v3 v3.0.12 // indirect diff --git a/machinery/src/capture/gortsplib.go b/machinery/src/capture/gortsplib.go index a19a534..4ecc4a3 100644 --- a/machinery/src/capture/gortsplib.go +++ b/machinery/src/capture/gortsplib.go @@ -101,6 +101,8 @@ type Golibrtsp struct { Client gortsplib.Client clientStarted bool VideoDecoderMutex *sync.Mutex + keyframeMu sync.Mutex + keyframeReady bool VideoH264Index int8 VideoH264Media *description.Media @@ -813,6 +815,10 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e } } + g.keyframeMu.Lock() + g.keyframeReady = err == nil && g.VideoH264Media != nil && g.VideoH264Decoder != nil + g.keyframeMu.Unlock() + return } @@ -1713,6 +1719,10 @@ func (g *Golibrtsp) Close(ctxOtel context.Context) error { _, span := tracer.Start(ctxOtel, "Close") defer span.End() + g.keyframeMu.Lock() + g.keyframeReady = false + g.keyframeMu.Unlock() + if !g.clientStarted { return nil } diff --git a/machinery/src/capture/keyframe.go b/machinery/src/capture/keyframe.go new file mode 100644 index 0000000..8de9ce3 --- /dev/null +++ b/machinery/src/capture/keyframe.go @@ -0,0 +1,35 @@ +package capture + +import ( + "errors" + "fmt" + + "github.com/pion/rtcp" +) + +func (g *Golibrtsp) RequestKeyframe() error { + g.keyframeMu.Lock() + defer g.keyframeMu.Unlock() + + if !g.keyframeReady { + return errors.New("RTSP H264 keyframe requests are unavailable") + } + + media, ok := g.Client.Stats().Session.Medias[g.VideoH264Media] + if !ok { + return errors.New("RTSP video media is not set up") + } + video, ok := media.Formats[g.VideoH264Forma] + if !ok || video.InboundRTPPackets == 0 { + return errors.New("RTSP video RTP has not arrived yet") + } + + // Browser and camera SSRCs differ: originate feedback on the RTSP session. + if err := g.Client.WritePacketRTCP(g.VideoH264Media, &rtcp.PictureLossIndication{ + SenderSSRC: video.LocalSSRC, + MediaSSRC: video.RemoteSSRC, + }); err != nil { + return fmt.Errorf("request camera keyframe: %w", err) + } + return nil +} diff --git a/machinery/src/capture/keyframe_test.go b/machinery/src/capture/keyframe_test.go new file mode 100644 index 0000000..3e91923 --- /dev/null +++ b/machinery/src/capture/keyframe_test.go @@ -0,0 +1,164 @@ +package capture + +import ( + "context" + "net" + "sync" + "testing" + "time" + + "github.com/bluenviron/gortsplib/v5" + "github.com/bluenviron/gortsplib/v5/pkg/base" + "github.com/bluenviron/gortsplib/v5/pkg/description" + "github.com/bluenviron/gortsplib/v5/pkg/format" + "github.com/pion/rtcp" + "github.com/pion/rtp" +) + +type keyframeCamera struct { + stream *gortsplib.ServerStream + feedback chan rtcp.Packet +} + +func (c *keyframeCamera) OnDescribe(*gortsplib.ServerHandlerOnDescribeCtx) (*base.Response, *gortsplib.ServerStream, error) { + return &base.Response{StatusCode: base.StatusOK}, c.stream, nil +} + +func (c *keyframeCamera) OnSetup(*gortsplib.ServerHandlerOnSetupCtx) (*base.Response, *gortsplib.ServerStream, error) { + return &base.Response{StatusCode: base.StatusOK}, c.stream, nil +} + +func (c *keyframeCamera) OnPlay(ctx *gortsplib.ServerHandlerOnPlayCtx) (*base.Response, error) { + ctx.Session.OnPacketRTCPAny(func(_ *description.Media, pkt rtcp.Packet) { + c.feedback <- pkt + }) + return &base.Response{StatusCode: base.StatusOK}, nil +} + +func connectedKeyframeCamera(t *testing.T) (*Golibrtsp, *keyframeCamera) { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { listener.Close() }) + camera := &keyframeCamera{feedback: make(chan rtcp.Packet, 32)} + server := &gortsplib.Server{ + RTSPAddress: "127.0.0.1:0", + Handler: camera, + Listen: func(string, string) (net.Listener, error) { + return listener, nil + }, + } + if err := server.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(server.Close) + media := &description.Media{ + Type: description.MediaTypeVideo, + Formats: []format.Format{&format.H264{ + PayloadTyp: 96, + PacketizationMode: 1, + }}, + } + camera.stream = &gortsplib.ServerStream{ + Server: server, + Desc: &description.Session{Medias: []*description.Media{media}}, + } + if err := camera.stream.Initialize(); err != nil { + t.Fatal(err) + } + t.Cleanup(camera.stream.Close) + + client := &Golibrtsp{Url: "rtsp://" + listener.Addr().String() + "/camera"} + t.Cleanup(func() { + if err := client.Close(context.Background()); err != nil { + t.Error(err) + } + }) + if err := client.Connect(context.Background(), context.Background()); err != nil { + t.Fatal(err) + } + if err := client.RequestKeyframe(); err == nil { + t.Fatal("requested a keyframe before the camera SSRC was known") + } + received := make(chan struct{}, 1) + client.Client.OnPacketRTP(client.VideoH264Media, client.VideoH264Forma, func(*rtp.Packet) { + received <- struct{}{} + }) + if _, err := client.Client.Play(nil); err != nil { + t.Fatal(err) + } + if err := camera.stream.WritePacketRTP(media, &rtp.Packet{ + Header: rtp.Header{ + Version: 2, + PayloadType: 96, + SSRC: 0x12345678, + SequenceNumber: 1, + Timestamp: 90000, + Marker: true, + }, + Payload: []byte{0x65, 0x88, 0x84}, + }); err != nil { + t.Fatal(err) + } + select { + case <-received: + case <-time.After(3 * time.Second): + t.Fatal("test camera RTP did not arrive") + } + return client, camera +} + +func TestRTSPKeyframeRequestUsesCameraSessionSSRCs(t *testing.T) { + client, camera := connectedKeyframeCamera(t) + want := client.Client.Stats().Session.Medias[client.VideoH264Media].Formats[client.VideoH264Forma] + if err := client.RequestKeyframe(); err != nil { + t.Fatal(err) + } + select { + case pkt := <-camera.feedback: + pli, ok := pkt.(*rtcp.PictureLossIndication) + if !ok { + t.Fatalf("camera received %T, want PLI", pkt) + } + if pli.MediaSSRC != want.RemoteSSRC || pli.SenderSSRC != want.LocalSSRC { + t.Fatalf("camera PLI SSRCs = sender %x media %x, want sender %x media %x", + pli.SenderSSRC, pli.MediaSSRC, want.LocalSSRC, want.RemoteSSRC) + } + case <-time.After(3 * time.Second): + t.Fatal("camera did not receive RTCP PLI") + } +} + +func TestRTSPKeyframeRequestUnavailableBeforeConnectAndAfterClose(t *testing.T) { + if err := (&Golibrtsp{}).RequestKeyframe(); err == nil { + t.Fatal("uninitialized client accepted a keyframe request") + } + client, _ := connectedKeyframeCamera(t) + if err := client.Close(context.Background()); err != nil { + t.Fatal(err) + } + if err := client.RequestKeyframe(); err == nil { + t.Fatal("closed client accepted a keyframe request") + } +} + +func TestRTSPKeyframeRequestConcurrentWithClose(t *testing.T) { + client, _ := connectedKeyframeCamera(t) + var wg sync.WaitGroup + for range 4 { + wg.Add(1) + go func() { + defer wg.Done() + for range 5 { + // Requests may race with Close and legitimately become unavailable. + _ = client.RequestKeyframe() + } + }() + } + if err := client.Close(context.Background()); err != nil { + t.Fatal(err) + } + wg.Wait() +} diff --git a/machinery/src/capture/rtsp_client.go b/machinery/src/capture/rtsp_client.go index 8b5896f..1a13a5e 100644 --- a/machinery/src/capture/rtsp_client.go +++ b/machinery/src/capture/rtsp_client.go @@ -72,6 +72,10 @@ func (c *Capture) ClearClients(main, sub, backchannel *Golibrtsp) { } } +type KeyframeRequester interface { + RequestKeyframe() error +} + // RTSPClient is a interface that abstracts the RTSP client implementation. type RTSPClient interface { // Connect to the RTSP server. diff --git a/machinery/src/cloud/cloud.go b/machinery/src/cloud/cloud.go index 25943d4..c3d00e8 100644 --- a/machinery/src/cloud/cloud.go +++ b/machinery/src/cloud/cloud.go @@ -1285,7 +1285,7 @@ func HandleLiveStreamHD( // viewer can pick the resolution it needs per peer connection without // the agent re-negotiating the RTSP source. mainStreams, _ := rtspClient.GetStreams() - mainVideoBroadcaster := webrtc.NewVideoBroadcaster(mainStreams) + mainVideoBroadcaster := webrtc.NewVideoBroadcaster(mainStreams, rtspClient) mainAudioBroadcaster := webrtc.NewAudioBroadcaster(mainStreams) if mainVideoBroadcaster == nil && mainAudioBroadcaster == nil { @@ -1316,7 +1316,7 @@ func HandleLiveStreamHD( var subAudioBroadcaster *webrtc.TrackBroadcaster if subStreamEnabled && rtspSubClient != nil && subQueue != nil { subStreams, _ := rtspSubClient.GetStreams() - subVideoBroadcaster = webrtc.NewVideoBroadcaster(subStreams) + subVideoBroadcaster = webrtc.NewVideoBroadcaster(subStreams, rtspSubClient) subAudioBroadcaster = webrtc.NewAudioBroadcaster(subStreams) defer func() { if subVideoBroadcaster != nil { diff --git a/machinery/src/webrtc/broadcaster.go b/machinery/src/webrtc/broadcaster.go index a29e452..8ca672d 100644 --- a/machinery/src/webrtc/broadcaster.go +++ b/machinery/src/webrtc/broadcaster.go @@ -3,7 +3,9 @@ package webrtc import ( "io" "sync" + "time" + "github.com/pion/rtcp" pionWebRTC "github.com/pion/webrtc/v4" pionMedia "github.com/pion/webrtc/v4/pkg/media" log "github.com/sirupsen/logrus" @@ -12,14 +14,17 @@ import ( const ( // peerSampleBuffer controls how many samples can be buffered per peer before // dropping. Keeps slow peers from blocking the broadcaster. - peerSampleBuffer = 60 + peerSampleBuffer = 60 + keyframeRequestInterval = time.Second ) // peerTrack is a per-peer track with its own non-blocking sample channel. type peerTrack struct { - track *pionWebRTC.TrackLocalStaticSample - samples chan pionMedia.Sample - done chan struct{} + track *pionWebRTC.TrackLocalStaticSample + samples chan pionMedia.Sample + done chan struct{} + connected bool + needsKeyframe bool } // TrackBroadcaster fans out media samples to multiple peer-specific tracks @@ -27,11 +32,17 @@ type peerTrack struct { // goroutine that drains samples independently, so a slow/congested peer // cannot stall the others. type TrackBroadcaster struct { - mu sync.RWMutex - peers map[string]*peerTrack - mimeType string - id string - streamID string + mu sync.RWMutex + peers map[string]*peerTrack + mimeType string + id string + streamID string + requestKeyframe func() error + requestTimer *time.Timer + requestInFlight bool + lastRequest time.Time + requests sync.WaitGroup + closed bool } // NewTrackBroadcaster creates a new broadcaster for either video or audio. @@ -57,12 +68,18 @@ func (b *TrackBroadcaster) AddPeer(sessionKey string) (*pionWebRTC.TrackLocalSta } pt := &peerTrack{ - track: track, - samples: make(chan pionMedia.Sample, peerSampleBuffer), - done: make(chan struct{}), + track: track, + samples: make(chan pionMedia.Sample, peerSampleBuffer), + done: make(chan struct{}), + connected: b.mimeType != pionWebRTC.MimeTypeH264, + needsKeyframe: b.mimeType == pionWebRTC.MimeTypeH264, } b.mu.Lock() + if b.closed { + b.mu.Unlock() + return nil, io.ErrClosedPipe + } b.peers[sessionKey] = pt b.mu.Unlock() @@ -114,14 +131,32 @@ func (b *TrackBroadcaster) RemovePeer(sessionKey string) { // WriteSample fans out a sample to all connected peers without blocking. // If a peer's buffer is full (slow consumer), the sample is dropped for // that peer only — other peers are unaffected. -func (b *TrackBroadcaster) WriteSample(sample pionMedia.Sample) { - b.mu.RLock() - defer b.mu.RUnlock() +func (b *TrackBroadcaster) WriteSample(sample pionMedia.Sample, isKeyframe bool) { + b.mu.Lock() + defer b.mu.Unlock() + if b.closed { + return + } for sessionKey, pt := range b.peers { + video := b.mimeType == pionWebRTC.MimeTypeH264 + if video && (!pt.connected || (pt.needsKeyframe && !isKeyframe)) { + continue + } select { case pt.samples <- sample: + if video && isKeyframe { + pt.needsKeyframe = false + } default: + if video { + pt.needsKeyframe = true + drainPeerSamples(pt) + if isKeyframe { + pt.samples <- sample + pt.needsKeyframe = false + } + } log.WithFields(log.Fields{ "component": "webrtc", "event": "slow_peer_sample_dropped", @@ -129,6 +164,105 @@ func (b *TrackBroadcaster) WriteSample(sample pionMedia.Sample) { }).Warn("Dropping WebRTC sample for slow peer") } } + b.scheduleKeyframeRequestLocked() +} + +func drainPeerSamples(pt *peerTrack) { + for { + select { + case <-pt.samples: + default: + return + } + } +} + +func (b *TrackBroadcaster) SetPeerConnected(sessionKey string, connected bool) { + b.mu.Lock() + defer b.mu.Unlock() + + pt, ok := b.peers[sessionKey] + if !ok || b.mimeType != pionWebRTC.MimeTypeH264 || pt.connected == connected { + return + } + pt.connected = connected + pt.needsKeyframe = true + drainPeerSamples(pt) + b.scheduleKeyframeRequestLocked() +} + +func (b *TrackBroadcaster) HandleRTCP(sessionKey string, data []byte) error { + feedback, err := rtcp.Unmarshal(data) + if err != nil { + return err + } + for _, packet := range feedback { + switch packet.(type) { + case *rtcp.PictureLossIndication, *rtcp.FullIntraRequest: + b.mu.Lock() + if pt, ok := b.peers[sessionKey]; ok && pt.connected && b.mimeType == pionWebRTC.MimeTypeH264 { + pt.needsKeyframe = true + drainPeerSamples(pt) + b.scheduleKeyframeRequestLocked() + } + b.mu.Unlock() + return nil + } + } + return nil +} + +func (b *TrackBroadcaster) waitingForKeyframeLocked() bool { + for _, pt := range b.peers { + if pt.connected && pt.needsKeyframe { + return true + } + } + return false +} + +func (b *TrackBroadcaster) scheduleKeyframeRequestLocked() { + if b.closed || b.requestKeyframe == nil || b.requestInFlight || !b.waitingForKeyframeLocked() { + return + } + if b.requestTimer == nil { + delay := time.Until(b.lastRequest.Add(keyframeRequestInterval)) + if delay < 0 { + delay = 0 + } + b.requestTimer = time.AfterFunc(delay, b.sendKeyframeRequest) + } +} + +func (b *TrackBroadcaster) sendKeyframeRequest() { + b.mu.Lock() + b.requestTimer = nil + if b.closed || !b.waitingForKeyframeLocked() { + b.mu.Unlock() + return + } + b.lastRequest = time.Now() + b.requestInFlight = true + b.requests.Add(1) + b.mu.Unlock() + defer b.requests.Done() + + if err := b.requestKeyframe(); err != nil { + log.WithError(err).WithFields(log.Fields{ + "component": "webrtc", + "event": "keyframe_request_failed", + }).Warn("Camera keyframe request failed; waiting for a periodic IDR") + } else { + log.WithFields(log.Fields{ + "component": "webrtc", + "event": "keyframe_requested", + }).Debug("Requested camera keyframe") + } + + b.mu.Lock() + b.requestInFlight = false + b.scheduleKeyframeRequestLocked() + b.mu.Unlock() } // PeerCount returns the current number of connected peers. @@ -141,11 +275,17 @@ func (b *TrackBroadcaster) PeerCount() int { // Close removes all peers and stops all writer goroutines. func (b *TrackBroadcaster) Close() { b.mu.Lock() + b.closed = true + if b.requestTimer != nil { + b.requestTimer.Stop() + b.requestTimer = nil + } keys := make([]string, 0, len(b.peers)) for k := range b.peers { keys = append(keys, k) } b.mu.Unlock() + b.requests.Wait() for _, key := range keys { b.RemovePeer(key) diff --git a/machinery/src/webrtc/broadcaster_test.go b/machinery/src/webrtc/broadcaster_test.go new file mode 100644 index 0000000..669eefb --- /dev/null +++ b/machinery/src/webrtc/broadcaster_test.go @@ -0,0 +1,360 @@ +package webrtc + +import ( + "bytes" + "errors" + "fmt" + "io" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/kerberos-io/agent/machinery/src/capture" + "github.com/kerberos-io/agent/machinery/src/models" + "github.com/kerberos-io/agent/machinery/src/packets" + "github.com/pion/rtcp" + pionWebRTC "github.com/pion/webrtc/v4" + pionMedia "github.com/pion/webrtc/v4/pkg/media" +) + +func bufferedTestPeer(capacity int) *peerTrack { + done := make(chan struct{}) + close(done) + return &peerTrack{ + samples: make(chan pionMedia.Sample, capacity), + done: done, + needsKeyframe: true, + } +} + +func testVideoBroadcaster(t *testing.T) (*TrackBroadcaster, *peerTrack) { + t.Helper() + b := NewTrackBroadcaster(pionWebRTC.MimeTypeH264, "video", "test") + pt := bufferedTestPeer(peerSampleBuffer) + b.peers["viewer"] = pt + t.Cleanup(b.Close) + return b, pt +} + +func TestVideoPeerStartsAndReconnectsAtKeyframe(t *testing.T) { + b, pt := testVideoBroadcaster(t) + sample := pionMedia.Sample{Data: []byte{1}} + b.WriteSample(sample, true) + if len(pt.samples) != 0 { + t.Fatal("sent a keyframe before the transport connected") + } + + b.SetPeerConnected("viewer", true) + b.WriteSample(sample, false) + if len(pt.samples) != 0 { + t.Fatal("sent a dependent frame before the first IDR") + } + b.WriteSample(sample, true) + b.WriteSample(sample, false) + if len(pt.samples) != 2 || pt.needsKeyframe { + t.Fatal("did not release video after the first IDR") + } + + b.SetPeerConnected("viewer", false) + b.WriteSample(sample, true) + if len(pt.samples) != 0 { + t.Fatal("retained or sent video while disconnected") + } + b.SetPeerConnected("viewer", true) + b.WriteSample(sample, false) + if len(pt.samples) != 0 { + t.Fatal("reconnected viewer resumed on a dependent frame") + } + b.WriteSample(sample, true) + if len(pt.samples) != 1 { + t.Fatal("reconnected viewer did not resume at an IDR") + } +} + +func TestLateViewerDoesNotResetExistingViewer(t *testing.T) { + b, first := testVideoBroadcaster(t) + b.SetPeerConnected("viewer", true) + b.WriteSample(pionMedia.Sample{}, true) + <-first.samples + + late := bufferedTestPeer(peerSampleBuffer) + b.peers["late"] = late + b.SetPeerConnected("late", true) + b.WriteSample(pionMedia.Sample{}, false) + if len(first.samples) != 1 || len(late.samples) != 0 { + t.Fatal("late viewer either interrupted an existing viewer or received a dependent frame") + } + b.WriteSample(pionMedia.Sample{}, true) + if len(late.samples) != 1 { + t.Fatal("late viewer did not receive the next IDR") + } +} + +func TestVideoFeedbackRequestsFreshKeyframe(t *testing.T) { + for _, feedback := range []rtcp.Packet{ + &rtcp.PictureLossIndication{MediaSSRC: 123}, + &rtcp.FullIntraRequest{FIR: []rtcp.FIREntry{{SSRC: 123, SequenceNumber: 1}}}, + } { + t.Run(fmt.Sprintf("%T", feedback), func(t *testing.T) { + b, pt := testVideoBroadcaster(t) + b.SetPeerConnected("viewer", true) + b.WriteSample(pionMedia.Sample{}, true) + b.WriteSample(pionMedia.Sample{}, false) + + rr, err := rtcp.Marshal([]rtcp.Packet{&rtcp.ReceiverReport{}}) + if err != nil { + t.Fatal(err) + } + if err := b.HandleRTCP("viewer", rr); err != nil || len(pt.samples) != 2 { + t.Fatal("receiver report reset the video gate", err) + } + data, err := rtcp.Marshal([]rtcp.Packet{&rtcp.ReceiverReport{}, feedback}) + if err != nil { + t.Fatal(err) + } + if err := b.HandleRTCP("viewer", data); err != nil { + t.Fatal(err) + } + b.WriteSample(pionMedia.Sample{}, false) + if len(pt.samples) != 0 || !pt.needsKeyframe { + t.Fatal("PLI/FIR did not discard stale queued frames and wait for an IDR") + } + b.WriteSample(pionMedia.Sample{}, true) + if len(pt.samples) != 1 { + t.Fatal("PLI/FIR recovery did not resume at an IDR") + } + if err := b.HandleRTCP("viewer", []byte{0}); err == nil { + t.Fatal("malformed RTCP was accepted") + } + }) + } +} + +func TestSlowVideoPeerRecoversWithoutBlockingOthers(t *testing.T) { + for _, overflowIsKeyframe := range []bool{false, true} { + t.Run(map[bool]string{false: "dependent-frame-dropped", true: "new-IDR-replaces-backlog"}[overflowIsKeyframe], func(t *testing.T) { + b, fast := testVideoBroadcaster(t) + slow := bufferedTestPeer(2) + b.peers["slow"] = slow + b.SetPeerConnected("viewer", true) + b.SetPeerConnected("slow", true) + b.WriteSample(pionMedia.Sample{}, true) + b.WriteSample(pionMedia.Sample{}, false) + b.WriteSample(pionMedia.Sample{}, overflowIsKeyframe) + if len(fast.samples) != 3 { + t.Fatal("slow viewer interrupted another viewer") + } + if overflowIsKeyframe { + if len(slow.samples) != 1 || slow.needsKeyframe { + t.Fatal("fresh IDR did not replace the stale backlog") + } + } else { + b.WriteSample(pionMedia.Sample{}, false) + if len(slow.samples) != 0 || !slow.needsKeyframe { + t.Fatal("slow viewer forwarded dependent frames after a drop") + } + b.WriteSample(pionMedia.Sample{}, true) + if len(slow.samples) != 1 { + t.Fatal("slow viewer did not recover at the next IDR") + } + } + }) + } +} + +func TestAudioDoesNotWaitForVideoKeyframe(t *testing.T) { + b := NewTrackBroadcaster(pionWebRTC.MimeTypePCMU, "audio", "test") + pt := bufferedTestPeer(2) + pt.connected = true + pt.needsKeyframe = false + b.peers["viewer"] = pt + t.Cleanup(b.Close) + + b.SetPeerConnected("viewer", false) + for range 3 { + b.WriteSample(pionMedia.Sample{}, false) + } + if len(pt.samples) != 2 { + t.Fatal("audio queue behavior changed") + } +} + +func TestVideoSampleBufferPreservesKeyframeFlag(t *testing.T) { + b, pt := testVideoBroadcaster(t) + b.SetPeerConnected("viewer", true) + state := &streamState{} + first := packets.Packet{Data: []byte{0x65}, IsKeyFrame: true, Time: 1} + processVideoPacket(first, state, b, models.Config{}) + processVideoPacket(packets.Packet{Data: []byte{0x41}, Time: 34}, state, b, models.Config{}) + if len(pt.samples) != 1 || !bytes.Equal((<-pt.samples).Data, first.Data) { + t.Fatal("buffered IDR lost its keyframe flag") + } + + b.SetPeerConnected("viewer", false) + b.SetPeerConnected("viewer", true) + writeFinalSamples(state, b, nil) + if len(pt.samples) != 0 { + t.Fatal("final dependent frame was mislabeled as an IDR") + } + state.lastVideoSample = &pionMedia.Sample{Data: first.Data} + state.lastVideoKeyframe = true + writeFinalSamples(state, b, nil) + if len(pt.samples) != 1 { + t.Fatal("final IDR lost its keyframe flag") + } +} + +func TestKeyframeRequestsCoalesceAndRetryUntilIDR(t *testing.T) { + b, _ := testVideoBroadcaster(t) + calls := make(chan time.Time, 10) + var count atomic.Int32 + b.requestKeyframe = func() error { + calls <- time.Now() + if count.Add(1) == 1 { + return errors.New("camera is not ready yet") + } + b.WriteSample(pionMedia.Sample{}, true) + return nil + } + for _, id := range []string{"viewer", "second", "third"} { + if id != "viewer" { + b.mu.Lock() + b.peers[id] = bufferedTestPeer(peerSampleBuffer) + b.mu.Unlock() + } + b.SetPeerConnected(id, true) + } + var first time.Time + select { + case first = <-calls: + case <-time.After(2 * time.Second): + t.Fatal("connected viewer did not request a keyframe") + } + feedback, err := rtcp.Marshal([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: 123}}) + if err != nil { + t.Fatal(err) + } + for range 100 { + if err := b.HandleRTCP("viewer", feedback); err != nil { + t.Fatal(err) + } + } + select { + case second := <-calls: + if second.Sub(first) < keyframeRequestInterval-5*time.Millisecond { + t.Fatal("feedback bypassed the per-stream request rate limit") + } + case <-time.After(2 * time.Second): + t.Fatal("failed camera request was not retried") + } + select { + case <-calls: + t.Fatal("requests continued after all viewers received an IDR") + case <-time.After(keyframeRequestInterval + 100*time.Millisecond): + } +} + +func TestKeyframeRequestsStopOnDisconnectAndClose(t *testing.T) { + b, _ := testVideoBroadcaster(t) + var calls atomic.Int32 + b.requestKeyframe = func() error { calls.Add(1); return nil } + b.lastRequest = time.Now() + b.SetPeerConnected("viewer", true) + b.SetPeerConnected("viewer", false) + time.Sleep(keyframeRequestInterval + 50*time.Millisecond) + if calls.Load() != 0 { + t.Fatal("disconnected viewer requested a keyframe") + } + b.SetPeerConnected("viewer", true) + b.Close() + if _, err := b.AddPeer("closed"); !errors.Is(err, io.ErrClosedPipe) { + t.Fatalf("AddPeer() after Close() = %v, want closed pipe", err) + } +} + +func TestBroadcasterCloseWaitsForKeyframeRequest(t *testing.T) { + b, _ := testVideoBroadcaster(t) + started, release, closed := make(chan struct{}), make(chan struct{}), make(chan struct{}) + unblock := sync.OnceFunc(func() { close(release) }) + defer unblock() + b.requestKeyframe = func() error { + close(started) + <-release + return nil + } + b.SetPeerConnected("viewer", true) + select { + case <-started: + case <-time.After(2 * time.Second): + t.Fatal("request did not start") + } + go func() { b.Close(); close(closed) }() + select { + case <-closed: + t.Fatal("Close returned before the in-flight camera request completed") + case <-time.After(30 * time.Millisecond): + } + unblock() + select { + case <-closed: + case <-time.After(2 * time.Second): + t.Fatal("Close did not finish") + } +} + +func TestConcurrentVideoFeedbackAndPeerLifecycle(t *testing.T) { + b := NewTrackBroadcaster(pionWebRTC.MimeTypeH264, "video", "test") + t.Cleanup(b.Close) + data, err := rtcp.Marshal([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: 123}}) + if err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + for _, id := range []string{"first", "second", "third"} { + if _, err := b.AddPeer(id); err != nil { + t.Fatal(err) + } + wg.Add(1) + go func() { + defer wg.Done() + for j := range 200 { + b.SetPeerConnected(id, j%5 != 0) + b.WriteSample(pionMedia.Sample{Data: []byte{0, 0, 0, 1, 0x65}, Duration: time.Millisecond}, j%10 == 0) + if err := b.HandleRTCP(id, data); err != nil { + t.Error(err) + } + } + b.RemovePeer(id) + }() + } + wg.Wait() +} + +type keyframeTestSource struct { + capture.RTSPClient + request func() error +} + +func (s keyframeTestSource) RequestKeyframe() error { return s.request() } + +func TestVideoBroadcastersRequestTheirOwnSource(t *testing.T) { + streams := []packets.Stream{{Name: "H264", IsVideo: true}} + var mainCalls, subCalls atomic.Int32 + main := NewVideoBroadcaster(streams, keyframeTestSource{request: func() error { mainCalls.Add(1); return nil }}) + sub := NewVideoBroadcaster(streams, keyframeTestSource{request: func() error { subCalls.Add(1); return nil }}) + t.Cleanup(main.Close) + t.Cleanup(sub.Close) + if err := main.requestKeyframe(); err != nil { + t.Fatal(err) + } + if mainCalls.Load() != 1 || subCalls.Load() != 0 { + t.Fatal("main feedback reached the wrong source") + } + if err := sub.requestKeyframe(); err != nil { + t.Fatal(err) + } + if mainCalls.Load() != 1 || subCalls.Load() != 1 { + t.Fatal("sub feedback reached the wrong source") + } +} diff --git a/machinery/src/webrtc/main.go b/machinery/src/webrtc/main.go index 93b7831..bbcdd0b 100644 --- a/machinery/src/webrtc/main.go +++ b/machinery/src/webrtc/main.go @@ -18,7 +18,6 @@ import ( mqtt "github.com/eclipse/paho.mqtt.golang" "github.com/pion/interceptor" - "github.com/pion/interceptor/pkg/intervalpli" pionWebRTC "github.com/pion/webrtc/v4" pionMedia "github.com/pion/webrtc/v4/pkg/media" ) @@ -429,16 +428,6 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati panic(err) } - // Register a intervalpli factory - // This interceptor sends a PLI every 3 seconds. A PLI causes a video keyframe to be generated by the sender. - // This makes our video seekable and more error resilent, but at a cost of lower picture quality and higher bitrates - // A real world application should process incoming RTCP packets from viewers and forward them to senders - intervalPliFactory, err := intervalpli.NewReceiverInterceptor() - if err != nil { - panic(err) - } - interceptorRegistry.Add(intervalPliFactory) - api := pionWebRTC.NewAPI( pionWebRTC.WithMediaEngine(mediaEngine), pionWebRTC.WithInterceptorRegistry(interceptorRegistry), @@ -519,9 +508,16 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati case <-ctx.Done(): return default: - if _, _, rtcpErr := videoSender.Read(rtcpBuf); rtcpErr != nil { + n, _, rtcpErr := videoSender.Read(rtcpBuf) + if rtcpErr != nil { + if ctx.Err() == nil && rtcpErr != io.EOF && rtcpErr != io.ErrClosedPipe { + log.WithError(rtcpErr).Warn("WebRTC video RTCP reader failed") + } return } + if err := videoBroadcaster.HandleRTCP(sessionKey, rtcpBuf[:n]); err != nil { + log.WithError(err).Warn("Ignoring malformed WebRTC video RTCP feedback") + } } } }() @@ -580,6 +576,9 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati switch connectionState { case pionWebRTC.PeerConnectionStateDisconnected: + if videoBroadcaster != nil { + videoBroadcaster.SetPeerConnected(sessionKey, false) + } // Disconnected is a transient state that can recover. // Start a grace period timer; if we don't recover, then cleanup. wrapper.disconnectMu.Lock() @@ -619,6 +618,9 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati cleanupPeerConnection(sessionKey, wrapper) case pionWebRTC.PeerConnectionStateConnected: + if videoBroadcaster != nil { + videoBroadcaster.SetPeerConnected(sessionKey, true) + } // Cancel any pending disconnect timer — connection recovered wrapper.disconnectMu.Lock() if wrapper.disconnectTimer != nil { @@ -760,11 +762,17 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati } } -func NewVideoBroadcaster(streams []packets.Stream) *TrackBroadcaster { +func NewVideoBroadcaster(streams []packets.Stream, rtspClient capture.RTSPClient) *TrackBroadcaster { // Verify H264 is available (same check as NewVideoTrack) for _, s := range streams { if s.Name == "H264" { - return NewTrackBroadcaster(pionWebRTC.MimeTypeH264, "video", trackStreamID) + b := NewTrackBroadcaster(pionWebRTC.MimeTypeH264, "video", trackStreamID) + if requester, ok := rtspClient.(capture.KeyframeRequester); ok { + b.requestKeyframe = requester.RequestKeyframe + } else { + log.Warn("RTSP source does not support keyframe requests; waiting for periodic IDRs") + } + return b } } log.Error("webrtc.main.NewVideoBroadcaster(): no H264 stream found") @@ -858,18 +866,19 @@ func NewAudioTrack(streams []packets.Stream) *pionWebRTC.TrackLocalStaticSample // streamState holds state information for the streaming process type streamState struct { - lastKeepAlive int64 - peerCount int64 - start bool - catchingUp bool - receivedKeyFrame bool - lastAudioSample *pionMedia.Sample - lastVideoSample *pionMedia.Sample - audioPacketsSeen int64 - aacPacketsSeen int64 - audioSamplesSent int64 - aacNoOutput int64 - aacErrors int64 + lastKeepAlive int64 + peerCount int64 + start bool + catchingUp bool + receivedKeyFrame bool + lastAudioSample *pionMedia.Sample + lastVideoSample *pionMedia.Sample + lastVideoKeyframe bool + audioPacketsSeen int64 + aacPacketsSeen int64 + audioSamplesSent int64 + aacNoOutput int64 + aacErrors int64 } // codecSupport tracks which codecs are available in the stream @@ -943,11 +952,11 @@ func updateStreamState(communication *models.Communication, state *streamState) // writeFinalSamples writes any remaining buffered samples func writeFinalSamples(state *streamState, videoBroadcaster, audioBroadcaster *TrackBroadcaster) { if state.lastVideoSample != nil && videoBroadcaster != nil { - videoBroadcaster.WriteSample(*state.lastVideoSample) + videoBroadcaster.WriteSample(*state.lastVideoSample, state.lastVideoKeyframe) } if state.lastAudioSample != nil && audioBroadcaster != nil { - audioBroadcaster.WriteSample(*state.lastAudioSample) + audioBroadcaster.WriteSample(*state.lastAudioSample, false) } } @@ -1011,10 +1020,11 @@ func processVideoPacket(pkt packets.Packet, state *streamState, videoBroadcaster if state.lastVideoSample != nil { state.lastVideoSample.Duration = sampleDuration(pkt, state.lastVideoSample.PacketTimestamp, 33*time.Millisecond) - videoBroadcaster.WriteSample(*state.lastVideoSample) + videoBroadcaster.WriteSample(*state.lastVideoSample, state.lastVideoKeyframe) } state.lastVideoSample = &sample + state.lastVideoKeyframe = pkt.IsKeyFrame } // processAudioPacket processes an audio packet and writes samples to the broadcaster. @@ -1097,7 +1107,7 @@ func processAudioPacket(pkt packets.Packet, state *streamState, audioBroadcaster "samples_queued": state.audioSamplesSent, }).Debug("WebRTC audio sample queued") } - audioBroadcaster.WriteSample(*state.lastAudioSample) + audioBroadcaster.WriteSample(*state.lastAudioSample, false) } state.lastAudioSample = &sample diff --git a/machinery/src/webrtc/main_test.go b/machinery/src/webrtc/main_test.go index b4f86a3..0472364 100644 --- a/machinery/src/webrtc/main_test.go +++ b/machinery/src/webrtc/main_test.go @@ -1,6 +1,18 @@ package webrtc -import "testing" +import ( + "encoding/base64" + "encoding/json" + "sync" + "testing" + "time" + + "github.com/kerberos-io/agent/machinery/src/models" + "github.com/kerberos-io/agent/machinery/src/packets" + "github.com/pion/rtcp" + pionWebRTC "github.com/pion/webrtc/v4" + pionMedia "github.com/pion/webrtc/v4/pkg/media" +) func TestBuildICEServersOmitsEmptyURLs(t *testing.T) { webRTC := CreateWebRTC("camera", []string{""}, []string{""}, "", "") @@ -26,6 +38,7 @@ func TestBuildICEServersIncludesConfiguredURLs(t *testing.T) { if len(iceServers) != 2 { t.Fatalf("buildICEServers() returned %d servers, want 2", len(iceServers)) } + if got := iceServers[0].URLs[0]; got != "stun:turn-fra1.kerberos.io:3478" { t.Fatalf("STUN URL = %q, want trimmed URL", got) } @@ -36,3 +49,138 @@ func TestBuildICEServersIncludesConfiguredURLs(t *testing.T) { t.Fatal("TURN credentials were not preserved") } } + +func TestWebRTCConnectionRequestsStartupAndRecoveryKeyframes(t *testing.T) { + viewer, err := pionWebRTC.NewPeerConnection(pionWebRTC.Configuration{}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := viewer.Close(); err != nil { + t.Error(err) + } + }) + if _, err := viewer.AddTransceiverFromKind(pionWebRTC.RTPCodecTypeVideo, pionWebRTC.RTPTransceiverInit{ + Direction: pionWebRTC.RTPTransceiverDirectionRecvonly, + }); err != nil { + t.Fatal(err) + } + tracks := make(chan *pionWebRTC.TrackRemote, 1) + viewer.OnTrack(func(track *pionWebRTC.TrackRemote, _ *pionWebRTC.RTPReceiver) { + tracks <- track + }) + offer, err := viewer.CreateOffer(nil) + if err != nil { + t.Fatal(err) + } + gathered := pionWebRTC.GatheringCompletePromise(viewer) + if err := viewer.SetLocalDescription(offer); err != nil { + t.Fatal(err) + } + select { + case <-gathered: + case <-time.After(5 * time.Second): + t.Fatal("viewer ICE gathering timed out") + } + + requests := make(chan time.Time, 10) + b := NewVideoBroadcaster([]packets.Stream{{Name: "H264", IsVideo: true}}, keyframeTestSource{ + request: func() error { requests <- time.Now(); return nil }, + }) + t.Cleanup(b.Close) + const sessionKey = "unit-camera/startup-recovery" + t.Cleanup(func() { + globalConnectionManager.mu.RLock() + wrapper := globalConnectionManager.peerConnections[sessionKey] + globalConnectionManager.mu.RUnlock() + if wrapper != nil { + cleanupPeerConnection(sessionKey, wrapper) + } + }) + var signalingMu sync.Mutex + var pending []pionWebRTC.ICECandidateInit + signaling := &models.LiveHDSignalingCallbacks{ + SendCandidate: func(_ string, data string) error { + var candidate pionWebRTC.ICECandidateInit + if err := json.Unmarshal([]byte(data), &candidate); err != nil { + return err + } + signalingMu.Lock() + defer signalingMu.Unlock() + if viewer.RemoteDescription() == nil { + pending = append(pending, candidate) + return nil + } + return viewer.AddICECandidate(candidate) + }, + SendAnswer: func(_ string, data string) error { + answer, err := base64.StdEncoding.DecodeString(data) + if err != nil { + return err + } + signalingMu.Lock() + defer signalingMu.Unlock() + if err := viewer.SetRemoteDescription(pionWebRTC.SessionDescription{ + Type: pionWebRTC.SDPTypeAnswer, SDP: string(answer), + }); err != nil { + return err + } + for _, candidate := range pending { + if err := viewer.AddICECandidate(candidate); err != nil { + return err + } + } + pending = nil + return nil + }, + } + InitializeWebRTCConnection( + &models.Configuration{Config: models.Config{Key: "unit-camera"}}, + &models.Communication{}, nil, b, nil, + models.LiveHDHandshake{ + Signaling: signaling, + Payload: models.RequestHDStreamPayload{ + SessionID: "startup-recovery", + SessionDescription: base64.StdEncoding.EncodeToString([]byte(viewer.LocalDescription().SDP)), + }, + }, + ) + var firstRequest time.Time + select { + case firstRequest = <-requests: + case <-time.After(5 * time.Second): + t.Fatal("connected transport did not request a startup keyframe") + } + b.WriteSample(pionMedia.Sample{Data: []byte{0, 0, 0, 1, 0x41, 0x80}, Duration: time.Second / 30}, false) + select { + case <-tracks: + t.Fatal("viewer received dependent video before its startup IDR") + case <-time.After(100 * time.Millisecond): + } + b.WriteSample(pionMedia.Sample{Data: []byte{0, 0, 0, 1, 0x65, 0x88, 0x84}, Duration: time.Second / 30}, true) + var track *pionWebRTC.TrackRemote + select { + case track = <-tracks: + case <-time.After(3 * time.Second): + t.Fatal("viewer did not receive the requested startup IDR") + } + + for _, feedback := range []rtcp.Packet{ + &rtcp.PictureLossIndication{SenderSSRC: 42, MediaSSRC: uint32(track.SSRC())}, + &rtcp.FullIntraRequest{SenderSSRC: 42, FIR: []rtcp.FIREntry{{SSRC: uint32(track.SSRC()), SequenceNumber: 1}}}, + } { + if err := viewer.WriteRTCP([]rtcp.Packet{feedback}); err != nil { + t.Fatal(err) + } + select { + case at := <-requests: + if at.Sub(firstRequest) < keyframeRequestInterval-5*time.Millisecond { + t.Fatal("real viewer feedback bypassed keyframe request throttling") + } + firstRequest = at + case <-time.After(3 * time.Second): + t.Fatalf("real %T feedback did not request a recovery keyframe", feedback) + } + b.WriteSample(pionMedia.Sample{Data: []byte{0, 0, 0, 1, 0x65, 0x88, 0x84}, Duration: time.Second / 30}, true) + } +} From 8c21638ff25400550bc7ae22d4084d4fddc6a223 Mon Sep 17 00:00:00 2001 From: cedricve Date: Wed, 18 Nov 2026 07:14:55 +0000 Subject: [PATCH 2/2] fix: preserve per-viewer RTP time across keyframe recovery Accumulate gated and discarded sample durations, including previously queued skips, and advance Pion's clock without filler packets before the recovery IDR. Verify exact outgoing RTP timestamp deltas for PLI, FIR, reconnects, and queue overflow. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- machinery/README.md | 3 + machinery/src/webrtc/broadcaster.go | 42 ++++-- machinery/src/webrtc/broadcaster_test.go | 2 +- machinery/src/webrtc/rtp_clock_test.go | 163 +++++++++++++++++++++++ 4 files changed, 199 insertions(+), 11 deletions(-) create mode 100644 machinery/src/webrtc/rtp_clock_test.go diff --git a/machinery/README.md b/machinery/README.md index 64c41c4..871acd6 100644 --- a/machinery/README.md +++ b/machinery/README.md @@ -33,6 +33,9 @@ Each H264 viewer waits until its WebRTC transport is connected and a fresh IDR arrives before receiving video. Reconnects, viewer PLI/FIR feedback, and a full per-viewer sample buffer reset that viewer's keyframe gate without stalling other viewers. Audio continues to use the existing independent sample queues. +Each viewer's skipped and discarded video durations advance its RTP clock before +recovery, without sending filler packets, so an IDR wait does not compress the +video timeline relative to audio. The main and sub streams request keyframes from their own RTSP sources. Viewer PLI and FIR feedback is translated into an RTSP PLI using the camera session's diff --git a/machinery/src/webrtc/broadcaster.go b/machinery/src/webrtc/broadcaster.go index 8ca672d..4b4fac2 100644 --- a/machinery/src/webrtc/broadcaster.go +++ b/machinery/src/webrtc/broadcaster.go @@ -18,13 +18,19 @@ const ( keyframeRequestInterval = time.Second ) +type peerSample struct { + pionMedia.Sample + skippedDuration time.Duration +} + // peerTrack is a per-peer track with its own non-blocking sample channel. type peerTrack struct { - track *pionWebRTC.TrackLocalStaticSample - samples chan pionMedia.Sample - done chan struct{} - connected bool - needsKeyframe bool + track *pionWebRTC.TrackLocalStaticSample + samples chan peerSample + done chan struct{} + connected bool + needsKeyframe bool + skippedDuration time.Duration } // TrackBroadcaster fans out media samples to multiple peer-specific tracks @@ -69,7 +75,7 @@ func (b *TrackBroadcaster) AddPeer(sessionKey string) (*pionWebRTC.TrackLocalSta pt := &peerTrack{ track: track, - samples: make(chan pionMedia.Sample, peerSampleBuffer), + samples: make(chan peerSample, peerSampleBuffer), done: make(chan struct{}), connected: b.mimeType != pionWebRTC.MimeTypeH264, needsKeyframe: b.mimeType == pionWebRTC.MimeTypeH264, @@ -87,7 +93,7 @@ func (b *TrackBroadcaster) AddPeer(sessionKey string) (*pionWebRTC.TrackLocalSta go func() { defer close(pt.done) for sample := range pt.samples { - if err := pt.track.WriteSample(sample); err != nil { + if err := writePeerSample(pt.track, sample); err != nil { if err == io.ErrClosedPipe { return } @@ -108,6 +114,16 @@ func (b *TrackBroadcaster) AddPeer(sessionKey string) (*pionWebRTC.TrackLocalSta return track, nil } +func writePeerSample(track *pionWebRTC.TrackLocalStaticSample, sample peerSample) error { + if sample.skippedDuration > 0 { + // An empty sample advances Pion's RTP clock without sending a packet. + if err := track.WriteSample(pionMedia.Sample{Duration: sample.skippedDuration}); err != nil { + return err + } + } + return track.WriteSample(sample.Sample) +} + // RemovePeer stops the writer goroutine and removes the peer. func (b *TrackBroadcaster) RemovePeer(sessionKey string) { b.mu.Lock() @@ -141,10 +157,12 @@ func (b *TrackBroadcaster) WriteSample(sample pionMedia.Sample, isKeyframe bool) for sessionKey, pt := range b.peers { video := b.mimeType == pionWebRTC.MimeTypeH264 if video && (!pt.connected || (pt.needsKeyframe && !isKeyframe)) { + pt.skippedDuration += sample.Duration continue } select { - case pt.samples <- sample: + case pt.samples <- peerSample{Sample: sample, skippedDuration: pt.skippedDuration}: + pt.skippedDuration = 0 if video && isKeyframe { pt.needsKeyframe = false } @@ -153,8 +171,11 @@ func (b *TrackBroadcaster) WriteSample(sample pionMedia.Sample, isKeyframe bool) pt.needsKeyframe = true drainPeerSamples(pt) if isKeyframe { - pt.samples <- sample + pt.samples <- peerSample{Sample: sample, skippedDuration: pt.skippedDuration} + pt.skippedDuration = 0 pt.needsKeyframe = false + } else { + pt.skippedDuration += sample.Duration } } log.WithFields(log.Fields{ @@ -170,7 +191,8 @@ func (b *TrackBroadcaster) WriteSample(sample pionMedia.Sample, isKeyframe bool) func drainPeerSamples(pt *peerTrack) { for { select { - case <-pt.samples: + case sample := <-pt.samples: + pt.skippedDuration += sample.skippedDuration + sample.Duration default: return } diff --git a/machinery/src/webrtc/broadcaster_test.go b/machinery/src/webrtc/broadcaster_test.go index 669eefb..0aefad6 100644 --- a/machinery/src/webrtc/broadcaster_test.go +++ b/machinery/src/webrtc/broadcaster_test.go @@ -22,7 +22,7 @@ func bufferedTestPeer(capacity int) *peerTrack { done := make(chan struct{}) close(done) return &peerTrack{ - samples: make(chan pionMedia.Sample, capacity), + samples: make(chan peerSample, capacity), done: done, needsKeyframe: true, } diff --git a/machinery/src/webrtc/rtp_clock_test.go b/machinery/src/webrtc/rtp_clock_test.go new file mode 100644 index 0000000..14a6913 --- /dev/null +++ b/machinery/src/webrtc/rtp_clock_test.go @@ -0,0 +1,163 @@ +package webrtc + +import ( + "errors" + "testing" + "time" + + "github.com/pion/interceptor" + "github.com/pion/rtcp" + "github.com/pion/rtp" + pionWebRTC "github.com/pion/webrtc/v4" + pionMedia "github.com/pion/webrtc/v4/pkg/media" +) + +type clockTestWriter struct { + timestamps []uint32 +} + +func (w *clockTestWriter) WriteRTP(header *rtp.Header, payload []byte) (int, error) { + w.timestamps = append(w.timestamps, header.Timestamp) + return len(payload), nil +} + +func (*clockTestWriter) Write([]byte) (int, error) { + return 0, errors.New("unexpected raw RTP write") +} + +type clockTestContext struct { + writer *clockTestWriter +} + +func (clockTestContext) ID() string { return "clock-test" } +func (clockTestContext) SSRC() pionWebRTC.SSRC { return 123 } +func (clockTestContext) SSRCRetransmission() pionWebRTC.SSRC { return 0 } +func (clockTestContext) SSRCForwardErrorCorrection() pionWebRTC.SSRC { return 0 } +func (clockTestContext) RTCPReader() interceptor.RTCPReader { return nil } +func (clockTestContext) HeaderExtensions() []pionWebRTC.RTPHeaderExtensionParameter { + return nil +} +func (c clockTestContext) WriteStream() pionWebRTC.TrackLocalWriter { return c.writer } +func (clockTestContext) CodecParameters() []pionWebRTC.RTPCodecParameters { + return []pionWebRTC.RTPCodecParameters{{ + RTPCodecCapability: pionWebRTC.RTPCodecCapability{MimeType: pionWebRTC.MimeTypeH264, ClockRate: 90000}, + PayloadType: 96, + }} +} + +func clockTestPeer(t *testing.T) (*TrackBroadcaster, *peerTrack, *clockTestWriter) { + t.Helper() + b, pt := testVideoBroadcaster(t) + pt.samples = make(chan peerSample, 2) + track, err := pionWebRTC.NewTrackLocalStaticSample( + pionWebRTC.RTPCodecCapability{MimeType: pionWebRTC.MimeTypeH264}, + "video", "clock-test", pionWebRTC.WithRTPTimestamp(1000), + ) + if err != nil { + t.Fatal(err) + } + writer := &clockTestWriter{} + if _, err := track.Bind(clockTestContext{writer: writer}); err != nil { + t.Fatal(err) + } + pt.track = track + b.SetPeerConnected("viewer", true) + return b, pt, writer +} + +func clockVideoSample(duration time.Duration) pionMedia.Sample { + return pionMedia.Sample{ + Data: []byte{0, 0, 0, 1, 0x65, 0x88, 0x84}, + Duration: duration, + } +} + +func writeClockSample(t *testing.T, pt *peerTrack) { + t.Helper() + select { + case sample := <-pt.samples: + if err := writePeerSample(pt.track, sample); err != nil { + t.Fatal(err) + } + default: + t.Fatal("expected queued video sample") + } +} + +func TestVideoRecoveryPreservesRTPClock(t *testing.T) { + for _, scenario := range []string{"PLI", "FIR", "reconnect", "overflow"} { + t.Run(scenario, func(t *testing.T) { + b, pt, writer := clockTestPeer(t) + b.WriteSample(clockVideoSample(40*time.Millisecond), true) + writeClockSample(t, pt) + b.WriteSample(clockVideoSample(70*time.Millisecond), false) + + switch scenario { + case "PLI", "FIR": + var feedback rtcp.Packet = &rtcp.PictureLossIndication{MediaSSRC: 123} + if scenario == "FIR" { + feedback = &rtcp.FullIntraRequest{FIR: []rtcp.FIREntry{{SSRC: 123, SequenceNumber: 1}}} + } + data, err := rtcp.Marshal([]rtcp.Packet{feedback}) + if err != nil { + t.Fatal(err) + } + if err := b.HandleRTCP("viewer", data); err != nil { + t.Fatal(err) + } + b.WriteSample(clockVideoSample(30*time.Millisecond), false) + case "reconnect": + b.SetPeerConnected("viewer", false) + b.WriteSample(clockVideoSample(30*time.Millisecond), false) + b.SetPeerConnected("viewer", true) + case "overflow": + b.WriteSample(clockVideoSample(30*time.Millisecond), false) + } + b.WriteSample(clockVideoSample(1900*time.Millisecond), false) + b.WriteSample(clockVideoSample(40*time.Millisecond), true) + writeClockSample(t, pt) + + if len(writer.timestamps) != 2 { + t.Fatalf("sent %d RTP packets, want only the two IDRs", len(writer.timestamps)) + } + // The initial 40 ms frame plus 2000 ms of discarded video at 90 kHz. + if got := writer.timestamps[1] - writer.timestamps[0]; got != 183600 { + t.Fatalf("recovery RTP timestamp delta = %d, want 183600", got) + } + }) + } +} + +func TestOverflowIDRPreservesDiscardedRTPDuration(t *testing.T) { + b, pt, writer := clockTestPeer(t) + b.WriteSample(clockVideoSample(40*time.Millisecond), true) + writeClockSample(t, pt) + b.WriteSample(clockVideoSample(70*time.Millisecond), false) + b.WriteSample(clockVideoSample(30*time.Millisecond), false) + b.WriteSample(clockVideoSample(40*time.Millisecond), true) + writeClockSample(t, pt) + if len(writer.timestamps) != 2 || writer.timestamps[1]-writer.timestamps[0] != 12600 { + t.Fatalf("overflow IDR compressed the RTP clock: %v", writer.timestamps) + } +} + +func TestRepeatedFeedbackPreservesAlreadyQueuedSkippedTime(t *testing.T) { + b, pt, writer := clockTestPeer(t) + b.WriteSample(clockVideoSample(40*time.Millisecond), true) + writeClockSample(t, pt) + data, err := rtcp.Marshal([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: 123}}) + if err != nil { + t.Fatal(err) + } + for range 2 { + if err := b.HandleRTCP("viewer", data); err != nil { + t.Fatal(err) + } + b.WriteSample(clockVideoSample(time.Second), false) + b.WriteSample(clockVideoSample(40*time.Millisecond), true) + } + writeClockSample(t, pt) + if len(writer.timestamps) != 2 || writer.timestamps[1]-writer.timestamps[0] != 187200 { + t.Fatalf("repeated feedback lost queued skipped time: %v", writer.timestamps) + } +}