diff --git a/machinery/README.md b/machinery/README.md index 1f3a31f..871acd6 100644 --- a/machinery/README.md +++ b/machinery/README.md @@ -26,3 +26,28 @@ Kerberos Open source (v3) is a cutting edge video surveillance management system ## Security reporting For sensitive vulnerabilities, use private disclosure channels documented in [../SECURITY.md](../SECURITY.md). + +## WebRTC startup and keyframe recovery + +Each H264 viewer waits until its WebRTC transport is connected and a fresh IDR +arrives before receiving video. Reconnects, viewer PLI/FIR feedback, and a full +per-viewer sample buffer reset that viewer's keyframe gate without stalling +other viewers. Audio continues to use the existing independent sample queues. +Each viewer's skipped and discarded video durations advance its RTP clock before +recovery, without sending filler packets, so an IDR wait does not compress the +video timeline relative to audio. + +The main and sub streams request keyframes from their own RTSP sources. Viewer +PLI and FIR feedback is translated into an RTSP PLI using the camera session's +SSRCs, never forwarded with the browser's SSRCs. Requests are coalesced and +limited to one per second per broadcaster, retrying while connected viewers +still need an IDR. Requests stop once all waiting viewers receive an IDR or +the broadcaster closes. + +Camera support for RTCP PLI varies. Unsupported sources, cameras that ignore +feedback, and failed requests fall back to the next periodic IDR; failures are +logged as `keyframe_request_failed`. No encoder settings are changed. A camera +with a long GOP can therefore still start slowly; configure a shorter IDR +interval (for example 1-2 seconds) when camera feedback is unavailable, accounting +for the bandwidth tradeoff. An isolated cached IDR must not be replayed followed +by arbitrary live P-frames: the dependent reference chain would be incomplete. diff --git a/machinery/go.mod b/machinery/go.mod index dffbea8..32d8b04 100644 --- a/machinery/go.mod +++ b/machinery/go.mod @@ -30,6 +30,7 @@ require ( github.com/moq-dev/moq-go v0.5.7 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 github.com/pion/interceptor v0.1.47 + github.com/pion/rtcp v1.2.17 github.com/pion/rtp v1.10.5 github.com/pion/webrtc/v4 v4.2.18 github.com/sirupsen/logrus v1.9.3 @@ -104,7 +105,6 @@ require ( github.com/pion/logging v0.2.4 // indirect github.com/pion/mdns/v2 v2.1.0 // indirect github.com/pion/randutil v0.1.0 // indirect - github.com/pion/rtcp v1.2.17 // indirect github.com/pion/sctp v1.11.1 // indirect github.com/pion/sdp/v3 v3.0.19 // indirect github.com/pion/srtp/v3 v3.0.12 // indirect diff --git a/machinery/src/capture/gortsplib.go b/machinery/src/capture/gortsplib.go index a19a534..4ecc4a3 100644 --- a/machinery/src/capture/gortsplib.go +++ b/machinery/src/capture/gortsplib.go @@ -101,6 +101,8 @@ type Golibrtsp struct { Client gortsplib.Client clientStarted bool VideoDecoderMutex *sync.Mutex + keyframeMu sync.Mutex + keyframeReady bool VideoH264Index int8 VideoH264Media *description.Media @@ -813,6 +815,10 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e } } + g.keyframeMu.Lock() + g.keyframeReady = err == nil && g.VideoH264Media != nil && g.VideoH264Decoder != nil + g.keyframeMu.Unlock() + return } @@ -1713,6 +1719,10 @@ func (g *Golibrtsp) Close(ctxOtel context.Context) error { _, span := tracer.Start(ctxOtel, "Close") defer span.End() + g.keyframeMu.Lock() + g.keyframeReady = false + g.keyframeMu.Unlock() + if !g.clientStarted { return nil } diff --git a/machinery/src/capture/keyframe.go b/machinery/src/capture/keyframe.go new file mode 100644 index 0000000..8de9ce3 --- /dev/null +++ b/machinery/src/capture/keyframe.go @@ -0,0 +1,35 @@ +package capture + +import ( + "errors" + "fmt" + + "github.com/pion/rtcp" +) + +func (g *Golibrtsp) RequestKeyframe() error { + g.keyframeMu.Lock() + defer g.keyframeMu.Unlock() + + if !g.keyframeReady { + return errors.New("RTSP H264 keyframe requests are unavailable") + } + + media, ok := g.Client.Stats().Session.Medias[g.VideoH264Media] + if !ok { + return errors.New("RTSP video media is not set up") + } + video, ok := media.Formats[g.VideoH264Forma] + if !ok || video.InboundRTPPackets == 0 { + return errors.New("RTSP video RTP has not arrived yet") + } + + // Browser and camera SSRCs differ: originate feedback on the RTSP session. + if err := g.Client.WritePacketRTCP(g.VideoH264Media, &rtcp.PictureLossIndication{ + SenderSSRC: video.LocalSSRC, + MediaSSRC: video.RemoteSSRC, + }); err != nil { + return fmt.Errorf("request camera keyframe: %w", err) + } + return nil +} diff --git a/machinery/src/capture/keyframe_test.go b/machinery/src/capture/keyframe_test.go new file mode 100644 index 0000000..3e91923 --- /dev/null +++ b/machinery/src/capture/keyframe_test.go @@ -0,0 +1,164 @@ +package capture + +import ( + "context" + "net" + "sync" + "testing" + "time" + + "github.com/bluenviron/gortsplib/v5" + "github.com/bluenviron/gortsplib/v5/pkg/base" + "github.com/bluenviron/gortsplib/v5/pkg/description" + "github.com/bluenviron/gortsplib/v5/pkg/format" + "github.com/pion/rtcp" + "github.com/pion/rtp" +) + +type keyframeCamera struct { + stream *gortsplib.ServerStream + feedback chan rtcp.Packet +} + +func (c *keyframeCamera) OnDescribe(*gortsplib.ServerHandlerOnDescribeCtx) (*base.Response, *gortsplib.ServerStream, error) { + return &base.Response{StatusCode: base.StatusOK}, c.stream, nil +} + +func (c *keyframeCamera) OnSetup(*gortsplib.ServerHandlerOnSetupCtx) (*base.Response, *gortsplib.ServerStream, error) { + return &base.Response{StatusCode: base.StatusOK}, c.stream, nil +} + +func (c *keyframeCamera) OnPlay(ctx *gortsplib.ServerHandlerOnPlayCtx) (*base.Response, error) { + ctx.Session.OnPacketRTCPAny(func(_ *description.Media, pkt rtcp.Packet) { + c.feedback <- pkt + }) + return &base.Response{StatusCode: base.StatusOK}, nil +} + +func connectedKeyframeCamera(t *testing.T) (*Golibrtsp, *keyframeCamera) { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { listener.Close() }) + camera := &keyframeCamera{feedback: make(chan rtcp.Packet, 32)} + server := &gortsplib.Server{ + RTSPAddress: "127.0.0.1:0", + Handler: camera, + Listen: func(string, string) (net.Listener, error) { + return listener, nil + }, + } + if err := server.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(server.Close) + media := &description.Media{ + Type: description.MediaTypeVideo, + Formats: []format.Format{&format.H264{ + PayloadTyp: 96, + PacketizationMode: 1, + }}, + } + camera.stream = &gortsplib.ServerStream{ + Server: server, + Desc: &description.Session{Medias: []*description.Media{media}}, + } + if err := camera.stream.Initialize(); err != nil { + t.Fatal(err) + } + t.Cleanup(camera.stream.Close) + + client := &Golibrtsp{Url: "rtsp://" + listener.Addr().String() + "/camera"} + t.Cleanup(func() { + if err := client.Close(context.Background()); err != nil { + t.Error(err) + } + }) + if err := client.Connect(context.Background(), context.Background()); err != nil { + t.Fatal(err) + } + if err := client.RequestKeyframe(); err == nil { + t.Fatal("requested a keyframe before the camera SSRC was known") + } + received := make(chan struct{}, 1) + client.Client.OnPacketRTP(client.VideoH264Media, client.VideoH264Forma, func(*rtp.Packet) { + received <- struct{}{} + }) + if _, err := client.Client.Play(nil); err != nil { + t.Fatal(err) + } + if err := camera.stream.WritePacketRTP(media, &rtp.Packet{ + Header: rtp.Header{ + Version: 2, + PayloadType: 96, + SSRC: 0x12345678, + SequenceNumber: 1, + Timestamp: 90000, + Marker: true, + }, + Payload: []byte{0x65, 0x88, 0x84}, + }); err != nil { + t.Fatal(err) + } + select { + case <-received: + case <-time.After(3 * time.Second): + t.Fatal("test camera RTP did not arrive") + } + return client, camera +} + +func TestRTSPKeyframeRequestUsesCameraSessionSSRCs(t *testing.T) { + client, camera := connectedKeyframeCamera(t) + want := client.Client.Stats().Session.Medias[client.VideoH264Media].Formats[client.VideoH264Forma] + if err := client.RequestKeyframe(); err != nil { + t.Fatal(err) + } + select { + case pkt := <-camera.feedback: + pli, ok := pkt.(*rtcp.PictureLossIndication) + if !ok { + t.Fatalf("camera received %T, want PLI", pkt) + } + if pli.MediaSSRC != want.RemoteSSRC || pli.SenderSSRC != want.LocalSSRC { + t.Fatalf("camera PLI SSRCs = sender %x media %x, want sender %x media %x", + pli.SenderSSRC, pli.MediaSSRC, want.LocalSSRC, want.RemoteSSRC) + } + case <-time.After(3 * time.Second): + t.Fatal("camera did not receive RTCP PLI") + } +} + +func TestRTSPKeyframeRequestUnavailableBeforeConnectAndAfterClose(t *testing.T) { + if err := (&Golibrtsp{}).RequestKeyframe(); err == nil { + t.Fatal("uninitialized client accepted a keyframe request") + } + client, _ := connectedKeyframeCamera(t) + if err := client.Close(context.Background()); err != nil { + t.Fatal(err) + } + if err := client.RequestKeyframe(); err == nil { + t.Fatal("closed client accepted a keyframe request") + } +} + +func TestRTSPKeyframeRequestConcurrentWithClose(t *testing.T) { + client, _ := connectedKeyframeCamera(t) + var wg sync.WaitGroup + for range 4 { + wg.Add(1) + go func() { + defer wg.Done() + for range 5 { + // Requests may race with Close and legitimately become unavailable. + _ = client.RequestKeyframe() + } + }() + } + if err := client.Close(context.Background()); err != nil { + t.Fatal(err) + } + wg.Wait() +} diff --git a/machinery/src/capture/rtsp_client.go b/machinery/src/capture/rtsp_client.go index 8b5896f..1a13a5e 100644 --- a/machinery/src/capture/rtsp_client.go +++ b/machinery/src/capture/rtsp_client.go @@ -72,6 +72,10 @@ func (c *Capture) ClearClients(main, sub, backchannel *Golibrtsp) { } } +type KeyframeRequester interface { + RequestKeyframe() error +} + // RTSPClient is a interface that abstracts the RTSP client implementation. type RTSPClient interface { // Connect to the RTSP server. diff --git a/machinery/src/cloud/cloud.go b/machinery/src/cloud/cloud.go index 25943d4..c3d00e8 100644 --- a/machinery/src/cloud/cloud.go +++ b/machinery/src/cloud/cloud.go @@ -1285,7 +1285,7 @@ func HandleLiveStreamHD( // viewer can pick the resolution it needs per peer connection without // the agent re-negotiating the RTSP source. mainStreams, _ := rtspClient.GetStreams() - mainVideoBroadcaster := webrtc.NewVideoBroadcaster(mainStreams) + mainVideoBroadcaster := webrtc.NewVideoBroadcaster(mainStreams, rtspClient) mainAudioBroadcaster := webrtc.NewAudioBroadcaster(mainStreams) if mainVideoBroadcaster == nil && mainAudioBroadcaster == nil { @@ -1316,7 +1316,7 @@ func HandleLiveStreamHD( var subAudioBroadcaster *webrtc.TrackBroadcaster if subStreamEnabled && rtspSubClient != nil && subQueue != nil { subStreams, _ := rtspSubClient.GetStreams() - subVideoBroadcaster = webrtc.NewVideoBroadcaster(subStreams) + subVideoBroadcaster = webrtc.NewVideoBroadcaster(subStreams, rtspSubClient) subAudioBroadcaster = webrtc.NewAudioBroadcaster(subStreams) defer func() { if subVideoBroadcaster != nil { diff --git a/machinery/src/webrtc/broadcaster.go b/machinery/src/webrtc/broadcaster.go index a29e452..4b4fac2 100644 --- a/machinery/src/webrtc/broadcaster.go +++ b/machinery/src/webrtc/broadcaster.go @@ -3,7 +3,9 @@ package webrtc import ( "io" "sync" + "time" + "github.com/pion/rtcp" pionWebRTC "github.com/pion/webrtc/v4" pionMedia "github.com/pion/webrtc/v4/pkg/media" log "github.com/sirupsen/logrus" @@ -12,14 +14,23 @@ import ( const ( // peerSampleBuffer controls how many samples can be buffered per peer before // dropping. Keeps slow peers from blocking the broadcaster. - peerSampleBuffer = 60 + peerSampleBuffer = 60 + keyframeRequestInterval = time.Second ) +type peerSample struct { + pionMedia.Sample + skippedDuration time.Duration +} + // peerTrack is a per-peer track with its own non-blocking sample channel. type peerTrack struct { - track *pionWebRTC.TrackLocalStaticSample - samples chan pionMedia.Sample - done chan struct{} + track *pionWebRTC.TrackLocalStaticSample + samples chan peerSample + done chan struct{} + connected bool + needsKeyframe bool + skippedDuration time.Duration } // TrackBroadcaster fans out media samples to multiple peer-specific tracks @@ -27,11 +38,17 @@ type peerTrack struct { // goroutine that drains samples independently, so a slow/congested peer // cannot stall the others. type TrackBroadcaster struct { - mu sync.RWMutex - peers map[string]*peerTrack - mimeType string - id string - streamID string + mu sync.RWMutex + peers map[string]*peerTrack + mimeType string + id string + streamID string + requestKeyframe func() error + requestTimer *time.Timer + requestInFlight bool + lastRequest time.Time + requests sync.WaitGroup + closed bool } // NewTrackBroadcaster creates a new broadcaster for either video or audio. @@ -57,12 +74,18 @@ func (b *TrackBroadcaster) AddPeer(sessionKey string) (*pionWebRTC.TrackLocalSta } pt := &peerTrack{ - track: track, - samples: make(chan pionMedia.Sample, peerSampleBuffer), - done: make(chan struct{}), + track: track, + samples: make(chan peerSample, peerSampleBuffer), + done: make(chan struct{}), + connected: b.mimeType != pionWebRTC.MimeTypeH264, + needsKeyframe: b.mimeType == pionWebRTC.MimeTypeH264, } b.mu.Lock() + if b.closed { + b.mu.Unlock() + return nil, io.ErrClosedPipe + } b.peers[sessionKey] = pt b.mu.Unlock() @@ -70,7 +93,7 @@ func (b *TrackBroadcaster) AddPeer(sessionKey string) (*pionWebRTC.TrackLocalSta go func() { defer close(pt.done) for sample := range pt.samples { - if err := pt.track.WriteSample(sample); err != nil { + if err := writePeerSample(pt.track, sample); err != nil { if err == io.ErrClosedPipe { return } @@ -91,6 +114,16 @@ func (b *TrackBroadcaster) AddPeer(sessionKey string) (*pionWebRTC.TrackLocalSta return track, nil } +func writePeerSample(track *pionWebRTC.TrackLocalStaticSample, sample peerSample) error { + if sample.skippedDuration > 0 { + // An empty sample advances Pion's RTP clock without sending a packet. + if err := track.WriteSample(pionMedia.Sample{Duration: sample.skippedDuration}); err != nil { + return err + } + } + return track.WriteSample(sample.Sample) +} + // RemovePeer stops the writer goroutine and removes the peer. func (b *TrackBroadcaster) RemovePeer(sessionKey string) { b.mu.Lock() @@ -114,14 +147,37 @@ func (b *TrackBroadcaster) RemovePeer(sessionKey string) { // WriteSample fans out a sample to all connected peers without blocking. // If a peer's buffer is full (slow consumer), the sample is dropped for // that peer only — other peers are unaffected. -func (b *TrackBroadcaster) WriteSample(sample pionMedia.Sample) { - b.mu.RLock() - defer b.mu.RUnlock() +func (b *TrackBroadcaster) WriteSample(sample pionMedia.Sample, isKeyframe bool) { + b.mu.Lock() + defer b.mu.Unlock() + if b.closed { + return + } for sessionKey, pt := range b.peers { + video := b.mimeType == pionWebRTC.MimeTypeH264 + if video && (!pt.connected || (pt.needsKeyframe && !isKeyframe)) { + pt.skippedDuration += sample.Duration + continue + } select { - case pt.samples <- sample: + case pt.samples <- peerSample{Sample: sample, skippedDuration: pt.skippedDuration}: + pt.skippedDuration = 0 + if video && isKeyframe { + pt.needsKeyframe = false + } default: + if video { + pt.needsKeyframe = true + drainPeerSamples(pt) + if isKeyframe { + pt.samples <- peerSample{Sample: sample, skippedDuration: pt.skippedDuration} + pt.skippedDuration = 0 + pt.needsKeyframe = false + } else { + pt.skippedDuration += sample.Duration + } + } log.WithFields(log.Fields{ "component": "webrtc", "event": "slow_peer_sample_dropped", @@ -129,6 +185,106 @@ func (b *TrackBroadcaster) WriteSample(sample pionMedia.Sample) { }).Warn("Dropping WebRTC sample for slow peer") } } + b.scheduleKeyframeRequestLocked() +} + +func drainPeerSamples(pt *peerTrack) { + for { + select { + case sample := <-pt.samples: + pt.skippedDuration += sample.skippedDuration + sample.Duration + default: + return + } + } +} + +func (b *TrackBroadcaster) SetPeerConnected(sessionKey string, connected bool) { + b.mu.Lock() + defer b.mu.Unlock() + + pt, ok := b.peers[sessionKey] + if !ok || b.mimeType != pionWebRTC.MimeTypeH264 || pt.connected == connected { + return + } + pt.connected = connected + pt.needsKeyframe = true + drainPeerSamples(pt) + b.scheduleKeyframeRequestLocked() +} + +func (b *TrackBroadcaster) HandleRTCP(sessionKey string, data []byte) error { + feedback, err := rtcp.Unmarshal(data) + if err != nil { + return err + } + for _, packet := range feedback { + switch packet.(type) { + case *rtcp.PictureLossIndication, *rtcp.FullIntraRequest: + b.mu.Lock() + if pt, ok := b.peers[sessionKey]; ok && pt.connected && b.mimeType == pionWebRTC.MimeTypeH264 { + pt.needsKeyframe = true + drainPeerSamples(pt) + b.scheduleKeyframeRequestLocked() + } + b.mu.Unlock() + return nil + } + } + return nil +} + +func (b *TrackBroadcaster) waitingForKeyframeLocked() bool { + for _, pt := range b.peers { + if pt.connected && pt.needsKeyframe { + return true + } + } + return false +} + +func (b *TrackBroadcaster) scheduleKeyframeRequestLocked() { + if b.closed || b.requestKeyframe == nil || b.requestInFlight || !b.waitingForKeyframeLocked() { + return + } + if b.requestTimer == nil { + delay := time.Until(b.lastRequest.Add(keyframeRequestInterval)) + if delay < 0 { + delay = 0 + } + b.requestTimer = time.AfterFunc(delay, b.sendKeyframeRequest) + } +} + +func (b *TrackBroadcaster) sendKeyframeRequest() { + b.mu.Lock() + b.requestTimer = nil + if b.closed || !b.waitingForKeyframeLocked() { + b.mu.Unlock() + return + } + b.lastRequest = time.Now() + b.requestInFlight = true + b.requests.Add(1) + b.mu.Unlock() + defer b.requests.Done() + + if err := b.requestKeyframe(); err != nil { + log.WithError(err).WithFields(log.Fields{ + "component": "webrtc", + "event": "keyframe_request_failed", + }).Warn("Camera keyframe request failed; waiting for a periodic IDR") + } else { + log.WithFields(log.Fields{ + "component": "webrtc", + "event": "keyframe_requested", + }).Debug("Requested camera keyframe") + } + + b.mu.Lock() + b.requestInFlight = false + b.scheduleKeyframeRequestLocked() + b.mu.Unlock() } // PeerCount returns the current number of connected peers. @@ -141,11 +297,17 @@ func (b *TrackBroadcaster) PeerCount() int { // Close removes all peers and stops all writer goroutines. func (b *TrackBroadcaster) Close() { b.mu.Lock() + b.closed = true + if b.requestTimer != nil { + b.requestTimer.Stop() + b.requestTimer = nil + } keys := make([]string, 0, len(b.peers)) for k := range b.peers { keys = append(keys, k) } b.mu.Unlock() + b.requests.Wait() for _, key := range keys { b.RemovePeer(key) diff --git a/machinery/src/webrtc/broadcaster_test.go b/machinery/src/webrtc/broadcaster_test.go new file mode 100644 index 0000000..0aefad6 --- /dev/null +++ b/machinery/src/webrtc/broadcaster_test.go @@ -0,0 +1,360 @@ +package webrtc + +import ( + "bytes" + "errors" + "fmt" + "io" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/kerberos-io/agent/machinery/src/capture" + "github.com/kerberos-io/agent/machinery/src/models" + "github.com/kerberos-io/agent/machinery/src/packets" + "github.com/pion/rtcp" + pionWebRTC "github.com/pion/webrtc/v4" + pionMedia "github.com/pion/webrtc/v4/pkg/media" +) + +func bufferedTestPeer(capacity int) *peerTrack { + done := make(chan struct{}) + close(done) + return &peerTrack{ + samples: make(chan peerSample, capacity), + done: done, + needsKeyframe: true, + } +} + +func testVideoBroadcaster(t *testing.T) (*TrackBroadcaster, *peerTrack) { + t.Helper() + b := NewTrackBroadcaster(pionWebRTC.MimeTypeH264, "video", "test") + pt := bufferedTestPeer(peerSampleBuffer) + b.peers["viewer"] = pt + t.Cleanup(b.Close) + return b, pt +} + +func TestVideoPeerStartsAndReconnectsAtKeyframe(t *testing.T) { + b, pt := testVideoBroadcaster(t) + sample := pionMedia.Sample{Data: []byte{1}} + b.WriteSample(sample, true) + if len(pt.samples) != 0 { + t.Fatal("sent a keyframe before the transport connected") + } + + b.SetPeerConnected("viewer", true) + b.WriteSample(sample, false) + if len(pt.samples) != 0 { + t.Fatal("sent a dependent frame before the first IDR") + } + b.WriteSample(sample, true) + b.WriteSample(sample, false) + if len(pt.samples) != 2 || pt.needsKeyframe { + t.Fatal("did not release video after the first IDR") + } + + b.SetPeerConnected("viewer", false) + b.WriteSample(sample, true) + if len(pt.samples) != 0 { + t.Fatal("retained or sent video while disconnected") + } + b.SetPeerConnected("viewer", true) + b.WriteSample(sample, false) + if len(pt.samples) != 0 { + t.Fatal("reconnected viewer resumed on a dependent frame") + } + b.WriteSample(sample, true) + if len(pt.samples) != 1 { + t.Fatal("reconnected viewer did not resume at an IDR") + } +} + +func TestLateViewerDoesNotResetExistingViewer(t *testing.T) { + b, first := testVideoBroadcaster(t) + b.SetPeerConnected("viewer", true) + b.WriteSample(pionMedia.Sample{}, true) + <-first.samples + + late := bufferedTestPeer(peerSampleBuffer) + b.peers["late"] = late + b.SetPeerConnected("late", true) + b.WriteSample(pionMedia.Sample{}, false) + if len(first.samples) != 1 || len(late.samples) != 0 { + t.Fatal("late viewer either interrupted an existing viewer or received a dependent frame") + } + b.WriteSample(pionMedia.Sample{}, true) + if len(late.samples) != 1 { + t.Fatal("late viewer did not receive the next IDR") + } +} + +func TestVideoFeedbackRequestsFreshKeyframe(t *testing.T) { + for _, feedback := range []rtcp.Packet{ + &rtcp.PictureLossIndication{MediaSSRC: 123}, + &rtcp.FullIntraRequest{FIR: []rtcp.FIREntry{{SSRC: 123, SequenceNumber: 1}}}, + } { + t.Run(fmt.Sprintf("%T", feedback), func(t *testing.T) { + b, pt := testVideoBroadcaster(t) + b.SetPeerConnected("viewer", true) + b.WriteSample(pionMedia.Sample{}, true) + b.WriteSample(pionMedia.Sample{}, false) + + rr, err := rtcp.Marshal([]rtcp.Packet{&rtcp.ReceiverReport{}}) + if err != nil { + t.Fatal(err) + } + if err := b.HandleRTCP("viewer", rr); err != nil || len(pt.samples) != 2 { + t.Fatal("receiver report reset the video gate", err) + } + data, err := rtcp.Marshal([]rtcp.Packet{&rtcp.ReceiverReport{}, feedback}) + if err != nil { + t.Fatal(err) + } + if err := b.HandleRTCP("viewer", data); err != nil { + t.Fatal(err) + } + b.WriteSample(pionMedia.Sample{}, false) + if len(pt.samples) != 0 || !pt.needsKeyframe { + t.Fatal("PLI/FIR did not discard stale queued frames and wait for an IDR") + } + b.WriteSample(pionMedia.Sample{}, true) + if len(pt.samples) != 1 { + t.Fatal("PLI/FIR recovery did not resume at an IDR") + } + if err := b.HandleRTCP("viewer", []byte{0}); err == nil { + t.Fatal("malformed RTCP was accepted") + } + }) + } +} + +func TestSlowVideoPeerRecoversWithoutBlockingOthers(t *testing.T) { + for _, overflowIsKeyframe := range []bool{false, true} { + t.Run(map[bool]string{false: "dependent-frame-dropped", true: "new-IDR-replaces-backlog"}[overflowIsKeyframe], func(t *testing.T) { + b, fast := testVideoBroadcaster(t) + slow := bufferedTestPeer(2) + b.peers["slow"] = slow + b.SetPeerConnected("viewer", true) + b.SetPeerConnected("slow", true) + b.WriteSample(pionMedia.Sample{}, true) + b.WriteSample(pionMedia.Sample{}, false) + b.WriteSample(pionMedia.Sample{}, overflowIsKeyframe) + if len(fast.samples) != 3 { + t.Fatal("slow viewer interrupted another viewer") + } + if overflowIsKeyframe { + if len(slow.samples) != 1 || slow.needsKeyframe { + t.Fatal("fresh IDR did not replace the stale backlog") + } + } else { + b.WriteSample(pionMedia.Sample{}, false) + if len(slow.samples) != 0 || !slow.needsKeyframe { + t.Fatal("slow viewer forwarded dependent frames after a drop") + } + b.WriteSample(pionMedia.Sample{}, true) + if len(slow.samples) != 1 { + t.Fatal("slow viewer did not recover at the next IDR") + } + } + }) + } +} + +func TestAudioDoesNotWaitForVideoKeyframe(t *testing.T) { + b := NewTrackBroadcaster(pionWebRTC.MimeTypePCMU, "audio", "test") + pt := bufferedTestPeer(2) + pt.connected = true + pt.needsKeyframe = false + b.peers["viewer"] = pt + t.Cleanup(b.Close) + + b.SetPeerConnected("viewer", false) + for range 3 { + b.WriteSample(pionMedia.Sample{}, false) + } + if len(pt.samples) != 2 { + t.Fatal("audio queue behavior changed") + } +} + +func TestVideoSampleBufferPreservesKeyframeFlag(t *testing.T) { + b, pt := testVideoBroadcaster(t) + b.SetPeerConnected("viewer", true) + state := &streamState{} + first := packets.Packet{Data: []byte{0x65}, IsKeyFrame: true, Time: 1} + processVideoPacket(first, state, b, models.Config{}) + processVideoPacket(packets.Packet{Data: []byte{0x41}, Time: 34}, state, b, models.Config{}) + if len(pt.samples) != 1 || !bytes.Equal((<-pt.samples).Data, first.Data) { + t.Fatal("buffered IDR lost its keyframe flag") + } + + b.SetPeerConnected("viewer", false) + b.SetPeerConnected("viewer", true) + writeFinalSamples(state, b, nil) + if len(pt.samples) != 0 { + t.Fatal("final dependent frame was mislabeled as an IDR") + } + state.lastVideoSample = &pionMedia.Sample{Data: first.Data} + state.lastVideoKeyframe = true + writeFinalSamples(state, b, nil) + if len(pt.samples) != 1 { + t.Fatal("final IDR lost its keyframe flag") + } +} + +func TestKeyframeRequestsCoalesceAndRetryUntilIDR(t *testing.T) { + b, _ := testVideoBroadcaster(t) + calls := make(chan time.Time, 10) + var count atomic.Int32 + b.requestKeyframe = func() error { + calls <- time.Now() + if count.Add(1) == 1 { + return errors.New("camera is not ready yet") + } + b.WriteSample(pionMedia.Sample{}, true) + return nil + } + for _, id := range []string{"viewer", "second", "third"} { + if id != "viewer" { + b.mu.Lock() + b.peers[id] = bufferedTestPeer(peerSampleBuffer) + b.mu.Unlock() + } + b.SetPeerConnected(id, true) + } + var first time.Time + select { + case first = <-calls: + case <-time.After(2 * time.Second): + t.Fatal("connected viewer did not request a keyframe") + } + feedback, err := rtcp.Marshal([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: 123}}) + if err != nil { + t.Fatal(err) + } + for range 100 { + if err := b.HandleRTCP("viewer", feedback); err != nil { + t.Fatal(err) + } + } + select { + case second := <-calls: + if second.Sub(first) < keyframeRequestInterval-5*time.Millisecond { + t.Fatal("feedback bypassed the per-stream request rate limit") + } + case <-time.After(2 * time.Second): + t.Fatal("failed camera request was not retried") + } + select { + case <-calls: + t.Fatal("requests continued after all viewers received an IDR") + case <-time.After(keyframeRequestInterval + 100*time.Millisecond): + } +} + +func TestKeyframeRequestsStopOnDisconnectAndClose(t *testing.T) { + b, _ := testVideoBroadcaster(t) + var calls atomic.Int32 + b.requestKeyframe = func() error { calls.Add(1); return nil } + b.lastRequest = time.Now() + b.SetPeerConnected("viewer", true) + b.SetPeerConnected("viewer", false) + time.Sleep(keyframeRequestInterval + 50*time.Millisecond) + if calls.Load() != 0 { + t.Fatal("disconnected viewer requested a keyframe") + } + b.SetPeerConnected("viewer", true) + b.Close() + if _, err := b.AddPeer("closed"); !errors.Is(err, io.ErrClosedPipe) { + t.Fatalf("AddPeer() after Close() = %v, want closed pipe", err) + } +} + +func TestBroadcasterCloseWaitsForKeyframeRequest(t *testing.T) { + b, _ := testVideoBroadcaster(t) + started, release, closed := make(chan struct{}), make(chan struct{}), make(chan struct{}) + unblock := sync.OnceFunc(func() { close(release) }) + defer unblock() + b.requestKeyframe = func() error { + close(started) + <-release + return nil + } + b.SetPeerConnected("viewer", true) + select { + case <-started: + case <-time.After(2 * time.Second): + t.Fatal("request did not start") + } + go func() { b.Close(); close(closed) }() + select { + case <-closed: + t.Fatal("Close returned before the in-flight camera request completed") + case <-time.After(30 * time.Millisecond): + } + unblock() + select { + case <-closed: + case <-time.After(2 * time.Second): + t.Fatal("Close did not finish") + } +} + +func TestConcurrentVideoFeedbackAndPeerLifecycle(t *testing.T) { + b := NewTrackBroadcaster(pionWebRTC.MimeTypeH264, "video", "test") + t.Cleanup(b.Close) + data, err := rtcp.Marshal([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: 123}}) + if err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + for _, id := range []string{"first", "second", "third"} { + if _, err := b.AddPeer(id); err != nil { + t.Fatal(err) + } + wg.Add(1) + go func() { + defer wg.Done() + for j := range 200 { + b.SetPeerConnected(id, j%5 != 0) + b.WriteSample(pionMedia.Sample{Data: []byte{0, 0, 0, 1, 0x65}, Duration: time.Millisecond}, j%10 == 0) + if err := b.HandleRTCP(id, data); err != nil { + t.Error(err) + } + } + b.RemovePeer(id) + }() + } + wg.Wait() +} + +type keyframeTestSource struct { + capture.RTSPClient + request func() error +} + +func (s keyframeTestSource) RequestKeyframe() error { return s.request() } + +func TestVideoBroadcastersRequestTheirOwnSource(t *testing.T) { + streams := []packets.Stream{{Name: "H264", IsVideo: true}} + var mainCalls, subCalls atomic.Int32 + main := NewVideoBroadcaster(streams, keyframeTestSource{request: func() error { mainCalls.Add(1); return nil }}) + sub := NewVideoBroadcaster(streams, keyframeTestSource{request: func() error { subCalls.Add(1); return nil }}) + t.Cleanup(main.Close) + t.Cleanup(sub.Close) + if err := main.requestKeyframe(); err != nil { + t.Fatal(err) + } + if mainCalls.Load() != 1 || subCalls.Load() != 0 { + t.Fatal("main feedback reached the wrong source") + } + if err := sub.requestKeyframe(); err != nil { + t.Fatal(err) + } + if mainCalls.Load() != 1 || subCalls.Load() != 1 { + t.Fatal("sub feedback reached the wrong source") + } +} diff --git a/machinery/src/webrtc/main.go b/machinery/src/webrtc/main.go index 93b7831..bbcdd0b 100644 --- a/machinery/src/webrtc/main.go +++ b/machinery/src/webrtc/main.go @@ -18,7 +18,6 @@ import ( mqtt "github.com/eclipse/paho.mqtt.golang" "github.com/pion/interceptor" - "github.com/pion/interceptor/pkg/intervalpli" pionWebRTC "github.com/pion/webrtc/v4" pionMedia "github.com/pion/webrtc/v4/pkg/media" ) @@ -429,16 +428,6 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati panic(err) } - // Register a intervalpli factory - // This interceptor sends a PLI every 3 seconds. A PLI causes a video keyframe to be generated by the sender. - // This makes our video seekable and more error resilent, but at a cost of lower picture quality and higher bitrates - // A real world application should process incoming RTCP packets from viewers and forward them to senders - intervalPliFactory, err := intervalpli.NewReceiverInterceptor() - if err != nil { - panic(err) - } - interceptorRegistry.Add(intervalPliFactory) - api := pionWebRTC.NewAPI( pionWebRTC.WithMediaEngine(mediaEngine), pionWebRTC.WithInterceptorRegistry(interceptorRegistry), @@ -519,9 +508,16 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati case <-ctx.Done(): return default: - if _, _, rtcpErr := videoSender.Read(rtcpBuf); rtcpErr != nil { + n, _, rtcpErr := videoSender.Read(rtcpBuf) + if rtcpErr != nil { + if ctx.Err() == nil && rtcpErr != io.EOF && rtcpErr != io.ErrClosedPipe { + log.WithError(rtcpErr).Warn("WebRTC video RTCP reader failed") + } return } + if err := videoBroadcaster.HandleRTCP(sessionKey, rtcpBuf[:n]); err != nil { + log.WithError(err).Warn("Ignoring malformed WebRTC video RTCP feedback") + } } } }() @@ -580,6 +576,9 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati switch connectionState { case pionWebRTC.PeerConnectionStateDisconnected: + if videoBroadcaster != nil { + videoBroadcaster.SetPeerConnected(sessionKey, false) + } // Disconnected is a transient state that can recover. // Start a grace period timer; if we don't recover, then cleanup. wrapper.disconnectMu.Lock() @@ -619,6 +618,9 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati cleanupPeerConnection(sessionKey, wrapper) case pionWebRTC.PeerConnectionStateConnected: + if videoBroadcaster != nil { + videoBroadcaster.SetPeerConnected(sessionKey, true) + } // Cancel any pending disconnect timer — connection recovered wrapper.disconnectMu.Lock() if wrapper.disconnectTimer != nil { @@ -760,11 +762,17 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati } } -func NewVideoBroadcaster(streams []packets.Stream) *TrackBroadcaster { +func NewVideoBroadcaster(streams []packets.Stream, rtspClient capture.RTSPClient) *TrackBroadcaster { // Verify H264 is available (same check as NewVideoTrack) for _, s := range streams { if s.Name == "H264" { - return NewTrackBroadcaster(pionWebRTC.MimeTypeH264, "video", trackStreamID) + b := NewTrackBroadcaster(pionWebRTC.MimeTypeH264, "video", trackStreamID) + if requester, ok := rtspClient.(capture.KeyframeRequester); ok { + b.requestKeyframe = requester.RequestKeyframe + } else { + log.Warn("RTSP source does not support keyframe requests; waiting for periodic IDRs") + } + return b } } log.Error("webrtc.main.NewVideoBroadcaster(): no H264 stream found") @@ -858,18 +866,19 @@ func NewAudioTrack(streams []packets.Stream) *pionWebRTC.TrackLocalStaticSample // streamState holds state information for the streaming process type streamState struct { - lastKeepAlive int64 - peerCount int64 - start bool - catchingUp bool - receivedKeyFrame bool - lastAudioSample *pionMedia.Sample - lastVideoSample *pionMedia.Sample - audioPacketsSeen int64 - aacPacketsSeen int64 - audioSamplesSent int64 - aacNoOutput int64 - aacErrors int64 + lastKeepAlive int64 + peerCount int64 + start bool + catchingUp bool + receivedKeyFrame bool + lastAudioSample *pionMedia.Sample + lastVideoSample *pionMedia.Sample + lastVideoKeyframe bool + audioPacketsSeen int64 + aacPacketsSeen int64 + audioSamplesSent int64 + aacNoOutput int64 + aacErrors int64 } // codecSupport tracks which codecs are available in the stream @@ -943,11 +952,11 @@ func updateStreamState(communication *models.Communication, state *streamState) // writeFinalSamples writes any remaining buffered samples func writeFinalSamples(state *streamState, videoBroadcaster, audioBroadcaster *TrackBroadcaster) { if state.lastVideoSample != nil && videoBroadcaster != nil { - videoBroadcaster.WriteSample(*state.lastVideoSample) + videoBroadcaster.WriteSample(*state.lastVideoSample, state.lastVideoKeyframe) } if state.lastAudioSample != nil && audioBroadcaster != nil { - audioBroadcaster.WriteSample(*state.lastAudioSample) + audioBroadcaster.WriteSample(*state.lastAudioSample, false) } } @@ -1011,10 +1020,11 @@ func processVideoPacket(pkt packets.Packet, state *streamState, videoBroadcaster if state.lastVideoSample != nil { state.lastVideoSample.Duration = sampleDuration(pkt, state.lastVideoSample.PacketTimestamp, 33*time.Millisecond) - videoBroadcaster.WriteSample(*state.lastVideoSample) + videoBroadcaster.WriteSample(*state.lastVideoSample, state.lastVideoKeyframe) } state.lastVideoSample = &sample + state.lastVideoKeyframe = pkt.IsKeyFrame } // processAudioPacket processes an audio packet and writes samples to the broadcaster. @@ -1097,7 +1107,7 @@ func processAudioPacket(pkt packets.Packet, state *streamState, audioBroadcaster "samples_queued": state.audioSamplesSent, }).Debug("WebRTC audio sample queued") } - audioBroadcaster.WriteSample(*state.lastAudioSample) + audioBroadcaster.WriteSample(*state.lastAudioSample, false) } state.lastAudioSample = &sample diff --git a/machinery/src/webrtc/main_test.go b/machinery/src/webrtc/main_test.go index b4f86a3..0472364 100644 --- a/machinery/src/webrtc/main_test.go +++ b/machinery/src/webrtc/main_test.go @@ -1,6 +1,18 @@ package webrtc -import "testing" +import ( + "encoding/base64" + "encoding/json" + "sync" + "testing" + "time" + + "github.com/kerberos-io/agent/machinery/src/models" + "github.com/kerberos-io/agent/machinery/src/packets" + "github.com/pion/rtcp" + pionWebRTC "github.com/pion/webrtc/v4" + pionMedia "github.com/pion/webrtc/v4/pkg/media" +) func TestBuildICEServersOmitsEmptyURLs(t *testing.T) { webRTC := CreateWebRTC("camera", []string{""}, []string{""}, "", "") @@ -26,6 +38,7 @@ func TestBuildICEServersIncludesConfiguredURLs(t *testing.T) { if len(iceServers) != 2 { t.Fatalf("buildICEServers() returned %d servers, want 2", len(iceServers)) } + if got := iceServers[0].URLs[0]; got != "stun:turn-fra1.kerberos.io:3478" { t.Fatalf("STUN URL = %q, want trimmed URL", got) } @@ -36,3 +49,138 @@ func TestBuildICEServersIncludesConfiguredURLs(t *testing.T) { t.Fatal("TURN credentials were not preserved") } } + +func TestWebRTCConnectionRequestsStartupAndRecoveryKeyframes(t *testing.T) { + viewer, err := pionWebRTC.NewPeerConnection(pionWebRTC.Configuration{}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := viewer.Close(); err != nil { + t.Error(err) + } + }) + if _, err := viewer.AddTransceiverFromKind(pionWebRTC.RTPCodecTypeVideo, pionWebRTC.RTPTransceiverInit{ + Direction: pionWebRTC.RTPTransceiverDirectionRecvonly, + }); err != nil { + t.Fatal(err) + } + tracks := make(chan *pionWebRTC.TrackRemote, 1) + viewer.OnTrack(func(track *pionWebRTC.TrackRemote, _ *pionWebRTC.RTPReceiver) { + tracks <- track + }) + offer, err := viewer.CreateOffer(nil) + if err != nil { + t.Fatal(err) + } + gathered := pionWebRTC.GatheringCompletePromise(viewer) + if err := viewer.SetLocalDescription(offer); err != nil { + t.Fatal(err) + } + select { + case <-gathered: + case <-time.After(5 * time.Second): + t.Fatal("viewer ICE gathering timed out") + } + + requests := make(chan time.Time, 10) + b := NewVideoBroadcaster([]packets.Stream{{Name: "H264", IsVideo: true}}, keyframeTestSource{ + request: func() error { requests <- time.Now(); return nil }, + }) + t.Cleanup(b.Close) + const sessionKey = "unit-camera/startup-recovery" + t.Cleanup(func() { + globalConnectionManager.mu.RLock() + wrapper := globalConnectionManager.peerConnections[sessionKey] + globalConnectionManager.mu.RUnlock() + if wrapper != nil { + cleanupPeerConnection(sessionKey, wrapper) + } + }) + var signalingMu sync.Mutex + var pending []pionWebRTC.ICECandidateInit + signaling := &models.LiveHDSignalingCallbacks{ + SendCandidate: func(_ string, data string) error { + var candidate pionWebRTC.ICECandidateInit + if err := json.Unmarshal([]byte(data), &candidate); err != nil { + return err + } + signalingMu.Lock() + defer signalingMu.Unlock() + if viewer.RemoteDescription() == nil { + pending = append(pending, candidate) + return nil + } + return viewer.AddICECandidate(candidate) + }, + SendAnswer: func(_ string, data string) error { + answer, err := base64.StdEncoding.DecodeString(data) + if err != nil { + return err + } + signalingMu.Lock() + defer signalingMu.Unlock() + if err := viewer.SetRemoteDescription(pionWebRTC.SessionDescription{ + Type: pionWebRTC.SDPTypeAnswer, SDP: string(answer), + }); err != nil { + return err + } + for _, candidate := range pending { + if err := viewer.AddICECandidate(candidate); err != nil { + return err + } + } + pending = nil + return nil + }, + } + InitializeWebRTCConnection( + &models.Configuration{Config: models.Config{Key: "unit-camera"}}, + &models.Communication{}, nil, b, nil, + models.LiveHDHandshake{ + Signaling: signaling, + Payload: models.RequestHDStreamPayload{ + SessionID: "startup-recovery", + SessionDescription: base64.StdEncoding.EncodeToString([]byte(viewer.LocalDescription().SDP)), + }, + }, + ) + var firstRequest time.Time + select { + case firstRequest = <-requests: + case <-time.After(5 * time.Second): + t.Fatal("connected transport did not request a startup keyframe") + } + b.WriteSample(pionMedia.Sample{Data: []byte{0, 0, 0, 1, 0x41, 0x80}, Duration: time.Second / 30}, false) + select { + case <-tracks: + t.Fatal("viewer received dependent video before its startup IDR") + case <-time.After(100 * time.Millisecond): + } + b.WriteSample(pionMedia.Sample{Data: []byte{0, 0, 0, 1, 0x65, 0x88, 0x84}, Duration: time.Second / 30}, true) + var track *pionWebRTC.TrackRemote + select { + case track = <-tracks: + case <-time.After(3 * time.Second): + t.Fatal("viewer did not receive the requested startup IDR") + } + + for _, feedback := range []rtcp.Packet{ + &rtcp.PictureLossIndication{SenderSSRC: 42, MediaSSRC: uint32(track.SSRC())}, + &rtcp.FullIntraRequest{SenderSSRC: 42, FIR: []rtcp.FIREntry{{SSRC: uint32(track.SSRC()), SequenceNumber: 1}}}, + } { + if err := viewer.WriteRTCP([]rtcp.Packet{feedback}); err != nil { + t.Fatal(err) + } + select { + case at := <-requests: + if at.Sub(firstRequest) < keyframeRequestInterval-5*time.Millisecond { + t.Fatal("real viewer feedback bypassed keyframe request throttling") + } + firstRequest = at + case <-time.After(3 * time.Second): + t.Fatalf("real %T feedback did not request a recovery keyframe", feedback) + } + b.WriteSample(pionMedia.Sample{Data: []byte{0, 0, 0, 1, 0x65, 0x88, 0x84}, Duration: time.Second / 30}, true) + } +} diff --git a/machinery/src/webrtc/rtp_clock_test.go b/machinery/src/webrtc/rtp_clock_test.go new file mode 100644 index 0000000..14a6913 --- /dev/null +++ b/machinery/src/webrtc/rtp_clock_test.go @@ -0,0 +1,163 @@ +package webrtc + +import ( + "errors" + "testing" + "time" + + "github.com/pion/interceptor" + "github.com/pion/rtcp" + "github.com/pion/rtp" + pionWebRTC "github.com/pion/webrtc/v4" + pionMedia "github.com/pion/webrtc/v4/pkg/media" +) + +type clockTestWriter struct { + timestamps []uint32 +} + +func (w *clockTestWriter) WriteRTP(header *rtp.Header, payload []byte) (int, error) { + w.timestamps = append(w.timestamps, header.Timestamp) + return len(payload), nil +} + +func (*clockTestWriter) Write([]byte) (int, error) { + return 0, errors.New("unexpected raw RTP write") +} + +type clockTestContext struct { + writer *clockTestWriter +} + +func (clockTestContext) ID() string { return "clock-test" } +func (clockTestContext) SSRC() pionWebRTC.SSRC { return 123 } +func (clockTestContext) SSRCRetransmission() pionWebRTC.SSRC { return 0 } +func (clockTestContext) SSRCForwardErrorCorrection() pionWebRTC.SSRC { return 0 } +func (clockTestContext) RTCPReader() interceptor.RTCPReader { return nil } +func (clockTestContext) HeaderExtensions() []pionWebRTC.RTPHeaderExtensionParameter { + return nil +} +func (c clockTestContext) WriteStream() pionWebRTC.TrackLocalWriter { return c.writer } +func (clockTestContext) CodecParameters() []pionWebRTC.RTPCodecParameters { + return []pionWebRTC.RTPCodecParameters{{ + RTPCodecCapability: pionWebRTC.RTPCodecCapability{MimeType: pionWebRTC.MimeTypeH264, ClockRate: 90000}, + PayloadType: 96, + }} +} + +func clockTestPeer(t *testing.T) (*TrackBroadcaster, *peerTrack, *clockTestWriter) { + t.Helper() + b, pt := testVideoBroadcaster(t) + pt.samples = make(chan peerSample, 2) + track, err := pionWebRTC.NewTrackLocalStaticSample( + pionWebRTC.RTPCodecCapability{MimeType: pionWebRTC.MimeTypeH264}, + "video", "clock-test", pionWebRTC.WithRTPTimestamp(1000), + ) + if err != nil { + t.Fatal(err) + } + writer := &clockTestWriter{} + if _, err := track.Bind(clockTestContext{writer: writer}); err != nil { + t.Fatal(err) + } + pt.track = track + b.SetPeerConnected("viewer", true) + return b, pt, writer +} + +func clockVideoSample(duration time.Duration) pionMedia.Sample { + return pionMedia.Sample{ + Data: []byte{0, 0, 0, 1, 0x65, 0x88, 0x84}, + Duration: duration, + } +} + +func writeClockSample(t *testing.T, pt *peerTrack) { + t.Helper() + select { + case sample := <-pt.samples: + if err := writePeerSample(pt.track, sample); err != nil { + t.Fatal(err) + } + default: + t.Fatal("expected queued video sample") + } +} + +func TestVideoRecoveryPreservesRTPClock(t *testing.T) { + for _, scenario := range []string{"PLI", "FIR", "reconnect", "overflow"} { + t.Run(scenario, func(t *testing.T) { + b, pt, writer := clockTestPeer(t) + b.WriteSample(clockVideoSample(40*time.Millisecond), true) + writeClockSample(t, pt) + b.WriteSample(clockVideoSample(70*time.Millisecond), false) + + switch scenario { + case "PLI", "FIR": + var feedback rtcp.Packet = &rtcp.PictureLossIndication{MediaSSRC: 123} + if scenario == "FIR" { + feedback = &rtcp.FullIntraRequest{FIR: []rtcp.FIREntry{{SSRC: 123, SequenceNumber: 1}}} + } + data, err := rtcp.Marshal([]rtcp.Packet{feedback}) + if err != nil { + t.Fatal(err) + } + if err := b.HandleRTCP("viewer", data); err != nil { + t.Fatal(err) + } + b.WriteSample(clockVideoSample(30*time.Millisecond), false) + case "reconnect": + b.SetPeerConnected("viewer", false) + b.WriteSample(clockVideoSample(30*time.Millisecond), false) + b.SetPeerConnected("viewer", true) + case "overflow": + b.WriteSample(clockVideoSample(30*time.Millisecond), false) + } + b.WriteSample(clockVideoSample(1900*time.Millisecond), false) + b.WriteSample(clockVideoSample(40*time.Millisecond), true) + writeClockSample(t, pt) + + if len(writer.timestamps) != 2 { + t.Fatalf("sent %d RTP packets, want only the two IDRs", len(writer.timestamps)) + } + // The initial 40 ms frame plus 2000 ms of discarded video at 90 kHz. + if got := writer.timestamps[1] - writer.timestamps[0]; got != 183600 { + t.Fatalf("recovery RTP timestamp delta = %d, want 183600", got) + } + }) + } +} + +func TestOverflowIDRPreservesDiscardedRTPDuration(t *testing.T) { + b, pt, writer := clockTestPeer(t) + b.WriteSample(clockVideoSample(40*time.Millisecond), true) + writeClockSample(t, pt) + b.WriteSample(clockVideoSample(70*time.Millisecond), false) + b.WriteSample(clockVideoSample(30*time.Millisecond), false) + b.WriteSample(clockVideoSample(40*time.Millisecond), true) + writeClockSample(t, pt) + if len(writer.timestamps) != 2 || writer.timestamps[1]-writer.timestamps[0] != 12600 { + t.Fatalf("overflow IDR compressed the RTP clock: %v", writer.timestamps) + } +} + +func TestRepeatedFeedbackPreservesAlreadyQueuedSkippedTime(t *testing.T) { + b, pt, writer := clockTestPeer(t) + b.WriteSample(clockVideoSample(40*time.Millisecond), true) + writeClockSample(t, pt) + data, err := rtcp.Marshal([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: 123}}) + if err != nil { + t.Fatal(err) + } + for range 2 { + if err := b.HandleRTCP("viewer", data); err != nil { + t.Fatal(err) + } + b.WriteSample(clockVideoSample(time.Second), false) + b.WriteSample(clockVideoSample(40*time.Millisecond), true) + } + writeClockSample(t, pt) + if len(writer.timestamps) != 2 || writer.timestamps[1]-writer.timestamps[0] != 187200 { + t.Fatalf("repeated feedback lost queued skipped time: %v", writer.timestamps) + } +}