mirror of
https://github.com/kerberos-io/agent.git
synced 2026-11-18 22:20:50 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bc8bc84cae | ||
|
|
8c21638ff2 | ||
|
|
ef81ce5b32 | ||
|
|
8016b3ba93 | ||
|
|
4dc6fe7e18 |
@@ -26,3 +26,28 @@ Kerberos Open source (v3) is a cutting edge video surveillance management system
|
||||
## Security reporting
|
||||
|
||||
For sensitive vulnerabilities, use private disclosure channels documented in [../SECURITY.md](../SECURITY.md).
|
||||
|
||||
## WebRTC startup and keyframe recovery
|
||||
|
||||
Each H264 viewer waits until its WebRTC transport is connected and a fresh IDR
|
||||
arrives before receiving video. Reconnects, viewer PLI/FIR feedback, and a full
|
||||
per-viewer sample buffer reset that viewer's keyframe gate without stalling
|
||||
other viewers. Audio continues to use the existing independent sample queues.
|
||||
Each viewer's skipped and discarded video durations advance its RTP clock before
|
||||
recovery, without sending filler packets, so an IDR wait does not compress the
|
||||
video timeline relative to audio.
|
||||
|
||||
The main and sub streams request keyframes from their own RTSP sources. Viewer
|
||||
PLI and FIR feedback is translated into an RTSP PLI using the camera session's
|
||||
SSRCs, never forwarded with the browser's SSRCs. Requests are coalesced and
|
||||
limited to one per second per broadcaster, retrying while connected viewers
|
||||
still need an IDR. Requests stop once all waiting viewers receive an IDR or
|
||||
the broadcaster closes.
|
||||
|
||||
Camera support for RTCP PLI varies. Unsupported sources, cameras that ignore
|
||||
feedback, and failed requests fall back to the next periodic IDR; failures are
|
||||
logged as `keyframe_request_failed`. No encoder settings are changed. A camera
|
||||
with a long GOP can therefore still start slowly; configure a shorter IDR
|
||||
interval (for example 1-2 seconds) when camera feedback is unavailable, accounting
|
||||
for the bandwidth tradeoff. An isolated cached IDR must not be replayed followed
|
||||
by arbitrary live P-frames: the dependent reference chain would be incomplete.
|
||||
|
||||
+1
-1
@@ -30,6 +30,7 @@ require (
|
||||
github.com/moq-dev/moq-go v0.5.7
|
||||
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
|
||||
github.com/pion/interceptor v0.1.47
|
||||
github.com/pion/rtcp v1.2.17
|
||||
github.com/pion/rtp v1.10.5
|
||||
github.com/pion/webrtc/v4 v4.2.18
|
||||
github.com/sirupsen/logrus v1.9.3
|
||||
@@ -104,7 +105,6 @@ require (
|
||||
github.com/pion/logging v0.2.4 // indirect
|
||||
github.com/pion/mdns/v2 v2.1.0 // indirect
|
||||
github.com/pion/randutil v0.1.0 // indirect
|
||||
github.com/pion/rtcp v1.2.17 // indirect
|
||||
github.com/pion/sctp v1.11.1 // indirect
|
||||
github.com/pion/sdp/v3 v3.0.19 // indirect
|
||||
github.com/pion/srtp/v3 v3.0.12 // indirect
|
||||
|
||||
@@ -101,6 +101,8 @@ type Golibrtsp struct {
|
||||
Client gortsplib.Client
|
||||
clientStarted bool
|
||||
VideoDecoderMutex *sync.Mutex
|
||||
keyframeMu sync.Mutex
|
||||
keyframeReady bool
|
||||
|
||||
VideoH264Index int8
|
||||
VideoH264Media *description.Media
|
||||
@@ -813,6 +815,10 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
|
||||
}
|
||||
}
|
||||
|
||||
g.keyframeMu.Lock()
|
||||
g.keyframeReady = err == nil && g.VideoH264Media != nil && g.VideoH264Decoder != nil
|
||||
g.keyframeMu.Unlock()
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1713,6 +1719,10 @@ func (g *Golibrtsp) Close(ctxOtel context.Context) error {
|
||||
_, span := tracer.Start(ctxOtel, "Close")
|
||||
defer span.End()
|
||||
|
||||
g.keyframeMu.Lock()
|
||||
g.keyframeReady = false
|
||||
g.keyframeMu.Unlock()
|
||||
|
||||
if !g.clientStarted {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
package capture
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/pion/rtcp"
|
||||
)
|
||||
|
||||
func (g *Golibrtsp) RequestKeyframe() error {
|
||||
g.keyframeMu.Lock()
|
||||
defer g.keyframeMu.Unlock()
|
||||
|
||||
if !g.keyframeReady {
|
||||
return errors.New("RTSP H264 keyframe requests are unavailable")
|
||||
}
|
||||
|
||||
media, ok := g.Client.Stats().Session.Medias[g.VideoH264Media]
|
||||
if !ok {
|
||||
return errors.New("RTSP video media is not set up")
|
||||
}
|
||||
video, ok := media.Formats[g.VideoH264Forma]
|
||||
if !ok || video.InboundRTPPackets == 0 {
|
||||
return errors.New("RTSP video RTP has not arrived yet")
|
||||
}
|
||||
|
||||
// Browser and camera SSRCs differ: originate feedback on the RTSP session.
|
||||
if err := g.Client.WritePacketRTCP(g.VideoH264Media, &rtcp.PictureLossIndication{
|
||||
SenderSSRC: video.LocalSSRC,
|
||||
MediaSSRC: video.RemoteSSRC,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("request camera keyframe: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
package capture
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/bluenviron/gortsplib/v5"
|
||||
"github.com/bluenviron/gortsplib/v5/pkg/base"
|
||||
"github.com/bluenviron/gortsplib/v5/pkg/description"
|
||||
"github.com/bluenviron/gortsplib/v5/pkg/format"
|
||||
"github.com/pion/rtcp"
|
||||
"github.com/pion/rtp"
|
||||
)
|
||||
|
||||
type keyframeCamera struct {
|
||||
stream *gortsplib.ServerStream
|
||||
feedback chan rtcp.Packet
|
||||
}
|
||||
|
||||
func (c *keyframeCamera) OnDescribe(*gortsplib.ServerHandlerOnDescribeCtx) (*base.Response, *gortsplib.ServerStream, error) {
|
||||
return &base.Response{StatusCode: base.StatusOK}, c.stream, nil
|
||||
}
|
||||
|
||||
func (c *keyframeCamera) OnSetup(*gortsplib.ServerHandlerOnSetupCtx) (*base.Response, *gortsplib.ServerStream, error) {
|
||||
return &base.Response{StatusCode: base.StatusOK}, c.stream, nil
|
||||
}
|
||||
|
||||
func (c *keyframeCamera) OnPlay(ctx *gortsplib.ServerHandlerOnPlayCtx) (*base.Response, error) {
|
||||
ctx.Session.OnPacketRTCPAny(func(_ *description.Media, pkt rtcp.Packet) {
|
||||
c.feedback <- pkt
|
||||
})
|
||||
return &base.Response{StatusCode: base.StatusOK}, nil
|
||||
}
|
||||
|
||||
func connectedKeyframeCamera(t *testing.T) (*Golibrtsp, *keyframeCamera) {
|
||||
t.Helper()
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { listener.Close() })
|
||||
camera := &keyframeCamera{feedback: make(chan rtcp.Packet, 32)}
|
||||
server := &gortsplib.Server{
|
||||
RTSPAddress: "127.0.0.1:0",
|
||||
Handler: camera,
|
||||
Listen: func(string, string) (net.Listener, error) {
|
||||
return listener, nil
|
||||
},
|
||||
}
|
||||
if err := server.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(server.Close)
|
||||
media := &description.Media{
|
||||
Type: description.MediaTypeVideo,
|
||||
Formats: []format.Format{&format.H264{
|
||||
PayloadTyp: 96,
|
||||
PacketizationMode: 1,
|
||||
}},
|
||||
}
|
||||
camera.stream = &gortsplib.ServerStream{
|
||||
Server: server,
|
||||
Desc: &description.Session{Medias: []*description.Media{media}},
|
||||
}
|
||||
if err := camera.stream.Initialize(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(camera.stream.Close)
|
||||
|
||||
client := &Golibrtsp{Url: "rtsp://" + listener.Addr().String() + "/camera"}
|
||||
t.Cleanup(func() {
|
||||
if err := client.Close(context.Background()); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
})
|
||||
if err := client.Connect(context.Background(), context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := client.RequestKeyframe(); err == nil {
|
||||
t.Fatal("requested a keyframe before the camera SSRC was known")
|
||||
}
|
||||
received := make(chan struct{}, 1)
|
||||
client.Client.OnPacketRTP(client.VideoH264Media, client.VideoH264Forma, func(*rtp.Packet) {
|
||||
received <- struct{}{}
|
||||
})
|
||||
if _, err := client.Client.Play(nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := camera.stream.WritePacketRTP(media, &rtp.Packet{
|
||||
Header: rtp.Header{
|
||||
Version: 2,
|
||||
PayloadType: 96,
|
||||
SSRC: 0x12345678,
|
||||
SequenceNumber: 1,
|
||||
Timestamp: 90000,
|
||||
Marker: true,
|
||||
},
|
||||
Payload: []byte{0x65, 0x88, 0x84},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-received:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("test camera RTP did not arrive")
|
||||
}
|
||||
return client, camera
|
||||
}
|
||||
|
||||
func TestRTSPKeyframeRequestUsesCameraSessionSSRCs(t *testing.T) {
|
||||
client, camera := connectedKeyframeCamera(t)
|
||||
want := client.Client.Stats().Session.Medias[client.VideoH264Media].Formats[client.VideoH264Forma]
|
||||
if err := client.RequestKeyframe(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case pkt := <-camera.feedback:
|
||||
pli, ok := pkt.(*rtcp.PictureLossIndication)
|
||||
if !ok {
|
||||
t.Fatalf("camera received %T, want PLI", pkt)
|
||||
}
|
||||
if pli.MediaSSRC != want.RemoteSSRC || pli.SenderSSRC != want.LocalSSRC {
|
||||
t.Fatalf("camera PLI SSRCs = sender %x media %x, want sender %x media %x",
|
||||
pli.SenderSSRC, pli.MediaSSRC, want.LocalSSRC, want.RemoteSSRC)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("camera did not receive RTCP PLI")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRTSPKeyframeRequestUnavailableBeforeConnectAndAfterClose(t *testing.T) {
|
||||
if err := (&Golibrtsp{}).RequestKeyframe(); err == nil {
|
||||
t.Fatal("uninitialized client accepted a keyframe request")
|
||||
}
|
||||
client, _ := connectedKeyframeCamera(t)
|
||||
if err := client.Close(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := client.RequestKeyframe(); err == nil {
|
||||
t.Fatal("closed client accepted a keyframe request")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRTSPKeyframeRequestConcurrentWithClose(t *testing.T) {
|
||||
client, _ := connectedKeyframeCamera(t)
|
||||
var wg sync.WaitGroup
|
||||
for range 4 {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for range 5 {
|
||||
// Requests may race with Close and legitimately become unavailable.
|
||||
_ = client.RequestKeyframe()
|
||||
}
|
||||
}()
|
||||
}
|
||||
if err := client.Close(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
@@ -72,6 +72,10 @@ func (c *Capture) ClearClients(main, sub, backchannel *Golibrtsp) {
|
||||
}
|
||||
}
|
||||
|
||||
type KeyframeRequester interface {
|
||||
RequestKeyframe() error
|
||||
}
|
||||
|
||||
// RTSPClient is a interface that abstracts the RTSP client implementation.
|
||||
type RTSPClient interface {
|
||||
// Connect to the RTSP server.
|
||||
|
||||
@@ -1285,7 +1285,7 @@ func HandleLiveStreamHD(
|
||||
// viewer can pick the resolution it needs per peer connection without
|
||||
// the agent re-negotiating the RTSP source.
|
||||
mainStreams, _ := rtspClient.GetStreams()
|
||||
mainVideoBroadcaster := webrtc.NewVideoBroadcaster(mainStreams)
|
||||
mainVideoBroadcaster := webrtc.NewVideoBroadcaster(mainStreams, rtspClient)
|
||||
mainAudioBroadcaster := webrtc.NewAudioBroadcaster(mainStreams)
|
||||
|
||||
if mainVideoBroadcaster == nil && mainAudioBroadcaster == nil {
|
||||
@@ -1316,7 +1316,7 @@ func HandleLiveStreamHD(
|
||||
var subAudioBroadcaster *webrtc.TrackBroadcaster
|
||||
if subStreamEnabled && rtspSubClient != nil && subQueue != nil {
|
||||
subStreams, _ := rtspSubClient.GetStreams()
|
||||
subVideoBroadcaster = webrtc.NewVideoBroadcaster(subStreams)
|
||||
subVideoBroadcaster = webrtc.NewVideoBroadcaster(subStreams, rtspSubClient)
|
||||
subAudioBroadcaster = webrtc.NewAudioBroadcaster(subStreams)
|
||||
defer func() {
|
||||
if subVideoBroadcaster != nil {
|
||||
@@ -1380,6 +1380,10 @@ func HandleRealtimeProcessing(processingCursor *packets.QueueCursor, configurati
|
||||
|
||||
// Check if we need to enable the realtime processing
|
||||
if config.RealtimeProcessing == "true" {
|
||||
if mqttClient == nil {
|
||||
log.Warn("cloud.RealtimeProcessing(): stopping as MQTT client is not configured.")
|
||||
return
|
||||
}
|
||||
|
||||
hubKey := ""
|
||||
if config.Cloud == "s3" && config.S3 != nil && config.S3.Publickey != "" {
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
package cloud
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/kerberos-io/agent/machinery/src/models"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/sirupsen/logrus/hooks/test"
|
||||
)
|
||||
|
||||
func TestHandleRealtimeProcessingWithoutMQTTClient(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
config models.Config
|
||||
wantWarning bool
|
||||
}{
|
||||
{
|
||||
name: "enabled without Hub credentials",
|
||||
config: models.Config{RealtimeProcessing: "true"},
|
||||
wantWarning: true,
|
||||
},
|
||||
{
|
||||
name: "disabled",
|
||||
config: models.Config{RealtimeProcessing: "false"},
|
||||
},
|
||||
{
|
||||
name: "offline",
|
||||
config: models.Config{RealtimeProcessing: "true", Offline: "true"},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
hook := test.NewGlobal()
|
||||
defer hook.Reset()
|
||||
|
||||
configuration := &models.Configuration{Config: tt.config}
|
||||
// No cursor or decoder should be accessed without an MQTT client.
|
||||
HandleRealtimeProcessing(nil, configuration, nil, nil, nil)
|
||||
|
||||
warnings := 0
|
||||
for _, entry := range hook.AllEntries() {
|
||||
if entry.Level == log.WarnLevel {
|
||||
warnings++
|
||||
if entry.Message != "cloud.RealtimeProcessing(): stopping as MQTT client is not configured." {
|
||||
t.Errorf("unexpected warning: %s", entry.Message)
|
||||
}
|
||||
}
|
||||
}
|
||||
wantWarnings := 0
|
||||
if tt.wantWarning {
|
||||
wantWarnings = 1
|
||||
}
|
||||
if warnings != wantWarnings {
|
||||
t.Fatalf("warnings = %d, want %d", warnings, wantWarnings)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,9 @@ package webrtc
|
||||
import (
|
||||
"io"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/pion/rtcp"
|
||||
pionWebRTC "github.com/pion/webrtc/v4"
|
||||
pionMedia "github.com/pion/webrtc/v4/pkg/media"
|
||||
log "github.com/sirupsen/logrus"
|
||||
@@ -12,14 +14,23 @@ import (
|
||||
const (
|
||||
// peerSampleBuffer controls how many samples can be buffered per peer before
|
||||
// dropping. Keeps slow peers from blocking the broadcaster.
|
||||
peerSampleBuffer = 60
|
||||
peerSampleBuffer = 60
|
||||
keyframeRequestInterval = time.Second
|
||||
)
|
||||
|
||||
type peerSample struct {
|
||||
pionMedia.Sample
|
||||
skippedDuration time.Duration
|
||||
}
|
||||
|
||||
// peerTrack is a per-peer track with its own non-blocking sample channel.
|
||||
type peerTrack struct {
|
||||
track *pionWebRTC.TrackLocalStaticSample
|
||||
samples chan pionMedia.Sample
|
||||
done chan struct{}
|
||||
track *pionWebRTC.TrackLocalStaticSample
|
||||
samples chan peerSample
|
||||
done chan struct{}
|
||||
connected bool
|
||||
needsKeyframe bool
|
||||
skippedDuration time.Duration
|
||||
}
|
||||
|
||||
// TrackBroadcaster fans out media samples to multiple peer-specific tracks
|
||||
@@ -27,11 +38,17 @@ type peerTrack struct {
|
||||
// goroutine that drains samples independently, so a slow/congested peer
|
||||
// cannot stall the others.
|
||||
type TrackBroadcaster struct {
|
||||
mu sync.RWMutex
|
||||
peers map[string]*peerTrack
|
||||
mimeType string
|
||||
id string
|
||||
streamID string
|
||||
mu sync.RWMutex
|
||||
peers map[string]*peerTrack
|
||||
mimeType string
|
||||
id string
|
||||
streamID string
|
||||
requestKeyframe func() error
|
||||
requestTimer *time.Timer
|
||||
requestInFlight bool
|
||||
lastRequest time.Time
|
||||
requests sync.WaitGroup
|
||||
closed bool
|
||||
}
|
||||
|
||||
// NewTrackBroadcaster creates a new broadcaster for either video or audio.
|
||||
@@ -57,12 +74,18 @@ func (b *TrackBroadcaster) AddPeer(sessionKey string) (*pionWebRTC.TrackLocalSta
|
||||
}
|
||||
|
||||
pt := &peerTrack{
|
||||
track: track,
|
||||
samples: make(chan pionMedia.Sample, peerSampleBuffer),
|
||||
done: make(chan struct{}),
|
||||
track: track,
|
||||
samples: make(chan peerSample, peerSampleBuffer),
|
||||
done: make(chan struct{}),
|
||||
connected: b.mimeType != pionWebRTC.MimeTypeH264,
|
||||
needsKeyframe: b.mimeType == pionWebRTC.MimeTypeH264,
|
||||
}
|
||||
|
||||
b.mu.Lock()
|
||||
if b.closed {
|
||||
b.mu.Unlock()
|
||||
return nil, io.ErrClosedPipe
|
||||
}
|
||||
b.peers[sessionKey] = pt
|
||||
b.mu.Unlock()
|
||||
|
||||
@@ -70,7 +93,7 @@ func (b *TrackBroadcaster) AddPeer(sessionKey string) (*pionWebRTC.TrackLocalSta
|
||||
go func() {
|
||||
defer close(pt.done)
|
||||
for sample := range pt.samples {
|
||||
if err := pt.track.WriteSample(sample); err != nil {
|
||||
if err := writePeerSample(pt.track, sample); err != nil {
|
||||
if err == io.ErrClosedPipe {
|
||||
return
|
||||
}
|
||||
@@ -91,6 +114,16 @@ func (b *TrackBroadcaster) AddPeer(sessionKey string) (*pionWebRTC.TrackLocalSta
|
||||
return track, nil
|
||||
}
|
||||
|
||||
func writePeerSample(track *pionWebRTC.TrackLocalStaticSample, sample peerSample) error {
|
||||
if sample.skippedDuration > 0 {
|
||||
// An empty sample advances Pion's RTP clock without sending a packet.
|
||||
if err := track.WriteSample(pionMedia.Sample{Duration: sample.skippedDuration}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return track.WriteSample(sample.Sample)
|
||||
}
|
||||
|
||||
// RemovePeer stops the writer goroutine and removes the peer.
|
||||
func (b *TrackBroadcaster) RemovePeer(sessionKey string) {
|
||||
b.mu.Lock()
|
||||
@@ -114,14 +147,37 @@ func (b *TrackBroadcaster) RemovePeer(sessionKey string) {
|
||||
// WriteSample fans out a sample to all connected peers without blocking.
|
||||
// If a peer's buffer is full (slow consumer), the sample is dropped for
|
||||
// that peer only — other peers are unaffected.
|
||||
func (b *TrackBroadcaster) WriteSample(sample pionMedia.Sample) {
|
||||
b.mu.RLock()
|
||||
defer b.mu.RUnlock()
|
||||
func (b *TrackBroadcaster) WriteSample(sample pionMedia.Sample, isKeyframe bool) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.closed {
|
||||
return
|
||||
}
|
||||
|
||||
for sessionKey, pt := range b.peers {
|
||||
video := b.mimeType == pionWebRTC.MimeTypeH264
|
||||
if video && (!pt.connected || (pt.needsKeyframe && !isKeyframe)) {
|
||||
pt.skippedDuration += sample.Duration
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case pt.samples <- sample:
|
||||
case pt.samples <- peerSample{Sample: sample, skippedDuration: pt.skippedDuration}:
|
||||
pt.skippedDuration = 0
|
||||
if video && isKeyframe {
|
||||
pt.needsKeyframe = false
|
||||
}
|
||||
default:
|
||||
if video {
|
||||
pt.needsKeyframe = true
|
||||
drainPeerSamples(pt)
|
||||
if isKeyframe {
|
||||
pt.samples <- peerSample{Sample: sample, skippedDuration: pt.skippedDuration}
|
||||
pt.skippedDuration = 0
|
||||
pt.needsKeyframe = false
|
||||
} else {
|
||||
pt.skippedDuration += sample.Duration
|
||||
}
|
||||
}
|
||||
log.WithFields(log.Fields{
|
||||
"component": "webrtc",
|
||||
"event": "slow_peer_sample_dropped",
|
||||
@@ -129,6 +185,106 @@ func (b *TrackBroadcaster) WriteSample(sample pionMedia.Sample) {
|
||||
}).Warn("Dropping WebRTC sample for slow peer")
|
||||
}
|
||||
}
|
||||
b.scheduleKeyframeRequestLocked()
|
||||
}
|
||||
|
||||
func drainPeerSamples(pt *peerTrack) {
|
||||
for {
|
||||
select {
|
||||
case sample := <-pt.samples:
|
||||
pt.skippedDuration += sample.skippedDuration + sample.Duration
|
||||
default:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (b *TrackBroadcaster) SetPeerConnected(sessionKey string, connected bool) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
|
||||
pt, ok := b.peers[sessionKey]
|
||||
if !ok || b.mimeType != pionWebRTC.MimeTypeH264 || pt.connected == connected {
|
||||
return
|
||||
}
|
||||
pt.connected = connected
|
||||
pt.needsKeyframe = true
|
||||
drainPeerSamples(pt)
|
||||
b.scheduleKeyframeRequestLocked()
|
||||
}
|
||||
|
||||
func (b *TrackBroadcaster) HandleRTCP(sessionKey string, data []byte) error {
|
||||
feedback, err := rtcp.Unmarshal(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, packet := range feedback {
|
||||
switch packet.(type) {
|
||||
case *rtcp.PictureLossIndication, *rtcp.FullIntraRequest:
|
||||
b.mu.Lock()
|
||||
if pt, ok := b.peers[sessionKey]; ok && pt.connected && b.mimeType == pionWebRTC.MimeTypeH264 {
|
||||
pt.needsKeyframe = true
|
||||
drainPeerSamples(pt)
|
||||
b.scheduleKeyframeRequestLocked()
|
||||
}
|
||||
b.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *TrackBroadcaster) waitingForKeyframeLocked() bool {
|
||||
for _, pt := range b.peers {
|
||||
if pt.connected && pt.needsKeyframe {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (b *TrackBroadcaster) scheduleKeyframeRequestLocked() {
|
||||
if b.closed || b.requestKeyframe == nil || b.requestInFlight || !b.waitingForKeyframeLocked() {
|
||||
return
|
||||
}
|
||||
if b.requestTimer == nil {
|
||||
delay := time.Until(b.lastRequest.Add(keyframeRequestInterval))
|
||||
if delay < 0 {
|
||||
delay = 0
|
||||
}
|
||||
b.requestTimer = time.AfterFunc(delay, b.sendKeyframeRequest)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *TrackBroadcaster) sendKeyframeRequest() {
|
||||
b.mu.Lock()
|
||||
b.requestTimer = nil
|
||||
if b.closed || !b.waitingForKeyframeLocked() {
|
||||
b.mu.Unlock()
|
||||
return
|
||||
}
|
||||
b.lastRequest = time.Now()
|
||||
b.requestInFlight = true
|
||||
b.requests.Add(1)
|
||||
b.mu.Unlock()
|
||||
defer b.requests.Done()
|
||||
|
||||
if err := b.requestKeyframe(); err != nil {
|
||||
log.WithError(err).WithFields(log.Fields{
|
||||
"component": "webrtc",
|
||||
"event": "keyframe_request_failed",
|
||||
}).Warn("Camera keyframe request failed; waiting for a periodic IDR")
|
||||
} else {
|
||||
log.WithFields(log.Fields{
|
||||
"component": "webrtc",
|
||||
"event": "keyframe_requested",
|
||||
}).Debug("Requested camera keyframe")
|
||||
}
|
||||
|
||||
b.mu.Lock()
|
||||
b.requestInFlight = false
|
||||
b.scheduleKeyframeRequestLocked()
|
||||
b.mu.Unlock()
|
||||
}
|
||||
|
||||
// PeerCount returns the current number of connected peers.
|
||||
@@ -141,11 +297,17 @@ func (b *TrackBroadcaster) PeerCount() int {
|
||||
// Close removes all peers and stops all writer goroutines.
|
||||
func (b *TrackBroadcaster) Close() {
|
||||
b.mu.Lock()
|
||||
b.closed = true
|
||||
if b.requestTimer != nil {
|
||||
b.requestTimer.Stop()
|
||||
b.requestTimer = nil
|
||||
}
|
||||
keys := make([]string, 0, len(b.peers))
|
||||
for k := range b.peers {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
b.mu.Unlock()
|
||||
b.requests.Wait()
|
||||
|
||||
for _, key := range keys {
|
||||
b.RemovePeer(key)
|
||||
|
||||
@@ -0,0 +1,360 @@
|
||||
package webrtc
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kerberos-io/agent/machinery/src/capture"
|
||||
"github.com/kerberos-io/agent/machinery/src/models"
|
||||
"github.com/kerberos-io/agent/machinery/src/packets"
|
||||
"github.com/pion/rtcp"
|
||||
pionWebRTC "github.com/pion/webrtc/v4"
|
||||
pionMedia "github.com/pion/webrtc/v4/pkg/media"
|
||||
)
|
||||
|
||||
func bufferedTestPeer(capacity int) *peerTrack {
|
||||
done := make(chan struct{})
|
||||
close(done)
|
||||
return &peerTrack{
|
||||
samples: make(chan peerSample, capacity),
|
||||
done: done,
|
||||
needsKeyframe: true,
|
||||
}
|
||||
}
|
||||
|
||||
func testVideoBroadcaster(t *testing.T) (*TrackBroadcaster, *peerTrack) {
|
||||
t.Helper()
|
||||
b := NewTrackBroadcaster(pionWebRTC.MimeTypeH264, "video", "test")
|
||||
pt := bufferedTestPeer(peerSampleBuffer)
|
||||
b.peers["viewer"] = pt
|
||||
t.Cleanup(b.Close)
|
||||
return b, pt
|
||||
}
|
||||
|
||||
func TestVideoPeerStartsAndReconnectsAtKeyframe(t *testing.T) {
|
||||
b, pt := testVideoBroadcaster(t)
|
||||
sample := pionMedia.Sample{Data: []byte{1}}
|
||||
b.WriteSample(sample, true)
|
||||
if len(pt.samples) != 0 {
|
||||
t.Fatal("sent a keyframe before the transport connected")
|
||||
}
|
||||
|
||||
b.SetPeerConnected("viewer", true)
|
||||
b.WriteSample(sample, false)
|
||||
if len(pt.samples) != 0 {
|
||||
t.Fatal("sent a dependent frame before the first IDR")
|
||||
}
|
||||
b.WriteSample(sample, true)
|
||||
b.WriteSample(sample, false)
|
||||
if len(pt.samples) != 2 || pt.needsKeyframe {
|
||||
t.Fatal("did not release video after the first IDR")
|
||||
}
|
||||
|
||||
b.SetPeerConnected("viewer", false)
|
||||
b.WriteSample(sample, true)
|
||||
if len(pt.samples) != 0 {
|
||||
t.Fatal("retained or sent video while disconnected")
|
||||
}
|
||||
b.SetPeerConnected("viewer", true)
|
||||
b.WriteSample(sample, false)
|
||||
if len(pt.samples) != 0 {
|
||||
t.Fatal("reconnected viewer resumed on a dependent frame")
|
||||
}
|
||||
b.WriteSample(sample, true)
|
||||
if len(pt.samples) != 1 {
|
||||
t.Fatal("reconnected viewer did not resume at an IDR")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLateViewerDoesNotResetExistingViewer(t *testing.T) {
|
||||
b, first := testVideoBroadcaster(t)
|
||||
b.SetPeerConnected("viewer", true)
|
||||
b.WriteSample(pionMedia.Sample{}, true)
|
||||
<-first.samples
|
||||
|
||||
late := bufferedTestPeer(peerSampleBuffer)
|
||||
b.peers["late"] = late
|
||||
b.SetPeerConnected("late", true)
|
||||
b.WriteSample(pionMedia.Sample{}, false)
|
||||
if len(first.samples) != 1 || len(late.samples) != 0 {
|
||||
t.Fatal("late viewer either interrupted an existing viewer or received a dependent frame")
|
||||
}
|
||||
b.WriteSample(pionMedia.Sample{}, true)
|
||||
if len(late.samples) != 1 {
|
||||
t.Fatal("late viewer did not receive the next IDR")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVideoFeedbackRequestsFreshKeyframe(t *testing.T) {
|
||||
for _, feedback := range []rtcp.Packet{
|
||||
&rtcp.PictureLossIndication{MediaSSRC: 123},
|
||||
&rtcp.FullIntraRequest{FIR: []rtcp.FIREntry{{SSRC: 123, SequenceNumber: 1}}},
|
||||
} {
|
||||
t.Run(fmt.Sprintf("%T", feedback), func(t *testing.T) {
|
||||
b, pt := testVideoBroadcaster(t)
|
||||
b.SetPeerConnected("viewer", true)
|
||||
b.WriteSample(pionMedia.Sample{}, true)
|
||||
b.WriteSample(pionMedia.Sample{}, false)
|
||||
|
||||
rr, err := rtcp.Marshal([]rtcp.Packet{&rtcp.ReceiverReport{}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := b.HandleRTCP("viewer", rr); err != nil || len(pt.samples) != 2 {
|
||||
t.Fatal("receiver report reset the video gate", err)
|
||||
}
|
||||
data, err := rtcp.Marshal([]rtcp.Packet{&rtcp.ReceiverReport{}, feedback})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := b.HandleRTCP("viewer", data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.WriteSample(pionMedia.Sample{}, false)
|
||||
if len(pt.samples) != 0 || !pt.needsKeyframe {
|
||||
t.Fatal("PLI/FIR did not discard stale queued frames and wait for an IDR")
|
||||
}
|
||||
b.WriteSample(pionMedia.Sample{}, true)
|
||||
if len(pt.samples) != 1 {
|
||||
t.Fatal("PLI/FIR recovery did not resume at an IDR")
|
||||
}
|
||||
if err := b.HandleRTCP("viewer", []byte{0}); err == nil {
|
||||
t.Fatal("malformed RTCP was accepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSlowVideoPeerRecoversWithoutBlockingOthers(t *testing.T) {
|
||||
for _, overflowIsKeyframe := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "dependent-frame-dropped", true: "new-IDR-replaces-backlog"}[overflowIsKeyframe], func(t *testing.T) {
|
||||
b, fast := testVideoBroadcaster(t)
|
||||
slow := bufferedTestPeer(2)
|
||||
b.peers["slow"] = slow
|
||||
b.SetPeerConnected("viewer", true)
|
||||
b.SetPeerConnected("slow", true)
|
||||
b.WriteSample(pionMedia.Sample{}, true)
|
||||
b.WriteSample(pionMedia.Sample{}, false)
|
||||
b.WriteSample(pionMedia.Sample{}, overflowIsKeyframe)
|
||||
if len(fast.samples) != 3 {
|
||||
t.Fatal("slow viewer interrupted another viewer")
|
||||
}
|
||||
if overflowIsKeyframe {
|
||||
if len(slow.samples) != 1 || slow.needsKeyframe {
|
||||
t.Fatal("fresh IDR did not replace the stale backlog")
|
||||
}
|
||||
} else {
|
||||
b.WriteSample(pionMedia.Sample{}, false)
|
||||
if len(slow.samples) != 0 || !slow.needsKeyframe {
|
||||
t.Fatal("slow viewer forwarded dependent frames after a drop")
|
||||
}
|
||||
b.WriteSample(pionMedia.Sample{}, true)
|
||||
if len(slow.samples) != 1 {
|
||||
t.Fatal("slow viewer did not recover at the next IDR")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAudioDoesNotWaitForVideoKeyframe(t *testing.T) {
|
||||
b := NewTrackBroadcaster(pionWebRTC.MimeTypePCMU, "audio", "test")
|
||||
pt := bufferedTestPeer(2)
|
||||
pt.connected = true
|
||||
pt.needsKeyframe = false
|
||||
b.peers["viewer"] = pt
|
||||
t.Cleanup(b.Close)
|
||||
|
||||
b.SetPeerConnected("viewer", false)
|
||||
for range 3 {
|
||||
b.WriteSample(pionMedia.Sample{}, false)
|
||||
}
|
||||
if len(pt.samples) != 2 {
|
||||
t.Fatal("audio queue behavior changed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVideoSampleBufferPreservesKeyframeFlag(t *testing.T) {
|
||||
b, pt := testVideoBroadcaster(t)
|
||||
b.SetPeerConnected("viewer", true)
|
||||
state := &streamState{}
|
||||
first := packets.Packet{Data: []byte{0x65}, IsKeyFrame: true, Time: 1}
|
||||
processVideoPacket(first, state, b, models.Config{})
|
||||
processVideoPacket(packets.Packet{Data: []byte{0x41}, Time: 34}, state, b, models.Config{})
|
||||
if len(pt.samples) != 1 || !bytes.Equal((<-pt.samples).Data, first.Data) {
|
||||
t.Fatal("buffered IDR lost its keyframe flag")
|
||||
}
|
||||
|
||||
b.SetPeerConnected("viewer", false)
|
||||
b.SetPeerConnected("viewer", true)
|
||||
writeFinalSamples(state, b, nil)
|
||||
if len(pt.samples) != 0 {
|
||||
t.Fatal("final dependent frame was mislabeled as an IDR")
|
||||
}
|
||||
state.lastVideoSample = &pionMedia.Sample{Data: first.Data}
|
||||
state.lastVideoKeyframe = true
|
||||
writeFinalSamples(state, b, nil)
|
||||
if len(pt.samples) != 1 {
|
||||
t.Fatal("final IDR lost its keyframe flag")
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeyframeRequestsCoalesceAndRetryUntilIDR(t *testing.T) {
|
||||
b, _ := testVideoBroadcaster(t)
|
||||
calls := make(chan time.Time, 10)
|
||||
var count atomic.Int32
|
||||
b.requestKeyframe = func() error {
|
||||
calls <- time.Now()
|
||||
if count.Add(1) == 1 {
|
||||
return errors.New("camera is not ready yet")
|
||||
}
|
||||
b.WriteSample(pionMedia.Sample{}, true)
|
||||
return nil
|
||||
}
|
||||
for _, id := range []string{"viewer", "second", "third"} {
|
||||
if id != "viewer" {
|
||||
b.mu.Lock()
|
||||
b.peers[id] = bufferedTestPeer(peerSampleBuffer)
|
||||
b.mu.Unlock()
|
||||
}
|
||||
b.SetPeerConnected(id, true)
|
||||
}
|
||||
var first time.Time
|
||||
select {
|
||||
case first = <-calls:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("connected viewer did not request a keyframe")
|
||||
}
|
||||
feedback, err := rtcp.Marshal([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: 123}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for range 100 {
|
||||
if err := b.HandleRTCP("viewer", feedback); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
select {
|
||||
case second := <-calls:
|
||||
if second.Sub(first) < keyframeRequestInterval-5*time.Millisecond {
|
||||
t.Fatal("feedback bypassed the per-stream request rate limit")
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("failed camera request was not retried")
|
||||
}
|
||||
select {
|
||||
case <-calls:
|
||||
t.Fatal("requests continued after all viewers received an IDR")
|
||||
case <-time.After(keyframeRequestInterval + 100*time.Millisecond):
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeyframeRequestsStopOnDisconnectAndClose(t *testing.T) {
|
||||
b, _ := testVideoBroadcaster(t)
|
||||
var calls atomic.Int32
|
||||
b.requestKeyframe = func() error { calls.Add(1); return nil }
|
||||
b.lastRequest = time.Now()
|
||||
b.SetPeerConnected("viewer", true)
|
||||
b.SetPeerConnected("viewer", false)
|
||||
time.Sleep(keyframeRequestInterval + 50*time.Millisecond)
|
||||
if calls.Load() != 0 {
|
||||
t.Fatal("disconnected viewer requested a keyframe")
|
||||
}
|
||||
b.SetPeerConnected("viewer", true)
|
||||
b.Close()
|
||||
if _, err := b.AddPeer("closed"); !errors.Is(err, io.ErrClosedPipe) {
|
||||
t.Fatalf("AddPeer() after Close() = %v, want closed pipe", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBroadcasterCloseWaitsForKeyframeRequest(t *testing.T) {
|
||||
b, _ := testVideoBroadcaster(t)
|
||||
started, release, closed := make(chan struct{}), make(chan struct{}), make(chan struct{})
|
||||
unblock := sync.OnceFunc(func() { close(release) })
|
||||
defer unblock()
|
||||
b.requestKeyframe = func() error {
|
||||
close(started)
|
||||
<-release
|
||||
return nil
|
||||
}
|
||||
b.SetPeerConnected("viewer", true)
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("request did not start")
|
||||
}
|
||||
go func() { b.Close(); close(closed) }()
|
||||
select {
|
||||
case <-closed:
|
||||
t.Fatal("Close returned before the in-flight camera request completed")
|
||||
case <-time.After(30 * time.Millisecond):
|
||||
}
|
||||
unblock()
|
||||
select {
|
||||
case <-closed:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("Close did not finish")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConcurrentVideoFeedbackAndPeerLifecycle(t *testing.T) {
|
||||
b := NewTrackBroadcaster(pionWebRTC.MimeTypeH264, "video", "test")
|
||||
t.Cleanup(b.Close)
|
||||
data, err := rtcp.Marshal([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: 123}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
for _, id := range []string{"first", "second", "third"} {
|
||||
if _, err := b.AddPeer(id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for j := range 200 {
|
||||
b.SetPeerConnected(id, j%5 != 0)
|
||||
b.WriteSample(pionMedia.Sample{Data: []byte{0, 0, 0, 1, 0x65}, Duration: time.Millisecond}, j%10 == 0)
|
||||
if err := b.HandleRTCP(id, data); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
b.RemovePeer(id)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
type keyframeTestSource struct {
|
||||
capture.RTSPClient
|
||||
request func() error
|
||||
}
|
||||
|
||||
func (s keyframeTestSource) RequestKeyframe() error { return s.request() }
|
||||
|
||||
func TestVideoBroadcastersRequestTheirOwnSource(t *testing.T) {
|
||||
streams := []packets.Stream{{Name: "H264", IsVideo: true}}
|
||||
var mainCalls, subCalls atomic.Int32
|
||||
main := NewVideoBroadcaster(streams, keyframeTestSource{request: func() error { mainCalls.Add(1); return nil }})
|
||||
sub := NewVideoBroadcaster(streams, keyframeTestSource{request: func() error { subCalls.Add(1); return nil }})
|
||||
t.Cleanup(main.Close)
|
||||
t.Cleanup(sub.Close)
|
||||
if err := main.requestKeyframe(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if mainCalls.Load() != 1 || subCalls.Load() != 0 {
|
||||
t.Fatal("main feedback reached the wrong source")
|
||||
}
|
||||
if err := sub.requestKeyframe(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if mainCalls.Load() != 1 || subCalls.Load() != 1 {
|
||||
t.Fatal("sub feedback reached the wrong source")
|
||||
}
|
||||
}
|
||||
@@ -18,7 +18,6 @@ import (
|
||||
|
||||
mqtt "github.com/eclipse/paho.mqtt.golang"
|
||||
"github.com/pion/interceptor"
|
||||
"github.com/pion/interceptor/pkg/intervalpli"
|
||||
pionWebRTC "github.com/pion/webrtc/v4"
|
||||
pionMedia "github.com/pion/webrtc/v4/pkg/media"
|
||||
)
|
||||
@@ -429,16 +428,6 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// Register a intervalpli factory
|
||||
// This interceptor sends a PLI every 3 seconds. A PLI causes a video keyframe to be generated by the sender.
|
||||
// This makes our video seekable and more error resilent, but at a cost of lower picture quality and higher bitrates
|
||||
// A real world application should process incoming RTCP packets from viewers and forward them to senders
|
||||
intervalPliFactory, err := intervalpli.NewReceiverInterceptor()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
interceptorRegistry.Add(intervalPliFactory)
|
||||
|
||||
api := pionWebRTC.NewAPI(
|
||||
pionWebRTC.WithMediaEngine(mediaEngine),
|
||||
pionWebRTC.WithInterceptorRegistry(interceptorRegistry),
|
||||
@@ -519,9 +508,16 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
if _, _, rtcpErr := videoSender.Read(rtcpBuf); rtcpErr != nil {
|
||||
n, _, rtcpErr := videoSender.Read(rtcpBuf)
|
||||
if rtcpErr != nil {
|
||||
if ctx.Err() == nil && rtcpErr != io.EOF && rtcpErr != io.ErrClosedPipe {
|
||||
log.WithError(rtcpErr).Warn("WebRTC video RTCP reader failed")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err := videoBroadcaster.HandleRTCP(sessionKey, rtcpBuf[:n]); err != nil {
|
||||
log.WithError(err).Warn("Ignoring malformed WebRTC video RTCP feedback")
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
@@ -580,6 +576,9 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
|
||||
|
||||
switch connectionState {
|
||||
case pionWebRTC.PeerConnectionStateDisconnected:
|
||||
if videoBroadcaster != nil {
|
||||
videoBroadcaster.SetPeerConnected(sessionKey, false)
|
||||
}
|
||||
// Disconnected is a transient state that can recover.
|
||||
// Start a grace period timer; if we don't recover, then cleanup.
|
||||
wrapper.disconnectMu.Lock()
|
||||
@@ -619,6 +618,9 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
|
||||
cleanupPeerConnection(sessionKey, wrapper)
|
||||
|
||||
case pionWebRTC.PeerConnectionStateConnected:
|
||||
if videoBroadcaster != nil {
|
||||
videoBroadcaster.SetPeerConnected(sessionKey, true)
|
||||
}
|
||||
// Cancel any pending disconnect timer — connection recovered
|
||||
wrapper.disconnectMu.Lock()
|
||||
if wrapper.disconnectTimer != nil {
|
||||
@@ -760,11 +762,17 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
|
||||
}
|
||||
}
|
||||
|
||||
func NewVideoBroadcaster(streams []packets.Stream) *TrackBroadcaster {
|
||||
func NewVideoBroadcaster(streams []packets.Stream, rtspClient capture.RTSPClient) *TrackBroadcaster {
|
||||
// Verify H264 is available (same check as NewVideoTrack)
|
||||
for _, s := range streams {
|
||||
if s.Name == "H264" {
|
||||
return NewTrackBroadcaster(pionWebRTC.MimeTypeH264, "video", trackStreamID)
|
||||
b := NewTrackBroadcaster(pionWebRTC.MimeTypeH264, "video", trackStreamID)
|
||||
if requester, ok := rtspClient.(capture.KeyframeRequester); ok {
|
||||
b.requestKeyframe = requester.RequestKeyframe
|
||||
} else {
|
||||
log.Warn("RTSP source does not support keyframe requests; waiting for periodic IDRs")
|
||||
}
|
||||
return b
|
||||
}
|
||||
}
|
||||
log.Error("webrtc.main.NewVideoBroadcaster(): no H264 stream found")
|
||||
@@ -858,18 +866,19 @@ func NewAudioTrack(streams []packets.Stream) *pionWebRTC.TrackLocalStaticSample
|
||||
|
||||
// streamState holds state information for the streaming process
|
||||
type streamState struct {
|
||||
lastKeepAlive int64
|
||||
peerCount int64
|
||||
start bool
|
||||
catchingUp bool
|
||||
receivedKeyFrame bool
|
||||
lastAudioSample *pionMedia.Sample
|
||||
lastVideoSample *pionMedia.Sample
|
||||
audioPacketsSeen int64
|
||||
aacPacketsSeen int64
|
||||
audioSamplesSent int64
|
||||
aacNoOutput int64
|
||||
aacErrors int64
|
||||
lastKeepAlive int64
|
||||
peerCount int64
|
||||
start bool
|
||||
catchingUp bool
|
||||
receivedKeyFrame bool
|
||||
lastAudioSample *pionMedia.Sample
|
||||
lastVideoSample *pionMedia.Sample
|
||||
lastVideoKeyframe bool
|
||||
audioPacketsSeen int64
|
||||
aacPacketsSeen int64
|
||||
audioSamplesSent int64
|
||||
aacNoOutput int64
|
||||
aacErrors int64
|
||||
}
|
||||
|
||||
// codecSupport tracks which codecs are available in the stream
|
||||
@@ -943,11 +952,11 @@ func updateStreamState(communication *models.Communication, state *streamState)
|
||||
// writeFinalSamples writes any remaining buffered samples
|
||||
func writeFinalSamples(state *streamState, videoBroadcaster, audioBroadcaster *TrackBroadcaster) {
|
||||
if state.lastVideoSample != nil && videoBroadcaster != nil {
|
||||
videoBroadcaster.WriteSample(*state.lastVideoSample)
|
||||
videoBroadcaster.WriteSample(*state.lastVideoSample, state.lastVideoKeyframe)
|
||||
}
|
||||
|
||||
if state.lastAudioSample != nil && audioBroadcaster != nil {
|
||||
audioBroadcaster.WriteSample(*state.lastAudioSample)
|
||||
audioBroadcaster.WriteSample(*state.lastAudioSample, false)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1011,10 +1020,11 @@ func processVideoPacket(pkt packets.Packet, state *streamState, videoBroadcaster
|
||||
|
||||
if state.lastVideoSample != nil {
|
||||
state.lastVideoSample.Duration = sampleDuration(pkt, state.lastVideoSample.PacketTimestamp, 33*time.Millisecond)
|
||||
videoBroadcaster.WriteSample(*state.lastVideoSample)
|
||||
videoBroadcaster.WriteSample(*state.lastVideoSample, state.lastVideoKeyframe)
|
||||
}
|
||||
|
||||
state.lastVideoSample = &sample
|
||||
state.lastVideoKeyframe = pkt.IsKeyFrame
|
||||
}
|
||||
|
||||
// processAudioPacket processes an audio packet and writes samples to the broadcaster.
|
||||
@@ -1097,7 +1107,7 @@ func processAudioPacket(pkt packets.Packet, state *streamState, audioBroadcaster
|
||||
"samples_queued": state.audioSamplesSent,
|
||||
}).Debug("WebRTC audio sample queued")
|
||||
}
|
||||
audioBroadcaster.WriteSample(*state.lastAudioSample)
|
||||
audioBroadcaster.WriteSample(*state.lastAudioSample, false)
|
||||
}
|
||||
|
||||
state.lastAudioSample = &sample
|
||||
|
||||
@@ -1,6 +1,18 @@
|
||||
package webrtc
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kerberos-io/agent/machinery/src/models"
|
||||
"github.com/kerberos-io/agent/machinery/src/packets"
|
||||
"github.com/pion/rtcp"
|
||||
pionWebRTC "github.com/pion/webrtc/v4"
|
||||
pionMedia "github.com/pion/webrtc/v4/pkg/media"
|
||||
)
|
||||
|
||||
func TestBuildICEServersOmitsEmptyURLs(t *testing.T) {
|
||||
webRTC := CreateWebRTC("camera", []string{""}, []string{""}, "", "")
|
||||
@@ -26,6 +38,7 @@ func TestBuildICEServersIncludesConfiguredURLs(t *testing.T) {
|
||||
if len(iceServers) != 2 {
|
||||
t.Fatalf("buildICEServers() returned %d servers, want 2", len(iceServers))
|
||||
}
|
||||
|
||||
if got := iceServers[0].URLs[0]; got != "stun:turn-fra1.kerberos.io:3478" {
|
||||
t.Fatalf("STUN URL = %q, want trimmed URL", got)
|
||||
}
|
||||
@@ -36,3 +49,138 @@ func TestBuildICEServersIncludesConfiguredURLs(t *testing.T) {
|
||||
t.Fatal("TURN credentials were not preserved")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebRTCConnectionRequestsStartupAndRecoveryKeyframes(t *testing.T) {
|
||||
viewer, err := pionWebRTC.NewPeerConnection(pionWebRTC.Configuration{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := viewer.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
})
|
||||
if _, err := viewer.AddTransceiverFromKind(pionWebRTC.RTPCodecTypeVideo, pionWebRTC.RTPTransceiverInit{
|
||||
Direction: pionWebRTC.RTPTransceiverDirectionRecvonly,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tracks := make(chan *pionWebRTC.TrackRemote, 1)
|
||||
viewer.OnTrack(func(track *pionWebRTC.TrackRemote, _ *pionWebRTC.RTPReceiver) {
|
||||
tracks <- track
|
||||
})
|
||||
offer, err := viewer.CreateOffer(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gathered := pionWebRTC.GatheringCompletePromise(viewer)
|
||||
if err := viewer.SetLocalDescription(offer); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-gathered:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("viewer ICE gathering timed out")
|
||||
}
|
||||
|
||||
requests := make(chan time.Time, 10)
|
||||
b := NewVideoBroadcaster([]packets.Stream{{Name: "H264", IsVideo: true}}, keyframeTestSource{
|
||||
request: func() error { requests <- time.Now(); return nil },
|
||||
})
|
||||
t.Cleanup(b.Close)
|
||||
const sessionKey = "unit-camera/startup-recovery"
|
||||
t.Cleanup(func() {
|
||||
globalConnectionManager.mu.RLock()
|
||||
wrapper := globalConnectionManager.peerConnections[sessionKey]
|
||||
globalConnectionManager.mu.RUnlock()
|
||||
if wrapper != nil {
|
||||
cleanupPeerConnection(sessionKey, wrapper)
|
||||
}
|
||||
})
|
||||
var signalingMu sync.Mutex
|
||||
var pending []pionWebRTC.ICECandidateInit
|
||||
signaling := &models.LiveHDSignalingCallbacks{
|
||||
SendCandidate: func(_ string, data string) error {
|
||||
var candidate pionWebRTC.ICECandidateInit
|
||||
if err := json.Unmarshal([]byte(data), &candidate); err != nil {
|
||||
return err
|
||||
}
|
||||
signalingMu.Lock()
|
||||
defer signalingMu.Unlock()
|
||||
if viewer.RemoteDescription() == nil {
|
||||
pending = append(pending, candidate)
|
||||
return nil
|
||||
}
|
||||
return viewer.AddICECandidate(candidate)
|
||||
},
|
||||
SendAnswer: func(_ string, data string) error {
|
||||
answer, err := base64.StdEncoding.DecodeString(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
signalingMu.Lock()
|
||||
defer signalingMu.Unlock()
|
||||
if err := viewer.SetRemoteDescription(pionWebRTC.SessionDescription{
|
||||
Type: pionWebRTC.SDPTypeAnswer, SDP: string(answer),
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, candidate := range pending {
|
||||
if err := viewer.AddICECandidate(candidate); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
pending = nil
|
||||
return nil
|
||||
},
|
||||
}
|
||||
InitializeWebRTCConnection(
|
||||
&models.Configuration{Config: models.Config{Key: "unit-camera"}},
|
||||
&models.Communication{}, nil, b, nil,
|
||||
models.LiveHDHandshake{
|
||||
Signaling: signaling,
|
||||
Payload: models.RequestHDStreamPayload{
|
||||
SessionID: "startup-recovery",
|
||||
SessionDescription: base64.StdEncoding.EncodeToString([]byte(viewer.LocalDescription().SDP)),
|
||||
},
|
||||
},
|
||||
)
|
||||
var firstRequest time.Time
|
||||
select {
|
||||
case firstRequest = <-requests:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("connected transport did not request a startup keyframe")
|
||||
}
|
||||
b.WriteSample(pionMedia.Sample{Data: []byte{0, 0, 0, 1, 0x41, 0x80}, Duration: time.Second / 30}, false)
|
||||
select {
|
||||
case <-tracks:
|
||||
t.Fatal("viewer received dependent video before its startup IDR")
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
b.WriteSample(pionMedia.Sample{Data: []byte{0, 0, 0, 1, 0x65, 0x88, 0x84}, Duration: time.Second / 30}, true)
|
||||
var track *pionWebRTC.TrackRemote
|
||||
select {
|
||||
case track = <-tracks:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("viewer did not receive the requested startup IDR")
|
||||
}
|
||||
|
||||
for _, feedback := range []rtcp.Packet{
|
||||
&rtcp.PictureLossIndication{SenderSSRC: 42, MediaSSRC: uint32(track.SSRC())},
|
||||
&rtcp.FullIntraRequest{SenderSSRC: 42, FIR: []rtcp.FIREntry{{SSRC: uint32(track.SSRC()), SequenceNumber: 1}}},
|
||||
} {
|
||||
if err := viewer.WriteRTCP([]rtcp.Packet{feedback}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case at := <-requests:
|
||||
if at.Sub(firstRequest) < keyframeRequestInterval-5*time.Millisecond {
|
||||
t.Fatal("real viewer feedback bypassed keyframe request throttling")
|
||||
}
|
||||
firstRequest = at
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatalf("real %T feedback did not request a recovery keyframe", feedback)
|
||||
}
|
||||
b.WriteSample(pionMedia.Sample{Data: []byte{0, 0, 0, 1, 0x65, 0x88, 0x84}, Duration: time.Second / 30}, true)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
package webrtc
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/pion/interceptor"
|
||||
"github.com/pion/rtcp"
|
||||
"github.com/pion/rtp"
|
||||
pionWebRTC "github.com/pion/webrtc/v4"
|
||||
pionMedia "github.com/pion/webrtc/v4/pkg/media"
|
||||
)
|
||||
|
||||
type clockTestWriter struct {
|
||||
timestamps []uint32
|
||||
}
|
||||
|
||||
func (w *clockTestWriter) WriteRTP(header *rtp.Header, payload []byte) (int, error) {
|
||||
w.timestamps = append(w.timestamps, header.Timestamp)
|
||||
return len(payload), nil
|
||||
}
|
||||
|
||||
func (*clockTestWriter) Write([]byte) (int, error) {
|
||||
return 0, errors.New("unexpected raw RTP write")
|
||||
}
|
||||
|
||||
type clockTestContext struct {
|
||||
writer *clockTestWriter
|
||||
}
|
||||
|
||||
func (clockTestContext) ID() string { return "clock-test" }
|
||||
func (clockTestContext) SSRC() pionWebRTC.SSRC { return 123 }
|
||||
func (clockTestContext) SSRCRetransmission() pionWebRTC.SSRC { return 0 }
|
||||
func (clockTestContext) SSRCForwardErrorCorrection() pionWebRTC.SSRC { return 0 }
|
||||
func (clockTestContext) RTCPReader() interceptor.RTCPReader { return nil }
|
||||
func (clockTestContext) HeaderExtensions() []pionWebRTC.RTPHeaderExtensionParameter {
|
||||
return nil
|
||||
}
|
||||
func (c clockTestContext) WriteStream() pionWebRTC.TrackLocalWriter { return c.writer }
|
||||
func (clockTestContext) CodecParameters() []pionWebRTC.RTPCodecParameters {
|
||||
return []pionWebRTC.RTPCodecParameters{{
|
||||
RTPCodecCapability: pionWebRTC.RTPCodecCapability{MimeType: pionWebRTC.MimeTypeH264, ClockRate: 90000},
|
||||
PayloadType: 96,
|
||||
}}
|
||||
}
|
||||
|
||||
func clockTestPeer(t *testing.T) (*TrackBroadcaster, *peerTrack, *clockTestWriter) {
|
||||
t.Helper()
|
||||
b, pt := testVideoBroadcaster(t)
|
||||
pt.samples = make(chan peerSample, 2)
|
||||
track, err := pionWebRTC.NewTrackLocalStaticSample(
|
||||
pionWebRTC.RTPCodecCapability{MimeType: pionWebRTC.MimeTypeH264},
|
||||
"video", "clock-test", pionWebRTC.WithRTPTimestamp(1000),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writer := &clockTestWriter{}
|
||||
if _, err := track.Bind(clockTestContext{writer: writer}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pt.track = track
|
||||
b.SetPeerConnected("viewer", true)
|
||||
return b, pt, writer
|
||||
}
|
||||
|
||||
func clockVideoSample(duration time.Duration) pionMedia.Sample {
|
||||
return pionMedia.Sample{
|
||||
Data: []byte{0, 0, 0, 1, 0x65, 0x88, 0x84},
|
||||
Duration: duration,
|
||||
}
|
||||
}
|
||||
|
||||
func writeClockSample(t *testing.T, pt *peerTrack) {
|
||||
t.Helper()
|
||||
select {
|
||||
case sample := <-pt.samples:
|
||||
if err := writePeerSample(pt.track, sample); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
default:
|
||||
t.Fatal("expected queued video sample")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVideoRecoveryPreservesRTPClock(t *testing.T) {
|
||||
for _, scenario := range []string{"PLI", "FIR", "reconnect", "overflow"} {
|
||||
t.Run(scenario, func(t *testing.T) {
|
||||
b, pt, writer := clockTestPeer(t)
|
||||
b.WriteSample(clockVideoSample(40*time.Millisecond), true)
|
||||
writeClockSample(t, pt)
|
||||
b.WriteSample(clockVideoSample(70*time.Millisecond), false)
|
||||
|
||||
switch scenario {
|
||||
case "PLI", "FIR":
|
||||
var feedback rtcp.Packet = &rtcp.PictureLossIndication{MediaSSRC: 123}
|
||||
if scenario == "FIR" {
|
||||
feedback = &rtcp.FullIntraRequest{FIR: []rtcp.FIREntry{{SSRC: 123, SequenceNumber: 1}}}
|
||||
}
|
||||
data, err := rtcp.Marshal([]rtcp.Packet{feedback})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := b.HandleRTCP("viewer", data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.WriteSample(clockVideoSample(30*time.Millisecond), false)
|
||||
case "reconnect":
|
||||
b.SetPeerConnected("viewer", false)
|
||||
b.WriteSample(clockVideoSample(30*time.Millisecond), false)
|
||||
b.SetPeerConnected("viewer", true)
|
||||
case "overflow":
|
||||
b.WriteSample(clockVideoSample(30*time.Millisecond), false)
|
||||
}
|
||||
b.WriteSample(clockVideoSample(1900*time.Millisecond), false)
|
||||
b.WriteSample(clockVideoSample(40*time.Millisecond), true)
|
||||
writeClockSample(t, pt)
|
||||
|
||||
if len(writer.timestamps) != 2 {
|
||||
t.Fatalf("sent %d RTP packets, want only the two IDRs", len(writer.timestamps))
|
||||
}
|
||||
// The initial 40 ms frame plus 2000 ms of discarded video at 90 kHz.
|
||||
if got := writer.timestamps[1] - writer.timestamps[0]; got != 183600 {
|
||||
t.Fatalf("recovery RTP timestamp delta = %d, want 183600", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOverflowIDRPreservesDiscardedRTPDuration(t *testing.T) {
|
||||
b, pt, writer := clockTestPeer(t)
|
||||
b.WriteSample(clockVideoSample(40*time.Millisecond), true)
|
||||
writeClockSample(t, pt)
|
||||
b.WriteSample(clockVideoSample(70*time.Millisecond), false)
|
||||
b.WriteSample(clockVideoSample(30*time.Millisecond), false)
|
||||
b.WriteSample(clockVideoSample(40*time.Millisecond), true)
|
||||
writeClockSample(t, pt)
|
||||
if len(writer.timestamps) != 2 || writer.timestamps[1]-writer.timestamps[0] != 12600 {
|
||||
t.Fatalf("overflow IDR compressed the RTP clock: %v", writer.timestamps)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepeatedFeedbackPreservesAlreadyQueuedSkippedTime(t *testing.T) {
|
||||
b, pt, writer := clockTestPeer(t)
|
||||
b.WriteSample(clockVideoSample(40*time.Millisecond), true)
|
||||
writeClockSample(t, pt)
|
||||
data, err := rtcp.Marshal([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: 123}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for range 2 {
|
||||
if err := b.HandleRTCP("viewer", data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.WriteSample(clockVideoSample(time.Second), false)
|
||||
b.WriteSample(clockVideoSample(40*time.Millisecond), true)
|
||||
}
|
||||
writeClockSample(t, pt)
|
||||
if len(writer.timestamps) != 2 || writer.timestamps[1]-writer.timestamps[0] != 187200 {
|
||||
t.Fatalf("repeated feedback lost queued skipped time: %v", writer.timestamps)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user