Merge pull request #234 from kerberos-io/fix/fps-gop-size

fix/fps-gop-size

machinery/src/capture/gortsplib.go

@@ -85,12 +85,8 @@ type Golibrtsp struct {
 
 	Streams []packets.Stream
 
-	// FPS calculation fields
-	lastFrameTime    time.Time
-	frameTimeBuffer  []time.Duration
-	frameBufferSize  int
-	frameBufferIndex int
-	fpsMutex         sync.Mutex
+	// Per-stream FPS calculation (keyed by stream index)
+	fpsTrackers map[int8]*fpsTracker
 
 	// I-frame interval tracking fields
 	packetsSinceLastKeyframe int
@@ -101,6 +97,78 @@ type Golibrtsp struct {
 	keyframeMutex            sync.Mutex
 }
 
+// fpsTracker holds per-stream state for PTS-based FPS calculation.
+// Each video stream (H264 / H265) gets its own tracker so PTS
+// samples from different codecs never interleave.
+type fpsTracker struct {
+	mu              sync.Mutex
+	lastPTS         time.Duration
+	hasPTS          bool
+	frameTimeBuffer []time.Duration
+	bufferSize      int
+	bufferIndex     int
+	cachedFPS       float64 // latest computed FPS
+}
+
+func newFPSTracker(bufferSize int) *fpsTracker {
+	return &fpsTracker{
+		frameTimeBuffer: make([]time.Duration, bufferSize),
+		bufferSize:      bufferSize,
+	}
+}
+
+// update records a new PTS sample and returns the latest FPS estimate.
+// It must be called once per complete decoded frame (after Decode()
+// succeeds), not on every RTP packet fragment.
+func (ft *fpsTracker) update(pts time.Duration) float64 {
+	ft.mu.Lock()
+	defer ft.mu.Unlock()
+
+	if !ft.hasPTS {
+		ft.lastPTS = pts
+		ft.hasPTS = true
+		return 0
+	}
+
+	interval := pts - ft.lastPTS
+	ft.lastPTS = pts
+
+	// Skip invalid intervals (zero, negative, or very large which
+	// indicate a PTS discontinuity or wrap).
+	if interval <= 0 || interval > 5*time.Second {
+		return ft.cachedFPS
+	}
+
+	ft.frameTimeBuffer[ft.bufferIndex] = interval
+	ft.bufferIndex = (ft.bufferIndex + 1) % ft.bufferSize
+
+	var totalInterval time.Duration
+	validSamples := 0
+	for _, iv := range ft.frameTimeBuffer {
+		if iv > 0 {
+			totalInterval += iv
+			validSamples++
+		}
+	}
+	if validSamples == 0 {
+		return ft.cachedFPS
+	}
+	avgInterval := totalInterval / time.Duration(validSamples)
+	if avgInterval == 0 {
+		return ft.cachedFPS
+	}
+
+	ft.cachedFPS = float64(time.Second) / float64(avgInterval)
+	return ft.cachedFPS
+}
+
+// fps returns the most recent FPS estimate without recording a new sample.
+func (ft *fpsTracker) fps() float64 {
+	ft.mu.Lock()
+	defer ft.mu.Unlock()
+	return ft.cachedFPS
+}
+
 // Init function
 var H264FrameDecoder *Decoder
 var H265FrameDecoder *Decoder
@@ -548,18 +616,17 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 
 			if len(rtppkt.Payload) > 0 {
 
-				// decode timestamp
-				pts, ok := g.Client.PacketPTS(g.VideoH264Media, rtppkt)
-				pts2, ok := g.Client.PacketPTS2(g.VideoH264Media, rtppkt)
-				if !ok {
-					log.Log.Debug("capture.golibrtsp.Start(): " + "unable to get PTS")
+				// decode timestamps — validate each call separately
+				pts, okPTS := g.Client.PacketPTS(g.VideoH264Media, rtppkt)
+				pts2, okPTS2 := g.Client.PacketPTS2(g.VideoH264Media, rtppkt)
+				if !okPTS2 {
+					log.Log.Debug("capture.golibrtsp.Start(): unable to get PTS2 from PacketPTS2")
 					return
 				}
 
-				// Extract access units from RTP packets
-				// We need to do this, because the decoder expects a full
-				// access unit. Once we have a full access unit, we can
-				// decode it, and know if it's a keyframe or not.
+				// Extract access units from RTP packets.
+				// We need a complete access unit to determine whether
+				// this is a keyframe.
 				au, errDecode := g.VideoH264Decoder.Decode(rtppkt)
 				if errDecode != nil {
 					if errDecode != rtph264.ErrNonStartingPacketAndNoPrevious && errDecode != rtph264.ErrMorePacketsNeeded {
@@ -568,6 +635,15 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 					return
 				}
 
+				// Frame is complete — update per-stream FPS from PTS.
+				if okPTS {
+					if ft := g.fpsTrackers[g.VideoH264Index]; ft != nil {
+						if ptsFPS := ft.update(pts); ptsFPS > 0 && ptsFPS <= 120 {
+							g.Streams[g.VideoH264Index].FPS = ptsFPS
+						}
+					}
+				}
+
 				// We'll need to read out a few things.
 				// prepend an AUD. This is required by some players
 				filteredAU = [][]byte{
@@ -651,7 +727,11 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 				keyframeInterval := g.trackKeyframeInterval(idrPresent)
 				if idrPresent && keyframeInterval > 0 {
 					avgInterval := g.getAverageKeyframeInterval()
-					gopDuration := float64(keyframeInterval) / g.Streams[g.VideoH265Index].FPS
+					fps := g.Streams[g.VideoH264Index].FPS
+					if fps <= 0 {
+						fps = 25.0 // Default fallback FPS
+					}
+					gopDuration := float64(keyframeInterval) / fps
 					gopSize := int(avgInterval) // Store GOP size in a separate variable
 					g.Streams[g.VideoH264Index].GopSize = gopSize
 					log.Log.Debug(fmt.Sprintf("capture.golibrtsp.Start(%s): Keyframe interval=%d packets, Avg=%.1f, GOP=%.1fs, GOPSize=%d",
@@ -716,18 +796,17 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 
 			if len(rtppkt.Payload) > 0 {
 
-				// decode timestamp
-				pts, ok := g.Client.PacketPTS(g.VideoH265Media, rtppkt)
-				pts2, ok := g.Client.PacketPTS2(g.VideoH265Media, rtppkt)
-				if !ok {
-					log.Log.Debug("capture.golibrtsp.Start(): " + "unable to get PTS")
+				// decode timestamps — validate each call separately
+				pts, okPTS := g.Client.PacketPTS(g.VideoH265Media, rtppkt)
+				pts2, okPTS2 := g.Client.PacketPTS2(g.VideoH265Media, rtppkt)
+				if !okPTS2 {
+					log.Log.Debug("capture.golibrtsp.Start(): unable to get PTS")
 					return
 				}
 
-				// Extract access units from RTP packets
-				// We need to do this, because the decoder expects a full
-				// access unit. Once we have a full access unit, we can
-				// decode it, and know if it's a keyframe or not.
+				// Extract access units from RTP packets.
+				// We need a complete access unit to determine whether
+				// this is a keyframe.
 				au, errDecode := g.VideoH265Decoder.Decode(rtppkt)
 				if errDecode != nil {
 					if errDecode != rtph265.ErrNonStartingPacketAndNoPrevious && errDecode != rtph265.ErrMorePacketsNeeded {
@@ -736,6 +815,15 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 					return
 				}
 
+				// Frame is complete — update per-stream FPS from PTS.
+				if okPTS {
+					if ft := g.fpsTrackers[g.VideoH265Index]; ft != nil {
+						if ptsFPS := ft.update(pts); ptsFPS > 0 && ptsFPS <= 120 {
+							g.Streams[g.VideoH265Index].FPS = ptsFPS
+						}
+					}
+				}
+
 				filteredAU = [][]byte{
 					{byte(h265.NALUType_AUD_NUT) << 1, 1, 0x50},
 				}
@@ -796,7 +884,11 @@ func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets
 				keyframeInterval := g.trackKeyframeInterval(isRandomAccess)
 				if isRandomAccess && keyframeInterval > 0 {
 					avgInterval := g.getAverageKeyframeInterval()
-					gopDuration := float64(keyframeInterval) / g.Streams[g.VideoH265Index].FPS
+					fps := g.Streams[g.VideoH265Index].FPS
+					if fps <= 0 {
+						fps = 25.0 // Default fallback FPS
+					}
+					gopDuration := float64(keyframeInterval) / fps
 					gopSize := int(avgInterval) // Store GOP size in a separate variable
 					g.Streams[g.VideoH265Index].GopSize = gopSize
 					log.Log.Debug(fmt.Sprintf("capture.golibrtsp.Start(%s): Keyframe interval=%d packets, Avg=%.1f, GOP=%.1fs, GOPSize=%d",
@@ -1179,10 +1271,11 @@ func WriteMPEG4Audio(forma *format.MPEG4Audio, aus [][]byte) ([]byte, error) {
 
 // Initialize FPS calculation buffers
 func (g *Golibrtsp) initFPSCalculation() {
-	g.frameBufferSize = 30 // Store last 30 frame intervals
-	g.frameTimeBuffer = make([]time.Duration, g.frameBufferSize)
-	g.frameBufferIndex = 0
-	g.lastFrameTime = time.Time{}
+	// Ensure the per-stream FPS trackers map exists. Individual trackers
+	// can be created lazily when a given stream index is first used.
+	if g.fpsTrackers == nil {
+		g.fpsTrackers = make(map[int8]*fpsTracker)
+	}
 
 	// Initialize I-frame interval tracking
 	g.keyframeBufferSize = 10 // Store last 10 keyframe intervals
@@ -1192,50 +1285,11 @@ func (g *Golibrtsp) initFPSCalculation() {
 	g.lastKeyframePacketCount = 0
 }
 
-// Calculate FPS from frame timestamps
-func (g *Golibrtsp) calculateFPSFromTimestamps() float64 {
-	g.fpsMutex.Lock()
-	defer g.fpsMutex.Unlock()
-
-	if g.lastFrameTime.IsZero() {
-		g.lastFrameTime = time.Now()
-		return 0
-	}
-
-	now := time.Now()
-	interval := now.Sub(g.lastFrameTime)
-	g.lastFrameTime = now
-
-	// Store the interval
-	g.frameTimeBuffer[g.frameBufferIndex] = interval
-	g.frameBufferIndex = (g.frameBufferIndex + 1) % g.frameBufferSize
-
-	// Calculate average FPS from stored intervals
-	var totalInterval time.Duration
-	validSamples := 0
-
-	for _, interval := range g.frameTimeBuffer {
-		if interval > 0 {
-			totalInterval += interval
-			validSamples++
-		}
-	}
-
-	if validSamples == 0 {
-		return 0
-	}
-
-	avgInterval := totalInterval / time.Duration(validSamples)
-	if avgInterval == 0 {
-		return 0
-	}
-
-	return float64(time.Second) / float64(avgInterval)
-}
-
-// Get enhanced FPS information from SPS with fallback
+// Get enhanced FPS information from SPS with fallback to PTS-based calculation.
+// The PTS-based FPS is computed per completed frame via fpsTracker.update(),
+// so by the time this is called we already have a good estimate.
 func (g *Golibrtsp) getEnhancedFPS(sps *h264.SPS, streamIndex int8) float64 {
-	// First try to get FPS from SPS
+	// First try to get FPS from SPS VUI parameters
 	spsFPS := sps.FPS()
 
 	// Check if SPS FPS is reasonable (between 1 and 120 fps)
@@ -1244,11 +1298,13 @@ func (g *Golibrtsp) getEnhancedFPS(sps *h264.SPS, streamIndex int8) float64 {
 		return spsFPS
 	}
 
-	// Fallback to timestamp-based calculation
-	timestampFPS := g.calculateFPSFromTimestamps()
-	if timestampFPS > 0 && timestampFPS <= 120 {
-		log.Log.Debug(fmt.Sprintf("capture.golibrtsp.getEnhancedFPS(): Timestamp FPS: %.2f", timestampFPS))
-		return timestampFPS
+	// Fallback to PTS-based FPS (already calculated per-frame)
+	if ft := g.fpsTrackers[streamIndex]; ft != nil {
+		ptsFPS := ft.fps()
+		if ptsFPS > 0 && ptsFPS <= 120 {
+			log.Log.Debug(fmt.Sprintf("capture.golibrtsp.getEnhancedFPS(): PTS FPS: %.2f", ptsFPS))
+			return ptsFPS
+		}
 	}
 
 	// Return SPS FPS even if it seems unreasonable, or default

machinery/src/video/mp4.go

@@ -150,6 +150,43 @@ func (mp4 *MP4) AddAudioTrack(codec string) uint32 {
 func (mp4 *MP4) AddMediaSegment(segNr int) {
 }
 
+// flushPendingVideoSample writes the pending video sample to the current fragment.
+// If nextPTS is provided (non-zero), it calculates duration from the PTS difference.
+// If nextPTS is 0 (e.g., at Close time), it uses the last known duration.
+// Returns true if a sample was flushed, false if there was no pending sample.
+func (mp4 *MP4) flushPendingVideoSample(nextPTS uint64) bool {
+	if mp4.VideoFullSample == nil || mp4.MultiTrackFragment == nil {
+		return false
+	}
+
+	var duration uint64
+	if nextPTS > 0 && nextPTS > mp4.VideoFullSample.DecodeTime {
+		duration = nextPTS - mp4.VideoFullSample.DecodeTime
+	} else {
+		// No valid nextPTS (Close case) or PTS went backwards (jitter/discontinuity)
+		if nextPTS > 0 {
+			log.Log.Warning(fmt.Sprintf("mp4.flushPendingVideoSample(): video PTS went backwards or zero duration (nextPTS=%d, prevDTS=%d), using last known duration", nextPTS, mp4.VideoFullSample.DecodeTime))
+		}
+		duration = mp4.LastVideoSampleDTS
+		if duration == 0 {
+			duration = 33 // Default ~30fps frame duration
+		}
+	}
+
+	mp4.LastVideoSampleDTS = duration
+	mp4.VideoTotalDuration += duration
+	mp4.VideoFullSample.DecodeTime = mp4.VideoTotalDuration - duration
+	mp4.VideoFullSample.Sample.Dur = uint32(duration)
+
+	err := mp4.MultiTrackFragment.AddFullSampleToTrack(*mp4.VideoFullSample, uint32(mp4.VideoTrack))
+	if err != nil {
+		log.Log.Error("mp4.flushPendingVideoSample(): error adding sample: " + err.Error())
+	}
+
+	mp4.VideoFullSample = nil
+	return true
+}
+
 func (mp4 *MP4) AddSampleToTrack(trackID uint32, isKeyframe bool, data []byte, pts uint64) error {
 
 	if isKeyframe {
@@ -166,6 +203,13 @@ func (mp4 *MP4) AddSampleToTrack(trackID uint32, isKeyframe bool, data []byte, p
 		if shouldFlush {
 			// Write the previous segment to the file
 			if mp4.Start {
+				// IMPORTANT: Add any pending video sample to the current segment BEFORE flushing.
+				// This ensures the segment contains all frames up to (but not including) this keyframe,
+				// and the new segment will start cleanly with this keyframe.
+				if trackID == uint32(mp4.VideoTrack) {
+					mp4.flushPendingVideoSample(pts)
+				}
+
 				mp4.MoofBoxes = mp4.MoofBoxes + 1
 				mp4.MoofBoxSizes = append(mp4.MoofBoxSizes, int64(mp4.Segment.Size()))
 				// Track the segment's duration and base decode time for sidx.
@@ -221,18 +265,10 @@ func (mp4 *MP4) AddSampleToTrack(trackID uint32, isKeyframe bool, data []byte, p
 			}
 
 			if err == nil {
+				// Flush previous pending sample before storing the new one
 				if mp4.VideoFullSample != nil {
-					duration := pts - mp4.VideoFullSample.DecodeTime
-					log.Log.Debug("Adding sample to track " + fmt.Sprintf("%d, PTS: %d, Duration: %d, size: %d, Keyframe: %t", trackID, pts, duration, len(lengthPrefixed), isKeyframe))
-
-					mp4.LastVideoSampleDTS = duration
-					mp4.VideoTotalDuration += duration
-					mp4.VideoFullSample.DecodeTime = mp4.VideoTotalDuration - duration
-					mp4.VideoFullSample.Sample.Dur = uint32(duration)
-					err := mp4.MultiTrackFragment.AddFullSampleToTrack(*mp4.VideoFullSample, trackID)
-					if err != nil {
-						log.Log.Error("mp4.AddSampleToTrack(): error adding sample to track " + fmt.Sprintf("%d: %v", trackID, err))
-					}
+					log.Log.Debug("Adding sample to track " + fmt.Sprintf("%d, PTS: %d, size: %d, Keyframe: %t", trackID, pts, len(lengthPrefixed), isKeyframe))
+					mp4.flushPendingVideoSample(pts)
 				}
 
 				// Set the sample data
@@ -304,21 +340,8 @@ func (mp4 *MP4) Close(config *models.Config) {
 
 	// Add final pending samples before closing
 	if mp4.Segment != nil {
-		// Add final video sample if pending
-		if mp4.VideoFullSample != nil {
-			duration := mp4.LastVideoSampleDTS
-			if duration == 0 {
-				duration = 33 // Default ~30fps frame duration
-			}
-			mp4.VideoTotalDuration += duration
-			mp4.VideoFullSample.DecodeTime = mp4.VideoTotalDuration - duration
-			mp4.VideoFullSample.Sample.Dur = uint32(duration)
-			err := mp4.MultiTrackFragment.AddFullSampleToTrack(*mp4.VideoFullSample, uint32(mp4.VideoTrack))
-			if err != nil {
-				log.Log.Error("mp4.Close(): error adding final video sample: " + err.Error())
-			}
-			mp4.VideoFullSample = nil
-		}
+		// Add final video sample if pending (pass 0 as nextPTS to use last known duration)
+		mp4.flushPendingVideoSample(0)
 
 		// Add final audio sample if pending
 		if mp4.AudioFullSample != nil && mp4.AudioTrack > 0 {

machinery/src/webrtc/main.go

@@ -49,6 +49,7 @@ type peerConnectionWrapper struct {
 	conn      *pionWebRTC.PeerConnection
 	cancelCtx context.CancelFunc
 	done      chan struct{}
+	closeOnce sync.Once
 }
 
 var globalConnectionManager = NewConnectionManager()
@@ -339,18 +340,20 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 
 				switch connectionState {
 				case pionWebRTC.PeerConnectionStateDisconnected, pionWebRTC.PeerConnectionStateClosed:
-					count := globalConnectionManager.DecrementPeerCount()
-					log.Log.Info("webrtc.main.InitializeWebRTCConnection(): Peer disconnected. Active peers: " + string(rune(count)))
+					wrapper.closeOnce.Do(func() {
+						count := globalConnectionManager.DecrementPeerCount()
+						log.Log.Info("webrtc.main.InitializeWebRTCConnection(): Peer disconnected. Active peers: " + string(rune(count)))
 
-					// Clean up resources
-					globalConnectionManager.CloseCandidateChannel(sessionKey)
+						// Clean up resources
+						globalConnectionManager.CloseCandidateChannel(sessionKey)
 
-					if err := peerConnection.Close(); err != nil {
-						log.Log.Error("webrtc.main.InitializeWebRTCConnection(): error closing peer connection: " + err.Error())
-					}
+						if err := peerConnection.Close(); err != nil {
+							log.Log.Error("webrtc.main.InitializeWebRTCConnection(): error closing peer connection: " + err.Error())
+						}
 
-					globalConnectionManager.RemovePeerConnection(handshake.SessionID)
-					close(wrapper.done)
+						globalConnectionManager.RemovePeerConnection(handshake.SessionID)
+						close(wrapper.done)
+					})
 
 				case pionWebRTC.PeerConnectionStateConnected:
 					count := globalConnectionManager.IncrementPeerCount()