Merge pull request #270 from kerberos-io/fix/media-looping-boundary

fix/media-looping-boundary

machinery/src/video/mp4.go

@@ -32,6 +32,15 @@ const MacEpochOffset uint64 = 2082844800
 // resulting in ~3 second fragments (assuming a typical GOP interval).
 const FragmentDurationMs = 3000
 
+// MinNormalGOPMs is the minimum spacing we expect between two consecutive
+// IDRs of a healthy source (typical encoders produce IDRs every 1000ms or
+// more). When two keyframes arrive closer than this, we treat the second one
+// as an upstream restart/loop-seam and force a fresh fragment so the seam
+// IDR cannot end up as a mid-fragment sync sample. The check only runs when
+// the current fragment has not yet reached FragmentDurationMs, so it never
+// fires during normal multi-GOP fragments at intended GOP boundaries.
+const MinNormalGOPMs = 950
+
 type MP4 struct {
 	// FileName is the name of the file
 	FileName                string
@@ -56,6 +65,7 @@ type MP4 struct {
 	FreeBoxSize             int64
 	FragmentStartRawPTS     uint64            // Raw PTS for timing when to flush fragments
 	FragmentStartDTS        uint64            // Accumulated VideoTotalDuration at fragment start (matches tfdt)
+	LastKeyframeRawPTS      uint64            // Raw PTS of the most recently seen keyframe (in any fragment)
 	MoofBoxes               int64             // Number of moof boxes in the file
 	MoofBoxSizes            []int64           // Sizes of each moof box
 	SegmentDurations        []uint64          // Duration of each segment in timescale units
@@ -307,6 +317,27 @@ func (mp4 *MP4) AddSampleToTrack(trackID uint32, isKeyframe bool, data []byte, p
 		}
 		shouldFlush := !mp4.Start || elapsed >= FragmentDurationMs
 
+		// Detect upstream source-loop / restart discontinuity. When an MP4 is
+		// looped through virtual-rtsp (ffmpeg `-stream_loop -1 -re`) the loop
+		// seam emits a fresh IDR much sooner than a normal GOP would. PTS keeps
+		// growing monotonically, so the timing-only `elapsed` check above does
+		// not catch it and the seam IDR ends up as a mid-fragment sync sample.
+		// MSE-based players (Video.js / Chromium / Firefox) reject the resulting
+		// fragment with a "media corruption" error because the inner IDR resets
+		// frame_num/POC inside what they expect to be a single GOP. Force a
+		// fragment boundary whenever two consecutive keyframes arrive much
+		// closer than a normal GOP (here: < 500 ms apart). This isolates the
+		// seam IDR into its own fragment so each fragment stays a clean GOP.
+		if !shouldFlush && trackID == uint32(mp4.VideoTrack) && mp4.Start &&
+			mp4.LastKeyframeRawPTS > 0 && pts > mp4.LastKeyframeRawPTS &&
+			pts-mp4.LastKeyframeRawPTS < MinNormalGOPMs {
+			log.Log.Warning(fmt.Sprintf("mp4.AddSampleToTrack(): forcing fragment flush at unexpectedly close keyframe (gap=%d ms, fragment elapsed=%d ms) - likely upstream loop/restart discontinuity", pts-mp4.LastKeyframeRawPTS, elapsed))
+			shouldFlush = true
+		}
+		if trackID == uint32(mp4.VideoTrack) {
+			mp4.LastKeyframeRawPTS = pts
+		}
+
 		if shouldFlush {
 			// Write the previous segment to the file
 			if mp4.Start {

machinery/src/video/mp4_loopseam_test.go

@@ -0,0 +1,99 @@
+package video
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	mp4ff "github.com/Eyevinn/mp4ff/mp4"
+	"github.com/kerberos-io/agent/machinery/src/models"
+)
+
+// TestMP4LoopSeamIsolation reproduces the loop-seam pattern from the
+// failing virtual-rtsp recordings: ~1s GOPs, but at the source-MP4
+// loop boundary an IDR arrives prematurely (~200-870ms after the
+// previous IDR). Without the fix this seam IDR ends up bunched into
+// the same fragment as the prior GOP's IDR which trips macOS
+// VideoToolbox (kVTVideoDecoderBadDataErr / -12909). The fix forces
+// a fragment flush whenever two IDRs arrive closer than MinNormalGOPMs.
+func TestMP4LoopSeamIsolation(t *testing.T) {
+	tmpFile := "/tmp/test_loop_seam.mp4"
+	defer os.Remove(tmpFile)
+
+	sps := []byte{0x67, 0x42, 0xc0, 0x1e, 0xd9, 0x00, 0xa0, 0x47, 0xfe, 0xc8}
+	pps := []byte{0x68, 0xce, 0x38, 0x80}
+	mp4Video := NewMP4(tmpFile, [][]byte{sps}, [][]byte{pps}, nil, 30)
+	mp4Video.SetWidth(1920)
+	mp4Video.SetHeight(1080)
+	v := mp4Video.AddVideoTrack("H264")
+
+	mk := func(k bool) []byte {
+		nt := byte(0x01)
+		if k {
+			nt = 0x65
+		}
+		f := []byte{0, 0, 0, 1, nt}
+		for i := 0; i < 200; i++ {
+			f = append(f, byte(i))
+		}
+		return f
+	}
+
+	frameDur := uint64(33)
+	pts := uint64(0)
+	emit := func(n int, gopLen int) {
+		for f := 0; f < n; f++ {
+			isKey := (f % gopLen) == 0
+			mp4Video.AddSampleToTrack(v, isKey, mk(isKey), pts)
+			pts += frameDur
+		}
+	}
+
+	// 17 seconds of normal content (last "good" IDR at sec 17).
+	emit(17*30, 30)
+	// Seam: IDR arrives ~867ms after previous (vs normal 1000ms).
+	pts -= 100
+	emit(13*30, 30)
+
+	mp4Video.Close(&models.Config{Signing: &models.Signing{PrivateKey: ""}})
+
+	f, _ := os.Open(tmpFile)
+	defer f.Close()
+	parsed, err := mp4ff.DecodeFile(f)
+	if err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+
+	fragIdx := 0
+	for _, seg := range parsed.Segments {
+		for _, fr := range seg.Fragments {
+			for _, traf := range fr.Moof.Trafs {
+				if traf.Tfhd.TrackID != 1 {
+					continue
+				}
+				tfdt := traf.Tfdt.BaseMediaDecodeTime()
+				offset := uint64(0)
+				var keys []uint64
+				for _, trun := range traf.Truns {
+					for _, s := range trun.Samples {
+						if (s.Flags>>24)&0x03 == 0x02 {
+							keys = append(keys, offset)
+						}
+						offset += uint64(s.Dur)
+					}
+				}
+				fmt.Printf("frag %d tfdt=%d samples_dur=%d keys@%v\n",
+					fragIdx, tfdt, offset, keys)
+				for i := 1; i < len(keys); i++ {
+					gap := keys[i] - keys[i-1]
+					if gap < MinNormalGOPMs {
+						t.Errorf("frag %d (tfdt=%d): two IDRs only %d ms apart "+
+							"in same fragment (< %d) - seam was not isolated",
+							fragIdx, tfdt, gap, MinNormalGOPMs)
+					}
+				}
+				fragIdx++
+			}
+		}
+	}
+}