Enhance audio/video synchronization and improve flushing timeout in AAC transcoding

README.md

@@ -65,7 +65,6 @@ There are a myriad of cameras out there (USB, IP and other cameras), and it migh
 
 ### Contributing
 
-1. [Security vulnerability reporting](#security-vulnerability-reporting)
 1. [Contribute with Codespaces](#contribute-with-codespaces)
 2. [Develop and build](#develop-and-build)
 3. [Building from source](#building-from-source)
@@ -302,10 +301,6 @@ If we talk about video encoders and decoders (codecs) there are 2 major video co
 
 Conclusion: depending on the use case you might choose one over the other, and you can use both at the same time. For example you can use H264 (main stream) for livestreaming, and H265 (sub stream) for recording. If you wish to play recordings in a cross-platform and cross-browser environment, you might opt for H264 for better support.
 
-## Security vulnerability reporting
-
-If you found a potential security vulnerability, please use the private channels described in [SECURITY.md](SECURITY.md). Avoid opening public GitHub issues for sensitive findings.
-
 ## Contribute with Codespaces
 
 One of the major blockers for letting you contribute to an Open Source project is to set up your local development machine. Why? Because you might already have some tools and libraries installed that are used for other projects, and the libraries you would need for Kerberos Agent, for example FFmpeg, might require a different version. Welcome to dependency hell...

SECURITY.md

@@ -1,40 +0,0 @@
-# Security Policy
-
-## Supported Versions
-
-We only provide security fixes for the latest release series on the `master` branch.
-
-## Reporting a Vulnerability
-
-Please do **not** open a public GitHub issue for potential security vulnerabilities.
-
-Use one of the private channels below:
-
-1. Preferred: GitHub private vulnerability reporting
-   - https://github.com/kerberos-io/agent/security/advisories/new
-2. Fallback: Email
-   - support@kerberos.io
-   - Optional CC: support@uug.ai
-
-Please include:
-
-- A short summary and impact.
-- Reproduction steps or proof of concept.
-- Affected version(s), commit hash, or deployment details.
-- Any proposed mitigation/workaround.
-- Your preferred attribution name.
-
-For faster triage, use this subject format in email:
-
-`[Security][Kerberos Agent] <short title>`
-
-## Response Expectations
-
-- Acknowledgement target: within 3 business days.
-- Triage/update target: within 7 business days after acknowledgement.
-
-If you do not receive a response in time, please resend your report and include your original timestamp.
-
-## Disclosure and Credits
-
-We follow coordinated disclosure. After a fix is available, we will credit reporters unless they prefer to stay anonymous.

machinery/README.md

@@ -22,8 +22,4 @@ https://brianmacdonald.github.io/Ethonate/address#0xf4a759C9436E2280Ea9cdd23d314
 
 [**Docker Hub**](https://hub.docker.com/r/kerberos/agent) | [**Documentation**](https://doc.kerberos.io) | [**Website**](https://kerberos.io)
 
-Kerberos Open source (v3) is a cutting edge video surveillance management system made available as Open Source under the MIT License. This means that all the source code is available for you or your company, and you can use, transform and distribute the source code; as long you keep a reference of the original license. Kerberos Open Source (v3) can be used for commercial usage (which was not the case for v2). Read more [about the license here](LICENSE).
-
-## Security reporting
-
-For sensitive vulnerabilities, use private disclosure channels documented in [../SECURITY.md](../SECURITY.md).
+Kerberos Open source (v3) is a cutting edge video surveillance management system made available as Open Source under the MIT License. This means that all the source code is available for you or your company, and you can use, transform and distribute the source code; as long you keep a reference of the original license. Kerberos Open Source (v3) can be used for commercial usage (which was not the case for v2). Read more [about the license here](LICENSE).

machinery/src/capture/gortsplib.go

@@ -339,6 +339,40 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 		}
 	}
 
+	// Look for audio stream.
+	// find the LPCM media and format
+	audioFormaLPCM, audioMediLPCM := FindLPCM(desc, false)
+	g.AudioLPCMMedia = audioMediLPCM
+	g.AudioLPCMForma = audioFormaLPCM
+	if audioMediLPCM == nil {
+		log.Log.Debug("capture.golibrtsp.Connect(LPCM): " + "audio media not found")
+	} else {
+		_, err = g.Client.Setup(desc.BaseURL, audioMediLPCM, 0, 0)
+		if err != nil {
+			log.Log.Error("capture.golibrtsp.Connect(LPCM): " + err.Error())
+		} else {
+			audiortpDec, err := audioFormaLPCM.CreateDecoder()
+			if err != nil {
+				log.Log.Error("capture.golibrtsp.Connect(LPCM): " + err.Error())
+			} else {
+				g.AudioLPCMDecoder = audiortpDec
+				streamIndex := len(g.Streams)
+				g.Streams = append(g.Streams, packets.Stream{
+					Index:         streamIndex,
+					Name:          "LPCM",
+					IsVideo:       false,
+					IsAudio:       true,
+					IsBackChannel: false,
+					SampleRate:    audioFormaLPCM.SampleRate,
+					Channels:      audioFormaLPCM.ChannelCount,
+					BitDepth:      audioFormaLPCM.BitDepth,
+				})
+
+				g.AudioLPCMIndex = int8(len(g.Streams)) - 1
+			}
+		}
+	}
+
 	// Look for audio stream.
 	// find the G711 media and format
 	audioForma, audioMedi := FindPCMU(desc, false)
@@ -367,6 +401,8 @@ func (g *Golibrtsp) Connect(ctx context.Context, ctxOtel context.Context) (err e
 					IsVideo:       false,
 					IsAudio:       true,
 					IsBackChannel: false,
+					SampleRate:    defaultPCMUSampleRate,
+					Channels:      defaultPCMUChannels,
 				})
 
 				// Set the index for the audio
@@ -509,6 +545,8 @@ func (g *Golibrtsp) ConnectBackChannel(ctx context.Context, ctxRunAgent context.
 				IsVideo:       false,
 				IsAudio:       true,
 				IsBackChannel: true,
+				SampleRate:    defaultPCMUSampleRate,
+				Channels:      defaultPCMUChannels,
 			})
 			// Set the index for the audio
 			g.AudioG711IndexBackChannel = int8(len(g.Streams)) - 1
@@ -521,6 +559,39 @@ func (g *Golibrtsp) ConnectBackChannel(ctx context.Context, ctxRunAgent context.
 func (g *Golibrtsp) Start(ctx context.Context, streamType string, queue *packets.Queue, configuration *models.Configuration, communication *models.Communication) (err error) {
 	log.Log.Debug("capture.golibrtsp.Start(): started")
 
+	// called when a MULAW audio RTP packet arrives
+	if g.AudioLPCMMedia != nil && g.AudioLPCMForma != nil {
+		g.Client.OnPacketRTP(g.AudioLPCMMedia, g.AudioLPCMForma, func(rtppkt *rtp.Packet) {
+			pts, ok := g.Client.PacketPTS(g.AudioLPCMMedia, rtppkt)
+			pts2, ok := g.Client.PacketPTS2(g.AudioLPCMMedia, rtppkt)
+			if !ok {
+				log.Log.Debug("capture.golibrtsp.Start(): " + "unable to get PTS")
+				return
+			}
+
+			op, err := g.AudioLPCMDecoder.Decode(rtppkt)
+			if err != nil {
+				log.Log.Error("capture.golibrtsp.Start(): " + err.Error())
+				return
+			}
+
+			pkt := packets.Packet{
+				IsKeyFrame:      false,
+				Packet:          rtppkt,
+				Data:            op,
+				Time:            pts2,
+				TimeLegacy:      pts,
+				CompositionTime: pts2,
+				CurrentTime:     time.Now().UnixMilli(),
+				Idx:             g.AudioLPCMIndex,
+				IsVideo:         false,
+				IsAudio:         true,
+				Codec:           "LPCM",
+			}
+			queue.WritePacket(pkt)
+		})
+	}
+
 	// called when a MULAW audio RTP packet arrives
 	if g.AudioG711Media != nil && g.AudioG711Forma != nil {
 		g.Client.OnPacketRTP(g.AudioG711Media, g.AudioG711Forma, func(rtppkt *rtp.Packet) {
@@ -1259,6 +1330,21 @@ func FindPCMU(desc *description.Session, isBackChannel bool) (*format.G711, *des
 	return nil, nil
 }
 
+func FindLPCM(desc *description.Session, isBackChannel bool) (*format.LPCM, *description.Media) {
+	for _, media := range desc.Medias {
+		if media.IsBackChannel == isBackChannel {
+			for _, forma := range media.Formats {
+				if lpcm, ok := forma.(*format.LPCM); ok {
+					if lpcm.SampleRate > 0 && lpcm.ChannelCount > 0 && lpcm.BitDepth > 0 {
+						return lpcm, media
+					}
+				}
+			}
+		}
+	}
+	return nil, nil
+}
+
 func FindOPUS(desc *description.Session, isBackChannel bool) (*format.Opus, *description.Media) {
 	for _, media := range desc.Medias {
 		if media.IsBackChannel == isBackChannel {

machinery/src/capture/main.go

@@ -87,13 +87,31 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 		// We only expect one audio and one video codec.
 		// If there are multiple audio or video streams, we will use the first one.
 		audioCodec := ""
+		audioBitDepth := 0
 		videoCodec := ""
+		configuredAudioSampleRate := config.Capture.IPCamera.SampleRate
+		configuredAudioChannels := config.Capture.IPCamera.Channels
 		audioStreams, _ := rtspClient.GetAudioStreams()
 		videoStreams, _ := rtspClient.GetVideoStreams()
 		if len(audioStreams) > 0 {
 			audioCodec = audioStreams[0].Name
-			config.Capture.IPCamera.SampleRate = audioStreams[0].SampleRate
-			config.Capture.IPCamera.Channels = audioStreams[0].Channels
+			resolvedSampleRate := audioStreams[0].SampleRate
+			resolvedChannels := audioStreams[0].Channels
+
+			if audioCodec == "LPCM" {
+				if configuredAudioSampleRate > 0 && configuredAudioSampleRate != resolvedSampleRate {
+					log.Log.Warning("capture.main.HandleRecordStream(): LPCM sample rate mismatch between configuration and RTSP stream; using configured value " + strconv.Itoa(configuredAudioSampleRate) + " instead of detected value " + strconv.Itoa(resolvedSampleRate))
+					resolvedSampleRate = configuredAudioSampleRate
+				}
+				if configuredAudioChannels > 0 && configuredAudioChannels != resolvedChannels {
+					log.Log.Warning("capture.main.HandleRecordStream(): LPCM channel count mismatch between configuration and RTSP stream; using configured value " + strconv.Itoa(configuredAudioChannels) + " instead of detected value " + strconv.Itoa(resolvedChannels))
+					resolvedChannels = configuredAudioChannels
+				}
+			}
+
+			config.Capture.IPCamera.SampleRate = resolvedSampleRate
+			config.Capture.IPCamera.Channels = resolvedChannels
+			audioBitDepth = audioStreams[0].BitDepth
 		}
 		if len(videoStreams) > 0 {
 			videoCodec = videoStreams[0].Name
@@ -105,13 +123,15 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 			//var cws *cacheWriterSeeker
 			var mp4Video *video.MP4
 			var videoTrack uint32
-			var audioTrack uint32
+			var audioWriter *recordingAudioWriter
 			var name string
 
 			// Do not do anything!
 			log.Log.Info("capture.main.HandleRecordStream(continuous): start recording")
 
 			start := false
+			rolloverRequested := false
+			rolloverMaxLogged := false
 
 			// If continuous record the full length
 			postRecording = maxRecordingPeriod
@@ -135,29 +155,42 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 
 				nextPkt, cursorError = recordingCursor.ReadPacket()
 
-				now := time.Now().UnixMilli()
+				packetTime := pkt.CurrentTime
+				hardMaxReached := packetTime-startRecording > maxRecordingPeriod-500
+				postRecordingElapsed := startRecording+postRecording-packetTime <= 0
+				if start && (postRecordingElapsed || hardMaxReached) {
+					rolloverRequested = true
+					if hardMaxReached && !rolloverMaxLogged {
+						log.Log.Info("capture.main.HandleRecordStream(continuous): max recording period reached, waiting for next keyframe to roll over without dropping frames")
+						rolloverMaxLogged = true
+					}
+				}
 
-				if start && // If already recording and current frame is a keyframe and we should stop recording
-					nextPkt.IsKeyFrame && (startRecording+postRecording-now <= 0 || now-startRecording > maxRecordingPeriod-500) {
+				closeOnCurrentKeyframe := rolloverRequested && pkt.IsKeyFrame && pkt.CurrentTime > startRecording
+				closeOnNextKeyframe := rolloverRequested && nextPkt.IsKeyFrame
 
-					pts := convertPTS(pkt.TimeLegacy)
-					if pkt.IsVideo {
-						// Write the last packet
-						if err := mp4Video.AddSampleToTrack(videoTrack, pkt.IsKeyFrame, pkt.Data, pts); err != nil {
-							log.Log.Error("capture.main.HandleRecordStream(continuous): " + err.Error())
-						}
-					} else if pkt.IsAudio {
-						// Write the last packet
-						if pkt.Codec == "AAC" {
-							if err := mp4Video.AddSampleToTrack(audioTrack, pkt.IsKeyFrame, pkt.Data, pts); err != nil {
+				if start && (closeOnCurrentKeyframe || closeOnNextKeyframe) {
+
+					if !closeOnCurrentKeyframe {
+						pts := convertPTS(pkt.TimeLegacy)
+						if pkt.IsVideo {
+							// Write the last packet before the rollover keyframe.
+							if err := mp4Video.AddSampleToTrack(videoTrack, pkt.IsKeyFrame, pkt.Data, pts); err != nil {
+								log.Log.Error("capture.main.HandleRecordStream(continuous): " + err.Error())
+							}
+						} else if pkt.IsAudio {
+							if err := audioWriter.WritePacket(pkt); err != nil {
 								log.Log.Error("capture.main.HandleRecordStream(continuous): " + err.Error())
 							}
-						} else if pkt.Codec == "PCM_MULAW" {
-							// TODO: transcode to AAC, some work to do..
-							log.Log.Debug("capture.main.HandleRecordStream(continuous): no AAC audio codec detected, skipping audio track.")
 						}
 					}
 
+					if err := audioWriter.Flush(); err != nil {
+						log.Log.Error("capture.main.HandleRecordStream(continuous): " + err.Error())
+					}
+					audioWriter.Close()
+					audioWriter = nil
+
 					// Close mp4
 					if len(mp4Video.SPSNALUs) == 0 && len(configuration.Config.Capture.IPCamera.SPSNALUs) > 0 {
 						mp4Video.SPSNALUs = configuration.Config.Capture.IPCamera.SPSNALUs
@@ -177,6 +210,8 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 
 					// Cleanup muxer
 					start = false
+					rolloverRequested = false
+					rolloverMaxLogged = false
 
 					// Update the name of the recording with the duration.
 					// We will update the name of the recording with the duration in milliseconds.
@@ -305,11 +340,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 					} else if videoCodec == "H265" {
 						videoTrack = mp4Video.AddVideoTrack("H265")
 					}
-					if audioCodec == "AAC" {
-						audioTrack = mp4Video.AddAudioTrack("AAC")
-					} else if audioCodec == "PCM_MULAW" {
-						log.Log.Debug("capture.main.HandleRecordStream(continuous): no AAC audio codec detected, skipping audio track.")
-					}
+					audioWriter = newRecordingAudioWriter(mp4Video, audioCodec, config.Capture.IPCamera.SampleRate, config.Capture.IPCamera.Channels, audioBitDepth, "capture.main.HandleRecordStream(continuous)")
 
 					pts := convertPTS(pkt.TimeLegacy)
 					if pkt.IsVideo {
@@ -317,15 +348,8 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 							log.Log.Error("capture.main.HandleRecordStream(continuous): " + err.Error())
 						}
 					} else if pkt.IsAudio {
-						if pkt.Codec == "AAC" {
-							if err := mp4Video.AddSampleToTrack(audioTrack, pkt.IsKeyFrame, pkt.Data, pts); err != nil {
-								log.Log.Error("capture.main.HandleRecordStream(continuous): " + err.Error())
-							}
-						} else if pkt.Codec == "PCM_MULAW" {
-							// TODO: transcode to AAC, some work to do..
-							// We might need to use ffmpeg to transcode the audio to AAC.
-							// For now we will skip the audio track.
-							log.Log.Debug("capture.main.HandleRecordStream(continuous): no AAC audio codec detected, skipping audio track.")
+						if err := audioWriter.WritePacket(pkt); err != nil {
+							log.Log.Error("capture.main.HandleRecordStream(continuous): " + err.Error())
 						}
 					}
 					recordingStatus = "started"
@@ -339,13 +363,8 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 							log.Log.Error("capture.main.HandleRecordStream(continuous): " + err.Error())
 						}
 					} else if pkt.IsAudio {
-						if pkt.Codec == "AAC" {
-							if err := mp4Video.AddSampleToTrack(audioTrack, pkt.IsKeyFrame, pkt.Data, pts); err != nil {
-								log.Log.Error("capture.main.HandleRecordStream(continuous): " + err.Error())
-							}
-						} else if pkt.Codec == "PCM_MULAW" {
-							// TODO: transcode to AAC, some work to do..
-							log.Log.Debug("capture.main.HandleRecordStream(continuous): no AAC audio codec detected, skipping audio track.")
+						if err := audioWriter.WritePacket(pkt); err != nil {
+							log.Log.Error("capture.main.HandleRecordStream(continuous): " + err.Error())
 						}
 					}
 				}
@@ -355,6 +374,10 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 			// We might have interrupted the recording while restarting the agent.
 			// If this happens we need to check to properly close the recording.
 			if cursorError != nil {
+				if audioWriter != nil {
+					audioWriter.Close()
+					audioWriter = nil
+				}
 				if recordingStatus == "started" {
 
 					log.Log.Info("capture.main.HandleRecordStream(continuous): Recording finished: file save: " + name)
@@ -434,7 +457,6 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 			var displayTime int64 = 0       // display time in milliseconds
 
 			var videoTrack uint32
-			var audioTrack uint32
 
 			for motion := range communication.HandleMotion {
 
@@ -448,6 +470,8 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 				motionTimestamp := now
 
 				start := false
+				rolloverRequested := false
+				rolloverMaxLogged := false
 
 				if cursorError == nil {
 					pkt, cursorError = recordingCursor.ReadPacket()
@@ -520,6 +544,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 				}
 				// Create the MP4 only once the first keyframe arrives.
 				var mp4Video *video.MP4
+				var audioWriter *recordingAudioWriter
 
 				for cursorError == nil {
 
@@ -528,20 +553,33 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 						log.Log.Error("capture.main.HandleRecordStream(motiondetection): " + cursorError.Error())
 					}
 
-					now = time.Now().UnixMilli()
 					select {
 					case motion := <-communication.HandleMotion:
-						motionTimestamp = now
+						motionTimestamp = pkt.CurrentTime
 						log.Log.Info("capture.main.HandleRecordStream(motiondetection): motion detected while recording. Expanding recording.")
 						numberOfChanges := motion.NumberOfChanges
 						log.Log.Info("capture.main.HandleRecordStream(motiondetection): Received message with recording data, detected changes to save: " + strconv.Itoa(numberOfChanges))
 					default:
 					}
 
-					if start && (motionTimestamp+postRecording-now < 0 || now-startRecording > maxRecordingPeriod-500) && nextPkt.IsKeyFrame {
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): timestamp+postRecording-now < 0  - " + strconv.FormatInt(motionTimestamp+postRecording-now, 10) + " < 0")
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): now-startRecording > maxRecordingPeriod-500 - " + strconv.FormatInt(now-startRecording, 10) + " > " + strconv.FormatInt(maxRecordingPeriod-500, 10))
-						log.Log.Info("capture.main.HandleRecordStream(motiondetection): closing recording (timestamp: " + strconv.FormatInt(motionTimestamp, 10) + ", postRecording: " + strconv.FormatInt(postRecording, 10) + ", now: " + strconv.FormatInt(now, 10) + ", startRecording: " + strconv.FormatInt(startRecording, 10) + ", maxRecordingPeriod: " + strconv.FormatInt(maxRecordingPeriod, 10))
+					packetTime := pkt.CurrentTime
+					hardMaxReached := packetTime-startRecording > maxRecordingPeriod-500
+					postRecordingElapsed := motionTimestamp+postRecording-packetTime < 0
+					if start && (postRecordingElapsed || hardMaxReached) {
+						rolloverRequested = true
+						if hardMaxReached && !rolloverMaxLogged {
+							log.Log.Info("capture.main.HandleRecordStream(motiondetection): max recording period reached, waiting for next keyframe to close without dropping frames")
+							rolloverMaxLogged = true
+						}
+					}
+
+					closeOnCurrentKeyframe := rolloverRequested && pkt.IsKeyFrame && pkt.CurrentTime > startRecording
+					closeOnNextKeyframe := rolloverRequested && nextPkt.IsKeyFrame
+
+					if start && (closeOnCurrentKeyframe || closeOnNextKeyframe) {
+						log.Log.Info("capture.main.HandleRecordStream(motiondetection): timestamp+postRecording-packetTime < 0  - " + strconv.FormatInt(motionTimestamp+postRecording-packetTime, 10) + " < 0")
+						log.Log.Info("capture.main.HandleRecordStream(motiondetection): packetTime-startRecording > maxRecordingPeriod-500 - " + strconv.FormatInt(packetTime-startRecording, 10) + " > " + strconv.FormatInt(maxRecordingPeriod-500, 10))
+						log.Log.Info("capture.main.HandleRecordStream(motiondetection): closing recording (timestamp: " + strconv.FormatInt(motionTimestamp, 10) + ", postRecording: " + strconv.FormatInt(postRecording, 10) + ", packetTime: " + strconv.FormatInt(packetTime, 10) + ", startRecording: " + strconv.FormatInt(startRecording, 10) + ", maxRecordingPeriod: " + strconv.FormatInt(maxRecordingPeriod, 10))
 						break
 					}
 					if pkt.IsKeyFrame && !start && pkt.CurrentTime >= startRecording {
@@ -550,7 +588,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 						log.Log.Debug("capture.main.HandleRecordStream(motiondetection): write frames")
 						log.Log.Debug("capture.main.HandleRecordStream(motiondetection): recording started on keyframe")
 
-						// Align duration timers with the first keyframe.
+						// Align duration timers with the first keyframe so audio/video start together.
 						startRecording = pkt.CurrentTime
 
 						// Create a video file, and set the dimensions.
@@ -563,11 +601,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 						} else if videoCodec == "H265" {
 							videoTrack = mp4Video.AddVideoTrack("H265")
 						}
-						if audioCodec == "AAC" {
-							audioTrack = mp4Video.AddAudioTrack("AAC")
-						} else if audioCodec == "PCM_MULAW" {
-							log.Log.Debug("capture.main.HandleRecordStream(continuous): no AAC audio codec detected, skipping audio track.")
-						}
+						audioWriter = newRecordingAudioWriter(mp4Video, audioCodec, config.Capture.IPCamera.SampleRate, config.Capture.IPCamera.Channels, audioBitDepth, "capture.main.HandleRecordStream(motiondetection)")
 						start = true
 					}
 					if start {
@@ -581,17 +615,10 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 							}
 						} else if pkt.IsAudio {
 							log.Log.Debug("capture.main.HandleRecordStream(motiondetection): add audio sample")
-							if pkt.Codec == "AAC" {
-								if mp4Video != nil {
-									if err := mp4Video.AddSampleToTrack(audioTrack, pkt.IsKeyFrame, pkt.Data, pts); err != nil {
-										log.Log.Error("capture.main.HandleRecordStream(motiondetection): " + err.Error())
-									}
+							if mp4Video != nil {
+								if err := audioWriter.WritePacket(pkt); err != nil {
+									log.Log.Error("capture.main.HandleRecordStream(motiondetection): " + err.Error())
 								}
-							} else if pkt.Codec == "PCM_MULAW" {
-								// TODO: transcode to AAC, some work to do..
-								// We might need to use ffmpeg to transcode the audio to AAC.
-								// For now we will skip the audio track.
-								log.Log.Debug("capture.main.HandleRecordStream(motiondetection): no AAC audio codec detected, skipping audio track.")
 							}
 						}
 					}
@@ -604,10 +631,18 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
 				lastRecordingTime = pkt.CurrentTime
 
 				if mp4Video == nil {
+					if audioWriter != nil {
+						audioWriter.Close()
+					}
 					log.Log.Warning("capture.main.HandleRecordStream(motiondetection): recording closed without keyframe; no MP4 created")
 					continue
 				}
 
+				if err := audioWriter.Flush(); err != nil {
+					log.Log.Error("capture.main.HandleRecordStream(motiondetection): " + err.Error())
+				}
+				audioWriter.Close()
+
 				// This will close the recording and write the last packet.
 				if len(mp4Video.SPSNALUs) == 0 && len(configuration.Config.Capture.IPCamera.SPSNALUs) > 0 {
 					mp4Video.SPSNALUs = configuration.Config.Capture.IPCamera.SPSNALUs

machinery/src/capture/pcmu_to_aac_transcoder.go

@@ -0,0 +1,566 @@
+package capture
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"os/exec"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/kerberos-io/agent/machinery/src/log"
+	"github.com/kerberos-io/agent/machinery/src/packets"
+	"github.com/kerberos-io/agent/machinery/src/video"
+)
+
+const (
+	defaultPCMUSampleRate = 8000
+	defaultPCMUChannels   = 1
+)
+
+type audioToAACTranscoder interface {
+	Transcode([]byte) ([]byte, error)
+	Flush() ([]byte, error)
+	Close()
+}
+
+func PCMUToAACTranscodingAvailable() bool {
+	_, err := exec.LookPath("ffmpeg")
+	return err == nil
+}
+
+type ffmpegToAACTranscoder struct {
+	cmd    *exec.Cmd
+	stdin  io.WriteCloser
+	stdout io.ReadCloser
+	stderr *bytes.Buffer
+
+	mu          sync.Mutex
+	outMu       sync.Mutex
+	outBuf      bytes.Buffer
+	adtsBuf     []byte
+	closed      bool
+	stdinClosed bool
+	closeOnce   sync.Once
+	stdoutDone  chan struct{}
+	waitDone    chan struct{}
+	waitErr     error
+}
+
+func newFFmpegToAACTranscoder(inputFormat string, sampleRate int, channels int) (*ffmpegToAACTranscoder, error) {
+	ffmpegPath, err := exec.LookPath("ffmpeg")
+	if err != nil {
+		return nil, errors.New("audio to AAC transcoding not available: ffmpeg binary not found in PATH")
+	}
+
+	cmd := exec.Command(
+		ffmpegPath,
+		"-hide_banner",
+		"-loglevel", "error",
+		"-fflags", "+nobuffer",
+		"-flags", "low_delay",
+		"-f", inputFormat,
+		"-ar", intToString(sampleRate),
+		"-ac", intToString(channels),
+		"-i", "pipe:0",
+		"-vn",
+		"-ac", "1",
+		"-ar", intToString(sampleRate),
+		"-c:a", "aac",
+		"-profile:a", "aac_low",
+		"-b:a", "32k",
+		"-f", "adts",
+		"pipe:1",
+	)
+
+	stdin, err := cmd.StdinPipe()
+	if err != nil {
+		return nil, err
+	}
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, err
+	}
+	stderr := &bytes.Buffer{}
+	cmd.Stderr = stderr
+
+	if err := cmd.Start(); err != nil {
+		return nil, err
+	}
+
+	t := &ffmpegToAACTranscoder{
+		cmd:        cmd,
+		stdin:      stdin,
+		stdout:     stdout,
+		stderr:     stderr,
+		stdoutDone: make(chan struct{}),
+		waitDone:   make(chan struct{}),
+	}
+
+	go func() {
+		defer close(t.stdoutDone)
+		buf := make([]byte, 4096)
+		for {
+			n, readErr := stdout.Read(buf)
+			if n > 0 {
+				t.outMu.Lock()
+				_, _ = t.outBuf.Write(buf[:n])
+				t.outMu.Unlock()
+			}
+			if readErr != nil {
+				if readErr != io.EOF {
+					log.Log.Warning("capture.pcmu_to_aac: stdout reader stopped: " + readErr.Error())
+				}
+				return
+			}
+		}
+	}()
+
+	go func() {
+		t.waitErr = cmd.Wait()
+		close(t.waitDone)
+	}()
+
+	log.Log.Info("capture.audio_to_aac: " + strings.ToUpper(inputFormat) + " -> AAC transcoder initialised (ffmpeg process)")
+	return t, nil
+}
+
+func NewPCMUToAACTranscoder(sampleRate int, channels int) (*ffmpegToAACTranscoder, error) {
+	if sampleRate <= 0 {
+		sampleRate = defaultPCMUSampleRate
+	}
+	if channels <= 0 {
+		channels = defaultPCMUChannels
+	}
+
+	return newFFmpegToAACTranscoder("mulaw", sampleRate, channels)
+}
+
+func NewLPCMToAACTranscoder(sampleRate int, channels int, bitDepth int) (*ffmpegToAACTranscoder, error) {
+	inputFormat, err := lpcmFFmpegInputFormat(bitDepth)
+	if err != nil {
+		return nil, err
+	}
+	if sampleRate <= 0 {
+		return nil, errors.New("LPCM to AAC transcoding requires a valid sample rate")
+	}
+	if channels <= 0 {
+		return nil, errors.New("LPCM to AAC transcoding requires a valid channel count")
+	}
+
+	return newFFmpegToAACTranscoder(inputFormat, sampleRate, channels)
+}
+
+func (t *ffmpegToAACTranscoder) Transcode(input []byte) ([]byte, error) {
+	if t == nil || len(input) == 0 {
+		return nil, nil
+	}
+
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	if t.closed {
+		return nil, errors.New("audio to AAC transcoder is closed")
+	}
+	if t.stdinClosed {
+		return nil, errors.New("audio to AAC transcoder input is closed")
+	}
+
+	if _, err := t.stdin.Write(input); err != nil {
+		return nil, err
+	}
+
+	// Do not block the recording loop waiting for the encoder to emit output.
+	// FFmpeg can buffer for a while, and polling here per RTP packet causes the
+	// recorder to fall behind real time and drop trailing media before close.
+	return t.readAvailable(), nil
+}
+
+func (t *ffmpegToAACTranscoder) Flush() ([]byte, error) {
+	if t == nil {
+		return nil, nil
+	}
+
+	t.mu.Lock()
+	if t.closed {
+		defer t.mu.Unlock()
+		return t.readAvailable(), nil
+	}
+	if !t.stdinClosed && t.stdin != nil {
+		if err := t.stdin.Close(); err != nil {
+			t.mu.Unlock()
+			return nil, err
+		}
+		t.stdinClosed = true
+	}
+	t.mu.Unlock()
+
+	processExited := false
+	readerFinished := false
+	deadline := time.Now().Add(15 * time.Second)
+	timedOut := false
+	for !processExited || !readerFinished {
+		if !processExited {
+			select {
+			case <-t.waitDone:
+				processExited = true
+			default:
+			}
+		}
+		if !readerFinished {
+			select {
+			case <-t.stdoutDone:
+				readerFinished = true
+			default:
+			}
+		}
+
+		if processExited && readerFinished {
+			break
+		}
+		if time.Now().After(deadline) {
+			timedOut = true
+			break
+		}
+
+		time.Sleep(15 * time.Millisecond)
+	}
+
+	if timedOut {
+		log.Log.Warning("capture.audio_to_aac: flush timed out before ffmpeg fully drained (process_exited=" + strconv.FormatBool(processExited) + ", stdout_done=" + strconv.FormatBool(readerFinished) + ", buffered=" + intToString(t.bufferedLen()) + ")")
+	}
+
+	if processExited && t.waitErr != nil {
+		return t.readAvailable(), t.waitErr
+	}
+
+	return t.readAvailable(), nil
+}
+
+func (t *ffmpegToAACTranscoder) Close() {
+	if t == nil {
+		return
+	}
+
+	t.closeOnce.Do(func() {
+		t.mu.Lock()
+		t.closed = true
+		if t.stdin != nil && !t.stdinClosed {
+			_ = t.stdin.Close()
+			t.stdinClosed = true
+		}
+		t.mu.Unlock()
+
+		processExited := false
+		waitTimeout := 250 * time.Millisecond
+		if t.stdinClosed {
+			waitTimeout = 2 * time.Second
+		}
+		select {
+		case <-t.waitDone:
+			processExited = true
+		case <-time.After(waitTimeout):
+		}
+
+		if !processExited {
+			if t.stdout != nil {
+				_ = t.stdout.Close()
+			}
+			if t.cmd != nil && t.cmd.Process != nil {
+				_ = t.cmd.Process.Kill()
+				<-t.waitDone
+			}
+		}
+
+		if stderr := t.stderrString(); stderr != "" {
+			log.Log.Info("capture.audio_to_aac: ffmpeg stderr on close: " + stderr)
+		}
+	})
+}
+
+func (t *ffmpegToAACTranscoder) readAvailable() []byte {
+	t.outMu.Lock()
+	defer t.outMu.Unlock()
+
+	if t.outBuf.Len() > 0 {
+		t.adtsBuf = append(t.adtsBuf, t.outBuf.Bytes()...)
+		t.outBuf.Reset()
+	}
+
+	return drainCompleteADTSFrames(&t.adtsBuf)
+}
+
+func (t *ffmpegToAACTranscoder) bufferedLen() int {
+	t.outMu.Lock()
+	defer t.outMu.Unlock()
+	return t.outBuf.Len()
+}
+
+func (t *ffmpegToAACTranscoder) stderrString() string {
+	if t == nil || t.stderr == nil {
+		return ""
+	}
+	return strings.TrimSpace(t.stderr.String())
+}
+
+func lpcmFFmpegInputFormat(bitDepth int) (string, error) {
+	switch bitDepth {
+	case 8:
+		return "u8", nil
+	case 16:
+		return "s16be", nil
+	case 24:
+		return "s24be", nil
+	default:
+		return "", errors.New("unsupported LPCM bit depth: " + intToString(bitDepth))
+	}
+}
+
+type recordingAudioWriter struct {
+	mp4        *video.MP4
+	trackID    uint32
+	transcoder audioToAACTranscoder
+	lastPTS    uint64
+	logPrefix  string
+
+	transcodedSampleRate int
+	aacBasePTS           uint64
+	aacFrameCursor       uint64
+	aacClockStarted      bool
+	loggedAACParams      bool
+}
+
+func newRecordingAudioWriter(mp4Video *video.MP4, audioCodec string, sampleRate int, channels int, bitDepth int, logPrefix string) *recordingAudioWriter {
+	writer := &recordingAudioWriter{
+		mp4:       mp4Video,
+		logPrefix: logPrefix,
+	}
+
+	switch audioCodec {
+	case "AAC":
+		writer.trackID = mp4Video.AddAudioTrack("AAC")
+	case "PCM_MULAW":
+		if sampleRate <= 0 {
+			sampleRate = defaultPCMUSampleRate
+		}
+		if channels <= 0 {
+			channels = defaultPCMUChannels
+		}
+
+		if !PCMUToAACTranscodingAvailable() {
+			log.Log.Warning(logPrefix + ": ffmpeg not available, skipping PCM_MULAW audio recording.")
+			return writer
+		}
+
+		transcoder, err := NewPCMUToAACTranscoder(sampleRate, channels)
+		if err != nil {
+			log.Log.Error(logPrefix + ": failed to create PCM_MULAW to AAC transcoder: " + err.Error())
+			return writer
+		}
+
+		writer.trackID = mp4Video.AddAudioTrack("AAC")
+		writer.transcoder = transcoder
+		writer.transcodedSampleRate = sampleRate
+		log.Log.Info(logPrefix + ": recording PCM_MULAW audio as AAC (input_rate=" + intToString(sampleRate) + ", channels=" + intToString(channels) + ").")
+	case "LPCM":
+		transcoder, err := NewLPCMToAACTranscoder(sampleRate, channels, bitDepth)
+		if err != nil {
+			log.Log.Error(logPrefix + ": failed to create LPCM to AAC transcoder: " + err.Error())
+			return writer
+		}
+
+		writer.trackID = mp4Video.AddAudioTrack("AAC")
+		writer.transcoder = transcoder
+		writer.transcodedSampleRate = sampleRate
+		log.Log.Info(logPrefix + ": recording LPCM audio as AAC (input_rate=" + intToString(sampleRate) + ", channels=" + intToString(channels) + ", bit_depth=" + intToString(bitDepth) + ").")
+	}
+
+	return writer
+}
+
+func (w *recordingAudioWriter) TrackID() uint32 {
+	if w == nil {
+		return 0
+	}
+	return w.trackID
+}
+
+func (w *recordingAudioWriter) WritePacket(pkt packets.Packet) error {
+	if w == nil || w.mp4 == nil || !pkt.IsAudio || w.trackID == 0 {
+		return nil
+	}
+
+	pts := convertPTS(pkt.TimeLegacy)
+	if pts > 0 {
+		w.lastPTS = pts
+	}
+
+	switch pkt.Codec {
+	case "AAC":
+		return w.mp4.AddSampleToTrack(w.trackID, pkt.IsKeyFrame, pkt.Data, pts)
+	case "PCM_MULAW", "LPCM":
+		if w.transcoder == nil {
+			return nil
+		}
+		if !w.aacClockStarted {
+			w.aacBasePTS = pts
+			w.aacClockStarted = true
+		}
+
+		adts, err := w.transcoder.Transcode(pkt.Data)
+		if err != nil {
+			return err
+		}
+		if len(adts) == 0 {
+			return nil
+		}
+
+		return w.writeTranscodedADTS(adts)
+	default:
+		return nil
+	}
+}
+
+func (w *recordingAudioWriter) Flush() error {
+	if w == nil || w.transcoder == nil || w.mp4 == nil || w.trackID == 0 {
+		return nil
+	}
+
+	adts, err := w.transcoder.Flush()
+	if err != nil {
+		return err
+	}
+	if len(adts) == 0 {
+		return nil
+	}
+
+	return w.writeTranscodedADTS(adts)
+}
+
+func (w *recordingAudioWriter) Close() {
+	if w != nil && w.transcoder != nil {
+		w.transcoder.Close()
+		w.transcoder = nil
+	}
+}
+
+func (w *recordingAudioWriter) writeTranscodedADTS(adts []byte) error {
+	if w == nil || w.mp4 == nil || w.trackID == 0 || len(adts) == 0 {
+		return nil
+	}
+	if w.transcodedSampleRate <= 0 {
+		return errors.New("transcoded AAC sample rate is not set")
+	}
+
+	var writeErr error
+	video.SplitAACFrame(adts, func(started bool, aac []byte) {
+		if writeErr != nil || len(aac) < 7 {
+			return
+		}
+
+		if !w.loggedAACParams {
+			log.Log.Info(w.logPrefix + ": first AAC frame parameters (aac_rate=" + intToString(int(video.AACSampleRateFromADTS(aac))) + ", channels=" + intToString(int(video.AACChannelCountFromADTS(aac))) + ", samples_per_frame=" + intToString(aacSamplesPerFrame(aac)) + ").")
+			w.loggedAACParams = true
+		}
+
+		pts := w.transcodedPTS()
+		if err := w.mp4.AddSampleToTrack(w.trackID, false, aac, pts); err != nil {
+			writeErr = err
+			return
+		}
+
+		w.lastPTS = pts
+		w.aacFrameCursor += uint64(aacSamplesPerFrame(aac))
+	})
+
+	return writeErr
+}
+
+func (w *recordingAudioWriter) transcodedPTS() uint64 {
+	if w == nil {
+		return 0
+	}
+	if !w.aacClockStarted {
+		w.aacClockStarted = true
+	}
+	return w.aacBasePTS + (w.aacFrameCursor*1000)/uint64(w.transcodedSampleRate)
+}
+
+func aacSamplesPerFrame(aac []byte) int {
+	if len(aac) < 7 {
+		return 1024
+	}
+	rawBlocks := int(aac[6]&0x03) + 1
+	return rawBlocks * 1024
+}
+
+func intToString(v int) string {
+	return strconv.Itoa(v)
+}
+
+func drainCompleteADTSFrames(buffer *[]byte) []byte {
+	if buffer == nil || len(*buffer) == 0 {
+		return nil
+	}
+
+	data := *buffer
+	start := video.FindSyncword(data, 0)
+	if start < 0 {
+		// Keep the tail in case the syncword is split across reads.
+		if len(data) > 1 {
+			*buffer = append([]byte{}, data[len(data)-1:]...)
+		}
+		return nil
+	}
+	if start > 0 {
+		data = data[start:]
+	}
+
+	var out bytes.Buffer
+	offset := 0
+	for {
+		if len(data[offset:]) < 7 {
+			break
+		}
+
+		var adts video.ADTS_Frame_Header
+		adts.Decode(data[offset:])
+		frameLen := int(adts.Variable_Header.Frame_length)
+		if frameLen < 7 {
+			next := video.FindSyncword(data, offset+1)
+			if next < 0 {
+				break
+			}
+			offset = next
+			continue
+		}
+		if offset+frameLen > len(data) {
+			break
+		}
+
+		_, _ = out.Write(data[offset : offset+frameLen])
+		offset += frameLen
+
+		next := video.FindSyncword(data, offset)
+		if next < 0 {
+			break
+		}
+		if next > offset {
+			offset = next
+		}
+	}
+
+	if offset < len(data) {
+		*buffer = append([]byte{}, data[offset:]...)
+	} else {
+		*buffer = nil
+	}
+
+	if out.Len() == 0 {
+		return nil
+	}
+
+	return out.Bytes()
+}

machinery/src/packets/stream.go

@@ -49,6 +49,9 @@ type Stream struct {
 	// Channels is the number of audio channels.
 	Channels int
 
+	// BitDepth is the number of bits per audio sample for PCM-based streams.
+	BitDepth int
+
 	// GopSize is the size of the GOP (Group of Pictures).
 	GopSize int
 }

machinery/src/video/mp4.go

@@ -13,6 +13,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/Eyevinn/mp4ff/aac"
 	"github.com/Eyevinn/mp4ff/avc"
 	mp4ff "github.com/Eyevinn/mp4ff/mp4"
 	"github.com/kerberos-io/agent/machinery/src/encryption"
@@ -32,24 +33,6 @@ const MacEpochOffset uint64 = 2082844800
 // resulting in ~3 second fragments (assuming a typical GOP interval).
 const FragmentDurationMs = 3000
 
-// MinNormalGOPMs is the maximum spacing between two consecutive IDRs that
-// we still consider an anomalous "loop/restart seam". When two keyframes
-// arrive closer than this, we treat the second one as an upstream
-// restart/loop-seam and force a fresh fragment so the seam IDR cannot end
-// up as a mid-fragment sync sample. The check only runs when the current
-// fragment has not yet reached FragmentDurationMs.
-//
-// This must be set well below the smallest plausible *legitimate* GOP
-// length. Typical IP cameras use GOP intervals of 1000-2000 ms, and the
-// arrival timing of consecutive IDRs can jitter by a few hundred ms due to
-// network/RTSP buffering. A threshold close to 1 s (e.g. 950) caused
-// false positives on cameras with ~1 s GOPs (warnings like
-// "gap=800 ms / 300 ms / 200 ms" while the stream itself was healthy).
-// 400 ms is comfortably below any realistic GOP yet still catches the
-// virtual-rtsp / ffmpeg loop-seam pattern (seam IDRs typically arrive
-// 100-200 ms after the prior IDR).
-const MinNormalGOPMs = 400
-
 type MP4 struct {
 	// FileName is the name of the file
 	FileName                string
@@ -64,9 +47,12 @@ type MP4 struct {
 	SegmentCount            int
 	SampleCount             int
 	StartPTS                uint64
+	MediaStartPTS           uint64
 	VideoTotalDuration      uint64
 	AudioTotalDuration      uint64
 	AudioPTS                uint64
+	AudioSampleRate         uint32
+	AudioChannels           uint16
 	Start                   bool
 	SPSNALUs                [][]byte // SPS NALUs for H264
 	PPSNALUs                [][]byte // PPS NALUs for H264
@@ -74,8 +60,6 @@ type MP4 struct {
 	FreeBoxSize             int64
 	FragmentStartRawPTS     uint64            // Raw PTS for timing when to flush fragments
 	FragmentStartDTS        uint64            // Accumulated VideoTotalDuration at fragment start (matches tfdt)
-	LastKeyframeRawPTS      uint64            // Raw PTS of the most recently seen keyframe (in any fragment)
-	LastKeyframeGapMs       uint64            // Gap (ms) between the previous two consecutive keyframes
 	MoofBoxes               int64             // Number of moof boxes in the file
 	MoofBoxSizes            []int64           // Sizes of each moof box
 	SegmentDurations        []uint64          // Duration of each segment in timescale units
@@ -253,24 +237,6 @@ func (mp4 *MP4) flushPendingVideoSample(nextPTS uint64) bool {
 	var duration uint64
 	if nextPTS > 0 && nextPTS > mp4.VideoFullSample.DecodeTime {
 		duration = nextPTS - mp4.VideoFullSample.DecodeTime
-		// Guard against forward PTS jumps (e.g. when looping a source MP4
-		// through virtual-rtsp the upstream ffmpeg may insert a large offset
-		// at the loop boundary, or the RTSP stream may stall briefly).
-		// Without this clamp the sample gets a huge duration which appears
-		// as a discontinuity in the trun/sidx/mvhd and causes browsers
-		// (Video.js / MSE) to abort playback with a "media corruption"
-		// error around the loop boundary.
-		var maxPlausible uint64 = 1000 // 1 second hard ceiling
-		if mp4.LastVideoSampleDTS > 0 && mp4.LastVideoSampleDTS*10 < maxPlausible {
-			maxPlausible = mp4.LastVideoSampleDTS * 10
-		}
-		if duration > maxPlausible {
-			log.Log.Warning(fmt.Sprintf("mp4.flushPendingVideoSample(): video PTS jumped forward (nextPTS=%d, prevDTS=%d, gap=%d ms) - clamping to %d ms (likely source loop/stall discontinuity)", nextPTS, mp4.VideoFullSample.DecodeTime, duration, maxPlausible))
-			duration = mp4.LastVideoSampleDTS
-			if duration == 0 {
-				duration = 33
-			}
-		}
 	} else {
 		// No valid nextPTS (Close case) or PTS went backwards (jitter/discontinuity)
 		if nextPTS > 0 {
@@ -304,6 +270,49 @@ func (mp4 *MP4) flushPendingVideoSample(nextPTS uint64) bool {
 	return true
 }
 
+func (mp4 *MP4) alignAudioTimeline(rawPTS uint64) {
+	if mp4 == nil || mp4.AudioSampleRate == 0 || mp4.AudioPTS != 0 {
+		return
+	}
+	if rawPTS <= mp4.MediaStartPTS {
+		return
+	}
+
+	offsetMs := rawPTS - mp4.MediaStartPTS
+	mp4.AudioPTS = (offsetMs*uint64(mp4.AudioSampleRate) + 500) / 1000
+}
+
+func (mp4 *MP4) appendPendingAudioSample(trackID uint32) {
+	if mp4 == nil || mp4.AudioFullSample == nil {
+		return
+	}
+
+	SplitAACFrame(mp4.AudioFullSample.Data, func(started bool, aac []byte) {
+		sampleToAdd := *mp4.AudioFullSample
+		dts := aacFrameDurationSamples(aac)
+		if dts == 0 {
+			dts = mp4.LastAudioSampleDTS
+			if dts == 0 {
+				dts = 1024
+			}
+		}
+
+		mp4.alignAudioTimeline(sampleToAdd.DecodeTime)
+		mp4.LastAudioSampleDTS = dts
+		sampleToAdd.Data = aac[7:]
+		sampleToAdd.DecodeTime = mp4.AudioPTS
+		sampleToAdd.Sample.Dur = uint32(dts)
+		sampleToAdd.Sample.Size = uint32(len(aac[7:]))
+		mp4.AudioTotalDuration += dts
+		mp4.AudioPTS += dts
+
+		err := mp4.MultiTrackFragment.AddFullSampleToTrack(sampleToAdd, trackID)
+		if err != nil {
+			log.Log.Error("mp4.appendPendingAudioSample(): error adding sample to track " + fmt.Sprintf("%d: %v", trackID, err))
+		}
+	})
+}
+
 func (mp4 *MP4) AddSampleToTrack(trackID uint32, isKeyframe bool, data []byte, pts uint64) error {
 
 	if isKeyframe && trackID == uint32(mp4.VideoTrack) {
@@ -327,37 +336,6 @@ func (mp4 *MP4) AddSampleToTrack(trackID uint32, isKeyframe bool, data []byte, p
 		}
 		shouldFlush := !mp4.Start || elapsed >= FragmentDurationMs
 
-		// Detect upstream source-loop / restart discontinuity. When an MP4 is
-		// looped through virtual-rtsp (ffmpeg `-stream_loop -1 -re`) the loop
-		// seam emits a fresh IDR much sooner than a normal GOP would. PTS keeps
-		// growing monotonically, so the timing-only `elapsed` check above does
-		// not catch it and the seam IDR ends up as a mid-fragment sync sample.
-		// MSE-based players (Video.js / Chromium / Firefox) reject the resulting
-		// fragment with a "media corruption" error because the inner IDR resets
-		// frame_num/POC inside what they expect to be a single GOP. Force a
-		// fragment boundary whenever two consecutive keyframes arrive much
-		// closer than a normal GOP.
-		//
-		// We only flag this as a seam when it is a *sudden* anomaly: the
-		// previous keyframe gap must have been healthy (>= MinNormalGOPMs).
-		// This avoids false positives on cameras that legitimately emit
-		// short-interval IDRs (short GOP, motion-triggered recovery IDRs,
-		// all-intra streams) where every keyframe would otherwise be flagged
-		// in a cascade, producing many tiny fragments and log spam.
-		if trackID == uint32(mp4.VideoTrack) && mp4.Start &&
-			mp4.LastKeyframeRawPTS > 0 && pts > mp4.LastKeyframeRawPTS {
-			gap := pts - mp4.LastKeyframeRawPTS
-			if !shouldFlush && gap < MinNormalGOPMs &&
-				(mp4.LastKeyframeGapMs == 0 || mp4.LastKeyframeGapMs >= MinNormalGOPMs) {
-				log.Log.Warning(fmt.Sprintf("mp4.AddSampleToTrack(): forcing fragment flush at unexpectedly close keyframe (gap=%d ms, fragment elapsed=%d ms) - likely upstream loop/restart discontinuity", gap, elapsed))
-				shouldFlush = true
-			}
-			mp4.LastKeyframeGapMs = gap
-		}
-		if trackID == uint32(mp4.VideoTrack) {
-			mp4.LastKeyframeRawPTS = pts
-		}
-
 		if shouldFlush {
 			// Write the previous segment to the file
 			if mp4.Start {
@@ -389,6 +367,9 @@ func (mp4 *MP4) AddSampleToTrack(trackID uint32, isKeyframe bool, data []byte, p
 
 			// Increment the segment count
 			mp4.SegmentCount = mp4.SegmentCount + 1
+			if mp4.MediaStartPTS == 0 {
+				mp4.MediaStartPTS = pts
+			}
 
 			// Create a new media segment
 			seg := mp4ff.NewMediaSegment()
@@ -452,37 +433,7 @@ func (mp4 *MP4) AddSampleToTrack(trackID uint32, isKeyframe bool, data []byte, p
 			}
 		} else if trackID == uint32(mp4.AudioTrack) {
 			if mp4.AudioFullSample != nil {
-				SplitAACFrame(mp4.AudioFullSample.Data, func(started bool, aac []byte) {
-					sampleToAdd := *mp4.AudioFullSample
-					dts := pts - mp4.AudioFullSample.DecodeTime
-					if pts < mp4.AudioFullSample.DecodeTime {
-						//log.Printf("Warning: PTS %d is less than previous sample's DecodeTime %d, resetting AudioFullSample", pts, mp4.AudioFullSample.DecodeTime)
-						dts = 1
-					}
-					if started {
-						dts = 1
-					}
-					// Guard against forward PTS jumps (e.g. virtual-rtsp loop
-					// boundary or upstream stalls). Without this clamp the
-					// audio trun would carry an enormous sample duration that
-					// renders the recording unplayable in browsers.
-					if mp4.LastAudioSampleDTS > 0 && dts > mp4.LastAudioSampleDTS*10 {
-						log.Log.Warning(fmt.Sprintf("mp4.AddSampleToTrack(): audio PTS jumped forward (pts=%d, prevDTS=%d, gap=%d) - clamping to last known duration", pts, mp4.AudioFullSample.DecodeTime, dts))
-						dts = mp4.LastAudioSampleDTS
-					}
-					mp4.LastAudioSampleDTS = dts
-					//fmt.Printf("Adding sample to track %d, PTS: %d, Duration: %d, size: %d\n", trackID, pts, dts, len(aac[7:]))
-					mp4.AudioTotalDuration += dts
-					mp4.AudioPTS += dts
-					sampleToAdd.Data = aac[7:] // Remove the ADTS header (first 7 bytes)
-					sampleToAdd.DecodeTime = mp4.AudioPTS - dts
-					sampleToAdd.Sample.Dur = uint32(dts)
-					sampleToAdd.Sample.Size = uint32(len(aac[7:]))
-					err := mp4.MultiTrackFragment.AddFullSampleToTrack(sampleToAdd, trackID)
-					if err != nil {
-						log.Log.Error("mp4.AddSampleToTrack(): error adding sample to track " + fmt.Sprintf("%d: %v", trackID, err))
-					}
-				})
+				mp4.appendPendingAudioSample(trackID)
 			}
 
 			// Set the sample data
@@ -495,6 +446,12 @@ func (mp4 *MP4) AddSampleToTrack(trackID uint32, isKeyframe bool, data []byte, p
 				Flags:                 0,
 				CompositionTimeOffset: 0, // No composition time offset for audio
 			}
+			if mp4.AudioSampleRate == 0 {
+				mp4.AudioSampleRate = aacSampleRate(data)
+			}
+			if mp4.AudioChannels == 0 {
+				mp4.AudioChannels = aacChannelCount(data)
+			}
 			mp4.AudioFullSample = &fullSample
 			mp4.SampleType = "audio"
 		}
@@ -524,23 +481,7 @@ func (mp4 *MP4) Close(config *models.Config) {
 
 		// Add final audio sample if pending
 		if mp4.AudioFullSample != nil && mp4.AudioTrack > 0 {
-			SplitAACFrame(mp4.AudioFullSample.Data, func(started bool, aac []byte) {
-				sampleToAdd := *mp4.AudioFullSample
-				dts := mp4.LastAudioSampleDTS
-				if dts == 0 {
-					dts = 1024 // Default AAC frame duration
-				}
-				mp4.AudioTotalDuration += dts
-				mp4.AudioPTS += dts
-				sampleToAdd.Data = aac[7:]
-				sampleToAdd.DecodeTime = mp4.AudioPTS - dts
-				sampleToAdd.Sample.Dur = uint32(dts)
-				sampleToAdd.Sample.Size = uint32(len(aac[7:]))
-				err := mp4.MultiTrackFragment.AddFullSampleToTrack(sampleToAdd, uint32(mp4.AudioTrack))
-				if err != nil {
-					log.Log.Error("mp4.Close(): error adding final audio sample: " + err.Error())
-				}
-			})
+			mp4.appendPendingAudioSample(uint32(mp4.AudioTrack))
 			mp4.AudioFullSample = nil
 		}
 	}
@@ -603,15 +544,26 @@ func (mp4 *MP4) Close(config *models.Config) {
 	// QuickTime requires timestamps in Mac HFS format (seconds since 1904-01-01),
 	// so we convert from Unix epoch by adding MacEpochOffset.
 	videoTimescale := uint32(1000)
-	audioTimescale := uint32(1000)
+	audioTimescale := mp4.AudioSampleRate
+	if audioTimescale == 0 {
+		if config.Capture.IPCamera.SampleRate > 0 {
+			audioTimescale = uint32(config.Capture.IPCamera.SampleRate)
+		} else {
+			audioTimescale = uint32(1000)
+		}
+	}
 	macTime := mp4.StartTime + MacEpochOffset
 	nextTrackID := uint32(len(mp4.TrackIDs) + 1)
+	audioMovieDuration := mp4.AudioTotalDuration
+	if audioTimescale != 0 && audioTimescale != videoTimescale {
+		audioMovieDuration = (mp4.AudioTotalDuration*uint64(videoTimescale) + uint64(audioTimescale/2)) / uint64(audioTimescale)
+	}
 
 	// mvhd.Duration must be the duration of the longest track.
 	// Start with video; if audio is longer, we update below.
 	movDuration := actualVideoDuration
-	if mp4.AudioTotalDuration > movDuration {
-		movDuration = mp4.AudioTotalDuration
+	if audioMovieDuration > movDuration {
+		movDuration = audioMovieDuration
 	}
 
 	mvhd := &mp4ff.MvhdBox{
@@ -686,16 +638,23 @@ func (mp4 *MP4) Close(config *models.Config) {
 		// Add an audio track to the moov box
 		init.AddEmptyTrack(audioTimescale, "audio", "und")
 
-		// Check if the same sample rate is set, otherwise we default to 48000
-		audioSampleRate := 48000
-		if config.Capture.IPCamera.SampleRate > 0 {
-			audioSampleRate = config.Capture.IPCamera.SampleRate
+		audioSampleRate := int(audioTimescale)
+		if audioSampleRate == 0 {
+			audioSampleRate = 48000
 		}
-		// Set the audio descriptor
-		err := init.Moov.Traks[1].SetAACDescriptor(29, audioSampleRate)
+		audioChannels := mp4.AudioChannels
+		if audioChannels == 0 {
+			if config.Capture.IPCamera.Channels > 0 {
+				audioChannels = uint16(config.Capture.IPCamera.Channels)
+			} else {
+				audioChannels = 1
+			}
+		}
+		// Set the audio descriptor to match the AAC-LC stream actually produced by ffmpeg/camera.
+		err := setAACLCDescriptor(init.Moov.Traks[1], audioSampleRate, audioChannels)
 		if err != nil {
 		}
-		init.Moov.Traks[1].Tkhd.Duration = mp4.AudioTotalDuration
+		init.Moov.Traks[1].Tkhd.Duration = audioMovieDuration
 		init.Moov.Traks[1].Tkhd.CreationTime = macTime
 		init.Moov.Traks[1].Tkhd.ModificationTime = macTime
 		init.Moov.Traks[1].Mdia.Hdlr.Name = "agent " + utils.VERSION
@@ -1338,7 +1297,7 @@ func (frame *ADTS_Frame_Header) Decode(aac []byte) {
 	frame.Fix_Header.Profile = aac[2] >> 6 & 0x03
 	frame.Fix_Header.Sampling_frequency_index = aac[2] >> 2 & 0x0F
 	frame.Fix_Header.Private_bit = aac[2] >> 1 & 0x01
-	frame.Fix_Header.Channel_configuration = (aac[2] & 0x01 << 2) | (aac[3] >> 6)
+	frame.Fix_Header.Channel_configuration = ((aac[2] & 0x01) << 2) | (aac[3] >> 6)
 	frame.Fix_Header.Originalorcopy = aac[3] >> 5 & 0x01
 	frame.Fix_Header.Home = aac[3] >> 4 & 0x01
 	frame.Variable_Header.Copyright_identification_bit = aac[3] >> 3 & 0x01
@@ -1380,6 +1339,89 @@ func AACSampleIdxToSample(idx int) int {
 	return AAC_Sampling_Idx[idx]
 }
 
+func aacFrameDurationSamples(aac []byte) uint64 {
+	if len(aac) < 7 {
+		return 0
+	}
+
+	var header ADTS_Frame_Header
+	header.Decode(aac)
+
+	rawBlocks := uint64(header.Variable_Header.Number_of_raw_data_blocks_in_frame) + 1
+	return rawBlocks * 1024
+}
+
+func aacSampleRate(aac []byte) uint32 {
+	if len(aac) < 7 {
+		return 0
+	}
+
+	var header ADTS_Frame_Header
+	header.Decode(aac)
+
+	sampleRateIdx := int(header.Fix_Header.Sampling_frequency_index)
+	if sampleRateIdx < 0 || sampleRateIdx >= len(AAC_Sampling_Idx) {
+		return 0
+	}
+
+	sampleRate := AACSampleIdxToSample(sampleRateIdx)
+	if sampleRate <= 0 {
+		return 0
+	}
+
+	return uint32(sampleRate)
+}
+
+func aacChannelCount(aacBytes []byte) uint16 {
+	if len(aacBytes) < 7 {
+		return 0
+	}
+
+	var header ADTS_Frame_Header
+	header.Decode(aacBytes)
+	if header.Fix_Header.Channel_configuration == 0 {
+		return 0
+	}
+
+	return uint16(header.Fix_Header.Channel_configuration)
+}
+
+func AACSampleRateFromADTS(aac []byte) uint32 {
+	return aacSampleRate(aac)
+}
+
+func AACChannelCountFromADTS(aac []byte) uint16 {
+	return aacChannelCount(aac)
+}
+
+func setAACLCDescriptor(trak *mp4ff.TrakBox, sampleRate int, channels uint16) error {
+	if trak == nil {
+		return errors.New("nil trak for AAC descriptor")
+	}
+	if sampleRate <= 0 {
+		return errors.New("invalid AAC sample rate")
+	}
+	if channels == 0 {
+		channels = 1
+	}
+
+	asc := &aac.AudioSpecificConfig{
+		ObjectType:           aac.AAClc,
+		ChannelConfiguration: byte(channels),
+		SamplingFrequency:    sampleRate,
+	}
+
+	buf := &bytes.Buffer{}
+	if err := asc.Encode(buf); err != nil {
+		return err
+	}
+
+	esds := mp4ff.CreateEsdsBox(buf.Bytes())
+	mp4a := mp4ff.CreateAudioSampleEntryBox("mp4a", channels, 16, uint16(sampleRate), esds)
+	trak.Mdia.Minf.Stbl.Stsd.AddChild(mp4a)
+	return nil
+}
+
 // +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
 // |  audio object type(5 bits)  |  sampling frequency index(4 bits) |   channel configuration(4 bits)  | GA framelength flag(1 bits) |  GA Depends on core coder(1 bits) | GA Extension Flag(1 bits) |
 // +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+

machinery/src/video/mp4_loopseam_test.go

@@ -1,102 +0,0 @@
-package video
-
-import (
-	"fmt"
-	"os"
-	"testing"
-
-	mp4ff "github.com/Eyevinn/mp4ff/mp4"
-	"github.com/kerberos-io/agent/machinery/src/models"
-)
-
-// TestMP4LoopSeamIsolation reproduces the loop-seam pattern from the
-// failing virtual-rtsp recordings: ~1s GOPs, but at the source-MP4
-// loop boundary an IDR arrives prematurely (~200-870ms after the
-// previous IDR). Without the fix this seam IDR ends up bunched into
-// the same fragment as the prior GOP's IDR which trips macOS
-// VideoToolbox (kVTVideoDecoderBadDataErr / -12909). The fix forces
-// a fragment flush whenever two IDRs arrive closer than MinNormalGOPMs.
-func TestMP4LoopSeamIsolation(t *testing.T) {
-	tmpFile := "/tmp/test_loop_seam.mp4"
-	defer os.Remove(tmpFile)
-
-	sps := []byte{0x67, 0x42, 0xc0, 0x1e, 0xd9, 0x00, 0xa0, 0x47, 0xfe, 0xc8}
-	pps := []byte{0x68, 0xce, 0x38, 0x80}
-	mp4Video := NewMP4(tmpFile, [][]byte{sps}, [][]byte{pps}, nil, 30)
-	mp4Video.SetWidth(1920)
-	mp4Video.SetHeight(1080)
-	v := mp4Video.AddVideoTrack("H264")
-
-	mk := func(k bool) []byte {
-		nt := byte(0x01)
-		if k {
-			nt = 0x65
-		}
-		f := []byte{0, 0, 0, 1, nt}
-		for i := 0; i < 200; i++ {
-			f = append(f, byte(i))
-		}
-		return f
-	}
-
-	frameDur := uint64(33)
-	pts := uint64(0)
-	emit := func(n int, gopLen int) {
-		for f := 0; f < n; f++ {
-			isKey := (f % gopLen) == 0
-			mp4Video.AddSampleToTrack(v, isKey, mk(isKey), pts)
-			pts += frameDur
-		}
-	}
-
-	// 17 seconds of normal content (last "good" IDR at sec 17).
-	emit(17*30, 30)
-	// Seam: IDR arrives ~150ms after previous (vs normal ~1000ms).
-	// This matches the realistic virtual-rtsp / ffmpeg `-stream_loop`
-	// loop boundary, where the new clip's first IDR is emitted shortly
-	// after the previous clip's final IDR.
-	pts -= 820
-	emit(13*30, 30)
-
-	mp4Video.Close(&models.Config{Signing: &models.Signing{PrivateKey: ""}})
-
-	f, _ := os.Open(tmpFile)
-	defer f.Close()
-	parsed, err := mp4ff.DecodeFile(f)
-	if err != nil {
-		t.Fatalf("decode: %v", err)
-	}
-
-	fragIdx := 0
-	for _, seg := range parsed.Segments {
-		for _, fr := range seg.Fragments {
-			for _, traf := range fr.Moof.Trafs {
-				if traf.Tfhd.TrackID != 1 {
-					continue
-				}
-				tfdt := traf.Tfdt.BaseMediaDecodeTime()
-				offset := uint64(0)
-				var keys []uint64
-				for _, trun := range traf.Truns {
-					for _, s := range trun.Samples {
-						if (s.Flags>>24)&0x03 == 0x02 {
-							keys = append(keys, offset)
-						}
-						offset += uint64(s.Dur)
-					}
-				}
-				fmt.Printf("frag %d tfdt=%d samples_dur=%d keys@%v\n",
-					fragIdx, tfdt, offset, keys)
-				for i := 1; i < len(keys); i++ {
-					gap := keys[i] - keys[i-1]
-					if gap < MinNormalGOPMs {
-						t.Errorf("frag %d (tfdt=%d): two IDRs only %d ms apart "+
-							"in same fragment (< %d) - seam was not isolated",
-							fragIdx, tfdt, gap, MinNormalGOPMs)
-					}
-				}
-				fragIdx++
-			}
-		}
-	}
-}