Compare commits

...

1 Commits

Author SHA1 Message Date
Cédric Verstraeten
041aac9e7c Add CPU starvation monitor; update release secrets
Pass GHCR credentials to the reusable release workflow and add a process-wide CPU-starvation probe.

- .github/workflows/release-bump.yml: explicitly map USERNAME/TOKEN to GHCR_USERNAME/GHCR_TOKEN so the reusable workflow gets the required secrets (use a PAT with write:packages for the uug-ai GHCR).
- machinery/src/components/Kerberos.go: start a new goroutine in Bootstrap and add monitorCPUStarvation(), which measures scheduler wake-up overshoot (1s interval, warn above 250ms, 30s summaries) to detect CPU throttling/starvation and log warnings/info. This helps disambiguate RTP loss causes (process starvation vs upstream/camera/network).
- machinery/src/capture/gortsplib.go: update a health log message to clarify that observed RTP packet loss can indicate camera-skipped sequence numbers (TCP transport: camera dropped frames) and that the root cause is the network path or receiver CPU starvation, not the downstream pipeline.
2026-06-11 15:02:45 +02:00
3 changed files with 61 additions and 2 deletions

View File

@@ -43,7 +43,13 @@ jobs:
{"architecture":"amd64","runner":"ubuntu-24.04"},
{"architecture":"arm64","runner":"ubuntu-24.04-arm"}
]
secrets: inherit
# The reusable workflow declares USERNAME/TOKEN as required secrets, so they
# must be passed explicitly (secrets: inherit only forwards secrets that
# already share those exact names). Map them to a dedicated PAT that has
# write:packages access on the uug-ai GitHub Container Registry.
secrets:
USERNAME: ${{ secrets.GHCR_USERNAME }}
TOKEN: ${{ secrets.GHCR_TOKEN }}
# Everything below mirrors the agent's own release-create.yml pipeline and
# publishes the multi-arch image to the kerberos/agent Docker Hub repo, driven

View File

@@ -204,7 +204,7 @@ func (h *streamHealth) observeLost(streamType string, lost uint64) {
h.lost += lost
h.mu.Unlock()
log.Log.Warning(fmt.Sprintf(
"capture.golibrtsp.health(%s): %d RTP packet(s) lost — sender-side gap (receiver not draining TCP fast enough)",
"capture.golibrtsp.health(%s): %d RTP packet(s) lost — camera skipped sequence numbers (TCP transport: camera dropped frames; cause is network path or receiver CPU starvation, not the downstream pipeline)",
streamType, lost))
}

View File

@@ -80,6 +80,12 @@ func Bootstrap(ctx context.Context, configDirectory string, configuration *model
// do several checks to see if the agent is still operational.
go ControlAgent(communication)
// Independent CPU-starvation probe. It runs once for the whole process and
// is decoupled from the network, so it disambiguates the two remaining
// causes of RTP loss + stream stalls: a starved process (k8s CPU throttling)
// vs. an upstream network/camera problem.
go monitorCPUStarvation()
// Handle heartbeats
go cloud.HandleHeartBeat(configuration, communication, uptimeStart)
@@ -478,6 +484,53 @@ func packetAgeString(timer *atomic.Value) string {
return strconv.FormatInt(age, 10) + "s"
}
// monitorCPUStarvation runs a single, process-wide probe that is independent of
// the camera and network. It sleeps for a fixed interval and measures how much
// the wake-up overshoots the requested duration. A goroutine that asks to sleep
// 1s but wakes up after 2s could not get scheduled onto a CPU — the classic
// signature of CPU starvation / cgroup (Kubernetes) CPU throttling.
//
// This disambiguates the two remaining causes of RTP loss + stream stalls:
// - large overshoot here => the process is starved (CPU limit / noisy
// neighbour); the RTSP read loop can't drain the socket, so the camera
// drops frames. Fix: raise/remove the CPU limit.
// - overshoot stays small => the process is healthy; the loss is upstream
// (network path or camera). Fix: investigate the camera link.
func monitorCPUStarvation() {
const (
interval = 1 * time.Second
warnAbove = 250 * time.Millisecond
summaryEvery = 30 * time.Second
)
var (
maxOvershoot time.Duration
windowStart = time.Now()
)
for {
start := time.Now()
time.Sleep(interval)
overshoot := time.Since(start) - interval
if overshoot < 0 {
overshoot = 0
}
if overshoot > maxOvershoot {
maxOvershoot = overshoot
}
if overshoot >= warnAbove {
log.Log.Warning(fmt.Sprintf(
"components.Kerberos.monitorCPUStarvation(): scheduler overshoot %dms (asked %dms) — process not getting CPU; likely CPU throttling / starvation",
overshoot.Milliseconds(), interval.Milliseconds()))
}
if time.Since(windowStart) >= summaryEvery {
log.Log.Info(fmt.Sprintf(
"components.Kerberos.monitorCPUStarvation(): %.0fs window — maxSchedulerOvershoot=%dms (>=%dms indicates CPU starvation)",
time.Since(windowStart).Seconds(), maxOvershoot.Milliseconds(), warnAbove.Milliseconds()))
windowStart = time.Now()
maxOvershoot = 0
}
}
}
// ControlAgent will check if the camera is still connected, if not it will restart the agent.
// In the other thread we are keeping track of the number of packets received, and particular the keyframe packets.
// Once we are not receiving any packets anymore, we will restart the agent.