mirror of
https://github.com/kerberos-io/agent.git
synced 2026-08-23 15:08:32 +00:00
Compare commits
1 Commits
v3.11.0
...
feature/se
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
041aac9e7c |
8
.github/workflows/release-bump.yml
vendored
8
.github/workflows/release-bump.yml
vendored
@@ -43,7 +43,13 @@ jobs:
|
||||
{"architecture":"amd64","runner":"ubuntu-24.04"},
|
||||
{"architecture":"arm64","runner":"ubuntu-24.04-arm"}
|
||||
]
|
||||
secrets: inherit
|
||||
# The reusable workflow declares USERNAME/TOKEN as required secrets, so they
|
||||
# must be passed explicitly (secrets: inherit only forwards secrets that
|
||||
# already share those exact names). Map them to a dedicated PAT that has
|
||||
# write:packages access on the uug-ai GitHub Container Registry.
|
||||
secrets:
|
||||
USERNAME: ${{ secrets.GHCR_USERNAME }}
|
||||
TOKEN: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
# Everything below mirrors the agent's own release-create.yml pipeline and
|
||||
# publishes the multi-arch image to the kerberos/agent Docker Hub repo, driven
|
||||
|
||||
@@ -204,7 +204,7 @@ func (h *streamHealth) observeLost(streamType string, lost uint64) {
|
||||
h.lost += lost
|
||||
h.mu.Unlock()
|
||||
log.Log.Warning(fmt.Sprintf(
|
||||
"capture.golibrtsp.health(%s): %d RTP packet(s) lost — sender-side gap (receiver not draining TCP fast enough)",
|
||||
"capture.golibrtsp.health(%s): %d RTP packet(s) lost — camera skipped sequence numbers (TCP transport: camera dropped frames; cause is network path or receiver CPU starvation, not the downstream pipeline)",
|
||||
streamType, lost))
|
||||
}
|
||||
|
||||
|
||||
@@ -80,6 +80,12 @@ func Bootstrap(ctx context.Context, configDirectory string, configuration *model
|
||||
// do several checks to see if the agent is still operational.
|
||||
go ControlAgent(communication)
|
||||
|
||||
// Independent CPU-starvation probe. It runs once for the whole process and
|
||||
// is decoupled from the network, so it disambiguates the two remaining
|
||||
// causes of RTP loss + stream stalls: a starved process (k8s CPU throttling)
|
||||
// vs. an upstream network/camera problem.
|
||||
go monitorCPUStarvation()
|
||||
|
||||
// Handle heartbeats
|
||||
go cloud.HandleHeartBeat(configuration, communication, uptimeStart)
|
||||
|
||||
@@ -478,6 +484,53 @@ func packetAgeString(timer *atomic.Value) string {
|
||||
return strconv.FormatInt(age, 10) + "s"
|
||||
}
|
||||
|
||||
// monitorCPUStarvation runs a single, process-wide probe that is independent of
|
||||
// the camera and network. It sleeps for a fixed interval and measures how much
|
||||
// the wake-up overshoots the requested duration. A goroutine that asks to sleep
|
||||
// 1s but wakes up after 2s could not get scheduled onto a CPU — the classic
|
||||
// signature of CPU starvation / cgroup (Kubernetes) CPU throttling.
|
||||
//
|
||||
// This disambiguates the two remaining causes of RTP loss + stream stalls:
|
||||
// - large overshoot here => the process is starved (CPU limit / noisy
|
||||
// neighbour); the RTSP read loop can't drain the socket, so the camera
|
||||
// drops frames. Fix: raise/remove the CPU limit.
|
||||
// - overshoot stays small => the process is healthy; the loss is upstream
|
||||
// (network path or camera). Fix: investigate the camera link.
|
||||
func monitorCPUStarvation() {
|
||||
const (
|
||||
interval = 1 * time.Second
|
||||
warnAbove = 250 * time.Millisecond
|
||||
summaryEvery = 30 * time.Second
|
||||
)
|
||||
var (
|
||||
maxOvershoot time.Duration
|
||||
windowStart = time.Now()
|
||||
)
|
||||
for {
|
||||
start := time.Now()
|
||||
time.Sleep(interval)
|
||||
overshoot := time.Since(start) - interval
|
||||
if overshoot < 0 {
|
||||
overshoot = 0
|
||||
}
|
||||
if overshoot > maxOvershoot {
|
||||
maxOvershoot = overshoot
|
||||
}
|
||||
if overshoot >= warnAbove {
|
||||
log.Log.Warning(fmt.Sprintf(
|
||||
"components.Kerberos.monitorCPUStarvation(): scheduler overshoot %dms (asked %dms) — process not getting CPU; likely CPU throttling / starvation",
|
||||
overshoot.Milliseconds(), interval.Milliseconds()))
|
||||
}
|
||||
if time.Since(windowStart) >= summaryEvery {
|
||||
log.Log.Info(fmt.Sprintf(
|
||||
"components.Kerberos.monitorCPUStarvation(): %.0fs window — maxSchedulerOvershoot=%dms (>=%dms indicates CPU starvation)",
|
||||
time.Since(windowStart).Seconds(), maxOvershoot.Milliseconds(), warnAbove.Milliseconds()))
|
||||
windowStart = time.Now()
|
||||
maxOvershoot = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ControlAgent will check if the camera is still connected, if not it will restart the agent.
|
||||
// In the other thread we are keeping track of the number of packets received, and particular the keyframe packets.
|
||||
// Once we are not receiving any packets anymore, we will restart the agent.
|
||||
|
||||
Reference in New Issue
Block a user