mirror of
https://github.com/kerberos-io/agent.git
synced 2026-11-15 21:50:47 +00:00
Adopt dated, millisecond-based recording paths
Store new recordings in UTC date directories with stable timestamp-device filenames, moving recording attributes into upload checkpoints. Use checkpoint metadata for S3 keys and HTTP/tus uploads, rejecting invalid S3 checkpoints instead of silently falling back. Support flat and dated layouts across listing, playback, uploads, cleanup, and decryption while retaining legacy filename compatibility.
This commit is contained in:
@@ -80,22 +80,53 @@ There are a myriad of cameras out there (USB, IP and other cameras), and it migh
|
||||
|
||||
Direct S3 uploads store recordings as
|
||||
`<s3.username>/yyyy/mm/dd/<timestamp>-<deviceKey>.mp4`. The timestamp is the
|
||||
recording's Unix timestamp in seconds, derived from its local filename, and the
|
||||
date is its UTC date, not the upload date. The device key is the configured Agent
|
||||
key and must be non-empty and contain no path separators. Retried uploads use
|
||||
the same object key, even after midnight. Recordings from the same device starting
|
||||
in the same second share an object key.
|
||||
recording's Unix timestamp in milliseconds, and the date is its UTC date, not the upload
|
||||
date. Timestamp and device key come from the recording's `.metadata` upload
|
||||
checkpoint, which stores the timestamp in milliseconds. The uploader also
|
||||
preserves checkpoint duration, FPS (when available), and encryption status as S3
|
||||
object metadata. The device key must be non-empty and contain no path separators.
|
||||
Retried uploads use the same object key, even after midnight. Millisecond
|
||||
precision keeps recordings starting within the same second distinct.
|
||||
|
||||
This replaces the previous `<s3.username>/<filename>.mp4` layout for new uploads.
|
||||
Existing objects are not moved; consumers of direct S3 uploads must support both
|
||||
layouts when reading historical recordings. Local Agent recordings and uploads
|
||||
through Kerberos Vault or Kerberos Hub are unchanged.
|
||||
layouts when reading historical recordings. Updated Kerberos Vault deployments
|
||||
use the same filename and dated layout under the authenticated account directory
|
||||
for both direct Vault uploads and Hub-proxied uploads.
|
||||
|
||||
The shorter S3 filename cannot be decoded by legacy filename-only consumers.
|
||||
Only historical recordings with missing, empty, or FPS-only queue markers fall
|
||||
back to the legacy filename format and configured Agent key. A structured
|
||||
checkpoint takes precedence over both the filename and current configured key;
|
||||
an invalid or unreadable checkpoint is reported as an upload error, not silently
|
||||
replaced with filename-derived values.
|
||||
|
||||
The shorter filename cannot be decoded by legacy filename-only consumers.
|
||||
Integrations must preserve structured recording metadata, including the device
|
||||
key (`productid`) and recording timestamp (`event-timestamp`), as well as the full
|
||||
object key.
|
||||
|
||||
### Local recording layout
|
||||
|
||||
New local recordings are stored under
|
||||
`data/recordings/yyyy/mm/dd/<timestamp>-<deviceKey>.mp4`, using the recording's UTC
|
||||
start date and Unix-millisecond timestamp. For example:
|
||||
`data/recordings/2026/01/02/1767355200123-device-key.mp4`.
|
||||
The filename no longer embeds the camera name, region, change count, or duration,
|
||||
and it does not change when the recording is finalized.
|
||||
Those attributes, together with FPS and encryption status, are captured in the
|
||||
`.metadata` upload checkpoint and sent as structured HTTP/tus metadata.
|
||||
|
||||
Existing flat recordings and dated recordings with older long filenames remain
|
||||
supported without a migration. Recording lists, day filters, playback, uploaders,
|
||||
and automatic cleanup handle both filename formats and layouts. Upload queue
|
||||
checkpoints remain flat in `data/cloud/`. Successful-upload removal also resolves
|
||||
both local layouts. Deploy the updated Vault alongside Agent for canonical
|
||||
`<account-directory>/yyyy/mm/dd/<timestamp>-<deviceKey>.mp4` storage keys and
|
||||
metadata-driven Hub integration events.
|
||||
Directory decryption also processes dated subdirectories and writes each output
|
||||
to a sibling `decrypted/` directory; those outputs are excluded from recording
|
||||
lists and automatic cleanup.
|
||||
|
||||
## Quickstart - Docker
|
||||
|
||||
The easiest way to get your Kerberos Agent up and running is to use our public image on [Docker hub](https://hub.docker.com/r/kerberos/agent). Once you have selected a specific tag, run `docker` command below, which will open the web interface of your Kerberos agent on port `80`, and off you go. For a more configurable and persistent deployment have a look at [Running and automating a Kerberos Agent](#running-and-automating-a-kerberos-agent).
|
||||
|
||||
@@ -10,11 +10,58 @@ import (
|
||||
"github.com/kerberos-io/agent/machinery/src/models"
|
||||
)
|
||||
|
||||
func TestPickRecordingToCleanupAcrossDatedDirectories(t *testing.T) {
|
||||
for _, legacyMarker := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "metadata", true: "legacy"}[legacyMarker], func(t *testing.T) {
|
||||
recordingsDir, cloudDir := newCleanupDirs(t)
|
||||
pendingName := "2026/01/01/oldest_pending.mp4"
|
||||
safeName := "2026/01/02/uploaded.mp4"
|
||||
writeRecording(t, recordingsDir, pendingName, 30)
|
||||
if legacyMarker {
|
||||
if err := os.WriteFile(filepath.Join(cloudDir, filepath.Base(pendingName)), nil, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
} else {
|
||||
markPending(t, cloudDir, pendingName)
|
||||
}
|
||||
writeRecording(t, recordingsDir, safeName, 20)
|
||||
writeRecording(t, recordingsDir, "flat_uploaded.mp4", 10)
|
||||
name, pending, err := pickRecordingToCleanup(recordingsDir, cloudDir)
|
||||
if err != nil || pending || name != safeName {
|
||||
t.Fatalf("cleanup = (%q, %v, %v), want (%q, false, nil)", name, pending, err, safeName)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupDatedPendingRecordingRemovesFlatMarker(t *testing.T) {
|
||||
recordingsDir, cloudDir := newCleanupDirs(t)
|
||||
name := "2026/01/01/pending.mp4"
|
||||
if err := os.MkdirAll(filepath.Dir(filepath.Join(recordingsDir, name)), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeSizedRecording(t, recordingsDir, name, 2*1000*1000)
|
||||
markPending(t, cloudDir, name)
|
||||
config := &models.Configuration{Config: models.Config{AutoClean: "true", MaxDirectorySize: 1}}
|
||||
CleanupRecordingDirectory(filepath.Dir(filepath.Dir(recordingsDir)), config)
|
||||
for _, filePath := range []string{
|
||||
filepath.Join(recordingsDir, name),
|
||||
filepath.Join(cloudDir, models.RecordingUploadMetadataFileName(name)),
|
||||
} {
|
||||
if _, err := os.Stat(filePath); !os.IsNotExist(err) {
|
||||
t.Fatalf("expected %s to be removed, got %v", filePath, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// writeRecording creates a file under recordingsDir and sets its modtime so the
|
||||
// tests can control the "oldest" ordering deterministically.
|
||||
func writeRecording(t *testing.T, recordingsDir, name string, ageMinutes int) {
|
||||
t.Helper()
|
||||
full := filepath.Join(recordingsDir, name)
|
||||
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
|
||||
t.Fatalf("mkdir recording directory: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(full, []byte("data"), 0o644); err != nil {
|
||||
t.Fatalf("write recording %s: %v", name, err)
|
||||
}
|
||||
|
||||
+35
-161
@@ -53,13 +53,16 @@ func publishRecordingState(mqttClient mqtt.Client, hubKey string, configuration
|
||||
}
|
||||
}
|
||||
|
||||
func recordingUploadMetadata(name, deviceKey string, timestamp int64, mp4Video *video.MP4, encrypted bool) models.RecordingUploadMetadata {
|
||||
func recordingUploadMetadata(name string, config models.Config, timestamp int64, mp4Video *video.MP4, encrypted bool, coordinates string, changes int) models.RecordingUploadMetadata {
|
||||
metadata := models.RecordingUploadMetadata{
|
||||
FileName: filepath.Base(name),
|
||||
DeviceKey: deviceKey,
|
||||
Timestamp: timestamp,
|
||||
Duration: mp4Video.VideoTotalDuration,
|
||||
Encrypted: encrypted,
|
||||
FileName: filepath.Base(name),
|
||||
DeviceKey: config.Key,
|
||||
Timestamp: timestamp,
|
||||
Duration: mp4Video.VideoTotalDuration,
|
||||
Encrypted: encrypted,
|
||||
DeviceName: config.Name,
|
||||
RegionCoordinates: coordinates,
|
||||
NumberOfChanges: strconv.Itoa(changes),
|
||||
}
|
||||
value := mp4Video.AverageFPS()
|
||||
if value > 0 && value <= 240 && !math.IsInf(value, 0) && !math.IsNaN(value) {
|
||||
@@ -190,7 +193,7 @@ func CleanupRecordingDirectory(configDirectory string, configuration *models.Con
|
||||
return
|
||||
}
|
||||
|
||||
if err := os.Remove(recordingsDirectory + "/" + name); err != nil {
|
||||
if err := utils.RemoveRecording(recordingsDirectory, name); err != nil {
|
||||
log.Info("HandleRecordStream: something went wrong, " + err.Error())
|
||||
return
|
||||
}
|
||||
@@ -275,7 +278,7 @@ func defaultReserveMB(totalMB int64) int64 {
|
||||
// overall with pending=true, signalling the caller that it is about to drop an
|
||||
// un-uploaded recording as a last resort.
|
||||
func pickRecordingToCleanup(recordingsDirectory, cloudDirectory string) (string, bool, error) {
|
||||
entries, err := os.ReadDir(recordingsDirectory)
|
||||
entries, err := utils.ReadRecordingDirectory(recordingsDirectory)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
@@ -284,14 +287,7 @@ func pickRecordingToCleanup(recordingsDirectory, cloudDirectory string) (string,
|
||||
var oldestSafeTime, oldestAnyTime time.Time
|
||||
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
info, err := entry.Info()
|
||||
if err != nil || !info.Mode().IsRegular() {
|
||||
continue
|
||||
}
|
||||
modTime := info.ModTime()
|
||||
modTime := entry.ModTime()
|
||||
|
||||
if oldestAnyName == "" || modTime.Before(oldestAnyTime) {
|
||||
oldestAnyName = entry.Name()
|
||||
@@ -449,39 +445,6 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
|
||||
// Cleanup muxer
|
||||
start = false
|
||||
|
||||
// Update the name of the recording with the duration.
|
||||
// We will update the name of the recording with the duration in milliseconds.
|
||||
if mp4Video.VideoTotalDuration > 0 {
|
||||
duration := mp4Video.VideoTotalDuration
|
||||
// Update the name with the duration in milliseconds.
|
||||
startRecordingSeconds := startRecording / 1000 // convert to seconds
|
||||
startRecordingMilliseconds := startRecording % 1000 // convert to milliseconds
|
||||
s := strconv.FormatInt(startRecordingSeconds, 10) + "_" +
|
||||
strconv.Itoa(len(strconv.FormatInt(startRecordingMilliseconds, 10))) + "-" +
|
||||
strconv.FormatInt(startRecordingMilliseconds, 10) + "_" +
|
||||
config.Name + "_" +
|
||||
"0-0-0-0" + "_" + // region coordinates, we
|
||||
"-1" + "_" + // token
|
||||
strconv.FormatInt(int64(duration), 10) // + "_" + // duration of recording
|
||||
//utils.VERSION // version of the agent
|
||||
|
||||
oldName := name
|
||||
name = s + ".mp4"
|
||||
fullName = configDirectory + "/data/recordings/" + name
|
||||
log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
|
||||
|
||||
// Rename the file to the new name.
|
||||
err := os.Rename(
|
||||
configDirectory+"/data/recordings/"+oldName,
|
||||
configDirectory+"/data/recordings/"+s+".mp4")
|
||||
|
||||
if err != nil {
|
||||
log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
|
||||
}
|
||||
} else {
|
||||
log.Info("capture.main.HandleRecordStream(continuous): no video data recorded, not renaming file.")
|
||||
}
|
||||
|
||||
encrypted := false
|
||||
// Check if we need to encrypt the recording.
|
||||
if config.Encryption != nil && config.Encryption.Enabled == "true" && config.Encryption.Recordings == "true" && config.Encryption.SymmetricKey != "" {
|
||||
@@ -506,7 +469,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
|
||||
}
|
||||
}
|
||||
|
||||
queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, startRecording, mp4Video, encrypted))
|
||||
queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config, startRecording, mp4Video, encrypted, "0-0-0-0", -1))
|
||||
|
||||
recordingStatus = "idle"
|
||||
|
||||
@@ -531,29 +494,17 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
|
||||
|
||||
start = true
|
||||
|
||||
// timestamp_microseconds_instanceName_regionCoordinates_numberOfChanges_token
|
||||
// 1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4
|
||||
// - Timestamp
|
||||
// - Size + - + microseconds
|
||||
// - device
|
||||
// - Region
|
||||
// - Number of changes
|
||||
// - Token
|
||||
|
||||
startRecording = pkt.CurrentTime
|
||||
startRecordingSeconds := startRecording / 1000 // convert to seconds
|
||||
startRecordingMilliseconds := startRecording % 1000 // convert to milliseconds
|
||||
s := strconv.FormatInt(startRecordingSeconds, 10) + "_" + // start timestamp in seconds
|
||||
strconv.Itoa(len(strconv.FormatInt(startRecordingMilliseconds, 10))) + "-" + // length of milliseconds
|
||||
strconv.FormatInt(startRecordingMilliseconds, 10) + "_" + // milliseconds
|
||||
config.Name + "_" + // device name
|
||||
"0-0-0-0" + "_" + // region coordinates, we will not use this for continuous recording
|
||||
"0" + "_" + // token
|
||||
"0" + "_" //+ // duration of recording in milliseconds
|
||||
//utils.VERSION // version of the agent
|
||||
|
||||
name = s + ".mp4"
|
||||
fullName = configDirectory + "/data/recordings/" + name
|
||||
name, err = utils.RecordingFileName(startRecording, config.Key)
|
||||
if err == nil {
|
||||
fullName, err = utils.CreateRecordingPath(configDirectory+"/data/recordings", name, time.UnixMilli(startRecording))
|
||||
}
|
||||
if err != nil {
|
||||
log.WithError(err).Error("capture: could not create continuous recording path")
|
||||
start = false
|
||||
pkt = nextPkt
|
||||
continue
|
||||
}
|
||||
|
||||
// Running...
|
||||
log.Info("capture.main.HandleRecordStream(continuous): recording started")
|
||||
@@ -609,39 +560,6 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
|
||||
// Cleanup muxer
|
||||
start = false
|
||||
|
||||
// Update the name of the recording with the duration.
|
||||
// We will update the name of the recording with the duration in milliseconds.
|
||||
if mp4Video.VideoTotalDuration > 0 {
|
||||
duration := mp4Video.VideoTotalDuration
|
||||
// Update the name with the duration in milliseconds.
|
||||
startRecordingSeconds := startRecording / 1000 // convert to seconds
|
||||
startRecordingMilliseconds := startRecording % 1000 // convert to milliseconds
|
||||
s := strconv.FormatInt(startRecordingSeconds, 10) + "_" +
|
||||
strconv.Itoa(len(strconv.FormatInt(startRecordingMilliseconds, 10))) + "-" +
|
||||
strconv.FormatInt(startRecordingMilliseconds, 10) + "_" +
|
||||
config.Name + "_" +
|
||||
"0-0-0-0" + "_" + // region coordinates, we
|
||||
"-1" + "_" + // token
|
||||
strconv.FormatInt(int64(duration), 10) // + "_" + // duration of recording
|
||||
//utils.VERSION // version of the agent
|
||||
|
||||
oldName := name
|
||||
name = s + ".mp4"
|
||||
fullName = configDirectory + "/data/recordings/" + name
|
||||
log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
|
||||
|
||||
// Rename the file to the new name.
|
||||
err := os.Rename(
|
||||
configDirectory+"/data/recordings/"+oldName,
|
||||
configDirectory+"/data/recordings/"+s+".mp4")
|
||||
|
||||
if err != nil {
|
||||
log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
|
||||
}
|
||||
} else {
|
||||
log.Info("capture.main.HandleRecordStream(continuous): no video data recorded, not renaming file.")
|
||||
}
|
||||
|
||||
encrypted := false
|
||||
// Check if we need to encrypt the recording.
|
||||
if config.Encryption != nil && config.Encryption.Enabled == "true" && config.Encryption.Recordings == "true" && config.Encryption.SymmetricKey != "" {
|
||||
@@ -666,7 +584,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
|
||||
}
|
||||
}
|
||||
|
||||
queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, startRecording, mp4Video, encrypted))
|
||||
queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config, startRecording, mp4Video, encrypted, "0-0-0-0", -1))
|
||||
|
||||
recordingStatus = "idle"
|
||||
|
||||
@@ -721,17 +639,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
|
||||
continue
|
||||
}
|
||||
|
||||
// timestamp_microseconds_instanceName_regionCoordinates_numberOfChanges_token
|
||||
// 1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4
|
||||
// - Timestamp
|
||||
// - Size + - + microseconds
|
||||
// - device
|
||||
// - Region
|
||||
// - Number of changes
|
||||
// - Token
|
||||
|
||||
displayTimeSeconds := displayTime / 1000 // convert to seconds
|
||||
displayTimeMilliseconds := displayTime % 1000 // convert to milliseconds
|
||||
displayTimeSeconds := displayTime / 1000
|
||||
motionRectangleString := "0-0-0-0"
|
||||
if motion.Rectangle.X != 0 || motion.Rectangle.Y != 0 ||
|
||||
motion.Rectangle.Width != 0 || motion.Rectangle.Height != 0 {
|
||||
@@ -742,17 +650,15 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
|
||||
// Get the number of changes from the motion detection.
|
||||
numberOfChanges := motion.NumberOfChanges
|
||||
|
||||
s := strconv.FormatInt(displayTimeSeconds, 10) + "_" + // start timestamp in seconds
|
||||
strconv.Itoa(len(strconv.FormatInt(displayTimeMilliseconds, 10))) + "-" + // length of milliseconds
|
||||
strconv.FormatInt(displayTimeMilliseconds, 10) + "_" + // milliseconds
|
||||
config.Name + "_" + // device name
|
||||
motionRectangleString + "_" + // region coordinates, we will not use this for continuous recording
|
||||
strconv.Itoa(numberOfChanges) + "_" + // number of changes
|
||||
"0" // + "_" + // duration of recording in milliseconds
|
||||
//utils.VERSION // version of the agent
|
||||
|
||||
name := s + ".mp4"
|
||||
fullName := configDirectory + "/data/recordings/" + name
|
||||
name, err := utils.RecordingFileName(displayTime, config.Key)
|
||||
var fullName string
|
||||
if err == nil {
|
||||
fullName, err = utils.CreateRecordingPath(configDirectory+"/data/recordings", name, time.UnixMilli(displayTime))
|
||||
}
|
||||
if err != nil {
|
||||
log.WithError(err).Error("capture: could not create motion recording path")
|
||||
continue
|
||||
}
|
||||
|
||||
// Running...
|
||||
log.Info("capture.main.HandleRecordStream(motiondetection): recording started (" + name + ")" + " at " + strconv.FormatInt(displayTimeSeconds, 10) + " unix")
|
||||
@@ -880,38 +786,6 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
|
||||
communication.TrySendMotion(models.MotionDataPartial{Timestamp: time.Now().Unix(), NumberOfChanges: 100000000})
|
||||
}
|
||||
|
||||
// Update the name of the recording with the duration.
|
||||
// We will update the name of the recording with the duration in milliseconds.
|
||||
if mp4Video.VideoTotalDuration > 0 {
|
||||
duration := mp4Video.VideoTotalDuration
|
||||
|
||||
// Update the name with the duration in milliseconds.
|
||||
s := strconv.FormatInt(displayTimeSeconds, 10) + "_" +
|
||||
strconv.Itoa(len(strconv.FormatInt(displayTimeMilliseconds, 10))) + "-" +
|
||||
strconv.FormatInt(displayTimeMilliseconds, 10) + "_" +
|
||||
config.Name + "_" +
|
||||
motionRectangleString + "_" +
|
||||
strconv.Itoa(numberOfChanges) + "_" + // number of changes
|
||||
strconv.FormatInt(int64(duration), 10) // + "_" + // duration of recording in milliseconds
|
||||
//utils.VERSION // version of the agent
|
||||
|
||||
oldName := name
|
||||
name = s + ".mp4"
|
||||
fullName = configDirectory + "/data/recordings/" + name
|
||||
log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name)
|
||||
|
||||
// Rename the file to the new name.
|
||||
err := os.Rename(
|
||||
configDirectory+"/data/recordings/"+oldName,
|
||||
configDirectory+"/data/recordings/"+s+".mp4")
|
||||
|
||||
if err != nil {
|
||||
log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error())
|
||||
}
|
||||
} else {
|
||||
log.Info("capture.main.HandleRecordStream(motiondetection): no video data recorded, not renaming file.")
|
||||
}
|
||||
|
||||
encrypted := false
|
||||
// Check if we need to encrypt the recording.
|
||||
if config.Encryption != nil && config.Encryption.Enabled == "true" && config.Encryption.Recordings == "true" && config.Encryption.SymmetricKey != "" {
|
||||
@@ -936,7 +810,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
|
||||
}
|
||||
}
|
||||
|
||||
queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, displayTime, mp4Video, encrypted))
|
||||
queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config, displayTime, mp4Video, encrypted, motionRectangleString, numberOfChanges))
|
||||
|
||||
// Clean up the recording directory if necessary.
|
||||
CleanupRecordingDirectory(configDirectory, configuration)
|
||||
|
||||
@@ -42,10 +42,11 @@ func TestQueueRecordingForUploadStoresFinalizedMetadata(t *testing.T) {
|
||||
}
|
||||
|
||||
mp4Video := &video.MP4{VideoTotalDuration: 20452, SampleCount: 613}
|
||||
metadata := recordingUploadMetadata("recording.mp4", "device-key", 1785934709414, mp4Video, true)
|
||||
config := models.Config{Key: "device-key", Name: "camera-name"}
|
||||
metadata := recordingUploadMetadata("1785934709414-device-key.mp4", config, 1785934709414, mp4Video, true, "1-2-3-4", 57)
|
||||
queueRecordingForUpload(configDirectory, metadata)
|
||||
|
||||
got, err := os.ReadFile(filepath.Join(configDirectory, "data", "cloud", "recording.metadata"))
|
||||
got, err := os.ReadFile(filepath.Join(configDirectory, "data", "cloud", "1785934709414-device-key.metadata"))
|
||||
if err != nil {
|
||||
t.Fatalf("read upload marker: %v", err)
|
||||
}
|
||||
@@ -54,7 +55,8 @@ func TestQueueRecordingForUploadStoresFinalizedMetadata(t *testing.T) {
|
||||
t.Fatalf("decode upload marker: %v", err)
|
||||
}
|
||||
expectedFPS := mp4Video.AverageFPS()
|
||||
if stored.FileName != "recording.mp4" || stored.DeviceKey != "device-key" || stored.Timestamp != 1785934709414 || stored.Duration != 20452 || math.Abs(stored.FPS-expectedFPS) > 1e-9 || !stored.Encrypted {
|
||||
if stored.FileName != "1785934709414-device-key.mp4" || stored.DeviceKey != "device-key" || stored.Timestamp != 1785934709414 || stored.Duration != 20452 || math.Abs(stored.FPS-expectedFPS) > 1e-9 || !stored.Encrypted ||
|
||||
stored.DeviceName != "camera-name" || stored.RegionCoordinates != "1-2-3-4" || stored.NumberOfChanges != "57" {
|
||||
t.Fatalf("upload marker = %+v", stored)
|
||||
}
|
||||
if stored.FPS == math.Floor(stored.FPS) {
|
||||
|
||||
@@ -125,7 +125,7 @@ func HandleUpload(configDirectory string, configuration *models.Configuration, c
|
||||
// Check if we need to remove the original recording
|
||||
// removeAfterUpload is set to false by default
|
||||
if config.RemoveAfterUpload != "false" {
|
||||
err := os.Remove(configDirectory + "/data/recordings/" + fileName)
|
||||
err := utils.RemoveRecording(configDirectory+"/data/recordings", fileName)
|
||||
if err != nil {
|
||||
log.Error("HandleUpload: " + err.Error())
|
||||
}
|
||||
|
||||
@@ -5,13 +5,13 @@ package cloud
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"os"
|
||||
|
||||
"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox"
|
||||
"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
|
||||
"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/users"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/kerberos-io/agent/machinery/src/models"
|
||||
"github.com/kerberos-io/agent/machinery/src/utils"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
@@ -37,14 +37,12 @@ func UploadDropbox(configuration *models.Configuration, fileName string) (bool,
|
||||
// Upload to Dropbox
|
||||
log.Info("UploadDropbox: Uploading to Dropbox")
|
||||
log.Info("UploadDropbox: Upload started for " + fileName)
|
||||
fullname := "data/recordings/" + fileName
|
||||
|
||||
dConfig := dropbox.Config{
|
||||
Token: token,
|
||||
LogLevel: dropbox.LogInfo, // if needed, set the desired logging level. Default is off
|
||||
}
|
||||
|
||||
file, err := os.OpenFile(fullname, os.O_RDWR, 0755)
|
||||
file, err := utils.OpenRecording("data/recordings", fileName)
|
||||
if file != nil {
|
||||
defer func() {
|
||||
if cerr := file.Close(); cerr != nil {
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"os"
|
||||
|
||||
"github.com/kerberos-io/agent/machinery/src/models"
|
||||
"github.com/kerberos-io/agent/machinery/src/utils"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
@@ -78,10 +79,8 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo
|
||||
}).Info("Resumable Hub upload unavailable; using legacy upload")
|
||||
}
|
||||
|
||||
fullname := "data/recordings/" + fileName
|
||||
|
||||
// Check if we still have the file otherwise we abort the request.
|
||||
file, err := os.OpenFile(fullname, os.O_RDWR, 0755)
|
||||
file, err := utils.OpenRecording("data/recordings", fileName)
|
||||
if file != nil {
|
||||
defer file.Close()
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/kerberos-io/agent/machinery/src/models"
|
||||
"github.com/kerberos-io/agent/machinery/src/utils"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
@@ -35,10 +36,23 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string) (
|
||||
// This can happen when the file was already removed (e.g. cleanup, or an
|
||||
// earlier successful upload). Skip it so the watcher drops the marker
|
||||
// instead of retrying indefinitely.
|
||||
info, err := os.Stat("data/recordings/" + fileName)
|
||||
filePath, err := utils.ResolveRecordingPath("data/recordings", fileName)
|
||||
if err != nil {
|
||||
log.Info("UploadKerberosVault: skipping " + fileName + ", file doesn't exist anymore")
|
||||
return false, false, nil
|
||||
if os.IsNotExist(err) {
|
||||
log.Info("UploadKerberosVault: skipping " + fileName + ", file doesn't exist anymore")
|
||||
return false, false, nil
|
||||
}
|
||||
log.WithError(err).Error("UploadKerberosVault: could not resolve recording path")
|
||||
return false, true, err
|
||||
}
|
||||
info, err := os.Stat(filePath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
log.Info("UploadKerberosVault: skipping " + fileName + ", file doesn't exist anymore")
|
||||
return false, false, nil
|
||||
}
|
||||
log.WithError(err).Error("UploadKerberosVault: could not stat recording")
|
||||
return false, true, err
|
||||
}
|
||||
if info.Size() == 0 {
|
||||
log.Warn("UploadKerberosVault: skipping " + fileName + ", recording is empty")
|
||||
@@ -182,9 +196,7 @@ func uploadVaultLegacy(vault models.KStorage, publicKey, deviceKey, fileName, la
|
||||
if err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
fullname := "data/recordings/" + fileName
|
||||
|
||||
file, err := os.Open(fullname)
|
||||
file, err := utils.OpenRecording("data/recordings", fileName)
|
||||
if file != nil {
|
||||
defer file.Close()
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package cloud
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
"os"
|
||||
@@ -10,12 +11,16 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/kerberos-io/agent/machinery/src/models"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const recordingFPSHeader = "X-Kerberos-Storage-Fps"
|
||||
const recordingDurationHeader = "X-Kerberos-Storage-Duration"
|
||||
const recordingTimestampHeader = "X-Kerberos-Storage-Timestamp"
|
||||
const recordingEncryptedHeader = "X-Kerberos-Storage-Encrypted"
|
||||
const recordingDeviceNameHeader = "X-Kerberos-Storage-DeviceName"
|
||||
const recordingRegionHeader = "X-Kerberos-Storage-RegionCoordinates"
|
||||
const recordingChangesHeader = "X-Kerberos-Storage-NumberOfChanges"
|
||||
|
||||
// queuedRecordingFPS reads the FPS snapshot written into the upload marker
|
||||
// when the recording was finalized. Historical empty markers intentionally
|
||||
@@ -61,6 +66,14 @@ func decodeRecordingUploadMetadata(value []byte) (models.RecordingUploadMetadata
|
||||
}
|
||||
|
||||
func readRecordingUploadMetadata(fileName string) ([]byte, bool) {
|
||||
value, err := readRecordingUploadMetadataFile(fileName)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
log.WithError(err).Error("Failed to read recording upload metadata")
|
||||
}
|
||||
return value, err == nil
|
||||
}
|
||||
|
||||
func readRecordingUploadMetadataFile(fileName string) ([]byte, error) {
|
||||
markerNames := []string{
|
||||
models.RecordingUploadMetadataFileName(fileName),
|
||||
filepath.Base(fileName),
|
||||
@@ -68,10 +81,13 @@ func readRecordingUploadMetadata(fileName string) ([]byte, bool) {
|
||||
for _, markerName := range markerNames {
|
||||
value, err := os.ReadFile(filepath.Join("data", "cloud", markerName))
|
||||
if err == nil {
|
||||
return value, true
|
||||
return value, nil
|
||||
}
|
||||
if !os.IsNotExist(err) {
|
||||
return nil, fmt.Errorf("read recording upload metadata %s: %w", markerName, err)
|
||||
}
|
||||
}
|
||||
return nil, false
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
|
||||
func setQueuedRecordingMetadataHeaders(header http.Header, fileName string) {
|
||||
@@ -79,6 +95,18 @@ func setQueuedRecordingMetadataHeaders(header http.Header, fileName string) {
|
||||
header.Set(recordingFPSHeader, fps)
|
||||
}
|
||||
if metadata, ok := queuedRecordingMetadata(fileName); ok {
|
||||
if metadata.DeviceKey != "" {
|
||||
header.Set("X-Kerberos-Storage-Device", metadata.DeviceKey)
|
||||
}
|
||||
if metadata.DeviceName != "" {
|
||||
header.Set(recordingDeviceNameHeader, metadata.DeviceName)
|
||||
}
|
||||
if metadata.RegionCoordinates != "" {
|
||||
header.Set(recordingRegionHeader, metadata.RegionCoordinates)
|
||||
}
|
||||
if metadata.NumberOfChanges != "" {
|
||||
header.Set(recordingChangesHeader, metadata.NumberOfChanges)
|
||||
}
|
||||
if metadata.Duration > 0 {
|
||||
header.Set(recordingDurationHeader, strconv.FormatUint(metadata.Duration, 10))
|
||||
}
|
||||
|
||||
+98
-46
@@ -2,15 +2,20 @@ package cloud
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/kerberos-io/agent/machinery/src/models"
|
||||
"github.com/kerberos-io/agent/machinery/src/utils"
|
||||
"github.com/minio/minio-go/v6"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
@@ -19,15 +24,6 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
|
||||
|
||||
config := configuration.Config
|
||||
|
||||
// timestamp_microseconds_instanceName_regionCoordinates_numberOfChanges_token
|
||||
// 1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4
|
||||
// - Timestamp
|
||||
// - Size + - + microseconds
|
||||
// - device
|
||||
// - Region
|
||||
// - Number of changes
|
||||
// - Token
|
||||
|
||||
if config.S3 == nil {
|
||||
errorMessage := "UploadS3: Uploading Failed, as no settings found"
|
||||
log.Error(errorMessage)
|
||||
@@ -72,34 +68,14 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
|
||||
s3Client.SetCustomTransport(transport)
|
||||
}
|
||||
|
||||
fileParts := strings.Split(fileName, "_")
|
||||
if len(fileParts) < 6 {
|
||||
errorMessage := "UploadS3: " + fileName + " is not a valid name."
|
||||
log.Error(errorMessage)
|
||||
return false, true, errors.New(errorMessage)
|
||||
}
|
||||
|
||||
deviceKey := config.Key
|
||||
if deviceKey == "" || strings.ContainsAny(deviceKey, "/\\") {
|
||||
errorMessage := "UploadS3: missing or invalid device key."
|
||||
log.Error(errorMessage)
|
||||
return false, true, errors.New(errorMessage)
|
||||
}
|
||||
startRecording, err := strconv.ParseInt(fileParts[0], 10, 64)
|
||||
objectName, userMetadata, err := s3RecordingUploadDetails(config, fileName, aws_access_key_id)
|
||||
if err != nil {
|
||||
errorMessage := "UploadS3: invalid recording timestamp in " + fileName + ": " + err.Error()
|
||||
log.Error(errorMessage)
|
||||
return false, true, errors.New(errorMessage)
|
||||
log.WithError(err).Error("UploadS3: invalid recording upload metadata")
|
||||
return false, true, err
|
||||
}
|
||||
devicename := fileParts[2]
|
||||
coordinates := fileParts[3]
|
||||
//numberOfChanges := fileParts[4]
|
||||
token, _ := strconv.Atoi(fileParts[5])
|
||||
|
||||
log.Info("UploadS3: Upload started for " + fileName)
|
||||
fullname := "data/recordings/" + fileName
|
||||
|
||||
file, err := os.OpenFile(fullname, os.O_RDWR, 0755)
|
||||
file, err := utils.OpenRecording("data/recordings", fileName)
|
||||
if file != nil {
|
||||
defer file.Close()
|
||||
}
|
||||
@@ -118,23 +94,13 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
|
||||
}
|
||||
|
||||
n, err := s3Client.PutObject(config.S3.Bucket,
|
||||
s3RecordingObjectName(config.S3.Username, deviceKey, startRecording),
|
||||
objectName,
|
||||
file,
|
||||
fileInfo.Size(),
|
||||
minio.PutObjectOptions{
|
||||
ContentType: "video/mp4",
|
||||
StorageClass: "ONEZONE_IA",
|
||||
UserMetadata: map[string]string{
|
||||
"event-timestamp": strconv.FormatInt(startRecording, 10),
|
||||
"event-microseconds": deviceKey,
|
||||
"event-instancename": devicename,
|
||||
"event-regioncoordinates": coordinates,
|
||||
"event-numberofchanges": deviceKey,
|
||||
"event-token": strconv.Itoa(token),
|
||||
"productid": deviceKey,
|
||||
"publickey": aws_access_key_id,
|
||||
"uploadtime": "now",
|
||||
},
|
||||
UserMetadata: userMetadata,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -148,6 +114,92 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
|
||||
}
|
||||
|
||||
func s3RecordingObjectName(username, deviceKey string, startRecording int64) string {
|
||||
return username + "/" + time.Unix(startRecording, 0).UTC().Format("2006/01/02") +
|
||||
return username + "/" + time.UnixMilli(startRecording).UTC().Format("2006/01/02") +
|
||||
"/" + strconv.FormatInt(startRecording, 10) + "-" + deviceKey + ".mp4"
|
||||
}
|
||||
|
||||
func s3RecordingUploadDetails(config models.Config, fileName, publicKey string) (string, map[string]string, error) {
|
||||
value, err := readRecordingUploadMetadataFile(fileName)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return "", nil, err
|
||||
}
|
||||
|
||||
deviceKey := config.Key
|
||||
var startRecording int64
|
||||
userMetadata := map[string]string{
|
||||
"publickey": publicKey,
|
||||
"uploadtime": "now",
|
||||
}
|
||||
marker := strings.TrimSpace(string(value))
|
||||
_, fpsErr := strconv.ParseFloat(marker, 64)
|
||||
if err == nil && marker != "" && fpsErr != nil {
|
||||
var metadata models.RecordingUploadMetadata
|
||||
if err := json.Unmarshal(value, &metadata); err != nil {
|
||||
return "", nil, fmt.Errorf("decode recording upload metadata for %s: %w", fileName, err)
|
||||
}
|
||||
if metadata.FileName != "" && metadata.FileName != filepath.Base(fileName) {
|
||||
return "", nil, fmt.Errorf("recording upload metadata filename does not match %s", fileName)
|
||||
}
|
||||
if metadata.Timestamp <= 0 {
|
||||
return "", nil, fmt.Errorf("invalid recording timestamp in upload metadata for %s", fileName)
|
||||
}
|
||||
deviceKey = metadata.DeviceKey
|
||||
startRecording = metadata.Timestamp
|
||||
userMetadata["event-instancename"] = config.Name
|
||||
if metadata.DeviceName != "" {
|
||||
userMetadata["event-instancename"] = metadata.DeviceName
|
||||
}
|
||||
if metadata.RegionCoordinates != "" {
|
||||
userMetadata["event-regioncoordinates"] = metadata.RegionCoordinates
|
||||
}
|
||||
if metadata.NumberOfChanges != "" {
|
||||
userMetadata["event-numberofchanges"] = metadata.NumberOfChanges
|
||||
}
|
||||
userMetadata["event-microseconds"] = strconv.FormatInt(metadata.Timestamp%1000*1000, 10)
|
||||
userMetadata["duration"] = strconv.FormatUint(metadata.Duration, 10)
|
||||
userMetadata["encrypted"] = strconv.FormatBool(metadata.Encrypted)
|
||||
if metadata.FPS > 0 && metadata.FPS <= 240 {
|
||||
userMetadata["fps"] = strconv.FormatFloat(metadata.FPS, 'f', -1, 64)
|
||||
}
|
||||
} else {
|
||||
// Empty and FPS-only markers predate structured recording metadata.
|
||||
fileParts := strings.Split(fileName, "_")
|
||||
if len(fileParts) < 6 {
|
||||
return "", nil, fmt.Errorf("%s is not a valid legacy recording name", fileName)
|
||||
}
|
||||
startRecording, err = strconv.ParseInt(fileParts[0], 10, 64)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("invalid recording timestamp in %s: %w", fileName, err)
|
||||
}
|
||||
if startRecording <= 0 || startRecording > math.MaxInt64/1000 {
|
||||
return "", nil, fmt.Errorf("invalid recording timestamp in %s", fileName)
|
||||
}
|
||||
startRecording *= 1000
|
||||
_, precision, found := strings.Cut(fileParts[1], "-")
|
||||
if found {
|
||||
fraction, err := strconv.ParseInt(precision, 10, 64)
|
||||
if err != nil || fraction < 0 || fraction >= 1000000 {
|
||||
return "", nil, fmt.Errorf("invalid recording precision in %s", fileName)
|
||||
}
|
||||
if len(precision) > 3 {
|
||||
fraction /= 1000
|
||||
}
|
||||
startRecording += fraction
|
||||
}
|
||||
token, _ := strconv.Atoi(fileParts[5])
|
||||
userMetadata["event-microseconds"] = deviceKey
|
||||
userMetadata["event-instancename"] = fileParts[2]
|
||||
userMetadata["event-regioncoordinates"] = fileParts[3]
|
||||
userMetadata["event-numberofchanges"] = deviceKey
|
||||
userMetadata["event-token"] = strconv.Itoa(token)
|
||||
}
|
||||
if deviceKey == "" || strings.ContainsAny(deviceKey, "/\\") {
|
||||
return "", nil, errors.New("missing or invalid recording device key")
|
||||
}
|
||||
if startRecording <= 0 {
|
||||
return "", nil, fmt.Errorf("invalid recording timestamp for %s", fileName)
|
||||
}
|
||||
userMetadata["event-timestamp"] = strconv.FormatInt(startRecording/1000, 10)
|
||||
userMetadata["productid"] = deviceKey
|
||||
return s3RecordingObjectName(config.S3.Username, deviceKey, startRecording), userMetadata, nil
|
||||
}
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
package cloud
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -9,6 +13,136 @@ import (
|
||||
"github.com/kerberos-io/agent/machinery/src/models"
|
||||
)
|
||||
|
||||
func TestS3RecordingUploadDetailsUsesCheckpoint(t *testing.T) {
|
||||
for _, fileName := range []string{
|
||||
"1767355200123-checkpoint-key.mp4",
|
||||
"1564859471_6-474162_old-camera_0-0-0-0_1153_27.mp4",
|
||||
} {
|
||||
t.Run(fileName, func(t *testing.T) {
|
||||
withRecording(t, fileName, []byte("recording"))
|
||||
recorded := time.Date(2026, time.January, 2, 12, 0, 0, 123000000, time.UTC)
|
||||
metadata := models.RecordingUploadMetadata{
|
||||
FileName: fileName,
|
||||
DeviceKey: "checkpoint-key",
|
||||
Timestamp: recorded.UnixMilli(),
|
||||
Duration: 20452,
|
||||
FPS: 29.97,
|
||||
Encrypted: true,
|
||||
DeviceName: "checkpoint-camera",
|
||||
RegionCoordinates: "1-2-3-4",
|
||||
NumberOfChanges: "57",
|
||||
}
|
||||
payload, err := json.Marshal(metadata)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
withQueuedRecordingFPS(t, fileName, string(payload))
|
||||
config := models.Config{
|
||||
Key: "changed-config-key",
|
||||
Name: "camera-name",
|
||||
S3: &models.S3{Username: "user"},
|
||||
}
|
||||
objectName, userMetadata, err := s3RecordingUploadDetails(config, fileName, "public-key")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantObjectName := "user/2026/01/02/" + strconv.FormatInt(recorded.UnixMilli(), 10) + "-checkpoint-key.mp4"
|
||||
if objectName != wantObjectName {
|
||||
t.Fatalf("object name = %q, want %q", objectName, wantObjectName)
|
||||
}
|
||||
wantMetadata := map[string]string{
|
||||
"publickey": "public-key",
|
||||
"uploadtime": "now",
|
||||
"event-instancename": "checkpoint-camera",
|
||||
"event-regioncoordinates": "1-2-3-4",
|
||||
"event-numberofchanges": "57",
|
||||
"event-microseconds": "123000",
|
||||
"event-timestamp": strconv.FormatInt(recorded.Unix(), 10),
|
||||
"productid": "checkpoint-key",
|
||||
"duration": "20452",
|
||||
"fps": "29.97",
|
||||
"encrypted": "true",
|
||||
}
|
||||
if !reflect.DeepEqual(userMetadata, wantMetadata) {
|
||||
t.Fatalf("metadata = %#v, want %#v", userMetadata, wantMetadata)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3RecordingUploadDetailsLegacyMarkers(t *testing.T) {
|
||||
for _, marker := range []string{"missing", "empty", "fps-only", "old-marker-name"} {
|
||||
t.Run(marker, func(t *testing.T) {
|
||||
const fileName = "1564859471_6-474162_camera_0-0-0-0_1153_27.mp4"
|
||||
withRecording(t, fileName, []byte("recording"))
|
||||
switch marker {
|
||||
case "empty":
|
||||
withQueuedRecordingFPS(t, fileName, "")
|
||||
case "fps-only", "old-marker-name":
|
||||
withQueuedRecordingFPS(t, fileName, "29.97")
|
||||
if marker == "old-marker-name" {
|
||||
err := os.Rename(filepath.Join("data/cloud", models.RecordingUploadMetadataFileName(fileName)),
|
||||
filepath.Join("data/cloud", fileName))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
config := models.Config{Key: "legacy-key", S3: &models.S3{Username: "user"}}
|
||||
objectName, metadata, err := s3RecordingUploadDetails(config, fileName, "public-key")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := "user/2019/08/03/1564859471474-legacy-key.mp4"; objectName != want {
|
||||
t.Fatalf("object name = %q, want %q", objectName, want)
|
||||
}
|
||||
if metadata["productid"] != "legacy-key" || metadata["event-instancename"] != "camera" ||
|
||||
metadata["event-regioncoordinates"] != "0-0-0-0" || metadata["event-timestamp"] != "1564859471" {
|
||||
t.Fatalf("unexpected legacy metadata: %#v", metadata)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3RecordingUploadDetailsRejectsInvalidCheckpoint(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
marker string
|
||||
want string
|
||||
}{
|
||||
{"malformed JSON", `{"timestamp":`, "decode"},
|
||||
{"missing timestamp", `{"device_key":"camera"}`, "timestamp"},
|
||||
{"missing device key", `{"timestamp":1787015373000}`, "device key"},
|
||||
{"invalid device key", `{"timestamp":1787015373000,"device_key":"camera/key"}`, "device key"},
|
||||
{"mismatched filename", `{"timestamp":1787015373000,"device_key":"camera","filename":"other.mp4"}`, "filename"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
const fileName = "1564859471_6-474162_camera_0-0-0-0_1153_27.mp4"
|
||||
withRecording(t, fileName, []byte("recording"))
|
||||
withQueuedRecordingFPS(t, fileName, tt.marker)
|
||||
config := models.Config{Key: "legacy-key", S3: &models.S3{Username: "user"}}
|
||||
_, _, err := s3RecordingUploadDetails(config, fileName, "public-key")
|
||||
if err == nil || !strings.Contains(err.Error(), tt.want) {
|
||||
t.Fatalf("error = %v, want error containing %q instead of legacy fallback", err, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3RecordingUploadDetailsReportsCheckpointReadError(t *testing.T) {
|
||||
const fileName = "1564859471_6-474162_camera_0-0-0-0_1153_27.mp4"
|
||||
withRecording(t, fileName, []byte("recording"))
|
||||
if err := os.MkdirAll(filepath.Join("data/cloud", models.RecordingUploadMetadataFileName(fileName)), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
config := models.Config{Key: "legacy-key", S3: &models.S3{Username: "user"}}
|
||||
_, _, err := s3RecordingUploadDetails(config, fileName, "public-key")
|
||||
if err == nil || !strings.Contains(err.Error(), "read recording upload metadata") {
|
||||
t.Fatalf("error = %v, want checkpoint read error instead of legacy fallback", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3RecordingObjectName(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -34,9 +168,9 @@ func TestS3RecordingObjectName(t *testing.T) {
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
const deviceKey = "camera-key-123"
|
||||
want := "user/" + tt.directory + "/" + strconv.FormatInt(tt.recorded.Unix(), 10) + "-" + deviceKey + ".mp4"
|
||||
want := "user/" + tt.directory + "/" + strconv.FormatInt(tt.recorded.UnixMilli(), 10) + "-" + deviceKey + ".mp4"
|
||||
for attempt := 0; attempt < 2; attempt++ {
|
||||
if got := s3RecordingObjectName("user", deviceKey, tt.recorded.Unix()); got != want {
|
||||
if got := s3RecordingObjectName("user", deviceKey, tt.recorded.UnixMilli()); got != want {
|
||||
t.Fatalf("s3RecordingObjectName() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/kerberos-io/agent/machinery/src/models"
|
||||
"github.com/kerberos-io/agent/machinery/src/utils"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
@@ -106,7 +107,7 @@ func logTusUploadProgress(label string, offset, size int64, loggedBucket *int64)
|
||||
// creation request; it is empty on HEAD/PATCH/DELETE.
|
||||
type tusHeaderFunc func(h http.Header, fileName string)
|
||||
|
||||
// runTusUpload performs a resumable (tus) upload of data/recordings/<fileName>
|
||||
// runTusUpload performs a resumable (tus) upload of a flat or dated recording.
|
||||
// to baseURL, sending target-specific authentication/routing headers via
|
||||
// setHeaders on every request. It encapsulates the create/resume/chunk/finalize
|
||||
// state machine shared by the Kerberos Vault (direct) and Kerberos Hub (proxied)
|
||||
@@ -120,9 +121,7 @@ type tusHeaderFunc func(h http.Header, fileName string)
|
||||
// should fall back to the legacy single-POST upload (older deployments).
|
||||
// - body: a short message for logging.
|
||||
func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tusHeaderFunc) (uploaded bool, responded bool, supported bool, body string, err error) {
|
||||
fullname := "data/recordings/" + fileName
|
||||
|
||||
file, ferr := os.Open(fullname)
|
||||
file, ferr := utils.OpenRecording("data/recordings", fileName)
|
||||
if file != nil {
|
||||
defer file.Close()
|
||||
}
|
||||
@@ -358,6 +357,12 @@ func addRecordingTusMetadata(values map[string]string, fileName string) {
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if metadata.DeviceKey != "" {
|
||||
values["device"] = metadata.DeviceKey
|
||||
}
|
||||
values["device_name"] = metadata.DeviceName
|
||||
values["region_coordinates"] = metadata.RegionCoordinates
|
||||
values["number_of_changes"] = metadata.NumberOfChanges
|
||||
if metadata.Duration > 0 {
|
||||
values["duration"] = strconv.FormatUint(metadata.Duration, 10)
|
||||
}
|
||||
@@ -500,6 +505,7 @@ func setVaultTusHeaders(h http.Header, vault models.KStorage, publicKey, deviceK
|
||||
h.Set("X-Kerberos-Storage-Capture", "IPCamera")
|
||||
if fileName != "" {
|
||||
h.Set("X-Kerberos-Storage-FileName", fileName)
|
||||
setQueuedRecordingMetadataHeaders(h, fileName)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -516,6 +522,7 @@ func setHubTusHeaders(h http.Header, config *models.Config, fileName string) {
|
||||
h.Set("X-Kerberos-Storage-Capture", "IPCamera")
|
||||
if fileName != "" {
|
||||
h.Set("X-Kerberos-Storage-FileName", fileName)
|
||||
setQueuedRecordingMetadataHeaders(h, fileName)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -18,8 +18,86 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/kerberos-io/agent/machinery/src/models"
|
||||
"github.com/kerberos-io/agent/machinery/src/utils"
|
||||
)
|
||||
|
||||
func TestUploadKerberosVaultReadsDatedLocalRecordings(t *testing.T) {
|
||||
previousCount, previousTimeout := kstorageRetryCount, kstorageRetryTimeout
|
||||
t.Cleanup(func() { kstorageRetryCount, kstorageRetryTimeout = previousCount, previousTimeout })
|
||||
for _, transport := range []string{"resumable", "legacy"} {
|
||||
t.Run(transport, func(t *testing.T) {
|
||||
const fileName = "1564859471123-device-key.mp4"
|
||||
payload := bytes.Repeat([]byte("video"), 1024)
|
||||
withRecording(t, fileName, payload)
|
||||
datedPath, err := utils.CreateRecordingPath("data/recordings", fileName, time.Unix(1564859471, 0))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Rename(filepath.Join("data/recordings", fileName), datedPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
withQueuedRecordingFPS(t, fileName, `{"filename":"1564859471123-device-key.mp4","device_key":"device-key","timestamp":1564859471123,"duration":27,"fps":25,"device_name":"camera-name","region_coordinates":"1-2-3-4","number_of_changes":"57"}`)
|
||||
fake := newFakeTus()
|
||||
var legacyBody []byte
|
||||
var legacyHeaders http.Header
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if transport == "resumable" {
|
||||
fake.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
body, readErr := io.ReadAll(r.Body)
|
||||
if readErr != nil {
|
||||
t.Error(readErr)
|
||||
}
|
||||
legacyBody = body
|
||||
legacyHeaders = r.Header.Clone()
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
if transport == "legacy" {
|
||||
t.Setenv("AGENT_DISABLE_RESUMABLE_UPLOAD", "true")
|
||||
} else {
|
||||
t.Setenv("AGENT_DISABLE_RESUMABLE_UPLOAD", "false")
|
||||
}
|
||||
kstorageRetryCount, kstorageRetryTimeout = 0, 0
|
||||
vault := testVault(server.URL)
|
||||
config := &models.Configuration{Config: models.Config{
|
||||
Key: "changed-config-key",
|
||||
KStorage: &vault,
|
||||
KStorageSecondary: &models.KStorage{},
|
||||
}}
|
||||
uploaded, configured, err := UploadKerberosVault(config, fileName)
|
||||
if err != nil || !uploaded || !configured {
|
||||
t.Fatalf("UploadKerberosVault() = (%v, %v, %v), want success", uploaded, configured, err)
|
||||
}
|
||||
if transport == "resumable" {
|
||||
if got := fake.totalBytes(); got != int64(len(payload)) {
|
||||
t.Fatalf("uploaded %d bytes, want %d", got, len(payload))
|
||||
}
|
||||
posts := fake.requestsForMethod(http.MethodPost)
|
||||
if len(posts) != 1 {
|
||||
t.Fatalf("got %d creation requests, want 1", len(posts))
|
||||
}
|
||||
metadata := decodeTusMetadata(posts[0].header.Get("Upload-Metadata"))
|
||||
if metadata["filename"] != fileName || metadata["directory"] != "dir" || metadata["timestamp"] != "1564859471123" ||
|
||||
metadata["device"] != "device-key" || metadata["device_name"] != "camera-name" ||
|
||||
metadata["region_coordinates"] != "1-2-3-4" || metadata["number_of_changes"] != "57" {
|
||||
t.Fatalf("unexpected upload metadata: %#v", metadata)
|
||||
}
|
||||
} else if !bytes.Equal(legacyBody, payload) ||
|
||||
legacyHeaders.Get("X-Kerberos-Storage-FileName") != fileName ||
|
||||
legacyHeaders.Get("X-Kerberos-Storage-Directory") != "dir" ||
|
||||
legacyHeaders.Get("X-Kerberos-Storage-Device") != "device-key" ||
|
||||
legacyHeaders.Get(recordingDeviceNameHeader) != "camera-name" ||
|
||||
legacyHeaders.Get(recordingRegionHeader) != "1-2-3-4" ||
|
||||
legacyHeaders.Get(recordingChangesHeader) != "57" ||
|
||||
legacyHeaders.Get(recordingTimestampHeader) != "1564859471123" {
|
||||
t.Fatalf("unexpected legacy upload: %d bytes, headers %#v", len(legacyBody), legacyHeaders)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// fakeUpload tracks the state of a single resumable upload on the fake server.
|
||||
type fakeUpload struct {
|
||||
size int64
|
||||
|
||||
@@ -964,7 +964,7 @@ func GetDashboard(c *gin.Context, configDirectory string, configuration *models.
|
||||
// All days stored in this agent.
|
||||
days := []string{}
|
||||
latestEvents := []models.Media{}
|
||||
files, err := utils.ReadDirectory(recordingDirectory)
|
||||
files, err := utils.ReadRecordingDirectory(recordingDirectory)
|
||||
if err == nil {
|
||||
events := utils.GetSortedDirectory(files)
|
||||
|
||||
@@ -975,6 +975,8 @@ func GetDashboard(c *gin.Context, configDirectory string, configuration *models.
|
||||
var eventFilter models.EventFilter
|
||||
eventFilter.NumberOfElements = 5
|
||||
latestEvents = utils.GetMediaFormatted(events, recordingDirectory, configuration, eventFilter) // will get 5 latest recordings.
|
||||
} else {
|
||||
log.WithError(err).Error("Could not list recordings for Agent status")
|
||||
}
|
||||
|
||||
c.JSON(200, gin.H{
|
||||
@@ -1007,7 +1009,7 @@ func GetLatestEvents(c *gin.Context, configDirectory string, configuration *mode
|
||||
eventFilter.NumberOfElements = 10
|
||||
}
|
||||
recordingDirectory := configDirectory + "/data/recordings"
|
||||
files, err := utils.ReadDirectory(recordingDirectory)
|
||||
files, err := utils.ReadRecordingDirectory(recordingDirectory)
|
||||
if err == nil {
|
||||
events := utils.GetSortedDirectory(files)
|
||||
// We will get all recordings from the directory (as defined by the filter).
|
||||
@@ -1036,7 +1038,7 @@ func GetLatestEvents(c *gin.Context, configDirectory string, configuration *mode
|
||||
// @Success 200
|
||||
func GetDays(c *gin.Context, configDirectory string, configuration *models.Configuration, communication *models.Communication) {
|
||||
recordingDirectory := configDirectory + "/data/recordings"
|
||||
files, err := utils.ReadDirectory(recordingDirectory)
|
||||
files, err := utils.ReadRecordingDirectory(recordingDirectory)
|
||||
if err == nil {
|
||||
events := utils.GetSortedDirectory(files)
|
||||
days := utils.GetDays(events, recordingDirectory, configuration)
|
||||
|
||||
@@ -11,12 +11,15 @@ const RecordingUploadMetadataExtension = ".metadata"
|
||||
// with a recording. New optional fields can be added without changing the queue
|
||||
// mechanism or breaking older agents.
|
||||
type RecordingUploadMetadata struct {
|
||||
FileName string `json:"filename"`
|
||||
DeviceKey string `json:"device_key"`
|
||||
Timestamp int64 `json:"timestamp"` // Unix milliseconds.
|
||||
Duration uint64 `json:"duration"` // Milliseconds.
|
||||
FPS float64 `json:"fps,omitempty"`
|
||||
Encrypted bool `json:"encrypted,omitempty"`
|
||||
FileName string `json:"filename"`
|
||||
DeviceKey string `json:"device_key"`
|
||||
Timestamp int64 `json:"timestamp"` // Unix milliseconds.
|
||||
Duration uint64 `json:"duration"` // Milliseconds.
|
||||
FPS float64 `json:"fps,omitempty"`
|
||||
Encrypted bool `json:"encrypted,omitempty"`
|
||||
DeviceName string `json:"device_name,omitempty"`
|
||||
RegionCoordinates string `json:"region_coordinates,omitempty"`
|
||||
NumberOfChanges string `json:"number_of_changes,omitempty"`
|
||||
}
|
||||
|
||||
// RecordingUploadMetadataFileName returns the queue marker name associated
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/kerberos-io/agent/machinery/src/encryption"
|
||||
"github.com/kerberos-io/agent/machinery/src/models"
|
||||
)
|
||||
|
||||
func TestFilesServesFlatAndDatedRecordings(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
for _, layout := range []string{"flat", "dated"} {
|
||||
for _, encrypted := range []bool{false, true} {
|
||||
name := layout + "-plain"
|
||||
if encrypted {
|
||||
name = layout + "-encrypted"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
const fileName = "1767355200123-device-key.mp4"
|
||||
directory := t.TempDir()
|
||||
relativePath := fileName
|
||||
if layout == "dated" {
|
||||
relativePath = "2026/01/02/" + fileName
|
||||
}
|
||||
filePath := filepath.Join(directory, "data/recordings", relativePath)
|
||||
if err := os.MkdirAll(filepath.Dir(filePath), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
config := &models.Configuration{Config: models.Config{Encryption: &models.Encryption{}}}
|
||||
content := []byte("recording-payload")
|
||||
if encrypted {
|
||||
config.Config.Encryption.Recordings = "true"
|
||||
config.Config.Encryption.SymmetricKey = "0123456789abcdef0123456789abcdef"
|
||||
ciphertext, err := encryption.AesEncrypt(content, config.Config.Encryption.SymmetricKey)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
content = []byte(ciphertext)
|
||||
}
|
||||
if err := os.WriteFile(filePath, content, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
router := gin.New()
|
||||
router.GET("/file/*filepath", func(c *gin.Context) { Files(c, directory, config) })
|
||||
for _, reference := range []string{relativePath, fileName} {
|
||||
response := httptest.NewRecorder()
|
||||
router.ServeHTTP(response, httptest.NewRequest("GET", "/file/"+reference, nil))
|
||||
if response.Code != 200 || response.Body.String() != "recording-payload" {
|
||||
t.Fatalf("GET %s = %d %q", reference, response.Code, response.Body.String())
|
||||
}
|
||||
if got := response.Header().Get("Content-Type"); got != "video/mp4" {
|
||||
t.Fatalf("Content-Type = %q, want video/mp4", got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilesRejectsMissingAndOutsideRecordings(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(directory, "data/recordings"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(directory, "data/outside.mp4"), []byte("not a recording"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
router := gin.New()
|
||||
router.GET("/file/*filepath", func(c *gin.Context) { Files(c, directory, &models.Configuration{}) })
|
||||
for _, reference := range []string{"missing.mp4", "../outside.mp4", "2026/01/02/missing.mp4"} {
|
||||
response := httptest.NewRecorder()
|
||||
router.ServeHTTP(response, httptest.NewRequest("GET", "/file/"+reference, nil))
|
||||
if response.Code != 404 {
|
||||
t.Fatalf("GET %s = %d, want 404", reference, response.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
jwt "github.com/appleboy/gin-jwt/v2"
|
||||
@@ -19,6 +20,7 @@ import (
|
||||
"github.com/kerberos-io/agent/machinery/src/capture"
|
||||
"github.com/kerberos-io/agent/machinery/src/encryption"
|
||||
"github.com/kerberos-io/agent/machinery/src/models"
|
||||
"github.com/kerberos-io/agent/machinery/src/utils"
|
||||
swaggerFiles "github.com/swaggo/files"
|
||||
ginSwagger "github.com/swaggo/gin-swagger"
|
||||
)
|
||||
@@ -160,9 +162,9 @@ func requestLogger() gin.HandlerFunc {
|
||||
func Files(c *gin.Context, configDirectory string, configuration *models.Configuration) {
|
||||
|
||||
// Get File
|
||||
filePath := configDirectory + "/data/recordings" + c.Param("filepath")
|
||||
_, err := os.Open(filePath)
|
||||
filePath, err := utils.ResolveRecordingPath(configDirectory+"/data/recordings", strings.TrimPrefix(c.Param("filepath"), "/"))
|
||||
if err != nil {
|
||||
log.WithError(err).Warn("Could not resolve recording for playback")
|
||||
c.JSON(404, gin.H{"error": "File not found"})
|
||||
return
|
||||
}
|
||||
|
||||
+45
-80
@@ -191,51 +191,32 @@ func GetMediaFormatted(files []os.FileInfo, recordingDirectory string, configura
|
||||
count := 0
|
||||
for _, file := range files {
|
||||
fileName := file.Name()
|
||||
fileParts := strings.Split(fileName, "_")
|
||||
if len(fileParts) == 6 {
|
||||
timestamp := fileParts[0]
|
||||
timestampInt, err := strconv.ParseInt(timestamp, 10, 64)
|
||||
if err == nil {
|
||||
|
||||
if eventFilter.TimestampOffsetStart > 0 {
|
||||
// TimestampOffsetStart represents the newest lower bound to include.
|
||||
if timestampInt < eventFilter.TimestampOffsetStart {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// If we have an offset we will check if we should skip or not
|
||||
if eventFilter.TimestampOffsetEnd > 0 {
|
||||
// Medias are sorted from new to older. TimestampOffsetEnd holds the oldest
|
||||
// timestamp of the previous batch of events. By doing this check, we make sure
|
||||
// to skip the previous batch.
|
||||
if timestampInt >= eventFilter.TimestampOffsetEnd {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
loc, _ := time.LoadLocation(configuration.Config.Timezone)
|
||||
time := time.Unix(timestampInt, 0).In(loc)
|
||||
day := time.Format("02-01-2006")
|
||||
timeString := time.Format("15:04:05")
|
||||
shortDay := time.Format("Jan _2")
|
||||
|
||||
media := models.Media{
|
||||
Key: fileName,
|
||||
Path: recordingDirectory + "/" + fileName,
|
||||
CameraName: configuration.Config.Name,
|
||||
CameraKey: configuration.Config.Key,
|
||||
Day: day,
|
||||
ShortDay: shortDay,
|
||||
Time: timeString,
|
||||
Timestamp: timestamp,
|
||||
}
|
||||
filePaths = append(filePaths, media)
|
||||
count = count + 1
|
||||
if eventFilter.NumberOfElements > 0 && count >= eventFilter.NumberOfElements {
|
||||
break
|
||||
}
|
||||
}
|
||||
recordedAt, err := RecordingStartTime(fileName)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
timestamp := recordedAt.Unix()
|
||||
if eventFilter.TimestampOffsetStart > 0 && timestamp < eventFilter.TimestampOffsetStart {
|
||||
continue
|
||||
}
|
||||
if eventFilter.TimestampOffsetEnd > 0 && timestamp >= eventFilter.TimestampOffsetEnd {
|
||||
continue
|
||||
}
|
||||
loc, _ := time.LoadLocation(configuration.Config.Timezone)
|
||||
localTime := recordedAt.In(loc)
|
||||
filePaths = append(filePaths, models.Media{
|
||||
Key: fileName,
|
||||
Path: recordingDirectory + "/" + fileName,
|
||||
CameraName: configuration.Config.Name,
|
||||
CameraKey: configuration.Config.Key,
|
||||
Day: localTime.Format("02-01-2006"),
|
||||
ShortDay: localTime.Format("Jan _2"),
|
||||
Time: localTime.Format("15:04:05"),
|
||||
Timestamp: strconv.FormatInt(timestamp, 10),
|
||||
})
|
||||
count++
|
||||
if eventFilter.NumberOfElements > 0 && count >= eventFilter.NumberOfElements {
|
||||
break
|
||||
}
|
||||
}
|
||||
return filePaths
|
||||
@@ -244,18 +225,12 @@ func GetMediaFormatted(files []os.FileInfo, recordingDirectory string, configura
|
||||
func GetDays(files []os.FileInfo, recordingDirectory string, configuration *models.Configuration) []string {
|
||||
days := []string{}
|
||||
for _, file := range files {
|
||||
fileName := file.Name()
|
||||
fileParts := strings.Split(fileName, "_")
|
||||
if len(fileParts) == 6 {
|
||||
timestamp := fileParts[0]
|
||||
timestampInt, err := strconv.ParseInt(timestamp, 10, 64)
|
||||
if err == nil {
|
||||
loc, _ := time.LoadLocation(configuration.Config.Timezone)
|
||||
time := time.Unix(timestampInt, 0).In(loc)
|
||||
day := time.Format("02-01-2006")
|
||||
days = append(days, day)
|
||||
}
|
||||
recordedAt, err := RecordingStartTime(file.Name())
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
loc, _ := time.LoadLocation(configuration.Config.Timezone)
|
||||
days = append(days, recordedAt.In(loc).Format("02-01-2006"))
|
||||
}
|
||||
uniqueDays := Unique(days)
|
||||
return uniqueDays
|
||||
@@ -278,10 +253,13 @@ func NumberOfFilesInDirectory(path string) int {
|
||||
return len(files)
|
||||
}
|
||||
|
||||
// NumberOfMP4sInDirectory returns the count of all files with mp4 extension in current directory
|
||||
// NumberOfMP4sInDirectory counts MP4 files, including dated subdirectories.
|
||||
func NumberOfMP4sInDirectory(path string) int {
|
||||
pattern := filepath.Join(path, "*.mp4")
|
||||
files, _ := filepath.Glob(pattern)
|
||||
files, err := ReadRecordingDirectory(path)
|
||||
if err != nil {
|
||||
log.WithError(err).Error("Could not count recordings")
|
||||
return 0
|
||||
}
|
||||
return len(files)
|
||||
}
|
||||
|
||||
@@ -409,25 +387,13 @@ func Decrypt(directoryOrFile string, symmetricKey []byte) {
|
||||
|
||||
var files []string
|
||||
if fileInfo.IsDir() {
|
||||
// Create decrypted directory
|
||||
err = os.MkdirAll(directoryOrFile+"/decrypted", 0755)
|
||||
if err != nil {
|
||||
log.Fatal(err.Error())
|
||||
return
|
||||
}
|
||||
dir, err := os.ReadDir(directoryOrFile)
|
||||
dir, err := ReadRecordingDirectory(directoryOrFile)
|
||||
if err != nil {
|
||||
log.Fatal(err.Error())
|
||||
return
|
||||
}
|
||||
for _, file := range dir {
|
||||
// Check if file is not a directory
|
||||
if !file.IsDir() {
|
||||
// Check if an mp4 file
|
||||
if strings.HasSuffix(file.Name(), ".mp4") {
|
||||
files = append(files, directoryOrFile+"/"+file.Name())
|
||||
}
|
||||
}
|
||||
files = append(files, filepath.Join(directoryOrFile, file.Name()))
|
||||
}
|
||||
} else {
|
||||
files = append(files, directoryOrFile)
|
||||
@@ -449,13 +415,12 @@ func Decrypt(directoryOrFile string, symmetricKey []byte) {
|
||||
return
|
||||
}
|
||||
|
||||
// Write decrypted content to file with appended .decrypted
|
||||
// Get filename split by / and get last element.
|
||||
fileParts := strings.Split(file, "/")
|
||||
fileName := fileParts[len(fileParts)-1]
|
||||
pathToFile := strings.Join(fileParts[:len(fileParts)-1], "/")
|
||||
|
||||
err = os.WriteFile(pathToFile+"/decrypted/"+fileName, []byte(decrypted), 0644)
|
||||
outputDirectory := filepath.Join(filepath.Dir(file), "decrypted")
|
||||
if err := os.MkdirAll(outputDirectory, 0o755); err != nil {
|
||||
log.Fatal(err.Error())
|
||||
return
|
||||
}
|
||||
err = os.WriteFile(filepath.Join(outputDirectory, filepath.Base(file)), []byte(decrypted), 0644)
|
||||
if err != nil {
|
||||
log.Fatal(err.Error())
|
||||
return
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func RecordingFileName(timestamp int64, deviceKey string) (string, error) {
|
||||
if timestamp <= 0 {
|
||||
return "", fmt.Errorf("invalid recording timestamp %d", timestamp)
|
||||
}
|
||||
if deviceKey == "" || strings.ContainsAny(deviceKey, "/\\") {
|
||||
return "", fmt.Errorf("missing or invalid recording device key")
|
||||
}
|
||||
return strconv.FormatInt(timestamp, 10) + "-" + deviceKey + ".mp4", nil
|
||||
}
|
||||
|
||||
func RecordingStartTime(name string) (time.Time, error) {
|
||||
base := filepath.Base(name)
|
||||
if !strings.HasSuffix(base, ".mp4") {
|
||||
return time.Time{}, fmt.Errorf("invalid recording filename %q", name)
|
||||
}
|
||||
timestamp, deviceKey, short := strings.Cut(strings.TrimSuffix(base, ".mp4"), "-")
|
||||
if short && deviceKey != "" {
|
||||
if millis, err := strconv.ParseInt(timestamp, 10, 64); err == nil && millis > 0 {
|
||||
return time.UnixMilli(millis), nil
|
||||
}
|
||||
}
|
||||
parts := strings.Split(base, "_")
|
||||
if len(parts) == 6 {
|
||||
seconds, err := strconv.ParseInt(parts[0], 10, 64)
|
||||
if err == nil && seconds > 0 {
|
||||
return time.Unix(seconds, 0), nil
|
||||
}
|
||||
}
|
||||
return time.Time{}, fmt.Errorf("invalid recording filename %q", name)
|
||||
}
|
||||
|
||||
func CreateRecordingPath(directory, fileName string, recordedAt time.Time) (string, error) {
|
||||
if !filepath.IsLocal(fileName) || fileName == "." || filepath.Base(fileName) != fileName || strings.ContainsAny(fileName, "/\\") {
|
||||
return "", fmt.Errorf("invalid recording filename %q", fileName)
|
||||
}
|
||||
if recordedAt.UnixMilli() <= 0 {
|
||||
return "", fmt.Errorf("invalid recording start time %s", recordedAt)
|
||||
}
|
||||
directory = filepath.Join(directory, recordedAt.UTC().Format("2006/01/02"))
|
||||
if err := os.MkdirAll(directory, 0o755); err != nil {
|
||||
return "", fmt.Errorf("create recording directory: %w", err)
|
||||
}
|
||||
return filepath.Join(directory, fileName), nil
|
||||
}
|
||||
|
||||
// ResolveRecordingPath accepts relative paths from listings and historical
|
||||
// basename-only references from upload markers and playback URLs.
|
||||
func ResolveRecordingPath(directory, name string) (string, error) {
|
||||
if !filepath.IsLocal(name) {
|
||||
return "", fmt.Errorf("invalid recording path %q", name)
|
||||
}
|
||||
filePath := filepath.Join(directory, name)
|
||||
info, err := os.Stat(filePath)
|
||||
if os.IsNotExist(err) && filepath.Base(name) == name {
|
||||
recordedAt, parseErr := RecordingStartTime(name)
|
||||
if parseErr == nil {
|
||||
filePath = filepath.Join(directory, recordedAt.UTC().Format("2006/01/02"), name)
|
||||
info, err = os.Stat(filePath)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return "", fmt.Errorf("recording %q is not a regular file", name)
|
||||
}
|
||||
return filePath, nil
|
||||
}
|
||||
|
||||
func OpenRecording(directory, name string) (*os.File, error) {
|
||||
filePath, err := ResolveRecordingPath(directory, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return os.Open(filePath)
|
||||
}
|
||||
|
||||
func RemoveRecording(directory, name string) error {
|
||||
filePath, err := ResolveRecordingPath(directory, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Remove(filePath)
|
||||
}
|
||||
|
||||
type recordingFileInfo struct {
|
||||
os.FileInfo
|
||||
relativePath string
|
||||
}
|
||||
|
||||
func (file recordingFileInfo) Name() string {
|
||||
return file.relativePath
|
||||
}
|
||||
|
||||
// ReadRecordingDirectory returns MP4s with root-relative names, preserving the
|
||||
// file information used by the existing sorting and media formatting helpers.
|
||||
func ReadRecordingDirectory(directory string) ([]os.FileInfo, error) {
|
||||
files := []os.FileInfo{}
|
||||
err := filepath.WalkDir(directory, func(filePath string, entry fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.IsDir() && entry.Name() == "decrypted" && filePath != directory {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
if !entry.Type().IsRegular() || filepath.Ext(entry.Name()) != ".mp4" {
|
||||
return nil
|
||||
}
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
name, err := filepath.Rel(directory, filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
files = append(files, recordingFileInfo{FileInfo: info, relativePath: filepath.ToSlash(name)})
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return files, nil
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kerberos-io/agent/machinery/src/encryption"
|
||||
"github.com/kerberos-io/agent/machinery/src/models"
|
||||
)
|
||||
|
||||
func TestShortRecordingNamePreservesMillisecondsAndDeviceKey(t *testing.T) {
|
||||
const timestamp = int64(1767355200123)
|
||||
const deviceKey = "device-key_with-hyphens"
|
||||
name, err := RecordingFileName(timestamp, deviceKey)
|
||||
if err != nil || name != "1767355200123-device-key_with-hyphens.mp4" {
|
||||
t.Fatalf("RecordingFileName() = %q, %v", name, err)
|
||||
}
|
||||
other, err := RecordingFileName(timestamp+1, deviceKey)
|
||||
if err != nil || other == name {
|
||||
t.Fatalf("recordings within the same second must have distinct names: %q, %v", other, err)
|
||||
}
|
||||
directory := t.TempDir()
|
||||
filePath, err := CreateRecordingPath(directory, name, time.UnixMilli(timestamp))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := filepath.Join(directory, "2026/01/02", name); filePath != want {
|
||||
t.Fatalf("path = %q, want %q", filePath, want)
|
||||
}
|
||||
if err := os.WriteFile(filePath, []byte("video"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resolved, err := ResolveRecordingPath(directory, name)
|
||||
if err != nil || resolved != filePath {
|
||||
t.Fatalf("ResolveRecordingPath() = %q, %v", resolved, err)
|
||||
}
|
||||
files, err := ReadRecordingDirectory(directory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
config := &models.Configuration{Config: models.Config{Timezone: "UTC"}}
|
||||
media := GetMediaFormatted(files, directory, config, models.EventFilter{TimestampOffsetStart: timestamp / 1000})
|
||||
if len(media) != 1 || media[0].Key != "2026/01/02/"+name || media[0].Timestamp != "1767355200" {
|
||||
t.Fatalf("unexpected short-name media: %+v", media)
|
||||
}
|
||||
if days := GetDays(files, directory, config); !reflect.DeepEqual(days, []string{"02-01-2026"}) {
|
||||
t.Fatalf("unexpected short-name days: %v", days)
|
||||
}
|
||||
if err := RemoveRecording(directory, name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, invalidKey := range []string{"", "camera/key", `camera\key`} {
|
||||
if _, err := RecordingFileName(timestamp, invalidKey); err == nil {
|
||||
t.Errorf("accepted invalid device key %q", invalidKey)
|
||||
}
|
||||
}
|
||||
if _, err := RecordingFileName(0, deviceKey); err == nil {
|
||||
t.Fatal("accepted invalid recording timestamp")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateRecordingPathUsesRecordingUTCDate(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
recordedAt := time.Date(2026, time.January, 1, 1, 0, 0, 0, time.FixedZone("UTC+2", 2*60*60))
|
||||
name := strconv.FormatInt(recordedAt.Unix(), 10) + "_3-123_camera_0-0-0-0_1_1000.mp4"
|
||||
filePath, err := CreateRecordingPath(directory, name, recordedAt)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := filepath.Join(directory, "2025/12/31", name); filePath != want {
|
||||
t.Fatalf("path = %q, want %q", filePath, want)
|
||||
}
|
||||
if err := os.WriteFile(filePath, []byte("video"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, reference := range []string{name, "2025/12/31/" + name} {
|
||||
got, err := ResolveRecordingPath(directory, reference)
|
||||
if err != nil || got != filePath {
|
||||
t.Fatalf("ResolveRecordingPath(%q) = %q, %v; want %q", reference, got, err, filePath)
|
||||
}
|
||||
file, err := OpenRecording(directory, reference)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
content, err := io.ReadAll(file)
|
||||
file.Close()
|
||||
if err != nil || string(content) != "video" {
|
||||
t.Fatalf("recording content = %q, %v", content, err)
|
||||
}
|
||||
}
|
||||
if err := RemoveRecording(directory, name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(filePath); !os.IsNotExist(err) {
|
||||
t.Fatalf("recording still exists after removal: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordingPathsKeepFlatFilesAccessible(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
name := "legacy-recording.mp4"
|
||||
filePath := filepath.Join(directory, name)
|
||||
if err := os.WriteFile(filePath, []byte("legacy"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := ResolveRecordingPath(directory, name)
|
||||
if err != nil || got != filePath {
|
||||
t.Fatalf("ResolveRecordingPath() = %q, %v; want %q", got, err, filePath)
|
||||
}
|
||||
if err := RemoveRecording(directory, name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordingPathsReportInvalidAndMissingPaths(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
for _, name := range []string{"", ".", "..", "../outside.mp4", "/outside.mp4", "nested/recording.mp4", `nested\recording.mp4`} {
|
||||
if _, err := CreateRecordingPath(directory, name, time.Now()); err == nil {
|
||||
t.Errorf("CreateRecordingPath(%q) should reject invalid filename", name)
|
||||
}
|
||||
}
|
||||
for _, name := range []string{"", ".", "../outside.mp4", "/outside.mp4"} {
|
||||
if _, err := ResolveRecordingPath(directory, name); err == nil {
|
||||
t.Errorf("ResolveRecordingPath(%q) should reject invalid path", name)
|
||||
}
|
||||
}
|
||||
for _, name := range []string{"missing.mp4", "1700000000_missing.mp4", "2026/01/01/missing.mp4"} {
|
||||
if _, err := ResolveRecordingPath(directory, name); !os.IsNotExist(err) {
|
||||
t.Errorf("ResolveRecordingPath(%q) error = %v, want not-exist", name, err)
|
||||
}
|
||||
}
|
||||
blocked := filepath.Join(directory, "not-a-directory")
|
||||
if err := os.WriteFile(blocked, nil, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := CreateRecordingPath(blocked, "recording.mp4", time.Now()); err == nil {
|
||||
t.Fatal("expected directory creation error")
|
||||
}
|
||||
if _, err := CreateRecordingPath(directory, "recording.mp4", time.Time{}); err == nil {
|
||||
t.Fatal("expected invalid recording start time error")
|
||||
}
|
||||
if _, err := ReadRecordingDirectory(filepath.Join(directory, "missing")); !os.IsNotExist(err) {
|
||||
t.Fatalf("ReadRecordingDirectory() error = %v, want not-exist", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadRecordingDirectoryIncludesFlatAndDatedMedia(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
names := []string{
|
||||
"1700000000_6_7_8_9_10.mp4",
|
||||
"2026/01/02/1767355200_6_7_8_9_10.mp4",
|
||||
}
|
||||
for index, name := range names {
|
||||
filePath := filepath.Join(directory, name)
|
||||
if err := os.MkdirAll(filepath.Dir(filePath), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filePath, []byte("video"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
modified := time.Unix(int64(index+1), 0)
|
||||
if err := os.Chtimes(filePath, modified, modified); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(directory, "2026/01/02/ignore.txt"), nil, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(directory, "2026/01/02/decrypted"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(directory, "2026/01/02/decrypted/output.mp4"), nil, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
files, err := ReadRecordingDirectory(directory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(files) != 2 || NumberOfMP4sInDirectory(directory) != 2 {
|
||||
t.Fatalf("expected 2 MP4 recordings, got %d", len(files))
|
||||
}
|
||||
config := &models.Configuration{Config: models.Config{Timezone: "UTC", Name: "camera", Key: "device"}}
|
||||
files = GetSortedDirectory(files)
|
||||
media := GetMediaFormatted(files, directory, config, models.EventFilter{})
|
||||
if len(media) != 2 || media[0].Key != names[1] || media[1].Key != names[0] {
|
||||
t.Fatalf("unexpected media ordering or relative keys: %+v", media)
|
||||
}
|
||||
if media[0].Path != filepath.Join(directory, names[1]) || media[0].Timestamp != "1767355200" {
|
||||
t.Fatalf("unexpected nested media path or timestamp: %+v", media[0])
|
||||
}
|
||||
if got, want := GetDays(files, directory, config), []string{"02-01-2026", "14-11-2023"}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("days = %v, want %v", got, want)
|
||||
}
|
||||
filtered := GetMediaFormatted(files, directory, config, models.EventFilter{TimestampOffsetStart: 1767355200})
|
||||
if len(filtered) != 1 || filtered[0].Key != names[1] {
|
||||
t.Fatalf("unexpected filtered nested media: %+v", filtered)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecryptDirectoryIncludesDatedRecordingsAndSkipsOutputs(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
key := []byte("0123456789abcdef0123456789abcdef")
|
||||
payload := []byte("recording-payload")
|
||||
ciphertext, err := encryption.AesEncrypt(payload, string(key))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
names := []string{"legacy.mp4", "2026/01/02/recording.mp4"}
|
||||
for _, name := range names {
|
||||
filePath := filepath.Join(directory, name)
|
||||
if err := os.MkdirAll(filepath.Dir(filePath), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filePath, []byte(ciphertext), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
Decrypt(directory, key)
|
||||
Decrypt(directory, key)
|
||||
for _, name := range names {
|
||||
output := filepath.Join(directory, filepath.Dir(name), "decrypted", filepath.Base(name))
|
||||
content, err := os.ReadFile(output)
|
||||
if err != nil || string(content) != string(payload) {
|
||||
t.Fatalf("decrypted %q = %q, %v", name, content, err)
|
||||
}
|
||||
}
|
||||
if got := NumberOfMP4sInDirectory(directory); got != len(names) {
|
||||
t.Fatalf("recording count = %d, want %d (excluding decrypted outputs)", got, len(names))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user