diff --git a/README.md b/README.md index fbe5f30..2cc0d5a 100644 --- a/README.md +++ b/README.md @@ -80,22 +80,53 @@ There are a myriad of cameras out there (USB, IP and other cameras), and it migh Direct S3 uploads store recordings as `/yyyy/mm/dd/-.mp4`. The timestamp is the -recording's Unix timestamp in seconds, derived from its local filename, and the -date is its UTC date, not the upload date. The device key is the configured Agent -key and must be non-empty and contain no path separators. Retried uploads use -the same object key, even after midnight. Recordings from the same device starting -in the same second share an object key. +recording's Unix timestamp in milliseconds, and the date is its UTC date, not the upload +date. Timestamp and device key come from the recording's `.metadata` upload +checkpoint, which stores the timestamp in milliseconds. The uploader also +preserves checkpoint duration, FPS (when available), and encryption status as S3 +object metadata. The device key must be non-empty and contain no path separators. +Retried uploads use the same object key, even after midnight. Millisecond +precision keeps recordings starting within the same second distinct. This replaces the previous `/.mp4` layout for new uploads. Existing objects are not moved; consumers of direct S3 uploads must support both -layouts when reading historical recordings. Local Agent recordings and uploads -through Kerberos Vault or Kerberos Hub are unchanged. +layouts when reading historical recordings. Updated Kerberos Vault deployments +use the same filename and dated layout under the authenticated account directory +for both direct Vault uploads and Hub-proxied uploads. -The shorter S3 filename cannot be decoded by legacy filename-only consumers. +Only historical recordings with missing, empty, or FPS-only queue markers fall +back to the legacy filename format and configured Agent key. A structured +checkpoint takes precedence over both the filename and current configured key; +an invalid or unreadable checkpoint is reported as an upload error, not silently +replaced with filename-derived values. + +The shorter filename cannot be decoded by legacy filename-only consumers. Integrations must preserve structured recording metadata, including the device key (`productid`) and recording timestamp (`event-timestamp`), as well as the full object key. +### Local recording layout + +New local recordings are stored under +`data/recordings/yyyy/mm/dd/-.mp4`, using the recording's UTC +start date and Unix-millisecond timestamp. For example: +`data/recordings/2026/01/02/1767355200123-device-key.mp4`. +The filename no longer embeds the camera name, region, change count, or duration, +and it does not change when the recording is finalized. +Those attributes, together with FPS and encryption status, are captured in the +`.metadata` upload checkpoint and sent as structured HTTP/tus metadata. + +Existing flat recordings and dated recordings with older long filenames remain +supported without a migration. Recording lists, day filters, playback, uploaders, +and automatic cleanup handle both filename formats and layouts. Upload queue +checkpoints remain flat in `data/cloud/`. Successful-upload removal also resolves +both local layouts. Deploy the updated Vault alongside Agent for canonical +`/yyyy/mm/dd/-.mp4` storage keys and +metadata-driven Hub integration events. +Directory decryption also processes dated subdirectories and writes each output +to a sibling `decrypted/` directory; those outputs are excluded from recording +lists and automatic cleanup. + ## Quickstart - Docker The easiest way to get your Kerberos Agent up and running is to use our public image on [Docker hub](https://hub.docker.com/r/kerberos/agent). Once you have selected a specific tag, run `docker` command below, which will open the web interface of your Kerberos agent on port `80`, and off you go. For a more configurable and persistent deployment have a look at [Running and automating a Kerberos Agent](#running-and-automating-a-kerberos-agent). diff --git a/machinery/src/capture/cleanup_test.go b/machinery/src/capture/cleanup_test.go index 2d750b3..58f4075 100644 --- a/machinery/src/capture/cleanup_test.go +++ b/machinery/src/capture/cleanup_test.go @@ -10,11 +10,58 @@ import ( "github.com/kerberos-io/agent/machinery/src/models" ) +func TestPickRecordingToCleanupAcrossDatedDirectories(t *testing.T) { + for _, legacyMarker := range []bool{false, true} { + t.Run(map[bool]string{false: "metadata", true: "legacy"}[legacyMarker], func(t *testing.T) { + recordingsDir, cloudDir := newCleanupDirs(t) + pendingName := "2026/01/01/oldest_pending.mp4" + safeName := "2026/01/02/uploaded.mp4" + writeRecording(t, recordingsDir, pendingName, 30) + if legacyMarker { + if err := os.WriteFile(filepath.Join(cloudDir, filepath.Base(pendingName)), nil, 0o644); err != nil { + t.Fatal(err) + } + } else { + markPending(t, cloudDir, pendingName) + } + writeRecording(t, recordingsDir, safeName, 20) + writeRecording(t, recordingsDir, "flat_uploaded.mp4", 10) + name, pending, err := pickRecordingToCleanup(recordingsDir, cloudDir) + if err != nil || pending || name != safeName { + t.Fatalf("cleanup = (%q, %v, %v), want (%q, false, nil)", name, pending, err, safeName) + } + }) + } +} + +func TestCleanupDatedPendingRecordingRemovesFlatMarker(t *testing.T) { + recordingsDir, cloudDir := newCleanupDirs(t) + name := "2026/01/01/pending.mp4" + if err := os.MkdirAll(filepath.Dir(filepath.Join(recordingsDir, name)), 0o755); err != nil { + t.Fatal(err) + } + writeSizedRecording(t, recordingsDir, name, 2*1000*1000) + markPending(t, cloudDir, name) + config := &models.Configuration{Config: models.Config{AutoClean: "true", MaxDirectorySize: 1}} + CleanupRecordingDirectory(filepath.Dir(filepath.Dir(recordingsDir)), config) + for _, filePath := range []string{ + filepath.Join(recordingsDir, name), + filepath.Join(cloudDir, models.RecordingUploadMetadataFileName(name)), + } { + if _, err := os.Stat(filePath); !os.IsNotExist(err) { + t.Fatalf("expected %s to be removed, got %v", filePath, err) + } + } +} + // writeRecording creates a file under recordingsDir and sets its modtime so the // tests can control the "oldest" ordering deterministically. func writeRecording(t *testing.T, recordingsDir, name string, ageMinutes int) { t.Helper() full := filepath.Join(recordingsDir, name) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatalf("mkdir recording directory: %v", err) + } if err := os.WriteFile(full, []byte("data"), 0o644); err != nil { t.Fatalf("write recording %s: %v", name, err) } diff --git a/machinery/src/capture/main.go b/machinery/src/capture/main.go index 18daff8..958487b 100644 --- a/machinery/src/capture/main.go +++ b/machinery/src/capture/main.go @@ -53,13 +53,16 @@ func publishRecordingState(mqttClient mqtt.Client, hubKey string, configuration } } -func recordingUploadMetadata(name, deviceKey string, timestamp int64, mp4Video *video.MP4, encrypted bool) models.RecordingUploadMetadata { +func recordingUploadMetadata(name string, config models.Config, timestamp int64, mp4Video *video.MP4, encrypted bool, coordinates string, changes int) models.RecordingUploadMetadata { metadata := models.RecordingUploadMetadata{ - FileName: filepath.Base(name), - DeviceKey: deviceKey, - Timestamp: timestamp, - Duration: mp4Video.VideoTotalDuration, - Encrypted: encrypted, + FileName: filepath.Base(name), + DeviceKey: config.Key, + Timestamp: timestamp, + Duration: mp4Video.VideoTotalDuration, + Encrypted: encrypted, + DeviceName: config.Name, + RegionCoordinates: coordinates, + NumberOfChanges: strconv.Itoa(changes), } value := mp4Video.AverageFPS() if value > 0 && value <= 240 && !math.IsInf(value, 0) && !math.IsNaN(value) { @@ -190,7 +193,7 @@ func CleanupRecordingDirectory(configDirectory string, configuration *models.Con return } - if err := os.Remove(recordingsDirectory + "/" + name); err != nil { + if err := utils.RemoveRecording(recordingsDirectory, name); err != nil { log.Info("HandleRecordStream: something went wrong, " + err.Error()) return } @@ -275,7 +278,7 @@ func defaultReserveMB(totalMB int64) int64 { // overall with pending=true, signalling the caller that it is about to drop an // un-uploaded recording as a last resort. func pickRecordingToCleanup(recordingsDirectory, cloudDirectory string) (string, bool, error) { - entries, err := os.ReadDir(recordingsDirectory) + entries, err := utils.ReadRecordingDirectory(recordingsDirectory) if err != nil { return "", false, err } @@ -284,14 +287,7 @@ func pickRecordingToCleanup(recordingsDirectory, cloudDirectory string) (string, var oldestSafeTime, oldestAnyTime time.Time for _, entry := range entries { - if entry.IsDir() { - continue - } - info, err := entry.Info() - if err != nil || !info.Mode().IsRegular() { - continue - } - modTime := info.ModTime() + modTime := entry.ModTime() if oldestAnyName == "" || modTime.Before(oldestAnyTime) { oldestAnyName = entry.Name() @@ -449,39 +445,6 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat // Cleanup muxer start = false - // Update the name of the recording with the duration. - // We will update the name of the recording with the duration in milliseconds. - if mp4Video.VideoTotalDuration > 0 { - duration := mp4Video.VideoTotalDuration - // Update the name with the duration in milliseconds. - startRecordingSeconds := startRecording / 1000 // convert to seconds - startRecordingMilliseconds := startRecording % 1000 // convert to milliseconds - s := strconv.FormatInt(startRecordingSeconds, 10) + "_" + - strconv.Itoa(len(strconv.FormatInt(startRecordingMilliseconds, 10))) + "-" + - strconv.FormatInt(startRecordingMilliseconds, 10) + "_" + - config.Name + "_" + - "0-0-0-0" + "_" + // region coordinates, we - "-1" + "_" + // token - strconv.FormatInt(int64(duration), 10) // + "_" + // duration of recording - //utils.VERSION // version of the agent - - oldName := name - name = s + ".mp4" - fullName = configDirectory + "/data/recordings/" + name - log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name) - - // Rename the file to the new name. - err := os.Rename( - configDirectory+"/data/recordings/"+oldName, - configDirectory+"/data/recordings/"+s+".mp4") - - if err != nil { - log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error()) - } - } else { - log.Info("capture.main.HandleRecordStream(continuous): no video data recorded, not renaming file.") - } - encrypted := false // Check if we need to encrypt the recording. if config.Encryption != nil && config.Encryption.Enabled == "true" && config.Encryption.Recordings == "true" && config.Encryption.SymmetricKey != "" { @@ -506,7 +469,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat } } - queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, startRecording, mp4Video, encrypted)) + queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config, startRecording, mp4Video, encrypted, "0-0-0-0", -1)) recordingStatus = "idle" @@ -531,29 +494,17 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat start = true - // timestamp_microseconds_instanceName_regionCoordinates_numberOfChanges_token - // 1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4 - // - Timestamp - // - Size + - + microseconds - // - device - // - Region - // - Number of changes - // - Token - startRecording = pkt.CurrentTime - startRecordingSeconds := startRecording / 1000 // convert to seconds - startRecordingMilliseconds := startRecording % 1000 // convert to milliseconds - s := strconv.FormatInt(startRecordingSeconds, 10) + "_" + // start timestamp in seconds - strconv.Itoa(len(strconv.FormatInt(startRecordingMilliseconds, 10))) + "-" + // length of milliseconds - strconv.FormatInt(startRecordingMilliseconds, 10) + "_" + // milliseconds - config.Name + "_" + // device name - "0-0-0-0" + "_" + // region coordinates, we will not use this for continuous recording - "0" + "_" + // token - "0" + "_" //+ // duration of recording in milliseconds - //utils.VERSION // version of the agent - - name = s + ".mp4" - fullName = configDirectory + "/data/recordings/" + name + name, err = utils.RecordingFileName(startRecording, config.Key) + if err == nil { + fullName, err = utils.CreateRecordingPath(configDirectory+"/data/recordings", name, time.UnixMilli(startRecording)) + } + if err != nil { + log.WithError(err).Error("capture: could not create continuous recording path") + start = false + pkt = nextPkt + continue + } // Running... log.Info("capture.main.HandleRecordStream(continuous): recording started") @@ -609,39 +560,6 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat // Cleanup muxer start = false - // Update the name of the recording with the duration. - // We will update the name of the recording with the duration in milliseconds. - if mp4Video.VideoTotalDuration > 0 { - duration := mp4Video.VideoTotalDuration - // Update the name with the duration in milliseconds. - startRecordingSeconds := startRecording / 1000 // convert to seconds - startRecordingMilliseconds := startRecording % 1000 // convert to milliseconds - s := strconv.FormatInt(startRecordingSeconds, 10) + "_" + - strconv.Itoa(len(strconv.FormatInt(startRecordingMilliseconds, 10))) + "-" + - strconv.FormatInt(startRecordingMilliseconds, 10) + "_" + - config.Name + "_" + - "0-0-0-0" + "_" + // region coordinates, we - "-1" + "_" + // token - strconv.FormatInt(int64(duration), 10) // + "_" + // duration of recording - //utils.VERSION // version of the agent - - oldName := name - name = s + ".mp4" - fullName = configDirectory + "/data/recordings/" + name - log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name) - - // Rename the file to the new name. - err := os.Rename( - configDirectory+"/data/recordings/"+oldName, - configDirectory+"/data/recordings/"+s+".mp4") - - if err != nil { - log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error()) - } - } else { - log.Info("capture.main.HandleRecordStream(continuous): no video data recorded, not renaming file.") - } - encrypted := false // Check if we need to encrypt the recording. if config.Encryption != nil && config.Encryption.Enabled == "true" && config.Encryption.Recordings == "true" && config.Encryption.SymmetricKey != "" { @@ -666,7 +584,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat } } - queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, startRecording, mp4Video, encrypted)) + queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config, startRecording, mp4Video, encrypted, "0-0-0-0", -1)) recordingStatus = "idle" @@ -721,17 +639,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat continue } - // timestamp_microseconds_instanceName_regionCoordinates_numberOfChanges_token - // 1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4 - // - Timestamp - // - Size + - + microseconds - // - device - // - Region - // - Number of changes - // - Token - - displayTimeSeconds := displayTime / 1000 // convert to seconds - displayTimeMilliseconds := displayTime % 1000 // convert to milliseconds + displayTimeSeconds := displayTime / 1000 motionRectangleString := "0-0-0-0" if motion.Rectangle.X != 0 || motion.Rectangle.Y != 0 || motion.Rectangle.Width != 0 || motion.Rectangle.Height != 0 { @@ -742,17 +650,15 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat // Get the number of changes from the motion detection. numberOfChanges := motion.NumberOfChanges - s := strconv.FormatInt(displayTimeSeconds, 10) + "_" + // start timestamp in seconds - strconv.Itoa(len(strconv.FormatInt(displayTimeMilliseconds, 10))) + "-" + // length of milliseconds - strconv.FormatInt(displayTimeMilliseconds, 10) + "_" + // milliseconds - config.Name + "_" + // device name - motionRectangleString + "_" + // region coordinates, we will not use this for continuous recording - strconv.Itoa(numberOfChanges) + "_" + // number of changes - "0" // + "_" + // duration of recording in milliseconds - //utils.VERSION // version of the agent - - name := s + ".mp4" - fullName := configDirectory + "/data/recordings/" + name + name, err := utils.RecordingFileName(displayTime, config.Key) + var fullName string + if err == nil { + fullName, err = utils.CreateRecordingPath(configDirectory+"/data/recordings", name, time.UnixMilli(displayTime)) + } + if err != nil { + log.WithError(err).Error("capture: could not create motion recording path") + continue + } // Running... log.Info("capture.main.HandleRecordStream(motiondetection): recording started (" + name + ")" + " at " + strconv.FormatInt(displayTimeSeconds, 10) + " unix") @@ -880,38 +786,6 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat communication.TrySendMotion(models.MotionDataPartial{Timestamp: time.Now().Unix(), NumberOfChanges: 100000000}) } - // Update the name of the recording with the duration. - // We will update the name of the recording with the duration in milliseconds. - if mp4Video.VideoTotalDuration > 0 { - duration := mp4Video.VideoTotalDuration - - // Update the name with the duration in milliseconds. - s := strconv.FormatInt(displayTimeSeconds, 10) + "_" + - strconv.Itoa(len(strconv.FormatInt(displayTimeMilliseconds, 10))) + "-" + - strconv.FormatInt(displayTimeMilliseconds, 10) + "_" + - config.Name + "_" + - motionRectangleString + "_" + - strconv.Itoa(numberOfChanges) + "_" + // number of changes - strconv.FormatInt(int64(duration), 10) // + "_" + // duration of recording in milliseconds - //utils.VERSION // version of the agent - - oldName := name - name = s + ".mp4" - fullName = configDirectory + "/data/recordings/" + name - log.Info("capture.main.HandleRecordStream(motiondetection): renamed file from: " + oldName + " to: " + name) - - // Rename the file to the new name. - err := os.Rename( - configDirectory+"/data/recordings/"+oldName, - configDirectory+"/data/recordings/"+s+".mp4") - - if err != nil { - log.Error("capture.main.HandleRecordStream(motiondetection): error renaming file: " + err.Error()) - } - } else { - log.Info("capture.main.HandleRecordStream(motiondetection): no video data recorded, not renaming file.") - } - encrypted := false // Check if we need to encrypt the recording. if config.Encryption != nil && config.Encryption.Enabled == "true" && config.Encryption.Recordings == "true" && config.Encryption.SymmetricKey != "" { @@ -936,7 +810,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat } } - queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, displayTime, mp4Video, encrypted)) + queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config, displayTime, mp4Video, encrypted, motionRectangleString, numberOfChanges)) // Clean up the recording directory if necessary. CleanupRecordingDirectory(configDirectory, configuration) diff --git a/machinery/src/capture/main_test.go b/machinery/src/capture/main_test.go index ea0dd82..d51900c 100644 --- a/machinery/src/capture/main_test.go +++ b/machinery/src/capture/main_test.go @@ -42,10 +42,11 @@ func TestQueueRecordingForUploadStoresFinalizedMetadata(t *testing.T) { } mp4Video := &video.MP4{VideoTotalDuration: 20452, SampleCount: 613} - metadata := recordingUploadMetadata("recording.mp4", "device-key", 1785934709414, mp4Video, true) + config := models.Config{Key: "device-key", Name: "camera-name"} + metadata := recordingUploadMetadata("1785934709414-device-key.mp4", config, 1785934709414, mp4Video, true, "1-2-3-4", 57) queueRecordingForUpload(configDirectory, metadata) - got, err := os.ReadFile(filepath.Join(configDirectory, "data", "cloud", "recording.metadata")) + got, err := os.ReadFile(filepath.Join(configDirectory, "data", "cloud", "1785934709414-device-key.metadata")) if err != nil { t.Fatalf("read upload marker: %v", err) } @@ -54,7 +55,8 @@ func TestQueueRecordingForUploadStoresFinalizedMetadata(t *testing.T) { t.Fatalf("decode upload marker: %v", err) } expectedFPS := mp4Video.AverageFPS() - if stored.FileName != "recording.mp4" || stored.DeviceKey != "device-key" || stored.Timestamp != 1785934709414 || stored.Duration != 20452 || math.Abs(stored.FPS-expectedFPS) > 1e-9 || !stored.Encrypted { + if stored.FileName != "1785934709414-device-key.mp4" || stored.DeviceKey != "device-key" || stored.Timestamp != 1785934709414 || stored.Duration != 20452 || math.Abs(stored.FPS-expectedFPS) > 1e-9 || !stored.Encrypted || + stored.DeviceName != "camera-name" || stored.RegionCoordinates != "1-2-3-4" || stored.NumberOfChanges != "57" { t.Fatalf("upload marker = %+v", stored) } if stored.FPS == math.Floor(stored.FPS) { diff --git a/machinery/src/cloud/cloud.go b/machinery/src/cloud/cloud.go index 6cc51db..4b09cd4 100644 --- a/machinery/src/cloud/cloud.go +++ b/machinery/src/cloud/cloud.go @@ -125,7 +125,7 @@ func HandleUpload(configDirectory string, configuration *models.Configuration, c // Check if we need to remove the original recording // removeAfterUpload is set to false by default if config.RemoveAfterUpload != "false" { - err := os.Remove(configDirectory + "/data/recordings/" + fileName) + err := utils.RemoveRecording(configDirectory+"/data/recordings", fileName) if err != nil { log.Error("HandleUpload: " + err.Error()) } diff --git a/machinery/src/cloud/dropbox.go b/machinery/src/cloud/dropbox.go index 23b161d..9d63ab6 100644 --- a/machinery/src/cloud/dropbox.go +++ b/machinery/src/cloud/dropbox.go @@ -5,13 +5,13 @@ package cloud import ( "bytes" "errors" - "os" "github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox" "github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files" "github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/users" "github.com/gin-gonic/gin" "github.com/kerberos-io/agent/machinery/src/models" + "github.com/kerberos-io/agent/machinery/src/utils" log "github.com/sirupsen/logrus" ) @@ -37,14 +37,12 @@ func UploadDropbox(configuration *models.Configuration, fileName string) (bool, // Upload to Dropbox log.Info("UploadDropbox: Uploading to Dropbox") log.Info("UploadDropbox: Upload started for " + fileName) - fullname := "data/recordings/" + fileName - dConfig := dropbox.Config{ Token: token, LogLevel: dropbox.LogInfo, // if needed, set the desired logging level. Default is off } - file, err := os.OpenFile(fullname, os.O_RDWR, 0755) + file, err := utils.OpenRecording("data/recordings", fileName) if file != nil { defer func() { if cerr := file.Close(); cerr != nil { diff --git a/machinery/src/cloud/kerberos_hub.go b/machinery/src/cloud/kerberos_hub.go index 25f479b..6f7749a 100644 --- a/machinery/src/cloud/kerberos_hub.go +++ b/machinery/src/cloud/kerberos_hub.go @@ -9,6 +9,7 @@ import ( "os" "github.com/kerberos-io/agent/machinery/src/models" + "github.com/kerberos-io/agent/machinery/src/utils" log "github.com/sirupsen/logrus" ) @@ -78,10 +79,8 @@ func UploadKerberosHub(configuration *models.Configuration, fileName string) (bo }).Info("Resumable Hub upload unavailable; using legacy upload") } - fullname := "data/recordings/" + fileName - // Check if we still have the file otherwise we abort the request. - file, err := os.OpenFile(fullname, os.O_RDWR, 0755) + file, err := utils.OpenRecording("data/recordings", fileName) if file != nil { defer file.Close() } diff --git a/machinery/src/cloud/kerberos_vault.go b/machinery/src/cloud/kerberos_vault.go index b995e72..b26672c 100644 --- a/machinery/src/cloud/kerberos_vault.go +++ b/machinery/src/cloud/kerberos_vault.go @@ -10,6 +10,7 @@ import ( "time" "github.com/kerberos-io/agent/machinery/src/models" + "github.com/kerberos-io/agent/machinery/src/utils" log "github.com/sirupsen/logrus" ) @@ -35,10 +36,23 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string) ( // This can happen when the file was already removed (e.g. cleanup, or an // earlier successful upload). Skip it so the watcher drops the marker // instead of retrying indefinitely. - info, err := os.Stat("data/recordings/" + fileName) + filePath, err := utils.ResolveRecordingPath("data/recordings", fileName) if err != nil { - log.Info("UploadKerberosVault: skipping " + fileName + ", file doesn't exist anymore") - return false, false, nil + if os.IsNotExist(err) { + log.Info("UploadKerberosVault: skipping " + fileName + ", file doesn't exist anymore") + return false, false, nil + } + log.WithError(err).Error("UploadKerberosVault: could not resolve recording path") + return false, true, err + } + info, err := os.Stat(filePath) + if err != nil { + if os.IsNotExist(err) { + log.Info("UploadKerberosVault: skipping " + fileName + ", file doesn't exist anymore") + return false, false, nil + } + log.WithError(err).Error("UploadKerberosVault: could not stat recording") + return false, true, err } if info.Size() == 0 { log.Warn("UploadKerberosVault: skipping " + fileName + ", recording is empty") @@ -182,9 +196,7 @@ func uploadVaultLegacy(vault models.KStorage, publicKey, deviceKey, fileName, la if err != nil { return false, false, "", err } - fullname := "data/recordings/" + fileName - - file, err := os.Open(fullname) + file, err := utils.OpenRecording("data/recordings", fileName) if file != nil { defer file.Close() } diff --git a/machinery/src/cloud/recording_metadata.go b/machinery/src/cloud/recording_metadata.go index 3773637..e5e9937 100644 --- a/machinery/src/cloud/recording_metadata.go +++ b/machinery/src/cloud/recording_metadata.go @@ -2,6 +2,7 @@ package cloud import ( "encoding/json" + "fmt" "math" "net/http" "os" @@ -10,12 +11,16 @@ import ( "strings" "github.com/kerberos-io/agent/machinery/src/models" + log "github.com/sirupsen/logrus" ) const recordingFPSHeader = "X-Kerberos-Storage-Fps" const recordingDurationHeader = "X-Kerberos-Storage-Duration" const recordingTimestampHeader = "X-Kerberos-Storage-Timestamp" const recordingEncryptedHeader = "X-Kerberos-Storage-Encrypted" +const recordingDeviceNameHeader = "X-Kerberos-Storage-DeviceName" +const recordingRegionHeader = "X-Kerberos-Storage-RegionCoordinates" +const recordingChangesHeader = "X-Kerberos-Storage-NumberOfChanges" // queuedRecordingFPS reads the FPS snapshot written into the upload marker // when the recording was finalized. Historical empty markers intentionally @@ -61,6 +66,14 @@ func decodeRecordingUploadMetadata(value []byte) (models.RecordingUploadMetadata } func readRecordingUploadMetadata(fileName string) ([]byte, bool) { + value, err := readRecordingUploadMetadataFile(fileName) + if err != nil && !os.IsNotExist(err) { + log.WithError(err).Error("Failed to read recording upload metadata") + } + return value, err == nil +} + +func readRecordingUploadMetadataFile(fileName string) ([]byte, error) { markerNames := []string{ models.RecordingUploadMetadataFileName(fileName), filepath.Base(fileName), @@ -68,10 +81,13 @@ func readRecordingUploadMetadata(fileName string) ([]byte, bool) { for _, markerName := range markerNames { value, err := os.ReadFile(filepath.Join("data", "cloud", markerName)) if err == nil { - return value, true + return value, nil + } + if !os.IsNotExist(err) { + return nil, fmt.Errorf("read recording upload metadata %s: %w", markerName, err) } } - return nil, false + return nil, os.ErrNotExist } func setQueuedRecordingMetadataHeaders(header http.Header, fileName string) { @@ -79,6 +95,18 @@ func setQueuedRecordingMetadataHeaders(header http.Header, fileName string) { header.Set(recordingFPSHeader, fps) } if metadata, ok := queuedRecordingMetadata(fileName); ok { + if metadata.DeviceKey != "" { + header.Set("X-Kerberos-Storage-Device", metadata.DeviceKey) + } + if metadata.DeviceName != "" { + header.Set(recordingDeviceNameHeader, metadata.DeviceName) + } + if metadata.RegionCoordinates != "" { + header.Set(recordingRegionHeader, metadata.RegionCoordinates) + } + if metadata.NumberOfChanges != "" { + header.Set(recordingChangesHeader, metadata.NumberOfChanges) + } if metadata.Duration > 0 { header.Set(recordingDurationHeader, strconv.FormatUint(metadata.Duration, 10)) } diff --git a/machinery/src/cloud/s3.go b/machinery/src/cloud/s3.go index f152154..aa0c508 100644 --- a/machinery/src/cloud/s3.go +++ b/machinery/src/cloud/s3.go @@ -2,15 +2,20 @@ package cloud import ( "crypto/tls" + "encoding/json" "errors" + "fmt" + "math" "net/http" "net/url" "os" + "path/filepath" "strconv" "strings" "time" "github.com/kerberos-io/agent/machinery/src/models" + "github.com/kerberos-io/agent/machinery/src/utils" "github.com/minio/minio-go/v6" log "github.com/sirupsen/logrus" ) @@ -19,15 +24,6 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool, config := configuration.Config - // timestamp_microseconds_instanceName_regionCoordinates_numberOfChanges_token - // 1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4 - // - Timestamp - // - Size + - + microseconds - // - device - // - Region - // - Number of changes - // - Token - if config.S3 == nil { errorMessage := "UploadS3: Uploading Failed, as no settings found" log.Error(errorMessage) @@ -72,34 +68,14 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool, s3Client.SetCustomTransport(transport) } - fileParts := strings.Split(fileName, "_") - if len(fileParts) < 6 { - errorMessage := "UploadS3: " + fileName + " is not a valid name." - log.Error(errorMessage) - return false, true, errors.New(errorMessage) - } - - deviceKey := config.Key - if deviceKey == "" || strings.ContainsAny(deviceKey, "/\\") { - errorMessage := "UploadS3: missing or invalid device key." - log.Error(errorMessage) - return false, true, errors.New(errorMessage) - } - startRecording, err := strconv.ParseInt(fileParts[0], 10, 64) + objectName, userMetadata, err := s3RecordingUploadDetails(config, fileName, aws_access_key_id) if err != nil { - errorMessage := "UploadS3: invalid recording timestamp in " + fileName + ": " + err.Error() - log.Error(errorMessage) - return false, true, errors.New(errorMessage) + log.WithError(err).Error("UploadS3: invalid recording upload metadata") + return false, true, err } - devicename := fileParts[2] - coordinates := fileParts[3] - //numberOfChanges := fileParts[4] - token, _ := strconv.Atoi(fileParts[5]) log.Info("UploadS3: Upload started for " + fileName) - fullname := "data/recordings/" + fileName - - file, err := os.OpenFile(fullname, os.O_RDWR, 0755) + file, err := utils.OpenRecording("data/recordings", fileName) if file != nil { defer file.Close() } @@ -118,23 +94,13 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool, } n, err := s3Client.PutObject(config.S3.Bucket, - s3RecordingObjectName(config.S3.Username, deviceKey, startRecording), + objectName, file, fileInfo.Size(), minio.PutObjectOptions{ ContentType: "video/mp4", StorageClass: "ONEZONE_IA", - UserMetadata: map[string]string{ - "event-timestamp": strconv.FormatInt(startRecording, 10), - "event-microseconds": deviceKey, - "event-instancename": devicename, - "event-regioncoordinates": coordinates, - "event-numberofchanges": deviceKey, - "event-token": strconv.Itoa(token), - "productid": deviceKey, - "publickey": aws_access_key_id, - "uploadtime": "now", - }, + UserMetadata: userMetadata, }) if err != nil { @@ -148,6 +114,92 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool, } func s3RecordingObjectName(username, deviceKey string, startRecording int64) string { - return username + "/" + time.Unix(startRecording, 0).UTC().Format("2006/01/02") + + return username + "/" + time.UnixMilli(startRecording).UTC().Format("2006/01/02") + "/" + strconv.FormatInt(startRecording, 10) + "-" + deviceKey + ".mp4" } + +func s3RecordingUploadDetails(config models.Config, fileName, publicKey string) (string, map[string]string, error) { + value, err := readRecordingUploadMetadataFile(fileName) + if err != nil && !os.IsNotExist(err) { + return "", nil, err + } + + deviceKey := config.Key + var startRecording int64 + userMetadata := map[string]string{ + "publickey": publicKey, + "uploadtime": "now", + } + marker := strings.TrimSpace(string(value)) + _, fpsErr := strconv.ParseFloat(marker, 64) + if err == nil && marker != "" && fpsErr != nil { + var metadata models.RecordingUploadMetadata + if err := json.Unmarshal(value, &metadata); err != nil { + return "", nil, fmt.Errorf("decode recording upload metadata for %s: %w", fileName, err) + } + if metadata.FileName != "" && metadata.FileName != filepath.Base(fileName) { + return "", nil, fmt.Errorf("recording upload metadata filename does not match %s", fileName) + } + if metadata.Timestamp <= 0 { + return "", nil, fmt.Errorf("invalid recording timestamp in upload metadata for %s", fileName) + } + deviceKey = metadata.DeviceKey + startRecording = metadata.Timestamp + userMetadata["event-instancename"] = config.Name + if metadata.DeviceName != "" { + userMetadata["event-instancename"] = metadata.DeviceName + } + if metadata.RegionCoordinates != "" { + userMetadata["event-regioncoordinates"] = metadata.RegionCoordinates + } + if metadata.NumberOfChanges != "" { + userMetadata["event-numberofchanges"] = metadata.NumberOfChanges + } + userMetadata["event-microseconds"] = strconv.FormatInt(metadata.Timestamp%1000*1000, 10) + userMetadata["duration"] = strconv.FormatUint(metadata.Duration, 10) + userMetadata["encrypted"] = strconv.FormatBool(metadata.Encrypted) + if metadata.FPS > 0 && metadata.FPS <= 240 { + userMetadata["fps"] = strconv.FormatFloat(metadata.FPS, 'f', -1, 64) + } + } else { + // Empty and FPS-only markers predate structured recording metadata. + fileParts := strings.Split(fileName, "_") + if len(fileParts) < 6 { + return "", nil, fmt.Errorf("%s is not a valid legacy recording name", fileName) + } + startRecording, err = strconv.ParseInt(fileParts[0], 10, 64) + if err != nil { + return "", nil, fmt.Errorf("invalid recording timestamp in %s: %w", fileName, err) + } + if startRecording <= 0 || startRecording > math.MaxInt64/1000 { + return "", nil, fmt.Errorf("invalid recording timestamp in %s", fileName) + } + startRecording *= 1000 + _, precision, found := strings.Cut(fileParts[1], "-") + if found { + fraction, err := strconv.ParseInt(precision, 10, 64) + if err != nil || fraction < 0 || fraction >= 1000000 { + return "", nil, fmt.Errorf("invalid recording precision in %s", fileName) + } + if len(precision) > 3 { + fraction /= 1000 + } + startRecording += fraction + } + token, _ := strconv.Atoi(fileParts[5]) + userMetadata["event-microseconds"] = deviceKey + userMetadata["event-instancename"] = fileParts[2] + userMetadata["event-regioncoordinates"] = fileParts[3] + userMetadata["event-numberofchanges"] = deviceKey + userMetadata["event-token"] = strconv.Itoa(token) + } + if deviceKey == "" || strings.ContainsAny(deviceKey, "/\\") { + return "", nil, errors.New("missing or invalid recording device key") + } + if startRecording <= 0 { + return "", nil, fmt.Errorf("invalid recording timestamp for %s", fileName) + } + userMetadata["event-timestamp"] = strconv.FormatInt(startRecording/1000, 10) + userMetadata["productid"] = deviceKey + return s3RecordingObjectName(config.S3.Username, deviceKey, startRecording), userMetadata, nil +} diff --git a/machinery/src/cloud/s3_test.go b/machinery/src/cloud/s3_test.go index bf2cbb5..954f846 100644 --- a/machinery/src/cloud/s3_test.go +++ b/machinery/src/cloud/s3_test.go @@ -1,6 +1,10 @@ package cloud import ( + "encoding/json" + "os" + "path/filepath" + "reflect" "strconv" "strings" "testing" @@ -9,6 +13,136 @@ import ( "github.com/kerberos-io/agent/machinery/src/models" ) +func TestS3RecordingUploadDetailsUsesCheckpoint(t *testing.T) { + for _, fileName := range []string{ + "1767355200123-checkpoint-key.mp4", + "1564859471_6-474162_old-camera_0-0-0-0_1153_27.mp4", + } { + t.Run(fileName, func(t *testing.T) { + withRecording(t, fileName, []byte("recording")) + recorded := time.Date(2026, time.January, 2, 12, 0, 0, 123000000, time.UTC) + metadata := models.RecordingUploadMetadata{ + FileName: fileName, + DeviceKey: "checkpoint-key", + Timestamp: recorded.UnixMilli(), + Duration: 20452, + FPS: 29.97, + Encrypted: true, + DeviceName: "checkpoint-camera", + RegionCoordinates: "1-2-3-4", + NumberOfChanges: "57", + } + payload, err := json.Marshal(metadata) + if err != nil { + t.Fatal(err) + } + withQueuedRecordingFPS(t, fileName, string(payload)) + config := models.Config{ + Key: "changed-config-key", + Name: "camera-name", + S3: &models.S3{Username: "user"}, + } + objectName, userMetadata, err := s3RecordingUploadDetails(config, fileName, "public-key") + if err != nil { + t.Fatal(err) + } + wantObjectName := "user/2026/01/02/" + strconv.FormatInt(recorded.UnixMilli(), 10) + "-checkpoint-key.mp4" + if objectName != wantObjectName { + t.Fatalf("object name = %q, want %q", objectName, wantObjectName) + } + wantMetadata := map[string]string{ + "publickey": "public-key", + "uploadtime": "now", + "event-instancename": "checkpoint-camera", + "event-regioncoordinates": "1-2-3-4", + "event-numberofchanges": "57", + "event-microseconds": "123000", + "event-timestamp": strconv.FormatInt(recorded.Unix(), 10), + "productid": "checkpoint-key", + "duration": "20452", + "fps": "29.97", + "encrypted": "true", + } + if !reflect.DeepEqual(userMetadata, wantMetadata) { + t.Fatalf("metadata = %#v, want %#v", userMetadata, wantMetadata) + } + }) + } +} + +func TestS3RecordingUploadDetailsLegacyMarkers(t *testing.T) { + for _, marker := range []string{"missing", "empty", "fps-only", "old-marker-name"} { + t.Run(marker, func(t *testing.T) { + const fileName = "1564859471_6-474162_camera_0-0-0-0_1153_27.mp4" + withRecording(t, fileName, []byte("recording")) + switch marker { + case "empty": + withQueuedRecordingFPS(t, fileName, "") + case "fps-only", "old-marker-name": + withQueuedRecordingFPS(t, fileName, "29.97") + if marker == "old-marker-name" { + err := os.Rename(filepath.Join("data/cloud", models.RecordingUploadMetadataFileName(fileName)), + filepath.Join("data/cloud", fileName)) + if err != nil { + t.Fatal(err) + } + } + } + config := models.Config{Key: "legacy-key", S3: &models.S3{Username: "user"}} + objectName, metadata, err := s3RecordingUploadDetails(config, fileName, "public-key") + if err != nil { + t.Fatal(err) + } + if want := "user/2019/08/03/1564859471474-legacy-key.mp4"; objectName != want { + t.Fatalf("object name = %q, want %q", objectName, want) + } + if metadata["productid"] != "legacy-key" || metadata["event-instancename"] != "camera" || + metadata["event-regioncoordinates"] != "0-0-0-0" || metadata["event-timestamp"] != "1564859471" { + t.Fatalf("unexpected legacy metadata: %#v", metadata) + } + }) + } +} + +func TestS3RecordingUploadDetailsRejectsInvalidCheckpoint(t *testing.T) { + tests := []struct { + name string + marker string + want string + }{ + {"malformed JSON", `{"timestamp":`, "decode"}, + {"missing timestamp", `{"device_key":"camera"}`, "timestamp"}, + {"missing device key", `{"timestamp":1787015373000}`, "device key"}, + {"invalid device key", `{"timestamp":1787015373000,"device_key":"camera/key"}`, "device key"}, + {"mismatched filename", `{"timestamp":1787015373000,"device_key":"camera","filename":"other.mp4"}`, "filename"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + const fileName = "1564859471_6-474162_camera_0-0-0-0_1153_27.mp4" + withRecording(t, fileName, []byte("recording")) + withQueuedRecordingFPS(t, fileName, tt.marker) + config := models.Config{Key: "legacy-key", S3: &models.S3{Username: "user"}} + _, _, err := s3RecordingUploadDetails(config, fileName, "public-key") + if err == nil || !strings.Contains(err.Error(), tt.want) { + t.Fatalf("error = %v, want error containing %q instead of legacy fallback", err, tt.want) + } + }) + } +} + +func TestS3RecordingUploadDetailsReportsCheckpointReadError(t *testing.T) { + const fileName = "1564859471_6-474162_camera_0-0-0-0_1153_27.mp4" + withRecording(t, fileName, []byte("recording")) + if err := os.MkdirAll(filepath.Join("data/cloud", models.RecordingUploadMetadataFileName(fileName)), 0o755); err != nil { + t.Fatal(err) + } + config := models.Config{Key: "legacy-key", S3: &models.S3{Username: "user"}} + _, _, err := s3RecordingUploadDetails(config, fileName, "public-key") + if err == nil || !strings.Contains(err.Error(), "read recording upload metadata") { + t.Fatalf("error = %v, want checkpoint read error instead of legacy fallback", err) + } +} + func TestS3RecordingObjectName(t *testing.T) { tests := []struct { name string @@ -34,9 +168,9 @@ func TestS3RecordingObjectName(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { const deviceKey = "camera-key-123" - want := "user/" + tt.directory + "/" + strconv.FormatInt(tt.recorded.Unix(), 10) + "-" + deviceKey + ".mp4" + want := "user/" + tt.directory + "/" + strconv.FormatInt(tt.recorded.UnixMilli(), 10) + "-" + deviceKey + ".mp4" for attempt := 0; attempt < 2; attempt++ { - if got := s3RecordingObjectName("user", deviceKey, tt.recorded.Unix()); got != want { + if got := s3RecordingObjectName("user", deviceKey, tt.recorded.UnixMilli()); got != want { t.Fatalf("s3RecordingObjectName() = %q, want %q", got, want) } } diff --git a/machinery/src/cloud/tus_client.go b/machinery/src/cloud/tus_client.go index c0dd6c9..5297b75 100644 --- a/machinery/src/cloud/tus_client.go +++ b/machinery/src/cloud/tus_client.go @@ -16,6 +16,7 @@ import ( "time" "github.com/kerberos-io/agent/machinery/src/models" + "github.com/kerberos-io/agent/machinery/src/utils" log "github.com/sirupsen/logrus" ) @@ -106,7 +107,7 @@ func logTusUploadProgress(label string, offset, size int64, loggedBucket *int64) // creation request; it is empty on HEAD/PATCH/DELETE. type tusHeaderFunc func(h http.Header, fileName string) -// runTusUpload performs a resumable (tus) upload of data/recordings/ +// runTusUpload performs a resumable (tus) upload of a flat or dated recording. // to baseURL, sending target-specific authentication/routing headers via // setHeaders on every request. It encapsulates the create/resume/chunk/finalize // state machine shared by the Kerberos Vault (direct) and Kerberos Hub (proxied) @@ -120,9 +121,7 @@ type tusHeaderFunc func(h http.Header, fileName string) // should fall back to the legacy single-POST upload (older deployments). // - body: a short message for logging. func runTusUpload(baseURL, metadata, fileName, label, slot string, setHeaders tusHeaderFunc) (uploaded bool, responded bool, supported bool, body string, err error) { - fullname := "data/recordings/" + fileName - - file, ferr := os.Open(fullname) + file, ferr := utils.OpenRecording("data/recordings", fileName) if file != nil { defer file.Close() } @@ -358,6 +357,12 @@ func addRecordingTusMetadata(values map[string]string, fileName string) { if !ok { return } + if metadata.DeviceKey != "" { + values["device"] = metadata.DeviceKey + } + values["device_name"] = metadata.DeviceName + values["region_coordinates"] = metadata.RegionCoordinates + values["number_of_changes"] = metadata.NumberOfChanges if metadata.Duration > 0 { values["duration"] = strconv.FormatUint(metadata.Duration, 10) } @@ -500,6 +505,7 @@ func setVaultTusHeaders(h http.Header, vault models.KStorage, publicKey, deviceK h.Set("X-Kerberos-Storage-Capture", "IPCamera") if fileName != "" { h.Set("X-Kerberos-Storage-FileName", fileName) + setQueuedRecordingMetadataHeaders(h, fileName) } } @@ -516,6 +522,7 @@ func setHubTusHeaders(h http.Header, config *models.Config, fileName string) { h.Set("X-Kerberos-Storage-Capture", "IPCamera") if fileName != "" { h.Set("X-Kerberos-Storage-FileName", fileName) + setQueuedRecordingMetadataHeaders(h, fileName) } } diff --git a/machinery/src/cloud/tus_client_test.go b/machinery/src/cloud/tus_client_test.go index aec2500..08389d2 100644 --- a/machinery/src/cloud/tus_client_test.go +++ b/machinery/src/cloud/tus_client_test.go @@ -18,8 +18,86 @@ import ( "time" "github.com/kerberos-io/agent/machinery/src/models" + "github.com/kerberos-io/agent/machinery/src/utils" ) +func TestUploadKerberosVaultReadsDatedLocalRecordings(t *testing.T) { + previousCount, previousTimeout := kstorageRetryCount, kstorageRetryTimeout + t.Cleanup(func() { kstorageRetryCount, kstorageRetryTimeout = previousCount, previousTimeout }) + for _, transport := range []string{"resumable", "legacy"} { + t.Run(transport, func(t *testing.T) { + const fileName = "1564859471123-device-key.mp4" + payload := bytes.Repeat([]byte("video"), 1024) + withRecording(t, fileName, payload) + datedPath, err := utils.CreateRecordingPath("data/recordings", fileName, time.Unix(1564859471, 0)) + if err != nil { + t.Fatal(err) + } + if err := os.Rename(filepath.Join("data/recordings", fileName), datedPath); err != nil { + t.Fatal(err) + } + withQueuedRecordingFPS(t, fileName, `{"filename":"1564859471123-device-key.mp4","device_key":"device-key","timestamp":1564859471123,"duration":27,"fps":25,"device_name":"camera-name","region_coordinates":"1-2-3-4","number_of_changes":"57"}`) + fake := newFakeTus() + var legacyBody []byte + var legacyHeaders http.Header + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if transport == "resumable" { + fake.ServeHTTP(w, r) + return + } + body, readErr := io.ReadAll(r.Body) + if readErr != nil { + t.Error(readErr) + } + legacyBody = body + legacyHeaders = r.Header.Clone() + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + if transport == "legacy" { + t.Setenv("AGENT_DISABLE_RESUMABLE_UPLOAD", "true") + } else { + t.Setenv("AGENT_DISABLE_RESUMABLE_UPLOAD", "false") + } + kstorageRetryCount, kstorageRetryTimeout = 0, 0 + vault := testVault(server.URL) + config := &models.Configuration{Config: models.Config{ + Key: "changed-config-key", + KStorage: &vault, + KStorageSecondary: &models.KStorage{}, + }} + uploaded, configured, err := UploadKerberosVault(config, fileName) + if err != nil || !uploaded || !configured { + t.Fatalf("UploadKerberosVault() = (%v, %v, %v), want success", uploaded, configured, err) + } + if transport == "resumable" { + if got := fake.totalBytes(); got != int64(len(payload)) { + t.Fatalf("uploaded %d bytes, want %d", got, len(payload)) + } + posts := fake.requestsForMethod(http.MethodPost) + if len(posts) != 1 { + t.Fatalf("got %d creation requests, want 1", len(posts)) + } + metadata := decodeTusMetadata(posts[0].header.Get("Upload-Metadata")) + if metadata["filename"] != fileName || metadata["directory"] != "dir" || metadata["timestamp"] != "1564859471123" || + metadata["device"] != "device-key" || metadata["device_name"] != "camera-name" || + metadata["region_coordinates"] != "1-2-3-4" || metadata["number_of_changes"] != "57" { + t.Fatalf("unexpected upload metadata: %#v", metadata) + } + } else if !bytes.Equal(legacyBody, payload) || + legacyHeaders.Get("X-Kerberos-Storage-FileName") != fileName || + legacyHeaders.Get("X-Kerberos-Storage-Directory") != "dir" || + legacyHeaders.Get("X-Kerberos-Storage-Device") != "device-key" || + legacyHeaders.Get(recordingDeviceNameHeader) != "camera-name" || + legacyHeaders.Get(recordingRegionHeader) != "1-2-3-4" || + legacyHeaders.Get(recordingChangesHeader) != "57" || + legacyHeaders.Get(recordingTimestampHeader) != "1564859471123" { + t.Fatalf("unexpected legacy upload: %d bytes, headers %#v", len(legacyBody), legacyHeaders) + } + }) + } +} + // fakeUpload tracks the state of a single resumable upload on the fake server. type fakeUpload struct { size int64 diff --git a/machinery/src/components/kerberos.go b/machinery/src/components/kerberos.go index c17c117..7a26e41 100644 --- a/machinery/src/components/kerberos.go +++ b/machinery/src/components/kerberos.go @@ -964,7 +964,7 @@ func GetDashboard(c *gin.Context, configDirectory string, configuration *models. // All days stored in this agent. days := []string{} latestEvents := []models.Media{} - files, err := utils.ReadDirectory(recordingDirectory) + files, err := utils.ReadRecordingDirectory(recordingDirectory) if err == nil { events := utils.GetSortedDirectory(files) @@ -975,6 +975,8 @@ func GetDashboard(c *gin.Context, configDirectory string, configuration *models. var eventFilter models.EventFilter eventFilter.NumberOfElements = 5 latestEvents = utils.GetMediaFormatted(events, recordingDirectory, configuration, eventFilter) // will get 5 latest recordings. + } else { + log.WithError(err).Error("Could not list recordings for Agent status") } c.JSON(200, gin.H{ @@ -1007,7 +1009,7 @@ func GetLatestEvents(c *gin.Context, configDirectory string, configuration *mode eventFilter.NumberOfElements = 10 } recordingDirectory := configDirectory + "/data/recordings" - files, err := utils.ReadDirectory(recordingDirectory) + files, err := utils.ReadRecordingDirectory(recordingDirectory) if err == nil { events := utils.GetSortedDirectory(files) // We will get all recordings from the directory (as defined by the filter). @@ -1036,7 +1038,7 @@ func GetLatestEvents(c *gin.Context, configDirectory string, configuration *mode // @Success 200 func GetDays(c *gin.Context, configDirectory string, configuration *models.Configuration, communication *models.Communication) { recordingDirectory := configDirectory + "/data/recordings" - files, err := utils.ReadDirectory(recordingDirectory) + files, err := utils.ReadRecordingDirectory(recordingDirectory) if err == nil { events := utils.GetSortedDirectory(files) days := utils.GetDays(events, recordingDirectory, configuration) diff --git a/machinery/src/models/recording_upload_metadata.go b/machinery/src/models/recording_upload_metadata.go index ac253d4..61bbae5 100644 --- a/machinery/src/models/recording_upload_metadata.go +++ b/machinery/src/models/recording_upload_metadata.go @@ -11,12 +11,15 @@ const RecordingUploadMetadataExtension = ".metadata" // with a recording. New optional fields can be added without changing the queue // mechanism or breaking older agents. type RecordingUploadMetadata struct { - FileName string `json:"filename"` - DeviceKey string `json:"device_key"` - Timestamp int64 `json:"timestamp"` // Unix milliseconds. - Duration uint64 `json:"duration"` // Milliseconds. - FPS float64 `json:"fps,omitempty"` - Encrypted bool `json:"encrypted,omitempty"` + FileName string `json:"filename"` + DeviceKey string `json:"device_key"` + Timestamp int64 `json:"timestamp"` // Unix milliseconds. + Duration uint64 `json:"duration"` // Milliseconds. + FPS float64 `json:"fps,omitempty"` + Encrypted bool `json:"encrypted,omitempty"` + DeviceName string `json:"device_name,omitempty"` + RegionCoordinates string `json:"region_coordinates,omitempty"` + NumberOfChanges string `json:"number_of_changes,omitempty"` } // RecordingUploadMetadataFileName returns the queue marker name associated diff --git a/machinery/src/routers/http/recordings_test.go b/machinery/src/routers/http/recordings_test.go new file mode 100644 index 0000000..5b9e277 --- /dev/null +++ b/machinery/src/routers/http/recordings_test.go @@ -0,0 +1,81 @@ +package http + +import ( + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/gin-gonic/gin" + "github.com/kerberos-io/agent/machinery/src/encryption" + "github.com/kerberos-io/agent/machinery/src/models" +) + +func TestFilesServesFlatAndDatedRecordings(t *testing.T) { + gin.SetMode(gin.TestMode) + for _, layout := range []string{"flat", "dated"} { + for _, encrypted := range []bool{false, true} { + name := layout + "-plain" + if encrypted { + name = layout + "-encrypted" + } + t.Run(name, func(t *testing.T) { + const fileName = "1767355200123-device-key.mp4" + directory := t.TempDir() + relativePath := fileName + if layout == "dated" { + relativePath = "2026/01/02/" + fileName + } + filePath := filepath.Join(directory, "data/recordings", relativePath) + if err := os.MkdirAll(filepath.Dir(filePath), 0o755); err != nil { + t.Fatal(err) + } + config := &models.Configuration{Config: models.Config{Encryption: &models.Encryption{}}} + content := []byte("recording-payload") + if encrypted { + config.Config.Encryption.Recordings = "true" + config.Config.Encryption.SymmetricKey = "0123456789abcdef0123456789abcdef" + ciphertext, err := encryption.AesEncrypt(content, config.Config.Encryption.SymmetricKey) + if err != nil { + t.Fatal(err) + } + content = []byte(ciphertext) + } + if err := os.WriteFile(filePath, content, 0o644); err != nil { + t.Fatal(err) + } + router := gin.New() + router.GET("/file/*filepath", func(c *gin.Context) { Files(c, directory, config) }) + for _, reference := range []string{relativePath, fileName} { + response := httptest.NewRecorder() + router.ServeHTTP(response, httptest.NewRequest("GET", "/file/"+reference, nil)) + if response.Code != 200 || response.Body.String() != "recording-payload" { + t.Fatalf("GET %s = %d %q", reference, response.Code, response.Body.String()) + } + if got := response.Header().Get("Content-Type"); got != "video/mp4" { + t.Fatalf("Content-Type = %q, want video/mp4", got) + } + } + }) + } + } +} + +func TestFilesRejectsMissingAndOutsideRecordings(t *testing.T) { + directory := t.TempDir() + if err := os.MkdirAll(filepath.Join(directory, "data/recordings"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(directory, "data/outside.mp4"), []byte("not a recording"), 0o644); err != nil { + t.Fatal(err) + } + router := gin.New() + router.GET("/file/*filepath", func(c *gin.Context) { Files(c, directory, &models.Configuration{}) }) + for _, reference := range []string{"missing.mp4", "../outside.mp4", "2026/01/02/missing.mp4"} { + response := httptest.NewRecorder() + router.ServeHTTP(response, httptest.NewRequest("GET", "/file/"+reference, nil)) + if response.Code != 404 { + t.Fatalf("GET %s = %d, want 404", reference, response.Code) + } + } +} diff --git a/machinery/src/routers/http/server.go b/machinery/src/routers/http/server.go index 54839db..83917d2 100644 --- a/machinery/src/routers/http/server.go +++ b/machinery/src/routers/http/server.go @@ -5,6 +5,7 @@ import ( "io" "os" "strconv" + "strings" "time" jwt "github.com/appleboy/gin-jwt/v2" @@ -19,6 +20,7 @@ import ( "github.com/kerberos-io/agent/machinery/src/capture" "github.com/kerberos-io/agent/machinery/src/encryption" "github.com/kerberos-io/agent/machinery/src/models" + "github.com/kerberos-io/agent/machinery/src/utils" swaggerFiles "github.com/swaggo/files" ginSwagger "github.com/swaggo/gin-swagger" ) @@ -160,9 +162,9 @@ func requestLogger() gin.HandlerFunc { func Files(c *gin.Context, configDirectory string, configuration *models.Configuration) { // Get File - filePath := configDirectory + "/data/recordings" + c.Param("filepath") - _, err := os.Open(filePath) + filePath, err := utils.ResolveRecordingPath(configDirectory+"/data/recordings", strings.TrimPrefix(c.Param("filepath"), "/")) if err != nil { + log.WithError(err).Warn("Could not resolve recording for playback") c.JSON(404, gin.H{"error": "File not found"}) return } diff --git a/machinery/src/utils/main.go b/machinery/src/utils/main.go index b4bf52e..3e764c9 100644 --- a/machinery/src/utils/main.go +++ b/machinery/src/utils/main.go @@ -191,51 +191,32 @@ func GetMediaFormatted(files []os.FileInfo, recordingDirectory string, configura count := 0 for _, file := range files { fileName := file.Name() - fileParts := strings.Split(fileName, "_") - if len(fileParts) == 6 { - timestamp := fileParts[0] - timestampInt, err := strconv.ParseInt(timestamp, 10, 64) - if err == nil { - - if eventFilter.TimestampOffsetStart > 0 { - // TimestampOffsetStart represents the newest lower bound to include. - if timestampInt < eventFilter.TimestampOffsetStart { - continue - } - } - - // If we have an offset we will check if we should skip or not - if eventFilter.TimestampOffsetEnd > 0 { - // Medias are sorted from new to older. TimestampOffsetEnd holds the oldest - // timestamp of the previous batch of events. By doing this check, we make sure - // to skip the previous batch. - if timestampInt >= eventFilter.TimestampOffsetEnd { - continue - } - } - - loc, _ := time.LoadLocation(configuration.Config.Timezone) - time := time.Unix(timestampInt, 0).In(loc) - day := time.Format("02-01-2006") - timeString := time.Format("15:04:05") - shortDay := time.Format("Jan _2") - - media := models.Media{ - Key: fileName, - Path: recordingDirectory + "/" + fileName, - CameraName: configuration.Config.Name, - CameraKey: configuration.Config.Key, - Day: day, - ShortDay: shortDay, - Time: timeString, - Timestamp: timestamp, - } - filePaths = append(filePaths, media) - count = count + 1 - if eventFilter.NumberOfElements > 0 && count >= eventFilter.NumberOfElements { - break - } - } + recordedAt, err := RecordingStartTime(fileName) + if err != nil { + continue + } + timestamp := recordedAt.Unix() + if eventFilter.TimestampOffsetStart > 0 && timestamp < eventFilter.TimestampOffsetStart { + continue + } + if eventFilter.TimestampOffsetEnd > 0 && timestamp >= eventFilter.TimestampOffsetEnd { + continue + } + loc, _ := time.LoadLocation(configuration.Config.Timezone) + localTime := recordedAt.In(loc) + filePaths = append(filePaths, models.Media{ + Key: fileName, + Path: recordingDirectory + "/" + fileName, + CameraName: configuration.Config.Name, + CameraKey: configuration.Config.Key, + Day: localTime.Format("02-01-2006"), + ShortDay: localTime.Format("Jan _2"), + Time: localTime.Format("15:04:05"), + Timestamp: strconv.FormatInt(timestamp, 10), + }) + count++ + if eventFilter.NumberOfElements > 0 && count >= eventFilter.NumberOfElements { + break } } return filePaths @@ -244,18 +225,12 @@ func GetMediaFormatted(files []os.FileInfo, recordingDirectory string, configura func GetDays(files []os.FileInfo, recordingDirectory string, configuration *models.Configuration) []string { days := []string{} for _, file := range files { - fileName := file.Name() - fileParts := strings.Split(fileName, "_") - if len(fileParts) == 6 { - timestamp := fileParts[0] - timestampInt, err := strconv.ParseInt(timestamp, 10, 64) - if err == nil { - loc, _ := time.LoadLocation(configuration.Config.Timezone) - time := time.Unix(timestampInt, 0).In(loc) - day := time.Format("02-01-2006") - days = append(days, day) - } + recordedAt, err := RecordingStartTime(file.Name()) + if err != nil { + continue } + loc, _ := time.LoadLocation(configuration.Config.Timezone) + days = append(days, recordedAt.In(loc).Format("02-01-2006")) } uniqueDays := Unique(days) return uniqueDays @@ -278,10 +253,13 @@ func NumberOfFilesInDirectory(path string) int { return len(files) } -// NumberOfMP4sInDirectory returns the count of all files with mp4 extension in current directory +// NumberOfMP4sInDirectory counts MP4 files, including dated subdirectories. func NumberOfMP4sInDirectory(path string) int { - pattern := filepath.Join(path, "*.mp4") - files, _ := filepath.Glob(pattern) + files, err := ReadRecordingDirectory(path) + if err != nil { + log.WithError(err).Error("Could not count recordings") + return 0 + } return len(files) } @@ -409,25 +387,13 @@ func Decrypt(directoryOrFile string, symmetricKey []byte) { var files []string if fileInfo.IsDir() { - // Create decrypted directory - err = os.MkdirAll(directoryOrFile+"/decrypted", 0755) - if err != nil { - log.Fatal(err.Error()) - return - } - dir, err := os.ReadDir(directoryOrFile) + dir, err := ReadRecordingDirectory(directoryOrFile) if err != nil { log.Fatal(err.Error()) return } for _, file := range dir { - // Check if file is not a directory - if !file.IsDir() { - // Check if an mp4 file - if strings.HasSuffix(file.Name(), ".mp4") { - files = append(files, directoryOrFile+"/"+file.Name()) - } - } + files = append(files, filepath.Join(directoryOrFile, file.Name())) } } else { files = append(files, directoryOrFile) @@ -449,13 +415,12 @@ func Decrypt(directoryOrFile string, symmetricKey []byte) { return } - // Write decrypted content to file with appended .decrypted - // Get filename split by / and get last element. - fileParts := strings.Split(file, "/") - fileName := fileParts[len(fileParts)-1] - pathToFile := strings.Join(fileParts[:len(fileParts)-1], "/") - - err = os.WriteFile(pathToFile+"/decrypted/"+fileName, []byte(decrypted), 0644) + outputDirectory := filepath.Join(filepath.Dir(file), "decrypted") + if err := os.MkdirAll(outputDirectory, 0o755); err != nil { + log.Fatal(err.Error()) + return + } + err = os.WriteFile(filepath.Join(outputDirectory, filepath.Base(file)), []byte(decrypted), 0644) if err != nil { log.Fatal(err.Error()) return diff --git a/machinery/src/utils/recordings.go b/machinery/src/utils/recordings.go new file mode 100644 index 0000000..e7b15fa --- /dev/null +++ b/machinery/src/utils/recordings.go @@ -0,0 +1,136 @@ +package utils + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + "strconv" + "strings" + "time" +) + +func RecordingFileName(timestamp int64, deviceKey string) (string, error) { + if timestamp <= 0 { + return "", fmt.Errorf("invalid recording timestamp %d", timestamp) + } + if deviceKey == "" || strings.ContainsAny(deviceKey, "/\\") { + return "", fmt.Errorf("missing or invalid recording device key") + } + return strconv.FormatInt(timestamp, 10) + "-" + deviceKey + ".mp4", nil +} + +func RecordingStartTime(name string) (time.Time, error) { + base := filepath.Base(name) + if !strings.HasSuffix(base, ".mp4") { + return time.Time{}, fmt.Errorf("invalid recording filename %q", name) + } + timestamp, deviceKey, short := strings.Cut(strings.TrimSuffix(base, ".mp4"), "-") + if short && deviceKey != "" { + if millis, err := strconv.ParseInt(timestamp, 10, 64); err == nil && millis > 0 { + return time.UnixMilli(millis), nil + } + } + parts := strings.Split(base, "_") + if len(parts) == 6 { + seconds, err := strconv.ParseInt(parts[0], 10, 64) + if err == nil && seconds > 0 { + return time.Unix(seconds, 0), nil + } + } + return time.Time{}, fmt.Errorf("invalid recording filename %q", name) +} + +func CreateRecordingPath(directory, fileName string, recordedAt time.Time) (string, error) { + if !filepath.IsLocal(fileName) || fileName == "." || filepath.Base(fileName) != fileName || strings.ContainsAny(fileName, "/\\") { + return "", fmt.Errorf("invalid recording filename %q", fileName) + } + if recordedAt.UnixMilli() <= 0 { + return "", fmt.Errorf("invalid recording start time %s", recordedAt) + } + directory = filepath.Join(directory, recordedAt.UTC().Format("2006/01/02")) + if err := os.MkdirAll(directory, 0o755); err != nil { + return "", fmt.Errorf("create recording directory: %w", err) + } + return filepath.Join(directory, fileName), nil +} + +// ResolveRecordingPath accepts relative paths from listings and historical +// basename-only references from upload markers and playback URLs. +func ResolveRecordingPath(directory, name string) (string, error) { + if !filepath.IsLocal(name) { + return "", fmt.Errorf("invalid recording path %q", name) + } + filePath := filepath.Join(directory, name) + info, err := os.Stat(filePath) + if os.IsNotExist(err) && filepath.Base(name) == name { + recordedAt, parseErr := RecordingStartTime(name) + if parseErr == nil { + filePath = filepath.Join(directory, recordedAt.UTC().Format("2006/01/02"), name) + info, err = os.Stat(filePath) + } + } + if err != nil { + return "", err + } + if !info.Mode().IsRegular() { + return "", fmt.Errorf("recording %q is not a regular file", name) + } + return filePath, nil +} + +func OpenRecording(directory, name string) (*os.File, error) { + filePath, err := ResolveRecordingPath(directory, name) + if err != nil { + return nil, err + } + return os.Open(filePath) +} + +func RemoveRecording(directory, name string) error { + filePath, err := ResolveRecordingPath(directory, name) + if err != nil { + return err + } + return os.Remove(filePath) +} + +type recordingFileInfo struct { + os.FileInfo + relativePath string +} + +func (file recordingFileInfo) Name() string { + return file.relativePath +} + +// ReadRecordingDirectory returns MP4s with root-relative names, preserving the +// file information used by the existing sorting and media formatting helpers. +func ReadRecordingDirectory(directory string) ([]os.FileInfo, error) { + files := []os.FileInfo{} + err := filepath.WalkDir(directory, func(filePath string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() && entry.Name() == "decrypted" && filePath != directory { + return filepath.SkipDir + } + if !entry.Type().IsRegular() || filepath.Ext(entry.Name()) != ".mp4" { + return nil + } + info, err := entry.Info() + if err != nil { + return err + } + name, err := filepath.Rel(directory, filePath) + if err != nil { + return err + } + files = append(files, recordingFileInfo{FileInfo: info, relativePath: filepath.ToSlash(name)}) + return nil + }) + if err != nil { + return nil, err + } + return files, nil +} diff --git a/machinery/src/utils/recordings_test.go b/machinery/src/utils/recordings_test.go new file mode 100644 index 0000000..a72e632 --- /dev/null +++ b/machinery/src/utils/recordings_test.go @@ -0,0 +1,235 @@ +package utils + +import ( + "io" + "os" + "path/filepath" + "reflect" + "strconv" + "testing" + "time" + + "github.com/kerberos-io/agent/machinery/src/encryption" + "github.com/kerberos-io/agent/machinery/src/models" +) + +func TestShortRecordingNamePreservesMillisecondsAndDeviceKey(t *testing.T) { + const timestamp = int64(1767355200123) + const deviceKey = "device-key_with-hyphens" + name, err := RecordingFileName(timestamp, deviceKey) + if err != nil || name != "1767355200123-device-key_with-hyphens.mp4" { + t.Fatalf("RecordingFileName() = %q, %v", name, err) + } + other, err := RecordingFileName(timestamp+1, deviceKey) + if err != nil || other == name { + t.Fatalf("recordings within the same second must have distinct names: %q, %v", other, err) + } + directory := t.TempDir() + filePath, err := CreateRecordingPath(directory, name, time.UnixMilli(timestamp)) + if err != nil { + t.Fatal(err) + } + if want := filepath.Join(directory, "2026/01/02", name); filePath != want { + t.Fatalf("path = %q, want %q", filePath, want) + } + if err := os.WriteFile(filePath, []byte("video"), 0o644); err != nil { + t.Fatal(err) + } + resolved, err := ResolveRecordingPath(directory, name) + if err != nil || resolved != filePath { + t.Fatalf("ResolveRecordingPath() = %q, %v", resolved, err) + } + files, err := ReadRecordingDirectory(directory) + if err != nil { + t.Fatal(err) + } + config := &models.Configuration{Config: models.Config{Timezone: "UTC"}} + media := GetMediaFormatted(files, directory, config, models.EventFilter{TimestampOffsetStart: timestamp / 1000}) + if len(media) != 1 || media[0].Key != "2026/01/02/"+name || media[0].Timestamp != "1767355200" { + t.Fatalf("unexpected short-name media: %+v", media) + } + if days := GetDays(files, directory, config); !reflect.DeepEqual(days, []string{"02-01-2026"}) { + t.Fatalf("unexpected short-name days: %v", days) + } + if err := RemoveRecording(directory, name); err != nil { + t.Fatal(err) + } + for _, invalidKey := range []string{"", "camera/key", `camera\key`} { + if _, err := RecordingFileName(timestamp, invalidKey); err == nil { + t.Errorf("accepted invalid device key %q", invalidKey) + } + } + if _, err := RecordingFileName(0, deviceKey); err == nil { + t.Fatal("accepted invalid recording timestamp") + } +} + +func TestCreateRecordingPathUsesRecordingUTCDate(t *testing.T) { + directory := t.TempDir() + recordedAt := time.Date(2026, time.January, 1, 1, 0, 0, 0, time.FixedZone("UTC+2", 2*60*60)) + name := strconv.FormatInt(recordedAt.Unix(), 10) + "_3-123_camera_0-0-0-0_1_1000.mp4" + filePath, err := CreateRecordingPath(directory, name, recordedAt) + if err != nil { + t.Fatal(err) + } + if want := filepath.Join(directory, "2025/12/31", name); filePath != want { + t.Fatalf("path = %q, want %q", filePath, want) + } + if err := os.WriteFile(filePath, []byte("video"), 0o644); err != nil { + t.Fatal(err) + } + for _, reference := range []string{name, "2025/12/31/" + name} { + got, err := ResolveRecordingPath(directory, reference) + if err != nil || got != filePath { + t.Fatalf("ResolveRecordingPath(%q) = %q, %v; want %q", reference, got, err, filePath) + } + file, err := OpenRecording(directory, reference) + if err != nil { + t.Fatal(err) + } + content, err := io.ReadAll(file) + file.Close() + if err != nil || string(content) != "video" { + t.Fatalf("recording content = %q, %v", content, err) + } + } + if err := RemoveRecording(directory, name); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filePath); !os.IsNotExist(err) { + t.Fatalf("recording still exists after removal: %v", err) + } +} + +func TestRecordingPathsKeepFlatFilesAccessible(t *testing.T) { + directory := t.TempDir() + name := "legacy-recording.mp4" + filePath := filepath.Join(directory, name) + if err := os.WriteFile(filePath, []byte("legacy"), 0o644); err != nil { + t.Fatal(err) + } + got, err := ResolveRecordingPath(directory, name) + if err != nil || got != filePath { + t.Fatalf("ResolveRecordingPath() = %q, %v; want %q", got, err, filePath) + } + if err := RemoveRecording(directory, name); err != nil { + t.Fatal(err) + } +} + +func TestRecordingPathsReportInvalidAndMissingPaths(t *testing.T) { + directory := t.TempDir() + for _, name := range []string{"", ".", "..", "../outside.mp4", "/outside.mp4", "nested/recording.mp4", `nested\recording.mp4`} { + if _, err := CreateRecordingPath(directory, name, time.Now()); err == nil { + t.Errorf("CreateRecordingPath(%q) should reject invalid filename", name) + } + } + for _, name := range []string{"", ".", "../outside.mp4", "/outside.mp4"} { + if _, err := ResolveRecordingPath(directory, name); err == nil { + t.Errorf("ResolveRecordingPath(%q) should reject invalid path", name) + } + } + for _, name := range []string{"missing.mp4", "1700000000_missing.mp4", "2026/01/01/missing.mp4"} { + if _, err := ResolveRecordingPath(directory, name); !os.IsNotExist(err) { + t.Errorf("ResolveRecordingPath(%q) error = %v, want not-exist", name, err) + } + } + blocked := filepath.Join(directory, "not-a-directory") + if err := os.WriteFile(blocked, nil, 0o644); err != nil { + t.Fatal(err) + } + if _, err := CreateRecordingPath(blocked, "recording.mp4", time.Now()); err == nil { + t.Fatal("expected directory creation error") + } + if _, err := CreateRecordingPath(directory, "recording.mp4", time.Time{}); err == nil { + t.Fatal("expected invalid recording start time error") + } + if _, err := ReadRecordingDirectory(filepath.Join(directory, "missing")); !os.IsNotExist(err) { + t.Fatalf("ReadRecordingDirectory() error = %v, want not-exist", err) + } +} + +func TestReadRecordingDirectoryIncludesFlatAndDatedMedia(t *testing.T) { + directory := t.TempDir() + names := []string{ + "1700000000_6_7_8_9_10.mp4", + "2026/01/02/1767355200_6_7_8_9_10.mp4", + } + for index, name := range names { + filePath := filepath.Join(directory, name) + if err := os.MkdirAll(filepath.Dir(filePath), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filePath, []byte("video"), 0o644); err != nil { + t.Fatal(err) + } + modified := time.Unix(int64(index+1), 0) + if err := os.Chtimes(filePath, modified, modified); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(directory, "2026/01/02/ignore.txt"), nil, 0o644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(directory, "2026/01/02/decrypted"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(directory, "2026/01/02/decrypted/output.mp4"), nil, 0o644); err != nil { + t.Fatal(err) + } + files, err := ReadRecordingDirectory(directory) + if err != nil { + t.Fatal(err) + } + if len(files) != 2 || NumberOfMP4sInDirectory(directory) != 2 { + t.Fatalf("expected 2 MP4 recordings, got %d", len(files)) + } + config := &models.Configuration{Config: models.Config{Timezone: "UTC", Name: "camera", Key: "device"}} + files = GetSortedDirectory(files) + media := GetMediaFormatted(files, directory, config, models.EventFilter{}) + if len(media) != 2 || media[0].Key != names[1] || media[1].Key != names[0] { + t.Fatalf("unexpected media ordering or relative keys: %+v", media) + } + if media[0].Path != filepath.Join(directory, names[1]) || media[0].Timestamp != "1767355200" { + t.Fatalf("unexpected nested media path or timestamp: %+v", media[0]) + } + if got, want := GetDays(files, directory, config), []string{"02-01-2026", "14-11-2023"}; !reflect.DeepEqual(got, want) { + t.Fatalf("days = %v, want %v", got, want) + } + filtered := GetMediaFormatted(files, directory, config, models.EventFilter{TimestampOffsetStart: 1767355200}) + if len(filtered) != 1 || filtered[0].Key != names[1] { + t.Fatalf("unexpected filtered nested media: %+v", filtered) + } +} + +func TestDecryptDirectoryIncludesDatedRecordingsAndSkipsOutputs(t *testing.T) { + directory := t.TempDir() + key := []byte("0123456789abcdef0123456789abcdef") + payload := []byte("recording-payload") + ciphertext, err := encryption.AesEncrypt(payload, string(key)) + if err != nil { + t.Fatal(err) + } + names := []string{"legacy.mp4", "2026/01/02/recording.mp4"} + for _, name := range names { + filePath := filepath.Join(directory, name) + if err := os.MkdirAll(filepath.Dir(filePath), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filePath, []byte(ciphertext), 0o644); err != nil { + t.Fatal(err) + } + } + Decrypt(directory, key) + Decrypt(directory, key) + for _, name := range names { + output := filepath.Join(directory, filepath.Dir(name), "decrypted", filepath.Base(name)) + content, err := os.ReadFile(output) + if err != nil || string(content) != string(payload) { + t.Fatalf("decrypted %q = %q, %v", name, content, err) + } + } + if got := NumberOfMP4sInDirectory(directory); got != len(names) { + t.Fatalf("recording count = %d, want %d (excluding decrypted outputs)", got, len(names)) + } +}