Compare commits

..

1 Commits

Author SHA1 Message Date
cedricve
0e6d7e3fb1 feat: enhance recording metadata structure and handling for uploads 2026-09-11 05:43:07 +00:00
10 changed files with 245 additions and 88 deletions

View File

@@ -53,12 +53,15 @@ func publishRecordingState(mqttClient mqtt.Client, hubKey string, configuration
}
}
func recordingUploadMetadata(name, deviceKey string, timestamp int64, mp4Video *video.MP4) models.RecordingUploadMetadata {
func recordingUploadMetadata(name, deviceKey, deviceName, regionCoordinates, numberOfChanges string, timestamp int64, mp4Video *video.MP4) models.RecordingUploadMetadata {
metadata := models.RecordingUploadMetadata{
FileName: filepath.Base(name),
DeviceKey: deviceKey,
Timestamp: timestamp,
Duration: mp4Video.VideoTotalDuration,
FileName: filepath.Base(name),
DeviceKey: deviceKey,
DeviceName: deviceName,
Timestamp: timestamp,
Duration: mp4Video.VideoTotalDuration,
RegionCoordinates: regionCoordinates,
NumberOfChanges: numberOfChanges,
}
value := mp4Video.AverageFPS()
if value > 0 && value <= 240 && !math.IsInf(value, 0) && !math.IsNaN(value) {
@@ -502,7 +505,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
}
}
queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, startRecording, mp4Video))
queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, config.Name, "0-0-0-0", "-1", startRecording, mp4Video))
recordingStatus = "idle"
@@ -659,7 +662,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
}
}
queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, startRecording, mp4Video))
queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, config.Name, "0-0-0-0", "-1", startRecording, mp4Video))
recordingStatus = "idle"
@@ -926,7 +929,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat
}
}
queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, displayTime, mp4Video))
queueRecordingForUpload(configDirectory, recordingUploadMetadata(name, config.Key, config.Name, motionRectangleString, strconv.Itoa(numberOfChanges), displayTime, mp4Video))
// Clean up the recording directory if necessary.
CleanupRecordingDirectory(configDirectory, configuration)

View File

@@ -42,7 +42,7 @@ func TestQueueRecordingForUploadStoresFinalizedMetadata(t *testing.T) {
}
mp4Video := &video.MP4{VideoTotalDuration: 20452, SampleCount: 613}
metadata := recordingUploadMetadata("recording.mp4", "device-key", 1785934709414, mp4Video)
metadata := recordingUploadMetadata("recording.mp4", "device-key", "camera-name", "1-2-3-4", "57", 1785934709414, mp4Video)
queueRecordingForUpload(configDirectory, metadata)
got, err := os.ReadFile(filepath.Join(configDirectory, "data", "cloud", "recording.metadata"))
@@ -54,7 +54,7 @@ func TestQueueRecordingForUploadStoresFinalizedMetadata(t *testing.T) {
t.Fatalf("decode upload marker: %v", err)
}
expectedFPS := mp4Video.AverageFPS()
if stored.FileName != "recording.mp4" || stored.DeviceKey != "device-key" || stored.Timestamp != 1785934709414 || stored.Duration != 20452 || math.Abs(stored.FPS-expectedFPS) > 1e-9 {
if stored.FileName != "recording.mp4" || stored.DeviceKey != "device-key" || stored.DeviceName != "camera-name" || stored.Timestamp != 1785934709414 || stored.Duration != 20452 || stored.RegionCoordinates != "1-2-3-4" || stored.NumberOfChanges != "57" || math.Abs(stored.FPS-expectedFPS) > 1e-9 {
t.Fatalf("upload marker = %+v", stored)
}
if stored.FPS == math.Floor(stored.FPS) {

View File

@@ -15,6 +15,9 @@ import (
const recordingFPSHeader = "X-Kerberos-Storage-Fps"
const recordingDurationHeader = "X-Kerberos-Storage-Duration"
const recordingTimestampHeader = "X-Kerberos-Storage-Timestamp"
const recordingDeviceNameHeader = "X-Kerberos-Storage-DeviceName"
const recordingRegionCoordinatesHeader = "X-Kerberos-Storage-RegionCoordinates"
const recordingNumberOfChangesHeader = "X-Kerberos-Storage-NumberOfChanges"
// queuedRecordingFPS reads the FPS snapshot written into the upload marker
// when the recording was finalized. Historical empty markers intentionally
@@ -84,5 +87,14 @@ func setQueuedRecordingMetadataHeaders(header http.Header, fileName string) {
if metadata.Timestamp > 0 {
header.Set(recordingTimestampHeader, strconv.FormatInt(metadata.Timestamp, 10))
}
if metadata.DeviceName != "" {
header.Set(recordingDeviceNameHeader, metadata.DeviceName)
}
if metadata.RegionCoordinates != "" {
header.Set(recordingRegionCoordinatesHeader, metadata.RegionCoordinates)
}
if metadata.NumberOfChanges != "" {
header.Set(recordingNumberOfChangesHeader, metadata.NumberOfChanges)
}
}
}

View File

@@ -6,6 +6,7 @@ import (
"net/http"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
@@ -71,20 +72,16 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
s3Client.SetCustomTransport(transport)
}
fileParts := strings.Split(fileName, "_")
if len(fileParts) == 1 {
recordingMetadata, ok := queuedRecordingMetadata(fileName)
if !ok {
recordingMetadata, ok = legacyS3RecordingMetadata(fileName, config.Key)
}
if !ok {
errorMessage := "UploadS3: " + fileName + " is not a valid name."
log.Error(errorMessage)
return false, true, errors.New(errorMessage)
}
deviceKey := config.Key
startRecording, _ := strconv.ParseInt(fileParts[0], 10, 64)
devicename := fileParts[2]
coordinates := fileParts[3]
//numberOfChanges := fileParts[4]
token, _ := strconv.Atoi(fileParts[5])
log.Info("UploadS3: Upload started for " + fileName)
fullname := "data/recordings/" + fileName
@@ -113,17 +110,7 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
minio.PutObjectOptions{
ContentType: "video/mp4",
StorageClass: "ONEZONE_IA",
UserMetadata: map[string]string{
"event-timestamp": strconv.FormatInt(startRecording, 10),
"event-microseconds": deviceKey,
"event-instancename": devicename,
"event-regioncoordinates": coordinates,
"event-numberofchanges": deviceKey,
"event-token": strconv.Itoa(token),
"productid": deviceKey,
"publickey": aws_access_key_id,
"uploadtime": "now",
},
UserMetadata: s3ObjectMetadata(recordingMetadata, config.Key, aws_access_key_id),
})
if err != nil {
@@ -135,3 +122,44 @@ func UploadS3(configuration *models.Configuration, fileName string) (bool, bool,
return true, true, nil
}
}
func s3ObjectMetadata(metadata models.RecordingUploadMetadata, deviceKey string, publicKey string) map[string]string {
return map[string]string{
"event-timestamp": strconv.FormatInt(metadata.Timestamp/1000, 10),
"event-microseconds": strconv.FormatInt(metadata.Timestamp%1000, 10),
"event-instancename": metadata.DeviceName,
"event-regioncoordinates": metadata.RegionCoordinates,
"event-numberofchanges": metadata.NumberOfChanges,
"event-duration": strconv.FormatUint(metadata.Duration, 10),
"event-token": strconv.FormatUint(metadata.Duration, 10),
"productid": deviceKey,
"publickey": publicKey,
"uploadtime": "now",
}
}
func legacyS3RecordingMetadata(fileName string, deviceKey string) (models.RecordingUploadMetadata, bool) {
fileParts := strings.Split(fileName, "_")
if len(fileParts) < 6 {
return models.RecordingUploadMetadata{}, false
}
seconds, secondsErr := strconv.ParseInt(fileParts[0], 10, 64)
milliseconds := int64(0)
precisionParts := strings.SplitN(fileParts[1], "-", 2)
if len(precisionParts) == 2 {
milliseconds, _ = strconv.ParseInt(precisionParts[1], 10, 64)
}
duration, durationErr := strconv.ParseUint(strings.TrimSuffix(fileParts[5], filepath.Ext(fileParts[5])), 10, 64)
if secondsErr != nil || durationErr != nil || milliseconds < 0 || milliseconds >= 1000 {
return models.RecordingUploadMetadata{}, false
}
return models.RecordingUploadMetadata{
FileName: fileName,
DeviceKey: deviceKey,
DeviceName: fileParts[2],
Timestamp: seconds*1000 + milliseconds,
Duration: duration,
RegionCoordinates: fileParts[3],
NumberOfChanges: fileParts[4],
}, true
}

View File

@@ -0,0 +1,40 @@
package cloud
import (
"testing"
"github.com/kerberos-io/agent/machinery/src/models"
)
func TestS3ObjectMetadataUsesStructuredRecordingMetadata(t *testing.T) {
metadata := s3ObjectMetadata(models.RecordingUploadMetadata{
Timestamp: 1785934709414,
Duration: 20452,
DeviceName: "camera-name",
RegionCoordinates: "1-2-3-4",
NumberOfChanges: "57",
}, "device-key", "public-key")
for key, want := range map[string]string{
"event-timestamp": "1785934709",
"event-microseconds": "414",
"event-instancename": "camera-name",
"event-regioncoordinates": "1-2-3-4",
"event-numberofchanges": "57",
"event-duration": "20452",
"event-token": "20452",
"productid": "device-key",
"publickey": "public-key",
} {
if metadata[key] != want {
t.Errorf("metadata[%q] = %q, want %q", key, metadata[key], want)
}
}
}
func TestLegacyS3RecordingMetadata(t *testing.T) {
metadata, ok := legacyS3RecordingMetadata("1785934709_3-414_camera_1-2-3-4_57_20452.mp4", "device-key")
if !ok || metadata.Timestamp != 1785934709414 || metadata.Duration != 20452 || metadata.DeviceName != "camera" || metadata.RegionCoordinates != "1-2-3-4" || metadata.NumberOfChanges != "57" {
t.Fatalf("legacyS3RecordingMetadata() = %#v/%v", metadata, ok)
}
}

View File

@@ -361,6 +361,9 @@ func addRecordingTusMetadata(values map[string]string, fileName string) {
if metadata.Timestamp > 0 {
values["timestamp"] = strconv.FormatInt(metadata.Timestamp, 10)
}
values["device_name"] = metadata.DeviceName
values["region_coordinates"] = metadata.RegionCoordinates
values["number_of_changes"] = metadata.NumberOfChanges
}
// tusCreate performs the tus "creation" request (POST). On success it returns

View File

@@ -302,7 +302,7 @@ func TestUploadVaultResumable_HappyPath(t *testing.T) {
fileName := "1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4"
payload := bytes.Repeat([]byte("x"), 4096)
withRecording(t, fileName, payload)
withQueuedRecordingFPS(t, fileName, `{"filename":"recording.mp4","device_key":"device-key","timestamp":1785934709414,"duration":20452,"fps":29.97}`)
withQueuedRecordingFPS(t, fileName, `{"filename":"recording.mp4","device_key":"device-key","device_name":"camera-name","timestamp":1785934709414,"duration":20452,"fps":29.97,"region_coordinates":"1-2-3-4","number_of_changes":"57"}`)
uploaded, responded, supported, _, err := uploadVaultResumable(testVault(ts.URL), "pk", "dev", fileName, "test", "primary")
if err != nil {
@@ -328,6 +328,15 @@ func TestUploadVaultResumable_HappyPath(t *testing.T) {
if got := metadata["timestamp"]; got != "1785934709414" {
t.Fatalf("POST metadata timestamp = %q, want %q", got, "1785934709414")
}
if got := metadata["device_name"]; got != "camera-name" {
t.Fatalf("POST metadata device_name = %q, want camera-name", got)
}
if got := metadata["region_coordinates"]; got != "1-2-3-4" {
t.Fatalf("POST metadata region_coordinates = %q, want 1-2-3-4", got)
}
if got := metadata["number_of_changes"]; got != "57" {
t.Fatalf("POST metadata number_of_changes = %q, want 57", got)
}
}
func TestQueuedRecordingFPSValidation(t *testing.T) {
@@ -386,7 +395,7 @@ func TestQueuedRecordingFPSAllowsMissingHistoricalMarker(t *testing.T) {
func TestQueuedRecordingMetadataHeaders(t *testing.T) {
fileName := "recording.mp4"
withRecording(t, fileName, []byte("recording"))
withQueuedRecordingFPS(t, fileName, `{"filename":"recording.mp4","device_key":"device-key","timestamp":1785934709414,"duration":20452,"fps":25}`)
withQueuedRecordingFPS(t, fileName, `{"filename":"recording.mp4","device_key":"device-key","device_name":"camera-name","timestamp":1785934709414,"duration":20452,"fps":25,"region_coordinates":"1-2-3-4","number_of_changes":"57"}`)
header := make(http.Header)
setQueuedRecordingMetadataHeaders(header, fileName)
@@ -399,6 +408,15 @@ func TestQueuedRecordingMetadataHeaders(t *testing.T) {
if got := header.Get(recordingTimestampHeader); got != "1785934709414" {
t.Fatalf("timestamp header = %q", got)
}
if got := header.Get(recordingDeviceNameHeader); got != "camera-name" {
t.Fatalf("device name header = %q", got)
}
if got := header.Get(recordingRegionCoordinatesHeader); got != "1-2-3-4" {
t.Fatalf("region coordinates header = %q", got)
}
if got := header.Get(recordingNumberOfChangesHeader); got != "57" {
t.Fatalf("number of changes header = %q", got)
}
}
func TestQueuedRecordingFPSAllowsLegacyMarkerFileName(t *testing.T) {

View File

@@ -11,11 +11,14 @@ const RecordingUploadMetadataExtension = ".metadata"
// with a recording. New optional fields can be added without changing the queue
// mechanism or breaking older agents.
type RecordingUploadMetadata struct {
FileName string `json:"filename"`
DeviceKey string `json:"device_key"`
Timestamp int64 `json:"timestamp"` // Unix milliseconds.
Duration uint64 `json:"duration"` // Milliseconds.
FPS float64 `json:"fps,omitempty"`
FileName string `json:"filename"`
DeviceKey string `json:"device_key"`
DeviceName string `json:"device_name,omitempty"`
Timestamp int64 `json:"timestamp"` // Unix milliseconds.
Duration uint64 `json:"duration"` // Milliseconds.
FPS float64 `json:"fps,omitempty"`
RegionCoordinates string `json:"region_coordinates,omitempty"`
NumberOfChanges string `json:"number_of_changes,omitempty"`
}
// RecordingUploadMetadataFileName returns the queue marker name associated

View File

@@ -2,6 +2,7 @@ package utils
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"image"
@@ -191,50 +192,47 @@ func GetMediaFormatted(files []os.FileInfo, recordingDirectory string, configura
count := 0
for _, file := range files {
fileName := file.Name()
fileParts := strings.Split(fileName, "_")
if len(fileParts) == 6 {
timestamp := fileParts[0]
timestampInt, err := strconv.ParseInt(timestamp, 10, 64)
if err == nil {
timestampInt, ok := recordingTimestamp(file, recordingDirectory)
if ok {
timestamp := strconv.FormatInt(timestampInt, 10)
if eventFilter.TimestampOffsetStart > 0 {
// TimestampOffsetStart represents the newest lower bound to include.
if timestampInt < eventFilter.TimestampOffsetStart {
continue
}
if eventFilter.TimestampOffsetStart > 0 {
// TimestampOffsetStart represents the newest lower bound to include.
if timestampInt < eventFilter.TimestampOffsetStart {
continue
}
}
// If we have an offset we will check if we should skip or not
if eventFilter.TimestampOffsetEnd > 0 {
// Medias are sorted from new to older. TimestampOffsetEnd holds the oldest
// timestamp of the previous batch of events. By doing this check, we make sure
// to skip the previous batch.
if timestampInt >= eventFilter.TimestampOffsetEnd {
continue
}
// If we have an offset we will check if we should skip or not
if eventFilter.TimestampOffsetEnd > 0 {
// Medias are sorted from new to older. TimestampOffsetEnd holds the oldest
// timestamp of the previous batch of events. By doing this check, we make sure
// to skip the previous batch.
if timestampInt >= eventFilter.TimestampOffsetEnd {
continue
}
}
loc, _ := time.LoadLocation(configuration.Config.Timezone)
time := time.Unix(timestampInt, 0).In(loc)
day := time.Format("02-01-2006")
timeString := time.Format("15:04:05")
shortDay := time.Format("Jan _2")
loc, _ := time.LoadLocation(configuration.Config.Timezone)
mediaTime := time.Unix(timestampInt, 0).In(loc)
day := mediaTime.Format("02-01-2006")
timeString := mediaTime.Format("15:04:05")
shortDay := mediaTime.Format("Jan _2")
media := models.Media{
Key: fileName,
Path: recordingDirectory + "/" + fileName,
CameraName: configuration.Config.Name,
CameraKey: configuration.Config.Key,
Day: day,
ShortDay: shortDay,
Time: timeString,
Timestamp: timestamp,
}
filePaths = append(filePaths, media)
count = count + 1
if eventFilter.NumberOfElements > 0 && count >= eventFilter.NumberOfElements {
break
}
media := models.Media{
Key: fileName,
Path: recordingDirectory + "/" + fileName,
CameraName: configuration.Config.Name,
CameraKey: configuration.Config.Key,
Day: day,
ShortDay: shortDay,
Time: timeString,
Timestamp: timestamp,
}
filePaths = append(filePaths, media)
count = count + 1
if eventFilter.NumberOfElements > 0 && count >= eventFilter.NumberOfElements {
break
}
}
}
@@ -244,23 +242,44 @@ func GetMediaFormatted(files []os.FileInfo, recordingDirectory string, configura
func GetDays(files []os.FileInfo, recordingDirectory string, configuration *models.Configuration) []string {
days := []string{}
for _, file := range files {
fileName := file.Name()
fileParts := strings.Split(fileName, "_")
if len(fileParts) == 6 {
timestamp := fileParts[0]
timestampInt, err := strconv.ParseInt(timestamp, 10, 64)
if err == nil {
loc, _ := time.LoadLocation(configuration.Config.Timezone)
time := time.Unix(timestampInt, 0).In(loc)
day := time.Format("02-01-2006")
days = append(days, day)
}
if timestamp, ok := recordingTimestamp(file, recordingDirectory); ok {
loc, _ := time.LoadLocation(configuration.Config.Timezone)
mediaTime := time.Unix(timestamp, 0).In(loc)
days = append(days, mediaTime.Format("02-01-2006"))
}
}
uniqueDays := Unique(days)
return uniqueDays
}
func recordingTimestamp(file os.FileInfo, recordingDirectory string) (int64, bool) {
markerPath := filepath.Join(filepath.Dir(recordingDirectory), "cloud", models.RecordingUploadMetadataFileName(file.Name()))
if value, err := os.ReadFile(markerPath); err == nil {
var metadata models.RecordingUploadMetadata
if json.Unmarshal(value, &metadata) == nil && metadata.Timestamp > 0 {
return metadata.Timestamp / 1000, true
}
}
if timestamp, ok := legacyRecordingTimestampFromFilename(file.Name()); ok {
return timestamp, true
}
if timestamp := file.ModTime().Unix(); timestamp > 0 {
return timestamp, true
}
return 0, false
}
func legacyRecordingTimestampFromFilename(fileName string) (int64, bool) {
fileParts := strings.Split(fileName, "_")
if len(fileParts) != 6 {
return 0, false
}
timestamp, err := strconv.ParseInt(fileParts[0], 10, 64)
return timestamp, err == nil
}
func Unique(intSlice []string) []string {
keys := make(map[string]bool)
list := []string{}

View File

@@ -7,6 +7,7 @@ import (
"image/color"
"image/jpeg"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
@@ -108,13 +109,14 @@ func TestConfigurationLogFieldsOmitCredentialsAndEndpoints(t *testing.T) {
}
type stubFileInfo struct {
name string
name string
modTime time.Time
}
func (s stubFileInfo) Name() string { return s.name }
func (s stubFileInfo) Size() int64 { return 0 }
func (s stubFileInfo) Mode() os.FileMode { return 0 }
func (s stubFileInfo) ModTime() time.Time { return time.Unix(0, 0) }
func (s stubFileInfo) ModTime() time.Time { return s.modTime }
func (s stubFileInfo) IsDir() bool { return false }
func (s stubFileInfo) Sys() interface{} { return nil }
@@ -151,3 +153,32 @@ func TestGetMediaFormattedHonorsTimestampRange(t *testing.T) {
t.Fatalf("expected camera key to be preserved, got %s", media[0].CameraKey)
}
}
func TestGetMediaFormattedSupportsOpaqueFileNames(t *testing.T) {
root := t.TempDir()
recordingDirectory := filepath.Join(root, "recordings")
cloudDirectory := filepath.Join(root, "cloud")
if err := os.MkdirAll(cloudDirectory, 0755); err != nil {
t.Fatal(err)
}
fileName := "opaque recording.mp4"
marker := []byte(`{"filename":"opaque recording.mp4","timestamp":1785934709414}`)
if err := os.WriteFile(filepath.Join(cloudDirectory, models.RecordingUploadMetadataFileName(fileName)), marker, 0644); err != nil {
t.Fatal(err)
}
configuration := &models.Configuration{}
configuration.Config.Timezone = "UTC"
media := GetMediaFormatted([]os.FileInfo{stubFileInfo{name: fileName}}, recordingDirectory, configuration, models.EventFilter{})
if len(media) != 1 || media[0].Timestamp != "1785934709" {
t.Fatalf("GetMediaFormatted() = %#v", media)
}
}
func TestRecordingTimestampFallsBackToModificationTime(t *testing.T) {
want := time.Unix(1785934709, 0)
timestamp, ok := recordingTimestamp(stubFileInfo{name: "opaque.mp4", modTime: want}, "/tmp/recordings")
if !ok || timestamp != want.Unix() {
t.Fatalf("recordingTimestamp() = %d/%v, want %d/true", timestamp, ok, want.Unix())
}
}