diff --git a/README-FILE-STRUCTURE.md b/README-FILE-STRUCTURE.md index ee95d85..86ccb46 100644 --- a/README-FILE-STRUCTURE.md +++ b/README-FILE-STRUCTURE.md @@ -25,6 +25,27 @@ Directory decryption also processes dated subdirectories and writes each output to a sibling `decrypted/` directory; those outputs are excluded from recording lists and automatic cleanup. +## Recording encryption format + +When recording encryption is enabled, recordings are written in a versioned +format: + +| Bytes | Field | +|---|---| +| 8 | Magic `KRBREC1\0` | +| 4 | PBKDF2 iteration count (big-endian) | +| 16 | Salt | +| 12 | Nonce | +| rest | AES-256-GCM ciphertext and 16-byte tag | + +- **Key derivation:** the key is derived from the configured symmetric key with + PBKDF2-HMAC-SHA256, currently 600,000 iterations. +- **Authentication:** the header is authenticated together with the payload, so + any change to a recording is detected. +- **Older recordings:** recordings from releases before 3.14.5 use the + OpenSSL/CryptoJS `Salted__` format. Playback and `-action decrypt` still read + them, so no migration is needed. + ## Direct S3 recording layout Direct S3 uploads store recordings as diff --git a/machinery/src/capture/main.go b/machinery/src/capture/main.go index df0aaf9..7d3c553 100644 --- a/machinery/src/capture/main.go +++ b/machinery/src/capture/main.go @@ -452,7 +452,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat contents, err := os.ReadFile(fullName) if err == nil { // encrypt - encryptedContents, err := encryption.AesEncrypt(contents, config.Encryption.SymmetricKey) + encryptedContents, err := encryption.EncryptRecording(contents, config.Encryption.SymmetricKey) if err == nil { // write back to file err := os.WriteFile(fullName, []byte(encryptedContents), 0644) @@ -567,7 +567,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat contents, err := os.ReadFile(fullName) if err == nil { // encrypt - encryptedContents, err := encryption.AesEncrypt(contents, config.Encryption.SymmetricKey) + encryptedContents, err := encryption.EncryptRecording(contents, config.Encryption.SymmetricKey) if err == nil { // write back to file err := os.WriteFile(fullName, []byte(encryptedContents), 0644) @@ -793,7 +793,7 @@ func HandleRecordStream(queue *packets.Queue, configDirectory string, configurat contents, err := os.ReadFile(fullName) if err == nil { // encrypt - encryptedContents, err := encryption.AesEncrypt(contents, config.Encryption.SymmetricKey) + encryptedContents, err := encryption.EncryptRecording(contents, config.Encryption.SymmetricKey) if err == nil { // write back to file err := os.WriteFile(fullName, []byte(encryptedContents), 0644) diff --git a/machinery/src/encryption/main.go b/machinery/src/encryption/main.go index 9c1f995..7c2db6a 100644 --- a/machinery/src/encryption/main.go +++ b/machinery/src/encryption/main.go @@ -26,6 +26,9 @@ func SignWithPrivateKey(data []byte, privateKey *rsa.PrivateKey) ([]byte, error) return signature, err } +// AesEncrypt writes the OpenSSL/CryptoJS "Salted__" format (MD5-based key +// derivation). It is kept for MQTT messages with Kerberos Hub, whose keys are +// random. Recordings use EncryptRecording instead. func AesEncrypt(content []byte, password string) ([]byte, error) { salt := make([]byte, 8) _, err := rand.Read(salt) @@ -33,6 +36,9 @@ func AesEncrypt(content []byte, password string) ([]byte, error) { return nil, err } key, iv, err := DefaultEvpKDF([]byte(password), salt) + if err != nil { + return nil, err + } block, err := aes.NewCipher(key) if err != nil { @@ -50,13 +56,18 @@ func AesEncrypt(content []byte, password string) ([]byte, error) { return cipherText, nil } +// AesDecrypt reads the OpenSSL/CryptoJS "Salted__" format used for MQTT +// messages with Kerberos Hub and by recordings from older releases. func AesDecrypt(cipherText []byte, password string) ([]byte, error) { - if string(cipherText[:8]) != "Salted__" { + if len(cipherText) < 16+aes.BlockSize || string(cipherText[:8]) != "Salted__" { return nil, errors.New("invalid crypto js aes encryption") } + if (len(cipherText)-16)%aes.BlockSize != 0 { + return nil, errLegacyCiphertext + } salt := cipherText[8:16] - cipherBytes := cipherText[16:] + cipherBytes := append([]byte(nil), cipherText[16:]...) key, iv, err := DefaultEvpKDF([]byte(password), salt) if err != nil { return nil, err @@ -70,8 +81,7 @@ func AesDecrypt(cipherText []byte, password string) ([]byte, error) { mode := cipher.NewCBCDecrypter(block, iv) mode.CryptBlocks(cipherBytes, cipherBytes) - result := PKCS5UnPadding(cipherBytes) - return result, nil + return pkcs7Unpad(cipherBytes, aes.BlockSize) } func EvpKDF(password []byte, salt []byte, keySize int, iterations int, hashAlgorithm string) ([]byte, error) { @@ -113,6 +123,25 @@ func DefaultEvpKDF(password []byte, salt []byte) (key []byte, iv []byte, err err return derivedKeyBytes[:keySize*4], derivedKeyBytes[keySize*4:], nil } +// pkcs7Unpad validates the padding instead of trusting the last byte, so a +// wrong key or a truncated file returns an error rather than panicking. +func pkcs7Unpad(src []byte, blockSize int) ([]byte, error) { + length := len(src) + if length == 0 || length%blockSize != 0 { + return nil, errLegacyCiphertext + } + padding := int(src[length-1]) + if padding == 0 || padding > blockSize || padding > length { + return nil, errLegacyCiphertext + } + for _, b := range src[length-padding:] { + if int(b) != padding { + return nil, errLegacyCiphertext + } + } + return src[:length-padding], nil +} + func PKCS5UnPadding(src []byte) []byte { length := len(src) unpadding := int(src[length-1]) diff --git a/machinery/src/encryption/recording.go b/machinery/src/encryption/recording.go new file mode 100644 index 0000000..e0ab84a --- /dev/null +++ b/machinery/src/encryption/recording.go @@ -0,0 +1,167 @@ +package encryption + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/pbkdf2" + "crypto/rand" + "crypto/sha256" + "encoding/binary" + "errors" + "strconv" + "sync" +) + +// Recording encryption format, version 1: +// +// magic 8 bytes "KRBREC1\x00" +// iterations 4 bytes big-endian PBKDF2-HMAC-SHA256 iteration count +// salt 16 bytes +// nonce 12 bytes +// payload AES-256-GCM ciphertext followed by the 16-byte tag +// +// The header (magic through nonce) is authenticated as additional data, so +// any change to it or to the payload is detected on decryption. +// +// Recordings encrypted by older releases use the OpenSSL/CryptoJS "Salted__" +// format (a single MD5 round to derive the key). DecryptRecording still reads +// those, so existing recordings remain playable. +const ( + recordingMagic = "KRBREC1\x00" + recordingSaltSize = 16 + recordingNonceSize = 12 + recordingHeaderSize = len(recordingMagic) + 4 + recordingSaltSize + recordingNonceSize + recordingKeySize = 32 + recordingIterations = 600_000 + recordingMaxIterations = 10_000_000 + recordingKeyCacheLimit = 16 + legacyOpenSSLSaltHeader = "Salted__" +) + +var ( + ErrRecordingFormat = errors.New("unknown recording encryption format") + ErrRecordingDecrypt = errors.New("recording could not be decrypted: wrong key or corrupted file") + errLegacyCiphertext = errors.New("invalid legacy ciphertext") + recordingKeyCacheMu sync.Mutex + recordingKeyCache = map[string][]byte{} + recordingSaltOnce sync.Once + recordingProcessSalt []byte + recordingProcessError error +) + +// processSalt returns a random salt generated once per process. Deriving the +// key with PBKDF2 is deliberately slow, so encryption reuses one salt and the +// cached key for every recording instead of paying that cost per file. Each +// recording still gets a unique random nonce. +func processSalt() ([]byte, error) { + recordingSaltOnce.Do(func() { + salt := make([]byte, recordingSaltSize) + if _, err := rand.Read(salt); err != nil { + recordingProcessError = err + return + } + recordingProcessSalt = salt + }) + return recordingProcessSalt, recordingProcessError +} + +func recordingKey(password string, salt []byte, iterations int) ([]byte, error) { + cacheKey := string(salt) + "\x00" + strconv.Itoa(iterations) + "\x00" + password + recordingKeyCacheMu.Lock() + if key, ok := recordingKeyCache[cacheKey]; ok { + recordingKeyCacheMu.Unlock() + return key, nil + } + recordingKeyCacheMu.Unlock() + + key, err := pbkdf2.Key(sha256.New, password, salt, iterations, recordingKeySize) + if err != nil { + return nil, err + } + + recordingKeyCacheMu.Lock() + if len(recordingKeyCache) >= recordingKeyCacheLimit { + recordingKeyCache = map[string][]byte{} + } + recordingKeyCache[cacheKey] = key + recordingKeyCacheMu.Unlock() + return key, nil +} + +// EncryptRecording encrypts a recording with a key derived from password. +func EncryptRecording(content []byte, password string) ([]byte, error) { + if password == "" { + return nil, errors.New("empty encryption key") + } + salt, err := processSalt() + if err != nil { + return nil, err + } + key, err := recordingKey(password, salt, recordingIterations) + if err != nil { + return nil, err + } + gcm, err := newRecordingGCM(key) + if err != nil { + return nil, err + } + + out := make([]byte, recordingHeaderSize, recordingHeaderSize+len(content)+gcm.Overhead()) + copy(out, recordingMagic) + binary.BigEndian.PutUint32(out[len(recordingMagic):], uint32(recordingIterations)) + copy(out[len(recordingMagic)+4:], salt) + nonce := out[len(recordingMagic)+4+recordingSaltSize : recordingHeaderSize] + if _, err := rand.Read(nonce); err != nil { + return nil, err + } + header := out[:recordingHeaderSize] + return gcm.Seal(out, nonce, content, header), nil +} + +// DecryptRecording decrypts a recording written by EncryptRecording, or by +// older releases in the legacy OpenSSL "Salted__" format. +func DecryptRecording(data []byte, password string) ([]byte, error) { + switch { + case bytes.HasPrefix(data, []byte(recordingMagic)): + return decryptRecordingV1(data, password) + case bytes.HasPrefix(data, []byte(legacyOpenSSLSaltHeader)): + return AesDecrypt(data, password) + default: + return nil, ErrRecordingFormat + } +} + +func decryptRecordingV1(data []byte, password string) ([]byte, error) { + if len(data) < recordingHeaderSize+16 { + return nil, ErrRecordingDecrypt + } + iterations := int(binary.BigEndian.Uint32(data[len(recordingMagic):])) + if iterations < 1 || iterations > recordingMaxIterations { + return nil, ErrRecordingDecrypt + } + salt := data[len(recordingMagic)+4 : len(recordingMagic)+4+recordingSaltSize] + nonce := data[len(recordingMagic)+4+recordingSaltSize : recordingHeaderSize] + + key, err := recordingKey(password, salt, iterations) + if err != nil { + return nil, err + } + gcm, err := newRecordingGCM(key) + if err != nil { + return nil, err + } + plain, err := gcm.Open(nil, nonce, data[recordingHeaderSize:], data[:recordingHeaderSize]) + if err != nil { + return nil, ErrRecordingDecrypt + } + return plain, nil +} + +func newRecordingGCM(key []byte) (cipher.AEAD, error) { + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + return cipher.NewGCM(block) +} diff --git a/machinery/src/encryption/recording_test.go b/machinery/src/encryption/recording_test.go new file mode 100644 index 0000000..51fffd5 --- /dev/null +++ b/machinery/src/encryption/recording_test.go @@ -0,0 +1,168 @@ +package encryption + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/pbkdf2" + "crypto/sha256" + "encoding/binary" + "errors" + "testing" + "time" +) + +const testRecordingKey = "correct horse battery staple" + +func TestEncryptRecordingRoundTrip(t *testing.T) { + content := bytes.Repeat([]byte("mp4-bytes"), 1000) + encrypted, err := EncryptRecording(content, testRecordingKey) + if err != nil { + t.Fatal(err) + } + if !bytes.HasPrefix(encrypted, []byte(recordingMagic)) { + t.Fatalf("missing format header: %q", encrypted[:8]) + } + if got := binary.BigEndian.Uint32(encrypted[len(recordingMagic):]); got != recordingIterations { + t.Fatalf("iterations in header = %d, want %d", got, recordingIterations) + } + decrypted, err := DecryptRecording(encrypted, testRecordingKey) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(decrypted, content) { + t.Fatal("round trip changed the content") + } +} + +// TestRecordingFormatMatchesSpecification decrypts with only the documented +// layout and the standard library, so the format is pinned independently of +// the implementation. +func TestRecordingFormatMatchesSpecification(t *testing.T) { + encrypted, err := EncryptRecording([]byte("payload"), testRecordingKey) + if err != nil { + t.Fatal(err) + } + iterations := int(binary.BigEndian.Uint32(encrypted[8:12])) + salt, nonce := encrypted[12:28], encrypted[28:40] + key, err := pbkdf2.Key(sha256.New, testRecordingKey, salt, iterations, 32) + if err != nil { + t.Fatal(err) + } + block, _ := aes.NewCipher(key) + gcm, _ := cipher.NewGCM(block) + plain, err := gcm.Open(nil, nonce, encrypted[40:], encrypted[:40]) + if err != nil || string(plain) != "payload" { + t.Fatalf("independent PBKDF2-SHA256 + AES-256-GCM decryption failed: %q, %v", plain, err) + } +} + +func TestDecryptRecordingReadsLegacyFormat(t *testing.T) { + legacy, err := AesEncrypt([]byte("old recording"), testRecordingKey) + if err != nil { + t.Fatal(err) + } + got, err := DecryptRecording(legacy, testRecordingKey) + if err != nil || string(got) != "old recording" { + t.Fatalf("legacy recording not readable: %q, %v", got, err) + } +} + +func TestEncryptRecordingUsesFreshNonces(t *testing.T) { + a, _ := EncryptRecording([]byte("same"), testRecordingKey) + b, _ := EncryptRecording([]byte("same"), testRecordingKey) + if bytes.Equal(a[28:40], b[28:40]) { + t.Fatal("two recordings share a nonce") + } + if bytes.Equal(a, b) { + t.Fatal("identical plaintexts produced identical ciphertexts") + } +} + +func TestEncryptRecordingCachesDerivedKey(t *testing.T) { + if _, err := EncryptRecording([]byte("warm"), testRecordingKey); err != nil { + t.Fatal(err) + } + start := time.Now() + for i := 0; i < 20; i++ { + if _, err := EncryptRecording([]byte("frame"), testRecordingKey); err != nil { + t.Fatal(err) + } + } + // 20 fresh PBKDF2 derivations at 600k iterations would take seconds. + if elapsed := time.Since(start); elapsed > 500*time.Millisecond { + t.Fatalf("20 encryptions took %v; the derived key is not being reused", elapsed) + } +} + +func TestDecryptRecordingRejectsWrongKeyAndTampering(t *testing.T) { + encrypted, err := EncryptRecording([]byte("secret footage"), testRecordingKey) + if err != nil { + t.Fatal(err) + } + if _, err := DecryptRecording(encrypted, "wrong key"); !errors.Is(err, ErrRecordingDecrypt) { + t.Fatalf("wrong key: err = %v", err) + } + + payload := append([]byte(nil), encrypted...) + payload[len(payload)-1] ^= 0x01 + if _, err := DecryptRecording(payload, testRecordingKey); !errors.Is(err, ErrRecordingDecrypt) { + t.Fatalf("modified payload: err = %v", err) + } + + header := append([]byte(nil), encrypted...) + header[30] ^= 0x01 // inside the nonce, which is authenticated + if _, err := DecryptRecording(header, testRecordingKey); !errors.Is(err, ErrRecordingDecrypt) { + t.Fatalf("modified header: err = %v", err) + } +} + +func TestDecryptRecordingRejectsExcessiveIterations(t *testing.T) { + encrypted, _ := EncryptRecording([]byte("x"), testRecordingKey) + crafted := append([]byte(nil), encrypted...) + binary.BigEndian.PutUint32(crafted[8:12], 0xFFFFFFFF) + start := time.Now() + if _, err := DecryptRecording(crafted, testRecordingKey); !errors.Is(err, ErrRecordingDecrypt) { + t.Fatalf("crafted iteration count: err = %v", err) + } + if time.Since(start) > 100*time.Millisecond { + t.Fatal("crafted iteration count was not rejected before key derivation") + } +} + +func TestDecryptRecordingNeverPanicsOnMalformedInput(t *testing.T) { + legacy, _ := AesEncrypt([]byte("old recording"), testRecordingKey) + current, _ := EncryptRecording([]byte("new recording"), testRecordingKey) + inputs := [][]byte{nil, {}, []byte("Salted__"), []byte("KRBREC1\x00"), []byte("not a recording")} + for n := 0; n < len(legacy); n++ { + inputs = append(inputs, legacy[:n]) + } + for n := 0; n < 60 && n < len(current); n++ { + inputs = append(inputs, current[:n]) + } + for _, input := range inputs { + func() { + defer func() { + if r := recover(); r != nil { + t.Fatalf("DecryptRecording panicked on %d bytes: %v", len(input), r) + } + }() + _, _ = DecryptRecording(input, testRecordingKey) + _, _ = DecryptRecording(input, "wrong key") + }() + } + if _, err := DecryptRecording([]byte("plain mp4"), testRecordingKey); !errors.Is(err, ErrRecordingFormat) { + t.Fatalf("unknown format: err = %v", err) + } +} + +func TestLegacyDecryptWithWrongKeyReturnsError(t *testing.T) { + legacy, _ := AesEncrypt([]byte("old recording"), testRecordingKey) + // With the wrong key the padding is garbage; it used to be trusted and + // could slice out of range. + for i := 0; i < 50; i++ { + if out, err := AesDecrypt(legacy, "wrong key "+string(rune('a'+i))); err == nil && string(out) == "old recording" { + t.Fatal("wrong key decrypted the recording") + } + } +} diff --git a/machinery/src/routers/http/recordings_test.go b/machinery/src/routers/http/recordings_test.go index 5b9e277..9b9db0b 100644 --- a/machinery/src/routers/http/recordings_test.go +++ b/machinery/src/routers/http/recordings_test.go @@ -14,11 +14,10 @@ import ( func TestFilesServesFlatAndDatedRecordings(t *testing.T) { gin.SetMode(gin.TestMode) for _, layout := range []string{"flat", "dated"} { - for _, encrypted := range []bool{false, true} { - name := layout + "-plain" - if encrypted { - name = layout + "-encrypted" - } + // "legacy" is the OpenSSL format written by older releases; "v1" is + // the current PBKDF2 + AES-GCM recording format. + for _, format := range []string{"plain", "legacy", "v1"} { + name := layout + "-" + format t.Run(name, func(t *testing.T) { const fileName = "1767355200123-device-key.mp4" directory := t.TempDir() @@ -32,14 +31,18 @@ func TestFilesServesFlatAndDatedRecordings(t *testing.T) { } config := &models.Configuration{Config: models.Config{Encryption: &models.Encryption{}}} content := []byte("recording-payload") - if encrypted { + if format != "plain" { config.Config.Encryption.Recordings = "true" config.Config.Encryption.SymmetricKey = "0123456789abcdef0123456789abcdef" - ciphertext, err := encryption.AesEncrypt(content, config.Config.Encryption.SymmetricKey) + encrypt := encryption.EncryptRecording + if format == "legacy" { + encrypt = encryption.AesEncrypt + } + ciphertext, err := encrypt(content, config.Config.Encryption.SymmetricKey) if err != nil { t.Fatal(err) } - content = []byte(ciphertext) + content = ciphertext } if err := os.WriteFile(filePath, content, 0o644); err != nil { t.Fatal(err) diff --git a/machinery/src/routers/http/server.go b/machinery/src/routers/http/server.go index 0402e81..d109046 100644 --- a/machinery/src/routers/http/server.go +++ b/machinery/src/routers/http/server.go @@ -199,7 +199,7 @@ func Files(c *gin.Context, configDirectory string, configuration *models.Configu } // Decrypt file - contents, err = encryption.AesDecrypt(contents, symmetricKey) + contents, err = encryption.DecryptRecording(contents, symmetricKey) if err != nil { c.JSON(404, gin.H{"error": "File not found"}) return diff --git a/machinery/src/utils/main.go b/machinery/src/utils/main.go index 3e764c9..9d7e5c3 100644 --- a/machinery/src/utils/main.go +++ b/machinery/src/utils/main.go @@ -409,7 +409,7 @@ func Decrypt(directoryOrFile string, symmetricKey []byte) { return } // Decrypt using AES key - decrypted, err := encryption.AesDecrypt(content, string(symmetricKey)) + decrypted, err := encryption.DecryptRecording(content, string(symmetricKey)) if err != nil { log.Fatal("Something went wrong while decrypting: " + err.Error()) return