diff --git a/packages/db/src/FileFlatDb/Cache.ts b/packages/db/src/FileFlatDb/Cache.ts new file mode 100644 index 000000000..780aa6f60 --- /dev/null +++ b/packages/db/src/FileFlatDb/Cache.ts @@ -0,0 +1,57 @@ +// Copyright 2017-2018 @polkadot/db authors & contributors +// This software may be modified and distributed under the terms +// of the Apache-2.0 license. See the LICENSE file for details. + +import { BaseDbOptions } from '../types'; + +import fs from 'fs'; +import { LRUMap } from 'lru_map'; + +import File from './File'; +import defaults from './defaults'; + +export default class Cache extends File { + protected _lruBranch: LRUMap; + protected _lruData: LRUMap; + + constructor (base: string, file: string, options: BaseDbOptions) { + super(base, file, options); + + this._lruBranch = new LRUMap(defaults.LRU_BRANCH_COUNT); + this._lruData = new LRUMap(defaults.LRU_DATA_COUNT); + } + + protected _cacheBranch (branchAt: number, branch: Buffer): void { + this._lruBranch.set(branchAt, branch); + } + + protected _cacheData (dataAt: number, data: Buffer): void { + this._lruData.set(dataAt, data); + } + + protected _getCachedBranch (branchAt: number): Buffer { + let branch = this._lruBranch.get(branchAt); + + if (!branch) { + branch = Buffer.alloc(defaults.BRANCH_SIZE); + + fs.readSync(this._fd, branch, 0, defaults.BRANCH_SIZE, branchAt); + this._cacheBranch(branchAt, branch); + } + + return branch; + } + + protected _getCachedData (dataAt: number, length: number): Buffer { + let data = this._lruData.get(dataAt); + + if (!data) { + data = Buffer.alloc(length); + + fs.readSync(this._fd, data, 0, length, dataAt); + this._cacheData(dataAt, data); + } + + return data; + } +} diff --git a/packages/db/src/FileFlatDb/File.ts b/packages/db/src/FileFlatDb/File.ts new file mode 100644 index 000000000..fc59fcb5d --- /dev/null +++ b/packages/db/src/FileFlatDb/File.ts @@ -0,0 +1,61 @@ +// Copyright 2017-2018 @polkadot/db authors & contributors +// This software may be modified and distributed under the terms +// of the Apache-2.0 license. See the LICENSE file for details. + +import { BaseDbOptions } from '../types'; + +import fs from 'fs'; +import mkdirp from 'mkdirp'; +import path from 'path'; +import { assert, isUndefined } from '@polkadot/util/index'; + +import Serialize from './Serialize'; +import defaults from './defaults'; + +export default class File extends Serialize { + protected _fd: number = -1; + protected _fileSize: number = 0; + protected _path: string; + protected _file: string; + + constructor (base: string, file: string, options: BaseDbOptions) { + super(); + + this._file = file; + this._path = path.join(base, file); + this._isCompressed = options && !isUndefined(options.isCompressed) + ? options.isCompressed + : false; + + mkdirp.sync(base); + } + + protected assertOpen (open: boolean = true): void { + const test = open + ? this._fd !== -1 + : this._fd === -1; + + assert(test, `Expected ${open ? 'an open' : 'a closed'} database`); + } + + protected _close (): void { + fs.closeSync(this._fd); + + this._fd = -1; + } + + protected _open (file: string, startEmpty: boolean = false): void { + const isExisting = fs.existsSync(file); + + if (!isExisting || startEmpty) { + if (isExisting) { + fs.renameSync(file, `${file}.${Date.now()}`); + } + + fs.writeFileSync(file, Buffer.alloc(defaults.BRANCH_SIZE)); + } + + this._fd = fs.openSync(file, 'r+'); + this._fileSize = fs.fstatSync(this._fd).size; + } +} diff --git a/packages/db/src/FileFlatDb/Impl.ts b/packages/db/src/FileFlatDb/Impl.ts new file mode 100644 index 000000000..37a928040 --- /dev/null +++ b/packages/db/src/FileFlatDb/Impl.ts @@ -0,0 +1,252 @@ +// Copyright 2017-2018 @polkadot/db authors & contributors +// This software may be modified and distributed under the terms +// of the Apache-2.0 license. See the LICENSE file for details. + +import { Key, NibbleBuffer, Slot, Value } from './types'; + +import fs from 'fs'; +import { logger } from '@polkadot/util/index'; + +import Cache from './Cache'; +import defaults from './defaults'; + +const l = logger('db/flat'); + +export default class Impl extends Cache { + protected _getKeyValue (keyAt: number): Buffer { + return this._getCachedData(keyAt, defaults.KEY_TOTAL_SIZE); + } + + protected _retrieveBranch (doCreate: boolean, branch: Buffer, entryIndex: number, keyIndex: number, key: NibbleBuffer): Key | null { + const nextBranchAt = branch.readUIntBE(entryIndex + 1, defaults.UINT_SIZE); + + l.debug(() => ['findKey/isBranch', { nextBranchAt }]); + + return this._findKey(key, doCreate, keyIndex + 1, nextBranchAt); + } + + protected _retrieveEmpty (doCreate: boolean, branch: Buffer, branchAt: number, entryIndex: number, key: NibbleBuffer): Key | null { + l.debug(() => ['findKey/isEmpty']); + + return doCreate + ? this.writeNewLeaf(branch, branchAt, entryIndex, key) + : null; + } + + protected _retrieveLeaf (doCreate: boolean, branch: Buffer, branchAt: number, entryIndex: number, keyIndex: number, key: NibbleBuffer): Key | null { + const keyAt = branch.readUIntBE(entryIndex + 1, defaults.UINT_SIZE); + const keyValue = this._getKeyValue(keyAt); + const prevKey = this._serializeKey(keyValue.subarray(0, defaults.KEY_SIZE)); + let matchIndex = keyIndex; + + l.debug(() => ['findKey/isLeaf', { keyAt, branch, branchAt, entryIndex, keyValue }]); + + while (matchIndex < defaults.KEY_SIZE) { + if (prevKey.nibbles[matchIndex] !== key.nibbles[matchIndex]) { + break; + } + + matchIndex++; + } + + if (matchIndex !== defaults.KEY_SIZE) { + return doCreate + ? this.writeNewBranch(branch, branchAt, entryIndex, key, keyAt, prevKey, matchIndex, matchIndex - keyIndex - 1) + : null; + } + + return { + key, + keyAt, + keyValue + }; + } + + protected _findKey (key: NibbleBuffer, doCreate: boolean, keyIndex: number, branchAt: number): Key | null { + const entryIndex = key.nibbles[keyIndex] * defaults.ENTRY_SIZE; + const branch = this._getCachedBranch(branchAt); + + l.debug(() => ['findKey', { key, doCreate, keyIndex, branchAt, branch, entryIndex }]); + + const entryType = branch[entryIndex]; + + switch (entryType) { + case Slot.BRANCH: + return this._retrieveBranch(doCreate, branch, entryIndex, keyIndex, key); + + case Slot.EMPTY: + return this._retrieveEmpty(doCreate, branch, branchAt, entryIndex, key); + + case Slot.LEAF: + return this._retrieveLeaf(doCreate, branch, branchAt, entryIndex, keyIndex, key); + + default: + throw new Error(`Unhandled entry type ${entryType}`); + } + } + + protected _extractValueInfo (keyValue: Buffer): { valueAt: number, valueLength: number } { + return { + valueLength: keyValue.readUIntBE(defaults.KEY_SIZE, defaults.UINT_SIZE), + valueAt: keyValue.readUIntBE(defaults.KEY_SIZE + defaults.UINT_SIZE, defaults.UINT_SIZE) + }; + } + + protected _readValue (keyValue: Buffer): Value { + l.debug(() => ['readValue', { keyValue }]); + + const { valueAt, valueLength } = this._extractValueInfo(keyValue); + const value = this._getCachedData(valueAt, valueLength); + + return { + value, + valueAt + }; + } + + protected _writeValue (keyAt: number, keyValue: Buffer, value: Buffer): Value { + l.debug(() => ['writeValue', { keyAt, keyValue, value }]); + + const current = this._extractValueInfo(keyValue); + const valueAt = value.length > current.valueLength + ? this._writeNewBuffer(value) + : this._writeUpdatedBuffer(value, current.valueAt); + + keyValue.writeUIntBE(value.length, defaults.KEY_SIZE, defaults.UINT_SIZE); + keyValue.writeUIntBE(valueAt, defaults.KEY_SIZE + defaults.UINT_SIZE, defaults.UINT_SIZE); + fs.writeSync(this._fd, keyValue, defaults.KEY_SIZE, 2 * defaults.UINT_SIZE, keyAt + defaults.KEY_SIZE); + + return { + value, + valueAt + }; + } + + protected _writeNewKey (key: NibbleBuffer): Key { + l.debug(() => ['writeNewKey', { key }]); + + const keyValue = Buffer.alloc(defaults.KEY_TOTAL_SIZE); + + keyValue.set(key.buffer, 0); + + const keyAt = this._writeNewBuffer(keyValue); + + return { + key, + keyAt, + keyValue + }; + } + + protected writeNewKey (key: NibbleBuffer): Key { + const result = this._writeNewKey(key); + + l.debug(() => ['writeNewKey', '=>', { result }]); + + return result; + } + + protected _writeNewBranch (branch: Buffer, branchAt: number, entryIndex: number, key: NibbleBuffer, prevAt: number, prevKey: NibbleBuffer, matchIndex: number, depth: number): Key { + l.debug(() => ['writeNewBranch', { branch, branchAt, entryIndex, key, prevAt, prevKey, matchIndex, depth }]); + + const { keyAt, keyValue } = this.writeNewKey(key); + const keyIndex = key.nibbles[matchIndex] * defaults.ENTRY_SIZE; + const prevIndex = prevKey.nibbles[matchIndex] * defaults.ENTRY_SIZE; + const buffers: Array = []; + let newBranchAt = this._fileSize; + let newBranch = Buffer.alloc(defaults.BRANCH_SIZE); + + newBranch.set([Slot.LEAF], keyIndex); + newBranch.writeUIntBE(keyAt, keyIndex + 1, defaults.UINT_SIZE); + newBranch.set([Slot.LEAF], prevIndex); + newBranch.writeUIntBE(prevAt, prevIndex + 1, defaults.UINT_SIZE); + buffers.push(newBranch); + + for (let offset = 1; depth > 0; depth--, offset++) { + const branchIndex = key.nibbles[matchIndex - offset] * defaults.ENTRY_SIZE; + + newBranch = Buffer.alloc(defaults.BRANCH_SIZE); + newBranch.set([Slot.BRANCH], branchIndex); + newBranch.writeUIntBE(newBranchAt, branchIndex + 1, defaults.UINT_SIZE); + buffers.push(newBranch); + newBranchAt += defaults.BRANCH_SIZE; + } + + this._writeNewBuffers(buffers); + + branch.set([Slot.BRANCH], entryIndex); + branch.writeUIntBE(newBranchAt, entryIndex + 1, defaults.UINT_SIZE); + fs.writeSync(this._fd, branch, entryIndex, defaults.ENTRY_SIZE, branchAt + entryIndex); + + return { + key, + keyAt, + keyValue + }; + } + + protected writeNewBranch (branch: Buffer, branchAt: number, entryIndex: number, key: NibbleBuffer, prevAt: number, prevValue: NibbleBuffer, matchIndex: number, depth: number): Key { + const result = this._writeNewBranch(branch, branchAt, entryIndex, key, prevAt, prevValue, matchIndex, depth); + + l.debug(() => ['writeNewBranch', '=>', { result }]); + + return result; + } + + protected _writeNewLeaf (branch: Buffer, branchAt: number, entryIndex: number, key: NibbleBuffer): Key { + l.debug(() => ['writeNewLeaf', { branch, branchAt, entryIndex, key }]); + + const { keyAt, keyValue } = this.writeNewKey(key); + + branch.set([Slot.LEAF], entryIndex); + branch.writeUIntBE(keyAt, entryIndex + 1, defaults.UINT_SIZE); + + fs.writeSync(this._fd, branch, entryIndex, defaults.ENTRY_SIZE, branchAt + entryIndex); + + return { + key, + keyAt, + keyValue + }; + } + + protected writeNewLeaf (branch: Buffer, branchAt: number, entryIndex: number, key: NibbleBuffer): Key { + const result = this._writeNewLeaf(branch, branchAt, entryIndex, key); + + l.debug(() => ['writeNewLeaf', '=>', { result }]); + + return result; + } + + protected _writeUpdatedBuffer (buffer: Buffer, bufferAt: number): number { + fs.writeSync(this._fd, buffer, 0, buffer.length, bufferAt); + this._cacheData(bufferAt, buffer); + + return bufferAt; + } + + protected _writeNewBuffer (buffer: Buffer, withCache: boolean = true): number { + const startAt = this._fileSize; + + fs.writeSync(this._fd, buffer, 0, buffer.length, startAt); + + if (withCache) { + this._cacheData(startAt, buffer); + } + + this._fileSize += buffer.length; + + return startAt; + } + + protected _writeNewBuffers (buffers: Array): number { + let bufferAt = this._fileSize; + + buffers.forEach((buffer) => { + this._cacheData(bufferAt, buffer); + bufferAt += buffer.length; + }); + + return this._writeNewBuffer(Buffer.concat(buffers), false); + } +} diff --git a/packages/db/src/FileFlatDb/Serialize.ts b/packages/db/src/FileFlatDb/Serialize.ts new file mode 100644 index 000000000..be352d326 --- /dev/null +++ b/packages/db/src/FileFlatDb/Serialize.ts @@ -0,0 +1,50 @@ +// Copyright 2017-2018 @polkadot/db authors & contributors +// This software may be modified and distributed under the terms +// of the Apache-2.0 license. See the LICENSE file for details. + +import { NibbleBuffer } from './types'; + +import snappy from 'snappy'; +import { toNibbles } from '@polkadot/trie-codec/util'; +import { assert, bufferToU8a, u8aToBuffer, u8aToHex } from '@polkadot/util/index'; + +import defaults from './defaults'; + +export default class Serialize { + protected _isCompressed: boolean = false; + + protected _deserializeValue (value: Buffer): Uint8Array | null { + return bufferToU8a( + this._isCompressed + ? snappy.uncompressSync(value) + : value + ); + } + + protected _serializeValue (value: Uint8Array): Buffer { + return this._isCompressed + ? snappy.compressSync( + u8aToBuffer(value) + ) + : u8aToBuffer(value); + } + + protected _serializeKey (u8a: Uint8Array): NibbleBuffer { + assert(u8a.length <= defaults.KEY_SIZE, `${u8aToHex(u8a)} too large, expected <= 32 bytes`); + + let buffer; + + if (u8a.length === defaults.KEY_SIZE) { + buffer = u8aToBuffer(u8a); + } else { + buffer = Buffer.alloc(defaults.KEY_SIZE); + + buffer.set(u8a, 0); + } + + return { + buffer, + nibbles: toNibbles(u8a) + }; + } +} diff --git a/packages/db/src/FileFlatDb/defaults.ts b/packages/db/src/FileFlatDb/defaults.ts index 691f62202..afba6bbfb 100644 --- a/packages/db/src/FileFlatDb/defaults.ts +++ b/packages/db/src/FileFlatDb/defaults.ts @@ -10,6 +10,8 @@ const ENTRY_NUM = 16; // nibbles, 256 for bytes (where serialize would be noop) const ENTRY_SIZE = 1 + UINT_SIZE; const BRANCH_SIZE = ENTRY_NUM * ENTRY_SIZE; const DEFAULT_FILE = 'store.db'; +const LRU_BRANCH_COUNT = 16384; // * 96 = bytes +const LRU_DATA_COUNT = 8192; export default { BRANCH_SIZE, @@ -18,5 +20,7 @@ export default { ENTRY_SIZE, KEY_SIZE, KEY_TOTAL_SIZE, + LRU_BRANCH_COUNT, + LRU_DATA_COUNT, UINT_SIZE }; diff --git a/packages/db/src/FileFlatDb/index.ts b/packages/db/src/FileFlatDb/index.ts index 2a2560b89..0c829a149 100644 --- a/packages/db/src/FileFlatDb/index.ts +++ b/packages/db/src/FileFlatDb/index.ts @@ -3,52 +3,27 @@ // of the Apache-2.0 license. See the LICENSE file for details. import { BaseDb, BaseDbOptions, ProgressCb } from '../types'; -import { Key, NibbleBuffer, Slot, Value } from './types'; +import { Key, NibbleBuffer, Value } from './types'; import fs from 'fs'; -import { LRUMap } from 'lru_map'; -import mkdirp from 'mkdirp'; import path from 'path'; -import snappy from 'snappy'; -import { toNibbles } from '@polkadot/trie-codec/util'; -import { assert, bufferToU8a, isUndefined, logger, u8aToBuffer, u8aToHex } from '@polkadot/util/index'; +import { logger } from '@polkadot/util/index'; -import Compact from './Compact'; import defaults from './defaults'; - -const LRU_BRANCH_COUNT = 16384; // * 96 = bytes -const LRU_DATA_COUNT = 8192; +import Compact from './Compact'; +import Impl from './Impl'; const l = logger('db/flat'); -export default class FileFlatDb implements BaseDb { - private _fd: number; - private _fileSize: number = 0; - private _path: string; - private _file: string; - private _isCompressed: boolean; - private _lruBranch: LRUMap; - private _lruData: LRUMap; - +export default class FileFlatDb extends Impl implements BaseDb { constructor (base: string, file: string = defaults.DEFAULT_FILE, options: BaseDbOptions = {}) { - this._fd = -1; - this._file = file; - this._path = path.join(base, file); - this._lruBranch = new LRUMap(LRU_BRANCH_COUNT); - this._lruData = new LRUMap(LRU_DATA_COUNT); - this._isCompressed = options && !isUndefined(options.isCompressed) - ? options.isCompressed - : false; - - mkdirp.sync(base); + super(base, file, options); } open (): void { this.assertOpen(false); - this._fd = this._open(this._path); - this._fileSize = fs.fstatSync(this._fd).size; - + this._open(this._path); this._lruBranch.clear(); this._lruData.clear(); } @@ -56,9 +31,7 @@ export default class FileFlatDb implements BaseDb { close (): void { this.assertOpen(); - fs.closeSync(this._fd); - - this._fd = -1; + this._close(); this._lruBranch.clear(); this._lruData.clear(); } @@ -72,8 +45,7 @@ export default class FileFlatDb implements BaseDb { empty (): void { this.assertOpen(false); - this._fd = this._open(this._path, true); - this._fileSize = fs.fstatSync(this._fd).size; + this._open(this._path, true); } size (): number { @@ -138,138 +110,7 @@ export default class FileFlatDb implements BaseDb { ); } - private _deserializeValue (value: Buffer): Uint8Array | null { - return bufferToU8a( - this._isCompressed - ? snappy.uncompressSync(value) - : value - ); - } - - private _serializeValue (value: Uint8Array): Buffer { - return this._isCompressed - ? snappy.compressSync( - u8aToBuffer(value) - ) - : u8aToBuffer(value); - } - - private _serializeKey (u8a: Uint8Array): NibbleBuffer { - if (u8a.length > defaults.KEY_SIZE) { - throw new Error(`${u8aToHex(u8a)} too large, expected <= 32 bytes`); - } - - let buffer; - - if (u8a.length === defaults.KEY_SIZE) { - buffer = u8aToBuffer(u8a); - } else { - buffer = Buffer.alloc(defaults.KEY_SIZE); - - buffer.set(u8a, 0); - } - - return { - buffer, - nibbles: toNibbles(u8a) - }; - } - - private _cacheBranch (branchAt: number, branch: Buffer): void { - this._lruBranch.set(branchAt, branch); - } - - private _cacheData (dataAt: number, data: Buffer): void { - this._lruData.set(dataAt, data); - } - - private _getCachedBranch (branchAt: number): Buffer { - let branch = this._lruBranch.get(branchAt); - - if (!branch) { - branch = Buffer.alloc(defaults.BRANCH_SIZE); - - fs.readSync(this._fd, branch, 0, defaults.BRANCH_SIZE, branchAt); - this._cacheBranch(branchAt, branch); - } - - return branch; - } - - private _getCachedData (dataAt: number, length: number): Buffer { - let data = this._lruData.get(dataAt); - - if (!data) { - data = Buffer.alloc(length); - - fs.readSync(this._fd, data, 0, length, dataAt); - this._cacheData(dataAt, data); - } - - return data; - } - - private _getKeyValue (keyAt: number): Buffer { - return this._getCachedData(keyAt, defaults.KEY_TOTAL_SIZE); - } - - private _findKey (key: NibbleBuffer, doCreate: boolean, keyIndex: number, branchAt: number): Key | null { - const entryIndex = key.nibbles[keyIndex] * defaults.ENTRY_SIZE; - const branch = this._getCachedBranch(branchAt); - - l.debug(() => ['findKey', { key, doCreate, keyIndex, branchAt, branch, entryIndex }]); - - const entryType = branch[entryIndex]; - - if (entryType === Slot.BRANCH) { - const nextBranchAt = branch.readUIntBE(entryIndex + 1, defaults.UINT_SIZE); - - l.debug(() => ['findKey/isBranch', { nextBranchAt }]); - - return this._findKey(key, doCreate, keyIndex + 1, nextBranchAt); - } - - if (entryType === Slot.EMPTY) { - l.debug(() => ['findKey/isEmpty']); - - return doCreate - ? this.writeNewLeaf(branch, branchAt, entryIndex, key) - : null; - } - - if (entryType === Slot.LEAF) { - const keyAt = branch.readUIntBE(entryIndex + 1, defaults.UINT_SIZE); - const keyValue = this._getKeyValue(keyAt); - const prevKey = this._serializeKey(keyValue.subarray(0, defaults.KEY_SIZE)); - let matchIndex = keyIndex; - - l.debug(() => ['findKey/isLeaf', { keyAt, branch, branchAt, entryIndex, keyValue }]); - - while (matchIndex < defaults.KEY_SIZE) { - if (prevKey.nibbles[matchIndex] !== key.nibbles[matchIndex]) { - break; - } - - matchIndex++; - } - - if (matchIndex !== defaults.KEY_SIZE) { - return doCreate - ? this.writeNewBranch(branch, branchAt, entryIndex, key, keyAt, prevKey, matchIndex, matchIndex - keyIndex - 1) - : null; - } - - return { - key, - keyAt, - keyValue - }; - } - - throw new Error(`Unhandled entry type ${entryType}`); - } - - private findKey (key: NibbleBuffer, doCreate: boolean): Key | null { + protected findKey (key: NibbleBuffer, doCreate: boolean): Key | null { const result = this._findKey(key, doCreate, 0, 0); l.debug(() => ['findKey', { result }]); @@ -277,26 +118,7 @@ export default class FileFlatDb implements BaseDb { return result; } - private _extractValueInfo (keyValue: Buffer): { valueAt: number, valueLength: number } { - return { - valueLength: keyValue.readUIntBE(defaults.KEY_SIZE, defaults.UINT_SIZE), - valueAt: keyValue.readUIntBE(defaults.KEY_SIZE + defaults.UINT_SIZE, defaults.UINT_SIZE) - }; - } - - private _readValue (keyValue: Buffer): Value { - l.debug(() => ['readValue', { keyValue }]); - - const { valueAt, valueLength } = this._extractValueInfo(keyValue); - const value = this._getCachedData(valueAt, valueLength); - - return { - value, - valueAt - }; - } - - private readValue ({ keyValue }: Key): Value { + protected readValue ({ keyValue }: Key): Value { const result = this._readValue(keyValue); l.debug(() => ['readValue', { result }]); @@ -304,179 +126,11 @@ export default class FileFlatDb implements BaseDb { return result; } - private _writeValue (keyAt: number, keyValue: Buffer, value: Buffer): Value { - l.debug(() => ['writeValue', { keyAt, keyValue, value }]); - - const current = this._extractValueInfo(keyValue); - const valueAt = value.length > current.valueLength - ? this._writeNewBuffer(value) - : this._writeUpdatedBuffer(value, current.valueAt); - - keyValue.writeUIntBE(value.length, defaults.KEY_SIZE, defaults.UINT_SIZE); - keyValue.writeUIntBE(valueAt, defaults.KEY_SIZE + defaults.UINT_SIZE, defaults.UINT_SIZE); - fs.writeSync(this._fd, keyValue, defaults.KEY_SIZE, 2 * defaults.UINT_SIZE, keyAt + defaults.KEY_SIZE); - - return { - value, - valueAt - }; - } - - private writeValue ({ keyAt, keyValue }: Key, value: Buffer): Value { + protected writeValue ({ keyAt, keyValue }: Key, value: Buffer): Value { const result = this._writeValue(keyAt, keyValue, value); l.debug(() => ['writeValue', '=>', { result }]); return result; } - - private _writeNewKey (key: NibbleBuffer): Key { - l.debug(() => ['writeNewKey', { key }]); - - const keyValue = Buffer.alloc(defaults.KEY_TOTAL_SIZE); - - keyValue.set(key.buffer, 0); - - const keyAt = this._writeNewBuffer(keyValue); - - return { - key, - keyAt, - keyValue - }; - } - - private writeNewKey (key: NibbleBuffer): Key { - const result = this._writeNewKey(key); - - l.debug(() => ['writeNewKey', '=>', { result }]); - - return result; - } - - private _writeNewBranch (branch: Buffer, branchAt: number, entryIndex: number, key: NibbleBuffer, prevAt: number, prevKey: NibbleBuffer, matchIndex: number, depth: number): Key { - l.debug(() => ['writeNewBranch', { branch, branchAt, entryIndex, key, prevAt, prevKey, matchIndex, depth }]); - - const { keyAt, keyValue } = this.writeNewKey(key); - const keyIndex = key.nibbles[matchIndex] * defaults.ENTRY_SIZE; - const prevIndex = prevKey.nibbles[matchIndex] * defaults.ENTRY_SIZE; - const buffers: Array = []; - let newBranchAt = this._fileSize; - let newBranch = Buffer.alloc(defaults.BRANCH_SIZE); - - newBranch.set([Slot.LEAF], keyIndex); - newBranch.writeUIntBE(keyAt, keyIndex + 1, defaults.UINT_SIZE); - newBranch.set([Slot.LEAF], prevIndex); - newBranch.writeUIntBE(prevAt, prevIndex + 1, defaults.UINT_SIZE); - buffers.push(newBranch); - - for (let offset = 1; depth > 0; depth--, offset++) { - const branchIndex = key.nibbles[matchIndex - offset] * defaults.ENTRY_SIZE; - - newBranch = Buffer.alloc(defaults.BRANCH_SIZE); - newBranch.set([Slot.BRANCH], branchIndex); - newBranch.writeUIntBE(newBranchAt, branchIndex + 1, defaults.UINT_SIZE); - buffers.push(newBranch); - newBranchAt += defaults.BRANCH_SIZE; - } - - this._writeNewBuffers(buffers); - - branch.set([Slot.BRANCH], entryIndex); - branch.writeUIntBE(newBranchAt, entryIndex + 1, defaults.UINT_SIZE); - fs.writeSync(this._fd, branch, entryIndex, defaults.ENTRY_SIZE, branchAt + entryIndex); - - return { - key, - keyAt, - keyValue - }; - } - - private writeNewBranch (branch: Buffer, branchAt: number, entryIndex: number, key: NibbleBuffer, prevAt: number, prevValue: NibbleBuffer, matchIndex: number, depth: number): Key { - const result = this._writeNewBranch(branch, branchAt, entryIndex, key, prevAt, prevValue, matchIndex, depth); - - l.debug(() => ['writeNewBranch', '=>', { result }]); - - return result; - } - - private _writeNewLeaf (branch: Buffer, branchAt: number, entryIndex: number, key: NibbleBuffer): Key { - l.debug(() => ['writeNewLeaf', { branch, branchAt, entryIndex, key }]); - - const { keyAt, keyValue } = this.writeNewKey(key); - - branch.set([Slot.LEAF], entryIndex); - branch.writeUIntBE(keyAt, entryIndex + 1, defaults.UINT_SIZE); - - fs.writeSync(this._fd, branch, entryIndex, defaults.ENTRY_SIZE, branchAt + entryIndex); - - return { - key, - keyAt, - keyValue - }; - } - - private writeNewLeaf (branch: Buffer, branchAt: number, entryIndex: number, key: NibbleBuffer): Key { - const result = this._writeNewLeaf(branch, branchAt, entryIndex, key); - - l.debug(() => ['writeNewLeaf', '=>', { result }]); - - return result; - } - - private _writeUpdatedBuffer (buffer: Buffer, bufferAt: number): number { - fs.writeSync(this._fd, buffer, 0, buffer.length, bufferAt); - this._cacheData(bufferAt, buffer); - - return bufferAt; - } - - private _writeNewBuffer (buffer: Buffer, withCache: boolean = true): number { - const startAt = this._fileSize; - - fs.writeSync(this._fd, buffer, 0, buffer.length, startAt); - - if (withCache) { - this._cacheData(startAt, buffer); - } - - this._fileSize += buffer.length; - - return startAt; - } - - private _writeNewBuffers (buffers: Array): number { - let bufferAt = this._fileSize; - - buffers.forEach((buffer) => { - this._cacheData(bufferAt, buffer); - bufferAt += buffer.length; - }); - - return this._writeNewBuffer(Buffer.concat(buffers), false); - } - - private assertOpen (open: boolean = true): void { - const test = open - ? this._fd !== -1 - : this._fd === -1; - - assert(test, `Expected ${open ? 'an open' : 'a closed'} database`); - } - - private _open (file: string, startEmpty: boolean = false): number { - const isExisting = fs.existsSync(file); - - if (!isExisting || startEmpty) { - if (isExisting) { - fs.renameSync(file, `${file}.${Date.now()}`); - } - - fs.writeFileSync(file, Buffer.alloc(defaults.BRANCH_SIZE)); - } - - return fs.openSync(file, 'r+'); - } } diff --git a/packages/trie-codec/src/NodeHeader.ts b/packages/trie-codec/src/NodeHeader.ts index 39599ee5a..d69dee9c6 100644 --- a/packages/trie-codec/src/NodeHeader.ts +++ b/packages/trie-codec/src/NodeHeader.ts @@ -85,40 +85,19 @@ export default class NodeHeader { : EMPTY_TRIE; if (!input || firstByte === EMPTY_TRIE) { - return [ - NODE_TYPE_NULL, - new Null() - ]; + return [NODE_TYPE_NULL, new Null()]; } else if (firstByte === BRANCH_NODE_NO_VALUE) { - return [ - NODE_TYPE_BRANCH, - new BranchHeader(false) - ]; + return [NODE_TYPE_BRANCH, new BranchHeader(false)]; } else if (firstByte === BRANCH_NODE_WITH_VALUE) { - return [ - NODE_TYPE_BRANCH, - new BranchHeader(true) - ]; + return [NODE_TYPE_BRANCH, new BranchHeader(true)]; } else if (firstByte >= EXTENSION_NODE_OFFSET && firstByte <= EXTENSION_NODE_SMALL_MAX) { - return [ - NODE_TYPE_EXT, - new ExtensionHeader(firstByte - EXTENSION_NODE_OFFSET) - ]; + return [NODE_TYPE_EXT, new ExtensionHeader(firstByte - EXTENSION_NODE_OFFSET)]; } else if (firstByte === EXTENSION_NODE_BIG) { - return [ - NODE_TYPE_EXT, - new ExtensionHeader(input[1] + EXTENSION_NODE_THRESHOLD) - ]; + return [NODE_TYPE_EXT, new ExtensionHeader(input[1] + EXTENSION_NODE_THRESHOLD)]; } else if (firstByte >= LEAF_NODE_OFFSET && firstByte <= LEAF_NODE_SMALL_MAX) { - return [ - NODE_TYPE_LEAF, - new LeafHeader(firstByte - LEAF_NODE_OFFSET) - ]; + return [NODE_TYPE_LEAF, new LeafHeader(firstByte - LEAF_NODE_OFFSET)]; } else if (firstByte === LEAF_NODE_BIG) { - return [ - NODE_TYPE_LEAF, - new LeafHeader(input[1] + LEAF_NODE_THRESHOLD) - ]; + return [NODE_TYPE_LEAF, new LeafHeader(input[1] + LEAF_NODE_THRESHOLD)]; } throw new Error('Unreachable'); @@ -158,9 +137,7 @@ export default class NodeHeader { const nodeType = this.nodeType; if (nodeType === NODE_TYPE_NULL) { - return new Uint8Array([ - EMPTY_TRIE - ]); + return new Uint8Array([EMPTY_TRIE]); } else if (nodeType === NODE_TYPE_BRANCH) { return new Uint8Array([ (this.value as BranchHeader).valueOf() === true @@ -174,10 +151,7 @@ export default class NodeHeader { return new Uint8Array([EXTENSION_NODE_OFFSET + nibbleCount]); } - return new Uint8Array([ - EXTENSION_NODE_BIG, - nibbleCount - EXTENSION_NODE_THRESHOLD - ]); + return new Uint8Array([EXTENSION_NODE_BIG, nibbleCount - EXTENSION_NODE_THRESHOLD]); } else if (nodeType === NODE_TYPE_LEAF) { const nibbleCount = (this.value as LeafHeader).toNumber(); @@ -185,10 +159,7 @@ export default class NodeHeader { return new Uint8Array([LEAF_NODE_OFFSET + nibbleCount]); } - return new Uint8Array([ - LEAF_NODE_BIG, - nibbleCount - LEAF_NODE_THRESHOLD - ]); + return new Uint8Array([LEAF_NODE_BIG, nibbleCount - LEAF_NODE_THRESHOLD]); } throw new Error('Unreachable'); diff --git a/packages/trie-codec/src/decode.ts b/packages/trie-codec/src/decode.ts index 98f6d93cd..5267a5f9e 100644 --- a/packages/trie-codec/src/decode.ts +++ b/packages/trie-codec/src/decode.ts @@ -13,71 +13,80 @@ const l = logger('trie/codec'); l.noop(); +function _decodeBranch (header: NodeHeader, input: Uint8Array): Array { + let offset = header.encodedLength; + const branch = header.value as BranchHeader; + const bitmap = input[offset] + (input[offset + 1] * 256); + let value: null | Uint8Array = null; + + offset += 2; + + if (branch.valueOf() === true) { + const [length, bytes] = compactStripLength(input.subarray(offset)); + + value = bytes; + offset += length; + } + + let cursor = 1; + + return [ + null, null, null, null, + null, null, null, null, + null, null, null, null, + null, null, null, null, + value + ].map((value, index) => { + let result: null | Uint8Array | [Uint8Array, Uint8Array] = value; + + if ((index < 16) && (bitmap & cursor)) { + const [length, bytes] = compactStripLength(input.subarray(offset)); + + result = bytes.length === 32 + ? bytes + : decode(bytes) as any; + offset += length; + } + + cursor = cursor << 1; + + return result; + }); +} + +function _decodeKv (header: NodeHeader, input: Uint8Array): Array { + let offset = header.encodedLength; + const nibbleCount = (header.value as NibbleHeader).toNumber(); + const nibbleLength = Math.floor((nibbleCount + 1) / 2); + const nibbleData = input.subarray(offset, offset + nibbleLength); + + // for odd, ignore the first nibble, data starts at offset 1 + const nibbles = toNibbles(nibbleData).subarray(nibbleCount % 2); + + offset += nibbleData.length; + + const [, value] = compactStripLength(input.subarray(offset)); + + return [ + encodeNibbles( + header.nodeType === NODE_TYPE_LEAF + ? addNibblesTerminator(nibbles) + : nibbles + ), + value + ]; +} + function _decode (input: null | Uint8Array): Uint8Array | null | Array { const header = new NodeHeader(input); const nodeType = header.nodeType; - let offset = header.encodedLength; if (!input || nodeType === NODE_TYPE_NULL) { return input; } else if (nodeType === NODE_TYPE_BRANCH) { - const branch = header.value as BranchHeader; - const bitmap = input[offset] + (input[offset + 1] * 256); - let value: null | Uint8Array = null; - - offset += 2; - - if (branch.valueOf() === true) { - const [length, bytes] = compactStripLength(input.subarray(offset)); - - value = bytes; - offset += length; - } - - let cursor = 1; - - return [ - null, null, null, null, - null, null, null, null, - null, null, null, null, - null, null, null, null, - value - ].map((value, index) => { - let result: null | Uint8Array | [Uint8Array, Uint8Array] = value; - - if ((index < 16) && (bitmap & cursor)) { - const [length, bytes] = compactStripLength(input.subarray(offset)); - - result = bytes.length === 32 - ? bytes - : decode(bytes) as any; - offset += length; - } - - cursor = cursor << 1; - - return result; - }); + return _decodeBranch(header, input); } else if (nodeType === NODE_TYPE_EXT || nodeType === NODE_TYPE_LEAF) { - const nibbleCount = (header.value as NibbleHeader).toNumber(); - const nibbleLength = Math.floor((nibbleCount + 1) / 2); - const nibbleData = input.subarray(offset, offset + nibbleLength); - - // for odd, ignore the first nibble, data starts at offset 1 - const nibbles = toNibbles(nibbleData).subarray(nibbleCount % 2); - - offset += nibbleData.length; - - const [, value] = compactStripLength(input.subarray(offset)); - - return [ - encodeNibbles( - nodeType === NODE_TYPE_LEAF - ? addNibblesTerminator(nibbles) - : nibbles - ), - value - ]; + return _decodeKv(header, input); } throw new Error('Unreachable'); diff --git a/packages/trie-codec/src/encode.ts b/packages/trie-codec/src/encode.ts index dd4c9e4f5..0e488ca51 100644 --- a/packages/trie-codec/src/encode.ts +++ b/packages/trie-codec/src/encode.ts @@ -44,6 +44,41 @@ function encodeValue (input: null | Uint8Array | Array): Uint return compactAddLength(encoded); } +function _encodeBranch (header: NodeHeader, input: Array): Uint8Array { + let valuesU8a = EMPTY; + let bitmap = 0; + + input.reduce((cursor, value, index) => { + if ((index < BRANCH_VALUE_INDEX) && value) { + bitmap = bitmap | cursor; + + valuesU8a = u8aConcat( + valuesU8a, + encodeValue(value) + ); + } + + return cursor << 1; + }, 1); + + return u8aConcat( + header.toU8a(), + new Uint8Array([(bitmap % 256), Math.floor(bitmap / 256)]), + encodeValue(input[BRANCH_VALUE_INDEX]), + valuesU8a + ); +} + +function _encodeKv (header: NodeHeader, input: Array): Uint8Array { + const [key, value] = input; + + return u8aConcat( + header.toU8a(), + encodeKey(key), + encodeValue(value) + ); +} + function _encode (input?: null | Uint8Array | Array): Uint8Array { if (isU8a(input)) { return input; @@ -51,41 +86,13 @@ function _encode (input?: null | Uint8Array | Array): Uint8Ar const header = new NodeHeader(input); const nodeType = header.nodeType; - const u8aHeader = header.toU8a(); if (!input || nodeType === NODE_TYPE_NULL) { - return u8aHeader; + return header.toU8a(); } else if (nodeType === NODE_TYPE_BRANCH) { - let valuesU8a = EMPTY; - let bitmap = 0; - - input.reduce((cursor, value, index) => { - if ((index < BRANCH_VALUE_INDEX) && value) { - bitmap = bitmap | cursor; - - valuesU8a = u8aConcat( - valuesU8a, - encodeValue(value) - ); - } - - return cursor << 1; - }, 1); - - return u8aConcat( - u8aHeader, - new Uint8Array([(bitmap % 256), Math.floor(bitmap / 256)]), - encodeValue(input[BRANCH_VALUE_INDEX]), - valuesU8a - ); + return _encodeBranch(header, input); } else if (nodeType === NODE_TYPE_EXT || nodeType === NODE_TYPE_LEAF) { - const [key, value] = input; - - return u8aConcat( - u8aHeader, - encodeKey(key), - encodeValue(value) - ); + return _encodeKv(header, input); } throw new Error('Unreachable'); diff --git a/packages/trie-db/src/Impl.ts b/packages/trie-db/src/Impl.ts new file mode 100644 index 000000000..52081febd --- /dev/null +++ b/packages/trie-db/src/Impl.ts @@ -0,0 +1,419 @@ +// Copyright 2017-2018 @polkadot/trie-db authors & contributors +// This software may be modified and distributed under the terms +// of the Apache-2.0 license. See the LICENSE file for details. + +import { TxDb, ProgressCb } from '@polkadot/db/types'; +import { Codec } from '@polkadot/trie-codec/types'; +import { TrieDb, Node, NodeBranch, NodeEncodedOrEmpty, NodeKv, NodeNotEmpty, NodeType } from './types'; + +import substrateCodec from '@polkadot/trie-codec/index'; +import { decodeNibbles, encodeNibbles, extractNodeKey } from '@polkadot/trie-codec/nibbles'; +import { isNull, logger , u8aConcat, u8aToHex } from '@polkadot/util/index'; + +import { isBranchNode, isEmptyNode, isExtensionNode, isKvNode, isLeafNode } from './util/is'; +import { keyEquals, keyStartsWith, computeExtensionKey, computeLeafKey, consumeCommonPrefix } from './util/key'; +import { getNodeType, decodeNode, encodeNode } from './util/node'; +import constants, { Constants } from './constants'; + +const l = logger('trie/db'); + +/** + * # @polkadot/common/trie-db + * + * ## Overview + * + * @name Trie + * @summary Re-implementation of a Patricia Trie + * @example See [Polkadot-JS Common Trie-DB Examples](https://polkadot.js.org/api/common/examples/trie-db/) + */ +export default class Impl { + readonly db: TxDb; + protected codec: Codec; + protected constants: Constants; + protected txRoot: Uint8Array; + protected rootHash: Uint8Array; + + constructor (db: TxDb, rootHash?: Uint8Array, codec: Codec = substrateCodec) { + this.db = db; + this.codec = codec; + this.constants = constants(codec); + this.rootHash = rootHash || this.constants.EMPTY_HASH; + this.txRoot = this.rootHash; + + l.log(`Created with ${codec.type} codec, root ${u8aToHex(this.rootHash, 64)}`); + } + + protected createCheckpoint (): Uint8Array { + this.txRoot = this.rootHash; + + return this.txRoot; + } + + protected commitCheckpoint (): Uint8Array { + return this.rootHash; + } + + protected revertCheckpoint (): Uint8Array { + this.rootHash = this.txRoot; + + return this.rootHash; + } + + protected _snapshot (dest: TrieDb, fn: ProgressCb, root: Uint8Array, keys: number, percent: number, depth: number): number { + // l.debug(() => ['snapshot', { root }]); + + const node = this._getNode(root); + + if (isNull(node)) { + return keys; + } + + keys++; + dest.db.put(root, encodeNode(this.codec, node)); + fn({ keys, percent }); + + node.forEach((u8a) => { + if (u8a && u8a.length === 32) { + keys = this._snapshot(dest, fn, u8a, keys, percent, depth + 1); + } + + percent += (100 / node.length) / Math.pow(16, depth); + }); + + return keys; + } + + protected _getNode (hash: Uint8Array | null): Node { + // l.debug(() => ['_getNode', { hash }]); + + if (!hash || hash.length === 0 || keyEquals(hash, this.constants.EMPTY_HASH)) { + return null; + } else if (hash.length < 32) { + return decodeNode(this.codec, hash); + } + + return decodeNode(this.codec, this.db.get(hash)); + } + + protected _del (node: Node, trieKey: Uint8Array): Node { + // l.debug(() => ['_del', { node, trieKey }]); + + if (isEmptyNode(node)) { + return null; + } else if (isBranchNode(node)) { + return this._delBranchNode(node, trieKey); + } else if (isKvNode(node)) { + return this._delKvNode(node, trieKey); + } + + throw new Error('Unreachable'); + } + + protected _delBranchNode (node: NodeBranch, trieKey: Uint8Array): Node { + // l.debug(() => ['_delBranchNode', { node, trieKey }]); + + if (trieKey.length === 0) { + node[node.length - 1] = null; + + return this._normaliseBranchNode(node); + } + + const nodeToDelete = this._getNode(node[trieKey[0]]); + const subNode = this._del(nodeToDelete, trieKey.subarray(1)); + const encodedSubNode = this._persistNode(subNode); + + if (keyEquals(encodedSubNode, node[trieKey[0]])) { + return node; + } + + node[trieKey[0]] = encodedSubNode; + + if (isNull(encodedSubNode)) { + return this._normaliseBranchNode(node); + } + + return node; + } + + protected _delKvNode (node: NodeNotEmpty, trieKey: Uint8Array): Node { + // l.debug(() => ['_delKvNode', { node, trieKey }]); + + const currentKey = extractNodeKey(node); + const nodeType = getNodeType(node); + + if (!keyStartsWith(trieKey, currentKey)) { + return node; + } + + if (nodeType === NodeType.LEAF) { + return keyEquals(trieKey, currentKey) + ? null + : node; + } + + const subKey = trieKey.subarray(currentKey.length); + const subNode = this._getNode(node[1]); + const newSub = this._del(subNode, subKey); + const encodedNewSub = this._persistNode(newSub); + + if (keyEquals(encodedNewSub, node[1])) { + return node; + } else if (isNull(newSub)) { + return null; + } + + if (isKvNode(newSub)) { + const subNibbles = decodeNibbles(newSub[0]); + const newKey = u8aConcat(currentKey, subNibbles); + + return [encodeNibbles(newKey), newSub[1]]; + } else if (isBranchNode(newSub)) { + return [encodeNibbles(currentKey), encodedNewSub]; + } + + throw new Error('Unreachable'); + } + + protected _get (node: Node, trieKey: Uint8Array): NodeEncodedOrEmpty { + // l.debug(() => ['_get', { node, trieKey }]); + + if (isEmptyNode(node)) { + return null; + } else if (isBranchNode(node)) { + return this._getBranchNode(node, trieKey); + } else if (isKvNode(node)) { + return this._getKvNode(node, trieKey); + } + + throw new Error('Invalid NodeType'); + } + + protected _getBranchNode (node: NodeBranch, trieKey: Uint8Array): NodeEncodedOrEmpty { + // l.debug(() => ['_getBranchNode', { node, trieKey }]); + + if (trieKey.length === 0) { + return node[16]; + } + + const subNode = this._getNode(node[trieKey[0]]); + + return this._get(subNode, trieKey.subarray(1)); + } + + protected _getKvNode (node: NodeKv, trieKey: Uint8Array): NodeEncodedOrEmpty { + // l.debug(() => ['_getKvNode', { node, trieKey }]); + + const currentKey = extractNodeKey(node); + const nodeType = getNodeType(node); + + // l.debug(() => [{ currentKey, trieKey, nodeType }]); + + if (nodeType === NodeType.LEAF) { + if (keyEquals(trieKey, currentKey)) { + return node[1]; + } + + return null; + } else if (nodeType === NodeType.EXTENSION) { + if (keyStartsWith(trieKey, currentKey)) { + const subNode = this._getNode(node[1]); + + return this._get(subNode, trieKey.subarray(currentKey ? currentKey.length : 0)); + } + + return null; + } + + throw new Error('Unreachable'); + } + + protected _nodeToDbMapping (node: Node): NodeNotEmpty { + // l.debug(() => ['_nodeToDbMapping', { node }]); + + if (isEmptyNode(node)) { + return [null, null]; + } + + const encoded = encodeNode(this.codec, node); + + if (encoded.length < 32) { + // FIXME typings... + // @ts-ignore Ok, this is not correct - this comes back as an embedded node, which mean that our definitions here are completely wrong (or what we think it is, does not align with what it should be as per the spec and implementation) + return [node, null]; + } + + return [this.codec.hashing(encoded), encoded]; + } + + protected _normaliseBranchNode (node: NodeNotEmpty): Node { + // l.debug(() => ['_normaliseBranchNode', { node }]); + + const mapped = node + .map((value, index) => ({ index, value })) + .filter(({ value }) => !!value && value.length !== 0); + + if (mapped.length >= 2) { + return node; + } + + if (node[16]) { + return [computeLeafKey(new Uint8Array()), node[16]]; + } + + const [{ index, value }] = mapped; + const subNode = this._getNode(value); + + if (isBranchNode(subNode)) { + return [encodeNibbles(new Uint8Array([index])), this._persistNode(subNode)]; + } else if (isKvNode(subNode)) { + const subNibbles = decodeNibbles(subNode[0]); + const newKey = u8aConcat(new Uint8Array([index]), subNibbles); + + return [encodeNibbles(newKey), subNode[1]]; + } + + throw new Error('Unreachable'); + } + + protected _persistNode (node: Node): NodeEncodedOrEmpty { + const [key, value] = this._nodeToDbMapping(node); + + // l.debug(() => ['_persistNode', { node, key, value }]); + + if (value) { + this.db.put(key as Uint8Array, value); + } + + return key; + } + + protected _put (node: Node, trieKey: Uint8Array, value: Uint8Array): NodeNotEmpty { + // l.debug(() => ['_put', { node, trieKey, value }]); + + if (isEmptyNode(node)) { + return [ + computeLeafKey(trieKey), + value + ]; + } else if (isKvNode(node)) { + return this._putKvNode(node, trieKey, value); + } else if (isBranchNode(node)) { + return this._putBranchNode(node, trieKey, value); + } + + throw new Error('Unreachable'); + } + + protected _putBranchNode (node: NodeBranch, trieKey: Uint8Array, value: Uint8Array): NodeNotEmpty { + // l.debug(() => ['_putBranchNode', { node, trieKey, value }]); + + if (trieKey && trieKey.length) { + const subNode = this._getNode(node[trieKey[0]]); + const newNode = this._put(subNode, trieKey.subarray(1), value); + + node[trieKey[0]] = this._persistNode(newNode); + } else { + node[node.length - 1] = value; + } + + return node; + } + + protected _putKvNode (node: NodeKv, trieKey: Uint8Array, value: Uint8Array): NodeNotEmpty { + // l.debug(() => ['_putKvNode', { node, trieKey, value }]); + + const currentKey = extractNodeKey(node); + const [commonPrefix, currentRemainder, trieRemainder] = consumeCommonPrefix(currentKey, trieKey); + const isExtension = isExtensionNode(node); + const isLeaf = isLeafNode(node); + let newNode: NodeNotEmpty; + + // l.debug(() => ['_putKvNode', { currentKey, commonPrefix, currentRemainder, trieRemainder }]); + + if (currentRemainder.length === 0 && trieRemainder.length === 0) { + if (isLeaf) { + return [ + node[0], + value + ]; + } + + const subNode = this._getNode(node[1]); + + newNode = this._put(subNode, trieRemainder, value); + } else if (currentRemainder.length === 0) { + if (isExtension) { + const subNode = this._getNode(node[1]); + + newNode = this._put(subNode, trieRemainder, value); + } else { + const subPosition = trieRemainder[0]; + const subKey = computeLeafKey(trieRemainder.subarray(1)); + const subNode: NodeKv = [subKey, value]; + + newNode = [ + null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, + node[1] + ]; + newNode[subPosition] = this._persistNode(subNode); + } + } else { + newNode = [ + null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, + null + ]; + + if (currentRemainder.length === 1 && isExtension) { + newNode[currentRemainder[0]] = node[1]; + } else { + const computedKey = isExtension + ? computeExtensionKey(currentRemainder.subarray(1)) + : computeLeafKey(currentRemainder.subarray(1)); + + newNode[currentRemainder[0]] = this._persistNode([ + computedKey, + node[1] + ]); + } + + if (trieRemainder.length) { + newNode[trieRemainder[0]] = this._persistNode([ + computeLeafKey(trieRemainder.subarray(1)), + value + ]); + } else { + newNode[16] = value; + } + } + + // l.debug(() => ['newNode', newNode]); + + if (commonPrefix.length) { + return [ + computeExtensionKey(commonPrefix), + this._persistNode(newNode) + ]; + } + + return newNode; + } + + protected _setRootNode (node: Node): void { + // l.debug(() => ['_setRootNode', { node }]); + + if (isEmptyNode(node)) { + this.rootHash = this.constants.EMPTY_HASH; + } else { + const encoded = encodeNode(this.codec, node); + const rootHash = this.codec.hashing(encoded); + + // l.debug(() => ['_setRootNode', { encoded, rootHash }]); + + this.db.put(rootHash, encoded); + + this.rootHash = rootHash; + } + } +} diff --git a/packages/trie-db/src/index.ts b/packages/trie-db/src/index.ts index 937371b4d..69057ff27 100644 --- a/packages/trie-db/src/index.ts +++ b/packages/trie-db/src/index.ts @@ -4,18 +4,14 @@ import { TxDb, ProgressCb } from '@polkadot/db/types'; import { Codec } from '@polkadot/trie-codec/types'; -import { TrieDb, Node, NodeBranch, NodeEncodedOrEmpty, NodeKv, NodeNotEmpty, NodeType } from './types'; +import { TrieDb, Node } from './types'; import MemoryDb from '@polkadot/db/Memory'; -import substrateCodec from '@polkadot/trie-codec/index'; -import { decodeNibbles, encodeNibbles, extractNodeKey } from '@polkadot/trie-codec/nibbles'; import { toNibbles } from '@polkadot/trie-codec/util'; -import { isNull, logger , u8aConcat, u8aToHex } from '@polkadot/util/index'; +import { isNull, logger, u8aToHex } from '@polkadot/util/index'; -import { isBranchNode, isEmptyNode, isExtensionNode, isKvNode, isLeafNode } from './util/is'; -import { keyEquals, keyStartsWith, computeExtensionKey, computeLeafKey, consumeCommonPrefix } from './util/key'; -import { getNodeType, decodeNode, encodeNode } from './util/node'; -import constants, { Constants } from './constants'; +import Impl from './Impl'; +import constants from './constants'; const l = logger('trie/db'); @@ -28,37 +24,11 @@ const l = logger('trie/db'); * @summary Re-implementation of a Patricia Trie * @example See [Polkadot-JS Common Trie-DB Examples](https://polkadot.js.org/api/common/examples/trie-db/) */ -export default class Trie implements TrieDb { - readonly db: TxDb; - private codec: Codec; - private constants: Constants; - private txRoot: Uint8Array; - private rootHash: Uint8Array; +export default class Trie extends Impl implements TrieDb { + constructor (db: TxDb = new MemoryDb(), rootHash?: Uint8Array, codec?: Codec) { + super(db, rootHash, codec); - constructor (db: TxDb = new MemoryDb(), rootHash?: Uint8Array, codec: Codec = substrateCodec) { - this.db = db; - this.codec = codec; - this.constants = constants(codec); - this.rootHash = rootHash || this.constants.EMPTY_HASH; - this.txRoot = this.rootHash; - - l.log(`Created with ${codec.type} codec, root ${u8aToHex(this.rootHash, 64)}`); - } - - private createCheckpoint (): Uint8Array { - this.txRoot = this.rootHash; - - return this.txRoot; - } - - private commitCheckpoint (): Uint8Array { - return this.rootHash; - } - - private revertCheckpoint (): Uint8Array { - this.rootHash = this.txRoot; - - return this.rootHash; + l.log(`Created with ${this.codec.type} codec, root ${u8aToHex(this.rootHash, 64)}`); } transaction (fn: () => boolean): boolean { @@ -184,391 +154,4 @@ export default class Trie implements TrieDb { return keys; } - - private _snapshot (dest: TrieDb, fn: ProgressCb, root: Uint8Array, keys: number, percent: number, depth: number): number { - // l.debug(() => ['snapshot', { root }]); - - const node = this._getNode(root); - - if (isNull(node)) { - return keys; - } - - keys++; - dest.db.put(root, encodeNode(this.codec, node)); - fn({ keys, percent }); - - node.forEach((u8a) => { - if (u8a && u8a.length === 32) { - keys = this._snapshot(dest, fn, u8a, keys, percent, depth + 1); - } - - percent += (100 / node.length) / Math.pow(16, depth); - }); - - return keys; - } - - private _getNode (hash: Uint8Array | null): Node { - // l.debug(() => ['_getNode', { hash }]); - - if (!hash || hash.length === 0 || keyEquals(hash, this.constants.EMPTY_HASH)) { - return null; - } else if (hash.length < 32) { - return decodeNode(this.codec, hash); - } - - return decodeNode(this.codec, this.db.get(hash)); - } - - private _del (node: Node, trieKey: Uint8Array): Node { - // l.debug(() => ['_del', { node, trieKey }]); - - if (isEmptyNode(node)) { - return null; - } else if (isBranchNode(node)) { - return this._delBranchNode(node, trieKey); - } else if (isKvNode(node)) { - return this._delKvNode(node, trieKey); - } - - throw new Error('Unreachable'); - } - - private _delBranchNode (node: NodeBranch, trieKey: Uint8Array): Node { - // l.debug(() => ['_delBranchNode', { node, trieKey }]); - - if (trieKey.length === 0) { - node[node.length - 1] = null; - - return this._normaliseBranchNode(node); - } - - const nodeToDelete = this._getNode(node[trieKey[0]]); - const subNode = this._del(nodeToDelete, trieKey.subarray(1)); - const encodedSubNode = this._persistNode(subNode); - - if (keyEquals(encodedSubNode, node[trieKey[0]])) { - return node; - } - - node[trieKey[0]] = encodedSubNode; - - if (isNull(encodedSubNode)) { - return this._normaliseBranchNode(node); - } - - return node; - } - - private _delKvNode (node: NodeNotEmpty, trieKey: Uint8Array): Node { - // l.debug(() => ['_delKvNode', { node, trieKey }]); - - const currentKey = extractNodeKey(node); - const nodeType = getNodeType(node); - - if (!keyStartsWith(trieKey, currentKey)) { - return node; - } - - if (nodeType === NodeType.LEAF) { - return keyEquals(trieKey, currentKey) - ? null - : node; - } - - const subKey = trieKey.subarray(currentKey.length); - const subNode = this._getNode(node[1]); - const newSub = this._del(subNode, subKey); - const encodedNewSub = this._persistNode(newSub); - - if (keyEquals(encodedNewSub, node[1])) { - return node; - } else if (isNull(newSub)) { - return null; - } - - if (isKvNode(newSub)) { - const subNibbles = decodeNibbles(newSub[0]); - const newKey = u8aConcat(currentKey, subNibbles); - - return [ - encodeNibbles(newKey), - newSub[1] - ]; - } else if (isBranchNode(newSub)) { - return [ - encodeNibbles(currentKey), - encodedNewSub - ]; - } - - throw new Error('Unreachable'); - } - - private _get (node: Node, trieKey: Uint8Array): NodeEncodedOrEmpty { - // l.debug(() => ['_get', { node, trieKey }]); - - if (isEmptyNode(node)) { - return null; - } else if (isBranchNode(node)) { - return this._getBranchNode(node, trieKey); - } else if (isKvNode(node)) { - return this._getKvNode(node, trieKey); - } - - throw new Error('Invalid NodeType'); - } - - private _getBranchNode (node: NodeBranch, trieKey: Uint8Array): NodeEncodedOrEmpty { - // l.debug(() => ['_getBranchNode', { node, trieKey }]); - - if (trieKey.length === 0) { - return node[16]; - } - - const subNode = this._getNode(node[trieKey[0]]); - - return this._get(subNode, trieKey.subarray(1)); - } - - private _getKvNode (node: NodeKv, trieKey: Uint8Array): NodeEncodedOrEmpty { - // l.debug(() => ['_getKvNode', { node, trieKey }]); - - const currentKey = extractNodeKey(node); - const nodeType = getNodeType(node); - - // l.debug(() => [{ currentKey, trieKey, nodeType }]); - - if (nodeType === NodeType.LEAF) { - if (keyEquals(trieKey, currentKey)) { - return node[1]; - } - - return null; - } else if (nodeType === NodeType.EXTENSION) { - if (keyStartsWith(trieKey, currentKey)) { - const subNode = this._getNode(node[1]); - - return this._get(subNode, trieKey.subarray(currentKey ? currentKey.length : 0)); - } - - return null; - } - - throw new Error('Unreachable'); - } - - private _nodeToDbMapping (node: Node): NodeNotEmpty { - // l.debug(() => ['_nodeToDbMapping', { node }]); - - if (isEmptyNode(node)) { - return [ - null, - null - ]; - } - - const encoded = encodeNode(this.codec, node); - - if (encoded.length < 32) { - return [ - // FIXME typings... - // @ts-ignore Ok, this is not correct - this comes back as an embedded node, which mean that our definitions here are completely wrong (or what we think it is, does not align with what it should be as per the spec and implementation) - node, - null - ]; - } - - return [ - this.codec.hashing(encoded), - encoded - ]; - } - - private _normaliseBranchNode (node: NodeNotEmpty): Node { - // l.debug(() => ['_normaliseBranchNode', { node }]); - - const mapped = node - .map((value, index) => ({ - index, - value - })) - .filter(({ value }) => - !!value && value.length !== 0 - ); - - if (mapped.length >= 2) { - return node; - } - - if (node[16]) { - return [ - computeLeafKey(new Uint8Array()), - node[16] - ]; - } - - const [{ index, value }] = mapped; - const subNode = this._getNode(value); - - if (isBranchNode(subNode)) { - return [ - encodeNibbles(new Uint8Array([index])), - this._persistNode(subNode) - ]; - } else if (isKvNode(subNode)) { - const subNibbles = decodeNibbles(subNode[0]); - const newKey = u8aConcat(new Uint8Array([index]), subNibbles); - - return [ - encodeNibbles(newKey), - subNode[1] - ]; - } - - throw new Error('Unreachable'); - } - - private _persistNode (node: Node): NodeEncodedOrEmpty { - const [key, value] = this._nodeToDbMapping(node); - - // l.debug(() => ['_persistNode', { node, key, value }]); - - if (value) { - this.db.put(key as Uint8Array, value); - } - - return key; - } - - private _put (node: Node, trieKey: Uint8Array, value: Uint8Array): NodeNotEmpty { - // l.debug(() => ['_put', { node, trieKey, value }]); - - if (isEmptyNode(node)) { - return [ - computeLeafKey(trieKey), - value - ]; - } else if (isKvNode(node)) { - return this._putKvNode(node, trieKey, value); - } else if (isBranchNode(node)) { - return this._putBranchNode(node, trieKey, value); - } - - throw new Error('Unreachable'); - } - - private _putBranchNode (node: NodeBranch, trieKey: Uint8Array, value: Uint8Array): NodeNotEmpty { - // l.debug(() => ['_putBranchNode', { node, trieKey, value }]); - - if (trieKey && trieKey.length) { - const subNode = this._getNode(node[trieKey[0]]); - const newNode = this._put(subNode, trieKey.subarray(1), value); - - node[trieKey[0]] = this._persistNode(newNode); - } else { - node[node.length - 1] = value; - } - - return node; - } - - private _putKvNode (node: NodeKv, trieKey: Uint8Array, value: Uint8Array): NodeNotEmpty { - // l.debug(() => ['_putKvNode', { node, trieKey, value }]); - - const currentKey = extractNodeKey(node); - const [commonPrefix, currentRemainder, trieRemainder] = consumeCommonPrefix(currentKey, trieKey); - const isExtension = isExtensionNode(node); - const isLeaf = isLeafNode(node); - let newNode: NodeNotEmpty; - - // l.debug(() => ['_putKvNode', { currentKey, commonPrefix, currentRemainder, trieRemainder }]); - - if (currentRemainder.length === 0 && trieRemainder.length === 0) { - if (isLeaf) { - return [ - node[0], - value - ]; - } - - const subNode = this._getNode(node[1]); - - newNode = this._put(subNode, trieRemainder, value); - } else if (currentRemainder.length === 0) { - if (isExtension) { - const subNode = this._getNode(node[1]); - - newNode = this._put(subNode, trieRemainder, value); - } else { - const subPosition = trieRemainder[0]; - const subKey = computeLeafKey(trieRemainder.subarray(1)); - const subNode: NodeKv = [subKey, value]; - - newNode = [ - null, null, null, null, null, null, null, null, - null, null, null, null, null, null, null, null, - node[1] - ]; - newNode[subPosition] = this._persistNode(subNode); - } - } else { - newNode = [ - null, null, null, null, null, null, null, null, - null, null, null, null, null, null, null, null, - null - ]; - - if (currentRemainder.length === 1 && isExtension) { - newNode[currentRemainder[0]] = node[1]; - } else { - const computedKey = isExtension - ? computeExtensionKey(currentRemainder.subarray(1)) - : computeLeafKey(currentRemainder.subarray(1)); - - newNode[currentRemainder[0]] = this._persistNode([ - computedKey, - node[1] - ]); - } - - if (trieRemainder.length) { - newNode[trieRemainder[0]] = this._persistNode([ - computeLeafKey(trieRemainder.subarray(1)), - value - ]); - } else { - newNode[16] = value; - } - } - - // l.debug(() => ['newNode', newNode]); - - if (commonPrefix.length) { - return [ - computeExtensionKey(commonPrefix), - this._persistNode(newNode) - ]; - } - - return newNode; - } - - private _setRootNode (node: Node): void { - // l.debug(() => ['_setRootNode', { node }]); - - if (isEmptyNode(node)) { - this.rootHash = this.constants.EMPTY_HASH; - } else { - const encoded = encodeNode(this.codec, node); - const rootHash = this.codec.hashing(encoded); - - // l.debug(() => ['_setRootNode', { encoded, rootHash }]); - - this.db.put(rootHash, encoded); - - this.rootHash = rootHash; - } - } } diff --git a/packages/trie-hash/test/data.ts b/packages/trie-hash/test/data.ts index d8e5ef9de..b86eeb385 100644 --- a/packages/trie-hash/test/data.ts +++ b/packages/trie-hash/test/data.ts @@ -17,14 +17,8 @@ const data: { [index: string]: TestData } = { simpleBranch: { desc: 'a branch', input: [ - { - k: Uint8Array.from([0xaa]), - v: Uint8Array.from([0x10]) - }, - { - k: Uint8Array.from([0xba]), - v: Uint8Array.from([0x11]) - } + { k: Uint8Array.from([0xaa]), v: Uint8Array.from([0x10]) }, + { k: Uint8Array.from([0xba]), v: Uint8Array.from([0x11]) } ], output: Uint8Array.from([ 0xfe, 0x00, 0x0c, 0x10, 0x02, 0x0a, 0x04, 0x10, 0x10, 0x02, 0x0a, 0x04, 0x11 @@ -34,14 +28,8 @@ const data: { [index: string]: TestData } = { extensionBranch: { desc: 'extension and branch', input: [ - { - k: Uint8Array.from([0xaa]), - v: Uint8Array.from([0x10]) - }, - { - k: Uint8Array.from([0xab]), - v: Uint8Array.from([0x11]) - } + { k: Uint8Array.from([0xaa]), v: Uint8Array.from([0x10]) }, + { k: Uint8Array.from([0xab]), v: Uint8Array.from([0x11]) } ], output: hexToU8a('0x810a2cfe000c0c0104100c010411'), root: hexToU8a('0x3f6764bc812ca86b84cf88b2045828fd605c8abbe1e9c657ff87adb715c644bf') @@ -49,18 +37,9 @@ const data: { [index: string]: TestData } = { extensionBranchValue: { desc: 'extension and branch with value', input: [ - { - k: Uint8Array.from([0xaa]), - v: Uint8Array.from([0xa0]) - }, - { - k: Uint8Array.from([0xaa, 0xaa]), - v: Uint8Array.from([0xaa]) - }, - { - k: Uint8Array.from([0xaa, 0xbb]), - v: Uint8Array.from([0xab]) - } + { k: Uint8Array.from([0xaa]), v: Uint8Array.from([0xa0]) }, + { k: Uint8Array.from([0xaa, 0xaa]), v: Uint8Array.from([0xaa]) }, + { k: Uint8Array.from([0xaa, 0xbb]), v: Uint8Array.from([0xab]) } ], output: Uint8Array.from([ 0x82, 0xaa, 0x3c, 0xff, 0x00, 0x0c, 0x04, 0xa0, 0x10, 0x02, 0x0a, 0x04, 0xaa, 0x10, 0x02, 0x0b, @@ -71,22 +50,10 @@ const data: { [index: string]: TestData } = { midExtensionBranchValue: { desc: 'medium extension and branch with value', input: [ - { - k: Uint8Array.from([0xaa]), - v: Uint8Array.from([0xa0]) - }, - { - k: Uint8Array.from([0xaa, 0xaa]), - v: Uint8Array.from([0xaa]) - }, - { - k: Uint8Array.from([0xaa, 0xbb]), - v: Uint8Array.from([0xab]) - }, - { - k: Uint8Array.from([0xbb]), - v: Uint8Array.from([0xb0]) - } + { k: Uint8Array.from([0xaa]), v: Uint8Array.from([0xa0]) }, + { k: Uint8Array.from([0xaa, 0xaa]), v: Uint8Array.from([0xaa]) }, + { k: Uint8Array.from([0xaa, 0xbb]), v: Uint8Array.from([0xab]) }, + { k: Uint8Array.from([0xbb]), v: Uint8Array.from([0xb0]) } ], output: Uint8Array.from([ 0xfe, 0x00, 0x0c, 0x48, 0x81, 0x0a, 0x3c, 0xff, 0x00, 0x0c, 0x04, 0xa0, 0x10, 0x02, 0x0a, 0x04, @@ -97,30 +64,12 @@ const data: { [index: string]: TestData } = { bigExtensionBranchValue: { desc: 'bigger extension and branch with value', input: [ - { - k: Uint8Array.from([0xaa]), - v: Uint8Array.from([0xa0]) - }, - { - k: Uint8Array.from([0xaa, 0xaa]), - v: Uint8Array.from([0xaa]) - }, - { - k: Uint8Array.from([0xaa, 0xbb]), - v: Uint8Array.from([0xab]) - }, - { - k: Uint8Array.from([0xbb]), - v: Uint8Array.from([0xb0]) - }, - { - k: Uint8Array.from([0xbb, 0xbb]), - v: Uint8Array.from([0xbb]) - }, - { - k: Uint8Array.from([0xbb, 0xcc]), - v: Uint8Array.from([0xbc]) - } + { k: Uint8Array.from([0xaa]), v: Uint8Array.from([0xa0]) }, + { k: Uint8Array.from([0xaa, 0xaa]), v: Uint8Array.from([0xaa]) }, + { k: Uint8Array.from([0xaa, 0xbb]), v: Uint8Array.from([0xab]) }, + { k: Uint8Array.from([0xbb]), v: Uint8Array.from([0xb0]) }, + { k: Uint8Array.from([0xbb, 0xbb]), v: Uint8Array.from([0xbb]) }, + { k: Uint8Array.from([0xbb, 0xcc]), v: Uint8Array.from([0xbc]) } ], output: Uint8Array.from([ 0xfe, 0x00, 0x0c, 0x48, 0x81, 0x0a, 0x3c, 0xff, 0x00, 0x0c, 0x04, 0xa0, 0x10, 0x02, 0x0a, 0x04, @@ -172,18 +121,13 @@ const data: { [index: string]: TestData } = { empty: { desc: 'empty into [0]', input: [], - output: Uint8Array.from([ - 0 - ]), + output: Uint8Array.from([0]), root: hexToU8a('0x03170a2e7597b7b7e3d84c05391d139a62b157e78786d8c082f29dcf4c111314') }, singleValue: { desc: 'a single tuple', input: [ - { - k: Uint8Array.from([0xaa]), - v: Uint8Array.from([0xbb]) - } + { k: Uint8Array.from([0xaa]), v: Uint8Array.from([0xbb]) } ], output: Uint8Array.from([ 0x03, // leaf (0x01) with (+) key of 2 nibbles (0x02) @@ -196,10 +140,7 @@ const data: { [index: string]: TestData } = { singleValue2: { desc: 'a single tuple (prepare for next)', input: [ - { - k: Uint8Array.from([0x48, 0x19]), - v: Uint8Array.from([0xfe]) - } + { k: Uint8Array.from([0x48, 0x19]), v: Uint8Array.from([0xfe]) } ], output: Uint8Array.from([ 0x05, 0x48, 0x19, 0x04, 0xfe @@ -209,14 +150,8 @@ const data: { [index: string]: TestData } = { twoValues: { desc: '2 disjointed tuple keys', input: [ - { - k: Uint8Array.from([0x48, 0x19]), - v: Uint8Array.from([0xfe]) - }, - { - k: Uint8Array.from([0x13, 0x14]), - v: Uint8Array.from([0xff]) - } + { k: Uint8Array.from([0x48, 0x19]), v: Uint8Array.from([0xfe]) }, + { k: Uint8Array.from([0x13, 0x14]), v: Uint8Array.from([0xff]) } ], output: Uint8Array.from([ 0xfe, // branch, no value