Re-implementation of Patricia-Merkle Trie (#203)
* Start of trie-db re-implementation * WIP commit * More WIP * WIP, seems complete, untested * Ported trie-db operational * Add Ethereum trie tests * Use u8aConcat to construct u8a * Basic RocksDb interface * Combine engines into DB, expand tests * Update Trie with transaction wrapper * LevelDb (sync) interface * Remove unused (for now) package (compatibility) * trie-d -> trie-db-old, trie-db-ext -> trie-db * Version bump * Skip old trie tests (package to be removed) * Skip tests (fixed) * Remove non-used RocksDb spec * Skip FileFlat tests (deprecated?)
This commit is contained in:
Jaco Greeff
@@ -0,0 +1,17 @@
|
||||
// Copyright 2017-2018 @polkadot/db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the ISC license. See the LICENSE file for details.
|
||||
|
||||
import LevelDb from './engines/LevelDb';
|
||||
import LruDb from './engines/LruDb';
|
||||
import TransactionDb from './engines/TransactionDb';
|
||||
|
||||
export default class DiskDb extends TransactionDb {
|
||||
constructor (location: string) {
|
||||
super(
|
||||
new LruDb(
|
||||
new LevelDb(location)
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
// Copyright 2017-2018 @polkadot/db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the ISC license. See the LICENSE file for details.
|
||||
|
||||
import MemoryDb from './engines/MemoryDb';
|
||||
import TransactionDb from './engines/TransactionDb';
|
||||
|
||||
export default class Memory extends TransactionDb {
|
||||
constructor () {
|
||||
super(new MemoryDb());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
// Copyright 2017-2018 @polkadot/db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the ISC license. See the LICENSE file for details.
|
||||
|
||||
import mkdirp from 'mkdirp';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import rimraf from 'rimraf';
|
||||
|
||||
import FileFlatDb from './FileFlatDb';
|
||||
|
||||
const KEY_A = new Uint8Array([
|
||||
1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
|
||||
]);
|
||||
const KEY_B = new Uint8Array([
|
||||
2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
|
||||
]);
|
||||
const KEY_C = new Uint8Array([
|
||||
1, 2, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
|
||||
]);
|
||||
const KEY_D = new Uint8Array([
|
||||
1, 2, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
|
||||
]);
|
||||
const KEY_E = new Uint8Array([
|
||||
1, 2, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
|
||||
]);
|
||||
const VAL_A = new Uint8Array([0x42, 1, 0x69]);
|
||||
const VAL_B = new Uint8Array([0x42, 1, 2, 0x69]);
|
||||
const VAL_C = new Uint8Array([0x42, 1, 2, 3, 0x69]);
|
||||
const VAL_D = new Uint8Array([0x42, 1, 2, 3, 4, 0x69]);
|
||||
const VAL_E = new Uint8Array([0x42, 1, 2, 3, 4, 5, 0x69]);
|
||||
|
||||
describe.skip('FileFlatDb (compacting)', () => {
|
||||
const combined = new FileFlatDb(process.cwd());
|
||||
|
||||
it('compacts without failing', () => {
|
||||
expect(
|
||||
combined.maintain(() => {})
|
||||
).not.toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// NOTE Skipped, doesn't seem to be too happy on CI (cwd issues?)
|
||||
describe.skip('FileFlatDb (basics)', () => {
|
||||
const testGet = (key, value) =>
|
||||
expect(store.get(key)).toEqual(value);
|
||||
|
||||
const location = path.join(os.tmpdir(), '--test-FileFlatDb');
|
||||
let store;
|
||||
|
||||
beforeAll(() => {
|
||||
mkdirp.sync(location);
|
||||
|
||||
store = new FileFlatDb(location);
|
||||
store.open();
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
store.close();
|
||||
rimraf.sync(location);
|
||||
});
|
||||
|
||||
it('writes an entry', () => {
|
||||
console.error('A: execute');
|
||||
|
||||
store.put(KEY_A, VAL_A);
|
||||
|
||||
console.error('A: expectations');
|
||||
|
||||
testGet(KEY_A, VAL_A);
|
||||
});
|
||||
|
||||
it('writes an entry (additional)', () => {
|
||||
console.error('B: execute');
|
||||
|
||||
store.put(KEY_B, VAL_B);
|
||||
|
||||
console.error('A: expectations');
|
||||
|
||||
testGet(KEY_A, VAL_A);
|
||||
testGet(KEY_B, VAL_B);
|
||||
});
|
||||
|
||||
it('writes an entry (expanding the tree)', () => {
|
||||
console.error('C: execute');
|
||||
|
||||
store.put(KEY_C, VAL_C);
|
||||
|
||||
console.error('C: expectations');
|
||||
|
||||
testGet(KEY_A, VAL_A);
|
||||
testGet(KEY_B, VAL_B);
|
||||
testGet(KEY_C, VAL_C);
|
||||
});
|
||||
|
||||
it('writes an entry (expanding the tree, again)', () => {
|
||||
console.error('D: execute');
|
||||
|
||||
store.put(KEY_D, VAL_D);
|
||||
|
||||
console.error('D: expectations');
|
||||
|
||||
testGet(KEY_A, VAL_A);
|
||||
testGet(KEY_B, VAL_B);
|
||||
testGet(KEY_C, VAL_C);
|
||||
testGet(KEY_D, VAL_D);
|
||||
});
|
||||
|
||||
it('writes an entry (expanding the tree, yet again)', () => {
|
||||
console.error('E: execute');
|
||||
|
||||
store.put(KEY_E, VAL_E);
|
||||
|
||||
console.error('E: expectations');
|
||||
|
||||
testGet(KEY_A, VAL_A);
|
||||
testGet(KEY_B, VAL_B);
|
||||
testGet(KEY_C, VAL_C);
|
||||
testGet(KEY_D, VAL_D);
|
||||
testGet(KEY_E, VAL_E);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,534 @@
|
||||
// Copyright 2017-2018 @polkadot/db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the ISC license. See the LICENSE file for details.
|
||||
|
||||
import { BaseDb, ProgressCb } from '../types';
|
||||
|
||||
import fs from 'fs';
|
||||
// import snappy from 'snappy';
|
||||
import assert from '@polkadot/util/assert';
|
||||
import logger from '@polkadot/util/logger';
|
||||
import bufferToU8a from '@polkadot/util/buffer/toU8a';
|
||||
import u8aToBuffer from '@polkadot/util/u8a/toBuffer';
|
||||
import u8aToHex from '@polkadot/util/u8a/toHex';
|
||||
|
||||
enum Slot {
|
||||
EMPTY = 0,
|
||||
BRANCH = 1,
|
||||
LEAF = 2
|
||||
}
|
||||
|
||||
type Entry = {
|
||||
entry: Buffer,
|
||||
entryAt: number
|
||||
};
|
||||
|
||||
type Key = {
|
||||
key: Buffer,
|
||||
keyAt: number,
|
||||
keyValue: Buffer
|
||||
};
|
||||
|
||||
type Leaf = Entry & Key;
|
||||
|
||||
type Branch = Leaf & {
|
||||
branch: Buffer,
|
||||
branchAt: number
|
||||
};
|
||||
|
||||
type Value = Key & {
|
||||
value: Buffer,
|
||||
valueAt: number
|
||||
};
|
||||
|
||||
// NOTE 1,099,511,627,776 filesize (max allowed here is 6 as per Nodejs)
|
||||
const UINT_SIZE = 5;
|
||||
const KEY_SIZE = 32;
|
||||
const KEY_TOTAL_SIZE = KEY_SIZE + UINT_SIZE + UINT_SIZE;
|
||||
const ENTRY_NUM = 256;
|
||||
const ENTRY_SIZE = 1 + UINT_SIZE;
|
||||
const HEADER_SIZE = ENTRY_NUM * ENTRY_SIZE;
|
||||
const DEFAULT_FILE = 'store.db';
|
||||
|
||||
const l = logger('db/flat');
|
||||
|
||||
export default class FileFlatDb implements BaseDb {
|
||||
_fd: number;
|
||||
_file: string;
|
||||
|
||||
constructor (location: string, file: string = DEFAULT_FILE) {
|
||||
if (!fs.existsSync(location)) {
|
||||
throw new Error(`Unable to open ${location}`);
|
||||
}
|
||||
|
||||
this._file = `${location}/${file}`;
|
||||
this._fd = -1;
|
||||
}
|
||||
|
||||
open (): void {
|
||||
this._fd = this._open(this._file);
|
||||
}
|
||||
|
||||
close (): void {
|
||||
this.assertOpen();
|
||||
|
||||
fs.closeSync(this._fd);
|
||||
}
|
||||
|
||||
maintain (fn: ProgressCb): void {
|
||||
assert(this._fd === -1, 'Database cannot be open for compacting');
|
||||
|
||||
l.log('compacting database');
|
||||
|
||||
const start = Date.now();
|
||||
const newFile = `${this._file}.compacted`;
|
||||
const newFd = this._open(newFile, true);
|
||||
const oldFd = this._open(this._file);
|
||||
const keys = this._compact(fn, newFd, oldFd);
|
||||
|
||||
fs.closeSync(oldFd);
|
||||
fs.closeSync(newFd);
|
||||
|
||||
const newStat = fs.lstatSync(newFile);
|
||||
const oldStat = fs.lstatSync(this._file);
|
||||
const percentage = 100 * (newStat.size / oldStat.size);
|
||||
const sizeMB = newStat.size / (1024 * 1024);
|
||||
const elapsed = (Date.now() - start) / 1000;
|
||||
|
||||
fs.unlinkSync(this._file);
|
||||
fs.renameSync(newFile, this._file);
|
||||
|
||||
l.log(`compacted in ${elapsed.toFixed(2)}s, ${(keys / 1000).toFixed(2)}k keys, ${sizeMB.toFixed(2)}MB (${percentage.toFixed(2)}%)`);
|
||||
}
|
||||
|
||||
del (key: Uint8Array): void {
|
||||
throw new Error('delete not implemented, only stubbed');
|
||||
}
|
||||
|
||||
get (key: Uint8Array): Uint8Array | null {
|
||||
this.assertOpen();
|
||||
|
||||
l.debug(() => ['get', { key }]);
|
||||
|
||||
const desc = this.findKey(this._serializeKey(key), false);
|
||||
|
||||
if (!desc) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const result = this.readValue(desc);
|
||||
|
||||
return result && result.value
|
||||
? this._deserializeValue(result.value)
|
||||
: null;
|
||||
}
|
||||
|
||||
put (key: Uint8Array, value: Uint8Array): void {
|
||||
this.assertOpen();
|
||||
|
||||
l.debug(() => ['set', { key, value }]);
|
||||
|
||||
const desc = this.findKey(this._serializeKey(key), true);
|
||||
|
||||
if (!desc) {
|
||||
throw new Error('Unable to create key');
|
||||
}
|
||||
|
||||
this.writeValue(
|
||||
desc,
|
||||
this._serializeValue(value)
|
||||
);
|
||||
}
|
||||
|
||||
private _deserializeValue (value: Buffer): Uint8Array | null {
|
||||
// return bufferToU8a(
|
||||
// snappy.uncompressSync(value)
|
||||
// );
|
||||
|
||||
return bufferToU8a(value);
|
||||
}
|
||||
|
||||
private _serializeValue (value: Uint8Array): Buffer {
|
||||
// return snappy.compressSync(
|
||||
// u8aToBuffer(value)
|
||||
// );
|
||||
|
||||
return u8aToBuffer(value);
|
||||
}
|
||||
|
||||
private _serializeKey (key: Uint8Array): Buffer {
|
||||
if (key.length === KEY_SIZE) {
|
||||
return u8aToBuffer(key);
|
||||
} else if (key.length < KEY_SIZE) {
|
||||
const paddedKey = Buffer.alloc(KEY_SIZE);
|
||||
|
||||
paddedKey.set(key, 0);
|
||||
|
||||
return paddedKey;
|
||||
}
|
||||
|
||||
throw new Error(`${u8aToHex(key)} too large, expected <= 32 bytes`);
|
||||
}
|
||||
|
||||
private _compactReadEntry (fd: number, at: number, index: number): Buffer {
|
||||
const entry = Buffer.alloc(ENTRY_SIZE);
|
||||
const entryAt = at + (index * ENTRY_SIZE);
|
||||
|
||||
fs.readSync(fd, entry, 0, ENTRY_SIZE, entryAt);
|
||||
|
||||
return entry;
|
||||
}
|
||||
|
||||
private _compactReadKey (fd: number, at: number): [Buffer, Buffer] {
|
||||
const key = Buffer.alloc(KEY_TOTAL_SIZE);
|
||||
|
||||
fs.readSync(fd, key, 0, KEY_TOTAL_SIZE, at);
|
||||
|
||||
const valueLength = key.readUIntBE(KEY_SIZE, UINT_SIZE);
|
||||
const valueAt = key.readUIntBE(KEY_SIZE + UINT_SIZE, UINT_SIZE);
|
||||
const value = Buffer.alloc(valueLength);
|
||||
|
||||
fs.readSync(fd, value, 0, valueLength, valueAt);
|
||||
|
||||
return [key, value];
|
||||
}
|
||||
|
||||
private _compactWriteKey (fd: number, key: Buffer, value: Buffer): number {
|
||||
const stats = fs.fstatSync(fd);
|
||||
const at = stats.size;
|
||||
|
||||
fs.writeSync(fd, value, 0, value.length, at);
|
||||
key.writeUIntBE(at, KEY_SIZE + UINT_SIZE, UINT_SIZE);
|
||||
fs.writeSync(fd, key, 0, KEY_TOTAL_SIZE);
|
||||
|
||||
return at + value.length;
|
||||
}
|
||||
|
||||
private _compactUpdateLink (fd: number, at: number, index: number, pointer: number, type: Slot): void {
|
||||
const entry = Buffer.alloc(ENTRY_SIZE);
|
||||
|
||||
entry.set([type], 0);
|
||||
entry.writeUIntBE(pointer, 1, UINT_SIZE);
|
||||
|
||||
fs.writeSync(fd, entry, 0, ENTRY_SIZE, at + (index * ENTRY_SIZE));
|
||||
}
|
||||
|
||||
private _compactWriteHeader (fd: number, at: number, index: number): number {
|
||||
const stats = fs.fstatSync(fd);
|
||||
const headerAt = stats.size;
|
||||
const header = Buffer.alloc(HEADER_SIZE);
|
||||
|
||||
fs.writeSync(fd, header, 0, HEADER_SIZE, headerAt);
|
||||
|
||||
this._compactUpdateLink(fd, at, index, headerAt, Slot.BRANCH);
|
||||
|
||||
return headerAt;
|
||||
}
|
||||
|
||||
private _compact (fn: ProgressCb, newFd: number, oldFd: number): number {
|
||||
// l.debug(() => ['_compact', debug({ newFd, oldFd, newAt, oldAt })]);
|
||||
|
||||
let keys = 0;
|
||||
let percent = 0;
|
||||
|
||||
const doCompact = (newAt: number, oldAt: number, depth: number) => {
|
||||
const increment = (100 / ENTRY_NUM) / Math.pow(ENTRY_NUM, depth);
|
||||
|
||||
for (let index = 0; index < ENTRY_NUM; index++) {
|
||||
const entry = this._compactReadEntry(oldFd, oldAt, index);
|
||||
const dataAt = entry.readUIntBE(1, UINT_SIZE);
|
||||
const entryType = entry[0];
|
||||
|
||||
if (entryType === Slot.EMPTY) {
|
||||
// l.debug(() => '_compact/isEmpty');
|
||||
percent += increment;
|
||||
} else if (entryType === Slot.LEAF) {
|
||||
// l.debug(() => '_compact/isLeaf');
|
||||
|
||||
const [key, value] = this._compactReadKey(oldFd, dataAt);
|
||||
const keyAt = this._compactWriteKey(newFd, key, value);
|
||||
|
||||
this._compactUpdateLink(newFd, newAt, index, keyAt, Slot.LEAF);
|
||||
|
||||
keys++;
|
||||
percent += increment;
|
||||
} else if (entryType === Slot.BRANCH) {
|
||||
// l.debug(() => '_compact/isBranch');
|
||||
|
||||
const headerAt = this._compactWriteHeader(newFd, newAt, index);
|
||||
|
||||
doCompact(headerAt, dataAt, depth + 1);
|
||||
} else {
|
||||
throw new Error(`Unknown entry type, ${entryType}`);
|
||||
}
|
||||
|
||||
fn({
|
||||
isCompleted: depth === 0 && index === (ENTRY_NUM - 1),
|
||||
keys,
|
||||
percent
|
||||
});
|
||||
}
|
||||
|
||||
// l.debug(() => ['_compact', '=>', `${depth}: ${keys} keys written`]);
|
||||
};
|
||||
|
||||
doCompact(0, 0, 0);
|
||||
|
||||
return keys;
|
||||
}
|
||||
|
||||
private _findKey (key: Buffer, doCreate: boolean, keyIndex: number, diskAt: number): Leaf | null {
|
||||
const entry = Buffer.alloc(ENTRY_SIZE);
|
||||
const entryAt = diskAt + (key[keyIndex] * ENTRY_SIZE);
|
||||
|
||||
fs.readSync(this._fd, entry, 0, ENTRY_SIZE, entryAt);
|
||||
|
||||
l.debug(() => ['findKey', { key, doCreate, keyIndex, diskAt, entry, entryAt }]);
|
||||
|
||||
const entryType = entry[0];
|
||||
|
||||
if (entryType === Slot.BRANCH) {
|
||||
const branchAt = entry.readUIntBE(1, UINT_SIZE);
|
||||
|
||||
l.debug(() => ['findKey/isBranch', { branchAt }]);
|
||||
|
||||
return this._findKey(key, doCreate, keyIndex + 1, branchAt);
|
||||
}
|
||||
|
||||
if (entryType === Slot.EMPTY) {
|
||||
l.debug(() => ['findKey/isEmpty']);
|
||||
|
||||
return doCreate
|
||||
? this.writeNewLeaf(entry, entryAt, key)
|
||||
: null;
|
||||
}
|
||||
|
||||
if (entryType === Slot.LEAF) {
|
||||
const keyAt = entry.readUIntBE(1, UINT_SIZE);
|
||||
const keyValue = Buffer.alloc(KEY_TOTAL_SIZE);
|
||||
|
||||
fs.readSync(this._fd, keyValue, 0, KEY_TOTAL_SIZE, keyAt);
|
||||
|
||||
l.debug(() => ['findKey/isLeaf', { keyAt, entry, keyValue }]);
|
||||
|
||||
let matchIndex = keyIndex;
|
||||
|
||||
while (matchIndex < KEY_SIZE) {
|
||||
if (keyValue[matchIndex] !== key[matchIndex]) {
|
||||
break;
|
||||
}
|
||||
|
||||
matchIndex++;
|
||||
}
|
||||
|
||||
if (matchIndex !== KEY_SIZE) {
|
||||
return doCreate
|
||||
? this.writeNewBranch(entry, entryAt, key, keyAt, keyValue, matchIndex, matchIndex - keyIndex - 1)
|
||||
: null;
|
||||
}
|
||||
|
||||
return {
|
||||
entry,
|
||||
entryAt,
|
||||
key,
|
||||
keyAt,
|
||||
keyValue
|
||||
};
|
||||
}
|
||||
|
||||
throw new Error(`Unhandled entry type ${entryType}`);
|
||||
}
|
||||
|
||||
private findKey (key: Buffer, doCreate: boolean): Leaf | null {
|
||||
const result = this._findKey(key, doCreate, 0, 0);
|
||||
|
||||
l.debug(() => ['findKey', { result }]);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private _extractValueInfo (keyValue: Buffer): { valueAt: number, valueLength: number } {
|
||||
return {
|
||||
valueLength: keyValue.readUIntBE(KEY_SIZE, UINT_SIZE),
|
||||
valueAt: keyValue.readUIntBE(KEY_SIZE + UINT_SIZE, UINT_SIZE)
|
||||
};
|
||||
}
|
||||
|
||||
private _readValue (key: Buffer, keyAt: number, keyValue: Buffer): Value {
|
||||
l.debug(() => ['readValue', { key, keyAt, keyValue }]);
|
||||
|
||||
const { valueAt, valueLength } = this._extractValueInfo(keyValue);
|
||||
const value = Buffer.alloc(valueLength);
|
||||
|
||||
fs.readSync(this._fd, value, 0, valueLength, valueAt);
|
||||
|
||||
return {
|
||||
key,
|
||||
keyAt,
|
||||
keyValue,
|
||||
value,
|
||||
valueAt
|
||||
};
|
||||
}
|
||||
|
||||
private readValue ({ key, keyAt, keyValue }: Key): Value {
|
||||
const result = this._readValue(key, keyAt, keyValue);
|
||||
|
||||
l.debug(() => ['readValue', { result }]);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private _writeValue (key: Buffer, keyAt: number, keyValue: Buffer, value: Buffer): Value {
|
||||
l.debug(() => ['writeValue', { keyAt, keyValue, value }]);
|
||||
|
||||
let { valueAt, valueLength } = this._extractValueInfo(keyValue);
|
||||
|
||||
if (valueLength < value.length) {
|
||||
const stats = fs.fstatSync(this._fd);
|
||||
|
||||
valueAt = stats.size;
|
||||
}
|
||||
|
||||
fs.writeSync(this._fd, value, 0, value.length, valueAt);
|
||||
|
||||
keyValue.writeUIntBE(value.length, KEY_SIZE, UINT_SIZE);
|
||||
keyValue.writeUIntBE(valueAt, KEY_SIZE + UINT_SIZE, UINT_SIZE);
|
||||
|
||||
fs.writeSync(this._fd, keyValue, KEY_SIZE, 2 * UINT_SIZE, keyAt + KEY_SIZE);
|
||||
|
||||
return {
|
||||
key,
|
||||
keyAt,
|
||||
keyValue,
|
||||
value,
|
||||
valueAt
|
||||
};
|
||||
}
|
||||
|
||||
private writeValue ({ key, keyAt, keyValue }: Key, value: Buffer): Value {
|
||||
const result = this._writeValue(key, keyAt, keyValue, value);
|
||||
|
||||
l.debug(() => ['writeValue', '=>', { result }]);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private _writeNewKey (key: Buffer): Key {
|
||||
l.debug(() => ['writeNewKey', { key }]);
|
||||
|
||||
const stats = fs.fstatSync(this._fd);
|
||||
const keyAt = stats.size;
|
||||
const keyValue = Buffer.alloc(KEY_TOTAL_SIZE);
|
||||
|
||||
keyValue.set(key, 0);
|
||||
|
||||
fs.writeSync(this._fd, keyValue, 0, KEY_TOTAL_SIZE, keyAt);
|
||||
|
||||
return {
|
||||
key,
|
||||
keyAt,
|
||||
keyValue
|
||||
};
|
||||
}
|
||||
|
||||
private writeNewKey (key: Buffer): Key {
|
||||
const result = this._writeNewKey(key);
|
||||
|
||||
l.debug(() => ['writeNewKey', '=>', { result }]);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private _writeNewBranch (entry: Buffer, entryAt: number, key: Buffer, prevAt: number, prevValue: Buffer, matchIndex: number, depth: number): Branch {
|
||||
l.debug(() => ['writeNewBranch', { entry, entryAt, key, prevValue, matchIndex, depth }]);
|
||||
|
||||
const { keyAt, keyValue } = this.writeNewKey(key);
|
||||
const branch = Buffer.alloc(HEADER_SIZE);
|
||||
const keyIndex = ENTRY_SIZE * key[matchIndex];
|
||||
const prevIndex = ENTRY_SIZE * prevValue[matchIndex];
|
||||
const stats = fs.fstatSync(this._fd);
|
||||
let branchAt = stats.size;
|
||||
|
||||
branch.set([Slot.LEAF], keyIndex);
|
||||
branch.writeUIntBE(keyAt, keyIndex + 1, UINT_SIZE);
|
||||
branch.set([Slot.LEAF], prevIndex);
|
||||
branch.writeUIntBE(prevAt, prevIndex + 1, UINT_SIZE);
|
||||
fs.writeSync(this._fd, branch, 0, HEADER_SIZE, branchAt);
|
||||
|
||||
let intermediateAt = branchAt;
|
||||
|
||||
for (let offset = 1; depth > 0; depth--, offset++) {
|
||||
const intermediate = Buffer.alloc(HEADER_SIZE);
|
||||
const intermediateIndex = ENTRY_SIZE * key[matchIndex - offset];
|
||||
const stats = fs.fstatSync(this._fd);
|
||||
|
||||
intermediate.set([Slot.BRANCH], intermediateIndex);
|
||||
intermediate.writeUIntBE(intermediateAt, intermediateIndex + 1, UINT_SIZE);
|
||||
intermediateAt = stats.size;
|
||||
|
||||
fs.writeSync(this._fd, intermediate, 0, HEADER_SIZE, intermediateAt);
|
||||
}
|
||||
|
||||
entry.set([Slot.BRANCH], 0);
|
||||
entry.writeUIntBE(intermediateAt, 1, UINT_SIZE);
|
||||
fs.writeSync(this._fd, entry, 0, ENTRY_SIZE, entryAt);
|
||||
|
||||
return {
|
||||
branch,
|
||||
branchAt,
|
||||
entry: branch.slice(keyIndex, keyIndex + ENTRY_SIZE),
|
||||
entryAt: branchAt + keyIndex,
|
||||
key,
|
||||
keyAt,
|
||||
keyValue
|
||||
};
|
||||
}
|
||||
|
||||
private writeNewBranch (entry: Buffer, entryAt: number, key: Buffer, prevAt: number, prevValue: Buffer, matchIndex: number, depth: number): Leaf {
|
||||
const result = this._writeNewBranch(entry, entryAt, key, prevAt, prevValue, matchIndex, depth);
|
||||
|
||||
l.debug(() => ['writeNewBranch', '=>', { result }]);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private _writeNewLeaf (entry: Buffer, entryAt: number, key: Buffer): Leaf {
|
||||
l.debug(() => ['writeNewLeaf', { entry, entryAt, key }]);
|
||||
|
||||
const { keyAt, keyValue } = this.writeNewKey(key);
|
||||
|
||||
entry.set([Slot.LEAF], 0);
|
||||
entry.writeUIntBE(keyAt, 1, UINT_SIZE);
|
||||
|
||||
fs.writeSync(this._fd, entry, 0, ENTRY_SIZE, entryAt);
|
||||
|
||||
return {
|
||||
entry,
|
||||
entryAt,
|
||||
key,
|
||||
keyAt,
|
||||
keyValue
|
||||
};
|
||||
}
|
||||
|
||||
private writeNewLeaf (entry: Buffer, entryAt: number, key: Buffer): Leaf {
|
||||
const result = this._writeNewLeaf(entry, entryAt, key);
|
||||
|
||||
l.debug(() => ['writeNewLeaf', '=>', { result }]);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private assertOpen (): void {
|
||||
assert(this._fd !== -1, 'Expected an open database');
|
||||
}
|
||||
|
||||
private _open (file: string, startEmpty: boolean = false): number {
|
||||
if (!fs.existsSync(file) || startEmpty) {
|
||||
fs.writeFileSync(file, Buffer.alloc(HEADER_SIZE));
|
||||
}
|
||||
|
||||
return fs.openSync(file, 'a+');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
// Copyright 2017-2018 @polkadot/db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the ISC license. See the LICENSE file for details.
|
||||
|
||||
import { BaseDb, ProgressCb } from '../types';
|
||||
|
||||
import fs from 'fs';
|
||||
import mkdirp from 'mkdirp';
|
||||
import logger from '@polkadot/util/logger';
|
||||
import bufferToU8a from '@polkadot/util/buffer/toU8a';
|
||||
import u8aToBuffer from '@polkadot/util/u8a/toBuffer';
|
||||
import u8aToHex from '@polkadot/util/u8a/toHex';
|
||||
|
||||
type FilePath = {
|
||||
directory: string,
|
||||
file: string
|
||||
};
|
||||
|
||||
const DIR_DEPTH = 1;
|
||||
|
||||
const l = logger('db/tree');
|
||||
|
||||
export default class FileTreeDb implements BaseDb {
|
||||
_location: string;
|
||||
|
||||
constructor (location: string) {
|
||||
this._location = location;
|
||||
}
|
||||
|
||||
private _getFilePath (key: Uint8Array): FilePath {
|
||||
// NOTE We want to limit the number of entries in any specific directory. Split the
|
||||
// key into parts and use this to construct the path and the actual filename. We want
|
||||
// to limit the entries per directory, but at the same time minimize the number of
|
||||
// directories we need to create (when non-existent as well as the size overhead)
|
||||
const parts = u8aToHex(key).match(/.{1,6}/g) || [];
|
||||
const directory = `${this._location}/${parts.slice(0, DIR_DEPTH).join('/')}`;
|
||||
const file = `${directory}/${parts.slice(DIR_DEPTH).join('')}`;
|
||||
|
||||
return {
|
||||
directory,
|
||||
file
|
||||
};
|
||||
}
|
||||
|
||||
open (): void {
|
||||
// noop
|
||||
}
|
||||
|
||||
close (): void {
|
||||
// noop
|
||||
}
|
||||
|
||||
maintain (fn: ProgressCb): void {
|
||||
fn({
|
||||
isCompleted: true,
|
||||
keys: 0,
|
||||
percent: 100
|
||||
});
|
||||
}
|
||||
|
||||
del (key: Uint8Array): void {
|
||||
l.debug(() => ['del', { key }]);
|
||||
|
||||
const { file } = this._getFilePath(key);
|
||||
|
||||
if (fs.existsSync(file)) {
|
||||
fs.unlinkSync(file);
|
||||
}
|
||||
}
|
||||
|
||||
get (key: Uint8Array): Uint8Array | null {
|
||||
l.debug(() => ['get', { key }]);
|
||||
|
||||
const { file } = this._getFilePath(key);
|
||||
|
||||
if (!fs.existsSync(file)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return bufferToU8a(
|
||||
fs.readFileSync(file)
|
||||
);
|
||||
}
|
||||
|
||||
put (key: Uint8Array, value: Uint8Array): void {
|
||||
l.debug(() => ['set', { key, value }]);
|
||||
|
||||
const { directory, file } = this._getFilePath(key);
|
||||
|
||||
if (!fs.existsSync(directory)) {
|
||||
mkdirp.sync(directory);
|
||||
}
|
||||
|
||||
fs.writeFileSync(file, u8aToBuffer(value));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
// Copyright 2017-2018 @polkadot/db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the ISC license. See the LICENSE file for details.
|
||||
|
||||
import mkdirp from 'mkdirp';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import toU8a from '@polkadot/util/u8a/fromString';
|
||||
|
||||
import LevelDb from './LevelDb';
|
||||
|
||||
describe('LevelDb', () => {
|
||||
const location = path.join(os.tmpdir(), '--test-LevelDb');
|
||||
let db;
|
||||
|
||||
beforeAll(() => {
|
||||
mkdirp.sync(location);
|
||||
|
||||
db = new LevelDb(location);
|
||||
db.open();
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
db.close();
|
||||
rimraf.sync(location);
|
||||
});
|
||||
|
||||
it('sets & retrieves', () => {
|
||||
const key = toU8a('key0');
|
||||
const value = toU8a('value0');
|
||||
|
||||
db.put(key, value);
|
||||
|
||||
expect(db.get(key)).toEqual(value);
|
||||
});
|
||||
|
||||
it('retrieves null for non-existing', () => {
|
||||
const key = toU8a('key1');
|
||||
|
||||
expect(db.get(key)).toEqual(null);
|
||||
});
|
||||
|
||||
it('overrides values', () => {
|
||||
const key = toU8a('key2');
|
||||
const value = toU8a('value1');
|
||||
const replace = toU8a('value2');
|
||||
|
||||
db.put(key, value);
|
||||
|
||||
expect(db.get(key)).toEqual(value);
|
||||
|
||||
db.put(key, replace);
|
||||
|
||||
expect(db.get(key)).toEqual(replace);
|
||||
});
|
||||
|
||||
it('deletes values', () => {
|
||||
const key = toU8a('key2');
|
||||
|
||||
db.del(key);
|
||||
|
||||
expect(db.get(key)).toEqual(null);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,83 @@
|
||||
// Copyright 2017-2018 @polkadot/db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the ISC license. See the LICENSE file for details.
|
||||
|
||||
import { BaseDb, ProgressCb } from '../types';
|
||||
|
||||
import leveldb from 'nosql-leveldb';
|
||||
import logger from '@polkadot/util/logger';
|
||||
import bufferToU8a from '@polkadot/util/buffer/toU8a';
|
||||
import u8aToBuffer from '@polkadot/util/u8a/toBuffer';
|
||||
|
||||
// import assert from '@polkadot/util/assert';
|
||||
// import u8aToHex from '@polkadot/util/u8a/toHex';
|
||||
|
||||
const l = logger('db/leveldb');
|
||||
|
||||
export default class Level implements BaseDb {
|
||||
private db: LevelDb;
|
||||
|
||||
constructor (location: string) {
|
||||
this.db = leveldb(location);
|
||||
}
|
||||
|
||||
close (): void {
|
||||
l.debug(() => ['close']);
|
||||
|
||||
this.db.closeSync();
|
||||
}
|
||||
|
||||
open (): void {
|
||||
l.debug(() => ['open']);
|
||||
|
||||
this.db.openSync();
|
||||
}
|
||||
|
||||
maintain (fn: ProgressCb): void {
|
||||
fn({
|
||||
isCompleted: true,
|
||||
keys: 0,
|
||||
percent: 100
|
||||
});
|
||||
}
|
||||
|
||||
del (key: Uint8Array): void {
|
||||
this.db.delSync(
|
||||
this._serializeKey(key)
|
||||
);
|
||||
}
|
||||
|
||||
get (key: Uint8Array): Uint8Array | null {
|
||||
try {
|
||||
return this._deserializeValue(
|
||||
this.db.getSync(
|
||||
this._serializeKey(key),
|
||||
{ asBuffer: true }
|
||||
)
|
||||
);
|
||||
} catch (error) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
put (key: Uint8Array, value: Uint8Array): void {
|
||||
this.db.putSync(
|
||||
this._serializeKey(key),
|
||||
this._serializeValue(value)
|
||||
);
|
||||
}
|
||||
|
||||
private _deserializeValue (value: Buffer): Uint8Array | null {
|
||||
return value
|
||||
? bufferToU8a(value)
|
||||
: null;
|
||||
}
|
||||
|
||||
private _serializeKey (key: Uint8Array): Buffer {
|
||||
return u8aToBuffer(key);
|
||||
}
|
||||
|
||||
private _serializeValue (value: Uint8Array): Buffer {
|
||||
return u8aToBuffer(value);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
// Copyright 2017-2018 @polkadot/db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the ISC license. See the LICENSE file for details.
|
||||
|
||||
import toU8a from '@polkadot/util/u8a/fromString';
|
||||
|
||||
import LruDb from './LruDb';
|
||||
import MemoryDb from './MemoryDb';
|
||||
|
||||
describe('LruDb', () => {
|
||||
const memory = new MemoryDb();
|
||||
const db = new LruDb(memory);
|
||||
|
||||
it('retrieves an item from the backing when not available (caching it)', () => {
|
||||
const key = toU8a('test1');
|
||||
const value = toU8a('value1');
|
||||
|
||||
memory.put(key, value);
|
||||
|
||||
expect(db.get(key)).toEqual(value);
|
||||
expect(db._getLru(key)).toEqual({ value });
|
||||
});
|
||||
|
||||
it('replaces an item (caching and backing)', () => {
|
||||
const key = toU8a('test1');
|
||||
const value = toU8a('test');
|
||||
|
||||
db.put(key, value);
|
||||
|
||||
expect(db.get(key)).toEqual(value);
|
||||
expect(memory.get(key)).toEqual(value);
|
||||
expect(db._getLru(key)).toEqual({ value });
|
||||
});
|
||||
|
||||
it('retrieves item from LRU when available', () => {
|
||||
const key = toU8a('test1');
|
||||
const value = toU8a('test');
|
||||
|
||||
memory.del(key);
|
||||
|
||||
expect(db.get(key)).toEqual(value);
|
||||
});
|
||||
|
||||
it('puts item both in LRU and backing', () => {
|
||||
const key = toU8a('test0');
|
||||
const value = toU8a('value0');
|
||||
|
||||
db.put(key, value);
|
||||
|
||||
expect(memory.get(key)).toEqual(value);
|
||||
expect(db._getLru(key)).toEqual({ value });
|
||||
});
|
||||
|
||||
it('deletes an item from both backing and db', () => {
|
||||
const key = toU8a('test0');
|
||||
|
||||
db.del(key);
|
||||
|
||||
expect(memory.get(key)).toEqual(null);
|
||||
expect(db._getLru(key)).toEqual({ value: null });
|
||||
expect(db.get(key)).toEqual(null);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,78 @@
|
||||
// Copyright 2017-2018 @polkadot/db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the ISC license. See the LICENSE file for details.
|
||||
|
||||
import { BaseDb, ProgressCb } from '../types';
|
||||
|
||||
import { LRUMap } from 'lru_map';
|
||||
import logger from '@polkadot/util/logger';
|
||||
|
||||
type CachedValue = {
|
||||
value: Uint8Array | null
|
||||
};
|
||||
|
||||
const DEFAULT_ITEM_COUNT = 4096;
|
||||
|
||||
const l = logger('db/lru');
|
||||
|
||||
export default class LruDb implements BaseDb {
|
||||
private backing: BaseDb;
|
||||
private lru: LRUMap<string, CachedValue>;
|
||||
|
||||
constructor (backing: BaseDb, itemCount: number = DEFAULT_ITEM_COUNT) {
|
||||
this.backing = backing;
|
||||
this.lru = new LRUMap(itemCount);
|
||||
}
|
||||
|
||||
close (): void {
|
||||
l.debug(() => ['close']);
|
||||
|
||||
this.lru.clear();
|
||||
this.backing.close();
|
||||
}
|
||||
|
||||
open (): void {
|
||||
l.debug(() => ['open']);
|
||||
|
||||
this.lru.clear();
|
||||
this.backing.open();
|
||||
}
|
||||
|
||||
maintain (fn: ProgressCb): void {
|
||||
this.backing.maintain(fn);
|
||||
}
|
||||
|
||||
del (key: Uint8Array): void {
|
||||
const keyStr = key.toString();
|
||||
|
||||
this.backing.del(key);
|
||||
this.lru.set(keyStr, { value: null });
|
||||
}
|
||||
|
||||
get (key: Uint8Array): Uint8Array | null {
|
||||
const keyStr = key.toString();
|
||||
const cached = this.lru.get(keyStr);
|
||||
|
||||
if (cached) {
|
||||
return cached.value;
|
||||
}
|
||||
|
||||
const value = this.backing.get(key);
|
||||
|
||||
this.lru.set(keyStr, { value });
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
// Convenience methods used in tests
|
||||
_getLru (key: Uint8Array): CachedValue | undefined {
|
||||
return this.lru.get(key.toString());
|
||||
}
|
||||
|
||||
put (key: Uint8Array, value: Uint8Array): void {
|
||||
const keyStr = key.toString();
|
||||
|
||||
this.backing.put(key, value);
|
||||
this.lru.set(keyStr, { value });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
// Copyright 2017-2018 @polkadot/db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the ISC license. See the LICENSE file for details.
|
||||
|
||||
import { BaseDb, ProgressCb } from '../types';
|
||||
|
||||
// import logger from '@polkadot/util/logger';
|
||||
// import u8aToHex from '@polkadot/util/u8a/toHex';
|
||||
|
||||
type Storage = {
|
||||
[index: string]: Uint8Array
|
||||
};
|
||||
|
||||
// const l = logger('db/memory');
|
||||
|
||||
export default class MemoryDb implements BaseDb {
|
||||
private storage: Storage;
|
||||
|
||||
constructor () {
|
||||
this.storage = {};
|
||||
}
|
||||
|
||||
close (): void {
|
||||
// noop
|
||||
}
|
||||
|
||||
open (): void {
|
||||
// noop
|
||||
}
|
||||
|
||||
maintain (fn: ProgressCb): void {
|
||||
fn({
|
||||
isCompleted: true,
|
||||
keys: Object.keys(this.storage).length,
|
||||
percent: 100
|
||||
});
|
||||
}
|
||||
|
||||
del (key: Uint8Array): void {
|
||||
delete this.storage[key.toString()];
|
||||
}
|
||||
|
||||
get (key: Uint8Array): Uint8Array | null {
|
||||
// l.debug(() => ['get', u8aToHex(key)]);
|
||||
|
||||
return this.storage[key.toString()] || null;
|
||||
}
|
||||
|
||||
put (key: Uint8Array, value: Uint8Array): void {
|
||||
// l.debug(() => ['put', u8aToHex(key), u8aToHex(value)]);
|
||||
|
||||
this.storage[key.toString()] = value;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
// Copyright 2017-2018 @polkadot/db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the ISC license. See the LICENSE file for details.
|
||||
|
||||
import { BaseDb, ProgressCb } from '../types';
|
||||
|
||||
import rocksdb from 'rocksdb-node';
|
||||
import assert from '@polkadot/util/assert';
|
||||
import logger from '@polkadot/util/logger';
|
||||
import isUndefined from '@polkadot/util/is/undefined';
|
||||
import bufferToU8a from '@polkadot/util/buffer/toU8a';
|
||||
import u8aToBuffer from '@polkadot/util/u8a/toBuffer';
|
||||
|
||||
// import assert from '@polkadot/util/assert';
|
||||
// import u8aToHex from '@polkadot/util/u8a/toHex';
|
||||
|
||||
const l = logger('db/rocksdb');
|
||||
|
||||
export default class Rocks implements BaseDb {
|
||||
// @ts-ignore Guard with assertOpen
|
||||
private db: RocksDb;
|
||||
private location: string;
|
||||
|
||||
constructor (location: string) {
|
||||
this.location = location;
|
||||
}
|
||||
|
||||
private assertOpen () {
|
||||
assert(!isUndefined(this.db), 'Database needs to be open');
|
||||
}
|
||||
|
||||
close (): void {
|
||||
l.debug(() => ['close']);
|
||||
|
||||
this.assertOpen();
|
||||
this.db.close();
|
||||
|
||||
// @ts-ignore caught with guard
|
||||
this.db = undefined;
|
||||
}
|
||||
|
||||
open (): void {
|
||||
l.debug(() => ['open']);
|
||||
|
||||
this.db = rocksdb.open(
|
||||
{ create_if_missing: true },
|
||||
this.location
|
||||
);
|
||||
}
|
||||
|
||||
maintain (fn: ProgressCb): void {
|
||||
fn({
|
||||
isCompleted: true,
|
||||
keys: 0,
|
||||
percent: 100
|
||||
});
|
||||
}
|
||||
|
||||
del (key: Uint8Array): void {
|
||||
this.assertOpen();
|
||||
this.db.del(u8aToBuffer(key));
|
||||
}
|
||||
|
||||
get (key: Uint8Array): Uint8Array | null {
|
||||
this.assertOpen();
|
||||
|
||||
return bufferToU8a(
|
||||
this.db.get(
|
||||
{ buffer: true },
|
||||
u8aToBuffer(key)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
put (key: Uint8Array, value: Uint8Array): void {
|
||||
this.assertOpen();
|
||||
|
||||
this.db.put(
|
||||
u8aToBuffer(key),
|
||||
u8aToBuffer(value)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
// Copyright 2017-2018 @polkadot/db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the ISC license. See the LICENSE file for details.
|
||||
|
||||
import toU8a from '@polkadot/util/u8a/fromString';
|
||||
|
||||
import MemoryDb from './MemoryDb';
|
||||
import TransactionDb from './TransactionDb';
|
||||
|
||||
describe('TransactionDb', () => {
|
||||
const memory = new MemoryDb();
|
||||
const db = new TransactionDb(memory);
|
||||
|
||||
it('passed through values immediately when not in transaction', () => {
|
||||
const key = toU8a('test0');
|
||||
const value = toU8a('value0');
|
||||
|
||||
db.put(key, value);
|
||||
|
||||
expect(memory.get(key)).toEqual(value);
|
||||
});
|
||||
|
||||
it('commits when transaction passes (result = true, put)', () => {
|
||||
const key = toU8a('test1');
|
||||
const value = toU8a('value1');
|
||||
|
||||
expect(
|
||||
db.transaction(() => {
|
||||
db.put(key, value);
|
||||
|
||||
return true;
|
||||
})
|
||||
).toEqual(true);
|
||||
expect(memory.get(key)).toEqual(value);
|
||||
});
|
||||
|
||||
it('commits when transaction passes (result = true, del)', () => {
|
||||
const key = toU8a('test0');
|
||||
|
||||
expect(
|
||||
db.transaction(() => {
|
||||
db.del(key);
|
||||
|
||||
return true;
|
||||
})
|
||||
).toEqual(true);
|
||||
expect(memory.get(key)).toEqual(null);
|
||||
});
|
||||
|
||||
it('does not commit when transaction fails (result = false)', () => {
|
||||
const key = toU8a('test2');
|
||||
const value = toU8a('value2');
|
||||
|
||||
expect(
|
||||
db.transaction(() => {
|
||||
db.put(key, value);
|
||||
|
||||
return false;
|
||||
})
|
||||
).toEqual(false);
|
||||
expect(memory.get(key)).toEqual(null);
|
||||
});
|
||||
|
||||
it('does not commit when transaction throws', () => {
|
||||
const key = toU8a('test2');
|
||||
const value = toU8a('value2');
|
||||
|
||||
expect(
|
||||
() => {
|
||||
db.transaction(() => {
|
||||
db.put(key, value);
|
||||
|
||||
throw new Error('test');
|
||||
})
|
||||
}
|
||||
).toThrow(/test/);
|
||||
expect(memory.get(key)).toEqual(null);
|
||||
});
|
||||
|
||||
it('inside the transaction, the value is set', () => {
|
||||
const key = toU8a('test2');
|
||||
const value = toU8a('value2');
|
||||
|
||||
db.transaction(() => {
|
||||
db.put(key, value);
|
||||
|
||||
expect(db.get(key)).toEqual(value);
|
||||
|
||||
return false;
|
||||
});
|
||||
expect(memory.get(key)).toEqual(null);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,142 @@
|
||||
// Copyright 2017-2018 @polkadot/db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the ISC license. See the LICENSE file for details.
|
||||
|
||||
import { BaseDb, TxDb, ProgressCb } from '../types';
|
||||
|
||||
import assert from '@polkadot/util/assert';
|
||||
import logger from '@polkadot/util/logger';
|
||||
import isNull from '@polkadot/util/is/null';
|
||||
|
||||
type Overlay = {
|
||||
[index: string]: {
|
||||
key: Uint8Array,
|
||||
value: Uint8Array | null
|
||||
}
|
||||
};
|
||||
|
||||
const l = logger('db/transact');
|
||||
|
||||
export default class TransactionDb implements TxDb {
|
||||
private backing: BaseDb;
|
||||
private txOverlay: Overlay;
|
||||
private txStarted: boolean;
|
||||
|
||||
constructor (backing: BaseDb) {
|
||||
this.backing = backing;
|
||||
this.txOverlay = {};
|
||||
this.txStarted = false;
|
||||
}
|
||||
|
||||
transaction (fn: () => boolean): boolean {
|
||||
l.debug(() => ['transaction']);
|
||||
|
||||
try {
|
||||
this.createTx();
|
||||
|
||||
const result = fn();
|
||||
|
||||
if (result) {
|
||||
this.commitTx();
|
||||
} else {
|
||||
this.revertTx();
|
||||
}
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
this.revertTx();
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
close (): void {
|
||||
this.backing.open();
|
||||
}
|
||||
|
||||
open (): void {
|
||||
this.backing.close();
|
||||
}
|
||||
|
||||
maintain (fn: ProgressCb): void {
|
||||
assert(this.txStarted === false, 'Cannot maintain inside an open transaction');
|
||||
|
||||
this.backing.maintain(fn);
|
||||
}
|
||||
|
||||
del (key: Uint8Array): void {
|
||||
if (this.txStarted) {
|
||||
this.txOverlay[key.toString()] = {
|
||||
key,
|
||||
value: null
|
||||
};
|
||||
return;
|
||||
}
|
||||
|
||||
this.backing.del(key);
|
||||
}
|
||||
|
||||
get (key: Uint8Array): Uint8Array | null {
|
||||
// l.debug(() => ['get', u8aToHex(key)]);
|
||||
|
||||
if (this.txStarted) {
|
||||
const value = this.txOverlay[key.toString()];
|
||||
|
||||
if (value) {
|
||||
return value.value;
|
||||
}
|
||||
}
|
||||
|
||||
return this.backing.get(key);
|
||||
}
|
||||
|
||||
put (key: Uint8Array, value: Uint8Array): void {
|
||||
// l.debug(() => ['put', u8aToHex(key), u8aToHex(value)]);
|
||||
|
||||
if (this.txStarted) {
|
||||
this.txOverlay[key.toString()] = {
|
||||
key,
|
||||
value
|
||||
};
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
return this.backing.put(key, value);
|
||||
}
|
||||
|
||||
private createTx (): void {
|
||||
l.debug(() => ['createTx']);
|
||||
|
||||
assert(!this.txStarted, 'Cannot create a transaction when one is already active');
|
||||
|
||||
this.txOverlay = {};
|
||||
this.txStarted = true;
|
||||
}
|
||||
|
||||
private commitTx (): void {
|
||||
l.debug(() => ['commitTx', Object.keys(this.txOverlay).length, 'keys']);
|
||||
|
||||
assert(this.txStarted, 'Cannot commit when not in transaction');
|
||||
|
||||
Object.values(this.txOverlay).forEach(({ key, value }) => {
|
||||
if (isNull(value)) {
|
||||
this.backing.del(key);
|
||||
} else {
|
||||
this.backing.put(key, value);
|
||||
}
|
||||
});
|
||||
|
||||
this.txOverlay = {};
|
||||
this.txStarted = false;
|
||||
}
|
||||
|
||||
private revertTx (): void {
|
||||
l.debug(() => ['revertTx']);
|
||||
|
||||
assert(this.txStarted, 'Cannot revert when not in transaction');
|
||||
|
||||
this.txOverlay = {};
|
||||
this.txStarted = false;
|
||||
}
|
||||
}
|
||||
Vendored
+34
@@ -0,0 +1,34 @@
|
||||
// Copyright 2017-2018 @polkadot/db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the ISC license. See the LICENSE file for details.
|
||||
|
||||
type LevelDbOptions$Open = {
|
||||
createIfMissing?: boolean,
|
||||
errorIfExists?: boolean,
|
||||
compression?: boolean,
|
||||
cacheSize?: boolean
|
||||
};
|
||||
|
||||
type LevelDbOptions$Get ={
|
||||
asBuffer?: boolean,
|
||||
fillCache?: boolean
|
||||
};
|
||||
|
||||
type LevelDbOptions$Put ={
|
||||
sync?: boolean
|
||||
};
|
||||
|
||||
declare interface LevelDb {
|
||||
openSync (options?: LevelDbOptions$Open): void;
|
||||
closeSync (): void;
|
||||
|
||||
delSync (key: Buffer): void;
|
||||
getSync (key: Buffer, options?: LevelDbOptions$Get): Buffer;
|
||||
putSync (key: Buffer, value: Buffer, options?: LevelDbOptions$Put): void;
|
||||
}
|
||||
|
||||
declare module 'nosql-leveldb' {
|
||||
const leveldb: (location: string) => LevelDb;
|
||||
|
||||
export default leveldb;
|
||||
}
|
||||
Vendored
+27
@@ -0,0 +1,27 @@
|
||||
// Copyright 2017-2018 @polkadot/db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the ISC license. See the LICENSE file for details.
|
||||
|
||||
type RocksDbOptions$Open = {
|
||||
create_if_missing?: boolean
|
||||
};
|
||||
|
||||
type RocksDbOptions$Get ={
|
||||
buffer: boolean
|
||||
};
|
||||
|
||||
declare interface RocksDb {
|
||||
close (): void;
|
||||
|
||||
del (key: Buffer): void;
|
||||
get (options: RocksDbOptions$Get, key: Buffer): Buffer;
|
||||
put (key: Buffer, value: Buffer): void;
|
||||
}
|
||||
|
||||
declare module 'rocksdb-node' {
|
||||
const rocksdb: {
|
||||
open (options: RocksDbOptions$Open, location: string): RocksDb;
|
||||
};
|
||||
|
||||
export default rocksdb;
|
||||
}
|
||||
Vendored
+14
@@ -0,0 +1,14 @@
|
||||
// Copyright 2017-2018 @polkadot/db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the ISC license. See the LICENSE file for details.
|
||||
|
||||
declare interface Snappy {
|
||||
compressSync (input: Buffer): Buffer;
|
||||
uncompressSync (input: Buffer): Buffer;
|
||||
}
|
||||
|
||||
declare module 'snappy' {
|
||||
const snappy: Snappy;
|
||||
|
||||
export default snappy;
|
||||
}
|
||||
Vendored
+26
@@ -0,0 +1,26 @@
|
||||
// Copyright 2017-2018 @polkadot/db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the ISC license. See the LICENSE file for details.
|
||||
|
||||
export type ProgressValue = {
|
||||
isCompleted?: boolean,
|
||||
keys: number,
|
||||
percent: number
|
||||
};
|
||||
|
||||
export type ProgressCb = (progress: ProgressValue) => void;
|
||||
|
||||
export interface BaseDb {
|
||||
close (): void;
|
||||
open (): void;
|
||||
|
||||
maintain (fn: ProgressCb): void;
|
||||
|
||||
del (key: Uint8Array): void;
|
||||
get (key: Uint8Array): Uint8Array | null;
|
||||
put (key: Uint8Array, value: Uint8Array): void;
|
||||
}
|
||||
|
||||
export interface TxDb extends BaseDb {
|
||||
transaction (fn: () => boolean): boolean;
|
||||
}
|
||||
Reference in New Issue
Block a user