quantus/common
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Add entries() method to trie (#418)

Browse Source 
* Add entries() method to trie

* Add inv. getEntry for a single node (or root)
This commit is contained in:
Jaco Greeff
2019-06-08 08:15:36 +02:00
committed by GitHub
parent 65ad76ace0
commit 1e561a4046
4 changed files with 109 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/trie-db/src/Impl.ts
+43 -6
View File
@@ -4,7 +4,7 @@
import { TxDb, ProgressCb } from '@polkadot/db/types';
import { Codec } from '@polkadot/trie-codec/types';
import { EncodedPath, TrieDb, Node, NodeBranch, NodeEncodedOrEmpty, NodeKv, NodeNotEmpty, NodeType } from './types';
import { EncodedPath, TrieDb, Node, NodeBranch, NodeEncodedOrEmpty, NodeKv, NodeNotEmpty, NodeType, TrieEntry } from './types';
import substrateCodec from '@polkadot/trie-codec';
import { decodeNibbles, encodeNibbles, extractNodeKey } from '@polkadot/trie-codec/nibbles';
@@ -45,19 +45,42 @@ export default class Impl extends Checkpoint {
this.constants = _constants;
}
protected _entry (root: Uint8Array): TrieEntry | null {
const [encoded, node] = this._getNodeRaw(root);
if (isNull(encoded) || isNull(node)) {
return null;
}
return [root, encoded, node.filter((u8a) => u8a && u8a.length === 32) as Array<Uint8Array>];
}
protected _entries (root: Uint8Array, entries: Array<TrieEntry> = []): Array<TrieEntry> {
// l.debug(() => ['entries', { root }]);
const entry = this._entry(root);
if (!entry) {
return entries;
}
entries.push(entry);
entry[2].forEach((u8a) => this._entries(u8a, entries));
return entries;
}
protected _snapshot (dest: TrieDb, fn: ProgressCb | undefined, root: Uint8Array, keys: number, percent: number, depth: number): number {
// l.debug(() => ['snapshot', { root }]);
const node = this._getNode(root);
const [encoded, node] = this._getNodeRaw(root);
if (isNull(node)) {
if (isNull(encoded) || isNull(node)) {
return keys;
}
keys++;
dest.db.put(root, encodeNode(this.codec, node));
dest.db.put(root, encoded);
fn && fn({ keys, percent });
fn && fn({ keys: ++keys, percent });
node.forEach((u8a) => {
if (u8a && u8a.length === 32) {
@@ -70,6 +93,20 @@ export default class Impl extends Checkpoint {
return keys;
}
protected _getNodeRaw (hash: Uint8Array | null): [Uint8Array | null, Node] {
// l.debug(() => ['_getNode', { hash }]);
if (!hash || hash.length === 0 || keyEquals(hash, this.constants.EMPTY_HASH)) {
return [null, null];
} else if (hash.length < 32) {
return [hash, decodeNode(this.codec, hash)];
}
const raw = this.db.get(hash);
return [raw, decodeNode(this.codec, raw)];
}
protected _getNode (hash: Uint8Array | null): Node {
// l.debug(() => ['_getNode', { hash }]);
packages/trie-db/src/index.ts
+23 -12
View File
@@ -4,7 +4,7 @@
import { TxDb, ProgressCb } from '@polkadot/db/types';
import { Codec } from '@polkadot/trie-codec/types';
import { TrieDb, Node } from './types';
import { TrieDb, Node, TrieEntry } from './types';
import MemoryDb from '@polkadot/db/Memory';
import { toNibbles } from '@polkadot/trie-codec/util';
@@ -121,30 +121,41 @@ export default class Trie extends Impl implements TrieDb {
return this.rootHash;
}
getNode (hash?: Uint8Array): Node {
return this._getNode(hash || this.rootHash);
}
setRoot (rootHash: Uint8Array): void {
this.rootHash = rootHash;
// return this._setRootNode(rootNode);
}
snapshot (dest: TrieDb, fn?: ProgressCb): number {
const start = Date.now();
getEntry (hash?: Uint8Array): TrieEntry | null {
return this._entry(hash || this.rootHash);
}
getNode (hash?: Uint8Array): Node {
return this._getNode(hash || this.rootHash);
}
entries (): Array<TrieEntry> {
l.debug(() => 'retreiving trie entries');
const start = Date.now();
const entries = this._entries(this.rootHash);
const elapsed = (Date.now() - start) / 1000;
l.debug(() => `entries retrieved in ${elapsed.toFixed(2)}s, ${(entries.length / 1000).toFixed(2)}k keys`);
return entries;
}
snapshot (dest: TrieDb, fn?: ProgressCb): number {
l.debug(() => 'creating current state snapshot');
const start = Date.now();
const keys = this._snapshot(dest, fn, this.rootHash, 0, 0, 0);
const elapsed = (Date.now() - start) / 1000;
dest.setRoot(this.rootHash);
const newSize = dest.db.size();
const percentage = 100 * (newSize / this.db.size());
const sizeMB = newSize / (1024 * 1024);
l.debug(() => `snapshot created in ${elapsed.toFixed(2)}s, ${(keys / 1000).toFixed(2)}k keys, ${sizeMB.toFixed(2)}MB (${percentage.toFixed(2)}%)`);
l.debug(() => `snapshot created in ${elapsed.toFixed(2)}s, ${(keys / 1000).toFixed(2)}k keys`);
fn && fn({
isCompleted: true,
packages/trie-db/src/snapshot.spec.ts
+34 -18
View File
@@ -8,6 +8,16 @@ import { u8aToHex, u8aToU8a as toU8a } from '@polkadot/util';
import Trie from '.';
describe('snapshots', () => {
const values = [
{ k: toU8a('test'), v: toU8a('one') },
{ k: toU8a('one'), v: toU8a('testing') },
{ k: toU8a('two'), v: toU8a('testing with a much longer value here') },
{ k: toU8a('twzei'), v: toU8a('und Deutch') },
{ k: toU8a('do'), v: toU8a('do it') },
{ k: toU8a('dog'), v: toU8a('doggie') },
{ k: toU8a('dogge'), v: toU8a('bigger dog') },
{ k: toU8a('dodge'), v: toU8a('coin') }
];
let trie: Trie;
let back: Trie;
@@ -17,10 +27,7 @@ describe('snapshots', () => {
});
it('creates a snapshot of the (relevant) trie data', () => {
const values = [
{ k: toU8a('test'), v: toU8a('one') }
];
const root = trieRoot(values);
const root = trieRoot([values[0]]);
trie.put(values[0].k, values[0].v);
trie.put(values[0].k, toU8a('two'));
@@ -32,29 +39,38 @@ describe('snapshots', () => {
trie.snapshot(back, () => void 0);
expect(u8aToHex(back.getRoot())).toEqual(u8aToHex(root));
expect(trie.get(values[0].k)).toEqual(values[0].v);
expect(back.get(values[0].k)).toEqual(values[0].v);
});
it('creates a snapshot of the (relevant) data', () => {
const values = [
{ k: toU8a('one'), v: toU8a('testing') },
{ k: toU8a('two'), v: toU8a('testing with a much longer value here') },
{ k: toU8a('twzei'), v: toU8a('und Deutch') },
{ k: toU8a('do'), v: toU8a('do it') },
{ k: toU8a('dog'), v: toU8a('doggie') },
{ k: toU8a('dogge'), v: toU8a('bigger dog') },
{ k: toU8a('dodge'), v: toU8a('coin') }
];
const root = trieRoot(values);
values.forEach(({ k, v }) =>
trie.put(k, v)
);
values.forEach(({ k, v }) => trie.put(k, v));
trie.snapshot(back, () => void 0);
expect(u8aToHex(back.getRoot())).toEqual(u8aToHex(root));
values.forEach(({ k, v }) =>
expect(trie.get(k)).toEqual(v)
expect(back.get(k)).toEqual(v)
);
});
it('retrieves entries, which can re-create', () => {
const root = trieRoot(values);
values.forEach(({ k, v }) => trie.put(k, v));
const entries = trie.entries();
entries.forEach(([key, encoded, children]) => {
back.db.put(key, encoded);
console.log(u8aToHex(key), ...children.map((child) => `\n\t${u8aToHex(child)}`));
});
back.setRoot(root);
values.forEach(({ k, v }) =>
expect(back.get(k)).toEqual(v)
);
});
});
packages/trie-db/src/types.ts
+9 -1
View File
@@ -33,10 +33,18 @@ export type NodeNotEmpty = NodeKv | NodeBranch;
export type Node = NodeEmpty | NodeNotEmpty;
// root, encoded, childroots
export type TrieEntry = [Uint8Array, Uint8Array, Array<Uint8Array>];
export interface TrieDb extends TxDb {
readonly db: TxDb;
getRoot (): Uint8Array;
setRoot (rootHash: Uint8Array): void;
snapshot (dest: TrieDb, fn: ProgressCb): number;
getEntry (root?: Uint8Array): TrieEntry | null;
getNode (hash?: Uint8Array): Node;
entries (): Array<TrieEntry>;
snapshot (dest: TrieDb, fn?: ProgressCb): number;
}