From 1e561a4046daac897ada6142d4395d2105030754 Mon Sep 17 00:00:00 2001 From: Jaco Greeff Date: Sat, 8 Jun 2019 08:15:36 +0200 Subject: [PATCH] Add entries() method to trie (#418) * Add entries() method to trie * Add inv. getEntry for a single node (or root) --- packages/trie-db/src/Impl.ts | 49 +++++++++++++++++++++---- packages/trie-db/src/index.ts | 35 +++++++++++------- packages/trie-db/src/snapshot.spec.ts | 52 +++++++++++++++++---------- packages/trie-db/src/types.ts | 10 +++++- 4 files changed, 109 insertions(+), 37 deletions(-) diff --git a/packages/trie-db/src/Impl.ts b/packages/trie-db/src/Impl.ts index e114229d6..35b700459 100644 --- a/packages/trie-db/src/Impl.ts +++ b/packages/trie-db/src/Impl.ts @@ -4,7 +4,7 @@ import { TxDb, ProgressCb } from '@polkadot/db/types'; import { Codec } from '@polkadot/trie-codec/types'; -import { EncodedPath, TrieDb, Node, NodeBranch, NodeEncodedOrEmpty, NodeKv, NodeNotEmpty, NodeType } from './types'; +import { EncodedPath, TrieDb, Node, NodeBranch, NodeEncodedOrEmpty, NodeKv, NodeNotEmpty, NodeType, TrieEntry } from './types'; import substrateCodec from '@polkadot/trie-codec'; import { decodeNibbles, encodeNibbles, extractNodeKey } from '@polkadot/trie-codec/nibbles'; @@ -45,19 +45,42 @@ export default class Impl extends Checkpoint { this.constants = _constants; } + protected _entry (root: Uint8Array): TrieEntry | null { + const [encoded, node] = this._getNodeRaw(root); + + if (isNull(encoded) || isNull(node)) { + return null; + } + + return [root, encoded, node.filter((u8a) => u8a && u8a.length === 32) as Array]; + } + + protected _entries (root: Uint8Array, entries: Array = []): Array { + // l.debug(() => ['entries', { root }]); + const entry = this._entry(root); + + if (!entry) { + return entries; + } + + entries.push(entry); + entry[2].forEach((u8a) => this._entries(u8a, entries)); + + return entries; + } + protected _snapshot (dest: TrieDb, fn: ProgressCb | undefined, root: Uint8Array, keys: number, percent: number, depth: number): number { // l.debug(() => ['snapshot', { root }]); - const node = this._getNode(root); + const [encoded, node] = this._getNodeRaw(root); - if (isNull(node)) { + if (isNull(encoded) || isNull(node)) { return keys; } - keys++; - dest.db.put(root, encodeNode(this.codec, node)); + dest.db.put(root, encoded); - fn && fn({ keys, percent }); + fn && fn({ keys: ++keys, percent }); node.forEach((u8a) => { if (u8a && u8a.length === 32) { @@ -70,6 +93,20 @@ export default class Impl extends Checkpoint { return keys; } + protected _getNodeRaw (hash: Uint8Array | null): [Uint8Array | null, Node] { + // l.debug(() => ['_getNode', { hash }]); + + if (!hash || hash.length === 0 || keyEquals(hash, this.constants.EMPTY_HASH)) { + return [null, null]; + } else if (hash.length < 32) { + return [hash, decodeNode(this.codec, hash)]; + } + + const raw = this.db.get(hash); + + return [raw, decodeNode(this.codec, raw)]; + } + protected _getNode (hash: Uint8Array | null): Node { // l.debug(() => ['_getNode', { hash }]); diff --git a/packages/trie-db/src/index.ts b/packages/trie-db/src/index.ts index ad6a6bce5..09b14a679 100644 --- a/packages/trie-db/src/index.ts +++ b/packages/trie-db/src/index.ts @@ -4,7 +4,7 @@ import { TxDb, ProgressCb } from '@polkadot/db/types'; import { Codec } from '@polkadot/trie-codec/types'; -import { TrieDb, Node } from './types'; +import { TrieDb, Node, TrieEntry } from './types'; import MemoryDb from '@polkadot/db/Memory'; import { toNibbles } from '@polkadot/trie-codec/util'; @@ -121,30 +121,41 @@ export default class Trie extends Impl implements TrieDb { return this.rootHash; } - getNode (hash?: Uint8Array): Node { - return this._getNode(hash || this.rootHash); - } - setRoot (rootHash: Uint8Array): void { this.rootHash = rootHash; // return this._setRootNode(rootNode); } - snapshot (dest: TrieDb, fn?: ProgressCb): number { - const start = Date.now(); + getEntry (hash?: Uint8Array): TrieEntry | null { + return this._entry(hash || this.rootHash); + } + getNode (hash?: Uint8Array): Node { + return this._getNode(hash || this.rootHash); + } + + entries (): Array { + l.debug(() => 'retreiving trie entries'); + + const start = Date.now(); + const entries = this._entries(this.rootHash); + const elapsed = (Date.now() - start) / 1000; + + l.debug(() => `entries retrieved in ${elapsed.toFixed(2)}s, ${(entries.length / 1000).toFixed(2)}k keys`); + + return entries; + } + + snapshot (dest: TrieDb, fn?: ProgressCb): number { l.debug(() => 'creating current state snapshot'); + const start = Date.now(); const keys = this._snapshot(dest, fn, this.rootHash, 0, 0, 0); const elapsed = (Date.now() - start) / 1000; dest.setRoot(this.rootHash); - const newSize = dest.db.size(); - const percentage = 100 * (newSize / this.db.size()); - const sizeMB = newSize / (1024 * 1024); - - l.debug(() => `snapshot created in ${elapsed.toFixed(2)}s, ${(keys / 1000).toFixed(2)}k keys, ${sizeMB.toFixed(2)}MB (${percentage.toFixed(2)}%)`); + l.debug(() => `snapshot created in ${elapsed.toFixed(2)}s, ${(keys / 1000).toFixed(2)}k keys`); fn && fn({ isCompleted: true, diff --git a/packages/trie-db/src/snapshot.spec.ts b/packages/trie-db/src/snapshot.spec.ts index f15bcafcb..d20256442 100644 --- a/packages/trie-db/src/snapshot.spec.ts +++ b/packages/trie-db/src/snapshot.spec.ts @@ -8,6 +8,16 @@ import { u8aToHex, u8aToU8a as toU8a } from '@polkadot/util'; import Trie from '.'; describe('snapshots', () => { + const values = [ + { k: toU8a('test'), v: toU8a('one') }, + { k: toU8a('one'), v: toU8a('testing') }, + { k: toU8a('two'), v: toU8a('testing with a much longer value here') }, + { k: toU8a('twzei'), v: toU8a('und Deutch') }, + { k: toU8a('do'), v: toU8a('do it') }, + { k: toU8a('dog'), v: toU8a('doggie') }, + { k: toU8a('dogge'), v: toU8a('bigger dog') }, + { k: toU8a('dodge'), v: toU8a('coin') } + ]; let trie: Trie; let back: Trie; @@ -17,10 +27,7 @@ describe('snapshots', () => { }); it('creates a snapshot of the (relevant) trie data', () => { - const values = [ - { k: toU8a('test'), v: toU8a('one') } - ]; - const root = trieRoot(values); + const root = trieRoot([values[0]]); trie.put(values[0].k, values[0].v); trie.put(values[0].k, toU8a('two')); @@ -32,29 +39,38 @@ describe('snapshots', () => { trie.snapshot(back, () => void 0); expect(u8aToHex(back.getRoot())).toEqual(u8aToHex(root)); - expect(trie.get(values[0].k)).toEqual(values[0].v); + expect(back.get(values[0].k)).toEqual(values[0].v); }); it('creates a snapshot of the (relevant) data', () => { - const values = [ - { k: toU8a('one'), v: toU8a('testing') }, - { k: toU8a('two'), v: toU8a('testing with a much longer value here') }, - { k: toU8a('twzei'), v: toU8a('und Deutch') }, - { k: toU8a('do'), v: toU8a('do it') }, - { k: toU8a('dog'), v: toU8a('doggie') }, - { k: toU8a('dogge'), v: toU8a('bigger dog') }, - { k: toU8a('dodge'), v: toU8a('coin') } - ]; const root = trieRoot(values); - values.forEach(({ k, v }) => - trie.put(k, v) - ); + values.forEach(({ k, v }) => trie.put(k, v)); trie.snapshot(back, () => void 0); expect(u8aToHex(back.getRoot())).toEqual(u8aToHex(root)); + values.forEach(({ k, v }) => - expect(trie.get(k)).toEqual(v) + expect(back.get(k)).toEqual(v) + ); + }); + + it('retrieves entries, which can re-create', () => { + const root = trieRoot(values); + + values.forEach(({ k, v }) => trie.put(k, v)); + + const entries = trie.entries(); + + entries.forEach(([key, encoded, children]) => { + back.db.put(key, encoded); + console.log(u8aToHex(key), ...children.map((child) => `\n\t${u8aToHex(child)}`)); + }); + + back.setRoot(root); + + values.forEach(({ k, v }) => + expect(back.get(k)).toEqual(v) ); }); }); diff --git a/packages/trie-db/src/types.ts b/packages/trie-db/src/types.ts index 8808b41c3..f9e9b523b 100644 --- a/packages/trie-db/src/types.ts +++ b/packages/trie-db/src/types.ts @@ -33,10 +33,18 @@ export type NodeNotEmpty = NodeKv | NodeBranch; export type Node = NodeEmpty | NodeNotEmpty; +// root, encoded, childroots +export type TrieEntry = [Uint8Array, Uint8Array, Array]; + export interface TrieDb extends TxDb { readonly db: TxDb; getRoot (): Uint8Array; setRoot (rootHash: Uint8Array): void; - snapshot (dest: TrieDb, fn: ProgressCb): number; + + getEntry (root?: Uint8Array): TrieEntry | null; + getNode (hash?: Uint8Array): Node; + + entries (): Array; + snapshot (dest: TrieDb, fn?: ProgressCb): number; }