Re-implementation of Patricia-Merkle Trie (#203)
* Start of trie-db re-implementation * WIP commit * More WIP * WIP, seems complete, untested * Ported trie-db operational * Add Ethereum trie tests * Use u8aConcat to construct u8a * Basic RocksDb interface * Combine engines into DB, expand tests * Update Trie with transaction wrapper * LevelDb (sync) interface * Remove unused (for now) package (compatibility) * trie-d -> trie-db-old, trie-db-ext -> trie-db * Version bump * Skip old trie tests (package to be removed) * Skip tests (fixed) * Remove non-used RocksDb spec * Skip FileFlat tests (deprecated?)
This commit is contained in:
Jaco Greeff
@@ -1,837 +0,0 @@
|
||||
// Copyright 2015-2018 authors & contributors of https://github.com/ethereumjs/merkle-patricia-tree
|
||||
// Copyright 2017-2018 @polkadot/trie-db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the MPL-2.0 license. See the LICENSE file for details.
|
||||
|
||||
import { HashFn, NodeFactory, SemaphorePromise } from './types';
|
||||
|
||||
import async from 'async';
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
import levelup from 'levelup';
|
||||
import memdown from 'memdown';
|
||||
// import logger from '@polkadot/util/logger';
|
||||
import assert from '@polkadot/util/assert';
|
||||
import isU8a from '@polkadot/util/is/u8a';
|
||||
import promisify from '@polkadot/util/promisify';
|
||||
// import u8aToHex from '@polkadot/util/u8a/toHex';
|
||||
import keccakAsU8a from '@polkadot/util-crypto/keccak/asU8a';
|
||||
import decodeRlp from '@polkadot/util-rlp/decode';
|
||||
|
||||
import { EMPTY_ROOT_U8A, EMPTY_ROOT_STR } from './constants';
|
||||
import encoder from './encoder';
|
||||
import { nibblesMatchingLength, nibblesIsEqual, nibblesFromU8a } from './nibbles';
|
||||
import ReadStream from './streams/TrieRead';
|
||||
import createFactory from './nodeFactory';
|
||||
import isRawNode from './util/isRawNode';
|
||||
import prioritizedTaskExecutor from './util/taskExecutor';
|
||||
import semaphore from './util/semaphore';
|
||||
|
||||
type RawPutFn = (key: Uint8Array, value: Uint8Array) => Promise<any>;
|
||||
|
||||
// const l = logger('trie/base');
|
||||
|
||||
export default class BaseTrie {
|
||||
// @ts-ignore we are assigning via this.root in constructor
|
||||
_root: Uint8Array;
|
||||
_getDBs: any[]; // FIXME
|
||||
_putDBs: any[]; // FIXME
|
||||
dbDown: any; // FIXME
|
||||
db: any; // FIXME
|
||||
hashing: HashFn;
|
||||
nodeFactory: NodeFactory;
|
||||
putRaw: RawPutFn;
|
||||
semaphore: SemaphorePromise;
|
||||
|
||||
constructor (db: any, root: Uint8Array = EMPTY_ROOT_U8A, hashing: HashFn = keccakAsU8a) {
|
||||
this.putRaw = this._putRaw;
|
||||
|
||||
this.semaphore = semaphore(1);
|
||||
this.dbDown = db || memdown();
|
||||
this.db = levelup(encoder(this.dbDown));
|
||||
this._getDBs = [this.db];
|
||||
this._putDBs = [this.db];
|
||||
this.hashing = hashing;
|
||||
this.nodeFactory = createFactory(hashing);
|
||||
this.root = root;
|
||||
|
||||
// l.debug(() => ['Created BaseTrie', typeof db, u8aToHex(root)]);
|
||||
}
|
||||
|
||||
get root (): Uint8Array {
|
||||
return this._root;
|
||||
}
|
||||
|
||||
set root (value: Uint8Array) {
|
||||
if (value) {
|
||||
assert(isU8a(value) && value.length === 32, 'Invalid root length. Roots are Uint8Array 32 bytes');
|
||||
} else {
|
||||
value = EMPTY_ROOT_U8A;
|
||||
}
|
||||
|
||||
this._root = value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets a value given a `key`
|
||||
* @method get
|
||||
* @param {Buffer|String} key - the key to search for
|
||||
* @param {Function} cb A callback `Function` which is given the arguments `err` - for errors that may have occured and `value` - the found value in a `Buffer` or if no value was found `null`
|
||||
*/
|
||||
async get (key: Uint8Array): Promise<Uint8Array | null> {
|
||||
// l.debug(() => ['get', 'key ->', u8aToHex(key)]);
|
||||
|
||||
const result = await this.findPath(key);
|
||||
|
||||
let value = null;
|
||||
|
||||
if (result && result.node && result.remainder.length === 0) {
|
||||
value = result.node.value;
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stores a given `value` at the given `key`
|
||||
* @method put
|
||||
* @param {Buffer|String} key
|
||||
* @param {Buffer|String} Value
|
||||
* @returns {Function} Promise<void>
|
||||
*/
|
||||
async put (key: Uint8Array, value: Uint8Array) {
|
||||
// l.debug(() => ['put', 'key ->', u8aToHex(key), 'value ->', u8aToHex(value, 256)]);
|
||||
|
||||
if (!value || value.toString() === '') {
|
||||
return this.del(key);
|
||||
}
|
||||
|
||||
return this.semaphore(async () => {
|
||||
if (this.root.toString() === EMPTY_ROOT_STR) {
|
||||
return this._createInitialNode(key, value); // if no root initialize this trie
|
||||
}
|
||||
|
||||
// first try to find the give key or its nearst node
|
||||
const result = await this.findPath(key);
|
||||
|
||||
return this._updateNode(key, value, result && result.remainder, result && result.stack);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* deletes a value given a `key`
|
||||
* @method del
|
||||
* @param {Buffer|String} key
|
||||
* @param {Function} callback the callback `Function`
|
||||
*/
|
||||
del (key: Uint8Array) {
|
||||
// l.debug(() => ['del', 'key ->', u8aToHex(key)]);
|
||||
|
||||
return this.semaphore(async () => {
|
||||
const result = await this.findPath(key);
|
||||
|
||||
if (result && result.node) {
|
||||
return this._deleteNode(key, result.stack);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves a raw value in the underlying db
|
||||
* @method getRaw
|
||||
* @param {Buffer} key
|
||||
* @param {Function} callback A callback `Function`, which is given the arguments `err` - for errors that may have occured and `value` - the found value in a `Buffer` or if no value was found `null`.
|
||||
*/
|
||||
async getRaw (key: Uint8Array): Promise<Uint8Array> {
|
||||
let found: Uint8Array;
|
||||
|
||||
await Promise.all(
|
||||
this._getDBs.map(async (db) => {
|
||||
if (!found) {
|
||||
try {
|
||||
found = await db.get(key);
|
||||
} catch (error) {
|
||||
// l.error('getRaw', 'key ->', u8aToHex(key), error);
|
||||
// not doing anything
|
||||
}
|
||||
}
|
||||
|
||||
return found;
|
||||
})
|
||||
);
|
||||
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
return found;
|
||||
}
|
||||
|
||||
// retrieves a node from dbs by hash
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
async _lookupNode (node) {
|
||||
if (isRawNode(node)) {
|
||||
return this.nodeFactory.fromRaw(node);
|
||||
}
|
||||
|
||||
let value = await this.getRaw(node);
|
||||
|
||||
if (value) {
|
||||
try {
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
value = this.nodeFactory.fromRaw(decodeRlp(value));
|
||||
} catch (error) {
|
||||
// l.error('_lookupNode decoding', typeof value, value);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
// TODO: remove the proxy method when changing the caching
|
||||
async _putRaw (key: Uint8Array, value: Uint8Array) {
|
||||
return Promise.all(
|
||||
this._putDBs.map((db) => {
|
||||
return db.put(key, value);
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes a raw value in the underlying db
|
||||
* @method delRaw
|
||||
* @param {Buffer|String} key
|
||||
* @param {Function} callback A callback `Function`, which is given the argument `err` - for errors that may have occured
|
||||
*/
|
||||
delRaw (key: Uint8Array) {
|
||||
return Promise.all(
|
||||
this._putDBs.map((db) => {
|
||||
return db.del(key);
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
// writes a single node to dbs
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
_putNode (node) {
|
||||
const hash = node.hash();
|
||||
const serialized = node.serialize();
|
||||
|
||||
return this._putRaw(hash, serialized);
|
||||
}
|
||||
|
||||
// writes many nodes to db
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
_batchNodes (opStack) {
|
||||
return Promise.all(
|
||||
this._putDBs.map((db) => {
|
||||
return db.batch(opStack);
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Trys to find a path to the node for the given key
|
||||
* It returns a `stack` of nodes to the closet node
|
||||
* @method findPath
|
||||
* @param {String|Buffer} - key - the search key
|
||||
* @param {Function} - cb - the callback function. Its is given the following
|
||||
* arguments
|
||||
* - err - any errors encontered
|
||||
* - node - the last node found
|
||||
* - remainder - the remaining key nibbles not accounted for
|
||||
* - stack - an array of nodes that forms the path to node we are searching for
|
||||
*/
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
findPath (_targetKey) {
|
||||
const stack: any[] = []; // FIXME
|
||||
const targetKey = nibblesFromU8a(_targetKey);
|
||||
|
||||
// l.debug(() => ['findPath', targetKey.join(', ')]);
|
||||
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
const processNode = (nodeRef, node, keyProgress, walkController) => {
|
||||
const nodeKey = node.key || [];
|
||||
const remainder = targetKey.slice(nibblesMatchingLength(keyProgress, targetKey));
|
||||
const matchingLen = nibblesMatchingLength(remainder, nodeKey);
|
||||
|
||||
stack.push(node);
|
||||
|
||||
if (node.type === 'branch') {
|
||||
if (remainder.length === 0) {
|
||||
walkController.return(null, { node, remainder: [], stack });
|
||||
// we exhausted the key without finding a node
|
||||
} else {
|
||||
const branchIndex = remainder[0];
|
||||
const branchNode = node.getValue(branchIndex);
|
||||
|
||||
if (!branchNode) {
|
||||
// there are no more nodes to find and we didn't find the key
|
||||
walkController.return(null, { node: null, remainder, stack });
|
||||
} else {
|
||||
// node found, continuing search
|
||||
walkController.only(branchIndex);
|
||||
}
|
||||
}
|
||||
} else if (node.type === 'leaf') {
|
||||
if (nibblesIsEqual(remainder, nodeKey)) {
|
||||
// keys match, return node with empty key
|
||||
walkController.return(null, { node, remainder: [], stack });
|
||||
} else {
|
||||
// reached leaf but keys dont match
|
||||
walkController.return(null, { node: null, remainder, stack });
|
||||
}
|
||||
} else if (node.type === 'extention') {
|
||||
if (matchingLen !== nodeKey.length) {
|
||||
// keys dont match, fail
|
||||
walkController.return(null, { node: null, remainder, stack });
|
||||
} else {
|
||||
// keys match, continue search
|
||||
walkController.next();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
return promisify(this, this._walkTrie, this.root, processNode);
|
||||
}
|
||||
|
||||
/*
|
||||
* Finds all nodes that store k,v values
|
||||
*/
|
||||
// NOTE seems unused
|
||||
// _findNode = function (key, root, stack, cb) {
|
||||
// this.findPath(key, (...args) => {
|
||||
// cb.apply(null, args);
|
||||
// });
|
||||
// };
|
||||
|
||||
/*
|
||||
* Finds all nodes that store k,v values
|
||||
*/
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
_findValueNodes (onFound, cb) {
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
this._walkTrie(this.root, (nodeRef, node, key, walkController) => { // tslint:disable-line
|
||||
let fullKey = key;
|
||||
|
||||
if (node.key) {
|
||||
fullKey = key.concat(node.key);
|
||||
}
|
||||
|
||||
if (node.type === 'leaf') {
|
||||
// found leaf node!
|
||||
onFound(nodeRef, node, fullKey, walkController.next);
|
||||
} else if (node.type === 'branch' && node.value) {
|
||||
// found branch with value
|
||||
onFound(nodeRef, node, fullKey, walkController.next);
|
||||
} else {
|
||||
// keep looking for value nodes
|
||||
walkController.next();
|
||||
}
|
||||
}, cb);
|
||||
}
|
||||
|
||||
/*
|
||||
* Finds all nodes that are stored directly in the db
|
||||
* (some nodes are stored raw inside other nodes)
|
||||
*/
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
_findDbNodes (onFound, cb) {
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
this._walkTrie(this.root, (nodeRef, node, key, walkController) => { // tslint:disable-line
|
||||
if (isRawNode(nodeRef)) {
|
||||
walkController.next();
|
||||
} else {
|
||||
onFound(nodeRef, node, key, walkController.next);
|
||||
}
|
||||
}, cb);
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates a node
|
||||
* @method _updateNode
|
||||
* @param {Buffer} key
|
||||
* @param {Buffer| String} value
|
||||
* @param {Array} keyRemainder
|
||||
* @param {Array} stack -
|
||||
* @param {Function} cb - the callback
|
||||
*/
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
async _updateNode (key, value, keyRemainder, stack) {
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
const toSave = [];
|
||||
const lastNode = stack.pop();
|
||||
|
||||
// add the new nodes
|
||||
key = nibblesFromU8a(key);
|
||||
|
||||
// Check if the last node is a leaf and the key matches to this
|
||||
let matchLeaf = false;
|
||||
|
||||
if (lastNode.type === 'leaf') {
|
||||
let l = 0;
|
||||
|
||||
for (let i = 0; i < stack.length; i++) {
|
||||
const n = stack[i];
|
||||
|
||||
if (n.type === 'branch') {
|
||||
l++;
|
||||
} else {
|
||||
l += n.key.length;
|
||||
}
|
||||
}
|
||||
|
||||
if ((nibblesMatchingLength(lastNode.key, key.slice(l)) === lastNode.key.length) && (keyRemainder.length === 0)) {
|
||||
matchLeaf = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (matchLeaf) {
|
||||
// just updating a found value
|
||||
lastNode.value = value;
|
||||
stack.push(lastNode);
|
||||
} else if (lastNode.type === 'branch') {
|
||||
stack.push(lastNode);
|
||||
|
||||
if (keyRemainder !== 0) {
|
||||
// add an extention to a branch node
|
||||
keyRemainder.shift();
|
||||
// create a new leaf
|
||||
stack.push(
|
||||
this.nodeFactory.fromLeaf(keyRemainder, value)
|
||||
);
|
||||
} else {
|
||||
lastNode.value = value;
|
||||
}
|
||||
} else {
|
||||
// create a branch node
|
||||
const lastKey = lastNode.key;
|
||||
const matchingLength = nibblesMatchingLength(lastKey, keyRemainder);
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
const newBranchNode = this.nodeFactory.fromBranch();
|
||||
|
||||
// create a new extention node
|
||||
if (matchingLength !== 0) {
|
||||
const newKey = lastNode.key.slice(0, matchingLength);
|
||||
const newExtNode = this.nodeFactory.fromExtention(newKey, value);
|
||||
|
||||
stack.push(newExtNode);
|
||||
lastKey.splice(0, matchingLength);
|
||||
keyRemainder.splice(0, matchingLength);
|
||||
}
|
||||
|
||||
stack.push(newBranchNode);
|
||||
|
||||
if (lastKey.length !== 0) {
|
||||
const branchKey = lastKey.shift();
|
||||
|
||||
if (lastKey.length !== 0 || lastNode.type === 'leaf') {
|
||||
// shriking extention or leaf
|
||||
lastNode.key = lastKey;
|
||||
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
const formatedNode = this._formatNode(lastNode, false, toSave);
|
||||
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
newBranchNode.setValue(branchKey, formatedNode);
|
||||
} else {
|
||||
// remove extention or attaching
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
this._formatNode(lastNode, false, true, toSave);
|
||||
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
newBranchNode.setValue(branchKey, lastNode.value);
|
||||
}
|
||||
} else {
|
||||
newBranchNode.value = lastNode.value;
|
||||
}
|
||||
|
||||
if (keyRemainder.length !== 0) {
|
||||
keyRemainder.shift();
|
||||
|
||||
// add a leaf node to the new branch node
|
||||
stack.push(
|
||||
this.nodeFactory.fromLeaf(keyRemainder, value)
|
||||
);
|
||||
} else {
|
||||
newBranchNode.value = value;
|
||||
}
|
||||
}
|
||||
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
return this._saveStack(key, stack, toSave);
|
||||
}
|
||||
|
||||
// walk tree
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
async _walkTrie (root, onNode, onDone) {
|
||||
root = root || this.root;
|
||||
|
||||
let isAborted: boolean = false;
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
let returnValues: any[] = [];
|
||||
|
||||
if (root.toString() === EMPTY_ROOT_STR) {
|
||||
onDone();
|
||||
return;
|
||||
}
|
||||
|
||||
// the maximum pool size should be high enough to utilise the parallelizability of reading nodes from disk and
|
||||
// low enough to utilize the prioritisation of node lookup.
|
||||
const maxPoolSize = 500;
|
||||
const taskExecutor = prioritizedTaskExecutor(maxPoolSize);
|
||||
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
const processNode = (nodeRef, node, key, cb) => {
|
||||
if (!node || isAborted) {
|
||||
return cb();
|
||||
}
|
||||
|
||||
const self = this;
|
||||
let isStopped = false;
|
||||
|
||||
key = key || [];
|
||||
|
||||
const walkController = {
|
||||
stop: function () {
|
||||
isStopped = true;
|
||||
cb();
|
||||
},
|
||||
// end all traversal and return values to the onDone cb
|
||||
return: function () {
|
||||
isAborted = true;
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
returnValues = arguments;
|
||||
cb();
|
||||
},
|
||||
next: async function () {
|
||||
if (isAborted || isStopped) {
|
||||
return cb();
|
||||
}
|
||||
|
||||
const children = node.getChildren();
|
||||
|
||||
// @ts-ignore FIXME callback
|
||||
async.forEachOf(children, (childData: any[], index: number, cb) => {
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
const keyExtension = childData[0];
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
const childRef = childData[1];
|
||||
const childKey = key.concat(keyExtension);
|
||||
const priority = childKey.length;
|
||||
|
||||
// tslint:disable-next-line
|
||||
taskExecutor(priority, async (taskCallback) => {
|
||||
const childNode = await self._lookupNode(childRef);
|
||||
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
taskCallback();
|
||||
processNode(childRef, childNode, childKey, cb);
|
||||
});
|
||||
}, cb);
|
||||
},
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
only: function (childIndex) {
|
||||
const childRef = node.getValue(childIndex);
|
||||
const childKey = key.slice();
|
||||
|
||||
childKey.push(childIndex);
|
||||
|
||||
const priority = childKey.length;
|
||||
|
||||
// tslint:disable-next-line
|
||||
taskExecutor(priority, async (taskCallback) => {
|
||||
const childNode = await self._lookupNode(childRef);
|
||||
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
taskCallback();
|
||||
processNode(childRef, childNode, childKey, cb);
|
||||
});
|
||||
}
|
||||
};
|
||||
onNode(nodeRef, node, key, walkController);
|
||||
};
|
||||
|
||||
const node = await this._lookupNode(root);
|
||||
|
||||
processNode(root, node, null, (error: Error | null) => {
|
||||
if (error) {
|
||||
onDone(error);
|
||||
return;
|
||||
}
|
||||
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
onDone.apply(null, returnValues);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* saves a stack
|
||||
* @method _saveStack
|
||||
* @param {Array} key - the key. Should follow the stack
|
||||
* @param {Array} stack - a stack of nodes to the value given by the key
|
||||
* @param {Array} opStack - a stack of levelup operations to commit at the end of this funciton
|
||||
* @param {Function} cb
|
||||
*/
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
_saveStack (key, stack, opStack: any[]) {
|
||||
let lastRoot;
|
||||
|
||||
// update nodes
|
||||
while (stack.length) {
|
||||
const node = stack.pop();
|
||||
|
||||
if (node.type === 'leaf') {
|
||||
key.splice(key.length - node.key.length);
|
||||
} else if (node.type === 'extention') {
|
||||
key.splice(key.length - node.key.length);
|
||||
if (lastRoot) {
|
||||
node.value = lastRoot;
|
||||
}
|
||||
} else if (node.type === 'branch') {
|
||||
if (lastRoot) {
|
||||
const branchKey = key.pop();
|
||||
|
||||
node.setValue(branchKey, lastRoot);
|
||||
}
|
||||
}
|
||||
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
lastRoot = this._formatNode(node, stack.length === 0, opStack);
|
||||
}
|
||||
|
||||
if (lastRoot) {
|
||||
this.root = lastRoot;
|
||||
}
|
||||
|
||||
return this._batchNodes(opStack);
|
||||
}
|
||||
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
async _deleteNode (key: Uint8Array, stack) {
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
const processBranchNode = (key, branchKey, branchNode, parentNode, stack) => {
|
||||
// branchNode is the node ON the branch node not THE branch node
|
||||
const branchNodeKey = branchNode.key;
|
||||
|
||||
if (!parentNode || parentNode.type === 'branch') {
|
||||
// branch->?
|
||||
if (parentNode) {
|
||||
stack.push(parentNode);
|
||||
}
|
||||
|
||||
if (branchNode.type === 'branch') {
|
||||
// create an extention node
|
||||
// branch->extention->branch
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
stack.push(this.nodeFactory.fromExtention([branchKey], null));
|
||||
key.push(branchKey);
|
||||
} else {
|
||||
// branch key is an extention or a leaf
|
||||
// branch->(leaf or extention)
|
||||
branchNodeKey.unshift(branchKey);
|
||||
branchNode.key = branchNodeKey;
|
||||
|
||||
// hackery. This is equvilant to array.concat except we need keep the
|
||||
// rerfance to the `key` that was passed in.
|
||||
branchNodeKey.unshift(0);
|
||||
branchNodeKey.unshift(key.length);
|
||||
key.splice.apply(key, branchNodeKey);
|
||||
}
|
||||
stack.push(branchNode);
|
||||
} else {
|
||||
// parent is a extention
|
||||
let parentKey = parentNode.key;
|
||||
|
||||
if (branchNode.type === 'branch') {
|
||||
// ext->branch
|
||||
parentKey.push(branchKey);
|
||||
key.push(branchKey);
|
||||
parentNode.key = parentKey;
|
||||
stack.push(parentNode);
|
||||
} else {
|
||||
// branch node is an leaf or extention and parent node is an exstention
|
||||
// add two keys together
|
||||
// dont push the parent node
|
||||
branchNodeKey.unshift(branchKey);
|
||||
key = key.concat(branchNodeKey);
|
||||
parentKey = parentKey.concat(branchNodeKey);
|
||||
branchNode.key = parentKey;
|
||||
}
|
||||
stack.push(branchNode);
|
||||
}
|
||||
|
||||
return key;
|
||||
};
|
||||
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
const opStack: any[] = [];
|
||||
let lastNode = stack.pop();
|
||||
let parentNode = stack.pop();
|
||||
|
||||
if (!Array.isArray(key)) {
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
key = nibblesFromU8a(key);
|
||||
}
|
||||
|
||||
if (!parentNode) {
|
||||
// the root here has to be a leaf.
|
||||
this.root = EMPTY_ROOT_U8A;
|
||||
} else {
|
||||
if (lastNode.type === 'branch') {
|
||||
lastNode.value = null;
|
||||
} else {
|
||||
// the lastNode has to be a leaf if its not a branch. And a leaf's parent
|
||||
// if it has one must be a branch.
|
||||
const lastNodeKey = lastNode.key;
|
||||
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
key.splice(key.length - lastNodeKey.length);
|
||||
// delete the value
|
||||
this._formatNode(lastNode, false, true, opStack);
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
parentNode.setValue(key.pop(), null);
|
||||
lastNode = parentNode;
|
||||
parentNode = stack.pop();
|
||||
}
|
||||
|
||||
// nodes on the branch
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
const branchNodes = [];
|
||||
|
||||
// count the number of nodes on the branch
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
lastNode.raw.forEach(function (node, i) {
|
||||
const val = lastNode.getValue(i);
|
||||
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
if (val) branchNodes.push([i, val]);
|
||||
});
|
||||
|
||||
// if there is only one branch node left, collapse the branch node
|
||||
if (branchNodes.length === 1) {
|
||||
// add the one remaing branch node to node above it
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
const branchNode = branchNodes[0][1];
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
const branchNodeKey = branchNodes[0][0];
|
||||
|
||||
// look up node
|
||||
const foundNode = await this._lookupNode(branchNode);
|
||||
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
key = processBranchNode(key, branchNodeKey, foundNode, parentNode, stack, opStack);
|
||||
|
||||
return this._saveStack(key, stack, opStack);
|
||||
} else {
|
||||
// simple removing a leaf and recaluclation the stack
|
||||
if (parentNode) {
|
||||
stack.push(parentNode);
|
||||
}
|
||||
|
||||
stack.push(lastNode);
|
||||
|
||||
return this._saveStack(key, stack, opStack);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Creates the initial node from an empty tree
|
||||
_createInitialNode (key: Uint8Array, value: Uint8Array) {
|
||||
const newNode = this.nodeFactory.fromLeaf(key, value);
|
||||
|
||||
this.root = newNode.hash();
|
||||
|
||||
return this._putNode(newNode);
|
||||
}
|
||||
|
||||
// formats node to be saved by levelup.batch.
|
||||
// returns either the hash that will be used key or the rawNode
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
_formatNode (node: any, topLevel: any, remove: any, opStack: any) {
|
||||
if (arguments.length === 3) {
|
||||
opStack = remove;
|
||||
remove = false;
|
||||
}
|
||||
|
||||
const rlpNode = node.serialize();
|
||||
|
||||
if (rlpNode.length >= 32 || topLevel) {
|
||||
const hashRoot = node.hash();
|
||||
|
||||
// FIXME Actually 2 things wrong here - this is on checkpoint, so one higher-level class...
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
if (remove && this.isCheckpoint) {
|
||||
opStack.push({
|
||||
type: 'del',
|
||||
key: hashRoot
|
||||
});
|
||||
} else {
|
||||
opStack.push({
|
||||
type: 'put',
|
||||
key: hashRoot,
|
||||
value: rlpNode
|
||||
});
|
||||
}
|
||||
|
||||
return hashRoot;
|
||||
}
|
||||
|
||||
return node.raw;
|
||||
}
|
||||
|
||||
/**
|
||||
* The `data` event is given an `Object` hat has two properties; the `key` and the `value`. Both should be Buffers.
|
||||
* @method createReadStream
|
||||
* @return {stream.Readable} Returns a [stream](https://nodejs.org/dist/latest-v5.x/docs/api/stream.html#stream_class_stream_readable) of the contents of the `trie`
|
||||
*/
|
||||
createReadStream () {
|
||||
return new ReadStream(this);
|
||||
}
|
||||
|
||||
// creates a new trie backed by the same db
|
||||
// and starting at the same root
|
||||
copy () {
|
||||
// l.debug(() => ['Copying BaseTrie', typeof this.dbDown, u8aToHex(this.root)]);
|
||||
|
||||
return new BaseTrie(this.dbDown, this.root, this.hashing);
|
||||
}
|
||||
|
||||
/**
|
||||
* The given hash of operations (key additions or deletions) are executed on the DB
|
||||
* @method batch
|
||||
* @example
|
||||
* const ops = [
|
||||
* { type: 'del', key: 'father' }
|
||||
* , { type: 'put', key: 'name', value: 'Yuri Irsenovich Kim' }
|
||||
* , { type: 'put', key: 'dob', value: '16 February 1941' }
|
||||
* , { type: 'put', key: 'spouse', value: 'Kim Young-sook' }
|
||||
* , { type: 'put', key: 'occupation', value: 'Clown' }
|
||||
* ]
|
||||
* trie.batch(ops)
|
||||
* @param {Array} ops
|
||||
* @param {Function} cb
|
||||
*/
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
batch (ops: any[]) {
|
||||
return Promise.all(
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
ops.map((op) => {
|
||||
if (op.type === 'put') {
|
||||
return this.put(op.key, op.value);
|
||||
} else if (op.type === 'del') {
|
||||
return this.del(op.key);
|
||||
}
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if a given root exists
|
||||
* @method checkRoot
|
||||
* @param {Buffer} root
|
||||
* @param {Function} cb
|
||||
*/
|
||||
async checkRoot (root: Uint8Array): Promise<boolean> {
|
||||
const value = await this._lookupNode(root);
|
||||
|
||||
return !!value;
|
||||
}
|
||||
}
|
||||
@@ -1,163 +0,0 @@
|
||||
// Copyright 2015-2018 authors & contributors of https://github.com/ethereumjs/merkle-patricia-tree
|
||||
// Copyright 2017-2018 @polkadot/trie-db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the MPL-2.0 license. See the LICENSE file for details.
|
||||
|
||||
import { HashFn } from './types';
|
||||
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
import levelws from 'level-ws';
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
import levelup from 'levelup';
|
||||
import memdown from 'memdown';
|
||||
import assert from '@polkadot/util/assert';
|
||||
// import logger from '@polkadot/util/logger';
|
||||
// import u8aToHex from '@polkadot/util/u8a/toHex';
|
||||
|
||||
import BaseTrie from './BaseTrie';
|
||||
import encoder from './encoder';
|
||||
import ScratchReadStream from './streams/ScratchRead';
|
||||
|
||||
// const l = logger('trie/checkpt');
|
||||
|
||||
function putRaw (key: Uint8Array, value: Uint8Array) {
|
||||
return Promise.all(
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
this.__putDBs.map((db) => {
|
||||
return db.put(key, value);
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
export default class CheckpointTrie extends BaseTrie {
|
||||
_checkpoints: Array<Uint8Array>;
|
||||
_scratch: any; // FIXME
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
__putDBs: any[]; // tslint:disable-line
|
||||
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
constructor (db: any, root?: Uint8Array, hashing?: HashFn) {
|
||||
super(db, root, hashing);
|
||||
|
||||
this._scratch = null;
|
||||
this._checkpoints = [];
|
||||
|
||||
// l.debug(() => 'Created CheckpointTrie');
|
||||
}
|
||||
|
||||
get isCheckpoint () {
|
||||
return !!this._checkpoints.length;
|
||||
}
|
||||
|
||||
copy () {
|
||||
// l.debug(() => ['Copying CheckpointTrie', typeof this.dbDown, u8aToHex(this.root)]);
|
||||
|
||||
return new CheckpointTrie(this.dbDown, this.root, this.hashing);
|
||||
}
|
||||
|
||||
async checkpoint () {
|
||||
// l.debug(() => 'Marking checkpoint');
|
||||
|
||||
const wasCheckpoint = this.isCheckpoint;
|
||||
|
||||
this._checkpoints.push(this.root);
|
||||
|
||||
if (!wasCheckpoint && this.isCheckpoint) {
|
||||
this._enterCpMode();
|
||||
}
|
||||
}
|
||||
|
||||
commit () {
|
||||
// l.debug(() => 'Committing checkpoint');
|
||||
|
||||
return this.semaphore(async () => {
|
||||
assert(this.isCheckpoint, 'trying to commit when not checkpointed');
|
||||
|
||||
this._checkpoints.pop();
|
||||
|
||||
if (this.isCheckpoint) {
|
||||
return;
|
||||
}
|
||||
|
||||
return this._exitCpMode(true);
|
||||
});
|
||||
}
|
||||
|
||||
revert () {
|
||||
// l.debug(() => 'Reverting checkpoint');
|
||||
|
||||
return this.semaphore(async () => {
|
||||
if (!this.isCheckpoint) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.root = this._checkpoints.pop() as Uint8Array;
|
||||
|
||||
if (this.isCheckpoint) {
|
||||
return;
|
||||
}
|
||||
|
||||
return this._exitCpMode(false);
|
||||
});
|
||||
}
|
||||
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
createScratchReadStream (scratch) {
|
||||
const trie = this.copy();
|
||||
|
||||
scratch = scratch || this._scratch;
|
||||
|
||||
// only read from the scratch
|
||||
trie._getDBs = [scratch];
|
||||
trie._scratch = scratch;
|
||||
|
||||
return new ScratchReadStream(trie);
|
||||
}
|
||||
|
||||
_enterCpMode () {
|
||||
// l.debug(() => 'Entering checkpoint mode');
|
||||
|
||||
this._scratch = levelup(encoder(memdown()));
|
||||
|
||||
this._getDBs = [this._scratch].concat(this._getDBs);
|
||||
this.__putDBs = this._putDBs;
|
||||
this._putDBs = [this._scratch];
|
||||
this._putRaw = this.putRaw;
|
||||
this.putRaw = putRaw;
|
||||
}
|
||||
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
_exitCpMode (commitState) {
|
||||
// l.debug(() => 'Exiting checkpoint mode');
|
||||
|
||||
const scratch = this._scratch;
|
||||
this._scratch = null;
|
||||
this._getDBs = this._getDBs.slice(1);
|
||||
this._putDBs = this.__putDBs;
|
||||
this.putRaw = this._putRaw;
|
||||
|
||||
if (!commitState) {
|
||||
return;
|
||||
}
|
||||
|
||||
return Promise.all(
|
||||
this._putDBs.map((db) => {
|
||||
return new Promise((resolve) => {
|
||||
if (!db.createWriteStream) {
|
||||
db = levelws(db);
|
||||
}
|
||||
|
||||
this
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
.createScratchReadStream(scratch)
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
.pipe(
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
db.createWriteStream(encoder.options)
|
||||
)
|
||||
.on('close', resolve);
|
||||
});
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,173 +0,0 @@
|
||||
// Copyright 2015-2018 authors & contributors of https://github.com/ethereumjs/merkle-patricia-tree
|
||||
// Copyright 2017-2018 @polkadot/trie-db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the MPL-2.0 license. See the LICENSE file for details.
|
||||
|
||||
import { DecodedRlp, Trie$Node$Type, HashFn } from './types';
|
||||
|
||||
import isU8a from '@polkadot/util/is/u8a';
|
||||
import u8aToHex from '@polkadot/util/u8a/toHex';
|
||||
import encodeRlp from '@polkadot/util-rlp/encode';
|
||||
|
||||
import nibblesFromU8a from './nibbles/fromU8a';
|
||||
import nibblesToU8a from './nibbles/toU8a';
|
||||
|
||||
function addHexPrefix (key: Array<number>, isTerminator: boolean): Array<number> {
|
||||
// odd
|
||||
if (key.length % 2) {
|
||||
key.unshift(1);
|
||||
} else {
|
||||
// even
|
||||
key.unshift(0);
|
||||
key.unshift(0);
|
||||
}
|
||||
|
||||
if (isTerminator) {
|
||||
key[0] += 2;
|
||||
}
|
||||
|
||||
return key;
|
||||
}
|
||||
|
||||
function removeHexPrefix (key: Uint8Array): Uint8Array {
|
||||
return key[0] % 2
|
||||
? key.slice(1)
|
||||
: key.slice(2);
|
||||
}
|
||||
|
||||
export default class TrieNode {
|
||||
hashing: HashFn;
|
||||
raw: DecodedRlp;
|
||||
type: Trie$Node$Type;
|
||||
|
||||
constructor (hashing: HashFn, type: Trie$Node$Type, raw: DecodedRlp) {
|
||||
this.hashing = hashing;
|
||||
this.raw = raw;
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
get value () {
|
||||
return this.getValue();
|
||||
}
|
||||
|
||||
set value (v) {
|
||||
this.setValue(v);
|
||||
}
|
||||
|
||||
get key () {
|
||||
return this.getKey();
|
||||
}
|
||||
|
||||
set key (k) {
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
this.setKey(k);
|
||||
}
|
||||
|
||||
setValue (key: number | Uint8Array, value?: Uint8Array) {
|
||||
if (this.type !== 'branch') {
|
||||
this.raw[1] = key as Uint8Array;
|
||||
return;
|
||||
}
|
||||
|
||||
if (arguments.length === 1) {
|
||||
value = key as Uint8Array;
|
||||
key = 16;
|
||||
}
|
||||
|
||||
this.raw[key as number] = value as Uint8Array;
|
||||
}
|
||||
|
||||
getValue (key?: number) {
|
||||
if (this.type !== 'branch') {
|
||||
return this.raw[1];
|
||||
}
|
||||
|
||||
if (arguments.length === 0) {
|
||||
key = 16;
|
||||
}
|
||||
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
const val = this.raw[key];
|
||||
|
||||
if (val !== null && val !== undefined && val.length !== 0) {
|
||||
return val;
|
||||
}
|
||||
}
|
||||
|
||||
setKey (key: Array<number> | Uint8Array) {
|
||||
if (this.type !== 'branch') {
|
||||
if (Array.isArray(key)) {
|
||||
key = key.slice(0); // copy the key
|
||||
} else {
|
||||
key = nibblesFromU8a(key);
|
||||
}
|
||||
|
||||
key = addHexPrefix(key, this.type === 'leaf');
|
||||
this.raw[0] = nibblesToU8a(key);
|
||||
}
|
||||
}
|
||||
|
||||
getKey () {
|
||||
if (this.type !== 'branch') {
|
||||
return removeHexPrefix(
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
nibblesFromU8a(this.raw[0])
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
serialize (): Uint8Array {
|
||||
return encodeRlp(this.raw);
|
||||
}
|
||||
|
||||
hash (): Uint8Array {
|
||||
return this.hashing(this.serialize());
|
||||
}
|
||||
|
||||
toString (): string {
|
||||
return this.raw.reduce((out, el, index) => {
|
||||
if (index !== 0) {
|
||||
out += ', ';
|
||||
}
|
||||
|
||||
if (isU8a(el)) {
|
||||
out += u8aToHex(el);
|
||||
} else if (el) {
|
||||
out += 'object';
|
||||
} else {
|
||||
out += 'empty';
|
||||
}
|
||||
|
||||
return out;
|
||||
}, `${this.type}: [`) + ']';
|
||||
}
|
||||
|
||||
getChildren () {
|
||||
const children = [];
|
||||
|
||||
switch (this.type) {
|
||||
case 'leaf':
|
||||
// no children
|
||||
break;
|
||||
case 'extention':
|
||||
// one child
|
||||
children.push([this.key, this.getValue()]);
|
||||
break;
|
||||
case 'branch':
|
||||
let end = 16;
|
||||
|
||||
for (let index = 0; index < end; index++) {
|
||||
const value = this.getValue(index);
|
||||
|
||||
if (value) {
|
||||
children.push([
|
||||
[index], value
|
||||
]);
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
return children;
|
||||
}
|
||||
}
|
||||
@@ -1,13 +1,20 @@
|
||||
// Copyright 2017-2018 @polkadot/trie-db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the MPL-2.0 license. See the LICENSE file for details.
|
||||
// of the ISC license. See the LICENSE file for details.
|
||||
|
||||
import hashing from '@polkadot/util-crypto/keccak/asU8a';
|
||||
import trieRoot from '@polkadot/trie-hash/root';
|
||||
|
||||
const EMPTY_ROOT_U8A = trieRoot([]);
|
||||
const EMPTY_ROOT_STR = EMPTY_ROOT_U8A.toString();
|
||||
const EMPTY_NODE = new Uint8Array();
|
||||
const EMPTY_U8A = trieRoot([]);
|
||||
const EMPTY_U8A_STR = trieRoot([]);
|
||||
const EMPTY_HASH = hashing(EMPTY_U8A);
|
||||
const EMPTY_HASH_STR = EMPTY_HASH.toString();
|
||||
|
||||
export {
|
||||
EMPTY_ROOT_U8A,
|
||||
EMPTY_ROOT_STR
|
||||
EMPTY_NODE,
|
||||
EMPTY_HASH,
|
||||
EMPTY_HASH_STR,
|
||||
EMPTY_U8A,
|
||||
EMPTY_U8A_STR
|
||||
};
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
// Copyright 2017-2018 @polkadot/trie-db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the MPL-2.0 license. See the LICENSE file for details.
|
||||
|
||||
import { Logger } from '@polkadot/util/types';
|
||||
|
||||
import assert from '@polkadot/util/assert';
|
||||
import isBuffer from '@polkadot/util/is/buffer';
|
||||
import isU8a from '@polkadot/util/is/u8a';
|
||||
import toU8a from '@polkadot/util/u8a/toU8a';
|
||||
|
||||
export default function decode (l: Logger) {
|
||||
return (value: Buffer | Uint8Array | null): Uint8Array => {
|
||||
// l.debug(() => ['decoding', value]);
|
||||
|
||||
if (!value || !value.length) {
|
||||
return new Uint8Array(0);
|
||||
}
|
||||
|
||||
const isBufferIn = isBuffer(value);
|
||||
const isU8aIn = isU8a(value);
|
||||
|
||||
assert(isBufferIn || isU8aIn, `Expected a Buffer | Uint8Array | null input type, received '${typeof value}'`);
|
||||
|
||||
return isBufferIn
|
||||
? toU8a(value)
|
||||
: value;
|
||||
};
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
// Copyright 2017-2018 @polkadot/trie-db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the MPL-2.0 license. See the LICENSE file for details.
|
||||
|
||||
import { Logger } from '@polkadot/util/types';
|
||||
|
||||
import assert from '@polkadot/util/assert';
|
||||
import isBuffer from '@polkadot/util/is/buffer';
|
||||
import isU8a from '@polkadot/util/is/u8a';
|
||||
import u8aToBuffer from '@polkadot/util/u8a/toBuffer';
|
||||
|
||||
export default function encode (l: Logger) {
|
||||
return (value: Buffer | Uint8Array | null): Buffer => {
|
||||
// l.debug(() => ['encoding', value ? value.toString() : value]);
|
||||
|
||||
if (!value || value.length === 0) {
|
||||
return Buffer.alloc(0);
|
||||
}
|
||||
|
||||
const isBufferIn = isBuffer(value);
|
||||
const isU8aIn = isU8a(value);
|
||||
|
||||
assert(isBufferIn || isU8aIn, `Expected Buffer | Uint8Array | null input, received '${typeof value}'`);
|
||||
|
||||
return isBufferIn
|
||||
? value as Buffer
|
||||
: u8aToBuffer(value);
|
||||
};
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
// Copyright 2017-2018 @polkadot/trie-db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the MPL-2.0 license. See the LICENSE file for details.
|
||||
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
import { LevelUp$AbstractStorage } from 'levelup';
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
import encodingDown, { EncoderBufferOptions, EncodingBuffer } from 'encoding-down';
|
||||
|
||||
import logger from '@polkadot/util/logger';
|
||||
|
||||
import decode from './decode';
|
||||
import encode from './encode';
|
||||
|
||||
const l = logger('trie/codec');
|
||||
|
||||
const encoding: EncodingBuffer<Uint8Array> = {
|
||||
buffer: true,
|
||||
decode: decode(l),
|
||||
encode: encode(l),
|
||||
type: 'Uint8Array'
|
||||
};
|
||||
|
||||
const options: EncoderBufferOptions<Uint8Array> = {
|
||||
keyEncoding: encoding,
|
||||
valueEncoding: encoding
|
||||
};
|
||||
|
||||
function encoder (db: LevelUp$AbstractStorage) {
|
||||
return encodingDown(db, options);
|
||||
}
|
||||
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
encoder.options = options;
|
||||
|
||||
export default encoder;
|
||||
@@ -0,0 +1,59 @@
|
||||
// Copyright 2017-2018 @polkadot/trie-db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the ISC license. See the LICENSE file for details.
|
||||
|
||||
import toU8a from '@polkadot/util/u8a/toU8a';
|
||||
import u8aToHex from '@polkadot/util/u8a/toHex';
|
||||
|
||||
import Trie from './index';
|
||||
import getTests from '../../../test/getTests';
|
||||
|
||||
const trietest = getTests('TrieTests/trietest.json');
|
||||
const trieanyorder = getTests('TrieTests/trieanyorder.json');
|
||||
|
||||
describe('official tests', () => {
|
||||
let trie;
|
||||
|
||||
beforeEach(() => {
|
||||
trie = new Trie();
|
||||
});
|
||||
|
||||
// FIXME (maybe) These all have empty/null values, we don't allow those atm as a key nor value
|
||||
describe.skip('ordered tests', () => {
|
||||
const testNames = Object.keys(trietest);
|
||||
|
||||
trietest.forEach(({ name, input, root }) => {
|
||||
it(name, () => {
|
||||
input.map((input) =>
|
||||
trie.put(toU8a(input[0]), toU8a(input[1]))
|
||||
);
|
||||
|
||||
expect(
|
||||
u8aToHex(trie.getRoot())
|
||||
).toEqual(
|
||||
root
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('unordered tests', () => {
|
||||
const testNames = Object.keys(trieanyorder);
|
||||
|
||||
trieanyorder.forEach(({ name, input, root }) => {
|
||||
const keys = Object.keys(input);
|
||||
|
||||
it(name, () => {
|
||||
keys.map((key) =>
|
||||
trie.put(toU8a(key), toU8a(input[key]))
|
||||
)
|
||||
|
||||
expect(
|
||||
u8aToHex(trie.getRoot())
|
||||
).toEqual(
|
||||
root
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,267 @@
|
||||
// Copyright 2015-2018 authors & contributors of https://github.com/ethereumjs/merkle-patricia-tree
|
||||
// Copyright 2017-2018 @polkadot/trie-db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the MPL-2.0 license. See the LICENSE file for details.
|
||||
|
||||
import u8aToHex from '@polkadot/util/u8a/toHex';
|
||||
import toU8a from '@polkadot/util/u8a/toU8a';
|
||||
|
||||
import Trie from './index';
|
||||
|
||||
describe('simple save and retrive', () => {
|
||||
const trie = new Trie();
|
||||
|
||||
it('starts with a valid root', () => {
|
||||
expect(
|
||||
trie.getRoot()
|
||||
).toEqual(
|
||||
new Uint8Array([
|
||||
86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33
|
||||
])
|
||||
)
|
||||
});
|
||||
|
||||
it('save a value', () => {
|
||||
trie.put(toU8a('test'), toU8a('one'));
|
||||
|
||||
expect(
|
||||
trie.getRoot()
|
||||
).toEqual(
|
||||
new Uint8Array([
|
||||
43, 119, 232, 84, 123, 197, 94, 42, 149, 34, 124, 147, 159, 159, 157, 103, 149, 45, 225, 233, 112, 160, 23, 224, 145, 11, 229, 16, 176, 144, 175, 243
|
||||
])
|
||||
);
|
||||
});
|
||||
|
||||
it('should get a value', () => {
|
||||
expect(
|
||||
trie.get(toU8a('test'))
|
||||
).toEqual(toU8a('one'));
|
||||
});
|
||||
|
||||
it('should update a value', () => {
|
||||
trie.put(toU8a('test'), toU8a('two'));
|
||||
|
||||
expect(
|
||||
trie.get(toU8a('test'))
|
||||
).toEqual(toU8a('two'));
|
||||
expect(
|
||||
trie.getRoot()
|
||||
).toEqual(
|
||||
new Uint8Array([
|
||||
127, 237, 74, 184, 190, 46, 103, 129, 65, 141, 13, 96, 61, 232, 146, 13, 56, 32, 59, 29, 246, 58, 175, 140, 72, 182, 196, 103, 230, 245, 229, 148
|
||||
])
|
||||
);
|
||||
});
|
||||
|
||||
it('should delete a value', () => {
|
||||
trie.del(toU8a('test'));
|
||||
|
||||
expect(
|
||||
trie.get(toU8a('test'))
|
||||
).toEqual(null);
|
||||
expect(
|
||||
trie.getRoot()
|
||||
).toEqual(
|
||||
new Uint8Array([
|
||||
86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33
|
||||
])
|
||||
);
|
||||
});
|
||||
|
||||
it('should recreate a value', () => {
|
||||
trie.put(toU8a('test'), toU8a('one'));
|
||||
|
||||
expect(
|
||||
trie.getRoot()
|
||||
).toEqual(
|
||||
new Uint8Array([
|
||||
43, 119, 232, 84, 123, 197, 94, 42, 149, 34, 124, 147, 159, 159, 157, 103, 149, 45, 225, 233, 112, 160, 23, 224, 145, 11, 229, 16, 176, 144, 175, 243
|
||||
])
|
||||
);
|
||||
});
|
||||
|
||||
it('should get updated a value', () => {
|
||||
expect(
|
||||
trie.get(toU8a('test'))
|
||||
).toEqual(toU8a('one'));
|
||||
});
|
||||
|
||||
it('should create a branch here', () => {
|
||||
trie.put(toU8a('doge'), toU8a('coin'));
|
||||
|
||||
expect(
|
||||
trie.getRoot()
|
||||
).toEqual(
|
||||
toU8a('0xde8a34a8c1d558682eae1528b47523a483dd8685d6db14b291451a66066bf0fc')
|
||||
);
|
||||
});
|
||||
|
||||
it('should get a value that is in a branch', () => {
|
||||
expect(
|
||||
trie.get(toU8a('doge'))
|
||||
).toEqual(toU8a('coin'));
|
||||
});
|
||||
|
||||
it('should delete from a branch', () => {
|
||||
trie.del(toU8a('doge'));
|
||||
|
||||
expect(
|
||||
trie.get(toU8a('doge'))
|
||||
).toEqual(null);
|
||||
expect(
|
||||
trie.getRoot()
|
||||
).toEqual(
|
||||
new Uint8Array([
|
||||
43, 119, 232, 84, 123, 197, 94, 42, 149, 34, 124, 147, 159, 159, 157, 103, 149, 45, 225, 233, 112, 160, 23, 224, 145, 11, 229, 16, 176, 144, 175, 243
|
||||
])
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('storing longer values', () => {
|
||||
const trie = new Trie();
|
||||
const longString = 'this will be a really really really long value';
|
||||
const longStringRoot = '0xb173e2db29e79c78963cff5196f8a983fbe0171388972106b114ef7f5c24dfa3';
|
||||
|
||||
it('should store a longer string', () => {
|
||||
trie.put(toU8a('done'), toU8a(longString));
|
||||
|
||||
expect(
|
||||
trie.getRoot()
|
||||
).toEqual(new Uint8Array([
|
||||
79, 170, 48, 228, 218, 25, 91, 13, 86, 114, 153, 98, 252, 127, 54, 124, 242, 163, 153, 215, 249, 196, 65, 209, 186, 130, 39, 6, 181, 246, 231, 111
|
||||
]));
|
||||
});
|
||||
|
||||
it('should retrieve a longer value (first pass)', () => {
|
||||
expect(
|
||||
trie.get(toU8a('done'))
|
||||
).toEqual(toU8a(longString));
|
||||
});
|
||||
|
||||
it('should store subsequent values', () => {
|
||||
trie.put(toU8a('doge'), toU8a('coin'));
|
||||
|
||||
expect(longStringRoot).toEqual(u8aToHex(trie.getRoot()));
|
||||
});
|
||||
|
||||
it('should retrieve a longer value (second pass)', () => {
|
||||
expect(
|
||||
trie.get(toU8a('done'))
|
||||
).toEqual(toU8a(longString));
|
||||
});
|
||||
|
||||
it('should retrieve subsequent values', () => {
|
||||
expect(trie.get(toU8a('doge'))).toEqual(toU8a('coin'));
|
||||
});
|
||||
|
||||
it('should when being modiefied delete the old value', () => {
|
||||
trie.put(toU8a('done'), toU8a('test'));
|
||||
|
||||
expect(
|
||||
trie.getRoot()
|
||||
).toEqual(new Uint8Array([
|
||||
219, 23, 9, 241, 116, 187, 56, 113, 244, 253, 200, 19, 242, 27, 188, 158, 217, 68, 67, 249, 222, 154, 243, 190, 184, 56, 173, 54, 214, 42, 91, 206
|
||||
]));
|
||||
});
|
||||
});
|
||||
|
||||
describe('testing Extentions and branches', () => {
|
||||
const trie = new Trie();
|
||||
|
||||
it('should store a value', () => {
|
||||
trie.put(toU8a('doge'), toU8a('coin'));
|
||||
});
|
||||
|
||||
it('should create extention to store this value', () => {
|
||||
trie.put(toU8a('do'), toU8a('verb'));
|
||||
|
||||
expect(
|
||||
toU8a('0xf803dfcb7e8f1afd45e88eedb4699a7138d6c07b71243d9ae9bff720c99925f9')
|
||||
).toEqual(trie.getRoot());
|
||||
});
|
||||
|
||||
it('should store this value under the extention ', () => {
|
||||
trie.put(toU8a('done'), toU8a('finished'));
|
||||
|
||||
expect(
|
||||
'0x409cff4d820b394ed3fb1cd4497bdd19ffa68d30ae34157337a7043c94a3e8cb'
|
||||
).toEqual(u8aToHex(trie.getRoot()));
|
||||
});
|
||||
});
|
||||
|
||||
describe('testing Extentions and branches - reverse', () => {
|
||||
const trie = new Trie();
|
||||
|
||||
it('should create extention to store this value', () => {
|
||||
trie.put(toU8a('do'), toU8a('verb'));
|
||||
|
||||
const value = trie.get(toU8a('do'));
|
||||
|
||||
expect(value).toEqual(toU8a('verb'));
|
||||
});
|
||||
|
||||
it('should store a value', () => {
|
||||
trie.put(toU8a('doge'), toU8a('coin'));
|
||||
});
|
||||
|
||||
it('should store this value under the extention ', () => {
|
||||
trie.put(toU8a('done'), toU8a('finished'));
|
||||
|
||||
expect(
|
||||
'0x409cff4d820b394ed3fb1cd4497bdd19ffa68d30ae34157337a7043c94a3e8cb'
|
||||
).toEqual(u8aToHex(trie.getRoot()));
|
||||
});
|
||||
});
|
||||
|
||||
describe('testing deletions cases', () => {
|
||||
const trie = new Trie();
|
||||
|
||||
it('should delete from a branch->branch-branch', () => {
|
||||
trie.put(new Uint8Array([11, 11, 11]), toU8a('first')),
|
||||
trie.put(new Uint8Array([12, 22, 22]), toU8a('create the first branch')),
|
||||
trie.put(new Uint8Array([12, 34, 44]), toU8a('create the last branch'))
|
||||
trie.del(new Uint8Array([12, 22, 22]));
|
||||
|
||||
const value = trie.get(new Uint8Array([12, 22, 22]));
|
||||
|
||||
expect(null).toEqual(value);
|
||||
});
|
||||
|
||||
it('should delete from a branch->branch-extention', () => {
|
||||
trie.put(new Uint8Array([11, 11, 11]), toU8a('first')),
|
||||
trie.put(new Uint8Array([12, 22, 22]), toU8a('create the first branch')),
|
||||
trie.put(new Uint8Array([12, 33, 33]), toU8a('create the middle branch')),
|
||||
trie.put(new Uint8Array([12, 33, 44]), toU8a('create the last branch'))
|
||||
trie.del(new Uint8Array([12, 22, 22]));
|
||||
|
||||
const value = trie.get(new Uint8Array([12, 22, 22]));
|
||||
|
||||
expect(null).toEqual(value);
|
||||
});
|
||||
|
||||
it('should delete from a extention->branch-extention', () => {
|
||||
trie.put(new Uint8Array([11, 11, 11]), toU8a('first'));
|
||||
trie.put(new Uint8Array([12, 22, 22]), toU8a('create the first branch'));
|
||||
trie.put(new Uint8Array([12, 33, 33]), toU8a('create the middle branch'));
|
||||
trie.put(new Uint8Array([12, 33, 44]), toU8a('create the last branch'));
|
||||
trie.del(new Uint8Array([11, 11, 11]));
|
||||
|
||||
const value = trie.get(new Uint8Array([11, 11, 11]));
|
||||
|
||||
expect(null).toEqual(value);
|
||||
});
|
||||
|
||||
it('should delete from a extention->branch-branch', () => {
|
||||
trie.put(new Uint8Array([11, 11, 11]), toU8a('first'));
|
||||
trie.put(new Uint8Array([12, 22, 22]), toU8a('create the first branch'));
|
||||
trie.put(new Uint8Array([12, 33, 33]), toU8a('create the middle branch'));
|
||||
trie.put(new Uint8Array([12, 34, 44]), toU8a('create the last branch'));
|
||||
trie.del(new Uint8Array([11, 11, 11]));
|
||||
|
||||
const value = trie.get(new Uint8Array([11, 11, 11]));
|
||||
|
||||
expect(null).toEqual(value);
|
||||
});
|
||||
});
|
||||
+460
-13
@@ -1,23 +1,470 @@
|
||||
// Copyright 2015-2018 authors & contributors of https://github.com/ethereumjs/merkle-patricia-tree
|
||||
// Copyright 2017-2018 @polkadot/trie-db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the MPL-2.0 license. See the LICENSE file for details.
|
||||
// of the ISC license. See the LICENSE file for details.
|
||||
|
||||
import { HashFn } from './types';
|
||||
import { TxDb } from '@polkadot/db/types';
|
||||
import { Node, NodeBranch, NodeEncodedOrEmpty, NodeKv, NodeNotEmpty, NodeType } from './types';
|
||||
|
||||
// import logger from '@polkadot/util/logger';
|
||||
import CheckpointTrie from './CheckpointTrie';
|
||||
import { prove, verifyProof } from './proof';
|
||||
import MemoryDb from '@polkadot/db/Memory';
|
||||
import isNull from '@polkadot/util/is/null';
|
||||
import logger from '@polkadot/util/logger';
|
||||
import u8aConcat from '@polkadot/util/u8a/concat';
|
||||
import hashing from '@polkadot/util-crypto/keccak/asU8a';
|
||||
import toNibbles from '@polkadot/trie-hash/util/asNibbles';
|
||||
|
||||
// const l = logger('trie');
|
||||
import { isBranchNode, isEmptyNode, isExtensionNode, isKvNode, isLeafNode } from './util/is';
|
||||
import { extractKey, keyEquals, keyStartsWith, computeExtensionKey, computeLeafKey, consumeCommonPrefix } from './util/key';
|
||||
import { decodeNibbles, encodeNibbles } from './util/nibbles';
|
||||
import { getNodeType, decodeNode, encodeNode } from './util/node';
|
||||
import { EMPTY_HASH, EMPTY_U8A } from './constants';
|
||||
|
||||
export default class Trie extends CheckpointTrie {
|
||||
static prove = prove;
|
||||
static verifyProof = verifyProof;
|
||||
const l = logger('trie/db');
|
||||
|
||||
constructor (db: any, root?: Uint8Array, hashing?: HashFn) {
|
||||
super(db, root, hashing);
|
||||
export default class Trie {
|
||||
private db: TxDb;
|
||||
private txRoot: Uint8Array;
|
||||
private rootHash: Uint8Array;
|
||||
|
||||
// l.debug('Created Trie');
|
||||
constructor (db?: TxDb, rootHash: Uint8Array = EMPTY_HASH) {
|
||||
this.db = db || new MemoryDb();
|
||||
this.rootHash = rootHash;
|
||||
this.txRoot = rootHash;
|
||||
}
|
||||
|
||||
private createCheckpoint (): Uint8Array {
|
||||
this.txRoot = this.rootHash;
|
||||
|
||||
return this.txRoot;
|
||||
}
|
||||
|
||||
private commitCheckpoint (): Uint8Array {
|
||||
return this.rootHash;
|
||||
}
|
||||
|
||||
private revertCheckpoint (): Uint8Array {
|
||||
this.rootHash = this.txRoot;
|
||||
|
||||
return this.rootHash;
|
||||
}
|
||||
|
||||
transaction (fn: () => boolean): Uint8Array {
|
||||
try {
|
||||
this.createCheckpoint();
|
||||
|
||||
return this.db.transaction(fn)
|
||||
? this.commitCheckpoint()
|
||||
: this.revertCheckpoint();
|
||||
} catch (error) {
|
||||
this.revertCheckpoint();
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
del (key: Uint8Array) {
|
||||
l.debug(() => ['del', { key }]);
|
||||
|
||||
this._setRootNode(
|
||||
this._del(
|
||||
this.getNode(this.rootHash),
|
||||
toNibbles(key)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
get (key: Uint8Array): Uint8Array | null {
|
||||
l.debug(() => ['get', { key }]);
|
||||
|
||||
return this._get(
|
||||
this.getNode(this.rootHash),
|
||||
toNibbles(key)
|
||||
);
|
||||
}
|
||||
|
||||
put (key: Uint8Array, value: Uint8Array): void {
|
||||
l.debug(() => ['put', { key, value }]);
|
||||
|
||||
this._setRootNode(
|
||||
this._put(
|
||||
this.getNode(this.rootHash),
|
||||
toNibbles(key),
|
||||
value
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
getRoot (): Uint8Array {
|
||||
const rootNode = this.getNode(this.rootHash);
|
||||
|
||||
if (isNull(rootNode)) {
|
||||
return EMPTY_U8A;
|
||||
}
|
||||
|
||||
return this.rootHash;
|
||||
}
|
||||
|
||||
setRoot (rootHash: Uint8Array): void {
|
||||
this.rootHash = rootHash;
|
||||
// return this._setRootNode(rootNode);
|
||||
}
|
||||
|
||||
private getNode (hash: Uint8Array | null): Node {
|
||||
if (!hash || hash.length === 0 || keyEquals(hash, EMPTY_HASH)) {
|
||||
return null;
|
||||
} else if (hash.length < 32) {
|
||||
return decodeNode(hash);
|
||||
}
|
||||
|
||||
return decodeNode(
|
||||
this.db.get(hash)
|
||||
);
|
||||
}
|
||||
|
||||
private _del (node: Node, trieKey: Uint8Array): Node {
|
||||
// l.debug(() => ['_del', { node, trieKey }]);
|
||||
|
||||
if (isEmptyNode(node)) {
|
||||
return null;
|
||||
} else if (isBranchNode(node)) {
|
||||
return this._delBranchNode(node, trieKey);
|
||||
} else if (isKvNode(node)) {
|
||||
return this._delKvNode(node, trieKey);
|
||||
}
|
||||
|
||||
throw new Error('Unreachable');
|
||||
}
|
||||
|
||||
private _delBranchNode (node: NodeBranch, trieKey: Uint8Array): Node {
|
||||
// l.debug(() => ['_delBranchNode', { node, trieKey }]);
|
||||
|
||||
if (trieKey.length === 0) {
|
||||
node[node.length - 1] = null;
|
||||
|
||||
return this._normaliseBranchNode(node);
|
||||
}
|
||||
|
||||
const nodeToDelete = this.getNode(node[trieKey[0]]);
|
||||
const subNode = this._del(nodeToDelete, trieKey.subarray(1));
|
||||
const encodedSubNode = this._persistNode(subNode);
|
||||
|
||||
if (keyEquals(encodedSubNode, node[trieKey[0]])) {
|
||||
return node;
|
||||
}
|
||||
|
||||
node[trieKey[0]] = encodedSubNode;
|
||||
|
||||
if (isNull(encodedSubNode)) {
|
||||
return this._normaliseBranchNode(node);
|
||||
}
|
||||
|
||||
return node;
|
||||
}
|
||||
|
||||
private _delKvNode (node: NodeNotEmpty, trieKey: Uint8Array): Node {
|
||||
// l.debug(() => ['_delKvNode', { node, trieKey }]);
|
||||
|
||||
const currentKey = extractKey(node);
|
||||
const nodeType = getNodeType(node);
|
||||
|
||||
if (!keyStartsWith(trieKey, currentKey)) {
|
||||
return node;
|
||||
}
|
||||
|
||||
if (nodeType === NodeType.LEAF) {
|
||||
return keyEquals(trieKey, currentKey)
|
||||
? null
|
||||
: node;
|
||||
}
|
||||
|
||||
const subKey = trieKey.subarray(currentKey.length);
|
||||
const subNode = this.getNode(node[1]);
|
||||
const newSub = this._del(subNode, subKey);
|
||||
const encodedNewSub = this._persistNode(newSub);
|
||||
|
||||
if (keyEquals(encodedNewSub, node[1])) {
|
||||
return node;
|
||||
} else if (isNull(newSub)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (isKvNode(newSub)) {
|
||||
const subNibbles = decodeNibbles(newSub[0]);
|
||||
const newKey = u8aConcat(currentKey, subNibbles);
|
||||
|
||||
return [
|
||||
encodeNibbles(newKey),
|
||||
newSub[1]
|
||||
];
|
||||
} else if (isBranchNode(newSub)) {
|
||||
return [
|
||||
encodeNibbles(currentKey),
|
||||
encodedNewSub
|
||||
];
|
||||
}
|
||||
|
||||
throw new Error('Unreachable');
|
||||
}
|
||||
|
||||
private _get (node: Node, trieKey: Uint8Array): NodeEncodedOrEmpty {
|
||||
// l.debug(() => ['_get', { node, trieKey }]);
|
||||
|
||||
if (isEmptyNode(node)) {
|
||||
return null;
|
||||
} else if (isBranchNode(node)) {
|
||||
return this._getBranchNode(node, trieKey);
|
||||
} else if (isKvNode(node)) {
|
||||
return this._getKvNode(node, trieKey);
|
||||
}
|
||||
|
||||
throw new Error('Invalid NodeType');
|
||||
}
|
||||
|
||||
private _getBranchNode (node: NodeBranch, trieKey: Uint8Array): NodeEncodedOrEmpty {
|
||||
// l.debug(() => ['_getBranchNode', { node, trieKey }]);
|
||||
|
||||
if (trieKey.length === 0) {
|
||||
return node[16];
|
||||
}
|
||||
|
||||
const subNode = this.getNode(node[trieKey[0]]);
|
||||
|
||||
return this._get(subNode, trieKey.subarray(1));
|
||||
}
|
||||
|
||||
private _getKvNode (node: NodeKv, trieKey: Uint8Array): NodeEncodedOrEmpty {
|
||||
// l.debug(() => ['_getKvNode', { node, trieKey }]);
|
||||
|
||||
const currentKey = extractKey(node);
|
||||
const nodeType = getNodeType(node);
|
||||
|
||||
l.debug(() => [{ currentKey, trieKey, nodeType }]);
|
||||
|
||||
if (nodeType === NodeType.LEAF) {
|
||||
if (keyEquals(trieKey, currentKey)) {
|
||||
return node[1];
|
||||
}
|
||||
|
||||
return null;
|
||||
} else if (nodeType === NodeType.EXTENSION) {
|
||||
if (keyStartsWith(trieKey, currentKey)) {
|
||||
const subNode = this.getNode(node[1]);
|
||||
|
||||
return this._get(subNode, trieKey.subarray(currentKey ? currentKey.length : 0));
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
throw new Error('Unreachable');
|
||||
}
|
||||
|
||||
private _nodeToDbMapping (node: Node): NodeNotEmpty {
|
||||
// l.debug(() => ['_nodeToDbMapping', { node }]);
|
||||
|
||||
if (isEmptyNode(node)) {
|
||||
return [
|
||||
null,
|
||||
null
|
||||
];
|
||||
}
|
||||
|
||||
const encoded = encodeNode(node);
|
||||
|
||||
if (encoded.length < 32) {
|
||||
return [
|
||||
// @ts-ignore Ok, this is not correct - this comes back as an embedded node, which mean that our definitions here are completely wrong (or what we think it is, does not align with what it should be as per the spec and implementation)
|
||||
node,
|
||||
null
|
||||
];
|
||||
}
|
||||
|
||||
return [
|
||||
hashing(encoded),
|
||||
encoded
|
||||
];
|
||||
}
|
||||
|
||||
private _normaliseBranchNode (node: NodeNotEmpty): Node {
|
||||
// l.debug(() => ['_normaliseBranchNode', { node }]);
|
||||
|
||||
const mapped = node
|
||||
.map((value, index) => ({
|
||||
index,
|
||||
value
|
||||
}))
|
||||
.filter(({ value }) =>
|
||||
!!value && value.length !== 0
|
||||
);
|
||||
|
||||
if (mapped.length >= 2) {
|
||||
return node;
|
||||
}
|
||||
|
||||
if (node[16]) {
|
||||
return [
|
||||
computeLeafKey(new Uint8Array()),
|
||||
node[16]
|
||||
];
|
||||
}
|
||||
|
||||
const [{ index, value }] = mapped;
|
||||
const subNode = this.getNode(value);
|
||||
|
||||
if (isBranchNode(subNode)) {
|
||||
return [
|
||||
encodeNibbles(new Uint8Array([index])),
|
||||
this._persistNode(subNode)
|
||||
];
|
||||
} else if (isKvNode(subNode)) {
|
||||
const subNibbles = decodeNibbles(subNode[0]);
|
||||
const newKey = u8aConcat(new Uint8Array([index]), subNibbles);
|
||||
|
||||
return [
|
||||
encodeNibbles(newKey),
|
||||
subNode[1]
|
||||
];
|
||||
}
|
||||
|
||||
throw new Error('Unreachable');
|
||||
}
|
||||
|
||||
private _persistNode (node: Node): NodeEncodedOrEmpty {
|
||||
const [key, value] = this._nodeToDbMapping(node);
|
||||
|
||||
// l.debug(() => ['_persistNode', { node, key, value }]);
|
||||
|
||||
if (value) {
|
||||
this.db.put(key as Uint8Array, value);
|
||||
}
|
||||
|
||||
return key;
|
||||
}
|
||||
|
||||
private _put (node: Node, trieKey: Uint8Array, value: Uint8Array): NodeNotEmpty {
|
||||
// l.debug(() => ['_put', { node, trieKey, value }]);
|
||||
|
||||
if (isEmptyNode(node)) {
|
||||
return [
|
||||
computeLeafKey(trieKey),
|
||||
value
|
||||
];
|
||||
} else if (isKvNode(node)) {
|
||||
return this._putKvNode(node, trieKey, value);
|
||||
} else if (isBranchNode(node)) {
|
||||
return this._putBranchNode(node, trieKey, value);
|
||||
}
|
||||
|
||||
throw new Error('Unreachable');
|
||||
}
|
||||
|
||||
private _putBranchNode (node: NodeBranch, trieKey: Uint8Array, value: Uint8Array): NodeNotEmpty {
|
||||
// l.debug(() => ['_putBranchNode', { node, trieKey, value }]);
|
||||
|
||||
if (trieKey && trieKey.length) {
|
||||
const subNode = this.getNode(node[trieKey[0]]);
|
||||
const newNode = this._put(subNode, trieKey.subarray(1), value);
|
||||
|
||||
node[trieKey[0]] = this._persistNode(newNode);
|
||||
} else {
|
||||
node[node.length - 1] = value;
|
||||
}
|
||||
|
||||
return node;
|
||||
}
|
||||
|
||||
private _putKvNode (node: NodeKv, trieKey: Uint8Array, value: Uint8Array): NodeNotEmpty {
|
||||
// l.debug(() => ['_putKvNode', { node, trieKey, value }]);
|
||||
|
||||
const currentKey = extractKey(node);
|
||||
const [commonPrefix, currentRemainder, trieRemainder] = consumeCommonPrefix(currentKey, trieKey);
|
||||
const isExtension = isExtensionNode(node);
|
||||
const isLeaf = isLeafNode(node);
|
||||
let newNode: NodeNotEmpty;
|
||||
|
||||
l.debug(() => ['_putKvNode', { currentKey, commonPrefix, currentRemainder, trieRemainder }]);
|
||||
|
||||
if (currentRemainder.length === 0 && trieRemainder.length === 0) {
|
||||
if (isLeaf) {
|
||||
return [
|
||||
node[0],
|
||||
value
|
||||
];
|
||||
}
|
||||
|
||||
const subNode = this.getNode(node[1]);
|
||||
|
||||
newNode = this._put(subNode, trieRemainder, value);
|
||||
} else if (currentRemainder.length === 0) {
|
||||
if (isExtension) {
|
||||
const subNode = this.getNode(node[1]);
|
||||
|
||||
newNode = this._put(subNode, trieRemainder, value);
|
||||
} else {
|
||||
const subPosition = trieRemainder[0];
|
||||
const subKey = computeLeafKey(trieRemainder.subarray(1));
|
||||
const subNode: NodeKv = [subKey, value];
|
||||
|
||||
newNode = [
|
||||
null, null, null, null, null, null, null, null,
|
||||
null, null, null, null, null, null, null, null,
|
||||
node[1]
|
||||
];
|
||||
newNode[subPosition] = this._persistNode(subNode);
|
||||
}
|
||||
} else {
|
||||
newNode = [
|
||||
null, null, null, null, null, null, null, null,
|
||||
null, null, null, null, null, null, null, null,
|
||||
null
|
||||
];
|
||||
|
||||
if (currentRemainder.length === 1 && isExtension) {
|
||||
newNode[currentRemainder[0]] = node[1];
|
||||
} else {
|
||||
const computedKey = isExtension
|
||||
? computeExtensionKey(currentRemainder.subarray(1))
|
||||
: computeLeafKey(currentRemainder.subarray(1));
|
||||
|
||||
newNode[currentRemainder[0]] = this._persistNode([
|
||||
computedKey,
|
||||
node[1]
|
||||
]);
|
||||
}
|
||||
|
||||
if (trieRemainder.length) {
|
||||
newNode[trieRemainder[0]] = this._persistNode([
|
||||
computeLeafKey(trieRemainder.subarray(1)),
|
||||
value
|
||||
]);
|
||||
} else {
|
||||
newNode[16] = value;
|
||||
}
|
||||
}
|
||||
|
||||
l.log(() => ['newNode', newNode]);
|
||||
|
||||
if (commonPrefix.length) {
|
||||
return [
|
||||
computeExtensionKey(commonPrefix),
|
||||
this._persistNode(newNode)
|
||||
];
|
||||
}
|
||||
|
||||
return newNode;
|
||||
}
|
||||
|
||||
private _setRootNode (node: Node): void {
|
||||
// l.debug(() => ['_setRootNode', { node }]);
|
||||
|
||||
if (isEmptyNode(node)) {
|
||||
this.rootHash = EMPTY_HASH;
|
||||
} else {
|
||||
const encoded = encodeNode(node);
|
||||
const newHash = hashing(encoded);
|
||||
|
||||
this.db.put(newHash, encoded);
|
||||
|
||||
this.rootHash = newHash;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
// Copyright 2015-2018 authors & contributors of https://github.com/ethereumjs/merkle-patricia-tree
|
||||
// Copyright 2017-2018 @polkadot/trie-db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the MPL-2.0 license. See the LICENSE file for details.
|
||||
|
||||
export default function nibblesFromU8a (key: Uint8Array | Buffer): Array<number> {
|
||||
const nibbles = [];
|
||||
|
||||
for (let i = 0; i < key.length; i++) {
|
||||
const q = i * 2;
|
||||
|
||||
nibbles[q] = key[i] >> 4;
|
||||
nibbles[q + 1] = key[i] % 16;
|
||||
}
|
||||
|
||||
return nibbles;
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
// Copyright 2017-2018 @polkadot/trie-db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the MPL-2.0 license. See the LICENSE file for details.
|
||||
|
||||
import nibblesFromU8a from './fromU8a';
|
||||
import nibblesIsEqual from './isEqual';
|
||||
import nibblesMatchingLength from './matchingLength';
|
||||
import nibblesToU8a from './toU8a';
|
||||
|
||||
export {
|
||||
nibblesFromU8a,
|
||||
nibblesIsEqual,
|
||||
nibblesMatchingLength,
|
||||
nibblesToU8a
|
||||
};
|
||||
@@ -1,12 +0,0 @@
|
||||
// Copyright 2015-2018 authors & contributors of https://github.com/ethereumjs/merkle-patricia-tree
|
||||
// Copyright 2017-2018 @polkadot/trie-db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the MPL-2.0 license. See the LICENSE file for details.
|
||||
|
||||
import matchingLength from './matchingLength';
|
||||
|
||||
export default function isEqual (keyA: Array<number>, keyB: Array<number>) {
|
||||
const length = matchingLength(keyA, keyB);
|
||||
|
||||
return length === keyA.length && length === keyB.length;
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
// Copyright 2015-2018 authors & contributors of https://github.com/ethereumjs/merkle-patricia-tree
|
||||
// Copyright 2017-2018 @polkadot/trie-db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the MPL-2.0 license. See the LICENSE file for details.
|
||||
|
||||
export default function matchingLength (nib1: Array<number>, nib2: Array<number>): number {
|
||||
let index = 0;
|
||||
|
||||
while (nib1[index] === nib2[index] && nib1.length > index) {
|
||||
index++;
|
||||
}
|
||||
|
||||
return index;
|
||||
}
|
||||
@@ -1,16 +0,0 @@
|
||||
// Copyright 2015-2018 authors & contributors of https://github.com/ethereumjs/merkle-patricia-tree
|
||||
// Copyright 2017-2018 @polkadot/trie-db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the MPL-2.0 license. See the LICENSE file for details.
|
||||
|
||||
export default function nibblesToU8a (arr: Array<number>): Uint8Array {
|
||||
const u8a = new Uint8Array(arr.length / 2);
|
||||
|
||||
for (let i = 0; i < u8a.length; i++) {
|
||||
const q = i * 2;
|
||||
|
||||
u8a[i] = (arr[q] << 4) + arr[q + 1];
|
||||
}
|
||||
|
||||
return u8a;
|
||||
}
|
||||
@@ -1,91 +0,0 @@
|
||||
// Copyright 2015-2018 authors & contributors of https://github.com/ethereumjs/merkle-patricia-tree
|
||||
// Copyright 2017-2018 @polkadot/trie-db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the MPL-2.0 license. See the LICENSE file for details.
|
||||
|
||||
import { Logger } from '@polkadot/util/types';
|
||||
import { HashFn, DecodedRlp, NodeFactory, Trie$Node, Trie$Node$Type } from './types';
|
||||
|
||||
import logger from '@polkadot/util/logger';
|
||||
// import u8aToHex from '@polkadot/util/u8a/toHex';
|
||||
|
||||
import nibblesFromU8a from './nibbles/fromU8a';
|
||||
// import rlpToString from './util/rlpToString';
|
||||
|
||||
import TrieNode from './Node';
|
||||
|
||||
const l = logger('trie/node');
|
||||
|
||||
/**
|
||||
* Determines the node type.
|
||||
* @returns {String} - the node type
|
||||
* - leaf - if the node is a leaf
|
||||
* - branch - if the node is a branch
|
||||
* - extention - if the node is an extention
|
||||
*/
|
||||
function getNodeType (node: DecodedRlp): Trie$Node$Type {
|
||||
if (node.length === 17) {
|
||||
return 'branch';
|
||||
}
|
||||
|
||||
if (node.length === 2) {
|
||||
const key = nibblesFromU8a(node[0] as Uint8Array);
|
||||
|
||||
if (key[0] > 1) {
|
||||
return 'leaf';
|
||||
}
|
||||
|
||||
return 'extention';
|
||||
}
|
||||
|
||||
throw new Error(`Unable to determine Node type`);
|
||||
}
|
||||
|
||||
function fromBranch (l: Logger, hashing: HashFn) {
|
||||
return (rlp: DecodedRlp): Trie$Node => {
|
||||
// l.debug(() => ['fromBranch']);
|
||||
|
||||
const node = new TrieNode(hashing, 'branch', Array.apply(null, Array(17)));
|
||||
|
||||
// NOTE Removed since
|
||||
// (1) 'branch' is never called with any keys,
|
||||
// (b) `this.set` points to what exactly?
|
||||
// if (key) {
|
||||
// key.forEach(function (keyVal) {
|
||||
// this.set.apply(this, keyVal);
|
||||
// });
|
||||
// }
|
||||
|
||||
return node;
|
||||
};
|
||||
}
|
||||
|
||||
function fromRaw (l: Logger, hashing: HashFn) {
|
||||
return (rlp: DecodedRlp): Trie$Node => {
|
||||
// l.debug(() => ['fromRaw ->', rlpToString(rlp)]);
|
||||
|
||||
const node = new TrieNode(hashing, getNodeType(rlp), rlp);
|
||||
|
||||
return node;
|
||||
};
|
||||
}
|
||||
|
||||
function fromType (l: Logger, hashing: HashFn, type: Trie$Node$Type) {
|
||||
return (key: Uint8Array, value: Uint8Array): Trie$Node => {
|
||||
// l.debug(() => ['fromType', 'type ->', type, 'key ->', u8aToHex(key), 'value ->', u8aToHex(value, 256)]);
|
||||
|
||||
const node = new TrieNode(hashing, type, Array(2));
|
||||
|
||||
node.setValue(value);
|
||||
node.setKey(key);
|
||||
|
||||
return node;
|
||||
};
|
||||
}
|
||||
|
||||
export default (hashing: HashFn): NodeFactory => ({
|
||||
fromBranch: fromBranch(l, hashing),
|
||||
fromExtention: fromType(l, hashing, 'extention'),
|
||||
fromLeaf: fromType(l, hashing, 'leaf'),
|
||||
fromRaw: fromRaw(l, hashing)
|
||||
});
|
||||
@@ -1,11 +0,0 @@
|
||||
// Copyright 2017-2018 @polkadot/trie-db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the MPL-2.0 license. See the LICENSE file for details.
|
||||
|
||||
import prove from './prove';
|
||||
import verifyProof from './verifyProof';
|
||||
|
||||
export {
|
||||
prove,
|
||||
verifyProof
|
||||
};
|
||||
@@ -1,35 +0,0 @@
|
||||
// Copyright 2015-2018 authors & contributors of https://github.com/ethereumjs/merkle-patricia-tree
|
||||
// Copyright 2017-2018 @polkadot/trie-db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the MPL-2.0 license. See the LICENSE file for details.
|
||||
|
||||
import assert from '@polkadot/util/assert';
|
||||
|
||||
/**
|
||||
* Returns a merkle proof for a given key
|
||||
* @method Trie.prove
|
||||
* @param {Trie} trie
|
||||
* @param {String} key
|
||||
* @return Promise<{Array.<TrieNode>}> `proof`
|
||||
*/
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
export default async function prove (trie, key) {
|
||||
const result = await trie.findPath(key);
|
||||
|
||||
assert(result, 'findPath does not contain a valid path');
|
||||
|
||||
const { remainder, stack } = result;
|
||||
|
||||
assert(remainder.length === 0, 'Node does not contain the key');
|
||||
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
return stack.reduce((proofs, node, index) => {
|
||||
const rlp = node.serialize();
|
||||
|
||||
if ((rlp.length >= 32) || (index === 0)) {
|
||||
proofs.push(rlp);
|
||||
}
|
||||
|
||||
return proofs;
|
||||
}, []);
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
// Copyright 2015-2018 authors & contributors of https://github.com/ethereumjs/merkle-patricia-tree
|
||||
// Copyright 2017-2018 @polkadot/trie-db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the MPL-2.0 license. See the LICENSE file for details.
|
||||
|
||||
import { HashFn } from '../types';
|
||||
|
||||
import assert from '@polkadot/util/assert';
|
||||
import decodeRlp from '@polkadot/util-rlp/decode';
|
||||
import keccakAsU8a from '@polkadot/util-crypto/keccak/asU8a';
|
||||
|
||||
import { nibblesMatchingLength, nibblesFromU8a } from '../nibbles';
|
||||
import createNodeFactory from '../nodeFactory';
|
||||
|
||||
/**
|
||||
* Verifies a merkle proof for a given key
|
||||
* @method Trie.verifyProof
|
||||
* @param {Buffer | Uint8Array} rootHash
|
||||
* @param {String} key
|
||||
* @param {Array.<TrieNode>} proof
|
||||
* @returns Promise<{String}>
|
||||
*/
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
export default async function verifyProof (rootHash, _key, proofs, hashing: HashFn = keccakAsU8a) {
|
||||
const nodeFactory = createNodeFactory(hashing);
|
||||
const lastIndex = proofs.length - 1;
|
||||
let key = nibblesFromU8a(_key);
|
||||
let wantedHash = rootHash;
|
||||
|
||||
for (let i = 0; i < proofs.length; i++) {
|
||||
const proof = proofs[i];
|
||||
const hash = hashing(proof);
|
||||
|
||||
assert(hash.toString() === wantedHash.toString(), `Bad proof node ${i}: hash mismatch`);
|
||||
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
const node = nodeFactory.fromRaw(decodeRlp(proof));
|
||||
let cld;
|
||||
|
||||
if (node.type === 'branch') {
|
||||
if (key.length === 0) {
|
||||
assert(i === lastIndex, 'Additional nodes at end of proof (branch)');
|
||||
|
||||
return node.value;
|
||||
}
|
||||
|
||||
cld = node.raw[key[0]];
|
||||
key = key.slice(1);
|
||||
|
||||
if (cld.length === 2) {
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
const embeddedNode = nodeFactory.fromRaw(cld);
|
||||
|
||||
assert(i === lastIndex, 'Additional nodes at end of proof (embeddedNode)');
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
assert(nibblesMatchingLength(embeddedNode.key, key) === embeddedNode.key.length, 'Key length does not match with the proof one (embeddedNode)');
|
||||
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
key = key.slice(embeddedNode.key.length);
|
||||
|
||||
assert(key.length === 0, 'Key does not match with the proof one (embeddedNode)');
|
||||
|
||||
return embeddedNode.value;
|
||||
} else {
|
||||
wantedHash = cld;
|
||||
}
|
||||
} else if ((node.type === 'extention') || (node.type === 'leaf')) {
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
assert(nibblesMatchingLength(node.key, key) === node.key.length, 'Key does not match with the proof one (extention|leaf)');
|
||||
|
||||
cld = node.value;
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
key = key.slice(node.key.length);
|
||||
|
||||
if (key.length === 0) {
|
||||
assert(i === lastIndex, 'Additional nodes at end of proof (extention|leaf)');
|
||||
|
||||
return cld;
|
||||
} else {
|
||||
wantedHash = cld;
|
||||
}
|
||||
} else {
|
||||
throw new Error('Invalid node type');
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error('Unexpected end of proof');
|
||||
}
|
||||
@@ -1,58 +0,0 @@
|
||||
// Copyright 2015-2018 authors & contributors of https://github.com/ethereumjs/merkle-patricia-tree
|
||||
// Copyright 2017-2018 @polkadot/trie-db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the MPL-2.0 license. See the LICENSE file for details.
|
||||
|
||||
import { Trie$Base } from '../types';
|
||||
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
import { Readable } from 'readable-stream';
|
||||
// import logger from '@polkadot/util/logger';
|
||||
// import u8aToHex from '@polkadot/util/u8a/toHex';
|
||||
|
||||
// import rlpToString from '../util/rlpToString';
|
||||
|
||||
// const l = logger('trie/scratch');
|
||||
|
||||
export default class ScratchReadStream extends Readable {
|
||||
trie: Trie$Base;
|
||||
next: null; // FIXME
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
_started: boolean;
|
||||
|
||||
constructor (trie: Trie$Base) {
|
||||
super({ objectMode: true });
|
||||
|
||||
this.trie = trie;
|
||||
this.next = null;
|
||||
}
|
||||
|
||||
_read () {
|
||||
if (this._started) {
|
||||
return;
|
||||
}
|
||||
|
||||
const onDone = () => {
|
||||
// l.debug(() => 'onDone');
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
this.push(null);
|
||||
};
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
const onFound = (nodeRef, node, key, next) => {
|
||||
// l.debug(() => ['onFound', u8aToHex(nodeRef), node.type, rlpToString(node.raw)]);
|
||||
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
this.push({
|
||||
key: nodeRef,
|
||||
value: node.serialize()
|
||||
});
|
||||
|
||||
next();
|
||||
};
|
||||
|
||||
// l.debug(() => 'Starting stream');
|
||||
|
||||
this._started = true;
|
||||
this.trie._findDbNodes(onFound, onDone);
|
||||
}
|
||||
}
|
||||
@@ -1,59 +0,0 @@
|
||||
// Copyright 2015-2018 authors & contributors of https://github.com/ethereumjs/merkle-patricia-tree
|
||||
// Copyright 2017-2018 @polkadot/trie-db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the MPL-2.0 license. See the LICENSE file for details.
|
||||
|
||||
import { Trie$Base } from '../types';
|
||||
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
import { Readable } from 'readable-stream';
|
||||
// import logger from '@polkadot/util/logger';
|
||||
// import u8aToHex from '@polkadot/util/u8a/toHex';
|
||||
|
||||
import { nibblesToU8a } from '../nibbles';
|
||||
|
||||
// const l = logger('trie/read');
|
||||
|
||||
export default class TrieReadStream extends Readable {
|
||||
trie: Trie$Base;
|
||||
next: null; // FIXME
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
_started: boolean;
|
||||
|
||||
constructor (trie: Trie$Base) {
|
||||
super({ objectMode: true });
|
||||
|
||||
this.trie = trie;
|
||||
this.next = null;
|
||||
this._started = false;
|
||||
}
|
||||
|
||||
_read () {
|
||||
if (this._started) {
|
||||
return;
|
||||
}
|
||||
|
||||
const onDone = () => {
|
||||
// l.debug(() => 'onDone');
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
this.push(null);
|
||||
};
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
const onFound = (nodeRef, node, nibbles, next) => {
|
||||
// l.debug(() => ['onFound', nibbles, u8aToHex(node.value)]);
|
||||
|
||||
// @ts-ignore FIXME, we need to properly check the full file
|
||||
this.push({
|
||||
key: nibblesToU8a(nibbles),
|
||||
value: node.value
|
||||
});
|
||||
|
||||
next();
|
||||
};
|
||||
|
||||
// l.debug(() => 'Starting stream');
|
||||
|
||||
this._started = true;
|
||||
this.trie._findValueNodes(onFound, onDone);
|
||||
}
|
||||
}
|
||||
Vendored
-37
@@ -1,37 +0,0 @@
|
||||
// Copyright 2017-2018 @polkadot/trie-db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the MPL-2.0 license. See the LICENSE file for details.
|
||||
|
||||
export type Callback = (error?: Error | null, result?: any) => void;
|
||||
|
||||
// NOTE This is actually DecodedRlp = Array<Uint8Array | DecodedRlp>, but since the decoder is relatively simple, we get away with the definition below
|
||||
export type DecodedRlp = Array<Uint8Array | Array<Uint8Array>>;
|
||||
|
||||
export type HashFn = (data: Uint8Array) => Uint8Array;
|
||||
|
||||
export type Trie$Node$Type = 'branch' | 'extention' | 'leaf';
|
||||
|
||||
export interface Trie$Node {
|
||||
hash: () => Uint8Array,
|
||||
serialize: () => Uint8Array,
|
||||
|
||||
raw: DecodedRlp,
|
||||
type: Trie$Node$Type,
|
||||
value: Uint8Array
|
||||
}
|
||||
|
||||
type Trie$Find$OnResult = (nodeRef: Uint8Array, node: Trie$Node, nibbles: Array<number>, next: () => void) => void;
|
||||
|
||||
export type Trie$Base = {
|
||||
_findDbNodes (onResult: Trie$Find$OnResult, onDone: Callback): void,
|
||||
_findValueNodes (onResult: Trie$Find$OnResult, onDone: Callback): void
|
||||
};
|
||||
|
||||
export type SemaphorePromise = (fn: () => Promise<any>) => Promise<any>;
|
||||
|
||||
type NodeFactory = {
|
||||
fromBranch (rlp: DecodedRlp): Trie$Node
|
||||
fromRaw (rlp: DecodedRlp): Trie$Node,
|
||||
fromExtention (key: Uint8Array, value: Uint8Array): Trie$Node,
|
||||
fromLeaf (key: Uint8Array, value: Uint8Array): Trie$Node
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
// Copyright 2017-2018 @polkadot/trie-db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the ISC license. See the LICENSE file for details.
|
||||
|
||||
export enum NodeType {
|
||||
EMPTY = 0,
|
||||
LEAF = 1,
|
||||
EXTENSION = 2,
|
||||
BRANCH = 3
|
||||
}
|
||||
|
||||
export type NodeEmpty = null;
|
||||
|
||||
export type NodeEncoded = Uint8Array;
|
||||
|
||||
export type NodeEncodedOrEmpty = NodeEncoded | NodeEmpty;
|
||||
|
||||
export type NodeBranch = [
|
||||
NodeEncodedOrEmpty, NodeEncodedOrEmpty, NodeEncodedOrEmpty, NodeEncodedOrEmpty,
|
||||
NodeEncodedOrEmpty, NodeEncodedOrEmpty, NodeEncodedOrEmpty, NodeEncodedOrEmpty,
|
||||
NodeEncodedOrEmpty, NodeEncodedOrEmpty, NodeEncodedOrEmpty, NodeEncodedOrEmpty,
|
||||
NodeEncodedOrEmpty, NodeEncodedOrEmpty, NodeEncodedOrEmpty, NodeEncodedOrEmpty,
|
||||
NodeEncodedOrEmpty
|
||||
];
|
||||
|
||||
export type EncodedPath = Uint8Array | null;
|
||||
|
||||
export type NodeKv = [EncodedPath, NodeEncodedOrEmpty];
|
||||
|
||||
export type NodeNotEmpty = NodeKv | NodeBranch;
|
||||
|
||||
export type Node = NodeEmpty | NodeNotEmpty;
|
||||
@@ -0,0 +1,29 @@
|
||||
// Copyright 2017-2018 @polkadot/trie-db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the ISC license. See the LICENSE file for details.
|
||||
|
||||
import { Node, NodeBranch, NodeKv, NodeEmpty, NodeType } from '../types';
|
||||
|
||||
import isNull from '@polkadot/util/is/null';
|
||||
|
||||
import { getNodeType } from './node';
|
||||
|
||||
export function isEmptyNode (node: Node): node is NodeEmpty {
|
||||
return isNull(node);
|
||||
}
|
||||
|
||||
export function isKvNode (node: Node): node is NodeKv {
|
||||
return !isEmptyNode(node) && node.length === 2;
|
||||
}
|
||||
|
||||
export function isExtensionNode (node: Node): node is NodeKv {
|
||||
return getNodeType(node) === NodeType.EXTENSION;
|
||||
}
|
||||
|
||||
export function isLeafNode (node: Node): node is NodeKv {
|
||||
return getNodeType(node) === NodeType.LEAF;
|
||||
}
|
||||
|
||||
export function isBranchNode (node: Node): node is NodeBranch {
|
||||
return !isEmptyNode(node) && node.length === 17;
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
// Copyright 2015-2018 authors & contributors of https://github.com/ethereumjs/merkle-patricia-tree
|
||||
// Copyright 2017-2018 @polkadot/trie-db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the MPL-2.0 license. See the LICENSE file for details.
|
||||
|
||||
import isU8a from '@polkadot/util/is/u8a';
|
||||
|
||||
export default function isRawNode (node: Array<number> | Buffer | Uint8Array) {
|
||||
return Array.isArray(node) && !isU8a(node);
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
// Copyright 2017-2018 @polkadot/trie-db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the ISC license. See the LICENSE file for details.
|
||||
|
||||
import isNull from '@polkadot/util/is/null';
|
||||
|
||||
import { EncodedPath, Node } from '../types';
|
||||
|
||||
import { isKvNode } from './is';
|
||||
import { addNibblesTerminator, decodeNibbles, encodeNibbles, removeNibblesTerminator } from './nibbles';
|
||||
|
||||
export function extractKey (node: Node): Uint8Array {
|
||||
if (!isKvNode(node)) {
|
||||
throw new Error('Can only extract keys from KV branches');
|
||||
}
|
||||
|
||||
const [prefixedKey] = node;
|
||||
|
||||
return removeNibblesTerminator(
|
||||
decodeNibbles(prefixedKey)
|
||||
);
|
||||
}
|
||||
|
||||
export function keyEquals (key: Uint8Array | null, test: Uint8Array | null): boolean {
|
||||
if (isNull(key) && isNull(test)) {
|
||||
return true;
|
||||
} else if (isNull(key) || isNull(test) || (key.length !== test.length)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return keyStartsWith(key, test);
|
||||
}
|
||||
|
||||
export function keyStartsWith (key: Uint8Array | null, partial: Uint8Array | null): boolean {
|
||||
if (isNull(key) && isNull(partial)) {
|
||||
return true;
|
||||
} else if (isNull(key) || isNull(partial) || (key.length < partial.length)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (let index = 0; index < partial.length; index++) {
|
||||
if (key[index] !== partial[index]) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
export function computeExtensionKey (nibbles: Uint8Array): EncodedPath {
|
||||
return encodeNibbles(nibbles);
|
||||
}
|
||||
|
||||
export function computeLeafKey (nibbles: Uint8Array): EncodedPath {
|
||||
return encodeNibbles(
|
||||
addNibblesTerminator(nibbles)
|
||||
);
|
||||
}
|
||||
|
||||
export function getCommonPrefixLength (left: Uint8Array, right: Uint8Array): number {
|
||||
for (let index = 0; index < left.length && index < right.length; index++) {
|
||||
if (left[index] !== right[index]) {
|
||||
return index;
|
||||
}
|
||||
}
|
||||
|
||||
return Math.min(left.length, right.length);
|
||||
}
|
||||
|
||||
export function consumeCommonPrefix (left: Uint8Array, right: Uint8Array): [Uint8Array, Uint8Array, Uint8Array] {
|
||||
const length = getCommonPrefixLength(left, right);
|
||||
|
||||
return [
|
||||
left.subarray(0, length),
|
||||
left.subarray(length),
|
||||
right.subarray(length)
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
// Copyright 2017-2018 @polkadot/trie-db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the ISC license. See the LICENSE file for details.
|
||||
|
||||
import { NodeEncodedOrEmpty, NodeEncoded } from '../types';
|
||||
|
||||
import fromNibbles from '@polkadot/trie-hash/util/fromNibbles';
|
||||
import toNibbles from '@polkadot/trie-hash/util/asNibbles';
|
||||
|
||||
const HP_FLAG_2 = 2;
|
||||
const HP_FLAG_0 = 0;
|
||||
|
||||
const NIBBLE_TERMINATOR = 16;
|
||||
|
||||
const NEEDS_TERMINATOR = [HP_FLAG_2, HP_FLAG_2 + 1];
|
||||
const IS_ODD_LENGTH = [HP_FLAG_0 + 1, HP_FLAG_2 + 1];
|
||||
|
||||
export function isNibblesTerminated (nibbles: Uint8Array): boolean {
|
||||
return nibbles[nibbles.length - 1] === NIBBLE_TERMINATOR;
|
||||
}
|
||||
|
||||
export function addNibblesTerminator (nibbles: Uint8Array): Uint8Array {
|
||||
if (isNibblesTerminated(nibbles)) {
|
||||
return nibbles;
|
||||
}
|
||||
|
||||
const terminated = new Uint8Array(nibbles.length + 1);
|
||||
|
||||
terminated.set(nibbles);
|
||||
terminated[nibbles.length] = NIBBLE_TERMINATOR;
|
||||
|
||||
return terminated;
|
||||
}
|
||||
|
||||
export function removeNibblesTerminator (nibbles: Uint8Array): Uint8Array {
|
||||
if (isNibblesTerminated(nibbles)) {
|
||||
return nibbles.subarray(0, nibbles.length - 1);
|
||||
}
|
||||
|
||||
return nibbles;
|
||||
}
|
||||
|
||||
export function decodeNibbles (value: NodeEncodedOrEmpty): Uint8Array {
|
||||
const nibblesWithFlag = toNibbles(value);
|
||||
const [flag] = nibblesWithFlag;
|
||||
|
||||
const rawNibbles = IS_ODD_LENGTH.includes(flag)
|
||||
? nibblesWithFlag.subarray(1)
|
||||
: nibblesWithFlag.subarray(2);
|
||||
|
||||
return NEEDS_TERMINATOR.includes(flag)
|
||||
? addNibblesTerminator(rawNibbles)
|
||||
: rawNibbles;
|
||||
}
|
||||
|
||||
export function encodeNibbles (nibbles: Uint8Array): NodeEncoded {
|
||||
const flag = isNibblesTerminated(nibbles)
|
||||
? HP_FLAG_2
|
||||
: HP_FLAG_0;
|
||||
const rawNibbles = removeNibblesTerminator(nibbles);
|
||||
const prefix = rawNibbles.length % 2
|
||||
? [flag + 1]
|
||||
: [flag, 0];
|
||||
const prefixed = new Uint8Array(prefix.length + rawNibbles.length);
|
||||
|
||||
prefixed.set(prefix);
|
||||
prefixed.set(rawNibbles, prefix.length);
|
||||
|
||||
return fromNibbles(prefixed);
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
// Copyright 2017-2018 @polkadot/trie-db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the ISC license. See the LICENSE file for details.
|
||||
|
||||
import { Node, NodeType, NodeNotEmpty } from '../types';
|
||||
|
||||
import isNull from '@polkadot/util/is/null';
|
||||
import nodeDecode from '@polkadot/util-rlp/decode';
|
||||
import nodeEncode from '@polkadot/util-rlp/encode';
|
||||
|
||||
import { isBranchNode, isEmptyNode, isKvNode } from './is';
|
||||
import { decodeNibbles, isNibblesTerminated } from './nibbles';
|
||||
|
||||
export function getNodeType (node: Node): NodeType {
|
||||
if (isEmptyNode(node)) {
|
||||
return NodeType.EMPTY;
|
||||
} else if (isKvNode(node)) {
|
||||
const [key] = node;
|
||||
const nibbles = decodeNibbles(key);
|
||||
|
||||
return isNibblesTerminated(nibbles)
|
||||
? NodeType.LEAF
|
||||
: NodeType.EXTENSION;
|
||||
} else if (isBranchNode(node)) {
|
||||
return NodeType.BRANCH;
|
||||
}
|
||||
|
||||
throw new Error(`Unable to determine node type`);
|
||||
}
|
||||
|
||||
export function decodeNode (encoded: Uint8Array | Node): Node {
|
||||
if (isNull(encoded) || encoded.length === 0) {
|
||||
return null;
|
||||
} else if (Array.isArray(encoded)) {
|
||||
return encoded;
|
||||
}
|
||||
|
||||
const decoded = (nodeDecode(encoded) as NodeNotEmpty)
|
||||
.map((value) =>
|
||||
value && value.length
|
||||
? value
|
||||
: null
|
||||
);
|
||||
|
||||
return decoded as NodeNotEmpty;
|
||||
}
|
||||
|
||||
export function encodeNode (node: Node): Uint8Array {
|
||||
return nodeEncode(node);
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
// Copyright 2017-2018 @polkadot/trie-db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the MPL-2.0 license. See the LICENSE file for details.
|
||||
|
||||
import { DecodedRlp } from '../types';
|
||||
|
||||
import assert from '@polkadot/util/assert';
|
||||
import isU8a from '@polkadot/util/is/u8a';
|
||||
import u8aToHex from '@polkadot/util/u8a/toHex';
|
||||
|
||||
export default function rlpToString (rlp: Uint8Array | DecodedRlp): string {
|
||||
if (Array.isArray(rlp)) {
|
||||
return `[ ${rlp.map(rlpToString).join(', ')} ]`;
|
||||
}
|
||||
|
||||
assert(isU8a(rlp), `Expected Array | Uint8Array inside rlp, found type '${typeof rlp}'`);
|
||||
|
||||
return u8aToHex(rlp);
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
// Copyright 2017-2018 @polkadot/trie-db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the MPL-2.0 license. See the LICENSE file for details.
|
||||
|
||||
import { SemaphorePromise } from '../types';
|
||||
|
||||
import semaphore from 'semaphore';
|
||||
|
||||
export default function semaphorePromise (max: number): SemaphorePromise {
|
||||
const _semaphore = semaphore(max);
|
||||
|
||||
return (fn: () => Promise<any>): Promise<any> => {
|
||||
return new Promise((resolve, reject) => {
|
||||
_semaphore.take(async () => {
|
||||
try {
|
||||
const result = await fn();
|
||||
|
||||
_semaphore.leave();
|
||||
resolve(result);
|
||||
} catch (error) {
|
||||
_semaphore.leave();
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
});
|
||||
};
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
// Copyright 2015-2018 authors & contributors of https://github.com/ethereumjs/merkle-patricia-tree
|
||||
// Copyright 2017-2018 @polkadot/trie-db authors & contributors
|
||||
// This software may be modified and distributed under the terms
|
||||
// of the MPL-2.0 license. See the LICENSE file for details.
|
||||
|
||||
type Task$Fn = (taskCb: (value: any) => any) => any;
|
||||
|
||||
type Task = {
|
||||
fn: Task$Fn,
|
||||
priority: number,
|
||||
reject: (error: Error) => void,
|
||||
resolve: (value: any) => void
|
||||
};
|
||||
|
||||
// https://github.com/ethereumjs/merkle-patricia-tree/pull/38
|
||||
export default function taskExecutor (maxPoolSize: number) {
|
||||
let currentPoolSize = 0;
|
||||
const queue: Array<Task> = [];
|
||||
|
||||
const execute = (task: Task | undefined) => {
|
||||
if (task === undefined) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (currentPoolSize < maxPoolSize) {
|
||||
currentPoolSize++;
|
||||
|
||||
task.fn((value: any): void => {
|
||||
currentPoolSize--;
|
||||
|
||||
task.resolve(value);
|
||||
|
||||
if (queue.length > 0) {
|
||||
queue.sort((a, b) =>
|
||||
b.priority - a.priority
|
||||
);
|
||||
|
||||
execute(queue.shift());
|
||||
}
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
queue.push(task);
|
||||
};
|
||||
|
||||
return (priority: number, fn: Task$Fn): Promise<any> => {
|
||||
return new Promise((resolve, reject) => {
|
||||
execute({ fn, reject, resolve, priority });
|
||||
});
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user