From e1135a665bb58dcac1eef3b790ec092f843ec79d Mon Sep 17 00:00:00 2001 From: Jaco Greeff Date: Sat, 1 Sep 2018 22:17:57 +0200 Subject: [PATCH] Re-implementation of Patricia-Merkle Trie (#203) * Start of trie-db re-implementation * WIP commit * More WIP * WIP, seems complete, untested * Ported trie-db operational * Add Ethereum trie tests * Use u8aConcat to construct u8a * Basic RocksDb interface * Combine engines into DB, expand tests * Update Trie with transaction wrapper * LevelDb (sync) interface * Remove unused (for now) package (compatibility) * trie-d -> trie-db-old, trie-db-ext -> trie-db * Version bump * Skip old trie tests (package to be removed) * Skip tests (fixed) * Remove non-used RocksDb spec * Skip FileFlat tests (deprecated?) --- jest.config.js | 4 +- lerna.json | 2 +- packages/db-diskdown/README.md | 3 - packages/db-diskdown/package.json | 40 -- packages/db-diskdown/src/index.ts | 115 ----- .../src/store/CombinedCompact.spec.js | 19 - packages/db-diskdown/src/types.d.ts | 18 - packages/{db-diskdown => db}/LICENSE | 2 +- packages/db/README.md | 3 + packages/db/package.json | 39 ++ packages/db/src/Disk.ts | 17 + packages/db/src/Memory.ts | 12 + .../src/engines/FileFlatDb.spec.js} | 57 ++- .../src/engines/FileFlatDb.ts} | 127 +++-- .../src/engines/FileTreeDb.ts} | 56 ++- packages/db/src/engines/LevelDb.spec.js | 64 +++ packages/db/src/engines/LevelDb.ts | 83 +++ packages/db/src/engines/LruDb.spec.js | 63 +++ packages/db/src/engines/LruDb.ts | 78 +++ packages/db/src/engines/MemoryDb.ts | 54 ++ packages/db/src/engines/RocksDb.ts | 83 +++ packages/db/src/engines/TransactionDb.spec.js | 93 ++++ packages/db/src/engines/TransactionDb.ts | 142 ++++++ packages/db/src/nosql-leveldb.d.ts | 34 ++ packages/db/src/rocksdb-node.d.ts | 27 + packages/db/src/snappy.d.ts | 14 + packages/db/src/types.d.ts | 26 + packages/trie-db-old/LICENSE | 373 ++++++++++++++ packages/trie-db-old/README.md | 30 ++ packages/trie-db-old/package.json | 45 ++ .../{trie-db => trie-db-old}/src/BaseTrie.ts | 24 +- .../src/CheckpointTrie.ts | 0 packages/{trie-db => trie-db-old}/src/Node.ts | 28 +- packages/trie-db-old/src/constants.ts | 13 + .../src/encoder/decode.ts | 0 .../src/encoder/encode.ts | 0 .../src/encoder/index.ts | 0 packages/trie-db-old/src/index.ts | 23 + .../src/nibbles/fromU8a.ts | 0 .../src/nibbles/index.ts | 4 +- .../src/nibbles/isEqual.ts | 0 .../src/nibbles/matchingLength.ts | 0 .../src/nodeFactory.ts | 3 +- .../src/proof/index.ts | 0 .../src/proof/prove.ts | 0 .../src/proof/verifyProof.ts | 0 .../src/streams/ScratchRead.ts | 0 .../src/streams/TrieRead.ts | 3 +- .../{trie-db => trie-db-old}/src/types.d.ts | 0 .../src/util/isRawNode.ts | 0 .../src/util/rlpToString.ts | 0 .../src/util/semaphore.ts | 0 .../src/util/taskExecutor.ts | 0 .../test/checkpoints.spec.js | 2 +- .../test/ethereum.spec.js | 2 +- .../test/index.spec.js | 42 +- .../test/offical.spec.js | 6 +- .../test/polkadotRefactor.spec.js | 2 +- .../test/proof.spec.js | 2 +- .../test/rawOPs.spec.js | 2 +- .../test/streams.spec.js | 4 +- .../test/taskExecutor.spec.js | 2 +- packages/trie-db/LICENSE | 382 +------------- packages/trie-db/README.md | 29 +- packages/trie-db/package.json | 48 +- packages/trie-db/src/constants.ts | 17 +- packages/trie-db/src/ethereum.spec.js | 59 +++ packages/trie-db/src/index.spec.js | 267 ++++++++++ packages/trie-db/src/index.ts | 473 +++++++++++++++++- packages/trie-db/src/nibbles/toU8a.ts | 16 - packages/trie-db/src/types.ts | 32 ++ packages/trie-db/src/util/is.ts | 29 ++ packages/trie-db/src/util/key.ts | 78 +++ packages/trie-db/src/util/nibbles.ts | 70 +++ packages/trie-db/src/util/node.ts | 50 ++ packages/trie-hash/package.json | 8 +- packages/trie-hash/src/util/asNibbles.ts | 14 +- .../trie-hash/src/util/fromNibbles.spec.js | 39 ++ packages/trie-hash/src/util/fromNibbles.ts | 26 + packages/util-crypto/package.json | 4 +- packages/util-keyring/package.json | 6 +- packages/util-rlp/package.json | 4 +- packages/util/package.json | 2 +- packages/util/src/is/object.ts | 6 +- packages/util/src/logger.ts | 47 +- tsconfig.json | 4 +- yarn.lock | 319 +++++++++++- 87 files changed, 3091 insertions(+), 823 deletions(-) delete mode 100644 packages/db-diskdown/README.md delete mode 100644 packages/db-diskdown/package.json delete mode 100644 packages/db-diskdown/src/index.ts delete mode 100644 packages/db-diskdown/src/store/CombinedCompact.spec.js delete mode 100644 packages/db-diskdown/src/types.d.ts rename packages/{db-diskdown => db}/LICENSE (91%) create mode 100644 packages/db/README.md create mode 100644 packages/db/package.json create mode 100644 packages/db/src/Disk.ts create mode 100644 packages/db/src/Memory.ts rename packages/{db-diskdown/src/store/Combined.spec.js => db/src/engines/FileFlatDb.spec.js} (63%) rename packages/{db-diskdown/src/store/Combined.ts => db/src/engines/FileFlatDb.ts} (80%) rename packages/{db-diskdown/src/store/Scatter.ts => db/src/engines/FileTreeDb.ts} (54%) create mode 100644 packages/db/src/engines/LevelDb.spec.js create mode 100644 packages/db/src/engines/LevelDb.ts create mode 100644 packages/db/src/engines/LruDb.spec.js create mode 100644 packages/db/src/engines/LruDb.ts create mode 100644 packages/db/src/engines/MemoryDb.ts create mode 100644 packages/db/src/engines/RocksDb.ts create mode 100644 packages/db/src/engines/TransactionDb.spec.js create mode 100644 packages/db/src/engines/TransactionDb.ts create mode 100644 packages/db/src/nosql-leveldb.d.ts create mode 100644 packages/db/src/rocksdb-node.d.ts create mode 100644 packages/db/src/snappy.d.ts create mode 100644 packages/db/src/types.d.ts create mode 100644 packages/trie-db-old/LICENSE create mode 100644 packages/trie-db-old/README.md create mode 100644 packages/trie-db-old/package.json rename packages/{trie-db => trie-db-old}/src/BaseTrie.ts (97%) rename packages/{trie-db => trie-db-old}/src/CheckpointTrie.ts (100%) rename packages/{trie-db => trie-db-old}/src/Node.ts (89%) create mode 100644 packages/trie-db-old/src/constants.ts rename packages/{trie-db => trie-db-old}/src/encoder/decode.ts (100%) rename packages/{trie-db => trie-db-old}/src/encoder/encode.ts (100%) rename packages/{trie-db => trie-db-old}/src/encoder/index.ts (100%) create mode 100644 packages/trie-db-old/src/index.ts rename packages/{trie-db => trie-db-old}/src/nibbles/fromU8a.ts (100%) rename packages/{trie-db => trie-db-old}/src/nibbles/index.ts (83%) rename packages/{trie-db => trie-db-old}/src/nibbles/isEqual.ts (100%) rename packages/{trie-db => trie-db-old}/src/nibbles/matchingLength.ts (100%) rename packages/{trie-db => trie-db-old}/src/nodeFactory.ts (96%) rename packages/{trie-db => trie-db-old}/src/proof/index.ts (100%) rename packages/{trie-db => trie-db-old}/src/proof/prove.ts (100%) rename packages/{trie-db => trie-db-old}/src/proof/verifyProof.ts (100%) rename packages/{trie-db => trie-db-old}/src/streams/ScratchRead.ts (100%) rename packages/{trie-db => trie-db-old}/src/streams/TrieRead.ts (96%) rename packages/{trie-db => trie-db-old}/src/types.d.ts (100%) rename packages/{trie-db => trie-db-old}/src/util/isRawNode.ts (100%) rename packages/{trie-db => trie-db-old}/src/util/rlpToString.ts (100%) rename packages/{trie-db => trie-db-old}/src/util/semaphore.ts (100%) rename packages/{trie-db => trie-db-old}/src/util/taskExecutor.ts (100%) rename packages/{trie-db => trie-db-old}/test/checkpoints.spec.js (97%) rename packages/{trie-db => trie-db-old}/test/ethereum.spec.js (95%) rename packages/{trie-db => trie-db-old}/test/index.spec.js (75%) rename packages/{trie-db => trie-db-old}/test/offical.spec.js (92%) rename packages/{trie-db => trie-db-old}/test/polkadotRefactor.spec.js (97%) rename packages/{trie-db => trie-db-old}/test/proof.spec.js (98%) rename packages/{trie-db => trie-db-old}/test/rawOPs.spec.js (97%) rename packages/{trie-db => trie-db-old}/test/streams.spec.js (97%) rename packages/{trie-db => trie-db-old}/test/taskExecutor.spec.js (95%) create mode 100644 packages/trie-db/src/ethereum.spec.js create mode 100644 packages/trie-db/src/index.spec.js delete mode 100644 packages/trie-db/src/nibbles/toU8a.ts create mode 100644 packages/trie-db/src/types.ts create mode 100644 packages/trie-db/src/util/is.ts create mode 100644 packages/trie-db/src/util/key.ts create mode 100644 packages/trie-db/src/util/nibbles.ts create mode 100644 packages/trie-db/src/util/node.ts create mode 100644 packages/trie-hash/src/util/fromNibbles.spec.js create mode 100644 packages/trie-hash/src/util/fromNibbles.ts diff --git a/jest.config.js b/jest.config.js index 535090d68..6ef8a93fd 100644 --- a/jest.config.js +++ b/jest.config.js @@ -2,8 +2,8 @@ const config = require('@polkadot/dev/config/jest'); module.exports = Object.assign({}, config, { moduleNameMapper: { - '@polkadot/db-(diskdown)(.*)$': '/packages/db-$1/src/$2', - '@polkadot/trie-(db|hash)(.*)$': '/packages/trie-$1/src/$2', + '@polkadot/db(.*)$': '/packages/db/src/$1', + '@polkadot/trie-(db-old|db|hash)(.*)$': '/packages/trie-$1/src/$2', '@polkadot/util-(crypto|keyring|rlp)(.*)$': '/packages/util-$1/src/$2', '@polkadot/util(.*)$': '/packages/util/src/$1' } diff --git a/lerna.json b/lerna.json index 964241190..471bd1567 100644 --- a/lerna.json +++ b/lerna.json @@ -9,5 +9,5 @@ "packages": [ "packages/*" ], - "version": "0.28.25" + "version": "0.29.0" } diff --git a/packages/db-diskdown/README.md b/packages/db-diskdown/README.md deleted file mode 100644 index 5436fd947..000000000 --- a/packages/db-diskdown/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# @polkadot/db-diskdown - -Simple AbstractDOWN file-based key-value store implementation diff --git a/packages/db-diskdown/package.json b/packages/db-diskdown/package.json deleted file mode 100644 index 27e977d28..000000000 --- a/packages/db-diskdown/package.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "name": "@polkadot/db-diskdown", - "version": "0.28.25", - "description": "Simple file-base key-value storage system", - "main": "index.js", - "engines": { - "node": ">=8.0" - }, - "publishConfig": { - "access": "public", - "registry": "https://registry.npmjs.org" - }, - "repository": { - "type": "git", - "url": "git+https://github.com/polkadot-js/client.git" - }, - "keywords": [ - "AbstractLevelDOWN" - ], - "author": "Jaco Greeff ", - "maintainers": [ - "Jaco Greeff " - ], - "contributors": [], - "license": "ISC", - "bugs": { - "url": "https://github.com/polkadot-js/client/issues" - }, - "homepage": "https://github.com/polkadot-js/client/tree/master/packages/db-diskdown#readme", - "dependencies": { - "@babel/runtime": "^7.0.0-rc.1", - "@polkadot/util": "^0.28.25", - "@types/mkdirp": "^0.5.2", - "abstract-leveldown": "^5.0.0", - "mkdirp": "^0.5.1" - }, - "devDependencies": { - "rimraf": "^2.6.2" - } -} diff --git a/packages/db-diskdown/src/index.ts b/packages/db-diskdown/src/index.ts deleted file mode 100644 index b56d3f8f7..000000000 --- a/packages/db-diskdown/src/index.ts +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2017-2018 @polkadot/db-diskdown authors & contributors -// This software may be modified and distributed under the terms -// of the ISC license. See the LICENSE file for details. - -import { DiskStore, ProgressValue } from './types'; - -import { AbstractLevelDOWN } from 'abstract-leveldown'; -import isUndefined from '@polkadot/util/is/undefined'; -import logger from '@polkadot/util/logger'; - -import Combined from './store/Combined'; - -const KEY_LENGTH = 32; - -const l = logger('disk/scatter'); -const noop = () => - undefined; - -class DiskDown extends AbstractLevelDOWN { - _disk: DiskStore; - - constructor (location: string) { - super(location); - - this._disk = new Combined(location); - } - - compact (progress: (value: ProgressValue) => void): void { - l.debug(() => ['compact']); - - this._disk.compact(progress); - } - - _open (options: any, callback: Function) { - l.debug(() => ['_open', options]); - - this._disk.open(); - - process.nextTick(callback); - } - - _close (callback: Function) { - l.debug(() => ['_close']); - - this._disk.close(); - - process.nextTick(callback); - } - - _batch (array: Array, options: any, callback: Function) { - l.debug(() => ['_batch', array]); - - array.forEach(({ key, type, value }) => { - switch (type) { - case 'del': - return this._del(key, {}, noop); - - case 'put': - return this._put(key, value, {}, noop); - - default: - // ignore - } - }); - - process.nextTick(callback); - } - - _del (key: Buffer, options: any, callback: Function) { - l.debug(() => ['_del', key]); - - this._disk.delete(key); - - process.nextTick(callback); - } - - _get (key: Buffer, options: any, callback: Function) { - l.debug(() => ['_get', key.toString('hex')]); - - const value = this._disk.get(key); - - if (isUndefined(value)) { - process.nextTick(callback, new Error('NotFound')); - return; - } - - process.nextTick(callback, null, value); - } - - _put (key: Buffer, value: Buffer, options: any, callback: Function) { - l.debug(() => ['_put', key.toString('hex'), value]); - - this._disk.set(key, value); - - process.nextTick(callback); - } - - _serializeKey (key: Buffer): Buffer { - if (key.length === KEY_LENGTH) { - return key; - } else if (key.length < KEY_LENGTH) { - const paddedKey = Buffer.alloc(KEY_LENGTH); - - paddedKey.set(key, 0); - - return paddedKey; - } - - throw new Error(`${key.toString()} too large, expected <= 32 bytes`); - } -} - -export default function (location: string) { - return new DiskDown(location); -} diff --git a/packages/db-diskdown/src/store/CombinedCompact.spec.js b/packages/db-diskdown/src/store/CombinedCompact.spec.js deleted file mode 100644 index 9a53c284e..000000000 --- a/packages/db-diskdown/src/store/CombinedCompact.spec.js +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2017-2018 @polkadot/db-diskdown authors & contributors -// This software may be modified and distributed under the terms -// of the ISC license. See the LICENSE file for details. - -import mkdirp from 'mkdirp'; -import path from 'path'; -import rimraf from 'rimraf'; - -import Combined from './Combined'; - -describe.skip('Combined (compacting)', () => { - const combined = new Combined(process.cwd()); - - it('compacts without failing', () => { - expect( - combined.compact() - ).not.toBe(0); - }); -}); diff --git a/packages/db-diskdown/src/types.d.ts b/packages/db-diskdown/src/types.d.ts deleted file mode 100644 index c6e37c0e5..000000000 --- a/packages/db-diskdown/src/types.d.ts +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2017-2018 @polkadot/db-diskdown authors & contributors -// This software may be modified and distributed under the terms -// of the ISC license. See the LICENSE file for details. - -export type ProgressValue = { - depth: number, - keys: number, - percent: number -}; - -export interface DiskStore { - close (): void; - compact (progress: (value: ProgressValue) => void): void, - delete (key: Buffer): void; - get (key: Buffer): Buffer | undefined; - open (): void, - set (key: Buffer, value: Buffer): void; -} diff --git a/packages/db-diskdown/LICENSE b/packages/db/LICENSE similarity index 91% rename from packages/db-diskdown/LICENSE rename to packages/db/LICENSE index cab468ba7..16204d375 100644 --- a/packages/db-diskdown/LICENSE +++ b/packages/db/LICENSE @@ -1,6 +1,6 @@ ISC License (ISC) -Copyright 2017-2018 @polkadot/db-diskdown authors & contributors +Copyright 2017-2018 @polkadot/db authors & contributors Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above diff --git a/packages/db/README.md b/packages/db/README.md new file mode 100644 index 000000000..865ade2af --- /dev/null +++ b/packages/db/README.md @@ -0,0 +1,3 @@ +# @polkadot/db + +This is an implementation of a basic sync in-memory and on-disk database with overlays diff --git a/packages/db/package.json b/packages/db/package.json new file mode 100644 index 000000000..060fecd04 --- /dev/null +++ b/packages/db/package.json @@ -0,0 +1,39 @@ +{ + "name": "@polkadot/db", + "version": "0.29.0", + "description": "Implementation of a basic sync in-memory and on-disk databases with overlays", + "main": "index.js", + "keywords": [ + "Polkadot" + ], + "author": "Jaco Greeff ", + "maintainers": [ + "Jaco Greeff " + ], + "contributors": [], + "license": "ISC", + "engines": { + "node": ">=6.4" + }, + "publishConfig": { + "access": "public", + "registry": "https://registry.npmjs.org" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/polkadot-js/common.git" + }, + "bugs": { + "url": "https://github.com/polkadot-js/common/issues" + }, + "homepage": "https://github.com/polkadot-js/common/tree/master/packages/db#readme", + "dependencies": { + "@babel/runtime": "^7.0.0-rc.1", + "@polkadot/util": "^0.29.0", + "@types/mkdirp": "^0.5.2", + "lru_map": "^0.3.3", + "mkdirp": "^0.5.1", + "nosql-leveldb": "^3.0.1", + "snappy": "^6.0.4" + } +} diff --git a/packages/db/src/Disk.ts b/packages/db/src/Disk.ts new file mode 100644 index 000000000..0e47c5120 --- /dev/null +++ b/packages/db/src/Disk.ts @@ -0,0 +1,17 @@ +// Copyright 2017-2018 @polkadot/db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +import LevelDb from './engines/LevelDb'; +import LruDb from './engines/LruDb'; +import TransactionDb from './engines/TransactionDb'; + +export default class DiskDb extends TransactionDb { + constructor (location: string) { + super( + new LruDb( + new LevelDb(location) + ) + ); + } +} diff --git a/packages/db/src/Memory.ts b/packages/db/src/Memory.ts new file mode 100644 index 000000000..6ad42b071 --- /dev/null +++ b/packages/db/src/Memory.ts @@ -0,0 +1,12 @@ +// Copyright 2017-2018 @polkadot/db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +import MemoryDb from './engines/MemoryDb'; +import TransactionDb from './engines/TransactionDb'; + +export default class Memory extends TransactionDb { + constructor () { + super(new MemoryDb()); + } +} diff --git a/packages/db-diskdown/src/store/Combined.spec.js b/packages/db/src/engines/FileFlatDb.spec.js similarity index 63% rename from packages/db-diskdown/src/store/Combined.spec.js rename to packages/db/src/engines/FileFlatDb.spec.js index 9f7bad0be..6109d653a 100644 --- a/packages/db-diskdown/src/store/Combined.spec.js +++ b/packages/db/src/engines/FileFlatDb.spec.js @@ -1,61 +1,74 @@ -// Copyright 2017-2018 @polkadot/db-diskdown authors & contributors +// Copyright 2017-2018 @polkadot/db authors & contributors // This software may be modified and distributed under the terms // of the ISC license. See the LICENSE file for details. import mkdirp from 'mkdirp'; +import os from 'os'; import path from 'path'; import rimraf from 'rimraf'; -import Combined from './Combined'; +import FileFlatDb from './FileFlatDb'; -const KEY_A = Buffer.from([ +const KEY_A = new Uint8Array([ 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ]); -const KEY_B = Buffer.from([ +const KEY_B = new Uint8Array([ 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ]); -const KEY_C = Buffer.from([ +const KEY_C = new Uint8Array([ 1, 2, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ]); -const KEY_D = Buffer.from([ +const KEY_D = new Uint8Array([ 1, 2, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ]); -const KEY_E = Buffer.from([ +const KEY_E = new Uint8Array([ 1, 2, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ]); -const VAL_A = Buffer.from([0x42, 1, 0x69]); -const VAL_B = Buffer.from([0x42, 1, 2, 0x69]); -const VAL_C = Buffer.from([0x42, 1, 2, 3, 0x69]); -const VAL_D = Buffer.from([0x42, 1, 2, 3, 4, 0x69]); -const VAL_E = Buffer.from([0x42, 1, 2, 3, 4, 5, 0x69]); +const VAL_A = new Uint8Array([0x42, 1, 0x69]); +const VAL_B = new Uint8Array([0x42, 1, 2, 0x69]); +const VAL_C = new Uint8Array([0x42, 1, 2, 3, 0x69]); +const VAL_D = new Uint8Array([0x42, 1, 2, 3, 4, 0x69]); +const VAL_E = new Uint8Array([0x42, 1, 2, 3, 4, 5, 0x69]); + +describe.skip('FileFlatDb (compacting)', () => { + const combined = new FileFlatDb(process.cwd()); + + it('compacts without failing', () => { + expect( + combined.maintain(() => {}) + ).not.toBe(0); + }); +}); // NOTE Skipped, doesn't seem to be too happy on CI (cwd issues?) -describe.skip('StoreCombined', () => { - const testGet = (key, val) => - expect(store.get(key).equals(val)).toEqual(true); +describe.skip('FileFlatDb (basics)', () => { + const testGet = (key, value) => + expect(store.get(key)).toEqual(value); - const location = path.join(process.cwd(), '--test-Combined'); + const location = path.join(os.tmpdir(), '--test-FileFlatDb'); let store; beforeAll(() => { mkdirp.sync(location); - store = new Combined(location); + store = new FileFlatDb(location); + store.open(); }); afterAll(() => { + store.close(); rimraf.sync(location); }); it('writes an entry', () => { console.error('A: execute'); - store.set(KEY_A, VAL_A); + store.put(KEY_A, VAL_A); console.error('A: expectations'); @@ -65,7 +78,7 @@ describe.skip('StoreCombined', () => { it('writes an entry (additional)', () => { console.error('B: execute'); - store.set(KEY_B, VAL_B); + store.put(KEY_B, VAL_B); console.error('A: expectations'); @@ -76,7 +89,7 @@ describe.skip('StoreCombined', () => { it('writes an entry (expanding the tree)', () => { console.error('C: execute'); - store.set(KEY_C, VAL_C); + store.put(KEY_C, VAL_C); console.error('C: expectations'); @@ -88,7 +101,7 @@ describe.skip('StoreCombined', () => { it('writes an entry (expanding the tree, again)', () => { console.error('D: execute'); - store.set(KEY_D, VAL_D); + store.put(KEY_D, VAL_D); console.error('D: expectations'); @@ -101,7 +114,7 @@ describe.skip('StoreCombined', () => { it('writes an entry (expanding the tree, yet again)', () => { console.error('E: execute'); - store.set(KEY_E, VAL_E); + store.put(KEY_E, VAL_E); console.error('E: expectations'); diff --git a/packages/db-diskdown/src/store/Combined.ts b/packages/db/src/engines/FileFlatDb.ts similarity index 80% rename from packages/db-diskdown/src/store/Combined.ts rename to packages/db/src/engines/FileFlatDb.ts index 4a563b5e9..1943e08b4 100644 --- a/packages/db-diskdown/src/store/Combined.ts +++ b/packages/db/src/engines/FileFlatDb.ts @@ -1,12 +1,16 @@ -// Copyright 2017-2018 @polkadot/db-diskdown authors & contributors +// Copyright 2017-2018 @polkadot/db authors & contributors // This software may be modified and distributed under the terms // of the ISC license. See the LICENSE file for details. -import { DiskStore, ProgressValue } from '../types'; +import { BaseDb, ProgressCb } from '../types'; import fs from 'fs'; +// import snappy from 'snappy'; import assert from '@polkadot/util/assert'; import logger from '@polkadot/util/logger'; +import bufferToU8a from '@polkadot/util/buffer/toU8a'; +import u8aToBuffer from '@polkadot/util/u8a/toBuffer'; +import u8aToHex from '@polkadot/util/u8a/toHex'; enum Slot { EMPTY = 0, @@ -44,33 +48,20 @@ const KEY_TOTAL_SIZE = KEY_SIZE + UINT_SIZE + UINT_SIZE; const ENTRY_NUM = 256; const ENTRY_SIZE = 1 + UINT_SIZE; const HEADER_SIZE = ENTRY_NUM * ENTRY_SIZE; +const DEFAULT_FILE = 'store.db'; -const l = logger('disk/combined'); +const l = logger('db/flat'); -function debug (obj: any) { - return JSON.stringify( - !obj - ? obj - : Object.keys(obj).reduce((result, key) => { - result[key] = Buffer.isBuffer(obj[key]) - ? obj[key].toString('hex') - : obj[key]; - - return result; - }, {} as any) - ); -} - -export default class Combined implements DiskStore { +export default class FileFlatDb implements BaseDb { _fd: number; _file: string; - constructor (location: string) { + constructor (location: string, file: string = DEFAULT_FILE) { if (!fs.existsSync(location)) { throw new Error(`Unable to open ${location}`); } - this._file = `${location}/store.db`; + this._file = `${location}/${file}`; this._fd = -1; } @@ -84,7 +75,7 @@ export default class Combined implements DiskStore { fs.closeSync(this._fd); } - compact (progress: (value: ProgressValue) => void): void { + maintain (fn: ProgressCb): void { assert(this._fd === -1, 'Database cannot be open for compacting'); l.log('compacting database'); @@ -93,7 +84,7 @@ export default class Combined implements DiskStore { const newFile = `${this._file}.compacted`; const newFd = this._open(newFile, true); const oldFd = this._open(this._file); - const keys = this._compact(progress, newFd, oldFd); + const keys = this._compact(fn, newFd, oldFd); fs.closeSync(oldFd); fs.closeSync(newFd); @@ -105,42 +96,78 @@ export default class Combined implements DiskStore { const elapsed = (Date.now() - start) / 1000; fs.unlinkSync(this._file); - // fs.renameSync(this._file, `${this._file}.old`); fs.renameSync(newFile, this._file); l.log(`compacted in ${elapsed.toFixed(2)}s, ${(keys / 1000).toFixed(2)}k keys, ${sizeMB.toFixed(2)}MB (${percentage.toFixed(2)}%)`); } - delete (key: Buffer): void { + del (key: Uint8Array): void { throw new Error('delete not implemented, only stubbed'); } - get (key: Buffer): Buffer | undefined { + get (key: Uint8Array): Uint8Array | null { this.assertOpen(); - l.debug(() => ['get', debug({ key })]); + l.debug(() => ['get', { key }]); - const desc = this.findKey(key, false); + const desc = this.findKey(this._serializeKey(key), false); if (!desc) { - return; + return null; } - return this.readValue(desc).value; + const result = this.readValue(desc); + + return result && result.value + ? this._deserializeValue(result.value) + : null; } - set (key: Buffer, value: Buffer): void { + put (key: Uint8Array, value: Uint8Array): void { this.assertOpen(); - l.debug(() => ['set', debug({ key, value })]); + l.debug(() => ['set', { key, value }]); - const desc = this.findKey(key, true); + const desc = this.findKey(this._serializeKey(key), true); if (!desc) { throw new Error('Unable to create key'); } - this.writeValue(desc, value); + this.writeValue( + desc, + this._serializeValue(value) + ); + } + + private _deserializeValue (value: Buffer): Uint8Array | null { + // return bufferToU8a( + // snappy.uncompressSync(value) + // ); + + return bufferToU8a(value); + } + + private _serializeValue (value: Uint8Array): Buffer { + // return snappy.compressSync( + // u8aToBuffer(value) + // ); + + return u8aToBuffer(value); + } + + private _serializeKey (key: Uint8Array): Buffer { + if (key.length === KEY_SIZE) { + return u8aToBuffer(key); + } else if (key.length < KEY_SIZE) { + const paddedKey = Buffer.alloc(KEY_SIZE); + + paddedKey.set(key, 0); + + return paddedKey; + } + + throw new Error(`${u8aToHex(key)} too large, expected <= 32 bytes`); } private _compactReadEntry (fd: number, at: number, index: number): Buffer { @@ -198,7 +225,7 @@ export default class Combined implements DiskStore { return headerAt; } - private _compact (progress: (value: ProgressValue) => void, newFd: number, oldFd: number): number { + private _compact (fn: ProgressCb, newFd: number, oldFd: number): number { // l.debug(() => ['_compact', debug({ newFd, oldFd, newAt, oldAt })]); let keys = 0; @@ -235,8 +262,8 @@ export default class Combined implements DiskStore { throw new Error(`Unknown entry type, ${entryType}`); } - progress({ - depth, + fn({ + isCompleted: depth === 0 && index === (ENTRY_NUM - 1), keys, percent }); @@ -256,14 +283,14 @@ export default class Combined implements DiskStore { fs.readSync(this._fd, entry, 0, ENTRY_SIZE, entryAt); - l.debug(() => ['findKey', debug({ key, doCreate, keyIndex, diskAt, entry, entryAt })]); + l.debug(() => ['findKey', { key, doCreate, keyIndex, diskAt, entry, entryAt }]); const entryType = entry[0]; if (entryType === Slot.BRANCH) { const branchAt = entry.readUIntBE(1, UINT_SIZE); - l.debug(() => ['findKey/isBranch', debug({ branchAt })]); + l.debug(() => ['findKey/isBranch', { branchAt }]); return this._findKey(key, doCreate, keyIndex + 1, branchAt); } @@ -282,7 +309,7 @@ export default class Combined implements DiskStore { fs.readSync(this._fd, keyValue, 0, KEY_TOTAL_SIZE, keyAt); - l.debug(() => ['findKey/isLeaf', debug({ keyAt, entry, keyValue })]); + l.debug(() => ['findKey/isLeaf', { keyAt, entry, keyValue }]); let matchIndex = keyIndex; @@ -315,7 +342,7 @@ export default class Combined implements DiskStore { private findKey (key: Buffer, doCreate: boolean): Leaf | null { const result = this._findKey(key, doCreate, 0, 0); - l.debug(() => ['findKey', '=>', debug(result)]); + l.debug(() => ['findKey', { result }]); return result; } @@ -328,7 +355,7 @@ export default class Combined implements DiskStore { } private _readValue (key: Buffer, keyAt: number, keyValue: Buffer): Value { - l.debug(() => ['readValue', debug({ key, keyAt, keyValue })]); + l.debug(() => ['readValue', { key, keyAt, keyValue }]); const { valueAt, valueLength } = this._extractValueInfo(keyValue); const value = Buffer.alloc(valueLength); @@ -347,13 +374,13 @@ export default class Combined implements DiskStore { private readValue ({ key, keyAt, keyValue }: Key): Value { const result = this._readValue(key, keyAt, keyValue); - l.debug(() => ['readValue', '=>', debug(result)]); + l.debug(() => ['readValue', { result }]); return result; } private _writeValue (key: Buffer, keyAt: number, keyValue: Buffer, value: Buffer): Value { - l.debug(() => ['writeValue', debug({ keyAt, keyValue, value })]); + l.debug(() => ['writeValue', { keyAt, keyValue, value }]); let { valueAt, valueLength } = this._extractValueInfo(keyValue); @@ -382,13 +409,13 @@ export default class Combined implements DiskStore { private writeValue ({ key, keyAt, keyValue }: Key, value: Buffer): Value { const result = this._writeValue(key, keyAt, keyValue, value); - l.debug(() => ['writeValue', '=>', debug(result)]); + l.debug(() => ['writeValue', '=>', { result }]); return result; } private _writeNewKey (key: Buffer): Key { - l.debug(() => ['writeNewKey', debug({ key })]); + l.debug(() => ['writeNewKey', { key }]); const stats = fs.fstatSync(this._fd); const keyAt = stats.size; @@ -408,13 +435,13 @@ export default class Combined implements DiskStore { private writeNewKey (key: Buffer): Key { const result = this._writeNewKey(key); - l.debug(() => ['writeNewKey', '=>', debug(result)]); + l.debug(() => ['writeNewKey', '=>', { result }]); return result; } private _writeNewBranch (entry: Buffer, entryAt: number, key: Buffer, prevAt: number, prevValue: Buffer, matchIndex: number, depth: number): Branch { - l.debug(() => ['writeNewBranch', debug({ entry, entryAt, key, prevValue, matchIndex, depth })]); + l.debug(() => ['writeNewBranch', { entry, entryAt, key, prevValue, matchIndex, depth }]); const { keyAt, keyValue } = this.writeNewKey(key); const branch = Buffer.alloc(HEADER_SIZE); @@ -461,13 +488,13 @@ export default class Combined implements DiskStore { private writeNewBranch (entry: Buffer, entryAt: number, key: Buffer, prevAt: number, prevValue: Buffer, matchIndex: number, depth: number): Leaf { const result = this._writeNewBranch(entry, entryAt, key, prevAt, prevValue, matchIndex, depth); - l.debug(() => ['writeNewBranch', '=>', debug(result)]); + l.debug(() => ['writeNewBranch', '=>', { result }]); return result; } private _writeNewLeaf (entry: Buffer, entryAt: number, key: Buffer): Leaf { - l.debug(() => ['writeNewLeaf', debug({ entry, entryAt, key })]); + l.debug(() => ['writeNewLeaf', { entry, entryAt, key }]); const { keyAt, keyValue } = this.writeNewKey(key); @@ -488,7 +515,7 @@ export default class Combined implements DiskStore { private writeNewLeaf (entry: Buffer, entryAt: number, key: Buffer): Leaf { const result = this._writeNewLeaf(entry, entryAt, key); - l.debug(() => ['writeNewLeaf', '=>', debug(result)]); + l.debug(() => ['writeNewLeaf', '=>', { result }]); return result; } diff --git a/packages/db-diskdown/src/store/Scatter.ts b/packages/db/src/engines/FileTreeDb.ts similarity index 54% rename from packages/db-diskdown/src/store/Scatter.ts rename to packages/db/src/engines/FileTreeDb.ts index 8652b56e6..c518fd6c1 100644 --- a/packages/db-diskdown/src/store/Scatter.ts +++ b/packages/db/src/engines/FileTreeDb.ts @@ -1,12 +1,15 @@ -// Copyright 2017-2018 @polkadot/db-diskdown authors & contributors +// Copyright 2017-2018 @polkadot/db authors & contributors // This software may be modified and distributed under the terms // of the ISC license. See the LICENSE file for details. -import { DiskStore, ProgressValue } from '../types'; +import { BaseDb, ProgressCb } from '../types'; import fs from 'fs'; import mkdirp from 'mkdirp'; import logger from '@polkadot/util/logger'; +import bufferToU8a from '@polkadot/util/buffer/toU8a'; +import u8aToBuffer from '@polkadot/util/u8a/toBuffer'; +import u8aToHex from '@polkadot/util/u8a/toHex'; type FilePath = { directory: string, @@ -15,28 +18,21 @@ type FilePath = { const DIR_DEPTH = 1; -const l = logger('disk/scatter'); +const l = logger('db/tree'); -export default class Scatter implements DiskStore { +export default class FileTreeDb implements BaseDb { _location: string; constructor (location: string) { this._location = location; } - private _keyToString (key: Buffer): string { - return key - .toString('base64') - .replace(/\+/g, '-') - .replace(/\//g, '_'); - } - - private _getFilePath (key: Buffer): FilePath { + private _getFilePath (key: Uint8Array): FilePath { // NOTE We want to limit the number of entries in any specific directory. Split the // key into parts and use this to construct the path and the actual filename. We want // to limit the entries per directory, but at the same time minimize the number of // directories we need to create (when non-existent as well as the size overhead) - const parts = this._keyToString(key).match(/.{1,3}/g) || []; + const parts = u8aToHex(key).match(/.{1,6}/g) || []; const directory = `${this._location}/${parts.slice(0, DIR_DEPTH).join('/')}`; const file = `${directory}/${parts.slice(DIR_DEPTH).join('')}`; @@ -54,28 +50,40 @@ export default class Scatter implements DiskStore { // noop } - compact (progress: (value: ProgressValue) => void): void { - // noop + maintain (fn: ProgressCb): void { + fn({ + isCompleted: true, + keys: 0, + percent: 100 + }); } - delete (key: Buffer): void { - throw new Error('delete not implemented, only stubbed'); + del (key: Uint8Array): void { + l.debug(() => ['del', { key }]); + + const { file } = this._getFilePath(key); + + if (fs.existsSync(file)) { + fs.unlinkSync(file); + } } - get (key: Buffer): Buffer | undefined { - l.debug(() => ['get', key.toString('hex')]); + get (key: Uint8Array): Uint8Array | null { + l.debug(() => ['get', { key }]); const { file } = this._getFilePath(key); if (!fs.existsSync(file)) { - return; + return null; } - return fs.readFileSync(file); + return bufferToU8a( + fs.readFileSync(file) + ); } - set (key: Buffer, value: Buffer): void { - l.debug(() => ['set', key.toString('hex'), value]); + put (key: Uint8Array, value: Uint8Array): void { + l.debug(() => ['set', { key, value }]); const { directory, file } = this._getFilePath(key); @@ -83,6 +91,6 @@ export default class Scatter implements DiskStore { mkdirp.sync(directory); } - fs.writeFileSync(file, value); + fs.writeFileSync(file, u8aToBuffer(value)); } } diff --git a/packages/db/src/engines/LevelDb.spec.js b/packages/db/src/engines/LevelDb.spec.js new file mode 100644 index 000000000..2e372f6d8 --- /dev/null +++ b/packages/db/src/engines/LevelDb.spec.js @@ -0,0 +1,64 @@ +// Copyright 2017-2018 @polkadot/db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +import mkdirp from 'mkdirp'; +import os from 'os'; +import path from 'path'; +import toU8a from '@polkadot/util/u8a/fromString'; + +import LevelDb from './LevelDb'; + +describe('LevelDb', () => { + const location = path.join(os.tmpdir(), '--test-LevelDb'); + let db; + + beforeAll(() => { + mkdirp.sync(location); + + db = new LevelDb(location); + db.open(); + }); + + afterAll(() => { + db.close(); + rimraf.sync(location); + }); + + it('sets & retrieves', () => { + const key = toU8a('key0'); + const value = toU8a('value0'); + + db.put(key, value); + + expect(db.get(key)).toEqual(value); + }); + + it('retrieves null for non-existing', () => { + const key = toU8a('key1'); + + expect(db.get(key)).toEqual(null); + }); + + it('overrides values', () => { + const key = toU8a('key2'); + const value = toU8a('value1'); + const replace = toU8a('value2'); + + db.put(key, value); + + expect(db.get(key)).toEqual(value); + + db.put(key, replace); + + expect(db.get(key)).toEqual(replace); + }); + + it('deletes values', () => { + const key = toU8a('key2'); + + db.del(key); + + expect(db.get(key)).toEqual(null); + }); +}); diff --git a/packages/db/src/engines/LevelDb.ts b/packages/db/src/engines/LevelDb.ts new file mode 100644 index 000000000..5d04f557c --- /dev/null +++ b/packages/db/src/engines/LevelDb.ts @@ -0,0 +1,83 @@ +// Copyright 2017-2018 @polkadot/db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +import { BaseDb, ProgressCb } from '../types'; + +import leveldb from 'nosql-leveldb'; +import logger from '@polkadot/util/logger'; +import bufferToU8a from '@polkadot/util/buffer/toU8a'; +import u8aToBuffer from '@polkadot/util/u8a/toBuffer'; + +// import assert from '@polkadot/util/assert'; +// import u8aToHex from '@polkadot/util/u8a/toHex'; + +const l = logger('db/leveldb'); + +export default class Level implements BaseDb { + private db: LevelDb; + + constructor (location: string) { + this.db = leveldb(location); + } + + close (): void { + l.debug(() => ['close']); + + this.db.closeSync(); + } + + open (): void { + l.debug(() => ['open']); + + this.db.openSync(); + } + + maintain (fn: ProgressCb): void { + fn({ + isCompleted: true, + keys: 0, + percent: 100 + }); + } + + del (key: Uint8Array): void { + this.db.delSync( + this._serializeKey(key) + ); + } + + get (key: Uint8Array): Uint8Array | null { + try { + return this._deserializeValue( + this.db.getSync( + this._serializeKey(key), + { asBuffer: true } + ) + ); + } catch (error) { + return null; + } + } + + put (key: Uint8Array, value: Uint8Array): void { + this.db.putSync( + this._serializeKey(key), + this._serializeValue(value) + ); + } + + private _deserializeValue (value: Buffer): Uint8Array | null { + return value + ? bufferToU8a(value) + : null; + } + + private _serializeKey (key: Uint8Array): Buffer { + return u8aToBuffer(key); + } + + private _serializeValue (value: Uint8Array): Buffer { + return u8aToBuffer(value); + } +} diff --git a/packages/db/src/engines/LruDb.spec.js b/packages/db/src/engines/LruDb.spec.js new file mode 100644 index 000000000..0efdd0427 --- /dev/null +++ b/packages/db/src/engines/LruDb.spec.js @@ -0,0 +1,63 @@ +// Copyright 2017-2018 @polkadot/db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +import toU8a from '@polkadot/util/u8a/fromString'; + +import LruDb from './LruDb'; +import MemoryDb from './MemoryDb'; + +describe('LruDb', () => { + const memory = new MemoryDb(); + const db = new LruDb(memory); + + it('retrieves an item from the backing when not available (caching it)', () => { + const key = toU8a('test1'); + const value = toU8a('value1'); + + memory.put(key, value); + + expect(db.get(key)).toEqual(value); + expect(db._getLru(key)).toEqual({ value }); + }); + + it('replaces an item (caching and backing)', () => { + const key = toU8a('test1'); + const value = toU8a('test'); + + db.put(key, value); + + expect(db.get(key)).toEqual(value); + expect(memory.get(key)).toEqual(value); + expect(db._getLru(key)).toEqual({ value }); + }); + + it('retrieves item from LRU when available', () => { + const key = toU8a('test1'); + const value = toU8a('test'); + + memory.del(key); + + expect(db.get(key)).toEqual(value); + }); + + it('puts item both in LRU and backing', () => { + const key = toU8a('test0'); + const value = toU8a('value0'); + + db.put(key, value); + + expect(memory.get(key)).toEqual(value); + expect(db._getLru(key)).toEqual({ value }); + }); + + it('deletes an item from both backing and db', () => { + const key = toU8a('test0'); + + db.del(key); + + expect(memory.get(key)).toEqual(null); + expect(db._getLru(key)).toEqual({ value: null }); + expect(db.get(key)).toEqual(null); + }); +}); diff --git a/packages/db/src/engines/LruDb.ts b/packages/db/src/engines/LruDb.ts new file mode 100644 index 000000000..a5cdde2f3 --- /dev/null +++ b/packages/db/src/engines/LruDb.ts @@ -0,0 +1,78 @@ +// Copyright 2017-2018 @polkadot/db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +import { BaseDb, ProgressCb } from '../types'; + +import { LRUMap } from 'lru_map'; +import logger from '@polkadot/util/logger'; + +type CachedValue = { + value: Uint8Array | null +}; + +const DEFAULT_ITEM_COUNT = 4096; + +const l = logger('db/lru'); + +export default class LruDb implements BaseDb { + private backing: BaseDb; + private lru: LRUMap; + + constructor (backing: BaseDb, itemCount: number = DEFAULT_ITEM_COUNT) { + this.backing = backing; + this.lru = new LRUMap(itemCount); + } + + close (): void { + l.debug(() => ['close']); + + this.lru.clear(); + this.backing.close(); + } + + open (): void { + l.debug(() => ['open']); + + this.lru.clear(); + this.backing.open(); + } + + maintain (fn: ProgressCb): void { + this.backing.maintain(fn); + } + + del (key: Uint8Array): void { + const keyStr = key.toString(); + + this.backing.del(key); + this.lru.set(keyStr, { value: null }); + } + + get (key: Uint8Array): Uint8Array | null { + const keyStr = key.toString(); + const cached = this.lru.get(keyStr); + + if (cached) { + return cached.value; + } + + const value = this.backing.get(key); + + this.lru.set(keyStr, { value }); + + return value; + } + + // Convenience methods used in tests + _getLru (key: Uint8Array): CachedValue | undefined { + return this.lru.get(key.toString()); + } + + put (key: Uint8Array, value: Uint8Array): void { + const keyStr = key.toString(); + + this.backing.put(key, value); + this.lru.set(keyStr, { value }); + } +} diff --git a/packages/db/src/engines/MemoryDb.ts b/packages/db/src/engines/MemoryDb.ts new file mode 100644 index 000000000..67dbca767 --- /dev/null +++ b/packages/db/src/engines/MemoryDb.ts @@ -0,0 +1,54 @@ +// Copyright 2017-2018 @polkadot/db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +import { BaseDb, ProgressCb } from '../types'; + +// import logger from '@polkadot/util/logger'; +// import u8aToHex from '@polkadot/util/u8a/toHex'; + +type Storage = { + [index: string]: Uint8Array +}; + +// const l = logger('db/memory'); + +export default class MemoryDb implements BaseDb { + private storage: Storage; + + constructor () { + this.storage = {}; + } + + close (): void { + // noop + } + + open (): void { + // noop + } + + maintain (fn: ProgressCb): void { + fn({ + isCompleted: true, + keys: Object.keys(this.storage).length, + percent: 100 + }); + } + + del (key: Uint8Array): void { + delete this.storage[key.toString()]; + } + + get (key: Uint8Array): Uint8Array | null { + // l.debug(() => ['get', u8aToHex(key)]); + + return this.storage[key.toString()] || null; + } + + put (key: Uint8Array, value: Uint8Array): void { + // l.debug(() => ['put', u8aToHex(key), u8aToHex(value)]); + + this.storage[key.toString()] = value; + } +} diff --git a/packages/db/src/engines/RocksDb.ts b/packages/db/src/engines/RocksDb.ts new file mode 100644 index 000000000..9eb69cbf5 --- /dev/null +++ b/packages/db/src/engines/RocksDb.ts @@ -0,0 +1,83 @@ +// Copyright 2017-2018 @polkadot/db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +import { BaseDb, ProgressCb } from '../types'; + +import rocksdb from 'rocksdb-node'; +import assert from '@polkadot/util/assert'; +import logger from '@polkadot/util/logger'; +import isUndefined from '@polkadot/util/is/undefined'; +import bufferToU8a from '@polkadot/util/buffer/toU8a'; +import u8aToBuffer from '@polkadot/util/u8a/toBuffer'; + +// import assert from '@polkadot/util/assert'; +// import u8aToHex from '@polkadot/util/u8a/toHex'; + +const l = logger('db/rocksdb'); + +export default class Rocks implements BaseDb { + // @ts-ignore Guard with assertOpen + private db: RocksDb; + private location: string; + + constructor (location: string) { + this.location = location; + } + + private assertOpen () { + assert(!isUndefined(this.db), 'Database needs to be open'); + } + + close (): void { + l.debug(() => ['close']); + + this.assertOpen(); + this.db.close(); + + // @ts-ignore caught with guard + this.db = undefined; + } + + open (): void { + l.debug(() => ['open']); + + this.db = rocksdb.open( + { create_if_missing: true }, + this.location + ); + } + + maintain (fn: ProgressCb): void { + fn({ + isCompleted: true, + keys: 0, + percent: 100 + }); + } + + del (key: Uint8Array): void { + this.assertOpen(); + this.db.del(u8aToBuffer(key)); + } + + get (key: Uint8Array): Uint8Array | null { + this.assertOpen(); + + return bufferToU8a( + this.db.get( + { buffer: true }, + u8aToBuffer(key) + ) + ); + } + + put (key: Uint8Array, value: Uint8Array): void { + this.assertOpen(); + + this.db.put( + u8aToBuffer(key), + u8aToBuffer(value) + ); + } +} diff --git a/packages/db/src/engines/TransactionDb.spec.js b/packages/db/src/engines/TransactionDb.spec.js new file mode 100644 index 000000000..f03500c0c --- /dev/null +++ b/packages/db/src/engines/TransactionDb.spec.js @@ -0,0 +1,93 @@ +// Copyright 2017-2018 @polkadot/db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +import toU8a from '@polkadot/util/u8a/fromString'; + +import MemoryDb from './MemoryDb'; +import TransactionDb from './TransactionDb'; + +describe('TransactionDb', () => { + const memory = new MemoryDb(); + const db = new TransactionDb(memory); + + it('passed through values immediately when not in transaction', () => { + const key = toU8a('test0'); + const value = toU8a('value0'); + + db.put(key, value); + + expect(memory.get(key)).toEqual(value); + }); + + it('commits when transaction passes (result = true, put)', () => { + const key = toU8a('test1'); + const value = toU8a('value1'); + + expect( + db.transaction(() => { + db.put(key, value); + + return true; + }) + ).toEqual(true); + expect(memory.get(key)).toEqual(value); + }); + + it('commits when transaction passes (result = true, del)', () => { + const key = toU8a('test0'); + + expect( + db.transaction(() => { + db.del(key); + + return true; + }) + ).toEqual(true); + expect(memory.get(key)).toEqual(null); + }); + + it('does not commit when transaction fails (result = false)', () => { + const key = toU8a('test2'); + const value = toU8a('value2'); + + expect( + db.transaction(() => { + db.put(key, value); + + return false; + }) + ).toEqual(false); + expect(memory.get(key)).toEqual(null); + }); + + it('does not commit when transaction throws', () => { + const key = toU8a('test2'); + const value = toU8a('value2'); + + expect( + () => { + db.transaction(() => { + db.put(key, value); + + throw new Error('test'); + }) + } + ).toThrow(/test/); + expect(memory.get(key)).toEqual(null); + }); + + it('inside the transaction, the value is set', () => { + const key = toU8a('test2'); + const value = toU8a('value2'); + + db.transaction(() => { + db.put(key, value); + + expect(db.get(key)).toEqual(value); + + return false; + }); + expect(memory.get(key)).toEqual(null); + }); +}); diff --git a/packages/db/src/engines/TransactionDb.ts b/packages/db/src/engines/TransactionDb.ts new file mode 100644 index 000000000..552a73924 --- /dev/null +++ b/packages/db/src/engines/TransactionDb.ts @@ -0,0 +1,142 @@ +// Copyright 2017-2018 @polkadot/db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +import { BaseDb, TxDb, ProgressCb } from '../types'; + +import assert from '@polkadot/util/assert'; +import logger from '@polkadot/util/logger'; +import isNull from '@polkadot/util/is/null'; + +type Overlay = { + [index: string]: { + key: Uint8Array, + value: Uint8Array | null + } +}; + +const l = logger('db/transact'); + +export default class TransactionDb implements TxDb { + private backing: BaseDb; + private txOverlay: Overlay; + private txStarted: boolean; + + constructor (backing: BaseDb) { + this.backing = backing; + this.txOverlay = {}; + this.txStarted = false; + } + + transaction (fn: () => boolean): boolean { + l.debug(() => ['transaction']); + + try { + this.createTx(); + + const result = fn(); + + if (result) { + this.commitTx(); + } else { + this.revertTx(); + } + + return result; + } catch (error) { + this.revertTx(); + + throw error; + } + } + + close (): void { + this.backing.open(); + } + + open (): void { + this.backing.close(); + } + + maintain (fn: ProgressCb): void { + assert(this.txStarted === false, 'Cannot maintain inside an open transaction'); + + this.backing.maintain(fn); + } + + del (key: Uint8Array): void { + if (this.txStarted) { + this.txOverlay[key.toString()] = { + key, + value: null + }; + return; + } + + this.backing.del(key); + } + + get (key: Uint8Array): Uint8Array | null { + // l.debug(() => ['get', u8aToHex(key)]); + + if (this.txStarted) { + const value = this.txOverlay[key.toString()]; + + if (value) { + return value.value; + } + } + + return this.backing.get(key); + } + + put (key: Uint8Array, value: Uint8Array): void { + // l.debug(() => ['put', u8aToHex(key), u8aToHex(value)]); + + if (this.txStarted) { + this.txOverlay[key.toString()] = { + key, + value + }; + + return; + } + + return this.backing.put(key, value); + } + + private createTx (): void { + l.debug(() => ['createTx']); + + assert(!this.txStarted, 'Cannot create a transaction when one is already active'); + + this.txOverlay = {}; + this.txStarted = true; + } + + private commitTx (): void { + l.debug(() => ['commitTx', Object.keys(this.txOverlay).length, 'keys']); + + assert(this.txStarted, 'Cannot commit when not in transaction'); + + Object.values(this.txOverlay).forEach(({ key, value }) => { + if (isNull(value)) { + this.backing.del(key); + } else { + this.backing.put(key, value); + } + }); + + this.txOverlay = {}; + this.txStarted = false; + } + + private revertTx (): void { + l.debug(() => ['revertTx']); + + assert(this.txStarted, 'Cannot revert when not in transaction'); + + this.txOverlay = {}; + this.txStarted = false; + } +} diff --git a/packages/db/src/nosql-leveldb.d.ts b/packages/db/src/nosql-leveldb.d.ts new file mode 100644 index 000000000..62e9a7a61 --- /dev/null +++ b/packages/db/src/nosql-leveldb.d.ts @@ -0,0 +1,34 @@ +// Copyright 2017-2018 @polkadot/db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +type LevelDbOptions$Open = { + createIfMissing?: boolean, + errorIfExists?: boolean, + compression?: boolean, + cacheSize?: boolean +}; + +type LevelDbOptions$Get ={ + asBuffer?: boolean, + fillCache?: boolean +}; + +type LevelDbOptions$Put ={ + sync?: boolean +}; + +declare interface LevelDb { + openSync (options?: LevelDbOptions$Open): void; + closeSync (): void; + + delSync (key: Buffer): void; + getSync (key: Buffer, options?: LevelDbOptions$Get): Buffer; + putSync (key: Buffer, value: Buffer, options?: LevelDbOptions$Put): void; +} + +declare module 'nosql-leveldb' { + const leveldb: (location: string) => LevelDb; + + export default leveldb; +} diff --git a/packages/db/src/rocksdb-node.d.ts b/packages/db/src/rocksdb-node.d.ts new file mode 100644 index 000000000..758a3d0d1 --- /dev/null +++ b/packages/db/src/rocksdb-node.d.ts @@ -0,0 +1,27 @@ +// Copyright 2017-2018 @polkadot/db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +type RocksDbOptions$Open = { + create_if_missing?: boolean +}; + +type RocksDbOptions$Get ={ + buffer: boolean +}; + +declare interface RocksDb { + close (): void; + + del (key: Buffer): void; + get (options: RocksDbOptions$Get, key: Buffer): Buffer; + put (key: Buffer, value: Buffer): void; +} + +declare module 'rocksdb-node' { + const rocksdb: { + open (options: RocksDbOptions$Open, location: string): RocksDb; + }; + + export default rocksdb; +} diff --git a/packages/db/src/snappy.d.ts b/packages/db/src/snappy.d.ts new file mode 100644 index 000000000..8284aabbc --- /dev/null +++ b/packages/db/src/snappy.d.ts @@ -0,0 +1,14 @@ +// Copyright 2017-2018 @polkadot/db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +declare interface Snappy { + compressSync (input: Buffer): Buffer; + uncompressSync (input: Buffer): Buffer; +} + +declare module 'snappy' { + const snappy: Snappy; + + export default snappy; +} diff --git a/packages/db/src/types.d.ts b/packages/db/src/types.d.ts new file mode 100644 index 000000000..1847e3f00 --- /dev/null +++ b/packages/db/src/types.d.ts @@ -0,0 +1,26 @@ +// Copyright 2017-2018 @polkadot/db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +export type ProgressValue = { + isCompleted?: boolean, + keys: number, + percent: number +}; + +export type ProgressCb = (progress: ProgressValue) => void; + +export interface BaseDb { + close (): void; + open (): void; + + maintain (fn: ProgressCb): void; + + del (key: Uint8Array): void; + get (key: Uint8Array): Uint8Array | null; + put (key: Uint8Array, value: Uint8Array): void; +} + +export interface TxDb extends BaseDb { + transaction (fn: () => boolean): boolean; +} diff --git a/packages/trie-db-old/LICENSE b/packages/trie-db-old/LICENSE new file mode 100644 index 000000000..a612ad981 --- /dev/null +++ b/packages/trie-db-old/LICENSE @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/packages/trie-db-old/README.md b/packages/trie-db-old/README.md new file mode 100644 index 000000000..a748624ea --- /dev/null +++ b/packages/trie-db-old/README.md @@ -0,0 +1,30 @@ +[![polkadotjs](https://img.shields.io/badge/polkadot-js-orange.svg?style=flat-square)](https://polkadot.js.org) +![mpl-2](https://img.shields.io/badge/license-MPL-lightgrey.svg?style=flat-square) +[![style](https://img.shields.io/badge/code%20style-semistandard-lightgrey.svg?style=flat-square)](https://github.com/Flet/semistandard) +[![npm](https://img.shields.io/npm/v/@polkadot/trie-db.svg?style=flat-square)](https://www.npmjs.com/package/@polkadot/trie-db) +[![travis](https://img.shields.io/travis/polkadot-js/trie.svg?style=flat-square)](https://travis-ci.org/polkadot-js/trie) +[![maintainability](https://img.shields.io/codeclimate/maintainability/polkadot-js/trie.svg?style=flat-square)](https://codeclimate.com/github/polkadot-js/trie/maintainability) +[![coverage](https://img.shields.io/coveralls/polkadot-js/trie.svg?style=flat-square)](https://coveralls.io/github/polkadot-js/trie?branch=master) +[![dependency](https://david-dm.org/polkadot-js/trie.svg?style=flat-square&path=packages/trie)](https://david-dm.org/polkadot-js/trie?path=packages/trie-db) +[![devDependency](https://david-dm.org/polkadot-js/trie/dev-status.svg?style=flat-square&path=packages/trie-db)](https://david-dm.org/polkadot-js/trie?path=packages/trie-db#info=devDependencies) + +# @polkadot/trie-db + +An Implementation of the [Ethereum Merkle Patricia Tree](https://github.com/ethereum/wiki/wiki/Patricia-Tree) that uses Blake2 for hashing. + +# Adapted from Ethereum + +The has been adapted from the [original EthereumJs implementation](https://github.com/ethereumjs/merkle-patricia-tree) with the following changes - + +- Introduce [TypeScript](https://www.typescriptlang.org/) into the mix +- ES6-ify as applicable (Classes, `self` -> `this`, `function() {}` -> `() => {}`) +- Moves to a Promise-based interface (as applicable) +- Support only Uint8Array key/values (Buffer/String in the original) +- Use utilities from [@polkadot/util](https://github.com/polkadot-js/common) +- Supports blake2 hashing (sha3 in the original), aligning with the Polkadot tech stack) +- Updated the build steps with sources moved to [src/](src/) +- Tests updated to use [Jest](https://facebook.github.io/jest/) +- Upgrade interfaces to latest [levelup](https://github.com/Level/levelup) +- Removed the secure interface (not applicable for Substrate chains) + +... well at least the above is the _idea_ behind the fork ... diff --git a/packages/trie-db-old/package.json b/packages/trie-db-old/package.json new file mode 100644 index 000000000..4e07b5f46 --- /dev/null +++ b/packages/trie-db-old/package.json @@ -0,0 +1,45 @@ +{ + "name": "@polkadot/trie-db-old", + "version": "0.29.0", + "description": "This is an implementation of the modified merkle patricia tree as speficed in Ethereum's yellow paper, adpated for Polkadot.", + "main": "index.js", + "maintainers": [ + "Jaco Greeff " + ], + "contributors": [ + "mjbecze ", + "Aaron Kumavis (https://github.com/kumavis)" + ], + "repository": { + "type": "git", + "url": "https://github.com/polkadot-js/common.git" + }, + "bugs": { + "url": "https://github.com/polkadot-js/common/issues" + }, + "keywords": [ + "merkle", + "radix", + "trie", + "ethereum" + ], + "license": "MPL-2.0", + "dependencies": { + "@babel/runtime": "^7.0.0-rc.1", + "@polkadot/trie-hash": "^0.29.0", + "@polkadot/util": "^0.29.0", + "@polkadot/util-crypto": "^0.29.0", + "@polkadot/util-rlp": "^0.29.0", + "@types/async": "^2.0.49", + "@types/semaphore": "^1.1.0", + "async": "^2.6.1", + "encoding-down": "^5.0.3", + "level-ws": "0.1.0", + "levelup": "^3.1.1", + "memdown": "^3.0.0", + "semaphore": ">=1.0.1" + }, + "devDependencies": { + "ethereumjs-testing": "1.0.4" + } +} diff --git a/packages/trie-db/src/BaseTrie.ts b/packages/trie-db-old/src/BaseTrie.ts similarity index 97% rename from packages/trie-db/src/BaseTrie.ts rename to packages/trie-db-old/src/BaseTrie.ts index 24ee4a4d3..751afde46 100644 --- a/packages/trie-db/src/BaseTrie.ts +++ b/packages/trie-db-old/src/BaseTrie.ts @@ -9,11 +9,11 @@ import async from 'async'; // @ts-ignore FIXME, we need to properly check the full file import levelup from 'levelup'; import memdown from 'memdown'; -// import logger from '@polkadot/util/logger'; +import logger from '@polkadot/util/logger'; import assert from '@polkadot/util/assert'; import isU8a from '@polkadot/util/is/u8a'; import promisify from '@polkadot/util/promisify'; -// import u8aToHex from '@polkadot/util/u8a/toHex'; +import u8aToHex from '@polkadot/util/u8a/toHex'; import keccakAsU8a from '@polkadot/util-crypto/keccak/asU8a'; import decodeRlp from '@polkadot/util-rlp/decode'; @@ -28,7 +28,7 @@ import semaphore from './util/semaphore'; type RawPutFn = (key: Uint8Array, value: Uint8Array) => Promise; -// const l = logger('trie/base'); +const l = logger('trie/base'); export default class BaseTrie { // @ts-ignore we are assigning via this.root in constructor @@ -78,7 +78,7 @@ export default class BaseTrie { * @param {Function} cb A callback `Function` which is given the arguments `err` - for errors that may have occured and `value` - the found value in a `Buffer` or if no value was found `null` */ async get (key: Uint8Array): Promise { - // l.debug(() => ['get', 'key ->', u8aToHex(key)]); + l.debug(() => ['get', 'key ->', u8aToHex(key)]); const result = await this.findPath(key); @@ -99,7 +99,7 @@ export default class BaseTrie { * @returns {Function} Promise */ async put (key: Uint8Array, value: Uint8Array) { - // l.debug(() => ['put', 'key ->', u8aToHex(key), 'value ->', u8aToHex(value, 256)]); + l.debug(() => ['put', 'key ->', u8aToHex(key), 'value ->', u8aToHex(value, 256)]); if (!value || value.toString() === '') { return this.del(key); @@ -113,6 +113,8 @@ export default class BaseTrie { // first try to find the give key or its nearst node const result = await this.findPath(key); + l.debug(() => ['put: path', result]); + return this._updateNode(key, value, result && result.remainder, result && result.stack); }); } @@ -124,7 +126,7 @@ export default class BaseTrie { * @param {Function} callback the callback `Function` */ del (key: Uint8Array) { - // l.debug(() => ['del', 'key ->', u8aToHex(key)]); + l.debug(() => ['del', 'key ->', u8aToHex(key)]); return this.semaphore(async () => { const result = await this.findPath(key); @@ -572,7 +574,7 @@ export default class BaseTrie { */ // @ts-ignore FIXME, we need to properly check the full file _saveStack (key, stack, opStack: any[]) { - let lastRoot; + let lastRoot: any; // update nodes while (stack.length) { @@ -595,12 +597,16 @@ export default class BaseTrie { // @ts-ignore FIXME, we need to properly check the full file lastRoot = this._formatNode(node, stack.length === 0, opStack); + + l.debug(() => ['_saveStack', { lastRoot, node }]); } if (lastRoot) { this.root = lastRoot; } + l.debug(() => ['_saveStack', opStack]); + return this._batchNodes(opStack); } @@ -756,6 +762,8 @@ export default class BaseTrie { if (rlpNode.length >= 32 || topLevel) { const hashRoot = node.hash(); + l.debug(() => ['_formatNode', { hashRoot }]); + // FIXME Actually 2 things wrong here - this is on checkpoint, so one higher-level class... // @ts-ignore FIXME, we need to properly check the full file if (remove && this.isCheckpoint) { @@ -774,6 +782,8 @@ export default class BaseTrie { return hashRoot; } + l.debug(() => ['_formatNode', { raw: node.raw }]); + return node.raw; } diff --git a/packages/trie-db/src/CheckpointTrie.ts b/packages/trie-db-old/src/CheckpointTrie.ts similarity index 100% rename from packages/trie-db/src/CheckpointTrie.ts rename to packages/trie-db-old/src/CheckpointTrie.ts diff --git a/packages/trie-db/src/Node.ts b/packages/trie-db-old/src/Node.ts similarity index 89% rename from packages/trie-db/src/Node.ts rename to packages/trie-db-old/src/Node.ts index 9c11131f5..565650fc8 100644 --- a/packages/trie-db/src/Node.ts +++ b/packages/trie-db-old/src/Node.ts @@ -8,9 +8,9 @@ import { DecodedRlp, Trie$Node$Type, HashFn } from './types'; import isU8a from '@polkadot/util/is/u8a'; import u8aToHex from '@polkadot/util/u8a/toHex'; import encodeRlp from '@polkadot/util-rlp/encode'; +import nibblesToU8a from '@polkadot/trie-hash/util/fromNibbles'; import nibblesFromU8a from './nibbles/fromU8a'; -import nibblesToU8a from './nibbles/toU8a'; function addHexPrefix (key: Array, isTerminator: boolean): Array { // odd @@ -125,20 +125,20 @@ export default class TrieNode { } toString (): string { + const format = (value: any): string => { + if (isU8a(value)) { + return u8aToHex(value); + } else if (Array.isArray(value)) { + return `[${value.map((v) => format(v))}]`; + } else if (value) { + return 'object'; + } + + return 'empty'; + }; + return this.raw.reduce((out, el, index) => { - if (index !== 0) { - out += ', '; - } - - if (isU8a(el)) { - out += u8aToHex(el); - } else if (el) { - out += 'object'; - } else { - out += 'empty'; - } - - return out; + return `${out}${index ? ', ' : ''}${format(el)}`; }, `${this.type}: [`) + ']'; } diff --git a/packages/trie-db-old/src/constants.ts b/packages/trie-db-old/src/constants.ts new file mode 100644 index 000000000..3741cb114 --- /dev/null +++ b/packages/trie-db-old/src/constants.ts @@ -0,0 +1,13 @@ +// Copyright 2017-2018 @polkadot/trie-db authors & contributors +// This software may be modified and distributed under the terms +// of the MPL-2.0 license. See the LICENSE file for details. + +import trieRoot from '@polkadot/trie-hash/root'; + +const EMPTY_ROOT_U8A = trieRoot([]); +const EMPTY_ROOT_STR = EMPTY_ROOT_U8A.toString(); + +export { + EMPTY_ROOT_U8A, + EMPTY_ROOT_STR +}; diff --git a/packages/trie-db/src/encoder/decode.ts b/packages/trie-db-old/src/encoder/decode.ts similarity index 100% rename from packages/trie-db/src/encoder/decode.ts rename to packages/trie-db-old/src/encoder/decode.ts diff --git a/packages/trie-db/src/encoder/encode.ts b/packages/trie-db-old/src/encoder/encode.ts similarity index 100% rename from packages/trie-db/src/encoder/encode.ts rename to packages/trie-db-old/src/encoder/encode.ts diff --git a/packages/trie-db/src/encoder/index.ts b/packages/trie-db-old/src/encoder/index.ts similarity index 100% rename from packages/trie-db/src/encoder/index.ts rename to packages/trie-db-old/src/encoder/index.ts diff --git a/packages/trie-db-old/src/index.ts b/packages/trie-db-old/src/index.ts new file mode 100644 index 000000000..ba2352fca --- /dev/null +++ b/packages/trie-db-old/src/index.ts @@ -0,0 +1,23 @@ +// Copyright 2015-2018 authors & contributors of https://github.com/ethereumjs/merkle-patricia-tree +// Copyright 2017-2018 @polkadot/trie-db authors & contributors +// This software may be modified and distributed under the terms +// of the MPL-2.0 license. See the LICENSE file for details. + +import { HashFn } from './types'; + +// import logger from '@polkadot/util/logger'; +import CheckpointTrie from './CheckpointTrie'; +import { prove, verifyProof } from './proof'; + +// const l = logger('trie'); + +export default class Trie extends CheckpointTrie { + static prove = prove; + static verifyProof = verifyProof; + + constructor (db: any, root?: Uint8Array, hashing?: HashFn) { + super(db, root, hashing); + + // l.debug('Created Trie'); + } +} diff --git a/packages/trie-db/src/nibbles/fromU8a.ts b/packages/trie-db-old/src/nibbles/fromU8a.ts similarity index 100% rename from packages/trie-db/src/nibbles/fromU8a.ts rename to packages/trie-db-old/src/nibbles/fromU8a.ts diff --git a/packages/trie-db/src/nibbles/index.ts b/packages/trie-db-old/src/nibbles/index.ts similarity index 83% rename from packages/trie-db/src/nibbles/index.ts rename to packages/trie-db-old/src/nibbles/index.ts index 19df12f41..ab6d32c6f 100644 --- a/packages/trie-db/src/nibbles/index.ts +++ b/packages/trie-db-old/src/nibbles/index.ts @@ -5,11 +5,9 @@ import nibblesFromU8a from './fromU8a'; import nibblesIsEqual from './isEqual'; import nibblesMatchingLength from './matchingLength'; -import nibblesToU8a from './toU8a'; export { nibblesFromU8a, nibblesIsEqual, - nibblesMatchingLength, - nibblesToU8a + nibblesMatchingLength }; diff --git a/packages/trie-db/src/nibbles/isEqual.ts b/packages/trie-db-old/src/nibbles/isEqual.ts similarity index 100% rename from packages/trie-db/src/nibbles/isEqual.ts rename to packages/trie-db-old/src/nibbles/isEqual.ts diff --git a/packages/trie-db/src/nibbles/matchingLength.ts b/packages/trie-db-old/src/nibbles/matchingLength.ts similarity index 100% rename from packages/trie-db/src/nibbles/matchingLength.ts rename to packages/trie-db-old/src/nibbles/matchingLength.ts diff --git a/packages/trie-db/src/nodeFactory.ts b/packages/trie-db-old/src/nodeFactory.ts similarity index 96% rename from packages/trie-db/src/nodeFactory.ts rename to packages/trie-db-old/src/nodeFactory.ts index b2f2c4b9c..cea2e2bc2 100644 --- a/packages/trie-db/src/nodeFactory.ts +++ b/packages/trie-db-old/src/nodeFactory.ts @@ -7,9 +7,8 @@ import { Logger } from '@polkadot/util/types'; import { HashFn, DecodedRlp, NodeFactory, Trie$Node, Trie$Node$Type } from './types'; import logger from '@polkadot/util/logger'; -// import u8aToHex from '@polkadot/util/u8a/toHex'; +import nibblesFromU8a from '@polkadot/trie-hash/util/asNibbles'; -import nibblesFromU8a from './nibbles/fromU8a'; // import rlpToString from './util/rlpToString'; import TrieNode from './Node'; diff --git a/packages/trie-db/src/proof/index.ts b/packages/trie-db-old/src/proof/index.ts similarity index 100% rename from packages/trie-db/src/proof/index.ts rename to packages/trie-db-old/src/proof/index.ts diff --git a/packages/trie-db/src/proof/prove.ts b/packages/trie-db-old/src/proof/prove.ts similarity index 100% rename from packages/trie-db/src/proof/prove.ts rename to packages/trie-db-old/src/proof/prove.ts diff --git a/packages/trie-db/src/proof/verifyProof.ts b/packages/trie-db-old/src/proof/verifyProof.ts similarity index 100% rename from packages/trie-db/src/proof/verifyProof.ts rename to packages/trie-db-old/src/proof/verifyProof.ts diff --git a/packages/trie-db/src/streams/ScratchRead.ts b/packages/trie-db-old/src/streams/ScratchRead.ts similarity index 100% rename from packages/trie-db/src/streams/ScratchRead.ts rename to packages/trie-db-old/src/streams/ScratchRead.ts diff --git a/packages/trie-db/src/streams/TrieRead.ts b/packages/trie-db-old/src/streams/TrieRead.ts similarity index 96% rename from packages/trie-db/src/streams/TrieRead.ts rename to packages/trie-db-old/src/streams/TrieRead.ts index 30b010d02..dec858b5b 100644 --- a/packages/trie-db/src/streams/TrieRead.ts +++ b/packages/trie-db-old/src/streams/TrieRead.ts @@ -7,11 +7,10 @@ import { Trie$Base } from '../types'; // @ts-ignore FIXME, we need to properly check the full file import { Readable } from 'readable-stream'; +import nibblesToU8a from '@polkadot/trie-hash/util/fromNibbles'; // import logger from '@polkadot/util/logger'; // import u8aToHex from '@polkadot/util/u8a/toHex'; -import { nibblesToU8a } from '../nibbles'; - // const l = logger('trie/read'); export default class TrieReadStream extends Readable { diff --git a/packages/trie-db/src/types.d.ts b/packages/trie-db-old/src/types.d.ts similarity index 100% rename from packages/trie-db/src/types.d.ts rename to packages/trie-db-old/src/types.d.ts diff --git a/packages/trie-db/src/util/isRawNode.ts b/packages/trie-db-old/src/util/isRawNode.ts similarity index 100% rename from packages/trie-db/src/util/isRawNode.ts rename to packages/trie-db-old/src/util/isRawNode.ts diff --git a/packages/trie-db/src/util/rlpToString.ts b/packages/trie-db-old/src/util/rlpToString.ts similarity index 100% rename from packages/trie-db/src/util/rlpToString.ts rename to packages/trie-db-old/src/util/rlpToString.ts diff --git a/packages/trie-db/src/util/semaphore.ts b/packages/trie-db-old/src/util/semaphore.ts similarity index 100% rename from packages/trie-db/src/util/semaphore.ts rename to packages/trie-db-old/src/util/semaphore.ts diff --git a/packages/trie-db/src/util/taskExecutor.ts b/packages/trie-db-old/src/util/taskExecutor.ts similarity index 100% rename from packages/trie-db/src/util/taskExecutor.ts rename to packages/trie-db-old/src/util/taskExecutor.ts diff --git a/packages/trie-db/test/checkpoints.spec.js b/packages/trie-db-old/test/checkpoints.spec.js similarity index 97% rename from packages/trie-db/test/checkpoints.spec.js rename to packages/trie-db-old/test/checkpoints.spec.js index b274d545c..19ad46cdb 100644 --- a/packages/trie-db/test/checkpoints.spec.js +++ b/packages/trie-db-old/test/checkpoints.spec.js @@ -13,7 +13,7 @@ import Trie from '../src/index'; // mkdirp(DISKPATH); -describe('testing checkpoints', () => { +describe.skip('testing checkpoints', () => { let trie, preRoot, postRoot; it('sets up the trie', async () => { diff --git a/packages/trie-db/test/ethereum.spec.js b/packages/trie-db-old/test/ethereum.spec.js similarity index 95% rename from packages/trie-db/test/ethereum.spec.js rename to packages/trie-db-old/test/ethereum.spec.js index 61eca5fac..6cf7f955a 100644 --- a/packages/trie-db/test/ethereum.spec.js +++ b/packages/trie-db-old/test/ethereum.spec.js @@ -10,7 +10,7 @@ import encodeRlp from '@polkadot/util-rlp/encode'; import Trie from '../src/index'; -describe('it should create the genesis state root from ethereum', () => { +describe.skip('it should create the genesis state root from ethereum', () => { const trie4 = new Trie(); const g = toU8a('0x8a40bfaa73256b60764c1bf40675a99083efb075'); const j = toU8a('0xe6716f9544a56c530d868e4bfbacb172315bdead'); diff --git a/packages/trie-db/test/index.spec.js b/packages/trie-db-old/test/index.spec.js similarity index 75% rename from packages/trie-db/test/index.spec.js rename to packages/trie-db-old/test/index.spec.js index 08637c069..d58dfef98 100644 --- a/packages/trie-db/test/index.spec.js +++ b/packages/trie-db-old/test/index.spec.js @@ -8,7 +8,7 @@ import toU8a from '@polkadot/util/u8a/toU8a'; import Trie from '../src/index'; -describe('simple save and retrive', () => { +describe.skip('simple save and retrive', () => { it('should not crash if given a non-existant root', async () => { const root = toU8a('0x3f4399b08efe68945c1cf90ffe85bbe3ce978959da753f9e649f034015b8817d'); const trie = new Trie(null, root); @@ -19,8 +19,26 @@ describe('simple save and retrive', () => { var trie = new Trie(); + it('starts with a valid root', () => { + expect( + trie.root + ).toEqual( + new Uint8Array([ + 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33 + ]) + ) + }); + it('save a value', async () => { await trie.put(toU8a('test'), toU8a('one')); + + expect( + trie.root + ).toEqual( + new Uint8Array([ + 43, 119, 232, 84, 123, 197, 94, 42, 149, 34, 124, 147, 159, 159, 157, 103, 149, 45, 225, 233, 112, 160, 23, 224, 145, 11, 229, 16, 176, 144, 175, 243 + ]) + ); }); it('should get a value', async () => { @@ -35,6 +53,7 @@ describe('simple save and retrive', () => { const value = await trie.get(toU8a('test')); expect(value).toEqual(toU8a('two')); + expect(trie.root).toEqual(new Uint8Array([127, 237, 74, 184, 190, 46, 103, 129, 65, 141, 13, 96, 61, 232, 146, 13, 56, 32, 59, 29, 246, 58, 175, 140, 72, 182, 196, 103, 230, 245, 229, 148])); }); it('should delete a value', async () => { @@ -43,10 +62,13 @@ describe('simple save and retrive', () => { const value = await trie.get(toU8a('test')); expect(value).toEqual(null); + expect(trie.root).toEqual(new Uint8Array([86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33])); }); it('should recreate a value', async () => { await trie.put(toU8a('test'), toU8a('one')); + + expect(trie.root).toEqual(new Uint8Array([43, 119, 232, 84, 123, 197, 94, 42, 149, 34, 124, 147, 159, 159, 157, 103, 149, 45, 225, 233, 112, 160, 23, 224, 145, 11, 229, 16, 176, 144, 175, 243])); }); it('should get updated a value', async () => { @@ -75,16 +97,22 @@ describe('simple save and retrive', () => { const value = await trie.get(toU8a('doge')); expect(value).toEqual(null); + expect(trie.root).toEqual(new Uint8Array([43, 119, 232, 84, 123, 197, 94, 42, 149, 34, 124, 147, 159, 159, 157, 103, 149, 45, 225, 233, 112, 160, 23, 224, 145, 11, 229, 16, 176, 144, 175, 243])); }); }); -describe('storing longer values', () => { +describe.skip('storing longer values', () => { const trie = new Trie(); const longString = 'this will be a really really really long value'; const longStringRoot = '0xb173e2db29e79c78963cff5196f8a983fbe0171388972106b114ef7f5c24dfa3'; it('should store a longer string', async () => { await trie.put(toU8a('done'), toU8a(longString)); + + expect(trie.root).toEqual(new Uint8Array([79, 170, 48, 228, 218, 25, 91, 13, 86, 114, 153, 98, 252, 127, 54, 124, 242, 163, 153, 215, 249, 196, 65, 209, 186, 130, 39, 6, 181, 246, 231, 111])); + }); + + it('stores subsequent values', async () => { await trie.put(toU8a('doge'), toU8a('coin')); expect(longStringRoot).toEqual(u8aToHex(trie.root)); @@ -98,10 +126,14 @@ describe('storing longer values', () => { it('should when being modiefied delete the old value', async () => { await trie.put(toU8a('done'), toU8a('test')); + + expect(trie.root).toEqual(new Uint8Array([ + 219, 23, 9, 241, 116, 187, 56, 113, 244, 253, 200, 19, 242, 27, 188, 158, 217, 68, 67, 249, 222, 154, 243, 190, 184, 56, 173, 54, 214, 42, 91, 206 + ])); }); }); -describe('testing Extentions and branches', () => { +describe.skip('testing Extentions and branches', () => { const trie = new Trie(); it('should store a value', async () => { @@ -125,7 +157,7 @@ describe('testing Extentions and branches', () => { }); }); -describe('testing Extentions and branches - reverse', () => { +describe.skip('testing Extentions and branches - reverse', () => { const trie = new Trie(); it('should create extention to store this value', async () => { @@ -149,7 +181,7 @@ describe('testing Extentions and branches - reverse', () => { }); }); -describe('testing deletions cases', () => { +describe.skip('testing deletions cases', () => { const trie = new Trie(); it('should delete from a branch->branch-branch', async () => { diff --git a/packages/trie-db/test/offical.spec.js b/packages/trie-db-old/test/offical.spec.js similarity index 92% rename from packages/trie-db/test/offical.spec.js rename to packages/trie-db-old/test/offical.spec.js index 1516c476f..f008f3d9c 100644 --- a/packages/trie-db/test/offical.spec.js +++ b/packages/trie-db-old/test/offical.spec.js @@ -12,14 +12,14 @@ import getTests from '../../../test/getTests'; const trietest = getTests('TrieTests/trietest.json'); const trieanyorder = getTests('TrieTests/trieanyorder.json'); -describe('official tests', () => { +describe.skip('official tests', () => { let trie; beforeEach(() => { trie = new Trie(); }); - describe('ordered tests', () => { + describe.skip('ordered tests', () => { const testNames = Object.keys(trietest); trietest.forEach(({ name, input, root }) => { @@ -39,7 +39,7 @@ describe('official tests', () => { }); }); - describe('unordered tests', () => { + describe.skip('unordered tests', () => { const testNames = Object.keys(trieanyorder); trieanyorder.forEach(({ name, input, root }) => { diff --git a/packages/trie-db/test/polkadotRefactor.spec.js b/packages/trie-db-old/test/polkadotRefactor.spec.js similarity index 97% rename from packages/trie-db/test/polkadotRefactor.spec.js rename to packages/trie-db-old/test/polkadotRefactor.spec.js index f924dd560..8f1d95a89 100644 --- a/packages/trie-db/test/polkadotRefactor.spec.js +++ b/packages/trie-db-old/test/polkadotRefactor.spec.js @@ -11,7 +11,7 @@ const EMPTY_ROOT = '0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e3 const HELLO_ROOT = '0x0a915659b88f80bfa200570bd2767a6ab8cb0a4e44fd240cc5ed7a27728c4531'; const FOO_ROOT = '0xed4e32371288ee83be74f78fb492f01261b7c3dc9c422581bb705c4376492dc6'; -describe('@polkadot refactoring', () => { +describe.skip('@polkadot refactoring', () => { const trie = new Trie(); it('has the correct starting trieroot', () => { diff --git a/packages/trie-db/test/proof.spec.js b/packages/trie-db-old/test/proof.spec.js similarity index 98% rename from packages/trie-db/test/proof.spec.js rename to packages/trie-db-old/test/proof.spec.js index 18dc13736..2e8a3c30b 100644 --- a/packages/trie-db/test/proof.spec.js +++ b/packages/trie-db-old/test/proof.spec.js @@ -8,7 +8,7 @@ import u8aToUtf8 from '@polkadot/util/u8a/toUtf8'; import Trie from '../src/index'; -describe('simple merkle proofs generation and verification', () => { +describe.skip('simple merkle proofs generation and verification', () => { it('create a merkle proof and verify it with a single short key', async () => { const trie = new Trie(); diff --git a/packages/trie-db/test/rawOPs.spec.js b/packages/trie-db-old/test/rawOPs.spec.js similarity index 97% rename from packages/trie-db/test/rawOPs.spec.js rename to packages/trie-db-old/test/rawOPs.spec.js index 4300831a0..071de7c6a 100644 --- a/packages/trie-db/test/rawOPs.spec.js +++ b/packages/trie-db-old/test/rawOPs.spec.js @@ -7,7 +7,7 @@ import randomAsU8a from '@polkadot/util-crypto/random/asU8a'; import Trie from '../src/index'; -describe('put & get raw functions', () => { +describe.skip('put & get raw functions', () => { var trie = new Trie(); var key = randomAsU8a(32); var val = randomAsU8a(32); diff --git a/packages/trie-db/test/streams.spec.js b/packages/trie-db-old/test/streams.spec.js similarity index 97% rename from packages/trie-db/test/streams.spec.js rename to packages/trie-db-old/test/streams.spec.js index daab147ea..4fc916523 100644 --- a/packages/trie-db/test/streams.spec.js +++ b/packages/trie-db-old/test/streams.spec.js @@ -7,7 +7,7 @@ import toU8a from '@polkadot/util/u8a/toU8a'; import Trie from '../src/index'; -describe('kv stream test', () => { +describe.skip('kv stream test', () => { var trie = new Trie(); var init = [{ type: 'del', @@ -112,7 +112,7 @@ describe('kv stream test', () => { }); }); -describe('db stream test', () => { +describe.skip('db stream test', () => { var trie = new Trie(); var init = [{ type: 'put', diff --git a/packages/trie-db/test/taskExecutor.spec.js b/packages/trie-db-old/test/taskExecutor.spec.js similarity index 95% rename from packages/trie-db/test/taskExecutor.spec.js rename to packages/trie-db-old/test/taskExecutor.spec.js index 701a17785..2d52c2798 100644 --- a/packages/trie-db/test/taskExecutor.spec.js +++ b/packages/trie-db-old/test/taskExecutor.spec.js @@ -7,7 +7,7 @@ import taskExecutor from '../src/util/taskExecutor'; const executor = taskExecutor(2); -describe('taskExecutor', () => { +describe.skip('taskExecutor', () => { it('executes in order', async () => { var tasks = [1, 2, 3, 4, 5, 6, 7]; var executionOrder = []; diff --git a/packages/trie-db/LICENSE b/packages/trie-db/LICENSE index a612ad981..5cf408dd2 100644 --- a/packages/trie-db/LICENSE +++ b/packages/trie-db/LICENSE @@ -1,373 +1,15 @@ -Mozilla Public License Version 2.0 -================================== +ISC License (ISC) -1. Definitions --------------- +Copyright 2017-2018 @polkadot/trie-db authors & contributors -1.1. "Contributor" - means each individual or legal entity that creates, contributes to - the creation of, or owns Covered Software. +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. -1.2. "Contributor Version" - means the combination of the Contributions of others (if any) used - by a Contributor and that particular Contributor's Contribution. - -1.3. "Contribution" - means Covered Software of a particular Contributor. - -1.4. "Covered Software" - means Source Code Form to which the initial Contributor has attached - the notice in Exhibit A, the Executable Form of such Source Code - Form, and Modifications of such Source Code Form, in each case - including portions thereof. - -1.5. "Incompatible With Secondary Licenses" - means - - (a) that the initial Contributor has attached the notice described - in Exhibit B to the Covered Software; or - - (b) that the Covered Software was made available under the terms of - version 1.1 or earlier of the License, but not also under the - terms of a Secondary License. - -1.6. "Executable Form" - means any form of the work other than Source Code Form. - -1.7. "Larger Work" - means a work that combines Covered Software with other material, in - a separate file or files, that is not Covered Software. - -1.8. "License" - means this document. - -1.9. "Licensable" - means having the right to grant, to the maximum extent possible, - whether at the time of the initial grant or subsequently, any and - all of the rights conveyed by this License. - -1.10. "Modifications" - means any of the following: - - (a) any file in Source Code Form that results from an addition to, - deletion from, or modification of the contents of Covered - Software; or - - (b) any new file in Source Code Form that contains any Covered - Software. - -1.11. "Patent Claims" of a Contributor - means any patent claim(s), including without limitation, method, - process, and apparatus claims, in any patent Licensable by such - Contributor that would be infringed, but for the grant of the - License, by the making, using, selling, offering for sale, having - made, import, or transfer of either its Contributions or its - Contributor Version. - -1.12. "Secondary License" - means either the GNU General Public License, Version 2.0, the GNU - Lesser General Public License, Version 2.1, the GNU Affero General - Public License, Version 3.0, or any later versions of those - licenses. - -1.13. "Source Code Form" - means the form of the work preferred for making modifications. - -1.14. "You" (or "Your") - means an individual or a legal entity exercising rights under this - License. For legal entities, "You" includes any entity that - controls, is controlled by, or is under common control with You. For - purposes of this definition, "control" means (a) the power, direct - or indirect, to cause the direction or management of such entity, - whether by contract or otherwise, or (b) ownership of more than - fifty percent (50%) of the outstanding shares or beneficial - ownership of such entity. - -2. License Grants and Conditions --------------------------------- - -2.1. Grants - -Each Contributor hereby grants You a world-wide, royalty-free, -non-exclusive license: - -(a) under intellectual property rights (other than patent or trademark) - Licensable by such Contributor to use, reproduce, make available, - modify, display, perform, distribute, and otherwise exploit its - Contributions, either on an unmodified basis, with Modifications, or - as part of a Larger Work; and - -(b) under Patent Claims of such Contributor to make, use, sell, offer - for sale, have made, import, and otherwise transfer either its - Contributions or its Contributor Version. - -2.2. Effective Date - -The licenses granted in Section 2.1 with respect to any Contribution -become effective for each Contribution on the date the Contributor first -distributes such Contribution. - -2.3. Limitations on Grant Scope - -The licenses granted in this Section 2 are the only rights granted under -this License. No additional rights or licenses will be implied from the -distribution or licensing of Covered Software under this License. -Notwithstanding Section 2.1(b) above, no patent license is granted by a -Contributor: - -(a) for any code that a Contributor has removed from Covered Software; - or - -(b) for infringements caused by: (i) Your and any other third party's - modifications of Covered Software, or (ii) the combination of its - Contributions with other software (except as part of its Contributor - Version); or - -(c) under Patent Claims infringed by Covered Software in the absence of - its Contributions. - -This License does not grant any rights in the trademarks, service marks, -or logos of any Contributor (except as may be necessary to comply with -the notice requirements in Section 3.4). - -2.4. Subsequent Licenses - -No Contributor makes additional grants as a result of Your choice to -distribute the Covered Software under a subsequent version of this -License (see Section 10.2) or under the terms of a Secondary License (if -permitted under the terms of Section 3.3). - -2.5. Representation - -Each Contributor represents that the Contributor believes its -Contributions are its original creation(s) or it has sufficient rights -to grant the rights to its Contributions conveyed by this License. - -2.6. Fair Use - -This License is not intended to limit any rights You have under -applicable copyright doctrines of fair use, fair dealing, or other -equivalents. - -2.7. Conditions - -Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted -in Section 2.1. - -3. Responsibilities -------------------- - -3.1. Distribution of Source Form - -All distribution of Covered Software in Source Code Form, including any -Modifications that You create or to which You contribute, must be under -the terms of this License. You must inform recipients that the Source -Code Form of the Covered Software is governed by the terms of this -License, and how they can obtain a copy of this License. You may not -attempt to alter or restrict the recipients' rights in the Source Code -Form. - -3.2. Distribution of Executable Form - -If You distribute Covered Software in Executable Form then: - -(a) such Covered Software must also be made available in Source Code - Form, as described in Section 3.1, and You must inform recipients of - the Executable Form how they can obtain a copy of such Source Code - Form by reasonable means in a timely manner, at a charge no more - than the cost of distribution to the recipient; and - -(b) You may distribute such Executable Form under the terms of this - License, or sublicense it under different terms, provided that the - license for the Executable Form does not attempt to limit or alter - the recipients' rights in the Source Code Form under this License. - -3.3. Distribution of a Larger Work - -You may create and distribute a Larger Work under terms of Your choice, -provided that You also comply with the requirements of this License for -the Covered Software. If the Larger Work is a combination of Covered -Software with a work governed by one or more Secondary Licenses, and the -Covered Software is not Incompatible With Secondary Licenses, this -License permits You to additionally distribute such Covered Software -under the terms of such Secondary License(s), so that the recipient of -the Larger Work may, at their option, further distribute the Covered -Software under the terms of either this License or such Secondary -License(s). - -3.4. Notices - -You may not remove or alter the substance of any license notices -(including copyright notices, patent notices, disclaimers of warranty, -or limitations of liability) contained within the Source Code Form of -the Covered Software, except that You may alter any license notices to -the extent required to remedy known factual inaccuracies. - -3.5. Application of Additional Terms - -You may choose to offer, and to charge a fee for, warranty, support, -indemnity or liability obligations to one or more recipients of Covered -Software. However, You may do so only on Your own behalf, and not on -behalf of any Contributor. You must make it absolutely clear that any -such warranty, support, indemnity, or liability obligation is offered by -You alone, and You hereby agree to indemnify every Contributor for any -liability incurred by such Contributor as a result of warranty, support, -indemnity or liability terms You offer. You may include additional -disclaimers of warranty and limitations of liability specific to any -jurisdiction. - -4. Inability to Comply Due to Statute or Regulation ---------------------------------------------------- - -If it is impossible for You to comply with any of the terms of this -License with respect to some or all of the Covered Software due to -statute, judicial order, or regulation then You must: (a) comply with -the terms of this License to the maximum extent possible; and (b) -describe the limitations and the code they affect. Such description must -be placed in a text file included with all distributions of the Covered -Software under this License. Except to the extent prohibited by statute -or regulation, such description must be sufficiently detailed for a -recipient of ordinary skill to be able to understand it. - -5. Termination --------------- - -5.1. The rights granted under this License will terminate automatically -if You fail to comply with any of its terms. However, if You become -compliant, then the rights granted under this License from a particular -Contributor are reinstated (a) provisionally, unless and until such -Contributor explicitly and finally terminates Your grants, and (b) on an -ongoing basis, if such Contributor fails to notify You of the -non-compliance by some reasonable means prior to 60 days after You have -come back into compliance. Moreover, Your grants from a particular -Contributor are reinstated on an ongoing basis if such Contributor -notifies You of the non-compliance by some reasonable means, this is the -first time You have received notice of non-compliance with this License -from such Contributor, and You become compliant prior to 30 days after -Your receipt of the notice. - -5.2. If You initiate litigation against any entity by asserting a patent -infringement claim (excluding declaratory judgment actions, -counter-claims, and cross-claims) alleging that a Contributor Version -directly or indirectly infringes any patent, then the rights granted to -You by any and all Contributors for the Covered Software under Section -2.1 of this License shall terminate. - -5.3. In the event of termination under Sections 5.1 or 5.2 above, all -end user license agreements (excluding distributors and resellers) which -have been validly granted by You or Your distributors under this License -prior to termination shall survive termination. - -************************************************************************ -* * -* 6. Disclaimer of Warranty * -* ------------------------- * -* * -* Covered Software is provided under this License on an "as is" * -* basis, without warranty of any kind, either expressed, implied, or * -* statutory, including, without limitation, warranties that the * -* Covered Software is free of defects, merchantable, fit for a * -* particular purpose or non-infringing. The entire risk as to the * -* quality and performance of the Covered Software is with You. * -* Should any Covered Software prove defective in any respect, You * -* (not any Contributor) assume the cost of any necessary servicing, * -* repair, or correction. This disclaimer of warranty constitutes an * -* essential part of this License. No use of any Covered Software is * -* authorized under this License except under this disclaimer. * -* * -************************************************************************ - -************************************************************************ -* * -* 7. Limitation of Liability * -* -------------------------- * -* * -* Under no circumstances and under no legal theory, whether tort * -* (including negligence), contract, or otherwise, shall any * -* Contributor, or anyone who distributes Covered Software as * -* permitted above, be liable to You for any direct, indirect, * -* special, incidental, or consequential damages of any character * -* including, without limitation, damages for lost profits, loss of * -* goodwill, work stoppage, computer failure or malfunction, or any * -* and all other commercial damages or losses, even if such party * -* shall have been informed of the possibility of such damages. This * -* limitation of liability shall not apply to liability for death or * -* personal injury resulting from such party's negligence to the * -* extent applicable law prohibits such limitation. Some * -* jurisdictions do not allow the exclusion or limitation of * -* incidental or consequential damages, so this exclusion and * -* limitation may not apply to You. * -* * -************************************************************************ - -8. Litigation -------------- - -Any litigation relating to this License may be brought only in the -courts of a jurisdiction where the defendant maintains its principal -place of business and such litigation shall be governed by laws of that -jurisdiction, without reference to its conflict-of-law provisions. -Nothing in this Section shall prevent a party's ability to bring -cross-claims or counter-claims. - -9. Miscellaneous ----------------- - -This License represents the complete agreement concerning the subject -matter hereof. If any provision of this License is held to be -unenforceable, such provision shall be reformed only to the extent -necessary to make it enforceable. Any law or regulation which provides -that the language of a contract shall be construed against the drafter -shall not be used to construe this License against a Contributor. - -10. Versions of the License ---------------------------- - -10.1. New Versions - -Mozilla Foundation is the license steward. Except as provided in Section -10.3, no one other than the license steward has the right to modify or -publish new versions of this License. Each version will be given a -distinguishing version number. - -10.2. Effect of New Versions - -You may distribute the Covered Software under the terms of the version -of the License under which You originally received the Covered Software, -or under the terms of any subsequent version published by the license -steward. - -10.3. Modified Versions - -If you create software not governed by this License, and you want to -create a new license for such software, you may create and use a -modified version of this License if you rename the license and remove -any references to the name of the license steward (except to note that -such modified license differs from this License). - -10.4. Distributing Source Code Form that is Incompatible With Secondary -Licenses - -If You choose to distribute Source Code Form that is Incompatible With -Secondary Licenses under the terms of this version of the License, the -notice described in Exhibit B of this License must be attached. - -Exhibit A - Source Code Form License Notice -------------------------------------------- - - This Source Code Form is subject to the terms of the Mozilla Public - License, v. 2.0. If a copy of the MPL was not distributed with this - file, You can obtain one at http://mozilla.org/MPL/2.0/. - -If it is not possible or desirable to put the notice in a particular -file, then You may include the notice in a location (such as a LICENSE -file in a relevant directory) where a recipient would be likely to look -for such a notice. - -You may add additional accurate notices of copyright ownership. - -Exhibit B - "Incompatible With Secondary Licenses" Notice ---------------------------------------------------------- - - This Source Code Form is "Incompatible With Secondary Licenses", as - defined by the Mozilla Public License, v. 2.0. +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. diff --git a/packages/trie-db/README.md b/packages/trie-db/README.md index a748624ea..63731190e 100644 --- a/packages/trie-db/README.md +++ b/packages/trie-db/README.md @@ -1,30 +1,5 @@ -[![polkadotjs](https://img.shields.io/badge/polkadot-js-orange.svg?style=flat-square)](https://polkadot.js.org) -![mpl-2](https://img.shields.io/badge/license-MPL-lightgrey.svg?style=flat-square) -[![style](https://img.shields.io/badge/code%20style-semistandard-lightgrey.svg?style=flat-square)](https://github.com/Flet/semistandard) -[![npm](https://img.shields.io/npm/v/@polkadot/trie-db.svg?style=flat-square)](https://www.npmjs.com/package/@polkadot/trie-db) -[![travis](https://img.shields.io/travis/polkadot-js/trie.svg?style=flat-square)](https://travis-ci.org/polkadot-js/trie) -[![maintainability](https://img.shields.io/codeclimate/maintainability/polkadot-js/trie.svg?style=flat-square)](https://codeclimate.com/github/polkadot-js/trie/maintainability) -[![coverage](https://img.shields.io/coveralls/polkadot-js/trie.svg?style=flat-square)](https://coveralls.io/github/polkadot-js/trie?branch=master) -[![dependency](https://david-dm.org/polkadot-js/trie.svg?style=flat-square&path=packages/trie)](https://david-dm.org/polkadot-js/trie?path=packages/trie-db) -[![devDependency](https://david-dm.org/polkadot-js/trie/dev-status.svg?style=flat-square&path=packages/trie-db)](https://david-dm.org/polkadot-js/trie?path=packages/trie-db#info=devDependencies) - # @polkadot/trie-db -An Implementation of the [Ethereum Merkle Patricia Tree](https://github.com/ethereum/wiki/wiki/Patricia-Tree) that uses Blake2 for hashing. +A re-implementation of a Patricia Trie. Unlike other implementations, this allows for the specification of a hash function as well as encoder/decoder and operates in a sync fashion by default. -# Adapted from Ethereum - -The has been adapted from the [original EthereumJs implementation](https://github.com/ethereumjs/merkle-patricia-tree) with the following changes - - -- Introduce [TypeScript](https://www.typescriptlang.org/) into the mix -- ES6-ify as applicable (Classes, `self` -> `this`, `function() {}` -> `() => {}`) -- Moves to a Promise-based interface (as applicable) -- Support only Uint8Array key/values (Buffer/String in the original) -- Use utilities from [@polkadot/util](https://github.com/polkadot-js/common) -- Supports blake2 hashing (sha3 in the original), aligning with the Polkadot tech stack) -- Updated the build steps with sources moved to [src/](src/) -- Tests updated to use [Jest](https://facebook.github.io/jest/) -- Upgrade interfaces to latest [levelup](https://github.com/Level/levelup) -- Removed the secure interface (not applicable for Substrate chains) - -... well at least the above is the _idea_ behind the fork ... +This is a JavaScript port of the (Python Ethereum Trie)[https://github.com/ethereum/py-trie] with additions to support transactions (checkpoints) and multiple hashing (npot only Keccak, but also Blake2 as found in Polkadot). diff --git a/packages/trie-db/package.json b/packages/trie-db/package.json index 4ff9f3bfb..f5c82f145 100644 --- a/packages/trie-db/package.json +++ b/packages/trie-db/package.json @@ -1,45 +1,41 @@ { "name": "@polkadot/trie-db", - "version": "0.28.25", - "description": "This is an implementation of the modified merkle patricia tree as speficed in Ethereum's yellow paper, adpated for Polkadot.", + "version": "0.29.0", + "description": "This is an implementation of the modified merkle patricia tree as speficed in Ethereum's yellow paper, adpated for Polkadot", "main": "index.js", + "keywords": [ + "Polkadot" + ], + "author": "Jaco Greeff ", "maintainers": [ "Jaco Greeff " ], - "contributors": [ - "mjbecze ", - "Aaron Kumavis (https://github.com/kumavis)" - ], + "contributors": [], + "license": "ISC", + "engines": { + "node": ">=6.4" + }, + "publishConfig": { + "access": "public", + "registry": "https://registry.npmjs.org" + }, "repository": { "type": "git", - "url": "https://github.com/polkadot-js/common.git" + "url": "git+https://github.com/polkadot-js/common.git" }, "bugs": { "url": "https://github.com/polkadot-js/common/issues" }, - "keywords": [ - "merkle", - "radix", - "trie", - "ethereum" - ], - "license": "MPL-2.0", + "homepage": "https://github.com/polkadot-js/common/tree/master/packages/trie-db#readme", "dependencies": { "@babel/runtime": "^7.0.0-rc.1", - "@polkadot/trie-hash": "^0.28.25", - "@polkadot/util": "^0.28.25", - "@polkadot/util-crypto": "^0.28.25", - "@polkadot/util-rlp": "^0.28.25", - "@types/async": "^2.0.49", - "@types/semaphore": "^1.1.0", - "async": "^2.6.1", - "encoding-down": "^5.0.3", - "level-ws": "0.1.0", - "levelup": "^3.1.1", - "memdown": "^3.0.0", - "semaphore": ">=1.0.1" + "@polkadot/trie-hash": "^0.29.0", + "@polkadot/util": "^0.29.0", + "@polkadot/util-crypto": "^0.29.0", + "@polkadot/util-rlp": "^0.29.0" }, "devDependencies": { + "@polkadot/db": "^0.29.0", "ethereumjs-testing": "1.0.4" } } diff --git a/packages/trie-db/src/constants.ts b/packages/trie-db/src/constants.ts index 3741cb114..5b1a36f88 100644 --- a/packages/trie-db/src/constants.ts +++ b/packages/trie-db/src/constants.ts @@ -1,13 +1,20 @@ // Copyright 2017-2018 @polkadot/trie-db authors & contributors // This software may be modified and distributed under the terms -// of the MPL-2.0 license. See the LICENSE file for details. +// of the ISC license. See the LICENSE file for details. +import hashing from '@polkadot/util-crypto/keccak/asU8a'; import trieRoot from '@polkadot/trie-hash/root'; -const EMPTY_ROOT_U8A = trieRoot([]); -const EMPTY_ROOT_STR = EMPTY_ROOT_U8A.toString(); +const EMPTY_NODE = new Uint8Array(); +const EMPTY_U8A = trieRoot([]); +const EMPTY_U8A_STR = trieRoot([]); +const EMPTY_HASH = hashing(EMPTY_U8A); +const EMPTY_HASH_STR = EMPTY_HASH.toString(); export { - EMPTY_ROOT_U8A, - EMPTY_ROOT_STR + EMPTY_NODE, + EMPTY_HASH, + EMPTY_HASH_STR, + EMPTY_U8A, + EMPTY_U8A_STR }; diff --git a/packages/trie-db/src/ethereum.spec.js b/packages/trie-db/src/ethereum.spec.js new file mode 100644 index 000000000..146e1727b --- /dev/null +++ b/packages/trie-db/src/ethereum.spec.js @@ -0,0 +1,59 @@ +// Copyright 2017-2018 @polkadot/trie-db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +import toU8a from '@polkadot/util/u8a/toU8a'; +import u8aToHex from '@polkadot/util/u8a/toHex'; + +import Trie from './index'; +import getTests from '../../../test/getTests'; + +const trietest = getTests('TrieTests/trietest.json'); +const trieanyorder = getTests('TrieTests/trieanyorder.json'); + +describe('official tests', () => { + let trie; + + beforeEach(() => { + trie = new Trie(); + }); + + // FIXME (maybe) These all have empty/null values, we don't allow those atm as a key nor value + describe.skip('ordered tests', () => { + const testNames = Object.keys(trietest); + + trietest.forEach(({ name, input, root }) => { + it(name, () => { + input.map((input) => + trie.put(toU8a(input[0]), toU8a(input[1])) + ); + + expect( + u8aToHex(trie.getRoot()) + ).toEqual( + root + ); + }); + }); + }); + + describe('unordered tests', () => { + const testNames = Object.keys(trieanyorder); + + trieanyorder.forEach(({ name, input, root }) => { + const keys = Object.keys(input); + + it(name, () => { + keys.map((key) => + trie.put(toU8a(key), toU8a(input[key])) + ) + + expect( + u8aToHex(trie.getRoot()) + ).toEqual( + root + ); + }); + }); + }); +}); diff --git a/packages/trie-db/src/index.spec.js b/packages/trie-db/src/index.spec.js new file mode 100644 index 000000000..9644c1b8f --- /dev/null +++ b/packages/trie-db/src/index.spec.js @@ -0,0 +1,267 @@ +// Copyright 2015-2018 authors & contributors of https://github.com/ethereumjs/merkle-patricia-tree +// Copyright 2017-2018 @polkadot/trie-db authors & contributors +// This software may be modified and distributed under the terms +// of the MPL-2.0 license. See the LICENSE file for details. + +import u8aToHex from '@polkadot/util/u8a/toHex'; +import toU8a from '@polkadot/util/u8a/toU8a'; + +import Trie from './index'; + +describe('simple save and retrive', () => { + const trie = new Trie(); + + it('starts with a valid root', () => { + expect( + trie.getRoot() + ).toEqual( + new Uint8Array([ + 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33 + ]) + ) + }); + + it('save a value', () => { + trie.put(toU8a('test'), toU8a('one')); + + expect( + trie.getRoot() + ).toEqual( + new Uint8Array([ + 43, 119, 232, 84, 123, 197, 94, 42, 149, 34, 124, 147, 159, 159, 157, 103, 149, 45, 225, 233, 112, 160, 23, 224, 145, 11, 229, 16, 176, 144, 175, 243 + ]) + ); + }); + + it('should get a value', () => { + expect( + trie.get(toU8a('test')) + ).toEqual(toU8a('one')); + }); + + it('should update a value', () => { + trie.put(toU8a('test'), toU8a('two')); + + expect( + trie.get(toU8a('test')) + ).toEqual(toU8a('two')); + expect( + trie.getRoot() + ).toEqual( + new Uint8Array([ + 127, 237, 74, 184, 190, 46, 103, 129, 65, 141, 13, 96, 61, 232, 146, 13, 56, 32, 59, 29, 246, 58, 175, 140, 72, 182, 196, 103, 230, 245, 229, 148 + ]) + ); + }); + + it('should delete a value', () => { + trie.del(toU8a('test')); + + expect( + trie.get(toU8a('test')) + ).toEqual(null); + expect( + trie.getRoot() + ).toEqual( + new Uint8Array([ + 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33 + ]) + ); + }); + + it('should recreate a value', () => { + trie.put(toU8a('test'), toU8a('one')); + + expect( + trie.getRoot() + ).toEqual( + new Uint8Array([ + 43, 119, 232, 84, 123, 197, 94, 42, 149, 34, 124, 147, 159, 159, 157, 103, 149, 45, 225, 233, 112, 160, 23, 224, 145, 11, 229, 16, 176, 144, 175, 243 + ]) + ); + }); + + it('should get updated a value', () => { + expect( + trie.get(toU8a('test')) + ).toEqual(toU8a('one')); + }); + + it('should create a branch here', () => { + trie.put(toU8a('doge'), toU8a('coin')); + + expect( + trie.getRoot() + ).toEqual( + toU8a('0xde8a34a8c1d558682eae1528b47523a483dd8685d6db14b291451a66066bf0fc') + ); + }); + + it('should get a value that is in a branch', () => { + expect( + trie.get(toU8a('doge')) + ).toEqual(toU8a('coin')); + }); + + it('should delete from a branch', () => { + trie.del(toU8a('doge')); + + expect( + trie.get(toU8a('doge')) + ).toEqual(null); + expect( + trie.getRoot() + ).toEqual( + new Uint8Array([ + 43, 119, 232, 84, 123, 197, 94, 42, 149, 34, 124, 147, 159, 159, 157, 103, 149, 45, 225, 233, 112, 160, 23, 224, 145, 11, 229, 16, 176, 144, 175, 243 + ]) + ); + }); +}); + +describe('storing longer values', () => { + const trie = new Trie(); + const longString = 'this will be a really really really long value'; + const longStringRoot = '0xb173e2db29e79c78963cff5196f8a983fbe0171388972106b114ef7f5c24dfa3'; + + it('should store a longer string', () => { + trie.put(toU8a('done'), toU8a(longString)); + + expect( + trie.getRoot() + ).toEqual(new Uint8Array([ + 79, 170, 48, 228, 218, 25, 91, 13, 86, 114, 153, 98, 252, 127, 54, 124, 242, 163, 153, 215, 249, 196, 65, 209, 186, 130, 39, 6, 181, 246, 231, 111 + ])); + }); + + it('should retrieve a longer value (first pass)', () => { + expect( + trie.get(toU8a('done')) + ).toEqual(toU8a(longString)); + }); + + it('should store subsequent values', () => { + trie.put(toU8a('doge'), toU8a('coin')); + + expect(longStringRoot).toEqual(u8aToHex(trie.getRoot())); + }); + + it('should retrieve a longer value (second pass)', () => { + expect( + trie.get(toU8a('done')) + ).toEqual(toU8a(longString)); + }); + + it('should retrieve subsequent values', () => { + expect(trie.get(toU8a('doge'))).toEqual(toU8a('coin')); + }); + + it('should when being modiefied delete the old value', () => { + trie.put(toU8a('done'), toU8a('test')); + + expect( + trie.getRoot() + ).toEqual(new Uint8Array([ + 219, 23, 9, 241, 116, 187, 56, 113, 244, 253, 200, 19, 242, 27, 188, 158, 217, 68, 67, 249, 222, 154, 243, 190, 184, 56, 173, 54, 214, 42, 91, 206 + ])); + }); +}); + +describe('testing Extentions and branches', () => { + const trie = new Trie(); + + it('should store a value', () => { + trie.put(toU8a('doge'), toU8a('coin')); + }); + + it('should create extention to store this value', () => { + trie.put(toU8a('do'), toU8a('verb')); + + expect( + toU8a('0xf803dfcb7e8f1afd45e88eedb4699a7138d6c07b71243d9ae9bff720c99925f9') + ).toEqual(trie.getRoot()); + }); + + it('should store this value under the extention ', () => { + trie.put(toU8a('done'), toU8a('finished')); + + expect( + '0x409cff4d820b394ed3fb1cd4497bdd19ffa68d30ae34157337a7043c94a3e8cb' + ).toEqual(u8aToHex(trie.getRoot())); + }); +}); + +describe('testing Extentions and branches - reverse', () => { + const trie = new Trie(); + + it('should create extention to store this value', () => { + trie.put(toU8a('do'), toU8a('verb')); + + const value = trie.get(toU8a('do')); + + expect(value).toEqual(toU8a('verb')); + }); + + it('should store a value', () => { + trie.put(toU8a('doge'), toU8a('coin')); + }); + + it('should store this value under the extention ', () => { + trie.put(toU8a('done'), toU8a('finished')); + + expect( + '0x409cff4d820b394ed3fb1cd4497bdd19ffa68d30ae34157337a7043c94a3e8cb' + ).toEqual(u8aToHex(trie.getRoot())); + }); +}); + +describe('testing deletions cases', () => { + const trie = new Trie(); + + it('should delete from a branch->branch-branch', () => { + trie.put(new Uint8Array([11, 11, 11]), toU8a('first')), + trie.put(new Uint8Array([12, 22, 22]), toU8a('create the first branch')), + trie.put(new Uint8Array([12, 34, 44]), toU8a('create the last branch')) + trie.del(new Uint8Array([12, 22, 22])); + + const value = trie.get(new Uint8Array([12, 22, 22])); + + expect(null).toEqual(value); + }); + + it('should delete from a branch->branch-extention', () => { + trie.put(new Uint8Array([11, 11, 11]), toU8a('first')), + trie.put(new Uint8Array([12, 22, 22]), toU8a('create the first branch')), + trie.put(new Uint8Array([12, 33, 33]), toU8a('create the middle branch')), + trie.put(new Uint8Array([12, 33, 44]), toU8a('create the last branch')) + trie.del(new Uint8Array([12, 22, 22])); + + const value = trie.get(new Uint8Array([12, 22, 22])); + + expect(null).toEqual(value); + }); + + it('should delete from a extention->branch-extention', () => { + trie.put(new Uint8Array([11, 11, 11]), toU8a('first')); + trie.put(new Uint8Array([12, 22, 22]), toU8a('create the first branch')); + trie.put(new Uint8Array([12, 33, 33]), toU8a('create the middle branch')); + trie.put(new Uint8Array([12, 33, 44]), toU8a('create the last branch')); + trie.del(new Uint8Array([11, 11, 11])); + + const value = trie.get(new Uint8Array([11, 11, 11])); + + expect(null).toEqual(value); + }); + + it('should delete from a extention->branch-branch', () => { + trie.put(new Uint8Array([11, 11, 11]), toU8a('first')); + trie.put(new Uint8Array([12, 22, 22]), toU8a('create the first branch')); + trie.put(new Uint8Array([12, 33, 33]), toU8a('create the middle branch')); + trie.put(new Uint8Array([12, 34, 44]), toU8a('create the last branch')); + trie.del(new Uint8Array([11, 11, 11])); + + const value = trie.get(new Uint8Array([11, 11, 11])); + + expect(null).toEqual(value); + }); +}); diff --git a/packages/trie-db/src/index.ts b/packages/trie-db/src/index.ts index ba2352fca..9849a9925 100644 --- a/packages/trie-db/src/index.ts +++ b/packages/trie-db/src/index.ts @@ -1,23 +1,470 @@ -// Copyright 2015-2018 authors & contributors of https://github.com/ethereumjs/merkle-patricia-tree // Copyright 2017-2018 @polkadot/trie-db authors & contributors // This software may be modified and distributed under the terms -// of the MPL-2.0 license. See the LICENSE file for details. +// of the ISC license. See the LICENSE file for details. -import { HashFn } from './types'; +import { TxDb } from '@polkadot/db/types'; +import { Node, NodeBranch, NodeEncodedOrEmpty, NodeKv, NodeNotEmpty, NodeType } from './types'; -// import logger from '@polkadot/util/logger'; -import CheckpointTrie from './CheckpointTrie'; -import { prove, verifyProof } from './proof'; +import MemoryDb from '@polkadot/db/Memory'; +import isNull from '@polkadot/util/is/null'; +import logger from '@polkadot/util/logger'; +import u8aConcat from '@polkadot/util/u8a/concat'; +import hashing from '@polkadot/util-crypto/keccak/asU8a'; +import toNibbles from '@polkadot/trie-hash/util/asNibbles'; -// const l = logger('trie'); +import { isBranchNode, isEmptyNode, isExtensionNode, isKvNode, isLeafNode } from './util/is'; +import { extractKey, keyEquals, keyStartsWith, computeExtensionKey, computeLeafKey, consumeCommonPrefix } from './util/key'; +import { decodeNibbles, encodeNibbles } from './util/nibbles'; +import { getNodeType, decodeNode, encodeNode } from './util/node'; +import { EMPTY_HASH, EMPTY_U8A } from './constants'; -export default class Trie extends CheckpointTrie { - static prove = prove; - static verifyProof = verifyProof; +const l = logger('trie/db'); - constructor (db: any, root?: Uint8Array, hashing?: HashFn) { - super(db, root, hashing); +export default class Trie { + private db: TxDb; + private txRoot: Uint8Array; + private rootHash: Uint8Array; - // l.debug('Created Trie'); + constructor (db?: TxDb, rootHash: Uint8Array = EMPTY_HASH) { + this.db = db || new MemoryDb(); + this.rootHash = rootHash; + this.txRoot = rootHash; + } + + private createCheckpoint (): Uint8Array { + this.txRoot = this.rootHash; + + return this.txRoot; + } + + private commitCheckpoint (): Uint8Array { + return this.rootHash; + } + + private revertCheckpoint (): Uint8Array { + this.rootHash = this.txRoot; + + return this.rootHash; + } + + transaction (fn: () => boolean): Uint8Array { + try { + this.createCheckpoint(); + + return this.db.transaction(fn) + ? this.commitCheckpoint() + : this.revertCheckpoint(); + } catch (error) { + this.revertCheckpoint(); + + throw error; + } + } + + del (key: Uint8Array) { + l.debug(() => ['del', { key }]); + + this._setRootNode( + this._del( + this.getNode(this.rootHash), + toNibbles(key) + ) + ); + } + + get (key: Uint8Array): Uint8Array | null { + l.debug(() => ['get', { key }]); + + return this._get( + this.getNode(this.rootHash), + toNibbles(key) + ); + } + + put (key: Uint8Array, value: Uint8Array): void { + l.debug(() => ['put', { key, value }]); + + this._setRootNode( + this._put( + this.getNode(this.rootHash), + toNibbles(key), + value + ) + ); + } + + getRoot (): Uint8Array { + const rootNode = this.getNode(this.rootHash); + + if (isNull(rootNode)) { + return EMPTY_U8A; + } + + return this.rootHash; + } + + setRoot (rootHash: Uint8Array): void { + this.rootHash = rootHash; + // return this._setRootNode(rootNode); + } + + private getNode (hash: Uint8Array | null): Node { + if (!hash || hash.length === 0 || keyEquals(hash, EMPTY_HASH)) { + return null; + } else if (hash.length < 32) { + return decodeNode(hash); + } + + return decodeNode( + this.db.get(hash) + ); + } + + private _del (node: Node, trieKey: Uint8Array): Node { + // l.debug(() => ['_del', { node, trieKey }]); + + if (isEmptyNode(node)) { + return null; + } else if (isBranchNode(node)) { + return this._delBranchNode(node, trieKey); + } else if (isKvNode(node)) { + return this._delKvNode(node, trieKey); + } + + throw new Error('Unreachable'); + } + + private _delBranchNode (node: NodeBranch, trieKey: Uint8Array): Node { + // l.debug(() => ['_delBranchNode', { node, trieKey }]); + + if (trieKey.length === 0) { + node[node.length - 1] = null; + + return this._normaliseBranchNode(node); + } + + const nodeToDelete = this.getNode(node[trieKey[0]]); + const subNode = this._del(nodeToDelete, trieKey.subarray(1)); + const encodedSubNode = this._persistNode(subNode); + + if (keyEquals(encodedSubNode, node[trieKey[0]])) { + return node; + } + + node[trieKey[0]] = encodedSubNode; + + if (isNull(encodedSubNode)) { + return this._normaliseBranchNode(node); + } + + return node; + } + + private _delKvNode (node: NodeNotEmpty, trieKey: Uint8Array): Node { + // l.debug(() => ['_delKvNode', { node, trieKey }]); + + const currentKey = extractKey(node); + const nodeType = getNodeType(node); + + if (!keyStartsWith(trieKey, currentKey)) { + return node; + } + + if (nodeType === NodeType.LEAF) { + return keyEquals(trieKey, currentKey) + ? null + : node; + } + + const subKey = trieKey.subarray(currentKey.length); + const subNode = this.getNode(node[1]); + const newSub = this._del(subNode, subKey); + const encodedNewSub = this._persistNode(newSub); + + if (keyEquals(encodedNewSub, node[1])) { + return node; + } else if (isNull(newSub)) { + return null; + } + + if (isKvNode(newSub)) { + const subNibbles = decodeNibbles(newSub[0]); + const newKey = u8aConcat(currentKey, subNibbles); + + return [ + encodeNibbles(newKey), + newSub[1] + ]; + } else if (isBranchNode(newSub)) { + return [ + encodeNibbles(currentKey), + encodedNewSub + ]; + } + + throw new Error('Unreachable'); + } + + private _get (node: Node, trieKey: Uint8Array): NodeEncodedOrEmpty { + // l.debug(() => ['_get', { node, trieKey }]); + + if (isEmptyNode(node)) { + return null; + } else if (isBranchNode(node)) { + return this._getBranchNode(node, trieKey); + } else if (isKvNode(node)) { + return this._getKvNode(node, trieKey); + } + + throw new Error('Invalid NodeType'); + } + + private _getBranchNode (node: NodeBranch, trieKey: Uint8Array): NodeEncodedOrEmpty { + // l.debug(() => ['_getBranchNode', { node, trieKey }]); + + if (trieKey.length === 0) { + return node[16]; + } + + const subNode = this.getNode(node[trieKey[0]]); + + return this._get(subNode, trieKey.subarray(1)); + } + + private _getKvNode (node: NodeKv, trieKey: Uint8Array): NodeEncodedOrEmpty { + // l.debug(() => ['_getKvNode', { node, trieKey }]); + + const currentKey = extractKey(node); + const nodeType = getNodeType(node); + + l.debug(() => [{ currentKey, trieKey, nodeType }]); + + if (nodeType === NodeType.LEAF) { + if (keyEquals(trieKey, currentKey)) { + return node[1]; + } + + return null; + } else if (nodeType === NodeType.EXTENSION) { + if (keyStartsWith(trieKey, currentKey)) { + const subNode = this.getNode(node[1]); + + return this._get(subNode, trieKey.subarray(currentKey ? currentKey.length : 0)); + } + + return null; + } + + throw new Error('Unreachable'); + } + + private _nodeToDbMapping (node: Node): NodeNotEmpty { + // l.debug(() => ['_nodeToDbMapping', { node }]); + + if (isEmptyNode(node)) { + return [ + null, + null + ]; + } + + const encoded = encodeNode(node); + + if (encoded.length < 32) { + return [ + // @ts-ignore Ok, this is not correct - this comes back as an embedded node, which mean that our definitions here are completely wrong (or what we think it is, does not align with what it should be as per the spec and implementation) + node, + null + ]; + } + + return [ + hashing(encoded), + encoded + ]; + } + + private _normaliseBranchNode (node: NodeNotEmpty): Node { + // l.debug(() => ['_normaliseBranchNode', { node }]); + + const mapped = node + .map((value, index) => ({ + index, + value + })) + .filter(({ value }) => + !!value && value.length !== 0 + ); + + if (mapped.length >= 2) { + return node; + } + + if (node[16]) { + return [ + computeLeafKey(new Uint8Array()), + node[16] + ]; + } + + const [{ index, value }] = mapped; + const subNode = this.getNode(value); + + if (isBranchNode(subNode)) { + return [ + encodeNibbles(new Uint8Array([index])), + this._persistNode(subNode) + ]; + } else if (isKvNode(subNode)) { + const subNibbles = decodeNibbles(subNode[0]); + const newKey = u8aConcat(new Uint8Array([index]), subNibbles); + + return [ + encodeNibbles(newKey), + subNode[1] + ]; + } + + throw new Error('Unreachable'); + } + + private _persistNode (node: Node): NodeEncodedOrEmpty { + const [key, value] = this._nodeToDbMapping(node); + + // l.debug(() => ['_persistNode', { node, key, value }]); + + if (value) { + this.db.put(key as Uint8Array, value); + } + + return key; + } + + private _put (node: Node, trieKey: Uint8Array, value: Uint8Array): NodeNotEmpty { + // l.debug(() => ['_put', { node, trieKey, value }]); + + if (isEmptyNode(node)) { + return [ + computeLeafKey(trieKey), + value + ]; + } else if (isKvNode(node)) { + return this._putKvNode(node, trieKey, value); + } else if (isBranchNode(node)) { + return this._putBranchNode(node, trieKey, value); + } + + throw new Error('Unreachable'); + } + + private _putBranchNode (node: NodeBranch, trieKey: Uint8Array, value: Uint8Array): NodeNotEmpty { + // l.debug(() => ['_putBranchNode', { node, trieKey, value }]); + + if (trieKey && trieKey.length) { + const subNode = this.getNode(node[trieKey[0]]); + const newNode = this._put(subNode, trieKey.subarray(1), value); + + node[trieKey[0]] = this._persistNode(newNode); + } else { + node[node.length - 1] = value; + } + + return node; + } + + private _putKvNode (node: NodeKv, trieKey: Uint8Array, value: Uint8Array): NodeNotEmpty { + // l.debug(() => ['_putKvNode', { node, trieKey, value }]); + + const currentKey = extractKey(node); + const [commonPrefix, currentRemainder, trieRemainder] = consumeCommonPrefix(currentKey, trieKey); + const isExtension = isExtensionNode(node); + const isLeaf = isLeafNode(node); + let newNode: NodeNotEmpty; + + l.debug(() => ['_putKvNode', { currentKey, commonPrefix, currentRemainder, trieRemainder }]); + + if (currentRemainder.length === 0 && trieRemainder.length === 0) { + if (isLeaf) { + return [ + node[0], + value + ]; + } + + const subNode = this.getNode(node[1]); + + newNode = this._put(subNode, trieRemainder, value); + } else if (currentRemainder.length === 0) { + if (isExtension) { + const subNode = this.getNode(node[1]); + + newNode = this._put(subNode, trieRemainder, value); + } else { + const subPosition = trieRemainder[0]; + const subKey = computeLeafKey(trieRemainder.subarray(1)); + const subNode: NodeKv = [subKey, value]; + + newNode = [ + null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, + node[1] + ]; + newNode[subPosition] = this._persistNode(subNode); + } + } else { + newNode = [ + null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, + null + ]; + + if (currentRemainder.length === 1 && isExtension) { + newNode[currentRemainder[0]] = node[1]; + } else { + const computedKey = isExtension + ? computeExtensionKey(currentRemainder.subarray(1)) + : computeLeafKey(currentRemainder.subarray(1)); + + newNode[currentRemainder[0]] = this._persistNode([ + computedKey, + node[1] + ]); + } + + if (trieRemainder.length) { + newNode[trieRemainder[0]] = this._persistNode([ + computeLeafKey(trieRemainder.subarray(1)), + value + ]); + } else { + newNode[16] = value; + } + } + + l.log(() => ['newNode', newNode]); + + if (commonPrefix.length) { + return [ + computeExtensionKey(commonPrefix), + this._persistNode(newNode) + ]; + } + + return newNode; + } + + private _setRootNode (node: Node): void { + // l.debug(() => ['_setRootNode', { node }]); + + if (isEmptyNode(node)) { + this.rootHash = EMPTY_HASH; + } else { + const encoded = encodeNode(node); + const newHash = hashing(encoded); + + this.db.put(newHash, encoded); + + this.rootHash = newHash; + } } } diff --git a/packages/trie-db/src/nibbles/toU8a.ts b/packages/trie-db/src/nibbles/toU8a.ts deleted file mode 100644 index 2b151cfe1..000000000 --- a/packages/trie-db/src/nibbles/toU8a.ts +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2015-2018 authors & contributors of https://github.com/ethereumjs/merkle-patricia-tree -// Copyright 2017-2018 @polkadot/trie-db authors & contributors -// This software may be modified and distributed under the terms -// of the MPL-2.0 license. See the LICENSE file for details. - -export default function nibblesToU8a (arr: Array): Uint8Array { - const u8a = new Uint8Array(arr.length / 2); - - for (let i = 0; i < u8a.length; i++) { - const q = i * 2; - - u8a[i] = (arr[q] << 4) + arr[q + 1]; - } - - return u8a; -} diff --git a/packages/trie-db/src/types.ts b/packages/trie-db/src/types.ts new file mode 100644 index 000000000..9000a048a --- /dev/null +++ b/packages/trie-db/src/types.ts @@ -0,0 +1,32 @@ +// Copyright 2017-2018 @polkadot/trie-db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +export enum NodeType { + EMPTY = 0, + LEAF = 1, + EXTENSION = 2, + BRANCH = 3 +} + +export type NodeEmpty = null; + +export type NodeEncoded = Uint8Array; + +export type NodeEncodedOrEmpty = NodeEncoded | NodeEmpty; + +export type NodeBranch = [ + NodeEncodedOrEmpty, NodeEncodedOrEmpty, NodeEncodedOrEmpty, NodeEncodedOrEmpty, + NodeEncodedOrEmpty, NodeEncodedOrEmpty, NodeEncodedOrEmpty, NodeEncodedOrEmpty, + NodeEncodedOrEmpty, NodeEncodedOrEmpty, NodeEncodedOrEmpty, NodeEncodedOrEmpty, + NodeEncodedOrEmpty, NodeEncodedOrEmpty, NodeEncodedOrEmpty, NodeEncodedOrEmpty, + NodeEncodedOrEmpty +]; + +export type EncodedPath = Uint8Array | null; + +export type NodeKv = [EncodedPath, NodeEncodedOrEmpty]; + +export type NodeNotEmpty = NodeKv | NodeBranch; + +export type Node = NodeEmpty | NodeNotEmpty; diff --git a/packages/trie-db/src/util/is.ts b/packages/trie-db/src/util/is.ts new file mode 100644 index 000000000..78a86a92a --- /dev/null +++ b/packages/trie-db/src/util/is.ts @@ -0,0 +1,29 @@ +// Copyright 2017-2018 @polkadot/trie-db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +import { Node, NodeBranch, NodeKv, NodeEmpty, NodeType } from '../types'; + +import isNull from '@polkadot/util/is/null'; + +import { getNodeType } from './node'; + +export function isEmptyNode (node: Node): node is NodeEmpty { + return isNull(node); +} + +export function isKvNode (node: Node): node is NodeKv { + return !isEmptyNode(node) && node.length === 2; +} + +export function isExtensionNode (node: Node): node is NodeKv { + return getNodeType(node) === NodeType.EXTENSION; +} + +export function isLeafNode (node: Node): node is NodeKv { + return getNodeType(node) === NodeType.LEAF; +} + +export function isBranchNode (node: Node): node is NodeBranch { + return !isEmptyNode(node) && node.length === 17; +} diff --git a/packages/trie-db/src/util/key.ts b/packages/trie-db/src/util/key.ts new file mode 100644 index 000000000..5c0df4be2 --- /dev/null +++ b/packages/trie-db/src/util/key.ts @@ -0,0 +1,78 @@ +// Copyright 2017-2018 @polkadot/trie-db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +import isNull from '@polkadot/util/is/null'; + +import { EncodedPath, Node } from '../types'; + +import { isKvNode } from './is'; +import { addNibblesTerminator, decodeNibbles, encodeNibbles, removeNibblesTerminator } from './nibbles'; + +export function extractKey (node: Node): Uint8Array { + if (!isKvNode(node)) { + throw new Error('Can only extract keys from KV branches'); + } + + const [prefixedKey] = node; + + return removeNibblesTerminator( + decodeNibbles(prefixedKey) + ); +} + +export function keyEquals (key: Uint8Array | null, test: Uint8Array | null): boolean { + if (isNull(key) && isNull(test)) { + return true; + } else if (isNull(key) || isNull(test) || (key.length !== test.length)) { + return false; + } + + return keyStartsWith(key, test); +} + +export function keyStartsWith (key: Uint8Array | null, partial: Uint8Array | null): boolean { + if (isNull(key) && isNull(partial)) { + return true; + } else if (isNull(key) || isNull(partial) || (key.length < partial.length)) { + return false; + } + + for (let index = 0; index < partial.length; index++) { + if (key[index] !== partial[index]) { + return false; + } + } + + return true; +} + +export function computeExtensionKey (nibbles: Uint8Array): EncodedPath { + return encodeNibbles(nibbles); +} + +export function computeLeafKey (nibbles: Uint8Array): EncodedPath { + return encodeNibbles( + addNibblesTerminator(nibbles) + ); +} + +export function getCommonPrefixLength (left: Uint8Array, right: Uint8Array): number { + for (let index = 0; index < left.length && index < right.length; index++) { + if (left[index] !== right[index]) { + return index; + } + } + + return Math.min(left.length, right.length); +} + +export function consumeCommonPrefix (left: Uint8Array, right: Uint8Array): [Uint8Array, Uint8Array, Uint8Array] { + const length = getCommonPrefixLength(left, right); + + return [ + left.subarray(0, length), + left.subarray(length), + right.subarray(length) + ]; +} diff --git a/packages/trie-db/src/util/nibbles.ts b/packages/trie-db/src/util/nibbles.ts new file mode 100644 index 000000000..82491dc30 --- /dev/null +++ b/packages/trie-db/src/util/nibbles.ts @@ -0,0 +1,70 @@ +// Copyright 2017-2018 @polkadot/trie-db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +import { NodeEncodedOrEmpty, NodeEncoded } from '../types'; + +import fromNibbles from '@polkadot/trie-hash/util/fromNibbles'; +import toNibbles from '@polkadot/trie-hash/util/asNibbles'; + +const HP_FLAG_2 = 2; +const HP_FLAG_0 = 0; + +const NIBBLE_TERMINATOR = 16; + +const NEEDS_TERMINATOR = [HP_FLAG_2, HP_FLAG_2 + 1]; +const IS_ODD_LENGTH = [HP_FLAG_0 + 1, HP_FLAG_2 + 1]; + +export function isNibblesTerminated (nibbles: Uint8Array): boolean { + return nibbles[nibbles.length - 1] === NIBBLE_TERMINATOR; +} + +export function addNibblesTerminator (nibbles: Uint8Array): Uint8Array { + if (isNibblesTerminated(nibbles)) { + return nibbles; + } + + const terminated = new Uint8Array(nibbles.length + 1); + + terminated.set(nibbles); + terminated[nibbles.length] = NIBBLE_TERMINATOR; + + return terminated; +} + +export function removeNibblesTerminator (nibbles: Uint8Array): Uint8Array { + if (isNibblesTerminated(nibbles)) { + return nibbles.subarray(0, nibbles.length - 1); + } + + return nibbles; +} + +export function decodeNibbles (value: NodeEncodedOrEmpty): Uint8Array { + const nibblesWithFlag = toNibbles(value); + const [flag] = nibblesWithFlag; + + const rawNibbles = IS_ODD_LENGTH.includes(flag) + ? nibblesWithFlag.subarray(1) + : nibblesWithFlag.subarray(2); + + return NEEDS_TERMINATOR.includes(flag) + ? addNibblesTerminator(rawNibbles) + : rawNibbles; +} + +export function encodeNibbles (nibbles: Uint8Array): NodeEncoded { + const flag = isNibblesTerminated(nibbles) + ? HP_FLAG_2 + : HP_FLAG_0; + const rawNibbles = removeNibblesTerminator(nibbles); + const prefix = rawNibbles.length % 2 + ? [flag + 1] + : [flag, 0]; + const prefixed = new Uint8Array(prefix.length + rawNibbles.length); + + prefixed.set(prefix); + prefixed.set(rawNibbles, prefix.length); + + return fromNibbles(prefixed); +} diff --git a/packages/trie-db/src/util/node.ts b/packages/trie-db/src/util/node.ts new file mode 100644 index 000000000..03f952db9 --- /dev/null +++ b/packages/trie-db/src/util/node.ts @@ -0,0 +1,50 @@ +// Copyright 2017-2018 @polkadot/trie-db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +import { Node, NodeType, NodeNotEmpty } from '../types'; + +import isNull from '@polkadot/util/is/null'; +import nodeDecode from '@polkadot/util-rlp/decode'; +import nodeEncode from '@polkadot/util-rlp/encode'; + +import { isBranchNode, isEmptyNode, isKvNode } from './is'; +import { decodeNibbles, isNibblesTerminated } from './nibbles'; + +export function getNodeType (node: Node): NodeType { + if (isEmptyNode(node)) { + return NodeType.EMPTY; + } else if (isKvNode(node)) { + const [key] = node; + const nibbles = decodeNibbles(key); + + return isNibblesTerminated(nibbles) + ? NodeType.LEAF + : NodeType.EXTENSION; + } else if (isBranchNode(node)) { + return NodeType.BRANCH; + } + + throw new Error(`Unable to determine node type`); +} + +export function decodeNode (encoded: Uint8Array | Node): Node { + if (isNull(encoded) || encoded.length === 0) { + return null; + } else if (Array.isArray(encoded)) { + return encoded; + } + + const decoded = (nodeDecode(encoded) as NodeNotEmpty) + .map((value) => + value && value.length + ? value + : null + ); + + return decoded as NodeNotEmpty; +} + +export function encodeNode (node: Node): Uint8Array { + return nodeEncode(node); +} diff --git a/packages/trie-hash/package.json b/packages/trie-hash/package.json index df2c55576..0c4d9acd1 100644 --- a/packages/trie-hash/package.json +++ b/packages/trie-hash/package.json @@ -1,6 +1,6 @@ { "name": "@polkadot/trie-hash", - "version": "0.28.25", + "version": "0.29.0", "description": "Creation of triehash values", "main": "index.js", "keywords": [ @@ -29,8 +29,8 @@ "homepage": "https://github.com/polkadot-js/common/tree/master/packages/trie-hash#readme", "dependencies": { "@babel/runtime": "^7.0.0-rc.1", - "@polkadot/util": "^0.28.25", - "@polkadot/util-crypto": "^0.28.25", - "@polkadot/util-rlp": "^0.28.25" + "@polkadot/util": "^0.29.0", + "@polkadot/util-crypto": "^0.29.0", + "@polkadot/util-rlp": "^0.29.0" } } diff --git a/packages/trie-hash/src/util/asNibbles.ts b/packages/trie-hash/src/util/asNibbles.ts index dcda01aae..e9afaf7df 100644 --- a/packages/trie-hash/src/util/asNibbles.ts +++ b/packages/trie-hash/src/util/asNibbles.ts @@ -4,7 +4,7 @@ /** * @name asNibbles - * @signature asNibbles (pairs: Trie$Pairs): Uint8Array + * @signature asNibbles (bytes: Uint8Array | Array): Uint8Array * @summary Converts the input to Nibbles. * @description * From an `Uint8Array` input, calculate and return a list of nibbles that makes up the input. @@ -13,14 +13,18 @@ * * asNibbles(new Uint8Array([0x41, 0x20]) // => Uint8Array([4, 1, 2, 0]) */ -export default function asNibbles (bytes: Uint8Array | Array): Uint8Array { +export default function asNibbles (bytes: Uint8Array | Array | null): Uint8Array { + if (bytes === null) { + return new Uint8Array(); + } + // HACK TypeScript gets a little bit confused as to what to apply, hence casting here although the reduces function for both types does exactly the same - return (bytes as number[]).reduce((result, byte, index) => { - result.set( + return (bytes as number[]).reduce((nibbles, byte, index) => { + nibbles.set( [byte >> 4, byte & 0b1111], index * 2 ); - return result; + return nibbles; }, new Uint8Array(bytes.length * 2)); } diff --git a/packages/trie-hash/src/util/fromNibbles.spec.js b/packages/trie-hash/src/util/fromNibbles.spec.js new file mode 100644 index 000000000..2841c165f --- /dev/null +++ b/packages/trie-hash/src/util/fromNibbles.spec.js @@ -0,0 +1,39 @@ +// Copyright 2017-2018 @polkadot/trie-hash authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +import u8aFromString from '@polkadot/util/u8a/fromString'; + +import fromNibbles from './fromNibbles'; + +describe('fromNibbles', () => { + it('converts a single', () => { + expect( + fromNibbles( + new Uint8Array([4, 1]) + ) + ).toEqual( + u8aFromString('A') + ); + }); + + it('converts an number of values', () => { + expect( + fromNibbles( + new Uint8Array([3, 1, 2, 3, 4, 5]) + ) + ).toEqual( + new Uint8Array([0x31, 0x23, 0x45]) + ); + }); + + it('converts an number of values (array)', () => { + expect( + fromNibbles( + [3, 1, 2, 3, 4, 5, 7, 8] + ) + ).toEqual( + new Uint8Array([0x31, 0x23, 0x45, 0x78]) + ); + }); +}); diff --git a/packages/trie-hash/src/util/fromNibbles.ts b/packages/trie-hash/src/util/fromNibbles.ts new file mode 100644 index 000000000..cc83e840b --- /dev/null +++ b/packages/trie-hash/src/util/fromNibbles.ts @@ -0,0 +1,26 @@ +// Copyright 2017-2018 @polkadot/trie-hash authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +/** + * @name fromNibbles + * @signature fromNibbles (nibbles: Uint8Array): Uint8Array + * @summary Converts the input to Nibbles. + * @description + * From an `Uint8Array` input, calculate and return a list of nibbles that makes up the input. + * @example + * import { fromNibbles } from '@polkadot/trie-hash/util'; + * + * asNibbles(new Uint8Array([4, 1, 2, 0])) // => Uint8Array([0x41, 0x20] + */ +export default function fromNibbles (nibbles: Uint8Array | Array): Uint8Array { + const u8a = new Uint8Array(nibbles.length / 2); + + for (let index = 0; index < u8a.length; index++) { + const nibIndex = index * 2; + + u8a[index] = (nibbles[nibIndex] << 4) + nibbles[nibIndex + 1]; + } + + return u8a; +} diff --git a/packages/util-crypto/package.json b/packages/util-crypto/package.json index 25ff63b3c..6f423f4f4 100644 --- a/packages/util-crypto/package.json +++ b/packages/util-crypto/package.json @@ -1,6 +1,6 @@ { "name": "@polkadot/util-crypto", - "version": "0.28.25", + "version": "0.29.0", "description": "A collection of useful crypto utilities for @polkadot", "main": "index.js", "keywords": [ @@ -29,7 +29,7 @@ "homepage": "https://github.com/polkadot-js/common/tree/master/packages/util-crypto#readme", "dependencies": { "@babel/runtime": "^7.0.0-rc.1", - "@polkadot/util": "^0.28.25", + "@polkadot/util": "^0.29.0", "blakejs": "^1.1.0", "js-sha3": "^0.8.0", "tweetnacl": "^1.0.0", diff --git a/packages/util-keyring/package.json b/packages/util-keyring/package.json index bad591fc9..c1a554b53 100644 --- a/packages/util-keyring/package.json +++ b/packages/util-keyring/package.json @@ -1,6 +1,6 @@ { "name": "@polkadot/util-keyring", - "version": "0.28.25", + "version": "0.29.0", "description": "Keyring management", "main": "index.js", "engines": { @@ -31,8 +31,8 @@ "homepage": "https://github.com/polkadot-js/client/tree/master/packages/util-keyring#readme", "dependencies": { "@babel/runtime": "^7.0.0-rc.1", - "@polkadot/util": "^0.28.25", - "@polkadot/util-crypto": "^0.28.25", + "@polkadot/util": "^0.29.0", + "@polkadot/util-crypto": "^0.29.0", "@types/bs58": "^3.0.30", "bs58": "^4.0.1" } diff --git a/packages/util-rlp/package.json b/packages/util-rlp/package.json index 83ff195e4..f60c8e330 100644 --- a/packages/util-rlp/package.json +++ b/packages/util-rlp/package.json @@ -1,6 +1,6 @@ { "name": "@polkadot/util-rlp", - "version": "0.28.25", + "version": "0.29.0", "description": "RLP encoding and decoding", "main": "index.js", "keywords": [ @@ -31,7 +31,7 @@ "homepage": "https://github.com/polkadot-js/common/tree/master/packages/trie-hash#readme", "dependencies": { "@babel/runtime": "^7.0.0-rc.1", - "@polkadot/util": "^0.28.25" + "@polkadot/util": "^0.29.0" }, "devDependencies": { "ethereumjs-testing": "1.0.4" diff --git a/packages/util/package.json b/packages/util/package.json index a4e88773a..c8cd99bab 100644 --- a/packages/util/package.json +++ b/packages/util/package.json @@ -1,6 +1,6 @@ { "name": "@polkadot/util", - "version": "0.28.25", + "version": "0.29.0", "description": "A collection of useful utilities for @polkadot", "main": "index.js", "keywords": [ diff --git a/packages/util/src/is/object.ts b/packages/util/src/is/object.ts index 016607961..223fcb001 100644 --- a/packages/util/src/is/object.ts +++ b/packages/util/src/is/object.ts @@ -2,6 +2,10 @@ // This software may be modified and distributed under the terms // of the ISC license. See the LICENSE file for details. +type ObjectIndexed = { + [index: string]: any +}; + /** * @name isObject * @signature isObject (value: any): boolean @@ -14,6 +18,6 @@ * isObject({}); // => true * isObject('something'); // => false */ -export default function isObject (value: any): value is object { +export default function isObject (value: any): value is ObjectIndexed { return typeof value === 'object'; } diff --git a/packages/util/src/logger.ts b/packages/util/src/logger.ts index 5899d398c..b24adbd0b 100644 --- a/packages/util/src/logger.ts +++ b/packages/util/src/logger.ts @@ -7,7 +7,14 @@ import { Logger, Logger$Data } from './types'; import chalk from 'chalk'; import moment from 'moment'; +import isBn from './is/bn'; +import isBuffer from './is/buffer'; import isFunction from './is/function'; +import isObject from './is/object'; +import isNull from './is/null'; +import isU8a from '@polkadot/util/is/u8a'; +import isUndefined from './is/undefined'; +import u8aToHex from '@polkadot/util/u8a/toHex'; type ConsoleType = 'error' | 'log' | 'warn'; type LogType = ConsoleType | 'debug'; @@ -26,11 +33,45 @@ const chalked = { warn: chalk.yellow }; +function formatValue (value: any): any { + if (isNull(value)) { + return null; + } else if (isUndefined(value)) { + return undefined; + } + + if (Array.isArray(value)) { + return value.map(formatValue); + } + + if (isBn(value)) { + return value.toString(); + } + + if (isBuffer(value)) { + return `0x${value.toString('hex')}`; + } + + if (isU8a(value)) { + return u8aToHex(value); + } + + if (isObject(value) && value.constructor === Object) { + return Object.keys(value).reduce((result, key) => { + result[key] = formatValue(value[key]); + + return result; + }, {} as { [index: string]: any }); + } + + return value; +} + function apply (log: LogType, type: string, values: Logger$Data): void { if (values.length === 1 && isFunction(values[0])) { const fnResult = (values[0] as Function)(); - return apply(log, type, fnResult); + return apply(log, type, Array.isArray(fnResult) ? fnResult : [fnResult]); } const chalk = (value: string): string => @@ -40,7 +81,9 @@ function apply (log: LogType, type: string, values: Logger$Data): void { console[logTo[log]].apply( console, [ chalk(moment().format('YYYY-MM-DD HH:mm:ss')), chalk(type) - ].concat(values) + ].concat( + values.map(formatValue) + ) ); } diff --git a/tsconfig.json b/tsconfig.json index d1e4dc7f9..7db06df88 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,13 +3,13 @@ "compilerOptions": { "baseUrl": ".", "paths": { - "@polkadot/db-diskdown/*": [ "packages/db-diskdown/src/*" ], + "@polkadot/db/*": [ "packages/db/src/*" ], + "@polkadot/trie-db-old/*": [ "packages/trie-db-old/src/*" ], "@polkadot/trie-db/*": [ "packages/trie-db/src/*" ], "@polkadot/trie-hash/*": [ "packages/trie-hash/src/*" ], "@polkadot/util-crypto/*": [ "packages/util-crypto/src/*" ], "@polkadot/util-keyring/*": [ "packages/util-keyring/src/*" ], "@polkadot/util-rlp/*": [ "packages/util-rlp/src/*" ], - "@polkadot/util-triedb/*": [ "packages/util-triedb/src/*" ], "@polkadot/util/*": [ "packages/util/src/*" ] }, "typeRoots": [ diff --git a/yarn.lock b/yarn.lock index 208c6860d..3762f59d5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1407,12 +1407,50 @@ abbrev@1: version "1.1.1" resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" +abstract-error@^1.0.1, abstract-error@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/abstract-error/-/abstract-error-1.0.2.tgz#5d9e5da97ead2a24a0dfefeeb8df2877c2636b1f" + dependencies: + inherits-ex "~1.1.2" + +abstract-iterator@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/abstract-iterator/-/abstract-iterator-3.0.1.tgz#223097b403ac1cdfa80a8529e683c83bcc8361a0" + dependencies: + abstract-error "^1.0.1" + abstract-object "^2.1.5" + inherits-ex "^1.0.6" + minimatch "^3.0.4" + util-ex "^0.2.9" + xtend "~4.0.0" + abstract-leveldown@^5.0.0, abstract-leveldown@~5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/abstract-leveldown/-/abstract-leveldown-5.0.0.tgz#f7128e1f86ccabf7d2893077ce5d06d798e386c6" dependencies: xtend "~4.0.0" +abstract-nosql@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/abstract-nosql/-/abstract-nosql-3.0.0.tgz#3f25915daf54b45976e988957e414ac6f9eb68b6" + dependencies: + abstract-error "^1.0.1" + abstract-object "~2.1.5" + buffer-codec "~1.6.2" + events-ex "^1.1.2" + inherits-ex "^1.1.1" + util-ex "^0.2.8" + xtend "~4.0.0" + +abstract-object@^2.1.5, abstract-object@~2.1.5: + version "2.1.9" + resolved "https://registry.yarnpkg.com/abstract-object/-/abstract-object-2.1.9.tgz#fa9fc8888ecccc42c3da6c4f61b9ceaa1a1a1a14" + dependencies: + custom-ability "^1.4.2" + events-ex "^1.1.5" + inherits-ex "^1.1.2" + util-ex "^0.3.10" + acorn-globals@^4.0.0, acorn-globals@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/acorn-globals/-/acorn-globals-4.1.0.tgz#ab716025dbe17c54d3ef81d32ece2b2d99fe2538" @@ -1832,10 +1870,21 @@ binary-extensions@^1.0.0: version "1.11.0" resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-1.11.0.tgz#46aa1751fb6a2f93ee5e689bb1087d4b14c6c205" +bindings@^1.3.0, bindings@~1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.3.0.tgz#b346f6ecf6a95f5a815c5839fc7cdb22502f1ed7" + bindings@~1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.2.1.tgz#14ad6113812d2d37d72e67b4cacb4bb726505f11" +bl@^1.0.0: + version "1.2.2" + resolved "https://registry.yarnpkg.com/bl/-/bl-1.2.2.tgz#a160911717103c07410cef63ef51b397c025af9c" + dependencies: + readable-stream "^2.3.5" + safe-buffer "^5.1.1" + blakejs@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/blakejs/-/blakejs-1.1.0.tgz#69df92ef953aa88ca51a32df6ab1c54a155fc7a5" @@ -1931,6 +1980,29 @@ bser@^2.0.0: dependencies: node-int64 "^0.4.0" +buffer-alloc-unsafe@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz#bd7dc26ae2972d0eda253be061dba992349c19f0" + +buffer-alloc@^1.1.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/buffer-alloc/-/buffer-alloc-1.2.0.tgz#890dd90d923a873e08e10e5fd51a57e5b7cce0ec" + dependencies: + buffer-alloc-unsafe "^1.1.0" + buffer-fill "^1.0.0" + +buffer-codec@~1.6.2: + version "1.6.4" + resolved "https://registry.yarnpkg.com/buffer-codec/-/buffer-codec-1.6.4.tgz#ef79ff722538bf98f7e1e46ab36416f437c4789c" + dependencies: + abstract-error "^1.0.2" + custom-factory "^1.5.0" + util-ex "^0.3.14" + +buffer-fill@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/buffer-fill/-/buffer-fill-1.0.0.tgz#f8f78b76789888ef39f205cd637f68e702122b2c" + buffer-from@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.0.tgz#87fcaa3a298358e0ade6e442cfce840740d1ad04" @@ -2472,10 +2544,30 @@ currently-unhandled@^0.4.1: dependencies: array-find-index "^1.0.1" +custom-ability@^1.4.0, custom-ability@^1.4.2: + version "1.5.0" + resolved "https://registry.yarnpkg.com/custom-ability/-/custom-ability-1.5.0.tgz#2feb5a5919ba02ce6a30938f151065a7165afef3" + dependencies: + inherits-ex "~1.2.0" + util-ex "^0.3.15" + +custom-factory@^1.5.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/custom-factory/-/custom-factory-1.6.0.tgz#04827a930a9b5136711447fcb7ba62cba9fed659" + dependencies: + depd "^1.1.2" + inherits-ex "^1.2.3" + cyclist@~0.2.2: version "0.2.2" resolved "https://registry.yarnpkg.com/cyclist/-/cyclist-0.2.2.tgz#1b33792e11e914a2fd6d6ed6447464444e5fa640" +d@1: + version "1.0.0" + resolved "https://registry.yarnpkg.com/d/-/d-1.0.0.tgz#754bb5bfe55451da69a58b94d45f4c5b0462d58f" + dependencies: + es5-ext "^0.10.9" + dargs@^4.0.1: version "4.1.0" resolved "https://registry.yarnpkg.com/dargs/-/dargs-4.1.0.tgz#03a9dbb4b5c2f139bf14ae53f0b8a2a6a86f4e17" @@ -2544,10 +2636,20 @@ decode-uri-component@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/decode-uri-component/-/decode-uri-component-0.2.0.tgz#eb3913333458775cb84cd1a1fae062106bb87545" +decompress-response@^3.3.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-3.3.0.tgz#80a4dd323748384bfa248083622aedec982adff3" + dependencies: + mimic-response "^1.0.0" + dedent@^0.7.0: version "0.7.0" resolved "https://registry.yarnpkg.com/dedent/-/dedent-0.7.0.tgz#2495ddbaf6eb874abb0e1be9df22d2e5a544326c" +deep-extend@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.6.0.tgz#c4fa7c95404a17a9c3e8ca7e1537312b736330ac" + deep-extend@~0.4.0: version "0.4.2" resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.4.2.tgz#48b699c27e334bf89f10892be432f6e4c7d34a7f" @@ -2621,6 +2723,10 @@ delegates@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" +depd@^1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9" + detect-indent@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/detect-indent/-/detect-indent-4.0.0.tgz#f76d064352cdf43a1cb6ce619c4ee3a9475de208" @@ -2631,7 +2737,7 @@ detect-indent@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/detect-indent/-/detect-indent-5.0.0.tgz#3871cc0a6a002e8c3e5b3cf7f336264675f06b9d" -detect-libc@^1.0.2: +detect-libc@^1.0.2, detect-libc@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-1.0.3.tgz#fa137c4bd698edf55cd5cd02ac559f91a4c4ba9b" @@ -2790,6 +2896,22 @@ es-to-primitive@^1.1.1: is-date-object "^1.0.1" is-symbol "^1.0.1" +es5-ext@^0.10.35, es5-ext@^0.10.9, es5-ext@~0.10.14, es5-ext@~0.10.7: + version "0.10.46" + resolved "https://registry.yarnpkg.com/es5-ext/-/es5-ext-0.10.46.tgz#efd99f67c5a7ec789baa3daa7f79870388f7f572" + dependencies: + es6-iterator "~2.0.3" + es6-symbol "~3.1.1" + next-tick "1" + +es6-iterator@~2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/es6-iterator/-/es6-iterator-2.0.3.tgz#a7de889141a05a94b0854403b2d0a0fbfa98f3b7" + dependencies: + d "1" + es5-ext "^0.10.35" + es6-symbol "^3.1.1" + es6-promise@^4.0.3: version "4.2.4" resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-4.2.4.tgz#dc4221c2b16518760bd8c39a52d8f356fc00ed29" @@ -2800,6 +2922,13 @@ es6-promisify@^5.0.0: dependencies: es6-promise "^4.0.3" +es6-symbol@^3.1.1, es6-symbol@~3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/es6-symbol/-/es6-symbol-3.1.1.tgz#bf00ef4fdab6ba1b46ecb7b629b4c7ed5715cc77" + dependencies: + d "1" + es5-ext "~0.10.14" + escape-string-regexp@^1.0.2, escape-string-regexp@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" @@ -3010,6 +3139,14 @@ ethereumjs-testing@1.0.4: dependencies: node-dir "^0.1.16" +events-ex@^1.1.2, events-ex@^1.1.5: + version "1.1.5" + resolved "https://registry.yarnpkg.com/events-ex/-/events-ex-1.1.5.tgz#5fd8b9c03d341ab9c313ef22b0b5b79a5f5c2405" + dependencies: + custom-ability "^1.4.0" + es5-ext "~0.10.7" + util-ex "^0.3.10" + exec-sh@^0.2.0: version "0.2.1" resolved "https://registry.yarnpkg.com/exec-sh/-/exec-sh-0.2.1.tgz#163b98a6e89e6b65b47c2a28d215bc1f63989c38" @@ -3068,6 +3205,10 @@ expand-range@^1.8.1: dependencies: fill-range "^2.1.0" +expand-template@^1.0.2: + version "1.1.1" + resolved "https://registry.yarnpkg.com/expand-template/-/expand-template-1.1.1.tgz#981f188c0c3a87d2e28f559bc541426ff94f21dd" + expect@^23.5.0: version "23.5.0" resolved "https://registry.yarnpkg.com/expect/-/expect-23.5.0.tgz#18999a0eef8f8acf99023fde766d9c323c2562ed" @@ -3139,6 +3280,10 @@ fast-deep-equal@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz#7b05218ddf9667bf7f370bf7fdb2cb15fdd0aa49" +fast-future@~1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/fast-future/-/fast-future-1.0.2.tgz#8435a9aaa02d79248d17d704e76259301d99280a" + fast-glob@^2.0.2: version "2.2.2" resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-2.2.2.tgz#71723338ac9b4e0e2fff1d6748a2a13d5ed352bf" @@ -3309,6 +3454,10 @@ from2@^2.1.0: inherits "^2.0.1" readable-stream "^2.0.0" +fs-constants@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad" + fs-extra@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-6.0.1.tgz#8abc128f7946e310135ddc93b98bddb410e7a34b" @@ -3480,6 +3629,10 @@ gitconfiglocal@^1.0.0: dependencies: ini "^1.3.2" +github-from-package@0.0.0: + version "0.0.0" + resolved "https://registry.yarnpkg.com/github-from-package/-/github-from-package-0.0.0.tgz#97fb5d96bfde8973313f20e8288ef9a167fa64ce" + glob-base@^0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/glob-base/-/glob-base-0.3.0.tgz#dbb164f6221b1c0b1ccf82aea328b497df0ea3c4" @@ -3819,6 +3972,24 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" +inherits-ex@^1.0.6, inherits-ex@^1.1.1, inherits-ex@^1.1.2, inherits-ex@^1.2.3, inherits-ex@~1.2.0: + version "1.2.3" + resolved "https://registry.yarnpkg.com/inherits-ex/-/inherits-ex-1.2.3.tgz#ba0da6258e9b855f98429e82ea97c4cc0e4e459d" + dependencies: + xtend "^4.0.0" + +inherits-ex@~1.0.6: + version "1.0.9" + resolved "https://registry.yarnpkg.com/inherits-ex/-/inherits-ex-1.0.9.tgz#5c9ff02326f45f2c3ebbde27462dee011d9fedd9" + dependencies: + xtend "^4.0.0" + +inherits-ex@~1.1.2: + version "1.1.8" + resolved "https://registry.yarnpkg.com/inherits-ex/-/inherits-ex-1.1.8.tgz#f709a9a59c58a93ddfe01d3530e67247bae29be3" + dependencies: + xtend "^4.0.0" + inherits@2, inherits@^2.0.1, inherits@^2.0.3, inherits@~2.0.0, inherits@~2.0.1, inherits@~2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" @@ -4920,6 +5091,10 @@ lru-cache@^4.1.2, lru-cache@^4.1.3: pseudomap "^1.0.2" yallist "^2.1.2" +lru_map@^0.3.3: + version "0.3.3" + resolved "https://registry.yarnpkg.com/lru_map/-/lru_map-0.3.3.tgz#b5c8351b9464cbd750335a79650a0ec0e56118dd" + ltgt@~2.2.0: version "2.2.1" resolved "https://registry.yarnpkg.com/ltgt/-/ltgt-2.2.1.tgz#f35ca91c493f7b73da0e07495304f17b31f87ee5" @@ -5118,6 +5293,10 @@ mimic-fn@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-1.1.0.tgz#e667783d92e89dbd342818b5230b9d62a672ad18" +mimic-response@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-1.0.1.tgz#4923538878eef42063cb8a3e3b0798781487ab1b" + minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.3, minimatch@^3.0.4: version "3.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" @@ -5243,6 +5422,10 @@ nan@^2.0.7, nan@^2.3.0: version "2.8.0" resolved "https://registry.yarnpkg.com/nan/-/nan-2.8.0.tgz#ed715f3fe9de02b57a5e6252d90a96675e1f085a" +nan@^2.10.0: + version "2.11.0" + resolved "https://registry.yarnpkg.com/nan/-/nan-2.11.0.tgz#574e360e4d954ab16966ec102c0c049fd961a099" + nan@^2.9.2: version "2.10.0" resolved "https://registry.yarnpkg.com/nan/-/nan-2.10.0.tgz#96d0cd610ebd58d4b4de9cc0c6828cda99c7548f" @@ -5280,10 +5463,20 @@ needle@^2.2.0: iconv-lite "^0.4.4" sax "^1.2.4" +next-tick@1: + version "1.0.0" + resolved "https://registry.yarnpkg.com/next-tick/-/next-tick-1.0.0.tgz#ca86d1fe8828169b0120208e3dc8424b9db8342c" + nice-try@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.4.tgz#d93962f6c52f2c1558c0fbda6d512819f1efe1c4" +node-abi@^2.2.0: + version "2.4.3" + resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-2.4.3.tgz#43666b7b17e57863e572409edbb82115ac7af28b" + dependencies: + semver "^5.4.1" + node-dir@^0.1.16: version "0.1.17" resolved "https://registry.yarnpkg.com/node-dir/-/node-dir-0.1.17.tgz#5f5665d93351335caabef8f1c554516cf5f1e4e5" @@ -5370,6 +5563,10 @@ noms@0.0.0: inherits "^2.0.1" readable-stream "~1.0.31" +noop-logger@^0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/noop-logger/-/noop-logger-0.1.1.tgz#94a2b1633c4f1317553007d8966fd0e841b6a4c2" + "nopt@2 || 3": version "3.0.6" resolved "https://registry.yarnpkg.com/nopt/-/nopt-3.0.6.tgz#c6465dbf08abcd4db359317f79ac68a646b28ff9" @@ -5398,6 +5595,20 @@ normalize-path@^2.0.0, normalize-path@^2.0.1, normalize-path@^2.1.1: dependencies: remove-trailing-separator "^1.0.1" +nosql-leveldb@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/nosql-leveldb/-/nosql-leveldb-3.0.1.tgz#b6a590c1b4c1dc029c180f9979dbc000c0530bc1" + dependencies: + abstract-error "^1.0.1" + abstract-iterator "^3.0.1" + abstract-nosql "^3.0.0" + bindings "~1.3.0" + fast-future "~1.0.0" + inherits-ex "^1.0.6" + nan "^2.10.0" + prebuild-install "^4.0.0" + util-ex "^0.2.9" + npm-bundled@^1.0.1: version "1.0.3" resolved "https://registry.yarnpkg.com/npm-bundled/-/npm-bundled-1.0.3.tgz#7e71703d973af3370a9591bafe3a63aca0be2308" @@ -5461,7 +5672,7 @@ npm-run-path@^2.0.0: dependencies: path-key "^2.0.0" -"npmlog@0 || 1 || 2 || 3 || 4", npmlog@^4.0.2, npmlog@^4.1.2: +"npmlog@0 || 1 || 2 || 3 || 4", npmlog@^4.0.1, npmlog@^4.0.2, npmlog@^4.1.2: version "4.1.2" resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.1.2.tgz#08a7f2a8bf734604779a9efa4ad5cc717abb954b" dependencies: @@ -5562,7 +5773,7 @@ optionator@^0.8.1, optionator@^0.8.2: type-check "~0.3.2" wordwrap "~1.0.0" -os-homedir@^1.0.0: +os-homedir@^1.0.0, os-homedir@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/os-homedir/-/os-homedir-1.0.2.tgz#ffbc4988336e0e833de0c168c7ef152121aa7fb3" @@ -5840,6 +6051,26 @@ posix-character-classes@^0.1.0: version "0.1.1" resolved "https://registry.yarnpkg.com/posix-character-classes/-/posix-character-classes-0.1.1.tgz#01eac0fe3b5af71a2a6c02feabb8c1fef7e00eab" +prebuild-install@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/prebuild-install/-/prebuild-install-4.0.0.tgz#206ce8106ce5efa4b6cf062fc8a0a7d93c17f3a8" + dependencies: + detect-libc "^1.0.3" + expand-template "^1.0.2" + github-from-package "0.0.0" + minimist "^1.2.0" + mkdirp "^0.5.1" + node-abi "^2.2.0" + noop-logger "^0.1.1" + npmlog "^4.0.1" + os-homedir "^1.0.1" + pump "^2.0.1" + rc "^1.1.6" + simple-get "^2.7.0" + tar-fs "^1.13.0" + tunnel-agent "^0.6.0" + which-pm-runs "^1.0.0" + prelude-ls@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54" @@ -5917,7 +6148,14 @@ psl@^1.1.24: version "1.1.29" resolved "https://registry.yarnpkg.com/psl/-/psl-1.1.29.tgz#60f580d360170bb722a797cc704411e6da850c67" -pump@^2.0.0: +pump@^1.0.0: + version "1.0.3" + resolved "https://registry.yarnpkg.com/pump/-/pump-1.0.3.tgz#5dfe8311c33bbf6fc18261f9f34702c47c08a954" + dependencies: + end-of-stream "^1.1.0" + once "^1.3.1" + +pump@^2.0.0, pump@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/pump/-/pump-2.0.1.tgz#12399add6e4cf7526d973cbc8b5ce2e2908b3909" dependencies: @@ -5978,6 +6216,15 @@ randomatic@^1.1.3: is-number "^3.0.0" kind-of "^4.0.0" +rc@^1.1.6: + version "1.2.8" + resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.8.tgz#cd924bf5200a075b83c188cd6b9e211b7fc0d3ed" + dependencies: + deep-extend "^0.6.0" + ini "~1.3.0" + minimist "^1.2.0" + strip-json-comments "~2.0.1" + rc@^1.1.7: version "1.2.2" resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.2.tgz#d8ce9cb57e8d64d9c7badd9876c7c34cbe3c7077" @@ -6065,7 +6312,7 @@ read@1, read@~1.0.1: dependencies: mute-stream "~0.0.4" -"readable-stream@1 || 2", readable-stream@^2.0.0, readable-stream@^2.0.4, readable-stream@^2.0.5, readable-stream@^2.2.8: +"readable-stream@1 || 2", readable-stream@^2.0.0, readable-stream@^2.0.4, readable-stream@^2.0.5, readable-stream@^2.2.8, readable-stream@^2.3.0, readable-stream@^2.3.5: version "2.3.6" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf" dependencies: @@ -6565,6 +6812,18 @@ signal-exit@^3.0.0, signal-exit@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.2.tgz#b5fdc08f1287ea1178628e415e25132b73646c6d" +simple-concat@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/simple-concat/-/simple-concat-1.0.0.tgz#7344cbb8b6e26fb27d66b2fc86f9f6d5997521c6" + +simple-get@^2.7.0: + version "2.8.1" + resolved "https://registry.yarnpkg.com/simple-get/-/simple-get-2.8.1.tgz#0e22e91d4575d87620620bc91308d57a77f44b5d" + dependencies: + decompress-response "^3.3.0" + once "^1.3.1" + simple-concat "^1.0.0" + sisteransi@^0.1.1: version "0.1.1" resolved "https://registry.yarnpkg.com/sisteransi/-/sisteransi-0.1.1.tgz#5431447d5f7d1675aac667ccd0b865a4994cb3ce" @@ -6614,6 +6873,13 @@ snapdragon@^0.8.1: source-map-resolve "^0.5.0" use "^3.1.0" +snappy@^6.0.4: + version "6.0.4" + resolved "https://registry.yarnpkg.com/snappy/-/snappy-6.0.4.tgz#8914f84a273270868828f3e2f10f61631aceabe5" + dependencies: + bindings "^1.3.0" + nan "^2.10.0" + sntp@1.x.x: version "1.0.9" resolved "https://registry.yarnpkg.com/sntp/-/sntp-1.0.9.tgz#6541184cc90aeea6c6e7b35e2659082443c66198" @@ -6926,6 +7192,15 @@ table@^4.0.3: slice-ansi "1.0.0" string-width "^2.1.1" +tar-fs@^1.13.0: + version "1.16.3" + resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-1.16.3.tgz#966a628841da2c4010406a82167cbd5e0c72d509" + dependencies: + chownr "^1.0.1" + mkdirp "^0.5.1" + pump "^1.0.0" + tar-stream "^1.1.2" + tar-pack@^3.4.0: version "3.4.1" resolved "https://registry.yarnpkg.com/tar-pack/-/tar-pack-3.4.1.tgz#e1dbc03a9b9d3ba07e896ad027317eb679a10a1f" @@ -6939,6 +7214,18 @@ tar-pack@^3.4.0: tar "^2.2.1" uid-number "^0.0.6" +tar-stream@^1.1.2: + version "1.6.1" + resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-1.6.1.tgz#f84ef1696269d6223ca48f6e1eeede3f7e81f395" + dependencies: + bl "^1.0.0" + buffer-alloc "^1.1.0" + end-of-stream "^1.0.0" + fs-constants "^1.0.0" + readable-stream "^2.3.0" + to-buffer "^1.1.0" + xtend "^4.0.0" + tar@^2.0.0, tar@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/tar/-/tar-2.2.1.tgz#8e4d2a256c0e2185c6b18ad694aec968b83cb1d1" @@ -7029,6 +7316,10 @@ tmpl@1.0.x: version "1.0.4" resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.4.tgz#23640dd7b42d00433911140820e5cf440e521dd1" +to-buffer@^1.1.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/to-buffer/-/to-buffer-1.1.1.tgz#493bd48f62d7c43fcded313a03dcadb2e1213a80" + to-fast-properties@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-1.0.3.tgz#b83571fa4d8c25b82e231b06e3a3055de4ca1a47" @@ -7351,6 +7642,20 @@ util-deprecate@~1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" +util-ex@^0.2.8, util-ex@^0.2.9: + version "0.2.9" + resolved "https://registry.yarnpkg.com/util-ex/-/util-ex-0.2.9.tgz#91a688073de0179c4fa7e9ab05102a60f550402e" + dependencies: + inherits-ex "~1.0.6" + xtend "^4.0.0" + +util-ex@^0.3.10, util-ex@^0.3.14, util-ex@^0.3.15: + version "0.3.15" + resolved "https://registry.yarnpkg.com/util-ex/-/util-ex-0.3.15.tgz#f9261cda13c4327d0740cbe67be1227ded8b0058" + dependencies: + inherits-ex "^1.1.2" + xtend "^4.0.0" + util.promisify@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/util.promisify/-/util.promisify-1.0.0.tgz#440f7165a459c9a16dc145eb8e72f35687097030" @@ -7471,6 +7776,10 @@ which-module@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/which-module/-/which-module-2.0.0.tgz#d9ef07dce77b9902b8a3a8fa4b31c3e3f7e6e87a" +which-pm-runs@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/which-pm-runs/-/which-pm-runs-1.0.0.tgz#670b3afbc552e0b55df6b7780ca74615f23ad1cb" + which@1, which@^1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a"