Add db-diskdown (simple disk-based LRU KV map) (#193)

* Add db-diskdown (simple disk-based LRU KV map) * TODOs * Split path based on key mapping * Swap back to in-memory trie
2018-08-24 11:11:23 +02:00
parent 33a732f28c
commit 5f3928a78e
11 changed files with 315 additions and 21 deletions
--- a/jest.config.js
+++ b/jest.config.js
@@ -2,6 +2,7 @@ const config = require('@polkadot/dev/config/jest');

 module.exports = Object.assign({}, config, {
  moduleNameMapper: {
+    '@polkadot/db-(diskdown)(.*)$': '<rootDir>/packages/db-$1/src/$2',
    '@polkadot/trie-(db|hash)(.*)$': '<rootDir>/packages/trie-$1/src/$2',
    '@polkadot/util-(crypto|keyring|rlp)(.*)$': '<rootDir>/packages/util-$1/src/$2',
    '@polkadot/util(.*)$': '<rootDir>/packages/util/src/$1'
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,4 @@
 {
-  "lerna": "2.11.0",
  "npmClient": "yarn",
  "useWorkspaces": true,
  "command": {
--- a/package.json
+++ b/package.json
@@ -20,7 +20,7 @@
    "test": "jest --coverage"
  },
  "devDependencies": {
-    "@polkadot/dev": "^0.20.17",
-    "@polkadot/ts": "^0.1.13"
+    "@polkadot/dev": "^0.20.18",
+    "@polkadot/ts": "^0.1.17"
  }
 }
--- a/packages/db-diskdown/LICENSE
+++ b/packages/db-diskdown/LICENSE
@@ -0,0 +1,15 @@
+ISC License (ISC)
+
+Copyright 2017-2018 @polkadot/db-diskdown authors & contributors
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.
--- a/packages/db-diskdown/README.md
+++ b/packages/db-diskdown/README.md
@@ -0,0 +1,3 @@
+# @polkadot/db-diskdown
+
+Simple AbstractDOWN file-based key-value store implementation
--- a/packages/db-diskdown/package.json
+++ b/packages/db-diskdown/package.json
@@ -0,0 +1,38 @@
+{
+  "name": "@polkadot/db-diskdown",
+  "version": "0.28.4",
+  "description": "Simple file-base key-value storage system",
+  "main": "index.js",
+  "engines": {
+    "node": ">=8.0"
+  },
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/polkadot-js/client.git"
+  },
+  "keywords": [
+    "AbstractLevelDOWN"
+  ],
+  "author": "Jaco Greeff <jacogr@gmail.com>",
+  "maintainers": [
+    "Jaco Greeff <jacogr@gmail.com>"
+  ],
+  "contributors": [],
+  "license": "ISC",
+  "bugs": {
+    "url": "https://github.com/polkadot-js/client/issues"
+  },
+  "homepage": "https://github.com/polkadot-js/client/tree/master/packages/db-diskdown#readme",
+  "dependencies": {
+    "@babel/runtime": "^7.0.0-rc.1",
+    "@polkadot/util": "^0.28.4",
+    "@types/mkdirp": "^0.5.2",
+    "abstract-leveldown": "^5.0.0",
+    "lru_map": "^0.3.3",
+    "mkdirp": "^0.5.1"
+  }
+}
--- a/packages/db-diskdown/src/index.ts
+++ b/packages/db-diskdown/src/index.ts
@@ -0,0 +1,154 @@
+// Copyright 2017-2018 @polkadot/db-diskdown authors & contributors
+// This software may be modified and distributed under the terms
+// of the ISC license. See the LICENSE file for details.
+
+import { AbstractLevelDOWN } from 'abstract-leveldown';
+import fs from 'fs';
+import { LRUMap } from 'lru_map';
+import logger from '@polkadot/util/logger';
+import mkdirp from 'mkdirp';
+import isUndefined from '@polkadot/util/is/undefined';
+
+type FilePathEntry = {
+  exists: boolean,
+  path: string
+};
+
+type FilePath = {
+  directory: FilePathEntry,
+  file: FilePathEntry
+};
+
+const LRU_SIZE = 4096;
+
+const l = logger('db-diskdown');
+const noop = () =>
+  undefined;
+
+class DiskDown extends AbstractLevelDOWN {
+  location: string;
+  _store: LRUMap<string, Buffer>;
+
+  constructor (location: string) {
+    super(location);
+
+    this.location = location;
+    this._store = new LRUMap(LRU_SIZE);
+  }
+
+  __getFilePath (key: Buffer, doExistence: boolean): FilePath {
+    // NOTE We want to limit the number of entries in any specific directory. Split the
+    // key into parts and use this to construct the path and the actual filename. We want
+    // to limit the entries per directory, but at the same time minimize the number of
+    // directories we need to create (when non-existent)
+    const parts = key.toString('hex').match(/.{1,4}/g) || [];
+    const directoryPath = `${this.location}/${parts.slice(0, 4).join('/')}`;
+    const filePath = `${directoryPath}/${parts.slice(4).join('')}`;
+    let directoryExists = false;
+    let fileExists = true;
+
+    if (doExistence) {
+      fileExists = fs.existsSync(filePath);
+
+      if (fileExists) {
+        directoryExists = true;
+      } else {
+        directoryExists = fs.existsSync(directoryPath);
+      }
+    }
+
+    return {
+      directory: {
+        exists: directoryExists,
+        path: directoryPath
+      },
+      file: {
+        exists: fileExists,
+        path: filePath
+      }
+    };
+  }
+
+  _batch (array: Array<any>, options: any, callback: Function) {
+    l.debug(() => ['_batch', array]);
+
+    array.forEach(({ key, type, value }) => {
+      switch (type) {
+        case 'del':
+          return this._del(key, {}, noop);
+
+        case 'put':
+          return this._put(key, value, {}, noop);
+
+        default:
+          // ignore
+      }
+    });
+
+    process.nextTick(callback);
+  }
+
+  _del (key: Buffer, options: any, callback: Function) {
+    l.debug(() => ['_del', key]);
+
+    const filePath = this.__getFilePath(key, true);
+
+    this._store.delete(key.toString());
+
+    if (filePath.file.exists) {
+      fs.unlinkSync(filePath.file.path);
+    }
+
+    process.nextTick(callback);
+  }
+
+  _get (key: Buffer, options: any, callback: Function) {
+    l.debug(() => ['_get', key.toString('hex')]);
+
+    const filePath = this.__getFilePath(key, true);
+    let value = this._store.get(key.toString());
+    const returnValue = () =>
+      process.nextTick(callback, null, value);
+
+    if (!isUndefined(value)) {
+      return returnValue();
+    }
+
+    if (filePath.file.exists) {
+      value = fs.readFileSync(filePath.file.path);
+
+      return returnValue();
+    }
+
+    // 'NotFound' error, consistent with LevelDOWN API
+    process.nextTick(callback, new Error('NotFound'));
+  }
+
+  _open (options: any, callback: Function) {
+    l.debug(() => ['_open', options]);
+
+    this._store.clear();
+
+    process.nextTick(callback, null, this);
+  }
+
+  _put (key: Buffer, value: Buffer, options: any, callback: Function) {
+    l.debug(() => ['_put', key.toString('hex'), value]);
+
+    const filePath = this.__getFilePath(key, false);
+
+    this._store.set(key.toString(), value);
+
+    if (!filePath.directory.exists) {
+      mkdirp.sync(filePath.directory.path);
+    }
+
+    fs.writeFileSync(filePath.file.path, value);
+
+    process.nextTick(callback);
+  }
+}
+
+export default function (location: string) {
+  return new DiskDown(location);
+}
--- a/packages/db-diskdown/src/types.d.ts
+++ b/packages/db-diskdown/src/types.d.ts
@@ -0,0 +1,3 @@
+// Copyright 2017-2018 @polkadot/db-diskdown authors & contributors
+// This software may be modified and distributed under the terms
+// of the ISC license. See the LICENSE file for details.
--- a/packages/trie-db/test/checkpoints.spec.js
+++ b/packages/trie-db/test/checkpoints.spec.js
@@ -3,14 +3,21 @@
 // This software may be modified and distributed under the terms
 // of the MPL-2.0 license. See the LICENSE file for details.

+import mkdirp from 'mkdirp';
 import toU8a from '@polkadot/util/u8a/toU8a';
+// import diskdown from '@polkadot/db-diskdown/index';

 import Trie from '../src/index';

+// const DISKPATH = `${process.cwd()}/--test--db--`;
+
+// mkdirp(DISKPATH);
+
 describe('testing checkpoints', () => {
  let trie, preRoot, postRoot;

  it('sets up the trie', async () => {
+    // trie = new Trie(diskdown(DISKPATH));
    trie = new Trie();

    await trie.put(toU8a('do'), toU8a('verb'));
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -3,7 +3,8 @@
  "compilerOptions": {
    "baseUrl": ".",
    "paths": {
-      "@polkadot/trie-db/*": [ "packages/trie-sb/src/*" ],
+      "@polkadot/db-diskdown/*": [ "packages/db-diskdown/src/*" ],
+      "@polkadot/trie-db/*": [ "packages/trie-db/src/*" ],
      "@polkadot/trie-hash/*": [ "packages/trie-hash/src/*" ],
      "@polkadot/util-crypto/*": [ "packages/util-crypto/src/*" ],
      "@polkadot/util-keyring/*": [ "packages/util-keyring/src/*" ],
--- a/yarn.lock
+++ b/yarn.lock
@@ -1239,9 +1239,9 @@
  version "1.1.0"
  resolved "https://registry.yarnpkg.com/@nodelib/fs.stat/-/fs.stat-1.1.0.tgz#50c1e2260ac0ed9439a181de3725a0168d59c48a"

-"@polkadot/dev@^0.20.17":
-  version "0.20.17"
-  resolved "https://registry.yarnpkg.com/@polkadot/dev/-/dev-0.20.17.tgz#30c89e0c2c03402eca3a4da8bb02e3d499849c96"
+"@polkadot/dev@^0.20.18":
+  version "0.20.18"
+  resolved "https://registry.yarnpkg.com/@polkadot/dev/-/dev-0.20.18.tgz#fb08df68a53f47cebdd0cea9366c0714b40879a6"
  dependencies:
    "@babel/cli" "^7.0.0-rc.1"
    "@babel/core" "^7.0.0-rc.1"
@@ -1285,9 +1285,9 @@
    typedoc-plugin-markdown "^1.1.13"
    typescript "^3.0.1"

-"@polkadot/ts@^0.1.13":
-  version "0.1.13"
-  resolved "https://registry.yarnpkg.com/@polkadot/ts/-/ts-0.1.13.tgz#31540153a5381f1442e3f24f615eefd7442dbeac"
+"@polkadot/ts@^0.1.17":
+  version "0.1.17"
+  resolved "https://registry.yarnpkg.com/@polkadot/ts/-/ts-0.1.17.tgz#0d85085038ae7e6757e059d07c8e9c54596919b3"

 "@types/async@^2.0.49":
  version "2.0.49"
@@ -1359,6 +1359,12 @@
  version "3.0.3"
  resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d"

+"@types/mkdirp@^0.5.2":
+  version "0.5.2"
+  resolved "https://registry.yarnpkg.com/@types/mkdirp/-/mkdirp-0.5.2.tgz#503aacfe5cc2703d5484326b1b27efa67a339c1f"
+  dependencies:
+    "@types/node" "*"
+
 "@types/node@*":
  version "8.5.2"
  resolved "https://registry.yarnpkg.com/@types/node/-/node-8.5.2.tgz#83b8103fa9a2c2e83d78f701a9aa7c9539739aa5"
@@ -1681,7 +1687,31 @@ babel-code-frame@^6.22.0, babel-code-frame@^6.26.0:
    esutils "^2.0.2"
    js-tokens "^3.0.2"

-babel-core@^6.0.0, babel-core@^7.0.0-bridge.0:
+babel-core@^6.0.0, babel-core@^6.26.0:
+  version "6.26.3"
+  resolved "https://registry.yarnpkg.com/babel-core/-/babel-core-6.26.3.tgz#b2e2f09e342d0f0c88e2f02e067794125e75c207"
+  dependencies:
+    babel-code-frame "^6.26.0"
+    babel-generator "^6.26.0"
+    babel-helpers "^6.24.1"
+    babel-messages "^6.23.0"
+    babel-register "^6.26.0"
+    babel-runtime "^6.26.0"
+    babel-template "^6.26.0"
+    babel-traverse "^6.26.0"
+    babel-types "^6.26.0"
+    babylon "^6.18.0"
+    convert-source-map "^1.5.1"
+    debug "^2.6.9"
+    json5 "^0.5.1"
+    lodash "^4.17.4"
+    minimatch "^3.0.4"
+    path-is-absolute "^1.0.1"
+    private "^0.1.8"
+    slash "^1.0.0"
+    source-map "^0.5.7"
+
+babel-core@^7.0.0-bridge.0:
  version "7.0.0-bridge.0"
  resolved "https://registry.yarnpkg.com/babel-core/-/babel-core-7.0.0-bridge.0.tgz#95a492ddd90f9b4e9a4a1da14eb335b87b634ece"

@@ -1709,6 +1739,26 @@ babel-generator@^6.18.0:
    source-map "^0.5.6"
    trim-right "^1.0.1"

+babel-generator@^6.26.0:
+  version "6.26.1"
+  resolved "https://registry.yarnpkg.com/babel-generator/-/babel-generator-6.26.1.tgz#1844408d3b8f0d35a404ea7ac180f087a601bd90"
+  dependencies:
+    babel-messages "^6.23.0"
+    babel-runtime "^6.26.0"
+    babel-types "^6.26.0"
+    detect-indent "^4.0.0"
+    jsesc "^1.3.0"
+    lodash "^4.17.4"
+    source-map "^0.5.7"
+    trim-right "^1.0.1"
+
+babel-helpers@^6.24.1:
+  version "6.24.1"
+  resolved "https://registry.yarnpkg.com/babel-helpers/-/babel-helpers-6.24.1.tgz#3471de9caec388e5c850e597e58a26ddf37602b2"
+  dependencies:
+    babel-runtime "^6.22.0"
+    babel-template "^6.24.1"
+
 babel-jest@^23.4.2:
  version "23.4.2"
  resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-23.4.2.tgz#f276de67798a5d68f2d6e87ff518c2f6e1609877"
@@ -1746,6 +1796,18 @@ babel-preset-jest@^23.2.0:
    babel-plugin-jest-hoist "^23.2.0"
    babel-plugin-syntax-object-rest-spread "^6.13.0"

+babel-register@^6.26.0:
+  version "6.26.0"
+  resolved "https://registry.yarnpkg.com/babel-register/-/babel-register-6.26.0.tgz#6ed021173e2fcb486d7acb45c6009a856f647071"
+  dependencies:
+    babel-core "^6.26.0"
+    babel-runtime "^6.26.0"
+    core-js "^2.5.0"
+    home-or-tmp "^2.0.0"
+    lodash "^4.17.4"
+    mkdirp "^0.5.1"
+    source-map-support "^0.4.15"
+
 babel-runtime@^6.22.0, babel-runtime@^6.26.0:
  version "6.26.0"
  resolved "https://registry.yarnpkg.com/babel-runtime/-/babel-runtime-6.26.0.tgz#965c7058668e82b55d7bfe04ff2337bc8b5647fe"
@@ -1753,7 +1815,7 @@ babel-runtime@^6.22.0, babel-runtime@^6.26.0:
    core-js "^2.4.0"
    regenerator-runtime "^0.11.0"

-babel-template@^6.16.0:
+babel-template@^6.16.0, babel-template@^6.24.1, babel-template@^6.26.0:
  version "6.26.0"
  resolved "https://registry.yarnpkg.com/babel-template/-/babel-template-6.26.0.tgz#de03e2d16396b069f46dd9fff8521fb1a0e35e02"
  dependencies:
@@ -2350,7 +2412,7 @@ conventional-recommended-bump@^2.0.6:
    meow "^4.0.0"
    q "^1.5.1"

-convert-source-map@^1.1.0, convert-source-map@^1.4.0:
+convert-source-map@^1.1.0, convert-source-map@^1.4.0, convert-source-map@^1.5.1:
  version "1.5.1"
  resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-1.5.1.tgz#b8278097b9bc229365de5c62cf5fcaed8b5599e5"

@@ -2384,7 +2446,7 @@ core-js@^2.4.0:
  version "2.5.1"
  resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.5.1.tgz#ae6874dc66937789b80754ff5428df66819ca50b"

-core-js@^2.5.7:
+core-js@^2.5.0, core-js@^2.5.7:
  version "2.5.7"
  resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.5.7.tgz#f972608ff0cead68b841a16a932d0b183791814e"

@@ -2507,7 +2569,7 @@ debug@3.1.0, debug@^3.1.0:
  dependencies:
    ms "2.0.0"

-debug@^2.1.2, debug@^2.2.0, debug@^2.3.3, debug@^2.6.8:
+debug@^2.1.2, debug@^2.2.0, debug@^2.3.3, debug@^2.6.8, debug@^2.6.9:
  version "2.6.9"
  resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
  dependencies:
@@ -3689,6 +3751,13 @@ hoek@4.x.x:
  version "4.2.0"
  resolved "https://registry.yarnpkg.com/hoek/-/hoek-4.2.0.tgz#72d9d0754f7fe25ca2d01ad8f8f9a9449a89526d"

+home-or-tmp@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/home-or-tmp/-/home-or-tmp-2.0.0.tgz#e36c3f2d2cae7d746a857e38d18d5f32a7882db8"
+  dependencies:
+    os-homedir "^1.0.0"
+    os-tmpdir "^1.0.1"
+
 home-or-tmp@^3.0.0:
  version "3.0.0"
  resolved "https://registry.yarnpkg.com/home-or-tmp/-/home-or-tmp-3.0.0.tgz#57a8fe24cf33cdd524860a15821ddc25c86671fb"
@@ -4668,7 +4737,7 @@ json-stringify-safe@^5.0.1, json-stringify-safe@~5.0.1:
  version "5.0.1"
  resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb"

-json5@^0.5.0:
+json5@^0.5.0, json5@^0.5.1:
  version "0.5.1"
  resolved "https://registry.yarnpkg.com/json5/-/json5-0.5.1.tgz#1eade7acc012034ad84e2396767ead9fa5495821"

@@ -4914,6 +4983,10 @@ lru-cache@^4.1.2, lru-cache@^4.1.3:
    pseudomap "^1.0.2"
    yallist "^2.1.2"

+lru_map@^0.3.3:
+  version "0.3.3"
+  resolved "https://registry.yarnpkg.com/lru_map/-/lru_map-0.3.3.tgz#b5c8351b9464cbd750335a79650a0ec0e56118dd"
+
 ltgt@~2.2.0:
  version "2.2.1"
  resolved "https://registry.yarnpkg.com/ltgt/-/ltgt-2.2.1.tgz#f35ca91c493f7b73da0e07495304f17b31f87ee5"
@@ -5568,7 +5641,7 @@ os-locale@^2.0.0:
    lcid "^1.0.0"
    mem "^1.1.0"

-os-tmpdir@^1.0.0, os-tmpdir@~1.0.2:
+os-tmpdir@^1.0.0, os-tmpdir@^1.0.1, os-tmpdir@~1.0.2:
  version "1.0.2"
  resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274"

@@ -5738,7 +5811,7 @@ path-exists@^3.0.0:
  version "3.0.0"
  resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-3.0.0.tgz#ce0ebeaa5f78cb18925ea7d810d7b59b010fd515"

-path-is-absolute@^1.0.0:
+path-is-absolute@^1.0.0, path-is-absolute@^1.0.1:
  version "1.0.1"
  resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f"

@@ -5849,7 +5922,7 @@ pretty-format@^23.5.0:
    ansi-regex "^3.0.0"
    ansi-styles "^3.2.0"

-private@^0.1.6:
+private@^0.1.6, private@^0.1.8:
  version "0.1.8"
  resolved "https://registry.yarnpkg.com/private/-/private-0.1.8.tgz#2381edb3689f7a53d653190060fcf822d2f368ff"

@@ -6650,7 +6723,7 @@ source-map-resolve@^0.5.0:
    source-map-url "^0.4.0"
    urix "^0.1.0"

-source-map-support@^0.4.2:
+source-map-support@^0.4.15, source-map-support@^0.4.2:
  version "0.4.18"
  resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.4.18.tgz#0286a6de8be42641338594e97ccea75f0a2c585f"
  dependencies:
@@ -6673,7 +6746,7 @@ source-map@^0.4.4:
  dependencies:
    amdefine ">=0.0.4"

-source-map@^0.5.0, source-map@^0.5.3, source-map@^0.5.6, source-map@~0.5.1, source-map@~0.5.6:
+source-map@^0.5.0, source-map@^0.5.3, source-map@^0.5.6, source-map@^0.5.7, source-map@~0.5.1, source-map@~0.5.6:
  version "0.5.7"
  resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.7.tgz#8a039d2d1021d22d1ea14c80d8ea468ba2ef3fcc"