Re-add RocksDb interface (#356)

* Re-add RocksDb (still in testing)

* Add bytes_per_sync option

* Move tests (unused)

* Move tests

.travis.yml

@@ -3,21 +3,22 @@ node_js:
   - "10"
 env:
   - GH_PAGES_SRC="build-docs"
-# compiler:
-#   - clang
-#   - gcc
-# addons:
-#   apt:
-#     sources:
-#       - ubuntu-toolchain-r-test
-#     packages:
-#       - curl
-#       - g++-8
-#       - libbz2-dev
-#       - libgflags-dev
-#       - libsnappy-dev
-#       - mingw-w64
-#       - zlib1g-dev
+# compiler & addons are both used for RocksDb
+compiler:
+  - clang
+  - gcc
+addons:
+  apt:
+    sources:
+      - ubuntu-toolchain-r-test
+    packages:
+      - curl
+      - g++-8
+      - libbz2-dev
+      - libgflags-dev
+      - libsnappy-dev
+      - mingw-w64
+      - zlib1g-dev
 before_install:
   - curl -o- -L https://yarnpkg.com/install.sh | bash
   - export PATH=$HOME/.yarn/bin:$(yarn global bin):$PATH
@@ -26,17 +27,17 @@ cache:
   directories:
     - node_modules
 install:
-  # - export ROCKSDB=$HOME/rocksdb-master
-  # - pushd $HOME
-  # - curl -sLO https://github.com/facebook/rocksdb/archive/master.zip
-  # - unzip -q master.zip
-  # - cd rocksdb-master
-  # - make shared_lib
-  # - popd
-  # - export CFLAGS="-I${ROCKSDB}/include/"
-  # - export CXXFLAGS="-I${ROCKSDB}/include/"
-  # - export LDFLAGS="-L${ROCKSDB} -lrocksdb -lstdc++ -lm -lz"
-  # - export LD_LIBRARY_PATH="${ROCKSDB}:/usr/local/lib"
+  - export ROCKSDB=$HOME/rocksdb-master
+  - pushd $HOME
+  - curl -sLO https://github.com/facebook/rocksdb/archive/master.zip
+  - unzip -q master.zip
+  - cd rocksdb-master
+  - make shared_lib
+  - popd
+  - export CFLAGS="-I${ROCKSDB}/include/"
+  - export CXXFLAGS="-I${ROCKSDB}/include/"
+  - export LDFLAGS="-L${ROCKSDB} -lrocksdb -lstdc++ -lm -lz"
+  - export LD_LIBRARY_PATH="${ROCKSDB}:/usr/local/lib"
   - yarn
 script:
   - yarn polkadot-dev-build-travis

packages/db/package.json

@@ -37,6 +37,7 @@
     "lru_map": "^0.3.3",
     "mkdirp": "^0.5.1",
     "node-lmdb": "^0.6.2",
+    "rocksdb-node": "jacogr/rocksdb-node#00ebfdbf0462ee615f4f48f1db403fd4df8471ea",
     "snappy": "^6.1.2"
   }
 }

packages/db/src/engines/RocksDb.ts

@@ -18,6 +18,13 @@ import { bufferToU8a, logger, u8aToBuffer, u8aToHex } from '@polkadot/util/index
 
 const l = logger('db/rocksdb');
 
+const OPEN_OPTIONS = {
+  allow_mmap_reads: true,
+  allow_mmap_writes: true,
+  bytes_per_sync: 1 * 1024 * 1024, // 1MB default
+  create_if_missing: true
+};
+
 export default class Rocks implements BaseDb {
   private db: RocksDb;
 
@@ -26,7 +33,7 @@ export default class Rocks implements BaseDb {
 
     mkdirp.sync(location);
 
-    this.db = rocksdb.open({ create_if_missing: true }, location);
+    this.db = rocksdb.open(OPEN_OPTIONS, location);
   }
 
   close (): void {

packages/db/src/engines/unused/RocksDb.spec.ts-unused

@@ -7,7 +7,7 @@ import path from 'path';
 import rimraf from 'rimraf';
 import { stringToU8a } from '@polkadot/util/index';
 
-import RocksDb from './RocksDb';
+import RocksDb from '../RocksDb';
 
 describe.skip('RocksDb', () => {
   const base = path.join(os.tmpdir(), '--test--');

yarn.lock

@@ -7546,6 +7546,11 @@ mute-stream@~0.0.4:
   resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.8.tgz#1630c42b2251ff81e2a283de96a5497ea92e5e0d"
   integrity sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==
 
+nan@2.11.0:
+  version "2.11.0"
+  resolved "https://registry.yarnpkg.com/nan/-/nan-2.11.0.tgz#574e360e4d954ab16966ec102c0c049fd961a099"
+  integrity sha512-F4miItu2rGnV2ySkXOQoA8FKz/SR2Q2sWP0sbTxNxz/tuokeC8WxOhPMcwi0qIyGtVn/rrSeLbvVkznqCdwYnw==
+
 nan@^2.12.0, nan@^2.12.1, nan@^2.2.1, nan@^2.9.2:
   version "2.12.1"
   resolved "https://registry.yarnpkg.com/nan/-/nan-2.12.1.tgz#7b1aa193e9aa86057e3c7bbd0ac448e770925552"
@@ -9538,6 +9543,12 @@ ripemd160@^2.0.0, ripemd160@^2.0.1:
     hash-base "^3.0.0"
     inherits "^2.0.1"
 
+rocksdb-node@jacogr/rocksdb-node#00ebfdbf0462ee615f4f48f1db403fd4df8471ea:
+  version "1.7.2"
+  resolved "https://codeload.github.com/jacogr/rocksdb-node/tar.gz/00ebfdbf0462ee615f4f48f1db403fd4df8471ea"
+  dependencies:
+    nan "2.11.0"
+
 rsvp@^3.3.3:
   version "3.6.2"
   resolved "https://registry.yarnpkg.com/rsvp/-/rsvp-3.6.2.tgz#2e96491599a96cde1b515d5674a8f7a91452926a"