diff --git a/.travis.yml b/.travis.yml index 3fca86022..57d80e6be 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,5 +5,18 @@ cache: yarn: true directories: - node_modules +install: + - export ROCKSDB=$HOME/rocksdb-master + - pushd $HOME + - curl -sLO https://github.com/facebook/rocksdb/archive/master.zip + - unzip master.zip + - cd rocksdb-master + - make shared_lib + - popd + - export CFLAGS="-I${ROCKSDB}/include/" + - export CXXFLAGS="-I${ROCKSDB}/include/" + - export LDFLAGS="-L${ROCKSDB} -lrocksdb -lstdc++ -lm -lz" + - export LD_LIBRARY_PATH="${ROCKSDB}:/usr/local/lib" + - yarn script: - yarn polkadot-dev-build-travis diff --git a/package.json b/package.json index d3307a6d3..0261e0dd0 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,6 @@ }, "devDependencies": { "@polkadot/dev": "^0.20.18", - "@polkadot/ts": "^0.1.21" + "@polkadot/ts": "^0.1.22" } } diff --git a/packages/db/package.json b/packages/db/package.json index 2c2289be5..056621b0b 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -35,6 +35,7 @@ "lru_map": "^0.3.3", "mkdirp": "^0.5.1", "nosql-leveldb": "^3.0.1", + "rocksdb-node": "jacogr/rocksdb-node#00ebfdbf0462ee615f4f48f1db403fd4df8471ea", "snappy": "^6.0.4" } } diff --git a/packages/db/src/engines/LevelDb.spec.js b/packages/db/src/engines/LevelDb.spec.js index 2bc507362..b9a67cea0 100644 --- a/packages/db/src/engines/LevelDb.spec.js +++ b/packages/db/src/engines/LevelDb.spec.js @@ -9,7 +9,7 @@ import toU8a from '@polkadot/util/u8a/fromString'; import LevelDb from './LevelDb'; -describe('LevelDb', () => { +describe.skip('LevelDb', () => { const base = path.join(os.tmpdir(), '--test--'); const name = 'LevelDb'; const location = path.join(base, name); diff --git a/packages/db/src/engines/LevelDb.ts b/packages/db/src/engines/LevelDb.ts index 04d04f3ee..4202bc51d 100644 --- a/packages/db/src/engines/LevelDb.ts +++ b/packages/db/src/engines/LevelDb.ts @@ -7,10 +7,11 @@ import { BaseDb, ProgressCb } from '../types'; import leveldb, { LevelDb } from 'nosql-leveldb'; import mkdirp from 'mkdirp'; import path from 'path'; -import snappy from 'snappy'; +// import snappy from 'snappy'; import logger from '@polkadot/util/logger'; import bufferToU8a from '@polkadot/util/buffer/toU8a'; import u8aToBuffer from '@polkadot/util/u8a/toBuffer'; +import u8aToHex from '@polkadot/util/u8a/toHex'; // import assert from '@polkadot/util/assert'; // import u8aToHex from '@polkadot/util/u8a/toHex'; @@ -38,7 +39,6 @@ export default class Level implements BaseDb { l.debug(() => ['open']); this.db.openSync({ - compression: false, createIfMissing: true }); } @@ -79,19 +79,22 @@ export default class Level implements BaseDb { private _deserializeValue (value: Buffer): Uint8Array | null { return value - ? bufferToU8a( - snappy.uncompressSync(value) - ) + ? bufferToU8a(value) + // ? bufferToU8a( + // snappy.uncompressSync(value) + // ) : null; } - private _serializeKey (key: Uint8Array): Buffer { - return u8aToBuffer(key); + private _serializeKey (key: Uint8Array): string { + return u8aToHex(key); } private _serializeValue (value: Uint8Array): Buffer { - return snappy.compressSync( - u8aToBuffer(value) - ); + return u8aToBuffer(value); + + // return snappy.compressSync( + // u8aToBuffer(value) + // ); } } diff --git a/packages/db/src/engines/RocksDb.spec.js b/packages/db/src/engines/RocksDb.spec.js new file mode 100644 index 000000000..dcf902040 --- /dev/null +++ b/packages/db/src/engines/RocksDb.spec.js @@ -0,0 +1,64 @@ +// Copyright 2017-2018 @polkadot/db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +import mkdirp from 'mkdirp'; +import os from 'os'; +import path from 'path'; +import toU8a from '@polkadot/util/u8a/fromString'; + +import RocksDb from './RocksDb'; + +describe.skip('RocksDb', () => { + const base = path.join(os.tmpdir(), '--test--'); + const name = 'RocksDb'; + const location = path.join(base, name); + let db; + + beforeAll(() => { + db = new RocksDb(base, name); + db.open(); + }); + + afterAll(() => { + db.close(); + rimraf.sync(location); + }); + + it('sets & retrieves', () => { + const key = toU8a('key0'); + const value = toU8a('value0'); + + db.put(key, value); + + expect(db.get(key)).toEqual(value); + }); + + it('retrieves null for non-existing', () => { + const key = toU8a('key1'); + + expect(db.get(key)).toEqual(null); + }); + + it('overrides values', () => { + const key = toU8a('key2'); + const value = toU8a('value1'); + const replace = toU8a('value2'); + + db.put(key, value); + + expect(db.get(key)).toEqual(value); + + db.put(key, replace); + + expect(db.get(key)).toEqual(replace); + }); + + it('deletes values', () => { + const key = toU8a('key2'); + + db.del(key); + + expect(db.get(key)).toEqual(null); + }); +}); diff --git a/packages/db/src/engines/RocksDb.ts b/packages/db/src/engines/RocksDb.ts new file mode 100644 index 000000000..f9cf58c8e --- /dev/null +++ b/packages/db/src/engines/RocksDb.ts @@ -0,0 +1,98 @@ +// Copyright 2017-2018 @polkadot/db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +import { BaseDb, ProgressCb } from '../types'; + +import rocksdb, { RocksDb } from 'rocksdb-node'; +import mkdirp from 'mkdirp'; +import path from 'path'; +// import snappy from 'snappy'; +import logger from '@polkadot/util/logger'; +import bufferToU8a from '@polkadot/util/buffer/toU8a'; +import u8aToBuffer from '@polkadot/util/u8a/toBuffer'; +import u8aToHex from '@polkadot/util/u8a/toHex'; + +// import assert from '@polkadot/util/assert'; +// import u8aToHex from '@polkadot/util/u8a/toHex'; + +const l = logger('db/rocksdb'); + +export default class Rocks implements BaseDb { + private db: RocksDb; + + constructor (base: string, name: string) { + const location = path.join(base, name); + + mkdirp.sync(location); + + this.db = rocksdb.open({ create_if_missing: true }, location); + } + + close (): void { + l.debug(() => ['close']); + + this.db.close(); + } + + open (): void { + l.debug(() => ['open']); + } + + maintain (fn: ProgressCb): void { + fn({ + isCompleted: true, + keys: 0, + percent: 100 + }); + } + + del (key: Uint8Array): void { + this.db.del( + {}, + this._serializeKey(key) + ); + } + + get (key: Uint8Array): Uint8Array | null { + try { + return this._deserializeValue( + this.db.get( + { buffer: true }, + this._serializeKey(key) + ) as Buffer + ); + } catch (error) { + return null; + } + } + + put (key: Uint8Array, value: Uint8Array): void { + this.db.put( + {}, + this._serializeKey(key), + this._serializeValue(value) + ); + } + + private _deserializeValue (value: Buffer): Uint8Array | null { + return value + ? bufferToU8a(value) + // ? bufferToU8a( + // snappy.uncompressSync(value) + // ) + : null; + } + + private _serializeKey (key: Uint8Array): string { + return u8aToHex(key); + } + + private _serializeValue (value: Uint8Array): Buffer { + return u8aToBuffer(value); + + // return snappy.compressSync( + // u8aToBuffer(value) + // ); + } +} diff --git a/yarn.lock b/yarn.lock index cd93cf477..c18c05d75 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1285,9 +1285,9 @@ typedoc-plugin-markdown "^1.1.13" typescript "^3.0.1" -"@polkadot/ts@^0.1.21": - version "0.1.21" - resolved "https://registry.yarnpkg.com/@polkadot/ts/-/ts-0.1.21.tgz#8941ae51a6ab74281c0d782c7346f9b2fd1b0dd0" +"@polkadot/ts@^0.1.22": + version "0.1.22" + resolved "https://registry.yarnpkg.com/@polkadot/ts/-/ts-0.1.22.tgz#8f89d87257ea515bfc79685f5fda70b531a3107b" "@types/base-x@*": version "1.0.29" @@ -5328,14 +5328,14 @@ mute-stream@0.0.7, mute-stream@~0.0.4: version "0.0.7" resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.7.tgz#3075ce93bc21b8fab43e1bc4da7e8115ed1e7bab" +nan@2.11.0, nan@^2.10.0: + version "2.11.0" + resolved "https://registry.yarnpkg.com/nan/-/nan-2.11.0.tgz#574e360e4d954ab16966ec102c0c049fd961a099" + nan@^2.0.7, nan@^2.3.0: version "2.8.0" resolved "https://registry.yarnpkg.com/nan/-/nan-2.8.0.tgz#ed715f3fe9de02b57a5e6252d90a96675e1f085a" -nan@^2.10.0: - version "2.11.0" - resolved "https://registry.yarnpkg.com/nan/-/nan-2.11.0.tgz#574e360e4d954ab16966ec102c0c049fd961a099" - nan@^2.9.2: version "2.10.0" resolved "https://registry.yarnpkg.com/nan/-/nan-2.10.0.tgz#96d0cd610ebd58d4b4de9cc0c6828cda99c7548f" @@ -6596,6 +6596,12 @@ rimraf@2, rimraf@^2.2.8, rimraf@^2.5.1, rimraf@^2.5.4, rimraf@^2.6.1, rimraf@^2. dependencies: glob "^7.0.5" +rocksdb-node@jacogr/rocksdb-node#00ebfdbf0462ee615f4f48f1db403fd4df8471ea: + version "1.7.2" + resolved "https://codeload.github.com/jacogr/rocksdb-node/tar.gz/00ebfdbf0462ee615f4f48f1db403fd4df8471ea" + dependencies: + nan "2.11.0" + run-async@^2.2.0: version "2.3.0" resolved "https://registry.yarnpkg.com/run-async/-/run-async-2.3.0.tgz#0371ab4ae0bdd720d4166d7dfda64ff7a445a6c0"