diff --git a/packages/db-diskdown/package.json b/packages/db-diskdown/package.json index 88dba0a22..6350cfbbf 100644 --- a/packages/db-diskdown/package.json +++ b/packages/db-diskdown/package.json @@ -32,6 +32,7 @@ "@polkadot/util": "^0.28.17", "@types/mkdirp": "^0.5.2", "abstract-leveldown": "^5.0.0", + "lru_map": "^0.3.3", "mkdirp": "^0.5.1" }, "devDependencies": { diff --git a/packages/db-diskdown/src/index.ts b/packages/db-diskdown/src/index.ts index a59584cb5..d1fcdbb5d 100644 --- a/packages/db-diskdown/src/index.ts +++ b/packages/db-diskdown/src/index.ts @@ -5,11 +5,13 @@ import { DiskStore } from './types'; import { AbstractLevelDOWN } from 'abstract-leveldown'; +import { LRUMap } from 'lru_map'; import isUndefined from '@polkadot/util/is/undefined'; import logger from '@polkadot/util/logger'; import Combined from './store/Combined'; +const LRU_SIZE = 16384; const KEY_LENGTH = 32; const l = logger('disk/scatter'); @@ -18,11 +20,13 @@ const noop = () => class DiskDown extends AbstractLevelDOWN { _disk: DiskStore; + _store: LRUMap; constructor (location: string) { super(location); this._disk = new Combined(location); + this._store = new LRUMap(LRU_SIZE); } compact (progress: (message: string) => void): void { @@ -69,6 +73,7 @@ class DiskDown extends AbstractLevelDOWN { _del (key: Buffer, options: any, callback: Function) { l.debug(() => ['_del', key]); + this._store.delete(key.toString()); this._disk.delete(key); process.nextTick(callback); @@ -77,7 +82,7 @@ class DiskDown extends AbstractLevelDOWN { _get (key: Buffer, options: any, callback: Function) { l.debug(() => ['_get', key.toString('hex')]); - const value = this._disk.get(key); + const value = this._store.get(key.toString()) || this._disk.get(key); if (isUndefined(value)) { process.nextTick(callback, new Error('NotFound')); @@ -90,6 +95,7 @@ class DiskDown extends AbstractLevelDOWN { _put (key: Buffer, value: Buffer, options: any, callback: Function) { l.debug(() => ['_put', key.toString('hex'), value]); + this._store.set(key.toString(), value); this._disk.set(key, value); process.nextTick(callback); diff --git a/packages/db-diskdown/src/store/Combined.ts b/packages/db-diskdown/src/store/Combined.ts index 26626d8c9..93a5aa3dd 100644 --- a/packages/db-diskdown/src/store/Combined.ts +++ b/packages/db-diskdown/src/store/Combined.ts @@ -101,14 +101,14 @@ export default class Combined implements DiskStore { const newStat = fs.lstatSync(newFile); const oldStat = fs.lstatSync(this._file); const percentage = 100 * (newStat.size / oldStat.size); - const sizeMb = newStat.size / (1024 * 1024); + const sizeMB = newStat.size / (1024 * 1024); const elapsed = (Date.now() - start) / 1000; fs.unlinkSync(this._file); // fs.renameSync(this._file, `${this._file}.old`); fs.renameSync(newFile, this._file); - l.log(`compacted in ${elapsed.toFixed(2)}s, ${(count / 1000).toFixed(2)}k keys, ${sizeMb.toFixed(2)}MB (${percentage.toFixed(2)}%)`); + l.log(`compacted in ${elapsed.toFixed(2)}s, ${(count / 1000).toFixed(2)}k keys, ${sizeMB.toFixed(2)}MB (${percentage.toFixed(2)}%)`); } delete (key: Buffer): void { diff --git a/yarn.lock b/yarn.lock index 208c6860d..0e42038a0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4920,6 +4920,10 @@ lru-cache@^4.1.2, lru-cache@^4.1.3: pseudomap "^1.0.2" yallist "^2.1.2" +lru_map@^0.3.3: + version "0.3.3" + resolved "https://registry.yarnpkg.com/lru_map/-/lru_map-0.3.3.tgz#b5c8351b9464cbd750335a79650a0ec0e56118dd" + ltgt@~2.2.0: version "2.2.1" resolved "https://registry.yarnpkg.com/ltgt/-/ltgt-2.2.1.tgz#f35ca91c493f7b73da0e07495304f17b31f87ee5"