From 7ff9ff8b34b35e2c14f10b63486fe68eec6e4c30 Mon Sep 17 00:00:00 2001 From: Jaco Greeff Date: Wed, 29 Aug 2018 16:11:04 +0200 Subject: [PATCH] Revert "Revert "diskdown to overwrite values when changed (if space available) (#202)"" This reverts commit bd0eb4cb342b5dd5753e6a9c9b848b89bc719bbb. --- packages/db-diskdown/package.json | 1 - packages/db-diskdown/src/index.ts | 9 --------- packages/db-diskdown/src/store/Combined.ts | 21 ++++++++++++++++----- yarn.lock | 4 ---- 4 files changed, 16 insertions(+), 19 deletions(-) diff --git a/packages/db-diskdown/package.json b/packages/db-diskdown/package.json index b0206cf34..782637538 100644 --- a/packages/db-diskdown/package.json +++ b/packages/db-diskdown/package.json @@ -32,7 +32,6 @@ "@polkadot/util": "^0.28.23", "@types/mkdirp": "^0.5.2", "abstract-leveldown": "^5.0.0", - "lru_map": "^0.3.3", "mkdirp": "^0.5.1" }, "devDependencies": { diff --git a/packages/db-diskdown/src/index.ts b/packages/db-diskdown/src/index.ts index d8fd37c9a..b56d3f8f7 100644 --- a/packages/db-diskdown/src/index.ts +++ b/packages/db-diskdown/src/index.ts @@ -5,13 +5,11 @@ import { DiskStore, ProgressValue } from './types'; import { AbstractLevelDOWN } from 'abstract-leveldown'; -// import { LRUMap } from 'lru_map'; import isUndefined from '@polkadot/util/is/undefined'; import logger from '@polkadot/util/logger'; import Combined from './store/Combined'; -// const LRU_SIZE = 16384; const KEY_LENGTH = 32; const l = logger('disk/scatter'); @@ -20,13 +18,11 @@ const noop = () => class DiskDown extends AbstractLevelDOWN { _disk: DiskStore; - // _store: LRUMap; constructor (location: string) { super(location); this._disk = new Combined(location); - // this._store = new LRUMap(LRU_SIZE); } compact (progress: (value: ProgressValue) => void): void { @@ -73,7 +69,6 @@ class DiskDown extends AbstractLevelDOWN { _del (key: Buffer, options: any, callback: Function) { l.debug(() => ['_del', key]); - // this._store.delete(key.toString()); this._disk.delete(key); process.nextTick(callback); @@ -82,9 +77,6 @@ class DiskDown extends AbstractLevelDOWN { _get (key: Buffer, options: any, callback: Function) { l.debug(() => ['_get', key.toString('hex')]); - // FIXME _If_ LRU is re-instated, it needs to get set once a value is - // retrieved from disk. (As of now, not needed since removed) - // const value = this._store.get(key.toString()) || this._disk.get(key); const value = this._disk.get(key); if (isUndefined(value)) { @@ -98,7 +90,6 @@ class DiskDown extends AbstractLevelDOWN { _put (key: Buffer, value: Buffer, options: any, callback: Function) { l.debug(() => ['_put', key.toString('hex'), value]); - // this._store.set(key.toString(), value); this._disk.set(key, value); process.nextTick(callback); diff --git a/packages/db-diskdown/src/store/Combined.ts b/packages/db-diskdown/src/store/Combined.ts index 743a95109..c5eae8f33 100644 --- a/packages/db-diskdown/src/store/Combined.ts +++ b/packages/db-diskdown/src/store/Combined.ts @@ -320,14 +320,20 @@ export default class Combined implements DiskStore { return result; } + private _extractValueInfo (keyValue: Buffer): { valueAt: number, valueLength: number } { + return { + valueLength: keyValue.readUIntBE(KEY_SIZE, UINT_SIZE), + valueAt: keyValue.readUIntBE(KEY_SIZE + UINT_SIZE, UINT_SIZE) + }; + } + private _readValue (key: Buffer, keyAt: number, keyValue: Buffer): Value { l.debug(() => ['readValue', debug({ key, keyAt, keyValue })]); - const length = keyValue.readUIntBE(KEY_SIZE, UINT_SIZE); - const valueAt = keyValue.readUIntBE(KEY_SIZE + UINT_SIZE, UINT_SIZE); + const { valueAt, valueLength } = this._extractValueInfo(keyValue); const value = Buffer.alloc(length); - fs.readSync(this._fd, value, 0, length, valueAt); + fs.readSync(this._fd, value, 0, valueLength, valueAt); return { key, @@ -349,8 +355,13 @@ export default class Combined implements DiskStore { private _writeValue (key: Buffer, keyAt: number, keyValue: Buffer, value: Buffer): Value { l.debug(() => ['writeValue', debug({ keyAt, keyValue, value })]); - const stats = fs.fstatSync(this._fd); - const valueAt = stats.size; + let { valueAt, valueLength } = this._extractValueInfo(keyValue); + + if (valueLength < value.length) { + const stats = fs.fstatSync(this._fd); + + valueAt = stats.size; + } fs.writeSync(this._fd, value, 0, value.length, valueAt); diff --git a/yarn.lock b/yarn.lock index 0e42038a0..208c6860d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4920,10 +4920,6 @@ lru-cache@^4.1.2, lru-cache@^4.1.3: pseudomap "^1.0.2" yallist "^2.1.2" -lru_map@^0.3.3: - version "0.3.3" - resolved "https://registry.yarnpkg.com/lru_map/-/lru_map-0.3.3.tgz#b5c8351b9464cbd750335a79650a0ec0e56118dd" - ltgt@~2.2.0: version "2.2.1" resolved "https://registry.yarnpkg.com/ltgt/-/ltgt-2.2.1.tgz#f35ca91c493f7b73da0e07495304f17b31f87ee5"