FlatDb code refactor & optimisations (#213)

* Move Compactor into own class

* Combine multiple buffer writes

* Skip failing CI test (again)

* Clean up value > current.length

* Test code linting

package.json

@@ -21,6 +21,6 @@
   },
   "devDependencies": {
     "@polkadot/dev": "^0.20.18",
-    "@polkadot/ts": "^0.1.23"
+    "@polkadot/ts": "^0.1.24"
   }
 }

packages/db/src/Disk.ts

@@ -4,7 +4,7 @@
 
 import { BaseDbOptions } from './types';
 
-import FileFlatDb from './engines/FileFlatDb';
+import FileFlatDb from './FileFlatDb';
 import LruDb from './engines/LruDb';
 import TransactionDb from './engines/TransactionDb';
 

packages/db/src/FileFlatDb/Compact.spec.js

@@ -0,0 +1,20 @@
+// Copyright 2017-2018 @polkadot/db authors & contributors
+// This software may be modified and distributed under the terms
+// of the ISC license. See the LICENSE file for details.
+
+import mkdirp from 'mkdirp';
+import os from 'os';
+import path from 'path';
+import rimraf from 'rimraf';
+
+import FileFlatDb from './index';
+
+describe.skip('FileFlatDb (compacting)', () => {
+  const combined = new FileFlatDb(process.cwd());
+
+  it('compacts without failing', () => {
+    expect(
+      combined.maintain(() => {})
+    ).not.toBe(0);
+  });
+});

packages/db/src/FileFlatDb/Compact.ts

@@ -0,0 +1,165 @@
+// Copyright 2017-2018 @polkadot/db authors & contributors
+// This software may be modified and distributed under the terms
+// of the ISC license. See the LICENSE file for details.
+
+import { ProgressCb } from '../types';
+import { Slot } from './types';
+
+import fs from 'fs';
+import assert from '@polkadot/util/assert';
+import logger from '@polkadot/util/logger';
+
+import defaults from './defaults';
+
+const l = logger('db/compact');
+
+export default class Compact {
+  private _fd: number;
+  private _file: string;
+
+  constructor (file: string) {
+    this._fd = -1;
+    this._file = file;
+  }
+
+  maintain (fn: ProgressCb): void {
+    assert(this._fd === -1, 'Database cannot be open for compacting');
+
+    l.log('compacting database');
+
+    const start = Date.now();
+    const newFile = `${this._file}.compacted`;
+    const newFd = this._open(newFile, true);
+    const oldFd = this._open(this._file);
+    const keys = this._compact(fn, newFd, oldFd);
+
+    fs.closeSync(oldFd);
+    fs.closeSync(newFd);
+
+    const newStat = fs.lstatSync(newFile);
+    const oldStat = fs.lstatSync(this._file);
+    const percentage = 100 * (newStat.size / oldStat.size);
+    const sizeMB = newStat.size / (1024 * 1024);
+    const elapsed = (Date.now() - start) / 1000;
+
+    fs.unlinkSync(this._file);
+    fs.renameSync(newFile, this._file);
+
+    l.log(`compacted in ${elapsed.toFixed(2)}s, ${(keys / 1000).toFixed(2)}k keys, ${sizeMB.toFixed(2)}MB (${percentage.toFixed(2)}%)`);
+  }
+
+  private _open (file: string, startEmpty: boolean = false): number {
+    if (!fs.existsSync(file) || startEmpty) {
+      fs.writeFileSync(file, Buffer.alloc(defaults.BRANCH_SIZE));
+    }
+
+    return fs.openSync(file, 'a+');
+  }
+
+  private _compact (fn: ProgressCb, newFd: number, oldFd: number): number {
+    // l.debug(() => ['_compact', debug({ newFd, oldFd, newAt, oldAt })]);
+
+    let keys = 0;
+    let percent = 0;
+
+    const doCompact = (newAt: number, oldAt: number, depth: number) => {
+      const increment = (100 / defaults.ENTRY_NUM) / Math.pow(defaults.ENTRY_NUM, depth);
+
+      for (let index = 0; index < defaults.ENTRY_NUM; index++) {
+        const entry = this._compactReadEntry(oldFd, oldAt, index);
+        const dataAt = entry.readUIntBE(1, defaults.UINT_SIZE);
+        const entryType = entry[0];
+
+        if (entryType === Slot.EMPTY) {
+          // l.debug(() => '_compact/isEmpty');
+          percent += increment;
+        } else if (entryType === Slot.LEAF) {
+          // l.debug(() => '_compact/isLeaf');
+
+          const [key, value] = this._compactReadKey(oldFd, dataAt);
+          const keyAt = this._compactWriteKey(newFd, key, value);
+
+          this._compactUpdateLink(newFd, newAt, index, keyAt, Slot.LEAF);
+
+          keys++;
+          percent += increment;
+        } else if (entryType === Slot.BRANCH) {
+          // l.debug(() => '_compact/isBranch');
+
+          const headerAt = this._compactWriteHeader(newFd, newAt, index);
+
+          doCompact(headerAt, dataAt, depth + 1);
+        } else {
+          throw new Error(`Unknown entry type, ${entryType}`);
+        }
+
+        fn({
+          isCompleted: depth === 0 && index === (defaults.ENTRY_NUM - 1),
+          keys,
+          percent
+        });
+      }
+
+      // l.debug(() => ['_compact', '=>', `${depth}: ${keys} keys written`]);
+    };
+
+    doCompact(0, 0, 0);
+
+    return keys;
+  }
+
+  private _compactReadEntry (fd: number, at: number, index: number): Buffer {
+    const entry = Buffer.alloc(defaults.ENTRY_SIZE);
+    const entryAt = at + (index * defaults.ENTRY_SIZE);
+
+    fs.readSync(fd, entry, 0, defaults.ENTRY_SIZE, entryAt);
+
+    return entry;
+  }
+
+  private _compactReadKey (fd: number, at: number): [Buffer, Buffer] {
+    const key = Buffer.alloc(defaults.KEY_TOTAL_SIZE);
+
+    fs.readSync(fd, key, 0, defaults.KEY_TOTAL_SIZE, at);
+
+    const valueLength = key.readUIntBE(defaults.KEY_SIZE, defaults.UINT_SIZE);
+    const valueAt = key.readUIntBE(defaults.KEY_SIZE + defaults.UINT_SIZE, defaults.UINT_SIZE);
+    const value = Buffer.alloc(valueLength);
+
+    fs.readSync(fd, value, 0, valueLength, valueAt);
+
+    return [key, value];
+  }
+
+  private _compactWriteKey (fd: number, key: Buffer, value: Buffer): number {
+    const valueAt = fs.fstatSync(fd).size;
+    const keyAt = valueAt + value.length;
+
+    key.writeUIntBE(valueAt, defaults.KEY_SIZE + defaults.UINT_SIZE, defaults.UINT_SIZE);
+
+    fs.writeSync(fd, value, 0, value.length, valueAt);
+    fs.writeSync(fd, key, 0, defaults.KEY_TOTAL_SIZE, keyAt);
+
+    return keyAt;
+  }
+
+  private _compactUpdateLink (fd: number, at: number, index: number, pointer: number, type: Slot): void {
+    const entry = Buffer.alloc(defaults.ENTRY_SIZE);
+
+    entry.set([type], 0);
+    entry.writeUIntBE(pointer, 1, defaults.UINT_SIZE);
+
+    fs.writeSync(fd, entry, 0, defaults.ENTRY_SIZE, at + (index * defaults.ENTRY_SIZE));
+  }
+
+  private _compactWriteHeader (fd: number, at: number, index: number): number {
+    const headerAt = fs.fstatSync(fd).size;
+    const header = Buffer.alloc(defaults.BRANCH_SIZE);
+
+    fs.writeSync(fd, header, 0, defaults.BRANCH_SIZE, headerAt);
+
+    this._compactUpdateLink(fd, at, index, headerAt, Slot.BRANCH);
+
+    return headerAt;
+  }
+}

packages/db/src/FileFlatDb/defaults.ts

@@ -0,0 +1,22 @@
+// Copyright 2017-2018 @polkadot/db authors & contributors
+// This software may be modified and distributed under the terms
+// of the ISC license. See the LICENSE file for details.
+
+// NOTE 1,099,511,627,776 filesize (max allowed here is 6 as per Nodejs)
+const UINT_SIZE = 5;
+const KEY_SIZE = 32;
+const KEY_TOTAL_SIZE = KEY_SIZE + UINT_SIZE + UINT_SIZE;
+const ENTRY_NUM = 16; // nibbles, 256 for bytes (where serialize would be noop)
+const ENTRY_SIZE = 1 + UINT_SIZE;
+const BRANCH_SIZE = ENTRY_NUM * ENTRY_SIZE;
+const DEFAULT_FILE = 'store.db';
+
+export default {
+  BRANCH_SIZE,
+  DEFAULT_FILE,
+  ENTRY_NUM,
+  ENTRY_SIZE,
+  KEY_SIZE,
+  KEY_TOTAL_SIZE,
+  UINT_SIZE
+};

packages/db/src/FileFlatDb/index.spec.js

@@ -7,7 +7,7 @@ import os from 'os';
 import path from 'path';
 import rimraf from 'rimraf';
 
-import FileFlatDb from './FileFlatDb';
+import FileFlatDb from './index';
 
 const KEY_A = new Uint8Array([
   0x10, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -40,16 +40,6 @@ const VAL_D = new Uint8Array([0x42, 1, 2, 3, 4, 0x69]);
 const VAL_E = new Uint8Array([0x42, 1, 2, 3, 4, 5, 0x69]);
 const VAL_F = new Uint8Array([0x42, 1, 2, 3, 4, 5, 6, 0x69]);
 
-describe.skip('FileFlatDb (compacting)', () => {
-  const combined = new FileFlatDb(process.cwd());
-
-  it('compacts without failing', () => {
-    expect(
-      combined.maintain(() => {})
-    ).not.toBe(0);
-  });
-});
-
 // NOTE Skipped, doesn't seem to be too happy on CI (cwd issues?)
 describe.skip('FileFlatDb (basics)', () => {
   const testGet = (key, value) =>
@@ -103,7 +93,7 @@ describe.skip('FileFlatDb (basics)', () => {
     testGet(KEY_C, VAL_C);
   });
 
-  it.skip('writes an entry (expanding the tree, again)', () => {
+  it('writes an entry (expanding the tree, again)', () => {
     console.error('D: execute');
 
     store.put(KEY_D, VAL_D);
@@ -116,7 +106,7 @@ describe.skip('FileFlatDb (basics)', () => {
     testGet(KEY_D, VAL_D);
   });
 
-  it.skip('writes an entry (expanding the tree, yet again)', () => {
+  it('writes an entry (expanding the tree, yet again)', () => {
     console.error('E: execute');
 
     store.put(KEY_E, VAL_E);
@@ -130,7 +120,7 @@ describe.skip('FileFlatDb (basics)', () => {
     testGet(KEY_E, VAL_E);
   });
 
-  it.skip('writes an entry, expanding the top-level', () => {
+  it('writes an entry, expanding the top-level', () => {
     console.error('F: execute');
 
     store.put(KEY_F, VAL_F);
@@ -144,4 +134,24 @@ describe.skip('FileFlatDb (basics)', () => {
     testGet(KEY_E, VAL_E);
     testGet(KEY_F, VAL_F);
   });
+
+  it('overrides with smaller values', () => {
+    console.error('G: execute');
+
+    store.put(KEY_F, VAL_A);
+
+    console.error('G: expectations');
+
+    testGet(KEY_F, VAL_A);
+  });
+
+  it('overrides with larger values', () => {
+    console.error('H: execute');
+
+    store.put(KEY_A, VAL_F);
+
+    console.error('H: expectations');
+
+    testGet(KEY_A, VAL_F);
+  });
 });

packages/db/src/FileFlatDb/index.ts

@@ -3,6 +3,7 @@
 // of the ISC license. See the LICENSE file for details.
 
 import { BaseDb, BaseDbOptions, ProgressCb } from '../types';
+import { Key, NibbleBuffer, Slot, Value } from './types';
 
 import fs from 'fs';
 import { LRUMap } from 'lru_map';
@@ -17,49 +18,23 @@ import bufferToU8a from '@polkadot/util/buffer/toU8a';
 import u8aToBuffer from '@polkadot/util/u8a/toBuffer';
 import u8aToHex from '@polkadot/util/u8a/toHex';
 
-enum Slot {
-  EMPTY = 0,
-  BRANCH = 1,
-  LEAF = 2
-}
+import Compact from './Compact';
+import defaults from './defaults';
 
-type NibbleBuffer = {
-  buffer: Buffer,
-  nibbles: Uint8Array
-};
-
-type Key = {
-  key: NibbleBuffer,
-  keyAt: number,
-  keyValue: Buffer
-};
-
-type Value = {
-  value: Buffer,
-  valueAt: number
-};
-
-// NOTE 1,099,511,627,776 filesize (max allowed here is 6 as per Nodejs)
-const UINT_SIZE = 5;
-const KEY_SIZE = 32;
-const KEY_TOTAL_SIZE = KEY_SIZE + UINT_SIZE + UINT_SIZE;
-const ENTRY_NUM = 16; // nibbles, 256 for bytes (where serialize would be noop)
-const ENTRY_SIZE = 1 + UINT_SIZE;
-const BRANCH_SIZE = ENTRY_NUM * ENTRY_SIZE;
 const LRU_BRANCH_COUNT = 2048;
 const LRU_DATA_COUNT = 4096;
-const DEFAULT_FILE = 'store.db';
 
 const l = logger('db/flat');
 
 export default class FileFlatDb implements BaseDb {
   private _fd: number;
+  private _fileSize: number = 0;
   private _file: string;
   private _isCompressed: boolean;
   private _lruBranch: LRUMap<number, Buffer>;
   private _lruData: LRUMap<number, Buffer>;
 
-  constructor (base: string, file: string = DEFAULT_FILE, options: BaseDbOptions = {}) {
+  constructor (base: string, file: string = defaults.DEFAULT_FILE, options: BaseDbOptions = {}) {
     this._fd = -1;
     this._file = path.join(base, file);
     this._lruBranch = new LRUMap(LRU_BRANCH_COUNT);
@@ -73,6 +48,7 @@ export default class FileFlatDb implements BaseDb {
 
   open (): void {
     this._fd = this._open(this._file);
+    this._fileSize = fs.fstatSync(this._fd).size;
 
     this._lruBranch.clear();
     this._lruData.clear();
@@ -90,27 +66,9 @@ export default class FileFlatDb implements BaseDb {
   maintain (fn: ProgressCb): void {
     assert(this._fd === -1, 'Database cannot be open for compacting');
 
-    l.log('compacting database');
+    const compactor = new Compact(this._file);
 
-    const start = Date.now();
-    const newFile = `${this._file}.compacted`;
-    const newFd = this._open(newFile, true);
-    const oldFd = this._open(this._file);
-    const keys = this._compact(fn, newFd, oldFd);
-
-    fs.closeSync(oldFd);
-    fs.closeSync(newFd);
-
-    const newStat = fs.lstatSync(newFile);
-    const oldStat = fs.lstatSync(this._file);
-    const percentage = 100 * (newStat.size / oldStat.size);
-    const sizeMB = newStat.size / (1024 * 1024);
-    const elapsed = (Date.now() - start) / 1000;
-
-    fs.unlinkSync(this._file);
-    fs.renameSync(newFile, this._file);
-
-    l.log(`compacted in ${elapsed.toFixed(2)}s, ${(keys / 1000).toFixed(2)}k keys, ${sizeMB.toFixed(2)}MB (${percentage.toFixed(2)}%)`);
+    compactor.maintain(fn);
   }
 
   del (key: Uint8Array): void {
@@ -169,16 +127,16 @@ export default class FileFlatDb implements BaseDb {
   }
 
   private _serializeKey (u8a: Uint8Array): NibbleBuffer {
-    if (u8a.length > KEY_SIZE) {
+    if (u8a.length > defaults.KEY_SIZE) {
       throw new Error(`${u8aToHex(u8a)} too large, expected <= 32 bytes`);
     }
 
     let buffer;
 
-    if (u8a.length === KEY_SIZE) {
+    if (u8a.length === defaults.KEY_SIZE) {
       buffer = u8aToBuffer(u8a);
     } else {
-      buffer = Buffer.alloc(KEY_SIZE);
+      buffer = Buffer.alloc(defaults.KEY_SIZE);
 
       buffer.set(u8a, 0);
     }
@@ -201,9 +159,9 @@ export default class FileFlatDb implements BaseDb {
     let branch = this._lruBranch.get(branchAt);
 
     if (!branch) {
-      branch = Buffer.alloc(BRANCH_SIZE);
+      branch = Buffer.alloc(defaults.BRANCH_SIZE);
 
-      fs.readSync(this._fd, branch, 0, BRANCH_SIZE, branchAt);
+      fs.readSync(this._fd, branch, 0, defaults.BRANCH_SIZE, branchAt);
       this._cacheBranch(branchAt, branch);
     }
 
@@ -224,11 +182,11 @@ export default class FileFlatDb implements BaseDb {
   }
 
   private _getKeyValue (keyAt: number): Buffer {
-    return this._getCachedData(keyAt, KEY_TOTAL_SIZE);
+    return this._getCachedData(keyAt, defaults.KEY_TOTAL_SIZE);
   }
 
   private _findKey (key: NibbleBuffer, doCreate: boolean, keyIndex: number, branchAt: number): Key | null {
-    const entryIndex = key.nibbles[keyIndex] * ENTRY_SIZE;
+    const entryIndex = key.nibbles[keyIndex] * defaults.ENTRY_SIZE;
     const branch = this._getCachedBranch(branchAt);
 
     l.debug(() => ['findKey', { key, doCreate, keyIndex, branchAt, branch, entryIndex }]);
@@ -236,7 +194,7 @@ export default class FileFlatDb implements BaseDb {
     const entryType = branch[entryIndex];
 
     if (entryType === Slot.BRANCH) {
-      const nextBranchAt = branch.readUIntBE(entryIndex + 1, UINT_SIZE);
+      const nextBranchAt = branch.readUIntBE(entryIndex + 1, defaults.UINT_SIZE);
 
       l.debug(() => ['findKey/isBranch', { nextBranchAt }]);
 
@@ -252,14 +210,14 @@ export default class FileFlatDb implements BaseDb {
     }
 
     if (entryType === Slot.LEAF) {
-      const keyAt = branch.readUIntBE(entryIndex + 1, UINT_SIZE);
+      const keyAt = branch.readUIntBE(entryIndex + 1, defaults.UINT_SIZE);
       const keyValue = this._getKeyValue(keyAt);
-      const prevKey = this._serializeKey(keyValue.subarray(0, KEY_SIZE));
+      const prevKey = this._serializeKey(keyValue.subarray(0, defaults.KEY_SIZE));
       let matchIndex = keyIndex;
 
       l.debug(() => ['findKey/isLeaf', { keyAt, branch, branchAt, entryIndex, keyValue }]);
 
-      while (matchIndex < KEY_SIZE) {
+      while (matchIndex < defaults.KEY_SIZE) {
         if (prevKey.nibbles[matchIndex] !== key.nibbles[matchIndex]) {
           break;
         }
@@ -267,7 +225,7 @@ export default class FileFlatDb implements BaseDb {
         matchIndex++;
       }
 
-      if (matchIndex !== KEY_SIZE) {
+      if (matchIndex !== defaults.KEY_SIZE) {
         return doCreate
           ? this.writeNewBranch(branch, branchAt, entryIndex, key, keyAt, prevKey, matchIndex, matchIndex - keyIndex - 1)
           : null;
@@ -293,8 +251,8 @@ export default class FileFlatDb implements BaseDb {
 
   private _extractValueInfo (keyValue: Buffer): { valueAt: number, valueLength: number } {
     return {
-      valueLength: keyValue.readUIntBE(KEY_SIZE, UINT_SIZE),
-      valueAt: keyValue.readUIntBE(KEY_SIZE + UINT_SIZE, UINT_SIZE)
+      valueLength: keyValue.readUIntBE(defaults.KEY_SIZE, defaults.UINT_SIZE),
+      valueAt: keyValue.readUIntBE(defaults.KEY_SIZE + defaults.UINT_SIZE, defaults.UINT_SIZE)
     };
   }
 
@@ -321,18 +279,14 @@ export default class FileFlatDb implements BaseDb {
   private _writeValue (keyAt: number, keyValue: Buffer, value: Buffer): Value {
     l.debug(() => ['writeValue', { keyAt, keyValue, value }]);
 
-    let { valueAt, valueLength } = this._extractValueInfo(keyValue);
+    const current = this._extractValueInfo(keyValue);
+    const valueAt = value.length > current.valueLength
+      ? this._writeNewBuffer(value)
+      : this._writeUpdatedBuffer(value, current.valueAt);
 
-    if (valueLength < value.length) {
-      valueAt = fs.fstatSync(this._fd).size;
-      this._cacheData(valueAt, value);
-    }
-
-    keyValue.writeUIntBE(value.length, KEY_SIZE, UINT_SIZE);
-    keyValue.writeUIntBE(valueAt, KEY_SIZE + UINT_SIZE, UINT_SIZE);
-
-    fs.writeSync(this._fd, value, 0, value.length, valueAt);
-    fs.writeSync(this._fd, keyValue, KEY_SIZE, 2 * UINT_SIZE, keyAt + KEY_SIZE);
+    keyValue.writeUIntBE(value.length, defaults.KEY_SIZE, defaults.UINT_SIZE);
+    keyValue.writeUIntBE(valueAt, defaults.KEY_SIZE + defaults.UINT_SIZE, defaults.UINT_SIZE);
+    fs.writeSync(this._fd, keyValue, defaults.KEY_SIZE, 2 * defaults.UINT_SIZE, keyAt + defaults.KEY_SIZE);
 
     return {
       value,
@@ -351,13 +305,11 @@ export default class FileFlatDb implements BaseDb {
   private _writeNewKey (key: NibbleBuffer): Key {
     l.debug(() => ['writeNewKey', { key }]);
 
-    const keyAt = fs.fstatSync(this._fd).size;
-    const keyValue = Buffer.alloc(KEY_TOTAL_SIZE);
+    const keyValue = Buffer.alloc(defaults.KEY_TOTAL_SIZE);
 
     keyValue.set(key.buffer, 0);
 
-    fs.writeSync(this._fd, keyValue, 0, KEY_TOTAL_SIZE, keyAt);
-    this._cacheData(keyAt, keyValue);
+    const keyAt = this._writeNewBuffer(keyValue);
 
     return {
       key,
@@ -378,35 +330,33 @@ export default class FileFlatDb implements BaseDb {
     l.debug(() => ['writeNewBranch', { branch, branchAt, entryIndex, key, prevAt, prevKey, matchIndex, depth }]);
 
     const { keyAt, keyValue } = this.writeNewKey(key);
-    const keyIndex = key.nibbles[matchIndex] * ENTRY_SIZE;
-    const prevIndex = prevKey.nibbles[matchIndex] * ENTRY_SIZE;
-    let newBranchAt = fs.fstatSync(this._fd).size;
-    let newBranch = Buffer.alloc(BRANCH_SIZE);
+    const keyIndex = key.nibbles[matchIndex] * defaults.ENTRY_SIZE;
+    const prevIndex = prevKey.nibbles[matchIndex] * defaults.ENTRY_SIZE;
+    const buffers = [];
+    let newBranchAt = this._fileSize;
+    let newBranch = Buffer.alloc(defaults.BRANCH_SIZE);
 
-    // FIXME Combine this along with all the newBranch writes in the loop into a single write
     newBranch.set([Slot.LEAF], keyIndex);
-    newBranch.writeUIntBE(keyAt, keyIndex + 1, UINT_SIZE);
+    newBranch.writeUIntBE(keyAt, keyIndex + 1, defaults.UINT_SIZE);
     newBranch.set([Slot.LEAF], prevIndex);
-    newBranch.writeUIntBE(prevAt, prevIndex + 1, UINT_SIZE);
-
-    fs.writeSync(this._fd, newBranch, 0, BRANCH_SIZE, newBranchAt);
-    this._cacheBranch(newBranchAt, newBranch);
+    newBranch.writeUIntBE(prevAt, prevIndex + 1, defaults.UINT_SIZE);
+    buffers.push(newBranch);
 
     for (let offset = 1; depth > 0; depth--, offset++) {
-      const branchIndex = key.nibbles[matchIndex - offset] * ENTRY_SIZE;
+      const branchIndex = key.nibbles[matchIndex - offset] * defaults.ENTRY_SIZE;
 
-      newBranch = Buffer.alloc(BRANCH_SIZE);
+      newBranch = Buffer.alloc(defaults.BRANCH_SIZE);
       newBranch.set([Slot.BRANCH], branchIndex);
-      newBranch.writeUIntBE(newBranchAt, branchIndex + 1, UINT_SIZE);
-      newBranchAt += BRANCH_SIZE;
-
-      fs.writeSync(this._fd, newBranch, 0, BRANCH_SIZE, newBranchAt);
-      this._cacheBranch(newBranchAt, newBranch);
+      newBranch.writeUIntBE(newBranchAt, branchIndex + 1, defaults.UINT_SIZE);
+      buffers.push(newBranch);
+      newBranchAt += defaults.BRANCH_SIZE;
     }
 
+    this._writeNewBuffers(buffers);
+
     branch.set([Slot.BRANCH], entryIndex);
-    branch.writeUIntBE(newBranchAt, entryIndex + 1, UINT_SIZE);
-    fs.writeSync(this._fd, branch, entryIndex, ENTRY_SIZE, branchAt + entryIndex);
+    branch.writeUIntBE(newBranchAt, entryIndex + 1, defaults.UINT_SIZE);
+    fs.writeSync(this._fd, branch, entryIndex, defaults.ENTRY_SIZE, branchAt + entryIndex);
 
     return {
       key,
@@ -429,9 +379,9 @@ export default class FileFlatDb implements BaseDb {
     const { keyAt, keyValue } = this.writeNewKey(key);
 
     branch.set([Slot.LEAF], entryIndex);
-    branch.writeUIntBE(keyAt, entryIndex + 1, UINT_SIZE);
+    branch.writeUIntBE(keyAt, entryIndex + 1, defaults.UINT_SIZE);
 
-    fs.writeSync(this._fd, branch, entryIndex, ENTRY_SIZE, branchAt + entryIndex);
+    fs.writeSync(this._fd, branch, entryIndex, defaults.ENTRY_SIZE, branchAt + entryIndex);
 
     return {
       key,
@@ -448,122 +398,47 @@ export default class FileFlatDb implements BaseDb {
     return result;
   }
 
+  private _writeUpdatedBuffer (buffer: Buffer, bufferAt: number): number {
+    fs.writeSync(this._fd, buffer, 0, buffer.length, bufferAt);
+    this._cacheData(bufferAt, buffer);
+
+    return bufferAt;
+  }
+
+  private _writeNewBuffer (buffer: Buffer, withCache: boolean = true): number {
+    const startAt = this._fileSize;
+
+    fs.writeSync(this._fd, buffer, 0, buffer.length, startAt);
+
+    if (withCache) {
+      this._cacheData(startAt, buffer);
+    }
+
+    this._fileSize += buffer.length;
+
+    return startAt;
+  }
+
+  private _writeNewBuffers (buffers: Array<Buffer>): number {
+    let bufferAt = this._fileSize;
+
+    buffers.forEach((buffer) => {
+      this._cacheData(bufferAt, buffer);
+      bufferAt += buffer.length;
+    });
+
+    return this._writeNewBuffer(Buffer.concat(buffers), false);
+  }
+
   private assertOpen (): void {
     assert(this._fd !== -1, 'Expected an open database');
   }
 
   private _open (file: string, startEmpty: boolean = false): number {
     if (!fs.existsSync(file) || startEmpty) {
-      fs.writeFileSync(file, Buffer.alloc(BRANCH_SIZE));
+      fs.writeFileSync(file, Buffer.alloc(defaults.BRANCH_SIZE));
     }
 
     return fs.openSync(file, 'a+');
   }
-
-  private _compactReadEntry (fd: number, at: number, index: number): Buffer {
-    const entry = Buffer.alloc(ENTRY_SIZE);
-    const entryAt = at + (index * ENTRY_SIZE);
-
-    fs.readSync(fd, entry, 0, ENTRY_SIZE, entryAt);
-
-    return entry;
-  }
-
-  private _compactReadKey (fd: number, at: number): [Buffer, Buffer] {
-    const key = Buffer.alloc(KEY_TOTAL_SIZE);
-
-    fs.readSync(fd, key, 0, KEY_TOTAL_SIZE, at);
-
-    const valueLength = key.readUIntBE(KEY_SIZE, UINT_SIZE);
-    const valueAt = key.readUIntBE(KEY_SIZE + UINT_SIZE, UINT_SIZE);
-    const value = Buffer.alloc(valueLength);
-
-    fs.readSync(fd, value, 0, valueLength, valueAt);
-
-    return [key, value];
-  }
-
-  private _compactWriteKey (fd: number, key: Buffer, value: Buffer): number {
-    const valueAt = fs.fstatSync(fd).size;
-    const keyAt = valueAt + value.length;
-
-    key.writeUIntBE(valueAt, KEY_SIZE + UINT_SIZE, UINT_SIZE);
-
-    fs.writeSync(fd, value, 0, value.length, valueAt);
-    fs.writeSync(fd, key, 0, KEY_TOTAL_SIZE, keyAt);
-
-    return keyAt;
-  }
-
-  private _compactUpdateLink (fd: number, at: number, index: number, pointer: number, type: Slot): void {
-    const entry = Buffer.alloc(ENTRY_SIZE);
-
-    entry.set([type], 0);
-    entry.writeUIntBE(pointer, 1, UINT_SIZE);
-
-    fs.writeSync(fd, entry, 0, ENTRY_SIZE, at + (index * ENTRY_SIZE));
-  }
-
-  private _compactWriteHeader (fd: number, at: number, index: number): number {
-    const headerAt = fs.fstatSync(fd).size;
-    const header = Buffer.alloc(BRANCH_SIZE);
-
-    fs.writeSync(fd, header, 0, BRANCH_SIZE, headerAt);
-
-    this._compactUpdateLink(fd, at, index, headerAt, Slot.BRANCH);
-
-    return headerAt;
-  }
-
-  private _compact (fn: ProgressCb, newFd: number, oldFd: number): number {
-    // l.debug(() => ['_compact', debug({ newFd, oldFd, newAt, oldAt })]);
-
-    let keys = 0;
-    let percent = 0;
-
-    const doCompact = (newAt: number, oldAt: number, depth: number) => {
-      const increment = (100 / ENTRY_NUM) / Math.pow(ENTRY_NUM, depth);
-
-      for (let index = 0; index < ENTRY_NUM; index++) {
-        const entry = this._compactReadEntry(oldFd, oldAt, index);
-        const dataAt = entry.readUIntBE(1, UINT_SIZE);
-        const entryType = entry[0];
-
-        if (entryType === Slot.EMPTY) {
-          // l.debug(() => '_compact/isEmpty');
-          percent += increment;
-        } else if (entryType === Slot.LEAF) {
-          // l.debug(() => '_compact/isLeaf');
-
-          const [key, value] = this._compactReadKey(oldFd, dataAt);
-          const keyAt = this._compactWriteKey(newFd, key, value);
-
-          this._compactUpdateLink(newFd, newAt, index, keyAt, Slot.LEAF);
-
-          keys++;
-          percent += increment;
-        } else if (entryType === Slot.BRANCH) {
-          // l.debug(() => '_compact/isBranch');
-
-          const headerAt = this._compactWriteHeader(newFd, newAt, index);
-
-          doCompact(headerAt, dataAt, depth + 1);
-        } else {
-          throw new Error(`Unknown entry type, ${entryType}`);
-        }
-
-        fn({
-          isCompleted: depth === 0 && index === (ENTRY_NUM - 1),
-          keys,
-          percent
-        });
-      }
-
-      // l.debug(() => ['_compact', '=>', `${depth}: ${keys} keys written`]);
-    };
-
-    doCompact(0, 0, 0);
-
-    return keys;
-  }
 }

packages/db/src/FileFlatDb/types.ts

@@ -0,0 +1,25 @@
+// Copyright 2017-2018 @polkadot/db authors & contributors
+// This software may be modified and distributed under the terms
+// of the ISC license. See the LICENSE file for details.
+
+export enum Slot {
+  EMPTY = 0,
+  BRANCH = 1,
+  LEAF = 2
+}
+
+export type NibbleBuffer = {
+  buffer: Buffer,
+  nibbles: Uint8Array
+};
+
+export type Key = {
+  key: NibbleBuffer,
+  keyAt: number,
+  keyValue: Buffer
+};
+
+export type Value = {
+  value: Buffer,
+  valueAt: number
+};

yarn.lock

@@ -1285,9 +1285,9 @@
     typedoc-plugin-markdown "^1.1.13"
     typescript "^3.0.1"
 
-"@polkadot/ts@^0.1.23":
-  version "0.1.23"
-  resolved "https://registry.yarnpkg.com/@polkadot/ts/-/ts-0.1.23.tgz#8515963f514ef62c1fa97b656a18553c01c32530"
+"@polkadot/ts@^0.1.24":
+  version "0.1.24"
+  resolved "https://registry.yarnpkg.com/@polkadot/ts/-/ts-0.1.24.tgz#a5202626f394114831851b476beba8a3a334ba79"
 
 "@types/base-x@*":
   version "1.0.29"