Fix FileDb nibble length, use to 256-entry headers (#383)

.gitignore

@@ -1,3 +1,4 @@
+--test-*
 _book/
 build/
 build-docs/

packages/db/src/FileFlatDb/Impl.ts

@@ -41,15 +41,15 @@ export default class Impl extends Cache {
 
     l.debug(() => ['findKey/isLeaf', { keyAt, branch, branchAt, entryIndex, keyValue }]);
 
-    while (matchIndex < defaults.KEY_SIZE) {
-      if (prevKey.nibbles[matchIndex] !== key.nibbles[matchIndex]) {
+    while (matchIndex < defaults.KEY_PARTS_LENGTH) {
+      if (prevKey.parts[matchIndex] !== key.parts[matchIndex]) {
         break;
       }
 
       matchIndex++;
     }
 
-    if (matchIndex !== defaults.KEY_SIZE) {
+    if (matchIndex !== defaults.KEY_PARTS_LENGTH) {
       return doCreate
         ? this.writeNewBranch(branch, branchAt, entryIndex, key, keyAt, prevKey, matchIndex, matchIndex - keyIndex - 1)
         : null;
@@ -63,7 +63,7 @@ export default class Impl extends Cache {
   }
 
   protected _findKey (key: NibbleBuffer, doCreate: boolean, keyIndex: number, branchAt: number): Key | null {
-    const entryIndex = key.nibbles[keyIndex] * defaults.ENTRY_SIZE;
+    const entryIndex = key.parts[keyIndex] * defaults.ENTRY_SIZE;
     const branch = this._getCachedBranch(branchAt);
 
     l.debug(() => ['findKey', { key, doCreate, keyIndex, branchAt, branch, entryIndex }]);
@@ -150,8 +150,8 @@ export default class Impl extends Cache {
     // l.debug(() => ['writeNewBranch', { branch, branchAt, entryIndex, key, prevAt, prevKey, matchIndex, depth }]);
 
     const { keyAt, keyValue } = this.writeNewKey(key);
-    const keyIndex = key.nibbles[matchIndex] * defaults.ENTRY_SIZE;
-    const prevIndex = prevKey.nibbles[matchIndex] * defaults.ENTRY_SIZE;
+    const keyIndex = key.parts[matchIndex] * defaults.ENTRY_SIZE;
+    const prevIndex = prevKey.parts[matchIndex] * defaults.ENTRY_SIZE;
     const buffers: Array<Buffer> = [];
     let newBranchAt = this._fileSize;
     let newBranch = Buffer.alloc(defaults.BRANCH_SIZE);
@@ -163,7 +163,7 @@ export default class Impl extends Cache {
     buffers.push(newBranch);
 
     for (let offset = 1; depth > 0; depth--, offset++) {
-      const branchIndex = key.nibbles[matchIndex - offset] * defaults.ENTRY_SIZE;
+      const branchIndex = key.parts[matchIndex - offset] * defaults.ENTRY_SIZE;
 
       newBranch = Buffer.alloc(defaults.BRANCH_SIZE);
       newBranch.set([Slot.BRANCH], branchIndex);

packages/db/src/FileFlatDb/Serialize.ts

@@ -33,18 +33,23 @@ export default class Serialize {
     assert(u8a.length <= defaults.KEY_SIZE, `${u8aToHex(u8a)} too large, expected <= 32 bytes`);
 
     let buffer;
+    let full;
 
     if (u8a.length === defaults.KEY_SIZE) {
-      buffer = u8aToBuffer(u8a);
+      full = u8a;
     } else {
-      buffer = Buffer.alloc(defaults.KEY_SIZE);
+      full = new Uint8Array(defaults.KEY_SIZE);
 
-      buffer.set(u8a, 0);
+      full.set(u8a, 0);
     }
 
+    buffer = u8aToBuffer(full);
+
     return {
       buffer,
-      nibbles: toNibbles(u8a)
+      parts: defaults.KEY_IS_NIBBLES
+        ? toNibbles(full)
+        : full
     };
   }
 }

packages/db/src/FileFlatDb/ViaTrie.spec.ts

@@ -0,0 +1,54 @@
+// Copyright 2017-2019 @polkadot/db authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import mkdirp from 'mkdirp';
+import path from 'path';
+import rimraf from 'rimraf';
+import { alexander } from '@polkadot/chainspec';
+import TrieDb from '@polkadot/trie-db';
+import { hexToU8a, u8aToHex } from '@polkadot/util';
+
+import DiskDb from '../Disk';
+
+describe.skip('DB via a trie', () => {
+  const location = path.join(process.cwd(), '--test-FileFlatDb-ViaTrie');
+  const { genesis: { raw }, genesisRoot } = alexander;
+
+  beforeAll(() => {
+    rimraf.sync(location);
+    mkdirp.sync(location);
+  });
+
+  it('creates a trie with the correct root', () => {
+    const trie = new TrieDb(new DiskDb(location, 'test.db', { isNative: true }));
+
+    trie.open();
+    trie.transaction(() => {
+      Object.keys(raw).forEach((key) => {
+        trie.put(hexToU8a(key), hexToU8a(raw[key]));
+      });
+
+      return true;
+    });
+
+    expect(
+      u8aToHex(trie.getRoot())
+    ).toEqual(genesisRoot);
+  });
+
+  describe('re-opens and has all the correct keys', () => {
+    const trie = new TrieDb(new DiskDb(location, 'test.db', { isNative: true }));
+
+    trie.open();
+    trie.setRoot(hexToU8a(genesisRoot));
+
+    Object.keys(raw).forEach((key) => {
+      it(key, () => {
+        expect(
+          u8aToHex(trie.get(hexToU8a(key)))
+        ).toEqual(raw[key]);
+      });
+    });
+  });
+});

packages/db/src/FileFlatDb/defaults.ts

@@ -2,22 +2,42 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
-// NOTE 1,099,511,627,776 filesize (max allowed here is 6 as per Nodejs)
+// Allows for 1,099,511,627,776 filesize (max allowed here is 6 as per Nodejs)
 const UINT_SIZE = 5;
-const KEY_SIZE = 32;
-const KEY_TOTAL_SIZE = KEY_SIZE + UINT_SIZE + UINT_SIZE;
-const ENTRY_NUM = 16; // nibbles, 256 for bytes (where serialize would be noop)
-const ENTRY_SIZE = 1 + UINT_SIZE;
-const BRANCH_SIZE = ENTRY_NUM * ENTRY_SIZE;
 const DEFAULT_FILE = 'store.db';
-const LRU_BRANCH_COUNT = 16384; // * 96 = bytes
-const LRU_DATA_COUNT = 8192;
+const ONE_K = 1024;
+
+// 16 for nibbles (key -> asNibbles), 256 for bytes (key -> asU8a)
+// There is a tradeof here:
+//   - with nibbles there are more writes and more read branching traversals
+//   - with bytes there are less writes, however more (initial) disk space
+const ENTRY_NUM: 16 | 256 = 256;
+const ENTRY_SIZE = 1 + UINT_SIZE;
+
+// the size of a branch entry, 96 for nibbles, 1,536 for bytes
+const BRANCH_SIZE = ENTRY_NUM * ENTRY_SIZE;
+
+// key calculations
+const KEY_IS_NIBBLES = (ENTRY_NUM as any) === 16;
+const KEY_SIZE = 32;
+const KEY_PARTS_LENGTH = KEY_IS_NIBBLES
+  ? KEY_SIZE * 2
+  : KEY_SIZE;
+const KEY_TOTAL_SIZE = KEY_SIZE + UINT_SIZE + UINT_SIZE;
+
+// LRU caches so things don't need to be read directly from disk when fresh
+const LRU_BRANCH_COUNT = KEY_IS_NIBBLES
+  ? 32 * ONE_K
+  : 4 * ONE_K;
+const LRU_DATA_COUNT = 32 * ONE_K;
 
 export default {
   BRANCH_SIZE,
   DEFAULT_FILE,
   ENTRY_NUM,
   ENTRY_SIZE,
+  KEY_IS_NIBBLES,
+  KEY_PARTS_LENGTH,
   KEY_SIZE,
   KEY_TOTAL_SIZE,
   LRU_BRANCH_COUNT,

packages/db/src/FileFlatDb/index.spec.ts

@@ -41,7 +41,7 @@ const VAL_F = new Uint8Array([0x42, 1, 2, 3, 4, 5, 6, 0x69]);
 
 // NOTE Skipped, doesn't seem to be too happy on CI (cwd issues?)
 describe.skip('FileFlatDb (basics)', () => {
-  const location = path.join(process.cwd(), '--test-FileFlatDb');
+  const location = path.join(process.cwd(), '--test-FileFlatDb-index');
   let store: FileFlatDb;
 
   const testGet = (key: Uint8Array, value: Uint8Array) =>

packages/db/src/FileFlatDb/types.ts

@@ -10,7 +10,7 @@ export enum Slot {
 
 export type NibbleBuffer = {
   buffer: Buffer,
-  nibbles: Uint8Array
+  parts: Uint8Array
 };
 
 export type Key = {