Use Read & Write transactions (interleave) (#354)

* Use Read & Write transactions (interleave)

* Rename toSeedEntropy -> toMiniSecret

packages/db/src/engines/LmDb.ts

@@ -14,16 +14,26 @@ const GB = 1 * 1024 * 1024 * 1024;
 
 const l = logger('db/lmdb');
 
+// NOTE we manage 2 transaction types, one for read and one for write. For
+// triedbs we assume that the transaction _only_ contains writes (managed by
+// the transaction overlay). At the same time have a read that we keep open
+// so we don't need to re-create the transaction, otherwise wastefull.
+//
+// There is reasoning behind this madness, reads cannot overlap with writes,
+// so the database grows more than needed while making actual reads interleaved
+// with writes. So manage these completely seperately.
 export default class LmDb implements BaseDb {
   private _env: any;
   private _dbi: any | null;
   private _path: string;
-  private _txn: any | null;
+  private _rtxn: any | null;
+  private _wtxn: any | null;
 
   constructor (base: string, name: string, options?: BaseDbOptions) {
     this._env = new lmdb.Env();
     this._path = path.join(base, name);
-    this._txn = null;
+    this._rtxn = null;
+    this._wtxn = null;
 
     mkdirp.sync(this._path);
 
@@ -60,6 +70,8 @@ export default class LmDb implements BaseDb {
   }
 
   close (): void {
+    this.exitRead();
+
     this._dbi.close();
     this._dbi = null;
     this._env.close();
@@ -70,6 +82,8 @@ export default class LmDb implements BaseDb {
       create: true,
       keyIsBuffer: true
     });
+
+    this.enterRead();
   }
 
   drop (): void {
@@ -100,47 +114,49 @@ export default class LmDb implements BaseDb {
       : 0;
   }
 
+  private enterRead (): void {
+    this._rtxn = this._env.beginTxn({ readOnly: true });
+  }
+
+  private exitRead (): void {
+    this._rtxn.abort();
+    this._rtxn = null;
+  }
+
   txCommit (): void {
-    this._txn.commit();
-    this._txn = null;
+    this._wtxn.commit();
+    this._wtxn = null;
+    this.enterRead();
   }
 
   txRevert (): void {
-    this._txn.abort();
-    this._txn = null;
+    this._wtxn.abort();
+    this._wtxn = null;
+    this.enterRead();
   }
 
   txStart (): void {
+    this.exitRead();
+
+    // grow between transactions
     this.growMapSize();
 
-    this._txn = this._env.beginTxn();
+    this._wtxn = this._env.beginTxn();
   }
 
   del (key: Uint8Array): void {
-    this._txn.del(this._dbi, u8aToBuffer(key));
+    this._wtxn.del(this._dbi, u8aToBuffer(key));
   }
 
   get (_key: Uint8Array): Uint8Array | null {
-    const key = u8aToBuffer(_key);
-    const value = this._txn
-      ? this._txn.getBinary(this._dbi, key)
-      : this.txnGet(key);
+    const value = this._rtxn.getBinary(this._dbi, u8aToBuffer(_key));
 
     return value
       ? bufferToU8a(value)
       : null;
   }
 
-  private txnGet (key: Buffer): Buffer {
-    const txn = this._env.beginTxn();
-    const value = txn.getBinary(this._dbi, key);
-
-    txn.abort();
-
-    return value;
-  }
-
   put (key: Uint8Array, value: Uint8Array): void {
-    this._txn.putBinary(this._dbi, u8aToBuffer(key), u8aToBuffer(value));
+    this._wtxn.putBinary(this._dbi, u8aToBuffer(key), u8aToBuffer(value));
   }
 }

packages/db/src/engines/TransactionDb.ts

@@ -126,10 +126,6 @@ export default class TransactionDb implements TxDb {
 
     this.txOverlay = {};
     this.txStarted = true;
-
-    if (this.backing.txStart) {
-      this.backing.txStart();
-    }
   }
 
   private commitTx (): void {
@@ -137,6 +133,10 @@ export default class TransactionDb implements TxDb {
 
     assert(this.txStarted, 'Cannot commit when not in transaction');
 
+    if (this.backing.txStart) {
+      this.backing.txStart();
+    }
+
     Object.values(this.txOverlay).forEach(({ key, value }) => {
       if (isNull(value)) {
         this.backing.del(key);
@@ -145,12 +145,12 @@ export default class TransactionDb implements TxDb {
       }
     });
 
-    this.txOverlay = {};
-    this.txStarted = false;
-
     if (this.backing.txCommit) {
       this.backing.txCommit();
     }
+
+    this.txOverlay = {};
+    this.txStarted = false;
   }
 
   private revertTx (): void {
@@ -160,9 +160,5 @@ export default class TransactionDb implements TxDb {
 
     this.txOverlay = {};
     this.txStarted = false;
-
-    if (this.backing.txRevert) {
-      this.backing.txRevert();
-    }
   }
 }

packages/util-crypto/src/mnemonic/index.ts

@@ -8,6 +8,6 @@
 
 export { default as mnemonicGenerate } from './generate';
 export { default as mnemonicToEntropy } from './toEntropy';
+export { default as mnemonicToMiniSecret } from './toMiniSecret';
 export { default as mnemonicToSeed } from './toSeed';
-export { default as mnemonicToSeedEntropy } from './toSeedEntropy';
 export { default as mnemonicValidate } from './validate';

packages/util-crypto/src/mnemonic/toMiniSecret.spec.ts

@@ -4,14 +4,14 @@
 
 import { u8aToHex } from '@polkadot/util/index';
 
-import toSeedEntropy from './toSeedEntropy';
+import mnemonicToMiniSecret from './toMiniSecret';
 import tests from '../schnorrkel/keypair/testing';
 
-describe('mnemonicToSeedEntropy', () => {
+describe('mnemonicToMiniSecret', () => {
   tests.forEach(([mnemonic, , seed], index) => {
     it(`Created correct seed for ${index}`, () => {
       expect(
-        u8aToHex(toSeedEntropy(mnemonic, 'Substrate'))
+        u8aToHex(mnemonicToMiniSecret(mnemonic, 'Substrate'))
       ).toEqual(
         // mini returned here, only check first 32-bytes (64 hex + 2 prefix)
         seed.substr(0, 66)

packages/util-crypto/src/mnemonic/toMiniSecret.ts

@@ -7,7 +7,7 @@ import { bufferToU8a, stringToU8a, u8aToBuffer } from '@polkadot/util/index';
 
 import toEntropy from './toEntropy';
 
-export default function toSeedEntropy (mnemonic: string, password: string = ''): Uint8Array {
+export default function toMiniSecret (mnemonic: string, password: string = ''): Uint8Array {
   const entropy = u8aToBuffer(toEntropy(mnemonic));
   const salt = u8aToBuffer(stringToU8a(`mnemonic${password}`));
 

packages/util-crypto/src/schnorrkel/keypair/fromSeed.spec.ts

@@ -6,7 +6,7 @@ import '../test-polyfill';
 
 import { stringToU8a, u8aToHex } from '@polkadot/util/index';
 
-import { mnemonicToSeedEntropy } from '../../mnemonic';
+import { mnemonicToMiniSecret } from '../../mnemonic';
 import { schnorrkelKeypairFromSeed, schnorrkelWaitReady } from '../index';
 import tests from './testing';
 
@@ -29,7 +29,7 @@ describe('schnorrkelKeypairFromSeed', () => {
 
   tests.forEach(([mnemonic, , , secret], index) => {
     it(`creates valid against known (${index})`, () => {
-      const seed = mnemonicToSeedEntropy(mnemonic, 'Substrate');
+      const seed = mnemonicToMiniSecret(mnemonic, 'Substrate');
       const pair = schnorrkelKeypairFromSeed(seed);
 
       expect(