[CI Skip] 1.15.1

skip-checks: true

.123trigger

@@ -2,3 +2,4 @@
 1.4.2
 1.6.2
 1.11.2
+1.12.2

.eslintignore

@@ -1,3 +1,4 @@
+.eslintrc.js
 **/build/*
 **/coverage/*
 **/node_modules/*

CHANGELOG.md

@@ -1,3 +1,49 @@
+# 1.15.1 May 23, 2020
+
+- Adjust `keyPrefix()` with DoubleMap args (Thanks to https://github.com/KiChjang)
+- Added `WeightToFeeCoefficient` type (Thanks to https://github.com/danforbes)
+- Spelling fixes for console logs (Thanks to https://github.com/JoshOrndorff)
+- Spelling fixes in cookbook (Thanks to https://github.com/pawelnguyen)
+- Add `transactionVersion` getter in payloads (Thanks to https://github.com/amaurymartiny)
+- Adjust `stakerRewards` derive ordering for improved on-payout refreshes
+- Baseline implementation for derive caching infrastructure
+- Adjust `Multiplier` type to `Fixed128` (as per latest Substrate)
+- Updated to latest Substrate metadata & add new types
+
+# 1.14.1 May 14, 2020
+
+- **Important** `DispatchInfo` changed `paysFee` to an enum, if using an older version use, `DispatchInfo: 'DispatchInfoTo244'`
+- Align imOnline `Heartbeat` with Substrate (Thanks to https://github.com/arjanz)
+- Adust EVM types to include `Vicinity` (Thanks to https://github.com/drewstone)
+- Add `derive.democracy.locks` to return extended lock info based on votes
+- Ensure metadata is only requested once upon startup (bypass on upgrade subscriptions)
+- Expand testing around `ExtrinsicEra` construction
+- Expand `Registry` to direct access to available signed extensions (in addition to exposed types)
+- Cater for new `CheckSpecVersion` and `CheckTxVersion` signed extensions (with `SignerPayload` adjustments)
+- Cater for new `PrevalidateAttests` signed extensions (Polkadot claims)
+- Don't re-calculate submittable extrinsic hash on status checks (optimization)
+- Support for both old/new Tuples in `democracy.depositOf` in derives
+- Fix democracy `ProxyState` types
+- Extends types for Polkadot claims
+- Update `@polkadot/util` to 2.10
+
+# 1.13.1 May 6, 2020
+
+- Add support for ECDSA keypairs in extrinsic signers (Thanks to https://github.com/akru)
+- Rework type generation to use templating for better maintenance (Thanks to https://github.com/xlc)
+- Adjust extrinsic `sign/signAsync` signature to optional options (Thanks to https://github.com/KiChjang)
+- Support round-robin endpoints in the `WsProvider` with array of urls (Thanks to https://github.com/hoani)
+- Remove static test-only metadata for Polkadot dev chains (not updated regularly, Substrate dev to remain)
+- Revert error swallow for wrong preimage data (temp. override for Kusama upgrade with no migration)
+- Fix `.encodedLength` calculation on `[Type; N]` types
+- Add new `rpc.grandpa.roundState` RPC
+- Update types & metadata as per latest Substrate master
+- Adjust Centrifuge starting version
+
+# 1.12.2 Apr 30, 2020
+
+- Update `@polkadot/util` to stable 2.9.1 (sadly missed in the previous version, stable should match to stable)
+
 # 1.12.1 Apr 29, 2020
 
 - Fix `CompactAssignments` types (Thanks to https://github.com/arjanz)

docs/.vuepress/config.js

@@ -20,7 +20,8 @@ module.exports = {
       },
       { text: 'GitHub', link: 'https://github.com/polkadot-js/api' }
     ],
-    search: false,
+    search: true,
+    searchMaxSuggestions: 10,
     sidebar: [
       {
         title: 'Getting started',
@@ -97,5 +98,10 @@ module.exports = {
       ['/types/', '@polkadot/types'],
       '/CONTRIBUTING.md'
     ]
-  }
+  },
+  plugins: [
+    ['@vuepress/search', {
+      searchMaxSuggestions: 10
+    }]
+  ]
 };

docs/.vuepress/styles/palette.styl

@@ -1,5 +1,8 @@
 $accentColor = #fd8824
 $textColor = #2c3e50
+$borderColor = #eaecef
+$codeBgColor = #282c34
+$arrowBgColor = #ccc
 
 .home .hero .description,
 .home .feature h2,

docs/cookbook/blocks.md

@@ -25,7 +25,7 @@ signedBlock.block.extrinsics.forEach((ex, index) => {
 
 ## How do I extract the block author?
 
-Th block author is encoded inside the consensus logs for the block. To extract, you need to decode the log (which the API does do) and then map the index of the validator to the list of session validators. This extraction is however available on the api derive for new head subscriptions, which returns an extended header with the author populated (assuming that the digest logs are known).
+The block author is encoded inside the consensus logs for the block. To extract, you need to decode the log (which the API does do) and then map the index of the validator to the list of session validators. This extraction is however available on the api derive for new head subscriptions, which returns an extended header with the author populated (assuming that the digest logs are known).
 
 ```js
 // subscribe to all new headers (with extended info)

docs/start/FAQ.md

@@ -63,3 +63,16 @@ Due to these customizations and differences that bleed through to the transactio
 Recently Substrate master updated the `Weight` type from `u32` -> `u64`. This type is used in the `DispatchInfo` struct in the `system.ExtrinsicSuccess` events, to return the applied call weights as well as the resulting fees. Since the API master branch tracks Substrate master, this means the change has been applied by default, with the default set to `u64`.
 
 If you are on a chain that has not been upgraded yet, you need to add `Weight: 'u32'` to your types to allow for successful parsing of all events. Without this override, parsing will fail. As soon as one event in the `Vec<EventRecord>` structure from `system.events` fails to parse, all subsequent events are affected and the decoding will return an error.
+
+## When querying older blocks & events, it fails on parsing
+
+Upon initialization, the API is decorated with the [current on-chain metadata](basics.md#metadata) and (if [available](https://github.com/polkadot-js/api/tree/master/packages/types-known/src)) the specific types for the runtime version is added. This allows the API to decode all current (as per the on-chain info) state and extrinsics.
+
+When the chain is upgraded, new metadata (indicating new events, storage and extrinsics) is added on-chain. Additionally this generally goes hand in-hand with changes in types, i.e. fields may be addedd or removed or state that was used could now be migrated to new locations. In most cases, this would mean that the current chain information cannot be used to parse older entries pre-upgrade. A typical consequence of this is that any `.at(...)` or `.range(...)` queries pre-upgrade may fail.
+
+As of the current time, there is no automatic injection of older metadata (based on the runtime version at a specific block) nor decoration of older types if they are available. It can be emulated with what the API will do to handle these cases when it adds a `.historic(...)` query type -
+
+- querying the runtime version at the parent of older block you wish to retrieve
+- retrieving and injecting the metadata from the parent block (if the runtime version has changed)
+- inject any specific older types for the state at that point
+- finally retrieve the actual entry via an `.at(...)` query

docs/start/api.query.subs.md

@@ -26,7 +26,7 @@ If we had a query with parameters, i.e. where we wish to perform a query for a s
 
 // Subscribe to balance changes for our account
 const unsub = await api.query.system.account(ADDR, ({ nonce, data: balance }) => {
-  console.log(`free balance is ${balance.freee} with ${balance.reserved} reserved and a nonce of ${nonce}`);
+  console.log(`free balance is ${balance.free} with ${balance.reserved} reserved and a nonce of ${nonce}`);
 });
 ```
 

docs/start/rpc.custom.md

@@ -1,6 +1,6 @@
 # Custom RPC
 
-In previous sections we looked at the injection of types, as in use and defined in modules from the node. Another area that can be customized is RC definitions, and like with the types, the API provides the capability to add user-defined RPCs (in addition to the Polkadot/Substrate base) to allow use of these RPCs via the API.
+In previous sections we looked at the injection of types, as in use and defined in modules from the node. Another area that can be customized is RPC definitions, and like with the types, the API provides the capability to add user-defined RPCs (in addition to the Polkadot/Substrate base) to allow use of these RPCs via the API.
 
 ## Custom definitions
 
@@ -25,9 +25,14 @@ const api = await ApiPromise.create({
           }
         ],
         type: 'Balance'
-      }
-    }
-  }
+      },
+      anotherMethod: { ... },
+      ...
+    },
+    anotherModule: { ... },
+    ...
+  },
+  ...
 });
 ```
 

lerna.json

@@ -9,5 +9,5 @@
   "packages": [
     "packages/*"
   ],
-  "version": "1.12.1"
+  "version": "1.15.1"
 }

package.json

@@ -7,7 +7,7 @@
     "packages/*"
   ],
   "resolutions": {
-    "typescript": "^3.8.3"
+    "babel-core": "^7.0.0-bridge.0"
   },
   "scripts": {
     "build": "yarn build:interfaces && polkadot-dev-build-ts && (cd packages/typegen && copyfiles scripts/* build)",
@@ -24,13 +24,14 @@
     "test:watch": "polkadot-dev-run-test --watch"
   },
   "devDependencies": {
-    "@babel/core": "^7.9.0",
+    "@babel/core": "^7.9.6",
     "@babel/register": "^7.9.0",
-    "@babel/runtime": "^7.9.2",
-    "@polkadot/dev": "^0.52.11",
-    "@polkadot/ts": "^0.3.18",
+    "@babel/runtime": "^7.9.6",
+    "@polkadot/dev": "^0.53.5",
+    "@polkadot/ts": "^0.3.22",
     "@polkadot/typegen": "workspace:packages/typegen",
-    "@types/jest": "^25.2.1",
+    "@types/jest": "^25.2.3",
+    "@vuepress/plugin-search": "^1.5.0",
     "copyfiles": "^2.2.0"
   }
 }

packages/api-contract/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/api-contract",
-  "version": "1.12.1",
+  "version": "1.15.1",
   "description": "Interfaces for interacting with contracts and contract ABIs",
   "main": "index.js",
   "keywords": [
@@ -26,12 +26,12 @@
   },
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/api-contract#readme",
   "dependencies": {
-    "@babel/runtime": "^7.9.2",
-    "@polkadot/api": "1.12.1",
-    "@polkadot/rpc-core": "1.12.1",
-    "@polkadot/types": "1.12.1",
-    "@polkadot/util": "^2.9.0-beta.3",
-    "bn.js": "^5.1.1",
+    "@babel/runtime": "^7.9.6",
+    "@polkadot/api": "1.15.1",
+    "@polkadot/rpc-core": "1.15.1",
+    "@polkadot/types": "1.15.1",
+    "@polkadot/util": "^2.10.1",
+    "bn.js": "^5.1.2",
     "rxjs": "^6.5.5"
   }
 }

packages/api-derive/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/api-derive",
-  "version": "1.12.1",
+  "version": "1.15.1",
   "description": "Common functions used across Polkadot, derived from RPC calls and storage queries.",
   "main": "index.js",
   "keywords": [
@@ -27,18 +27,18 @@
   },
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/api-derive#readme",
   "dependencies": {
-    "@babel/runtime": "^7.9.2",
-    "@polkadot/api": "1.12.1",
-    "@polkadot/rpc-core": "1.12.1",
-    "@polkadot/rpc-provider": "1.12.1",
-    "@polkadot/types": "1.12.1",
-    "@polkadot/util": "^2.9.0-beta.3",
-    "@polkadot/util-crypto": "^2.9.0-beta.3",
-    "bn.js": "^5.1.1",
+    "@babel/runtime": "^7.9.6",
+    "@polkadot/api": "1.15.1",
+    "@polkadot/rpc-core": "1.15.1",
+    "@polkadot/rpc-provider": "1.15.1",
+    "@polkadot/types": "1.15.1",
+    "@polkadot/util": "^2.10.1",
+    "@polkadot/util-crypto": "^2.10.1",
+    "bn.js": "^5.1.2",
     "memoizee": "^0.4.14",
     "rxjs": "^6.5.5"
   },
   "devDependencies": {
-    "@polkadot/keyring": "^2.9.0-beta.3"
+    "@polkadot/keyring": "^2.10.1"
   }
 }

packages/api-derive/src/accounts/indexToId.ts

@@ -10,6 +10,7 @@ import { map, startWith } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
 import { ENUMSET_SIZE } from '@polkadot/types/generic/AccountIndex';
 import { Option, Vec } from '@polkadot/types';
+import { isFunction } from '@polkadot/util';
 
 import { memo } from '../util';
 
@@ -51,7 +52,7 @@ function query (api: ApiInterfaceRx, accountIndex: AccountIndex | string): Obser
 export function indexToId (api: ApiInterfaceRx): (accountIndex: AccountIndex | string) => Observable<AccountId | undefined> {
   return memo((accountIndex: AccountIndex | string): Observable<AccountId | undefined> =>
     api.query.indices
-      ? api.query.indices.accounts
+      ? isFunction(api.query.indices.accounts)
         ? query(api, accountIndex)
         : queryEnumSet(api, accountIndex)
       : of(undefined)

packages/api-derive/src/accounts/indexes.ts

@@ -10,6 +10,7 @@ import { Observable, of } from 'rxjs';
 import { map, startWith, switchMap } from 'rxjs/operators';
 import { ENUMSET_SIZE } from '@polkadot/types/generic/AccountIndex';
 import { Vec } from '@polkadot/types';
+import { isFunction } from '@polkadot/util';
 
 import { memo } from '../util';
 
@@ -72,7 +73,7 @@ function queryAccounts (api: ApiInterfaceRx): Observable<AccountIndexes> {
 export function indexes (api: ApiInterfaceRx): () => Observable<AccountIndexes> {
   return memo((): Observable<AccountIndexes> =>
     (api.query.indices
-      ? api.query.indices.accounts
+      ? isFunction(api.query.indices.accounts)
         ? queryAccounts(api)
         : queryEnumSet(api)
       : of({} as AccountIndexes)

packages/api-derive/src/balances/account.ts

@@ -9,6 +9,7 @@ import { DeriveBalancesAccount } from '../types';
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
+import { isFunction } from '@polkadot/util';
 
 import { memo } from '../util';
 
@@ -86,9 +87,9 @@ export function account (api: ApiInterfaceRx): (address: AccountIndex | AccountI
         (accountId
           ? combineLatest([
             of(accountId),
-            api.query.system.account
+            isFunction(api.query.system.account)
               ? queryCurrent(api, accountId)
-              : api.query.balances.account
+              : isFunction(api.query.balances.account)
                 ? queryBalancesAccount(api, accountId)
                 : queryBalancesFree(api, accountId)
           ])

packages/api-derive/src/balances/all.ts

@@ -10,7 +10,7 @@ import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
 import { Option, Vec } from '@polkadot/types';
-import { bnMax } from '@polkadot/util';
+import { bnMax, isFunction } from '@polkadot/util';
 
 import { memo } from '../util';
 
@@ -135,7 +135,7 @@ export function all (api: ApiInterfaceRx): (address: AccountIndex | AccountId |
           ? combineLatest([
             of(account),
             api.derive.chain.bestNumber(),
-            api.query.balances.account
+            isFunction(api.query.balances.account)
               ? queryCurrent(api, account.accountId)
               : queryOld(api, account.accountId)
           ])

packages/api-derive/src/balances/fees.ts

@@ -28,12 +28,12 @@ export function fees (api: ApiInterfaceRx): () => Observable<DeriveFees> {
   return memo((): Observable<DeriveFees> =>
     of([
       // deprecated - remove
-      (api.consts.balances?.creationFee as Balance) || api.registry.createType('Balance'),
-      (api.consts.balances?.transferFee as Balance) || api.registry.createType('Balance'),
+      (api.consts.balances?.creationFee as unknown as Balance) || api.registry.createType('Balance'),
+      (api.consts.balances?.transferFee as unknown as Balance) || api.registry.createType('Balance'),
 
       // current
       (api.consts.balances?.existentialDeposit as Balance) || api.registry.createType('Balance'),
-      (api.consts.transactionPayment?.transactionBaseFee as Balance) || api.registry.createType('Balance'),
+      (api.consts.transactionPayment?.transactionBaseFee as unknown as Balance) || api.registry.createType('Balance'),
       (api.consts.transactionPayment?.transactionByteFee as Balance) || api.registry.createType('Balance')
     ]).pipe(
       map(([creationFee, transferFee, existentialDeposit, transactionBaseFee, transactionByteFee]): DeriveFees => ({

packages/api-derive/src/collective/proposals.ts

@@ -9,6 +9,7 @@ import { DeriveCollectiveProposal } from '../types';
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { Option } from '@polkadot/types';
+import { isFunction } from '@polkadot/util';
 
 import { memo } from '../util';
 
@@ -30,7 +31,7 @@ function parse ([hashes, proposals, votes]: Result): DeriveCollectiveProposal[]
 
 export function proposals (api: ApiInterfaceRx, section: 'council' | 'technicalCommittee'): () => Observable<DeriveCollectiveProposal[]> {
   return memo((): Observable<DeriveCollectiveProposal[]> =>
-    api.query[section]?.proposals
+    isFunction(api.query[section]?.proposals)
       ? api.query[section].proposals().pipe(
         switchMap((hashes: Hash[]): Observable<Result> =>
           hashes.length

packages/api-derive/src/democracy/dispatchQueue.ts

@@ -10,7 +10,7 @@ import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
 import { Option, Vec, u64 } from '@polkadot/types';
-import { stringToHex } from '@polkadot/util';
+import { isFunction, stringToHex } from '@polkadot/util';
 
 import { memo } from '../util';
 
@@ -96,7 +96,7 @@ function queryScheduler (api: ApiInterfaceRx): Observable<DeriveDispatch[]> {
 
 export function dispatchQueue (api: ApiInterfaceRx): () => Observable<DeriveDispatch[]> {
   return memo((): Observable<DeriveDispatch[]> =>
-    api.query.scheduler?.agenda
+    isFunction(api.query.scheduler?.agenda)
       ? queryScheduler(api)
       : api.query.democracy.dispatchQueue
         ? queryQueue(api)

packages/api-derive/src/democracy/index.ts

@@ -3,6 +3,7 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 export * from './dispatchQueue';
+export * from './locks';
 export * from './nextExternal';
 export * from './preimage';
 export * from './preimages';

packages/api-derive/src/democracy/locks.ts

@@ -0,0 +1,66 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { AccountId, ReferendumInfo, ReferendumInfoTo239, VotingDirect, VotingDirectVote } from '@polkadot/types/interfaces';
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { DeriveDemocracyLock } from '../types';
+
+import BN from 'bn.js';
+import { Observable, of } from 'rxjs';
+import { map, switchMap } from 'rxjs/operators';
+import { Option } from '@polkadot/types';
+import { isUndefined } from '@polkadot/util';
+
+import { memo } from '../util';
+
+const LOCKUPS = [0, 1, 2, 4, 8, 16, 32];
+const ZERO = new BN(0);
+
+function parseLock (api: ApiInterfaceRx, [referendumId, accountVote]: VotingDirectVote, referendum: ReferendumInfo): DeriveDemocracyLock {
+  const { balance, vote } = accountVote.asStandard;
+  let unlockAt = ZERO;
+
+  if (referendum.isFinished) {
+    const { approved, end } = referendum.asFinished;
+
+    if ((approved.isTrue && vote.isAye) || (approved.isFalse && vote.isNay)) {
+      unlockAt = end.add(api.consts.democracy.enactmentPeriod.muln(LOCKUPS[vote.conviction.index]));
+    }
+  }
+
+  return { balance, isFinished: referendum.isFinished, referendumId, unlockAt, vote };
+}
+
+function directLocks (api: ApiInterfaceRx, { votes }: VotingDirect): Observable<DeriveDemocracyLock[]> {
+  if (!votes.length) {
+    return of([]);
+  }
+
+  return api.query.democracy.referendumInfoOf.multi<Option<ReferendumInfo | ReferendumInfoTo239>>(votes.map(([referendumId]) => referendumId)).pipe(
+    map((referendums) =>
+      votes
+        .map((vote, index): [VotingDirectVote, ReferendumInfo | ReferendumInfoTo239 | null] =>
+          [vote, referendums[index].unwrapOr(null)]
+        )
+        .filter((item): item is [VotingDirectVote, ReferendumInfo] =>
+          !!item[1] && isUndefined((item[1] as ReferendumInfoTo239).end) && item[0][1].isStandard
+        )
+        .map(([directVote, referendum]) =>
+          parseLock(api, directVote, referendum)
+        )
+    )
+  );
+}
+
+export function locks (api: ApiInterfaceRx): (accountId: string | AccountId) => Observable<DeriveDemocracyLock[]> {
+  return memo((accountId: string | AccountId): Observable<DeriveDemocracyLock[]> =>
+    api.query.democracy.votingOf(accountId).pipe(
+      switchMap((voting): Observable<DeriveDemocracyLock[]> =>
+        voting.isDirect
+          ? directLocks(api, voting.asDirect)
+          : of([])
+      )
+    )
+  );
+}

packages/api-derive/src/democracy/proposals.ts

@@ -10,27 +10,41 @@ import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
 import { Option, Vec } from '@polkadot/types';
+import { isFunction } from '@polkadot/util';
 
 import { memo } from '../util';
 
-type Depositors = Option<ITuple<[Balance, Vec<AccountId>]>>;
+type DepositorsNew = Option<ITuple<[Vec<AccountId>, Balance]>>;
+type DepositorsOld = Option<ITuple<[Balance, Vec<AccountId>]>>;
+type Depositors = DepositorsNew | DepositorsOld;
 type Proposals = Vec<ITuple<[PropIndex, Hash, AccountId]>>;
+type Result = [Proposals, (DeriveProposalImage | undefined)[], Depositors[]];
 
-function parse ([proposals, images, depositors]: [Proposals, (DeriveProposalImage | undefined)[], Depositors[]]): DeriveProposal[] {
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+function isNewDepositors (depositors: ITuple<[Vec<AccountId>, Balance]> | ITuple<[Balance, Vec<AccountId>]>): depositors is ITuple<[Vec<AccountId>, Balance]> {
+  // Detect balance...
+  // eslint-disable-next-line @typescript-eslint/unbound-method
+  return isFunction((depositors[1] as Balance).mul);
+}
+
+function parse ([proposals, images, optDepositors]: Result): DeriveProposal[] {
   return proposals
     .filter(([, , proposer], index): boolean =>
-      !!(depositors[index]?.isSome) && !proposer.isEmpty
+      !!(optDepositors[index]?.isSome) && !proposer.isEmpty
     )
     .map(([index, imageHash, proposer], proposalIndex): DeriveProposal => {
-      const [balance, seconds] = depositors[proposalIndex].unwrap();
+      const depositors = optDepositors[proposalIndex].unwrap();
 
       return {
-        balance,
+        ...(
+          isNewDepositors(depositors)
+            ? { balance: depositors[1], seconds: depositors[0] }
+            : { balance: depositors[0], seconds: depositors[1] }
+        ),
         image: images[proposalIndex],
         imageHash,
         index,
-        proposer,
-        seconds
+        proposer
       };
     });
 }

packages/api-derive/src/democracy/referendumsInfo.ts

@@ -10,6 +10,7 @@ import BN from 'bn.js';
 import { Observable, of, combineLatest } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { Option, Vec } from '@polkadot/types';
+import { isFunction } from '@polkadot/util';
 
 import { memo } from '../util';
 import { calcVotes, getStatus, parseImage } from './util';
@@ -95,7 +96,7 @@ export function _referendumVotes (api: ApiInterfaceRx): (referendum: DeriveRefer
   return memo((referendum: DeriveReferendum): Observable<DeriveReferendumVotes> =>
     combineLatest([
       api.derive.democracy.sqrtElectorate(),
-      api.query.democracy.votingOf
+      isFunction(api.query.democracy.votingOf)
         ? votesCurr(api, referendum.index)
         : votesPrev(api, referendum.index)
     ]).pipe(

packages/api-derive/src/democracy/types.ts

@@ -2,11 +2,19 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
+import type BN from 'bn.js';
 import { AccountId, Balance, BlockNumber, Hash, Proposal, PropIndex, ReferendumInfoTo239, ReferendumStatus, Vote, ReferendumIndex, VoteThreshold } from '@polkadot/types/interfaces';
 
-import BN from 'bn.js';
 import { Vec } from '@polkadot/types';
 
+export interface DeriveDemocracyLock {
+  balance: Balance;
+  isFinished: boolean;
+  referendumId: ReferendumIndex;
+  unlockAt: BN;
+  vote: Vote;
+}
+
 export interface DeriveDispatch {
   at: BlockNumber;
   index: ReferendumIndex;

packages/api-derive/src/session/indexes.ts

@@ -9,6 +9,7 @@ import { DeriveSessionIndexes } from '../types';
 import { Observable, of } from 'rxjs';
 import { map } from 'rxjs/operators';
 import { Option, u32 } from '@polkadot/types';
+import { isFunction } from '@polkadot/util';
 
 import { memo } from '../util';
 
@@ -79,7 +80,7 @@ export function indexes (api: ApiInterfaceRx): () => Observable<DeriveSessionInd
   return memo((): Observable<DeriveSessionIndexes> =>
     (
       api.query.session && api.query.staking
-        ? api.query.staking.activeEra
+        ? isFunction(api.query.staking.activeEra)
           ? query(api)
           : queryNoActive(api)
         : empty(api)

packages/api-derive/src/session/progress.ts

@@ -9,6 +9,7 @@ import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
 import { Option, u64 } from '@polkadot/types';
+import { isFunction } from '@polkadot/util';
 
 import { memo } from '../util';
 
@@ -77,9 +78,10 @@ function queryBabeNoHistory (api: ApiInterfaceRx): Observable<[DeriveSessionInfo
 export function progress (api: ApiInterfaceRx): () => Observable<DeriveSessionProgress> {
   return memo((): Observable<DeriveSessionProgress> =>
     api.consts.babe
-      ? (api.query.staking.erasStartSessionIndex
-        ? queryBabe(api) // 2.x with Babe
-        : queryBabeNoHistory(api)
+      ? (
+        isFunction(api.query.staking.erasStartSessionIndex)
+          ? queryBabe(api) // 2.x with Babe
+          : queryBabeNoHistory(api)
       ).pipe(
         map(([info, slots]: [DeriveSessionInfo, ResultSlotsFlat]): DeriveSessionProgress =>
           createDerive(api, info, slots)

packages/api-derive/src/staking/erasExposure.ts

@@ -10,10 +10,12 @@ import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { StorageKey } from '@polkadot/types';
 
-import { memo } from '../util';
+import { deriveCache, memo } from '../util';
 
 type KeysAndExposures = [StorageKey, Exposure][];
 
+const CACHE_KEY = 'eraExposure';
+
 function mapStakers (era: EraIndex, stakers: KeysAndExposures): DeriveEraExposure {
   const nominators: DeriveEraNominatorExposure = {};
   const validators: DeriveEraValidatorExposure = {};
@@ -34,26 +36,45 @@ function mapStakers (era: EraIndex, stakers: KeysAndExposures): DeriveEraExposur
   return { era, nominators, validators };
 }
 
+export function _eraExposure (api: ApiInterfaceRx): (era: EraIndex, withActive: boolean) => Observable<DeriveEraExposure> {
+  return memo((era: EraIndex, withActive: boolean): Observable<DeriveEraExposure> => {
+    const cacheKey = `${CACHE_KEY}-${era}`;
+    const cached = withActive
+      ? undefined
+      : deriveCache.get<DeriveEraExposure>(cacheKey);
+
+    return cached
+      ? of(cached)
+      : api.query.staking.erasStakersClipped.entries(era).pipe(
+        map((stakers): DeriveEraExposure => {
+          const value = mapStakers(era, stakers);
+
+          !withActive && deriveCache.set(cacheKey, value);
+
+          return value;
+        })
+      );
+  });
+}
+
 export function eraExposure (api: ApiInterfaceRx): (era: EraIndex) => Observable<DeriveEraExposure> {
   return memo((era: EraIndex): Observable<DeriveEraExposure> =>
-    api.query.staking.erasStakersClipped.entries(era).pipe(
-      map((stakers) => mapStakers(era, stakers))
-    )
+    api.derive.staking._eraExposure(era, true)
   );
 }
 
-export function _erasExposure (api: ApiInterfaceRx): (eras: EraIndex[]) => Observable<DeriveEraExposure[]> {
-  return memo((eras: EraIndex[]): Observable<DeriveEraExposure[]> =>
+export function _erasExposure (api: ApiInterfaceRx): (eras: EraIndex[], withActive: boolean) => Observable<DeriveEraExposure[]> {
+  return memo((eras: EraIndex[], withActive: boolean): Observable<DeriveEraExposure[]> =>
     eras.length
-      ? combineLatest(eras.map((era) => api.derive.staking.eraExposure(era)))
+      ? combineLatest(eras.map((era) => api.derive.staking._eraExposure(era, withActive)))
       : of([])
   );
 }
 
 export function erasExposure (api: ApiInterfaceRx): (withActive?: boolean) => Observable<DeriveEraExposure[]> {
-  return memo((withActive?: boolean): Observable<DeriveEraExposure[]> =>
+  return memo((withActive = false): Observable<DeriveEraExposure[]> =>
     api.derive.staking.erasHistoric(withActive).pipe(
-      switchMap((eras) => api.derive.staking._erasExposure(eras))
+      switchMap((eras) => api.derive.staking._erasExposure(eras, withActive))
     )
   );
 }

packages/api-derive/src/staking/erasPoints.ts

@@ -9,7 +9,9 @@ import { DeriveEraPoints, DeriveEraValPoints } from '../types';
 import { Observable, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 
-import { memo } from '../util';
+import { deriveCache, memo } from '../util';
+
+const CACHE_KEY = 'eraPoints';
 
 function mapValidators ({ individual }: EraRewardPoints): DeriveEraValPoints {
   return [...individual.entries()]
@@ -21,26 +23,48 @@ function mapValidators ({ individual }: EraRewardPoints): DeriveEraValPoints {
     }, {});
 }
 
-export function _erasPoints (api: ApiInterfaceRx): (eras: EraIndex[]) => Observable<DeriveEraPoints[]> {
-  return memo((eras: EraIndex[]): Observable<DeriveEraPoints[]> =>
-    eras.length
-      ? api.query.staking.erasRewardPoints.multi<EraRewardPoints>(eras).pipe(
-        map((points) =>
-          eras.map((era, index): DeriveEraPoints => ({
-            era,
-            eraPoints: points[index].total,
-            validators: mapValidators(points[index])
-          }))
-        )
-      )
-      : of([])
-  );
+function mapPoints (eras: EraIndex[], points: EraRewardPoints[]): DeriveEraPoints[] {
+  return eras.map((era, index): DeriveEraPoints => ({
+    era,
+    eraPoints: points[index].total,
+    validators: mapValidators(points[index])
+  }));
+}
+
+export function _erasPoints (api: ApiInterfaceRx): (eras: EraIndex[], withActive: boolean) => Observable<DeriveEraPoints[]> {
+  return memo((eras: EraIndex[], withActive: boolean): Observable<DeriveEraPoints[]> => {
+    if (!eras.length) {
+      return of([]);
+    }
+
+    const cached: DeriveEraPoints[] = withActive
+      ? []
+      : eras
+        .map((era) => deriveCache.get(`${CACHE_KEY}-${era}`))
+        .filter((value): value is DeriveEraPoints => !!value);
+    const remaining = eras.filter((era) => !cached.some((cached) => era.eq(cached.era)));
+
+    return !remaining.length
+      ? of(cached)
+      : api.query.staking.erasRewardPoints.multi<EraRewardPoints>(remaining).pipe(
+        map((points): DeriveEraPoints[] => {
+          const query = mapPoints(remaining, points);
+
+          !withActive && query.forEach((q) => deriveCache.set(`${CACHE_KEY}-${q.era}`, q));
+
+          return eras.map((era): DeriveEraPoints =>
+            cached.find((cached) => era.eq(cached.era)) ||
+            query.find((query) => era.eq(query.era)) as DeriveEraPoints
+          );
+        })
+      );
+  });
 }
 
 export function erasPoints (api: ApiInterfaceRx): (withActive?: boolean) => Observable<DeriveEraPoints[]> {
-  return memo((withActive?: boolean): Observable<DeriveEraPoints[]> =>
+  return memo((withActive = false): Observable<DeriveEraPoints[]> =>
     api.derive.staking.erasHistoric(withActive).pipe(
-      switchMap((eras) => api.derive.staking._erasPoints(eras))
+      switchMap((eras) => api.derive.staking._erasPoints(eras, withActive))
     )
   );
 }

packages/api-derive/src/staking/erasPrefs.ts

@@ -10,7 +10,9 @@ import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { StorageKey } from '@polkadot/types';
 
-import { memo } from '../util';
+import { deriveCache, memo } from '../util';
+
+const CACHE_KEY = 'eraPrefs';
 
 function mapPrefs (era: EraIndex, all: [StorageKey, ValidatorPrefs][]): DeriveEraPrefs {
   const validators: DeriveEraValPrefs = {};
@@ -22,26 +24,45 @@ function mapPrefs (era: EraIndex, all: [StorageKey, ValidatorPrefs][]): DeriveEr
   return { era, validators };
 }
 
+export function _eraPrefs (api: ApiInterfaceRx): (era: EraIndex, withActive: boolean) => Observable<DeriveEraPrefs> {
+  return memo((era: EraIndex, withActive: boolean): Observable<DeriveEraPrefs> => {
+    const cacheKey = `${CACHE_KEY}-${era}`;
+    const cached = withActive
+      ? undefined
+      : deriveCache.get<DeriveEraPrefs>(cacheKey);
+
+    return cached
+      ? of(cached)
+      : api.query.staking.erasValidatorPrefs.entries(era).pipe(
+        map((prefs): DeriveEraPrefs => {
+          const value = mapPrefs(era, prefs);
+
+          !withActive && deriveCache.set(cacheKey, value);
+
+          return value;
+        })
+      );
+  });
+}
+
 export function eraPrefs (api: ApiInterfaceRx): (era: EraIndex) => Observable<DeriveEraPrefs> {
   return memo((era: EraIndex): Observable<DeriveEraPrefs> =>
-    api.query.staking.erasValidatorPrefs.entries(era).pipe(
-      map((prefs) => mapPrefs(era, prefs))
-    )
+    api.derive.staking._eraPrefs(era, true)
   );
 }
 
-export function _erasPrefs (api: ApiInterfaceRx): (eras: EraIndex[]) => Observable<DeriveEraPrefs[]> {
-  return memo((eras: EraIndex[]): Observable<DeriveEraPrefs[]> =>
+export function _erasPrefs (api: ApiInterfaceRx): (eras: EraIndex[], withActive: boolean) => Observable<DeriveEraPrefs[]> {
+  return memo((eras: EraIndex[], withActive: boolean): Observable<DeriveEraPrefs[]> =>
     eras.length
-      ? combineLatest(eras.map((era) => api.derive.staking.eraPrefs(era)))
+      ? combineLatest(eras.map((era) => api.derive.staking._eraPrefs(era, withActive)))
       : of([])
   );
 }
 
 export function erasPrefs (api: ApiInterfaceRx): (withActive?: boolean) => Observable<DeriveEraPrefs[]> {
-  return memo((withActive?: boolean): Observable<DeriveEraPrefs[]> =>
+  return memo((withActive = false): Observable<DeriveEraPrefs[]> =>
     api.derive.staking.erasHistoric(withActive).pipe(
-      switchMap((eras) => api.derive.staking._erasPrefs(eras))
+      switchMap((eras) => api.derive.staking._erasPrefs(eras, withActive))
     )
   );
 }

packages/api-derive/src/staking/erasRewards.ts

@@ -10,25 +10,53 @@ import { Observable, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { Option } from '@polkadot/types';
 
-import { memo } from '../util';
+import { deriveCache, memo } from '../util';
 
-export function _erasRewards (api: ApiInterfaceRx): (eras: EraIndex[]) => Observable<DeriveEraRewards[]> {
-  return memo((eras: EraIndex[]): Observable<DeriveEraRewards[]> =>
-    eras.length
-      ? api.query.staking.erasValidatorReward.multi<Option<Balance>>(eras).pipe(
-        map((rewards) => eras.map((era, index): DeriveEraRewards => ({
-          era,
-          eraReward: rewards[index].unwrapOrDefault()
-        })))
-      )
-      : of([])
-  );
+const CACHE_KEY = 'eraRewards';
+
+function mapRewards (eras: EraIndex[], optRewards: Option<Balance>[]): DeriveEraRewards[] {
+  return eras.map((era, index): DeriveEraRewards => ({
+    era,
+    eraReward: optRewards[index].unwrapOrDefault()
+  }));
+}
+
+export function _erasRewards (api: ApiInterfaceRx): (eras: EraIndex[], withActive: boolean) => Observable<DeriveEraRewards[]> {
+  return memo((eras: EraIndex[], withActive: boolean): Observable<DeriveEraRewards[]> => {
+    if (!eras.length) {
+      return of([]);
+    }
+
+    const cached: DeriveEraRewards[] = withActive
+      ? []
+      : eras
+        .map((era) => deriveCache.get(`${CACHE_KEY}-${era}`))
+        .filter((value): value is DeriveEraRewards => !!value);
+    const remaining = eras.filter((era) => !cached.some((cached) => era.eq(cached.era)));
+
+    if (!remaining.length) {
+      return of(cached);
+    }
+
+    return api.query.staking.erasValidatorReward.multi<Option<Balance>>(remaining).pipe(
+      map((optRewards) => {
+        const query = mapRewards(remaining, optRewards);
+
+        !withActive && query.forEach((q) => deriveCache.set(`${CACHE_KEY}-${q.era}`, q));
+
+        return eras.map((era): DeriveEraRewards =>
+          cached.find((cached) => era.eq(cached.era)) ||
+          query.find((query) => era.eq(query.era)) as DeriveEraRewards
+        );
+      })
+    );
+  });
 }
 
 export function erasRewards (api: ApiInterfaceRx): (withActive?: boolean) => Observable<DeriveEraRewards[]> {
-  return memo((withActive?: boolean): Observable<DeriveEraRewards[]> =>
+  return memo((withActive = false): Observable<DeriveEraRewards[]> =>
     api.derive.staking.erasHistoric(withActive).pipe(
-      switchMap((eras) => api.derive.staking._erasRewards(eras))
+      switchMap((eras) => api.derive.staking._erasRewards(eras, withActive))
     )
   );
 }

packages/api-derive/src/staking/erasSlashes.ts

@@ -11,7 +11,9 @@ import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { Option, StorageKey } from '@polkadot/types';
 
-import { memo } from '../util';
+import { deriveCache, memo } from '../util';
+
+const CACHE_KEY = 'eraSlashes';
 
 function mapSlashes (era: EraIndex, noms: [StorageKey, Option<BalanceOf>][], vals: [StorageKey, Option<ITuple<[Perbill, BalanceOf]>>][]): DeriveEraSlashes {
   const nominators: DeriveEraValSlash = {};
@@ -28,31 +30,50 @@ function mapSlashes (era: EraIndex, noms: [StorageKey, Option<BalanceOf>][], val
   return { era, nominators, validators };
 }
 
+export function _eraSlashes (api: ApiInterfaceRx): (era: EraIndex, withActive: boolean) => Observable<DeriveEraSlashes> {
+  return memo((era: EraIndex, withActive: boolean): Observable<DeriveEraSlashes> => {
+    const cacheKey = `${CACHE_KEY}-${era}`;
+    const cached = withActive
+      ? undefined
+      : deriveCache.get<DeriveEraSlashes>(cacheKey);
+
+    return cached
+      ? of(cached)
+      : combineLatest([
+        api.query.staking.nominatorSlashInEra.entries(era),
+        api.query.staking.validatorSlashInEra.entries(era)
+      ]).pipe(
+        map(([noms, vals]): DeriveEraSlashes => {
+          const value = mapSlashes(era, noms, vals);
+
+          !withActive && deriveCache.set(cacheKey, value);
+
+          return value;
+        })
+      );
+  });
+}
+
 export function eraSlashes (api: ApiInterfaceRx): (era: EraIndex) => Observable<DeriveEraSlashes> {
   return memo((era: EraIndex): Observable<DeriveEraSlashes> =>
-    combineLatest([
-      api.query.staking.nominatorSlashInEra.entries(era),
-      api.query.staking.validatorSlashInEra.entries(era)
-    ]).pipe(
-      map(([noms, vals]) => mapSlashes(era, noms, vals))
-    )
+    api.derive.staking._eraSlashes(era, true)
   );
 }
 
-export function _erasSlashes (api: ApiInterfaceRx): (eras: EraIndex[]) => Observable<DeriveEraSlashes[]> {
-  return memo((eras: EraIndex[]): Observable<DeriveEraSlashes[]> =>
+export function _erasSlashes (api: ApiInterfaceRx): (eras: EraIndex[], withActive: boolean) => Observable<DeriveEraSlashes[]> {
+  return memo((eras: EraIndex[], withActive: boolean): Observable<DeriveEraSlashes[]> =>
     eras.length
       ? combineLatest(
-        eras.map((era) => api.derive.staking.eraSlashes(era))
+        eras.map((era) => api.derive.staking._eraSlashes(era, withActive))
       )
       : of([])
   );
 }
 
 export function erasSlashes (api: ApiInterfaceRx): (withActive?: boolean) => Observable<DeriveEraSlashes[]> {
-  return memo((withActive?: boolean): Observable<DeriveEraSlashes[]> =>
+  return memo((withActive = false): Observable<DeriveEraSlashes[]> =>
     api.derive.staking.erasHistoric(withActive).pipe(
-      switchMap((eras) => api.derive.staking._erasSlashes(eras))
+      switchMap((eras) => api.derive.staking._erasSlashes(eras, withActive))
     )
   );
 }

packages/api-derive/src/staking/ownSlashes.ts

@@ -11,7 +11,7 @@ import { map, switchMap } from 'rxjs/operators';
 import { memo } from '../util';
 
 export function ownSlashes (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean) => Observable<DeriveStakerSlashes[]> {
-  return memo((accountId: Uint8Array | string, withActive?: boolean): Observable<DeriveStakerSlashes[]> => {
+  return memo((accountId: Uint8Array | string, withActive = false): Observable<DeriveStakerSlashes[]> => {
     return api.derive.staking.erasHistoric(withActive).pipe(
       switchMap((eras) =>
         eras.length

packages/api-derive/src/staking/query.ts

@@ -10,6 +10,7 @@ import { DeriveStakingQuery } from '../types';
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { Option, Vec } from '@polkadot/types';
+import { isFunction } from '@polkadot/util';
 
 import { memo } from '../util';
 
@@ -87,7 +88,7 @@ export function queryWithQueued (api: ApiInterfaceRx): (accountId: Uint8Array |
     const stashId = api.registry.createType('AccountId', accountId);
 
     return (
-      api.query.staking.erasStakers
+      isFunction(api.query.staking.erasStakers)
         ? retrieveCurr(api, stashId)
         : retrievePrev(api, stashId)
     ).pipe(

packages/api-derive/src/staking/stakerExposure.ts

@@ -11,11 +11,11 @@ import { map, switchMap } from 'rxjs/operators';
 
 import { memo } from '../util';
 
-export function _stakerExposure (api: ApiInterfaceRx): (accountId: Uint8Array | string, eras: EraIndex[]) => Observable<DeriveStakerExposure[]> {
-  return memo((accountId: Uint8Array | string, eras: EraIndex[]): Observable<DeriveStakerExposure[]> => {
+export function _stakerExposure (api: ApiInterfaceRx): (accountId: Uint8Array | string, eras: EraIndex[], withActive: boolean) => Observable<DeriveStakerExposure[]> {
+  return memo((accountId: Uint8Array | string, eras: EraIndex[], withActive: boolean): Observable<DeriveStakerExposure[]> => {
     const stakerId = api.registry.createType('AccountId', accountId).toString();
 
-    return api.derive.staking._erasExposure(eras).pipe(
+    return api.derive.staking._erasExposure(eras, withActive).pipe(
       map((exposures): DeriveStakerExposure[] =>
         exposures.map(({ era, nominators: allNominators, validators: allValidators }): DeriveStakerExposure => {
           const isValidator = !!allValidators[stakerId];
@@ -38,9 +38,9 @@ export function _stakerExposure (api: ApiInterfaceRx): (accountId: Uint8Array |
 }
 
 export function stakerExposure (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean) => Observable<DeriveStakerExposure[]> {
-  return memo((accountId: Uint8Array | string, withActive?: boolean): Observable<DeriveStakerExposure[]> =>
+  return memo((accountId: Uint8Array | string, withActive = false): Observable<DeriveStakerExposure[]> =>
     api.derive.staking.erasHistoric(withActive).pipe(
-      switchMap((eras) => api.derive.staking._stakerExposure(accountId, eras))
+      switchMap((eras) => api.derive.staking._stakerExposure(accountId, eras, withActive))
     )
   );
 }

packages/api-derive/src/staking/stakerPoints.ts

@@ -11,11 +11,11 @@ import { map, switchMap } from 'rxjs/operators';
 
 import { memo } from '../util';
 
-export function _stakerPoints (api: ApiInterfaceRx): (accountId: Uint8Array | string, eras: EraIndex[]) => Observable<DeriveStakerPoints[]> {
-  return memo((accountId: Uint8Array | string, eras: EraIndex[]): Observable<DeriveStakerPoints[]> => {
+export function _stakerPoints (api: ApiInterfaceRx): (accountId: Uint8Array | string, eras: EraIndex[], withActive: boolean) => Observable<DeriveStakerPoints[]> {
+  return memo((accountId: Uint8Array | string, eras: EraIndex[], withActive: boolean): Observable<DeriveStakerPoints[]> => {
     const stakerId = api.registry.createType('AccountId', accountId).toString();
 
-    return api.derive.staking._erasPoints(eras).pipe(
+    return api.derive.staking._erasPoints(eras, withActive).pipe(
       map((points): DeriveStakerPoints[] =>
         points.map(({ era, eraPoints, validators }): DeriveStakerPoints => ({
           era,
@@ -28,9 +28,9 @@ export function _stakerPoints (api: ApiInterfaceRx): (accountId: Uint8Array | st
 }
 
 export function stakerPoints (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean) => Observable<DeriveStakerPoints[]> {
-  return memo((accountId: Uint8Array | string, withActive?: boolean): Observable<DeriveStakerPoints[]> =>
+  return memo((accountId: Uint8Array | string, withActive = false): Observable<DeriveStakerPoints[]> =>
     api.derive.staking.erasHistoric(withActive).pipe(
-      switchMap((eras) => api.derive.staking._stakerPoints(accountId, eras))
+      switchMap((eras) => api.derive.staking._stakerPoints(accountId, eras, withActive))
     )
   );
 }

packages/api-derive/src/staking/stakerRewards.ts

@@ -10,17 +10,18 @@ import BN from 'bn.js';
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { Option } from '@polkadot/types';
+import { isFunction } from '@polkadot/util';
 
 import { memo } from '../util';
 
-type Result = [DeriveEraPoints[], DeriveEraPrefs[], DeriveEraRewards[], DeriveStakerExposure[]];
+type ErasResult = [{ unwrapOr: (value: BN) => BN }, DeriveEraPoints[], DeriveEraPrefs[], DeriveEraRewards[]];
 
 const ZERO = new BN(0);
 const MIN_ONE = new BN(-1);
 const COMM_DIV = new BN(1_000_000_000);
 const MAX_ERAS = new BN(1_000_000_000);
 
-function parseRewards (api: ApiInterfaceRx, stashId: AccountId, [erasPoints, erasPrefs, erasRewards, exposures]: Result): DeriveStakerReward[] {
+function parseRewards (api: ApiInterfaceRx, stashId: AccountId, [, erasPoints, erasPrefs, erasRewards]: ErasResult, exposures: DeriveStakerExposure[]): DeriveStakerReward[] {
   return exposures.map(({ era, isEmpty, isValidator, nominating, validators: eraValidators }): DeriveStakerReward => {
     const { eraPoints, validators: allValPoints } = erasPoints.find((p) => p.era.eq(era)) || { eraPoints: ZERO, validators: {} };
     const { eraReward } = erasRewards.find((r) => r.era.eq(era)) || { eraReward: api.registry.createType('Balance') };
@@ -97,28 +98,29 @@ function filterEras (eras: EraIndex[], stakingLedger: StakingLedger): EraIndex[]
   return eras.filter((era) => filterEra(era, stakingLedger));
 }
 
-function filterRewards (api: ApiInterfaceRx, rewards: DeriveStakerReward[], stakingLedger: StakingLedger, withActive?: boolean): Observable<DeriveStakerReward[]> {
+function filterRewards (api: ApiInterfaceRx, eras: EraIndex[], migrateEra: BN, rewards: DeriveStakerReward[], stakingLedger: StakingLedger, withActive: boolean): Observable<DeriveStakerReward[]> {
   if (withActive) {
     return of(rewards);
   }
 
   const validators = uniqValidators(rewards);
 
-  return combineLatest([
-    api.query.staking.migrateEra
-      ? api.query.staking.migrateEra<Option<EraIndex>>()
-      : of({ unwrapOr: () => api.tx.staking.payoutStakers ? ZERO : MAX_ERAS }),
-    api.tx.staking.payoutStakers
+  return (
+    isFunction(api.tx.staking.payoutStakers)
       ? api.derive.staking.queryMulti(validators)
       : of([])
-  ]).pipe(
-    map(([optMigrate, queryValidators]): DeriveStakerReward[] => {
-      const migrateEra: BN = optMigrate.unwrapOr(ZERO);
+  ).pipe(
+    map((queryValidators): DeriveStakerReward[] => {
+      const filter = withActive
+        ? eras
+        : filterEras(eras, stakingLedger);
 
       return rewards
         .filter(({ isEmpty }) => !isEmpty)
         .filter((reward): boolean => {
-          if (reward.era.lt(migrateEra)) {
+          if (!filter.some((filter) => reward.era.eq(filter))) {
+            return false;
+          } else if (reward.era.lt(migrateEra)) {
             // we filter again here, the actual ledger may have changed, e.g. something has been claimed
             return filterEra(reward.era, stakingLedger);
           }
@@ -154,35 +156,41 @@ function filterRewards (api: ApiInterfaceRx, rewards: DeriveStakerReward[], stak
   );
 }
 
-export function _stakerRewards (api: ApiInterfaceRx): (accountId: Uint8Array | string, eras: EraIndex[], withActive?: boolean) => Observable<DeriveStakerReward[]> {
-  return memo((accountId: Uint8Array | string, _eras: EraIndex[], withActive?: boolean): Observable<DeriveStakerReward[]> =>
-    api.derive.staking.query(accountId).pipe(
-      switchMap(({ stakingLedger, stashId }): Observable<DeriveStakerReward[]> => {
+export function _stakerRewardsEras (api: ApiInterfaceRx): (eras: EraIndex[], withActive: boolean) => Observable<ErasResult> {
+  return memo((eras: EraIndex[], withActive: boolean): Observable<ErasResult> =>
+    combineLatest([
+      isFunction(api.query.staking.migrateEra)
+        ? api.query.staking.migrateEra<Option<EraIndex>>()
+        : of({ unwrapOr: () => isFunction(api.tx.staking.payoutStakers) ? ZERO : MAX_ERAS }),
+      api.derive.staking._erasPoints(eras, withActive),
+      api.derive.staking._erasPrefs(eras, withActive),
+      api.derive.staking._erasRewards(eras, withActive)
+    ])
+  );
+}
+
+export function _stakerRewards (api: ApiInterfaceRx): (accountId: Uint8Array | string, eras: EraIndex[], withActive: boolean) => Observable<DeriveStakerReward[]> {
+  return memo((accountId: Uint8Array | string, eras: EraIndex[], withActive: boolean): Observable<DeriveStakerReward[]> =>
+    combineLatest([
+      api.derive.staking.query(accountId),
+      api.derive.staking._stakerExposure(accountId, eras, withActive),
+      api.derive.staking._stakerRewardsEras(eras, withActive)
+    ]).pipe(
+      switchMap(([{ stakingLedger, stashId }, exposures, erasResult]): Observable<DeriveStakerReward[]> => {
+        const migrateEra: BN = erasResult[0].unwrapOr(ZERO);
+
         if (!stashId || !stakingLedger) {
           return of([]);
         }
 
-        const eras = withActive
-          ? _eras
-          : filterEras(_eras, stakingLedger);
-
-        return combineLatest([
-          api.derive.staking._erasPoints(eras),
-          api.derive.staking._erasPrefs(eras),
-          api.derive.staking._erasRewards(eras),
-          api.derive.staking._stakerExposure(stashId, eras)
-        ]).pipe(
-          switchMap((result): Observable<DeriveStakerReward[]> =>
-            filterRewards(api, parseRewards(api, stashId, result), stakingLedger, withActive)
-          )
-        );
+        return filterRewards(api, eras, migrateEra, parseRewards(api, stashId, erasResult, exposures), stakingLedger, withActive);
       })
     )
   );
 }
 
 export function stakerRewards (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean) => Observable<DeriveStakerReward[]> {
-  return memo((accountId: Uint8Array | string, withActive?: boolean): Observable<DeriveStakerReward[]> =>
+  return memo((accountId: Uint8Array | string, withActive = false): Observable<DeriveStakerReward[]> =>
     api.derive.staking.erasHistoric(withActive).pipe(
       switchMap((eras) => api.derive.staking._stakerRewards(accountId, eras, withActive))
     )
@@ -190,7 +198,7 @@ export function stakerRewards (api: ApiInterfaceRx): (accountId: Uint8Array | st
 }
 
 export function stakerRewardsMulti (api: ApiInterfaceRx): (accountIds: (Uint8Array | string)[], withActive?: boolean) => Observable<DeriveStakerReward[][]> {
-  return memo((accountIds: (Uint8Array | string)[], withActive?: boolean): Observable<DeriveStakerReward[][]> =>
+  return memo((accountIds: (Uint8Array | string)[], withActive = false): Observable<DeriveStakerReward[][]> =>
     accountIds.length
       ? api.derive.staking.erasHistoric(withActive).pipe(
         switchMap((eras) =>

packages/api-derive/src/staking/stakerSlashes.ts

@@ -11,11 +11,11 @@ import { map, switchMap } from 'rxjs/operators';
 
 import { memo } from '../util';
 
-export function _stakerSlashes (api: ApiInterfaceRx): (accountId: Uint8Array | string, eras: EraIndex[]) => Observable<DeriveStakerSlashes[]> {
-  return memo((accountId: Uint8Array | string, eras: EraIndex[]): Observable<DeriveStakerSlashes[]> => {
+export function _stakerSlashes (api: ApiInterfaceRx): (accountId: Uint8Array | string, eras: EraIndex[], withActive: boolean) => Observable<DeriveStakerSlashes[]> {
+  return memo((accountId: Uint8Array | string, eras: EraIndex[], withActive: boolean): Observable<DeriveStakerSlashes[]> => {
     const stakerId = api.registry.createType('AccountId', accountId).toString();
 
-    return api.derive.staking._erasSlashes(eras).pipe(
+    return api.derive.staking._erasSlashes(eras, withActive).pipe(
       map((slashes): DeriveStakerSlashes[] =>
         slashes.map(({ era, nominators, validators }): DeriveStakerSlashes => ({
           era,
@@ -27,9 +27,9 @@ export function _stakerSlashes (api: ApiInterfaceRx): (accountId: Uint8Array | s
 }
 
 export function stakerSlashes (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean) => Observable<DeriveStakerSlashes[]> {
-  return memo((accountId: Uint8Array | string, withActive?: boolean): Observable<DeriveStakerSlashes[]> =>
+  return memo((accountId: Uint8Array | string, withActive = false): Observable<DeriveStakerSlashes[]> =>
     api.derive.staking.erasHistoric(withActive).pipe(
-      switchMap((eras) => api.derive.staking._stakerSlashes(accountId, eras))
+      switchMap((eras) => api.derive.staking._stakerSlashes(accountId, eras, withActive))
     )
   );
 }

packages/api-derive/src/staking/types.ts

@@ -2,11 +2,10 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
+import type BN from 'bn.js';
 import { AccountId, Balance, EraIndex, Exposure, Keys, RewardDestination, RewardPoint, StakingLedger, ValidatorPrefs } from '@polkadot/types/interfaces';
 import { DeriveSessionIndexes } from '../session/types';
 
-import BN from 'bn.js';
-
 export type DeriveEraValPoints = Record<string, RewardPoint>;
 
 export type DeriveEraValPrefs = Record<string, ValidatorPrefs>;

packages/api-derive/src/tx/constants.ts

@@ -4,4 +4,11 @@
 
 import BN from 'bn.js';
 
+export const FALLBACK_MAX_HASH_COUNT = 250;
+
+// default here to 5 min eras, adjusted based on the actual blocktime
+export const FALLBACK_PERIOD = new BN(6 * 1000);
+
 export const MAX_FINALITY_LAG = new BN(5);
+
+export const MORTAL_PERIOD = new BN(5 * 60 * 1000);

packages/api-derive/src/tx/signingInfo.ts

@@ -5,23 +5,19 @@
 import { Header, Index } from '@polkadot/types/interfaces';
 import { AnyNumber, Codec, IExtrinsicEra } from '@polkadot/types/types';
 
-import BN from 'bn.js';
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
 import { isNumber, isUndefined } from '@polkadot/util';
 
+import { FALLBACK_PERIOD, MAX_FINALITY_LAG, MORTAL_PERIOD } from './constants';
+
 interface Result {
   header: Header | null;
   mortalLength: number;
   nonce: Index;
 }
 
-// default here to 5 min eras, adjusted based on the actual blocktime
-const FALLBACK_PERIOD = new BN(6 * 1000);
-const MAX_FINALITY_LAG = new BN(5);
-const MORTAL_PERIOD = new BN(5 * 60 * 1000);
-
 function latestNonce (api: ApiInterfaceRx, address: string): Observable<Index> {
   return api.derive.balances.account(address).pipe(
     map(({ accountNonce }): Index => accountNonce)

packages/api-derive/src/types.ts

@@ -2,9 +2,9 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
+import type BN from 'bn.js';
 import { AccountId, Balance, BalanceLock, BalanceLockTo212, BalanceOf, Bid, BidKind, BlockNumber, Hash, Index, Proposal, ProposalIndex, SetIndex, SocietyVote, StrikeCount, TreasuryProposal, Votes, VoteIndex, VouchingStatus } from '@polkadot/types/interfaces';
 
-import BN from 'bn.js';
 import { u32 } from '@polkadot/types';
 
 export * from './accounts/types';

packages/api-derive/src/util/cache.ts

@@ -0,0 +1,66 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { DeriveCache } from './types';
+
+import { deriveNoopCache } from './cacheImpl';
+
+interface CacheValue<T> {
+  v: T;
+  x: number;
+}
+
+const CHACHE_EXPIRY = 7 * (24 * 60) * (60 * 1000);
+
+let deriveCache: DeriveCache;
+
+function wrapCache (keyStart: string, cache: DeriveCache): DeriveCache {
+  return {
+    del: (partial: string): void => cache.del(`${keyStart}${partial}`),
+    forEach: cache.forEach,
+    get: <T = any> (partial: string): T | undefined => {
+      const key = `${keyStart}${partial}`;
+      const cached = cache.get<CacheValue<T>>(key);
+
+      if (cached) {
+        cached.x = Date.now();
+        cache.set(key, cached);
+
+        return cached.v;
+      }
+
+      return undefined;
+    },
+    set: (partial: string, v: any): void => {
+      cache.set(`${keyStart}${partial}`, { v, x: Date.now() });
+    }
+  };
+}
+
+function clearCache (cache: DeriveCache): void {
+  // clear all expired values
+  const now = Date.now();
+  const all: any[] = [];
+
+  cache.forEach((key: string, { x }: CacheValue<any>): void => {
+    ((now - x) > CHACHE_EXPIRY) && all.push(key);
+  });
+
+  // don't do delete inside loop, just in-case
+  all.forEach((key) => cache.del(key));
+}
+
+export function setDeriveCache (prefix = '', cache?: DeriveCache): void {
+  deriveCache = cache
+    ? wrapCache(`derive:${prefix}:`, cache)
+    : deriveNoopCache;
+
+  if (cache) {
+    clearCache(cache);
+  }
+}
+
+setDeriveCache();
+
+export { deriveCache };

packages/api-derive/src/util/cacheImpl.ts

@@ -0,0 +1,35 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { DeriveCache } from './types';
+
+const mapCache: Map<string, any> = new Map();
+
+const deriveMapCache: DeriveCache = {
+  del: (key: string): void => {
+    mapCache.delete(key);
+  },
+  forEach: (cb: (key: string, value: any) => void): void => {
+    const entries = mapCache.entries();
+
+    for (const entry in entries) {
+      cb(entry[0], entry[1]);
+    }
+  },
+  get: <T = any> (key: string): T | undefined => {
+    return mapCache.get(key);
+  },
+  set: (key: string, value: any): void => {
+    mapCache.set(key, value);
+  }
+};
+
+const deriveNoopCache: DeriveCache = {
+  del: (): void => undefined,
+  forEach: () => undefined,
+  get: (): undefined => undefined,
+  set: (_: string, value: any): any => value
+};
+
+export { deriveMapCache, deriveNoopCache };

packages/api-derive/src/util/index.ts

@@ -5,6 +5,8 @@
 import { drr } from '@polkadot/rpc-core/rxjs';
 
 export * from './approvalFlagsToBools';
+export * from './cache';
+export * from './cacheImpl';
 export * from './memo';
 
 export { drr };

packages/api-derive/src/util/types.ts

@@ -0,0 +1,10 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+export interface DeriveCache {
+  del: (key: string) => void;
+  forEach: (cb: (key: string, value: any) => void) => void;
+  get: <T = any> (key: string) => T | undefined;
+  set: (key: string, value: any) => void;
+}

packages/api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/api",
-  "version": "1.12.1",
+  "version": "1.15.1",
   "description": "Promise and RxJS wrappers around the Polkadot JS RPC",
   "main": "index.js",
   "keywords": [
@@ -26,21 +26,21 @@
   },
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/api#readme",
   "dependencies": {
-    "@babel/runtime": "^7.9.2",
-    "@polkadot/api-derive": "1.12.1",
-    "@polkadot/keyring": "^2.9.0-beta.3",
-    "@polkadot/metadata": "1.12.1",
-    "@polkadot/rpc-core": "1.12.1",
-    "@polkadot/rpc-provider": "1.12.1",
-    "@polkadot/types": "1.12.1",
-    "@polkadot/types-known": "1.12.1",
-    "@polkadot/util": "^2.9.0-beta.3",
-    "@polkadot/util-crypto": "^2.9.0-beta.3",
-    "bn.js": "^5.1.1",
-    "eventemitter3": "^4.0.0",
+    "@babel/runtime": "^7.9.6",
+    "@polkadot/api-derive": "1.15.1",
+    "@polkadot/keyring": "^2.10.1",
+    "@polkadot/metadata": "1.15.1",
+    "@polkadot/rpc-core": "1.15.1",
+    "@polkadot/rpc-provider": "1.15.1",
+    "@polkadot/types": "1.15.1",
+    "@polkadot/types-known": "1.15.1",
+    "@polkadot/util": "^2.10.1",
+    "@polkadot/util-crypto": "^2.10.1",
+    "bn.js": "^5.1.2",
+    "eventemitter3": "^4.0.4",
     "rxjs": "^6.5.5"
   },
   "devDependencies": {
-    "@polkadot/keyring": "^2.9.0-beta.3"
+    "@polkadot/keyring": "^2.10.1"
   }
 }

packages/api/src/augment/consts.ts

@@ -2,10 +2,12 @@
 /* eslint-disable */
 
 import { Codec } from '@polkadot/types/types';
+import { Vec } from '@polkadot/types/codec';
 import { u32, u64 } from '@polkadot/types/primitive';
 import { Balance, BalanceOf, BlockNumber, LockIdentifier, ModuleId, Moment, Percent, Permill, RuntimeDbWeight, Weight } from '@polkadot/types/interfaces/runtime';
 import { SessionIndex } from '@polkadot/types/interfaces/session';
 import { EraIndex } from '@polkadot/types/interfaces/staking';
+import { WeightToFeeCoefficient } from '@polkadot/types/interfaces/support';
 
 declare module '@polkadot/metadata/Decorated/consts/types' {
   export interface Constants {
@@ -51,6 +53,7 @@ declare module '@polkadot/metadata/Decorated/consts/types' {
       /**
        * The amount of funds a contract should deposit in order to offset
        * the cost of one byte.
+       * 
        * Let's suppose the deposit is 1,000 BU (balance units)/byte and the rent is 1 BU/byte/day,
        * then a contract with 1,000,000 BU that uses 1,000 bytes of storage would pay no rent.
        * But if the balance reduced to 500,000 BU and the storage stayed the same at 1,000,
@@ -59,6 +62,7 @@ declare module '@polkadot/metadata/Decorated/consts/types' {
       rentDepositOffset: AugmentedConst<BalanceOf>;
       /**
        * Number of block delay an extrinsic claim surcharge has.
+       * 
        * When claim surcharge is called by an extrinsic the rent is checked
        * for current_block - delay
        **/
@@ -86,6 +90,7 @@ declare module '@polkadot/metadata/Decorated/consts/types' {
       cooloffPeriod: AugmentedConst<BlockNumber>;
       /**
        * The minimum period of locking and the period between a proposal being approved and enacted.
+       * 
        * It should generally be a little more than the unstake period to ensure that
        * voting stakers have an opportunity to remove themselves from the system in the case where
        * they are on the losing side of a vote.
@@ -99,6 +104,10 @@ declare module '@polkadot/metadata/Decorated/consts/types' {
        * How often (in blocks) new public referenda are launched.
        **/
       launchPeriod: AugmentedConst<BlockNumber>;
+      /**
+       * The maximum number of votes for an account.
+       **/
+      maxVotes: AugmentedConst<u32>;
       /**
        * The minimum amount to be used as a deposit for a public referendum proposal.
        **/
@@ -213,6 +222,10 @@ declare module '@polkadot/metadata/Decorated/consts/types' {
        * The base weight of executing a block, independent of the transactions in the block.
        **/
       blockExecutionWeight: AugmentedConst<Weight>;
+      /**
+       * The maximum number of blocks to allow in mortal eras.
+       **/
+      blockHashCount: AugmentedConst<BlockNumber>;
       /**
        * The weight of runtime database operations the runtime can invoke.
        **/
@@ -246,6 +259,10 @@ declare module '@polkadot/metadata/Decorated/consts/types' {
        * The fee to be paid for making a transaction; the per-byte portion.
        **/
       transactionByteFee: AugmentedConst<BalanceOf>;
+      /**
+       * The polynomial that is applied in order to derive fee from weight.
+       **/
+      weightToFee: AugmentedConst<Vec<WeightToFeeCoefficient>>;
     };
     treasury: {
       [index: string]: AugmentedConst<object & Codec>;

packages/api/src/augment/query.ts

@@ -5,19 +5,19 @@ import { AnyNumber, ITuple, Observable } from '@polkadot/types/types';
 import { Option, U8aFixed, Vec } from '@polkadot/types/codec';
 import { Bytes, Data, bool, u32, u64 } from '@polkadot/types/primitive';
 import { UncleEntryItem } from '@polkadot/types/interfaces/authorship';
-import { BabeAuthorityWeight, MaybeVrf, Randomness } from '@polkadot/types/interfaces/babe';
+import { BabeAuthorityWeight, MaybeRandomness, NextConfigDescriptor, Randomness } from '@polkadot/types/interfaces/babe';
 import { AccountData, BalanceLock, ReleasesBalances } from '@polkadot/types/interfaces/balances';
 import { ProposalIndex, Votes } from '@polkadot/types/interfaces/collective';
-import { AuthorityId, RawVRFOutput } from '@polkadot/types/interfaces/consensus';
+import { AuthorityId } from '@polkadot/types/interfaces/consensus';
 import { CodeHash, ContractInfo, PrefabWasmModule, Schedule } from '@polkadot/types/interfaces/contracts';
-import { PreimageStatus, PropIndex, Proposal, ProxyState, ReferendumIndex, ReferendumInfo, Voting } from '@polkadot/types/interfaces/democracy';
+import { PreimageStatus, PropIndex, Proposal, ProxyState, ReferendumIndex, ReferendumInfo, ReleasesDemocracy, Voting } from '@polkadot/types/interfaces/democracy';
 import { VoteThreshold } from '@polkadot/types/interfaces/elections';
 import { SetId, StoredPendingChange, StoredState } from '@polkadot/types/interfaces/grandpa';
 import { RegistrarInfo, Registration } from '@polkadot/types/interfaces/identity';
 import { AuthIndex } from '@polkadot/types/interfaces/imOnline';
 import { DeferredOffenceOf, Kind, OffenceDetails, OpaqueTimeSlot, ReportIdOf } from '@polkadot/types/interfaces/offences';
 import { ActiveRecovery, RecoveryConfig } from '@polkadot/types/interfaces/recovery';
-import { AccountId, AccountIndex, Balance, BalanceOf, BlockNumber, Hash, KeyTypeId, Moment, Perbill, ValidatorId, Weight } from '@polkadot/types/interfaces/runtime';
+import { AccountId, AccountIndex, Balance, BalanceOf, BlockNumber, ExtrinsicsWeight, Hash, KeyTypeId, Moment, Perbill, ValidatorId } from '@polkadot/types/interfaces/runtime';
 import { Scheduled, TaskAddress } from '@polkadot/types/interfaces/scheduler';
 import { Keys, SessionIndex } from '@polkadot/types/interfaces/session';
 import { Bid, BidKind, SocietyVote, StrikeCount, VouchingStatus } from '@polkadot/types/interfaces/society';
@@ -69,21 +69,28 @@ declare module '@polkadot/api/types/storage' {
        * Temporary value (cleared at block finalization) which is `Some`
        * if per-block initialization has already been called for current block.
        **/
-      initialized: AugmentedQuery<ApiType, () => Observable<Option<MaybeVrf>>> & QueryableStorageEntry<ApiType>;
+      initialized: AugmentedQuery<ApiType, () => Observable<Option<MaybeRandomness>>> & QueryableStorageEntry<ApiType>;
       /**
        * How late the current block is compared to its parent.
+       * 
        * This entry is populated as part of block execution and is cleaned up
        * on block finalization. Querying this storage entry outside of block
        * execution context should always yield zero.
        **/
       lateness: AugmentedQuery<ApiType, () => Observable<BlockNumber>> & QueryableStorageEntry<ApiType>;
+      /**
+       * Next epoch configuration, if changed.
+       **/
+      nextEpochConfig: AugmentedQuery<ApiType, () => Observable<Option<NextConfigDescriptor>>> & QueryableStorageEntry<ApiType>;
       /**
        * Next epoch randomness.
        **/
       nextRandomness: AugmentedQuery<ApiType, () => Observable<Randomness>> & QueryableStorageEntry<ApiType>;
       /**
        * The epoch randomness for the *current* epoch.
+       * 
        * # Security
+       * 
        * This MUST NOT be used for gambling, as it can be influenced by a
        * malicious validator in the short term. It MAY be used in many
        * cryptographic protocols, however, so long as one remembers that this
@@ -94,22 +101,29 @@ declare module '@polkadot/api/types/storage' {
       randomness: AugmentedQuery<ApiType, () => Observable<Randomness>> & QueryableStorageEntry<ApiType>;
       /**
        * Randomness under construction.
+       * 
        * We make a tradeoff between storage accesses and list length.
        * We store the under-construction randomness in segments of up to
        * `UNDER_CONSTRUCTION_SEGMENT_LENGTH`.
+       * 
        * Once a segment reaches this length, we begin the next one.
        * We reset all segments and return to `0` at the beginning of every
        * epoch.
        **/
       segmentIndex: AugmentedQuery<ApiType, () => Observable<u32>> & QueryableStorageEntry<ApiType>;
-      underConstruction: AugmentedQuery<ApiType, (arg: u32 | AnyNumber | Uint8Array) => Observable<Vec<RawVRFOutput>>> & QueryableStorageEntry<ApiType>;
+      /**
+       * TWOX-NOTE: `SegmentIndex` is an increasing integer, so this is okay.
+       **/
+      underConstruction: AugmentedQuery<ApiType, (arg: u32 | AnyNumber | Uint8Array) => Observable<Vec<Randomness>>> & QueryableStorageEntry<ApiType>;
     };
     balances: {
       [index: string]: QueryableStorageEntry<ApiType>;
       /**
        * The balance of an account.
+       * 
        * NOTE: THIS MAY NEVER BE IN EXISTENCE AND YET HAVE A `total().is_zero()`. If the total
        * is ever zero, then the entry *MUST* be removed.
+       * 
        * NOTE: This is only used in the case that this module is used to store balances.
        **/
       account: AugmentedQuery<ApiType, (arg: AccountId | string | Uint8Array) => Observable<AccountData>> & QueryableStorageEntry<ApiType>;
@@ -120,6 +134,7 @@ declare module '@polkadot/api/types/storage' {
       locks: AugmentedQuery<ApiType, (arg: AccountId | string | Uint8Array) => Observable<Vec<BalanceLock>>> & QueryableStorageEntry<ApiType>;
       /**
        * Storage version of the pallet.
+       * 
        * This is set to v2.0.0 for new networks.
        **/
       storageVersion: AugmentedQuery<ApiType, () => Observable<ReleasesBalances>> & QueryableStorageEntry<ApiType>;
@@ -140,6 +155,8 @@ declare module '@polkadot/api/types/storage' {
       codeStorage: AugmentedQuery<ApiType, (arg: CodeHash | string | Uint8Array) => Observable<Option<PrefabWasmModule>>> & QueryableStorageEntry<ApiType>;
       /**
        * The code associated with a given account.
+       * 
+       * TWOX-NOTE: SAFE since `AccountId` is a secure hash.
        **/
       contractInfoOf: AugmentedQuery<ApiType, (arg: AccountId | string | Uint8Array) => Observable<Option<ContractInfo>>> & QueryableStorageEntry<ApiType>;
       /**
@@ -192,8 +209,10 @@ declare module '@polkadot/api/types/storage' {
       cancellations: AugmentedQuery<ApiType, (arg: Hash | string | Uint8Array) => Observable<bool>> & QueryableStorageEntry<ApiType>;
       /**
        * Those who have locked a deposit.
+       * 
+       * TWOX-NOTE: Safe, as increasing integer keys are safe.
        **/
-      depositOf: AugmentedQuery<ApiType, (arg: PropIndex | AnyNumber | Uint8Array) => Observable<Option<ITuple<[BalanceOf, Vec<AccountId>]>>>> & QueryableStorageEntry<ApiType>;
+      depositOf: AugmentedQuery<ApiType, (arg: PropIndex | AnyNumber | Uint8Array) => Observable<Option<ITuple<[Vec<AccountId>, BalanceOf]>>>> & QueryableStorageEntry<ApiType>;
       /**
        * True if the last referendum tabled was submitted externally. False if it was a public
        * proposal.
@@ -202,6 +221,8 @@ declare module '@polkadot/api/types/storage' {
       /**
        * Accounts for which there are locks in action which may be removed at some point in the
        * future. The value is the block number at which the lock expires and may be removed.
+       * 
+       * TWOX-NOTE: OK ― `AccountId` is a secure hash.
        **/
       locks: AugmentedQuery<ApiType, (arg: AccountId | string | Uint8Array) => Observable<Option<BlockNumber>>> & QueryableStorageEntry<ApiType>;
       /**
@@ -224,6 +245,8 @@ declare module '@polkadot/api/types/storage' {
       /**
        * Who is able to vote for whom. Value is the fund-holding account, key is the
        * vote-transaction-sending account.
+       * 
+       * TWOX-NOTE: OK ― `AccountId` is a secure hash.
        **/
       proxy: AugmentedQuery<ApiType, (arg: AccountId | string | Uint8Array) => Observable<Option<ProxyState>>> & QueryableStorageEntry<ApiType>;
       /**
@@ -240,11 +263,21 @@ declare module '@polkadot/api/types/storage' {
       referendumCount: AugmentedQuery<ApiType, () => Observable<ReferendumIndex>> & QueryableStorageEntry<ApiType>;
       /**
        * Information concerning any given referendum.
+       * 
+       * TWOX-NOTE: SAFE as indexes are not under an attacker’s control.
        **/
       referendumInfoOf: AugmentedQuery<ApiType, (arg: ReferendumIndex | AnyNumber | Uint8Array) => Observable<Option<ReferendumInfo>>> & QueryableStorageEntry<ApiType>;
+      /**
+       * Storage version of the pallet.
+       * 
+       * New networks start with last version.
+       **/
+      storageVersion: AugmentedQuery<ApiType, () => Observable<Option<ReleasesDemocracy>>> & QueryableStorageEntry<ApiType>;
       /**
        * All votes for a particular voter. We store the balance for the number of votes that we
        * have recorded. The second item is the total amount of delegations, that will be added.
+       * 
+       * TWOX-NOTE: SAFE as `AccountId`s are crypto hashes anyway.
        **/
       votingOf: AugmentedQuery<ApiType, (arg: AccountId | string | Uint8Array) => Observable<Voting>> & QueryableStorageEntry<ApiType>;
     };
@@ -269,6 +302,8 @@ declare module '@polkadot/api/types/storage' {
       runnersUp: AugmentedQuery<ApiType, () => Observable<Vec<ITuple<[AccountId, BalanceOf]>>>> & QueryableStorageEntry<ApiType>;
       /**
        * Votes and locked stake of a particular voter.
+       * 
+       * TWOX-NOTE: SAFE as `AccountId` is a crypto hash
        **/
       voting: AugmentedQuery<ApiType, (arg: AccountId | string | Uint8Array) => Observable<ITuple<[BalanceOf, Vec<AccountId>]>>> & QueryableStorageEntry<ApiType>;
     };
@@ -290,6 +325,8 @@ declare module '@polkadot/api/types/storage' {
       /**
        * A mapping from grandpa set ID to the index of the *most recent* session for which its
        * members were responsible.
+       * 
+       * TWOX-NOTE: `SetId` is not under user control.
        **/
       setIdSession: AugmentedQuery<ApiType, (arg: SetId | AnyNumber | Uint8Array) => Observable<Option<SessionIndex>>> & QueryableStorageEntry<ApiType>;
       /**
@@ -305,17 +342,23 @@ declare module '@polkadot/api/types/storage' {
       [index: string]: QueryableStorageEntry<ApiType>;
       /**
        * Information that is pertinent to identify the entity behind an account.
+       * 
+       * TWOX-NOTE: OK ― `AccountId` is a secure hash.
        **/
       identityOf: AugmentedQuery<ApiType, (arg: AccountId | string | Uint8Array) => Observable<Option<Registration>>> & QueryableStorageEntry<ApiType>;
       /**
        * The set of registrars. Not expected to get very big as can only be added through a
        * special origin (likely a council motion).
+       * 
        * The index into this can be cast to `RegistrarIndex` to get a valid value.
        **/
       registrars: AugmentedQuery<ApiType, () => Observable<Vec<Option<RegistrarInfo>>>> & QueryableStorageEntry<ApiType>;
       /**
        * Alternative "sub" identities of this account.
+       * 
        * The first item is the deposit, the second is a vector of the accounts.
+       * 
+       * TWOX-NOTE: OK ― `AccountId` is a secure hash.
        **/
       subsOf: AugmentedQuery<ApiType, (arg: AccountId | string | Uint8Array) => Observable<ITuple<[BalanceOf, Vec<AccountId>]>>> & QueryableStorageEntry<ApiType>;
       /**
@@ -333,6 +376,7 @@ declare module '@polkadot/api/types/storage' {
       authoredBlocks: AugmentedQueryDoubleMap<ApiType, (key1: SessionIndex | AnyNumber | Uint8Array, key2: ValidatorId | string | Uint8Array) => Observable<u32>> & QueryableStorageEntry<ApiType>;
       /**
        * The block number after which it's ok to send heartbeats in current session.
+       * 
        * At the beginning of each session we set this to a value that should
        * fall roughly in the middle of the session duration.
        * The idea is to first wait for the validators to produce a block
@@ -373,7 +417,9 @@ declare module '@polkadot/api/types/storage' {
       reports: AugmentedQuery<ApiType, (arg: ReportIdOf | string | Uint8Array) => Observable<Option<OffenceDetails>>> & QueryableStorageEntry<ApiType>;
       /**
        * Enumerates all reports of a kind along with the time they happened.
+       * 
        * All reports are sorted by the time of offence.
+       * 
        * Note that the actual type of this mapping is `Vec<u8>`, this is because values of
        * different types are not supported at the moment so we are doing the manual serialization.
        **/
@@ -392,12 +438,14 @@ declare module '@polkadot/api/types/storage' {
       [index: string]: QueryableStorageEntry<ApiType>;
       /**
        * Active recovery attempts.
+       * 
        * First account is the account to be recovered, and the second account
        * is the user trying to recover the account.
        **/
       activeRecoveries: AugmentedQueryDoubleMap<ApiType, (key1: AccountId | string | Uint8Array, key2: AccountId | string | Uint8Array) => Observable<Option<ActiveRecovery>>> & QueryableStorageEntry<ApiType>;
       /**
        * The list of allowed proxy accounts.
+       * 
        * Map from the user who can access it to the recovered account.
        **/
       proxy: AugmentedQuery<ApiType, (arg: AccountId | string | Uint8Array) => Observable<Option<AccountId>>> & QueryableStorageEntry<ApiType>;
@@ -425,6 +473,7 @@ declare module '@polkadot/api/types/storage' {
       currentIndex: AugmentedQuery<ApiType, () => Observable<SessionIndex>> & QueryableStorageEntry<ApiType>;
       /**
        * Indices of disabled validators.
+       * 
        * The set is cleared when `on_session_ending` returns a new set of identities.
        **/
       disabledValidators: AugmentedQuery<ApiType, () => Observable<Vec<u32>>> & QueryableStorageEntry<ApiType>;
@@ -523,6 +572,7 @@ declare module '@polkadot/api/types/storage' {
       [index: string]: QueryableStorageEntry<ApiType>;
       /**
        * The active era information, it holds index and start.
+       * 
        * The active era is the era currently rewarded.
        * Validator set of this era must be equal to `SessionInterface::validators`.
        **/
@@ -533,6 +583,7 @@ declare module '@polkadot/api/types/storage' {
       bonded: AugmentedQuery<ApiType, (arg: AccountId | string | Uint8Array) => Observable<Option<AccountId>>> & QueryableStorageEntry<ApiType>;
       /**
        * A mapping from still-bonded eras to the first session index of that era.
+       * 
        * Must contains information for eras for the range:
        * `[active_era - bounding_duration; active_era]`
        **/
@@ -544,6 +595,7 @@ declare module '@polkadot/api/types/storage' {
       canceledSlashPayout: AugmentedQuery<ApiType, () => Observable<BalanceOf>> & QueryableStorageEntry<ApiType>;
       /**
        * The current era index.
+       * 
        * This is the latest planned era, depending on how the Session pallet queues the validator
        * set, it might be active or not.
        **/
@@ -564,18 +616,23 @@ declare module '@polkadot/api/types/storage' {
       erasRewardPoints: AugmentedQuery<ApiType, (arg: EraIndex | AnyNumber | Uint8Array) => Observable<EraRewardPoints>> & QueryableStorageEntry<ApiType>;
       /**
        * Exposure of validator at era.
+       * 
        * This is keyed first by the era index to allow bulk deletion and then the stash account.
+       * 
        * Is it removed after `HISTORY_DEPTH` eras.
        * If stakers hasn't been set or has been removed then empty exposure is returned.
        **/
       erasStakers: AugmentedQueryDoubleMap<ApiType, (key1: EraIndex | AnyNumber | Uint8Array, key2: AccountId | string | Uint8Array) => Observable<Exposure>> & QueryableStorageEntry<ApiType>;
       /**
        * Clipped Exposure of validator at era.
+       * 
        * This is similar to [`ErasStakers`] but number of nominators exposed is reduced to the
        * `T::MaxNominatorRewardedPerValidator` biggest stakers.
        * (Note: the field `total` and `own` of the exposure remains unchanged).
        * This is used to limit the i/o cost for the nominator payout.
+       * 
        * This is keyed fist by the era index to allow bulk deletion and then the stash account.
+       * 
        * Is it removed after `HISTORY_DEPTH` eras.
        * If stakers hasn't been set or has been removed then empty exposure is returned.
        **/
@@ -591,12 +648,15 @@ declare module '@polkadot/api/types/storage' {
       erasTotalStake: AugmentedQuery<ApiType, (arg: EraIndex | AnyNumber | Uint8Array) => Observable<BalanceOf>> & QueryableStorageEntry<ApiType>;
       /**
        * Similar to `ErasStakers`, this holds the preferences of validators.
+       * 
        * This is keyed first by the era index to allow bulk deletion and then the stash account.
+       * 
        * Is it removed after `HISTORY_DEPTH` eras.
        **/
       erasValidatorPrefs: AugmentedQueryDoubleMap<ApiType, (key1: EraIndex | AnyNumber | Uint8Array, key2: AccountId | string | Uint8Array) => Observable<ValidatorPrefs>> & QueryableStorageEntry<ApiType>;
       /**
        * The total validator era payout for the last `HISTORY_DEPTH` eras.
+       * 
        * Eras that haven't finished yet or has been removed doesn't have reward.
        **/
       erasValidatorReward: AugmentedQuery<ApiType, (arg: EraIndex | AnyNumber | Uint8Array) => Observable<Option<BalanceOf>>> & QueryableStorageEntry<ApiType>;
@@ -606,7 +666,9 @@ declare module '@polkadot/api/types/storage' {
       forceEra: AugmentedQuery<ApiType, () => Observable<Forcing>> & QueryableStorageEntry<ApiType>;
       /**
        * Number of eras to keep in history.
+       * 
        * Information is kept for eras in `[current_era - history_depth; current_era]`.
+       * 
        * Must be more than the number of eras delayed by session otherwise.
        * I.e. active era must always be in history.
        * I.e. `active_era > current_era - history_depth` must be guaranteed.
@@ -663,6 +725,7 @@ declare module '@polkadot/api/types/storage' {
       slashingSpans: AugmentedQuery<ApiType, (arg: AccountId | string | Uint8Array) => Observable<Option<SlashingSpans>>> & QueryableStorageEntry<ApiType>;
       /**
        * The percentage of the slash that is distributed to reporters.
+       * 
        * The rest of the slashed value is handled by the `Slash`.
        **/
       slashRewardFraction: AugmentedQuery<ApiType, () => Observable<Perbill>> & QueryableStorageEntry<ApiType>;
@@ -684,6 +747,7 @@ declare module '@polkadot/api/types/storage' {
       /**
        * True if network has been upgraded to this version.
        * Storage version of the pallet.
+       * 
        * This is set to v3.0.0 for new networks.
        **/
       storageVersion: AugmentedQuery<ApiType, () => Observable<ReleasesStaking>> & QueryableStorageEntry<ApiType>;
@@ -722,14 +786,14 @@ declare module '@polkadot/api/types/storage' {
        * Total length (in bytes) for all extrinsics put together, for the current block.
        **/
       allExtrinsicsLen: AugmentedQuery<ApiType, () => Observable<Option<u32>>> & QueryableStorageEntry<ApiType>;
-      /**
-       * Total weight for all extrinsics put together, for the current block.
-       **/
-      allExtrinsicsWeight: AugmentedQuery<ApiType, () => Observable<Option<Weight>>> & QueryableStorageEntry<ApiType>;
       /**
        * Map of block numbers to block hashes.
        **/
       blockHash: AugmentedQuery<ApiType, (arg: BlockNumber | AnyNumber | Uint8Array) => Observable<Hash>> & QueryableStorageEntry<ApiType>;
+      /**
+       * The current weight for the block.
+       **/
+      blockWeight: AugmentedQuery<ApiType, () => Observable<ExtrinsicsWeight>> & QueryableStorageEntry<ApiType>;
       /**
        * Digest of the current block, also part of the block header.
        **/
@@ -745,9 +809,11 @@ declare module '@polkadot/api/types/storage' {
       /**
        * Mapping between a topic (represented by T::Hash) and a vector of indexes
        * of events in the `<Events<T>>` list.
+       * 
        * All topic vectors have deterministic storage locations depending on the topic. This
        * allows light-clients to leverage the changes trie storage tracking mechanism and
        * in case of changes fetch the list of events of interest.
+       * 
        * The value has the type `(T::BlockNumber, EventIndex)` because if we used only just
        * the `EventIndex` then in case if the topic has the same contents on the next block
        * no notification will be triggered thus the event might be lost.

packages/api/src/augment/rpc.ts

@@ -13,6 +13,7 @@ import { AuthorityId } from '@polkadot/types/interfaces/consensus';
 import { ContractCallRequest, ContractExecResult } from '@polkadot/types/interfaces/contracts';
 import { CreatedBlock } from '@polkadot/types/interfaces/engine';
 import { Extrinsic } from '@polkadot/types/interfaces/extrinsics';
+import { ReportedRoundStates } from '@polkadot/types/interfaces/grandpa';
 import { StorageKind } from '@polkadot/types/interfaces/offchain';
 import { RuntimeDispatchInfo } from '@polkadot/types/interfaces/payment';
 import { RpcMethods } from '@polkadot/types/interfaces/rpc';
@@ -140,6 +141,12 @@ declare module '@polkadot/rpc-core/types.jsonrpc' {
        **/
       finalizeBlock: AugmentedRpc<(hash: BlockHash | string | Uint8Array, justification?: Justification | string | Uint8Array) => Observable<bool>>;
     };
+    grandpa: {
+      /**
+       * Returns the state of the current best round state as well as the ongoing background rounds
+       **/
+      roundState: AugmentedRpc<() => Observable<ReportedRoundStates>>;
+    };
     offchain: {
       /**
        * Get offchain local storage under given key and prefix

packages/api/src/base/Decorate.ts

@@ -15,14 +15,13 @@ import { map, switchMap, take, tap, toArray } from 'rxjs/operators';
 import decorateDerive, { ExactDerive } from '@polkadot/api-derive';
 import { memo } from '@polkadot/api-derive/util';
 import DecoratedMeta from '@polkadot/metadata/Decorated';
-import getHasher from '@polkadot/metadata/Decorated/storage/fromMetadata/getHasher';
 import RpcCore from '@polkadot/rpc-core';
 import { WsProvider } from '@polkadot/rpc-provider';
 import { Metadata, Null, Option, Raw, Text, TypeRegistry, u64 } from '@polkadot/types';
 import Linkage, { LinkageResult } from '@polkadot/types/codec/Linkage';
 import { DEFAULT_VERSION as EXTRINSIC_DEFAULT_VERSION } from '@polkadot/types/extrinsic/constants';
 import StorageKey, { StorageEntry, unwrapStorageType } from '@polkadot/types/primitive/StorageKey';
-import { assert, compactStripLength, isNull, isUndefined, logger, u8aConcat, u8aToHex } from '@polkadot/util';
+import { assert, compactStripLength, logger, u8aToHex } from '@polkadot/util';
 
 import { createSubmittable } from '../submittable';
 import augmentObject from '../util/augmentObject';
@@ -344,8 +343,8 @@ export default abstract class Decorate<ApiType extends ApiTypes> extends Events
     decorated.key = (arg1?: Arg, arg2?: Arg): string =>
       u8aToHex(compactStripLength(creator(creator.meta.type.isDoubleMap ? [arg1, arg2] : arg1))[1]);
 
-    decorated.keyPrefix = (): string =>
-      u8aToHex(creator.keyPrefix);
+    decorated.keyPrefix = (key1?: Arg): string =>
+      u8aToHex(creator.keyPrefix(key1));
 
     decorated.range = decorateMethod((range: [Hash, Hash?], arg1?: Arg, arg2?: Arg): Observable<[Hash, Codec][]> =>
       this._decorateStorageRange(decorated, [arg1, arg2], range));
@@ -392,6 +391,9 @@ export default abstract class Decorate<ApiType extends ApiTypes> extends Events
     const result: Map<Codec, ITuple<[Codec, Linkage<Codec>]> | null> = new Map();
     let subject: BehaviorSubject<LinkageResult>;
     let head: Codec | null = null;
+    const iterKey = creator.iterKey;
+
+    assert(iterKey, 'iterKey field is missing');
 
     // retrieve a value based on the key, iterating if it has a next entry. Since
     // entries can be re-linked in the middle of a list, we subscribe here to make
@@ -455,7 +457,7 @@ export default abstract class Decorate<ApiType extends ApiTypes> extends Events
           .subscribeStorage<[[Codec, Linkage<Codec>]]>([[creator, ...args]])
           .pipe(map(([data]): [Codec, Linkage<Codec>] => data))
         : this._rpcCore.state
-          .subscribeStorage<[LinkageResult]>([creator.iterKey])
+          .subscribeStorage<[LinkageResult]>([iterKey()])
           .pipe(switchMap(([key]): Observable<LinkageResult> => getNext(head = key)))
     );
   }
@@ -463,14 +465,7 @@ export default abstract class Decorate<ApiType extends ApiTypes> extends Events
   private _retrieveMapKeys ({ iterKey, meta }: StorageEntry, arg?: Arg): Observable<StorageKey[]> {
     assert(iterKey && (meta.type.isMap || meta.type.isDoubleMap), 'keys can only be retrieved on maps, linked maps and double maps');
 
-    const headKey = this.createType('Raw', u8aConcat(
-      iterKey,
-      meta.type.isDoubleMap && !isUndefined(arg) && !isNull(arg)
-        ? getHasher(meta.type.asDoubleMap.hasher)(
-          this.createType(meta.type.asDoubleMap.key1.toString() as 'Raw', arg).toU8a()
-        )
-        : new Uint8Array([])
-    )).toHex();
+    const headKey = iterKey(arg).toHex();
     const startSubject = new BehaviorSubject<string>(headKey);
 
     return this._rpcCore.state.getKeysPaged

packages/api/src/base/Init.ts

@@ -5,7 +5,7 @@
 import { SignedBlock, RuntimeVersion } from '@polkadot/types/interfaces';
 import { ApiBase, ApiOptions, ApiTypes, DecorateMethod } from '../types';
 
-import { Subscription, combineLatest, of } from 'rxjs';
+import { Observable, Subscription, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { Metadata, Text } from '@polkadot/types';
 import { LATEST_EXTRINSIC_VERSION } from '@polkadot/types/extrinsic/Extrinsic';
@@ -124,28 +124,25 @@ export default abstract class Init<ApiType extends ApiTypes> extends Decorate<Ap
     }
 
     this.#updateSub = this._rpcCore.state.subscribeRuntimeVersion().pipe(
-      switchMap((version: RuntimeVersion) =>
-        combineLatest(
-          of(version),
-          this._rpcCore.state.getMetadata()
-        )
-      ),
-      map(([version, metadata]: [RuntimeVersion, Metadata]): boolean => {
-        if (this._runtimeVersion?.specVersion.eq(version.specVersion)) {
-          return false;
-        }
+      switchMap((version: RuntimeVersion): Observable<boolean> =>
+        // only retrieve the metadata when the on-chain version has been changed
+        this._runtimeVersion?.specVersion.eq(version.specVersion)
+          ? of(false)
+          : this._rpcCore.state.getMetadata().pipe(
+            map((metadata: Metadata): boolean => {
+              l.log(`Runtime version updated to ${version.specVersion}`);
 
-        l.log(`Runtime version updated to ${version.specVersion}`);
+              this._runtimeMetadata = metadata;
+              this._runtimeVersion = version;
+              this._rx.runtimeVersion = version;
 
-        this._runtimeMetadata = metadata;
-        this._runtimeVersion = version;
-        this._rx.runtimeVersion = version;
+              this.registerTypes(getSpecTypes(this.registry, this._runtimeChain as Text, version.specName, version.specVersion));
+              this.injectMetadata(metadata, false);
 
-        this.registerTypes(getSpecTypes(this.registry, this._runtimeChain as Text, version.specName, version.specVersion));
-        this.injectMetadata(metadata, false);
-
-        return true;
-      })
+              return true;
+            })
+          )
+      )
     ).subscribe();
   }
 
@@ -159,6 +156,7 @@ export default abstract class Init<ApiType extends ApiTypes> extends Decorate<Ap
     // set our chain version & genesisHash as returned
     this._runtimeChain = chain;
     this._runtimeVersion = runtimeVersion;
+    this._rx.runtimeVersion = runtimeVersion;
 
     // do the setup for the specific chain
     this.registry.setChainProperties(chainProps);

packages/api/src/checkTypes.manual.ts

@@ -159,7 +159,7 @@ async function tx (api: ApiPromise, keyring: TestKeyringMap): Promise<void> {
     });
 
   // it allows for query & then using the submittable
-  const second = api.tx.democracy.second(123);
+  const second = api.tx.democracy.second(123, 5);
 
   second.signAndSend('123', (result): void => {
     console.log(result);

packages/api/src/promise/Api.spec.ts

@@ -13,7 +13,7 @@ import { hexToU8a } from '@polkadot/util';
 import { SingleAccountSigner } from '../../test/util';
 import ApiPromise from './Api';
 
-const TRANSFER_SIG = '0xa1d493ab6b74550da434fbc153a739429be68c9d30e59e631fb5d08f348504deefb5577f4824e138741ad102f479109827e8fa9552812a3e5f505a24d10e820f';
+const TRANSFER_SIG = '0x7a10e5ed9a14284eca7bea53f81631981dddda5a3d2dee973b136475947264801465726e4829ae3994d9058df638d959d4e043c7f1924299546790dda1dea20a';
 
 describe('ApiPromise', (): void => {
   const registry = new TypeRegistry();

packages/api/src/rx/Api.spec.ts

@@ -49,5 +49,22 @@ describe('ApiRx', (): void => {
         })
       );
     });
+
+    it('allows the second argument to signAsync to be omitted', () => {
+      const signer = new SingleAccountSigner(registry, keyring.alice_session);
+
+      ApiRx.create({ provider, registry, signer }).pipe(
+        switchMap((api: ApiRx): Observable<SubmittableExtrinsic<'rxjs'>> =>
+          api.tx.balances
+            .transfer(keyring.eve.address, 12345)
+            .signAsync(keyring.alice_session)
+        ),
+        map((tx: SubmittableExtrinsic<'rxjs'>): void => {
+          expect(tx.signature.toHex()).toEqual(
+            '0x97f3cfe5088fcd575313e983f45d02b0f630e7b94ff9a3ac50e20cd096a8f554fda73d42ead891b5a1d3ce5607d83f20b0c6570b555e949cfb5763d0abcd590b'
+          );
+        })
+      );
+    });
   });
 });

packages/api/src/submittable/createClass.ts

@@ -60,7 +60,7 @@ export default function createClass <ApiType extends ApiTypes> ({ api, apiType,
     }
 
     // sign a transaction, returning the this to allow chaining, i.e. .sign(...).send()
-    public sign (account: IKeyringPair, optionsOrNonce: Partial<SignerOptions>): this {
+    public sign (account: IKeyringPair, optionsOrNonce?: Partial<SignerOptions>): this {
       super.sign(account, this.#makeSignOptions(this.#optionsOrNonce(optionsOrNonce), {}));
 
       return this;
@@ -69,7 +69,7 @@ export default function createClass <ApiType extends ApiTypes> ({ api, apiType,
     // signs a transaction, returning `this` to allow chaining. E.g.: `sign(...).send()`
     //
     // also supports signing through external signers
-    public signAsync (account: AddressOrPair, optionsOrNonce: Partial<SignerOptions>): SubmittableThis<ApiType, this> {
+    public signAsync (account: AddressOrPair, optionsOrNonce?: Partial<SignerOptions>): SubmittableThis<ApiType, this> {
       return decorateMethod(
         (): Observable<this> =>
           this.#observeSign(account, optionsOrNonce).pipe(mapTo(this))
@@ -148,6 +148,7 @@ export default function createClass <ApiType extends ApiTypes> ({ api, apiType,
         ...options,
         ...extras,
         runtimeVersion: api.runtimeVersion,
+        signedExtensions: api.registry.signedExtensions,
         version: api.extrinsicType
       } as SignatureOptions;
     }
@@ -164,7 +165,7 @@ export default function createClass <ApiType extends ApiTypes> ({ api, apiType,
       return [options, statusCb];
     }
 
-    #observeSign = (account: AddressOrPair, optionsOrNonce: Partial<SignerOptions>): Observable<number | undefined> => {
+    #observeSign = (account: AddressOrPair, optionsOrNonce?: Partial<SignerOptions>): Observable<number | undefined> => {
       const address = isKeyringPair(account) ? account.address : account.toString();
       const options = this.#optionsOrNonce(optionsOrNonce);
       let updateId: number | undefined;
@@ -184,7 +185,7 @@ export default function createClass <ApiType extends ApiTypes> ({ api, apiType,
       );
     }
 
-    #observeStatus = (status: ExtrinsicStatus): Observable<ISubmittableResult> => {
+    #observeStatus = (hash: Hash, status: ExtrinsicStatus): Observable<ISubmittableResult> => {
       if (!status.isFinalized && !status.isInBlock) {
         return of(new SubmittableResult({ status }));
       }
@@ -196,7 +197,7 @@ export default function createClass <ApiType extends ApiTypes> ({ api, apiType,
       return api.derive.tx.events(blockHash).pipe(
         map(({ block, events }): ISubmittableResult =>
           new SubmittableResult({
-            events: filterEvents(this.hash, block, events, status),
+            events: filterEvents(hash, block, events, status),
             status
           })
         )
@@ -212,9 +213,11 @@ export default function createClass <ApiType extends ApiTypes> ({ api, apiType,
     }
 
     #observeSubscribe = (updateId = -1): Observable<ISubmittableResult> => {
+      const hash = this.hash;
+
       return api.rpc.author.submitAndWatchExtrinsic(this).pipe(
         switchMap((status): Observable<ISubmittableResult> =>
-          this.#observeStatus(status)
+          this.#observeStatus(hash, status)
         ),
         tap((status): void => {
           this.#updateSigner(updateId, status);
@@ -224,7 +227,7 @@ export default function createClass <ApiType extends ApiTypes> ({ api, apiType,
 
     // NOTE here we actually override nonce if it was specified (backwards compat for
     // the previous signature - don't let userspace break, but allow then time to upgrade)
-    #optionsOrNonce = (optionsOrNonce: Partial<SignerOptions>): Partial<SignerOptions> => {
+    #optionsOrNonce = (optionsOrNonce: Partial<SignerOptions> = {}): Partial<SignerOptions> => {
       return isBn(optionsOrNonce) || isNumber(optionsOrNonce)
         ? { nonce: optionsOrNonce }
         : optionsOrNonce;

packages/api/src/submittable/types.ts

@@ -50,9 +50,9 @@ export interface SubmittableExtrinsic<ApiType extends ApiTypes> extends Extrinsi
 
   send(statusCb: Callback<ISubmittableResult>): SubmittableResultSubscription<ApiType>;
 
-  sign(account: IKeyringPair, _options: Partial<SignatureOptions>): this;
+  sign(account: IKeyringPair, _options?: Partial<SignatureOptions>): this;
 
-  signAsync(account: AddressOrPair, _options: Partial<SignatureOptions>): SubmittableThis<ApiType, this>;
+  signAsync(account: AddressOrPair, _options?: Partial<SignatureOptions>): SubmittableThis<ApiType, this>;
 
   signAndSend(account: AddressOrPair, options?: Partial<SignerOptions>): SubmittableResultResult<ApiType>;
 

packages/api/src/types/storage.ts

@@ -40,6 +40,7 @@ export interface StorageEntryBase<ApiType extends ApiTypes, F extends AnyFunctio
 
 export interface StorageEntryDoubleMap<ApiType extends ApiTypes, F extends AnyFunction> extends StorageEntryBase<ApiType, F> {
   entries: <T extends Codec | any = ObsInnerType<ReturnType<F>>>(arg?: Parameters<F>[0]) => PromiseOrObs<ApiType, [StorageKey, T][]>;
+  keyPrefix: (key1?: Parameters<F>[0]) => string;
 }
 
 interface StorageEntryObservableMulti {

packages/metadata/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/metadata",
-  "version": "1.12.1",
+  "version": "1.15.1",
   "description": "Helpers to extract information from runtime metadata",
   "main": "index.js",
   "publishConfig": {
@@ -26,14 +26,14 @@
   },
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/type-metadata#readme",
   "dependencies": {
-    "@babel/runtime": "^7.9.2",
-    "@polkadot/types": "1.12.1",
-    "@polkadot/types-known": "1.12.1",
-    "@polkadot/util": "^2.9.0-beta.3",
-    "@polkadot/util-crypto": "^2.9.0-beta.3",
-    "bn.js": "^5.1.1"
+    "@babel/runtime": "^7.9.6",
+    "@polkadot/types": "1.15.1",
+    "@polkadot/types-known": "1.15.1",
+    "@polkadot/util": "^2.10.1",
+    "@polkadot/util-crypto": "^2.10.1",
+    "bn.js": "^5.1.2"
   },
   "devDependencies": {
-    "@polkadot/keyring": "^2.9.0-beta.3"
+    "@polkadot/keyring": "^2.10.1"
   }
 }

packages/metadata/src/Decorated/extrinsics/index.spec.ts

@@ -29,7 +29,8 @@ describe('extrinsics', (): void => {
           implName: 'test',
           implVersion: new BN(123),
           specName: 'test',
-          specVersion: new BN(123)
+          specVersion: new BN(123),
+          transactionVersion: new BN(123)
         }
       }).toHex()
     ).toEqual(

packages/metadata/src/Decorated/storage/fromMetadata/createFunction.spec.ts

@@ -4,7 +4,7 @@
 
 import { Text, TypeRegistry } from '@polkadot/types';
 import { StorageEntry } from '@polkadot/types/primitive/StorageKey';
-import { stringToU8a, u8aConcat, u8aToHex } from '@polkadot/util';
+import { assert, stringToU8a, u8aConcat, u8aToHex } from '@polkadot/util';
 
 import createFunction from './createFunction';
 
@@ -104,6 +104,17 @@ describe('createFunction', (): void => {
         (): Uint8Array => storageFn(['5DXUeE5N5LtkW97F2PzqYPyqNkxqSWESdGSPTX6AvkUAhwKP'])
       ).toThrow(/requires two arguments/);
     });
+
+    it('accepts an optional parameter for key construction', (): void => {
+      const iterKey = storageFn.iterKey;
+
+      assert(iterKey, 'storageFn.iterKey is undefined');
+
+      const result = iterKey('5DXUeE5N5LtkW97F2PzqYPyqNkxqSWESdGSPTX6AvkUAhwKP');
+
+      expect(u8aToHex(result)).toEqual('0x223416315e3dddca3b5a47fd0ac8e4916482b9ade7bc6657aaca787ba1add3b4c4303117deb55aad9858c8a873273280f78d172b398d5e77e43a2db5e42163e9');
+      expect(u8aToHex(iterKey())).toEqual('0x223416315e3dddca3b5a47fd0ac8e4916482b9ade7bc6657aaca787ba1add3b4');
+    });
   });
 
   it('allows creates double map function with a Null type key', (): void => {

packages/metadata/src/Decorated/storage/fromMetadata/createFunction.ts

@@ -142,7 +142,7 @@ function expandWithMeta ({ meta, method, prefix, section }: CreateItemFn, storag
 }
 
 /** @internal */
-function extendHeadMeta (registry: Registry, { meta: { documentation, name, type }, section }: CreateItemFn, { method }: StorageEntry, iterFn: (arg?: any) => Raw): StorageKey {
+function extendHeadMeta (registry: Registry, { meta: { documentation, name, type }, section }: CreateItemFn, { method }: StorageEntry, iterFn: (arg?: any) => Raw): (arg?: any) => StorageKey {
   const outputType = type.isMap
     ? type.asMap.key.toString()
     : type.asDoubleMap.key1.toString();
@@ -157,7 +157,12 @@ function extendHeadMeta (registry: Registry, { meta: { documentation, name, type
     type: registry.createType('StorageEntryTypeLatest', registry.createType('PlainTypeLatest', type.isMap ? type.asMap.key : type.asDoubleMap.key1), 0)
   });
 
-  return registry.createType('StorageKey', iterFn, { method, section });
+  const prefixKey = registry.createType('StorageKey', iterFn, { method, section });
+
+  return (arg?: any): StorageKey =>
+    !isUndefined(arg) && !isNull(arg)
+      ? registry.createType('StorageKey', iterFn(arg), { method, section })
+      : prefixKey;
 }
 
 // attach the head key hashing for linked maps
@@ -177,8 +182,12 @@ function extendLinkedMap (registry: Registry, itemFn: CreateItemFn, storageFn: S
 // attach the full list hashing for prefixed maps
 /** @internal */
 function extendPrefixedMap (registry: Registry, itemFn: CreateItemFn, storageFn: StorageEntry): StorageEntry {
-  storageFn.iterKey = extendHeadMeta(registry, itemFn, storageFn, (): Raw =>
-    new Raw(registry, createPrefixedKey(itemFn))
+  const { meta: { type } } = itemFn;
+
+  storageFn.iterKey = extendHeadMeta(registry, itemFn, storageFn, (arg?: any): Raw =>
+    type.isDoubleMap && !isUndefined(arg) && !isNull(arg)
+      ? new Raw(registry, u8aConcat(createPrefixedKey(itemFn), getHasher(type.asDoubleMap.hasher)(registry.createType(type.asDoubleMap.key1.toString() as 'Raw', arg).toU8a())))
+      : new Raw(registry, createPrefixedKey(itemFn))
   );
 
   return storageFn;
@@ -219,7 +228,8 @@ export default function createFunction (registry: Registry, itemFn: CreateItemFn
     extendDoubleMap(registry, itemFn, storageFn);
   }
 
-  storageFn.keyPrefix = storageFn.iterKey || compactStripLength(storageFn())[1];
+  storageFn.keyPrefix = (arg?: any): Uint8Array =>
+    (storageFn.iterKey && storageFn.iterKey(arg)) || compactStripLength(storageFn())[1];
 
   return storageFn;
 }

packages/metadata/src/Metadata/v11/Metadata.spec.ts

@@ -4,8 +4,6 @@
 
 import { TypeRegistry } from '@polkadot/types';
 
-import polkadotJson from './static-polkadot.json';
-import polkadotData from './static.polkadot';
 import substrateJson from './static-substrate.json';
 import substrateData from './static';
 import { decodeLatestSubstrate, defaultValues, toLatest } from '../util/testUtil';
@@ -19,13 +17,3 @@ describe('MetadataV11 (substrate)', (): void => {
 
   defaultValues(registry, substrateData);
 });
-
-describe('MetadataV11 (polkadot)', (): void => {
-  const registry = new TypeRegistry();
-
-  decodeLatestSubstrate(registry, 11, polkadotData, polkadotJson);
-
-  toLatest(registry, 11, polkadotData);
-
-  defaultValues(registry, polkadotData);
-});

packages/rpc-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/rpc-core",
-  "version": "1.12.1",
+  "version": "1.15.1",
   "description": "A JavaScript wrapper for the Polkadot JsonRPC interface",
   "main": "index.js",
   "keywords": [
@@ -26,15 +26,15 @@
   },
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/rpc-core#readme",
   "dependencies": {
-    "@babel/runtime": "^7.9.2",
-    "@polkadot/metadata": "1.12.1",
-    "@polkadot/rpc-provider": "1.12.1",
-    "@polkadot/types": "1.12.1",
-    "@polkadot/util": "^2.9.0-beta.3",
+    "@babel/runtime": "^7.9.6",
+    "@polkadot/metadata": "1.15.1",
+    "@polkadot/rpc-provider": "1.15.1",
+    "@polkadot/types": "1.15.1",
+    "@polkadot/util": "^2.10.1",
     "memoizee": "^0.4.14",
     "rxjs": "^6.5.5"
   },
   "devDependencies": {
-    "@polkadot/keyring": "^2.9.0-beta.3"
+    "@polkadot/keyring": "^2.10.1"
   }
 }

packages/rpc-core/src/formatting.spec.FIXME-ts

@@ -153,7 +153,7 @@ function formattingTests (version: string, decorated: Metadata, encodedValues: [
       }
 
       api.state
-        .subscribeStorage([iterKey])
+        .subscribeStorage([iterKey()])
         .subscribe((value: Codec[]): void => {
           expect(value).toHaveLength(1);
           // console.error('head fallback', value);

packages/rpc-core/src/index.ts

@@ -89,6 +89,8 @@ export default class Rpc implements RpcInterface {
 
   public readonly engine!: RpcInterface['engine'];
 
+  public readonly grandpa!: RpcInterface['grandpa'];
+
   public readonly offchain!: RpcInterface['offchain'];
 
   public readonly payment!: RpcInterface['payment'];
@@ -436,24 +438,13 @@ export default class Rpc implements RpcInterface {
     this.#storageCache.set(hexKey, value);
 
     if (meta.modifier.isOptional) {
-      try {
-        return new Option(
-          this.registry,
-          createClass(this.registry, type),
-          isEmpty
-            ? null
-            : createTypeUnsafe(this.registry, type, [input], true)
-        );
-      } catch (error) {
-        if (type === 'PreimageStatus') {
-          return new Option(
-            this.registry,
-            createClass(this.registry, type)
-          );
-        }
-
-        throw error;
-      }
+      return new Option(
+        this.registry,
+        createClass(this.registry, type),
+        isEmpty
+          ? null
+          : createTypeUnsafe(this.registry, type, [input], true)
+      );
     }
 
     return createTypeUnsafe(this.registry, type, [

packages/rpc-provider/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/rpc-provider",
-  "version": "1.12.1",
+  "version": "1.15.1",
   "description": "Transport providers for the API",
   "main": "index.js",
   "keywords": [
@@ -26,18 +26,18 @@
   },
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/rpc-provider#readme",
   "dependencies": {
-    "@babel/runtime": "^7.9.2",
-    "@polkadot/metadata": "1.12.1",
-    "@polkadot/types": "1.12.1",
-    "@polkadot/util": "^2.9.0-beta.3",
-    "@polkadot/util-crypto": "^2.9.0-beta.3",
-    "bn.js": "^5.1.1",
-    "eventemitter3": "^4.0.0",
+    "@babel/runtime": "^7.9.6",
+    "@polkadot/metadata": "1.15.1",
+    "@polkadot/types": "1.15.1",
+    "@polkadot/util": "^2.10.1",
+    "@polkadot/util-crypto": "^2.10.1",
+    "bn.js": "^5.1.2",
+    "eventemitter3": "^4.0.4",
     "isomorphic-fetch": "^2.2.1",
     "websocket": "^1.0.31"
   },
   "devDependencies": {
-    "@polkadot/keyring": "^2.9.0-beta.3",
+    "@polkadot/keyring": "^2.10.1",
     "mock-socket": "^9.0.3",
     "nock": "^12.0.3"
   }

packages/rpc-provider/src/ws/Provider.ts

@@ -65,7 +65,7 @@ const l = logger('api-ws');
 export default class WsProvider implements WSProviderInterface {
   readonly #coder: Coder;
 
-  readonly #endpoint: string;
+  readonly #endpoints: string[];
 
   readonly #eventemitter: EventEmitter;
 
@@ -77,6 +77,8 @@ export default class WsProvider implements WSProviderInterface {
 
   #autoConnectMs: number;
 
+  #endpointIndex: number;
+
   #isConnected = false;
 
   #subscriptions: Record<string, WsStateSubscription> = {};
@@ -84,16 +86,25 @@ export default class WsProvider implements WSProviderInterface {
   #websocket: WebSocket | null;
 
   /**
-   * @param {string}  endpoint    The endpoint url. Usually `ws://ip:9944` or `wss://ip:9944`
+   * @param {string | string[]}  endpoint    The endpoint url. Usually `ws://ip:9944` or `wss://ip:9944`, may provide an array of endpoint strings.
    * @param {boolean} autoConnect Whether to connect automatically or not.
    */
-  constructor (endpoint: string = defaults.WS_URL, autoConnectMs: number | false = 1000) {
-    assert(/^(wss|ws):\/\//.test(endpoint), `Endpoint should start with 'ws://', received '${endpoint}'`);
+  constructor (endpoint: string | string[] = defaults.WS_URL, autoConnectMs: number | false = 1000) {
+    const endpoints = Array.isArray(endpoint)
+      ? endpoint
+      : [endpoint];
+
+    assert(endpoints.length !== 0, 'WsProvider requires at least one Endpoint');
+
+    endpoints.forEach((endpoint) => {
+      assert(/^(wss|ws):\/\//.test(endpoint), `Endpoint should start with 'ws://', received '${endpoint}'`);
+    });
 
     this.#eventemitter = new EventEmitter();
     this.#autoConnectMs = autoConnectMs || 0;
     this.#coder = new Coder();
-    this.#endpoint = endpoint;
+    this.#endpointIndex = -1;
+    this.#endpoints = endpoints;
     this.#websocket = null;
 
     if (autoConnectMs > 0) {
@@ -112,7 +123,7 @@ export default class WsProvider implements WSProviderInterface {
    * @description Returns a clone of the object
    */
   public clone (): WsProvider {
-    return new WsProvider(this.#endpoint);
+    return new WsProvider(this.#endpoints);
   }
 
   /**
@@ -122,9 +133,11 @@ export default class WsProvider implements WSProviderInterface {
    */
   public async connect (): Promise<void> {
     try {
+      this.#endpointIndex = (this.#endpointIndex + 1) % this.#endpoints.length;
+
       const WS = await getWSClass();
 
-      this.#websocket = new WS(this.#endpoint);
+      this.#websocket = new WS(this.#endpoints[this.#endpointIndex]);
       this.#websocket.onclose = this.#onSocketClose;
       this.#websocket.onerror = this.#onSocketError;
       this.#websocket.onmessage = this.#onSocketMessage;
@@ -268,7 +281,7 @@ export default class WsProvider implements WSProviderInterface {
 
   #onSocketClose = (event: CloseEvent): void => {
     if (this.#autoConnectMs > 0) {
-      l.error(`disconnected from ${this.#endpoint} code: '${event.code}' reason: '${event.reason}'`);
+      l.error(`disconnected from ${this.#endpoints[this.#endpointIndex]} code: '${event.code}' reason: '${event.reason}'`);
     }
 
     this.#isConnected = false;
@@ -363,7 +376,7 @@ export default class WsProvider implements WSProviderInterface {
   #onSocketOpen = (): boolean => {
     assert(!isNull(this.#websocket), 'WebSocket cannot be null in onOpen');
 
-    l.debug((): any[] => ['connected to', this.#endpoint]);
+    l.debug((): any[] => ['connected to', this.#endpoints[this.#endpointIndex]]);
 
     this.#isConnected = true;
 

packages/rpc-provider/src/ws/connect.spec.ts

@@ -6,17 +6,25 @@ import WsProvider from './';
 import { Mock } from './../mock/types';
 import { mockWs, TEST_WS_URL } from '../../test/mockWs';
 
+function sleepMs (ms = 0): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
 describe('onConnect', (): void => {
-  let mock: Mock;
+  let mocks: Mock[];
 
   beforeEach((): void => {
-    mock = mockWs([]);
+    jest.setTimeout(30000);
+    mocks = [mockWs([])];
   });
 
   afterEach((): void => {
-    if (mock) {
-      mock.done();
-    }
+    jest.setTimeout(5000);
+    mocks.forEach((m) => {
+      if (m) {
+        m.done();
+      }
+    });
   });
 
   it('Does not connect when autoConnect is false', (): void => {
@@ -25,19 +33,102 @@ describe('onConnect', (): void => {
     expect(provider.isConnected()).toBe(false);
   });
 
-  it('Does connect when autoConnect is true', (): void => {
-    const provider: WsProvider = new WsProvider(TEST_WS_URL, 1000);
+  it('Does connect when autoConnect is true', async () => {
+    const provider: WsProvider = new WsProvider(TEST_WS_URL, 1);
 
-    expect(provider.isConnected()).not.toBe(true);
+    await sleepMs(100); // Hack to give the provider time to connect
+    expect(provider.isConnected()).toBe(true);
   });
 
-  it('Creates a new WebSocket instance by calling the connect() method', (): void => {
-    const provider: WsProvider = new WsProvider(TEST_WS_URL, 0);
+  it('Creates a new WebSocket instance by calling the connect() method', async () => {
+    const provider: WsProvider = new WsProvider(TEST_WS_URL, false);
 
     expect(provider.isConnected()).toBe(false);
+    expect(mocks[0].server.clients().length).toBe(0);
 
-    provider.connect();
+    await provider.connect();
+    await sleepMs(100); // Hack to give the provider time to connect
 
-    expect(provider.isConnected()).not.toBe(true);
+    expect(provider.isConnected()).toBe(true);
+    expect(mocks[0].server.clients().length).toBe(1);
+  });
+
+  it('Connects to first endpoint when an array is given', async () => {
+    const provider: WsProvider = new WsProvider([TEST_WS_URL], 1);
+
+    await provider.connect();
+    await sleepMs(100); // Hack to give the provider time to connect
+
+    expect(provider.isConnected()).toBe(true);
+  });
+
+  it('Connects to the second endpoint when the first is unreachable', async () => {
+    /* eslint-disable @typescript-eslint/no-empty-function */
+    jest.spyOn(console, 'error').mockImplementation((): void => {});
+
+    const endpoints: string[] = ['ws://localhost:9956', TEST_WS_URL];
+    const provider: WsProvider = new WsProvider(endpoints, 1);
+
+    await sleepMs(100); // Hack to give the provider time to connect
+
+    expect(provider.isConnected()).toBe(true);
+  });
+
+  it('Connects to the second endpoint when the first is dropped', async () => {
+    const endpoints: string[] = [TEST_WS_URL, 'ws://localhost:9957'];
+
+    mocks.push(mockWs([], endpoints[1]));
+
+    const provider: WsProvider = new WsProvider(endpoints, 1);
+
+    await sleepMs(100); // Hack to give the provider time to connect
+    // Check that first server is connected
+    expect(mocks[0].server.clients().length).toBe(1);
+    expect(mocks[1].server.clients().length).toBe(0);
+
+    // Close connection from first server
+    mocks[0].server.clients()[0].close();
+    await sleepMs(100);
+
+    // Check that second server is connected
+    expect(mocks[1].server.clients().length).toBe(1);
+    expect(provider.isConnected()).toBe(true);
+  });
+
+  it('Round-robin of endpoints on WsProvider', async () => {
+    const rounds = 5;
+    const endpoints: string[] = [
+      TEST_WS_URL,
+      'ws://localhost:9956',
+      'ws://localhost:9957',
+      'ws://invalid:9956',
+      'ws://localhost:9958'
+    ];
+
+    mocks.push(mockWs([], endpoints[1]));
+    mocks.push(mockWs([], endpoints[2]));
+    mocks.push(mockWs([], endpoints[4]));
+
+    const mockNext = [
+      mocks[1],
+      mocks[2],
+      mocks[3],
+      mocks[0]
+    ];
+    const provider: WsProvider = new WsProvider(endpoints, 1);
+
+    for (let round = 0; round < rounds; round++) {
+      for (let index = 0; index < mocks.length; index++) {
+        await sleepMs(100); // Hack to give the provider time to connect
+
+        // Check that first server is connected and the next one isn't
+        expect(mocks[index].server.clients().length).toBe(1);
+        expect(mockNext[index].server.clients().length).toBe(0);
+        expect(provider.isConnected()).toBe(true);
+
+        // Close connection from first server
+        mocks[index].server.clients()[0].close();
+      }
+    }
   });
 });

packages/rpc-provider/src/ws/index.spec.ts

@@ -29,3 +29,42 @@ describe('Ws', (): void => {
     ).toEqual(false);
   });
 });
+
+describe('Endpoint Parsing', (): void => {
+  it('Succeeds when WsProvider endpoint is a valid string', () => {
+    /* eslint-disable no-new */
+    new WsProvider(TEST_WS_URL, 0);
+  });
+
+  it('Throws when WsProvider endpoint is an invalid string', () => {
+    expect(() => {
+      /* eslint-disable no-new */
+      new WsProvider('http://127.0.0.1:9955', 0);
+    }).toThrowError(/^Endpoint should start with /);
+  });
+
+  it('Succeeds when WsProvider endpoint is a valid array', () => {
+    const endpoints: string[] = ['ws://127.0.0.1:9955', 'wss://testnet.io:9944', 'ws://mychain.com:9933'];
+
+    /* eslint-disable no-new */
+    new WsProvider(endpoints, 0);
+  });
+
+  it('Throws when WsProvider endpoint is an empty array', () => {
+    const endpoints: string[] = [];
+
+    expect(() => {
+      /* eslint-disable no-new */
+      new WsProvider(endpoints, 0);
+    }).toThrowError('WsProvider requires at least one Endpoint');
+  });
+
+  it('Throws when WsProvider endpoint is an invalid array', () => {
+    const endpoints: string[] = ['ws://127.0.0.1:9955', 'http://bad.co:9944', 'ws://mychain.com:9933'];
+
+    expect(() => {
+      /* eslint-disable no-new */
+      new WsProvider(endpoints, 0);
+    }).toThrowError(/^Endpoint should start with /);
+  });
+});

packages/rpc-provider/test/mockWs.ts

@@ -6,8 +6,6 @@ import { Server } from 'mock-socket';
 
 const TEST_WS_URL = 'ws://localhost:9955';
 
-let server: Server;
-
 interface Scope {
   body: { [index: string]: {} };
   requests: number;
@@ -52,8 +50,8 @@ function createReply ({ id, reply: { result } }: ReplyDef): any {
 }
 
 // scope definition returned
-function mockWs (requests: { method: string }[]): Scope {
-  server = new Server(TEST_WS_URL);
+function mockWs (requests: { method: string }[], wsUrl: string = TEST_WS_URL): Scope {
+  const server = new Server(wsUrl);
 
   let requestCount = 0;
   const scope: Scope = {

packages/typegen/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/typegen",
-  "version": "1.12.1",
+  "version": "1.15.1",
   "description": "Type generation scripts",
   "main": "index.js",
   "bin": {
@@ -33,19 +33,20 @@
   },
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/typegen#readme",
   "dependencies": {
-    "@babel/core": "^7.9.0",
+    "@babel/core": "^7.9.6",
     "@babel/register": "^7.9.0",
-    "@babel/runtime": "^7.9.2",
-    "@polkadot/api": "1.12.1",
-    "@polkadot/metadata": "1.12.1",
-    "@polkadot/rpc-provider": "1.12.1",
-    "@polkadot/types": "1.12.1",
-    "@polkadot/util": "^2.9.0-beta.3",
+    "@babel/runtime": "^7.9.6",
+    "@polkadot/api": "1.15.1",
+    "@polkadot/metadata": "1.15.1",
+    "@polkadot/rpc-provider": "1.15.1",
+    "@polkadot/types": "1.15.1",
+    "@polkadot/util": "^2.10.1",
+    "handlebars": "^4.7.6",
     "websocket": "^1.0.31",
     "yargs": "^15.3.1"
   },
   "devDependencies": {
     "@types/websocket": "^1.0.0",
-    "@types/yargs": "^15.0.4"
+    "@types/yargs": "^15.0.5"
   }
 }

packages/typegen/src/generate/consts.ts

@@ -2,7 +2,7 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
-import { ModuleMetadataLatest } from '@polkadot/types/interfaces/metadata';
+import Handlebars from 'handlebars';
 
 import staticData from '@polkadot/metadata/Metadata/static';
 import Metadata from '@polkadot/metadata/Metadata';
@@ -10,29 +10,10 @@ import { TypeRegistry } from '@polkadot/types/create';
 import * as defaultDefs from '@polkadot/types/interfaces/definitions';
 import { stringCamelCase } from '@polkadot/util';
 
-import { FOOTER, HEADER, TypeImports, createDocComments, createImportCode, createImports, indent, registerDefinitions, setImports, writeFile } from '../util';
+import { createImports, compareName, readTemplate, registerDefinitions, setImports, writeFile } from '../util';
 
-// Generate types for one module
-/** @internal */
-function generateModule (allDefs: object, { constants, name }: ModuleMetadataLatest, imports: TypeImports, isStrict: boolean): string[] {
-  if (!constants.length) {
-    return [];
-  }
-
-  if (!isStrict) {
-    setImports(allDefs, imports, ['Codec']);
-  }
-
-  return [indent(4)(`${stringCamelCase(name.toString())}: {`)]
-    .concat(isStrict ? '' : indent(6)('[index: string]: AugmentedConst<object & Codec>;'))
-    .concat(constants.sort((a, b) => a.name.localeCompare(b.name.toString())).map(({ documentation, name, type }): string => {
-      setImports(allDefs, imports, [type.toString()]);
-
-      return createDocComments(6, documentation) +
-        indent(6)(`${stringCamelCase(name.toString())}: AugmentedConst<${type}>;`);
-    }))
-    .concat([indent(4)('};')]);
-}
+const template = readTemplate('consts');
+const generateForMetaTemplate = Handlebars.compile(template);
 
 /** @internal */
 function generateForMeta (meta: Metadata, dest: string, extraTypes: Record<string, Record<string, { types: Record<string, any> }>>, isStrict: boolean): void {
@@ -42,27 +23,47 @@ function generateForMeta (meta: Metadata, dest: string, extraTypes: Record<strin
     const allDefs = Object.entries(allTypes).reduce((defs, [path, obj]) => {
       return Object.entries(obj).reduce((defs, [key, value]) => ({ ...defs, [`${path}/${key}`]: value }), defs);
     }, {});
-    const body = meta.asLatest.modules.sort((a, b) => a.name.localeCompare(b.name.toString())).reduce((acc, mod): string[] => {
-      return acc.concat(generateModule(allDefs, mod, imports, isStrict));
-    }, [] as string[]);
-    const header = createImportCode(HEADER('chain'), imports, [
+
+    const modules = meta.asLatest.modules
+      .sort(compareName)
+      .filter((mod) => mod.constants.length > 0)
+      .map(({ constants, name }) => {
+        if (!isStrict) {
+          setImports(allDefs, imports, ['Codec']);
+        }
+
+        const items = constants
+          .sort(compareName)
+          .map(({ documentation, name, type }) => {
+            setImports(allDefs, imports, [type.toString()]);
+
+            return {
+              docs: documentation,
+              name: stringCamelCase(name.toString()),
+              type: type.toString()
+            };
+          });
+
+        return {
+          items,
+          name: stringCamelCase(name.toString())
+        };
+      });
+
+    const types = [
       ...Object.keys(imports.localTypes).sort().map((packagePath): { file: string; types: string[] } => ({
         file: packagePath,
         types: Object.keys(imports.localTypes[packagePath])
       }))
-    ]);
-    const interfaceStart = [
-      "declare module '@polkadot/metadata/Decorated/consts/types' {",
-      indent(2)('export interface Constants {\n')
-    ].join('\n');
-    const interfaceEnd = `\n${indent(2)('}')}\n}`;
+    ];
 
-    return header
-      .concat(interfaceStart)
-      .concat(isStrict ? '' : indent(4)('[index: string]: ModuleConstants;\n'))
-      .concat(body.join('\n'))
-      .concat(interfaceEnd)
-      .concat(FOOTER);
+    return generateForMetaTemplate({
+      headerType: 'chain',
+      imports,
+      isStrict,
+      modules,
+      types
+    });
   });
 }
 

packages/typegen/src/generate/interfaceRegistry.ts

@@ -2,17 +2,22 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
+import Handlebars from 'handlebars';
+
 import Raw from '@polkadot/types/codec/Raw';
 import * as defaultDefinitions from '@polkadot/types/interfaces/definitions';
 import * as defaultPrimitives from '@polkadot/types/primitive';
 
-import { FOOTER, HEADER, createImportCode, createImports, getDerivedTypes, indent, setImports, writeFile } from '../util';
+import { createImports, getDerivedTypes, readTemplate, setImports, writeFile } from '../util';
 
 const primitiveClasses = {
   ...defaultPrimitives,
   Raw
 };
 
+const template = readTemplate('interfaceRegistry');
+const generateInterfaceTypesTemplate = Handlebars.compile(template);
+
 /** @internal */
 export function generateInterfaceTypes (importDefinitions: { [importPath: string]: object }, dest: string): void {
   writeFile(dest, (): string => {
@@ -21,52 +26,43 @@ export function generateInterfaceTypes (importDefinitions: { [importPath: string
     const imports = createImports(importDefinitions);
     const definitions = imports.definitions;
 
-    const primitives = Object
+    const items: string[] = [];
+
+    Object
       .keys(primitiveClasses)
       .filter((name): boolean => !!name.indexOf('Generic'))
-      .reduce((accumulator, primitiveName): string => {
+      .forEach((primitiveName) => {
         setImports(definitions, imports, [primitiveName]);
 
-        return [
-          accumulator,
-          getDerivedTypes(definitions, primitiveName, primitiveName, imports).map(indent(4)).join('\n')
-        ].join('\n');
-      }, '');
+        items.push(...getDerivedTypes(definitions, primitiveName, primitiveName, imports));
+      });
 
     const existingTypes: Record<string, boolean> = {};
 
-    // eslint-disable-next-line @typescript-eslint/no-unused-vars
-    const srml = Object.entries(definitions).reduce((accumulator, [_defName, { types }]): string => {
+    Object.entries(definitions).forEach(([, { types }]) => {
       setImports(definitions, imports, Object.keys(types));
 
       const uniqueTypes = Object.keys(types).filter((type) => !existingTypes[type]);
 
-      uniqueTypes.forEach((type) => { existingTypes[type] = true; });
+      uniqueTypes.forEach((type) => {
+        existingTypes[type] = true;
+        items.push(...getDerivedTypes(definitions, type, types[type], imports));
+      });
+    });
 
-      return [
-        accumulator,
-        ...uniqueTypes.map((type): string =>
-          getDerivedTypes(definitions, type, types[type], imports).map(indent(4)).join('\n')
-        )
-      ].join('\n');
-    }, '');
-
-    const header = createImportCode(HEADER('defs'), imports, [
+    const types = [
       ...Object.keys(imports.localTypes).sort().map((packagePath): { file: string; types: string[] } => ({
         file: packagePath,
         types: Object.keys(imports.localTypes[packagePath])
       }))
-    ]);
+    ];
 
-    const interfaceStart = "declare module '@polkadot/types/types/registry' {\n  export interface InterfaceTypes {";
-    const interfaceEnd = '\n  }\n}';
-
-    return header
-      .concat(interfaceStart)
-      .concat(primitives)
-      .concat(srml)
-      .concat(interfaceEnd)
-      .concat(FOOTER);
+    return generateInterfaceTypesTemplate({
+      headerType: 'defs',
+      imports,
+      items,
+      types
+    });
   });
 }
 

packages/typegen/src/generate/query.ts

@@ -2,7 +2,9 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
-import { ModuleMetadataLatest, StorageEntryMetadataLatest } from '@polkadot/types/interfaces/metadata';
+import Handlebars from 'handlebars';
+
+import { StorageEntryMetadataLatest } from '@polkadot/types/interfaces/metadata';
 import { Registry } from '@polkadot/types/types';
 
 import staticData from '@polkadot/metadata/Metadata/static';
@@ -12,7 +14,7 @@ import { unwrapStorageType } from '@polkadot/types/primitive/StorageKey';
 import { TypeRegistry } from '@polkadot/types/create';
 import { stringCamelCase } from '@polkadot/util';
 
-import { FOOTER, HEADER, TypeImports, createDocComments, createImportCode, createImports, formatType, getSimilarTypes, indent, registerDefinitions, setImports, writeFile } from '../util';
+import { TypeImports, createImports, compareName, formatType, getSimilarTypes, readTemplate, registerDefinitions, setImports, writeFile } from '../util';
 
 // From a storage entry metadata, we return [args, returnType]
 /** @internal */
@@ -60,28 +62,8 @@ function entrySignature (allDefs: object, registry: Registry, storageEntry: Stor
   throw new Error(`entryArgs: Cannot parse args of entry ${storageEntry.name}`);
 }
 
-// Generate types for one module
-/** @internal */
-function generateModule (allDefs: object, registry: Registry, { name, storage }: ModuleMetadataLatest, imports: TypeImports, isStrict: boolean): string[] {
-  if (storage.isNone) {
-    return [];
-  }
-
-  return [indent(4)(`${stringCamelCase(name.toString())}: {`)]
-    .concat(isStrict ? '' : indent(6)('[index: string]: QueryableStorageEntry<ApiType>;'))
-    .concat(storage.unwrap().items.sort((a, b) => a.name.localeCompare(b.name.toString())).map((storageEntry): string => {
-      const [args, returnType] = entrySignature(allDefs, registry, storageEntry, imports);
-      let entryType = 'AugmentedQuery';
-
-      if (storageEntry.type.isDoubleMap) {
-        entryType = `${entryType}DoubleMap`;
-      }
-
-      return createDocComments(6, storageEntry.documentation) +
-      indent(6)(`${stringCamelCase(storageEntry.name.toString())}: ${entryType}<ApiType, (${args}) => Observable<${returnType}>>${isStrict ? '' : ' & QueryableStorageEntry<ApiType>'};`);
-    }))
-    .concat([indent(4)('};')]);
-}
+const template = readTemplate('query');
+const generateForMetaTemplate = Handlebars.compile(template);
 
 /** @internal */
 function generateForMeta (registry: Registry, meta: Metadata, dest: string, extraTypes: Record<string, Record<string, { types: Record<string, any> }>>, isStrict: boolean): void {
@@ -91,13 +73,39 @@ function generateForMeta (registry: Registry, meta: Metadata, dest: string, extr
     const allDefs = Object.entries(allTypes).reduce((defs, [path, obj]) => {
       return Object.entries(obj).reduce((defs, [key, value]) => ({ ...defs, [`${path}/${key}`]: value }), defs);
     }, {});
-    const body = meta.asLatest.modules.sort((a, b) => a.name.localeCompare(b.name.toString())).reduce((acc: string[], mod): string[] => {
-      return acc.concat(generateModule(allDefs, registry, mod, imports, isStrict));
-    }, []);
+
+    const modules = meta.asLatest.modules
+      .sort(compareName)
+      .filter((mod) => !mod.storage.isNone)
+      .map(({ name, storage }) => {
+        const items = storage.unwrap().items
+          .sort(compareName)
+          .map((storageEntry) => {
+            const [args, returnType] = entrySignature(allDefs, registry, storageEntry, imports);
+            let entryType = 'AugmentedQuery';
+
+            if (storageEntry.type.isDoubleMap) {
+              entryType = `${entryType}DoubleMap`;
+            }
+
+            return {
+              args,
+              docs: storageEntry.documentation,
+              entryType,
+              name: stringCamelCase(storageEntry.name.toString()),
+              returnType
+            };
+          });
+
+        return {
+          items,
+          name: stringCamelCase(name.toString())
+        };
+      });
 
     imports.typesTypes.Observable = true;
 
-    const header = createImportCode(HEADER('chain'), imports, [
+    const types = [
       ...Object.keys(imports.localTypes).sort().map((packagePath): { file: string; types: string[] } => ({
         file: packagePath,
         types: Object.keys(imports.localTypes[packagePath])
@@ -106,24 +114,15 @@ function generateForMeta (registry: Registry, meta: Metadata, dest: string, extr
         file: '@polkadot/api/types',
         types: ['ApiTypes']
       }
-    ]);
-    const interfaceStart = [
-      "declare module '@polkadot/api/types/storage' {",
-      indent(2)('export interface AugmentedQueries<ApiType> {\n')
-    ].join('\n');
-    const interfaceEnd = `\n${indent(2)('}')}\n\n`;
-    const queryableStorageInterface = indent(2)('export interface QueryableStorage<ApiType extends ApiTypes> extends AugmentedQueries<ApiType> {')
-      .concat('\n')
-      .concat(isStrict ? '' : indent(4)('[index: string]: QueryableModuleStorage<ApiType>;\n'))
-      .concat(indent(2)('}\n'));
+    ];
 
-    return header
-      .concat(interfaceStart)
-      .concat(body.join('\n'))
-      .concat(interfaceEnd)
-      .concat(queryableStorageInterface)
-      .concat('}')
-      .concat(FOOTER);
+    return generateForMetaTemplate({
+      headerType: 'chain',
+      imports,
+      isStrict,
+      modules,
+      types
+    });
   });
 }
 

packages/typegen/src/generate/rpc.ts

@@ -2,13 +2,18 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
+import Handlebars from 'handlebars';
+
 import { TypeRegistry } from '@polkadot/types/create';
 import * as definitions from '@polkadot/types/interfaces/definitions';
 
-import { FOOTER, HEADER, createDocComments, createImportCode, createImports, getSimilarTypes, setImports, writeFile, indent } from '../util';
+import { createImports, getSimilarTypes, readTemplate, setImports, writeFile } from '../util';
 
 const StorageKeyTye = 'StorageKey | string | Uint8Array | any';
 
+const template = readTemplate('rpc');
+const generateRpcTypesTemplate = Handlebars.compile(template);
+
 /** @internal */
 export default function generateRpcTypes (dest = 'packages/api/src/augment/rpc.ts'): void {
   writeFile(dest, (): string => {
@@ -22,62 +27,80 @@ export default function generateRpcTypes (dest = 'packages/api/src/augment/rpc.t
       .keys(definitions)
       .filter((key) => Object.keys(definitions[key as 'babe'].rpc || {}).length !== 0)
       .sort();
-    const body = rpcKeys.reduce<string[]>((allSections, section): string[] => {
-      const allMethods = Object.keys(definitions[section as 'babe'].rpc).sort().map((methodName): string => {
+    const modules = rpcKeys.map((section) => {
+      const allMethods = Object.keys(definitions[section as 'babe'].rpc).sort().map((methodName) => {
         const def = definitions[section as 'babe'].rpc[methodName];
 
+        let args;
+        let type;
+        let generic;
+
         // These are too hard to type with generics, do manual overrides
         if (section === 'state') {
           setImports(allDefs, imports, ['Codec', 'Hash', 'StorageKey', 'Vec']);
 
           if (methodName === 'getStorage') {
-            return createDocComments(6, [def.description]) + indent(6)(`getStorage: AugmentedRpc<<T = Codec>(key: ${StorageKeyTye}, block?: Hash | Uint8Array | string) => Observable<T>>;`);
+            generic = 'T = Codec';
+            args = [`key: ${StorageKeyTye}, block?: Hash | Uint8Array | string`];
+            type = 'T';
           } else if (methodName === 'queryStorage') {
-            return createDocComments(6, [def.description]) + indent(6)(`queryStorage: AugmentedRpc<<T = Codec[]>(keys: Vec<StorageKey> | (${StorageKeyTye})[], fromBlock?: Hash | Uint8Array | string, toBlock?: Hash | Uint8Array | string) => Observable<[Hash, T][]>>;`);
+            generic = 'T = Codec[]';
+            args = [`keys: Vec<StorageKey> | (${StorageKeyTye})[], fromBlock?: Hash | Uint8Array | string, toBlock?: Hash | Uint8Array | string`];
+            type = '[Hash, T][]';
           } else if (methodName === 'queryStorageAt') {
-            return createDocComments(6, [def.description]) + indent(6)(`queryStorageAt: AugmentedRpc<<T = Codec[]>(keys: Vec<StorageKey> | (${StorageKeyTye})[], at?: Hash | Uint8Array | string) => Observable<T>>;`);
+            generic = 'T = Codec[]';
+            args = [`keys: Vec<StorageKey> | (${StorageKeyTye})[], at?: Hash | Uint8Array | string`];
+            type = 'T';
           } else if (methodName === 'subscribeStorage') {
-            return createDocComments(6, [def.description]) + indent(6)(`subscribeStorage: AugmentedRpc<<T = Codec[]>(keys?: Vec<StorageKey> | (${StorageKeyTye})[]) => Observable<T>>;`);
+            generic = 'T = Codec[]';
+            args = [`keys?: Vec<StorageKey> | (${StorageKeyTye})[]`];
+            type = 'T';
           }
         }
 
-        setImports(allDefs, imports, [def.type]);
+        if (args === undefined) {
+          setImports(allDefs, imports, [def.type]);
+          args = def.params.map((param: any) => {
+            const similarTypes = getSimilarTypes(definitions, registry, param.type, imports);
 
-        const args = def.params.map((param: any): string => {
-          const similarTypes = getSimilarTypes(definitions, registry, param.type, imports);
+            setImports(allDefs, imports, [param.type, ...similarTypes]);
 
-          setImports(allDefs, imports, [param.type, ...similarTypes]);
+            return `${param.name}${param.isOptional ? '?' : ''}: ${similarTypes.join(' | ')}`;
+          });
 
-          return `${param.name}${param.isOptional ? '?' : ''}: ${similarTypes.join(' | ')}`;
-        });
+          type = def.type;
+          generic = '';
+        }
 
-        return createDocComments(6, [def.description]) + indent(6)(`${methodName}: AugmentedRpc<(${args.join(', ')}) => Observable<${def.type}>>;`);
+        return {
+          args: args.join(', '),
+          docs: [def.description],
+          generic,
+          name: methodName,
+          type
+        };
       });
 
-      return allSections.concat(
-        [
-          `    ${section}: {`,
-          ...allMethods,
-          '    };'
-        ].join('\n')
-      );
-    }, []).join('\n');
+      return {
+        items: allMethods,
+        name: section
+      };
+    });
 
     imports.typesTypes.Observable = true;
 
-    const header = createImportCode(HEADER('chain'), imports, [
+    const types = [
       ...Object.keys(imports.localTypes).sort().map((packagePath): { file: string; types: string[] } => ({
         file: packagePath,
         types: Object.keys(imports.localTypes[packagePath])
       }))
-    ]);
-    const interfaceStart = "declare module '@polkadot/rpc-core/types.jsonrpc' {\n  export interface RpcInterface {\n";
-    const interfaceEnd = '\n  }\n}';
+    ];
 
-    return header
-      .concat(interfaceStart)
-      .concat(body)
-      .concat(interfaceEnd)
-      .concat(FOOTER);
+    return generateRpcTypesTemplate({
+      headerType: 'chain',
+      imports,
+      modules,
+      types
+    });
   });
 }

packages/typegen/src/generate/tsDef.ts

@@ -2,6 +2,8 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
+import Handlebars from 'handlebars';
+
 import { TypeDef, TypeDefInfo } from '@polkadot/types/create/types';
 
 import path from 'path';
@@ -9,7 +11,7 @@ import { getTypeDef } from '@polkadot/types/create';
 import * as defaultDefinitions from '@polkadot/types/interfaces/definitions';
 import { isString, stringCamelCase, stringUpperFirst } from '@polkadot/util';
 
-import { FOOTER, HEADER, TypeImports, createImportCode, createImports, exportInterface, exportType, formatType, setImports, writeFile } from '../util';
+import { TypeImports, createImports, exportInterface, exportType, readTemplate, formatType, setImports, writeFile } from '../util';
 
 interface Imports extends TypeImports {
   interfaces: [string, string][];
@@ -55,6 +57,7 @@ function tsEnum (definitions: object, { name: enumName, sub }: TypeDef, imports:
     const isGetter = createGetter(definitions, `is${getter}`, 'boolean', imports);
 
     switch (info) {
+      case TypeDefInfo.Compact:
       case TypeDefInfo.Plain:
       case TypeDefInfo.Tuple:
       case TypeDefInfo.Vec:
@@ -181,22 +184,37 @@ function generateInterfaces (definitions: object, { types }: { types: Record<str
   });
 }
 
+const templateIndex = readTemplate('tsDef/index');
+const generateTsDefIndexTemplate = Handlebars.compile(templateIndex);
+
+const templateModuleTypes = readTemplate('tsDef/moduleTypes');
+const generateTsDefModuleTypesTemplate = Handlebars.compile(templateModuleTypes);
+
+const templateTypes = readTemplate('tsDef/types');
+const generateTsDefTypesTemplate = Handlebars.compile(templateTypes);
+
 /** @internal */
 function generateTsDefFor (importDefinitions: { [importPath: string]: object }, defName: string, { types }: { types: Record<string, any> }, outputDir: string): void {
   const imports = { ...createImports(importDefinitions, { types }), interfaces: [] } as Imports;
   const definitions = imports.definitions;
   const interfaces = generateInterfaces(definitions, { types }, imports);
-  const sortedDefs = interfaces.sort((a, b): number => a[0].localeCompare(b[0])).map(([, definition]): string => definition).join('\n\n');
+  const items = interfaces.sort((a, b): number => a[0].localeCompare(b[0])).map(([, definition]): string => definition);
 
-  const header = createImportCode(HEADER('defs'), imports, [
+  const importTypes = [
     ...Object.keys(imports.localTypes).sort().map((packagePath): { file: string; types: string[] } => ({
       file: packagePath,
       types: Object.keys(imports.localTypes[packagePath])
     }))
-  ]);
+  ];
 
-  writeFile(path.join(outputDir, defName, 'types.ts'), () => header.concat(sortedDefs).concat(`\n\nexport type PHANTOM_${defName.toUpperCase()} = '${defName}';`).concat(FOOTER), true);
-  writeFile(path.join(outputDir, defName, 'index.ts'), () => HEADER('defs').concat('export * from \'./types\';').concat(FOOTER), true);
+  writeFile(path.join(outputDir, defName, 'types.ts'), () => generateTsDefModuleTypesTemplate({
+    headerType: 'defs',
+    imports,
+    items,
+    name: defName,
+    types: importTypes
+  }), true);
+  writeFile(path.join(outputDir, defName, 'index.ts'), () => generateTsDefIndexTemplate({ headerType: 'defs' }), true);
 }
 
 /** @internal */
@@ -210,17 +228,13 @@ export function generateTsDef (importDefinitions: { [importPath: string]: object
       generateTsDefFor(importDefinitions, defName, obj, outputDir);
     });
 
-    return HEADER('defs')
-      .concat(
-        Object
-          .keys(definitions)
-          .map((moduleName): string => `export * from './${moduleName}/types';`)
-          .join('\n')
-      )
-      .concat(FOOTER);
+    return generateTsDefTypesTemplate({
+      headerType: 'defs',
+      items: Object.keys(definitions)
+    });
   });
 
-  writeFile(path.join(outputDir, 'index.ts'), () => HEADER('defs').concat('export * from \'./types\';').concat(FOOTER), true);
+  writeFile(path.join(outputDir, 'index.ts'), () => generateTsDefIndexTemplate({ headerType: 'defs' }), true);
 }
 
 /** @internal */

packages/typegen/src/generate/tx.ts

@@ -2,7 +2,8 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
-import { ModuleMetadataLatest } from '@polkadot/types/interfaces/metadata';
+import Handlebars from 'handlebars';
+
 import { Registry } from '@polkadot/types/types';
 
 import staticData from '@polkadot/metadata/Metadata/static';
@@ -12,7 +13,7 @@ import { Text } from '@polkadot/types/primitive';
 import { TypeRegistry } from '@polkadot/types/create';
 import { stringCamelCase } from '@polkadot/util';
 
-import { FOOTER, HEADER, TypeImports, createDocComments, createImportCode, createImports, formatType, getSimilarTypes, indent, registerDefinitions, setImports, writeFile } from '../util';
+import { createImports, compareName, formatType, getSimilarTypes, readTemplate, registerDefinitions, setImports, writeFile } from '../util';
 
 const MAPPED_NAMES: Record<string, string> = {
   new: 'updated'
@@ -24,39 +25,8 @@ function mapName (_name: Text): string {
   return MAPPED_NAMES[name] || name;
 }
 
-// Generate types for one module
-/** @internal */
-// eslint-disable-next-line @typescript-eslint/no-unused-vars
-function generateModule (registry: Registry, allDefs: object, { calls, name }: ModuleMetadataLatest, imports: TypeImports, isStrict: boolean): string[] {
-  const allCalls = calls.unwrapOr(null);
-
-  if (!allCalls?.length) {
-    return [];
-  }
-
-  setImports(allDefs, imports, ['Call', 'Extrinsic', 'SubmittableExtrinsic']);
-
-  return [indent(4)(`${stringCamelCase(name.toString())}: {`)]
-    .concat(isStrict ? '' : indent(6)('[index: string]: SubmittableExtrinsicFunction<ApiType>;'))
-    .concat(allCalls.sort((a, b) => a.name.localeCompare(b.name.toString())).map(({ args, documentation, name }): string => {
-      const params = args
-        .map(({ name, type }): [string, string, string] => {
-          const typeStr = type.toString();
-          const similarTypes = getSimilarTypes(allDefs, registry, typeStr, imports).map((type): string => formatType(allDefs, type, imports));
-          const nameStr = mapName(name);
-
-          setImports(allDefs, imports, [...similarTypes.filter((type): boolean => !type.startsWith('(') && !type.startsWith('{')), typeStr]);
-
-          return [`${nameStr}: ${similarTypes.join(' | ')}`, nameStr, typeStr];
-        })
-        .map(([full]): string => full)
-        .join(', ');
-
-      return createDocComments(6, documentation) +
-      indent(6)(`${stringCamelCase(name.toString())}: AugmentedSubmittable<(${params}) => SubmittableExtrinsic<ApiType>>;`);
-    }))
-    .concat([indent(4)('};')]);
-}
+const template = readTemplate('tx');
+const generateForMetaTemplate = Handlebars.compile(template);
 
 /** @internal */
 function generateForMeta (registry: Registry, meta: Metadata, dest: string, extraTypes: Record<string, Record<string, { types: Record<string, any> }>>, isStrict: boolean): void {
@@ -66,10 +36,42 @@ function generateForMeta (registry: Registry, meta: Metadata, dest: string, extr
     const allDefs = Object.entries(allTypes).reduce((defs, [path, obj]) => {
       return Object.entries(obj).reduce((defs, [key, value]) => ({ ...defs, [`${path}/${key}`]: value }), defs);
     }, {});
-    const body = meta.asLatest.modules.sort((a, b) => a.name.localeCompare(b.name.toString())).reduce((acc, mod): string[] => {
-      return acc.concat(generateModule(registry, allDefs, mod, imports, isStrict));
-    }, [] as string[]);
-    const header = createImportCode(HEADER('chain'), imports, [
+
+    const modules = meta.asLatest.modules
+      .sort(compareName)
+      .filter(({ calls }) => calls.unwrapOr([]).length)
+      .map(({ calls, name }) => {
+        setImports(allDefs, imports, ['Call', 'Extrinsic', 'SubmittableExtrinsic']);
+
+        const items = calls.unwrap()
+          .sort(compareName)
+          .map(({ args, documentation, name }) => {
+            const params = args
+              .map(({ name, type }) => {
+                const typeStr = type.toString();
+                const similarTypes = getSimilarTypes(allDefs, registry, typeStr, imports).map((type): string => formatType(allDefs, type, imports));
+                const nameStr = mapName(name);
+
+                setImports(allDefs, imports, [...similarTypes.filter((type): boolean => !type.startsWith('(') && !type.startsWith('{')), typeStr]);
+
+                return `${nameStr}: ${similarTypes.join(' | ')}`;
+              })
+              .join(', ');
+
+            return {
+              docs: documentation,
+              name: stringCamelCase(name.toString()),
+              params
+            };
+          });
+
+        return {
+          items,
+          name: stringCamelCase(name.toString())
+        };
+      });
+
+    const types = [
       ...Object.keys(imports.localTypes).sort().map((packagePath): { file: string; types: string[] } => ({
         file: packagePath,
         types: Object.keys(imports.localTypes[packagePath])
@@ -78,25 +80,15 @@ function generateForMeta (registry: Registry, meta: Metadata, dest: string, extr
         file: '@polkadot/api/types',
         types: ['ApiTypes', 'SubmittableExtrinsic']
       }
-    ]);
-    const interfaceStart = [
-      "declare module '@polkadot/api/types/submittable' {",
-      indent(2)('export interface AugmentedSubmittables<ApiType> {\n')
-    ].join('\n');
-    const interfaceEnd = `\n${indent(2)('}')}\n\n`;
-    const submittableExtrinsicsInterface = indent(2)('export interface SubmittableExtrinsics<ApiType extends ApiTypes> extends AugmentedSubmittables<ApiType> {')
-      .concat('\n')
-      .concat(indent(4)('(extrinsic: Call | Extrinsic | Uint8Array | string): SubmittableExtrinsic<ApiType>;\n'))
-      .concat(isStrict ? '' : indent(4)('[index: string]: SubmittableModuleExtrinsics<ApiType>;\n'))
-      .concat(indent(2)('}\n'));
+    ];
 
-    return header
-      .concat(interfaceStart)
-      .concat(body.join('\n'))
-      .concat(interfaceEnd)
-      .concat(submittableExtrinsicsInterface)
-      .concat('}')
-      .concat(FOOTER);
+    return generateForMetaTemplate({
+      headerType: 'chain',
+      imports,
+      isStrict,
+      modules,
+      types
+    });
   });
 }
 

packages/typegen/src/metadataMd.ts

@@ -196,8 +196,7 @@ function addStorage (metadata: MetadataLatest): string {
       };
     });
 
-  const options = { encoding: 'utf8', flags: 'r' };
-  const knownSection = JSON.parse(fs.readFileSync('docs/substrate/storage-known-section.json', options));
+  const knownSection = JSON.parse(fs.readFileSync('docs/substrate/storage-known-section.json', 'utf8'));
 
   return renderPage({
     description: DESC_STORAGE,
@@ -283,8 +282,7 @@ function addErrors (metadata: MetadataLatest): string {
 
 /** @internal */
 function writeFile (name: string, ...chunks: any[]): void {
-  const options = { encoding: 'utf8', flags: 'w' };
-  const writeStream = fs.createWriteStream(name, options);
+  const writeStream = fs.createWriteStream(name, { encoding: 'utf8', flags: 'w' });
 
   writeStream.on('finish', (): void => {
     console.log(`Completed writing ${name}`);

packages/typegen/src/templates/consts.hbs

@@ -0,0 +1,23 @@
+{{> header }}
+
+{{{ imports }}}
+
+declare module '@polkadot/metadata/Decorated/consts/types' {
+  export interface Constants {
+    {{#unless isStrict}}
+    [index: string]: ModuleConstants;
+    {{/unless}}
+    {{#each modules}}
+    {{{name}}}: {
+      {{#unless @root.isStrict}}
+      [index: string]: AugmentedConst<object & Codec>;
+      {{/unless}}
+      {{~#each items}}
+      {{> docs}}
+      {{{name}}}: AugmentedConst<{{{type}}}>;
+      {{/each}}
+    };
+    {{/each}}
+  }
+}
+{{> footer }}

packages/typegen/src/templates/docs.hbs

@@ -0,0 +1,7 @@
+{{#if docs.length}}
+/**
+{{#each docs}}
+ * {{#trim}}{{{this}}}{{/trim}}
+{{/each}}
+ **/
+{{/if}}

packages/typegen/src/templates/header.hbs

@@ -0,0 +1,2 @@
+// Auto-generated via `yarn polkadot-types-from-{{headerType}}`, do not edit
+/* eslint-disable */

packages/typegen/src/templates/interfaceRegistry.hbs

@@ -0,0 +1,12 @@
+{{> header }}
+
+{{{ imports }}}
+
+declare module '@polkadot/types/types/registry' {
+  export interface InterfaceTypes {
+    {{#each items}}
+    {{{this}}}
+    {{/each}}
+  }
+}
+{{> footer }}

packages/typegen/src/templates/query.hbs

@@ -0,0 +1,26 @@
+{{> header }}
+
+{{{ imports }}}
+
+declare module '@polkadot/api/types/storage' {
+  export interface AugmentedQueries<ApiType> {
+    {{#each modules}}
+    {{{name}}}: {
+      {{#unless @root.isStrict}}
+      [index: string]: QueryableStorageEntry<ApiType>;
+      {{/unless}}
+      {{~#each items}}
+      {{> docs}}
+      {{{name}}}: {{{entryType}}}<ApiType, ({{{args}}}) => Observable<{{{returnType}}}>>{{#unless @root.isStrict}} & QueryableStorageEntry<ApiType>{{/unless}};
+      {{/each}}
+    };
+    {{/each}}
+  }
+
+  export interface QueryableStorage<ApiType extends ApiTypes> extends AugmentedQueries<ApiType> {
+    {{#unless isStrict}}
+    [index: string]: QueryableModuleStorage<ApiType>;
+    {{/unless}}
+  }
+}
+{{> footer }}

packages/typegen/src/templates/rpc.hbs

@@ -0,0 +1,17 @@
+{{> header }}
+
+{{{ imports }}}
+
+declare module '@polkadot/rpc-core/types.jsonrpc' {
+  export interface RpcInterface {
+    {{#each modules}}
+    {{{name}}}: {
+      {{#each items}}
+      {{> docs}}
+      {{{name}}}: AugmentedRpc<{{#if generic}}<{{{generic}}}>{{/if}}({{{args}}}) => Observable<{{{type}}}>>;
+      {{/each}}
+    };
+    {{/each}}
+  }
+}
+{{> footer }}

packages/typegen/src/templates/tsDef/index.hbs

@@ -0,0 +1,4 @@
+{{> header }}
+
+export * from './types';
+{{> footer }}

packages/typegen/src/templates/tsDef/moduleTypes.hbs

@@ -0,0 +1,11 @@
+{{> header }}
+
+{{{ imports }}}
+
+{{#each items}}
+{{{this}}}
+
+{{/each}}
+
+export type PHANTOM_{{#upper}}{{{name}}}{{/upper}} = '{{{name}}}';
+{{> footer }}

packages/typegen/src/templates/tsDef/types.hbs

@@ -0,0 +1,6 @@
+{{> header }}
+
+{{#each items}}
+export * from './{{{this}}}/types';
+{{/each}}
+{{> footer }}

packages/typegen/src/templates/tx.hbs

@@ -0,0 +1,27 @@
+{{> header }}
+
+{{{ imports }}}
+
+declare module '@polkadot/api/types/submittable' {
+  export interface AugmentedSubmittables<ApiType> {
+    {{#each modules}}
+    {{{name}}}: {
+      {{#unless @root.isStrict}}
+      [index: string]: SubmittableExtrinsicFunction<ApiType>;
+      {{/unless}}
+      {{~#each items}}
+      {{> docs}}
+      {{{name}}}: AugmentedSubmittable<({{{params}}}) => SubmittableExtrinsic<ApiType>>;
+      {{/each}}
+    };
+    {{/each}}
+  }
+
+  export interface SubmittableExtrinsics<ApiType extends ApiTypes> extends AugmentedSubmittables<ApiType> {
+    (extrinsic: Call | Extrinsic | Uint8Array | string): SubmittableExtrinsic<ApiType>;
+    {{#unless isStrict}}
+    [index: string]: SubmittableModuleExtrinsics<ApiType>;
+    {{/unless}}
+  }
+}
+{{> footer }}

packages/typegen/src/util/docs.ts

@@ -2,22 +2,10 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
-import { Text } from '@polkadot/types';
+import Handlebars from 'handlebars';
 
-import { indent } from './formatting';
+import { readTemplate } from './file';
 
-type AnyString = Text | string;
-type Arg = [AnyString, AnyString];
-
-export function createDocComments (spaces: number, docs: AnyString[], args: Arg[] = []): string {
-  const contents = [
-    ...docs.map((doc): string => doc.toString().trim()).filter((doc): boolean => !!doc),
-    ...args.map(([name, type]): string => `@param ${name} ${type}`)
-  ].map((d): string => ` * ${d}`);
-
-  return contents.length
-    ? ['/**', ...contents, ' **/\n']
-      .map((s): string => indent(spaces)(s))
-      .join('\n')
-    : '';
-}
+Handlebars.registerPartial({
+  docs: Handlebars.compile(readTemplate('docs'))
+});

packages/typegen/src/util/file.ts

@@ -19,3 +19,7 @@ export function writeFile (dest: string, generator: () => string, noLog?: boolea
 
   !noLog && console.log('');
 }
+
+export function readTemplate (path: string): string {
+  return fs.readFileSync(`${__dirname}/../templates/${path}.hbs`).toString();
+}

packages/typegen/src/util/formatting.spec.ts

@@ -1,11 +0,0 @@
-// Copyright 2017-2020 @polkadot/typegen authors & contributors
-// This software may be modified and distributed under the terms
-// of the Apache-2.0 license. See the LICENSE file for details.
-
-import { indent } from './formatting';
-
-describe('formatting', () => {
-  it('should indent correctly', () => {
-    expect(indent(2)('foo')).toBe('  foo');
-  });
-});

packages/typegen/src/util/formatting.ts

@@ -2,59 +2,74 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
+import Handlebars from 'handlebars';
+
 import { TypeDef, TypeDefInfo } from '@polkadot/types/create/types';
 
 import { getTypeDef } from '@polkadot/types/create';
 import { paramsNotation } from '@polkadot/types/codec/utils';
 
 import { setImports, TypeImports } from './imports';
-
-export const HEADER = (type: 'chain' | 'defs'): string => `// Auto-generated via \`yarn polkadot-types-from-${type}\`, do not edit\n/* eslint-disable */\n\n`;
-export const FOOTER = '\n';
+import { readTemplate } from './file';
 
 const TYPES_NON_PRIMITIVE = ['Metadata'];
 
-// creates the import lines
-/** @internal */
-export function createImportCode (header: string, imports: TypeImports, checks: { file: string; types: string[] }[]): string {
-  return [
-    {
-      file: '@polkadot/types/types',
-      types: Object.keys(imports.typesTypes)
-    },
-    {
-      file: '@polkadot/types/codec',
-      types: Object
-        .keys(imports.codecTypes)
-        .filter((name): boolean => name !== 'Tuple')
-    },
-    {
-      file: '@polkadot/types/extrinsic',
-      types: Object.keys(imports.extrinsicTypes)
-    },
-    {
-      file: '@polkadot/types/generic',
-      types: Object.keys(imports.genericTypes)
-    },
-    {
-      file: '@polkadot/types/primitive',
-      types: Object
-        .keys(imports.primitiveTypes)
-        .filter((name): boolean => !TYPES_NON_PRIMITIVE.includes(name))
-    },
-    {
-      file: '@polkadot/types',
-      types: Object
-        .keys(imports.primitiveTypes)
-        .filter((name): boolean => TYPES_NON_PRIMITIVE.includes(name))
-    },
-    ...checks
-  ].reduce((result, { file, types }): string => {
-    return types.length
-      ? `${result}import { ${types.sort().join(', ')} } from '${file}';\n`
-      : result;
-  }, header) + '\n';
-}
+export const HEADER = (type: 'chain' | 'defs'): string => `// Auto-generated via \`yarn polkadot-types-from-${type}\`, do not edit\n/* eslint-disable */\n\n`;
+
+Handlebars.registerPartial({
+  footer: Handlebars.compile(readTemplate('footer')),
+  header: Handlebars.compile(readTemplate('header'))
+});
+
+Handlebars.registerHelper({
+  imports () {
+    const { imports, types } = this as any;
+
+    return [
+      {
+        file: '@polkadot/types/types',
+        types: Object.keys(imports.typesTypes)
+      },
+      {
+        file: '@polkadot/types/codec',
+        types: Object
+          .keys(imports.codecTypes)
+          .filter((name): boolean => name !== 'Tuple')
+      },
+      {
+        file: '@polkadot/types/extrinsic',
+        types: Object.keys(imports.extrinsicTypes)
+      },
+      {
+        file: '@polkadot/types/generic',
+        types: Object.keys(imports.genericTypes)
+      },
+      {
+        file: '@polkadot/types/primitive',
+        types: Object
+          .keys(imports.primitiveTypes)
+          .filter((name): boolean => !TYPES_NON_PRIMITIVE.includes(name))
+      },
+      {
+        file: '@polkadot/types',
+        types: Object
+          .keys(imports.primitiveTypes)
+          .filter((name): boolean => TYPES_NON_PRIMITIVE.includes(name))
+      },
+      ...types
+    ].reduce((result, { file, types }): string => {
+      return types.length
+        ? `${result}import { ${types.sort().join(', ')} } from '${file}';\n`
+        : result;
+    }, '');
+  },
+  trim (options) {
+    return options.fn(this).trim();
+  },
+  upper (options) {
+    return options.fn(this).toUpperCase();
+  }
+});
 
 // helper to generate a `export interface <Name> extends <Base> {<Body>}
 /** @internal */
@@ -273,13 +288,3 @@ export function formatType (definitions: object, type: string | TypeDef, imports
     }
   }
 }
-
-/**
- * Indent a string with `n` spaces before.
- */
-/** @internal */
-export function indent (n: number, char = ' '): (str: string) => string {
-  return function (str: string): string {
-    return `${char.repeat(n)}${str}`;
-  };
-}