[CI Skip] 1.7.1

skip-checks: true

.github/workflows/pr-any.yml

@@ -2,110 +2,18 @@ name: PR
 on: [pull_request]
 
 jobs:
-  lint:
-    name: Linting
-    runs-on: ubuntu-latest
+  pr:
     strategy:
       matrix:
-        node-version: [12.x]
-    steps:
-    - uses: actions/checkout@v1
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v1
-      with:
-        node-version: ${{ matrix.node-version }}
-    - name: Config yarn cache
-      id: yarn-cache-dir-path
-      run: echo "::set-output name=dir::$(yarn cache dir)"
-    - uses: actions/cache@v1
-      id: yarn-cache
-      with:
-        path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-        key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-        restore-keys: |
-          ${{ runner.os }}-yarn-
-    - name: lint
-      run: |
-        yarn install --immutable | grep -v 'YN0013'
-        yarn lint
-
-  test:
-    name: Testing
+        step: ['lint', 'test', 'build', 'docs']
+    name: ${{ matrix.step }}
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [12.x]
     steps:
     - uses: actions/checkout@v1
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v1
+    - uses: actions/setup-node@v1
       with:
-        node-version: ${{ matrix.node-version }}
-    - name: Config yarn cache
-      id: yarn-cache-dir-path
-      run: echo "::set-output name=dir::$(yarn cache dir)"
-    - uses: actions/cache@v1
-      id: yarn-cache
-      with:
-        path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-        key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-        restore-keys: |
-          ${{ runner.os }}-yarn-
-    - name: test
+        node-version: '12.x'
+    - name: ${{ matrix.step }}
       run: |
         yarn install --immutable | grep -v 'YN0013'
-        yarn test
-
-  build_code:
-    name: Build Code
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [12.x]
-    steps:
-    - uses: actions/checkout@v1
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v1
-      with:
-        node-version: ${{ matrix.node-version }}
-    - name: Config yarn cache
-      id: yarn-cache-dir-path
-      run: echo "::set-output name=dir::$(yarn cache dir)"
-    - uses: actions/cache@v1
-      id: yarn-cache
-      with:
-        path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-        key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-        restore-keys: |
-          ${{ runner.os }}-yarn-
-    - name: build
-      run: |
-        yarn install --immutable | grep -v 'YN0013'
-        yarn build
-
-  build_docs:
-    name: Build Docs
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [12.x]
-    steps:
-    - uses: actions/checkout@v1
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v1
-      with:
-        node-version: ${{ matrix.node-version }}
-    - name: Config yarn cache
-      id: yarn-cache-dir-path
-      run: echo "::set-output name=dir::$(yarn cache dir)"
-    - uses: actions/cache@v1
-      id: yarn-cache
-      with:
-        path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-        key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-        restore-keys: |
-          ${{ runner.os }}-yarn-
-    - name: build
-      run: |
-        yarn install --immutable | grep -v 'YN0013'
-        yarn docs
+        yarn ${{ matrix.step }}

.github/workflows/push-master.yml

@@ -5,82 +5,36 @@ on:
       - master
 
 jobs:
-  build_code:
-    name: Build Code
-    if: "! contains(github.event.head_commit.message, '[CI Skip]')"
-    runs-on: ubuntu-latest
+  master:
     strategy:
       matrix:
-        node-version: [12.x]
+        step: ['build:release', 'docs:release']
+    name: ${{ matrix.step }}
+    if: "! contains(github.event.head_commit.message, '[CI Skip]')"
+    runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v1
       with:
         token: ${{ secrets.GH_PAT }}
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v1
+    - uses: actions/setup-node@v1
       with:
-        node-version: ${{ matrix.node-version }}
-    - name: Config yarn cache
-      id: yarn-cache-dir-path
-      run: echo "::set-output name=dir::$(yarn cache dir)"
-    - uses: actions/cache@v1
-      id: yarn-cache
-      with:
-        path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-        key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-        restore-keys: |
-          ${{ runner.os }}-yarn-
-    - name: build
+        node-version: '12.x'
+    - name: ${{ matrix.step }}
       env:
         CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
+        GH_PAGES_SRC: build-docs
         GH_PAT: ${{ secrets.GH_PAT }}
         GH_TOKEN: ${{ secrets.GH_PAT }}
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
       run: |
         yarn install --immutable | grep -v 'YN0013'
-        yarn polkadot-ci-ghact-build
-
-  build_docs:
-    name: Build Docs
-    if: "! contains(github.event.head_commit.message, '[CI Skip]')"
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [12.x]
-    steps:
-    - uses: actions/checkout@v1
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v1
-      with:
-        node-version: ${{ matrix.node-version }}
-    - name: Config yarn cache
-      id: yarn-cache-dir-path
-      run: echo "::set-output name=dir::$(yarn cache dir)"
-    - uses: actions/cache@v1
-      id: yarn-cache
-      with:
-        path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-        key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-        restore-keys: |
-          ${{ runner.os }}-yarn-
-    - name: build
-      env:
-        GH_PAT: ${{ secrets.GH_PAT }}
-        GH_TOKEN: ${{ secrets.GH_PAT }}
-        GH_PAGES_SRC: build-docs
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: |
-        yarn install --immutable | grep -v 'YN0013'
-        yarn polkadot-ci-ghact-docs
+        yarn ${{ matrix.step }}
 
   dummy:
     name: Dummy
     if: "contains(github.event.head_commit.message, '[CI Skip]')"
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [12.x]
     steps:
     - name: dummy
       run: |

.yarnrc.yml

@@ -1,4 +1,5 @@
 enableImmutableInstalls: false
+
 enableProgressBars: false
 
 nodeLinker: node-modules
@@ -7,4 +8,4 @@ plugins:
   - path: .yarn/plugins/@yarnpkg/plugin-interactive-tools.js
     spec: "@yarnpkg/plugin-interactive-tools"
 
-yarnPath: .yarn/releases/yarn-berry.js
+yarnPath: .yarn/releases/yarn-rc.js

CHANGELOG.md

@@ -1,6 +1,16 @@
-# 1.6.2 Feb 27, 2020
+# 1.7.1 Mar 17, 2020
 
-- Revert checks for arg, not working on `.at()` queries (proper fix in next version)
+- **Important** Current versions of Polkadot/Substrate have dropped support for linked maps on storage entries. If you are using these queries to retriev all entries, for instance `staking.{nominators,validators}()` and using an upgraded chain, you need to swap to retrieving entries via `.entries()` or `.keys()` for the keys.
+- Pull in support and types for latest Polkadot/Substrate
+- Add support for the new Substrate `state_getKeysPaged` RPC, including use in storage keys
+- Move `derive.staking.controllers` to `derive.staking.stashes`, reflecting actual content
+- Cater for adjusted storage (non-linked mapped) for `derive.staking.stashes` queries (with old-compat)
+- Expanded `derive.staking.*` derives, including addition of `derive.staking.own*`
+- Re-add fixed (with tests) checks for query args, previously disabled in 1.6.2
+
+# 1.6.2 Mar 12, 2020
+
+- Revert checks for query args, not working on `.at()` queries (proper fix in next version)
 
 # 1.6.1 Mar 12, 2020
 

copy-to-apps.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 function copy_folder () {
   SRC="packages/$1/build"

docs/start/FAQ.md

@@ -34,7 +34,7 @@ To submit these transactions, it needs to be send as a [wrapped transaction](api
 
 ## How do I call a function with a Tuple input
 
-Tuples, as defined in the Polkadot/Substrate types appear as `(TypeA, TypeB)`. For instance we may have an `(AccountId, u64)` input as defined in the metadata or as part of the user types. To specify a Tuple as an input, wrap it is an  array format, for instance to call `query.module.get((u32, u64))` where a `(u32, u64)` Tuple input is expected, you would do  `query.module.get([123, 456])`
+Tuples, as defined in the Polkadot/Substrate types appear as `(TypeA, TypeB)`. For instance we may have an `(AccountId, u64)` input as defined in the metadata or as part of the user types. To specify a Tuple as an input, wrap it in an  array format, for instance to call `query.module.get((u32, u64))` where a `(u32, u64)` Tuple input is expected, you would do  `query.module.get([123, 456])`
 
 ## How long do transactions live
 

docs/start/api.query.other.md

@@ -22,7 +22,7 @@ const [{ data: balanceNow }, { data: balancePrev }] = await Promise.all([
 console.log(`The delta was ${balanceNow.free.sub(balancePrev.free)}`);
 ```
 
-In the above example, we introduce the `.at(<hash>[, ...params])` query. For all `.at` queries, the first parameter is always the block hash at which we want to make the query, in our example we use both the last retrieved block and the parent thereof. The params are optional as per the type of query made, for instance to retrieve the timestamp for a previous block, it would be -
+In the above example, we introduce the `.at(<hash>[, ...params]): Type` query. For all `.at` queries, the first parameter is always the block hash at which we want to make the query, in our example we use both the last retrieved block and the parent thereof. The params are optional as per the type of query made, for instance to retrieve the timestamp for a previous block, it would be -
 
 ```js
 ...
@@ -35,6 +35,46 @@ The `.at` queries are all single-shot, i.e. there are no subscription option to
 
 An additional point to take care of (briefly mentioned above), is state pruning. By default a Polkadot/Substrate node will only keep state for the last 256 blocks, unless it is explicitly run in archive mode. This means that querying state further back than the pruning period will result in an error returned from the Node. (Generally most public RPC nodes only run with default settings, which includes aggressive state pruning)
 
+## State for a range of blocks
+
+In addition to the `.at` queries, you can also query state starting at a specific historic block and up to either a specified or the latest blocks. This is done via the `.range([from, to?], <...opt params>): [Hash, Type][]` query. As an example -
+
+```js
+...
+// Retrieve the current block header
+const lastHdr = await api.rpc.chain.getHeader();
+const startHdr = await api.rpc.chain.getBlockHash(lastHdr.number.unwrap().subn(500));
+
+// retrieve the range of changes
+const changes = await api.query.system.account.range([startHdr]);
+
+changes.forEach(([hash, value]) => {
+  console.log(hash.toHex(), value.toHuman());
+});
+```
+
+## Map keys & entries
+
+When working maps and double-maps, it is possible to retrieve a list of all the keys and entries for the map. For this we can use the `.entries(<args>): [StorageKey, Type][]` queries. For example we may want to know the current list of validator exposures at a current era in the staking module -
+
+```js
+...
+// Retrieve the active era
+const activeEra = await api.query.staking.activeEra();
+
+// retrieve all exposures int the active era
+const exposures = await api.query.staking.erasStakers.entries(activeEra.index);
+
+exposures.forEach(([key, exposure]) => {
+  console.log('key arguments:', key.args.map((k) => k.toHuman()));
+  console.log('     exposure:', exposure.toHuman());
+});
+```
+
+Here we are querying a double-map, so we supply 1 argument. No arguments on double-maps will be very costly, retrieving all the eras and associated entries. Additionally when `twox64_concat` & `blake2_concat` is used, the key `.args` will contain decoded values of the params, in this case it will contain the actual `AccountId` of the staker. (Since that was not supplied)
+
+In the same way as above we can simply do `.keys(activeEra.index): StorageKey[]` to retrieve all the keys here, including the individual keys args decoding, as available on maps with decodable hashing functions.
+
 ## State entries
 
 In addition to using `api.query` to make actual on-chain queries, it can also be used to retrieve some information on the state entries. For instance to retrieve both the hash and size of an existing entry, we can make the following calls -

docs/start/api.tx.subs.md

@@ -62,6 +62,23 @@ const unsub = await api.tx.balances
 
 Be aware that when a transaction status is `isFinalized`, it means it is included, but it may still have failed - for instance if you try to send a larger amount that you have free, the transaction is included in a block, however from a end-user perspective the transaction failed since the transfer did not occur. In these cases a `system.ExtrinsicFailed` event will be available in the events array.
 
+## Payment information
+
+The Polkadot/Substrate RPC endpoints exposes weight/payment information that takes an encoded extrinsic and calculates the on-chain weight fees for it. A wrapper for this is available on the tx itself, taking exactly the same parameters as you would pass to a normal `.signAndSend` operation, specifically `.paymentInfo(sender, <any options>)`. To expand on our previous example -
+
+```js
+// construct a transaction
+const transfer = api.tx.balances.transfer(BOB, 12345);
+
+// retrieve the payment info
+const { partialFee, weight } = await transfer.paymentInfo(alice);
+
+console.log(`transaction will have a weight of ${weight}, with ${partialFee.toHuman()} weight fees`);
+
+// send the tx
+transfer.signAndSend(alice, ({ events = [], status }) => { ... });
+```
+
 ## Complex transactions
 
 In many cases transactions can carry quite complex information, be it for passing objects or proposing changes. In the next section we will take a dive [into complex transactions, including those wrapped for sudo](api.tx.wrap.md).

docs/start/typescript.user.md

@@ -1,6 +1,6 @@
 # TypeScript user generated
 
-In the previous section we looked at the TypeScript definitions that are available and is generated from both the chain and definitions. Here we will expand upon the use of the infrastructure created to define types as part of the `@polkadot/types` library and see how to use them to generate your onw definitions and chain types.
+In the previous section we looked at the TypeScript definitions that are available and is generated from both the chain and definitions. Here we will expand upon the use of the infrastructure created to define types as part of the `@polkadot/types` library and see how to use them to generate your own definitions and chain types.
 
 ## Definitions
 

lerna.json

@@ -9,5 +9,5 @@
   "packages": [
     "packages/*"
   ],
-  "version": "1.6.2"
+  "version": "1.7.1"
 }

package.json

@@ -12,7 +12,9 @@
   "scripts": {
     "build": "yarn build:interfaces && polkadot-dev-build-ts && (cd packages/typegen && copyfiles scripts/* build)",
     "build:interfaces": "polkadot-types-internal-interfaces",
+    "build:release": "polkadot-ci-ghact-build",
     "docs": "polkadot-types-internal-metadata && polkadot-dev-build-docs",
+    "docs:release": "polkadot-ci-ghact-docs",
     "chain:info": "polkadot-types-chain-info",
     "lint": "polkadot-dev-run-lint",
     "clean": "polkadot-dev-clean-build",
@@ -22,12 +24,13 @@
     "test:watch": "polkadot-dev-run-test --watch"
   },
   "devDependencies": {
-    "@babel/core": "^7.8.4",
-    "@babel/register": "^7.8.3",
-    "@babel/runtime": "^7.8.4",
-    "@polkadot/dev": "^0.50.30",
-    "@polkadot/ts": "^0.3.7",
+    "@babel/core": "^7.8.7",
+    "@babel/register": "^7.8.6",
+    "@babel/runtime": "^7.8.7",
+    "@polkadot/dev": "^0.51.5",
+    "@polkadot/ts": "^0.3.13",
     "@polkadot/typegen": "workspace:packages/typegen",
+    "@types/jest": "^25.1.4",
     "copyfiles": "^2.2.0"
   }
 }

packages/api-contract/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/api-contract",
-  "version": "1.6.2",
+  "version": "1.7.1",
   "description": "Interfaces for interacting with contracts and contract ABIs",
   "main": "index.js",
   "keywords": [
@@ -26,10 +26,10 @@
   },
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/api-contract#readme",
   "dependencies": {
-    "@babel/runtime": "^7.8.4",
-    "@polkadot/api": "1.6.2",
-    "@polkadot/rpc-core": "1.6.2",
-    "@polkadot/types": "1.6.2",
+    "@babel/runtime": "^7.8.7",
+    "@polkadot/api": "1.7.1",
+    "@polkadot/rpc-core": "1.7.1",
+    "@polkadot/types": "1.7.1",
     "@polkadot/util": "^2.6.2",
     "bn.js": "^5.1.1",
     "rxjs": "^6.5.4"

packages/api-derive/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/api-derive",
-  "version": "1.6.2",
+  "version": "1.7.1",
   "description": "Common functions used across Polkadot, derived from RPC calls and storage queries.",
   "main": "index.js",
   "keywords": [
@@ -27,11 +27,11 @@
   },
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/api-derive#readme",
   "dependencies": {
-    "@babel/runtime": "^7.8.4",
-    "@polkadot/api": "1.6.2",
-    "@polkadot/rpc-core": "1.6.2",
-    "@polkadot/rpc-provider": "1.6.2",
-    "@polkadot/types": "1.6.2",
+    "@babel/runtime": "^7.8.7",
+    "@polkadot/api": "1.7.1",
+    "@polkadot/rpc-core": "1.7.1",
+    "@polkadot/rpc-provider": "1.7.1",
+    "@polkadot/types": "1.7.1",
     "@polkadot/util": "^2.6.2",
     "@polkadot/util-crypto": "^2.6.2",
     "bn.js": "^5.1.1",

packages/api-derive/src/accounts/idAndIndex.ts

@@ -7,7 +7,6 @@ import { AccountId, AccountIndex, Address } from '@polkadot/types/interfaces';
 import { Observable, of } from 'rxjs';
 import { map } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { createType } from '@polkadot/types';
 import { isU8a } from '@polkadot/util';
 import { decodeAddress } from '@polkadot/util-crypto';
 
@@ -23,14 +22,14 @@ function retrieve (api: ApiInterfaceRx, address: Address | AccountId | AccountIn
       : decodeAddress((address || '').toString());
 
     if (decoded.length === 32) {
-      const accountId = createType(api.registry, 'AccountId', decoded);
+      const accountId = api.registry.createType('AccountId', decoded);
 
       return api.derive.accounts.idToIndex(accountId).pipe(
         map((accountIndex): AccountIdAndIndex => [accountId, accountIndex])
       );
     }
 
-    const accountIndex = createType(api.registry, 'AccountIndex', decoded);
+    const accountIndex = api.registry.createType('AccountIndex', decoded);
 
     return api.derive.accounts.indexToId(accountIndex).pipe(
       map((accountId): AccountIdAndIndex => [accountId, accountIndex])

packages/api-derive/src/accounts/indexToId.ts

@@ -9,14 +9,14 @@ import { Observable, of } from 'rxjs';
 import { map, startWith } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
 import { ENUMSET_SIZE } from '@polkadot/types/generic/AccountIndex';
-import { Option, Vec, createType } from '@polkadot/types';
+import { Option, Vec } from '@polkadot/types';
 
 import { memo } from '../util';
 
 function queryEnumSet (api: ApiInterfaceRx, _accountIndex: AccountIndex | string): Observable<AccountId | undefined> {
   const accountIndex = _accountIndex instanceof api.registry.createClass('AccountIndex')
     ? _accountIndex
-    : createType(api.registry, 'AccountIndex', _accountIndex);
+    : api.registry.createType('AccountIndex', _accountIndex);
 
   return api.query.indices.enumSet<Vec<AccountId>>(accountIndex.div(ENUMSET_SIZE)).pipe(
     startWith([]),

packages/api-derive/src/accounts/indexes.ts

@@ -9,7 +9,7 @@ import { AccountIndexes } from '../types';
 import { Observable, of } from 'rxjs';
 import { map, startWith, switchMap } from 'rxjs/operators';
 import { ENUMSET_SIZE } from '@polkadot/types/generic/AccountIndex';
-import { Vec, createType } from '@polkadot/types';
+import { Vec } from '@polkadot/types';
 
 import { memo } from '../util';
 
@@ -31,7 +31,7 @@ function queryEnumSet (api: ApiInterfaceRx): Observable<AccountIndexes> {
           // 64 (0..63 in first) is [1][0] (the first index value in set 2)
           const index = (outerIndex * enumsetSize) + innerIndex;
 
-          indexes[accountId.toString()] = createType(api.registry, 'AccountIndex', index);
+          indexes[accountId.toString()] = api.registry.createType('AccountIndex', index);
         });
 
         return indexes;

packages/api-derive/src/balances/account.ts

@@ -6,10 +6,9 @@ import { AccountId, AccountData, AccountIndex, AccountInfo, Address, Balance, In
 import { ITuple } from '@polkadot/types/types';
 import { DerivedBalancesAccount } from '../types';
 
-import { combineLatest, of, Observable } from 'rxjs';
+import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { createType } from '@polkadot/types';
 
 import { memo } from '../util';
 
@@ -23,7 +22,7 @@ function calcBalances (api: ApiInterfaceRx, [accountId, [freeBalance, reservedBa
     frozenFee,
     frozenMisc,
     reservedBalance,
-    votingBalance: createType(api.registry, 'Balance', freeBalance.add(reservedBalance))
+    votingBalance: api.registry.createType('Balance', freeBalance.add(reservedBalance))
   };
 }
 
@@ -35,7 +34,7 @@ function queryBalancesFree (api: ApiInterfaceRx, accountId: AccountId): Observab
     [api.query.system.accountNonce, accountId]
   ]).pipe(
     map(([freeBalance, reservedBalance, accountNonce]): Result =>
-      [freeBalance, reservedBalance, createType(api.registry, 'Balance'), createType(api.registry, 'Balance'), accountNonce]
+      [freeBalance, reservedBalance, api.registry.createType('Balance'), api.registry.createType('Balance'), accountNonce]
     )
   );
 }
@@ -93,7 +92,7 @@ export function account (api: ApiInterfaceRx): (address: AccountIndex | AccountI
                 ? queryBalancesAccount(api, accountId)
                 : queryBalancesFree(api, accountId)
           ])
-          : of([createType(api.registry, 'AccountId'), [createType(api.registry, 'Balance'), createType(api.registry, 'Balance'), createType(api.registry, 'Balance'), createType(api.registry, 'Balance'), createType(api.registry, 'Index')]])
+          : of([api.registry.createType('AccountId'), [api.registry.createType('Balance'), api.registry.createType('Balance'), api.registry.createType('Balance'), api.registry.createType('Balance'), api.registry.createType('Index')]])
         )
       ),
       map((result): DerivedBalancesAccount => calcBalances(api, result))

packages/api-derive/src/balances/all.ts

@@ -6,10 +6,10 @@ import { AccountId, AccountIndex, Address, Balance, BalanceLock, BalanceLockTo21
 import { DerivedBalancesAccount, DerivedBalancesAll } from '../types';
 
 import BN from 'bn.js';
-import { combineLatest, of, Observable } from 'rxjs';
+import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { Option, Vec, createType } from '@polkadot/types';
+import { Option, Vec } from '@polkadot/types';
 import { bnMax } from '@polkadot/util';
 
 import { memo } from '../util';
@@ -18,7 +18,7 @@ type ResultBalance = [VestingInfo | null, (BalanceLock | BalanceLockTo212)[]];
 type Result = [DerivedBalancesAccount, BlockNumber, ResultBalance];
 
 function calcBalances (api: ApiInterfaceRx, [{ accountId, accountNonce, freeBalance, frozenFee, frozenMisc, reservedBalance, votingBalance }, bestNumber, [vesting, locks]]: Result): DerivedBalancesAll {
-  let lockedBalance = createType(api.registry, 'Balance');
+  let lockedBalance = api.registry.createType('Balance');
   let lockedBreakdown: (BalanceLock | BalanceLockTo212)[] = [];
   let allLocked = false;
 
@@ -32,16 +32,16 @@ function calcBalances (api: ApiInterfaceRx, [{ accountId, accountNonce, freeBala
 
     // get the maximum of the locks according to https://github.com/paritytech/substrate/blob/master/srml/balances/src/lib.rs#L699
     if (notAll.length) {
-      lockedBalance = createType(api.registry, 'Balance', bnMax(...notAll.map(({ amount }): Balance => amount)));
+      lockedBalance = api.registry.createType('Balance', bnMax(...notAll.map(({ amount }): Balance => amount)));
     }
   }
 
   // Calculate the vesting balances,
   //  - offset = balance locked at startingBlock
   //  - perBlock is the unlock amount
-  const { locked: vestingTotal, perBlock, startingBlock } = vesting || createType(api.registry, 'VestingInfo');
+  const { locked: vestingTotal, perBlock, startingBlock } = vesting || api.registry.createType('VestingInfo');
   const isStarted = bestNumber.gt(startingBlock);
-  const vestedBalance = createType(api.registry, 'Balance', isStarted ? perBlock.mul(bestNumber.sub(startingBlock)) : 0);
+  const vestedBalance = api.registry.createType('Balance', isStarted ? perBlock.mul(bestNumber.sub(startingBlock)) : 0);
   const isVesting = isStarted && vestedBalance.lt(vestingTotal);
 
   // The available balance & vested has an interplay here
@@ -53,7 +53,7 @@ function calcBalances (api: ApiInterfaceRx, [{ accountId, accountNonce, freeBala
   // ""
   const floating = freeBalance.sub(lockedBalance);
   const extraReceived = isVesting ? freeBalance.sub(vestingTotal) : new BN(0);
-  const availableBalance = createType(api.registry, 'Balance', allLocked ? 0 : bnMax(new BN(0), isVesting && floating.gt(vestedBalance) ? vestedBalance.add(extraReceived) : floating));
+  const availableBalance = api.registry.createType('Balance', allLocked ? 0 : bnMax(new BN(0), isVesting && floating.gt(vestedBalance) ? vestedBalance.add(extraReceived) : floating));
 
   return {
     accountId,
@@ -84,7 +84,7 @@ function queryOld (api: ApiInterfaceRx, accountId: AccountId): Observable<Result
       if (optVesting.isSome) {
         const { offset: locked, perBlock, startingBlock } = optVesting.unwrap();
 
-        vestingNew = createType(api.registry, 'VestingInfo', { locked, perBlock, startingBlock });
+        vestingNew = api.registry.createType('VestingInfo', { locked, perBlock, startingBlock });
       }
 
       return [vestingNew, locks];
@@ -102,7 +102,7 @@ function queryCurrent (api: ApiInterfaceRx, accountId: AccountId): Observable<Re
       ])
       : api.query.balances.locks(accountId).pipe(
         map((locks): [Vec<BalanceLock>, Option<VestingInfo>] =>
-          [locks, createType(api.registry, 'Option<VestingInfo>')]
+          [locks, api.registry.createType('Option<VestingInfo>')]
         )
       )
   ).pipe(
@@ -139,7 +139,7 @@ export function all (api: ApiInterfaceRx): (address: AccountIndex | AccountId |
               ? queryCurrent(api, account.accountId)
               : queryOld(api, account.accountId)
           ])
-          : of([account, createType(api.registry, 'BlockNumber'), [null, createType(api.registry, 'Vec<BalanceLock>')]])
+          : of([account, api.registry.createType('BlockNumber'), [null, api.registry.createType('Vec<BalanceLock>')]])
         )
       ),
       map((result): DerivedBalancesAll => calcBalances(api, result))

packages/api-derive/src/balances/fees.ts

@@ -8,7 +8,6 @@ import { DerivedFees } from '../types';
 
 import { Observable, of } from 'rxjs';
 import { map } from 'rxjs/operators';
-import { createType } from '@polkadot/types';
 
 import { memo } from '../util';
 
@@ -29,13 +28,13 @@ export function fees (api: ApiInterfaceRx): () => Observable<DerivedFees> {
   return memo((): Observable<DerivedFees> =>
     of([
       // deprecated - remove
-      (api.consts.balances?.creationFee as Balance) || createType(api.registry, 'Balance'),
-      (api.consts.balances?.transferFee as Balance) || createType(api.registry, 'Balance'),
+      (api.consts.balances?.creationFee as Balance) || api.registry.createType('Balance'),
+      (api.consts.balances?.transferFee as Balance) || api.registry.createType('Balance'),
 
       // current
-      (api.consts.balances?.existentialDeposit as Balance) || createType(api.registry, 'Balance'),
-      (api.consts.transactionPayment?.transactionBaseFee as Balance) || createType(api.registry, 'Balance'),
-      (api.consts.transactionPayment?.transactionByteFee as Balance) || createType(api.registry, 'Balance')
+      (api.consts.balances?.existentialDeposit as Balance) || api.registry.createType('Balance'),
+      (api.consts.transactionPayment?.transactionBaseFee as Balance) || api.registry.createType('Balance'),
+      (api.consts.transactionPayment?.transactionByteFee as Balance) || api.registry.createType('Balance')
     ]).pipe(
       map(([creationFee, transferFee, existentialDeposit, transactionBaseFee, transactionByteFee]): DerivedFees => ({
         creationFee,

packages/api-derive/src/balances/votingBalances.ts

@@ -5,7 +5,7 @@
 import { AccountId, AccountIndex, Address } from '@polkadot/types/interfaces';
 import { DerivedBalancesAccount } from '../types';
 
-import { combineLatest, Observable, of } from 'rxjs';
+import { Observable, combineLatest, of } from 'rxjs';
 import { ApiInterfaceRx } from '@polkadot/api/types';
 
 import { memo } from '../util';
@@ -16,7 +16,8 @@ export function votingBalances (api: ApiInterfaceRx): (addresses?: (AccountId |
       ? of([] as DerivedBalancesAccount[])
       : combineLatest(
         addresses.map((accountId): Observable<DerivedBalancesAccount> =>
-          api.derive.balances.account(accountId))
+          api.derive.balances.account(accountId)
+        )
       )
   );
 }

packages/api-derive/src/chain/bestNumberLag.ts

@@ -7,7 +7,6 @@ import { BlockNumber } from '@polkadot/types/interfaces';
 import { Observable, combineLatest } from 'rxjs';
 import { map } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { createType } from '@polkadot/types';
 
 import { memo } from '../util';
 
@@ -31,7 +30,7 @@ export function bestNumberLag (api: ApiInterfaceRx): () => Observable<BlockNumbe
       api.derive.chain.bestNumberFinalized()
     ]).pipe(
       map(([bestNumber, bestNumberFinalized]): BlockNumber =>
-        createType(api.registry, 'BlockNumber', bestNumber.sub(bestNumberFinalized))
+        api.registry.createType('BlockNumber', bestNumber.sub(bestNumberFinalized))
       )
     ));
 }

packages/api-derive/src/collective/proposals.ts

@@ -6,7 +6,7 @@ import { ApiInterfaceRx } from '@polkadot/api/types';
 import { Hash, Proposal, Votes } from '@polkadot/types/interfaces';
 import { DerivedCollectiveProposal } from '../types';
 
-import { combineLatest, Observable, of } from 'rxjs';
+import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { Option } from '@polkadot/types';
 

packages/api-derive/src/contracts/fees.ts

@@ -8,7 +8,6 @@ import { DerivedContractFees } from '../types';
 import BN from 'bn.js';
 import { Observable, of } from 'rxjs';
 import { map } from 'rxjs/operators';
-import { createType } from '@polkadot/types';
 
 import { memo } from '../util';
 
@@ -18,8 +17,8 @@ type ResultV2 = [BN, BN, BN, BN, BN, BN, BN, BN, BN];
 function queryConstants (api: ApiInterfaceRx): Observable<ResultV2> {
   return of([
     // deprecated
-    api.consts.contracts.creationFee || createType(api.registry, 'Balance'),
-    api.consts.contracts.transferFee || createType(api.registry, 'Balance'),
+    api.consts.contracts.creationFee || api.registry.createType('Balance'),
+    api.consts.contracts.transferFee || api.registry.createType('Balance'),
 
     // current
     api.consts.contracts.callBaseFee,

packages/api-derive/src/democracy/proposals.ts

@@ -9,7 +9,7 @@ import { DeriveProposal } from '../types';
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { Bytes, Option, Vec, createType } from '@polkadot/types';
+import { Bytes, Option, Vec } from '@polkadot/types';
 
 import { memo } from '../util';
 
@@ -37,7 +37,7 @@ function parse (api: ApiInterfaceRx, { depositors, proposals, preimages }: Resul
       // we could end up in a situation where the proposal is non-decodable, e.g. after an upgrade
       if (preimage) {
         try {
-          proposal = createType(api.registry, 'Proposal', preimage[0].toU8a(true));
+          proposal = api.registry.createType('Proposal', preimage[0].toU8a(true));
         } catch (error) {
           console.error(error);
         }

packages/api-derive/src/democracy/referendumInfo.ts

@@ -10,7 +10,7 @@ import BN from 'bn.js';
 import { Observable, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { Option, createType } from '@polkadot/types';
+import { Option } from '@polkadot/types';
 
 import { memo } from '../util';
 
@@ -25,11 +25,11 @@ function constructInfo (api: ApiInterfaceRx, index: BN | number, _info: Option<R
   }
 
   return {
-    index: createType(api.registry, 'PropIndex', index),
+    index: api.registry.createType('PropIndex', index),
     info,
     hash: info.proposalHash,
     proposal: preImage
-      ? createType(api.registry, 'Proposal', preImage[0].toU8a(true))
+      ? api.registry.createType('Proposal', preImage[0].toU8a(true))
       : undefined,
     preimage: preImage
       ? {

packages/api-derive/src/democracy/referendumInfos.ts

@@ -23,7 +23,7 @@ export function referendumInfos (api: ApiInterfaceRx): (ids?: (BN | number)[]) =
     ).pipe(
       switchMap((infos): Observable<(DerivedReferendum | null)[]> =>
         combineLatest(
-          ...ids.map((id, index): Observable<DerivedReferendum | null> =>
+          ids.map((id, index): Observable<DerivedReferendum | null> =>
             retrieveInfo(api, id, infos[index])
           )
         )

packages/api-derive/src/democracy/referendumVotesFor.ts

@@ -5,10 +5,10 @@
 import { AccountId, Vote } from '@polkadot/types/interfaces';
 
 import BN from 'bn.js';
-import { combineLatest, Observable, of } from 'rxjs';
+import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { Vec, createType } from '@polkadot/types';
+import { Vec } from '@polkadot/types';
 
 import { DerivedBalancesAccount, DerivedReferendumVote } from '../types';
 import { memo } from '../util';
@@ -26,8 +26,8 @@ export function referendumVotesFor (api: ApiInterfaceRx): (referendumId: BN | nu
       map(([votersFor, votes, balances]): DerivedReferendumVote[] =>
         votersFor.map((accountId, index): DerivedReferendumVote => ({
           accountId,
-          balance: balances[index].votingBalance || createType(api.registry, 'Balance'),
-          vote: votes[index] || createType(api.registry, 'Vote')
+          balance: balances[index].votingBalance || api.registry.createType('Balance'),
+          vote: votes[index] || api.registry.createType('Vote')
         } as DerivedReferendumVote))
       )
     ));

packages/api-derive/src/elections/info.ts

@@ -8,7 +8,7 @@ import { ITuple } from '@polkadot/types/types';
 import { Observable } from 'rxjs';
 import { map } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { createType, Vec, u32 } from '@polkadot/types';
+import { Vec, u32 } from '@polkadot/types';
 
 import { DerivedElectionsInfo } from '../types';
 import { memo } from '../util';
@@ -28,12 +28,12 @@ function queryElections (api: ApiInterfaceRx): Observable<DerivedElectionsInfo>
   ]).pipe(
     map(([councilMembers, candidates, members, runnersUp]): DerivedElectionsInfo => ({
       candidates,
-      candidateCount: createType(api.registry, 'u32', candidates.length),
+      candidateCount: api.registry.createType('u32', candidates.length),
       candidacyBond: api.consts[section].candidacyBond as Balance,
       desiredSeats: api.consts[section].desiredMembers as u32,
       members: members.length
         ? members.sort(sortAccounts)
-        : councilMembers.map((accountId): [AccountId, Balance] => [accountId, createType(api.registry, 'Balance')]),
+        : councilMembers.map((accountId): [AccountId, Balance] => [accountId, api.registry.createType('Balance')]),
       runnersUp: runnersUp.sort(sortAccounts),
       termDuration: api.consts[section].termDuration as BlockNumber,
       votingBond: api.consts[section].votingBond as Balance

packages/api-derive/src/imOnline/receivedHeartbeats.ts

@@ -5,7 +5,7 @@
 import { AccountId } from '@polkadot/types/interfaces';
 import { DerivedHeartbeats } from '../types';
 
-import { of, Observable, combineLatest } from 'rxjs';
+import { Observable, of, combineLatest } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
 import { Bytes, Option, u32 } from '@polkadot/types';

packages/api-derive/src/index.spec.ts

@@ -61,7 +61,7 @@ describe('derive', (): void => {
     testFunction(api)('session', 'sessionProgress', []);
 
     testFunction(api)('staking', 'account', []);
-    testFunction(api)('staking', 'controllers', []);
+    testFunction(api)('staking', 'stashes', []);
   });
 
   describe('custom', (): void => {

packages/api-derive/src/parachains/info.ts

@@ -9,7 +9,6 @@ import { Active, DidUpdate, Heads, ParaInfoResult, PendingSwap, RelayDispatchQue
 import { Observable, of } from 'rxjs';
 import { map } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { createType } from '@polkadot/types';
 
 import { memo } from '../util';
 
@@ -92,7 +91,7 @@ export function info (api: ApiInterfaceRx): (id: ParaId | number) => Observable<
       ])
         .pipe(
           map((result: Result): DeriveParachainFull | null =>
-            parse(createType(api.registry, 'ParaId', id), result)
+            parse(api.registry.createType('ParaId', id), result)
           )
         )
       : of(null)

packages/api-derive/src/session/indexes.ts

@@ -8,7 +8,7 @@ import { DeriveSessionIndexes } from '../types';
 
 import { Observable, of } from 'rxjs';
 import { map } from 'rxjs/operators';
-import { createType, Option, u32 } from '@polkadot/types';
+import { Option, u32 } from '@polkadot/types';
 
 import { memo } from '../util';
 
@@ -34,7 +34,7 @@ function queryNoActive (api: ApiInterfaceRx): Observable<Result> {
   ]).pipe(
     map(([currentEra, currentIndex, validatorCount]): Result => [
       currentEra,
-      createType(api.registry, 'Option<Moment>'),
+      api.registry.createType('Option<Moment>'),
       currentEra,
       currentIndex,
       validatorCount
@@ -56,7 +56,7 @@ function query (api: ApiInterfaceRx): Observable<Result> {
       return [
         activeEra,
         activeEraStart,
-        currentEra.unwrapOr(createType(api.registry, 'EraIndex')),
+        currentEra.unwrapOr(api.registry.createType('EraIndex')),
         currentIndex,
         validatorCount
       ];
@@ -67,11 +67,11 @@ function query (api: ApiInterfaceRx): Observable<Result> {
 // empty set when none is available
 function empty (api: ApiInterfaceRx): Observable<Result> {
   return of([
-    createType(api.registry, 'EraIndex'),
-    createType(api.registry, 'Option<Moment>'),
-    createType(api.registry, 'EraIndex'),
-    createType(api.registry, 'SessionIndex', 1),
-    createType(api.registry, 'u32')
+    api.registry.createType('EraIndex'),
+    api.registry.createType('Option<Moment>'),
+    api.registry.createType('EraIndex'),
+    api.registry.createType('SessionIndex', 1),
+    api.registry.createType('u32')
   ]);
 }
 

packages/api-derive/src/session/info.ts

@@ -8,7 +8,7 @@ import { DerivedSessionInfo, DeriveSessionIndexes } from '../types';
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { createType, Option, u64 } from '@polkadot/types';
+import { Option, u64 } from '@polkadot/types';
 
 import { memo } from '../util';
 
@@ -27,12 +27,12 @@ function createDerived (api: ApiInterfaceRx, [[hasBabe, epochDuration, sessionsP
     activeEraStart,
     currentEra,
     currentIndex,
-    eraLength: createType(api.registry, 'BlockNumber', sessionsPerEra.mul(epochDuration)),
-    eraProgress: createType(api.registry, 'BlockNumber', eraProgress),
+    eraLength: api.registry.createType('BlockNumber', sessionsPerEra.mul(epochDuration)),
+    eraProgress: api.registry.createType('BlockNumber', eraProgress),
     isEpoch: hasBabe,
     sessionLength: epochDuration,
     sessionsPerEra,
-    sessionProgress: createType(api.registry, 'BlockNumber', sessionProgress),
+    sessionProgress: api.registry.createType('BlockNumber', sessionProgress),
     validatorCount
   };
 }
@@ -43,15 +43,15 @@ function queryAura (api: ApiInterfaceRx): Observable<DerivedSessionInfo> {
       createDerived(api, [
         [
           false,
-          createType(api.registry, 'u64', 1),
-          api.consts.staking?.sessionsPerEra || createType(api.registry, 'SessionIndex', 1)
+          api.registry.createType('u64', 1),
+          api.consts.staking?.sessionsPerEra || api.registry.createType('SessionIndex', 1)
         ],
         indexes,
         [
-          createType(api.registry, 'u64', 1),
-          createType(api.registry, 'u64', 1),
-          createType(api.registry, 'u64', 1),
-          createType(api.registry, 'SessionIndex', 1)
+          api.registry.createType('u64', 1),
+          api.registry.createType('u64', 1),
+          api.registry.createType('u64', 1),
+          api.registry.createType('SessionIndex', 1)
         ]
       ])
     )
@@ -72,7 +72,7 @@ function queryBabe (api: ApiInterfaceRx): Observable<[DeriveSessionIndexes, Resu
       ])
     ),
     map(([indexes, [currentSlot, epochIndex, genesisSlot, optStartIndex]]): [DeriveSessionIndexes, ResultSlotsFlat] => [
-      indexes, [currentSlot, epochIndex, genesisSlot, optStartIndex.unwrapOr(createType(api.registry, 'SessionIndex', 1))]
+      indexes, [currentSlot, epochIndex, genesisSlot, optStartIndex.unwrapOr(api.registry.createType('SessionIndex', 1))]
     ])
   );
 }

packages/api-derive/src/society/candidates.ts

@@ -7,7 +7,7 @@ import { AccountId, BalanceOf, Bid, BidKind } from '@polkadot/types/interfaces';
 import { ITuple } from '@polkadot/types/types';
 import { DeriveSocietyCandidate } from '../types';
 
-import { combineLatest, of, Observable } from 'rxjs';
+import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { Option, Vec } from '@polkadot/types';
 

packages/api-derive/src/society/members.ts

@@ -6,7 +6,7 @@ import { ApiInterfaceRx } from '@polkadot/api/types';
 import { AccountId } from '@polkadot/types/interfaces';
 import { DeriveSocietyMember } from '../types';
 
-import { combineLatest, Observable } from 'rxjs';
+import { Observable, combineLatest } from 'rxjs';
 import { switchMap } from 'rxjs/operators';
 
 import { memo } from '../util';
@@ -18,9 +18,11 @@ export function members (api: ApiInterfaceRx): () => Observable<DeriveSocietyMem
   return memo((): Observable<DeriveSocietyMember[]> =>
     api.query.society.members<AccountId[]>().pipe(
       switchMap((members): Observable<DeriveSocietyMember[]> =>
-        combineLatest(members.map((accountId): Observable<DeriveSocietyMember> =>
-          api.derive.society.member(accountId)
-        ))
+        combineLatest(
+          members.map((accountId): Observable<DeriveSocietyMember> =>
+            api.derive.society.member(accountId)
+          )
+        )
       )
     )
   );

packages/api-derive/src/staking/account.ts

@@ -7,9 +7,8 @@ import { Balance, BlockNumber, StakingLedger, UnlockChunk } from '@polkadot/type
 import { DerivedSessionInfo, DerivedStakingAccount, DerivedStakingQuery, DerivedUnlocking } from '../types';
 
 import BN from 'bn.js';
-import { combineLatest, Observable } from 'rxjs';
+import { Observable, combineLatest } from 'rxjs';
 import { map } from 'rxjs/operators';
-import { createType } from '@polkadot/types';
 
 import { isUndefined } from '@polkadot/util';
 
@@ -33,7 +32,7 @@ function remainingBlocks (api: ApiInterfaceRx, era: BN, sessionInfo: DerivedSess
   const remaining = era.sub(sessionInfo.currentEra);
 
   // on the Rust side the current-era >= era-for-unlock (removal done on >)
-  return createType(api.registry, 'BlockNumber', remaining.gtn(0)
+  return api.registry.createType('BlockNumber', remaining.gtn(0)
     ? remaining
       .subn(1)
       .mul(sessionInfo.eraLength)
@@ -58,7 +57,7 @@ function calculateUnlocking (api: ApiInterfaceRx, stakingLedger: StakingLedger |
   // group the unlock chunks that have the same era and sum their values
   const groupedResult = groupByEra(unlockingChunks);
   const results = Object.entries(groupedResult).map(([eraString, value]): DerivedUnlocking => ({
-    value: createType(api.registry, 'Balance', value),
+    value: api.registry.createType('Balance', value),
     remainingBlocks: remainingBlocks(api, new BN(eraString), sessionInfo)
   }));
 
@@ -67,10 +66,10 @@ function calculateUnlocking (api: ApiInterfaceRx, stakingLedger: StakingLedger |
 
 function redeemableSum (api: ApiInterfaceRx, stakingLedger: StakingLedger | undefined, sessionInfo: DerivedSessionInfo): Balance {
   if (isUndefined(stakingLedger)) {
-    return createType(api.registry, 'Balance');
+    return api.registry.createType('Balance');
   }
 
-  return createType(api.registry, 'Balance', stakingLedger.unlocking.reduce((total, { era, value }): BN => {
+  return api.registry.createType('Balance', stakingLedger.unlocking.reduce((total, { era, value }): BN => {
     return remainingBlocks(api, era.unwrap(), sessionInfo).eqn(0)
       ? total.add(value.unwrap())
       : total;

packages/api-derive/src/staking/controllers.ts

@@ -1,33 +0,0 @@
-// Copyright 2017-2020 @polkadot/api-derive authors & contributors
-// This software may be modified and distributed under the terms
-// of the Apache-2.0 license. See the LICENSE file for details.
-
-import { ApiInterfaceRx } from '@polkadot/api/types';
-import { AccountId, ValidatorPrefs } from '@polkadot/types/interfaces';
-import { ITuple } from '@polkadot/types/types';
-
-import { Observable, combineLatest, of } from 'rxjs';
-import { switchMap } from 'rxjs/operators';
-import { Option, Vec } from '@polkadot/types';
-
-import { memo } from '../util';
-
-type DeriveControllers = [AccountId[], Option<AccountId>[]];
-
-/**
- * @description From the list of stash accounts, retrieve the list of controllers
- */
-export function controllers (api: ApiInterfaceRx): () => Observable<DeriveControllers> {
-  return memo((): Observable<[AccountId[], Option<AccountId>[]]> =>
-    api.query.staking.validators<ITuple<[Vec<AccountId>, Vec<ValidatorPrefs>]>>().pipe(
-      switchMap(([stashIds]): Observable<DeriveControllers> =>
-        combineLatest([
-          of(stashIds),
-          // for V2, don't return all the controllers, we call bonded at a later point
-          api.consts.session
-            ? of([])
-            : api.query.staking.bonded.multi<Option<AccountId>>(stashIds)
-        ])
-      )
-    ));
-}

packages/api-derive/src/staking/electedInfo.ts

@@ -17,9 +17,11 @@ export function electedInfo (api: ApiInterfaceRx): () => Observable<DerivedStaki
       switchMap(({ nextElected }): Observable<[AccountId[], DerivedStakingQuery[]]> =>
         combineLatest([
           of(nextElected),
-          combineLatest(nextElected.map((accountId): Observable<DerivedStakingQuery> =>
-            api.derive.staking.query(accountId)
-          ))
+          combineLatest(
+            nextElected.map((accountId): Observable<DerivedStakingQuery> =>
+              api.derive.staking.query(accountId)
+            )
+          )
         ])
       ),
       map(([nextElected, info]): DerivedStakingElected => ({

packages/api-derive/src/staking/erasExposure.ts

@@ -8,12 +8,14 @@ import { DeriveEraExposure, DeriveEraNominatorExposure, DeriveEraValidatorExposu
 
 import BN from 'bn.js';
 import { Observable, combineLatest, of } from 'rxjs';
-import { map, switchMap, take } from 'rxjs/operators';
+import { map, switchMap } from 'rxjs/operators';
 import { StorageKey } from '@polkadot/types';
 
 import { memo } from '../util';
 
-function mapStakers (era: EraIndex, stakers: [StorageKey, Exposure][]): DeriveEraExposure {
+type KeysAndExposures = [StorageKey, Exposure][];
+
+function mapStakers (era: EraIndex, stakers: KeysAndExposures): DeriveEraExposure {
   const nominators: DeriveEraNominatorExposure = {};
   const validators: DeriveEraValidatorExposure = {};
 
@@ -33,27 +35,23 @@ function mapStakers (era: EraIndex, stakers: [StorageKey, Exposure][]): DeriveEr
   return { era, nominators, validators };
 }
 
+export function eraExposure (api: ApiInterfaceRx): (era: EraIndex) => Observable<DeriveEraExposure> {
+  return memo((era: EraIndex): Observable<DeriveEraExposure> =>
+    api.query.staking.erasStakersClipped.entries(era).pipe(
+      map((stakers) => mapStakers(era, stakers))
+    )
+  );
+}
+
 export function erasExposure (api: ApiInterfaceRx): (withActive?: boolean | BN | number) => Observable<DeriveEraExposure[]> {
   return memo((withActive?: boolean | BN | number): Observable<DeriveEraExposure[]> =>
     api.derive.staking.erasHistoric(withActive).pipe(
-      switchMap((eras): Observable<[EraIndex[], [StorageKey, Exposure][][]]> =>
-        combineLatest([
-          of(eras),
-          eras.length
-            ? combineLatest(
-              // we could just do entries over the full set, however the set can be quite large - split it into
-              // batches - may need to re-visit this, or alternatively use pages keys for exceptionally large sets
-              eras.map((era): Observable<[StorageKey, Exposure][]> =>
-                api.query.staking.erasStakersClipped.entries(era).pipe(take(1))
-              )
-            )
-            : of([])
-        ])
-      ),
-      map(([eras, erasStakers]): DeriveEraExposure[] =>
-        eras.map((era, index): DeriveEraExposure =>
-          mapStakers(era, erasStakers[index])
-        )
+      switchMap((eras): Observable<DeriveEraExposure[]> =>
+        eras.length
+          ? combineLatest(
+            eras.map((era) => api.derive.staking.eraExposure(era))
+          )
+          : of([])
       )
     )
   );

packages/api-derive/src/staking/erasPrefs.ts

@@ -0,0 +1,46 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { EraIndex, ValidatorPrefs } from '@polkadot/types/interfaces';
+import { DeriveEraPrefs, DeriveEraValPrefs } from '../types';
+
+import BN from 'bn.js';
+import { Observable, combineLatest, of } from 'rxjs';
+import { map, switchMap } from 'rxjs/operators';
+import { StorageKey } from '@polkadot/types';
+
+import { memo } from '../util';
+
+function mapPrefs (era: EraIndex, all: [StorageKey, ValidatorPrefs][]): DeriveEraPrefs {
+  const validators: DeriveEraValPrefs = {};
+
+  all.forEach(([key, prefs]): void => {
+    validators[key.args[1].toString()] = prefs;
+  });
+
+  return { era, validators };
+}
+
+export function eraPrefs (api: ApiInterfaceRx): (era: EraIndex) => Observable<DeriveEraPrefs> {
+  return memo((era: EraIndex): Observable<DeriveEraPrefs> =>
+    api.query.staking.erasValidatorPrefs.entries(era).pipe(
+      map((prefs) => mapPrefs(era, prefs))
+    )
+  );
+}
+
+export function erasPrefs (api: ApiInterfaceRx): (withActive?: boolean | BN | number) => Observable<DeriveEraPrefs[]> {
+  return memo((withActive?: boolean | BN | number): Observable<DeriveEraPrefs[]> =>
+    api.derive.staking.erasHistoric(withActive).pipe(
+      switchMap((eras): Observable<DeriveEraPrefs[]> =>
+        eras.length
+          ? combineLatest(
+            eras.map((era) => api.derive.staking.eraPrefs(era))
+          )
+          : of([])
+      )
+    )
+  );
+}

packages/api-derive/src/staking/erasSlashes.ts

@@ -0,0 +1,55 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { BalanceOf, EraIndex, Perbill } from '@polkadot/types/interfaces';
+import { ITuple } from '@polkadot/types/types';
+import { DeriveEraSlashes, DeriveEraValSlash } from '../types';
+
+import BN from 'bn.js';
+import { Observable, combineLatest, of } from 'rxjs';
+import { map, switchMap } from 'rxjs/operators';
+import { Option, StorageKey } from '@polkadot/types';
+
+import { memo } from '../util';
+
+function mapSlashes (era: EraIndex, noms: [StorageKey, Option<BalanceOf>][], vals: [StorageKey, Option<ITuple<[Perbill, BalanceOf]>>][]): DeriveEraSlashes {
+  const nominators: DeriveEraValSlash = {};
+  const validators: DeriveEraValSlash = {};
+
+  noms.forEach(([key, optBalance]): void => {
+    nominators[key.args[1].toString()] = optBalance.unwrap();
+  });
+
+  vals.forEach(([key, optRes]): void => {
+    validators[key.args[1].toString()] = optRes.unwrapOrDefault()[1];
+  });
+
+  return { era, nominators, validators };
+}
+
+export function eraSlashes (api: ApiInterfaceRx): (era: EraIndex) => Observable<DeriveEraSlashes> {
+  return memo((era: EraIndex): Observable<DeriveEraSlashes> =>
+    combineLatest([
+      api.query.staking.nominatorSlashInEra.entries(era),
+      api.query.staking.validatorSlashInEra.entries(era)
+    ]).pipe(
+      map(([noms, vals]) => mapSlashes(era, noms, vals))
+    )
+  );
+}
+
+export function erasSlashes (api: ApiInterfaceRx): (withActive?: boolean | BN | number) => Observable<DeriveEraSlashes[]> {
+  return memo((withActive?: boolean | BN | number): Observable<DeriveEraSlashes[]> =>
+    api.derive.staking.erasHistoric(withActive).pipe(
+      switchMap((eras): Observable<DeriveEraSlashes[]> =>
+        eras.length
+          ? combineLatest(
+            eras.map((era) => api.derive.staking.eraSlashes(era))
+          )
+          : of([])
+      )
+    )
+  );
+}

packages/api-derive/src/staking/index.ts

@@ -6,12 +6,17 @@ export * from './account';
 export * from './erasExposure';
 export * from './erasHistoric';
 export * from './erasPoints';
+export * from './erasPrefs';
 export * from './erasRewards';
-export * from './controllers';
+export * from './erasSlashes';
 export * from './electedInfo';
 export * from './overview';
+export * from './ownExposure';
+export * from './ownSlashes';
 export * from './query';
 export * from './stakerExposure';
 export * from './stakerPoints';
 export * from './stakerRewards';
+export * from './stakerSlashes';
+export * from './stashes';
 export * from './validators';

packages/api-derive/src/staking/overview.ts

@@ -2,24 +2,23 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
-import { AccountId, EraIndex, EraPoints, EraRewardPoints, RewardPoint } from '@polkadot/types/interfaces';
+import { AccountId, EraPoints, EraRewardPoints, RewardPoint } from '@polkadot/types/interfaces';
 import { ApiInterfaceRx } from '@polkadot/api/types';
 import { DerivedStakingOverview } from '../types';
 
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
-import { createType } from '@polkadot/types';
 
 import { memo } from '../util';
 
-function retrievePointsPrev (api: ApiInterfaceRx, activeEra: EraIndex, currentElected: AccountId[]): Observable<EraRewardPoints> {
-  return api.query.staking.currentEraPointsEarned<EraPoints>(activeEra).pipe(
+function retrievePointsPrev (api: ApiInterfaceRx, currentElected: AccountId[]): Observable<EraRewardPoints> {
+  return api.query.staking.currentEraPointsEarned<EraPoints>().pipe(
     map(({ individual, total }): EraRewardPoints =>
-      createType(api.registry, 'EraRewardPoints', {
+      api.registry.createType('EraRewardPoints', {
         total,
         individual: new Map<AccountId, RewardPoint>(
           individual
-            .map((points): RewardPoint => createType(api.registry, 'RewardPoint', points))
+            .map((points): RewardPoint => api.registry.createType('RewardPoint', points))
             .map((points, index): [AccountId, RewardPoint] => [currentElected[index], points])
         )
       })
@@ -42,8 +41,8 @@ export function overview (api: ApiInterfaceRx): () => Observable<DerivedStakingO
           api.query.staking.erasRewardPoints
             ? api.query.staking.erasRewardPoints<EraRewardPoints>(indexes.activeEra)
             : api.query.staking.currentEraPointsEarned
-              ? retrievePointsPrev(api, indexes.activeEra, nextElected)
-              : of(createType(api.registry, 'EraRewardPoints'))
+              ? retrievePointsPrev(api, nextElected)
+              : of(api.registry.createType('EraRewardPoints'))
         ])
       ),
       map(([info, eraPoints]): DerivedStakingOverview => ({

packages/api-derive/src/staking/ownExposure.ts

@@ -0,0 +1,37 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { DeriveOwnExposure } from '../types';
+
+import BN from 'bn.js';
+import { Observable, combineLatest, of } from 'rxjs';
+import { map, switchMap } from 'rxjs/operators';
+
+import { memo } from '../util';
+
+export function ownExposure (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean | BN | number) => Observable<DeriveOwnExposure[]> {
+  return memo((accountId: Uint8Array | string, withActive?: boolean | BN | number): Observable<DeriveOwnExposure[]> => {
+    return api.derive.staking.erasHistoric(withActive).pipe(
+      switchMap((eras) =>
+        eras.length
+          ? combineLatest(
+            eras.map((era) => combineLatest([
+              of(era),
+              api.query.staking.erasStakersClipped(era, accountId),
+              api.query.staking.erasStakers(era, accountId)
+            ]))
+          )
+          : of([])
+      ),
+      map((result): DeriveOwnExposure[] =>
+        result.map(([era, clipped, exposure]): DeriveOwnExposure => ({
+          clipped,
+          era,
+          exposure
+        }))
+      )
+    );
+  });
+}

packages/api-derive/src/staking/ownSlashes.ts

@@ -0,0 +1,40 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { DeriveStakerSlashes } from '../types';
+
+import BN from 'bn.js';
+import { Observable, combineLatest, of } from 'rxjs';
+import { map, switchMap } from 'rxjs/operators';
+
+import { memo } from '../util';
+
+export function ownSlashes (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean | BN | number) => Observable<DeriveStakerSlashes[]> {
+  return memo((accountId: Uint8Array | string, withActive?: boolean | BN | number): Observable<DeriveStakerSlashes[]> => {
+    return api.derive.staking.erasHistoric(withActive).pipe(
+      switchMap((eras) =>
+        eras.length
+          ? combineLatest(
+            eras.map((era) => combineLatest([
+              of(era),
+              api.query.staking.nominatorSlashInEra(era, accountId),
+              api.query.staking.validatorSlashInEra(era, accountId)
+            ]))
+          )
+          : of([])
+      ),
+      map((result): DeriveStakerSlashes[] =>
+        result.map(([era, optNom, optVal]): DeriveStakerSlashes => ({
+          era,
+          total: optVal.isSome
+            ? optVal.unwrap()[1]
+            : optNom.isSome
+              ? optNom.unwrap()
+              : api.registry.createType('Balance')
+        }))
+      )
+    );
+  });
+}

packages/api-derive/src/staking/query.ts

@@ -7,9 +7,9 @@ import { AccountId, Exposure, Keys, Nominations, RewardDestination, ValidatorPre
 import { ITuple } from '@polkadot/types/types';
 import { DerivedStakingQuery } from '../types';
 
-import { combineLatest, Observable, of } from 'rxjs';
+import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
-import { createType, Option, Vec } from '@polkadot/types';
+import { Option, Vec } from '@polkadot/types';
 
 import { memo } from '../util';
 
@@ -80,7 +80,7 @@ function retrieveController (api: ApiInterfaceRx, stashId: AccountId, [queuedKey
  */
 export function query (api: ApiInterfaceRx): (accountId: Uint8Array | string) => Observable<DerivedStakingQuery> {
   return memo((accountId: Uint8Array | string): Observable<DerivedStakingQuery> => {
-    const stashId = createType(api.registry, 'AccountId', accountId);
+    const stashId = api.registry.createType('AccountId', accountId);
 
     return combineLatest([
       api.query.session.queuedKeys<Vec<ITuple<[AccountId, Keys]>>>(),

packages/api-derive/src/staking/stakerRewards.ts

@@ -4,7 +4,7 @@
 
 import { ApiInterfaceRx } from '@polkadot/api/types';
 import { Balance } from '@polkadot/types/interfaces';
-import { DeriveEraPoints, DeriveEraRewards, DeriveStakerExposure, DeriveStakerReward } from '../types';
+import { DeriveEraPoints, DeriveEraPrefs, DeriveEraRewards, DeriveEraValPrefs, DeriveStakerExposure, DeriveStakerReward } from '../types';
 
 import BN from 'bn.js';
 import { Observable, combineLatest } from 'rxjs';
@@ -12,18 +12,24 @@ import { map } from 'rxjs/operators';
 
 import { memo } from '../util';
 
-type Result = [DeriveEraPoints[], DeriveEraRewards[], DeriveStakerExposure[]];
-const ZERO = new BN(0);
+type Result = [DeriveEraPoints[], DeriveEraPrefs[], DeriveEraRewards[], DeriveStakerExposure[]];
 
-function parseRewards (api: ApiInterfaceRx, stakerId: string, [erasPoints, erasRewards, exposures]: Result): DeriveStakerReward[] {
+const ZERO = new BN(0);
+const COMM_DIV = new BN(1_000_000_000);
+
+function parseRewards (api: ApiInterfaceRx, stakerId: string, [erasPoints, erasPrefs, erasRewards, exposures]: Result): DeriveStakerReward[] {
   return exposures.map(({ era, isEmpty, isValidator, nominating, validators: eraValidators }): DeriveStakerReward => {
     const { eraPoints, validators: allValPoints } = erasPoints.find((p) => p.era.eq(era)) || { eraPoints: new BN(0), validators: {} };
-    const { eraReward } = erasRewards.find((r) => r.era.eq(era)) || { eraReward: new BN(0) };
+    const { eraReward } = erasRewards.find((r) => r.era.eq(era)) || { eraReward: ZERO };
+    const { validators: allValPrefs } = erasPrefs.find((p) => p.era.eq(era)) || { validators: {} as DeriveEraValPrefs };
     const validators: Record<string, Balance> = {};
     let total = ZERO;
 
     Object.entries(eraValidators).forEach(([validatorId, exposure]): void => {
-      const valPoints = allValPoints[validatorId] || new BN(0);
+      const valPoints = allValPoints[validatorId] || ZERO;
+      const valComm = allValPrefs[validatorId]?.commission.unwrap() || ZERO;
+      const avail = eraReward.mul(valPoints).div(eraPoints);
+      const valCut = valComm.mul(avail).div(COMM_DIV);
       const expTotal = exposure.total.unwrap();
       let value: BN | undefined;
 
@@ -40,7 +46,7 @@ function parseRewards (api: ApiInterfaceRx, stakerId: string, [erasPoints, erasR
             : ZERO;
         }
 
-        value = eraReward.mul(valPoints).mul(staked).div(eraPoints).div(expTotal);
+        value = avail.sub(valCut).mul(staked).div(expTotal).add(validatorId === stakerId ? valCut : ZERO);
         total = total.add(value);
       }
 
@@ -64,6 +70,7 @@ export function stakerRewards (api: ApiInterfaceRx): (accountId: Uint8Array | st
 
     return combineLatest([
       api.derive.staking.erasPoints(startEra),
+      api.derive.staking.erasPrefs(startEra),
       api.derive.staking.erasRewards(startEra),
       api.derive.staking.stakerExposure(stakerId, startEra)
     ]).pipe(

packages/api-derive/src/staking/stakerSlashes.ts

@@ -0,0 +1,27 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { DeriveStakerSlashes } from '../types';
+
+import BN from 'bn.js';
+import { Observable } from 'rxjs';
+import { map } from 'rxjs/operators';
+
+import { memo } from '../util';
+
+export function stakerSlashes (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean | BN | number) => Observable<DeriveStakerSlashes[]> {
+  return memo((accountId: Uint8Array | string, withActive?: boolean | BN | number): Observable<DeriveStakerSlashes[]> => {
+    const stakerId = api.registry.createType('AccountId', accountId).toString();
+
+    return api.derive.staking.erasSlashes(withActive).pipe(
+      map((slashes): DeriveStakerSlashes[] =>
+        slashes.map(({ era, nominators, validators }): DeriveStakerSlashes => ({
+          era,
+          total: nominators[stakerId] || validators[stakerId] || api.registry.createType('Balance')
+        }))
+      )
+    );
+  });
+}

packages/api-derive/src/staking/stashes.ts

@@ -0,0 +1,28 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { AccountId, ValidatorPrefs } from '@polkadot/types/interfaces';
+import { ITuple } from '@polkadot/types/types';
+
+import { Observable } from 'rxjs';
+import { map } from 'rxjs/operators';
+import { Vec } from '@polkadot/types';
+
+import { memo } from '../util';
+
+/**
+ * @description Retrieve the list of all validator stashes
+ */
+export function stashes (api: ApiInterfaceRx): () => Observable<AccountId[]> {
+  return memo((): Observable<AccountId[]> =>
+    api.query.staking.validators.creator.meta.type.asMap.linked.isTrue
+      ? api.query.staking.validators<ITuple<[Vec<AccountId>, Vec<ValidatorPrefs>]>>().pipe(
+        map(([stashIds]) => stashIds)
+      )
+      : api.query.staking.validators.keys().pipe(
+        map((keys) => keys.map((key) => key.args[0] as AccountId))
+      )
+  );
+}

packages/api-derive/src/staking/types.ts

@@ -7,17 +7,32 @@ import { DeriveSessionIndexes } from '../session/types';
 
 export type DeriveEraValPoints = Record<string, RewardPoint>;
 
+export type DeriveEraValPrefs = Record<string, ValidatorPrefs>;
+
+export type DeriveEraValSlash = Record<string, Balance>;
+
 export interface DeriveEraPoints {
   era: EraIndex;
   eraPoints: RewardPoint;
   validators: DeriveEraValPoints;
 }
 
+export interface DeriveEraPrefs {
+  era: EraIndex;
+  validators: DeriveEraValPrefs;
+}
+
 export interface DeriveEraRewards {
   era: EraIndex;
   eraReward: Balance;
 }
 
+export interface DeriveEraSlashes {
+  era: EraIndex;
+  nominators: DeriveEraValSlash;
+  validators: DeriveEraValSlash;
+}
+
 export interface DeriveStakerPoints {
   era: EraIndex;
   eraPoints: RewardPoint;
@@ -34,6 +49,14 @@ export interface DeriveEraExposure {
   validators: DeriveEraValidatorExposure;
 }
 
+export interface DeriveOwnExposure {
+  clipped: Exposure;
+  era: EraIndex;
+  exposure: Exposure;
+}
+
+export type DeriveOwnSlashes = DeriveStakerSlashes;
+
 export interface DeriveStakerExposure {
   era: EraIndex;
   isEmpty: boolean;
@@ -51,6 +74,11 @@ export interface DeriveStakerReward {
   total: Balance;
 }
 
+export interface DeriveStakerSlashes {
+  era: EraIndex;
+  total: Balance;
+}
+
 export interface DerivedStakingElected {
   nextElected: AccountId[];
   info: DerivedStakingQuery[];

packages/api-derive/src/staking/validators.ts

@@ -7,21 +7,25 @@ import { AccountId } from '@polkadot/types/interfaces';
 import { DeriveStakingValidators } from '../types';
 
 import { Observable, combineLatest, of } from 'rxjs';
-import { map, switchMap, take } from 'rxjs/operators';
+import { map, switchMap } from 'rxjs/operators';
 import { StorageKey } from '@polkadot/types';
 
 import { memo } from '../util';
 
-function queryNextStakers (api: ApiInterfaceRx): Observable<AccountId[]> {
-  // only populate for next era in the last session, so track both here - entries are not
-  // subscriptions, so we need a trigger - currentIndex acts as that trigger to refresh
-  return api.derive.session.indexes().pipe(
-    switchMap(({ activeEra }): Observable<StorageKey[]> =>
-      api.query.staking.erasStakers.keys(activeEra.addn(1)).pipe(take(1))
-    ),
-    map((keys): AccountId[] =>
-      keys.map((key): AccountId => key.args[1] as AccountId)
-    )
+export function nextElected (api: ApiInterfaceRx): () => Observable<AccountId[]> {
+  return memo((): Observable<AccountId[]> =>
+    api.query.staking.erasStakers
+      ? api.derive.session.indexes().pipe(
+        // only populate for next era in the last session, so track both here - entries are not
+        // subscriptions, so we need a trigger - currentIndex acts as that trigger to refresh
+        switchMap(({ currentEra }): Observable<StorageKey[]> =>
+          api.query.staking.erasStakers.keys(currentEra)
+        ),
+        map((keys): AccountId[] =>
+          keys.map((key): AccountId => key.args[1] as AccountId)
+        )
+      )
+      : api.query.staking.currentElected<AccountId[]>()
   );
 }
 
@@ -38,9 +42,7 @@ export function validators (api: ApiInterfaceRx): () => Observable<DeriveStaking
         ? api.query.session.validators()
         : of([]),
       api.query.staking
-        ? api.query.staking.erasStakers
-          ? queryNextStakers(api)
-          : api.query.staking.currentElected<AccountId[]>()
+        ? api.derive.staking.nextElected()
         : of([])
     ]).pipe(
       map(([validators, nextElected]): DeriveStakingValidators => ({

packages/api-derive/src/treasury/proposals.ts

@@ -8,7 +8,7 @@ import { DerivedCollectiveProposals, DerivedTreasuryProposal, DerivedTreasuryPro
 
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
-import { createType, Option } from '@polkadot/types';
+import { Option } from '@polkadot/types';
 
 import { memo } from '../util';
 
@@ -55,7 +55,7 @@ function retrieveProposals (api: ApiInterfaceRx, proposalCount: ProposalIndex, a
     const isApproval = approvalIds.some((id): boolean => id.eqn(index));
 
     if (!isApproval) {
-      proposalIds.push(createType(api.registry, 'ProposalIndex', index));
+      proposalIds.push(api.registry.createType('ProposalIndex', index));
     }
   }
 
@@ -87,7 +87,7 @@ export function proposals (api: ApiInterfaceRx): () => Observable<DerivedTreasur
       )
       : of({
         approvals: [],
-        proposalCount: createType(api.registry, 'ProposalIndex'),
+        proposalCount: api.registry.createType('ProposalIndex'),
         proposals: []
       } as DerivedTreasuryProposals)
   );

packages/api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/api",
-  "version": "1.6.2",
+  "version": "1.7.1",
   "description": "Promise and RxJS wrappers around the Polkadot JS RPC",
   "main": "index.js",
   "keywords": [
@@ -26,13 +26,13 @@
   },
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/api#readme",
   "dependencies": {
-    "@babel/runtime": "^7.8.4",
-    "@polkadot/api-derive": "1.6.2",
+    "@babel/runtime": "^7.8.7",
+    "@polkadot/api-derive": "1.7.1",
     "@polkadot/keyring": "^2.6.2",
-    "@polkadot/metadata": "1.6.2",
-    "@polkadot/rpc-core": "1.6.2",
-    "@polkadot/rpc-provider": "1.6.2",
-    "@polkadot/types": "1.6.2",
+    "@polkadot/metadata": "1.7.1",
+    "@polkadot/rpc-core": "1.7.1",
+    "@polkadot/rpc-provider": "1.7.1",
+    "@polkadot/types": "1.7.1",
     "@polkadot/util": "^2.6.2",
     "@polkadot/util-crypto": "^2.6.2",
     "bn.js": "^5.1.1",

packages/api/src/augment/consts.ts

@@ -2,7 +2,7 @@
 /* eslint-disable @typescript-eslint/no-empty-interface */
 
 import { Codec } from '@polkadot/types/types';
-import { Bytes, u32, u64 } from '@polkadot/types/primitive';
+import { u32, u64 } from '@polkadot/types/primitive';
 import { Gas } from '@polkadot/types/interfaces/contracts';
 import { Balance, BalanceOf, BlockNumber, Moment, Percent, Permill } from '@polkadot/types/interfaces/runtime';
 import { SessionIndex } from '@polkadot/types/interfaces/session';
@@ -27,16 +27,6 @@ declare module '@polkadot/metadata/Decorated/consts/types' {
        **/
       expectedBlockTime: AugmentedConst<Moment>;
     };
-    timestamp: {
-      [index: string]: AugmentedConst<object & Codec>;
-      /**
-       * The minimum period between blocks. Beware that this is different to the *expected* period
-       * that the block production apparatus provides. Your chosen consensus system will generally
-       * work with this to determine a sensible block time. e.g. For Aura, it will be double this
-       * period on default settings.
-       **/
-      minimumPeriod: AugmentedConst<Moment>;
-    };
     balances: {
       [index: string]: AugmentedConst<object & Codec>;
       /**
@@ -44,8 +34,70 @@ declare module '@polkadot/metadata/Decorated/consts/types' {
        **/
       existentialDeposit: AugmentedConst<Balance>;
     };
-    transactionPayment: {
+    contracts: {
       [index: string]: AugmentedConst<object & Codec>;
+      /**
+       * The maximum amount of gas that could be expended per block. A reasonable
+       * default value is 10_000_000.
+       **/
+      blockGasLimit: AugmentedConst<Gas>;
+      /**
+       * The base fee charged for calling into a contract. A reasonable default
+       * value is 135.
+       **/
+      callBaseFee: AugmentedConst<Gas>;
+      /**
+       * The fee required to instantiate a contract instance. A reasonable default value
+       * is 21.
+       **/
+      contractFee: AugmentedConst<BalanceOf>;
+      /**
+       * The base fee charged for instantiating a contract. A reasonable default value
+       * is 175.
+       **/
+      instantiateBaseFee: AugmentedConst<Gas>;
+      /**
+       * The maximum nesting level of a call/instantiate stack. A reasonable default
+       * value is 100.
+       **/
+      maxDepth: AugmentedConst<u32>;
+      /**
+       * The maximum size of a storage value in bytes. A reasonable default is 16 KiB.
+       **/
+      maxValueSize: AugmentedConst<u32>;
+      /**
+       * Price of a byte of storage per one block interval. Should be greater than 0.
+       **/
+      rentByteFee: AugmentedConst<BalanceOf>;
+      /**
+       * The amount of funds a contract should deposit in order to offset
+       * the cost of one byte.
+       * Let's suppose the deposit is 1,000 BU (balance units)/byte and the rent is 1 BU/byte/day,
+       * then a contract with 1,000,000 BU that uses 1,000 bytes of storage would pay no rent.
+       * But if the balance reduced to 500,000 BU and the storage stayed the same at 1,000,
+       * then it would pay 500 BU/day.
+       **/
+      rentDepositOffset: AugmentedConst<BalanceOf>;
+      /**
+       * Number of block delay an extrinsic claim surcharge has.
+       * When claim surcharge is called by an extrinsic the rent is checked
+       * for current_block - delay
+       **/
+      signedClaimHandicap: AugmentedConst<BlockNumber>;
+      /**
+       * Size of a contract at the time of instantiation. This is a simple way to ensure that
+       * empty contracts eventually gets deleted.
+       **/
+      storageSizeOffset: AugmentedConst<u32>;
+      /**
+       * Reward that is received by the party whose touch has led
+       * to removal of a contract.
+       **/
+      surchargeReward: AugmentedConst<BalanceOf>;
+      /**
+       * The minimum amount required to generate a tombstone.
+       **/
+      tombstoneDeposit: AugmentedConst<BalanceOf>;
       /**
        * The fee to be paid for making a transaction; the base.
        **/
@@ -55,27 +107,16 @@ declare module '@polkadot/metadata/Decorated/consts/types' {
        **/
       transactionByteFee: AugmentedConst<BalanceOf>;
     };
-    staking: {
-      [index: string]: AugmentedConst<object & Codec>;
-      /**
-       * Number of sessions per era.
-       **/
-      sessionsPerEra: AugmentedConst<SessionIndex>;
-      /**
-       * Number of eras that staked funds must remain bonded for.
-       **/
-      bondingDuration: AugmentedConst<EraIndex>;
-    };
-    session: {
-      [index: string]: AugmentedConst<object & Codec>;
-      /**
-       * Used as first key for `NextKeys` and `KeyOwner` to put all the data into the same branch
-       * of the trie.
-       **/
-      dedupKeyPrefix: AugmentedConst<Bytes>;
-    };
     democracy: {
       [index: string]: AugmentedConst<object & Codec>;
+      /**
+       * Period in blocks where an external proposal may not be re-submitted after being vetoed.
+       **/
+      cooloffPeriod: AugmentedConst<BlockNumber>;
+      /**
+       * Minimum voting period allowed for an emergency referendum.
+       **/
+      emergencyVotingPeriod: AugmentedConst<BlockNumber>;
       /**
        * The minimum period of locking and the period between a proposal being approved and enacted.
        * It should generally be a little more than the unstake period to ensure that
@@ -87,48 +128,105 @@ declare module '@polkadot/metadata/Decorated/consts/types' {
        * How often (in blocks) new public referenda are launched.
        **/
       launchPeriod: AugmentedConst<BlockNumber>;
-      /**
-       * How often (in blocks) to check for new votes.
-       **/
-      votingPeriod: AugmentedConst<BlockNumber>;
       /**
        * The minimum amount to be used as a deposit for a public referendum proposal.
        **/
       minimumDeposit: AugmentedConst<BalanceOf>;
-      /**
-       * Minimum voting period allowed for an emergency referendum.
-       **/
-      emergencyVotingPeriod: AugmentedConst<BlockNumber>;
-      /**
-       * Period in blocks where an external proposal may not be re-submitted after being vetoed.
-       **/
-      cooloffPeriod: AugmentedConst<BlockNumber>;
       /**
        * The amount of balance that must be deposited per byte of preimage stored.
        **/
       preimageByteDeposit: AugmentedConst<BalanceOf>;
+      /**
+       * How often (in blocks) to check for new votes.
+       **/
+      votingPeriod: AugmentedConst<BlockNumber>;
     };
     elections: {
       [index: string]: AugmentedConst<object & Codec>;
       candidacyBond: AugmentedConst<BalanceOf>;
-      votingBond: AugmentedConst<BalanceOf>;
       desiredMembers: AugmentedConst<u32>;
       desiredRunnersUp: AugmentedConst<u32>;
       termDuration: AugmentedConst<BlockNumber>;
+      votingBond: AugmentedConst<BalanceOf>;
     };
     finalityTracker: {
       [index: string]: AugmentedConst<object & Codec>;
-      /**
-       * The number of recent samples to keep from this chain. Default is 101.
-       **/
-      windowSize: AugmentedConst<BlockNumber>;
       /**
        * The delay after which point things become suspicious. Default is 1000.
        **/
       reportLatency: AugmentedConst<BlockNumber>;
+      /**
+       * The number of recent samples to keep from this chain. Default is 101.
+       **/
+      windowSize: AugmentedConst<BlockNumber>;
+    };
+    society: {
+      [index: string]: AugmentedConst<object & Codec>;
+      /**
+       * The minimum amount of a deposit required for a bid to be made.
+       **/
+      candidateDeposit: AugmentedConst<BalanceOf>;
+      /**
+       * The number of blocks between membership challenges.
+       **/
+      challengePeriod: AugmentedConst<BlockNumber>;
+      /**
+       * The number of times a member may vote the wrong way (or not at all, when they are a skeptic)
+       * before they become suspended.
+       **/
+      maxStrikes: AugmentedConst<u32>;
+      /**
+       * The amount of incentive paid within each period. Doesn't include VoterTip.
+       **/
+      periodSpend: AugmentedConst<BalanceOf>;
+      /**
+       * The number of blocks between candidate/membership rotation periods.
+       **/
+      rotationPeriod: AugmentedConst<BlockNumber>;
+      /**
+       * The amount of the unpaid reward that gets deducted in the case that either a skeptic
+       * doesn't vote or someone votes in the wrong way.
+       **/
+      wrongSideDeduction: AugmentedConst<BalanceOf>;
+    };
+    staking: {
+      [index: string]: AugmentedConst<object & Codec>;
+      /**
+       * Number of eras that staked funds must remain bonded for.
+       **/
+      bondingDuration: AugmentedConst<EraIndex>;
+      /**
+       * Number of sessions per era.
+       **/
+      sessionsPerEra: AugmentedConst<SessionIndex>;
+    };
+    timestamp: {
+      [index: string]: AugmentedConst<object & Codec>;
+      /**
+       * The minimum period between blocks. Beware that this is different to the *expected* period
+       * that the block production apparatus provides. Your chosen consensus system will generally
+       * work with this to determine a sensible block time. e.g. For Aura, it will be double this
+       * period on default settings.
+       **/
+      minimumPeriod: AugmentedConst<Moment>;
+    };
+    transactionPayment: {
+      [index: string]: AugmentedConst<object & Codec>;
+      /**
+       * The fee to be paid for making a transaction; the base.
+       **/
+      transactionBaseFee: AugmentedConst<BalanceOf>;
+      /**
+       * The fee to be paid for making a transaction; the per-byte portion.
+       **/
+      transactionByteFee: AugmentedConst<BalanceOf>;
     };
     treasury: {
       [index: string]: AugmentedConst<object & Codec>;
+      /**
+       * Percentage of spare funds (if any) that are burnt per spend period.
+       **/
+      burn: AugmentedConst<Permill>;
       /**
        * Fraction of a proposal's value that should be bonded in order to place the proposal.
        * An accepted proposal gets these back. A rejected proposal does not.
@@ -142,10 +240,6 @@ declare module '@polkadot/metadata/Decorated/consts/types' {
        * Period between successive spends.
        **/
       spendPeriod: AugmentedConst<BlockNumber>;
-      /**
-       * Percentage of spare funds (if any) that are burnt per spend period.
-       **/
-      burn: AugmentedConst<Permill>;
       /**
        * The period for which a tip remains open after is has achieved threshold tippers.
        **/
@@ -163,108 +257,6 @@ declare module '@polkadot/metadata/Decorated/consts/types' {
        **/
       tipReportDepositPerByte: AugmentedConst<BalanceOf>;
     };
-    contracts: {
-      [index: string]: AugmentedConst<object & Codec>;
-      /**
-       * Number of block delay an extrinsic claim surcharge has.
-       * When claim surcharge is called by an extrinsic the rent is checked
-       * for current_block - delay
-       **/
-      signedClaimHandicap: AugmentedConst<BlockNumber>;
-      /**
-       * The minimum amount required to generate a tombstone.
-       **/
-      tombstoneDeposit: AugmentedConst<BalanceOf>;
-      /**
-       * Size of a contract at the time of instantiation. This is a simple way to ensure that
-       * empty contracts eventually gets deleted.
-       **/
-      storageSizeOffset: AugmentedConst<u32>;
-      /**
-       * Price of a byte of storage per one block interval. Should be greater than 0.
-       **/
-      rentByteFee: AugmentedConst<BalanceOf>;
-      /**
-       * The amount of funds a contract should deposit in order to offset
-       * the cost of one byte.
-       * Let's suppose the deposit is 1,000 BU (balance units)/byte and the rent is 1 BU/byte/day,
-       * then a contract with 1,000,000 BU that uses 1,000 bytes of storage would pay no rent.
-       * But if the balance reduced to 500,000 BU and the storage stayed the same at 1,000,
-       * then it would pay 500 BU/day.
-       **/
-      rentDepositOffset: AugmentedConst<BalanceOf>;
-      /**
-       * Reward that is received by the party whose touch has led
-       * to removal of a contract.
-       **/
-      surchargeReward: AugmentedConst<BalanceOf>;
-      /**
-       * The fee to be paid for making a transaction; the base.
-       **/
-      transactionBaseFee: AugmentedConst<BalanceOf>;
-      /**
-       * The fee to be paid for making a transaction; the per-byte portion.
-       **/
-      transactionByteFee: AugmentedConst<BalanceOf>;
-      /**
-       * The fee required to instantiate a contract instance. A reasonable default value
-       * is 21.
-       **/
-      contractFee: AugmentedConst<BalanceOf>;
-      /**
-       * The base fee charged for calling into a contract. A reasonable default
-       * value is 135.
-       **/
-      callBaseFee: AugmentedConst<Gas>;
-      /**
-       * The base fee charged for instantiating a contract. A reasonable default value
-       * is 175.
-       **/
-      instantiateBaseFee: AugmentedConst<Gas>;
-      /**
-       * The maximum nesting level of a call/instantiate stack. A reasonable default
-       * value is 100.
-       **/
-      maxDepth: AugmentedConst<u32>;
-      /**
-       * The maximum size of a storage value in bytes. A reasonable default is 16 KiB.
-       **/
-      maxValueSize: AugmentedConst<u32>;
-      /**
-       * The maximum amount of gas that could be expended per block. A reasonable
-       * default value is 10_000_000.
-       **/
-      blockGasLimit: AugmentedConst<Gas>;
-    };
-    society: {
-      [index: string]: AugmentedConst<object & Codec>;
-      /**
-       * The minimum amount of a deposit required for a bid to be made.
-       **/
-      candidateDeposit: AugmentedConst<BalanceOf>;
-      /**
-       * The amount of the unpaid reward that gets deducted in the case that either a skeptic
-       * doesn't vote or someone votes in the wrong way.
-       **/
-      wrongSideDeduction: AugmentedConst<BalanceOf>;
-      /**
-       * The number of times a member may vote the wrong way (or not at all, when they are a skeptic)
-       * before they become suspended.
-       **/
-      maxStrikes: AugmentedConst<u32>;
-      /**
-       * The amount of incentive paid within each period. Doesn't include VoterTip.
-       **/
-      periodSpend: AugmentedConst<BalanceOf>;
-      /**
-       * The number of blocks between candidate/membership rotation periods.
-       **/
-      rotationPeriod: AugmentedConst<BlockNumber>;
-      /**
-       * The number of blocks between membership challenges.
-       **/
-      challengePeriod: AugmentedConst<BlockNumber>;
-    };
     vesting: {
       [index: string]: AugmentedConst<object & Codec>;
       /**

packages/api/src/augment/rpc.ts

@@ -130,6 +130,10 @@ declare module '@polkadot/rpc-core/types.jsonrpc' {
        * Retrieves the keys with a certain prefix
        **/
       getKeys: AugmentedRpc<(key: StorageKey | string | Uint8Array | any, at?: BlockHash | string | Uint8Array) => Observable<Vec<StorageKey>>>;
+      /**
+       * Returns the keys with prefix with pagination support.
+       **/
+      getKeysPaged: AugmentedRpc<(key: StorageKey | string | Uint8Array | any, count: u32 | AnyNumber | Uint8Array, startKey?: StorageKey | string | Uint8Array | any, at?: BlockHash | string | Uint8Array) => Observable<Vec<StorageKey>>>;
       /**
        * Returns the runtime metadata
        **/

packages/api/src/base/Decorate.ts

@@ -10,8 +10,8 @@ import { SubmittableExtrinsic } from '../submittable/types';
 import { ApiInterfaceRx, ApiOptions, ApiTypes, DecorateMethod, DecoratedRpc, DecoratedRpcSection, QueryableModuleStorage, QueryableStorage, QueryableStorageEntry, QueryableStorageMulti, QueryableStorageMultiArg, SubmittableExtrinsicFunction, SubmittableExtrinsics, SubmittableModuleExtrinsics } from '../types';
 
 import BN from 'bn.js';
-import { BehaviorSubject, Observable, combineLatest, of } from 'rxjs';
-import { map, switchMap, take } from 'rxjs/operators';
+import { BehaviorSubject, Observable, combineLatest, from, of } from 'rxjs';
+import { concatMap, map, switchMap, take, tap, toArray } from 'rxjs/operators';
 import decorateDerive, { ExactDerive } from '@polkadot/api-derive';
 import { memo } from '@polkadot/api-derive/util';
 import DecoratedMeta from '@polkadot/metadata/Decorated';
@@ -40,6 +40,9 @@ interface MetaDecoration {
 
 type LinkageData = ITuple<[Codec, Linkage<Codec>]>;
 
+const PAGE_SIZE_KEYS = 256;
+const PAGE_SIZE_VALS = PAGE_SIZE_KEYS / 2;
+
 export default abstract class Decorate<ApiType extends ApiTypes> extends Events {
   public readonly registry: Registry;
 
@@ -433,43 +436,79 @@ export default abstract class Decorate<ApiType extends ApiTypes> extends Events
   private retrieveMapKeys ({ iterKey, meta }: StorageEntry, arg?: Arg): Observable<StorageKey[]> {
     assert(iterKey && (meta.type.isMap || meta.type.isDoubleMap), 'keys can only be retrieved on maps, linked maps and double maps');
 
-    const startKey = this.createType('Raw', u8aConcat(
+    const headKey = this.createType('Raw', u8aConcat(
       iterKey,
       meta.type.isDoubleMap && !isUndefined(arg) && !isNull(arg)
         ? getHasher(meta.type.asDoubleMap.hasher)(
           this.createType(meta.type.asDoubleMap.key1.toString() as 'Raw', arg).toU8a()
         )
         : new Uint8Array([])
-    ));
+    )).toHex();
+    const startSubject = new BehaviorSubject<string>(headKey);
 
-    return this._rpcCore.state.getKeys(startKey).pipe(
-      map((keys): StorageKey[] =>
-        keys.map((key) => key.decodeArgsFromMeta(meta))
+    return this._rpcCore.state.getKeysPaged
+      ? startSubject.pipe(
+        switchMap((startKey) =>
+          this._rpcCore.state.getKeysPaged(headKey, PAGE_SIZE_KEYS, startKey).pipe(
+            map((keys): StorageKey[] =>
+              keys.map((key) => key.decodeArgsFromMeta(meta))
+            )
+          )
+        ),
+        tap((keys): void => {
+          keys.length === PAGE_SIZE_KEYS
+            ? startSubject.next(keys[PAGE_SIZE_KEYS - 1].toHex())
+            : startSubject.complete();
+        }),
+        toArray(), // toArray since we want to startSubject to be completed
+        map((keysArr: StorageKey[][]): StorageKey[] =>
+          keysArr.reduce((result: StorageKey[], keys): StorageKey[] =>
+            result.concat(keys), []
+          )
+        )
       )
-    );
+      : this._rpcCore.state.getKeys(headKey).pipe(
+        map((keys): StorageKey[] =>
+          keys.map((key) => key.decodeArgsFromMeta(meta))
+        )
+      );
   }
 
   private retrieveMapEntries (entry: StorageEntry, arg?: Arg): Observable<[StorageKey, Codec][]> {
     const outputType = unwrapStorageType(entry.meta.type, entry.meta.modifier.isOptional);
 
-    return this.retrieveMapKeys(entry, arg)
-      .pipe(
-        switchMap((keys): Observable<[StorageKey[], Option<Raw>[]]> =>
-          combineLatest([
-            of(keys),
-            // Since we have a default constructed key, we have Option<Raw> as the result
-            // Don't keep the subscription here active, retrieve and get out of Dodge
-            // TODO Once we have a multiple key RPC available, use that via fallback
-            this._rpcCore.state.subscribeStorage<Option<Raw>[]>(keys).pipe(take(1))
-          ])
-        ),
-        map(([keys, values]): [StorageKey, Codec][] =>
-          keys.map((key, index): [StorageKey, Codec] => [
-            key,
-            this.createType(outputType, values[index].isSome ? values[index].unwrap().toHex() : undefined)
-          ])
-        )
-      );
+    return this.retrieveMapKeys(entry, arg).pipe(
+      switchMap((keys): Observable<[StorageKey[], Option<Raw>[]]> =>
+        combineLatest([
+          of(keys),
+          from(Array(Math.ceil(keys.length / PAGE_SIZE_VALS)).fill(0)).pipe(
+            // FIXME New RPC to take care of this in the works...
+            concatMap((_, index): Observable<Option<Raw>[]> =>
+              this._rpcCore.state.subscribeStorage<Option<Raw>[]>(
+                keys.slice(index * PAGE_SIZE_VALS, (index * PAGE_SIZE_VALS) + PAGE_SIZE_VALS)
+              ).pipe(take(1))
+            ),
+            toArray(),
+            map((valsArr: Option<Raw>[][]): Option<Raw>[] =>
+              valsArr.reduce((result: Option<Raw>[], vals): Option<Raw>[] =>
+                result.concat(vals), []
+              )
+            )
+          )
+        ])
+      ),
+      map(([keys, vals]): [StorageKey, Codec][] =>
+        keys.map((key, index): [StorageKey, Codec] => [
+          key,
+          this.createType(
+            outputType,
+            vals[index].isSome
+              ? vals[index].unwrap().toHex()
+              : undefined
+          )
+        ])
+      )
+    );
   }
 
   protected decorateDeriveRx (decorateMethod: DecorateMethod<ApiType>): DeriveAllSections<'rxjs', ExactDerive> {

packages/api/src/util/validate.spec.ts

@@ -11,7 +11,7 @@ import { TypeRegistry } from '@polkadot/types';
 
 import { extractStorageArgs } from './validate';
 
-describe.skip('extractStorageArgs', (): void => {
+describe('extractStorageArgs', (): void => {
   const registry = new TypeRegistry();
   let storage: Storage;
 
@@ -27,6 +27,12 @@ describe.skip('extractStorageArgs', (): void => {
     ).toEqual([storage.timestamp.now]);
   });
 
+  it('validates no-arg plain (with undefined, undefined)', (): void => {
+    expect(
+      extractStorageArgs(storage.timestamp.now, [undefined, undefined])
+    ).toEqual([storage.timestamp.now]);
+  });
+
   it('validates no-arg plain (failing when there are args)', (): void => {
     expect(
       (): any[] =>
@@ -74,19 +80,20 @@ describe.skip('extractStorageArgs', (): void => {
     ).toThrow('staking.erasStakers(EraIndex, AccountId) is a doublemap, requiring 2 arguments, 1 found');
   });
 
-  it('validates linked map, no args', (): void => {
+  // Linked maps have been removed
+  it.skip('validates linked map, no args', (): void => {
     expect(
       extractStorageArgs(storage.staking.validators, [])
     ).toEqual([storage.staking.validators]);
   });
 
-  it('validates linked map, single arg', (): void => {
+  it.skip('validates linked map, single arg', (): void => {
     expect(
       extractStorageArgs(storage.staking.validators, [123])
     ).toEqual([storage.staking.validators, 123]);
   });
 
-  it('validates linked map (failing with extra args)', (): void => {
+  it.skip('validates linked map (failing with extra args)', (): void => {
     expect(
       (): any[] =>
         extractStorageArgs(storage.staking.validators, [123, 456])

packages/api/src/util/validate.ts

@@ -3,53 +3,48 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 import { StorageEntry } from '@polkadot/types/primitive/StorageKey';
+import { Type } from '@polkadot/types';
+import { assert, isUndefined } from '@polkadot/util';
 
-// FIXME This breaks with .at(hash, ...)
+function sig ({ method, section }: StorageEntry, ...args: Type[]): string {
+  return `${section}.${method}(${args.join(', ')})`;
+}
 
-// import { Type } from '@polkadot/types';
-// import { assert } from '@polkadot/util';
+function doDoubleMap (creator: StorageEntry, args: any[]): [StorageEntry, [any, any]] {
+  const { key1, key2 } = creator.meta.type.asDoubleMap;
 
-// function sig ({ method, section }: StorageEntry, ...args: Type[]): string {
-//   return `${section}.${method}(${args.join(', ')})`;
-// }
+  assert(args.length === 2, `${sig(creator, key1, key2)} is a doublemap, requiring 2 arguments, ${args.length} found`);
 
-// function doDoubleMap (creator: StorageEntry, args: any[]): [StorageEntry, [any, any]] {
-//   const { key1, key2 } = creator.meta.type.asDoubleMap;
+  // pass as tuple
+  return [creator, args as [any, any]];
+}
 
-//   assert(args.length === 2, `${sig(creator, key1, key2)} is a doublemap, requiring 2 arguments, ${args.length} found`);
+function doMap (creator: StorageEntry, args: any[]): [StorageEntry] | [StorageEntry, any] {
+  const { key, linked } = creator.meta.type.asMap;
 
-//   // pass as tuple
-//   return [creator, args as [any, any]];
-// }
+  linked.isTrue
+    ? assert(args.length <= 1, `${sig(creator, key)} is a linked map, requiring either 0 arguments (retrieving all) or 1 argument, ${args.length} found`)
+    : assert(args.length === 1, `${sig(creator, key)} is a map, requiring 1 argument, ${args.length} found`);
 
-// function doMap (creator: StorageEntry, args: any[]): [StorageEntry] | [StorageEntry, any] {
-//   const { key, linked } = creator.meta.type.asMap;
-
-//   linked.isTrue
-//     ? assert(args.length <= 1, `${sig(creator, key)} is a linked map, requiring either 0 arguments (retrieving all) or 1 argument, ${args.length} found`)
-//     : assert(args.length === 1, `${sig(creator, key)} is a map, requiring 1 argument, ${args.length} found`);
-
-//   // expand
-//   return args.length
-//     ? [creator, args[0]]
-//     : [creator];
-// }
+  // expand
+  return args.length
+    ? [creator, args[0]]
+    : [creator];
+}
 
 // sets up the arguments in the form of [creator, args] ready to be used in a storage
 // call. Additionally, it verifies that the correct number of arguments have been passed
-export function extractStorageArgs (creator: StorageEntry, args: any[]): any[] {
-  return creator.meta.type.isDoubleMap
-    ? [creator, args]
-    : [creator, ...args];
+export function extractStorageArgs (creator: StorageEntry, _args: any[]): any[] {
+  const args = _args.filter((arg) => !isUndefined(arg));
 
-  // if (creator.meta.type.isDoubleMap) {
-  //   return doDoubleMap(creator, args);
-  // } else if (creator.meta.type.isMap) {
-  //   return doMap(creator, args);
-  // }
+  if (creator.meta.type.isDoubleMap) {
+    return doDoubleMap(creator, args);
+  } else if (creator.meta.type.isMap) {
+    return doMap(creator, args);
+  }
 
-  // assert(args.length === 0, `${sig(creator)} does not take any arguments, ${args.length} found`);
+  assert(args.length === 0, `${sig(creator)} does not take any arguments, ${args.length} found`);
 
-  // // no args
-  // return [creator, ...args];
+  // no args
+  return [creator];
 }

packages/api/test/api.spec.ts

@@ -5,6 +5,7 @@
 import { ApiPromise, WsProvider } from '@polkadot/api';
 
 function createApi (): Promise<ApiPromise> {
+  jest.setTimeout(30000);
   process.env.NODE_ENV = 'test';
 
   const provider = new WsProvider('wss://kusama-rpc.polkadot.io');
@@ -24,12 +25,28 @@ describe.skip('misc quick tests', (): void => {
     ));
   });
 
+  it.skip('handles map keys', async (): Promise<void> => {
+    const api = await createApi();
+
+    console.time('map.keys');
+
+    const keys = await api.query.system.account.keys();
+
+    console.error('# keys', keys.length);
+
+    console.timeEnd('map.keys');
+  });
+
   it.skip('handles map entries', async (): Promise<void> => {
     const api = await createApi();
 
-    console.error(JSON.stringify(
-      await api.query.indices.accounts.entries()
-    ));
+    console.time('map.entries');
+
+    const entries = await api.query.system.account.entries(); // api.query.indices.accounts.entries();
+
+    console.error('# entries', entries.length);
+
+    console.timeEnd('map.entries');
   });
 
   it.skip('handles doublemap entries', async (): Promise<void> => {

packages/jsonrpc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/jsonrpc",
-  "version": "1.6.2",
+  "version": "1.7.1",
   "description": "Method definitions for the Polkadot RPC layer",
   "main": "index.js",
   "publishConfig": {
@@ -26,8 +26,8 @@
   },
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/jsonrpc#readme",
   "dependencies": {
-    "@babel/runtime": "^7.8.4",
-    "@polkadot/types": "1.6.2",
+    "@babel/runtime": "^7.8.7",
+    "@polkadot/types": "1.7.1",
     "@polkadot/util": "^2.6.2"
   }
 }

packages/jsonrpc/src/state.ts

@@ -30,6 +30,17 @@ const getKeys: RpcMethodOpt = {
   type: 'Vec<StorageKey>'
 };
 
+const getKeysPaged: RpcMethodOpt = {
+  description: 'Returns the keys with prefix with pagination support.',
+  params: [
+    createParam('key', 'StorageKey'),
+    createParam('count', 'u32'),
+    createParam('startKey', 'StorageKey', { isOptional: true }),
+    createParam('at', 'BlockHash', { isOptional: true })
+  ],
+  type: 'Vec<StorageKey>'
+};
+
 const getStorage: RpcMethodOpt = {
   description: 'Retrieves the storage for a key',
   params: KEY_QUERY_PARAMS,
@@ -148,6 +159,7 @@ export default {
     getChildStorageHash: createMethod(section, 'getChildStorageHash', getChildStorageHash),
     getChildStorageSize: createMethod(section, 'getChildStorageSize', getChildStorageSize),
     getKeys: createMethod(section, 'getKeys', getKeys),
+    getKeysPaged: createMethod(section, 'getKeysPaged', getKeysPaged),
     getMetadata: createMethod(section, 'getMetadata', getMetadata),
     getRuntimeVersion: createMethod(section, 'getRuntimeVersion', getRuntimeVersion),
     getStorage: createMethod(section, 'getStorage', getStorage),

packages/metadata/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/metadata",
-  "version": "1.6.2",
+  "version": "1.7.1",
   "description": "Helpers to extract information from runtime metadata",
   "main": "index.js",
   "publishConfig": {
@@ -26,8 +26,8 @@
   },
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/type-metadata#readme",
   "dependencies": {
-    "@babel/runtime": "^7.8.4",
-    "@polkadot/types": "1.6.2",
+    "@babel/runtime": "^7.8.7",
+    "@polkadot/types": "1.7.1",
     "@polkadot/util": "^2.6.2",
     "@polkadot/util-crypto": "^2.6.2",
     "bn.js": "^5.1.1"

packages/metadata/src/Decorated/Decorated.spec.ts

@@ -18,7 +18,7 @@ describe('Decorated', () => {
         decorated.query.system.account('5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY')
       )
     ).toEqual(
-      '0x010126aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da92e3fb4c297a84c5cebc0e78257d213d0927ccc7596044c6ba013dd05522aacba'
+      '0x410126aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da9de1e86a9a8c739864cf3cc5ec2bea59fd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d'
     );
   });
 

packages/metadata/src/Decorated/consts/fromMetadata/fromMetadata.spec.ts

@@ -19,7 +19,8 @@ describe('fromMetadata', (): void => {
     expect(consts.democracy.cooloffPeriod.toNumber()).toEqual(28 * 24 * 60 * (60 / 3));
   });
 
-  it('correctly handles bytes', (): void => {
+  // removed from session
+  it.skip('correctly handles bytes', (): void => {
     // 0x34 removes as the length prefix removed
     expect(consts.session.dedupKeyPrefix.toHex()).toEqual('0x3a73657373696f6e3a6b657973');
   });

packages/metadata/src/Decorated/storage/fromMetadata/fromMetadata.spec.ts

@@ -31,8 +31,7 @@ describe('fromMetadata', (): void => {
           decorated.query.system.account(keyring.alice.address)
         )
       ).toEqual(
-        // new storage key format
-        '0x010126aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da92e3fb4c297a84c5cebc0e78257d213d0927ccc7596044c6ba013dd05522aacba'
+        '0x410126aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da9de1e86a9a8c739864cf3cc5ec2bea59fd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d'
       );
     });
   });

packages/metadata/src/Decorated/storage/fromMetadata/getHasher.ts

@@ -2,6 +2,7 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
+import metadataDefs from '@polkadot/types/interfaces/metadata/definitions';
 import { StorageHasher } from '@polkadot/types/interfaces';
 import { u8aConcat, u8aToU8a } from '@polkadot/util';
 import { blake2AsU8a, xxhashAsU8a } from '@polkadot/util-crypto';
@@ -10,24 +11,26 @@ export type HasherInput = string | Buffer | Uint8Array;
 
 export type HasherFunction = (data: HasherInput) => Uint8Array;
 
-type HasherCheck = 'isBlake2128' | 'isBlake2128Concat' | 'isBlake2256' | 'isTwox128' | 'isTwox256' | 'isTwox64Concat';
+const DEFAULT_FN = (data: HasherInput): Uint8Array => xxhashAsU8a(data, 128);
 
-const DEFAULT = (data: HasherInput): Uint8Array => xxhashAsU8a(data, 128);
-
-const map: Record<HasherCheck, HasherFunction> = {
-  isBlake2128: (data: HasherInput): Uint8Array => blake2AsU8a(data, 128),
-  isBlake2128Concat: (data: HasherInput): Uint8Array => u8aConcat(blake2AsU8a(data, 128), u8aToU8a(data)),
-  isBlake2256: (data: HasherInput): Uint8Array => blake2AsU8a(data, 256),
-  isTwox128: (data: HasherInput): Uint8Array => xxhashAsU8a(data, 128),
-  isTwox256: (data: HasherInput): Uint8Array => xxhashAsU8a(data, 256),
-  isTwox64Concat: (data: HasherInput): Uint8Array => u8aConcat(xxhashAsU8a(data, 64), u8aToU8a(data))
+const HASHERS: Record<keyof typeof metadataDefs.types.StorageHasherV11._enum, HasherFunction> = {
+  Blake2_128: (data: HasherInput): Uint8Array => // eslint-disable-line @typescript-eslint/camelcase
+    blake2AsU8a(data, 128),
+  Blake2_128Concat: (data: HasherInput): Uint8Array => // eslint-disable-line @typescript-eslint/camelcase
+    u8aConcat(blake2AsU8a(data, 128), u8aToU8a(data)),
+  Blake2_256: (data: HasherInput): Uint8Array => // eslint-disable-line @typescript-eslint/camelcase
+    blake2AsU8a(data, 256),
+  Identity: (data: HasherInput): Uint8Array =>
+    u8aToU8a(data),
+  Twox128: (data: HasherInput): Uint8Array =>
+    xxhashAsU8a(data, 128),
+  Twox256: (data: HasherInput): Uint8Array =>
+    xxhashAsU8a(data, 256),
+  Twox64Concat: (data: HasherInput): Uint8Array =>
+    u8aConcat(xxhashAsU8a(data, 64), u8aToU8a(data))
 };
 
 /** @internal */
 export default function getHasher (hasher?: StorageHasher): HasherFunction {
-  const [, fn] = (hasher && Object.entries(map).find(([check]): boolean =>
-    hasher[check as HasherCheck]
-  )) || [undefined, DEFAULT];
-
-  return fn;
+  return HASHERS[hasher?.type as 'Identity'] || DEFAULT_FN;
 }

packages/metadata/src/Metadata/v11/static-substrate.json

@@ -13,7 +13,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Blake2_128Concat",
                     "key": "AccountId",
                     "value": "AccountInfo",
                     "linked": false
@@ -62,7 +62,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "BlockNumber",
                     "value": "Hash",
                     "linked": false
@@ -160,7 +160,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Blake2_128Concat",
                     "key": "Hash",
                     "value": "Vec<(BlockNumber,EventIndex)>",
                     "linked": false
@@ -181,14 +181,14 @@
                 ]
               },
               {
-                "name": "RuntimeUpgraded",
-                "modifier": "Default",
+                "name": "LastRuntimeUpgrade",
+                "modifier": "Optional",
                 "type": {
-                  "Plain": "bool"
+                  "Plain": "LastRuntimeUpgradeInfo"
                 },
                 "fallback": "0x00",
                 "documentation": [
-                  " A bool to track if the runtime was upgraded last block."
+                  " Stores the `spec_version` and `spec_name` of when the last runtime upgrade happened."
                 ]
               }
             ]
@@ -309,6 +309,16 @@
                 " Kill the sending account, assuming there are no references outstanding and the composite",
                 " data is equal to its default value."
               ]
+            },
+            {
+              "name": "migrate_accounts",
+              "args": [
+                {
+                  "name": "accounts",
+                  "type": "Vec<AccountId>"
+                }
+              ],
+              "documentation": []
             }
           ],
           "events": [
@@ -367,26 +377,12 @@
               ]
             },
             {
-              "name": "SpecVersionNotAllowedToDecrease",
+              "name": "SpecVersionNeedsToIncrease",
               "documentation": [
                 " The specification version is not allowed to decrease between the current runtime",
                 " and the new runtime."
               ]
             },
-            {
-              "name": "ImplVersionNotAllowedToDecrease",
-              "documentation": [
-                " The implementation version is not allowed to decrease between the current runtime",
-                " and the new runtime."
-              ]
-            },
-            {
-              "name": "SpecOrImplVersionNeedToIncrease",
-              "documentation": [
-                " The specification or the implementation version need to increase between the",
-                " current runtime and the new runtime."
-              ]
-            },
             {
               "name": "FailedToExtractRuntimeVersion",
               "documentation": [
@@ -899,7 +895,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "u32",
                     "value": "Vec<[u8;32]>",
                     "linked": false
@@ -928,7 +924,7 @@
             {
               "name": "EpochDuration",
               "type": "u64",
-              "value": "0x2800000000000000",
+              "value": "0xc800000000000000",
               "documentation": [
                 " The number of **slots** that an epoch takes. We couple sessions to",
                 " epochs, i.e. we start a new session once the new epoch begins."
@@ -1303,7 +1299,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Blake2_128Concat",
                     "key": "AccountId",
                     "value": "AccountData",
                     "linked": false
@@ -1324,7 +1320,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Blake2_128Concat",
                     "key": "AccountId",
                     "value": "Vec<BalanceLock>",
                     "linked": false
@@ -1585,7 +1581,7 @@
         {
           "name": "TransactionPayment",
           "storage": {
-            "prefix": "Balances",
+            "prefix": "TransactionPayment",
             "items": [
               {
                 "name": "NextFeeMultiplier",
@@ -1682,7 +1678,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "AccountId",
                     "linked": false
@@ -1698,7 +1694,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Blake2_128Concat",
                     "key": "AccountId",
                     "value": "StakingLedger",
                     "linked": false
@@ -1714,7 +1710,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "RewardDestination",
                     "linked": false
@@ -1730,10 +1726,10 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "ValidatorPrefs",
-                    "linked": true
+                    "linked": false
                   }
                 },
                 "fallback": "0x00",
@@ -1746,10 +1742,10 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "Nominations",
-                    "linked": true
+                    "linked": false
                   }
                 },
                 "fallback": "0x00",
@@ -1790,7 +1786,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "EraIndex",
                     "value": "SessionIndex",
                     "linked": false
@@ -1841,6 +1837,7 @@
                   "",
                   " This is similar to [`ErasStakers`] but number of nominators exposed is reduce to the",
                   " `T::MaxNominatorRewardedPerValidator` biggest stakers.",
+                  " (Note: the field `total` and `own` of the exposure remains unchanged).",
                   " This is used to limit the i/o cost for the nominator payout.",
                   "",
                   " This is keyed fist by the era index to allow bulk deletion and then the stash account.",
@@ -1865,7 +1862,7 @@
                 "documentation": [
                   " Similarly to `ErasStakers` this holds the preferences of validators.",
                   "",
-                  " This is keyed fist by the era index to allow bulk deletion and then the stash account.",
+                  " This is keyed first by the era index to allow bulk deletion and then the stash account.",
                   "",
                   " Is it removed after `HISTORY_DEPTH` eras."
                 ]
@@ -1875,7 +1872,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "EraIndex",
                     "value": "BalanceOf",
                     "linked": false
@@ -1893,7 +1890,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "EraIndex",
                     "value": "EraRewardPoints",
                     "linked": false
@@ -1910,7 +1907,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "EraIndex",
                     "value": "BalanceOf",
                     "linked": false
@@ -1963,7 +1960,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "EraIndex",
                     "value": "Vec<UnappliedSlash>",
                     "linked": false
@@ -1993,11 +1990,11 @@
                 "modifier": "Optional",
                 "type": {
                   "DoubleMap": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key1": "EraIndex",
                     "key2": "AccountId",
                     "value": "(Perbill,BalanceOf)",
-                    "key2Hasher": "Twox128"
+                    "key2Hasher": "Twox64Concat"
                   }
                 },
                 "fallback": "0x00",
@@ -2011,11 +2008,11 @@
                 "modifier": "Optional",
                 "type": {
                   "DoubleMap": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key1": "EraIndex",
                     "key2": "AccountId",
                     "value": "BalanceOf",
-                    "key2Hasher": "Twox128"
+                    "key2Hasher": "Twox64Concat"
                   }
                 },
                 "fallback": "0x00",
@@ -2028,7 +2025,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "SlashingSpans",
                     "linked": false
@@ -2044,7 +2041,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "(AccountId,SpanIndex)",
                     "value": "SpanRecord",
                     "linked": false
@@ -2073,7 +2070,7 @@
                 "type": {
                   "Plain": "ReleasesStaking"
                 },
-                "fallback": "0x00",
+                "fallback": "0x01",
                 "documentation": [
                   " Storage version of the pallet.",
                   "",
@@ -2107,6 +2104,8 @@
                 "",
                 " The dispatch origin for this call must be _Signed_ by the stash account.",
                 "",
+                " Emits `Bonded`.",
+                "",
                 " # <weight>",
                 " - Independent of the arguments. Moderate complexity.",
                 " - O(1).",
@@ -2135,6 +2134,8 @@
                 "",
                 " The dispatch origin for this call must be _Signed_ by the stash, not the controller.",
                 "",
+                " Emits `Bonded`.",
+                "",
                 " # <weight>",
                 " - Independent of the arguments. Insignificant complexity.",
                 " - O(1).",
@@ -2164,6 +2165,8 @@
                 "",
                 " The dispatch origin for this call must be _Signed_ by the controller, not the stash.",
                 "",
+                " Emits `Unbonded`.",
+                "",
                 " See also [`Call::withdraw_unbonded`].",
                 "",
                 " # <weight>",
@@ -2188,6 +2191,8 @@
                 "",
                 " The dispatch origin for this call must be _Signed_ by the controller, not the stash.",
                 "",
+                " Emits `Withdrawn`.",
+                "",
                 " See also [`Call::unbond`].",
                 "",
                 " # <weight>",
@@ -2414,7 +2419,10 @@
                 " - `who` is the controller account of the nominator to pay out.",
                 " - `era` may not be lower than one following the most recently paid era. If it is higher,",
                 "   then it indicates an instruction to skip the payout of all previous eras.",
-                " - `validators` is the list of all validators that `who` had exposure to during `era`.",
+                " - `validators` is the list of all validators that `who` had exposure to during `era`,",
+                "   alongside the index of `who` in the clipped exposure of the validator.",
+                "   I.e. each element is a tuple of",
+                "   `(validator, index of `who` in clipped exposure of validator)`.",
                 "   If it is incomplete, then less than the full reward will be paid out.",
                 "   It must not exceed `MAX_NOMINATIONS`.",
                 "",
@@ -2540,6 +2548,40 @@
                 " An old slashing report from a prior era was discarded because it could",
                 " not be processed."
               ]
+            },
+            {
+              "name": "Bonded",
+              "args": [
+                "AccountId",
+                "Balance"
+              ],
+              "documentation": [
+                " An account has bonded this amount.",
+                "",
+                " NOTE: This event is only emitted when funds are bonded via a dispatchable. Notably,",
+                " it will not be emitted for staking rewards when they are added to stake."
+              ]
+            },
+            {
+              "name": "Unbonded",
+              "args": [
+                "AccountId",
+                "Balance"
+              ],
+              "documentation": [
+                " An account has unbonded this amount."
+              ]
+            },
+            {
+              "name": "Withdrawn",
+              "args": [
+                "AccountId",
+                "Balance"
+              ],
+              "documentation": [
+                " An account has called `withdraw_unbonded` and removed unbonding chunks worth `Balance`",
+                " from the unlocking queue."
+              ]
             }
           ],
           "constants": [
@@ -2638,6 +2680,12 @@
               "documentation": [
                 " Invalid number of nominations."
               ]
+            },
+            {
+              "name": "NotSortedAndUnique",
+              "documentation": [
+                " Items are not sorted and unique."
+              ]
             }
           ]
         },
@@ -2709,40 +2757,32 @@
                 "name": "NextKeys",
                 "modifier": "Optional",
                 "type": {
-                  "DoubleMap": {
+                  "Map": {
                     "hasher": "Twox64Concat",
-                    "key1": "Bytes",
-                    "key2": "ValidatorId",
+                    "key": "ValidatorId",
                     "value": "Keys",
-                    "key2Hasher": "Blake2_256"
+                    "linked": false
                   }
                 },
                 "fallback": "0x00",
                 "documentation": [
-                  " The next session keys for a validator.",
-                  "",
-                  " The first key is always `DEDUP_KEY_PREFIX` to have all the data in the same branch of",
-                  " the trie. Having all data in the same branch should prevent slowing down other queries."
+                  " The next session keys for a validator."
                 ]
               },
               {
                 "name": "KeyOwner",
                 "modifier": "Optional",
                 "type": {
-                  "DoubleMap": {
+                  "Map": {
                     "hasher": "Twox64Concat",
-                    "key1": "Bytes",
-                    "key2": "(KeyTypeId,Bytes)",
+                    "key": "(KeyTypeId,Bytes)",
                     "value": "ValidatorId",
-                    "key2Hasher": "Blake2_256"
+                    "linked": false
                   }
                 },
                 "fallback": "0x00",
                 "documentation": [
-                  " The owner of a key. The second key is the `KeyTypeId` + the encoded key.",
-                  "",
-                  " The first key is always `DEDUP_KEY_PREFIX` to have all the data in the same branch of",
-                  " the trie. Having all data in the same branch should prevent slowing down other queries."
+                  " The owner of a key. The key is the `KeyTypeId` + the encoded key."
                 ]
               }
             ]
@@ -2804,17 +2844,7 @@
               ]
             }
           ],
-          "constants": [
-            {
-              "name": "DEDUP_KEY_PREFIX",
-              "type": "Bytes",
-              "value": "0x343a73657373696f6e3a6b657973",
-              "documentation": [
-                " Used as first key for `NextKeys` and `KeyOwner` to put all the data into the same branch",
-                " of the trie."
-              ]
-            }
-          ],
+          "constants": [],
           "errors": [
             {
               "name": "InvalidProof",
@@ -2874,7 +2904,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Identity",
                     "key": "Hash",
                     "value": "(Bytes,AccountId,BalanceOf,BlockNumber)",
                     "linked": false
@@ -2891,7 +2921,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "PropIndex",
                     "value": "(BalanceOf,Vec<AccountId>)",
                     "linked": false
@@ -2930,7 +2960,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "ReferendumIndex",
                     "value": "ReferendumInfo",
                     "linked": false
@@ -2957,7 +2987,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "ReferendumIndex",
                     "value": "Vec<AccountId>",
                     "linked": false
@@ -2973,7 +3003,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "(ReferendumIndex,AccountId)",
                     "value": "Vote",
                     "linked": false
@@ -2992,7 +3022,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "ProxyState",
                     "linked": false
@@ -3009,10 +3039,10 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "(AccountId,Conviction)",
-                    "linked": true
+                    "linked": false
                   }
                 },
                 "fallback": "0x000000000000000000000000000000000000000000000000000000000000000000",
@@ -3025,7 +3055,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "BlockNumber",
                     "linked": false
@@ -3068,7 +3098,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Identity",
                     "key": "Hash",
                     "value": "(BlockNumber,Vec<AccountId>)",
                     "linked": false
@@ -3085,7 +3115,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Identity",
                     "key": "Hash",
                     "value": "bool",
                     "linked": false
@@ -3260,7 +3290,7 @@
                 " an external referendum.",
                 "",
                 " The dispatch of this call must be `ExternalMajorityOrigin`.",
-                " ",
+                "",
                 " - `proposal_hash`: The preimage hash of the proposal.",
                 "",
                 " Unlike `external_propose`, blacklisting has no effect on this and it may replace a",
@@ -3347,7 +3377,7 @@
                 " Veto and blacklist the external proposal hash.",
                 "",
                 " The dispatch origin of this call must be `VetoOrigin`.",
-                " ",
+                "",
                 " - `proposal_hash`: The preimage hash of the proposal to veto and blacklist.",
                 "",
                 " Emits `Vetoed`.",
@@ -3543,7 +3573,7 @@
                 " Emits `PreimageNoted`.",
                 "",
                 " # <weight>",
-                " - Dependent on the size of `encoded_proposal` but protected by a ",
+                " - Dependent on the size of `encoded_proposal` but protected by a",
                 "   required deposit.",
                 " # </weight>"
               ]
@@ -4062,7 +4092,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Identity",
                     "key": "Hash",
                     "value": "Proposal",
                     "linked": false
@@ -4078,7 +4108,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Identity",
                     "key": "Hash",
                     "value": "Votes",
                     "linked": false
@@ -4377,7 +4407,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Identity",
                     "key": "Hash",
                     "value": "Proposal",
                     "linked": false
@@ -4393,7 +4423,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Identity",
                     "key": "Hash",
                     "value": "Votes",
                     "linked": false
@@ -4714,10 +4744,10 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "Vec<AccountId>",
-                    "linked": true
+                    "linked": false
                   }
                 },
                 "fallback": "0x00",
@@ -4730,7 +4760,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "BalanceOf",
                     "linked": false
@@ -5361,7 +5391,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "SetId",
                     "value": "SessionIndex",
                     "linked": false
@@ -5369,7 +5399,8 @@
                 },
                 "fallback": "0x00",
                 "documentation": [
-                  " A mapping from grandpa set ID to the index of the *most recent* session for which its members were responsible."
+                  " A mapping from grandpa set ID to the index of the *most recent* session for which its",
+                  " members were responsible."
                 ]
               }
             ]
@@ -5464,7 +5495,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "ProposalIndex",
                     "value": "TreasuryProposal",
                     "linked": false
@@ -5509,7 +5540,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Twox64Concat",
+                    "hasher": "Identity",
                     "key": "Hash",
                     "value": "Bytes",
                     "linked": false
@@ -5974,7 +6005,7 @@
         {
           "name": "Contracts",
           "storage": {
-            "prefix": "Contract",
+            "prefix": "Contracts",
             "items": [
               {
                 "name": "GasSpent",
@@ -6003,7 +6034,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Identity",
                     "key": "CodeHash",
                     "value": "Bytes",
                     "linked": false
@@ -6019,7 +6050,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Identity",
                     "key": "CodeHash",
                     "value": "PrefabWasmModule",
                     "linked": false
@@ -6046,7 +6077,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "ContractInfo",
                     "linked": false
@@ -6616,17 +6647,17 @@
                 "modifier": "Optional",
                 "type": {
                   "DoubleMap": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key1": "SessionIndex",
                     "key2": "AuthIndex",
                     "value": "Bytes",
-                    "key2Hasher": "Blake2_256"
+                    "key2Hasher": "Twox64Concat"
                   }
                 },
                 "fallback": "0x00",
                 "documentation": [
-                  " For each session index, we keep a mapping of `AuthIndex`",
-                  " to `offchain::OpaqueNetworkState`."
+                  " For each session index, we keep a mapping of `AuthIndex` to",
+                  " `offchain::OpaqueNetworkState`."
                 ]
               },
               {
@@ -6634,11 +6665,11 @@
                 "modifier": "Default",
                 "type": {
                   "DoubleMap": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key1": "SessionIndex",
                     "key2": "ValidatorId",
                     "value": "u32",
-                    "key2Hasher": "Blake2_256"
+                    "key2Hasher": "Twox64Concat"
                   }
                 },
                 "fallback": "0x00000000",
@@ -6726,7 +6757,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "ReportIdOf",
                     "value": "OffenceDetails",
                     "linked": false
@@ -6742,11 +6773,11 @@
                 "modifier": "Default",
                 "type": {
                   "DoubleMap": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key1": "Kind",
                     "key2": "OpaqueTimeSlot",
                     "value": "Vec<ReportIdOf>",
-                    "key2Hasher": "Blake2_256"
+                    "key2Hasher": "Twox64Concat"
                   }
                 },
                 "fallback": "0x00",
@@ -6759,7 +6790,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "Kind",
                     "value": "Bytes",
                     "linked": false
@@ -6822,14 +6853,14 @@
         {
           "name": "Identity",
           "storage": {
-            "prefix": "Sudo",
+            "prefix": "Identity",
             "items": [
               {
                 "name": "IdentityOf",
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "Registration",
                     "linked": false
@@ -6845,7 +6876,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Blake2_128Concat",
                     "key": "AccountId",
                     "value": "(AccountId,Data)",
                     "linked": false
@@ -6862,7 +6893,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "(BalanceOf,Vec<AccountId>)",
                     "linked": false
@@ -7399,7 +7430,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "(BalanceOf,BidKind)",
                     "linked": false
@@ -7448,7 +7479,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "bool",
                     "linked": false
@@ -7475,7 +7506,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "VouchingStatus",
                     "linked": false
@@ -7491,7 +7522,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "Vec<(BlockNumber,BalanceOf)>",
                     "linked": false
@@ -7507,7 +7538,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "StrikeCount",
                     "linked": false
@@ -8329,7 +8360,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "RecoveryConfig",
                     "linked": false
@@ -8365,7 +8396,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Blake2_128Concat",
                     "key": "AccountId",
                     "value": "AccountId",
                     "linked": false
@@ -8839,7 +8870,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Blake2_128Concat",
                     "key": "AccountId",
                     "value": "VestingInfo",
                     "linked": false
@@ -8912,7 +8943,7 @@
                 }
               ],
               "documentation": [
-                " Create a vested transfer. ",
+                " Create a vested transfer.",
                 "",
                 " The dispatch origin for this call must be _Signed_.",
                 "",

packages/rpc-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/rpc-core",
-  "version": "1.6.2",
+  "version": "1.7.1",
   "description": "A JavaScript wrapper for the Polkadot JsonRPC interface",
   "main": "index.js",
   "keywords": [
@@ -26,11 +26,11 @@
   },
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/rpc-core#readme",
   "dependencies": {
-    "@babel/runtime": "^7.8.4",
-    "@polkadot/jsonrpc": "1.6.2",
-    "@polkadot/metadata": "1.6.2",
-    "@polkadot/rpc-provider": "1.6.2",
-    "@polkadot/types": "1.6.2",
+    "@babel/runtime": "^7.8.7",
+    "@polkadot/jsonrpc": "1.7.1",
+    "@polkadot/metadata": "1.7.1",
+    "@polkadot/rpc-provider": "1.7.1",
+    "@polkadot/types": "1.7.1",
     "@polkadot/util": "^2.6.2",
     "memoizee": "^0.4.14",
     "rxjs": "^6.5.4"

packages/rpc-provider/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/rpc-provider",
-  "version": "1.6.2",
+  "version": "1.7.1",
   "description": "Transport providers for the API",
   "main": "index.js",
   "keywords": [
@@ -26,10 +26,10 @@
   },
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/rpc-provider#readme",
   "dependencies": {
-    "@babel/runtime": "^7.8.4",
-    "@polkadot/jsonrpc": "1.6.2",
-    "@polkadot/metadata": "1.6.2",
-    "@polkadot/types": "1.6.2",
+    "@babel/runtime": "^7.8.7",
+    "@polkadot/jsonrpc": "1.7.1",
+    "@polkadot/metadata": "1.7.1",
+    "@polkadot/types": "1.7.1",
     "@polkadot/util": "^2.6.2",
     "@polkadot/util-crypto": "^2.6.2",
     "bn.js": "^5.1.1",
@@ -40,6 +40,6 @@
   "devDependencies": {
     "@polkadot/keyring": "^2.6.2",
     "mock-socket": "^9.0.3",
-    "nock": "^12.0.1"
+    "nock": "^12.0.2"
   }
 }

packages/rpc-provider/src/mock/index.ts

@@ -60,6 +60,7 @@ export default class Mock implements ProviderInterface {
     chain_getBlockHash: (blockNumber: number): string => '0x1234',
     chain_getHeader: (): any => this.registry.createType('Header', rpcHeader.result).toJSON(),
     state_getKeys: (): string[] => [],
+    state_getKeysPaged: (): string[] => [],
     state_getRuntimeVersion: (): string => this.registry.createType('RuntimeVersion').toHex(),
     state_getMetadata: (): string => rpcMetadata,
     state_getStorage: (storage: MockStateDb, params: any[]): string => u8aToHex(storage[(params[0] as string)]),

packages/typegen/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/typegen",
-  "version": "1.6.2",
+  "version": "1.7.1",
   "description": "Type generation scripts",
   "main": "index.js",
   "bin": {
@@ -33,20 +33,20 @@
   },
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/typegen#readme",
   "dependencies": {
-    "@babel/core": "^7.8.3",
-    "@babel/register": "^7.8.3",
-    "@babel/runtime": "^7.8.4",
-    "@polkadot/api": "1.6.2",
-    "@polkadot/jsonrpc": "1.6.2",
-    "@polkadot/metadata": "1.6.2",
-    "@polkadot/rpc-provider": "1.6.2",
-    "@polkadot/types": "1.6.2",
+    "@babel/core": "^7.8.7",
+    "@babel/register": "^7.8.6",
+    "@babel/runtime": "^7.8.7",
+    "@polkadot/api": "1.7.1",
+    "@polkadot/jsonrpc": "1.7.1",
+    "@polkadot/metadata": "1.7.1",
+    "@polkadot/rpc-provider": "1.7.1",
+    "@polkadot/types": "1.7.1",
     "@polkadot/util": "^2.6.2",
     "websocket": "^1.0.31",
-    "yargs": "^15.1.0"
+    "yargs": "^15.3.0"
   },
   "devDependencies": {
     "@types/websocket": "^1.0.0",
-    "@types/yargs": "^15.0.3"
+    "@types/yargs": "^15.0.4"
   }
 }

packages/typegen/src/generate/consts.ts

@@ -25,7 +25,7 @@ function generateModule (allDefs: object, { constants, name }: ModuleMetadataLat
 
   return [indent(4)(`${stringCamelCase(name.toString())}: {`)]
     .concat(isStrict ? '' : indent(6)('[index: string]: AugmentedConst<object & Codec>;'))
-    .concat(constants.map(({ documentation, name, type }): string => {
+    .concat(constants.sort((a, b) => a.name.localeCompare(b.name.toString())).map(({ documentation, name, type }): string => {
       setImports(allDefs, imports, [type.toString()]);
 
       return createDocComments(6, documentation) +
@@ -42,7 +42,7 @@ function generateForMeta (meta: Metadata, dest: string, extraTypes: Record<strin
     const allDefs = Object.entries(allTypes).reduce((defs, [path, obj]) => {
       return Object.entries(obj).reduce((defs, [key, value]) => ({ ...defs, [`${path}/${key}`]: value }), defs);
     }, {});
-    const body = meta.asLatest.modules.reduce((acc, mod): string[] => {
+    const body = meta.asLatest.modules.sort((a, b) => a.name.localeCompare(b.name.toString())).reduce((acc, mod): string[] => {
       return acc.concat(generateModule(allDefs, mod, imports, isStrict));
     }, [] as string[]);
     const header = createImportCode(HEADER('chain'), imports, [

packages/typegen/src/generate/query.ts

@@ -69,7 +69,7 @@ function generateModule (allDefs: object, registry: Registry, { name, storage }:
 
   return [indent(4)(`${stringLowerFirst(name.toString())}: {`)]
     .concat(isStrict ? '' : indent(6)('[index: string]: QueryableStorageEntry<ApiType>;'))
-    .concat(storage.unwrap().items.map((storageEntry): string => {
+    .concat(storage.unwrap().items.sort((a, b) => a.name.localeCompare(b.name.toString())).map((storageEntry): string => {
       const [args, returnType] = entrySignature(allDefs, registry, storageEntry, imports);
       let entryType = 'AugmentedQuery';
 
@@ -91,7 +91,7 @@ function generateForMeta (registry: Registry, meta: Metadata, dest: string, extr
     const allDefs = Object.entries(allTypes).reduce((defs, [path, obj]) => {
       return Object.entries(obj).reduce((defs, [key, value]) => ({ ...defs, [`${path}/${key}`]: value }), defs);
     }, {});
-    const body = meta.asLatest.modules.reduce((acc: string[], mod): string[] => {
+    const body = meta.asLatest.modules.sort((a, b) => a.name.localeCompare(b.name.toString())).reduce((acc: string[], mod): string[] => {
       return acc.concat(generateModule(allDefs, registry, mod, imports, isStrict));
     }, []);
     const header = createImportCode(HEADER('chain'), imports, [

packages/typegen/src/generate/tx.ts

@@ -38,7 +38,7 @@ function generateModule (registry: Registry, allDefs: object, { calls, name }: M
 
   return [indent(4)(`${stringCamelCase(name.toString())}: {`)]
     .concat(isStrict ? '' : indent(6)('[index: string]: SubmittableExtrinsicFunction<ApiType>;'))
-    .concat(allCalls.map(({ args, documentation, name }): string => {
+    .concat(allCalls.sort((a, b) => a.name.localeCompare(b.name.toString())).map(({ args, documentation, name }): string => {
       const params = args
         .map(({ name, type }): [string, string, string] => {
           const typeStr = type.toString();
@@ -66,7 +66,7 @@ function generateForMeta (registry: Registry, meta: Metadata, dest: string, extr
     const allDefs = Object.entries(allTypes).reduce((defs, [path, obj]) => {
       return Object.entries(obj).reduce((defs, [key, value]) => ({ ...defs, [`${path}/${key}`]: value }), defs);
     }, {});
-    const body = meta.asLatest.modules.reduce((acc, mod): string[] => {
+    const body = meta.asLatest.modules.sort((a, b) => a.name.localeCompare(b.name.toString())).reduce((acc, mod): string[] => {
       return acc.concat(generateModule(registry, allDefs, mod, imports, isStrict));
     }, [] as string[]);
     const header = createImportCode(HEADER('chain'), imports, [

packages/types/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/types",
-  "version": "1.6.2",
+  "version": "1.7.1",
   "description": "Implementation of the Parity codec",
   "main": "index.js",
   "keywords": [
@@ -26,8 +26,8 @@
   },
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/types#readme",
   "dependencies": {
-    "@babel/runtime": "^7.8.4",
-    "@polkadot/metadata": "1.6.2",
+    "@babel/runtime": "^7.8.7",
+    "@polkadot/metadata": "1.7.1",
     "@polkadot/util": "^2.6.2",
     "@polkadot/util-crypto": "^2.6.2",
     "@types/bn.js": "^4.11.6",

packages/types/src/augment/registry.ts

@@ -30,7 +30,7 @@ import { AccountId, AccountIdOf, AccountIndex, Address, AssetId, Balance, Balanc
 import { FullIdentification, IdentificationTuple, Keys, SessionIndex, SessionKeys1, SessionKeys2, SessionKeys3, SessionKeys4, SessionKeys5, SessionKeys6 } from '@polkadot/types/interfaces/session';
 import { Bid, BidKind, SocietyJudgement, SocietyVote, StrikeCount, VouchingStatus } from '@polkadot/types/interfaces/society';
 import { ActiveEraInfo, CompactAssignments, CompactScore, ElectionCompute, ElectionResult, ElectionStatus, EraIndex, EraPoints, EraRewardPoints, EraRewards, Exposure, Forcing, IndividualExposure, KeyType, MomentOf, Nominations, PhragmenScore, Points, ReleasesStaking, RewardDestination, RewardPoint, SlashJournalEntry, SlashingSpans, SlashingSpansTo204, SpanIndex, SpanRecord, StakingLedger, StakingLedgerTo223, UnappliedSlash, UnappliedSlashOther, UnlockChunk, ValidatorIndex, ValidatorPrefs, ValidatorPrefsTo145, ValidatorPrefsTo196 } from '@polkadot/types/interfaces/staking';
-import { AccountInfo, DigestOf, DispatchError, DispatchErrorModule, DispatchErrorTo198, DispatchResult, DispatchResultOf, DispatchResultTo198, Event, EventId, EventIndex, EventRecord, EventRecordTo76, Key, Phase, RefCount } from '@polkadot/types/interfaces/system';
+import { AccountInfo, DigestOf, DispatchError, DispatchErrorModule, DispatchErrorTo198, DispatchResult, DispatchResultOf, DispatchResultTo198, Event, EventId, EventIndex, EventRecord, EventRecordTo76, Key, LastRuntimeUpgradeInfo, Phase, RefCount } from '@polkadot/types/interfaces/system';
 import { OpenTip, OpenTipFinder, OpenTipTip, TreasuryProposal } from '@polkadot/types/interfaces/treasury';
 import { Multiplier } from '@polkadot/types/interfaces/txpayment';
 import { Multisig, Timepoint } from '@polkadot/types/interfaces/utility';
@@ -850,6 +850,9 @@ declare module '@polkadot/types/types/registry' {
     Key: Key;
     'Option<Key>': Option<Key>;
     'Vec<Key>': Vec<Key>;
+    LastRuntimeUpgradeInfo: LastRuntimeUpgradeInfo;
+    'Option<LastRuntimeUpgradeInfo>': Option<LastRuntimeUpgradeInfo>;
+    'Vec<LastRuntimeUpgradeInfo>': Vec<LastRuntimeUpgradeInfo>;
     Phase: Phase;
     'Option<Phase>': Option<Phase>;
     'Vec<Phase>': Vec<Phase>;

packages/types/src/interfaces/extrinsics/definitions.ts

@@ -37,7 +37,7 @@ export default {
     Signature: 'H512',
     SignerPayload: 'GenericSignerPayload',
     EcdsaSignature: '[u8; 65]',
-    Ed25519Signature: 'Signature',
-    Sr25519Signature: 'Signature'
+    Ed25519Signature: 'H512',
+    Sr25519Signature: 'H512'
   }
 };

packages/types/src/interfaces/extrinsics/types.ts

@@ -9,7 +9,7 @@ import { H512 } from '@polkadot/types/interfaces/runtime';
 export interface EcdsaSignature extends U8aFixed {}
 
 /** @name Ed25519Signature */
-export interface Ed25519Signature extends Signature {}
+export interface Ed25519Signature extends H512 {}
 
 /** @name Extrinsic */
 export interface Extrinsic extends GenericExtrinsic {}
@@ -85,4 +85,4 @@ export interface Signature extends H512 {}
 export interface SignerPayload extends GenericSignerPayload {}
 
 /** @name Sr25519Signature */
-export interface Sr25519Signature extends Signature {}
+export interface Sr25519Signature extends H512 {}

packages/types/src/interfaces/metadata/definitions.ts

@@ -431,7 +431,13 @@ export default {
     },
 
     // v11
-    DoubleMapTypeV11: 'DoubleMapTypeV10',
+    DoubleMapTypeV11: {
+      hasher: 'StorageHasherV11',
+      key1: 'Type',
+      key2: 'Type',
+      value: 'Type',
+      key2Hasher: 'StorageHasherV11'
+    },
     ErrorMetadataV11: 'ErrorMetadataV10',
     EventMetadataV11: 'EventMetadataV10',
     ExtrinsicMetadataV11: {
@@ -440,19 +446,57 @@ export default {
     },
     FunctionArgumentMetadataV11: 'FunctionArgumentMetadataV10',
     FunctionMetadataV11: 'FunctionMetadataV10',
-    MapTypeV11: 'MapTypeV10',
+    MapTypeV11: {
+      hasher: 'StorageHasherV11',
+      key: 'Type',
+      value: 'Type',
+      linked: 'bool'
+    },
     MetadataV11: {
       modules: 'Vec<ModuleMetadataV11>',
       extrinsic: 'ExtrinsicMetadataV11'
     },
     ModuleConstantMetadataV11: 'ModuleConstantMetadataV10',
-    ModuleMetadataV11: 'ModuleMetadataV10',
+    ModuleMetadataV11: {
+      name: 'Text',
+      storage: 'Option<StorageMetadataV11>',
+      calls: 'Option<Vec<FunctionMetadataV11>>',
+      events: 'Option<Vec<EventMetadataV11>>',
+      constants: 'Vec<ModuleConstantMetadataV11>',
+      errors: 'Vec<ErrorMetadataV11>'
+    },
     PlainTypeV11: 'Type',
     StorageEntryModifierV11: 'StorageEntryModifierV10',
-    StorageEntryMetadataV11: 'StorageEntryMetadataV10',
-    StorageEntryTypeV11: 'StorageEntryTypeV10',
-    StorageMetadataV11: 'StorageMetadataV10',
-    StorageHasherV11: 'StorageHasherV10',
+    StorageEntryMetadataV11: {
+      name: 'Text',
+      modifier: 'StorageEntryModifierV11',
+      type: 'StorageEntryTypeV11',
+      fallback: 'Bytes',
+      documentation: 'Vec<Text>'
+    },
+    StorageEntryTypeV11: {
+      _enum: {
+        Plain: 'PlainTypeV11',
+        Map: 'MapTypeV11',
+        DoubleMap: 'DoubleMapTypeV11'
+      }
+    },
+    StorageMetadataV11: {
+      prefix: 'Text',
+      items: 'Vec<StorageEntryMetadataV11>'
+    },
+    StorageHasherV11: {
+      _enum: {
+        Blake2_128: null, // eslint-disable-line @typescript-eslint/camelcase
+        Blake2_256: null, // eslint-disable-line @typescript-eslint/camelcase
+        Blake2_128Concat: null, // eslint-disable-line @typescript-eslint/camelcase
+        Twox128: null,
+        Twox256: null,
+        Twox64Concat: null,
+        // new in v11
+        Identity: null
+      }
+    },
 
     // This always maps to the latest
     DoubleMapTypeLatest: 'DoubleMapTypeV11',

packages/types/src/interfaces/metadata/types.ts

@@ -24,7 +24,13 @@ export interface DoubleMapTypeV10 extends Struct {
 }
 
 /** @name DoubleMapTypeV11 */
-export interface DoubleMapTypeV11 extends DoubleMapTypeV10 {}
+export interface DoubleMapTypeV11 extends Struct {
+  readonly hasher: StorageHasherV11;
+  readonly key1: Type;
+  readonly key2: Type;
+  readonly value: Type;
+  readonly key2Hasher: StorageHasherV11;
+}
 
 /** @name DoubleMapTypeV3 */
 export interface DoubleMapTypeV3 extends Struct {
@@ -239,7 +245,12 @@ export interface MapTypeV10 extends Struct {
 }
 
 /** @name MapTypeV11 */
-export interface MapTypeV11 extends MapTypeV10 {}
+export interface MapTypeV11 extends Struct {
+  readonly hasher: StorageHasherV11;
+  readonly key: Type;
+  readonly value: Type;
+  readonly linked: bool;
+}
 
 /** @name MapTypeV2 */
 export interface MapTypeV2 extends Struct {
@@ -421,7 +432,14 @@ export interface ModuleMetadataV10 extends Struct {
 }
 
 /** @name ModuleMetadataV11 */
-export interface ModuleMetadataV11 extends ModuleMetadataV10 {}
+export interface ModuleMetadataV11 extends Struct {
+  readonly name: Text;
+  readonly storage: Option<StorageMetadataV11>;
+  readonly calls: Option<Vec<FunctionMetadataV11>>;
+  readonly events: Option<Vec<EventMetadataV11>>;
+  readonly constants: Vec<ModuleConstantMetadataV11>;
+  readonly errors: Vec<ErrorMetadataV11>;
+}
 
 /** @name ModuleMetadataV2 */
 export interface ModuleMetadataV2 extends Struct {
@@ -572,7 +590,13 @@ export interface StorageEntryMetadataV10 extends Struct {
 }
 
 /** @name StorageEntryMetadataV11 */
-export interface StorageEntryMetadataV11 extends StorageEntryMetadataV10 {}
+export interface StorageEntryMetadataV11 extends Struct {
+  readonly name: Text;
+  readonly modifier: StorageEntryModifierV11;
+  readonly type: StorageEntryTypeV11;
+  readonly fallback: Bytes;
+  readonly documentation: Vec<Text>;
+}
 
 /** @name StorageEntryMetadataV6 */
 export interface StorageEntryMetadataV6 extends StorageFunctionMetadataV5 {}
@@ -621,7 +645,14 @@ export interface StorageEntryTypeV10 extends Enum {
 }
 
 /** @name StorageEntryTypeV11 */
-export interface StorageEntryTypeV11 extends StorageEntryTypeV10 {}
+export interface StorageEntryTypeV11 extends Enum {
+  readonly isPlain: boolean;
+  readonly asPlain: PlainTypeV11;
+  readonly isMap: boolean;
+  readonly asMap: MapTypeV11;
+  readonly isDoubleMap: boolean;
+  readonly asDoubleMap: DoubleMapTypeV11;
+}
 
 /** @name StorageEntryTypeV6 */
 export interface StorageEntryTypeV6 extends StorageFunctionTypeV5 {}
@@ -768,7 +799,15 @@ export interface StorageHasherV10 extends Enum {
 }
 
 /** @name StorageHasherV11 */
-export interface StorageHasherV11 extends StorageHasherV10 {}
+export interface StorageHasherV11 extends Enum {
+  readonly isBlake2128: boolean;
+  readonly isBlake2256: boolean;
+  readonly isBlake2128Concat: boolean;
+  readonly isTwox128: boolean;
+  readonly isTwox256: boolean;
+  readonly isTwox64Concat: boolean;
+  readonly isIdentity: boolean;
+}
 
 /** @name StorageHasherV4 */
 export interface StorageHasherV4 extends Enum {
@@ -810,7 +849,10 @@ export interface StorageMetadataV10 extends Struct {
 }
 
 /** @name StorageMetadataV11 */
-export interface StorageMetadataV11 extends StorageMetadataV10 {}
+export interface StorageMetadataV11 extends Struct {
+  readonly prefix: Text;
+  readonly items: Vec<StorageEntryMetadataV11>;
+}
 
 /** @name StorageMetadataV7 */
 export interface StorageMetadataV7 extends Struct {

packages/types/src/interfaces/system/definitions.ts

@@ -42,6 +42,10 @@ export default {
       event: 'Event'
     },
     Key: 'Bytes',
+    LastRuntimeUpgradeInfo: {
+      specVersion: 'Compact<u32>',
+      specName: 'Text'
+    },
     Phase: {
       _enum: {
         ApplyExtrinsic: 'u32',

packages/types/src/interfaces/system/types.ts

@@ -2,7 +2,7 @@
 /* eslint-disable @typescript-eslint/no-empty-interface */
 
 import { ITuple } from '@polkadot/types/types';
-import { Enum, Option, Result, Struct, U8aFixed, Vec } from '@polkadot/types/codec';
+import { Compact, Enum, Option, Result, Struct, U8aFixed, Vec } from '@polkadot/types/codec';
 import { GenericEvent } from '@polkadot/types/generic';
 import { Bytes, Text, u32, u8 } from '@polkadot/types/primitive';
 import { AccountData } from '@polkadot/types/interfaces/balances';
@@ -83,6 +83,12 @@ export interface EventRecordTo76 extends Struct {
 /** @name Key */
 export interface Key extends Bytes {}
 
+/** @name LastRuntimeUpgradeInfo */
+export interface LastRuntimeUpgradeInfo extends Struct {
+  readonly specVersion: Compact<u32>;
+  readonly specName: Text;
+}
+
 /** @name Phase */
 export interface Phase extends Enum {
   readonly isApplyExtrinsic: boolean;

packages/types/src/primitive/StorageKey.ts

@@ -32,39 +32,45 @@ interface StorageKeyExtra {
   section: string;
 }
 
-// order important, matching enum
-const HASHER_MAP: Record<keyof typeof metadataDefs.types.StorageHasherV10._enum, [number, boolean]> = {
+const HASHER_MAP: Record<keyof typeof metadataDefs.types.StorageHasherV11._enum, [number, boolean]> = {
+  // opaque
   Blake2_128: [16, false], // eslint-disable-line @typescript-eslint/camelcase
   Blake2_256: [32, false], // eslint-disable-line @typescript-eslint/camelcase
-  Blake2_128Concat: [16, true], // eslint-disable-line @typescript-eslint/camelcase
   Twox128: [16, false],
   Twox256: [32, false],
-  Twox64Concat: [8, true]
-};
-const HASHER_OPTS = Object.values(HASHER_MAP);
 
-function getStorageType (type: StorageEntryTypeLatest): string {
+  // can decode
+  Blake2_128Concat: [16, true], // eslint-disable-line @typescript-eslint/camelcase
+  Twox64Concat: [8, true],
+  Identity: [0, true]
+};
+
+function getStorageType (type: StorageEntryTypeLatest, isOptionalLinked?: boolean): [boolean, string] {
   if (type.isPlain) {
-    return type.asPlain.toString() as keyof InterfaceTypes;
+    return [false, type.asPlain.toString()];
   } else if (type.isDoubleMap) {
-    return type.asDoubleMap.value.toString() as keyof InterfaceTypes;
+    return [false, type.asDoubleMap.value.toString()];
   }
 
   const map = type.asMap;
 
   if (map.linked.isTrue) {
-    return `(${map.value.toString()}, Linkage<${map.key.toString()}>)` as keyof InterfaceTypes;
+    const [pre, post] = isOptionalLinked
+      ? ['Option<', '>']
+      : ['', ''];
+
+    return [true, `(${pre}${map.value.toString()}${post}, Linkage<${map.key.toString()}>)`];
   }
 
-  return map.value.toString();
+  return [false, map.value.toString()];
 }
 
 // we unwrap the type here, turning into an output usable for createType
 /** @internal */
 export function unwrapStorageType (type: StorageEntryTypeLatest, isOptional?: boolean): keyof InterfaceTypes {
-  const outputType = getStorageType(type);
+  const [hasWrapper, outputType] = getStorageType(type, isOptional);
 
-  return isOptional
+  return isOptional && !hasWrapper
     ? `Option<${outputType}>` as keyof InterfaceTypes
     : outputType as keyof InterfaceTypes;
 }
@@ -104,14 +110,13 @@ function decodeStorageKey (value?: AnyU8a | StorageKey | StorageEntry | [Storage
 
 function decodeHashers (registry: Registry, value: Uint8Array, hashers: [StorageHasher, string][]): Codec[] {
   // the storage entry is xxhashAsU8a(prefix, 128) + xxhashAsU8a(method, 128), 256 bits total
-  const encoded = value.subarray(32);
-  let offset = 0;
+  let offset = 32;
 
   return hashers.reduce((result: Codec[], [hasher, type]): Codec[] => {
-    const [hashLen, canDecode] = HASHER_OPTS[hasher.index];
+    const [hashLen, canDecode] = HASHER_MAP[hasher.type as 'Identity'];
     const decoded = canDecode
-      ? registry.createType(type as 'Raw', encoded.subarray(offset + hashLen))
-      : registry.createType('Raw', encoded.subarray(offset, offset + hashLen));
+      ? registry.createType(type as 'Raw', value.subarray(offset + hashLen))
+      : registry.createType('Raw', value.subarray(offset, offset + hashLen));
 
     offset += hashLen + (canDecode ? decoded.encodedLength : 0);
     result.push(decoded);