[CI Skip] 1.7.1

skip-checks: true

.123trigger

@@ -1,3 +1,3 @@
-19
- 1.4.2
+20
  1.4.2
+1.6.2

.github/workflows/pr-any.yml

@@ -2,110 +2,18 @@ name: PR
 on: [pull_request]
 
 jobs:
-  lint:
-    name: Linting
-    runs-on: ubuntu-latest
+  pr:
     strategy:
       matrix:
-        node-version: [12.x]
-    steps:
-    - uses: actions/checkout@v1
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v1
-      with:
-        node-version: ${{ matrix.node-version }}
-    - name: Config yarn cache
-      id: yarn-cache-dir-path
-      run: echo "::set-output name=dir::$(yarn cache dir)"
-    - uses: actions/cache@v1
-      id: yarn-cache
-      with:
-        path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-        key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-        restore-keys: |
-          ${{ runner.os }}-yarn-
-    - name: lint
-      run: |
-        yarn install --immutable | grep -v 'YN0013'
-        yarn lint
-
-  test:
-    name: Testing
+        step: ['lint', 'test', 'build', 'docs']
+    name: ${{ matrix.step }}
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [12.x]
     steps:
     - uses: actions/checkout@v1
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v1
+    - uses: actions/setup-node@v1
       with:
-        node-version: ${{ matrix.node-version }}
-    - name: Config yarn cache
-      id: yarn-cache-dir-path
-      run: echo "::set-output name=dir::$(yarn cache dir)"
-    - uses: actions/cache@v1
-      id: yarn-cache
-      with:
-        path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-        key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-        restore-keys: |
-          ${{ runner.os }}-yarn-
-    - name: test
+        node-version: '12.x'
+    - name: ${{ matrix.step }}
       run: |
         yarn install --immutable | grep -v 'YN0013'
-        yarn test
-
-  build_code:
-    name: Build Code
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [12.x]
-    steps:
-    - uses: actions/checkout@v1
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v1
-      with:
-        node-version: ${{ matrix.node-version }}
-    - name: Config yarn cache
-      id: yarn-cache-dir-path
-      run: echo "::set-output name=dir::$(yarn cache dir)"
-    - uses: actions/cache@v1
-      id: yarn-cache
-      with:
-        path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-        key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-        restore-keys: |
-          ${{ runner.os }}-yarn-
-    - name: build
-      run: |
-        yarn install --immutable | grep -v 'YN0013'
-        yarn build
-
-  build_docs:
-    name: Build Docs
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [12.x]
-    steps:
-    - uses: actions/checkout@v1
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v1
-      with:
-        node-version: ${{ matrix.node-version }}
-    - name: Config yarn cache
-      id: yarn-cache-dir-path
-      run: echo "::set-output name=dir::$(yarn cache dir)"
-    - uses: actions/cache@v1
-      id: yarn-cache
-      with:
-        path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-        key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-        restore-keys: |
-          ${{ runner.os }}-yarn-
-    - name: build
-      run: |
-        yarn install --immutable | grep -v 'YN0013'
-        yarn docs
+        yarn ${{ matrix.step }}

.github/workflows/push-master.yml

@@ -5,82 +5,36 @@ on:
       - master
 
 jobs:
-  build_code:
-    name: Build Code
-    if: "! contains(github.event.head_commit.message, '[CI Skip]')"
-    runs-on: ubuntu-latest
+  master:
     strategy:
       matrix:
-        node-version: [12.x]
+        step: ['build:release', 'docs:release']
+    name: ${{ matrix.step }}
+    if: "! contains(github.event.head_commit.message, '[CI Skip]')"
+    runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v1
       with:
         token: ${{ secrets.GH_PAT }}
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v1
+    - uses: actions/setup-node@v1
       with:
-        node-version: ${{ matrix.node-version }}
-    - name: Config yarn cache
-      id: yarn-cache-dir-path
-      run: echo "::set-output name=dir::$(yarn cache dir)"
-    - uses: actions/cache@v1
-      id: yarn-cache
-      with:
-        path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-        key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-        restore-keys: |
-          ${{ runner.os }}-yarn-
-    - name: build
+        node-version: '12.x'
+    - name: ${{ matrix.step }}
       env:
         CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
+        GH_PAGES_SRC: build-docs
         GH_PAT: ${{ secrets.GH_PAT }}
         GH_TOKEN: ${{ secrets.GH_PAT }}
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
       run: |
         yarn install --immutable | grep -v 'YN0013'
-        yarn polkadot-ci-ghact-build
-
-  build_docs:
-    name: Build Docs
-    if: "! contains(github.event.head_commit.message, '[CI Skip]')"
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [12.x]
-    steps:
-    - uses: actions/checkout@v1
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v1
-      with:
-        node-version: ${{ matrix.node-version }}
-    - name: Config yarn cache
-      id: yarn-cache-dir-path
-      run: echo "::set-output name=dir::$(yarn cache dir)"
-    - uses: actions/cache@v1
-      id: yarn-cache
-      with:
-        path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-        key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-        restore-keys: |
-          ${{ runner.os }}-yarn-
-    - name: build
-      env:
-        GH_PAT: ${{ secrets.GH_PAT }}
-        GH_TOKEN: ${{ secrets.GH_PAT }}
-        GH_PAGES_SRC: build-docs
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: |
-        yarn install --immutable | grep -v 'YN0013'
-        yarn polkadot-ci-ghact-docs
+        yarn ${{ matrix.step }}
 
   dummy:
     name: Dummy
     if: "contains(github.event.head_commit.message, '[CI Skip]')"
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [12.x]
     steps:
     - name: dummy
       run: |

.yarnrc.yml

@@ -1,4 +1,5 @@
 enableImmutableInstalls: false
+
 enableProgressBars: false
 
 nodeLinker: node-modules
@@ -7,4 +8,4 @@ plugins:
   - path: .yarn/plugins/@yarnpkg/plugin-interactive-tools.js
     spec: "@yarnpkg/plugin-interactive-tools"
 
-yarnPath: .yarn/releases/yarn-berry.js
+yarnPath: .yarn/releases/yarn-rc.js

CHANGELOG.md

@@ -1,3 +1,31 @@
+# 1.7.1 Mar 17, 2020
+
+- **Important** Current versions of Polkadot/Substrate have dropped support for linked maps on storage entries. If you are using these queries to retriev all entries, for instance `staking.{nominators,validators}()` and using an upgraded chain, you need to swap to retrieving entries via `.entries()` or `.keys()` for the keys.
+- Pull in support and types for latest Polkadot/Substrate
+- Add support for the new Substrate `state_getKeysPaged` RPC, including use in storage keys
+- Move `derive.staking.controllers` to `derive.staking.stashes`, reflecting actual content
+- Cater for adjusted storage (non-linked mapped) for `derive.staking.stashes` queries (with old-compat)
+- Expanded `derive.staking.*` derives, including addition of `derive.staking.own*`
+- Re-add fixed (with tests) checks for query args, previously disabled in 1.6.2
+
+# 1.6.2 Mar 12, 2020
+
+- Revert checks for query args, not working on `.at()` queries (proper fix in next version)
+
+# 1.6.1 Mar 12, 2020
+
+- **Breaking change** `api.rpc.state.queryStorage(...)` now fully decodes the `Vec<StorageChangeSet>` and returns a decoded `[Hash, Codec[]][]` when using this RPC.
+- `StorageKey` now has an `.args` property, decoded from meta where `twox64_concat` or `blake128_concat` are used on maps
+- Fix `api.query.*.*.entries` type conversions to return exact types (not `Option` in some cases)
+- Add `api.query.*.*.keys` to retrieve only the storage keys, similar to `.entries`
+- Full linked map retrievals will now use direct getStorage queries for faster operation
+- Underlying rpc-core interfaces now unwraps `Error("...")` when found in responses
+- Added `derive.eras*` interfaces for queries to new Substrate staking interfaces
+- Update `derive.account` to cater for new indices module storage (detected with fallbacks)
+- Adjust derive queries for session without module prefix (DoubleMap -> Map), detected on use
+- Add runtime validation for map arguments to `api.query.*`
+- TypeScript interfaces for linked maps now correctly generates as `[Type, Linkage<Next>]`
+
 # 1.5.1 Mar 06, 2020
 
 - **Important** Substrate master has updated staking, on older chains supply `StakingLedger: 'StakingLedgerTo223'`

copy-to-apps.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 function copy_folder () {
   SRC="packages/$1/build"

docs/start/FAQ.md

@@ -34,7 +34,7 @@ To submit these transactions, it needs to be send as a [wrapped transaction](api
 
 ## How do I call a function with a Tuple input
 
-Tuples, as defined in the Polkadot/Substrate types appear as `(TypeA, TypeB)`. For instance we may have an `(AccountId, u64)` input as defined in the metadata or as part of the user types. To specify a Tuple as an input, wrap it is an  array format, for instance to call `query.module.get((u32, u64))` where a `(u32, u64)` Tuple input is expected, you would do  `query.module.get([123, 456])`
+Tuples, as defined in the Polkadot/Substrate types appear as `(TypeA, TypeB)`. For instance we may have an `(AccountId, u64)` input as defined in the metadata or as part of the user types. To specify a Tuple as an input, wrap it in an  array format, for instance to call `query.module.get((u32, u64))` where a `(u32, u64)` Tuple input is expected, you would do  `query.module.get([123, 456])`
 
 ## How long do transactions live
 

docs/start/api.query.other.md

@@ -22,7 +22,7 @@ const [{ data: balanceNow }, { data: balancePrev }] = await Promise.all([
 console.log(`The delta was ${balanceNow.free.sub(balancePrev.free)}`);
 ```
 
-In the above example, we introduce the `.at(<hash>[, ...params])` query. For all `.at` queries, the first parameter is always the block hash at which we want to make the query, in our example we use both the last retrieved block and the parent thereof. The params are optional as per the type of query made, for instance to retrieve the timestamp for a previous block, it would be -
+In the above example, we introduce the `.at(<hash>[, ...params]): Type` query. For all `.at` queries, the first parameter is always the block hash at which we want to make the query, in our example we use both the last retrieved block and the parent thereof. The params are optional as per the type of query made, for instance to retrieve the timestamp for a previous block, it would be -
 
 ```js
 ...
@@ -35,6 +35,46 @@ The `.at` queries are all single-shot, i.e. there are no subscription option to
 
 An additional point to take care of (briefly mentioned above), is state pruning. By default a Polkadot/Substrate node will only keep state for the last 256 blocks, unless it is explicitly run in archive mode. This means that querying state further back than the pruning period will result in an error returned from the Node. (Generally most public RPC nodes only run with default settings, which includes aggressive state pruning)
 
+## State for a range of blocks
+
+In addition to the `.at` queries, you can also query state starting at a specific historic block and up to either a specified or the latest blocks. This is done via the `.range([from, to?], <...opt params>): [Hash, Type][]` query. As an example -
+
+```js
+...
+// Retrieve the current block header
+const lastHdr = await api.rpc.chain.getHeader();
+const startHdr = await api.rpc.chain.getBlockHash(lastHdr.number.unwrap().subn(500));
+
+// retrieve the range of changes
+const changes = await api.query.system.account.range([startHdr]);
+
+changes.forEach(([hash, value]) => {
+  console.log(hash.toHex(), value.toHuman());
+});
+```
+
+## Map keys & entries
+
+When working maps and double-maps, it is possible to retrieve a list of all the keys and entries for the map. For this we can use the `.entries(<args>): [StorageKey, Type][]` queries. For example we may want to know the current list of validator exposures at a current era in the staking module -
+
+```js
+...
+// Retrieve the active era
+const activeEra = await api.query.staking.activeEra();
+
+// retrieve all exposures int the active era
+const exposures = await api.query.staking.erasStakers.entries(activeEra.index);
+
+exposures.forEach(([key, exposure]) => {
+  console.log('key arguments:', key.args.map((k) => k.toHuman()));
+  console.log('     exposure:', exposure.toHuman());
+});
+```
+
+Here we are querying a double-map, so we supply 1 argument. No arguments on double-maps will be very costly, retrieving all the eras and associated entries. Additionally when `twox64_concat` & `blake2_concat` is used, the key `.args` will contain decoded values of the params, in this case it will contain the actual `AccountId` of the staker. (Since that was not supplied)
+
+In the same way as above we can simply do `.keys(activeEra.index): StorageKey[]` to retrieve all the keys here, including the individual keys args decoding, as available on maps with decodable hashing functions.
+
 ## State entries
 
 In addition to using `api.query` to make actual on-chain queries, it can also be used to retrieve some information on the state entries. For instance to retrieve both the hash and size of an existing entry, we can make the following calls -

docs/start/api.tx.subs.md

@@ -62,6 +62,23 @@ const unsub = await api.tx.balances
 
 Be aware that when a transaction status is `isFinalized`, it means it is included, but it may still have failed - for instance if you try to send a larger amount that you have free, the transaction is included in a block, however from a end-user perspective the transaction failed since the transfer did not occur. In these cases a `system.ExtrinsicFailed` event will be available in the events array.
 
+## Payment information
+
+The Polkadot/Substrate RPC endpoints exposes weight/payment information that takes an encoded extrinsic and calculates the on-chain weight fees for it. A wrapper for this is available on the tx itself, taking exactly the same parameters as you would pass to a normal `.signAndSend` operation, specifically `.paymentInfo(sender, <any options>)`. To expand on our previous example -
+
+```js
+// construct a transaction
+const transfer = api.tx.balances.transfer(BOB, 12345);
+
+// retrieve the payment info
+const { partialFee, weight } = await transfer.paymentInfo(alice);
+
+console.log(`transaction will have a weight of ${weight}, with ${partialFee.toHuman()} weight fees`);
+
+// send the tx
+transfer.signAndSend(alice, ({ events = [], status }) => { ... });
+```
+
 ## Complex transactions
 
 In many cases transactions can carry quite complex information, be it for passing objects or proposing changes. In the next section we will take a dive [into complex transactions, including those wrapped for sudo](api.tx.wrap.md).

docs/start/typescript.user.md

@@ -1,6 +1,6 @@
 # TypeScript user generated
 
-In the previous section we looked at the TypeScript definitions that are available and is generated from both the chain and definitions. Here we will expand upon the use of the infrastructure created to define types as part of the `@polkadot/types` library and see how to use them to generate your onw definitions and chain types.
+In the previous section we looked at the TypeScript definitions that are available and is generated from both the chain and definitions. Here we will expand upon the use of the infrastructure created to define types as part of the `@polkadot/types` library and see how to use them to generate your own definitions and chain types.
 
 ## Definitions
 

lerna.json

@@ -9,5 +9,5 @@
   "packages": [
     "packages/*"
   ],
-  "version": "1.5.1"
+  "version": "1.7.1"
 }

package.json

@@ -12,7 +12,9 @@
   "scripts": {
     "build": "yarn build:interfaces && polkadot-dev-build-ts && (cd packages/typegen && copyfiles scripts/* build)",
     "build:interfaces": "polkadot-types-internal-interfaces",
+    "build:release": "polkadot-ci-ghact-build",
     "docs": "polkadot-types-internal-metadata && polkadot-dev-build-docs",
+    "docs:release": "polkadot-ci-ghact-docs",
     "chain:info": "polkadot-types-chain-info",
     "lint": "polkadot-dev-run-lint",
     "clean": "polkadot-dev-clean-build",
@@ -22,12 +24,13 @@
     "test:watch": "polkadot-dev-run-test --watch"
   },
   "devDependencies": {
-    "@babel/core": "^7.8.4",
-    "@babel/register": "^7.8.3",
-    "@babel/runtime": "^7.8.4",
-    "@polkadot/dev": "^0.50.27",
-    "@polkadot/ts": "^0.3.7",
+    "@babel/core": "^7.8.7",
+    "@babel/register": "^7.8.6",
+    "@babel/runtime": "^7.8.7",
+    "@polkadot/dev": "^0.51.5",
+    "@polkadot/ts": "^0.3.13",
     "@polkadot/typegen": "workspace:packages/typegen",
+    "@types/jest": "^25.1.4",
     "copyfiles": "^2.2.0"
   }
 }

packages/api-contract/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/api-contract",
-  "version": "1.5.1",
+  "version": "1.7.1",
   "description": "Interfaces for interacting with contracts and contract ABIs",
   "main": "index.js",
   "keywords": [
@@ -26,11 +26,11 @@
   },
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/api-contract#readme",
   "dependencies": {
-    "@babel/runtime": "^7.8.4",
-    "@polkadot/api": "1.5.1",
-    "@polkadot/rpc-core": "1.5.1",
-    "@polkadot/types": "1.5.1",
-    "@polkadot/util": "^2.6.1",
+    "@babel/runtime": "^7.8.7",
+    "@polkadot/api": "1.7.1",
+    "@polkadot/rpc-core": "1.7.1",
+    "@polkadot/types": "1.7.1",
+    "@polkadot/util": "^2.6.2",
     "bn.js": "^5.1.1",
     "rxjs": "^6.5.4"
   }

packages/api-derive/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/api-derive",
-  "version": "1.5.1",
+  "version": "1.7.1",
   "description": "Common functions used across Polkadot, derived from RPC calls and storage queries.",
   "main": "index.js",
   "keywords": [
@@ -27,18 +27,18 @@
   },
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/api-derive#readme",
   "dependencies": {
-    "@babel/runtime": "^7.8.4",
-    "@polkadot/api": "1.5.1",
-    "@polkadot/rpc-core": "1.5.1",
-    "@polkadot/rpc-provider": "1.5.1",
-    "@polkadot/types": "1.5.1",
-    "@polkadot/util": "^2.6.1",
-    "@polkadot/util-crypto": "^2.6.1",
+    "@babel/runtime": "^7.8.7",
+    "@polkadot/api": "1.7.1",
+    "@polkadot/rpc-core": "1.7.1",
+    "@polkadot/rpc-provider": "1.7.1",
+    "@polkadot/types": "1.7.1",
+    "@polkadot/util": "^2.6.2",
+    "@polkadot/util-crypto": "^2.6.2",
     "bn.js": "^5.1.1",
     "memoizee": "^0.4.14",
     "rxjs": "^6.5.4"
   },
   "devDependencies": {
-    "@polkadot/keyring": "^2.6.1"
+    "@polkadot/keyring": "^2.6.2"
   }
 }

packages/api-derive/src/accounts/idAndIndex.ts

@@ -7,7 +7,6 @@ import { AccountId, AccountIndex, Address } from '@polkadot/types/interfaces';
 import { Observable, of } from 'rxjs';
 import { map } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { createType } from '@polkadot/types';
 import { isU8a } from '@polkadot/util';
 import { decodeAddress } from '@polkadot/util-crypto';
 
@@ -23,14 +22,14 @@ function retrieve (api: ApiInterfaceRx, address: Address | AccountId | AccountIn
       : decodeAddress((address || '').toString());
 
     if (decoded.length === 32) {
-      const accountId = createType(api.registry, 'AccountId', decoded);
+      const accountId = api.registry.createType('AccountId', decoded);
 
       return api.derive.accounts.idToIndex(accountId).pipe(
         map((accountIndex): AccountIdAndIndex => [accountId, accountIndex])
       );
     }
 
-    const accountIndex = createType(api.registry, 'AccountIndex', decoded);
+    const accountIndex = api.registry.createType('AccountIndex', decoded);
 
     return api.derive.accounts.indexToId(accountIndex).pipe(
       map((accountId): AccountIdAndIndex => [accountId, accountIndex])

packages/api-derive/src/accounts/idToIndex.ts

@@ -6,7 +6,7 @@ import { AccountId, AccountIndex } from '@polkadot/types/interfaces';
 import { AccountIndexes } from '../types';
 
 import { Observable } from 'rxjs';
-import { map, startWith } from 'rxjs/operators';
+import { map } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
 
 import { memo } from '../util';
@@ -28,7 +28,6 @@ import { memo } from '../util';
 export function idToIndex (api: ApiInterfaceRx): (accountId: AccountId | string) => Observable<AccountIndex | undefined> {
   return memo((accountId: AccountId | string): Observable<AccountIndex | undefined> =>
     api.derive.accounts.indexes().pipe(
-      startWith({}),
       map((indexes: AccountIndexes): AccountIndex | undefined =>
         (indexes || {})[accountId.toString()]
       )

packages/api-derive/src/accounts/indexToId.ts

@@ -9,14 +9,14 @@ import { Observable, of } from 'rxjs';
 import { map, startWith } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
 import { ENUMSET_SIZE } from '@polkadot/types/generic/AccountIndex';
-import { Option, Vec, createType } from '@polkadot/types';
+import { Option, Vec } from '@polkadot/types';
 
 import { memo } from '../util';
 
 function queryEnumSet (api: ApiInterfaceRx, _accountIndex: AccountIndex | string): Observable<AccountId | undefined> {
   const accountIndex = _accountIndex instanceof api.registry.createClass('AccountIndex')
     ? _accountIndex
-    : createType(api.registry, 'AccountIndex', _accountIndex);
+    : api.registry.createType('AccountIndex', _accountIndex);
 
   return api.query.indices.enumSet<Vec<AccountId>>(accountIndex.div(ENUMSET_SIZE)).pipe(
     startWith([]),

packages/api-derive/src/accounts/indexes.ts

@@ -7,9 +7,9 @@ import { AccountId, AccountIndex } from '@polkadot/types/interfaces';
 import { AccountIndexes } from '../types';
 
 import { Observable, of } from 'rxjs';
-import { map, switchMap } from 'rxjs/operators';
+import { map, startWith, switchMap } from 'rxjs/operators';
 import { ENUMSET_SIZE } from '@polkadot/types/generic/AccountIndex';
-import { Vec, createType } from '@polkadot/types';
+import { Vec } from '@polkadot/types';
 
 import { memo } from '../util';
 
@@ -25,24 +25,33 @@ function queryEnumSet (api: ApiInterfaceRx): Observable<AccountIndexes> {
       api.query.indices.enumSet.multi<Vec<AccountId>>([...Array(next.toNumber() + 1).keys()])
     ),
     map((all: AccountId[][]): AccountIndexes =>
-      all.reduce((result: AccountIndexes, list, outerIndex): AccountIndexes => {
+      all.reduce((indexes: AccountIndexes, list, outerIndex): AccountIndexes => {
         (list || []).forEach((accountId, innerIndex): void => {
           // re-create the index based on position 0 is [0][0] and likewise
           // 64 (0..63 in first) is [1][0] (the first index value in set 2)
           const index = (outerIndex * enumsetSize) + innerIndex;
 
-          result[accountId.toString()] = createType(api.registry, 'AccountIndex', index);
+          indexes[accountId.toString()] = api.registry.createType('AccountIndex', index);
         });
 
-        return result;
+        return indexes;
       }, {})
     )
   );
 }
 
-// TODO Reverse lookup (via query keys)
-function query (): Observable<AccountIndexes> {
-  return of({} as AccountIndexes);
+function queryAccounts (api: ApiInterfaceRx): Observable<AccountIndexes> {
+  return api.query.indices.accounts.entries().pipe(
+    map((entries): AccountIndexes =>
+      entries.reduce((indexes: AccountIndexes, [key, idOpt]): AccountIndexes => {
+        if (idOpt.isSome) {
+          indexes[idOpt.unwrap()[0].toString()] = key.args[0] as AccountIndex;
+        }
+
+        return indexes;
+      }, {})
+    )
+  );
 }
 
 /**
@@ -62,10 +71,11 @@ function query (): Observable<AccountIndexes> {
  */
 export function indexes (api: ApiInterfaceRx): () => Observable<AccountIndexes> {
   return memo((): Observable<AccountIndexes> =>
-    api.query.indices
+    (api.query.indices
       ? api.query.indices.accounts
-        ? query()
+        ? queryAccounts(api)
         : queryEnumSet(api)
       : of({} as AccountIndexes)
+    ).pipe(startWith({}))
   );
 }

packages/api-derive/src/balances/account.ts

@@ -6,10 +6,9 @@ import { AccountId, AccountData, AccountIndex, AccountInfo, Address, Balance, In
 import { ITuple } from '@polkadot/types/types';
 import { DerivedBalancesAccount } from '../types';
 
-import { combineLatest, of, Observable } from 'rxjs';
+import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { createType } from '@polkadot/types';
 
 import { memo } from '../util';
 
@@ -23,7 +22,7 @@ function calcBalances (api: ApiInterfaceRx, [accountId, [freeBalance, reservedBa
     frozenFee,
     frozenMisc,
     reservedBalance,
-    votingBalance: createType(api.registry, 'Balance', freeBalance.add(reservedBalance))
+    votingBalance: api.registry.createType('Balance', freeBalance.add(reservedBalance))
   };
 }
 
@@ -35,7 +34,7 @@ function queryBalancesFree (api: ApiInterfaceRx, accountId: AccountId): Observab
     [api.query.system.accountNonce, accountId]
   ]).pipe(
     map(([freeBalance, reservedBalance, accountNonce]): Result =>
-      [freeBalance, reservedBalance, createType(api.registry, 'Balance'), createType(api.registry, 'Balance'), accountNonce]
+      [freeBalance, reservedBalance, api.registry.createType('Balance'), api.registry.createType('Balance'), accountNonce]
     )
   );
 }
@@ -93,7 +92,7 @@ export function account (api: ApiInterfaceRx): (address: AccountIndex | AccountI
                 ? queryBalancesAccount(api, accountId)
                 : queryBalancesFree(api, accountId)
           ])
-          : of([createType(api.registry, 'AccountId'), [createType(api.registry, 'Balance'), createType(api.registry, 'Balance'), createType(api.registry, 'Balance'), createType(api.registry, 'Balance'), createType(api.registry, 'Index')]])
+          : of([api.registry.createType('AccountId'), [api.registry.createType('Balance'), api.registry.createType('Balance'), api.registry.createType('Balance'), api.registry.createType('Balance'), api.registry.createType('Index')]])
         )
       ),
       map((result): DerivedBalancesAccount => calcBalances(api, result))

packages/api-derive/src/balances/all.ts

@@ -6,10 +6,10 @@ import { AccountId, AccountIndex, Address, Balance, BalanceLock, BalanceLockTo21
 import { DerivedBalancesAccount, DerivedBalancesAll } from '../types';
 
 import BN from 'bn.js';
-import { combineLatest, of, Observable } from 'rxjs';
+import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { Option, Vec, createType } from '@polkadot/types';
+import { Option, Vec } from '@polkadot/types';
 import { bnMax } from '@polkadot/util';
 
 import { memo } from '../util';
@@ -18,8 +18,9 @@ type ResultBalance = [VestingInfo | null, (BalanceLock | BalanceLockTo212)[]];
 type Result = [DerivedBalancesAccount, BlockNumber, ResultBalance];
 
 function calcBalances (api: ApiInterfaceRx, [{ accountId, accountNonce, freeBalance, frozenFee, frozenMisc, reservedBalance, votingBalance }, bestNumber, [vesting, locks]]: Result): DerivedBalancesAll {
-  let lockedBalance = createType(api.registry, 'Balance');
+  let lockedBalance = api.registry.createType('Balance');
   let lockedBreakdown: (BalanceLock | BalanceLockTo212)[] = [];
+  let allLocked = false;
 
   if (Array.isArray(locks)) {
     // only get the locks that are valid until passed the current block
@@ -27,18 +28,20 @@ function calcBalances (api: ApiInterfaceRx, [{ accountId, accountNonce, freeBala
 
     const notAll = lockedBreakdown.filter(({ amount }): boolean => !amount.isMax());
 
+    allLocked = lockedBreakdown.some(({ amount }): boolean => amount.isMax());
+
     // get the maximum of the locks according to https://github.com/paritytech/substrate/blob/master/srml/balances/src/lib.rs#L699
     if (notAll.length) {
-      lockedBalance = createType(api.registry, 'Balance', bnMax(...notAll.map(({ amount }): Balance => amount)));
+      lockedBalance = api.registry.createType('Balance', bnMax(...notAll.map(({ amount }): Balance => amount)));
     }
   }
 
   // Calculate the vesting balances,
   //  - offset = balance locked at startingBlock
   //  - perBlock is the unlock amount
-  const { locked: vestingTotal, perBlock, startingBlock } = vesting || createType(api.registry, 'VestingInfo');
+  const { locked: vestingTotal, perBlock, startingBlock } = vesting || api.registry.createType('VestingInfo');
   const isStarted = bestNumber.gt(startingBlock);
-  const vestedBalance = createType(api.registry, 'Balance', isStarted ? perBlock.mul(bestNumber.sub(startingBlock)) : 0);
+  const vestedBalance = api.registry.createType('Balance', isStarted ? perBlock.mul(bestNumber.sub(startingBlock)) : 0);
   const isVesting = isStarted && vestedBalance.lt(vestingTotal);
 
   // The available balance & vested has an interplay here
@@ -50,7 +53,7 @@ function calcBalances (api: ApiInterfaceRx, [{ accountId, accountNonce, freeBala
   // ""
   const floating = freeBalance.sub(lockedBalance);
   const extraReceived = isVesting ? freeBalance.sub(vestingTotal) : new BN(0);
-  const availableBalance = createType(api.registry, 'Balance', bnMax(new BN(0), isVesting && floating.gt(vestedBalance) ? vestedBalance.add(extraReceived) : floating));
+  const availableBalance = api.registry.createType('Balance', allLocked ? 0 : bnMax(new BN(0), isVesting && floating.gt(vestedBalance) ? vestedBalance.add(extraReceived) : floating));
 
   return {
     accountId,
@@ -81,7 +84,7 @@ function queryOld (api: ApiInterfaceRx, accountId: AccountId): Observable<Result
       if (optVesting.isSome) {
         const { offset: locked, perBlock, startingBlock } = optVesting.unwrap();
 
-        vestingNew = createType(api.registry, 'VestingInfo', { locked, perBlock, startingBlock });
+        vestingNew = api.registry.createType('VestingInfo', { locked, perBlock, startingBlock });
       }
 
       return [vestingNew, locks];
@@ -99,7 +102,7 @@ function queryCurrent (api: ApiInterfaceRx, accountId: AccountId): Observable<Re
       ])
       : api.query.balances.locks(accountId).pipe(
         map((locks): [Vec<BalanceLock>, Option<VestingInfo>] =>
-          [locks, createType(api.registry, 'Option<VestingInfo>')]
+          [locks, api.registry.createType('Option<VestingInfo>')]
         )
       )
   ).pipe(
@@ -136,7 +139,7 @@ export function all (api: ApiInterfaceRx): (address: AccountIndex | AccountId |
               ? queryCurrent(api, account.accountId)
               : queryOld(api, account.accountId)
           ])
-          : of([account, createType(api.registry, 'BlockNumber'), [null, createType(api.registry, 'Vec<BalanceLock>')]])
+          : of([account, api.registry.createType('BlockNumber'), [null, api.registry.createType('Vec<BalanceLock>')]])
         )
       ),
       map((result): DerivedBalancesAll => calcBalances(api, result))

packages/api-derive/src/balances/fees.ts

@@ -8,7 +8,6 @@ import { DerivedFees } from '../types';
 
 import { Observable, of } from 'rxjs';
 import { map } from 'rxjs/operators';
-import { createType } from '@polkadot/types';
 
 import { memo } from '../util';
 
@@ -29,13 +28,13 @@ export function fees (api: ApiInterfaceRx): () => Observable<DerivedFees> {
   return memo((): Observable<DerivedFees> =>
     of([
       // deprecated - remove
-      (api.consts.balances.creationFee as Balance) || createType(api.registry, 'Balance'),
-      (api.consts.balances.transferFee as Balance) || createType(api.registry, 'Balance'),
+      (api.consts.balances?.creationFee as Balance) || api.registry.createType('Balance'),
+      (api.consts.balances?.transferFee as Balance) || api.registry.createType('Balance'),
 
       // current
-      api.consts.balances.existentialDeposit,
-      api.consts.transactionPayment.transactionBaseFee,
-      api.consts.transactionPayment.transactionByteFee
+      (api.consts.balances?.existentialDeposit as Balance) || api.registry.createType('Balance'),
+      (api.consts.transactionPayment?.transactionBaseFee as Balance) || api.registry.createType('Balance'),
+      (api.consts.transactionPayment?.transactionByteFee as Balance) || api.registry.createType('Balance')
     ]).pipe(
       map(([creationFee, transferFee, existentialDeposit, transactionBaseFee, transactionByteFee]): DerivedFees => ({
         creationFee,

packages/api-derive/src/balances/votingBalances.ts

@@ -5,7 +5,7 @@
 import { AccountId, AccountIndex, Address } from '@polkadot/types/interfaces';
 import { DerivedBalancesAccount } from '../types';
 
-import { combineLatest, Observable, of } from 'rxjs';
+import { Observable, combineLatest, of } from 'rxjs';
 import { ApiInterfaceRx } from '@polkadot/api/types';
 
 import { memo } from '../util';
@@ -16,7 +16,8 @@ export function votingBalances (api: ApiInterfaceRx): (addresses?: (AccountId |
       ? of([] as DerivedBalancesAccount[])
       : combineLatest(
         addresses.map((accountId): Observable<DerivedBalancesAccount> =>
-          api.derive.balances.account(accountId))
+          api.derive.balances.account(accountId)
+        )
       )
   );
 }

packages/api-derive/src/chain/bestNumberLag.ts

@@ -7,7 +7,6 @@ import { BlockNumber } from '@polkadot/types/interfaces';
 import { Observable, combineLatest } from 'rxjs';
 import { map } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { createType } from '@polkadot/types';
 
 import { memo } from '../util';
 
@@ -31,7 +30,7 @@ export function bestNumberLag (api: ApiInterfaceRx): () => Observable<BlockNumbe
       api.derive.chain.bestNumberFinalized()
     ]).pipe(
       map(([bestNumber, bestNumberFinalized]): BlockNumber =>
-        createType(api.registry, 'BlockNumber', bestNumber.sub(bestNumberFinalized))
+        api.registry.createType('BlockNumber', bestNumber.sub(bestNumberFinalized))
       )
     ));
 }

packages/api-derive/src/collective/proposals.ts

@@ -6,7 +6,7 @@ import { ApiInterfaceRx } from '@polkadot/api/types';
 import { Hash, Proposal, Votes } from '@polkadot/types/interfaces';
 import { DerivedCollectiveProposal } from '../types';
 
-import { combineLatest, Observable, of } from 'rxjs';
+import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { Option } from '@polkadot/types';
 
@@ -28,7 +28,7 @@ function parse ([hashes, proposals, votes]: Result): DerivedCollectiveProposal[]
 
 export function proposals (api: ApiInterfaceRx, section: 'council' | 'technicalCommittee'): () => Observable<DerivedCollectiveProposal[]> {
   return (): Observable<DerivedCollectiveProposal[]> =>
-    api.query[section]
+    api.query[section]?.proposals
       ? api.query[section].proposals().pipe(
         switchMap((hashes: Hash[]): Observable<Result> =>
           hashes.length

packages/api-derive/src/contracts/fees.ts

@@ -8,7 +8,6 @@ import { DerivedContractFees } from '../types';
 import BN from 'bn.js';
 import { Observable, of } from 'rxjs';
 import { map } from 'rxjs/operators';
-import { createType } from '@polkadot/types';
 
 import { memo } from '../util';
 
@@ -18,8 +17,8 @@ type ResultV2 = [BN, BN, BN, BN, BN, BN, BN, BN, BN];
 function queryConstants (api: ApiInterfaceRx): Observable<ResultV2> {
   return of([
     // deprecated
-    api.consts.contracts.creationFee || createType(api.registry, 'Balance'),
-    api.consts.contracts.transferFee || createType(api.registry, 'Balance'),
+    api.consts.contracts.creationFee || api.registry.createType('Balance'),
+    api.consts.contracts.transferFee || api.registry.createType('Balance'),
 
     // current
     api.consts.contracts.callBaseFee,

packages/api-derive/src/democracy/proposals.ts

@@ -9,7 +9,7 @@ import { DeriveProposal } from '../types';
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { Bytes, Option, Vec, createType } from '@polkadot/types';
+import { Bytes, Option, Vec } from '@polkadot/types';
 
 import { memo } from '../util';
 
@@ -37,7 +37,7 @@ function parse (api: ApiInterfaceRx, { depositors, proposals, preimages }: Resul
       // we could end up in a situation where the proposal is non-decodable, e.g. after an upgrade
       if (preimage) {
         try {
-          proposal = createType(api.registry, 'Proposal', preimage[0].toU8a(true));
+          proposal = api.registry.createType('Proposal', preimage[0].toU8a(true));
         } catch (error) {
           console.error(error);
         }
@@ -63,7 +63,7 @@ function parse (api: ApiInterfaceRx, { depositors, proposals, preimages }: Resul
 
 export function proposals (api: ApiInterfaceRx): () => Observable<DeriveProposal[]> {
   return memo((): Observable<DeriveProposal[]> =>
-    api.query.democracy?.publicProps
+    api.query.democracy?.publicProps && api.query.democracy?.preimages
       ? api.query.democracy.publicProps<Proposals>().pipe(
         switchMap((proposals) =>
           combineLatest([

packages/api-derive/src/democracy/referendumInfo.ts

@@ -10,7 +10,7 @@ import BN from 'bn.js';
 import { Observable, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { Option, createType } from '@polkadot/types';
+import { Option } from '@polkadot/types';
 
 import { memo } from '../util';
 
@@ -25,11 +25,11 @@ function constructInfo (api: ApiInterfaceRx, index: BN | number, _info: Option<R
   }
 
   return {
-    index: createType(api.registry, 'PropIndex', index),
+    index: api.registry.createType('PropIndex', index),
     info,
     hash: info.proposalHash,
     proposal: preImage
-      ? createType(api.registry, 'Proposal', preImage[0].toU8a(true))
+      ? api.registry.createType('Proposal', preImage[0].toU8a(true))
       : undefined,
     preimage: preImage
       ? {

packages/api-derive/src/democracy/referendumInfos.ts

@@ -23,7 +23,7 @@ export function referendumInfos (api: ApiInterfaceRx): (ids?: (BN | number)[]) =
     ).pipe(
       switchMap((infos): Observable<(DerivedReferendum | null)[]> =>
         combineLatest(
-          ...ids.map((id, index): Observable<DerivedReferendum | null> =>
+          ids.map((id, index): Observable<DerivedReferendum | null> =>
             retrieveInfo(api, id, infos[index])
           )
         )

packages/api-derive/src/democracy/referendumVotesFor.ts

@@ -5,10 +5,10 @@
 import { AccountId, Vote } from '@polkadot/types/interfaces';
 
 import BN from 'bn.js';
-import { combineLatest, Observable, of } from 'rxjs';
+import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { Vec, createType } from '@polkadot/types';
+import { Vec } from '@polkadot/types';
 
 import { DerivedBalancesAccount, DerivedReferendumVote } from '../types';
 import { memo } from '../util';
@@ -26,8 +26,8 @@ export function referendumVotesFor (api: ApiInterfaceRx): (referendumId: BN | nu
       map(([votersFor, votes, balances]): DerivedReferendumVote[] =>
         votersFor.map((accountId, index): DerivedReferendumVote => ({
           accountId,
-          balance: balances[index].votingBalance || createType(api.registry, 'Balance'),
-          vote: votes[index] || createType(api.registry, 'Vote')
+          balance: balances[index].votingBalance || api.registry.createType('Balance'),
+          vote: votes[index] || api.registry.createType('Vote')
         } as DerivedReferendumVote))
       )
     ));

packages/api-derive/src/elections/info.ts

@@ -8,7 +8,7 @@ import { ITuple } from '@polkadot/types/types';
 import { Observable } from 'rxjs';
 import { map } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { createType, Vec, u32 } from '@polkadot/types';
+import { Vec, u32 } from '@polkadot/types';
 
 import { DerivedElectionsInfo } from '../types';
 import { memo } from '../util';
@@ -28,12 +28,12 @@ function queryElections (api: ApiInterfaceRx): Observable<DerivedElectionsInfo>
   ]).pipe(
     map(([councilMembers, candidates, members, runnersUp]): DerivedElectionsInfo => ({
       candidates,
-      candidateCount: createType(api.registry, 'u32', candidates.length),
+      candidateCount: api.registry.createType('u32', candidates.length),
       candidacyBond: api.consts[section].candidacyBond as Balance,
       desiredSeats: api.consts[section].desiredMembers as u32,
       members: members.length
         ? members.sort(sortAccounts)
-        : councilMembers.map((accountId): [AccountId, Balance] => [accountId, createType(api.registry, 'Balance')]),
+        : councilMembers.map((accountId): [AccountId, Balance] => [accountId, api.registry.createType('Balance')]),
       runnersUp: runnersUp.sort(sortAccounts),
       termDuration: api.consts[section].termDuration as BlockNumber,
       votingBond: api.consts[section].votingBond as Balance

packages/api-derive/src/imOnline/receivedHeartbeats.ts

@@ -5,7 +5,7 @@
 import { AccountId } from '@polkadot/types/interfaces';
 import { DerivedHeartbeats } from '../types';
 
-import { of, Observable, combineLatest } from 'rxjs';
+import { Observable, of, combineLatest } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
 import { Bytes, Option, u32 } from '@polkadot/types';

packages/api-derive/src/index.spec.ts

@@ -61,7 +61,7 @@ describe('derive', (): void => {
     testFunction(api)('session', 'sessionProgress', []);
 
     testFunction(api)('staking', 'account', []);
-    testFunction(api)('staking', 'controllers', []);
+    testFunction(api)('staking', 'stashes', []);
   });
 
   describe('custom', (): void => {

packages/api-derive/src/parachains/info.ts

@@ -4,12 +4,11 @@
 
 import { CollatorId, ParaId } from '@polkadot/types/interfaces';
 import { DeriveParachainInfo, DeriveParachainFull, DeriveParachainActive } from '../types';
-import { Active, DidUpdate, Heads, ParaInfoResult, PendingSwap, RelayDispatchQueue, RetryQueue, SelectedThreads, Watermarks } from './types';
+import { Active, DidUpdate, Heads, ParaInfoResult, PendingSwap, RelayDispatchQueue, RetryQueue, SelectedThreads } from './types';
 
 import { Observable, of } from 'rxjs';
 import { map } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { createType } from '@polkadot/types';
 
 import { memo } from '../util';
 
@@ -21,8 +20,7 @@ type Result = [
   ParaInfoResult,
   PendingSwap,
   Heads,
-  RelayDispatchQueue,
-  Watermarks
+  RelayDispatchQueue
 ];
 
 function parseActive (id: ParaId, active: Active): DeriveParachainActive | null {
@@ -58,7 +56,7 @@ function parseCollators (id: ParaId, collatorQueue: SelectedThreads | RetryQueue
     });
 }
 
-function parse (id: ParaId, [active, retryQueue, selectedThreads, didUpdate, info, pendingSwap, heads, relayDispatchQueue, watermarks]: Result): DeriveParachainFull | null {
+function parse (id: ParaId, [active, retryQueue, selectedThreads, didUpdate, info, pendingSwap, heads, relayDispatchQueue]: Result): DeriveParachainFull | null {
   if (info.isNone) {
     return null;
   }
@@ -74,8 +72,7 @@ function parse (id: ParaId, [active, retryQueue, selectedThreads, didUpdate, inf
     pendingSwapId: pendingSwap.unwrapOr(null),
     relayDispatchQueue,
     retryCollators: parseCollators(id, retryQueue),
-    selectedCollators: parseCollators(id, selectedThreads),
-    watermark: watermarks.unwrapOr(null)
+    selectedCollators: parseCollators(id, selectedThreads)
   };
 }
 
@@ -90,12 +87,11 @@ export function info (api: ApiInterfaceRx): (id: ParaId | number) => Observable<
         [api.query.registrar.paras, id],
         [api.query.registrar.pendingSwap, id],
         [api.query.parachains.heads, id],
-        [api.query.parachains.relayDispatchQueue, id],
-        [api.query.parachains.watermarks, id]
+        [api.query.parachains.relayDispatchQueue, id]
       ])
         .pipe(
           map((result: Result): DeriveParachainFull | null =>
-            parse(createType(api.registry, 'ParaId', id), result)
+            parse(api.registry.createType('ParaId', id), result)
           )
         )
       : of(null)

packages/api-derive/src/parachains/overview.ts

@@ -4,7 +4,7 @@
 
 import { ParaId } from '@polkadot/types/interfaces';
 import { DeriveParachain, DeriveParachainInfo } from '../types';
-import { DidUpdate, ParaInfoResult, PendingSwap, RelayDispatchQueueSize, Watermarks } from './types';
+import { DidUpdate, ParaInfoResult, PendingSwap, RelayDispatchQueueSize } from './types';
 
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
@@ -17,11 +17,10 @@ type Result = [
   DidUpdate,
   ParaInfoResult[],
   PendingSwap[],
-  RelayDispatchQueueSize[],
-  Watermarks[],
+  RelayDispatchQueueSize[]
 ];
 
-function parse ([ids, didUpdate, infos, pendingSwaps, relayDispatchQueueSizes, watermarks]: Result): DeriveParachain[] {
+function parse ([ids, didUpdate, infos, pendingSwaps, relayDispatchQueueSizes]: Result): DeriveParachain[] {
   return ids.map((id, index): DeriveParachain => {
     return {
       didUpdate: didUpdate.isSome
@@ -30,8 +29,7 @@ function parse ([ids, didUpdate, infos, pendingSwaps, relayDispatchQueueSizes, w
       id,
       info: { id, ...infos[index].unwrapOr(null) } as DeriveParachainInfo,
       pendingSwapId: pendingSwaps[index].unwrapOr(null),
-      relayDispatchQueueSize: relayDispatchQueueSizes[index][0].toNumber(),
-      watermark: watermarks[index].unwrapOr(null)
+      relayDispatchQueueSize: relayDispatchQueueSizes[index][0].toNumber()
     };
   });
 }
@@ -46,8 +44,7 @@ export function overview (api: ApiInterfaceRx): () => Observable<DeriveParachain
             api.query.parachains.didUpdate<DidUpdate>(),
             api.query.registrar.paras.multi<ParaInfoResult>(paraIds),
             api.query.registrar.pendingSwap.multi<PendingSwap>(paraIds),
-            api.query.parachains.relayDispatchQueueSize.multi<RelayDispatchQueueSize>(paraIds),
-            api.query.parachains.watermarks.multi<Watermarks>(paraIds)
+            api.query.parachains.relayDispatchQueueSize.multi<RelayDispatchQueueSize>(paraIds)
           ])
         ),
         map((result: Result): DeriveParachain[] =>

packages/api-derive/src/parachains/types.ts

@@ -2,7 +2,7 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
-import { BlockNumber, CollatorId, ParaId, ParaInfo, Retriable, UpwardMessage } from '@polkadot/types/interfaces';
+import { CollatorId, ParaId, ParaInfo, Retriable, UpwardMessage } from '@polkadot/types/interfaces';
 import { ITuple } from '@polkadot/types/types';
 
 import { Bytes, Option, u32, Vec } from '@polkadot/types';
@@ -16,7 +16,6 @@ export type Code = Bytes;
 export type Heads = Bytes;
 export type RelayDispatchQueue = Vec<UpwardMessage>
 export type RelayDispatchQueueSize = ITuple<[u32, u32]>;
-export type Watermarks = Option<BlockNumber>;
 export type DidUpdate = Option<Vec<ParaId>>;
 
 export interface DeriveParachainActive {
@@ -38,7 +37,6 @@ export interface DeriveParachain {
   id: ParaId;
   info: DeriveParachainInfo | null;
   relayDispatchQueueSize?: number;
-  watermark: BlockNumber | null;
 }
 
 export interface DeriveParachainFull extends DeriveParachain {

packages/api-derive/src/session/indexes.ts

@@ -8,7 +8,7 @@ import { DeriveSessionIndexes } from '../types';
 
 import { Observable, of } from 'rxjs';
 import { map } from 'rxjs/operators';
-import { createType, Option, u32 } from '@polkadot/types';
+import { Option, u32 } from '@polkadot/types';
 
 import { memo } from '../util';
 
@@ -34,7 +34,7 @@ function queryNoActive (api: ApiInterfaceRx): Observable<Result> {
   ]).pipe(
     map(([currentEra, currentIndex, validatorCount]): Result => [
       currentEra,
-      createType(api.registry, 'Option<Moment>'),
+      api.registry.createType('Option<Moment>'),
       currentEra,
       currentIndex,
       validatorCount
@@ -56,7 +56,7 @@ function query (api: ApiInterfaceRx): Observable<Result> {
       return [
         activeEra,
         activeEraStart,
-        currentEra.unwrapOr(createType(api.registry, 'EraIndex')),
+        currentEra.unwrapOr(api.registry.createType('EraIndex')),
         currentIndex,
         validatorCount
       ];
@@ -67,11 +67,11 @@ function query (api: ApiInterfaceRx): Observable<Result> {
 // empty set when none is available
 function empty (api: ApiInterfaceRx): Observable<Result> {
   return of([
-    createType(api.registry, 'EraIndex'),
-    createType(api.registry, 'Option<Moment>'),
-    createType(api.registry, 'EraIndex'),
-    createType(api.registry, 'SessionIndex', 1),
-    createType(api.registry, 'u32')
+    api.registry.createType('EraIndex'),
+    api.registry.createType('Option<Moment>'),
+    api.registry.createType('EraIndex'),
+    api.registry.createType('SessionIndex', 1),
+    api.registry.createType('u32')
   ]);
 }
 

packages/api-derive/src/session/info.ts

@@ -8,7 +8,7 @@ import { DerivedSessionInfo, DeriveSessionIndexes } from '../types';
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { createType, Option, u64 } from '@polkadot/types';
+import { Option, u64 } from '@polkadot/types';
 
 import { memo } from '../util';
 
@@ -27,12 +27,12 @@ function createDerived (api: ApiInterfaceRx, [[hasBabe, epochDuration, sessionsP
     activeEraStart,
     currentEra,
     currentIndex,
-    eraLength: createType(api.registry, 'BlockNumber', sessionsPerEra.mul(epochDuration)),
-    eraProgress: createType(api.registry, 'BlockNumber', eraProgress),
+    eraLength: api.registry.createType('BlockNumber', sessionsPerEra.mul(epochDuration)),
+    eraProgress: api.registry.createType('BlockNumber', eraProgress),
     isEpoch: hasBabe,
     sessionLength: epochDuration,
     sessionsPerEra,
-    sessionProgress: createType(api.registry, 'BlockNumber', sessionProgress),
+    sessionProgress: api.registry.createType('BlockNumber', sessionProgress),
     validatorCount
   };
 }
@@ -43,15 +43,15 @@ function queryAura (api: ApiInterfaceRx): Observable<DerivedSessionInfo> {
       createDerived(api, [
         [
           false,
-          createType(api.registry, 'u64', 1),
-          api.consts.staking?.sessionsPerEra || createType(api.registry, 'SessionIndex', 1)
+          api.registry.createType('u64', 1),
+          api.consts.staking?.sessionsPerEra || api.registry.createType('SessionIndex', 1)
         ],
         indexes,
         [
-          createType(api.registry, 'u64', 1),
-          createType(api.registry, 'u64', 1),
-          createType(api.registry, 'u64', 1),
-          createType(api.registry, 'SessionIndex', 1)
+          api.registry.createType('u64', 1),
+          api.registry.createType('u64', 1),
+          api.registry.createType('u64', 1),
+          api.registry.createType('SessionIndex', 1)
         ]
       ])
     )
@@ -72,7 +72,7 @@ function queryBabe (api: ApiInterfaceRx): Observable<[DeriveSessionIndexes, Resu
       ])
     ),
     map(([indexes, [currentSlot, epochIndex, genesisSlot, optStartIndex]]): [DeriveSessionIndexes, ResultSlotsFlat] => [
-      indexes, [currentSlot, epochIndex, genesisSlot, optStartIndex.unwrapOr(createType(api.registry, 'SessionIndex', 1))]
+      indexes, [currentSlot, epochIndex, genesisSlot, optStartIndex.unwrapOr(api.registry.createType('SessionIndex', 1))]
     ])
   );
 }

packages/api-derive/src/society/candidates.ts

@@ -7,7 +7,7 @@ import { AccountId, BalanceOf, Bid, BidKind } from '@polkadot/types/interfaces';
 import { ITuple } from '@polkadot/types/types';
 import { DeriveSocietyCandidate } from '../types';
 
-import { combineLatest, of, Observable } from 'rxjs';
+import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { Option, Vec } from '@polkadot/types';
 

packages/api-derive/src/society/member.ts

@@ -27,12 +27,12 @@ export function member (api: ApiInterfaceRx): (accountId: AccountId) => Observab
       [api.query.society.suspendedMembers, accountId],
       [api.query.society.vouching, accountId]
     ]).pipe(
-      map(([payouts, strikes, defenderVote, suspended, vouching]: Result): DeriveSocietyMember => ({
+      map(([payouts, strikes, defenderVotes, suspended, vouching]: Result): DeriveSocietyMember => ({
         accountId,
         payouts,
         isSuspended: suspended.isTrue,
         strikes,
-        vote: defenderVote.unwrapOr(undefined),
+        vote: defenderVotes.unwrapOr(undefined),
         vouching: vouching.unwrapOr(undefined)
       }))
     )

packages/api-derive/src/society/members.ts

@@ -6,7 +6,7 @@ import { ApiInterfaceRx } from '@polkadot/api/types';
 import { AccountId } from '@polkadot/types/interfaces';
 import { DeriveSocietyMember } from '../types';
 
-import { combineLatest, Observable } from 'rxjs';
+import { Observable, combineLatest } from 'rxjs';
 import { switchMap } from 'rxjs/operators';
 
 import { memo } from '../util';
@@ -18,9 +18,11 @@ export function members (api: ApiInterfaceRx): () => Observable<DeriveSocietyMem
   return memo((): Observable<DeriveSocietyMember[]> =>
     api.query.society.members<AccountId[]>().pipe(
       switchMap((members): Observable<DeriveSocietyMember[]> =>
-        combineLatest(members.map((accountId): Observable<DeriveSocietyMember> =>
-          api.derive.society.member(accountId)
-        ))
+        combineLatest(
+          members.map((accountId): Observable<DeriveSocietyMember> =>
+            api.derive.society.member(accountId)
+          )
+        )
       )
     )
   );

packages/api-derive/src/staking/account.ts

@@ -7,9 +7,8 @@ import { Balance, BlockNumber, StakingLedger, UnlockChunk } from '@polkadot/type
 import { DerivedSessionInfo, DerivedStakingAccount, DerivedStakingQuery, DerivedUnlocking } from '../types';
 
 import BN from 'bn.js';
-import { combineLatest, Observable } from 'rxjs';
+import { Observable, combineLatest } from 'rxjs';
 import { map } from 'rxjs/operators';
-import { createType } from '@polkadot/types';
 
 import { isUndefined } from '@polkadot/util';
 
@@ -33,7 +32,7 @@ function remainingBlocks (api: ApiInterfaceRx, era: BN, sessionInfo: DerivedSess
   const remaining = era.sub(sessionInfo.currentEra);
 
   // on the Rust side the current-era >= era-for-unlock (removal done on >)
-  return createType(api.registry, 'BlockNumber', remaining.gtn(0)
+  return api.registry.createType('BlockNumber', remaining.gtn(0)
     ? remaining
       .subn(1)
       .mul(sessionInfo.eraLength)
@@ -58,7 +57,7 @@ function calculateUnlocking (api: ApiInterfaceRx, stakingLedger: StakingLedger |
   // group the unlock chunks that have the same era and sum their values
   const groupedResult = groupByEra(unlockingChunks);
   const results = Object.entries(groupedResult).map(([eraString, value]): DerivedUnlocking => ({
-    value: createType(api.registry, 'Balance', value),
+    value: api.registry.createType('Balance', value),
     remainingBlocks: remainingBlocks(api, new BN(eraString), sessionInfo)
   }));
 
@@ -67,10 +66,10 @@ function calculateUnlocking (api: ApiInterfaceRx, stakingLedger: StakingLedger |
 
 function redeemableSum (api: ApiInterfaceRx, stakingLedger: StakingLedger | undefined, sessionInfo: DerivedSessionInfo): Balance {
   if (isUndefined(stakingLedger)) {
-    return createType(api.registry, 'Balance');
+    return api.registry.createType('Balance');
   }
 
-  return createType(api.registry, 'Balance', stakingLedger.unlocking.reduce((total, { era, value }): BN => {
+  return api.registry.createType('Balance', stakingLedger.unlocking.reduce((total, { era, value }): BN => {
     return remainingBlocks(api, era.unwrap(), sessionInfo).eqn(0)
       ? total.add(value.unwrap())
       : total;

packages/api-derive/src/staking/controllers.ts

@@ -1,33 +0,0 @@
-// Copyright 2017-2020 @polkadot/api-derive authors & contributors
-// This software may be modified and distributed under the terms
-// of the Apache-2.0 license. See the LICENSE file for details.
-
-import { ApiInterfaceRx } from '@polkadot/api/types';
-import { AccountId, ValidatorPrefs } from '@polkadot/types/interfaces';
-import { ITuple } from '@polkadot/types/types';
-
-import { Observable, combineLatest, of } from 'rxjs';
-import { switchMap } from 'rxjs/operators';
-import { Option, Vec } from '@polkadot/types';
-
-import { memo } from '../util';
-
-type DeriveControllers = [AccountId[], Option<AccountId>[]];
-
-/**
- * @description From the list of stash accounts, retrieve the list of controllers
- */
-export function controllers (api: ApiInterfaceRx): () => Observable<DeriveControllers> {
-  return memo((): Observable<[AccountId[], Option<AccountId>[]]> =>
-    api.query.staking.validators<ITuple<[Vec<AccountId>, Vec<ValidatorPrefs>]>>().pipe(
-      switchMap(([stashIds]): Observable<DeriveControllers> =>
-        combineLatest([
-          of(stashIds),
-          // for V2, don't return all the controllers, we call bonded at a later point
-          api.consts.session
-            ? of([])
-            : api.query.staking.bonded.multi<Option<AccountId>>(stashIds)
-        ])
-      )
-    ));
-}

packages/api-derive/src/staking/electedInfo.ts

@@ -17,9 +17,11 @@ export function electedInfo (api: ApiInterfaceRx): () => Observable<DerivedStaki
       switchMap(({ nextElected }): Observable<[AccountId[], DerivedStakingQuery[]]> =>
         combineLatest([
           of(nextElected),
-          combineLatest(nextElected.map((accountId): Observable<DerivedStakingQuery> =>
-            api.derive.staking.query(accountId)
-          ))
+          combineLatest(
+            nextElected.map((accountId): Observable<DerivedStakingQuery> =>
+              api.derive.staking.query(accountId)
+            )
+          )
         ])
       ),
       map(([nextElected, info]): DerivedStakingElected => ({

packages/api-derive/src/staking/erasExposure.ts

@@ -4,52 +4,54 @@
 
 import { ApiInterfaceRx } from '@polkadot/api/types';
 import { EraIndex, Exposure } from '@polkadot/types/interfaces';
-import { DeriveEraExposure } from '../types';
+import { DeriveEraExposure, DeriveEraNominatorExposure, DeriveEraValidatorExposure } from '../types';
 
+import BN from 'bn.js';
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
+import { StorageKey } from '@polkadot/types';
 
 import { memo } from '../util';
 
-function queryClipped (api: ApiInterfaceRx): Observable<[[EraIndex, string][], Exposure[]]> {
-  return api.derive.staking.erasPoints().pipe(
-    switchMap((allPoints): Observable<[[EraIndex, string][], Exposure[]]> => {
-      const indexes: [EraIndex, string][] = [];
+type KeysAndExposures = [StorageKey, Exposure][];
 
-      allPoints.forEach(({ all, era }): void => {
-        Object.keys(all).forEach((validatorId): void => {
-          indexes.push([era, validatorId]);
-        });
-      });
+function mapStakers (era: EraIndex, stakers: KeysAndExposures): DeriveEraExposure {
+  const nominators: DeriveEraNominatorExposure = {};
+  const validators: DeriveEraValidatorExposure = {};
 
-      // FIXME use prefix queries, get going first
-      return combineLatest([
-        of(indexes),
-        indexes.length
-          ? api.query.staking.erasStakersClipped.multi<Exposure>(indexes)
-          : of([] as Exposure[])
-      ]);
-    })
+  stakers.forEach(([key, exposure]): void => {
+    const validatorId = key.args[1].toString();
+
+    validators[validatorId] = exposure;
+
+    exposure.others.forEach(({ who }, index): void => {
+      const nominatorId = who.toString();
+
+      nominators[nominatorId] = nominators[nominatorId] || [];
+      nominators[nominatorId].push([validatorId, index]);
+    });
+  });
+
+  return { era, nominators, validators };
+}
+
+export function eraExposure (api: ApiInterfaceRx): (era: EraIndex) => Observable<DeriveEraExposure> {
+  return memo((era: EraIndex): Observable<DeriveEraExposure> =>
+    api.query.staking.erasStakersClipped.entries(era).pipe(
+      map((stakers) => mapStakers(era, stakers))
+    )
   );
 }
 
-export function erasExposure (api: ApiInterfaceRx): () => Observable<DeriveEraExposure[]> {
-  return memo((): Observable<DeriveEraExposure[]> =>
-    queryClipped(api).pipe(
-      map(([indexes, exposures]): DeriveEraExposure[] =>
-        indexes.reduce((result: DeriveEraExposure[], [era, validatorId], index): DeriveEraExposure[] => {
-          let entry = result.find((entry): boolean => entry.era.eq(era));
-
-          if (!entry) {
-            entry = { all: {}, era };
-
-            result.push(entry);
-          }
-
-          entry.all[validatorId] = exposures[index];
-
-          return result;
-        }, [])
+export function erasExposure (api: ApiInterfaceRx): (withActive?: boolean | BN | number) => Observable<DeriveEraExposure[]> {
+  return memo((withActive?: boolean | BN | number): Observable<DeriveEraExposure[]> =>
+    api.derive.staking.erasHistoric(withActive).pipe(
+      switchMap((eras): Observable<DeriveEraExposure[]> =>
+        eras.length
+          ? combineLatest(
+            eras.map((era) => api.derive.staking.eraExposure(era))
+          )
+          : of([])
       )
     )
   );

packages/api-derive/src/staking/erasHistoric.ts

@@ -0,0 +1,55 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { ActiveEraInfo, EraIndex } from '@polkadot/types/interfaces';
+
+import BN from 'bn.js';
+import { Observable, of } from 'rxjs';
+import { map } from 'rxjs/operators';
+import { Option, u32 } from '@polkadot/types';
+import { isBoolean, isUndefined, bnToBn } from '@polkadot/util';
+
+import { memo } from '../util';
+
+export function erasHistoric (api: ApiInterfaceRx): (withActive?: boolean | BN | number) => Observable<EraIndex[]> {
+  return memo((withActive?: boolean | BN | number): Observable<EraIndex[]> =>
+    api.query.staking?.activeEra
+      ? api.queryMulti<[Option<ActiveEraInfo>, u32]>([
+        api.query.staking.activeEra,
+        api.query.staking.historyDepth
+      ]).pipe(
+        map(([activeEraOpt, historyDepth]): EraIndex[] => {
+          const result: (EraIndex | null)[] = [];
+          const max = historyDepth.toNumber();
+          const activeEra = activeEraOpt.unwrapOrDefault().index.toBn();
+          let lastEra = activeEra;
+
+          while (lastEra.gten(0) && (result.length < max)) {
+            result.push(
+              ((lastEra !== activeEra) || (withActive === true))
+                ? api.registry.createType('EraIndex', lastEra)
+                : null
+            );
+
+            lastEra = lastEra.subn(1);
+          }
+
+          const startEra = isUndefined(withActive) || isBoolean(withActive)
+            ? new BN(0)
+            : bnToBn(withActive);
+
+          // go from oldest to newest
+          return result
+            .filter((era): era is EraIndex =>
+              era
+                ? era.gte(startEra)
+                : false
+            )
+            .reverse();
+        })
+      )
+      : of([])
+  );
+}

packages/api-derive/src/staking/erasPoints.ts

@@ -3,61 +3,41 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { ActiveEraInfo, EraIndex, EraRewardPoints, RewardPoint } from '@polkadot/types/interfaces';
-import { DeriveEraPointsAll } from '../types';
+import { EraIndex, EraRewardPoints } from '@polkadot/types/interfaces';
+import { DeriveEraPoints, DeriveEraValPoints } from '../types';
 
+import BN from 'bn.js';
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
-import { Option, u32 } from '@polkadot/types';
 
 import { memo } from '../util';
 
-function getAvailableIndexes (api: ApiInterfaceRx): Observable<EraIndex[]> {
-  return api.query.staking?.activeEra
-    ? api.queryMulti<[Option<ActiveEraInfo>, u32]>([
-      api.query.staking.activeEra,
-      api.query.staking.historyDepth
-    ]).pipe(
-      map(([activeEraOpt, historyDepth]): EraIndex[] => {
-        const result: EraIndex[] = [];
-        const max = historyDepth.toNumber();
-        let lastEra = activeEraOpt.unwrapOrDefault().index.subn(1);
+function mapValidators ({ individual }: EraRewardPoints): DeriveEraValPoints {
+  return [...individual.entries()]
+    .filter(([, points]): boolean => points.gtn(0))
+    .reduce((result: DeriveEraValPoints, [validatorId, points]): DeriveEraValPoints => {
+      result[validatorId.toString()] = points;
 
-        while (lastEra.gten(0) && result.length < max) {
-          result.push(api.registry.createType('EraIndex', lastEra));
-
-          lastEra = lastEra.subn(1);
-        }
-
-        // go from oldest to newest
-        return result.reverse();
-      })
-    )
-    : of([]);
+      return result;
+    }, {});
 }
 
-export function erasPoints (api: ApiInterfaceRx): () => Observable<DeriveEraPointsAll[]> {
-  return memo((): Observable<DeriveEraPointsAll[]> =>
-    getAvailableIndexes(api).pipe(
-      switchMap((indexes): Observable<[EraIndex[], EraRewardPoints[]]> =>
+export function erasPoints (api: ApiInterfaceRx): (withActive?: boolean | BN | number) => Observable<DeriveEraPoints[]> {
+  return memo((withActive?: boolean | number): Observable<DeriveEraPoints[]> =>
+    api.derive.staking.erasHistoric(withActive).pipe(
+      switchMap((eras): Observable<[EraIndex[], EraRewardPoints[]]> =>
         combineLatest([
-          of(indexes),
-          indexes.length
-            ? api.query.staking.erasRewardPoints.multi<EraRewardPoints>(indexes)
+          of(eras),
+          eras.length
+            ? api.query.staking.erasRewardPoints.multi<EraRewardPoints>(eras)
             : of([])
         ])
       ),
-      map(([eras, rewards]): DeriveEraPointsAll[] =>
-        eras.map((era, index): DeriveEraPointsAll => ({
-          all: [...rewards[index].individual.entries()]
-            .filter(([, points]): boolean => points.gtn(0))
-            .reduce((all: Record<string, RewardPoint>, [validatorId, points]): Record<string, RewardPoint> => {
-              all[validatorId.toString()] = points;
-
-              return all;
-            }, {}),
+      map(([eras, points]): DeriveEraPoints[] =>
+        eras.map((era, index): DeriveEraPoints => ({
           era,
-          total: rewards[index].total
+          eraPoints: points[index].total,
+          validators: mapValidators(points[index])
         }))
       )
     )

packages/api-derive/src/staking/erasPrefs.ts

@@ -0,0 +1,46 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { EraIndex, ValidatorPrefs } from '@polkadot/types/interfaces';
+import { DeriveEraPrefs, DeriveEraValPrefs } from '../types';
+
+import BN from 'bn.js';
+import { Observable, combineLatest, of } from 'rxjs';
+import { map, switchMap } from 'rxjs/operators';
+import { StorageKey } from '@polkadot/types';
+
+import { memo } from '../util';
+
+function mapPrefs (era: EraIndex, all: [StorageKey, ValidatorPrefs][]): DeriveEraPrefs {
+  const validators: DeriveEraValPrefs = {};
+
+  all.forEach(([key, prefs]): void => {
+    validators[key.args[1].toString()] = prefs;
+  });
+
+  return { era, validators };
+}
+
+export function eraPrefs (api: ApiInterfaceRx): (era: EraIndex) => Observable<DeriveEraPrefs> {
+  return memo((era: EraIndex): Observable<DeriveEraPrefs> =>
+    api.query.staking.erasValidatorPrefs.entries(era).pipe(
+      map((prefs) => mapPrefs(era, prefs))
+    )
+  );
+}
+
+export function erasPrefs (api: ApiInterfaceRx): (withActive?: boolean | BN | number) => Observable<DeriveEraPrefs[]> {
+  return memo((withActive?: boolean | BN | number): Observable<DeriveEraPrefs[]> =>
+    api.derive.staking.erasHistoric(withActive).pipe(
+      switchMap((eras): Observable<DeriveEraPrefs[]> =>
+        eras.length
+          ? combineLatest(
+            eras.map((era) => api.derive.staking.eraPrefs(era))
+          )
+          : of([])
+      )
+    )
+  );
+}

packages/api-derive/src/staking/erasRewards.ts

@@ -3,33 +3,32 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { DeriveEraRewardsAll } from '../types';
+import { Balance, EraIndex } from '@polkadot/types/interfaces';
+import { DeriveEraRewards } from '../types';
 
-import { Observable } from 'rxjs';
-import { map } from 'rxjs/operators';
+import BN from 'bn.js';
+import { Observable, combineLatest, of } from 'rxjs';
+import { map, switchMap } from 'rxjs/operators';
+import { Option } from '@polkadot/types';
 
 import { memo } from '../util';
 
-export function erasRewards (api: ApiInterfaceRx): () => Observable<DeriveEraRewardsAll[]> {
-  return memo((): Observable<DeriveEraRewardsAll[]> =>
-    api.derive.staking.erasExposure().pipe(
-      map((exposures): DeriveEraRewardsAll[] =>
-        exposures.map(({ era, all }): DeriveEraRewardsAll =>
-          Object
-            .entries(all)
-            .reduce((rewards: DeriveEraRewardsAll, [validatorId, exposure]): DeriveEraRewardsAll => {
-              rewards.validators[validatorId] = exposure;
-
-              exposure.others.forEach(({ who }, index): void => {
-                const nominatorId = who.toString();
-
-                rewards.nominators[nominatorId] = rewards.nominators[nominatorId] || [];
-                rewards.nominators[nominatorId].push([validatorId, index]);
-              });
-
-              return rewards;
-            }, { era, nominators: {}, validators: {} })
-        )
+export function erasRewards (api: ApiInterfaceRx): (withActive?: boolean | BN | number) => Observable<DeriveEraRewards[]> {
+  return memo((withActive?: boolean | BN | number): Observable<DeriveEraRewards[]> =>
+    api.derive.staking.erasHistoric(withActive).pipe(
+      switchMap((eras): Observable<[EraIndex[], Option<Balance>[]]> =>
+        combineLatest([
+          of(eras),
+          eras.length
+            ? api.query.staking.erasValidatorReward.multi<Option<Balance>>(eras)
+            : of([])
+        ])
+      ),
+      map(([eras, rewards]): DeriveEraRewards[] =>
+        eras.map((era, index): DeriveEraRewards => ({
+          era,
+          eraReward: rewards[index].unwrapOrDefault()
+        }))
       )
     )
   );

packages/api-derive/src/staking/erasSlashes.ts

@@ -0,0 +1,55 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { BalanceOf, EraIndex, Perbill } from '@polkadot/types/interfaces';
+import { ITuple } from '@polkadot/types/types';
+import { DeriveEraSlashes, DeriveEraValSlash } from '../types';
+
+import BN from 'bn.js';
+import { Observable, combineLatest, of } from 'rxjs';
+import { map, switchMap } from 'rxjs/operators';
+import { Option, StorageKey } from '@polkadot/types';
+
+import { memo } from '../util';
+
+function mapSlashes (era: EraIndex, noms: [StorageKey, Option<BalanceOf>][], vals: [StorageKey, Option<ITuple<[Perbill, BalanceOf]>>][]): DeriveEraSlashes {
+  const nominators: DeriveEraValSlash = {};
+  const validators: DeriveEraValSlash = {};
+
+  noms.forEach(([key, optBalance]): void => {
+    nominators[key.args[1].toString()] = optBalance.unwrap();
+  });
+
+  vals.forEach(([key, optRes]): void => {
+    validators[key.args[1].toString()] = optRes.unwrapOrDefault()[1];
+  });
+
+  return { era, nominators, validators };
+}
+
+export function eraSlashes (api: ApiInterfaceRx): (era: EraIndex) => Observable<DeriveEraSlashes> {
+  return memo((era: EraIndex): Observable<DeriveEraSlashes> =>
+    combineLatest([
+      api.query.staking.nominatorSlashInEra.entries(era),
+      api.query.staking.validatorSlashInEra.entries(era)
+    ]).pipe(
+      map(([noms, vals]) => mapSlashes(era, noms, vals))
+    )
+  );
+}
+
+export function erasSlashes (api: ApiInterfaceRx): (withActive?: boolean | BN | number) => Observable<DeriveEraSlashes[]> {
+  return memo((withActive?: boolean | BN | number): Observable<DeriveEraSlashes[]> =>
+    api.derive.staking.erasHistoric(withActive).pipe(
+      switchMap((eras): Observable<DeriveEraSlashes[]> =>
+        eras.length
+          ? combineLatest(
+            eras.map((era) => api.derive.staking.eraSlashes(era))
+          )
+          : of([])
+      )
+    )
+  );
+}

packages/api-derive/src/staking/index.ts

@@ -4,12 +4,19 @@
 
 export * from './account';
 export * from './erasExposure';
+export * from './erasHistoric';
 export * from './erasPoints';
+export * from './erasPrefs';
 export * from './erasRewards';
-export * from './controllers';
+export * from './erasSlashes';
 export * from './electedInfo';
 export * from './overview';
+export * from './ownExposure';
+export * from './ownSlashes';
 export * from './query';
+export * from './stakerExposure';
+export * from './stakerPoints';
 export * from './stakerRewards';
+export * from './stakerSlashes';
+export * from './stashes';
 export * from './validators';
-export * from './validatorPoints';

packages/api-derive/src/staking/overview.ts

@@ -2,24 +2,23 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
-import { AccountId, EraIndex, EraPoints, EraRewardPoints, RewardPoint } from '@polkadot/types/interfaces';
+import { AccountId, EraPoints, EraRewardPoints, RewardPoint } from '@polkadot/types/interfaces';
 import { ApiInterfaceRx } from '@polkadot/api/types';
 import { DerivedStakingOverview } from '../types';
 
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
-import { createType } from '@polkadot/types';
 
 import { memo } from '../util';
 
-function retrievePointsPrev (api: ApiInterfaceRx, activeEra: EraIndex, currentElected: AccountId[]): Observable<EraRewardPoints> {
-  return api.query.staking.currentEraPointsEarned<EraPoints>(activeEra).pipe(
+function retrievePointsPrev (api: ApiInterfaceRx, currentElected: AccountId[]): Observable<EraRewardPoints> {
+  return api.query.staking.currentEraPointsEarned<EraPoints>().pipe(
     map(({ individual, total }): EraRewardPoints =>
-      createType(api.registry, 'EraRewardPoints', {
+      api.registry.createType('EraRewardPoints', {
         total,
         individual: new Map<AccountId, RewardPoint>(
           individual
-            .map((points): RewardPoint => createType(api.registry, 'RewardPoint', points))
+            .map((points): RewardPoint => api.registry.createType('RewardPoint', points))
             .map((points, index): [AccountId, RewardPoint] => [currentElected[index], points])
         )
       })
@@ -41,7 +40,9 @@ export function overview (api: ApiInterfaceRx): () => Observable<DerivedStakingO
           of({ ...indexes, nextElected, validators }),
           api.query.staking.erasRewardPoints
             ? api.query.staking.erasRewardPoints<EraRewardPoints>(indexes.activeEra)
-            : retrievePointsPrev(api, indexes.activeEra, nextElected)
+            : api.query.staking.currentEraPointsEarned
+              ? retrievePointsPrev(api, nextElected)
+              : of(api.registry.createType('EraRewardPoints'))
         ])
       ),
       map(([info, eraPoints]): DerivedStakingOverview => ({

packages/api-derive/src/staking/ownExposure.ts

@@ -0,0 +1,37 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { DeriveOwnExposure } from '../types';
+
+import BN from 'bn.js';
+import { Observable, combineLatest, of } from 'rxjs';
+import { map, switchMap } from 'rxjs/operators';
+
+import { memo } from '../util';
+
+export function ownExposure (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean | BN | number) => Observable<DeriveOwnExposure[]> {
+  return memo((accountId: Uint8Array | string, withActive?: boolean | BN | number): Observable<DeriveOwnExposure[]> => {
+    return api.derive.staking.erasHistoric(withActive).pipe(
+      switchMap((eras) =>
+        eras.length
+          ? combineLatest(
+            eras.map((era) => combineLatest([
+              of(era),
+              api.query.staking.erasStakersClipped(era, accountId),
+              api.query.staking.erasStakers(era, accountId)
+            ]))
+          )
+          : of([])
+      ),
+      map((result): DeriveOwnExposure[] =>
+        result.map(([era, clipped, exposure]): DeriveOwnExposure => ({
+          clipped,
+          era,
+          exposure
+        }))
+      )
+    );
+  });
+}

packages/api-derive/src/staking/ownSlashes.ts

@@ -0,0 +1,40 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { DeriveStakerSlashes } from '../types';
+
+import BN from 'bn.js';
+import { Observable, combineLatest, of } from 'rxjs';
+import { map, switchMap } from 'rxjs/operators';
+
+import { memo } from '../util';
+
+export function ownSlashes (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean | BN | number) => Observable<DeriveStakerSlashes[]> {
+  return memo((accountId: Uint8Array | string, withActive?: boolean | BN | number): Observable<DeriveStakerSlashes[]> => {
+    return api.derive.staking.erasHistoric(withActive).pipe(
+      switchMap((eras) =>
+        eras.length
+          ? combineLatest(
+            eras.map((era) => combineLatest([
+              of(era),
+              api.query.staking.nominatorSlashInEra(era, accountId),
+              api.query.staking.validatorSlashInEra(era, accountId)
+            ]))
+          )
+          : of([])
+      ),
+      map((result): DeriveStakerSlashes[] =>
+        result.map(([era, optNom, optVal]): DeriveStakerSlashes => ({
+          era,
+          total: optVal.isSome
+            ? optVal.unwrap()[1]
+            : optNom.isSome
+              ? optNom.unwrap()
+              : api.registry.createType('Balance')
+        }))
+      )
+    );
+  });
+}

packages/api-derive/src/staking/query.ts

@@ -7,9 +7,9 @@ import { AccountId, Exposure, Keys, Nominations, RewardDestination, ValidatorPre
 import { ITuple } from '@polkadot/types/types';
 import { DerivedStakingQuery } from '../types';
 
-import { combineLatest, Observable, of } from 'rxjs';
+import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
-import { createType, Option, Vec } from '@polkadot/types';
+import { Option, Vec } from '@polkadot/types';
 
 import { memo } from '../util';
 
@@ -46,7 +46,9 @@ function retrieveCurr (api: ApiInterfaceRx, stashId: AccountId): Observable<Mult
         [api.query.staking.nominators, stashId],
         [api.query.staking.payee, stashId],
         [api.query.staking.validators, stashId],
-        [api.query.session.nextKeys, [api.consts.session.dedupKeyPrefix, stashId]],
+        api.consts.session?.dedupKeyPrefix
+          ? [api.query.session.nextKeys, [api.consts.session.dedupKeyPrefix, stashId]]
+          : [api.query.session.nextKeys, stashId],
         [api.query.staking.erasStakers, [activeEra, stashId]]
       ])
     )
@@ -78,7 +80,7 @@ function retrieveController (api: ApiInterfaceRx, stashId: AccountId, [queuedKey
  */
 export function query (api: ApiInterfaceRx): (accountId: Uint8Array | string) => Observable<DerivedStakingQuery> {
   return memo((accountId: Uint8Array | string): Observable<DerivedStakingQuery> => {
-    const stashId = createType(api.registry, 'AccountId', accountId);
+    const stashId = api.registry.createType('AccountId', accountId);
 
     return combineLatest([
       api.query.session.queuedKeys<Vec<ITuple<[AccountId, Keys]>>>(),
@@ -92,3 +94,11 @@ export function query (api: ApiInterfaceRx): (accountId: Uint8Array | string) =>
     );
   });
 }
+
+export function queryMulti (api: ApiInterfaceRx): (...accountIds: (Uint8Array | string)[]) => Observable<DerivedStakingQuery[]> {
+  return memo((...accountIds: (Uint8Array | string)[]): Observable<DerivedStakingQuery[]> =>
+    combineLatest(
+      accountIds.map((accountId) => api.derive.staking.query(accountId))
+    )
+  );
+}

packages/api-derive/src/staking/stakerExposure.ts

@@ -0,0 +1,40 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { DeriveStakerExposure, DeriveEraValidatorExposure } from '../types';
+
+import BN from 'bn.js';
+import { Observable } from 'rxjs';
+import { map } from 'rxjs/operators';
+
+import { memo } from '../util';
+
+export function stakerExposure (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean | BN | number) => Observable<DeriveStakerExposure[]> {
+  return memo((accountId: Uint8Array | string, withActive?: boolean | BN | number): Observable<DeriveStakerExposure[]> => {
+    const stakerId = api.registry.createType('AccountId', accountId).toString();
+
+    return api.derive.staking.erasExposure(withActive).pipe(
+      map((exposures): DeriveStakerExposure[] =>
+        exposures.map(({ era, nominators: allNominators, validators: allValidators }): DeriveStakerExposure => {
+          const isValidator = !!allValidators[stakerId];
+          const validators: DeriveEraValidatorExposure = {};
+          let nominating: [string, number][] = [];
+
+          if (isValidator) {
+            validators[stakerId] = allValidators[stakerId];
+          } else if (allNominators[stakerId]) {
+            nominating = allNominators[stakerId];
+
+            nominating.forEach(([validatorId]): void => {
+              validators[validatorId] = allValidators[validatorId];
+            });
+          }
+
+          return { era, isEmpty: !Object.keys(validators).length, isValidator, nominating, validators };
+        })
+      )
+    );
+  });
+}

packages/api-derive/src/staking/stakerPoints.ts

@@ -0,0 +1,30 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { DeriveStakerPoints } from '../types';
+
+import BN from 'bn.js';
+import { Observable } from 'rxjs';
+import { map } from 'rxjs/operators';
+
+import { memo } from '../util';
+
+export function stakerPoints (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean | BN | number) => Observable<DeriveStakerPoints[]> {
+  return memo((accountId: Uint8Array | string, withActive?: boolean | BN | number): Observable<DeriveStakerPoints[]> => {
+    const stakerId = api.registry.createType('AccountId', accountId).toString();
+
+    return api.derive.staking.erasPoints(withActive).pipe(
+      map((points): DeriveStakerPoints[] =>
+        points.map(({ era, eraPoints, validators }): DeriveStakerPoints => ({
+          era,
+          eraPoints,
+          points: validators[stakerId]
+            ? validators[stakerId]
+            : api.registry.createType('RewardPoint')
+        }))
+      )
+    );
+  });
+}

packages/api-derive/src/staking/stakerRewards.ts

@@ -3,40 +3,90 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { Exposure } from '@polkadot/types/interfaces';
-import { DeriveStakerReward } from '../types';
+import { Balance } from '@polkadot/types/interfaces';
+import { DeriveEraPoints, DeriveEraPrefs, DeriveEraRewards, DeriveEraValPrefs, DeriveStakerExposure, DeriveStakerReward } from '../types';
 
-import { Observable } from 'rxjs';
+import BN from 'bn.js';
+import { Observable, combineLatest } from 'rxjs';
 import { map } from 'rxjs/operators';
 
 import { memo } from '../util';
 
-export function stakerRewards (api: ApiInterfaceRx): (accountId: Uint8Array | string) => Observable<DeriveStakerReward[]> {
-  return memo((accountId: Uint8Array | string): Observable<DeriveStakerReward[]> => {
+type Result = [DeriveEraPoints[], DeriveEraPrefs[], DeriveEraRewards[], DeriveStakerExposure[]];
+
+const ZERO = new BN(0);
+const COMM_DIV = new BN(1_000_000_000);
+
+function parseRewards (api: ApiInterfaceRx, stakerId: string, [erasPoints, erasPrefs, erasRewards, exposures]: Result): DeriveStakerReward[] {
+  return exposures.map(({ era, isEmpty, isValidator, nominating, validators: eraValidators }): DeriveStakerReward => {
+    const { eraPoints, validators: allValPoints } = erasPoints.find((p) => p.era.eq(era)) || { eraPoints: new BN(0), validators: {} };
+    const { eraReward } = erasRewards.find((r) => r.era.eq(era)) || { eraReward: ZERO };
+    const { validators: allValPrefs } = erasPrefs.find((p) => p.era.eq(era)) || { validators: {} as DeriveEraValPrefs };
+    const validators: Record<string, Balance> = {};
+    let total = ZERO;
+
+    Object.entries(eraValidators).forEach(([validatorId, exposure]): void => {
+      const valPoints = allValPoints[validatorId] || ZERO;
+      const valComm = allValPrefs[validatorId]?.commission.unwrap() || ZERO;
+      const avail = eraReward.mul(valPoints).div(eraPoints);
+      const valCut = valComm.mul(avail).div(COMM_DIV);
+      const expTotal = exposure.total.unwrap();
+      let value: BN | undefined;
+
+      if (!expTotal.isZero() && !valPoints.isZero()) {
+        let staked: BN;
+
+        if (validatorId === stakerId) {
+          staked = exposure.own.unwrap();
+        } else {
+          const stakerExp = exposure.others.find(({ who }): boolean => who.eq(stakerId));
+
+          staked = stakerExp
+            ? stakerExp.value.unwrap()
+            : ZERO;
+        }
+
+        value = avail.sub(valCut).mul(staked).div(expTotal).add(validatorId === stakerId ? valCut : ZERO);
+        total = total.add(value);
+      }
+
+      validators[validatorId] = api.registry.createType('Balance', value);
+    });
+
+    return {
+      era,
+      isEmpty,
+      isValidator,
+      nominating,
+      total: api.registry.createType('Balance', total),
+      validators
+    };
+  });
+}
+
+export function stakerRewards (api: ApiInterfaceRx): (accountId: Uint8Array | string, startEra: BN | number) => Observable<DeriveStakerReward[]> {
+  return memo((accountId: Uint8Array | string, startEra: BN | number): Observable<DeriveStakerReward[]> => {
     const stakerId = api.registry.createType('AccountId', accountId).toString();
 
-    return api.derive.staking.erasRewards().pipe(
-      map((rewards): DeriveStakerReward[] =>
-        rewards
-          .map(({ era, nominators: allNominators, validators: allValidators }): DeriveStakerReward => {
-            const isValidator = !!allValidators[stakerId];
-            const validators: Record<string, Exposure> = {};
-            let nominating: [string, number][] = [];
-
-            if (isValidator) {
-              validators[stakerId] = allValidators[stakerId];
-            } else if (allNominators[stakerId]) {
-              nominating = allNominators[stakerId];
-
-              nominating.forEach(([validatorId]): void => {
-                validators[validatorId] = allValidators[validatorId];
-              });
-            }
-
-            return { era, isValidator, nominating, validators };
-          })
-          .filter(({ validators }): boolean => Object.keys(validators).length !== 0)
+    return combineLatest([
+      api.derive.staking.erasPoints(startEra),
+      api.derive.staking.erasPrefs(startEra),
+      api.derive.staking.erasRewards(startEra),
+      api.derive.staking.stakerExposure(stakerId, startEra)
+    ]).pipe(
+      map((result): DeriveStakerReward[] =>
+        parseRewards(api, stakerId, result)
       )
     );
   });
 }
+
+export function stakerRewardsMulti (api: ApiInterfaceRx): (...params: [Uint8Array | string, BN | number][]) => Observable<DeriveStakerReward[][]> {
+  return memo((...params: [Uint8Array | string, BN | number][]): Observable<DeriveStakerReward[][]> =>
+    combineLatest(
+      params.map(([accountId, startEra]) =>
+        api.derive.staking.stakerRewards(accountId, startEra)
+      )
+    )
+  );
+}

packages/api-derive/src/staking/stakerSlashes.ts

@@ -0,0 +1,27 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { DeriveStakerSlashes } from '../types';
+
+import BN from 'bn.js';
+import { Observable } from 'rxjs';
+import { map } from 'rxjs/operators';
+
+import { memo } from '../util';
+
+export function stakerSlashes (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean | BN | number) => Observable<DeriveStakerSlashes[]> {
+  return memo((accountId: Uint8Array | string, withActive?: boolean | BN | number): Observable<DeriveStakerSlashes[]> => {
+    const stakerId = api.registry.createType('AccountId', accountId).toString();
+
+    return api.derive.staking.erasSlashes(withActive).pipe(
+      map((slashes): DeriveStakerSlashes[] =>
+        slashes.map(({ era, nominators, validators }): DeriveStakerSlashes => ({
+          era,
+          total: nominators[stakerId] || validators[stakerId] || api.registry.createType('Balance')
+        }))
+      )
+    );
+  });
+}

packages/api-derive/src/staking/stashes.ts

@@ -0,0 +1,28 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { AccountId, ValidatorPrefs } from '@polkadot/types/interfaces';
+import { ITuple } from '@polkadot/types/types';
+
+import { Observable } from 'rxjs';
+import { map } from 'rxjs/operators';
+import { Vec } from '@polkadot/types';
+
+import { memo } from '../util';
+
+/**
+ * @description Retrieve the list of all validator stashes
+ */
+export function stashes (api: ApiInterfaceRx): () => Observable<AccountId[]> {
+  return memo((): Observable<AccountId[]> =>
+    api.query.staking.validators.creator.meta.type.asMap.linked.isTrue
+      ? api.query.staking.validators<ITuple<[Vec<AccountId>, Vec<ValidatorPrefs>]>>().pipe(
+        map(([stashIds]) => stashIds)
+      )
+      : api.query.staking.validators.keys().pipe(
+        map((keys) => keys.map((key) => key.args[0] as AccountId))
+      )
+  );
+}

packages/api-derive/src/staking/types.ts

@@ -5,34 +5,78 @@
 import { AccountId, Balance, BlockNumber, EraIndex, EraRewardPoints, Exposure, Keys, RewardDestination, RewardPoint, StakingLedger, ValidatorPrefs } from '@polkadot/types/interfaces';
 import { DeriveSessionIndexes } from '../session/types';
 
-export interface DeriveEraPointsAll {
-  all: Record<string, RewardPoint>;
-  era: EraIndex;
-  total: RewardPoint;
-}
+export type DeriveEraValPoints = Record<string, RewardPoint>;
+
+export type DeriveEraValPrefs = Record<string, ValidatorPrefs>;
+
+export type DeriveEraValSlash = Record<string, Balance>;
 
 export interface DeriveEraPoints {
   era: EraIndex;
-  own: RewardPoint;
-  total: RewardPoint;
+  eraPoints: RewardPoint;
+  validators: DeriveEraValPoints;
 }
 
+export interface DeriveEraPrefs {
+  era: EraIndex;
+  validators: DeriveEraValPrefs;
+}
+
+export interface DeriveEraRewards {
+  era: EraIndex;
+  eraReward: Balance;
+}
+
+export interface DeriveEraSlashes {
+  era: EraIndex;
+  nominators: DeriveEraValSlash;
+  validators: DeriveEraValSlash;
+}
+
+export interface DeriveStakerPoints {
+  era: EraIndex;
+  eraPoints: RewardPoint;
+  points: RewardPoint;
+}
+
+export type DeriveEraNominatorExposure = Record<string, [string, number][]>;
+
+export type DeriveEraValidatorExposure = Record<string, Exposure>;
+
 export interface DeriveEraExposure {
-  all: Record<string, Exposure>;
   era: EraIndex;
+  nominators: DeriveEraNominatorExposure;
+  validators: DeriveEraValidatorExposure;
 }
 
-export interface DeriveEraRewardsAll {
+export interface DeriveOwnExposure {
+  clipped: Exposure;
   era: EraIndex;
-  nominators: Record<string, [string, number][]>;
-  validators: Record<string, Exposure>;
+  exposure: Exposure;
+}
+
+export type DeriveOwnSlashes = DeriveStakerSlashes;
+
+export interface DeriveStakerExposure {
+  era: EraIndex;
+  isEmpty: boolean;
+  isValidator: boolean;
+  nominating: [string, number][];
+  validators: DeriveEraValidatorExposure;
 }
 
 export interface DeriveStakerReward {
   era: EraIndex;
+  isEmpty: boolean;
   isValidator: boolean;
   nominating: [string, number][];
-  validators: Record<string, Exposure>;
+  validators: Record<string, Balance>;
+  total: Balance;
+}
+
+export interface DeriveStakerSlashes {
+  era: EraIndex;
+  total: Balance;
 }
 
 export interface DerivedStakingElected {

packages/api-derive/src/staking/validatorPoints.ts

@@ -1,27 +0,0 @@
-// Copyright 2017-2020 @polkadot/api-derive authors & contributors
-// This software may be modified and distributed under the terms
-// of the Apache-2.0 license. See the LICENSE file for details.
-
-import { ApiInterfaceRx } from '@polkadot/api/types';
-import { DeriveEraPoints } from '../types';
-
-import { Observable } from 'rxjs';
-import { map } from 'rxjs/operators';
-
-import { memo } from '../util';
-
-export function validatorPoints (api: ApiInterfaceRx): (accountId: Uint8Array | string) => Observable<DeriveEraPoints[]> {
-  return memo((_accountId: Uint8Array | string): Observable<DeriveEraPoints[]> => {
-    const accountId = api.registry.createType('AccountId', _accountId).toString();
-
-    return api.derive.staking.erasPoints().pipe(
-      map((all): DeriveEraPoints[] =>
-        all.map(({ all, era, total }): DeriveEraPoints => ({
-          era,
-          own: all[accountId] || api.registry.createType('RewardPoint'),
-          total
-        }))
-      )
-    );
-  });
-}

packages/api-derive/src/staking/validators.ts

@@ -7,10 +7,28 @@ import { AccountId } from '@polkadot/types/interfaces';
 import { DeriveStakingValidators } from '../types';
 
 import { Observable, combineLatest, of } from 'rxjs';
-import { map } from 'rxjs/operators';
+import { map, switchMap } from 'rxjs/operators';
+import { StorageKey } from '@polkadot/types';
 
 import { memo } from '../util';
 
+export function nextElected (api: ApiInterfaceRx): () => Observable<AccountId[]> {
+  return memo((): Observable<AccountId[]> =>
+    api.query.staking.erasStakers
+      ? api.derive.session.indexes().pipe(
+        // only populate for next era in the last session, so track both here - entries are not
+        // subscriptions, so we need a trigger - currentIndex acts as that trigger to refresh
+        switchMap(({ currentEra }): Observable<StorageKey[]> =>
+          api.query.staking.erasStakers.keys(currentEra)
+        ),
+        map((keys): AccountId[] =>
+          keys.map((key): AccountId => key.args[1] as AccountId)
+        )
+      )
+      : api.query.staking.currentElected<AccountId[]>()
+  );
+}
+
 /**
  * @description Retrieve latest list of validators
  */
@@ -23,15 +41,14 @@ export function validators (api: ApiInterfaceRx): () => Observable<DeriveStaking
       api.query.session
         ? api.query.session.validators()
         : of([]),
-      // FIXME need a replacement for currentElected in new
-      api.query.staking?.currentElected
-        ? api.query.staking.erasStakers
-          ? of(null)
-          : api.query.staking.currentElected<AccountId[]>()
+      api.query.staking
+        ? api.derive.staking.nextElected()
         : of([])
     ]).pipe(
       map(([validators, nextElected]): DeriveStakingValidators => ({
-        nextElected: nextElected || validators,
+        nextElected: nextElected.length
+          ? nextElected
+          : validators,
         validators
       }))
     ));

packages/api-derive/src/treasury/proposals.ts

@@ -8,7 +8,7 @@ import { DerivedCollectiveProposals, DerivedTreasuryProposal, DerivedTreasuryPro
 
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
-import { createType, Option } from '@polkadot/types';
+import { Option } from '@polkadot/types';
 
 import { memo } from '../util';
 
@@ -55,7 +55,7 @@ function retrieveProposals (api: ApiInterfaceRx, proposalCount: ProposalIndex, a
     const isApproval = approvalIds.some((id): boolean => id.eqn(index));
 
     if (!isApproval) {
-      proposalIds.push(createType(api.registry, 'ProposalIndex', index));
+      proposalIds.push(api.registry.createType('ProposalIndex', index));
     }
   }
 
@@ -87,7 +87,7 @@ export function proposals (api: ApiInterfaceRx): () => Observable<DerivedTreasur
       )
       : of({
         approvals: [],
-        proposalCount: createType(api.registry, 'ProposalIndex'),
+        proposalCount: api.registry.createType('ProposalIndex'),
         proposals: []
       } as DerivedTreasuryProposals)
   );

packages/api-derive/src/util/index.ts

@@ -2,5 +2,9 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
+import { drr } from '@polkadot/rpc-core/rxjs';
+
 export * from './approvalFlagsToBools';
 export * from './memo';
+
+export { drr };

packages/api/README.md

@@ -95,6 +95,7 @@ Some of the users of the API (let us know if you are missing from the list), inc
 - [Polkabot](https://gitlab.com/Polkabot) Polkabot is a Matrix chatbot that keeps an eye on the Polkadot network. You can see Polkabot in action in https://matrix.to/#/#polkadot-network-status:matrix.org
 - [Polkawallet.io](https://polkawallet.io) and [Polkawallet (Github)](https://github.com/polkawallet-io/polkawallet-RN/) A mobile wallet for the Polkadot network to manage funds and make transactions, available on both Androind and iOS
 - [PolkaStats.io](https://polkastats.io), [PolkaStats frontend GitHub repository](https://github.com/Colm3na/polkastats-v2) and [PolkaStats backend GitHub repository](https://github.com/Colm3na/polkastats-backend-v2) Polkadot network statistics (currently Kusama and Alexander). Shows network information and staking details from validators and intentions.
+- [Polkadot API Server (GitHub)](https://github.com/SimplyVC/polkadot_api_server) A lightweight server for querying Polkadot nodes from any language, built primarily as a backend for [PANIC for Polkadot (GitHub)](https://github.com/SimplyVC/panic_polkadot/), a validator monitoring and alerting tool.
 
 ## Classes
 

packages/api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/api",
-  "version": "1.5.1",
+  "version": "1.7.1",
   "description": "Promise and RxJS wrappers around the Polkadot JS RPC",
   "main": "index.js",
   "keywords": [
@@ -26,20 +26,20 @@
   },
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/api#readme",
   "dependencies": {
-    "@babel/runtime": "^7.8.4",
-    "@polkadot/api-derive": "1.5.1",
-    "@polkadot/keyring": "^2.6.1",
-    "@polkadot/metadata": "1.5.1",
-    "@polkadot/rpc-core": "1.5.1",
-    "@polkadot/rpc-provider": "1.5.1",
-    "@polkadot/types": "1.5.1",
-    "@polkadot/util": "^2.6.1",
-    "@polkadot/util-crypto": "^2.6.1",
+    "@babel/runtime": "^7.8.7",
+    "@polkadot/api-derive": "1.7.1",
+    "@polkadot/keyring": "^2.6.2",
+    "@polkadot/metadata": "1.7.1",
+    "@polkadot/rpc-core": "1.7.1",
+    "@polkadot/rpc-provider": "1.7.1",
+    "@polkadot/types": "1.7.1",
+    "@polkadot/util": "^2.6.2",
+    "@polkadot/util-crypto": "^2.6.2",
     "bn.js": "^5.1.1",
     "eventemitter3": "^4.0.0",
     "rxjs": "^6.5.4"
   },
   "devDependencies": {
-    "@polkadot/keyring": "^2.6.1"
+    "@polkadot/keyring": "^2.6.2"
   }
 }

packages/api/src/augment/consts.ts

@@ -2,7 +2,7 @@
 /* eslint-disable @typescript-eslint/no-empty-interface */
 
 import { Codec } from '@polkadot/types/types';
-import { Bytes, u32, u64 } from '@polkadot/types/primitive';
+import { u32, u64 } from '@polkadot/types/primitive';
 import { Gas } from '@polkadot/types/interfaces/contracts';
 import { Balance, BalanceOf, BlockNumber, Moment, Percent, Permill } from '@polkadot/types/interfaces/runtime';
 import { SessionIndex } from '@polkadot/types/interfaces/session';
@@ -27,16 +27,6 @@ declare module '@polkadot/metadata/Decorated/consts/types' {
        **/
       expectedBlockTime: AugmentedConst<Moment>;
     };
-    timestamp: {
-      [index: string]: AugmentedConst<object & Codec>;
-      /**
-       * The minimum period between blocks. Beware that this is different to the *expected* period
-       * that the block production apparatus provides. Your chosen consensus system will generally
-       * work with this to determine a sensible block time. e.g. For Aura, it will be double this
-       * period on default settings.
-       **/
-      minimumPeriod: AugmentedConst<Moment>;
-    };
     balances: {
       [index: string]: AugmentedConst<object & Codec>;
       /**
@@ -44,8 +34,70 @@ declare module '@polkadot/metadata/Decorated/consts/types' {
        **/
       existentialDeposit: AugmentedConst<Balance>;
     };
-    transactionPayment: {
+    contracts: {
       [index: string]: AugmentedConst<object & Codec>;
+      /**
+       * The maximum amount of gas that could be expended per block. A reasonable
+       * default value is 10_000_000.
+       **/
+      blockGasLimit: AugmentedConst<Gas>;
+      /**
+       * The base fee charged for calling into a contract. A reasonable default
+       * value is 135.
+       **/
+      callBaseFee: AugmentedConst<Gas>;
+      /**
+       * The fee required to instantiate a contract instance. A reasonable default value
+       * is 21.
+       **/
+      contractFee: AugmentedConst<BalanceOf>;
+      /**
+       * The base fee charged for instantiating a contract. A reasonable default value
+       * is 175.
+       **/
+      instantiateBaseFee: AugmentedConst<Gas>;
+      /**
+       * The maximum nesting level of a call/instantiate stack. A reasonable default
+       * value is 100.
+       **/
+      maxDepth: AugmentedConst<u32>;
+      /**
+       * The maximum size of a storage value in bytes. A reasonable default is 16 KiB.
+       **/
+      maxValueSize: AugmentedConst<u32>;
+      /**
+       * Price of a byte of storage per one block interval. Should be greater than 0.
+       **/
+      rentByteFee: AugmentedConst<BalanceOf>;
+      /**
+       * The amount of funds a contract should deposit in order to offset
+       * the cost of one byte.
+       * Let's suppose the deposit is 1,000 BU (balance units)/byte and the rent is 1 BU/byte/day,
+       * then a contract with 1,000,000 BU that uses 1,000 bytes of storage would pay no rent.
+       * But if the balance reduced to 500,000 BU and the storage stayed the same at 1,000,
+       * then it would pay 500 BU/day.
+       **/
+      rentDepositOffset: AugmentedConst<BalanceOf>;
+      /**
+       * Number of block delay an extrinsic claim surcharge has.
+       * When claim surcharge is called by an extrinsic the rent is checked
+       * for current_block - delay
+       **/
+      signedClaimHandicap: AugmentedConst<BlockNumber>;
+      /**
+       * Size of a contract at the time of instantiation. This is a simple way to ensure that
+       * empty contracts eventually gets deleted.
+       **/
+      storageSizeOffset: AugmentedConst<u32>;
+      /**
+       * Reward that is received by the party whose touch has led
+       * to removal of a contract.
+       **/
+      surchargeReward: AugmentedConst<BalanceOf>;
+      /**
+       * The minimum amount required to generate a tombstone.
+       **/
+      tombstoneDeposit: AugmentedConst<BalanceOf>;
       /**
        * The fee to be paid for making a transaction; the base.
        **/
@@ -55,27 +107,16 @@ declare module '@polkadot/metadata/Decorated/consts/types' {
        **/
       transactionByteFee: AugmentedConst<BalanceOf>;
     };
-    staking: {
-      [index: string]: AugmentedConst<object & Codec>;
-      /**
-       * Number of sessions per era.
-       **/
-      sessionsPerEra: AugmentedConst<SessionIndex>;
-      /**
-       * Number of eras that staked funds must remain bonded for.
-       **/
-      bondingDuration: AugmentedConst<EraIndex>;
-    };
-    session: {
-      [index: string]: AugmentedConst<object & Codec>;
-      /**
-       * Used as first key for `NextKeys` and `KeyOwner` to put all the data into the same branch
-       * of the trie.
-       **/
-      dedupKeyPrefix: AugmentedConst<Bytes>;
-    };
     democracy: {
       [index: string]: AugmentedConst<object & Codec>;
+      /**
+       * Period in blocks where an external proposal may not be re-submitted after being vetoed.
+       **/
+      cooloffPeriod: AugmentedConst<BlockNumber>;
+      /**
+       * Minimum voting period allowed for an emergency referendum.
+       **/
+      emergencyVotingPeriod: AugmentedConst<BlockNumber>;
       /**
        * The minimum period of locking and the period between a proposal being approved and enacted.
        * It should generally be a little more than the unstake period to ensure that
@@ -87,48 +128,105 @@ declare module '@polkadot/metadata/Decorated/consts/types' {
        * How often (in blocks) new public referenda are launched.
        **/
       launchPeriod: AugmentedConst<BlockNumber>;
-      /**
-       * How often (in blocks) to check for new votes.
-       **/
-      votingPeriod: AugmentedConst<BlockNumber>;
       /**
        * The minimum amount to be used as a deposit for a public referendum proposal.
        **/
       minimumDeposit: AugmentedConst<BalanceOf>;
-      /**
-       * Minimum voting period allowed for an emergency referendum.
-       **/
-      emergencyVotingPeriod: AugmentedConst<BlockNumber>;
-      /**
-       * Period in blocks where an external proposal may not be re-submitted after being vetoed.
-       **/
-      cooloffPeriod: AugmentedConst<BlockNumber>;
       /**
        * The amount of balance that must be deposited per byte of preimage stored.
        **/
       preimageByteDeposit: AugmentedConst<BalanceOf>;
+      /**
+       * How often (in blocks) to check for new votes.
+       **/
+      votingPeriod: AugmentedConst<BlockNumber>;
     };
     elections: {
       [index: string]: AugmentedConst<object & Codec>;
       candidacyBond: AugmentedConst<BalanceOf>;
-      votingBond: AugmentedConst<BalanceOf>;
       desiredMembers: AugmentedConst<u32>;
       desiredRunnersUp: AugmentedConst<u32>;
       termDuration: AugmentedConst<BlockNumber>;
+      votingBond: AugmentedConst<BalanceOf>;
     };
     finalityTracker: {
       [index: string]: AugmentedConst<object & Codec>;
-      /**
-       * The number of recent samples to keep from this chain. Default is 101.
-       **/
-      windowSize: AugmentedConst<BlockNumber>;
       /**
        * The delay after which point things become suspicious. Default is 1000.
        **/
       reportLatency: AugmentedConst<BlockNumber>;
+      /**
+       * The number of recent samples to keep from this chain. Default is 101.
+       **/
+      windowSize: AugmentedConst<BlockNumber>;
+    };
+    society: {
+      [index: string]: AugmentedConst<object & Codec>;
+      /**
+       * The minimum amount of a deposit required for a bid to be made.
+       **/
+      candidateDeposit: AugmentedConst<BalanceOf>;
+      /**
+       * The number of blocks between membership challenges.
+       **/
+      challengePeriod: AugmentedConst<BlockNumber>;
+      /**
+       * The number of times a member may vote the wrong way (or not at all, when they are a skeptic)
+       * before they become suspended.
+       **/
+      maxStrikes: AugmentedConst<u32>;
+      /**
+       * The amount of incentive paid within each period. Doesn't include VoterTip.
+       **/
+      periodSpend: AugmentedConst<BalanceOf>;
+      /**
+       * The number of blocks between candidate/membership rotation periods.
+       **/
+      rotationPeriod: AugmentedConst<BlockNumber>;
+      /**
+       * The amount of the unpaid reward that gets deducted in the case that either a skeptic
+       * doesn't vote or someone votes in the wrong way.
+       **/
+      wrongSideDeduction: AugmentedConst<BalanceOf>;
+    };
+    staking: {
+      [index: string]: AugmentedConst<object & Codec>;
+      /**
+       * Number of eras that staked funds must remain bonded for.
+       **/
+      bondingDuration: AugmentedConst<EraIndex>;
+      /**
+       * Number of sessions per era.
+       **/
+      sessionsPerEra: AugmentedConst<SessionIndex>;
+    };
+    timestamp: {
+      [index: string]: AugmentedConst<object & Codec>;
+      /**
+       * The minimum period between blocks. Beware that this is different to the *expected* period
+       * that the block production apparatus provides. Your chosen consensus system will generally
+       * work with this to determine a sensible block time. e.g. For Aura, it will be double this
+       * period on default settings.
+       **/
+      minimumPeriod: AugmentedConst<Moment>;
+    };
+    transactionPayment: {
+      [index: string]: AugmentedConst<object & Codec>;
+      /**
+       * The fee to be paid for making a transaction; the base.
+       **/
+      transactionBaseFee: AugmentedConst<BalanceOf>;
+      /**
+       * The fee to be paid for making a transaction; the per-byte portion.
+       **/
+      transactionByteFee: AugmentedConst<BalanceOf>;
     };
     treasury: {
       [index: string]: AugmentedConst<object & Codec>;
+      /**
+       * Percentage of spare funds (if any) that are burnt per spend period.
+       **/
+      burn: AugmentedConst<Permill>;
       /**
        * Fraction of a proposal's value that should be bonded in order to place the proposal.
        * An accepted proposal gets these back. A rejected proposal does not.
@@ -142,10 +240,6 @@ declare module '@polkadot/metadata/Decorated/consts/types' {
        * Period between successive spends.
        **/
       spendPeriod: AugmentedConst<BlockNumber>;
-      /**
-       * Percentage of spare funds (if any) that are burnt per spend period.
-       **/
-      burn: AugmentedConst<Permill>;
       /**
        * The period for which a tip remains open after is has achieved threshold tippers.
        **/
@@ -163,108 +257,6 @@ declare module '@polkadot/metadata/Decorated/consts/types' {
        **/
       tipReportDepositPerByte: AugmentedConst<BalanceOf>;
     };
-    contracts: {
-      [index: string]: AugmentedConst<object & Codec>;
-      /**
-       * Number of block delay an extrinsic claim surcharge has.
-       * When claim surcharge is called by an extrinsic the rent is checked
-       * for current_block - delay
-       **/
-      signedClaimHandicap: AugmentedConst<BlockNumber>;
-      /**
-       * The minimum amount required to generate a tombstone.
-       **/
-      tombstoneDeposit: AugmentedConst<BalanceOf>;
-      /**
-       * Size of a contract at the time of instantiation. This is a simple way to ensure that
-       * empty contracts eventually gets deleted.
-       **/
-      storageSizeOffset: AugmentedConst<u32>;
-      /**
-       * Price of a byte of storage per one block interval. Should be greater than 0.
-       **/
-      rentByteFee: AugmentedConst<BalanceOf>;
-      /**
-       * The amount of funds a contract should deposit in order to offset
-       * the cost of one byte.
-       * Let's suppose the deposit is 1,000 BU (balance units)/byte and the rent is 1 BU/byte/day,
-       * then a contract with 1,000,000 BU that uses 1,000 bytes of storage would pay no rent.
-       * But if the balance reduced to 500,000 BU and the storage stayed the same at 1,000,
-       * then it would pay 500 BU/day.
-       **/
-      rentDepositOffset: AugmentedConst<BalanceOf>;
-      /**
-       * Reward that is received by the party whose touch has led
-       * to removal of a contract.
-       **/
-      surchargeReward: AugmentedConst<BalanceOf>;
-      /**
-       * The fee to be paid for making a transaction; the base.
-       **/
-      transactionBaseFee: AugmentedConst<BalanceOf>;
-      /**
-       * The fee to be paid for making a transaction; the per-byte portion.
-       **/
-      transactionByteFee: AugmentedConst<BalanceOf>;
-      /**
-       * The fee required to instantiate a contract instance. A reasonable default value
-       * is 21.
-       **/
-      contractFee: AugmentedConst<BalanceOf>;
-      /**
-       * The base fee charged for calling into a contract. A reasonable default
-       * value is 135.
-       **/
-      callBaseFee: AugmentedConst<Gas>;
-      /**
-       * The base fee charged for instantiating a contract. A reasonable default value
-       * is 175.
-       **/
-      instantiateBaseFee: AugmentedConst<Gas>;
-      /**
-       * The maximum nesting level of a call/instantiate stack. A reasonable default
-       * value is 100.
-       **/
-      maxDepth: AugmentedConst<u32>;
-      /**
-       * The maximum size of a storage value in bytes. A reasonable default is 16 KiB.
-       **/
-      maxValueSize: AugmentedConst<u32>;
-      /**
-       * The maximum amount of gas that could be expended per block. A reasonable
-       * default value is 10_000_000.
-       **/
-      blockGasLimit: AugmentedConst<Gas>;
-    };
-    society: {
-      [index: string]: AugmentedConst<object & Codec>;
-      /**
-       * The minimum amount of a deposit required for a bid to be made.
-       **/
-      candidateDeposit: AugmentedConst<BalanceOf>;
-      /**
-       * The amount of the unpaid reward that gets deducted in the case that either a skeptic
-       * doesn't vote or someone votes in the wrong way.
-       **/
-      wrongSideDeduction: AugmentedConst<BalanceOf>;
-      /**
-       * The number of times a member may vote the wrong way (or not at all, when they are a skeptic)
-       * before they become suspended.
-       **/
-      maxStrikes: AugmentedConst<u32>;
-      /**
-       * The amount of incentive paid within each period. Doesn't include VoterTip.
-       **/
-      periodSpend: AugmentedConst<BalanceOf>;
-      /**
-       * The number of blocks between candidate/membership rotation periods.
-       **/
-      rotationPeriod: AugmentedConst<BlockNumber>;
-      /**
-       * The number of blocks between membership challenges.
-       **/
-      challengePeriod: AugmentedConst<BlockNumber>;
-    };
     vesting: {
       [index: string]: AugmentedConst<object & Codec>;
       /**

packages/api/src/augment/rpc.ts

@@ -7,7 +7,7 @@ import { Bytes, StorageKey, Text, bool, u32, u64 } from '@polkadot/types/primiti
 import { Metadata } from '@polkadot/types';
 import { ContractCallRequest, ContractExecResult } from '@polkadot/types/interfaces/contracts';
 import { Extrinsic } from '@polkadot/types/interfaces/extrinsics';
-import { BlockHash, ChainProperties, ExtrinsicOrHash, ExtrinsicStatus, Health, NetworkState, PeerInfo, RpcMethods, RuntimeDispatchInfo, RuntimeVersion, StorageChangeSet } from '@polkadot/types/interfaces/rpc';
+import { BlockHash, ChainProperties, ExtrinsicOrHash, ExtrinsicStatus, Health, NetworkState, PeerInfo, RpcMethods, RuntimeDispatchInfo, RuntimeVersion } from '@polkadot/types/interfaces/rpc';
 import { AccountId, BlockNumber, H256, Hash, Header, Index, SignedBlock, StorageData } from '@polkadot/types/interfaces/runtime';
 import { Observable } from 'rxjs';
 
@@ -130,6 +130,10 @@ declare module '@polkadot/rpc-core/types.jsonrpc' {
        * Retrieves the keys with a certain prefix
        **/
       getKeys: AugmentedRpc<(key: StorageKey | string | Uint8Array | any, at?: BlockHash | string | Uint8Array) => Observable<Vec<StorageKey>>>;
+      /**
+       * Returns the keys with prefix with pagination support.
+       **/
+      getKeysPaged: AugmentedRpc<(key: StorageKey | string | Uint8Array | any, count: u32 | AnyNumber | Uint8Array, startKey?: StorageKey | string | Uint8Array | any, at?: BlockHash | string | Uint8Array) => Observable<Vec<StorageKey>>>;
       /**
        * Returns the runtime metadata
        **/
@@ -138,7 +142,10 @@ declare module '@polkadot/rpc-core/types.jsonrpc' {
        * Get the runtime version
        **/
       getRuntimeVersion: AugmentedRpc<(at?: BlockHash | string | Uint8Array) => Observable<RuntimeVersion>>;
-      getStorage: AugmentedRpc<<T = Codec>(key: any, block?: Hash | Uint8Array | string) => Observable<T>>;
+      /**
+       * Retrieves the storage for a key
+       **/
+      getStorage: AugmentedRpc<<T = Codec>(key: StorageKey | string | Uint8Array | any, block?: Hash | Uint8Array | string) => Observable<T>>;
       /**
        * Retrieves the storage hash
        **/
@@ -150,12 +157,15 @@ declare module '@polkadot/rpc-core/types.jsonrpc' {
       /**
        * Query historical storage entries (by key) starting from a start block
        **/
-      queryStorage: AugmentedRpc<(keys: Vec<StorageKey> | (StorageKey | string | Uint8Array | any)[], startBlock: Hash | string | Uint8Array, at?: BlockHash | string | Uint8Array) => Observable<Vec<StorageChangeSet>>>;
+      queryStorage: AugmentedRpc<<T = Codec[]>(keys: Vec<StorageKey> | (StorageKey | string | Uint8Array | any)[], fromBlock?: Hash | Uint8Array | string, toBlock?: Hash | Uint8Array | string) => Observable<[Hash, T][]>>;
       /**
        * Retrieves the runtime version via subscription
        **/
       subscribeRuntimeVersion: AugmentedRpc<() => Observable<RuntimeVersion>>;
-      subscribeStorage: AugmentedRpc<<T = Codec[]>(keys: any[]) => Observable<T>>;
+      /**
+       * Subscribes to storage changes for the provided keys
+       **/
+      subscribeStorage: AugmentedRpc<<T = Codec[]>(keys: Vec<StorageKey> | (StorageKey | string | Uint8Array | any)[]) => Observable<T>>;
     };
     system: {
       /**

packages/api/src/base/Decorate.ts

@@ -4,28 +4,30 @@
 
 import { Constants, Storage } from '@polkadot/metadata/Decorated/types';
 import { RpcInterface } from '@polkadot/rpc-core/types';
-import { Call, Hash, RuntimeVersion, StorageChangeSet } from '@polkadot/types/interfaces';
+import { Call, Hash, RuntimeVersion } from '@polkadot/types/interfaces';
 import { AnyFunction, CallFunction, Codec, CodecArg as Arg, ITuple, InterfaceTypes, ModulesWithCalls, Registry, RegistryTypes } from '@polkadot/types/types';
 import { SubmittableExtrinsic } from '../submittable/types';
 import { ApiInterfaceRx, ApiOptions, ApiTypes, DecorateMethod, DecoratedRpc, DecoratedRpcSection, QueryableModuleStorage, QueryableStorage, QueryableStorageEntry, QueryableStorageMulti, QueryableStorageMultiArg, SubmittableExtrinsicFunction, SubmittableExtrinsics, SubmittableModuleExtrinsics } from '../types';
 
 import BN from 'bn.js';
-import { BehaviorSubject, Observable, combineLatest, of } from 'rxjs';
-import { map, switchMap } from 'rxjs/operators';
+import { BehaviorSubject, Observable, combineLatest, from, of } from 'rxjs';
+import { concatMap, map, switchMap, take, tap, toArray } from 'rxjs/operators';
 import decorateDerive, { ExactDerive } from '@polkadot/api-derive';
+import { memo } from '@polkadot/api-derive/util';
 import DecoratedMeta from '@polkadot/metadata/Decorated';
 import getHasher from '@polkadot/metadata/Decorated/storage/fromMetadata/getHasher';
 import RpcCore from '@polkadot/rpc-core';
 import { WsProvider } from '@polkadot/rpc-provider';
-import { Metadata, Null, Option, Text, TypeRegistry, u64, Vec } from '@polkadot/types';
+import { Metadata, Null, Option, Raw, Text, TypeRegistry, u64 } from '@polkadot/types';
 import Linkage, { LinkageResult } from '@polkadot/types/codec/Linkage';
 import { DEFAULT_VERSION as EXTRINSIC_DEFAULT_VERSION } from '@polkadot/types/extrinsic/constants';
 import StorageKey, { StorageEntry, unwrapStorageType } from '@polkadot/types/primitive/StorageKey';
 import { assert, compactStripLength, isNull, isUndefined, u8aConcat, u8aToHex } from '@polkadot/util';
 
 import { createSubmittable } from '../submittable';
-import { decorateSections, DeriveAllSections } from '../util/decorate';
 import augmentObject from '../util/augmentObject';
+import { decorateSections, DeriveAllSections } from '../util/decorate';
+import { extractStorageArgs } from '../util/validate';
 import Events from './Events';
 
 interface MetaDecoration {
@@ -38,6 +40,9 @@ interface MetaDecoration {
 
 type LinkageData = ITuple<[Codec, Linkage<Codec>]>;
 
+const PAGE_SIZE_KEYS = 256;
+const PAGE_SIZE_VALS = PAGE_SIZE_KEYS / 2;
+
 export default abstract class Decorate<ApiType extends ApiTypes> extends Events {
   public readonly registry: Registry;
 
@@ -291,9 +296,7 @@ export default abstract class Decorate<ApiType extends ApiTypes> extends Events
 
   private decorateStorageEntry<ApiType extends ApiTypes> (creator: StorageEntry, decorateMethod: DecorateMethod<ApiType>): QueryableStorageEntry<ApiType> {
     // get the storage arguments, with DoubleMap as an array entry, otherwise spread
-    const getArgs = (...args: any[]): any[] => creator.meta.type.isDoubleMap
-      ? [creator, args]
-      : [creator, ...args];
+    const getArgs = (...args: any[]): any[] => extractStorageArgs(creator, args);
 
     // FIXME We probably want to be able to query the full list with non-subs as well
     const decorated = this.hasSubscriptions && creator.iterKey && creator.meta.type.isMap && creator.meta.type.asMap.linked.isTrue
@@ -314,8 +317,8 @@ export default abstract class Decorate<ApiType extends ApiTypes> extends Events
     decorated.at = decorateMethod((hash: Hash, arg1?: Arg, arg2?: Arg): Observable<Codec> =>
       this._rpcCore.state.getStorage(getArgs(arg1, arg2), hash));
 
-    decorated.entries = decorateMethod((doubleMapArg?: Arg): Observable<[StorageKey, Codec][]> =>
-      this.retrieveMapEntries(creator, doubleMapArg));
+    decorated.entries = decorateMethod(memo((doubleMapArg?: Arg): Observable<[StorageKey, Codec][]> =>
+      this.retrieveMapEntries(creator, doubleMapArg)));
 
     decorated.hash = decorateMethod((arg1?: Arg, arg2?: Arg): Observable<Hash> =>
       this._rpcCore.state.getStorageHash(getArgs(arg1, arg2)));
@@ -326,6 +329,9 @@ export default abstract class Decorate<ApiType extends ApiTypes> extends Events
     decorated.keyPrefix = (): string =>
       u8aToHex(creator.keyPrefix);
 
+    decorated.keys = decorateMethod(memo((doubleMapArg?: Arg): Observable<StorageKey[]> =>
+      this.retrieveMapKeys(creator, doubleMapArg)));
+
     if (this.hasSubscriptions) {
       // When using double map storage function, user need to pass double map key as an array
       decorated.multi = decorateMethod((args: (Arg | Arg[])[]): Observable<Codec[]> =>
@@ -333,8 +339,8 @@ export default abstract class Decorate<ApiType extends ApiTypes> extends Events
           args.map((arg: Arg[] | Arg): [StorageEntry, Arg | Arg[]] => [creator, arg])));
     }
 
-    decorated.range = decorateMethod((range: [Hash, Hash?], arg1?: Arg, arg2?: Arg): Observable<[Hash, Codec][]> =>
-      this.decorateStorageRange(decorated, [arg1, arg2], range));
+    decorated.range = decorateMethod(memo((range: [Hash, Hash?], arg1?: Arg, arg2?: Arg): Observable<[Hash, Codec][]> =>
+      this.decorateStorageRange(decorated, [arg1, arg2], range)));
 
     decorated.size = decorateMethod((arg1?: Arg, arg2?: Arg): Observable<u64> =>
       this._rpcCore.state.getStorageSize(getArgs(arg1, arg2)));
@@ -343,17 +349,15 @@ export default abstract class Decorate<ApiType extends ApiTypes> extends Events
   }
 
   private decorateStorageRange<ApiType extends ApiTypes> (decorated: QueryableStorageEntry<ApiType>, args: [Arg?, Arg?], range: [Hash, Hash?]): Observable<[Hash, Codec][]> {
-    const outputType = unwrapStorageType(decorated.creator.meta.type);
+    const outputType = unwrapStorageType(decorated.creator.meta.type, decorated.creator.meta.modifier.isOptional);
 
     return this._rpcCore.state
-      .queryStorage([decorated.key(...args)], ...range)
-      .pipe(map((result: StorageChangeSet[]): [Hash, Codec][] =>
-        result
-          .filter((change): change is StorageChangeSet => !!change.changes.length)
-          .map(({ block, changes: [[, value]] }): [Hash, Codec] => [
-            block,
-            this.createType(outputType, value.unwrapOrDefault())
-          ])
+      .queryStorage<[Option<Raw>]>([decorated.key(...args)], ...range)
+      .pipe(map((result: [Hash, [Option<Raw>]][]): [Hash, Codec][] =>
+        result.map(([blockHash, [value]]): [Hash, Codec] => [
+          blockHash,
+          this.createType(outputType, value.isSome ? value.unwrap().toHex() : undefined)
+        ])
       ));
   }
 
@@ -366,8 +370,8 @@ export default abstract class Decorate<ApiType extends ApiTypes> extends Events
     // entries can be re-linked in the middle of a list, we subscribe here to make
     // sure we catch any updates, no matter the list position
     const getNext = (key: Codec): Observable<LinkageResult> =>
-      this._rpcCore.state.subscribeStorage<[LinkageData | Option<LinkageData>]>([[creator, key]]).pipe(
-        switchMap(([_data]: [LinkageData | Option<LinkageData>]): Observable<LinkageResult> => {
+      this._rpcCore.state.getStorage<LinkageData | Option<LinkageData>>([creator, key]).pipe(
+        switchMap((_data: LinkageData | Option<LinkageData>): Observable<LinkageResult> => {
           const data = creator.meta.modifier.isOptional
             ? (_data as Option<LinkageData>).unwrapOr(null)
             : _data as LinkageData;
@@ -429,42 +433,84 @@ export default abstract class Decorate<ApiType extends ApiTypes> extends Events
     );
   }
 
-  private retrieveMapEntries (creator: StorageEntry, arg?: Arg): Observable<[StorageKey, Codec][]> {
-    assert(creator.iterKey && (creator.meta.type.isMap || creator.meta.type.isDoubleMap), 'entries can only be retrieved on maps, linked maps and double maps');
+  private retrieveMapKeys ({ iterKey, meta }: StorageEntry, arg?: Arg): Observable<StorageKey[]> {
+    assert(iterKey && (meta.type.isMap || meta.type.isDoubleMap), 'keys can only be retrieved on maps, linked maps and double maps');
 
-    const outputType = creator.meta.type.isMap
-      ? creator.meta.type.asMap.value.toString()
-      : creator.meta.type.asDoubleMap.value.toString();
+    const headKey = this.createType('Raw', u8aConcat(
+      iterKey,
+      meta.type.isDoubleMap && !isUndefined(arg) && !isNull(arg)
+        ? getHasher(meta.type.asDoubleMap.hasher)(
+          this.createType(meta.type.asDoubleMap.key1.toString() as 'Raw', arg).toU8a()
+        )
+        : new Uint8Array([])
+    )).toHex();
+    const startSubject = new BehaviorSubject<string>(headKey);
 
-    return this._rpcCore.state
-      // FIXME This should really be some sort of subscription, so we can get stuff as
-      // it changes (as of now it is a one-shot query). Not sure how to do this though...
-      .getKeys(
-        this.createType('Raw', u8aConcat(
-          creator.iterKey,
-          creator.meta.type.isDoubleMap && !isUndefined(arg) && !isNull(arg)
-            ? getHasher(creator.meta.type.asDoubleMap.hasher)(
-              this.createType(creator.meta.type.asDoubleMap.key1.toString() as 'Raw', arg).toU8a()
+    return this._rpcCore.state.getKeysPaged
+      ? startSubject.pipe(
+        switchMap((startKey) =>
+          this._rpcCore.state.getKeysPaged(headKey, PAGE_SIZE_KEYS, startKey).pipe(
+            map((keys): StorageKey[] =>
+              keys.map((key) => key.decodeArgsFromMeta(meta))
             )
-            : new Uint8Array([])
-        ))
-      )
-      .pipe(
-        switchMap((keys): Observable<[StorageKey[], Vec<Codec>]> =>
-          combineLatest([
-            of(keys),
-            // Not 100% sure about this, surely this is just a vec?
-            this._rpcCore.state.subscribeStorage<Vec<Codec>>(
-              keys.map((key) => key.setOutputType(outputType))
-            )
-          ])
+          )
         ),
-        map(([keys, values]): [StorageKey, Codec][] =>
-          keys.map((key, index): [StorageKey, Codec] => [key, values[index]])
+        tap((keys): void => {
+          keys.length === PAGE_SIZE_KEYS
+            ? startSubject.next(keys[PAGE_SIZE_KEYS - 1].toHex())
+            : startSubject.complete();
+        }),
+        toArray(), // toArray since we want to startSubject to be completed
+        map((keysArr: StorageKey[][]): StorageKey[] =>
+          keysArr.reduce((result: StorageKey[], keys): StorageKey[] =>
+            result.concat(keys), []
+          )
+        )
+      )
+      : this._rpcCore.state.getKeys(headKey).pipe(
+        map((keys): StorageKey[] =>
+          keys.map((key) => key.decodeArgsFromMeta(meta))
         )
       );
   }
 
+  private retrieveMapEntries (entry: StorageEntry, arg?: Arg): Observable<[StorageKey, Codec][]> {
+    const outputType = unwrapStorageType(entry.meta.type, entry.meta.modifier.isOptional);
+
+    return this.retrieveMapKeys(entry, arg).pipe(
+      switchMap((keys): Observable<[StorageKey[], Option<Raw>[]]> =>
+        combineLatest([
+          of(keys),
+          from(Array(Math.ceil(keys.length / PAGE_SIZE_VALS)).fill(0)).pipe(
+            // FIXME New RPC to take care of this in the works...
+            concatMap((_, index): Observable<Option<Raw>[]> =>
+              this._rpcCore.state.subscribeStorage<Option<Raw>[]>(
+                keys.slice(index * PAGE_SIZE_VALS, (index * PAGE_SIZE_VALS) + PAGE_SIZE_VALS)
+              ).pipe(take(1))
+            ),
+            toArray(),
+            map((valsArr: Option<Raw>[][]): Option<Raw>[] =>
+              valsArr.reduce((result: Option<Raw>[], vals): Option<Raw>[] =>
+                result.concat(vals), []
+              )
+            )
+          )
+        ])
+      ),
+      map(([keys, vals]): [StorageKey, Codec][] =>
+        keys.map((key, index): [StorageKey, Codec] => [
+          key,
+          this.createType(
+            outputType,
+            vals[index].isSome
+              ? vals[index].unwrap().toHex()
+              : undefined
+          )
+        ])
+      )
+    );
+  }
+
   protected decorateDeriveRx (decorateMethod: DecorateMethod<ApiType>): DeriveAllSections<'rxjs', ExactDerive> {
     // Pull in derive from api-derive
     const derive = decorateDerive(this._rx, this._options.derives);

packages/api/src/index.ts

@@ -6,7 +6,7 @@ import { detectPackage } from '@polkadot/util';
 
 // FIXME This really should be `import(...).then(...)`, but need to check rejections
 // eslint-disable-next-line @typescript-eslint/no-var-requires
-detectPackage(require('./package.json'), __dirname);
+detectPackage(require('./package.json'), typeof __dirname !== 'undefined' && __dirname);
 
 export { Keyring } from '@polkadot/keyring';
 export { WsProvider } from '@polkadot/rpc-provider';

packages/api/src/types/storage.ts

@@ -32,6 +32,7 @@ export interface StorageEntryBase<ApiType extends ApiTypes, F extends AnyFunctio
   hash: (...args: Parameters<F>) => PromiseOrObs<ApiType, Hash>;
   key: (...args: Parameters<F>) => string;
   keyPrefix: () => string;
+  keys: (arg?: any) => PromiseOrObs<ApiType, StorageKey[]>;
   range: <T extends Codec | any = ObsInnerType<ReturnType<F>>>([from, to]: [Hash | Uint8Array | string, Hash | Uint8Array | string | undefined] | [Hash | Uint8Array | string], ...args: Parameters<F>) => PromiseOrObs<ApiType, [Hash, T][]>;
   size: (...args: Parameters<F>) => PromiseOrObs<ApiType, u64>;
   multi: ApiType extends 'rxjs' ? StorageEntryObservableMulti : StorageEntryPromiseMulti;

packages/api/src/util/validate.spec.ts

@@ -0,0 +1,102 @@
+// Copyright 2017-2019 @polkadot/api authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { Storage } from '@polkadot/metadata/Decorated/types';
+
+import storageFromMeta from '@polkadot/metadata/Decorated/storage/fromMetadata';
+import Metadata from '@polkadot/metadata/Metadata';
+import metaStatic from '@polkadot/metadata/Metadata/static';
+import { TypeRegistry } from '@polkadot/types';
+
+import { extractStorageArgs } from './validate';
+
+describe('extractStorageArgs', (): void => {
+  const registry = new TypeRegistry();
+  let storage: Storage;
+
+  beforeEach((): void => {
+    const metadata = new Metadata(registry, metaStatic);
+
+    storage = storageFromMeta(registry, metadata);
+  });
+
+  it('validates no-arg plain', (): void => {
+    expect(
+      extractStorageArgs(storage.timestamp.now, [])
+    ).toEqual([storage.timestamp.now]);
+  });
+
+  it('validates no-arg plain (with undefined, undefined)', (): void => {
+    expect(
+      extractStorageArgs(storage.timestamp.now, [undefined, undefined])
+    ).toEqual([storage.timestamp.now]);
+  });
+
+  it('validates no-arg plain (failing when there are args)', (): void => {
+    expect(
+      (): any[] =>
+        extractStorageArgs(storage.timestamp.now, [123, 456])
+    ).toThrow('timestamp.now() does not take any arguments, 2 found');
+  });
+
+  it('validates map, 1 arg', (): void => {
+    expect(
+      extractStorageArgs(storage.staking.payee, ['abc'])
+    ).toEqual([storage.staking.payee, 'abc']);
+  });
+
+  it('validates map, 1 arg (failing with no args)', (): void => {
+    expect(
+      (): any =>
+        extractStorageArgs(storage.staking.payee, [])
+    ).toThrow('staking.payee(AccountId) is a map, requiring 1 argument, 0 found');
+  });
+
+  it('validates map, 1 arg (failing with no args)', (): void => {
+    expect(
+      (): any =>
+        extractStorageArgs(storage.staking.payee, ['abc', 'def'])
+    ).toThrow('staking.payee(AccountId) is a map, requiring 1 argument, 2 found');
+  });
+
+  it('validates doublemap, 2 args', (): void => {
+    expect(
+      extractStorageArgs(storage.staking.erasStakers, [1, '0x1234'])
+    ).toEqual([storage.staking.erasStakers, [1, '0x1234']]);
+  });
+
+  it('validates doublemap, 2 args (failing with no args)', (): void => {
+    expect(
+      (): any =>
+        extractStorageArgs(storage.staking.erasStakers, [])
+    ).toThrow('staking.erasStakers(EraIndex, AccountId) is a doublemap, requiring 2 arguments, 0 found');
+  });
+
+  it('validates doublemap, 2 args (failing with 1 arg)', (): void => {
+    expect(
+      (): any =>
+        extractStorageArgs(storage.staking.erasStakers, [123])
+    ).toThrow('staking.erasStakers(EraIndex, AccountId) is a doublemap, requiring 2 arguments, 1 found');
+  });
+
+  // Linked maps have been removed
+  it.skip('validates linked map, no args', (): void => {
+    expect(
+      extractStorageArgs(storage.staking.validators, [])
+    ).toEqual([storage.staking.validators]);
+  });
+
+  it.skip('validates linked map, single arg', (): void => {
+    expect(
+      extractStorageArgs(storage.staking.validators, [123])
+    ).toEqual([storage.staking.validators, 123]);
+  });
+
+  it.skip('validates linked map (failing with extra args)', (): void => {
+    expect(
+      (): any[] =>
+        extractStorageArgs(storage.staking.validators, [123, 456])
+    ).toThrow('staking.validators(AccountId) is a linked map, requiring either 0 arguments (retrieving all) or 1 argument, 2 found');
+  });
+});

packages/api/src/util/validate.ts

@@ -0,0 +1,50 @@
+// Copyright 2017-2019 @polkadot/api authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { StorageEntry } from '@polkadot/types/primitive/StorageKey';
+import { Type } from '@polkadot/types';
+import { assert, isUndefined } from '@polkadot/util';
+
+function sig ({ method, section }: StorageEntry, ...args: Type[]): string {
+  return `${section}.${method}(${args.join(', ')})`;
+}
+
+function doDoubleMap (creator: StorageEntry, args: any[]): [StorageEntry, [any, any]] {
+  const { key1, key2 } = creator.meta.type.asDoubleMap;
+
+  assert(args.length === 2, `${sig(creator, key1, key2)} is a doublemap, requiring 2 arguments, ${args.length} found`);
+
+  // pass as tuple
+  return [creator, args as [any, any]];
+}
+
+function doMap (creator: StorageEntry, args: any[]): [StorageEntry] | [StorageEntry, any] {
+  const { key, linked } = creator.meta.type.asMap;
+
+  linked.isTrue
+    ? assert(args.length <= 1, `${sig(creator, key)} is a linked map, requiring either 0 arguments (retrieving all) or 1 argument, ${args.length} found`)
+    : assert(args.length === 1, `${sig(creator, key)} is a map, requiring 1 argument, ${args.length} found`);
+
+  // expand
+  return args.length
+    ? [creator, args[0]]
+    : [creator];
+}
+
+// sets up the arguments in the form of [creator, args] ready to be used in a storage
+// call. Additionally, it verifies that the correct number of arguments have been passed
+export function extractStorageArgs (creator: StorageEntry, _args: any[]): any[] {
+  const args = _args.filter((arg) => !isUndefined(arg));
+
+  if (creator.meta.type.isDoubleMap) {
+    return doDoubleMap(creator, args);
+  } else if (creator.meta.type.isMap) {
+    return doMap(creator, args);
+  }
+
+  assert(args.length === 0, `${sig(creator)} does not take any arguments, ${args.length} found`);
+
+  // no args
+  return [creator];
+}

packages/api/test/api.spec.ts

@@ -0,0 +1,78 @@
+// Copyright 2017-2020 @polkadot/api authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { ApiPromise, WsProvider } from '@polkadot/api';
+
+function createApi (): Promise<ApiPromise> {
+  jest.setTimeout(30000);
+  process.env.NODE_ENV = 'test';
+
+  const provider = new WsProvider('wss://kusama-rpc.polkadot.io');
+
+  return new ApiPromise({ provider }).isReady;
+}
+
+describe.skip('misc quick tests', (): void => {
+  it.skip('retrieves balances correctly', async (): Promise<void> => {
+    const api = await createApi();
+
+    console.error(JSON.stringify(
+      await api.query.system.account('FPzukZw2mphWsXDqdkNzLaxnanjZEWH9i2vqwobTdtde5me')
+    ));
+    console.error(JSON.stringify(
+      await api.query.system.account('HUewJvzVuEeyaxH2vx9XiyAPKrpu1Zj5r5Pi9VrGiBVty7q')
+    ));
+  });
+
+  it.skip('handles map keys', async (): Promise<void> => {
+    const api = await createApi();
+
+    console.time('map.keys');
+
+    const keys = await api.query.system.account.keys();
+
+    console.error('# keys', keys.length);
+
+    console.timeEnd('map.keys');
+  });
+
+  it.skip('handles map entries', async (): Promise<void> => {
+    const api = await createApi();
+
+    console.time('map.entries');
+
+    const entries = await api.query.system.account.entries(); // api.query.indices.accounts.entries();
+
+    console.error('# entries', entries.length);
+
+    console.timeEnd('map.entries');
+  });
+
+  it.skip('handles doublemap entries', async (): Promise<void> => {
+    const api = await createApi();
+    const activeEra = await api.query.staking.activeEra();
+
+    console.error(JSON.stringify(
+      await api.query.staking.erasStakers.entries(activeEra.unwrapOrDefault().index)
+    ));
+  });
+
+  it.skip('does something in society', async (): Promise<void> => {
+    const api = await createApi();
+
+    console.error(JSON.stringify(
+      await api.query.society.defenderVotes('Dab4bfYTZRUDMWjYAUQuFbDreQ9mt7nULWu3Dw7jodbzVe9')
+    ));
+  });
+
+  it.skip('allows for range queries', async (): Promise<void> => {
+    const api = await createApi();
+    const header = await api.rpc.chain.getHeader();
+
+    console.error(JSON.stringify(
+      (await api.query.staking.activeEra.range([header.parentHash, header.hash], 'Dab4bfYTZRUDMWjYAUQuFbDreQ9mt7nULWu3Dw7jodbzVe9'))
+        .map(([block, value]) => [block, value.toRawType(), value.toHuman()])
+    ));
+  });
+});

packages/jsonrpc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/jsonrpc",
-  "version": "1.5.1",
+  "version": "1.7.1",
   "description": "Method definitions for the Polkadot RPC layer",
   "main": "index.js",
   "publishConfig": {
@@ -26,8 +26,8 @@
   },
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/jsonrpc#readme",
   "dependencies": {
-    "@babel/runtime": "^7.8.4",
-    "@polkadot/types": "1.5.1",
-    "@polkadot/util": "^2.6.1"
+    "@babel/runtime": "^7.8.7",
+    "@polkadot/types": "1.7.1",
+    "@polkadot/util": "^2.6.2"
   }
 }

packages/jsonrpc/src/state.ts

@@ -30,6 +30,17 @@ const getKeys: RpcMethodOpt = {
   type: 'Vec<StorageKey>'
 };
 
+const getKeysPaged: RpcMethodOpt = {
+  description: 'Returns the keys with prefix with pagination support.',
+  params: [
+    createParam('key', 'StorageKey'),
+    createParam('count', 'u32'),
+    createParam('startKey', 'StorageKey', { isOptional: true }),
+    createParam('at', 'BlockHash', { isOptional: true })
+  ],
+  type: 'Vec<StorageKey>'
+};
+
 const getStorage: RpcMethodOpt = {
   description: 'Retrieves the storage for a key',
   params: KEY_QUERY_PARAMS,
@@ -148,6 +159,7 @@ export default {
     getChildStorageHash: createMethod(section, 'getChildStorageHash', getChildStorageHash),
     getChildStorageSize: createMethod(section, 'getChildStorageSize', getChildStorageSize),
     getKeys: createMethod(section, 'getKeys', getKeys),
+    getKeysPaged: createMethod(section, 'getKeysPaged', getKeysPaged),
     getMetadata: createMethod(section, 'getMetadata', getMetadata),
     getRuntimeVersion: createMethod(section, 'getRuntimeVersion', getRuntimeVersion),
     getStorage: createMethod(section, 'getStorage', getStorage),

packages/metadata/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/metadata",
-  "version": "1.5.1",
+  "version": "1.7.1",
   "description": "Helpers to extract information from runtime metadata",
   "main": "index.js",
   "publishConfig": {
@@ -26,13 +26,13 @@
   },
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/type-metadata#readme",
   "dependencies": {
-    "@babel/runtime": "^7.8.4",
-    "@polkadot/types": "1.5.1",
-    "@polkadot/util": "^2.6.1",
-    "@polkadot/util-crypto": "^2.6.1",
+    "@babel/runtime": "^7.8.7",
+    "@polkadot/types": "1.7.1",
+    "@polkadot/util": "^2.6.2",
+    "@polkadot/util-crypto": "^2.6.2",
     "bn.js": "^5.1.1"
   },
   "devDependencies": {
-    "@polkadot/keyring": "^2.6.1"
+    "@polkadot/keyring": "^2.6.2"
   }
 }

packages/metadata/src/Decorated/Decorated.spec.ts

@@ -18,7 +18,7 @@ describe('Decorated', () => {
         decorated.query.system.account('5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY')
       )
     ).toEqual(
-      '0x010126aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da92e3fb4c297a84c5cebc0e78257d213d0927ccc7596044c6ba013dd05522aacba'
+      '0x410126aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da9de1e86a9a8c739864cf3cc5ec2bea59fd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d'
     );
   });
 

packages/metadata/src/Decorated/consts/fromMetadata/fromMetadata.spec.ts

@@ -19,7 +19,8 @@ describe('fromMetadata', (): void => {
     expect(consts.democracy.cooloffPeriod.toNumber()).toEqual(28 * 24 * 60 * (60 / 3));
   });
 
-  it('correctly handles bytes', (): void => {
+  // removed from session
+  it.skip('correctly handles bytes', (): void => {
     // 0x34 removes as the length prefix removed
     expect(consts.session.dedupKeyPrefix.toHex()).toEqual('0x3a73657373696f6e3a6b657973');
   });

packages/metadata/src/Decorated/storage/fromMetadata/fromMetadata.spec.ts

@@ -31,8 +31,7 @@ describe('fromMetadata', (): void => {
           decorated.query.system.account(keyring.alice.address)
         )
       ).toEqual(
-        // new storage key format
-        '0x010126aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da92e3fb4c297a84c5cebc0e78257d213d0927ccc7596044c6ba013dd05522aacba'
+        '0x410126aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da9de1e86a9a8c739864cf3cc5ec2bea59fd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d'
       );
     });
   });

packages/metadata/src/Decorated/storage/fromMetadata/getHasher.ts

@@ -2,6 +2,7 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
+import metadataDefs from '@polkadot/types/interfaces/metadata/definitions';
 import { StorageHasher } from '@polkadot/types/interfaces';
 import { u8aConcat, u8aToU8a } from '@polkadot/util';
 import { blake2AsU8a, xxhashAsU8a } from '@polkadot/util-crypto';
@@ -10,24 +11,26 @@ export type HasherInput = string | Buffer | Uint8Array;
 
 export type HasherFunction = (data: HasherInput) => Uint8Array;
 
-type HasherCheck = 'isBlake2128' | 'isBlake2128Concat' | 'isBlake2256' | 'isTwox128' | 'isTwox256' | 'isTwox64Concat';
+const DEFAULT_FN = (data: HasherInput): Uint8Array => xxhashAsU8a(data, 128);
 
-const DEFAULT = (data: HasherInput): Uint8Array => xxhashAsU8a(data, 128);
-
-const map: Record<HasherCheck, HasherFunction> = {
-  isBlake2128: (data: HasherInput): Uint8Array => blake2AsU8a(data, 128),
-  isBlake2128Concat: (data: HasherInput): Uint8Array => u8aConcat(blake2AsU8a(data, 128), u8aToU8a(data)),
-  isBlake2256: (data: HasherInput): Uint8Array => blake2AsU8a(data, 256),
-  isTwox128: (data: HasherInput): Uint8Array => xxhashAsU8a(data, 128),
-  isTwox256: (data: HasherInput): Uint8Array => xxhashAsU8a(data, 256),
-  isTwox64Concat: (data: HasherInput): Uint8Array => u8aConcat(xxhashAsU8a(data, 64), u8aToU8a(data))
+const HASHERS: Record<keyof typeof metadataDefs.types.StorageHasherV11._enum, HasherFunction> = {
+  Blake2_128: (data: HasherInput): Uint8Array => // eslint-disable-line @typescript-eslint/camelcase
+    blake2AsU8a(data, 128),
+  Blake2_128Concat: (data: HasherInput): Uint8Array => // eslint-disable-line @typescript-eslint/camelcase
+    u8aConcat(blake2AsU8a(data, 128), u8aToU8a(data)),
+  Blake2_256: (data: HasherInput): Uint8Array => // eslint-disable-line @typescript-eslint/camelcase
+    blake2AsU8a(data, 256),
+  Identity: (data: HasherInput): Uint8Array =>
+    u8aToU8a(data),
+  Twox128: (data: HasherInput): Uint8Array =>
+    xxhashAsU8a(data, 128),
+  Twox256: (data: HasherInput): Uint8Array =>
+    xxhashAsU8a(data, 256),
+  Twox64Concat: (data: HasherInput): Uint8Array =>
+    u8aConcat(xxhashAsU8a(data, 64), u8aToU8a(data))
 };
 
 /** @internal */
 export default function getHasher (hasher?: StorageHasher): HasherFunction {
-  const [, fn] = (hasher && Object.entries(map).find(([check]): boolean =>
-    hasher[check as HasherCheck]
-  )) || [undefined, DEFAULT];
-
-  return fn;
+  return HASHERS[hasher?.type as 'Identity'] || DEFAULT_FN;
 }

packages/metadata/src/Metadata/v11/static-substrate.json

@@ -13,7 +13,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Blake2_128Concat",
                     "key": "AccountId",
                     "value": "AccountInfo",
                     "linked": false
@@ -62,7 +62,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "BlockNumber",
                     "value": "Hash",
                     "linked": false
@@ -160,7 +160,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Blake2_128Concat",
                     "key": "Hash",
                     "value": "Vec<(BlockNumber,EventIndex)>",
                     "linked": false
@@ -181,14 +181,14 @@
                 ]
               },
               {
-                "name": "RuntimeUpgraded",
-                "modifier": "Default",
+                "name": "LastRuntimeUpgrade",
+                "modifier": "Optional",
                 "type": {
-                  "Plain": "bool"
+                  "Plain": "LastRuntimeUpgradeInfo"
                 },
                 "fallback": "0x00",
                 "documentation": [
-                  " A bool to track if the runtime was upgraded last block."
+                  " Stores the `spec_version` and `spec_name` of when the last runtime upgrade happened."
                 ]
               }
             ]
@@ -309,6 +309,16 @@
                 " Kill the sending account, assuming there are no references outstanding and the composite",
                 " data is equal to its default value."
               ]
+            },
+            {
+              "name": "migrate_accounts",
+              "args": [
+                {
+                  "name": "accounts",
+                  "type": "Vec<AccountId>"
+                }
+              ],
+              "documentation": []
             }
           ],
           "events": [
@@ -367,26 +377,12 @@
               ]
             },
             {
-              "name": "SpecVersionNotAllowedToDecrease",
+              "name": "SpecVersionNeedsToIncrease",
               "documentation": [
                 " The specification version is not allowed to decrease between the current runtime",
                 " and the new runtime."
               ]
             },
-            {
-              "name": "ImplVersionNotAllowedToDecrease",
-              "documentation": [
-                " The implementation version is not allowed to decrease between the current runtime",
-                " and the new runtime."
-              ]
-            },
-            {
-              "name": "SpecOrImplVersionNeedToIncrease",
-              "documentation": [
-                " The specification or the implementation version need to increase between the",
-                " current runtime and the new runtime."
-              ]
-            },
             {
               "name": "FailedToExtractRuntimeVersion",
               "documentation": [
@@ -899,7 +895,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "u32",
                     "value": "Vec<[u8;32]>",
                     "linked": false
@@ -928,7 +924,7 @@
             {
               "name": "EpochDuration",
               "type": "u64",
-              "value": "0x2800000000000000",
+              "value": "0xc800000000000000",
               "documentation": [
                 " The number of **slots** that an epoch takes. We couple sessions to",
                 " epochs, i.e. we start a new session once the new epoch begins."
@@ -1303,7 +1299,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Blake2_128Concat",
                     "key": "AccountId",
                     "value": "AccountData",
                     "linked": false
@@ -1324,7 +1320,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Blake2_128Concat",
                     "key": "AccountId",
                     "value": "Vec<BalanceLock>",
                     "linked": false
@@ -1585,7 +1581,7 @@
         {
           "name": "TransactionPayment",
           "storage": {
-            "prefix": "Balances",
+            "prefix": "TransactionPayment",
             "items": [
               {
                 "name": "NextFeeMultiplier",
@@ -1682,7 +1678,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "AccountId",
                     "linked": false
@@ -1698,7 +1694,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Blake2_128Concat",
                     "key": "AccountId",
                     "value": "StakingLedger",
                     "linked": false
@@ -1714,7 +1710,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "RewardDestination",
                     "linked": false
@@ -1730,10 +1726,10 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "ValidatorPrefs",
-                    "linked": true
+                    "linked": false
                   }
                 },
                 "fallback": "0x00",
@@ -1746,10 +1742,10 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "Nominations",
-                    "linked": true
+                    "linked": false
                   }
                 },
                 "fallback": "0x00",
@@ -1790,7 +1786,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "EraIndex",
                     "value": "SessionIndex",
                     "linked": false
@@ -1841,6 +1837,7 @@
                   "",
                   " This is similar to [`ErasStakers`] but number of nominators exposed is reduce to the",
                   " `T::MaxNominatorRewardedPerValidator` biggest stakers.",
+                  " (Note: the field `total` and `own` of the exposure remains unchanged).",
                   " This is used to limit the i/o cost for the nominator payout.",
                   "",
                   " This is keyed fist by the era index to allow bulk deletion and then the stash account.",
@@ -1865,7 +1862,7 @@
                 "documentation": [
                   " Similarly to `ErasStakers` this holds the preferences of validators.",
                   "",
-                  " This is keyed fist by the era index to allow bulk deletion and then the stash account.",
+                  " This is keyed first by the era index to allow bulk deletion and then the stash account.",
                   "",
                   " Is it removed after `HISTORY_DEPTH` eras."
                 ]
@@ -1875,7 +1872,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "EraIndex",
                     "value": "BalanceOf",
                     "linked": false
@@ -1893,7 +1890,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "EraIndex",
                     "value": "EraRewardPoints",
                     "linked": false
@@ -1910,7 +1907,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "EraIndex",
                     "value": "BalanceOf",
                     "linked": false
@@ -1963,7 +1960,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "EraIndex",
                     "value": "Vec<UnappliedSlash>",
                     "linked": false
@@ -1993,11 +1990,11 @@
                 "modifier": "Optional",
                 "type": {
                   "DoubleMap": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key1": "EraIndex",
                     "key2": "AccountId",
                     "value": "(Perbill,BalanceOf)",
-                    "key2Hasher": "Twox128"
+                    "key2Hasher": "Twox64Concat"
                   }
                 },
                 "fallback": "0x00",
@@ -2011,11 +2008,11 @@
                 "modifier": "Optional",
                 "type": {
                   "DoubleMap": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key1": "EraIndex",
                     "key2": "AccountId",
                     "value": "BalanceOf",
-                    "key2Hasher": "Twox128"
+                    "key2Hasher": "Twox64Concat"
                   }
                 },
                 "fallback": "0x00",
@@ -2028,7 +2025,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "SlashingSpans",
                     "linked": false
@@ -2044,7 +2041,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "(AccountId,SpanIndex)",
                     "value": "SpanRecord",
                     "linked": false
@@ -2073,7 +2070,7 @@
                 "type": {
                   "Plain": "ReleasesStaking"
                 },
-                "fallback": "0x00",
+                "fallback": "0x01",
                 "documentation": [
                   " Storage version of the pallet.",
                   "",
@@ -2107,6 +2104,8 @@
                 "",
                 " The dispatch origin for this call must be _Signed_ by the stash account.",
                 "",
+                " Emits `Bonded`.",
+                "",
                 " # <weight>",
                 " - Independent of the arguments. Moderate complexity.",
                 " - O(1).",
@@ -2135,6 +2134,8 @@
                 "",
                 " The dispatch origin for this call must be _Signed_ by the stash, not the controller.",
                 "",
+                " Emits `Bonded`.",
+                "",
                 " # <weight>",
                 " - Independent of the arguments. Insignificant complexity.",
                 " - O(1).",
@@ -2164,6 +2165,8 @@
                 "",
                 " The dispatch origin for this call must be _Signed_ by the controller, not the stash.",
                 "",
+                " Emits `Unbonded`.",
+                "",
                 " See also [`Call::withdraw_unbonded`].",
                 "",
                 " # <weight>",
@@ -2188,6 +2191,8 @@
                 "",
                 " The dispatch origin for this call must be _Signed_ by the controller, not the stash.",
                 "",
+                " Emits `Withdrawn`.",
+                "",
                 " See also [`Call::unbond`].",
                 "",
                 " # <weight>",
@@ -2414,7 +2419,10 @@
                 " - `who` is the controller account of the nominator to pay out.",
                 " - `era` may not be lower than one following the most recently paid era. If it is higher,",
                 "   then it indicates an instruction to skip the payout of all previous eras.",
-                " - `validators` is the list of all validators that `who` had exposure to during `era`.",
+                " - `validators` is the list of all validators that `who` had exposure to during `era`,",
+                "   alongside the index of `who` in the clipped exposure of the validator.",
+                "   I.e. each element is a tuple of",
+                "   `(validator, index of `who` in clipped exposure of validator)`.",
                 "   If it is incomplete, then less than the full reward will be paid out.",
                 "   It must not exceed `MAX_NOMINATIONS`.",
                 "",
@@ -2540,6 +2548,40 @@
                 " An old slashing report from a prior era was discarded because it could",
                 " not be processed."
               ]
+            },
+            {
+              "name": "Bonded",
+              "args": [
+                "AccountId",
+                "Balance"
+              ],
+              "documentation": [
+                " An account has bonded this amount.",
+                "",
+                " NOTE: This event is only emitted when funds are bonded via a dispatchable. Notably,",
+                " it will not be emitted for staking rewards when they are added to stake."
+              ]
+            },
+            {
+              "name": "Unbonded",
+              "args": [
+                "AccountId",
+                "Balance"
+              ],
+              "documentation": [
+                " An account has unbonded this amount."
+              ]
+            },
+            {
+              "name": "Withdrawn",
+              "args": [
+                "AccountId",
+                "Balance"
+              ],
+              "documentation": [
+                " An account has called `withdraw_unbonded` and removed unbonding chunks worth `Balance`",
+                " from the unlocking queue."
+              ]
             }
           ],
           "constants": [
@@ -2638,6 +2680,12 @@
               "documentation": [
                 " Invalid number of nominations."
               ]
+            },
+            {
+              "name": "NotSortedAndUnique",
+              "documentation": [
+                " Items are not sorted and unique."
+              ]
             }
           ]
         },
@@ -2709,40 +2757,32 @@
                 "name": "NextKeys",
                 "modifier": "Optional",
                 "type": {
-                  "DoubleMap": {
+                  "Map": {
                     "hasher": "Twox64Concat",
-                    "key1": "Bytes",
-                    "key2": "ValidatorId",
+                    "key": "ValidatorId",
                     "value": "Keys",
-                    "key2Hasher": "Blake2_256"
+                    "linked": false
                   }
                 },
                 "fallback": "0x00",
                 "documentation": [
-                  " The next session keys for a validator.",
-                  "",
-                  " The first key is always `DEDUP_KEY_PREFIX` to have all the data in the same branch of",
-                  " the trie. Having all data in the same branch should prevent slowing down other queries."
+                  " The next session keys for a validator."
                 ]
               },
               {
                 "name": "KeyOwner",
                 "modifier": "Optional",
                 "type": {
-                  "DoubleMap": {
+                  "Map": {
                     "hasher": "Twox64Concat",
-                    "key1": "Bytes",
-                    "key2": "(KeyTypeId,Bytes)",
+                    "key": "(KeyTypeId,Bytes)",
                     "value": "ValidatorId",
-                    "key2Hasher": "Blake2_256"
+                    "linked": false
                   }
                 },
                 "fallback": "0x00",
                 "documentation": [
-                  " The owner of a key. The second key is the `KeyTypeId` + the encoded key.",
-                  "",
-                  " The first key is always `DEDUP_KEY_PREFIX` to have all the data in the same branch of",
-                  " the trie. Having all data in the same branch should prevent slowing down other queries."
+                  " The owner of a key. The key is the `KeyTypeId` + the encoded key."
                 ]
               }
             ]
@@ -2804,17 +2844,7 @@
               ]
             }
           ],
-          "constants": [
-            {
-              "name": "DEDUP_KEY_PREFIX",
-              "type": "Bytes",
-              "value": "0x343a73657373696f6e3a6b657973",
-              "documentation": [
-                " Used as first key for `NextKeys` and `KeyOwner` to put all the data into the same branch",
-                " of the trie."
-              ]
-            }
-          ],
+          "constants": [],
           "errors": [
             {
               "name": "InvalidProof",
@@ -2874,7 +2904,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Identity",
                     "key": "Hash",
                     "value": "(Bytes,AccountId,BalanceOf,BlockNumber)",
                     "linked": false
@@ -2891,7 +2921,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "PropIndex",
                     "value": "(BalanceOf,Vec<AccountId>)",
                     "linked": false
@@ -2930,7 +2960,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "ReferendumIndex",
                     "value": "ReferendumInfo",
                     "linked": false
@@ -2957,7 +2987,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "ReferendumIndex",
                     "value": "Vec<AccountId>",
                     "linked": false
@@ -2973,7 +3003,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "(ReferendumIndex,AccountId)",
                     "value": "Vote",
                     "linked": false
@@ -2992,7 +3022,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "ProxyState",
                     "linked": false
@@ -3009,10 +3039,10 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "(AccountId,Conviction)",
-                    "linked": true
+                    "linked": false
                   }
                 },
                 "fallback": "0x000000000000000000000000000000000000000000000000000000000000000000",
@@ -3025,7 +3055,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "BlockNumber",
                     "linked": false
@@ -3068,7 +3098,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Identity",
                     "key": "Hash",
                     "value": "(BlockNumber,Vec<AccountId>)",
                     "linked": false
@@ -3085,7 +3115,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Identity",
                     "key": "Hash",
                     "value": "bool",
                     "linked": false
@@ -3260,7 +3290,7 @@
                 " an external referendum.",
                 "",
                 " The dispatch of this call must be `ExternalMajorityOrigin`.",
-                " ",
+                "",
                 " - `proposal_hash`: The preimage hash of the proposal.",
                 "",
                 " Unlike `external_propose`, blacklisting has no effect on this and it may replace a",
@@ -3347,7 +3377,7 @@
                 " Veto and blacklist the external proposal hash.",
                 "",
                 " The dispatch origin of this call must be `VetoOrigin`.",
-                " ",
+                "",
                 " - `proposal_hash`: The preimage hash of the proposal to veto and blacklist.",
                 "",
                 " Emits `Vetoed`.",
@@ -3543,7 +3573,7 @@
                 " Emits `PreimageNoted`.",
                 "",
                 " # <weight>",
-                " - Dependent on the size of `encoded_proposal` but protected by a ",
+                " - Dependent on the size of `encoded_proposal` but protected by a",
                 "   required deposit.",
                 " # </weight>"
               ]
@@ -4062,7 +4092,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Identity",
                     "key": "Hash",
                     "value": "Proposal",
                     "linked": false
@@ -4078,7 +4108,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Identity",
                     "key": "Hash",
                     "value": "Votes",
                     "linked": false
@@ -4377,7 +4407,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Identity",
                     "key": "Hash",
                     "value": "Proposal",
                     "linked": false
@@ -4393,7 +4423,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Identity",
                     "key": "Hash",
                     "value": "Votes",
                     "linked": false
@@ -4714,10 +4744,10 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "Vec<AccountId>",
-                    "linked": true
+                    "linked": false
                   }
                 },
                 "fallback": "0x00",
@@ -4730,7 +4760,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "BalanceOf",
                     "linked": false
@@ -5361,7 +5391,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "SetId",
                     "value": "SessionIndex",
                     "linked": false
@@ -5369,7 +5399,8 @@
                 },
                 "fallback": "0x00",
                 "documentation": [
-                  " A mapping from grandpa set ID to the index of the *most recent* session for which its members were responsible."
+                  " A mapping from grandpa set ID to the index of the *most recent* session for which its",
+                  " members were responsible."
                 ]
               }
             ]
@@ -5464,7 +5495,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "ProposalIndex",
                     "value": "TreasuryProposal",
                     "linked": false
@@ -5509,7 +5540,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Twox64Concat",
+                    "hasher": "Identity",
                     "key": "Hash",
                     "value": "Bytes",
                     "linked": false
@@ -5974,7 +6005,7 @@
         {
           "name": "Contracts",
           "storage": {
-            "prefix": "Contract",
+            "prefix": "Contracts",
             "items": [
               {
                 "name": "GasSpent",
@@ -6003,7 +6034,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Identity",
                     "key": "CodeHash",
                     "value": "Bytes",
                     "linked": false
@@ -6019,7 +6050,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Identity",
                     "key": "CodeHash",
                     "value": "PrefabWasmModule",
                     "linked": false
@@ -6046,7 +6077,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "ContractInfo",
                     "linked": false
@@ -6616,17 +6647,17 @@
                 "modifier": "Optional",
                 "type": {
                   "DoubleMap": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key1": "SessionIndex",
                     "key2": "AuthIndex",
                     "value": "Bytes",
-                    "key2Hasher": "Blake2_256"
+                    "key2Hasher": "Twox64Concat"
                   }
                 },
                 "fallback": "0x00",
                 "documentation": [
-                  " For each session index, we keep a mapping of `AuthIndex`",
-                  " to `offchain::OpaqueNetworkState`."
+                  " For each session index, we keep a mapping of `AuthIndex` to",
+                  " `offchain::OpaqueNetworkState`."
                 ]
               },
               {
@@ -6634,11 +6665,11 @@
                 "modifier": "Default",
                 "type": {
                   "DoubleMap": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key1": "SessionIndex",
                     "key2": "ValidatorId",
                     "value": "u32",
-                    "key2Hasher": "Blake2_256"
+                    "key2Hasher": "Twox64Concat"
                   }
                 },
                 "fallback": "0x00000000",
@@ -6726,7 +6757,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "ReportIdOf",
                     "value": "OffenceDetails",
                     "linked": false
@@ -6742,11 +6773,11 @@
                 "modifier": "Default",
                 "type": {
                   "DoubleMap": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key1": "Kind",
                     "key2": "OpaqueTimeSlot",
                     "value": "Vec<ReportIdOf>",
-                    "key2Hasher": "Blake2_256"
+                    "key2Hasher": "Twox64Concat"
                   }
                 },
                 "fallback": "0x00",
@@ -6759,7 +6790,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "Kind",
                     "value": "Bytes",
                     "linked": false
@@ -6822,14 +6853,14 @@
         {
           "name": "Identity",
           "storage": {
-            "prefix": "Sudo",
+            "prefix": "Identity",
             "items": [
               {
                 "name": "IdentityOf",
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "Registration",
                     "linked": false
@@ -6845,7 +6876,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Blake2_128Concat",
                     "key": "AccountId",
                     "value": "(AccountId,Data)",
                     "linked": false
@@ -6862,7 +6893,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "(BalanceOf,Vec<AccountId>)",
                     "linked": false
@@ -7399,7 +7430,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "(BalanceOf,BidKind)",
                     "linked": false
@@ -7448,7 +7479,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "bool",
                     "linked": false
@@ -7475,7 +7506,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "VouchingStatus",
                     "linked": false
@@ -7491,7 +7522,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "Vec<(BlockNumber,BalanceOf)>",
                     "linked": false
@@ -7507,7 +7538,7 @@
                 "modifier": "Default",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "StrikeCount",
                     "linked": false
@@ -8329,7 +8360,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Twox64Concat",
                     "key": "AccountId",
                     "value": "RecoveryConfig",
                     "linked": false
@@ -8365,7 +8396,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Blake2_128Concat",
                     "key": "AccountId",
                     "value": "AccountId",
                     "linked": false
@@ -8839,7 +8870,7 @@
                 "modifier": "Optional",
                 "type": {
                   "Map": {
-                    "hasher": "Blake2_256",
+                    "hasher": "Blake2_128Concat",
                     "key": "AccountId",
                     "value": "VestingInfo",
                     "linked": false
@@ -8912,7 +8943,7 @@
                 }
               ],
               "documentation": [
-                " Create a vested transfer. ",
+                " Create a vested transfer.",
                 "",
                 " The dispatch origin for this call must be _Signed_.",
                 "",

packages/rpc-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/rpc-core",
-  "version": "1.5.1",
+  "version": "1.7.1",
   "description": "A JavaScript wrapper for the Polkadot JsonRPC interface",
   "main": "index.js",
   "keywords": [
@@ -26,16 +26,16 @@
   },
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/rpc-core#readme",
   "dependencies": {
-    "@babel/runtime": "^7.8.4",
-    "@polkadot/jsonrpc": "1.5.1",
-    "@polkadot/metadata": "1.5.1",
-    "@polkadot/rpc-provider": "1.5.1",
-    "@polkadot/types": "1.5.1",
-    "@polkadot/util": "^2.6.1",
+    "@babel/runtime": "^7.8.7",
+    "@polkadot/jsonrpc": "1.7.1",
+    "@polkadot/metadata": "1.7.1",
+    "@polkadot/rpc-provider": "1.7.1",
+    "@polkadot/types": "1.7.1",
+    "@polkadot/util": "^2.6.2",
     "memoizee": "^0.4.14",
     "rxjs": "^6.5.4"
   },
   "devDependencies": {
-    "@polkadot/keyring": "^2.6.1"
+    "@polkadot/keyring": "^2.6.2"
   }
 }

packages/rpc-core/src/index.ts

@@ -4,6 +4,7 @@
 
 import { ProviderInterface, ProviderInterfaceCallback } from '@polkadot/rpc-provider/types';
 import { RpcMethod, RpcSection, RpcParam } from '@polkadot/jsonrpc/types';
+import { Hash } from '@polkadot/types/interfaces';
 import { AnyJson, Codec, Registry } from '@polkadot/types/types';
 import { RpcInterface, RpcInterfaceMethod, UserRpc } from './types';
 
@@ -349,26 +350,12 @@ export default class Rpc implements RpcInterface {
         throw error;
       }
     } else if (method.type === 'StorageChangeSet') {
-      // For StorageChangeSet, the changes has the [key, value] mappings
-      const changes: [string, string | null][] = result.changes;
-      const keys = params[0] as Vec<StorageKey>;
-      const withCache = keys.length !== 1;
-
-      // multiple return values (via state.storage subscription), decode the values
-      // one at a time, all based on the query types. Three values can be returned -
-      //   - Codec - There is a valid value, non-empty
-      //   - null - The storage key is empty
-      return keys.reduce((results, key: StorageKey): Codec[] => {
-        try {
-          results.push(this.formatStorageSet(key, changes, withCache));
-        } catch (error) {
-          console.error(`Unable to decode storage ${key.section}.${key.method}:`, error.message);
-
-          throw error;
-        }
-
-        return results;
-      }, [] as Codec[]);
+      return this.formatStorageSet(params[0] as Vec<StorageKey>, result.changes);
+    } else if (method.type === 'Vec<StorageChangeSet>') {
+      return result.map(({ block, changes }: { block: string; changes: [string, string | null][] }): [Hash, Codec[]] => [
+        this.registry.createType('Hash', block),
+        this.formatStorageSet(params[0] as Vec<StorageKey>, changes)
+      ]);
     }
 
     return createTypeUnsafe(this.registry, method.type, [result]);
@@ -402,7 +389,28 @@ export default class Rpc implements RpcInterface {
     return createTypeUnsafe(this.registry, type, [isEmpty ? meta.fallback : input], true);
   }
 
-  private formatStorageSet (key: StorageKey, changes: [string, string | null][], witCache: boolean): Codec {
+  private formatStorageSet (keys: Vec<StorageKey>, changes: [string, string | null][]): Codec[] {
+    // For StorageChangeSet, the changes has the [key, value] mappings
+    const withCache = keys.length !== 1;
+
+    // multiple return values (via state.storage subscription), decode the values
+    // one at a time, all based on the query types. Three values can be returned -
+    //   - Codec - There is a valid value, non-empty
+    //   - null - The storage key is empty
+    return keys.reduce((results: Codec[], key: StorageKey): Codec[] => {
+      try {
+        results.push(this.formatStorageSetEntry(key, changes, withCache));
+      } catch (error) {
+        console.error(`Unable to decode storage ${key.section}.${key.method}:`, error.message);
+
+        throw error;
+      }
+
+      return results;
+    }, []);
+  }
+
+  private formatStorageSetEntry (key: StorageKey, changes: [string, string | null][], witCache: boolean): Codec {
     // Fallback to Raw (i.e. just the encoding) if we don't have a specific type
     const type = key.outputType || 'Raw';
     const hexKey = key.toHex();

packages/rpc-provider/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/rpc-provider",
-  "version": "1.5.1",
+  "version": "1.7.1",
   "description": "Transport providers for the API",
   "main": "index.js",
   "keywords": [
@@ -26,20 +26,20 @@
   },
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/rpc-provider#readme",
   "dependencies": {
-    "@babel/runtime": "^7.8.4",
-    "@polkadot/jsonrpc": "1.5.1",
-    "@polkadot/metadata": "1.5.1",
-    "@polkadot/types": "1.5.1",
-    "@polkadot/util": "^2.6.1",
-    "@polkadot/util-crypto": "^2.6.1",
+    "@babel/runtime": "^7.8.7",
+    "@polkadot/jsonrpc": "1.7.1",
+    "@polkadot/metadata": "1.7.1",
+    "@polkadot/types": "1.7.1",
+    "@polkadot/util": "^2.6.2",
+    "@polkadot/util-crypto": "^2.6.2",
     "bn.js": "^5.1.1",
     "eventemitter3": "^4.0.0",
     "isomorphic-fetch": "^2.2.1",
     "websocket": "^1.0.31"
   },
   "devDependencies": {
-    "@polkadot/keyring": "^2.6.1",
+    "@polkadot/keyring": "^2.6.2",
     "mock-socket": "^9.0.3",
-    "nock": "^12.0.1"
+    "nock": "^12.0.2"
   }
 }

packages/rpc-provider/src/coder/decodeResponse.spec.ts

@@ -42,6 +42,12 @@ describe('decodeResponse', (): void => {
     ).toThrow(/123: test error/);
   });
 
+  it('throws any error found, with data', (): void => {
+    expect(
+      (): any => coder.decodeResponse({ id: 1, jsonrpc: '2.0', error: { code: 123, data: 'Error("Some random error description")', message: 'test error' } } as JsonRpcResponse)
+    ).toThrow(/123: test error: Some random error description/);
+  });
+
   it('returns the result', (): void => {
     expect(
       coder.decodeResponse({ id: 1, jsonrpc: '2.0', result: 'some result' } as JsonRpcResponse)

packages/rpc-provider/src/coder/index.ts

@@ -6,6 +6,20 @@ import { JsonRpcRequest, JsonRpcResponse, JsonRpcResponseBaseError } from '../ty
 
 import { assert, isUndefined, isNumber, isString } from '@polkadot/util';
 
+function formatErrorData (data?: string | number): string {
+  if (isUndefined(data)) {
+    return '';
+  }
+
+  const formatted = isString(data)
+    ? data.replace('Error("', '').replace('")', '')
+    : JSON.stringify(data);
+
+  // We need some sort of cut-off here since these can be very large and
+  // very nested, pick a number and trim the result display to it
+  return `: ${formatted.substr(0, 35)}`;
+}
+
 /** @internal */
 export default class RpcCoder {
   #id = 0;
@@ -54,13 +68,7 @@ export default class RpcCoder {
     if (error) {
       const { code, data, message } = error;
 
-      // We need some sort of cut-off here since these can be very large and
-      // very nested, pick a number and trim the result display to it
-      const _data = isUndefined(data)
-        ? ''
-        : `: ${isString(data) ? data : JSON.stringify(data)}`.substr(0, 35);
-
-      throw new Error(`${code}: ${message}${_data}`);
+      throw new Error(`${code}: ${message}${formatErrorData(data)}`);
     }
   }
 }

packages/rpc-provider/src/mock/index.ts

@@ -59,6 +59,8 @@ export default class Mock implements ProviderInterface {
     // eslint-disable-next-line @typescript-eslint/no-unused-vars
     chain_getBlockHash: (blockNumber: number): string => '0x1234',
     chain_getHeader: (): any => this.registry.createType('Header', rpcHeader.result).toJSON(),
+    state_getKeys: (): string[] => [],
+    state_getKeysPaged: (): string[] => [],
     state_getRuntimeVersion: (): string => this.registry.createType('RuntimeVersion').toHex(),
     state_getMetadata: (): string => rpcMetadata,
     state_getStorage: (storage: MockStateDb, params: any[]): string => u8aToHex(storage[(params[0] as string)]),

packages/typegen/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/typegen",
-  "version": "1.5.1",
+  "version": "1.7.1",
   "description": "Type generation scripts",
   "main": "index.js",
   "bin": {
@@ -33,20 +33,20 @@
   },
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/typegen#readme",
   "dependencies": {
-    "@babel/core": "^7.8.3",
-    "@babel/register": "^7.8.3",
-    "@babel/runtime": "^7.8.4",
-    "@polkadot/api": "1.5.1",
-    "@polkadot/jsonrpc": "1.5.1",
-    "@polkadot/metadata": "1.5.1",
-    "@polkadot/rpc-provider": "1.5.1",
-    "@polkadot/types": "1.5.1",
-    "@polkadot/util": "^2.6.1",
+    "@babel/core": "^7.8.7",
+    "@babel/register": "^7.8.6",
+    "@babel/runtime": "^7.8.7",
+    "@polkadot/api": "1.7.1",
+    "@polkadot/jsonrpc": "1.7.1",
+    "@polkadot/metadata": "1.7.1",
+    "@polkadot/rpc-provider": "1.7.1",
+    "@polkadot/types": "1.7.1",
+    "@polkadot/util": "^2.6.2",
     "websocket": "^1.0.31",
-    "yargs": "^15.1.0"
+    "yargs": "^15.3.0"
   },
   "devDependencies": {
     "@types/websocket": "^1.0.0",
-    "@types/yargs": "^15.0.3"
+    "@types/yargs": "^15.0.4"
   }
 }

packages/typegen/src/generate/consts.ts

@@ -25,7 +25,7 @@ function generateModule (allDefs: object, { constants, name }: ModuleMetadataLat
 
   return [indent(4)(`${stringCamelCase(name.toString())}: {`)]
     .concat(isStrict ? '' : indent(6)('[index: string]: AugmentedConst<object & Codec>;'))
-    .concat(constants.map(({ documentation, name, type }): string => {
+    .concat(constants.sort((a, b) => a.name.localeCompare(b.name.toString())).map(({ documentation, name, type }): string => {
       setImports(allDefs, imports, [type.toString()]);
 
       return createDocComments(6, documentation) +
@@ -42,7 +42,7 @@ function generateForMeta (meta: Metadata, dest: string, extraTypes: Record<strin
     const allDefs = Object.entries(allTypes).reduce((defs, [path, obj]) => {
       return Object.entries(obj).reduce((defs, [key, value]) => ({ ...defs, [`${path}/${key}`]: value }), defs);
     }, {});
-    const body = meta.asLatest.modules.reduce((acc, mod): string[] => {
+    const body = meta.asLatest.modules.sort((a, b) => a.name.localeCompare(b.name.toString())).reduce((acc, mod): string[] => {
       return acc.concat(generateModule(allDefs, mod, imports, isStrict));
     }, [] as string[]);
     const header = createImportCode(HEADER('chain'), imports, [

packages/typegen/src/generate/query.ts

@@ -8,30 +8,22 @@ import * as defaultDefs from '@polkadot/types/interfaces/definitions';
 
 import staticData from '@polkadot/metadata/Metadata/static';
 import Metadata from '@polkadot/metadata/Metadata';
+import { unwrapStorageType } from '@polkadot/types/primitive/StorageKey';
 import { TypeRegistry } from '@polkadot/types/create';
 import { stringLowerFirst } from '@polkadot/util';
 
 import { FOOTER, HEADER, TypeImports, createDocComments, createImportCode, createImports, formatType, getSimilarTypes, indent, registerDefinitions, setImports, writeFile } from '../util';
 
-// If the StorageEntry returns T, output `Option<T>` if the modifier is optional
-/** @internal */
-function addModifier (storageEntry: StorageEntryMetadataLatest, returnType: string): string {
-  if (storageEntry.modifier.isOptional) {
-    return `Option<${returnType}>`;
-  }
-
-  return returnType;
-}
-
 // From a storage entry metadata, we return [args, returnType]
 /** @internal */
 function entrySignature (allDefs: object, registry: Registry, storageEntry: StorageEntryMetadataLatest, imports: TypeImports): [string, string] {
   const format = (type: string): string => formatType(allDefs, type, imports);
+  const outputType = unwrapStorageType(storageEntry.type, storageEntry.modifier.isOptional);
 
   if (storageEntry.type.isPlain) {
     setImports(allDefs, imports, [storageEntry.type.asPlain.toString()]);
 
-    return ['', formatType(allDefs, addModifier(storageEntry, storageEntry.type.asPlain.toString()), imports)];
+    return ['', formatType(allDefs, outputType, imports)];
   } else if (storageEntry.type.isMap) {
     // Find similar types of the `key` type
     const similarTypes = getSimilarTypes(allDefs, registry, storageEntry.type.asMap.key.toString(), imports);
@@ -43,7 +35,7 @@ function entrySignature (allDefs: object, registry: Registry, storageEntry: Stor
 
     return [
       `arg: ${similarTypes.map(format).join(' | ')}`,
-      formatType(allDefs, addModifier(storageEntry, storageEntry.type.asMap.value.toString()), imports)
+      formatType(allDefs, outputType, imports)
     ];
   } else if (storageEntry.type.isDoubleMap) {
     // Find similar types of `key1` and `key2` types
@@ -61,7 +53,7 @@ function entrySignature (allDefs: object, registry: Registry, storageEntry: Stor
 
     return [
       `key1: ${key1Types}, key2: ${key2Types}`,
-      formatType(allDefs, addModifier(storageEntry, storageEntry.type.asDoubleMap.value.toString()), imports)
+      formatType(allDefs, outputType, imports)
     ];
   }
 
@@ -77,7 +69,7 @@ function generateModule (allDefs: object, registry: Registry, { name, storage }:
 
   return [indent(4)(`${stringLowerFirst(name.toString())}: {`)]
     .concat(isStrict ? '' : indent(6)('[index: string]: QueryableStorageEntry<ApiType>;'))
-    .concat(storage.unwrap().items.map((storageEntry): string => {
+    .concat(storage.unwrap().items.sort((a, b) => a.name.localeCompare(b.name.toString())).map((storageEntry): string => {
       const [args, returnType] = entrySignature(allDefs, registry, storageEntry, imports);
       let entryType = 'AugmentedQuery';
 
@@ -99,7 +91,7 @@ function generateForMeta (registry: Registry, meta: Metadata, dest: string, extr
     const allDefs = Object.entries(allTypes).reduce((defs, [path, obj]) => {
       return Object.entries(obj).reduce((defs, [key, value]) => ({ ...defs, [`${path}/${key}`]: value }), defs);
     }, {});
-    const body = meta.asLatest.modules.reduce((acc: string[], mod): string[] => {
+    const body = meta.asLatest.modules.sort((a, b) => a.name.localeCompare(b.name.toString())).reduce((acc: string[], mod): string[] => {
       return acc.concat(generateModule(allDefs, registry, mod, imports, isStrict));
     }, []);
     const header = createImportCode(HEADER('chain'), imports, [

packages/typegen/src/generate/rpc.ts

@@ -8,6 +8,8 @@ import { TypeRegistry } from '@polkadot/types/create';
 
 import { FOOTER, HEADER, createDocComments, createImportCode, createImports, getSimilarTypes, setImports, writeFile, indent } from '../util';
 
+const StorageKeyTye = 'StorageKey | string | Uint8Array | any';
+
 /** @internal */
 export default function generateRpcTypes (dest = 'packages/api/src/augment/rpc.ts'): void {
   writeFile(dest, (): string => {
@@ -21,19 +23,21 @@ export default function generateRpcTypes (dest = 'packages/api/src/augment/rpc.t
       const allMethods = Object.keys(interfaces[section].methods).sort().map((key): string => {
         const method = interfaces[section].methods[key];
 
-        setImports(allDefs, imports, [method.type]);
-
-        // FIXME These 2 are too hard to type, I give up
+        // These are too hard to type with generics, do manual overrides
         if (section === 'state') {
-          if (method.method === 'getStorage') {
-            setImports(allDefs, imports, ['Codec']);
+          setImports(allDefs, imports, ['Codec', 'Hash', 'StorageKey', 'Vec']);
 
-            return indent(6)('getStorage: AugmentedRpc<<T = Codec>(key: any, block?: Hash | Uint8Array | string) => Observable<T>>;');
+          if (method.method === 'getStorage') {
+            return createDocComments(6, [method.description]) + indent(6)(`getStorage: AugmentedRpc<<T = Codec>(key: ${StorageKeyTye}, block?: Hash | Uint8Array | string) => Observable<T>>;`);
+          } else if (method.method === 'queryStorage') {
+            return createDocComments(6, [method.description]) + indent(6)(`queryStorage: AugmentedRpc<<T = Codec[]>(keys: Vec<StorageKey> | (${StorageKeyTye})[], fromBlock?: Hash | Uint8Array | string, toBlock?: Hash | Uint8Array | string) => Observable<[Hash, T][]>>;`);
           } else if (method.method === 'subscribeStorage') {
-            return indent(6)('subscribeStorage: AugmentedRpc<<T = Codec[]>(keys: any[]) => Observable<T>>;');
+            return createDocComments(6, [method.description]) + indent(6)(`subscribeStorage: AugmentedRpc<<T = Codec[]>(keys: Vec<StorageKey> | (${StorageKeyTye})[]) => Observable<T>>;`);
           }
         }
 
+        setImports(allDefs, imports, [method.type]);
+
         const args = method.params.map((param): string => {
           const similarTypes = getSimilarTypes(definitions, registry, param.type, imports);
 

packages/typegen/src/generate/tx.ts

@@ -38,7 +38,7 @@ function generateModule (registry: Registry, allDefs: object, { calls, name }: M
 
   return [indent(4)(`${stringCamelCase(name.toString())}: {`)]
     .concat(isStrict ? '' : indent(6)('[index: string]: SubmittableExtrinsicFunction<ApiType>;'))
-    .concat(allCalls.map(({ args, documentation, name }): string => {
+    .concat(allCalls.sort((a, b) => a.name.localeCompare(b.name.toString())).map(({ args, documentation, name }): string => {
       const params = args
         .map(({ name, type }): [string, string, string] => {
           const typeStr = type.toString();
@@ -66,7 +66,7 @@ function generateForMeta (registry: Registry, meta: Metadata, dest: string, extr
     const allDefs = Object.entries(allTypes).reduce((defs, [path, obj]) => {
       return Object.entries(obj).reduce((defs, [key, value]) => ({ ...defs, [`${path}/${key}`]: value }), defs);
     }, {});
-    const body = meta.asLatest.modules.reduce((acc, mod): string[] => {
+    const body = meta.asLatest.modules.sort((a, b) => a.name.localeCompare(b.name.toString())).reduce((acc, mod): string[] => {
       return acc.concat(generateModule(registry, allDefs, mod, imports, isStrict));
     }, [] as string[]);
     const header = createImportCode(HEADER('chain'), imports, [

packages/typegen/src/metadataMd.ts

@@ -3,7 +3,6 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 import { MetadataLatest } from '@polkadot/types/interfaces/metadata';
-import { InterfaceTypes } from '@polkadot/types/types';
 
 import fs from 'fs';
 import interfaces from '@polkadot/jsonrpc';
@@ -64,20 +63,27 @@ function renderPage (page: Page): string {
   }
 
   // index
-  page.sections.forEach(section => {
+  page.sections.forEach((section) => {
     md += `- **[${stringCamelCase(section.name)}](#${stringCamelCase(section.name).toLowerCase()})**\n\n`;
   });
 
   // contents
   page.sections.forEach(section => {
     md += `\n___\n\n\n## ${section.name}\n`;
-    if (section.description) { md += `\n_${section.description}_\n`; }
+
+    if (section.description) {
+      md += `\n_${section.description}_\n`;
+    }
 
     section.items.forEach((item) => {
       md += ` \n### ${item.name}`;
 
       Object.keys(item).filter(i => i !== 'name').forEach(bullet => {
-        md += `\n- **${bullet}**: ${item[bullet] instanceof Vec ? documentationVecToMarkdown(item[bullet] as Vec<Text>, 2) : item[bullet]}`;
+        md += `\n- **${bullet}**: ${
+          item[bullet] instanceof Vec
+            ? documentationVecToMarkdown(item[bullet] as Vec<Text>, 2)
+            : item[bullet]
+        }`;
       });
 
       md += '\n';
@@ -116,8 +122,8 @@ function addRpc (): string {
 
               return {
                 name: `${methodName}(${args}): ${type}`,
-                jsonrpc: `${sectionName}_${methodName}`,
-                interface: `api.rpc.${sectionName}.${methodName}`,
+                jsonrpc: '`' + `${sectionName}_${methodName}` + '`',
+                interface: '`' + `api.rpc.${sectionName}.${methodName}` + '`',
                 ...(method.description && { summary: method.description })
               };
             })
@@ -146,7 +152,7 @@ function addConstants (metadata: MetadataLatest): string {
 
               return {
                 name: `${methodName}: ` + '`' + func.type + '`',
-                interface: `api.consts.${sectionName}.${methodName}`,
+                interface: '`' + `api.consts.${sectionName}.${methodName}` + '`',
                 ...(func.documentation.length && { summary: func.documentation })
               };
             })
@@ -174,15 +180,11 @@ function addStorage (metadata: MetadataLatest): string {
                 ? ('`' + func.type.asDoubleMap.key1.toString() + ', ' + func.type.asDoubleMap.key2.toString() + '`')
                 : '';
             const methodName = stringLowerFirst(func.name.toString());
-            let result = unwrapStorageType(func.type);
-
-            if (func.modifier.isOptional) {
-              result = `Option<${result}>` as keyof InterfaceTypes;
-            }
+            const outputType = unwrapStorageType(func.type, func.modifier.isOptional);
 
             return {
-              name: `${methodName}(${arg}): ` + '`' + result + '`',
-              interface: `api.query.${sectionName}.${methodName}`,
+              name: `${methodName}(${arg}): ` + '`' + outputType + '`',
+              interface: '`' + `api.query.${sectionName}.${methodName}` + '`',
               ...(func.documentation.length && { summary: func.documentation })
             };
           })
@@ -220,7 +222,7 @@ function addExtrinsics (metadata: MetadataLatest): string {
 
               return {
                 name: `${methodName}(${args})`,
-                interface: `api.tx.${sectionName}.${methodName}`,
+                interface: '`' + `api.tx.${sectionName}.${methodName}` + '`',
                 ...(func.documentation.length && { summary: func.documentation })
               };
             })

packages/typegen/src/util/formatting.ts

@@ -105,6 +105,14 @@ function formatHashMap (key: string, val: string): string {
   return `HashMap<${key}, ${val}>`;
 }
 
+/**
+ * Given the inner `T`, return a `Vec<T>` string
+ */
+/** @internal */
+function formatLinkage (inner: string): string {
+  return paramsNotation('Linkage', inner);
+}
+
 /**
  * Given the inner `O` & `E`, return a `Result<O, E>`  string
  */
@@ -221,6 +229,13 @@ export function formatType (definitions: object, type: string | TypeDef, imports
 
       return formatHashMap(formatType(definitions, keyDef.type, imports), formatType(definitions, valDef.type, imports));
     }
+    case TypeDefInfo.Linkage: {
+      const type = (typeDef.sub as TypeDef).type;
+
+      setImports(definitions, imports, ['Linkage']);
+
+      return formatLinkage(formatType(definitions, type, imports));
+    }
     case TypeDefInfo.Result: {
       setImports(definitions, imports, ['Result']);