[CI Skip] 1.9.1

skip-checks: true

.123trigger

@@ -1,3 +1,3 @@
-20
+21
  1.4.2
-1.6.2
+1.6.2

.github/workflows/push-master.yml

@@ -10,7 +10,7 @@ jobs:
       matrix:
         step: ['build:release', 'docs:release']
     name: ${{ matrix.step }}
-    if: "! contains(github.event.head_commit.message, '[CI Skip]')"
+    if: "! contains(github.event.head_commit.message, '[CI Skip]') && github.repository == 'polkadot-js/api'"
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v1

CHANGELOG.md

@@ -1,6 +1,21 @@
+# 1.9.1 Apr 1, 2020
+
+**Breaking change** (TypeScript with derives only) The return type names for `api.derive.*` have been adjusted for consistency, all starting with `Derive*` (previously a mixture of `Derived*` and `Derive*`)
+**Breaking change** (derives only) Additional cleanups of democracy proposal & referenda derives, including support for new Polkadot/Substrate vote retrievals
+- Tx signing via `signAndSend` & `signAsync` now tries to use the last finalized block (depending on finalized drift)
+- Add support for `api.rpc.childstate.*` Substrate RPCs, older nodes with still populate `api.rpc.state.*` with the child methods
+- Adjust `Error.message` on RPC failures to not include the (already known) method signature
+- Split `@polkadot/types/known` into `@polkadot/types-known`, allowing for base API overrides
+- Extended `api.derive.democracy` to cater for dispatchQueue and externals
+- Extended `api.derive.council` to cater for new phragmen elections
+- Use new `queryStorageAt` for query `.entries`, removing unneeded subscriptions (use as available)
+- Fix `BTreeSet` initialization when it forms part of query interfaces
+- Fix encoding for `Vec<u8>`-related types as part of constants
+- Internal adjustment of interface type locations (no impact on imports via `@polkadot/types/interfaces`)
+
 # 1.8.1 Mar 22, 2020
 
-- **Breaking change** The format for any custom RPCs have been changed alongside API-internal changes to allow for better RPC management. If you are currently using custom RPCs (or planning to do so), look at the [updated documentation](https://polkadot.js.org/api/start/rpc.custom.html)
+- **Breaking change** The format for any custom RPCs have been changed (alongside API-internal changes) to allow for better RPC management. If you are currently using custom RPCs (or planning to do so), look at the [updated documentation](https://polkadot.js.org/api/start/rpc.custom.html)
 - **Breaking change** Alongside API RPC changes, the `@polkadot/jsonrpc` package has been removed. Since it was never documented and only used internally, this should not have adverse impacts. All RPC definitions itself has now been moved to the relevant modules inside `@polkadot/types/interfaces`
 - **Important** Substrate has an updated democracy module. If using an older chain add the `ReferendumInfo: 'ReferendumInfoTo239'` type when using referendums
 - The `isRetracted` Extrinsic status is now a warning, not a fatal error, correctly aligning with Polkadot/Substrate
@@ -8,6 +23,7 @@
 - Add support for all known substrate RPC methods
 - Extend account derives to include status flags, e.g. `isCouncil`, `isSudo`, ...
 - Adjusted referendum derives to cater for bot new/old democracy. Derived `referendumInfo` now includes `status` field, not `info`
+- Add `initWasm` flag to API options. When set to `false` does not init the WASM portion (leaving it to the user elsewhere)
 
 # 1.7.1 Mar 17, 2020
 

docs/start/FAQ.md

@@ -40,7 +40,7 @@ Tuples, as defined in the Polkadot/Substrate types appear as `(TypeA, TypeB)`. F
 
 Polkadot/Substrate supports both immortal and mortal transactions. For immortal, this means that the transaction has an infinite lifetime, for mortals, the transactions expire after a defined period. By default the API sends mortal transactions when no explicit extrinsic era has been specified. This means that all transaction has a limited lifetime and will expire if not included in the period set.
 
-The length for this transaction validity is set to 50 blocks, which translates to 5 minutes assuming a default of 6 second blocktimes. (10 blocks per minute in this default configuration).
+The length for this transaction validity is set to 5 minutes, calculated based on the blocktime for the chain. (10 blocks per minute in this default 6s Substrate configuration).
 
 ## My chain does not support system.account queries
 
@@ -49,3 +49,11 @@ The API always tracks the latest Substrate master in terms of examples. This mea
 It is possible that you are connecting to an older chain that has not been upgraded yet. For these chains, this storage entry won't be available (yet). To query the nonce on older chains, you can do a query to `api.query.system.accountNonce(<account>)` and balances can be retrieved via `api.query.balances.freeBalance(<account>)`.
 
 Likewise, if your chain has been upgraded recently and you are still using the old `system.accountNonce` or `balances.freeBalance` queries in your code (which is now not available in the chain metadata), you need to update it to query the new location.
+
+## I cannot send transactions from my node-template-based chain
+
+For the API defaults, the API always the type definitions as specified by the Substrate master fully-featured node. This means that any customizations needs to be applied as types should there be differences in specific user-implementations.
+
+The Substrate node-template has added customizations for some types in the default template, specifically around the `Address` and `Lookup` types, removing indices lookups from addresses. This means that the transaction payload saves 2 bytes for a transfer and is an approach followed by other chains as well, notably Polkadot & Kusama.
+
+Due to these customizations and differences that bleed through to the transaction formats, out-of-the-box the node-template will have issues when sending transactions. To fix this, you would need to add [the customized Address types into your API](types.extend.md#impact-on-extrinsics) instances (or UIs), allowing the API to have the information required to adjust the encoding.

jest.config.js

@@ -9,6 +9,7 @@ module.exports = Object.assign({}, config, {
     '@polkadot/api(.*)$': '<rootDir>/packages/api/src/$1',
     '@polkadot/metadata(.*)$': '<rootDir>/packages/metadata/src/$1',
     '@polkadot/rpc-(core|provider)(.*)$': '<rootDir>/packages/rpc-$1/src/$2',
+    '@polkadot/types-known(.*)$': '<rootDir>/packages/types-known/src/$1',
     '@polkadot/types(.*)$': '<rootDir>/packages/types/src/$1'
   },
   modulePathIgnorePatterns: [
@@ -18,7 +19,8 @@ module.exports = Object.assign({}, config, {
     '<rootDir>/packages/metadata/build',
     '<rootDir>/packages/rpc-core/build',
     '<rootDir>/packages/rpc-provider/build',
-    '<rootDir>/packages/types/build'
+    '<rootDir>/packages/types/build',
+    '<rootDir>/packages/types-known/build'
   ],
   resolver: './jest.resolver.js'
 });

lerna.json

@@ -9,5 +9,5 @@
   "packages": [
     "packages/*"
   ],
-  "version": "1.8.1"
+  "version": "1.9.1"
 }

package.json

@@ -27,7 +27,7 @@
     "@babel/core": "^7.9.0",
     "@babel/register": "^7.9.0",
     "@babel/runtime": "^7.9.2",
-    "@polkadot/dev": "^0.51.7",
+    "@polkadot/dev": "^0.51.9",
     "@polkadot/ts": "^0.3.14",
     "@polkadot/typegen": "workspace:packages/typegen",
     "@types/jest": "^25.1.4",

packages/api-contract/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/api-contract",
-  "version": "1.8.1",
+  "version": "1.9.1",
   "description": "Interfaces for interacting with contracts and contract ABIs",
   "main": "index.js",
   "keywords": [
@@ -27,10 +27,10 @@
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/api-contract#readme",
   "dependencies": {
     "@babel/runtime": "^7.9.2",
-    "@polkadot/api": "1.8.1",
-    "@polkadot/rpc-core": "1.8.1",
-    "@polkadot/types": "1.8.1",
-    "@polkadot/util": "^2.6.2",
+    "@polkadot/api": "1.9.1",
+    "@polkadot/rpc-core": "1.9.1",
+    "@polkadot/types": "1.9.1",
+    "@polkadot/util": "^2.7.1",
     "bn.js": "^5.1.1",
     "rxjs": "^6.5.4"
   }

packages/api-derive/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/api-derive",
-  "version": "1.8.1",
+  "version": "1.9.1",
   "description": "Common functions used across Polkadot, derived from RPC calls and storage queries.",
   "main": "index.js",
   "keywords": [
@@ -28,17 +28,17 @@
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/api-derive#readme",
   "dependencies": {
     "@babel/runtime": "^7.9.2",
-    "@polkadot/api": "1.8.1",
-    "@polkadot/rpc-core": "1.8.1",
-    "@polkadot/rpc-provider": "1.8.1",
-    "@polkadot/types": "1.8.1",
-    "@polkadot/util": "^2.6.2",
-    "@polkadot/util-crypto": "^2.6.2",
+    "@polkadot/api": "1.9.1",
+    "@polkadot/rpc-core": "1.9.1",
+    "@polkadot/rpc-provider": "1.9.1",
+    "@polkadot/types": "1.9.1",
+    "@polkadot/util": "^2.7.1",
+    "@polkadot/util-crypto": "^2.7.1",
     "bn.js": "^5.1.1",
     "memoizee": "^0.4.14",
     "rxjs": "^6.5.4"
   },
   "devDependencies": {
-    "@polkadot/keyring": "^2.6.2"
+    "@polkadot/keyring": "^2.7.1"
   }
 }

packages/api-derive/src/balances/account.ts

@@ -4,7 +4,7 @@
 
 import { AccountId, AccountData, AccountIndex, AccountInfo, Address, Balance, Index } from '@polkadot/types/interfaces';
 import { ITuple } from '@polkadot/types/types';
-import { DerivedBalancesAccount } from '../types';
+import { DeriveBalancesAccount } from '../types';
 
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
@@ -14,7 +14,7 @@ import { memo } from '../util';
 
 type Result = [Balance, Balance, Balance, Balance, Index];
 
-function calcBalances (api: ApiInterfaceRx, [accountId, [freeBalance, reservedBalance, frozenFee, frozenMisc, accountNonce]]: [AccountId, Result]): DerivedBalancesAccount {
+function calcBalances (api: ApiInterfaceRx, [accountId, [freeBalance, reservedBalance, frozenFee, frozenMisc, accountNonce]]: [AccountId, Result]): DeriveBalancesAccount {
   return {
     accountId,
     accountNonce,
@@ -79,8 +79,8 @@ function queryCurrent (api: ApiInterfaceRx, accountId: AccountId): Observable<Re
  * });
  * ```
  */
-export function account (api: ApiInterfaceRx): (address: AccountIndex | AccountId | Address | string) => Observable<DerivedBalancesAccount> {
-  return memo((address: AccountIndex | AccountId | Address | string): Observable<DerivedBalancesAccount> =>
+export function account (api: ApiInterfaceRx): (address: AccountIndex | AccountId | Address | string) => Observable<DeriveBalancesAccount> {
+  return memo((address: AccountIndex | AccountId | Address | string): Observable<DeriveBalancesAccount> =>
     api.derive.accounts.info(address).pipe(
       switchMap(({ accountId }): Observable<[AccountId, Result]> =>
         (accountId
@@ -95,6 +95,6 @@ export function account (api: ApiInterfaceRx): (address: AccountIndex | AccountI
           : of([api.registry.createType('AccountId'), [api.registry.createType('Balance'), api.registry.createType('Balance'), api.registry.createType('Balance'), api.registry.createType('Balance'), api.registry.createType('Index')]])
         )
       ),
-      map((result): DerivedBalancesAccount => calcBalances(api, result))
+      map((result): DeriveBalancesAccount => calcBalances(api, result))
     ));
 }

packages/api-derive/src/balances/all.ts

@@ -3,7 +3,7 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 import { AccountId, AccountIndex, Address, Balance, BalanceLock, BalanceLockTo212, BlockNumber, VestingInfo, VestingSchedule } from '@polkadot/types/interfaces';
-import { DerivedBalancesAccount, DerivedBalancesAll } from '../types';
+import { DeriveBalancesAccount, DeriveBalancesAll } from '../types';
 
 import BN from 'bn.js';
 import { Observable, combineLatest, of } from 'rxjs';
@@ -15,9 +15,9 @@ import { bnMax } from '@polkadot/util';
 import { memo } from '../util';
 
 type ResultBalance = [VestingInfo | null, (BalanceLock | BalanceLockTo212)[]];
-type Result = [DerivedBalancesAccount, BlockNumber, ResultBalance];
+type Result = [DeriveBalancesAccount, BlockNumber, ResultBalance];
 
-function calcBalances (api: ApiInterfaceRx, [{ accountId, accountNonce, freeBalance, frozenFee, frozenMisc, reservedBalance, votingBalance }, bestNumber, [vesting, locks]]: Result): DerivedBalancesAll {
+function calcBalances (api: ApiInterfaceRx, [{ accountId, accountNonce, freeBalance, frozenFee, frozenMisc, reservedBalance, votingBalance }, bestNumber, [vesting, locks]]: Result): DeriveBalancesAll {
   let lockedBalance = api.registry.createType('Balance');
   let lockedBreakdown: (BalanceLock | BalanceLockTo212)[] = [];
   let allLocked = false;
@@ -127,8 +127,8 @@ function queryCurrent (api: ApiInterfaceRx, accountId: AccountId): Observable<Re
  * });
  * ```
  */
-export function all (api: ApiInterfaceRx): (address: AccountIndex | AccountId | Address | string) => Observable<DerivedBalancesAll> {
-  return memo((address: AccountIndex | AccountId | Address | string): Observable<DerivedBalancesAll> =>
+export function all (api: ApiInterfaceRx): (address: AccountIndex | AccountId | Address | string) => Observable<DeriveBalancesAll> {
+  return memo((address: AccountIndex | AccountId | Address | string): Observable<DeriveBalancesAll> =>
     api.derive.balances.account(address).pipe(
       switchMap((account): Observable<Result> =>
         (!account.accountId.isEmpty
@@ -142,6 +142,6 @@ export function all (api: ApiInterfaceRx): (address: AccountIndex | AccountId |
           : of([account, api.registry.createType('BlockNumber'), [null, api.registry.createType('Vec<BalanceLock>')]])
         )
       ),
-      map((result): DerivedBalancesAll => calcBalances(api, result))
+      map((result): DeriveBalancesAll => calcBalances(api, result))
     ));
 }

packages/api-derive/src/balances/fees.ts

@@ -4,7 +4,7 @@
 
 import { ApiInterfaceRx } from '@polkadot/api/types';
 import { Balance } from '@polkadot/types/interfaces';
-import { DerivedFees } from '../types';
+import { DeriveFees } from '../types';
 
 import { Observable, of } from 'rxjs';
 import { map } from 'rxjs/operators';
@@ -24,8 +24,8 @@ import { memo } from '../util';
  * });
  * ```
  */
-export function fees (api: ApiInterfaceRx): () => Observable<DerivedFees> {
-  return memo((): Observable<DerivedFees> =>
+export function fees (api: ApiInterfaceRx): () => Observable<DeriveFees> {
+  return memo((): Observable<DeriveFees> =>
     of([
       // deprecated - remove
       (api.consts.balances?.creationFee as Balance) || api.registry.createType('Balance'),
@@ -36,7 +36,7 @@ export function fees (api: ApiInterfaceRx): () => Observable<DerivedFees> {
       (api.consts.transactionPayment?.transactionBaseFee as Balance) || api.registry.createType('Balance'),
       (api.consts.transactionPayment?.transactionByteFee as Balance) || api.registry.createType('Balance')
     ]).pipe(
-      map(([creationFee, transferFee, existentialDeposit, transactionBaseFee, transactionByteFee]): DerivedFees => ({
+      map(([creationFee, transferFee, existentialDeposit, transactionBaseFee, transactionByteFee]): DeriveFees => ({
         creationFee,
         existentialDeposit,
         transactionBaseFee,

packages/api-derive/src/balances/votingBalances.ts

@@ -3,19 +3,19 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 import { AccountId, AccountIndex, Address } from '@polkadot/types/interfaces';
-import { DerivedBalancesAccount } from '../types';
+import { DeriveBalancesAccount } from '../types';
 
 import { Observable, combineLatest, of } from 'rxjs';
 import { ApiInterfaceRx } from '@polkadot/api/types';
 
 import { memo } from '../util';
 
-export function votingBalances (api: ApiInterfaceRx): (addresses?: (AccountId | AccountIndex | Address | string)[]) => Observable<DerivedBalancesAccount[]> {
-  return memo((addresses?: (AccountId | AccountIndex | Address | string)[]): Observable<DerivedBalancesAccount[]> =>
+export function votingBalances (api: ApiInterfaceRx): (addresses?: (AccountId | AccountIndex | Address | string)[]) => Observable<DeriveBalancesAccount[]> {
+  return memo((addresses?: (AccountId | AccountIndex | Address | string)[]): Observable<DeriveBalancesAccount[]> =>
     !addresses || !addresses.length
-      ? of([] as DerivedBalancesAccount[])
+      ? of([] as DeriveBalancesAccount[])
       : combineLatest(
-        addresses.map((accountId): Observable<DerivedBalancesAccount> =>
+        addresses.map((accountId): Observable<DeriveBalancesAccount> =>
           api.derive.balances.account(accountId)
         )
       )

packages/api-derive/src/collective/proposals.ts

@@ -4,7 +4,7 @@
 
 import { ApiInterfaceRx } from '@polkadot/api/types';
 import { Hash, Proposal, Votes } from '@polkadot/types/interfaces';
-import { DerivedCollectiveProposal } from '../types';
+import { DeriveCollectiveProposal } from '../types';
 
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
@@ -12,9 +12,9 @@ import { Option } from '@polkadot/types';
 
 type Result = [Hash[], Option<Proposal>[], Option<Votes>[]];
 
-function parse ([hashes, proposals, votes]: Result): DerivedCollectiveProposal[] {
+function parse ([hashes, proposals, votes]: Result): DeriveCollectiveProposal[] {
   return proposals
-    .map((proposalOpt, index): DerivedCollectiveProposal | null =>
+    .map((proposalOpt, index): DeriveCollectiveProposal | null =>
       proposalOpt.isSome
         ? {
           hash: hashes[index],
@@ -23,11 +23,11 @@ function parse ([hashes, proposals, votes]: Result): DerivedCollectiveProposal[]
         }
         : null
     )
-    .filter((proposal): proposal is DerivedCollectiveProposal => !!proposal);
+    .filter((proposal): proposal is DeriveCollectiveProposal => !!proposal);
 }
 
-export function proposals (api: ApiInterfaceRx, section: 'council' | 'technicalCommittee'): () => Observable<DerivedCollectiveProposal[]> {
-  return (): Observable<DerivedCollectiveProposal[]> =>
+export function proposals (api: ApiInterfaceRx, section: 'council' | 'technicalCommittee'): () => Observable<DeriveCollectiveProposal[]> {
+  return (): Observable<DeriveCollectiveProposal[]> =>
     api.query[section]?.proposals
       ? api.query[section].proposals().pipe(
         switchMap((hashes: Hash[]): Observable<Result> =>
@@ -41,5 +41,5 @@ export function proposals (api: ApiInterfaceRx, section: 'council' | 'technicalC
         ),
         map(parse)
       )
-      : of([] as DerivedCollectiveProposal[]);
+      : of([] as DeriveCollectiveProposal[]);
 }

packages/api-derive/src/contracts/fees.ts

@@ -3,7 +3,7 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { DerivedContractFees } from '../types';
+import { DeriveContractFees } from '../types';
 
 import BN from 'bn.js';
 import { Observable, of } from 'rxjs';
@@ -44,10 +44,10 @@ function queryConstants (api: ApiInterfaceRx): Observable<ResultV2> {
  * });
  * ```
  */
-export function fees (api: ApiInterfaceRx): () => Observable<DerivedContractFees> {
-  return memo((): Observable<DerivedContractFees> => {
+export function fees (api: ApiInterfaceRx): () => Observable<DeriveContractFees> {
+  return memo((): Observable<DeriveContractFees> => {
     return queryConstants(api).pipe(
-      map(([creationFee, transferFee, callBaseFee, contractFee, rentByteFee, rentDepositOffset, tombstoneDeposit, transactionBaseFee, transactionByteFee]): DerivedContractFees => ({
+      map(([creationFee, transferFee, callBaseFee, contractFee, rentByteFee, rentDepositOffset, tombstoneDeposit, transactionBaseFee, transactionByteFee]): DeriveContractFees => ({
         callBaseFee,
         contractFee,
         creationFee,

packages/api-derive/src/council/index.ts

@@ -3,3 +3,5 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 export * from './proposals';
+export * from './votes';
+export * from './votesOf';

packages/api-derive/src/council/proposals.ts

@@ -3,12 +3,12 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { DerivedCollectiveProposals } from '../types';
+import { DeriveCollectiveProposals } from '../types';
 
 import { Observable } from 'rxjs';
 import { proposals as collectiveProposals } from '../collective';
 import { memo } from '../util';
 
-export function proposals (api: ApiInterfaceRx): () => Observable<DerivedCollectiveProposals> {
+export function proposals (api: ApiInterfaceRx): () => Observable<DeriveCollectiveProposals> {
   return memo(collectiveProposals(api, 'council'));
 }

packages/api-derive/src/council/types.ts

@@ -0,0 +1,12 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { AccountId, Balance } from '@polkadot/types/interfaces';
+
+export interface DeriveCouncilVote {
+  stake: Balance;
+  votes: AccountId[];
+}
+
+export type DeriveCouncilVotes = [AccountId, DeriveCouncilVote][];

packages/api-derive/src/council/votes.ts

@@ -0,0 +1,88 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { AccountId, Balance } from '@polkadot/types/interfaces';
+import { ITuple } from '@polkadot/types/types';
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { DeriveCouncilVote, DeriveCouncilVotes } from '../types';
+
+import { Observable, combineLatest } from 'rxjs';
+import { map } from 'rxjs/operators';
+import { Vec } from '@polkadot/types';
+
+function retrieveStakeOf (api: ApiInterfaceRx): Observable<[AccountId, Balance][]> {
+  const elections = (api.query.electionsPhragmen || api.query.elections);
+
+  return elections.stakeOf.creator.meta.type.asMap.linked.isTrue
+    ? elections.stakeOf<ITuple<[Vec<AccountId>, Vec<Balance>]>>().pipe(
+      map(([voters, stake]) =>
+        voters.map((voter, index): [AccountId, Balance] => [voter, stake[index]])
+      )
+    )
+    : elections.stakeOf.entries<Balance>().pipe(
+      map((entries) =>
+        entries.map(([key, stake]) => [key.args[0] as AccountId, stake])
+      )
+    );
+}
+
+function retrieveVoteOf (api: ApiInterfaceRx): Observable<[AccountId, AccountId[]][]> {
+  const elections = (api.query.electionsPhragmen || api.query.elections);
+
+  return elections.votesOf.creator.meta.type.asMap.linked.isTrue
+    ? elections.votesOf<ITuple<[Vec<AccountId>, Vec<Vec<AccountId>>]>>().pipe(
+      map(([voters, votes]) =>
+        voters.map((voter, index) => [voter, votes[index]])
+      )
+    )
+    : elections.votesOf.entries<Vec<AccountId>>().pipe(
+      map((entries) =>
+        entries.map(([key, votes]) => [key.args[0] as AccountId, votes])
+      )
+    );
+}
+
+function retrievePrev (api: ApiInterfaceRx): Observable<DeriveCouncilVotes> {
+  return combineLatest([retrieveStakeOf(api), retrieveVoteOf(api)]).pipe(
+    map(([stakes, votes]): DeriveCouncilVotes => {
+      const result: DeriveCouncilVotes = [];
+
+      votes.forEach(([voter, votes]): void => {
+        result.push([voter, { stake: api.registry.createType('Balance'), votes }]);
+      });
+
+      stakes.forEach(([staker, stake]): void => {
+        const entry = result.find(([voter]) => voter.eq(staker));
+
+        if (entry) {
+          entry[1].stake = stake;
+        } else {
+          result.push([staker, { stake, votes: [] }]);
+        }
+      });
+
+      return result;
+    })
+  );
+}
+
+function retrieveCurrent (api: ApiInterfaceRx): Observable<DeriveCouncilVotes> {
+  const elections = (api.query.electionsPhragmen || api.query.elections);
+
+  return elections.voting.entries<ITuple<[Balance, Vec<AccountId>]>>().pipe(
+    map((entries): DeriveCouncilVotes =>
+      entries.map(([key, [stake, votes]]): [AccountId, DeriveCouncilVote] =>
+        [key.args[0] as AccountId, { stake, votes }]
+      )
+    )
+  );
+}
+
+export function votes (api: ApiInterfaceRx): () => Observable<DeriveCouncilVotes> {
+  // since we are using entries, we are not adding a memo
+  return (): Observable<DeriveCouncilVotes> =>
+    (api.query.electionsPhragmen || api.query.elections).stakeOf
+      ? retrievePrev(api)
+      : retrieveCurrent(api);
+}

packages/api-derive/src/council/votesOf.ts

@@ -0,0 +1,23 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { AccountId } from '@polkadot/types/interfaces';
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { DeriveCouncilVote } from '../types';
+
+import { Observable } from 'rxjs';
+import { map } from 'rxjs/operators';
+
+export function votesOf (api: ApiInterfaceRx): (accountId: string | Uint8Array | AccountId) => Observable<DeriveCouncilVote> {
+  // no memo, votes actually pulls the entries without a memo (using entries)
+  return (accountId: string | Uint8Array | AccountId): Observable<DeriveCouncilVote> =>
+    api.derive.council.votes().pipe(
+      map((votes): DeriveCouncilVote =>
+        (
+          votes.find(([from]) => from.eq(accountId)) ||
+          [null, { stake: api.registry.createType('Balance'), votes: [] as AccountId[] }]
+        )[1]
+      )
+    );
+}

packages/api-derive/src/democracy/_referendumInfo.ts

@@ -0,0 +1,32 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { ReferendumInfo, ReferendumInfoTo239 } from '@polkadot/types/interfaces';
+import { DeriveReferendum } from '../types';
+
+import BN from 'bn.js';
+import { Observable, of } from 'rxjs';
+import { map } from 'rxjs/operators';
+import { Option } from '@polkadot/types';
+
+import { memo } from '../util';
+import { getStatus, parseImage } from './util';
+
+export function _referendumInfo (api: ApiInterfaceRx): (index: BN, info: Option<ReferendumInfo | ReferendumInfoTo239>) => Observable<DeriveReferendum | null> {
+  return memo((index: BN, info: Option<ReferendumInfo | ReferendumInfoTo239>): Observable<DeriveReferendum | null> => {
+    const status = getStatus(info);
+
+    return status
+      ? api.query.democracy.preimages(status.proposalHash).pipe(
+        map((preimage): DeriveReferendum => ({
+          index: api.registry.createType('ReferendumIndex', index),
+          image: parseImage(api, preimage),
+          imageHash: status.proposalHash,
+          status
+        }))
+      )
+      : of(null);
+  });
+}

packages/api-derive/src/democracy/_referendumVotes.ts

@@ -0,0 +1,84 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { AccountId, Vote, Voting, VotingDirectVote } from '@polkadot/types/interfaces';
+import { DeriveBalancesAccount, DeriveReferendum, DeriveReferendumVote, DeriveReferendumVotes } from '../types';
+
+import BN from 'bn.js';
+import { Observable, of, combineLatest } from 'rxjs';
+import { map, switchMap } from 'rxjs/operators';
+import { Vec } from '@polkadot/types';
+
+import { memo } from '../util';
+import { calcVotes } from './util';
+
+function votesPrev (api: ApiInterfaceRx, referendumId: BN): Observable<DeriveReferendumVote[]> {
+  return api.query.democracy.votersFor<Vec<AccountId>>(referendumId).pipe(
+    switchMap((votersFor): Observable<[Vec<AccountId>, Vote[], DeriveBalancesAccount[]]> =>
+      combineLatest([
+        of(votersFor),
+        votersFor.length
+          ? api.query.democracy.voteOf.multi<Vote>(
+            votersFor.map((accountId): [BN | number, AccountId] =>
+              [referendumId, accountId]
+            )
+          )
+          : of([]),
+        api.derive.balances.votingBalances(votersFor)
+      ])
+    ),
+    map(([votersFor, votes, balances]): DeriveReferendumVote[] =>
+      votersFor.map((accountId, index): DeriveReferendumVote => ({
+        accountId,
+        balance: balances[index].votingBalance || api.registry.createType('Balance'),
+        vote: votes[index] || api.registry.createType('Vote')
+      }))
+    )
+  );
+}
+
+function votesCurr (api: ApiInterfaceRx, referendumId: BN): Observable<DeriveReferendumVote[]> {
+  return api.query.democracy.votingOf.entries<Voting>().pipe(
+    map((allVoting): DeriveReferendumVote[] =>
+      allVoting
+        .map(([key, voting]): [AccountId, Voting] => [key.args[0] as AccountId, voting])
+        // FIXME We are ignoring delegated votes
+        .filter(([, voting]) => voting.isDirect)
+        .map(([accountId, voting]): [AccountId, VotingDirectVote[]] => [
+          accountId,
+          voting.asDirect.votes.filter(([idx]) => idx.eq(referendumId))
+        ])
+        .filter(([, directVotes]) => !!directVotes.length)
+        .reduce((result: DeriveReferendumVote[], [accountId, votes]) =>
+          // FIXME We are ignoring split votes
+          votes.reduce((result: DeriveReferendumVote[], [, vote]): DeriveReferendumVote[] => {
+            if (vote.isStandard) {
+              result.push({
+                accountId,
+                ...vote.asStandard
+              });
+            }
+
+            return result;
+          }, result), []
+        )
+    )
+  );
+}
+
+export function _referendumVotes (api: ApiInterfaceRx): (referendum: DeriveReferendum) => Observable<DeriveReferendumVotes> {
+  return memo((referendum: DeriveReferendum): Observable<DeriveReferendumVotes> =>
+    combineLatest([
+      api.derive.democracy.sqrtElectorate(),
+      api.query.democracy.votingOf
+        ? votesCurr(api, referendum.index)
+        : votesPrev(api, referendum.index)
+    ]).pipe(
+      map(([sqrtElectorate, votes]) =>
+        calcVotes(sqrtElectorate, referendum, votes)
+      )
+    )
+  );
+}

packages/api-derive/src/democracy/_referendumsVotes.ts

@@ -0,0 +1,22 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { DeriveReferendum, DeriveReferendumVotes } from '../types';
+
+import { Observable, of, combineLatest } from 'rxjs';
+
+import { memo } from '../util';
+
+export function _referendumsVotes (api: ApiInterfaceRx): (referendums: DeriveReferendum[]) => Observable<DeriveReferendumVotes[]> {
+  return memo((referendums: DeriveReferendum[]): Observable<DeriveReferendumVotes[]> =>
+    referendums.length
+      ? combineLatest(
+        referendums.map((referendum): Observable<DeriveReferendumVotes> =>
+          api.derive.democracy._referendumVotes(referendum)
+        )
+      )
+      : of([])
+  );
+}

packages/api-derive/src/democracy/dispatchQueue.ts

@@ -0,0 +1,38 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { BlockNumber, Hash, ReferendumIndex } from '@polkadot/types/interfaces';
+import { ITuple } from '@polkadot/types/types';
+import { DeriveDispatch } from '../types';
+
+import { Observable, combineLatest, of } from 'rxjs';
+import { map, switchMap } from 'rxjs/operators';
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { Vec } from '@polkadot/types';
+
+import { memo } from '../util';
+
+export function dispatchQueue (api: ApiInterfaceRx): () => Observable<DeriveDispatch[]> {
+  return memo((): Observable<DeriveDispatch[]> =>
+    api.query.democracy?.dispatchQueue
+      ? api.query.democracy.dispatchQueue<Vec<ITuple<[BlockNumber, Hash, ReferendumIndex]>>>().pipe(
+        switchMap((dispatches) =>
+          combineLatest([
+            of(dispatches),
+            api.derive.democracy.preimages(
+              dispatches.map(([, hash]) => hash))
+          ])
+        ),
+        map(([dispatches, images]) =>
+          dispatches.map(([at, imageHash, index], dispatchIndex): DeriveDispatch => ({
+            at,
+            index,
+            image: images[dispatchIndex],
+            imageHash
+          }))
+        )
+      )
+      : of([])
+  );
+}

packages/api-derive/src/democracy/index.ts

@@ -2,8 +2,15 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
+export * from './_referendumsVotes';
+export * from './_referendumVotes';
+export * from './_referendumInfo';
+export * from './dispatchQueue';
+export * from './nextExternal';
+export * from './preimage';
+export * from './preimages';
 export * from './proposals';
-export * from './referendumInfos';
 export * from './referendums';
-export * from './referendumVotesFor';
-export * from './votes';
+export * from './referendumsActive';
+export * from './referendumsInfo';
+export * from './sqrtElectorate';

packages/api-derive/src/democracy/nextExternal.ts

@@ -0,0 +1,40 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { Hash, VoteThreshold } from '@polkadot/types/interfaces';
+import { ITuple } from '@polkadot/types/types';
+import { DeriveProposalExternal } from '../types';
+
+import { Observable, of } from 'rxjs';
+import { map, switchMap } from 'rxjs/operators';
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { Option } from '@polkadot/types';
+
+import { memo } from '../util';
+
+function withImage (api: ApiInterfaceRx, nextOpt: Option<ITuple<[Hash, VoteThreshold]>>): Observable<DeriveProposalExternal | null> {
+  if (nextOpt.isNone) {
+    return of(null);
+  }
+
+  const [imageHash, threshold] = nextOpt.unwrap();
+
+  return api.derive.democracy.preimage(imageHash).pipe(
+    map((image): DeriveProposalExternal => ({
+      image,
+      imageHash,
+      threshold
+    }))
+  );
+}
+
+export function nextExternal (api: ApiInterfaceRx): () => Observable<DeriveProposalExternal | null> {
+  return memo((): Observable<DeriveProposalExternal | null> =>
+    api.query.democracy?.nextExternal
+      ? api.query.democracy.nextExternal<Option<ITuple<[Hash, VoteThreshold]>>>().pipe(
+        switchMap((nextOpt) => withImage(api, nextOpt))
+      )
+      : of(null)
+  );
+}

packages/api-derive/src/democracy/preimage.ts

@@ -0,0 +1,25 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { AccountId, Balance, BlockNumber, Hash } from '@polkadot/types/interfaces';
+import { ITuple } from '@polkadot/types/types';
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { DeriveProposalImage } from '../types';
+
+import { Observable } from 'rxjs';
+import { map } from 'rxjs/operators';
+import { Bytes, Option } from '@polkadot/types';
+
+import { memo } from '../util';
+import { parseImage } from './util';
+
+type PreImage = Option<ITuple<[Bytes, AccountId, Balance, BlockNumber]>>;
+
+export function preimage (api: ApiInterfaceRx): (hash: Hash) => Observable<DeriveProposalImage | undefined> {
+  return memo((hashes: Hash[]): Observable<DeriveProposalImage | undefined> =>
+    api.query.democracy.preimages<PreImage>(hashes).pipe(
+      map((imageOpt) => parseImage(api, imageOpt))
+    )
+  );
+}

packages/api-derive/src/democracy/preimages.ts

@@ -0,0 +1,29 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { AccountId, Balance, BlockNumber, Hash } from '@polkadot/types/interfaces';
+import { ITuple } from '@polkadot/types/types';
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { DeriveProposalImage } from '../types';
+
+import { Observable } from 'rxjs';
+import { map } from 'rxjs/operators';
+import { Bytes, Option } from '@polkadot/types';
+
+import { memo } from '../util';
+import { parseImage } from './util';
+
+type PreImage = Option<ITuple<[Bytes, AccountId, Balance, BlockNumber]>>;
+
+export function preimages (api: ApiInterfaceRx): (hashes: Hash[]) => Observable<(DeriveProposalImage | undefined)[]> {
+  return memo((hashes: Hash[]): Observable<(DeriveProposalImage | undefined)[]> =>
+    api.query.democracy.preimages.multi<PreImage>(hashes).pipe(
+      map((images): (DeriveProposalImage | undefined)[] =>
+        images.map((imageOpt) =>
+          parseImage(api, imageOpt)
+        )
+      )
+    )
+  );
+}

packages/api-derive/src/democracy/proposals.ts

@@ -2,61 +2,35 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
-import { AccountId, Balance, BlockNumber, Hash, PropIndex, Proposal } from '@polkadot/types/interfaces';
+import { AccountId, Balance, Hash, PropIndex } from '@polkadot/types/interfaces';
 import { ITuple } from '@polkadot/types/types';
-import { DeriveProposal } from '../types';
+import { DeriveProposalImage, DeriveProposal } from '../types';
 
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { Bytes, Option, Vec } from '@polkadot/types';
+import { Option, Vec } from '@polkadot/types';
 
 import { memo } from '../util';
 
-export type PreImage = Option<ITuple<[Bytes, AccountId, Balance, BlockNumber]>>;
-
 type Depositors = Option<ITuple<[Balance, Vec<AccountId>]>>;
 type Proposals = Vec<ITuple<[PropIndex, Hash, AccountId]>>;
 
-interface Result {
-  depositors: Depositors[];
-  preimages: PreImage[];
-  proposals: Proposals;
-}
-
-function parse (api: ApiInterfaceRx, { depositors, proposals, preimages }: Result): DeriveProposal[] {
+function parse ([proposals, images, depositors]: [Proposals, (DeriveProposalImage | undefined)[], Depositors[]]): DeriveProposal[] {
   return proposals
     .filter(([, , proposer], index): boolean =>
       !!(depositors[index]?.isSome) && !proposer.isEmpty
     )
-    .map(([propIndex, hash, proposer], index): DeriveProposal => {
-      const preimage = preimages[index].unwrapOr(null);
-      const depositor = depositors[index].unwrap();
-      let proposal: undefined | Proposal;
-
-      // we could end up in a situation where the proposal is non-decodable, e.g. after an upgrade
-      if (preimage) {
-        try {
-          proposal = api.registry.createType('Proposal', preimage[0].toU8a(true));
-        } catch (error) {
-          console.error(error);
-        }
-      }
+    .map(([index, imageHash, proposer], proposalIndex): DeriveProposal => {
+      const [balance, seconds] = depositors[proposalIndex].unwrap();
 
       return {
-        balance: depositor[0],
-        hash,
-        index: propIndex,
-        preimage: preimage
-          ? {
-            at: preimage[3],
-            balance: preimage[2],
-            proposer: preimage[1]
-          }
-          : undefined,
-        proposal,
+        balance,
+        index,
+        image: images[proposalIndex],
+        imageHash,
         proposer,
-        seconds: depositor[1]
+        seconds
       };
     });
 }
@@ -68,16 +42,14 @@ export function proposals (api: ApiInterfaceRx): () => Observable<DeriveProposal
         switchMap((proposals) =>
           combineLatest([
             of(proposals),
-            api.query.democracy.preimages.multi<PreImage>(
+            api.derive.democracy.preimages(
               proposals.map(([, hash]): Hash => hash)),
             api.query.democracy.depositOf.multi<Depositors>(
               proposals.map(([index]): PropIndex => index))
           ])
         ),
-        map(([proposals, preimages, depositors]): DeriveProposal[] =>
-          parse(api, { depositors, proposals, preimages })
-        )
+        map(parse)
       )
-      : of([] as DeriveProposal[])
+      : of([])
   );
 }

packages/api-derive/src/democracy/referendumInfo.ts

@@ -1,80 +0,0 @@
-// Copyright 2017-2020 @polkadot/api-derive authors & contributors
-// This software may be modified and distributed under the terms
-// of the Apache-2.0 license. See the LICENSE file for details.
-
-import { ReferendumInfo, ReferendumInfoTo239, ReferendumStatus } from '@polkadot/types/interfaces';
-import { DerivedReferendum } from '../types';
-import { PreImage } from './proposals';
-
-import BN from 'bn.js';
-import { Observable, of } from 'rxjs';
-import { map, switchMap } from 'rxjs/operators';
-import { ApiInterfaceRx } from '@polkadot/api/types';
-import { Option } from '@polkadot/types';
-
-import { memo } from '../util';
-
-function isOld (info: ReferendumInfo | ReferendumInfoTo239): info is ReferendumInfoTo239 {
-  return !!(info as ReferendumInfoTo239).proposalHash;
-}
-
-function getStatus (info: Option<ReferendumInfo | ReferendumInfoTo239>): ReferendumStatus | ReferendumInfoTo239 | null {
-  if (info.isNone) {
-    return null;
-  }
-
-  const unwrapped = info.unwrap();
-
-  if (isOld(unwrapped)) {
-    return unwrapped;
-  } else if (unwrapped.isOngoing) {
-    return unwrapped.asOngoing;
-  }
-
-  // done, we don't include it here... only currently active
-  return null;
-}
-
-function constructInfo (api: ApiInterfaceRx, index: BN | number, status: ReferendumStatus | ReferendumInfoTo239, _preimage?: PreImage): DerivedReferendum | null {
-  const preImage = _preimage?.isSome
-    ? _preimage.unwrap()
-    : null;
-
-  return {
-    index: api.registry.createType('PropIndex', index),
-    hash: status.proposalHash,
-    proposal: preImage
-      ? api.registry.createType('Proposal', preImage[0].toU8a(true))
-      : undefined,
-    preimage: preImage
-      ? {
-        at: preImage[3],
-        balance: preImage[2],
-        proposer: preImage[1]
-      }
-      : undefined,
-    status
-  };
-}
-
-export function retrieveInfo (api: ApiInterfaceRx, index: BN | number, info: Option<ReferendumInfo | ReferendumInfoTo239>): Observable<DerivedReferendum | null> {
-  const status = getStatus(info);
-
-  return status
-    ? api.query.democracy.preimages(status.proposalHash).pipe(
-      map((preimage?: PreImage): DerivedReferendum | null =>
-        constructInfo(api, index, status, preimage)
-      )
-    )
-    : of(null);
-}
-
-export function referendumInfo (api: ApiInterfaceRx): (index: BN | number) => Observable<DerivedReferendum | null> {
-  return memo((index: BN | number): Observable<DerivedReferendum | null> =>
-    api.query.democracy.referendumInfoOf<Option<ReferendumInfo | ReferendumInfoTo239>>(index).pipe(
-      switchMap((info): Observable<DerivedReferendum | null> =>
-        retrieveInfo(api, index, info)
-      )
-    )
-  );
-}

packages/api-derive/src/democracy/referendumInfos.ts

@@ -1,35 +0,0 @@
-// Copyright 2017-2020 @polkadot/api-derive authors & contributors
-// This software may be modified and distributed under the terms
-// of the Apache-2.0 license. See the LICENSE file for details.
-
-import { ReferendumInfo } from '@polkadot/types/interfaces/democracy';
-import { DerivedReferendum } from '../types';
-
-import BN from 'bn.js';
-import { Observable, combineLatest, of } from 'rxjs';
-import { map, switchMap } from 'rxjs/operators';
-import { ApiInterfaceRx } from '@polkadot/api/types';
-import { Option } from '@polkadot/types';
-
-import { memo } from '../util';
-import { retrieveInfo } from './referendumInfo';
-
-export function referendumInfos (api: ApiInterfaceRx): (ids?: (BN | number)[]) => Observable<DerivedReferendum[]> {
-  return memo((ids: (BN | number)[] = []): Observable<DerivedReferendum[]> =>
-    (
-      !ids?.length
-        ? of([] as Option<ReferendumInfo>[])
-        : api.query.democracy.referendumInfoOf.multi<Option<ReferendumInfo>>(ids)
-    ).pipe(
-      switchMap((infos): Observable<(DerivedReferendum | null)[]> =>
-        combineLatest(
-          ids.map((id, index): Observable<DerivedReferendum | null> =>
-            retrieveInfo(api, id, infos[index])
-          )
-        )
-      ),
-      map((infos): DerivedReferendum[] =>
-        infos.filter((referendum): referendum is DerivedReferendum => !!referendum)
-      )
-    ));
-}

packages/api-derive/src/democracy/referendumVotesFor.ts

@@ -1,34 +0,0 @@
-// Copyright 2017-2020 @polkadot/api-derive authors & contributors
-// This software may be modified and distributed under the terms
-// of the Apache-2.0 license. See the LICENSE file for details.
-
-import { AccountId, Vote } from '@polkadot/types/interfaces';
-
-import BN from 'bn.js';
-import { Observable, combineLatest, of } from 'rxjs';
-import { map, switchMap } from 'rxjs/operators';
-import { ApiInterfaceRx } from '@polkadot/api/types';
-import { Vec } from '@polkadot/types';
-
-import { DerivedBalancesAccount, DerivedReferendumVote } from '../types';
-import { memo } from '../util';
-
-export function referendumVotesFor (api: ApiInterfaceRx): (referendumId: BN | number) => Observable<DerivedReferendumVote[]> {
-  return memo((referendumId: BN | number): Observable<DerivedReferendumVote[]> =>
-    api.query.democracy.votersFor<Vec<AccountId>>(referendumId).pipe(
-      switchMap((votersFor): Observable<[Vec<AccountId>, Vote[], DerivedBalancesAccount[]]> =>
-        combineLatest([
-          of(votersFor),
-          api.derive.democracy.votes(referendumId, votersFor),
-          api.derive.balances.votingBalances(votersFor)
-        ])
-      ),
-      map(([votersFor, votes, balances]): DerivedReferendumVote[] =>
-        votersFor.map((accountId, index): DerivedReferendumVote => ({
-          accountId,
-          balance: balances[index].votingBalance || api.registry.createType('Balance'),
-          vote: votes[index] || api.registry.createType('Vote')
-        } as DerivedReferendumVote))
-      )
-    ));
-}

packages/api-derive/src/democracy/referendums.ts

@@ -2,33 +2,29 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
-import { ReferendumIndex } from '@polkadot/types/interfaces/democracy';
-import { DerivedReferendum } from '../types';
-
-import BN from 'bn.js';
-import { Observable, of } from 'rxjs';
-import { switchMap } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
+import { DeriveReferendumExt } from '../types';
+
+import { Observable, combineLatest, of } from 'rxjs';
+import { map, switchMap } from 'rxjs/operators';
 
 import { memo } from '../util';
 
-export function referendums (api: ApiInterfaceRx): () => Observable<DerivedReferendum[]> {
-  return memo((): Observable<DerivedReferendum[]> =>
-    api.query.democracy?.lowestUnbaked
-      ? api.queryMulti<[ReferendumIndex, ReferendumIndex]>([
-        api.query.democracy.lowestUnbaked,
-        api.query.democracy.referendumCount
-      ]).pipe(
-        switchMap(([earliest, referendumCount]): Observable<DerivedReferendum[]> =>
-          referendumCount?.gt(earliest) && referendumCount?.gtn(0)
-            ? api.derive.democracy.referendumInfos(
-              [...Array(referendumCount.sub(earliest).toNumber())].map((_, i): BN =>
-                earliest.addn(i)
-              )
-            )
-            : of([])
-        )
+export function referendums (api: ApiInterfaceRx): () => Observable<DeriveReferendumExt[]> {
+  return memo((): Observable<DeriveReferendumExt[]> =>
+    api.derive.democracy.referendumsActive().pipe(
+      switchMap((referendums) =>
+        combineLatest([
+          of(referendums),
+          api.derive.democracy._referendumsVotes(referendums)
+        ])
+      ),
+      map(([referendums, votes]) =>
+        referendums.map((referendum, index): DeriveReferendumExt => ({
+          ...referendum,
+          ...votes[index]
+        }))
       )
-      : of([] as DerivedReferendum[])
+    )
   );
 }

packages/api-derive/src/democracy/referendumsActive.ts

@@ -0,0 +1,32 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { ReferendumIndex } from '@polkadot/types/interfaces';
+import { DeriveReferendum } from '../types';
+
+import BN from 'bn.js';
+import { Observable, of } from 'rxjs';
+import { switchMap } from 'rxjs/operators';
+
+import { memo } from '../util';
+
+export function referendumsActive (api: ApiInterfaceRx): () => Observable<DeriveReferendum[]> {
+  return memo((): Observable<DeriveReferendum[]> =>
+    api.query.democracy?.lowestUnbaked
+      ? api.queryMulti<[ReferendumIndex, ReferendumIndex]>([
+        api.query.democracy.lowestUnbaked,
+        api.query.democracy.referendumCount
+      ]).pipe(
+        switchMap(([first, total]): Observable<DeriveReferendum[]> =>
+          total.gt(first)
+            ? api.derive.democracy.referendumsInfo(
+              [...Array(total.sub(first).toNumber())].map((_, i): BN => first.addn(i))
+            )
+            : of([])
+        )
+      )
+      : of([])
+  );
+}

packages/api-derive/src/democracy/referendumsInfo.ts

@@ -0,0 +1,33 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { ReferendumInfo } from '@polkadot/types/interfaces';
+import { DeriveReferendum } from '../types';
+
+import BN from 'bn.js';
+import { Observable, of, combineLatest } from 'rxjs';
+import { map, switchMap } from 'rxjs/operators';
+import { Option } from '@polkadot/types';
+
+import { memo } from '../util';
+
+export function referendumsInfo (api: ApiInterfaceRx): (ids: BN[]) => Observable<DeriveReferendum[]> {
+  return memo((ids: BN[]): Observable<DeriveReferendum[]> =>
+    ids.length
+      ? api.query.democracy.referendumInfoOf.multi<Option<ReferendumInfo>>(ids).pipe(
+        switchMap((infos): Observable<(DeriveReferendum | null)[]> =>
+          combineLatest(
+            ids.map((id, index): Observable<DeriveReferendum | null> =>
+              api.derive.democracy._referendumInfo(id, infos[index])
+            )
+          )
+        ),
+        map((infos) =>
+          infos.filter((referendum): referendum is DeriveReferendum => !!referendum)
+        )
+      )
+      : of([])
+  );
+}

packages/api-derive/src/democracy/sqrtElectorate.ts

@@ -0,0 +1,22 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { ApiInterfaceRx } from '@polkadot/api/types';
+
+import BN from 'bn.js';
+import { Observable } from 'rxjs';
+import { map } from 'rxjs/operators';
+import { bnSqrt } from '@polkadot/util';
+
+import { memo } from '../util';
+
+export function sqrtElectorate (api: ApiInterfaceRx): () => Observable<BN> {
+  return memo((): Observable<BN> =>
+    api.query.balances.totalIssuance().pipe(
+      map((totalIssuance) =>
+        bnSqrt(totalIssuance)
+      )
+    )
+  );
+}

packages/api-derive/src/democracy/types.ts

@@ -0,0 +1,69 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { AccountId, Balance, BlockNumber, Hash, Proposal, PropIndex, ReferendumInfoTo239, ReferendumStatus, Vote, ReferendumIndex, VoteThreshold } from '@polkadot/types/interfaces';
+
+import BN from 'bn.js';
+import { Vec } from '@polkadot/types';
+
+export interface DeriveDispatch {
+  at: BlockNumber;
+  index: ReferendumIndex;
+  imageHash: Hash;
+  image?: DeriveProposalImage;
+}
+
+export interface DeriveProposalImage {
+  at: BlockNumber;
+  balance: Balance;
+  proposal?: Proposal;
+  proposer: AccountId;
+}
+
+export interface DeriveProposal {
+  balance?: Balance;
+  index: PropIndex;
+  image?: DeriveProposalImage;
+  imageHash: Hash;
+  proposer: AccountId;
+  seconds: Vec<AccountId>;
+}
+
+export interface DeriveProposalExternal {
+  image?: DeriveProposalImage;
+  imageHash: Hash;
+  threshold: VoteThreshold;
+}
+
+export interface DeriveReferendum {
+  index: ReferendumIndex;
+  image?: DeriveProposalImage;
+  imageHash: Hash;
+  status: ReferendumStatus | ReferendumInfoTo239;
+}
+
+export interface DeriveReferendumVote {
+  accountId: AccountId;
+  balance: Balance;
+  vote: Vote;
+}
+
+export interface DeriveReferendumVoteState {
+  allAye: DeriveReferendumVote[];
+  allNay: DeriveReferendumVote[];
+  voteCount: number;
+  voteCountAye: number;
+  voteCountNay: number;
+  votedAye: BN;
+  votedNay: BN;
+  votedTotal: BN;
+}
+
+export interface DeriveReferendumVotes extends DeriveReferendumVoteState {
+  isPassing: boolean;
+  votes: DeriveReferendumVote[];
+}
+
+export interface DeriveReferendumExt extends DeriveReferendum, DeriveReferendumVotes {
+}

packages/api-derive/src/democracy/util.ts

@@ -0,0 +1,163 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { AccountId, Balance, BlockNumber, Proposal, ReferendumInfo, ReferendumInfoTo239, ReferendumStatus, Tally, VoteThreshold } from '@polkadot/types/interfaces';
+import { ITuple } from '@polkadot/types/types';
+import { DeriveProposalImage, DeriveReferendum, DeriveReferendumVote, DeriveReferendumVotes, DeriveReferendumVoteState } from '../types';
+
+import BN from 'bn.js';
+import { Bytes, Option } from '@polkadot/types';
+import { bnSqrt } from '@polkadot/util';
+
+function isOldInfo (info: ReferendumInfo | ReferendumInfoTo239): info is ReferendumInfoTo239 {
+  return !!(info as ReferendumInfoTo239).proposalHash;
+}
+
+function isCurrentStatus (status: ReferendumStatus | ReferendumInfoTo239): status is ReferendumStatus {
+  return !!(status as ReferendumStatus).tally;
+}
+
+export function compareRationals (n1: BN, d1: BN, n2: BN, d2: BN): boolean {
+  while (true) {
+    const q1 = n1.div(d1);
+    const q2 = n2.div(d2);
+
+    if (q1.lt(q2)) {
+      return true;
+    } else if (q2.lt(q1)) {
+      return false;
+    }
+
+    const r1 = n1.mod(d1);
+    const r2 = n2.mod(d2);
+
+    if (r2.isZero()) {
+      return false;
+    } else if (r1.isZero()) {
+      return true;
+    }
+
+    n1 = d2;
+    n2 = d1;
+    d1 = r2;
+    d2 = r1;
+  }
+}
+
+function isPassing (threshold: VoteThreshold, sqrtElectorate: BN, { votedAye, votedNay, votedTotal }: DeriveReferendumVoteState): boolean {
+  const sqrtVoters = bnSqrt(votedTotal);
+
+  return sqrtVoters.isZero()
+    ? false
+    : threshold.isSimplemajority
+      ? votedAye.gt(votedNay)
+      : threshold.isSupermajorityapproval
+        ? compareRationals(votedNay, sqrtVoters, votedAye, sqrtElectorate)
+        : compareRationals(votedNay, sqrtElectorate, votedAye, sqrtVoters);
+}
+
+function calcVotesPrev (votesFor: DeriveReferendumVote[]): DeriveReferendumVoteState {
+  return votesFor.reduce((state: DeriveReferendumVoteState, derived): DeriveReferendumVoteState => {
+    const { balance, vote } = derived;
+    const isDefault = vote.conviction.index === 0;
+    const counted = balance
+      .muln(isDefault ? 1 : vote.conviction.index)
+      .divn(isDefault ? 10 : 1);
+
+    if (vote.isAye) {
+      state.allAye.push(derived);
+      state.voteCountAye++;
+      state.votedAye = state.votedAye.add(counted);
+    } else {
+      state.allNay.push(derived);
+      state.voteCountNay++;
+      state.votedNay = state.votedNay.add(counted);
+    }
+
+    state.voteCount++;
+    state.votedTotal = state.votedTotal.add(counted);
+
+    return state;
+  }, {
+    allAye: [],
+    allNay: [],
+    voteCount: 0,
+    voteCountAye: 0,
+    voteCountNay: 0,
+    votedAye: new BN(0),
+    votedNay: new BN(0),
+    votedTotal: new BN(0)
+  });
+}
+
+function calcVotesCurrent (tally: Tally, votes: DeriveReferendumVote[]): DeriveReferendumVoteState {
+  const allAye: DeriveReferendumVote[] = [];
+  const allNay: DeriveReferendumVote[] = [];
+
+  votes.forEach((derived): void => {
+    if (derived.vote.isAye) {
+      allAye.push(derived);
+    } else {
+      allNay.push(derived);
+    }
+  });
+
+  return {
+    allAye,
+    allNay,
+    voteCount: allAye.length + allNay.length,
+    voteCountAye: allAye.length,
+    voteCountNay: allNay.length,
+    votedAye: tally.ayes,
+    votedNay: tally.nays,
+    votedTotal: tally.turnout
+  };
+}
+
+export function calcVotes (sqrtElectorate: BN, referendum: DeriveReferendum, votes: DeriveReferendumVote[]): DeriveReferendumVotes {
+  const state = isCurrentStatus(referendum.status)
+    ? calcVotesCurrent(referendum.status.tally, votes)
+    : calcVotesPrev(votes);
+
+  return {
+    ...state,
+    isPassing: isPassing(referendum.status.threshold, sqrtElectorate, state),
+    votes
+  };
+}
+
+export function getStatus (info: Option<ReferendumInfo | ReferendumInfoTo239>): ReferendumStatus | ReferendumInfoTo239 | null {
+  if (info.isNone) {
+    return null;
+  }
+
+  const unwrapped = info.unwrap();
+
+  if (isOldInfo(unwrapped)) {
+    return unwrapped;
+  } else if (unwrapped.isOngoing) {
+    return unwrapped.asOngoing;
+  }
+
+  // done, we don't include it here... only currently active
+  return null;
+}
+
+export function parseImage (api: ApiInterfaceRx, imageOpt: Option<ITuple<[Bytes, AccountId, Balance, BlockNumber]>>): DeriveProposalImage | undefined {
+  if (imageOpt.isNone) {
+    return;
+  }
+
+  let proposal: Proposal | undefined;
+  const [bytes, proposer, balance, at] = imageOpt.unwrap();
+
+  try {
+    proposal = api.registry.createType('Proposal', bytes.toU8a(true));
+  } catch (error) {
+    console.error(error);
+  }
+
+  return { at, balance, proposal, proposer };
+}

packages/api-derive/src/democracy/votes.ts

@@ -1,23 +0,0 @@
-// Copyright 2017-2020 @polkadot/api-derive authors & contributors
-// This software may be modified and distributed under the terms
-// of the Apache-2.0 license. See the LICENSE file for details.
-
-import { AccountId, Vote } from '@polkadot/types/interfaces';
-
-import BN from 'bn.js';
-import { Observable, of } from 'rxjs';
-import { ApiInterfaceRx } from '@polkadot/api/types';
-
-import { memo } from '../util';
-
-export function votes (api: ApiInterfaceRx): (referendumId: BN | number, accountIds?: AccountId[]) => Observable<Vote[]> {
-  return memo((referendumId: BN | number, accountIds: AccountId[] = []): Observable<Vote[]> =>
-    !accountIds || !accountIds.length
-      ? of([] as Vote[])
-      : api.query.democracy.voteOf.multi<Vote>(
-        accountIds.map((accountId): [BN | number, AccountId] =>
-          [referendumId, accountId]
-        )
-      )
-  );
-}

packages/api-derive/src/elections/info.ts

@@ -10,14 +10,14 @@ import { map } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
 import { Vec, u32 } from '@polkadot/types';
 
-import { DerivedElectionsInfo } from '../types';
+import { DeriveElectionsInfo } from '../types';
 import { memo } from '../util';
 
 function sortAccounts ([, balanceA]: ITuple<[AccountId, Balance]>, [, balanceB]: ITuple<[AccountId, Balance]>): number {
   return balanceB.cmp(balanceA);
 }
 
-function queryElections (api: ApiInterfaceRx): Observable<DerivedElectionsInfo> {
+function queryElections (api: ApiInterfaceRx): Observable<DeriveElectionsInfo> {
   const section = api.query.electionsPhragmen ? 'electionsPhragmen' : 'elections';
 
   return api.queryMulti<[Vec<AccountId>, Vec<AccountId>, Vec<ITuple<[AccountId, Balance]>>, Vec<ITuple<[AccountId, Balance]>>]>([
@@ -26,7 +26,7 @@ function queryElections (api: ApiInterfaceRx): Observable<DerivedElectionsInfo>
     api.query[section].members,
     api.query[section].runnersUp
   ]).pipe(
-    map(([councilMembers, candidates, members, runnersUp]): DerivedElectionsInfo => ({
+    map(([councilMembers, candidates, members, runnersUp]): DeriveElectionsInfo => ({
       candidates,
       candidateCount: api.registry.createType('u32', candidates.length),
       candidacyBond: api.consts[section].candidacyBond as Balance,
@@ -54,6 +54,6 @@ function queryElections (api: ApiInterfaceRx): Observable<DerivedElectionsInfo>
  * });
  * ```
  */
-export function info (api: ApiInterfaceRx): () => Observable<DerivedElectionsInfo> {
-  return memo((): Observable<DerivedElectionsInfo> => queryElections(api));
+export function info (api: ApiInterfaceRx): () => Observable<DeriveElectionsInfo> {
+  return memo((): Observable<DeriveElectionsInfo> => queryElections(api));
 }

packages/api-derive/src/imOnline/receivedHeartbeats.ts

@@ -3,7 +3,7 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 import { AccountId } from '@polkadot/types/interfaces';
-import { DerivedHeartbeats } from '../types';
+import { DeriveHeartbeats } from '../types';
 
 import { Observable, of, combineLatest } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
@@ -15,8 +15,8 @@ import { memo } from '../util';
 /**
  * @description Return a boolean array indicating whether the passed accounts had received heartbeats in the current session
  */
-export function receivedHeartbeats (api: ApiInterfaceRx): () => Observable<DerivedHeartbeats> {
-  return memo((): Observable<DerivedHeartbeats> =>
+export function receivedHeartbeats (api: ApiInterfaceRx): () => Observable<DeriveHeartbeats> {
+  return memo((): Observable<DeriveHeartbeats> =>
     api.query.imOnline?.receivedHeartbeats
       ? api.derive.staking.overview().pipe(
         switchMap(({ currentIndex, validators }): Observable<[AccountId[], Option<Bytes>[], u32[]]> =>
@@ -28,8 +28,8 @@ export function receivedHeartbeats (api: ApiInterfaceRx): () => Observable<Deriv
               validators.map((address) => [currentIndex, address]))
           ])
         ),
-        map(([validators, heartbeats, numBlocks]): DerivedHeartbeats =>
-          validators.reduce((result: DerivedHeartbeats, validator, index): DerivedHeartbeats => ({
+        map(([validators, heartbeats, numBlocks]): DeriveHeartbeats =>
+          validators.reduce((result: DeriveHeartbeats, validator, index): DeriveHeartbeats => ({
             ...result,
             [validator.toString()]: {
               blockCount: numBlocks[index],

packages/api-derive/src/index.spec.ts

@@ -49,10 +49,8 @@ describe('derive', (): void => {
     testFunction(api)('chain', 'bestNumber', []);
     testFunction(api)('chain', 'bestNumberFinalized', []);
 
-    testFunction(api)('democracy', 'referendumInfos', []);
+    testFunction(api)('democracy', 'proposals', []);
     testFunction(api)('democracy', 'referendums', []);
-    testFunction(api)('democracy', 'referendumVotesFor', []);
-    testFunction(api)('democracy', 'votes', []);
 
     testFunction(api)('elections', 'info', []);
 

packages/api-derive/src/index.ts

@@ -21,10 +21,11 @@ import * as society from './society';
 import * as staking from './staking';
 import * as technicalCommittee from './technicalCommittee';
 import * as treasury from './treasury';
+import * as tx from './tx';
 
 export * from './type';
 
-export const derive = { accounts, balances, chain, contracts, council, democracy, elections, imOnline, parachains, session, society, staking, technicalCommittee, treasury };
+export const derive = { accounts, balances, chain, contracts, council, democracy, elections, imOnline, parachains, session, society, staking, technicalCommittee, treasury, tx };
 
 type DeriveSection<Section> = {
   [Method in keyof Section]: Section[Method] extends AnyFunction

packages/api-derive/src/session/eraProgress.ts

@@ -12,7 +12,7 @@ import { memo } from '../util';
 
 export function eraProgress (api: ApiInterfaceRx): () => Observable<BlockNumber> {
   return memo((): Observable<BlockNumber> =>
-    api.derive.session.info().pipe(
+    api.derive.session.progress().pipe(
       map((info) => info.eraProgress)
     ));
 }

packages/api-derive/src/session/index.ts

@@ -6,4 +6,5 @@ export * from './eraLength';
 export * from './eraProgress';
 export * from './indexes';
 export * from './info';
+export * from './progress';
 export * from './sessionProgress';

packages/api-derive/src/session/info.ts

@@ -3,110 +3,69 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 import { SessionIndex } from '@polkadot/types/interfaces';
-import { DerivedSessionInfo, DeriveSessionIndexes } from '../types';
+import { DeriveSessionInfo, DeriveSessionIndexes } from '../types';
 
-import { Observable, combineLatest, of } from 'rxjs';
-import { map, switchMap } from 'rxjs/operators';
+import { Observable } from 'rxjs';
+import { map } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { Option, u64 } from '@polkadot/types';
+import { u64 } from '@polkadot/types';
 
 import { memo } from '../util';
 
-type ResultSlots = [u64, u64, u64, Option<SessionIndex>];
-type ResultSlotsFlat = [u64, u64, u64, SessionIndex];
 type ResultType = [boolean, u64, SessionIndex];
-type Result = [ResultType, DeriveSessionIndexes, ResultSlotsFlat];
-
-function createDerived (api: ApiInterfaceRx, [[hasBabe, epochDuration, sessionsPerEra], { activeEra, activeEraStart, currentEra, currentIndex, validatorCount }, [currentSlot, epochIndex, epochOrGenesisStartSlot, activeEraStartSessionIndex]]: Result): DerivedSessionInfo {
-  const epochStartSlot = epochIndex.mul(epochDuration).add(epochOrGenesisStartSlot);
-  const sessionProgress = currentSlot.sub(epochStartSlot);
-  const eraProgress = currentIndex.sub(activeEraStartSessionIndex).mul(epochDuration).add(sessionProgress);
+type Result = [ResultType, DeriveSessionIndexes];
 
+function createDerive (api: ApiInterfaceRx, [[hasBabe, epochDuration, sessionsPerEra], { activeEra, activeEraStart, currentEra, currentIndex, validatorCount }]: Result): DeriveSessionInfo {
   return {
     activeEra,
     activeEraStart,
     currentEra,
     currentIndex,
     eraLength: api.registry.createType('BlockNumber', sessionsPerEra.mul(epochDuration)),
-    eraProgress: api.registry.createType('BlockNumber', eraProgress),
     isEpoch: hasBabe,
     sessionLength: epochDuration,
     sessionsPerEra,
-    sessionProgress: api.registry.createType('BlockNumber', sessionProgress),
     validatorCount
   };
 }
 
-function queryAura (api: ApiInterfaceRx): Observable<DerivedSessionInfo> {
+function queryAura (api: ApiInterfaceRx): Observable<DeriveSessionInfo> {
   return api.derive.session.indexes().pipe(
-    map((indexes): DerivedSessionInfo =>
-      createDerived(api, [
+    map((indexes): DeriveSessionInfo =>
+      createDerive(api, [
         [
           false,
           api.registry.createType('u64', 1),
           api.consts.staking?.sessionsPerEra || api.registry.createType('SessionIndex', 1)
         ],
-        indexes,
-        [
-          api.registry.createType('u64', 1),
-          api.registry.createType('u64', 1),
-          api.registry.createType('u64', 1),
-          api.registry.createType('SessionIndex', 1)
-        ]
+        indexes
       ])
     )
   );
 }
 
-function queryBabe (api: ApiInterfaceRx): Observable<[DeriveSessionIndexes, ResultSlotsFlat]> {
+function queryBabe (api: ApiInterfaceRx): Observable<DeriveSessionInfo> {
   return api.derive.session.indexes().pipe(
-    switchMap((indexes): Observable<[DeriveSessionIndexes, ResultSlots]> =>
-      combineLatest([
-        of(indexes),
-        api.queryMulti<ResultSlots>([
-          api.query.babe.currentSlot,
-          api.query.babe.epochIndex,
-          api.query.babe.genesisSlot,
-          [api.query.staking.erasStartSessionIndex, indexes.activeEra]
-        ])
+    map((indexes) =>
+      createDerive(api, [
+        [
+          true,
+          api.consts.babe.epochDuration,
+          api.consts.staking.sessionsPerEra
+        ],
+        indexes
       ])
-    ),
-    map(([indexes, [currentSlot, epochIndex, genesisSlot, optStartIndex]]): [DeriveSessionIndexes, ResultSlotsFlat] => [
-      indexes, [currentSlot, epochIndex, genesisSlot, optStartIndex.unwrapOr(api.registry.createType('SessionIndex', 1))]
-    ])
+    )
   );
 }
 
-function queryBabeNoHistory (api: ApiInterfaceRx): Observable<[DeriveSessionIndexes, ResultSlotsFlat]> {
-  return combineLatest([
-    api.derive.session.indexes(),
-    api.queryMulti<ResultSlotsFlat>([
-      api.query.babe.currentSlot,
-      api.query.babe.epochIndex,
-      api.query.babe.genesisSlot,
-      api.query.staking.currentEraStartSessionIndex
-    ])
-  ]);
-}
-
 /**
  * @description Retrieves all the session and era query and calculates specific values on it as the length of the session and eras
  */
-export function info (api: ApiInterfaceRx): () => Observable<DerivedSessionInfo> {
-  return memo((): Observable<DerivedSessionInfo> =>
+export function info (api: ApiInterfaceRx): () => Observable<DeriveSessionInfo> {
+  return memo((): Observable<DeriveSessionInfo> =>
     api.consts.babe
-      ? (api.query.staking.erasStartSessionIndex
-        ? queryBabe(api) // 2.x with Babe
-        : queryBabeNoHistory(api)
-      ).pipe(
-        map(([indexes, slots]: [DeriveSessionIndexes, ResultSlotsFlat]): DerivedSessionInfo =>
-          createDerived(api, [
-            [true, api.consts.babe.epochDuration, api.consts.staking.sessionsPerEra],
-            indexes,
-            slots
-          ])
-        )
-      )
+      ? queryBabe(api)
       : queryAura(api)
   );
 }

packages/api-derive/src/session/progress.ts

@@ -0,0 +1,90 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { SessionIndex } from '@polkadot/types/interfaces';
+import { DeriveSessionInfo, DeriveSessionProgress } from '../types';
+
+import { Observable, combineLatest, of } from 'rxjs';
+import { map, switchMap } from 'rxjs/operators';
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { Option, u64 } from '@polkadot/types';
+
+import { memo } from '../util';
+
+type ResultSlots = [u64, u64, u64, Option<SessionIndex>];
+type ResultSlotsFlat = [u64, u64, u64, SessionIndex];
+
+function createDerive (api: ApiInterfaceRx, info: DeriveSessionInfo, [currentSlot, epochIndex, epochOrGenesisStartSlot, activeEraStartSessionIndex]: ResultSlotsFlat): DeriveSessionProgress {
+  const epochStartSlot = epochIndex.mul(info.sessionLength).add(epochOrGenesisStartSlot);
+  const sessionProgress = currentSlot.sub(epochStartSlot);
+  const eraProgress = info.currentIndex.sub(activeEraStartSessionIndex).mul(info.sessionLength).add(sessionProgress);
+
+  return {
+    ...info,
+    eraProgress: api.registry.createType('BlockNumber', eraProgress),
+    sessionProgress: api.registry.createType('BlockNumber', sessionProgress)
+  };
+}
+
+function queryAura (api: ApiInterfaceRx): Observable<DeriveSessionProgress> {
+  return api.derive.session.info().pipe(
+    map((info): DeriveSessionProgress =>
+      createDerive(api, info, [
+        api.registry.createType('u64', 1),
+        api.registry.createType('u64', 1),
+        api.registry.createType('u64', 1),
+        api.registry.createType('SessionIndex', 1)
+      ])
+    )
+  );
+}
+
+function queryBabe (api: ApiInterfaceRx): Observable<[DeriveSessionInfo, ResultSlotsFlat]> {
+  return api.derive.session.info().pipe(
+    switchMap((info): Observable<[DeriveSessionInfo, ResultSlots]> =>
+      combineLatest([
+        of(info),
+        api.queryMulti<ResultSlots>([
+          api.query.babe.currentSlot,
+          api.query.babe.epochIndex,
+          api.query.babe.genesisSlot,
+          [api.query.staking.erasStartSessionIndex, info.activeEra]
+        ])
+      ])
+    ),
+    map(([info, [currentSlot, epochIndex, genesisSlot, optStartIndex]]): [DeriveSessionInfo, ResultSlotsFlat] => [
+      info, [currentSlot, epochIndex, genesisSlot, optStartIndex.unwrapOr(api.registry.createType('SessionIndex', 1))]
+    ])
+  );
+}
+
+function queryBabeNoHistory (api: ApiInterfaceRx): Observable<[DeriveSessionInfo, ResultSlotsFlat]> {
+  return combineLatest([
+    api.derive.session.info(),
+    api.queryMulti<ResultSlotsFlat>([
+      api.query.babe.currentSlot,
+      api.query.babe.epochIndex,
+      api.query.babe.genesisSlot,
+      api.query.staking.currentEraStartSessionIndex
+    ])
+  ]);
+}
+
+/**
+ * @description Retrieves all the session and era query and calculates specific values on it as the length of the session and eras
+ */
+export function progress (api: ApiInterfaceRx): () => Observable<DeriveSessionProgress> {
+  return memo((): Observable<DeriveSessionProgress> =>
+    api.consts.babe
+      ? (api.query.staking.erasStartSessionIndex
+        ? queryBabe(api) // 2.x with Babe
+        : queryBabeNoHistory(api)
+      ).pipe(
+        map(([info, slots]: [DeriveSessionInfo, ResultSlotsFlat]): DeriveSessionProgress =>
+          createDerive(api, info, slots)
+        )
+      )
+      : queryAura(api)
+  );
+}

packages/api-derive/src/session/sessionProgress.ts

@@ -12,7 +12,7 @@ import { memo } from '../util';
 
 export function sessionProgress (api: ApiInterfaceRx): () => Observable<BlockNumber> {
   return memo((): Observable<BlockNumber> =>
-    api.derive.session.info().pipe(
+    api.derive.session.progress().pipe(
       map((info) => info.sessionProgress)
     ));
 }

packages/api-derive/src/session/types.ts

@@ -14,11 +14,14 @@ export interface DeriveSessionIndexes {
   validatorCount: u32;
 }
 
-export interface DerivedSessionInfo extends DeriveSessionIndexes {
+export interface DeriveSessionInfo extends DeriveSessionIndexes {
   eraLength: BlockNumber;
-  eraProgress: BlockNumber;
   isEpoch: boolean;
   sessionLength: BlockNumber;
   sessionsPerEra: SessionIndex;
+}
+
+export interface DeriveSessionProgress extends DeriveSessionInfo {
+  eraProgress: BlockNumber;
   sessionProgress: BlockNumber;
 }

packages/api-derive/src/staking/account.ts

@@ -3,8 +3,8 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { Balance, BlockNumber, StakingLedger, UnlockChunk } from '@polkadot/types/interfaces';
-import { DerivedSessionInfo, DerivedStakingAccount, DerivedStakingQuery, DerivedUnlocking } from '../types';
+import { Balance, StakingLedger, UnlockChunk } from '@polkadot/types/interfaces';
+import { DeriveSessionInfo, DeriveStakingAccount, DeriveStakingQuery, DeriveUnlocking } from '../types';
 
 import BN from 'bn.js';
 import { Observable, combineLatest } from 'rxjs';
@@ -27,27 +27,13 @@ function groupByEra (list: UnlockChunk[]): Record<string, BN> {
   }, {});
 }
 
-// calculate the remaining blocks in a specific unlock era
-function remainingBlocks (api: ApiInterfaceRx, era: BN, sessionInfo: DerivedSessionInfo): BlockNumber {
-  const remaining = era.sub(sessionInfo.currentEra);
-
-  // on the Rust side the current-era >= era-for-unlock (removal done on >)
-  return api.registry.createType('BlockNumber', remaining.gtn(0)
-    ? remaining
-      .subn(1)
-      .mul(sessionInfo.eraLength)
-      .add(sessionInfo.eraLength.sub(sessionInfo.eraProgress))
-    : 0
-  );
-}
-
-function calculateUnlocking (api: ApiInterfaceRx, stakingLedger: StakingLedger | undefined, sessionInfo: DerivedSessionInfo): DerivedUnlocking[] | undefined {
+function calculateUnlocking (api: ApiInterfaceRx, stakingLedger: StakingLedger | undefined, sessionInfo: DeriveSessionInfo): DeriveUnlocking[] | undefined {
   if (isUndefined(stakingLedger)) {
     return undefined;
   }
 
   const unlockingChunks = stakingLedger.unlocking.filter(({ era }): boolean =>
-    remainingBlocks(api, era.unwrap(), sessionInfo).gtn(0)
+    era.unwrap().sub(sessionInfo.activeEra).gtn(0)
   );
 
   if (!unlockingChunks.length) {
@@ -56,27 +42,27 @@ function calculateUnlocking (api: ApiInterfaceRx, stakingLedger: StakingLedger |
 
   // group the unlock chunks that have the same era and sum their values
   const groupedResult = groupByEra(unlockingChunks);
-  const results = Object.entries(groupedResult).map(([eraString, value]): DerivedUnlocking => ({
-    value: api.registry.createType('Balance', value),
-    remainingBlocks: remainingBlocks(api, new BN(eraString), sessionInfo)
+  const results = Object.entries(groupedResult).map(([eraString, value]): DeriveUnlocking => ({
+    remainingEras: new BN(eraString).sub(sessionInfo.activeEra),
+    value: api.registry.createType('Balance', value)
   }));
 
   return results.length ? results : undefined;
 }
 
-function redeemableSum (api: ApiInterfaceRx, stakingLedger: StakingLedger | undefined, sessionInfo: DerivedSessionInfo): Balance {
+function redeemableSum (api: ApiInterfaceRx, stakingLedger: StakingLedger | undefined, sessionInfo: DeriveSessionInfo): Balance {
   if (isUndefined(stakingLedger)) {
     return api.registry.createType('Balance');
   }
 
   return api.registry.createType('Balance', stakingLedger.unlocking.reduce((total, { era, value }): BN => {
-    return remainingBlocks(api, era.unwrap(), sessionInfo).eqn(0)
+    return sessionInfo.activeEra.gte(era.unwrap())
       ? total.add(value.unwrap())
       : total;
   }, new BN(0)));
 }
 
-function parseResult (api: ApiInterfaceRx, sessionInfo: DerivedSessionInfo, query: DerivedStakingQuery): DerivedStakingAccount {
+function parseResult (api: ApiInterfaceRx, sessionInfo: DeriveSessionInfo, query: DeriveStakingQuery): DeriveStakingAccount {
   return {
     ...query,
     redeemable: redeemableSum(api, query.stakingLedger, sessionInfo),
@@ -87,13 +73,13 @@ function parseResult (api: ApiInterfaceRx, sessionInfo: DerivedSessionInfo, quer
 /**
  * @description From a stash, retrieve the controllerId and fill in all the relevant staking details
  */
-export function account (api: ApiInterfaceRx): (accountId: Uint8Array | string) => Observable<DerivedStakingAccount> {
-  return memo((accountId: Uint8Array | string): Observable<DerivedStakingAccount> =>
+export function account (api: ApiInterfaceRx): (accountId: Uint8Array | string) => Observable<DeriveStakingAccount> {
+  return memo((accountId: Uint8Array | string): Observable<DeriveStakingAccount> =>
     combineLatest([
       api.derive.session.info(),
       api.derive.staking.query(accountId)
     ]).pipe(
-      map(([sessionInfo, query]: [DerivedSessionInfo, DerivedStakingQuery]) =>
+      map(([sessionInfo, query]: [DeriveSessionInfo, DeriveStakingQuery]) =>
         parseResult(api, sessionInfo, query))
     ));
 }

packages/api-derive/src/staking/electedInfo.ts

@@ -4,27 +4,27 @@
 
 import { ApiInterfaceRx } from '@polkadot/api/types';
 import { AccountId } from '@polkadot/types/interfaces';
-import { DerivedStakingQuery, DerivedStakingElected } from '../types';
+import { DeriveStakingQuery, DeriveStakingElected } from '../types';
 
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 
 import { memo } from '../util';
 
-export function electedInfo (api: ApiInterfaceRx): () => Observable<DerivedStakingElected> {
-  return memo((): Observable<DerivedStakingElected> =>
+export function electedInfo (api: ApiInterfaceRx): () => Observable<DeriveStakingElected> {
+  return memo((): Observable<DeriveStakingElected> =>
     api.derive.staking.validators().pipe(
-      switchMap(({ nextElected }): Observable<[AccountId[], DerivedStakingQuery[]]> =>
+      switchMap(({ nextElected }): Observable<[AccountId[], DeriveStakingQuery[]]> =>
         combineLatest([
           of(nextElected),
           combineLatest(
-            nextElected.map((accountId): Observable<DerivedStakingQuery> =>
+            nextElected.map((accountId): Observable<DeriveStakingQuery> =>
               api.derive.staking.query(accountId)
             )
           )
         ])
       ),
-      map(([nextElected, info]): DerivedStakingElected => ({
+      map(([nextElected, info]): DeriveStakingElected => ({
         info,
         nextElected
       }))

packages/api-derive/src/staking/overview.ts

@@ -4,7 +4,7 @@
 
 import { AccountId, EraPoints, EraRewardPoints, RewardPoint } from '@polkadot/types/interfaces';
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { DerivedStakingOverview } from '../types';
+import { DeriveStakingOverview } from '../types';
 
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
@@ -29,8 +29,8 @@ function retrievePointsPrev (api: ApiInterfaceRx, currentElected: AccountId[]):
 /**
  * @description Retrieve the staking overview, including elected and points earned
  */
-export function overview (api: ApiInterfaceRx): () => Observable<DerivedStakingOverview> {
-  return memo((): Observable<DerivedStakingOverview> =>
+export function overview (api: ApiInterfaceRx): () => Observable<DeriveStakingOverview> {
+  return memo((): Observable<DeriveStakingOverview> =>
     combineLatest([
       api.derive.session.indexes(),
       api.derive.staking.validators()
@@ -45,7 +45,7 @@ export function overview (api: ApiInterfaceRx): () => Observable<DerivedStakingO
               : of(api.registry.createType('EraRewardPoints'))
         ])
       ),
-      map(([info, eraPoints]): DerivedStakingOverview => ({
+      map(([info, eraPoints]): DeriveStakingOverview => ({
         ...info, eraPoints
       }))
     ));

packages/api-derive/src/staking/query.ts

@@ -5,7 +5,7 @@
 import { ApiInterfaceRx } from '@polkadot/api/types';
 import { AccountId, Exposure, Keys, Nominations, RewardDestination, ValidatorPrefs } from '@polkadot/types/interfaces';
 import { ITuple } from '@polkadot/types/types';
-import { DerivedStakingQuery } from '../types';
+import { DeriveStakingQuery } from '../types';
 
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
@@ -55,13 +55,13 @@ function retrieveCurr (api: ApiInterfaceRx, stashId: AccountId): Observable<Mult
   );
 }
 
-function retrieveController (api: ApiInterfaceRx, stashId: AccountId, [queuedKeys, [controllerIdOpt, nominatorsOpt, rewardDestination, validatorPrefs, nextKeys, exposure]]: [Vec<ITuple<[AccountId, Keys]>>, MultiResult]): Observable<DerivedStakingQuery> {
+function retrieveController (api: ApiInterfaceRx, stashId: AccountId, [queuedKeys, [controllerIdOpt, nominatorsOpt, rewardDestination, validatorPrefs, nextKeys, exposure]]: [Vec<ITuple<[AccountId, Keys]>>, MultiResult]): Observable<DeriveStakingQuery> {
   const controllerId = controllerIdOpt.unwrapOr(null);
   const nominators = nominatorsOpt.unwrapOr(null);
 
   return controllerId
     ? api.query.staking.ledger(controllerId).pipe(
-      map((stakingLedgerOpt): DerivedStakingQuery => ({
+      map((stakingLedgerOpt): DeriveStakingQuery => ({
         accountId: stashId,
         controllerId,
         exposure,
@@ -85,8 +85,8 @@ function retrieveController (api: ApiInterfaceRx, stashId: AccountId, [queuedKey
 /**
  * @description From a stash, retrieve the controllerId and all relevant details
  */
-export function query (api: ApiInterfaceRx): (accountId: Uint8Array | string) => Observable<DerivedStakingQuery> {
-  return memo((accountId: Uint8Array | string): Observable<DerivedStakingQuery> => {
+export function query (api: ApiInterfaceRx): (accountId: Uint8Array | string) => Observable<DeriveStakingQuery> {
+  return memo((accountId: Uint8Array | string): Observable<DeriveStakingQuery> => {
     const stashId = api.registry.createType('AccountId', accountId);
 
     return combineLatest([
@@ -95,15 +95,15 @@ export function query (api: ApiInterfaceRx): (accountId: Uint8Array | string) =>
         ? retrieveCurr(api, stashId)
         : retrievePrev(api, stashId)
     ]).pipe(
-      switchMap((result): Observable<DerivedStakingQuery> =>
+      switchMap((result): Observable<DeriveStakingQuery> =>
         retrieveController(api, stashId, result)
       )
     );
   });
 }
 
-export function queryMulti (api: ApiInterfaceRx): (...accountIds: (Uint8Array | string)[]) => Observable<DerivedStakingQuery[]> {
-  return memo((...accountIds: (Uint8Array | string)[]): Observable<DerivedStakingQuery[]> =>
+export function queryMulti (api: ApiInterfaceRx): (...accountIds: (Uint8Array | string)[]) => Observable<DeriveStakingQuery[]> {
+  return memo((...accountIds: (Uint8Array | string)[]): Observable<DeriveStakingQuery[]> =>
     combineLatest(
       accountIds.map((accountId) => api.derive.staking.query(accountId))
     )

packages/api-derive/src/staking/types.ts

@@ -2,9 +2,11 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
-import { AccountId, Balance, BlockNumber, EraIndex, EraRewardPoints, Exposure, Keys, RewardDestination, RewardPoint, StakingLedger, ValidatorPrefs } from '@polkadot/types/interfaces';
+import { AccountId, Balance, EraIndex, EraRewardPoints, Exposure, Keys, RewardDestination, RewardPoint, StakingLedger, ValidatorPrefs } from '@polkadot/types/interfaces';
 import { DeriveSessionIndexes } from '../session/types';
 
+import BN from 'bn.js';
+
 export type DeriveEraValPoints = Record<string, RewardPoint>;
 
 export type DeriveEraValPrefs = Record<string, ValidatorPrefs>;
@@ -79,9 +81,9 @@ export interface DeriveStakerSlashes {
   total: Balance;
 }
 
-export interface DerivedStakingElected {
+export interface DeriveStakingElected {
   nextElected: AccountId[];
-  info: DerivedStakingQuery[];
+  info: DeriveStakingQuery[];
 }
 
 export interface DeriveStakingValidators {
@@ -89,7 +91,7 @@ export interface DeriveStakingValidators {
   validators: AccountId[];
 }
 
-export interface DerivedStakingStash {
+export interface DeriveStakingStash {
   controllerId?: AccountId;
   exposure?: Exposure;
   nominators?: AccountId[];
@@ -100,25 +102,25 @@ export interface DerivedStakingStash {
   validatorPrefs?: ValidatorPrefs;
 }
 
-export interface DerivedStakingQuery extends DerivedStakingStash {
+export interface DeriveStakingQuery extends DeriveStakingStash {
   accountId: AccountId;
   nextSessionIds: AccountId[];
   sessionIds: AccountId[];
   stakingLedger?: StakingLedger;
 }
 
-export interface DerivedStakingAccount extends DerivedStakingQuery {
+export interface DeriveStakingAccount extends DeriveStakingQuery {
   redeemable?: Balance;
-  unlocking?: DerivedUnlocking[];
+  unlocking?: DeriveUnlocking[];
 }
 
-export interface DerivedStakingOverview extends DeriveSessionIndexes {
+export interface DeriveStakingOverview extends DeriveSessionIndexes {
   eraPoints: EraRewardPoints;
   nextElected: AccountId[];
   validators: AccountId[];
 }
 
-export type DerivedUnlocking = {
-  remainingBlocks: BlockNumber;
+export type DeriveUnlocking = {
+  remainingEras: BN;
   value: Balance;
 };

packages/api-derive/src/technicalCommittee/proposals.ts

@@ -3,12 +3,12 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { DerivedCollectiveProposals } from '../types';
+import { DeriveCollectiveProposals } from '../types';
 
 import { Observable } from 'rxjs';
 import { proposals as collectiveProposals } from '../collective';
 import { memo } from '../util';
 
-export function proposals (api: ApiInterfaceRx): () => Observable<DerivedCollectiveProposals> {
+export function proposals (api: ApiInterfaceRx): () => Observable<DeriveCollectiveProposals> {
   return memo(collectiveProposals(api, 'technicalCommittee'));
 }

packages/api-derive/src/treasury/proposals.ts

@@ -4,7 +4,7 @@
 
 import { ApiInterfaceRx } from '@polkadot/api/types';
 import { ProposalIndex, TreasuryProposal } from '@polkadot/types/interfaces';
-import { DerivedCollectiveProposals, DerivedTreasuryProposal, DerivedTreasuryProposals } from '../types';
+import { DeriveCollectiveProposals, DeriveTreasuryProposal, DeriveTreasuryProposals } from '../types';
 
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
@@ -16,13 +16,13 @@ interface Result {
   allIds: ProposalIndex[];
   allProposals: Option<TreasuryProposal>[];
   approvalIds: ProposalIndex[];
-  councilProposals: DerivedCollectiveProposals;
+  councilProposals: DeriveCollectiveProposals;
   proposalCount: ProposalIndex;
 }
 
-function parseResult (_api: ApiInterfaceRx, { allIds, allProposals, approvalIds, councilProposals, proposalCount }: Result): DerivedTreasuryProposals {
-  const approvals: DerivedTreasuryProposal[] = [];
-  const proposals: DerivedTreasuryProposal[] = [];
+function parseResult (_api: ApiInterfaceRx, { allIds, allProposals, approvalIds, councilProposals, proposalCount }: Result): DeriveTreasuryProposals {
+  const approvals: DeriveTreasuryProposal[] = [];
+  const proposals: DeriveTreasuryProposal[] = [];
   const councilTreasury = councilProposals.filter(({ proposal: { methodName, sectionName } }): boolean =>
     sectionName === 'treasury' &&
     ['approveProposal', 'rejectProposal'].includes(methodName)
@@ -47,7 +47,7 @@ function parseResult (_api: ApiInterfaceRx, { allIds, allProposals, approvalIds,
   return { approvals, proposalCount, proposals };
 }
 
-function retrieveProposals (api: ApiInterfaceRx, proposalCount: ProposalIndex, approvalIds: ProposalIndex[]): Observable<DerivedTreasuryProposals> {
+function retrieveProposals (api: ApiInterfaceRx, proposalCount: ProposalIndex, approvalIds: ProposalIndex[]): Observable<DeriveTreasuryProposals> {
   const proposalIds: ProposalIndex[] = [];
   const count = proposalCount.toNumber();
 
@@ -65,7 +65,7 @@ function retrieveProposals (api: ApiInterfaceRx, proposalCount: ProposalIndex, a
     api.query.treasury.proposals.multi<Option<TreasuryProposal>>(allIds),
     api.derive.council.proposals()
   ]).pipe(
-    map(([allProposals, councilProposals]: [Option<TreasuryProposal>[], DerivedCollectiveProposals]): DerivedTreasuryProposals =>
+    map(([allProposals, councilProposals]: [Option<TreasuryProposal>[], DeriveCollectiveProposals]): DeriveTreasuryProposals =>
       parseResult(api, { allIds, allProposals, approvalIds, councilProposals, proposalCount })
     )
   );
@@ -74,8 +74,8 @@ function retrieveProposals (api: ApiInterfaceRx, proposalCount: ProposalIndex, a
 /**
  * @description Retrieve all active and approved treasury proposals, along with their info
  */
-export function proposals (api: ApiInterfaceRx): () => Observable<DerivedTreasuryProposals> {
-  return memo((): Observable<DerivedTreasuryProposals> =>
+export function proposals (api: ApiInterfaceRx): () => Observable<DeriveTreasuryProposals> {
+  return memo((): Observable<DeriveTreasuryProposals> =>
     api.query.treasury
       ? combineLatest([
         api.query.treasury.proposalCount(),
@@ -89,6 +89,6 @@ export function proposals (api: ApiInterfaceRx): () => Observable<DerivedTreasur
         approvals: [],
         proposalCount: api.registry.createType('ProposalIndex'),
         proposals: []
-      } as DerivedTreasuryProposals)
+      } as DeriveTreasuryProposals)
   );
 }

packages/api-derive/src/tx/constants.ts

@@ -0,0 +1,7 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import BN from 'bn.js';
+
+export const MAX_FINALITY_LAG = new BN(5);

packages/api-derive/src/tx/events.ts

@@ -0,0 +1,30 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { EventRecord, Hash, SignedBlock } from '@polkadot/types/interfaces';
+
+import { Observable, combineLatest } from 'rxjs';
+import { map } from 'rxjs/operators';
+import { ApiInterfaceRx } from '@polkadot/api/types';
+
+import { memo } from '../util';
+
+interface Result {
+  block: SignedBlock;
+  events: EventRecord[];
+}
+
+export function events (api: ApiInterfaceRx): (at: Hash) => Observable<Result> {
+  return memo((at: Hash): Observable<Result> =>
+    combineLatest([
+      api.query.system.events.at(at),
+      api.rpc.chain.getBlock(at)
+    ]).pipe(
+      map(([events, block]) => ({
+        block,
+        events
+      }))
+    )
+  );
+}

packages/api-derive/src/tx/index.ts

@@ -0,0 +1,6 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+export * from './events';
+export * from './signingInfo';

packages/api-derive/src/tx/signingInfo.ts

@@ -0,0 +1,69 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { Header, Index } from '@polkadot/types/interfaces';
+import { AnyNumber, Codec, IExtrinsicEra } from '@polkadot/types/types';
+
+import BN from 'bn.js';
+import { Observable, combineLatest, of } from 'rxjs';
+import { map, switchMap } from 'rxjs/operators';
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { isNumber, isUndefined } from '@polkadot/util';
+
+interface Result {
+  header: Header | null;
+  mortalLength: number;
+  nonce: Index;
+}
+
+// default here to 5 min eras, adjusted based on the actual blocktime
+const FALLBACK_PERIOD = new BN(6 * 1000);
+const MAX_FINALITY_LAG = new BN(5);
+const MORTAL_PERIOD = new BN(5 * 60 * 1000);
+
+function latestNonce (api: ApiInterfaceRx, address: string): Observable<Index> {
+  return api.derive.balances.account(address).pipe(
+    map(({ accountNonce }): Index => accountNonce)
+  );
+}
+
+function signingHeader (api: ApiInterfaceRx): Observable<Header> {
+  return combineLatest([
+    api.rpc.chain.getHeader(),
+    api.rpc.chain.getFinalizedHead().pipe(
+      switchMap((hash) => api.rpc.chain.getHeader(hash))
+    )
+  ]).pipe(
+    map(([current, finalized]): Header =>
+      // determine the hash to use, current when lag > max, else finalized
+      current.number.unwrap().sub(finalized.number.unwrap()).gt(MAX_FINALITY_LAG)
+        ? current
+        : finalized
+    )
+  );
+}
+
+export function signingInfo (api: ApiInterfaceRx): (address: string, nonce?: AnyNumber | Codec, era?: IExtrinsicEra | number) => Observable<Result> {
+  // no memo, we want to do this fresh on each run
+  return (address: string, nonce?: AnyNumber | Codec, era?: IExtrinsicEra | number): Observable<Result> =>
+    combineLatest([
+      // retrieve nonce if none was specified
+      isUndefined(nonce)
+        ? latestNonce(api, address)
+        : of(api.registry.createType('Index', nonce)),
+      // if no era (create) or era > 0 (mortal), do block retrieval
+      (isUndefined(era) || (isNumber(era) && era > 0))
+        ? signingHeader(api)
+        : of(null)
+    ]).pipe(
+      map(([nonce, header]) => ({
+        header,
+        nonce,
+        mortalLength: MORTAL_PERIOD
+          .div(api.consts.babe?.expectedBlockTime || api.consts.timestamp?.minimumPeriod.muln(2) || FALLBACK_PERIOD)
+          .add(MAX_FINALITY_LAG)
+          .toNumber()
+      }))
+    );
+}

packages/api-derive/src/types.ts

@@ -2,11 +2,13 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
-import { AccountId, AccountIndex, Balance, BalanceLock, BalanceLockTo212, BalanceOf, Bid, BidKind, BlockNumber, Hash, Index, Proposal, PropIndex, ProposalIndex, ReferendumInfoTo239, ReferendumStatus, RegistrationJudgement, SetIndex, SocietyVote, StrikeCount, TreasuryProposal, Vote, Votes, VoteIndex, VouchingStatus } from '@polkadot/types/interfaces';
+import { AccountId, AccountIndex, Balance, BalanceLock, BalanceLockTo212, BalanceOf, Bid, BidKind, BlockNumber, Hash, Index, Proposal, ProposalIndex, RegistrationJudgement, SetIndex, SocietyVote, StrikeCount, TreasuryProposal, Votes, VoteIndex, VouchingStatus } from '@polkadot/types/interfaces';
 
 import BN from 'bn.js';
-import { u32, Vec } from '@polkadot/types';
+import { u32 } from '@polkadot/types';
 
+export * from './council/types';
+export * from './democracy/types';
 export * from './parachains/types';
 export * from './session/types';
 export * from './staking/types';
@@ -42,7 +44,7 @@ export interface DeriveAccountInfo extends DeriveAccountFlags {
   nickname?: string;
 }
 
-export interface DerivedBalancesAccount {
+export interface DeriveBalancesAccount {
   accountId: AccountId;
   accountNonce: Index;
   freeBalance: Balance;
@@ -52,7 +54,7 @@ export interface DerivedBalancesAccount {
   votingBalance: Balance;
 }
 
-export interface DerivedBalancesAll extends DerivedBalancesAccount {
+export interface DeriveBalancesAll extends DeriveBalancesAccount {
   isVesting: boolean;
   lockedBalance: Balance;
   lockedBreakdown: (BalanceLock | BalanceLockTo212)[];
@@ -62,9 +64,9 @@ export interface DerivedBalancesAll extends DerivedBalancesAccount {
   vestingTotal: Balance;
 }
 
-export type DerivedBalancesMap = Record<string, DerivedBalancesAll>;
+export type DeriveBalancesMap = Record<string, DeriveBalancesAll>;
 
-export interface DerivedContractFees {
+export interface DeriveContractFees {
   callBaseFee: BN;
   contractFee: BN;
   creationFee: BN;
@@ -76,15 +78,15 @@ export interface DerivedContractFees {
   transferFee: BN;
 }
 
-export interface DerivedCollectiveProposal {
+export interface DeriveCollectiveProposal {
   hash: Hash;
   proposal: Proposal;
   votes: Votes | null;
 }
 
-export type DerivedCollectiveProposals = DerivedCollectiveProposal[];
+export type DeriveCollectiveProposals = DeriveCollectiveProposal[];
 
-export interface DerivedElectionsInfo {
+export interface DeriveElectionsInfo {
   candidates: AccountId[];
   candidateCount: u32;
   candidacyBond?: Balance;
@@ -98,7 +100,7 @@ export interface DerivedElectionsInfo {
   votingBond?: Balance;
 }
 
-export interface DerivedFees {
+export interface DeriveFees {
   creationFee: Balance;
   existentialDeposit: Balance;
   transactionBaseFee: Balance;
@@ -106,50 +108,20 @@ export interface DerivedFees {
   transferFee: Balance;
 }
 
-export interface DerivedHeartbeatAuthor {
+export interface DeriveHeartbeatAuthor {
   blockCount: u32;
   hasMessage: boolean;
   isOnline: boolean;
 }
 
-export type DerivedHeartbeats = Record<string, DerivedHeartbeatAuthor>;
+export type DeriveHeartbeats = Record<string, DeriveHeartbeatAuthor>;
 
 export interface RecentlyOffline {
   blockNumber: BlockNumber;
   count: BN;
 }
 
-export interface DeriveProposalPreImage {
-  at: BlockNumber;
-  balance: Balance;
-  proposer: AccountId;
-}
-
-export interface DeriveProposal {
-  balance?: Balance;
-  hash: Hash;
-  index: PropIndex;
-  preimage?: DeriveProposalPreImage;
-  proposal?: Proposal;
-  proposer: AccountId;
-  seconds: Vec<AccountId>;
-}
-
-export interface DerivedReferendum {
-  hash: Hash;
-  index: PropIndex;
-  preimage?: DeriveProposalPreImage;
-  proposal?: Proposal;
-  status: ReferendumStatus | ReferendumInfoTo239;
-}
-
-export type DerivedRecentlyOffline = Record<string, RecentlyOffline[]>;
-
-export interface DerivedReferendumVote {
-  accountId: AccountId;
-  balance: Balance;
-  vote: Vote;
-}
+export type DeriveRecentlyOffline = Record<string, RecentlyOffline[]>;
 
 export interface DeriveSociety {
   bids: Bid[];
@@ -177,16 +149,16 @@ export interface DeriveSocietyMember {
   vouching?: VouchingStatus;
 }
 
-export interface DerivedTreasuryProposal {
-  council: DerivedCollectiveProposal[];
+export interface DeriveTreasuryProposal {
+  council: DeriveCollectiveProposal[];
   id: ProposalIndex;
   proposal: TreasuryProposal;
 }
 
-export interface DerivedTreasuryProposals {
-  approvals: DerivedTreasuryProposal[];
+export interface DeriveTreasuryProposals {
+  approvals: DeriveTreasuryProposal[];
   proposalCount: ProposalIndex;
-  proposals: DerivedTreasuryProposal[];
+  proposals: DeriveTreasuryProposal[];
 }
 
 export interface VoterPosition {
@@ -195,4 +167,4 @@ export interface VoterPosition {
   setIndex: SetIndex;
 }
 
-export type DerivedVoterPositions = Record<string, VoterPosition>;
+export type DeriveVoterPositions = Record<string, VoterPosition>;

packages/api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/api",
-  "version": "1.8.1",
+  "version": "1.9.1",
   "description": "Promise and RxJS wrappers around the Polkadot JS RPC",
   "main": "index.js",
   "keywords": [
@@ -27,19 +27,20 @@
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/api#readme",
   "dependencies": {
     "@babel/runtime": "^7.9.2",
-    "@polkadot/api-derive": "1.8.1",
-    "@polkadot/keyring": "^2.6.2",
-    "@polkadot/metadata": "1.8.1",
-    "@polkadot/rpc-core": "1.8.1",
-    "@polkadot/rpc-provider": "1.8.1",
-    "@polkadot/types": "1.8.1",
-    "@polkadot/util": "^2.6.2",
-    "@polkadot/util-crypto": "^2.6.2",
+    "@polkadot/api-derive": "1.9.1",
+    "@polkadot/keyring": "^2.7.1",
+    "@polkadot/metadata": "1.9.1",
+    "@polkadot/rpc-core": "1.9.1",
+    "@polkadot/rpc-provider": "1.9.1",
+    "@polkadot/types": "1.9.1",
+    "@polkadot/types-known": "1.9.1",
+    "@polkadot/util": "^2.7.1",
+    "@polkadot/util-crypto": "^2.7.1",
     "bn.js": "^5.1.1",
     "eventemitter3": "^4.0.0",
     "rxjs": "^6.5.4"
   },
   "devDependencies": {
-    "@polkadot/keyring": "^2.6.2"
+    "@polkadot/keyring": "^2.7.1"
   }
 }

packages/api/src/augment/consts.ts

@@ -113,10 +113,6 @@ declare module '@polkadot/metadata/Decorated/consts/types' {
        * Period in blocks where an external proposal may not be re-submitted after being vetoed.
        **/
       cooloffPeriod: AugmentedConst<BlockNumber>;
-      /**
-       * Minimum voting period allowed for an emergency referendum.
-       **/
-      emergencyVotingPeriod: AugmentedConst<BlockNumber>;
       /**
        * The minimum period of locking and the period between a proposal being approved and enacted.
        * It should generally be a little more than the unstake period to ensure that
@@ -124,6 +120,10 @@ declare module '@polkadot/metadata/Decorated/consts/types' {
        * they are on the losing side of a vote.
        **/
       enactmentPeriod: AugmentedConst<BlockNumber>;
+      /**
+       * Minimum voting period allowed for an emergency referendum.
+       **/
+      fastTrackVotingPeriod: AugmentedConst<BlockNumber>;
       /**
        * How often (in blocks) new public referenda are launched.
        **/

packages/api/src/augment/query.ts

@@ -5,23 +5,23 @@ import { AnyNumber, ITuple } from '@polkadot/types/types';
 import { Option, U8aFixed, Vec } from '@polkadot/types/codec';
 import { Bytes, Data, bool, u32, u64 } from '@polkadot/types/primitive';
 import { UncleEntryItem } from '@polkadot/types/interfaces/authorship';
-import { BabeAuthorityWeight, MaybeVrf } from '@polkadot/types/interfaces/babe';
+import { BabeAuthorityWeight, MaybeVrf, Randomness } from '@polkadot/types/interfaces/babe';
 import { AccountData, BalanceLock, ReleasesBalances } from '@polkadot/types/interfaces/balances';
 import { ProposalIndex, Votes } from '@polkadot/types/interfaces/collective';
-import { AuthorityId } from '@polkadot/types/interfaces/consensus';
+import { AuthorityId, RawVRFOutput } from '@polkadot/types/interfaces/consensus';
 import { CodeHash, ContractInfo, Gas, PrefabWasmModule, Schedule } from '@polkadot/types/interfaces/contracts';
-import { Conviction, PropIndex, Proposal, ProxyState, ReferendumIndex, ReferendumInfo } from '@polkadot/types/interfaces/democracy';
-import { Vote, VoteThreshold } from '@polkadot/types/interfaces/elections';
-import { AuthorityList, SetId, StoredPendingChange, StoredState } from '@polkadot/types/interfaces/grandpa';
+import { PropIndex, Proposal, ProxyState, ReferendumIndex, ReferendumInfo, Voting } from '@polkadot/types/interfaces/democracy';
+import { VoteThreshold } from '@polkadot/types/interfaces/elections';
+import { SetId, StoredPendingChange, StoredState } from '@polkadot/types/interfaces/grandpa';
 import { RegistrarInfo, Registration } from '@polkadot/types/interfaces/identity';
 import { AuthIndex } from '@polkadot/types/interfaces/imOnline';
-import { Kind, OffenceDetails, OpaqueTimeSlot, ReportIdOf } from '@polkadot/types/interfaces/offences';
+import { DeferredOffenceOf, Kind, OffenceDetails, OpaqueTimeSlot, ReportIdOf } from '@polkadot/types/interfaces/offences';
 import { ActiveRecovery, RecoveryConfig } from '@polkadot/types/interfaces/recovery';
 import { AccountId, AccountIndex, Balance, BalanceOf, BlockNumber, Hash, KeyTypeId, Moment, Perbill, ValidatorId, Weight } from '@polkadot/types/interfaces/runtime';
 import { Keys, SessionIndex } from '@polkadot/types/interfaces/session';
 import { Bid, BidKind, SocietyVote, StrikeCount, VouchingStatus } from '@polkadot/types/interfaces/society';
-import { ActiveEraInfo, EraIndex, EraRewardPoints, Exposure, Forcing, Nominations, ReleasesStaking, RewardDestination, SlashingSpans, SpanIndex, SpanRecord, StakingLedger, UnappliedSlash, ValidatorPrefs } from '@polkadot/types/interfaces/staking';
-import { AccountInfo, DigestOf, EventIndex, EventRecord, LastRuntimeUpgradeInfo } from '@polkadot/types/interfaces/system';
+import { ActiveEraInfo, ElectionResult, ElectionStatus, EraIndex, EraRewardPoints, Exposure, Forcing, Nominations, PhragmenScore, ReleasesStaking, RewardDestination, SlashingSpans, SpanIndex, SpanRecord, StakingLedger, UnappliedSlash, ValidatorPrefs } from '@polkadot/types/interfaces/staking';
+import { AccountInfo, DigestOf, EventIndex, EventRecord, LastRuntimeUpgradeInfo, Phase } from '@polkadot/types/interfaces/system';
 import { OpenTip, TreasuryProposal } from '@polkadot/types/interfaces/treasury';
 import { Multiplier } from '@polkadot/types/interfaces/txpayment';
 import { Multisig } from '@polkadot/types/interfaces/utility';
@@ -73,7 +73,7 @@ declare module '@polkadot/api/types/storage' {
       /**
        * Next epoch randomness.
        **/
-      nextRandomness: AugmentedQuery<ApiType, () => Observable<U8aFixed>> & QueryableStorageEntry<ApiType>;
+      nextRandomness: AugmentedQuery<ApiType, () => Observable<Randomness>> & QueryableStorageEntry<ApiType>;
       /**
        * The epoch randomness for the *current* epoch.
        * # Security
@@ -84,7 +84,7 @@ declare module '@polkadot/api/types/storage' {
        * used where a number is needed that cannot have been chosen by an
        * adversary, for purposes such as public-coin zero-knowledge proofs.
        **/
-      randomness: AugmentedQuery<ApiType, () => Observable<U8aFixed>> & QueryableStorageEntry<ApiType>;
+      randomness: AugmentedQuery<ApiType, () => Observable<Randomness>> & QueryableStorageEntry<ApiType>;
       /**
        * Randomness under construction.
        * We make a tradeoff between storage accesses and list length.
@@ -95,7 +95,7 @@ declare module '@polkadot/api/types/storage' {
        * epoch.
        **/
       segmentIndex: AugmentedQuery<ApiType, () => Observable<u32>> & QueryableStorageEntry<ApiType>;
-      underConstruction: AugmentedQuery<ApiType, (arg: u32 | AnyNumber | Uint8Array) => Observable<Vec<U8aFixed>>> & QueryableStorageEntry<ApiType>;
+      underConstruction: AugmentedQuery<ApiType, (arg: u32 | AnyNumber | Uint8Array) => Observable<Vec<RawVRFOutput>>> & QueryableStorageEntry<ApiType>;
     };
     balances: {
       [index: string]: QueryableStorageEntry<ApiType>;
@@ -191,10 +191,6 @@ declare module '@polkadot/api/types/storage' {
        * Record of all proposals that have been subject to emergency cancellation.
        **/
       cancellations: AugmentedQuery<ApiType, (arg: Hash | string | Uint8Array) => Observable<bool>> & QueryableStorageEntry<ApiType>;
-      /**
-       * Get the account (and lock periods) to which another account is delegating vote.
-       **/
-      delegations: AugmentedQuery<ApiType, (arg: AccountId | string | Uint8Array) => Observable<ITuple<[AccountId, Conviction]>>> & QueryableStorageEntry<ApiType>;
       /**
        * Those who have locked a deposit.
        **/
@@ -252,22 +248,16 @@ declare module '@polkadot/api/types/storage' {
        **/
       referendumInfoOf: AugmentedQuery<ApiType, (arg: ReferendumIndex | AnyNumber | Uint8Array) => Observable<Option<ReferendumInfo>>> & QueryableStorageEntry<ApiType>;
       /**
-       * Get the vote in a given referendum of a particular voter. The result is meaningful only
-       * if `voters_for` includes the voter when called with the referendum (you'll get the
-       * default `Vote` value otherwise). If you don't want to check `voters_for`, then you can
-       * also check for simple existence with `VoteOf::contains_key` first.
+       * All votes for a particular voter. We store the balance for the number of votes that we
+       * have recorded. The second item is the total amount of delegations, that will be added.
        **/
-      voteOf: AugmentedQuery<ApiType, (arg: ITuple<[ReferendumIndex, AccountId]> | [ReferendumIndex | AnyNumber | Uint8Array, AccountId | string | Uint8Array]) => Observable<Vote>> & QueryableStorageEntry<ApiType>;
-      /**
-       * Get the voters for the current proposal.
-       **/
-      votersFor: AugmentedQuery<ApiType, (arg: ReferendumIndex | AnyNumber | Uint8Array) => Observable<Vec<AccountId>>> & QueryableStorageEntry<ApiType>;
+      votingOf: AugmentedQuery<ApiType, (arg: AccountId | string | Uint8Array) => Observable<Voting>> & QueryableStorageEntry<ApiType>;
     };
     elections: {
       [index: string]: QueryableStorageEntry<ApiType>;
       /**
-       * The present candidate list. Sorted based on account-id. A current member or a runner can
-       * never enter this vector and is always implicitly assumed to be a candidate.
+       * The present candidate list. Sorted based on account-id. A current member or runner-up
+       * can never enter this vector and is always implicitly assumed to be a candidate.
        **/
       candidates: AugmentedQuery<ApiType, () => Observable<Vec<AccountId>>> & QueryableStorageEntry<ApiType>;
       /**
@@ -283,22 +273,12 @@ declare module '@polkadot/api/types/storage' {
        **/
       runnersUp: AugmentedQuery<ApiType, () => Observable<Vec<ITuple<[AccountId, BalanceOf]>>>> & QueryableStorageEntry<ApiType>;
       /**
-       * Locked stake of a voter.
+       * Votes and locked stake of a particular voter.
        **/
-      stakeOf: AugmentedQuery<ApiType, (arg: AccountId | string | Uint8Array) => Observable<BalanceOf>> & QueryableStorageEntry<ApiType>;
-      /**
-       * Votes of a particular voter, with the round index of the votes.
-       **/
-      votesOf: AugmentedQuery<ApiType, (arg: AccountId | string | Uint8Array) => Observable<Vec<AccountId>>> & QueryableStorageEntry<ApiType>;
+      voting: AugmentedQuery<ApiType, (arg: AccountId | string | Uint8Array) => Observable<ITuple<[BalanceOf, Vec<AccountId>]>>> & QueryableStorageEntry<ApiType>;
     };
     grandpa: {
       [index: string]: QueryableStorageEntry<ApiType>;
-      /**
-       * DEPRECATED
-       * This used to store the current authority set, which has been migrated to the well-known
-       * GRANDPA_AUTHORITIES_KEY unhashed key.
-       **/
-      authorities: AugmentedQuery<ApiType, () => Observable<AuthorityList>> & QueryableStorageEntry<ApiType>;
       /**
        * The number of changes (both in terms of keys and underlying economic responsibilities)
        * in the "set" of Grandpa validators from genesis.
@@ -387,6 +367,11 @@ declare module '@polkadot/api/types/storage' {
        * A vector of reports of the same kind that happened at the same time slot.
        **/
       concurrentReportsIndex: AugmentedQueryDoubleMap<ApiType, (key1: Kind | string | Uint8Array, key2: OpaqueTimeSlot | string | Uint8Array) => Observable<Vec<ReportIdOf>>> & QueryableStorageEntry<ApiType>;
+      /**
+       * Deferred reports that have been rejected by the offence handler and need to be submitted
+       * at a later time.
+       **/
+      deferredOffences: AugmentedQuery<ApiType, () => Observable<Vec<DeferredOffenceOf>>> & QueryableStorageEntry<ApiType>;
       /**
        * The primary structure that holds all offence records keyed by report identifiers.
        **/
@@ -561,6 +546,11 @@ declare module '@polkadot/api/types/storage' {
        * The earliest era for which we have a pending, unapplied slash.
        **/
       earliestUnappliedSlash: AugmentedQuery<ApiType, () => Observable<Option<EraIndex>>> & QueryableStorageEntry<ApiType>;
+      /**
+       * Flag to control the execution of the offchain election. When `Open(_)`, we accept
+       * solutions to be submitted.
+       **/
+      eraElectionStatus: AugmentedQuery<ApiType, () => Observable<ElectionStatus>> & QueryableStorageEntry<ApiType>;
       /**
        * Rewards for the last `HISTORY_DEPTH` eras.
        * If reward hasn't been set or has been removed then 0 reward is returned.
@@ -605,7 +595,7 @@ declare module '@polkadot/api/types/storage' {
        **/
       erasValidatorReward: AugmentedQuery<ApiType, (arg: EraIndex | AnyNumber | Uint8Array) => Observable<Option<BalanceOf>>> & QueryableStorageEntry<ApiType>;
       /**
-       * True if the next session change will be a new era regardless of index.
+       * Mode of era forcing.
        **/
       forceEra: AugmentedQuery<ApiType, () => Observable<Forcing>> & QueryableStorageEntry<ApiType>;
       /**
@@ -622,6 +612,10 @@ declare module '@polkadot/api/types/storage' {
        * invulnerables) and restricted to testnets.
        **/
       invulnerables: AugmentedQuery<ApiType, () => Observable<Vec<AccountId>>> & QueryableStorageEntry<ApiType>;
+      /**
+       * True if the current planned session is final.
+       **/
+      isCurrentSessionFinal: AugmentedQuery<ApiType, () => Observable<bool>> & QueryableStorageEntry<ApiType>;
       /**
        * Map from all (unlocked) "controller" accounts to the info regarding the staking.
        **/
@@ -642,6 +636,16 @@ declare module '@polkadot/api/types/storage' {
        * Where the reward payment should be made. Keyed by stash.
        **/
       payee: AugmentedQuery<ApiType, (arg: AccountId | string | Uint8Array) => Observable<RewardDestination>> & QueryableStorageEntry<ApiType>;
+      /**
+       * The next validator set. At the end of an era, if this is available (potentially from the
+       * result of an offchain worker), it is immediately used. Otherwise, the on-chain election
+       * is executed.
+       **/
+      queuedElected: AugmentedQuery<ApiType, () => Observable<Option<ElectionResult>>> & QueryableStorageEntry<ApiType>;
+      /**
+       * The score of the current [`QueuedElected`].
+       **/
+      queuedScore: AugmentedQuery<ApiType, () => Observable<Option<PhragmenScore>>> & QueryableStorageEntry<ApiType>;
       /**
        * Slashing spans for stash accounts.
        **/
@@ -651,14 +655,25 @@ declare module '@polkadot/api/types/storage' {
        * The rest of the slashed value is handled by the `Slash`.
        **/
       slashRewardFraction: AugmentedQuery<ApiType, () => Observable<Perbill>> & QueryableStorageEntry<ApiType>;
+      /**
+       * Snapshot of nominators at the beginning of the current election window. This should only
+       * have a value when [`EraElectionStatus`] == `ElectionStatus::Open(_)`.
+       **/
+      snapshotNominators: AugmentedQuery<ApiType, () => Observable<Option<Vec<AccountId>>>> & QueryableStorageEntry<ApiType>;
+      /**
+       * Snapshot of validators at the beginning of the current election window. This should only
+       * have a value when [`EraElectionStatus`] == `ElectionStatus::Open(_)`.
+       **/
+      snapshotValidators: AugmentedQuery<ApiType, () => Observable<Option<Vec<AccountId>>>> & QueryableStorageEntry<ApiType>;
       /**
        * Records information about the maximum slash of a stash within a slashing span,
        * as well as how much reward has been paid out.
        **/
       spanSlash: AugmentedQuery<ApiType, (arg: ITuple<[AccountId, SpanIndex]> | [AccountId | string | Uint8Array, SpanIndex | AnyNumber | Uint8Array]) => Observable<SpanRecord>> & QueryableStorageEntry<ApiType>;
       /**
+       * True if network has been upgraded to this version.
        * Storage version of the pallet.
-       * This is set to v2.0.0 for new networks.
+       * This is set to v3.0.0 for new networks.
        **/
       storageVersion: AugmentedQuery<ApiType, () => Observable<ReleasesStaking>> & QueryableStorageEntry<ApiType>;
       /**
@@ -727,6 +742,10 @@ declare module '@polkadot/api/types/storage' {
        * no notification will be triggered thus the event might be lost.
        **/
       eventTopics: AugmentedQuery<ApiType, (arg: Hash | string | Uint8Array) => Observable<Vec<ITuple<[BlockNumber, EventIndex]>>>> & QueryableStorageEntry<ApiType>;
+      /**
+       * The execution phase of the block.
+       **/
+      executionPhase: AugmentedQuery<ApiType, () => Observable<Option<Phase>>> & QueryableStorageEntry<ApiType>;
       /**
        * Total extrinsics count for the current block.
        **/

packages/api/src/augment/rpc.ts

@@ -5,15 +5,20 @@ import { AnyNumber, Codec, IExtrinsic } from '@polkadot/types/types';
 import { HashMap, Option, Vec } from '@polkadot/types/codec';
 import { Bytes, Null, StorageKey, Text, bool, u32, u64 } from '@polkadot/types/primitive';
 import { Metadata } from '@polkadot/types';
+import { ExtrinsicOrHash, ExtrinsicStatus } from '@polkadot/types/interfaces/author';
 import { EpochAuthorship } from '@polkadot/types/interfaces/babe';
+import { BlockHash } from '@polkadot/types/interfaces/chain';
+import { PrefixedStorageKey } from '@polkadot/types/interfaces/childstate';
 import { AuthorityId } from '@polkadot/types/interfaces/consensus';
 import { ContractCallRequest, ContractExecResult } from '@polkadot/types/interfaces/contracts';
 import { CreatedBlock } from '@polkadot/types/interfaces/engine';
 import { Extrinsic } from '@polkadot/types/interfaces/extrinsics';
 import { StorageKind } from '@polkadot/types/interfaces/offchain';
-import { BlockHash, ChainProperties, ExtrinsicOrHash, ExtrinsicStatus, Health, NetworkState, PeerInfo, RpcMethods, RuntimeDispatchInfo, RuntimeVersion } from '@polkadot/types/interfaces/rpc';
+import { RuntimeDispatchInfo } from '@polkadot/types/interfaces/payment';
+import { RpcMethods } from '@polkadot/types/interfaces/rpc';
 import { AccountId, BlockNumber, H256, Hash, Header, Index, Justification, KeyValue, SignedBlock, StorageData } from '@polkadot/types/interfaces/runtime';
-import { NodeRole } from '@polkadot/types/interfaces/system';
+import { RuntimeVersion } from '@polkadot/types/interfaces/state';
+import { ChainProperties, Health, NetworkState, NodeRole, PeerInfo } from '@polkadot/types/interfaces/system';
 import { Observable } from 'rxjs';
 
 declare module '@polkadot/rpc-core/types.jsonrpc' {
@@ -44,7 +49,7 @@ declare module '@polkadot/rpc-core/types.jsonrpc' {
       /**
        * Remove given extrinsic from the pool and temporarily ban it to prevent reimporting
        **/
-      removeExtrinsic: AugmentedRpc<(bytesOrHash: Vec<ExtrinsicOrHash> | (ExtrinsicOrHash | { hash: any } | { extrinsic: any } | string | Uint8Array)[]) => Observable<Vec<Hash>>>;
+      removeExtrinsic: AugmentedRpc<(bytesOrHash: Vec<ExtrinsicOrHash> | (ExtrinsicOrHash | { Hash: any } | { Extrinsic: any } | string | Uint8Array)[]) => Observable<Vec<Hash>>>;
       /**
        * Generate new session keys and returns the corresponding public keys
        **/
@@ -94,6 +99,24 @@ declare module '@polkadot/rpc-core/types.jsonrpc' {
        **/
       subscribeNewHeads: AugmentedRpc<() => Observable<Header>>;
     };
+    childstate: {
+      /**
+       * Returns the keys with prefix from a child storage, leave empty to get all the keys
+       **/
+      getKeys: AugmentedRpc<(childKey: PrefixedStorageKey | string | Uint8Array, prefix: StorageKey | string | Uint8Array | any, at?: Hash | string | Uint8Array) => Observable<Vec<StorageKey>>>;
+      /**
+       * Returns a child storage entry at a specific block state
+       **/
+      getStorage: AugmentedRpc<(childKey: PrefixedStorageKey | string | Uint8Array, key: StorageKey | string | Uint8Array | any, at?: Hash | string | Uint8Array) => Observable<Option<StorageData>>>;
+      /**
+       * Returns the hash of a child storage entry at a block state
+       **/
+      getStorageHash: AugmentedRpc<(childKey: PrefixedStorageKey | string | Uint8Array, key: StorageKey | string | Uint8Array | any, at?: Hash | string | Uint8Array) => Observable<Option<Hash>>>;
+      /**
+       * Returns the size of a child storage entry at a block state
+       **/
+      getStorageSize: AugmentedRpc<(childKey: PrefixedStorageKey | string | Uint8Array, key: StorageKey | string | Uint8Array | any, at?: Hash | string | Uint8Array) => Observable<Option<u64>>>;
+    };
     contracts: {
       /**
        * Executes a call to a contract
@@ -197,6 +220,10 @@ declare module '@polkadot/rpc-core/types.jsonrpc' {
        * Query historical storage entries (by key) starting from a start block
        **/
       queryStorage: AugmentedRpc<<T = Codec[]>(keys: Vec<StorageKey> | (StorageKey | string | Uint8Array | any)[], fromBlock?: Hash | Uint8Array | string, toBlock?: Hash | Uint8Array | string) => Observable<[Hash, T][]>>;
+      /**
+       * Query storage entries (by key) starting at block hash given as the second parameter
+       **/
+      queryStorageAt: AugmentedRpc<<T = Codec[]>(keys: Vec<StorageKey> | (StorageKey | string | Uint8Array | any)[], at?: Hash | Uint8Array | string) => Observable<T>>;
       /**
        * Retrieves the runtime version via subscription
        **/

packages/api/src/augment/tx.ts

@@ -6,15 +6,14 @@ import { Compact, Option, U8aFixed, Vec } from '@polkadot/types/codec';
 import { Bytes, Data, bool, u16, u32, u64 } from '@polkadot/types/primitive';
 import { MemberCount, ProposalIndex } from '@polkadot/types/interfaces/collective';
 import { CodeHash, Gas, Schedule } from '@polkadot/types/interfaces/contracts';
-import { Conviction, PropIndex, Proposal, ReferendumIndex } from '@polkadot/types/interfaces/democracy';
-import { Vote } from '@polkadot/types/interfaces/elections';
+import { AccountVote, Conviction, PropIndex, Proposal, ReferendumIndex } from '@polkadot/types/interfaces/democracy';
 import { Extrinsic, Signature } from '@polkadot/types/interfaces/extrinsics';
 import { IdentityFields, IdentityInfo, IdentityJudgement, RegistrarIndex } from '@polkadot/types/interfaces/identity';
 import { Heartbeat } from '@polkadot/types/interfaces/imOnline';
 import { AccountId, AccountIndex, Address, Balance, BalanceOf, BlockNumber, Call, ChangesTrieConfiguration, Hash, Header, KeyValue, LookupSource, Moment, Perbill } from '@polkadot/types/interfaces/runtime';
 import { Keys } from '@polkadot/types/interfaces/session';
 import { SocietyJudgement } from '@polkadot/types/interfaces/society';
-import { EraIndex, RewardDestination, ValidatorPrefs } from '@polkadot/types/interfaces/staking';
+import { CompactAssignments, EraIndex, PhragmenScore, RewardDestination, ValidatorIndex, ValidatorPrefs } from '@polkadot/types/interfaces/staking';
 import { Key } from '@polkadot/types/interfaces/system';
 import { Timepoint } from '@polkadot/types/interfaces/utility';
 import { VestingInfo } from '@polkadot/types/interfaces/vesting';
@@ -179,6 +178,7 @@ declare module '@polkadot/api/types/submittable' {
        * - `which`: The index of the referendum to cancel.
        * # <weight>
        * - One DB change.
+       * - O(d) where d is the items in the dispatch queue.
        * # </weight>
        **/
       cancelQueued: AugmentedSubmittable<(which: ReferendumIndex | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
@@ -221,25 +221,30 @@ declare module '@polkadot/api/types/submittable' {
        **/
       deactivateProxy: AugmentedSubmittable<(proxy: AccountId | string | Uint8Array) => SubmittableExtrinsic<ApiType>>;
       /**
-       * Delegate vote.
-       * Currency is locked indefinitely for as long as it's delegated.
-       * The dispatch origin of this call must be _Signed_.
-       * - `to`: The account to make a delegate of the sender.
-       * - `conviction`: The conviction that will be attached to the delegated
-       * votes.
+       * Delegate the voting power (with some given conviction) of the sending account.
+       * The balance delegated is locked for as long as it's delegated, and thereafter for the
+       * time appropriate for the conviction's lock period.
+       * The dispatch origin of this call must be _Signed_, and the signing account must either:
+       * - be delegating already; or
+       * - have no voting activity (if there is, then it will need to be removed/consolidated
+       * through `reap_vote` or `unvote`).
+       * - `to`: The account whose voting the `target` account's voting power will follow.
+       * - `conviction`: The conviction that will be attached to the delegated votes. When the
+       * account is undelegated, the funds will be locked for the corresponding period.
+       * - `balance`: The amount of the account's balance to be used in delegating. This must
+       * not be more than the account's current balance.
        * Emits `Delegated`.
        * # <weight>
-       * - One extra DB entry.
        * # </weight>
        **/
-      delegate: AugmentedSubmittable<(to: AccountId | string | Uint8Array, conviction: Conviction | ('None' | 'Locked1x' | 'Locked2x' | 'Locked3x' | 'Locked4x' | 'Locked5x' | 'Locked6x') | number | Uint8Array) => SubmittableExtrinsic<ApiType>>;
+      delegate: AugmentedSubmittable<(to: AccountId | string | Uint8Array, conviction: Conviction | ('None' | 'Locked1x' | 'Locked2x' | 'Locked3x' | 'Locked4x' | 'Locked5x' | 'Locked6x') | number | Uint8Array, balance: BalanceOf | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
       /**
        * Schedule an emergency cancellation of a referendum. Cannot happen twice to the same
        * referendum.
        * The dispatch origin of this call must be `CancellationOrigin`.
        * -`ref_index`: The index of the referendum to cancel.
        * # <weight>
-       * - Depends on size of storage vec `VotersFor` for this referendum.
+       * - `O(1)`.
        * # </weight>
        **/
       emergencyCancel: AugmentedSubmittable<(refIndex: ReferendumIndex | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
@@ -287,7 +292,7 @@ declare module '@polkadot/api/types/submittable' {
        * The dispatch of this call must be `FastTrackOrigin`.
        * - `proposal_hash`: The hash of the current external proposal.
        * - `voting_period`: The period that is allowed for voting on this proposal. Increased to
-       * `EmergencyVotingPeriod` if too low.
+       * `FastTrackVotingPeriod` if too low.
        * - `delay`: The number of block after voting has ended in approval and this should be
        * enacted. This doesn't have a minimum amount.
        * Emits `Started`.
@@ -305,7 +310,7 @@ declare module '@polkadot/api/types/submittable' {
        * - `encoded_proposal`: The preimage of a proposal.
        * Emits `PreimageNoted`.
        * # <weight>
-       * - Dependent on the size of `encoded_proposal`.
+       * - Dependent on the size of `encoded_proposal` and length of dispatch queue.
        * # </weight>
        **/
       noteImminentPreimage: AugmentedSubmittable<(encodedProposal: Bytes | string | Uint8Array) => SubmittableExtrinsic<ApiType>>;
@@ -340,11 +345,57 @@ declare module '@polkadot/api/types/submittable' {
        * - `value`: The amount of deposit (must be at least `MinimumDeposit`).
        * Emits `Proposed`.
        * # <weight>
-       * - `O(1)`.
+       * - `O(P)`
+       * - P is the number proposals in the `PublicProps` vec.
        * - Two DB changes, one DB entry.
        * # </weight>
        **/
       propose: AugmentedSubmittable<(proposalHash: Hash | string | Uint8Array, value: Compact<BalanceOf> | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
+      /**
+       * Delegate the voting power (with some given conviction) of a proxied account.
+       * The balance delegated is locked for as long as it's delegated, and thereafter for the
+       * time appropriate for the conviction's lock period.
+       * The dispatch origin of this call must be _Signed_, and the signing account must have
+       * been set as the proxy account for `target`.
+       * - `target`: The account whole voting power shall be delegated and whose balance locked.
+       * This account must either:
+       * - be delegating already; or
+       * - have no voting activity (if there is, then it will need to be removed/consolidated
+       * through `reap_vote` or `unvote`).
+       * - `to`: The account whose voting the `target` account's voting power will follow.
+       * - `conviction`: The conviction that will be attached to the delegated votes. When the
+       * account is undelegated, the funds will be locked for the corresponding period.
+       * - `balance`: The amount of the account's balance to be used in delegating. This must
+       * not be more than the account's current balance.
+       * Emits `Delegated`.
+       * # <weight>
+       * # </weight>
+       **/
+      proxyDelegate: AugmentedSubmittable<(to: AccountId | string | Uint8Array, conviction: Conviction | ('None' | 'Locked1x' | 'Locked2x' | 'Locked3x' | 'Locked4x' | 'Locked5x' | 'Locked6x') | number | Uint8Array, balance: BalanceOf | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
+      /**
+       * Remove a proxied vote for a referendum.
+       * Exactly equivalent to `remove_vote` except that it operates on the account that the
+       * sender is a proxy for.
+       * The dispatch origin of this call must be _Signed_ and the signing account must be a
+       * proxy for some other account which has a registered vote for the referendum of `index`.
+       * - `index`: The index of referendum of the vote to be removed.
+       * # <weight>
+       * - `O(R + log R)` where R is the number of referenda that `target` has voted on.
+       * # </weight>
+       **/
+      proxyRemoveVote: AugmentedSubmittable<(index: ReferendumIndex | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
+      /**
+       * Undelegate the voting power of a proxied account.
+       * Tokens may be unlocked following once an amount of time consistent with the lock period
+       * of the conviction with which the delegation was issued.
+       * The dispatch origin of this call must be _Signed_ and the signing account must be a
+       * proxy for some other account which is currently delegating.
+       * Emits `Undelegated`.
+       * # <weight>
+       * - O(1).
+       * # </weight>
+       **/
+      proxyUndelegate: AugmentedSubmittable<() => SubmittableExtrinsic<ApiType>>;
       /**
        * Vote in a referendum on behalf of a stash. If `vote.is_aye()`, the vote is to enact
        * the proposal; otherwise it is a vote to keep the status quo.
@@ -356,7 +407,7 @@ declare module '@polkadot/api/types/submittable' {
        * - One DB change, one DB entry.
        * # </weight>
        **/
-      proxyVote: AugmentedSubmittable<(refIndex: Compact<ReferendumIndex> | AnyNumber | Uint8Array, vote: Vote | { aye: boolean; conviction?: ('None' | 'Locked1x' | 'Locked2x' | 'Locked3x' | 'Locked4x' | 'Locked5x' | 'Locked6x') | number } | boolean | string | Uint8Array) => SubmittableExtrinsic<ApiType>>;
+      proxyVote: AugmentedSubmittable<(refIndex: Compact<ReferendumIndex> | AnyNumber | Uint8Array, vote: AccountVote | { Standard: any } | { Split: any } | string | Uint8Array) => SubmittableExtrinsic<ApiType>>;
       /**
        * Remove an expired proposal preimage and collect the deposit.
        * The dispatch origin of this call must be _Signed_.
@@ -370,23 +421,65 @@ declare module '@polkadot/api/types/submittable' {
        * # </weight>
        **/
       reapPreimage: AugmentedSubmittable<(proposalHash: Hash | string | Uint8Array) => SubmittableExtrinsic<ApiType>>;
+      /**
+       * Remove a vote for a referendum.
+       * If the `target` is equal to the signer, then this function is exactly equivalent to
+       * `remove_vote`. If not equal to the signer, then the vote must have expired,
+       * either because the referendum was cancelled, because the voter lost the referendum or
+       * because the conviction period is over.
+       * The dispatch origin of this call must be _Signed_.
+       * - `target`: The account of the vote to be removed; this account must have voted for
+       * referendum `index`.
+       * - `index`: The index of referendum of the vote to be removed.
+       * # <weight>
+       * - `O(R + log R)` where R is the number of referenda that `target` has voted on.
+       * # </weight>
+       **/
+      removeOtherVote: AugmentedSubmittable<(target: AccountId | string | Uint8Array, index: ReferendumIndex | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
+      /**
+       * Remove a vote for a referendum.
+       * If:
+       * - the referendum was cancelled, or
+       * - the referendum is ongoing, or
+       * - the referendum has ended such that
+       * - the vote of the account was in opposition to the result; or
+       * - there was no conviction to the account's vote; or
+       * - the account made a split vote
+       * ...then the vote is removed cleanly and a following call to `unlock` may result in more
+       * funds being available.
+       * If, however, the referendum has ended and:
+       * - it finished corresponding to the vote of the account, and
+       * - the account made a standard vote with conviction, and
+       * - the lock period of the conviction is not over
+       * ...then the lock will be aggregated into the overall account's lock, which may involve
+       * *overlocking* (where the two locks are combined into a single lock that is the maximum
+       * of both the amount locked and the time is it locked for).
+       * The dispatch origin of this call must be _Signed_, and the signer must have a vote
+       * registered for referendum `index`.
+       * - `index`: The index of referendum of the vote to be removed.
+       * # <weight>
+       * - `O(R + log R)` where R is the number of referenda that `target` has voted on.
+       * # </weight>
+       **/
+      removeVote: AugmentedSubmittable<(index: ReferendumIndex | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
       /**
        * Signals agreement with a particular proposal.
        * The dispatch origin of this call must be _Signed_ and the sender
        * must have funds to cover the deposit, equal to the original deposit.
        * - `proposal`: The index of the proposal to second.
        * # <weight>
-       * - `O(1)`.
+       * - `O(S)`.
+       * - S is the number of seconds a proposal already has.
        * - One DB entry.
        * # </weight>
        **/
       second: AugmentedSubmittable<(proposal: Compact<PropIndex> | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
       /**
-       * Undelegate vote.
-       * Must be sent from an account that has called delegate previously.
-       * The tokens will be reduced from an indefinite lock to the maximum
-       * possible according to the conviction of the prior delegation.
-       * The dispatch origin of this call must be _Signed_.
+       * Undelegate the voting power of the sending account.
+       * Tokens may be unlocked following once an amount of time consistent with the lock period
+       * of the conviction with which the delegation was issued.
+       * The dispatch origin of this call must be _Signed_ and the signing account must be
+       * currently delegating.
        * Emits `Undelegated`.
        * # <weight>
        * - O(1).
@@ -397,7 +490,6 @@ declare module '@polkadot/api/types/submittable' {
        * Unlock tokens that have an expired lock.
        * The dispatch origin of this call must be _Signed_.
        * - `target`: The account to remove the lock on.
-       * Emits `Unlocked`.
        * # <weight>
        * - `O(1)`.
        * # </weight>
@@ -413,6 +505,7 @@ declare module '@polkadot/api/types/submittable' {
        * - One DB clear.
        * - Performs a binary search on `existing_vetoers` which should not
        * be very large.
+       * - O(log v), v is number of `existing_vetoers`
        * # </weight>
        **/
       vetoExternal: AugmentedSubmittable<(proposalHash: Hash | string | Uint8Array) => SubmittableExtrinsic<ApiType>>;
@@ -423,11 +516,12 @@ declare module '@polkadot/api/types/submittable' {
        * - `ref_index`: The index of the referendum to vote for.
        * - `vote`: The vote configuration.
        * # <weight>
-       * - `O(1)`.
+       * - `O(R)`.
+       * - R is the number of referendums the voter has voted on.
        * - One DB change, one DB entry.
        * # </weight>
        **/
-      vote: AugmentedSubmittable<(refIndex: Compact<ReferendumIndex> | AnyNumber | Uint8Array, vote: Vote | { aye: boolean; conviction?: ('None' | 'Locked1x' | 'Locked2x' | 'Locked3x' | 'Locked4x' | 'Locked5x' | 'Locked6x') | number } | boolean | string | Uint8Array) => SubmittableExtrinsic<ApiType>>;
+      vote: AugmentedSubmittable<(refIndex: Compact<ReferendumIndex> | AnyNumber | Uint8Array, vote: AccountVote | { Standard: any } | { Split: any } | string | Uint8Array) => SubmittableExtrinsic<ApiType>>;
     };
     elections: {
       [index: string]: SubmittableExtrinsicFunction<ApiType>;
@@ -601,7 +695,7 @@ declare module '@polkadot/api/types/submittable' {
        * - One event.
        * # </weight>
        **/
-      provideJudgement: AugmentedSubmittable<(regIndex: Compact<RegistrarIndex> | AnyNumber | Uint8Array, target: LookupSource | Address | AccountId | AccountIndex | string | Uint8Array, judgement: IdentityJudgement | { unknown: any } | { feePaid: any } | { reasonable: any } | { knownGood: any } | { outOfDate: any } | { lowQuality: any } | { erroneous: any } | string | Uint8Array) => SubmittableExtrinsic<ApiType>>;
+      provideJudgement: AugmentedSubmittable<(regIndex: Compact<RegistrarIndex> | AnyNumber | Uint8Array, target: LookupSource | Address | AccountId | AccountIndex | string | Uint8Array, judgement: IdentityJudgement | { Unknown: any } | { FeePaid: any } | { Reasonable: any } | { KnownGood: any } | { OutOfDate: any } | { LowQuality: any } | { Erroneous: any } | string | Uint8Array) => SubmittableExtrinsic<ApiType>>;
       /**
        * Request a judgement from a registrar.
        * Payment: At most `max_fee` will be reserved for payment to the registrar if judgement
@@ -688,7 +782,7 @@ declare module '@polkadot/api/types/submittable' {
        * one storage-exists.
        * # </weight>
        **/
-      setSubs: AugmentedSubmittable<(subs: Vec<ITuple<[AccountId, Data]>> | ([AccountId | string | Uint8Array, Data | { none: any } | { raw: any } | { blakeTwo256: any } | { sha256: any } | { keccak256: any } | { shaThree256: any } | string | Uint8Array])[]) => SubmittableExtrinsic<ApiType>>;
+      setSubs: AugmentedSubmittable<(subs: Vec<ITuple<[AccountId, Data]>> | ([AccountId | string | Uint8Array, Data | { None: any } | { Raw: any } | { BlakeTwo256: any } | { Sha256: any } | { Keccak256: any } | { ShaThree256: any } | string | Uint8Array])[]) => SubmittableExtrinsic<ApiType>>;
     };
     imOnline: {
       [index: string]: SubmittableExtrinsicFunction<ApiType>;
@@ -1220,8 +1314,8 @@ declare module '@polkadot/api/types/submittable' {
        * - Independent of the arguments. Moderate complexity.
        * - O(1).
        * - Three extra DB entries.
-       * NOTE: Two of the storage writes (`Self::bonded`, `Self::payee`) are _never_ cleaned unless
-       * the `origin` falls below _existential deposit_ and gets removed as dust.
+       * NOTE: Two of the storage writes (`Self::bonded`, `Self::payee`) are _never_ cleaned
+       * unless the `origin` falls below _existential deposit_ and gets removed as dust.
        * # </weight>
        **/
       bond: AugmentedSubmittable<(controller: LookupSource | Address | AccountId | AccountIndex | string | Uint8Array, value: Compact<BalanceOf> | AnyNumber | Uint8Array, payee: RewardDestination | ('Staked' | 'Stash' | 'Controller') | number | Uint8Array) => SubmittableExtrinsic<ApiType>>;
@@ -1292,7 +1386,7 @@ declare module '@polkadot/api/types/submittable' {
        * The dispatch origin for this call must be _Signed_ by the controller, not the stash.
        * # <weight>
        * - The transaction's complexity is proportional to the size of `targets`,
-       * which is capped at `MAX_NOMINATIONS`.
+       * which is capped at CompactAssignments::LIMIT.
        * - Both the reads and writes follow a similar pattern.
        * # </weight>
        **/
@@ -1387,6 +1481,69 @@ declare module '@polkadot/api/types/submittable' {
        * The ideal number of validators.
        **/
       setValidatorCount: AugmentedSubmittable<(updated: Compact<u32> | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
+      /**
+       * Submit a phragmen result to the chain. If the solution:
+       * 1. is valid.
+       * 2. has a better score than a potentially existing solution on chain.
+       * then, it will be _put_ on chain.
+       * A solution consists of two pieces of data:
+       * 1. `winners`: a flat vector of all the winners of the round.
+       * 2. `assignments`: the compact version of an assignment vector that encodes the edge
+       * weights.
+       * Both of which may be computed using [`phragmen`], or any other algorithm.
+       * Additionally, the submitter must provide:
+       * - The `score` that they claim their solution has.
+       * Both validators and nominators will be represented by indices in the solution. The
+       * indices should respect the corresponding types ([`ValidatorIndex`] and
+       * [`NominatorIndex`]). Moreover, they should be valid when used to index into
+       * [`SnapshotValidators`] and [`SnapshotNominators`]. Any invalid index will cause the
+       * solution to be rejected. These two storage items are set during the election window and
+       * may be used to determine the indices.
+       * A solution is valid if:
+       * 0. It is submitted when [`EraElectionStatus`] is `Open`.
+       * 1. Its claimed score is equal to the score computed on-chain.
+       * 2. Presents the correct number of winners.
+       * 3. All indexes must be value according to the snapshot vectors. All edge values must
+       * also be correct and should not overflow the granularity of the ratio type (i.e. 256
+       * or billion).
+       * 4. For each edge, all targets are actually nominated by the voter.
+       * 5. Has correct self-votes.
+       * A solutions score is consisted of 3 parameters:
+       * 1. `min { support.total }` for each support of a winner. This value should be maximized.
+       * 2. `sum { support.total }` for each support of a winner. This value should be minimized.
+       * 3. `sum { support.total^2 }` for each support of a winner. This value should be
+       * minimized (to ensure less variance)
+       * # <weight>
+       * E: number of edges. m: size of winner committee. n: number of nominators. d: edge degree
+       * (16 for now) v: number of on-chain validator candidates.
+       * NOTE: given a solution which is reduced, we can enable a new check the ensure `|E| < n +
+       * m`. We don't do this _yet_, but our offchain worker code executes it nonetheless.
+       * major steps (all done in `check_and_replace_solution`):
+       * - Storage: O(1) read `ElectionStatus`.
+       * - Storage: O(1) read `PhragmenScore`.
+       * - Storage: O(1) read `ValidatorCount`.
+       * - Storage: O(1) length read from `SnapshotValidators`.
+       * - Storage: O(v) reads of `AccountId` to fetch `snapshot_validators`.
+       * - Memory: O(m) iterations to map winner index to validator id.
+       * - Storage: O(n) reads `AccountId` to fetch `snapshot_nominators`.
+       * - Memory: O(n + m) reads to map index to `AccountId` for un-compact.
+       * - Storage: O(e) accountid reads from `Nomination` to read correct nominations.
+       * - Storage: O(e) calls into `slashable_balance_of_extended` to convert ratio to staked.
+       * - Memory: build_support_map. O(e).
+       * - Memory: evaluate_support: O(E).
+       * - Storage: O(e) writes to `QueuedElected`.
+       * - Storage: O(1) write to `QueuedScore`
+       * The weight of this call is 1/10th of the blocks total weight.
+       * # </weight>
+       **/
+      submitElectionSolution: AugmentedSubmittable<(winners: Vec<ValidatorIndex> | (ValidatorIndex | AnyNumber | Uint8Array)[], compactAssignments: CompactAssignments | { votes1?: any; votes2?: any; votes3?: any; votes4?: any; votes5?: any; votes6?: any; votes7?: any; votes8?: any; votes9?: any; votes10?: any; votes11?: any; votes12?: any; votes13?: any; votes14?: any; votes15?: any; votes16?: any } | string | Uint8Array, score: PhragmenScore, era: EraIndex | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
+      /**
+       * Unsigned version of `submit_election_solution`.
+       * Note that this must pass the [`ValidateUnsigned`] check which only allows transactions
+       * from the local node to be included. In other words, only the block author can include a
+       * transaction in the block.
+       **/
+      submitElectionSolutionUnsigned: AugmentedSubmittable<(winners: Vec<ValidatorIndex> | (ValidatorIndex | AnyNumber | Uint8Array)[], compactAssignments: CompactAssignments | { votes1?: any; votes2?: any; votes3?: any; votes4?: any; votes5?: any; votes6?: any; votes7?: any; votes8?: any; votes9?: any; votes10?: any; votes11?: any; votes12?: any; votes13?: any; votes14?: any; votes15?: any; votes16?: any } | string | Uint8Array, score: PhragmenScore, era: EraIndex | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
       /**
        * Schedule a portion of the stash to be unlocked ready for transfer out after the bond
        * period ends. If this leaves an amount actively bonded less than
@@ -1488,7 +1645,6 @@ declare module '@polkadot/api/types/submittable' {
        * Kill some items from storage.
        **/
       killStorage: AugmentedSubmittable<(keys: Vec<Key> | (Key | string | Uint8Array)[]) => SubmittableExtrinsic<ApiType>>;
-      migrateAccounts: AugmentedSubmittable<(accounts: Vec<AccountId> | (AccountId | string | Uint8Array)[]) => SubmittableExtrinsic<ApiType>>;
       /**
        * Make some on-chain remark.
        **/

packages/api/src/base/Decorate.ts

@@ -41,7 +41,7 @@ interface MetaDecoration {
 type LinkageData = ITuple<[Codec, Linkage<Codec>]>;
 
 const PAGE_SIZE_KEYS = 256;
-const PAGE_SIZE_VALS = PAGE_SIZE_KEYS / 2;
+const PAGE_SIZE_VALS = PAGE_SIZE_KEYS;
 
 const l = logger('api/init');
 
@@ -248,6 +248,7 @@ export default abstract class Decorate<ApiType extends ApiTypes> extends Events
       // eslint-disable-next-line @typescript-eslint/no-unused-vars
       .forEach(([_, { method, section }]): void => {
         delete (this._rpc as any)[section][method];
+        delete (this._rpcCore as any)[section][method];
         delete (this._rx.rpc as any)[section][method];
       });
   }
@@ -502,12 +503,13 @@ export default abstract class Decorate<ApiType extends ApiTypes> extends Events
         combineLatest([
           of(keys),
           from(Array(Math.ceil(keys.length / PAGE_SIZE_VALS)).fill(0)).pipe(
-            // FIXME New RPC to take care of this in the works...
-            concatMap((_, index): Observable<Option<Raw>[]> =>
-              this._rpcCore.state.subscribeStorage<Option<Raw>[]>(
-                keys.slice(index * PAGE_SIZE_VALS, (index * PAGE_SIZE_VALS) + PAGE_SIZE_VALS)
-              ).pipe(take(1))
-            ),
+            concatMap((_, index): Observable<Option<Raw>[]> => {
+              const keyset = keys.slice(index * PAGE_SIZE_VALS, (index * PAGE_SIZE_VALS) + PAGE_SIZE_VALS);
+
+              return this._rpcCore.state.queryStorageAt
+                ? this._rpcCore.state.queryStorageAt<Option<Raw>[]>(keyset)
+                : this._rpcCore.state.subscribeStorage<Option<Raw>[]>(keyset).pipe(take(1));
+            }),
             toArray(),
             map((valsArr: Option<Raw>[][]): Option<Raw>[] =>
               valsArr.reduce((result: Option<Raw>[], vals): Option<Raw>[] =>

packages/api/src/base/Init.ts

@@ -9,7 +9,7 @@ import { Subscription, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { Metadata, Text } from '@polkadot/types';
 import { LATEST_EXTRINSIC_VERSION } from '@polkadot/types/extrinsic/Extrinsic';
-import { getMetadataTypes, getSpecTypes, getUserTypes } from '@polkadot/types/known';
+import { getMetadataTypes, getSpecTypes } from '@polkadot/types-known';
 import { logger } from '@polkadot/util';
 import { cryptoWaitReady } from '@polkadot/util-crypto';
 
@@ -140,7 +140,7 @@ export default abstract class Init<ApiType extends ApiTypes> extends Decorate<Ap
         this._runtimeMetadata = metadata;
         this._runtimeVersion = version;
 
-        this.registerTypes(getSpecTypes(this.registry, this._runtimeChain as Text, version));
+        this.registerTypes(getSpecTypes(this.registry, this._runtimeChain as Text, version.specName, version.specVersion));
         this.injectMetadata(metadata, false);
 
         return true;
@@ -161,8 +161,7 @@ export default abstract class Init<ApiType extends ApiTypes> extends Decorate<Ap
 
     // do the setup for the specific chain
     this.registry.setChainProperties(chainProps);
-    this.registerTypes(getSpecTypes(this.registry, chain, runtimeVersion));
-    this.registerTypes(getUserTypes(this.registry, chain, runtimeVersion));
+    this.registerTypes(getSpecTypes(this.registry, chain, runtimeVersion.specName, runtimeVersion.specVersion));
     this.subscribeUpdates();
 
     // filter the RPC methods (this does an rpc-methods call)

packages/api/src/checkTypes.manual.ts

@@ -159,6 +159,9 @@ async function tx (api: ApiPromise, keyring: TestKeyringMap): Promise<void> {
   second.signAndSend('123', (result): void => {
     console.log(result);
   });
+
+  // it handles enum inputs correctly
+  await api.tx.democracy.proxyVote(123, { Split: { yay: 123, nay: 456 } }).signAndSend(keyring.alice);
 }
 
 async function main (): Promise<void> {

packages/api/src/promise/Api.spec.ts

@@ -13,7 +13,7 @@ import { hexToU8a } from '@polkadot/util';
 import { SingleAccountSigner } from '../../test/util';
 import ApiPromise from './Api';
 
-const TRANSFER_SIG = '0x62bfcd07a9f50ba32ed9e2c41ca11267ac62a24c95949e25efdccc4989d48573f84754dbbaa1d0937ceb888ad8a2cf71daced1545a839d9d6cceac33a5771401';
+const TRANSFER_SIG = '0xa1d493ab6b74550da434fbc153a739429be68c9d30e59e631fb5d08f348504deefb5577f4824e138741ad102f479109827e8fa9552812a3e5f505a24d10e820f';
 
 describe('ApiPromise', (): void => {
   const registry = new TypeRegistry();

packages/api/src/submittable/createClass.ts

@@ -8,9 +8,9 @@ import { Callback, Codec, Constructor, IKeyringPair, Registry, SignatureOptions,
 import { ApiInterfaceRx, ApiTypes, SignerResult } from '../types';
 import { AddressOrPair, SignerOptions, SubmittableExtrinsic, SubmittablePaymentResult, SubmittableResultResult, SubmittableResultSubscription, SubmittableThis } from './types';
 
-import { Observable, combineLatest, of } from 'rxjs';
+import { Observable, of } from 'rxjs';
 import { first, map, mapTo, mergeMap, switchMap, tap } from 'rxjs/operators';
-import { assert, isBn, isFunction, isNumber, isUndefined } from '@polkadot/util';
+import { assert, isBn, isFunction, isNumber } from '@polkadot/util';
 
 import { filterEvents, isKeyringPair } from '../util';
 import ApiBase from '../base';
@@ -22,13 +22,6 @@ interface SubmittableOptions<ApiType extends ApiTypes> {
   decorateMethod: ApiBase<ApiType>['decorateMethod'];
 }
 
-// The default for 6s allowing for 5min eras. When translating this to faster blocks -
-//   - 4s = (10 / 15) * 5 = 3.33m
-//   - 2s = (10 / 30) * 5 = 1.66m
-const BLOCKTIME = 6;
-const ONE_MINUTE = 60 / BLOCKTIME;
-const DEFAULT_MORTAL_LENGTH = 5 * ONE_MINUTE;
-
 export default function createClass <ApiType extends ApiTypes> ({ api, apiType, decorateMethod }: SubmittableOptions<ApiType>): Constructor<SubmittableExtrinsic<ApiType>> {
   // an instance of the base extrinsic for us to extend
   const ExtrinsicBase = api.registry.createClass('Extrinsic');
@@ -49,11 +42,11 @@ export default function createClass <ApiType extends ApiTypes> ({ api, apiType,
 
       return decorateMethod(
         (): Observable<RuntimeDispatchInfo> =>
-          this.#getPrelimState(address, allOptions).pipe(
+          api.derive.tx.signingInfo(address, allOptions.nonce, allOptions.era).pipe(
             first(),
-            switchMap(([nonce, header]): Observable<RuntimeDispatchInfo> => {
+            switchMap((signingInfo): Observable<RuntimeDispatchInfo> => {
               // setup our options (same way as in signAndSend)
-              const eraOptions = this.#makeEraOptions(allOptions, { header, nonce });
+              const eraOptions = this.#makeEraOptions(allOptions, signingInfo);
               const signOptions = this.#makeSignOptions(eraOptions, {});
 
               // add a fake signature to the extrinsic
@@ -125,23 +118,7 @@ export default function createClass <ApiType extends ApiTypes> ({ api, apiType,
       )(statusCb);
     }
 
-    #getPrelimState = (address: string, options: Partial<SignerOptions>): Observable<[Index, Header | null]> => {
-      return combineLatest([
-        // if we have a nonce already, don't retrieve the latest, use what is there
-        isUndefined(options.nonce)
-          ? api.derive.balances.account(address).pipe(
-            map(({ accountNonce }): Index => accountNonce)
-          )
-          : of(this.registry.createType('Index', options.nonce)),
-        // if we have an era provided already or eraLength is <= 0 (immortal)
-        // don't get the latest block, just pass null, handle in mergeMap
-        (isUndefined(options.era) || (isNumber(options.era) && options.era > 0))
-          ? api.rpc.chain.getHeader()
-          : of(null)
-      ]);
-    }
-
-    #makeEraOptions = (options: Partial<SignerOptions>, { header, nonce }: { header: Header | null; nonce: Index }): SignatureOptions => {
+    #makeEraOptions = (options: Partial<SignerOptions>, { header, mortalLength, nonce }: { header: Header | null; mortalLength: number; nonce: Index }): SignatureOptions => {
       if (!header) {
         if (isNumber(options.era)) {
           // since we have no header, it is immortal, remove any option overrides
@@ -157,7 +134,7 @@ export default function createClass <ApiType extends ApiTypes> ({ api, apiType,
         blockHash: header.hash,
         era: this.registry.createType('ExtrinsicEra', {
           current: header.number,
-          period: options.era || DEFAULT_MORTAL_LENGTH
+          period: options.era || mortalLength
         }),
         nonce
       });
@@ -191,15 +168,15 @@ export default function createClass <ApiType extends ApiTypes> ({ api, apiType,
       const options = this.#optionsOrNonce(optionsOrNonce);
       let updateId: number | undefined;
 
-      return this.#getPrelimState(address, options).pipe(
+      return api.derive.tx.signingInfo(address, options.nonce, options.era).pipe(
         first(),
-        mergeMap(async ([nonce, header]): Promise<void> => {
-          const eraOptions = this.#makeEraOptions(options, { header, nonce });
+        mergeMap(async (signingInfo): Promise<void> => {
+          const eraOptions = this.#makeEraOptions(options, signingInfo);
 
           if (isKeyringPair(account)) {
             this.sign(account, eraOptions);
           } else {
-            updateId = await this.#signViaSigner(address, eraOptions, header);
+            updateId = await this.#signViaSigner(address, eraOptions, signingInfo.header);
           }
         }),
         mapTo(updateId)
@@ -215,13 +192,10 @@ export default function createClass <ApiType extends ApiTypes> ({ api, apiType,
         ? status.asInBlock
         : status.asFinalized;
 
-      return combineLatest([
-        api.rpc.chain.getBlock(blockHash),
-        api.query.system.events.at(blockHash)
-      ]).pipe(
-        map(([signedBlock, allEvents]): ISubmittableResult =>
+      return api.derive.tx.events(blockHash).pipe(
+        map(({ block, events }): ISubmittableResult =>
           new SubmittableResult({
-            events: filterEvents(this.hash, signedBlock, allEvents, status),
+            events: filterEvents(this.hash, block, events, status),
             status
           })
         )
@@ -229,26 +203,22 @@ export default function createClass <ApiType extends ApiTypes> ({ api, apiType,
     }
 
     #observeSend = (updateId = -1): Observable<Hash> => {
-      return api.rpc.author
-        .submitExtrinsic(this)
-        .pipe(
-          tap((hash): void => {
-            this.#updateSigner(updateId, hash);
-          })
-        );
+      return api.rpc.author.submitExtrinsic(this).pipe(
+        tap((hash): void => {
+          this.#updateSigner(updateId, hash);
+        })
+      );
     }
 
     #observeSubscribe = (updateId = -1): Observable<ISubmittableResult> => {
-      return api.rpc.author
-        .submitAndWatchExtrinsic(this)
-        .pipe(
-          switchMap((status): Observable<ISubmittableResult> =>
-            this.#observeStatus(status)
-          ),
-          tap((status): void => {
-            this.#updateSigner(updateId, status);
-          })
-        );
+      return api.rpc.author.submitAndWatchExtrinsic(this).pipe(
+        switchMap((status): Observable<ISubmittableResult> =>
+          this.#observeStatus(status)
+        ),
+        tap((status): void => {
+          this.#updateSigner(updateId, status);
+        })
+      );
     }
 
     // NOTE here we actually override nonce if it was specified (backwards compat for

packages/metadata/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/metadata",
-  "version": "1.8.1",
+  "version": "1.9.1",
   "description": "Helpers to extract information from runtime metadata",
   "main": "index.js",
   "publishConfig": {
@@ -27,12 +27,13 @@
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/type-metadata#readme",
   "dependencies": {
     "@babel/runtime": "^7.9.2",
-    "@polkadot/types": "1.8.1",
-    "@polkadot/util": "^2.6.2",
-    "@polkadot/util-crypto": "^2.6.2",
+    "@polkadot/types": "1.9.1",
+    "@polkadot/types-known": "1.9.1",
+    "@polkadot/util": "^2.7.1",
+    "@polkadot/util-crypto": "^2.7.1",
     "bn.js": "^5.1.1"
   },
   "devDependencies": {
-    "@polkadot/keyring": "^2.6.2"
+    "@polkadot/keyring": "^2.7.1"
   }
 }

packages/metadata/src/Decorated/consts/fromMetadata/fromMetadata.spec.ts

@@ -2,25 +2,34 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
+import { Constants } from '../../types';
+
 import { Metadata, TypeRegistry } from '@polkadot/types';
 
 import rpcMetadata from '../../../Metadata/static';
+import rpcMetadataV10 from '../../../Metadata/v10/static';
 import fromMetadata from '../fromMetadata';
 
-// Use the pre-generated metadata
-const registry = new TypeRegistry();
-const metadata = new Metadata(registry, rpcMetadata);
-const consts = fromMetadata(registry, metadata);
+function init (meta: string): [Constants, TypeRegistry] {
+  const registry = new TypeRegistry();
+  const metadata = new Metadata(registry, meta);
+
+  return [fromMetadata(registry, metadata), registry];
+}
 
 describe('fromMetadata', (): void => {
   it('should return constants with the correct type and value', (): void => {
+    const [consts, registry] = init(rpcMetadata);
+
     expect(consts.democracy.cooloffPeriod).toBeInstanceOf(registry.createClass('BlockNumber'));
     // 3 second blocks, 28 days
     expect(consts.democracy.cooloffPeriod.toNumber()).toEqual(28 * 24 * 60 * (60 / 3));
   });
 
   // removed from session
-  it.skip('correctly handles bytes', (): void => {
+  it('correctly handles bytes', (): void => {
+    const [consts] = init(rpcMetadataV10);
+
     // 0x34 removes as the length prefix removed
     expect(consts.session.dedupKeyPrefix.toHex()).toEqual('0x3a73657373696f6e3a6b657973');
   });

packages/metadata/src/Decorated/consts/fromMetadata/index.ts

@@ -6,12 +6,10 @@ import { Registry } from '@polkadot/types/types';
 import { Constants, ConstantCodec, ModuleConstants } from '../../types';
 
 import { createTypeUnsafe } from '@polkadot/types/create';
-import { stringCamelCase } from '@polkadot/util';
+import { hexToU8a, stringCamelCase } from '@polkadot/util';
 
 import Metadata from '../../../Metadata';
 
-const AS_STRIPPED = ['Bytes'];
-
 /** @internal */
 export default function fromMetadata (registry: Registry, metadata: Metadata): Constants {
   return metadata.asLatest.modules.reduce((result: Constants, moduleMetadata): Constants => {
@@ -23,19 +21,10 @@ export default function fromMetadata (registry: Registry, metadata: Metadata): C
 
     // For access, we change the index names, i.e. Democracy.EnactmentPeriod -> democracy.enactmentPeriod
     result[stringCamelCase(name.toString())] = moduleMetadata.constants.reduce((newModule: ModuleConstants, meta): ModuleConstants => {
-      // in the case of Bytes, the data has a length prefix encoded when received,
-      // leading to double-encoding unless removed
+      // convert to the natural type as received
       const type = meta.type.toString();
-      const codec: ConstantCodec = createTypeUnsafe(registry, type, [
-        AS_STRIPPED.includes(type)
-          ? meta.value.toU8a(true)
-          : meta.value
-      ]);
+      const codec: ConstantCodec = createTypeUnsafe(registry, type, [hexToU8a(meta.value.toHex())]);
 
-      // This is not a perfect idea, however as it stands with number-only constants on the metadata
-      // does not have any effect. However, this could become problematic in cases where items are
-      // exposed that contain their own metadata. As of now, the compatibility with current, e.g.
-      // storage is the driving factor, one consistent way of handling interfaces
       codec.meta = meta;
       newModule[stringCamelCase(meta.name.toString())] = codec;
 

packages/metadata/src/Metadata/util/toCallsOnly.ts

@@ -2,16 +2,39 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
-import { MetadataLatest } from '@polkadot/types/interfaces/metadata';
+import { FunctionMetadataLatest, MetadataLatest } from '@polkadot/types/interfaces/metadata';
 import { AnyJson, Registry } from '@polkadot/types/types';
 
+import { Option, Text, Vec } from '@polkadot/types';
+
+interface ModuleMetadataTrimmed {
+  name: Text;
+  calls: Option<Vec<FunctionMetadataLatest>>;
+}
+
+function mapCalls (registry: Registry, _calls: Option<Vec<FunctionMetadataLatest>>): Option<Vec<FunctionMetadataLatest>> {
+  const calls = _calls.unwrapOr(null);
+
+  return registry.createType(
+    'Option<Vec<FunctionMetadataLatest>>' as any,
+    calls
+      ? calls.map(({ args, documentation, name }) =>
+        registry.createType('FunctionMetadataLatest', {
+          args,
+          name,
+          documentation: documentation.map((doc) => doc.toString().trim())
+        })
+      )
+      : null
+  );
+}
+
 /** @internal */
 export default function toCallsOnly (registry: Registry, { extrinsic, modules }: MetadataLatest): AnyJson {
   return registry.createType('MetadataLatest', {
-    // FIXME, this needs typing, not any
-    modules: modules.map(({ calls, name }): any => ({
+    modules: modules.map(({ calls, name }): ModuleMetadataTrimmed => ({
       name,
-      calls
+      calls: mapCalls(registry, calls)
     })),
     extrinsic
   }).toJSON();

packages/metadata/src/Metadata/util/validateTypes.ts

@@ -17,6 +17,7 @@ function extractTypes (types: string[]): any[] {
       case TypeDefInfo.Plain:
         return decoded.type;
 
+      case TypeDefInfo.BTreeSet:
       case TypeDefInfo.Compact:
       case TypeDefInfo.Option:
       case TypeDefInfo.Vec:
@@ -24,7 +25,7 @@ function extractTypes (types: string[]): any[] {
         return extractTypes([(decoded.sub as TypeDef).type]);
 
       case TypeDefInfo.BTreeMap:
-      case TypeDefInfo.BTreeSet:
+      case TypeDefInfo.HashMap:
       case TypeDefInfo.Result:
       case TypeDefInfo.Tuple:
         return extractTypes((decoded.sub as TypeDef[]).map((sub): string => sub.type));

packages/metadata/src/Metadata/v11/toLatest.ts

@@ -2,11 +2,10 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
-import { OverrideModuleType } from '@polkadot/types/known/types';
 import { FunctionMetadataV11, FunctionMetadataLatest, MetadataV11, MetadataLatest, ModuleMetadataLatest, StorageMetadataV11, StorageMetadataLatest, StorageEntryMetadataLatest } from '@polkadot/types/interfaces/metadata';
-import { Registry } from '@polkadot/types/types';
+import { Registry, OverrideModuleType } from '@polkadot/types/types';
 
-import { getModuleTypes } from '@polkadot/types/known';
+import { getModuleTypes } from '@polkadot/types-known';
 import { Type } from '@polkadot/types/primitive';
 import { stringCamelCase } from '@polkadot/util';
 

packages/rpc-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/rpc-core",
-  "version": "1.8.1",
+  "version": "1.9.1",
   "description": "A JavaScript wrapper for the Polkadot JsonRPC interface",
   "main": "index.js",
   "keywords": [
@@ -27,14 +27,14 @@
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/rpc-core#readme",
   "dependencies": {
     "@babel/runtime": "^7.9.2",
-    "@polkadot/metadata": "1.8.1",
-    "@polkadot/rpc-provider": "1.8.1",
-    "@polkadot/types": "1.8.1",
-    "@polkadot/util": "^2.6.2",
+    "@polkadot/metadata": "1.9.1",
+    "@polkadot/rpc-provider": "1.9.1",
+    "@polkadot/types": "1.9.1",
+    "@polkadot/util": "^2.7.1",
     "memoizee": "^0.4.14",
     "rxjs": "^6.5.4"
   },
   "devDependencies": {
-    "@polkadot/keyring": "^2.6.2"
+    "@polkadot/keyring": "^2.7.1"
   }
 }

packages/rpc-core/src/index.ts

@@ -12,7 +12,7 @@ import { combineLatest, from, Observable, Observer, of, throwError } from 'rxjs'
 import { catchError, map, publishReplay, refCount, switchMap } from 'rxjs/operators';
 import jsonrpc from '@polkadot/types/interfaces/jsonrpc';
 import { Option, StorageKey, Vec, createClass, createTypeUnsafe } from '@polkadot/types';
-import { assert, isFunction, isNull, isNumber, isUndefined, logger, u8aToU8a } from '@polkadot/util';
+import { assert, hexToU8a, isFunction, isNull, isNumber, isUndefined, logger, u8aToU8a } from '@polkadot/util';
 
 import { drr } from './rxjs';
 
@@ -29,12 +29,12 @@ const EMPTY_META = {
 
 // utility method to create a nicely-formatted error
 /** @internal */
-function createErrorMessage (method: string, { params, type }: DefinitionRpc, error: Error): string {
+function logErrorMessage (method: string, { params, type }: DefinitionRpc, error: Error): void {
   const inputs = params.map(({ isOptional, name, type }): string =>
     `${name}${isOptional ? '?' : ''}: ${type}`
   ).join(', ');
 
-  return `${method}(${inputs}): ${type}:: ${error.message}`;
+  l.error(`${method}(${inputs}): ${type}:: ${error.message}`);
 }
 
 /**
@@ -83,6 +83,8 @@ export default class Rpc implements RpcInterface {
 
   public readonly chain!: RpcInterface['chain'];
 
+  public readonly childstate!: RpcInterface['childstate'];
+
   public readonly contracts!: RpcInterface['contracts'];
 
   public readonly engine!: RpcInterface['engine'];
@@ -188,15 +190,12 @@ export default class Rpc implements RpcInterface {
         map(([params, result]): any =>
           isRaw
             ? this.registry.createType('Raw', result)
-            : this.formatOutput(def, params, result)
+            : this.formatOutput(method, def, params, result)
         ),
         catchError((error): any => {
-          const message = createErrorMessage(method, def, error);
+          logErrorMessage(method, def, error);
 
-          // don't scare with old nodes, this is handled transparently
-          rpcName !== 'rpc_methods' && l.error(message);
-
-          return throwError(new Error(message));
+          return throwError(error);
         }),
         publishReplay(1), // create a Replay(1)
         refCount() // Unsubscribe WS when there are no more subscribers
@@ -232,11 +231,9 @@ export default class Rpc implements RpcInterface {
         // Have at least an empty promise, as used in the unsubscribe
         let subscriptionPromise: Promise<number | void> = Promise.resolve();
         const errorHandler = (error: Error): void => {
-          const message = createErrorMessage(method, def, error);
+          logErrorMessage(method, def, error);
 
-          l.error(message);
-
-          observer.error(new Error(message));
+          observer.error(error);
         };
 
         try {
@@ -244,7 +241,7 @@ export default class Rpc implements RpcInterface {
           const paramsJson = params.map((param): AnyJson => param.toJSON());
           const update = (error?: Error | null, result?: any): void => {
             if (error) {
-              l.error(createErrorMessage(method, def, error));
+              logErrorMessage(method, def, error);
               return;
             }
 
@@ -252,7 +249,7 @@ export default class Rpc implements RpcInterface {
               observer.next(
                 isRaw
                   ? this.registry.createType('Raw', result)
-                  : this.formatOutput(def, params, result)
+                  : this.formatOutput(method, def, params, result)
               );
             } catch (error) {
               observer.error(error);
@@ -285,7 +282,7 @@ export default class Rpc implements RpcInterface {
                 ? this.provider.unsubscribe(subType, unsubName, subscriptionId)
                 : Promise.resolve(false)
             )
-            .catch((error: Error): void => l.error(createErrorMessage(method, def, error)));
+            .catch((error: Error): void => logErrorMessage(method, def, error));
         };
       }).pipe(drr());
     };
@@ -322,7 +319,7 @@ export default class Rpc implements RpcInterface {
     return ['0x3a636f6465'].includes(key.toHex());
   }
 
-  private formatOutput (rpc: DefinitionRpc, params: Codec[], result?: any): Codec | Codec[] {
+  private formatOutput (method: string, rpc: DefinitionRpc, params: Codec[], result?: any): Codec | Codec[] {
     if (rpc.type === 'StorageData') {
       const key = params[0] as StorageKey;
 
@@ -336,10 +333,15 @@ export default class Rpc implements RpcInterface {
     } else if (rpc.type === 'StorageChangeSet') {
       return this.formatStorageSet(params[0] as Vec<StorageKey>, result.changes);
     } else if (rpc.type === 'Vec<StorageChangeSet>') {
-      return result.map(({ block, changes }: { block: string; changes: [string, string | null][] }): [Hash, Codec[]] => [
+      const mapped = result.map(({ block, changes }: { block: string; changes: [string, string | null][] }): [Hash, Codec[]] => [
         this.registry.createType('Hash', block),
         this.formatStorageSet(params[0] as Vec<StorageKey>, changes)
       ]);
+
+      // we only query at a specific block, not a range - flatten
+      return method === 'queryStorageAt'
+        ? mapped[0][1]
+        : mapped;
     }
 
     return createTypeUnsafe(this.registry, rpc.type, [result]);
@@ -370,7 +372,13 @@ export default class Rpc implements RpcInterface {
       );
     }
 
-    return createTypeUnsafe(this.registry, type, [isEmpty ? meta.fallback : input], true);
+    return createTypeUnsafe(this.registry, type, [
+      isEmpty
+        ? meta.fallback
+          ? hexToU8a(meta.fallback.toHex())
+          : undefined
+        : input
+    ], true);
   }
 
   private formatStorageSet (keys: Vec<StorageKey>, changes: [string, string | null][]): Codec[] {
@@ -428,6 +436,12 @@ export default class Rpc implements RpcInterface {
       );
     }
 
-    return createTypeUnsafe(this.registry, type, [isEmpty ? meta.fallback : input], true);
+    return createTypeUnsafe(this.registry, type, [
+      isEmpty
+        ? meta.fallback
+          ? hexToU8a(meta.fallback.toHex())
+          : undefined
+        : input
+    ], true);
   }
 }

packages/rpc-core/src/methodSend.spec.ts

@@ -39,19 +39,6 @@ describe('methodSend', (): void => {
     rpc = new Rpc(registry, provider);
   });
 
-  it('wraps errors with the call signature', (done): void => {
-    // private access
-    const method = (rpc as any).createMethodSend('test', 'blah', methods.blah);
-
-    method().subscribe(
-      (): void => undefined,
-      (error: Error): void => {
-        expect(error.message).toMatch(/blah\(foo: Bytes\): Bytes/);
-        done();
-      }
-    );
-  });
-
   it('checks for mismatched parameters', (done): void => {
     // private method
     const method = (rpc as any).createMethodSend('test', 'bleh', methods.bleh);

packages/rpc-provider/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/rpc-provider",
-  "version": "1.8.1",
+  "version": "1.9.1",
   "description": "Transport providers for the API",
   "main": "index.js",
   "keywords": [
@@ -27,17 +27,17 @@
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/rpc-provider#readme",
   "dependencies": {
     "@babel/runtime": "^7.9.2",
-    "@polkadot/metadata": "1.8.1",
-    "@polkadot/types": "1.8.1",
-    "@polkadot/util": "^2.6.2",
-    "@polkadot/util-crypto": "^2.6.2",
+    "@polkadot/metadata": "1.9.1",
+    "@polkadot/types": "1.9.1",
+    "@polkadot/util": "^2.7.1",
+    "@polkadot/util-crypto": "^2.7.1",
     "bn.js": "^5.1.1",
     "eventemitter3": "^4.0.0",
     "isomorphic-fetch": "^2.2.1",
     "websocket": "^1.0.31"
   },
   "devDependencies": {
-    "@polkadot/keyring": "^2.6.2",
+    "@polkadot/keyring": "^2.7.1",
     "mock-socket": "^9.0.3",
     "nock": "^12.0.3"
   }

packages/rpc-provider/src/coder/index.ts

@@ -17,7 +17,7 @@ function formatErrorData (data?: string | number): string {
 
   // We need some sort of cut-off here since these can be very large and
   // very nested, pick a number and trim the result display to it
-  return `: ${formatted.substr(0, 35)}`;
+  return `: ${formatted.substr(0, 100)}`;
 }
 
 /** @internal */

packages/rpc-provider/src/mock/index.ts

@@ -54,7 +54,8 @@ export default class Mock implements ProviderInterface {
     chain_getBlock: (hash: string): any => this.registry.createType('SignedBlock', rpcSignedBlock.result).toJSON(),
     // eslint-disable-next-line @typescript-eslint/no-unused-vars
     chain_getBlockHash: (blockNumber: number): string => '0x1234',
-    chain_getHeader: (): any => this.registry.createType('Header', rpcHeader.result).toJSON(),
+    chain_getFinalizedHead: () => this.registry.createType('Header', rpcHeader.result).hash,
+    chain_getHeader: () => this.registry.createType('Header', rpcHeader.result).toJSON(),
     state_getKeys: (): string[] => [],
     state_getKeysPaged: (): string[] => [],
     state_getRuntimeVersion: (): string => this.registry.createType('RuntimeVersion').toHex(),

packages/typegen/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/typegen",
-  "version": "1.8.1",
+  "version": "1.9.1",
   "description": "Type generation scripts",
   "main": "index.js",
   "bin": {
@@ -36,11 +36,11 @@
     "@babel/core": "^7.9.0",
     "@babel/register": "^7.9.0",
     "@babel/runtime": "^7.9.2",
-    "@polkadot/api": "1.8.1",
-    "@polkadot/metadata": "1.8.1",
-    "@polkadot/rpc-provider": "1.8.1",
-    "@polkadot/types": "1.8.1",
-    "@polkadot/util": "^2.6.2",
+    "@polkadot/api": "1.9.1",
+    "@polkadot/metadata": "1.9.1",
+    "@polkadot/rpc-provider": "1.9.1",
+    "@polkadot/types": "1.9.1",
+    "@polkadot/util": "^2.7.1",
     "websocket": "^1.0.31",
     "yargs": "^15.3.1"
   },

packages/typegen/scripts/polkadot-types-chain-info.js

@@ -19,6 +19,7 @@ try {
           '^@polkadot/api(.*)': './packages/api/src/\\1',
           '^@polkadot/rpc-core(.*)': './packages/rpc-core/src\\1',
           '^@polkadot/rpc-provider(.*)': './packages/rpc-provider/src\\1',
+          '^@polkadot/types-known(.*)': './packages/types-known/src\\1',
           '^@polkadot/types(.*)': './packages/types/src\\1'
         }
       }]

packages/typegen/scripts/polkadot-types-from-chain.js

@@ -15,6 +15,7 @@ try {
       ['module-resolver', {
         alias: {
           '^@polkadot/metadata(.*)': './packages/metadata/src\\1',
+          '^@polkadot/types-known(.*)': './packages/types-known/src\\1',
           '^@polkadot/types(.*)': './packages/types/src\\1'
         }
       }]

packages/typegen/scripts/polkadot-types-from-defs.js

@@ -16,6 +16,7 @@ try {
         alias: {
           '^@polkadot/metadata(.*)': './packages/metadata/src\\1',
           '^@polkadot/typegen(.*)': './packages/typegen/src\\1',
+          '^@polkadot/types-known(.*)': './packages/types-known/src\\1',
           '^@polkadot/types(.*)': './packages/types/src\\1'
         }
       }]

packages/typegen/scripts/polkadot-types-internal-interfaces.js

@@ -15,6 +15,7 @@ try {
       ['module-resolver', {
         alias: {
           '^@polkadot/metadata(.*)': './packages/metadata/src\\1',
+          '^@polkadot/types-known(.*)': './packages/types-known/src\\1',
           '^@polkadot/types(.*)': './packages/types/src\\1'
         }
       }]

packages/typegen/scripts/polkadot-types-internal-metadata.js

@@ -15,6 +15,7 @@ try {
       ['module-resolver', {
         alias: {
           '^@polkadot/metadata(.*)': './packages/metadata/src\\1',
+          '^@polkadot/types-known(.*)': './packages/types-known/src\\1',
           '^@polkadot/types(.*)': './packages/types/src\\1'
         }
       }]

packages/typegen/src/generate/rpc.ts

@@ -34,6 +34,8 @@ export default function generateRpcTypes (dest = 'packages/api/src/augment/rpc.t
             return createDocComments(6, [def.description]) + indent(6)(`getStorage: AugmentedRpc<<T = Codec>(key: ${StorageKeyTye}, block?: Hash | Uint8Array | string) => Observable<T>>;`);
           } else if (methodName === 'queryStorage') {
             return createDocComments(6, [def.description]) + indent(6)(`queryStorage: AugmentedRpc<<T = Codec[]>(keys: Vec<StorageKey> | (${StorageKeyTye})[], fromBlock?: Hash | Uint8Array | string, toBlock?: Hash | Uint8Array | string) => Observable<[Hash, T][]>>;`);
+          } else if (methodName === 'queryStorageAt') {
+            return createDocComments(6, [def.description]) + indent(6)(`queryStorageAt: AugmentedRpc<<T = Codec[]>(keys: Vec<StorageKey> | (${StorageKeyTye})[], at?: Hash | Uint8Array | string) => Observable<T>>;`);
           } else if (methodName === 'subscribeStorage') {
             return createDocComments(6, [def.description]) + indent(6)(`subscribeStorage: AugmentedRpc<<T = Codec[]>(keys: Vec<StorageKey> | (${StorageKeyTye})[]) => Observable<T>>;`);
           }

packages/typegen/src/util/derived.ts

@@ -20,7 +20,7 @@ import GenericAddress from '@polkadot/types/generic/Address';
 import Vote from '@polkadot/types/generic/Vote';
 import Null from '@polkadot/types/primitive/Null';
 import * as primitiveClasses from '@polkadot/types/primitive';
-import { isChildClass, stringLowerFirst } from '@polkadot/util';
+import { isChildClass } from '@polkadot/util';
 
 import { isCompactEncodable } from './class';
 import { formatType } from './formatting';
@@ -53,6 +53,7 @@ export function getDerivedTypes (definitions: object, type: string, primitiveNam
       sub: def
     }
   ];
+
   if (isCompact) {
     types.unshift({
       info: TypeDefInfo.Compact,
@@ -110,7 +111,7 @@ export function getSimilarTypes (definitions: object, registry: Registry, _type:
       possibleTypes.push(arrayToStrType(e.defKeys), 'number');
     } else {
       // TODO We don't really want any here, these should be expanded
-      possibleTypes.push(...e.defKeys.map((key): string => `{ ${stringLowerFirst(key)}: any }`), 'string');
+      possibleTypes.push(...e.defKeys.map((key): string => `{ ${key}: any }`), 'string');
     }
 
     possibleTypes.push('Uint8Array');

packages/types-known/LICENSE

@@ -0,0 +1,201 @@
+                              Apache License
+                        Version 2.0, January 2004
+                    http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+  "License" shall mean the terms and conditions for use, reproduction,
+  and distribution as defined by Sections 1 through 9 of this document.
+
+  "Licensor" shall mean the copyright owner or entity authorized by
+  the copyright owner that is granting the License.
+
+  "Legal Entity" shall mean the union of the acting entity and all
+  other entities that control, are controlled by, or are under common
+  control with that entity. For the purposes of this definition,
+  "control" means (i) the power, direct or indirect, to cause the
+  direction or management of such entity, whether by contract or
+  otherwise, or (ii) ownership of fifty percent (50%) or more of the
+  outstanding shares, or (iii) beneficial ownership of such entity.
+
+  "You" (or "Your") shall mean an individual or Legal Entity
+  exercising permissions granted by this License.
+
+  "Source" form shall mean the preferred form for making modifications,
+  including but not limited to software source code, documentation
+  source, and configuration files.
+
+  "Object" form shall mean any form resulting from mechanical
+  transformation or translation of a Source form, including but
+  not limited to compiled object code, generated documentation,
+  and conversions to other media types.
+
+  "Work" shall mean the work of authorship, whether in Source or
+  Object form, made available under the License, as indicated by a
+  copyright notice that is included in or attached to the work
+  (an example is provided in the Appendix below).
+
+  "Derivative Works" shall mean any work, whether in Source or Object
+  form, that is based on (or derived from) the Work and for which the
+  editorial revisions, annotations, elaborations, or other modifications
+  represent, as a whole, an original work of authorship. For the purposes
+  of this License, Derivative Works shall not include works that remain
+  separable from, or merely link (or bind by name) to the interfaces of,
+  the Work and Derivative Works thereof.
+
+  "Contribution" shall mean any work of authorship, including
+  the original version of the Work and any modifications or additions
+  to that Work or Derivative Works thereof, that is intentionally
+  submitted to Licensor for inclusion in the Work by the copyright owner
+  or by an individual or Legal Entity authorized to submit on behalf of
+  the copyright owner. For the purposes of this definition, "submitted"
+  means any form of electronic, verbal, or written communication sent
+  to the Licensor or its representatives, including but not limited to
+  communication on electronic mailing lists, source code control systems,
+  and issue tracking systems that are managed by, or on behalf of, the
+  Licensor for the purpose of discussing and improving the Work, but
+  excluding communication that is conspicuously marked or otherwise
+  designated in writing by the copyright owner as "Not a Contribution."
+
+  "Contributor" shall mean Licensor and any individual or Legal Entity
+  on behalf of whom a Contribution has been received by Licensor and
+  subsequently incorporated within the Work.
+
+2. Grant of Copyright License. Subject to the terms and conditions of
+  this License, each Contributor hereby grants to You a perpetual,
+  worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+  copyright license to reproduce, prepare Derivative Works of,
+  publicly display, publicly perform, sublicense, and distribute the
+  Work and such Derivative Works in Source or Object form.
+
+3. Grant of Patent License. Subject to the terms and conditions of
+  this License, each Contributor hereby grants to You a perpetual,
+  worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+  (except as stated in this section) patent license to make, have made,
+  use, offer to sell, sell, import, and otherwise transfer the Work,
+  where such license applies only to those patent claims licensable
+  by such Contributor that are necessarily infringed by their
+  Contribution(s) alone or by combination of their Contribution(s)
+  with the Work to which such Contribution(s) was submitted. If You
+  institute patent litigation against any entity (including a
+  cross-claim or counterclaim in a lawsuit) alleging that the Work
+  or a Contribution incorporated within the Work constitutes direct
+  or contributory patent infringement, then any patent licenses
+  granted to You under this License for that Work shall terminate
+  as of the date such litigation is filed.
+
+4. Redistribution. You may reproduce and distribute copies of the
+  Work or Derivative Works thereof in any medium, with or without
+  modifications, and in Source or Object form, provided that You
+  meet the following conditions:
+
+  (a) You must give any other recipients of the Work or
+      Derivative Works a copy of this License; and
+
+  (b) You must cause any modified files to carry prominent notices
+      stating that You changed the files; and
+
+  (c) You must retain, in the Source form of any Derivative Works
+      that You distribute, all copyright, patent, trademark, and
+      attribution notices from the Source form of the Work,
+      excluding those notices that do not pertain to any part of
+      the Derivative Works; and
+
+  (d) If the Work includes a "NOTICE" text file as part of its
+      distribution, then any Derivative Works that You distribute must
+      include a readable copy of the attribution notices contained
+      within such NOTICE file, excluding those notices that do not
+      pertain to any part of the Derivative Works, in at least one
+      of the following places: within a NOTICE text file distributed
+      as part of the Derivative Works; within the Source form or
+      documentation, if provided along with the Derivative Works; or,
+      within a display generated by the Derivative Works, if and
+      wherever such third-party notices normally appear. The contents
+      of the NOTICE file are for informational purposes only and
+      do not modify the License. You may add Your own attribution
+      notices within Derivative Works that You distribute, alongside
+      or as an addendum to the NOTICE text from the Work, provided
+      that such additional attribution notices cannot be construed
+      as modifying the License.
+
+  You may add Your own copyright statement to Your modifications and
+  may provide additional or different license terms and conditions
+  for use, reproduction, or distribution of Your modifications, or
+  for any such Derivative Works as a whole, provided Your use,
+  reproduction, and distribution of the Work otherwise complies with
+  the conditions stated in this License.
+
+5. Submission of Contributions. Unless You explicitly state otherwise,
+  any Contribution intentionally submitted for inclusion in the Work
+  by You to the Licensor shall be under the terms and conditions of
+  this License, without any additional terms or conditions.
+  Notwithstanding the above, nothing herein shall supersede or modify
+  the terms of any separate license agreement you may have executed
+  with Licensor regarding such Contributions.
+
+6. Trademarks. This License does not grant permission to use the trade
+  names, trademarks, service marks, or product names of the Licensor,
+  except as required for reasonable and customary use in describing the
+  origin of the Work and reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty. Unless required by applicable law or
+  agreed to in writing, Licensor provides the Work (and each
+  Contributor provides its Contributions) on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+  implied, including, without limitation, any warranties or conditions
+  of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+  PARTICULAR PURPOSE. You are solely responsible for determining the
+  appropriateness of using or redistributing the Work and assume any
+  risks associated with Your exercise of permissions under this License.
+
+8. Limitation of Liability. In no event and under no legal theory,
+  whether in tort (including negligence), contract, or otherwise,
+  unless required by applicable law (such as deliberate and grossly
+  negligent acts) or agreed to in writing, shall any Contributor be
+  liable to You for damages, including any direct, indirect, special,
+  incidental, or consequential damages of any character arising as a
+  result of this License or out of the use or inability to use the
+  Work (including but not limited to damages for loss of goodwill,
+  work stoppage, computer failure or malfunction, or any and all
+  other commercial damages or losses), even if such Contributor
+  has been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability. While redistributing
+  the Work or Derivative Works thereof, You may choose to offer,
+  and charge a fee for, acceptance of support, warranty, indemnity,
+  or other liability obligations and/or rights consistent with this
+  License. However, in accepting such obligations, You may act only
+  on Your own behalf and on Your sole responsibility, not on behalf
+  of any other Contributor, and only if You agree to indemnify,
+  defend, and hold each Contributor harmless for any liability
+  incurred by, or claims asserted against, such Contributor by reason
+  of your accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work.
+
+  To apply the Apache License to your work, attach the following
+  boilerplate notice, with the fields enclosed by brackets "[]"
+  replaced with your own identifying information. (Don't include
+  the brackets!)  The text should be enclosed in the appropriate
+  comment syntax for the file format. We also recommend that a
+  file or class name and description of purpose be included on the
+  same "printed page" as the copyright notice for easier
+  identification within third-party archives.
+
+Copyright [yyyy] [name of copyright owner]
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

packages/types-known/README.md

@@ -0,0 +1,3 @@
+# @polkadot/types-known
+
+Specific known base type overrides (e.g. `Address`, etc) for chains & specs. This does not contain user-specfic types.

packages/types-known/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "@polkadot/types-known",
+  "version": "1.9.1",
+  "description": "A JavaScript wrapper for the Polkadot JsonRPC interface",
+  "main": "index.js",
+  "keywords": [
+    "Polkadot",
+    "JsonRPC"
+  ],
+  "author": "Jaco Greeff <jacogr@gmail.com>",
+  "maintainers": [
+    "Jaco Greeff <jacogr@gmail.com>"
+  ],
+  "contributors": [],
+  "license": "Apache-2.0",
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/polkadot-js/api.git"
+  },
+  "bugs": {
+    "url": "https://github.com/polkadot-js/api/issues"
+  },
+  "homepage": "https://github.com/polkadot-js/api/tree/master/packages/types-known#readme",
+  "dependencies": {
+    "@babel/runtime": "^7.9.2",
+    "@polkadot/types": "1.9.1",
+    "@polkadot/util": "^2.7.1",
+    "bn.js": "^5.1.1"
+  },
+  "devDependencies": {
+    "@types/bn.js": "^4.11.6"
+  }
+}

packages/types-known/src/chain/index.ts

@@ -0,0 +1,14 @@
+// Copyright 2017-2020 @polkadot/types authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { OverrideVersionedType } from '@polkadot/types/types';
+
+import Westend from './westend';
+
+// Type overrides based on specific nodes
+const typesChain: Record<string, OverrideVersionedType[]> = {
+  Westend
+};
+
+export default typesChain;

packages/types-known/src/chain/westend.ts

@@ -0,0 +1,21 @@
+// Copyright 2017-2020 @polkadot/types authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { OverrideVersionedType } from '@polkadot/types/types';
+
+const versioned: OverrideVersionedType[] = [
+  {
+    minmax: [1000, undefined],
+    types: {
+      Address: 'GenericAddress',
+      Keys: 'SessionKeys5',
+      // previous substrate versions
+      BalanceLock: 'BalanceLockTo212',
+      ReferendumInfo: 'ReferendumInfoTo239',
+      StakingLedger: 'StakingLedgerTo223'
+    }
+  }
+];
+
+export default versioned;

packages/types-known/src/index.spec.ts

@@ -2,7 +2,7 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
-import { TypeRegistry } from '../create/registry';
+import { TypeRegistry } from '@polkadot/types/create/registry';
 import { getModuleTypes } from './';
 
 const registry = new TypeRegistry();

packages/types-known/src/index.ts

@@ -0,0 +1,61 @@
+// Copyright 2017-2020 @polkadot/types authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { Registry, RegistryTypes, OverrideModuleType, OverrideVersionedType } from '@polkadot/types/types';
+
+import BN from 'bn.js';
+import { Text } from '@polkadot/types';
+import { bnToBn, isUndefined } from '@polkadot/util';
+
+import typesChain from './chain';
+import typesMeta from './metadata';
+import typesModules from './modules';
+import typesSpec from './spec';
+
+// flatten a VersionedType[] into a Record<string, string>
+/** @internal */
+function filterVersions (versions: OverrideVersionedType[] = [], version: number): RegistryTypes {
+  return versions
+    .filter(({ minmax: [min, max] }): boolean =>
+      (isUndefined(min) || version >= min) &&
+      (isUndefined(max) || version <= max)
+    )
+    .reduce((result: RegistryTypes, { types }): RegistryTypes => ({
+      ...result,
+      ...types
+    }), {});
+}
+
+/**
+ * @description Based on the metadata version, return the registry types
+ */
+export function getMetadataTypes (_registry: Registry, version: number): RegistryTypes {
+  return filterVersions(typesMeta, version);
+}
+
+/**
+ * @description Get types for specific modules (metadata override)
+ */
+export function getModuleTypes ({ knownTypes }: Registry, section: string): OverrideModuleType {
+  return {
+    ...(typesModules[section] || {}),
+    ...(knownTypes.typesAlias?.[section] || {})
+  };
+}
+
+/**
+ * @description Based on the chain and runtimeVersion, get the applicable types (ready for registration)
+ */
+export function getSpecTypes ({ knownTypes }: Registry, chainName: Text | string, specName: Text | string, specVersion: BN | number): RegistryTypes {
+  const _chainName = chainName.toString();
+  const _specName = specName.toString();
+  const _specVersion = bnToBn(specVersion).toNumber();
+
+  return {
+    ...filterVersions(typesSpec[_specName], _specVersion),
+    ...filterVersions(typesChain[_chainName], _specVersion),
+    ...(knownTypes.typesSpec?.[_specName] || {}),
+    ...(knownTypes.typesChain?.[_chainName] || {})
+  };
+}