[CI Skip] 1.12.1

skip-checks: true

.123trigger

@@ -1,3 +1,4 @@
-21
- 1.4.2
+24
+1.4.2
 1.6.2
+1.11.2

.github/workflows/pr-any.yml

@@ -10,9 +10,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v1
-    - uses: actions/setup-node@v1
-      with:
-        node-version: '12.x'
     - name: ${{ matrix.step }}
       run: |
         yarn install --immutable | grep -v 'YN0013'

.github/workflows/push-master.yml

@@ -16,9 +16,6 @@ jobs:
     - uses: actions/checkout@v1
       with:
         token: ${{ secrets.GH_PAT }}
-    - uses: actions/setup-node@v1
-      with:
-        node-version: '12.x'
     - name: ${{ matrix.step }}
       env:
         CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}

CHANGELOG.md

@@ -1,7 +1,54 @@
+# 1.12.1 Apr 29, 2020
+
+- Fix `CompactAssignments` types (Thanks to https://github.com/arjanz)
+- Provider ayo-connect now allows for interval config (Thanks to https://github.com/jnaviask)
+- Cleanup generation for c-like enums (Thanks to https://github.com/monitz87)
+- Allow subscription to all storage keys, alignment with Substrate (Thanks to https://github.com/xlc)
+- Cater for vote delegations in democracy derives
+- Cleanup account derive flags where to allow for optional council modules
+- Allow generic types in ApiPromise.combineLatest
+- Update to latest substrate metadata
+
+# 1.11.2 Apr 22, 2020
+
+- Small patch to add missing Polkadot master types (will be in next Kusama update)
+
+# 1.11.1 Apr 21, 2020
+
+- **Important** Substrate master has an updated `Weight` (to `u64`), on older chains apply `Weight: 'u32'` (This does affect event decoding if wrongly set)
+- Allow user type augmentation to work with different versions of rxjs (Thanks to https://github.com/monitz87)
+- Align augmented generation with camelCase (Thanks to https://github.com/monitz87)
+- Fix bitfield endian encoding (Thanks to investigation by https://github.com/xlc)
+- Add account flag derives via `accounts.flags(accountId)` (Thanks to https://github.com/kwingram25)
+- Fix entries decoding for `Option<Number>` (& simplify, decoding correctly on rpc layer)
+- Add `system.{localPeerId, localListenAddresses}` & `state.getReadProof` RPCs
+- Ensure proper propagation of `api.runtimeVersion` (used for signing) after upgrade
+- Field ordering fix for deprecated `Linkage` (used in linked maps)
+- `derive.account.info` now returns all field enum types decoded (not only raw)
+- Add banner with FAQ entries for node-template to aid with common issues
+- `Digest` has been moved to definitions, removal of `GenericDigest` type
+- Support `Fixed128` type
+- Add `transactionVersion` to `RuntimeVersion` struct
+- Enhance API support for non-subscription providers (skip subscription to runtimeVersion)
+- Adjust types for Kusama 1057 (notably the new `StakingLedger`)
+
+# 1.10.1 Apr 13, 2020
+
+- **Important** Substrate master has updated staking, on older chains apply `StakingLedger: 'StakingLedgerTo240'`
+- Add `rpc.system.chainType` to retrieve the chain type (Development. Local, Live & Custom)
+- Adjust types for Westend after the reset
+- Adjust democracy derives, including the support of the frame-scheduler for scheduled deployments
+- Allow staking derives to work with new staking payouts, specifically `tx.staking.payoutStakers`
+- Cleanups to derives for democracy & staking to be more performant
+- Add Centrifuge base types to `@polkadot/types-known`
+- Adjust with all the latest substrate master types
+- Internally all private/protected methods/variables are now `_` prefixed (Aligning with the `private/protected` indicators in the TS definitions)
+- Update metadata/docs-from-metadata to the latest Substrate
+
 # 1.9.1 Apr 1, 2020
 
-**Breaking change** (TypeScript with derives only) The return type names for `api.derive.*` have been adjusted for consistency, all starting with `Derive*` (previously a mixture of `Derived*` and `Derive*`)
-**Breaking change** (derives only) Additional cleanups of democracy proposal & referenda derives, including support for new Polkadot/Substrate vote retrievals
+- **Breaking change** (TypeScript with derives only) The return type names for `api.derive.*` have been adjusted for consistency, all starting with `Derive*` (previously a mixture of `Derived*` and `Derive*`)
+- **Breaking change** (derives only) Additional cleanups of democracy proposal & referenda derives, including support for new Polkadot/Substrate vote retrievals
 - Tx signing via `signAndSend` & `signAsync` now tries to use the last finalized block (depending on finalized drift)
 - Add support for `api.rpc.childstate.*` Substrate RPCs, older nodes with still populate `api.rpc.state.*` with the child methods
 - Adjust `Error.message` on RPC failures to not include the (already known) method signature

babel.config.js

@@ -1 +1,5 @@
+// Copyright 2017-2020 @polkadot/api authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
 module.exports = require('@polkadot/dev/config/babel');

copy-to-apps.sh

@@ -19,4 +19,5 @@ copy_folder "metadata"
 copy_folder "rpc-core"
 copy_folder "rpc-provider"
 copy_folder "types"
+copy_folder "types-known"
 copy_folder "typegen"

docs/.vuepress/config.js

@@ -50,6 +50,17 @@ module.exports = {
           ['start/FAQ.md', 'FAQ']
         ]
       },
+      {
+        title: 'Cookbook',
+        path: '/cookbook/',
+        collapsable: false,
+        sidebarDepth: 0,
+        children: [
+          ['cookbook/blocks.md', 'Blocks'],
+          ['cookbook/storage.md', 'Storage'],
+          ['cookbook/tx.md', 'Transactions']
+        ]
+      },
       {
         title: 'Examples (Promise API)',
         path: '/examples/promise/',

docs/README.md

@@ -13,7 +13,7 @@ features:
 - title: Types
   details: Various type interfaces to make using returned values from API calls transparent in your code.
 
-footer: Apache-2 Licensed | Copyright © 2017-2019 polkadot-js authors and contributors
+footer: Apache-2 Licensed | Copyright © 2017-2020 polkadot-js authors and contributors
 ---
 
 ## Welcome
@@ -21,16 +21,9 @@ footer: Apache-2 Licensed | Copyright © 2017-2019 polkadot-js authors and contr
 The API provides application developers the ability to query a node and interact with the Polkadot or Substrate chains using Javascript. Here you will find documentation and examples to get you started.
 
 ::: tip Getting started & Examples
-[Jump right in](/start/) and get an overview on using the API in your projects, from installation all the way through to making it do magic.  Already understand how things work and just want the examples? [The ApiPromise examples](/examples/promise/) provide some basic recipies.
+[Jump right in](/start/) and get an overview on using the API in your projects, from installation all the way through to making it do magic. Have things working and want tips? The [cookbook](/cookbook/) provides some tips and tricks. Getting started and want some full examples? [The ApiPromise examples](/examples/promise/) provide some basic examples.
 :::
 
-## Available packages
-
-This repository split up into a number of internal packages, namely -
-
-- [api](api/README.md) Promise and RxJS-based APIs
-- [types](types/README.md) Type encoding and decoding wrappers
-
 ## The Polkadot Project
 
 You can read more about the Polkadot Network at [https://polkadot.network/](https://polkadot.network/) and more about the polkadot-js projects at [https://polkadot.js.org](https://polkadot.js.org)

docs/cookbook/README.md

@@ -0,0 +1,19 @@
+# Cookbook
+
+The cookbook hosts small code snippets in a question-answer format. It does not walk you through the whole setup, like the [basic examples](../examples/), but rather aims to answer some questions around specific uses. It also does not replace the [getting started FAQ](../start/FAQ.md), rather this is focussed explicitly on code samples, instead of general trouble-shooting for common mishaps.
+
+These questions are added as they do come up and the hope is that the snippets here is will be useful to others, instead of just the original person who asked on a specific use-case. As of today, it is only making a start and not fully comprehensive yet - check back early, check back often.
+
+## Sections
+
+The following cookbook sections are available -
+
+- [Blocks](blocks.md) - Snippets around working with blocks and headers
+- [Storage](storage.md) - Snippets for working with storage
+- [Transactions](tx.md) - Snippets for working with transactions
+
+... more to come.
+
+## Contribute
+
+And as always, if you have an addition to make, or a question to ask any [any PRs](https://github.com/polkadot-js/api/tree/master/docs) or [issues](https://github.com/polkadot-js/api/issues) are appreciated. The initial content was generate from exactly such questions, either in Riot chat or as submitted as issues.

docs/cookbook/blocks.md

@@ -0,0 +1,44 @@
+# Blocks
+
+The building blocks for each blockchain. As such there are a number of examples for working with blocks and headers, that could be useful.
+
+## How do I retrieve the header/extrinsic hash from blocks?
+
+A block hash refers to the hash over the header, the extrinsic hash refers to the hash of the encoded extrinsic. Since all objects returned by the API implements the `.hash => Hash` getter, we can simply use this to view the actual hash.
+
+```js
+// returns Hash
+const blockHash = await this.api.rpc.chain.getBlockHash(blockNumber);
+// returns SignedBlock
+const signedBlock = await this.api.rpc.chain.getBlock(blockHash);
+
+// the hash for the block, always via header (Hash -> toHex()) - will be
+// the same as blockHash above (also available on any header retrieved,
+// subscription or once-off)
+console.log(signedBlock.header.hash.toHex());
+
+// the hash for each extrinsic in the block
+signedBlock.block.extrinsics.forEach((ex, index) => {
+  console.log(index, ex.hash.toHex());
+});
+```
+
+## How do I extract the block author?
+
+Th block author is encoded inside the consensus logs for the block. To extract, you need to decode the log (which the API does do) and then map the index of the validator to the list of session validators. This extraction is however available on the api derive for new head subscriptions, which returns an extended header with the author populated (assuming that the digest logs are known).
+
+```js
+// subscribe to all new headers (with extended info)
+api.derive.chain.subscribeNewHeads((header) => {
+  console.log(`#${header.number}: ${header.author}`);
+});
+```
+
+For a single header only, the derives also contain a `getHeader`, which once again returns a header extended with the author -
+
+```js
+// retrieve the last header (hash optional)
+const header = await api.derive.chain.getHeader();
+
+console.log(`#${header.number}: ${header.author}`);
+```

docs/cookbook/storage.md

@@ -0,0 +1,55 @@
+# Storage
+
+Here you will find snippets for working with storage.
+
+## How do I check for storage existence?
+
+In the metadata, for each storage item a fallback is provided. This means that when an entry does not exist, the fallback (which is the default value for the type) will be provided. This means, that querying for a non-existent key (unless an option), will yield a value -
+
+```js
+// retrieve Option<StakingLedger>
+const ledger = await api.query.staking.ledger('EoukLS2Rzh6dZvMQSkqFy4zGvqeo14ron28Ue3yopVc8e3Q');
+// retrieve ValidatorPrefs (will yield the default value)
+const prefs = await api.query.staking.validators('EoukLS2Rzh6dZvMQSkqFy4zGvqeo14ron28Ue3yopVc8e3Q');
+
+console.log(ledger.isNone, ledger.isSome); // true, false
+console.log(JSON.stringify(prefs.toHuman())); // {"commission":"0"}
+```
+
+In the second case, the non-existent prefs returns the default/fallback value for the storage item. So in this case we don't know if the value is set to 0 or unset. Existence can be checked by using the storage size, which would be zero if nothing is stored.
+
+```js
+// exists
+const sizeY = await api.query.staking.validators.size('DB2mp5nNhbFN86J9hxoAog8JALMhDXgwvWMxrRMLNUFMEY4');
+// non existent
+const sizeN = await api.query.staking.validators.size('EoukLS2Rzh6dZvMQSkqFy4zGvqeo14ron28Ue3yopVc8e3Q');
+
+console.log(sizeY.isZero(), sizeY.toNumber()); // false 4
+console.log(sizeN.isZero(), sizeY.toNumber()); // true 0
+```
+
+## How do I use .entries()/.keys() on double maps?
+
+As [explained elsewhere](../start/api.query.other.md#map-keys-entries) each map-type storage entry exposes the entries/keys helpers to retrieve the whole list. In the case of double maps, with the addition of a single argument, you can retrieve either all entries or a subset based on the first map key.
+
+In both these cases, entries/keys operate the same way, `.entries()` retrieving `(StorageKey, Codec)[]` and `.keys()` retrieving `StorageKey[]`
+
+```js
+// Retrieves the entries for all slashes, in all eras (no arg)
+const allEntries = await api.query.staking.nominatorSlashInEra.entries();
+
+// nominatorSlashInEra(EraIndex, AccountId) for the types of the key args
+allEntries.forEach(([{ args: [era, nominatorId] }, value]) => {
+  console.log(`${era}: ${nominatorId} slashed ${value.toHuman()}`);
+});
+```
+
+While we can retrieve only the keys for a specific era, using a argument for the first part of the doublemap (as defined here, an `EraIndex`) -
+
+```js
+// Retrieves the keys for the slashed validators in era 652
+const slashedKeys = await api.query.staking.nominatorSlashInEra.keys(652);
+
+// key args still contains [EraIndex, AccountId] decoded
+console.log(`slashed: ${slashedKeys.map(({ args: [era, nominatorId] }) => nominatorId)`);
+```

docs/cookbook/tx.md

@@ -0,0 +1,112 @@
+# Transactions
+
+A blockchain is no fun if you are not submitting transactions. Or at least if somebody is not submitting any. Here you will find some snippets for dealing with some common issues.
+
+## How do I estimate the transaction fees?
+
+In addition to the `signAndSend` helper on transactions, `.paymentInfo` (with the exact same parameters) are also exposed. Using the same sender, it applies a dummy signature to the transaction and then gets the fee estimation via RPC.
+
+```js
+// estimate the fees as RuntimeDispatchInfo, using the signer (either
+// address or locked/unlocked keypair) (When overrides are applied, e.g
+//  nonce, the format would be `paymentInfo(sender, { nonce })`)
+const info = await api.tx.balances
+  .transfer(recipient, 123)
+  .paymentInfo(sender);
+
+// log relevant info, partialFee is Balance, estimated for current
+console.log(`
+  class=${class.toString()},
+  weight=${weight.toString()},
+  partialFee=${partialFee.toHuman()}
+`);
+```
+
+## How do I get the decoded enum for an ExtrinsicFailed event?
+
+Assuming you are sending a tx via `.signAndSend`, the callback yields information around the tx pool status as well as any events when `isInBlock` or `isFinalized`. If an extrinsic fails via `system.ExtrinsicFailed` event, you can retrieve the error, if defined as an enum on a module.
+
+```js
+api.tx.balances
+  .transfer(recipient, 123)
+  .signAndSend(sender, ({ status, events }) => {
+    if (status.isInBlock || status.isFinalized) {
+      events
+        // find/filter for failed events
+        .filter(({ section, method }) =>
+          section === 'system' &&
+          method === 'ExtrinsicFailed'
+        )
+        // we know that data for system.ExtrinsicFailed is
+        // (DispatchError, DispatchInfo)
+        .forEach(({ data: [error, info] }) => {
+          if (error.isModule) {
+            // for module errors, we have the section indexed, lookup
+            const decoded = api.registry.findMetaError(error.asModule);
+            const { documentation, method, section } = decoded;
+
+            console.log(`${section}.${method}: ${documentation.join(' ')}`);
+          } else {
+            // Other, CannotLookup, BadOrigin, no extra info
+            console.log(error.toString());
+          }
+        });
+    }
+  });
+```
+
+## How do I send an unsigned extrinsic?
+
+For most runtime modules, transactions need to be signed and validation for this happens node-side. There are however modules that accepts unsigned extrinsics, an example would be the Polkadot/Kusama token claims (which is here used as an example).
+
+```js
+// construct the transaction, exactly as per normal
+const utx = api.tx.claims.claim(beneficiary, ethSignature);
+
+// send it without calling sign, pass callback with status/events
+tx.send(({ status }) => {
+  if (status.isInBlock) {
+    console.log(`included in ${status.asInBlock}`);
+  }
+});
+```
+
+The signing is indicated by the first byte in the transaction, so in this case we have called `.send` on it (no `.sign` or `.signAndSend`), so it will be sent using the unsigned state, without signature attached.
+
+## How can I batch transactions?
+
+Polkadot/Substrate provides a `utility.batch` method that can be used to send a number of transactions at once. These are then executed from a single sender (single nonce specified) in sequence. This is very useful in a number of cases, for instance if you wish to create a payout for a validator for multiple eras, you can use this method. Likewise, you can send a number of transfers at once. Or even batch different types of transactions.
+
+```js
+// construct a list of transactions we want to batch
+const txs = [
+  api.tx.balances.transfer(addrBob, 12345),
+  api.tx.balances.transfer(addrEve, 12345),
+  api.tx.staking.unbond(12345)
+];
+
+// construct the batch and send the transactions
+api.tx.utility
+  .batch(txs)
+  .signAndSend(sender, ({ status }) => {
+    if (status.isInBlock) {
+      console.log(`included in ${status.asInBlock}`);
+    }
+  });
+```
+
+## How do I take the pending tx pool into account in my nonce?
+
+The `system.account` query will always contain the current state, i.e. it will reflect the nonce for the last known block. As such when sending multiple transactions in quick succession (see batching above), there may be transactions in the pool that has the same nonce that `signAndSend` would apply - this call doesn't do any magic, it simply reads the state for the nonce. Since we can specify options to the `signandSend` operation, we can override the nonce, either by manually incrementing it o querying it via `nextNonce`.
+
+```js
+for (let i = 0; i < 10; i++) {
+  // retrieve the nextNonce, taking txs in the pool into account
+  const nonce = await api.rpc.account.nextNonce(sender);
+
+  // send, just retrieving the hash, not waiting on status
+  const txhash = await api.tx.balances
+    .transfer(recipient, 123)
+    .signAndSend(sender, { nonce });
+}
+```

docs/examples/promise/01_simple_connect/index.js

@@ -1,6 +1,7 @@
+/* eslint-disable header/header */
 /* eslint-disable @typescript-eslint/require-await */
 /* eslint-disable @typescript-eslint/unbound-method */
-/* eslint-disable @typescript-eslint/no-var-requires */
+
 // Required imports
 const { ApiPromise, WsProvider } = require('@polkadot/api');
 

docs/examples/promise/02_listen_to_blocks/index.js

@@ -1,6 +1,7 @@
+/* eslint-disable header/header */
 /* eslint-disable @typescript-eslint/require-await */
 /* eslint-disable @typescript-eslint/unbound-method */
-/* eslint-disable @typescript-eslint/no-var-requires */
+
 // Import the API
 const { ApiPromise } = require('@polkadot/api');
 

docs/examples/promise/03_listen_to_balance_change/index.js

@@ -1,6 +1,7 @@
+/* eslint-disable header/header */
 /* eslint-disable @typescript-eslint/require-await */
 /* eslint-disable @typescript-eslint/unbound-method */
-/* eslint-disable @typescript-eslint/no-var-requires */
+
 // Import the API
 const { ApiPromise } = require('@polkadot/api');
 
@@ -13,13 +14,13 @@ async function main () {
 
   // Retrieve the initial balance. Since the call has no callback, it is simply a promise
   // that resolves to the current on-chain value
-  let { nonce: previousNonce, data: { free: previousFree } } = await api.query.system.account(Alice);
+  let { data: { free: previousFree }, nonce: previousNonce } = await api.query.system.account(Alice);
 
   console.log(`${Alice} has a balance of ${previousFree}, nonce ${previousNonce}`);
   console.log(`You may leave this example running and start example 06 or transfer any value to ${Alice}`);
 
   // Here we subscribe to any balance changes and update the on-screen value
-  api.query.system.account(Alice, ({ nonce: currentNonce, data: { free: currentFree } }) => {
+  api.query.system.account(Alice, ({ data: { free: currentFree }, nonce: currentNonce }) => {
     // Calculate the delta
     const change = currentFree.sub(previousFree);
 

docs/examples/promise/03_listen_to_multiple_balances_change/index.js

@@ -1,6 +1,7 @@
+/* eslint-disable header/header */
 /* eslint-disable @typescript-eslint/require-await */
 /* eslint-disable @typescript-eslint/unbound-method */
-/* eslint-disable @typescript-eslint/no-var-requires */
+
 // Import the API
 const { ApiPromise } = require('@polkadot/api');
 

docs/examples/promise/04_unsubscribe/index.js

@@ -1,6 +1,7 @@
+/* eslint-disable header/header */
 /* eslint-disable @typescript-eslint/require-await */
 /* eslint-disable @typescript-eslint/unbound-method */
-/* eslint-disable @typescript-eslint/no-var-requires */
+
 // Import the API
 const { ApiPromise } = require('@polkadot/api');
 

docs/examples/promise/05_read_storage/index.js

@@ -1,6 +1,7 @@
+/* eslint-disable header/header */
 /* eslint-disable @typescript-eslint/require-await */
 /* eslint-disable @typescript-eslint/unbound-method */
-/* eslint-disable @typescript-eslint/no-var-requires */
+
 // Import the API
 const { ApiPromise } = require('@polkadot/api');
 

docs/examples/promise/05_read_storage_at/index.js

@@ -1,6 +1,7 @@
+/* eslint-disable header/header */
 /* eslint-disable @typescript-eslint/require-await */
 /* eslint-disable @typescript-eslint/unbound-method */
-/* eslint-disable @typescript-eslint/no-var-requires */
+
 // Import the API
 const { ApiPromise } = require('@polkadot/api');
 

docs/examples/promise/06_make_transfer/index.js

@@ -1,6 +1,7 @@
+/* eslint-disable header/header */
 /* eslint-disable @typescript-eslint/require-await */
 /* eslint-disable @typescript-eslint/unbound-method */
-/* eslint-disable @typescript-eslint/no-var-requires */
+
 // Import the API, Keyring and some utility functions
 const { ApiPromise } = require('@polkadot/api');
 const { Keyring } = require('@polkadot/keyring');

docs/examples/promise/07_make_transfer_with_allowed_block_permissions_only/index.js

@@ -1,6 +1,7 @@
+/* eslint-disable header/header */
 /* eslint-disable @typescript-eslint/require-await */
 /* eslint-disable @typescript-eslint/unbound-method */
-/* eslint-disable @typescript-eslint/no-var-requires */
+
 // Import the API, Keyring and some utility functions
 const { ApiPromise } = require('@polkadot/api');
 const { Keyring } = require('@polkadot/keyring');

docs/examples/promise/08_system_events/index.js

@@ -1,6 +1,7 @@
+/* eslint-disable header/header */
 /* eslint-disable @typescript-eslint/require-await */
 /* eslint-disable @typescript-eslint/unbound-method */
-/* eslint-disable @typescript-eslint/no-var-requires */
+
 // Import the API
 const { ApiPromise } = require('@polkadot/api');
 

docs/examples/promise/09_transfer_events/index.js

@@ -1,6 +1,7 @@
+/* eslint-disable header/header */
 /* eslint-disable @typescript-eslint/require-await */
-/* eslint-disable @typescript-eslint/unbound-method */
 /* eslint-disable @typescript-eslint/no-var-requires */
+
 // Import the API & Provider and some utility functions
 const { ApiPromise } = require('@polkadot/api');
 
@@ -44,7 +45,7 @@ async function main () {
         console.log('Included at block hash', status.asInBlock.toHex());
         console.log('Events:');
 
-        events.forEach(({ phase, event: { data, method, section } }) => {
+        events.forEach(({ event: { data, method, section }, phase }) => {
           console.log('\t', phase.toString(), `: ${section}.${method}`, data.toString());
         });
       } else if (status.isFinalized) {

docs/examples/promise/10_upgrade_chain/index.js

@@ -1,6 +1,7 @@
+/* eslint-disable header/header */
 /* eslint-disable @typescript-eslint/require-await */
 /* eslint-disable @typescript-eslint/unbound-method */
-/* eslint-disable @typescript-eslint/no-var-requires */
+
 // Import the API & Provider and some utility functions
 const { ApiPromise, WsProvider } = require('@polkadot/api');
 

docs/examples/rx/01_simple_connect/index.js

@@ -1,6 +1,7 @@
+/* eslint-disable header/header */
 /* eslint-disable @typescript-eslint/require-await */
 /* eslint-disable @typescript-eslint/unbound-method */
-/* eslint-disable @typescript-eslint/no-var-requires */
+
 // Required imports
 const { zip } = require('rxjs');
 const { ApiRx } = require('@polkadot/api');

docs/examples/rx/02_listen_to_blocks/index.js

@@ -1,6 +1,7 @@
+/* eslint-disable header/header */
 /* eslint-disable @typescript-eslint/require-await */
 /* eslint-disable @typescript-eslint/unbound-method */
-/* eslint-disable @typescript-eslint/no-var-requires */
+
 // Import the API
 const { ApiRx } = require('@polkadot/api');
 const { switchMap } = require('rxjs/operators');

docs/examples/rx/03_listen_to_balance_change/index.js

@@ -1,6 +1,7 @@
+/* eslint-disable header/header */
 /* eslint-disable @typescript-eslint/require-await */
 /* eslint-disable @typescript-eslint/unbound-method */
-/* eslint-disable @typescript-eslint/no-var-requires */
+
 // Import the API and operators from RxJs
 const { ApiRx } = require('@polkadot/api');
 const { pairwise, startWith } = require('rxjs/operators');
@@ -27,6 +28,7 @@ async function main () {
         // then balance[1] is the initial value of Alice account.
         console.log(`Alice ${Alice} has a balance of ${balance[1].data.free}`);
         console.log('You may leave this example running and start the "Make a transfer" example or transfer any value to Alice address');
+
         return;
       }
 

docs/examples/rx/04_unsubscribe/index.js

@@ -1,6 +1,7 @@
+/* eslint-disable header/header */
 /* eslint-disable @typescript-eslint/require-await */
 /* eslint-disable @typescript-eslint/unbound-method */
-/* eslint-disable @typescript-eslint/no-var-requires */
+
 // Import the API
 const { ApiRx } = require('@polkadot/api');
 const { switchMap } = require('rxjs/operators');

docs/examples/rx/05_read_storage/index.js

@@ -1,6 +1,7 @@
+/* eslint-disable header/header */
 /* eslint-disable @typescript-eslint/require-await */
 /* eslint-disable @typescript-eslint/unbound-method */
-/* eslint-disable @typescript-eslint/no-var-requires */
+
 // Import the API
 const { ApiRx } = require('@polkadot/api');
 const { combineLatest, of } = require('rxjs');

docs/examples/rx/06_make_transfer/index.js

@@ -1,6 +1,8 @@
+
+/* eslint-disable header/header */
 /* eslint-disable @typescript-eslint/require-await */
 /* eslint-disable @typescript-eslint/unbound-method */
-/* eslint-disable @typescript-eslint/no-var-requires */
+
 // Import the API, Keyring and some utility functions
 const { ApiRx } = require('@polkadot/api');
 const { Keyring } = require('@polkadot/keyring');

docs/examples/rx/08_system_events/index.js

@@ -1,6 +1,7 @@
+/* eslint-disable header/header */
 /* eslint-disable @typescript-eslint/require-await */
 /* eslint-disable @typescript-eslint/unbound-method */
-/* eslint-disable @typescript-eslint/no-var-requires */
+
 // Import the API and selected RxJs operators
 const { switchMap } = require('rxjs/operators');
 const { ApiRx } = require('@polkadot/api');

docs/examples/rx/09_transfer_events/index.js

@@ -1,6 +1,7 @@
+/* eslint-disable header/header */
 /* eslint-disable @typescript-eslint/require-await */
 /* eslint-disable @typescript-eslint/unbound-method */
-/* eslint-disable @typescript-eslint/no-var-requires */
+
 // Import the API and some utility functions
 const { ApiRx } = require('@polkadot/api');
 
@@ -48,7 +49,7 @@ async function main () {
         console.log('Included at block hash', status.asInBlock.toHex());
         console.log('Events:');
 
-        events.forEach(({ phase, event: { data, method, section } }) => {
+        events.forEach(({ event: { data, method, section }, phase }) => {
           console.log('\t', phase.toString(), `: ${section}.${method}`, data.toString());
         });
       } else if (status.isFinalized) {

docs/examples/rx/10_upgrade_chain/index.js

@@ -1,6 +1,7 @@
+/* eslint-disable header/header */
 /* eslint-disable @typescript-eslint/require-await */
 /* eslint-disable @typescript-eslint/unbound-method */
-/* eslint-disable @typescript-eslint/no-var-requires */
+
 // Import the API & Provider and some utility functions
 const { ApiRx, WsProvider } = require('@polkadot/api');
 // Import the test keyring (already has dev keys for Alice, Bob, Charlie, Eve & Ferdie)

docs/start/FAQ.md

@@ -52,8 +52,14 @@ Likewise, if your chain has been upgraded recently and you are still using the o
 
 ## I cannot send transactions from my node-template-based chain
 
-For the API defaults, the API always the type definitions as specified by the Substrate master fully-featured node. This means that any customizations needs to be applied as types should there be differences in specific user-implementations.
+The API always injects the default type definitions as specified by the Substrate master fully-featured node. This means that any customizations to chains needs needs to be applied as types, should there be differences in specific user-implementations.
 
-The Substrate node-template has added customizations for some types in the default template, specifically around the `Address` and `Lookup` types, removing indices lookups from addresses. This means that the transaction payload saves 2 bytes for a transfer and is an approach followed by other chains as well, notably Polkadot & Kusama.
+The Substrate node-template has added customizations for some types in the default template, specifically around the `Address` and `Lookup` types, removing any lookups based on indices. This means that the transaction payload saves 2 bytes for a transfer and is an approach followed by other chains as well, notably Polkadot & Kusama.
 
-Due to these customizations and differences that bleed through to the transaction formats, out-of-the-box the node-template will have issues when sending transactions. To fix this, you would need to add [the customized Address types into your API](types.extend.md#impact-on-extrinsics) instances (or UIs), allowing the API to have the information required to adjust the encoding.
+Due to these customizations and differences that bleed through to the transaction formats, out-of-the-box chains based on the node-template will have issues when sending transactions. To fix this, you would need to add [the customized Address types into your API](types.extend.md#impact-on-extrinsics) instances (or UIs), allowing the API to have the information required to adjust the encoding.
+
+## Using a non-current-master node, I have issues parsing events
+
+Recently Substrate master updated the `Weight` type from `u32` -> `u64`. This type is used in the `DispatchInfo` struct in the `system.ExtrinsicSuccess` events, to return the applied call weights as well as the resulting fees. Since the API master branch tracks Substrate master, this means the change has been applied by default, with the default set to `u64`.
+
+If you are on a chain that has not been upgraded yet, you need to add `Weight: 'u32'` to your types to allow for successful parsing of all events. Without this override, parsing will fail. As soon as one event in the `Vec<EventRecord>` structure from `system.events` fails to parse, all subsequent events are affected and the decoding will return an error.

docs/start/api.query.multi.md

@@ -23,7 +23,7 @@ A couple of items to note in the example above: we don't call `account` directly
 ...
 
 // Retrieve a snapshot of the validators
-const validators = await api.query.session.validators();
+const validatorKeys = await api.query.session.validators.keys();
 
 // Subscribe to the balances for these accounts
 const unsub = await api.query.balances.account.multi(validators, (balances) => {

docs/start/api.query.other.md

@@ -62,7 +62,7 @@ When working maps and double-maps, it is possible to retrieve a list of all the
 // Retrieve the active era
 const activeEra = await api.query.staking.activeEra();
 
-// retrieve all exposures int the active era
+// retrieve all exposures for the active era
 const exposures = await api.query.staking.erasStakers.entries(activeEra.index);
 
 exposures.forEach(([key, exposure]) => {
@@ -71,9 +71,24 @@ exposures.forEach(([key, exposure]) => {
 });
 ```
 
-Here we are querying a double-map, so we supply 1 argument. No arguments on double-maps will be very costly, retrieving all the eras and associated entries. Additionally when `twox64_concat` & `blake2_concat` is used, the key `.args` will contain decoded values of the params, in this case it will contain the actual `AccountId` of the staker. (Since that was not supplied)
+To understand the usage of the `key.args`, you need to understand that map/doublemap keys are stored alongside their lookups. This means that the raw key has hashed parts as well as the raw data. The API will decode the keys and provide the raw key arguments in args. This would mean -
 
-In the same way as above we can simply do `.keys(activeEra.index): StorageKey[]` to retrieve all the keys here, including the individual keys args decoding, as available on maps with decodable hashing functions.
+- if we are querying `api.query.staking.validators(validatorId: AccountId)` via `entries`, the `key.args` would be `[AccountId]`
+- if we are querying `api.query.staking.erasStakers(era: EraIndex, validatorId: AccountId)` via `entries`, the `key.args` would be `[EraIndex, AccountId]`
+
+the same applies to `.keys()` - here the list of keys also have the decoded args, as specified. You can think of `.args` as a tuple with the same types as the types required to retrieve a single entry in the map.
+
+In the first example we are querying a double-map, so we supply 1 argument. No arguments on double-maps will be very costly, retrieving all the eras and associated entries. In the same way as above we can simply do `.keys(activeEra.index): StorageKey[]` to retrieve all the keys here, including the individual keys args (available on maps with decodable hashing functions) -
+
+```js
+// retrieve all the nominator keys
+const keys = await api.query.staking.nominators.keys();
+
+// extract the first key argument [AccountId] as string
+const nominatorIds = keys.map(({ args: [nominatorId] }) => nominatorId);
+
+console.log('all nominators:', nominatorIds.join(', '));
+```
 
 ## State entries
 

docs/start/rpc.custom.md

@@ -17,6 +17,11 @@ const api = await ApiPromise.create({
           {
             name: 'index',
             type: 'u64'
+          },
+          {
+            name: 'at',
+            type: 'Hash',
+            isOptional: true
           }
         ],
         type: 'Balance'
@@ -26,7 +31,9 @@ const api = await ApiPromise.create({
 });
 ```
 
-In the above example we have defined a new method, which is now available on the RPCs as `api.rpc.firstModule.testMethod(u64) => Promise<Balance>`. Do be aware that while defined, the method will only appear if it is in the list as returned by `api.rpc.rpc.methods()`, which is the list of known RPCs the node exposes. When making changes to the node, always ensure that it does expose the RPC method correctly, otherwise it will not be decorated.
+In the above example we have defined a new method, which is now available on the RPCs as `api.rpc.firstModule.testMethod(index: u64, at?: Hash) => Promise<Balance>`. In the case of optional params, we have added the `isOptional: true` flag alongside the `name` & `type` in the param definition.
+
+Be aware that while defined, the method will only appear on the API if it is in the list as returned by `api.rpc.rpc.methods()`, which is the list of known RPCs the node exposes. When making changes to the node, always ensure that it does expose the RPC method correctly, otherwise it will not be decorated.
 
 ## Definition breakdown
 

docs/start/types.basics.md

@@ -57,7 +57,7 @@ Be aware that in the JS version naming defaults to `camelCase` where names of fi
 
 Each enum has additional getters which are injected based on the fields wrapped. These take the form of `.is<Name>` and `.as<Name>` to allow you to check is the enum is a certain value or to retrieve the underlying value as a specific type.
 
-As a real-world example, when an extrinsic is applied, the `Phase` enum has one of two states, `ApplyExtrinsic(u32)` or `Finalization`. In this case `.isApplyExtrinsic` would be `true` when an extrinsic is being applied, and `.asApplyExtrinsic` would return the value as a `u32` (which is the index of the extrinsic in the block, as it is being applied). When `isisApplyExtrinsic` is `false` and `asApplyExtrinsic` is called, the getter will throw.
+As a real-world example, when an extrinsic is applied, the `Phase` enum has one of two states, `ApplyExtrinsic(u32)` or `Finalization`. In this case `.isApplyExtrinsic` would be `true` when an extrinsic is being applied, and `.asApplyExtrinsic` would return the value as a `u32` (which is the index of the extrinsic in the block, as it is being applied). When `isApplyExtrinsic` is `false` and `asApplyExtrinsic` is called, the getter will throw.
 
 ## Working with Option&lt;Type&gt;
 
@@ -67,6 +67,7 @@ An `Option<Type>` attempts to mimic the Rust approach of having `None` and `Some
 - `.isSome` - this is `true` is a value is wrapped, i.e. if a `Option<u32>` has an actual underlying `u32`
 - `.unwrap()` - when `isSome`, this will return the wrapped value, i.e. for `Option<u32>`, this would return the `u32`. When the value is `isNone`, this call will throw an exception.
 - `.unwrapOr(<default value>)` - this extends `unwrap()`, returning the wrapped value when `isSome` and in the case of `isNone` it will return the `<default value>` passed.
+- `.unwrapOrDefault()` - returns either the rapped value when `isSome`, or the default for the type when `isNone`
 
 ## Working with Tuples
 

docs/start/types.extend.md

@@ -26,6 +26,15 @@ const api = await ApiPromise.create({
 
 The above introduces the `types` registry, effectively allowing overrides and the definition of new types. The override above would mean that immediately the API will treat all occurrences of `Balance` not as the default, but rather as the defined size.
 
+## Field ordering
+
+When defining any custom structures or types, it is critical that the following rules are applied -
+
+- Map exactly to what is defined in the Rust code, i.e. defining a `SaleType` cannot be `u16` on the one end and `u32` on the other end. If mismatches occur, the serialization will fail.
+- Ensure that the field order is maintained in all definitions. The SCALE serialization is binary and contains no field names in the serialization, only the encoded values. Any decoding is therefore done based on the size of the type and the order thereof in the definitions.
+
+These rules apply everywhere. Always ensure that the types match exactly between the environments and that the ordering is maintained, be it for structs, tuples or enums.
+
 ## User-defined structs
 
 Registration also applies to any type that can be found on a specific chain, i.e. we can add any types that is available on a specific node -

docs/start/typescript.user.md

@@ -4,7 +4,7 @@ In the previous section we looked at the TypeScript definitions that are availab
 
 ## Definitions
 
-A large part of the type definitions are defined not as classes, but rather as structures from which type definitions are extracted and that can be used as-is to let the API know how to decode a type. By now you should be familiar with [type extensions](types.extend.md), but effectively the [types library does exactly the same](https://github.com/polkadot-js/api/tree/master/packages/types/src/interfaces).
+A large part of the type definitions are defined not as classes, but rather as structures from which type definitions are extracted and that can be used as-is to let the API know how to decode a type. By now you should be familiar with [type extensions](types.extend.md) and the [rules around fields](types.extend.md#field-ordering), but effectively the [types library does exactly the same](https://github.com/polkadot-js/api/tree/master/packages/types/src/interfaces).
 
 To create TypeScript-aware definitions and keep your files in a single place, you can follow the same approach. Assuming you have a package named `@MeInc/stuff` where you have the definitions in an `stuff/interfaces` folder. Each sub-folder would indicate a runtime module, e.g.
 

jest.config.js

@@ -1,15 +1,19 @@
-/* eslint-disable @typescript-eslint/camelcase */
-/* eslint-disable @typescript-eslint/no-var-requires */
+// Copyright 2017-2020 @polkadot/api authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
 const config = require('@polkadot/dev/config/jest');
 
 module.exports = Object.assign({}, config, {
   moduleNameMapper: {
     '@polkadot/api-contract(.*)$': '<rootDir>/packages/api-contract/src/$1',
     '@polkadot/api-derive(.*)$': '<rootDir>/packages/api-derive/src/$1',
+    // eslint-disable-next-line sort-keys
     '@polkadot/api(.*)$': '<rootDir>/packages/api/src/$1',
     '@polkadot/metadata(.*)$': '<rootDir>/packages/metadata/src/$1',
     '@polkadot/rpc-(core|provider)(.*)$': '<rootDir>/packages/rpc-$1/src/$2',
     '@polkadot/types-known(.*)$': '<rootDir>/packages/types-known/src/$1',
+    // eslint-disable-next-line sort-keys
     '@polkadot/types(.*)$': '<rootDir>/packages/types/src/$1'
   },
   modulePathIgnorePatterns: [

jest.resolver.js

@@ -1,8 +1,7 @@
-// Copyright 2017-2020 @polkadot/util authors & contributors
+// Copyright 2017-2020 @polkadot/api authors & contributors
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
-// eslint-disable-next-line @typescript-eslint/no-var-requires
 const path = require('path');
 
 module.exports = function resolver (file, config) {

lerna.json

@@ -9,5 +9,5 @@
   "packages": [
     "packages/*"
   ],
-  "version": "1.9.1"
+  "version": "1.12.1"
 }

package.json

@@ -27,10 +27,10 @@
     "@babel/core": "^7.9.0",
     "@babel/register": "^7.9.0",
     "@babel/runtime": "^7.9.2",
-    "@polkadot/dev": "^0.51.9",
-    "@polkadot/ts": "^0.3.14",
+    "@polkadot/dev": "^0.52.11",
+    "@polkadot/ts": "^0.3.18",
     "@polkadot/typegen": "workspace:packages/typegen",
-    "@types/jest": "^25.1.4",
+    "@types/jest": "^25.2.1",
     "copyfiles": "^2.2.0"
   }
 }

packages/api-contract/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/api-contract",
-  "version": "1.9.1",
+  "version": "1.12.1",
   "description": "Interfaces for interacting with contracts and contract ABIs",
   "main": "index.js",
   "keywords": [
@@ -27,11 +27,11 @@
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/api-contract#readme",
   "dependencies": {
     "@babel/runtime": "^7.9.2",
-    "@polkadot/api": "1.9.1",
-    "@polkadot/rpc-core": "1.9.1",
-    "@polkadot/types": "1.9.1",
-    "@polkadot/util": "^2.7.1",
+    "@polkadot/api": "1.12.1",
+    "@polkadot/rpc-core": "1.12.1",
+    "@polkadot/types": "1.12.1",
+    "@polkadot/util": "^2.9.0-beta.3",
     "bn.js": "^5.1.1",
-    "rxjs": "^6.5.4"
+    "rxjs": "^6.5.5"
   }
 }

packages/api-contract/src/Abi.ts

@@ -18,10 +18,10 @@ export default class ContractAbi extends ContractRegistry implements InterfaceAb
 
   constructor (registry: Registry, abi: ContractABIPre) {
     super(registry, abi);
-    [this.abi, this.constructors, this.messages] = this.decodeAbi(abi);
+    [this.abi, this.constructors, this.messages] = this._decodeAbi(abi);
   }
 
-  private decodeAbi (abiPre: ContractABIPre): [ContractABI, ContractABIFn[], AbiMessages] {
+  private _decodeAbi (abiPre: ContractABIPre): [ContractABI, ContractABIFn[], AbiMessages] {
     this.validateAbi(abiPre);
 
     const abi = this.convertAbi(abiPre);

packages/api-contract/src/ContractRegistry.ts

@@ -61,7 +61,7 @@ export default class ContractRegistry extends MetaRegistry {
       const unknownKeys = Object.keys(arg).filter((key): boolean => !['name', 'type'].includes(key));
 
       assert(unknownKeys.length === 0, `Unknown keys ${unknownKeys.join(', ')} found in ABI args for ${name}`);
-      assert(isNumber(arg.name) && isString(this.stringAt(arg.name)), `${name} args should have valid name `);
+      assert(isNumber(arg.name) && isString(this._stringAt(arg.name)), `${name} args should have valid name `);
       assert(isNumber(arg.type.ty) && isObject(this.typeDefAt(arg.type.ty)), `${name} args should have valid type`);
     });
   }
@@ -83,9 +83,9 @@ export default class ContractRegistry extends MetaRegistry {
 
       assert(unknownKeys.length === 0, `Unknown keys ${unknownKeys.join(', ')} found in ABI args for messages.${message.name}`);
 
-      const { name, selector, return_type: returnType } = message;
+      const { name, return_type: returnType, selector } = message;
 
-      assert(isNumber(name) && isString(this.stringAt(name)), `Expected name for ${fnname}`);
+      assert(isNumber(name) && isString(this._stringAt(name)), `Expected name for ${fnname}`);
       assert(isNull(returnType) || (isNumber(returnType.ty) && isObject(this.typeDefAt(returnType.ty))), `Expected return_type for ${fnname}`);
 
       parseSelector(this.registry, fnname, selector);
@@ -100,7 +100,7 @@ export default class ContractRegistry extends MetaRegistry {
 
     const { contract: { constructors, messages, name } } = abi;
 
-    assert(constructors && messages && isString(this.stringAt(name)), 'ABI should have constructors, messages & name sections');
+    assert(constructors && messages && isString(this._stringAt(name)), 'ABI should have constructors, messages & name sections');
 
     this.validateConstructors(abi);
     this.validateMessages(abi);
@@ -109,13 +109,16 @@ export default class ContractRegistry extends MetaRegistry {
   public createMessage (name: string, message: Partial<ContractABIMessage> & ContractABIMessageBase): ContractABIFn {
     const args = message.args.map(({ name, type }): ContractABIFnArg => {
       assert(isObject(type), `Invalid type at index ${type}`);
+
       return {
         name: stringCamelCase(name),
         type
       };
     });
+
     const Clazz = createArgClass(this.registry, args, isUndefined(message.selector) ? {} : { __selector: 'u32' });
     const baseStruct: { [index: string]: any } = { __selector: isUndefined(message.selector) ? undefined : parseSelector(this.registry, name, message.selector) };
+
     const encoder = (...params: CodecArg[]): Uint8Array => {
       assert(params.length === args.length, `Expected ${args.length} arguments to contract ${name}, found ${params.length}`);
 
@@ -136,6 +139,7 @@ export default class ContractRegistry extends MetaRegistry {
     fn.args = args;
     fn.isConstant = !(message.mutates || false);
     fn.type = message.returnType || null;
+
     return fn;
   }
 
@@ -147,19 +151,20 @@ export default class ContractRegistry extends MetaRegistry {
   }
 
   public convertArgs (args: ContractABIArgBasePre[]): any[] {
-    return args.map(({ name, type, ...arg }) => ({ ...arg, name: this.stringAt(name), type: this.convertType(type) }));
+    return args.map(({ name, type, ...arg }) => ({ ...arg, name: this._stringAt(name), type: this.convertType(type) }));
   }
 
-  public convertType ({ ty, display_name: displayNameIndices }: ContractABITypePre): TypeDef {
+  public convertType ({ display_name: displayNameIndices, ty }: ContractABITypePre): TypeDef {
     const displayName = this.stringsAt(displayNameIndices).join('::');
+
     return this.typeDefAt(ty, { displayName });
   }
 
-  public convertContract ({ constructors, messages, name, events, ...contract }: ContractABIContractPre): ContractABIContract {
+  public convertContract ({ constructors, events, messages, name, ...contract }: ContractABIContractPre): ContractABIContract {
     return {
       constructors: this.convertConstructors(constructors),
       messages: messages.map((message) => this.convertMessage(message)),
-      name: this.stringAt(name),
+      name: this._stringAt(name),
       ...(events
         ? { events: events.map((event) => this.convertEvent(event)) }
         : {}),
@@ -179,7 +184,7 @@ export default class ContractRegistry extends MetaRegistry {
     return {
       ...message,
       args: this.convertArgs(args),
-      name: this.stringAt(name),
+      name: this._stringAt(name),
       returnType: returnType ? this.convertType(returnType) : null
     };
   }
@@ -202,13 +207,13 @@ export default class ContractRegistry extends MetaRegistry {
     }
   }
 
-  public convertStorageStruct ({ 'struct.type': structType, 'struct.fields': structFields }: ContractABIStorageStructPre): ContractABIStorageStruct {
+  public convertStorageStruct ({ 'struct.fields': structFields, 'struct.type': structType }: ContractABIStorageStructPre): ContractABIStorageStruct {
     return {
-      'struct.type': this.typeDefAt(structType),
-      'struct.fields': structFields.map(({ name, layout }) => ({
-        name: this.stringAt(name),
-        layout: this.convertStorageLayout(layout)
-      }))
+      'struct.fields': structFields.map(({ layout, name }) => ({
+        layout: this.convertStorageLayout(layout),
+        name: this._stringAt(name)
+      })),
+      'struct.type': this.typeDefAt(structType)
     };
   }
 

packages/api-contract/src/MetaRegistry.ts

@@ -48,7 +48,7 @@ class MetadataRegistryLookup {
     this._types = types;
   }
 
-  protected hasItemAt (index: number, variant: MetaRegistryItem): boolean {
+  protected _hasItemAt (index: number, variant: MetaRegistryItem): boolean {
     switch (variant) {
       case MetaRegistryItem.String:
         return this._strings && !!this._strings[index - 1];
@@ -59,8 +59,8 @@ class MetadataRegistryLookup {
     }
   }
 
-  protected itemAt (index: number, variant: MetaRegistryItem): any {
-    assert(this.hasItemAt(index, variant), `MetaRegistry: Invalid ${variant} index ${index} found in metadata`);
+  protected _itemAt (index: number, variant: MetaRegistryItem): any {
+    assert(this._hasItemAt(index, variant), `MetaRegistry: Invalid ${variant} index ${index} found in metadata`);
 
     switch (variant) {
       case MetaRegistryItem.String:
@@ -72,33 +72,33 @@ class MetadataRegistryLookup {
     }
   }
 
-  protected itemsAt (indices: number[], variant: MetaRegistryItem): any[] {
-    return indices.map((index: number): string => this.itemAt(index, variant));
+  protected _itemsAt (indices: number[], variant: MetaRegistryItem): any[] {
+    return indices.map((index: number): string => this._itemAt(index, variant));
   }
 
-  protected stringAt (index: StringIndex): string {
-    return this.itemAt(index, MetaRegistryItem.String);
+  protected _stringAt (index: StringIndex): string {
+    return this._itemAt(index, MetaRegistryItem.String);
   }
 
   public stringsAt (indices: StringIndex[]): string[] {
-    return this.itemsAt(indices, MetaRegistryItem.String);
+    return this._itemsAt(indices, MetaRegistryItem.String);
   }
 
   public typeAt (index: TypeIndex): MetaType {
-    return this.itemAt(index, MetaRegistryItem.Type);
+    return this._itemAt(index, MetaRegistryItem.Type);
   }
 
   public typesAt (indices: TypeIndex[]): MetaType[] {
-    return this.itemsAt(indices, MetaRegistryItem.Type);
+    return this._itemsAt(indices, MetaRegistryItem.Type);
   }
 
   public hasTypeDefAt (index: TypeIndex): boolean {
-    return this.hasItemAt(index, MetaRegistryItem.TypeDef);
+    return this._hasItemAt(index, MetaRegistryItem.TypeDef);
   }
 
   public typeDefAt (index: TypeIndex, extra: Pick<TypeDef, never> = {}): TypeDef {
     return {
-      ...this.itemAt(index, MetaRegistryItem.TypeDef),
+      ...this._itemAt(index, MetaRegistryItem.TypeDef),
       ...extra
     };
   }
@@ -116,7 +116,7 @@ export default class MetaRegistry extends MetadataRegistryLookup {
     this.typeDefs[typeIndex - 1] = this.typeDefFromMetaType(this.typeAt(typeIndex), typeIndex);
   }
 
-  private typeDefIdFields ({ id }: MetaType): Pick<TypeDef, never> {
+  private _typeDefIdFields ({ id }: MetaType): Pick<TypeDef, never> {
     const { 'custom.name': nameIndex, 'custom.namespace': namespaceIndices, 'custom.params': paramsIndices } = id as MetaTypeIdCustom;
 
     if (!nameIndex) {
@@ -124,7 +124,7 @@ export default class MetaRegistry extends MetadataRegistryLookup {
     }
 
     return {
-      name: this.stringAt(nameIndex),
+      name: this._stringAt(nameIndex),
       ...(namespaceIndices && namespaceIndices.length
         ? { namespace: this.stringsAt(namespaceIndices).join('::') }
         : {}
@@ -136,33 +136,33 @@ export default class MetaRegistry extends MetadataRegistryLookup {
     };
   }
 
-  private typeDefDefFields (metaType: MetaType, typeIndex?: TypeIndex): Pick<TypeDef, never> {
+  private _typeDefDefFields (metaType: MetaType, typeIndex?: TypeIndex): Pick<TypeDef, never> {
     let typeDef;
 
     switch (detectedType(metaType)) {
       case MetaTypeInfo.BuiltinPlain:
-        typeDef = this.typeDefForBuiltinPlain(metaType.id as string);
+        typeDef = this._typeDefForBuiltinPlain(metaType.id as string);
         break;
       case MetaTypeInfo.BuiltinTuple:
-        typeDef = this.typeDefForBuiltinTuple(metaType.id as TypeIndex[]);
+        typeDef = this._typeDefForBuiltinTuple(metaType.id as TypeIndex[]);
         break;
       case MetaTypeInfo.BuiltinVec:
-        typeDef = this.typeDefForBuiltinVec(metaType.id as MetaTypeIdVec, typeIndex);
+        typeDef = this._typeDefForBuiltinVec(metaType.id as MetaTypeIdVec, typeIndex);
         break;
       case MetaTypeInfo.BuiltinVecFixed:
-        typeDef = this.typeDefForBuiltinVecFixed(metaType.id as MetaTypeIdVecFixed, typeIndex);
+        typeDef = this._typeDefForBuiltinVecFixed(metaType.id as MetaTypeIdVecFixed, typeIndex);
         break;
       case MetaTypeInfo.Enum:
-        typeDef = this.typeDefForEnum(metaType.def as MetaTypeDefEnum, metaType.id as MetaTypeIdCustom, typeIndex);
+        typeDef = this._typeDefForEnum(metaType.def as MetaTypeDefEnum, metaType.id as MetaTypeIdCustom, typeIndex);
         break;
       case MetaTypeInfo.ClikeEnum:
-        typeDef = this.typeDefForClikeEnum(metaType.def as MetaTypeDefClikeEnum);
+        typeDef = this._typeDefForClikeEnum(metaType.def as MetaTypeDefClikeEnum);
         break;
       case MetaTypeInfo.Struct:
         typeDef = this.typeDefForStruct(metaType.def as MetaTypeDefStruct);
         break;
       case MetaTypeInfo.TupleStruct:
-        typeDef = this.typeDefForTupleStruct(metaType.def as MetaTypeDefTupleStruct);
+        typeDef = this._typeDefForTupleStruct(metaType.def as MetaTypeDefTupleStruct);
         break;
       case MetaTypeInfo.Null:
       default:
@@ -176,8 +176,8 @@ export default class MetaRegistry extends MetadataRegistryLookup {
     return withTypeString({
       info: TypeDefInfo.Null,
       type: '',
-      ...this.typeDefDefFields(metaType, typeIndex),
-      ...this.typeDefIdFields(metaType)
+      ...this._typeDefDefFields(metaType, typeIndex),
+      ...this._typeDefIdFields(metaType)
     }) as TypeDef;
   }
 
@@ -189,13 +189,13 @@ export default class MetaRegistry extends MetadataRegistryLookup {
     return this.typeDefAt(typeIndex);
   }
 
-  private typeDefForEnumVariant (variant: MetaTypeDefEnumVariant): Pick<TypeDef, any> {
+  private _typeDefForEnumVariant (variant: MetaTypeDefEnumVariant): Pick<TypeDef, any> {
     const { 'unit_variant.name': unitNameIndex } = variant as MetaTypeDefEnumVariantUnit;
 
     if (unitNameIndex) {
       return {
         info: TypeDefInfo.Plain,
-        name: this.stringAt(unitNameIndex),
+        name: this._stringAt(unitNameIndex),
         type: 'Null'
       };
     }
@@ -203,7 +203,7 @@ export default class MetaRegistry extends MetadataRegistryLookup {
     const { 'tuple_struct_variant.name': tupleStructVariantNameIndex } = variant as MetaTypeDefEnumVariantTupleStruct;
 
     if (tupleStructVariantNameIndex) {
-      return this.typeDefForTupleStruct(variant as MetaTypeDefEnumVariantTupleStruct);
+      return this._typeDefForTupleStruct(variant as MetaTypeDefEnumVariantTupleStruct);
     }
 
     const { 'struct_variant.name': structVariantNameIndex } = variant as MetaTypeDefEnumVariantStruct;
@@ -218,14 +218,14 @@ export default class MetaRegistry extends MetadataRegistryLookup {
     };
   }
 
-  private typeDefForBuiltinPlain (id: string): Pick<TypeDef, never> {
+  private _typeDefForBuiltinPlain (id: string): Pick<TypeDef, never> {
     return {
       info: TypeDefInfo.Plain,
       type: id
     };
   }
 
-  private typeDefForBuiltinTuple (id: TypeIndex[]): Pick<TypeDef, never> {
+  private _typeDefForBuiltinTuple (id: TypeIndex[]): Pick<TypeDef, never> {
     const sub = id.map((tupleTypeIndex: number): TypeDef => this.typeDefFromMetaTypeAt(tupleTypeIndex));
 
     return {
@@ -234,48 +234,52 @@ export default class MetaRegistry extends MetadataRegistryLookup {
     };
   }
 
-  private typeDefForBuiltinVec (id: MetaTypeIdVec, typeIndex?: TypeIndex): Pick<TypeDef, never> {
+  private _typeDefForBuiltinVec (id: MetaTypeIdVec, typeIndex?: TypeIndex): Pick<TypeDef, never> {
     const { 'slice.type': vecTypeIndex } = id;
 
     assert(!typeIndex || vecTypeIndex !== typeIndex, `MetaRegistry: self-referencing registry type at index ${typeIndex}`);
 
     const type = displayType(this.typeDefFromMetaTypeAt(vecTypeIndex));
+
     assert(type && type.length > 0, `MetaRegistry: Invalid builtin Vec type found at index ${typeIndex}`);
 
     return {
       info: TypeDefInfo.Vec,
-      type: `Vec<${type}>`,
-      sub: this.typeDefFromMetaTypeAt(vecTypeIndex)
+      sub: this.typeDefFromMetaTypeAt(vecTypeIndex),
+      type: `Vec<${type}>`
     };
   }
 
-  private typeDefForBuiltinVecFixed (id: MetaTypeIdVecFixed, typeIndex?: TypeIndex): Pick<TypeDef, never> {
-    const { 'array.type': vecTypeIndex, 'array.len': vecLength } = id;
+  private _typeDefForBuiltinVecFixed (id: MetaTypeIdVecFixed, typeIndex?: TypeIndex): Pick<TypeDef, never> {
+    const { 'array.len': vecLength, 'array.type': vecTypeIndex } = id;
 
     assert(!vecLength || vecLength <= 256, 'MetaRegistry: Only support for [Type; <length>], where length <= 256');
     assert(!typeIndex || vecTypeIndex !== typeIndex, `MetaRegistry: self-referencing registry type at index ${typeIndex}`);
 
     const type = displayType(this.typeDefFromMetaTypeAt(vecTypeIndex));
+
     assert(type && type.length > 0, `MetaRegistry: Invalid vector type found at index ${typeIndex}`);
 
     return {
       info: TypeDefInfo.VecFixed,
       length: vecLength, // ex: { type: type }
-      type: `[${type};${vecLength}]`,
-      sub: this.typeDefFromMetaTypeAt(vecTypeIndex)
+      sub: this.typeDefFromMetaTypeAt(vecTypeIndex),
+      type: `[${type};${vecLength}]`
     };
   }
 
-  private typeDefForEnum (def: MetaTypeDefEnum, id: MetaTypeIdCustom, typeIndex?: TypeIndex): Pick<TypeDef, any> {
-    const name = id && this.stringAt(id['custom.name']);
+  private _typeDefForEnum (def: MetaTypeDefEnum, id: MetaTypeIdCustom, typeIndex?: TypeIndex): Pick<TypeDef, any> {
+    const name = id && this._stringAt(id['custom.name']);
 
     switch (name) {
       case 'Option':
         return this.typeDefForOption(id, typeIndex);
+
       case 'Result':
         return this.typeDefForResult(id, typeIndex);
+
       default: {
-        const sub = def['enum.variants'].map((variant) => this.typeDefForEnumVariant(variant));
+        const sub = def['enum.variants'].map((variant) => this._typeDefForEnumVariant(variant));
 
         return {
           info: TypeDefInfo.Enum,
@@ -285,13 +289,13 @@ export default class MetaRegistry extends MetadataRegistryLookup {
     }
   }
 
-  private typeDefForClikeEnum (def: MetaTypeDefClikeEnum): Pick<TypeDef, never> {
+  private _typeDefForClikeEnum (def: MetaTypeDefClikeEnum): Pick<TypeDef, never> {
     return {
       info: TypeDefInfo.Enum,
-      sub: def['clike_enum.variants'].map(({ name: nameIndex, discriminant }): TypeDef => ({
+      sub: def['clike_enum.variants'].map(({ discriminant, name: nameIndex }): TypeDef => ({
         ext: { discriminant },
         info: TypeDefInfo.Plain,
-        name: this.stringAt(nameIndex),
+        name: this._stringAt(nameIndex),
         type: 'Null'
       }))
     };
@@ -322,24 +326,24 @@ export default class MetaRegistry extends MetadataRegistryLookup {
     return withTypeString({
       info: TypeDefInfo.Struct,
       ...(structNameIndex
-        ? { name: this.stringAt(structNameIndex) }
+        ? { name: this._stringAt(structNameIndex) }
         : {}
       ),
       sub: structFields.map((field: MetaTypeDefStructField): TypeDef => ({
         ...this.typeDefFromMetaTypeAt(field.type),
-        name: this.stringAt(field.name)
+        name: this._stringAt(field.name)
       }))
     });
   }
 
-  private typeDefForTupleStruct (def: MetaTypeDefTupleStruct | MetaTypeDefEnumVariantTupleStruct): Pick<TypeDef, any> {
+  private _typeDefForTupleStruct (def: MetaTypeDefTupleStruct | MetaTypeDefEnumVariantTupleStruct): Pick<TypeDef, any> {
     const tupleStructTypes = (def as MetaTypeDefTupleStruct)['tuple_struct.types'] || (def as MetaTypeDefEnumVariantTupleStruct)['tuple_struct_variant.types'];
     const tupleStructNameIndex = (def as MetaTypeDefEnumVariantTupleStruct)['tuple_struct_variant.name'];
 
     return withTypeString({
       info: TypeDefInfo.Tuple,
       ...(tupleStructNameIndex
-        ? { name: this.stringAt(tupleStructNameIndex) }
+        ? { name: this._stringAt(tupleStructNameIndex) }
         : {}
       ),
       sub: tupleStructTypes.map((fieldIndex: number, index: number): TypeDef => ({

packages/api-contract/src/base/Blueprint.ts

@@ -49,16 +49,16 @@ export default class Blueprint<ApiType extends ApiTypes> extends BaseWithTx<ApiT
     return {
       signAndSend: this.decorateMethod(
         (account: IKeyringPair | string | AccountId | Address): BlueprintCreateResultSubscription<ApiType> => {
-          return this.apiContracts
+          return this._apiContracts
             .create(endowment, maxGas, this.codeHash, this.abi.constructors[constructorIndex](...params))
             .signAndSend(account)
-            .pipe(map(this.createResult));
+            .pipe(map(this._createResult));
         }
       )
     };
   }
 
-  private createResult = (result: SubmittableResult): BlueprintCreateResult<ApiType> => {
+  private _createResult = (result: SubmittableResult): BlueprintCreateResult<ApiType> => {
     let contract: Contract<ApiType> | undefined;
 
     if (result.isInBlock) {

packages/api-contract/src/base/Code.ts

@@ -48,15 +48,15 @@ export default class Code<ApiType extends ApiTypes> extends BaseWithTx<ApiType>
     return {
       signAndSend: this.decorateMethod(
         (account: IKeyringPair | string | AccountId | Address): CodePutCodeResultSubscription<ApiType> =>
-          this.apiContracts
+          this._apiContracts
             .putCode(maxGas, compactAddLength(this.code))
             .signAndSend(account)
-            .pipe(map(this.createResult))
+            .pipe(map(this._createResult))
       )
     };
   }
 
-  private createResult = (result: ISubmittableResult): CodePutCodeResult<ApiType> => {
+  private _createResult = (result: ISubmittableResult): CodePutCodeResult<ApiType> => {
     let blueprint: Blueprint<ApiType> | undefined;
 
     if (result.isInBlock) {

packages/api-contract/src/base/Contract.ts

@@ -33,35 +33,41 @@ export type ContractCallResult<CallType extends ContractCallTypes> = CallType ex
 export default class Contract<ApiType extends ApiTypes> extends BaseWithTxAndRpcCall<ApiType> {
   public readonly address: Address;
 
+  constructor (api: ApiObject<ApiType>, abi: ContractABIPre | Abi, decorateMethod: DecorateMethod<ApiType>, address: string | AccountId | Address) {
+    super(api, abi, decorateMethod);
+
+    this.address = this.registry.createType('Address', address);
+  }
+
   public call (as: 'rpc', message: string, value: BN | number, gasLimit: BN | number, ...params: any[]): ContractCall<ApiType, 'rpc'>;
   public call (as: 'tx', message: string, value: BN | number, gasLimit: BN | number, ...params: any[]): ContractCall<ApiType, 'tx'>;
   public call<CallType extends ContractCallTypes> (as: CallType, message: string, value: BN | number, gasLimit: BN | number, ...params: any[]): ContractCall<ApiType, CallType> {
-    const { fn, def } = this.getMessage(message);
+    const { def, fn } = this.getMessage(message);
 
     return {
       send: this.decorateMethod(
         as === 'rpc' && this.hasRpcContractsCall
           ? (account: IKeyringPair | string | AccountId | Address): ContractCallResult<'rpc'> =>
-            this.rpcContractsCall(
+            this._rpcContractsCall(
               this.registry.createType('ContractCallRequest', {
-                origin: account,
                 dest: this.address.toString(),
-                value,
                 gasLimit,
-                inputData: fn(...params)
+                inputData: fn(...params),
+                origin: account,
+                value
               })
             ).pipe(map((result: ContractExecResult): ContractCallOutcome =>
-              this.createOutcome(result, this.registry.createType('AccountId', account), def, params)
+              this._createOutcome(result, this.registry.createType('AccountId', account), def, params)
             ))
           : (account: IKeyringPair | string | AccountId | Address): ContractCallResult<'tx'> =>
-            this.apiContracts
+            this._apiContracts
               .call(this.address, value, gasLimit, fn(...params))
               .signAndSend(account)
       )
     };
   }
 
-  private createOutcome (result: ContractExecResult, origin: AccountId, message: ContractABIMessage, params: any[]): ContractCallOutcome {
+  private _createOutcome (result: ContractExecResult, origin: AccountId, message: ContractABIMessage, params: any[]): ContractCallOutcome {
     let output: Codec | null = null;
 
     if (result.isSuccess) {
@@ -73,19 +79,13 @@ export default class Contract<ApiType extends ApiTypes> extends BaseWithTxAndRpc
     }
 
     return {
-      time: Date.now(),
+      isSuccess: result.isSuccess,
       message,
       origin,
+      output,
       params,
       result,
-      isSuccess: result.isSuccess,
-      output
+      time: Date.now()
     };
   }
-
-  constructor (api: ApiObject<ApiType>, abi: ContractABIPre | Abi, decorateMethod: DecorateMethod<ApiType>, address: string | AccountId | Address) {
-    super(api, abi, decorateMethod);
-
-    this.address = this.registry.createType('Address', address);
-  }
 }

packages/api-contract/src/base/util.ts

@@ -31,9 +31,9 @@ export abstract class Base<ApiType extends ApiTypes> implements ContractBase<Api
   public get messages (): ContractMessage[] {
     return this.abi.abi.contract.messages.map(
       (def: ContractABIMessage, index): ContractMessage => ({
-        index,
         def,
-        fn: this.abi.messages[def.name] || this.abi.messages[stringCamelCase(name)]
+        fn: this.abi.messages[def.name] || this.abi.messages[stringCamelCase(name)],
+        index
       })
     );
   }
@@ -49,12 +49,13 @@ export abstract class Base<ApiType extends ApiTypes> implements ContractBase<Api
     assert(!!def, `Attempted to access a contract message that does not exist: ${typeof nameOrIndex === 'number' ? `index ${nameOrIndex}` : nameOrIndex}`);
 
     const fn = this.abi.messages[def.name] || this.abi.messages[stringCamelCase(def.name)];
-    return { index, def, fn };
+
+    return { def, fn, index };
   }
 }
 
 export abstract class BaseWithTx<ApiType extends ApiTypes> extends Base<ApiType> {
-  protected get apiContracts (): SubmittableModuleExtrinsics<'rxjs'> {
+  protected get _apiContracts (): SubmittableModuleExtrinsics<'rxjs'> {
     return this.api.rx.tx.contracts;
   }
 
@@ -70,7 +71,7 @@ export abstract class BaseWithTxAndRpcCall<ApiType extends ApiTypes> extends Bas
     return isFunction(this.api.rx.rpc.contracts?.call);
   }
 
-  protected get rpcContractsCall (): DecoratedRpc<'rxjs', RpcInterface>['contracts']['call'] {
+  protected get _rpcContractsCall (): DecoratedRpc<'rxjs', RpcInterface>['contracts']['call'] {
     assert(this.hasRpcContractsCall, 'You need to connect to a node with the contracts.call RPC method.');
 
     return this.api.rx.rpc.contracts.call;

packages/api-derive/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/api-derive",
-  "version": "1.9.1",
+  "version": "1.12.1",
   "description": "Common functions used across Polkadot, derived from RPC calls and storage queries.",
   "main": "index.js",
   "keywords": [
@@ -28,17 +28,17 @@
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/api-derive#readme",
   "dependencies": {
     "@babel/runtime": "^7.9.2",
-    "@polkadot/api": "1.9.1",
-    "@polkadot/rpc-core": "1.9.1",
-    "@polkadot/rpc-provider": "1.9.1",
-    "@polkadot/types": "1.9.1",
-    "@polkadot/util": "^2.7.1",
-    "@polkadot/util-crypto": "^2.7.1",
+    "@polkadot/api": "1.12.1",
+    "@polkadot/rpc-core": "1.12.1",
+    "@polkadot/rpc-provider": "1.12.1",
+    "@polkadot/types": "1.12.1",
+    "@polkadot/util": "^2.9.0-beta.3",
+    "@polkadot/util-crypto": "^2.9.0-beta.3",
     "bn.js": "^5.1.1",
     "memoizee": "^0.4.14",
-    "rxjs": "^6.5.4"
+    "rxjs": "^6.5.5"
   },
   "devDependencies": {
-    "@polkadot/keyring": "^2.7.1"
+    "@polkadot/keyring": "^2.9.0-beta.3"
   }
 }

packages/api-derive/src/accounts/flags.ts

@@ -0,0 +1,69 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { AccountId, Address, Balance } from '@polkadot/types/interfaces';
+import { ITuple } from '@polkadot/types/types';
+import { DeriveAccountFlags } from '../types';
+
+import { Observable, combineLatest, of } from 'rxjs';
+import { map } from 'rxjs/operators';
+import { Vec } from '@polkadot/types';
+
+import { memo } from '../util';
+
+type FlagsIntermediate = [
+  Vec<ITuple<[AccountId, Balance]>> | undefined,
+  AccountId[],
+  AccountId[],
+  AccountId[],
+  AccountId | undefined
+];
+
+function isIncludedFn (accountId: AccountId | Address | string): (_: AccountId | Address |string) => boolean {
+  return function (id: AccountId | Address | string): boolean {
+    return id.toString() === accountId.toString();
+  };
+}
+
+/**
+ * @name info
+ * @description Returns account membership flags
+ */
+export function flags (api: ApiInterfaceRx): (address?: AccountId | Address | string | null) => Observable<DeriveAccountFlags> {
+  return memo((address?: AccountId | Address | string | null): Observable<DeriveAccountFlags> => {
+    const councilSection = api.query.electionsPhragmen ? 'electionsPhragmen' : 'elections';
+
+    return (
+      combineLatest([
+        address && api.query[councilSection]?.members
+          ? api.query[councilSection].members()
+          : of(undefined),
+        address && api.query.council?.members
+          ? api.query.council.members()
+          : of([]),
+        address && api.query.technicalCommittee?.members
+          ? api.query.technicalCommittee.members()
+          : of([]),
+        address && api.query.society?.members
+          ? api.query.society.members()
+          : of([]),
+        address && api.query.sudo?.key
+          ? api.query.sudo.key()
+          : of(undefined)
+      ]) as Observable<FlagsIntermediate>
+    ).pipe(
+      map(([electionsMembers, councilMembers, technicalCommitteeMembers, societyMembers, sudoKey]): DeriveAccountFlags => {
+        const isIncluded = address ? isIncludedFn(address) : (): boolean => false;
+
+        return {
+          isCouncil: (electionsMembers?.map(([id]: ITuple<[AccountId, Balance]>) => id) || councilMembers || []).some(isIncluded),
+          isSociety: (societyMembers || []).some(isIncluded),
+          isSudo: sudoKey?.toString() === address?.toString(),
+          isTechCommittee: (technicalCommitteeMembers || []).some(isIncluded)
+        };
+      })
+    );
+  });
+}

packages/api-derive/src/accounts/idAndIndex.ts

@@ -3,6 +3,7 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 import { AccountId, AccountIndex, Address } from '@polkadot/types/interfaces';
+import { AccountIdAndIndex } from '../types';
 
 import { Observable, of } from 'rxjs';
 import { map } from 'rxjs/operators';
@@ -12,8 +13,6 @@ import { decodeAddress } from '@polkadot/util-crypto';
 
 import { memo } from '../util';
 
-export type AccountIdAndIndex = [AccountId?, AccountIndex?];
-
 function retrieve (api: ApiInterfaceRx, address: Address | AccountId | AccountIndex | string | null | undefined): Observable<AccountIdAndIndex> {
   try {
     // yes, this can fail, don't care too much, catch will catch it

packages/api-derive/src/accounts/index.ts

@@ -2,6 +2,7 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
+export * from './flags';
 export * from './idAndIndex';
 export * from './idToIndex';
 export * from './indexToId';

packages/api-derive/src/accounts/info.ts

@@ -5,11 +5,11 @@
 import { ApiInterfaceRx } from '@polkadot/api/types';
 import { AccountId, AccountIndex, Address, Balance, Registration } from '@polkadot/types/interfaces';
 import { ITuple } from '@polkadot/types/types';
-import { DeriveAccountFlags, DeriveAccountInfo, DeriveAccountRegistration } from '../types';
+import { DeriveAccountInfo, DeriveAccountRegistration } from '../types';
 
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
-import { Bytes, Data, Option, u32, Vec } from '@polkadot/types';
+import { Bytes, Data, Option, u32 } from '@polkadot/types';
 import { u8aToString } from '@polkadot/util';
 
 import { memo } from '../util';
@@ -17,15 +17,9 @@ import { memo } from '../util';
 function dataAsString (data: Data): string | undefined {
   return data.isRaw
     ? u8aToString(data.asRaw.toU8a(true))
-    : data.isSha256
-      ? data.asSha256.toHex()
-      : undefined;
-}
-
-function includedWrapper (accountId?: AccountId): (_: AccountId) => boolean {
-  return function (id: AccountId): boolean {
-    return id.eq(accountId);
-  };
+    : data.isNone
+      ? undefined
+      : data.toHex();
 }
 
 function retrieveNick (api: ApiInterfaceRx, accountId?: AccountId): Observable<string | undefined> {
@@ -59,6 +53,7 @@ function extractIdentity (identityOfOpt?: Option<Registration>, superOf?: [Accou
       : undefined,
     email: dataAsString(info.email),
     image: dataAsString(info.image),
+    judgements,
     legal: dataAsString(info.legal),
     other: info.additional.reduce((other: Record<string, string>, [_key, _value]): Record<string, string> => {
       const key = dataAsString(_key);
@@ -78,8 +73,7 @@ function extractIdentity (identityOfOpt?: Option<Registration>, superOf?: [Accou
       : undefined,
     riot: dataAsString(info.riot),
     twitter: dataAsString(info.twitter),
-    web: dataAsString(info.web),
-    judgements
+    web: dataAsString(info.web)
   };
 }
 
@@ -115,39 +109,6 @@ function retrieveIdentity (api: ApiInterfaceRx, accountId?: AccountId): Observab
   );
 }
 
-function retrieveFlags (api: ApiInterfaceRx, accountId?: AccountId): Observable<DeriveAccountFlags> {
-  const councilSection = api.query.electionsPhragmen ? 'electionsPhragmen' : 'elections';
-
-  return combineLatest([
-    api.query[councilSection]?.members
-      ? api.query[councilSection].members<Vec<ITuple<[AccountId, Balance]>>>()
-      : of(undefined),
-    api.query.council?.members
-      ? api.query.council.members()
-      : of([]),
-    api.query.technicalCommittee?.members
-      ? api.query.technicalCommittee.members()
-      : of([]),
-    api.query.society?.members
-      ? api.query.society.members()
-      : of([]),
-    api.query.sudo?.key
-      ? api.query.sudo.key()
-      : of(undefined)
-  ]).pipe(
-    map(([electionsMembers, councilMembers, technicalCommitteeMembers, societyMembers, sudoKey]): DeriveAccountFlags => {
-      const checkIncluded = includedWrapper(accountId);
-
-      return {
-        isCouncil: (electionsMembers?.map(([id]) => id) || councilMembers || []).some(checkIncluded),
-        isTechCommittee: technicalCommitteeMembers.some(checkIncluded),
-        isSociety: societyMembers.some(checkIncluded),
-        isSudo: !!sudoKey && sudoKey.eq(accountId)
-      };
-    })
-  );
-}
-
 /**
  * @name info
  * @description Returns aux. info with regards to an account, current that includes the accountId, accountIndex and nickname
@@ -155,16 +116,15 @@ function retrieveFlags (api: ApiInterfaceRx, accountId?: AccountId): Observable<
 export function info (api: ApiInterfaceRx): (address?: AccountIndex | AccountId | Address | string | null) => Observable<DeriveAccountInfo> {
   return memo((address?: AccountIndex | AccountId | Address | string | null): Observable<DeriveAccountInfo> =>
     api.derive.accounts.idAndIndex(address).pipe(
-      switchMap(([accountId, accountIndex]): Observable<[Partial<DeriveAccountInfo>, DeriveAccountFlags, DeriveAccountRegistration, string?]> =>
+      switchMap(([accountId, accountIndex]): Observable<[Partial<DeriveAccountInfo>, DeriveAccountRegistration, string?]> =>
         combineLatest([
           of({ accountId, accountIndex }),
-          retrieveFlags(api, accountId),
           retrieveIdentity(api, accountId),
           retrieveNick(api, accountId)
         ])
       ),
-      map(([{ accountId, accountIndex }, flags, identity, nickname]): DeriveAccountInfo => ({
-        accountId, accountIndex, ...flags, identity, nickname
+      map(([{ accountId, accountIndex }, identity, nickname]): DeriveAccountInfo => ({
+        accountId, accountIndex, identity, nickname
       }))
     ));
 }

packages/api-derive/src/accounts/types.ts

@@ -0,0 +1,38 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { AccountId, AccountIndex, RegistrationJudgement } from '@polkadot/types/interfaces';
+
+export type AccountIdAndIndex = [AccountId?, AccountIndex?];
+
+export type AccountIndexes = Record<string, AccountIndex>;
+
+export interface DeriveAccountRegistration {
+  display?: string;
+  displayParent?: string;
+  email?: string;
+  image?: string;
+  legal?: string;
+  other?: Record<string, string>;
+  parent?: AccountId;
+  pgp?: string;
+  riot?: string;
+  twitter?: string;
+  web?: string;
+  judgements: RegistrationJudgement[];
+}
+
+export interface DeriveAccountFlags {
+  isCouncil: boolean;
+  isSociety: boolean;
+  isSudo: boolean;
+  isTechCommittee: boolean;
+}
+
+export interface DeriveAccountInfo {
+  accountId?: AccountId;
+  accountIndex?: AccountIndex;
+  identity: DeriveAccountRegistration;
+  nickname?: string;
+}

packages/api-derive/src/balances/account.ts

@@ -44,7 +44,7 @@ function queryBalancesAccount (api: ApiInterfaceRx, accountId: AccountId): Obser
     [api.query.balances.account, accountId],
     [api.query.system.accountNonce, accountId]
   ]).pipe(
-    map(([{ free, reserved, miscFrozen, feeFrozen }, accountNonce]): Result =>
+    map(([{ feeFrozen, free, miscFrozen, reserved }, accountNonce]): Result =>
       [free, reserved, feeFrozen, miscFrozen, accountNonce]
     )
   );
@@ -54,7 +54,7 @@ function queryCurrent (api: ApiInterfaceRx, accountId: AccountId): Observable<Re
   // AccountInfo is current, support old, eg. Edgeware
   return api.query.system.account<AccountInfo | ITuple<[Index, AccountData]>>(accountId).pipe(
     map((infoOrTuple): Result => {
-      const { free, reserved, miscFrozen, feeFrozen } = (infoOrTuple as AccountInfo).nonce
+      const { feeFrozen, free, miscFrozen, reserved } = (infoOrTuple as AccountInfo).nonce
         ? (infoOrTuple as AccountInfo).data
         : (infoOrTuple as [Index, AccountData])[1];
       const accountNonce = (infoOrTuple as AccountInfo).nonce || (infoOrTuple as [Index, AccountData])[0];

packages/api-derive/src/collective/proposals.ts

@@ -10,6 +10,8 @@ import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { Option } from '@polkadot/types';
 
+import { memo } from '../util';
+
 type Result = [Hash[], Option<Proposal>[], Option<Votes>[]];
 
 function parse ([hashes, proposals, votes]: Result): DeriveCollectiveProposal[] {
@@ -27,7 +29,7 @@ function parse ([hashes, proposals, votes]: Result): DeriveCollectiveProposal[]
 }
 
 export function proposals (api: ApiInterfaceRx, section: 'council' | 'technicalCommittee'): () => Observable<DeriveCollectiveProposal[]> {
-  return (): Observable<DeriveCollectiveProposal[]> =>
+  return memo((): Observable<DeriveCollectiveProposal[]> =>
     api.query[section]?.proposals
       ? api.query[section].proposals().pipe(
         switchMap((hashes: Hash[]): Observable<Result> =>
@@ -41,5 +43,6 @@ export function proposals (api: ApiInterfaceRx, section: 'council' | 'technicalC
         ),
         map(parse)
       )
-      : of([] as DeriveCollectiveProposal[]);
+      : of([] as DeriveCollectiveProposal[])
+  );
 }

packages/api-derive/src/council/votes.ts

@@ -11,6 +11,8 @@ import { Observable, combineLatest } from 'rxjs';
 import { map } from 'rxjs/operators';
 import { Vec } from '@polkadot/types';
 
+import { memo } from '../util';
+
 function retrieveStakeOf (api: ApiInterfaceRx): Observable<[AccountId, Balance][]> {
   const elections = (api.query.electionsPhragmen || api.query.elections);
 
@@ -80,9 +82,9 @@ function retrieveCurrent (api: ApiInterfaceRx): Observable<DeriveCouncilVotes> {
 }
 
 export function votes (api: ApiInterfaceRx): () => Observable<DeriveCouncilVotes> {
-  // since we are using entries, we are not adding a memo
-  return (): Observable<DeriveCouncilVotes> =>
+  return memo((): Observable<DeriveCouncilVotes> =>
     (api.query.electionsPhragmen || api.query.elections).stakeOf
       ? retrievePrev(api)
-      : retrieveCurrent(api);
+      : retrieveCurrent(api)
+  );
 }

packages/api-derive/src/council/votesOf.ts

@@ -8,10 +8,10 @@ import { DeriveCouncilVote } from '../types';
 
 import { Observable } from 'rxjs';
 import { map } from 'rxjs/operators';
+import { memo } from '../util';
 
 export function votesOf (api: ApiInterfaceRx): (accountId: string | Uint8Array | AccountId) => Observable<DeriveCouncilVote> {
-  // no memo, votes actually pulls the entries without a memo (using entries)
-  return (accountId: string | Uint8Array | AccountId): Observable<DeriveCouncilVote> =>
+  return memo((accountId: string | Uint8Array | AccountId): Observable<DeriveCouncilVote> =>
     api.derive.council.votes().pipe(
       map((votes): DeriveCouncilVote =>
         (
@@ -19,5 +19,6 @@ export function votesOf (api: ApiInterfaceRx): (accountId: string | Uint8Array |
           [null, { stake: api.registry.createType('Balance'), votes: [] as AccountId[] }]
         )[1]
       )
-    );
+    )
+  );
 }

packages/api-derive/src/democracy/_referendumInfo.ts

@@ -1,32 +0,0 @@
-// Copyright 2017-2020 @polkadot/api-derive authors & contributors
-// This software may be modified and distributed under the terms
-// of the Apache-2.0 license. See the LICENSE file for details.
-
-import { ApiInterfaceRx } from '@polkadot/api/types';
-import { ReferendumInfo, ReferendumInfoTo239 } from '@polkadot/types/interfaces';
-import { DeriveReferendum } from '../types';
-
-import BN from 'bn.js';
-import { Observable, of } from 'rxjs';
-import { map } from 'rxjs/operators';
-import { Option } from '@polkadot/types';
-
-import { memo } from '../util';
-import { getStatus, parseImage } from './util';
-
-export function _referendumInfo (api: ApiInterfaceRx): (index: BN, info: Option<ReferendumInfo | ReferendumInfoTo239>) => Observable<DeriveReferendum | null> {
-  return memo((index: BN, info: Option<ReferendumInfo | ReferendumInfoTo239>): Observable<DeriveReferendum | null> => {
-    const status = getStatus(info);
-
-    return status
-      ? api.query.democracy.preimages(status.proposalHash).pipe(
-        map((preimage): DeriveReferendum => ({
-          index: api.registry.createType('ReferendumIndex', index),
-          image: parseImage(api, preimage),
-          imageHash: status.proposalHash,
-          status
-        }))
-      )
-      : of(null);
-  });
-}

packages/api-derive/src/democracy/_referendumVotes.ts

@@ -1,84 +0,0 @@
-// Copyright 2017-2020 @polkadot/api-derive authors & contributors
-// This software may be modified and distributed under the terms
-// of the Apache-2.0 license. See the LICENSE file for details.
-
-import { ApiInterfaceRx } from '@polkadot/api/types';
-import { AccountId, Vote, Voting, VotingDirectVote } from '@polkadot/types/interfaces';
-import { DeriveBalancesAccount, DeriveReferendum, DeriveReferendumVote, DeriveReferendumVotes } from '../types';
-
-import BN from 'bn.js';
-import { Observable, of, combineLatest } from 'rxjs';
-import { map, switchMap } from 'rxjs/operators';
-import { Vec } from '@polkadot/types';
-
-import { memo } from '../util';
-import { calcVotes } from './util';
-
-function votesPrev (api: ApiInterfaceRx, referendumId: BN): Observable<DeriveReferendumVote[]> {
-  return api.query.democracy.votersFor<Vec<AccountId>>(referendumId).pipe(
-    switchMap((votersFor): Observable<[Vec<AccountId>, Vote[], DeriveBalancesAccount[]]> =>
-      combineLatest([
-        of(votersFor),
-        votersFor.length
-          ? api.query.democracy.voteOf.multi<Vote>(
-            votersFor.map((accountId): [BN | number, AccountId] =>
-              [referendumId, accountId]
-            )
-          )
-          : of([]),
-        api.derive.balances.votingBalances(votersFor)
-      ])
-    ),
-    map(([votersFor, votes, balances]): DeriveReferendumVote[] =>
-      votersFor.map((accountId, index): DeriveReferendumVote => ({
-        accountId,
-        balance: balances[index].votingBalance || api.registry.createType('Balance'),
-        vote: votes[index] || api.registry.createType('Vote')
-      }))
-    )
-  );
-}
-
-function votesCurr (api: ApiInterfaceRx, referendumId: BN): Observable<DeriveReferendumVote[]> {
-  return api.query.democracy.votingOf.entries<Voting>().pipe(
-    map((allVoting): DeriveReferendumVote[] =>
-      allVoting
-        .map(([key, voting]): [AccountId, Voting] => [key.args[0] as AccountId, voting])
-        // FIXME We are ignoring delegated votes
-        .filter(([, voting]) => voting.isDirect)
-        .map(([accountId, voting]): [AccountId, VotingDirectVote[]] => [
-          accountId,
-          voting.asDirect.votes.filter(([idx]) => idx.eq(referendumId))
-        ])
-        .filter(([, directVotes]) => !!directVotes.length)
-        .reduce((result: DeriveReferendumVote[], [accountId, votes]) =>
-          // FIXME We are ignoring split votes
-          votes.reduce((result: DeriveReferendumVote[], [, vote]): DeriveReferendumVote[] => {
-            if (vote.isStandard) {
-              result.push({
-                accountId,
-                ...vote.asStandard
-              });
-            }
-
-            return result;
-          }, result), []
-        )
-    )
-  );
-}
-
-export function _referendumVotes (api: ApiInterfaceRx): (referendum: DeriveReferendum) => Observable<DeriveReferendumVotes> {
-  return memo((referendum: DeriveReferendum): Observable<DeriveReferendumVotes> =>
-    combineLatest([
-      api.derive.democracy.sqrtElectorate(),
-      api.query.democracy.votingOf
-        ? votesCurr(api, referendum.index)
-        : votesPrev(api, referendum.index)
-    ]).pipe(
-      map(([sqrtElectorate, votes]) =>
-        calcVotes(sqrtElectorate, referendum, votes)
-      )
-    )
-  );
-}

packages/api-derive/src/democracy/_referendumsVotes.ts

@@ -1,22 +0,0 @@
-// Copyright 2017-2020 @polkadot/api-derive authors & contributors
-// This software may be modified and distributed under the terms
-// of the Apache-2.0 license. See the LICENSE file for details.
-
-import { ApiInterfaceRx } from '@polkadot/api/types';
-import { DeriveReferendum, DeriveReferendumVotes } from '../types';
-
-import { Observable, of, combineLatest } from 'rxjs';
-
-import { memo } from '../util';
-
-export function _referendumsVotes (api: ApiInterfaceRx): (referendums: DeriveReferendum[]) => Observable<DeriveReferendumVotes[]> {
-  return memo((referendums: DeriveReferendum[]): Observable<DeriveReferendumVotes[]> =>
-    referendums.length
-      ? combineLatest(
-        referendums.map((referendum): Observable<DeriveReferendumVotes> =>
-          api.derive.democracy._referendumVotes(referendum)
-        )
-      )
-      : of([])
-  );
-}

packages/api-derive/src/democracy/dispatchQueue.ts

@@ -2,37 +2,104 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
-import { BlockNumber, Hash, ReferendumIndex } from '@polkadot/types/interfaces';
+import { BlockNumber, Hash, ReferendumIndex, Scheduled } from '@polkadot/types/interfaces';
 import { ITuple } from '@polkadot/types/types';
-import { DeriveDispatch } from '../types';
+import { DeriveDispatch, DeriveProposalImage } from '../types';
 
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { Vec } from '@polkadot/types';
+import { Option, Vec, u64 } from '@polkadot/types';
+import { stringToHex } from '@polkadot/util';
 
 import { memo } from '../util';
 
-export function dispatchQueue (api: ApiInterfaceRx): () => Observable<DeriveDispatch[]> {
-  return memo((): Observable<DeriveDispatch[]> =>
-    api.query.democracy?.dispatchQueue
-      ? api.query.democracy.dispatchQueue<Vec<ITuple<[BlockNumber, Hash, ReferendumIndex]>>>().pipe(
-        switchMap((dispatches) =>
-          combineLatest([
-            of(dispatches),
-            api.derive.democracy.preimages(
-              dispatches.map(([, hash]) => hash))
-          ])
-        ),
-        map(([dispatches, images]) =>
-          dispatches.map(([at, imageHash, index], dispatchIndex): DeriveDispatch => ({
-            at,
-            index,
-            image: images[dispatchIndex],
-            imageHash
-          }))
-        )
-      )
-      : of([])
+const DEMOCRACY_ID = stringToHex('democrac');
+
+interface SchedulerInfo {
+  at: BlockNumber;
+  imageHash: Hash;
+  index: ReferendumIndex;
+}
+
+function queryQueue (api: ApiInterfaceRx): Observable<DeriveDispatch[]> {
+  return api.query.democracy.dispatchQueue<Vec<ITuple<[BlockNumber, Hash, ReferendumIndex]>>>().pipe(
+    switchMap((dispatches) =>
+      combineLatest([
+        of(dispatches),
+        api.derive.democracy.preimages(
+          dispatches.map(([, hash]) => hash))
+      ])
+    ),
+    map(([dispatches, images]) =>
+      dispatches.map(([at, imageHash, index], dispatchIndex): DeriveDispatch => ({
+        at,
+        image: images[dispatchIndex],
+        imageHash,
+        index
+      }))
+    )
+  );
+}
+
+function schedulerEntries (api: ApiInterfaceRx): Observable<[BlockNumber[], Option<Scheduled>[][]]> {
+  // We don't get entries, but rather we get the keys (triggered via finished referendums) and
+  // the subscribe to those keys - this means we pickup when the schedulers actually executes
+  // at a block, the entry for that block will become empty
+  return api.derive.democracy.referendumsFinished().pipe(
+    switchMap(() =>
+      api.query.scheduler.agenda.keys()
+    ),
+    switchMap((keys) => {
+      const blockNumbers = keys.map((key) => key.args[0] as BlockNumber);
+
+      return combineLatest([
+        of(blockNumbers),
+        api.query.scheduler.agenda.multi<Vec<Option<Scheduled>>>(blockNumbers)
+      ]);
+    })
+  );
+}
+
+function queryScheduler (api: ApiInterfaceRx): Observable<DeriveDispatch[]> {
+  return schedulerEntries(api).pipe(
+    switchMap(([blockNumbers, agendas]): Observable<[SchedulerInfo[], (DeriveProposalImage | undefined)[]]> => {
+      const result: SchedulerInfo[] = [];
+
+      blockNumbers.forEach((at, index): void => {
+        agendas[index].filter((optScheduled) => optScheduled.isSome).forEach((optScheduled): void => {
+          const scheduled = optScheduled.unwrap();
+
+          if (scheduled.maybeId.isSome) {
+            const id = scheduled.maybeId.unwrap().toHex();
+
+            if (id.startsWith(DEMOCRACY_ID)) {
+              const [, index] = api.registry.createType('(u64, ReferendumIndex)' as any, id) as ITuple<[u64, ReferendumIndex]>;
+              const imageHash = scheduled.call.args[0] as Hash;
+
+              result.push({ at, imageHash, index });
+            }
+          }
+        });
+      });
+
+      return combineLatest([
+        of(result),
+        api.derive.democracy.preimages(result.map(({ imageHash }) => imageHash))
+      ]);
+    }),
+    map(([infos, images]): DeriveDispatch[] =>
+      infos.map((info, index) => ({ ...info, image: images[index] }))
+    )
+  );
+}
+
+export function dispatchQueue (api: ApiInterfaceRx): () => Observable<DeriveDispatch[]> {
+  return memo((): Observable<DeriveDispatch[]> =>
+    api.query.scheduler?.agenda
+      ? queryScheduler(api)
+      : api.query.democracy.dispatchQueue
+        ? queryQueue(api)
+        : of([])
   );
 }

packages/api-derive/src/democracy/index.ts

@@ -2,15 +2,14 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
-export * from './_referendumsVotes';
-export * from './_referendumVotes';
-export * from './_referendumInfo';
 export * from './dispatchQueue';
 export * from './nextExternal';
 export * from './preimage';
 export * from './preimages';
 export * from './proposals';
+export * from './referendumIds';
 export * from './referendums';
 export * from './referendumsActive';
+export * from './referendumsFinished';
 export * from './referendumsInfo';
 export * from './sqrtElectorate';

packages/api-derive/src/democracy/preimage.ts

@@ -17,8 +17,8 @@ import { parseImage } from './util';
 type PreImage = Option<ITuple<[Bytes, AccountId, Balance, BlockNumber]>>;
 
 export function preimage (api: ApiInterfaceRx): (hash: Hash) => Observable<DeriveProposalImage | undefined> {
-  return memo((hashes: Hash[]): Observable<DeriveProposalImage | undefined> =>
-    api.query.democracy.preimages<PreImage>(hashes).pipe(
+  return memo((hash: Hash): Observable<DeriveProposalImage | undefined> =>
+    api.query.democracy.preimages<PreImage>(hash).pipe(
       map((imageOpt) => parseImage(api, imageOpt))
     )
   );

packages/api-derive/src/democracy/preimages.ts

@@ -20,9 +20,7 @@ export function preimages (api: ApiInterfaceRx): (hashes: Hash[]) => Observable<
   return memo((hashes: Hash[]): Observable<(DeriveProposalImage | undefined)[]> =>
     api.query.democracy.preimages.multi<PreImage>(hashes).pipe(
       map((images): (DeriveProposalImage | undefined)[] =>
-        images.map((imageOpt) =>
-          parseImage(api, imageOpt)
-        )
+        images.map((imageOpt) => parseImage(api, imageOpt))
       )
     )
   );

packages/api-derive/src/democracy/proposals.ts

@@ -26,9 +26,9 @@ function parse ([proposals, images, depositors]: [Proposals, (DeriveProposalImag
 
       return {
         balance,
-        index,
         image: images[proposalIndex],
         imageHash,
+        index,
         proposer,
         seconds
       };

packages/api-derive/src/democracy/referendumIds.ts

@@ -0,0 +1,29 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { ReferendumIndex } from '@polkadot/types/interfaces';
+
+import BN from 'bn.js';
+import { Observable, of } from 'rxjs';
+import { map } from 'rxjs/operators';
+
+import { memo } from '../util';
+
+export function referendumIds (api: ApiInterfaceRx): () => Observable<BN[]> {
+  return memo((): Observable<BN[]> =>
+    api.query.democracy?.lowestUnbaked
+      ? api.queryMulti<[ReferendumIndex, ReferendumIndex]>([
+        api.query.democracy.lowestUnbaked,
+        api.query.democracy.referendumCount
+      ]).pipe(
+        map(([first, total]): BN[] =>
+          total.gt(first)
+            ? [...Array(total.sub(first).toNumber())].map((_, i): BN => first.addn(i))
+            : []
+        )
+      )
+      : of([])
+  );
+}

packages/api-derive/src/democracy/referendumsActive.ts

@@ -3,10 +3,8 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { ReferendumIndex } from '@polkadot/types/interfaces';
 import { DeriveReferendum } from '../types';
 
-import BN from 'bn.js';
 import { Observable, of } from 'rxjs';
 import { switchMap } from 'rxjs/operators';
 
@@ -14,19 +12,12 @@ import { memo } from '../util';
 
 export function referendumsActive (api: ApiInterfaceRx): () => Observable<DeriveReferendum[]> {
   return memo((): Observable<DeriveReferendum[]> =>
-    api.query.democracy?.lowestUnbaked
-      ? api.queryMulti<[ReferendumIndex, ReferendumIndex]>([
-        api.query.democracy.lowestUnbaked,
-        api.query.democracy.referendumCount
-      ]).pipe(
-        switchMap(([first, total]): Observable<DeriveReferendum[]> =>
-          total.gt(first)
-            ? api.derive.democracy.referendumsInfo(
-              [...Array(total.sub(first).toNumber())].map((_, i): BN => first.addn(i))
-            )
-            : of([])
-        )
+    api.derive.democracy.referendumIds().pipe(
+      switchMap((ids): Observable<DeriveReferendum[]> =>
+        ids.length
+          ? api.derive.democracy.referendumsInfo(ids)
+          : of([])
       )
-      : of([])
+    )
   );
 }

packages/api-derive/src/democracy/referendumsFinished.ts

@@ -0,0 +1,27 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { ReferendumInfo, ReferendumInfoFinished } from '@polkadot/types/interfaces';
+
+import { Observable } from 'rxjs';
+import { map, switchMap } from 'rxjs/operators';
+import { Option } from '@polkadot/types';
+
+import { memo } from '../util';
+
+export function referendumsFinished (api: ApiInterfaceRx): () => Observable<ReferendumInfoFinished[]> {
+  return memo((): Observable<ReferendumInfoFinished[]> =>
+    api.derive.democracy.referendumIds().pipe(
+      switchMap((ids) => api.query.democracy.referendumInfoOf.multi<Option<ReferendumInfo>>(ids)),
+      map((infos): ReferendumInfoFinished[] =>
+        infos
+          .filter((optInfo) => optInfo.isSome)
+          .map((optInfo) => optInfo.unwrap())
+          .filter((info) => info.isFinished)
+          .map((info) => info.asFinished)
+      )
+    )
+  );
+}

packages/api-derive/src/democracy/referendumsInfo.ts

@@ -3,15 +3,137 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { ReferendumInfo } from '@polkadot/types/interfaces';
-import { DeriveReferendum } from '../types';
+import { AccountId, ReferendumInfo, ReferendumInfoTo239, Vote, Voting, VotingDirectVote, VotingDelegating } from '@polkadot/types/interfaces';
+import { DeriveBalancesAccount, DeriveReferendum, DeriveReferendumVote, DeriveReferendumVotes } from '../types';
 
 import BN from 'bn.js';
 import { Observable, of, combineLatest } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
-import { Option } from '@polkadot/types';
+import { Option, Vec } from '@polkadot/types';
 
 import { memo } from '../util';
+import { calcVotes, getStatus, parseImage } from './util';
+
+function votesPrev (api: ApiInterfaceRx, referendumId: BN): Observable<DeriveReferendumVote[]> {
+  return api.query.democracy.votersFor<Vec<AccountId>>(referendumId).pipe(
+    switchMap((votersFor): Observable<[Vec<AccountId>, Vote[], DeriveBalancesAccount[]]> =>
+      combineLatest([
+        of(votersFor),
+        votersFor.length
+          ? api.query.democracy.voteOf.multi<Vote>(
+            votersFor.map((accountId): [BN | number, AccountId] =>
+              [referendumId, accountId]
+            )
+          )
+          : of([]),
+        api.derive.balances.votingBalances(votersFor)
+      ])
+    ),
+    map(([votersFor, votes, balances]): DeriveReferendumVote[] =>
+      votersFor.map((accountId, index): DeriveReferendumVote => ({
+        accountId,
+        balance: balances[index].votingBalance || api.registry.createType('Balance'),
+        isDelegating: false,
+        vote: votes[index] || api.registry.createType('Vote')
+      }))
+    )
+  );
+}
+
+function votesCurr (api: ApiInterfaceRx, referendumId: BN): Observable<DeriveReferendumVote[]> {
+  return api.query.democracy.votingOf.entries<Voting>().pipe(
+    map((allVoting): DeriveReferendumVote[] => {
+      const mapped = allVoting.map(([key, voting]): [AccountId, Voting] => [key.args[0] as AccountId, voting]);
+      const votes = mapped
+        .filter(([, voting]) => voting.isDirect)
+        .map(([accountId, voting]): [AccountId, VotingDirectVote[]] => [
+          accountId,
+          voting.asDirect.votes.filter(([idx]) => idx.eq(referendumId))
+        ])
+        .filter(([, directVotes]) => !!directVotes.length)
+        .reduce((result: DeriveReferendumVote[], [accountId, votes]) =>
+          // FIXME We are ignoring split votes
+          votes.reduce((result: DeriveReferendumVote[], [, vote]): DeriveReferendumVote[] => {
+            if (vote.isStandard) {
+              result.push({
+                accountId,
+                isDelegating: false,
+                ...vote.asStandard
+              });
+            }
+
+            return result;
+          }, result), []
+        );
+      const delegations = mapped
+        .filter(([, voting]) => voting.isDelegating)
+        .map(([accountId, voting]): [AccountId, VotingDelegating] => [accountId, voting.asDelegating]);
+
+      // add delegations
+      delegations.forEach(([accountId, { balance, conviction, target }]): void => {
+        // Are we delegating to a delegator
+        const toDelegator = delegations.find(([accountId]) => accountId.eq(target));
+        const to = votes.find(({ accountId }) => accountId.eq(toDelegator ? toDelegator[0] : target));
+
+        // this delegation has a target
+        if (to) {
+          votes.push({
+            accountId,
+            balance,
+            isDelegating: true,
+            vote: api.registry.createType('Vote', { aye: to.vote.isAye, conviction })
+          });
+        }
+      });
+
+      return votes;
+    })
+  );
+}
+
+export function _referendumVotes (api: ApiInterfaceRx): (referendum: DeriveReferendum) => Observable<DeriveReferendumVotes> {
+  return memo((referendum: DeriveReferendum): Observable<DeriveReferendumVotes> =>
+    combineLatest([
+      api.derive.democracy.sqrtElectorate(),
+      api.query.democracy.votingOf
+        ? votesCurr(api, referendum.index)
+        : votesPrev(api, referendum.index)
+    ]).pipe(
+      map(([sqrtElectorate, votes]) =>
+        calcVotes(sqrtElectorate, referendum, votes)
+      )
+    )
+  );
+}
+
+export function _referendumsVotes (api: ApiInterfaceRx): (referendums: DeriveReferendum[]) => Observable<DeriveReferendumVotes[]> {
+  return memo((referendums: DeriveReferendum[]): Observable<DeriveReferendumVotes[]> =>
+    referendums.length
+      ? combineLatest(
+        referendums.map((referendum): Observable<DeriveReferendumVotes> =>
+          api.derive.democracy._referendumVotes(referendum)
+        )
+      )
+      : of([])
+  );
+}
+
+export function _referendumInfo (api: ApiInterfaceRx): (index: BN, info: Option<ReferendumInfo | ReferendumInfoTo239>) => Observable<DeriveReferendum | null> {
+  return memo((index: BN, info: Option<ReferendumInfo | ReferendumInfoTo239>): Observable<DeriveReferendum | null> => {
+    const status = getStatus(info);
+
+    return status
+      ? api.query.democracy.preimages(status.proposalHash).pipe(
+        map((preimage): DeriveReferendum => ({
+          image: parseImage(api, preimage),
+          imageHash: status.proposalHash,
+          index: api.registry.createType('ReferendumIndex', index),
+          status
+        }))
+      )
+      : of(null);
+  });
+}
 
 export function referendumsInfo (api: ApiInterfaceRx): (ids: BN[]) => Observable<DeriveReferendum[]> {
   return memo((ids: BN[]): Observable<DeriveReferendum[]> =>

packages/api-derive/src/democracy/types.ts

@@ -46,6 +46,7 @@ export interface DeriveReferendum {
 export interface DeriveReferendumVote {
   accountId: AccountId;
   balance: Balance;
+  isDelegating: boolean;
   vote: Vote;
 }
 

packages/api-derive/src/democracy/util.ts

@@ -3,7 +3,7 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { AccountId, Balance, BlockNumber, Proposal, ReferendumInfo, ReferendumInfoTo239, ReferendumStatus, Tally, VoteThreshold } from '@polkadot/types/interfaces';
+import { AccountId, Balance, BlockNumber, PreimageStatus, Proposal, ReferendumInfo, ReferendumInfoTo239, ReferendumStatus, Tally, VoteThreshold } from '@polkadot/types/interfaces';
 import { ITuple } from '@polkadot/types/types';
 import { DeriveProposalImage, DeriveReferendum, DeriveReferendumVote, DeriveReferendumVotes, DeriveReferendumVoteState } from '../types';
 
@@ -11,6 +11,15 @@ import BN from 'bn.js';
 import { Bytes, Option } from '@polkadot/types';
 import { bnSqrt } from '@polkadot/util';
 
+type PreimageInfo = [Bytes, AccountId, Balance, BlockNumber];
+type OldPreimage = ITuple<PreimageInfo>;
+
+interface ApproxState {
+  votedAye: BN;
+  votedNay: BN;
+  votedTotal: BN;
+}
+
 function isOldInfo (info: ReferendumInfo | ReferendumInfoTo239): info is ReferendumInfoTo239 {
   return !!(info as ReferendumInfoTo239).proposalHash;
 }
@@ -19,6 +28,10 @@ function isCurrentStatus (status: ReferendumStatus | ReferendumInfoTo239): statu
   return !!(status as ReferendumStatus).tally;
 }
 
+function isCurrentPreimage (api: ApiInterfaceRx, imageOpt: Option<OldPreimage> | Option<PreimageStatus>): imageOpt is Option<PreimageStatus> {
+  return !!imageOpt && !api.query.democracy.dispatchQueue;
+}
+
 export function compareRationals (n1: BN, d1: BN, n2: BN, d2: BN): boolean {
   while (true) {
     const q1 = n1.div(d1);
@@ -46,7 +59,7 @@ export function compareRationals (n1: BN, d1: BN, n2: BN, d2: BN): boolean {
   }
 }
 
-function isPassing (threshold: VoteThreshold, sqrtElectorate: BN, { votedAye, votedNay, votedTotal }: DeriveReferendumVoteState): boolean {
+export function calcPassing (threshold: VoteThreshold, sqrtElectorate: BN, { votedAye, votedNay, votedTotal }: ApproxState): boolean {
   const sqrtVoters = bnSqrt(votedTotal);
 
   return sqrtVoters.isZero()
@@ -123,7 +136,7 @@ export function calcVotes (sqrtElectorate: BN, referendum: DeriveReferendum, vot
 
   return {
     ...state,
-    isPassing: isPassing(referendum.status.threshold, sqrtElectorate, state),
+    isPassing: calcPassing(referendum.status.threshold, sqrtElectorate, state),
     votes
   };
 }
@@ -145,13 +158,8 @@ export function getStatus (info: Option<ReferendumInfo | ReferendumInfoTo239>):
   return null;
 }
 
-export function parseImage (api: ApiInterfaceRx, imageOpt: Option<ITuple<[Bytes, AccountId, Balance, BlockNumber]>>): DeriveProposalImage | undefined {
-  if (imageOpt.isNone) {
-    return;
-  }
-
+function constructProposal (api: ApiInterfaceRx, [bytes, proposer, balance, at]: PreimageInfo): DeriveProposalImage {
   let proposal: Proposal | undefined;
-  const [bytes, proposer, balance, at] = imageOpt.unwrap();
 
   try {
     proposal = api.registry.createType('Proposal', bytes.toU8a(true));
@@ -161,3 +169,23 @@ export function parseImage (api: ApiInterfaceRx, imageOpt: Option<ITuple<[Bytes,
 
   return { at, balance, proposal, proposer };
 }
+
+export function parseImage (api: ApiInterfaceRx, imageOpt: Option<OldPreimage> | Option<PreimageStatus>): DeriveProposalImage | undefined {
+  if (imageOpt.isNone) {
+    return;
+  }
+
+  if (isCurrentPreimage(api, imageOpt)) {
+    const status = imageOpt.unwrap();
+
+    if (status.isMissing) {
+      return;
+    }
+
+    const { data, deposit, provider, since } = status.asAvailable;
+
+    return constructProposal(api, [data, provider, deposit, since]);
+  }
+
+  return constructProposal(api, imageOpt.unwrap());
+}

packages/api-derive/src/elections/info.ts

@@ -27,9 +27,9 @@ function queryElections (api: ApiInterfaceRx): Observable<DeriveElectionsInfo> {
     api.query[section].runnersUp
   ]).pipe(
     map(([councilMembers, candidates, members, runnersUp]): DeriveElectionsInfo => ({
-      candidates,
-      candidateCount: api.registry.createType('u32', candidates.length),
       candidacyBond: api.consts[section].candidacyBond as Balance,
+      candidateCount: api.registry.createType('u32', candidates.length),
+      candidates,
       desiredSeats: api.consts[section].desiredMembers as u32,
       members: members.length
         ? members.sort(sortAccounts)

packages/api-derive/src/index.ts

@@ -77,6 +77,7 @@ function injectFunctions<AllSections> (api: ApiInterfaceRx, allSections: AllSect
           const methodName = _methodName as keyof typeof section;
           // Not sure what to do here, casting as any. Though the final types are good
           const method = (section[methodName] as any)(api);
+
           // idem
           (sectionAcc as any)[methodName] = method;
 

packages/api-derive/src/parachains/info.ts

@@ -44,6 +44,7 @@ function parseActive (id: ParaId, active: Active): DeriveParachainActive | null
       )
     };
   }
+
   return null;
 }
 

packages/api-derive/src/society/candidates.ts

@@ -30,7 +30,7 @@ export function candidates (api: ApiInterfaceRx): () => Observable<DeriveSociety
         ])
       ),
       map(([candidates, suspended]: Result): DeriveSocietyCandidate[] =>
-        candidates.map(({ who, kind, value }, index) => ({
+        candidates.map(({ kind, value, who }, index) => ({
           accountId: who,
           isSuspended: suspended[index].isSome,
           kind,

packages/api-derive/src/society/info.ts

@@ -30,9 +30,9 @@ export function info (api: ApiInterfaceRx): () => Observable<DeriveSociety> {
       map(([bids, defender, founder, head, maxMembers, pot]: Result): DeriveSociety => ({
         bids,
         defender: defender.unwrapOr(undefined),
+        founder: founder.unwrapOr(undefined),
         hasDefender: (defender.isSome && head.isSome && !head.eq(defender)) || false,
         head: head.unwrapOr(undefined),
-        founder: founder.unwrapOr(undefined),
         maxMembers,
         pot
       }))

packages/api-derive/src/society/member.ts

@@ -29,8 +29,8 @@ export function member (api: ApiInterfaceRx): (accountId: AccountId) => Observab
     ]).pipe(
       map(([payouts, strikes, defenderVotes, suspended, vouching]: Result): DeriveSocietyMember => ({
         accountId,
-        payouts,
         isSuspended: suspended.isTrue,
+        payouts,
         strikes,
         vote: defenderVotes.unwrapOr(undefined),
         vouching: vouching.unwrapOr(undefined)

packages/api-derive/src/staking/account.ts

@@ -8,7 +8,7 @@ import { DeriveSessionInfo, DeriveStakingAccount, DeriveStakingQuery, DeriveUnlo
 
 import BN from 'bn.js';
 import { Observable, combineLatest } from 'rxjs';
-import { map } from 'rxjs/operators';
+import { map, switchMap } from 'rxjs/operators';
 
 import { isUndefined } from '@polkadot/util';
 
@@ -70,16 +70,31 @@ function parseResult (api: ApiInterfaceRx, sessionInfo: DeriveSessionInfo, query
   };
 }
 
+export function _account (api: ApiInterfaceRx): (sessionInfo: DeriveSessionInfo, accountId: Uint8Array | string) => Observable<DeriveStakingAccount> {
+  return memo((sessionInfo: DeriveSessionInfo, accountId: Uint8Array | string): Observable<DeriveStakingAccount> =>
+    api.derive.staking.query(accountId).pipe(
+      map((query) => parseResult(api, sessionInfo, query))
+    ));
+}
+
 /**
  * @description From a stash, retrieve the controllerId and fill in all the relevant staking details
  */
 export function account (api: ApiInterfaceRx): (accountId: Uint8Array | string) => Observable<DeriveStakingAccount> {
   return memo((accountId: Uint8Array | string): Observable<DeriveStakingAccount> =>
-    combineLatest([
-      api.derive.session.info(),
-      api.derive.staking.query(accountId)
-    ]).pipe(
-      map(([sessionInfo, query]: [DeriveSessionInfo, DeriveStakingQuery]) =>
-        parseResult(api, sessionInfo, query))
+    api.derive.session.info().pipe(
+      switchMap((sessionInfo) => api.derive.staking._account(sessionInfo, accountId))
+    ));
+}
+
+/**
+ * @description From a list of stashes, fill in all the relevant staking details
+ */
+export function accounts (api: ApiInterfaceRx): (accountIds: (Uint8Array | string)[]) => Observable<DeriveStakingAccount[]> {
+  return memo((accountIds: (Uint8Array | string)[]): Observable<DeriveStakingAccount[]> =>
+    api.derive.session.info().pipe(
+      switchMap((sessionInfo) =>
+        combineLatest(accountIds.map((accountId) => api.derive.staking._account(sessionInfo, accountId)))
+      )
     ));
 }

packages/api-derive/src/staking/currentPoints.ts

@@ -0,0 +1,42 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { AccountId, EraPoints, EraRewardPoints, RewardPoint } from '@polkadot/types/interfaces';
+import { ApiInterfaceRx } from '@polkadot/api/types';
+
+import { Observable, of } from 'rxjs';
+import { map, switchMap } from 'rxjs/operators';
+
+import { memo } from '../util';
+
+function retrievePointsPrev (api: ApiInterfaceRx, currentElected: AccountId[]): Observable<EraRewardPoints> {
+  return api.query.staking.currentEraPointsEarned<EraPoints>().pipe(
+    map(({ individual, total }): EraRewardPoints =>
+      api.registry.createType('EraRewardPoints', {
+        individual: new Map<AccountId, RewardPoint>(
+          individual
+            .map((points) => api.registry.createType('RewardPoint', points))
+            .map((points, index): [AccountId, RewardPoint] => [currentElected[index], points])
+        ),
+        total
+      })
+    )
+  );
+}
+
+/**
+ * @description Retrieve the staking overview, including elected and points earned
+ */
+export function currentPoints (api: ApiInterfaceRx): () => Observable<EraRewardPoints> {
+  return memo((): Observable<EraRewardPoints> =>
+    api.derive.staking.overview().pipe(
+      switchMap(({ activeEra, nextElected }) =>
+        api.query.staking.erasRewardPoints
+          ? api.query.staking.erasRewardPoints<EraRewardPoints>(activeEra)
+          : api.query.staking.currentEraPointsEarned
+            ? retrievePointsPrev(api, nextElected)
+            : of(api.registry.createType('EraRewardPoints'))
+      )
+    ));
+}

packages/api-derive/src/staking/electedInfo.ts

@@ -18,9 +18,7 @@ export function electedInfo (api: ApiInterfaceRx): () => Observable<DeriveStakin
         combineLatest([
           of(nextElected),
           combineLatest(
-            nextElected.map((accountId): Observable<DeriveStakingQuery> =>
-              api.derive.staking.query(accountId)
-            )
+            nextElected.map((accountId) => api.derive.staking.query(accountId))
           )
         ])
       ),

packages/api-derive/src/staking/erasExposure.ts

@@ -6,7 +6,6 @@ import { ApiInterfaceRx } from '@polkadot/api/types';
 import { EraIndex, Exposure } from '@polkadot/types/interfaces';
 import { DeriveEraExposure, DeriveEraNominatorExposure, DeriveEraValidatorExposure } from '../types';
 
-import BN from 'bn.js';
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { StorageKey } from '@polkadot/types';
@@ -24,11 +23,11 @@ function mapStakers (era: EraIndex, stakers: KeysAndExposures): DeriveEraExposur
 
     validators[validatorId] = exposure;
 
-    exposure.others.forEach(({ who }, index): void => {
+    exposure.others.forEach(({ who }, validatorIndex): void => {
       const nominatorId = who.toString();
 
       nominators[nominatorId] = nominators[nominatorId] || [];
-      nominators[nominatorId].push([validatorId, index]);
+      nominators[nominatorId].push({ validatorId, validatorIndex });
     });
   });
 
@@ -43,16 +42,18 @@ export function eraExposure (api: ApiInterfaceRx): (era: EraIndex) => Observable
   );
 }
 
-export function erasExposure (api: ApiInterfaceRx): (withActive?: boolean | BN | number) => Observable<DeriveEraExposure[]> {
-  return memo((withActive?: boolean | BN | number): Observable<DeriveEraExposure[]> =>
+export function _erasExposure (api: ApiInterfaceRx): (eras: EraIndex[]) => Observable<DeriveEraExposure[]> {
+  return memo((eras: EraIndex[]): Observable<DeriveEraExposure[]> =>
+    eras.length
+      ? combineLatest(eras.map((era) => api.derive.staking.eraExposure(era)))
+      : of([])
+  );
+}
+
+export function erasExposure (api: ApiInterfaceRx): (withActive?: boolean) => Observable<DeriveEraExposure[]> {
+  return memo((withActive?: boolean): Observable<DeriveEraExposure[]> =>
     api.derive.staking.erasHistoric(withActive).pipe(
-      switchMap((eras): Observable<DeriveEraExposure[]> =>
-        eras.length
-          ? combineLatest(
-            eras.map((era) => api.derive.staking.eraExposure(era))
-          )
-          : of([])
-      )
+      switchMap((eras) => api.derive.staking._erasExposure(eras))
     )
   );
 }

packages/api-derive/src/staking/erasHistoric.ts

@@ -9,45 +9,32 @@ import BN from 'bn.js';
 import { Observable, of } from 'rxjs';
 import { map } from 'rxjs/operators';
 import { Option, u32 } from '@polkadot/types';
-import { isBoolean, isUndefined, bnToBn } from '@polkadot/util';
 
 import { memo } from '../util';
 
-export function erasHistoric (api: ApiInterfaceRx): (withActive?: boolean | BN | number) => Observable<EraIndex[]> {
-  return memo((withActive?: boolean | BN | number): Observable<EraIndex[]> =>
+export function erasHistoric (api: ApiInterfaceRx): (withActive?: boolean) => Observable<EraIndex[]> {
+  return memo((withActive?: boolean): Observable<EraIndex[]> =>
     api.query.staking?.activeEra
       ? api.queryMulti<[Option<ActiveEraInfo>, u32]>([
         api.query.staking.activeEra,
         api.query.staking.historyDepth
       ]).pipe(
         map(([activeEraOpt, historyDepth]): EraIndex[] => {
-          const result: (EraIndex | null)[] = [];
+          const result: EraIndex[] = [];
           const max = historyDepth.toNumber();
-          const activeEra = activeEraOpt.unwrapOrDefault().index.toBn();
+          const activeEra: BN = activeEraOpt.unwrapOrDefault().index;
           let lastEra = activeEra;
 
           while (lastEra.gten(0) && (result.length < max)) {
-            result.push(
-              ((lastEra !== activeEra) || (withActive === true))
-                ? api.registry.createType('EraIndex', lastEra)
-                : null
-            );
+            if ((lastEra !== activeEra) || (withActive === true)) {
+              result.push(api.registry.createType('EraIndex', lastEra));
+            }
 
             lastEra = lastEra.subn(1);
           }
 
-          const startEra = isUndefined(withActive) || isBoolean(withActive)
-            ? new BN(0)
-            : bnToBn(withActive);
-
           // go from oldest to newest
-          return result
-            .filter((era): era is EraIndex =>
-              era
-                ? era.gte(startEra)
-                : false
-            )
-            .reverse();
+          return result.reverse();
         })
       )
       : of([])

packages/api-derive/src/staking/erasPoints.ts

@@ -6,8 +6,7 @@ import { ApiInterfaceRx } from '@polkadot/api/types';
 import { EraIndex, EraRewardPoints } from '@polkadot/types/interfaces';
 import { DeriveEraPoints, DeriveEraValPoints } from '../types';
 
-import BN from 'bn.js';
-import { Observable, combineLatest, of } from 'rxjs';
+import { Observable, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 
 import { memo } from '../util';
@@ -22,24 +21,26 @@ function mapValidators ({ individual }: EraRewardPoints): DeriveEraValPoints {
     }, {});
 }
 
-export function erasPoints (api: ApiInterfaceRx): (withActive?: boolean | BN | number) => Observable<DeriveEraPoints[]> {
-  return memo((withActive?: boolean | number): Observable<DeriveEraPoints[]> =>
-    api.derive.staking.erasHistoric(withActive).pipe(
-      switchMap((eras): Observable<[EraIndex[], EraRewardPoints[]]> =>
-        combineLatest([
-          of(eras),
-          eras.length
-            ? api.query.staking.erasRewardPoints.multi<EraRewardPoints>(eras)
-            : of([])
-        ])
-      ),
-      map(([eras, points]): DeriveEraPoints[] =>
-        eras.map((era, index): DeriveEraPoints => ({
-          era,
-          eraPoints: points[index].total,
-          validators: mapValidators(points[index])
-        }))
+export function _erasPoints (api: ApiInterfaceRx): (eras: EraIndex[]) => Observable<DeriveEraPoints[]> {
+  return memo((eras: EraIndex[]): Observable<DeriveEraPoints[]> =>
+    eras.length
+      ? api.query.staking.erasRewardPoints.multi<EraRewardPoints>(eras).pipe(
+        map((points) =>
+          eras.map((era, index): DeriveEraPoints => ({
+            era,
+            eraPoints: points[index].total,
+            validators: mapValidators(points[index])
+          }))
+        )
       )
+      : of([])
+  );
+}
+
+export function erasPoints (api: ApiInterfaceRx): (withActive?: boolean) => Observable<DeriveEraPoints[]> {
+  return memo((withActive?: boolean): Observable<DeriveEraPoints[]> =>
+    api.derive.staking.erasHistoric(withActive).pipe(
+      switchMap((eras) => api.derive.staking._erasPoints(eras))
     )
   );
 }

packages/api-derive/src/staking/erasPrefs.ts

@@ -6,7 +6,6 @@ import { ApiInterfaceRx } from '@polkadot/api/types';
 import { EraIndex, ValidatorPrefs } from '@polkadot/types/interfaces';
 import { DeriveEraPrefs, DeriveEraValPrefs } from '../types';
 
-import BN from 'bn.js';
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { StorageKey } from '@polkadot/types';
@@ -31,16 +30,18 @@ export function eraPrefs (api: ApiInterfaceRx): (era: EraIndex) => Observable<De
   );
 }
 
-export function erasPrefs (api: ApiInterfaceRx): (withActive?: boolean | BN | number) => Observable<DeriveEraPrefs[]> {
-  return memo((withActive?: boolean | BN | number): Observable<DeriveEraPrefs[]> =>
+export function _erasPrefs (api: ApiInterfaceRx): (eras: EraIndex[]) => Observable<DeriveEraPrefs[]> {
+  return memo((eras: EraIndex[]): Observable<DeriveEraPrefs[]> =>
+    eras.length
+      ? combineLatest(eras.map((era) => api.derive.staking.eraPrefs(era)))
+      : of([])
+  );
+}
+
+export function erasPrefs (api: ApiInterfaceRx): (withActive?: boolean) => Observable<DeriveEraPrefs[]> {
+  return memo((withActive?: boolean): Observable<DeriveEraPrefs[]> =>
     api.derive.staking.erasHistoric(withActive).pipe(
-      switchMap((eras): Observable<DeriveEraPrefs[]> =>
-        eras.length
-          ? combineLatest(
-            eras.map((era) => api.derive.staking.eraPrefs(era))
-          )
-          : of([])
-      )
+      switchMap((eras) => api.derive.staking._erasPrefs(eras))
     )
   );
 }

packages/api-derive/src/staking/erasRewards.ts

@@ -6,30 +6,29 @@ import { ApiInterfaceRx } from '@polkadot/api/types';
 import { Balance, EraIndex } from '@polkadot/types/interfaces';
 import { DeriveEraRewards } from '../types';
 
-import BN from 'bn.js';
-import { Observable, combineLatest, of } from 'rxjs';
+import { Observable, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { Option } from '@polkadot/types';
 
 import { memo } from '../util';
 
-export function erasRewards (api: ApiInterfaceRx): (withActive?: boolean | BN | number) => Observable<DeriveEraRewards[]> {
-  return memo((withActive?: boolean | BN | number): Observable<DeriveEraRewards[]> =>
-    api.derive.staking.erasHistoric(withActive).pipe(
-      switchMap((eras): Observable<[EraIndex[], Option<Balance>[]]> =>
-        combineLatest([
-          of(eras),
-          eras.length
-            ? api.query.staking.erasValidatorReward.multi<Option<Balance>>(eras)
-            : of([])
-        ])
-      ),
-      map(([eras, rewards]): DeriveEraRewards[] =>
-        eras.map((era, index): DeriveEraRewards => ({
+export function _erasRewards (api: ApiInterfaceRx): (eras: EraIndex[]) => Observable<DeriveEraRewards[]> {
+  return memo((eras: EraIndex[]): Observable<DeriveEraRewards[]> =>
+    eras.length
+      ? api.query.staking.erasValidatorReward.multi<Option<Balance>>(eras).pipe(
+        map((rewards) => eras.map((era, index): DeriveEraRewards => ({
           era,
           eraReward: rewards[index].unwrapOrDefault()
-        }))
+        })))
       )
+      : of([])
+  );
+}
+
+export function erasRewards (api: ApiInterfaceRx): (withActive?: boolean) => Observable<DeriveEraRewards[]> {
+  return memo((withActive?: boolean): Observable<DeriveEraRewards[]> =>
+    api.derive.staking.erasHistoric(withActive).pipe(
+      switchMap((eras) => api.derive.staking._erasRewards(eras))
     )
   );
 }

packages/api-derive/src/staking/erasSlashes.ts

@@ -7,7 +7,6 @@ import { BalanceOf, EraIndex, Perbill } from '@polkadot/types/interfaces';
 import { ITuple } from '@polkadot/types/types';
 import { DeriveEraSlashes, DeriveEraValSlash } from '../types';
 
-import BN from 'bn.js';
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { Option, StorageKey } from '@polkadot/types';
@@ -40,16 +39,20 @@ export function eraSlashes (api: ApiInterfaceRx): (era: EraIndex) => Observable<
   );
 }
 
-export function erasSlashes (api: ApiInterfaceRx): (withActive?: boolean | BN | number) => Observable<DeriveEraSlashes[]> {
-  return memo((withActive?: boolean | BN | number): Observable<DeriveEraSlashes[]> =>
-    api.derive.staking.erasHistoric(withActive).pipe(
-      switchMap((eras): Observable<DeriveEraSlashes[]> =>
-        eras.length
-          ? combineLatest(
-            eras.map((era) => api.derive.staking.eraSlashes(era))
-          )
-          : of([])
+export function _erasSlashes (api: ApiInterfaceRx): (eras: EraIndex[]) => Observable<DeriveEraSlashes[]> {
+  return memo((eras: EraIndex[]): Observable<DeriveEraSlashes[]> =>
+    eras.length
+      ? combineLatest(
+        eras.map((era) => api.derive.staking.eraSlashes(era))
       )
+      : of([])
+  );
+}
+
+export function erasSlashes (api: ApiInterfaceRx): (withActive?: boolean) => Observable<DeriveEraSlashes[]> {
+  return memo((withActive?: boolean): Observable<DeriveEraSlashes[]> =>
+    api.derive.staking.erasHistoric(withActive).pipe(
+      switchMap((eras) => api.derive.staking._erasSlashes(eras))
     )
   );
 }

packages/api-derive/src/staking/index.ts

@@ -3,6 +3,7 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 export * from './account';
+export * from './currentPoints';
 export * from './erasExposure';
 export * from './erasHistoric';
 export * from './erasPoints';

packages/api-derive/src/staking/overview.ts

@@ -2,30 +2,14 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
-import { AccountId, EraPoints, EraRewardPoints, RewardPoint } from '@polkadot/types/interfaces';
 import { ApiInterfaceRx } from '@polkadot/api/types';
 import { DeriveStakingOverview } from '../types';
 
-import { Observable, combineLatest, of } from 'rxjs';
-import { map, switchMap } from 'rxjs/operators';
+import { Observable, combineLatest } from 'rxjs';
+import { map } from 'rxjs/operators';
 
 import { memo } from '../util';
 
-function retrievePointsPrev (api: ApiInterfaceRx, currentElected: AccountId[]): Observable<EraRewardPoints> {
-  return api.query.staking.currentEraPointsEarned<EraPoints>().pipe(
-    map(({ individual, total }): EraRewardPoints =>
-      api.registry.createType('EraRewardPoints', {
-        total,
-        individual: new Map<AccountId, RewardPoint>(
-          individual
-            .map((points): RewardPoint => api.registry.createType('RewardPoint', points))
-            .map((points, index): [AccountId, RewardPoint] => [currentElected[index], points])
-        )
-      })
-    )
-  );
-}
-
 /**
  * @description Retrieve the staking overview, including elected and points earned
  */
@@ -35,18 +19,10 @@ export function overview (api: ApiInterfaceRx): () => Observable<DeriveStakingOv
       api.derive.session.indexes(),
       api.derive.staking.validators()
     ]).pipe(
-      switchMap(([indexes, { nextElected, validators }]) =>
-        combineLatest([
-          of({ ...indexes, nextElected, validators }),
-          api.query.staking.erasRewardPoints
-            ? api.query.staking.erasRewardPoints<EraRewardPoints>(indexes.activeEra)
-            : api.query.staking.currentEraPointsEarned
-              ? retrievePointsPrev(api, nextElected)
-              : of(api.registry.createType('EraRewardPoints'))
-        ])
-      ),
-      map(([info, eraPoints]): DeriveStakingOverview => ({
-        ...info, eraPoints
+      map(([indexes, { nextElected, validators }]): DeriveStakingOverview => ({
+        ...indexes,
+        nextElected,
+        validators
       }))
     ));
 }

packages/api-derive/src/staking/ownExposure.ts

@@ -5,14 +5,13 @@
 import { ApiInterfaceRx } from '@polkadot/api/types';
 import { DeriveOwnExposure } from '../types';
 
-import BN from 'bn.js';
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 
 import { memo } from '../util';
 
-export function ownExposure (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean | BN | number) => Observable<DeriveOwnExposure[]> {
-  return memo((accountId: Uint8Array | string, withActive?: boolean | BN | number): Observable<DeriveOwnExposure[]> => {
+export function ownExposure (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean) => Observable<DeriveOwnExposure[]> {
+  return memo((accountId: Uint8Array | string, withActive?: boolean): Observable<DeriveOwnExposure[]> => {
     return api.derive.staking.erasHistoric(withActive).pipe(
       switchMap((eras) =>
         eras.length

packages/api-derive/src/staking/ownSlashes.ts

@@ -5,14 +5,13 @@
 import { ApiInterfaceRx } from '@polkadot/api/types';
 import { DeriveStakerSlashes } from '../types';
 
-import BN from 'bn.js';
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 
 import { memo } from '../util';
 
-export function ownSlashes (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean | BN | number) => Observable<DeriveStakerSlashes[]> {
-  return memo((accountId: Uint8Array | string, withActive?: boolean | BN | number): Observable<DeriveStakerSlashes[]> => {
+export function ownSlashes (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean) => Observable<DeriveStakerSlashes[]> {
+  return memo((accountId: Uint8Array | string, withActive?: boolean): Observable<DeriveStakerSlashes[]> => {
     return api.derive.staking.erasHistoric(withActive).pipe(
       switchMap((eras) =>
         eras.length
@@ -30,9 +29,7 @@ export function ownSlashes (api: ApiInterfaceRx): (accountId: Uint8Array | strin
           era,
           total: optVal.isSome
             ? optVal.unwrap()[1]
-            : optNom.isSome
-              ? optNom.unwrap()
-              : api.registry.createType('Balance')
+            : optNom.unwrapOrDefault()
         }))
       )
     );

packages/api-derive/src/staking/query.ts

@@ -55,7 +55,7 @@ function retrieveCurr (api: ApiInterfaceRx, stashId: AccountId): Observable<Mult
   );
 }
 
-function retrieveController (api: ApiInterfaceRx, stashId: AccountId, [queuedKeys, [controllerIdOpt, nominatorsOpt, rewardDestination, validatorPrefs, nextKeys, exposure]]: [Vec<ITuple<[AccountId, Keys]>>, MultiResult]): Observable<DeriveStakingQuery> {
+function retrieveController (api: ApiInterfaceRx, stashId: AccountId, queuedKeys: Vec<ITuple<[AccountId, Keys]>>, [controllerIdOpt, nominatorsOpt, rewardDestination, validatorPrefs, nextKeys, exposure]: MultiResult): Observable<DeriveStakingQuery> {
   const controllerId = controllerIdOpt.unwrapOr(null);
   const nominators = nominatorsOpt.unwrapOr(null);
 
@@ -82,30 +82,41 @@ function retrieveController (api: ApiInterfaceRx, stashId: AccountId, [queuedKey
     : of({ accountId: stashId, nextSessionIds: [], sessionIds: [] });
 }
 
-/**
- * @description From a stash, retrieve the controllerId and all relevant details
- */
-export function query (api: ApiInterfaceRx): (accountId: Uint8Array | string) => Observable<DeriveStakingQuery> {
-  return memo((accountId: Uint8Array | string): Observable<DeriveStakingQuery> => {
+export function queryWithQueued (api: ApiInterfaceRx): (accountId: Uint8Array | string, queuedKeys: Vec<ITuple<[AccountId, Keys]>>) => Observable<DeriveStakingQuery> {
+  return memo((accountId: Uint8Array | string, queuedKeys: Vec<ITuple<[AccountId, Keys]>>): Observable<DeriveStakingQuery> => {
     const stashId = api.registry.createType('AccountId', accountId);
 
-    return combineLatest([
-      api.query.session.queuedKeys<Vec<ITuple<[AccountId, Keys]>>>(),
+    return (
       api.query.staking.erasStakers
         ? retrieveCurr(api, stashId)
         : retrievePrev(api, stashId)
-    ]).pipe(
+    ).pipe(
       switchMap((result): Observable<DeriveStakingQuery> =>
-        retrieveController(api, stashId, result)
+        retrieveController(api, stashId, queuedKeys, result)
       )
     );
   });
 }
 
-export function queryMulti (api: ApiInterfaceRx): (...accountIds: (Uint8Array | string)[]) => Observable<DeriveStakingQuery[]> {
-  return memo((...accountIds: (Uint8Array | string)[]): Observable<DeriveStakingQuery[]> =>
-    combineLatest(
-      accountIds.map((accountId) => api.derive.staking.query(accountId))
+/**
+ * @description From a stash, retrieve the controllerId and all relevant details
+ */
+export function query (api: ApiInterfaceRx): (accountId: Uint8Array | string) => Observable<DeriveStakingQuery> {
+  return memo((accountId: Uint8Array | string): Observable<DeriveStakingQuery> =>
+    api.query.session.queuedKeys<Vec<ITuple<[AccountId, Keys]>>>().pipe(
+      switchMap((queuedKeys) => api.derive.staking.queryWithQueued(accountId, queuedKeys))
     )
   );
 }
+
+export function queryMulti (api: ApiInterfaceRx): (accountIds: (Uint8Array | string)[]) => Observable<DeriveStakingQuery[]> {
+  return memo((accountIds: (Uint8Array | string)[]): Observable<DeriveStakingQuery[]> =>
+    accountIds.length
+      ? api.query.session.queuedKeys<Vec<ITuple<[AccountId, Keys]>>>().pipe(
+        switchMap((queuedKeys) => combineLatest(
+          accountIds.map((acc) => api.derive.staking.queryWithQueued(acc, queuedKeys))
+        ))
+      )
+      : of([])
+  );
+}

packages/api-derive/src/staking/stakerExposure.ts

@@ -3,31 +3,29 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 import { ApiInterfaceRx } from '@polkadot/api/types';
+import { EraIndex } from '@polkadot/types/interfaces';
 import { DeriveStakerExposure, DeriveEraValidatorExposure } from '../types';
 
-import BN from 'bn.js';
 import { Observable } from 'rxjs';
-import { map } from 'rxjs/operators';
+import { map, switchMap } from 'rxjs/operators';
 
 import { memo } from '../util';
 
-export function stakerExposure (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean | BN | number) => Observable<DeriveStakerExposure[]> {
-  return memo((accountId: Uint8Array | string, withActive?: boolean | BN | number): Observable<DeriveStakerExposure[]> => {
+export function _stakerExposure (api: ApiInterfaceRx): (accountId: Uint8Array | string, eras: EraIndex[]) => Observable<DeriveStakerExposure[]> {
+  return memo((accountId: Uint8Array | string, eras: EraIndex[]): Observable<DeriveStakerExposure[]> => {
     const stakerId = api.registry.createType('AccountId', accountId).toString();
 
-    return api.derive.staking.erasExposure(withActive).pipe(
+    return api.derive.staking._erasExposure(eras).pipe(
       map((exposures): DeriveStakerExposure[] =>
         exposures.map(({ era, nominators: allNominators, validators: allValidators }): DeriveStakerExposure => {
           const isValidator = !!allValidators[stakerId];
           const validators: DeriveEraValidatorExposure = {};
-          let nominating: [string, number][] = [];
+          const nominating = allNominators[stakerId] || [];
 
           if (isValidator) {
             validators[stakerId] = allValidators[stakerId];
-          } else if (allNominators[stakerId]) {
-            nominating = allNominators[stakerId];
-
-            nominating.forEach(([validatorId]): void => {
+          } else if (nominating) {
+            nominating.forEach(({ validatorId }): void => {
               validators[validatorId] = allValidators[validatorId];
             });
           }
@@ -38,3 +36,11 @@ export function stakerExposure (api: ApiInterfaceRx): (accountId: Uint8Array | s
     );
   });
 }
+
+export function stakerExposure (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean) => Observable<DeriveStakerExposure[]> {
+  return memo((accountId: Uint8Array | string, withActive?: boolean): Observable<DeriveStakerExposure[]> =>
+    api.derive.staking.erasHistoric(withActive).pipe(
+      switchMap((eras) => api.derive.staking._stakerExposure(accountId, eras))
+    )
+  );
+}

packages/api-derive/src/staking/stakerPoints.ts

@@ -3,28 +3,34 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 import { ApiInterfaceRx } from '@polkadot/api/types';
+import { EraIndex } from '@polkadot/types/interfaces';
 import { DeriveStakerPoints } from '../types';
 
-import BN from 'bn.js';
 import { Observable } from 'rxjs';
-import { map } from 'rxjs/operators';
+import { map, switchMap } from 'rxjs/operators';
 
 import { memo } from '../util';
 
-export function stakerPoints (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean | BN | number) => Observable<DeriveStakerPoints[]> {
-  return memo((accountId: Uint8Array | string, withActive?: boolean | BN | number): Observable<DeriveStakerPoints[]> => {
+export function _stakerPoints (api: ApiInterfaceRx): (accountId: Uint8Array | string, eras: EraIndex[]) => Observable<DeriveStakerPoints[]> {
+  return memo((accountId: Uint8Array | string, eras: EraIndex[]): Observable<DeriveStakerPoints[]> => {
     const stakerId = api.registry.createType('AccountId', accountId).toString();
 
-    return api.derive.staking.erasPoints(withActive).pipe(
+    return api.derive.staking._erasPoints(eras).pipe(
       map((points): DeriveStakerPoints[] =>
         points.map(({ era, eraPoints, validators }): DeriveStakerPoints => ({
           era,
           eraPoints,
-          points: validators[stakerId]
-            ? validators[stakerId]
-            : api.registry.createType('RewardPoint')
+          points: validators[stakerId] || api.registry.createType('RewardPoint')
         }))
       )
     );
   });
 }
+
+export function stakerPoints (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean) => Observable<DeriveStakerPoints[]> {
+  return memo((accountId: Uint8Array | string, withActive?: boolean): Observable<DeriveStakerPoints[]> =>
+    api.derive.staking.erasHistoric(withActive).pipe(
+      switchMap((eras) => api.derive.staking._stakerPoints(accountId, eras))
+    )
+  );
+}

packages/api-derive/src/staking/stakerRewards.ts

@@ -3,27 +3,30 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { Balance } from '@polkadot/types/interfaces';
-import { DeriveEraPoints, DeriveEraPrefs, DeriveEraRewards, DeriveEraValPrefs, DeriveStakerExposure, DeriveStakerReward } from '../types';
+import { AccountId, EraIndex, StakingLedger, StakingLedgerTo240 } from '@polkadot/types/interfaces';
+import { DeriveEraPoints, DeriveEraPrefs, DeriveEraRewards, DeriveEraValPrefs, DeriveStakerExposure, DeriveStakerReward, DeriveStakerRewardValidator } from '../types';
 
 import BN from 'bn.js';
-import { Observable, combineLatest } from 'rxjs';
-import { map } from 'rxjs/operators';
+import { Observable, combineLatest, of } from 'rxjs';
+import { map, switchMap } from 'rxjs/operators';
+import { Option } from '@polkadot/types';
 
 import { memo } from '../util';
 
 type Result = [DeriveEraPoints[], DeriveEraPrefs[], DeriveEraRewards[], DeriveStakerExposure[]];
 
 const ZERO = new BN(0);
+const MIN_ONE = new BN(-1);
 const COMM_DIV = new BN(1_000_000_000);
+const MAX_ERAS = new BN(1_000_000_000);
 
-function parseRewards (api: ApiInterfaceRx, stakerId: string, [erasPoints, erasPrefs, erasRewards, exposures]: Result): DeriveStakerReward[] {
+function parseRewards (api: ApiInterfaceRx, stashId: AccountId, [erasPoints, erasPrefs, erasRewards, exposures]: Result): DeriveStakerReward[] {
   return exposures.map(({ era, isEmpty, isValidator, nominating, validators: eraValidators }): DeriveStakerReward => {
-    const { eraPoints, validators: allValPoints } = erasPoints.find((p) => p.era.eq(era)) || { eraPoints: new BN(0), validators: {} };
-    const { eraReward } = erasRewards.find((r) => r.era.eq(era)) || { eraReward: ZERO };
+    const { eraPoints, validators: allValPoints } = erasPoints.find((p) => p.era.eq(era)) || { eraPoints: ZERO, validators: {} };
+    const { eraReward } = erasRewards.find((r) => r.era.eq(era)) || { eraReward: api.registry.createType('Balance') };
     const { validators: allValPrefs } = erasPrefs.find((p) => p.era.eq(era)) || { validators: {} as DeriveEraValPrefs };
-    const validators: Record<string, Balance> = {};
-    let total = ZERO;
+    const validators: Record<string, DeriveStakerRewardValidator> = {};
+    const stakerId = stashId.toString();
 
     Object.entries(eraValidators).forEach(([validatorId, exposure]): void => {
       const valPoints = allValPoints[validatorId] || ZERO;
@@ -47,46 +50,153 @@ function parseRewards (api: ApiInterfaceRx, stakerId: string, [erasPoints, erasP
         }
 
         value = avail.sub(valCut).mul(staked).div(expTotal).add(validatorId === stakerId ? valCut : ZERO);
-        total = total.add(value);
       }
 
-      validators[validatorId] = api.registry.createType('Balance', value);
+      validators[validatorId] = {
+        total: api.registry.createType('Balance', avail),
+        value: api.registry.createType('Balance', value)
+      };
     });
 
     return {
       era,
+      eraReward,
       isEmpty,
       isValidator,
       nominating,
-      total: api.registry.createType('Balance', total),
       validators
     };
   });
 }
 
-export function stakerRewards (api: ApiInterfaceRx): (accountId: Uint8Array | string, startEra: BN | number) => Observable<DeriveStakerReward[]> {
-  return memo((accountId: Uint8Array | string, startEra: BN | number): Observable<DeriveStakerReward[]> => {
-    const stakerId = api.registry.createType('AccountId', accountId).toString();
+function uniqValidators (rewards: DeriveStakerReward[]): string[] {
+  const uniq: string[] = [];
 
-    return combineLatest([
-      api.derive.staking.erasPoints(startEra),
-      api.derive.staking.erasPrefs(startEra),
-      api.derive.staking.erasRewards(startEra),
-      api.derive.staking.stakerExposure(stakerId, startEra)
-    ]).pipe(
-      map((result): DeriveStakerReward[] =>
-        parseRewards(api, stakerId, result)
-      )
-    );
+  rewards.forEach(({ validators }): void => {
+    Object.keys(validators).forEach((validatorId): void => {
+      if (!uniq.includes(validatorId)) {
+        uniq.push(validatorId);
+      }
+    });
   });
+
+  return uniq;
 }
 
-export function stakerRewardsMulti (api: ApiInterfaceRx): (...params: [Uint8Array | string, BN | number][]) => Observable<DeriveStakerReward[][]> {
-  return memo((...params: [Uint8Array | string, BN | number][]): Observable<DeriveStakerReward[][]> =>
-    combineLatest(
-      params.map(([accountId, startEra]) =>
-        api.derive.staking.stakerRewards(accountId, startEra)
-      )
+function isOldLedger (ledger?: StakingLedger | StakingLedgerTo240): ledger is StakingLedgerTo240 {
+  return !!(ledger as StakingLedgerTo240)?.lastReward;
+}
+
+function filterEra (era: EraIndex, stakingLedger: StakingLedger): boolean {
+  return isOldLedger(stakingLedger)
+    ? era.gt(stakingLedger.lastReward.unwrapOr(MIN_ONE))
+    : !stakingLedger.claimedRewards.some((e) => e.eq(era));
+}
+
+function filterEras (eras: EraIndex[], stakingLedger: StakingLedger): EraIndex[] {
+  return eras.filter((era) => filterEra(era, stakingLedger));
+}
+
+function filterRewards (api: ApiInterfaceRx, rewards: DeriveStakerReward[], stakingLedger: StakingLedger, withActive?: boolean): Observable<DeriveStakerReward[]> {
+  if (withActive) {
+    return of(rewards);
+  }
+
+  const validators = uniqValidators(rewards);
+
+  return combineLatest([
+    api.query.staking.migrateEra
+      ? api.query.staking.migrateEra<Option<EraIndex>>()
+      : of({ unwrapOr: () => api.tx.staking.payoutStakers ? ZERO : MAX_ERAS }),
+    api.tx.staking.payoutStakers
+      ? api.derive.staking.queryMulti(validators)
+      : of([])
+  ]).pipe(
+    map(([optMigrate, queryValidators]): DeriveStakerReward[] => {
+      const migrateEra: BN = optMigrate.unwrapOr(ZERO);
+
+      return rewards
+        .filter(({ isEmpty }) => !isEmpty)
+        .filter((reward): boolean => {
+          if (reward.era.lt(migrateEra)) {
+            // we filter again here, the actual ledger may have changed, e.g. something has been claimed
+            return filterEra(reward.era, stakingLedger);
+          }
+
+          reward.isStakerPayout = true;
+
+          const rm: string[] = [];
+
+          Object.keys(reward.validators).forEach((validatorId): void => {
+            const index = validators.indexOf(validatorId);
+
+            if (index !== -1) {
+              const valLedger = queryValidators[index].stakingLedger;
+
+              if (valLedger?.claimedRewards.some((era) => reward.era.eq(era))) {
+                rm.push(validatorId);
+              }
+            }
+          });
+
+          rm.forEach((validatorId): void => {
+            delete reward.validators[validatorId];
+          });
+
+          return true;
+        })
+        .filter(({ validators }) => Object.keys(validators).length !== 0)
+        .map((reward) => ({
+          ...reward,
+          nominators: reward.nominating.filter(({ validatorId }) => !!reward.validators[validatorId])
+        }));
+    })
+  );
+}
+
+export function _stakerRewards (api: ApiInterfaceRx): (accountId: Uint8Array | string, eras: EraIndex[], withActive?: boolean) => Observable<DeriveStakerReward[]> {
+  return memo((accountId: Uint8Array | string, _eras: EraIndex[], withActive?: boolean): Observable<DeriveStakerReward[]> =>
+    api.derive.staking.query(accountId).pipe(
+      switchMap(({ stakingLedger, stashId }): Observable<DeriveStakerReward[]> => {
+        if (!stashId || !stakingLedger) {
+          return of([]);
+        }
+
+        const eras = withActive
+          ? _eras
+          : filterEras(_eras, stakingLedger);
+
+        return combineLatest([
+          api.derive.staking._erasPoints(eras),
+          api.derive.staking._erasPrefs(eras),
+          api.derive.staking._erasRewards(eras),
+          api.derive.staking._stakerExposure(stashId, eras)
+        ]).pipe(
+          switchMap((result): Observable<DeriveStakerReward[]> =>
+            filterRewards(api, parseRewards(api, stashId, result), stakingLedger, withActive)
+          )
+        );
+      })
     )
   );
 }
+
+export function stakerRewards (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean) => Observable<DeriveStakerReward[]> {
+  return memo((accountId: Uint8Array | string, withActive?: boolean): Observable<DeriveStakerReward[]> =>
+    api.derive.staking.erasHistoric(withActive).pipe(
+      switchMap((eras) => api.derive.staking._stakerRewards(accountId, eras, withActive))
+    )
+  );
+}
+
+export function stakerRewardsMulti (api: ApiInterfaceRx): (accountIds: (Uint8Array | string)[], withActive?: boolean) => Observable<DeriveStakerReward[][]> {
+  return memo((accountIds: (Uint8Array | string)[], withActive?: boolean): Observable<DeriveStakerReward[][]> =>
+    accountIds.length
+      ? api.derive.staking.erasHistoric(withActive).pipe(
+        switchMap((eras) =>
+          combineLatest(accountIds.map((acc) => api.derive.staking._stakerRewards(acc, eras, withActive)))
+        )
+      )
+      : of([])
+  );
+}

packages/api-derive/src/staking/stakerSlashes.ts

@@ -3,19 +3,19 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 import { ApiInterfaceRx } from '@polkadot/api/types';
+import { EraIndex } from '@polkadot/types/interfaces';
 import { DeriveStakerSlashes } from '../types';
 
-import BN from 'bn.js';
 import { Observable } from 'rxjs';
-import { map } from 'rxjs/operators';
+import { map, switchMap } from 'rxjs/operators';
 
 import { memo } from '../util';
 
-export function stakerSlashes (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean | BN | number) => Observable<DeriveStakerSlashes[]> {
-  return memo((accountId: Uint8Array | string, withActive?: boolean | BN | number): Observable<DeriveStakerSlashes[]> => {
+export function _stakerSlashes (api: ApiInterfaceRx): (accountId: Uint8Array | string, eras: EraIndex[]) => Observable<DeriveStakerSlashes[]> {
+  return memo((accountId: Uint8Array | string, eras: EraIndex[]): Observable<DeriveStakerSlashes[]> => {
     const stakerId = api.registry.createType('AccountId', accountId).toString();
 
-    return api.derive.staking.erasSlashes(withActive).pipe(
+    return api.derive.staking._erasSlashes(eras).pipe(
       map((slashes): DeriveStakerSlashes[] =>
         slashes.map(({ era, nominators, validators }): DeriveStakerSlashes => ({
           era,
@@ -25,3 +25,11 @@ export function stakerSlashes (api: ApiInterfaceRx): (accountId: Uint8Array | st
     );
   });
 }
+
+export function stakerSlashes (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean) => Observable<DeriveStakerSlashes[]> {
+  return memo((accountId: Uint8Array | string, withActive?: boolean): Observable<DeriveStakerSlashes[]> =>
+    api.derive.staking.erasHistoric(withActive).pipe(
+      switchMap((eras) => api.derive.staking._stakerSlashes(accountId, eras))
+    )
+  );
+}