[CI Skip] release/stable 2.9.1

skip-checks: true

.123trigger

@@ -4,4 +4,4 @@
 1.11.2
 1.12.2
 1.17.2
-1.31.2
+1.31.22.8.2

CHANGELOG.md

@@ -1,5 +1,41 @@
 # CHANGELOG
 
+## 2.9.1 Nov 30, 2020
+
+Upgrade priority: Low. No major changes when on at least 2.8.2. Users encouraged to upgrade to at least that patch.
+
+- **Breaking change** The API will now not attempt to queue calls made while not connected and send on connection. Rather when making a call and the RPC node is not connected, an error will be thrown.
+
+Contributed:
+
+- Add typegen capability for custom RPCs (Thanks to https://github.com/holygits)
+
+Changes:
+
+- Remove rpc-core send queue, only allow sends while connected
+- Adjust staking derive for `electedInfo` to contain details of current validators
+- Extract identity-only derive from `accounts.info` for simpler reusability
+- Add types for Substrate master node (to allow for consistent swapping to `MultiAddress`)
+- Adjust `Bytes` with `.slice` to be consistent for `Uint8array` extensions (already released in 2.8.2 as a bugfix)
+
+
+## 2.8.1 Nov 23, 2020
+
+Upgrade priority: Low. Recommended if using Substrate master with new `MultiAddress`.
+
+Contributed:
+
+- Export `derive.chain.getBlock` (Thanks to https://github.com/lovesh)
+
+Changes:
+
+- Expand `MultiAddress` construction with multi type detection
+- Rename `GenericLookupSource` to `IndicesLookupSource` (internal use)
+- Cleanup list retrievals with `Raw` data results for undecodable entries
+- Export actual version via `RegistryMetadata`
+- Dedupe bn.js (one version shared between dependencies)
+
+
 ## 2.7.1 Nov 16, 2020
 
 Upgrade priority: Medium. Recommended for users of Polkadot/Kusama and api-contract users.

README.md

@@ -28,19 +28,3 @@ The API is split up into a number of internal packages -
 Type definitions for interfaces as exposed by Polkadot & Substrate clients -
 
 - [@polkadot/types](packages/types/) Codecs for all Polkadot and Substrate primitives
-
-## development
-
-Contributions are welcome!
-
-To start off, this repo (along with others in the [@polkadot](https://github.com/polkadot-js/) family) uses yarn workspaces to organise the code. As such, after cloning, its dependencies _should_ be installed via `yarn`, not via npm; the latter will result in broken dependencies.
-
-To get started -
-
-1. Clone the repo locally, via `git clone https://github.com/polkadot-js/api <optional local path>`
-2. Ensure that you have a recent version of Node.js, for development purposes [Node 10](https://nodejs.org/en/) is recommended.
-3. Ensure that you have a recent version of Yarn, for development purposes [Yarn >=1.10.1](https://yarnpkg.com/docs/install) is required.
-4. Install the dependencies by running `yarn`
-5. Build the everything via `yarn run build`
-6. You can also launch the API Docs, via `yarn vuepress dev docs`
-7. Access the docs via [http://localhost:8080](http://localhost:8080)

package.json

@@ -7,7 +7,7 @@
     "packages/*"
   ],
   "resolutions": {
-    "typescript": "^4.0.5"
+    "typescript": "^4.1.2"
   },
   "scripts": {
     "build": "yarn build:interfaces && polkadot-dev-build-ts && (cd packages/typegen && copyfiles scripts/* build)",
@@ -23,14 +23,14 @@
     "test:watch": "polkadot-dev-run-test --watch"
   },
   "devDependencies": {
-    "@babel/core": "^7.12.3",
+    "@babel/core": "^7.12.8",
     "@babel/register": "^7.12.1",
     "@babel/runtime": "^7.12.5",
-    "@polkadot/dev": "^0.60.2",
+    "@polkadot/dev": "^0.60.5",
     "@polkadot/ts": "^0.3.55",
     "@polkadot/typegen": "workspace:packages/typegen",
     "@types/jest": "^26.0.15",
     "copyfiles": "^2.4.0"
   },
-  "version": "2.7.1"
+  "version": "2.9.1"
 }

packages/api-contract/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/api-contract",
-  "version": "2.7.1",
+  "version": "2.9.1",
   "description": "Interfaces for interacting with contracts and contract ABIs",
   "main": "index.js",
   "sideEffects": false,
@@ -17,10 +17,10 @@
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/api-contract#readme",
   "dependencies": {
     "@babel/runtime": "^7.12.5",
-    "@polkadot/api": "2.7.1",
-    "@polkadot/types": "2.7.1",
-    "@polkadot/util": "^4.1.1",
-    "bn.js": "^5.1.3",
+    "@polkadot/api": "2.9.1",
+    "@polkadot/types": "2.9.1",
+    "@polkadot/util": "^4.2.1",
+    "bn.js": "^4.11.9",
     "rxjs": "^6.6.3"
   }
 }

packages/api-derive/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/api-derive",
-  "version": "2.7.1",
+  "version": "2.9.1",
   "description": "Common functions used across Polkadot, derived from RPC calls and storage queries.",
   "main": "index.js",
   "sideEffects": false,
@@ -18,17 +18,17 @@
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/api-derive#readme",
   "dependencies": {
     "@babel/runtime": "^7.12.5",
-    "@polkadot/api": "2.7.1",
-    "@polkadot/rpc-core": "2.7.1",
-    "@polkadot/types": "2.7.1",
-    "@polkadot/util": "^4.1.1",
-    "@polkadot/util-crypto": "^4.1.1",
-    "bn.js": "^5.1.3",
+    "@polkadot/api": "2.9.1",
+    "@polkadot/rpc-core": "2.9.1",
+    "@polkadot/types": "2.9.1",
+    "@polkadot/util": "^4.2.1",
+    "@polkadot/util-crypto": "^4.2.1",
+    "bn.js": "^4.11.9",
     "memoizee": "^0.4.14",
     "rxjs": "^6.6.3"
   },
   "devDependencies": {
-    "@polkadot/keyring": "^4.1.1",
-    "@polkadot/rpc-provider": "2.7.1"
+    "@polkadot/keyring": "^4.2.1",
+    "@polkadot/rpc-provider": "2.9.1"
   }
 }

packages/api-derive/src/accounts/idAndIndex.ts

@@ -12,7 +12,7 @@ import { decodeAddress } from '@polkadot/util-crypto';
 
 import { memo } from '../util';
 
-function retrieve (api: ApiInterfaceRx, address: Address | AccountId | AccountIndex | string | null | undefined): Observable<AccountIdAndIndex> {
+function retrieve (api: ApiInterfaceRx, address: Address | AccountId | AccountIndex | Uint8Array | string | null | undefined): Observable<AccountIdAndIndex> {
   try {
     // yes, this can fail, don't care too much, catch will catch it
     const decoded = isU8a(address)
@@ -39,7 +39,7 @@ function retrieve (api: ApiInterfaceRx, address: Address | AccountId | AccountIn
 
 /**
  * @name idAndIndex
- * @param {(Address | AccountId | AccountIndex | string | null)} address - An accounts address in various formats.
+ * @param {(Address | AccountId | AccountIndex | Uint8Array | string | null)} address - An accounts address in various formats.
  * @description  An array containing the [[AccountId]] and [[AccountIndex]] as optional values.
  * @example
  * <BR>
@@ -50,7 +50,7 @@ function retrieve (api: ApiInterfaceRx, address: Address | AccountId | AccountIn
  * });
  * ```
  */
-export function idAndIndex (instanceId: string, api: ApiInterfaceRx): (address?: Address | AccountId | AccountIndex | string | null) => Observable<AccountIdAndIndex> {
-  return memo(instanceId, (address?: Address | AccountId | AccountIndex | string | null): Observable<AccountIdAndIndex> =>
+export function idAndIndex (instanceId: string, api: ApiInterfaceRx): (address?: Address | AccountId | AccountIndex | Uint8Array | string | null) => Observable<AccountIdAndIndex> {
+  return memo(instanceId, (address?: Address | AccountId | AccountIndex | Uint8Array | string | null): Observable<AccountIdAndIndex> =>
     retrieve(api, address));
 }

packages/api-derive/src/accounts/identity.ts

@@ -0,0 +1,126 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { AccountId, IdentityInfoAdditional, Registration } from '@polkadot/types/interfaces';
+import { ITuple } from '@polkadot/types/types';
+import { DeriveAccountRegistration, DeriveHasIdentity } from '../types';
+
+import { Observable, combineLatest, of } from 'rxjs';
+import { map, switchMap } from 'rxjs/operators';
+import { Data, Option } from '@polkadot/types';
+import { u8aToString } from '@polkadot/util';
+
+import { memo } from '../util';
+
+const UNDEF_HEX = { toHex: () => undefined };
+
+function dataAsString (data: Data): string | undefined {
+  return data.isRaw
+    ? u8aToString(data.asRaw.toU8a(true))
+    : data.isNone
+      ? undefined
+      : data.toHex();
+}
+
+function extractOther (additional: IdentityInfoAdditional[]): Record<string, string> {
+  return additional.reduce((other: Record<string, string>, [_key, _value]): Record<string, string> => {
+    const key = dataAsString(_key);
+    const value = dataAsString(_value);
+
+    if (key && value) {
+      other[key] = value;
+    }
+
+    return other;
+  }, {});
+}
+
+function extractIdentity (identityOfOpt?: Option<Registration>, superOf?: [AccountId, Data]): DeriveAccountRegistration {
+  if (!identityOfOpt?.isSome) {
+    return { judgements: [] };
+  }
+
+  const { info, judgements } = identityOfOpt.unwrap();
+  const topDisplay = dataAsString(info.display);
+
+  return {
+    display: (superOf && dataAsString(superOf[1])) || topDisplay,
+    displayParent: superOf && topDisplay,
+    email: dataAsString(info.email),
+    image: dataAsString(info.image),
+    judgements,
+    legal: dataAsString(info.legal),
+    other: extractOther(info.additional),
+    parent: superOf && superOf[0],
+    pgp: info.pgpFingerprint.unwrapOr(UNDEF_HEX).toHex(),
+    riot: dataAsString(info.riot),
+    twitter: dataAsString(info.twitter),
+    web: dataAsString(info.web)
+  };
+}
+
+function getParent (api: ApiInterfaceRx, identityOfOpt: Option<Registration> | undefined, superOfOpt: Option<ITuple<[AccountId, Data]>> | undefined): Observable<[Option<Registration> | undefined, [AccountId, Data] | undefined]> {
+  if (identityOfOpt?.isSome) {
+    // this identity has something set
+    return of([identityOfOpt, undefined]);
+  } else if (superOfOpt?.isSome) {
+    const superOf = superOfOpt.unwrap();
+
+    // we have a super
+    return combineLatest([
+      api.query.identity.identityOf<Option<Registration>>(superOf[0]),
+      of(superOf)
+    ]);
+  }
+
+  // nothing of value returned
+  return of([undefined, undefined]);
+}
+
+export function _identityBase (instanceId: string, api: ApiInterfaceRx): (accountId?: AccountId | Uint8Array | string) => Observable<[Option<Registration> | undefined, Option<ITuple<[AccountId, Data]>> | undefined]> {
+  return memo(instanceId, (accountId?: AccountId | Uint8Array | string): Observable<[Option<Registration> | undefined, Option<ITuple<[AccountId, Data]>> | undefined]> =>
+    accountId && api.query.identity?.identityOf
+      ? api.queryMulti<[Option<Registration>, Option<ITuple<[AccountId, Data]>>]>([
+        [api.query.identity.identityOf, accountId],
+        [api.query.identity.superOf, accountId]
+      ])
+      : of([undefined, undefined])
+  );
+}
+
+/**
+ * @name identity
+ * @description Returns identity info for an account
+ */
+export function identity (instanceId: string, api: ApiInterfaceRx): (accountId?: AccountId | Uint8Array | string) => Observable<DeriveAccountRegistration> {
+  return memo(instanceId, (accountId?: AccountId | Uint8Array | string): Observable<DeriveAccountRegistration> =>
+    api.derive.accounts._identityBase(accountId).pipe(
+      switchMap(([identityOfOpt, superOfOpt]) => getParent(api, identityOfOpt, superOfOpt)),
+      map(([identityOfOpt, superOf]) => extractIdentity(identityOfOpt, superOf))
+    )
+  );
+}
+
+export function hasIdentity (instanceId: string, api: ApiInterfaceRx): (accountId: AccountId | Uint8Array | string) => Observable<DeriveHasIdentity> {
+  return memo(instanceId, (accountId: AccountId | Uint8Array | string): Observable<DeriveHasIdentity> =>
+    api.derive.accounts._identityBase(accountId).pipe(
+      map(([identityOfOpt, superOfOpt]: [Option<Registration> | undefined, Option<ITuple<[AccountId, Data]>> | undefined]): DeriveHasIdentity => {
+        const parentId = superOfOpt && superOfOpt.isSome
+          ? superOfOpt.unwrap()[0].toString()
+          : null;
+        const hasIdentity = !!parentId || !!(identityOfOpt && identityOfOpt.isSome);
+
+        return { hasIdentity, parentId };
+      })
+    )
+  );
+}
+
+export function hasIdentityMulti (instanceId: string, api: ApiInterfaceRx): (accountIds: (AccountId | Uint8Array | string)[]) => Observable<DeriveHasIdentity[]> {
+  return memo(instanceId, (accountIds: (AccountId | Uint8Array | string)[]): Observable<DeriveHasIdentity[]> =>
+    combineLatest(
+      accountIds.map((accountId) => api.derive.accounts.hasIdentity(accountId))
+    )
+  );
+}

packages/api-derive/src/accounts/index.ts

@@ -5,6 +5,7 @@ export * from './accountId';
 export * from './flags';
 export * from './idAndIndex';
 export * from './idToIndex';
+export * from './identity';
 export * from './indexToId';
 export * from './indexes';
 export * from './info';

packages/api-derive/src/accounts/info.ts

@@ -2,27 +2,17 @@
 // SPDX-License-Identifier: Apache-2.0
 
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { AccountId, AccountIndex, Address, Balance, IdentityInfoAdditional, Registration } from '@polkadot/types/interfaces';
+import { AccountId, AccountIndex, Address, Balance } from '@polkadot/types/interfaces';
 import { ITuple } from '@polkadot/types/types';
 import { DeriveAccountInfo, DeriveAccountRegistration } from '../types';
 
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
-import { Bytes, Data, Option, u32 } from '@polkadot/types';
+import { Bytes, Option, u32 } from '@polkadot/types';
 import { u8aToString } from '@polkadot/util';
 
 import { memo } from '../util';
 
-const UNDEF_HEX = { toHex: () => undefined };
-
-function dataAsString (data: Data): string | undefined {
-  return data.isRaw
-    ? u8aToString(data.asRaw.toU8a(true))
-    : data.isNone
-      ? undefined
-      : data.toHex();
-}
-
 function retrieveNick (api: ApiInterfaceRx, accountId?: AccountId): Observable<string | undefined> {
   return ((
     accountId && api.query.nicks?.nameOf
@@ -37,86 +27,17 @@ function retrieveNick (api: ApiInterfaceRx, accountId?: AccountId): Observable<s
   );
 }
 
-function extractOther (additional: IdentityInfoAdditional[]): Record<string, string> {
-  return additional.reduce((other: Record<string, string>, [_key, _value]): Record<string, string> => {
-    const key = dataAsString(_key);
-    const value = dataAsString(_value);
-
-    if (key && value) {
-      other[key] = value;
-    }
-
-    return other;
-  }, {});
-}
-
-function extractIdentity (identityOfOpt?: Option<Registration>, superOf?: [AccountId, Data]): DeriveAccountRegistration {
-  if (!identityOfOpt?.isSome) {
-    return { judgements: [] };
-  }
-
-  const { info, judgements } = identityOfOpt.unwrap();
-  const topDisplay = dataAsString(info.display);
-
-  return {
-    display: (superOf && dataAsString(superOf[1])) || topDisplay,
-    displayParent: superOf && topDisplay,
-    email: dataAsString(info.email),
-    image: dataAsString(info.image),
-    judgements,
-    legal: dataAsString(info.legal),
-    other: extractOther(info.additional),
-    parent: superOf && superOf[0],
-    pgp: info.pgpFingerprint.unwrapOr(UNDEF_HEX).toHex(),
-    riot: dataAsString(info.riot),
-    twitter: dataAsString(info.twitter),
-    web: dataAsString(info.web)
-  };
-}
-
-function retrieveIdentity (api: ApiInterfaceRx, accountId?: AccountId): Observable<DeriveAccountRegistration> {
-  return ((
-    accountId && api.query.identity?.identityOf
-      ? api.queryMulti([
-        [api.query.identity.identityOf, accountId],
-        [api.query.identity.superOf, accountId]
-      ])
-      : of([undefined, undefined])
-  ) as Observable<[Option<Registration> | undefined, Option<ITuple<[AccountId, Data]>> | undefined]>).pipe(
-    switchMap(([identityOfOpt, superOfOpt]): Observable<[Option<Registration> | undefined, [AccountId, Data] | undefined]> => {
-      if (identityOfOpt?.isSome) {
-        // this identity has something set
-        return of([identityOfOpt, undefined]);
-      } else if (superOfOpt?.isSome) {
-        const superOf = superOfOpt.unwrap();
-
-        // we have a super
-        return combineLatest([
-          api.query.identity.identityOf<Option<Registration>>(superOf[0]),
-          of(superOf)
-        ]);
-      }
-
-      // nothing of value returned
-      return of([undefined, undefined]);
-    }),
-    map(([identityOfOpt, superOf]): DeriveAccountRegistration =>
-      extractIdentity(identityOfOpt, superOf)
-    )
-  );
-}
-
 /**
  * @name info
  * @description Returns aux. info with regards to an account, current that includes the accountId, accountIndex and nickname
  */
-export function info (instanceId: string, api: ApiInterfaceRx): (address?: AccountIndex | AccountId | Address | string | null) => Observable<DeriveAccountInfo> {
-  return memo(instanceId, (address?: AccountIndex | AccountId | Address | string | null): Observable<DeriveAccountInfo> =>
+export function info (instanceId: string, api: ApiInterfaceRx): (address?: AccountIndex | AccountId | Address | Uint8Array | string | null) => Observable<DeriveAccountInfo> {
+  return memo(instanceId, (address?: AccountIndex | AccountId | Address | Uint8Array | string | null): Observable<DeriveAccountInfo> =>
     api.derive.accounts.idAndIndex(address).pipe(
       switchMap(([accountId, accountIndex]): Observable<[Partial<DeriveAccountInfo>, DeriveAccountRegistration, string?]> =>
         combineLatest([
           of({ accountId, accountIndex }),
-          retrieveIdentity(api, accountId),
+          api.derive.accounts.identity(accountId),
           retrieveNick(api, accountId)
         ])
       ),

packages/api-derive/src/accounts/types.ts

@@ -35,3 +35,8 @@ export interface DeriveAccountInfo {
   identity: DeriveAccountRegistration;
   nickname?: string;
 }
+
+export interface DeriveHasIdentity {
+  hasIdentity: boolean;
+  parentId: string | null;
+}

packages/api-derive/src/chain/getBlock.ts

@@ -17,12 +17,12 @@ import { memo } from '../util';
  * <BR>
  *
  * ```javascript
- * const { author, number } = await api.derive.chain.getHeader('0x123...456');
+ * const { author, block } = await api.derive.chain.getBlock('0x123...456');
  *
- * console.log(`block #${number} was authored by ${author}`);
+ * console.log(`block #${block.header.number} was authored by ${author}`);
  * ```
  */
-export function getHeader (instanceId: string, api: ApiInterfaceRx): (hash: Uint8Array | string) => Observable<SignedBlockExtended | undefined> {
+export function getBlock (instanceId: string, api: ApiInterfaceRx): (hash: Uint8Array | string) => Observable<SignedBlockExtended | undefined> {
   return memo(instanceId, (hash: Uint8Array | string): Observable<SignedBlockExtended | undefined> =>
     combineLatest([
       api.rpc.chain.getBlock(hash),
@@ -35,7 +35,7 @@ export function getHeader (instanceId: string, api: ApiInterfaceRx): (hash: Uint
       ),
       catchError((): Observable<undefined> =>
         // where rpc.chain.getHeader throws, we will land here - it can happen that
-        // we supplied an invalid hash. (Due to defaults, storeage will have an
+        // we supplied an invalid hash. (Due to defaults, storage will have an
         // empty value, so only the RPC is affected). So return undefined
         of()
       )

packages/api-derive/src/chain/index.ts

@@ -5,5 +5,6 @@ export * from './bestNumber';
 export * from './bestNumberFinalized';
 export * from './bestNumberLag';
 export * from './getHeader';
+export * from './getBlock';
 export * from './subscribeNewBlocks';
 export * from './subscribeNewHeads';

packages/api-derive/src/staking/electedInfo.ts

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 import { ApiInterfaceRx } from '@polkadot/api/types';
+import { AccountId } from '@polkadot/types/interfaces';
 import { DeriveStakingElected } from '../types';
 
 import { Observable } from 'rxjs';
@@ -9,14 +10,19 @@ import { map, switchMap } from 'rxjs/operators';
 
 import { memo } from '../util';
 
+function combineAccounts (nextElected: AccountId[], validators: AccountId[]): AccountId[] {
+  return [...nextElected].concat(...validators.filter((v) => !nextElected.find((n) => n.eq(v))));
+}
+
 export function electedInfo (instanceId: string, api: ApiInterfaceRx): () => Observable<DeriveStakingElected> {
   return memo(instanceId, (): Observable<DeriveStakingElected> =>
     api.derive.staking.validators().pipe(
-      switchMap(({ nextElected }): Observable<DeriveStakingElected> =>
-        api.derive.staking.queryMulti(nextElected).pipe(
+      switchMap(({ nextElected, validators }): Observable<DeriveStakingElected> =>
+        api.derive.staking.queryMulti(combineAccounts(nextElected, validators)).pipe(
           map((info): DeriveStakingElected => ({
             info,
-            nextElected
+            nextElected,
+            validators
           }))
         )
       )

packages/api-derive/src/staking/types.ts

@@ -120,8 +120,9 @@ export interface DeriveStakingQuery extends DeriveStakingStash {
 }
 
 export interface DeriveStakingElected {
-  nextElected: AccountId[];
   info: DeriveStakingQuery[];
+  nextElected: AccountId[];
+  validators: AccountId[];
 }
 
 export interface DeriveStakingWaiting {

packages/api-derive/src/staking/validators.ts

@@ -30,7 +30,6 @@ export function validators (instanceId: string, api: ApiInterfaceRx): () => Obse
   return memo(instanceId, (): Observable<DeriveStakingValidators> =>
     // Sadly the node-template is (for some obscure reason) not comprehensive, so while the derive works
     // in all actual real-world deployed chains, it does create some confusion for limited template chains
-    // NOTE: Not doing multi queries here, since we have validators as a single in the derived newHead
     combineLatest([
       api.query.session
         ? api.query.session.validators()

packages/api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/api",
-  "version": "2.7.1",
+  "version": "2.9.1",
   "description": "Promise and RxJS wrappers around the Polkadot JS RPC",
   "main": "index.js",
   "sideEffects": false,
@@ -17,16 +17,16 @@
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/api#readme",
   "dependencies": {
     "@babel/runtime": "^7.12.5",
-    "@polkadot/api-derive": "2.7.1",
-    "@polkadot/keyring": "^4.1.1",
-    "@polkadot/metadata": "2.7.1",
-    "@polkadot/rpc-core": "2.7.1",
-    "@polkadot/rpc-provider": "2.7.1",
-    "@polkadot/types": "2.7.1",
-    "@polkadot/types-known": "2.7.1",
-    "@polkadot/util": "^4.1.1",
-    "@polkadot/util-crypto": "^4.1.1",
-    "bn.js": "^5.1.3",
+    "@polkadot/api-derive": "2.9.1",
+    "@polkadot/keyring": "^4.2.1",
+    "@polkadot/metadata": "2.9.1",
+    "@polkadot/rpc-core": "2.9.1",
+    "@polkadot/rpc-provider": "2.9.1",
+    "@polkadot/types": "2.9.1",
+    "@polkadot/types-known": "2.9.1",
+    "@polkadot/util": "^4.2.1",
+    "@polkadot/util-crypto": "^4.2.1",
+    "bn.js": "^4.11.9",
     "eventemitter3": "^4.0.7",
     "rxjs": "^6.6.3"
   }

packages/api/src/base/Decorate.ts

@@ -468,10 +468,10 @@ export abstract class Decorate<ApiType extends ApiTypes> extends Events {
   private _retrieveMapEntries (entry: StorageEntry, at: Hash | Uint8Array | string | null, arg?: Arg): Observable<[StorageKey, Codec][]> {
     const query = this._rpcCore.state.queryStorageAt
       ? at
-        ? (keyset: StorageKey[]) => this._rpcCore.state.queryStorageAt<Codec[]>(keyset, at)
-        : (keyset: StorageKey[]) => this._rpcCore.state.queryStorageAt<Codec[]>(keyset)
+        ? (keyset: StorageKey[]) => this._rpcCore.state.queryStorageAt(keyset, at)
+        : (keyset: StorageKey[]) => this._rpcCore.state.queryStorageAt(keyset)
       // this is horrible, but need older support (which now doesn't work with at)
-      : (keyset: StorageKey[]) => this._rpcCore.state.subscribeStorage<Codec[]>(keyset).pipe(take(1));
+      : (keyset: StorageKey[]) => this._rpcCore.state.subscribeStorage(keyset).pipe(take(1));
 
     return this._retrieveMapKeys(entry, at, arg).pipe(
       switchMap((keys) =>

packages/metadata/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/metadata",
-  "version": "2.7.1",
+  "version": "2.9.1",
   "description": "Helpers to extract information from runtime metadata",
   "main": "index.js",
   "sideEffects": false,
@@ -16,13 +16,13 @@
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/type-metadata#readme",
   "dependencies": {
     "@babel/runtime": "^7.12.5",
-    "@polkadot/types": "2.7.1",
-    "@polkadot/types-known": "2.7.1",
-    "@polkadot/util": "^4.1.1",
-    "@polkadot/util-crypto": "^4.1.1",
-    "bn.js": "^5.1.3"
+    "@polkadot/types": "2.9.1",
+    "@polkadot/types-known": "2.9.1",
+    "@polkadot/util": "^4.2.1",
+    "@polkadot/util-crypto": "^4.2.1",
+    "bn.js": "^4.11.9"
   },
   "devDependencies": {
-    "@polkadot/keyring": "^4.1.1"
+    "@polkadot/keyring": "^4.2.1"
   }
 }

packages/rpc-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/rpc-core",
-  "version": "2.7.1",
+  "version": "2.9.1",
   "description": "A JavaScript wrapper for the Polkadot JsonRPC interface",
   "main": "index.js",
   "sideEffects": false,
@@ -17,14 +17,14 @@
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/rpc-core#readme",
   "dependencies": {
     "@babel/runtime": "^7.12.5",
-    "@polkadot/metadata": "2.7.1",
-    "@polkadot/rpc-provider": "2.7.1",
-    "@polkadot/types": "2.7.1",
-    "@polkadot/util": "^4.1.1",
+    "@polkadot/metadata": "2.9.1",
+    "@polkadot/rpc-provider": "2.9.1",
+    "@polkadot/types": "2.9.1",
+    "@polkadot/util": "^4.2.1",
     "memoizee": "^0.4.14",
     "rxjs": "^6.6.3"
   },
   "devDependencies": {
-    "@polkadot/keyring": "^4.1.1"
+    "@polkadot/keyring": "^4.2.1"
   }
 }

packages/rpc-core/src/index.ts

@@ -382,13 +382,7 @@ export class RpcCore implements RpcInterface {
     if (rpc.type === 'StorageData') {
       const key = params[0] as StorageKey;
 
-      try {
-        return this._formatStorageData(registry, key, result);
-      } catch (error) {
-        l.error(`Unable to decode storage ${key.section || 'unknown'}.${key.method || 'unknown'}:`, (error as Error).message);
-
-        throw error;
-      }
+      return this._formatStorageData(registry, key, result);
     } else if (rpc.type === 'StorageChangeSet') {
       const keys = params[0] as Vec<StorageKey>;
 
@@ -426,22 +420,32 @@ export class RpcCore implements RpcInterface {
         : u8aToU8a(value);
 
     if (meta.modifier.isOptional) {
-      return new Option(
-        registry,
-        createClass(registry, type),
-        isEmpty
-          ? null
-          : createTypeUnsafe(registry, type, [input], true)
-      );
+      let inner = null;
+
+      if (!isEmpty) {
+        try {
+          inner = createTypeUnsafe(registry, type, [input], true);
+        } catch (error) {
+          l.error(`Unable to decode storage ${key.section || 'unknown'}.${key.method || 'unknown'}:`, (error as Error).message);
+        }
+      }
+
+      return new Option(registry, createClass(registry, type), inner);
     }
 
-    return createTypeUnsafe(registry, type, [
-      isEmpty
-        ? meta.fallback
-          ? hexToU8a(meta.fallback.toHex())
-          : undefined
-        : input
-    ], true);
+    try {
+      return createTypeUnsafe(registry, type, [
+        isEmpty
+          ? meta.fallback
+            ? hexToU8a(meta.fallback.toHex())
+            : undefined
+          : input
+      ], true);
+    } catch (error) {
+      l.error(`Unable to decode storage ${key.section || 'unknown'}.${key.method || 'unknown'}:`, (error as Error).message);
+
+      return registry.createType('Raw', input);
+    }
   }
 
   private _formatStorageSet (registry: Registry, keys: Vec<StorageKey>, changes: [string, string | null][]): Codec[] {
@@ -453,24 +457,18 @@ export class RpcCore implements RpcInterface {
     //   - Codec - There is a valid value, non-empty
     //   - null - The storage key is empty
     return keys.reduce((results: Codec[], key: StorageKey, index: number): Codec[] => {
-      try {
-        results.push(this._formatStorageSetEntry(registry, key, changes, withCache));
-      } catch (error) {
-        l.error(`Unable to decode storage ${key.section || 'unknown'}.${key.method || 'unknown'}: entry ${index + 1}/${keys.length}:`, (error as Error).message);
-
-        throw error;
-      }
+      results.push(this._formatStorageSetEntry(registry, key, changes, withCache, index));
 
       return results;
     }, []);
   }
 
-  private _formatStorageSetEntry (registry: Registry, key: StorageKey, changes: [string, string | null][], witCache: boolean): Codec {
+  private _formatStorageSetEntry (registry: Registry, key: StorageKey, changes: [string, string | null][], witCache: boolean, entryIndex: number): Codec {
     // Fallback to Raw (i.e. just the encoding) if we don't have a specific type
     const type = key.outputType || 'Raw';
     const hexKey = key.toHex();
     const meta = key.meta || EMPTY_META;
-    const found = changes.find(([key]): boolean => key === hexKey);
+    const found = changes.find(([key]) => key === hexKey);
 
     // if we don't find the value, this is our fallback
     //   - in the case of an array of values, fill the hole from the cache
@@ -490,21 +488,31 @@ export class RpcCore implements RpcInterface {
     this.#storageCache.set(hexKey, value);
 
     if (meta.modifier.isOptional) {
-      return new Option(
-        registry,
-        createClass(registry, type),
-        isEmpty
-          ? null
-          : createTypeUnsafe(registry, type, [input], true)
-      );
+      let inner = null;
+
+      if (!isEmpty) {
+        try {
+          inner = createTypeUnsafe(registry, type, [input], true);
+        } catch (error) {
+          l.error(`Unable to decode storage ${key.section || 'unknown'}.${key.method || 'unknown'}: entry ${entryIndex}:`, (error as Error).message);
+        }
+      }
+
+      return new Option(registry, createClass(registry, type), inner);
     }
 
-    return createTypeUnsafe(registry, type, [
-      isEmpty
-        ? meta.fallback
-          ? hexToU8a(meta.fallback.toHex())
-          : undefined
-        : input
-    ], true);
+    try {
+      return createTypeUnsafe(registry, type, [
+        isEmpty
+          ? meta.fallback
+            ? hexToU8a(meta.fallback.toHex())
+            : undefined
+          : input
+      ], true);
+    } catch (error) {
+      l.error(`Unable to decode storage ${key.section || 'unknown'}.${key.method || 'unknown'}: entry ${entryIndex}:`, (error as Error).message);
+
+      return registry.createType('Raw', input);
+    }
   }
 }

packages/rpc-provider/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/rpc-provider",
-  "version": "2.7.1",
+  "version": "2.9.1",
   "description": "Transport providers for the API",
   "main": "index.js",
   "sideEffects": false,
@@ -17,17 +17,17 @@
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/rpc-provider#readme",
   "dependencies": {
     "@babel/runtime": "^7.12.5",
-    "@polkadot/types": "2.7.1",
-    "@polkadot/util": "^4.1.1",
-    "@polkadot/util-crypto": "^4.1.1",
-    "@polkadot/x-fetch": "^4.1.1",
-    "@polkadot/x-ws": "^4.1.1",
-    "bn.js": "^5.1.3",
+    "@polkadot/types": "2.9.1",
+    "@polkadot/util": "^4.2.1",
+    "@polkadot/util-crypto": "^4.2.1",
+    "@polkadot/x-fetch": "^4.2.1",
+    "@polkadot/x-ws": "^4.2.1",
+    "bn.js": "^4.11.9",
     "eventemitter3": "^4.0.7"
   },
   "devDependencies": {
-    "@polkadot/keyring": "^4.1.1",
-    "@polkadot/metadata": "2.7.1",
+    "@polkadot/keyring": "^4.2.1",
+    "@polkadot/metadata": "2.9.1",
     "mock-socket": "^9.0.3",
     "nock": "^13.0.5"
   }

packages/rpc-provider/src/ws/index.ts

@@ -71,7 +71,7 @@ export class WsProvider implements ProviderInterface {
 
   readonly #handlers: Record<string, WsStateAwaiting> = {};
 
-  readonly #queued: Record<string, string> = {};
+  readonly #isReadyPromise: Promise<WsProvider>;
 
   readonly #waitingForId: Record<string, JsonRpcResponse> = {};
 
@@ -113,6 +113,12 @@ export class WsProvider implements ProviderInterface {
         // does not throw
       });
     }
+
+    this.#isReadyPromise = new Promise((resolve): void => {
+      this.#eventemitter.once('connected', (): void => {
+        resolve(this);
+      });
+    });
   }
 
   /**
@@ -130,6 +136,13 @@ export class WsProvider implements ProviderInterface {
     return this.#isConnected;
   }
 
+  /**
+   * @description Promise that resolves the first time we are connected and loaded
+   */
+  public get isReady (): Promise<WsProvider> {
+    return this.#isReadyPromise;
+  }
+
   /**
    * @description Returns a clone of the object
    */
@@ -225,12 +238,14 @@ export class WsProvider implements ProviderInterface {
   /**
    * @summary Send JSON data using WebSockets to configured HTTP Endpoint or queue.
    * @param method The RPC methods to execute
-   * @param params Encoded paramaters as appliucable for the method
+   * @param params Encoded parameters as applicable for the method
    * @param subscription Subscription details (internally used)
    */
   public send (method: string, params: any[], subscription?: SubscriptionHandler): Promise<any> {
     return new Promise((resolve, reject): void => {
       try {
+        assert(this.isConnected && !isNull(this.#websocket), 'WebSocket is not connected');
+
         const json = this.#coder.encodeJson(method, params);
         const id = this.#coder.getId();
 
@@ -240,7 +255,7 @@ export class WsProvider implements ProviderInterface {
             : resolve(result);
         };
 
-        l.debug((): string[] => ['calling', method, json]);
+        l.debug(() => ['calling', method, json]);
 
         this.#handlers[id] = {
           callback,
@@ -249,11 +264,7 @@ export class WsProvider implements ProviderInterface {
           subscription
         };
 
-        if (this.isConnected && !isNull(this.#websocket)) {
-          this.#websocket.send(json);
-        } else {
-          this.#queued[id] = json;
-        }
+        this.#websocket.send(json);
       } catch (error) {
         reject(error);
       }
@@ -300,7 +311,7 @@ export class WsProvider implements ProviderInterface {
     // a slight complication in solving - since we cannot rely on the send id, but rather
     // need to find the actual subscription id to map it
     if (isUndefined(this.#subscriptions[subscription])) {
-      l.debug((): string => `Unable to find active subscription=${subscription}`);
+      l.debug(() => `Unable to find active subscription=${subscription}`);
 
       return false;
     }
@@ -334,15 +345,14 @@ export class WsProvider implements ProviderInterface {
   }
 
   #onSocketError = (error: Event): void => {
-    l.debug((): any => ['socket error', error]);
+    l.debug(() => ['socket error', error]);
     this.#emit('error', error);
   }
 
-  #onSocketMessage = (message: MessageEvent): void => {
-    // eslint-disable-next-line @typescript-eslint/no-unsafe-return
+  #onSocketMessage = (message: MessageEvent<string>): void => {
     l.debug(() => ['received', message.data]);
 
-    const response = JSON.parse(message.data as string) as JsonRpcResponse;
+    const response = JSON.parse(message.data) as JsonRpcResponse;
 
     return isUndefined(response.method)
       ? this.#onSocketMessageResult(response)
@@ -353,7 +363,7 @@ export class WsProvider implements ProviderInterface {
     const handler = this.#handlers[response.id];
 
     if (!handler) {
-      l.debug((): string => `Unable to find handler for id=${response.id}`);
+      l.debug(() => `Unable to find handler for id=${response.id}`);
 
       return;
     }
@@ -396,7 +406,7 @@ export class WsProvider implements ProviderInterface {
       // store the JSON, we could have out-of-order subid coming in
       this.#waitingForId[subId] = response;
 
-      l.debug((): string => `Unable to find handler for subscription=${subId}`);
+      l.debug(() => `Unable to find handler for subscription=${subId}`);
 
       return;
     }
@@ -416,12 +426,11 @@ export class WsProvider implements ProviderInterface {
   #onSocketOpen = (): boolean => {
     assert(!isNull(this.#websocket), 'WebSocket cannot be null in onOpen');
 
-    l.debug((): any[] => ['connected to', this.#endpoints[this.#endpointIndex]]);
+    l.debug(() => ['connected to', this.#endpoints[this.#endpointIndex]]);
 
     this.#isConnected = true;
 
     this.#emit('connected');
-    this.#sendQueue();
     this.#resubscribe();
 
     return true;
@@ -432,8 +441,7 @@ export class WsProvider implements ProviderInterface {
 
     this.#subscriptions = {};
 
-    // eslint-disable-next-line @typescript-eslint/no-misused-promises
-    Object.keys(subscriptions).forEach(async (id): Promise<void> => {
+    Promise.all(Object.keys(subscriptions).map(async (id): Promise<void> => {
       const { callback, method, params, type } = subscriptions[id];
 
       // only re-create subscriptions which are not in author (only area where
@@ -448,21 +456,6 @@ export class WsProvider implements ProviderInterface {
       } catch (error) {
         l.error(error);
       }
-    });
-  }
-
-  #sendQueue = (): void => {
-    Object.keys(this.#queued).forEach((id): void => {
-      try {
-        // we have done the websocket check in onSocketOpen, if an issue, will catch it
-        (this.#websocket as WebSocket).send(
-          this.#queued[id]
-        );
-
-        delete this.#queued[id];
-      } catch (error) {
-        l.error(error);
-      }
-    });
+    })).catch(l.error);
   }
 }

packages/rpc-provider/src/ws/onOpen.spec.ts

@@ -1,43 +0,0 @@
-// Copyright 2017-2020 @polkadot/rpc-provider authors & contributors
-// SPDX-License-Identifier: Apache-2.0
-
-import { Mock } from './../mock/types';
-import { mockWs, TEST_WS_URL } from '../../test/mockWs';
-
-import { WsProvider } from './';
-
-let ws: WsProvider;
-let mock: Mock;
-
-function createWs (requests: any[], autoConnect: number | undefined = 1000): WsProvider {
-  mock = mockWs(requests);
-  ws = new WsProvider(TEST_WS_URL, autoConnect);
-
-  return ws;
-}
-
-describe('onOpen', (): void => {
-  afterEach((): void => {
-    if (mock) {
-      mock.done();
-    }
-  });
-
-  it('sends messages when connected', (): Promise<void> => {
-    const ws = createWs([{
-      id: 1,
-      method: 'test_queue',
-      reply: {
-        result: 'ok'
-      }
-    }], 0);
-    const sendPromise = ws.send('test_queue', []);
-
-    // eslint-disable-next-line @typescript-eslint/no-floating-promises
-    ws.connect();
-
-    return sendPromise.then((result): void => {
-      expect(result).toEqual('ok');
-    });
-  });
-});

packages/rpc-provider/src/ws/send.spec.ts

@@ -17,10 +17,10 @@ function createMock (requests: any[]): void {
   mock = mockWs(requests);
 }
 
-function createWs (autoConnect = 1000): WsProvider {
+function createWs (autoConnect = 1000): Promise<WsProvider> {
   provider = new WsProvider(TEST_WS_URL, autoConnect);
 
-  return provider;
+  return provider.isReady;
 }
 
 describe('send', (): void => {
@@ -47,11 +47,13 @@ describe('send', (): void => {
       }
     }]);
 
-    return createWs()
-      .send('test_encoding', [{ error: 'send error' }])
-      .catch((error): void => {
-        expect((error as Error).message).toEqual('send error');
-      });
+    return createWs().then((ws) =>
+      ws
+        .send('test_encoding', [{ error: 'send error' }])
+        .catch((error): void => {
+          expect((error as Error).message).toEqual('send error');
+        })
+    );
   });
 
   it('passes the body through correctly', (): Promise<void> => {
@@ -63,14 +65,16 @@ describe('send', (): void => {
       }
     }]);
 
-    return createWs()
-      .send('test_body', ['param'])
-      .then((): void => {
-        expect(
-          // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
-          (mock.body as any).test_body
-        ).toEqual('{"id":1,"jsonrpc":"2.0","method":"test_body","params":["param"]}');
-      });
+    return createWs().then((ws) =>
+      ws
+        .send('test_body', ['param'])
+        .then((): void => {
+          expect(
+            // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+            (mock.body as any).test_body
+          ).toEqual('{"id":1,"jsonrpc":"2.0","method":"test_body","params":["param"]}');
+        })
+    );
   });
 
   it('throws error when !response.ok', (): Promise<any> => {
@@ -82,11 +86,13 @@ describe('send', (): void => {
       id: 1
     }]);
 
-    return createWs()
-      .send('test_error', [])
-      .catch((error): void => {
-        expect((error as Error).message).toMatch(/666: error/);
-      });
+    return createWs().then((ws) =>
+      ws
+        .send('test_error', [])
+        .catch((error): void => {
+          expect((error as Error).message).toMatch(/666: error/);
+        })
+    );
   });
 
   it('adds subscriptions', (): Promise<void> => {
@@ -98,10 +104,12 @@ describe('send', (): void => {
       }
     }]);
 
-    return createWs()
-      .send('test_sub', [])
-      .then((id): void => {
-        expect(id).toEqual(1);
-      });
+    return createWs().then((ws) =>
+      ws
+        .send('test_sub', [])
+        .then((id): void => {
+          expect(id).toEqual(1);
+        })
+    );
   });
 });

packages/rpc-provider/src/ws/subscribe.spec.ts

@@ -17,10 +17,10 @@ function createMock (requests: any[]): void {
   mock = mockWs(requests);
 }
 
-function createWs (autoConnect = 1000): WsProvider {
+function createWs (autoConnect = 1000): Promise<WsProvider> {
   ws = new WsProvider(TEST_WS_URL, autoConnect);
 
-  return ws;
+  return ws.isReady;
 }
 
 describe('subscribe', (): void => {
@@ -47,12 +47,14 @@ describe('subscribe', (): void => {
       }
     }]);
 
-    return createWs()
-      .subscribe('type', 'test_sub', [], (cb): void => {
-        expect(cb).toEqual(expect.anything());
-      })
-      .then((id): void => {
-        expect(id).toEqual(1);
-      });
+    return createWs().then((ws) =>
+      ws
+        .subscribe('type', 'test_sub', [], (cb): void => {
+          expect(cb).toEqual(expect.anything());
+        })
+        .then((id): void => {
+          expect(id).toEqual(1);
+        })
+    );
   });
 });

packages/rpc-provider/src/ws/unsubscribe.spec.ts

@@ -17,10 +17,10 @@ function createMock (requests: any): void {
   mock = mockWs(requests);
 }
 
-function createWs (autoConnect = 1000): WsProvider {
+function createWs (autoConnect = 1000): Promise<WsProvider> {
   ws = new WsProvider(TEST_WS_URL, autoConnect);
 
-  return ws;
+  return ws.isReady;
 }
 
 describe('subscribe', (): void => {
@@ -56,15 +56,15 @@ describe('subscribe', (): void => {
       }
     ]);
 
-    const ws = createWs();
-
-    return ws
-      .subscribe('test', 'subscribe_test', [], (cb): void => {
-        expect(cb).toEqual(expect.anything());
-      })
-      .then((id): Promise<boolean> => {
-        return ws.unsubscribe('test', 'subscribe_test', id);
-      });
+    return createWs().then((ws) =>
+      ws
+        .subscribe('test', 'subscribe_test', [], (cb): void => {
+          expect(cb).toEqual(expect.anything());
+        })
+        .then((id): Promise<boolean> => {
+          return ws.unsubscribe('test', 'subscribe_test', id);
+        })
+    );
   });
 
   it('fails when sub not found', (): Promise<void> => {
@@ -76,17 +76,17 @@ describe('subscribe', (): void => {
       }
     }]);
 
-    const ws = createWs();
-
-    return ws
-      .subscribe('test', 'subscribe_test', [], (cb): void => {
-        expect(cb).toEqual(expect.anything());
-      })
-      .then((): Promise<boolean> => {
-        return ws.unsubscribe('test', 'subscribe_test', 111);
-      })
-      .then((result): void => {
-        expect(result).toBe(false);
-      });
+    return createWs().then((ws) =>
+      ws
+        .subscribe('test', 'subscribe_test', [], (cb): void => {
+          expect(cb).toEqual(expect.anything());
+        })
+        .then((): Promise<boolean> => {
+          return ws.unsubscribe('test', 'subscribe_test', 111);
+        })
+        .then((result): void => {
+          expect(result).toBe(false);
+        })
+    );
   });
 });

packages/typegen/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/typegen",
-  "version": "2.7.1",
+  "version": "2.9.1",
   "description": "Type generation scripts",
   "main": "index.js",
   "sideEffects": false,
@@ -23,20 +23,20 @@
   "bugs": "https://github.com/polkadot-js/api/issues",
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/typegen#readme",
   "dependencies": {
-    "@babel/core": "^7.12.3",
+    "@babel/core": "^7.12.8",
     "@babel/register": "^7.12.1",
     "@babel/runtime": "^7.12.5",
-    "@polkadot/api": "2.7.1",
-    "@polkadot/metadata": "2.7.1",
-    "@polkadot/rpc-provider": "2.7.1",
-    "@polkadot/types": "2.7.1",
-    "@polkadot/util": "^4.1.1",
+    "@polkadot/api": "2.9.1",
+    "@polkadot/metadata": "2.9.1",
+    "@polkadot/rpc-provider": "2.9.1",
+    "@polkadot/types": "2.9.1",
+    "@polkadot/util": "^4.2.1",
     "handlebars": "^4.7.6",
     "websocket": "^1.0.32",
-    "yargs": "^16.1.0"
+    "yargs": "^16.1.1"
   },
   "devDependencies": {
     "@types/websocket": "^1.0.1",
-    "@types/yargs": "^15.0.9"
+    "@types/yargs": "^15.0.10"
   }
 }

packages/typegen/src/fromChain.ts

@@ -6,7 +6,7 @@ import yargs from 'yargs';
 import { formatNumber } from '@polkadot/util';
 import WS from '@polkadot/x-ws';
 
-import { generateDefaultConsts, generateDefaultQuery, generateDefaultTx } from './generate';
+import { generateDefaultConsts, generateDefaultQuery, generateDefaultRpc, generateDefaultTx } from './generate';
 import { HEADER, writeFile } from './util';
 
 function generate (metaHex: string, pkg: string | undefined, output: string, isStrict?: boolean): void {
@@ -19,6 +19,7 @@ function generate (metaHex: string, pkg: string | undefined, output: string, isS
 
   generateDefaultConsts(path.join(process.cwd(), output, 'augment-api-consts.ts'), metaHex, extraTypes, isStrict);
   generateDefaultQuery(path.join(process.cwd(), output, 'augment-api-query.ts'), metaHex, extraTypes, isStrict);
+  generateDefaultRpc(path.join(process.cwd(), output, 'augment-api-rpc.ts'), extraTypes);
   generateDefaultTx(path.join(process.cwd(), output, 'augment-api-tx.ts'), metaHex, extraTypes, isStrict);
 
   writeFile(path.join(process.cwd(), output, 'augment-api.ts'), (): string =>
@@ -26,7 +27,7 @@ function generate (metaHex: string, pkg: string | undefined, output: string, isS
       HEADER('chain'),
       ...[
         '@polkadot/api/augment/rpc',
-        ...['consts', 'query', 'tx'].filter((key) => !!key).map((key) => `./augment-api-${key}`)
+        ...['consts', 'query', 'tx', 'rpc'].filter((key) => !!key).map((key) => `./augment-api-${key}`)
       ].map((path) => `import '${path}';\n`)
     ].join('')
   );

packages/typegen/src/generate/rpc.ts

@@ -7,7 +7,7 @@ import { TypeRegistry } from '@polkadot/types/create';
 import { Definitions } from '@polkadot/types/types';
 import * as defaultDefinitions from '@polkadot/types/interfaces/definitions';
 
-import { createImports, getSimilarTypes, formatType, readTemplate, setImports, writeFile } from '../util';
+import { createImports, getSimilarTypes, formatType, readTemplate, registerDefinitions, setImports, writeFile } from '../util';
 
 const StorageKeyTye = 'StorageKey | string | Uint8Array | any';
 
@@ -15,12 +15,12 @@ const template = readTemplate('rpc');
 const generateRpcTypesTemplate = Handlebars.compile(template);
 
 /** @internal */
-export function generateRpcTypes (importDefinitions: { [importPath: string]: Record<string, Definitions> }, dest: string): void {
+export function generateRpcTypes (registry: TypeRegistry, importDefinitions: Record<string, Definitions>, dest: string, extraTypes: Record<string, Record<string, { types: Record<string, any> }>> = {}): void {
   writeFile(dest, (): string => {
-    const registry = new TypeRegistry();
-    const imports = createImports(importDefinitions);
+    const allTypes: Record<string, Record<string, { types: Record<string, any> }>> = { '@polkadot/types/interfaces': importDefinitions, ...extraTypes };
+    const imports = createImports(allTypes);
     const definitions = imports.definitions as Record<string, Definitions>;
-    const allDefs = Object.entries(importDefinitions).reduce((defs, [path, obj]) => {
+    const allDefs = Object.entries(allTypes).reduce((defs, [path, obj]) => {
       return Object.entries(obj).reduce((defs, [key, value]) => ({ ...defs, [`${path}/${key}`]: value }), defs);
     }, {});
 
@@ -111,9 +111,15 @@ export function generateRpcTypes (importDefinitions: { [importPath: string]: Rec
   });
 }
 
-export function generateDefaultRpc (): void {
+export function generateDefaultRpc (dest = 'packages/api/src/augment/rpc.ts', extraTypes: Record<string, Record<string, { types: Record<string, any> }>> = {}): void {
+  const registry = new TypeRegistry();
+
+  registerDefinitions(registry, extraTypes);
+
   generateRpcTypes(
-    { '@polkadot/types/interfaces': defaultDefinitions },
-    'packages/api/src/augment/rpc.ts'
+    registry,
+    defaultDefinitions,
+    dest,
+    extraTypes
   );
 }

packages/types-known/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/types-known",
-  "version": "2.7.1",
+  "version": "2.9.1",
   "description": "A JavaScript wrapper for the Polkadot JsonRPC interface",
   "main": "index.js",
   "sideEffects": false,
@@ -17,9 +17,9 @@
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/types-known#readme",
   "dependencies": {
     "@babel/runtime": "^7.12.5",
-    "@polkadot/types": "2.7.1",
-    "@polkadot/util": "^4.1.1",
-    "bn.js": "^5.1.3"
+    "@polkadot/types": "2.9.1",
+    "@polkadot/util": "^4.2.1",
+    "bn.js": "^4.11.9"
   },
   "devDependencies": {
     "@types/bn.js": "^4.11.6"

packages/types-known/src/spec/centrifuge-chain.ts

@@ -1,10 +1,15 @@
 // Copyright 2017-2020 @polkadot/types-known authors & contributors
 // SPDX-License-Identifier: Apache-2.0
 
+/* eslint-disable sort-keys */
+
 import { OverrideVersionedType } from '@polkadot/types/types';
 
 const sharedTypes = {
+  // substrate
   Address: 'LookupSource',
+  LookupSource: 'IndicesLookupSource',
+
   AnchorData: {
     anchoredBlock: 'u64',
     docRoot: 'H256',

packages/types-known/src/spec/index.ts

@@ -5,6 +5,7 @@ import { OverrideVersionedType } from '@polkadot/types/types';
 
 import centrifugeChain from './centrifuge-chain';
 import kusama from './kusama';
+import node from './node';
 import nodeTemplate from './node-template';
 import polkadot from './polkadot';
 import rococo from './rococo';
@@ -14,6 +15,7 @@ import westend from './westend';
 const typesSpec: Record<string, OverrideVersionedType[]> = {
   'centrifuge-chain': centrifugeChain,
   kusama,
+  node,
   'node-template': nodeTemplate,
   polkadot,
   rococo,

packages/types-known/src/spec/kusama.ts

@@ -1,6 +1,8 @@
 // Copyright 2017-2020 @polkadot/types-known authors & contributors
 // SPDX-License-Identifier: Apache-2.0
 
+/* eslint-disable sort-keys */
+
 import { OverrideVersionedType } from '@polkadot/types/types';
 
 const sharedTypes = {
@@ -23,6 +25,7 @@ const versioned: OverrideVersionedType[] = [
       DispatchError: 'DispatchErrorTo198',
       DispatchInfo: 'DispatchInfoTo244',
       Keys: 'SessionKeys5',
+      LookupSource: 'IndicesLookupSource',
       Multiplier: 'Fixed64',
       OpenTip: 'OpenTipTo225',
       RefCount: 'RefCountTo259',
@@ -42,6 +45,7 @@ const versioned: OverrideVersionedType[] = [
       CompactAssignments: 'CompactAssignmentsTo257',
       DispatchInfo: 'DispatchInfoTo244',
       Keys: 'SessionKeys5',
+      LookupSource: 'IndicesLookupSource',
       Multiplier: 'Fixed64',
       OpenTip: 'OpenTipTo225',
       RefCount: 'RefCountTo259',
@@ -62,6 +66,7 @@ const versioned: OverrideVersionedType[] = [
       CompactAssignments: 'CompactAssignmentsTo257',
       DispatchInfo: 'DispatchInfoTo244',
       Keys: 'SessionKeys5',
+      LookupSource: 'IndicesLookupSource',
       Multiplier: 'Fixed64',
       OpenTip: 'OpenTipTo225',
       RefCount: 'RefCountTo259',

packages/types-known/src/spec/node-template.ts

@@ -1,6 +1,8 @@
 // Copyright 2017-2020 @polkadot/types-known authors & contributors
 // SPDX-License-Identifier: Apache-2.0
 
+/* eslint-disable sort-keys */
+
 import { OverrideVersionedType } from '@polkadot/types/types';
 
 const versioned: OverrideVersionedType[] = [

packages/types-known/src/spec/node.ts

@@ -0,0 +1,26 @@
+// Copyright 2017-2020 @polkadot/types-known authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+
+/* eslint-disable sort-keys */
+
+import { OverrideVersionedType } from '@polkadot/types/types';
+
+// these are override types for Polkadot
+const versioned: OverrideVersionedType[] = [
+  {
+    minmax: [0, 259],
+    types: {
+      Address: 'LookupSource',
+      LookupSource: 'IndicesLookupSource'
+    }
+  },
+  {
+    minmax: [260, undefined],
+    types: {
+      Address: 'MultiAddress',
+      LookupSource: 'MultiAddress'
+    }
+  }
+];
+
+export default versioned;

packages/types-known/src/spec/polkadot.ts

@@ -1,6 +1,8 @@
 // Copyright 2017-2020 @polkadot/types-known authors & contributors
 // SPDX-License-Identifier: Apache-2.0
 
+/* eslint-disable sort-keys */
+
 import { OverrideVersionedType } from '@polkadot/types/types';
 
 const sharedTypes = {

packages/types-known/src/spec/rococo.ts

@@ -1,6 +1,8 @@
 // Copyright 2017-2020 @polkadot/types-known authors & contributors
 // SPDX-License-Identifier: Apache-2.0
 
+/* eslint-disable sort-keys */
+
 import { OverrideVersionedType } from '@polkadot/types/types';
 
 // structs need to be in order

packages/types-known/src/spec/westend.ts

@@ -1,6 +1,8 @@
 // Copyright 2017-2020 @polkadot/types-known authors & contributors
 // SPDX-License-Identifier: Apache-2.0
 
+/* eslint-disable sort-keys */
+
 import { OverrideVersionedType } from '@polkadot/types/types';
 
 const sharedTypes = {

packages/types/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polkadot/types",
-  "version": "2.7.1",
+  "version": "2.9.1",
   "description": "Implementation of the Parity codec",
   "main": "index.js",
   "sideEffects": false,
@@ -17,16 +17,16 @@
   "homepage": "https://github.com/polkadot-js/api/tree/master/packages/types#readme",
   "dependencies": {
     "@babel/runtime": "^7.12.5",
-    "@polkadot/metadata": "2.7.1",
-    "@polkadot/util": "^4.1.1",
-    "@polkadot/util-crypto": "^4.1.1",
+    "@polkadot/metadata": "2.9.1",
+    "@polkadot/util": "^4.2.1",
+    "@polkadot/util-crypto": "^4.2.1",
     "@types/bn.js": "^4.11.6",
-    "bn.js": "^5.1.3",
+    "bn.js": "^4.11.9",
     "memoizee": "^0.4.14",
     "rxjs": "^6.6.3"
   },
   "devDependencies": {
-    "@polkadot/keyring": "^4.1.1",
+    "@polkadot/keyring": "^4.2.1",
     "@types/memoizee": "^0.4.5"
   }
 }

packages/types/src/augment/registry.ts

@@ -35,7 +35,7 @@ import type { ProxyAnnouncement, ProxyDefinition, ProxyType } from '@polkadot/ty
 import type { AccountStatus, AccountValidity } from '@polkadot/types/interfaces/purchase';
 import type { ActiveRecovery, RecoveryConfig } from '@polkadot/types/interfaces/recovery';
 import type { RpcMethods } from '@polkadot/types/interfaces/rpc';
-import type { AccountId, AccountIdOf, AccountIndex, Address, AssetId, Balance, BalanceOf, Block, BlockNumber, Call, CallHash, CallHashOf, ChangesTrieConfiguration, Consensus, ConsensusEngineId, Digest, DigestItem, ExtrinsicsWeight, Fixed128, Fixed64, FixedI128, FixedI64, FixedU128, FixedU64, GenericAddress, H160, H256, H512, Hash, Header, I32F32, Index, Justification, KeyTypeId, KeyValue, LockIdentifier, LookupSource, LookupTarget, ModuleId, Moment, MultiAddress, OpaqueCall, Origin, OriginCaller, PalletVersion, PalletsOrigin, Pays, PerU16, Perbill, Percent, Permill, Perquintill, Phantom, PhantomData, PreRuntime, Releases, RuntimeDbWeight, Seal, SealV0, SignedBlock, StorageData, TransactionPriority, U32F32, ValidatorId, Weight, WeightMultiplier } from '@polkadot/types/interfaces/runtime';
+import type { AccountId, AccountIdOf, AccountIndex, Address, AssetId, Balance, BalanceOf, Block, BlockNumber, Call, CallHash, CallHashOf, ChangesTrieConfiguration, Consensus, ConsensusEngineId, Digest, DigestItem, ExtrinsicsWeight, Fixed128, Fixed64, FixedI128, FixedI64, FixedU128, FixedU64, H160, H256, H512, Hash, Header, I32F32, Index, IndicesLookupSource, Justification, KeyTypeId, KeyValue, LockIdentifier, LookupSource, LookupTarget, ModuleId, Moment, MultiAddress, OpaqueCall, Origin, OriginCaller, PalletVersion, PalletsOrigin, Pays, PerU16, Perbill, Percent, Permill, Perquintill, Phantom, PhantomData, PreRuntime, Releases, RuntimeDbWeight, Seal, SealV0, SignedBlock, StorageData, TransactionPriority, U32F32, ValidatorId, Weight, WeightMultiplier } from '@polkadot/types/interfaces/runtime';
 import type { SiField, SiLookupTypeId, SiPath, SiType, SiTypeDef, SiTypeDefArray, SiTypeDefComposite, SiTypeDefPrimitive, SiTypeDefSequence, SiTypeDefTuple, SiTypeDefVariant, SiVariant } from '@polkadot/types/interfaces/scaleInfo';
 import type { Period, Priority, SchedulePeriod, SchedulePriority, Scheduled, ScheduledTo254, TaskAddress } from '@polkadot/types/interfaces/scheduler';
 import type { FullIdentification, IdentificationTuple, Keys, MembershipProof, SessionIndex, SessionKeys1, SessionKeys2, SessionKeys3, SessionKeys4, SessionKeys5, SessionKeys6, ValidatorCount } from '@polkadot/types/interfaces/session';
@@ -283,9 +283,6 @@ declare module '@polkadot/types/types/registry' {
     ExtrinsicsWeight: ExtrinsicsWeight;
     'Option<ExtrinsicsWeight>': Option<ExtrinsicsWeight>;
     'Vec<ExtrinsicsWeight>': Vec<ExtrinsicsWeight>;
-    GenericAddress: GenericAddress;
-    'Option<GenericAddress>': Option<GenericAddress>;
-    'Vec<GenericAddress>': Vec<GenericAddress>;
     H160: H160;
     'Option<H160>': Option<H160>;
     'Vec<H160>': Vec<H160>;
@@ -301,6 +298,9 @@ declare module '@polkadot/types/types/registry' {
     Header: Header;
     'Option<Header>': Option<Header>;
     'Vec<Header>': Vec<Header>;
+    IndicesLookupSource: IndicesLookupSource;
+    'Option<IndicesLookupSource>': Option<IndicesLookupSource>;
+    'Vec<IndicesLookupSource>': Vec<IndicesLookupSource>;
     Index: Index;
     'Compact<Index>': Compact<Index>;
     'Option<Index>': Option<Index>;

packages/types/src/codec/Raw.ts

@@ -95,12 +95,23 @@ export class Raw extends Uint8Array implements IU8a {
     return this.eq(decodeU8a(other));
   }
 
+  /**
+   * @description Create a new slice from the actual buffer. (compat)
+   * @param start The position to start at
+   * @param end The position to end at
+   */
+  public slice (start?: number, end?: number): Uint8Array {
+    // Like subarray below, we have to follow this approach since we are extending the TypeArray.
+    // This happens especially when it comes to further extensions, the length may be an override
+    return Uint8Array.from(this).slice(start, end);
+  }
+
   /**
    * @description Create a new subarray from the actual buffer. (compat)
    * @param begin The position to start at
    * @param end The position to end at
    */
-  public subarray (begin: number, end?: number): Uint8Array {
+  public subarray (begin?: number, end?: number): Uint8Array {
     return Uint8Array.from(this).subarray(begin, end);
   }
 

packages/types/src/generic/LookupSource.spec.ts

@@ -2,12 +2,12 @@
 // SPDX-License-Identifier: Apache-2.0
 
 import { TypeRegistry } from '../create';
-import { GenericAccountId as AccountId, GenericAccountIndex as AccountIndex, GenericLookupSource as LookuoSource } from '.';
+import { GenericAccountId as AccountId, GenericAccountIndex as AccountIndex, GenericLookupSource as LookupSource } from '.';
 
 describe('LookupSource', (): void => {
   const registry = new TypeRegistry();
 
-  const testDecode = (type: string, input: LookuoSource | AccountId | AccountIndex | number[] | Uint8Array, expected: string): void =>
+  const testDecode = (type: string, input: LookupSource | AccountId | AccountIndex | number[] | Uint8Array, expected: string): void =>
     it(`can decode from ${type}`, (): void => {
       const a = registry.createType('Address', input);
 

packages/types/src/generic/MultiAddress.ts

@@ -3,10 +3,40 @@
 
 import { Registry } from '../types';
 
-import { isString, u8aConcat } from '@polkadot/util';
+import { isBn, isNumber, isString, isU8a } from '@polkadot/util';
 import { decodeAddress } from '@polkadot/util-crypto';
 
 import { Enum } from '../codec/Enum';
+import { GenericAccountId } from './AccountId';
+import { GenericAccountIndex } from './AccountIndex';
+
+function decodeMultiU8a (registry: Registry, value?: unknown): unknown {
+  if (isU8a(value) && value.length <= 32) {
+    if (value.length === 32) {
+      return { id: value };
+    } else if (value.length === 20) {
+      return { Address20: value };
+    } else {
+      return decodeMultiAny(registry, registry.createType('AccountIndex', value));
+    }
+  }
+
+  return value;
+}
+
+function decodeMultiAny (registry: Registry, value?: unknown): unknown {
+  if (value instanceof GenericMultiAddress) {
+    return value;
+  } else if (value instanceof GenericAccountId) {
+    return { Id: value };
+  } else if (value instanceof GenericAccountIndex || isNumber(value) || isBn(value)) {
+    return { Index: registry.createType('Compact<AccountIndex>', value) };
+  } else if (isString(value)) {
+    return decodeMultiU8a(registry, decodeAddress(value.toString()));
+  }
+
+  return decodeMultiU8a(registry, value);
+}
 
 export class GenericMultiAddress extends Enum {
   constructor (registry: Registry, value?: unknown) {
@@ -18,21 +48,7 @@ export class GenericMultiAddress extends Enum {
       Address32: 'H256',
       // eslint-disable-next-line sort-keys
       Address20: 'H160'
-    }, GenericMultiAddress._decodeMultiAddress(value as string));
-  }
-
-  private static _decodeMultiAddress (value?: unknown): unknown {
-    if (isString(value)) {
-      try {
-        const u8a = decodeAddress(value.toString());
-
-        return u8aConcat(new Uint8Array(u8a.length <= 8 ? 1 : 0), u8a);
-      } catch (error) {
-        // ignore, not a valid ss58 address
-      }
-    }
-
-    return value;
+    }, decodeMultiAny(registry, value));
   }
 
   /**

packages/types/src/interfaces/runtime/definitions.ts

@@ -29,7 +29,7 @@ export default {
     AccountId: 'GenericAccountId',
     AccountIdOf: 'AccountId',
     AccountIndex: 'GenericAccountIndex',
-    Address: 'GenericAddress',
+    Address: 'LookupSource',
     AssetId: 'u32',
     Balance: 'UInt<128, Balance>',
     BalanceOf: 'Balance',
@@ -61,7 +61,6 @@ export default {
       normal: 'Weight',
       operational: 'Weight'
     },
-    GenericAddress: 'LookupSource',
     H160: '[u8; 20; H160]',
     H256: '[u8; 32; H256]',
     H512: '[u8; 64; H512]',
@@ -73,12 +72,13 @@ export default {
       extrinsicsRoot: 'Hash',
       digest: 'Digest'
     },
+    IndicesLookupSource: 'GenericLookupSource',
     Index: 'u32',
     Justification: 'Bytes',
     KeyValue: '(StorageKey, StorageData)',
     KeyTypeId: 'u32',
     LockIdentifier: '[u8; 8]',
-    LookupSource: 'GenericLookupSource',
+    LookupSource: 'IndicesLookupSource',
     LookupTarget: 'AccountId',
     ModuleId: 'LockIdentifier',
     MultiAddress: 'GenericMultiAddress',

packages/types/src/interfaces/runtime/types.ts

@@ -17,7 +17,7 @@ export interface AccountIdOf extends AccountId {}
 export interface AccountIndex extends GenericAccountIndex {}
 
 /** @name Address */
-export interface Address extends GenericAddress {}
+export interface Address extends LookupSource {}
 
 /** @name AssetId */
 export interface AssetId extends u32 {}
@@ -102,9 +102,6 @@ export interface FixedU128 extends UInt {}
 /** @name FixedU64 */
 export interface FixedU64 extends UInt {}
 
-/** @name GenericAddress */
-export interface GenericAddress extends LookupSource {}
-
 /** @name H160 */
 export interface H160 extends U8aFixed {}
 
@@ -132,6 +129,9 @@ export interface I32F32 extends Int {}
 /** @name Index */
 export interface Index extends u32 {}
 
+/** @name IndicesLookupSource */
+export interface IndicesLookupSource extends GenericLookupSource {}
+
 /** @name Justification */
 export interface Justification extends Bytes {}
 
@@ -145,7 +145,7 @@ export interface KeyValue extends ITuple<[StorageKey, StorageData]> {}
 export interface LockIdentifier extends U8aFixed {}
 
 /** @name LookupSource */
-export interface LookupSource extends GenericLookupSource {}
+export interface LookupSource extends IndicesLookupSource {}
 
 /** @name LookupTarget */
 export interface LookupTarget extends AccountId {}

packages/types/src/types/registry.ts

@@ -92,6 +92,7 @@ export interface RegistryMetadataLatest {
 
 export interface RegistryMetadata {
   asLatest: RegistryMetadataLatest;
+  version: number;
 }
 
 export interface OverrideVersionedType {