Type capability detection (#3075)

* Type capability detection

* re-add Observable export

* Remove unused exports

* Decorate capabilities

* log capabilities

* Detect capabilities for historic

* Re-add logging

* Log

* Rework ValidatorPrefs types

packages/api/src/base/Init.ts

@@ -14,12 +14,13 @@ import { Metadata } from '@polkadot/metadata';
 import { TypeRegistry } from '@polkadot/types/create';
 import { LATEST_EXTRINSIC_VERSION } from '@polkadot/types/extrinsic/Extrinsic';
 import { getSpecAlias, getSpecRpc, getSpecTypes, getUpgradeVersion } from '@polkadot/types-known';
-import { assert, BN_ZERO, logger, u8aEq, u8aToU8a } from '@polkadot/util';
+import { assert, BN_ZERO, logger, u8aEq, u8aToHex, u8aToU8a } from '@polkadot/util';
 import { cryptoWaitReady } from '@polkadot/util-crypto';
 import { of } from '@polkadot/x-rxjs';
 import { map, switchMap } from '@polkadot/x-rxjs/operators';
 
 import { Decorate } from './Decorate';
+import { detectedCapabilities } from './util';
 
 const KEEPALIVE_INTERVAL = 15000;
 const DEFAULT_BLOCKNUMBER = { unwrap: () => BN_ZERO };
@@ -135,6 +136,11 @@ export abstract class Init<ApiType extends ApiTypes> extends Decorate<ApiType> {
     // nothing has been found, construct new
     const metadata = await this._rpcCore.state.getMetadata(header.parentHash).toPromise();
     const registry = this._initRegistry(new TypeRegistry(), this._runtimeChain as Text, version, metadata);
+
+    // For now, since this is new we ignore the capability lookups (this could be useful once proven)
+    // this._detectCapabilities(registry, blockHash);
+
+    // add our new registry
     const result = { isDefault: false, lastBlockHash, metadata, metadataConsts: null, registry, specVersion: version.specVersion };
 
     this.#registries.push(result);
@@ -180,6 +186,19 @@ export abstract class Init<ApiType extends ApiTypes> extends Decorate<ApiType> {
     return [source.genesisHash, source.runtimeMetadata];
   }
 
+  private _detectCapabilities (registry?: Registry, blockHash?: string | Uint8Array): void {
+    detectedCapabilities(this._rx, blockHash)
+      .toPromise()
+      .then((types): void => {
+        if (Object.keys(types).length) {
+          (registry || this.registry).register(types as Record<string, string>);
+
+          l.log(`Capabilities detected${blockHash ? ` (${u8aToHex(u8aToU8a(blockHash))})` : ''}: ${JSON.stringify(types)}`);
+        }
+      })
+      .catch(l.error);
+  }
+
   // subscribe to metadata updates, inject the types on changes
   private _subscribeUpdates (): void {
     if (this.#updateSub || !this.hasSubscriptions) {
@@ -212,6 +231,7 @@ export abstract class Init<ApiType extends ApiTypes> extends Decorate<ApiType> {
               // clear the registry types to ensure that we override correctly
               this._initRegistry(thisRegistry.registry.init(), this._runtimeChain as Text, version, metadata);
               this.injectMetadata(metadata, false, thisRegistry.registry);
+              this._detectCapabilities(thisRegistry.registry);
 
               return true;
             })
@@ -279,7 +299,9 @@ export abstract class Init<ApiType extends ApiTypes> extends Decorate<ApiType> {
     this._rx.genesisHash = this._genesisHash;
     this._rx.runtimeVersion = this._runtimeVersion;
 
+    // inject metadata and adjust the types as detected
     this.injectMetadata(metadata, true);
+    this._detectCapabilities();
 
     // derive is last, since it uses the decorated rx
     this._rx.derive = this._decorateDeriveRx(this._rxDecorateMethod);

packages/api/src/base/util.ts

@@ -0,0 +1,69 @@
+// Copyright 2017-2021 @polkadot/api-derive authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+
+import type { ApiInterfaceRx } from '@polkadot/api/types';
+import type { bool } from '@polkadot/types';
+import type { Releases } from '@polkadot/types/interfaces';
+
+import { combineLatest, Observable, of } from '@polkadot/x-rxjs';
+import { map, take } from '@polkadot/x-rxjs/operators';
+
+// the order and types needs to map with the all array setup below
+type Extracted = [bool | null, bool | null, Releases | null];
+
+interface DetectedTypes {
+  AccountInfo: 'AccountInfoWithRefCount' | 'AccountInfoWithProviders',
+  ValidatorPrefs: 'ValidatorPrefsWithBlocked' | 'ValidatorPrefsWithCommission';
+}
+
+function mapCapabilities ([systemRefcount32, systemRefcountDual, stakingVersion]: Extracted): Partial<DetectedTypes> {
+  const types: Partial<DetectedTypes> = {};
+
+  // AccountInfo
+  if (systemRefcountDual && systemRefcountDual.isTrue) {
+    types.AccountInfo = 'AccountInfoWithProviders';
+  } else if (systemRefcount32 && systemRefcount32.isTrue) {
+    types.AccountInfo = 'AccountInfoWithRefCount';
+  }
+
+  // ValidatorPrefs
+  if (stakingVersion) {
+    if (stakingVersion.index >= 4) { // v1 = index 0, V5 = index 4
+      types.ValidatorPrefs = 'ValidatorPrefsWithBlocked';
+    } else {
+      types.ValidatorPrefs = 'ValidatorPrefsWithCommission';
+    }
+  }
+
+  return types;
+}
+
+/**
+ * @description Query the chain for the specific capabilities
+ */
+export function detectedCapabilities (api: ApiInterfaceRx, blockHash?: Uint8Array | string | undefined): Observable<Partial<DetectedTypes>> {
+  const all = [
+    api.query.system?.upgradedToU32RefCount,
+    api.query.system?.upgradedToDualRefCount,
+    api.query.staking?.storageVersion
+  ];
+  const included = all.map((c) => !!c);
+  const filtered = all.filter((_, index) => included[index]);
+
+  return (
+    filtered.length
+      ? blockHash
+        ? combineLatest(filtered.map((c) => c.at(blockHash)))
+        : api.queryMulti(filtered)
+      : of([])
+  ).pipe(
+    map((results): Partial<DetectedTypes> => {
+      let offset = -1;
+
+      return mapCapabilities(
+        included.map((isIncluded) => isIncluded ? results[++offset] : null) as Extracted
+      );
+    }),
+    take(1)
+  );
+}

packages/types/src/augment/registry.ts

@@ -43,7 +43,7 @@ import type { SiField, SiLookupTypeId, SiPath, SiType, SiTypeDef, SiTypeDefArray
 import type { Period, Priority, SchedulePeriod, SchedulePriority, Scheduled, ScheduledTo254, TaskAddress } from '@polkadot/types/interfaces/scheduler';
 import type { FullIdentification, IdentificationTuple, Keys, MembershipProof, SessionIndex, SessionKeys1, SessionKeys2, SessionKeys3, SessionKeys4, SessionKeys5, SessionKeys6, SessionKeys7, SessionKeys8, SessionKeys9, ValidatorCount } from '@polkadot/types/interfaces/session';
 import type { Bid, BidKind, SocietyJudgement, SocietyVote, StrikeCount, VouchingStatus } from '@polkadot/types/interfaces/society';
-import type { ActiveEraInfo, CompactAssignments, CompactAssignmentsTo257, CompactScore, CompactScoreCompact, ElectionCompute, ElectionResult, ElectionScore, ElectionSize, ElectionStatus, EraIndex, EraPoints, EraRewardPoints, EraRewards, Exposure, Forcing, IndividualExposure, KeyType, MomentOf, Nominations, NominatorIndex, NominatorIndexCompact, OffchainAccuracy, OffchainAccuracyCompact, PhragmenScore, Points, RewardDestination, RewardDestinationTo257, RewardPoint, SlashJournalEntry, SlashingSpans, SlashingSpansTo204, SpanIndex, SpanRecord, StakingLedger, StakingLedgerTo223, StakingLedgerTo240, UnappliedSlash, UnappliedSlashOther, UnlockChunk, ValidatorIndex, ValidatorIndexCompact, ValidatorPrefs, ValidatorPrefsBlocked, ValidatorPrefsTo145, ValidatorPrefsTo196 } from '@polkadot/types/interfaces/staking';
+import type { ActiveEraInfo, CompactAssignments, CompactAssignmentsTo257, CompactScore, CompactScoreCompact, ElectionCompute, ElectionResult, ElectionScore, ElectionSize, ElectionStatus, EraIndex, EraPoints, EraRewardPoints, EraRewards, Exposure, Forcing, IndividualExposure, KeyType, MomentOf, Nominations, NominatorIndex, NominatorIndexCompact, OffchainAccuracy, OffchainAccuracyCompact, PhragmenScore, Points, RewardDestination, RewardDestinationTo257, RewardPoint, SlashJournalEntry, SlashingSpans, SlashingSpansTo204, SpanIndex, SpanRecord, StakingLedger, StakingLedgerTo223, StakingLedgerTo240, UnappliedSlash, UnappliedSlashOther, UnlockChunk, ValidatorIndex, ValidatorIndexCompact, ValidatorPrefs, ValidatorPrefsTo145, ValidatorPrefsTo196, ValidatorPrefsWithBlocked, ValidatorPrefsWithCommission } from '@polkadot/types/interfaces/staking';
 import type { ApiId, KeyValueOption, ReadProof, RuntimeVersion, RuntimeVersionApi, StorageChangeSet } from '@polkadot/types/interfaces/state';
 import type { WeightToFeeCoefficient } from '@polkadot/types/interfaces/support';
 import type { AccountInfo, AccountInfoWithProviders, AccountInfoWithRefCount, ApplyExtrinsicResult, BlockWeights, ChainProperties, ChainType, ConsumedWeight, DigestOf, DispatchClass, DispatchError, DispatchErrorModule, DispatchErrorTo198, DispatchInfo, DispatchInfoTo190, DispatchInfoTo244, DispatchOutcome, DispatchResult, DispatchResultOf, DispatchResultTo198, Event, EventId, EventIndex, EventRecord, Health, InvalidTransaction, Key, LastRuntimeUpgradeInfo, NetworkState, NetworkStatePeerset, NetworkStatePeersetInfo, NodeRole, NotConnectedPeer, Peer, PeerEndpoint, PeerEndpointAddr, PeerInfo, PeerPing, PerDispatchClass, Phase, RawOrigin, RefCount, RefCountTo259, SyncState, SystemOrigin, TransactionValidityError, UnknownTransaction, WeightPerClass } from '@polkadot/types/interfaces/system';
@@ -804,9 +804,10 @@ declare module '@polkadot/types/types/registry' {
     'Option<ValidatorIndex>': Option<ValidatorIndex>;
     'Option<ValidatorIndexCompact>': Option<ValidatorIndexCompact>;
     'Option<ValidatorPrefs>': Option<ValidatorPrefs>;
-    'Option<ValidatorPrefsBlocked>': Option<ValidatorPrefsBlocked>;
     'Option<ValidatorPrefsTo145>': Option<ValidatorPrefsTo145>;
     'Option<ValidatorPrefsTo196>': Option<ValidatorPrefsTo196>;
+    'Option<ValidatorPrefsWithBlocked>': Option<ValidatorPrefsWithBlocked>;
+    'Option<ValidatorPrefsWithCommission>': Option<ValidatorPrefsWithCommission>;
     'Option<ValidatorSignature>': Option<ValidatorSignature>;
     'Option<ValidityAttestation>': Option<ValidityAttestation>;
     'Option<VersionedMultiAsset>': Option<VersionedMultiAsset>;
@@ -1511,9 +1512,10 @@ declare module '@polkadot/types/types/registry' {
     'Vec<ValidatorIndex>': Vec<ValidatorIndex>;
     'Vec<ValidatorIndexCompact>': Vec<ValidatorIndexCompact>;
     'Vec<ValidatorPrefs>': Vec<ValidatorPrefs>;
-    'Vec<ValidatorPrefsBlocked>': Vec<ValidatorPrefsBlocked>;
     'Vec<ValidatorPrefsTo145>': Vec<ValidatorPrefsTo145>;
     'Vec<ValidatorPrefsTo196>': Vec<ValidatorPrefsTo196>;
+    'Vec<ValidatorPrefsWithBlocked>': Vec<ValidatorPrefsWithBlocked>;
+    'Vec<ValidatorPrefsWithCommission>': Vec<ValidatorPrefsWithCommission>;
     'Vec<ValidatorSignature>': Vec<ValidatorSignature>;
     'Vec<ValidityAttestation>': Vec<ValidityAttestation>;
     'Vec<VersionedMultiAsset>': Vec<VersionedMultiAsset>;
@@ -2218,9 +2220,10 @@ declare module '@polkadot/types/types/registry' {
     ValidatorIndex: ValidatorIndex;
     ValidatorIndexCompact: ValidatorIndexCompact;
     ValidatorPrefs: ValidatorPrefs;
-    ValidatorPrefsBlocked: ValidatorPrefsBlocked;
     ValidatorPrefsTo145: ValidatorPrefsTo145;
     ValidatorPrefsTo196: ValidatorPrefsTo196;
+    ValidatorPrefsWithBlocked: ValidatorPrefsWithBlocked;
+    ValidatorPrefsWithCommission: ValidatorPrefsWithCommission;
     ValidatorSignature: ValidatorSignature;
     ValidityAttestation: ValidityAttestation;
     VersionedMultiAsset: VersionedMultiAsset;

packages/types/src/interfaces/staking/definitions.ts

@@ -190,10 +190,11 @@ export default {
     },
     ValidatorIndex: 'u16',
     ValidatorIndexCompact: 'Compact<ValidatorIndex>',
-    ValidatorPrefs: {
+    ValidatorPrefs: 'ValidatorPrefsWithCommission',
+    ValidatorPrefsWithCommission: {
       commission: 'Compact<Perbill>'
     },
-    ValidatorPrefsBlocked: {
+    ValidatorPrefsWithBlocked: {
       commission: 'Compact<Perbill>',
       blocked: 'bool'
     },

packages/types/src/interfaces/staking/types.ts

@@ -262,15 +262,7 @@ export interface ValidatorIndex extends u16 {}
 export interface ValidatorIndexCompact extends Compact<ValidatorIndex> {}
 
 /** @name ValidatorPrefs */
-export interface ValidatorPrefs extends Struct {
-  readonly commission: Compact<Perbill>;
-}
-
-/** @name ValidatorPrefsBlocked */
-export interface ValidatorPrefsBlocked extends Struct {
-  readonly commission: Compact<Perbill>;
-  readonly blocked: bool;
-}
+export interface ValidatorPrefs extends ValidatorPrefsWithCommission {}
 
 /** @name ValidatorPrefsTo145 */
 export interface ValidatorPrefsTo145 extends Struct {
@@ -283,4 +275,15 @@ export interface ValidatorPrefsTo196 extends Struct {
   readonly validatorPayment: Compact<Balance>;
 }
 
+/** @name ValidatorPrefsWithBlocked */
+export interface ValidatorPrefsWithBlocked extends Struct {
+  readonly commission: Compact<Perbill>;
+  readonly blocked: bool;
+}
+
+/** @name ValidatorPrefsWithCommission */
+export interface ValidatorPrefsWithCommission extends Struct {
+  readonly commission: Compact<Perbill>;
+}
+
 export type PHANTOM_STAKING = 'staking';

packages/types/src/types/index.ts

@@ -13,4 +13,5 @@ export * from './extrinsic';
 export * from './interfaces';
 export * from './registry';
 
+// used inside augmented definitions
 export type { Observable } from '@polkadot/x-rxjs';