Raw getHeader value for metadata determination (#3490)

* Raw getHeader value for metadata determination

* Don't cache raw/json RPC calls

* getRuntimeVersion via .json RPC

* Minimal RuntimeVersion type

* getMetadata with isHistoric

* User .raw for Metadata

* Adjust types

* Updates

* Remove (now-unused) isCached

* Update packages/api/src/base/Init.ts

* Update packages/api/src/base/Init.ts

* Apply suggestions from code review

* Only decorate registry on scale

* tidy typings

* typing

* Linting updates

* CHANGELOG

* V2 contracts

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## master
 
+- **Breaking change** Support for the contracts V2 compatibility has been dropped in `@polkadot/api-contract`
+
 Contributed:
 
 - Expose RPC definitions via `.meta` (Thanks to https://github.com/jiqiang90)
@@ -17,6 +19,7 @@ Changes:
 - Remove all type overrides for node/node-template (API tracks latest)
 - Contracts ABI now supports metadata version where indexes are 0-based
 - Remove support for the old-style V2 contracts from `api-contract`
+- Adjust historic decoration, return as-at registry via `getHeader`, `getMetadata` & `getRuntimeVersion`
 
 
 ## 4.9.1 May 6, 2021

packages/api/src/base/Init.ts

@@ -1,6 +1,7 @@
 // Copyright 2017-2021 @polkadot/api authors & contributors
 // SPDX-License-Identifier: Apache-2.0
 
+import type { RpcInterfaceMethod } from '@polkadot/rpc-core/types';
 import type { Text } from '@polkadot/types';
 import type { ChainProperties, Hash, RuntimeVersion } from '@polkadot/types/interfaces';
 import type { Registry } from '@polkadot/types/types';
@@ -22,7 +23,6 @@ import { detectedCapabilities } from './capabilities';
 import { Decorate } from './Decorate';
 
 const KEEPALIVE_INTERVAL = 10000;
-const DEFAULT_BLOCKNUMBER = { unwrap: () => BN_ZERO };
 
 const l = logger('api/init');
 
@@ -116,17 +116,21 @@ export abstract class Init<ApiType extends ApiTypes> extends Decorate<ApiType> {
 
     // We have to assume that on the RPC layer the calls used here does not call back into
     // the registry swap, so getHeader & getRuntimeVersion should not be historic
-    const header = this._genesisHash.eq(blockHash)
-      ? { number: DEFAULT_BLOCKNUMBER, parentHash: this._genesisHash }
-      : await this._rpcCore.chain.getHeader(blockHash).toPromise();
+    const header = this.registry.createType('HeaderPartial',
+      this._genesisHash.eq(blockHash)
+        ? { number: BN_ZERO, parentHash: this._genesisHash }
+        : await (this._rpcCore.chain.getHeader as RpcInterfaceMethod).json(blockHash).toPromise()
+    );
 
-    assert(header?.parentHash && !header.parentHash.isEmpty, 'Unable to retrieve header and parent from supplied hash');
+    assert(!header.parentHash.isEmpty, 'Unable to retrieve header and parent from supplied hash');
 
     // get the runtime version, either on-chain or via an known upgrade history
-    const [firstVersion, lastVersion] = getUpgradeVersion(this._genesisHash, header.number.unwrap());
-    const version = (firstVersion && (lastVersion || firstVersion.specVersion.eq(this._runtimeVersion.specVersion)))
-      ? { specName: this._runtimeVersion.specName, specVersion: firstVersion.specVersion }
-      : await this._rpcCore.state.getRuntimeVersion(header.parentHash).toPromise();
+    const [firstVersion, lastVersion] = getUpgradeVersion(this._genesisHash, header.number);
+    const version = this.registry.createType('RuntimeVersionPartial',
+      (firstVersion && (lastVersion || firstVersion.specVersion.eq(this._runtimeVersion.specVersion)))
+        ? { specName: this._runtimeVersion.specName, specVersion: firstVersion.specVersion }
+        : await (this._rpcCore.state.getRuntimeVersion as RpcInterfaceMethod).json(header.parentHash).toPromise()
+    );
 
     // check for pre-existing registries
     const existingViaVersion = this.#registries.find(({ specVersion }) => specVersion.eq(version.specVersion));
@@ -138,7 +142,9 @@ export abstract class Init<ApiType extends ApiTypes> extends Decorate<ApiType> {
     }
 
     // nothing has been found, construct new
-    const metadata = await this._rpcCore.state.getMetadata(header.parentHash).toPromise();
+    const metadata = new Metadata(this.registry,
+      await (this._rpcCore.state.getMetadata as RpcInterfaceMethod).raw(header.parentHash).toPromise()
+    );
     const registry = this._initRegistry(new TypeRegistry(blockHash), this._runtimeChain as Text, version, metadata);
 
     // add our new registry

packages/rpc-core/src/index.ts

@@ -186,7 +186,7 @@ export class RpcCore {
       }, {} as RpcInterface[Section]);
   }
 
-  private _memomize <T = unknown> (creator: (outputAs: OutputType) => (...values: unknown[]) => Observable<T>, def: DefinitionRpc): Memoized<RpcInterfaceMethod> {
+  private _memomize (creator: <T> (outputAs: OutputType) => (...values: unknown[]) => Observable<T>, def: DefinitionRpc): Memoized<RpcInterfaceMethod> {
     const memoized = memoize(creator('scale') as RpcInterfaceMethod, {
       getInstanceId: () => this.#instanceId
     });
@@ -201,30 +201,29 @@ export class RpcCore {
   private _createMethodSend (section: string, method: string, def: DefinitionRpc): RpcInterfaceMethod {
     const rpcName = def.endpoint || `${section}_${method}`;
     const hashIndex = def.params.findIndex(({ isHistoric }) => isHistoric);
-    const cacheIndex = def.params.findIndex(({ isCached }) => isCached);
     let memoized: null | Memoized<RpcInterfaceMethod> = null;
 
     // execute the RPC call, doing a registry swap for historic as applicable
-    const callWithRegistry = async (outputAs: OutputType, values: unknown[]): Promise<Codec | Codec[]> => {
+    const callWithRegistry = async <T> (outputAs: OutputType, values: unknown[]): Promise<T> => {
       const blockHash = hashIndex === -1
         ? null
         : values[hashIndex] as (Uint8Array | string | null | undefined);
-      const { registry } = blockHash && this.#getBlockRegistry
+      const { registry } = outputAs === 'scale' && blockHash && this.#getBlockRegistry
         ? await this.#getBlockRegistry(u8aToU8a(blockHash))
         : { registry: this.#registryDefault };
       const params = this._formatInputs(registry, null, def, values);
       const data = await this.provider.send<AnyJson>(rpcName, params.map((param) => param.toJSON()));
 
       return outputAs === 'scale'
-        ? this._formatOutput(registry, blockHash, method, def, params, data)
-        : registry.createType(outputAs === 'raw' ? 'Raw' : 'Json', data);
+        ? this._formatOutput(registry, blockHash, method, def, params, data) as unknown as T
+        : registry.createType(outputAs === 'raw' ? 'Raw' : 'Json', data) as unknown as T;
     };
 
-    const creator = (outputAs: OutputType) => (...values: unknown[]): Observable<Codec | Codec[]> => {
-      const isDelayed = (hashIndex !== -1 && !!values[hashIndex]) || (cacheIndex !== -1 && !!values[cacheIndex]);
+    const creator = <T> (outputAs: OutputType) => (...values: unknown[]): Observable<T> => {
+      const isDelayed = outputAs === 'scale' && hashIndex !== -1 && !!values[hashIndex];
 
-      return new Observable((observer: Observer<Codec | Codec[]>): VoidCallback => {
-        callWithRegistry(outputAs, values)
+      return new Observable((observer: Observer<T>): VoidCallback => {
+        callWithRegistry<T>(outputAs, values)
           .then((value): void => {
             observer.next(value);
             observer.complete();
@@ -273,8 +272,8 @@ export class RpcCore {
     const subType = `${section}_${updateType}`;
     let memoized: null | Memoized<RpcInterfaceMethod> = null;
 
-    const creator = (outputAs: OutputType) => (...values: unknown[]): Observable<Codec | Codec[]> => {
-      return new Observable((observer: Observer<Codec | Codec[]>): VoidCallback => {
+    const creator = <T> (outputAs: OutputType) => (...values: unknown[]): Observable<T> => {
+      return new Observable((observer: Observer<T>): VoidCallback => {
         // Have at least an empty promise, as used in the unsubscribe
         let subscriptionPromise: Promise<number | string | null> = Promise.resolve(null);
         const registry = this.#registryDefault;
@@ -289,7 +288,7 @@ export class RpcCore {
           const params = this._formatInputs(registry, null, def, values);
           const paramsJson = params.map((param): AnyJson => param.toJSON());
 
-          const update = (error?: Error | null, result?: Codec | Codec[]): void => {
+          const update = (error?: Error | null, result?: T): void => {
             if (error) {
               logErrorMessage(method, def, error);
 
@@ -299,8 +298,8 @@ export class RpcCore {
             try {
               observer.next(
                 outputAs === 'scale'
-                  ? this._formatOutput(registry, null, method, def, params, result)
-                  : registry.createType(outputAs === 'raw' ? 'Raw' : 'Json', result)
+                  ? this._formatOutput(registry, null, method, def, params, result) as unknown as T
+                  : registry.createType(outputAs === 'raw' ? 'Raw' : 'Json', result) as unknown as T
               );
             } catch (error) {
               observer.error(error);

packages/rpc-core/src/types.ts

@@ -1,15 +1,15 @@
 // Copyright 2017-2021 @polkadot/rpc-core authors & contributors
 // SPDX-License-Identifier: Apache-2.0
 
+import type { Json, Raw } from '@polkadot/types';
+import type { Codec, DefinitionRpc } from '@polkadot/types/types';
 import type { Observable } from '@polkadot/x-rxjs';
 
-import { DefinitionRpc } from '@polkadot/types/types';
-
 export * from './types.jsonrpc';
 
 export interface RpcInterfaceMethod {
-  (...params: any[]): Observable<any>;
-  json (...params: any[]): Observable<any>;
-  raw (...params: any[]): Observable<any>;
+  <T extends Codec> (...params: unknown[]): Observable<T>;
+  json (...params: unknown[]): Observable<Json>;
+  raw (...params: unknown[]): Observable<Raw>;
   meta: DefinitionRpc;
 }

packages/types/src/augment/registry.ts

@@ -42,13 +42,13 @@ import type { ProxyAnnouncement, ProxyDefinition, ProxyType } from '@polkadot/ty
 import type { AccountStatus, AccountValidity } from '@polkadot/types/interfaces/purchase';
 import type { ActiveRecovery, RecoveryConfig } from '@polkadot/types/interfaces/recovery';
 import type { RpcMethods } from '@polkadot/types/interfaces/rpc';
-import type { AccountId, AccountIdOf, AccountIndex, Address, AssetId, Balance, BalanceOf, Block, BlockNumber, Call, CallHash, CallHashOf, ChangesTrieConfiguration, ChangesTrieSignal, CodecHash, Consensus, ConsensusEngineId, Digest, DigestItem, EncodedJustification, ExtrinsicsWeight, Fixed128, Fixed64, FixedI128, FixedI64, FixedU128, FixedU64, H1024, H128, H160, H2048, H256, H32, H512, H64, Hash, Header, I32F32, Index, IndicesLookupSource, Justification, Justifications, KeyTypeId, KeyValue, LockIdentifier, LookupSource, LookupTarget, ModuleId, Moment, MultiAddress, MultiSigner, OpaqueCall, Origin, OriginCaller, PalletId, PalletVersion, PalletsOrigin, Pays, PerU16, Perbill, Percent, Permill, Perquintill, Phantom, PhantomData, PreRuntime, Releases, RuntimeDbWeight, Seal, SealV0, SignedBlock, SignedBlockWithJustification, SignedBlockWithJustifications, Slot, StorageData, StorageProof, TransactionPriority, U32F32, ValidatorId, ValidatorIdOf, Weight, WeightMultiplier } from '@polkadot/types/interfaces/runtime';
+import type { AccountId, AccountIdOf, AccountIndex, Address, AssetId, Balance, BalanceOf, Block, BlockNumber, Call, CallHash, CallHashOf, ChangesTrieConfiguration, ChangesTrieSignal, CodecHash, Consensus, ConsensusEngineId, Digest, DigestItem, EncodedJustification, ExtrinsicsWeight, Fixed128, Fixed64, FixedI128, FixedI64, FixedU128, FixedU64, H1024, H128, H160, H2048, H256, H32, H512, H64, Hash, Header, HeaderPartial, I32F32, Index, IndicesLookupSource, Justification, Justifications, KeyTypeId, KeyValue, LockIdentifier, LookupSource, LookupTarget, ModuleId, Moment, MultiAddress, MultiSigner, OpaqueCall, Origin, OriginCaller, PalletId, PalletVersion, PalletsOrigin, Pays, PerU16, Perbill, Percent, Permill, Perquintill, Phantom, PhantomData, PreRuntime, Releases, RuntimeDbWeight, Seal, SealV0, SignedBlock, SignedBlockWithJustification, SignedBlockWithJustifications, Slot, StorageData, StorageProof, TransactionPriority, U32F32, ValidatorId, ValidatorIdOf, Weight, WeightMultiplier } from '@polkadot/types/interfaces/runtime';
 import type { SiField, SiLookupTypeId, SiPath, SiType, SiTypeDef, SiTypeDefArray, SiTypeDefComposite, SiTypeDefPrimitive, SiTypeDefSequence, SiTypeDefTuple, SiTypeDefVariant, SiVariant } from '@polkadot/types/interfaces/scaleInfo';
 import type { Period, Priority, SchedulePeriod, SchedulePriority, Scheduled, ScheduledTo254, TaskAddress } from '@polkadot/types/interfaces/scheduler';
 import type { BeefyKey, FullIdentification, IdentificationTuple, Keys, MembershipProof, SessionIndex, SessionKeys1, SessionKeys10, SessionKeys10B, SessionKeys2, SessionKeys3, SessionKeys4, SessionKeys5, SessionKeys6, SessionKeys6B, SessionKeys7, SessionKeys7B, SessionKeys8, SessionKeys8B, SessionKeys9, SessionKeys9B, ValidatorCount } from '@polkadot/types/interfaces/session';
 import type { Bid, BidKind, SocietyJudgement, SocietyVote, StrikeCount, VouchingStatus } from '@polkadot/types/interfaces/society';
 import type { ActiveEraInfo, CompactAssignments, CompactAssignmentsTo257, CompactScore, CompactScoreCompact, ElectionCompute, ElectionPhase, ElectionResult, ElectionScore, ElectionSize, ElectionStatus, EraIndex, EraPoints, EraRewardPoints, EraRewards, Exposure, ExtendedBalance, Forcing, IndividualExposure, KeyType, MomentOf, Nominations, NominatorIndex, NominatorIndexCompact, OffchainAccuracy, OffchainAccuracyCompact, PhragmenScore, Points, RawSolution, ReadySolution, RewardDestination, RewardPoint, RoundSnapshot, SeatHolder, SlashJournalEntry, SlashingSpans, SlashingSpansTo204, SolutionOrSnapshotSize, SolutionSupport, SolutionSupports, SpanIndex, SpanRecord, StakingLedger, StakingLedgerTo223, StakingLedgerTo240, UnappliedSlash, UnappliedSlashOther, UnlockChunk, ValidatorIndex, ValidatorIndexCompact, ValidatorPrefs, ValidatorPrefsTo145, ValidatorPrefsTo196, ValidatorPrefsWithBlocked, ValidatorPrefsWithCommission, VoteWeight, Voter } from '@polkadot/types/interfaces/staking';
-import type { ApiId, BlockTrace, BlockTraceEvent, BlockTraceEventData, BlockTraceSpan, KeyValueOption, ReadProof, RuntimeVersion, RuntimeVersionApi, StorageChangeSet, TraceBlockResponse, TraceError } from '@polkadot/types/interfaces/state';
+import type { ApiId, BlockTrace, BlockTraceEvent, BlockTraceEventData, BlockTraceSpan, KeyValueOption, ReadProof, RuntimeVersion, RuntimeVersionApi, RuntimeVersionPartial, StorageChangeSet, TraceBlockResponse, TraceError } from '@polkadot/types/interfaces/state';
 import type { WeightToFeeCoefficient } from '@polkadot/types/interfaces/support';
 import type { AccountInfo, AccountInfoWithDualRefCount, AccountInfoWithProviders, AccountInfoWithRefCount, AccountInfoWithTripleRefCount, ApplyExtrinsicResult, ArithmeticError, BlockLength, BlockWeights, ChainProperties, ChainType, ConsumedWeight, DigestOf, DispatchClass, DispatchError, DispatchErrorModule, DispatchErrorTo198, DispatchInfo, DispatchInfoTo190, DispatchInfoTo244, DispatchOutcome, DispatchResult, DispatchResultOf, DispatchResultTo198, Event, EventId, EventIndex, EventRecord, Health, InvalidTransaction, Key, LastRuntimeUpgradeInfo, NetworkState, NetworkStatePeerset, NetworkStatePeersetInfo, NodeRole, NotConnectedPeer, Peer, PeerEndpoint, PeerEndpointAddr, PeerInfo, PeerPing, PerDispatchClassU32, PerDispatchClassWeight, PerDispatchClassWeightsPerClass, Phase, RawOrigin, RefCount, RefCountTo259, SyncState, SystemOrigin, TokenError, TransactionValidityError, UnknownTransaction, WeightPerClass } from '@polkadot/types/interfaces/system';
 import type { Bounty, BountyIndex, BountyStatus, BountyStatusActive, BountyStatusCuratorProposed, BountyStatusPendingPayout, OpenTip, OpenTipFinderTo225, OpenTipTip, OpenTipTo225, TreasuryProposal } from '@polkadot/types/interfaces/treasury';
@@ -482,6 +482,7 @@ declare module '@polkadot/types/types/registry' {
     'Option<Hash>': Option<Hash>;
     'Option<HeadData>': Option<HeadData>;
     'Option<Header>': Option<Header>;
+    'Option<HeaderPartial>': Option<HeaderPartial>;
     'Option<Health>': Option<Health>;
     'Option<Heartbeat>': Option<Heartbeat>;
     'Option<HeartbeatTo244>': Option<HeartbeatTo244>;
@@ -756,6 +757,7 @@ declare module '@polkadot/types/types/registry' {
     'Option<RuntimeDispatchInfo>': Option<RuntimeDispatchInfo>;
     'Option<RuntimeVersion>': Option<RuntimeVersion>;
     'Option<RuntimeVersionApi>': Option<RuntimeVersionApi>;
+    'Option<RuntimeVersionPartial>': Option<RuntimeVersionPartial>;
     'Option<Schedule>': Option<Schedule>;
     'Option<Scheduled>': Option<Scheduled>;
     'Option<ScheduledTo254>': Option<ScheduledTo254>;
@@ -1317,6 +1319,7 @@ declare module '@polkadot/types/types/registry' {
     'Vec<Hash>': Vec<Hash>;
     'Vec<HeadData>': Vec<HeadData>;
     'Vec<Header>': Vec<Header>;
+    'Vec<HeaderPartial>': Vec<HeaderPartial>;
     'Vec<Health>': Vec<Health>;
     'Vec<Heartbeat>': Vec<Heartbeat>;
     'Vec<HeartbeatTo244>': Vec<HeartbeatTo244>;
@@ -1591,6 +1594,7 @@ declare module '@polkadot/types/types/registry' {
     'Vec<RuntimeDispatchInfo>': Vec<RuntimeDispatchInfo>;
     'Vec<RuntimeVersion>': Vec<RuntimeVersion>;
     'Vec<RuntimeVersionApi>': Vec<RuntimeVersionApi>;
+    'Vec<RuntimeVersionPartial>': Vec<RuntimeVersionPartial>;
     'Vec<Schedule>': Vec<Schedule>;
     'Vec<Scheduled>': Vec<Scheduled>;
     'Vec<ScheduledTo254>': Vec<ScheduledTo254>;
@@ -2152,6 +2156,7 @@ declare module '@polkadot/types/types/registry' {
     Hash: Hash;
     HeadData: HeadData;
     Header: Header;
+    HeaderPartial: HeaderPartial;
     Health: Health;
     Heartbeat: Heartbeat;
     HeartbeatTo244: HeartbeatTo244;
@@ -2426,6 +2431,7 @@ declare module '@polkadot/types/types/registry' {
     RuntimeDispatchInfo: RuntimeDispatchInfo;
     RuntimeVersion: RuntimeVersion;
     RuntimeVersionApi: RuntimeVersionApi;
+    RuntimeVersionPartial: RuntimeVersionPartial;
     Schedule: Schedule;
     Scheduled: Scheduled;
     ScheduledTo254: ScheduledTo254;

packages/types/src/interfaces/chain/definitions.ts

@@ -15,7 +15,7 @@ export default {
         {
           name: 'hash',
           type: 'BlockHash',
-          isCached: true,
+          isHistoric: true,
           isOptional: true
         }
       ],

packages/types/src/interfaces/runtime/definitions.ts

@@ -84,6 +84,11 @@ export default {
       extrinsicsRoot: 'Hash',
       digest: 'Digest'
     },
+    HeaderPartial: {
+      parentHash: 'Hash',
+      // since we only parse JSON with this, having non-compact works
+      number: 'BlockNumber'
+    },
     IndicesLookupSource: 'GenericLookupSource',
     Index: 'u32',
     Justification: '(ConsensusEngineId, EncodedJustification)',

packages/types/src/interfaces/runtime/types.ts

@@ -152,6 +152,12 @@ export interface Header extends Struct {
   readonly digest: Digest;
 }
 
+/** @name HeaderPartial */
+export interface HeaderPartial extends Struct {
+  readonly parentHash: Hash;
+  readonly number: BlockNumber;
+}
+
 /** @name I32F32 */
 export interface I32F32 extends Int {}
 

packages/types/src/interfaces/state/definitions.ts

@@ -256,7 +256,7 @@ export default {
         {
           name: 'at',
           type: 'BlockHash',
-          isCached: true,
+          isHistoric: true,
           isOptional: true
         }
       ],
@@ -269,7 +269,7 @@ export default {
         {
           name: 'at',
           type: 'BlockHash',
-          isCached: true,
+          isHistoric: true,
           isOptional: true
         }
       ],
@@ -412,6 +412,10 @@ export default {
       apis: 'Vec<RuntimeVersionApi>',
       transactionVersion: 'u32'
     },
+    RuntimeVersionPartial: {
+      specName: 'Text',
+      specVersion: 'u32'
+    },
     StorageChangeSet: {
       block: 'Hash',
       changes: 'Vec<KeyValueOption>'

packages/types/src/interfaces/state/types.ts

@@ -62,6 +62,12 @@ export interface RuntimeVersion extends Struct {
 /** @name RuntimeVersionApi */
 export interface RuntimeVersionApi extends ITuple<[ApiId, u32]> {}
 
+/** @name RuntimeVersionPartial */
+export interface RuntimeVersionPartial extends Struct {
+  readonly specName: Text;
+  readonly specVersion: u32;
+}
+
 /** @name StorageChangeSet */
 export interface StorageChangeSet extends Struct {
   readonly block: Hash;

packages/types/src/types/definitions.ts

@@ -12,7 +12,6 @@ export type DefinitionTypeStruct = Record<string, DefinitionTypeType> | { _alias
 export type DefinitionType = string | DefinitionTypeEnum | DefinitionTypeSet | DefinitionTypeStruct;
 
 export interface DefinitionRpcParam {
-  isCached?: boolean;
   isHistoric?: boolean;
   isOptional?: boolean;
   name: string;