diff --git a/CHANGELOG.md b/CHANGELOG.md index 375a92e2bd..b73274e91a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ Contributed: Changes: +- Added `await api.at()` to retrieve API instances at a point - Added `fallbackType` for type classes (only struct, as defined) - Allow `instances` API config to drive collective locations - Align static metadata with latest Substrate diff --git a/jest.config.cjs b/jest.config.cjs index 34a85f8803..3b7f188d7b 100644 --- a/jest.config.cjs +++ b/jest.config.cjs @@ -27,6 +27,6 @@ module.exports = { '/packages/types/build', '/packages/types-known/build' ], - testTimeout: 10000, + testTimeout: 30000, transformIgnorePatterns: ['/node_modules/(?!@polkadot|@babel/runtime/helpers/esm/)'] }; diff --git a/packages/api-derive/test/helpers.ts b/packages/api-derive/test/helpers.ts index 25486abd16..81dd2d8169 100644 --- a/packages/api-derive/test/helpers.ts +++ b/packages/api-derive/test/helpers.ts @@ -14,7 +14,7 @@ export function createApiWithAugmentations (): ApiPromise { const api = new ApiPromise({ provider: new WsProvider('ws://', false), registry }); - api.injectMetadata(metadata, true); + api.injectMetadata(metadata, true, registry); return api; } diff --git a/packages/api/src/base/Decorate.ts b/packages/api/src/base/Decorate.ts index 9ba2df20f1..9212c540e7 100644 --- a/packages/api/src/base/Decorate.ts +++ b/packages/api/src/base/Decorate.ts @@ -8,7 +8,8 @@ import type { Call, Hash, RpcMethods, RuntimeVersion } from '@polkadot/types/int import type { StorageEntry } from '@polkadot/types/primitive/types'; import type { AnyFunction, AnyTuple, CallFunction, Codec, CodecArg as Arg, DefinitionRpc, DefinitionRpcSub, IMethod, InterfaceTypes, IStorageKey, Registry, RegistryTypes } from '@polkadot/types/types'; import type { SubmittableExtrinsic } from '../submittable/types'; -import type { ApiInterfaceRx, ApiOptions, ApiTypes, DecoratedErrors, DecoratedEvents, DecoratedRpc, DecoratedRpcSection, DecorateMethod, PaginationOptions, QueryableConsts, QueryableModuleStorage, QueryableStorage, QueryableStorageEntry, QueryableStorageMulti, QueryableStorageMultiArg, SubmittableExtrinsicFunction, SubmittableExtrinsics, SubmittableModuleExtrinsics } from '../types'; +import type { ApiDecoration, ApiInterfaceRx, ApiOptions, ApiTypes, DecoratedErrors, DecoratedEvents, DecoratedRpc, DecoratedRpcSection, DecorateMethod, PaginationOptions, QueryableConsts, QueryableModuleStorage, QueryableModuleStorageAt, QueryableStorage, QueryableStorageAt, QueryableStorageEntry, QueryableStorageEntryAt, QueryableStorageMulti, QueryableStorageMultiArg, SubmittableExtrinsicFunction, SubmittableExtrinsics, SubmittableModuleExtrinsics } from '../types'; +import type { VersionedRegistry } from './types'; import { decorateDerive, ExactDerive } from '@polkadot/api-derive'; import { memo } from '@polkadot/api-derive/util'; @@ -18,7 +19,7 @@ import { WsProvider } from '@polkadot/rpc-provider'; import { TypeRegistry } from '@polkadot/types/create'; import { DEFAULT_VERSION as EXTRINSIC_DEFAULT_VERSION } from '@polkadot/types/extrinsic/constants'; import { unwrapStorageType } from '@polkadot/types/primitive/StorageKey'; -import { arrayChunk, arrayFlatten, assert, BN, compactStripLength, logger, u8aToHex } from '@polkadot/util'; +import { arrayChunk, arrayFlatten, assert, BN, BN_ZERO, compactStripLength, logger, u8aToHex } from '@polkadot/util'; import { BehaviorSubject, combineLatest, Observable, of } from '@polkadot/x-rxjs'; import { map, switchMap, tap, toArray } from '@polkadot/x-rxjs/operators'; @@ -36,6 +37,11 @@ interface MetaDecoration { toJSON: () => any; } +interface FullDecoration { + decoratedApi: ApiDecoration; + decoratedMeta: DecoratedMeta; +} + // the max amount of keys/values that we will retrieve at once const PAGE_SIZE_K = 1000; // limit aligned with the 1k on the node (trie lookups are heavy) const PAGE_SIZE_V = 250; // limited since the data may be very large (e.g. misfiring elections) @@ -186,8 +192,43 @@ export abstract class Decorate extends Events { return this._rpcCore.provider.hasSubscriptions || !!this._rpcCore.state.queryStorageAt; } - public injectMetadata (metadata: Metadata, fromEmpty?: boolean, registry?: Registry): void { - const decoratedMeta = expandMetadata(registry || this.#registry, metadata); + protected _injectDecorated (registry: VersionedRegistry, fromEmpty?: boolean, blockHash?: Uint8Array): FullDecoration { + const decoratedMeta = expandMetadata(registry.registry, registry.metadata); + + // clear the decoration, we are redoing it here + if (fromEmpty || !registry.decoration) { + registry.decoration = { + consts: {}, + errors: {}, + events: {}, + query: {} + } as ApiDecoration; + } + + // adjust the versioned registry + augmentObject('consts', decoratedMeta.consts, registry.decoration.consts, fromEmpty); + augmentObject('errors', decoratedMeta.errors, registry.decoration.errors, fromEmpty); + augmentObject('events', decoratedMeta.events, registry.decoration.events, fromEmpty); + + const storage = blockHash + ? this._decorateStorageAt(decoratedMeta, this._decorateMethod, blockHash) + : this._decorateStorage(decoratedMeta, this._decorateMethod); + + augmentObject('query', storage, registry.decoration.query, fromEmpty); + + return { + decoratedApi: registry.decoration, + decoratedMeta + }; + } + + protected _injectMetadata (registry: VersionedRegistry, fromEmpty?: boolean): void { + const { decoratedApi, decoratedMeta } = this._injectDecorated(registry, fromEmpty); + + this._consts = decoratedApi.consts; + this._errors = decoratedApi.errors; + this._events = decoratedApi.events; + this._query = decoratedApi.query; if (fromEmpty || !this._extrinsics) { this._extrinsics = this._decorateExtrinsics(decoratedMeta, this._decorateMethod); @@ -197,17 +238,19 @@ export abstract class Decorate extends Events { augmentObject(null, this._decorateExtrinsics(decoratedMeta, this._rxDecorateMethod), this._rx.tx, false); } - // this API - augmentObject('query', this._decorateStorage(decoratedMeta, this._decorateMethod), this._query, fromEmpty); - augmentObject('consts', decoratedMeta.consts, this._consts, fromEmpty); - augmentObject('errors', decoratedMeta.errors, this._errors, fromEmpty); - augmentObject('events', decoratedMeta.events, this._events, fromEmpty); - // rx augmentObject(null, this._decorateStorage(decoratedMeta, this._rxDecorateMethod), this._rx.query, fromEmpty); augmentObject(null, decoratedMeta.consts, this._rx.consts, fromEmpty); } + /** + * @deprecated + * backwards compatible endpoint for metadata injection, may be removed in the future (However, it is still useful for testing injection) + */ + public injectMetadata (metadata: Metadata, fromEmpty?: boolean, registry?: Registry): void { + this._injectMetadata({ metadata, registry: registry || this.#registry, specVersion: BN_ZERO }, fromEmpty); + } + private _decorateFunctionMeta (input: MetaDecoration, output: MetaDecoration): MetaDecoration { output.meta = input.meta; output.method = input.method; @@ -353,6 +396,18 @@ export abstract class Decorate extends Events { }, {} as QueryableStorage); } + protected _decorateStorageAt ({ query }: DecoratedMeta, decorateMethod: DecorateMethod, blockHash: Uint8Array): QueryableStorageAt { + return Object.entries(query).reduce((out, [name, section]): QueryableStorageAt => { + out[name] = Object.entries(section).reduce((out, [name, method]): QueryableModuleStorageAt => { + out[name] = this._decorateStorageEntryAt(method, decorateMethod, blockHash); + + return out; + }, {} as QueryableModuleStorageAt); + + return out; + }, {} as QueryableStorageAt); + } + private _decorateStorageEntry (creator: StorageEntry, decorateMethod: DecorateMethod): QueryableStorageEntry { // get the storage arguments, with DoubleMap as an array entry, otherwise spread const getArgs = (args: unknown[]): unknown[] => extractStorageArgs(creator, args); @@ -371,7 +426,8 @@ export abstract class Decorate extends Events { this._rpcCore.state.getStorageHash(getArgs(args))); decorated.is = (key: IStorageKey): key is IStorageKey => - key.section === creator.section && key.method === creator.method; + key.section === creator.section && + key.method === creator.method; decorated.key = (...args: Arg[]): string => u8aToHex(compactStripLength(creator( @@ -435,6 +491,59 @@ export abstract class Decorate extends Events { return this._decorateFunctionMeta(creator as any, decorated) as unknown as QueryableStorageEntry; } + private _decorateStorageEntryAt (creator: StorageEntry, decorateMethod: DecorateMethod, blockHash: Uint8Array): QueryableStorageEntryAt { + // get the storage arguments, with DoubleMap as an array entry, otherwise spread + const getArgs = (args: unknown[]): unknown[] => extractStorageArgs(creator, args); + + // Disable this where it occurs for each field we are decorating + /* eslint-disable @typescript-eslint/no-unsafe-member-access,@typescript-eslint/no-unsafe-assignment */ + + const decorated = decorateMethod((...args: Arg[]): Observable => + this._rpcCore.state.getStorage(getArgs(args), blockHash)); + + decorated.creator = creator; + + decorated.hash = decorateMethod((...args: Arg[]): Observable => + this._rpcCore.state.getStorageHash(getArgs(args), blockHash)); + + decorated.is = (key: IStorageKey): key is IStorageKey => + key.section === creator.section && + key.method === creator.method; + + decorated.key = (...args: Arg[]): string => + u8aToHex(compactStripLength(creator( + creator.meta.type.isPlain + ? undefined + : creator.meta.type.isMap + ? args[0] + : creator.meta.type.isDoubleMap + ? [args[0], args[1]] + : args + ))[1]); + + decorated.keyPrefix = (...keys: Arg[]): string => + u8aToHex(creator.keyPrefix(...keys)); + + decorated.size = decorateMethod((...args: Arg[]): Observable => + this._rpcCore.state.getStorageSize(getArgs(args), blockHash)); + + // FIXME NMap support + // .keys() & .entries() only available on map types + if (creator.iterKey && (creator.meta.type.isMap || creator.meta.type.isDoubleMap)) { + decorated.entries = decorateMethod( + memo(this.#instanceId, (...args: Arg[]): Observable<[StorageKey, Codec][]> => + this._retrieveMapEntries(creator, blockHash, args))); + + decorated.keys = decorateMethod( + memo(this.#instanceId, (...args: Arg[]): Observable => + this._retrieveMapKeys(creator, blockHash, args))); + } + + /* eslint-enable @typescript-eslint/no-unsafe-member-access,@typescript-eslint/no-unsafe-assignment */ + + return this._decorateFunctionMeta(creator as any, decorated) as unknown as QueryableStorageEntry; + } + // Decorate the base storage call. In the case or rxjs or promise-without-callback (await) // we make a subscription, alternatively we push this through a single-shot query private _decorateStorageCall (creator: StorageEntry, decorateMethod: DecorateMethod): ReturnType> { diff --git a/packages/api/src/base/Getters.ts b/packages/api/src/base/Getters.ts index 7dc31fc933..ff71447c73 100644 --- a/packages/api/src/base/Getters.ts +++ b/packages/api/src/base/Getters.ts @@ -5,7 +5,7 @@ import type { Metadata } from '@polkadot/metadata'; import type { RpcInterface } from '@polkadot/rpc-core/types'; import type { Text } from '@polkadot/types'; import type { Hash, RuntimeVersion } from '@polkadot/types/interfaces'; -import type { ApiInterfaceRx, ApiTypes, DecoratedErrors, DecoratedEvents, DecoratedRpc, QueryableConsts, QueryableStorage, QueryableStorageMulti, SubmittableExtrinsics } from '../types'; +import type { ApiDecoration, ApiInterfaceRx, ApiTypes, DecoratedErrors, DecoratedEvents, DecoratedRpc, QueryableConsts, QueryableStorage, QueryableStorageMulti, SubmittableExtrinsics } from '../types'; import { assertReturn } from '@polkadot/util'; @@ -21,7 +21,7 @@ function assertResult (value: T | undefined): T { return assertReturn(value, 'Api needs to be initialized before using, listen on \'ready\''); } -export abstract class Getters extends Init { +export abstract class Getters extends Init implements ApiDecoration { /** * @description Contains the parameter types (constants) of all modules. * diff --git a/packages/api/src/base/Init.ts b/packages/api/src/base/Init.ts index 7b8d214008..b00bb922a2 100644 --- a/packages/api/src/base/Init.ts +++ b/packages/api/src/base/Init.ts @@ -7,7 +7,7 @@ import type { ChainProperties, Hash, RuntimeVersion } from '@polkadot/types/inte import type { Registry } from '@polkadot/types/types'; import type { BN } from '@polkadot/util'; import type { Observable, Subscription } from '@polkadot/x-rxjs'; -import type { ApiBase, ApiOptions, ApiTypes, DecorateMethod } from '../types'; +import type { ApiBase, ApiDecoration, ApiOptions, ApiTypes, DecorateMethod } from '../types'; import type { VersionedRegistry } from './types'; import { Metadata } from '@polkadot/metadata'; @@ -28,7 +28,7 @@ const l = logger('api/init'); export abstract class Init extends Decorate { #healthTimer: NodeJS.Timeout | null = null; - #registries: VersionedRegistry[] = []; + #registries: VersionedRegistry[] = []; #updateSub?: Subscription | null = null; @@ -44,7 +44,7 @@ export abstract class Init extends Decorate { if (!options.source) { this.registerTypes(options.types); } else { - this.#registries = options.source.#registries; + this.#registries = options.source.#registries as VersionedRegistry[]; } this._rpc = this._decorateRpc(this._rpcCore, this._decorateMethod); @@ -98,10 +98,34 @@ export abstract class Init extends Decorate { return registry; } + /** + * @description Returns the default versioned registry + */ + private _getDefaultRegistry (): VersionedRegistry { + // get the default registry version + const thisRegistry = this.#registries.find(({ isDefault }) => isDefault); + + assert(thisRegistry, 'Initialization error, cannot find the default registry'); + + return thisRegistry; + } + + /** + * @description Returns a decorated API instance at a specific point in time + */ + public async at (blockHash: Uint8Array | string): Promise> { + const u8aHash = u8aToU8a(blockHash); + const registry = await this.getBlockRegistry(u8aHash); + + return registry.decoration + ? registry.decoration + : this._injectDecorated(registry, true, u8aHash).decoratedApi; + } + /** * @description Sets up a registry based on the block hash defined */ - public async getBlockRegistry (blockHash: Uint8Array): Promise { + public async getBlockRegistry (blockHash: Uint8Array): Promise> { const existingViaHash = this.#registries.find(({ lastBlockHash }) => lastBlockHash && u8aEq(lastBlockHash, blockHash) ); @@ -147,7 +171,7 @@ export abstract class Init extends Decorate { const registry = this._initRegistry(new TypeRegistry(blockHash), this._runtimeChain as Text, version, metadata); // add our new registry - const result = { isDefault: false, lastBlockHash: blockHash, metadata, metadataConsts: null, registry, specVersion: version.specVersion }; + const result = { lastBlockHash: blockHash, metadata, registry, specVersion: version.specVersion }; this.#registries.push(result); @@ -230,18 +254,15 @@ export abstract class Init extends Decorate { this._rx.runtimeVersion = version; // update the default registry version - const thisRegistry = this.#registries.find(({ isDefault }) => isDefault); - - assert(thisRegistry, 'Initialization error, cannot find the default registry'); + const thisRegistry = this._getDefaultRegistry(); // setup the data as per the current versions thisRegistry.metadata = metadata; - thisRegistry.metadataConsts = null; thisRegistry.specVersion = version.specVersion; // clear the registry types to ensure that we override correctly this._initRegistry(thisRegistry.registry.init(), this._runtimeChain as Text, version, metadata); - this.injectMetadata(metadata, false, thisRegistry.registry); + this._injectMetadata(thisRegistry, false); return this._detectCapabilities(thisRegistry.registry); }) @@ -282,7 +303,7 @@ export abstract class Init extends Decorate { // setup the initial registry, when we have none if (!this.#registries.length) { - this.#registries.push({ isDefault: true, lastBlockHash: null, metadata, metadataConsts: null, registry: this.registry, specVersion: runtimeVersion.specVersion }); + this.#registries.push({ isDefault: true, metadata, registry: this.registry, specVersion: runtimeVersion.specVersion }); } // get unique types & validate @@ -298,7 +319,7 @@ export abstract class Init extends Decorate { this._rx.runtimeVersion = this._runtimeVersion as RuntimeVersion; // must be set here // inject metadata and adjust the types as detected - this.injectMetadata(metadata, true); + this._injectMetadata(this._getDefaultRegistry(), true); // derive is last, since it uses the decorated rx this._rx.derive = this._decorateDeriveRx(this._rxDecorateMethod); diff --git a/packages/api/src/base/types.ts b/packages/api/src/base/types.ts index 21d8ec02e8..ad1c433de1 100644 --- a/packages/api/src/base/types.ts +++ b/packages/api/src/base/types.ts @@ -2,15 +2,15 @@ // SPDX-License-Identifier: Apache-2.0 import type { Metadata } from '@polkadot/metadata'; -import type { Constants } from '@polkadot/metadata/decorate/types'; import type { Registry } from '@polkadot/types/types'; import type { BN } from '@polkadot/util'; +import type { ApiDecoration, ApiTypes } from '../types'; -export interface VersionedRegistry { - isDefault: boolean; - lastBlockHash: Uint8Array | null; +export interface VersionedRegistry { + decoration?: ApiDecoration | null; + isDefault?: boolean; + lastBlockHash?: Uint8Array | null; metadata: Metadata; - metadataConsts: Constants | null; registry: Registry; specVersion: BN; } diff --git a/packages/api/src/checkTypes.manual.ts b/packages/api/src/checkTypes.manual.ts index 49589e357a..0f33849ced 100644 --- a/packages/api/src/checkTypes.manual.ts +++ b/packages/api/src/checkTypes.manual.ts @@ -239,6 +239,16 @@ async function tx (api: ApiPromise, pairs: TestKeyringMap): Promise { } } +async function at (api: ApiPromise): Promise { + const apiAt = await api.at('0x1234'); + + // get old balances + console.log(await apiAt.query.balances.freeBalance('0x1234')); + + // get some constants + console.log(apiAt.consts.balances.existentialDeposit); +} + async function main (): Promise { const api = await ApiPromise.create(); const pairs = createTestPairs(); @@ -253,7 +263,8 @@ async function main (): Promise { queryExtra(api, pairs), rpc(api), types(api), - tx(api, pairs) + tx(api, pairs), + at(api) ]); } diff --git a/packages/api/src/types/index.ts b/packages/api/src/types/index.ts index 7b49cf5203..33483895b1 100644 --- a/packages/api/src/types/index.ts +++ b/packages/api/src/types/index.ts @@ -14,7 +14,10 @@ import type { DefinitionRpc, DefinitionRpcSub, RegisteredTypes, Registry, Signat import type { BN } from '@polkadot/util'; import type { ApiBase } from '../base'; import type { DeriveAllSections } from '../util/decorate'; +import type { ApiTypes } from './base'; import type { QueryableConsts } from './consts'; +import type { DecoratedErrors } from './errors'; +import type { DecoratedEvents } from './events'; import type { DecoratedRpc } from './rpc'; import type { QueryableStorage, QueryableStorageMulti } from './storage'; import type { SubmittableExtrinsics } from './submittable'; @@ -103,3 +106,10 @@ export interface SignerOptions extends SignatureOptions { blockNumber: BN; genesisHash: Hash; } + +export interface ApiDecoration { + consts: QueryableConsts; + errors: DecoratedErrors; + events: DecoratedEvents; + query: QueryableStorage; +} diff --git a/packages/api/src/types/storage.ts b/packages/api/src/types/storage.ts index ceaa4a1f76..a7a4583fa1 100644 --- a/packages/api/src/types/storage.ts +++ b/packages/api/src/types/storage.ts @@ -26,6 +26,11 @@ export interface StorageEntryPromiseOverloads { (arg1: CodecArg, arg2: CodecArg, arg3: CodecArg, callback: Callback): UnsubscribePromise; } +export interface StorageEntryPromiseOverloadsAt { + (arg1?: CodecArg, arg2?: CodecArg, arg3?: CodecArg): Promise; + (arg1?: CodecArg, arg2?: CodecArg, arg3?: CodecArg): Promise; +} + // This is the most generic typings we can have for a storage entry function type GenericStorageEntryFunction = (...args: CodecArg[]) => Observable @@ -36,6 +41,17 @@ export type QueryableStorageEntry & StorageEntryPromiseOverloads; +export type QueryableStorageEntryAt = + ApiType extends 'rxjs' + // eslint-disable-next-line no-use-before-define + ? AugmentedQueryAt<'rxjs', GenericStorageEntryFunction, A> + // eslint-disable-next-line no-use-before-define + : AugmentedQueryAt<'promise', GenericStorageEntryFunction, A> & StorageEntryPromiseOverloadsAt; + +export interface QueryableStorageAt extends AugmentedQueries { + [key: string]: QueryableModuleStorageAt; +} + export interface StorageEntryBase { at: >>(hash: Hash | Uint8Array | string, ...args: Parameters) => PromiseOrObs; creator: StorageEntry; @@ -56,10 +72,24 @@ export interface StorageEntryBase { + entries: >, K extends AnyTuple = A>(...args: DropLast>) => PromiseOrObs, T][]>; + hash: (...args: Parameters) => PromiseOrObs; + is: (key: IStorageKey) => key is IStorageKey; + key: (...args: Parameters) => string; + keyPrefix: (...args: DropLast>) => string; + keys: (...args: DropLast>) => PromiseOrObs[]>; + size: (...args: Parameters) => PromiseOrObs; +} + export interface QueryableModuleStorage { [index: string]: QueryableStorageEntry; } +export interface QueryableModuleStorageAt { + [index: string]: QueryableStorageEntryAt; +} + export type QueryableStorageMultiArg = QueryableStorageEntry | [QueryableStorageEntry, ...CodecArg[]]; @@ -83,5 +113,7 @@ export interface AugmentedQueries { } export type AugmentedQuery = MethodResult & StorageEntryBase; +export type AugmentedQueryAt = MethodResult & StorageEntryBaseAt; + // backwards compatibility-only export type AugmentedQueryDoubleMap = AugmentedQuery;