From c243da79e180ba7af58ffb971e5bbb64353ba0bb Mon Sep 17 00:00:00 2001 From: Jaco Date: Wed, 5 May 2021 22:12:50 +0300 Subject: [PATCH] Prepare for auto-capability detection for historic queries (#3474) * Add auto-capability detection for historic queries * Detect * Detection via storage length * Capabilities via storage sizes * Update CHANGELOG.md * Explicit * CHANGELOG * Westend upgrade block * Pass registry explcitly * Updates * Adjust * .at remove capabilities call --- CHANGELOG.md | 3 +- packages/api/src/base/Decorate.ts | 18 +++----- packages/api/src/base/Init.ts | 23 ++++++---- packages/api/src/base/capabilities.ts | 57 +++++++++++++++---------- packages/api/src/promise/index.spec.ts | 12 ++---- packages/rpc-provider/src/mock/index.ts | 4 +- 6 files changed, 64 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d02ca5454b..84f7fc0080 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,6 @@ Upgrade priority: Medium. Required for the next v0.9.0 Polkadot/Kusama chains Changes: - Re-add isPedantic check on storage decoding -- Adjust capability detection (only on full-known historic `AccountInfo`) - `ConsensusEngineId` now implements `[u8;4]` (mapping closer to Rust source) - Additional Xcm types - Adjust parachain types @@ -17,6 +16,8 @@ Changes: - Adjust `ContractInfo` type as per latest Substrate - Adjust `DigestItem` type as per latest Substrate - Adjust re-encoding of `BitVec` based on decoded length +- Prepare for auto-capability detection with historic `.at` queries +- Add known upgrade block for Westend 900 - Remove all deprecated interfaces inside `@polkadot/api-contracts` diff --git a/packages/api/src/base/Decorate.ts b/packages/api/src/base/Decorate.ts index 800fd323d4..9a8a46bfe3 100644 --- a/packages/api/src/base/Decorate.ts +++ b/packages/api/src/base/Decorate.ts @@ -262,7 +262,8 @@ export abstract class Decorate extends Events { return allKeys; }, []); - const unknown = exposed.filter((key) => !allKeys.includes(key)); + const unknown = exposed.filter((k) => !allKeys.includes(k)); + const deletion = allKnown.filter(([k]) => hasResults && !exposed.includes(k) && k !== 'rpc_methods'); if (unknown.length) { l.warn(`RPC methods not decorated: ${unknown.join(', ')}`); @@ -270,17 +271,10 @@ export abstract class Decorate extends Events { // loop through all entries we have (populated in decorate) and filter as required // only remove when we have results and method missing, or with no results if optional - allKnown - .filter(([key]): boolean => hasResults - ? !exposed.includes(key) && key !== 'rpc_methods' // rpc_methods doesn't appear, v1 - : key === 'rpc_methods' // we didn't find this one, remove - ) - // eslint-disable-next-line @typescript-eslint/no-unused-vars - .forEach(([_, { method, section }]): void => { - delete (this._rpc as Record>)[section][method]; - delete (this._rpcCore as unknown as Record>)[section][method]; - delete (this._rx.rpc as Record>)[section][method]; - }); + deletion.forEach(([, { method, section }]): void => { + delete (this._rpc as Record>)[section][method]; + delete (this._rx.rpc as Record>)[section][method]; + }); } protected _decorateRpc (rpc: RpcCore & RpcInterface, decorateMethod: DecorateMethod, input: Partial> = {}): DecoratedRpc { diff --git a/packages/api/src/base/Init.ts b/packages/api/src/base/Init.ts index ca9842cd71..ff6ec83010 100644 --- a/packages/api/src/base/Init.ts +++ b/packages/api/src/base/Init.ts @@ -105,7 +105,9 @@ export abstract class Init extends Decorate { public async getBlockRegistry (blockHash: string | Uint8Array): Promise { // shortcut in the case where we have an immediate-same request const lastBlockHash = u8aToU8a(blockHash); - const existingViaHash = this.#registries.find((r) => r.lastBlockHash && u8aEq(lastBlockHash, r.lastBlockHash)); + const existingViaHash = this.#registries.find(({ lastBlockHash }) => + lastBlockHash && u8aEq(lastBlockHash, lastBlockHash) + ); if (existingViaHash) { return existingViaHash; @@ -129,7 +131,7 @@ export abstract class Init extends Decorate { : await this._rpcCore.state.getRuntimeVersion(header.parentHash).toPromise(); // check for pre-existing registries - const existingViaVersion = this.#registries.find((r) => r.specVersion.eq(version.specVersion)); + const existingViaVersion = this.#registries.find(({ specVersion }) => specVersion.eq(version.specVersion)); if (existingViaVersion) { existingViaVersion.lastBlockHash = lastBlockHash; @@ -141,14 +143,14 @@ export abstract class Init extends Decorate { const metadata = await this._rpcCore.state.getMetadata(header.parentHash).toPromise(); const registry = this._initRegistry(new TypeRegistry(blockHash), this._runtimeChain as Text, version, metadata); - // For now, since this is new we ignore the capability lookups (this could be useful once proven) - // this._detectCapabilities(registry, blockHash); - // add our new registry const result = { isDefault: false, lastBlockHash, metadata, metadataConsts: null, registry, specVersion: version.specVersion }; this.#registries.push(result); + // TODO This could be useful for historic, disabled due to cross-looping, i.e. .at queries + // this._detectCapabilities(registry, blockHash); + return result; } @@ -190,17 +192,17 @@ export abstract class Init extends Decorate { return [source.genesisHash, source.runtimeMetadata]; } - private _detectCapabilities (registry?: Registry, blockHash?: string | Uint8Array): void { + private _detectCapabilities (registry: Registry, blockHash?: string | Uint8Array): void { detectedCapabilities(this._rx, blockHash) .toPromise() .then((types): void => { if (Object.keys(types).length) { - (registry || this.registry).register(types as Record); + registry.register(types as Record); l.debug(() => `Capabilities detected${blockHash ? ` (${u8aToHex(u8aToU8a(blockHash))})` : ''}: ${stringify(types)}`); } }) - .catch(l.error); + .catch(undefined); } // subscribe to metadata updates, inject the types on changes @@ -235,6 +237,7 @@ export abstract class Init extends Decorate { // clear the registry types to ensure that we override correctly this._initRegistry(thisRegistry.registry.init(), this._runtimeChain as Text, version, metadata); this.injectMetadata(metadata, false, thisRegistry.registry); + this._detectCapabilities(thisRegistry.registry); return true; @@ -293,12 +296,14 @@ export abstract class Init extends Decorate { // inject metadata and adjust the types as detected this.injectMetadata(metadata, true); - this._detectCapabilities(); // derive is last, since it uses the decorated rx this._rx.derive = this._decorateDeriveRx(this._rxDecorateMethod); this._derive = this._decorateDerive(this._decorateMethod); + // detect the on-chain capabilities + this._detectCapabilities(this.registry); + return true; } diff --git a/packages/api/src/base/capabilities.ts b/packages/api/src/base/capabilities.ts index 9d169ff2a8..c5a5a6ef1e 100644 --- a/packages/api/src/base/capabilities.ts +++ b/packages/api/src/base/capabilities.ts @@ -2,18 +2,18 @@ // SPDX-License-Identifier: Apache-2.0 import type { ApiInterfaceRx } from '@polkadot/api/types'; -import type { bool, Raw, u32 } from '@polkadot/types'; +import type { Raw, u32 } from '@polkadot/types'; import type { Releases } from '@polkadot/types/interfaces'; import type { InterfaceTypes } from '@polkadot/types/types'; import { assert, compactFromU8a } from '@polkadot/util'; import { combineLatest, Observable, of } from '@polkadot/x-rxjs'; -import { map, take } from '@polkadot/x-rxjs/operators'; +import { catchError, map, take } from '@polkadot/x-rxjs/operators'; // the order and types needs to map with the all array setup below -type ExtractedQ = [bool | null, bool | null, bool | null, Releases | null]; +type ExtractedQ = [Releases | null]; -type ExtractedR = [Raw | null]; +type ExtractedR = [Raw | null, Raw | null]; type ExtractedC = [u32 | null, u32 | null]; @@ -38,20 +38,27 @@ interface AvailableMap { interface Constants { accountIdLength: number; + refcount1Length: number; + refcount2Length: number; + refcount3Length: number; } const NumberMap = ['Zero', 'One', 'Two', 'Three', 'Four', 'Five', 'Six', 'Seven', 'Eight', 'Nine', 'Ten']; -function mapCapabilities ({ accountIdLength }: Constants, [leasePeriodsPerSlot, slotRangeCount]: ExtractedC, [systemRefcount32, systemRefcountDual, systemRefcountTriple, stakingVersion]: ExtractedQ, [keys]: ExtractedR): Partial { +function mapCapabilities ({ accountIdLength, refcount1Length, refcount2Length, refcount3Length }: Constants, [leasePeriodsPerSlot, slotRangeCount]: ExtractedC, [stakingVersion]: ExtractedQ, [keys, accountInfo]: ExtractedR): Partial { const types: Partial = {}; // AccountInfo - if (systemRefcountTriple && systemRefcountTriple.isTrue) { - types.AccountInfo = 'AccountInfoWithTripleRefCount'; - } else if (systemRefcountDual && systemRefcountDual.isTrue) { - types.AccountInfo = 'AccountInfoWithDualRefCount'; - } else if (systemRefcount32 && systemRefcount32.isTrue) { - types.AccountInfo = 'AccountInfoWithRefCount'; + if (accountInfo) { + const length = accountInfo.length; + + if (length === refcount1Length) { + types.AccountInfo = 'AccountInfoWithRefCount'; + } else if (length === refcount2Length) { + types.AccountInfo = 'AccountInfoWithDualRefCount'; + } else if (length === refcount3Length) { + types.AccountInfo = 'AccountInfoWithTripleRefCount'; + } } // ValidatorPrefs @@ -88,7 +95,7 @@ function mapCapabilities ({ accountIdLength }: Constants, [leasePeriodsPerSlot, types.Keys = `SessionKeys${numIds - 1}`; } } catch (error) { - console.error(error); + console.error((error as Error).message); } } @@ -111,9 +118,12 @@ function mapCapabilities ({ accountIdLength }: Constants, [leasePeriodsPerSlot, function filterEntries (original: T[]): AvailableMap { const included = original.map((c) => !!c); - const filtered = original.filter((_, index) => included[index]); - return { filtered, included, original }; + return { + filtered: original.filter((_, index) => included[index]), + included, + original + }; } function extractResults (results: unknown[], map: AvailableMap): R { @@ -130,18 +140,17 @@ function extractResults (results: unknown[], map: AvailableMap< * @description Query the chain for the specific capabilities */ export function detectedCapabilities (api: ApiInterfaceRx, blockHash?: Uint8Array | string | undefined): Observable> { + const emptyAccountId = api.registry.createType('AccountId'); const consts = filterEntries([ api.consts.auctions?.leasePeriodsPerSlot, api.consts.auctions?.slotRangeCount ]); const queries = filterEntries([ - api.query.system?.upgradedToU32RefCount, - api.query.system?.upgradedToDualRefCount, - api.query.system?.upgradedToTripleRefCount, api.query.staking?.storageVersion ]); const raws = filterEntries([ - api.query.session?.queuedKeys + api.query.session?.queuedKeys.key(), + api.query.system?.account.key(emptyAccountId) ]); return combineLatest([ @@ -158,20 +167,24 @@ export function detectedCapabilities (api: ApiInterfaceRx, blockHash?: Uint8Arra : of([]), raws.filtered.length ? blockHash - ? combineLatest(raws.filtered.map((k) => api.rpc.state.getStorage.raw(k.key(), blockHash))) - : combineLatest(raws.filtered.map((k) => api.rpc.state.getStorage.raw(k.key()))) + ? combineLatest(raws.filtered.map((k) => api.rpc.state.getStorage.raw(k, blockHash))) + : combineLatest(raws.filtered.map((k) => api.rpc.state.getStorage.raw(k))) : of([]) ]).pipe( map(([cResults, qResults, rResults]): Partial => mapCapabilities( { - accountIdLength: api.registry.createType('AccountId').encodedLength + accountIdLength: emptyAccountId.encodedLength, + refcount1Length: api.registry.createType('AccountInfoWithRefCount').encodedLength, + refcount2Length: api.registry.createType('AccountInfoWithDualRefCount').encodedLength, + refcount3Length: api.registry.createType('AccountInfoWithTripleRefCount').encodedLength }, extractResults(cResults, consts), extractResults(qResults, queries), extractResults(rResults, raws) ) ), - take(1) + take(1), + catchError(() => of({})) ); } diff --git a/packages/api/src/promise/index.spec.ts b/packages/api/src/promise/index.spec.ts index 9331c5efc8..3e3cd5bcd4 100644 --- a/packages/api/src/promise/index.spec.ts +++ b/packages/api/src/promise/index.spec.ts @@ -1,7 +1,7 @@ // Copyright 2017-2021 @polkadot/api authors & contributors // SPDX-License-Identifier: Apache-2.0 -import type { ApiOptions, SubmittableExtrinsic } from '../types'; +import type { SubmittableExtrinsic } from '../types'; import { createPair } from '@polkadot/keyring/pair'; import { createTestKeyring } from '@polkadot/keyring/testing'; @@ -44,7 +44,7 @@ describe('ApiPromise', (): void => { } beforeEach((): void => { - jest.setTimeout(3000000); + jest.setTimeout(10000); provider = new MockProvider(registry); }); @@ -53,14 +53,10 @@ describe('ApiPromise', (): void => { const rpcData = await provider.send('state_getMetadata', []); const genesisHash = registry.createType('Hash', await provider.send('chain_getBlockHash', [])).toHex(); const specVersion = 0; - const metadata: any = {}; - const key = `${genesisHash}-${specVersion}`; - - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - metadata[key] = rpcData; + const metadata = { [`${genesisHash}-${specVersion}`]: rpcData as string }; // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - const api = await ApiPromise.create({ metadata, provider, registry } as ApiOptions); + const api = await ApiPromise.create({ metadata, provider, registry }); expect(api.genesisHash).toBeDefined(); expect(api.runtimeMetadata).toBeDefined(); diff --git a/packages/rpc-provider/src/mock/index.ts b/packages/rpc-provider/src/mock/index.ts index d5aeb8933f..5fdc3b46ad 100644 --- a/packages/rpc-provider/src/mock/index.ts +++ b/packages/rpc-provider/src/mock/index.ts @@ -122,6 +122,8 @@ export class MockProvider implements ProviderInterface { // eslint-disable-next-line @typescript-eslint/require-await public async send (method: string, params: any[]): Promise { + l.debug(() => ['send', method, params]); + assert(this.requests[method], () => `provider.send: Invalid method '${method}'`); return this.requests[method](this.db, params); @@ -129,7 +131,7 @@ export class MockProvider implements ProviderInterface { // eslint-disable-next-line @typescript-eslint/require-await public async subscribe (type: string, method: string, ...params: unknown[]): Promise { - l.debug((): any => ['subscribe', method, params]); + l.debug(() => ['subscribe', method, params]); assert(this.subscriptions[method], () => `provider.subscribe: Invalid method '${method}'`);