From adee447cd0c297a3208c4b21cc21ed7823c0187e Mon Sep 17 00:00:00 2001 From: Jaco Greeff Date: Fri, 6 Mar 2020 13:47:40 +0100 Subject: [PATCH] Add .args to decoded StorageKeys (#2004) * Add .args to decoded StorageKeys * Small opts * Formatting --- CHANGELOG.md | 5 ++ .../api-derive/src/staking/erasExposure.ts | 4 +- packages/api-derive/src/staking/validators.ts | 5 +- packages/api/src/base/Decorate.ts | 28 ++++---- packages/typegen/src/metadataMd.ts | 23 ++++--- .../types/src/primitive/StorageKey.spec.ts | 18 +++++ packages/types/src/primitive/StorageKey.ts | 65 ++++++++++++++++++- 7 files changed, 121 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a57cf3df3..be6c2a2ee0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +# 1.6.0-beta.x + +- `StorageKey` now have an `.args` property, decoded from meta in the cases twox64concat is used on maps +- Fix storage `.entries` type conversions to return exact types (not `Option` in some cases) + # 1.5.1 Mar 06, 2020 - **Important** Substrate master has updated staking, on older chains supply `StakingLedger: 'StakingLedgerTo223'` diff --git a/packages/api-derive/src/staking/erasExposure.ts b/packages/api-derive/src/staking/erasExposure.ts index 77be9619a5..0fdaba42d2 100644 --- a/packages/api-derive/src/staking/erasExposure.ts +++ b/packages/api-derive/src/staking/erasExposure.ts @@ -18,6 +18,8 @@ function queryClipped (api: ApiInterfaceRx): Observable<[EraIndex, [AccountId, E combineLatest([ of(allPoints), combineLatest( + // we could just do entries over the full set, however the resultset can be quite large - split it into + // batches - may need to re-visit this, or alternatively use pages keys for exceptionally large sets allPoints.map(({ era }): Observable<[StorageKey, Exposure][]> => api.query.staking.erasStakersClipped.entries(era) ) @@ -28,7 +30,7 @@ function queryClipped (api: ApiInterfaceRx): Observable<[EraIndex, [AccountId, E allPoints.map(({ era }, index): [EraIndex, [AccountId, Exposure][]] => [ era, erasStakers[index].map(([key, exposure]): [AccountId, Exposure] => [ - api.registry.createType('AccountId', key.toU8a().slice(-32)), + key.args[1] as AccountId, exposure ]) ]) diff --git a/packages/api-derive/src/staking/validators.ts b/packages/api-derive/src/staking/validators.ts index 5aa1cf68dc..ca183f8048 100644 --- a/packages/api-derive/src/staking/validators.ts +++ b/packages/api-derive/src/staking/validators.ts @@ -20,10 +20,7 @@ function queryNextStakers (api: ApiInterfaceRx): Observable { api.query.staking.erasStakers.entries(activeEra.addn(1)) ), map((entries): AccountId[] => - // key is from concat with AccountId bytes right at the end - entries.map(([key]): AccountId => - api.registry.createType('AccountId', key.toU8a().slice(-32)) - ) + entries.map(([key]): AccountId => key.args[1] as AccountId) ) ); } diff --git a/packages/api/src/base/Decorate.ts b/packages/api/src/base/Decorate.ts index 490c84d15b..0d2b0a2664 100644 --- a/packages/api/src/base/Decorate.ts +++ b/packages/api/src/base/Decorate.ts @@ -17,7 +17,7 @@ import DecoratedMeta from '@polkadot/metadata/Decorated'; import getHasher from '@polkadot/metadata/Decorated/storage/fromMetadata/getHasher'; import RpcCore from '@polkadot/rpc-core'; import { WsProvider } from '@polkadot/rpc-provider'; -import { Metadata, Null, Option, Text, TypeRegistry, u64, Vec } from '@polkadot/types'; +import { Metadata, Null, Option, Text, TypeRegistry, u64 } from '@polkadot/types'; import Linkage, { LinkageResult } from '@polkadot/types/codec/Linkage'; import { DEFAULT_VERSION as EXTRINSIC_DEFAULT_VERSION } from '@polkadot/types/extrinsic/constants'; import StorageKey, { StorageEntry, unwrapStorageType } from '@polkadot/types/primitive/StorageKey'; @@ -429,31 +429,33 @@ export default abstract class Decorate extends Events ); } - private retrieveMapEntries (creator: StorageEntry, arg?: Arg): Observable<[StorageKey, Codec][]> { - assert(creator.iterKey && (creator.meta.type.isMap || creator.meta.type.isDoubleMap), 'entries can only be retrieved on maps, linked maps and double maps'); + private retrieveMapEntries ({ iterKey, meta }: StorageEntry, arg?: Arg): Observable<[StorageKey, Codec][]> { + assert(iterKey && (meta.type.isMap || meta.type.isDoubleMap), 'entries can only be retrieved on maps, linked maps and double maps'); - const outputType: any = creator.meta.type.isMap - ? creator.meta.type.asMap.value.toString() - : creator.meta.type.asDoubleMap.value.toString(); + let outputType: any = unwrapStorageType(meta.type); + + if (meta.modifier.isOptional) { + outputType = `Option<${outputType}>`; + } return this._rpcCore.state // TODO This should really be some sort of subscription, so we can get stuff as // it changes (as of now it is a one-shot query). Not available on Substrate. .getKeys( this.createType('Raw', u8aConcat( - creator.iterKey, - creator.meta.type.isDoubleMap && !isUndefined(arg) && !isNull(arg) - ? getHasher(creator.meta.type.asDoubleMap.hasher)( - this.createType(creator.meta.type.asDoubleMap.key1.toString() as 'Raw', arg).toU8a() + iterKey, + meta.type.isDoubleMap && !isUndefined(arg) && !isNull(arg) + ? getHasher(meta.type.asDoubleMap.hasher)( + this.createType(meta.type.asDoubleMap.key1.toString() as 'Raw', arg).toU8a() ) : new Uint8Array([]) )) ) .pipe( - switchMap((keys): Observable<[StorageKey[], Vec]> => + switchMap((keys): Observable<[StorageKey[], Codec[]]> => combineLatest([ - of(keys), - this._rpcCore.state.subscribeStorage>(keys) + of(keys.map((key): StorageKey => key.decodeArgsFromMeta(meta))), + this._rpcCore.state.subscribeStorage(keys) ]) ), map(([keys, values]): [StorageKey, Codec][] => diff --git a/packages/typegen/src/metadataMd.ts b/packages/typegen/src/metadataMd.ts index be73f6c0b2..2a95a87148 100644 --- a/packages/typegen/src/metadataMd.ts +++ b/packages/typegen/src/metadataMd.ts @@ -64,20 +64,27 @@ function renderPage (page: Page): string { } // index - page.sections.forEach(section => { + page.sections.forEach((section) => { md += `- **[${stringCamelCase(section.name)}](#${stringCamelCase(section.name).toLowerCase()})**\n\n`; }); // contents page.sections.forEach(section => { md += `\n___\n\n\n## ${section.name}\n`; - if (section.description) { md += `\n_${section.description}_\n`; } + + if (section.description) { + md += `\n_${section.description}_\n`; + } section.items.forEach((item) => { md += ` \n### ${item.name}`; Object.keys(item).filter(i => i !== 'name').forEach(bullet => { - md += `\n- **${bullet}**: ${item[bullet] instanceof Vec ? documentationVecToMarkdown(item[bullet] as Vec, 2) : item[bullet]}`; + md += `\n- **${bullet}**: ${ + item[bullet] instanceof Vec + ? documentationVecToMarkdown(item[bullet] as Vec, 2) + : item[bullet] + }`; }); md += '\n'; @@ -116,8 +123,8 @@ function addRpc (): string { return { name: `${methodName}(${args}): ${type}`, - jsonrpc: `${sectionName}_${methodName}`, - interface: `api.rpc.${sectionName}.${methodName}`, + jsonrpc: '`' + `${sectionName}_${methodName}` + '`', + interface: '`' + `api.rpc.${sectionName}.${methodName}` + '`', ...(method.description && { summary: method.description }) }; }) @@ -146,7 +153,7 @@ function addConstants (metadata: MetadataLatest): string { return { name: `${methodName}: ` + '`' + func.type + '`', - interface: `api.consts.${sectionName}.${methodName}`, + interface: '`' + `api.consts.${sectionName}.${methodName}` + '`', ...(func.documentation.length && { summary: func.documentation }) }; }) @@ -182,7 +189,7 @@ function addStorage (metadata: MetadataLatest): string { return { name: `${methodName}(${arg}): ` + '`' + result + '`', - interface: `api.query.${sectionName}.${methodName}`, + interface: '`' + `api.query.${sectionName}.${methodName}` + '`', ...(func.documentation.length && { summary: func.documentation }) }; }) @@ -220,7 +227,7 @@ function addExtrinsics (metadata: MetadataLatest): string { return { name: `${methodName}(${args})`, - interface: `api.tx.${sectionName}.${methodName}`, + interface: '`' + `api.tx.${sectionName}.${methodName}` + '`', ...(func.documentation.length && { summary: func.documentation }) }; }) diff --git a/packages/types/src/primitive/StorageKey.spec.ts b/packages/types/src/primitive/StorageKey.spec.ts index 5688a61395..cbb3969082 100644 --- a/packages/types/src/primitive/StorageKey.spec.ts +++ b/packages/types/src/primitive/StorageKey.spec.ts @@ -9,6 +9,7 @@ import rpcDataV5 from '@polkadot/metadata/Metadata/v5/static'; import rpcDataV6 from '@polkadot/metadata/Metadata/v6/static'; import rpcDataV7 from '@polkadot/metadata/Metadata/v7/static'; import rpcDataV8 from '@polkadot/metadata/Metadata/v8/static'; +import rpcDataV11 from '@polkadot/metadata/Metadata/v11/static'; import { hexToU8a } from '@polkadot/util'; import { TypeRegistry } from '../create'; @@ -230,4 +231,21 @@ describe('StorageKey', (): void => { ).toEqual(Uint8Array.from([1, 1, 48, 7, 201, 255, 112, 39, 246, 89, 0, 171, 205, 252, 164, 253, 177, 7, 234, 212, 126, 42, 158, 53, 88, 224, 27, 105, 27, 15, 74, 95, 133, 24, 212, 135, 50, 102, 20, 240, 102, 65, 99, 8, 191, 106, 164, 229, 4, 29, 25, 73, 146, 142, 75, 38, 237, 233, 142, 60, 235, 179, 106, 59, 23, 38])); }); }); + + describe('with MetadataV11', (): void => { + const metadata = new Metadata(registry, rpcDataV11); + + it('should allow decoding of a key', (): void => { + const key = new StorageKey(registry, '0x5f3e4907f716ac89b6347d15ececedca8bde0a0ea8864605e3b68ed9cb2da01b66ccada06515787c10000000e535263148daaf49be5ddb1579b72e84524fc29e78609e3caf42e85aa118ebfe0b0ad404b5bdd25f'); + + key.setMeta(metadata.query.staking.erasStakers.meta); + + expect( + key.args.map((arg) => arg.toHuman()) + ).toEqual([ + '16', + '5GNJqTPyNqANBkUVMN1LPPrxXnFouWXoe2wNSmmEoLctxiZY' + ]); + }); + }); }); diff --git a/packages/types/src/primitive/StorageKey.ts b/packages/types/src/primitive/StorageKey.ts index e5737f024a..e4337509c3 100644 --- a/packages/types/src/primitive/StorageKey.ts +++ b/packages/types/src/primitive/StorageKey.ts @@ -2,7 +2,7 @@ // This software may be modified and distributed under the terms // of the Apache-2.0 license. See the LICENSE file for details. -import { StorageEntryMetadataLatest, StorageEntryTypeLatest } from '../interfaces/metadata'; +import { StorageEntryMetadataLatest, StorageEntryTypeLatest, StorageHasher } from '../interfaces/metadata'; import { AnyU8a, Codec, InterfaceTypes, Registry } from '../types'; import { assert, isFunction, isString, isU8a } from '@polkadot/util'; @@ -82,6 +82,46 @@ function decodeStorageKey (value?: AnyU8a | StorageKey | StorageEntry | [Storage throw new Error(`Unable to convert input ${value} to StorageKey`); } +function decodeHashers (registry: Registry, encoded: Uint8Array, hashers: [StorageHasher, string][]): Codec[] { + let offset = 0; + + return hashers.reduce((result: Codec[], [hasher, type]): Codec[] => { + const decoded = hasher.isTwox64Concat + ? registry.createType(type as any, encoded.subarray(offset + 8)) + : registry.createType('Null'); + + offset += hasher.isTwox64Concat ? (8 + decoded.encodedLength) : 0; + result.push(decoded); + + return result; + }, []); +} + +/** @internal */ +function decodeArgsFromMeta (registry: Registry, value: Uint8Array, meta?: StorageEntryMetadataLatest): Codec[] { + if (!meta || !(meta.type.isDoubleMap || meta.type.isMap)) { + return []; + } + + // the prefix is xxhashAsU8a(prefix, 128) + xxhashAsU8a(method, 128), 256 bits total + const encoded = value.subarray(32); + + if (meta.type.isMap) { + const mapInfo = meta.type.asMap; + + return decodeHashers(registry, encoded, [ + [mapInfo.hasher, mapInfo.value.toString()] + ]); + } + + const mapInfo = meta.type.asDoubleMap; + + return decodeHashers(registry, encoded, [ + [mapInfo.hasher, mapInfo.key1.toString()], + [mapInfo.key2Hasher, mapInfo.key2.toString()] + ]); +} + /** * @name StorageKey * @description @@ -89,6 +129,10 @@ function decodeStorageKey (value?: AnyU8a | StorageKey | StorageEntry | [Storage * constructed by passing in a raw key or a StorageEntry with (optional) arguments. */ export default class StorageKey extends Bytes { + // eslint-disable-next-line @typescript-eslint/ban-ts-ignore + // @ts-ignore This is assigned via this.decodeArgsFromMeta() + private _args: Codec[]; + private _meta?: StorageEntryMetadataLatest; private readonly _method?: string; @@ -106,6 +150,9 @@ export default class StorageKey extends Bytes { this._method = override.method || method; this._outputType = StorageKey.getType(value as StorageKey); this._section = override.section || section; + + // decode the args (as applicable based on the key and the hashers, after all init) + this.decodeArgsFromMeta(); } public static getMeta (value: StorageKey | StorageEntry | [StorageEntry, any]): StorageEntryMetadataLatest | undefined { @@ -139,6 +186,13 @@ export default class StorageKey extends Bytes { return 'Raw'; } + /** + * @description Return the decoded arguments (applicable to map/doublemap with decodable values) + */ + public get args (): Codec[] { + return this._args; + } + /** * @description The metadata or `undefined` when not available */ @@ -173,6 +227,15 @@ export default class StorageKey extends Bytes { public setMeta (meta?: StorageEntryMetadataLatest): this { this._meta = meta; + return this.decodeArgsFromMeta(); + } + + /** + * @description Decode the args embedded in the key (assuming we have decodable hashers) + */ + public decodeArgsFromMeta (meta?: StorageEntryMetadataLatest): this { + this._args = decodeArgsFromMeta(this.registry, this.toU8a(true), meta || this.meta); + return this; } }