From 6e2ad7789e6fffcb406e6c8e3a4d8c17d9911a70 Mon Sep 17 00:00:00 2001 From: Jaco Greeff Date: Mon, 27 Jan 2020 17:13:53 +0100 Subject: [PATCH] Expose .entries() for DoubleMaps (#1750) * Expose .entries() for DoubleMaps * DoubleMap vs Map test * Fix decoration * Remove prefixed map * Allow SoubleMap entries() with arguments * DoubleMap with & without params * Remove debug log * Update CHANGELOG * ... and working... finally. * Cleanup debug logs --- CHANGELOG.md | 14 +++++++ packages/api/src/base/Decorate.ts | 38 ++++++++++++------- .../storage/fromMetadata/createFunction.ts | 17 +++++++-- packages/rpc-core/src/formatting.spec.ts | 9 +---- packages/types/src/interfaceRegistry.ts | 8 +++- .../src/interfaces/metadata/definitions.ts | 2 + .../types/src/interfaces/metadata/types.ts | 6 +++ packages/types/src/primitive/StorageKey.ts | 8 ++-- 8 files changed, 72 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65b49845bf..31c60d8099 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,17 @@ +# 1.0.0-beta-x + +- **Breaking change** Drop support for Substrate v1 chain in all derives +- Support `.entries(arg?: any)` lookups on DoubleMaps (in addition to previously supported maps) +- Remove un-deployed support for v11 metadata (this was decided against on Substrate) +- Allow v9 metadata to parse even in cases where it was wrongly deployed pre-v10 +- Allow type-aliasses on a per-module basis (e.g. `Judgement` in identity as well as society) +- Allow passing `{ signer }` to both `api.tx.*` and `api.sign` (in addition to `api.setSigner`) +- Add derives for the society module +- Don't map empy tuples, e.. `()` to `Null`, rather keep them empty +- Add lookups to the metadata erros via `findMetaError` +- Update metadata and types for the lastest Substrate & Polkadot versions +- Remove old known types for Substrate V1 + # 0.100.1 - **Important** This will the the last API version with Substrate 1.x support. Although you will still be able to use subsequent versions with older chans, dependent libraries such as sr25519 may not be compatible. diff --git a/packages/api/src/base/Decorate.ts b/packages/api/src/base/Decorate.ts index 0505af6495..c915235cd0 100644 --- a/packages/api/src/base/Decorate.ts +++ b/packages/api/src/base/Decorate.ts @@ -4,27 +4,24 @@ import { Constants, Storage } from '@polkadot/metadata/Decorated/types'; import { RpcInterface } from '@polkadot/rpc-core/jsonrpc.types'; +import { InterfaceRegistry } from '@polkadot/types/interfaceRegistry'; import { Call, Hash, RuntimeVersion } from '@polkadot/types/interfaces'; -import { AnyFunction, CallFunction, Codec, CodecArg as Arg, ITuple, ModulesWithCalls, Registry } from '@polkadot/types/types'; +import { AnyFunction, CallFunction, Codec, CodecArg as Arg, ITuple, InterfaceTypes, ModulesWithCalls, Registry } from '@polkadot/types/types'; import { SubmittableExtrinsic } from '../submittable/types'; -import { - ApiInterfaceRx, ApiOptions, ApiTypes, - DecorateMethod, DecoratedRpc, DecoratedRpcSection, - QueryableModuleStorage, QueryableStorage, QueryableStorageEntry, QueryableStorageMulti, QueryableStorageMultiArg, - SubmittableExtrinsicFunction, SubmittableExtrinsics, SubmittableModuleExtrinsics -} from '../types'; +import { ApiInterfaceRx, ApiOptions, ApiTypes, DecorateMethod, DecoratedRpc, DecoratedRpcSection, QueryableModuleStorage, QueryableStorage, QueryableStorageEntry, QueryableStorageMulti, QueryableStorageMultiArg, SubmittableExtrinsicFunction, SubmittableExtrinsics, SubmittableModuleExtrinsics } from '../types'; import BN from 'bn.js'; import { BehaviorSubject, Observable, combineLatest, of } from 'rxjs'; import { map, switchMap } from 'rxjs/operators'; import decorateDerive, { ExactDerive } from '@polkadot/api-derive'; +import getHasher from '@polkadot/metadata/Decorated/storage/fromMetadata/getHasher'; import RpcCore from '@polkadot/rpc-core'; import { WsProvider } from '@polkadot/rpc-provider'; import { Metadata, Null, Option, TypeRegistry, u64, Vec } from '@polkadot/types'; import Linkage, { LinkageResult } from '@polkadot/types/codec/Linkage'; import { DEFAULT_VERSION as EXTRINSIC_DEFAULT_VERSION } from '@polkadot/types/primitive/Extrinsic/constants'; import StorageKey, { StorageEntry } from '@polkadot/types/primitive/StorageKey'; -import { assert, compactStripLength, u8aToHex } from '@polkadot/util'; +import { assert, compactStripLength, isNull, isUndefined, u8aConcat, u8aToHex } from '@polkadot/util'; import { createSubmittable } from '../submittable'; import { decorateSections, DeriveAllSections } from '../util/decorate'; @@ -132,6 +129,8 @@ export default abstract class Decorate extends Events this._rx.registry = this.registry; } + public abstract createType (type: K, ...params: any[]): InterfaceRegistry[K]; + /** * @returns `true` if the API operates with subscriptions */ @@ -275,8 +274,8 @@ export default abstract class Decorate extends Events decorated.at = decorateMethod((hash: Hash, arg1?: Arg, arg2?: Arg): Observable => this._rpcCore.state.getStorage(getArgs(arg1, arg2), hash)); - decorated.entries = decorateMethod((): Observable<[StorageKey, Codec][]> => - this.retrieveMapEntries(creator)); + decorated.entries = decorateMethod((doubleMapArg?: Arg): Observable<[StorageKey, Codec][]> => + this.retrieveMapEntries(creator, doubleMapArg)); decorated.hash = decorateMethod((arg1?: Arg, arg2?: Arg): Observable => this._rpcCore.state.getStorageHash(getArgs(arg1, arg2))); @@ -369,15 +368,26 @@ export default abstract class Decorate extends Events ); } - private retrieveMapEntries (creator: StorageEntry): Observable<[StorageKey, Codec][]> { - assert(creator.meta.type.isMap, 'entries can only be retrieved on maps'); + private retrieveMapEntries (creator: StorageEntry, arg?: Arg): Observable<[StorageKey, Codec][]> { + assert(creator.iterKey && (creator.meta.type.isMap || creator.meta.type.isDoubleMap), 'entries can only be retrieved on maps, linked maps and double maps'); - const outputType = creator.meta.type.asMap.value.toString(); + const outputType = creator.meta.type.isMap + ? creator.meta.type.asMap.value.toString() + : creator.meta.type.asDoubleMap.value.toString(); return this._rpcCore.state // FIXME This should really be some sort of subscription, so we can get stuff as // it changes (as of now it is a one-shot query). Not sure how to do this though... - .getKeys(creator.iterKey) + .getKeys( + this.createType('Raw', u8aConcat( + creator.iterKey, + creator.meta.type.isDoubleMap && !isUndefined(arg) && !isNull(arg) + ? getHasher(creator.meta.type.asDoubleMap.hasher)( + this.createType(creator.meta.type.asDoubleMap.key1.toString() as 'Raw', arg).toU8a() + ) + : new Uint8Array([]) + )) + ) .pipe( switchMap((keys): Observable<[StorageKey[], Vec]> => combineLatest([ diff --git a/packages/metadata/src/Decorated/storage/fromMetadata/createFunction.ts b/packages/metadata/src/Decorated/storage/fromMetadata/createFunction.ts index a318440d54..dafc867f39 100644 --- a/packages/metadata/src/Decorated/storage/fromMetadata/createFunction.ts +++ b/packages/metadata/src/Decorated/storage/fromMetadata/createFunction.ts @@ -141,16 +141,17 @@ function expandWithMeta ({ meta, method, prefix, section }: CreateItemFn, storag } /** @internal */ -function extendHeadMeta (registry: Registry, { meta: { documentation, name, type }, section }: CreateItemFn, { method }: StorageEntry, iterFn: () => Raw): StorageKey { - const map = type.asMap; - const outputType = map.key.toString(); +function extendHeadMeta (registry: Registry, { meta: { documentation, name, type }, section }: CreateItemFn, { method }: StorageEntry, iterFn: (arg?: any) => Raw): StorageKey { + const outputType = type.isMap + ? type.asMap.key.toString() + : type.asDoubleMap.key1.toString(); // metadata with a fallback value using the type of the key, the normal // meta fallback only applies to actual entry values, create one for head (iterFn as IterFn).meta = createType(registry, 'StorageEntryMetadataLatest', { name, modifier: createType(registry, 'StorageEntryModifierLatest', 1), // required - type: createType(registry, 'StorageEntryTypeLatest', createType(registry, 'PlainTypeLatest', map.key), 0), + type: createType(registry, 'StorageEntryTypeLatest', createType(registry, 'PlainTypeLatest', type.isMap ? type.asMap.key : type.asDoubleMap.key1), 0), fallback: createType(registry, 'Bytes', createTypeUnsafe(registry, outputType).toHex()), documentation }); @@ -182,6 +183,12 @@ function extendPrefixedMap (registry: Registry, itemFn: CreateItemFn, storageFn: return storageFn; } +// attach the full list hashing for double maps +/** @internal */ +function extendDoubleMap (registry: Registry, itemFn: CreateItemFn, storageFn: StorageEntry): StorageEntry { + return extendPrefixedMap(registry, itemFn, storageFn); +} + /** @internal */ export default function createFunction (registry: Registry, itemFn: CreateItemFn, options: CreateItemOptions): StorageEntry { const { meta: { type } } = itemFn; @@ -207,6 +214,8 @@ export default function createFunction (registry: Registry, itemFn: CreateItemFn } else { extendPrefixedMap(registry, itemFn, storageFn); } + } else if (type.isDoubleMap) { + extendDoubleMap(registry, itemFn, storageFn); } return storageFn; diff --git a/packages/rpc-core/src/formatting.spec.ts b/packages/rpc-core/src/formatting.spec.ts index bddb60ee89..5230fcbf26 100644 --- a/packages/rpc-core/src/formatting.spec.ts +++ b/packages/rpc-core/src/formatting.spec.ts @@ -163,18 +163,11 @@ function formattingTests (version: string, decorated: Metadata, encodedValues: [ }); it('handles fallbacks for linked maps (new metadata only)', (done): void => { - const iterKey = decorated.query.staking.validators && decorated.query.staking.validators.iterKey; - - // skip for old - if (!iterKey) { - return done(); - } - api.state .subscribeStorage([[decorated.query.staking.validators, '0x00']]) .subscribe(([value]: Codec[]): void => { expect(value).toBeDefined(); - // console.error('linked falklback', value); + // console.error('linked fallback', value); done(); }); diff --git a/packages/types/src/interfaceRegistry.ts b/packages/types/src/interfaceRegistry.ts index b0c6271823..38f20cb731 100644 --- a/packages/types/src/interfaceRegistry.ts +++ b/packages/types/src/interfaceRegistry.ts @@ -30,7 +30,7 @@ import { Multisig, Timepoint } from '@polkadot/types/interfaces/utility'; import { BlockAttestations, IncludedBlocks, MoreAttestations } from '@polkadot/types/interfaces/attestations'; import { EthereumAddress } from '@polkadot/types/interfaces/claims'; import { AttestedCandidate, AuctionIndex, BalanceUpload, Bidder, CandidateReceipt, CollatorId, CollatorSignature, EgressQueueRoot, HeadData, IncomingParachain, IncomingParachainDeploy, IncomingParachainFixed, LeasePeriod, LeasePeriodOf, NewBidder, ParaId, ParaIdOf, ParaInfo, ParaScheduling, ParachainDispatchOrigin, Retriable, SlotRange, SubId, UpwardMessage, ValidatorIndex, ValidityAttestation, ValidityVote, WinningData, WinningDataEntry } from '@polkadot/types/interfaces/parachains'; -import { CallMetadataV0, DoubleMapTypeV10, DoubleMapTypeV3, DoubleMapTypeV4, DoubleMapTypeV5, DoubleMapTypeV6, DoubleMapTypeV7, DoubleMapTypeV8, DoubleMapTypeV9, ErrorMetadataV10, ErrorMetadataV8, ErrorMetadataV9, EventMetadataLatest, EventMetadataV0, EventMetadataV1, EventMetadataV10, EventMetadataV2, EventMetadataV3, EventMetadataV4, EventMetadataV5, EventMetadataV6, EventMetadataV7, EventMetadataV8, EventMetadataV9, FunctionArgumentMetadataLatest, FunctionArgumentMetadataV0, FunctionArgumentMetadataV1, FunctionArgumentMetadataV10, FunctionArgumentMetadataV2, FunctionArgumentMetadataV3, FunctionArgumentMetadataV4, FunctionArgumentMetadataV5, FunctionArgumentMetadataV6, FunctionArgumentMetadataV7, FunctionArgumentMetadataV8, FunctionArgumentMetadataV9, FunctionMetadataLatest, FunctionMetadataV0, FunctionMetadataV1, FunctionMetadataV10, FunctionMetadataV2, FunctionMetadataV3, FunctionMetadataV4, FunctionMetadataV5, FunctionMetadataV6, FunctionMetadataV7, FunctionMetadataV8, FunctionMetadataV9, MapTypeV0, MapTypeV10, MapTypeV2, MapTypeV3, MapTypeV4, MapTypeV5, MapTypeV6, MapTypeV7, MapTypeV8, MapTypeV9, MetadataAll, MetadataLatest, MetadataV0, MetadataV1, MetadataV10, MetadataV2, MetadataV3, MetadataV4, MetadataV5, MetadataV6, MetadataV7, MetadataV8, MetadataV9, ModuleConstantMetadataLatest, ModuleConstantMetadataV10, ModuleConstantMetadataV6, ModuleConstantMetadataV7, ModuleConstantMetadataV8, ModuleConstantMetadataV9, ModuleMetadataLatest, ModuleMetadataV0, ModuleMetadataV1, ModuleMetadataV10, ModuleMetadataV2, ModuleMetadataV3, ModuleMetadataV4, ModuleMetadataV5, ModuleMetadataV6, ModuleMetadataV7, ModuleMetadataV8, ModuleMetadataV9, OuterDispatchCallV0, OuterDispatchMetadataV0, OuterEventEventMetadataEventsV0, OuterEventEventMetadataV0, OuterEventMetadataV0, PlainTypeLatest, PlainTypeV0, PlainTypeV10, PlainTypeV2, PlainTypeV3, PlainTypeV4, PlainTypeV5, PlainTypeV6, PlainTypeV7, PlainTypeV8, PlainTypeV9, RuntimeModuleMetadataV0, StorageEntryMetadataLatest, StorageEntryMetadataV10, StorageEntryMetadataV6, StorageEntryMetadataV7, StorageEntryMetadataV8, StorageEntryMetadataV9, StorageEntryModifierLatest, StorageEntryModifierV10, StorageEntryModifierV6, StorageEntryModifierV7, StorageEntryModifierV8, StorageEntryModifierV9, StorageEntryTypeLatest, StorageEntryTypeV10, StorageEntryTypeV6, StorageEntryTypeV7, StorageEntryTypeV8, StorageEntryTypeV9, StorageFunctionMetadataV0, StorageFunctionMetadataV1, StorageFunctionMetadataV2, StorageFunctionMetadataV3, StorageFunctionMetadataV4, StorageFunctionMetadataV5, StorageFunctionModifierV0, StorageFunctionModifierV1, StorageFunctionModifierV2, StorageFunctionModifierV3, StorageFunctionModifierV4, StorageFunctionModifierV5, StorageFunctionTypeV0, StorageFunctionTypeV1, StorageFunctionTypeV2, StorageFunctionTypeV3, StorageFunctionTypeV4, StorageFunctionTypeV5, StorageHasher, StorageHasherV10, StorageHasherV4, StorageHasherV5, StorageHasherV6, StorageHasherV7, StorageHasherV8, StorageHasherV9, StorageMetadataV0, StorageMetadataV10, StorageMetadataV7, StorageMetadataV8, StorageMetadataV9 } from '@polkadot/types/interfaces/metadata'; +import { CallMetadataV0, DoubleMapTypeLatest, DoubleMapTypeV10, DoubleMapTypeV3, DoubleMapTypeV4, DoubleMapTypeV5, DoubleMapTypeV6, DoubleMapTypeV7, DoubleMapTypeV8, DoubleMapTypeV9, ErrorMetadataV10, ErrorMetadataV8, ErrorMetadataV9, EventMetadataLatest, EventMetadataV0, EventMetadataV1, EventMetadataV10, EventMetadataV2, EventMetadataV3, EventMetadataV4, EventMetadataV5, EventMetadataV6, EventMetadataV7, EventMetadataV8, EventMetadataV9, FunctionArgumentMetadataLatest, FunctionArgumentMetadataV0, FunctionArgumentMetadataV1, FunctionArgumentMetadataV10, FunctionArgumentMetadataV2, FunctionArgumentMetadataV3, FunctionArgumentMetadataV4, FunctionArgumentMetadataV5, FunctionArgumentMetadataV6, FunctionArgumentMetadataV7, FunctionArgumentMetadataV8, FunctionArgumentMetadataV9, FunctionMetadataLatest, FunctionMetadataV0, FunctionMetadataV1, FunctionMetadataV10, FunctionMetadataV2, FunctionMetadataV3, FunctionMetadataV4, FunctionMetadataV5, FunctionMetadataV6, FunctionMetadataV7, FunctionMetadataV8, FunctionMetadataV9, MapTypeLatest, MapTypeV0, MapTypeV10, MapTypeV2, MapTypeV3, MapTypeV4, MapTypeV5, MapTypeV6, MapTypeV7, MapTypeV8, MapTypeV9, MetadataAll, MetadataLatest, MetadataV0, MetadataV1, MetadataV10, MetadataV2, MetadataV3, MetadataV4, MetadataV5, MetadataV6, MetadataV7, MetadataV8, MetadataV9, ModuleConstantMetadataLatest, ModuleConstantMetadataV10, ModuleConstantMetadataV6, ModuleConstantMetadataV7, ModuleConstantMetadataV8, ModuleConstantMetadataV9, ModuleMetadataLatest, ModuleMetadataV0, ModuleMetadataV1, ModuleMetadataV10, ModuleMetadataV2, ModuleMetadataV3, ModuleMetadataV4, ModuleMetadataV5, ModuleMetadataV6, ModuleMetadataV7, ModuleMetadataV8, ModuleMetadataV9, OuterDispatchCallV0, OuterDispatchMetadataV0, OuterEventEventMetadataEventsV0, OuterEventEventMetadataV0, OuterEventMetadataV0, PlainTypeLatest, PlainTypeV0, PlainTypeV10, PlainTypeV2, PlainTypeV3, PlainTypeV4, PlainTypeV5, PlainTypeV6, PlainTypeV7, PlainTypeV8, PlainTypeV9, RuntimeModuleMetadataV0, StorageEntryMetadataLatest, StorageEntryMetadataV10, StorageEntryMetadataV6, StorageEntryMetadataV7, StorageEntryMetadataV8, StorageEntryMetadataV9, StorageEntryModifierLatest, StorageEntryModifierV10, StorageEntryModifierV6, StorageEntryModifierV7, StorageEntryModifierV8, StorageEntryModifierV9, StorageEntryTypeLatest, StorageEntryTypeV10, StorageEntryTypeV6, StorageEntryTypeV7, StorageEntryTypeV8, StorageEntryTypeV9, StorageFunctionMetadataV0, StorageFunctionMetadataV1, StorageFunctionMetadataV2, StorageFunctionMetadataV3, StorageFunctionMetadataV4, StorageFunctionMetadataV5, StorageFunctionModifierV0, StorageFunctionModifierV1, StorageFunctionModifierV2, StorageFunctionModifierV3, StorageFunctionModifierV4, StorageFunctionModifierV5, StorageFunctionTypeV0, StorageFunctionTypeV1, StorageFunctionTypeV2, StorageFunctionTypeV3, StorageFunctionTypeV4, StorageFunctionTypeV5, StorageHasher, StorageHasherV10, StorageHasherV4, StorageHasherV5, StorageHasherV6, StorageHasherV7, StorageHasherV8, StorageHasherV9, StorageMetadataV0, StorageMetadataV10, StorageMetadataV7, StorageMetadataV8, StorageMetadataV9 } from '@polkadot/types/interfaces/metadata'; import { ApiId, BlockHash, ChainProperties, ExtrinsicOrHash, ExtrinsicStatus, Health, KeyValueOption, NetworkState, PeerInfo, RpcMethods, RuntimeDispatchInfo, RuntimeVersion, RuntimeVersionApi, StorageChangeSet } from '@polkadot/types/interfaces/rpc'; export interface InterfaceRegistry { @@ -1334,6 +1334,9 @@ export interface InterfaceRegistry { StorageHasherV10: StorageHasherV10; 'Option': Option; 'Vec': Vec; + DoubleMapTypeLatest: DoubleMapTypeLatest; + 'Option': Option; + 'Vec': Vec; EventMetadataLatest: EventMetadataLatest; 'Option': Option; 'Vec': Vec; @@ -1343,6 +1346,9 @@ export interface InterfaceRegistry { FunctionMetadataLatest: FunctionMetadataLatest; 'Option': Option; 'Vec': Vec; + MapTypeLatest: MapTypeLatest; + 'Option': Option; + 'Vec': Vec; MetadataLatest: MetadataLatest; 'Option': Option; 'Vec': Vec; diff --git a/packages/types/src/interfaces/metadata/definitions.ts b/packages/types/src/interfaces/metadata/definitions.ts index 1faaa863ad..5c48eec480 100644 --- a/packages/types/src/interfaces/metadata/definitions.ts +++ b/packages/types/src/interfaces/metadata/definitions.ts @@ -435,9 +435,11 @@ export default { }, // This always maps to the latest + DoubleMapTypeLatest: 'DoubleMapTypeV10', EventMetadataLatest: 'EventMetadataV10', FunctionArgumentMetadataLatest: 'FunctionArgumentMetadataV10', FunctionMetadataLatest: 'FunctionMetadataV10', + MapTypeLatest: 'MapTypeV10', MetadataLatest: 'MetadataV10', ModuleConstantMetadataLatest: 'ModuleConstantMetadataV10', ModuleMetadataLatest: 'ModuleMetadataV10', diff --git a/packages/types/src/interfaces/metadata/types.ts b/packages/types/src/interfaces/metadata/types.ts index 2843bd5c05..f083663eca 100644 --- a/packages/types/src/interfaces/metadata/types.ts +++ b/packages/types/src/interfaces/metadata/types.ts @@ -13,6 +13,9 @@ export interface CallMetadataV0 extends Struct { readonly functions: Vec; } +/** DoubleMapTypeV10 */ +export interface DoubleMapTypeLatest extends DoubleMapTypeV10 {} + /** Struct */ export interface DoubleMapTypeV10 extends Struct { /** StorageHasherV10 */ @@ -229,6 +232,9 @@ export interface FunctionMetadataV8 extends FunctionMetadataV7 {} /** FunctionMetadataV8 */ export interface FunctionMetadataV9 extends FunctionMetadataV8 {} +/** MapTypeV10 */ +export interface MapTypeLatest extends MapTypeV10 {} + /** Struct */ export interface MapTypeV0 extends Struct { /** Type */ diff --git a/packages/types/src/primitive/StorageKey.ts b/packages/types/src/primitive/StorageKey.ts index 5c94579acf..d13ffbbcc8 100644 --- a/packages/types/src/primitive/StorageKey.ts +++ b/packages/types/src/primitive/StorageKey.ts @@ -3,7 +3,7 @@ // of the Apache-2.0 license. See the LICENSE file for details. import { StorageEntryMetadataLatest, StorageEntryTypeLatest } from '../interfaces/metadata'; -import { AnyU8a, Registry } from '../types'; +import { AnyU8a, Codec, Registry } from '../types'; import { assert, isFunction, isString, isU8a } from '@polkadot/util'; @@ -11,7 +11,7 @@ import Bytes from './Bytes'; export interface StorageEntry { (arg?: any): Uint8Array; - iterKey?: Uint8Array; + iterKey?: Uint8Array & Codec; meta: StorageEntryMetadataLatest; method: string; prefix: string; @@ -127,7 +127,9 @@ export default class StorageKey extends Bytes { } else if (Array.isArray(value)) { const [fn] = value; - return unwrapStorageType(fn.meta.type); + if (fn.meta) { + return unwrapStorageType(fn.meta.type); + } } return undefined;