From 7fd45f63dfb63b13a407474bee41334c7be7c4af Mon Sep 17 00:00:00 2001 From: Jaco Greeff Date: Fri, 25 Sep 2020 10:19:28 +0200 Subject: [PATCH] Drop support for linked lists (#2643) * Drop support for linked lists * CHANGELOG * Remove all .linked. checks --- CHANGELOG.md | 2 + packages/api-derive/src/council/votes.ts | 32 ++----- packages/api-derive/src/staking/stashes.ts | 14 +-- packages/api/src/base/Decorate.ts | 91 +------------------ packages/api/src/util/validate.ts | 6 +- .../storage/fromMetadata/createFunction.ts | 22 +---- packages/types/src/primitive/StorageKey.ts | 16 +--- 7 files changed, 23 insertions(+), 160 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5be7f89851..388f8f95b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,12 +3,14 @@ ## 1.35.0-beta.x - **Breaking change** The `RefCount` type (used by `query.system.accounts`) has been changed to a `u32`. On older chains supply the `RefCount: 'RefCountTo259'` override +- **Breaking change** Support for the old-style linked-map retrievals have been dropped, only `.keys()/.entries()` are available now for map iteration. Al known live chains have been upgraded to versions that don't have linked lists. Changes: - Adjust `RefCount` type as per substrate 2.0 - Add bounty type definitions from treasury - Add support for `grandpa_proveFinality` RPC +- Drop support for linked maps (deprecated since early versions of Substrate 2) ## 1.34.1 Sep 21, 2020 diff --git a/packages/api-derive/src/council/votes.ts b/packages/api-derive/src/council/votes.ts index 04e4355cbb..7fd160aa8f 100644 --- a/packages/api-derive/src/council/votes.ts +++ b/packages/api-derive/src/council/votes.ts @@ -13,35 +13,19 @@ import { Vec } from '@polkadot/types'; import { memo } from '../util'; function retrieveStakeOf (api: ApiInterfaceRx): Observable<[AccountId, Balance][]> { - const elections = (api.query.electionsPhragmen || api.query.elections); - - return elections.stakeOf.creator.meta.type.asMap.linked.isTrue - ? elections.stakeOf, Vec]>>().pipe( - map(([voters, stake]) => - voters.map((voter, index): [AccountId, Balance] => [voter, stake[index]]) - ) + return (api.query.electionsPhragmen || api.query.elections).stakeOf.entries().pipe( + map((entries) => + entries.map(([key, stake]) => [key.args[0] as AccountId, stake]) ) - : elections.stakeOf.entries().pipe( - map((entries) => - entries.map(([key, stake]) => [key.args[0] as AccountId, stake]) - ) - ); + ); } function retrieveVoteOf (api: ApiInterfaceRx): Observable<[AccountId, AccountId[]][]> { - const elections = (api.query.electionsPhragmen || api.query.elections); - - return elections.votesOf.creator.meta.type.asMap.linked.isTrue - ? elections.votesOf, Vec>]>>().pipe( - map(([voters, votes]) => - voters.map((voter, index) => [voter, votes[index]]) - ) + return (api.query.electionsPhragmen || api.query.elections).votesOf.entries>().pipe( + map((entries) => + entries.map(([key, votes]) => [key.args[0] as AccountId, votes]) ) - : elections.votesOf.entries>().pipe( - map((entries) => - entries.map(([key, votes]) => [key.args[0] as AccountId, votes]) - ) - ); + ); } function retrievePrev (api: ApiInterfaceRx): Observable { diff --git a/packages/api-derive/src/staking/stashes.ts b/packages/api-derive/src/staking/stashes.ts index c861b8c42e..a6c61d9be6 100644 --- a/packages/api-derive/src/staking/stashes.ts +++ b/packages/api-derive/src/staking/stashes.ts @@ -2,12 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 import { ApiInterfaceRx } from '@polkadot/api/types'; -import { AccountId, ValidatorPrefs } from '@polkadot/types/interfaces'; -import { ITuple } from '@polkadot/types/types'; +import { AccountId } from '@polkadot/types/interfaces'; import { Observable } from 'rxjs'; import { map } from 'rxjs/operators'; -import { Vec } from '@polkadot/types'; import { memo } from '../util'; @@ -16,12 +14,8 @@ import { memo } from '../util'; */ export function stashes (instanceId: string, api: ApiInterfaceRx): () => Observable { return memo(instanceId, (): Observable => - api.query.staking.validators.creator.meta.type.asMap.linked.isTrue - ? api.query.staking.validators, Vec]>>().pipe( - map(([stashIds]) => stashIds) - ) - : api.query.staking.validators.keys().pipe( - map((keys) => keys.map((key) => key.args[0] as AccountId).filter((a) => a)) - ) + api.query.staking.validators.keys().pipe( + map((keys) => keys.map((key) => key.args[0] as AccountId).filter((a) => a)) + ) ); } diff --git a/packages/api/src/base/Decorate.ts b/packages/api/src/base/Decorate.ts index 52eea2c0f8..dcfe4372fe 100644 --- a/packages/api/src/base/Decorate.ts +++ b/packages/api/src/base/Decorate.ts @@ -4,7 +4,7 @@ import { Constants, Storage } from '@polkadot/metadata/Decorated/types'; import { RpcInterface } from '@polkadot/rpc-core/types'; import { Call, Hash, RuntimeVersion } from '@polkadot/types/interfaces'; -import { AnyFunction, CallFunction, Codec, CodecArg as Arg, ITuple, InterfaceTypes, ModulesWithCalls, Registry, RegistryTypes } from '@polkadot/types/types'; +import { AnyFunction, CallFunction, Codec, CodecArg as Arg, InterfaceTypes, ModulesWithCalls, Registry, RegistryTypes } from '@polkadot/types/types'; import { SubmittableExtrinsic } from '../submittable/types'; import { ApiInterfaceRx, ApiOptions, ApiTypes, DecorateMethod, DecoratedRpc, DecoratedRpcSection, PaginationOptions, QueryableConsts, QueryableModuleStorage, QueryableStorage, QueryableStorageEntry, QueryableStorageMulti, QueryableStorageMultiArg, SubmittableExtrinsicFunction, SubmittableExtrinsics, SubmittableModuleExtrinsics } from '../types'; @@ -16,8 +16,7 @@ import { memo } from '@polkadot/api-derive/util'; import DecoratedMeta from '@polkadot/metadata/Decorated'; import RpcCore from '@polkadot/rpc-core'; import { WsProvider } from '@polkadot/rpc-provider'; -import { Metadata, Null, Option, Raw, Text, TypeRegistry, u64 } from '@polkadot/types'; -import Linkage, { LinkageResult } from '@polkadot/types/codec/Linkage'; +import { Metadata, Option, Raw, Text, TypeRegistry, u64 } from '@polkadot/types'; import { DEFAULT_VERSION as EXTRINSIC_DEFAULT_VERSION } from '@polkadot/types/extrinsic/constants'; import StorageKey, { StorageEntry, unwrapStorageType } from '@polkadot/types/primitive/StorageKey'; import { assert, compactStripLength, logger, u8aToHex } from '@polkadot/util'; @@ -36,8 +35,6 @@ interface MetaDecoration { toJSON: () => any; } -type LinkageData = ITuple<[Codec, Linkage]>; - const PAGE_SIZE_KEYS = 256; const PAGE_SIZE_VALS = PAGE_SIZE_KEYS; @@ -336,10 +333,7 @@ export default abstract class Decorate extends Events // Disable this where it occurs for each field we are decorating /* eslint-disable @typescript-eslint/no-unsafe-member-access,@typescript-eslint/no-unsafe-assignment */ - // FIXME We probably want to be able to query the full list with non-subs as well - const decorated = this.hasSubscriptions && creator.iterKey && creator.meta.type.isMap && creator.meta.type.asMap.linked.isTrue - ? this._decorateStorageLinked(creator, decorateMethod) - : this._decorateStorageCall(creator, decorateMethod); + const decorated = this._decorateStorageCall(creator, decorateMethod); decorated.creator = creator; @@ -422,85 +416,6 @@ export default abstract class Decorate extends Events )); } - // This is deprecated/removed on substrate - we still have some live chains, eg. Edgeware needing it, so it - // has not been dropped yet. Slated for removal in API 2.0 - private _decorateStorageLinked (creator: StorageEntry, decorateMethod: DecorateMethod): ReturnType> { - const result = new Map]> | null>(); - let subject: BehaviorSubject; - let head: Codec | null = null; - const iterKey = creator.iterKey; - - assert(iterKey, 'iterKey field is missing'); - - // retrieve a value based on the key, iterating if it has a next entry. Since - // entries can be re-linked in the middle of a list, we subscribe here to make - // sure we catch any updates, no matter the list position - const getNext = (key: Codec): Observable => - this._rpcCore.state.getStorage>([creator, key]).pipe( - switchMap((_data: LinkageData | Option): Observable => { - const data = creator.meta.modifier.isOptional - ? (_data as Option).unwrapOr(null) - : _data as LinkageData; - - result.set(key, data); - - // iterate from this key to the linkages, constructing entries for all - // those found and available - if (data && data[1].next.isSome) { - return getNext(data[1].next.unwrap()); - } - - const [keys, vals]: [Codec[], Codec[]] = [[], []]; - let nextKey = head; - - // loop through the results collected, starting at the head an re-creating - // the list. Our map may have old entries, based on the linking these will - // not be returned in the final result - while (nextKey) { - const entry = result.get(nextKey); - - if (!entry) { - break; - } - - const [item, linkage] = entry; - - keys.push(nextKey); - vals.push(item); - - nextKey = linkage.next?.unwrapOr(null); - } - - const nextResult = vals.length - ? new LinkageResult(this.#registry, [keys[0].constructor as any, keys], [vals[0].constructor as any, vals]) - : new LinkageResult(this.#registry, [Null, []], [Null, []]); - - // we set our result into a subject so we have a single observable for - // which the value changes over time. Initially create, follow-up next - if (subject) { - subject.next(nextResult); - } else { - subject = new BehaviorSubject(nextResult); - } - - return subject; - })); - - // this handles the case where the head changes effectively, i.e. a new entry - // appears at the top of the list, the new getNext gets kicked off - // eslint-disable-next-line @typescript-eslint/no-unsafe-return - return decorateMethod((...args: unknown[]): Observable]> => - args.length - ? this._rpcCore.state - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - .subscribeStorage<[[Codec, Linkage]]>([[creator, ...args]]) - .pipe(map(([data]) => data)) - : this._rpcCore.state - .subscribeStorage<[LinkageResult]>([iterKey()]) - .pipe(switchMap(([key]) => getNext(head = key))) - ); - } - private _retrieveMapKeys ({ iterKey, meta }: StorageEntry, arg?: Arg): Observable { assert(iterKey && (meta.type.isMap || meta.type.isDoubleMap), 'keys can only be retrieved on maps, linked maps and double maps'); diff --git a/packages/api/src/util/validate.ts b/packages/api/src/util/validate.ts index ee438ee45c..93653a154a 100644 --- a/packages/api/src/util/validate.ts +++ b/packages/api/src/util/validate.ts @@ -20,11 +20,9 @@ function doDoubleMap (creator: StorageEntry, args: unknown[]): [StorageEntry, [a } function doMap (creator: StorageEntry, args: unknown[]): [StorageEntry] | [StorageEntry, any] { - const { key, linked } = creator.meta.type.asMap; + const { key } = creator.meta.type.asMap; - linked.isTrue - ? assert(args.length <= 1, `${sig(creator, key)} is a linked map, requiring either 0 arguments (retrieving all) or 1 argument, ${args.length} found`) - : assert(args.length === 1, `${sig(creator, key)} is a map, requiring 1 argument, ${args.length} found`); + assert(args.length === 1, `${sig(creator, key)} is a map, requiring 1 argument, ${args.length} found`); // expand return args.length diff --git a/packages/metadata/src/Decorated/storage/fromMetadata/createFunction.ts b/packages/metadata/src/Decorated/storage/fromMetadata/createFunction.ts index 7adb187af2..0d1ced823a 100644 --- a/packages/metadata/src/Decorated/storage/fromMetadata/createFunction.ts +++ b/packages/metadata/src/Decorated/storage/fromMetadata/createFunction.ts @@ -161,20 +161,6 @@ function extendHeadMeta (registry: Registry, { meta: { documentation, name, type : prefixKey; } -// attach the head key hashing for linked maps -/** @internal */ -function extendLinkedMap (registry: Registry, itemFn: CreateItemFn, storageFn: StorageEntry, stringKey: string, hasher: HasherFunction, metaVersion: number): StorageEntry { - const key = metaVersion <= 8 - ? hasher(`head of ${stringKey}`) - : u8aConcat(xxhashAsU8a(itemFn.prefix, 128), xxhashAsU8a(`HeadOf${itemFn.method}`, 128)); - - storageFn.iterKey = extendHeadMeta(registry, itemFn, storageFn, (): Raw => - new Raw(registry, key) - ); - - return storageFn; -} - // attach the full list hashing for prefixed maps /** @internal */ function extendPrefixedMap (registry: Registry, itemFn: CreateItemFn, storageFn: StorageEntry): StorageEntry { @@ -215,13 +201,7 @@ export default function createFunction (registry: Registry, itemFn: CreateItemFn const storageFn = expandWithMeta(itemFn, _storageFn as StorageEntry); if (type.isMap) { - const map = type.asMap; - - if (map.linked.isTrue) { - extendLinkedMap(registry, itemFn, storageFn, stringKey, hasher, options.metaVersion); - } else { - extendPrefixedMap(registry, itemFn, storageFn); - } + extendPrefixedMap(registry, itemFn, storageFn); } else if (type.isDoubleMap) { extendDoubleMap(registry, itemFn, storageFn); } diff --git a/packages/types/src/primitive/StorageKey.ts b/packages/types/src/primitive/StorageKey.ts index c19ef922d2..76e3761a42 100644 --- a/packages/types/src/primitive/StorageKey.ts +++ b/packages/types/src/primitive/StorageKey.ts @@ -42,30 +42,20 @@ const HASHER_MAP: Record = { Twox64Concat: [8, true] }; -function getStorageType (type: StorageEntryTypeLatest, isOptionalLinked?: boolean): [boolean, string] { +function getStorageType (type: StorageEntryTypeLatest): [boolean, string] { if (type.isPlain) { return [false, type.asPlain.toString()]; } else if (type.isDoubleMap) { return [false, type.asDoubleMap.value.toString()]; } - const map = type.asMap; - - if (map.linked.isTrue) { - const [pre, post] = isOptionalLinked - ? ['Option<', '>'] - : ['', '']; - - return [true, `(${pre}${map.value.toString()}${post}, Linkage<${map.key.toString()}>)`]; - } - - return [false, map.value.toString()]; + return [false, type.asMap.value.toString()]; } // we unwrap the type here, turning into an output usable for createType /** @internal */ export function unwrapStorageType (type: StorageEntryTypeLatest, isOptional?: boolean): keyof InterfaceTypes { - const [hasWrapper, outputType] = getStorageType(type, isOptional); + const [hasWrapper, outputType] = getStorageType(type); return isOptional && !hasWrapper ? `Option<${outputType}>` as keyof InterfaceTypes