From cda0ef3486e21bb7f636bcbd674b111bf9e53510 Mon Sep 17 00:00:00 2001 From: Jaco Greeff Date: Wed, 27 Mar 2019 15:34:11 +0100 Subject: [PATCH] Linked map support (#801) * able to get head * head actually working * Correctly get linkage * I think it is working * Results now return both keys and values --- packages/api/src/Base.ts | 96 ++++++++++++++++++- packages/api/src/types.ts | 1 + packages/api/test/e2e/promise-alex.spec.js | 38 ++++++++ packages/api/test/e2e/rx-queries.spec.js | 2 +- packages/rpc-core/src/index.ts | 44 +++++---- .../type-storage/src/utils/createFunction.ts | 31 ++++-- packages/types/src/Metadata/v0/Modules.ts | 10 +- packages/types/src/codec/Linkage.ts | 42 ++++++++ packages/types/src/codec/createType.ts | 85 +++++++++------- packages/types/src/primitive/StorageKey.ts | 5 +- 10 files changed, 284 insertions(+), 70 deletions(-) create mode 100644 packages/api/test/e2e/promise-alex.spec.js create mode 100644 packages/types/src/codec/Linkage.ts diff --git a/packages/api/src/Base.ts b/packages/api/src/Base.ts index 1d378db61c..bf0d7ea845 100644 --- a/packages/api/src/Base.ts +++ b/packages/api/src/Base.ts @@ -17,19 +17,21 @@ import { } from './types'; import EventEmitter from 'eventemitter3'; -import { map } from 'rxjs/operators'; +import { map, switchMap } from 'rxjs/operators'; import decorateDerive, { Derive as DeriveInterface } from '@polkadot/api-derive'; import extrinsicsFromMeta from '@polkadot/extrinsics/fromMetadata'; import RpcBase from '@polkadot/rpc-core'; import RpcRx from '@polkadot/rpc-rx'; import storageFromMeta from '@polkadot/storage/fromMetadata'; -import { Event, getTypeRegistry, Hash, Metadata, Method, RuntimeVersion } from '@polkadot/types'; +import { Event, getTypeRegistry, Hash, Metadata, Method, RuntimeVersion, Tuple, Null } from '@polkadot/types'; import { MethodFunction, ModulesWithMethods } from '@polkadot/types/primitive/Method'; import { StorageFunction } from '@polkadot/types/primitive/StorageKey'; import { assert, compactStripLength, isFunction, isObject, isUndefined, logger, u8aToHex } from '@polkadot/util'; import { cryptoWaitReady } from '@polkadot/util-crypto'; import { createSubmittableExtrinsic, SubmittableExtrinsic } from './SubmittableExtrinsic'; +import { Linkage, LinkageResult } from '@polkadot/types/codec/Linkage'; +import { Observable, of } from 'rxjs'; type MetaDecoration = { callIndex?: Uint8Array, @@ -491,6 +493,77 @@ export default abstract class ApiBase implement params = args.slice(0, args.length - 1); } + if (method.headKey && params.length === 0) { + // Fetch all values for this linked map + const result: Map = new Map(); + let head: Codec | null = null; + const getNext = (key: Codec): Observable => { + if (head === null) { + head = key; + } + + return this._rpcRx.state.subscribeStorage([[method, key]]) + .pipe( + switchMap(([data]: [Tuple]) => { + const linkage = data[1] as Linkage; + + if (linkage.next && linkage.previous) { + result.set(key, data); + + if (linkage.next && linkage.next.isSome) { + return getNext(linkage.next.unwrap()); + } + } + + const keys = []; + const values = []; + let nextKey = head; + + while (nextKey) { + const entry = result.get(nextKey); + + if (!entry) { + break; + } + + const [item, linkage] = entry as any as [Codec, Linkage]; + + keys.push(nextKey); + values.push(item); + + nextKey = linkage.next && linkage.next.unwrapOr(null); + } + + return of( + values.length + ? new LinkageResult( + [keys[0].constructor as any, keys], + [values[0].constructor as any, values] + ) + : new LinkageResult( + [Null, []], + [Null, []] + ) + ); + }) + ); + }; + + return onCall( + (arg: CodecArg) => this._rpcRx.state + .subscribeStorage([arg]) + .pipe( + switchMap((result: Array) => { + const key = result[0]; + + return getNext(key); + }) + ), + [method.headKey], + callback + ); + } + return onCall( (arg: CodecArg) => this._rpcRx.state .subscribeStorage([[method, arg]]) @@ -530,6 +603,25 @@ export default abstract class ApiBase implement decorated.key = (arg?: CodecArg): string => u8aToHex(compactStripLength(method(arg))[1]); + // Linked Map support + + if (method.headKey) { + decorated.head = (): C => + onCall( + (arg: CodecArg) => this._rpcRx.state + .getStorage(arg) + .pipe( + switchMap(key => this._rpcRx.state.getStorage([method, key])) + ) + , + [method.headKey] + ) as C; + } else { + decorated.head = () => { + throw new Error(`${method.name} is not LinkedMap`); + }; + } + return this.decorateFunctionMeta(method, decorated) as QueryableStorageFunction; } diff --git a/packages/api/src/types.ts b/packages/api/src/types.ts index e84e8343e4..6acc3acc4a 100644 --- a/packages/api/src/types.ts +++ b/packages/api/src/types.ts @@ -59,6 +59,7 @@ export interface QueryableStorageFunctionBase e hash: (arg?: CodecArg) => HashResult; key: (arg?: CodecArg) => string; size: (arg?: CodecArg) => U64Result; + head: () => CodecResult; } interface QueryableStorageFunctionPromise extends QueryableStorageFunctionBase { diff --git a/packages/api/test/e2e/promise-alex.spec.js b/packages/api/test/e2e/promise-alex.spec.js new file mode 100644 index 0000000000..bd023316f2 --- /dev/null +++ b/packages/api/test/e2e/promise-alex.spec.js @@ -0,0 +1,38 @@ +// Copyright 2017-2019 @polkadot/api authors & contributors +// This software may be modified and distributed under the terms +// of the Apache-2.0 license. See the LICENSE file for details. + +import Api from '../../src/promise'; +import WsProvider from '../../../rpc-provider/src/ws'; + +describe.skip('alex queries', () => { + let api; + + beforeAll(async () => { + api = await Api.create({ + provider: new WsProvider('wss://poc3-rpc.polkadot.io/') + }); + + return api; + }); + + beforeEach(() => { + jest.setTimeout(30000); + }); + + it('retrieves the list of validators', (done) => { + api.query.staking.validators((res) => { + console.log('api.query.staking.validators():', res.toJSON()); + + done(); + }); + }); + + it('retrieves a single value', (done) => { + api.query.staking.validators('5EF7wdkP9XZq38fu2ioAvxoEhHUJUc1E2KbuSQSMGofRKhCL', (res) => { + console.log('api.query.staking.validators(id):', res.toJSON()); + + done(); + }); + }); +}); diff --git a/packages/api/test/e2e/rx-queries.spec.js b/packages/api/test/e2e/rx-queries.spec.js index cffccc36af..1796f7a4a4 100644 --- a/packages/api/test/e2e/rx-queries.spec.js +++ b/packages/api/test/e2e/rx-queries.spec.js @@ -13,7 +13,7 @@ describe.skip('e2e queries', () => { beforeEach(async (done) => { api = await Api.create().toPromise(); - jest.setTimeout(30000); + jest.setTimeout(3000000); done(); }); diff --git a/packages/rpc-core/src/index.ts b/packages/rpc-core/src/index.ts index 02c5624e42..f5058272ff 100644 --- a/packages/rpc-core/src/index.ts +++ b/packages/rpc-core/src/index.ts @@ -194,9 +194,14 @@ export default class Rpc implements RpcInterface { const Clazz = createClass(type); const meta = key.meta || { default: undefined, modifier: { isOptional: true } }; - return meta.modifier.isOptional - ? new Option(Clazz, isNull(result) ? null : new Clazz(base)) - : new Clazz(base); + if (key.meta && key.meta.type.isMap && key.meta.type.asMap.isLinked) { + // linked map + return new Clazz(base); + } else { + return meta.modifier.isOptional + ? new Option(Clazz, isNull(result) ? null : new Clazz(base)) + : new Clazz(base); + } } else if (method.type === 'StorageChangeSet') { // multiple return values (via state.storage subscription), decode the values // one at a time, all based on the query types. Three values can be returned - @@ -215,21 +220,26 @@ export default class Rpc implements RpcInterface { ) || { value: null }; const meta = key.meta || { default: undefined, modifier: { isOptional: true } }; - // if we don't have a value, do not fill in the entry, it will be up to the + if (!value) { + // if we don't have a value, do not fill in the entry, it will be up to the // caller to sort this out, either ignoring or having a cache for older values - result.push( - !value - ? undefined - : ( - meta.modifier.isOptional - // create option either with the existing value, or empty when - // there is no value returned - ? new Option(Clazz, value.isNone ? null : new Clazz(value.unwrap())) - // for `null` we fallback to the default value, or create an empty type, - // otherwise we return the actual value as retrieved - : new Clazz(value.unwrapOr(meta.default)) - ) - ); + result.push(undefined); + } else { + if (key.meta && key.meta.type.isMap && key.meta.type.asMap.isLinked) { + // linked map + result.push(new Clazz(value.unwrapOr(null))); + } else { + if (meta.modifier.isOptional) { + // create option either with the existing value, or empty when + // there is no value returned + result.push(new Option(Clazz, value.isNone ? null : new Clazz(value.unwrap()))); + } else { + // for `null` we fallback to the default value, or create an empty type, + // otherwise we return the actual value as retrieved + result.push(new Clazz(value.unwrapOr(meta.default))); + } + } + } return result; }, [] as Array); diff --git a/packages/type-storage/src/utils/createFunction.ts b/packages/type-storage/src/utils/createFunction.ts index c016d24b42..7ed5e8fee6 100644 --- a/packages/type-storage/src/utils/createFunction.ts +++ b/packages/type-storage/src/utils/createFunction.ts @@ -2,11 +2,12 @@ // This software may be modified and distributed under the terms // of the Apache-2.0 license. See the LICENSE file for details. -import { StorageFunctionMetadata } from '@polkadot/types/Metadata/v0/Modules'; +import { StorageFunctionMetadata, StorageFunctionModifier, StorageFunctionType } from '@polkadot/types/Metadata/v0/Modules'; import { StorageFunction } from '@polkadot/types/primitive/StorageKey'; -import { Compact, Text, createType } from '@polkadot/types'; +import { Compact, Text, createType, StorageKey, Bytes, U8a } from '@polkadot/types'; import { assert, isNull, isUndefined, stringLowerFirst, stringToU8a, u8aConcat } from '@polkadot/util'; import { xxhashAsU8a } from '@polkadot/util-crypto'; +import { PlainType } from '@polkadot/types/Metadata/v2/Storage'; export interface CreateItemOptions { isUnhashed?: boolean; @@ -24,16 +25,16 @@ export interface CreateItemOptions { * by us manually at compile time. */ export default function createFunction (section: Text | string, method: Text | string, meta: StorageFunctionMetadata, options: CreateItemOptions = {}): StorageFunction { + const stringKey = options.key + ? options.key + : `${section.toString()} ${method.toString()}`; + const rawKey = stringToU8a(stringKey); + // Can only have zero or one argument: // - storage.balances.freeBalance(address) // - storage.timestamp.blockPeriod() const storageFn = (arg?: any): Uint8Array => { - let key = stringToU8a( - options.key - ? options.key - : `${section.toString()} ${method.toString()}` - ); - + let key = rawKey; if (meta.type.isMap) { assert(!isUndefined(arg) && !isNull(arg), `${meta.name} expects one argument`); @@ -51,6 +52,20 @@ export default function createFunction (section: Text | string, method: Text | s ); }; + if (meta.type.isMap && meta.type.asMap.isLinked) { + // TODO: there needs some better way to do this + const keyHash = new U8a(xxhashAsU8a(`head of ${stringKey}`, 128)); + const keyFn: any = () => keyHash; + keyFn.meta = new StorageFunctionMetadata({ + name: meta.name, + modifier: new StorageFunctionModifier('Required'), + type: new StorageFunctionType(new PlainType(meta.type.asMap.key), 0), + default: new Bytes(), + documentation: meta.documentation + }); + storageFn.headKey = new StorageKey(keyFn); + } + storageFn.meta = meta; storageFn.method = stringLowerFirst(method.toString()); storageFn.section = stringLowerFirst(section.toString()); diff --git a/packages/types/src/Metadata/v0/Modules.ts b/packages/types/src/Metadata/v0/Modules.ts index 3c316a8a74..eb72da7a67 100644 --- a/packages/types/src/Metadata/v0/Modules.ts +++ b/packages/types/src/Metadata/v0/Modules.ts @@ -207,9 +207,13 @@ export class StorageFunctionType extends EnumType { * @description Returns the string representation of the value */ toString (): string { - return this.isMap - ? this.asMap.value.toString() - : this.asType.toString(); + if (this.isMap) { + if (this.asMap.isLinked) { + return `(${this.asMap.value.toString()}, Linkage<${this.asMap.key.toString()}>)`; + } + return this.asMap.value.toString(); + } + return this.asType.toString(); } } diff --git a/packages/types/src/codec/Linkage.ts b/packages/types/src/codec/Linkage.ts new file mode 100644 index 0000000000..82116158dc --- /dev/null +++ b/packages/types/src/codec/Linkage.ts @@ -0,0 +1,42 @@ +// Copyright 2017-2019 @polkadot/types authors & contributors +// This software may be modified and distributed under the terms +// of the Apache-2.0 license. See the LICENSE file for details. + +import { Struct, Option, Tuple, Vector } from '.'; +import { Constructor, Codec } from '../types'; + +type TypeWithValues = [Constructor, Array]; + +export class Linkage extends Struct { + constructor (Type: Constructor, value?: any) { + super({ + previous: Option.with(Type), + next: Option.with(Type) + }, value); + } + + static withKey (Type: Constructor): Constructor> { + return class extends Linkage { + constructor (value?: any) { + super(Type, value); + } + }; + } + + get previous (): Option { + return this.get('previous') as Option; + } + + get next (): Option { + return this.get('next') as Option; + } +} + +export class LinkageResult extends Tuple { + constructor ([TypeKey, keys]: TypeWithValues, [TypeValue, values]: TypeWithValues) { + super({ + Keys: Vector.with(TypeKey), + Values: Vector.with(TypeValue) + }, [keys, values]); + } +} diff --git a/packages/types/src/codec/createType.ts b/packages/types/src/codec/createType.ts index 9d633f90be..2ddc862c45 100644 --- a/packages/types/src/codec/createType.ts +++ b/packages/types/src/codec/createType.ts @@ -14,6 +14,7 @@ import Tuple from './Tuple'; import UInt from './UInt'; import Vector from './Vector'; import getRegistry from './typeRegistry'; +import { Linkage } from './Linkage'; export enum TypeDefInfo { Compact, @@ -22,7 +23,8 @@ export enum TypeDefInfo { Plain, Struct, Tuple, - Vector + Vector, + Linkage } export type TypeDef = { @@ -130,6 +132,9 @@ export function getTypeDef (_type: Text | string, name?: string): TypeDef { } else if (startingWith(type, 'Vec<', '>')) { value.info = TypeDefInfo.Vector; value.sub = getTypeDef(subType); + } else if (startingWith(type, 'Linkage<', '>')) { + value.info = TypeDefInfo.Linkage; + value.sub = getTypeDef(subType); } return value; @@ -141,50 +146,56 @@ export function getTypeClass (value: TypeDef): Constructor { if (Type) { return Type; } - if (value.info === TypeDefInfo.Compact) { - assert(value.sub && !Array.isArray(value.sub), 'Expected subtype for Compact'); + switch (value.info) { + case TypeDefInfo.Compact: + assert(value.sub && !Array.isArray(value.sub), 'Expected subtype for Compact'); - return Compact.with( - getTypeClass(value.sub as TypeDef) as Constructor - ); - } else if (value.info === TypeDefInfo.Enum) { - assert(value.sub && Array.isArray(value.sub), 'Expected subtype for Enum'); + return Compact.with( + getTypeClass(value.sub as TypeDef) as Constructor + ); + case TypeDefInfo.Enum: + assert(value.sub && Array.isArray(value.sub), 'Expected subtype for Enum'); - return EnumType.with( - (value.sub as Array).reduce((result, sub, index) => { - result[sub.name as string] = getTypeClass(sub); + return EnumType.with( + (value.sub as Array).reduce((result, sub, index) => { + result[sub.name as string] = getTypeClass(sub); - return result; - }, {} as { [index: string]: Constructor }) - ); - } else if (value.info === TypeDefInfo.Option) { - assert(value.sub && !Array.isArray(value.sub), 'Expected subtype for Option'); + return result; + }, {} as { [index: string]: Constructor }) + ); + case TypeDefInfo.Option: + assert(value.sub && !Array.isArray(value.sub), 'Expected subtype for Option'); - return Option.with( - getTypeClass(value.sub as TypeDef) - ); - } else if (value.info === TypeDefInfo.Struct) { - assert(Array.isArray(value.sub), 'Expected nested subtypes for Struct'); + return Option.with( + getTypeClass(value.sub as TypeDef) + ); + case TypeDefInfo.Struct: + assert(Array.isArray(value.sub), 'Expected nested subtypes for Struct'); - return Struct.with( - (value.sub as Array).reduce((result, sub) => { - result[sub.name as string] = getTypeClass(sub); + return Struct.with( + (value.sub as Array).reduce((result, sub) => { + result[sub.name as string] = getTypeClass(sub); - return result; - }, {} as { [index: string]: Constructor }) - ); - } else if (value.info === TypeDefInfo.Tuple) { - assert(Array.isArray(value.sub), 'Expected nested subtypes for Tuple'); + return result; + }, {} as { [index: string]: Constructor }) + ); + case TypeDefInfo.Tuple: + assert(Array.isArray(value.sub), 'Expected nested subtypes for Tuple'); - return Tuple.with( - (value.sub as Array).map(getTypeClass) - ); - } else if (value.info === TypeDefInfo.Vector) { - assert(value.sub && !Array.isArray(value.sub), 'Expected subtype for Vector'); + return Tuple.with( + (value.sub as Array).map(getTypeClass) + ); + case TypeDefInfo.Vector: + assert(value.sub && !Array.isArray(value.sub), 'Expected subtype for Vector'); - return Vector.with( - getTypeClass(value.sub as TypeDef) - ); + return Vector.with( + getTypeClass(value.sub as TypeDef) + ); + case TypeDefInfo.Linkage: + assert(value.sub && !Array.isArray(value.sub), 'Expected subtype for Linkage'); + return Linkage.withKey( + getTypeClass(value.sub as TypeDef) + ); } throw new Error(`Unable to determine type from '${value.type}'`); diff --git a/packages/types/src/primitive/StorageKey.ts b/packages/types/src/primitive/StorageKey.ts index c1e0707f48..fcc55a9642 100644 --- a/packages/types/src/primitive/StorageKey.ts +++ b/packages/types/src/primitive/StorageKey.ts @@ -15,6 +15,7 @@ export interface StorageFunction { method: string; section: string; toJSON: () => any; + headKey?: Uint8Array; } /** @@ -38,10 +39,10 @@ export default class StorageKey extends Bytes { if (isFunction(value)) { return value(); } else if (Array.isArray(value)) { - const [fn, arg] = value; + const [fn, ...arg] = value; if (isFunction(fn)) { - return fn(arg); + return fn(...arg); } }