Linked map support (#801)

* able to get head

* head actually working

* Correctly get linkage

* I think it is working

* Results now return both keys and values

packages/api/src/Base.ts

@@ -17,19 +17,21 @@ import {
 } from './types';
 
 import EventEmitter from 'eventemitter3';
-import { map } from 'rxjs/operators';
+import { map, switchMap } from 'rxjs/operators';
 import decorateDerive, { Derive as DeriveInterface } from '@polkadot/api-derive';
 import extrinsicsFromMeta from '@polkadot/extrinsics/fromMetadata';
 import RpcBase from '@polkadot/rpc-core';
 import RpcRx from '@polkadot/rpc-rx';
 import storageFromMeta from '@polkadot/storage/fromMetadata';
-import { Event, getTypeRegistry, Hash, Metadata, Method, RuntimeVersion } from '@polkadot/types';
+import { Event, getTypeRegistry, Hash, Metadata, Method, RuntimeVersion, Tuple, Null } from '@polkadot/types';
 import { MethodFunction, ModulesWithMethods } from '@polkadot/types/primitive/Method';
 import { StorageFunction } from '@polkadot/types/primitive/StorageKey';
 import { assert, compactStripLength, isFunction, isObject, isUndefined, logger, u8aToHex } from '@polkadot/util';
 import { cryptoWaitReady } from '@polkadot/util-crypto';
 
 import { createSubmittableExtrinsic, SubmittableExtrinsic } from './SubmittableExtrinsic';
+import { Linkage, LinkageResult } from '@polkadot/types/codec/Linkage';
+import { Observable, of } from 'rxjs';
 
 type MetaDecoration = {
   callIndex?: Uint8Array,
@@ -491,6 +493,77 @@ export default abstract class ApiBase<CodecResult, SubscriptionResult> implement
         params = args.slice(0, args.length - 1);
       }
 
+      if (method.headKey && params.length === 0) {
+        // Fetch all values for this linked map
+        const result: Map<Codec, Tuple> = new Map();
+        let head: Codec | null = null;
+        const getNext = (key: Codec): Observable<any> => {
+          if (head === null) {
+            head = key;
+          }
+
+          return this._rpcRx.state.subscribeStorage([[method, key]])
+            .pipe(
+              switchMap(([data]: [Tuple]) => {
+                const linkage = data[1] as Linkage<Codec>;
+
+                if (linkage.next && linkage.previous) {
+                  result.set(key, data);
+
+                  if (linkage.next && linkage.next.isSome) {
+                    return getNext(linkage.next.unwrap());
+                  }
+                }
+
+                const keys = [];
+                const values = [];
+                let nextKey = head;
+
+                while (nextKey) {
+                  const entry = result.get(nextKey);
+
+                  if (!entry) {
+                    break;
+                  }
+
+                  const [item, linkage] = entry as any as [Codec, Linkage<Codec>];
+
+                  keys.push(nextKey);
+                  values.push(item);
+
+                  nextKey = linkage.next && linkage.next.unwrapOr(null);
+                }
+
+                return of(
+                  values.length
+                    ? new LinkageResult(
+                      [keys[0].constructor as any, keys],
+                      [values[0].constructor as any, values]
+                    )
+                    : new LinkageResult(
+                      [Null, []],
+                      [Null, []]
+                    )
+                );
+              })
+            );
+        };
+
+        return onCall(
+          (arg: CodecArg) => this._rpcRx.state
+            .subscribeStorage([arg])
+            .pipe(
+              switchMap((result: Array<Codec>) => {
+                const key = result[0];
+
+                return getNext(key);
+              })
+            ),
+          [method.headKey],
+          callback
+        );
+      }
+
       return onCall(
         (arg: CodecArg) => this._rpcRx.state
           .subscribeStorage([[method, arg]])
@@ -530,6 +603,25 @@ export default abstract class ApiBase<CodecResult, SubscriptionResult> implement
     decorated.key = (arg?: CodecArg): string =>
       u8aToHex(compactStripLength(method(arg))[1]);
 
+    // Linked Map support
+
+    if (method.headKey) {
+      decorated.head = (): C =>
+        onCall(
+          (arg: CodecArg) => this._rpcRx.state
+            .getStorage(arg)
+            .pipe(
+              switchMap(key => this._rpcRx.state.getStorage([method, key]))
+            )
+            ,
+          [method.headKey]
+        ) as C;
+    } else {
+      decorated.head = () => {
+        throw new Error(`${method.name} is not LinkedMap`);
+      };
+    }
+
     return this.decorateFunctionMeta(method, decorated) as QueryableStorageFunction<C, S>;
   }
 

packages/api/src/types.ts

@@ -59,6 +59,7 @@ export interface QueryableStorageFunctionBase<CodecResult, SubscriptionResult> e
   hash: (arg?: CodecArg) => HashResult<CodecResult, SubscriptionResult>;
   key: (arg?: CodecArg) => string;
   size: (arg?: CodecArg) => U64Result<CodecResult, SubscriptionResult>;
+  head: () => CodecResult;
 }
 
 interface QueryableStorageFunctionPromise<CodecResult, SubscriptionResult> extends QueryableStorageFunctionBase<CodecResult, SubscriptionResult> {

packages/api/test/e2e/promise-alex.spec.js

@@ -0,0 +1,38 @@
+// Copyright 2017-2019 @polkadot/api authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import Api from '../../src/promise';
+import WsProvider from '../../../rpc-provider/src/ws';
+
+describe.skip('alex queries', () => {
+  let api;
+
+  beforeAll(async () => {
+    api = await Api.create({
+      provider: new WsProvider('wss://poc3-rpc.polkadot.io/')
+    });
+
+    return api;
+  });
+
+  beforeEach(() => {
+    jest.setTimeout(30000);
+  });
+
+  it('retrieves the list of validators', (done) => {
+    api.query.staking.validators((res) => {
+      console.log('api.query.staking.validators():', res.toJSON());
+
+      done();
+    });
+  });
+
+  it('retrieves a single value', (done) => {
+    api.query.staking.validators('5EF7wdkP9XZq38fu2ioAvxoEhHUJUc1E2KbuSQSMGofRKhCL', (res) => {
+      console.log('api.query.staking.validators(id):', res.toJSON());
+
+      done();
+    });
+  });
+});

packages/api/test/e2e/rx-queries.spec.js

@@ -13,7 +13,7 @@ describe.skip('e2e queries', () => {
 
   beforeEach(async (done) => {
     api = await Api.create().toPromise();
-    jest.setTimeout(30000);
+    jest.setTimeout(3000000);
     done();
   });
 

packages/rpc-core/src/index.ts

@@ -194,9 +194,14 @@ export default class Rpc implements RpcInterface {
       const Clazz = createClass(type);
       const meta = key.meta || { default: undefined, modifier: { isOptional: true } };
 
-      return meta.modifier.isOptional
-        ? new Option(Clazz, isNull(result) ? null : new Clazz(base))
-        : new Clazz(base);
+      if (key.meta && key.meta.type.isMap && key.meta.type.asMap.isLinked) {
+        // linked map
+        return new Clazz(base);
+      } else {
+        return meta.modifier.isOptional
+          ? new Option(Clazz, isNull(result) ? null : new Clazz(base))
+          : new Clazz(base);
+      }
     } else if (method.type === 'StorageChangeSet') {
       // multiple return values (via state.storage subscription), decode the values
       // one at a time, all based on the query types. Three values can be returned -
@@ -215,21 +220,26 @@ export default class Rpc implements RpcInterface {
         ) || { value: null };
         const meta = key.meta || { default: undefined, modifier: { isOptional: true } };
 
-        // if we don't have a value, do not fill in the entry, it will be up to the
+        if (!value) {
+          // if we don't have a value, do not fill in the entry, it will be up to the
         // caller to sort this out, either ignoring or having a cache for older values
-        result.push(
-          !value
-            ? undefined
-            : (
-              meta.modifier.isOptional
-                // create option either with the existing value, or empty when
-                // there is no value returned
-                ? new Option(Clazz, value.isNone ? null : new Clazz(value.unwrap()))
-                // for `null` we fallback to the default value, or create an empty type,
-                // otherwise we return the actual value as retrieved
-                : new Clazz(value.unwrapOr(meta.default))
-            )
-        );
+          result.push(undefined);
+        } else {
+          if (key.meta && key.meta.type.isMap && key.meta.type.asMap.isLinked) {
+            // linked map
+            result.push(new Clazz(value.unwrapOr(null)));
+          } else {
+            if (meta.modifier.isOptional) {
+              // create option either with the existing value, or empty when
+              // there is no value returned
+              result.push(new Option(Clazz, value.isNone ? null : new Clazz(value.unwrap())));
+            } else {
+              // for `null` we fallback to the default value, or create an empty type,
+              // otherwise we return the actual value as retrieved
+              result.push(new Clazz(value.unwrapOr(meta.default)));
+            }
+          }
+        }
 
         return result;
       }, [] as Array<Codec | null | undefined>);

packages/type-storage/src/utils/createFunction.ts

@@ -2,11 +2,12 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
-import { StorageFunctionMetadata } from '@polkadot/types/Metadata/v0/Modules';
+import { StorageFunctionMetadata, StorageFunctionModifier, StorageFunctionType } from '@polkadot/types/Metadata/v0/Modules';
 import { StorageFunction } from '@polkadot/types/primitive/StorageKey';
-import { Compact, Text, createType } from '@polkadot/types';
+import { Compact, Text, createType, StorageKey, Bytes, U8a } from '@polkadot/types';
 import { assert, isNull, isUndefined, stringLowerFirst, stringToU8a, u8aConcat } from '@polkadot/util';
 import { xxhashAsU8a } from '@polkadot/util-crypto';
+import { PlainType } from '@polkadot/types/Metadata/v2/Storage';
 
 export interface CreateItemOptions {
   isUnhashed?: boolean;
@@ -24,16 +25,16 @@ export interface CreateItemOptions {
  * by us manually at compile time.
  */
 export default function createFunction (section: Text | string, method: Text | string, meta: StorageFunctionMetadata, options: CreateItemOptions = {}): StorageFunction {
+  const stringKey = options.key
+    ? options.key
+    : `${section.toString()} ${method.toString()}`;
+  const rawKey = stringToU8a(stringKey);
+
   // Can only have zero or one argument:
   // - storage.balances.freeBalance(address)
   // - storage.timestamp.blockPeriod()
   const storageFn = (arg?: any): Uint8Array => {
-    let key = stringToU8a(
-      options.key
-        ? options.key
-        : `${section.toString()} ${method.toString()}`
-    );
-
+    let key = rawKey;
     if (meta.type.isMap) {
       assert(!isUndefined(arg) && !isNull(arg), `${meta.name} expects one argument`);
 
@@ -51,6 +52,20 @@ export default function createFunction (section: Text | string, method: Text | s
     );
   };
 
+  if (meta.type.isMap && meta.type.asMap.isLinked) {
+    // TODO: there needs some better way to do this
+    const keyHash = new U8a(xxhashAsU8a(`head of ${stringKey}`, 128));
+    const keyFn: any = () => keyHash;
+    keyFn.meta = new StorageFunctionMetadata({
+      name: meta.name,
+      modifier: new StorageFunctionModifier('Required'),
+      type: new StorageFunctionType(new PlainType(meta.type.asMap.key), 0),
+      default: new Bytes(),
+      documentation: meta.documentation
+    });
+    storageFn.headKey = new StorageKey(keyFn);
+  }
+
   storageFn.meta = meta;
   storageFn.method = stringLowerFirst(method.toString());
   storageFn.section = stringLowerFirst(section.toString());

packages/types/src/Metadata/v0/Modules.ts

@@ -207,9 +207,13 @@ export class StorageFunctionType extends EnumType<PlainType | MapType> {
    * @description Returns the string representation of the value
    */
   toString (): string {
-    return this.isMap
-      ? this.asMap.value.toString()
-      : this.asType.toString();
+    if (this.isMap) {
+      if (this.asMap.isLinked) {
+        return `(${this.asMap.value.toString()}, Linkage<${this.asMap.key.toString()}>)`;
+      }
+      return this.asMap.value.toString();
+    }
+    return this.asType.toString();
   }
 }
 

packages/types/src/codec/Linkage.ts

@@ -0,0 +1,42 @@
+// Copyright 2017-2019 @polkadot/types authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { Struct, Option, Tuple, Vector } from '.';
+import { Constructor, Codec } from '../types';
+
+type TypeWithValues = [Constructor, Array<any>];
+
+export class Linkage<T extends Codec> extends Struct {
+  constructor (Type: Constructor, value?: any) {
+    super({
+      previous: Option.with(Type),
+      next: Option.with(Type)
+    }, value);
+  }
+
+  static withKey<O extends Codec> (Type: Constructor): Constructor<Linkage<O>> {
+    return class extends Linkage<O> {
+      constructor (value?: any) {
+        super(Type, value);
+      }
+    };
+  }
+
+  get previous (): Option<T> {
+    return this.get('previous') as Option<T>;
+  }
+
+  get next (): Option<T> {
+    return this.get('next') as Option<T>;
+  }
+}
+
+export class LinkageResult extends Tuple {
+  constructor ([TypeKey, keys]: TypeWithValues, [TypeValue, values]: TypeWithValues) {
+    super({
+      Keys: Vector.with(TypeKey),
+      Values: Vector.with(TypeValue)
+    }, [keys, values]);
+  }
+}

packages/types/src/codec/createType.ts

@@ -14,6 +14,7 @@ import Tuple from './Tuple';
 import UInt from './UInt';
 import Vector from './Vector';
 import getRegistry from './typeRegistry';
+import { Linkage } from './Linkage';
 
 export enum TypeDefInfo {
   Compact,
@@ -22,7 +23,8 @@ export enum TypeDefInfo {
   Plain,
   Struct,
   Tuple,
-  Vector
+  Vector,
+  Linkage
 }
 
 export type TypeDef = {
@@ -130,6 +132,9 @@ export function getTypeDef (_type: Text | string, name?: string): TypeDef {
   } else if (startingWith(type, 'Vec<', '>')) {
     value.info = TypeDefInfo.Vector;
     value.sub = getTypeDef(subType);
+  } else if (startingWith(type, 'Linkage<', '>')) {
+    value.info = TypeDefInfo.Linkage;
+    value.sub = getTypeDef(subType);
   }
 
   return value;
@@ -141,50 +146,56 @@ export function getTypeClass (value: TypeDef): Constructor {
   if (Type) {
     return Type;
   }
-  if (value.info === TypeDefInfo.Compact) {
-    assert(value.sub && !Array.isArray(value.sub), 'Expected subtype for Compact');
+  switch (value.info) {
+    case TypeDefInfo.Compact:
+      assert(value.sub && !Array.isArray(value.sub), 'Expected subtype for Compact');
 
-    return Compact.with(
-      getTypeClass(value.sub as TypeDef) as Constructor<UInt>
-    );
-  } else if (value.info === TypeDefInfo.Enum) {
-    assert(value.sub && Array.isArray(value.sub), 'Expected subtype for Enum');
+      return Compact.with(
+        getTypeClass(value.sub as TypeDef) as Constructor<UInt>
+      );
+    case TypeDefInfo.Enum:
+      assert(value.sub && Array.isArray(value.sub), 'Expected subtype for Enum');
 
-    return EnumType.with(
-      (value.sub as Array<TypeDef>).reduce((result, sub, index) => {
-        result[sub.name as string] = getTypeClass(sub);
+      return EnumType.with(
+        (value.sub as Array<TypeDef>).reduce((result, sub, index) => {
+          result[sub.name as string] = getTypeClass(sub);
 
-        return result;
-      }, {} as { [index: string]: Constructor })
-    );
-  } else if (value.info === TypeDefInfo.Option) {
-    assert(value.sub && !Array.isArray(value.sub), 'Expected subtype for Option');
+          return result;
+        }, {} as { [index: string]: Constructor })
+      );
+    case TypeDefInfo.Option:
+      assert(value.sub && !Array.isArray(value.sub), 'Expected subtype for Option');
 
-    return Option.with(
-      getTypeClass(value.sub as TypeDef)
-    );
-  } else if (value.info === TypeDefInfo.Struct) {
-    assert(Array.isArray(value.sub), 'Expected nested subtypes for Struct');
+      return Option.with(
+        getTypeClass(value.sub as TypeDef)
+      );
+    case TypeDefInfo.Struct:
+      assert(Array.isArray(value.sub), 'Expected nested subtypes for Struct');
 
-    return Struct.with(
-      (value.sub as Array<TypeDef>).reduce((result, sub) => {
-        result[sub.name as string] = getTypeClass(sub);
+      return Struct.with(
+        (value.sub as Array<TypeDef>).reduce((result, sub) => {
+          result[sub.name as string] = getTypeClass(sub);
 
-        return result;
-      }, {} as { [index: string]: Constructor })
-    );
-  } else if (value.info === TypeDefInfo.Tuple) {
-    assert(Array.isArray(value.sub), 'Expected nested subtypes for Tuple');
+          return result;
+        }, {} as { [index: string]: Constructor })
+      );
+    case TypeDefInfo.Tuple:
+      assert(Array.isArray(value.sub), 'Expected nested subtypes for Tuple');
 
-    return Tuple.with(
-      (value.sub as Array<TypeDef>).map(getTypeClass)
-    );
-  } else if (value.info === TypeDefInfo.Vector) {
-    assert(value.sub && !Array.isArray(value.sub), 'Expected subtype for Vector');
+      return Tuple.with(
+        (value.sub as Array<TypeDef>).map(getTypeClass)
+      );
+    case TypeDefInfo.Vector:
+      assert(value.sub && !Array.isArray(value.sub), 'Expected subtype for Vector');
 
-    return Vector.with(
-      getTypeClass(value.sub as TypeDef)
-    );
+      return Vector.with(
+        getTypeClass(value.sub as TypeDef)
+      );
+    case TypeDefInfo.Linkage:
+      assert(value.sub && !Array.isArray(value.sub), 'Expected subtype for Linkage');
+      return Linkage.withKey(
+        getTypeClass(value.sub as TypeDef)
+      );
   }
 
   throw new Error(`Unable to determine type from '${value.type}'`);

packages/types/src/primitive/StorageKey.ts

@@ -15,6 +15,7 @@ export interface StorageFunction {
   method: string;
   section: string;
   toJSON: () => any;
+  headKey?: Uint8Array;
 }
 
 /**
@@ -38,10 +39,10 @@ export default class StorageKey extends Bytes {
     if (isFunction(value)) {
       return value();
     } else if (Array.isArray(value)) {
-      const [fn, arg] = value;
+      const [fn, ...arg] = value;
 
       if (isFunction(fn)) {
-        return fn(arg);
+        return fn(...arg);
       }
     }