Initial DoubleMap support (support v3 Metadata) (#798)

* DoubleMap support

* Typo

* V3 static

* Swap static to V3

* Cleanup toString

* Remove V0 support, don't get to isReady when load fails

* Re-add v0 (old tests, this is a bit of a mission...)

packages/api/src/Base.ts

@@ -341,61 +341,59 @@ export default abstract class ApiBase<CodecResult, SubscriptionResult> implement
     this._rpcBase._provider.on('connected', async () => {
       this.emit('connected');
 
-      const [hasMeta, cryptoReady] = await Promise.all([
-        this.loadMeta(),
-        cryptoWaitReady()
-      ]);
+      try {
+        const [hasMeta, cryptoReady] = await Promise.all([
+          this.loadMeta(),
+          cryptoWaitReady()
+        ]);
 
-      if (hasMeta && !this._isReady && cryptoReady) {
-        this._isReady = true;
+        if (hasMeta && !this._isReady && cryptoReady) {
+          this._isReady = true;
 
-        this.emit('ready', this);
+          this.emit('ready', this);
+        }
+      } catch (error) {
+        l.error('FATAL: Unable to initialize the API: ', error.message);
       }
     });
   }
 
   private async loadMeta (): Promise<boolean> {
-    try {
-      // only load from on-chain if we are not a clone (default path), alternatively
-      // just use the values from the source instance provided
-      if (!this._options.source || !this._options.source._isReady) {
-        this._runtimeMetadata = await this._rpcBase.state.getMetadata();
-        this._runtimeVersion = await this._rpcBase.chain.getRuntimeVersion();
-        this._genesisHash = await this._rpcBase.chain.getBlockHash(0);
+    // only load from on-chain if we are not a clone (default path), alternatively
+    // just use the values from the source instance provided
+    if (!this._options.source || !this._options.source._isReady) {
+      this._runtimeMetadata = await this._rpcBase.state.getMetadata();
+      this._runtimeVersion = await this._rpcBase.chain.getRuntimeVersion();
+      this._genesisHash = await this._rpcBase.chain.getBlockHash(0);
 
-        // get unique types & validate
-        this.runtimeMetadata.getUniqTypes(false);
-      } else {
-        this._runtimeMetadata = this._options.source.runtimeMetadata;
-        this._runtimeVersion = this._options.source.runtimeVersion;
-        this._genesisHash = this._options.source.genesisHash;
-      }
-
-      const extrinsics = extrinsicsFromMeta(this.runtimeMetadata.asV0);
-      const storage = storageFromMeta(this.runtimeMetadata.asV0);
-
-      this._extrinsics = this.decorateExtrinsics(extrinsics, this.onCall);
-      this._query = this.decorateStorage(storage, this.onCall);
-      this._derive = this.decorateDerive(this._rx as ApiInterface$Rx, this.onCall);
-
-      this._rx.genesisHash = this._genesisHash;
-      this._rx.runtimeVersion = this._runtimeVersion;
-      this._rx.tx = this.decorateExtrinsics(extrinsics, rxOnCall);
-      this._rx.query = this.decorateStorage(storage, rxOnCall);
-      this._rx.derive = this.decorateDerive(this._rx as ApiInterface$Rx, rxOnCall);
-
-      // only inject if we are not a clone (global init)
-      if (!this._options.source) {
-        Event.injectMetadata(this.runtimeMetadata.asV0);
-        Method.injectMethods(extrinsics);
-      }
-
-      return true;
-    } catch (error) {
-      l.error('loadMeta', error);
-
-      return false;
+      // get unique types & validate
+      this.runtimeMetadata.getUniqTypes(false);
+    } else {
+      this._runtimeMetadata = this._options.source.runtimeMetadata;
+      this._runtimeVersion = this._options.source.runtimeVersion;
+      this._genesisHash = this._options.source.genesisHash;
     }
+
+    const extrinsics = extrinsicsFromMeta(this.runtimeMetadata.asV0);
+    const storage = storageFromMeta(this.runtimeMetadata.asV0);
+
+    this._extrinsics = this.decorateExtrinsics(extrinsics, this.onCall);
+    this._query = this.decorateStorage(storage, this.onCall);
+    this._derive = this.decorateDerive(this._rx as ApiInterface$Rx, this.onCall);
+
+    this._rx.genesisHash = this._genesisHash;
+    this._rx.runtimeVersion = this._runtimeVersion;
+    this._rx.tx = this.decorateExtrinsics(extrinsics, rxOnCall);
+    this._rx.query = this.decorateStorage(storage, rxOnCall);
+    this._rx.derive = this.decorateDerive(this._rx as ApiInterface$Rx, rxOnCall);
+
+    // only inject if we are not a clone (global init)
+    if (!this._options.source) {
+      Event.injectMetadata(this.runtimeMetadata.asV0);
+      Method.injectMethods(extrinsics);
+    }
+
+    return true;
   }
 
   private decorateFunctionMeta (input: MetaDecoration, output: MetaDecoration): MetaDecoration {

packages/types/src/Metadata/MagicNumber.ts

@@ -2,9 +2,10 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
-import U32 from '../primitive/U32';
 import { assert } from '@polkadot/util';
 
+import U32 from '../primitive/U32';
+
 export const MAGIC_NUMBER = new U32(0x6174656d); // `meta`, reversed for LE encoding
 export const MAGIC_ERROR = -61746;
 

packages/types/src/Metadata/MetadataVersioned.ts

@@ -9,19 +9,22 @@ import { assert, isUndefined } from '@polkadot/util';
 import EnumType from '../codec/EnumType';
 import Struct from '../codec/Struct';
 import Null from '../primitive/Null';
+import MagicNumber from './MagicNumber';
 import MetadataV0 from './v0';
 import MetadataV1 from './v1';
 import MetadataV2 from './v2';
+import MetadataV3 from './v3';
 import v1ToV0 from './v1/toV0';
 import v2ToV1 from './v2/toV1';
-import MagicNumber from './MagicNumber';
+import v3ToV2 from './v3/toV2';
 
 class MetadataEnum extends EnumType<Null | MetadataV1 | MetadataV2> {
   constructor (value?: any) {
     super({
       MetadataV0, // once rolled-out, can replace this with Null
       MetadataV1, // once rolled-out, can replace this with Null
-      MetadataV2
+      MetadataV2, // once rolled-out, can replace this with Null
+      MetadataV3
     }, value);
   }
 
@@ -40,12 +43,19 @@ class MetadataEnum extends EnumType<Null | MetadataV1 | MetadataV2> {
   }
 
   /**
-   * @description Returns the wrapped values as a V1 object
+   * @description Returns the wrapped values as a V2 object
    */
   get asV2 (): MetadataV2 {
     return this.value as MetadataV2;
   }
 
+  /**
+   * @description Returns the wrapped values as a V3 object
+   */
+  get asV3 (): MetadataV3 {
+    return this.value as MetadataV3;
+  }
+
   /**
    * @description The version this metadata represents
    */
@@ -62,6 +72,7 @@ class MetadataEnum extends EnumType<Null | MetadataV1 | MetadataV2> {
 export default class MetadataVersioned extends Struct implements MetadataInterface {
   private _convertedV0?: MetadataV0;
   private _convertedV1?: MetadataV1;
+  private _convertedV2?: MetadataV2;
 
   constructor (value?: any) {
     super({
@@ -99,10 +110,8 @@ export default class MetadataVersioned extends Struct implements MetadataInterfa
       return this.metadata.asV0;
     }
 
-    let v1 = this.asV1;
-
     if (isUndefined(this._convertedV0)) {
-      this._convertedV0 = v1ToV0(v1);
+      this._convertedV0 = v1ToV0(this.asV1);
     }
 
     return this._convertedV0;
@@ -116,7 +125,7 @@ export default class MetadataVersioned extends Struct implements MetadataInterfa
       return this.metadata.asV1;
     }
 
-    assert(this.metadata.version === 2, `Cannot convert metadata from v${this.metadata.version} to v1`);
+    assert([2, 3].includes(this.metadata.version), `Cannot convert metadata from v${this.metadata.version} to v1`);
 
     if (isUndefined(this._convertedV1)) {
       this._convertedV1 = v2ToV1(this.metadata.asV2);
@@ -126,12 +135,29 @@ export default class MetadataVersioned extends Struct implements MetadataInterfa
   }
 
   /**
-   * @description Returns the wrapped values as a V1 object
+   * @description Returns the wrapped values as a V2 object
    */
   get asV2 (): MetadataV2 {
-    assert(this.metadata.version === 2, `Cannot convert metadata from v${this.metadata.version} to v2`);
+    if (this.metadata.version === 2) {
+      return this.metadata.asV2;
+    }
 
-    return this.metadata.asV2;
+    assert(this.metadata.version === 3, `Cannot convert metadata from v${this.metadata.version} to v1`);
+
+    if (isUndefined(this._convertedV2)) {
+      this._convertedV2 = v3ToV2(this.metadata.asV3);
+    }
+
+    return this._convertedV2;
+  }
+
+  /**
+   * @description Returns the wrapped values as a V3 object
+   */
+  get asV3 (): MetadataV3 {
+    assert(this.metadata.version === 3, `Cannot convert metadata from v${this.metadata.version} to v3`);
+
+    return this.metadata.asV3;
   }
 
   getUniqTypes (throwError: boolean): Array<string> {

packages/types/src/Metadata/static.ts

@@ -2,6 +2,6 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
-import metadata from './v2/static';
+import metadata from './v3/static';
 
 export default metadata;

packages/types/src/Metadata/v0/Metadata.spec.ts

@@ -16,9 +16,6 @@ describe('Metadata', () => {
     const metadata = new Metadata(rpcData);
     const str = JSON.stringify(metadata.toJSON());
 
-    // console.error(str);
-    // console.error(metadata.getUniqTypes(true));
-
     expect(metadata.events.length).not.toBe(0);
     expect(str).toEqual(JSON.stringify(latestParsed));
   });

packages/types/src/Metadata/v1/Metadata.spec.ts

@@ -17,9 +17,6 @@ describe('MetadataV1', () => {
   it('decodes latest properly', () => {
     const str = JSON.stringify(metadata.toJSON());
 
-    // console.error(str);
-    // console.error(metadata.getUniqTypes(true));
-
     expect(metadata.version).toBe(1);
     expect(metadata.asV1.modules.length).not.toBe(0);
     expect(str).toEqual(JSON.stringify(latestParsed));
@@ -28,8 +25,6 @@ describe('MetadataV1', () => {
   it('converts v1 to V0', () => {
     const v0 = metadata.asV0;
 
-    // console.error(JSON.stringify(v0.toJSON()));
-
     expect(metadata.getUniqTypes(true)).toEqual(v0.getUniqTypes(true));
   });
 

packages/types/src/Metadata/v2/Metadata.spec.ts

@@ -17,9 +17,6 @@ describe('MetadataV2', () => {
   it('decodes latest properly', () => {
     const str = JSON.stringify(metadata.toJSON());
 
-    console.error(str);
-    console.error(metadata.getUniqTypes(true));
-
     expect(metadata.version).toBe(2);
     expect(metadata.asV2.modules.length).not.toBe(0);
     expect(str).toEqual(JSON.stringify(latestParsed));
@@ -28,16 +25,12 @@ describe('MetadataV2', () => {
   it('converts v2 to V1', () => {
     const v1 = metadata.asV1;
 
-    // console.error(JSON.stringify(v1.toJSON()));
-
     expect(metadata.getUniqTypes(true)).toEqual(v1.getUniqTypes(true));
   });
 
   it('converts v2 to V0', () => {
     const v1 = metadata.asV0;
 
-    // console.error(JSON.stringify(v0.toJSON()));
-
     expect(metadata.getUniqTypes(true)).toEqual(v1.getUniqTypes(true));
   });
 

packages/types/src/Metadata/v3/Metadata.spec.ts

@@ -0,0 +1,22 @@
+// Copyright 2017-2019 @polkadot/types authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import Metadata from '../Metadata';
+import latestParsed from './latest.substrate.v3.json';
+import rpcData from './static';
+
+describe('MetadataV3', () => {
+  const metadata = new Metadata(rpcData);
+
+  it('decodes latest properly', () => {
+    const str = JSON.stringify(metadata.toJSON());
+
+    console.error(str);
+    console.error(metadata.getUniqTypes(true));
+
+    expect(metadata.version).toBe(3);
+    expect(metadata.asV3.modules.length).not.toBe(0);
+    expect(str).toEqual(JSON.stringify(latestParsed));
+  });
+});

packages/types/src/Metadata/v3/Metadata.ts

@@ -0,0 +1,130 @@
+// Copyright 2017-2019 @polkadot/types authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { MetadataInterface } from '../types';
+
+import Option from '../../codec/Option';
+import Struct from '../../codec/Struct';
+import Vector from '../../codec/Vector';
+import Text from '../../primitive/Text';
+import { flattenUniq, validateTypes } from '../util';
+
+import { MetadataCall } from '../v1/Calls';
+import { MetadataEvent } from '../v1/Events';
+import { MetadataStorage } from './Storage';
+
+/**
+ * @name MetadataModule
+ * @description
+ * The definition of a module in the system
+ */
+export class MetadataModule extends Struct {
+  constructor (value?: any) {
+    super({
+      name: Text,
+      prefix: Text,
+      storage: Option.with(Vector.with(MetadataStorage)),
+      calls: Option.with(Vector.with(MetadataCall)),
+      events: Option.with(Vector.with(MetadataEvent))
+    }, value);
+  }
+
+  /**
+   * @description the module calls
+   */
+  get calls (): Option<Vector<MetadataCall>> {
+    return this.get('calls') as Option<Vector<MetadataCall>>;
+  }
+
+  /**
+   * @description the module events
+   */
+  get events (): Option<Vector<MetadataEvent>> {
+    return this.get('events') as Option<Vector<MetadataEvent>>;
+  }
+
+  /**
+   * @description the module name
+   */
+  get name (): Text {
+    return this.get('name') as Text;
+  }
+
+  /**
+   * @description the module prefix
+   */
+  get prefix (): Text {
+    return this.get('prefix') as Text;
+  }
+
+  /**
+   * @description the associated module storage
+   */
+  get storage (): Option<Vector<MetadataStorage>> {
+    return this.get('storage') as Option<Vector<MetadataStorage>>;
+  }
+}
+
+/**
+ * @name MetadataV3
+ * @description
+ * The runtime metadata as a decoded structure
+ */
+export default class MetadataV3 extends Struct implements MetadataInterface {
+  constructor (value?: any) {
+    super({
+      modules: Vector.with(MetadataModule)
+    }, value);
+  }
+
+  /**
+   * @description The associated modules for this structure
+   */
+  get modules (): Vector<MetadataModule> {
+    return this.get('modules') as Vector<MetadataModule>;
+  }
+
+  private get callNames () {
+    return this.modules.map((mod) =>
+      mod.calls.isNone
+        ? []
+        : mod.calls.unwrap().map((fn) =>
+          fn.args.map((arg) => arg.type.toString())
+        )
+    );
+  }
+
+  private get eventNames () {
+    return this.modules.map((mod) =>
+      mod.events.isNone
+        ? []
+        : mod.events.unwrap().map((event) =>
+          event.args.map((arg) => arg.toString())
+        )
+    );
+  }
+
+  private get storageNames () {
+    return this.modules.map((mod) =>
+      mod.storage.isNone
+        ? []
+        : mod.storage.unwrap().map((fn) =>
+          fn.type.isMap
+            ? [fn.type.asMap.key.toString(), fn.type.asMap.value.toString()]
+            : [fn.type.asType.toString()]
+        )
+    );
+  }
+
+  /**
+   * @description Helper to retrieve a list of all type that are found, sorted and de-deuplicated
+   */
+  getUniqTypes (throwError: boolean): Array<string> {
+    const types = flattenUniq([this.callNames, this.eventNames, this.storageNames]);
+
+    validateTypes(types, throwError);
+
+    return types;
+  }
+}

packages/types/src/Metadata/v3/Storage.ts

@@ -0,0 +1,160 @@
+// Copyright 2017-2019 @polkadot/types authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import EnumType from '../../codec/EnumType';
+import Struct from '../../codec/Struct';
+import Vector from '../../codec/Vector';
+import Bytes from '../../primitive/Bytes';
+import Text from '../../primitive/Text';
+import { MetadataStorageModifier } from '../v1/Storage';
+import { MapType, PlainType } from '../v2/Storage';
+
+export class DoubleMapType extends Struct {
+  constructor (value?: any) {
+    super({
+      key1: Text,
+      key2: Text,
+      value: Text,
+      keyHasher: Text
+    }, value);
+  }
+
+  /**
+   * @description The mapped key as [[Text]]
+   */
+  get key1 (): Text {
+    return this.get('key1') as Text;
+  }
+
+  /**
+   * @description The mapped key as [[Text]]
+   */
+  get key2 (): Text {
+    return this.get('key2') as Text;
+  }
+
+  /**
+   * @description The mapped key as [[Text]]
+   */
+  get keyHasher (): Text {
+    return this.get('keyHasher') as Text;
+  }
+
+  /**
+   * @description The mapped key as [[Text]]
+   */
+  get value (): Text {
+    return this.get('value') as Text;
+  }
+}
+
+export class MetadataStorageType extends EnumType<PlainType | MapType | DoubleMapType> {
+  constructor (value?: any, index?: number) {
+    super({
+      PlainType,
+      MapType,
+      DoubleMapType
+    }, value, index);
+  }
+
+  /**
+   * @description The value as a mapped value
+   */
+  get asDoubleMap (): DoubleMapType {
+    return this.value as DoubleMapType;
+  }
+
+  /**
+   * @description The value as a mapped value
+   */
+  get asMap (): MapType {
+    return this.value as MapType;
+  }
+
+  /**
+   * @description The value as a [[Type]] value
+   */
+  get asType (): PlainType {
+    return this.value as PlainType;
+  }
+
+  /**
+   * @description `true` if the storage entry is a doublemap
+   */
+  get isDoubleMap (): boolean {
+    return this.toNumber() === 2;
+  }
+
+  /**
+   * @description `true` if the storage entry is a map
+   */
+  get isMap (): boolean {
+    return this.toNumber() === 1;
+  }
+
+  /**
+   * @description Returns the string representation of the value
+   */
+  toString (): string {
+    if (this.isDoubleMap) {
+      return this.asDoubleMap.toString();
+    }
+
+    return this.isMap
+      ? this.asMap.value.toString()
+      : this.asType.toString();
+  }
+}
+
+/**
+ * @name MetadataModule
+ * @description
+ * The definition of a storage function
+ */
+export class MetadataStorage extends Struct {
+  constructor (value?: any) {
+    super({
+      name: Text,
+      modifier: MetadataStorageModifier,
+      type: MetadataStorageType,
+      fallback: Bytes,
+      docs: Vector.with(Text)
+    }, value);
+  }
+
+  /**
+   * @description The [[Text]] documentation
+   */
+  get docs (): Vector<Text> {
+    return this.get('docs') as Vector<Text>;
+  }
+
+  /**
+   * @description The [[Bytes]] fallback default
+   */
+  get fallback (): Bytes {
+    return this.get('fallback') as Bytes;
+  }
+
+  /**
+   * @description The [[MetadataArgument]] for arguments
+   */
+  get modifier (): MetadataStorageModifier {
+    return this.get('modifier') as MetadataStorageModifier;
+  }
+
+  /**
+   * @description The call name
+   */
+  get name (): Text {
+    return this.get('name') as Text;
+  }
+
+  /**
+   * @description The [[MetadataStorageType]]
+   */
+  get type (): MetadataStorageType {
+    return this.get('type') as MetadataStorageType;
+  }
+}

packages/types/src/Metadata/v3/index.ts

@@ -0,0 +1,11 @@
+// Copyright 2017-2019 @polkadot/types authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { default as MetadataV3, MetadataModule as MetadataModuleV3 } from './Metadata';
+
+export default MetadataV3;
+
+export {
+  MetadataModuleV3
+};

packages/types/src/Metadata/v3/toV2.ts

@@ -0,0 +1,10 @@
+// Copyright 2017-2019 @polkadot/types authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import MetadataV2 from '../v2/Metadata';
+import MetadataV3 from '.';
+
+export default function toV2 (v2: MetadataV3): MetadataV2 {
+  return new MetadataV2(v2);
+}