Make Metadata <-> hex encoding work (#1862)

* Make Metadata <-> hex encoding work

* Cleanup encoded, only toString exposes santized

* toHex is actual original toHex, remove override

* Update CHANGELOG

* CHANGELOG for .entries argument typing

* Update CHANGELOG.md

* Manual check type for DoubleMap

* Comment around super.toString usage

* Runtime version

packages/api/src/checkTypes.manual.ts

@@ -59,6 +59,10 @@ async function query (api: ApiPromise, keyring: TestKeyringMap): Promise<void> {
 
     multiUnsub();
   });
+
+  // check entries()
+  await api.query.system.account.entries(); // should not take a param
+  await api.query.staking.nominatorSlashInEra.entries(123); // should take a param
 }
 
 async function rpc (api: ApiPromise): Promise<void> {

packages/metadata/src/Metadata/Metadata.spec.ts

@@ -0,0 +1,18 @@
+// Copyright 2017-2020 @polkadot/metadata authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { TypeRegistry } from '@polkadot/types';
+
+import substrateData from './static';
+import Metadata from './Metadata';
+
+describe('Metadata', (): void => {
+  it('allows creation from hex with JSON equivalence', (): void => {
+    const test = new Metadata(new TypeRegistry(), substrateData);
+
+    expect(
+      new Metadata(new TypeRegistry(), test.toHex()).toJSON()
+    ).toEqual(test.toJSON());
+  });
+});

packages/metadata/src/Metadata/Metadata.ts

@@ -9,6 +9,32 @@ import { bnToU8a, isHex, hexToU8a, u8aConcat } from '@polkadot/util';
 import { MAGIC_NUMBER } from './MagicNumber';
 import MetadataVersioned from './MetadataVersioned';
 
+// first we try and parse using the versioned structure, if this does fail,
+// we adjust with the magic number and a manual version and re-try. As soon as
+// we remove support for V0, we will just do a new here
+function decodeMetadata (registry: Registry, _value: Uint8Array | string = new Uint8Array()): MetadataVersioned {
+  const value = isHex(_value)
+    ? hexToU8a(_value)
+    : _value;
+
+  try {
+    return new MetadataVersioned(registry, value);
+  } catch (error) {
+    if (error.message.indexOf('MagicNumber mismatch') === -1) {
+      throw error;
+    }
+  }
+
+  return new MetadataVersioned(
+    registry,
+    u8aConcat(
+      bnToU8a(MAGIC_NUMBER), // manually add the magic number
+      Uint8Array.from([0]), // add the version for the original
+      value // the actual data as retrieved
+    )
+  );
+}
+
 /**
  * @name Metadata
  * @description
@@ -16,32 +42,6 @@ import MetadataVersioned from './MetadataVersioned';
  */
 export default class Metadata extends MetadataVersioned {
   constructor (registry: Registry, value?: Uint8Array | string) {
-    super(registry, Metadata.decodeMetadata(registry, value));
-  }
-
-  // first we try and parse using the versioned structure, if this does fail,
-  // we adjust with the magic number and a manual version and re-try. As soon as
-  // we remove support for V0, we will just do a new here
-  private static decodeMetadata (registry: Registry, _value: Uint8Array | string = new Uint8Array()): MetadataVersioned {
-    const value = isHex(_value)
-      ? hexToU8a(_value)
-      : _value;
-
-    try {
-      return new MetadataVersioned(registry, value);
-    } catch (error) {
-      if (error.message.indexOf('MagicNumber mismatch') === -1) {
-        throw error;
-      }
-    }
-
-    return new MetadataVersioned(
-      registry,
-      u8aConcat(
-        bnToU8a(MAGIC_NUMBER), // manually add the magic number
-        Uint8Array.from([0]), // add the version for the original
-        value // the actual data as retrieved
-      )
-    );
+    super(registry, decodeMetadata(registry, value));
   }
 }

packages/types/src/codec/Struct.ts

@@ -12,6 +12,77 @@ import { compareMap, decodeU8a, mapToTypeMap } from './utils';
 
 type TypesDef<T = Codec> = Record<string, InterfaceTypes | Constructor<T>>;
 
+/** @internal */
+function decodeStructFromObject <T> (registry: Registry, Types: ConstructorDef, value: any, jsonMap: Map<any, string>): T {
+  return Object.keys(Types).reduce((raw, key, index): T => {
+    // The key in the JSON can be snake_case (or other cases), but in our
+    // Types, result or any other maps, it's camelCase
+    const jsonKey = (jsonMap.get(key as any) && !value[key]) ? jsonMap.get(key as any) : key;
+
+    try {
+      if (Array.isArray(value)) {
+        // TS2322: Type 'Codec' is not assignable to type 'T[keyof S]'.
+        (raw as any)[key] = value[index] instanceof Types[key]
+          ? value[index]
+          : new Types[key](registry, value[index]);
+      } else if (value instanceof Map) {
+        const mapped = value.get(jsonKey);
+
+        (raw as any)[key] = mapped instanceof Types[key]
+          ? mapped
+          : new Types[key](registry, mapped);
+      } else if (isObject(value)) {
+        (raw as any)[key] = value[jsonKey as string] instanceof Types[key]
+          ? value[jsonKey as string]
+          : new Types[key](registry, value[jsonKey as string]);
+      } else {
+        throw new Error(`Struct: cannot decode type ${Types[key].name} with value ${JSON.stringify(value)}`);
+      }
+    } catch (error) {
+      throw new Error(`Struct: failed on '${jsonKey}':: ${error.message}`);
+    }
+
+    return raw;
+  }, {} as T);
+}
+
+/**
+ * Decode input to pass into constructor.
+ *
+ * @param Types - Types definition.
+ * @param value - Value to decode, one of:
+ * - null
+ * - undefined
+ * - hex
+ * - Uint8Array
+ * - object with `{ key1: value1, key2: value2 }`, assuming `key1` and `key2`
+ * are also keys in `Types`
+ * - array with `[value1, value2]` assuming the array has the same length as
+ * `Object.keys(Types)`
+ * @param jsonMap
+ * @internal
+ */
+function decodeStruct <T> (registry: Registry, Types: ConstructorDef, value: any, jsonMap: Map<any, string>): T {
+  if (isHex(value)) {
+    return decodeStruct(registry, Types, hexToU8a(value as string), jsonMap);
+  } else if (isU8a(value)) {
+    const values = decodeU8a(registry, value, Object.values(Types));
+
+    // Transform array of values to {key: value} mapping
+    return Object.keys(Types).reduce((raw, key, index): T => {
+      // TS2322: Type 'Codec' is not assignable to type 'T[keyof S]'.
+      (raw as any)[key] = values[index];
+
+      return raw;
+    }, {} as T);
+  } else if (!value) {
+    return {} as T;
+  }
+
+  // We assume from here that value is a JS object (Array, Map, Object)
+  return decodeStructFromObject(registry, Types, value, jsonMap);
+}
+
 /**
  * @name Struct
  * @description
@@ -39,7 +110,7 @@ export default class Struct<
 
   constructor (registry: Registry, Types: S, value: V | Map<any, any> | any[] | string = {} as V, jsonMap: Map<keyof S, string> = new Map()) {
     const Clazzes = mapToTypeMap(registry, Types);
-    const decoded: T = Struct.decodeStruct(registry, Clazzes, value, jsonMap);
+    const decoded: T = decodeStruct(registry, Clazzes, value, jsonMap);
 
     super(Object.entries(decoded));
 
@@ -48,77 +119,6 @@ export default class Struct<
     this._Types = Clazzes;
   }
 
-  /**
-   * Decode input to pass into constructor.
-   *
-   * @param Types - Types definition.
-   * @param value - Value to decode, one of:
-   * - null
-   * - undefined
-   * - hex
-   * - Uint8Array
-   * - object with `{ key1: value1, key2: value2 }`, assuming `key1` and `key2`
-   * are also keys in `Types`
-   * - array with `[value1, value2]` assuming the array has the same length as
-   * `Object.keys(Types)`
-   * @param jsonMap
-   * @internal
-   */
-  private static decodeStruct <T> (registry: Registry, Types: ConstructorDef, value: any, jsonMap: Map<any, string>): T {
-    if (isHex(value)) {
-      return Struct.decodeStruct(registry, Types, hexToU8a(value as string), jsonMap);
-    } else if (isU8a(value)) {
-      const values = decodeU8a(registry, value, Object.values(Types));
-
-      // Transform array of values to {key: value} mapping
-      return Object.keys(Types).reduce((raw, key, index): T => {
-        // TS2322: Type 'Codec' is not assignable to type 'T[keyof S]'.
-        (raw as any)[key] = values[index];
-
-        return raw;
-      }, {} as T);
-    } else if (!value) {
-      return {} as T;
-    }
-
-    // We assume from here that value is a JS object (Array, Map, Object)
-    return Struct.decodeStructFromObject(registry, Types, value, jsonMap);
-  }
-
-  /** @internal */
-  private static decodeStructFromObject <T> (registry: Registry, Types: ConstructorDef, value: any, jsonMap: Map<any, string>): T {
-    return Object.keys(Types).reduce((raw, key, index): T => {
-      // The key in the JSON can be snake_case (or other cases), but in our
-      // Types, result or any other maps, it's camelCase
-      const jsonKey = (jsonMap.get(key as any) && !value[key]) ? jsonMap.get(key as any) : key;
-
-      try {
-        if (Array.isArray(value)) {
-          // TS2322: Type 'Codec' is not assignable to type 'T[keyof S]'.
-          (raw as any)[key] = value[index] instanceof Types[key]
-            ? value[index]
-            : new Types[key](registry, value[index]);
-        } else if (value instanceof Map) {
-          const mapped = value.get(jsonKey);
-
-          (raw as any)[key] = mapped instanceof Types[key]
-            ? mapped
-            : new Types[key](registry, mapped);
-        } else if (isObject(value)) {
-          (raw as any)[key] = value[jsonKey as string] instanceof Types[key]
-            ? value[jsonKey as string]
-            : new Types[key](registry, value[jsonKey as string]);
-        } else {
-          throw new Error(`Struct: cannot decode type ${Types[key].name} with value ${JSON.stringify(value)}`);
-        }
-      } catch (error) {
-        throw new Error(`Struct: failed on '${jsonKey}':: ${error.message}`);
-      }
-
-      return raw;
-    }, {} as T);
-  }
-
   public static with<S extends TypesDef> (Types: S): Constructor<Struct<S>> {
     return class extends Struct<S> {
       constructor (registry: Registry, value?: any, jsonMap?: Map<keyof S, string>) {
@@ -231,7 +231,7 @@ export default class Struct<
   /**
    * @description Converts the Object to JSON, typically used for RPC transfers
    */
-  public toJSON (): AnyJsonObject | string {
+  public toJSON (): AnyJsonObject {
     // FIXME the return type string is only used by Extrinsic (extends Struct),
     // but its toJSON is the hex value
     return [...this.keys()].reduce((json, key): any => {

packages/types/src/known/overrides.ts

@@ -38,7 +38,7 @@ const TYPES_POLKADOT_VERSIONED: OverrideVersionedType[] = [
     }
   },
   {
-    minmax: [1010, undefined],
+    minmax: [1004, undefined],
     types: {
       // Indices optional, not in transaction
       Address: 'AccountId',

packages/types/src/primitive/Text.ts

@@ -137,7 +137,9 @@ export default class Text extends String implements Codec {
    * @param isBare true when the value has none of the type-specific prefixes (internal)
    */
   public toU8a (isBare?: boolean): Uint8Array {
-    const encoded = stringToU8a(this.toString());
+    // NOTE Here we use the super toString (we are not taking overrides into account,
+    // rather encoding the original value the string was constructed with)
+    const encoded = stringToU8a(super.toString());
 
     return isBare
       ? encoded

packages/types/src/primitive/Type.spec.ts

@@ -71,7 +71,7 @@ describe('Type', (): void => {
     ).toEqual('()');
   });
 
-  it('has a length for the type', (): void => {
+  it('has the sanitized', (): void => {
     expect(
       new Type(
         registry,
@@ -102,4 +102,14 @@ describe('Type', (): void => {
       )
     );
   });
+
+  it('creates a decodable U8a for sanitized types', (): void => {
+    const original = '<T::InherentOfflineReport as InherentOfflineReport>::Inherent';
+    const expected = 'InherentOfflineReport';
+    const u8a = new Type(registry, original).toU8a();
+    const decoded = new Type(registry, u8a);
+
+    expect(decoded.encodedLength).toEqual(original.length + 1); // extra byte for length
+    expect(decoded.toString()).toEqual(expected);
+  });
 });

packages/types/src/primitive/Type.ts

@@ -15,27 +15,10 @@ import Text from './Text';
  * i.e. we remove the `T::` prefixes found in some types for consistency across implementation.
  */
 export default class Type extends Text {
-  private _originalLength: number;
-
   constructor (registry: Registry, value: Text | Uint8Array | string = '') {
-    // First decode it with Text
-    const textValue = new Text(registry, value);
+    super(registry, value);
 
-    // Then cleanup the textValue to get the @polkadot/types type, and pass the
-    // sanitized value to constructor
-    super(registry, sanitize(textValue.toString()));
-
-    this._originalLength = textValue.encodedLength;
-  }
-
-  /**
-   * @description The length of the value when encoded as a Uint8Array
-   */
-  public get encodedLength (): number {
-    // NOTE Length is used in the decoding calculations, so return the original (pre-cleanup)
-    // length of the data. Since toU8a is disabled, this does not affect encoding, but rather
-    // only the decoding leg, allowing the decoders to work with original pointers
-    return this._originalLength;
+    this.setOverride(sanitize(this.toString()));
   }
 
   /**