From c465252e693cbdb70ccb139b9844d739effd1bea Mon Sep 17 00:00:00 2001 From: Jaco Greeff Date: Sun, 16 Feb 2020 07:34:45 +0100 Subject: [PATCH] Make Metadata <-> hex encoding work (#1862) * Make Metadata <-> hex encoding work * Cleanup encoded, only toString exposes santized * toHex is actual original toHex, remove override * Update CHANGELOG * CHANGELOG for .entries argument typing * Update CHANGELOG.md * Manual check type for DoubleMap * Comment around super.toString usage * Runtime version --- CHANGELOG.md | 2 + packages/api/src/checkTypes.manual.ts | 4 + .../metadata/src/Metadata/Metadata.spec.ts | 18 +++ packages/metadata/src/Metadata/Metadata.ts | 54 +++---- packages/types/src/codec/Struct.ts | 146 +++++++++--------- packages/types/src/known/overrides.ts | 2 +- packages/types/src/primitive/Text.ts | 4 +- packages/types/src/primitive/Type.spec.ts | 12 +- packages/types/src/primitive/Type.ts | 21 +-- 9 files changed, 141 insertions(+), 122 deletions(-) create mode 100644 packages/metadata/src/Metadata/Metadata.spec.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 16f048b1b8..f02a3d6e17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # 1.2.0-beta.x - **Important** Update RPC status codes (latest Substrate 2), with the `isInBlock` (`isFinalized` now indicates finality) +- Storage `.entries(arg?: any)` now has the correct argument type for DoubleMap (Thanks to https://github.com/monitz87) - Swap Kusama and Polkadot aliasses for `Address` to `AccountId` (future update) - Add `LookupSource` and `LookupTarget` types instead of mapping these directly for aliassing - Add `BitVec` type primitive @@ -9,6 +10,7 @@ - Add derives for the parachains interfaces - Applied the latest types for Polkadot parachains - `registry.findMetaError` now can take in a `DispatchErrorModule` object for error lookups (from `asModule` in `DispatchError`) +- Metadata now correctly serializes to the original input data via `toHex()` & `toU8a()` - Metadata dcocumentation generation rework, including the output of errors from metadata - Update all examples and comments to use latest `system.account` queries (instead of `system.accountNonce` and `balances.freeBalance`/`balances.reservedBalance`) - Cleanup `codec/Date` tests to make it run properly on all timezones diff --git a/packages/api/src/checkTypes.manual.ts b/packages/api/src/checkTypes.manual.ts index 67ad1cc49b..082648c758 100644 --- a/packages/api/src/checkTypes.manual.ts +++ b/packages/api/src/checkTypes.manual.ts @@ -59,6 +59,10 @@ async function query (api: ApiPromise, keyring: TestKeyringMap): Promise { multiUnsub(); }); + + // check entries() + await api.query.system.account.entries(); // should not take a param + await api.query.staking.nominatorSlashInEra.entries(123); // should take a param } async function rpc (api: ApiPromise): Promise { diff --git a/packages/metadata/src/Metadata/Metadata.spec.ts b/packages/metadata/src/Metadata/Metadata.spec.ts new file mode 100644 index 0000000000..61bfcd29be --- /dev/null +++ b/packages/metadata/src/Metadata/Metadata.spec.ts @@ -0,0 +1,18 @@ +// Copyright 2017-2020 @polkadot/metadata authors & contributors +// This software may be modified and distributed under the terms +// of the Apache-2.0 license. See the LICENSE file for details. + +import { TypeRegistry } from '@polkadot/types'; + +import substrateData from './static'; +import Metadata from './Metadata'; + +describe('Metadata', (): void => { + it('allows creation from hex with JSON equivalence', (): void => { + const test = new Metadata(new TypeRegistry(), substrateData); + + expect( + new Metadata(new TypeRegistry(), test.toHex()).toJSON() + ).toEqual(test.toJSON()); + }); +}); diff --git a/packages/metadata/src/Metadata/Metadata.ts b/packages/metadata/src/Metadata/Metadata.ts index 92d6f1258a..4dab004158 100644 --- a/packages/metadata/src/Metadata/Metadata.ts +++ b/packages/metadata/src/Metadata/Metadata.ts @@ -9,6 +9,32 @@ import { bnToU8a, isHex, hexToU8a, u8aConcat } from '@polkadot/util'; import { MAGIC_NUMBER } from './MagicNumber'; import MetadataVersioned from './MetadataVersioned'; +// first we try and parse using the versioned structure, if this does fail, +// we adjust with the magic number and a manual version and re-try. As soon as +// we remove support for V0, we will just do a new here +function decodeMetadata (registry: Registry, _value: Uint8Array | string = new Uint8Array()): MetadataVersioned { + const value = isHex(_value) + ? hexToU8a(_value) + : _value; + + try { + return new MetadataVersioned(registry, value); + } catch (error) { + if (error.message.indexOf('MagicNumber mismatch') === -1) { + throw error; + } + } + + return new MetadataVersioned( + registry, + u8aConcat( + bnToU8a(MAGIC_NUMBER), // manually add the magic number + Uint8Array.from([0]), // add the version for the original + value // the actual data as retrieved + ) + ); +} + /** * @name Metadata * @description @@ -16,32 +42,6 @@ import MetadataVersioned from './MetadataVersioned'; */ export default class Metadata extends MetadataVersioned { constructor (registry: Registry, value?: Uint8Array | string) { - super(registry, Metadata.decodeMetadata(registry, value)); - } - - // first we try and parse using the versioned structure, if this does fail, - // we adjust with the magic number and a manual version and re-try. As soon as - // we remove support for V0, we will just do a new here - private static decodeMetadata (registry: Registry, _value: Uint8Array | string = new Uint8Array()): MetadataVersioned { - const value = isHex(_value) - ? hexToU8a(_value) - : _value; - - try { - return new MetadataVersioned(registry, value); - } catch (error) { - if (error.message.indexOf('MagicNumber mismatch') === -1) { - throw error; - } - } - - return new MetadataVersioned( - registry, - u8aConcat( - bnToU8a(MAGIC_NUMBER), // manually add the magic number - Uint8Array.from([0]), // add the version for the original - value // the actual data as retrieved - ) - ); + super(registry, decodeMetadata(registry, value)); } } diff --git a/packages/types/src/codec/Struct.ts b/packages/types/src/codec/Struct.ts index 16e722b205..d439f14677 100644 --- a/packages/types/src/codec/Struct.ts +++ b/packages/types/src/codec/Struct.ts @@ -12,6 +12,77 @@ import { compareMap, decodeU8a, mapToTypeMap } from './utils'; type TypesDef = Record>; +/** @internal */ +function decodeStructFromObject (registry: Registry, Types: ConstructorDef, value: any, jsonMap: Map): T { + return Object.keys(Types).reduce((raw, key, index): T => { + // The key in the JSON can be snake_case (or other cases), but in our + // Types, result or any other maps, it's camelCase + const jsonKey = (jsonMap.get(key as any) && !value[key]) ? jsonMap.get(key as any) : key; + + try { + if (Array.isArray(value)) { + // TS2322: Type 'Codec' is not assignable to type 'T[keyof S]'. + (raw as any)[key] = value[index] instanceof Types[key] + ? value[index] + : new Types[key](registry, value[index]); + } else if (value instanceof Map) { + const mapped = value.get(jsonKey); + + (raw as any)[key] = mapped instanceof Types[key] + ? mapped + : new Types[key](registry, mapped); + } else if (isObject(value)) { + (raw as any)[key] = value[jsonKey as string] instanceof Types[key] + ? value[jsonKey as string] + : new Types[key](registry, value[jsonKey as string]); + } else { + throw new Error(`Struct: cannot decode type ${Types[key].name} with value ${JSON.stringify(value)}`); + } + } catch (error) { + throw new Error(`Struct: failed on '${jsonKey}':: ${error.message}`); + } + + return raw; + }, {} as T); +} + +/** + * Decode input to pass into constructor. + * + * @param Types - Types definition. + * @param value - Value to decode, one of: + * - null + * - undefined + * - hex + * - Uint8Array + * - object with `{ key1: value1, key2: value2 }`, assuming `key1` and `key2` + * are also keys in `Types` + * - array with `[value1, value2]` assuming the array has the same length as + * `Object.keys(Types)` + * @param jsonMap + * @internal + */ +function decodeStruct (registry: Registry, Types: ConstructorDef, value: any, jsonMap: Map): T { + if (isHex(value)) { + return decodeStruct(registry, Types, hexToU8a(value as string), jsonMap); + } else if (isU8a(value)) { + const values = decodeU8a(registry, value, Object.values(Types)); + + // Transform array of values to {key: value} mapping + return Object.keys(Types).reduce((raw, key, index): T => { + // TS2322: Type 'Codec' is not assignable to type 'T[keyof S]'. + (raw as any)[key] = values[index]; + + return raw; + }, {} as T); + } else if (!value) { + return {} as T; + } + + // We assume from here that value is a JS object (Array, Map, Object) + return decodeStructFromObject(registry, Types, value, jsonMap); +} + /** * @name Struct * @description @@ -39,7 +110,7 @@ export default class Struct< constructor (registry: Registry, Types: S, value: V | Map | any[] | string = {} as V, jsonMap: Map = new Map()) { const Clazzes = mapToTypeMap(registry, Types); - const decoded: T = Struct.decodeStruct(registry, Clazzes, value, jsonMap); + const decoded: T = decodeStruct(registry, Clazzes, value, jsonMap); super(Object.entries(decoded)); @@ -48,77 +119,6 @@ export default class Struct< this._Types = Clazzes; } - /** - * Decode input to pass into constructor. - * - * @param Types - Types definition. - * @param value - Value to decode, one of: - * - null - * - undefined - * - hex - * - Uint8Array - * - object with `{ key1: value1, key2: value2 }`, assuming `key1` and `key2` - * are also keys in `Types` - * - array with `[value1, value2]` assuming the array has the same length as - * `Object.keys(Types)` - * @param jsonMap - * @internal - */ - private static decodeStruct (registry: Registry, Types: ConstructorDef, value: any, jsonMap: Map): T { - if (isHex(value)) { - return Struct.decodeStruct(registry, Types, hexToU8a(value as string), jsonMap); - } else if (isU8a(value)) { - const values = decodeU8a(registry, value, Object.values(Types)); - - // Transform array of values to {key: value} mapping - return Object.keys(Types).reduce((raw, key, index): T => { - // TS2322: Type 'Codec' is not assignable to type 'T[keyof S]'. - (raw as any)[key] = values[index]; - - return raw; - }, {} as T); - } else if (!value) { - return {} as T; - } - - // We assume from here that value is a JS object (Array, Map, Object) - return Struct.decodeStructFromObject(registry, Types, value, jsonMap); - } - - /** @internal */ - private static decodeStructFromObject (registry: Registry, Types: ConstructorDef, value: any, jsonMap: Map): T { - return Object.keys(Types).reduce((raw, key, index): T => { - // The key in the JSON can be snake_case (or other cases), but in our - // Types, result or any other maps, it's camelCase - const jsonKey = (jsonMap.get(key as any) && !value[key]) ? jsonMap.get(key as any) : key; - - try { - if (Array.isArray(value)) { - // TS2322: Type 'Codec' is not assignable to type 'T[keyof S]'. - (raw as any)[key] = value[index] instanceof Types[key] - ? value[index] - : new Types[key](registry, value[index]); - } else if (value instanceof Map) { - const mapped = value.get(jsonKey); - - (raw as any)[key] = mapped instanceof Types[key] - ? mapped - : new Types[key](registry, mapped); - } else if (isObject(value)) { - (raw as any)[key] = value[jsonKey as string] instanceof Types[key] - ? value[jsonKey as string] - : new Types[key](registry, value[jsonKey as string]); - } else { - throw new Error(`Struct: cannot decode type ${Types[key].name} with value ${JSON.stringify(value)}`); - } - } catch (error) { - throw new Error(`Struct: failed on '${jsonKey}':: ${error.message}`); - } - - return raw; - }, {} as T); - } - public static with (Types: S): Constructor> { return class extends Struct { constructor (registry: Registry, value?: any, jsonMap?: Map) { @@ -231,7 +231,7 @@ export default class Struct< /** * @description Converts the Object to JSON, typically used for RPC transfers */ - public toJSON (): AnyJsonObject | string { + public toJSON (): AnyJsonObject { // FIXME the return type string is only used by Extrinsic (extends Struct), // but its toJSON is the hex value return [...this.keys()].reduce((json, key): any => { diff --git a/packages/types/src/known/overrides.ts b/packages/types/src/known/overrides.ts index fe8db7994f..f78bb1745e 100644 --- a/packages/types/src/known/overrides.ts +++ b/packages/types/src/known/overrides.ts @@ -38,7 +38,7 @@ const TYPES_POLKADOT_VERSIONED: OverrideVersionedType[] = [ } }, { - minmax: [1010, undefined], + minmax: [1004, undefined], types: { // Indices optional, not in transaction Address: 'AccountId', diff --git a/packages/types/src/primitive/Text.ts b/packages/types/src/primitive/Text.ts index d875f3d756..53001c77da 100644 --- a/packages/types/src/primitive/Text.ts +++ b/packages/types/src/primitive/Text.ts @@ -137,7 +137,9 @@ export default class Text extends String implements Codec { * @param isBare true when the value has none of the type-specific prefixes (internal) */ public toU8a (isBare?: boolean): Uint8Array { - const encoded = stringToU8a(this.toString()); + // NOTE Here we use the super toString (we are not taking overrides into account, + // rather encoding the original value the string was constructed with) + const encoded = stringToU8a(super.toString()); return isBare ? encoded diff --git a/packages/types/src/primitive/Type.spec.ts b/packages/types/src/primitive/Type.spec.ts index d602111335..0c37da4900 100644 --- a/packages/types/src/primitive/Type.spec.ts +++ b/packages/types/src/primitive/Type.spec.ts @@ -71,7 +71,7 @@ describe('Type', (): void => { ).toEqual('()'); }); - it('has a length for the type', (): void => { + it('has the sanitized', (): void => { expect( new Type( registry, @@ -102,4 +102,14 @@ describe('Type', (): void => { ) ); }); + + it('creates a decodable U8a for sanitized types', (): void => { + const original = '::Inherent'; + const expected = 'InherentOfflineReport'; + const u8a = new Type(registry, original).toU8a(); + const decoded = new Type(registry, u8a); + + expect(decoded.encodedLength).toEqual(original.length + 1); // extra byte for length + expect(decoded.toString()).toEqual(expected); + }); }); diff --git a/packages/types/src/primitive/Type.ts b/packages/types/src/primitive/Type.ts index 483c43a78f..53b178f27a 100644 --- a/packages/types/src/primitive/Type.ts +++ b/packages/types/src/primitive/Type.ts @@ -15,27 +15,10 @@ import Text from './Text'; * i.e. we remove the `T::` prefixes found in some types for consistency across implementation. */ export default class Type extends Text { - private _originalLength: number; - constructor (registry: Registry, value: Text | Uint8Array | string = '') { - // First decode it with Text - const textValue = new Text(registry, value); + super(registry, value); - // Then cleanup the textValue to get the @polkadot/types type, and pass the - // sanitized value to constructor - super(registry, sanitize(textValue.toString())); - - this._originalLength = textValue.encodedLength; - } - - /** - * @description The length of the value when encoded as a Uint8Array - */ - public get encodedLength (): number { - // NOTE Length is used in the decoding calculations, so return the original (pre-cleanup) - // length of the data. Since toU8a is disabled, this does not affect encoding, but rather - // only the decoding leg, allowing the decoders to work with original pointers - return this._originalLength; + this.setOverride(sanitize(this.toString())); } /**