From 833b49d0120baca9994d0479434ec5dd53e1e726 Mon Sep 17 00:00:00 2001 From: Alex Wang Date: Tue, 21 May 2019 20:25:58 +1200 Subject: [PATCH] support double map storage function (#934) * support double map storage function * add twox_64_concat, clean comments * change Twox128Concat to Twox64Concat * update decodeStorageKey * fix: update decodeStorageKey * test: add e2e test --- packages/api/test/e2e/promise-queries.spec.js | 15 +++++ .../src/fromMetadata/createFunction.spec.ts | 60 ++++++++++++++++++- .../src/fromMetadata/createFunction.ts | 30 ++++++++-- .../src/fromMetadata/getHasher.ts | 28 ++++++++- packages/types/src/Metadata/v4/Storage.ts | 2 +- packages/types/src/codec/createType.spec.ts | 12 +++- packages/types/src/codec/createType.ts | 9 ++- packages/types/src/primitive/StorageHasher.ts | 6 +- .../types/src/primitive/StorageKey.spec.ts | 28 ++++++++- packages/types/src/primitive/StorageKey.ts | 14 ++++- 10 files changed, 182 insertions(+), 22 deletions(-) diff --git a/packages/api/test/e2e/promise-queries.spec.js b/packages/api/test/e2e/promise-queries.spec.js index 90a08f0513..ed89204da8 100644 --- a/packages/api/test/e2e/promise-queries.spec.js +++ b/packages/api/test/e2e/promise-queries.spec.js @@ -168,4 +168,19 @@ describe.skip('e2e queries', () => { done(); }); }); + + it('queries state using double map key', async () => { + // TODO Update ['any', '0x1234'] to the key of a known event topic and update '[]' to the expected value + const eventTopics = await api.query.system.eventTopics(['any', '0x1234']); + + expect(eventTopics.toString()).toEqual('[]'); + }); + + it('subscribes to queries using double map key', async (done) => { + // TODO Update ['any', '0x1234'] to the key of a known event topic and update '[]' to the expected value + api.query.system.eventTopics(['any', '0x1234'], (eventTopics) => { + expect(eventTopics.toString()).toEqual('[]'); + done(); + }); + }); }); diff --git a/packages/type-storage/src/fromMetadata/createFunction.spec.ts b/packages/type-storage/src/fromMetadata/createFunction.spec.ts index a0695230f3..b4fff1cacf 100644 --- a/packages/type-storage/src/fromMetadata/createFunction.spec.ts +++ b/packages/type-storage/src/fromMetadata/createFunction.spec.ts @@ -2,7 +2,9 @@ // This software may be modified and distributed under the terms // of the Apache-2.0 license. See the LICENSE file for details. -import { stringToU8a, u8aConcat } from '@polkadot/util'; +import { StorageHasher, Text } from '@polkadot/types'; +import { StorageFunction } from '@polkadot/types/primitive/StorageKey'; +import { stringToU8a, u8aConcat, u8aToHex } from '@polkadot/util'; import createFunction from './createFunction'; @@ -53,4 +55,60 @@ describe('createFunction', () => { ) ); }); + + describe('the created double map function', () => { + let storageFn: StorageFunction; + beforeAll(() => { + storageFn = createFunction( + 'GenericAsset', + 'FreeBalance', { + name: 'metaName', + type: { + isDoubleMap: true, + asDoubleMap: { + hasher: new StorageHasher('Blake2_256'), + key1: new Text('AccountId'), + key2: new Text('AccountId'), + value: new Text('Balance'), + key2Hasher: new Text('twox_128') + } + } + } as any + ); + }); + + it('should return correct key', () => { + const result = storageFn(['5DXUeE5N5LtkW97F2PzqYPyqNkxqSWESdGSPTX6AvkUAhwKP', '5DXUeE5N5LtkW97F2PzqYPyqNkxqSWESdGSPTX6AvkUAhwKP']); + expect(u8aToHex(result)).toEqual('0x00fa40e72d7173e69ee54b980345ea01cb81e64258502e0247af4303dee91ec0aec2ecd3a60ab080cff7b52a8f6d543b'); + }); + + it('needs two arguments', () => { + expect( + () => storageFn(['5DXUeE5N5LtkW97F2PzqYPyqNkxqSWESdGSPTX6AvkUAhwKP']) + ).toThrow(/metaName expects two arguments/); + }); + }); + + it('allows creates double map function with a Null type key', () => { + const storageFn = createFunction( + 'System', + 'EventTopics', + { + type: { + isDoubleMap: true, + asDoubleMap: { + hasher: new StorageHasher('Blake2_256'), + key1: new Text('Null'), + key2: new Text('Hash'), + value: new Text('Vec<(BlockNumber,EventIndex)>'), + key2Hasher: new Text('blake2_256') + } + } + } as any + ); + // the value of the Null type key does not effect the result + expect(u8aToHex(storageFn(['any', [1, 2, 3]]))).toEqual(u8aToHex(storageFn([[1, 2, 3], [1, 2, 3]]))); + // the value of the not Null type key does effect the result + expect(u8aToHex(storageFn(['any', [1, 2, 3]]))).not.toEqual(u8aToHex(storageFn(['any', [1, 2, 3, 4]]))); + }); }); diff --git a/packages/type-storage/src/fromMetadata/createFunction.ts b/packages/type-storage/src/fromMetadata/createFunction.ts index c8943befbf..2e1fdcecb7 100644 --- a/packages/type-storage/src/fromMetadata/createFunction.ts +++ b/packages/type-storage/src/fromMetadata/createFunction.ts @@ -7,7 +7,7 @@ import { PlainType, StorageFunctionMetadata, StorageFunctionModifier, StorageFun import { StorageFunction } from '@polkadot/types/primitive/StorageKey'; import { assert, isNull, isUndefined, stringLowerFirst, stringToU8a, u8aConcat } from '@polkadot/util'; -import getHasher from './getHasher'; +import getHasher, { HasherFunction } from './getHasher'; export interface CreateItemOptions { key?: string; @@ -31,16 +31,36 @@ export default function createFunction (section: Text | string, method: Text | s const rawKey = stringToU8a(stringKey); // Get the hashing function - // FIXME Hash correctly for double map too - const hasher = meta.type.isMap - ? getHasher(meta.type.asMap.hasher) - : getHasher(); + let hasher: HasherFunction; + let key2Hasher: HasherFunction; + + if (meta.type.isDoubleMap) { + hasher = getHasher(meta.type.asDoubleMap.hasher); + key2Hasher = getHasher(meta.type.asDoubleMap.key2Hasher); + } else if (meta.type.isMap) { + hasher = getHasher(meta.type.asMap.hasher); + } else { + hasher = getHasher(); + } // Can only have zero or one argument: // - storage.balances.freeBalance(address) // - storage.timestamp.blockPeriod() const storageFn = (arg?: any): Uint8Array => { let key = rawKey; + + if (meta.type.isDoubleMap) { + assert(!isUndefined(arg) && !isNull(arg) && !isUndefined(arg[0]) && !isNull(arg[0]) && !isUndefined(arg[1]) && !isNull(arg[1]), `${meta.name} expects two arguments`); + const type1 = meta.type.asDoubleMap.key1.toString(); + const type2 = meta.type.asDoubleMap.key2.toString(); + + const param1Encoded = u8aConcat(key, createType(type1, arg[0]).toU8a(true)); + const param1Hashed = hasher(param1Encoded); + const param2Hashed = key2Hasher(createType(type2, arg[1]).toU8a(true)); + + return u8aConcat(param1Hashed, param2Hashed); + } + if (meta.type.isMap) { assert(!isUndefined(arg) && !isNull(arg), `${meta.name} expects one argument`); diff --git a/packages/type-storage/src/fromMetadata/getHasher.ts b/packages/type-storage/src/fromMetadata/getHasher.ts index cfef3688db..17651e144d 100644 --- a/packages/type-storage/src/fromMetadata/getHasher.ts +++ b/packages/type-storage/src/fromMetadata/getHasher.ts @@ -2,19 +2,41 @@ // This software may be modified and distributed under the terms // of the Apache-2.0 license. See the LICENSE file for details. +import { Text } from '@polkadot/types'; import { StorageHasher } from '@polkadot/types/primitive'; import { blake2AsU8a, xxhashAsU8a } from '@polkadot/util-crypto'; type HasherInput = string | Buffer | Uint8Array; -type HasherFunction = (data: HasherInput) => Uint8Array; +export type HasherFunction = (data: HasherInput) => Uint8Array; -export default function getHasher (hasher?: StorageHasher): HasherFunction { +function toStorageHasher (text: Text): StorageHasher { + switch (text.toString()) { + case 'blake2_128': + return new StorageHasher('Blake2_128'); + case 'blake2_256': + return new StorageHasher('Blake2_256'); + case 'twox_128': + return new StorageHasher('Twox128'); + case 'twox_256': + return new StorageHasher('Twox256'); + case 'twox_64_concat': + return new StorageHasher('Twox64Concat'); + default: + throw new Error(`Invalid Storage hasher: ${text.toString()}`); + } +} + +export default function getHasher (hasherInput?: StorageHasher | Text): HasherFunction { // This one is the default for PlainType storage keys - if (!hasher) { + if (!hasherInput) { return (data: HasherInput) => xxhashAsU8a(data, 128); } + const hasher: StorageHasher = hasherInput instanceof Text + ? toStorageHasher(hasherInput) + : hasherInput; + if (hasher.isBlake2128) { return (data: HasherInput) => blake2AsU8a(data, 128); } diff --git a/packages/types/src/Metadata/v4/Storage.ts b/packages/types/src/Metadata/v4/Storage.ts index 581e58a1fa..9e0417cc2b 100644 --- a/packages/types/src/Metadata/v4/Storage.ts +++ b/packages/types/src/Metadata/v4/Storage.ts @@ -170,7 +170,7 @@ export class StorageFunctionType extends Enum { */ toString (): string { if (this.isDoubleMap) { - return `DoubleMap<${this.asDoubleMap.toString()}>`; + return `DoubleMap<${this.asDoubleMap.value.toString()}>`; } if (this.isMap) { diff --git a/packages/types/src/codec/createType.spec.ts b/packages/types/src/codec/createType.spec.ts index 1c80400ae3..6491fab6c5 100644 --- a/packages/types/src/codec/createType.spec.ts +++ b/packages/types/src/codec/createType.spec.ts @@ -63,10 +63,10 @@ describe('getTypeValue', () => { it('returns a type structure', () => { expect( - getTypeDef('(u32, Compact, Vec, Option, (Text, Vec<(Bool, u128)>))') + getTypeDef('(u32, Compact, Vec, Option, DoubleMap, (Text, Vec<(Bool, u128)>))') ).toEqual({ info: TypeDefInfo.Tuple, - type: '(u32, Compact, Vec, Option, (Text, Vec<(Bool, u128)>))', + type: '(u32, Compact, Vec, Option, DoubleMap, (Text, Vec<(Bool, u128)>))', sub: [ { info: TypeDefInfo.Plain, @@ -96,6 +96,14 @@ describe('getTypeValue', () => { type: 'u128' } }, + { + info: TypeDefInfo.DoubleMap, + type: 'DoubleMap', + sub: { + info: TypeDefInfo.Plain, + type: 'u128' + } + }, { info: TypeDefInfo.Tuple, type: '(Text, Vec<(Bool, u128)>)', diff --git a/packages/types/src/codec/createType.ts b/packages/types/src/codec/createType.ts index e08999a0f2..b8987a8895 100644 --- a/packages/types/src/codec/createType.ts +++ b/packages/types/src/codec/createType.ts @@ -26,6 +26,7 @@ export enum TypeDefInfo { Tuple, Vector, Linkage, + DoubleMap, // anything not full supported (keep this as the last entry) Null } @@ -147,8 +148,8 @@ export function getTypeDef (_type: Text | string, name?: string): TypeDef { value.info = TypeDefInfo.Linkage; value.sub = getTypeDef(subType); } else if (startingWith(type, 'DoubleMap<', '>')) { - // FIXME As a first step, only allow parsing of DoubleMap, don't actually init - value.info = TypeDefInfo.Null; + value.info = TypeDefInfo.DoubleMap; + value.sub = getTypeDef(subType); } return value; @@ -213,6 +214,10 @@ export function getTypeClass (value: TypeDef): Constructor { return Linkage.withKey( getTypeClass(value.sub as TypeDef) ); + case TypeDefInfo.DoubleMap: + assert(value.sub && !Array.isArray(value.sub), 'Expected subtype for DoubleMap'); + + return getTypeClass(value.sub as TypeDef); case TypeDefInfo.Null: return Null; } diff --git a/packages/types/src/primitive/StorageHasher.ts b/packages/types/src/primitive/StorageHasher.ts index 308ebe7bdc..7c7a04c688 100644 --- a/packages/types/src/primitive/StorageHasher.ts +++ b/packages/types/src/primitive/StorageHasher.ts @@ -11,7 +11,7 @@ export default class StorageHasher extends Enum { 'Blake2_256', 'Twox128', 'Twox256', - 'Twox128Concat' + 'Twox64Concat' ], value); } @@ -44,9 +44,9 @@ export default class StorageHasher extends Enum { } /** - * @description Is the enum isTwox128Concat? + * @description Is the enum isTwox64Concat? */ - get isTwox128Concat (): boolean { + get isTwox64Concat (): boolean { return this.toNumber() === 4; } diff --git a/packages/types/src/primitive/StorageKey.spec.ts b/packages/types/src/primitive/StorageKey.spec.ts index 1c72da0ca5..7c659419f6 100644 --- a/packages/types/src/primitive/StorageKey.spec.ts +++ b/packages/types/src/primitive/StorageKey.spec.ts @@ -58,8 +58,8 @@ describe('StorageKey', () => { expect( new StorageKey( storage - .balances - .freeBalance('5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY') + .balances + .freeBalance('5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY') ) .toU8a() ).toEqual( @@ -69,5 +69,29 @@ describe('StorageKey', () => { ]) ); }); + + it(`should correctly get the EventTopics double map storage key (hex)`, () => { + expect( + new StorageKey([ + storage + .system + .eventTopics, + ['any', [1, 2, 3]] + ]) + .toHex() + ).toBe('0x3007c9ff7027f65900abcdfca4fdb107ead47e2a9e3558e01b691b0f4a5f8518d487326614f066416308bf6aa4e5041d1949928e4b26ede98e3cebb36a3b1726'); + }); + + it(`should correctly get the EventTopics double map storage key (u8a)`, () => { + expect( + new StorageKey([ + storage + .system + .eventTopics, + ['any', [1, 2, 3]] + ]) + .toU8a() + ).toEqual(Uint8Array.from([1, 1, 48, 7, 201, 255, 112, 39, 246, 89, 0, 171, 205, 252, 164, 253, 177, 7, 234, 212, 126, 42, 158, 53, 88, 224, 27, 105, 27, 15, 74, 95, 133, 24, 212, 135, 50, 102, 20, 240, 102, 65, 99, 8, 191, 106, 164, 229, 4, 29, 25, 73, 146, 142, 75, 38, 237, 233, 142, 60, 235, 179, 106, 59, 23, 38])); + }); }); }); diff --git a/packages/types/src/primitive/StorageKey.ts b/packages/types/src/primitive/StorageKey.ts index 06fddb7726..98ff4b4840 100644 --- a/packages/types/src/primitive/StorageKey.ts +++ b/packages/types/src/primitive/StorageKey.ts @@ -2,12 +2,12 @@ // This software may be modified and distributed under the terms // of the Apache-2.0 license. See the LICENSE file for details. -import { AnyU8a } from '../types'; - import { assert, isFunction, isString, isU8a } from '@polkadot/util'; -import Bytes from './Bytes'; +import U8a from '../codec/U8a'; import { StorageFunctionMetadata as MetaV4 } from '../Metadata/v4/Storage'; +import { AnyU8a } from '../types'; +import Bytes from './Bytes'; export interface StorageFunction { (arg?: any): Uint8Array; @@ -70,6 +70,14 @@ export default class StorageKey extends Bytes { assert(isFunction(fn), 'Expected function input for key construction'); + if (fn.meta && fn.meta.type.isDoubleMap) { + return { + key: new U8a(fn(...arg)), // skip compact length check in decodeBytes + method: fn.method, + section: fn.section + }; + } + return { key: fn(...arg), method: fn.method,