diff --git a/packages/rpc-core/src/index.ts b/packages/rpc-core/src/index.ts index 3cef718023..281b212058 100644 --- a/packages/rpc-core/src/index.ts +++ b/packages/rpc-core/src/index.ts @@ -4,7 +4,6 @@ import { ProviderInterface } from '@polkadot/rpc-provider/types'; import { RpcMethod } from '@polkadot/jsonrpc/types'; -import { StorageChangeSet } from '@polkadot/types/interfaces'; import { AnyJson, Codec } from '@polkadot/types/types'; import { RpcInterface } from './jsonrpc.types'; import { RpcInterfaceMethod } from './types'; @@ -13,9 +12,9 @@ import memoizee from 'memoizee'; import { combineLatest, from, Observable, Observer, of, throwError } from 'rxjs'; import { catchError, map, publishReplay, refCount, switchMap } from 'rxjs/operators'; import interfaces from '@polkadot/jsonrpc'; -import { ClassOf, Option, StorageData, StorageKey, Vec, createClass } from '@polkadot/types'; +import { Option, StorageKey, Vec, createClass } from '@polkadot/types'; import { createTypeUnsafe } from '@polkadot/types/codec'; -import { assert, isFunction, isNull, isNumber, logger } from '@polkadot/util'; +import { assert, isFunction, isNull, isNumber, logger, u8aToU8a } from '@polkadot/util'; const l = logger('rpc-core'); @@ -52,7 +51,7 @@ const EMPTY_META = { * ``` */ export default class Rpc implements RpcInterface { - private _storageCache = new Map>(); + private _storageCache = new Map(); public readonly provider: ProviderInterface; @@ -282,31 +281,36 @@ export default class Rpc implements RpcInterface { ); } - private formatOutput (method: RpcMethod, params: Codec[], result?: any): Codec | (Codec | null | undefined)[] { - const base = createTypeUnsafe(method.type, [result]); + private treatAsHex (key: StorageKey): boolean { + // :code is problematic - it does not have the length attached, which is + // unlike all other storage entries where it is indeed properly encoded + return ['0x3a636f6465'].includes(key.toHex()); + } + private formatOutput (method: RpcMethod, params: Codec[], result?: any): Codec | Codec[] { if (method.type === 'StorageData') { const key = params[0] as StorageKey; try { - return this.formatStorageData(key, base, isNull(result)); + return this.formatStorageData(key, result); } catch (error) { console.error(`Unable to decode storage ${key.section}.${key.method}:`, error.message); throw error; } - } else if ((method.type as string) === 'StorageChangeSet') { + } else if (method.type === 'StorageChangeSet') { + // For StorageChangeSet, the changes has the [key, value] mappings + const changes: [string, string | null][] = result.changes; const keys = params[0] as Vec; const withCache = keys.length !== 1; // multiple return values (via state.storage subscription), decode the values // one at a time, all based on the query types. Three values can be returned - - // - Base - There is a valid value, non-empty - // - null - The storage key is empty (but in the resultset) - // - undefined - The storage value is not in the resultset - return keys.reduce((results, key: StorageKey): (Codec | undefined)[] => { + // - Codec - There is a valid value, non-empty + // - null - The storage key is empty + return keys.reduce((results, key: StorageKey): Codec[] => { try { - results.push(this.formatStorageSet(key, base as StorageChangeSet, withCache)); + results.push(this.formatStorageSet(key, changes, withCache)); } catch (error) { console.error(`Unable to decode storage ${key.section}.${key.method}:`, error.message); @@ -314,66 +318,77 @@ export default class Rpc implements RpcInterface { } return results; - }, [] as (Codec | undefined)[]); + }, [] as Codec[]); } - return base; + return createTypeUnsafe(method.type, [result]); } - private formatStorageData (key: StorageKey, base: Codec, isNull: boolean): Codec { + private formatStorageData (key: StorageKey, value: string | null): Codec { // single return value (via state.getStorage), decode the value based on the // outputType that we have specified. Fallback to Data on nothing const type = key.outputType || 'Data'; const meta = key.meta || EMPTY_META; + const isEmpty = isNull(value); + + // we convert to Uint8Array since it maps to the raw encoding, all + // data will be correctly encoded (incl. numbers, excl. :code) + const input = isEmpty + ? null + : this.treatAsHex(key) + ? value + : u8aToU8a(value); if (meta.modifier.isOptional) { return new Option( createClass(type), - isNull + isEmpty ? null - : createTypeUnsafe(type, [base], true) + : createTypeUnsafe(type, [input], true) ); } - return createTypeUnsafe(type, [isNull ? meta.fallback : base], true); + return createTypeUnsafe(type, [isEmpty ? meta.fallback : input], true); } - private formatStorageSet (key: StorageKey, base: StorageChangeSet, witCache: boolean): Codec { + private formatStorageSet (key: StorageKey, changes: [string, string | null][], witCache: boolean): Codec { // Fallback to Data (i.e. just the encoding) if we don't have a specific type const type = key.outputType || 'Data'; const hexKey = key.toHex(); const meta = key.meta || EMPTY_META; - // if we don't find the value, this is out fallback + // if we don't find the value, this is our fallback // - in the case of an array of values, fill the hole from the cache // - if a single result value, don't fill - it is not an update hole // - fallback to an empty option in all cases - const emptyVal = ( - witCache - ? this._storageCache.get(hexKey) - : null - ) || new Option(ClassOf('StorageData'), null); + const emptyVal = (witCache && this._storageCache.get(hexKey)) || null; // see if we have a result value for this specific key, fallback to the cache value // when the value in the set is not available, or is null/empty. - const [, value] = base.changes.find(([key, value]): boolean => - value.isSome && key.toHex() === hexKey + const [, value] = changes.find(([key, value]): boolean => + !isNull(value) && key === hexKey ) || [null, emptyVal]; + const isEmpty = isNull(value); + const input = isEmpty + ? null + : this.treatAsHex(key) + ? value + : u8aToU8a(value); // store the retrieved result - the only issue with this cache is that there is no - // clearning of it, so very long running processes (not just a couple of hours, longer) + // clearing of it, so very long running processes (not just a couple of hours, longer) // will increase memory beyond what is allowed. this._storageCache.set(hexKey, value); if (meta.modifier.isOptional) { return new Option( createClass(type), - value.isNone + isEmpty ? null - : createTypeUnsafe(type, [value.unwrap()], true) + : createTypeUnsafe(type, [input], true) ); } - return createTypeUnsafe(type, [value.unwrapOr(meta.fallback)], true); + return createTypeUnsafe(type, [isEmpty ? meta.fallback : input], true); } } diff --git a/packages/types/src/codec/Linkage.ts b/packages/types/src/codec/Linkage.ts index 4ef4f35547..1da6e48a67 100644 --- a/packages/types/src/codec/Linkage.ts +++ b/packages/types/src/codec/Linkage.ts @@ -51,8 +51,10 @@ export default class Linkage extends Struct { /** * @description Custom toU8a which with bare mode does not return the linkage if empty */ - public toU8a (isBare?: boolean): Uint8Array { - return isBare && this.isEmpty + public toU8a (): Uint8Array { + // As part of a storage query (where these appear), in the case of empty, the values + // are NOT populated by the node - follow the same logic, leaving it empty + return this.isEmpty ? EMPTY : super.toU8a(); } diff --git a/packages/types/src/codec/create/createType.ts b/packages/types/src/codec/create/createType.ts index 3a180dcf35..f02285c604 100644 --- a/packages/types/src/codec/create/createType.ts +++ b/packages/types/src/codec/create/createType.ts @@ -5,23 +5,24 @@ import { Codec, Constructor, InterfaceTypes } from '../../types'; import { FromReg } from './types'; +import { isU8a, u8aToHex } from '@polkadot/util'; + import { InterfaceRegistry } from '../../interfaceRegistry'; import { createClass } from './createClass'; +function u8aHasValue (value: Uint8Array): boolean { + return value.some((v): boolean => !!v); +} + // With isPedantic, actually check that the encoding matches that supplied. This // is much slower, but verifies that we have the correct types defined -function checkInstance (value: any, created: FromReg): void { - const inHex = value.toHex(true); +function checkInstance (value: Uint8Array, created: FromReg): void { + // For option, we are not adding the initial byte (this is via storage) const crHex = created.toHex(true); - const hasMatch = (inHex === crHex) || (value.toU8a(true).toString() === ( - created instanceof Uint8Array - // strip the input length - ? created.toU8a().toString() - // compare raw. without additions - : created.toU8a(true).toString() - )); + const inHex = u8aToHex(value); - if (!hasMatch) { + // if the hex doesn't match and the value for both is non-empty, complain... bitterly + if (inHex !== crHex && (u8aHasValue(value) || u8aHasValue(created.toU8a(true)))) { console.warn(`${created.toRawType()}:: Input doesn't match output, received ${inHex}, created ${crHex}`); } } @@ -32,7 +33,7 @@ function initType (Type: Con const created = new Type(...params); const [value] = params; - if (isPedantic && value && value.toU8a && !value.isEmpty) { + if (isPedantic && isU8a(value)) { checkInstance(value, created); } diff --git a/packages/types/src/interfaces/rpc/StorageChangeSet.spec.ts b/packages/types/src/interfaces/rpc/StorageChangeSet.spec.ts index 73f9cf54e0..6905f0764d 100644 --- a/packages/types/src/interfaces/rpc/StorageChangeSet.spec.ts +++ b/packages/types/src/interfaces/rpc/StorageChangeSet.spec.ts @@ -43,11 +43,5 @@ describe('StorageChangeSet', (): void => { expect(set.changes).toHaveLength(1); expect(set.changes[0][0].toHex()).toEqual('0x54bdbdb5e438d574dd4da05ee6131cee'); }); - - it('converts result data to Bytes', (): void => { - const bytes = createType('Bytes', set.changes[0][1].unwrap()); - - expect(bytes).toHaveLength(4452); - }); }); }); diff --git a/packages/types/src/primitive/Bytes.spec.ts b/packages/types/src/primitive/Bytes.spec.ts index 05e4b45b5d..21b66fd343 100644 --- a/packages/types/src/primitive/Bytes.spec.ts +++ b/packages/types/src/primitive/Bytes.spec.ts @@ -36,13 +36,5 @@ describe('Bytes', (): void => { ).toU8a() ).toEqual(CODE); }); - - it('creates via storagedata (with length prefix)', (): void => { - expect( - new Bytes( - createType('StorageData', '0x143a636f6465') - ).toU8a() - ).toEqual(CODE); - }); }); }); diff --git a/packages/types/src/primitive/Bytes.ts b/packages/types/src/primitive/Bytes.ts index 5fc0bc229c..7655d3feb0 100644 --- a/packages/types/src/primitive/Bytes.ts +++ b/packages/types/src/primitive/Bytes.ts @@ -6,7 +6,6 @@ import { AnyU8a } from '../types'; import { assert, isString, isU8a, u8aToU8a } from '@polkadot/util'; -import { ClassOf } from '../codec/create'; import Compact from '../codec/Compact'; import U8a from '../codec/U8a'; @@ -24,11 +23,7 @@ export default class Bytes extends U8a { private static decodeBytes (value?: AnyU8a): Uint8Array | undefined { if (Array.isArray(value) || isString(value)) { - return Bytes.decodeBytesU8a( - Compact.addLengthPrefix(u8aToU8a(value)) - ); - } else if (value instanceof ClassOf('StorageData')) { - return Bytes.decodeBytesStorage(value); + return u8aToU8a(value); } else if (!(value instanceof U8a) && isU8a(value)) { // We are ensuring we are not a U8a instance. In the case of a U8a we already have gotten // rid of the length, i.e. new Bytes(new Bytes(...)) will work as expected @@ -38,16 +33,6 @@ export default class Bytes extends U8a { return value; } - private static decodeBytesStorage (u8a: Uint8Array): Uint8Array { - // Here we cater for the actual StorageData that _could_ have a length prefix. In the - // case of `:code` it is not added, for others it is - const [offset, length] = Compact.decodeU8a(u8a); - - return u8a.length === length.addn(offset).toNumber() - ? u8a.subarray(offset) - : u8a; - } - private static decodeBytesU8a (value: Uint8Array): Uint8Array { if (!value.length) { return new Uint8Array();