quantus/api
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Don't decode and re-encode storage queries, single pass (#1433)

Browse Source 
* Don't decode and re-encode storage queries, single pass

* Remove old SorageData <-> Bytes HACK tests

* Add non-empty check to isPendantic
This commit is contained in:
Jaco Greeff
2019-10-02 12:38:16 +02:00
committed by GitHub
parent 275593b6fb
commit d0f9114fda
6 changed files with 64 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/rpc-core/src/index.ts
+47 -32
View File
@@ -4,7 +4,6 @@
import { ProviderInterface } from '@polkadot/rpc-provider/types';
import { RpcMethod } from '@polkadot/jsonrpc/types';
import { StorageChangeSet } from '@polkadot/types/interfaces';
import { AnyJson, Codec } from '@polkadot/types/types';
import { RpcInterface } from './jsonrpc.types';
import { RpcInterfaceMethod } from './types';
@@ -13,9 +12,9 @@ import memoizee from 'memoizee';
import { combineLatest, from, Observable, Observer, of, throwError } from 'rxjs';
import { catchError, map, publishReplay, refCount, switchMap } from 'rxjs/operators';
import interfaces from '@polkadot/jsonrpc';
import { ClassOf, Option, StorageData, StorageKey, Vec, createClass } from '@polkadot/types';
import { Option, StorageKey, Vec, createClass } from '@polkadot/types';
import { createTypeUnsafe } from '@polkadot/types/codec';
import { assert, isFunction, isNull, isNumber, logger } from '@polkadot/util';
import { assert, isFunction, isNull, isNumber, logger, u8aToU8a } from '@polkadot/util';
const l = logger('rpc-core');
@@ -52,7 +51,7 @@ const EMPTY_META = {
* ```
*/
export default class Rpc implements RpcInterface {
private _storageCache = new Map<string, Option<StorageData>>();
private _storageCache = new Map<string, string | null>();
public readonly provider: ProviderInterface;
@@ -282,31 +281,36 @@ export default class Rpc implements RpcInterface {
);
}
private formatOutput (method: RpcMethod, params: Codec[], result?: any): Codec | (Codec | null | undefined)[] {
const base = createTypeUnsafe(method.type, [result]);
private treatAsHex (key: StorageKey): boolean {
// :code is problematic - it does not have the length attached, which is
// unlike all other storage entries where it is indeed properly encoded
return ['0x3a636f6465'].includes(key.toHex());
}
private formatOutput (method: RpcMethod, params: Codec[], result?: any): Codec | Codec[] {
if (method.type === 'StorageData') {
const key = params[0] as StorageKey;
try {
return this.formatStorageData(key, base, isNull(result));
return this.formatStorageData(key, result);
} catch (error) {
console.error(`Unable to decode storage ${key.section}.${key.method}:`, error.message);
throw error;
}
} else if ((method.type as string) === 'StorageChangeSet') {
} else if (method.type === 'StorageChangeSet') {
// For StorageChangeSet, the changes has the [key, value] mappings
const changes: [string, string | null][] = result.changes;
const keys = params[0] as Vec<StorageKey>;
const withCache = keys.length !== 1;
// multiple return values (via state.storage subscription), decode the values
// one at a time, all based on the query types. Three values can be returned -
// - Base - There is a valid value, non-empty
// - null - The storage key is empty (but in the resultset)
// - undefined - The storage value is not in the resultset
return keys.reduce((results, key: StorageKey): (Codec | undefined)[] => {
// - Codec - There is a valid value, non-empty
// - null - The storage key is empty
return keys.reduce((results, key: StorageKey): Codec[] => {
try {
results.push(this.formatStorageSet(key, base as StorageChangeSet, withCache));
results.push(this.formatStorageSet(key, changes, withCache));
} catch (error) {
console.error(`Unable to decode storage ${key.section}.${key.method}:`, error.message);
@@ -314,66 +318,77 @@ export default class Rpc implements RpcInterface {
}
return results;
}, [] as (Codec | undefined)[]);
}, [] as Codec[]);
}
return base;
return createTypeUnsafe(method.type, [result]);
}
private formatStorageData (key: StorageKey, base: Codec, isNull: boolean): Codec {
private formatStorageData (key: StorageKey, value: string | null): Codec {
// single return value (via state.getStorage), decode the value based on the
// outputType that we have specified. Fallback to Data on nothing
const type = key.outputType || 'Data';
const meta = key.meta || EMPTY_META;
const isEmpty = isNull(value);
// we convert to Uint8Array since it maps to the raw encoding, all
// data will be correctly encoded (incl. numbers, excl. :code)
const input = isEmpty
? null
: this.treatAsHex(key)
? value
: u8aToU8a(value);
if (meta.modifier.isOptional) {
return new Option(
createClass(type),
isNull
isEmpty
? null
: createTypeUnsafe(type, [base], true)
: createTypeUnsafe(type, [input], true)
);
}
return createTypeUnsafe(type, [isNull ? meta.fallback : base], true);
return createTypeUnsafe(type, [isEmpty ? meta.fallback : input], true);
}
private formatStorageSet (key: StorageKey, base: StorageChangeSet, witCache: boolean): Codec {
private formatStorageSet (key: StorageKey, changes: [string, string | null][], witCache: boolean): Codec {
// Fallback to Data (i.e. just the encoding) if we don't have a specific type
const type = key.outputType || 'Data';
const hexKey = key.toHex();
const meta = key.meta || EMPTY_META;
// if we don't find the value, this is out fallback
// if we don't find the value, this is our fallback
// - in the case of an array of values, fill the hole from the cache
// - if a single result value, don't fill - it is not an update hole
// - fallback to an empty option in all cases
const emptyVal = (
witCache
? this._storageCache.get(hexKey)
: null
) || new Option<StorageData>(ClassOf('StorageData'), null);
const emptyVal = (witCache && this._storageCache.get(hexKey)) || null;
// see if we have a result value for this specific key, fallback to the cache value
// when the value in the set is not available, or is null/empty.
const [, value] = base.changes.find(([key, value]): boolean =>
value.isSome && key.toHex() === hexKey
const [, value] = changes.find(([key, value]): boolean =>
!isNull(value) && key === hexKey
) || [null, emptyVal];
const isEmpty = isNull(value);
const input = isEmpty
? null
: this.treatAsHex(key)
? value
: u8aToU8a(value);
// store the retrieved result - the only issue with this cache is that there is no
// clearning of it, so very long running processes (not just a couple of hours, longer)
// clearing of it, so very long running processes (not just a couple of hours, longer)
// will increase memory beyond what is allowed.
this._storageCache.set(hexKey, value);
if (meta.modifier.isOptional) {
return new Option(
createClass(type),
value.isNone
isEmpty
? null
: createTypeUnsafe(type, [value.unwrap()], true)
: createTypeUnsafe(type, [input], true)
);
}
return createTypeUnsafe(type, [value.unwrapOr(meta.fallback)], true);
return createTypeUnsafe(type, [isEmpty ? meta.fallback : input], true);
}
}
packages/types/src/codec/Linkage.ts
+4 -2
View File
@@ -51,8 +51,10 @@ export default class Linkage<T extends Codec> extends Struct {
/**
* @description Custom toU8a which with bare mode does not return the linkage if empty
*/
public toU8a (isBare?: boolean): Uint8Array {
return isBare && this.isEmpty
public toU8a (): Uint8Array {
// As part of a storage query (where these appear), in the case of empty, the values
// are NOT populated by the node - follow the same logic, leaving it empty
return this.isEmpty
? EMPTY
: super.toU8a();
}
packages/types/src/codec/create/createType.ts
+12 -11
View File
@@ -5,23 +5,24 @@
import { Codec, Constructor, InterfaceTypes } from '../../types';
import { FromReg } from './types';
import { isU8a, u8aToHex } from '@polkadot/util';
import { InterfaceRegistry } from '../../interfaceRegistry';
import { createClass } from './createClass';
function u8aHasValue (value: Uint8Array): boolean {
return value.some((v): boolean => !!v);
}
// With isPedantic, actually check that the encoding matches that supplied. This
// is much slower, but verifies that we have the correct types defined
function checkInstance<T extends Codec = Codec, K extends string = string> (value: any, created: FromReg<T, K>): void {
const inHex = value.toHex(true);
function checkInstance<T extends Codec = Codec, K extends string = string> (value: Uint8Array, created: FromReg<T, K>): void {
// For option, we are not adding the initial byte (this is via storage)
const crHex = created.toHex(true);
const hasMatch = (inHex === crHex) || (value.toU8a(true).toString() === (
created instanceof Uint8Array
// strip the input length
? created.toU8a().toString()
// compare raw. without additions
: created.toU8a(true).toString()
));
const inHex = u8aToHex(value);
if (!hasMatch) {
// if the hex doesn't match and the value for both is non-empty, complain... bitterly
if (inHex !== crHex && (u8aHasValue(value) || u8aHasValue(created.toU8a(true)))) {
console.warn(`${created.toRawType()}:: Input doesn't match output, received ${inHex}, created ${crHex}`);
}
}
@@ -32,7 +33,7 @@ function initType<T extends Codec = Codec, K extends string = string> (Type: Con
const created = new Type(...params);
const [value] = params;
if (isPedantic && value && value.toU8a && !value.isEmpty) {
if (isPedantic && isU8a(value)) {
checkInstance(value, created);
}
packages/types/src/interfaces/rpc/StorageChangeSet.spec.ts
-6
View File
@@ -43,11 +43,5 @@ describe('StorageChangeSet', (): void => {
expect(set.changes).toHaveLength(1);
expect(set.changes[0][0].toHex()).toEqual('0x54bdbdb5e438d574dd4da05ee6131cee');
});
it('converts result data to Bytes', (): void => {
const bytes = createType('Bytes', set.changes[0][1].unwrap());
expect(bytes).toHaveLength(4452);
});
});
});
packages/types/src/primitive/Bytes.spec.ts
-8
View File
@@ -36,13 +36,5 @@ describe('Bytes', (): void => {
).toU8a()
).toEqual(CODE);
});
it('creates via storagedata (with length prefix)', (): void => {
expect(
new Bytes(
createType('StorageData', '0x143a636f6465')
).toU8a()
).toEqual(CODE);
});
});
});
packages/types/src/primitive/Bytes.ts
+1 -16
View File
@@ -6,7 +6,6 @@ import { AnyU8a } from '../types';
import { assert, isString, isU8a, u8aToU8a } from '@polkadot/util';
import { ClassOf } from '../codec/create';
import Compact from '../codec/Compact';
import U8a from '../codec/U8a';
@@ -24,11 +23,7 @@ export default class Bytes extends U8a {
private static decodeBytes (value?: AnyU8a): Uint8Array | undefined {
if (Array.isArray(value) || isString(value)) {
return Bytes.decodeBytesU8a(
Compact.addLengthPrefix(u8aToU8a(value))
);
} else if (value instanceof ClassOf('StorageData')) {
return Bytes.decodeBytesStorage(value);
return u8aToU8a(value);
} else if (!(value instanceof U8a) && isU8a(value)) {
// We are ensuring we are not a U8a instance. In the case of a U8a we already have gotten
// rid of the length, i.e. new Bytes(new Bytes(...)) will work as expected
@@ -38,16 +33,6 @@ export default class Bytes extends U8a {
return value;
}
private static decodeBytesStorage (u8a: Uint8Array): Uint8Array {
// Here we cater for the actual StorageData that _could_ have a length prefix. In the
// case of `:code` it is not added, for others it is
const [offset, length] = Compact.decodeU8a(u8a);
return u8a.length === length.addn(offset).toNumber()
? u8a.subarray(offset)
: u8a;
}
private static decodeBytesU8a (value: Uint8Array): Uint8Array {
if (!value.length) {
return new Uint8Array();