From 477be907d8711751f74a4de036a9dc92af0b7bfe Mon Sep 17 00:00:00 2001 From: Jaco Greeff Date: Wed, 21 Nov 2018 15:41:19 +0100 Subject: [PATCH] Compact support for decoding and decoding (#256) --- packages/util/src/compact/addLength.spec.ts | 13 +++++ packages/util/src/compact/addLength.ts | 28 +++++++++ packages/util/src/compact/defaults.ts | 7 +++ packages/util/src/compact/fromU8a.spec.ts | 39 +++++++++++++ packages/util/src/compact/fromU8a.ts | 41 +++++++++++++ packages/util/src/compact/index.ts | 18 ++++++ packages/util/src/compact/stripLength.spec.ts | 16 ++++++ packages/util/src/compact/stripLength.ts | 30 ++++++++++ packages/util/src/compact/toU8a.spec.ts | 57 +++++++++++++++++++ packages/util/src/compact/toU8a.ts | 46 +++++++++++++++ packages/util/src/compact/types.d.ts | 5 ++ packages/util/src/index.ts | 1 + packages/util/src/logger.ts | 7 ++- packages/util/src/types.d.ts | 1 + packages/util/src/u8a/toU8a.spec.js | 8 +++ 15 files changed, 316 insertions(+), 1 deletion(-) create mode 100644 packages/util/src/compact/addLength.spec.ts create mode 100644 packages/util/src/compact/addLength.ts create mode 100644 packages/util/src/compact/defaults.ts create mode 100644 packages/util/src/compact/fromU8a.spec.ts create mode 100644 packages/util/src/compact/fromU8a.ts create mode 100644 packages/util/src/compact/index.ts create mode 100644 packages/util/src/compact/stripLength.spec.ts create mode 100644 packages/util/src/compact/stripLength.ts create mode 100644 packages/util/src/compact/toU8a.spec.ts create mode 100644 packages/util/src/compact/toU8a.ts create mode 100644 packages/util/src/compact/types.d.ts diff --git a/packages/util/src/compact/addLength.spec.ts b/packages/util/src/compact/addLength.spec.ts new file mode 100644 index 000000000..7142e9b74 --- /dev/null +++ b/packages/util/src/compact/addLength.spec.ts @@ -0,0 +1,13 @@ +// Copyright 2017-2018 @polkadot/util authors & contributors +// This software may be modified and distributed under the terms +// of the Apache-2.0 license. See the LICENSE file for details. + +import compactAddLength from './addLength'; + +describe('compactAddLength', () => { + it('correctly adds the length prefix', () => { + expect( + compactAddLength(Uint8Array.from([12, 13])) + ).toEqual(Uint8Array.from([2 << 2, 12, 13])); + }); +}); diff --git a/packages/util/src/compact/addLength.ts b/packages/util/src/compact/addLength.ts new file mode 100644 index 000000000..d964ed634 --- /dev/null +++ b/packages/util/src/compact/addLength.ts @@ -0,0 +1,28 @@ +// Copyright 2017-2018 @polkadot/util authors & contributors +// This software may be modified and distributed under the terms +// of the Apache-2.0 license. See the LICENSE file for details. + +import { BitLength } from './types'; + +import { u8aConcat } from '../u8a'; +import { DEFAULT_BITLENGTH } from './defaults'; +import compactToU8a from './toU8a'; + +/** + * @name compactAddLength + * @description Adds a length prefix to the input value + * @example + *
+ * + * ```javascript + * import { compactAddLength } from '@polkadot/util'; + * + * console.log(compactAddLength(new Uint8Array([0xde, 0xad, 0xbe, 0xef]))); // Uint8Array([4 << 2, 0xde, 0xad, 0xbe, 0xef]) + * ``` + */ +export default function compactAddLength (input: Uint8Array, bitLength: BitLength = DEFAULT_BITLENGTH): Uint8Array { + return u8aConcat( + compactToU8a(input.length, bitLength), + input + ); +} diff --git a/packages/util/src/compact/defaults.ts b/packages/util/src/compact/defaults.ts new file mode 100644 index 000000000..7b79a103e --- /dev/null +++ b/packages/util/src/compact/defaults.ts @@ -0,0 +1,7 @@ +// Copyright 2017-2018 @polkadot/util authors & contributors +// This software may be modified and distributed under the terms +// of the Apache-2.0 license. See the LICENSE file for details. + +import { BitLength } from './types'; + +export const DEFAULT_BITLENGTH: BitLength = 32; diff --git a/packages/util/src/compact/fromU8a.spec.ts b/packages/util/src/compact/fromU8a.spec.ts new file mode 100644 index 000000000..a903427eb --- /dev/null +++ b/packages/util/src/compact/fromU8a.spec.ts @@ -0,0 +1,39 @@ +// Copyright 2017-2018 @polkadot/util authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +import BN from 'bn.js'; + +import compactFromU8a from './fromU8a'; + +describe('compactFromU8a', () => { + it('decoded u8 value', () => { + expect( + compactFromU8a(new Uint8Array([0b11111100])) + ).toEqual([1, new BN(63)]); + }); + + it('decodes from same u16 encoded value', () => { + expect( + compactFromU8a(new Uint8Array([0b11111101, 0b00000111]), 32) + ).toEqual([2, new BN(511)]); + }); + + it('decodes from same u32 encoded value (short)', () => { + expect( + compactFromU8a(new Uint8Array([254, 255, 3, 0]), 32) + ).toEqual([4, new BN(0xffff)]); + }); + + it('decodes from same u32 encoded value (full)', () => { + expect( + compactFromU8a(new Uint8Array([3, 249, 255, 255, 255]), 32) + ).toEqual([5, new BN(0xfffffff9)]); + }); + + it('decodes from same u32 as u64 encoded value (full, default)', () => { + expect( + compactFromU8a(new Uint8Array([3, 249, 255, 255, 255]), 64) + ).toEqual([9, new BN(0xfffffff9)]); + }); +}); diff --git a/packages/util/src/compact/fromU8a.ts b/packages/util/src/compact/fromU8a.ts new file mode 100644 index 000000000..6abbb7d96 --- /dev/null +++ b/packages/util/src/compact/fromU8a.ts @@ -0,0 +1,41 @@ +// Copyright 2017-2018 @polkadot/util authors & contributors +// This software may be modified and distributed under the terms +// of the Apache-2.0 license. See the LICENSE file for details. + +import { BitLength } from './types'; + +import BN from 'bn.js'; + +import { u8aToBn, u8aToU8a } from '../u8a'; +import { DEFAULT_BITLENGTH } from './defaults'; + +/** + * @name compactFromU8a + * @description Retrievs the offset and encoded length from a compact-prefixed value + * @example + *
+ * + * ```javascript + * import { compactFromU8a } from '@polkadot/util'; + * + * const [offset, length] = compactFromU8a(new Uint8Array([254, 255, 3, 0]), 32)); + * + * console.log('value offset=', offset, 'length=', length); // 4, 0xffff + * ``` + */ +export default function compactFromU8a (_input: Uint8Array | string, bitLength: BitLength = DEFAULT_BITLENGTH): [number, BN] { + const input = u8aToU8a(_input); + const flag = input[0] & 0b11; + + if (flag === 0b00) { + return [1, new BN(input[0]).shrn(2)]; + } else if (flag === 0b01) { + return [2, u8aToBn(input.slice(0, 2), true).shrn(2)]; + } else if (flag === 0b10) { + return [4, u8aToBn(input.slice(0, 4), true).shrn(2)]; + } + + const offset = 1 + (bitLength / 8); + + return [offset, u8aToBn(input.subarray(1, offset), true)]; +} diff --git a/packages/util/src/compact/index.ts b/packages/util/src/compact/index.ts new file mode 100644 index 000000000..9fe7cd22a --- /dev/null +++ b/packages/util/src/compact/index.ts @@ -0,0 +1,18 @@ +// Copyright 2017-2018 @polkadot/util authors & contributors +// This software may be modified and distributed under the terms +// of the Apache-2.0 license. See the LICENSE file for details. + +import compactAddLength from './addLength'; +import compactStripLength from './stripLength'; +import compactFromU8a from './fromU8a'; +import compactToU8a from './toU8a'; + +/** + * @summary Encoding and decoding of parity-codec compact numbers + */ +export { + compactAddLength, + compactFromU8a, + compactStripLength, + compactToU8a +}; diff --git a/packages/util/src/compact/stripLength.spec.ts b/packages/util/src/compact/stripLength.spec.ts new file mode 100644 index 000000000..43c70c077 --- /dev/null +++ b/packages/util/src/compact/stripLength.spec.ts @@ -0,0 +1,16 @@ +// Copyright 2017-2018 @polkadot/util authors & contributors +// This software may be modified and distributed under the terms +// of the Apache-2.0 license. See the LICENSE file for details. + +import compactStripLength from './stripLength'; + +describe('compactStripLength', () => { + it('correctly removes the length prefix', () => { + expect( + compactStripLength(Uint8Array.from([2 << 2, 12, 13])) + ).toEqual([ + 3, + Uint8Array.from([12, 13]) + ]); + }); +}); diff --git a/packages/util/src/compact/stripLength.ts b/packages/util/src/compact/stripLength.ts new file mode 100644 index 000000000..35c64d123 --- /dev/null +++ b/packages/util/src/compact/stripLength.ts @@ -0,0 +1,30 @@ +// Copyright 2017-2018 @polkadot/util authors & contributors +// This software may be modified and distributed under the terms +// of the Apache-2.0 license. See the LICENSE file for details. + +import { BitLength } from './types'; + +import { DEFAULT_BITLENGTH } from './defaults'; +import compactFromU8a from './fromU8a'; + +/** + * @name compactStripLength + * @description Removes the length prefix, returning both the total length (including the value + compact encoding) and the decoded value with the correct length + * @example + *
+ * + * ```javascript + * import { compactStripLength } from '@polkadot/util'; + * + * console.log(compactStripLength(new Uint8Array([2 << 2, 0xde, 0xad]))); // [2, Uint8Array[0xde, 0xad]] + * ``` + */ +export default function compactStripLength (input: Uint8Array, bitLength: BitLength = DEFAULT_BITLENGTH): [number, Uint8Array] { + const [offset, length] = compactFromU8a(input, bitLength); + const total = offset + length.toNumber(); + + return [ + total, + input.subarray(offset, total) + ]; +} diff --git a/packages/util/src/compact/toU8a.spec.ts b/packages/util/src/compact/toU8a.spec.ts new file mode 100644 index 000000000..bdb95ff84 --- /dev/null +++ b/packages/util/src/compact/toU8a.spec.ts @@ -0,0 +1,57 @@ +// Copyright 2017-2018 @polkadot/util authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +import BN from 'bn.js'; + +import compactToU8a from './toU8a'; + +describe('encode', () => { + it('encodes short u8', () => { + expect( + compactToU8a(18, 8) + ).toEqual( + new Uint8Array([18 << 2]) + ); + }); + + it('encodes max u8 values', () => { + expect( + compactToU8a(new BN(63), 16) + ).toEqual( + new Uint8Array([0b11111100]) + ); + }); + + it('encodes basic u16 value', () => { + expect( + compactToU8a(511, 32) + ).toEqual( + new Uint8Array([0b11111101, 0b00000111]) + ); + }); + + it('encodes basic u16 (not at edge)', () => { + expect( + compactToU8a(111, 32) + ).toEqual( + new Uint8Array([0xbd, 0x01]) + ); + }); + + it('encodes basic u32 values (short)', () => { + expect( + compactToU8a(0xffff, 32) + ).toEqual( + new Uint8Array([254, 255, 3, 0]) + ); + }); + + it('encodes basic u32 values (full)', () => { + expect( + compactToU8a(0xfffffff9) + ).toEqual( + new Uint8Array([3, 249, 255, 255, 255]) + ); + }); +}); diff --git a/packages/util/src/compact/toU8a.ts b/packages/util/src/compact/toU8a.ts new file mode 100644 index 000000000..a6bc4ccfc --- /dev/null +++ b/packages/util/src/compact/toU8a.ts @@ -0,0 +1,46 @@ +// Copyright 2017-2018 @polkadot/util authors & contributors +// This software may be modified and distributed under the terms +// of the Apache-2.0 license. See the LICENSE file for details. + +import { BitLength } from './types'; + +import BN from 'bn.js'; + +import { bnToBn, bnToU8a } from '../bn'; +import { u8aConcat } from '../u8a'; +import { DEFAULT_BITLENGTH } from './defaults'; + +const MAX_U8 = new BN(2).pow(new BN(8 - 2)).subn(1); +const MAX_U16 = new BN(2).pow(new BN(16 - 2)).subn(1); +const MAX_U32 = new BN(2).pow(new BN(32 - 2)).subn(1); + +/** + * @name compactToU8a + * @description Encodes a number into a compact representation + * @example + *
+ * + * ```javascript + * import { compactToU8a } from '@polkadot/util'; + * + * console.log(compactToU8a(511, 32)); // Uint8Array([0b11111101, 0b00000111]) + * ``` + */ +export default function compactToU8a (_value: BN | number, bitLength: BitLength = DEFAULT_BITLENGTH): Uint8Array { + const value = bnToBn(_value); + + if (value.lte(MAX_U8)) { + return new Uint8Array([value.toNumber() << 2]); + } else if (value.lte(MAX_U16)) { + return bnToU8a(value.shln(2).addn(0b01), 16, true); + } else if (value.lte(MAX_U32)) { + return bnToU8a(value.shln(2).addn(0b10), 32, true); + } + + return u8aConcat( + new Uint8Array([ + 0b11 + ]), + bnToU8a(value, bitLength, true) + ); +} diff --git a/packages/util/src/compact/types.d.ts b/packages/util/src/compact/types.d.ts new file mode 100644 index 000000000..bebf1372a --- /dev/null +++ b/packages/util/src/compact/types.d.ts @@ -0,0 +1,5 @@ +// Copyright 2017-2018 @polkadot/util authors & contributors +// This software may be modified and distributed under the terms +// of the Apache-2.0 license. See the LICENSE file for details. + +export type BitLength = 8 | 16 | 32 | 64 | 128 | 256; diff --git a/packages/util/src/index.ts b/packages/util/src/index.ts index 55b9f5aa2..ce58d2d8f 100644 --- a/packages/util/src/index.ts +++ b/packages/util/src/index.ts @@ -9,6 +9,7 @@ import promisify from './promisify'; export * from './array'; export * from './bn'; export * from './buffer'; +export * from './compact'; export * from './ext'; export * from './hex'; export * from './is'; diff --git a/packages/util/src/logger.ts b/packages/util/src/logger.ts index fa454415e..3ea4c6416 100644 --- a/packages/util/src/logger.ts +++ b/packages/util/src/logger.ts @@ -87,6 +87,10 @@ function apply (log: LogType, type: string, values: Logger$Data): void { ); } +function noop (...values: Logger$Data): void { + // noop +} + /** * @name Logger * @signature logger (type: string): Logger @@ -118,9 +122,10 @@ export default function logger (_type: string): Logger { return { debug: isDebug ? (...values: Logger$Data): void => apply('debug', type, values) - : (...values: Logger$Data): void => void 0, + : noop, error: (...values: Logger$Data): void => apply('error', type, values), log: (...values: Logger$Data): void => apply('log', type, values), + noop, warn: (...values: Logger$Data): void => apply('warn', type, values) }; } diff --git a/packages/util/src/types.d.ts b/packages/util/src/types.d.ts index e13f0348c..6ec2a0b97 100644 --- a/packages/util/src/types.d.ts +++ b/packages/util/src/types.d.ts @@ -16,5 +16,6 @@ export type Logger = { debug: (...values: Logger$Data) => void, error: (...values: Logger$Data) => void, log: (...values: Logger$Data) => void, + noop: (...values: Logger$Data) => void, warn: (...values: Logger$Data) => void }; diff --git a/packages/util/src/u8a/toU8a.spec.js b/packages/util/src/u8a/toU8a.spec.js index beadf1099..07411543c 100644 --- a/packages/util/src/u8a/toU8a.spec.js +++ b/packages/util/src/u8a/toU8a.spec.js @@ -33,6 +33,14 @@ describe('u8aToU8a', () => { ); }); + it('creates from Array', () => { + expect( + u8aToU8a([128, 0, 10, 11, 12]) + ).toEqual( + new Uint8Array([128, 0, 10, 11, 12]) + ); + }); + it('returns a Uint8Array as-is (u8a input)', () => { expect( u8aToU8a(new Uint8Array([128, 0, 10]))