diff --git a/CHANGELOG.md b/CHANGELOG.md index d1d807035..4170b196c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ Changes: - Add strongly typed `object{Entries, Keys, Values}` utilities - Expand tests for uncovered functions - Sync with upstream Substrate ss58 registry +- Optimizations for hex <-> u8a functions +- Allow `u8aEq` to use `Uint{8, 16, 32}Array` with correct alignment ## 7.8.2 Nov 7, 2021 diff --git a/packages/util/src/compact/fromU8a.ts b/packages/util/src/compact/fromU8a.ts index 460530845..ad1480ecb 100644 --- a/packages/util/src/compact/fromU8a.ts +++ b/packages/util/src/compact/fromU8a.ts @@ -27,9 +27,9 @@ export function compactFromU8a (_input: U8aLike): [number, BN] { if (flag === 0b00) { return [1, new BN(input[0]).ishrn(2)]; } else if (flag === 0b01) { - return [2, u8aToBn(input.slice(0, 2), true).ishrn(2)]; + return [2, u8aToBn(input.subarray(0, 2), true).ishrn(2)]; } else if (flag === 0b10) { - return [4, u8aToBn(input.slice(0, 4), true).ishrn(2)]; + return [4, u8aToBn(input.subarray(0, 4), true).ishrn(2)]; } const offset = 1 + ( diff --git a/packages/util/src/hex/alphabet.ts b/packages/util/src/hex/alphabet.ts new file mode 100644 index 000000000..7fa9bb13e --- /dev/null +++ b/packages/util/src/hex/alphabet.ts @@ -0,0 +1,27 @@ +// Copyright 2017-2021 @polkadot/util authors & contributors +// SPDX-License-Identifier: Apache-2.0 + +const U8_TO_HEX = new Array(256); +const U16_TO_HEX = new Array(256 * 256); + +const HEX_TO_U8: Record = {}; +const HEX_TO_U16: Record = {}; + +for (let n = 0; n < 256; n++) { + const hex = n.toString(16).padStart(2, '0'); + + U8_TO_HEX[n] = hex; + HEX_TO_U8[hex] = n; +} + +for (let i = 0; i < 256; i++) { + for (let j = 0; j < 256; j++) { + const hex = U8_TO_HEX[i] + U8_TO_HEX[j]; + const n = (j * 256) + i; + + U16_TO_HEX[n] = hex; + HEX_TO_U16[hex] = n; + } +} + +export { HEX_TO_U16, HEX_TO_U8, U16_TO_HEX, U8_TO_HEX }; diff --git a/packages/util/src/hex/toU8a.spec.ts b/packages/util/src/hex/toU8a.spec.ts index 7f0ad515a..d7eedf565 100644 --- a/packages/util/src/hex/toU8a.spec.ts +++ b/packages/util/src/hex/toU8a.spec.ts @@ -1,6 +1,7 @@ // Copyright 2017-2021 @polkadot/util authors & contributors // SPDX-License-Identifier: Apache-2.0 +import { U16_TO_HEX } from './alphabet'; import { hexToU8a } from '.'; describe('hexToU8a', (): void => { @@ -20,7 +21,7 @@ describe('hexToU8a', (): void => { it('returns a Uint8Array with the correct values (bitLength > provided)', (): void => { expect( - hexToU8a('0x80000a', 32) + hexToU8a('0x80000A', 32) ).toEqual( new Uint8Array([0, 128, 0, 10]) ); @@ -39,4 +40,20 @@ describe('hexToU8a', (): void => { (): Uint8Array => hexToU8a('notahex') ).toThrow(/hex value to convert/); }); + + it.skip('performance', (): void => { + let a = '0x'; + + for (let i = 0; i < 8192; i++) { + a += U16_TO_HEX[i]; + } + + console.time('hexToU8a:performance'); + + for (let i = 0; i < 8192; i++) { + hexToU8a(a); + } + + console.timeEnd('hexToU8a:performance'); + }); }); diff --git a/packages/util/src/hex/toU8a.ts b/packages/util/src/hex/toU8a.ts index e6e083c33..42eca78b9 100644 --- a/packages/util/src/hex/toU8a.ts +++ b/packages/util/src/hex/toU8a.ts @@ -5,6 +5,7 @@ import type { HexString } from '../types'; import { assert } from '../assert'; import { isHex } from '../is/hex'; +import { HEX_TO_U8 } from './alphabet'; import { hexStripPrefix } from './stripPrefix'; /** @@ -29,7 +30,7 @@ export function hexToU8a (_value?: HexString | string | null, bitLength = -1): U assert(isHex(_value), () => `Expected hex value to convert, found '${_value}'`); - const value = hexStripPrefix(_value); + const value = hexStripPrefix(_value).toLowerCase(); const valLength = value.length / 2; const resultLength = Math.ceil( bitLength === -1 @@ -40,7 +41,9 @@ export function hexToU8a (_value?: HexString | string | null, bitLength = -1): U const offset = Math.max(0, resultLength - valLength); for (let index = 0; index < resultLength; index++) { - result[index + offset] = parseInt(value.substr(index * 2, 2), 16); + // TODO With some magic we can use the HEX_TO_U16 lookups here as well + // (and then finally HEX_TO_U8 for the overflows) + result[index + offset] = HEX_TO_U8[value.substr(index * 2, 2)]; } return result; diff --git a/packages/util/src/hex/toU8aBuffer.spec.ts b/packages/util/src/hex/toU8aBuffer.spec.ts index 7f0ad515a..1af52ab2d 100644 --- a/packages/util/src/hex/toU8aBuffer.spec.ts +++ b/packages/util/src/hex/toU8aBuffer.spec.ts @@ -1,7 +1,8 @@ // Copyright 2017-2021 @polkadot/util authors & contributors // SPDX-License-Identifier: Apache-2.0 -import { hexToU8a } from '.'; +import { U16_TO_HEX } from './alphabet'; +import { hexToU8a } from './toU8aBuffer'; describe('hexToU8a', (): void => { it('returns an empty Uint8Array when null provided', (): void => { @@ -20,7 +21,7 @@ describe('hexToU8a', (): void => { it('returns a Uint8Array with the correct values (bitLength > provided)', (): void => { expect( - hexToU8a('0x80000a', 32) + hexToU8a('0x80000A', 32) ).toEqual( new Uint8Array([0, 128, 0, 10]) ); @@ -39,4 +40,20 @@ describe('hexToU8a', (): void => { (): Uint8Array => hexToU8a('notahex') ).toThrow(/hex value to convert/); }); + + it.skip('performance', (): void => { + let a = '0x'; + + for (let i = 0; i < 8192; i++) { + a += U16_TO_HEX[i]; + } + + console.time('hexToU8a:performance'); + + for (let i = 0; i < 8192; i++) { + hexToU8a(a); + } + + console.timeEnd('hexToU8a:performance'); + }); }); diff --git a/packages/util/src/hex/toU8aBuffer.ts b/packages/util/src/hex/toU8aBuffer.ts index ccbe61842..14fc5c841 100644 --- a/packages/util/src/hex/toU8aBuffer.ts +++ b/packages/util/src/hex/toU8aBuffer.ts @@ -22,15 +22,16 @@ import { hexStripPrefix } from './stripPrefix'; * hexToU8a('0x80001f', 32); // Uint8Array([0x00, 0x80, 0x00, 0x1f]) * ``` */ -export function hexToU8a (value?: HexString | string | null, bitLength = -1): Uint8Array { - if (!value) { +export function hexToU8a (_value?: HexString | string | null, bitLength = -1): Uint8Array { + if (!_value) { return new Uint8Array(); } - assert(isHex(value), () => `Expected hex value to convert, found '${value}'`); + assert(isHex(_value), () => `Expected hex value to convert, found '${_value}'`); - const buf = Buffer.from(hexStripPrefix(value), 'hex'); - const valLength = buf.length / 2; + const value = hexStripPrefix(_value); + const buf = Buffer.from(value, 'hex'); + const valLength = value.length / 2; const resultLength = Math.ceil( bitLength === -1 ? valLength diff --git a/packages/util/src/u8a/eq.spec.ts b/packages/util/src/u8a/eq.spec.ts index 92c5a98f9..2b24a085f 100644 --- a/packages/util/src/u8a/eq.spec.ts +++ b/packages/util/src/u8a/eq.spec.ts @@ -39,4 +39,32 @@ describe('u8aEq', (): void => { ) ).toEqual(true); }); + + it('works on aubarray values', (): void => { + const a = new Uint8Array(16); + + for (let i = 0; i < a.length; i++) { + a[i] = i; + } + + expect( + u8aEq(a.subarray(0, 5), a.subarray(0, 5)) + ).toEqual(true); + }); + + it.skip('performance', (): void => { + const a = new Uint8Array(32768); + + for (let i = 0; i < a.length; i++) { + a[i] = i % 256; + } + + console.time('u8aEq:performance'); + + for (let i = 0; i < 65536; i++) { + u8aEq(a, a); + } + + console.timeEnd('u8aEq:performance'); + }); }); diff --git a/packages/util/src/u8a/eq.ts b/packages/util/src/u8a/eq.ts index 0ac7e05eb..708b2eb48 100644 --- a/packages/util/src/u8a/eq.ts +++ b/packages/util/src/u8a/eq.ts @@ -3,11 +3,59 @@ import type { HexString } from '../types'; -import { u8aCmp } from './cmp'; import { u8aToU8a } from './toU8a'; +interface Constructor { + BYTES_PER_ELEMENT: number; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + new(...args: any[]): T; +} + +// React-native does not have BigUint64Array, check others as well +const U64_BYTES = typeof BigUint64Array !== 'undefined' + ? BigUint64Array.BYTES_PER_ELEMENT + : 0; +const U32_BYTES = typeof Uint32Array !== 'undefined' + ? Uint32Array.BYTES_PER_ELEMENT + : 0; +const U16_BYTES = typeof Uint16Array !== 'undefined' + ? Uint16Array.BYTES_PER_ELEMENT + : 0; + +// Creates a Uint8Array, ensuring that the alignment is correct +function createArray (Clazz: Constructor, value: Uint8Array): T { + // The byteOffset needs to match the size of the data, i.e. for Uint32 it needs to be 4 + // NOTE: DataView doesn't have this limitation, but getters are slower + const align = Clazz.BYTES_PER_ELEMENT; + + return value.byteOffset % align + ? new Clazz(value.buffer.slice(value.byteOffset), 0, value.length / align) + : new Clazz(value.buffer, value.byteOffset, value.length / align); +} + +function equalsArray (a: T, b: T): boolean { + for (let i = 0; i < a.length; i++) { + if (a[i] !== b[i]) { + return false; + } + } + + return true; +} + function equals (a: Uint8Array, b: Uint8Array): boolean { - return (a.length === b.length) && (u8aCmp(a, b) === 0); + if (a.length === b.length) { + return U64_BYTES && (a.length % U64_BYTES) + ? U32_BYTES && (a.length % U32_BYTES) + ? U16_BYTES && (a.length % U16_BYTES) + ? equalsArray(a, b) + : equalsArray(createArray(Uint16Array, a), createArray(Uint16Array, b)) + : equalsArray(createArray(Uint32Array, a), createArray(Uint32Array, b)) + : equalsArray(createArray(BigUint64Array, a), createArray(BigUint64Array, b)); + } + + return false; } /** diff --git a/packages/util/src/u8a/toHex.spec.ts b/packages/util/src/u8a/toHex.spec.ts index 3bd5063f4..528dd67a5 100644 --- a/packages/util/src/u8a/toHex.spec.ts +++ b/packages/util/src/u8a/toHex.spec.ts @@ -57,4 +57,20 @@ describe('u8aToHex', (): void => { ) ).toEqual('0x8000…0c0d'); }); + + it.skip('performance', (): void => { + const a = new Uint8Array(32768); + + for (let i = 0; i < a.length; i++) { + a[i] = i % 256; + } + + console.time('u8aToHex:performance'); + + for (let i = 0; i < 65536; i++) { + u8aToHex(a); + } + + console.timeEnd('u8aToHex:performance'); + }); }); diff --git a/packages/util/src/u8a/toHex.ts b/packages/util/src/u8a/toHex.ts index d25c67017..ae5697952 100644 --- a/packages/util/src/u8a/toHex.ts +++ b/packages/util/src/u8a/toHex.ts @@ -1,23 +1,31 @@ // Copyright 2017-2021 @polkadot/util authors & contributors // SPDX-License-Identifier: Apache-2.0 -import type { HexDigit, HexString } from '../types'; +import type { HexString } from '../types'; -import { arrayRange } from '../array'; +import { U8_TO_HEX, U16_TO_HEX } from '../hex/alphabet'; -type HexByte = `${HexDigit}${HexDigit}`; - -const ALPHABET = arrayRange(256).map((n) => n.toString(16).padStart(2, '0') as HexByte); +const U16_SIZE = Uint16Array.BYTES_PER_ELEMENT; /** @internal */ function hex (value: Uint8Array): string { - const result = new Array(value.length); + const mod = value.length % U16_SIZE; - for (let i = 0; i < value.length; i++) { - result[i] = ALPHABET[value[i]]; + // We need alignment of the Uint16Array, see u8aEq for further details + const u16 = value.byteOffset % U16_SIZE + ? new Uint16Array(value.buffer.slice(value.byteOffset), 0, (value.length - mod) / U16_SIZE) + : new Uint16Array(value.buffer, value.byteOffset, (value.length - mod) / U16_SIZE); + let result = ''; + + for (let i = 0; i < u16.length; i++) { + result += U16_TO_HEX[u16[i]]; } - return result.join(''); + if (mod) { + result += U8_TO_HEX[value[value.length - 1]]; + } + + return result; } /** diff --git a/packages/util/src/u8a/toHexBuffer.spec.ts b/packages/util/src/u8a/toHexBuffer.spec.ts index ea16f174a..77af52cf5 100644 --- a/packages/util/src/u8a/toHexBuffer.spec.ts +++ b/packages/util/src/u8a/toHexBuffer.spec.ts @@ -57,4 +57,20 @@ describe('u8aToHex', (): void => { ) ).toEqual('0x8000…0c0d'); }); + + it.skip('performance', (): void => { + const a = new Uint8Array(32768); + + for (let i = 0; i < a.length; i++) { + a[i] = i % 256; + } + + console.time('u8aToHex:performance'); + + for (let i = 0; i < 65536; i++) { + u8aToHex(a); + } + + console.timeEnd('u8aToHex:performance'); + }); });