Performance optimizations (#1181)

* Performance optimizations

* Use slice internally to u8aEq

* Adjust

* formatting

* subarray

* Fixes

* Use DataView

* With alignment

* Use Float64Array when available

* Skip performance

* Renames

* Alignment comment

* Constants

* Use BYTES_PER_ELEMENT

* Adjust

CHANGELOG.md

@@ -8,6 +8,8 @@ Changes:
 - Add strongly typed `object{Entries, Keys, Values}` utilities
 - Expand tests for uncovered functions
 - Sync with upstream Substrate ss58 registry
+- Optimizations for hex <-> u8a functions
+- Allow `u8aEq` to use `Uint{8, 16, 32}Array` with correct alignment
 
 
 ## 7.8.2 Nov 7, 2021

packages/util/src/compact/fromU8a.ts

@@ -27,9 +27,9 @@ export function compactFromU8a (_input: U8aLike): [number, BN] {
   if (flag === 0b00) {
     return [1, new BN(input[0]).ishrn(2)];
   } else if (flag === 0b01) {
-    return [2, u8aToBn(input.slice(0, 2), true).ishrn(2)];
+    return [2, u8aToBn(input.subarray(0, 2), true).ishrn(2)];
   } else if (flag === 0b10) {
-    return [4, u8aToBn(input.slice(0, 4), true).ishrn(2)];
+    return [4, u8aToBn(input.subarray(0, 4), true).ishrn(2)];
   }
 
   const offset = 1 + (

packages/util/src/hex/alphabet.ts

@@ -0,0 +1,27 @@
+// Copyright 2017-2021 @polkadot/util authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+
+const U8_TO_HEX = new Array<string>(256);
+const U16_TO_HEX = new Array<string>(256 * 256);
+
+const HEX_TO_U8: Record<string, number> = {};
+const HEX_TO_U16: Record<string, number> = {};
+
+for (let n = 0; n < 256; n++) {
+  const hex = n.toString(16).padStart(2, '0');
+
+  U8_TO_HEX[n] = hex;
+  HEX_TO_U8[hex] = n;
+}
+
+for (let i = 0; i < 256; i++) {
+  for (let j = 0; j < 256; j++) {
+    const hex = U8_TO_HEX[i] + U8_TO_HEX[j];
+    const n = (j * 256) + i;
+
+    U16_TO_HEX[n] = hex;
+    HEX_TO_U16[hex] = n;
+  }
+}
+
+export { HEX_TO_U16, HEX_TO_U8, U16_TO_HEX, U8_TO_HEX };

packages/util/src/hex/toU8a.spec.ts

@@ -1,6 +1,7 @@
 // Copyright 2017-2021 @polkadot/util authors & contributors
 // SPDX-License-Identifier: Apache-2.0
 
+import { U16_TO_HEX } from './alphabet';
 import { hexToU8a } from '.';
 
 describe('hexToU8a', (): void => {
@@ -20,7 +21,7 @@ describe('hexToU8a', (): void => {
 
   it('returns a Uint8Array with the correct values (bitLength > provided)', (): void => {
     expect(
-      hexToU8a('0x80000a', 32)
+      hexToU8a('0x80000A', 32)
     ).toEqual(
       new Uint8Array([0, 128, 0, 10])
     );
@@ -39,4 +40,20 @@ describe('hexToU8a', (): void => {
       (): Uint8Array => hexToU8a('notahex')
     ).toThrow(/hex value to convert/);
   });
+
+  it.skip('performance', (): void => {
+    let a = '0x';
+
+    for (let i = 0; i < 8192; i++) {
+      a += U16_TO_HEX[i];
+    }
+
+    console.time('hexToU8a:performance');
+
+    for (let i = 0; i < 8192; i++) {
+      hexToU8a(a);
+    }
+
+    console.timeEnd('hexToU8a:performance');
+  });
 });

packages/util/src/hex/toU8a.ts

@@ -5,6 +5,7 @@ import type { HexString } from '../types';
 
 import { assert } from '../assert';
 import { isHex } from '../is/hex';
+import { HEX_TO_U8 } from './alphabet';
 import { hexStripPrefix } from './stripPrefix';
 
 /**
@@ -29,7 +30,7 @@ export function hexToU8a (_value?: HexString | string | null, bitLength = -1): U
 
   assert(isHex(_value), () => `Expected hex value to convert, found '${_value}'`);
 
-  const value = hexStripPrefix(_value);
+  const value = hexStripPrefix(_value).toLowerCase();
   const valLength = value.length / 2;
   const resultLength = Math.ceil(
     bitLength === -1
@@ -40,7 +41,9 @@ export function hexToU8a (_value?: HexString | string | null, bitLength = -1): U
   const offset = Math.max(0, resultLength - valLength);
 
   for (let index = 0; index < resultLength; index++) {
-    result[index + offset] = parseInt(value.substr(index * 2, 2), 16);
+    // TODO With some magic we can use the HEX_TO_U16 lookups here as well
+    // (and then finally HEX_TO_U8 for the overflows)
+    result[index + offset] = HEX_TO_U8[value.substr(index * 2, 2)];
   }
 
   return result;

packages/util/src/hex/toU8aBuffer.spec.ts

@@ -1,7 +1,8 @@
 // Copyright 2017-2021 @polkadot/util authors & contributors
 // SPDX-License-Identifier: Apache-2.0
 
-import { hexToU8a } from '.';
+import { U16_TO_HEX } from './alphabet';
+import { hexToU8a } from './toU8aBuffer';
 
 describe('hexToU8a', (): void => {
   it('returns an empty Uint8Array when null provided', (): void => {
@@ -20,7 +21,7 @@ describe('hexToU8a', (): void => {
 
   it('returns a Uint8Array with the correct values (bitLength > provided)', (): void => {
     expect(
-      hexToU8a('0x80000a', 32)
+      hexToU8a('0x80000A', 32)
     ).toEqual(
       new Uint8Array([0, 128, 0, 10])
     );
@@ -39,4 +40,20 @@ describe('hexToU8a', (): void => {
       (): Uint8Array => hexToU8a('notahex')
     ).toThrow(/hex value to convert/);
   });
+
+  it.skip('performance', (): void => {
+    let a = '0x';
+
+    for (let i = 0; i < 8192; i++) {
+      a += U16_TO_HEX[i];
+    }
+
+    console.time('hexToU8a:performance');
+
+    for (let i = 0; i < 8192; i++) {
+      hexToU8a(a);
+    }
+
+    console.timeEnd('hexToU8a:performance');
+  });
 });

packages/util/src/hex/toU8aBuffer.ts

@@ -22,15 +22,16 @@ import { hexStripPrefix } from './stripPrefix';
  * hexToU8a('0x80001f', 32); // Uint8Array([0x00, 0x80, 0x00, 0x1f])
  * ```
  */
-export function hexToU8a (value?: HexString | string | null, bitLength = -1): Uint8Array {
-  if (!value) {
+export function hexToU8a (_value?: HexString | string | null, bitLength = -1): Uint8Array {
+  if (!_value) {
     return new Uint8Array();
   }
 
-  assert(isHex(value), () => `Expected hex value to convert, found '${value}'`);
+  assert(isHex(_value), () => `Expected hex value to convert, found '${_value}'`);
 
-  const buf = Buffer.from(hexStripPrefix(value), 'hex');
-  const valLength = buf.length / 2;
+  const value = hexStripPrefix(_value);
+  const buf = Buffer.from(value, 'hex');
+  const valLength = value.length / 2;
   const resultLength = Math.ceil(
     bitLength === -1
       ? valLength

packages/util/src/u8a/eq.spec.ts

@@ -39,4 +39,32 @@ describe('u8aEq', (): void => {
       )
     ).toEqual(true);
   });
+
+  it('works on aubarray values', (): void => {
+    const a = new Uint8Array(16);
+
+    for (let i = 0; i < a.length; i++) {
+      a[i] = i;
+    }
+
+    expect(
+      u8aEq(a.subarray(0, 5), a.subarray(0, 5))
+    ).toEqual(true);
+  });
+
+  it.skip('performance', (): void => {
+    const a = new Uint8Array(32768);
+
+    for (let i = 0; i < a.length; i++) {
+      a[i] = i % 256;
+    }
+
+    console.time('u8aEq:performance');
+
+    for (let i = 0; i < 65536; i++) {
+      u8aEq(a, a);
+    }
+
+    console.timeEnd('u8aEq:performance');
+  });
 });

packages/util/src/u8a/eq.ts

@@ -3,11 +3,59 @@
 
 import type { HexString } from '../types';
 
-import { u8aCmp } from './cmp';
 import { u8aToU8a } from './toU8a';
 
+interface Constructor<T extends BigUint64Array | Uint32Array | Uint16Array | Uint8Array> {
+  BYTES_PER_ELEMENT: number;
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  new(...args: any[]): T;
+}
+
+// React-native does not have BigUint64Array, check others as well
+const U64_BYTES = typeof BigUint64Array !== 'undefined'
+  ? BigUint64Array.BYTES_PER_ELEMENT
+  : 0;
+const U32_BYTES = typeof Uint32Array !== 'undefined'
+  ? Uint32Array.BYTES_PER_ELEMENT
+  : 0;
+const U16_BYTES = typeof Uint16Array !== 'undefined'
+  ? Uint16Array.BYTES_PER_ELEMENT
+  : 0;
+
+// Creates a Uint8Array, ensuring that the alignment is correct
+function createArray <T extends BigUint64Array | Uint32Array | Uint16Array | Uint8Array> (Clazz: Constructor<T>, value: Uint8Array): T {
+  // The byteOffset needs to match the size of the data, i.e. for Uint32 it needs to be 4
+  // NOTE: DataView doesn't have this limitation, but getters are slower
+  const align = Clazz.BYTES_PER_ELEMENT;
+
+  return value.byteOffset % align
+    ? new Clazz(value.buffer.slice(value.byteOffset), 0, value.length / align)
+    : new Clazz(value.buffer, value.byteOffset, value.length / align);
+}
+
+function equalsArray <T extends BigUint64Array | Uint32Array | Uint16Array | Uint8Array> (a: T, b: T): boolean {
+  for (let i = 0; i < a.length; i++) {
+    if (a[i] !== b[i]) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
 function equals (a: Uint8Array, b: Uint8Array): boolean {
-  return (a.length === b.length) && (u8aCmp(a, b) === 0);
+  if (a.length === b.length) {
+    return U64_BYTES && (a.length % U64_BYTES)
+      ? U32_BYTES && (a.length % U32_BYTES)
+        ? U16_BYTES && (a.length % U16_BYTES)
+          ? equalsArray(a, b)
+          : equalsArray(createArray(Uint16Array, a), createArray(Uint16Array, b))
+        : equalsArray(createArray(Uint32Array, a), createArray(Uint32Array, b))
+      : equalsArray(createArray(BigUint64Array, a), createArray(BigUint64Array, b));
+  }
+
+  return false;
 }
 
 /**

packages/util/src/u8a/toHex.spec.ts

@@ -57,4 +57,20 @@ describe('u8aToHex', (): void => {
       )
     ).toEqual('0x8000…0c0d');
   });
+
+  it.skip('performance', (): void => {
+    const a = new Uint8Array(32768);
+
+    for (let i = 0; i < a.length; i++) {
+      a[i] = i % 256;
+    }
+
+    console.time('u8aToHex:performance');
+
+    for (let i = 0; i < 65536; i++) {
+      u8aToHex(a);
+    }
+
+    console.timeEnd('u8aToHex:performance');
+  });
 });

packages/util/src/u8a/toHex.ts

@@ -1,23 +1,31 @@
 // Copyright 2017-2021 @polkadot/util authors & contributors
 // SPDX-License-Identifier: Apache-2.0
 
-import type { HexDigit, HexString } from '../types';
+import type { HexString } from '../types';
 
-import { arrayRange } from '../array';
+import { U8_TO_HEX, U16_TO_HEX } from '../hex/alphabet';
 
-type HexByte = `${HexDigit}${HexDigit}`;
-
-const ALPHABET = arrayRange(256).map((n) => n.toString(16).padStart(2, '0') as HexByte);
+const U16_SIZE = Uint16Array.BYTES_PER_ELEMENT;
 
 /** @internal */
 function hex (value: Uint8Array): string {
-  const result = new Array<HexByte>(value.length);
+  const mod = value.length % U16_SIZE;
 
-  for (let i = 0; i < value.length; i++) {
-    result[i] = ALPHABET[value[i]];
+  // We need alignment of the Uint16Array, see u8aEq for further details
+  const u16 = value.byteOffset % U16_SIZE
+    ? new Uint16Array(value.buffer.slice(value.byteOffset), 0, (value.length - mod) / U16_SIZE)
+    : new Uint16Array(value.buffer, value.byteOffset, (value.length - mod) / U16_SIZE);
+  let result = '';
+
+  for (let i = 0; i < u16.length; i++) {
+    result += U16_TO_HEX[u16[i]];
   }
 
-  return result.join('');
+  if (mod) {
+    result += U8_TO_HEX[value[value.length - 1]];
+  }
+
+  return result;
 }
 
 /**

packages/util/src/u8a/toHexBuffer.spec.ts

@@ -57,4 +57,20 @@ describe('u8aToHex', (): void => {
       )
     ).toEqual('0x8000…0c0d');
   });
+
+  it.skip('performance', (): void => {
+    const a = new Uint8Array(32768);
+
+    for (let i = 0; i < a.length; i++) {
+      a[i] = i % 256;
+    }
+
+    console.time('u8aToHex:performance');
+
+    for (let i = 0; i < 65536; i++) {
+      u8aToHex(a);
+    }
+
+    console.timeEnd('u8aToHex:performance');
+  });
 });