xxhash64 replacement (#1220)

* xxhash64 replacement

* yarn.lock

* hasBigInt check

* extract common

* lint fixes

* Dedupe

* flatten

* Update changes

* Update packages/util-crypto/src/xxhash/asU8a.spec.ts

* Remove useless assignment

* flatten

* combine intermediate vars

* Align var naming

* lint

* Remove unneeded inner conversions

* Update packages/util-crypto/src/xxhash/xxhash64.ts

* Adjust

* bitLength type

* BitLength type

* additional BitLength

* Small cleanups

* Adjust

* Adjust

* blak2 bitLength adjust

* with 384

* renames

* Adjust

* Adjust

* localize

* Comments

* format performance outputs

* format performance outputs

* isWasmOnly revert

packages/util-crypto/package.json

@@ -39,16 +39,14 @@
     "js-sha3": "^0.8.0",
     "micro-base": "^0.9.0",
     "scryptsy": "^2.1.0",
-    "tweetnacl": "^1.0.3",
-    "xxhashjs": "^0.2.2"
+    "tweetnacl": "^1.0.3"
   },
   "devDependencies": {
     "@types/bn.js": "^4.11.6",
     "@types/create-hash": "^1.2.2",
     "@types/ed2curve": "^0.2.2",
     "@types/elliptic": "^6.4.14",
-    "@types/scryptsy": "^2.0.0",
-    "@types/xxhashjs": "^0.2.2"
+    "@types/scryptsy": "^2.0.0"
   },
   "peerDependencies": {
     "@polkadot/util": "7.9.3-3"

packages/util-crypto/src/blake2/asU8a.ts

@@ -10,6 +10,8 @@ import { blake2b, isReady } from '@polkadot/wasm-crypto';
 
 import { createAsHex } from '../helpers';
 
+type BitLength = 64 | 128 | 256 | 384 | 512;
+
 /**
  * @name blake2AsU8a
  * @summary Creates a blake2b u8a from the input.
@@ -24,7 +26,7 @@ import { createAsHex } from '../helpers';
  * blake2AsU8a('abc'); // => [0xba, 0x80, 0xa5, 0x3f, 0x98, 0x1c, 0x4d, 0x0d]
  * ```
  */
-export function blake2AsU8a (data: HexString | Uint8Array | string, bitLength = 256, key?: Uint8Array | null, onlyJs?: boolean): Uint8Array {
+export function blake2AsU8a (data: HexString | Uint8Array | string, bitLength: BitLength = 256, key?: Uint8Array | null, onlyJs?: boolean): Uint8Array {
   const byteLength = Math.ceil(bitLength / 8);
   const u8a = u8aToU8a(data);
 

packages/util-crypto/src/hmac/shaAsU8a.ts

@@ -6,14 +6,18 @@ import hash from 'hash.js';
 import { u8aToU8a } from '@polkadot/util';
 import { hmacSha256, hmacSha512, isReady } from '@polkadot/wasm-crypto';
 
-type HashFn = (key: Uint8Array | string, data: Uint8Array, onlyJs?: boolean) => Uint8Array;
+type BitLength = 256 | 512;
 
-function createSha (bitLength: 256 | 512 = 256): HashFn {
+function createSha (bitLength: BitLength): (key: Uint8Array | string, data: Uint8Array, onlyJs?: boolean) => Uint8Array {
   return (key: Uint8Array | string, data: Uint8Array, onlyJs?: boolean): Uint8Array =>
     hmacShaAsU8a(key, data, bitLength, onlyJs);
 }
 
-export function hmacShaAsU8a (key: Uint8Array | string, data: Uint8Array, bitLength: 256 | 512 = 256, onlyJs?: boolean): Uint8Array {
+/**
+ * @name hmacShaAsU8a
+ * @description creates a Hmac Sha (256/512) Uint8Array from the key & data
+ */
+export function hmacShaAsU8a (key: Uint8Array | string, data: Uint8Array, bitLength: BitLength = 256, onlyJs?: boolean): Uint8Array {
   const is256 = bitLength === 256;
   const u8aKey = u8aToU8a(key);
 
@@ -32,5 +36,14 @@ export function hmacShaAsU8a (key: Uint8Array | string, data: Uint8Array, bitLen
     );
 }
 
+/**
+ * @name hmacSha256AsU8a
+ * @description creates a Hmac Sha256 Uint8Array from the key & data
+ */
 export const hmacSha256AsU8a = createSha(256);
+
+/**
+ * @name hmacSha512AsU8a
+ * @description creates a Hmac Sha512 Uint8Array from the key & data
+ */
 export const hmacSha512AsU8a = createSha(512);

packages/util-crypto/src/keccak/asU8a.ts

@@ -10,6 +10,8 @@ import { isReady, keccak256, keccak512 } from '@polkadot/wasm-crypto';
 
 import { createAsHex, createBitHasher } from '../helpers';
 
+type BitLength = 256 | 512;
+
 /**
  * @name keccakAsU8a
  * @summary Creates a keccak Uint8Array from the input.
@@ -24,7 +26,7 @@ import { createAsHex, createBitHasher } from '../helpers';
  * keccakAsU8a('123'); // => Uint8Array
  * ```
  */
-export function keccakAsU8a (value: HexString | Buffer | Uint8Array | string, bitLength: 256 | 512 = 256, onlyJs?: boolean): Uint8Array {
+export function keccakAsU8a (value: HexString | Buffer | Uint8Array | string, bitLength: BitLength = 256, onlyJs?: boolean): Uint8Array {
   const is256 = bitLength === 256;
   const u8a = u8aToU8a(value);
 

packages/util-crypto/src/scrypt/encode.ts

@@ -21,11 +21,13 @@ interface Result {
 export function scryptEncode (passphrase?: HexString | Uint8Array | string, salt = randomAsU8a(), params = DEFAULT_PARAMS, onlyJs?: boolean): Result {
   const u8a = u8aToU8a(passphrase);
 
-  const password = !onlyJs && isReady()
-    ? scrypt(u8a, salt, Math.log2(params.N), params.r, params.p)
-    : bufferToU8a(
-      scryptsy(u8aToBuffer(u8a), u8aToBuffer(salt), params.N, params.r, params.p, 64)
-    );
-
-  return { params, password, salt };
+  return {
+    params,
+    password: !onlyJs && isReady()
+      ? scrypt(u8a, salt, Math.log2(params.N), params.r, params.p)
+      : bufferToU8a(
+        scryptsy(u8aToBuffer(u8a), u8aToBuffer(salt), params.N, params.r, params.p, 64)
+      ),
+    salt
+  };
 }

packages/util-crypto/src/sha/asU8a.ts

@@ -10,11 +10,13 @@ import { isReady, sha256, sha512 } from '@polkadot/wasm-crypto';
 
 import { createBitHasher } from '../helpers';
 
+type BitLength = 256 | 512;
+
 /**
  * @name shaAsU8a
  * @summary Creates a sha Uint8Array from the input.
  */
-export function shaAsU8a (value: HexString | Buffer | Uint8Array | string, bitLength: 256 | 512 = 256, onlyJs?: boolean): Uint8Array {
+export function shaAsU8a (value: HexString | Buffer | Uint8Array | string, bitLength: BitLength = 256, onlyJs?: boolean): Uint8Array {
   const is256 = bitLength === 256;
   const u8a = u8aToU8a(value);
 
@@ -29,5 +31,14 @@ export function shaAsU8a (value: HexString | Buffer | Uint8Array | string, bitLe
     );
 }
 
+/**
+ * @name sha256AsU8a
+ * @summary Creates a sha256 Uint8Array from the input.
+ */
 export const sha256AsU8a = createBitHasher(256, shaAsU8a);
+
+/**
+ * @name sha512AsU8a
+ * @summary Creates a sha512 Uint8Array from the input.
+ */
 export const sha512AsU8a = createBitHasher(512, shaAsU8a);

packages/util-crypto/src/xxhash/asU8a.ts

@@ -3,11 +3,13 @@
 
 import type { HexString } from '@polkadot/util/types';
 
-import { u8aToU8a } from '@polkadot/util';
+import { hasBigInt, u8aToU8a } from '@polkadot/util';
 import { isReady, twox } from '@polkadot/wasm-crypto';
 
 import { createAsHex } from '../helpers';
-import xxhash64AsBn from './xxhash64/asBn';
+import { xxhash64 } from './xxhash64';
+
+type BitLength = 64 | 128 | 192 | 256 | 320 | 384 | 448 | 512;
 
 /**
  * @name xxhashAsU8a
@@ -23,18 +25,18 @@ import xxhash64AsBn from './xxhash64/asBn';
  * xxhashAsU8a('abc'); // => 0x44bc2cf5ad770999
  * ```
  */
-export function xxhashAsU8a (data: HexString | Buffer | Uint8Array | string, bitLength = 64, onlyJs?: boolean): Uint8Array {
-  const iterations = Math.ceil(bitLength / 64);
+export function xxhashAsU8a (data: HexString | Buffer | Uint8Array | string, bitLength: BitLength = 64, onlyJs?: boolean): Uint8Array {
+  const rounds = Math.ceil(bitLength / 64);
   const u8a = u8aToU8a(data);
 
-  if (!onlyJs && isReady()) {
-    return twox(u8a, iterations);
+  if (!hasBigInt || (!onlyJs && isReady())) {
+    return twox(u8a, rounds);
   }
 
-  const result = new Uint8Array(Math.ceil(bitLength / 8));
+  const result = new Uint8Array(rounds * 8);
 
-  for (let seed = 0; seed < iterations; seed++) {
-    result.set(xxhash64AsBn(u8a, seed).toArray('le', 8), seed * 8);
+  for (let seed = 0; seed < rounds; seed++) {
+    result.set(xxhash64(u8a, seed).reverse(), seed * 8);
   }
 
   return result;

packages/util-crypto/src/xxhash/xxhash64.ts

@@ -0,0 +1,143 @@
+// Copyright 2017-2021 @polkadot/util-crypto authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+
+// Adapted from https://github.com/pierrec/js-xxhash/blob/0504e76f3d31a21ae8528a7f590c7289c9e431d2/lib/xxhash64.js
+//
+// xxHash64 implementation in pure Javascript
+// Copyright (C) 2016, Pierre Curto
+// MIT license
+//
+// Changes made:
+//   - converted to TypeScript
+//   - uses native JS BigInt (no external dependencies)
+//   - support only for Uint8Array inputs
+//   - no constructor function, straight fill & digest
+//   - update code removed, only called once, no streams
+
+interface State {
+  seed: bigint;
+  u8a: Uint8Array;
+  u8asize: number;
+  v1: bigint;
+  v2: bigint;
+  v3: bigint;
+  v4: bigint;
+}
+
+const P64_1 = 11400714785074694791n;
+const P64_2 = 14029467366897019727n;
+const P64_3 = 1609587929392839161n;
+const P64_4 = 9650029242287828579n;
+const P64_5 = 2870177450012600261n;
+
+// mask for a u64, all bits set
+const U64 = 2n ** 64n - 1n;
+
+function rotl (a: bigint, b: bigint): bigint {
+  const c = a & U64;
+
+  return ((c << b) | (c >> (64n - b))) & U64;
+}
+
+function fromU8a (u8a: Uint8Array, p: number, count: 2 | 4): bigint {
+  const bigints = new Array<bigint>(count);
+  let offset = 0;
+
+  for (let i = 0; i < count; i++, offset += 2) {
+    bigints[i] = BigInt(u8a[p + offset] | (u8a[p + 1 + offset] << 8));
+  }
+
+  let result = 0n;
+
+  for (let i = count - 1; i >= 0; i--) {
+    result = (result << 16n) + bigints[i];
+  }
+
+  return result;
+}
+
+function toU8a (h64: bigint): Uint8Array {
+  const result = new Uint8Array(8);
+
+  for (let i = 7; i >= 0; i--) {
+    result[i] = Number(h64 % 256n);
+
+    h64 = h64 / 256n;
+  }
+
+  return result;
+}
+
+function state (initSeed: bigint | number): State {
+  const seed = BigInt(initSeed);
+
+  return {
+    seed,
+    u8a: new Uint8Array(32),
+    u8asize: 0,
+    v1: seed + P64_1 + P64_2,
+    v2: seed + P64_2,
+    v3: seed,
+    v4: seed - P64_1
+  };
+}
+
+function init (state: State, input: Uint8Array): State {
+  if (input.length < 32) {
+    state.u8a.set(input);
+    state.u8asize = input.length;
+
+    return state;
+  }
+
+  const limit = input.length - 32;
+  let p = 0;
+
+  if (limit >= 0) {
+    const adjustV = (v: bigint) =>
+      P64_1 * rotl(v + P64_2 * fromU8a(input, p, 4), 31n);
+
+    do {
+      state.v1 = adjustV(state.v1); p += 8;
+      state.v2 = adjustV(state.v2); p += 8;
+      state.v3 = adjustV(state.v3); p += 8;
+      state.v4 = adjustV(state.v4); p += 8;
+    } while (p <= limit);
+  }
+
+  if (p < input.length) {
+    state.u8a.set(input.subarray(p, input.length));
+    state.u8asize = input.length - p;
+  }
+
+  return state;
+}
+
+export function xxhash64 (input: Uint8Array, initSeed: bigint | number): Uint8Array {
+  const { seed, u8a, u8asize, v1, v2, v3, v4 } = init(state(initSeed), input);
+  let p = 0;
+  let h64 = U64 & (BigInt(input.length) + (
+    input.length >= 32
+      ? (((((((((rotl(v1, 1n) + rotl(v2, 7n) + rotl(v3, 12n) + rotl(v4, 18n)) ^ (P64_1 * rotl(v1 * P64_2, 31n))) * P64_1 + P64_4) ^ (P64_1 * rotl(v2 * P64_2, 31n))) * P64_1 + P64_4) ^ (P64_1 * rotl(v3 * P64_2, 31n))) * P64_1 + P64_4) ^ (P64_1 * rotl(v4 * P64_2, 31n))) * P64_1 + P64_4)
+      : (seed + P64_5)
+  ));
+
+  while (p <= (u8asize - 8)) {
+    h64 = U64 & (P64_4 + P64_1 * rotl(h64 ^ (P64_1 * rotl(P64_2 * fromU8a(u8a, p, 4), 31n)), 27n));
+    p += 8;
+  }
+
+  if ((p + 4) <= u8asize) {
+    h64 = U64 & (P64_3 + P64_2 * rotl(h64 ^ (P64_1 * fromU8a(u8a, p, 2)), 23n));
+    p += 4;
+  }
+
+  while (p < u8asize) {
+    h64 = U64 & (P64_1 * rotl(h64 ^ (P64_5 * BigInt(u8a[p++])), 11n));
+  }
+
+  h64 = U64 & (P64_2 * (h64 ^ (h64 >> 33n)));
+  h64 = U64 & (P64_3 * (h64 ^ (h64 >> 29n)));
+
+  return toU8a(U64 & (h64 ^ (h64 >> 32n)));
+}

packages/util-crypto/src/xxhash/xxhash64/asBn.spec.ts

@@ -1,12 +0,0 @@
-// Copyright 2017-2021 @polkadot/util-crypto authors & contributors
-// SPDX-License-Identifier: Apache-2.0
-
-import xxhash64AsBn from './asBn';
-
-describe('xxhash64AsBn', (): void => {
-  it('creates the correct BN output', (): void => {
-    expect(
-      xxhash64AsBn('abcd', 0xabcd).toString(16)
-    ).toEqual('e29f70f8b8c96df7');
-  });
-});

packages/util-crypto/src/xxhash/xxhash64/asBn.ts

@@ -1,13 +0,0 @@
-// Copyright 2017-2021 @polkadot/util-crypto authors & contributors
-// SPDX-License-Identifier: Apache-2.0
-
-import { BN } from '@polkadot/util';
-
-import xxhash64AsRaw from './asRaw';
-
-export default function xxhash64AsBn (data: Buffer | Uint8Array | string, seed: number): BN {
-  return new BN(
-    xxhash64AsRaw(data, seed),
-    16
-  );
-}

packages/util-crypto/src/xxhash/xxhash64/asHex.spec.ts

@@ -1,12 +0,0 @@
-// Copyright 2017-2021 @polkadot/util-crypto authors & contributors
-// SPDX-License-Identifier: Apache-2.0
-
-import xxhash64AsHex from './asHex';
-
-describe('xxhash64AsHex', (): void => {
-  it('creates the correct hex output', (): void => {
-    expect(
-      xxhash64AsHex('abcd', 0xabcd)
-    ).toEqual('0xe29f70f8b8c96df7');
-  });
-});

packages/util-crypto/src/xxhash/xxhash64/asHex.ts

@@ -1,12 +0,0 @@
-// Copyright 2017-2021 @polkadot/util-crypto authors & contributors
-// SPDX-License-Identifier: Apache-2.0
-
-import { hexAddPrefix } from '@polkadot/util';
-
-import xxhash64AsRaw from './asRaw';
-
-export default function xxhash64AsHex (data: Buffer | Uint8Array | string, seed: number): string {
-  return hexAddPrefix(
-    xxhash64AsRaw(data, seed)
-  );
-}

packages/util-crypto/src/xxhash/xxhash64/asRaw.spec.ts

@@ -1,12 +0,0 @@
-// Copyright 2017-2021 @polkadot/util-crypto authors & contributors
-// SPDX-License-Identifier: Apache-2.0
-
-import xxhash64AsRaw from './asRaw';
-
-describe('xxhash64AsRaw', (): void => {
-  it('creates the correct non-prefixed hex output', (): void => {
-    expect(
-      xxhash64AsRaw('abcd', 0xabcd)
-    ).toEqual('e29f70f8b8c96df7');
-  });
-});

packages/util-crypto/src/xxhash/xxhash64/asRaw.ts

@@ -1,8 +0,0 @@
-// Copyright 2017-2021 @polkadot/util-crypto authors & contributors
-// SPDX-License-Identifier: Apache-2.0
-
-import xxhash64AsValue from './asValue';
-
-export default function xxhash64AsRaw (data: Buffer | Uint8Array | string, seed: number): string {
-  return xxhash64AsValue(data, seed).toString(16);
-}

packages/util-crypto/src/xxhash/xxhash64/asValue.spec.ts

@@ -1,24 +0,0 @@
-// Copyright 2017-2021 @polkadot/util-crypto authors & contributors
-// SPDX-License-Identifier: Apache-2.0
-
-import xxhash64AsValue from './asValue';
-
-describe('xxhash64AsValue', (): void => {
-  it('creates the correct output (string input)', (): void => {
-    expect(
-      xxhash64AsValue('abcd', 0xabcd).toString(16)
-    ).toEqual('e29f70f8b8c96df7');
-  });
-
-  it('creates the correct output (Buffer input)', (): void => {
-    expect(
-      xxhash64AsValue(Buffer.from([0x61, 0x62, 0x63, 0x64]), 0xabcd).toString(16)
-    ).toEqual('e29f70f8b8c96df7');
-  });
-
-  it('creates the correct output (Uint8Array input)', (): void => {
-    expect(
-      xxhash64AsValue(new Uint8Array([0x61, 0x62, 0x63, 0x64]), 0xabcd).toString(16)
-    ).toEqual('e29f70f8b8c96df7');
-  });
-});

packages/util-crypto/src/xxhash/xxhash64/asValue.ts

@@ -1,13 +0,0 @@
-// Copyright 2017-2021 @polkadot/util-crypto authors & contributors
-// SPDX-License-Identifier: Apache-2.0
-
-import xx from 'xxhashjs';
-
-import { u8aToU8a } from '@polkadot/util';
-
-export default function xxhash64AsValue (data: Buffer | Uint8Array | string, seed: number): number {
-  return xx.h64(
-    u8aToU8a(data).buffer,
-    seed
-  );
-}

packages/util-crypto/src/xxhash/xxhash64/xxhash64.d.ts

@@ -1,12 +0,0 @@
-// Copyright 2017-2021 @polkadot/util-crypto authors & contributors
-// SPDX-License-Identifier: Apache-2.0
-
-declare module 'xxhashjs' {
-  export interface HashInterface {
-    (data: string | Buffer | ArrayBuffer, seed: number): number;
-  }
-
-  const h64: HashInterface;
-
-  export = { h64 };
-}

packages/util/src/array/range.ts

@@ -21,9 +21,9 @@ import { assert } from '../assert';
 export function arrayRange (size: number, startAt = 0): number[] {
   assert(size > 0, 'Expected non-zero, positive number as a range size');
 
-  const result = new Array<number>(size).fill(0);
+  const result = new Array<number>(size);
 
-  for (let i = 0; i < result.length; i++) {
+  for (let i = 0; i < size; i++) {
     result[i] = i + startAt;
   }
 

yarn.lock

@@ -2093,7 +2093,6 @@ __metadata:
     "@types/ed2curve": ^0.2.2
     "@types/elliptic": ^6.4.14
     "@types/scryptsy": ^2.0.0
-    "@types/xxhashjs": ^0.2.2
     blakejs: ^1.1.1
     bn.js: ^4.12.0
     create-hash: ^1.2.0
@@ -2104,7 +2103,6 @@ __metadata:
     micro-base: ^0.9.0
     scryptsy: ^2.1.0
     tweetnacl: ^1.0.3
-    xxhashjs: ^0.2.2
   peerDependencies:
     "@polkadot/util": 7.9.3-3
   languageName: unknown
@@ -2650,15 +2648,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@types/xxhashjs@npm:^0.2.2":
-  version: 0.2.2
-  resolution: "@types/xxhashjs@npm:0.2.2"
-  dependencies:
-    "@types/node": "*"
-  checksum: 2195cd97c32dfb0ab37fd831368ad106c43c4d2450dda441b493875ad476b00ccee5e3a676971cf8a6f7518c23d527a6791b8b005f7e52c9d3b39bc7286f1615
-  languageName: node
-  linkType: hard
-
 "@types/yargs-parser@npm:*":
   version: 20.2.0
   resolution: "@types/yargs-parser@npm:20.2.0"
@@ -4142,13 +4131,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"cuint@npm:^0.2.2":
-  version: 0.2.2
-  resolution: "cuint@npm:0.2.2"
-  checksum: b8127a93a7f16ce120ffcb22108014327c9808b258ee20e7dbb4c6740d7cb0f0c12d18a054eb716b0f2470090666abaae8a082d3cd5ef0e94fa447dd155842c4
-  languageName: node
-  linkType: hard
-
 "d@npm:1, d@npm:^1.0.1":
   version: 1.0.1
   resolution: "d@npm:1.0.1"
@@ -10642,15 +10624,6 @@ resolve@^2.0.0-next.3:
   languageName: node
   linkType: hard
 
-"xxhashjs@npm:^0.2.2":
-  version: 0.2.2
-  resolution: "xxhashjs@npm:0.2.2"
-  dependencies:
-    cuint: ^0.2.2
-  checksum: cf6baf05bafe5651dbf108008bafdb1ebe972f65228633f00b56c49d7a1e614a821fe3345c4eb27462994c7c954d982eae05871be6a48146f30803dd87f3c3b6
-  languageName: node
-  linkType: hard
-
 "y18n@npm:^5.0.5":
   version: 5.0.5
   resolution: "y18n@npm:5.0.5"