diff --git a/CHANGELOG.md b/CHANGELOG.md index 89c1d65d2..125390ec4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,8 @@ ## 2.18.0-beta.x -- Add `isAscii` to allow detection of ASCII characters (including tab, newline) +- Add `isAscii` to allow detection of printable ASCII sequences (including tab, newline) +- Add `isUtf8` to allow detection of valid Utf8 sequences ## 2.17.1 Jul 6, 2020 diff --git a/packages/util/src/is/index.ts b/packages/util/src/is/index.ts index ffafd714d..8fb8554cf 100644 --- a/packages/util/src/is/index.ts +++ b/packages/util/src/is/index.ts @@ -26,3 +26,4 @@ export { default as isTestChain } from './testChain'; export { default as isToBn } from './toBn'; export { default as isU8a } from './u8a'; export { default as isUndefined } from './undefined'; +export { default as isUtf8 } from './utf8'; diff --git a/packages/util/src/is/utf8.spec.ts b/packages/util/src/is/utf8.spec.ts new file mode 100644 index 000000000..811c3695a --- /dev/null +++ b/packages/util/src/is/utf8.spec.ts @@ -0,0 +1,41 @@ +// Copyright 2017-2020 @polkadot/util authors & contributors +// This software may be modified and distributed under the terms +// of the Apache-2.0 license. See the LICENSE file for details. + +import { isUtf8 } from '.'; + +describe('isUtf8', (): void => { + it('returns true for an ASCII string', (): void => { + expect(isUtf8('Hello\tWorld!\n\rTesting')).toEqual(true); + }); + + it('returns true for ASCII bytes', (): void => { + expect(isUtf8(new Uint8Array([0x31, 0x32, 0x20, 10]))).toEqual(true); + }); + + it('returns true for empty string inputs', (): void => { + expect(isUtf8('')).toEqual(true); + }); + + it('returns true for empty U8a inputs', (): void => { + expect(isUtf8(new Uint8Array())).toEqual(true); + }); + + it('returns false for empty undefined inputs', (): void => { + expect(isUtf8()).toEqual(false); + }); + + it('it is valid for ru', (): void => { + expect(isUtf8('Приветствую, ми')).toEqual(true); + }); + + it('is valid for zh', (): void => { + expect(isUtf8('你好')).toEqual(true); + }); + + it('is invalid for something random', (): void => { + expect(isUtf8('0x7f07b1f87709608bee603bbc79a0dfc29cd315c1351a83aa31adf7458d7d3003')).toEqual(false); + expect(isUtf8('0xc9ec51351beec59b52db06b3dd4685004013035df080358c319553ced597ca05')).toEqual(false); + expect(isUtf8('0xd12cef06e90a05b63737d542aaee1389ff9f5c25e461b362180f025c3b2d5635')).toEqual(false); + }); +}); diff --git a/packages/util/src/is/utf8.ts b/packages/util/src/is/utf8.ts new file mode 100644 index 000000000..0c205b14c --- /dev/null +++ b/packages/util/src/is/utf8.ts @@ -0,0 +1,208 @@ +// Copyright 2017-2020 @polkadot/util authors & contributors +// This software may be modified and distributed under the terms +// of the Apache-2.0 license. See the LICENSE file for details. + +// Adapted from https://github.com/JulienPalard/is_utf8/blob/master/is_utf8.c + +import u8aToU8a from '../u8a/toU8a'; +import isString from './string'; + +/** + * @name isUtf8 + * @summary Tests if the input is valid Utf8 + * @description + * Checks to see if the input string or Uint8Array is valid Utf8 + */ +export default function isUtf8 (value?: number[] | Buffer | Uint8Array | string | null): boolean { + if (!value) { + return isString(value); + } + + const u8a = u8aToU8a(value); + const len = u8a.length; + let i = 0; + + while (i < len) { + if (u8a[i] <= 0x7F) /* 00..7F */ { + i += 1; + } else if (u8a[i] >= 0xC2 && u8a[i] <= 0xDF) /* C2..DF 80..BF */ { + if (i + 1 < len) /* Expect a 2nd byte */ { + if (u8a[i + 1] < 0x80 || u8a[i + 1] > 0xBF) { + // *message = "After a first byte between C2 and DF, expecting a 2nd byte between 80 and BF"; + // *faulty_bytes = 2; + return false; + } + } else { + // *message = "After a first byte between C2 and DF, expecting a 2nd byte."; + // *faulty_bytes = 1; + return false; + } + + i += 2; + } else if (u8a[i] === 0xE0) /* E0 A0..BF 80..BF */ { + if (i + 2 < len) /* Expect a 2nd and 3rd byte */ { + if (u8a[i + 1] < 0xA0 || u8a[i + 1] > 0xBF) { + // *message = "After a first byte of E0, expecting a 2nd byte between A0 and BF."; + // *faulty_bytes = 2; + return false; + } + + if (u8a[i + 2] < 0x80 || u8a[i + 2] > 0xBF) { + // *message = "After a first byte of E0, expecting a 3nd byte between 80 and BF."; + // *faulty_bytes = 3; + return false; + } + } else { + // *message = "After a first byte of E0, expecting two following bytes."; + // *faulty_bytes = 1; + return false; + } + + i += 3; + } else if (u8a[i] >= 0xE1 && u8a[i] <= 0xEC) /* E1..EC 80..BF 80..BF */ { + if (i + 2 < len) /* Expect a 2nd and 3rd byte */ { + if (u8a[i + 1] < 0x80 || u8a[i + 1] > 0xBF) { + // *message = "After a first byte between E1 and EC, expecting the 2nd byte between 80 and BF."; + // *faulty_bytes = 2; + return false; + } + + if (u8a[i + 2] < 0x80 || u8a[i + 2] > 0xBF) { + // *message = "After a first byte between E1 and EC, expecting the 3rd byte between 80 and BF."; + // *faulty_bytes = 3; + return false; + } + } else { + // *message = "After a first byte between E1 and EC, expecting two following bytes."; + // *faulty_bytes = 1; + return false; + } + + i += 3; + } else if (u8a[i] === 0xED) /* ED 80..9F 80..BF */ { + if (i + 2 < len) /* Expect a 2nd and 3rd byte */ { + if (u8a[i + 1] < 0x80 || u8a[i + 1] > 0x9F) { + // *message = "After a first byte of ED, expecting 2nd byte between 80 and 9F."; + // *faulty_bytes = 2; + return false; + } + + if (u8a[i + 2] < 0x80 || u8a[i + 2] > 0xBF) { + // *message = "After a first byte of ED, expecting 3rd byte between 80 and BF."; + // *faulty_bytes = 3; + return false; + } + } else { + // *message = "After a first byte of ED, expecting two following bytes."; + // *faulty_bytes = 1; + return false; + } + + i += 3; + } else if (u8a[i] >= 0xEE && u8a[i] <= 0xEF) /* EE..EF 80..BF 80..BF */ { + if (i + 2 < len) /* Expect a 2nd and 3rd byte */ { + if (u8a[i + 1] < 0x80 || u8a[i + 1] > 0xBF) { + // *message = "After a first byte between EE and EF, expecting 2nd byte between 80 and BF."; + // *faulty_bytes = 2; + return false; + } + + if (u8a[i + 2] < 0x80 || u8a[i + 2] > 0xBF) { + // *message = "After a first byte between EE and EF, expecting 3rd byte between 80 and BF."; + // *faulty_bytes = 3; + return false; + } + } else { + // *message = "After a first byte between EE and EF, two following bytes."; + // *faulty_bytes = 1; + return false; + } + + i += 3; + } else if (u8a[i] === 0xF0) /* F0 90..BF 80..BF 80..BF */ { + if (i + 3 < len) /* Expect a 2nd, 3rd 3th byte */ { + if (u8a[i + 1] < 0x90 || u8a[i + 1] > 0xBF) { + // *message = "After a first byte of F0, expecting 2nd byte between 90 and BF."; + // *faulty_bytes = 2; + return false; + } + + if (u8a[i + 2] < 0x80 || u8a[i + 2] > 0xBF) { + // *message = "After a first byte of F0, expecting 3rd byte between 80 and BF."; + // *faulty_bytes = 3; + return false; + } + + if (u8a[i + 3] < 0x80 || u8a[i + 3] > 0xBF) { + // *message = "After a first byte of F0, expecting 4th byte between 80 and BF."; + // *faulty_bytes = 4; + return false; + } + } else { + // *message = "After a first byte of F0, expecting three following bytes."; + // *faulty_bytes = 1; + return false; + } + + i += 4; + } else if (u8a[i] >= 0xF1 && u8a[i] <= 0xF3) /* F1..F3 80..BF 80..BF 80..BF */ { + if (i + 3 < len) /* Expect a 2nd, 3rd 3th byte */ { + if (u8a[i + 1] < 0x80 || u8a[i + 1] > 0xBF) { + // *message = "After a first byte of F1, F2, or F3, expecting a 2nd byte between 80 and BF."; + // *faulty_bytes = 2; + return false; + } + + if (u8a[i + 2] < 0x80 || u8a[i + 2] > 0xBF) { + // *message = "After a first byte of F1, F2, or F3, expecting a 3rd byte between 80 and BF."; + // *faulty_bytes = 3; + return false; + } + + if (u8a[i + 3] < 0x80 || u8a[i + 3] > 0xBF) { + // *message = "After a first byte of F1, F2, or F3, expecting a 4th byte between 80 and BF."; + // *faulty_bytes = 4; + return false; + } + } else { + // *message = "After a first byte of F1, F2, or F3, expecting three following bytes."; + // *faulty_bytes = 1; + return false; + } + + i += 4; + } else if (u8a[i] === 0xF4) /* F4 80..8F 80..BF 80..BF */ { + if (i + 3 < len) /* Expect a 2nd, 3rd 3th byte */ { + if (u8a[i + 1] < 0x80 || u8a[i + 1] > 0x8F) { + // *message = "After a first byte of F4, expecting 2nd byte between 80 and 8F."; + // *faulty_bytes = 2; + return false; + } + + if (u8a[i + 2] < 0x80 || u8a[i + 2] > 0xBF) { + // *message = "After a first byte of F4, expecting 3rd byte between 80 and BF."; + // *faulty_bytes = 3; + return false; + } + + if (u8a[i + 3] < 0x80 || u8a[i + 3] > 0xBF) { + // *message = "After a first byte of F4, expecting 4th byte between 80 and BF."; + // *faulty_bytes = 4; + return false; + } + } else { + // *message = "After a first byte of F4, expecting three following bytes."; + // *faulty_bytes = 1; + return false; + } + + i += 4; + } else { + // *message = "Expecting bytes in the following ranges: 00..7F C2..F4."; + // *faulty_bytes = 1; + return false; + } + } + + return true; +}