Custom encoding for identity Data (#1672)

* Custom encoding for identity Data

* Custom encoding for IdentityFields

* Allow empty U8a input to Data

* IdentityField tests

packages/types/src/codec/Set.ts

@@ -4,13 +4,14 @@
 
 import { Codec, Constructor, IHash, Registry } from '../types';
 
-import { assert, isU8a, isNumber, isUndefined, stringCamelCase, stringUpperFirst, u8aToHex } from '@polkadot/util';
+import BN from 'bn.js';
+import { assert, bnToBn, isU8a, isNumber, isString, isUndefined, stringCamelCase, stringUpperFirst, u8aToHex, isBn, u8aToU8a } from '@polkadot/util';
 import { blake2AsU8a } from '@polkadot/util-crypto';
 
 import Raw from './Raw';
 import { compareArray } from './utils';
 
-type SetValues = Record<string, number>;
+type SetValues = Record<string, number | BN>;
 
 /**
  * @name Set
@@ -22,17 +23,19 @@ type SetValues = Record<string, number>;
 export default class CodecSet extends Set<string> implements Codec {
   public readonly registry: Registry;
 
-  private _setValues: SetValues;
+  protected _setValues: SetValues;
 
-  constructor (registry: Registry, setValues: SetValues, value?: string[] | Set<string> | Uint8Array | number) {
+  constructor (registry: Registry, setValues: SetValues, value?: string[] | Set<string> | Uint8Array | BN | number | string) {
     super(CodecSet.decodeSet(setValues, value));
 
     this.registry = registry;
     this._setValues = setValues;
   }
 
-  public static decodeSet (setValues: SetValues, value: string[] | Set<string> | Uint8Array | number = 0): string[] {
-    if (isU8a(value)) {
+  public static decodeSet (setValues: SetValues, value: string[] | Set<string> | Uint8Array | BN | number | string = 0): string[] {
+    if (isString(value)) {
+      return CodecSet.decodeSet(setValues, u8aToU8a(value));
+    } else if (isU8a(value)) {
       return value.length === 0
         ? []
         : CodecSet.decodeSetNumber(setValues, value[0]);
@@ -55,9 +58,10 @@ export default class CodecSet extends Set<string> implements Codec {
     }, [] as string[]);
   }
 
-  private static decodeSetNumber (setValues: SetValues, value: number): string[] {
+  private static decodeSetNumber (setValues: SetValues, _value: BN | number): string[] {
+    const bn = bnToBn(_value);
     const result = Object.keys(setValues).reduce((result, key): string[] => {
-      if ((value & setValues[key]) === setValues[key]) {
+      if (bn.and(bnToBn(setValues[key])).eq(bnToBn(setValues[key]))) {
         result.push(key);
       }
 
@@ -66,15 +70,15 @@ export default class CodecSet extends Set<string> implements Codec {
 
     const computed = CodecSet.encodeSet(setValues, result);
 
-    assert(value === computed, `Set: Mismatch decoding '${value}', computed as '${computed}' with ${result}`);
+    assert(bn.eq(computed), `Set: Mismatch decoding '${bn}', computed as '${computed}' with ${result}`);
 
     return result;
   }
 
-  public static encodeSet (setValues: SetValues, value: string[]): number {
-    return value.reduce((result, value): number => {
-      return result | (setValues[value] || 0);
-    }, 0);
+  public static encodeSet (setValues: SetValues, value: string[]): BN {
+    return value.reduce((result, value): BN => {
+      return result.or(bnToBn(setValues[value] || 0));
+    }, new BN(0));
   }
 
   public static with (values: SetValues): Constructor<CodecSet> {
@@ -129,7 +133,7 @@ export default class CodecSet extends Set<string> implements Codec {
   /**
    * @description The encoded value for the set members
    */
-  public get valueEncoded (): number {
+  public get valueEncoded (): BN {
     return CodecSet.encodeSet(this._setValues, this.strings);
   }
 
@@ -155,8 +159,8 @@ export default class CodecSet extends Set<string> implements Codec {
       return compareArray(this.strings.sort(), other.sort());
     } else if (other instanceof Set) {
       return this.eq([...other.values()]);
-    } else if (isNumber(other)) {
-      return this.valueEncoded === other;
+    } else if (isNumber(other) || isBn(other)) {
+      return this.valueEncoded.eq(bnToBn(other));
     }
 
     return false;
@@ -180,7 +184,7 @@ export default class CodecSet extends Set<string> implements Codec {
    * @description The encoded value for the set members
    */
   public toNumber (): number {
-    return this.valueEncoded;
+    return this.valueEncoded.toNumber();
   }
 
   /**
@@ -204,6 +208,6 @@ export default class CodecSet extends Set<string> implements Codec {
    */
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
   public toU8a (isBare?: boolean): Uint8Array {
-    return new Uint8Array([this.valueEncoded]);
+    return new Uint8Array([this.valueEncoded.toNumber()]);
   }
 }

packages/types/src/interfaceRegistry.ts

@@ -2,7 +2,7 @@
 /* eslint-disable @typescript-eslint/no-empty-interface */
 
 import { Compact, Option, Raw, Vec } from '@polkadot/types/codec';
-import { Bytes, Fixed64, H160, H256, H512, Null, StorageData, StorageHasher, StorageKey, Text, Type, U256, bool, i128, i16, i256, i32, i64, i8, u128, u16, u256, u32, u64, u8, usize } from '@polkadot/types/primitive';
+import { Bytes, Data, Fixed64, H160, H256, H512, IdentityFields, Null, StorageData, StorageHasher, StorageKey, Text, Type, U256, bool, i128, i16, i256, i32, i64, i8, u128, u16, u256, u32, u64, u8, usize } from '@polkadot/types/primitive';
 import { AccountId, AccountIdOf, AccountIndex, Address, AssetId, Balance, BalanceOf, Block, BlockNumber, Call, Consensus, ConsensusEngineId, Digest, DigestItem, DispatchClass, DispatchInfo, EcdsaSignature, Ed25519Signature, Extrinsic, ExtrinsicEra, ExtrinsicPayload, ExtrinsicPayloadUnknown, ExtrinsicPayloadV1, ExtrinsicPayloadV2, ExtrinsicPayloadV3, ExtrinsicPayloadV4, ExtrinsicUnknown, ExtrinsicV1, ExtrinsicV2, ExtrinsicV3, ExtrinsicV4, Hash, Header, ImmortalEra, Index, Justification, KeyTypeId, KeyValue, LockIdentifier, Moment, MortalEra, MultiSignature, Origin, Perbill, Permill, Phantom, PhantomData, PreRuntime, Seal, SealV0, Signature, SignedBlock, SignerPayload, Sr25519Signature, ValidatorId, Weight, WeightMultiplier } from '@polkadot/types/interfaces/runtime';
 import { InclusionHeight, Uncle, UncleEntryItem } from '@polkadot/types/interfaces/authorship';
 import { RawAuraPreDigest } from '@polkadot/types/interfaces/aura';
@@ -16,7 +16,7 @@ import { AccountInfo, Amount, AssetOf, InherentOfflineReport, LockPeriods, NewAc
 import { ApprovalFlag, SetIndex, Vote, VoteIndex, VoteThreshold, VoterInfo } from '@polkadot/types/interfaces/elections';
 import { AssetOptions, Owner, PermissionLatest, PermissionVersions, PermissionsV1 } from '@polkadot/types/interfaces/genericAsset';
 import { AuthorityIndex, AuthorityList, AuthorityWeight, NextAuthority, PendingPause, PendingResume, SetId, StoredPendingChange, StoredState } from '@polkadot/types/interfaces/grandpa';
-import { Data, IdentityFields, IdentityInfo, IdentityInfoAdditional, Judgement, RegistrarIndex, RegistrarInfo, Registration, RegistrationJudgement } from '@polkadot/types/interfaces/identity';
+import { IdentityInfo, IdentityInfoAdditional, Judgement, RegistrarIndex, RegistrarInfo, Registration, RegistrationJudgement } from '@polkadot/types/interfaces/identity';
 import { AuthIndex, AuthoritySignature, Heartbeat, OpaqueMultiaddr, OpaqueNetworkState, OpaquePeerId } from '@polkadot/types/interfaces/imOnline';
 import { Kind, OffenceDetails, Offender, OpaqueTimeSlot, ReportIdOf, Reporter } from '@polkadot/types/interfaces/offences';
 import { FullIdentification, IdentificationTuple, Keys, SessionIndex, SessionKeys1, SessionKeys2, SessionKeys3, SessionKeys4, SessionKeys5, SessionKeys6 } from '@polkadot/types/interfaces/session';
@@ -38,6 +38,9 @@ export interface InterfaceRegistry {
   Bytes: Bytes;
   'Option<Bytes>': Option<Bytes>;
   'Vec<Bytes>': Vec<Bytes>;
+  Data: Data;
+  'Option<Data>': Option<Data>;
+  'Vec<Data>': Vec<Data>;
   H160: H160;
   'Option<H160>': Option<H160>;
   'Vec<H160>': Vec<H160>;
@@ -47,6 +50,9 @@ export interface InterfaceRegistry {
   H512: H512;
   'Option<H512>': Option<H512>;
   'Vec<H512>': Vec<H512>;
+  IdentityFields: IdentityFields;
+  'Option<IdentityFields>': Option<IdentityFields>;
+  'Vec<IdentityFields>': Vec<IdentityFields>;
   i8: i8;
   'Option<i8>': Option<i8>;
   'Vec<i8>': Vec<i8>;
@@ -535,12 +541,6 @@ export interface InterfaceRegistry {
   StoredState: StoredState;
   'Option<StoredState>': Option<StoredState>;
   'Vec<StoredState>': Vec<StoredState>;
-  Data: Data;
-  'Option<Data>': Option<Data>;
-  'Vec<Data>': Vec<Data>;
-  IdentityFields: IdentityFields;
-  'Option<IdentityFields>': Option<IdentityFields>;
-  'Vec<IdentityFields>': Vec<IdentityFields>;
   IdentityInfoAdditional: IdentityInfoAdditional;
   'Option<IdentityInfoAdditional>': Option<IdentityInfoAdditional>;
   'Vec<IdentityInfoAdditional>': Vec<IdentityInfoAdditional>;

packages/types/src/interfaces/identity/definitions.ts

@@ -4,27 +4,6 @@
 
 export default {
   types: {
-    Data: {
-      _enum: {
-        None: 'Null',
-        Raw: 'Bytes',
-        BlakeTwo256: 'H256',
-        Sha256: 'H256',
-        Keccak256: 'H256',
-        ShaThree256: 'H256'
-      }
-    },
-    IdentityFields: {
-      _set: {
-        Display: 0b0000000000000000000000000000000000000000000000000000000000000001,
-        Legal: 0b0000000000000000000000000000000000000000000000000000000000000010,
-        Web: 0b0000000000000000000000000000000000000000000000000000000000000100,
-        Riot: 0b0000000000000000000000000000000000000000000000000000000000001000,
-        Email: 0b0000000000000000000000000000000000000000000000000000000000010000,
-        PgpFingerprint: 0b0000000000000000000000000000000000000000000000000000000000100000,
-        Image: 0b0000000000000000000000000000000000000000000000000000000001000000
-      }
-    },
     IdentityInfoAdditional: '(Data, Data)',
     IdentityInfo: {
       additional: 'Vec<IdentityInfoAdditional>',

packages/types/src/interfaces/identity/types.ts

@@ -2,54 +2,10 @@
 /* eslint-disable @typescript-eslint/no-empty-interface */
 
 import { ITuple } from '@polkadot/types/types';
-import { Enum, Option, Set, Struct, Vec } from '@polkadot/types/codec';
-import { Bytes, H160, H256, u32 } from '@polkadot/types/primitive';
+import { Enum, Option, Struct, Vec } from '@polkadot/types/codec';
+import { Data, H160, IdentityFields, u32 } from '@polkadot/types/primitive';
 import { AccountId, Balance } from '@polkadot/types/interfaces/runtime';
 
-/** Enum */
-export interface Data extends Enum {
-  /** 0:: None */
-  readonly isNone: boolean;
-  /** 1:: Raw(Bytes) */
-  readonly isRaw: boolean;
-  /** Bytes */
-  readonly asRaw: Bytes;
-  /** 2:: BlakeTwo256(H256) */
-  readonly isBlakeTwo256: boolean;
-  /** H256 */
-  readonly asBlakeTwo256: H256;
-  /** 3:: Sha256(H256) */
-  readonly isSha256: boolean;
-  /** H256 */
-  readonly asSha256: H256;
-  /** 4:: Keccak256(H256) */
-  readonly isKeccak256: boolean;
-  /** H256 */
-  readonly asKeccak256: H256;
-  /** 5:: ShaThree256(H256) */
-  readonly isShaThree256: boolean;
-  /** H256 */
-  readonly asShaThree256: H256;
-}
-
-/** Set */
-export interface IdentityFields extends Set {
-  /** boolean */
-  readonly isDisplay: boolean;
-  /** boolean */
-  readonly isLegal: boolean;
-  /** boolean */
-  readonly isWeb: boolean;
-  /** boolean */
-  readonly isRiot: boolean;
-  /** boolean */
-  readonly isEmail: boolean;
-  /** boolean */
-  readonly isPgpFingerprint: boolean;
-  /** boolean */
-  readonly isImage: boolean;
-}
-
 /** Struct */
 export interface IdentityInfo extends Struct {
   /** Vec<IdentityInfoAdditional> */

packages/types/src/primitive/Data.spec.ts

@@ -0,0 +1,28 @@
+// Copyright 2017-2019 @polkadot/types authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { TypeRegistry } from '../codec/create';
+import Data from './Data';
+
+const registry = new TypeRegistry();
+
+describe('Data', (): void => {
+  it('encodes a normal None', (): void => {
+    expect(
+      new Data(registry).toHex()
+    ).toEqual('0x00');
+  });
+
+  it('encodes a hashed value correctly', (): void => {
+    expect(
+      new Data(registry, { Keccak256: '0x0102030405060708091011121314151617181920212223242526272829303132' }).toHex()
+    ).toEqual('0x240102030405060708091011121314151617181920212223242526272829303132');
+  });
+
+  it('encodes a Raw value correctly', (): void => {
+    expect(
+      new Data(registry, { Raw: '0x0102030405060708' }).toHex()
+    ).toEqual('0x090102030405060708');
+  });
+});

packages/types/src/primitive/Data.ts

@@ -0,0 +1,90 @@
+// Copyright 2017-2019 @polkadot/types authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { Registry } from '../types';
+
+import { isString, isU8a, u8aToU8a } from '@polkadot/util';
+
+import Enum from '../codec/Enum';
+import { createType } from '../codec/create/createType';
+
+function decodeDataU8a (registry: Registry, value: Uint8Array): [any, number | undefined] {
+  if (!value.length) {
+    return [undefined, undefined];
+  }
+
+  const indicator = value[0];
+
+  if (indicator === 0) {
+    return [null, 0];
+  } else if (indicator >= 1 && indicator <= 33) {
+    const length = indicator - 1;
+    const data = value.subarray(1, length + 1);
+
+    // in this case, we are passing a Raw back (since we have no length)
+    return [createType(registry, 'Raw', data), 1];
+  } else if (indicator >= 34 && indicator <= 37) {
+    return [value.subarray(1, 32 + 1), indicator - 32]; // 34 becomes 2
+  }
+
+  throw new Error(`Unable to decode Data, invalid indicator byte ${indicator}`);
+}
+
+function decodeData (registry: Registry, value?: Record<string, any> | Uint8Array | Enum | string): [any, number | undefined] {
+  if (!value) {
+    return [undefined, undefined];
+  } else if (isString(value)) {
+    return decodeDataU8a(registry, u8aToU8a(value));
+  } else if (isU8a(value)) {
+    return decodeDataU8a(registry, value);
+  }
+
+  // assume we have an Enum or an  object input, handle this via the normal Enum decoding
+  return [value, undefined];
+}
+
+/**
+ * @name Data
+ * @description
+ * A [[Data]] container with node, raw or hashed data
+ */
+export default class Data extends Enum {
+  constructor (registry: Registry, value?: Record<string, any> | Uint8Array | Enum | string) {
+    super(registry, {
+      None: 'Null', // 0
+      Raw: 'Bytes', // 1
+      BlakeTwo256: 'H256', // 2
+      Sha256: 'H256', // 3
+      Keccak256: 'H256', // 4
+      ShaThree256: 'H256' // 5
+    }, ...decodeData(registry, value));
+  }
+
+  /**
+   * @description Encodes the value as a Uint8Array as per the SCALE specifications
+   */
+  public toU8a (): Uint8Array {
+    if (this.index === 0) {
+      return new Uint8Array(1);
+    } else if (this.index === 1) {
+      // don't add the length, just the data
+      const data = this.raw.toU8a(true);
+      const length = Math.min(data.length, 32);
+      const u8a = new Uint8Array(length + 1);
+
+      u8a.set([data.length + 1], 0);
+      u8a.set(data.subarray(0, length), 1);
+
+      return u8a;
+    }
+
+    // otherwise we simply have a hash
+    const u8a = new Uint8Array(33);
+
+    u8a.set([this.index + 32], 0);
+    u8a.set(this.raw.toU8a(), 1);
+
+    return u8a;
+  }
+}

packages/types/src/primitive/Generic/index.ts

@@ -8,7 +8,7 @@ export { default as GenericAddress } from './Address';
 export { default as GenericBlock } from './Block';
 export { default as GenericCall } from './Call';
 export { default as GenericConsensusEngineId } from './ConsensusEngineId';
-export { default as GenericEvent } from './Event';
 export { default as GenericDigest, DigestItem as GenericDigestItem } from './Digest';
+export { default as GenericEvent } from './Event';
 export { default as GenericOrigin } from './Origin';
 export { default as GenericVote } from './Vote';

packages/types/src/primitive/IdentityFields.spec.ts

@@ -0,0 +1,28 @@
+// Copyright 2017-2019 @polkadot/types authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { TypeRegistry } from '../codec';
+import IdentityFields from './IdentityFields';
+
+describe('IdentityFields', (): void => {
+  const registry = new TypeRegistry();
+
+  it('has a custom u64 (64-bit) encodedLength', (): void => {
+    expect(
+      new IdentityFields(registry).encodedLength
+    ).toEqual(8);
+  });
+
+  it('encodes a sample', (): void => {
+    expect(
+      new IdentityFields(registry, ['Display', 'Legal']).valueEncoded.eqn(1 + 2)
+    ).toBe(true);
+  });
+
+  it('encodes to a valid hex value', (): void => {
+    expect(
+      new IdentityFields(registry, ['Display', 'Legal']).toHex()
+    ).toEqual('0x0300000000000000');
+  });
+});

packages/types/src/primitive/IdentityFields.ts

@@ -0,0 +1,49 @@
+// Copyright 2017-2019 @polkadot/types authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { Registry } from '../types';
+
+import BN from 'bn.js';
+import { isString, isU8a, u8aToU8a } from '@polkadot/util';
+
+import Set from '../codec/Set';
+import { createType } from '../codec/create/createType';
+
+function decodeFields (registry: Registry, value?: any): any {
+  if (isString(value)) {
+    return decodeFields(registry, u8aToU8a(value.toString()));
+  } else if (isU8a(value)) {
+    return createType(registry, 'u64', value);
+  }
+
+  return value;
+}
+
+export default class IdentityFields extends Set {
+  constructor (registry: Registry, value?: any) {
+    super(registry, {
+      Display: new BN(0b0000000000000000000000000000000000000000000000000000000000000001),
+      Legal: new BN(0b0000000000000000000000000000000000000000000000000000000000000010),
+      Web: new BN(0b0000000000000000000000000000000000000000000000000000000000000100),
+      Riot: new BN(0b0000000000000000000000000000000000000000000000000000000000001000),
+      Email: new BN(0b0000000000000000000000000000000000000000000000000000000000010000),
+      PgpFingerprint: new BN(0b0000000000000000000000000000000000000000000000000000000000100000),
+      Image: new BN(0b0000000000000000000000000000000000000000000000000000000001000000)
+    }, decodeFields(registry, value));
+  }
+
+  /**
+   * @description The length of the value when encoded as a Uint8Array
+   */
+  public get encodedLength (): number {
+    return 8;
+  }
+
+  /**
+   * @description Encodes the value as a Uint8Array as per the SCALE specifications
+   */
+  public toU8a (): Uint8Array {
+    return createType(this.registry, 'u64', this.valueEncoded).toU8a();
+  }
+}

packages/types/src/primitive/index.ts

@@ -10,9 +10,11 @@
  */
 export { default as bool } from './Bool';
 export { default as Bytes } from './Bytes';
+export { default as Data } from './Data';
 export { default as H160 } from './H160';
 export { default as H256 } from './H256';
 export { default as H512 } from './H512';
+export { default as IdentityFields } from './IdentityFields';
 export { default as i8 } from './I8';
 export { default as i16 } from './I16';
 export { default as i32 } from './I32';