refactor: Metadata overhaul (#913)

* Replace part1

* Replace part2

* Replace part3

* Replace part5

* Replace part6

* Fix small bugs

* Fix double map toString

* Small fixes

* Update to v4

* Fix docs script

* Grumbles

* Alphabetical

* Reads better

* comment

* module to modul

packages/type-storage/src/fromMetadata/createFunction.spec.ts

@@ -26,8 +26,8 @@ describe('createFunction', () => {
         'authorityCount',
         { type: {} } as any,
         {
-          isUnhashed: true,
-          key: ':auth:len'
+          key: ':auth:len',
+          skipHashing: true
         }
       ).method
     ).toEqual('authorityCount');
@@ -42,8 +42,8 @@ describe('createFunction', () => {
         'authorityCount',
         { type: {} } as any,
         {
-          isUnhashed: true,
-          key
+          key,
+          skipHashing: true
         }
       )()
     ).toEqual(

packages/type-storage/src/fromMetadata/createFunction.ts

@@ -10,8 +10,8 @@ import { assert, isNull, isUndefined, stringLowerFirst, stringToU8a, u8aConcat }
 import getHasher from './getHasher';
 
 export interface CreateItemOptions {
-  isUnhashed?: boolean;
   key?: string;
+  skipHashing?: boolean; // We don't hash the keys defined in ./substrate.ts
 }
 
 /**
@@ -52,14 +52,13 @@ export default function createFunction (section: Text | string, method: Text | s
 
     // StorageKey is a Bytes, so is length-prefixed
     return Compact.addLengthPrefix(
-      options.isUnhashed
+      options.skipHashing
         ? key
         : hasher(key)
     );
   };
 
   if (meta.type.isMap && meta.type.asMap.isLinked) {
-    // TODO: there needs some better way to do this
     const keyHash = new U8a(hasher(`head of ${stringKey}`));
     const keyFn: any = () => keyHash;
     keyFn.meta = new StorageFunctionMetadata({

packages/type-storage/src/fromMetadata/fromMetadata.spec.ts

@@ -7,26 +7,26 @@ import { Metadata } from '@plugnet/types';
 import json from '@plugnet/types/Metadata/v4/static';
 import { u8aToHex } from '@plugnet/util';
 
-import fromv4 from '.';
+import fromMetadata from '.';
 
 const keyring = testingPairs({ type: 'ed25519' });
 // Use the pre-generated metadata
-const metadata = new Metadata(json).asV4;
-const newStorage = fromv4(metadata);
+const metadata = new Metadata(json);
+const storage = fromMetadata(metadata);
 
-describe('fromv4', () => {
+describe('fromMetadata', () => {
   it('should throw if the storage function expects an argument', () => {
-    expect(() => newStorage.balances.freeBalance()).toThrowError(/expects one argument/);
+    expect(() => storage.balances.freeBalance()).toThrowError(/expects one argument/);
   });
 
   it('should return a value if the storage function does not expect an argument', () => {
-    expect(() => newStorage.timestamp.now()).not.toThrow();
+    expect(() => storage.timestamp.now()).not.toThrow();
   });
 
   it('should return the correct length-prefixed storage key', () => {
     expect(
       u8aToHex(
-        newStorage.balances.freeBalance(keyring.alice.address())
+        storage.balances.freeBalance(keyring.alice.address())
       )
     ).toEqual(
       '0x807f864e18e3dd8b58386310d2fe0919eef27c6e558564b7f67f22d99d20f587bb'

packages/type-storage/src/fromMetadata/fromMetadata.ts

@@ -1,27 +0,0 @@
-// Copyright 2017-2019 @polkadot/storage authors & contributors
-// This software may be modified and distributed under the terms
-// of the Apache-2.0 license. See the LICENSE file for details.
-
-import { Storage } from '../types';
-
-import Metadata from '@plugnet/types/Metadata';
-
-import fromV0 from './v0';
-import fromV4 from './v4';
-
-/**
- * Extend a storage object with the storage modules & module functions present
- * in the metadata.
- *
- * @param storage - A storage object to be extended.
- * @param metadata - The metadata to extend the storage object against.
- */
-export default function fromMetadata (metadata: Metadata): Storage {
-  // For v0-v3 we convert to v0, and use the fromV0 function to parse
-  if ([0, 1, 2, 3].includes(metadata.version)) {
-    return fromV0(metadata.asV0);
-  }
-
-  // For v4, we use fromV4 function
-  return fromV4(metadata.asV4);
-}

packages/type-storage/src/fromMetadata/index.ts

@@ -2,6 +2,36 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
-import fromMetadata from './fromMetadata';
+import { ModuleStorage, Storage } from '../types';
 
-export default fromMetadata;
+import Metadata from '@polkadot/types/Metadata';
+import { stringLowerFirst } from '@polkadot/util';
+
+import createFunction from './createFunction';
+import { storage } from './storage';
+
+/**
+ * Extend a storage object with the storage modules & module functions present
+ * in the metadata.
+ *
+ * @param storage - A storage object to be extended.
+ * @param metadata - The metadata to extend the storage object against.
+ */
+export default function fromMetadata (metadata: Metadata): Storage {
+  return metadata.asV4.modules.reduce((result, moduleMetadata) => {
+    if (moduleMetadata.storage.isNone) {
+      return result;
+    }
+
+    const prefix = moduleMetadata.prefix;
+
+    // For access, we change the index names, i.e. Balances.FreeBalance -> balances.freeBalance
+    result[stringLowerFirst(prefix.toString())] = moduleMetadata.storage.unwrap().reduce((newModule, storageFnMeta) => {
+      newModule[stringLowerFirst(storageFnMeta.name.toString())] = createFunction(prefix, storageFnMeta.name, storageFnMeta);
+
+      return newModule;
+    }, {} as ModuleStorage);
+
+    return result;
+  }, { ...storage });
+}

packages/type-storage/src/fromMetadata/storage.ts

@@ -2,7 +2,7 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
-import { Storage } from '../../types';
+import { Storage } from '../types';
 import * as substrate from './substrate';
 
 export const storage: Storage = {

packages/type-storage/src/fromMetadata/substrate.ts

@@ -25,8 +25,8 @@ const createRuntimeFunction = (method: string, key: string, { documentation, typ
       toJSON: (): any => key
     } as StorageFunctionMetadata,
     {
-      isUnhashed: true,
-      key
+      key,
+      skipHashing: true
     }
   );
 

packages/type-storage/src/fromMetadata/v0/createFunction.spec.ts

@@ -1,56 +0,0 @@
-// Copyright 2017-2019 @polkadot/storage authors & contributors
-// This software may be modified and distributed under the terms
-// of the Apache-2.0 license. See the LICENSE file for details.
-
-import { stringToU8a, u8aConcat } from '@plugnet/util';
-
-import createFunction from './createFunction';
-
-describe('createFunction', () => {
-  it('should create timestamp.now correctly', () => {
-    expect(
-      createFunction(
-        'Timestamp',
-        'Now',
-        { type: {} } as any
-      )()
-    ).toEqual(
-      Uint8Array.from([64, 14, 73, 68, 207, 217, 141, 111, 76, 195, 116, 209, 111, 90, 78, 63, 156]) // Length-prefixed
-    );
-  });
-
-  it('allows overrides on key (keeping name)', () => {
-    expect(
-      createFunction(
-        'Substrate',
-        'authorityCount',
-        { type: {} } as any,
-        {
-          isUnhashed: true,
-          key: ':auth:len'
-        }
-      ).method
-    ).toEqual('authorityCount');
-  });
-
-  it('allows overrides on key (unhashed)', () => {
-    const key = ':auth:len';
-
-    expect(
-      createFunction(
-        'Substrate',
-        'authorityCount',
-        { type: {} } as any,
-        {
-          isUnhashed: true,
-          key
-        }
-      )()
-    ).toEqual(
-      u8aConcat(
-        Uint8Array.from([key.length << 2]),
-        stringToU8a(':auth:len')
-      )
-    );
-  });
-});

packages/type-storage/src/fromMetadata/v0/createFunction.ts

@@ -1,74 +0,0 @@
-// Copyright 2017-2019 @polkadot/storage authors & contributors
-// This software may be modified and distributed under the terms
-// of the Apache-2.0 license. See the LICENSE file for details.
-
-import { PlainType, StorageFunctionMetadata, StorageFunctionModifier, StorageFunctionType } from '@plugnet/types/Metadata/v0/Storage';
-import { StorageFunction } from '@plugnet/types/primitive/StorageKey';
-import { Compact, Text, createType, StorageKey, Bytes, U8a } from '@plugnet/types';
-import { assert, isNull, isUndefined, stringLowerFirst, stringToU8a, u8aConcat } from '@plugnet/util';
-import { xxhashAsU8a } from '@plugnet/util-crypto';
-
-export interface CreateItemOptions {
-  isUnhashed?: boolean;
-  key?: string;
-}
-
-/**
- * From the schema of a function in the module's storage, generate the function
- * that will return the correct storage key.
- *
- * @param schema - The function's definition schema to create the function from.
- * The schema is taken from state_getMetadata.
- * @param options - Additional options when creating the function. These options
- * are not known at runtime (from state_getMetadata), they need to be supplied
- * by us manually at compile time.
- */
-export default function createFunction (section: Text | string, method: Text | string, meta: StorageFunctionMetadata, options: CreateItemOptions = {}): StorageFunction {
-  const stringKey = options.key
-    ? options.key
-    : `${section.toString()} ${method.toString()}`;
-  const rawKey = stringToU8a(stringKey);
-
-  // Can only have zero or one argument:
-  // - storage.balances.freeBalance(address)
-  // - storage.timestamp.blockPeriod()
-  const storageFn = (arg?: any): Uint8Array => {
-    let key = rawKey;
-    if (meta.type.isMap) {
-      assert(!isUndefined(arg) && !isNull(arg), `${meta.name} expects one argument`);
-
-      const type = meta.type.asMap.key.toString();
-      const param = createType(type, arg).toU8a(false);
-
-      key = u8aConcat(key, param);
-    }
-
-    // StorageKey is a Bytes, so is length-prefixed
-    return Compact.addLengthPrefix(
-      options.isUnhashed
-        ? key
-        : xxhashAsU8a(key, 128)
-    );
-  };
-
-  if (meta.type.isMap && meta.type.asMap.isLinked) {
-    // TODO: there needs some better way to do this
-    const keyHash = new U8a(xxhashAsU8a(`head of ${stringKey}`, 128));
-    const keyFn: any = () => keyHash;
-    keyFn.meta = new StorageFunctionMetadata({
-      name: meta.name,
-      modifier: new StorageFunctionModifier('Required'),
-      type: new StorageFunctionType(new PlainType(meta.type.asMap.key), 0),
-      fallback: new Bytes(),
-      documentation: meta.documentation
-    });
-    storageFn.headKey = new StorageKey(keyFn);
-  }
-
-  storageFn.meta = meta;
-  storageFn.method = stringLowerFirst(method.toString());
-  storageFn.section = stringLowerFirst(section.toString());
-  storageFn.toJSON = (): any => meta.toJSON();
-
-  return storageFn as StorageFunction;
-}

packages/type-storage/src/fromMetadata/v0/index.ts

@@ -1,37 +0,0 @@
-// Copyright 2017-2019 @polkadot/storage authors & contributors
-// This software may be modified and distributed under the terms
-// of the Apache-2.0 license. See the LICENSE file for details.
-
-import { ModuleStorage, Storage } from '../../types';
-
-import MetadataV0 from '@plugnet/types/Metadata/v0';
-import { stringLowerFirst } from '@plugnet/util';
-
-import createFunction from './createFunction';
-import { storage } from './storage';
-
-/**
- * Extend a storage object with the storage modules & module functions present
- * in the metadata.
- *
- * @param storage - A storage object to be extended.
- * @param metadata - The metadata to extend the storage object against.
- */
-export default function fromV0 (metadata: MetadataV0): Storage {
-  return metadata.modules.reduce((result, moduleMetadata) => {
-    if (moduleMetadata.storage.isNone) {
-      return result;
-    }
-
-    const prefix = moduleMetadata.storage.unwrap().prefix;
-
-    // For access, we change the index names, i.e. Balances.FreeBalance -> balances.freeBalance
-    result[stringLowerFirst(prefix.toString())] = moduleMetadata.storage.unwrap().functions.reduce((newModule, func) => {
-      newModule[stringLowerFirst(func.name.toString())] = createFunction(prefix, func.name, func);
-
-      return newModule;
-    }, {} as ModuleStorage);
-
-    return result;
-  }, { ...storage });
-}

packages/type-storage/src/fromMetadata/v0/substrate.ts

@@ -1,61 +0,0 @@
-// Copyright 2017-2019 @polkadot/storage authors & contributors
-// This software may be modified and distributed under the terms
-// of the Apache-2.0 license. See the LICENSE file for details.
-
-import { StorageFunctionMetadata, StorageFunctionModifier, StorageFunctionType } from '@plugnet/types/Metadata/v0/Storage';
-import { StorageFunction } from '@plugnet/types/primitive/StorageKey';
-import { Text, Vector } from '@plugnet/types';
-
-import createFunction from './createFunction';
-
-interface SubstrateMetadata {
-  documentation: string;
-  type: string;
-}
-
-// Small helper function to factorize code on this page.
-const createRuntimeFunction = (method: string, key: string, { documentation, type }: SubstrateMetadata): StorageFunction =>
-  createFunction(
-    new Text('Substrate'),
-    new Text(method),
-    {
-      documentation: new Vector(Text, [documentation]),
-      modifier: new StorageFunctionModifier(1), // default
-      type: new StorageFunctionType(type, 0),
-      toJSON: (): any => key
-    } as StorageFunctionMetadata,
-    {
-      isUnhashed: true,
-      key
-    }
-  );
-
-export const code = createRuntimeFunction('code', ':code', {
-  documentation: 'Wasm code of the runtime.',
-  type: 'Bytes'
-});
-
-export const heapPages = createRuntimeFunction('heapPages', ':heappages', {
-  documentation: 'Number of wasm linear memory pages required for execution of the runtime.',
-  type: 'u64'
-});
-
-export const authorityCount = createRuntimeFunction('authorityCount', ':auth:len', {
-  documentation: 'Number of authorities.',
-  type: 'u32'
-});
-
-export const authorityPrefix = createRuntimeFunction('authorityPrefix', ':auth:', {
-  documentation: 'Prefix under which authorities are stored.',
-  type: 'u32'
-});
-
-export const extrinsicIndex = createRuntimeFunction('extrinsicIndex', ':extrinsic_index', {
-  documentation: 'Current extrinsic index (u32) is stored under this key.',
-  type: 'u32'
-});
-
-export const changesTrieConfig = createRuntimeFunction('changesTrieConfig', ':changes_trie', {
-  documentation: 'Changes trie configuration is stored under this key.',
-  type: 'u32'
-});

packages/type-storage/src/fromMetadata/v0/v0.spec.ts

@@ -1,33 +0,0 @@
-// Copyright 2017-2019 @polkadot/storage authors & contributors
-// This software may be modified and distributed under the terms
-// of the Apache-2.0 license. See the LICENSE file for details.
-
-import { u8aToHex } from '@plugnet/util';
-import { Metadata } from '@plugnet/types';
-import json from '@plugnet/types/Metadata/v0/static';
-
-import fromv0 from '.';
-
-// Use the pre-generated metadata
-const metadata = new Metadata(json).asV0;
-const newStorage = fromv0(metadata);
-
-describe('fromv0', () => {
-  it('should throw if the storage function expects an argument', () => {
-    expect(() => newStorage.balances.freeBalance()).toThrowError(/expects one argument/);
-  });
-
-  it('should return a value if the storage function does not expect an argument', () => {
-    expect(() => newStorage.timestamp.now()).not.toThrow();
-  });
-
-  it('should return the correct length-prefixed storage key', () => {
-    expect(
-      u8aToHex(
-        newStorage.balances.freeBalance('5DkQbYAExs3M2sZgT1Ec3mKfZnAQCL4Dt9beTCknkCUn5jzo')
-      )
-    ).toEqual(
-      '0x404af2c53fce3ec33c6ccccf22e926f1a7'
-    );
-  });
-});

packages/type-storage/src/fromMetadata/v4/index.ts

@@ -1,37 +0,0 @@
-// Copyright 2017-2019 @polkadot/storage authors & contributors
-// This software may be modified and distributed under the terms
-// of the Apache-2.0 license. See the LICENSE file for details.
-
-import { ModuleStorage, Storage } from '../../types';
-
-import MetadataV4 from '@plugnet/types/Metadata/v4';
-import { stringLowerFirst } from '@plugnet/util';
-
-import createFunction from './createFunction';
-import { storage } from './storage';
-
-/**
- * Extend a storage object with the storage modules & module functions present
- * in the metadata.
- *
- * @param storage - A storage object to be extended.
- * @param metadata - The metadata to extend the storage object against.
- */
-export default function fromV4 (metadata: MetadataV4): Storage {
-  return metadata.modules.reduce((result, moduleMetadata) => {
-    if (moduleMetadata.storage.isNone) {
-      return result;
-    }
-
-    const prefix = moduleMetadata.prefix;
-
-    // For access, we change the index names, i.e. Balances.FreeBalance -> balances.freeBalance
-    result[stringLowerFirst(prefix.toString())] = moduleMetadata.storage.unwrap().reduce((newModule, storageFnMeta) => {
-      newModule[stringLowerFirst(storageFnMeta.name.toString())] = createFunction(prefix, storageFnMeta.name, storageFnMeta);
-
-      return newModule;
-    }, {} as ModuleStorage);
-
-    return result;
-  }, { ...storage });
-}

packages/type-storage/src/fromMetadata/v4/storage.spec.ts

@@ -1,17 +0,0 @@
-// Copyright 2017-2019 @polkadot/storage authors & contributors
-// This software may be modified and distributed under the terms
-// of the Apache-2.0 license. See the LICENSE file for details.
-
-import { storage } from './storage';
-
-describe('storage', () => {
-  it('should return well known keys', () => {
-    expect(typeof storage.substrate).toBe('object');
-    expect(storage.substrate.authorityCount).toBeTruthy();
-    expect(storage.substrate.authorityPrefix).toBeTruthy();
-    expect(storage.substrate.changesTrieConfig).toBeTruthy();
-    expect(storage.substrate.code).toBeTruthy();
-    expect(storage.substrate.extrinsicIndex).toBeTruthy();
-    expect(storage.substrate.heapPages).toBeTruthy();
-  });
-});

packages/type-storage/src/fromMetadata/v4/storage.ts

@@ -1,10 +0,0 @@
-// Copyright 2017-2019 @polkadot/storage authors & contributors
-// This software may be modified and distributed under the terms
-// of the Apache-2.0 license. See the LICENSE file for details.
-
-import { Storage } from '../../types';
-import * as substrate from './substrate';
-
-export const storage: Storage = {
-  substrate // Prefill storage with well known keys, as not returned by state_getMetadata
-};

packages/type-storage/src/fromMetadata/v4/substrate.spec.ts

@@ -1,33 +0,0 @@
-// Copyright 2017-2019 @polkadot/storage authors & contributors
-// This software may be modified and distributed under the terms
-// of the Apache-2.0 license. See the LICENSE file for details.
-
-import { authorityCount, authorityPrefix, changesTrieConfig, code, extrinsicIndex, heapPages } from './substrate';
-
-describe('substrate', () => {
-  it('authorityCount should return the correct storage key', () => {
-    expect(authorityCount()).toEqual(Uint8Array.from([36, 58, 97, 117, 116, 104, 58, 108, 101, 110])); // Length-prefixed
-  });
-
-  it('authorityPrefix should return the correct storage key', () => {
-    expect(authorityPrefix()).toEqual(Uint8Array.from([24, 58, 97, 117, 116, 104, 58])); // Length-prefixed
-  });
-
-  it('changesTrieConfig should return the correct storage key', () => {
-    expect(changesTrieConfig()).toEqual(Uint8Array.from([52, 58, 99, 104, 97, 110, 103, 101, 115, 95, 116, 114, 105, 101])); // Length-prefixed
-  });
-
-  it('code should return the correct storage key', () => {
-    expect(code()).toEqual(Uint8Array.from([20, 58, 99, 111, 100, 101])); // Length-prefixed
-  });
-
-  it('extrinsicIndex should return the correct storage key', () => {
-    expect(extrinsicIndex()).toEqual(
-      Uint8Array.from([64, 58, 101, 120, 116, 114, 105, 110, 115, 105, 99, 95, 105, 110, 100, 101, 120]) // Length-prefixed
-    );
-  });
-
-  it('heapPages should return the correct storage key', () => {
-    expect(heapPages()).toEqual(Uint8Array.from([40, 58, 104, 101, 97, 112, 112, 97, 103, 101, 115])); // Length-prefixed
-  });
-});

packages/type-storage/src/index.ts

@@ -2,6 +2,6 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
-import { storage } from './fromMetadata/v0/storage';
+import { storage } from './fromMetadata/storage';
 
 export default storage;

packages/type-storage/src/static.ts

@@ -3,7 +3,7 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 import { Metadata } from '@plugnet/types';
-import metadataRpc from '@plugnet/types/Metadata/v0/static';
+import metadataRpc from '@plugnet/types/Metadata/v4/static';
 
 import fromMetadata from './fromMetadata';