Parsing for metadata v14 (#3759)

* Adjust

* Better, need a BoundedVec solution...

* Composites

* Storage optional types

* Better... not perfect

* v14 tests passing

* Always perform asLatest

* Keep portable registry seperate

* Cleanup derived types

* Cleanups

* Adjust

* Cleanups

* Pass wraps

* Rename types (PortableRegistry) -> lookup

* Back on-track...

* SiLookup strings

* Allow building

* unsafe create on registry

* createTypeUnsafe typing

* Adjust

* Fix bool check

* Remove WrappedConstructor

* Additional

* Additional cleanups

* On-demand even type creation

* Rework

* Adjust

* Revert createClass to master (with adjustments)

* Running through v14

* Registry from master

* Run against v14

* Adjust

* Adjust

* Adjust generation (non-complete)

* Adjust

* Definition from metadata (round 1)

* Some linting fixes

* v13

* Adjust

* Dedupe

* Some linting

* Additional linting (md generation, not all)

* md additional

* calls update

* Additional linting

* Cleanups

* Remove extra arg

* Remove unneeded assert

* Revert < 12 index adjustments

* Cleanups

* v13 compat types

* Some tests

* Registry revert

* Additional test fixes

* Adjust sorting

* Adjust tests with lookups

* CodeClimate cleanups

* Additional tests

* Compat with prev-generation args

* Error & events compat args

* Fix Error/Event metadata creation

* Fix lookup circular

* Registry types

* Adjust

* Adjust lookup generation

* Short-circuit full type (with lookupIndex)

* Defaults for type expansion

* Tuple with extra spaces removed

* Fill in generated types

* Checks

* Adjust generation

* Adjust tests

* Adjust

* Adjust type encoding

* Adjust

* Use registry for name generation

* Update v13 lookup generation

* Adjust looup with typeDef (optional)

* Format lookup definitions

* Start with TS interface generation

* Adjust

* Adjust

* Adjust

* Don't short-cisrcuit type evaluation

* Adjust

* Correct plain imports

* Adjust

* Don't remove already-set lookupName

* Bytes

* Cleanups

* Adjust with unused

* Embed non-named structs

* Enum without name throws error

* Case-sensitivity on path duplication

* s/lookupNameOrig/lookupNameRoot/

* Don't export Lookup* type interfaces

* Shortcut lookup

* Adjust

* Revert api/Decorate

* Adjust import name generation

* Don't clobber root lookups

* Root exports

* Remove duplicated fields creation

* Adjust less-important name parts

* Cleanups & imports

* Adjust with correct imports

* Storage optionality

* Rebuild, fix (some) tests

* Extract identity BitFlags

* Additional cleanups

* Adjust

* No IdentityFields override

* v13 for extrinsic test

* Dedupe getUniqTypes

* Simplify?

* Adjust

* Update sorting

* Adjust expansion

* Dedupe

* Strip generics, sub-part * matching

* Name matching

* Align

* Adjust set matching

* hex/json split

* naming

* v13 for extrinsic injection test

* Adjust test signature

* Dedupe tests

* Adjust

* Update import

* augment @polkadot/types/lookup

* Sort import names

* Handle BitSequence in PortableRegistry

* Use Kusama-dev v14 metadata

* Merge bitSequence support

* Polkadot/Substrate split

* Template formatting

* Hobbling along

* Adjust

* Recode subs

* Fix circular in dual fashion

* Adjust

* Also generate Substrate

* Exclude Substrate types from Polkadot

* Remove duplicate phantoms

* Cleanup lookup typegen

* Additional

* Extract names with params (as applicable)

* Ensure path is always available

* renames

* Metadata v14 only has combined Map

* v14 Map conversion de-duped

* Substrate Map conversion

* v13 tests

* Apply suggestions from code review

* Housekeeping

* v14 bump

* NMap -> Map (& others removed)

* Adjust generation

* Decorate linting

* Tuple & non-tuple types

* hashers check

* Condense

* Additional asTuple check

* Consistent pre-v14 Map handling

* Fix metadata tests

* Fix (some) tests

* Fix test expectations

* Fail create early

* Additional fixes

* Additional

* Movement

* Remove CodecArg

* Dump it

* Apply param expansion

* Remove only

* Allow no-args to StorageKey

* linting

* multi fixes

* Map adjustments

* Rebuild

* Rebuild

* Rebuild

* Latest

* Bump metadata

* Rewrite

* Fix lookup formatting

* Rebuild after merge

* Latest substrate v14

* Rebuild

* Rebuild (+ no enum trailing ,)

* trigger

* Fix JSON checks

* Remove unneeded check

* Bump Kusama/Polkadot

* Dedupe testUtil

* Metadata v14

packages/typegen/src/extractChain.ts

@@ -12,7 +12,7 @@ import { ApiPromise, WsProvider } from '@polkadot/api';
 /** @internal */
 async function run (ws: string): Promise<void> {
   const provider = new WsProvider(ws);
-  const api = await ApiPromise.create({ provider });
+  const api = await ApiPromise.create({ provider, throwOnConnect: true });
   const [chain, props] = await Promise.all([
     api.rpc.system.chain(),
     api.rpc.system.properties()

packages/typegen/src/generate/consts.ts

@@ -6,6 +6,7 @@ import type { ExtraTypes } from './types';
 
 import Handlebars from 'handlebars';
 
+import lookupDefinitions from '@polkadot/types/augment/lookup/definitions';
 import * as defaultDefs from '@polkadot/types/interfaces/definitions';
 import { stringCamelCase } from '@polkadot/util';
 
@@ -17,14 +18,19 @@ const generateForMetaTemplate = Handlebars.compile(template);
 /** @internal */
 function generateForMeta (meta: Metadata, dest: string, extraTypes: ExtraTypes, isStrict: boolean): void {
   writeFile(dest, (): string => {
-    const allTypes = { '@polkadot/types/interfaces': defaultDefs, ...extraTypes };
+    const allTypes = {
+      '@polkadot/types/augment': { lookup: lookupDefinitions },
+      '@polkadot/types/interfaces': defaultDefs,
+      ...extraTypes
+    };
     const imports = createImports(allTypes);
     const allDefs = Object.entries(allTypes).reduce((defs, [path, obj]) => {
       return Object.entries(obj).reduce((defs, [key, value]) => ({ ...defs, [`${path}/${key}`]: value }), defs);
     }, {});
+    const { lookup, pallets, registry } = meta.asLatest;
 
-    const modules = meta.asLatest.modules
-      .filter((mod) => mod.constants.length > 0)
+    const modules = pallets
+      .filter(({ constants }) => constants.length > 0)
       .map(({ constants, name }) => {
         if (!isStrict) {
           setImports(allDefs, imports, ['Codec']);
@@ -32,7 +38,8 @@ function generateForMeta (meta: Metadata, dest: string, extraTypes: ExtraTypes,
 
         const items = constants
           .map(({ docs, name, type }) => {
-            const returnType = formatType(meta.registry, allDefs, type.toString(), imports);
+            const typeDef = lookup.getTypeDef(type);
+            const returnType = typeDef.lookupName || formatType(registry, allDefs, typeDef, imports);
 
             setImports(allDefs, imports, [returnType]);
 
@@ -57,8 +64,8 @@ function generateForMeta (meta: Metadata, dest: string, extraTypes: ExtraTypes,
       isStrict,
       modules,
       types: [
-        ...Object.keys(imports.localTypes).sort().map((packagePath): { file: string; types: string[] } => ({
-          file: packagePath,
+        ...Object.keys(imports.localTypes).sort().map<{ file: string; types: string[] }>((packagePath) => ({
+          file: packagePath.replace('@polkadot/types/augment', '@polkadot/types'),
           types: Object.keys(imports.localTypes[packagePath])
         })),
         {

packages/typegen/src/generate/errors.ts

@@ -17,11 +17,11 @@ const generateForMetaTemplate = Handlebars.compile(template);
 function generateForMeta (meta: Metadata, dest: string, isStrict: boolean): void {
   writeFile(dest, (): string => {
     const imports = createImports({});
-
-    const modules = meta.asLatest.modules
-      .filter((mod) => mod.errors.length > 0)
+    const { lookup, pallets } = meta.asLatest;
+    const modules = pallets
+      .filter(({ errors }) => errors.isSome)
       .map(({ errors, name }) => ({
-        items: errors
+        items: lookup.getSiType(errors.unwrap().type).def.asVariant.variants
           .map(({ docs, name }) => ({
             docs,
             name: name.toString()

packages/typegen/src/generate/events.ts

@@ -6,6 +6,7 @@ import type { ExtraTypes } from './types';
 
 import Handlebars from 'handlebars';
 
+import lookupDefinitions from '@polkadot/types/augment/lookup/definitions';
 import * as defaultDefs from '@polkadot/types/interfaces/definitions';
 import { stringCamelCase } from '@polkadot/util';
 
@@ -17,29 +18,34 @@ const generateForMetaTemplate = Handlebars.compile(template);
 /** @internal */
 function generateForMeta (meta: Metadata, dest: string, extraTypes: ExtraTypes, isStrict: boolean): void {
   writeFile(dest, (): string => {
-    const allTypes = { '@polkadot/types/interfaces': defaultDefs, ...extraTypes };
+    const allTypes = {
+      '@polkadot/types/augment': { lookup: lookupDefinitions },
+      '@polkadot/types/interfaces': defaultDefs,
+      ...extraTypes
+    };
     const imports = createImports(allTypes);
     const allDefs = Object.entries(allTypes).reduce((defs, [path, obj]) => {
       return Object.entries(obj).reduce((defs, [key, value]) => ({ ...defs, [`${path}/${key}`]: value }), defs);
     }, {});
-
-    const modules = meta.asLatest.modules
-      .filter((mod) => mod.events.isSome)
+    const { lookup, pallets, registry } = meta.asLatest;
+    const modules = pallets
+      .filter(({ events }) => events.isSome)
       .map(({ events, name }) => ({
-        items: events
-          .unwrap()
-          .sort(compareName)
-          .map(({ args, docs, name }) => {
-            const types = args.map((type) => formatType(meta.registry, allDefs, type.toString(), imports));
+        items: lookup.getSiType(events.unwrap().type).def.asVariant.variants
+          .map(({ docs, fields, name }) => {
+            const args = fields
+              .map(({ type }) => lookup.getTypeDef(type))
+              .map((typeDef) => typeDef.lookupName || formatType(registry, allDefs, typeDef, imports));
 
-            setImports(allDefs, imports, types);
+            setImports(allDefs, imports, args);
 
             return {
               docs,
               name: name.toString(),
-              type: types.join(', ')
+              type: args.join(', ')
             };
-          }).sort(compareName),
+          })
+          .sort(compareName),
         name: stringCamelCase(name)
       }))
       .sort(compareName);
@@ -51,7 +57,7 @@ function generateForMeta (meta: Metadata, dest: string, extraTypes: ExtraTypes,
       modules,
       types: [
         ...Object.keys(imports.localTypes).sort().map((packagePath): { file: string; types: string[] } => ({
-          file: packagePath,
+          file: packagePath.replace('@polkadot/types/augment', '@polkadot/types'),
           types: Object.keys(imports.localTypes[packagePath])
         })),
         {

packages/typegen/src/generate/index.ts

@@ -5,6 +5,7 @@ export { generateDefaultConsts } from './consts';
 export { generateDefaultErrors } from './errors';
 export { generateDefaultEvents } from './events';
 export { generateDefaultInterface } from './interfaceRegistry';
+export { generateDefaultLookup } from './lookup';
 export { generateDefaultQuery } from './query';
 export { generateDefaultRpc } from './rpc';
 export { generateDefaultTsDef } from './tsDef';

packages/typegen/src/generate/lookup.ts

@@ -0,0 +1,269 @@
+// Copyright 2017-2021 @polkadot/typegen authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+
+import type { PortableRegistry, PortableType, SiLookupTypeId, SiPath, SiTypeParameter } from '@polkadot/types/interfaces';
+import type { TypeDef } from '@polkadot/types/types';
+
+import Handlebars from 'handlebars';
+import path from 'path';
+
+import * as defaultDefinitions from '@polkadot/types/interfaces/definitions';
+import { Registry } from '@polkadot/types/types';
+import staticKusama from '@polkadot/types-support/metadata/static-kusama';
+import staticPolkadot from '@polkadot/types-support/metadata/static-polkadot';
+import staticSubstrate from '@polkadot/types-support/metadata/static-substrate';
+import { isString, stringify } from '@polkadot/util';
+
+import { createImports, exportType, initMeta, readTemplate, writeFile } from '../util';
+import { typeEncoders } from './tsDef';
+
+const MAP_ENUMS = ['Call', 'Event', 'Error', 'RawEvent'];
+const WITH_TYPEDEF = false;
+
+const generateLookupDefsTmpl = Handlebars.compile(readTemplate('lookup/defs'));
+const generateLookupDefsNamedTmpl = Handlebars.compile(readTemplate('lookup/defs-named'));
+const generateLookupIndexTmpl = Handlebars.compile(readTemplate('lookup/index'));
+const generateLookupTypesTmpl = Handlebars.compile(readTemplate('lookup/types'));
+
+function generateParamType (registry: Registry, { name, type }: SiTypeParameter): string {
+  if (type.isSome) {
+    const link = registry.lookup.types[type.unwrap().toNumber()];
+
+    if (link.type.path.length) {
+      return generateTypeDocs(registry, null, link.type.path, link.type.params);
+    }
+  }
+
+  return name.toString();
+}
+
+function generateTypeDocs (registry: Registry, id: SiLookupTypeId | null, path: SiPath, params: SiTypeParameter[]): string {
+  return `${id ? `${registry.createLookupType(id)}${path.length ? ': ' : ''}` : ''}${path.map((p) => p.toString()).join('::')}${params.length ? `<${params.map((p) => generateParamType(registry, p)).join(', ')}>` : ''}`;
+}
+
+function formatObject (lines: string[]): string[] {
+  const max = lines.length - 1;
+
+  return [
+    '{',
+    ...lines.map((l, index) =>
+      (l.endsWith(',') || l.endsWith('{') || index === max || lines[index + 1].endsWith('}') || lines[index + 1].endsWith('}'))
+        ? l
+        : `${l},`
+    ),
+    '}'
+  ];
+}
+
+function expandSet (parsed: Record<string, number>): string[] {
+  return formatObject(
+    Object.entries(parsed).reduce<string[]>((all, [k, v]) => {
+      all.push(`${k}: ${v}`);
+
+      return all;
+    }, [])
+  );
+}
+
+function expandObject (parsed: Record<string, string | Record<string, string>>): string[] {
+  if (parsed._set) {
+    return expandSet(parsed._set as unknown as Record<string, number>);
+  }
+
+  return formatObject(
+    Object.entries(parsed).reduce<string[]>((all, [k, v]) => {
+      const inner = isString(v)
+        ? expandType(v)
+        : Array.isArray(v)
+          ? [`[${(v as string[]).map((e) => `'${e}'`).join(', ')}]`]
+          : expandObject(v);
+
+      inner.forEach((l, index): void => {
+        all.push(`${
+          index === 0
+            ? `${k}: ${l}`
+            : `${l}`
+        }`);
+      });
+
+      return all;
+    }, [])
+  );
+}
+
+function expandType (encoded: string): string[] {
+  if (!encoded.startsWith('{')) {
+    return [`'${encoded}'`];
+  }
+
+  return expandObject(JSON.parse(encoded) as Record<string, string | Record<string, string>>);
+}
+
+function expandDefToString ({ lookupNameRoot, type }: TypeDef, indent: number): string {
+  if (lookupNameRoot) {
+    return `'${lookupNameRoot}'`;
+  }
+
+  const lines = expandType(type);
+  let inc = 0;
+
+  return lines.map((l, index) => {
+    let r: string;
+
+    if (l.endsWith('{')) {
+      r = index === 0
+        ? l
+        : `${' '.padStart(indent + inc)}${l}`;
+      inc += 2;
+    } else {
+      if (l.endsWith('},') || l.endsWith('}')) {
+        inc -= 2;
+      }
+
+      r = index === 0
+        ? l
+        : `${' '.padStart(indent + inc)}${l}`;
+    }
+
+    return r;
+  }).join('\n');
+}
+
+function getFilteredTypes (lookup: PortableRegistry, exclude: string[] = []): [PortableType, TypeDef][] {
+  const named = lookup.types.filter(({ id, type: { path } }) => {
+    const typeDef = lookup.getTypeDef(id);
+
+    return (
+      // We actually only want those with lookupName set
+      !!typeDef.lookupName &&
+      !(
+        path.length === 2 &&
+        (
+          (
+            // Ensure that we match {node, kusama, *}_runtime
+            path[0].toString().split('_')[1] === 'runtime' &&
+            !['Call', 'Event'].includes(path[1].toString())
+          ) ||
+          path[0].toString().startsWith('pallet_')
+        ) &&
+        // Ensure we strip generics, e.g. Event<T>
+        MAP_ENUMS.includes(path[1].toString().split('<')[0])
+      ) &&
+      !(
+        path.length >= 3 &&
+        path[path.length - 2].toString() === 'pallet' &&
+        // As above, cater for generics
+        MAP_ENUMS.includes(path[path.length - 1].toString().split('<')[0])
+      )
+    );
+  });
+  const names = named.map(({ id }) => lookup.getName(id));
+
+  return named
+    .filter((_, index) =>
+      !names.some((n, iindex) =>
+        index > iindex &&
+        n === names[index]
+      )
+    )
+    .map((p): [PortableType, TypeDef] => [p, lookup.getTypeDef(p.id)])
+    .filter(([, typeDef]) => !exclude.includes(typeDef.lookupName || '<invalid>'));
+}
+
+function generateLookupDefs (registry: Registry, filtered: [PortableType, TypeDef][], destDir: string, subPath?: string): void {
+  writeFile(path.join(destDir, `${subPath || 'definitions'}.ts`), (): string => {
+    const all = filtered.map(([{ id, type: { params, path } }, typeDef]) => {
+      const typeLookup = registry.createLookupType(id);
+      const def = expandDefToString(typeDef, subPath ? 2 : 4);
+
+      return {
+        docs: [
+          generateTypeDocs(registry, id, path, params),
+          WITH_TYPEDEF
+            ? `@typeDef ${stringify(typeDef)}`
+            : null
+        ].filter((d): d is string => !!d),
+        type: { def, typeLookup, typeName: typeDef.lookupName }
+      };
+    });
+    const max = all.length - 1;
+
+    return (subPath ? generateLookupDefsNamedTmpl : generateLookupDefsTmpl)({
+      defs: all.map(({ docs, type }, i) => {
+        const { def, typeLookup, typeName } = type;
+
+        return {
+          defs: [
+            [typeName || typeLookup, `${def}${i !== max ? ',' : ''}`]
+          ].map(([n, t]) => `${n}: ${t}`),
+          docs
+        };
+      })
+    });
+  });
+}
+
+function generateLookupTypes (registry: Registry, filtered: [PortableType, TypeDef][], destDir: string, subPath?: string): void {
+  const imports = {
+    ...createImports(
+      { '@polkadot/types/interfaces': defaultDefinitions },
+      { types: {} }
+    ),
+    interfaces: []
+  };
+  const items = filtered.map(([, typeDef]) => {
+    typeDef.name = typeDef.lookupName;
+
+    return typeDef.lookupNameRoot && typeDef.lookupName
+      ? exportType(typeDef.lookupIndex, typeDef.lookupName, typeDef.lookupNameRoot)
+      : typeEncoders[typeDef.info](registry, imports.definitions, typeDef, imports);
+  }).filter((t): t is string => !!t);
+
+  writeFile(path.join(destDir, `types${subPath ? `-${subPath}` : ''}.ts`), () => generateLookupTypesTmpl({
+    headerType: 'defs',
+    imports,
+    items: items.map((l) =>
+      l
+        .split('\n')
+        .map((l) => l.length ? `  ${l}` : '')
+        .join('\n')
+    ),
+    types: [
+      ...Object.keys(imports.localTypes).sort().map((packagePath): { file: string; types: string[] } => ({
+        file: packagePath,
+        types: Object.keys(imports.localTypes[packagePath])
+      }))
+    ]
+  }), true);
+  writeFile(path.join(destDir, 'index.ts'), () => generateLookupIndexTmpl({ headerType: 'defs' }), true);
+}
+
+function generateLookup (destDir: string, entries: [string | undefined, string][]): void {
+  entries.reduce<string[]>((exclude, [subPath, staticMeta]): string[] => {
+    const { lookup, registry } = initMeta(staticMeta).metadata.asLatest;
+    const filtered = getFilteredTypes(lookup, exclude);
+
+    generateLookupDefs(registry, filtered, destDir, subPath);
+    generateLookupTypes(registry, filtered, destDir, subPath);
+
+    return exclude.concat(
+      ...filtered
+        .map(([, typeDef]) => typeDef.lookupName)
+        .filter((n): n is string => !!n)
+    );
+  }, []);
+}
+
+// Generate `packages/types/src/lookup/*s`, the registry of all lookup types
+export function generateDefaultLookup (destDir = 'packages/types/src/augment/lookup', staticData?: string): void {
+  generateLookup(
+    destDir,
+    staticData
+      ? [[undefined, staticData]]
+      : [
+        ['substrate', staticSubstrate],
+        ['polkadot', staticPolkadot],
+        ['kusama', staticKusama]
+      ]
+  );
+}

packages/typegen/src/generate/query.ts

@@ -1,15 +1,16 @@
 // Copyright 2017-2021 @polkadot/typegen authors & contributors
 // SPDX-License-Identifier: Apache-2.0
 
-import type { StorageEntryMetadataLatest } from '@polkadot/types/interfaces/metadata';
+import type { PortableRegistry, StorageEntryMetadataLatest } from '@polkadot/types/interfaces/metadata';
 import type { Metadata } from '@polkadot/types/metadata/Metadata';
 import type { Registry } from '@polkadot/types/types';
 import type { ExtraTypes } from './types';
 
 import Handlebars from 'handlebars';
 
+import lookupDefinitions from '@polkadot/types/augment/lookup/definitions';
 import * as defaultDefs from '@polkadot/types/interfaces/definitions';
-import { unwrapStorageType } from '@polkadot/types/primitive/StorageKey';
+import { unwrapStorageSi } from '@polkadot/types/primitive/StorageKey';
 import { stringCamelCase } from '@polkadot/util';
 
 import { compareName, createImports, formatType, getSimilarTypes, initMeta, readTemplate, setImports, TypeImports, writeFile } from '../util';
@@ -17,69 +18,56 @@ import { ModuleTypes } from '../util/imports';
 
 // From a storage entry metadata, we return [args, returnType]
 /** @internal */
-function entrySignature (allDefs: Record<string, ModuleTypes>, registry: Registry, storageEntry: StorageEntryMetadataLatest, imports: TypeImports): [string, string, string] {
-  const outputType = unwrapStorageType(registry, storageEntry.type, storageEntry.modifier.isOptional);
+function entrySignature (lookup: PortableRegistry, allDefs: Record<string, ModuleTypes>, registry: Registry, storageEntry: StorageEntryMetadataLatest, imports: TypeImports): [boolean, string, string, string] {
+  const outputType = lookup.getTypeDef(unwrapStorageSi(storageEntry.type));
+
+  // else if (storageEntry.type.isMap) {
+  //   const map = storageEntry.type.asMap;
+  //   const keyDef = lookup.getTypeDef(map.key);
+  //   const valDef = lookup.getTypeDef(map.value);
+
+  //   // Find similar types of the `key` type
+  //   const similarTypes = getSimilarTypes(registry, allDefs, keyDef.lookupName || keyDef.type, imports);
+
+  //   setImports(allDefs, imports, [
+  //     ...similarTypes,
+  //     valDef.lookupName || valDef.type
+  //   ]);
+
+  //   return [
+  //     storageEntry.modifier.isOptional,
+  //     formatType(registry, allDefs, keyDef.lookupName || keyDef.type, imports),
+  //     `arg: ${similarTypes.join(' | ')}`,
+  //     formatType(registry, allDefs, outputType, imports)
+  //   ];
+  // }
 
   if (storageEntry.type.isPlain) {
-    setImports(allDefs, imports, [storageEntry.type.asPlain.toString()]);
+    const typeDef = lookup.getTypeDef(storageEntry.type.asPlain);
 
-    return ['', '', formatType(registry, allDefs, outputType, imports)];
+    setImports(allDefs, imports, [typeDef.lookupName || typeDef.type]);
+
+    return [storageEntry.modifier.isOptional, '', '', formatType(registry, allDefs, outputType, imports)];
   } else if (storageEntry.type.isMap) {
-    const map = storageEntry.type.asMap;
-
-    // Find similar types of the `key` type
-    const similarTypes = getSimilarTypes(registry, allDefs, map.key.toString(), imports);
-
-    setImports(allDefs, imports, [
-      ...similarTypes,
-      map.value.toString()
-    ]);
-
-    return [
-      formatType(registry, allDefs, map.key.toString(), imports),
-      `arg: ${similarTypes.join(' | ')}`,
-      formatType(registry, allDefs, outputType, imports)
-    ];
-  } else if (storageEntry.type.isDoubleMap) {
-    const dm = storageEntry.type.asDoubleMap;
-
-    // Find similar types of `key1` and `key2` types
-    const similarTypes1 = getSimilarTypes(registry, allDefs, dm.key1.toString(), imports);
-    const similarTypes2 = getSimilarTypes(registry, allDefs, dm.key2.toString(), imports);
-
-    setImports(allDefs, imports, [
-      ...similarTypes1,
-      ...similarTypes2,
-      dm.value.toString()
-    ]);
-
-    const key1Types = similarTypes1.join(' | ');
-    const key2Types = similarTypes2.join(' | ');
-
-    return [
-      [
-        formatType(registry, allDefs, dm.key1.toString(), imports),
-        formatType(registry, allDefs, dm.key2.toString(), imports)
-      ].join(', '),
-      `arg1: ${key1Types}, arg2: ${key2Types}`,
-      formatType(registry, allDefs, outputType, imports)
-    ];
-  } else if (storageEntry.type.isNMap) {
-    const nmap = storageEntry.type.asNMap;
-
-    // Find similar types for all keys
-    const similarTypes = nmap.keyVec.map((k) => getSimilarTypes(registry, allDefs, k.toString(), imports));
+    const { hashers, key, value } = storageEntry.type.asMap;
+    const keyDefs = hashers.length === 1
+      ? [lookup.getTypeDef(key)]
+      : lookup.getSiType(key).def.asTuple.map((k) => lookup.getTypeDef(k));
+    const similarTypes = keyDefs.map((k) => getSimilarTypes(registry, allDefs, k.lookupName || k.type, imports));
+    const keyTypes = similarTypes.map((t) => t.join(' | '));
+    const defValue = lookup.getTypeDef(value);
 
     setImports(allDefs, imports, [
       ...similarTypes.reduce<string[]>((all, t) => all.concat(t), []),
-      nmap.value.toString()
+      defValue.lookupName
+        ? undefined
+        : defValue.type
     ]);
 
-    const keyTypes = similarTypes.map((t) => t.join(' | '));
-
     return [
-      nmap.keyVec.map((k) => formatType(registry, allDefs, k.toString(), imports)).join(', '),
-      keyTypes.map((t, i) => `arg${i + 1}: ${t}`).join(', '),
+      storageEntry.modifier.isOptional,
+      keyDefs.map((k) => formatType(registry, allDefs, k.lookupName || k.type, imports)).join(', '),
+      keyTypes.map((t, i) => `arg${keyTypes.length === 1 ? '' : (i + 1)}: ${t}`).join(', '),
       formatType(registry, allDefs, outputType, imports)
     ];
   }
@@ -93,19 +81,25 @@ const generateForMetaTemplate = Handlebars.compile(template);
 /** @internal */
 function generateForMeta (registry: Registry, meta: Metadata, dest: string, extraTypes: ExtraTypes, isStrict: boolean): void {
   writeFile(dest, (): string => {
-    const allTypes: ExtraTypes = { '@polkadot/types/interfaces': defaultDefs, ...extraTypes };
+    const allTypes: ExtraTypes = {
+      '@polkadot/types/augment': { lookup: lookupDefinitions },
+      '@polkadot/types/interfaces': defaultDefs,
+      ...extraTypes
+    };
     const imports = createImports(allTypes);
     const allDefs = Object.entries(allTypes).reduce((defs, [path, obj]) => {
       return Object.entries(obj).reduce((defs, [key, value]) => ({ ...defs, [`${path}/${key}`]: value }), defs);
     }, {});
-
-    const modules = meta.asLatest.modules
-      .sort(compareName)
-      .filter((mod) => !mod.storage.isNone)
+    const { lookup, pallets } = meta.asLatest;
+    const modules = pallets
+      .filter(({ storage }) => storage.isSome)
       .map(({ name, storage }) => {
         const items = storage.unwrap().items
           .map((storageEntry) => {
-            const [args, params, returnType] = entrySignature(allDefs, registry, storageEntry, imports);
+            const [isOptional, args, params, _returnType] = entrySignature(lookup, allDefs, registry, storageEntry, imports);
+            const returnType = isOptional
+              ? `Option<${_returnType}>`
+              : _returnType;
 
             return {
               args,
@@ -134,7 +128,7 @@ function generateForMeta (registry: Registry, meta: Metadata, dest: string, extr
       modules,
       types: [
         ...Object.keys(imports.localTypes).sort().map((packagePath): { file: string; types: string[] } => ({
-          file: packagePath,
+          file: packagePath.replace('@polkadot/types/augment', '@polkadot/types'),
           types: Object.keys(imports.localTypes[packagePath])
         })),
         {

packages/typegen/src/generate/rpc.ts

@@ -132,7 +132,7 @@ export function generateRpcTypes (registry: TypeRegistry, importDefinitions: Rec
       modules,
       types: [
         ...Object.keys(imports.localTypes).sort().map((packagePath): { file: string; types: string[] } => ({
-          file: packagePath,
+          file: packagePath.replace('@polkadot/types/augment', '@polkadot/types'),
           types: Object.keys(imports.localTypes[packagePath])
         }))
       ]

packages/typegen/src/generate/tsDef.ts

@@ -35,7 +35,7 @@ function errorUnhandled (_: Registry, definitions: Record<string, ModuleTypes>,
 
 /** @internal */
 function tsExport (registry: Registry, definitions: Record<string, ModuleTypes>, def: TypeDef, imports: TypeImports): string {
-  return exportInterface(def.name, formatType(registry, definitions, def, imports));
+  return exportInterface(def.lookupIndex, def.name, formatType(registry, definitions, def, imports, false));
 }
 
 const tsBTreeMap = tsExport;
@@ -48,16 +48,16 @@ const tsPlain = tsExport;
 const tsTuple = tsExport;
 
 /** @internal */
-function tsEnum (registry: Registry, definitions: Record<string, ModuleTypes>, { name: enumName, sub }: TypeDef, imports: TypeImports): string {
+function tsEnum (registry: Registry, definitions: Record<string, ModuleTypes>, { lookupIndex, name: enumName, sub }: TypeDef, imports: TypeImports): string {
   setImports(definitions, imports, ['Enum']);
 
   const keys = (sub as TypeDef[]).map((def, index): string => {
-    const { info, name = `unknown${index}`, type } = def;
+    const { info, lookupName, name = `unknown${index}`, type } = def;
     const getter = stringUpperFirst(stringCamelCase(name.replace(' ', '_')));
     const isComplex = [TypeDefInfo.Result, TypeDefInfo.Struct, TypeDefInfo.Tuple, TypeDefInfo.Vec, TypeDefInfo.VecFixed].includes(info);
     const asGetter = type === 'Null' || info === TypeDefInfo.DoNotConstruct
       ? ''
-      : createGetter(definitions, `as${getter}`, isComplex ? formatType(registry, definitions, info === TypeDefInfo.Struct ? def : type, imports) : type, imports);
+      : createGetter(definitions, `as${getter}`, lookupName || (isComplex ? formatType(registry, definitions, info === TypeDefInfo.Struct ? def : type, imports, false) : type), imports);
     const isGetter = info === TypeDefInfo.DoNotConstruct
       ? ''
       : createGetter(definitions, `is${getter}`, 'boolean', imports);
@@ -66,6 +66,7 @@ function tsEnum (registry: Registry, definitions: Record<string, ModuleTypes>, {
       case TypeDefInfo.Compact:
       case TypeDefInfo.Plain:
       case TypeDefInfo.Result:
+      case TypeDefInfo.Si:
       case TypeDefInfo.Struct:
       case TypeDefInfo.Tuple:
       case TypeDefInfo.Vec:
@@ -74,39 +75,54 @@ function tsEnum (registry: Registry, definitions: Record<string, ModuleTypes>, {
         return `${isGetter}${asGetter}`;
 
       case TypeDefInfo.DoNotConstruct:
-        return '';
+      case TypeDefInfo.Null:
+        return `${isGetter}`;
 
       default:
-        throw new Error(`Enum: ${enumName || 'undefined'}: Unhandled nested "${TypeDefInfo[info]}" type`);
+        throw new Error(`Enum: ${enumName || 'undefined'}: Unhandled type ${TypeDefInfo[info]}, ${stringify(def)}`);
     }
   });
 
-  return exportInterface(enumName, 'Enum', keys.join(''));
+  return exportInterface(lookupIndex, enumName, 'Enum', keys.join(''));
 }
 
 function tsInt (_: Registry, definitions: Record<string, ModuleTypes>, def: TypeDef, imports: TypeImports, type: 'Int' | 'UInt' = 'Int'): string {
   setImports(definitions, imports, [type]);
 
-  return exportInterface(def.name, type);
+  return exportInterface(def.lookupIndex, def.name, type);
+}
+
+/** @internal */
+function tsNull (registry: Registry, definitions: Record<string, ModuleTypes>, { lookupIndex = -1, name }: TypeDef, imports: TypeImports): string {
+  setImports(definitions, imports, ['Null']);
+
+  // * @description extends [[${base}]]
+  const doc = `/** @name ${name || ''}${lookupIndex !== -1 ? ` (${lookupIndex})` : ''} */\n`;
+
+  return `${doc}export type ${name || ''} = Null;`;
 }
 
 /** @internal */
 function tsResultGetter (registry: Registry, definitions: Record<string, ModuleTypes>, resultName = '', getter: 'Ok' | 'Err' | 'Error', def: TypeDef, imports: TypeImports): string {
-  const { info, type } = def;
+  const { info, lookupName, type } = def;
   const asGetter = type === 'Null'
     ? ''
-    : (getter === 'Error' ? '  /** @deprecated Use asErr */\n' : '') + createGetter(definitions, `as${getter}`, info === TypeDefInfo.Tuple ? formatType(registry, definitions, def, imports) : type, imports);
+    : (getter === 'Error' ? '  /** @deprecated Use asErr */\n' : '') + createGetter(definitions, `as${getter}`, lookupName || (info === TypeDefInfo.Tuple ? formatType(registry, definitions, def, imports, false) : type), imports);
   const isGetter = (getter === 'Error' ? '  /** @deprecated Use isErr */\n' : '') + createGetter(definitions, `is${getter}`, 'boolean', imports);
 
   switch (info) {
     case TypeDefInfo.Plain:
+    case TypeDefInfo.Si:
     case TypeDefInfo.Tuple:
     case TypeDefInfo.Vec:
     case TypeDefInfo.Option:
       return `${isGetter}${asGetter}`;
 
+    case TypeDefInfo.Null:
+      return `${isGetter}`;
+
     default:
-      throw new Error(`Result: ${resultName}: Unhandled type ${TypeDefInfo[info]}`);
+      throw new Error(`Result: ${resultName}: Unhandled type ${TypeDefInfo[info]}, ${stringify(def)}`);
   }
 }
 
@@ -122,17 +138,22 @@ function tsResult (registry: Registry, definitions: Record<string, ModuleTypes>,
 
   setImports(definitions, imports, [def.type]);
 
-  return exportInterface(def.name, formatType(registry, definitions, def, imports), inner);
+  const fmtType = def.lookupName && def.name !== def.lookupName
+    ? def.lookupName
+    : formatType(registry, definitions, def, imports, false);
+
+  return exportInterface(def.lookupIndex, def.name, fmtType, inner);
 }
 
 /** @internal */
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
-function tsSi (registry: Registry, definitions: Record<string, ModuleTypes>, { type }: TypeDef, imports: TypeImports): string {
-  throw new Error('Unable to generate string definition from TypeDefInfo.Si');
+function tsSi (registry: Registry, definitions: Record<string, ModuleTypes>, typeDef: TypeDef, imports: TypeImports): string {
+  // FIXME
+  return `// SI: ${JSON.stringify(typeDef)}`;
 }
 
 /** @internal */
-function tsSet (_: Registry, definitions: Record<string, ModuleTypes>, { name: setName, sub }: TypeDef, imports: TypeImports): string {
+function tsSet (_: Registry, definitions: Record<string, ModuleTypes>, { lookupIndex, name: setName, sub }: TypeDef, imports: TypeImports): string {
   setImports(definitions, imports, ['Set']);
 
   const types = (sub as TypeDef[]).map(({ name }): string => {
@@ -141,20 +162,22 @@ function tsSet (_: Registry, definitions: Record<string, ModuleTypes>, { name: s
     return createGetter(definitions, `is${name}`, 'boolean', imports);
   });
 
-  return exportInterface(setName, 'Set', types.join(''));
+  return exportInterface(lookupIndex, setName, 'Set', types.join(''));
 }
 
 /** @internal */
-function tsStruct (registry: Registry, definitions: Record<string, ModuleTypes>, { name: structName, sub }: TypeDef, imports: TypeImports): string {
+function tsStruct (registry: Registry, definitions: Record<string, ModuleTypes>, { lookupIndex, name: structName, sub }: TypeDef, imports: TypeImports): string {
   setImports(definitions, imports, ['Struct']);
 
-  const keys = (sub as TypeDef[]).map((typedef): string => {
-    const returnType = formatType(registry, definitions, typedef, imports);
+  const keys = (sub as TypeDef[]).map((def): string => {
+    const fmtType = def.lookupName && def.name !== def.lookupName
+      ? def.lookupName
+      : formatType(registry, definitions, def, imports, false);
 
-    return createGetter(definitions, typedef.name, returnType, imports);
+    return createGetter(definitions, def.name, fmtType, imports);
   });
 
-  return exportInterface(structName, 'Struct', keys.join(''));
+  return exportInterface(lookupIndex, structName, 'Struct', keys.join(''));
 }
 
 /** @internal */
@@ -166,19 +189,29 @@ function tsUInt (registry: Registry, definitions: Record<string, ModuleTypes>, d
 function tsVec (registry: Registry, definitions: Record<string, ModuleTypes>, def: TypeDef, imports: TypeImports): string {
   const type = (def.sub as TypeDef).type;
 
-  if (def.info === TypeDefInfo.VecFixed && type === 'u8') {
-    setImports(definitions, imports, ['U8aFixed']);
+  if (type === 'u8') {
+    if (def.info === TypeDefInfo.VecFixed) {
+      setImports(definitions, imports, ['U8aFixed']);
 
-    return exportType(def.name, 'U8aFixed');
+      return exportType(def.lookupIndex, def.name, 'U8aFixed');
+    } else {
+      setImports(definitions, imports, ['Bytes']);
+
+      return exportType(def.lookupIndex, def.name, 'Bytes');
+    }
   }
 
-  return exportInterface(def.name, formatType(registry, definitions, def, imports));
+  const fmtType = def.lookupName && def.name !== def.lookupName
+    ? def.lookupName
+    : formatType(registry, definitions, def, imports, false);
+
+  return exportInterface(def.lookupIndex, def.name, fmtType);
 }
 
 // handlers are defined externally to use - this means that when we do a
 // `generators[typedef.info](...)` TS will show any unhandled types. Rather
 // we are being explicit in having no handlers where we do not support (yet)
-const encoders: Record<TypeDefInfo, (registry: Registry, definitions: Record<string, ModuleTypes>, def: TypeDef, imports: TypeImports) => string> = {
+export const typeEncoders: Record<TypeDefInfo, (registry: Registry, definitions: Record<string, ModuleTypes>, def: TypeDef, imports: TypeImports) => string> = {
   [TypeDefInfo.BTreeMap]: tsBTreeMap,
   [TypeDefInfo.BTreeSet]: tsBTreeSet,
   [TypeDefInfo.Compact]: tsCompact,
@@ -187,7 +220,7 @@ const encoders: Record<TypeDefInfo, (registry: Registry, definitions: Record<str
   [TypeDefInfo.HashMap]: tsHashMap,
   [TypeDefInfo.Int]: tsInt,
   [TypeDefInfo.Linkage]: errorUnhandled,
-  [TypeDefInfo.Null]: errorUnhandled,
+  [TypeDefInfo.Null]: tsNull,
   [TypeDefInfo.Option]: tsOption,
   [TypeDefInfo.Plain]: tsPlain,
   [TypeDefInfo.Range]: errorUnhandled,
@@ -206,7 +239,7 @@ function generateInterfaces (registry: Registry, definitions: Record<string, Mod
   return Object.entries(types).map(([name, type]): [string, string] => {
     const def = getTypeDef(isString(type) ? type : stringify(type), { name });
 
-    return [name, encoders[def.info](registry, definitions, def, imports)];
+    return [name, typeEncoders[def.info](registry, definitions, def, imports)];
   });
 }
 
@@ -220,25 +253,23 @@ const templateTypes = readTemplate('tsDef/types');
 const generateTsDefTypesTemplate = Handlebars.compile(templateTypes);
 
 /** @internal */
-function generateTsDefFor (registry: Registry, importDefinitions: { [importPath: string]: Record<string, ModuleTypes> }, defName: string, { types }: { types: Record<string, any> }, outputDir: string): void {
+export function generateTsDefFor (registry: Registry, importDefinitions: { [importPath: string]: Record<string, ModuleTypes> }, defName: string, { types }: { types: Record<string, any> }, outputDir: string): void {
   const imports = { ...createImports(importDefinitions, { types }), interfaces: [] } as Imports;
   const definitions = imports.definitions;
   const interfaces = generateInterfaces(registry, definitions, { types }, imports);
-  const items = interfaces.sort((a, b): number => a[0].localeCompare(b[0])).map(([, definition]): string => definition);
-
-  const importTypes = [
-    ...Object.keys(imports.localTypes).sort().map((packagePath): { file: string; types: string[] } => ({
-      file: packagePath,
-      types: Object.keys(imports.localTypes[packagePath])
-    }))
-  ];
+  const items = interfaces.sort((a, b) => a[0].localeCompare(b[0])).map(([, definition]) => definition);
 
   writeFile(path.join(outputDir, defName, 'types.ts'), () => generateTsDefModuleTypesTemplate({
     headerType: 'defs',
     imports,
     items,
     name: defName,
-    types: importTypes
+    types: [
+      ...Object.keys(imports.localTypes).sort().map((packagePath): { file: string; types: string[] } => ({
+        file: packagePath.replace('@polkadot/types/augment', '@polkadot/types'),
+        types: Object.keys(imports.localTypes[packagePath])
+      }))
+    ]
   }), true);
   writeFile(path.join(outputDir, defName, 'index.ts'), () => generateTsDefIndexTemplate({ headerType: 'defs' }), true);
 }

packages/typegen/src/generate/tx.ts

@@ -7,6 +7,7 @@ import type { ExtraTypes } from './types';
 
 import Handlebars from 'handlebars';
 
+import lookupDefinitions from '@polkadot/types/augment/lookup/definitions';
 import * as defaultDefs from '@polkadot/types/interfaces/definitions';
 import { Text } from '@polkadot/types/primitive';
 import { stringCamelCase } from '@polkadot/util';
@@ -30,32 +31,54 @@ const generateForMetaTemplate = Handlebars.compile(template);
 /** @internal */
 function generateForMeta (registry: Registry, meta: Metadata, dest: string, extraTypes: ExtraTypes, isStrict: boolean): void {
   writeFile(dest, (): string => {
-    const allTypes: ExtraTypes = { '@polkadot/types/interfaces': defaultDefs, ...extraTypes };
+    const allTypes: ExtraTypes = {
+      '@polkadot/types/augment': { lookup: lookupDefinitions },
+      '@polkadot/types/interfaces': defaultDefs,
+      ...extraTypes
+    };
     const imports = createImports(allTypes);
     const allDefs = Object.entries(allTypes).reduce((defs, [path, obj]) => {
       return Object.entries(obj).reduce((defs, [key, value]) => ({ ...defs, [`${path}/${key}`]: value }), defs);
     }, {});
-
-    const modules = meta.asLatest.modules
-      .filter(({ calls }) => calls.unwrapOr([]).length !== 0)
+    const { lookup, pallets } = meta.asLatest;
+    const modules = pallets
+      .sort(compareName)
+      .filter(({ calls }) => calls.isSome)
       .map(({ calls, name }) => {
         setImports(allDefs, imports, ['Call', 'Extrinsic', 'SubmittableExtrinsic']);
 
-        const items = calls.unwrap()
-          .map(({ args, docs, name }) => {
-            const params = args
-              .map(({ name, type }) => {
-                const typeStr = type.toString();
+        const sectionName = stringCamelCase(name);
+        const items = lookup.getSiType(calls.unwrap().type).def.asVariant.variants
+          .map(({ docs, fields, name }) => {
+            const typesInfo = fields.map(({ name, type, typeName }, index): [string, string, string] => {
+              const typeDef = registry.lookup.getTypeDef(type);
+
+              return [
+                name.isSome
+                  ? mapName(name.unwrap())
+                  : `param${index}`,
+                typeName.isSome
+                  ? typeName.toString()
+                  : typeDef.type,
+                typeDef.isFromSi
+                  ? typeDef.type
+                  : typeDef.lookupName || typeDef.type
+              ];
+            });
+            const params = typesInfo
+              .map(([name,, typeStr]) => {
                 const similarTypes = getSimilarTypes(registry, allDefs, typeStr, imports);
 
                 setImports(allDefs, imports, [typeStr, ...similarTypes]);
 
-                return `${mapName(name)}: ${similarTypes.join(' | ')}`;
+                return `${name}: ${similarTypes.join(' | ')}`;
               })
               .join(', ');
 
             return {
-              args: args.map(({ type }) => formatType(registry, allDefs, type.toString(), imports)).join(', '),
+              args: typesInfo.map(([,, typeStr]) =>
+                formatType(registry, allDefs, typeStr, imports)
+              ).join(', '),
               docs,
               name: stringCamelCase(name),
               params
@@ -65,7 +88,7 @@ function generateForMeta (registry: Registry, meta: Metadata, dest: string, extr
 
         return {
           items,
-          name: stringCamelCase(name)
+          name: sectionName
         };
       })
       .sort(compareName);
@@ -77,7 +100,7 @@ function generateForMeta (registry: Registry, meta: Metadata, dest: string, extr
       modules,
       types: [
         ...Object.keys(imports.localTypes).sort().map((packagePath): { file: string; types: string[] } => ({
-          file: packagePath,
+          file: packagePath.replace('@polkadot/types/augment', '@polkadot/types'),
           types: Object.keys(imports.localTypes[packagePath])
         })),
         {

packages/typegen/src/interfacesTs.ts

@@ -1,9 +1,10 @@
 // Copyright 2017-2021 @polkadot/typegen authors & contributors
 // SPDX-License-Identifier: Apache-2.0
 
-import { generateDefaultConsts, generateDefaultErrors, generateDefaultEvents, generateDefaultInterface, generateDefaultQuery, generateDefaultRpc, generateDefaultTsDef, generateDefaultTx } from './generate';
+import { generateDefaultConsts, generateDefaultErrors, generateDefaultEvents, generateDefaultInterface, generateDefaultLookup, generateDefaultQuery, generateDefaultRpc, generateDefaultTsDef, generateDefaultTx } from './generate';
 
 export function main (): void {
+  generateDefaultLookup();
   generateDefaultInterface();
 
   generateDefaultConsts();

packages/typegen/src/metadataMd.ts

@@ -6,7 +6,7 @@ import type { Codec, DefinitionRpcParam } from '@polkadot/types/types';
 
 import fs from 'fs';
 
-import { GenericCall as Call, Metadata, TypeRegistry, Vec } from '@polkadot/types';
+import { Metadata, TypeRegistry, Vec } from '@polkadot/types';
 import * as definitions from '@polkadot/types/interfaces/definitions';
 import { getStorage as getSubstrateStorage } from '@polkadot/types/metadata/decorate/storage/getStorage';
 import { Text } from '@polkadot/types/primitive';
@@ -45,9 +45,15 @@ const DESC_STORAGE = `The following sections contain Storage methods are part of
 /** @internal */
 function docsVecToMarkdown (docLines: Vec<Text>, indent = 0): string {
   const md = docLines
-    .map((docLine) => docLine && docLine.substring(1)) // trim the leading space
-    .reduce((md, docLine): string => // generate paragraphs
-      !docLine.trim().length
+    .map((docLine) =>
+      docLine
+        .toString()
+        .trimStart()
+        .replace(/^r"/g, '')
+        .trimStart()
+    )
+    .reduce((md, docLine) => // generate paragraphs
+      !docLine.length
         ? `${md}\n\n` // empty line
         : /^[*-]/.test(docLine.trimStart()) && !md.endsWith('\n\n')
           ? `${md}\n\n${docLine}` // line calling for a preceding linebreak
@@ -162,25 +168,25 @@ function addRpc (): string {
 }
 
 /** @internal */
-function addConstants (metadata: MetadataLatest): string {
+function addConstants ({ lookup, pallets }: MetadataLatest): string {
   return renderPage({
     description: DESC_CONSTANTS,
-    sections: metadata.modules
+    sections: pallets
       .sort(sortByName)
-      .filter((moduleMetadata) => !moduleMetadata.constants.isEmpty)
-      .map((moduleMetadata) => {
-        const sectionName = stringLowerFirst(moduleMetadata.name);
+      .filter(({ constants }) => !constants.isEmpty)
+      .map(({ constants, name }) => {
+        const sectionName = stringLowerFirst(name);
 
         return {
-          items: moduleMetadata.constants
+          items: constants
             .sort(sortByName)
-            .map((func) => {
-              const methodName = stringCamelCase(func.name);
+            .map(({ docs, name, type }) => {
+              const methodName = stringCamelCase(name);
 
               return {
                 interface: '`' + `api.consts.${sectionName}.${methodName}` + '`',
-                name: `${methodName}: ` + '`' + func.type.toString() + '`',
-                ...(func.docs.length && { summary: func.docs })
+                name: `${methodName}: ` + '`' + lookup.getTypeDef(type).type + '`',
+                ...(docs.length && { summary: docs })
               };
             }),
           name: sectionName
@@ -191,9 +197,9 @@ function addConstants (metadata: MetadataLatest): string {
 }
 
 /** @internal */
-function addStorage (metadata: MetadataLatest): string {
-  const { substrate } = getSubstrateStorage(metadata.registry);
-  const moduleSections = metadata.modules
+function addStorage ({ lookup, pallets, registry }: MetadataLatest): string {
+  const { substrate } = getSubstrateStorage(registry);
+  const moduleSections = pallets
     .sort(sortByName)
     .filter((moduleMetadata) => !moduleMetadata.storage.isNone)
     .map((moduleMetadata): Section => {
@@ -203,15 +209,18 @@ function addStorage (metadata: MetadataLatest): string {
         items: moduleMetadata.storage.unwrap().items
           .sort(sortByName)
           .map((func) => {
-            const arg = func.type.isMap
-              ? ('`' + func.type.asMap.key.toString() + '`')
-              : func.type.isDoubleMap
-                ? ('`' + func.type.asDoubleMap.key1.toString() + ', ' + func.type.asDoubleMap.key2.toString() + '`')
-                : func.type.isNMap
-                  ? ('`' + func.type.asNMap.keyVec.map((k) => k.toString()).join(', ') + '`')
-                  : '';
+            let arg = '';
+
+            if (func.type.isMap) {
+              const { hashers, key } = func.type.asMap;
+
+              arg = hashers.length === 1
+                ? ('`' + key.toString() + '`')
+                : ('`' + lookup.getSiType(key).def.asTuple.map((t) => t).join(', ') + '`');
+            }
+
             const methodName = stringLowerFirst(func.name);
-            const outputType = unwrapStorageType(metadata.registry, func.type, func.modifier.isOptional);
+            const outputType = unwrapStorageType(registry, func.type, func.modifier.isOptional);
 
             return {
               interface: '`' + `api.query.${sectionName}.${methodName}` + '`',
@@ -229,12 +238,10 @@ function addStorage (metadata: MetadataLatest): string {
       description: 'These are well-known keys that are always available to the runtime implementation of any Substrate-based network.',
       items: Object.entries(substrate).map(([name, { meta }]) => {
         const arg = meta.type.isMap
-          ? ('`' + meta.type.asMap.key.toString() + '`')
-          : meta.type.isDoubleMap
-            ? ('`' + meta.type.asDoubleMap.key1.toString() + ', ' + meta.type.asDoubleMap.key2.toString() + '`')
-            : '';
+          ? ('`' + lookup.getTypeDef(meta.type.asMap.key).type + '`')
+          : '';
         const methodName = stringLowerFirst(name);
-        const outputType = unwrapStorageType(metadata.registry, meta.type, meta.modifier.isOptional);
+        const outputType = unwrapStorageType(registry, meta.type, meta.modifier.isOptional);
 
         return {
           interface: '`' + `api.query.substrate.${methodName}` + '`',
@@ -249,26 +256,26 @@ function addStorage (metadata: MetadataLatest): string {
 }
 
 /** @internal */
-function addExtrinsics (metadata: MetadataLatest): string {
+function addExtrinsics ({ lookup, pallets }: MetadataLatest): string {
   return renderPage({
     description: DESC_EXTRINSICS,
-    sections: metadata.modules
+    sections: pallets
       .sort(sortByName)
-      .filter((meta) => !meta.calls.isNone && meta.calls.unwrap().length !== 0)
-      .map((meta) => {
-        const sectionName = stringCamelCase(meta.name);
+      .filter(({ calls }) => calls.isSome)
+      .map(({ calls, name }) => {
+        const sectionName = stringCamelCase(name);
 
         return {
-          items: meta.calls.unwrap()
+          items: lookup.getSiType(calls.unwrap().type).def.asVariant.variants
             .sort(sortByName)
-            .map((func) => {
-              const methodName = stringCamelCase(func.name);
-              const args = Call.filterOrigin(func).map(({ name, type }) => `${name.toString()}: ` + '`' + type.toString() + '`').join(', ');
+            .map(({ docs, fields, name }, index) => {
+              const methodName = stringCamelCase(name);
+              const args = fields.map(({ name, type }) => `${name.isSome ? name.toString() : `param${index}`}: ` + '`' + lookup.getTypeDef(type).type + '`').join(', ');
 
               return {
                 interface: '`' + `api.tx.${sectionName}.${methodName}` + '`',
                 name: `${methodName}(${args})`,
-                ...(func.docs.length && { summary: func.docs })
+                ...(docs.length && { summary: docs })
               };
             }),
           name: sectionName
@@ -279,23 +286,23 @@ function addExtrinsics (metadata: MetadataLatest): string {
 }
 
 /** @internal */
-function addEvents (metadata: MetadataLatest): string {
+function addEvents ({ lookup, pallets }: MetadataLatest): string {
   return renderPage({
     description: DESC_EVENTS,
-    sections: metadata.modules
+    sections: pallets
       .sort(sortByName)
-      .filter((meta) => !meta.events.isNone && meta.events.unwrap().length !== 0)
+      .filter(({ events }) => events.isSome)
       .map((meta) => ({
-        items: meta.events.unwrap()
+        items: lookup.getSiType(meta.events.unwrap().type).def.asVariant.variants
           .sort(sortByName)
-          .map((func) => {
-            const methodName = func.name.toString();
-            const args = func.args.map((type): string => '`' + type.toString() + '`').join(', ');
+          .map(({ docs, fields, name }) => {
+            const methodName = name.toString();
+            const args = fields.map(({ type }) => '`' + lookup.getTypeDef(type).type + '`').join(', ');
 
             return {
               interface: '`' + `api.events.${stringCamelCase(meta.name)}.${methodName}.is` + '`',
               name: `${methodName}(${args})`,
-              ...(func.docs.length && { summary: func.docs })
+              ...(docs.length && { summary: docs })
             };
           }),
         name: stringCamelCase(meta.name)
@@ -305,14 +312,14 @@ function addEvents (metadata: MetadataLatest): string {
 }
 
 /** @internal */
-function addErrors (metadata: MetadataLatest): string {
+function addErrors ({ lookup, pallets }: MetadataLatest): string {
   return renderPage({
     description: DESC_ERRORS,
-    sections: metadata.modules
+    sections: pallets
       .sort(sortByName)
-      .filter((moduleMetadata) => !moduleMetadata.errors.isEmpty)
+      .filter(({ errors }) => errors.isSome)
       .map((moduleMetadata) => ({
-        items: moduleMetadata.errors
+        items: lookup.getSiType(moduleMetadata.errors.unwrap().type).def.asVariant.variants
           .sort(sortByName)
           .map((error) => ({
             interface: '`' + `api.errors.${stringCamelCase(moduleMetadata.name)}.${error.name.toString()}.is` + '`',

packages/typegen/src/templates/lookup/defs-named.hbs

@@ -0,0 +1,15 @@
+{{> header }}
+
+/* eslint-disable sort-keys */
+
+import type { DefinitionsTypes } from '../../types';
+
+export default {
+  {{#each defs}}
+  {{> docs}}
+  {{#each defs}}
+  {{{this}}}
+  {{/each}}
+  {{/each}}
+} as DefinitionsTypes;
+{{> footer }}

packages/typegen/src/templates/lookup/defs.hbs

@@ -0,0 +1,18 @@
+{{> header }}
+
+/* eslint-disable sort-keys */
+
+import type { Definitions } from '../../types';
+
+export default {
+  rpc: {},
+  types: {
+    {{#each defs}}
+    {{> docs}}
+    {{#each defs}}
+    {{{this}}}
+    {{/each}}
+    {{/each}}
+  }
+} as Definitions;
+{{> footer }}

packages/typegen/src/templates/lookup/index.hbs

@@ -0,0 +1,4 @@
+{{> header }}
+
+export * from './types';
+{{> footer }}

packages/typegen/src/templates/lookup/types.hbs

@@ -0,0 +1,12 @@
+{{> header }}
+
+{{{ imports }}}
+
+declare module '@polkadot/types/lookup' {
+
+{{#each items}}
+{{{this}}}
+
+{{/each}}
+}
+{{> footer }}

packages/typegen/src/util/formatting.ts

@@ -55,7 +55,7 @@ Handlebars.registerHelper({
       ...types
     ];
 
-    return defs.reduce((result, { file, types }): string => {
+    return [...defs].sort((a, b) => a.file.localeCompare(b.file)).reduce((result, { file, types }): string => {
       return types.length
         ? `${result}import type { ${types.sort().join(', ')} } from '${file}';\n`
         : result;
@@ -71,9 +71,9 @@ Handlebars.registerHelper({
 
 // helper to generate a `export interface <Name> extends <Base> {<Body>}
 /** @internal */
-export function exportInterface (name = '', base: string, body = ''): string {
+export function exportInterface (lookupIndex = -1, name = '', base: string, body = ''): string {
   // * @description extends [[${base}]]
-  const doc = `/** @name ${name} */\n`;
+  const doc = `/** @name ${name}${lookupIndex !== -1 ? ` (${lookupIndex})` : ''} */\n`;
 
   return `${doc}export interface ${name} extends ${base} {${body.length ? '\n' : ''}${body}}`;
 }
@@ -82,19 +82,19 @@ export function exportInterface (name = '', base: string, body = ''): string {
 // but since we don't want type alias (TS doesn't preserve names) we use
 // interface here.
 /** @internal */
-export function exportType (name = '', base: string): string {
-  return exportInterface(name, base);
+export function exportType (lookupIndex = -1, name = '', base: string): string {
+  return exportInterface(lookupIndex, name, base);
 }
 
 const formatters: Record<TypeDefInfo, (registry: Registry, typeDef: TypeDef, definitions: Record<string, ModuleTypes>, imports: TypeImports, withShortcut: boolean) => string> = {
   [TypeDefInfo.Compact]: (registry: Registry, typeDef: TypeDef, definitions: Record<string, ModuleTypes>, imports: TypeImports, withShortcut: boolean) => {
-    setImports(definitions, imports, ['Compact']);
-
     const sub = typeDef.sub as TypeDef;
 
-    return paramsNotation('Compact', formatType(registry, definitions, sub.lookupName || sub.type, imports, withShortcut));
+    setImports(definitions, imports, ['Compact', sub.lookupName]);
+
+    return paramsNotation('Compact', sub.lookupName || formatType(registry, definitions, sub.type, imports, withShortcut));
   },
-  [TypeDefInfo.DoNotConstruct]: (registry: Registry, typeDef: TypeDef, definitions: Record<string, ModuleTypes>, imports: TypeImports, withShortcut: boolean) => {
+  [TypeDefInfo.DoNotConstruct]: (registry: Registry, { lookupName }: TypeDef, definitions: Record<string, ModuleTypes>, imports: TypeImports, withShortcut: boolean) => {
     setImports(definitions, imports, ['DoNotConstruct']);
 
     return 'DoNotConstruct';
@@ -118,11 +118,11 @@ const formatters: Record<TypeDefInfo, (registry: Registry, typeDef: TypeDef, def
     return 'Null';
   },
   [TypeDefInfo.Option]: (registry: Registry, typeDef: TypeDef, definitions: Record<string, ModuleTypes>, imports: TypeImports, withShortcut: boolean) => {
-    setImports(definitions, imports, ['Option']);
-
     const sub = (typeDef.sub as TypeDef);
 
-    return paramsNotation('Option', formatType(registry, definitions, sub.lookupName || sub.type, imports, withShortcut));
+    setImports(definitions, imports, ['Option', sub.lookupName]);
+
+    return paramsNotation('Option', sub.lookupName || formatType(registry, definitions, sub.type, imports, withShortcut));
   },
   [TypeDefInfo.Plain]: (registry: Registry, typeDef: TypeDef, definitions: Record<string, ModuleTypes>, imports: TypeImports, withShortcut: boolean) => {
     setImports(definitions, imports, [typeDef.type]);
@@ -139,31 +139,39 @@ const formatters: Record<TypeDefInfo, (registry: Registry, typeDef: TypeDef, def
     return formatType(registry, definitions, registry.lookup.getTypeDef(typeDef.type), imports, withShortcut);
   },
   [TypeDefInfo.Struct]: (registry: Registry, typeDef: TypeDef, definitions: Record<string, ModuleTypes>, imports: TypeImports, withShortcut: boolean) => {
-    setImports(definitions, imports, ['Struct']);
+    if (typeDef.lookupName) {
+      return typeDef.lookupName;
+    }
+
+    const sub = typeDef.sub as TypeDef[];
+
+    setImports(definitions, imports, ['Struct', ...sub.map(({ lookupName }) => lookupName)]);
 
     return `{${withShortcut ? ' ' : '\n'}${
-      ((typeDef.sub as TypeDef[]).map(({ lookupName, name, type }, index) => [
+      sub.map(({ lookupName, name, type }, index) => [
         name || `unknown${index}`,
-        formatType(registry, definitions, lookupName || type, imports, withShortcut)
-      ])).map(([k, t]) => `${withShortcut ? '' : '    readonly '}${k}: ${t};`).join(withShortcut ? ' ' : '\n')
+        lookupName || formatType(registry, definitions, type, imports, withShortcut)
+      ]).map(([k, t]) => `${withShortcut ? '' : '    readonly '}${k}: ${t};`).join(withShortcut ? ' ' : '\n')
     }${withShortcut ? ' ' : '\n  '}} & Struct`;
   },
   [TypeDefInfo.Tuple]: (registry: Registry, typeDef: TypeDef, definitions: Record<string, ModuleTypes>, imports: TypeImports, withShortcut: boolean) => {
-    setImports(definitions, imports, ['ITuple']);
+    const sub = typeDef.sub as TypeDef[];
+
+    setImports(definitions, imports, ['ITuple', ...sub.map(({ lookupName }) => lookupName)]);
 
     // `(a,b)` gets transformed into `ITuple<[a, b]>`
     return paramsNotation('ITuple', `[${
-      ((typeDef.sub as TypeDef[]).map(({ lookupName, type }) =>
-        formatType(registry, definitions, lookupName || type, imports, withShortcut)
-      )).join(', ')
+      sub.map(({ lookupName, type }) =>
+        lookupName || formatType(registry, definitions, type, imports, withShortcut)
+      ).join(', ')
     }]`);
   },
   [TypeDefInfo.Vec]: (registry: Registry, typeDef: TypeDef, definitions: Record<string, ModuleTypes>, imports: TypeImports, withShortcut: boolean) => {
-    setImports(definitions, imports, ['Vec']);
-
     const sub = (typeDef.sub as TypeDef);
 
-    return paramsNotation('Vec', formatType(registry, definitions, sub.lookupName || sub.type, imports, withShortcut));
+    setImports(definitions, imports, ['Vec', sub.lookupName]);
+
+    return paramsNotation('Vec', sub.lookupName || formatType(registry, definitions, sub.type, imports, withShortcut));
   },
   [TypeDefInfo.VecFixed]: (registry: Registry, typeDef: TypeDef, definitions: Record<string, ModuleTypes>, imports: TypeImports, withShortcut: boolean) => {
     const sub = (typeDef.sub as TypeDef);
@@ -174,44 +182,44 @@ const formatters: Record<TypeDefInfo, (registry: Registry, typeDef: TypeDef, def
       return 'U8aFixed';
     }
 
-    setImports(definitions, imports, ['Vec']);
+    setImports(definitions, imports, ['Vec', sub.lookupName]);
 
-    return paramsNotation('Vec', formatType(registry, definitions, sub.lookupName || sub.type, imports, withShortcut));
+    return paramsNotation('Vec', sub.lookupName || formatType(registry, definitions, sub.type, imports, withShortcut));
   },
   [TypeDefInfo.BTreeMap]: (registry: Registry, typeDef: TypeDef, definitions: Record<string, ModuleTypes>, imports: TypeImports, withShortcut: boolean) => {
-    setImports(definitions, imports, ['BTreeMap']);
-
     const [keyDef, valDef] = (typeDef.sub as TypeDef[]);
 
-    return `BTreeMap<${formatType(registry, definitions, keyDef.lookupName || keyDef.type, imports, withShortcut)}, ${formatType(registry, definitions, valDef.lookupName || valDef.type, imports, withShortcut)}>`;
+    setImports(definitions, imports, ['BTreeMap', keyDef.lookupName, valDef.lookupName]);
+
+    return `BTreeMap<${keyDef.lookupName || formatType(registry, definitions, keyDef.type, imports, withShortcut)}, ${valDef.lookupName || formatType(registry, definitions, valDef.type, imports, withShortcut)}>`;
   },
   [TypeDefInfo.BTreeSet]: (registry: Registry, typeDef: TypeDef, definitions: Record<string, ModuleTypes>, imports: TypeImports, withShortcut: boolean) => {
-    setImports(definitions, imports, ['BTreeSet']);
-
     const valDef = typeDef.sub as TypeDef;
 
-    return `BTreeSet<${formatType(registry, definitions, valDef.lookupName || valDef.type, imports, withShortcut)}>`;
+    setImports(definitions, imports, ['BTreeSet', valDef.lookupName]);
+
+    return `BTreeSet<${valDef.lookupName || formatType(registry, definitions, valDef.type, imports, withShortcut)}>`;
   },
   [TypeDefInfo.HashMap]: (registry: Registry, typeDef: TypeDef, definitions: Record<string, ModuleTypes>, imports: TypeImports, withShortcut: boolean) => {
-    setImports(definitions, imports, ['HashMap']);
-
     const [keyDef, valDef] = (typeDef.sub as TypeDef[]);
 
-    return `HashMap<${formatType(registry, definitions, keyDef.lookupName || keyDef.type, imports, withShortcut)}, ${formatType(registry, definitions, valDef.lookupName || valDef.type, imports, withShortcut)}>`;
+    setImports(definitions, imports, ['HashMap', keyDef.lookupName, valDef.lookupName]);
+
+    return `HashMap<${keyDef.lookupName || formatType(registry, definitions, keyDef.type, imports, withShortcut)}, ${valDef.lookupName || formatType(registry, definitions, valDef.type, imports, withShortcut)}>`;
   },
   [TypeDefInfo.Linkage]: (registry: Registry, typeDef: TypeDef, definitions: Record<string, ModuleTypes>, imports: TypeImports, withShortcut: boolean) => {
-    setImports(definitions, imports, ['Linkage']);
-
     const sub = (typeDef.sub as TypeDef);
 
-    return paramsNotation('Linkage', formatType(registry, definitions, sub.lookupName || sub.type, imports, withShortcut));
+    setImports(definitions, imports, ['Linkage', sub.lookupName]);
+
+    return paramsNotation('Linkage', sub.lookupName || formatType(registry, definitions, sub.type, imports, withShortcut));
   },
   [TypeDefInfo.Result]: (registry: Registry, typeDef: TypeDef, definitions: Record<string, ModuleTypes>, imports: TypeImports, withShortcut: boolean) => {
-    setImports(definitions, imports, ['Result']);
-
     const [okDef, errDef] = (typeDef.sub as TypeDef[]);
 
-    return `Result<${formatType(registry, definitions, okDef.lookupName || okDef.type, imports, withShortcut)}, ${formatType(registry, definitions, errDef.lookupName || errDef.type, imports, withShortcut)}>`;
+    setImports(definitions, imports, ['Result', okDef.lookupName, errDef.lookupName]);
+
+    return `Result<${okDef.lookupName || formatType(registry, definitions, okDef.type, imports, withShortcut)}, ${errDef.lookupName || formatType(registry, definitions, errDef.type, imports, withShortcut)}>`;
   }
 };
 
@@ -237,7 +245,7 @@ export function formatType (registry: Registry, definitions: Record<string, Modu
     typeDef = type;
   }
 
-  setImports(definitions, imports, [typeDef.type]);
+  setImports(definitions, imports, [typeDef.lookupName || typeDef.type]);
 
   return formatters[typeDef.info](registry, typeDef, definitions, imports, withShortcut);
 }

packages/typegen/src/util/imports.ts

@@ -36,10 +36,10 @@ export interface TypeImports {
 // Maps the types as found to the source location. This is used to generate the
 // imports in the output file, dep-duped and sorted
 /** @internal */
-export function setImports (allDefs: Record<string, ModuleTypes>, imports: TypeImports, types: string[]): void {
+export function setImports (allDefs: Record<string, ModuleTypes>, imports: TypeImports, types: (string | null | undefined)[]): void {
   const { codecTypes, extrinsicTypes, genericTypes, ignoredTypes, localTypes, metadataTypes, primitiveTypes, typesTypes } = imports;
 
-  types.forEach((type): void => {
+  types.filter((t): t is string => !!t).forEach((type): void => {
     if (ignoredTypes.includes(type)) {
       // do nothing
     } else if (['AnyNumber', 'CallFunction', 'Codec', 'IExtrinsic', 'ITuple'].includes(type)) {

packages/typegen/src/util/initMeta.ts

@@ -4,7 +4,7 @@
 import type { ExtraTypes } from '../generate/types';
 
 import { Metadata, TypeRegistry } from '@polkadot/types';
-import staticData from '@polkadot/types-support/metadata/static-substrate';
+import staticSubstrate from '@polkadot/types-support/metadata/static-substrate';
 
 import { registerDefinitions } from './register';
 
@@ -13,12 +13,12 @@ interface Result {
   registry: TypeRegistry;
 }
 
-export function initMeta (data = staticData, extraTypes: ExtraTypes = {}): Result {
+export function initMeta (staticMeta = staticSubstrate, extraTypes: ExtraTypes = {}): Result {
   const registry = new TypeRegistry();
 
   registerDefinitions(registry, extraTypes);
 
-  const metadata = new Metadata(registry, data);
+  const metadata = new Metadata(registry, staticMeta);
 
   registry.setMetadata(metadata);