diff --git a/CHANGELOG.md b/CHANGELOG.md index bbbba07993..e3d210c7c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # 1.0.0-beta-x - **Breaking change** Drop support for Substrate v1 chain in all derives +- Add `polkadot-types-from-chain` & `polkadot-types-from-defs` TypeScript generators (`@polkadot/types` bin) - Only decorate derives where relevant parent `api.query.*` is available - Support `.entries(arg?: any)` lookups on DoubleMaps (in addition to previously supported maps) - Remove un-deployed support for v11 metadata (this was decided against on Substrate) diff --git a/docs/start/typescript.user.md b/docs/start/typescript.user.md index 6ae67423b9..5f94af8bbf 100644 --- a/docs/start/typescript.user.md +++ b/docs/start/typescript.user.md @@ -14,7 +14,7 @@ In the root of your project (with the `@polkadot/api`), you can run `yarn polkad In the same way as the type library provides defaults from a substrate-base chain, you can also, directly from chain from metadata, generate a complete `api.{consts,query}.*` defintion for your specific chain. The command will create 2 files, `{consts, query}.types.ts` which you can either use to augment the TypeScript defintions, or replace those in `@polkadot/api/*` with your versions (copy, TypeScript replacement or browser/node aliassing). -In the root of your project, you can run `yarn polkadot-types-from-chain.js --endpoint wss:// --output ./stuff` and it will create the required output. +In the root of your project, you can run `yarn polkadot-types-from-chain.js --endpoint wss:// --output ./stuff` and it will create the required output. (Here you can specify an optional `--package @MeInc/stuff` to read definitions for the targetted output folder with the specified package name.) ## And that's a wrap diff --git a/packages/types/src/scripts/generateTypes/consts.ts b/packages/types/src/scripts/generateTypes/consts.ts index 0f0a80624d..420bea2cf5 100644 --- a/packages/types/src/scripts/generateTypes/consts.ts +++ b/packages/types/src/scripts/generateTypes/consts.ts @@ -3,7 +3,7 @@ // of the Apache-2.0 license. See the LICENSE file for details. import { ModuleMetadataLatest } from '../../interfaces/metadata'; -import * as definitions from '../../interfaces/definitions'; +import * as defaultDefs from '../../interfaces/definitions'; import fs from 'fs'; import staticData from '@polkadot/metadata/Metadata/static'; @@ -14,19 +14,19 @@ import { createImportCode, createImports, FOOTER, HEADER, indent, setImports, Ty // Generate types for one module /** @internal */ -function generateModule (modul: ModuleMetadataLatest, imports: TypeImports): string[] { +function generateModule (allDefs: object, modul: ModuleMetadataLatest, imports: TypeImports): string[] { if (!modul.constants.length) { return []; } - setImports(definitions, imports, ['Codec']); + setImports(allDefs, imports, ['Codec']); return [indent(4)(`${stringCamelCase(modul.name.toString())}: {`)] .concat(indent(6)('[index: string]: Codec;')) .concat( modul.constants .map((constant): string => { - setImports(definitions, imports, [constant.type.toString()]); + setImports(allDefs, imports, [constant.type.toString()]); return indent(6)(`${stringCamelCase(constant.name.toString())}: ${constant.type} & ConstantCodec;`); }) @@ -37,17 +37,19 @@ function generateModule (modul: ModuleMetadataLatest, imports: TypeImports): str // Generate `packages/api/src/consts.types.ts` for a particular // metadata /** @internal */ -function generateForMeta (meta: Metadata, dest: string): void { +function generateForMeta (meta: Metadata, dest: string, extraTypes: Record>): void { console.log(`Writing ${dest}`); - const imports = createImports({ '@polkadot/types/interfaces': definitions }); // Will hold all needed imports - + const allTypes: Record> = { '@polkadot/types/interfaces': defaultDefs, ...extraTypes }; + const imports = createImports(allTypes); + const allDefs = Object.entries(allTypes).reduce((defs, [, obj]) => { + return Object.entries(obj).reduce((defs, [key, value]) => ({ ...defs, [key]: value }), defs); + }, {}); const body = meta.asLatest.modules.reduce((acc, modul): string[] => { - const storageEntries = generateModule(modul, imports); + const storageEntries = generateModule(allDefs, modul, imports); return acc.concat(storageEntries); }, [] as string[]); - const header = createImportCode(HEADER, [ { file: '@polkadot/types/codec', @@ -58,7 +60,7 @@ function generateForMeta (meta: Metadata, dest: string): void { types: Object.keys(imports.primitiveTypes) }, ...Object.keys(imports.localTypes).map((moduleName): { file: string; types: string[] } => ({ - file: `@polkadot/types/interfaces/${moduleName}`, + file: `${imports.moduleToPackage[moduleName]}/${moduleName}`, types: Object.keys(imports.localTypes[moduleName]) })), { @@ -66,12 +68,10 @@ function generateForMeta (meta: Metadata, dest: string): void { types: Object.keys(imports.typesTypes) } ]); - - const interfaceStart = - [ - "declare module '@polkadot/metadata/Decorated/types' {", - indent(2)('export interface Constants {\n') - ].join('\n'); + const interfaceStart = [ + "declare module '@polkadot/metadata/Decorated/types' {", + indent(2)('export interface Constants {\n') + ].join('\n'); const interfaceEnd = `\n${indent(2)('}')}\n}`; fs.writeFileSync( @@ -87,8 +87,8 @@ function generateForMeta (meta: Metadata, dest: string): void { // Call `generateForMeta()` with current static metadata /** @internal */ -export default function generateConsts (dest = 'packages/api/src/consts.types.ts', data = staticData): void { +export default function generateConsts (dest = 'packages/api/src/consts.types.ts', data = staticData, extraTypes: Record> = {}): void { const registry = new TypeRegistry(); - return generateForMeta(new Metadata(registry, data), dest); + return generateForMeta(new Metadata(registry, data), dest, extraTypes); } diff --git a/packages/types/src/scripts/generateTypes/interfaceRegistry.ts b/packages/types/src/scripts/generateTypes/interfaceRegistry.ts index 024e580d41..9d1d6dbb88 100644 --- a/packages/types/src/scripts/generateTypes/interfaceRegistry.ts +++ b/packages/types/src/scripts/generateTypes/interfaceRegistry.ts @@ -82,9 +82,7 @@ export function generateInterfaceRegistry (importDefinitions: { [importPath: str // Generate `packages/types/src/interfaceRegistry.ts`, the registry of all interfaces export default function generateDefaultInterfaceRegistry (): void { generateInterfaceRegistry( - { - '@polkadot/types/interfaces': defaultDefinitions - }, + { '@polkadot/types/interfaces': defaultDefinitions }, 'packages/types/src/interfaceRegistry.ts' ); } diff --git a/packages/types/src/scripts/generateTypes/query.ts b/packages/types/src/scripts/generateTypes/query.ts index ed1e9bb5c0..12f3c98990 100644 --- a/packages/types/src/scripts/generateTypes/query.ts +++ b/packages/types/src/scripts/generateTypes/query.ts @@ -4,7 +4,7 @@ import { ModuleMetadataLatest, StorageEntryMetadataLatest } from '../../interfaces/metadata'; import { Registry } from '../../types'; -import * as defaultDefinitions from '../../interfaces/definitions'; +import * as defaultDefs from '../../interfaces/definitions'; import fs from 'fs'; import staticData from '@polkadot/metadata/Metadata/static'; @@ -26,41 +26,41 @@ function addModifier (storageEntry: StorageEntryMetadataLatest, returnType: stri // From a storage entry metadata, we return [args, returnType] /** @internal */ -function entrySignature (definitions: object, registry: Registry, storageEntry: StorageEntryMetadataLatest, imports: TypeImports): [string, string] { +function entrySignature (allDefs: object, registry: Registry, storageEntry: StorageEntryMetadataLatest, imports: TypeImports): [string, string] { if (storageEntry.type.isPlain) { - setImports(definitions, imports, [storageEntry.type.asPlain.toString()]); + setImports(allDefs, imports, [storageEntry.type.asPlain.toString()]); - return ['', formatType(definitions, addModifier(storageEntry, storageEntry.type.asPlain.toString()), imports)]; + return ['', formatType(allDefs, addModifier(storageEntry, storageEntry.type.asPlain.toString()), imports)]; } else if (storageEntry.type.isMap) { // Find similar types of the `key` type - const similarTypes = getSimilarTypes(definitions, registry, storageEntry.type.asMap.key.toString(), imports); + const similarTypes = getSimilarTypes(allDefs, registry, storageEntry.type.asMap.key.toString(), imports); - setImports(definitions, imports, [ + setImports(allDefs, imports, [ ...similarTypes, storageEntry.type.asMap.value.toString() ]); return [ - `arg: ${similarTypes.map((type) => formatType(definitions, type, imports)).join(' | ')}`, - formatType(definitions, addModifier(storageEntry, storageEntry.type.asMap.value.toString()), imports) + `arg: ${similarTypes.map((type) => formatType(allDefs, type, imports)).join(' | ')}`, + formatType(allDefs, addModifier(storageEntry, storageEntry.type.asMap.value.toString()), imports) ]; } else if (storageEntry.type.isDoubleMap) { // Find similartypes of `key1` and `key2` types - const similarTypes1 = getSimilarTypes(definitions, registry, storageEntry.type.asDoubleMap.key1.toString(), imports); - const similarTypes2 = getSimilarTypes(definitions, registry, storageEntry.type.asDoubleMap.key2.toString(), imports); + const similarTypes1 = getSimilarTypes(allDefs, registry, storageEntry.type.asDoubleMap.key1.toString(), imports); + const similarTypes2 = getSimilarTypes(allDefs, registry, storageEntry.type.asDoubleMap.key2.toString(), imports); - setImports(definitions, imports, [ + setImports(allDefs, imports, [ ...similarTypes1, ...similarTypes2, storageEntry.type.asDoubleMap.value.toString() ]); - const key1Types = similarTypes1.map((type) => formatType(definitions, type, imports)).join(' | '); - const key2Types = similarTypes2.map((type) => formatType(definitions, type, imports)).join(' | '); + const key1Types = similarTypes1.map((type) => formatType(allDefs, type, imports)).join(' | '); + const key2Types = similarTypes2.map((type) => formatType(allDefs, type, imports)).join(' | '); return [ `key1: ${key1Types}, key2: ${key2Types}`, - formatType(definitions, addModifier(storageEntry, storageEntry.type.asDoubleMap.value.toString()), imports) + formatType(allDefs, addModifier(storageEntry, storageEntry.type.asDoubleMap.value.toString()), imports) ]; } @@ -69,8 +69,8 @@ function entrySignature (definitions: object, registry: Registry, storageEntry: // Generate types for one storage entry in a module /** @internal */ -function generateEntry (definitions: object, registry: Registry, storageEntry: StorageEntryMetadataLatest, imports: TypeImports): string[] { - const [args, returnType] = entrySignature(definitions, registry, storageEntry, imports); +function generateEntry (allDefs: object, registry: Registry, storageEntry: StorageEntryMetadataLatest, imports: TypeImports): string[] { + const [args, returnType] = entrySignature(allDefs, registry, storageEntry, imports); return [ `${stringLowerFirst(storageEntry.name.toString())}: StorageEntryExact Observable<${returnType}>> & QueryableStorageEntry;` @@ -79,7 +79,7 @@ function generateEntry (definitions: object, registry: Registry, storageEntry: S // Generate types for one module /** @internal */ -function generateModule (definitions: object, registry: Registry, modul: ModuleMetadataLatest, imports: TypeImports): string[] { +function generateModule (allDefs: object, registry: Registry, modul: ModuleMetadataLatest, imports: TypeImports): string[] { if (modul.storage.isNone) { return []; } @@ -89,7 +89,7 @@ function generateModule (definitions: object, registry: Registry, modul: ModuleM .concat( modul.storage.unwrap().items .reduce((acc, storageEntry): string[] => { - return acc.concat(generateEntry(definitions, registry, storageEntry, imports).map(indent(6))); + return acc.concat(generateEntry(allDefs, registry, storageEntry, imports).map(indent(6))); }, [] as string[]) .join('\n') ) @@ -99,17 +99,19 @@ function generateModule (definitions: object, registry: Registry, modul: ModuleM // Generate `packages/api/src/query.types.ts` for a particular // metadata /** @internal */ -function generateForMeta (definitions: object, registry: Registry, meta: Metadata, dest: string): void { +function generateForMeta (registry: Registry, meta: Metadata, dest: string, extraTypes: Record>): void { console.log(`Writing ${dest}`); - const imports = createImports({ '@polkadot/types/interfaces': definitions }); // Will hold all needed imports - - const body = meta.asLatest.modules.reduce((acc, modul): string[] => { - const storageEntries = generateModule(definitions, registry, modul, imports); + const allTypes: Record> = { '@polkadot/types/interfaces': defaultDefs, ...extraTypes }; + const imports = createImports(allTypes); + const allDefs = Object.entries(allTypes).reduce((defs, [, obj]) => { + return Object.entries(obj).reduce((defs, [key, value]) => ({ ...defs, [key]: value }), defs); + }, {}); + const body = meta.asLatest.modules.reduce((acc, mod): string[] => { + const storageEntries = generateModule(allDefs, registry, mod, imports); return acc.concat(storageEntries); }, [] as string[]); - const header = createImportCode(HEADER, [ { file: 'rxjs', @@ -124,7 +126,7 @@ function generateForMeta (definitions: object, registry: Registry, meta: Metadat types: Object.keys(imports.primitiveTypes) }, ...Object.keys(imports.localTypes).map((moduleName): { file: string; types: string[] } => ({ - file: `@polkadot/types/interfaces/${moduleName}`, + file: `${imports.moduleToPackage[moduleName]}/${moduleName}`, types: Object.keys(imports.localTypes[moduleName]) })), { @@ -132,12 +134,10 @@ function generateForMeta (definitions: object, registry: Registry, meta: Metadat types: Object.keys(imports.typesTypes) } ]); - - const interfaceStart = - [ - "declare module '@polkadot/api/types' {", - indent(2)('export interface QueryableStorageExact {\n') - ].join('\n'); + const interfaceStart = [ + "declare module '@polkadot/api/types' {", + indent(2)('export interface QueryableStorageExact {\n') + ].join('\n'); const interfaceEnd = `\n${indent(2)('}')}\n}`; fs.writeFileSync( @@ -153,8 +153,8 @@ function generateForMeta (definitions: object, registry: Registry, meta: Metadat // Call `generateForMeta()` with current static metadata /** @internal */ -export default function generateQuery (dest = 'packages/api/src/query.types.ts', data = staticData): void { +export default function generateQuery (dest = 'packages/api/src/query.types.ts', data = staticData, extraTypes: Record> = {}): void { const registry = new TypeRegistry(); - return generateForMeta(defaultDefinitions, registry, new Metadata(registry, data), dest); + return generateForMeta(registry, new Metadata(registry, data), dest, extraTypes); } diff --git a/packages/types/src/scripts/generateTypes/tsDef.ts b/packages/types/src/scripts/generateTypes/tsDef.ts index fad9d8232b..fc088dc036 100644 --- a/packages/types/src/scripts/generateTypes/tsDef.ts +++ b/packages/types/src/scripts/generateTypes/tsDef.ts @@ -209,6 +209,7 @@ function generateTsDefFor (importDefinitions: { [importPath: string]: object }, /** @internal */ export function generateTsDef (importDefinitions: { [importPath: string]: object }, outputDir: string, generatingPackage: string): void { const definitions = importDefinitions[generatingPackage]; + Object.entries(definitions).forEach(([defName, obj]): void => { console.log(`Extracting interfaces for ${defName}`); diff --git a/packages/types/src/scripts/polkadot-types-from-chain.js b/packages/types/src/scripts/polkadot-types-from-chain.js index 7371376e45..549e074f63 100755 --- a/packages/types/src/scripts/polkadot-types-from-chain.js +++ b/packages/types/src/scripts/polkadot-types-from-chain.js @@ -26,7 +26,7 @@ const { formatNumber } = require('@polkadot/util'); const generateConst = require('./generateTypes/consts').default; const generateQuery = require('./generateTypes/query').default; -const { endpoint, output } = yargs.strict().options({ +const { endpoint, output, package } = yargs.strict().options({ endpoint: { description: 'The endpoint to connect to, e.g. wss://kusama-rpc.polkadot.io', type: 'string', @@ -36,6 +36,9 @@ const { endpoint, output } = yargs.strict().options({ description: 'The target directory to write the data to', type: 'string', required: true + }, + package: { + description: 'Optional package in output location (for extra definitions)' } }).argv; let websocket = null; @@ -63,11 +66,14 @@ function onSocketOpen () { function onSocketMessage (message) { const data = JSON.parse(message.data); const metaHex = data.result; + const extraTypes = package + ? { [package]: require(path.join(process.cwd(), output, 'definitions')) } + : {}; console.log(`Received metadata, ${formatNumber((metaHex.length - 2) / 2)} bytes`); - generateConst(path.join(process.cwd(), output, 'consts.types.ts'), metaHex); - generateQuery(path.join(process.cwd(), output, 'query.types.ts'), metaHex); + generateConst(path.join(process.cwd(), output, 'consts.types.ts'), metaHex, extraTypes); + generateQuery(path.join(process.cwd(), output, 'query.types.ts'), metaHex, extraTypes); process.exit(0); } diff --git a/packages/types/src/scripts/util/imports.ts b/packages/types/src/scripts/util/imports.ts index b4d65cd1bb..b1e385385d 100644 --- a/packages/types/src/scripts/util/imports.ts +++ b/packages/types/src/scripts/util/imports.ts @@ -29,7 +29,7 @@ export interface TypeImports { // Maps the types as found to the source location. This is used to generate the // imports in the output file, dep-duped and sorted /** @internal */ -export function setImports (definitions: object, imports: TypeImports, types: string[]): void { +export function setImports (allDefs: object, imports: TypeImports, types: string[]): void { const { codecTypes, localTypes, ignoredTypes, primitiveTypes, typesTypes } = imports; types.forEach((type): void => { @@ -45,18 +45,19 @@ export function setImports (definitions: object, imports: TypeImports, types: st // If the type is a bit special (tuple, fixed u8, nested type...), then we // need to parse it with `getTypeDef`. const typeDef = getTypeDef(type); - setImports(definitions, imports, [TypeDefInfo[typeDef.info]]); + + setImports(allDefs, imports, [TypeDefInfo[typeDef.info]]); // TypeDef.sub is a `TypeDef | TypeDef[]` if (Array.isArray(typeDef.sub)) { - typeDef.sub.forEach((subType): void => setImports(definitions, imports, [subType.type])); + typeDef.sub.forEach((subType): void => setImports(allDefs, imports, [subType.type])); } else if (typeDef.sub) { // typeDef.sub is a TypeDef in this case - setImports(definitions, imports, [typeDef.sub.type]); + setImports(allDefs, imports, [typeDef.sub.type]); } } else { // find this module inside the exports from the rest - const [moduleName] = Object.entries(definitions).find(([, { types }]): boolean => + const [moduleName] = Object.entries(allDefs).find(([, { types }]): boolean => Object.keys(types).includes(type) ) || [null]; @@ -69,37 +70,42 @@ export function setImports (definitions: object, imports: TypeImports, types: st // Create an Imports object, can be prefilled with `ignoredTypes` /** @internal */ -export function createImports (importDefinitions: { [importPath: string]: object }, { types }: { types: Record } = { types: {} }): TypeImports { +export function createImports (importDefinitions: Record, { types }: { types: Record } = { types: {} }): TypeImports { const definitions = {} as Record; const typeToModule = {} as Record; const moduleToPackage = {} as Record; - for (const [packagePath, packageDef] of Object.entries(importDefinitions)) { - for (const [name, moduleDef] of Object.entries(packageDef)) { + + Object.entries(importDefinitions).forEach(([packagePath, packageDef]): void => { + Object.entries(packageDef).forEach(([name, moduleDef]): void => { if (definitions[name]) { throw new Error(`Duplicated module: ${name}. Packages: ${packagePath}, ${moduleToPackage[name]}`); } + definitions[name] = moduleDef; moduleToPackage[name] = packagePath; - for (const type of Object.keys(moduleDef.types)) { + + Object.keys(moduleDef.types).forEach((type): void => { if (typeToModule[type]) { throw new Error(`Duplicated type: ${type}. Modules: ${name}, ${typeToModule[type]}`); } + typeToModule[type] = name; - } - } - } + }); + }); + }); - const codecTypes: TypeExist = {}; - const localTypes: TypeExistMap = Object.keys(definitions).reduce((localTypes: Record, moduleName): Record => { - localTypes[moduleName] = {}; + return { + codecTypes: {}, + localTypes: Object.keys(definitions).reduce((local: Record, mod): Record => { + local[mod] = {}; - return localTypes; - }, {}); - const ignoredTypes = Object.keys(types); - const primitiveTypes: TypeExist = {}; - const typesTypes: TypeExist = {}; - - const imports = { codecTypes, localTypes, ignoredTypes, primitiveTypes, typesTypes, definitions, typeToModule, moduleToPackage }; - - return imports; + return local; + }, {}), + ignoredTypes: Object.keys(types), + primitiveTypes: {}, + typesTypes: {}, + definitions, + typeToModule, + moduleToPackage + }; }