quantus/api
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Allow chain to use user-defined types (#1769)

Browse Source 
* Allow chain to use user-defined types

* --package docs on chain extract
This commit is contained in:
Jaco Greeff
2020-01-29 17:37:19 +01:00
committed by GitHub
parent 70e5e2b60a
commit 4c328964d5
8 changed files with 94 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+1
View File
@@ -1,6 +1,7 @@
# 1.0.0-beta-x
- **Breaking change** Drop support for Substrate v1 chain in all derives
- Add `polkadot-types-from-chain` & `polkadot-types-from-defs` TypeScript generators (`@polkadot/types` bin)
- Only decorate derives where relevant parent `api.query.*` is available
- Support `.entries(arg?: any)` lookups on DoubleMaps (in addition to previously supported maps)
- Remove un-deployed support for v11 metadata (this was decided against on Substrate)
docs/start/typescript.user.md
+1 -1
View File
@@ -14,7 +14,7 @@ In the root of your project (with the `@polkadot/api`), you can run `yarn polkad
In the same way as the type library provides defaults from a substrate-base chain, you can also, directly from chain from metadata, generate a complete `api.{consts,query}.*` defintion for your specific chain. The command will create 2 files, `{consts, query}.types.ts` which you can either use to augment the TypeScript defintions, or replace those in `@polkadot/api/*` with your versions (copy, TypeScript replacement or browser/node aliassing).
In the root of your project, you can run `yarn polkadot-types-from-chain.js --endpoint wss://<url> --output ./stuff` and it will create the required output.
In the root of your project, you can run `yarn polkadot-types-from-chain.js --endpoint wss://<url> --output ./stuff` and it will create the required output. (Here you can specify an optional `--package @MeInc/stuff` to read definitions for the targetted output folder with the specified package name.)
## And that's a wrap
packages/types/src/scripts/generateTypes/consts.ts
+18 -18
View File
@@ -3,7 +3,7 @@
// of the Apache-2.0 license. See the LICENSE file for details.
import { ModuleMetadataLatest } from '../../interfaces/metadata';
import * as definitions from '../../interfaces/definitions';
import * as defaultDefs from '../../interfaces/definitions';
import fs from 'fs';
import staticData from '@polkadot/metadata/Metadata/static';
@@ -14,19 +14,19 @@ import { createImportCode, createImports, FOOTER, HEADER, indent, setImports, Ty
// Generate types for one module
/** @internal */
function generateModule (modul: ModuleMetadataLatest, imports: TypeImports): string[] {
function generateModule (allDefs: object, modul: ModuleMetadataLatest, imports: TypeImports): string[] {
if (!modul.constants.length) {
return [];
}
setImports(definitions, imports, ['Codec']);
setImports(allDefs, imports, ['Codec']);
return [indent(4)(`${stringCamelCase(modul.name.toString())}: {`)]
.concat(indent(6)('[index: string]: Codec;'))
.concat(
modul.constants
.map((constant): string => {
setImports(definitions, imports, [constant.type.toString()]);
setImports(allDefs, imports, [constant.type.toString()]);
return indent(6)(`${stringCamelCase(constant.name.toString())}: ${constant.type} & ConstantCodec;`);
})
@@ -37,17 +37,19 @@ function generateModule (modul: ModuleMetadataLatest, imports: TypeImports): str
// Generate `packages/api/src/consts.types.ts` for a particular
// metadata
/** @internal */
function generateForMeta (meta: Metadata, dest: string): void {
function generateForMeta (meta: Metadata, dest: string, extraTypes: Record<string, Record<string, object>>): void {
console.log(`Writing ${dest}`);
const imports = createImports({ '@polkadot/types/interfaces': definitions }); // Will hold all needed imports
const allTypes: Record<string, Record<string, object>> = { '@polkadot/types/interfaces': defaultDefs, ...extraTypes };
const imports = createImports(allTypes);
const allDefs = Object.entries(allTypes).reduce((defs, [, obj]) => {
return Object.entries(obj).reduce((defs, [key, value]) => ({ ...defs, [key]: value }), defs);
}, {});
const body = meta.asLatest.modules.reduce((acc, modul): string[] => {
const storageEntries = generateModule(modul, imports);
const storageEntries = generateModule(allDefs, modul, imports);
return acc.concat(storageEntries);
}, [] as string[]);
const header = createImportCode(HEADER, [
{
file: '@polkadot/types/codec',
@@ -58,7 +60,7 @@ function generateForMeta (meta: Metadata, dest: string): void {
types: Object.keys(imports.primitiveTypes)
},
...Object.keys(imports.localTypes).map((moduleName): { file: string; types: string[] } => ({
file: `@polkadot/types/interfaces/${moduleName}`,
file: `${imports.moduleToPackage[moduleName]}/${moduleName}`,
types: Object.keys(imports.localTypes[moduleName])
})),
{
@@ -66,12 +68,10 @@ function generateForMeta (meta: Metadata, dest: string): void {
types: Object.keys(imports.typesTypes)
}
]);
const interfaceStart =
[
"declare module '@polkadot/metadata/Decorated/types' {",
indent(2)('export interface Constants {\n')
].join('\n');
const interfaceStart = [
"declare module '@polkadot/metadata/Decorated/types' {",
indent(2)('export interface Constants {\n')
].join('\n');
const interfaceEnd = `\n${indent(2)('}')}\n}`;
fs.writeFileSync(
@@ -87,8 +87,8 @@ function generateForMeta (meta: Metadata, dest: string): void {
// Call `generateForMeta()` with current static metadata
/** @internal */
export default function generateConsts (dest = 'packages/api/src/consts.types.ts', data = staticData): void {
export default function generateConsts (dest = 'packages/api/src/consts.types.ts', data = staticData, extraTypes: Record<string, Record<string, object>> = {}): void {
const registry = new TypeRegistry();
return generateForMeta(new Metadata(registry, data), dest);
return generateForMeta(new Metadata(registry, data), dest, extraTypes);
}
packages/types/src/scripts/generateTypes/interfaceRegistry.ts
+1 -3
View File
@@ -82,9 +82,7 @@ export function generateInterfaceRegistry (importDefinitions: { [importPath: str
// Generate `packages/types/src/interfaceRegistry.ts`, the registry of all interfaces
export default function generateDefaultInterfaceRegistry (): void {
generateInterfaceRegistry(
{
'@polkadot/types/interfaces': defaultDefinitions
},
{ '@polkadot/types/interfaces': defaultDefinitions },
'packages/types/src/interfaceRegistry.ts'
);
}
packages/types/src/scripts/generateTypes/query.ts
+33 -33
View File
@@ -4,7 +4,7 @@
import { ModuleMetadataLatest, StorageEntryMetadataLatest } from '../../interfaces/metadata';
import { Registry } from '../../types';
import * as defaultDefinitions from '../../interfaces/definitions';
import * as defaultDefs from '../../interfaces/definitions';
import fs from 'fs';
import staticData from '@polkadot/metadata/Metadata/static';
@@ -26,41 +26,41 @@ function addModifier (storageEntry: StorageEntryMetadataLatest, returnType: stri
// From a storage entry metadata, we return [args, returnType]
/** @internal */
function entrySignature (definitions: object, registry: Registry, storageEntry: StorageEntryMetadataLatest, imports: TypeImports): [string, string] {
function entrySignature (allDefs: object, registry: Registry, storageEntry: StorageEntryMetadataLatest, imports: TypeImports): [string, string] {
if (storageEntry.type.isPlain) {
setImports(definitions, imports, [storageEntry.type.asPlain.toString()]);
setImports(allDefs, imports, [storageEntry.type.asPlain.toString()]);
return ['', formatType(definitions, addModifier(storageEntry, storageEntry.type.asPlain.toString()), imports)];
return ['', formatType(allDefs, addModifier(storageEntry, storageEntry.type.asPlain.toString()), imports)];
} else if (storageEntry.type.isMap) {
// Find similar types of the `key` type
const similarTypes = getSimilarTypes(definitions, registry, storageEntry.type.asMap.key.toString(), imports);
const similarTypes = getSimilarTypes(allDefs, registry, storageEntry.type.asMap.key.toString(), imports);
setImports(definitions, imports, [
setImports(allDefs, imports, [
...similarTypes,
storageEntry.type.asMap.value.toString()
]);
return [
`arg: ${similarTypes.map((type) => formatType(definitions, type, imports)).join(' | ')}`,
formatType(definitions, addModifier(storageEntry, storageEntry.type.asMap.value.toString()), imports)
`arg: ${similarTypes.map((type) => formatType(allDefs, type, imports)).join(' | ')}`,
formatType(allDefs, addModifier(storageEntry, storageEntry.type.asMap.value.toString()), imports)
];
} else if (storageEntry.type.isDoubleMap) {
// Find similartypes of `key1` and `key2` types
const similarTypes1 = getSimilarTypes(definitions, registry, storageEntry.type.asDoubleMap.key1.toString(), imports);
const similarTypes2 = getSimilarTypes(definitions, registry, storageEntry.type.asDoubleMap.key2.toString(), imports);
const similarTypes1 = getSimilarTypes(allDefs, registry, storageEntry.type.asDoubleMap.key1.toString(), imports);
const similarTypes2 = getSimilarTypes(allDefs, registry, storageEntry.type.asDoubleMap.key2.toString(), imports);
setImports(definitions, imports, [
setImports(allDefs, imports, [
...similarTypes1,
...similarTypes2,
storageEntry.type.asDoubleMap.value.toString()
]);
const key1Types = similarTypes1.map((type) => formatType(definitions, type, imports)).join(' | ');
const key2Types = similarTypes2.map((type) => formatType(definitions, type, imports)).join(' | ');
const key1Types = similarTypes1.map((type) => formatType(allDefs, type, imports)).join(' | ');
const key2Types = similarTypes2.map((type) => formatType(allDefs, type, imports)).join(' | ');
return [
`key1: ${key1Types}, key2: ${key2Types}`,
formatType(definitions, addModifier(storageEntry, storageEntry.type.asDoubleMap.value.toString()), imports)
formatType(allDefs, addModifier(storageEntry, storageEntry.type.asDoubleMap.value.toString()), imports)
];
}
@@ -69,8 +69,8 @@ function entrySignature (definitions: object, registry: Registry, storageEntry:
// Generate types for one storage entry in a module
/** @internal */
function generateEntry (definitions: object, registry: Registry, storageEntry: StorageEntryMetadataLatest, imports: TypeImports): string[] {
const [args, returnType] = entrySignature(definitions, registry, storageEntry, imports);
function generateEntry (allDefs: object, registry: Registry, storageEntry: StorageEntryMetadataLatest, imports: TypeImports): string[] {
const [args, returnType] = entrySignature(allDefs, registry, storageEntry, imports);
return [
`${stringLowerFirst(storageEntry.name.toString())}: StorageEntryExact<ApiType, (${args}) => Observable<${returnType}>> & QueryableStorageEntry<ApiType>;`
@@ -79,7 +79,7 @@ function generateEntry (definitions: object, registry: Registry, storageEntry: S
// Generate types for one module
/** @internal */
function generateModule (definitions: object, registry: Registry, modul: ModuleMetadataLatest, imports: TypeImports): string[] {
function generateModule (allDefs: object, registry: Registry, modul: ModuleMetadataLatest, imports: TypeImports): string[] {
if (modul.storage.isNone) {
return [];
}
@@ -89,7 +89,7 @@ function generateModule (definitions: object, registry: Registry, modul: ModuleM
.concat(
modul.storage.unwrap().items
.reduce((acc, storageEntry): string[] => {
return acc.concat(generateEntry(definitions, registry, storageEntry, imports).map(indent(6)));
return acc.concat(generateEntry(allDefs, registry, storageEntry, imports).map(indent(6)));
}, [] as string[])
.join('\n')
)
@@ -99,17 +99,19 @@ function generateModule (definitions: object, registry: Registry, modul: ModuleM
// Generate `packages/api/src/query.types.ts` for a particular
// metadata
/** @internal */
function generateForMeta (definitions: object, registry: Registry, meta: Metadata, dest: string): void {
function generateForMeta (registry: Registry, meta: Metadata, dest: string, extraTypes: Record<string, Record<string, object>>): void {
console.log(`Writing ${dest}`);
const imports = createImports({ '@polkadot/types/interfaces': definitions }); // Will hold all needed imports
const body = meta.asLatest.modules.reduce((acc, modul): string[] => {
const storageEntries = generateModule(definitions, registry, modul, imports);
const allTypes: Record<string, Record<string, object>> = { '@polkadot/types/interfaces': defaultDefs, ...extraTypes };
const imports = createImports(allTypes);
const allDefs = Object.entries(allTypes).reduce((defs, [, obj]) => {
return Object.entries(obj).reduce((defs, [key, value]) => ({ ...defs, [key]: value }), defs);
}, {});
const body = meta.asLatest.modules.reduce((acc, mod): string[] => {
const storageEntries = generateModule(allDefs, registry, mod, imports);
return acc.concat(storageEntries);
}, [] as string[]);
const header = createImportCode(HEADER, [
{
file: 'rxjs',
@@ -124,7 +126,7 @@ function generateForMeta (definitions: object, registry: Registry, meta: Metadat
types: Object.keys(imports.primitiveTypes)
},
...Object.keys(imports.localTypes).map((moduleName): { file: string; types: string[] } => ({
file: `@polkadot/types/interfaces/${moduleName}`,
file: `${imports.moduleToPackage[moduleName]}/${moduleName}`,
types: Object.keys(imports.localTypes[moduleName])
})),
{
@@ -132,12 +134,10 @@ function generateForMeta (definitions: object, registry: Registry, meta: Metadat
types: Object.keys(imports.typesTypes)
}
]);
const interfaceStart =
[
"declare module '@polkadot/api/types' {",
indent(2)('export interface QueryableStorageExact<ApiType> {\n')
].join('\n');
const interfaceStart = [
"declare module '@polkadot/api/types' {",
indent(2)('export interface QueryableStorageExact<ApiType> {\n')
].join('\n');
const interfaceEnd = `\n${indent(2)('}')}\n}`;
fs.writeFileSync(
@@ -153,8 +153,8 @@ function generateForMeta (definitions: object, registry: Registry, meta: Metadat
// Call `generateForMeta()` with current static metadata
/** @internal */
export default function generateQuery (dest = 'packages/api/src/query.types.ts', data = staticData): void {
export default function generateQuery (dest = 'packages/api/src/query.types.ts', data = staticData, extraTypes: Record<string, Record<string, object>> = {}): void {
const registry = new TypeRegistry();
return generateForMeta(defaultDefinitions, registry, new Metadata(registry, data), dest);
return generateForMeta(registry, new Metadata(registry, data), dest, extraTypes);
}
packages/types/src/scripts/generateTypes/tsDef.ts
+1
View File
@@ -209,6 +209,7 @@ function generateTsDefFor (importDefinitions: { [importPath: string]: object },
/** @internal */
export function generateTsDef (importDefinitions: { [importPath: string]: object }, outputDir: string, generatingPackage: string): void {
const definitions = importDefinitions[generatingPackage];
Object.entries(definitions).forEach(([defName, obj]): void => {
console.log(`Extracting interfaces for ${defName}`);
packages/types/src/scripts/polkadot-types-from-chain.js
+9 -3
View File
@@ -26,7 +26,7 @@ const { formatNumber } = require('@polkadot/util');
const generateConst = require('./generateTypes/consts').default;
const generateQuery = require('./generateTypes/query').default;
const { endpoint, output } = yargs.strict().options({
const { endpoint, output, package } = yargs.strict().options({
endpoint: {
description: 'The endpoint to connect to, e.g. wss://kusama-rpc.polkadot.io',
type: 'string',
@@ -36,6 +36,9 @@ const { endpoint, output } = yargs.strict().options({
description: 'The target directory to write the data to',
type: 'string',
required: true
},
package: {
description: 'Optional package in output location (for extra definitions)'
}
}).argv;
let websocket = null;
@@ -63,11 +66,14 @@ function onSocketOpen () {
function onSocketMessage (message) {
const data = JSON.parse(message.data);
const metaHex = data.result;
const extraTypes = package
? { [package]: require(path.join(process.cwd(), output, 'definitions')) }
: {};
console.log(`Received metadata, ${formatNumber((metaHex.length - 2) / 2)} bytes`);
generateConst(path.join(process.cwd(), output, 'consts.types.ts'), metaHex);
generateQuery(path.join(process.cwd(), output, 'query.types.ts'), metaHex);
generateConst(path.join(process.cwd(), output, 'consts.types.ts'), metaHex, extraTypes);
generateQuery(path.join(process.cwd(), output, 'query.types.ts'), metaHex, extraTypes);
process.exit(0);
}
packages/types/src/scripts/util/imports.ts
+30 -24
View File
@@ -29,7 +29,7 @@ export interface TypeImports {
// Maps the types as found to the source location. This is used to generate the
// imports in the output file, dep-duped and sorted
/** @internal */
export function setImports (definitions: object, imports: TypeImports, types: string[]): void {
export function setImports (allDefs: object, imports: TypeImports, types: string[]): void {
const { codecTypes, localTypes, ignoredTypes, primitiveTypes, typesTypes } = imports;
types.forEach((type): void => {
@@ -45,18 +45,19 @@ export function setImports (definitions: object, imports: TypeImports, types: st
// If the type is a bit special (tuple, fixed u8, nested type...), then we
// need to parse it with `getTypeDef`.
const typeDef = getTypeDef(type);
setImports(definitions, imports, [TypeDefInfo[typeDef.info]]);
setImports(allDefs, imports, [TypeDefInfo[typeDef.info]]);
// TypeDef.sub is a `TypeDef | TypeDef[]`
if (Array.isArray(typeDef.sub)) {
typeDef.sub.forEach((subType): void => setImports(definitions, imports, [subType.type]));
typeDef.sub.forEach((subType): void => setImports(allDefs, imports, [subType.type]));
} else if (typeDef.sub) {
// typeDef.sub is a TypeDef in this case
setImports(definitions, imports, [typeDef.sub.type]);
setImports(allDefs, imports, [typeDef.sub.type]);
}
} else {
// find this module inside the exports from the rest
const [moduleName] = Object.entries(definitions).find(([, { types }]): boolean =>
const [moduleName] = Object.entries(allDefs).find(([, { types }]): boolean =>
Object.keys(types).includes(type)
) || [null];
@@ -69,37 +70,42 @@ export function setImports (definitions: object, imports: TypeImports, types: st
// Create an Imports object, can be prefilled with `ignoredTypes`
/** @internal */
export function createImports (importDefinitions: { [importPath: string]: object }, { types }: { types: Record<string, any> } = { types: {} }): TypeImports {
export function createImports (importDefinitions: Record<string, object>, { types }: { types: Record<string, any> } = { types: {} }): TypeImports {
const definitions = {} as Record<string, object>;
const typeToModule = {} as Record<string, string>;
const moduleToPackage = {} as Record<string, string>;
for (const [packagePath, packageDef] of Object.entries(importDefinitions)) {
for (const [name, moduleDef] of Object.entries(packageDef)) {
Object.entries(importDefinitions).forEach(([packagePath, packageDef]): void => {
Object.entries(packageDef).forEach(([name, moduleDef]): void => {
if (definitions[name]) {
throw new Error(`Duplicated module: ${name}. Packages: ${packagePath}, ${moduleToPackage[name]}`);
}
definitions[name] = moduleDef;
moduleToPackage[name] = packagePath;
for (const type of Object.keys(moduleDef.types)) {
Object.keys(moduleDef.types).forEach((type): void => {
if (typeToModule[type]) {
throw new Error(`Duplicated type: ${type}. Modules: ${name}, ${typeToModule[type]}`);
}
typeToModule[type] = name;
}
}
}
});
});
});
const codecTypes: TypeExist = {};
const localTypes: TypeExistMap = Object.keys(definitions).reduce((localTypes: Record<string, TypeExist>, moduleName): Record<string, TypeExist> => {
localTypes[moduleName] = {};
return {
codecTypes: {},
localTypes: Object.keys(definitions).reduce((local: Record<string, TypeExist>, mod): Record<string, TypeExist> => {
local[mod] = {};
return localTypes;
}, {});
const ignoredTypes = Object.keys(types);
const primitiveTypes: TypeExist = {};
const typesTypes: TypeExist = {};
const imports = { codecTypes, localTypes, ignoredTypes, primitiveTypes, typesTypes, definitions, typeToModule, moduleToPackage };
return imports;
return local;
}, {}),
ignoredTypes: Object.keys(types),
primitiveTypes: {},
typesTypes: {},
definitions,
typeToModule,
moduleToPackage
};
}