Better performant per-account staking queries (#1652)

* Better performant per-account staking queries

* Rename retrieval (no confusion)

* staking.info -> staking.account

* Update .info test
This commit is contained in:
Jaco Greeff
2019-12-09 10:14:51 +01:00
committed by GitHub
parent 48ef5af93f
commit 8a5a86e8b3
7 changed files with 150 additions and 132 deletions
+1 -1
View File
@@ -65,7 +65,7 @@ describe('derive', (): void => {
testFunction(api)('session', 'eraProgress', []);
testFunction(api)('session', 'sessionProgress', []);
testFunction(api)('staking', 'info', []);
testFunction(api)('staking', 'account', []);
testFunction(api)('staking', 'recentlyOffline', []);
testFunction(api)('staking', 'controllers', []);
});
+100
View File
@@ -0,0 +1,100 @@
// Copyright 2017-2019 @polkadot/api-derive authors & contributors
// This software may be modified and distributed under the terms
// of the Apache-2.0 license. See the LICENSE file for details.
import { ApiInterfaceRx } from '@polkadot/api/types';
import { Balance, BlockNumber, StakingLedger, UnlockChunk } from '@polkadot/types/interfaces';
import { DerivedSessionInfo, DerivedStakingAccount, DerivedStakingQuery, DerivedUnlocking } from '../types';
import BN from 'bn.js';
import { combineLatest, Observable } from 'rxjs';
import { map } from 'rxjs/operators';
import { createType } from '@polkadot/types';
import { isUndefined } from '@polkadot/util';
import { memo } from '../util';
// groups the supplied chunks by era, i.e. { [era]: BN(total of values) }
function groupByEra (list: UnlockChunk[]): Record<string, BN> {
return list.reduce((map: Record<string, BN>, { era, value }): Record<string, BN> => {
const key = era.toString();
map[key] = !map[key]
? value.unwrap()
: map[key].add(value.unwrap());
return map;
}, {});
}
// calculate the remaining blocks in a specific unlock era
function remainingBlocks (api: ApiInterfaceRx, era: BN, sessionInfo: DerivedSessionInfo): BlockNumber {
const remaining = era.sub(sessionInfo.currentEra);
// on the Rust side the current-era >= era-for-unlock (removal done on >)
return createType(api.registry, 'BlockNumber', remaining.gtn(0)
? remaining
.subn(1)
.mul(sessionInfo.eraLength)
.add(sessionInfo.eraLength.sub(sessionInfo.eraProgress))
: 0
);
}
function calculateUnlocking (api: ApiInterfaceRx, stakingLedger: StakingLedger | undefined, sessionInfo: DerivedSessionInfo): DerivedUnlocking[] | undefined {
if (isUndefined(stakingLedger)) {
return undefined;
}
const unlockingChunks = stakingLedger.unlocking.filter(({ era }): boolean =>
remainingBlocks(api, era.unwrap(), sessionInfo).gtn(0)
);
if (!unlockingChunks.length) {
return undefined;
}
// group the unlock chunks that have the same era and sum their values
const groupedResult = groupByEra(unlockingChunks);
const results = Object.entries(groupedResult).map(([eraString, value]): DerivedUnlocking => ({
value: createType(api.registry, 'Balance', value),
remainingBlocks: remainingBlocks(api, new BN(eraString), sessionInfo)
}));
return results.length ? results : undefined;
}
function redeemableSum (api: ApiInterfaceRx, stakingLedger: StakingLedger | undefined, sessionInfo: DerivedSessionInfo): Balance {
if (isUndefined(stakingLedger)) {
return createType(api.registry, 'Balance');
}
return createType(api.registry, 'Balance', stakingLedger.unlocking.reduce((total, { era, value }): BN => {
return remainingBlocks(api, era.unwrap(), sessionInfo).eqn(0)
? total.add(value.unwrap())
: total;
}, new BN(0)));
}
function parseResult (api: ApiInterfaceRx, sessionInfo: DerivedSessionInfo, query: DerivedStakingQuery): DerivedStakingAccount {
return {
...query,
redeemable: redeemableSum(api, query.stakingLedger, sessionInfo),
unlocking: calculateUnlocking(api, query.stakingLedger, sessionInfo)
};
}
/**
* @description From a stash, retrieve the controllerId and fill in all the relevant staking details
*/
export function account (api: ApiInterfaceRx): (accountId: Uint8Array | string) => Observable<DerivedStakingAccount> {
return memo((accountId: Uint8Array | string): Observable<DerivedStakingAccount> =>
combineLatest([
api.derive.session.info(),
api.derive.staking.query(accountId)
]).pipe(
map(([sessionInfo, query]: [DerivedSessionInfo, DerivedStakingQuery]) =>
parseResult(api, sessionInfo, query))
));
}
@@ -4,7 +4,7 @@
import { ApiInterfaceRx } from '@polkadot/api/types';
import { AccountId } from '@polkadot/types/interfaces';
import { DerivedStaking, DerivedStakingElected } from '../types';
import { DerivedStakingQuery, DerivedStakingElected } from '../types';
import { Observable, combineLatest, of } from 'rxjs';
import { map, switchMap } from 'rxjs/operators';
@@ -14,11 +14,11 @@ import { memo } from '../util';
export function electedInfo (api: ApiInterfaceRx): () => Observable<DerivedStakingElected> {
return memo((): Observable<DerivedStakingElected> =>
api.derive.staking.validators().pipe(
switchMap(({ currentElected }): Observable<[AccountId[], DerivedStaking[]]> =>
switchMap(({ currentElected }): Observable<[AccountId[], DerivedStakingQuery[]]> =>
combineLatest([
of(currentElected),
combineLatest(currentElected.map((accountId): Observable<DerivedStaking> =>
api.derive.staking.info(accountId)
combineLatest(currentElected.map((accountId): Observable<DerivedStakingQuery> =>
api.derive.staking.query(accountId)
))
])
),
+2 -1
View File
@@ -2,9 +2,10 @@
// This software may be modified and distributed under the terms
// of the Apache-2.0 license. See the LICENSE file for details.
export * from './account';
export * from './controllers';
export * from './electedInfo';
export * from './info';
export * from './overview';
export * from './query';
export * from './recentlyOffline';
export * from './validators';
@@ -3,27 +3,22 @@
// of the Apache-2.0 license. See the LICENSE file for details.
import { ApiInterfaceRx } from '@polkadot/api/types';
import { AccountId, Balance, BlockNumber, Exposure, Keys, Nominations, RewardDestination, StakingLedger, UnlockChunk, ValidatorPrefs } from '@polkadot/types/interfaces';
import { AccountId, Exposure, Keys, Nominations, RewardDestination, StakingLedger, ValidatorPrefs } from '@polkadot/types/interfaces';
import { ITuple } from '@polkadot/types/types';
import { DerivedRecentlyOffline, DerivedSessionInfo, DerivedStaking, DerivedUnlocking } from '../types';
import { DerivedStakingQuery } from '../types';
import BN from 'bn.js';
import { combineLatest, Observable, of } from 'rxjs';
import { map, switchMap } from 'rxjs/operators';
import { createType, Option, Vec } from '@polkadot/types';
import { isUndefined } from '@polkadot/util';
import { memo } from '../util';
interface ParseInput {
accountId: AccountId;
controllerId: AccountId;
recentlyOffline?: DerivedRecentlyOffline;
nominators: AccountId[];
rewardDestination: RewardDestination;
queuedKeys?: [AccountId, Keys][];
sessionInfo: DerivedSessionInfo;
stakers: Exposure;
stakingLedger: Option<StakingLedger>;
stashId: AccountId;
@@ -32,68 +27,6 @@ interface ParseInput {
nextKeyFor?: Option<AccountId>;
}
// groups the supplied chunks by era, i.e. { [era]: BN(total of values) }
function groupByEra (list: UnlockChunk[]): Record<string, BN> {
return list.reduce((map: Record<string, BN>, { era, value }): Record<string, BN> => {
const key = era.toString();
map[key] = !map[key]
? value.unwrap()
: map[key].add(value.unwrap());
return map;
}, {});
}
// calculate the remaining blocks in a specific unlock era
function remainingBlocks (api: ApiInterfaceRx, era: BN, sessionInfo: DerivedSessionInfo): BlockNumber {
const remaining = era.sub(sessionInfo.currentEra);
// on the Rust side the current-era >= era-for-unlock (removal done on >)
return createType(api.registry, 'BlockNumber', remaining.gtn(0)
? remaining
.subn(1)
.mul(sessionInfo.eraLength)
.add(sessionInfo.eraLength.sub(sessionInfo.eraProgress))
: 0
);
}
function calculateUnlocking (api: ApiInterfaceRx, stakingLedger: StakingLedger | undefined, sessionInfo: DerivedSessionInfo): DerivedUnlocking[] | undefined {
if (isUndefined(stakingLedger)) {
return undefined;
}
const unlockingChunks = stakingLedger.unlocking.filter(({ era }): boolean =>
remainingBlocks(api, era.unwrap(), sessionInfo).gtn(0)
);
if (!unlockingChunks.length) {
return undefined;
}
// group the unlock chunks that have the same era and sum their values
const groupedResult = groupByEra(unlockingChunks);
const results = Object.entries(groupedResult).map(([eraString, value]): DerivedUnlocking => ({
value: createType(api.registry, 'Balance', value),
remainingBlocks: remainingBlocks(api, new BN(eraString), sessionInfo)
}));
return results.length ? results : undefined;
}
function redeemableSum (api: ApiInterfaceRx, stakingLedger: StakingLedger | undefined, sessionInfo: DerivedSessionInfo): Balance {
if (isUndefined(stakingLedger)) {
return createType(api.registry, 'Balance');
}
return createType(api.registry, 'Balance', stakingLedger.unlocking.reduce((total, { era, value }): BN => {
return remainingBlocks(api, era.unwrap(), sessionInfo).eqn(0)
? total.add(value.unwrap())
: total;
}, new BN(0)));
}
function unwrapSessionIds (stashId: AccountId, queuedKeys: Option<AccountId> | [AccountId, Keys][], nextKeys: Option<Keys>): { nextSessionIds: AccountId[]; sessionIds: AccountId[] } {
// for 2.x we have a Vec<(ValidatorId,Keys)> of the keys
if (Array.isArray(queuedKeys)) {
@@ -119,7 +52,7 @@ function unwrapSessionIds (stashId: AccountId, queuedKeys: Option<AccountId> | [
};
}
function parseResult (api: ApiInterfaceRx, { accountId, controllerId, stashId, sessionInfo, recentlyOffline = {}, nextKeyFor, queuedKeys, stakingLedger, nominators, rewardDestination, stakers, validatorPrefs, nextKeys }: ParseInput): DerivedStaking {
function parseResult (api: ApiInterfaceRx, { accountId, controllerId, stashId, nextKeyFor, queuedKeys, stakingLedger, nominators, rewardDestination, stakers, validatorPrefs, nextKeys }: ParseInput): DerivedStakingQuery {
const _stakingLedger = stakingLedger.unwrapOr(undefined);
const _nextKeyFor = nextKeyFor || createType(api.registry, 'Option<AccountId>', null);
const _nextKeys = nextKeys || createType(api.registry, 'Option<Keys>', null);
@@ -128,13 +61,10 @@ function parseResult (api: ApiInterfaceRx, { accountId, controllerId, stashId, s
accountId,
controllerId,
nominators,
offline: recentlyOffline[stashId.toString()],
redeemable: redeemableSum(api, _stakingLedger, sessionInfo),
rewardDestination,
stakers,
stakingLedger: _stakingLedger,
stashId,
unlocking: calculateUnlocking(api, _stakingLedger, sessionInfo),
validatorPrefs,
...unwrapSessionIds(stashId, queuedKeys || _nextKeyFor, _nextKeys)
};
@@ -142,10 +72,8 @@ function parseResult (api: ApiInterfaceRx, { accountId, controllerId, stashId, s
type MultiResultV1 = [Option<AccountId>, Option<StakingLedger>, [Vec<AccountId>], RewardDestination, Exposure, ITuple<[ValidatorPrefs]>];
function retrieveInfoV1 (api: ApiInterfaceRx, accountId: AccountId, stashId: AccountId, controllerId: AccountId): Observable<DerivedStaking> {
return combineLatest([
api.derive.session.info(),
api.derive.staking.recentlyOffline(),
function retrieveInfoV1 (api: ApiInterfaceRx, accountId: AccountId, stashId: AccountId, controllerId: AccountId): Observable<DerivedStakingQuery> {
return (
api.queryMulti([
[api.query.session.nextKeyFor, controllerId],
[api.query.staking.ledger, controllerId],
@@ -154,18 +82,17 @@ function retrieveInfoV1 (api: ApiInterfaceRx, accountId: AccountId, stashId: Acc
[api.query.staking.stakers, stashId],
[api.query.staking.validators, stashId]
]) as Observable<MultiResultV1>
]).pipe(map(([sessionInfo, recentlyOffline, [nextKeyFor, stakingLedger, [nominators], rewardDestination, stakers, [validatorPrefs]]]: [DerivedSessionInfo, DerivedRecentlyOffline, MultiResultV1]): DerivedStaking =>
).pipe(map(([nextKeyFor, stakingLedger, [nominators], rewardDestination, stakers, [validatorPrefs]]: MultiResultV1): DerivedStakingQuery =>
parseResult(api, {
accountId, controllerId, stashId, sessionInfo, recentlyOffline, nextKeyFor, stakingLedger, nominators, rewardDestination, stakers, validatorPrefs
accountId, controllerId, stashId, nextKeyFor, stakingLedger, nominators, rewardDestination, stakers, validatorPrefs
})
));
}
type MultiResultV2 = [[Vec<AccountId>] | Option<ITuple<[Nominations]>>, RewardDestination, Exposure, [ValidatorPrefs], Option<Keys>, Option<StakingLedger>];
function retrieveInfoV2 (api: ApiInterfaceRx, accountId: AccountId, stashId: AccountId, controllerId: AccountId): Observable<DerivedStaking> {
function retrieveInfoV2 (api: ApiInterfaceRx, accountId: AccountId, stashId: AccountId, controllerId: AccountId): Observable<DerivedStakingQuery> {
return combineLatest([
api.derive.session.info(),
api.query.session.queuedKeys<Vec<ITuple<[AccountId, Keys]>>>(),
api.queryMulti([
[api.query.staking.nominators, stashId],
@@ -175,7 +102,7 @@ function retrieveInfoV2 (api: ApiInterfaceRx, accountId: AccountId, stashId: Acc
[api.query.session.nextKeys, [api.consts.session.dedupKeyPrefix, stashId]],
[api.query.staking.ledger, controllerId]
]) as Observable<MultiResultV2>
]).pipe(map(([sessionInfo, queuedKeys, [_nominators, rewardDestination, stakers, [validatorPrefs], nextKeys, stakingLedger]]: [DerivedSessionInfo, [AccountId, Keys][], MultiResultV2]): DerivedStaking => {
]).pipe(map(([queuedKeys, [_nominators, rewardDestination, stakers, [validatorPrefs], nextKeys, stakingLedger]]: [[AccountId, Keys][], MultiResultV2]): DerivedStakingQuery => {
// if we have staking.storageVersion it indicates the new structure, unwrap as needed
// FIXME We really want to be pulling all the new (valuable) info along
const nominators: AccountId[] = api.query.staking.storageVersion
@@ -185,17 +112,17 @@ function retrieveInfoV2 (api: ApiInterfaceRx, accountId: AccountId, stashId: Acc
: (_nominators as [Vec<AccountId>])[0];
return parseResult(api, {
accountId, controllerId, stashId, sessionInfo, queuedKeys, stakingLedger, nominators, rewardDestination, stakers, validatorPrefs, nextKeys
accountId, controllerId, stashId, queuedKeys, stakingLedger, nominators, rewardDestination, stakers, validatorPrefs, nextKeys
});
}));
}
function retrieveV1 (api: ApiInterfaceRx, accountId: AccountId): Observable<DerivedStaking> {
function retrieveV1 (api: ApiInterfaceRx, accountId: AccountId): Observable<DerivedStakingQuery> {
// depending on where we come from, this may be a controller or stash
return combineLatest([
api.query.staking.bonded<Option<AccountId>>(accountId),
api.query.staking.ledger<Option<StakingLedger>>(accountId)
]).pipe(switchMap(([bonded, stakingLedger]): Observable<DerivedStaking> =>
]).pipe(switchMap(([bonded, stakingLedger]): Observable<DerivedStakingQuery> =>
stakingLedger.isSome
? retrieveInfoV1(api, accountId, stakingLedger.unwrap().stash, accountId)
: bonded.isSome
@@ -204,10 +131,10 @@ function retrieveV1 (api: ApiInterfaceRx, accountId: AccountId): Observable<Deri
));
}
function retrieveV2 (api: ApiInterfaceRx, stashId: AccountId): Observable<DerivedStaking> {
function retrieveV2 (api: ApiInterfaceRx, stashId: AccountId): Observable<DerivedStakingQuery> {
return api.query.staking
.bonded<Option<AccountId>>(stashId)
.pipe(switchMap((controllerId): Observable<DerivedStaking> =>
.pipe(switchMap((controllerId): Observable<DerivedStakingQuery> =>
controllerId.isSome
? retrieveInfoV2(api, stashId, stashId, controllerId.unwrap())
: of({ accountId: stashId, nextSessionIds: [], sessionIds: [] })
@@ -217,11 +144,11 @@ function retrieveV2 (api: ApiInterfaceRx, stashId: AccountId): Observable<Derive
/**
* @description From a stash, retrieve the controllerId and fill in all the relevant staking details
*/
export function info (api: ApiInterfaceRx): (_accountId: Uint8Array | string) => Observable<DerivedStaking> {
const query = api.consts.session
export function query (api: ApiInterfaceRx): (_accountId: Uint8Array | string) => Observable<DerivedStakingQuery> {
const retrieve = api.consts.session
? retrieveV2
: retrieveV1;
return memo((accountId: Uint8Array | string): Observable<DerivedStaking> =>
query(api, createType(api.registry, 'AccountId', accountId)));
return memo((accountId: Uint8Array | string): Observable<DerivedStakingQuery> =>
retrieve(api, createType(api.registry, 'AccountId', accountId)));
}
+10 -19
View File
@@ -126,24 +126,9 @@ export interface DerivedSessionInfo extends DeriveSessionIndexes {
sessionProgress: BlockNumber;
}
export type DerivedStakingAccount = [AccountId, DerivedStakingOnlineStatus];
export type DerivedStakingAccounts = DerivedStakingAccount[];
export interface DerivedStakingElected {
currentElected: AccountId[];
info: DerivedStaking[];
}
export interface DerivedStakingOnlineStatus {
online?: {
isOnline: boolean;
blockNumber?: BlockNumber;
};
offline?: {
blockNumber: BlockNumber;
count: BN;
}[];
info: DerivedStakingQuery[];
}
export interface DeriveStakingValidators {
@@ -161,12 +146,15 @@ export interface DerivedStakingStash {
validatorPrefs?: ValidatorPrefs;
}
export interface DerivedStaking extends DerivedStakingOnlineStatus, DerivedStakingStash {
export interface DerivedStakingQuery extends DerivedStakingStash {
accountId: AccountId;
nextSessionIds: AccountId[];
redeemable?: Balance;
sessionIds: AccountId[];
stakingLedger?: StakingLedger;
}
export interface DerivedStakingAccount extends DerivedStakingQuery {
redeemable?: Balance;
unlocking?: DerivedUnlocking[];
}
@@ -176,7 +164,10 @@ export interface DerivedStakingOverview extends DeriveSessionIndexes {
validators: AccountId[];
}
export type DerivedUnlocking = { remainingBlocks: BlockNumber; value: Balance };
export type DerivedUnlocking = {
remainingBlocks: BlockNumber;
value: Balance;
};
export interface VoterPosition {
globalIndex: BN;
@@ -6,7 +6,7 @@ import ApiPromise from '@polkadot/api/promise/Api';
import testingPairs from '@polkadot/keyring/testingPairs';
import { WsProvider } from '@polkadot/rpc-provider';
import { SubmittableResult } from '@polkadot/api';
import { DerivedBalances, DerivedStaking } from '@polkadot/api-derive/types';
import { DerivedBalances, DerivedStakingAccount } from '@polkadot/api-derive/types';
import { describeE2E } from '../../util';
@@ -49,7 +49,7 @@ describeE2E({
it('retrieves all staking info (for controller)', (done): Promise<() => void> => {
const accountId = '5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY';
return api.derive.staking.info(accountId, (info: DerivedStaking): void => {
return api.derive.staking.account(accountId, (info: DerivedStakingAccount): void => {
console.error(JSON.stringify(info));
expect(info.accountId.eq(accountId)).toBe(true);
@@ -64,7 +64,7 @@ describeE2E({
it('retrieves all staking info (for stash)', (done): Promise<() => void> => {
const accountId = '5GNJqTPyNqANBkUVMN1LPPrxXnFouWXoe2wNSmmEoLctxiZY';
return api.derive.staking.info(accountId, (info: DerivedStaking): void => {
return api.derive.staking.account(accountId, (info: DerivedStakingAccount): void => {
console.error(JSON.stringify(info));
if (!info.stashId || !info.controllerId || !info.stakingLedger) {
@@ -95,7 +95,7 @@ describeE2E({
});
it('verifies that derive.staking.unlocking isn\'t empty/undefined', (): Promise<() => void> => {
return api.derive.staking.info(ALICE_STASH, (info: DerivedStaking): void => {
return api.derive.staking.account(ALICE_STASH, (info: DerivedStakingAccount): void => {
expect(info.unlocking).toBeDefined();
});
});
@@ -116,7 +116,7 @@ describeE2E({
});
it('verifies payee for ALICE_STASH', (done): Promise<() => void> => {
return api.derive.staking.info(ALICE_STASH, (info: DerivedStaking): void => {
return api.derive.staking.account(ALICE_STASH, (info: DerivedStakingAccount): void => {
if (!info.rewardDestination) {
return done.fail(new Error('rewardDestination is undefined.'));
} else {
@@ -127,7 +127,7 @@ describeE2E({
});
});
it('staking.info updates itself after changing reward destination', async (done): Promise<void> => {
it('staking.account updates itself after changing reward destination', async (done): Promise<void> => {
const stashId = testingPairs().alice_stash.address;
// start by setting the reqred to Staked, so we have a common starting point
@@ -139,19 +139,18 @@ describeE2E({
}
});
let count = 0; // The # of times we got a callback response from api.derive.staking.info
let count = 0; // The # of times we got a callback response from api.derive.staking.account
// Subscribe to staking.info
api.derive.staking
.info(stashId, (result): void => {
++count;
// Subscribe to staking.account
api.derive.staking.account(stashId, (result): void => {
++count;
console.error('***', count, JSON.stringify(result));
console.error('***', count, JSON.stringify(result));
if (count >= 2 && result.rewardDestination!.toString() === 'Stash') {
done();
}
});
if (count >= 2 && result.rewardDestination!.toString() === 'Stash') {
done();
}
});
// Wait a bit, and change reward destination
// eslint-disable-next-line @typescript-eslint/no-misused-promises