Additional derives for new staking points & rewards (#1994)

* Additional derives for new staking points & rewards

* Fix method name

* StakingLedger update

* Expand derives

* Cleanups

* Cleanup Compact decode

* Cleanup derives

* Update CHANGELOG

* Split types

packages/api-derive/src/parachains/types.ts

@@ -18,3 +18,33 @@ export type RelayDispatchQueue = Vec<UpwardMessage>
 export type RelayDispatchQueueSize = ITuple<[u32, u32]>;
 export type Watermarks = Option<BlockNumber>;
 export type DidUpdate = Option<Vec<ParaId>>;
+
+export interface DeriveParachainActive {
+  collatorId: CollatorId;
+  isRetriable: boolean;
+  retries: number;
+}
+
+export interface DeriveParachainInfo extends ParaInfo {
+  id: ParaId;
+  icon?: string;
+  name?: string;
+  owner?: string;
+}
+
+export interface DeriveParachain {
+  didUpdate: boolean;
+  pendingSwapId: ParaId | null;
+  id: ParaId;
+  info: DeriveParachainInfo | null;
+  relayDispatchQueueSize?: number;
+  watermark: BlockNumber | null;
+}
+
+export interface DeriveParachainFull extends DeriveParachain {
+  active: DeriveParachainActive | null;
+  heads: Bytes | null;
+  relayDispatchQueue: UpwardMessage[];
+  retryCollators: (CollatorId | null)[];
+  selectedCollators: (CollatorId | null)[];
+}

packages/api-derive/src/session/indexes.ts

@@ -44,28 +44,32 @@ function queryNoActive (api: ApiInterfaceRx): Observable<Result> {
 
 // query based on latest
 function query (api: ApiInterfaceRx): Observable<Result> {
-  return api.queryMulti<[ActiveEraInfo, Option<EraIndex>, SessionIndex, u32]>([
+  return api.queryMulti<[Option<ActiveEraInfo>, Option<EraIndex>, SessionIndex, u32]>([
     api.query.staking.activeEra,
     api.query.staking.currentEra,
     api.query.session.currentIndex,
     api.query.staking.validatorCount
   ]).pipe(
-    map(([{ index: activeEra, start: activeEraStart }, currentEra, currentIndex, validatorCount]): Result => [
-      activeEra,
-      activeEraStart,
-      currentEra.unwrapOr(createType(api.registry, 'EraIndex', 1)),
-      currentIndex,
-      validatorCount
-    ])
+    map(([activeOpt, currentEra, currentIndex, validatorCount]): Result => {
+      const { index: activeEra, start: activeEraStart } = activeOpt.unwrapOrDefault();
+
+      return [
+        activeEra,
+        activeEraStart,
+        currentEra.unwrapOr(createType(api.registry, 'EraIndex')),
+        currentIndex,
+        validatorCount
+      ];
+    })
   );
 }
 
 // empty set when none is available
 function empty (api: ApiInterfaceRx): Observable<Result> {
   return of([
-    createType(api.registry, 'EraIndex', 1),
+    createType(api.registry, 'EraIndex'),
     createType(api.registry, 'Option<Moment>'),
-    createType(api.registry, 'EraIndex', 1),
+    createType(api.registry, 'EraIndex'),
     createType(api.registry, 'SessionIndex', 1),
     createType(api.registry, 'u32')
   ]);

packages/api-derive/src/session/types.ts

@@ -0,0 +1,24 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { BlockNumber, EraIndex, Moment, SessionIndex } from '@polkadot/types/interfaces';
+
+import { Option, u32 } from '@polkadot/types';
+
+export interface DeriveSessionIndexes {
+  activeEra: EraIndex;
+  activeEraStart: Option<Moment>;
+  currentEra: EraIndex;
+  currentIndex: SessionIndex;
+  validatorCount: u32;
+}
+
+export interface DerivedSessionInfo extends DeriveSessionIndexes {
+  eraLength: BlockNumber;
+  eraProgress: BlockNumber;
+  isEpoch: boolean;
+  sessionLength: BlockNumber;
+  sessionsPerEra: SessionIndex;
+  sessionProgress: BlockNumber;
+}

packages/api-derive/src/staking/erasExposure.ts

@@ -0,0 +1,56 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { EraIndex, Exposure } from '@polkadot/types/interfaces';
+import { DeriveEraExposure } from '../types';
+
+import { Observable, combineLatest, of } from 'rxjs';
+import { map, switchMap } from 'rxjs/operators';
+
+import { memo } from '../util';
+
+function queryClipped (api: ApiInterfaceRx): Observable<[[EraIndex, string][], Exposure[]]> {
+  return api.derive.staking.erasPoints().pipe(
+    switchMap((allPoints): Observable<[[EraIndex, string][], Exposure[]]> => {
+      const indexes: [EraIndex, string][] = [];
+
+      allPoints.forEach(({ all, era }): void => {
+        Object.keys(all).forEach((validatorId): void => {
+          indexes.push([era, validatorId]);
+        });
+      });
+
+      // FIXME use prefix queries, get going first
+      return combineLatest([
+        of(indexes),
+        indexes.length
+          ? api.query.staking.erasStakersClipped.multi<Exposure>(indexes)
+          : of([] as Exposure[])
+      ]);
+    })
+  );
+}
+
+export function erasExposure (api: ApiInterfaceRx): () => Observable<DeriveEraExposure[]> {
+  return memo((): Observable<DeriveEraExposure[]> =>
+    queryClipped(api).pipe(
+      map(([indexes, exposures]): DeriveEraExposure[] =>
+        indexes.reduce((result: DeriveEraExposure[], [era, validatorId], index): DeriveEraExposure[] => {
+          let entry = result.find((entry): boolean => entry.era.eq(era));
+
+          if (!entry) {
+            entry = { all: {}, era };
+
+            result.push(entry);
+          }
+
+          entry.all[validatorId] = exposures[index];
+
+          return result;
+        }, [])
+      )
+    )
+  );
+}

packages/api-derive/src/staking/erasPoints.ts

@@ -0,0 +1,65 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { ActiveEraInfo, EraIndex, EraRewardPoints, RewardPoint } from '@polkadot/types/interfaces';
+import { DeriveEraPointsAll } from '../types';
+
+import { Observable, combineLatest, of } from 'rxjs';
+import { map, switchMap } from 'rxjs/operators';
+import { Option, u32 } from '@polkadot/types';
+
+import { memo } from '../util';
+
+function getAvailableIndexes (api: ApiInterfaceRx): Observable<EraIndex[]> {
+  return api.query.staking?.activeEra
+    ? api.queryMulti<[Option<ActiveEraInfo>, u32]>([
+      api.query.staking.activeEra,
+      api.query.staking.historyDepth
+    ]).pipe(
+      map(([activeEraOpt, historyDepth]): EraIndex[] => {
+        const result: EraIndex[] = [];
+        const max = historyDepth.toNumber();
+        let lastEra = activeEraOpt.unwrapOrDefault().index.subn(1);
+
+        while (lastEra.gten(0) && result.length < max) {
+          result.push(api.registry.createType('EraIndex', lastEra));
+
+          lastEra = lastEra.subn(1);
+        }
+
+        // go from oldest to newest
+        return result.reverse();
+      })
+    )
+    : of([]);
+}
+
+export function erasPoints (api: ApiInterfaceRx): () => Observable<DeriveEraPointsAll[]> {
+  return memo((): Observable<DeriveEraPointsAll[]> =>
+    getAvailableIndexes(api).pipe(
+      switchMap((indexes): Observable<[EraIndex[], EraRewardPoints[]]> =>
+        combineLatest([
+          of(indexes),
+          indexes.length
+            ? api.query.staking.erasRewardPoints.multi<EraRewardPoints>(indexes)
+            : of([])
+        ])
+      ),
+      map(([eras, rewards]): DeriveEraPointsAll[] =>
+        eras.map((era, index): DeriveEraPointsAll => ({
+          all: [...rewards[index].individual.entries()]
+            .filter(([, points]): boolean => points.gtn(0))
+            .reduce((all: Record<string, RewardPoint>, [validatorId, points]): Record<string, RewardPoint> => {
+              all[validatorId.toString()] = points;
+
+              return all;
+            }, {}),
+          era,
+          total: rewards[index].total
+        }))
+      )
+    )
+  );
+}

packages/api-derive/src/staking/erasRewards.ts

@@ -0,0 +1,36 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { DeriveEraRewardsAll } from '../types';
+
+import { Observable } from 'rxjs';
+import { map } from 'rxjs/operators';
+
+import { memo } from '../util';
+
+export function erasRewards (api: ApiInterfaceRx): () => Observable<DeriveEraRewardsAll[]> {
+  return memo((): Observable<DeriveEraRewardsAll[]> =>
+    api.derive.staking.erasExposure().pipe(
+      map((exposures): DeriveEraRewardsAll[] =>
+        exposures.map(({ era, all }): DeriveEraRewardsAll =>
+          Object
+            .entries(all)
+            .reduce((rewards: DeriveEraRewardsAll, [validatorId, exposure]): DeriveEraRewardsAll => {
+              rewards.validators[validatorId] = exposure;
+
+              exposure.others.forEach(({ who }, index): void => {
+                const nominatorId = who.toString();
+
+                rewards.nominators[nominatorId] = rewards.nominators[nominatorId] || [];
+                rewards.nominators[nominatorId].push([validatorId, index]);
+              });
+
+              return rewards;
+            }, { era, nominators: {}, validators: {} })
+        )
+      )
+    )
+  );
+}

packages/api-derive/src/staking/index.ts

@@ -3,8 +3,13 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 export * from './account';
+export * from './erasExposure';
+export * from './erasPoints';
+export * from './erasRewards';
 export * from './controllers';
 export * from './electedInfo';
 export * from './overview';
 export * from './query';
+export * from './stakerRewards';
 export * from './validators';
+export * from './validatorPoints';

packages/api-derive/src/staking/stakerRewards.ts

@@ -0,0 +1,42 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { Exposure } from '@polkadot/types/interfaces';
+import { DeriveStakerReward } from '../types';
+
+import { Observable } from 'rxjs';
+import { map } from 'rxjs/operators';
+
+import { memo } from '../util';
+
+export function stakerRewards (api: ApiInterfaceRx): (accountId: Uint8Array | string) => Observable<DeriveStakerReward[]> {
+  return memo((accountId: Uint8Array | string): Observable<DeriveStakerReward[]> => {
+    const stakerId = api.registry.createType('AccountId', accountId).toString();
+
+    return api.derive.staking.erasRewards().pipe(
+      map((rewards): DeriveStakerReward[] =>
+        rewards
+          .map(({ era, nominators: allNominators, validators: allValidators }): DeriveStakerReward => {
+            const isValidator = !!allValidators[stakerId];
+            const validators: Record<string, Exposure> = {};
+            let nominating: [string, number][] = [];
+
+            if (isValidator) {
+              validators[stakerId] = allValidators[stakerId];
+            } else if (allNominators[stakerId]) {
+              nominating = allNominators[stakerId];
+
+              nominating.forEach(([validatorId]): void => {
+                validators[validatorId] = allValidators[validatorId];
+              });
+            }
+
+            return { era, isValidator, nominating, validators };
+          })
+          .filter(({ validators }): boolean => Object.keys(validators).length !== 0)
+      )
+    );
+  });
+}

packages/api-derive/src/staking/types.ts

@@ -0,0 +1,80 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { AccountId, Balance, BlockNumber, EraIndex, EraRewardPoints, Exposure, Keys, RewardDestination, RewardPoint, StakingLedger, ValidatorPrefs } from '@polkadot/types/interfaces';
+import { DeriveSessionIndexes } from '../session/types';
+
+export interface DeriveEraPointsAll {
+  all: Record<string, RewardPoint>;
+  era: EraIndex;
+  total: RewardPoint;
+}
+
+export interface DeriveEraPoints {
+  era: EraIndex;
+  own: RewardPoint;
+  total: RewardPoint;
+}
+
+export interface DeriveEraExposure {
+  all: Record<string, Exposure>;
+  era: EraIndex;
+}
+
+export interface DeriveEraRewardsAll {
+  era: EraIndex;
+  nominators: Record<string, [string, number][]>;
+  validators: Record<string, Exposure>;
+}
+
+export interface DeriveStakerReward {
+  era: EraIndex;
+  isValidator: boolean;
+  nominating: [string, number][];
+  validators: Record<string, Exposure>;
+}
+
+export interface DerivedStakingElected {
+  nextElected: AccountId[];
+  info: DerivedStakingQuery[];
+}
+
+export interface DeriveStakingValidators {
+  nextElected: AccountId[];
+  validators: AccountId[];
+}
+
+export interface DerivedStakingStash {
+  controllerId?: AccountId;
+  exposure?: Exposure;
+  nominators?: AccountId[];
+  nominateAt?: EraIndex;
+  rewardDestination?: RewardDestination;
+  nextKeys?: Keys;
+  stashId?: AccountId;
+  validatorPrefs?: ValidatorPrefs;
+}
+
+export interface DerivedStakingQuery extends DerivedStakingStash {
+  accountId: AccountId;
+  nextSessionIds: AccountId[];
+  sessionIds: AccountId[];
+  stakingLedger?: StakingLedger;
+}
+
+export interface DerivedStakingAccount extends DerivedStakingQuery {
+  redeemable?: Balance;
+  unlocking?: DerivedUnlocking[];
+}
+
+export interface DerivedStakingOverview extends DeriveSessionIndexes {
+  eraPoints: EraRewardPoints;
+  nextElected: AccountId[];
+  validators: AccountId[];
+}
+
+export type DerivedUnlocking = {
+  remainingBlocks: BlockNumber;
+  value: Balance;
+};

packages/api-derive/src/staking/validatorPoints.ts

@@ -0,0 +1,27 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { DeriveEraPoints } from '../types';
+
+import { Observable } from 'rxjs';
+import { map } from 'rxjs/operators';
+
+import { memo } from '../util';
+
+export function validatorPoints (api: ApiInterfaceRx): (accountId: Uint8Array | string) => Observable<DeriveEraPoints[]> {
+  return memo((_accountId: Uint8Array | string): Observable<DeriveEraPoints[]> => {
+    const accountId = api.registry.createType('AccountId', _accountId).toString();
+
+    return api.derive.staking.erasPoints().pipe(
+      map((all): DeriveEraPoints[] =>
+        all.map(({ all, era, total }): DeriveEraPoints => ({
+          era,
+          own: all[accountId] || api.registry.createType('RewardPoint'),
+          total
+        }))
+      )
+    );
+  });
+}

packages/api-derive/src/types.ts

@@ -2,10 +2,14 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
-import { AccountId, AccountIndex, Balance, BalanceLock, BalanceLockTo212, BalanceOf, Bid, BidKind, BlockNumber, CollatorId, EraIndex, EraRewardPoints, Exposure, Hash, Index, Keys, Moment, ParaId, ParaInfo, Proposal, PropIndex, ProposalIndex, ReferendumInfo, RegistrationJudgement, RewardDestination, SessionIndex, SetIndex, SocietyVote, StakingLedger, StrikeCount, TreasuryProposal, UpwardMessage, ValidatorPrefs, Vote, Votes, VoteIndex, VouchingStatus } from '@polkadot/types/interfaces';
+import { AccountId, AccountIndex, Balance, BalanceLock, BalanceLockTo212, BalanceOf, Bid, BidKind, BlockNumber, Hash, Index, Proposal, PropIndex, ProposalIndex, ReferendumInfo, RegistrationJudgement, SetIndex, SocietyVote, StrikeCount, TreasuryProposal, Vote, Votes, VoteIndex, VouchingStatus } from '@polkadot/types/interfaces';
 
 import BN from 'bn.js';
-import { Bytes, Option, u32, Vec } from '@polkadot/types';
+import { u32, Vec } from '@polkadot/types';
+
+export * from './parachains/types';
+export * from './session/types';
+export * from './staking/types';
 
 export type AccountIndexes = Record<string, AccountIndex>;
 
@@ -108,36 +112,6 @@ export interface RecentlyOffline {
   count: BN;
 }
 
-export interface DeriveParachainActive {
-  collatorId: CollatorId;
-  isRetriable: boolean;
-  retries: number;
-}
-
-export interface DeriveParachainInfo extends ParaInfo {
-  id: ParaId;
-  icon?: string;
-  name?: string;
-  owner?: string;
-}
-
-export interface DeriveParachain {
-  didUpdate: boolean;
-  pendingSwapId: ParaId | null;
-  id: ParaId;
-  info: DeriveParachainInfo | null;
-  relayDispatchQueueSize?: number;
-  watermark: BlockNumber | null;
-}
-
-export interface DeriveParachainFull extends DeriveParachain {
-  active: DeriveParachainActive | null;
-  heads: Bytes | null;
-  relayDispatchQueue: UpwardMessage[];
-  retryCollators: (CollatorId | null)[];
-  selectedCollators: (CollatorId | null)[];
-}
-
 export interface DeriveProposalPreImage {
   at: BlockNumber;
   balance: Balance;
@@ -170,23 +144,6 @@ export interface DerivedReferendumVote {
   vote: Vote;
 }
 
-export interface DeriveSessionIndexes {
-  activeEra: EraIndex;
-  activeEraStart: Option<Moment>;
-  currentEra: EraIndex;
-  currentIndex: SessionIndex;
-  validatorCount: u32;
-}
-
-export interface DerivedSessionInfo extends DeriveSessionIndexes {
-  eraLength: BlockNumber;
-  eraProgress: BlockNumber;
-  isEpoch: boolean;
-  sessionLength: BlockNumber;
-  sessionsPerEra: SessionIndex;
-  sessionProgress: BlockNumber;
-}
-
 export interface DeriveSociety {
   bids: Bid[];
   defender?: AccountId;
@@ -213,45 +170,6 @@ export interface DeriveSocietyMember {
   vouching?: VouchingStatus;
 }
 
-export interface DerivedStakingElected {
-  nextElected: AccountId[];
-  info: DerivedStakingQuery[];
-}
-
-export interface DeriveStakingValidators {
-  nextElected: AccountId[];
-  validators: AccountId[];
-}
-
-export interface DerivedStakingStash {
-  controllerId?: AccountId;
-  exposure?: Exposure;
-  nominators?: AccountId[];
-  nominateAt?: EraIndex;
-  rewardDestination?: RewardDestination;
-  nextKeys?: Keys;
-  stashId?: AccountId;
-  validatorPrefs?: ValidatorPrefs;
-}
-
-export interface DerivedStakingQuery extends DerivedStakingStash {
-  accountId: AccountId;
-  nextSessionIds: AccountId[];
-  sessionIds: AccountId[];
-  stakingLedger?: StakingLedger;
-}
-
-export interface DerivedStakingAccount extends DerivedStakingQuery {
-  redeemable?: Balance;
-  unlocking?: DerivedUnlocking[];
-}
-
-export interface DerivedStakingOverview extends DeriveSessionIndexes {
-  eraPoints: EraRewardPoints;
-  nextElected: AccountId[];
-  validators: AccountId[];
-}
-
 export interface DerivedTreasuryProposal {
   council: DerivedCollectiveProposal[];
   id: ProposalIndex;
@@ -264,11 +182,6 @@ export interface DerivedTreasuryProposals {
   proposals: DerivedTreasuryProposal[];
 }
 
-export type DerivedUnlocking = {
-  remainingBlocks: BlockNumber;
-  value: Balance;
-};
-
 export interface VoterPosition {
   globalIndex: BN;
   index: BN;