Update StakingLedger (simpl payouts) (#2102)

* Update StakingLedger (simpl payouts)

* Chain defs

* Allow stakers queries with exclude filter

* Generate interfaces

* Return all exposures

* Rework exposures

* Cleanups

packages/api-derive/src/staking/erasExposure.ts

@@ -6,7 +6,6 @@ import { ApiInterfaceRx } from '@polkadot/api/types';
 import { EraIndex, Exposure } from '@polkadot/types/interfaces';
 import { DeriveEraExposure, DeriveEraNominatorExposure, DeriveEraValidatorExposure } from '../types';
 
-import BN from 'bn.js';
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { StorageKey } from '@polkadot/types';
@@ -24,11 +23,11 @@ function mapStakers (era: EraIndex, stakers: KeysAndExposures): DeriveEraExposur
 
     validators[validatorId] = exposure;
 
-    exposure.others.forEach(({ who }, index): void => {
+    exposure.others.forEach(({ who }, validatorIndex): void => {
       const nominatorId = who.toString();
 
       nominators[nominatorId] = nominators[nominatorId] || [];
-      nominators[nominatorId].push([validatorId, index]);
+      nominators[nominatorId].push({ validatorId, validatorIndex });
     });
   });
 
@@ -43,14 +42,12 @@ export function eraExposure (api: ApiInterfaceRx): (era: EraIndex) => Observable
   );
 }
 
-export function erasExposure (api: ApiInterfaceRx): (withActive?: boolean | BN | number) => Observable<DeriveEraExposure[]> {
-  return memo((withActive?: boolean | BN | number): Observable<DeriveEraExposure[]> =>
+export function erasExposure (api: ApiInterfaceRx): (withActive?: boolean) => Observable<DeriveEraExposure[]> {
+  return memo((withActive?: boolean): Observable<DeriveEraExposure[]> =>
     api.derive.staking.erasHistoric(withActive).pipe(
       switchMap((eras): Observable<DeriveEraExposure[]> =>
         eras.length
-          ? combineLatest(
-            eras.map((era) => api.derive.staking.eraExposure(era))
-          )
+          ? combineLatest(eras.map((era) => api.derive.staking.eraExposure(era)))
           : of([])
       )
     )

packages/api-derive/src/staking/erasHistoric.ts

@@ -5,49 +5,35 @@
 import { ApiInterfaceRx } from '@polkadot/api/types';
 import { ActiveEraInfo, EraIndex } from '@polkadot/types/interfaces';
 
-import BN from 'bn.js';
 import { Observable, of } from 'rxjs';
 import { map } from 'rxjs/operators';
 import { Option, u32 } from '@polkadot/types';
-import { isBoolean, isUndefined, bnToBn } from '@polkadot/util';
 
 import { memo } from '../util';
 
-export function erasHistoric (api: ApiInterfaceRx): (withActive?: boolean | BN | number) => Observable<EraIndex[]> {
-  return memo((withActive?: boolean | BN | number): Observable<EraIndex[]> =>
+export function erasHistoric (api: ApiInterfaceRx): (withActive?: boolean) => Observable<EraIndex[]> {
+  return memo((withActive?: boolean): Observable<EraIndex[]> =>
     api.query.staking?.activeEra
       ? api.queryMulti<[Option<ActiveEraInfo>, u32]>([
         api.query.staking.activeEra,
         api.query.staking.historyDepth
       ]).pipe(
         map(([activeEraOpt, historyDepth]): EraIndex[] => {
-          const result: (EraIndex | null)[] = [];
+          const result: EraIndex[] = [];
           const max = historyDepth.toNumber();
           const activeEra = activeEraOpt.unwrapOrDefault().index.toBn();
           let lastEra = activeEra;
 
           while (lastEra.gten(0) && (result.length < max)) {
-            result.push(
-              ((lastEra !== activeEra) || (withActive === true))
-                ? api.registry.createType('EraIndex', lastEra)
-                : null
-            );
+            if ((lastEra !== activeEra) || (withActive === true)) {
+              result.push(api.registry.createType('EraIndex', lastEra));
+            }
 
             lastEra = lastEra.subn(1);
           }
 
-          const startEra = isUndefined(withActive) || isBoolean(withActive)
-            ? new BN(0)
-            : bnToBn(withActive);
-
           // go from oldest to newest
-          return result
-            .filter((era): era is EraIndex =>
-              era
-                ? era.gte(startEra)
-                : false
-            )
-            .reverse();
+          return result.reverse();
         })
       )
       : of([])

packages/api-derive/src/staking/erasPoints.ts

@@ -6,7 +6,6 @@ import { ApiInterfaceRx } from '@polkadot/api/types';
 import { EraIndex, EraRewardPoints } from '@polkadot/types/interfaces';
 import { DeriveEraPoints, DeriveEraValPoints } from '../types';
 
-import BN from 'bn.js';
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 
@@ -22,8 +21,8 @@ function mapValidators ({ individual }: EraRewardPoints): DeriveEraValPoints {
     }, {});
 }
 
-export function erasPoints (api: ApiInterfaceRx): (withActive?: boolean | BN | number) => Observable<DeriveEraPoints[]> {
-  return memo((withActive?: boolean | number): Observable<DeriveEraPoints[]> =>
+export function erasPoints (api: ApiInterfaceRx): (withActive?: boolean) => Observable<DeriveEraPoints[]> {
+  return memo((withActive?: boolean): Observable<DeriveEraPoints[]> =>
     api.derive.staking.erasHistoric(withActive).pipe(
       switchMap((eras): Observable<[EraIndex[], EraRewardPoints[]]> =>
         combineLatest([

packages/api-derive/src/staking/erasPrefs.ts

@@ -6,7 +6,6 @@ import { ApiInterfaceRx } from '@polkadot/api/types';
 import { EraIndex, ValidatorPrefs } from '@polkadot/types/interfaces';
 import { DeriveEraPrefs, DeriveEraValPrefs } from '../types';
 
-import BN from 'bn.js';
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { StorageKey } from '@polkadot/types';
@@ -31,14 +30,12 @@ export function eraPrefs (api: ApiInterfaceRx): (era: EraIndex) => Observable<De
   );
 }
 
-export function erasPrefs (api: ApiInterfaceRx): (withActive?: boolean | BN | number) => Observable<DeriveEraPrefs[]> {
-  return memo((withActive?: boolean | BN | number): Observable<DeriveEraPrefs[]> =>
+export function erasPrefs (api: ApiInterfaceRx): (withActive?: boolean) => Observable<DeriveEraPrefs[]> {
+  return memo((withActive?: boolean): Observable<DeriveEraPrefs[]> =>
     api.derive.staking.erasHistoric(withActive).pipe(
       switchMap((eras): Observable<DeriveEraPrefs[]> =>
         eras.length
-          ? combineLatest(
-            eras.map((era) => api.derive.staking.eraPrefs(era))
-          )
+          ? combineLatest(eras.map((era) => api.derive.staking.eraPrefs(era)))
           : of([])
       )
     )

packages/api-derive/src/staking/erasRewards.ts

@@ -6,15 +6,14 @@ import { ApiInterfaceRx } from '@polkadot/api/types';
 import { Balance, EraIndex } from '@polkadot/types/interfaces';
 import { DeriveEraRewards } from '../types';
 
-import BN from 'bn.js';
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { Option } from '@polkadot/types';
 
 import { memo } from '../util';
 
-export function erasRewards (api: ApiInterfaceRx): (withActive?: boolean | BN | number) => Observable<DeriveEraRewards[]> {
-  return memo((withActive?: boolean | BN | number): Observable<DeriveEraRewards[]> =>
+export function erasRewards (api: ApiInterfaceRx): (withActive?: boolean) => Observable<DeriveEraRewards[]> {
+  return memo((withActive?: boolean): Observable<DeriveEraRewards[]> =>
     api.derive.staking.erasHistoric(withActive).pipe(
       switchMap((eras): Observable<[EraIndex[], Option<Balance>[]]> =>
         combineLatest([

packages/api-derive/src/staking/erasSlashes.ts

@@ -7,7 +7,6 @@ import { BalanceOf, EraIndex, Perbill } from '@polkadot/types/interfaces';
 import { ITuple } from '@polkadot/types/types';
 import { DeriveEraSlashes, DeriveEraValSlash } from '../types';
 
-import BN from 'bn.js';
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 import { Option, StorageKey } from '@polkadot/types';
@@ -40,8 +39,8 @@ export function eraSlashes (api: ApiInterfaceRx): (era: EraIndex) => Observable<
   );
 }
 
-export function erasSlashes (api: ApiInterfaceRx): (withActive?: boolean | BN | number) => Observable<DeriveEraSlashes[]> {
-  return memo((withActive?: boolean | BN | number): Observable<DeriveEraSlashes[]> =>
+export function erasSlashes (api: ApiInterfaceRx): (withActive?: boolean) => Observable<DeriveEraSlashes[]> {
+  return memo((withActive?: boolean): Observable<DeriveEraSlashes[]> =>
     api.derive.staking.erasHistoric(withActive).pipe(
       switchMap((eras): Observable<DeriveEraSlashes[]> =>
         eras.length

packages/api-derive/src/staking/ownExposure.ts

@@ -5,14 +5,13 @@
 import { ApiInterfaceRx } from '@polkadot/api/types';
 import { DeriveOwnExposure } from '../types';
 
-import BN from 'bn.js';
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 
 import { memo } from '../util';
 
-export function ownExposure (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean | BN | number) => Observable<DeriveOwnExposure[]> {
-  return memo((accountId: Uint8Array | string, withActive?: boolean | BN | number): Observable<DeriveOwnExposure[]> => {
+export function ownExposure (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean) => Observable<DeriveOwnExposure[]> {
+  return memo((accountId: Uint8Array | string, withActive?: boolean): Observable<DeriveOwnExposure[]> => {
     return api.derive.staking.erasHistoric(withActive).pipe(
       switchMap((eras) =>
         eras.length

packages/api-derive/src/staking/ownSlashes.ts

@@ -5,14 +5,13 @@
 import { ApiInterfaceRx } from '@polkadot/api/types';
 import { DeriveStakerSlashes } from '../types';
 
-import BN from 'bn.js';
 import { Observable, combineLatest, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
 
 import { memo } from '../util';
 
-export function ownSlashes (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean | BN | number) => Observable<DeriveStakerSlashes[]> {
-  return memo((accountId: Uint8Array | string, withActive?: boolean | BN | number): Observable<DeriveStakerSlashes[]> => {
+export function ownSlashes (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean) => Observable<DeriveStakerSlashes[]> {
+  return memo((accountId: Uint8Array | string, withActive?: boolean): Observable<DeriveStakerSlashes[]> => {
     return api.derive.staking.erasHistoric(withActive).pipe(
       switchMap((eras) =>
         eras.length

packages/api-derive/src/staking/query.ts

@@ -102,10 +102,10 @@ export function query (api: ApiInterfaceRx): (accountId: Uint8Array | string) =>
   });
 }
 
-export function queryMulti (api: ApiInterfaceRx): (...accountIds: (Uint8Array | string)[]) => Observable<DeriveStakingQuery[]> {
-  return memo((...accountIds: (Uint8Array | string)[]): Observable<DeriveStakingQuery[]> =>
-    combineLatest(
-      accountIds.map((accountId) => api.derive.staking.query(accountId))
-    )
+export function queryMulti (api: ApiInterfaceRx): (accountIds: (Uint8Array | string)[]) => Observable<DeriveStakingQuery[]> {
+  return memo((accountIds: (Uint8Array | string)[]): Observable<DeriveStakingQuery[]> =>
+    accountIds.length
+      ? combineLatest(accountIds.map((acc) => api.derive.staking.query(acc)))
+      : of([])
   );
 }

packages/api-derive/src/staking/stakerExposure.ts

@@ -3,16 +3,15 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 import { ApiInterfaceRx } from '@polkadot/api/types';
-import { DeriveStakerExposure, DeriveEraValidatorExposure } from '../types';
+import { DeriveStakerExposure, DeriveEraExposureNominating, DeriveEraValidatorExposure } from '../types';
 
-import BN from 'bn.js';
 import { Observable } from 'rxjs';
 import { map } from 'rxjs/operators';
 
 import { memo } from '../util';
 
-export function stakerExposure (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean | BN | number) => Observable<DeriveStakerExposure[]> {
-  return memo((accountId: Uint8Array | string, withActive?: boolean | BN | number): Observable<DeriveStakerExposure[]> => {
+export function stakerExposure (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean) => Observable<DeriveStakerExposure[]> {
+  return memo((accountId: Uint8Array | string, withActive?: boolean): Observable<DeriveStakerExposure[]> => {
     const stakerId = api.registry.createType('AccountId', accountId).toString();
 
     return api.derive.staking.erasExposure(withActive).pipe(
@@ -20,14 +19,14 @@ export function stakerExposure (api: ApiInterfaceRx): (accountId: Uint8Array | s
         exposures.map(({ era, nominators: allNominators, validators: allValidators }): DeriveStakerExposure => {
           const isValidator = !!allValidators[stakerId];
           const validators: DeriveEraValidatorExposure = {};
-          let nominating: [string, number][] = [];
+          let nominating: DeriveEraExposureNominating[] = [];
 
           if (isValidator) {
             validators[stakerId] = allValidators[stakerId];
           } else if (allNominators[stakerId]) {
             nominating = allNominators[stakerId];
 
-            nominating.forEach(([validatorId]): void => {
+            nominating.forEach(({ validatorId }): void => {
               validators[validatorId] = allValidators[validatorId];
             });
           }

packages/api-derive/src/staking/stakerPoints.ts

@@ -5,14 +5,13 @@
 import { ApiInterfaceRx } from '@polkadot/api/types';
 import { DeriveStakerPoints } from '../types';
 
-import BN from 'bn.js';
 import { Observable } from 'rxjs';
 import { map } from 'rxjs/operators';
 
 import { memo } from '../util';
 
-export function stakerPoints (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean | BN | number) => Observable<DeriveStakerPoints[]> {
-  return memo((accountId: Uint8Array | string, withActive?: boolean | BN | number): Observable<DeriveStakerPoints[]> => {
+export function stakerPoints (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean) => Observable<DeriveStakerPoints[]> {
+  return memo((accountId: Uint8Array | string, withActive?: boolean): Observable<DeriveStakerPoints[]> => {
     const stakerId = api.registry.createType('AccountId', accountId).toString();
 
     return api.derive.staking.erasPoints(withActive).pipe(

packages/api-derive/src/staking/stakerRewards.ts

@@ -3,11 +3,13 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 import { ApiInterfaceRx } from '@polkadot/api/types';
+import { EraIndex, StakingLedger, StakingLedgerTo240 } from '@polkadot/types/interfaces';
 import { DeriveEraPoints, DeriveEraPrefs, DeriveEraRewards, DeriveEraValPrefs, DeriveStakerExposure, DeriveStakerReward, DeriveStakerRewardValidator } from '../types';
 
 import BN from 'bn.js';
-import { Observable, combineLatest } from 'rxjs';
-import { map } from 'rxjs/operators';
+import { Observable, combineLatest, of } from 'rxjs';
+import { map, switchMap } from 'rxjs/operators';
+import { Option } from '@polkadot/types';
 
 import { memo } from '../util';
 
@@ -67,29 +69,102 @@ function parseRewards (api: ApiInterfaceRx, stakerId: string, [erasPoints, erasP
   });
 }
 
-export function stakerRewards (api: ApiInterfaceRx): (accountId: Uint8Array | string, startEra: BN | number) => Observable<DeriveStakerReward[]> {
-  return memo((accountId: Uint8Array | string, startEra: BN | number): Observable<DeriveStakerReward[]> => {
+function uniqValidators (rewards: DeriveStakerReward[]): string[] {
+  const uniq: string[] = [];
+
+  rewards.forEach(({ validators }): void => {
+    Object.keys(validators).forEach((validatorId): void => {
+      if (!uniq.includes(validatorId)) {
+        uniq.push(validatorId);
+      }
+    });
+  });
+
+  return uniq;
+}
+
+function isOldLedger (ledger: StakingLedger | StakingLedgerTo240): ledger is StakingLedgerTo240 {
+  return !!(ledger as StakingLedgerTo240).lastReward;
+}
+
+function filterRewards (api: ApiInterfaceRx, accountId: Uint8Array | string, rewards: DeriveStakerReward[], withActive?: boolean): Observable<DeriveStakerReward[]> {
+  if (withActive) {
+    return of(rewards);
+  }
+
+  const validators = uniqValidators(rewards);
+
+  return combineLatest([
+    api.query.staking.migrateEra
+      ? api.query.staking.migrateEra<Option<EraIndex>>()
+      : of(api.registry.createType('Option<EraIndex>', api.tx.staking.payoutStakers ? 0 : 1_000_000_000)),
+    api.derive.staking.query(accountId),
+    api.derive.staking.queryMulti(validators)
+  ]).pipe(
+    map(([optMigrate, { stakingLedger }, queryValidators]): DeriveStakerReward[] => {
+      const migrateEra = optMigrate.unwrapOr(new BN(-1));
+
+      return rewards
+        .filter(({ isEmpty }) => !isEmpty)
+        .filter((reward): boolean => {
+          if (reward.era.lte(migrateEra)) {
+            return !stakingLedger
+              ? true
+              : isOldLedger(stakingLedger)
+                ? reward.era.gt(stakingLedger.lastReward.unwrapOr(new BN(-1)))
+                : !stakingLedger.claimedRewards.some((claimEra) => reward.era.eq(claimEra));
+          }
+
+          reward.isStakerPayout = true;
+
+          const rm: string[] = [];
+
+          Object.keys(reward.validators).forEach((validatorId): void => {
+            const index = validators.indexOf(validatorId);
+
+            if (index !== -1) {
+              const valLedger = queryValidators[index].stakingLedger;
+
+              if (valLedger?.claimedRewards.some((era) => reward.era.eq(era))) {
+                rm.push(validatorId);
+              }
+            }
+          });
+
+          rm.forEach((validatorId): void => {
+            delete reward.validators[validatorId];
+          });
+
+          return true;
+        })
+        .filter(({ validators }) => Object.keys(validators).length !== 0)
+        .map((reward) => ({
+          ...reward,
+          nominators: reward.nominating.filter(({ validatorId }) => !!reward.validators[validatorId])
+        }));
+    })
+  );
+}
+
+export function stakerRewards (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean) => Observable<DeriveStakerReward[]> {
+  return memo((accountId: Uint8Array | string, withActive?: boolean): Observable<DeriveStakerReward[]> => {
     const stakerId = api.registry.createType('AccountId', accountId).toString();
 
     return combineLatest([
-      api.derive.staking.erasPoints(startEra),
-      api.derive.staking.erasPrefs(startEra),
-      api.derive.staking.erasRewards(startEra),
-      api.derive.staking.stakerExposure(stakerId, startEra)
+      api.derive.staking.erasPoints(withActive),
+      api.derive.staking.erasPrefs(withActive),
+      api.derive.staking.erasRewards(withActive),
+      api.derive.staking.stakerExposure(stakerId, withActive)
     ]).pipe(
-      map((result): DeriveStakerReward[] =>
-        parseRewards(api, stakerId, result)
-      )
+      switchMap((result) => filterRewards(api, accountId, parseRewards(api, stakerId, result), withActive))
     );
   });
 }
 
-export function stakerRewardsMulti (api: ApiInterfaceRx): (...params: [Uint8Array | string, BN | number][]) => Observable<DeriveStakerReward[][]> {
-  return memo((...params: [Uint8Array | string, BN | number][]): Observable<DeriveStakerReward[][]> =>
-    combineLatest(
-      params.map(([accountId, startEra]) =>
-        api.derive.staking.stakerRewards(accountId, startEra)
-      )
-    )
+export function stakerRewardsMulti (api: ApiInterfaceRx): (accountIds: (Uint8Array | string)[], withActive?: boolean) => Observable<DeriveStakerReward[][]> {
+  return memo((accountIds: (Uint8Array | string)[], withActive?: boolean): Observable<DeriveStakerReward[][]> =>
+    accountIds.length
+      ? combineLatest(accountIds.map((acc) => api.derive.staking.stakerRewards(acc, withActive)))
+      : of([])
   );
 }

packages/api-derive/src/staking/stakerSlashes.ts

@@ -5,14 +5,13 @@
 import { ApiInterfaceRx } from '@polkadot/api/types';
 import { DeriveStakerSlashes } from '../types';
 
-import BN from 'bn.js';
 import { Observable } from 'rxjs';
 import { map } from 'rxjs/operators';
 
 import { memo } from '../util';
 
-export function stakerSlashes (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean | BN | number) => Observable<DeriveStakerSlashes[]> {
-  return memo((accountId: Uint8Array | string, withActive?: boolean | BN | number): Observable<DeriveStakerSlashes[]> => {
+export function stakerSlashes (api: ApiInterfaceRx): (accountId: Uint8Array | string, withActive?: boolean) => Observable<DeriveStakerSlashes[]> {
+  return memo((accountId: Uint8Array | string, withActive?: boolean): Observable<DeriveStakerSlashes[]> => {
     const stakerId = api.registry.createType('AccountId', accountId).toString();
 
     return api.derive.staking.erasSlashes(withActive).pipe(

packages/api-derive/src/staking/types.ts

@@ -41,7 +41,7 @@ export interface DeriveStakerPoints {
   points: RewardPoint;
 }
 
-export type DeriveEraNominatorExposure = Record<string, [string, number][]>;
+export type DeriveEraNominatorExposure = Record<string, DeriveEraExposureNominating[]>;
 
 export type DeriveEraValidatorExposure = Record<string, Exposure>;
 
@@ -59,11 +59,16 @@ export interface DeriveOwnExposure {
 
 export type DeriveOwnSlashes = DeriveStakerSlashes;
 
+export interface DeriveEraExposureNominating {
+  validatorId: string;
+  validatorIndex: number;
+}
+
 export interface DeriveStakerExposure {
   era: EraIndex;
   isEmpty: boolean;
   isValidator: boolean;
-  nominating: [string, number][];
+  nominating: DeriveEraExposureNominating[];
   validators: DeriveEraValidatorExposure;
 }
 
@@ -75,9 +80,10 @@ export interface DeriveStakerRewardValidator {
 export interface DeriveStakerReward {
   era: EraIndex;
   eraReward: Balance;
+  isStakerPayout?: boolean;
   isEmpty: boolean;
   isValidator: boolean;
-  nominating: [string, number][];
+  nominating: DeriveEraExposureNominating[];
   total: Balance;
   validators: Record<string, DeriveStakerRewardValidator>;
 }