Optimize use of derive staking.query (with full results) (#2897)

package.json

@@ -27,7 +27,7 @@
     "@babel/register": "^7.12.1",
     "@babel/runtime": "^7.12.5",
     "@polkadot/dev": "^0.60.6",
-    "@polkadot/ts": "^0.3.55",
+    "@polkadot/ts": "^0.3.56",
     "@polkadot/typegen": "workspace:packages/typegen",
     "@types/jest": "^26.0.15",
     "copyfiles": "^2.4.0"

packages/api-derive/src/accounts/identity.ts

@@ -79,15 +79,13 @@ function getParent (api: ApiInterfaceRx, identityOfOpt: Option<Registration> | u
   return of([undefined, undefined]);
 }
 
-export function _identityBase (instanceId: string, api: ApiInterfaceRx): (accountId?: AccountId | Uint8Array | string) => Observable<[Option<Registration> | undefined, Option<ITuple<[AccountId, Data]>> | undefined]> {
+function getBase (api: ApiInterfaceRx, accountId?: AccountId | Uint8Array | string): Observable<[Option<Registration> | undefined, Option<ITuple<[AccountId, Data]>> | undefined]> {
-  return memo(instanceId, (accountId?: AccountId | Uint8Array | string): Observable<[Option<Registration> | undefined, Option<ITuple<[AccountId, Data]>> | undefined]> =>
+  return accountId && api.query.identity?.identityOf
-    accountId && api.query.identity?.identityOf
+    ? api.queryMulti<[Option<Registration>, Option<ITuple<[AccountId, Data]>>]>([
-      ? api.queryMulti<[Option<Registration>, Option<ITuple<[AccountId, Data]>>]>([
+      [api.query.identity.identityOf, accountId],
-        [api.query.identity.identityOf, accountId],
+      [api.query.identity.superOf, accountId]
-        [api.query.identity.superOf, accountId]
+    ])
-      ])
+    : of([undefined, undefined]);
-      : of([undefined, undefined])
-  );
 }
 
 /**
@@ -96,7 +94,7 @@ export function _identityBase (instanceId: string, api: ApiInterfaceRx): (accoun
  */
 export function identity (instanceId: string, api: ApiInterfaceRx): (accountId?: AccountId | Uint8Array | string) => Observable<DeriveAccountRegistration> {
   return memo(instanceId, (accountId?: AccountId | Uint8Array | string): Observable<DeriveAccountRegistration> =>
-    api.derive.accounts._identityBase(accountId).pipe(
+    getBase(api, accountId).pipe(
       switchMap(([identityOfOpt, superOfOpt]) => getParent(api, identityOfOpt, superOfOpt)),
       map(([identityOfOpt, superOf]) => extractIdentity(identityOfOpt, superOf))
     )
@@ -105,23 +103,31 @@ export function identity (instanceId: string, api: ApiInterfaceRx): (accountId?:
 
 export function hasIdentity (instanceId: string, api: ApiInterfaceRx): (accountId: AccountId | Uint8Array | string) => Observable<DeriveHasIdentity> {
   return memo(instanceId, (accountId: AccountId | Uint8Array | string): Observable<DeriveHasIdentity> =>
-    api.derive.accounts._identityBase(accountId).pipe(
+    api.derive.accounts.hasIdentityMulti([accountId]).pipe(
-      map(([identityOfOpt, superOfOpt]: [Option<Registration> | undefined, Option<ITuple<[AccountId, Data]>> | undefined]): DeriveHasIdentity => {
+      map(([first]) => first)
-        const parentId = superOfOpt && superOfOpt.isSome
-          ? superOfOpt.unwrap()[0].toString()
-          : null;
-        const hasIdentity = !!parentId || !!(identityOfOpt && identityOfOpt.isSome);
-
-        return { hasIdentity, parentId };
-      })
     )
   );
 }
 
 export function hasIdentityMulti (instanceId: string, api: ApiInterfaceRx): (accountIds: (AccountId | Uint8Array | string)[]) => Observable<DeriveHasIdentity[]> {
   return memo(instanceId, (accountIds: (AccountId | Uint8Array | string)[]): Observable<DeriveHasIdentity[]> =>
-    combineLatest(
+    api.query.identity?.identityOf
-      accountIds.map((accountId) => api.derive.accounts.hasIdentity(accountId))
+      ? combineLatest([
-    )
+        api.query.identity.identityOf.multi<Option<Registration>>(accountIds),
+        api.query.identity.superOf.multi<Option<ITuple<[AccountId, Data]>>>(accountIds)
+      ]).pipe(
+        map(([identities, supers]) =>
+          identities.map((identityOfOpt, index): DeriveHasIdentity => {
+            const superOfOpt = supers[index];
+            const parentId = superOfOpt && superOfOpt.isSome
+              ? superOfOpt.unwrap()[0].toString()
+              : null;
+            const hasIdentity = !!parentId || !!(identityOfOpt && identityOfOpt.isSome);
+
+            return { hasIdentity, parentId };
+          })
+        )
+      )
+      : of(accountIds.map(() => ({ hasIdentity: false, parentId: null })))
   );
 }

packages/api-derive/src/staking/account.ts

@@ -63,7 +63,7 @@ export function accounts (instanceId: string, api: ApiInterfaceRx): (accountIds:
       switchMap((sessionInfo) =>
         combineLatest([
           api.derive.staking.keysMulti(accountIds),
-          api.derive.staking.queryMulti(accountIds)
+          api.derive.staking.queryMulti(accountIds, true)
         ]).pipe(
           map(([keys, queries]) => queries.map((query, index) => parseResult(api, sessionInfo, keys[index], query)))
         )

packages/api-derive/src/staking/electedInfo.ts

@@ -18,7 +18,7 @@ export function electedInfo (instanceId: string, api: ApiInterfaceRx): () => Obs
   return memo(instanceId, (): Observable<DeriveStakingElected> =>
     api.derive.staking.validators().pipe(
       switchMap(({ nextElected, validators }): Observable<DeriveStakingElected> =>
-        api.derive.staking.queryMulti(combineAccounts(nextElected, validators)).pipe(
+        api.derive.staking.queryMulti(combineAccounts(nextElected, validators), false).pipe(
           map((info): DeriveStakingElected => ({
             info,
             nextElected,

packages/api-derive/src/staking/query.ts

@@ -14,14 +14,16 @@ import { isFunction } from '@polkadot/util';
 
 import { memo } from '../util';
 
-type MultiResult = [Option<AccountId>, Option<ITuple<[Nominations]> | Nominations>, RewardDestination, ITuple<[ValidatorPrefs]> | ValidatorPrefs, Exposure];
+type MultiResultPrev = [Option<AccountId>, Option<ITuple<[Nominations]>>, RewardDestination, ITuple<[ValidatorPrefs]> | ValidatorPrefs, Exposure];
 
-function parseController (stashId: AccountId, [controllerIdOpt, nominatorsOpt, rewardDestination, validatorPrefs, exposure]: MultiResult, stakingLedgerOpt: Option<StakingLedger>): DeriveStakingQuery {
+type MultiResultCombo = [Option<AccountId>, Option<ITuple<[Nominations]> | Nominations>, RewardDestination, ITuple<[ValidatorPrefs]> | ValidatorPrefs, Exposure];
-  const nominators = nominatorsOpt.unwrapOr(null);
+
+function parseDetails (stashId: AccountId, [controllerIdOpt, nominatorsOpt, rewardDestination, validatorPrefs, exposure]: MultiResultCombo, stakingLedgerOpt: Option<StakingLedger>): DeriveStakingQuery {
+  const nominators = nominatorsOpt && nominatorsOpt.unwrapOr(null);
 
   return {
     accountId: stashId,
-    controllerId: controllerIdOpt.unwrapOr(null),
+    controllerId: controllerIdOpt && controllerIdOpt.unwrapOr(null),
     exposure,
     nominators: nominators
       ? Array.isArray(nominators)
@@ -37,8 +39,8 @@ function parseController (stashId: AccountId, [controllerIdOpt, nominatorsOpt, r
   };
 }
 
-function retrievePrev (api: ApiInterfaceRx, stashId: AccountId): Observable<MultiResult> {
+function retrievePrev (api: ApiInterfaceRx, stashId: AccountId): Observable<MultiResultPrev> {
-  return api.queryMulti<MultiResult>([
+  return api.queryMulti<MultiResultPrev>([
     [api.query.staking.bonded, stashId],
     [api.query.staking.nominators, stashId],
     [api.query.staking.payee, stashId],
@@ -47,18 +49,26 @@ function retrievePrev (api: ApiInterfaceRx, stashId: AccountId): Observable<Mult
   ]);
 }
 
-function retrieveCurr (api: ApiInterfaceRx, stashIds: AccountId[], activeEra: EraIndex): Observable<MultiResult[]> {
+function retrieveCurr (api: ApiInterfaceRx, stashIds: AccountId[], activeEra: EraIndex, withFull: boolean): Observable<MultiResultCombo[]> {
+  const emptyCont = api.registry.createType('Option<AccountId>');
+  const emptyNoms = api.registry.createType('Option<Nominations>');
+  const emptyRewa = api.registry.createType('RewardDestination');
+
   return combineLatest([
-    api.query.staking.bonded.multi<Option<AccountId>>(stashIds),
+    withFull
-    api.query.staking.nominators
+      ? api.query.staking.bonded.multi<Option<AccountId>>(stashIds)
+      : of(stashIds.map(() => emptyCont)),
+    withFull && api.query.staking.nominators
       ? api.query.staking.nominators.multi<Option<Nominations>>(stashIds)
-      : of(stashIds.map(() => api.registry.createType('Option<Nominations>'))),
+      : of(stashIds.map(() => emptyNoms)),
-    api.query.staking.payee.multi<RewardDestination>(stashIds),
+    withFull
+      ? api.query.staking.payee.multi<RewardDestination>(stashIds)
+      : of(stashIds.map(() => emptyRewa)),
     api.query.staking.validators.multi<ValidatorPrefs>(stashIds),
     api.query.staking.erasStakers.multi<Exposure>(stashIds.map((stashId) => [activeEra, stashId]))
   ]).pipe(
-    map(([controllerIdOpt, nominatorsOpt, rewardDestination, validatorPrefs, exposure]): MultiResult[] =>
+    map(([controllerIdOpt, nominatorsOpt, rewardDestination, validatorPrefs, exposure]): MultiResultCombo[] =>
-      controllerIdOpt.map((controllerIdOpt, index): MultiResult =>
+      controllerIdOpt.map((controllerIdOpt, index): MultiResultCombo =>
         [controllerIdOpt, nominatorsOpt[index], rewardDestination[index], validatorPrefs[index], exposure[index]]
       )
     )
@@ -67,9 +77,10 @@ function retrieveCurr (api: ApiInterfaceRx, stashIds: AccountId[], activeEra: Er
 
 function retrieveControllers (api: ApiInterfaceRx, optControllerIds: Option<AccountId>[]): Observable<Option<StakingLedger>[]> {
   const ids = optControllerIds.filter((opt) => opt.isSome).map((opt) => opt.unwrap());
+  const emptyLed = api.registry.createType('Option<StakingLedger>');
 
   if (!ids.length) {
-    return of(optControllerIds.map(() => api.registry.createType('Option<StakingLedger>')));
+    return of(optControllerIds.map(() => emptyLed));
   }
 
   return api.query.staking.ledger.multi<Option<StakingLedger>>(ids).pipe(
@@ -79,7 +90,7 @@ function retrieveControllers (api: ApiInterfaceRx, optControllerIds: Option<Acco
       return optControllerIds.map((opt): Option<StakingLedger> =>
         opt.isSome
           ? optLedgers[++offset]
-          : api.registry.createType('Option<StakingLedger>')
+          : emptyLed
       );
     })
   );
@@ -88,33 +99,38 @@ function retrieveControllers (api: ApiInterfaceRx, optControllerIds: Option<Acco
 /**
  * @description From a stash, retrieve the controllerId and all relevant details
  */
-export function query (instanceId: string, api: ApiInterfaceRx): (accountId: Uint8Array | string) => Observable<DeriveStakingQuery> {
+export function query (instanceId: string, api: ApiInterfaceRx): (accountId: Uint8Array | string, withFull?: boolean) => Observable<DeriveStakingQuery> {
-  return memo(instanceId, (accountId: Uint8Array | string): Observable<DeriveStakingQuery> =>
+  return memo(instanceId, (accountId: Uint8Array | string, withFull?: boolean): Observable<DeriveStakingQuery> =>
-    api.derive.staking.queryMulti([accountId]).pipe(
+    api.derive.staking.queryMulti([accountId], withFull).pipe(
       map(([first]) => first)
     )
   );
 }
 
-export function queryMulti (instanceId: string, api: ApiInterfaceRx): (accountIds: (Uint8Array | string)[]) => Observable<DeriveStakingQuery[]> {
+export function queryMulti (instanceId: string, api: ApiInterfaceRx): (accountIds: (Uint8Array | string)[], withFull?: boolean) => Observable<DeriveStakingQuery[]> {
-  return memo(instanceId, (accountIds: (Uint8Array | string)[]): Observable<DeriveStakingQuery[]> =>
+  return memo(instanceId, (accountIds: (Uint8Array | string)[], withFull = true): Observable<DeriveStakingQuery[]> =>
     accountIds.length
       ? api.derive.session.indexes().pipe(
         switchMap(({ activeEra }): Observable<DeriveStakingQuery[]> => {
           const stashIds = accountIds.map((accountId) => api.registry.createType('AccountId', accountId));
+          const emptyLed = api.registry.createType('Option<StakingLedger>');
 
           return (
             isFunction(api.query.staking.erasStakers)
-              ? retrieveCurr(api, stashIds, activeEra)
+              ? retrieveCurr(api, stashIds, activeEra, withFull)
               : combineLatest(stashIds.map((stashId) => retrievePrev(api, stashId)))
           ).pipe(
-            switchMap((results): Observable<DeriveStakingQuery[]> =>
+            withFull
-              retrieveControllers(api, results.map(([optController]) => optController)).pipe(
+              ? switchMap((results): Observable<DeriveStakingQuery[]> =>
-                map((stakingLedgerOpts) =>
+                retrieveControllers(api, results.map(([optController]) => optController)).pipe(
-                  stashIds.map((stashId, index) => parseController(stashId, results[index], stakingLedgerOpts[index]))
+                  map((stakingLedgerOpts) =>
+                    stashIds.map((stashId, index) => parseDetails(stashId, results[index], stakingLedgerOpts[index]))
+                  )
                 )
               )
-            )
+              : map((results): DeriveStakingQuery[] =>
+                stashIds.map((stashId, index) => parseDetails(stashId, results[index], emptyLed))
+              )
           );
         })
       )

packages/api-derive/src/staking/stakerRewards.ts

@@ -127,7 +127,7 @@ function filterRewards (api: ApiInterfaceRx, eras: EraIndex[], { migrateEra, rew
 
   return (
     isFunction(api.tx.staking.payoutStakers)
-      ? api.derive.staking.queryMulti(validators)
+      ? api.derive.staking.queryMulti(validators, false)
       : of([])
   ).pipe(
     map((queryValidators): DeriveStakerReward[] =>
@@ -150,7 +150,7 @@ function filterRewards (api: ApiInterfaceRx, eras: EraIndex[], { migrateEra, rew
         .filter(({ validators }) => Object.keys(validators).length !== 0)
         .map((reward) => ({
           ...reward,
-          nominators: reward.nominating.filter((n) => !!reward.validators[n.validatorId])
+          nominators: reward.nominating.filter((n) => reward.validators[n.validatorId])
         }))
     )
   );

packages/api-derive/src/staking/validatorsFrom.ts

@@ -27,7 +27,8 @@ export function validatorsFrom (instanceId: string, api: ApiInterfaceRx): (stash
 
               return validatorIds;
             }, validatorIds);
-          }, [])
+          }, []),
+          false
         )
       )
     )

packages/api-derive/src/staking/waitingInfo.ts

@@ -20,7 +20,7 @@ export function waitingInfo (instanceId: string, api: ApiInterfaceRx): () => Obs
         const elected = nextElected.map((a) => a.toString());
         const waiting = stashes.filter((v) => !elected.includes(v.toString()));
 
-        return api.derive.staking.queryMulti(waiting).pipe(
+        return api.derive.staking.queryMulti(waiting, false).pipe(
           map((info): DeriveStakingWaiting => ({
             info,
             waiting

packages/api/src/base/Decorate.ts

@@ -392,8 +392,7 @@ export abstract class Decorate<ApiType extends ApiTypes> extends Events {
     if (this.hasSubscriptions) {
       // When using double map storage function, user need to pass double map key as an array
       decorated.multi = decorateMethod((args: (Arg | Arg[])[]): Observable<Codec[]> =>
-        this._rpcCore.state.subscribeStorage(
+        this._retrieveMulti(args.map((arg) => [creator, arg])));
-          args.map((arg: Arg[] | Arg): [StorageEntry, Arg | Arg[]] => [creator, arg])));
     }
 
     /* eslint-enable @typescript-eslint/no-unsafe-member-access,@typescript-eslint/no-unsafe-assignment */
@@ -430,6 +429,21 @@ export abstract class Decorate<ApiType extends ApiTypes> extends Events {
       ));
   }
 
+  // retrieve a set of values for a specific set of keys - here we chunk the keys into PAGE_SIZE_VALS sizes
+  private _retrieveMulti (keys: [StorageEntry, Arg | Arg[]][]): Observable<Codec[]> {
+    return combineLatest(
+      ...Array(Math.ceil(keys.length / PAGE_SIZE_VALS))
+        .fill(0)
+        .map((_, index): Observable<Codec[]> =>
+          this._rpcCore.state.subscribeStorage(keys.slice(index * PAGE_SIZE_VALS, (index * PAGE_SIZE_VALS) + PAGE_SIZE_VALS))
+        )
+    ).pipe(
+      map((valsArr): Codec[] =>
+        valsArr.reduce((result: Codec[], vals) => result.concat(vals), [])
+      )
+    );
+  }
+
   private _retrieveMapKeys ({ iterKey, meta, method, section }: StorageEntry, at: Hash | Uint8Array | string | null, arg?: Arg): Observable<StorageKey[]> {
     assert(iterKey && (meta.type.isMap || meta.type.isDoubleMap), 'keys can only be retrieved on maps, linked maps and double maps');
 

yarn.lock

@@ -1926,12 +1926,12 @@ __metadata:
   languageName: unknown
   linkType: soft
 
-"@polkadot/ts@npm:^0.3.55":
+"@polkadot/ts@npm:^0.3.56":
-  version: 0.3.55
+  version: 0.3.56
-  resolution: "@polkadot/ts@npm:0.3.55"
+  resolution: "@polkadot/ts@npm:0.3.56"
   dependencies:
     "@types/chrome": ^0.0.126
-  checksum: 36083562f98cdd0651045472147ae23283633fe9f792a967e9b6d2c52fcbf8f66789219f0d325aca7b1372f7588e5b846becceb45a90d1ea2b00648456a8f32c
+  checksum: 0b2e94c65ff27e5300ad7162e8d73bb7c1b476c351a02b84045a2ce5e5895c46b1daa1c059f50fb7f4c49f05966b1c12385014366d36d9311fd2c8d014501cc9
   languageName: node
   linkType: hard
 
@@ -9260,7 +9260,7 @@ fsevents@~2.1.2:
     "@babel/register": ^7.12.1
     "@babel/runtime": ^7.12.5
     "@polkadot/dev": ^0.60.6
-    "@polkadot/ts": ^0.3.55
+    "@polkadot/ts": ^0.3.56
     "@polkadot/typegen": "workspace:packages/typegen"
     "@types/jest": ^26.0.15
     copyfiles: ^2.4.0