From 8be52dec13e70d7cbd08de39d160c0aa87b55808 Mon Sep 17 00:00:00 2001
From: Jaco <jacogr@gmail.com>
Date: Sun, 6 Mar 2022 09:26:43 +0100
Subject: [PATCH] Apply chunking to era{Exposure, Prefs, Slashes} (#4620)

* Apply chunking to era{Exposure, Prefs, Slashes}

* CHANGELOG

* Failsafe return location

* typo
---
 CHANGELOG.md                            |  1 +
 packages/api-derive/src/staking/util.ts | 41 ++++++++++++++++++++++---
 2 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d513dbf4ed..9f0e937411 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,7 @@
 
 Changes:
 
+- Chunk derive retrieval of `era{Exposure, Prefs, Slashes}`
 - Kusama & Polkadot 9170 upgrade blocks
 - Update to latest Substrate, Kusama & Polkadot static metadata
 
diff --git a/packages/api-derive/src/staking/util.ts b/packages/api-derive/src/staking/util.ts
index f3e04408b7..8e589bdb8c 100644
--- a/packages/api-derive/src/staking/util.ts
+++ b/packages/api-derive/src/staking/util.ts
@@ -7,12 +7,40 @@ import type { EraIndex } from '@polkadot/types/interfaces';
 import type { ExactDerive } from '../derive';
 import type { DeriveApi } from '../types';
 
-import { combineLatest, of, switchMap } from 'rxjs';
+import { BehaviorSubject, combineLatest, map, of, switchMap, tap, toArray } from 'rxjs';
+
+import { arrayChunk, arrayFlatten } from '@polkadot/util';
 
 import { memo } from '../util';
 
 type ApplyReturn<T extends keyof ExactDerive['staking']> = ReturnType<ExactDerive['staking'][T]>;
 
+// only retrieve a maximum of 14 eras (84 / 6) at a time
+// (This is not empirically calculated. Rather smaller sizes take longer
+// time due to the serial nature, large sizes may tie up the RPCs)
+const ERA_CHUNK_SIZE = 14;
+
+function chunkEras <T> (eras: EraIndex[], fn: (eras: EraIndex[]) => Observable<T[]>): Observable<T[]> {
+  const chunked = arrayChunk(eras, ERA_CHUNK_SIZE);
+  let index = 0;
+  const startSubject = new BehaviorSubject<EraIndex[]>(chunked[index]);
+
+  return startSubject.pipe(
+    switchMap(fn),
+    tap((): void => {
+      setTimeout((): void => {
+        index++;
+
+        index === chunked.length
+          ? startSubject.complete()
+          : startSubject.next(chunked[index]);
+      }, 0);
+    }),
+    toArray(),
+    map(arrayFlatten)
+  );
+}
+
 export function filterEras <T extends { era: EraIndex }> (eras: EraIndex[], list: T[]): EraIndex[] {
   return eras.filter((e) => !list.some(({ era }) => e.eq(era)));
 }
@@ -53,9 +81,12 @@ export function combineEras <F extends '_eraExposure' | '_eraPrefs' | '_eraSlash
     // Cannot quite get the typing right, but it is right in the code
     // eslint-disable-next-line @typescript-eslint/no-unsafe-return
     memo(instanceId, (eras: EraIndex[], withActive: boolean) =>
-      // eslint-disable-next-line @typescript-eslint/no-unsafe-return
-      eras.length
-        ? combineLatest(eras.map((e) => api.derive.staking[fn](e, withActive)))
-        : of([])
+      !eras.length
+        ? of([])
+        : chunkEras(eras, (eras) =>
+          combineLatest(
+            eras.map((e) => api.derive.staking[fn](e, withActive))
+          )
+        )
     ) as any;
 }