Split page sizes for keys & values on maps (#3620)

* Ssplit page sizes for keys & values on maps

* Adjust semgrep

.github/workflows/semgrep.yml

@@ -7,6 +7,7 @@ on:
 jobs:
   semgrep:
     name: Scan
+    if: "! startsWith(github.event.head_commit.message, '[CI Skip]') && github.repository == 'polkadot-js/api'"
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -15,3 +16,9 @@ jobs:
           auditOn: push
           publishToken: ${{ secrets.SEMGREP_APP_TOKEN }}
           publishDeployment: 1395
+  dummy:
+    runs-on: ubuntu-latest
+    steps:
+    - name: dummy
+      run: |
+        echo "Dummy skip step"

CHANGELOG.md

@@ -9,6 +9,7 @@ Changes:
 - Add & adjust types for latest Cumulus code
 - Added 9031 upgrade block for Westend
 - Update static metadata to latest Substrate
+- Optimize map retrieval with split page sizes for keys & values
 - Sanitizing `BN` imports to `type`-only as required
 
 

packages/api/src/base/Decorate.ts

@@ -39,7 +39,8 @@ interface MetaDecoration {
 }
 
 // the max amount of keys/values that we will retrieve at once
-const PAGE_SIZE = 256;
+const PAGE_SIZE_K = 768; // limited since the trie lookups are heavy
+const PAGE_SIZE_V = 256; // limited since the data may be very large
 
 const l = logger('api/init');
 
@@ -471,15 +472,14 @@ export abstract class Decorate<ApiType extends ApiTypes> extends Events {
       return of([]);
     }
 
+    const queryCall = this.hasSubscriptions
+      ? this._rpcCore.state.subscribeStorage
+      : this._rpcCore.state.queryStorageAt;
+
     return combineLatest(
-      arrayChunk(keys, PAGE_SIZE).map((keys) =>
+      arrayChunk(keys, PAGE_SIZE_V).map((keys) => queryCall(keys))
-        (this.hasSubscriptions
-          ? this._rpcCore.state.subscribeStorage
-          : this._rpcCore.state.queryStorageAt
-        )(keys)
-      )
     ).pipe(
-      map((valsArr): Codec[] => arrayFlatten(valsArr))
+      map(arrayFlatten)
     );
   }
 
@@ -488,25 +488,22 @@ export abstract class Decorate<ApiType extends ApiTypes> extends Events {
 
     const headKey = iterKey(...args).toHex();
     const startSubject = new BehaviorSubject<string>(headKey);
-    const query = at
+    const queryCall = at
-      ? (startKey: string) => this._rpcCore.state.getKeysPaged(headKey, PAGE_SIZE, startKey, at)
+      ? (startKey: string) => this._rpcCore.state.getKeysPaged(headKey, PAGE_SIZE_K, startKey, at)
-      : (startKey: string) => this._rpcCore.state.getKeysPaged(headKey, PAGE_SIZE, startKey);
+      : (startKey: string) => this._rpcCore.state.getKeysPaged(headKey, PAGE_SIZE_K, startKey);
 
     return startSubject.pipe(
-      switchMap((startKey) =>
+      switchMap(queryCall),
-        query(startKey).pipe(
+      map((keys) => keys.map((key) => key.setMeta(meta, section, method))),
-          map((keys) => keys.map((key) => key.setMeta(meta, section, method)))
-        )
-      ),
       tap((keys): void => {
         setTimeout((): void => {
-          keys.length === PAGE_SIZE
+          keys.length === PAGE_SIZE_K
-            ? startSubject.next(keys[PAGE_SIZE - 1].toHex())
+            ? startSubject.next(keys[PAGE_SIZE_K - 1].toHex())
             : startSubject.complete();
         }, 0);
       }),
       toArray(), // toArray since we want to startSubject to be completed
-      map((keysArr: StorageKey[][]) => arrayFlatten(keysArr))
+      map(arrayFlatten)
     );
   }
 
@@ -528,7 +525,7 @@ export abstract class Decorate<ApiType extends ApiTypes> extends Events {
     return this._retrieveMapKeys(entry, at, args).pipe(
       switchMap((keys) =>
         keys.length
-          ? combineLatest(arrayChunk(keys, PAGE_SIZE).map(query)).pipe(
+          ? combineLatest(arrayChunk(keys, PAGE_SIZE_V).map(query)).pipe(
             map((valsArr) =>
               arrayFlatten(valsArr).map((value, index): [StorageKey, Codec] => [keys[index], value])
             )