diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index 9b23aa0f3b..c9ce3de2b1 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -7,6 +7,7 @@ on: jobs: semgrep: name: Scan + if: "! startsWith(github.event.head_commit.message, '[CI Skip]') && github.repository == 'polkadot-js/api'" runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -15,3 +16,9 @@ jobs: auditOn: push publishToken: ${{ secrets.SEMGREP_APP_TOKEN }} publishDeployment: 1395 + dummy: + runs-on: ubuntu-latest + steps: + - name: dummy + run: | + echo "Dummy skip step" diff --git a/CHANGELOG.md b/CHANGELOG.md index 79e3d72b48..72e0839f5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ Changes: - Add & adjust types for latest Cumulus code - Added 9031 upgrade block for Westend - Update static metadata to latest Substrate +- Optimize map retrieval with split page sizes for keys & values - Sanitizing `BN` imports to `type`-only as required diff --git a/packages/api/src/base/Decorate.ts b/packages/api/src/base/Decorate.ts index b427c7b23e..80d1e13361 100644 --- a/packages/api/src/base/Decorate.ts +++ b/packages/api/src/base/Decorate.ts @@ -39,7 +39,8 @@ interface MetaDecoration { } // the max amount of keys/values that we will retrieve at once -const PAGE_SIZE = 256; +const PAGE_SIZE_K = 768; // limited since the trie lookups are heavy +const PAGE_SIZE_V = 256; // limited since the data may be very large const l = logger('api/init'); @@ -471,15 +472,14 @@ export abstract class Decorate extends Events { return of([]); } + const queryCall = this.hasSubscriptions + ? this._rpcCore.state.subscribeStorage + : this._rpcCore.state.queryStorageAt; + return combineLatest( - arrayChunk(keys, PAGE_SIZE).map((keys) => - (this.hasSubscriptions - ? this._rpcCore.state.subscribeStorage - : this._rpcCore.state.queryStorageAt - )(keys) - ) + arrayChunk(keys, PAGE_SIZE_V).map((keys) => queryCall(keys)) ).pipe( - map((valsArr): Codec[] => arrayFlatten(valsArr)) + map(arrayFlatten) ); } @@ -488,25 +488,22 @@ export abstract class Decorate extends Events { const headKey = iterKey(...args).toHex(); const startSubject = new BehaviorSubject(headKey); - const query = at - ? (startKey: string) => this._rpcCore.state.getKeysPaged(headKey, PAGE_SIZE, startKey, at) - : (startKey: string) => this._rpcCore.state.getKeysPaged(headKey, PAGE_SIZE, startKey); + const queryCall = at + ? (startKey: string) => this._rpcCore.state.getKeysPaged(headKey, PAGE_SIZE_K, startKey, at) + : (startKey: string) => this._rpcCore.state.getKeysPaged(headKey, PAGE_SIZE_K, startKey); return startSubject.pipe( - switchMap((startKey) => - query(startKey).pipe( - map((keys) => keys.map((key) => key.setMeta(meta, section, method))) - ) - ), + switchMap(queryCall), + map((keys) => keys.map((key) => key.setMeta(meta, section, method))), tap((keys): void => { setTimeout((): void => { - keys.length === PAGE_SIZE - ? startSubject.next(keys[PAGE_SIZE - 1].toHex()) + keys.length === PAGE_SIZE_K + ? startSubject.next(keys[PAGE_SIZE_K - 1].toHex()) : startSubject.complete(); }, 0); }), toArray(), // toArray since we want to startSubject to be completed - map((keysArr: StorageKey[][]) => arrayFlatten(keysArr)) + map(arrayFlatten) ); } @@ -528,7 +525,7 @@ export abstract class Decorate extends Events { return this._retrieveMapKeys(entry, at, args).pipe( switchMap((keys) => keys.length - ? combineLatest(arrayChunk(keys, PAGE_SIZE).map(query)).pipe( + ? combineLatest(arrayChunk(keys, PAGE_SIZE_V).map(query)).pipe( map((valsArr) => arrayFlatten(valsArr).map((value, index): [StorageKey, Codec] => [keys[index], value]) )