feat: add external pagination support (#2325)

* feat: add external pagination support

* Ordering, no functionality changes

* fix: use correct assertion condition

Also moved the PaginationOptions type to the base types file

* Apply suggestions from code review

Co-authored-by: Jaco Greeff <jacogr@gmail.com>

packages/api/src/base/Decorate.ts

@@ -7,7 +7,7 @@ import { RpcInterface } from '@polkadot/rpc-core/types';
 import { Call, Hash, RuntimeVersion } from '@polkadot/types/interfaces';
 import { AnyFunction, CallFunction, Codec, CodecArg as Arg, ITuple, InterfaceTypes, ModulesWithCalls, Registry, RegistryTypes } from '@polkadot/types/types';
 import { SubmittableExtrinsic } from '../submittable/types';
-import { ApiInterfaceRx, ApiOptions, ApiTypes, DecorateMethod, DecoratedRpc, DecoratedRpcSection, QueryableModuleStorage, QueryableStorage, QueryableStorageEntry, QueryableStorageMulti, QueryableStorageMultiArg, SubmittableExtrinsicFunction, SubmittableExtrinsics, SubmittableModuleExtrinsics } from '../types';
+import { ApiInterfaceRx, ApiOptions, ApiTypes, DecorateMethod, DecoratedRpc, DecoratedRpcSection, PaginationOptions, QueryableModuleStorage, QueryableStorage, QueryableStorageEntry, QueryableStorageMulti, QueryableStorageMultiArg, SubmittableExtrinsicFunction, SubmittableExtrinsics, SubmittableModuleExtrinsics } from '../types';
 
 import BN from 'bn.js';
 import { BehaviorSubject, Observable, combineLatest, of } from 'rxjs';
@@ -371,10 +371,20 @@ export default abstract class Decorate<ApiType extends ApiTypes> extends Events
         memo((doubleMapArg?: Arg): Observable<[StorageKey, Codec][]> =>
           this._retrieveMapEntries(creator, doubleMapArg)));
 
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access,@typescript-eslint/no-unsafe-assignment
+      decorated.entriesPaged = decorateMethod(
+        memo((opts: PaginationOptions): Observable<[StorageKey, Codec][]> =>
+          this._retrieveMapEntriesPaged(creator, opts)));
+
       // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access,@typescript-eslint/no-unsafe-assignment
       decorated.keys = decorateMethod(
         memo((doubleMapArg?: Arg): Observable<StorageKey[]> =>
           this._retrieveMapKeys(creator, doubleMapArg)));
+
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access,@typescript-eslint/no-unsafe-assignment
+      decorated.keysPaged = decorateMethod(
+        memo((opts: PaginationOptions): Observable<StorageKey[]> =>
+          this._retrieveMapKeysPaged(creator, opts)));
     }
 
     // only support multi where subs are available
@@ -505,6 +515,19 @@ export default abstract class Decorate<ApiType extends ApiTypes> extends Events
       );
   }
 
+  private _retrieveMapKeysPaged ({ iterKey, meta }: StorageEntry, opts: PaginationOptions): Observable<StorageKey[]> {
+    assert(iterKey && (meta.type.isMap || meta.type.isDoubleMap), 'keys can only be retrieved on maps, linked maps and double maps');
+
+    const headKey = iterKey(opts.arg).toHex();
+    const getKeysPaged = this._rpcCore.state.getKeysPaged;
+
+    assert(getKeysPaged, 'Pagination not supported by the chain');
+
+    return getKeysPaged(headKey, opts.pageSize, opts.startKey || headKey).pipe(
+      map((keys) => keys.map((key) => key.setMeta(meta)))
+    );
+  }
+
   private _retrieveMapEntries (entry: StorageEntry, arg?: Arg): Observable<[StorageKey, Codec][]> {
     return this._retrieveMapKeys(entry, arg).pipe(
       switchMap((keys) =>
@@ -529,6 +552,24 @@ export default abstract class Decorate<ApiType extends ApiTypes> extends Events
     );
   }
 
+  private _retrieveMapEntriesPaged (entry: StorageEntry, opts: PaginationOptions): Observable<[StorageKey, Codec][]> {
+    return this._retrieveMapKeysPaged(entry, opts).pipe(
+      switchMap((keys) =>
+        combineLatest<[StorageKey[], ...Codec[][]]>([
+          of(keys),
+          this._rpcCore.state.queryStorageAt
+            ? this._rpcCore.state.queryStorageAt<Codec[]>(keys)
+            : this._rpcCore.state.subscribeStorage<Codec[]>(keys).pipe(take(1))
+        ])
+      ),
+      map(([keys, ...valsArr]): [StorageKey, Codec][] =>
+        valsArr
+          .reduce((result: Codec[], vals) => result.concat(vals), [])
+          .map((value, index) => [keys[index], value])
+      )
+    );
+  }
+
   protected _decorateDeriveRx (decorateMethod: DecorateMethod<ApiType>): DeriveAllSections<'rxjs', ExactDerive> {
     // Pull in derive from api-derive
     const derive = decorateDerive(this._rx, this._options.derives);

packages/api/src/types/base.ts

@@ -3,7 +3,7 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 import { Observable } from 'rxjs';
-import { AnyFunction, Callback, Codec } from '@polkadot/types/types';
+import { AnyFunction, Callback, Codec, CodecArg } from '@polkadot/types/types';
 
 // Prepend an element V onto the beginning of a tuple T.
 // Cons<1, [2,3,4]> is [1,2,3,4]
@@ -68,4 +68,10 @@ export interface DecorateMethodOptions {
   methodName?: string;
 }
 
+export interface PaginationOptions {
+  arg?: CodecArg;
+  pageSize: number;
+  startKey?: string;
+}
+
 export type DecorateMethod<ApiType extends ApiTypes> = <Method extends (...args: any[]) => Observable<any>>(method: Method, options?: DecorateMethodOptions) => any;

packages/api/src/types/storage.ts

@@ -8,7 +8,7 @@ import { Hash } from '@polkadot/types/interfaces';
 import { AnyFunction, Callback, Codec, CodecArg } from '@polkadot/types/types';
 import StorageKey, { StorageEntry } from '@polkadot/types/primitive/StorageKey';
 
-import { ApiTypes, MethodResult, ObsInnerType, PromiseOrObs, UnsubscribePromise } from './base';
+import { ApiTypes, MethodResult, ObsInnerType, PaginationOptions, PromiseOrObs, UnsubscribePromise } from './base';
 
 // eslint-disable-next-line @typescript-eslint/no-empty-interface
 export interface AugmentedQueries<ApiType extends ApiTypes> { }
@@ -29,10 +29,12 @@ export interface StorageEntryBase<ApiType extends ApiTypes, F extends AnyFunctio
   at: <T extends Codec | any = ObsInnerType<ReturnType<F>>>(hash: Hash | Uint8Array | string, ...args: Parameters<F>) => PromiseOrObs<ApiType, T>;
   creator: StorageEntry;
   entries: <T extends Codec | any = ObsInnerType<ReturnType<F>>>(arg?: any) => PromiseOrObs<ApiType, [StorageKey, T][]>;
+  entriesPaged: <T extends Codec | any = ObsInnerType<ReturnType<F>>>(opts: PaginationOptions) => PromiseOrObs<ApiType, [StorageKey, T][]>;
   hash: (...args: Parameters<F>) => PromiseOrObs<ApiType, Hash>;
   key: (...args: Parameters<F>) => string;
   keyPrefix: () => string;
   keys: (arg?: any) => PromiseOrObs<ApiType, StorageKey[]>;
+  keysPaged: (opts: PaginationOptions) => PromiseOrObs<ApiType, StorageKey[]>;
   range: <T extends Codec | any = ObsInnerType<ReturnType<F>>>([from, to]: [Hash | Uint8Array | string, Hash | Uint8Array | string | undefined] | [Hash | Uint8Array | string], ...args: Parameters<F>) => PromiseOrObs<ApiType, [Hash, T][]>;
   size: (...args: Parameters<F>) => PromiseOrObs<ApiType, u64>;
   multi: ApiType extends 'rxjs' ? StorageEntryObservableMulti : StorageEntryPromiseMulti;