Update Substrate with vesting (#1790)

* Update Substrate with vesting

* Update balances & vesting queries with compat)

* Split base account queries

* Linting, simpler referendum voters

* Fix (some) tests

packages/api-derive/src/balances/account.ts

@@ -0,0 +1,84 @@
+// Copyright 2017-2020 @polkadot/api-derive authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+import { AccountId, AccountData, AccountIndex, Address, Balance, Index } from '@polkadot/types/interfaces';
+import { DerivedBalancesAccount } from '../types';
+
+import { combineLatest, of, Observable } from 'rxjs';
+import { map, switchMap } from 'rxjs/operators';
+import { ApiInterfaceRx } from '@polkadot/api/types';
+import { createType } from '@polkadot/types';
+
+import { memo } from '../util';
+
+type Result = [Balance, Balance, Balance, Balance, Index];
+
+function calcBalances (api: ApiInterfaceRx, [accountId, [freeBalance, reservedBalance, frozenFree, frozenMisc, accountNonce]]: [AccountId, Result]): DerivedBalancesAccount {
+  return {
+    accountId,
+    accountNonce,
+    freeBalance,
+    frozenFree,
+    frozenMisc,
+    reservedBalance,
+    votingBalance: createType(api.registry, 'Balance', freeBalance.add(reservedBalance))
+  };
+}
+
+// old
+function queryOld (api: ApiInterfaceRx, accountId: AccountId): Observable<Result> {
+  return api.queryMulti<[Balance, Balance, Index]>([
+    [api.query.balances.freeBalance, accountId],
+    [api.query.balances.reservedBalance, accountId],
+    [api.query.system.accountNonce, accountId]
+  ]).pipe(
+    map(([freeBalance, reservedBalance, accountNonce]): Result =>
+      [freeBalance, reservedBalance, createType(api.registry, 'Balance'), createType(api.registry, 'Balance'), accountNonce]
+    )
+  );
+}
+
+function queryCurrent (api: ApiInterfaceRx, accountId: AccountId): Observable<Result> {
+  return api.queryMulti<[AccountData, Index]>([
+    [api.query.balances.account, accountId],
+    [api.query.system.accountNonce, accountId]
+  ]).pipe(
+    map(([{ free, reserved, miscFrozen, feeFrozen }, accountNonce]): Result =>
+      [free, reserved, feeFrozen, miscFrozen, accountNonce]
+    )
+  );
+}
+
+/**
+ * @name account
+ * @param {( AccountIndex | AccountId | Address | string )} address - An accounts Id in different formats.
+ * @returns An object containing the results of various balance queries
+ * @example
+ * <BR>
+ *
+ * ```javascript
+ * const ALICE = 'F7Hs';
+ *
+ * api.derive.balances.all(ALICE, ({ accountId, lockedBalance }) => {
+ *   console.log(`The account ${accountId} has a locked balance ${lockedBalance} units.`);
+ * });
+ * ```
+ */
+export function account (api: ApiInterfaceRx): (address: AccountIndex | AccountId | Address | string) => Observable<DerivedBalancesAccount> {
+  return memo((address: AccountIndex | AccountId | Address | string): Observable<DerivedBalancesAccount> =>
+    api.derive.accounts.info(address).pipe(
+      switchMap(({ accountId }): Observable<[AccountId, Result]> =>
+        (accountId
+          ? combineLatest([
+            of(accountId),
+            api.query.balances.account
+              ? queryCurrent(api, accountId)
+              : queryOld(api, accountId)
+          ])
+          : of([createType(api.registry, 'AccountId'), [createType(api.registry, 'Balance'), createType(api.registry, 'Balance'), createType(api.registry, 'Balance'), createType(api.registry, 'Balance'), createType(api.registry, 'Index')]])
+        )
+      ),
+      map((result): DerivedBalancesAccount => calcBalances(api, result))
+    ));
+}

packages/api-derive/src/balances/all.ts

@@ -2,8 +2,8 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
-import { AccountId, AccountIndex, Address, Balance, BalanceLock, BlockNumber, Index, VestingSchedule } from '@polkadot/types/interfaces';
-import { DerivedBalances } from '../types';
+import { AccountId, AccountIndex, Address, Balance, BalanceLock, BalanceLockTo212, BlockNumber, VestingInfo, VestingSchedule } from '@polkadot/types/interfaces';
+import { DerivedBalancesAccount, DerivedBalancesAll } from '../types';
 
 import BN from 'bn.js';
 import { combineLatest, of, Observable } from 'rxjs';
@@ -14,16 +14,16 @@ import { bnMax } from '@polkadot/util';
 
 import { memo } from '../util';
 
-type ResultBalance = [Balance, Balance, BalanceLock[], Option<VestingSchedule>, Index];
-type Result = [AccountId, BlockNumber, ResultBalance];
+type ResultBalance = [VestingInfo | null, (BalanceLock | BalanceLockTo212)[]];
+type Result = [DerivedBalancesAccount, BlockNumber, ResultBalance];
 
-function calcBalances (api: ApiInterfaceRx, [accountId, bestNumber, [freeBalance, reservedBalance, locks, vesting, accountNonce]]: Result): DerivedBalances {
+function calcBalances (api: ApiInterfaceRx, [{ accountId, accountNonce, freeBalance, frozenFree, frozenMisc, reservedBalance, votingBalance }, bestNumber, [vesting, locks]]: Result): DerivedBalancesAll {
   let lockedBalance = createType(api.registry, 'Balance');
-  let lockedBreakdown: BalanceLock[] = [];
+  let lockedBreakdown: (BalanceLock | BalanceLockTo212)[] = [];
 
   if (Array.isArray(locks)) {
     // only get the locks that are valid until passed the current block
-    lockedBreakdown = locks.filter(({ until }): boolean => bestNumber && until.gt(bestNumber));
+    lockedBreakdown = (locks as BalanceLockTo212[]).filter(({ until }): boolean => !until || (bestNumber && until.gt(bestNumber)));
 
     const notAll = lockedBreakdown.filter(({ amount }): boolean => !amount.isMax());
 
@@ -34,11 +34,12 @@ function calcBalances (api: ApiInterfaceRx, [accountId, bestNumber, [freeBalance
   }
 
   // Calculate the vesting balances,
-  //  - offset = balance locked at genesis,
+  //  - offset = balance locked at startingBlock
   //  - perBlock is the unlock amount
-  const { offset: vestingTotal, perBlock } = vesting.unwrapOr(createType(api.registry, 'VestingSchedule'));
-  const vestedBalance = createType(api.registry, 'Balance', perBlock.mul(bestNumber));
-  const isVesting = vestedBalance.lt(vestingTotal);
+  const { locked: vestingTotal, perBlock, startingBlock } = vesting || createType(api.registry, 'VestingInfo');
+  const isStarted = bestNumber.gt(startingBlock);
+  const vestedBalance = createType(api.registry, 'Balance', isStarted ? perBlock.mul(bestNumber.sub(startingBlock)) : 0);
+  const isVesting = isStarted && vestedBalance.lt(vestingTotal);
 
   // The available balance & vested has an interplay here
   // "
@@ -56,16 +57,58 @@ function calcBalances (api: ApiInterfaceRx, [accountId, bestNumber, [freeBalance
     accountNonce,
     availableBalance,
     freeBalance,
+    frozenFree,
+    frozenMisc,
     isVesting,
     lockedBalance,
     lockedBreakdown,
     reservedBalance,
     vestedBalance,
     vestingTotal,
-    votingBalance: createType(api.registry, 'Balance', freeBalance.add(reservedBalance))
+    votingBalance
   };
 }
 
+// old
+function queryOld (api: ApiInterfaceRx, accountId: AccountId): Observable<ResultBalance> {
+  return api.queryMulti<[Vec<BalanceLock>, Option<VestingSchedule>]>([
+    [api.query.balances.locks, accountId],
+    [api.query.balances.vesting, accountId]
+  ]).pipe(
+    map(([locks, optVesting]): ResultBalance => {
+      let vestingNew = null;
+
+      if (optVesting.isSome) {
+        const { offset: locked, perBlock, startingBlock } = optVesting.unwrap();
+
+        vestingNew = createType(api.registry, 'VestingInfo', { locked, perBlock, startingBlock });
+      }
+
+      return [vestingNew, locks];
+    })
+  );
+}
+
+// current (balances  vesting)
+function queryCurrent (api: ApiInterfaceRx, accountId: AccountId): Observable<ResultBalance> {
+  return (
+    api.query.vesting.vesting
+      ? api.queryMulti<[Vec<BalanceLock>, Option<VestingInfo>]>([
+        [api.query.balances.locks, accountId],
+        [api.query.vesting.vesting, accountId]
+      ])
+      : api.query.balances.locks(accountId).pipe(
+        map((locks): [Vec<BalanceLock>, Option<VestingInfo>] =>
+          [locks, createType(api.registry, 'Option<VestingInfo>')]
+        )
+      )
+  ).pipe(
+    map(([locks, optVesting]): ResultBalance =>
+      [optVesting.unwrapOr(null), locks]
+    )
+  );
+}
+
 /**
  * @name all
  * @param {( AccountIndex | AccountId | Address | string )} address - An accounts Id in different formats.
@@ -81,25 +124,21 @@ function calcBalances (api: ApiInterfaceRx, [accountId, bestNumber, [freeBalance
  * });
  * ```
  */
-export function all (api: ApiInterfaceRx): (address: AccountIndex | AccountId | Address | string) => Observable<DerivedBalances> {
-  return memo((address: AccountIndex | AccountId | Address | string): Observable<DerivedBalances> =>
-    api.derive.accounts.info(address).pipe(
-      switchMap(({ accountId }): Observable<Result> =>
-        (accountId
+export function all (api: ApiInterfaceRx): (address: AccountIndex | AccountId | Address | string) => Observable<DerivedBalancesAll> {
+  return memo((address: AccountIndex | AccountId | Address | string): Observable<DerivedBalancesAll> =>
+    api.derive.balances.account(address).pipe(
+      switchMap((account): Observable<Result> =>
+        (!account.accountId.isEmpty
           ? combineLatest([
-            of(accountId),
+            of(account),
             api.derive.chain.bestNumber(),
-            api.queryMulti<[Balance, Balance, Vec<BalanceLock>, Option<VestingSchedule>, Index]>([
-              [api.query.balances.freeBalance, accountId],
-              [api.query.balances.reservedBalance, accountId],
-              [api.query.balances.locks, accountId],
-              [api.query.balances.vesting, accountId],
-              [api.query.system.accountNonce, accountId]
-            ])
+            api.query.balances.account
+              ? queryCurrent(api, account.accountId)
+              : queryOld(api, account.accountId)
           ])
-          : of([createType(api.registry, 'AccountId'), createType(api.registry, 'BlockNumber'), [createType(api.registry, 'Balance'), createType(api.registry, 'Balance'), createType(api.registry, 'Vec<BalanceLock>'), createType(api.registry, 'Option<VestingSchedule>', null), createType(api.registry, 'Index')]])
+          : of([account, createType(api.registry, 'BlockNumber'), [null, createType(api.registry, 'Vec<BalanceLock>')]])
         )
       ),
-      map((result): DerivedBalances => calcBalances(api, result))
+      map((result): DerivedBalancesAll => calcBalances(api, result))
     ));
 }

packages/api-derive/src/balances/fees.ts

@@ -8,22 +8,20 @@ import { DerivedFees } from '../types';
 
 import { Observable, of } from 'rxjs';
 import { map } from 'rxjs/operators';
+import { createType } from '@polkadot/types';
 
 import { memo } from '../util';
 
 type Result = [Balance, Balance, Balance, Balance, Balance];
 
 function query (api: ApiInterfaceRx): Observable<Result> {
-  const paymentBase = api.consts.transactionPayment || api.consts.balances;
-
   return of([
-    // get values from api.const for substrate versions post spec_version: 101
-    // https://github.com/paritytech/substrate/pull/2883/files#diff-5e5e1c3aec9ddfde0a9054d062ab3db9R131
     api.consts.balances.creationFee,
     api.consts.balances.existentialDeposit,
-    api.consts.balances.transferFee,
-    paymentBase.transactionBaseFee as Balance,
-    paymentBase.transactionByteFee as Balance
+    // deprecated - remove
+    (api.consts.balances.transferFee as Balance) || createType(api.registry, 'Balance'),
+    api.consts.transactionPayment.transactionBaseFee,
+    api.consts.transactionPayment.transactionByteFee
   ]);
 }
 

packages/api-derive/src/balances/index.ts

@@ -4,6 +4,7 @@
 
 import { all } from './all';
 
+export * from './account';
 export * from './fees';
 export * from './votingBalances';
 

packages/api-derive/src/balances/votingBalances.ts

@@ -3,20 +3,20 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 import { AccountId, AccountIndex, Address } from '@polkadot/types/interfaces';
-import { DerivedBalances } from '../types';
+import { DerivedBalancesAccount } from '../types';
 
 import { combineLatest, Observable, of } from 'rxjs';
 import { ApiInterfaceRx } from '@polkadot/api/types';
 
 import { memo } from '../util';
 
-export function votingBalances (api: ApiInterfaceRx): (addresses?: (AccountId | AccountIndex | Address | string)[]) => Observable<DerivedBalances[]> {
-  return memo((addresses?: (AccountId | AccountIndex | Address | string)[]): Observable<DerivedBalances[]> =>
+export function votingBalances (api: ApiInterfaceRx): (addresses?: (AccountId | AccountIndex | Address | string)[]) => Observable<DerivedBalancesAccount[]> {
+  return memo((addresses?: (AccountId | AccountIndex | Address | string)[]): Observable<DerivedBalancesAccount[]> =>
     !addresses || !addresses.length
-      ? of([] as DerivedBalances[])
+      ? of([] as DerivedBalancesAccount[])
       : combineLatest(
-        addresses.map((accountId): Observable<DerivedBalances> =>
-          api.derive.balances.all(accountId))
+        addresses.map((accountId): Observable<DerivedBalancesAccount> =>
+          api.derive.balances.account(accountId))
       )
   );
 }

packages/api-derive/src/contracts/fees.ts

@@ -8,6 +8,7 @@ import { DerivedContractFees } from '../types';
 import BN from 'bn.js';
 import { Observable, of } from 'rxjs';
 import { map } from 'rxjs/operators';
+import { createType } from '@polkadot/types';
 
 import { memo } from '../util';
 
@@ -39,7 +40,7 @@ function queryConstants (api: ApiInterfaceRx): Observable<ResultV2> {
     api.consts.contracts.tombstoneDeposit,
     api.consts.contracts.transactionBaseFee,
     api.consts.contracts.transactionByteFee,
-    api.consts.contracts.transferFee
+    api.consts.contracts.transferFee || createType(api.registry, 'Balance')
   ]) as unknown as Observable<ResultV2>;
 }
 

packages/api-derive/src/democracy/referendumVotesFor.ts

@@ -10,13 +10,13 @@ import { map, switchMap } from 'rxjs/operators';
 import { ApiInterfaceRx } from '@polkadot/api/types';
 import { Vec, createType } from '@polkadot/types';
 
-import { DerivedBalances, DerivedReferendumVote } from '../types';
+import { DerivedBalancesAccount, DerivedReferendumVote } from '../types';
 import { memo } from '../util';
 
 export function referendumVotesFor (api: ApiInterfaceRx): (referendumId: BN | number) => Observable<DerivedReferendumVote[]> {
   return memo((referendumId: BN | number): Observable<DerivedReferendumVote[]> =>
     api.query.democracy.votersFor<Vec<AccountId>>(referendumId).pipe(
-      switchMap((votersFor): Observable<[Vec<AccountId>, Vote[], DerivedBalances[]]> =>
+      switchMap((votersFor): Observable<[Vec<AccountId>, Vote[], DerivedBalancesAccount[]]> =>
         combineLatest([
           of(votersFor),
           api.derive.democracy.votes(referendumId, votersFor),

packages/api-derive/src/types.ts

@@ -2,7 +2,7 @@
 // This software may be modified and distributed under the terms
 // of the Apache-2.0 license. See the LICENSE file for details.
 
-import { AccountId, AccountIndex, Balance, BalanceLock, BalanceOf, Bid, BidKind, BlockNumber, EraIndex, EraPoints, Exposure, Hash, Index, Keys, Proposal, PropIndex, ProposalIndex, ReferendumInfo, RegistrationJudgement, RewardDestination, SessionIndex, SetIndex, SocietyVote, StakingLedger, StrikeCount, TreasuryProposal, ValidatorPrefs, Vote, Votes, VoteIndex, VouchingStatus } from '@polkadot/types/interfaces';
+import { AccountId, AccountIndex, Balance, BalanceLock, BalanceLockTo212, BalanceOf, Bid, BidKind, BlockNumber, EraIndex, EraPoints, Exposure, Hash, Index, Keys, Proposal, PropIndex, ProposalIndex, ReferendumInfo, RegistrationJudgement, RewardDestination, SessionIndex, SetIndex, SocietyVote, StakingLedger, StrikeCount, TreasuryProposal, ValidatorPrefs, Vote, Votes, VoteIndex, VouchingStatus } from '@polkadot/types/interfaces';
 
 import BN from 'bn.js';
 import { u32, Vec } from '@polkadot/types';
@@ -30,21 +30,27 @@ export interface DeriveAccountInfo {
   nickname?: string;
 }
 
-export interface DerivedBalances {
+export interface DerivedBalancesAccount {
   accountId: AccountId;
   accountNonce: Index;
   freeBalance: Balance;
+  frozenFree: Balance;
+  frozenMisc: Balance;
+  reservedBalance: Balance;
+  votingBalance: Balance;
+}
+
+export interface DerivedBalancesAll extends DerivedBalancesAccount {
   isVesting: boolean;
   lockedBalance: Balance;
-  lockedBreakdown: BalanceLock[];
+  lockedBreakdown: (BalanceLock | BalanceLockTo212)[];
   availableBalance: Balance;
-  reservedBalance: Balance;
   votingBalance: Balance;
   vestedBalance: Balance;
   vestingTotal: Balance;
 }
 
-export type DerivedBalancesMap = Record<string, DerivedBalances>;
+export type DerivedBalancesMap = Record<string, DerivedBalancesAll>;
 
 export interface DerivedContractFees {
   callBaseFee: BN;