Send progress messages more often (#201)

This commit is contained in:
Jaco Greeff
2018-08-29 10:43:37 +02:00
committed by GitHub
parent b8e7e3fd2c
commit ea1549de04
4 changed files with 27 additions and 22 deletions
+2 -2
View File
@@ -2,7 +2,7 @@
// This software may be modified and distributed under the terms
// of the ISC license. See the LICENSE file for details.
import { DiskStore } from './types';
import { DiskStore, ProgressValue } from './types';
import { AbstractLevelDOWN } from 'abstract-leveldown';
// import { LRUMap } from 'lru_map';
@@ -29,7 +29,7 @@ class DiskDown extends AbstractLevelDOWN {
// this._store = new LRUMap(LRU_SIZE);
}
compact (progress: (message: string) => void): void {
compact (progress: (value: ProgressValue) => void): void {
l.debug(() => ['compact']);
this._disk.compact(progress);
+16 -17
View File
@@ -2,7 +2,7 @@
// This software may be modified and distributed under the terms
// of the ISC license. See the LICENSE file for details.
import { DiskStore } from '../types';
import { DiskStore, ProgressValue } from '../types';
import fs from 'fs';
import assert from '@polkadot/util/assert';
@@ -84,7 +84,7 @@ export default class Combined implements DiskStore {
fs.closeSync(this._fd);
}
compact (progress: (message: string) => void): void {
compact (progress: (value: ProgressValue) => void): void {
assert(this._fd === -1, 'Database cannot be open for compacting');
l.log('compacting database');
@@ -93,7 +93,7 @@ export default class Combined implements DiskStore {
const newFile = `${this._file}.compacted`;
const newFd = this._open(newFile, true);
const oldFd = this._open(this._file);
const count = this._compact(progress, newFd, oldFd);
const keys = this._compact(progress, newFd, oldFd);
fs.closeSync(oldFd);
fs.closeSync(newFd);
@@ -108,7 +108,7 @@ export default class Combined implements DiskStore {
// fs.renameSync(this._file, `${this._file}.old`);
fs.renameSync(newFile, this._file);
l.log(`compacted in ${elapsed.toFixed(2)}s, ${(count / 1000).toFixed(2)}k keys, ${sizeMB.toFixed(2)}MB (${percentage.toFixed(2)}%)`);
l.log(`compacted in ${elapsed.toFixed(2)}s, ${(keys / 1000).toFixed(2)}k keys, ${sizeMB.toFixed(2)}MB (${percentage.toFixed(2)}%)`);
}
delete (key: Buffer): void {
@@ -198,16 +198,15 @@ export default class Combined implements DiskStore {
return headerAt;
}
private _compact (progress: (message: string) => void, newFd: number, oldFd: number, newAt: number = 0, oldAt: number = 0, depth: number = 0): number {
private _compact (progress: (value: ProgressValue) => void, newFd: number, oldFd: number, newAt: number = 0, oldAt: number = 0, depth: number = 0, percent: number = 0): number {
// l.debug(() => ['_compact', debug({ newFd, oldFd, newAt, oldAt })]);
let count = 0;
let keys = 0;
for (let index = 0; index < ENTRY_NUM; index++) {
const entry = this._compactReadEntry(oldFd, oldAt, index);
const dataAt = entry.readUIntBE(1, UINT_SIZE);
const entryType = entry[0];
let indexCount = 0;
if (entryType === Slot.EMPTY) {
// l.debug(() => '_compact/isEmpty');
@@ -219,29 +218,29 @@ export default class Combined implements DiskStore {
this._compactUpdateLink(newFd, newAt, index, keyAt, Slot.LEAF);
indexCount++;
keys++;
} else if (entryType === Slot.BRANCH) {
// l.debug(() => '_compact/isBranch');
const headerAt = this._compactWriteHeader(newFd, newAt, index);
indexCount += this._compact(progress, newFd, oldFd, headerAt, dataAt, depth + 1);
keys += this._compact(progress, newFd, oldFd, headerAt, dataAt, depth + 1, percent);
} else {
throw new Error(`Unknown entry type, ${entryType}`);
}
count += indexCount;
percent += (100 / ENTRY_NUM) / Math.pow(ENTRY_NUM, depth);
if (depth === 0) {
const percentage = ` ${(100 * (index + 1) / ENTRY_NUM).toFixed(2)}`;
progress(`${percentage.slice(-6)}% compacted, ${count} keys`);
}
progress({
depth,
keys,
percent
});
}
// l.debug(() => ['_compact', '=>', `${depth}: ${count} keys written`]);
// l.debug(() => ['_compact', '=>', `${depth}: ${keys} keys written`]);
return count;
return keys;
}
private _findKey (key: Buffer, doCreate: boolean, keyIndex: number, diskAt: number): Leaf | null {
+2 -2
View File
@@ -2,7 +2,7 @@
// This software may be modified and distributed under the terms
// of the ISC license. See the LICENSE file for details.
import { DiskStore } from '../types';
import { DiskStore, ProgressValue } from '../types';
import fs from 'fs';
import mkdirp from 'mkdirp';
@@ -54,7 +54,7 @@ export default class Scatter implements DiskStore {
// noop
}
compact (progress: (message: string) => void): void {
compact (progress: (value: ProgressValue) => void): void {
// noop
}
+7 -1
View File
@@ -2,9 +2,15 @@
// This software may be modified and distributed under the terms
// of the ISC license. See the LICENSE file for details.
export type ProgressValue = {
depth: number,
keys: number,
percent: number
};
export interface DiskStore {
close (): void;
compact (progress: (message: string) => void): void,
compact (progress: (value: ProgressValue) => void): void,
delete (key: Buffer): void;
get (key: Buffer): Buffer | undefined;
open (): void,