1 | "use strict";
|
2 | Object.defineProperty(exports, "__esModule", { value: true });
|
3 | exports.transformLogProgress = void 0;
|
4 | const js_lib_1 = require("@naturalcycles/js-lib");
|
5 | const time_lib_1 = require("@naturalcycles/time-lib");
|
6 | const stream_1 = require("stream");
|
7 | const util_1 = require("util");
|
8 | const colors_1 = require("../../colors");
|
9 | const inspectOpt = {
|
10 | colors: true,
|
11 | breakLength: 200,
|
12 | };
|
13 |
|
14 |
|
15 |
|
16 | function transformLogProgress(opt = {}) {
|
17 | const { metric = 'progress', heapTotal: logHeapTotal = false, heapUsed: logHeapUsed = true, rss: logRss = true, logRPS = true, logEvery = 1000, extra, } = opt;
|
18 | const logProgress = opt.logProgress !== false && logEvery !== 0;
|
19 | const logEvery10 = logEvery * 10;
|
20 | const started = Date.now();
|
21 | let lastSecondStarted = Date.now();
|
22 | const sma = new js_lib_1.SimpleMovingAverage(10);
|
23 | let processedLastSecond = 0;
|
24 | let progress = 0;
|
25 | logStats();
|
26 | return new stream_1.Transform({
|
27 | objectMode: true,
|
28 | ...opt,
|
29 | transform(chunk, _encoding, cb) {
|
30 | progress++;
|
31 | processedLastSecond++;
|
32 | if (logProgress && progress % logEvery === 0) {
|
33 | logStats(chunk, false, progress % logEvery10 === 0);
|
34 | }
|
35 | cb(null, chunk);
|
36 | },
|
37 | final(cb) {
|
38 | logStats(undefined, true);
|
39 | cb();
|
40 | },
|
41 | });
|
42 | function logStats(chunk, final = false, tenx = false) {
|
43 | if (!logProgress)
|
44 | return;
|
45 | const mem = process.memoryUsage();
|
46 | const now = Date.now();
|
47 | const lastRPS = processedLastSecond / ((now - lastSecondStarted) / 1000) || 0;
|
48 | const rpsTotal = Math.round(progress / ((now - started) / 1000)) || 0;
|
49 | lastSecondStarted = now;
|
50 | processedLastSecond = 0;
|
51 | const rps10 = Math.round(sma.push(lastRPS));
|
52 | console.log(util_1.inspect({
|
53 | [final ? `${metric}_final` : metric]: progress,
|
54 | ...(extra ? extra(chunk, progress) : {}),
|
55 | ...(logHeapUsed ? { heapUsed: js_lib_1._mb(mem.heapUsed) } : {}),
|
56 | ...(logHeapTotal ? { heapTotal: js_lib_1._mb(mem.heapTotal) } : {}),
|
57 | ...(logRss ? { rss: js_lib_1._mb(mem.rss) } : {}),
|
58 | ...(opt.rssMinusHeap ? { rssMinusHeap: js_lib_1._mb(mem.rss - mem.heapTotal) } : {}),
|
59 | ...(opt.external ? { external: js_lib_1._mb(mem.external) } : {}),
|
60 | ...(opt.arrayBuffers ? { arrayBuffers: js_lib_1._mb(mem.arrayBuffers || 0) } : {}),
|
61 | ...(logRPS
|
62 | ? {
|
63 | rps10,
|
64 | rpsTotal,
|
65 | }
|
66 | : {}),
|
67 | }, inspectOpt));
|
68 | if (tenx) {
|
69 | let perHour = Math.round((progress * 1000 * 60 * 60) / (now - started)) || 0;
|
70 | if (perHour > 900) {
|
71 | perHour = Math.round(perHour / 1000) + 'K';
|
72 | }
|
73 | console.log(`${colors_1.dimGrey(time_lib_1.dayjs().toPretty())} ${colors_1.white(metric)} took ${colors_1.yellow(js_lib_1._since(started))} so far to process ${colors_1.yellow(progress)} rows, ~${colors_1.yellow(perHour)}/hour`);
|
74 | }
|
75 | else if (final) {
|
76 | console.log(`${colors_1.boldWhite(metric)} took ${colors_1.yellow(js_lib_1._since(started))} to process ${colors_1.yellow(progress)} rows with total RPS of ${colors_1.yellow(rpsTotal)}`);
|
77 | }
|
78 | }
|
79 | }
|
80 | exports.transformLogProgress = transformLogProgress;
|