1 | "use strict";
|
2 | Object.defineProperty(exports, "__esModule", { value: true });
|
3 | exports.pipelineFromNDJsonFile = void 0;
|
4 | const zlib_1 = require("zlib");
|
5 | const js_lib_1 = require("@naturalcycles/js-lib");
|
6 | const fs = require("fs-extra");
|
7 | const __1 = require("../..");
|
8 | const colors_1 = require("../../colors");
|
9 | const ndjson_model_1 = require("./ndjson.model");
|
10 | const transformJsonParse_1 = require("./transformJsonParse");
|
11 |
|
12 |
|
13 |
|
14 | async function pipelineFromNDJsonFile(streams, opt) {
|
15 | const { filePath, gzip, separator } = opt;
|
16 | const started = Date.now();
|
17 | let rows = 0;
|
18 | const { size: sizeBytes } = fs.statSync(filePath);
|
19 | console.log(`<< ${(0, colors_1.grey)(filePath)} ${(0, colors_1.dimWhite)((0, js_lib_1._hb)(sizeBytes))} started...`);
|
20 | await (0, __1._pipeline)([
|
21 | fs.createReadStream(filePath),
|
22 | ...(gzip ? [(0, zlib_1.createUnzip)(opt.zlibOptions)] : []),
|
23 | (0, __1.transformSplit)(separator),
|
24 | (0, transformJsonParse_1.transformJsonParse)(opt),
|
25 | (0, __1.transformTap)(() => rows++),
|
26 | ...streams,
|
27 | ]);
|
28 | const stats = ndjson_model_1.NDJsonStats.create({
|
29 | tookMillis: Date.now() - started,
|
30 | rows,
|
31 | sizeBytes,
|
32 | });
|
33 | console.log(`<< ${(0, colors_1.grey)(filePath)}\n` + stats.toPretty());
|
34 | return stats;
|
35 | }
|
36 | exports.pipelineFromNDJsonFile = pipelineFromNDJsonFile;
|