1 | "use strict";
|
2 | Object.defineProperty(exports, "__esModule", { value: true });
|
3 | exports.pipelineToNDJsonFile = void 0;
|
4 | const zlib_1 = require("zlib");
|
5 | const js_lib_1 = require("@naturalcycles/js-lib");
|
6 | const fs = require("fs-extra");
|
7 | const __1 = require("../..");
|
8 | const colors_1 = require("../../colors");
|
9 | const ndjson_model_1 = require("./ndjson.model");
|
10 | const transformToNDJson_1 = require("./transformToNDJson");
|
11 |
|
12 |
|
13 |
|
14 |
|
15 |
|
16 | async function pipelineToNDJsonFile(streams, opt) {
|
17 | const { filePath, gzip, protectFromOverwrite = false } = opt;
|
18 | if (protectFromOverwrite && fs.pathExistsSync(filePath)) {
|
19 | throw new js_lib_1.AppError(`pipelineToNDJsonFile: output file exists: ${filePath}`);
|
20 | }
|
21 | const started = Date.now();
|
22 | let rows = 0;
|
23 | fs.ensureFileSync(filePath);
|
24 | console.log(`>> ${(0, colors_1.grey)(filePath)} started...`);
|
25 | await (0, __1._pipeline)([
|
26 | ...streams,
|
27 | (0, __1.transformTap)(() => rows++),
|
28 | (0, transformToNDJson_1.transformToNDJson)(opt),
|
29 | ...(gzip ? [(0, zlib_1.createGzip)(opt.zlibOptions)] : []),
|
30 | fs.createWriteStream(filePath),
|
31 | ]);
|
32 | const { size: sizeBytes } = fs.statSync(filePath);
|
33 | const stats = ndjson_model_1.NDJsonStats.create({
|
34 | tookMillis: Date.now() - started,
|
35 | rows,
|
36 | sizeBytes,
|
37 | });
|
38 | console.log(`>> ${(0, colors_1.grey)(filePath)}\n` + stats.toPretty());
|
39 | return stats;
|
40 | }
|
41 | exports.pipelineToNDJsonFile = pipelineToNDJsonFile;
|