1 | "use strict";
|
2 | Object.defineProperty(exports, "__esModule", { value: true });
|
3 | exports.ndjsonMap = void 0;
|
4 | const fs_1 = require("fs");
|
5 | const zlib_1 = require("zlib");
|
6 | const js_lib_1 = require("@naturalcycles/js-lib");
|
7 | const __1 = require("../..");
|
8 |
|
9 |
|
10 |
|
11 |
|
12 | async function ndjsonMap(mapper, opt) {
|
13 | const { inputFilePath, outputFilePath, logEveryOutput = 100000, limitInput, limitOutput } = opt;
|
14 | (0, __1.requireFileToExist)(inputFilePath);
|
15 | console.log({
|
16 | inputFilePath,
|
17 | outputFilePath,
|
18 | });
|
19 | const transformUnzip = inputFilePath.endsWith('.gz') ? [(0, zlib_1.createUnzip)()] : [];
|
20 | const transformZip = outputFilePath.endsWith('.gz') ? [(0, zlib_1.createGzip)()] : [];
|
21 | const readable = (0, fs_1.createReadStream)(inputFilePath);
|
22 | await (0, __1._pipeline)([
|
23 | readable,
|
24 | ...transformUnzip,
|
25 | (0, __1.transformSplit)(),
|
26 | (0, __1.transformJsonParse)(),
|
27 | (0, __1.transformLimit)({ limit: limitInput, sourceReadable: readable }),
|
28 | (0, __1.transformLogProgress)({ metric: 'read', ...opt }),
|
29 | (0, __1.transformMap)(mapper, {
|
30 | flattenArrayOutput: true,
|
31 | errorMode: js_lib_1.ErrorMode.SUPPRESS,
|
32 | ...opt,
|
33 | }),
|
34 | (0, __1.transformLimit)({ limit: limitOutput, sourceReadable: readable }),
|
35 | (0, __1.transformLogProgress)({ metric: 'saved', logEvery: logEveryOutput }),
|
36 | (0, __1.transformToNDJson)(),
|
37 | ...transformZip,
|
38 | (0, fs_1.createWriteStream)(outputFilePath),
|
39 | ]);
|
40 | }
|
41 | exports.ndjsonMap = ndjsonMap;
|