@naturalcycles/nodejs-lib/dist/stream/ndjson/ndjsonMap.js

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.ndjsonMap = void 0;
const js_lib_1 = require("@naturalcycles/js-lib");
const fs_1 = require("fs");
const path = require("path");
const zlib_1 = require("zlib");
const __1 = require("../..");
/**
 * Unzips input file automatically, if it ends with `.gz`.
 * Zips output file automatically, if it ends with `.gz`.
 */
async function ndjsonMap(opt) {
    const { inputFilePath, outputFilePath, mapperFilePath, logEveryInput = 1000, logEveryOutput = 100000, limitInput, limitOutput, } = opt;
    __1.requireFileToExist(inputFilePath);
    __1.requireFileToExist(mapperFilePath);
    const resolvedMapperPath = path.resolve(mapperFilePath);
    console.log({
        inputFilePath,
        outputFilePath,
        mapperFilePath,
        resolvedMapperPath,
    });
    // This is to allow importing *.ts mappers
    try {
        require('ts-node/register/transpile-only');
        require('tsconfig-paths/register');
    }
    catch { } // require if exists
    const { mapper } = require(resolvedMapperPath);
    if (!mapper) {
        throw new Error(`Mapper file should export "mapper" function`);
    }
    const transformUnzip = inputFilePath.endsWith('.gz') ? [zlib_1.createUnzip()] : [];
    const transformZip = outputFilePath.endsWith('.gz') ? [zlib_1.createGzip()] : [];
    await __1._pipeline([
        fs_1.createReadStream(inputFilePath),
        ...transformUnzip,
        __1.transformSplit(),
        __1.transformJsonParse(),
        __1.transformLimit(limitInput),
        __1.transformLogProgress({ metric: 'read', logEvery: logEveryInput }),
        __1.transformMap(mapper, {
            flattenArrayOutput: true,
            errorMode: js_lib_1.ErrorMode.SUPPRESS,
            ...opt,
        }),
        __1.transformLimit(limitOutput),
        __1.transformLogProgress({ metric: 'saved', logEvery: logEveryOutput }),
        __1.transformToNDJson(),
        ...transformZip,
        fs_1.createWriteStream(outputFilePath),
    ]);
}
exports.ndjsonMap = ndjsonMap;