sitemap/dist/lib/sitemap-parser.js

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.ObjectStreamToJSON = exports.parseSitemap = exports.XMLToSitemapItemStream = void 0;
const sax_1 = __importDefault(require("sax"));
const stream_1 = require("stream");
const types_1 = require("./types");
function isValidTagName(tagName) {
    // This only works because the enum name and value are the same
    return tagName in types_1.TagNames;
}
function tagTemplate() {
    return {
        img: [],
        video: [],
        links: [],
        url: '',
    };
}
function videoTemplate() {
    return {
        tag: [],
        thumbnail_loc: '',
        title: '',
        description: '',
    };
}
const imageTemplate = {
    url: '',
};
const linkTemplate = {
    lang: '',
    url: '',
};
function newsTemplate() {
    return {
        publication: { name: '', language: '' },
        publication_date: '',
        title: '',
    };
}
const defaultLogger = (level, ...message) => console[level](...message);
const defaultStreamOpts = {
    logger: defaultLogger,
};
// TODO does this need to end with `options`
/**
 * Takes a stream of xml and transforms it into a stream of SitemapItems
 * Use this to parse existing sitemaps into config options compatible with this library
 */
class XMLToSitemapItemStream extends stream_1.Transform {
    constructor(opts = defaultStreamOpts) {
        var _a;
        opts.objectMode = true;
        super(opts);
        this.saxStream = sax_1.default.createStream(true, {
            xmlns: true,
            // eslint-disable-next-line @typescript-eslint/ban-ts-comment
            // @ts-ignore
            strictEntities: true,
            trim: true,
        });
        this.level = opts.level || types_1.ErrorLevel.WARN;
        if (this.level !== types_1.ErrorLevel.SILENT && opts.logger !== false) {
            this.logger = (_a = opts.logger) !== null && _a !== void 0 ? _a : defaultLogger;
        }
        else {
            this.logger = () => undefined;
        }
        let currentItem = tagTemplate();
        let currentTag;
        let currentVideo = videoTemplate();
        let currentImage = { ...imageTemplate };
        let currentLink = { ...linkTemplate };
        let dontpushCurrentLink = false;
        this.saxStream.on('opentagstart', (tag) => {
            currentTag = tag.name;
            if (currentTag.startsWith('news:') && !currentItem.news) {
                currentItem.news = newsTemplate();
            }
        });
        this.saxStream.on('opentag', (tag) => {
            if (isValidTagName(tag.name)) {
                if (tag.name === 'xhtml:link') {
                    if (typeof tag.attributes.rel === 'string' ||
                        typeof tag.attributes.href === 'string') {
                        return;
                    }
                    if (tag.attributes.rel.value === 'alternate' &&
                        tag.attributes.hreflang) {
                        currentLink.url = tag.attributes.href.value;
                        if (typeof tag.attributes.hreflang === 'string')
                            return;
                        currentLink.lang = tag.attributes.hreflang.value;
                    }
                    else if (tag.attributes.rel.value === 'alternate') {
                        dontpushCurrentLink = true;
                        currentItem.androidLink = tag.attributes.href.value;
                    }
                    else if (tag.attributes.rel.value === 'amphtml') {
                        dontpushCurrentLink = true;
                        currentItem.ampLink = tag.attributes.href.value;
                    }
                    else {
                        this.logger('log', 'unhandled attr for xhtml:link', tag.attributes);
                    }
                }
            }
            else {
                this.logger('warn', 'unhandled tag', tag.name);
            }
        });
        this.saxStream.on('text', (text) => {
            switch (currentTag) {
                case 'mobile:mobile':
                    break;
                case types_1.TagNames.loc:
                    currentItem.url = text;
                    break;
                case types_1.TagNames.changefreq:
                    if (types_1.isValidChangeFreq(text)) {
                        currentItem.changefreq = text;
                    }
                    break;
                case types_1.TagNames.priority:
                    currentItem.priority = parseFloat(text);
                    break;
                case types_1.TagNames.lastmod:
                    currentItem.lastmod = text;
                    break;
                case types_1.TagNames['video:thumbnail_loc']:
                    currentVideo.thumbnail_loc = text;
                    break;
                case types_1.TagNames['video:tag']:
                    currentVideo.tag.push(text);
                    break;
                case types_1.TagNames['video:duration']:
                    currentVideo.duration = parseInt(text, 10);
                    break;
                case types_1.TagNames['video:player_loc']:
                    currentVideo.player_loc = text;
                    break;
                case types_1.TagNames['video:content_loc']:
                    currentVideo.content_loc = text;
                    break;
                case types_1.TagNames['video:requires_subscription']:
                    if (types_1.isValidYesNo(text)) {
                        currentVideo.requires_subscription = text;
                    }
                    break;
                case types_1.TagNames['video:publication_date']:
                    currentVideo.publication_date = text;
                    break;
                case types_1.TagNames['video:id']:
                    currentVideo.id = text;
                    break;
                case types_1.TagNames['video:restriction']:
                    currentVideo.restriction = text;
                    break;
                case types_1.TagNames['video:view_count']:
                    currentVideo.view_count = parseInt(text, 10);
                    break;
                case types_1.TagNames['video:uploader']:
                    currentVideo.uploader = text;
                    break;
                case types_1.TagNames['video:family_friendly']:
                    if (types_1.isValidYesNo(text)) {
                        currentVideo.family_friendly = text;
                    }
                    break;
                case types_1.TagNames['video:expiration_date']:
                    currentVideo.expiration_date = text;
                    break;
                case types_1.TagNames['video:platform']:
                    currentVideo.platform = text;
                    break;
                case types_1.TagNames['video:price']:
                    currentVideo.price = text;
                    break;
                case types_1.TagNames['video:rating']:
                    currentVideo.rating = parseFloat(text);
                    break;
                case types_1.TagNames['video:category']:
                    currentVideo.category = text;
                    break;
                case types_1.TagNames['video:live']:
                    if (types_1.isValidYesNo(text)) {
                        currentVideo.live = text;
                    }
                    break;
                case types_1.TagNames['video:gallery_loc']:
                    currentVideo.gallery_loc = text;
                    break;
                case types_1.TagNames['image:loc']:
                    currentImage.url = text;
                    break;
                case types_1.TagNames['image:geo_location']:
                    currentImage.geoLocation = text;
                    break;
                case types_1.TagNames['image:license']:
                    currentImage.license = text;
                    break;
                case types_1.TagNames['news:access']:
                    if (!currentItem.news) {
                        currentItem.news = newsTemplate();
                    }
                    currentItem.news.access = text;
                    break;
                case types_1.TagNames['news:genres']:
                    if (!currentItem.news) {
                        currentItem.news = newsTemplate();
                    }
                    currentItem.news.genres = text;
                    break;
                case types_1.TagNames['news:publication_date']:
                    if (!currentItem.news) {
                        currentItem.news = newsTemplate();
                    }
                    currentItem.news.publication_date = text;
                    break;
                case types_1.TagNames['news:keywords']:
                    if (!currentItem.news) {
                        currentItem.news = newsTemplate();
                    }
                    currentItem.news.keywords = text;
                    break;
                case types_1.TagNames['news:stock_tickers']:
                    if (!currentItem.news) {
                        currentItem.news = newsTemplate();
                    }
                    currentItem.news.stock_tickers = text;
                    break;
                case types_1.TagNames['news:language']:
                    if (!currentItem.news) {
                        currentItem.news = newsTemplate();
                    }
                    currentItem.news.publication.language = text;
                    break;
                case types_1.TagNames['video:title']:
                    currentVideo.title += text;
                    break;
                case types_1.TagNames['video:description']:
                    currentVideo.description += text;
                    break;
                case types_1.TagNames['news:name']:
                    if (!currentItem.news) {
                        currentItem.news = newsTemplate();
                    }
                    currentItem.news.publication.name += text;
                    break;
                case types_1.TagNames['news:title']:
                    if (!currentItem.news) {
                        currentItem.news = newsTemplate();
                    }
                    currentItem.news.title += text;
                    break;
                case types_1.TagNames['image:caption']:
                    if (!currentImage.caption) {
                        currentImage.caption = text;
                    }
                    else {
                        currentImage.caption += text;
                    }
                    break;
                case types_1.TagNames['image:title']:
                    if (!currentImage.title) {
                        currentImage.title = text;
                    }
                    else {
                        currentImage.title += text;
                    }
                    break;
                default:
                    this.logger('log', 'unhandled text for tag:', currentTag, `'${text}'`);
                    break;
            }
        });
        this.saxStream.on('cdata', (text) => {
            switch (currentTag) {
                case types_1.TagNames['video:title']:
                    currentVideo.title += text;
                    break;
                case types_1.TagNames['video:description']:
                    currentVideo.description += text;
                    break;
                case types_1.TagNames['news:name']:
                    if (!currentItem.news) {
                        currentItem.news = newsTemplate();
                    }
                    currentItem.news.publication.name += text;
                    break;
                case types_1.TagNames['news:title']:
                    if (!currentItem.news) {
                        currentItem.news = newsTemplate();
                    }
                    currentItem.news.title += text;
                    break;
                case types_1.TagNames['image:caption']:
                    if (!currentImage.caption) {
                        currentImage.caption = text;
                    }
                    else {
                        currentImage.caption += text;
                    }
                    break;
                case types_1.TagNames['image:title']:
                    if (!currentImage.title) {
                        currentImage.title = text;
                    }
                    else {
                        currentImage.title += text;
                    }
                    break;
                default:
                    this.logger('log', 'unhandled cdata for tag:', currentTag);
                    break;
            }
        });
        this.saxStream.on('attribute', (attr) => {
            switch (currentTag) {
                case types_1.TagNames['urlset']:
                case types_1.TagNames['xhtml:link']:
                case types_1.TagNames['video:id']:
                    break;
                case types_1.TagNames['video:restriction']:
                    if (attr.name === 'relationship' && types_1.isAllowDeny(attr.value)) {
                        currentVideo['restriction:relationship'] = attr.value;
                    }
                    else {
                        this.logger('log', 'unhandled attr', currentTag, attr.name);
                    }
                    break;
                case types_1.TagNames['video:price']:
                    if (attr.name === 'type' && types_1.isPriceType(attr.value)) {
                        currentVideo['price:type'] = attr.value;
                    }
                    else if (attr.name === 'currency') {
                        currentVideo['price:currency'] = attr.value;
                    }
                    else if (attr.name === 'resolution' && types_1.isResolution(attr.value)) {
                        currentVideo['price:resolution'] = attr.value;
                    }
                    else {
                        this.logger('log', 'unhandled attr for video:price', attr.name);
                    }
                    break;
                case types_1.TagNames['video:player_loc']:
                    if (attr.name === 'autoplay') {
                        currentVideo['player_loc:autoplay'] = attr.value;
                    }
                    else if (attr.name === 'allow_embed' && types_1.isValidYesNo(attr.value)) {
                        currentVideo['player_loc:allow_embed'] = attr.value;
                    }
                    else {
                        this.logger('log', 'unhandled attr for video:player_loc', attr.name);
                    }
                    break;
                case types_1.TagNames['video:platform']:
                    if (attr.name === 'relationship' && types_1.isAllowDeny(attr.value)) {
                        currentVideo['platform:relationship'] = attr.value;
                    }
                    else {
                        this.logger('log', 'unhandled attr for video:platform', attr.name, attr.value);
                    }
                    break;
                case types_1.TagNames['video:gallery_loc']:
                    if (attr.name === 'title') {
                        currentVideo['gallery_loc:title'] = attr.value;
                    }
                    else {
                        this.logger('log', 'unhandled attr for video:galler_loc', attr.name);
                    }
                    break;
                case types_1.TagNames['video:uploader']:
                    if (attr.name === 'info') {
                        currentVideo['uploader:info'] = attr.value;
                    }
                    else {
                        this.logger('log', 'unhandled attr for video:uploader', attr.name);
                    }
                    break;
                default:
                    this.logger('log', 'unhandled attr', currentTag, attr.name);
            }
        });
        this.saxStream.on('closetag', (tag) => {
            switch (tag) {
                case types_1.TagNames.url:
                    this.push(currentItem);
                    currentItem = tagTemplate();
                    break;
                case types_1.TagNames['video:video']:
                    currentItem.video.push(currentVideo);
                    currentVideo = videoTemplate();
                    break;
                case types_1.TagNames['image:image']:
                    currentItem.img.push(currentImage);
                    currentImage = { ...imageTemplate };
                    break;
                case types_1.TagNames['xhtml:link']:
                    if (!dontpushCurrentLink) {
                        currentItem.links.push(currentLink);
                    }
                    currentLink = { ...linkTemplate };
                    break;
                default:
                    break;
            }
        });
    }
    _transform(data, encoding, callback) {
        // correcting the type here can be done without making it a breaking change
        // TODO fix this
        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
        // @ts-ignore
        this.saxStream.write(data, encoding);
        callback();
    }
}
exports.XMLToSitemapItemStream = XMLToSitemapItemStream;
/**
  Read xml and resolve with the configuration that would produce it or reject with
  an error
  ```
  const { createReadStream } = require('fs')
  const { parseSitemap, createSitemap } = require('sitemap')
  parseSitemap(createReadStream('./example.xml')).then(
    // produces the same xml
    // you can, of course, more practically modify it or store it
    (xmlConfig) => console.log(createSitemap(xmlConfig).toString()),
    (err) => console.log(err)
  )
  ```
  @param {Readable} xml what to parse
  @return {Promise<SitemapItem[]>} resolves with list of sitemap items that can be fed into a SitemapStream. Rejects with an Error object.
 */
async function parseSitemap(xml) {
    const urls = [];
    return new Promise((resolve, reject) => {
        xml
            .pipe(new XMLToSitemapItemStream())
            .on('data', (smi) => urls.push(smi))
            .on('end', () => {
            resolve(urls);
        })
            .on('error', (error) => {
            reject(error);
        });
    });
}
exports.parseSitemap = parseSitemap;
const defaultObjectStreamOpts = {
    lineSeparated: false,
};
/**
 * A Transform that converts a stream of objects into a JSON Array or a line
 * separated stringified JSON
 * @param [lineSeparated=false] whether to separate entries by a new line or comma
 */
class ObjectStreamToJSON extends stream_1.Transform {
    constructor(opts = defaultObjectStreamOpts) {
        opts.writableObjectMode = true;
        super(opts);
        this.lineSeparated = opts.lineSeparated;
        this.firstWritten = false;
    }
    _transform(chunk, encoding, cb) {
        if (!this.firstWritten) {
            this.firstWritten = true;
            if (!this.lineSeparated) {
                this.push('[');
            }
        }
        else if (this.lineSeparated) {
            this.push('\n');
        }
        else {
            this.push(',');
        }
        if (chunk) {
            this.push(JSON.stringify(chunk));
        }
        cb();
    }
    _flush(cb) {
        if (!this.lineSeparated) {
            this.push(']');
        }
        cb();
    }
}
exports.ObjectStreamToJSON = ObjectStreamToJSON;