1 | #!/usr/bin/env node
|
2 | "use strict";
|
3 | Object.defineProperty(exports, "__esModule", { value: true });
|
4 | const fs_1 = require("fs");
|
5 | const xmllint_1 = require("./lib/xmllint");
|
6 | const errors_1 = require("./lib/errors");
|
7 | const sitemap_parser_1 = require("./lib/sitemap-parser");
|
8 | const utils_1 = require("./lib/utils");
|
9 | const sitemap_stream_1 = require("./lib/sitemap-stream");
|
10 | const sitemap_index_stream_1 = require("./lib/sitemap-index-stream");
|
11 | const url_1 = require("url");
|
12 | const zlib_1 = require("zlib");
|
13 |
|
14 | const arg = require('arg');
|
15 | const pickStreamOrArg = (argv) => {
|
16 | if (!argv._.length) {
|
17 | return process.stdin;
|
18 | }
|
19 | else {
|
20 | return fs_1.createReadStream(argv._[0], { encoding: 'utf8' });
|
21 | }
|
22 | };
|
23 | const argSpec = {
|
24 | '--help': Boolean,
|
25 | '--version': Boolean,
|
26 | '--validate': Boolean,
|
27 | '--index': Boolean,
|
28 | '--index-base-url': String,
|
29 | '--limit': Number,
|
30 | '--parse': Boolean,
|
31 | '--single-line-json': Boolean,
|
32 | '--prepend': String,
|
33 | '--gzip': Boolean,
|
34 | '-h': '--help',
|
35 | };
|
36 | const argv = arg(argSpec);
|
37 | function getStream() {
|
38 | if (argv._ && argv._.length) {
|
39 | return fs_1.createReadStream(argv._[0]);
|
40 | }
|
41 | else {
|
42 | console.warn('Reading from stdin. If you are not piping anything in, this command is not doing anything');
|
43 | return process.stdin;
|
44 | }
|
45 | }
|
46 | if (argv['--version']) {
|
47 |
|
48 | const packagejson = require('../package.json');
|
49 | console.log(packagejson.version);
|
50 | }
|
51 | else if (argv['--help']) {
|
52 | console.log(`
|
53 | Turn a list of urls into a sitemap xml.
|
54 | Options:
|
55 | --help Print this text
|
56 | --version Print the version
|
57 | --validate Ensure the passed in file is conforms to the sitemap spec
|
58 | --index Create an index and stream that out. Writes out sitemaps along the way.
|
59 | --index-base-url Base url the sitemaps will be hosted eg. https://example.com/sitemaps/
|
60 | --limit=45000 Set a custom limit to the items per sitemap
|
61 | --parse Parse fed xml and spit out config
|
62 | --prepend=sitemap.xml Prepend the streamed in sitemap configs to sitemap.xml
|
63 | --gzip Compress output
|
64 | --single-line-json When used with parse, it spits out each entry as json rather than the whole json.
|
65 |
|
66 | # examples
|
67 |
|
68 | Generate a sitemap index file as well as sitemaps
|
69 | npx sitemap --gzip --index --index-base-url https://example.com/path/to/sitemaps/ < listofurls.txt > sitemap-index.xml.gz
|
70 |
|
71 | Add to a sitemap
|
72 | npx sitemap --prepend sitemap.xml < listofurls.json
|
73 |
|
74 | Turn an existing sitemap into configuration understood by the sitemap library
|
75 | npx sitemap --parse sitemap.xml
|
76 |
|
77 | Use XMLLib to validate your sitemap (requires xmllib)
|
78 | npx sitemap --validate sitemap.xml
|
79 | `);
|
80 | }
|
81 | else if (argv['--parse']) {
|
82 | let oStream = getStream()
|
83 | .pipe(new sitemap_parser_1.XMLToSitemapItemStream())
|
84 | .pipe(new sitemap_parser_1.ObjectStreamToJSON({ lineSeparated: !argv['--single-line-json'] }));
|
85 | if (argv['--gzip']) {
|
86 | oStream = oStream.pipe(zlib_1.createGzip());
|
87 | }
|
88 | oStream.pipe(process.stdout);
|
89 | }
|
90 | else if (argv['--validate']) {
|
91 | xmllint_1.xmlLint(getStream())
|
92 | .then(() => console.log('valid'))
|
93 | .catch(([error, stderr]) => {
|
94 | if (error instanceof errors_1.XMLLintUnavailable) {
|
95 | console.error(error.message);
|
96 | return;
|
97 | }
|
98 | else {
|
99 | console.log(stderr);
|
100 | }
|
101 | });
|
102 | }
|
103 | else if (argv['--index']) {
|
104 | const limit = argv['--limit'];
|
105 | const baseURL = argv['--index-base-url'];
|
106 | if (!baseURL) {
|
107 | throw new Error("You must specify where the sitemaps will be hosted. use --index-base-url 'https://example.com/path'");
|
108 | }
|
109 | const sms = new sitemap_index_stream_1.SitemapAndIndexStream({
|
110 | limit,
|
111 | getSitemapStream: (i) => {
|
112 | const sm = new sitemap_stream_1.SitemapStream();
|
113 | const path = `./sitemap-${i}.xml`;
|
114 | let ws;
|
115 | if (argv['--gzip']) {
|
116 | ws = sm.pipe(zlib_1.createGzip()).pipe(fs_1.createWriteStream(path));
|
117 | }
|
118 | else {
|
119 | ws = sm.pipe(fs_1.createWriteStream(path));
|
120 | }
|
121 | return [new url_1.URL(path, baseURL).toString(), sm, ws];
|
122 | },
|
123 | });
|
124 | let oStream = utils_1.lineSeparatedURLsToSitemapOptions(pickStreamOrArg(argv)).pipe(sms);
|
125 | if (argv['--gzip']) {
|
126 | oStream = oStream.pipe(zlib_1.createGzip());
|
127 | }
|
128 | oStream.pipe(process.stdout);
|
129 | }
|
130 | else {
|
131 | const sms = new sitemap_stream_1.SitemapStream();
|
132 | if (argv['--prepend']) {
|
133 | fs_1.createReadStream(argv['--prepend'])
|
134 | .pipe(new sitemap_parser_1.XMLToSitemapItemStream())
|
135 | .pipe(sms);
|
136 | }
|
137 | const oStream = utils_1.lineSeparatedURLsToSitemapOptions(pickStreamOrArg(argv)).pipe(sms);
|
138 | if (argv['--gzip']) {
|
139 | oStream.pipe(zlib_1.createGzip()).pipe(process.stdout);
|
140 | }
|
141 | else {
|
142 | oStream.pipe(process.stdout);
|
143 | }
|
144 | }
|