1 | /**
|
2 | Licensed to the Apache Software Foundation (ASF) under one
|
3 | or more contributor license agreements. See the NOTICE file
|
4 | distributed with this work for additional information
|
5 | regarding copyright ownership. The ASF licenses this file
|
6 | to you under the Apache License, Version 2.0 (the
|
7 | "License"); you may not use this file except in compliance
|
8 | with the License. You may obtain a copy of the License at
|
9 |
|
10 | http://www.apache.org/licenses/LICENSE-2.0
|
11 |
|
12 | Unless required by applicable law or agreed to in writing,
|
13 | software distributed under the License is distributed on an
|
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
15 | KIND, either express or implied. See the License for the
|
16 | specific language governing permissions and limitations
|
17 | under the License.
|
18 | */
|
19 |
|
20 | ;
|
21 |
|
22 | var fs = require('fs');
|
23 | var path = require('path');
|
24 | var shell = require('shelljs');
|
25 | var minimatch = require('minimatch');
|
26 |
|
27 | /**
|
28 | * Logging callback used in the FileUpdater methods.
|
29 | * @callback loggingCallback
|
30 | * @param {string} message A message describing a single file update operation.
|
31 | */
|
32 |
|
33 | /**
|
34 | * Updates a target file or directory with a source file or directory. (Directory updates are
|
35 | * not recursive.) Stats for target and source items must be passed in. This is an internal
|
36 | * helper function used by other methods in this module.
|
37 | *
|
38 | * @param {?string} sourcePath Source file or directory to be used to update the
|
39 | * destination. If the source is null, then the destination is deleted if it exists.
|
40 | * @param {?fs.Stats} sourceStats An instance of fs.Stats for the source path, or null if
|
41 | * the source does not exist.
|
42 | * @param {string} targetPath Required destination file or directory to be updated. If it does
|
43 | * not exist, it will be created.
|
44 | * @param {?fs.Stats} targetStats An instance of fs.Stats for the target path, or null if
|
45 | * the target does not exist.
|
46 | * @param {Object} [options] Optional additional parameters for the update.
|
47 | * @param {string} [options.rootDir] Optional root directory (such as a project) to which target
|
48 | * and source path parameters are relative; may be omitted if the paths are absolute. The
|
49 | * rootDir is always omitted from any logged paths, to make the logs easier to read.
|
50 | * @param {boolean} [options.all] If true, all files are copied regardless of last-modified times.
|
51 | * Otherwise, a file is copied if the source's last-modified time is greather than or
|
52 | * equal to the target's last-modified time, or if the file sizes are different.
|
53 | * @param {loggingCallback} [log] Optional logging callback that takes a string message
|
54 | * describing any file operations that are performed.
|
55 | * @return {boolean} true if any changes were made, or false if the force flag is not set
|
56 | * and everything was up to date
|
57 | */
|
58 | function updatePathWithStats (sourcePath, sourceStats, targetPath, targetStats, options, log) {
|
59 | var updated = false;
|
60 |
|
61 | var rootDir = (options && options.rootDir) || '';
|
62 | var copyAll = (options && options.all) || false;
|
63 |
|
64 | var targetFullPath = path.join(rootDir || '', targetPath);
|
65 |
|
66 | if (sourceStats) {
|
67 | var sourceFullPath = path.join(rootDir || '', sourcePath);
|
68 |
|
69 | if (targetStats) {
|
70 | // The target exists. But if the directory status doesn't match the source, delete it.
|
71 | if (targetStats.isDirectory() && !sourceStats.isDirectory()) {
|
72 | log('rmdir ' + targetPath + ' (source is a file)');
|
73 | shell.rm('-rf', targetFullPath);
|
74 | targetStats = null;
|
75 | updated = true;
|
76 | } else if (!targetStats.isDirectory() && sourceStats.isDirectory()) {
|
77 | log('delete ' + targetPath + ' (source is a directory)');
|
78 | shell.rm('-f', targetFullPath);
|
79 | targetStats = null;
|
80 | updated = true;
|
81 | }
|
82 | }
|
83 |
|
84 | if (!targetStats) {
|
85 | if (sourceStats.isDirectory()) {
|
86 | // The target directory does not exist, so it should be created.
|
87 | log('mkdir ' + targetPath);
|
88 | shell.mkdir('-p', targetFullPath);
|
89 | updated = true;
|
90 | } else if (sourceStats.isFile()) {
|
91 | // The target file does not exist, so it should be copied from the source.
|
92 | log('copy ' + sourcePath + ' ' + targetPath + (copyAll ? '' : ' (new file)'));
|
93 | shell.cp('-f', sourceFullPath, targetFullPath);
|
94 | updated = true;
|
95 | }
|
96 | } else if (sourceStats.isFile() && targetStats.isFile()) {
|
97 | // The source and target paths both exist and are files.
|
98 | if (copyAll) {
|
99 | // The caller specified all files should be copied.
|
100 | log('copy ' + sourcePath + ' ' + targetPath);
|
101 | shell.cp('-f', sourceFullPath, targetFullPath);
|
102 | updated = true;
|
103 | } else {
|
104 | // Copy if the source has been modified since it was copied to the target, or if
|
105 | // the file sizes are different. (The latter catches most cases in which something
|
106 | // was done to the file after copying.) Comparison is >= rather than > to allow
|
107 | // for timestamps lacking sub-second precision in some filesystems.
|
108 | if (sourceStats.mtime.getTime() >= targetStats.mtime.getTime() ||
|
109 | sourceStats.size !== targetStats.size) {
|
110 | log('copy ' + sourcePath + ' ' + targetPath + ' (updated file)');
|
111 | shell.cp('-f', sourceFullPath, targetFullPath);
|
112 | updated = true;
|
113 | }
|
114 | }
|
115 | }
|
116 | } else if (targetStats) {
|
117 | // The target exists but the source is null, so the target should be deleted.
|
118 | if (targetStats.isDirectory()) {
|
119 | log('rmdir ' + targetPath + (copyAll ? '' : ' (no source)'));
|
120 | shell.rm('-rf', targetFullPath);
|
121 | } else {
|
122 | log('delete ' + targetPath + (copyAll ? '' : ' (no source)'));
|
123 | shell.rm('-f', targetFullPath);
|
124 | }
|
125 | updated = true;
|
126 | }
|
127 |
|
128 | return updated;
|
129 | }
|
130 |
|
131 | /**
|
132 | * Helper for updatePath and updatePaths functions. Queries stats for source and target
|
133 | * and ensures target directory exists before copying a file.
|
134 | */
|
135 | function updatePathInternal (sourcePath, targetPath, options, log) {
|
136 | var rootDir = (options && options.rootDir) || '';
|
137 | var targetFullPath = path.join(rootDir, targetPath);
|
138 | var targetStats = fs.existsSync(targetFullPath) ? fs.statSync(targetFullPath) : null;
|
139 | var sourceStats = null;
|
140 |
|
141 | if (sourcePath) {
|
142 | // A non-null source path was specified. It should exist.
|
143 | var sourceFullPath = path.join(rootDir, sourcePath);
|
144 | if (!fs.existsSync(sourceFullPath)) {
|
145 | throw new Error('Source path does not exist: ' + sourcePath);
|
146 | }
|
147 |
|
148 | sourceStats = fs.statSync(sourceFullPath);
|
149 |
|
150 | // Create the target's parent directory if it doesn't exist.
|
151 | var parentDir = path.dirname(targetFullPath);
|
152 | if (!fs.existsSync(parentDir)) {
|
153 | shell.mkdir('-p', parentDir);
|
154 | }
|
155 | }
|
156 |
|
157 | return updatePathWithStats(sourcePath, sourceStats, targetPath, targetStats, options, log);
|
158 | }
|
159 |
|
160 | /**
|
161 | * Updates a target file or directory with a source file or directory. (Directory updates are
|
162 | * not recursive.)
|
163 | *
|
164 | * @param {?string} sourcePath Source file or directory to be used to update the
|
165 | * destination. If the source is null, then the destination is deleted if it exists.
|
166 | * @param {string} targetPath Required destination file or directory to be updated. If it does
|
167 | * not exist, it will be created.
|
168 | * @param {Object} [options] Optional additional parameters for the update.
|
169 | * @param {string} [options.rootDir] Optional root directory (such as a project) to which target
|
170 | * and source path parameters are relative; may be omitted if the paths are absolute. The
|
171 | * rootDir is always omitted from any logged paths, to make the logs easier to read.
|
172 | * @param {boolean} [options.all] If true, all files are copied regardless of last-modified times.
|
173 | * Otherwise, a file is copied if the source's last-modified time is greather than or
|
174 | * equal to the target's last-modified time, or if the file sizes are different.
|
175 | * @param {loggingCallback} [log] Optional logging callback that takes a string message
|
176 | * describing any file operations that are performed.
|
177 | * @return {boolean} true if any changes were made, or false if the force flag is not set
|
178 | * and everything was up to date
|
179 | */
|
180 | function updatePath (sourcePath, targetPath, options, log) {
|
181 | if (sourcePath !== null && typeof sourcePath !== 'string') {
|
182 | throw new Error('A source path (or null) is required.');
|
183 | }
|
184 |
|
185 | if (!targetPath || typeof targetPath !== 'string') {
|
186 | throw new Error('A target path is required.');
|
187 | }
|
188 |
|
189 | log = log || function (message) { };
|
190 |
|
191 | return updatePathInternal(sourcePath, targetPath, options, log);
|
192 | }
|
193 |
|
194 | /**
|
195 | * Updates files and directories based on a mapping from target paths to source paths. Targets
|
196 | * with null sources in the map are deleted.
|
197 | *
|
198 | * @param {Object} pathMap A dictionary mapping from target paths to source paths.
|
199 | * @param {Object} [options] Optional additional parameters for the update.
|
200 | * @param {string} [options.rootDir] Optional root directory (such as a project) to which target
|
201 | * and source path parameters are relative; may be omitted if the paths are absolute. The
|
202 | * rootDir is always omitted from any logged paths, to make the logs easier to read.
|
203 | * @param {boolean} [options.all] If true, all files are copied regardless of last-modified times.
|
204 | * Otherwise, a file is copied if the source's last-modified time is greather than or
|
205 | * equal to the target's last-modified time, or if the file sizes are different.
|
206 | * @param {loggingCallback} [log] Optional logging callback that takes a string message
|
207 | * describing any file operations that are performed.
|
208 | * @return {boolean} true if any changes were made, or false if the force flag is not set
|
209 | * and everything was up to date
|
210 | */
|
211 | function updatePaths (pathMap, options, log) {
|
212 | if (!pathMap || typeof pathMap !== 'object' || Array.isArray(pathMap)) {
|
213 | throw new Error('An object mapping from target paths to source paths is required.');
|
214 | }
|
215 |
|
216 | log = log || function (message) { };
|
217 |
|
218 | var updated = false;
|
219 |
|
220 | // Iterate in sorted order to ensure directories are created before files under them.
|
221 | Object.keys(pathMap).sort().forEach(function (targetPath) {
|
222 | var sourcePath = pathMap[targetPath];
|
223 | updated = updatePathInternal(sourcePath, targetPath, options, log) || updated;
|
224 | });
|
225 |
|
226 | return updated;
|
227 | }
|
228 |
|
229 | /**
|
230 | * Updates a target directory with merged files and subdirectories from source directories.
|
231 | *
|
232 | * @param {string|string[]} sourceDirs Required source directory or array of source directories
|
233 | * to be merged into the target. The directories are listed in order of precedence; files in
|
234 | * directories later in the array supersede files in directories earlier in the array
|
235 | * (regardless of timestamps).
|
236 | * @param {string} targetDir Required destination directory to be updated. If it does not exist,
|
237 | * it will be created. If it exists, newer files from source directories will be copied over,
|
238 | * and files missing in the source directories will be deleted.
|
239 | * @param {Object} [options] Optional additional parameters for the update.
|
240 | * @param {string} [options.rootDir] Optional root directory (such as a project) to which target
|
241 | * and source path parameters are relative; may be omitted if the paths are absolute. The
|
242 | * rootDir is always omitted from any logged paths, to make the logs easier to read.
|
243 | * @param {boolean} [options.all] If true, all files are copied regardless of last-modified times.
|
244 | * Otherwise, a file is copied if the source's last-modified time is greather than or
|
245 | * equal to the target's last-modified time, or if the file sizes are different.
|
246 | * @param {string|string[]} [options.include] Optional glob string or array of glob strings that
|
247 | * are tested against both target and source relative paths to determine if they are included
|
248 | * in the merge-and-update. If unspecified, all items are included.
|
249 | * @param {string|string[]} [options.exclude] Optional glob string or array of glob strings that
|
250 | * are tested against both target and source relative paths to determine if they are excluded
|
251 | * from the merge-and-update. Exclusions override inclusions. If unspecified, no items are
|
252 | * excluded.
|
253 | * @param {loggingCallback} [log] Optional logging callback that takes a string message
|
254 | * describing any file operations that are performed.
|
255 | * @return {boolean} true if any changes were made, or false if the force flag is not set
|
256 | * and everything was up to date
|
257 | */
|
258 | function mergeAndUpdateDir (sourceDirs, targetDir, options, log) {
|
259 | if (sourceDirs && typeof sourceDirs === 'string') {
|
260 | sourceDirs = [ sourceDirs ];
|
261 | } else if (!Array.isArray(sourceDirs)) {
|
262 | throw new Error('A source directory path or array of paths is required.');
|
263 | }
|
264 |
|
265 | if (!targetDir || typeof targetDir !== 'string') {
|
266 | throw new Error('A target directory path is required.');
|
267 | }
|
268 |
|
269 | log = log || function (message) { };
|
270 |
|
271 | var rootDir = (options && options.rootDir) || '';
|
272 |
|
273 | var include = (options && options.include) || [ '**' ];
|
274 | if (typeof include === 'string') {
|
275 | include = [ include ];
|
276 | } else if (!Array.isArray(include)) {
|
277 | throw new Error('Include parameter must be a glob string or array of glob strings.');
|
278 | }
|
279 |
|
280 | var exclude = (options && options.exclude) || [];
|
281 | if (typeof exclude === 'string') {
|
282 | exclude = [ exclude ];
|
283 | } else if (!Array.isArray(exclude)) {
|
284 | throw new Error('Exclude parameter must be a glob string or array of glob strings.');
|
285 | }
|
286 |
|
287 | // Scan the files in each of the source directories.
|
288 | var sourceMaps = sourceDirs.map(function (sourceDir) {
|
289 | return path.join(rootDir, sourceDir);
|
290 | }).map(function (sourcePath) {
|
291 | if (!fs.existsSync(sourcePath)) {
|
292 | throw new Error('Source directory does not exist: ' + sourcePath);
|
293 | }
|
294 | return mapDirectory(rootDir, path.relative(rootDir, sourcePath), include, exclude);
|
295 | });
|
296 |
|
297 | // Scan the files in the target directory, if it exists.
|
298 | var targetMap = {};
|
299 | var targetFullPath = path.join(rootDir, targetDir);
|
300 | if (fs.existsSync(targetFullPath)) {
|
301 | targetMap = mapDirectory(rootDir, targetDir, include, exclude);
|
302 | }
|
303 |
|
304 | var pathMap = mergePathMaps(sourceMaps, targetMap, targetDir);
|
305 |
|
306 | var updated = false;
|
307 |
|
308 | // Iterate in sorted order to ensure directories are created before files under them.
|
309 | Object.keys(pathMap).sort().forEach(function (subPath) {
|
310 | var entry = pathMap[subPath];
|
311 | updated = updatePathWithStats(
|
312 | entry.sourcePath,
|
313 | entry.sourceStats,
|
314 | entry.targetPath,
|
315 | entry.targetStats,
|
316 | options,
|
317 | log) || updated;
|
318 | });
|
319 |
|
320 | return updated;
|
321 | }
|
322 |
|
323 | /**
|
324 | * Creates a dictionary map of all files and directories under a path.
|
325 | */
|
326 | function mapDirectory (rootDir, subDir, include, exclude) {
|
327 | var dirMap = { '': { subDir: subDir, stats: fs.statSync(path.join(rootDir, subDir)) } };
|
328 | mapSubdirectory(rootDir, subDir, '', include, exclude, dirMap);
|
329 | return dirMap;
|
330 |
|
331 | function mapSubdirectory (rootDir, subDir, relativeDir, include, exclude, dirMap) {
|
332 | var itemMapped = false;
|
333 | var items = fs.readdirSync(path.join(rootDir, subDir, relativeDir));
|
334 |
|
335 | items.forEach(function (item) {
|
336 | var relativePath = path.join(relativeDir, item);
|
337 | if (!matchGlobArray(relativePath, exclude)) {
|
338 | // Stats obtained here (required at least to know where to recurse in directories)
|
339 | // are saved for later, where the modified times may also be used. This minimizes
|
340 | // the number of file I/O operations performed.
|
341 | var fullPath = path.join(rootDir, subDir, relativePath);
|
342 | var stats = fs.statSync(fullPath);
|
343 |
|
344 | if (stats.isDirectory()) {
|
345 | // Directories are included if either something under them is included or they
|
346 | // match an include glob.
|
347 | if (mapSubdirectory(rootDir, subDir, relativePath, include, exclude, dirMap) ||
|
348 | matchGlobArray(relativePath, include)) {
|
349 | dirMap[relativePath] = { subDir: subDir, stats: stats };
|
350 | itemMapped = true;
|
351 | }
|
352 | } else if (stats.isFile()) {
|
353 | // Files are included only if they match an include glob.
|
354 | if (matchGlobArray(relativePath, include)) {
|
355 | dirMap[relativePath] = { subDir: subDir, stats: stats };
|
356 | itemMapped = true;
|
357 | }
|
358 | }
|
359 | }
|
360 | });
|
361 | return itemMapped;
|
362 | }
|
363 |
|
364 | function matchGlobArray (path, globs) {
|
365 | return globs.some(function (elem) {
|
366 | return minimatch(path, elem, {dot: true});
|
367 | });
|
368 | }
|
369 | }
|
370 |
|
371 | /**
|
372 | * Merges together multiple source maps and a target map into a single mapping from
|
373 | * relative paths to objects with target and source paths and stats.
|
374 | */
|
375 | function mergePathMaps (sourceMaps, targetMap, targetDir) {
|
376 | // Merge multiple source maps together, along with target path info.
|
377 | // Entries in later source maps override those in earlier source maps.
|
378 | // Target stats will be filled in below for targets that exist.
|
379 | var pathMap = {};
|
380 | sourceMaps.forEach(function (sourceMap) {
|
381 | Object.keys(sourceMap).forEach(function (sourceSubPath) {
|
382 | var sourceEntry = sourceMap[sourceSubPath];
|
383 | pathMap[sourceSubPath] = {
|
384 | targetPath: path.join(targetDir, sourceSubPath),
|
385 | targetStats: null,
|
386 | sourcePath: path.join(sourceEntry.subDir, sourceSubPath),
|
387 | sourceStats: sourceEntry.stats
|
388 | };
|
389 | });
|
390 | });
|
391 |
|
392 | // Fill in target stats for targets that exist, and create entries
|
393 | // for targets that don't have any corresponding sources.
|
394 | Object.keys(targetMap).forEach(function (subPath) {
|
395 | var entry = pathMap[subPath];
|
396 | if (entry) {
|
397 | entry.targetStats = targetMap[subPath].stats;
|
398 | } else {
|
399 | pathMap[subPath] = {
|
400 | targetPath: path.join(targetDir, subPath),
|
401 | targetStats: targetMap[subPath].stats,
|
402 | sourcePath: null,
|
403 | sourceStats: null
|
404 | };
|
405 | }
|
406 | });
|
407 |
|
408 | return pathMap;
|
409 | }
|
410 |
|
411 | module.exports = {
|
412 | updatePath: updatePath,
|
413 | updatePaths: updatePaths,
|
414 | mergeAndUpdateDir: mergeAndUpdateDir
|
415 | };
|