1 |
|
2 |
|
3 |
|
4 |
|
5 |
|
6 |
|
7 |
|
8 |
|
9 |
|
10 |
|
11 |
|
12 |
|
13 |
|
14 |
|
15 | import * as path from 'path';
|
16 | import * as logging from 'plylog';
|
17 | import {Analyzer, FsUrlResolver, PackageRelativeUrl, ResolvedUrl, Severity, UrlLoader, Warning, WarningFilter, WarningPrinter} from 'polymer-analyzer';
|
18 | import {ProjectConfig} from 'polymer-project-config';
|
19 | import {PassThrough, Transform} from 'stream';
|
20 | import {src as vinylSrc} from 'vinyl-fs';
|
21 |
|
22 | import {LocalFsPath, pathFromUrl, urlFromPath} from './path-transformers';
|
23 | import {AsyncTransformStream, VinylReaderTransform} from './streams';
|
24 |
|
25 | import File = require('vinyl');
|
26 |
|
27 | const logger = logging.getLogger('cli.build.analyzer');
|
28 |
|
29 | export interface DocumentDeps {
|
30 | imports: PackageRelativeUrl[];
|
31 | scripts: PackageRelativeUrl[];
|
32 | styles: PackageRelativeUrl[];
|
33 | }
|
34 |
|
35 | export interface DepsIndex {
|
36 |
|
37 | depsToFragments: Map<PackageRelativeUrl, PackageRelativeUrl[]>;
|
38 |
|
39 |
|
40 | fragmentToDeps: Map<PackageRelativeUrl, PackageRelativeUrl[]>;
|
41 |
|
42 | fragmentToFullDeps: Map<PackageRelativeUrl, DocumentDeps>;
|
43 | }
|
44 |
|
45 |
|
46 |
|
47 |
|
48 |
|
49 |
|
50 | class ResolveTransform extends AsyncTransformStream<File, File> {
|
51 | private _buildAnalyzer: BuildAnalyzer;
|
52 |
|
53 | constructor(buildAnalyzer: BuildAnalyzer) {
|
54 | super({objectMode: true});
|
55 | this._buildAnalyzer = buildAnalyzer;
|
56 | }
|
57 |
|
58 | protected async *
|
59 | _transformIter(files: AsyncIterable<File>): AsyncIterable<File> {
|
60 | for await (const file of files) {
|
61 | this._buildAnalyzer.resolveFile(file);
|
62 | yield file;
|
63 | }
|
64 | }
|
65 | }
|
66 |
|
67 |
|
68 |
|
69 |
|
70 |
|
71 |
|
72 |
|
73 |
|
74 |
|
75 |
|
76 |
|
77 |
|
78 |
|
79 | class AnalyzeTransform extends AsyncTransformStream<File, File> {
|
80 | private _buildAnalyzer: BuildAnalyzer;
|
81 |
|
82 | constructor(buildAnalyzer: BuildAnalyzer) {
|
83 |
|
84 |
|
85 |
|
86 |
|
87 | super({objectMode: true, highWaterMark: 10000});
|
88 | this._buildAnalyzer = buildAnalyzer;
|
89 | }
|
90 |
|
91 | protected async *
|
92 | _transformIter(files: AsyncIterable<File>): AsyncIterable<File> {
|
93 | for await (const file of files) {
|
94 | await this._buildAnalyzer.analyzeFile(file);
|
95 | yield file;
|
96 | }
|
97 | }
|
98 | }
|
99 |
|
100 |
|
101 | export class BuildAnalyzer {
|
102 | config: ProjectConfig;
|
103 | loader: StreamLoader;
|
104 | analyzer: Analyzer;
|
105 | started: boolean = false;
|
106 | sourceFilesLoaded: boolean = false;
|
107 |
|
108 | private _sourcesStream!: NodeJS.ReadableStream;
|
109 | private _sourcesProcessingStream!: NodeJS.ReadWriteStream;
|
110 | private _dependenciesStream!: Transform;
|
111 | private _dependenciesProcessingStream!: NodeJS.ReadWriteStream;
|
112 | private _warningsFilter: WarningFilter;
|
113 |
|
114 | files = new Map<PackageRelativeUrl, File>();
|
115 | warnings = new Set<Warning>();
|
116 | allFragmentsToAnalyze: Set<LocalFsPath>;
|
117 |
|
118 | analyzeDependencies: Promise<DepsIndex>;
|
119 | _dependencyAnalysis: DepsIndex = {
|
120 | depsToFragments: new Map(),
|
121 | fragmentToDeps: new Map(),
|
122 | fragmentToFullDeps: new Map()
|
123 | };
|
124 | _resolveDependencyAnalysis!: (index: DepsIndex) => void;
|
125 |
|
126 | constructor(
|
127 | config: ProjectConfig,
|
128 |
|
129 | private readonly streamToWarnTo: (NodeJS.WriteStream|
|
130 | null) = process.stdout) {
|
131 | this.config = config;
|
132 |
|
133 | this.loader = new StreamLoader(this);
|
134 | this.analyzer = new Analyzer({
|
135 | urlLoader: this.loader,
|
136 |
|
137 |
|
138 | urlResolver: new FsUrlResolver(config.root),
|
139 |
|
140 | moduleResolution: config.moduleResolution === 'none' ?
|
141 | undefined :
|
142 | config.moduleResolution,
|
143 | });
|
144 |
|
145 | this.allFragmentsToAnalyze =
|
146 | new Set(this.config.allFragments.map((f) => f as LocalFsPath));
|
147 | this.analyzeDependencies = new Promise((resolve, _reject) => {
|
148 | this._resolveDependencyAnalysis = resolve;
|
149 | });
|
150 |
|
151 | const lintOptions: Partial<typeof config.lint> = (this.config.lint || {});
|
152 |
|
153 | const warningCodesToIgnore = new Set(lintOptions.ignoreWarnings || []);
|
154 |
|
155 |
|
156 | warningCodesToIgnore.add('not-loadable');
|
157 | this._warningsFilter = new WarningFilter(
|
158 | {warningCodesToIgnore, minimumSeverity: Severity.WARNING});
|
159 | }
|
160 |
|
161 | |
162 |
|
163 |
|
164 |
|
165 |
|
166 |
|
167 | startAnalysis(): void {
|
168 | if (this.started) {
|
169 | return;
|
170 | }
|
171 | this.started = true;
|
172 |
|
173 |
|
174 | this._dependenciesStream = new PassThrough({objectMode: true});
|
175 | this._sourcesStream = vinylSrc(this.config.sources, {
|
176 | cwdbase: true,
|
177 | nodir: true,
|
178 | });
|
179 |
|
180 |
|
181 |
|
182 |
|
183 | this._sourcesProcessingStream =
|
184 | this._sourcesStream
|
185 | .on('error',
|
186 | (err: Error) =>
|
187 | this._sourcesProcessingStream.emit('error', err))
|
188 | .pipe(new ResolveTransform(this))
|
189 | .on('error',
|
190 | (err: Error) =>
|
191 | this._sourcesProcessingStream.emit('error', err))
|
192 | .on('end', this.onSourcesStreamComplete.bind(this))
|
193 | .pipe(new AnalyzeTransform(this));
|
194 |
|
195 |
|
196 |
|
197 |
|
198 | this._dependenciesProcessingStream =
|
199 | this._dependenciesStream
|
200 | .on('error',
|
201 | (err: Error) =>
|
202 | this._dependenciesProcessingStream.emit('error', err))
|
203 | .pipe(new VinylReaderTransform())
|
204 | .on('error',
|
205 | (err: Error) =>
|
206 | this._dependenciesProcessingStream.emit('error', err))
|
207 | .pipe(new ResolveTransform(this));
|
208 | }
|
209 |
|
210 | |
211 |
|
212 |
|
213 |
|
214 | dependencies(): NodeJS.ReadableStream {
|
215 | this.startAnalysis();
|
216 | return this._dependenciesProcessingStream;
|
217 | }
|
218 |
|
219 | |
220 |
|
221 |
|
222 |
|
223 | sources(): NodeJS.ReadableStream {
|
224 | this.startAnalysis();
|
225 | return this._sourcesProcessingStream;
|
226 | }
|
227 |
|
228 | |
229 |
|
230 |
|
231 | resolveFile(file: File) {
|
232 | const filePath = file.path as LocalFsPath;
|
233 | this.addFile(file);
|
234 |
|
235 | if (this.loader.hasDeferredFile(filePath)) {
|
236 | this.loader.resolveDeferredFile(filePath, file);
|
237 | }
|
238 | }
|
239 |
|
240 | |
241 |
|
242 |
|
243 |
|
244 |
|
245 | async analyzeFile(file: File): Promise<void> {
|
246 | const filePath = file.path as LocalFsPath;
|
247 |
|
248 |
|
249 | if (this.config.isFragment(filePath)) {
|
250 | const deps = await this._getDependencies(this.analyzer.resolveUrl(
|
251 | urlFromPath(this.config.root as LocalFsPath, filePath))!);
|
252 | this._addDependencies(filePath, deps);
|
253 | this.allFragmentsToAnalyze.delete(filePath);
|
254 |
|
255 | if (this.allFragmentsToAnalyze.size === 0) {
|
256 | this._done();
|
257 | }
|
258 | }
|
259 | }
|
260 |
|
261 | |
262 |
|
263 |
|
264 | private onSourcesStreamComplete() {
|
265 |
|
266 |
|
267 | for (const filePath of this.loader.deferredFiles.keys()) {
|
268 | if (this.config.isSource(filePath)) {
|
269 | const err = new Error(`Not found: ${filePath}`);
|
270 | this.loader.rejectDeferredFile(filePath, err);
|
271 | }
|
272 | }
|
273 |
|
274 |
|
275 | this.sourceFilesLoaded = true;
|
276 | }
|
277 |
|
278 | |
279 |
|
280 |
|
281 |
|
282 | private emitAnalysisError(err: Error) {
|
283 | this._sourcesProcessingStream.emit('error', err);
|
284 | this._dependenciesProcessingStream.emit('error', err);
|
285 | }
|
286 |
|
287 | |
288 |
|
289 |
|
290 |
|
291 |
|
292 | private _done() {
|
293 | this.printWarnings();
|
294 | const allWarningCount = this.countWarningsByType();
|
295 | const errorWarningCount = allWarningCount.get(Severity.ERROR)!;
|
296 |
|
297 |
|
298 | if (errorWarningCount > 0) {
|
299 | this.emitAnalysisError(
|
300 | new Error(`${errorWarningCount} error(s) occurred during build.`));
|
301 | return;
|
302 | }
|
303 |
|
304 |
|
305 |
|
306 | for (const filePath of this.loader.deferredFiles.keys()) {
|
307 | const err = new Error(`Not found: ${filePath}`);
|
308 | this.loader.rejectDeferredFile(filePath, err);
|
309 | return;
|
310 | }
|
311 |
|
312 |
|
313 | this._dependenciesStream.end();
|
314 | this._resolveDependencyAnalysis(this._dependencyAnalysis);
|
315 | }
|
316 |
|
317 | getFile(filepath: LocalFsPath): File|undefined {
|
318 | const url = urlFromPath(this.config.root as LocalFsPath, filepath);
|
319 | return this.getFileByUrl(url as PackageRelativeUrl);
|
320 | }
|
321 |
|
322 | getFileByUrl(url: PackageRelativeUrl): File|undefined {
|
323 |
|
324 | if (url.startsWith('/')) {
|
325 | url = url.substring(1) as PackageRelativeUrl;
|
326 | }
|
327 | return this.files.get(url);
|
328 | }
|
329 |
|
330 | |
331 |
|
332 |
|
333 |
|
334 |
|
335 |
|
336 | addFile(file: File): void {
|
337 | logger.debug(`addFile: ${file.path}`);
|
338 |
|
339 |
|
340 | const filepath = path.normalize(file.path) as LocalFsPath;
|
341 |
|
342 | this.files.set(
|
343 | urlFromPath(this.config.root as LocalFsPath, filepath), file);
|
344 | }
|
345 |
|
346 | printWarnings(): void {
|
347 | if (this.streamToWarnTo === null) {
|
348 | return;
|
349 | }
|
350 | const warningPrinter = new WarningPrinter(this.streamToWarnTo);
|
351 | warningPrinter.printWarnings(this.warnings);
|
352 | }
|
353 |
|
354 | private countWarningsByType(): Map<Severity, number> {
|
355 | const errorCountMap = new Map<Severity, number>();
|
356 | errorCountMap.set(Severity.INFO, 0);
|
357 | errorCountMap.set(Severity.WARNING, 0);
|
358 | errorCountMap.set(Severity.ERROR, 0);
|
359 | for (const warning of this.warnings) {
|
360 | errorCountMap.set(
|
361 | warning.severity, errorCountMap.get(warning.severity)! + 1);
|
362 | }
|
363 | return errorCountMap;
|
364 | }
|
365 |
|
366 |
|
367 | |
368 |
|
369 |
|
370 | async _getDependencies(url: ResolvedUrl): Promise<DocumentDeps> {
|
371 | const analysis = await this.analyzer.analyze([url]);
|
372 | const result = analysis.getDocument(url);
|
373 |
|
374 | if (result.successful === false) {
|
375 | const message = result.error && result.error.message || 'unknown';
|
376 | throw new Error(`Unable to get document ${url}: ${message}`);
|
377 | }
|
378 |
|
379 | const doc = result.value;
|
380 | doc.getWarnings({imported: true})
|
381 | .filter((w) => !this._warningsFilter.shouldIgnore(w))
|
382 | .forEach((w) => this.warnings.add(w));
|
383 |
|
384 | const scripts = new Set<PackageRelativeUrl>();
|
385 | const styles = new Set<PackageRelativeUrl>();
|
386 | const imports = new Set<PackageRelativeUrl>();
|
387 |
|
388 | const importFeatures = doc.getFeatures(
|
389 | {kind: 'import', externalPackages: true, imported: true});
|
390 | for (const importFeature of importFeatures) {
|
391 | const importUrl = importFeature.url;
|
392 | if (!this.analyzer.canLoad(importUrl)) {
|
393 | logger.debug(`ignoring external dependency: ${importUrl}`);
|
394 | } else if (importFeature.type === 'html-script') {
|
395 | scripts.add(this.analyzer.urlResolver.relative(importUrl));
|
396 | } else if (importFeature.type === 'html-style') {
|
397 | styles.add(this.analyzer.urlResolver.relative(importUrl));
|
398 | } else if (importFeature.type === 'html-import') {
|
399 | imports.add(this.analyzer.urlResolver.relative(importUrl));
|
400 | } else {
|
401 | logger.debug(
|
402 | `unexpected import type encountered: ${importFeature.type}`);
|
403 | }
|
404 | }
|
405 |
|
406 | const deps = {
|
407 | scripts: [...scripts],
|
408 | styles: [...styles],
|
409 | imports: [...imports],
|
410 | };
|
411 | logger.debug(`dependencies analyzed for: ${url}`, deps);
|
412 | return deps;
|
413 | }
|
414 |
|
415 | _addDependencies(filePath: LocalFsPath, deps: DocumentDeps) {
|
416 |
|
417 | if (!this.allFragmentsToAnalyze.has(filePath)) {
|
418 | throw new Error(`Dependency analysis incorrectly called for ${filePath}`);
|
419 | }
|
420 |
|
421 | const relativeUrl = urlFromPath(this.config.root as LocalFsPath, filePath);
|
422 |
|
423 |
|
424 | this._dependencyAnalysis.fragmentToFullDeps.set(relativeUrl, deps);
|
425 | this._dependencyAnalysis.fragmentToDeps.set(relativeUrl, deps.imports);
|
426 | deps.imports.forEach((url) => {
|
427 | const entrypointList = this._dependencyAnalysis.depsToFragments.get(url);
|
428 | if (entrypointList) {
|
429 | entrypointList.push(relativeUrl);
|
430 | } else {
|
431 | this._dependencyAnalysis.depsToFragments.set(url, [relativeUrl]);
|
432 | }
|
433 | });
|
434 | }
|
435 |
|
436 | |
437 |
|
438 |
|
439 |
|
440 |
|
441 | sourcePathAnalyzed(filePath: LocalFsPath): void {
|
442 |
|
443 |
|
444 |
|
445 | if (this.sourceFilesLoaded) {
|
446 | throw new Error(`Not found: "${filePath}"`);
|
447 | }
|
448 |
|
449 |
|
450 |
|
451 | logger.debug('dependency is a source file, ignoring...', {dep: filePath});
|
452 | }
|
453 |
|
454 | |
455 |
|
456 |
|
457 |
|
458 | dependencyPathAnalyzed(filePath: LocalFsPath): void {
|
459 | if (this.getFile(filePath)) {
|
460 | logger.debug(
|
461 | 'dependency has already been pushed, ignoring...', {dep: filePath});
|
462 | return;
|
463 | }
|
464 |
|
465 | logger.debug(
|
466 | 'new dependency analyzed, pushing into dependency stream...', filePath);
|
467 | this._dependenciesStream.push(filePath);
|
468 | }
|
469 | }
|
470 |
|
471 | export type ResolveFileCallback = (a: string) => void;
|
472 | export type RejectFileCallback = (err: Error) => void;
|
473 | export type DeferredFileCallbacks = {
|
474 | resolve: ResolveFileCallback; reject: RejectFileCallback;
|
475 | };
|
476 |
|
477 | export class StreamLoader implements UrlLoader {
|
478 | config: ProjectConfig;
|
479 | private _buildAnalyzer: BuildAnalyzer;
|
480 |
|
481 |
|
482 |
|
483 |
|
484 | deferredFiles = new Map<LocalFsPath, DeferredFileCallbacks>();
|
485 |
|
486 | constructor(buildAnalyzer: BuildAnalyzer) {
|
487 | this._buildAnalyzer = buildAnalyzer;
|
488 | this.config = this._buildAnalyzer.config;
|
489 | }
|
490 |
|
491 | hasDeferredFile(filePath: LocalFsPath): boolean {
|
492 | return this.deferredFiles.has(filePath);
|
493 | }
|
494 |
|
495 | hasDeferredFiles(): boolean {
|
496 | return this.deferredFiles.size > 0;
|
497 | }
|
498 |
|
499 | resolveDeferredFile(filePath: LocalFsPath, file: File): void {
|
500 | const deferredCallbacks = this.deferredFiles.get(filePath);
|
501 | if (deferredCallbacks == null) {
|
502 | throw new Error(
|
503 | `Internal error: could not get deferredCallbacks for ${filePath}`);
|
504 | }
|
505 | deferredCallbacks.resolve(file.contents!.toString());
|
506 | this.deferredFiles.delete(filePath);
|
507 | }
|
508 |
|
509 | rejectDeferredFile(filePath: LocalFsPath, err: Error): void {
|
510 | const deferredCallbacks = this.deferredFiles.get(filePath);
|
511 | if (deferredCallbacks == null) {
|
512 | throw new Error(
|
513 | `Internal error: could not get deferredCallbacks for ${filePath}`);
|
514 | }
|
515 | deferredCallbacks.reject(err);
|
516 | this.deferredFiles.delete(filePath);
|
517 | }
|
518 |
|
519 |
|
520 | canLoad(url: ResolvedUrl): boolean {
|
521 | return url.startsWith('file:///');
|
522 | }
|
523 |
|
524 | async load(url: ResolvedUrl): Promise<string> {
|
525 | logger.debug(`loading: ${url}`);
|
526 | if (!this.canLoad(url)) {
|
527 | throw new Error('Unable to load ${url}.');
|
528 | }
|
529 |
|
530 | const urlPath = this._buildAnalyzer.analyzer.urlResolver.relative(url);
|
531 | const filePath = pathFromUrl(this.config.root as LocalFsPath, urlPath);
|
532 | const file = this._buildAnalyzer.getFile(filePath);
|
533 |
|
534 | if (file) {
|
535 | return file.contents!.toString();
|
536 | }
|
537 |
|
538 | return new Promise(
|
539 | (resolve: ResolveFileCallback, reject: RejectFileCallback) => {
|
540 | this.deferredFiles.set(filePath, {resolve, reject});
|
541 | try {
|
542 | if (this.config.isSource(filePath)) {
|
543 | this._buildAnalyzer.sourcePathAnalyzed(filePath);
|
544 | } else {
|
545 | this._buildAnalyzer.dependencyPathAnalyzed(filePath);
|
546 | }
|
547 | } catch (err) {
|
548 | this.rejectDeferredFile(filePath, err);
|
549 | }
|
550 | });
|
551 | }
|
552 | }
|