From 6b7a35cb9f9c772c676a5cffc40ab5224a540116 Mon Sep 17 00:00:00 2001 From: Hans Larsen Date: Sat, 26 Mar 2016 15:29:13 -0700 Subject: [PATCH] Move to Broccoli 1.0 --- ember-cli-build.js | 38 +- package.json | 3 +- src/tsconfig.json | 6 +- tools/broccoli/angular_builder.d.ts | 5 - tools/broccoli/angular_builder.ts | 167 --------- tools/broccoli/broccoli-check-imports.ts | 108 ------ tools/broccoli/broccoli-dartfmt.ts | 132 +++---- tools/broccoli/broccoli-dest-copy.ts | 36 -- tools/broccoli/broccoli-filter.d.ts | 14 - tools/broccoli/broccoli-flatten.spec.ts | 76 ---- tools/broccoli/broccoli-flatten.ts | 51 --- tools/broccoli/broccoli-lodash.ts | 49 --- tools/broccoli/broccoli-merge-trees.spec.ts | 91 ----- tools/broccoli/broccoli-merge-trees.ts | 137 ------- tools/broccoli/broccoli-replace.ts | 59 --- tools/broccoli/broccoli-tree-stabilizer.ts | 45 --- tools/broccoli/broccoli-ts2dart.ts | 106 ++++-- tools/broccoli/broccoli-typescript.ts | 370 ------------------- tools/broccoli/broccoli-writer.d.ts | 8 - tools/broccoli/broccoli.d.ts | 2 +- tools/broccoli/diffing-broccoli-plugin.ts | 190 ---------- tools/broccoli/multi_copy.ts | 49 --- tools/broccoli/tree-differ.spec.ts | 378 -------------------- tools/broccoli/tree-differ.ts | 176 --------- 24 files changed, 171 insertions(+), 2125 deletions(-) delete mode 100644 tools/broccoli/angular_builder.d.ts delete mode 100644 tools/broccoli/angular_builder.ts delete mode 100644 tools/broccoli/broccoli-check-imports.ts delete mode 100644 tools/broccoli/broccoli-dest-copy.ts delete mode 100644 tools/broccoli/broccoli-filter.d.ts delete mode 100644 tools/broccoli/broccoli-flatten.spec.ts delete mode 100644 tools/broccoli/broccoli-flatten.ts delete mode 100644 tools/broccoli/broccoli-lodash.ts delete mode 100644 tools/broccoli/broccoli-merge-trees.spec.ts delete mode 100644 tools/broccoli/broccoli-merge-trees.ts delete mode 100644 tools/broccoli/broccoli-replace.ts delete mode 100644 tools/broccoli/broccoli-tree-stabilizer.ts delete mode 100644 tools/broccoli/broccoli-typescript.ts delete mode 100644 tools/broccoli/broccoli-writer.d.ts delete mode 100644 tools/broccoli/diffing-broccoli-plugin.ts delete mode 100644 tools/broccoli/multi_copy.ts delete mode 100644 tools/broccoli/tree-differ.spec.ts delete mode 100644 tools/broccoli/tree-differ.ts diff --git a/ember-cli-build.js b/ember-cli-build.js index fe27aa6e98b..e5a5b438e96 100644 --- a/ember-cli-build.js +++ b/ember-cli-build.js @@ -1,3 +1,4 @@ +'use strict'; var fs = require('fs'); var path = require('path'); @@ -12,28 +13,38 @@ var BroccoliSass = require('broccoli-sass'); var broccoliAutoprefixer = require('broccoli-autoprefixer'); const BroccoliTs2Dart = require('./tools/broccoli/broccoli-ts2dart').default; -const BroccoliDestCopy = require('./tools/broccoli/broccoli-dest-copy').default; const BroccoliDartFmt = require('./tools/broccoli/broccoli-dartfmt').default; const BroccoliFunnel = require('broccoli-funnel'); -const BroccoliSource = require('broccoli-source'); var autoprefixerOptions = require('./build/autoprefixer-options'); +/** + * Almost the same as using a plain string for a tree; but unlike a plain + * string, Broccoli won't watch this. Used only internally. + */ +class UnwatchedTree { + constructor(dir) { this.dir = dir; } + read() { return this.dir; } + cleanup() { /* DoNothing */ } +} + + module.exports = function(defaults) { var demoAppCssTree = new BroccoliSass(['src/demo-app'], './demo-app.scss', 'demo-app/demo-app.css'); var demoCssTree = getCssTree('demo-app'); var componentCssTree = getCssTree('components'); var mainCssTree = new BroccoliSass(['src', 'src/core/style'], './main.scss', 'main.css'); - var angularAppTree = new Angular2App(defaults); + var angularAppTree = new Angular2App(defaults, { + vendorNpmFiles: [] + }); var dartAppTree = getDartTree('src/'); - return mergeTrees([ angularAppTree.toTree(), componentCssTree, mainCssTree, demoAppCssTree, - demoCssTree, + demoCssTree ].concat(dartAppTree || [])); }; @@ -50,15 +61,16 @@ function getDartTree(root) { return null; } - const ts2dart = new BroccoliTs2Dart(root, { + const ts2dart = new BroccoliTs2Dart([root], { generateLibraryName: true, generateSourceMap: false, translateBuiltins: true, - typingsRoot: '../../typings/browser/ambient/', - additionalFiles: [path.join(process.cwd(), root, 'typings.d.ts')], + basePath: path.join(__dirname, root), + typingsRoot: '../typings/browser/ambient/', + additionalFiles: [path.join(process.cwd(), root, 'typings.d.ts')] }); - const formatter = new BroccoliDartFmt(ts2dart, { dartSDK }); + const formatter = new BroccoliDartFmt([ts2dart], { dartSDK }); const dartSources = new BroccoliFunnel(root, { include: ['**/*.dart'], @@ -70,16 +82,16 @@ function getDartTree(root) { formatter ]); - const pubSpecTree = new BroccoliFunnel(new BroccoliSource.UnwatchedDir('.'), { + const pubSpecTree = new BroccoliFunnel(new UnwatchedTree('.'), { files: ['pubspec.yaml'], - destDir: 'dart', + destDir: 'dart' }); // Publishes the Dart files and pubspec inside a return mergeTrees([ - dartSources, pubSpecTree, - new BroccoliDestCopy(formatter, 'dart/lib'), + new BroccoliFunnel(formatter, { destDir: 'dart/lib' }), + allDartFiles ]); } diff --git a/package.json b/package.json index d3583e39952..c4ded827105 100644 --- a/package.json +++ b/package.json @@ -36,9 +36,10 @@ }, "devDependencies": { "add-stream": "^1.0.0", - "angular-cli": "^0.0.24", + "angular-cli": "^0.0.28", "angular-cli-github-pages": "^0.2.0", "broccoli-autoprefixer": "^4.1.0", + "broccoli-caching-writer": "^2.2.1", "broccoli-funnel": "^1.0.1", "broccoli-merge-trees": "^1.1.1", "broccoli-sass": "^0.7.0", diff --git a/src/tsconfig.json b/src/tsconfig.json index 9633bb74b26..3821d9fd92c 100644 --- a/src/tsconfig.json +++ b/src/tsconfig.json @@ -13,5 +13,9 @@ "sourceMap": true, "sourceRoot": "/", "target": "es5" - } + }, + "files": [ + "main.ts", + "typings.d.ts" + ] } diff --git a/tools/broccoli/angular_builder.d.ts b/tools/broccoli/angular_builder.d.ts deleted file mode 100644 index c7476f3c498..00000000000 --- a/tools/broccoli/angular_builder.d.ts +++ /dev/null @@ -1,5 +0,0 @@ -interface AngularBuilderOptions { - outputPath: string; - dartSDK?: any; - logs?: any; -} diff --git a/tools/broccoli/angular_builder.ts b/tools/broccoli/angular_builder.ts deleted file mode 100644 index 032278a9d04..00000000000 --- a/tools/broccoli/angular_builder.ts +++ /dev/null @@ -1,167 +0,0 @@ -var broccoli = require('broccoli'); -var fs = require('fs'); -var makeBrowserTree = require('./trees/browser_tree'); -var makeNodeTree = require('./trees/node_tree'); -var makeDartTree = require('./trees/dart_tree'); -var path = require('path'); -var printSlowTrees = require('broccoli-slow-trees'); -var Q = require('q'); - -type ProjectMap = { - [key: string]: boolean -}; - -type Options = { - projects: ProjectMap; -noTypeChecks: boolean; -generateEs6: boolean; -useBundles: boolean; -} -; - -/** - * BroccoliBuilder facade for all of our build pipelines. - */ -export class AngularBuilder { - private nodeBuilder: BroccoliBuilder; - private browserDevBuilder: BroccoliBuilder; - private browserProdBuilder: BroccoliBuilder; - private dartBuilder: BroccoliBuilder; - private outputPath: string; - private firstResult: BuildResult; - - constructor(public options: AngularBuilderOptions) { this.outputPath = options.outputPath; } - - - public rebuildBrowserDevTree(opts: Options): Promise { - this.browserDevBuilder = this.browserDevBuilder || this.makeBrowserDevBuilder(opts); - return this.rebuild(this.browserDevBuilder, 'js.dev'); - } - - - public rebuildBrowserProdTree(opts: Options): Promise { - this.browserProdBuilder = this.browserProdBuilder || this.makeBrowserProdBuilder(opts); - return this.rebuild(this.browserProdBuilder, 'js.prod'); - } - - - public rebuildNodeTree(opts: Options): Promise { - this.nodeBuilder = this.nodeBuilder || this.makeNodeBuilder(opts.projects); - return this.rebuild(this.nodeBuilder, 'js.cjs'); - } - - - public rebuildDartTree(projects: ProjectMap): Promise { - this.dartBuilder = this.dartBuilder || this.makeDartBuilder(projects); - return this.rebuild(this.dartBuilder, 'dart'); - } - - - cleanup(): Promise { - return Q.all([ - this.nodeBuilder && this.nodeBuilder.cleanup(), - this.browserDevBuilder && this.browserDevBuilder.cleanup(), - this.browserProdBuilder && this.browserProdBuilder.cleanup() - ]); - } - - - private makeBrowserDevBuilder(opts: Options): BroccoliBuilder { - let tree = makeBrowserTree( - { - name: 'dev', - typeAssertions: true, - sourceMaps: true, - projects: opts.projects, - noTypeChecks: opts.noTypeChecks, - generateEs6: opts.generateEs6, - useBundles: opts.useBundles - }, - path.join(this.outputPath, 'js', 'dev')); - return new broccoli.Builder(tree); - } - - - private makeBrowserProdBuilder(opts: Options): BroccoliBuilder { - let tree = makeBrowserTree( - { - name: 'prod', - typeAssertions: false, - sourceMaps: false, - projects: opts.projects, - noTypeChecks: opts.noTypeChecks, - generateEs6: opts.generateEs6, - useBundles: opts.useBundles - }, - path.join(this.outputPath, 'js', 'prod')); - return new broccoli.Builder(tree); - } - - - private makeNodeBuilder(projects: ProjectMap): BroccoliBuilder { - let tree = makeNodeTree(projects, path.join(this.outputPath, 'js', 'cjs')); - return new broccoli.Builder(tree); - } - - - private makeDartBuilder(projects: ProjectMap): BroccoliBuilder { - let options = { - outputPath: path.join(this.outputPath, 'dart'), - dartSDK: this.options.dartSDK, - logs: this.options.logs, - projects: projects - }; - let tree = makeDartTree(options); - return new broccoli.Builder(tree); - } - - - private rebuild(builder, name) { - return builder.build().then( - (result) => { - if (!this.firstResult) { - this.firstResult = result; - } - - printSlowTrees(result.graph); - writeBuildLog(result, name); - }, - (error) => { - // the build tree is the same during rebuilds, only leaf properties of the nodes change - // so let's traverse it and get updated values for input/cache/output paths - if (this.firstResult) { - writeBuildLog(this.firstResult, name); - } - throw error; - }); - } -} - - -function writeBuildLog(result: BuildResult, name: string) { - let logPath = `tmp/build.${name}.log`; - let prevLogPath = logPath + '.previous'; - let formattedLogContent = JSON.stringify(broccoliNodeToBuildNode(result.graph), null, 2); - - if (fs.existsSync(prevLogPath)) fs.unlinkSync(prevLogPath); - if (fs.existsSync(logPath)) fs.renameSync(logPath, prevLogPath); - fs.writeFileSync(logPath, formattedLogContent, {encoding: 'utf-8'}); -} - - -function broccoliNodeToBuildNode(broccoliNode) { - let tree = broccoliNode.tree.newStyleTree || broccoliNode.tree; - - return new BuildNode(tree.description || tree.constructor.name, - tree.inputPath ? [tree.inputPath] : tree.inputPaths, tree.cachePath, - tree.outputPath, broccoliNode.selfTime / (1000 * 1000 * 1000), - broccoliNode.totalTime / (1000 * 1000 * 1000), - broccoliNode.subtrees.map(broccoliNodeToBuildNode)); -} - - -class BuildNode { - constructor(public pluginName: string, public inputPaths: string[], public cachePath: string, - public outputPath: string, public selfTime: number, public totalTime: number, - public inputNodes: BroccoliNode[]) {} -} diff --git a/tools/broccoli/broccoli-check-imports.ts b/tools/broccoli/broccoli-check-imports.ts deleted file mode 100644 index f1d53c0076f..00000000000 --- a/tools/broccoli/broccoli-check-imports.ts +++ /dev/null @@ -1,108 +0,0 @@ -import fs = require('fs'); -import fse = require('fs-extra'); -import path = require('path'); -import {wrapDiffingPlugin, DiffingBroccoliPlugin, DiffResult} from './diffing-broccoli-plugin'; - -/** - * Checks that modules do not import files that are not supposed to import. - * - * This guarantees that platform-independent modules remain platoform-independent. - */ -class CheckImports implements DiffingBroccoliPlugin { - static IMPORT_DECL_REGEXP = new RegExp(`^import[^;]+;`, "mg"); - static IMPORT_PATH_REGEXP = new RegExp(`['"]([^'"]+)+['"]`, "m"); - - static ALLOWED_IMPORTS = { - "angular2/src/core": ["angular2/src/facade"], - "angular2/src/facade": ["rxjs"], - "angular2/src/common": ["angular2/core", "angular2/src/facade"], - "angular2/src/http": ["angular2/core", "angular2/src/facade", "rxjs"], - "angular2/src/upgrade": ["angular2/core", "angular2/src/facade", "angular2/platform/browser"] - //"angular2/src/render": [ - // "angular2/animate", - // "angular2/core", - // "angular2/src/facade", - //] - }; - - private initRun = true; - - constructor(private inputPath, private cachePath, private options) {} - - rebuild(treeDiff: DiffResult) { - const errors = this.checkAllPaths(treeDiff); - if (errors.length > 0) { - throw new Error( - `The following imports are not allowed because they break barrel boundaries:\n${errors.join("\n")}`); - } - this.symlinkInputAndCacheIfNeeded(); - return treeDiff; - } - - private checkAllPaths(treeDiff: DiffResult) { - const changesFiles = treeDiff.addedPaths.concat(treeDiff.changedPaths); - return flatMap(changesFiles, _ => this.checkFilePath(_)); - } - - private symlinkInputAndCacheIfNeeded() { - if (this.initRun) { - fs.rmdirSync(this.cachePath); - fs.symlinkSync(this.inputPath, this.cachePath); - } - this.initRun = false; - } - - private checkFilePath(filePath: string) { - const sourceFilePath = path.join(this.inputPath, filePath); - if (endsWith(sourceFilePath, ".ts") && fs.existsSync(sourceFilePath)) { - const content = fs.readFileSync(sourceFilePath, "UTF-8"); - const imports = content.match(CheckImports.IMPORT_DECL_REGEXP); - if (imports) { - return imports.filter(i => !this.isAllowedImport(filePath, i)) - .map(i => this.formatError(filePath, i)); - } else { - return []; - } - } - return []; - } - - private isAllowedImport(sourceFile: string, importDecl: string): boolean { - const res = CheckImports.IMPORT_PATH_REGEXP.exec(importDecl); - if (!res || res.length < 2) return true; // non-es6 import - const importPath = res[1]; - - if (startsWith(importPath, "./") || startsWith(importPath, "../")) return true; - - const c = CheckImports.ALLOWED_IMPORTS; - for (var prop in c) { - if (c.hasOwnProperty(prop) && startsWith(sourceFile, prop)) { - const allowedPaths = c[prop]; - return startsWith(importPath, prop) || - allowedPaths.filter(p => startsWith(importPath, p)).length > 0; - } - } - - return true; - } - - private formatError(filePath: string, importPath: string): string { - const i = importPath.replace(new RegExp(`\n`, 'g'), "\\n"); - return `${filePath}: ${i}`; - } -} - - -function startsWith(str: string, substring: string): boolean { - return str.substring(0, substring.length) === substring; -} - -function endsWith(str: string, substring: string): boolean { - return str.indexOf(substring, str.length - substring.length) !== -1; -} - -function flatMap(arr: T[], fn: (t: T) => U[]): U[] { - return [].concat(...arr.map(fn)); -} - -export default wrapDiffingPlugin(CheckImports); diff --git a/tools/broccoli/broccoli-dartfmt.ts b/tools/broccoli/broccoli-dartfmt.ts index c418d9baa5b..b6f7a8631de 100644 --- a/tools/broccoli/broccoli-dartfmt.ts +++ b/tools/broccoli/broccoli-dartfmt.ts @@ -1,92 +1,92 @@ /// /// +/// + +interface BroccoliCachingWriterPlugin extends BroccoliTree { + listFiles(): string[]; + listEntries(): any[]; +} + +interface Constructor { + new (...args: any[]): BroccoliCachingWriterPlugin; +} + +interface CacheEntry { + version: number; +} + + +import fs = require('fs'); import fse = require('fs-extra'); import path = require('path'); -import {wrapDiffingPlugin, DiffingBroccoliPlugin, DiffResult} from './diffing-broccoli-plugin'; -var spawn = require('child_process').spawn; -var exec = require('child_process').exec; -function processToPromise(process) { - return new Promise(function(resolve, reject) { - process.on('close', function(code) { - if (code) { - reject(code); +const Plugin: Constructor = require('broccoli-caching-writer'); +const exec = require('child_process').exec; + + +const promiseExec = (cmd: string): Promise => { + return new Promise((resolve, reject) => { + exec(cmd, (err, stdout, stderr) => { + if (err) { + console.error(shortenFormatterOutput(stderr)); + reject('Formatting failed.'); } else { resolve(); } }); }); -} +}; -class DartFormatter implements DiffingBroccoliPlugin { + +export default class DartFormatter extends Plugin { private DARTFMT: string; - private verbose: boolean; - private firstBuild: boolean = true; + private cache: {[filePath: string]: CacheEntry} = Object.create(null); - constructor(public inputPath: string, public cachePath: string, options) { + constructor(inputPaths: any[], options: any) { + super(inputPaths, options); if (!options.dartSDK) throw new Error("Missing Dart SDK"); this.DARTFMT = options.dartSDK.DARTFMT; } - rebuild(treeDiff: DiffResult): Promise { - let args = ['-w']; - let argsLength = 2; - let argPackages = []; - let firstBuild = this.firstBuild; - treeDiff.addedPaths.concat(treeDiff.changedPaths) - .forEach((changedFile) => { - let sourcePath = path.join(this.inputPath, changedFile); - let destPath = path.join(this.cachePath, changedFile); - if (!firstBuild && /\.dart$/.test(changedFile)) { - if ((argsLength + destPath.length + 2) >= 0x2000) { - // Win32 command line arguments length - argPackages.push(args); - args = ['-w']; - argsLength = 2; - } - args.push(destPath); - argsLength += destPath.length + 2; - } - fse.copySync(sourcePath, destPath); - }); - treeDiff.removedPaths.forEach((removedFile) => { - let destPath = path.join(this.cachePath, removedFile); - fse.removeSync(destPath); - }); + build(): Promise { + const dartEntries = this.listEntries().filter(entry => entry.relativePath.match(/\.dart$/)); - if (!firstBuild && args.length > 1) { - argPackages.push(args); - } + const changedEntries: any[] = dartEntries.map(entry => { + const filePath = entry.relativePath; + const inputPath = path.join(this.inputPaths[0], filePath); + const cachePath = path.join(this.cachePath, filePath); - let execute = (args) => { - if (args.length < 2) return Promise.resolve(); - return new Promise((resolve, reject) => { - exec(this.DARTFMT + ' ' + args.join(' '), (err, stdout, stderr) => { - if (this.verbose) { - console.log(stdout); - } - if (err) { - console.error(shortenFormatterOutput(stderr)); - reject('Formatting failed.'); - } else { + if (!this.cache[inputPath]) { + this.cache[inputPath] = { version: entry.mtime }; + } else if (this.cache[inputPath].version >= entry.mtime) { + // Ignore this, cache is as young as input. + return; + } + + fse.copySync(inputPath, cachePath); + }).filter(x => !!x); + + return Promise.all( + changedEntries.map(entry => { + const filePath = entry.relativePath; + const cachePath = path.join(this.cachePath, filePath); + return promiseExec(this.DARTFMT + ' ' + cachePath); + })) + .then(() => Promise.all( + Object.keys(this.cache).map(filePath => { + const cachePath = filePath.replace(this.inputPaths[0], this.cachePath); + const outputPath = filePath.replace(this.inputPaths[0], this.outputPath); + + return new Promise(resolve => { + fse.mkdirsSync(path.dirname(outputPath)); + fs.linkSync(cachePath, outputPath); resolve(); - } - }); - }); - }; - - if (firstBuild) { - // On firstBuild, format the entire cachePath - this.firstBuild = false; - return execute(['-w', this.cachePath]); - } - - return Promise.all(argPackages.map(execute)); + }); + }) + )); } } -export default wrapDiffingPlugin(DartFormatter); - var ARROW_LINE = /^(\s+)\^+/; var BEFORE_CHARS = 15; var stripAnsi = require('strip-ansi'); diff --git a/tools/broccoli/broccoli-dest-copy.ts b/tools/broccoli/broccoli-dest-copy.ts deleted file mode 100644 index d8e0426b85a..00000000000 --- a/tools/broccoli/broccoli-dest-copy.ts +++ /dev/null @@ -1,36 +0,0 @@ -/// -/// - -import fs = require('fs'); -import fse = require('fs-extra'); -import path = require('path'); -import {wrapDiffingPlugin, DiffingBroccoliPlugin, DiffResult} from './diffing-broccoli-plugin'; - -/** - * Intercepts each file as it is copied to the destination tempdir, - * and tees a copy to the given path outside the tmp dir. - */ -class DestCopy implements DiffingBroccoliPlugin { - constructor(private inputPath, private cachePath: string, private outputRoot: string) {} - - - rebuild(treeDiff: DiffResult) { - treeDiff.addedPaths.concat(treeDiff.changedPaths) - .forEach((changedFilePath) => { - var sourceFilePath = path.join(this.inputPath, changedFilePath); - var destFilePath = path.join(this.cachePath, this.outputRoot, changedFilePath); - var destDirPath = path.dirname(destFilePath); - - fse.mkdirsSync(destDirPath); - fse.copySync(sourceFilePath, destFilePath); - }); - - treeDiff.removedPaths.forEach((removedFilePath) => { - var destFilePath = path.join(this.outputRoot, removedFilePath); - // TODO: what about obsolete directories? we are not cleaning those up yet - fs.unlinkSync(destFilePath); - }); - } -} - -export default wrapDiffingPlugin(DestCopy); diff --git a/tools/broccoli/broccoli-filter.d.ts b/tools/broccoli/broccoli-filter.d.ts deleted file mode 100644 index 3a04ebea2a8..00000000000 --- a/tools/broccoli/broccoli-filter.d.ts +++ /dev/null @@ -1,14 +0,0 @@ -/// - -interface FilterOptions { - extensions?: string[] -} - -declare class Filter { - constructor(inputTree: any, options?: FilterOptions); - processString(contents: string, relativePath: string): string; - // NB: This function is probably not intended as part of the public API - processFile(srcDir: string, destDir: string, relativePath: string): Promise; -} - -export = Filter; diff --git a/tools/broccoli/broccoli-flatten.spec.ts b/tools/broccoli/broccoli-flatten.spec.ts deleted file mode 100644 index b58fd585d5e..00000000000 --- a/tools/broccoli/broccoli-flatten.spec.ts +++ /dev/null @@ -1,76 +0,0 @@ -/// -/// - -let mockfs = require('mock-fs'); -import fs = require('fs'); -import path = require('path'); -import {TreeDiffer} from './tree-differ'; -import {DiffingFlatten} from './broccoli-flatten'; - -describe('Flatten', () => { - afterEach(() => mockfs.restore()); - - function flatten(inputPaths) { return new DiffingFlatten(inputPaths, 'output', null); } - - function read(path) { return fs.readFileSync(path, {encoding: "utf-8"}); } - function rm(path) { return fs.unlinkSync(path); } - function write(path, content) { fs.writeFileSync(path, content, {encoding: "utf-8"}); } - - - it('should flatten files and be incremental', () => { - let testDir = { - 'input': { - 'dir1': { - 'file-1.txt': mockfs.file({content: 'file-1.txt content', mtime: new Date(1000)}), - 'file-2.txt': mockfs.file({content: 'file-2.txt content', mtime: new Date(1000)}), - 'subdir-1': { - 'file-1.1.txt': mockfs.file({content: 'file-1.1.txt content', mtime: new Date(1000)}) - }, - 'empty-dir': {} - }, - }, - 'output': {} - }; - mockfs(testDir); - - let differ = new TreeDiffer('testLabel', 'input'); - let flattenedTree = flatten('input'); - flattenedTree.rebuild(differ.diffTree()); - - expect(fs.readdirSync('output')).toEqual(['file-1.1.txt', 'file-1.txt', 'file-2.txt']); - // fails due to a mock-fs bug related to reading symlinks? - // expect(read('output/file-1.1.txt')).toBe('file-1.1.txt content'); - - - // delete a file - rm('input/dir1/file-1.txt'); - // add a new one - write('input/dir1/file-3.txt', 'file-3.txt content'); - - flattenedTree.rebuild(differ.diffTree()); - - expect(fs.readdirSync('output')).toEqual(['file-1.1.txt', 'file-2.txt', 'file-3.txt']); - }); - - - it('should throw an exception if duplicates are found', () => { - let testDir = { - 'input': { - 'dir1': { - 'file-1.txt': mockfs.file({content: 'file-1.txt content', mtime: new Date(1000)}), - 'subdir-1': - {'file-1.txt': mockfs.file({content: 'file-1.1.txt content', mtime: new Date(1000)})}, - 'empty-dir': {} - }, - }, - 'output': {} - }; - mockfs(testDir); - - let differ = new TreeDiffer('testLabel', 'input'); - let flattenedTree = flatten('input'); - expect(() => flattenedTree.rebuild(differ.diffTree())) - .toThrowError("Duplicate file 'file-1.txt' found in path 'dir1" + path.sep + "subdir-1" + - path.sep + "file-1.txt'"); - }); -}); diff --git a/tools/broccoli/broccoli-flatten.ts b/tools/broccoli/broccoli-flatten.ts deleted file mode 100644 index 079c15f671d..00000000000 --- a/tools/broccoli/broccoli-flatten.ts +++ /dev/null @@ -1,51 +0,0 @@ -import fs = require('fs'); -import fse = require('fs-extra'); -import path = require('path'); -import {wrapDiffingPlugin, DiffingBroccoliPlugin, DiffResult} from './diffing-broccoli-plugin'; -var symlinkOrCopy = require('symlink-or-copy').sync; - -var isWindows = process.platform === 'win32'; - - -/** - * Intercepts each changed file and replaces its contents with - * the associated changes. - */ -export class DiffingFlatten implements DiffingBroccoliPlugin { - constructor(private inputPath, private cachePath, private options) {} - - - rebuild(treeDiff: DiffResult) { - let pathsToUpdate = treeDiff.addedPaths; - - // since we need to run on Windows as well we can't rely on symlinks being available, - // which means that we need to respond to both added and changed paths - if (isWindows) { - pathsToUpdate = pathsToUpdate.concat(treeDiff.changedPaths); - } - - pathsToUpdate.forEach((changedFilePath) => { - var sourceFilePath = path.join(this.inputPath, changedFilePath); - var destFilePath = path.join(this.cachePath, path.basename(changedFilePath)); - var destDirPath = path.dirname(destFilePath); - - if (!fs.existsSync(destDirPath)) { - fse.mkdirpSync(destDirPath); - } - - if (!fs.existsSync(destFilePath)) { - symlinkOrCopy(sourceFilePath, destFilePath); - } else { - throw new Error(`Duplicate file '${path.basename(changedFilePath)}' ` + - `found in path '${changedFilePath}'`); - } - }); - - treeDiff.removedPaths.forEach((removedFilePath) => { - var destFilePath = path.join(this.cachePath, path.basename(removedFilePath)); - fs.unlinkSync(destFilePath); - }); - } -} - -export default wrapDiffingPlugin(DiffingFlatten); diff --git a/tools/broccoli/broccoli-lodash.ts b/tools/broccoli/broccoli-lodash.ts deleted file mode 100644 index 76388c2aafd..00000000000 --- a/tools/broccoli/broccoli-lodash.ts +++ /dev/null @@ -1,49 +0,0 @@ -import fs = require('fs'); -import fse = require('fs-extra'); -import path = require('path'); -var _ = require('lodash'); -import {wrapDiffingPlugin, DiffingBroccoliPlugin, DiffResult} from './diffing-broccoli-plugin'; - -interface LodashRendererOptions { - encoding?: string; - context?: any; - // files option unsupported --- use Funnel on inputTree instead. - files?: string[]; -} - -const kDefaultOptions: LodashRendererOptions = { - encoding: 'utf-8', - context: {}, - files: [] -}; - - -/** - * Intercepts each changed file and replaces its contents with - * the associated changes. - */ -export class LodashRenderer implements DiffingBroccoliPlugin { - constructor(private inputPath, private cachePath, - private options: LodashRendererOptions = kDefaultOptions) {} - - rebuild(treeDiff: DiffResult) { - let {encoding = 'utf-8', context = {}} = this.options; - let processFile = (relativePath) => { - let sourceFilePath = path.join(this.inputPath, relativePath); - let destFilePath = path.join(this.cachePath, relativePath); - let content = fs.readFileSync(sourceFilePath, {encoding}); - let transformedContent = _.template(content)(context); - fse.outputFileSync(destFilePath, transformedContent); - }; - - let removeFile = (relativePath) => { - let destFilePath = path.join(this.cachePath, relativePath); - fs.unlinkSync(destFilePath); - }; - - treeDiff.addedPaths.concat(treeDiff.changedPaths).forEach(processFile); - treeDiff.removedPaths.forEach(removeFile); - } -} - -export default wrapDiffingPlugin(LodashRenderer); diff --git a/tools/broccoli/broccoli-merge-trees.spec.ts b/tools/broccoli/broccoli-merge-trees.spec.ts deleted file mode 100644 index c1c659f5462..00000000000 --- a/tools/broccoli/broccoli-merge-trees.spec.ts +++ /dev/null @@ -1,91 +0,0 @@ -/// -/// - -let mockfs = require('mock-fs'); -import fs = require('fs'); -import {TreeDiffer} from './tree-differ'; -import {MergeTrees} from './broccoli-merge-trees'; - -describe('MergeTrees', () => { - afterEach(() => mockfs.restore()); - - function mergeTrees(inputPaths, cachePath, options) { - return new MergeTrees(inputPaths, cachePath, options); - } - - function MakeTreeDiffers(rootDirs) { - let treeDiffers = rootDirs.map((rootDir) => new TreeDiffer('MergeTrees', rootDir)); - treeDiffers.diffTrees = () => { return treeDiffers.map(tree => tree.diffTree()); }; - return treeDiffers; - } - - function read(path) { return fs.readFileSync(path, "utf-8"); } - - it('should copy the file from the right-most inputTree with overwrite=true', () => { - let testDir: any = { - 'tree1': {'foo.js': mockfs.file({content: 'tree1/foo.js content', mtime: new Date(1000)})}, - 'tree2': {'foo.js': mockfs.file({content: 'tree2/foo.js content', mtime: new Date(1000)})}, - 'tree3': {'foo.js': mockfs.file({content: 'tree3/foo.js content', mtime: new Date(1000)})} - }; - mockfs(testDir); - let treeDiffer = MakeTreeDiffers(['tree1', 'tree2', 'tree3']); - let treeMerger = mergeTrees(['tree1', 'tree2', 'tree3'], 'dest', {overwrite: true}); - treeMerger.rebuild(treeDiffer.diffTrees()); - expect(read('dest/foo.js')).toBe('tree3/foo.js content'); - - delete testDir.tree2['foo.js']; - delete testDir.tree3['foo.js']; - mockfs(testDir); - treeMerger.rebuild(treeDiffer.diffTrees()); - expect(read('dest/foo.js')).toBe('tree1/foo.js content'); - - testDir.tree2['foo.js'] = mockfs.file({content: 'tree2/foo.js content', mtime: new Date(1000)}); - mockfs(testDir); - treeMerger.rebuild(treeDiffer.diffTrees()); - expect(read('dest/foo.js')).toBe('tree2/foo.js content'); - }); - - it('should throw if duplicates are found during the initial build', () => { - let testDir: any = { - 'tree1': {'foo.js': mockfs.file({content: 'tree1/foo.js content', mtime: new Date(1000)})}, - 'tree2': {'foo.js': mockfs.file({content: 'tree2/foo.js content', mtime: new Date(1000)})}, - 'tree3': {'foo.js': mockfs.file({content: 'tree3/foo.js content', mtime: new Date(1000)})} - }; - mockfs(testDir); - let treeDiffer = MakeTreeDiffers(['tree1', 'tree2', 'tree3']); - let treeMerger = mergeTrees(['tree1', 'tree2', 'tree3'], 'dest', {}); - expect(() => treeMerger.rebuild(treeDiffer.diffTrees())) - .toThrowError( - 'Duplicate path found while merging trees. Path: "foo.js".\n' + - 'Either remove the duplicate or enable the "overwrite" option for this merge.'); - - testDir = { - 'tree1': {'foo.js': mockfs.file({content: 'tree1/foo.js content', mtime: new Date(1000)})}, - 'tree2': {}, - 'tree3': {} - }; - mockfs(testDir); - }); - - - it('should throw if duplicates are found during rebuild', () => { - let testDir = { - 'tree1': {'foo.js': mockfs.file({content: 'tree1/foo.js content', mtime: new Date(1000)})}, - 'tree2': {}, - 'tree3': {} - }; - mockfs(testDir); - - let treeDiffer = MakeTreeDiffers(['tree1', 'tree2', 'tree3']); - let treeMerger = mergeTrees(['tree1', 'tree2', 'tree3'], 'dest', {}); - expect(() => treeMerger.rebuild(treeDiffer.diffTrees())).not.toThrow(); - - - testDir.tree2['foo.js'] = mockfs.file({content: 'tree2/foo.js content', mtime: new Date(1000)}); - mockfs(testDir); - expect(() => treeMerger.rebuild(treeDiffer.diffTrees())) - .toThrowError( - 'Duplicate path found while merging trees. Path: "foo.js".\n' + - 'Either remove the duplicate or enable the "overwrite" option for this merge.'); - }); -}); diff --git a/tools/broccoli/broccoli-merge-trees.ts b/tools/broccoli/broccoli-merge-trees.ts deleted file mode 100644 index 60672c1a2b2..00000000000 --- a/tools/broccoli/broccoli-merge-trees.ts +++ /dev/null @@ -1,137 +0,0 @@ -import fs = require('fs'); -import fse = require('fs-extra'); -import path = require('path'); -var symlinkOrCopySync = require('symlink-or-copy').sync; -import {wrapDiffingPlugin, DiffingBroccoliPlugin, DiffResult} from './diffing-broccoli-plugin'; - -var isWindows = process.platform === 'win32'; - -interface MergeTreesOptions { - overwrite?: boolean; -} - -function outputFileSync(sourcePath, destPath) { - let dirname = path.dirname(destPath); - fse.mkdirsSync(dirname, {fs: fs}); - symlinkOrCopySync(sourcePath, destPath); -} - -function pathOverwrittenError(path) { - const msg = 'Either remove the duplicate or enable the "overwrite" option for this merge.'; - return new Error(`Duplicate path found while merging trees. Path: "${path}".\n${msg}`); -} - -export class MergeTrees implements DiffingBroccoliPlugin { - private pathCache: {[key: string]: number[]} = Object.create(null); - public options: MergeTreesOptions; - private firstBuild: boolean = true; - - constructor(public inputPaths: string[], public cachePath: string, - options: MergeTreesOptions = {}) { - this.options = options || {}; - } - - rebuild(treeDiffs: DiffResult[]) { - let overwrite = this.options.overwrite; - let pathsToEmit: string[] = []; - let pathsToRemove: string[] = []; - let emitted: {[key: string]: boolean} = Object.create(null); - let contains = (cache, val) => { - for (let i = 0, ii = cache.length; i < ii; ++i) { - if (cache[i] === val) return true; - } - return false; - }; - - let emit = (relativePath) => { - // ASSERT(!emitted[relativePath]); - pathsToEmit.push(relativePath); - emitted[relativePath] = true; - }; - - if (this.firstBuild) { - this.firstBuild = false; - - // Build initial cache - treeDiffs.reverse().forEach((treeDiff: DiffResult, index) => { - index = treeDiffs.length - 1 - index; - treeDiff.addedPaths.forEach((changedPath) => { - let cache = this.pathCache[changedPath]; - if (cache === undefined) { - this.pathCache[changedPath] = [index]; - pathsToEmit.push(changedPath); - } else if (overwrite) { - // ASSERT(contains(pathsToEmit, changedPath)); - cache.unshift(index); - } else { - throw pathOverwrittenError(changedPath); - } - }); - }); - - } else { - // Update cache - treeDiffs.reverse().forEach((treeDiff: DiffResult, index) => { - index = treeDiffs.length - 1 - index; - if (treeDiff.removedPaths) { - treeDiff.removedPaths.forEach((removedPath) => { - let cache = this.pathCache[removedPath]; - // ASSERT(cache !== undefined); - // ASSERT(contains(cache, index)); - if (cache[cache.length - 1] === index) { - pathsToRemove.push(path.join(this.cachePath, removedPath)); - cache.pop(); - if (cache.length === 0) { - this.pathCache[removedPath] = undefined; - } else if (!emitted[removedPath]) { - if (cache.length === 1 && !overwrite) { - throw pathOverwrittenError(removedPath); - } - emit(removedPath); - } - } - }); - } - - let pathsToUpdate = treeDiff.addedPaths; - - if (isWindows) { - pathsToUpdate = pathsToUpdate.concat(treeDiff.changedPaths); - } - - pathsToUpdate.forEach((changedPath) => { - let cache = this.pathCache[changedPath]; - if (cache === undefined) { - // File was added - this.pathCache[changedPath] = [index]; - emit(changedPath); - } else if (!contains(cache, index)) { - cache.push(index); - cache.sort((a, b) => a - b); - if (cache.length > 1 && !overwrite) { - throw pathOverwrittenError(changedPath); - } - if (cache[cache.length - 1] === index && !emitted[changedPath]) { - emit(changedPath); - } - } - }); - }); - } - - pathsToRemove.forEach((destPath) => fse.removeSync(destPath)); - pathsToEmit.forEach((emittedPath) => { - let cache = this.pathCache[emittedPath]; - let destPath = path.join(this.cachePath, emittedPath); - let sourceIndex = cache[cache.length - 1]; - let sourceInputPath = this.inputPaths[sourceIndex]; - let sourcePath = path.join(sourceInputPath, emittedPath); - if (cache.length > 1) { - fse.removeSync(destPath); - } - outputFileSync(sourcePath, destPath); - }); - } -} - -export default wrapDiffingPlugin(MergeTrees); diff --git a/tools/broccoli/broccoli-replace.ts b/tools/broccoli/broccoli-replace.ts deleted file mode 100644 index 7583dd5b3b7..00000000000 --- a/tools/broccoli/broccoli-replace.ts +++ /dev/null @@ -1,59 +0,0 @@ -import fs = require('fs'); -import fse = require('fs-extra'); -import path = require('path'); -import {wrapDiffingPlugin, DiffingBroccoliPlugin, DiffResult} from './diffing-broccoli-plugin'; - -var minimatch = require('minimatch'); -var FILE_ENCODING = {encoding: 'utf-8'}; - -/** - * Intercepts each changed file and replaces its contents with - * the associated changes. - */ -class DiffingReplace implements DiffingBroccoliPlugin { - constructor(private inputPath, private cachePath, private options) {} - - rebuild(treeDiff: DiffResult) { - var patterns = this.options.patterns; - var files = this.options.files; - - treeDiff.addedPaths.concat(treeDiff.changedPaths) - .forEach((changedFilePath) => { - var sourceFilePath = path.join(this.inputPath, changedFilePath); - var destFilePath = path.join(this.cachePath, changedFilePath); - var destDirPath = path.dirname(destFilePath); - - if (!fs.existsSync(destDirPath)) { - fse.mkdirpSync(destDirPath); - } - - var fileMatches = files.some((filePath) => minimatch(changedFilePath, filePath)); - if (fileMatches) { - var content = fs.readFileSync(sourceFilePath, FILE_ENCODING); - patterns.forEach((pattern) => { - var replacement = pattern.replacement; - if (typeof replacement === 'function') { - replacement = function(content) { - return pattern.replacement(content, changedFilePath); - }; - } - content = content.replace(pattern.match, replacement); - }); - fs.writeFileSync(destFilePath, content, FILE_ENCODING); - } else if (!fs.existsSync(destFilePath)) { - try { - fs.symlinkSync(sourceFilePath, destFilePath); - } catch (e) { - fs.writeFileSync(destFilePath, fs.readFileSync(sourceFilePath)); - } - } - }); - - treeDiff.removedPaths.forEach((removedFilePath) => { - var destFilePath = path.join(this.cachePath, removedFilePath); - fs.unlinkSync(destFilePath); - }); - } -} - -export default wrapDiffingPlugin(DiffingReplace); diff --git a/tools/broccoli/broccoli-tree-stabilizer.ts b/tools/broccoli/broccoli-tree-stabilizer.ts deleted file mode 100644 index 9a1bf97a4f0..00000000000 --- a/tools/broccoli/broccoli-tree-stabilizer.ts +++ /dev/null @@ -1,45 +0,0 @@ -/// -/// - -import fs = require('fs'); -let symlinkOrCopy = require('symlink-or-copy'); - - -/** - * Stabilizes the inputPath for the following plugins in the build tree. - * - * All broccoli plugins that inherit from `broccoli-writer` or `broccoli-filter` change their - * outputPath during each rebuild. - * - * This means that all following plugins in the build tree can't rely on their inputPath being - * immutable. This results in breakage of any plugin that is not expecting such behavior. - * - * For example all `DiffingBroccoliPlugin`s expect their inputPath to be stable. - * - * By inserting this plugin into the tree after any misbehaving plugin, we can stabilize the - * inputPath for the following plugin in the tree and correct the surprising behavior. - */ -class TreeStabilizer implements BroccoliTree { - inputPath: string; - outputPath: string; - - - constructor(public inputTree: BroccoliTree) {} - - - rebuild() { - fs.rmdirSync(this.outputPath); - - // TODO: investigate if we can use rename the directory instead to improve performance on - // Windows - symlinkOrCopy.sync(this.inputPath, this.outputPath); - } - - - cleanup() {} -} - - -export default function stabilizeTree(inputTree) { - return new TreeStabilizer(inputTree); -} diff --git a/tools/broccoli/broccoli-ts2dart.ts b/tools/broccoli/broccoli-ts2dart.ts index 6acd88878e9..65821dae783 100644 --- a/tools/broccoli/broccoli-ts2dart.ts +++ b/tools/broccoli/broccoli-ts2dart.ts @@ -1,60 +1,98 @@ /// /// +/// + +interface BroccoliCachingWriterPlugin extends BroccoliTree { + listFiles(): string[]; + listEntries(): any[]; +} + +interface Constructor { + new (...args: any[]): BroccoliCachingWriterPlugin; +} + +interface CacheEntry { + version: number; +} + import fs = require('fs'); import fse = require('fs-extra'); import path = require('path'); -import {wrapDiffingPlugin, DiffingBroccoliPlugin, DiffResult} from './diffing-broccoli-plugin'; -class TSToDartTranspiler implements DiffingBroccoliPlugin { - static includeExtensions = ['.ts']; +const Plugin: Constructor = require('broccoli-caching-writer'); +import TS2Dart = require('ts2dart'); - private transpiler: any /*ts2dart.Transpiler*/; + +export default class TSToDartTranspiler extends Plugin { + private transpiler: TS2Dart.Transpiler; private additionalFiles: string[]; + private basePath: string; + private cache: {[filePath: string]: CacheEntry} = Object.create(null); + + constructor(inputPaths: any[], public options: any /*ts2dart.TranspilerOptions*/) { + super(inputPaths, {}); - constructor(public inputPath: string, public cachePath: string, - public options: any /*ts2dart.TranspilerOptions*/) { // Additional files to include in the transpile call. These files could include, e.g., // .d.ts files. They will always be included in the transpile call below, even if they // weren't modified. this.additionalFiles = options.additionalFiles || []; + // Delete additionalFiles as the transpiler does not support that option. delete options.additionalFiles; - options.basePath = inputPath; - // Workaround for https://github.com/dart-lang/dart_style/issues/493 - var ts2dart = require('ts2dart'); - this.transpiler = new ts2dart.Transpiler(options); + this.basePath = options.basePath; + this.transpiler = new TS2Dart.Transpiler(options); } - rebuild(treeDiff: DiffResult) { - let toEmit = [path.resolve(this.inputPath, 'angular2/manual_typings/globals.d.ts')]; - let getDartFilePath = (path: string) => path.replace(/((\.js)|(\.ts))$/i, '.dart'); - treeDiff.addedPaths.concat(treeDiff.changedPaths) - .filter(name => !name.match(/\.d\.ts$/)) - .forEach((changedPath) => { - let inputFilePath = path.resolve(this.inputPath, changedPath); + build(): Promise { + const inputPath = this.inputPaths[0]; + const basePath = this.basePath || inputPath; + const getDartFilePath = (path: string) => path.replace(/((\.js)|(\.ts))$/i, '.dart'); - // Ignore files which don't need to be transpiled to Dart - let dartInputFilePath = getDartFilePath(inputFilePath); - if (fs.existsSync(dartInputFilePath)) return; + // List all the changed TS files. The `toEmit` variable will contain the list + // of relative paths to the basePath of files that have changed. + // For an initial run this is basically all the files. + const toEmit = this.listEntries() + .filter(entry => entry.relativePath.match(/\.ts$/) && !entry.relativePath.match(/\.d\.ts$/)) + .map(entry => { + const fullPath = path.resolve(inputPath, entry.relativePath); + const dartInputFilePath = getDartFilePath(fullPath); - // Prepare to rebuild - toEmit.push(path.resolve(this.inputPath, changedPath)); - }); + // If a dart file exist, just ignore the TS source. + if (fs.existsSync(dartInputFilePath)) { + return; + } - treeDiff.removedPaths.forEach((removedPath) => { - let absolutePath = path.resolve(this.inputPath, removedPath); + if (!this.cache[fullPath]) { + this.cache[fullPath] = { version: entry.mtime }; + } else if (this.cache[fullPath].version >= entry.mtime) { + // Do nothing. Cache is as recent as the original input. + return; + } + return path.normalize(fullPath.replace(inputPath, basePath)); + }) + .filter(x => !!x); - // Ignore files which don't need to be transpiled to Dart - let dartInputFilePath = getDartFilePath(absolutePath); - if (fs.existsSync(dartInputFilePath)) return; - - let dartOutputFilePath = getDartFilePath(removedPath); - fs.unlinkSync(path.join(this.cachePath, dartOutputFilePath)); - }); this.transpiler.transpile(this.additionalFiles.concat(toEmit), this.cachePath); + + const dirs: {[p: string]: boolean} = Object.create(null); + // Make a symbolic link in the output for the cache. + return Promise.all( + Object.keys(this.cache).map(filePath => { + return new Promise(resolve => { + const cachePath = getDartFilePath(filePath.replace(inputPath, this.cachePath)); + const outputPath = getDartFilePath(filePath.replace(inputPath, this.outputPath)); + + const dirPath = path.dirname(outputPath); + if (!dirs[dirPath]) { + dirs[dirPath] = true; + fse.mkdirsSync(path.dirname(outputPath)); + } + fs.linkSync(cachePath, outputPath); + resolve(); + }); + }) + ); } } - -export default wrapDiffingPlugin(TSToDartTranspiler); diff --git a/tools/broccoli/broccoli-typescript.ts b/tools/broccoli/broccoli-typescript.ts deleted file mode 100644 index 1913d02c502..00000000000 --- a/tools/broccoli/broccoli-typescript.ts +++ /dev/null @@ -1,370 +0,0 @@ -/// - -import fs = require('fs'); -import fse = require('fs-extra'); -import path = require('path'); -import * as ts from 'typescript'; -import {wrapDiffingPlugin, DiffingBroccoliPlugin, DiffResult} from './diffing-broccoli-plugin'; - - -type FileRegistry = ts.Map<{version: number}>; - -const FS_OPTS = { - encoding: 'utf-8' -}; - -// Sub-directory where the @internal typing files (.d.ts) are stored -export const INTERNAL_TYPINGS_PATH: string = 'internal_typings'; - -// Monkey patch the TS compiler to be able to re-emit files with @internal symbols -let tsEmitInternal: boolean = false; - -const originalEmitFiles: Function = (ts).emitFiles; - -(ts).emitFiles = function(resolver: any, host: any, targetSourceFile: any): any { - if (tsEmitInternal) { - const orignalgetCompilerOptions = host.getCompilerOptions; - host.getCompilerOptions = () => { - let options = clone(orignalgetCompilerOptions.call(host)); - options.stripInternal = false; - options.outDir = `${options.outDir}/${INTERNAL_TYPINGS_PATH}`; - return options; - } - } - return originalEmitFiles(resolver, host, targetSourceFile); -}; - -/** - * Broccoli plugin that implements incremental Typescript compiler. - * - * It instantiates a typescript compiler instance that keeps all the state about the project and - * can re-emit only the files that actually changed. - * - * Limitations: only files that map directly to the changed source file via naming conventions are - * re-emitted. This primarily affects code that uses `const enum`s, because changing the enum value - * requires global emit, which can affect many files. - */ -class DiffingTSCompiler implements DiffingBroccoliPlugin { - private tsOpts: ts.CompilerOptions; - private fileRegistry: FileRegistry = Object.create(null); - private rootFilePaths: string[]; - private tsServiceHost: ts.LanguageServiceHost; - private tsService: ts.LanguageService; - private firstRun: boolean = true; - private previousRunFailed: boolean = false; - // Whether to generate the @internal typing files (they are only generated when `stripInternal` is - // true) - private genInternalTypings: boolean = false; - - static includeExtensions = ['.ts']; - static excludeExtensions = ['.d.ts']; - - constructor(public inputPath: string, public cachePath: string, public options) { - if (options.rootFilePaths) { - this.rootFilePaths = options.rootFilePaths.splice(0); - delete options.rootFilePaths; - } else { - this.rootFilePaths = []; - } - - if (options.internalTypings) { - this.genInternalTypings = true; - delete options.internalTypings; - } - - // the conversion is a bit awkward, see https://github.com/Microsoft/TypeScript/issues/5276 - // in 1.8 use convertCompilerOptionsFromJson - this.tsOpts = - ts.parseJsonConfigFileContent({compilerOptions: options, files: []}, null, null).options; - - if ((this.tsOpts).stripInternal === false) { - // @internal are included in the generated .d.ts, do not generate them separately - this.genInternalTypings = false; - } - - // TODO: the above turns rootDir set to './' into an empty string - looks like a tsc bug - // check back when we upgrade to 1.7.x - if (this.tsOpts.rootDir === '') { - this.tsOpts.rootDir = './'; - } - this.tsOpts.outDir = this.cachePath; - - this.tsServiceHost = new CustomLanguageServiceHost(this.tsOpts, this.rootFilePaths, - this.fileRegistry, this.inputPath); - this.tsService = ts.createLanguageService(this.tsServiceHost, ts.createDocumentRegistry()); - } - - - rebuild(treeDiff: DiffResult) { - let pathsToEmit = []; - let pathsWithErrors = []; - let errorMessages = []; - - treeDiff.addedPaths.concat(treeDiff.changedPaths) - .forEach((tsFilePath) => { - if (!this.fileRegistry[tsFilePath]) { - this.fileRegistry[tsFilePath] = {version: 0}; - this.rootFilePaths.push(tsFilePath); - } else { - this.fileRegistry[tsFilePath].version++; - } - - pathsToEmit.push(tsFilePath); - }); - - treeDiff.removedPaths.forEach((tsFilePath) => { - console.log('removing outputs for', tsFilePath); - - this.rootFilePaths.splice(this.rootFilePaths.indexOf(tsFilePath), 1); - this.fileRegistry[tsFilePath] = null; - this.removeOutputFor(tsFilePath); - }); - - if (this.firstRun) { - this.firstRun = false; - this.doFullBuild(); - } else { - tsEmitInternal = false; - pathsToEmit.forEach((tsFilePath) => { - let output = this.tsService.getEmitOutput(tsFilePath); - - if (output.emitSkipped) { - let errorFound = this.collectErrors(tsFilePath); - if (errorFound) { - pathsWithErrors.push(tsFilePath); - errorMessages.push(errorFound); - } - } else { - output.outputFiles.forEach(o => { - let destDirPath = path.dirname(o.name); - fse.mkdirsSync(destDirPath); - fs.writeFileSync(o.name, this.fixSourceMapSources(o.text), FS_OPTS); - }); - } - }); - - if (pathsWithErrors.length) { - this.previousRunFailed = true; - var error = - new Error('Typescript found the following errors:\n' + errorMessages.join('\n')); - error['showStack'] = false; - throw error; - } else if (this.previousRunFailed) { - this.doFullBuild(); - } else if (this.genInternalTypings) { - // serialize the .d.ts files containing @internal symbols - tsEmitInternal = true; - pathsToEmit.forEach((tsFilePath) => { - let output = this.tsService.getEmitOutput(tsFilePath); - if (!output.emitSkipped) { - output.outputFiles.forEach(o => { - if (endsWith(o.name, '.d.ts')) { - let destDirPath = path.dirname(o.name); - fse.mkdirsSync(destDirPath); - fs.writeFileSync(o.name, this.fixSourceMapSources(o.text), FS_OPTS); - } - }); - } - }); - tsEmitInternal = false; - } - } - } - - private collectErrors(tsFilePath): String { - let allDiagnostics = this.tsService.getCompilerOptionsDiagnostics() - .concat(this.tsService.getSyntacticDiagnostics(tsFilePath)) - .concat(this.tsService.getSemanticDiagnostics(tsFilePath)); - let errors = []; - - allDiagnostics.forEach(diagnostic => { - let message = ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n"); - if (diagnostic.file) { - let {line, character} = diagnostic.file.getLineAndCharacterOfPosition(diagnostic.start); - errors.push(` ${diagnostic.file.fileName} (${line + 1},${character + 1}): ${message}`); - } else { - errors.push(` Error: ${message}`); - } - }); - - if (errors.length) { - return errors.join('\n'); - } - } - - private doFullBuild() { - let program = this.tsService.getProgram(); - - tsEmitInternal = false; - let emitResult = program.emit(undefined, (absoluteFilePath, fileContent) => { - fse.mkdirsSync(path.dirname(absoluteFilePath)); - fs.writeFileSync(absoluteFilePath, this.fixSourceMapSources(fileContent), FS_OPTS); - }); - - if (this.genInternalTypings) { - // serialize the .d.ts files containing @internal symbols - tsEmitInternal = true; - program.emit(undefined, (absoluteFilePath, fileContent) => { - if (endsWith(absoluteFilePath, '.d.ts')) { - fse.mkdirsSync(path.dirname(absoluteFilePath)); - fs.writeFileSync(absoluteFilePath, fileContent, FS_OPTS); - } - }); - tsEmitInternal = false; - } - - if (emitResult.emitSkipped) { - let allDiagnostics = ts.getPreEmitDiagnostics(program).concat(emitResult.diagnostics); - let errorMessages = []; - - allDiagnostics.forEach(diagnostic => { - var pos = ''; - if (diagnostic.file) { - var {line, character} = diagnostic.file.getLineAndCharacterOfPosition(diagnostic.start); - pos = `${diagnostic.file.fileName} (${line + 1}, ${character + 1}): ` - } - var message = ts.flattenDiagnosticMessageText(diagnostic.messageText, '\n'); - errorMessages.push(` ${pos}${message}`); - }); - - if (errorMessages.length) { - this.previousRunFailed = true; - var error = - new Error('Typescript found the following errors:\n' + errorMessages.join('\n')); - error['showStack'] = false; - throw error; - } else { - this.previousRunFailed = false; - } - } - } - - /** - * There is a bug in TypeScript 1.6, where the sourceRoot and inlineSourceMap properties - * are exclusive. This means that the sources property always contains relative paths - * (e.g, ../../../../angular2/src/di/injector.ts). - * - * Here, we normalize the sources property and remove the ../../../ - * - * This issue is fixed in https://github.com/Microsoft/TypeScript/pull/5620. - * Once we switch to TypeScript 1.8, we can remove this method. - */ - private fixSourceMapSources(content: string): string { - try { - const marker = "//# sourceMappingURL=data:application/json;base64,"; - const index = content.indexOf(marker); - if (index == -1) return content; - - const base = content.substring(0, index + marker.length); - const sourceMapBit = - new Buffer(content.substring(index + marker.length), 'base64').toString("utf8"); - const sourceMaps = JSON.parse(sourceMapBit); - const source = sourceMaps.sources[0]; - sourceMaps.sources = [source.substring(source.lastIndexOf("../") + 3)]; - return `${base}${new Buffer(JSON.stringify(sourceMaps)).toString('base64')}`; - } catch (e) { - return content; - } - } - - private removeOutputFor(tsFilePath: string) { - let absoluteJsFilePath = path.join(this.cachePath, tsFilePath.replace(/\.ts$/, '.js')); - let absoluteMapFilePath = path.join(this.cachePath, tsFilePath.replace(/.ts$/, '.js.map')); - let absoluteDtsFilePath = path.join(this.cachePath, tsFilePath.replace(/\.ts$/, '.d.ts')); - - if (fs.existsSync(absoluteJsFilePath)) { - fs.unlinkSync(absoluteJsFilePath); - if (fs.existsSync(absoluteMapFilePath)) { - // source map could be inline or not generated - fs.unlinkSync(absoluteMapFilePath); - } - fs.unlinkSync(absoluteDtsFilePath); - } - } -} - - -class CustomLanguageServiceHost implements ts.LanguageServiceHost { - private currentDirectory: string; - private defaultLibFilePath: string; - - - constructor(private compilerOptions: ts.CompilerOptions, private fileNames: string[], - private fileRegistry: FileRegistry, private treeInputPath: string) { - this.currentDirectory = process.cwd(); - this.defaultLibFilePath = ts.getDefaultLibFilePath(compilerOptions).replace(/\\/g, '/'); - } - - - getScriptFileNames(): string[] { return this.fileNames; } - - - getScriptVersion(fileName: string): string { - return this.fileRegistry[fileName] && this.fileRegistry[fileName].version.toString(); - } - - - /** - * This method is called quite a bit to lookup 3 kinds of paths: - * 1/ files in the fileRegistry - * - these are the files in our project that we are watching for changes - * - in the future we could add caching for these files and invalidate the cache when - * the file is changed lazily during lookup - * 2/ .d.ts and library files not in the fileRegistry - * - these are not our files, they come from tsd or typescript itself - * - these files change only rarely but since we need them very rarely, it's not worth the - * cache invalidation hassle to cache them - * 3/ bogus paths that typescript compiler tries to lookup during import resolution - * - these paths are tricky to cache since files come and go and paths that was bogus in the - * past might not be bogus later - * - * In the initial experiments the impact of this caching was insignificant (single digit %) and - * not worth the potential issues with stale cache records. - */ - getScriptSnapshot(tsFilePath: string): ts.IScriptSnapshot { - let absoluteTsFilePath; - - if (tsFilePath == this.defaultLibFilePath || path.isAbsolute(tsFilePath)) { - absoluteTsFilePath = tsFilePath; - } else if (this.compilerOptions.moduleResolution === ts.ModuleResolutionKind.NodeJs && - tsFilePath.match(/^node_modules/)) { - absoluteTsFilePath = path.resolve(tsFilePath); - } else if (tsFilePath.match(/^rxjs/)) { - absoluteTsFilePath = path.resolve('node_modules', tsFilePath); - } else { - absoluteTsFilePath = path.join(this.treeInputPath, tsFilePath); - } - - - if (!fs.existsSync(absoluteTsFilePath)) { - // TypeScript seems to request lots of bogus paths during import path lookup and resolution, - // so we we just return undefined when the path is not correct. - return undefined; - } - return ts.ScriptSnapshot.fromString(fs.readFileSync(absoluteTsFilePath, FS_OPTS)); - } - - - getCurrentDirectory(): string { return this.currentDirectory; } - - getCompilationSettings(): ts.CompilerOptions { return this.compilerOptions; } - - getDefaultLibFileName(options: ts.CompilerOptions): string { - // ignore options argument, options should not change during the lifetime of the plugin - return this.defaultLibFilePath; - } -} - -export default wrapDiffingPlugin(DiffingTSCompiler); - -function clone(object: T): T { - const result: any = {}; - for (const id in object) { - result[id] = (object)[id]; - } - return result; -} - -function endsWith(str: string, substring: string): boolean { - return str.indexOf(substring, str.length - substring.length) !== -1; -} diff --git a/tools/broccoli/broccoli-writer.d.ts b/tools/broccoli/broccoli-writer.d.ts deleted file mode 100644 index 5d821fa4f96..00000000000 --- a/tools/broccoli/broccoli-writer.d.ts +++ /dev/null @@ -1,8 +0,0 @@ -/// - -declare module "broccoli-writer" { - class Writer { - write(readTree: (tree) => Promise, destDir: string): Promise; - } - export = Writer; -} diff --git a/tools/broccoli/broccoli.d.ts b/tools/broccoli/broccoli.d.ts index d058488c82c..5c72bd18581 100644 --- a/tools/broccoli/broccoli.d.ts +++ b/tools/broccoli/broccoli.d.ts @@ -66,7 +66,7 @@ interface BroccoliTree { */ description?: string; - rebuild(): (Promise| void); + build(): (Promise| void); cleanup(): void; } diff --git a/tools/broccoli/diffing-broccoli-plugin.ts b/tools/broccoli/diffing-broccoli-plugin.ts deleted file mode 100644 index d61db4364f2..00000000000 --- a/tools/broccoli/diffing-broccoli-plugin.ts +++ /dev/null @@ -1,190 +0,0 @@ -/// -/// -/// - -import fs = require('fs'); -import fse = require('fs-extra'); -import path = require('path'); -import {TreeDiffer, DiffResult} from './tree-differ'; -import stabilizeTree from './broccoli-tree-stabilizer'; -let symlinkOrCopy = require('symlink-or-copy'); - - -export {DiffResult} from './tree-differ'; - - -/** - * Makes writing diffing plugins easy. - * - * Factory method that takes a class that implements the DiffingBroccoliPlugin interface and returns - * an instance of BroccoliTree. - * - * @param pluginClass - * @returns {DiffingBroccoliPlugin} - */ -export function wrapDiffingPlugin(pluginClass): DiffingPluginWrapperFactory { - return function() { return new DiffingPluginWrapper(pluginClass, arguments); }; -} - - -export interface DiffingBroccoliPlugin { - rebuild(diff: (DiffResult | DiffResult[])): (Promise| DiffResult | void); - cleanup ? () : void; -} - - -type DiffingPluginWrapperFactory = (inputTrees: (BroccoliTree | BroccoliTree[]), options?) => - BroccoliTree; - - -class DiffingPluginWrapper implements BroccoliTree { - treeDiffer: TreeDiffer = null; - treeDiffers: TreeDiffer[] = null; - initialized = false; - wrappedPlugin: DiffingBroccoliPlugin = null; - inputTree = null; - inputTrees = null; - description = null; - - // props monkey-patched by broccoli builder: - inputPath = null; - inputPaths = null; - cachePath = null; - outputPath = null; - - private diffResult: DiffResult = null; - - constructor(private pluginClass, private wrappedPluginArguments) { - if (Array.isArray(wrappedPluginArguments[0])) { - this.inputTrees = this.stabilizeTrees(wrappedPluginArguments[0]); - } else { - this.inputTree = this.stabilizeTree(wrappedPluginArguments[0]); - } - - this.description = this.pluginClass.name; - } - - private getDiffResult(): (DiffResult | DiffResult[]) { - let returnOrCalculateDiffResult = (tree, index) => { - // returnOrCalculateDiffResult will do one of two things: - // - // If `this.diffResult` is null, calculate a DiffResult using TreeDiffer - // for the input tree. - // - // Otherwise, `this.diffResult` was produced from the output of the - // inputTree's rebuild() method, and can be used without being checked. - // Set `this.diffResult` to null and return the previously stored value. - let diffResult = tree.diffResult; - if (diffResult) return diffResult; - let differ = index === false ? this.treeDiffer : this.treeDiffers[index]; - return differ.diffTree(); - }; - - if (this.inputTrees) { - return this.inputTrees.map(returnOrCalculateDiffResult); - } else if (this.inputTree) { - return returnOrCalculateDiffResult(this.inputTree, false); - } else { - throw new Error("Missing TreeDiffer"); - } - } - - private maybeStoreDiffResult(value: (DiffResult | void)) { - if (!(value instanceof DiffResult)) value = null; - this.diffResult = (value); - } - - rebuild(): (Promise| void) { - try { - let firstRun = !this.initialized; - this.init(); - - let diffResult = this.getDiffResult(); - - let result = this.wrappedPlugin.rebuild(diffResult); - - if (result) { - let resultPromise = >(result); - if (resultPromise.then) { - // rebuild() -> Promise<> - return resultPromise.then((result: (DiffResult | void)) => { - this.maybeStoreDiffResult(result); - this.relinkOutputAndCachePaths(); - }); - } - } - - this.maybeStoreDiffResult(<(DiffResult | void)>(result)); - this.relinkOutputAndCachePaths(); - } catch (e) { - e.message = `[${this.description}]: ${e.message}`; - throw e; - } - } - - - cleanup() { - if (this.wrappedPlugin && this.wrappedPlugin.cleanup) { - this.wrappedPlugin.cleanup(); - } - } - - - private relinkOutputAndCachePaths() { - // just symlink the cache and output tree - fs.rmdirSync(this.outputPath); - symlinkOrCopy.sync(this.cachePath, this.outputPath); - } - - - private init() { - if (!this.initialized) { - let includeExtensions = this.pluginClass.includeExtensions || []; - let excludeExtensions = this.pluginClass.excludeExtensions || []; - let description = this.description; - this.initialized = true; - if (this.inputPaths) { - this.treeDiffers = - this.inputPaths.map((inputPath) => new TreeDiffer( - description, inputPath, includeExtensions, excludeExtensions)); - } else if (this.inputPath) { - this.treeDiffer = - new TreeDiffer(description, this.inputPath, includeExtensions, excludeExtensions); - } - this.wrappedPlugin = new this.pluginClass(this.inputPaths || this.inputPath, this.cachePath, - this.wrappedPluginArguments[1]); - } - } - - - private stabilizeTrees(trees: BroccoliTree[]) { - // Prevent extensions to prevent array from being mutated from the outside. - // For-loop used to avoid re-allocating a new array. - var stableTrees = []; - for (let i = 0; i < trees.length; ++i) { - // ignore null/undefined input tries in order to support conditional build pipelines - if (trees[i]) { - stableTrees.push(this.stabilizeTree(trees[i])); - } - } - - if (stableTrees.length === 0) { - throw new Error('No input trees provided!'); - } - - return Object.freeze(stableTrees); - } - - - private stabilizeTree(tree: BroccoliTree) { - // Ignore all DiffingPlugins as they are already stable, for others we don't know for sure - // so we need to stabilize them. - // Since it's not safe to use instanceof operator in node, we are checking the constructor.name. - // - // New-style/rebuild trees should always be stable. - let isNewStyleTree = !!(tree['newStyleTree'] || typeof tree.rebuild === 'function' || - tree['isReadAPICompatTree'] || tree.constructor['name'] === 'Funnel'); - - return isNewStyleTree ? tree : stabilizeTree(tree); - } -} diff --git a/tools/broccoli/multi_copy.ts b/tools/broccoli/multi_copy.ts deleted file mode 100644 index 18573be43bf..00000000000 --- a/tools/broccoli/multi_copy.ts +++ /dev/null @@ -1,49 +0,0 @@ -/// -/// -/// - -import Writer = require('broccoli-writer'); -import fs = require('fs'); -import fsx = require('fs-extra'); -var minimatch = require('minimatch'); -var path = require('path'); -var glob = require('glob'); - -export interface MultiCopyOptions { - /** The path of the file to copy. */ - srcPath: string; - /** A list of glob patterns of folders to copy to, matched against the input tree. */ - targetPatterns: string[]; - /** List of glob patterns to *not* copy to, matched against the matches from `targetPatterns`. */ - exclude?: string[]; -} - -/** - * A writer that copies an input file from an input path into (potentially many) output locations - * given by glob patterns, . - */ -export class MultiCopy extends Writer { - constructor(private inputTree, private options: MultiCopyOptions) { super(); } - - write(readTree: (tree) => Promise, destDir: string): Promise { - return readTree(this.inputTree) - .then((inputPath: string) => { - var fileName = path.basename(this.options.srcPath); - var data = fs.readFileSync(path.join(inputPath, this.options.srcPath), 'utf-8'); - - this.options.targetPatterns.forEach(pattern => { - var paths: string[] = glob.sync(pattern); - paths = paths.filter(p => fs.statSync(p).isDirectory()); - if (this.options.exclude) { - paths = paths.filter(p => !this.options.exclude.some((excl) => minimatch(p, excl))); - } - paths.forEach(p => { - var folder = path.join(destDir, p); - fsx.mkdirsSync(folder); - var outputPath = path.join(folder, fileName); - fs.writeFileSync(outputPath, data); - }); - }); - }); - } -} diff --git a/tools/broccoli/tree-differ.spec.ts b/tools/broccoli/tree-differ.spec.ts deleted file mode 100644 index 0a2205ca522..00000000000 --- a/tools/broccoli/tree-differ.spec.ts +++ /dev/null @@ -1,378 +0,0 @@ -/// -/// - -let mockfs = require('mock-fs'); -import fs = require('fs'); -import path = require('path'); -import {TreeDiffer} from './tree-differ'; - - -describe('TreeDiffer', () => { - - afterEach(() => mockfs.restore()); - - - describe('diff of added and changed files', () => { - - it('should list all files (but no directories) during the first diff', () => { - let testDir = { - 'dir1': { - 'file-1.txt': mockfs.file({content: 'file-1.txt content', mtime: new Date(1000)}), - 'file-2.txt': mockfs.file({content: 'file-2.txt content', mtime: new Date(1000)}), - 'subdir-1': { - 'file-1.1.txt': mockfs.file({content: 'file-1.1.txt content', mtime: new Date(1000)}) - }, - 'empty-dir': {} - } - }; - mockfs(testDir); - - let differ = new TreeDiffer('testLabel', 'dir1'); - - let diffResult = differ.diffTree(); - - expect(diffResult.addedPaths) - .toEqual(['file-1.txt', 'file-2.txt', 'subdir-1' + path.sep + 'file-1.1.txt']); - - expect(diffResult.changedPaths).toEqual([]); - expect(diffResult.removedPaths).toEqual([]); - }); - - - it('should return empty diff if nothing has changed', () => { - let testDir = { - 'dir1': { - 'file-1.txt': mockfs.file({content: 'file-1.txt content', mtime: new Date(1000)}), - 'file-2.txt': mockfs.file({content: 'file-2.txt content', mtime: new Date(1000)}), - 'subdir-1': { - 'file-1.1.txt': mockfs.file({content: 'file-1.1.txt content', mtime: new Date(1000)}) - }, - } - }; - mockfs(testDir); - - let differ = new TreeDiffer('testLabel', 'dir1'); - - let diffResult = differ.diffTree(); - - expect(diffResult.addedPaths).not.toEqual([]); - expect(diffResult.changedPaths).toEqual([]); - expect(diffResult.removedPaths).toEqual([]); - - diffResult = differ.diffTree(); - - expect(diffResult.addedPaths).toEqual([]); - expect(diffResult.changedPaths).toEqual([]); - expect(diffResult.removedPaths).toEqual([]); - }); - - - it('should list only changed files during the subsequent diffs', () => { - let testDir = { - 'dir1': { - 'file-1.txt': mockfs.file({content: 'file-1.txt content', mtime: new Date(1000)}), - 'file-2.txt': mockfs.file({content: 'file-2.txt content', mtime: new Date(1000)}), - 'subdir-1': { - 'file-1.1.txt': - mockfs.file({content: 'file-1.1.txt content', mtime: new Date(1000)}) - } - } - }; - mockfs(testDir); - - let differ = new TreeDiffer('testLabel', 'dir1'); - - let diffResult = differ.diffTree(); - - expect(diffResult.addedPaths) - .toEqual(['file-1.txt', 'file-2.txt', 'subdir-1' + path.sep + 'file-1.1.txt']); - - // change two files - testDir['dir1']['file-1.txt'] = mockfs.file({content: 'new content', mtime: new Date(1000)}); - testDir['dir1']['subdir-1']['file-1.1.txt'] = - mockfs.file({content: 'file-1.1.txt content', mtime: new Date(9999)}); - mockfs(testDir); - - diffResult = differ.diffTree(); - - expect(diffResult.changedPaths) - .toEqual(['file-1.txt', 'subdir-1' + path.sep + 'file-1.1.txt']); - - expect(diffResult.removedPaths).toEqual([]); - - // change one file - testDir['dir1']['file-1.txt'] = mockfs.file({content: 'super new', mtime: new Date(1000)}); - mockfs(testDir); - - diffResult = differ.diffTree(); - expect(diffResult.changedPaths).toEqual(['file-1.txt']); - }); - - - it('should handle changes via symbolic links', () => { - let testDir = { - 'orig_path': { - 'file-1.txt': mockfs.file({content: 'file-1.txt content', mtime: new Date(1000)}), - 'file-2.txt': mockfs.file({content: 'file-2.txt content', mtime: new Date(1000)}), - 'subdir-1': { - 'file-1.1.txt': mockfs.file({content: 'file-1.1.txt content', mtime: new Date(1000)}) - } - }, - 'symlinks': { - 'file-1.txt': mockfs.symlink({path: '../orig_path/file-1.txt'}), - 'file-2.txt': mockfs.symlink({path: '../orig_path/file-2.txt'}), - 'subdir-1': - {'file-1.1.txt': mockfs.symlink({path: '../../orig_path/subdir-1/file-1.1.txt'})} - } - }; - mockfs(testDir); - - let differ = new TreeDiffer('testLabel', 'symlinks'); - - let diffResult = differ.diffTree(); - - expect(diffResult.addedPaths) - .toEqual(['file-1.txt', 'file-2.txt', 'subdir-1' + path.sep + 'file-1.1.txt']); - - // change two files - testDir['orig_path']['file-1.txt'] = - mockfs.file({content: 'new content', mtime: new Date(1000)}); - testDir['orig_path']['subdir-1']['file-1.1.txt'] = - mockfs.file({content: 'file-1.1.txt content', mtime: new Date(9999)}); - mockfs(testDir); - - diffResult = differ.diffTree(); - - expect(diffResult.addedPaths).toEqual([]); - expect(diffResult.changedPaths) - .toEqual(['file-1.txt', 'subdir-1' + path.sep + 'file-1.1.txt']); - expect(diffResult.removedPaths).toEqual([]); - - // change one file - testDir['orig_path']['file-1.txt'] = - mockfs.file({content: 'super new', mtime: new Date(1000)}); - mockfs(testDir); - - diffResult = differ.diffTree(); - expect(diffResult.changedPaths).toEqual(['file-1.txt']); - - // remove a link - delete testDir['orig_path']['file-1.txt']; - mockfs(testDir); - - diffResult = differ.diffTree(); - expect(diffResult.addedPaths).toEqual([]); - expect(diffResult.changedPaths).toEqual([]); - expect(diffResult.removedPaths).toEqual(['file-1.txt']); - - // don't report it as a removal twice - mockfs(testDir); - - diffResult = differ.diffTree(); - expect(diffResult.changedPaths).toEqual([]); - expect(diffResult.removedPaths).toEqual([]); - - // re-add it. - testDir['orig_path']['file-1.txt'] = - mockfs.file({content: 'super new', mtime: new Date(1000)}); - mockfs(testDir); - - diffResult = differ.diffTree(); - expect(diffResult.addedPaths).toEqual(['file-1.txt']); - expect(diffResult.changedPaths).toEqual([]); - expect(diffResult.removedPaths).toEqual([]); - }); - - - it("should throw an error if an extension isn't prefixed with doc", () => { - // includeExtensions - expect(() => new TreeDiffer('testLabel', 'dir1', ['js'])) - .toThrowError("Extension must begin with '.'. Was: 'js'"); - - // excludeExtentions - expect(() => new TreeDiffer('testLabel', 'dir1', [], ['js'])) - .toThrowError("Extension must begin with '.'. Was: 'js'"); - }); - - - it('should ignore files with extensions not listed in includeExtensions', () => { - let testDir = { - 'dir1': { - 'file-1.js': mockfs.file({content: 'file-1.js content', mtime: new Date(1000)}), - 'file-2.md': mockfs.file({content: 'file-2.md content', mtime: new Date(1000)}), - 'file-3.coffee': mockfs.file({content: 'file-3.coffee content', mtime: new Date(1000)}), - 'subdir-1': { - 'file-1.1.cc': mockfs.file({content: 'file-1.1.cc content', mtime: new Date(1000)}) - } - } - }; - mockfs(testDir); - - let differ = new TreeDiffer('testLabel', 'dir1', ['.js', '.coffee']); - - let diffResult = differ.diffTree(); - - expect(diffResult.addedPaths).toEqual(['file-1.js', 'file-3.coffee']); - expect(diffResult.changedPaths).toEqual([]); - expect(diffResult.removedPaths).toEqual([]); - - // change two files - testDir['dir1']['file-1.js'] = mockfs.file({content: 'new content', mtime: new Date(1000)}); - testDir['dir1']['file-3.coffee'] = - mockfs.file({content: 'new content', mtime: new Date(1000)}); - testDir['dir1']['subdir-1']['file-1.1.cc'] = - mockfs.file({content: 'file-1.1.cc content', mtime: new Date(9999)}); - mockfs(testDir); - - diffResult = differ.diffTree(); - - expect(diffResult.addedPaths).toEqual([]); - expect(diffResult.changedPaths).toEqual(['file-1.js', 'file-3.coffee']); - expect(diffResult.removedPaths).toEqual([]); - - // change one file - testDir['dir1']['file-1.js'] = mockfs.file({content: 'super new', mtime: new Date(1000)}); - mockfs(testDir); - - diffResult = differ.diffTree(); - expect(diffResult.changedPaths).toEqual(['file-1.js']); - }); - - - it('should ignore files with extensions listed in excludeExtensions', () => { - let testDir = { - 'dir1': { - 'file-1.ts': mockfs.file({content: 'file-1.ts content', mtime: new Date(1000)}), - 'file-1.cs': mockfs.file({content: 'file-1.cs content', mtime: new Date(1000)}), - 'file-1d.cs': mockfs.file({content: 'file-1d.cs content', mtime: new Date(1000)}), - 'file-1.d.cs': mockfs.file({content: 'file-1.d.cs content', mtime: new Date(1000)}), - 'file-2.md': mockfs.file({content: 'file-2.md content', mtime: new Date(1000)}), - 'file-3.ts': mockfs.file({content: 'file-3.ts content', mtime: new Date(1000)}), - 'file-4.d.ts': mockfs.file({content: 'file-4.d.ts content', mtime: new Date(1000)}), - 'subdir-1': { - 'file-1.1.cc': mockfs.file({content: 'file-1.1.cc content', mtime: new Date(1000)}) - } - } - }; - mockfs(testDir); - - let differ = new TreeDiffer('testLabel', 'dir1', ['.ts', '.cs'], ['.d.ts', '.d.cs']); - - let diffResult = differ.diffTree(); - - expect(diffResult.addedPaths).toEqual(['file-1.cs', 'file-1.ts', 'file-1d.cs', 'file-3.ts']); - - // change two files - testDir['dir1']['file-1.ts'] = mockfs.file({content: 'new content', mtime: new Date(1000)}); - testDir['dir1']['file-1.cs'] = mockfs.file({content: 'new content', mtime: new Date(1000)}); - testDir['dir1']['file-1.d.cs'] = mockfs.file({content: 'new content', mtime: new Date(1000)}); - testDir['dir1']['file-3.ts'] = mockfs.file({content: 'new content', mtime: new Date(1000)}); - testDir['dir1']['file-4.d.ts'] = mockfs.file({content: 'new content', mtime: new Date(1000)}); - testDir['dir1']['subdir-1']['file-1.1.cc'] = - mockfs.file({content: 'file-1.1.cc content', mtime: new Date(9999)}); - mockfs(testDir); - - diffResult = differ.diffTree(); - - expect(diffResult.addedPaths).toEqual([]); - expect(diffResult.changedPaths).toEqual(['file-1.cs', 'file-1.ts', 'file-3.ts']); - expect(diffResult.removedPaths).toEqual([]); - - // change one file - testDir['dir1']['file-4.d.ts'] = mockfs.file({content: 'super new', mtime: new Date(1000)}); - mockfs(testDir); - - diffResult = differ.diffTree(); - expect(diffResult.changedPaths).toEqual([]); - }); - }); - - describe('diff of new files', () => { - - it('should detect file additions', () => { - let testDir = { - 'dir1': - {'file-1.txt': mockfs.file({content: 'file-1.txt content', mtime: new Date(1000)})} - }; - mockfs(testDir); - - let differ = new TreeDiffer('testLabel', 'dir1'); - differ.diffTree(); - - testDir['dir1']['file-2.txt'] = 'new file'; - mockfs(testDir); - - let diffResult = differ.diffTree(); - expect(diffResult.addedPaths).toEqual(['file-2.txt']); - expect(diffResult.changedPaths).toEqual([]); - expect(diffResult.removedPaths).toEqual([]); - }); - - - it('should detect file additions mixed with file changes', () => { - let testDir = { - 'dir1': - {'file-1.txt': mockfs.file({content: 'file-1.txt content', mtime: new Date(1000)})} - }; - mockfs(testDir); - - let differ = new TreeDiffer('testLabel', 'dir1'); - differ.diffTree(); - - testDir['dir1']['file-1.txt'] = 'new content'; - testDir['dir1']['file-2.txt'] = 'new file'; - mockfs(testDir); - - let diffResult = differ.diffTree(); - expect(diffResult.addedPaths).toEqual(['file-2.txt']); - expect(diffResult.changedPaths).toEqual(['file-1.txt']); - }); - }); - - - describe('diff of removed files', () => { - - it('should detect file removals and report them as removed files', () => { - let testDir = { - 'dir1': - {'file-1.txt': mockfs.file({content: 'file-1.txt content', mtime: new Date(1000)})} - }; - mockfs(testDir); - - let differ = new TreeDiffer('testLabel', 'dir1'); - differ.diffTree(); - - delete testDir['dir1']['file-1.txt']; - mockfs(testDir); - - let diffResult = differ.diffTree(); - expect(diffResult.changedPaths).toEqual([]); - expect(diffResult.removedPaths).toEqual(['file-1.txt']); - }); - - - it('should detect file removals mixed with file changes and additions', () => { - let testDir = { - 'dir1': { - 'file-1.txt': mockfs.file({content: 'file-1.txt content', mtime: new Date(1000)}), - 'file-2.txt': mockfs.file({content: 'file-1.txt content', mtime: new Date(1000)}) - } - }; - - mockfs(testDir); - - let differ = new TreeDiffer('testLabel', 'dir1'); - differ.diffTree(); - - testDir['dir1']['file-1.txt'] = 'changed content'; - delete testDir['dir1']['file-2.txt']; - testDir['dir1']['file-3.txt'] = 'new content'; - mockfs(testDir); - - let diffResult = differ.diffTree(); - expect(diffResult.addedPaths).toEqual(['file-3.txt']); - expect(diffResult.changedPaths).toEqual(['file-1.txt']); - expect(diffResult.removedPaths).toEqual(['file-2.txt']); - }); - }); -}); diff --git a/tools/broccoli/tree-differ.ts b/tools/broccoli/tree-differ.ts deleted file mode 100644 index af3000670fd..00000000000 --- a/tools/broccoli/tree-differ.ts +++ /dev/null @@ -1,176 +0,0 @@ -/// - -import fs = require('fs'); -import path = require('path'); - - -function tryStatSync(path) { - try { - return fs.statSync(path); - } catch (e) { - if (e.code === "ENOENT") return null; - throw e; - } -} - - -export class TreeDiffer { - private fingerprints: {[key: string]: string} = Object.create(null); - private nextFingerprints: {[key: string]: string} = Object.create(null); - private rootDirName: string; - private include: RegExp = null; - private exclude: RegExp = null; - - constructor(private label: string, private rootPath: string, includeExtensions?: string[], - excludeExtensions?: string[]) { - this.rootDirName = path.basename(rootPath); - - let buildRegexp = (arr) => new RegExp(`(${arr.reduce(combine, "")})$`, "i"); - - this.include = (includeExtensions || []).length ? buildRegexp(includeExtensions) : null; - this.exclude = (excludeExtensions || []).length ? buildRegexp(excludeExtensions) : null; - - function combine(prev, curr) { - if (curr.charAt(0) !== ".") { - throw new Error(`Extension must begin with '.'. Was: '${curr}'`); - } - let kSpecialRegexpChars = /[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g; - curr = '(' + curr.replace(kSpecialRegexpChars, '\\$&') + ')'; - return prev ? (prev + '|' + curr) : curr; - } - } - - - public diffTree(): DiffResult { - let result = new DirtyCheckingDiffResult(this.label, this.rootDirName); - this.dirtyCheckPath(this.rootPath, result); - this.detectDeletionsAndUpdateFingerprints(result); - result.endTime = Date.now(); - return result; - } - - - private dirtyCheckPath(rootDir: string, result: DirtyCheckingDiffResult) { - fs.readdirSync(rootDir).forEach((segment) => { - let absolutePath = path.join(rootDir, segment); - let pathStat = fs.lstatSync(absolutePath); - if (pathStat.isSymbolicLink()) { - pathStat = tryStatSync(absolutePath); - if (pathStat === null) return; - } - - if (pathStat.isDirectory()) { - result.directoriesChecked++; - this.dirtyCheckPath(absolutePath, result); - } else { - if (!(this.include && !absolutePath.match(this.include)) && - !(this.exclude && absolutePath.match(this.exclude))) { - result.filesChecked++; - let relativeFilePath = path.relative(this.rootPath, absolutePath); - - switch (this.isFileDirty(absolutePath, pathStat)) { - case FileStatus.Added: - result.addedPaths.push(relativeFilePath); - break; - case FileStatus.Changed: - result.changedPaths.push(relativeFilePath); - } - } - } - }); - - return result; - } - - - private isFileDirty(path: string, stat: fs.Stats): FileStatus { - let oldFingerprint = this.fingerprints[path]; - let newFingerprint = `${stat.mtime.getTime()} # ${stat.size}`; - - this.nextFingerprints[path] = newFingerprint; - - if (oldFingerprint) { - this.fingerprints[path] = null; - - if (oldFingerprint === newFingerprint) { - // nothing changed - return FileStatus.Unchanged; - } - - return FileStatus.Changed; - } - - return FileStatus.Added; - } - - - private detectDeletionsAndUpdateFingerprints(result: DiffResult) { - for (let absolutePath in this.fingerprints) { - if (!(this.include && !absolutePath.match(this.include)) && - !(this.exclude && absolutePath.match(this.exclude))) { - if (this.fingerprints[absolutePath] !== null) { - let relativePath = path.relative(this.rootPath, absolutePath); - result.removedPaths.push(relativePath); - } - } - } - - this.fingerprints = this.nextFingerprints; - this.nextFingerprints = Object.create(null); - } -} - - -export class DiffResult { - public addedPaths: string[] = []; - public changedPaths: string[] = []; - public removedPaths: string[] = []; - - constructor(public label: string = '') {} - - log(verbose: boolean): void {} - - toString(): string { - // TODO(@caitp): more meaningful logging - return ''; - } -} - -class DirtyCheckingDiffResult extends DiffResult { - public filesChecked: number = 0; - public directoriesChecked: number = 0; - public startTime: number = Date.now(); - public endTime: number = null; - - constructor(label: string, public directoryName: string) { super(label); } - - toString() { - return `${pad(this.label, 30)}, ${pad(this.endTime - this.startTime, 5)}ms, ` + - `${pad(this.addedPaths.length + this.changedPaths.length + this.removedPaths.length, 5)} changes ` + - `(files: ${pad(this.filesChecked, 5)}, dirs: ${pad(this.directoriesChecked, 4)})`; - } - - log(verbose) { - let prefixedPaths = this.addedPaths.map(p => `+ ${p}`) - .concat(this.changedPaths.map(p => `* ${p}`)) - .concat(this.removedPaths.map(p => `- ${p}`)); - console.log(`Tree diff: ${this}` + ((verbose && prefixedPaths.length) ? - ` [\n ${prefixedPaths.join('\n ')}\n]` : - '')); - } -} - - -function pad(value, length) { - value = '' + value; - let whitespaceLength = (value.length < length) ? length - value.length : 0; - whitespaceLength = whitespaceLength + 1; - return new Array(whitespaceLength).join(' ') + value; -} - - -enum FileStatus { - Added, - Unchanged, - Changed -}