build: generate api docs for cdk using bazel (#14427)

src/BUILD.bazel

@@ -1,6 +1,8 @@
 package(default_visibility = ["//visibility:public"])
 
+load("//:packages.bzl", "CDK_PACKAGES")
 load("//tools:defaults.bzl", "ts_library")
+load("//tools/dgeni:index.bzl", "dgeni_api_docs")
 
 exports_files([
   "bazel-tsconfig-build.json",
@@ -12,3 +14,16 @@ ts_library(
   name = 'module-typings',
   srcs = [":module-typings.d.ts"]
 )
+
+dgeni_api_docs(
+  name = "api-docs",
+  srcs = ["//src/cdk/%s:source-files" % name for name in CDK_PACKAGES] + [
+    # Add all Angular packages to the sources because some Material exports use
+    # Angular exports and these should not cause any warnings when Dgeni uses the
+    # type checker to parse our TypeScript sources.
+    "@matdeps//@angular"
+  ],
+  entry_points = {
+    "cdk": CDK_PACKAGES,
+  },
+)

src/cdk/BUILD.bazel

@@ -1,6 +1,6 @@
 package(default_visibility=["//visibility:public"])
 
-load("//:packages.bzl", "CDK_TARGETS", "ROLLUP_GLOBALS")
+load("//:packages.bzl", "CDK_TARGETS", "CDK_PACKAGES", "ROLLUP_GLOBALS")
 load("//tools:defaults.bzl", "ng_module", "ng_package")
 
 # Root "@angular/cdk" entry-point that does not re-export individual entry-points.
@@ -13,6 +13,11 @@ ng_module(
   ],
 )
 
+filegroup(
+  name = "overviews",
+  srcs = ["//src/cdk/%s:overview" % name for name in CDK_PACKAGES]
+)
+
 # Creates the @angular/cdk package published to npm.
 ng_package(
   name = "npm_package",

src/cdk/a11y/BUILD.bazel

@@ -55,3 +55,8 @@ markdown_to_html(
   name = "overview",
   srcs = [":a11y.md"],
 )
+
+filegroup(
+  name = "source-files",
+  srcs = glob(["**/*.ts"]),
+)

src/cdk/accordion/BUILD.bazel

@@ -34,3 +34,9 @@ markdown_to_html(
   name = "overview",
   srcs = [],
 )
+
+filegroup(
+  name = "source-files",
+  srcs = glob(["**/*.ts"]),
+)
+

src/cdk/bidi/BUILD.bazel

@@ -31,3 +31,8 @@ markdown_to_html(
   name = "overview",
   srcs = [":bidi.md"],
 )
+
+filegroup(
+  name = "source-files",
+  srcs = glob(["**/*.ts"]),
+)

src/cdk/coercion/BUILD.bazel

@@ -27,3 +27,8 @@ markdown_to_html(
   name = "overview",
   srcs = [":coercion.md"],
 )
+
+filegroup(
+  name = "source-files",
+  srcs = glob(["**/*.ts"]),
+)

src/cdk/collections/BUILD.bazel

@@ -31,3 +31,8 @@ markdown_to_html(
   name = "overview",
   srcs = [":collections.md"],
 )
+
+filegroup(
+  name = "source-files",
+  srcs = glob(["**/*.ts"]),
+)

src/cdk/drag-drop/BUILD.bazel

@@ -38,3 +38,8 @@ markdown_to_html(
   name = "overview",
   srcs = [":drag-drop.md"],
 )
+
+filegroup(
+  name = "source-files",
+  srcs = glob(["**/*.ts"]),
+)

src/cdk/keycodes/BUILD.bazel

@@ -28,3 +28,8 @@ markdown_to_html(
   name = "overview",
   srcs = [":keycodes.md"],
 )
+
+filegroup(
+  name = "source-files",
+  srcs = glob(["**/*.ts"]),
+)

src/cdk/layout/BUILD.bazel

@@ -34,3 +34,8 @@ markdown_to_html(
   name = "overview",
   srcs = [":layout.md"],
 )
+
+filegroup(
+  name = "source-files",
+  srcs = glob(["**/*.ts"]),
+)

src/cdk/observers/BUILD.bazel

@@ -30,3 +30,8 @@ markdown_to_html(
   name = "overview",
   srcs = [":observers.md"],
 )
+
+filegroup(
+  name = "source-files",
+  srcs = glob(["**/*.ts"]),
+)

src/cdk/overlay/BUILD.bazel

@@ -63,3 +63,7 @@ markdown_to_html(
   srcs = [":overlay.md"],
 )
 
+filegroup(
+  name = "source-files",
+  srcs = glob(["**/*.ts"]),
+)

src/cdk/platform/BUILD.bazel

@@ -16,3 +16,8 @@ markdown_to_html(
   name = "overview",
   srcs = [":platform.md"],
 )
+
+filegroup(
+  name = "source-files",
+  srcs = glob(["**/*.ts"]),
+)

src/cdk/portal/BUILD.bazel

@@ -31,3 +31,8 @@ markdown_to_html(
   srcs = [":portal.md"],
 )
 
+filegroup(
+  name = "source-files",
+  srcs = glob(["**/*.ts"]),
+)
+

src/cdk/scrolling/BUILD.bazel

@@ -46,3 +46,8 @@ markdown_to_html(
   name = "overview",
   srcs = [":scrolling.md"],
 )
+
+filegroup(
+  name = "source-files",
+  srcs = glob(["**/*.ts"]),
+)

src/cdk/stepper/BUILD.bazel

@@ -23,3 +23,8 @@ markdown_to_html(
   name = "overview",
   srcs = [":stepper.md"],
 )
+
+filegroup(
+  name = "source-files",
+  srcs = glob(["**/*.ts"]),
+)

src/cdk/table/BUILD.bazel

@@ -40,3 +40,8 @@ markdown_to_html(
   name = "overview",
   srcs = [":table.md"],
 )
+
+filegroup(
+  name = "source-files",
+  srcs = glob(["**/*.ts"]),
+)

src/cdk/text-field/BUILD.bazel

@@ -52,3 +52,8 @@ markdown_to_html(
   name = "overview",
   srcs = [":text-field.md"],
 )
+
+filegroup(
+  name = "source-files",
+  srcs = glob(["**/*.ts"]),
+)

src/cdk/tree/BUILD.bazel

@@ -39,3 +39,8 @@ markdown_to_html(
   name = "overview",
   srcs = [":tree.md"],
 )
+
+filegroup(
+  name = "source-files",
+  srcs = glob(["**/*.ts"]),
+)

src/lib/BUILD.bazel

@@ -24,6 +24,11 @@ sass_bundle(
   output_name = "_theming.scss",
 )
 
+filegroup(
+  name = "overviews",
+  srcs = ["//src/lib/%s:overview" % name for name in MATERIAL_PACKAGES]
+)
+
 # Creates the @angular/material package published to npm.
 ng_package(
   name = "npm_package",

src/material-examples/BUILD.bazel

@@ -1,7 +1,6 @@
 package(default_visibility=["//visibility:public"])
 
-load("//:packages.bzl", "CDK_TARGETS", "MATERIAL_TARGETS", "ROLLUP_GLOBALS", "MATERIAL_PACKAGES",
-  "CDK_PACKAGES")
+load("//:packages.bzl", "CDK_TARGETS", "MATERIAL_TARGETS", "ROLLUP_GLOBALS")
 load("//tools:defaults.bzl", "ng_module", "ng_package")
 load("//tools/highlight-files:index.bzl", "highlight_files")
 load("//tools/package-docs-content:index.bzl", "package_docs_content")
@@ -28,16 +27,6 @@ filegroup(
   srcs = glob(["*/*.html", "*/*.css", "*/*.ts"])
 )
 
-filegroup(
-  name = "material-overviews",
-  srcs = ["//src/lib/%s:overview" % name for name in MATERIAL_PACKAGES]
-)
-
-filegroup(
-  name = "cdk-overviews",
-  srcs = ["//src/cdk/%s:overview" % name for name in CDK_PACKAGES]
-)
-
 highlight_files(
   name = "highlighted-source-files",
   srcs = [":example-source-files"]
@@ -59,8 +48,8 @@ package_docs_content(
     ":example-source-files": "examples-source",
 
     # Package the overviews for "@angular/material" and "@angular/cdk" into the docs content
-    ":material-overviews": "overviews/material",
-    ":cdk-overviews": "overviews/cdk",
+    "//src/lib:overviews": "overviews/material",
+    "//src/cdk:overviews": "overviews/cdk",
 
     # TODO(devversion): we need to also package the API html files here
   }

tools/dgeni/BUILD.bazel

@@ -0,0 +1,25 @@
+package(default_visibility = ["//visibility:public"])
+
+load("@build_bazel_rules_nodejs//:defs.bzl", "nodejs_binary")
+load("//tools:defaults.bzl", "ts_library")
+
+nodejs_binary(
+  name = "dgeni",
+  entry_point = "angular_material/tools/dgeni/bazel-bin.js",
+  data = [
+    "@matdeps//source-map-support",
+    "@matdeps//dgeni",
+    "@matdeps//dgeni-packages",
+    ":sources",
+  ],
+)
+
+ts_library(
+  name = "sources",
+  srcs = glob(["**/*.ts"]),
+  deps = [
+    "@matdeps//@types/node",
+    "//tools/highlight-files:sources",
+  ],
+  tsconfig = ":tsconfig.json",
+)

tools/dgeni/bazel-bin.ts

@@ -0,0 +1,106 @@
+import {Dgeni} from 'dgeni';
+import {ReadTypeScriptModules} from 'dgeni-packages/typescript/processors/readTypeScriptModules';
+import {TsParser} from 'dgeni-packages/typescript/services/TsParser';
+import {readFileSync} from 'fs';
+import {join, relative} from 'path';
+import {apiDocsPackage} from './index';
+
+/**
+ * Determines the command line arguments for the current Bazel action. Since this action can
+ * have a large set of input files, Bazel may write the arguments into a parameter file.
+ * This function is responsible for handling normal argument passing or Bazel parameter files.
+ * Read more here: https://docs.bazel.build/versions/master/skylark/lib/Args.html#use_param_file
+ */
+function getBazelActionArguments() {
+  const args = process.argv.slice(2);
+
+  // If Bazel uses a parameter file, we've specified that it passes the file in the following
+  // format: "arg0 arg1 --param-file={path_to_param_file}"
+  if (args[0].startsWith('--param-file=')) {
+    return readFileSync(args[0].split('=')[1], 'utf8').trim().split('\n');
+  }
+
+  return args;
+}
+
+if (require.main === module) {
+  const [
+    // Path that refers to the package where the current Bazel target is defined.
+    bazelLabelPackagePath,
+    // Path that is relative to the execroot and is the output directory for the docs.
+    outputDirPath,
+    // Remaining arguments that will be used to compute the entry points that need to be parsed.
+    ...entryPointArgs
+  ] = getBazelActionArguments();
+
+  const execRootPath = process.cwd();
+  const packagePath = join(execRootPath, bazelLabelPackagePath);
+
+  // Configure the Dgeni docs package to respect our passed options from the Bazel rule.
+  apiDocsPackage.config((readTypeScriptModules: ReadTypeScriptModules,
+                         tsParser: TsParser,
+                         templateFinder: any,
+                         writeFilesProcessor: any,
+                         readFilesProcessor: any) => {
+
+    // Set the base path for the "readFilesProcessor" to the execroot. This is necessary because
+    // otherwise the "writeFilesProcessor" is not able to write to the specified output path.
+    readFilesProcessor.basePath = execRootPath;
+
+    // Set the base path for parsing the TypeScript source files to the directory that includes
+    // all sources (also known as the path to the current Bazel target). This makes it easier for
+    // custom processors (such as the `entry-point-grouper) to compute entry-point paths.
+    readTypeScriptModules.basePath = packagePath;
+
+    // For each package we want to setup all entry points in Dgeni so that their API
+    // will be generated. Packages and their associated entry points are passed in pairs.
+    // The first argument will be always the package name, and the second argument will be a
+    // joined string containing names of all entry points for that specific package.
+    // e.g. "cdk" "platform,bidi,a11y"
+    for (let i = 0; i + 1 < entryPointArgs.length; i += 2) {
+      const packageName = entryPointArgs[i];
+      const entryPoints = entryPointArgs[i + 1].split(',');
+
+      // Walk through each entry point of the current package and add it to the
+      // "readTypeScriptModules" processor so that it will parse it. Additionally we want
+      // to setup path mapping for that entry-point, so that we are able to merge
+      // inherited class members across entry points or packages.
+      entryPoints.forEach(entryPointName => {
+        const entryPointPath = `${packageName}/${entryPointName}`;
+        const entryPointIndexPath = `${entryPointPath}/index.ts`;
+
+        tsParser.options.paths![`@angular/${entryPointPath}`] = [entryPointIndexPath];
+        readTypeScriptModules.sourceFiles.push(entryPointIndexPath);
+      });
+    }
+
+    // Base URL for the `tsParser`. The base URL refer to the directory that includes all
+    // package sources that need to be processed by Dgeni.
+    tsParser.options.baseUrl = packagePath;
+
+    // This is ensures that the Dgeni TypeScript processor is able to parse node modules such
+    // as the Angular packages which might be needed for doc items. e.g. if a class implements
+    // the "AfterViewInit" interface from "@angular/core". This needs to be relative to the
+    // "baseUrl" that has been specified for the "tsParser" compiler options.
+    tsParser.options.paths!['*'] = [relative(packagePath, 'external/matdeps/node_modules/*')];
+
+    // Since our base directory is the Bazel execroot, we need to make sure that Dgeni can
+    // find all templates needed to output the API docs.
+    templateFinder.templateFolders = [join(execRootPath, 'tools/dgeni/templates/')];
+
+    // The output path for files will be computed by joining the output folder with the base path
+    // from the "readFilesProcessors". Since the base path is the execroot, we can just use
+    // the output path passed from Bazel (e.g. $EXECROOT/bazel-out/bin/src/docs-content)
+    writeFilesProcessor.outputFolder = outputDirPath;
+  });
+
+  // Run the docs generation. The process will be automatically kept alive until Dgeni
+  // completed. In case the returned promise has been rejected, we need to manually exit the
+  // process with the proper exit code because Dgeni doesn't use native promises which would
+  // automatically cause the error to propagate. The error message will be automatically
+  // printed internally by Dgeni (so we don't want to repeat here)
+  new Dgeni([apiDocsPackage]).generate().catch(() => process.exit(1));
+}
+
+
+

tools/dgeni/index.bzl

@@ -0,0 +1,81 @@
+"""
+  Implementation of the "_dgeni_api_docs" rule. The implementation runs Dgeni with the
+  specified entry points and outputs the API docs into a package relative directory.
+"""
+def _dgeni_api_docs(ctx):
+  input_files = ctx.files.srcs;
+  args = ctx.actions.args()
+  expected_outputs = [];
+
+  output_directory = ctx.actions.declare_directory(ctx.attr.name)
+
+  # Do nothing if there are no input files. Bazel will throw if we schedule an action
+  # that returns no outputs.
+  if not input_files:
+    return None
+
+  # Support passing arguments through a parameter file. This is necessary because on Windows
+  # there is an argument limit and we need to handle a large amount of input files. Bazel
+  # switches between parameter file and normal argument passing based on the operating system.
+  # Read more here: https://docs.bazel.build/versions/master/skylark/lib/Args.html#use_param_file
+  args.use_param_file(param_file_arg = "--param-file=%s")
+
+  # Pass the path to the package that contains the current Bazel target that is being built.
+  # This will be used as a base path to resolve TypeScript source files within Dgeni.
+  args.add(ctx.label.package)
+
+  # Pass the path to the output directory. This will be used to instruct Dgeni where the docs
+  # output should be written to. (e.g. bazel-out/bin/src/docs-content)
+  args.add(output_directory.path)
+
+  # Pass each specified entry point and it's corresponding entry points to Dgeni. This will then
+  # be used to resolve the files that need to be parsed by Dgeni.
+  for package_name, entry_points in ctx.attr.entry_points.items():
+    args.add(package_name)
+    args.add_joined(entry_points, join_with = ",")
+
+  # Run the Dgeni bazel executable which builds the documentation output based on the
+  # configured rule attributes.
+  ctx.actions.run(
+    # Note that we want to add the dgeni template files as well. This makes sure that the
+    # templates are available in the sandbox execution and can be read by Dgeni.
+    inputs = input_files + ctx.files._dgeni_templates,
+    executable = ctx.executable._dgeni_bin,
+    outputs = [output_directory],
+    arguments = [args],
+  )
+
+  # TODO(devversion): We can construct a list of output files that will be generated. This would
+  # improve hermeticity and Bazel's caching mechanism for this rule.
+  return DefaultInfo(files = depset([output_directory]))
+
+"""
+  Rule definition for the "dgeni_api_docs" rule that can generate API documentation
+  for specified packages and their entry points.
+"""
+dgeni_api_docs = rule(
+  implementation = _dgeni_api_docs,
+  attrs = {
+    # List of labels that need to be available when Dgeni processes the entry points.
+    # This usually contains the source files and Angular packages which will be read
+    # by the dgeni-packages typeScript processor.
+    "srcs": attr.label_list(allow_files = True),
+
+    # String dictionary that defines a package and its entry points. e.g.
+    # { "cdk": ["a11y", "platform", "bidi"] }.
+    "entry_points": attr.string_list_dict(mandatory = True),
+
+    # Dgeni document templates that should be be available as inputs to the
+    # Bazel action. Dgeni tries to resolve templates from the execroot, so they
+    # need to be available in the sandbox.
+    "_dgeni_templates": attr.label(
+      default = Label("//tools/dgeni/templates"),
+    ),
+
+    # NodeJS binary target that runs Dgeni and parses the passed command arguments.
+    "_dgeni_bin": attr.label(
+      default = Label("//tools/dgeni"),
+      executable = True,
+      cfg = "host"
+  )},
+)

tools/dgeni/nunjucks-tags/highlight.ts

@@ -1,4 +1,4 @@
-const hljs = require('highlight.js');
+import {highlightCodeBlock} from '../../highlight-files/highlight-code-block';
 
 /**
  * Nunjucks extension that supports rendering highlighted content. Content that is placed in
@@ -32,6 +32,6 @@ export class HighlightNunjucksExtension {
   }
 
   render(_context: any, language: string, contentFn: () => string) {
-    return hljs.highlight(language, contentFn()).value;
+    return highlightCodeBlock(contentFn(), language);
   }
 }

tools/dgeni/templates/BUILD.bazel

@@ -0,0 +1,6 @@
+package(default_visibility = ["//visibility:public"])
+
+filegroup(
+  name = "templates",
+  srcs = glob(["*.html"])
+)

tools/dgeni/tsconfig.json

@@ -18,5 +18,8 @@
   },
   "files": [
     "index.ts"
-  ]
+  ],
+  "bazelOptions": {
+    "suppressTsconfigOverrideWarnings": true
+  }
 }

tools/highlight-files/BUILD.bazel

@@ -3,19 +3,21 @@ package(default_visibility = ["//visibility:public"])
 load("@build_bazel_rules_nodejs//:defs.bzl", "nodejs_binary")
 load("//tools:defaults.bzl", "ts_library")
 
+ts_library(
+  name = "sources",
+  srcs = glob(["**/*.ts"]),
+  deps = [
+    "@matdeps//@types/node",
+    "@matdeps//highlight.js",
+  ],
+  tsconfig = ":tsconfig.json",
+)
+
 nodejs_binary(
   name = "highlight-files",
   entry_point = "angular_material/tools/highlight-files/highlight-files.js",
   data = [
-    "@matdeps//highlight.js",
     "@matdeps//source-map-support",
     ":sources",
   ],
 )
-
-ts_library(
-  name = "sources",
-  srcs = glob(["**/*.ts"]),
-  deps = ["@matdeps//@types/node"],
-  tsconfig = ":tsconfig.json",
-)

tools/markdown-to-html/BUILD.bazel

@@ -9,6 +9,7 @@ ts_library(
   deps = [
     "@matdeps//@types/node",
     "@matdeps//@types/marked",
+    "@matdeps//marked",
     "//tools/highlight-files:sources",
   ],
   tsconfig = ":tsconfig.json",
@@ -18,8 +19,6 @@ nodejs_binary(
   name = "markdown-to-html",
   entry_point = "angular_material/tools/markdown-to-html/transform-markdown.js",
   data = [
-    "@matdeps//highlight.js",
-    "@matdeps//marked",
     "@matdeps//source-map-support",
     ":transform-markdown",
   ],