diff --git a/packages/compiler/src/ml_parser/lexer.ts b/packages/compiler/src/ml_parser/lexer.ts
index be34e8efa32..ce170b11961 100644
--- a/packages/compiler/src/ml_parser/lexer.ts
+++ b/packages/compiler/src/ml_parser/lexer.ts
@@ -89,6 +89,12 @@ export interface TokenizeOptions {
* If true, do not convert CRLF to LF.
*/
preserveLineEndings?: boolean;
+
+ // TODO(crisbeto): temporary option to limit access to the block syntax.
+ /**
+ * Whether the block syntax is enabled at the compiler level.
+ */
+ tokenizeBlocks?: boolean;
}
export function tokenize(
@@ -136,8 +142,8 @@ class _Tokenizer {
private _expansionCaseStack: TokenType[] = [];
private _inInterpolation: boolean = false;
private readonly _preserveLineEndings: boolean;
- private readonly _escapedString: boolean;
private readonly _i18nNormalizeLineEndingsInICUs: boolean;
+ private readonly _tokenizeBlocks: boolean;
tokens: Token[] = [];
errors: TokenError[] = [];
nonNormalizedIcuExpressions: Token[] = [];
@@ -159,8 +165,8 @@ class _Tokenizer {
this._cursor = options.escapedString ? new EscapedCharacterCursor(_file, range) :
new PlainCharacterCursor(_file, range);
this._preserveLineEndings = options.preserveLineEndings || false;
- this._escapedString = options.escapedString || false;
this._i18nNormalizeLineEndingsInICUs = options.i18nNormalizeLineEndingsInICUs || false;
+ this._tokenizeBlocks = options.tokenizeBlocks || false;
try {
this._cursor.init();
} catch (e) {
@@ -197,6 +203,12 @@ class _Tokenizer {
} else {
this._consumeTagOpen(start);
}
+ } else if (this._tokenizeBlocks && this._attemptStr('{#')) {
+ this._consumeBlockGroupOpen(start);
+ } else if (this._tokenizeBlocks && this._attemptStr('{/')) {
+ this._consumeBlockGroupClose(start);
+ } else if (this._tokenizeBlocks && this._attemptStr('{:')) {
+ this._consumeBlock(start);
} else if (!(this._tokenizeIcu && this._tokenizeExpansionForm())) {
// In (possibly interpolated) text the end of the text is given by `isTextEnd()`, while
// the premature end of an interpolation is given by the start of a new HTML element.
@@ -212,6 +224,72 @@ class _Tokenizer {
this._endToken([]);
}
+ private _consumeBlockGroupOpen(start: CharacterCursor) {
+ this._beginToken(TokenType.BLOCK_GROUP_OPEN_START, start);
+ const nameCursor = this._cursor.clone();
+ this._attemptCharCodeUntilFn(code => !isBlockNameChar(code));
+ this._endToken([this._cursor.getChars(nameCursor)]);
+ this._consumeBlockParameters();
+ this._beginToken(TokenType.BLOCK_GROUP_OPEN_END);
+ this._requireCharCode(chars.$RBRACE);
+ this._endToken([]);
+ }
+
+ private _consumeBlockGroupClose(start: CharacterCursor) {
+ this._beginToken(TokenType.BLOCK_GROUP_CLOSE, start);
+ const nameCursor = this._cursor.clone();
+ this._attemptCharCodeUntilFn(code => !isBlockNameChar(code));
+ const name = this._cursor.getChars(nameCursor);
+ this._requireCharCode(chars.$RBRACE);
+ this._endToken([name]);
+ }
+
+ private _consumeBlock(start: CharacterCursor) {
+ this._beginToken(TokenType.BLOCK_OPEN_START, start);
+ const nameCursor = this._cursor.clone();
+ this._attemptCharCodeUntilFn(code => !isBlockNameChar(code));
+ this._endToken([this._cursor.getChars(nameCursor)]);
+ this._consumeBlockParameters();
+ this._beginToken(TokenType.BLOCK_OPEN_END);
+ this._requireCharCode(chars.$RBRACE);
+ this._endToken([]);
+ }
+
+ private _consumeBlockParameters() {
+ // Trim the whitespace until the first parameter.
+ this._attemptCharCodeUntilFn(isBlockParameterChar);
+
+ while (this._cursor.peek() !== chars.$RBRACE && this._cursor.peek() !== chars.$EOF) {
+ this._beginToken(TokenType.BLOCK_PARAMETER);
+ const start = this._cursor.clone();
+ let inQuote: number|null = null;
+
+ // Consume the parameter until the next semicolon or brace.
+ // Note that we skip over semicolons/braces inside of strings.
+ while ((this._cursor.peek() !== chars.$SEMICOLON && this._cursor.peek() !== chars.$RBRACE &&
+ this._cursor.peek() !== chars.$EOF) ||
+ inQuote !== null) {
+ const char = this._cursor.peek();
+
+ // Skip to the next character if it was escaped.
+ if (char === chars.$BACKSLASH) {
+ this._cursor.advance();
+ } else if (char === inQuote) {
+ inQuote = null;
+ } else if (inQuote === null && chars.isQuote(char)) {
+ inQuote = char;
+ }
+
+ this._cursor.advance();
+ }
+
+ this._endToken([this._cursor.getChars(start)]);
+
+ // Skip to the next parameter.
+ this._attemptCharCodeUntilFn(isBlockParameterChar);
+ }
+ }
+
/**
* @returns whether an ICU token has been created
* @internal
@@ -578,7 +656,6 @@ class _Tokenizer {
}
private _consumeAttributeValue() {
- let value: string;
if (this._cursor.peek() === chars.$SQ || this._cursor.peek() === chars.$DQ) {
const quoteChar = this._cursor.peek();
this._consumeQuote(quoteChar);
@@ -795,7 +872,7 @@ class _Tokenizer {
}
private _isTextEnd(): boolean {
- if (this._isTagStart() || this._cursor.peek() === chars.$EOF) {
+ if (this._isTagStart() || this._isBlockStart() || this._cursor.peek() === chars.$EOF) {
return true;
}
@@ -833,6 +910,26 @@ class _Tokenizer {
return false;
}
+ private _isBlockStart(): boolean {
+ if (this._tokenizeBlocks && this._cursor.peek() === chars.$LBRACE) {
+ const tmp = this._cursor.clone();
+
+ // Check that the cursor is on a `{#`, `{/` or `{:`.
+ tmp.advance();
+ const next = tmp.peek();
+ if (next !== chars.$BANG && next !== chars.$SLASH && next !== chars.$COLON) {
+ return false;
+ }
+
+ // If it is, also verify that the next character is a valid block identifier.
+ tmp.advance();
+ if (isBlockNameChar(tmp.peek())) {
+ return true;
+ }
+ }
+ return false;
+ }
+
private _readUntil(char: number): string {
const start = this._cursor.clone();
this._attemptUntilChar(char);
@@ -900,6 +997,14 @@ function toUpperCaseCharCode(code: number): number {
return code >= chars.$a && code <= chars.$z ? code - chars.$a + chars.$A : code;
}
+function isBlockNameChar(code: number): boolean {
+ return chars.isAsciiLetter(code) || chars.isDigit(code) || code === chars.$_;
+}
+
+function isBlockParameterChar(code: number): boolean {
+ return code !== chars.$SEMICOLON && isNotWhitespace(code);
+}
+
function mergeTextTokens(srcTokens: Token[]): Token[] {
const dstTokens: Token[] = [];
let lastDstToken: Token|undefined = undefined;
diff --git a/packages/compiler/src/ml_parser/tokens.ts b/packages/compiler/src/ml_parser/tokens.ts
index 643dc250031..43c9fb9ae3e 100644
--- a/packages/compiler/src/ml_parser/tokens.ts
+++ b/packages/compiler/src/ml_parser/tokens.ts
@@ -33,7 +33,13 @@ export const enum TokenType {
EXPANSION_CASE_EXP_START,
EXPANSION_CASE_EXP_END,
EXPANSION_FORM_END,
- EOF
+ EOF,
+ BLOCK_GROUP_OPEN_START,
+ BLOCK_GROUP_OPEN_END,
+ BLOCK_GROUP_CLOSE,
+ BLOCK_PARAMETER,
+ BLOCK_OPEN_START,
+ BLOCK_OPEN_END,
}
export type Token = TagOpenStartToken|TagOpenEndToken|TagOpenEndVoidToken|TagCloseToken|
@@ -41,7 +47,8 @@ export type Token = TagOpenStartToken|TagOpenEndToken|TagOpenEndVoidToken|TagClo
CommentEndToken|CdataStartToken|CdataEndToken|AttributeNameToken|AttributeQuoteToken|
AttributeValueTextToken|AttributeValueInterpolationToken|DocTypeToken|ExpansionFormStartToken|
ExpansionCaseValueToken|ExpansionCaseExpressionStartToken|ExpansionCaseExpressionEndToken|
- ExpansionFormEndToken|EndOfFileToken;
+ ExpansionFormEndToken|EndOfFileToken|BlockGroupOpenStartToken|BlockGroupOpenEndToken|
+ BlockGroupCloseToken|BlockParameterToken|BlockOpenStartToken|BlockOpenEndToken;
export type InterpolatedTextToken = TextToken|InterpolationToken|EncodedEntityToken;
@@ -170,3 +177,33 @@ export interface EndOfFileToken extends TokenBase {
type: TokenType.EOF;
parts: [];
}
+
+export interface BlockGroupOpenStartToken extends TokenBase {
+ type: TokenType.BLOCK_GROUP_OPEN_START;
+ parts: [name: string];
+}
+
+export interface BlockGroupOpenEndToken extends TokenBase {
+ type: TokenType.BLOCK_GROUP_OPEN_END;
+ parts: [];
+}
+
+export interface BlockGroupCloseToken extends TokenBase {
+ type: TokenType.BLOCK_GROUP_CLOSE;
+ parts: [name: string];
+}
+
+export interface BlockParameterToken extends TokenBase {
+ type: TokenType.BLOCK_PARAMETER;
+ parts: [expression: string];
+}
+
+export interface BlockOpenStartToken extends TokenBase {
+ type: TokenType.BLOCK_OPEN_START;
+ parts: [name: string];
+}
+
+export interface BlockOpenEndToken extends TokenBase {
+ type: TokenType.BLOCK_OPEN_END;
+ parts: [];
+}
diff --git a/packages/compiler/test/ml_parser/lexer_spec.ts b/packages/compiler/test/ml_parser/lexer_spec.ts
index c1b73cc19d5..ea5a1a3c548 100644
--- a/packages/compiler/test/ml_parser/lexer_spec.ts
+++ b/packages/compiler/test/ml_parser/lexer_spec.ts
@@ -1814,6 +1814,222 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u
]);
});
});
+
+ describe('blocks', () => {
+ // TODO(crisbeto): temporary utility while blocks are disabled by default.
+ function tokenizeBlock(input: string, options: TokenizeOptions = {}): any[] {
+ return tokenizeAndHumanizeParts(input, {tokenizeBlocks: true, ...options});
+ }
+
+ it('should parse a block group', () => {
+ expect(tokenizeBlock('{#foo}hello{/foo}')).toEqual([
+ [TokenType.BLOCK_GROUP_OPEN_START, 'foo'],
+ [TokenType.BLOCK_GROUP_OPEN_END],
+ [TokenType.TEXT, 'hello'],
+ [TokenType.BLOCK_GROUP_CLOSE, 'foo'],
+ [TokenType.EOF],
+ ]);
+ });
+
+ it('should parse a block group with parameters', () => {
+ expect(tokenizeBlock('{#for item of items; track item.id}hello{/for}')).toEqual([
+ [TokenType.BLOCK_GROUP_OPEN_START, 'for'],
+ [TokenType.BLOCK_PARAMETER, 'item of items'],
+ [TokenType.BLOCK_PARAMETER, 'track item.id'],
+ [TokenType.BLOCK_GROUP_OPEN_END],
+ [TokenType.TEXT, 'hello'],
+ [TokenType.BLOCK_GROUP_CLOSE, 'for'],
+ [TokenType.EOF],
+ ]);
+ });
+
+ it('should parse a block group with a trailing semicolon after the parameters', () => {
+ expect(tokenizeBlock('{#for item of items;}hello{/for}')).toEqual([
+ [TokenType.BLOCK_GROUP_OPEN_START, 'for'],
+ [TokenType.BLOCK_PARAMETER, 'item of items'],
+ [TokenType.BLOCK_GROUP_OPEN_END],
+ [TokenType.TEXT, 'hello'],
+ [TokenType.BLOCK_GROUP_CLOSE, 'for'],
+ [TokenType.EOF],
+ ]);
+ });
+
+ it('should parse a block group with multiple trailing semicolons', () => {
+ expect(tokenizeBlock('{#for item of items;;;;;}hello{/for}')).toEqual([
+ [TokenType.BLOCK_GROUP_OPEN_START, 'for'],
+ [TokenType.BLOCK_PARAMETER, 'item of items'],
+ [TokenType.BLOCK_GROUP_OPEN_END],
+ [TokenType.TEXT, 'hello'],
+ [TokenType.BLOCK_GROUP_CLOSE, 'for'],
+ [TokenType.EOF],
+ ]);
+ });
+
+ it('should parse a block group with trailing whitespace', () => {
+ expect(tokenizeBlock('{#foo }hello{/foo}')).toEqual([
+ [TokenType.BLOCK_GROUP_OPEN_START, 'foo'],
+ [TokenType.BLOCK_GROUP_OPEN_END],
+ [TokenType.TEXT, 'hello'],
+ [TokenType.BLOCK_GROUP_CLOSE, 'foo'],
+ [TokenType.EOF],
+ ]);
+ });
+
+ it('should parse a block group with no trailing semicolon', () => {
+ expect(tokenizeBlock('{#for item of items}hello{/for}')).toEqual([
+ [TokenType.BLOCK_GROUP_OPEN_START, 'for'],
+ [TokenType.BLOCK_PARAMETER, 'item of items'],
+ [TokenType.BLOCK_GROUP_OPEN_END],
+ [TokenType.TEXT, 'hello'],
+ [TokenType.BLOCK_GROUP_CLOSE, 'for'],
+ [TokenType.EOF],
+ ]);
+ });
+
+ it('should handle semicolons and braces used in a block group parameter', () => {
+ expect(tokenizeBlock(`{#foo a === ";"; b === '}'}hello{/foo}`)).toEqual([
+ [TokenType.BLOCK_GROUP_OPEN_START, 'foo'],
+ [TokenType.BLOCK_PARAMETER, `a === ";"`],
+ [TokenType.BLOCK_PARAMETER, `b === '}'`],
+ [TokenType.BLOCK_GROUP_OPEN_END],
+ [TokenType.TEXT, 'hello'],
+ [TokenType.BLOCK_GROUP_CLOSE, 'foo'],
+ [TokenType.EOF],
+ ]);
+ });
+
+ it('should throw for invalid quotes in a parameter', () => {
+ const options = {tokenizeBlocks: true};
+
+ expect(tokenizeAndHumanizeErrors(`{#foo a === "}hello{/foo}`, options)).toEqual([
+ [TokenType.BLOCK_PARAMETER, 'Unexpected character "EOF"', '0:25']
+ ]);
+
+ expect(tokenizeAndHumanizeErrors(`{#foo a === "hi'}hello{/foo}`, options)).toEqual([
+ [TokenType.BLOCK_PARAMETER, 'Unexpected character "EOF"', '0:28']
+ ]);
+ });
+
+ it('should handle a semicolon used in a nested string inside a block group parameter', () => {
+ expect(tokenizeBlock(`{#if condition === "';'"}hello{/if}`)).toEqual([
+ [TokenType.BLOCK_GROUP_OPEN_START, 'if'],
+ [TokenType.BLOCK_PARAMETER, `condition === "';'"`],
+ [TokenType.BLOCK_GROUP_OPEN_END],
+ [TokenType.TEXT, 'hello'],
+ [TokenType.BLOCK_GROUP_CLOSE, 'if'],
+ [TokenType.EOF],
+ ]);
+ });
+
+ it('should handle a semicolon next to an escaped quote used in a block group parameter',
+ () => {
+ expect(tokenizeBlock('{#if condition === "\\";"}hello{/if}')).toEqual([
+ [TokenType.BLOCK_GROUP_OPEN_START, 'if'],
+ [TokenType.BLOCK_PARAMETER, 'condition === "\\";"'],
+ [TokenType.BLOCK_GROUP_OPEN_END],
+ [TokenType.TEXT, 'hello'],
+ [TokenType.BLOCK_GROUP_CLOSE, 'if'],
+ [TokenType.EOF],
+ ]);
+ });
+
+ it('should parse mixed text and html content in a block group', () => {
+ expect(tokenizeBlock('{#if a === 1}foo bar baz{/if}')).toEqual([
+ [TokenType.BLOCK_GROUP_OPEN_START, 'if'],
+ [TokenType.BLOCK_PARAMETER, 'a === 1'],
+ [TokenType.BLOCK_GROUP_OPEN_END],
+ [TokenType.TEXT, 'foo '],
+ [TokenType.TAG_OPEN_START, '', 'b'],
+ [TokenType.TAG_OPEN_END],
+ [TokenType.TEXT, 'bar'],
+ [TokenType.TAG_CLOSE, '', 'b'],
+ [TokenType.TEXT, ' baz'],
+ [TokenType.BLOCK_GROUP_CLOSE, 'if'],
+ [TokenType.EOF],
+ ]);
+ });
+
+ it('should parse a block group with blocks', () => {
+ expect(tokenizeBlock(
+ '{#if expr}' +
+ 'foo' +
+ '{:else if otherExpr === 1}' +
+ '' +
+ '{:else}' +
+ 'bar' +
+ '{/if}'))
+ .toEqual([
+ [TokenType.BLOCK_GROUP_OPEN_START, 'if'],
+ [TokenType.BLOCK_PARAMETER, 'expr'],
+ [TokenType.BLOCK_GROUP_OPEN_END],
+ [TokenType.TEXT, 'foo'],
+ [TokenType.BLOCK_OPEN_START, 'else'],
+ [TokenType.BLOCK_PARAMETER, 'if otherExpr === 1'],
+ [TokenType.BLOCK_OPEN_END],
+ [TokenType.TAG_OPEN_START, '', 'some-comp'],
+ [TokenType.TAG_OPEN_END_VOID],
+ [TokenType.BLOCK_OPEN_START, 'else'],
+ [TokenType.BLOCK_OPEN_END],
+ [TokenType.TEXT, 'bar'],
+ [TokenType.BLOCK_GROUP_CLOSE, 'if'],
+ [TokenType.EOF],
+ ]);
+ });
+
+ it('should parse nested block groups', () => {
+ expect(tokenizeBlock(
+ '{#if a}' +
+ 'hello a' +
+ '{:else}' +
+ '{#if b}' +
+ 'hello b' +
+ '{:else}' +
+ '{#if c}hello c{/if}' +
+ '{/if}' +
+ '{/if}'))
+ .toEqual([
+ [TokenType.BLOCK_GROUP_OPEN_START, 'if'], [TokenType.BLOCK_PARAMETER, 'a'],
+ [TokenType.BLOCK_GROUP_OPEN_END], [TokenType.TEXT, 'hello a'],
+ [TokenType.BLOCK_OPEN_START, 'else'], [TokenType.BLOCK_OPEN_END],
+ [TokenType.BLOCK_GROUP_OPEN_START, 'if'], [TokenType.BLOCK_PARAMETER, 'b'],
+ [TokenType.BLOCK_GROUP_OPEN_END], [TokenType.TEXT, 'hello b'],
+ [TokenType.BLOCK_OPEN_START, 'else'], [TokenType.BLOCK_OPEN_END],
+ [TokenType.BLOCK_GROUP_OPEN_START, 'if'], [TokenType.BLOCK_PARAMETER, 'c'],
+ [TokenType.BLOCK_GROUP_OPEN_END], [TokenType.TEXT, 'hello c'],
+ [TokenType.BLOCK_GROUP_CLOSE, 'if'], [TokenType.BLOCK_GROUP_CLOSE, 'if'],
+ [TokenType.BLOCK_GROUP_CLOSE, 'if'], [TokenType.EOF],
+ ]);
+ });
+
+ it('should parse a block group containing an expansion', () => {
+ const result = tokenizeBlock(
+ '{#foo}{one.two, three, =4 {four} =5 {five} foo {bar} }{/foo}',
+ {tokenizeExpansionForms: true});
+
+ expect(result).toEqual([
+ [TokenType.BLOCK_GROUP_OPEN_START, 'foo'],
+ [TokenType.BLOCK_GROUP_OPEN_END],
+ [TokenType.EXPANSION_FORM_START],
+ [TokenType.RAW_TEXT, 'one.two'],
+ [TokenType.RAW_TEXT, 'three'],
+ [TokenType.EXPANSION_CASE_VALUE, '=4'],
+ [TokenType.EXPANSION_CASE_EXP_START],
+ [TokenType.TEXT, 'four'],
+ [TokenType.EXPANSION_CASE_EXP_END],
+ [TokenType.EXPANSION_CASE_VALUE, '=5'],
+ [TokenType.EXPANSION_CASE_EXP_START],
+ [TokenType.TEXT, 'five'],
+ [TokenType.EXPANSION_CASE_EXP_END],
+ [TokenType.EXPANSION_CASE_VALUE, 'foo'],
+ [TokenType.EXPANSION_CASE_EXP_START],
+ [TokenType.TEXT, 'bar'],
+ [TokenType.EXPANSION_CASE_EXP_END],
+ [TokenType.EXPANSION_FORM_END],
+ [TokenType.BLOCK_GROUP_CLOSE, 'foo'],
+ [TokenType.EOF],
+ ]);
+ });
+ });
});
}