diff --git a/packages/compiler/src/ml_parser/lexer.ts b/packages/compiler/src/ml_parser/lexer.ts index be34e8efa32..ce170b11961 100644 --- a/packages/compiler/src/ml_parser/lexer.ts +++ b/packages/compiler/src/ml_parser/lexer.ts @@ -89,6 +89,12 @@ export interface TokenizeOptions { * If true, do not convert CRLF to LF. */ preserveLineEndings?: boolean; + + // TODO(crisbeto): temporary option to limit access to the block syntax. + /** + * Whether the block syntax is enabled at the compiler level. + */ + tokenizeBlocks?: boolean; } export function tokenize( @@ -136,8 +142,8 @@ class _Tokenizer { private _expansionCaseStack: TokenType[] = []; private _inInterpolation: boolean = false; private readonly _preserveLineEndings: boolean; - private readonly _escapedString: boolean; private readonly _i18nNormalizeLineEndingsInICUs: boolean; + private readonly _tokenizeBlocks: boolean; tokens: Token[] = []; errors: TokenError[] = []; nonNormalizedIcuExpressions: Token[] = []; @@ -159,8 +165,8 @@ class _Tokenizer { this._cursor = options.escapedString ? new EscapedCharacterCursor(_file, range) : new PlainCharacterCursor(_file, range); this._preserveLineEndings = options.preserveLineEndings || false; - this._escapedString = options.escapedString || false; this._i18nNormalizeLineEndingsInICUs = options.i18nNormalizeLineEndingsInICUs || false; + this._tokenizeBlocks = options.tokenizeBlocks || false; try { this._cursor.init(); } catch (e) { @@ -197,6 +203,12 @@ class _Tokenizer { } else { this._consumeTagOpen(start); } + } else if (this._tokenizeBlocks && this._attemptStr('{#')) { + this._consumeBlockGroupOpen(start); + } else if (this._tokenizeBlocks && this._attemptStr('{/')) { + this._consumeBlockGroupClose(start); + } else if (this._tokenizeBlocks && this._attemptStr('{:')) { + this._consumeBlock(start); } else if (!(this._tokenizeIcu && this._tokenizeExpansionForm())) { // In (possibly interpolated) text the end of the text is given by `isTextEnd()`, while // the premature end of an interpolation is given by the start of a new HTML element. @@ -212,6 +224,72 @@ class _Tokenizer { this._endToken([]); } + private _consumeBlockGroupOpen(start: CharacterCursor) { + this._beginToken(TokenType.BLOCK_GROUP_OPEN_START, start); + const nameCursor = this._cursor.clone(); + this._attemptCharCodeUntilFn(code => !isBlockNameChar(code)); + this._endToken([this._cursor.getChars(nameCursor)]); + this._consumeBlockParameters(); + this._beginToken(TokenType.BLOCK_GROUP_OPEN_END); + this._requireCharCode(chars.$RBRACE); + this._endToken([]); + } + + private _consumeBlockGroupClose(start: CharacterCursor) { + this._beginToken(TokenType.BLOCK_GROUP_CLOSE, start); + const nameCursor = this._cursor.clone(); + this._attemptCharCodeUntilFn(code => !isBlockNameChar(code)); + const name = this._cursor.getChars(nameCursor); + this._requireCharCode(chars.$RBRACE); + this._endToken([name]); + } + + private _consumeBlock(start: CharacterCursor) { + this._beginToken(TokenType.BLOCK_OPEN_START, start); + const nameCursor = this._cursor.clone(); + this._attemptCharCodeUntilFn(code => !isBlockNameChar(code)); + this._endToken([this._cursor.getChars(nameCursor)]); + this._consumeBlockParameters(); + this._beginToken(TokenType.BLOCK_OPEN_END); + this._requireCharCode(chars.$RBRACE); + this._endToken([]); + } + + private _consumeBlockParameters() { + // Trim the whitespace until the first parameter. + this._attemptCharCodeUntilFn(isBlockParameterChar); + + while (this._cursor.peek() !== chars.$RBRACE && this._cursor.peek() !== chars.$EOF) { + this._beginToken(TokenType.BLOCK_PARAMETER); + const start = this._cursor.clone(); + let inQuote: number|null = null; + + // Consume the parameter until the next semicolon or brace. + // Note that we skip over semicolons/braces inside of strings. + while ((this._cursor.peek() !== chars.$SEMICOLON && this._cursor.peek() !== chars.$RBRACE && + this._cursor.peek() !== chars.$EOF) || + inQuote !== null) { + const char = this._cursor.peek(); + + // Skip to the next character if it was escaped. + if (char === chars.$BACKSLASH) { + this._cursor.advance(); + } else if (char === inQuote) { + inQuote = null; + } else if (inQuote === null && chars.isQuote(char)) { + inQuote = char; + } + + this._cursor.advance(); + } + + this._endToken([this._cursor.getChars(start)]); + + // Skip to the next parameter. + this._attemptCharCodeUntilFn(isBlockParameterChar); + } + } + /** * @returns whether an ICU token has been created * @internal @@ -578,7 +656,6 @@ class _Tokenizer { } private _consumeAttributeValue() { - let value: string; if (this._cursor.peek() === chars.$SQ || this._cursor.peek() === chars.$DQ) { const quoteChar = this._cursor.peek(); this._consumeQuote(quoteChar); @@ -795,7 +872,7 @@ class _Tokenizer { } private _isTextEnd(): boolean { - if (this._isTagStart() || this._cursor.peek() === chars.$EOF) { + if (this._isTagStart() || this._isBlockStart() || this._cursor.peek() === chars.$EOF) { return true; } @@ -833,6 +910,26 @@ class _Tokenizer { return false; } + private _isBlockStart(): boolean { + if (this._tokenizeBlocks && this._cursor.peek() === chars.$LBRACE) { + const tmp = this._cursor.clone(); + + // Check that the cursor is on a `{#`, `{/` or `{:`. + tmp.advance(); + const next = tmp.peek(); + if (next !== chars.$BANG && next !== chars.$SLASH && next !== chars.$COLON) { + return false; + } + + // If it is, also verify that the next character is a valid block identifier. + tmp.advance(); + if (isBlockNameChar(tmp.peek())) { + return true; + } + } + return false; + } + private _readUntil(char: number): string { const start = this._cursor.clone(); this._attemptUntilChar(char); @@ -900,6 +997,14 @@ function toUpperCaseCharCode(code: number): number { return code >= chars.$a && code <= chars.$z ? code - chars.$a + chars.$A : code; } +function isBlockNameChar(code: number): boolean { + return chars.isAsciiLetter(code) || chars.isDigit(code) || code === chars.$_; +} + +function isBlockParameterChar(code: number): boolean { + return code !== chars.$SEMICOLON && isNotWhitespace(code); +} + function mergeTextTokens(srcTokens: Token[]): Token[] { const dstTokens: Token[] = []; let lastDstToken: Token|undefined = undefined; diff --git a/packages/compiler/src/ml_parser/tokens.ts b/packages/compiler/src/ml_parser/tokens.ts index 643dc250031..43c9fb9ae3e 100644 --- a/packages/compiler/src/ml_parser/tokens.ts +++ b/packages/compiler/src/ml_parser/tokens.ts @@ -33,7 +33,13 @@ export const enum TokenType { EXPANSION_CASE_EXP_START, EXPANSION_CASE_EXP_END, EXPANSION_FORM_END, - EOF + EOF, + BLOCK_GROUP_OPEN_START, + BLOCK_GROUP_OPEN_END, + BLOCK_GROUP_CLOSE, + BLOCK_PARAMETER, + BLOCK_OPEN_START, + BLOCK_OPEN_END, } export type Token = TagOpenStartToken|TagOpenEndToken|TagOpenEndVoidToken|TagCloseToken| @@ -41,7 +47,8 @@ export type Token = TagOpenStartToken|TagOpenEndToken|TagOpenEndVoidToken|TagClo CommentEndToken|CdataStartToken|CdataEndToken|AttributeNameToken|AttributeQuoteToken| AttributeValueTextToken|AttributeValueInterpolationToken|DocTypeToken|ExpansionFormStartToken| ExpansionCaseValueToken|ExpansionCaseExpressionStartToken|ExpansionCaseExpressionEndToken| - ExpansionFormEndToken|EndOfFileToken; + ExpansionFormEndToken|EndOfFileToken|BlockGroupOpenStartToken|BlockGroupOpenEndToken| + BlockGroupCloseToken|BlockParameterToken|BlockOpenStartToken|BlockOpenEndToken; export type InterpolatedTextToken = TextToken|InterpolationToken|EncodedEntityToken; @@ -170,3 +177,33 @@ export interface EndOfFileToken extends TokenBase { type: TokenType.EOF; parts: []; } + +export interface BlockGroupOpenStartToken extends TokenBase { + type: TokenType.BLOCK_GROUP_OPEN_START; + parts: [name: string]; +} + +export interface BlockGroupOpenEndToken extends TokenBase { + type: TokenType.BLOCK_GROUP_OPEN_END; + parts: []; +} + +export interface BlockGroupCloseToken extends TokenBase { + type: TokenType.BLOCK_GROUP_CLOSE; + parts: [name: string]; +} + +export interface BlockParameterToken extends TokenBase { + type: TokenType.BLOCK_PARAMETER; + parts: [expression: string]; +} + +export interface BlockOpenStartToken extends TokenBase { + type: TokenType.BLOCK_OPEN_START; + parts: [name: string]; +} + +export interface BlockOpenEndToken extends TokenBase { + type: TokenType.BLOCK_OPEN_END; + parts: []; +} diff --git a/packages/compiler/test/ml_parser/lexer_spec.ts b/packages/compiler/test/ml_parser/lexer_spec.ts index c1b73cc19d5..ea5a1a3c548 100644 --- a/packages/compiler/test/ml_parser/lexer_spec.ts +++ b/packages/compiler/test/ml_parser/lexer_spec.ts @@ -1814,6 +1814,222 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u ]); }); }); + + describe('blocks', () => { + // TODO(crisbeto): temporary utility while blocks are disabled by default. + function tokenizeBlock(input: string, options: TokenizeOptions = {}): any[] { + return tokenizeAndHumanizeParts(input, {tokenizeBlocks: true, ...options}); + } + + it('should parse a block group', () => { + expect(tokenizeBlock('{#foo}hello{/foo}')).toEqual([ + [TokenType.BLOCK_GROUP_OPEN_START, 'foo'], + [TokenType.BLOCK_GROUP_OPEN_END], + [TokenType.TEXT, 'hello'], + [TokenType.BLOCK_GROUP_CLOSE, 'foo'], + [TokenType.EOF], + ]); + }); + + it('should parse a block group with parameters', () => { + expect(tokenizeBlock('{#for item of items; track item.id}hello{/for}')).toEqual([ + [TokenType.BLOCK_GROUP_OPEN_START, 'for'], + [TokenType.BLOCK_PARAMETER, 'item of items'], + [TokenType.BLOCK_PARAMETER, 'track item.id'], + [TokenType.BLOCK_GROUP_OPEN_END], + [TokenType.TEXT, 'hello'], + [TokenType.BLOCK_GROUP_CLOSE, 'for'], + [TokenType.EOF], + ]); + }); + + it('should parse a block group with a trailing semicolon after the parameters', () => { + expect(tokenizeBlock('{#for item of items;}hello{/for}')).toEqual([ + [TokenType.BLOCK_GROUP_OPEN_START, 'for'], + [TokenType.BLOCK_PARAMETER, 'item of items'], + [TokenType.BLOCK_GROUP_OPEN_END], + [TokenType.TEXT, 'hello'], + [TokenType.BLOCK_GROUP_CLOSE, 'for'], + [TokenType.EOF], + ]); + }); + + it('should parse a block group with multiple trailing semicolons', () => { + expect(tokenizeBlock('{#for item of items;;;;;}hello{/for}')).toEqual([ + [TokenType.BLOCK_GROUP_OPEN_START, 'for'], + [TokenType.BLOCK_PARAMETER, 'item of items'], + [TokenType.BLOCK_GROUP_OPEN_END], + [TokenType.TEXT, 'hello'], + [TokenType.BLOCK_GROUP_CLOSE, 'for'], + [TokenType.EOF], + ]); + }); + + it('should parse a block group with trailing whitespace', () => { + expect(tokenizeBlock('{#foo }hello{/foo}')).toEqual([ + [TokenType.BLOCK_GROUP_OPEN_START, 'foo'], + [TokenType.BLOCK_GROUP_OPEN_END], + [TokenType.TEXT, 'hello'], + [TokenType.BLOCK_GROUP_CLOSE, 'foo'], + [TokenType.EOF], + ]); + }); + + it('should parse a block group with no trailing semicolon', () => { + expect(tokenizeBlock('{#for item of items}hello{/for}')).toEqual([ + [TokenType.BLOCK_GROUP_OPEN_START, 'for'], + [TokenType.BLOCK_PARAMETER, 'item of items'], + [TokenType.BLOCK_GROUP_OPEN_END], + [TokenType.TEXT, 'hello'], + [TokenType.BLOCK_GROUP_CLOSE, 'for'], + [TokenType.EOF], + ]); + }); + + it('should handle semicolons and braces used in a block group parameter', () => { + expect(tokenizeBlock(`{#foo a === ";"; b === '}'}hello{/foo}`)).toEqual([ + [TokenType.BLOCK_GROUP_OPEN_START, 'foo'], + [TokenType.BLOCK_PARAMETER, `a === ";"`], + [TokenType.BLOCK_PARAMETER, `b === '}'`], + [TokenType.BLOCK_GROUP_OPEN_END], + [TokenType.TEXT, 'hello'], + [TokenType.BLOCK_GROUP_CLOSE, 'foo'], + [TokenType.EOF], + ]); + }); + + it('should throw for invalid quotes in a parameter', () => { + const options = {tokenizeBlocks: true}; + + expect(tokenizeAndHumanizeErrors(`{#foo a === "}hello{/foo}`, options)).toEqual([ + [TokenType.BLOCK_PARAMETER, 'Unexpected character "EOF"', '0:25'] + ]); + + expect(tokenizeAndHumanizeErrors(`{#foo a === "hi'}hello{/foo}`, options)).toEqual([ + [TokenType.BLOCK_PARAMETER, 'Unexpected character "EOF"', '0:28'] + ]); + }); + + it('should handle a semicolon used in a nested string inside a block group parameter', () => { + expect(tokenizeBlock(`{#if condition === "';'"}hello{/if}`)).toEqual([ + [TokenType.BLOCK_GROUP_OPEN_START, 'if'], + [TokenType.BLOCK_PARAMETER, `condition === "';'"`], + [TokenType.BLOCK_GROUP_OPEN_END], + [TokenType.TEXT, 'hello'], + [TokenType.BLOCK_GROUP_CLOSE, 'if'], + [TokenType.EOF], + ]); + }); + + it('should handle a semicolon next to an escaped quote used in a block group parameter', + () => { + expect(tokenizeBlock('{#if condition === "\\";"}hello{/if}')).toEqual([ + [TokenType.BLOCK_GROUP_OPEN_START, 'if'], + [TokenType.BLOCK_PARAMETER, 'condition === "\\";"'], + [TokenType.BLOCK_GROUP_OPEN_END], + [TokenType.TEXT, 'hello'], + [TokenType.BLOCK_GROUP_CLOSE, 'if'], + [TokenType.EOF], + ]); + }); + + it('should parse mixed text and html content in a block group', () => { + expect(tokenizeBlock('{#if a === 1}foo bar baz{/if}')).toEqual([ + [TokenType.BLOCK_GROUP_OPEN_START, 'if'], + [TokenType.BLOCK_PARAMETER, 'a === 1'], + [TokenType.BLOCK_GROUP_OPEN_END], + [TokenType.TEXT, 'foo '], + [TokenType.TAG_OPEN_START, '', 'b'], + [TokenType.TAG_OPEN_END], + [TokenType.TEXT, 'bar'], + [TokenType.TAG_CLOSE, '', 'b'], + [TokenType.TEXT, ' baz'], + [TokenType.BLOCK_GROUP_CLOSE, 'if'], + [TokenType.EOF], + ]); + }); + + it('should parse a block group with blocks', () => { + expect(tokenizeBlock( + '{#if expr}' + + 'foo' + + '{:else if otherExpr === 1}' + + '' + + '{:else}' + + 'bar' + + '{/if}')) + .toEqual([ + [TokenType.BLOCK_GROUP_OPEN_START, 'if'], + [TokenType.BLOCK_PARAMETER, 'expr'], + [TokenType.BLOCK_GROUP_OPEN_END], + [TokenType.TEXT, 'foo'], + [TokenType.BLOCK_OPEN_START, 'else'], + [TokenType.BLOCK_PARAMETER, 'if otherExpr === 1'], + [TokenType.BLOCK_OPEN_END], + [TokenType.TAG_OPEN_START, '', 'some-comp'], + [TokenType.TAG_OPEN_END_VOID], + [TokenType.BLOCK_OPEN_START, 'else'], + [TokenType.BLOCK_OPEN_END], + [TokenType.TEXT, 'bar'], + [TokenType.BLOCK_GROUP_CLOSE, 'if'], + [TokenType.EOF], + ]); + }); + + it('should parse nested block groups', () => { + expect(tokenizeBlock( + '{#if a}' + + 'hello a' + + '{:else}' + + '{#if b}' + + 'hello b' + + '{:else}' + + '{#if c}hello c{/if}' + + '{/if}' + + '{/if}')) + .toEqual([ + [TokenType.BLOCK_GROUP_OPEN_START, 'if'], [TokenType.BLOCK_PARAMETER, 'a'], + [TokenType.BLOCK_GROUP_OPEN_END], [TokenType.TEXT, 'hello a'], + [TokenType.BLOCK_OPEN_START, 'else'], [TokenType.BLOCK_OPEN_END], + [TokenType.BLOCK_GROUP_OPEN_START, 'if'], [TokenType.BLOCK_PARAMETER, 'b'], + [TokenType.BLOCK_GROUP_OPEN_END], [TokenType.TEXT, 'hello b'], + [TokenType.BLOCK_OPEN_START, 'else'], [TokenType.BLOCK_OPEN_END], + [TokenType.BLOCK_GROUP_OPEN_START, 'if'], [TokenType.BLOCK_PARAMETER, 'c'], + [TokenType.BLOCK_GROUP_OPEN_END], [TokenType.TEXT, 'hello c'], + [TokenType.BLOCK_GROUP_CLOSE, 'if'], [TokenType.BLOCK_GROUP_CLOSE, 'if'], + [TokenType.BLOCK_GROUP_CLOSE, 'if'], [TokenType.EOF], + ]); + }); + + it('should parse a block group containing an expansion', () => { + const result = tokenizeBlock( + '{#foo}{one.two, three, =4 {four} =5 {five} foo {bar} }{/foo}', + {tokenizeExpansionForms: true}); + + expect(result).toEqual([ + [TokenType.BLOCK_GROUP_OPEN_START, 'foo'], + [TokenType.BLOCK_GROUP_OPEN_END], + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, 'one.two'], + [TokenType.RAW_TEXT, 'three'], + [TokenType.EXPANSION_CASE_VALUE, '=4'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'four'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_CASE_VALUE, '=5'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'five'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_CASE_VALUE, 'foo'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'bar'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], + [TokenType.BLOCK_GROUP_CLOSE, 'foo'], + [TokenType.EOF], + ]); + }); + }); }); }