From 353005b97fcffffb41af3ac3fa59a64c312418aa Mon Sep 17 00:00:00 2001 From: Kristiyan Kostadinov Date: Tue, 17 Dec 2024 08:28:07 +0200 Subject: [PATCH] refactor(compiler): tokenize template literals (#59230) Reworks the lexer to produce tokens for template literal expressions. PR Close #59230 --- .../compiler/src/expression_parser/lexer.ts | 176 ++++++++++--- .../test/expression_parser/lexer_spec.ts | 246 +++++++++++++++++- 2 files changed, 380 insertions(+), 42 deletions(-) diff --git a/packages/compiler/src/expression_parser/lexer.ts b/packages/compiler/src/expression_parser/lexer.ts index 22f4a5c401b..83de92c6694 100644 --- a/packages/compiler/src/expression_parser/lexer.ts +++ b/packages/compiler/src/expression_parser/lexer.ts @@ -19,6 +19,12 @@ export enum TokenType { Error, } +export enum StringTokenKind { + Plain, + TemplateLiteralPart, + TemplateLiteralEnd, +} + const KEYWORDS = [ 'var', 'let', @@ -56,7 +62,7 @@ export class Token { return this.type === TokenType.Number; } - isString(): boolean { + isString(): this is StringToken { return this.type === TokenType.String; } @@ -116,6 +122,22 @@ export class Token { return this.type === TokenType.Number ? this.numValue : -1; } + isTemplateLiteralPart(): this is StringToken { + return this.isString() && this.kind === StringTokenKind.TemplateLiteralPart; + } + + isTemplateLiteralEnd(): this is StringToken { + return this.isString() && this.kind === StringTokenKind.TemplateLiteralEnd; + } + + isTemplateLiteralInterpolationStart(): boolean { + return this.isOperator('${'); + } + + isTemplateLiteralInterpolationEnd(): boolean { + return this.isOperator('}'); + } + toString(): string | null { switch (this.type) { case TokenType.Character: @@ -134,6 +156,17 @@ export class Token { } } +export class StringToken extends Token { + constructor( + index: number, + end: number, + strValue: string, + readonly kind: StringTokenKind, + ) { + super(index, end, TokenType.String, 0, strValue); + } +} + function newCharacterToken(index: number, end: number, code: number): Token { return new Token(index, end, TokenType.Character, code, String.fromCharCode(code)); } @@ -154,10 +187,6 @@ function newOperatorToken(index: number, end: number, text: string): Token { return new Token(index, end, TokenType.Operator, 0, text); } -function newStringToken(index: number, end: number, text: string): Token { - return new Token(index, end, TokenType.String, 0, text); -} - function newNumberToken(index: number, end: number, n: number): Token { return new Token(index, end, TokenType.Number, n, ''); } @@ -173,6 +202,8 @@ class _Scanner { private readonly length: number; private peek = 0; private index = -1; + private literalInterpolationDepth = 0; + private braceDepth = 0; constructor(private readonly input: string) { this.length = input.length; @@ -235,17 +266,22 @@ class _Scanner { : newCharacterToken(start, this.index, chars.$PERIOD); case chars.$LPAREN: case chars.$RPAREN: - case chars.$LBRACE: - case chars.$RBRACE: case chars.$LBRACKET: case chars.$RBRACKET: case chars.$COMMA: case chars.$COLON: case chars.$SEMICOLON: return this.scanCharacter(start, peek); + case chars.$LBRACE: + return this.scanOpenBrace(start, peek); + case chars.$RBRACE: + return this.scanCloseBrace(start, peek); case chars.$SQ: case chars.$DQ: return this.scanString(); + case chars.$BT: + this.advance(); + return this.scanTemplateLiteralPart(start); case chars.$HASH: return this.scanPrivateIdentifier(); case chars.$PLUS: @@ -293,6 +329,25 @@ class _Scanner { return newOperatorToken(start, this.index, str); } + private scanOpenBrace(start: number, code: number): Token { + this.braceDepth++; + this.advance(); + return newCharacterToken(start, this.index, code); + } + + private scanCloseBrace(start: number, code: number): Token { + this.advance(); + + if (this.braceDepth === 0 && this.literalInterpolationDepth > 0) { + this.literalInterpolationDepth--; + this.tokens.push(newOperatorToken(start, this.index, '}')); + return this.scanTemplateLiteralPart(this.index); + } + + this.braceDepth--; + return newCharacterToken(start, this.index, code); + } + /** * Tokenize a 2/3 char long operator * @@ -388,36 +443,21 @@ class _Scanner { } private scanString(): Token { - const start: number = this.index; - const quote: number = this.peek; + const start = this.index; + const quote = this.peek; this.advance(); // Skip initial quote. - let buffer: string = ''; - let marker: number = this.index; - const input: string = this.input; + let buffer = ''; + let marker = this.index; + const input = this.input; while (this.peek != quote) { if (this.peek == chars.$BACKSLASH) { - buffer += input.substring(marker, this.index); - let unescapedCode: number; - this.advance(); // mutates this.peek - // @ts-expect-error see microsoft/TypeScript#9998 - if (this.peek == chars.$u) { - // 4 character hex code for unicode character. - const hex: string = input.substring(this.index + 1, this.index + 5); - if (/^[0-9a-f]+$/i.test(hex)) { - unescapedCode = parseInt(hex, 16); - } else { - return this.error(`Invalid unicode escape [\\u${hex}]`, 0); - } - for (let i: number = 0; i < 5; i++) { - this.advance(); - } - } else { - unescapedCode = unescape(this.peek); - this.advance(); + const result = this.scanStringBackslash(buffer, marker); + if (typeof result !== 'string') { + return result; // Error } - buffer += String.fromCharCode(unescapedCode); + buffer = result; marker = this.index; } else if (this.peek == chars.$EOF) { return this.error('Unterminated quote', 0); @@ -429,7 +469,7 @@ class _Scanner { const last: string = input.substring(marker, this.index); this.advance(); // Skip terminating quote. - return newStringToken(start, this.index, buffer + last); + return new StringToken(start, this.index, buffer + last, StringTokenKind.Plain); } private scanQuestion(start: number): Token { @@ -443,13 +483,81 @@ class _Scanner { return newOperatorToken(start, this.index, str); } - private error(message: string, offset: number): Token { + private scanTemplateLiteralPart(start: number): Token { + let buffer = ''; + let marker = this.index; + + while (this.peek !== chars.$BT) { + if (this.peek === chars.$BACKSLASH) { + const result = this.scanStringBackslash(buffer, marker); + if (typeof result !== 'string') { + return result; // Error + } + buffer = result; + marker = this.index; + } else if (this.peek === chars.$$) { + const dollar = this.index; + this.advance(); + + // @ts-expect-error + if (this.peek === chars.$LBRACE) { + this.literalInterpolationDepth++; + this.tokens.push( + new StringToken( + start, + dollar, + buffer + this.input.substring(marker, dollar), + StringTokenKind.TemplateLiteralPart, + ), + ); + this.advance(); + return newOperatorToken(dollar, this.index, this.input.substring(dollar, this.index)); + } + } else if (this.peek === chars.$EOF) { + return this.error('Unterminated template literal', 0); + } else { + this.advance(); + } + } + + const last = this.input.substring(marker, this.index); + this.advance(); + return new StringToken(start, this.index, buffer + last, StringTokenKind.TemplateLiteralEnd); + } + + private error(message: string, offset: number): Token & {type: TokenType.Error} { const position: number = this.index + offset; return newErrorToken( position, this.index, `Lexer Error: ${message} at column ${position} in expression [${this.input}]`, - ); + ) as Token & {type: TokenType.Error}; + } + + private scanStringBackslash( + buffer: string, + marker: number, + ): string | (Token & {type: TokenType.Error}) { + buffer += this.input.substring(marker, this.index); + let unescapedCode: number; + this.advance(); + if (this.peek === chars.$u) { + // 4 character hex code for unicode character. + const hex: string = this.input.substring(this.index + 1, this.index + 5); + if (/^[0-9a-f]+$/i.test(hex)) { + unescapedCode = parseInt(hex, 16); + } else { + return this.error(`Invalid unicode escape [\\u${hex}]`, 0); + } + for (let i = 0; i < 5; i++) { + this.advance(); + } + } else { + unescapedCode = unescape(this.peek); + this.advance(); + } + buffer += String.fromCharCode(unescapedCode); + return buffer; } } diff --git a/packages/compiler/test/expression_parser/lexer_spec.ts b/packages/compiler/test/expression_parser/lexer_spec.ts index 7d4223cc0b5..20545ad8c6d 100644 --- a/packages/compiler/test/expression_parser/lexer_spec.ts +++ b/packages/compiler/test/expression_parser/lexer_spec.ts @@ -6,7 +6,7 @@ * found in the LICENSE file at https://angular.dev/license */ -import {Lexer, Token} from '@angular/compiler/src/expression_parser/lexer'; +import {Lexer, StringTokenKind, Token} from '@angular/compiler/src/expression_parser/lexer'; function lex(text: string): any[] { return new Lexer().tokenize(text); @@ -35,9 +35,16 @@ function expectNumberToken(token: any, index: number, end: number, n: number) { expect(token.toNumber()).toEqual(n); } -function expectStringToken(token: any, index: number, end: number, str: string) { +function expectStringToken( + token: any, + index: number, + end: number, + str: string, + kind: StringTokenKind, +) { expectToken(token, index, end); expect(token.isString()).toBe(true); + expect(token.kind).toBe(kind); expect(token.toString()).toEqual(str); } @@ -152,11 +159,11 @@ describe('lexer', () => { }); it('should tokenize simple quoted strings', () => { - expectStringToken(lex('"a"')[0], 0, 3, 'a'); + expectStringToken(lex('"a"')[0], 0, 3, 'a', StringTokenKind.Plain); }); it('should tokenize quoted strings with escaped quotes', () => { - expectStringToken(lex('"a\\""')[0], 0, 5, 'a"'); + expectStringToken(lex('"a\\""')[0], 0, 5, 'a"', StringTokenKind.Plain); }); it('should tokenize a string', () => { @@ -174,9 +181,9 @@ describe('lexer', () => { expectOperatorToken(tokens[10], 14, 15, '|'); expectIdentifierToken(tokens[11], 15, 16, 'f'); expectCharacterToken(tokens[12], 16, 17, ':'); - expectStringToken(tokens[13], 17, 23, "a'c"); + expectStringToken(tokens[13], 17, 23, "a'c", StringTokenKind.Plain); expectCharacterToken(tokens[14], 23, 24, ':'); - expectStringToken(tokens[15], 24, 30, 'd"e'); + expectStringToken(tokens[15], 24, 30, 'd"e', StringTokenKind.Plain); }); it('should tokenize undefined', () => { @@ -200,8 +207,8 @@ describe('lexer', () => { it('should tokenize quoted string', () => { const str = '[\'\\\'\', "\\""]'; const tokens: Token[] = lex(str); - expectStringToken(tokens[1], 1, 5, "'"); - expectStringToken(tokens[3], 7, 11, '"'); + expectStringToken(tokens[1], 1, 5, "'", StringTokenKind.Plain); + expectStringToken(tokens[3], 7, 11, '"', StringTokenKind.Plain); }); it('should tokenize escaped quoted string', () => { @@ -376,5 +383,228 @@ describe('lexer', () => { 'Lexer Error: Invalid numeric separator at column 6 in expression [1_2_3._456]', ); }); + + describe('template literals', () => { + it('should tokenize template literal with no interpolations', () => { + const tokens: Token[] = lex('`hello world`'); + expect(tokens.length).toBe(1); + expectStringToken(tokens[0], 0, 13, 'hello world', StringTokenKind.TemplateLiteralEnd); + }); + + it('should tokenize template literal containing strings', () => { + expectStringToken(lex('`a "b" c`')[0], 0, 9, `a "b" c`, StringTokenKind.TemplateLiteralEnd); + expectStringToken(lex("`a 'b' c`")[0], 0, 9, `a 'b' c`, StringTokenKind.TemplateLiteralEnd); + expectStringToken( + lex('`a \\`b\\` c`')[0], + 0, + 11, + 'a `b` c', + StringTokenKind.TemplateLiteralEnd, + ); + expectStringToken( + lex('`a "\'\\`b\\`\'" c`')[0], + 0, + 15, + `a "'\`b\`'" c`, + StringTokenKind.TemplateLiteralEnd, + ); + }); + + it('should tokenize unicode inside a template string', () => { + const tokens: Token[] = lex('`\\u00A0`'); + expect(tokens.length).toBe(1); + expect(tokens[0].toString()).toBe('\u00a0'); + }); + + it('should tokenize template literal with an interpolation in the end', () => { + const tokens: Token[] = lex('`hello ${name}`'); + expect(tokens.length).toBe(5); + expectStringToken(tokens[0], 0, 7, 'hello ', StringTokenKind.TemplateLiteralPart); + expectOperatorToken(tokens[1], 7, 9, '${'); + expectIdentifierToken(tokens[2], 9, 13, 'name'); + expectOperatorToken(tokens[3], 13, 14, '}'); + expectStringToken(tokens[4], 14, 15, '', StringTokenKind.TemplateLiteralEnd); + }); + + it('should tokenize template literal with an interpolation in the beginning', () => { + const tokens: Token[] = lex('`${name} Johnson`'); + expect(tokens.length).toBe(5); + expectStringToken(tokens[0], 0, 1, '', StringTokenKind.TemplateLiteralPart); + expectOperatorToken(tokens[1], 1, 3, '${'); + expectIdentifierToken(tokens[2], 3, 7, 'name'); + expectOperatorToken(tokens[3], 7, 8, '}'); + expectStringToken(tokens[4], 8, 17, ' Johnson', StringTokenKind.TemplateLiteralEnd); + }); + + it('should tokenize template literal with an interpolation in the middle', () => { + const tokens: Token[] = lex('`foo${bar}baz`'); + expect(tokens.length).toBe(5); + expectStringToken(tokens[0], 0, 4, 'foo', StringTokenKind.TemplateLiteralPart); + expectOperatorToken(tokens[1], 4, 6, '${'); + expectIdentifierToken(tokens[2], 6, 9, 'bar'); + expectOperatorToken(tokens[3], 9, 10, '}'); + expectStringToken(tokens[4], 10, 14, 'baz', StringTokenKind.TemplateLiteralEnd); + }); + + it('should be able to use interpolation characters inside template string', () => { + expectStringToken(lex('`foo $`')[0], 0, 7, 'foo $', StringTokenKind.TemplateLiteralEnd); + expectStringToken(lex('`foo }`')[0], 0, 7, 'foo }', StringTokenKind.TemplateLiteralEnd); + expectStringToken( + lex('`foo $ {}`')[0], + 0, + 10, + 'foo $ {}', + StringTokenKind.TemplateLiteralEnd, + ); + expectStringToken( + lex('`foo \\${bar}`')[0], + 0, + 13, + 'foo ${bar}', + StringTokenKind.TemplateLiteralEnd, + ); + }); + + it('should tokenize template literal with several interpolations', () => { + const tokens: Token[] = lex('`${a} - ${b} - ${c}`'); + expect(tokens.length).toBe(13); + expectStringToken(tokens[0], 0, 1, '', StringTokenKind.TemplateLiteralPart); + expectOperatorToken(tokens[1], 1, 3, '${'); + expectIdentifierToken(tokens[2], 3, 4, 'a'); + expectOperatorToken(tokens[3], 4, 5, '}'); + expectStringToken(tokens[4], 5, 8, ' - ', StringTokenKind.TemplateLiteralPart); + expectOperatorToken(tokens[5], 8, 10, '${'); + expectIdentifierToken(tokens[6], 10, 11, 'b'); + expectOperatorToken(tokens[7], 11, 12, '}'); + expectStringToken(tokens[8], 12, 15, ' - ', StringTokenKind.TemplateLiteralPart); + expectOperatorToken(tokens[9], 15, 17, '${'); + expectIdentifierToken(tokens[10], 17, 18, 'c'); + expectOperatorToken(tokens[11], 18, 19, '}'); + }); + + it('should tokenize template literal with an object literal inside the interpolation', () => { + const tokens: Token[] = lex('`foo ${{$: true}} baz`'); + expect(tokens.length).toBe(9); + expectStringToken(tokens[0], 0, 5, 'foo ', StringTokenKind.TemplateLiteralPart); + expectOperatorToken(tokens[1], 5, 7, '${'); + expectCharacterToken(tokens[2], 7, 8, '{'); + expectIdentifierToken(tokens[3], 8, 9, '$'); + expectCharacterToken(tokens[4], 9, 10, ':'); + expectKeywordToken(tokens[5], 11, 15, 'true'); + expectCharacterToken(tokens[6], 15, 16, '}'); + expectOperatorToken(tokens[7], 16, 17, '}'); + expectStringToken(tokens[8], 17, 22, ' baz', StringTokenKind.TemplateLiteralEnd); + }); + + it('should tokenize template literal with template literals inside the interpolation', () => { + const tokens: Token[] = lex('`foo ${`hello ${`${a} - b`}`} baz`'); + expect(tokens.length).toBe(13); + expectStringToken(tokens[0], 0, 5, 'foo ', StringTokenKind.TemplateLiteralPart); + expectOperatorToken(tokens[1], 5, 7, '${'); + expectStringToken(tokens[2], 7, 14, 'hello ', StringTokenKind.TemplateLiteralPart); + expectOperatorToken(tokens[3], 14, 16, '${'); + expectStringToken(tokens[4], 16, 17, '', StringTokenKind.TemplateLiteralPart); + expectOperatorToken(tokens[5], 17, 19, '${'); + expectIdentifierToken(tokens[6], 19, 20, 'a'); + expectOperatorToken(tokens[7], 20, 21, '}'); + expectStringToken(tokens[8], 21, 26, ' - b', StringTokenKind.TemplateLiteralEnd); + expectOperatorToken(tokens[9], 26, 27, '}'); + expectStringToken(tokens[10], 27, 28, '', StringTokenKind.TemplateLiteralEnd); + expectOperatorToken(tokens[11], 28, 29, '}'); + expectStringToken(tokens[12], 29, 34, ' baz', StringTokenKind.TemplateLiteralEnd); + }); + + it('should tokenize two template literal right after each other', () => { + const tokens: Token[] = lex('`hello ${name}``see ${name} later`'); + expect(tokens.length).toBe(10); + expectStringToken(tokens[0], 0, 7, 'hello ', StringTokenKind.TemplateLiteralPart); + expectOperatorToken(tokens[1], 7, 9, '${'); + expectIdentifierToken(tokens[2], 9, 13, 'name'); + expectOperatorToken(tokens[3], 13, 14, '}'); + expectStringToken(tokens[4], 14, 15, '', StringTokenKind.TemplateLiteralEnd); + expectStringToken(tokens[5], 15, 20, 'see ', StringTokenKind.TemplateLiteralPart); + expectOperatorToken(tokens[6], 20, 22, '${'); + expectIdentifierToken(tokens[7], 22, 26, 'name'); + expectOperatorToken(tokens[8], 26, 27, '}'); + expectStringToken(tokens[9], 27, 34, ' later', StringTokenKind.TemplateLiteralEnd); + }); + + it('should tokenize a concatenated template literal', () => { + const tokens: Token[] = lex('`hello ${name}` + 123'); + expect(tokens.length).toBe(7); + expectStringToken(tokens[0], 0, 7, 'hello ', StringTokenKind.TemplateLiteralPart); + expectOperatorToken(tokens[1], 7, 9, '${'); + expectIdentifierToken(tokens[2], 9, 13, 'name'); + expectOperatorToken(tokens[3], 13, 14, '}'); + expectStringToken(tokens[4], 14, 15, '', StringTokenKind.TemplateLiteralEnd); + expectOperatorToken(tokens[5], 16, 17, '+'); + expectNumberToken(tokens[6], 18, 21, 123); + }); + + it('should tokenize a template literal with a pipe inside an interpolation', () => { + const tokens: Token[] = lex('`hello ${name | capitalize}!!!`'); + expect(tokens.length).toBe(7); + expectStringToken(tokens[0], 0, 7, 'hello ', StringTokenKind.TemplateLiteralPart); + expectOperatorToken(tokens[1], 7, 9, '${'); + expectIdentifierToken(tokens[2], 9, 13, 'name'); + expectOperatorToken(tokens[3], 14, 15, '|'); + expectIdentifierToken(tokens[4], 16, 26, 'capitalize'); + expectOperatorToken(tokens[5], 26, 27, '}'); + expectStringToken(tokens[6], 27, 31, '!!!', StringTokenKind.TemplateLiteralEnd); + }); + + it('should tokenize a template literal with a pipe inside a parenthesized interpolation', () => { + const tokens: Token[] = lex('`hello ${(name | capitalize)}!!!`'); + expect(tokens.length).toBe(9); + expectStringToken(tokens[0], 0, 7, 'hello ', StringTokenKind.TemplateLiteralPart); + expectOperatorToken(tokens[1], 7, 9, '${'); + expectCharacterToken(tokens[2], 9, 10, '('); + expectIdentifierToken(tokens[3], 10, 14, 'name'); + expectOperatorToken(tokens[4], 15, 16, '|'); + expectIdentifierToken(tokens[5], 17, 27, 'capitalize'); + expectCharacterToken(tokens[6], 27, 28, ')'); + expectOperatorToken(tokens[7], 28, 29, '}'); + expectStringToken(tokens[8], 29, 33, '!!!', StringTokenKind.TemplateLiteralEnd); + }); + + it('should produce an error if a template literal is not terminated', () => { + expectErrorToken( + lex('`hello')[0], + 6, + 6, + 'Lexer Error: Unterminated template literal at column 6 in expression [`hello]', + ); + }); + + it('should produce an error for an unterminated template literal with an interpolation', () => { + const tokens: Token[] = lex('`hello ${name}!'); + expect(tokens.length).toBe(5); + expectStringToken(tokens[0], 0, 7, 'hello ', StringTokenKind.TemplateLiteralPart); + expectOperatorToken(tokens[1], 7, 9, '${'); + expectIdentifierToken(tokens[2], 9, 13, 'name'); + expectOperatorToken(tokens[3], 13, 14, '}'); + expectErrorToken( + tokens[4], + 15, + 15, + 'Lexer Error: Unterminated template literal at column 15 in expression [`hello ${name}!]', + ); + }); + + it('should produce an error for an unterminate template literal interpolation', () => { + const tokens: Token[] = lex('`hello ${name!`'); + expect(tokens.length).toBe(5); + expectStringToken(tokens[0], 0, 7, 'hello ', StringTokenKind.TemplateLiteralPart); + expectOperatorToken(tokens[1], 7, 9, '${'); + expectIdentifierToken(tokens[2], 9, 13, 'name'); + expectOperatorToken(tokens[3], 13, 14, '!'); + expectErrorToken( + tokens[4], + 15, + 15, + 'Lexer Error: Unterminated template literal at column 15 in expression [`hello ${name!`]', + ); + }); + }); }); });