mirror of
https://github.com/angular/angular.git
synced 2026-09-14 13:54:52 +08:00
refactor(compiler): tokenize template literals (#59230)
Reworks the lexer to produce tokens for template literal expressions. PR Close #59230
This commit is contained in:
Kristiyan Kostadinov
committed by
Andrew Kushnir
Andrew Kushnir
parent
98998bbd42
commit
353005b97f
@@ -19,6 +19,12 @@ export enum TokenType {
|
||||
Error,
|
||||
}
|
||||
|
||||
export enum StringTokenKind {
|
||||
Plain,
|
||||
TemplateLiteralPart,
|
||||
TemplateLiteralEnd,
|
||||
}
|
||||
|
||||
const KEYWORDS = [
|
||||
'var',
|
||||
'let',
|
||||
@@ -56,7 +62,7 @@ export class Token {
|
||||
return this.type === TokenType.Number;
|
||||
}
|
||||
|
||||
isString(): boolean {
|
||||
isString(): this is StringToken {
|
||||
return this.type === TokenType.String;
|
||||
}
|
||||
|
||||
@@ -116,6 +122,22 @@ export class Token {
|
||||
return this.type === TokenType.Number ? this.numValue : -1;
|
||||
}
|
||||
|
||||
isTemplateLiteralPart(): this is StringToken {
|
||||
return this.isString() && this.kind === StringTokenKind.TemplateLiteralPart;
|
||||
}
|
||||
|
||||
isTemplateLiteralEnd(): this is StringToken {
|
||||
return this.isString() && this.kind === StringTokenKind.TemplateLiteralEnd;
|
||||
}
|
||||
|
||||
isTemplateLiteralInterpolationStart(): boolean {
|
||||
return this.isOperator('${');
|
||||
}
|
||||
|
||||
isTemplateLiteralInterpolationEnd(): boolean {
|
||||
return this.isOperator('}');
|
||||
}
|
||||
|
||||
toString(): string | null {
|
||||
switch (this.type) {
|
||||
case TokenType.Character:
|
||||
@@ -134,6 +156,17 @@ export class Token {
|
||||
}
|
||||
}
|
||||
|
||||
export class StringToken extends Token {
|
||||
constructor(
|
||||
index: number,
|
||||
end: number,
|
||||
strValue: string,
|
||||
readonly kind: StringTokenKind,
|
||||
) {
|
||||
super(index, end, TokenType.String, 0, strValue);
|
||||
}
|
||||
}
|
||||
|
||||
function newCharacterToken(index: number, end: number, code: number): Token {
|
||||
return new Token(index, end, TokenType.Character, code, String.fromCharCode(code));
|
||||
}
|
||||
@@ -154,10 +187,6 @@ function newOperatorToken(index: number, end: number, text: string): Token {
|
||||
return new Token(index, end, TokenType.Operator, 0, text);
|
||||
}
|
||||
|
||||
function newStringToken(index: number, end: number, text: string): Token {
|
||||
return new Token(index, end, TokenType.String, 0, text);
|
||||
}
|
||||
|
||||
function newNumberToken(index: number, end: number, n: number): Token {
|
||||
return new Token(index, end, TokenType.Number, n, '');
|
||||
}
|
||||
@@ -173,6 +202,8 @@ class _Scanner {
|
||||
private readonly length: number;
|
||||
private peek = 0;
|
||||
private index = -1;
|
||||
private literalInterpolationDepth = 0;
|
||||
private braceDepth = 0;
|
||||
|
||||
constructor(private readonly input: string) {
|
||||
this.length = input.length;
|
||||
@@ -235,17 +266,22 @@ class _Scanner {
|
||||
: newCharacterToken(start, this.index, chars.$PERIOD);
|
||||
case chars.$LPAREN:
|
||||
case chars.$RPAREN:
|
||||
case chars.$LBRACE:
|
||||
case chars.$RBRACE:
|
||||
case chars.$LBRACKET:
|
||||
case chars.$RBRACKET:
|
||||
case chars.$COMMA:
|
||||
case chars.$COLON:
|
||||
case chars.$SEMICOLON:
|
||||
return this.scanCharacter(start, peek);
|
||||
case chars.$LBRACE:
|
||||
return this.scanOpenBrace(start, peek);
|
||||
case chars.$RBRACE:
|
||||
return this.scanCloseBrace(start, peek);
|
||||
case chars.$SQ:
|
||||
case chars.$DQ:
|
||||
return this.scanString();
|
||||
case chars.$BT:
|
||||
this.advance();
|
||||
return this.scanTemplateLiteralPart(start);
|
||||
case chars.$HASH:
|
||||
return this.scanPrivateIdentifier();
|
||||
case chars.$PLUS:
|
||||
@@ -293,6 +329,25 @@ class _Scanner {
|
||||
return newOperatorToken(start, this.index, str);
|
||||
}
|
||||
|
||||
private scanOpenBrace(start: number, code: number): Token {
|
||||
this.braceDepth++;
|
||||
this.advance();
|
||||
return newCharacterToken(start, this.index, code);
|
||||
}
|
||||
|
||||
private scanCloseBrace(start: number, code: number): Token {
|
||||
this.advance();
|
||||
|
||||
if (this.braceDepth === 0 && this.literalInterpolationDepth > 0) {
|
||||
this.literalInterpolationDepth--;
|
||||
this.tokens.push(newOperatorToken(start, this.index, '}'));
|
||||
return this.scanTemplateLiteralPart(this.index);
|
||||
}
|
||||
|
||||
this.braceDepth--;
|
||||
return newCharacterToken(start, this.index, code);
|
||||
}
|
||||
|
||||
/**
|
||||
* Tokenize a 2/3 char long operator
|
||||
*
|
||||
@@ -388,36 +443,21 @@ class _Scanner {
|
||||
}
|
||||
|
||||
private scanString(): Token {
|
||||
const start: number = this.index;
|
||||
const quote: number = this.peek;
|
||||
const start = this.index;
|
||||
const quote = this.peek;
|
||||
this.advance(); // Skip initial quote.
|
||||
|
||||
let buffer: string = '';
|
||||
let marker: number = this.index;
|
||||
const input: string = this.input;
|
||||
let buffer = '';
|
||||
let marker = this.index;
|
||||
const input = this.input;
|
||||
|
||||
while (this.peek != quote) {
|
||||
if (this.peek == chars.$BACKSLASH) {
|
||||
buffer += input.substring(marker, this.index);
|
||||
let unescapedCode: number;
|
||||
this.advance(); // mutates this.peek
|
||||
// @ts-expect-error see microsoft/TypeScript#9998
|
||||
if (this.peek == chars.$u) {
|
||||
// 4 character hex code for unicode character.
|
||||
const hex: string = input.substring(this.index + 1, this.index + 5);
|
||||
if (/^[0-9a-f]+$/i.test(hex)) {
|
||||
unescapedCode = parseInt(hex, 16);
|
||||
} else {
|
||||
return this.error(`Invalid unicode escape [\\u${hex}]`, 0);
|
||||
}
|
||||
for (let i: number = 0; i < 5; i++) {
|
||||
this.advance();
|
||||
}
|
||||
} else {
|
||||
unescapedCode = unescape(this.peek);
|
||||
this.advance();
|
||||
const result = this.scanStringBackslash(buffer, marker);
|
||||
if (typeof result !== 'string') {
|
||||
return result; // Error
|
||||
}
|
||||
buffer += String.fromCharCode(unescapedCode);
|
||||
buffer = result;
|
||||
marker = this.index;
|
||||
} else if (this.peek == chars.$EOF) {
|
||||
return this.error('Unterminated quote', 0);
|
||||
@@ -429,7 +469,7 @@ class _Scanner {
|
||||
const last: string = input.substring(marker, this.index);
|
||||
this.advance(); // Skip terminating quote.
|
||||
|
||||
return newStringToken(start, this.index, buffer + last);
|
||||
return new StringToken(start, this.index, buffer + last, StringTokenKind.Plain);
|
||||
}
|
||||
|
||||
private scanQuestion(start: number): Token {
|
||||
@@ -443,13 +483,81 @@ class _Scanner {
|
||||
return newOperatorToken(start, this.index, str);
|
||||
}
|
||||
|
||||
private error(message: string, offset: number): Token {
|
||||
private scanTemplateLiteralPart(start: number): Token {
|
||||
let buffer = '';
|
||||
let marker = this.index;
|
||||
|
||||
while (this.peek !== chars.$BT) {
|
||||
if (this.peek === chars.$BACKSLASH) {
|
||||
const result = this.scanStringBackslash(buffer, marker);
|
||||
if (typeof result !== 'string') {
|
||||
return result; // Error
|
||||
}
|
||||
buffer = result;
|
||||
marker = this.index;
|
||||
} else if (this.peek === chars.$$) {
|
||||
const dollar = this.index;
|
||||
this.advance();
|
||||
|
||||
// @ts-expect-error
|
||||
if (this.peek === chars.$LBRACE) {
|
||||
this.literalInterpolationDepth++;
|
||||
this.tokens.push(
|
||||
new StringToken(
|
||||
start,
|
||||
dollar,
|
||||
buffer + this.input.substring(marker, dollar),
|
||||
StringTokenKind.TemplateLiteralPart,
|
||||
),
|
||||
);
|
||||
this.advance();
|
||||
return newOperatorToken(dollar, this.index, this.input.substring(dollar, this.index));
|
||||
}
|
||||
} else if (this.peek === chars.$EOF) {
|
||||
return this.error('Unterminated template literal', 0);
|
||||
} else {
|
||||
this.advance();
|
||||
}
|
||||
}
|
||||
|
||||
const last = this.input.substring(marker, this.index);
|
||||
this.advance();
|
||||
return new StringToken(start, this.index, buffer + last, StringTokenKind.TemplateLiteralEnd);
|
||||
}
|
||||
|
||||
private error(message: string, offset: number): Token & {type: TokenType.Error} {
|
||||
const position: number = this.index + offset;
|
||||
return newErrorToken(
|
||||
position,
|
||||
this.index,
|
||||
`Lexer Error: ${message} at column ${position} in expression [${this.input}]`,
|
||||
);
|
||||
) as Token & {type: TokenType.Error};
|
||||
}
|
||||
|
||||
private scanStringBackslash(
|
||||
buffer: string,
|
||||
marker: number,
|
||||
): string | (Token & {type: TokenType.Error}) {
|
||||
buffer += this.input.substring(marker, this.index);
|
||||
let unescapedCode: number;
|
||||
this.advance();
|
||||
if (this.peek === chars.$u) {
|
||||
// 4 character hex code for unicode character.
|
||||
const hex: string = this.input.substring(this.index + 1, this.index + 5);
|
||||
if (/^[0-9a-f]+$/i.test(hex)) {
|
||||
unescapedCode = parseInt(hex, 16);
|
||||
} else {
|
||||
return this.error(`Invalid unicode escape [\\u${hex}]`, 0);
|
||||
}
|
||||
for (let i = 0; i < 5; i++) {
|
||||
this.advance();
|
||||
}
|
||||
} else {
|
||||
unescapedCode = unescape(this.peek);
|
||||
this.advance();
|
||||
}
|
||||
buffer += String.fromCharCode(unescapedCode);
|
||||
return buffer;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
* found in the LICENSE file at https://angular.dev/license
|
||||
*/
|
||||
|
||||
import {Lexer, Token} from '@angular/compiler/src/expression_parser/lexer';
|
||||
import {Lexer, StringTokenKind, Token} from '@angular/compiler/src/expression_parser/lexer';
|
||||
|
||||
function lex(text: string): any[] {
|
||||
return new Lexer().tokenize(text);
|
||||
@@ -35,9 +35,16 @@ function expectNumberToken(token: any, index: number, end: number, n: number) {
|
||||
expect(token.toNumber()).toEqual(n);
|
||||
}
|
||||
|
||||
function expectStringToken(token: any, index: number, end: number, str: string) {
|
||||
function expectStringToken(
|
||||
token: any,
|
||||
index: number,
|
||||
end: number,
|
||||
str: string,
|
||||
kind: StringTokenKind,
|
||||
) {
|
||||
expectToken(token, index, end);
|
||||
expect(token.isString()).toBe(true);
|
||||
expect(token.kind).toBe(kind);
|
||||
expect(token.toString()).toEqual(str);
|
||||
}
|
||||
|
||||
@@ -152,11 +159,11 @@ describe('lexer', () => {
|
||||
});
|
||||
|
||||
it('should tokenize simple quoted strings', () => {
|
||||
expectStringToken(lex('"a"')[0], 0, 3, 'a');
|
||||
expectStringToken(lex('"a"')[0], 0, 3, 'a', StringTokenKind.Plain);
|
||||
});
|
||||
|
||||
it('should tokenize quoted strings with escaped quotes', () => {
|
||||
expectStringToken(lex('"a\\""')[0], 0, 5, 'a"');
|
||||
expectStringToken(lex('"a\\""')[0], 0, 5, 'a"', StringTokenKind.Plain);
|
||||
});
|
||||
|
||||
it('should tokenize a string', () => {
|
||||
@@ -174,9 +181,9 @@ describe('lexer', () => {
|
||||
expectOperatorToken(tokens[10], 14, 15, '|');
|
||||
expectIdentifierToken(tokens[11], 15, 16, 'f');
|
||||
expectCharacterToken(tokens[12], 16, 17, ':');
|
||||
expectStringToken(tokens[13], 17, 23, "a'c");
|
||||
expectStringToken(tokens[13], 17, 23, "a'c", StringTokenKind.Plain);
|
||||
expectCharacterToken(tokens[14], 23, 24, ':');
|
||||
expectStringToken(tokens[15], 24, 30, 'd"e');
|
||||
expectStringToken(tokens[15], 24, 30, 'd"e', StringTokenKind.Plain);
|
||||
});
|
||||
|
||||
it('should tokenize undefined', () => {
|
||||
@@ -200,8 +207,8 @@ describe('lexer', () => {
|
||||
it('should tokenize quoted string', () => {
|
||||
const str = '[\'\\\'\', "\\""]';
|
||||
const tokens: Token[] = lex(str);
|
||||
expectStringToken(tokens[1], 1, 5, "'");
|
||||
expectStringToken(tokens[3], 7, 11, '"');
|
||||
expectStringToken(tokens[1], 1, 5, "'", StringTokenKind.Plain);
|
||||
expectStringToken(tokens[3], 7, 11, '"', StringTokenKind.Plain);
|
||||
});
|
||||
|
||||
it('should tokenize escaped quoted string', () => {
|
||||
@@ -376,5 +383,228 @@ describe('lexer', () => {
|
||||
'Lexer Error: Invalid numeric separator at column 6 in expression [1_2_3._456]',
|
||||
);
|
||||
});
|
||||
|
||||
describe('template literals', () => {
|
||||
it('should tokenize template literal with no interpolations', () => {
|
||||
const tokens: Token[] = lex('`hello world`');
|
||||
expect(tokens.length).toBe(1);
|
||||
expectStringToken(tokens[0], 0, 13, 'hello world', StringTokenKind.TemplateLiteralEnd);
|
||||
});
|
||||
|
||||
it('should tokenize template literal containing strings', () => {
|
||||
expectStringToken(lex('`a "b" c`')[0], 0, 9, `a "b" c`, StringTokenKind.TemplateLiteralEnd);
|
||||
expectStringToken(lex("`a 'b' c`")[0], 0, 9, `a 'b' c`, StringTokenKind.TemplateLiteralEnd);
|
||||
expectStringToken(
|
||||
lex('`a \\`b\\` c`')[0],
|
||||
0,
|
||||
11,
|
||||
'a `b` c',
|
||||
StringTokenKind.TemplateLiteralEnd,
|
||||
);
|
||||
expectStringToken(
|
||||
lex('`a "\'\\`b\\`\'" c`')[0],
|
||||
0,
|
||||
15,
|
||||
`a "'\`b\`'" c`,
|
||||
StringTokenKind.TemplateLiteralEnd,
|
||||
);
|
||||
});
|
||||
|
||||
it('should tokenize unicode inside a template string', () => {
|
||||
const tokens: Token[] = lex('`\\u00A0`');
|
||||
expect(tokens.length).toBe(1);
|
||||
expect(tokens[0].toString()).toBe('\u00a0');
|
||||
});
|
||||
|
||||
it('should tokenize template literal with an interpolation in the end', () => {
|
||||
const tokens: Token[] = lex('`hello ${name}`');
|
||||
expect(tokens.length).toBe(5);
|
||||
expectStringToken(tokens[0], 0, 7, 'hello ', StringTokenKind.TemplateLiteralPart);
|
||||
expectOperatorToken(tokens[1], 7, 9, '${');
|
||||
expectIdentifierToken(tokens[2], 9, 13, 'name');
|
||||
expectOperatorToken(tokens[3], 13, 14, '}');
|
||||
expectStringToken(tokens[4], 14, 15, '', StringTokenKind.TemplateLiteralEnd);
|
||||
});
|
||||
|
||||
it('should tokenize template literal with an interpolation in the beginning', () => {
|
||||
const tokens: Token[] = lex('`${name} Johnson`');
|
||||
expect(tokens.length).toBe(5);
|
||||
expectStringToken(tokens[0], 0, 1, '', StringTokenKind.TemplateLiteralPart);
|
||||
expectOperatorToken(tokens[1], 1, 3, '${');
|
||||
expectIdentifierToken(tokens[2], 3, 7, 'name');
|
||||
expectOperatorToken(tokens[3], 7, 8, '}');
|
||||
expectStringToken(tokens[4], 8, 17, ' Johnson', StringTokenKind.TemplateLiteralEnd);
|
||||
});
|
||||
|
||||
it('should tokenize template literal with an interpolation in the middle', () => {
|
||||
const tokens: Token[] = lex('`foo${bar}baz`');
|
||||
expect(tokens.length).toBe(5);
|
||||
expectStringToken(tokens[0], 0, 4, 'foo', StringTokenKind.TemplateLiteralPart);
|
||||
expectOperatorToken(tokens[1], 4, 6, '${');
|
||||
expectIdentifierToken(tokens[2], 6, 9, 'bar');
|
||||
expectOperatorToken(tokens[3], 9, 10, '}');
|
||||
expectStringToken(tokens[4], 10, 14, 'baz', StringTokenKind.TemplateLiteralEnd);
|
||||
});
|
||||
|
||||
it('should be able to use interpolation characters inside template string', () => {
|
||||
expectStringToken(lex('`foo $`')[0], 0, 7, 'foo $', StringTokenKind.TemplateLiteralEnd);
|
||||
expectStringToken(lex('`foo }`')[0], 0, 7, 'foo }', StringTokenKind.TemplateLiteralEnd);
|
||||
expectStringToken(
|
||||
lex('`foo $ {}`')[0],
|
||||
0,
|
||||
10,
|
||||
'foo $ {}',
|
||||
StringTokenKind.TemplateLiteralEnd,
|
||||
);
|
||||
expectStringToken(
|
||||
lex('`foo \\${bar}`')[0],
|
||||
0,
|
||||
13,
|
||||
'foo ${bar}',
|
||||
StringTokenKind.TemplateLiteralEnd,
|
||||
);
|
||||
});
|
||||
|
||||
it('should tokenize template literal with several interpolations', () => {
|
||||
const tokens: Token[] = lex('`${a} - ${b} - ${c}`');
|
||||
expect(tokens.length).toBe(13);
|
||||
expectStringToken(tokens[0], 0, 1, '', StringTokenKind.TemplateLiteralPart);
|
||||
expectOperatorToken(tokens[1], 1, 3, '${');
|
||||
expectIdentifierToken(tokens[2], 3, 4, 'a');
|
||||
expectOperatorToken(tokens[3], 4, 5, '}');
|
||||
expectStringToken(tokens[4], 5, 8, ' - ', StringTokenKind.TemplateLiteralPart);
|
||||
expectOperatorToken(tokens[5], 8, 10, '${');
|
||||
expectIdentifierToken(tokens[6], 10, 11, 'b');
|
||||
expectOperatorToken(tokens[7], 11, 12, '}');
|
||||
expectStringToken(tokens[8], 12, 15, ' - ', StringTokenKind.TemplateLiteralPart);
|
||||
expectOperatorToken(tokens[9], 15, 17, '${');
|
||||
expectIdentifierToken(tokens[10], 17, 18, 'c');
|
||||
expectOperatorToken(tokens[11], 18, 19, '}');
|
||||
});
|
||||
|
||||
it('should tokenize template literal with an object literal inside the interpolation', () => {
|
||||
const tokens: Token[] = lex('`foo ${{$: true}} baz`');
|
||||
expect(tokens.length).toBe(9);
|
||||
expectStringToken(tokens[0], 0, 5, 'foo ', StringTokenKind.TemplateLiteralPart);
|
||||
expectOperatorToken(tokens[1], 5, 7, '${');
|
||||
expectCharacterToken(tokens[2], 7, 8, '{');
|
||||
expectIdentifierToken(tokens[3], 8, 9, '$');
|
||||
expectCharacterToken(tokens[4], 9, 10, ':');
|
||||
expectKeywordToken(tokens[5], 11, 15, 'true');
|
||||
expectCharacterToken(tokens[6], 15, 16, '}');
|
||||
expectOperatorToken(tokens[7], 16, 17, '}');
|
||||
expectStringToken(tokens[8], 17, 22, ' baz', StringTokenKind.TemplateLiteralEnd);
|
||||
});
|
||||
|
||||
it('should tokenize template literal with template literals inside the interpolation', () => {
|
||||
const tokens: Token[] = lex('`foo ${`hello ${`${a} - b`}`} baz`');
|
||||
expect(tokens.length).toBe(13);
|
||||
expectStringToken(tokens[0], 0, 5, 'foo ', StringTokenKind.TemplateLiteralPart);
|
||||
expectOperatorToken(tokens[1], 5, 7, '${');
|
||||
expectStringToken(tokens[2], 7, 14, 'hello ', StringTokenKind.TemplateLiteralPart);
|
||||
expectOperatorToken(tokens[3], 14, 16, '${');
|
||||
expectStringToken(tokens[4], 16, 17, '', StringTokenKind.TemplateLiteralPart);
|
||||
expectOperatorToken(tokens[5], 17, 19, '${');
|
||||
expectIdentifierToken(tokens[6], 19, 20, 'a');
|
||||
expectOperatorToken(tokens[7], 20, 21, '}');
|
||||
expectStringToken(tokens[8], 21, 26, ' - b', StringTokenKind.TemplateLiteralEnd);
|
||||
expectOperatorToken(tokens[9], 26, 27, '}');
|
||||
expectStringToken(tokens[10], 27, 28, '', StringTokenKind.TemplateLiteralEnd);
|
||||
expectOperatorToken(tokens[11], 28, 29, '}');
|
||||
expectStringToken(tokens[12], 29, 34, ' baz', StringTokenKind.TemplateLiteralEnd);
|
||||
});
|
||||
|
||||
it('should tokenize two template literal right after each other', () => {
|
||||
const tokens: Token[] = lex('`hello ${name}``see ${name} later`');
|
||||
expect(tokens.length).toBe(10);
|
||||
expectStringToken(tokens[0], 0, 7, 'hello ', StringTokenKind.TemplateLiteralPart);
|
||||
expectOperatorToken(tokens[1], 7, 9, '${');
|
||||
expectIdentifierToken(tokens[2], 9, 13, 'name');
|
||||
expectOperatorToken(tokens[3], 13, 14, '}');
|
||||
expectStringToken(tokens[4], 14, 15, '', StringTokenKind.TemplateLiteralEnd);
|
||||
expectStringToken(tokens[5], 15, 20, 'see ', StringTokenKind.TemplateLiteralPart);
|
||||
expectOperatorToken(tokens[6], 20, 22, '${');
|
||||
expectIdentifierToken(tokens[7], 22, 26, 'name');
|
||||
expectOperatorToken(tokens[8], 26, 27, '}');
|
||||
expectStringToken(tokens[9], 27, 34, ' later', StringTokenKind.TemplateLiteralEnd);
|
||||
});
|
||||
|
||||
it('should tokenize a concatenated template literal', () => {
|
||||
const tokens: Token[] = lex('`hello ${name}` + 123');
|
||||
expect(tokens.length).toBe(7);
|
||||
expectStringToken(tokens[0], 0, 7, 'hello ', StringTokenKind.TemplateLiteralPart);
|
||||
expectOperatorToken(tokens[1], 7, 9, '${');
|
||||
expectIdentifierToken(tokens[2], 9, 13, 'name');
|
||||
expectOperatorToken(tokens[3], 13, 14, '}');
|
||||
expectStringToken(tokens[4], 14, 15, '', StringTokenKind.TemplateLiteralEnd);
|
||||
expectOperatorToken(tokens[5], 16, 17, '+');
|
||||
expectNumberToken(tokens[6], 18, 21, 123);
|
||||
});
|
||||
|
||||
it('should tokenize a template literal with a pipe inside an interpolation', () => {
|
||||
const tokens: Token[] = lex('`hello ${name | capitalize}!!!`');
|
||||
expect(tokens.length).toBe(7);
|
||||
expectStringToken(tokens[0], 0, 7, 'hello ', StringTokenKind.TemplateLiteralPart);
|
||||
expectOperatorToken(tokens[1], 7, 9, '${');
|
||||
expectIdentifierToken(tokens[2], 9, 13, 'name');
|
||||
expectOperatorToken(tokens[3], 14, 15, '|');
|
||||
expectIdentifierToken(tokens[4], 16, 26, 'capitalize');
|
||||
expectOperatorToken(tokens[5], 26, 27, '}');
|
||||
expectStringToken(tokens[6], 27, 31, '!!!', StringTokenKind.TemplateLiteralEnd);
|
||||
});
|
||||
|
||||
it('should tokenize a template literal with a pipe inside a parenthesized interpolation', () => {
|
||||
const tokens: Token[] = lex('`hello ${(name | capitalize)}!!!`');
|
||||
expect(tokens.length).toBe(9);
|
||||
expectStringToken(tokens[0], 0, 7, 'hello ', StringTokenKind.TemplateLiteralPart);
|
||||
expectOperatorToken(tokens[1], 7, 9, '${');
|
||||
expectCharacterToken(tokens[2], 9, 10, '(');
|
||||
expectIdentifierToken(tokens[3], 10, 14, 'name');
|
||||
expectOperatorToken(tokens[4], 15, 16, '|');
|
||||
expectIdentifierToken(tokens[5], 17, 27, 'capitalize');
|
||||
expectCharacterToken(tokens[6], 27, 28, ')');
|
||||
expectOperatorToken(tokens[7], 28, 29, '}');
|
||||
expectStringToken(tokens[8], 29, 33, '!!!', StringTokenKind.TemplateLiteralEnd);
|
||||
});
|
||||
|
||||
it('should produce an error if a template literal is not terminated', () => {
|
||||
expectErrorToken(
|
||||
lex('`hello')[0],
|
||||
6,
|
||||
6,
|
||||
'Lexer Error: Unterminated template literal at column 6 in expression [`hello]',
|
||||
);
|
||||
});
|
||||
|
||||
it('should produce an error for an unterminated template literal with an interpolation', () => {
|
||||
const tokens: Token[] = lex('`hello ${name}!');
|
||||
expect(tokens.length).toBe(5);
|
||||
expectStringToken(tokens[0], 0, 7, 'hello ', StringTokenKind.TemplateLiteralPart);
|
||||
expectOperatorToken(tokens[1], 7, 9, '${');
|
||||
expectIdentifierToken(tokens[2], 9, 13, 'name');
|
||||
expectOperatorToken(tokens[3], 13, 14, '}');
|
||||
expectErrorToken(
|
||||
tokens[4],
|
||||
15,
|
||||
15,
|
||||
'Lexer Error: Unterminated template literal at column 15 in expression [`hello ${name}!]',
|
||||
);
|
||||
});
|
||||
|
||||
it('should produce an error for an unterminate template literal interpolation', () => {
|
||||
const tokens: Token[] = lex('`hello ${name!`');
|
||||
expect(tokens.length).toBe(5);
|
||||
expectStringToken(tokens[0], 0, 7, 'hello ', StringTokenKind.TemplateLiteralPart);
|
||||
expectOperatorToken(tokens[1], 7, 9, '${');
|
||||
expectIdentifierToken(tokens[2], 9, 13, 'name');
|
||||
expectOperatorToken(tokens[3], 13, 14, '!');
|
||||
expectErrorToken(
|
||||
tokens[4],
|
||||
15,
|
||||
15,
|
||||
'Lexer Error: Unterminated template literal at column 15 in expression [`hello ${name!`]',
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user