skills/angular__angular
1
0
Fork 0
mirror of https://github.com/angular/angular.git synced 2026-09-14 13:54:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor(compiler): remove TokenError (#62160)

Browse Source 
Replaces the `TokenError` class with `ParseError` to reduce the number of error classes we need to maintain.

PR Close #62160
This commit is contained in:
Kristiyan Kostadinov
2025-06-20 15:00:11 +02:00
parent bcde744e0a
commit 982f90ff35
4 changed files with 44 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/compiler/src/ml_parser/lexer.ts
+10 -22
View File
@@ -21,20 +21,10 @@ import {
TokenType,
} from './tokens';
export class TokenError extends ParseError {
constructor(
errorMsg: string,
public tokenType: TokenType | null,
span: ParseSourceSpan,
) {
super(span, errorMsg);
}
}
export class TokenizeResult {
constructor(
public tokens: Token[],
public errors: TokenError[],
public errors: ParseError[],
public nonNormalizedIcuExpressions: Token[],
) {}
}
@@ -171,7 +161,7 @@ class _Tokenizer {
private readonly _tokenizeLet: boolean;
private readonly _selectorlessEnabled: boolean;
tokens: Token[] = [];
errors: TokenError[] = [];
errors: ParseError[] = [];
nonNormalizedIcuExpressions: Token[] = [];
/**
@@ -499,17 +489,15 @@ class _Tokenizer {
private _endToken(parts: string[], end?: CharacterCursor): Token {
if (this._currentTokenStart === null) {
throw new TokenError(
'Programming error - attempted to end a token when there was no start to the token',
this._currentTokenType,
throw new ParseError(
this._cursor.getSpan(end),
'Programming error - attempted to end a token when there was no start to the token',
);
}
if (this._currentTokenType === null) {
throw new TokenError(
'Programming error - attempted to end a token which has no token type',
null,
throw new ParseError(
this._cursor.getSpan(this._currentTokenStart),
'Programming error - attempted to end a token which has no token type',
);
}
const token = {
@@ -526,11 +514,11 @@ class _Tokenizer {
return token;
}
private _createError(msg: string, span: ParseSourceSpan): TokenError {
private _createError(msg: string, span: ParseSourceSpan): ParseError {
if (this._isInExpansionForm()) {
msg += ` (Do you have an unescaped "{" in your template? Use "{{ '{' }}") to escape it.)`;
}
const error = new TokenError(msg, this._currentTokenType, span);
const error = new ParseError(span, msg);
this._currentTokenStart = null;
this._currentTokenType = null;
return error;
@@ -540,7 +528,7 @@ class _Tokenizer {
if (e instanceof CursorError) {
e = this._createError(e.msg, this._cursor.getSpan(e.cursor));
}
if (e instanceof TokenError) {
if (e instanceof ParseError) {
this.errors.push(e);
} else {
throw e;
@@ -816,7 +804,7 @@ class _Tokenizer {
this._consumeTagOpenEnd();
}
} catch (e) {
if (e instanceof TokenError) {
if (e instanceof ParseError) {
if (openToken) {
// We errored before we could close the opening tag, so it is incomplete.
openToken.type =
packages/compiler/src/ml_parser/parser.ts
+1 -4
View File
@@ -79,10 +79,7 @@ export class Parser {
const tokenizeResult = tokenize(source, url, this.getTagDefinition, options);
const parser = new _TreeBuilder(tokenizeResult.tokens, this.getTagDefinition);
parser.build();
return new ParseTreeResult(
parser.rootNodes,
(tokenizeResult.errors as ParseError[]).concat(parser.errors),
);
return new ParseTreeResult(parser.rootNodes, [...tokenizeResult.errors, ...parser.errors]);
}
}
packages/compiler/test/ml_parser/html_parser_spec.ts
+2 -3
View File
@@ -770,7 +770,6 @@ describe('HtmlParser', () => {
});
expect(humanizeErrors(p.errors)).toEqual([
[
TokenType.RAW_TEXT,
'Unexpected character "EOF" (Do you have an unescaped "{" in your template? Use "{{ \'{\' }}") to escape it.)',
'0:36',
],
@@ -1857,7 +1856,7 @@ describe('HtmlParser', () => {
const errors = parser.parse('<!-err--><div></p></div>', 'TestComp').errors;
expect(errors.length).toEqual(2);
expect(humanizeErrors(errors)).toEqual([
[TokenType.COMMENT_START, 'Unexpected character "e"', '0:3'],
['Unexpected character "e"', '0:3'],
[
'p',
'Unexpected closing tag "p". It may happen when the tag has already been closed by another tag. For more info see https://www.w3.org/TR/html5/syntax.html#closing-elements-that-have-implied-end-tags',
@@ -1876,6 +1875,6 @@ export function humanizeErrors(errors: ParseError[]): any[] {
return [<any>e.elementName, e.msg, humanizeLineColumn(e.span.start)];
}
// Tokenizer errors
return [(<any>e).tokenType, e.msg, humanizeLineColumn(e.span.start)];
return [e.msg, humanizeLineColumn(e.span.start)];
});
}
packages/compiler/test/ml_parser/lexer_spec.ts
+31 -60
View File
@@ -7,9 +7,9 @@
*/
import {getHtmlTagDefinition} from '../../src/ml_parser/html_tags';
import {TokenError, tokenize, TokenizeOptions, TokenizeResult} from '../../src/ml_parser/lexer';
import {tokenize, TokenizeOptions, TokenizeResult} from '../../src/ml_parser/lexer';
import {Token, TokenType} from '../../src/ml_parser/tokens';
import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_util';
import {ParseError, ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_util';
describe('HtmlLexer', () => {
describe('line/column numbers', () => {
@@ -112,15 +112,11 @@ describe('HtmlLexer', () => {
});
it('should report <!- without -', () => {
expect(tokenizeAndHumanizeErrors('<!-a')).toEqual([
[TokenType.COMMENT_START, 'Unexpected character "a"', '0:3'],
]);
expect(tokenizeAndHumanizeErrors('<!-a')).toEqual([['Unexpected character "a"', '0:3']]);
});
it('should report missing end comment', () => {
expect(tokenizeAndHumanizeErrors('<!--')).toEqual([
[TokenType.RAW_TEXT, 'Unexpected character "EOF"', '0:4'],
]);
expect(tokenizeAndHumanizeErrors('<!--')).toEqual([['Unexpected character "EOF"', '0:4']]);
});
it('should accept comments finishing by too many dashes (even number)', () => {
@@ -158,9 +154,7 @@ describe('HtmlLexer', () => {
});
it('should report missing end doctype', () => {
expect(tokenizeAndHumanizeErrors('<!')).toEqual([
[TokenType.DOC_TYPE, 'Unexpected character "EOF"', '0:2'],
]);
expect(tokenizeAndHumanizeErrors('<!')).toEqual([['Unexpected character "EOF"', '0:2']]);
});
});
@@ -184,14 +178,12 @@ describe('HtmlLexer', () => {
});
it('should report <![ without CDATA[', () => {
expect(tokenizeAndHumanizeErrors('<![a')).toEqual([
[TokenType.CDATA_START, 'Unexpected character "a"', '0:3'],
]);
expect(tokenizeAndHumanizeErrors('<![a')).toEqual([['Unexpected character "a"', '0:3']]);
});
it('should report missing end cdata', () => {
expect(tokenizeAndHumanizeErrors('<![CDATA[')).toEqual([
[TokenType.RAW_TEXT, 'Unexpected character "EOF"', '0:9'],
['Unexpected character "EOF"', '0:9'],
]);
});
});
@@ -1069,7 +1061,6 @@ describe('HtmlLexer', () => {
tokenizeAndHumanizeErrors(`<p>before { after</p>`, {tokenizeExpansionForms: true}),
).toEqual([
[
TokenType.RAW_TEXT,
`Unexpected character "EOF" (Do you have an unescaped "{" in your template? Use "{{ '{' }}") to escape it.)`,
'0:21',
],
@@ -1083,7 +1074,6 @@ describe('HtmlLexer', () => {
}),
).toEqual([
[
TokenType.RAW_TEXT,
`Unexpected character "EOF" (Do you have an unescaped "{" in your template? Use "{{ '{' }}") to escape it.)`,
'0:56',
],
@@ -1095,7 +1085,7 @@ describe('HtmlLexer', () => {
const file = new ParseSourceFile(src, 'file://');
const location = new ParseLocation(file, 12, 123, 456);
const span = new ParseSourceSpan(location, location);
const error = new TokenError('**ERROR**', null!, span);
const error = new ParseError(span, '**ERROR**');
expect(error.toString()).toEqual(
`**ERROR** ("\n222\n333\n[ERROR ->]E\n444\n555\n"): file://@123:456`,
);
@@ -1192,15 +1182,15 @@ describe('HtmlLexer', () => {
it('should report an error on an invalid hex sequence', () => {
expect(tokenizeAndHumanizeErrors('\\xGG', {escapedString: true})).toEqual([
[null, 'Invalid hexadecimal escape sequence', '0:2'],
['Invalid hexadecimal escape sequence', '0:2'],
]);
expect(tokenizeAndHumanizeErrors('abc \\x xyz', {escapedString: true})).toEqual([
[TokenType.TEXT, 'Invalid hexadecimal escape sequence', '0:6'],
['Invalid hexadecimal escape sequence', '0:6'],
]);
expect(tokenizeAndHumanizeErrors('abc\\x', {escapedString: true})).toEqual([
[TokenType.TEXT, 'Unexpected character "EOF"', '0:5'],
['Unexpected character "EOF"', '0:5'],
]);
});
@@ -1213,7 +1203,7 @@ describe('HtmlLexer', () => {
it('should error on an invalid fixed length Unicode sequence', () => {
expect(tokenizeAndHumanizeErrors('\\uGGGG', {escapedString: true})).toEqual([
[null, 'Invalid hexadecimal escape sequence', '0:2'],
['Invalid hexadecimal escape sequence', '0:2'],
]);
});
@@ -1225,7 +1215,7 @@ describe('HtmlLexer', () => {
it('should error on an invalid variable length Unicode sequence', () => {
expect(tokenizeAndHumanizeErrors('\\u{GG}', {escapedString: true})).toEqual([
[null, 'Invalid hexadecimal escape sequence', '0:3'],
['Invalid hexadecimal escape sequence', '0:3'],
]);
});
@@ -1655,11 +1645,11 @@ describe('HtmlLexer', () => {
it('should report invalid quotes in a parameter', () => {
expect(tokenizeAndHumanizeErrors(`@foo (a === ") {hello}`)).toEqual([
[TokenType.BLOCK_PARAMETER, 'Unexpected character "EOF"', '0:22'],
['Unexpected character "EOF"', '0:22'],
]);
expect(tokenizeAndHumanizeErrors(`@foo (a === "hi') {hello}`)).toEqual([
[TokenType.BLOCK_PARAMETER, 'Unexpected character "EOF"', '0:25'],
['Unexpected character "EOF"', '0:25'],
]);
});
@@ -2017,7 +2007,7 @@ describe('HtmlLexer', () => {
it('should handle @let declaration with invalid syntax in the value', () => {
expect(tokenizeAndHumanizeErrors(`@let foo = ";`)).toEqual([
[TokenType.LET_VALUE, 'Unexpected character "EOF"', '0:13'],
['Unexpected character "EOF"', '0:13'],
]);
expect(tokenizeAndHumanizeParts(`@let foo = {a: 1,;`)).toEqual([
@@ -2381,13 +2371,13 @@ describe('HtmlLexer', () => {
it('should report missing closing single quote', () => {
expect(tokenizeAndHumanizeErrors("<t a='b>")).toEqual([
[TokenType.ATTR_VALUE_TEXT, 'Unexpected character "EOF"', '0:8'],
['Unexpected character "EOF"', '0:8'],
]);
});
it('should report missing closing double quote', () => {
expect(tokenizeAndHumanizeErrors('<t a="b>')).toEqual([
[TokenType.ATTR_VALUE_TEXT, 'Unexpected character "EOF"', '0:8'],
['Unexpected character "EOF"', '0:8'],
]);
});
});
@@ -2422,15 +2412,11 @@ describe('HtmlLexer', () => {
});
it('should report missing name after </', () => {
expect(tokenizeAndHumanizeErrors('</')).toEqual([
[TokenType.TAG_CLOSE, 'Unexpected character "EOF"', '0:2'],
]);
expect(tokenizeAndHumanizeErrors('</')).toEqual([['Unexpected character "EOF"', '0:2']]);
});
it('should report missing >', () => {
expect(tokenizeAndHumanizeErrors('</test')).toEqual([
[TokenType.TAG_CLOSE, 'Unexpected character "EOF"', '0:6'],
]);
expect(tokenizeAndHumanizeErrors('</test')).toEqual([['Unexpected character "EOF"', '0:6']]);
});
});
@@ -2475,39 +2461,27 @@ describe('HtmlLexer', () => {
it('should report malformed/unknown entities', () => {
expect(tokenizeAndHumanizeErrors('&tbo;')).toEqual([
[
TokenType.ENCODED_ENTITY,
'Unknown entity "tbo" - use the "&#<decimal>;" or "&#x<hex>;" syntax',
'0:0',
],
['Unknown entity "tbo" - use the "&#<decimal>;" or "&#x<hex>;" syntax', '0:0'],
]);
expect(tokenizeAndHumanizeErrors('&#3sdf;')).toEqual([
[
TokenType.ENCODED_ENTITY,
'Unable to parse entity "&#3s" - decimal character reference entities must end with ";"',
'0:4',
],
]);
expect(tokenizeAndHumanizeErrors('&#xasdf;')).toEqual([
[
TokenType.ENCODED_ENTITY,
'Unable to parse entity "&#xas" - hexadecimal character reference entities must end with ";"',
'0:5',
],
]);
expect(tokenizeAndHumanizeErrors('&#xABC')).toEqual([
[TokenType.ENCODED_ENTITY, 'Unexpected character "EOF"', '0:6'],
]);
expect(tokenizeAndHumanizeErrors('&#xABC')).toEqual([['Unexpected character "EOF"', '0:6']]);
});
it('should not parse js object methods', () => {
expect(tokenizeAndHumanizeErrors('&valueOf;')).toEqual([
[
TokenType.ENCODED_ENTITY,
'Unknown entity "valueOf" - use the "&#<decimal>;" or "&#x<hex>;" syntax',
'0:0',
],
['Unknown entity "valueOf" - use the "&#<decimal>;" or "&#x<hex>;" syntax', '0:0'],
]);
});
});
@@ -3327,7 +3301,6 @@ describe('HtmlLexer', () => {
tokenizeAndHumanizeErrors(`<p>before { after</p>`, {tokenizeExpansionForms: true}),
).toEqual([
[
TokenType.RAW_TEXT,
`Unexpected character "EOF" (Do you have an unescaped "{" in your template? Use "{{ '{' }}") to escape it.)`,
'0:21',
],
@@ -3341,7 +3314,6 @@ describe('HtmlLexer', () => {
}),
).toEqual([
[
TokenType.RAW_TEXT,
`Unexpected character "EOF" (Do you have an unescaped "{" in your template? Use "{{ '{' }}") to escape it.)`,
'0:56',
],
@@ -3353,7 +3325,7 @@ describe('HtmlLexer', () => {
const file = new ParseSourceFile(src, 'file://');
const location = new ParseLocation(file, 12, 123, 456);
const span = new ParseSourceSpan(location, location);
const error = new TokenError('**ERROR**', null!, span);
const error = new ParseError(span, '**ERROR**');
expect(error.toString()).toEqual(
`**ERROR** ("\n222\n333\n[ERROR ->]E\n444\n555\n"): file://@123:456`,
);
@@ -3450,15 +3422,15 @@ describe('HtmlLexer', () => {
it('should report an error on an invalid hex sequence', () => {
expect(tokenizeAndHumanizeErrors('\\xGG', {escapedString: true})).toEqual([
[null, 'Invalid hexadecimal escape sequence', '0:2'],
['Invalid hexadecimal escape sequence', '0:2'],
]);
expect(tokenizeAndHumanizeErrors('abc \\x xyz', {escapedString: true})).toEqual([
[TokenType.TEXT, 'Invalid hexadecimal escape sequence', '0:6'],
['Invalid hexadecimal escape sequence', '0:6'],
]);
expect(tokenizeAndHumanizeErrors('abc\\x', {escapedString: true})).toEqual([
[TokenType.TEXT, 'Unexpected character "EOF"', '0:5'],
['Unexpected character "EOF"', '0:5'],
]);
});
@@ -3471,7 +3443,7 @@ describe('HtmlLexer', () => {
it('should error on an invalid fixed length Unicode sequence', () => {
expect(tokenizeAndHumanizeErrors('\\uGGGG', {escapedString: true})).toEqual([
[null, 'Invalid hexadecimal escape sequence', '0:2'],
['Invalid hexadecimal escape sequence', '0:2'],
]);
});
@@ -3483,7 +3455,7 @@ describe('HtmlLexer', () => {
it('should error on an invalid variable length Unicode sequence', () => {
expect(tokenizeAndHumanizeErrors('\\u{GG}', {escapedString: true})).toEqual([
[null, 'Invalid hexadecimal escape sequence', '0:3'],
['Invalid hexadecimal escape sequence', '0:3'],
]);
});
@@ -3864,11 +3836,11 @@ describe('HtmlLexer', () => {
it('should report invalid quotes in a parameter', () => {
expect(tokenizeAndHumanizeErrors(`@foo (a === ") {hello}`)).toEqual([
[TokenType.BLOCK_PARAMETER, 'Unexpected character "EOF"', '0:22'],
['Unexpected character "EOF"', '0:22'],
]);
expect(tokenizeAndHumanizeErrors(`@foo (a === "hi') {hello}`)).toEqual([
[TokenType.BLOCK_PARAMETER, 'Unexpected character "EOF"', '0:25'],
['Unexpected character "EOF"', '0:25'],
]);
});
@@ -4121,7 +4093,6 @@ function tokenizeAndHumanizeFullStart(input: string, options?: TokenizeOptions):
function tokenizeAndHumanizeErrors(input: string, options?: TokenizeOptions): any[] {
return tokenize(input, 'someUrl', getHtmlTagDefinition, options).errors.map((e) => [
<any>e.tokenType,
e.msg,
humanizeLineColumn(e.span.start),
]);