Skip to content

Commit

Permalink
Resolve issue #34
Browse files Browse the repository at this point in the history
  • Loading branch information
@ericmorand
ericmorand committed Nov 24, 2023
1 parent 6d36810 commit e520d2e
Show file tree
Hide file tree
Showing 8 changed files with 652 additions and 725 deletions.
98 changes: 49 additions & 49 deletions src/lib/Lexer.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import {Token} from "./Token";
import {TokenType} from "./TokenType";
import {SyntaxError} from "./SyntaxError";
import {TokenType} from "./TokenType";

enum LexerState {
COMMENT = 'COMMENT',
Expand Down Expand Up @@ -328,7 +328,7 @@ export class Lexer {
let match = this.statementStartRegExp.exec(data);

// push the text
this.pushToken(TokenType.TEXT, match ? data.substr(0, match.index) : data);
this.pushToken("TEXT", match ? data.substr(0, match.index) : data);

if (match) {
let tag: string = match[1];
Expand All @@ -339,7 +339,7 @@ export class Lexer {
this.currentVarBlockLine = this.line;
this.currentVarBlockColumn = this.column;

this.pushToken(TokenType.COMMENT_START, tag);
this.pushToken("COMMENT_START", tag);
this.pushModifier(modifier);
this.pushState(LexerState.COMMENT);
break;
Expand All @@ -348,38 +348,38 @@ export class Lexer {
this.currentVarBlockLine = this.line;
this.currentVarBlockColumn = this.column;

this.pushToken(TokenType.TAG_START, match[1]);
this.pushToken("TAG_START", match[1]);
this.pushModifier(match[2]);
this.pushToken(TokenType.WHITESPACE, match[3]);
this.pushToken(TokenType.NAME, match[4]); // verbatim itself
this.pushToken(TokenType.WHITESPACE, match[5]);
this.pushToken("WHITESPACE", match[3]);
this.pushToken("NAME", match[4]); // verbatim itself
this.pushToken("WHITESPACE", match[5]);
this.pushModifier(match[6]);
this.pushToken(TokenType.TAG_END, match[7]);
this.pushToken("TAG_END", match[7]);
this.pushState(LexerState.VERBATIM);
} else if ((match = this.lineTagRegExp.exec(this.source.substring(this.cursor))) !== null) {
this.pushToken(TokenType.TAG_START, match[1]);
this.pushToken("TAG_START", match[1]);

if (match[2].length > 0) {
this.pushToken(TokenType.WHITESPACE, match[2]);
this.pushToken("WHITESPACE", match[2]);
}

this.pushToken(TokenType.NAME, match[3]); // line itself
this.pushToken(TokenType.WHITESPACE, match[4]);
this.pushToken(TokenType.NUMBER, match[5]);
this.pushToken("NAME", match[3]); // line itself
this.pushToken("WHITESPACE", match[4]);
this.pushToken("NUMBER", match[5]);

if (match[6].length > 0) {
this.pushToken(TokenType.WHITESPACE, match[6]);
this.pushToken("WHITESPACE", match[6]);
}

this.pushToken(TokenType.TAG_END, match[7]);
this.pushToken("TAG_END", match[7]);

this.line = Number(match[5]);
this.column = 0;
} else {
this.currentVarBlockLine = this.line;
this.currentVarBlockColumn = this.column;

this.pushToken(TokenType.TAG_START, tag);
this.pushToken("TAG_START", tag);
this.pushModifier(modifier);
this.pushState(LexerState.TAG);
}
Expand All @@ -388,7 +388,7 @@ export class Lexer {
this.currentVarBlockLine = this.line;
this.currentVarBlockColumn = this.column;

this.pushToken(TokenType.VARIABLE_START, tag);
this.pushToken("VARIABLE_START", tag);
this.pushModifier(modifier);
this.pushState(LexerState.VARIABLE);

Expand All @@ -398,7 +398,7 @@ export class Lexer {
}
}

this.pushToken(TokenType.EOF, null);
this.pushToken("EOF", null);

if (this.state == LexerState.VARIABLE) {
throw new SyntaxError(`Unclosed variable opened at {${this.currentVarBlockLine}:${this.currentVarBlockColumn}}.`, this.line, this.column);
Expand All @@ -420,28 +420,28 @@ export class Lexer {

// test operator
if ((match = this.testOperatorRegExp.exec(candidate)) !== null) {
this.pushToken(TokenType.TEST_OPERATOR, match[0]);
this.pushToken("TEST_OPERATOR", match[0]);
}
// arrow
else if ((match = this.arrowOperatorRegExp.exec(candidate)) !== null) {
this.pushToken(TokenType.ARROW, match[0]);
this.pushToken("ARROW", match[0]);
}
// operator
else if ((match = this.operatorRegExp.exec(candidate)) !== null) {
this.pushToken(TokenType.OPERATOR, match[0]);
this.pushToken("OPERATOR", match[0]);
}
// name
else if ((match = this.nameRegExp.exec(candidate)) !== null) {
this.pushToken(TokenType.NAME, match[0]);
this.pushToken("NAME", match[0]);
}
// number
else if ((match = this.numberRegExp.exec(candidate)) !== null) {
this.pushToken(TokenType.NUMBER, match[0]);
this.pushToken("NUMBER", match[0]);
}
// opening bracket
else if (this.openingBracketRegExp.test(singleCharacterCandidate)) {
this.pushScope(singleCharacterCandidate);
this.pushToken(TokenType.PUNCTUATION, singleCharacterCandidate);
this.pushToken("PUNCTUATION", singleCharacterCandidate);
}
// closing bracket
else if (this.closingBracketRegExp.test(singleCharacterCandidate)) {
Expand All @@ -453,33 +453,33 @@ export class Lexer {
throw new SyntaxError(`Unclosed bracket "${this.scope.value}" opened at {${this.scope.line}:${this.scope.column}}.`, this.line, this.column);
}

this.pushToken(TokenType.PUNCTUATION, singleCharacterCandidate);
this.pushToken("PUNCTUATION", singleCharacterCandidate);
this.popScope();
}
// punctuation
else if (this.punctuationRegExp.test(singleCharacterCandidate)) {
this.pushToken(TokenType.PUNCTUATION, singleCharacterCandidate);
this.pushToken("PUNCTUATION", singleCharacterCandidate);
}
// string
else if ((match = this.stringRegExp.exec(candidate)) !== null) {
let openingBracket = match[1] || match[4];
let value = match[2] || match[5];
let closingBracket = match[3] || match[6];

this.pushToken(TokenType.OPENING_QUOTE, openingBracket);
this.pushToken("OPENING_QUOTE", openingBracket);

if (value !== undefined) {
this.pushToken(TokenType.STRING, value);
this.pushToken("STRING", value);
}

this.pushToken(TokenType.CLOSING_QUOTE, closingBracket);
this.pushToken("CLOSING_QUOTE", closingBracket);
}
// double quoted string
else if ((match = this.doubleQuotedStringDelimiterRegExp.exec(candidate)) !== null) {
let value = match[0];

this.pushScope(value, value);
this.pushToken(TokenType.OPENING_QUOTE, value);
this.pushToken("OPENING_QUOTE", value);
this.pushState(LexerState.DOUBLE_QUOTED_STRING);
}
// unlexable
Expand All @@ -499,7 +499,7 @@ export class Lexer {
let modifier = match[1] || match[3];

this.pushModifier(modifier);
this.pushToken(TokenType.TAG_END, tag);
this.pushToken("TAG_END", tag);
this.popState();
} else {
this.lexExpression();
Expand All @@ -513,7 +513,7 @@ export class Lexer {

if (!this.scope && ((match = this.variableEndRegExp.exec(this.source.substring(this.cursor))) !== null)) {
this.pushModifier(match[1]);
this.pushToken(TokenType.VARIABLE_END, match[2]);
this.pushToken("VARIABLE_END", match[2]);
this.popState();
} else {
this.lexExpression();
Expand All @@ -533,15 +533,15 @@ export class Lexer {

let text = this.source.substr(this.cursor, match.index);

this.pushToken(TokenType.TEXT, text);
this.pushToken("TEXT", text);

this.pushToken(TokenType.TAG_START, match[1]);
this.pushToken("TAG_START", match[1]);
this.pushModifier(match[2]);
this.pushToken(TokenType.WHITESPACE, match[3]);
this.pushToken(TokenType.NAME, match[4]); // endverbatim itself
this.pushToken(TokenType.WHITESPACE, match[5]);
this.pushToken("WHITESPACE", match[3]);
this.pushToken("NAME", match[4]); // endverbatim itself
this.pushToken("WHITESPACE", match[5]);
this.pushModifier(match[6]);
this.pushToken(TokenType.TAG_END, match[7]);
this.pushToken("TAG_END", match[7]);
this.popState();
}

Expand All @@ -550,7 +550,7 @@ export class Lexer {
let candidate: string = this.source.substring(this.cursor);

if ((match = this.whitespaceRegExp.exec(candidate)) !== null) {
this.pushToken(TokenType.WHITESPACE, match[0]);
this.pushToken("WHITESPACE", match[0]);
}
}

Expand All @@ -570,10 +570,10 @@ export class Lexer {
let modifier = match[2] || match[5];
let value = match[3] || match[6];

this.pushToken(TokenType.TEXT, text);
this.pushToken("TEXT", text);
this.lexWhitespace();
this.pushModifier(modifier);
this.pushToken(TokenType.COMMENT_END, value);
this.pushToken("COMMENT_END", value);
this.popState();
}

Expand All @@ -583,14 +583,14 @@ export class Lexer {
if ((match = this.interpolationStartRegExp.exec(this.source.substring(this.cursor))) !== null) {
let tag = match[1];

this.pushToken(TokenType.INTERPOLATION_START, tag);
this.pushToken(TokenType.WHITESPACE, match[2]);
this.pushToken("INTERPOLATION_START", tag);
this.pushToken("WHITESPACE", match[2]);
this.pushScope(tag, this.interpolationPair[1]);
this.pushState(LexerState.INTERPOLATION);
} else if (((match = this.doubleQuotedStringContentRegExp.exec(this.source.substring(this.cursor))) !== null) && (match[0].length > 0)) {
this.pushToken(TokenType.STRING, match[0]);
this.pushToken("STRING", match[0]);
} else {
this.pushToken(TokenType.CLOSING_QUOTE, this.scope.value);
this.pushToken("CLOSING_QUOTE", this.scope.value);
this.popScope();
this.popState();
}
Expand All @@ -603,8 +603,8 @@ export class Lexer {
let tag = match[2];
let whitespace = match[1] || '';

this.pushToken(TokenType.WHITESPACE, whitespace);
this.pushToken(TokenType.INTERPOLATION_END, tag);
this.pushToken("WHITESPACE", whitespace);
this.pushToken("INTERPOLATION_END", tag);
this.popScope();
this.popState();
} else {
Expand All @@ -627,7 +627,7 @@ export class Lexer {
}

private pushToken(type: TokenType, value: any) {
if ((type === TokenType.TEXT || type === TokenType.WHITESPACE) && (value.length < 1)) {
if ((type === "TEXT" || type === "WHITESPACE") && (value.length < 1)) {
return;
}

Expand All @@ -642,7 +642,7 @@ export class Lexer {

private pushModifier(modifier: string) {
if (modifier) {
this.pushToken(modifier === this.trimmingModifier ? TokenType.TRIMMING_MODIFIER : TokenType.LINE_TRIMMING_MODIFIER, modifier);
this.pushToken(modifier === this.trimmingModifier ? "TRIMMING_MODIFIER" : "LINE_TRIMMING_MODIFIER", modifier);
}
}

Expand Down
4 changes: 2 additions & 2 deletions src/lib/Token.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import {TokenType, typeToString} from "./TokenType";
import {TokenType} from "./TokenType";

export class Token {
private readonly _type: TokenType;
Expand Down Expand Up @@ -73,7 +73,7 @@ export class Token {
* @return {string}
*/
public toString(): string {
return `${typeToString(this.type, true)}(${this.value ? this.value : ''})`;
return `${this.type}(${this.value ? this.value : ''})`;
}

/**
Expand Down
40 changes: 20 additions & 20 deletions src/lib/TokenStream.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,48 +23,48 @@ type TokenVisitor = (token: Token, stream: TokenStream) => Token;
* @return {Token}
*/
export const astVisitor: TokenVisitor = (token: Token, stream: TokenStream): Token => {
if (!token.test(TokenType.WHITESPACE) &&
!token.test(TokenType.TRIMMING_MODIFIER) &&
!token.test(TokenType.LINE_TRIMMING_MODIFIER)) {
if (!token.test("WHITESPACE") &&
!token.test("TRIMMING_MODIFIER") &&
!token.test("LINE_TRIMMING_MODIFIER")) {
let tokenValue: string = token.value;
let tokenLine: number = token.line;
let tokenColumn: number = token.column;

if (token.test(TokenType.EOF)) {
if (token.test("EOF")) {
return token;
}

if (token.test(TokenType.NUMBER)) {
if (token.test("NUMBER")) {
return new Token(token.type, Number(token.value), token.line, token.column);
}

if (token.test(TokenType.OPENING_QUOTE)) {
if (token.test("OPENING_QUOTE")) {
let candidate = stream.look(1);

if (candidate.test(TokenType.CLOSING_QUOTE)) {
return new Token(TokenType.STRING, '', token.line, token.column);
if (candidate.test("CLOSING_QUOTE")) {
return new Token("STRING", '', token.line, token.column);
}
}

if (token.test(TokenType.STRING)) {
if (token.test("STRING")) {
let candidate = stream.look(-1);

if (candidate && candidate.test(TokenType.OPENING_QUOTE)) {
if (candidate && candidate.test("OPENING_QUOTE")) {
tokenLine = candidate.line;
tokenColumn = candidate.column;
}
}

if (!token.test(TokenType.OPENING_QUOTE) && !token.test(TokenType.CLOSING_QUOTE)) {
if (token.test(TokenType.TEXT) || token.test(TokenType.STRING)) {
if (!token.test("OPENING_QUOTE") && !token.test("CLOSING_QUOTE")) {
if (token.test("TEXT") || token.test("STRING")) {
// streamline line separators
tokenValue = tokenValue.replace(/\r\n|\r/g, '\n');
} else if (token.test(TokenType.OPERATOR)) {
} else if (token.test("OPERATOR")) {
// remove unnecessary operator spaces
tokenValue = tokenValue.replace(/\s+/, ' ');
}

if (token.test(TokenType.STRING)) {
if (token.test("STRING")) {
// strip C slashes
tokenValue = stripcslashes(tokenValue);
}
Expand All @@ -75,29 +75,29 @@ export const astVisitor: TokenVisitor = (token: Token, stream: TokenStream): Tok
wstCandidate = stream.look(2);

if (wstCandidate) {
if (wstCandidate.type === TokenType.TRIMMING_MODIFIER) {
if (wstCandidate.type === "TRIMMING_MODIFIER") {
tokenValue = tokenValue.replace(/\s*$/, '');
}

if (wstCandidate.type === TokenType.LINE_TRIMMING_MODIFIER) {
if (wstCandidate.type === "LINE_TRIMMING_MODIFIER") {
tokenValue = tokenValue.replace(/[ \t\0\x0B]*$/, '');
}
}

wstCandidate = stream.look(-2);

if (wstCandidate) {
if (wstCandidate.type === TokenType.TRIMMING_MODIFIER) {
if (wstCandidate.type === "TRIMMING_MODIFIER") {
tokenValue = tokenValue.replace(/^\s*/, '');
}

if (wstCandidate.type === TokenType.LINE_TRIMMING_MODIFIER) {
if (wstCandidate.type === "LINE_TRIMMING_MODIFIER") {
tokenValue = tokenValue.replace(/^[ \t\0\x0B]*/, '');
}
}

// don't push empty TEXT tokens
if (!token.test(TokenType.TEXT) || (tokenValue.length > 0)) {
// don't push empty "TEXT" tokens
if (!token.test("TEXT") || (tokenValue.length > 0)) {
return new Token(token.type, tokenValue, tokenLine, tokenColumn);
}
}
Expand Down
Loading

0 comments on commit e520d2e

Please sign in to comment.