Skip to content

Commit 969c2e5

Browse files
jashkenasjashkenas
committed
a number of refactors to the Lexer. It should be a good bit clearer to read now.
1 parent bb2bf7c commit 969c2e5

2 files changed

Lines changed: 110 additions & 59 deletions

File tree

‎lib/lexer.js‎

Lines changed: 55 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
(function(){
2-
var ACCESSORS, ASSIGNMENT, BEFORE_WHEN, CALLABLE, CODE, COFFEE_KEYWORDS, COMMENT, COMMENT_CLEANER, HEREDOC, HEREDOC_INDENT, IDENTIFIER, JS, JS_CLEANER, JS_FORBIDDEN, JS_KEYWORDS, KEYWORDS, LAST_DENT, LAST_DENTS, Lexer, MULTILINER, MULTI_DENT, NOT_REGEX, NO_NEWLINE, NUMBER, OPERATOR, REGEX, RESERVED, Rewriter, STRING, STRING_NEWLINES, WHITESPACE;
2+
var ACCESSORS, ASSIGNMENT, BEFORE_WHEN, CALLABLE, CODE, COFFEE_KEYWORDS, COMMENT, COMMENT_CLEANER, HEREDOC, HEREDOC_INDENT, IDENTIFIER, JS, JS_CLEANER, JS_FORBIDDEN, JS_KEYWORDS, KEYWORDS, LAST_DENT, LAST_DENTS, Lexer, MULTILINER, MULTI_DENT, NOT_REGEX, NO_NEWLINE, NUMBER, OPERATOR, REGEX, RESERVED, Rewriter, STRING, STRING_NEWLINES, WHITESPACE, include;
33
if ((typeof process !== "undefined" && process !== null)) {
44
Rewriter = require('./rewriter').Rewriter;
55
} else {
@@ -60,17 +60,17 @@
6060
// Scan by attempting to match tokens one character at a time. Slow and steady.
6161
Lexer.prototype.tokenize = function tokenize(code) {
6262
this.code = code;
63-
// Cleanup code by remove extra line breaks, TODO: chomp
63+
// The remainder of the source code.
6464
this.i = 0;
65-
// Current character position we're parsing
65+
// Current character position we're parsing.
6666
this.line = 1;
6767
// The current line.
6868
this.indent = 0;
6969
// The current indent level.
7070
this.indents = [];
7171
// The stack of all indent levels we are currently within.
7272
this.tokens = [];
73-
// Collection of all parsed tokens in the form [:TOKEN_TYPE, value]
73+
// Collection of all parsed tokens in the form ['TOKEN_TYPE', value]
7474
while (this.i < this.code.length) {
7575
this.chunk = this.code.slice(this.i);
7676
this.extract_next_token();
@@ -117,25 +117,15 @@
117117
if (!((id = this.match(IDENTIFIER, 1)))) {
118118
return false;
119119
}
120-
if (this.value() === '::') {
121-
this.tag(1, 'PROTOTYPE_ACCESS');
122-
}
123-
if (this.value() === '.' && !(this.value(2) === '.')) {
124-
if (this.tag(2) === '?') {
125-
this.tag(1, 'SOAK_ACCESS');
126-
this.tokens.splice(-2, 1);
127-
} else {
128-
this.tag(1, 'PROPERTY_ACCESS');
129-
}
130-
}
120+
this.name_access_type();
131121
tag = 'IDENTIFIER';
132-
if (KEYWORDS.indexOf(id) >= 0 && !((ACCESSORS.indexOf(this.tag()) >= 0) && !this.prev().spaced)) {
122+
if (include(KEYWORDS, id) && !(include(ACCESSORS, this.tag(0)) && !this.prev().spaced)) {
133123
tag = id.toUpperCase();
134124
}
135-
if (RESERVED.indexOf(id) >= 0) {
136-
throw new Error('SyntaxError: Reserved word "' + id + '" on line ' + this.line);
125+
if (include(RESERVED, id)) {
126+
this.identifier_error(id);
137127
}
138-
if (tag === 'WHEN' && BEFORE_WHEN.indexOf(this.tag()) >= 0) {
128+
if (tag === 'WHEN' && include(BEFORE_WHEN, this.tag())) {
139129
tag = 'LEADING_WHEN';
140130
}
141131
this.token(tag, id);
@@ -166,13 +156,11 @@
166156
};
167157
// Matches heredocs, adjusting indentation to the correct level.
168158
Lexer.prototype.heredoc_token = function heredoc_token() {
169-
var doc, indent, match;
159+
var doc, match;
170160
if (!((match = this.chunk.match(HEREDOC)))) {
171161
return false;
172162
}
173-
doc = match[2] || match[4];
174-
indent = (doc.match(HEREDOC_INDENT) || ['']).sort()[0];
175-
doc = doc.replace(new RegExp("^" + indent, 'gm'), '').replace(MULTILINER, "\\n").replace(/"/g, '\\"');
163+
doc = this.sanitize_heredoc(match[2] || match[4]);
176164
this.token('STRING', '"' + doc + '"');
177165
this.line += this.count(match[1], "\n");
178166
this.i += match[1].length;
@@ -194,7 +182,7 @@
194182
if (!((regex = this.match(REGEX, 1)))) {
195183
return false;
196184
}
197-
if (NOT_REGEX.indexOf(this.tag()) >= 0) {
185+
if (include(NOT_REGEX, this.tag())) {
198186
return false;
199187
}
200188
this.token('REGEX', regex);
@@ -221,9 +209,9 @@
221209
}
222210
this.line += indent.match(MULTILINER).length;
223211
this.i += indent.length;
224-
next_character = this.chunk.match(MULTI_DENT)[4];
225212
prev = this.prev(2);
226213
size = indent.match(LAST_DENTS).reverse()[0].match(LAST_DENT)[1].length;
214+
next_character = this.chunk.match(MULTI_DENT)[4];
227215
no_newlines = next_character === '.' || (this.value() && this.value().match(NO_NEWLINE) && prev && (prev[0] !== '.') && !this.value().match(CODE));
228216
if (size === this.indent) {
229217
if (no_newlines) {
@@ -243,7 +231,7 @@
243231
this.indent = size;
244232
return true;
245233
};
246-
// Record an oudent token or tokens, if we're moving back inwards past
234+
// Record an outdent token or tokens, if we're moving back inwards past
247235
// multiple recorded indents.
248236
Lexer.prototype.outdent_token = function outdent_token(move_out, no_newlines) {
249237
var last_indent;
@@ -257,7 +245,8 @@
257245
}
258246
return true;
259247
};
260-
// Matches and consumes non-meaningful whitespace.
248+
// Matches and consumes non-meaningful whitespace. Tag the previous token
249+
// as being "spaced", because there are some cases where it matters.
261250
Lexer.prototype.whitespace_token = function whitespace_token() {
262251
var prev, space;
263252
if (!((space = this.match(WHITESPACE, 1)))) {
@@ -300,8 +289,8 @@
300289
tag = value;
301290
if (value.match(ASSIGNMENT)) {
302291
tag = 'ASSIGN';
303-
if (JS_FORBIDDEN.indexOf(this.value()) >= 0) {
304-
throw new Error('SyntaxError: Reserved word "' + this.value() + '" on line ' + this.line + ' can\'t be assigned');
292+
if (include(JS_FORBIDDEN, this.value)) {
293+
this.assignment_error();
305294
}
306295
} else if (value === ';') {
307296
tag = 'TERMINATOR';
@@ -312,7 +301,7 @@
312301
} else if (value === ']' && this.soaked_index) {
313302
tag = 'SOAKED_INDEX_END';
314303
this.soaked_index = false;
315-
} else if (CALLABLE.indexOf(this.tag()) >= 0 && not_spaced) {
304+
} else if (include(CALLABLE, this.tag()) && not_spaced) {
316305
if (value === '(') {
317306
tag = 'CALL_START';
318307
}
@@ -324,6 +313,37 @@
324313
this.i += value.length;
325314
return true;
326315
};
316+
// Token Manipulators ==================================================
317+
// As we consume a new IDENTIFIER, look at the previous token to determine
318+
// if it's a special kind of access.
319+
Lexer.prototype.name_access_type = function name_access_type() {
320+
if (this.value() === '::') {
321+
this.tag(1, 'PROTOTYPE_ACCESS');
322+
}
323+
if (this.value() === '.' && !(this.value(2) === '.')) {
324+
if (this.tag(2) === '?') {
325+
this.tag(1, 'SOAK_ACCESS');
326+
return this.tokens.splice(-2, 1);
327+
} else {
328+
return this.tag(1, 'PROPERTY_ACCESS');
329+
}
330+
}
331+
};
332+
// Sanitize a heredoc by escaping double quotes and erasing all external
333+
// indentation on the left-hand side.
334+
Lexer.prototype.sanitize_heredoc = function sanitize_heredoc(doc) {
335+
var indent;
336+
indent = (doc.match(HEREDOC_INDENT) || ['']).sort()[0];
337+
return doc.replace(new RegExp("^" + indent, 'gm'), '').replace(MULTILINER, "\\n").replace(/"/g, '\\"');
338+
};
339+
// When you try to use a forbidden word in JavaScript as an identifier.
340+
Lexer.prototype.identifier_error = function identifier_error(word) {
341+
throw new Error('SyntaxError: Reserved word "' + word + '" on line ' + this.line);
342+
};
343+
// When you try to assign to a reserved word in JavaScript, like "function".
344+
Lexer.prototype.assignment_error = function assignment_error() {
345+
throw new Error('SyntaxError: Reserved word "' + this.value() + '" on line ' + this.line + ' can\'t be assigned');
346+
};
327347
// Helpers =============================================================
328348
// Add a token to the results, taking note of the line number.
329349
Lexer.prototype.token = function token(tag, value) {
@@ -408,4 +428,9 @@
408428
};
409429
return Lexer;
410430
}).call(this);
431+
// Helper functions:
432+
// Does a list include a value?
433+
include = function include(list, value) {
434+
return list.indexOf(value) >= 0;
435+
};
411436
})();

‎src/lexer.coffee‎

Lines changed: 55 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -89,12 +89,12 @@ exports.Lexer: class Lexer
8989

9090
# Scan by attempting to match tokens one character at a time. Slow and steady.
9191
tokenize: (code) ->
92-
@code : code # Cleanup code by remove extra line breaks, TODO: chomp
93-
@i : 0 # Current character position we're parsing
94-
@line : 1 # The current line.
95-
@indent : 0 # The current indent level.
96-
@indents : [] # The stack of all indent levels we are currently within.
97-
@tokens : [] # Collection of all parsed tokens in the form [:TOKEN_TYPE, value]
92+
@code : code # The remainder of the source code.
93+
@i : 0 # Current character position we're parsing.
94+
@line : 1 # The current line.
95+
@indent : 0 # The current indent level.
96+
@indents : [] # The stack of all indent levels we are currently within.
97+
@tokens : [] # Collection of all parsed tokens in the form ['TOKEN_TYPE', value]
9898
while @i < @code.length
9999
@chunk: @code.slice(@i)
100100
@extract_next_token()
@@ -120,18 +120,12 @@ exports.Lexer: class Lexer
120120
# Matches identifying literals: variables, keywords, method names, etc.
121121
identifier_token: ->
122122
return false unless id: @match IDENTIFIER, 1
123-
@tag(1, 'PROTOTYPE_ACCESS') if @value() is '::'
124-
if @value() is '.' and not (@value(2) is '.')
125-
if @tag(2) is '?'
126-
@tag(1, 'SOAK_ACCESS')
127-
@tokens.splice(-2, 1)
128-
else
129-
@tag(1, 'PROPERTY_ACCESS')
123+
@name_access_type()
130124
tag: 'IDENTIFIER'
131-
tag: id.toUpperCase() if KEYWORDS.indexOf(id) >= 0 and
132-
not ((ACCESSORS.indexOf(@tag()) >= 0) and not @prev().spaced)
133-
throw new Error('SyntaxError: Reserved word "' + id + '" on line ' + @line) if RESERVED.indexOf(id) >= 0
134-
tag: 'LEADING_WHEN' if tag is 'WHEN' and BEFORE_WHEN.indexOf(@tag()) >= 0
125+
tag: id.toUpperCase() if include(KEYWORDS, id) and
126+
not (include(ACCESSORS, @tag(0)) and not @prev().spaced)
127+
@identifier_error id if include RESERVED, id
128+
tag: 'LEADING_WHEN' if tag is 'WHEN' and include BEFORE_WHEN, @tag()
135129
@token(tag, id)
136130
@i += id.length
137131
true
@@ -155,11 +149,7 @@ exports.Lexer: class Lexer
155149
# Matches heredocs, adjusting indentation to the correct level.
156150
heredoc_token: ->
157151
return false unless match = @chunk.match(HEREDOC)
158-
doc: match[2] or match[4]
159-
indent: (doc.match(HEREDOC_INDENT) or ['']).sort()[0]
160-
doc: doc.replace(new RegExp("^" + indent, 'gm'), '')
161-
.replace(MULTILINER, "\\n")
162-
.replace(/"/g, '\\"')
152+
doc: @sanitize_heredoc match[2] or match[4]
163153
@token 'STRING', '"' + doc + '"'
164154
@line += @count match[1], "\n"
165155
@i += match[1].length
@@ -175,7 +165,7 @@ exports.Lexer: class Lexer
175165
# Matches regular expression literals.
176166
regex_token: ->
177167
return false unless regex: @match REGEX, 1
178-
return false if NOT_REGEX.indexOf(@tag()) >= 0
168+
return false if include NOT_REGEX, @tag()
179169
@token 'REGEX', regex
180170
@i += regex.length
181171
true
@@ -194,10 +184,11 @@ exports.Lexer: class Lexer
194184
return false unless indent: @match MULTI_DENT, 1
195185
@line += indent.match(MULTILINER).length
196186
@i += indent.length
197-
next_character: @chunk.match(MULTI_DENT)[4]
198187
prev: @prev(2)
199188
size: indent.match(LAST_DENTS).reverse()[0].match(LAST_DENT)[1].length
200-
no_newlines: next_character is '.' or (@value() and @value().match(NO_NEWLINE) and prev and (prev[0] isnt '.') and not @value().match(CODE))
189+
next_character: @chunk.match(MULTI_DENT)[4]
190+
no_newlines: next_character is '.' or (@value() and @value().match(NO_NEWLINE) and
191+
prev and (prev[0] isnt '.') and not @value().match(CODE))
201192
if size is @indent
202193
return @suppress_newlines(indent) if no_newlines
203194
return @newline_token(indent)
@@ -211,7 +202,7 @@ exports.Lexer: class Lexer
211202
@indent: size
212203
true
213204

214-
# Record an oudent token or tokens, if we're moving back inwards past
205+
# Record an outdent token or tokens, if we're moving back inwards past
215206
# multiple recorded indents.
216207
outdent_token: (move_out, no_newlines) ->
217208
while move_out > 0 and @indents.length
@@ -221,7 +212,8 @@ exports.Lexer: class Lexer
221212
@token 'TERMINATOR', "\n" unless @tag() is 'TERMINATOR' or no_newlines
222213
true
223214

224-
# Matches and consumes non-meaningful whitespace.
215+
# Matches and consumes non-meaningful whitespace. Tag the previous token
216+
# as being "spaced", because there are some cases where it matters.
225217
whitespace_token: ->
226218
return false unless space: @match WHITESPACE, 1
227219
prev: @prev()
@@ -252,7 +244,7 @@ exports.Lexer: class Lexer
252244
tag: value
253245
if value.match(ASSIGNMENT)
254246
tag: 'ASSIGN'
255-
throw new Error('SyntaxError: Reserved word "' + @value() + '" on line ' + @line + ' can\'t be assigned') if JS_FORBIDDEN.indexOf(@value()) >= 0
247+
@assignment_error() if include JS_FORBIDDEN, @value
256248
else if value is ';'
257249
tag: 'TERMINATOR'
258250
else if value is '[' and @tag() is '?' and not_spaced
@@ -262,13 +254,42 @@ exports.Lexer: class Lexer
262254
else if value is ']' and @soaked_index
263255
tag: 'SOAKED_INDEX_END'
264256
@soaked_index: false
265-
else if CALLABLE.indexOf(@tag()) >= 0 and not_spaced
257+
else if include(CALLABLE, @tag()) and not_spaced
266258
tag: 'CALL_START' if value is '('
267259
tag: 'INDEX_START' if value is '['
268260
@token tag, value
269261
@i += value.length
270262
true
271263

264+
# Token Manipulators ==================================================
265+
266+
# As we consume a new IDENTIFIER, look at the previous token to determine
267+
# if it's a special kind of access.
268+
name_access_type: ->
269+
@tag(1, 'PROTOTYPE_ACCESS') if @value() is '::'
270+
if @value() is '.' and not (@value(2) is '.')
271+
if @tag(2) is '?'
272+
@tag(1, 'SOAK_ACCESS')
273+
@tokens.splice(-2, 1)
274+
else
275+
@tag 1, 'PROPERTY_ACCESS'
276+
277+
# Sanitize a heredoc by escaping double quotes and erasing all external
278+
# indentation on the left-hand side.
279+
sanitize_heredoc: (doc) ->
280+
indent: (doc.match(HEREDOC_INDENT) or ['']).sort()[0]
281+
doc.replace(new RegExp("^" + indent, 'gm'), '')
282+
.replace(MULTILINER, "\\n")
283+
.replace(/"/g, '\\"')
284+
285+
# When you try to use a forbidden word in JavaScript as an identifier.
286+
identifier_error: (word) ->
287+
throw new Error 'SyntaxError: Reserved word "' + word + '" on line ' + @line
288+
289+
# When you try to assign to a reserved word in JavaScript, like "function".
290+
assignment_error: ->
291+
throw new Error 'SyntaxError: Reserved word "' + @value() + '" on line ' + @line + ' can\'t be assigned'
292+
272293
# Helpers =============================================================
273294

274295
# Add a token to the results, taking note of the line number.
@@ -327,3 +348,8 @@ exports.Lexer: class Lexer
327348
# axe it.
328349
close_indentation: ->
329350
@outdent_token(@indent)
351+
352+
# Helper functions:
353+
354+
# Does a list include a value?
355+
include: (list, value) -> list.indexOf(value) >= 0

0 commit comments

Comments
 (0)