size: 235 KiB

1--[=[===========================================================
2--=
3--= Dumb Lua Parser - Lua parsing library
4--= by Marcus 'ReFreezed' Thunström
5--=
6--= Tokenize Lua code or create ASTs (Abstract Syntax Trees)
7--= and convert the data back to Lua.
8--=
9--= Version: 2.3 (2022-06-23)
10--=
11--= License: MIT (see the bottom of this file)
12--= Website: http://refreezed.com/luaparser/
13--= Documentation: http://refreezed.com/luaparser/docs/
14--=
15--= Supported Lua versions: 5.1, 5.2, 5.3, 5.4, LuaJIT
16--=
17--==============================================================
18
191 - Usage
202 - API
21 2.1 - Functions
22 2.2 - Constants
23 2.3 - Settings
243 - Tokens
254 - AST
265 - Other Objects
27 5.1 - Stats
28 5.2 - Locations
296 - Notes
30
31
321 - Usage
33================================================================
34
35local parser = require("dumbParser")
36
37local tokens = parser.tokenizeFile("cool.lua")
38local ast = parser.parse(tokens)
39
40parser.simplify(ast)
41parser.printTree(ast)
42
43local lua = parser.toLua(ast, true)
44print(lua)
45
46
472 - API
48================================================================
49
50
512.1 - Functions
52----------------------------------------------------------------
53
54tokenize, tokenizeFile
55newToken, updateToken, cloneToken, concatTokens
56parse, parseExpression, parseFile
57newNode, newNodeFast, valueToAst, cloneNode, cloneTree, getChild, setChild, addChild, removeChild
58validateTree
59traverseTree, traverseTreeReverse
60updateReferences
61simplify, optimize, minify
62toLua
63printTokens, printNode, printTree
64formatMessage
65findDeclaredNames, findGlobalReferences, findShadows
66
67tokenize()
68 tokens = parser.tokenize( luaString [, pathForErrorMessages="?" ] )
69 tokens = parser.tokenize( luaString [, keepWhitespaceTokens=false, pathForErrorMessages="?" ] )
70 Convert a Lua string into an array of tokens. (See below for more info.)
71 Returns nil and a message on error.
72
73tokenizeFile()
74 tokens = parser.tokenizeFile( path [, keepWhitespaceTokens=false ] )
75 Convert the contents of a file into an array of tokens. (See below for more info.) Uses io.open().
76 Returns nil and a message on error.
77
78newToken()
79 token = parser.newToken( tokenType, tokenValue )
80 Create a new token. (See below or search for 'TokenCreation' for more info.)
81
82updateToken()
83 parser.updateToken( token, tokenValue )
84 Update the value and representation of an existing token. (Search for 'TokenModification' for more info.)
85
86cloneToken()
87 tokenClone = parser.cloneToken( token )
88 Clone an existing token.
89
90concatTokens()
91 luaString = parser.concatTokens( tokens )
92 Concatenate tokens. Whitespace is added between tokens when necessary.
93
94parse()
95 astNode = parser.parse( tokens )
96 astNode = parser.parse( luaString [, pathForErrorMessages="?" ] )
97 Convert tokens or Lua code into an AST representing a block of code. (See below for more info.)
98 Returns nil and a message on error.
99
100parseExpression()
101 astNode = parser.parseExpression( tokens )
102 astNode = parser.parseExpression( luaString [, pathForErrorMessages="?" ] )
103 Convert tokens or Lua code into an AST representing a value expression. (See below for more info.)
104 Returns nil and a message on error.
105
106parseFile()
107 astNode = parser.parseFile( path )
108 Convert a Lua file into an AST. (See below for more info.) Uses io.open().
109 Returns nil and a message on error.
110
111newNode()
112 astNode = parser.newNode( nodeType, arguments... )
113 Create a new AST node. (Search for 'NodeCreation' for more info.)
114
115newNodeFast()
116 astNode = parser.newNodeFast( nodeType, arguments... )
117 Same as newNode() but without any validation. (Search for 'NodeCreation' for more info.)
118
119valueToAst()
120 astNode = parser.valueToAst( value [, sortTableKeys=false ] )
121 Convert a Lua value (number, string, boolean, nil or table) to an AST.
122
123cloneNode()
124 astNode = parser.cloneNode( astNode )
125 Clone an existing AST node (but not any children).
126
127cloneTree()
128 astNode = parser.cloneTree( astNode )
129 Clone an existing AST node and its children.
130
131getChild()
132 childNode = parser.getChild( astNode, fieldName )
133 childNode = parser.getChild( astNode, fieldName, index ) -- If the node field is an array.
134 childNode = parser.getChild( astNode, fieldName, index, tableFieldKey ) -- If the node field is a table field array.
135 tableFieldKey = "key"|"value"
136 Get a child node. (Search for 'NodeFields' for field names.)
137
138 The result is the same as doing the following, but with more error checking:
139 childNode = astNode[fieldName]
140 childNode = astNode[fieldName][index]
141 childNode = astNode[fieldName][index][tableFieldKey]
142
143setChild()
144 parser.setChild( astNode, fieldName, childNode )
145 parser.setChild( astNode, fieldName, index, childNode ) -- If the node field is an array.
146 parser.setChild( astNode, fieldName, index, tableFieldKey, childNode ) -- If the node field is a table field array.
147 tableFieldKey = "key"|"value"
148 Set a child node. (Search for 'NodeFields' for field names.)
149
150 The result is the same as doing the following, but with more error checking:
151 astNode[fieldName] = childNode
152 astNode[fieldName][index] = childNode
153 astNode[fieldName][index][tableFieldKey] = childNode
154
155addChild()
156 parser.addChild( astNode, fieldName, [ index=atEnd, ] childNode )
157 parser.addChild( astNode, fieldName, [ index=atEnd, ] keyNode, valueNode ) -- If the node field is a table field array.
158 Add a child node to an array field. (Search for 'NodeFields' for field names.)
159
160 The result is the same as doing the following, but with more error checking:
161 table.insert(astNode[fieldName], index, childNode)
162 table.insert(astNode[fieldName], index, {key=keyNode, value=valueNode, generatedKey=false})
163
164removeChild()
165 parser.removeChild( astNode, fieldName [, index=last ] )
166 Remove a child node from an array field. (Search for 'NodeFields' for field names.)
167
168 The result is the same as doing the following, but with more error checking:
169 table.remove(astNode[fieldName], index)
170
171isExpression()
172 bool = parser.isExpression( astNode )
173 Returns true for expression nodes and false for statements.
174 Note that call nodes count as expressions for this function, i.e. return true.
175
176isStatement()
177 bool = parser.isStatement( astNode )
178 Returns true for statements and false for expression nodes.
179 Note that call nodes count as statements for this function, i.e. return true.
180
181validateTree()
182 isValid, errorMessages = parser.validateTree( astNode )
183 Check for errors in an AST (e.g. missing condition expressions for if statements).
184 errorMessages is a multi-line string if isValid is false.
185
186traverseTree()
187 didStop = parser.traverseTree( astNode, [ leavesFirst=false, ] callback [, topNodeParent=nil, topNodeContainer=nil, topNodeKey=nil ] )
188 action = callback( astNode, parent, container, key )
189 action = "stop"|"ignorechildren"|nil -- Returning nil (or nothing) means continue traversal.
190 Call a function on all nodes in an AST, going from astNode out to the leaf nodes (or from leaf nodes and inwards if leavesFirst is set).
191 container[key] is the position of the current node in the tree and can be used to replace the node.
192
193traverseTreeReverse()
194 didStop = parser.traverseTreeReverse( astNode, [ leavesFirst=false, ] callback [, topNodeParent=nil, topNodeContainer=nil, topNodeKey=nil ] )
195 action = callback( astNode, parent, container, key )
196 action = "stop"|"ignorechildren"|nil -- Returning nil (or nothing) means continue traversal.
197 Call a function on all nodes in reverse order in an AST, going from astNode out to the leaf nodes (or from leaf nodes and inwards if leavesFirst is set).
198 container[key] is the position of the current node in the tree and can be used to replace the node.
199
200updateReferences()
201 parser.updateReferences( astNode [, updateTopNodePositionInfo=true ] )
202 Update references between nodes in the tree.
203 This function sets 'parent'+'container'+'key' for all nodes, 'declaration' for identifiers and vararg nodes, and 'label' for goto nodes.
204 If 'updateTopNodePositionInfo' is false then 'parent', 'container' and 'key' will remain as-is for 'astNode' specifically.
205
206simplify()
207 stats = parser.simplify( astNode )
208 Simplify/fold expressions and statements involving constants ('1+2' becomes '3', 'false and func()' becomes 'false' etc.).
209 See the INT_SIZE constant for notes.
210 See below for more info about stats.
211
212optimize()
213 stats = parser.optimize( astNode )
214 Attempt to remove nodes that aren't useful, like unused variables, or variables that are essentially constants.
215 Calls simplify() internally.
216 This function can be quite slow!
217 See below for more info about stats.
218 Note: References may be out-of-date after calling this.
219
220minify()
221 stats = parser.minify( astNode [, optimize=false ] )
222 Replace local variable names with short names.
223 This function can be used to obfuscate the code to some extent.
224 If 'optimize' is set then optimize() is also called automatically.
225 See below for more info about stats.
226 Note: References may be out-of-date after calling this.
227
228toLua()
229 luaString = parser.toLua( astNode [, prettyOuput=false, nodeCallback ] )
230 nodeCallback = function( node, outputBuffer )
231 Convert an AST to Lua, optionally call a function on each node before they are turned into Lua.
232 Any node in the tree with a .pretty attribute will override the 'prettyOuput' flag for itself and its children.
233 Nodes can also have a .prefix and/or .suffix attribute with Lua code to output before/after the node (e.g. declaration.names[1].suffix="--[[foo]]").
234 outputBuffer is an array of Lua code that has been output so far.
235 Returns nil and a message on error.
236
237printTokens()
238 parser.printTokens( tokens )
239 Print tokens to stdout.
240
241printNode()
242 parser.printNode( astNode )
243 Print information about an AST node to stdout.
244
245printTree()
246 parser.printTree( astNode )
247 Print the structure of a whole AST to stdout.
248
249formatMessage()
250 message = parser.formatMessage( [ prefix="Info", ] token, formatString, ... )
251 message = parser.formatMessage( [ prefix="Info", ] astNode, formatString, ... )
252 message = parser.formatMessage( [ prefix="Info", ] location, formatString, ... )
253 Format a message to contain a code preview window with an arrow pointing at the target token, node or location.
254 This is used internally for formatting error messages.
255
256 -- Example:
257 if identifier.name ~= "good" then
258 print(parser.formatMessage("Error", identifier, "This identifier is not good!"))
259 print(parser.formatMessage(currentStatement, "Current statement."))
260 end
261
262findDeclaredNames()
263 identifiers = parser.findDeclaredNames( astNode )
264 Find all declared names in the tree (i.e. identifiers from AstDeclaration, AstFunction and AstFor nodes).
265
266findGlobalReferences()
267 identifiers = parser.findGlobalReferences( astNode )
268 Find all identifiers not referring to local variables in the tree.
269 Note: updateReferences() must be called at some point before you call this - otherwise all variables will be seen as globals!
270
271findShadows()
272 shadowSequences = parser.findShadows( astNode )
273 shadowSequences = { shadowSequence1, ... }
274 shadowSequence = { shadowingIdentifier, shadowedIdentifier1, ... }
275 Find local variable shadowing in the tree. Each shadowSequence is an array of declared identifiers where each identifier shadows the next one.
276 Note: updateReferences() must be called at some point before you call this - otherwise all variables will be seen as globals!
277 Note: Shadowing of globals cannot be detected by the function as that would require knowledge of all potential globals in your program. (See findGlobalReferences())
278
279
2802.2 - Constants
281----------------------------------------------------------------
282
283INT_SIZE, MAX_INT, MIN_INT
284VERSION
285
286INT_SIZE
287 parser.INT_SIZE = integer
288 How many bits integers have. In Lua 5.3 and later this is usually 64, and in earlier versions it's 32.
289 The int size may affect how bitwise operations involving only constants get simplified (see simplify()),
290 e.g. the expression '-1>>1' becomes 2147483647 in Lua 5.2 but 9223372036854775807 in Lua 5.3.
291
292MAX_INT
293 parser.MAX_INT = integer
294 The highest representable positive signed integer value, according to INT_SIZE.
295 This is the same value as math.maxinteger in Lua 5.3 and later.
296 This only affects simplification of some bitwise operations.
297
298MIN_INT
299 parser.MIN_INT = integer
300 The highest representable negative signed integer value, according to INT_SIZE.
301 This is the same value as math.mininteger in Lua 5.3 and later.
302 This only affects simplification of some bitwise operations.
303
304VERSION
305 parser.VERSION
306 The parser's version number (e.g. "1.0.2").
307
308
3092.3 - Settings
310----------------------------------------------------------------
311
312printIds, printLocations
313indentation
314constantNameReplacementStringMaxLength
315
316printIds
317 parser.printIds = bool
318 If AST node IDs should be printed. (All nodes gets assigned a unique ID when created.)
319 Default: false.
320
321printLocations
322 parser.printLocations = bool
323 If the file location (filename and line number) should be printed for each token or AST node.
324 Default: false.
325
326indentation
327 parser.indentation = string
328 The indentation used when printing ASTs (with printTree()).
329 Default: 4 spaces.
330
331constantNameReplacementStringMaxLength
332 parser.constantNameReplacementStringMaxLength = length
333 Normally optimize() replaces variable names that are effectively constants with their value.
334 The exception is if the value is a string that's longer than what this setting specifies.
335 Default: 200.
336
337 -- Example:
338 local ast = parser.parse[==[
339 local short = "a"
340 local long = "xy"
341 func(short, long)
342 ]==]
343 parser.constantNameReplacementStringMaxLength = 1
344 parser.optimize(ast)
345 print(parser.toLua(ast)) -- local long="xy";func("a",long);
346
347
3483 - Tokens
349================================================================
350
351Tokens are represented by tables.
352
353Token fields:
354
355 type -- Token type. (See below.)
356 value -- Token value. All token types have a string value, except "number" tokens which have a number value.
357 representation -- The token's code representation. (Strings have surrounding quotes, comments start with "--" etc.)
358
359 sourceString -- The original source string, or "" if there is none.
360 sourcePath -- Path to the source file, or "?" if there is none.
361
362 lineStart -- Start line number in sourceString, or 0 by default.
363 lineEnd -- End line number in sourceString, or 0 by default.
364 positionStart -- Start byte position in sourceString, or 0 by default.
365 positionEnd -- End byte position in sourceString, or 0 by default.
366
367Token types:
368
369 "comment" -- A comment.
370 "identifier" -- Word that is not a keyword.
371 "keyword" -- Lua keyword.
372 "number" -- Number literal.
373 "punctuation" -- Any punctuation, e.g. ".." or "(".
374 "string" -- String value.
375 "whitespace" -- Sequence of whitespace characters.
376
377
3784 - AST
379================================================================
380
381AST nodes are represented by tables.
382
383Node types:
384
385 "assignment" -- Assignment of one or more values to one or more variables.
386 "binary" -- Binary expression (operation with two operands, e.g. "+" or "and").
387 "block" -- List of statements. Blocks inside blocks are 'do...end' statements.
388 "break" -- Loop break statement.
389 "call" -- Function call.
390 "declaration" -- Declaration of one or more local variables, possibly with initial values.
391 "for" -- A 'for' loop.
392 "function" -- Anonymous function header and body.
393 "goto" -- A jump to a label.
394 "identifier" -- An identifier.
395 "if" -- If statement with a condition, a body if the condition is true, and possibly another body if the condition is false.
396 "label" -- Label for goto commands.
397 "literal" -- Number, string, boolean or nil literal.
398 "lookup" -- Field lookup on an object.
399 "repeat" -- A 'repeat' loop.
400 "return" -- Function/chunk return statement, possibly with values.
401 "table" -- Table constructor.
402 "unary" -- Unary expression (operation with one operand, e.g. "-" or "not").
403 "vararg" -- Vararg expression ("...").
404 "while" -- A 'while' loop.
405
406Node fields: (Search for 'NodeFields'.)
407
408
4095 - Other Objects
410================================================================
411
412
4135.1 - Stats
414----------------------------------------------------------------
415
416Some functions return a stats table which contains these fields:
417
418 nodeReplacements -- Array of locations. Locations of nodes that were replaced. (See below for location info.)
419 nodeRemovals -- Array of locations. Locations of nodes or tree branches that were removed. (See below for location info.)
420 nodeRemoveCount -- Number. How many nodes were removed, including subnodes of nodeRemovals.
421
422 renameCount -- Number. How many identifiers were renamed.
423 generatedNameCount -- Number. How many unique names were generated.
424
425
4265.2 - Locations
427----------------------------------------------------------------
428
429Locations are tables with these fields:
430
431 sourceString -- The original source string, or "" if there is none.
432 sourcePath -- Path to the source file, or "?" if there is none.
433
434 line -- Line number in sourceString, or 0 by default.
435 position -- Byte position in sourceString, or 0 by default.
436
437 node -- The node the location points to, or nil if there is none.
438 replacement -- The node that replaced 'node', or nil if there is none. (This is set for stats.nodeReplacements.)
439
440
4416 - Notes
442================================================================
443
444Special number notation rules.
445
446 The expression '-n' is parsed as a single number literal if 'n' is a
447 numeral (i.e. the result is a negative number).
448
449 The expression 'n/0' is parsed as a single number literal if 'n' is a
450 numeral. If 'n' is positive then the result is math.huge, if 'n' is
451 negative then the result is -math.huge, or if 'n' is 0 then the result is
452 NaN.
453
454
455-============================================================]=]
456
457local PARSER_VERSION = "2.3.0"
458
459local NORMALIZE_MINUS_ZERO, HANDLE_ENV -- Should HANDLE_ENV be a setting?
460do
461 local n = 0
462 NORMALIZE_MINUS_ZERO = tostring(-n) == "0" -- Lua 5.3+ normalizes -0 to 0.
463end
464do
465 local pcall = pcall
466 local _ENV = nil
467 HANDLE_ENV = not pcall(function() return _G end) -- Looking up the global _G will raise an error if _ENV is supported (Lua 5.2+).
468end
469
470local assert = assert
471local error = error
472local ipairs = ipairs
473local loadstring = loadstring or load
474local pairs = pairs
475local print = print
476local select = select
477local tonumber = tonumber
478local tostring = tostring
479local type = type
480
481local ioOpen = io.open
482local ioWrite = io.write
483
484local jit = jit
485
486local mathFloor = math.floor
487local mathMax = math.max
488local mathMin = math.min
489local mathType = math.type -- May be nil.
490
491local F = string.format
492local stringByte = string.byte
493local stringChar = string.char
494local stringFind = string.find
495local stringGmatch = string.gmatch
496local stringGsub = string.gsub
497local stringMatch = string.match
498local stringRep = string.rep
499local stringSub = string.sub
500
501local tableConcat = table.concat
502local tableInsert = table.insert
503local tableRemove = table.remove
504local tableSort = table.sort
505local tableUnpack = table.unpack or unpack
506
507local maybeWrapInt = (
508 (jit and function(n)
509 -- 'n' might be cdata (i.e. a 64-bit integer) here. We have to limit the range
510 -- with mod once before we convert it to a Lua number to not lose precision,
511 -- but the result might be negative (and still out of range somehow!) so we
512 -- have to use mod again. Gah!
513 return tonumber(n % 0x100000000) % 0x100000000 -- 0x100000000 == 2^32
514 end)
515 or (_VERSION == "Lua 5.2" and require"bit32".band)
516 or function(n) return n end
517)
518
519local parser
520
521
522
523local function newSet(values)
524 local set = {}
525 for _, v in ipairs(values) do
526 set[v] = true
527 end
528 return set
529end
530local function newCharSet(chars)
531 return newSet{ stringByte(chars, 1, #chars) }
532end
533
534local KEYWORDS = newSet{
535 "and", "break", "do", "else", "elseif", "end", "false", "for", "function", "goto", "if",
536 "in", "local", "nil", "not", "or", "repeat", "return", "then", "true", "until", "while",
537}
538local PUNCTUATION = newSet{
539 "+", "-", "*", "/", "%", "^", "#",
540 "&", "~", "|", "<<", ">>", "//",
541 "==", "~=", "<=", ">=", "<", ">", "=",
542 "(", ")", "{", "}", "[", "]", "::",
543 ";", ":", ",", ".", "..", "...",
544}
545local OPERATORS_UNARY = newSet{
546 "-", "not", "#", "~",
547}
548local OPERATORS_BINARY = newSet{
549 "+", "-", "*", "/", "//", "^", "%",
550 "&", "~", "|", ">>", "<<", "..",
551 "<", "<=", ">", ">=", "==", "~=",
552 "and", "or",
553}
554local OPERATOR_PRECEDENCE = {
555 ["or"] = 1,
556 ["and"] = 2,
557 ["<"] = 3, [">"] = 3, ["<="] = 3, [">="] = 3, ["~="] = 3, ["=="] = 3,
558 ["|"] = 4,
559 ["~"] = 5,
560 ["&"] = 6,
561 ["<<"] = 7, [">>"] = 7,
562 [".."] = 8,
563 ["+"] = 9, ["-"] = 9,
564 ["*"] = 10, ["/"] = 10, ["//"] = 10, ["%"] = 10,
565 unary = 11, -- "-", "not", "#", "~"
566 ["^"] = 12,
567}
568
569local EXPRESSION_NODES = newSet{ "binary", "call", "function", "identifier", "literal", "lookup", "table", "unary", "vararg" }
570local STATEMENT_NODES = newSet{ "assignment", "block", "break", "call", "declaration", "for", "goto", "if", "label", "repeat", "return", "while" }
571
572local TOKEN_BYTES = {
573 NAME_START = newCharSet"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_",
574 DASH = newCharSet"-",
575 NUM = newCharSet"0123456789",
576 QUOTE = newCharSet"\"'",
577 SQUARE = newCharSet"[",
578 DOT = newCharSet".",
579 PUNCT_TWO_CHARS = newCharSet".=~<>:/<>",
580 PUNCT_ONE_CHAR = newCharSet"+-*/%^#<>=(){}[];:,.&~|",
581}
582
583local NUMERAL_PATTERNS = {
584 HEX_FRAC_EXP = stringGsub("^( 0[Xx] (%x*) %.(%x+) [Pp]([-+]?%x+) )", " +", ""),
585 HEX_FRAC = stringGsub("^( 0[Xx] (%x*) %.(%x+) )", " +", ""),
586 HEX_EXP = stringGsub("^( 0[Xx] (%x+) %.? [Pp]([-+]?%x+) )", " +", ""),
587 HEX = stringGsub("^( 0[Xx] %x+ %.? )", " +", ""),
588 BIN = stringGsub("^( 0[Bb] [01]+ )", " +", ""),
589 DEC_FRAC_EXP = stringGsub("^( %d* %.%d+ [Ee][-+]?%d+ )", " +", ""),
590 DEC_FRAC = stringGsub("^( %d* %.%d+ )", " +", ""),
591 DEC_EXP = stringGsub("^( %d+ %.? [Ee][-+]?%d+ )", " +", ""),
592 DEC = stringGsub("^( %d+ %.? )", " +", ""),
593}
594
595local INT_SIZE, MAX_INT, MIN_INT
596do
597 local hex = F("%x", maybeWrapInt(-1))
598 INT_SIZE = #hex * 4 -- This should generally be 64 for Lua 5.3+ and 32 for earlier.
599 MAX_INT = math.maxinteger or tonumber(stringGsub(hex, "f", "7", 1), 16)
600 MIN_INT = math.mininteger or -MAX_INT-1
601end
602
603local nextSerialNumber = 1
604
605
606
607-- :NodeFields
608
609local function populateCommonNodeFields(token, node)
610 -- All nodes have these fields.
611 node.id = nextSerialNumber
612 nextSerialNumber = nextSerialNumber + 1
613
614 node.sourceString = token and token.sourceString or ""
615 node.sourcePath = token and token.sourcePath or "?"
616
617 node.token = token
618 node.line = token and token.lineStart or 0
619 node.position = token and token.positionStart or 0
620
621 -- These fields are set by updateReferences():
622 -- node.parent = nil -- Refers to the node's parent in the tree.
623 -- node.container = nil -- Refers to the specific table that the node is in, which could be the parent itself or a field in the parent.
624 -- node.key = nil -- Refers to the specific field in the container that the node is in (which is either a string or an integer).
625
626 -- toLua() uses these fields if present:
627 -- node.pretty = bool
628 -- node.prefix = luaString
629 -- node.suffix = luaString
630
631 return node
632end
633
634-- AST expressions.
635local function AstIdentifier (token,name)return populateCommonNodeFields(token,{
636 type = "identifier",
637 name = name, -- String.
638 attribute = "", -- "" | "close" | "const" -- Only used in declarations.
639 declaration = nil, -- AstIdentifier (whose parent is an AstDeclaration, AstFunction or AstFor). Updated by updateReferences(). This is nil for globals.
640})end
641local function AstVararg (token)return populateCommonNodeFields(token,{
642 type = "vararg",
643 declaration = nil, -- AstVararg (whose parent is an AstFunction). Updated by updateReferences(). This is nil in the main chunk (or in a non-vararg function, which is probably an error).
644 adjustToOne = false, -- True if parentheses surround the vararg.
645})end
646local function AstLiteral (token,v)return populateCommonNodeFields(token,{
647 type = "literal",
648 value = v, -- Number, string, boolean or nil.
649})end
650local function AstTable (token)return populateCommonNodeFields(token,{
651 type = "table",
652 fields = {}, -- Array of {key=expression, value=expression, generatedKey=bool}. generatedKey is true for implicit keys (i.e. {x,y}) and false for explicit keys (i.e. {a=x,b=y}). Note that the state of generatedKey affects the output of toLua()! 'key' may be nil if generatedKey is true.
653})end
654local function AstLookup (token)return populateCommonNodeFields(token,{
655 type = "lookup",
656 object = nil, -- Expression.
657 member = nil, -- Expression.
658})end
659local function AstUnary (token,op)return populateCommonNodeFields(token,{
660 type = "unary",
661 operator = op, -- "-" | "not" | "#" | "~"
662 expression = nil, -- Expression.
663})end
664local function AstBinary (token,op)return populateCommonNodeFields(token,{
665 type = "binary",
666 operator = op, -- "+" | "-" | "*" | "/" | "//" | "^" | "%" | "&" | "~" | "|" | ">>" | "<<" | ".." | "<" | "<=" | ">" | ">=" | "==" | "~=" | "and" | "or"
667 left = nil, -- Expression.
668 right = nil, -- Expression.
669})end
670local function AstCall (token)return populateCommonNodeFields(token,{ -- Calls can be both expressions and statements.
671 type = "call",
672 callee = nil, -- Expression.
673 arguments = {}, -- Array of expressions.
674 method = false, -- True if the call is a method call. Method calls must have a callee that is a lookup with a member expression that is a string literal that can pass as an identifier.
675 adjustToOne = false, -- True if parentheses surround the call.
676})end
677local function AstFunction (token)return populateCommonNodeFields(token,{
678 type = "function",
679 parameters = {}, -- Array of AstIdentifier and maybe an AstVararg at the end.
680 body = nil, -- AstBlock.
681})end
682
683-- AST statements.
684local function AstBreak (token)return populateCommonNodeFields(token,{
685 type = "break",
686})end
687local function AstReturn (token)return populateCommonNodeFields(token,{
688 type = "return",
689 values = {}, -- Array of expressions.
690})end
691local function AstLabel (token,name)return populateCommonNodeFields(token,{
692 type = "label",
693 name = name, -- String. The value must be able to pass as an identifier.
694})end
695local function AstGoto (token,name)return populateCommonNodeFields(token,{
696 type = "goto",
697 name = name, -- String. The value must be able to pass as an identifier.
698 label = nil, -- AstLabel. Updated by updateReferences().
699})end
700local function AstBlock (token)return populateCommonNodeFields(token,{
701 type = "block",
702 statements = {}, -- Array of statements.
703})end
704local function AstDeclaration (token)return populateCommonNodeFields(token,{
705 type = "declaration",
706 names = {}, -- Array of AstIdentifier.
707 values = {}, -- Array of expressions.
708})end
709local function AstAssignment (token)return populateCommonNodeFields(token,{
710 type = "assignment",
711 targets = {}, -- Mixed array of AstIdentifier and AstLookup.
712 values = {}, -- Array of expressions.
713})end
714local function AstIf (token)return populateCommonNodeFields(token,{
715 type = "if",
716 condition = nil, -- Expression.
717 bodyTrue = nil, -- AstBlock.
718 bodyFalse = nil, -- AstBlock or nil.
719})end
720local function AstWhile (token)return populateCommonNodeFields(token,{
721 type = "while",
722 condition = nil, -- Expression.
723 body = nil, -- AstBlock.
724})end
725local function AstRepeat (token)return populateCommonNodeFields(token,{
726 type = "repeat",
727 body = nil, -- AstBlock.
728 condition = nil, -- Expression.
729})end
730local function AstFor (token,kind)return populateCommonNodeFields(token,{
731 type = "for",
732 kind = kind, -- "numeric" | "generic"
733 names = {}, -- Array of AstIdentifier.
734 values = {}, -- Array of expressions.
735 body = nil, -- AstBlock.
736})end
737
738
739
740local CHILD_FIELDS = {
741 ["identifier"] = {},
742 ["vararg"] = {},
743 ["literal"] = {},
744 ["table"] = {fields="tablefields"},
745 ["lookup"] = {object="node", member="node"},
746 ["unary"] = {expressions="node"},
747 ["binary"] = {left="node", right="node"},
748 ["call"] = {callee="node", arguments="nodearray"},
749 ["function"] = {parameters="nodearray", body="node"},
750 ["break"] = {},
751 ["return"] = {values="nodearray"},
752 ["label"] = {},
753 ["goto"] = {},
754 ["block"] = {statements="nodearray"},
755 ["declaration"] = {names="nodearray", values="nodearray"},
756 ["assignment"] = {targets="nodearray", values="nodearray"},
757 ["if"] = {condition="node", bodyTrue="node", bodyFalse="node"},
758 ["while"] = {condition="node", body="node"},
759 ["repeat"] = {body="node", condition="node"},
760 ["for"] = {names="nodearray", values="nodearray", body="node"},
761}
762
763
764
765local function Stats()
766 return {
767 -- simplify() and optimize():
768 nodeReplacements = {--[[ location1, ... ]]},
769 nodeRemovals = {--[[ location1, ... ]]},
770 nodeRemoveCount = 0,
771
772 -- minify():
773 renameCount = 0,
774 generatedNameCount = 0,
775 }
776end
777
778-- location = Location( sourceLocation [, extraKey, extraValue ] )
779-- location = Location( sourceNode [, extraKey, extraValue ] )
780local function Location(sourceLocOrNode, extraK, extraV)
781 local loc = {
782 sourceString = sourceLocOrNode.sourceString,
783 sourcePath = sourceLocOrNode.sourcePath,
784
785 line = sourceLocOrNode.line,
786 position = sourceLocOrNode.position,
787
788 node = sourceLocOrNode.type and sourceLocOrNode or nil,
789 }
790 if extraK then
791 loc[extraK] = extraV
792 end
793 return loc
794end
795
796
797
798-- count = countString( haystack, needle [, plain=false ] )
799local function countString(haystack, needle, plain)
800 local count = 0
801 local pos = 1
802
803 while true do
804 local _, i2 = stringFind(haystack, needle, pos, plain)
805 if not i2 then return count end
806
807 count = count + 1
808 pos = i2 + 1
809 end
810end
811
812-- count = countSubString( haystack, startPosition, endPosition, needle [, plain=false ] )
813local function countSubString(haystack, pos, posEnd, needle, plain)
814 local count = 0
815
816 while true do
817 local _, i2 = stringFind(haystack, needle, pos, plain)
818 if not i2 or i2 > posEnd then return count end
819
820 count = count + 1
821 pos = i2 + 1
822 end
823end
824
825
826
827-- errorf( [ level=1, ] format, ... )
828local function errorf(level, s, ...)
829 if type(level) == "number" then
830 error(F(s, ...), (level == 0 and 0 or (1+level)))
831 else
832 error(F(level, s, ...), 2)
833 end
834end
835
836-- assertArg1( functionName, argumentNumber, value, expectedType [, level=2 ] )
837-- assertArg2( functionName, argumentNumber, value, expectedType1, expectedType2 [, level=2 ] )
838local function assertArg1(funcName, argNum, v, expectedType, level)
839 if type(v) == expectedType then return end
840 errorf(1+(level or 2), "Bad argument #%d to '%s'. (Expected %s, got %s)", argNum, funcName, expectedType, type(v))
841end
842local function assertArg2(funcName, argNum, v, expectedType1, expectedType2, level)
843 if type(v) == expectedType1 or type(v) == expectedType2 then return end
844 errorf(1+(level or 2), "Bad argument #%d to '%s'. (Expected %s or %s, got %s)", argNum, funcName, expectedType1, expectedType2, type(v))
845end
846
847
848
849local ensurePrintable
850do
851 local CONTROL_TO_READABLE = {
852 ["\0"] = "{NUL}",
853 ["\n"] = "{NL}",
854 ["\r"] = "{CR}",
855 }
856
857 --[[local]] function ensurePrintable(s)
858 return (stringGsub(s, "[%z\1-\31\127-\255]", function(c)
859 return CONTROL_TO_READABLE[c] or (stringByte(c) <= 31 or stringByte(c) >= 127) and F("{%d}", stringByte(c)) or nil
860 end))
861 end
862end
863
864
865
866local function removeUnordered(t, i)
867 local len = #t
868 if i > len or i < 1 then return end
869
870 -- Note: This does the correct thing if i==len too.
871 t[i] = t[len]
872 t[len] = nil
873end
874
875local function removeItemUnordered(t, v)
876 for i = 1, #t do
877 if t[i] == v then
878 removeUnordered(t, i)
879 return
880 end
881 end
882end
883
884
885
886local function getLineNumber(s, pos)
887 return 1 + countSubString(s, 1, pos-1, "\n", true)
888end
889
890
891
892local formatMessageInFile
893do
894 local function findStartOfLine(s, pos, canBeEmpty)
895 while pos > 1 do
896 if stringByte(s, pos-1) == 10--[[\n]] and (canBeEmpty or stringByte(s, pos) ~= 10--[[\n]]) then break end
897 pos = pos - 1
898 end
899 return mathMax(pos, 1)
900 end
901 local function findEndOfLine(s, pos)
902 while pos < #s do
903 if stringByte(s, pos+1) == 10--[[\n]] then break end
904 pos = pos + 1
905 end
906 return mathMin(pos, #s)
907 end
908
909 local function getSubTextLength(s, pos, posEnd)
910 local len = 0
911
912 while pos <= posEnd do
913 if stringByte(s, pos) == 9 then -- '\t'
914 len = len + 4
915 pos = pos + 1
916 else
917 local _, i2 = stringFind(s, "^[%z\1-\127\194-\253][\128-\191]*", pos)
918 if i2 and i2 <= posEnd then
919 len = len + 1
920 pos = i2 + 1
921 else
922 len = len + 1
923 pos = pos + 1
924 end
925 end
926 end
927
928 return len
929 end
930
931 --[[local]] function formatMessageInFile(prefix, contents, path, pos, agent, s, ...)
932 if agent ~= "" then
933 agent = "["..agent.."] "
934 end
935
936 s = F(s, ...)
937
938 if contents == "" then
939 return F("%s @ %s: %s%s", prefix, path, agent, s)
940 end
941
942 pos = mathMin(mathMax(pos, 1), #contents+1)
943 local ln = getLineNumber(contents, pos)
944
945 local lineStart = findStartOfLine(contents, pos, true)
946 local lineEnd = findEndOfLine (contents, pos-1)
947 local linePre1Start = findStartOfLine(contents, lineStart-1, false)
948 local linePre1End = findEndOfLine (contents, linePre1Start-1)
949 local linePre2Start = findStartOfLine(contents, linePre1Start-1, false)
950 local linePre2End = findEndOfLine (contents, linePre2Start-1)
951 -- print(F("pos %d | lines %d..%d, %d..%d, %d..%d", pos, linePre2Start,linePre2End+1, linePre1Start,linePre1End+1, lineStart,lineEnd+1)) -- DEBUG
952
953 return F("%s @ %s:%d: %s%s\n>\n%s%s%s>-%s^",
954 prefix, path, ln, agent, s,
955 (linePre2Start < linePre1Start and linePre2Start <= linePre2End) and F("> %s\n", (stringGsub(stringSub(contents, linePre2Start, linePre2End), "\t", " "))) or "",
956 (linePre1Start < lineStart and linePre1Start <= linePre1End) and F("> %s\n", (stringGsub(stringSub(contents, linePre1Start, linePre1End), "\t", " "))) or "",
957 ( lineStart <= lineEnd ) and F("> %s\n", (stringGsub(stringSub(contents, lineStart, lineEnd ), "\t", " "))) or ">\n",
958 stringRep("-", getSubTextLength(contents, lineStart, pos-1))
959 )
960 end
961end
962
963local function formatMessageAtToken(prefix, token, agent, s, ...)
964 return (formatMessageInFile(prefix, (token and token.sourceString or ""), (token and token.sourcePath or "?"), (token and token.positionStart or 0), agent, s, ...))
965end
966local function formatMessageAfterToken(prefix, token, agent, s, ...)
967 return (formatMessageInFile(prefix, (token and token.sourceString or ""), (token and token.sourcePath or "?"), (token and token.positionEnd+1 or 0), agent, s, ...))
968end
969
970local function formatMessageAtNode(prefix, node, agent, s, ...)
971 return (formatMessageInFile(prefix, node.sourceString, node.sourcePath, node.position, agent, s, ...))
972end
973
974local function formatMessageHelper(argNumOffset, prefix, nodeOrLocOrToken, s, ...)
975 assertArg1("formatMessage", 1+argNumOffset, prefix, "string", 3)
976 assertArg1("formatMessage", 2+argNumOffset, nodeOrLocOrToken, "table", 3)
977 assertArg1("formatMessage", 3+argNumOffset, s, "string", 3)
978
979 local formatter = nodeOrLocOrToken.representation and formatMessageAtToken or formatMessageAtNode
980 return (formatter(prefix, nodeOrLocOrToken, "", s, ...))
981end
982
983-- message = formatMessage( [ prefix="Info", ] token, s, ... )
984-- message = formatMessage( [ prefix="Info", ] astNode, s, ... )
985-- message = formatMessage( [ prefix="Info", ] location, s, ... )
986local function formatMessage(prefix, ...)
987 if type(prefix) == "string" then
988 return (formatMessageHelper(0, prefix, ...))
989 else
990 return (formatMessageHelper(-1, "Info", prefix, ...))
991 end
992
993end
994
995
996
997local function formatErrorInFile (...) return formatMessageInFile ("Error", ...) end
998local function formatErrorAtToken (...) return formatMessageAtToken ("Error", ...) end
999local function formatErrorAfterToken(...) return formatMessageAfterToken("Error", ...) end
1000local function formatErrorAtNode (...) return formatMessageAtNode ("Error", ...) end
1001
1002
1003
1004local function where(node, s, ...)
1005 if not node then
1006 print("[Where] No node here!")
1007 elseif s then
1008 print(formatMessageAtNode("Info", node, "Where", s, ...))
1009 else
1010 print(formatMessageAtNode("Info", node, "Where", "Here!"))
1011 end
1012end
1013
1014
1015
1016local function iprev(t, i)
1017 i = i-1
1018 local v = t[i]
1019
1020 if v ~= nil then return i, v end
1021end
1022
1023local function ipairsr(t)
1024 return iprev, t, #t+1
1025end
1026
1027
1028
1029-- index = indexOf( array, value [, startIndex=1, endIndex=#array ] )
1030local function indexOf(t, v, i1, i2)
1031 for i = (i1 or 1), (i2 or #t) do
1032 if t[i] == v then return i end
1033 end
1034 return nil
1035end
1036
1037-- item, index = itemWith1 ( array, key, value )
1038-- item, index = lastItemWith1( array, key, value )
1039local function itemWith1(t, k, v)
1040 for i, item in ipairs(t) do
1041 if item[k] == v then return item, i end
1042 end
1043 return nil
1044end
1045local function lastItemWith1(t, k, v)
1046 for i, item in ipairsr(t) do
1047 if item[k] == v then return item, i end
1048 end
1049 return nil
1050end
1051
1052
1053
1054-- text = getRelativeLocationText( sourcePathOfInterest, lineNumberOfInterest, otherSourcePath, otherLineNumber )
1055-- text = getRelativeLocationText( lineNumberOfInterest, otherLineNumber )
1056local function getRelativeLocationText(sourcePath, ln, otherSourcePath, otherLn)
1057 if type(sourcePath) ~= "string" then
1058 sourcePath, ln, otherSourcePath, otherLn = "", sourcePath, "", ln
1059 end
1060
1061 if not (ln > 0) then
1062 return "at <UnknownLocation>"
1063 end
1064
1065 if sourcePath ~= otherSourcePath then return F("at %s:%d", sourcePath, ln) end
1066 if ln+1 == otherLn and otherLn > 0 then return F("on the previous line") end
1067 if ln-1 == otherLn and otherLn > 0 then return F("on the next line") end
1068 if ln ~= otherLn then return F("on line %d", ln) end
1069 return "on the same line"
1070end
1071
1072-- text = getRelativeLocationTextForToken( tokens, tokenOfInterest, otherToken )
1073local function getRelativeLocationTextForToken(tokens, tokOfInterest, otherTok)
1074 return getRelativeLocationText((tokens[tokOfInterest] and tokens[tokOfInterest].lineStart or 0), (tokens[otherTok] and tokens[otherTok].lineStart or 0))
1075end
1076
1077--[[
1078-- text = getRelativeLocationTextForNode( nodeOfInterest, otherNode )
1079-- text = getRelativeLocationTextForNode( nodeOfInterest, otherSourcePath, otherLineNumber )
1080local function getRelativeLocationTextForNode(nodeOfInterest, otherSourcePath, otherLn)
1081 if type(otherSourcePath) == "table" then
1082 return getRelativeLocationTextForNode(nodeOfInterest, otherSourcePath.sourcePath, otherSourcePath.line)
1083 end
1084
1085 if not (nodeOfInterest.sourcePath ~= "?" and nodeOfInterest.line > 0) then
1086 return "at <UnknownLocation>"
1087 end
1088
1089 return getRelativeLocationText(nodeOfInterest.sourcePath, nodeOfInterest.line, otherSourcePath, otherLn)
1090end
1091]]
1092
1093
1094
1095local function formatNumber(n)
1096 -- @Speed: Cache!
1097
1098 -- 64-bit int in LuaJIT (is what we assume, anyway).
1099 if jit and type(n) == "cdata" then
1100 local nStr = tostring(n)
1101
1102 if stringFind(nStr, "i$") then
1103 if stringFind(nStr, "^0[-+]") then
1104 nStr = stringGsub(nStr, "^0%+?", "")
1105 else
1106 --
1107 -- LuaJIT doesn't seem to be able to parse nStr if we output it as-is.
1108 -- What is even the notation for complex numbers with a non-zero real part?
1109 -- Oh LuaJIT, you're so mysterious...
1110 --
1111 -- @Robustness: Make sure we don't choke when trying to simplify() complex numbers.
1112 --
1113 errorf(2, "Cannot output complex number '%s'.", nStr)
1114 end
1115 end
1116
1117 return nStr
1118 end
1119
1120 -- Int (including 64-bit ints in Lua 5.3+, and excluding whole floats).
1121 if n == mathFloor(n) and not (mathType and mathType(n) == "float") then
1122 local nStr = F("%.0f", n)
1123 if tonumber(nStr) == n then return nStr end
1124 end
1125
1126 -- Anything else.
1127 return (tostring(n)
1128 :gsub("(e[-+])0+(%d+)$", "%1%2") -- Remove unnecessary zeroes after 'e'.
1129 :gsub("e%+", "e" ) -- Remove plus after 'e'.
1130 )
1131end
1132
1133
1134
1135local tokenize
1136do
1137 local ERROR_UNFINISHED_VALUE = {}
1138
1139 -- success, equalSignCountIfLong|errorCode, ptr = parseStringlikeToken( s, ptr )
1140 local function parseStringlikeToken(s, ptr)
1141 local longEqualSigns = stringMatch(s, "^%[(=*)%[", ptr)
1142 local equalSignCountIfLong = longEqualSigns and #longEqualSigns
1143
1144 -- Single line (comment).
1145 if not equalSignCountIfLong then
1146 local i1, i2 = stringFind(s, "\n", ptr)
1147 ptr = i2 and i2 + 1 or #s + 1
1148
1149 -- Multiline.
1150 else
1151 ptr = ptr + 1 + #longEqualSigns + 1
1152
1153 local i1, i2 = stringFind(s, "%]"..longEqualSigns.."%]", ptr)
1154 if not i1 then
1155 return false, ERROR_UNFINISHED_VALUE, 0
1156 end
1157
1158 ptr = i2 + 1
1159 end
1160
1161 return true, equalSignCountIfLong, ptr
1162 end
1163
1164 local function codepointToString(cp, buffer)
1165 if cp < 0 or cp > 0x10ffff then
1166 -- This error is actually incorrect as Lua supports codepoints up to 2^31.
1167 -- This is probably an issue that no one will ever encounter!
1168 return false, F("Codepoint 0x%X (%.0f) is outside the valid range (0..10FFFF).", cp, cp)
1169 end
1170
1171 if cp < 128 then
1172 tableInsert(buffer, stringChar(cp))
1173 return true
1174 end
1175
1176 local suffix = cp % 64
1177 local c4 = 128 + suffix
1178 cp = (cp - suffix) / 64
1179
1180 if cp < 32 then
1181 tableInsert(buffer, stringChar(192+cp))
1182 tableInsert(buffer, stringChar(c4))
1183 return true
1184 end
1185
1186 suffix = cp % 64
1187 local c3 = 128 + suffix
1188 cp = (cp - suffix) / 64
1189
1190 if cp < 16 then
1191 tableInsert(buffer, stringChar(224+cp))
1192 tableInsert(buffer, stringChar(c3))
1193 tableInsert(buffer, stringChar(c4))
1194 return true
1195 end
1196
1197 suffix = cp % 64
1198 cp = (cp - suffix) / 64
1199
1200 tableInsert(buffer, stringChar(240+cp))
1201 tableInsert(buffer, stringChar(128+suffix))
1202 tableInsert(buffer, stringChar(c3))
1203 tableInsert(buffer, stringChar(c4))
1204 return true
1205 end
1206
1207 local function parseStringContents(s, path, ptrStart, ptrEnd)
1208 local ptr = ptrStart
1209 local buffer = {}
1210
1211 while ptr <= ptrEnd do
1212 local i1 = stringFind(s, "\\", ptr, true)
1213 if not i1 or i1 > ptrEnd then break end
1214
1215 if i1 > ptr then
1216 tableInsert(buffer, stringSub(s, ptr, i1-1))
1217 end
1218 ptr = i1 + 1
1219
1220 -- local b1, b2, b3 = stringByte(s, ptr, ptr+2)
1221
1222 if stringFind(s, "^a", ptr) then tableInsert(buffer, "\a") ; ptr = ptr + 1
1223 elseif stringFind(s, "^b", ptr) then tableInsert(buffer, "\b") ; ptr = ptr + 1
1224 elseif stringFind(s, "^t", ptr) then tableInsert(buffer, "\t") ; ptr = ptr + 1
1225 elseif stringFind(s, "^n", ptr) then tableInsert(buffer, "\n") ; ptr = ptr + 1
1226 elseif stringFind(s, "^v", ptr) then tableInsert(buffer, "\v") ; ptr = ptr + 1
1227 elseif stringFind(s, "^f", ptr) then tableInsert(buffer, "\f") ; ptr = ptr + 1
1228 elseif stringFind(s, "^r", ptr) then tableInsert(buffer, "\r") ; ptr = ptr + 1
1229 elseif stringFind(s, "^\\",ptr) then tableInsert(buffer, "\\") ; ptr = ptr + 1
1230 elseif stringFind(s, '^"', ptr) then tableInsert(buffer, "\"") ; ptr = ptr + 1
1231 elseif stringFind(s, "^'", ptr) then tableInsert(buffer, "\'") ; ptr = ptr + 1
1232 elseif stringFind(s, "^\n",ptr) then tableInsert(buffer, "\n") ; ptr = ptr + 1
1233
1234 elseif stringFind(s, "^z", ptr) then
1235 local i1, i2 = stringFind(s, "^%s*", ptr+1)
1236 ptr = i2 + 1
1237
1238 elseif stringFind(s, "^%d", ptr) then
1239 local nStr = stringMatch(s, "^%d%d?%d?", ptr)
1240 local byte = tonumber(nStr)
1241
1242 if byte > 255 then
1243 return nil, formatErrorInFile(
1244 s, path, ptr, "Tokenizer",
1245 "Byte value '%s' is out-of-range in decimal escape sequence. (String starting %s)",
1246 nStr, getRelativeLocationText(getLineNumber(s, ptrStart), getLineNumber(s, ptr))
1247 )
1248 end
1249
1250 tableInsert(buffer, stringChar(byte))
1251 ptr = ptr + #nStr
1252
1253 elseif stringFind(s, "^x%x%x", ptr) then
1254 local hexStr = stringSub(s, ptr+1, ptr+2)
1255 local byte = tonumber(hexStr, 16)
1256
1257 tableInsert(buffer, stringChar(byte))
1258 ptr = ptr + 3
1259
1260 elseif stringFind(s, "^u{%x+}", ptr) then
1261 local hexStr = stringMatch(s, "^%x+", ptr+2)
1262 local cp = tonumber(hexStr, 16)
1263
1264 local ok, err = codepointToString(cp, buffer)
1265 if not ok then
1266 return nil, formatErrorInFile(
1267 s, path, ptr+2, "Tokenizer",
1268 "%s (String starting %s)",
1269 err, getRelativeLocationText(getLineNumber(s, ptrStart), getLineNumber(s, ptr))
1270 )
1271 end
1272
1273 ptr = ptr + 3 + #hexStr
1274
1275 else
1276 return nil, formatErrorInFile(
1277 s, path, ptr-1, "Tokenizer",
1278 "Invalid escape sequence. (String starting %s)",
1279 getRelativeLocationText(getLineNumber(s, ptrStart), getLineNumber(s, ptr))
1280 )
1281 end
1282
1283 end
1284
1285 if ptr <= ptrEnd then
1286 tableInsert(buffer, stringSub(s, ptr, ptrEnd))
1287 end
1288
1289 return tableConcat(buffer)
1290 end
1291
1292 -- tokens, error = tokenize( luaString [, keepWhitespaceTokens=false ] [, pathForErrorMessages="?" ] )
1293 --[[local]] function tokenize(s, keepWhitespaceTokens, path)
1294 assertArg1("tokenize", 1, s, "string")
1295
1296 if type(keepWhitespaceTokens) == "string" then
1297 path = keepWhitespaceTokens
1298 keepWhitespaceTokens = false
1299 else
1300 assertArg2("tokenize", 2, keepWhitespaceTokens, "boolean","nil")
1301 assertArg2("tokenize", 3, path, "string","nil")
1302 end
1303
1304 if stringFind(s, "\r", 1, true) then
1305 s = stringGsub(s, "\r\n?", "\n")
1306 end
1307 path = path or "?"
1308
1309 local tokens = {}
1310 local count = 0
1311 local ptr = 1
1312 local ln = 1
1313
1314 local BYTES_NAME_START = TOKEN_BYTES.NAME_START
1315 local BYTES_DASH = TOKEN_BYTES.DASH
1316 local BYTES_NUM = TOKEN_BYTES.NUM
1317 local BYTES_QUOTE = TOKEN_BYTES.QUOTE
1318 local BYTES_SQUARE = TOKEN_BYTES.SQUARE
1319 local BYTES_DOT = TOKEN_BYTES.DOT
1320 local BYTES_PUNCT_TWO_CHARS = TOKEN_BYTES.PUNCT_TWO_CHARS
1321 local BYTES_PUNCT_ONE_CHAR = TOKEN_BYTES.PUNCT_ONE_CHAR
1322
1323 local NUM_HEX_FRAC_EXP = NUMERAL_PATTERNS.HEX_FRAC_EXP
1324 local NUM_HEX_FRAC = NUMERAL_PATTERNS.HEX_FRAC
1325 local NUM_HEX_EXP = NUMERAL_PATTERNS.HEX_EXP
1326 local NUM_HEX = NUMERAL_PATTERNS.HEX
1327 local NUM_BIN = NUMERAL_PATTERNS.BIN
1328 local NUM_DEC_FRAC_EXP = NUMERAL_PATTERNS.DEC_FRAC_EXP
1329 local NUM_DEC_FRAC = NUMERAL_PATTERNS.DEC_FRAC
1330 local NUM_DEC_EXP = NUMERAL_PATTERNS.DEC_EXP
1331 local NUM_DEC = NUMERAL_PATTERNS.DEC
1332
1333 while true do
1334 local i1, i2 = stringFind(s, "^[ \t\n]+", ptr)
1335
1336 if i1 then
1337 if keepWhitespaceTokens then
1338 local lnStart = ln
1339 local tokRepr = stringSub(s, i1, i2)
1340 ln = ln + countString(tokRepr, "\n", true)
1341 count = count + 1
1342
1343 tokens[count] = {
1344 type = "whitespace",
1345 value = tokRepr,
1346 representation = tokRepr,
1347
1348 sourceString = s,
1349 sourcePath = path,
1350
1351 lineStart = lnStart,
1352 lineEnd = ln,
1353 positionStart = i1,
1354 positionEnd = i2,
1355 }
1356
1357 else
1358 ln = ln + countSubString(s, i1, i2, "\n", true)
1359 end
1360
1361 ptr = i2 + 1
1362 end
1363
1364 if ptr > #s then break end
1365
1366 local ptrStart = ptr
1367 local lnStart = ln
1368 local b1, b2 = stringByte(s, ptr, ptr+1)
1369 local tokType, tokRepr, tokValue
1370
1371 -- Identifier/keyword.
1372 if BYTES_NAME_START[b1] then
1373 local i1, i2, word = stringFind(s, "^(.[%w_]*)", ptr)
1374 ptr = i2+1
1375 tokType = KEYWORDS[word] and "keyword" or "identifier"
1376 tokRepr = stringSub(s, ptrStart, ptr-1)
1377 tokValue = tokRepr
1378
1379 -- Comment.
1380 elseif BYTES_DASH[b1] and BYTES_DASH[b2] then
1381 ptr = ptr + 2
1382
1383 local ok, equalSignCountIfLong
1384 ok, equalSignCountIfLong, ptr = parseStringlikeToken(s, ptr)
1385
1386 if not ok then
1387 local errCode = equalSignCountIfLong
1388 if errCode == ERROR_UNFINISHED_VALUE then
1389 return nil, formatErrorInFile(s, path, ptrStart, "Tokenizer", "Unfinished long comment.")
1390 else
1391 return nil, formatErrorInFile(s, path, ptrStart, "Tokenizer", "Invalid comment.")
1392 end
1393 end
1394
1395 -- Check for nesting of [[...]] which is deprecated in Lua. Sigh...
1396 if equalSignCountIfLong and equalSignCountIfLong == 0 then
1397 local pos = stringFind(s, "[[", ptrStart+4, true)
1398 if pos and pos < ptr then
1399 return nil, formatErrorInFile(s, path, pos, "Tokenizer", "Cannot have nested comments. (Comment starting %s)", getRelativeLocationText(lnStart, getLineNumber(s, pos)))
1400 end
1401 end
1402
1403 tokType = "comment"
1404 tokRepr = stringSub(s, ptrStart, ptr-1)
1405 tokRepr = equalSignCountIfLong and tokRepr or (stringFind(tokRepr, "\n$") and tokRepr or tokRepr.."\n") -- Make sure there's a newline at the end of single-line comments. (It may be missing if we've reached EOF.)
1406 tokValue = equalSignCountIfLong and stringSub(tokRepr, 5+equalSignCountIfLong, -3-equalSignCountIfLong) or stringSub(tokRepr, 3, -2)
1407
1408 -- Number.
1409 elseif BYTES_NUM[b1] or (BYTES_DOT[b1] and BYTES_NUM[b2]) then
1410 local pat, maybeInt, kind, i1, i2, numStr = NUM_HEX_FRAC_EXP, false, "lua52hex", stringFind(s, NUM_HEX_FRAC_EXP, ptr)
1411 if not i1 then pat, maybeInt, kind, i1, i2, numStr = NUM_HEX_FRAC, false, "lua52hex", stringFind(s, NUM_HEX_FRAC, ptr)
1412 if not i1 then pat, maybeInt, kind, i1, i2, numStr = NUM_HEX_EXP, false, "lua52hex", stringFind(s, NUM_HEX_EXP, ptr)
1413 if not i1 then pat, maybeInt, kind, i1, i2, numStr = NUM_HEX, true, "", stringFind(s, NUM_HEX, ptr)
1414 if not i1 then pat, maybeInt, kind, i1, i2, numStr = NUM_BIN, true, "binary", stringFind(s, NUM_BIN, ptr) -- LuaJIT supports these, so why not.
1415 if not i1 then pat, maybeInt, kind, i1, i2, numStr = NUM_DEC_FRAC_EXP, false, "", stringFind(s, NUM_DEC_FRAC_EXP, ptr)
1416 if not i1 then pat, maybeInt, kind, i1, i2, numStr = NUM_DEC_FRAC, false, "", stringFind(s, NUM_DEC_FRAC, ptr)
1417 if not i1 then pat, maybeInt, kind, i1, i2, numStr = NUM_DEC_EXP, false, "", stringFind(s, NUM_DEC_EXP, ptr)
1418 if not i1 then pat, maybeInt, kind, i1, i2, numStr = NUM_DEC, true, "", stringFind(s, NUM_DEC, ptr)
1419 if not numStr then return nil, formatErrorInFile(s, path, ptrStart, "Tokenizer", "Malformed number.")
1420 end end end end end end end end end
1421
1422 local numStrFallback = numStr
1423
1424 if jit then
1425 if stringFind(s, "^[Ii]", i2+1) then -- Imaginary part of complex number.
1426 numStr = stringSub(s, i1, i2+1)
1427 i2 = i2 + 1
1428
1429 elseif not maybeInt or stringFind(numStr, ".", 1, true) then
1430 -- void
1431 elseif stringFind(s, "^[Uu][Ll][Ll]", i2+1) then -- Unsigned 64-bit integer.
1432 numStr = stringSub(s, i1, i2+3)
1433 i2 = i2 + 3
1434 elseif stringFind(s, "^[Ll][Ll]", i2+1) then -- Signed 64-bit integer.
1435 numStr = stringSub(s, i1, i2+2)
1436 i2 = i2 + 2
1437 end
1438 end
1439
1440 local n = tonumber(numStr)
1441
1442 if not n and jit then
1443 local chunk = loadstring("return "..numStr)
1444 n = chunk and chunk() or n
1445 end
1446
1447 n = n or tonumber(numStrFallback)
1448
1449 if not n then
1450 -- Note: We know we're not running LuaJIT here as it supports hexadecimal floats and binary notation, thus we use numStrFallback instead of numStr.
1451
1452 -- Support hexadecimal floats if we're running Lua 5.1.
1453 if kind == "lua52hex" then
1454 local _, intStr, fracStr, expStr
1455 if pat == NUM_HEX_FRAC_EXP then _, intStr, fracStr, expStr = stringMatch(numStrFallback, NUM_HEX_FRAC_EXP)
1456 elseif pat == NUM_HEX_FRAC then _, intStr, fracStr = stringMatch(numStrFallback, NUM_HEX_FRAC) ; expStr = "0"
1457 elseif pat == NUM_HEX_EXP then _, intStr, expStr = stringMatch(numStrFallback, NUM_HEX_EXP) ; fracStr = ""
1458 else return nil, formatErrorInFile(s, path, ptrStart, "Tokenizer", "Internal error parsing the number '%s'.", numStrFallback) end
1459
1460 n = tonumber(intStr, 16) or 0 -- intStr may be "".
1461
1462 local fracValue = 1
1463 for i = 1, #fracStr do
1464 fracValue = fracValue / 16
1465 n = n + tonumber(stringSub(fracStr, i, i), 16) * fracValue
1466 end
1467
1468 n = n * 2 ^ stringGsub(expStr, "^+", "")
1469
1470 elseif kind == "binary" then
1471 n = tonumber(stringSub(numStrFallback, 3), 2)
1472 end
1473 end
1474
1475 if not n then
1476 return nil, formatErrorInFile(s, path, ptrStart, "Tokenizer", "Invalid number.")
1477 end
1478
1479 ptr = i2+1
1480 tokType = "number"
1481 tokRepr = numStr
1482 tokValue = n
1483
1484 if stringFind(s, "^[%w_.]", ptr) then
1485 local after = stringMatch(s, "^%.?%d+", ptr) or stringMatch(s, "^[%w_.][%w_.]?", ptr)
1486 return nil, formatErrorInFile(s, path, ptrStart, "Tokenizer", "Malformed number near '%s%s'.", numStr, after)
1487 end
1488
1489 -- Quoted string.
1490 elseif BYTES_QUOTE[b1] then
1491 local quote = stringSub(s, ptr, ptr)
1492 local quoteByte = stringByte(quote)
1493 ptr = ptr + 1
1494
1495 local pat = "["..quote.."\\\n]"
1496
1497 while true do
1498 local i1 = stringFind(s, pat, ptr)
1499 if not i1 then
1500 return nil, formatErrorInFile(s, path, ptrStart, "Tokenizer", "Unfinished string.")
1501 end
1502
1503 ptr = i1
1504 local b1, b2 = stringByte(s, ptr, ptr+1)
1505
1506 -- '"'
1507 if b1 == quoteByte then
1508 ptr = ptr + 1
1509 break
1510
1511 -- '\'
1512 elseif b1 == 92 then
1513 ptr = ptr + 1
1514
1515 if b2 == 122 then -- 'z'
1516 ptr = ptr + 1
1517 local _, i2 = stringFind(s, "^%s*", ptr)
1518 ptr = i2 + 1
1519 else
1520 -- Note: We don't have to look for multiple characters after the escape, like \nnn - this algorithm works anyway.
1521 if ptr > #s then
1522 return nil, formatErrorInFile(
1523 s, path, ptr, "Tokenizer",
1524 "Unfinished string after escape character. (String starting %s)",
1525 getRelativeLocationText(lnStart, getLineNumber(s, ptr))
1526 )
1527 end
1528 ptr = ptr + 1 -- Just skip the next character, whatever it might be.
1529 end
1530
1531 -- '\n'
1532 elseif b1 == 10 then
1533 -- Lua, this is silly!
1534 return nil, formatErrorInFile(s, path, ptr, "Tokenizer", "Unescaped newline in string (starting %s).", getRelativeLocationText(lnStart, getLineNumber(s, ptr)))
1535
1536 else
1537 assert(false)
1538 end
1539 end
1540
1541 tokType = "string"
1542 tokRepr = stringSub(s, ptrStart, ptr-1)
1543
1544 local chunk = loadstring("return "..tokRepr, "@") -- Try to make Lua parse the string value before we fall back to our own parser which is probably slower.
1545 if chunk then
1546 tokValue = chunk()
1547 assert(type(tokValue) == "string")
1548 else
1549 local stringValue, err = parseStringContents(s, path, ptrStart+1, ptr-2)
1550 if not stringValue then return nil, err end
1551 tokValue = stringValue
1552 end
1553
1554 -- Long string.
1555 elseif BYTES_SQUARE[b1] and stringFind(s, "^=*%[", ptr+1) then
1556 local ok, equalSignCountIfLong
1557 ok, equalSignCountIfLong, ptr = parseStringlikeToken(s, ptr)
1558
1559 if not ok then
1560 local errCode = equalSignCountIfLong
1561 if errCode == ERROR_UNFINISHED_VALUE then
1562 return nil, formatErrorInFile(s, path, ptrStart, "Tokenizer", "Unfinished long string.")
1563 else
1564 return nil, formatErrorInFile(s, path, ptrStart, "Tokenizer", "Invalid long string.")
1565 end
1566 end
1567
1568 tokType = "string"
1569 tokRepr = stringSub(s, ptrStart, ptr-1)
1570
1571 local chunk, err = loadstring("return "..tokRepr, "@")
1572 if not chunk then
1573 err = stringGsub(err, "^:%d+: ", "")
1574 return nil, formatErrorInFile(s, path, ptrStart, "Tokenizer", "Could not convert long string token to value. (%s)", err)
1575 end
1576 tokValue = assert(chunk)()
1577 assert(type(tokValue) == "string")
1578
1579 -- Punctuation.
1580 elseif BYTES_DOT[b1] and stringFind(s, "^%.%.", ptr+1) then
1581 ptr = ptr + 3
1582 tokType = "punctuation"
1583 tokRepr = stringSub(s, ptrStart, ptr-1)
1584 tokValue = tokRepr
1585 elseif BYTES_PUNCT_TWO_CHARS[b1] and stringFind(s, "^%.%.", ptr) or stringFind(s, "^[=~<>]=", ptr) or stringFind(s, "^::", ptr) or stringFind(s, "^//", ptr) or stringFind(s, "^<<", ptr) or stringFind(s, "^>>", ptr) then
1586 ptr = ptr + 2
1587 tokType = "punctuation"
1588 tokRepr = stringSub(s, ptrStart, ptr-1)
1589 tokValue = tokRepr
1590 elseif BYTES_PUNCT_ONE_CHAR[b1] then
1591 ptr = ptr + 1
1592 tokType = "punctuation"
1593 tokRepr = stringSub(s, ptrStart, ptr-1)
1594 tokValue = tokRepr
1595
1596 else
1597 return nil, formatErrorInFile(s, path, ptrStart, "Tokenizer", "Unknown character.")
1598 end
1599 assert(tokType)
1600
1601 ln = ln + countString(tokRepr, "\n", true)
1602 count = count + 1
1603
1604 tokens[count] = {
1605 type = tokType,
1606 value = tokValue,
1607 representation = tokRepr,
1608
1609 sourceString = s,
1610 sourcePath = path,
1611
1612 lineStart = lnStart,
1613 lineEnd = ln,
1614 positionStart = ptrStart,
1615 positionEnd = ptr - 1,
1616 }
1617
1618 -- print(F("%4d %-11s '%s'", count, tokType, (stringGsub(tokRepr, "\n", "\\n"))))
1619 end
1620
1621 return tokens
1622 end
1623end
1624
1625-- tokens, error = tokenizeFile( path [, keepWhitespaceTokens=false ] )
1626local function tokenizeFile(path, keepWhitespaceTokens)
1627 assertArg1("tokenizeFile", 1, path, "string")
1628 assertArg2("tokenizeFile", 2, keepWhitespaceTokens, "boolean","nil")
1629
1630 local file, err = ioOpen(path, "r")
1631 if not file then return nil, F("Could not open file '%s'. (%s)", ensurePrintable(path), ensurePrintable(err)) end
1632
1633 local s = file:read("*a")
1634 file:close()
1635
1636 return tokenize(s, (keepWhitespaceTokens or false), path)
1637end
1638
1639--
1640-- :TokenCreation
1641--
1642-- commentToken = newToken( "comment", contents )
1643-- identifierToken = newToken( "identifier", name )
1644-- keywordToken = newToken( "keyword", name )
1645-- numberToken = newToken( "number", number )
1646-- punctuationToken = newToken( "punctuation", punctuationString )
1647-- stringToken = newToken( "string", stringValue )
1648-- whitespaceToken = newToken( "whitespace", contents )
1649--
1650local function newToken(tokType, tokValue)
1651 local tokRepr
1652
1653 if tokType == "keyword" then
1654 if type(tokValue) ~= "string" then errorf(2, "Expected string value for 'keyword' token. (Got %s)", type(tokValue)) end
1655 if not KEYWORDS[tokValue] then errorf(2, "Invalid keyword '%s'.", tokValue) end
1656 tokRepr = tokValue
1657
1658 elseif tokType == "identifier" then
1659 if type(tokValue) ~= "string" then errorf(2, "Expected string value for 'identifier' token. (Got %s)", type(tokValue)) end
1660 if not stringFind(tokValue, "^[%a_][%w_]*$") then errorf(2, "Invalid identifier '%s'.", tokValue) end
1661 if KEYWORDS[tokValue] then errorf(2, "Invalid identifier '%s'.", tokValue) end
1662 tokRepr = tokValue
1663
1664 elseif tokType == "number" then
1665 if type(tokValue) ~= "number" then
1666 errorf(2, "Expected number value for 'number' token. (Got %s)", type(tokValue))
1667 end
1668 tokRepr = (
1669 tokValue == 0 and NORMALIZE_MINUS_ZERO and "0" or -- Avoid '-0' sometimes.
1670 tokValue == 1/0 and "(1/0)" or
1671 tokValue == -1/0 and "(-1/0)" or
1672 tokValue ~= tokValue and "(0/0)" or
1673 formatNumber(tokValue)
1674 )
1675
1676 elseif tokType == "string" then
1677 if type(tokValue) ~= "string" then errorf(2, "Expected string value for 'string' token. (Got %s)", type(tokValue)) end
1678 tokRepr = stringGsub(F("%q", tokValue), "\n", "n")
1679
1680 elseif tokType == "punctuation" then
1681 if type(tokValue) ~= "string" then errorf(2, "Expected string value for 'punctuation' token. (Got %s)", type(tokValue)) end
1682 if not PUNCTUATION[tokValue] then errorf(2, "Invalid punctuation '%s'.", tokValue) end
1683 tokRepr = tokValue
1684
1685 elseif tokType == "comment" then
1686 if type(tokValue) ~= "string" then errorf(2, "Expected string value for 'comment' token. (Got %s)", type(tokValue)) end
1687
1688 if stringFind(tokValue, "\n") then
1689 local equalSigns = stringFind(tokValue, "[[", 1, true) and "=" or ""
1690
1691 while stringFind(tokValue, "]"..equalSigns.."]", 1, true) do
1692 equalSigns = equalSigns.."="
1693 end
1694
1695 tokRepr = F("--[%s[%s]%s]", equalSigns, tokValue, equalSigns)
1696
1697 else
1698 tokRepr = F("--%s\n", tokValue)
1699 end
1700
1701 elseif tokType == "whitespace" then
1702 if type(tokValue) ~= "string" then errorf(2, "Expected string value for 'whitespace' token. (Got %s)", type(tokValue)) end
1703 if tokValue == "" then errorf(2, "Value is empty.") end -- Having a token that is zero characters long would be weird, so we disallow it.
1704 if stringFind(tokValue, "[^ \t\n]") then errorf(2, "Value has non-whitespace characters.") end
1705 tokRepr = tokValue
1706
1707 else
1708 errorf(2, "Invalid token type '%s'.", tostring(tokType))
1709 end
1710
1711 return {
1712 type = tokType,
1713 value = tokValue,
1714 representation = tokRepr,
1715
1716 sourceString = "",
1717 sourcePath = "?",
1718
1719 lineStart = 0,
1720 lineEnd = 0,
1721 positionStart = 0,
1722 positionEnd = 0,
1723 }
1724end
1725
1726--
1727-- :TokenModification
1728--
1729-- updateToken( commentToken, contents )
1730-- updateToken( identifierToken, name )
1731-- updateToken( keywordToken, name )
1732-- updateToken( numberToken, number )
1733-- updateToken( punctuationToken, punctuationString )
1734-- updateToken( stringToken, stringValue )
1735-- updateToken( whitespaceToken, contents )
1736--
1737local function updateToken(tok, tokValue)
1738 -- @Copypaste from newToken().
1739
1740 if tok.type == "keyword" then
1741 if type(tokValue) ~= "string" then errorf(2, "Expected string value for 'keyword' token. (Got %s)", type(tokValue)) end
1742 if not KEYWORDS[tokValue] then errorf(2, "Invalid keyword '%s'.", tokValue) end
1743 tok.representation = tokValue
1744
1745 elseif tok.type == "identifier" then
1746 if type(tokValue) ~= "string" then errorf(2, "Expected string value for 'identifier' token. (Got %s)", type(tokValue)) end
1747 if not stringFind(tokValue, "^[%a_][%w_]*$") then errorf(2, "Invalid identifier '%s'.", tokValue) end
1748 if KEYWORDS[tokValue] then errorf(2, "Invalid identifier '%s'.", tokValue) end
1749 tok.representation = tokValue
1750
1751 elseif tok.type == "number" then
1752 if type(tokValue) ~= "number" then
1753 errorf(2, "Expected number value for 'number' token. (Got %s)", type(tokValue))
1754 end
1755 tok.representation = (
1756 tokValue == 0 and NORMALIZE_MINUS_ZERO and "0" or -- Avoid '-0' sometimes.
1757 tokValue == 1/0 and "(1/0)" or
1758 tokValue == -1/0 and "(-1/0)" or
1759 tokValue ~= tokValue and "(0/0)" or
1760 formatNumber(tokValue)
1761 )
1762
1763 elseif tok.type == "string" then
1764 if type(tokValue) ~= "string" then errorf(2, "Expected string value for 'string' token. (Got %s)", type(tokValue)) end
1765 tok.representation = stringGsub(F("%q", tokValue), "\n", "n")
1766
1767 elseif tok.type == "punctuation" then
1768 if type(tokValue) ~= "string" then errorf(2, "Expected string value for 'punctuation' token. (Got %s)", type(tokValue)) end
1769 if not PUNCTUATION[tokValue] then errorf(2, "Invalid punctuation '%s'.", tokValue) end
1770 tok.representation = tokValue
1771
1772 elseif tok.type == "comment" then
1773 if type(tokValue) ~= "string" then errorf(2, "Expected string value for 'comment' token. (Got %s)", type(tokValue)) end
1774
1775 if stringFind(tokValue, "\n") then
1776 local equalSigns = stringFind(tokValue, "[[", 1, true) and "=" or ""
1777
1778 while stringFind(tokValue, "]"..equalSigns.."]", 1, true) do
1779 equalSigns = equalSigns.."="
1780 end
1781
1782 tok.representation = F("--[%s[%s]%s]", equalSigns, tokValue, equalSigns)
1783
1784 else
1785 tok.representation = F("--%s\n", tokValue)
1786 end
1787
1788 elseif tok.type == "whitespace" then
1789 if type(tokValue) ~= "string" then errorf(2, "Expected string value for 'whitespace' token. (Got %s)", type(tokValue)) end
1790 if tokValue == "" then errorf(2, "Value is empty.") end -- Having a token that is zero characters long would be weird, so we disallow it.
1791 if stringFind(tokValue, "[^ \t\n]") then errorf(2, "Value has non-whitespace characters.") end
1792 tok.representation = tokValue
1793
1794 else
1795 errorf(2, "Internal error: Invalid token type '%s'.", tostring(tok.type))
1796 end
1797
1798 tok.value = tokValue
1799end
1800
1801local function cloneToken(tok)
1802 return {
1803 type = tok.type,
1804 value = tok.value,
1805 representation = tok.representation,
1806
1807 sourceString = tok.sourceString,
1808 sourcePath = tok.sourcePath,
1809
1810 lineStart = tok.lineStart,
1811 lineEnd = tok.lineEnd,
1812 positionStart = tok.positionStart,
1813 positionEnd = tok.positionEnd,
1814 }
1815end
1816
1817local function concatTokens(tokens)
1818 local parts = {}
1819
1820 for tok = 1, #tokens do
1821 local tokRepr = tokens[tok ].representation
1822 local lastTokRepr = tok > 1 and tokens[tok-1].representation
1823
1824 if lastTokRepr and (
1825 (stringFind(tokRepr, "^[%w_]") and stringFind(lastTokRepr, "[%w_]$")) or
1826 (stringFind(tokRepr, "^%." ) and stringFind(lastTokRepr, "%.$" )) or
1827 (stringFind(tokRepr, "^%-" ) and stringFind(lastTokRepr, "%-$" )) or
1828 (stringFind(tokRepr, "^/" ) and stringFind(lastTokRepr, "/$" )) or
1829
1830 (tok > 1 and tokens[tok-1].type == "number" and stringFind(tokRepr, "^[%w_.]")) or
1831 ( tokens[tok ].type == "number" and stringFind(lastTokRepr, "%.$") and not stringFind(lastTokRepr, "%.%.$"))
1832 ) then
1833 tableInsert(parts, " ")
1834 end
1835 tableInsert(parts, tokRepr)
1836 end
1837
1838 return tableConcat(parts)
1839end
1840
1841
1842
1843local function isToken(token, tokType, tokValue)
1844 return token ~= nil and token.type == tokType and token.value == tokValue
1845end
1846
1847local function isTokenType(token, tokType)
1848 return token ~= nil and token.type == tokType
1849end
1850
1851local function isTokenAnyValue(token, tokValueSet)
1852 return token ~= nil and tokValueSet[token.value] == true
1853end
1854
1855
1856
1857local function getLeftmostToken(node)
1858 if node.type == "binary" then
1859 return getLeftmostToken(node.left)
1860 else
1861 return node.token
1862 end
1863end
1864
1865
1866
1867local parseExpressionInternal, parseExpressionList, parseFunctionParametersAndBody, parseBlock
1868
1869local function parseIdentifier(tokens, tok) --> ident, token, error
1870 if not isTokenType(tokens[tok], "identifier") then
1871 return nil, tok, formatErrorAtToken(tokens[tok], "Parser", "Expected an identifier.")
1872 end
1873
1874 local ident = AstIdentifier(tokens[tok], tokens[tok].value)
1875 tok = tok + 1
1876
1877 return ident, tok
1878end
1879
1880local function parseNameList(tokens, tok, names, allowVararg, allowAttributes) --> success, token, error|nil
1881 while true do
1882 if allowVararg and isToken(tokens[tok], "punctuation", "...") then
1883 local vararg = AstVararg(tokens[tok])
1884 tok = tok + 1 -- '...'
1885
1886 tableInsert(names, vararg)
1887
1888 return true, tok
1889 end
1890
1891 local ident, tokNext, err = parseIdentifier(tokens, tok)
1892 if not ident then return false, tok, err end
1893 tok = tokNext
1894
1895 if allowAttributes and isToken(tokens[tok], "punctuation", "<") then
1896 tok = tok + 1 -- '<'
1897
1898 local attrIdent, tokNext, err = parseIdentifier(tokens, tok)
1899 if not attrIdent then
1900 return false, tok, err
1901 elseif not (attrIdent.name == "close" or attrIdent.name == "const") then
1902 return false, tok, formatErrorAtToken(tokens[tok], "Parser", "Expected 'close' or 'const' for attribute name.")
1903 end
1904 tok = tokNext
1905
1906 ident.attribute = attrIdent.name
1907
1908 if not isToken(tokens[tok], "punctuation", ">") then
1909 return nil, tok, formatErrorAfterToken(tokens[tok-1], "Parser", "Expected '>' after attribute name.")
1910 end
1911 tok = tok + 1 -- '>'
1912 end
1913
1914 tableInsert(names, ident)
1915
1916 if not isToken(tokens[tok], "punctuation", ",") then
1917 return true, tok
1918 end
1919 tok = tok + 1 -- ','
1920 end
1921
1922 return true, tok
1923end
1924
1925local function parseTable(tokens, tokStart) --> tableNode, token, error
1926 local tok = tokStart
1927 local tableNode = AstTable(tokens[tok])
1928 tok = tok + 1 -- '{'
1929
1930 local generatedIndex = 0
1931
1932 while true do
1933 if isToken(tokens[tok], "punctuation", "}") then
1934 tok = tok + 1 -- '}'
1935 break
1936
1937 elseif isToken(tokens[tok], "punctuation", "[") then
1938 tok = tok + 1 -- '['
1939
1940 local keyExpr, tokNext, err = parseExpressionInternal(tokens, tok, 0)
1941 if not keyExpr then return nil, tok, err end
1942 tok = tokNext
1943
1944 if not isToken(tokens[tok], "punctuation", "]") then
1945 return nil, tok, formatErrorAfterToken(tokens[tok-1], "Parser", "Expected ']' after key value.")
1946 end
1947 tok = tok + 1 -- ']'
1948
1949 if not isToken(tokens[tok], "punctuation", "=") then
1950 return nil, tok, formatErrorAfterToken(tokens[tok-1], "Parser", "Expected '=' after key.")
1951 end
1952 tok = tok + 1 -- '='
1953
1954 local valueExpr, tokNext, err = parseExpressionInternal(tokens, tok, 0)
1955 if not valueExpr then return nil, tok, err end
1956 tok = tokNext
1957
1958 local tableField = {key=keyExpr, value=valueExpr, generatedKey=false}
1959 tableInsert(tableNode.fields, tableField)
1960
1961 elseif isTokenType(tokens[tok], "identifier") and isToken(tokens[tok+1], "punctuation", "=") then
1962 local keyExpr = AstLiteral(tokens[tok], tokens[tok].value)
1963 tok = tok + 1 -- identifier
1964
1965 if not isToken(tokens[tok], "punctuation", "=") then
1966 return nil, tok, formatErrorAfterToken(tokens[tok-1], "Parser", "Expected '=' after key name.")
1967 end
1968 tok = tok + 1 -- '='
1969
1970 local valueExpr, tokNext, err = parseExpressionInternal(tokens, tok, 0)
1971 if not valueExpr then return nil, tok, err end
1972 tok = tokNext
1973
1974 local tableField = {key=keyExpr, value=valueExpr, generatedKey=false}
1975 tableInsert(tableNode.fields, tableField)
1976
1977 else
1978 generatedIndex = generatedIndex + 1
1979 local keyExpr = AstLiteral(tokens[tok], generatedIndex)
1980
1981 local valueExpr, tokNext, err = parseExpressionInternal(tokens, tok, 0)
1982 if not valueExpr then return nil, tok, err end
1983 tok = tokNext
1984
1985 local tableField = {key=keyExpr, value=valueExpr, generatedKey=true}
1986 tableInsert(tableNode.fields, tableField)
1987 end
1988
1989 if isToken(tokens[tok], "punctuation", ",") or isToken(tokens[tok], "punctuation", ";") then
1990 tok = tok + 1 -- ',' or ';'
1991 -- Continue...
1992
1993 elseif isToken(tokens[tok], "punctuation", "}") then
1994 tok = tok + 1 -- '}'
1995 break
1996
1997 else
1998 return nil, tok, formatErrorAfterToken(
1999 tokens[tok-1], "Parser",
2000 "Expected ',' or '}' after value in table constructor (starting %s).",
2001 getRelativeLocationTextForToken(tokens, tokStart, tok-1)
2002 )
2003 end
2004 end
2005
2006 return tableNode, tok
2007end
2008
2009--[[local]] function parseExpressionInternal(tokens, tokStart, lastPrecedence) --> expression, token, error
2010 local tok = tokStart
2011 local canParseLookupOrCall = false
2012 local currentToken = tokens[tok]
2013 local expr
2014
2015 -- identifier
2016 if isTokenType(currentToken, "identifier") then
2017 local ident, tokNext, err = parseIdentifier(tokens, tok)
2018 if not ident then return nil, tok, err end
2019 tok = tokNext
2020
2021 expr = ident
2022 canParseLookupOrCall = true
2023
2024 -- ...
2025 elseif isToken(currentToken, "punctuation", "...") then
2026 local vararg = AstVararg(currentToken)
2027 tok = tok + 1 -- '...'
2028 expr = vararg
2029
2030 -- literal
2031 elseif isTokenType(currentToken, "string") or isTokenType(currentToken, "number") then
2032 local literal = AstLiteral(currentToken, currentToken.value)
2033 tok = tok + 1 -- literal
2034 expr = literal
2035 elseif isToken(currentToken, "keyword", "true") then
2036 local literal = AstLiteral(currentToken, true)
2037 tok = tok + 1 -- 'true'
2038 expr = literal
2039 elseif isToken(currentToken, "keyword", "false") then
2040 local literal = AstLiteral(currentToken, false)
2041 tok = tok + 1 -- 'false'
2042 expr = literal
2043 elseif isToken(currentToken, "keyword", "nil") then
2044 local literal = AstLiteral(currentToken, nil)
2045 tok = tok + 1 -- 'nil'
2046 expr = literal
2047
2048 -- unary
2049 elseif
2050 (isToken(currentToken, "keyword", "not") or (isTokenType(currentToken, "punctuation") and isTokenAnyValue(currentToken, OPERATORS_UNARY)))
2051 and OPERATOR_PRECEDENCE.unary > lastPrecedence
2052 then
2053 local unary = AstUnary(currentToken, currentToken.value)
2054 tok = tok + 1 -- operator
2055
2056 local subExpr, tokNext, err = parseExpressionInternal(tokens, tok, OPERATOR_PRECEDENCE.unary-1)
2057 if not subExpr then return nil, tok, err end
2058 unary.expression = subExpr
2059 tok = tokNext
2060
2061 expr = unary
2062
2063 -- Special rule: Treat '-n' as one literal (but not '-n^n' because of operator precedence).
2064 if
2065 unary.operator == "-"
2066 and subExpr.type == "literal"
2067 and type(subExpr.value) == "number"
2068 and isTokenType(subExpr.token, "number")
2069 and not (subExpr.value == 0 and NORMALIZE_MINUS_ZERO) -- We cannot store -0 in Lua 5.3+, thus we need to keep the unary expression.
2070 then
2071 subExpr.value = -subExpr.value
2072 subExpr.token = unary.token
2073 expr = subExpr
2074 end
2075
2076 -- {...}
2077 elseif isToken(currentToken, "punctuation", "{") then
2078 local tableNode, tokNext, err = parseTable(tokens, tok)
2079 if not tableNode then return nil, tok, err end
2080 tok = tokNext
2081
2082 expr = tableNode
2083
2084 -- function
2085 elseif isToken(currentToken, "keyword", "function") then
2086 local funcTok = tok
2087 tok = tok + 1 -- 'function'
2088
2089 local func, tokNext, err = parseFunctionParametersAndBody(tokens, tok, funcTok)
2090 if not func then return nil, tok, err end
2091 func.token = tok
2092 tok = tokNext
2093
2094 expr = func
2095
2096 -- (...)
2097 elseif isToken(currentToken, "punctuation", "(") then
2098 tok = tok + 1 -- '('
2099
2100 local _expr, tokNext, err = parseExpressionInternal(tokens, tok, 0)
2101 if not _expr then return nil, tok, err end
2102 tok = tokNext
2103
2104 if _expr.type == "call" or _expr.type == "vararg" then
2105 _expr.adjustToOne = true
2106 end
2107
2108 if not isToken(tokens[tok], "punctuation", ")") then
2109 return nil, tok, formatErrorAfterToken(tokens[tok-1], "Parser", "Expected ')' (to end parenthesis expression starting %s).", getRelativeLocationTextForToken(tokens, tokStart, tok-1))
2110 end
2111 tok = tok + 1 -- ')'
2112
2113 expr = _expr
2114 canParseLookupOrCall = true
2115
2116 else
2117 return nil, tok, formatErrorAtToken(currentToken, "Parser", "Failed parsing expression.")
2118 end
2119
2120 assert(expr)
2121
2122 -- Binary expressions, including lookups and calls.
2123 while true do
2124 currentToken = tokens[tok]
2125
2126 -- a + b
2127 if
2128 (
2129 (isTokenType(currentToken, "punctuation") and isTokenAnyValue(currentToken, OPERATORS_BINARY))
2130 or isToken(currentToken, "keyword", "and")
2131 or isToken(currentToken, "keyword", "or")
2132 )
2133 and OPERATOR_PRECEDENCE[currentToken.value] > lastPrecedence
2134 then
2135 local rightAssociative = isToken(currentToken, "punctuation", "..") or isToken(currentToken, "punctuation", "^")
2136
2137 local tokOp = tok
2138 local binary = AstBinary(currentToken, currentToken.value)
2139 tok = tok + 1 -- operator
2140
2141 local lhsExpr = expr
2142
2143 local rhsExpr, tokNext, err = parseExpressionInternal(tokens, tok, OPERATOR_PRECEDENCE[binary.operator] + (rightAssociative and -1 or 0))
2144 if not rhsExpr then return nil, tok, err end
2145 tok = tokNext
2146
2147 binary.left = expr
2148 binary.right = rhsExpr
2149
2150 expr = binary
2151
2152 -- Special rule: Treat 'n/0' and '-n/0' as one literal (because that's how toLua() outputs infinity/NaN).
2153 if
2154 binary.operator == "/"
2155 and lhsExpr.type == "literal" and type(lhsExpr.value) == "number"
2156 and rhsExpr.type == "literal" and rhsExpr.value == 0
2157 and (
2158 isTokenType(lhsExpr.token, "number")
2159 or (
2160 isToken(lhsExpr.token, "punctuation", "-")
2161 and isTokenType(tokens[indexOf(tokens, lhsExpr.token, tokStart, tokOp-1) + 1], "number") -- @Speed: Don't use indexOf().
2162 )
2163 )
2164 and isTokenType(rhsExpr.token, "number")
2165 then
2166 lhsExpr.value = lhsExpr.value / 0
2167 expr = lhsExpr
2168 end
2169
2170 elseif not canParseLookupOrCall then
2171 break
2172
2173 -- t.k
2174 elseif isToken(currentToken, "punctuation", ".") then
2175 local lookup = AstLookup(currentToken)
2176 tok = tok + 1 -- '.'
2177
2178 local ident, tokNext, err = parseIdentifier(tokens, tok)
2179 if not ident then return nil, tok, err end
2180 tok = tokNext
2181
2182 local literal = AstLiteral(ident.token, ident.name)
2183
2184 lookup.object = expr
2185 lookup.member = literal
2186
2187 expr = lookup
2188
2189 -- t[k]
2190 elseif isToken(currentToken, "punctuation", "[") then
2191 local lookup = AstLookup(currentToken)
2192 tok = tok + 1 -- '['
2193
2194 local memberExpr, tokNext, err = parseExpressionInternal(tokens, tok, 0)
2195 if not memberExpr then return nil, tok, err end
2196 tok = tokNext
2197
2198 if not isToken(tokens[tok], "punctuation", "]") then
2199 return nil, tok, formatErrorAfterToken(tokens[tok-1], "Parser", "Expected ']' after lookup key value.")
2200 end
2201 tok = tok + 1 -- ']'
2202
2203 lookup.object = expr
2204 lookup.member = memberExpr
2205
2206 expr = lookup
2207
2208 -- f""
2209 elseif isTokenType(currentToken, "string") then
2210 local call = AstCall(currentToken)
2211
2212 local literal = AstLiteral(currentToken, currentToken.value)
2213 tok = tok + 1 -- string
2214 call.arguments[1] = literal
2215
2216 call.callee = expr
2217 expr = call
2218
2219 -- f{}
2220 elseif isToken(currentToken, "punctuation", "{") then
2221 local call = AstCall(currentToken)
2222
2223 local tableNode, tokNext, err = parseTable(tokens, tok)
2224 if not tableNode then return nil, tok, err end
2225 call.arguments[1] = tableNode
2226 tok = tokNext
2227
2228 call.callee = expr
2229 expr = call
2230
2231 -- f()
2232 elseif isToken(currentToken, "punctuation", "(") then
2233 if tok >= 2 and currentToken.lineStart > tokens[tok-1].lineEnd then
2234 return nil, tok, formatErrorAtToken(currentToken, "Parser", "Ambigous syntax. Is this a function call or a new statement?")
2235 end
2236
2237 local call = AstCall(currentToken)
2238 tok = tok + 1 -- '('
2239
2240 if not isToken(tokens[tok], "punctuation", ")") then
2241 local ok, tokNext, err = parseExpressionList(tokens, tok, call.arguments)
2242 if not ok then return nil, tok, err end
2243 tok = tokNext
2244 end
2245
2246 if not isToken(tokens[tok], "punctuation", ")") then
2247 return nil, tok, formatErrorAfterToken(tokens[tok-1], "Parser", "Expected ')' to end argument list for call.")
2248 end
2249 tok = tok + 1 -- ')'
2250
2251 call.callee = expr
2252 expr = call
2253
2254 -- o:m()
2255 elseif isToken(currentToken, "punctuation", ":") then
2256 do
2257 local lookup = AstLookup(currentToken)
2258 tok = tok + 1 -- ':'
2259
2260 local ident, tokNext, err = parseIdentifier(tokens, tok)
2261 if not ident then return nil, tok, err end
2262 tok = tokNext
2263
2264 local literal = AstLiteral(ident.token, ident.name)
2265
2266 lookup.object = expr
2267 lookup.member = literal
2268
2269 expr = lookup
2270 end
2271
2272 do
2273 local call = AstCall(tokens[tok])
2274 call.method = true
2275
2276 if isTokenType(tokens[tok], "string") then
2277 local literal = AstLiteral(tokens[tok], tokens[tok].value)
2278 tok = tok + 1 -- string
2279 call.arguments[1] = literal
2280
2281 elseif isToken(tokens[tok], "punctuation", "{") then
2282 local tableNode, tokNext, err = parseTable(tokens, tok)
2283 if not tableNode then return nil, tok, err end
2284 call.arguments[1] = tableNode
2285 tok = tokNext
2286
2287 elseif isToken(tokens[tok], "punctuation", "(") then
2288 if tok >= 2 and tokens[tok].lineStart > tokens[tok-1].lineEnd then
2289 return nil, tok, formatErrorAtToken(tokens[tok], "Parser", "Ambigous syntax. Is this a function call or a new statement?")
2290 end
2291
2292 tok = tok + 1 -- '('
2293
2294 if not isToken(tokens[tok], "punctuation", ")") then
2295 local ok, tokNext, err = parseExpressionList(tokens, tok, call.arguments)
2296 if not ok then return nil, tok, err end
2297 tok = tokNext
2298 end
2299
2300 if not isToken(tokens[tok], "punctuation", ")") then
2301 return nil, tok, formatErrorAfterToken(tokens[tok-1], "Parser", "Expected ')' after argument list for method call.")
2302 end
2303 tok = tok + 1 -- ')'
2304
2305 else
2306 return nil, tok, formatErrorAfterToken(tokens[tok-1], "Parser", "Expected '(' to start argument list for method call.")
2307 end
2308
2309 call.callee = expr
2310 expr = call
2311 end
2312
2313 else
2314 break
2315 end
2316
2317 assert(expr)
2318 end
2319
2320 return expr, tok
2321end
2322
2323--[[local]] function parseExpressionList(tokens, tok, expressions) --> success, token, error
2324 while true do
2325 local expr, tokNext, err = parseExpressionInternal(tokens, tok, 0)
2326 if not expr then return false, tok, err end
2327 tok = tokNext
2328
2329 tableInsert(expressions, expr)
2330
2331 if not isToken(tokens[tok], "punctuation", ",") then
2332 return true, tok
2333 end
2334 tok = tok + 1 -- ','
2335 end
2336end
2337
2338--[[local]] function parseFunctionParametersAndBody(tokens, tokStart, funcTok) --> func, token, error
2339 local tok = tokStart
2340 local func = AstFunction(tokens[funcTok])
2341
2342 if not isToken(tokens[tok], "punctuation", "(") then
2343 return nil, tok, formatErrorAfterToken(tokens[tok-1], "Parser", "Expected '(' to start parameter list for function.")
2344 end
2345 tok = tok + 1 -- '('
2346
2347 if not isToken(tokens[tok], "punctuation", ")") then
2348 local ok, tokNext, err = parseNameList(tokens, tok, func.parameters, true, false)
2349 if not ok then return nil, tok, err end
2350 tok = tokNext
2351 -- @Cleanup: Move the vararg parameter parsing here.
2352 end
2353
2354 if not isToken(tokens[tok], "punctuation", ")") then
2355 return nil, tok, formatErrorAfterToken(tokens[tok-1], "Parser", "Expected ')' to end parameter list for function.")
2356 end
2357 tok = tok + 1 -- ')'
2358
2359 local block, tokNext, err = parseBlock(tokens, tok, tok-1, true)
2360 if not block then return nil, tok, err end
2361 func.body = block
2362 tok = tokNext
2363
2364 if not isToken(tokens[tok], "keyword", "end") then
2365 return nil, tok, formatErrorAtToken(tokens[tok], "Parser", "Expected 'end' to end function (starting %s).", getRelativeLocationTextForToken(tokens, tokStart, tok))
2366 end
2367 tok = tok + 1 -- 'end'
2368
2369 return func, tok
2370end
2371
2372local BLOCK_END_TOKEN_TYPES = newSet{ "end", "else", "elseif", "until" }
2373
2374local function parseOneOrPossiblyMoreStatements(tokens, tokStart, statements) --> success, token, error -- The error message may be empty.
2375 --[[
2376 stat ::= ';'
2377 varlist '=' explist |
2378 functioncall |
2379 label |
2380 break |
2381 goto Name |
2382 do block end |
2383 while exp do block end |
2384 repeat block until exp |
2385 if exp then block {elseif exp then block} [else block] end |
2386 for Name '=' exp ',' exp [',' exp] do block end |
2387 for namelist in explist do block end |
2388 function funcname funcbody |
2389 local function Name funcbody |
2390 local attnamelist ['=' explist]
2391
2392 retstat ::= return [explist] [';']
2393 ]]
2394 local tok = tokStart
2395 local currentToken = tokens[tok]
2396
2397 -- do
2398 if isToken(currentToken, "keyword", "do") then
2399 tok = tok + 1 -- 'do'
2400
2401 local block, tokNext, err = parseBlock(tokens, tok, tok-1, true)
2402 if not block then return false, tok, err end
2403 block.token = tok - 1
2404 tok = tokNext
2405
2406 if not isToken(tokens[tok], "keyword", "end") then
2407 return false, tok, formatErrorAtToken(tokens[tok], "Parser", "Expected 'end' to end 'do' block (starting %s).", getRelativeLocationTextForToken(tokens, tokStart, tok))
2408 end
2409 tok = tok + 1 -- 'end'
2410
2411 tableInsert(statements, block)
2412 return true, tok
2413
2414 -- while
2415 elseif isToken(currentToken, "keyword", "while") then
2416 local whileLoop = AstWhile(currentToken)
2417 tok = tok + 1 -- 'while'
2418
2419 local expr, tokNext, err = parseExpressionInternal(tokens, tok, 0)
2420 if not expr then return false, tok, err end
2421 whileLoop.condition = expr
2422 tok = tokNext
2423
2424 if not isToken(tokens[tok], "keyword", "do") then
2425 return false, tok, formatErrorAfterToken(tokens[tok-1], "Parser", "Expected 'do' to start body for 'while' loop.")
2426 end
2427 tok = tok + 1 -- 'do'
2428
2429 local block, tokNext, err = parseBlock(tokens, tok, tok-1, true)
2430 if not block then return false, tok, err end
2431 block.token = tok - 1
2432 whileLoop.body = block
2433 tok = tokNext
2434
2435 if not isToken(tokens[tok], "keyword", "end") then
2436 return false, tok, formatErrorAtToken(tokens[tok], "Parser", "Expected 'end' to end 'while' loop (starting %s).", getRelativeLocationTextForToken(tokens, tokStart, tok))
2437 end
2438 tok = tok + 1 -- 'end'
2439
2440 tableInsert(statements, whileLoop)
2441 return true, tok
2442
2443 -- repeat
2444 elseif isToken(currentToken, "keyword", "repeat") then
2445 local repeatLoop = AstRepeat(currentToken)
2446 tok = tok + 1 -- 'repeat'
2447
2448 local block, tokNext, err = parseBlock(tokens, tok, tok-1, true)
2449 if not block then return false, tok, err end
2450 repeatLoop.body = block
2451 tok = tokNext
2452
2453 if not isToken(tokens[tok], "keyword", "until") then
2454 return false, tok, formatErrorAtToken(tokens[tok], "Parser", "Expected 'until' at the end of 'repeat' loop (starting %s).", getRelativeLocationTextForToken(tokens, tokStart, tok))
2455 end
2456 tok = tok + 1 -- 'until'
2457
2458 local expr, tokNext, err = parseExpressionInternal(tokens, tok, 0)
2459 if not expr then return false, tok, err end
2460 repeatLoop.condition = expr
2461 tok = tokNext
2462
2463 tableInsert(statements, repeatLoop)
2464 return true, tok
2465
2466 -- if
2467 elseif isToken(currentToken, "keyword", "if") then
2468 local ifNode = AstIf(currentToken)
2469 tok = tok + 1 -- 'if'
2470
2471 local expr, tokNext, err = parseExpressionInternal(tokens, tok, 0)
2472 if not expr then return false, tok, err end
2473 ifNode.condition = expr
2474 tok = tokNext
2475
2476 if not isToken(tokens[tok], "keyword", "then") then
2477 return false, tok, formatErrorAtToken(tokens[tok], "Parser", "Expected 'then' after 'if' condition.")
2478 end
2479 tok = tok + 1 -- 'then'
2480
2481 local block, tokNext, err = parseBlock(tokens, tok, tok-1, true)
2482 if not block then return false, tok, err end
2483 ifNode.bodyTrue = block
2484 tok = tokNext
2485
2486 local ifNodeLeaf = ifNode
2487
2488 while isToken(tokens[tok], "keyword", "elseif") do
2489 tok = tok + 1 -- 'elseif'
2490
2491 ifNodeLeaf.bodyFalse = AstBlock(tokens[tok])
2492 ifNodeLeaf.bodyFalse.statements[1] = AstIf (tokens[tok])
2493 ifNodeLeaf = ifNodeLeaf.bodyFalse.statements[1]
2494
2495 local expr, tokNext, err = parseExpressionInternal(tokens, tok, 0)
2496 if not expr then return false, tok, err end
2497 ifNodeLeaf.condition = expr
2498 tok = tokNext
2499
2500 if not isToken(tokens[tok], "keyword", "then") then
2501 return false, tok, formatErrorAtToken(tokens[tok], "Parser", "Expected 'then' after 'elseif' condition.")
2502 end
2503 tok = tok + 1 -- 'then'
2504
2505 local block, tokNext, err = parseBlock(tokens, tok, tok-1, true)
2506 if not block then return false, tok, err end
2507 ifNodeLeaf.bodyTrue = block
2508 tok = tokNext
2509 end
2510
2511 if isToken(tokens[tok], "keyword", "else") then
2512 tok = tok + 1 -- 'else'
2513
2514 local block, tokNext, err = parseBlock(tokens, tok, tok-1, true)
2515 if not block then return false, tok, err end
2516 ifNodeLeaf.bodyFalse = block
2517 tok = tokNext
2518 end
2519
2520 if not isToken(tokens[tok], "keyword", "end") then
2521 return false, tok, formatErrorAtToken(tokens[tok], "Parser", "Expected 'end' to end 'if' statement (starting %s).", getRelativeLocationTextForToken(tokens, tokStart, tok))
2522 end
2523 tok = tok + 1 -- 'end'
2524
2525 tableInsert(statements, ifNode)
2526 return true, tok
2527
2528 -- for
2529 elseif isToken(currentToken, "keyword", "for") then
2530 local forLoop = AstFor(currentToken, "")
2531 tok = tok + 1 -- 'for'
2532
2533 local ok, tokNext, err = parseNameList(tokens, tok, forLoop.names, false, false)
2534 if not ok then return false, tok, err end
2535 tok = tokNext
2536
2537 if isToken(tokens[tok], "keyword", "in") then
2538 forLoop.kind = "generic"
2539 tok = tok + 1 -- 'in'
2540
2541 elseif isToken(tokens[tok], "punctuation", "=") then
2542 if forLoop.names[2] then
2543 return false, tok, formatErrorAtToken(tokens[tok], "Parser", "Expected 'in' for generic loop.")
2544 end
2545
2546 forLoop.kind = "numeric"
2547 tok = tok + 1 -- '='
2548
2549 else
2550 return false, tok, formatErrorAfterToken(tokens[tok-1], "Parser", "Expected '=' or 'in' for 'for' loop.")
2551 end
2552
2553 local valuesStartTok = tok
2554
2555 local ok, tokNext, err = parseExpressionList(tokens, tok, forLoop.values, 0)
2556 if not ok then return false, tok, err end
2557 tok = tokNext
2558
2559 if forLoop.kind ~= "numeric" then
2560 -- void
2561 elseif not forLoop.values[2] then
2562 return false, tok, formatErrorAtToken(tokens[valuesStartTok], "Parser", "Numeric loop: Too few values.")
2563 elseif forLoop.values[4] then
2564 -- @Cleanup: Instead of using getLeftmostToken(), make parseExpressionList() return a list of expression start tokens.
2565 return false, tok, formatErrorAtToken(getLeftmostToken(forLoop.values[4]), "Parser", "Numeric loop: Too many values.")
2566 end
2567
2568 if not isToken(tokens[tok], "keyword", "do") then
2569 return false, tok, formatErrorAfterToken(tokens[tok-1], "Parser", "Expected 'do' to start body for 'for' loop.")
2570 end
2571 tok = tok + 1 -- 'do'
2572
2573 local block, tokNext, err = parseBlock(tokens, tok, tok-1, true)
2574 if not block then return false, tok, err end
2575 forLoop.body = block
2576 tok = tokNext
2577
2578 if not isToken(tokens[tok], "keyword", "end") then
2579 return false, tok, formatErrorAtToken(tokens[tok], "Parser", "Expected 'end' to end 'for' loop (starting %s).", getRelativeLocationTextForToken(tokens, tokStart, tok))
2580 end
2581 tok = tok + 1 -- 'end'
2582
2583 tableInsert(statements, forLoop)
2584 return true, tok
2585
2586 -- function
2587 elseif isToken(currentToken, "keyword", "function") then
2588 local funcTok = tok
2589 local assignment = AstAssignment(currentToken)
2590 tok = tok + 1 -- 'function'
2591
2592 local targetExpr, tokNext, err = parseIdentifier(tokens, tok)
2593 if not targetExpr then return false, tok, err end
2594 tok = tokNext
2595
2596 while isToken(tokens[tok], "punctuation", ".") do
2597 local lookup = AstLookup(tokens[tok])
2598 tok = tok + 1 -- '.'
2599
2600 local ident, tokNext, err = parseIdentifier(tokens, tok)
2601 if not ident then return false, tok, err end
2602 tok = tokNext
2603
2604 local literal = AstLiteral(ident.token, ident.name)
2605 lookup.member = literal
2606
2607 lookup.object = targetExpr
2608 lookup.member = literal
2609
2610 targetExpr = lookup
2611 end
2612
2613 local isMethod = isToken(tokens[tok], "punctuation", ":")
2614
2615 if isMethod then
2616 local lookup = AstLookup(tokens[tok])
2617 tok = tok + 1 -- ':'
2618
2619 local ident, tokNext, err = parseIdentifier(tokens, tok)
2620 if not ident then return false, tok, err end
2621 tok = tokNext
2622
2623 local literal = AstLiteral(ident.token, ident.name)
2624 lookup.member = literal
2625
2626 lookup.object = targetExpr
2627 lookup.member = literal
2628
2629 targetExpr = lookup
2630 end
2631
2632 local func, tokNext, err = parseFunctionParametersAndBody(tokens, tok, funcTok)
2633 if not func then return false, tok, err end
2634 tok = tokNext
2635
2636 if isMethod then
2637 local ident = AstIdentifier(func.token, "self")
2638 tableInsert(func.parameters, 1, ident)
2639 end
2640
2641 assignment.targets[1] = targetExpr
2642 assignment.values[1] = func
2643
2644 tableInsert(statements, assignment)
2645 return true, tok
2646
2647 -- local function
2648 elseif isToken(currentToken, "keyword", "local") and isToken(tokens[tok+1], "keyword", "function") then
2649 local funcTok = tok + 1
2650 local decl = AstDeclaration(currentToken)
2651 local assignment = AstAssignment(currentToken)
2652 tok = tok + 2 -- 'local function'
2653
2654 local ident, tokNext, err = parseIdentifier(tokens, tok)
2655 if not ident then return false, tok, err end
2656 local identCopy = parseIdentifier(tokens, tok)
2657 tok = tokNext
2658
2659 local func, tokNext, err = parseFunctionParametersAndBody(tokens, tok, funcTok)
2660 if not func then return false, tok, err end
2661 tok = tokNext
2662
2663 decl.names[1] = ident
2664 assignment.targets[1] = identCopy
2665 assignment.values[1] = func
2666
2667 tableInsert(statements, decl)
2668 tableInsert(statements, assignment)
2669 return true, tok
2670
2671 -- local
2672 elseif isToken(currentToken, "keyword", "local") then
2673 local decl = AstDeclaration(currentToken)
2674 tok = tok + 1 -- 'local'
2675
2676 local ok, tokNext, err = parseNameList(tokens, tok, decl.names, false, true)
2677 if not ok then return false, tok, err end
2678 tok = tokNext
2679
2680 if isToken(tokens[tok], "punctuation", "=") then
2681 tok = tok + 1 -- '='
2682
2683 local ok, tokNext, err = parseExpressionList(tokens, tok, decl.values)
2684 if not ok then return false, tok, err end
2685 tok = tokNext
2686 end
2687
2688 tableInsert(statements, decl)
2689 return true, tok
2690
2691 -- ::label::
2692 elseif isToken(currentToken, "punctuation", "::") then
2693 local label = AstLabel(currentToken, "")
2694 tok = tok + 1 -- '::'
2695
2696 local labelIdent, tokNext, err = parseIdentifier(tokens, tok)
2697 if not labelIdent then return false, tok, err end
2698 tok = tokNext
2699
2700 label.name = labelIdent.name
2701
2702 if not isToken(tokens[tok], "punctuation", "::") then
2703 return false, tok, formatErrorAfterToken(tokens[tok-1], "Parser", "Expected '::' after label name.")
2704 end
2705 tok = tok + 1 -- '::'
2706
2707 tableInsert(statements, label)
2708 return true, tok
2709
2710 -- goto
2711 elseif isToken(currentToken, "keyword", "goto") then
2712 local gotoNode = AstGoto(currentToken, "")
2713 tok = tok + 1 -- 'goto'
2714
2715 local labelIdent, tokNext, err = parseIdentifier(tokens, tok)
2716 if not labelIdent then return false, tok, err end
2717 tok = tokNext
2718
2719 gotoNode.name = labelIdent.name
2720
2721 tableInsert(statements, gotoNode)
2722 return true, tok
2723
2724 -- break
2725 elseif isToken(currentToken, "keyword", "break") then
2726 local breakNode = AstBreak(currentToken)
2727 tok = tok + 1 -- 'break'
2728
2729 tableInsert(statements, breakNode)
2730 return true, tok
2731
2732 -- return (last)
2733 elseif isToken(currentToken, "keyword", "return") then
2734 local returnNode = AstReturn(currentToken)
2735 tok = tok + 1 -- 'return'
2736
2737 if tokens[tok] and not (
2738 (isTokenType(tokens[tok], "keyword") and isTokenAnyValue(tokens[tok], BLOCK_END_TOKEN_TYPES))
2739 or isToken(tokens[tok], "punctuation", ";")
2740 or isTokenType(tokens[tok], "end")
2741 ) then
2742 local ok, tokNext, err = parseExpressionList(tokens, tok, returnNode.values)
2743 if not ok then return false, tok, err end
2744 tok = tokNext
2745 end
2746
2747 tableInsert(statements, returnNode)
2748 return true, tok
2749
2750 elseif isTokenType(currentToken, "keyword") then
2751 return false, tok, ""
2752
2753 else
2754 local lookahead, tokNext, err = parseExpressionInternal(tokens, tok, 0)
2755 if not lookahead then return false, tok, err end
2756
2757 if lookahead.type == "call" then
2758 local call = lookahead
2759 tok = tokNext
2760
2761 tableInsert(statements, call)
2762 return true, tok
2763
2764 elseif isToken(tokens[tokNext], "punctuation", "=") or isToken(tokens[tokNext], "punctuation", ",") then
2765 local assignment = AstAssignment(tokens[tokNext])
2766
2767 local ok, tokNext, err = parseExpressionList(tokens, tok, assignment.targets)
2768 if not ok then return false, tok, err end
2769 tok = tokNext
2770
2771 if not isToken(tokens[tok], "punctuation", "=") then
2772 return false, tok, formatErrorAfterToken(tokens[tok-1], "Parser", "Expected '=' for an assignment.")
2773 end
2774 tok = tok + 1 -- '='
2775
2776 for _, targetExpr in ipairs(assignment.targets) do
2777 if not (targetExpr.type == "identifier" or targetExpr.type == "lookup") then
2778 return false, tok, formatErrorAtNode(targetExpr, "Parser", "Invalid assignment target.")
2779 end
2780 end
2781
2782 local ok, tokNext, err = parseExpressionList(tokens, tok, assignment.values)
2783 if not ok then return false, tok, err end
2784 tok = tokNext
2785
2786 tableInsert(statements, assignment)
2787 return true, tok
2788
2789 else
2790 return false, tok, ""
2791 end
2792 end
2793
2794 assert(false)
2795end
2796
2797local statementErrorReported = false
2798
2799--[[local]] function parseBlock(tokens, tok, blockTok, stopAtEndKeyword) --> block, token, error
2800 local block = AstBlock(tokens[blockTok])
2801 local statements = block.statements
2802
2803 while tok <= #tokens and tokens[tok].type ~= "end" do
2804 while isToken(tokens[tok], "punctuation", ";") do
2805 -- Empty statements are valid in Lua 5.2+.
2806 tok = tok + 1 -- ';'
2807 end
2808
2809 local statementStartTok = tok
2810
2811 if stopAtEndKeyword and isTokenType(tokens[tok], "keyword") and isTokenAnyValue(tokens[tok], BLOCK_END_TOKEN_TYPES) then
2812 break
2813 end
2814
2815 local ok, tokNext, err = parseOneOrPossiblyMoreStatements(tokens, tok, statements)
2816 if not ok then
2817 if not statementErrorReported then
2818 statementErrorReported = true
2819 err = (err ~= "" and err.."\n" or "") .. formatErrorAtToken(tokens[tok], "Parser", "Failed parsing statement.")
2820 end
2821 return nil, tok, err
2822 end
2823 tok = tokNext
2824
2825 if isToken(tokens[tok], "punctuation", ";") then
2826 tok = tok + 1 -- ';'
2827 end
2828
2829 local lastAddedStatement = statements[#statements]
2830
2831 if lastAddedStatement.type == "return" then -- Note: 'break' statements are allowed in the middle of blocks as of Lua 5.2.
2832 break
2833
2834 elseif lastAddedStatement.type == "call" and lastAddedStatement.adjustToOne then
2835 statementErrorReported = true
2836 return nil, tok, formatErrorAtToken(tokens[statementStartTok], "Parser", "Syntax error.")
2837 end
2838 end
2839
2840 return block, tok
2841end
2842
2843-- block, error = tokensToAst( tokens, asBlock )
2844local function tokensToAst(tokens, asBlock)
2845 local tokensPurged = {}
2846 local count = 0
2847
2848 do
2849 -- Dummy start token.
2850 local token = tokens[1]
2851 count = count + 1
2852
2853 tokensPurged[count] = {
2854 type = "start",
2855 value = "",
2856 representation = "",
2857
2858 sourceString = token and token.sourceString or "",
2859 sourcePath = token and token.sourcePath or "?",
2860
2861 lineStart = 1,
2862 lineEnd = 1,
2863 positionStart = 1,
2864 positionEnd = 1,
2865 }
2866 end
2867
2868 -- Remove useless tokens.
2869 for tok = 1, #tokens do
2870 local tokType = tokens[tok].type
2871
2872 if not (tokType == "comment" or tokType == "whitespace") then
2873 count = count + 1
2874 tokensPurged[count] = tokens[tok]
2875 end
2876 end
2877
2878 do
2879 -- Dummy end token.
2880 local token = tokens[#tokens]
2881 local ln = token and 1+countString(token.sourceString, "\n", true) or 0
2882 local pos = token and #token.sourceString+1 or 0
2883 count = count + 1
2884
2885 tokensPurged[count] = {
2886 type = "end",
2887 value = "",
2888 representation = "",
2889
2890 sourceString = token and token.sourceString or "",
2891 sourcePath = token and token.sourcePath or "?",
2892
2893 lineStart = ln,
2894 lineEnd = ln,
2895 positionStart = pos,
2896 positionEnd = pos,
2897 }
2898 end
2899
2900 statementErrorReported = false
2901
2902 local ast, _, err
2903 if asBlock then
2904 ast, _, err = parseBlock(tokensPurged, 2, 2, false)
2905 else
2906 ast, _, err = parseExpressionInternal(tokensPurged, 2, 0)
2907 end
2908 if not ast then return nil, err end
2909
2910 return ast
2911end
2912
2913-- ast, error = parse( tokens )
2914-- ast, error = parse( luaString [, pathForErrorMessages="?" ] )
2915local function parse(luaOrTokens, path)
2916 assertArg2("parse", 1, luaOrTokens, "string","table")
2917
2918 -- ast, error = parse( tokens )
2919 if type(luaOrTokens) == "table" then
2920 assertArg1("parse", 2, path, "nil")
2921
2922 return tokensToAst(luaOrTokens, true)
2923
2924 -- ast, error = parse( luaString, pathForErrorMessages )
2925 else
2926 if path == nil then
2927 path = "?"
2928 else
2929 assertArg1("parse", 2, path, "string")
2930 end
2931
2932 local tokens, err = tokenize(luaOrTokens, path)
2933 if not tokens then return nil, err end
2934
2935 return tokensToAst(tokens, true)
2936 end
2937end
2938
2939-- ast, error = parseExpression( tokens )
2940-- ast, error = parseExpression( luaString [, pathForErrorMessages="?" ] )
2941local function parseExpression(luaOrTokens, path)
2942 assertArg2("parseExpression", 1, luaOrTokens, "string","table")
2943
2944 -- ast, error = parseExpression( tokens )
2945 if type(luaOrTokens) == "table" then
2946 assertArg1("parseExpression", 2, path, "nil")
2947
2948 return tokensToAst(luaOrTokens, false)
2949
2950 -- ast, error = parseExpression( luaString, pathForErrorMessages )
2951 else
2952 if path == nil then
2953 path = "?"
2954 else
2955 assertArg1("parseExpression", 2, path, "string")
2956 end
2957
2958 local tokens, err = tokenize(luaOrTokens, path)
2959 if not tokens then return nil, err end
2960
2961 return tokensToAst(tokens, false)
2962 end
2963end
2964
2965-- ast, error = parseFile( path )
2966local function parseFile(path)
2967 assertArg1("parseFile", 1, path, "string")
2968
2969 local tokens, err = tokenizeFile(path)
2970 if not tokens then return nil, err end
2971
2972 return tokensToAst(tokens, true)
2973end
2974
2975
2976
2977local nodeConstructors = {
2978 ["vararg"] = function() return AstVararg (nil) end,
2979 ["table"] = function() return AstTable (nil) end,
2980 ["lookup"] = function() return AstLookup (nil) end,
2981 ["call"] = function() return AstCall (nil) end,
2982 ["function"] = function() return AstFunction (nil) end,
2983 ["break"] = function() return AstBreak (nil) end,
2984 ["return"] = function() return AstReturn (nil) end,
2985 ["block"] = function() return AstBlock (nil) end,
2986 ["declaration"] = function() return AstDeclaration(nil) end,
2987 ["assignment"] = function() return AstAssignment (nil) end,
2988 ["if"] = function() return AstIf (nil) end,
2989 ["while"] = function() return AstWhile (nil) end,
2990 ["repeat"] = function() return AstRepeat (nil) end,
2991
2992 ["identifier"] = function(argCount, name, attribute)
2993 if argCount == 0 then
2994 errorf(3, "Missing name argument for identifier.")
2995 elseif type(name) ~= "string" then
2996 errorf(3, "Invalid name argument value type '%s'. (Expected string)", type(name))
2997 elseif not stringFind(name, "^[%a_][%w_]*$") or KEYWORDS[name] then
2998 errorf(3, "Invalid identifier name '%s'.", name)
2999 end
3000
3001 if attribute == nil or attribute == "" then
3002 -- void
3003 elseif type(attribute) ~= "string" then
3004 errorf(3, "Invalid attribute argument value type '%s'. (Expected string)", type(attribute))
3005 elseif not (attribute == "close" or attribute == "const") then
3006 errorf(3, "Invalid attribute name '%s'. (Must be 'close' or 'const'.)", attribute)
3007 end
3008
3009 local ident = AstIdentifier(nil, name)
3010 ident.attribute = attribute or ""
3011 return ident
3012 end,
3013
3014 ["label"] = function(argCount, name)
3015 if argCount == 0 then
3016 errorf(3, "Missing name argument for label.")
3017 elseif type(name) ~= "string" then
3018 errorf(3, "Invalid name argument value type '%s'. (Expected string)", type(name))
3019 elseif not stringFind(name, "^[%a_][%w_]*$") or KEYWORDS[name] then
3020 errorf(3, "Invalid label name '%s'.", name)
3021 end
3022
3023 return AstLabel(nil, name)
3024 end,
3025
3026 ["goto"] = function(argCount, name)
3027 if argCount == 0 then
3028 errorf(3, "Missing label name argument for goto.")
3029 elseif type(name) ~= "string" then
3030 errorf(3, "Invalid label name argument value type '%s'. (Expected string)", type(name))
3031 elseif not stringFind(name, "^[%a_][%w_]*$") or KEYWORDS[name] then
3032 errorf(3, "Invalid label name '%s'.", name)
3033 end
3034
3035 return AstGoto(nil, name)
3036 end,
3037
3038 ["literal"] = function(argCount, value)
3039 if argCount == 0 then
3040 errorf(3, "Missing value argument for literal.")
3041 elseif not (type(value) == "number" or type(value) == "string" or type(value) == "boolean" or type(value) == "nil") then
3042 errorf(3, "Invalid literal value type '%s'. (Expected number, string, boolean or nil)", type(value))
3043 end
3044
3045 return AstLiteral(nil, value)
3046 end,
3047
3048 ["unary"] = function(argCount, op)
3049 if argCount == 0 then
3050 errorf(3, "Missing operator argument for unary expression.")
3051 elseif not OPERATORS_UNARY[op] then
3052 errorf(3, "Invalid unary operator '%s'.", tostring(op))
3053 end
3054
3055 return AstUnary(nil, op)
3056 end,
3057
3058 ["binary"] = function(argCount, op)
3059 if argCount == 0 then
3060 errorf(3, "Missing operator argument for binary expression.")
3061 elseif not OPERATORS_BINARY[op] then
3062 errorf(3, "Invalid binary operator '%s'.", tostring(op))
3063 end
3064
3065 return AstBinary(nil, op)
3066 end,
3067
3068 ["for"] = function(argCount, kind)
3069 if argCount == 0 then
3070 errorf(3, "Missing kind argument for 'for' loop.")
3071 elseif not (kind == "numeric" or kind == "generic") then
3072 errorf(3, "Invalid for loop kind '%s'. (Must be 'numeric' or 'generic')", tostring(kind))
3073 end
3074
3075 return AstFor(nil, kind)
3076 end,
3077}
3078
3079local nodeConstructorsFast = {
3080 ["vararg"] = function() return AstVararg (nil) end,
3081 ["table"] = function() return AstTable (nil) end,
3082 ["lookup"] = function() return AstLookup (nil) end,
3083 ["call"] = function() return AstCall (nil) end,
3084 ["function"] = function() return AstFunction (nil) end,
3085 ["break"] = function() return AstBreak (nil) end,
3086 ["return"] = function() return AstReturn (nil) end,
3087 ["block"] = function() return AstBlock (nil) end,
3088 ["declaration"] = function() return AstDeclaration(nil) end,
3089 ["assignment"] = function() return AstAssignment (nil) end,
3090 ["if"] = function() return AstIf (nil) end,
3091 ["while"] = function() return AstWhile (nil) end,
3092 ["repeat"] = function() return AstRepeat (nil) end,
3093
3094 ["label"] = function(name) return AstLabel (nil, name) end,
3095 ["goto"] = function(name) return AstGoto (nil, name) end,
3096 ["literal"] = function(value) return AstLiteral(nil, value) end,
3097 ["unary"] = function(op) return AstUnary (nil, op) end,
3098 ["binary"] = function(op) return AstBinary (nil, op) end,
3099 ["for"] = function(kind) return AstFor (nil, kind) end,
3100
3101 ["identifier"] = function(name, attribute)
3102 local ident = AstIdentifier(nil, name)
3103 ident.attribute = attribute or ""
3104 return ident
3105 end,
3106}
3107
3108
3109
3110--
3111-- :NodeCreation
3112--
3113-- identifier = newNode( "identifier", name [, attributeName="" ] ) -- 'attributeName' can be "close", "const" or "".
3114-- vararg = newNode( "vararg" )
3115-- literal = newNode( "literal", value ) -- 'value' must be a number, a string, a boolean or nil.
3116-- tableNode = newNode( "table" )
3117-- lookup = newNode( "lookup" )
3118-- unary = newNode( "unary", unaryOperator )
3119-- binary = newNode( "binary", binaryOperator )
3120-- call = newNode( "call" )
3121-- functionNode = newNode( "function" )
3122-- breakNode = newNode( "break" )
3123-- returnNode = newNode( "return" )
3124-- label = newNode( "label", labelName )
3125-- gotoNode = newNode( "goto", labelName )
3126-- block = newNode( "block" )
3127-- declaration = newNode( "declaration" )
3128-- assignment = newNode( "assignment" )
3129-- ifNode = newNode( "if" )
3130-- whileLoop = newNode( "while" )
3131-- repeatLoop = newNode( "repeat" )
3132-- forLoop = newNode( "for", forLoopKind ) -- 'forLoopKind' can be "numeric" or "generic".
3133--
3134-- Search for 'NodeFields' for each node's fields.
3135--
3136local function newNode(nodeType, ...)
3137 if nodeConstructors[nodeType] then
3138 return (nodeConstructors[nodeType](select("#", ...), ...))
3139 else
3140 errorf(2, "Invalid node type '%s'.", tostring(nodeType))
3141 end
3142end
3143local function newNodeFast(nodeType, ...)
3144 return nodeConstructorsFast[nodeType](...)
3145end
3146
3147local cloneNodeArrayAndChildren
3148
3149local function cloneNodeAndMaybeChildren(node, cloneChildren)
3150 local nodeType = node.type
3151 local clone
3152
3153 if nodeType == "identifier" then
3154 clone = AstIdentifier(nil, node.name)
3155 clone.attribute = node.attribute
3156
3157 elseif nodeType == "vararg" then
3158 clone = AstVararg(nil)
3159 clone.adjustToOne = node.adjustToOne
3160
3161 elseif nodeType == "literal" then
3162 clone = AstLiteral(nil, node.value)
3163
3164 elseif nodeType == "break" then
3165 clone = AstBreak(nil)
3166
3167 elseif nodeType == "label" then
3168 clone = AstLabel(nil, node.name)
3169
3170 elseif nodeType == "goto" then
3171 clone = AstGoto(nil, node.name)
3172
3173 elseif nodeType == "lookup" then
3174 clone = AstLookup(nil)
3175
3176 if cloneChildren then
3177 clone.object = node.object and cloneNodeAndMaybeChildren(node.object, true)
3178 clone.member = node.member and cloneNodeAndMaybeChildren(node.member, true)
3179 end
3180
3181 elseif nodeType == "unary" then
3182 clone = AstUnary(nil, node.operator)
3183
3184 if cloneChildren then
3185 clone.expression = node.expression and cloneNodeAndMaybeChildren(node.expression, true)
3186 end
3187
3188 elseif nodeType == "binary" then
3189 clone = AstBinary(nil, node.operator)
3190
3191 if cloneChildren then
3192 clone.left = node.left and cloneNodeAndMaybeChildren(node.left, true)
3193 clone.right = node.right and cloneNodeAndMaybeChildren(node.right, true)
3194 end
3195
3196 elseif nodeType == "call" then
3197 clone = AstCall(nil)
3198 clone.method = node.method
3199 clone.adjustToOne = node.adjustToOne
3200
3201 if cloneChildren then
3202 clone.callee = node.callee and cloneNodeAndMaybeChildren(node.callee, true)
3203 cloneNodeArrayAndChildren(clone.arguments, node.arguments)
3204 end
3205
3206 elseif nodeType == "function" then
3207 clone = AstFunction(nil)
3208
3209 if cloneChildren then
3210 cloneNodeArrayAndChildren(clone.parameters, node.parameters)
3211 clone.body = node.body and cloneNodeAndMaybeChildren(node.body, true)
3212 end
3213
3214 elseif nodeType == "return" then
3215 clone = AstReturn(nil)
3216
3217 if cloneChildren then
3218 cloneNodeArrayAndChildren(clone.values, node.values)
3219 end
3220
3221 elseif nodeType == "block" then
3222 clone = AstBlock(nil)
3223
3224 if cloneChildren then
3225 cloneNodeArrayAndChildren(clone.statements, node.statements)
3226 end
3227
3228 elseif nodeType == "declaration" then
3229 clone = AstDeclaration(nil)
3230
3231 if cloneChildren then
3232 cloneNodeArrayAndChildren(clone.names, node.names)
3233 cloneNodeArrayAndChildren(clone.values, node.values)
3234 end
3235
3236 elseif nodeType == "assignment" then
3237 clone = AstAssignment(nil)
3238
3239 if cloneChildren then
3240 cloneNodeArrayAndChildren(clone.targets, node.targets)
3241 cloneNodeArrayAndChildren(clone.values, node.values)
3242 end
3243
3244 elseif nodeType == "if" then
3245 clone = AstIf(nil)
3246
3247 if cloneChildren then
3248 clone.condition = node.condition and cloneNodeAndMaybeChildren(node.condition, true)
3249 clone.bodyTrue = node.bodyTrue and cloneNodeAndMaybeChildren(node.bodyTrue, true)
3250 clone.bodyFalse = node.bodyFalse and cloneNodeAndMaybeChildren(node.bodyFalse, true)
3251 end
3252
3253 elseif nodeType == "while" then
3254 clone = AstWhile(nil)
3255
3256 if cloneChildren then
3257 clone.condition = node.condition and cloneNodeAndMaybeChildren(node.condition, true)
3258 clone.body = node.body and cloneNodeAndMaybeChildren(node.body, true)
3259 end
3260
3261 elseif nodeType == "repeat" then
3262 clone = AstRepeat(nil)
3263
3264 if cloneChildren then
3265 clone.body = node.body and cloneNodeAndMaybeChildren(node.body, true)
3266 clone.condition = node.condition and cloneNodeAndMaybeChildren(node.condition, true)
3267 end
3268
3269 elseif nodeType == "for" then
3270 clone = AstFor(nil, node.kind)
3271
3272 if cloneChildren then
3273 cloneNodeArrayAndChildren(clone.names, node.names)
3274 cloneNodeArrayAndChildren(clone.values, node.values)
3275 clone.body = node.body and cloneNodeAndMaybeChildren(node.body, true)
3276 end
3277
3278 elseif nodeType == "table" then
3279 clone = AstTable(nil)
3280
3281 if cloneChildren then
3282 for i, tableField in ipairs(node.fields) do
3283 clone.fields[i] = {
3284 key = tableField.key and cloneNodeAndMaybeChildren(tableField.key, true),
3285 value = tableField.value and cloneNodeAndMaybeChildren(tableField.value, true),
3286 generatedKey = tableField.generatedKey,
3287 }
3288 end
3289 end
3290
3291 else
3292 errorf("Invalid node type '%s'.", tostring(nodeType))
3293 end
3294
3295 clone.pretty = node.pretty
3296 clone.prefix = node.prefix
3297 clone.suffix = node.suffix
3298 -- Should we set node.token etc. too? @Incomplete
3299
3300 return clone
3301end
3302
3303--[[local]] function cloneNodeArrayAndChildren(cloneArray, sourceArray)
3304 for i, node in ipairs(sourceArray) do
3305 cloneArray[i] = cloneNodeAndMaybeChildren(node, true)
3306 end
3307end
3308
3309local function cloneNode(node)
3310 return (cloneNodeAndMaybeChildren(node, false))
3311end
3312
3313local function cloneTree(node)
3314 return (cloneNodeAndMaybeChildren(node, true))
3315end
3316
3317
3318
3319local INVOLVED_NEVER = newSet{ "function", "literal", "vararg" }
3320local INVOLVED_ALWAYS = newSet{ "break", "call", "goto", "label", "lookup", "return" }
3321
3322local mayAnyNodeBeInvolvedInJump
3323
3324local function mayNodeBeInvolvedInJump(node)
3325 if INVOLVED_NEVER[node.type] then
3326 return false
3327
3328 elseif INVOLVED_ALWAYS[node.type] then
3329 return true
3330
3331 elseif node.type == "identifier" then
3332 return (node.declaration == nil) -- Globals may invoke a metamethod on the environment.
3333
3334 elseif node.type == "binary" then
3335 return mayNodeBeInvolvedInJump(node.left) or mayNodeBeInvolvedInJump(node.right)
3336 elseif node.type == "unary" then
3337 return mayNodeBeInvolvedInJump(node.expression)
3338
3339 elseif node.type == "block" then
3340 return mayAnyNodeBeInvolvedInJump(node.statements)
3341
3342 elseif node.type == "if" then
3343 return mayNodeBeInvolvedInJump(node.condition) or mayNodeBeInvolvedInJump(node.bodyTrue) or (node.bodyFalse ~= nil and mayNodeBeInvolvedInJump(node.bodyFalse))
3344
3345 elseif node.type == "for" then
3346 return mayAnyNodeBeInvolvedInJump(node.values) or mayNodeBeInvolvedInJump(node.body)
3347 elseif node.type == "repeat" or node.type == "while" then
3348 return mayNodeBeInvolvedInJump(node.condition) or mayNodeBeInvolvedInJump(node.body)
3349
3350 elseif node.type == "declaration" then
3351 return mayAnyNodeBeInvolvedInJump(node.values)
3352 elseif node.type == "assignment" then
3353 return mayAnyNodeBeInvolvedInJump(node.targets) or mayAnyNodeBeInvolvedInJump(node.values) -- Targets may be identifiers or lookups.
3354
3355 elseif node.type == "table" then
3356 for _, tableField in ipairs(node.fields) do
3357 if mayNodeBeInvolvedInJump(tableField.key) then return true end
3358 if mayNodeBeInvolvedInJump(tableField.value) then return true end
3359 end
3360 return false
3361
3362 else
3363 errorf("Invalid/unhandled node type '%s'.", tostring(node.type))
3364 end
3365end
3366
3367--[[local]] function mayAnyNodeBeInvolvedInJump(nodes)
3368 for _, node in ipairs(nodes) do
3369 if mayNodeBeInvolvedInJump(node) then return true end
3370 end
3371 return false
3372end
3373
3374
3375
3376local printNode, printTree
3377do
3378 local function _printNode(node)
3379 local nodeType = node.type
3380
3381 ioWrite(nodeType)
3382
3383 if parser.printIds then ioWrite("#", node.id) end
3384
3385 -- if mayNodeBeInvolvedInJump(node) then ioWrite("[MAYJUMP]") end -- DEBUG
3386
3387 if nodeType == "identifier" then
3388 ioWrite(" (", node.name, ")")
3389
3390 if node.declaration then
3391 ioWrite(" (decl=", node.declaration.type)
3392 if parser.printIds then ioWrite("#", node.declaration.id) end
3393 ioWrite(")")
3394 end
3395
3396 elseif nodeType == "vararg" then
3397 if node.adjustToOne then ioWrite(" (adjustToOne)") end
3398
3399 if node.declaration then
3400 ioWrite(" (decl=", node.declaration.type)
3401 if parser.printIds then ioWrite("#", node.declaration.id) end
3402 ioWrite(")")
3403 end
3404
3405 elseif nodeType == "literal" then
3406 if node.value == nil or node.value == true or node.value == false then
3407 ioWrite(" (", tostring(node.value), ")")
3408 elseif type(node.value) == "string" then
3409 ioWrite(' (string="', ensurePrintable(node.value), '")')
3410 else
3411 ioWrite(" (", type(node.value), "=", tostring(node.value), ")")
3412 end
3413
3414 elseif nodeType == "unary" then
3415 ioWrite(" (", node.operator, ")")
3416
3417 elseif nodeType == "binary" then
3418 ioWrite(" (", node.operator, ")")
3419
3420 elseif nodeType == "call" then
3421 if node.method then ioWrite(" (method)" ) end
3422 if node.adjustToOne then ioWrite(" (adjustToOne)") end
3423
3424 elseif nodeType == "function" then
3425 if node.parameters[1] and node.parameters[#node.parameters].type == "vararg" then ioWrite(" (vararg)") end
3426
3427 elseif nodeType == "for" then
3428 ioWrite(" (", node.kind, ")")
3429
3430 elseif nodeType == "label" then
3431 ioWrite(" (", node.name, ")")
3432
3433 elseif nodeType == "goto" then
3434 ioWrite(" (", node.name, ")")
3435
3436 if node.label then
3437 ioWrite(" (label")
3438 if parser.printIds then ioWrite("#", node.label.id) end
3439 ioWrite(")")
3440 end
3441 end
3442
3443 if parser.printLocations then ioWrite(" @ ", node.sourcePath, ":", node.line) end
3444
3445 ioWrite("\n")
3446 end
3447
3448 local function _printTree(node, indent, key)
3449 for i = 1, indent do ioWrite(parser.indentation) end
3450 indent = indent+1
3451
3452 if key ~= nil then
3453 ioWrite(tostring(key), " ")
3454 end
3455
3456 _printNode(node)
3457
3458 local nodeType = node.type
3459
3460 if nodeType == "table" then
3461 for i, tableField in ipairs(node.fields) do
3462 if tableField.key then
3463 _printTree(tableField.key, indent, i..(tableField.generatedKey and "KEYGEN" or "KEY "))
3464 elseif tableField.generatedKey then
3465 for i = 1, indent do ioWrite(parser.indentation) end
3466 ioWrite(i, "KEYGEN -\n")
3467 end
3468 if tableField.value then _printTree(tableField.value, indent, i.."VALUE ") end
3469 end
3470
3471 elseif nodeType == "lookup" then
3472 if node.object then _printTree(node.object, indent, "OBJECT") end
3473 if node.member then _printTree(node.member, indent, "MEMBER") end
3474
3475 elseif nodeType == "unary" then
3476 if node.expression then _printTree(node.expression, indent, nil) end
3477
3478 elseif nodeType == "binary" then
3479 if node.left then _printTree(node.left, indent, nil) end
3480 for i = 1, indent do ioWrite(parser.indentation) end ; ioWrite(node.operator, "\n")
3481 if node.right then _printTree(node.right, indent, nil) end
3482
3483 elseif nodeType == "call" then
3484 if node.callee then _printTree(node.callee, indent, "CALLEE") end
3485 for i, expr in ipairs(node.arguments) do _printTree(expr, indent, "ARG"..i) end
3486
3487 elseif nodeType == "function" then
3488 for i, ident in ipairs(node.parameters) do _printTree(ident, indent, "PARAM"..i) end
3489 if node.body then _printTree(node.body, indent, "BODY") end
3490
3491 elseif nodeType == "return" then
3492 for i, expr in ipairs(node.values) do _printTree(expr, indent, tostring(i)) end
3493
3494 elseif nodeType == "block" then
3495 for i, statement in ipairs(node.statements) do _printTree(statement, indent, tostring(i)) end
3496
3497 elseif nodeType == "declaration" then
3498 for i, ident in ipairs(node.names) do _printTree(ident, indent, "NAME"..i) end
3499 for i, expr in ipairs(node.values) do _printTree(expr, indent, "VALUE"..i) end
3500
3501 elseif nodeType == "assignment" then
3502 for i, expr in ipairs(node.targets) do _printTree(expr, indent, "TARGET"..i) end
3503 for i, expr in ipairs(node.values) do _printTree(expr, indent, "VALUE"..i) end
3504
3505 elseif nodeType == "if" then
3506 if node.condition then _printTree(node.condition, indent, "CONDITION") end
3507 if node.bodyTrue then _printTree(node.bodyTrue, indent, "BODY" ) end
3508
3509 local i = 1
3510
3511 while node.bodyFalse do
3512 -- Automatically detect what looks like 'elseif'.
3513 if #node.bodyFalse.statements == 1 and node.bodyFalse.statements[1].type == "if" then
3514 i = i+1
3515 node = node.bodyFalse.statements[1]
3516
3517 if node.condition then _printTree(node.condition, indent, "ELSEIF" ) end
3518 if node.bodyTrue then _printTree(node.bodyTrue, indent, "BODY"..i) end
3519
3520 else
3521 _printTree(node.bodyFalse, indent, "ELSE")
3522 break
3523 end
3524 end
3525
3526 elseif nodeType == "while" then
3527 if node.condition then _printTree(node.condition, indent, "CONDITION") end
3528 if node.body then _printTree(node.body, indent, "BODY" ) end
3529
3530 elseif nodeType == "repeat" then
3531 if node.body then _printTree(node.body, indent, "BODY" ) end
3532 if node.condition then _printTree(node.condition, indent, "CONDITION") end
3533
3534 elseif nodeType == "for" then
3535 for i, ident in ipairs(node.names) do _printTree(ident, indent, "NAME"..i) end
3536 for i, expr in ipairs(node.values) do _printTree(expr, indent, "VALUE"..i) end
3537 if node.body then _printTree(node.body, indent, "BODY") end
3538 end
3539 end
3540
3541 --[[local]] function printNode(node)
3542 _printNode(node)
3543 end
3544
3545 --[[local]] function printTree(node)
3546 _printTree(node, 0, nil)
3547 end
3548end
3549
3550
3551
3552-- didStop = traverseTree( astNode, [ leavesFirst=false, ] callback [, topNodeParent=nil, topNodeContainer=nil, topNodeKey=nil ] )
3553-- action = callback( astNode, parent, container, key )
3554-- action = "stop"|"ignorechildren"|nil -- Returning nil (or nothing) means continue traversal.
3555local function traverseTree(node, leavesFirst, cb, parent, container, k)
3556 assertArg1("traverseTree", 1, node, "table")
3557
3558 if type(leavesFirst) == "boolean" then
3559 assertArg1("traverseTree", 3, cb, "function")
3560 else
3561 leavesFirst, cb, parent, container, k = false, leavesFirst, cb, parent, container
3562 assertArg1("traverseTree", 2, cb, "function")
3563 end
3564
3565 if not leavesFirst then
3566 local action = cb(node, parent, container, k)
3567 if action == "stop" then return true end
3568 if action == "ignorechildren" then return false end
3569 if action then errorf("Unknown traversal action '%s' returned from callback.", tostring(action)) end
3570 end
3571
3572 local nodeType = node.type
3573
3574 if nodeType == "identifier" or nodeType == "vararg" or nodeType == "literal" or nodeType == "break" or nodeType == "label" or nodeType == "goto" then
3575 -- void No child nodes.
3576
3577 elseif nodeType == "table" then
3578 for _, tableField in ipairs(node.fields) do
3579 if tableField.key and traverseTree(tableField.key, leavesFirst, cb, node, tableField, "key") then return true end
3580 if tableField.value and traverseTree(tableField.value, leavesFirst, cb, node, tableField, "value") then return true end
3581 end
3582
3583 elseif nodeType == "lookup" then
3584 if node.object and traverseTree(node.object, leavesFirst, cb, node, node, "object") then return true end
3585 if node.member and traverseTree(node.member, leavesFirst, cb, node, node, "member") then return true end
3586
3587 elseif nodeType == "unary" then
3588 if node.expression and traverseTree(node.expression, leavesFirst, cb, node, node, "expression") then return true end
3589
3590 elseif nodeType == "binary" then
3591 if node.left and traverseTree(node.left, leavesFirst, cb, node, node, "left") then return true end
3592 if node.right and traverseTree(node.right, leavesFirst, cb, node, node, "right") then return true end
3593
3594 elseif nodeType == "call" then
3595 if node.callee and traverseTree(node.callee, leavesFirst, cb, node, node, "callee") then return true end
3596 for i, expr in ipairs(node.arguments) do
3597 if traverseTree(expr, leavesFirst, cb, node, node.arguments, i) then return true end
3598 end
3599
3600 elseif nodeType == "function" then
3601 for i, name in ipairs(node.parameters) do
3602 if traverseTree(name, leavesFirst, cb, node, node.parameters, i) then return true end
3603 end
3604 if node.body and traverseTree(node.body, leavesFirst, cb, node, node, "body") then return true end
3605
3606 elseif nodeType == "return" then
3607 for i, expr in ipairs(node.values) do
3608 if traverseTree(expr, leavesFirst, cb, node, node.values, i) then return true end
3609 end
3610
3611 elseif nodeType == "block" then
3612 for i, statement in ipairs(node.statements) do
3613 if traverseTree(statement, leavesFirst, cb, node, node.statements, i) then return true end
3614 end
3615
3616 elseif nodeType == "declaration" then
3617 for i, ident in ipairs(node.names) do
3618 if traverseTree(ident, leavesFirst, cb, node, node.names, i) then return true end
3619 end
3620 for i, expr in ipairs(node.values) do
3621 if traverseTree(expr, leavesFirst, cb, node, node.values, i) then return true end
3622 end
3623
3624 elseif nodeType == "assignment" then
3625 for i, expr in ipairs(node.targets) do
3626 if traverseTree(expr, leavesFirst, cb, node, node.targets, i) then return true end
3627 end
3628 for i, expr in ipairs(node.values) do
3629 if traverseTree(expr, leavesFirst, cb, node, node.values, i) then return true end
3630 end
3631
3632 elseif nodeType == "if" then
3633 if node.condition and traverseTree(node.condition, leavesFirst, cb, node, node, "condition") then return true end
3634 if node.bodyTrue and traverseTree(node.bodyTrue, leavesFirst, cb, node, node, "bodyTrue") then return true end
3635 if node.bodyFalse and traverseTree(node.bodyFalse, leavesFirst, cb, node, node, "bodyFalse") then return true end
3636
3637 elseif nodeType == "while" then
3638 if node.condition and traverseTree(node.condition, leavesFirst, cb, node, node, "condition") then return true end
3639 if node.body and traverseTree(node.body, leavesFirst, cb, node, node, "body") then return true end
3640
3641 elseif nodeType == "repeat" then
3642 if node.body and traverseTree(node.body, leavesFirst, cb, node, node, "body") then return true end
3643 if node.condition and traverseTree(node.condition, leavesFirst, cb, node, node, "condition") then return true end
3644
3645 elseif nodeType == "for" then
3646 for i, ident in ipairs(node.names) do
3647 if traverseTree(ident, leavesFirst, cb, node, node.names, i) then return true end
3648 end
3649 for i, expr in ipairs(node.values) do
3650 if traverseTree(expr, leavesFirst, cb, node, node.values, i) then return true end
3651 end
3652 if node.body and traverseTree(node.body, leavesFirst, cb, node, node, "body") then return true end
3653
3654 else
3655 errorf("Invalid node type '%s'.", tostring(nodeType))
3656 end
3657
3658 if leavesFirst then
3659 local action = cb(node, parent, container, k)
3660 if action == "stop" then return true end
3661 if action == "ignorechildren" then errorf("Cannot ignore children when leavesFirst is set.") end
3662 if action then errorf("Unknown traversal action '%s' returned from callback.", tostring(action)) end
3663 end
3664
3665 return false
3666end
3667
3668-- didStop = traverseTreeReverse( astNode, [ leavesFirst=false, ] callback [, topNodeParent=nil, topNodeContainer=nil, topNodeKey=nil ] )
3669-- action = callback( astNode, parent, container, key )
3670-- action = "stop"|"ignorechildren"|nil -- Returning nil (or nothing) means continue traversal.
3671local function traverseTreeReverse(node, leavesFirst, cb, parent, container, k)
3672 assertArg1("traverseTreeReverse", 1, node, "table")
3673
3674 if type(leavesFirst) == "boolean" then
3675 assertArg1("traverseTreeReverse", 3, cb, "function")
3676 else
3677 leavesFirst, cb, parent, container, k = false, leavesFirst, cb, parent, container
3678 assertArg1("traverseTreeReverse", 2, cb, "function")
3679 end
3680
3681 if not leavesFirst then
3682 local action = cb(node, parent, container, k)
3683 if action == "stop" then return true end
3684 if action == "ignorechildren" then return false end
3685 if action then errorf("Unknown traversal action '%s' returned from callback.", tostring(action)) end
3686 end
3687
3688 local nodeType = node.type
3689
3690 if nodeType == "identifier" or nodeType == "vararg" or nodeType == "literal" or nodeType == "break" or nodeType == "label" or nodeType == "goto" then
3691 -- void No child nodes.
3692
3693 elseif nodeType == "table" then
3694 for _, tableField in ipairsr(node.fields) do
3695 if tableField.value and traverseTreeReverse(tableField.value, leavesFirst, cb, node, tableField, "value") then return true end
3696 if tableField.key and traverseTreeReverse(tableField.key, leavesFirst, cb, node, tableField, "key") then return true end
3697 end
3698
3699 elseif nodeType == "lookup" then
3700 if node.member and traverseTreeReverse(node.member, leavesFirst, cb, node, node, "member") then return true end
3701 if node.object and traverseTreeReverse(node.object, leavesFirst, cb, node, node, "object") then return true end
3702
3703 elseif nodeType == "unary" then
3704 if node.expression and traverseTreeReverse(node.expression, leavesFirst, cb, node, node, "expression") then return true end
3705
3706 elseif nodeType == "binary" then
3707 if node.right and traverseTreeReverse(node.right, leavesFirst, cb, node, node, "right") then return true end
3708 if node.left and traverseTreeReverse(node.left, leavesFirst, cb, node, node, "left") then return true end
3709
3710 elseif nodeType == "call" then
3711 for i, expr in ipairsr(node.arguments) do
3712 if traverseTreeReverse(expr, leavesFirst, cb, node, node.arguments, i) then return true end
3713 end
3714 if node.callee and traverseTreeReverse(node.callee, leavesFirst, cb, node, node, "callee") then return true end
3715
3716 elseif nodeType == "function" then
3717 if node.body and traverseTreeReverse(node.body, leavesFirst, cb, node, node, "body") then return true end
3718 for i, name in ipairsr(node.parameters) do
3719 if traverseTreeReverse(name, leavesFirst, cb, node, node.parameters, i) then return true end
3720 end
3721
3722 elseif nodeType == "return" then
3723 for i, expr in ipairsr(node.values) do
3724 if traverseTreeReverse(expr, leavesFirst, cb, node, node.values, i) then return true end
3725 end
3726
3727 elseif nodeType == "block" then
3728 for i, statement in ipairsr(node.statements) do
3729 if traverseTreeReverse(statement, leavesFirst, cb, node, node.statements, i) then return true end
3730 end
3731
3732 elseif nodeType == "declaration" then
3733 for i, expr in ipairsr(node.values) do
3734 if traverseTreeReverse(expr, leavesFirst, cb, node, node.values, i) then return true end
3735 end
3736 for i, ident in ipairsr(node.names) do
3737 if traverseTreeReverse(ident, leavesFirst, cb, node, node.names, i) then return true end
3738 end
3739
3740 elseif nodeType == "assignment" then
3741 for i, expr in ipairsr(node.values) do
3742 if traverseTreeReverse(expr, leavesFirst, cb, node, node.values, i) then return true end
3743 end
3744 for i, expr in ipairsr(node.targets) do
3745 if traverseTreeReverse(expr, leavesFirst, cb, node, node.targets, i) then return true end
3746 end
3747
3748 elseif nodeType == "if" then
3749 if node.bodyFalse and traverseTreeReverse(node.bodyFalse, leavesFirst, cb, node, node, "bodyFalse") then return true end
3750 if node.bodyTrue and traverseTreeReverse(node.bodyTrue, leavesFirst, cb, node, node, "bodyTrue") then return true end
3751 if node.condition and traverseTreeReverse(node.condition, leavesFirst, cb, node, node, "condition") then return true end
3752
3753 elseif nodeType == "while" then
3754 if node.body and traverseTreeReverse(node.body, leavesFirst, cb, node, node, "body") then return true end
3755 if node.condition and traverseTreeReverse(node.condition, leavesFirst, cb, node, node, "condition") then return true end
3756
3757 elseif nodeType == "repeat" then
3758 if node.condition and traverseTreeReverse(node.condition, leavesFirst, cb, node, node, "condition") then return true end
3759 if node.body and traverseTreeReverse(node.body, leavesFirst, cb, node, node, "body") then return true end
3760
3761 elseif nodeType == "for" then
3762 if node.body and traverseTreeReverse(node.body, leavesFirst, cb, node, node, "body") then return true end
3763 for i, expr in ipairsr(node.values) do
3764 if traverseTreeReverse(expr, leavesFirst, cb, node, node.values, i) then return true end
3765 end
3766 for i, ident in ipairsr(node.names) do
3767 if traverseTreeReverse(ident, leavesFirst, cb, node, node.names, i) then return true end
3768 end
3769
3770 else
3771 errorf("Invalid node type '%s'.", tostring(nodeType))
3772 end
3773
3774 if leavesFirst then
3775 local action = cb(node, parent, container, k)
3776 if action == "stop" then return true end
3777 if action == "ignorechildren" then errorf("Cannot ignore children when leavesFirst is set.") end
3778 if action then errorf("Unknown traversal action '%s' returned from callback.", tostring(action)) end
3779 end
3780
3781 return false
3782end
3783
3784
3785
3786-- declIdent | nil = findIdentifierDeclaration( ident )
3787-- declIdent.parent = decl | func | forLoop
3788local function findIdentifierDeclaration(ident)
3789 local name = ident.name
3790 local parent = ident
3791
3792 while true do
3793 local lastChild = parent
3794 parent = parent.parent
3795
3796 if not parent then return nil end
3797
3798 if parent.type == "declaration" then
3799 local decl = parent
3800
3801 if lastChild.container == decl.names then
3802 assert(lastChild == ident)
3803 return ident -- ident is the declaration node.
3804 end
3805
3806 elseif parent.type == "function" then
3807 local func = parent
3808
3809 if lastChild.container == func.parameters then
3810 assert(lastChild == ident)
3811 return ident -- ident is the declaration node.
3812 else
3813 local func = parent
3814 local declIdent = lastItemWith1(func.parameters, "name", name) -- Note: This will ignore any vararg parameter.
3815 if declIdent then return declIdent end
3816 end
3817
3818 elseif parent.type == "for" then
3819 local forLoop = parent
3820
3821 if lastChild.container == forLoop.names then
3822 assert(lastChild == ident)
3823 return ident -- ident is the declaration node.
3824 elseif lastChild.container ~= forLoop.values then
3825 local declIdent = lastItemWith1(forLoop.names, "name", name)
3826 if declIdent then return declIdent end
3827 end
3828
3829 elseif parent.type == "block" then
3830 local block = parent
3831
3832 -- Look through statements above lastChild.
3833 for i = lastChild.key-1, 1, -1 do
3834 local statement = block.statements[i]
3835
3836 if statement.type == "declaration" then
3837 local decl = statement
3838 local declIdent = lastItemWith1(decl.names, "name", name)
3839 if declIdent then return declIdent end
3840 end
3841 end
3842
3843 elseif parent.type == "repeat" then
3844 local repeatLoop = parent
3845
3846 -- Repeat loop conditions can see into the loop block.
3847 if lastChild == repeatLoop.condition then
3848 local block = repeatLoop.body
3849
3850 for i = #block.statements, 1, -1 do
3851 local statement = block.statements[i]
3852
3853 if statement.type == "declaration" then
3854 local decl = statement
3855 local declIdent = lastItemWith1(decl.names, "name", name)
3856 if declIdent then return declIdent end
3857 end
3858 end
3859 end
3860 end
3861 end
3862end
3863
3864-- declVararg|nil = findVarargDeclaration( vararg )
3865-- declVararg.parent = func
3866local function findVarargDeclaration(vararg)
3867 local parent = vararg
3868
3869 while true do
3870 parent = parent.parent
3871 if not parent then return nil end
3872
3873 if parent.type == "function" then
3874 local lastParam = parent.parameters[#parent.parameters]
3875
3876 if lastParam and lastParam.type == "vararg" then
3877 return lastParam
3878 else
3879 return nil
3880 end
3881 end
3882 end
3883end
3884
3885local function findLabel(gotoNode)
3886 local name = gotoNode.name
3887 local parent = gotoNode
3888
3889 while true do
3890 parent = parent.parent
3891 if not parent then return nil end
3892
3893 if parent.type == "block" then
3894 local block = parent
3895
3896 for _, statement in ipairs(block.statements) do
3897 if statement.type == "label" and statement.name == name then
3898 return statement
3899 end
3900 end
3901
3902 elseif parent.type == "function" then
3903 return nil
3904 end
3905 end
3906end
3907
3908local function updateReferences(node, updateTopNodePositionInfo)
3909 local topNodeParent = nil
3910 local topNodeContainer = nil
3911 local topNodeKey = nil
3912
3913 if updateTopNodePositionInfo == false then
3914 topNodeParent = node.parent
3915 topNodeContainer = node.container
3916 topNodeKey = node.key
3917 end
3918
3919 traverseTree(node, function(node, parent, container, key)
3920 node.parent = parent
3921 node.container = container
3922 node.key = key
3923
3924 if node.type == "identifier" then
3925 local ident = node
3926 ident.declaration = findIdentifierDeclaration(ident) -- We can call this because all parents and previous nodes already have their references updated at this point.
3927
3928 --[[ DEBUG
3929 print(F(
3930 "%-10s %-12s %s",
3931 ident.name,
3932 ( ident.declaration and ident.declaration.type or ""),
3933 tostring(ident.declaration and ident.declaration.id or "")
3934 ))
3935 --]]
3936
3937 elseif node.type == "vararg" then
3938 local vararg = node
3939 vararg.declaration = findVarargDeclaration(vararg) -- We can call this because all relevant 'parent' references have been updated at this point.
3940
3941 --[[ DEBUG
3942 print(F(
3943 "%-10s %-12s %s",
3944 "...",
3945 ( vararg.declaration and vararg.declaration.type or ""),
3946 tostring(vararg.declaration and vararg.declaration.id or "")
3947 ))
3948 --]]
3949
3950 elseif node.type == "goto" then
3951 local gotoNode = node
3952 gotoNode.label = findLabel(gotoNode) -- We can call this because all relevant 'parent' references have been updated at this point.
3953 end
3954 end, topNodeParent, topNodeContainer, topNodeKey)
3955end
3956
3957
3958
3959local function replace(node, replacement, parent, container, key, stats)
3960 tableInsert(stats.nodeReplacements, Location(container[key], "replacement", replacement))
3961
3962 container[key] = replacement
3963
3964 replacement.sourceString = node.sourceString
3965 replacement.sourcePath = node.sourcePath
3966
3967 replacement.token = node.token
3968 replacement.line = node.line
3969 replacement.position = node.position
3970
3971 replacement.parent = parent
3972 replacement.container = container
3973 replacement.key = key
3974end
3975
3976
3977
3978local PATTERN_INT_TO_HEX = F("%%0%dx", INT_SIZE/4)
3979
3980local HEX_DIGIT_TO_BITS = {
3981 ["0"]={0,0,0,0}, ["1"]={0,0,0,1}, ["2"]={0,0,1,0}, ["3"]={0,0,1,1}, ["4"]={0,1,0,0}, ["5"]={0,1,0,1}, ["6"]={0,1,1,0}, ["7"]={0,1,1,1},
3982 ["8"]={1,0,0,0}, ["9"]={1,0,0,1}, ["a"]={1,0,1,0}, ["b"]={1,0,1,1}, ["c"]={1,1,0,0}, ["d"]={1,1,0,1}, ["e"]={1,1,1,0}, ["f"]={1,1,1,1},
3983}
3984
3985local function intToBits(n, bitsOut)
3986 local hexNumber = stringSub(F(PATTERN_INT_TO_HEX, maybeWrapInt(n)), -INT_SIZE/4) -- The stringSub() call is probably not needed, but just to be safe.
3987 local i = 1
3988
3989 for hexDigit in stringGmatch(hexNumber, ".") do
3990 local bits = HEX_DIGIT_TO_BITS[hexDigit]
3991
3992 for iOffset = 1, 4 do
3993 bitsOut[i] = bits[iOffset]
3994 i = i + 1
3995 end
3996 end
3997end
3998
3999local function bitsToInt(bits)
4000 local n = 0
4001 local multi = 1
4002
4003 for i = INT_SIZE, 2, -1 do
4004 n = n + multi * bits[i]
4005 multi = multi * 2
4006 end
4007
4008 return (bits[1] == 1 and MIN_INT+n or n)
4009end
4010
4011local function isValueNumberOrString(v)
4012 return type(v) == "number" or type(v) == "string"
4013end
4014local function isValueFiniteNumber(v) -- Should we actually use the logic of isValueNumberLike() where this function is used? @Incomplete
4015 return type(v) == "number" and v == v and v ~= 1/0 and v ~= -1/0
4016end
4017local function isValueNumberLike(v)
4018 return tonumber(v) ~= nil
4019end
4020local function areValuesNumbersOrStringsAndOfSameType(v1, v2)
4021 local type1 = type(v1)
4022 return type1 == type(v2) and (type1 == "number" or type1 == "string")
4023end
4024
4025local bits1 = {}
4026local bits2 = {}
4027
4028local unaryFolders = {
4029 ["-"] = function(unary, expr)
4030 -- -numberLike -> number
4031 if expr.type == "literal" and isValueNumberLike(expr.value) then
4032 return AstLiteral(unary.token, -expr.value) -- This may convert a string to a number.
4033 end
4034
4035 return nil
4036 end,
4037
4038 ["not"] = function(unary, expr)
4039 -- not literal -> boolean
4040 if expr.type == "literal" then -- :SimplifyTruthfulValues
4041 return AstLiteral(unary.token, (not expr.value))
4042
4043 -- not (x == y) -> x ~= y
4044 -- not (x ~= y) -> x == y
4045 elseif expr.type == "binary" then
4046 if expr.operator == "==" then
4047 local binary = AstBinary(unary.token, "~=")
4048 binary.left = expr.left
4049 binary.right = expr.right
4050 return binary
4051 elseif expr.operator == "~=" then
4052 local binary = AstBinary(unary.token, "==")
4053 binary.left = expr.left
4054 binary.right = expr.right
4055 return binary
4056 end
4057 end
4058
4059 return nil
4060 end,
4061
4062 ["#"] = function(unary, expr)
4063 -- #string -> number
4064 if expr.type == "literal" and type(expr.value) == "string" then
4065 return AstLiteral(unary.token, #expr.value)
4066 end
4067
4068 -- We could get the length of tables containing only constants, but who in their right mind writes #{}?
4069
4070 return nil
4071 end,
4072
4073 ["~"] = function(unary, expr)
4074 -- ~number -> number
4075 if expr.type == "literal" and isValueFiniteNumber(expr.value) then
4076 intToBits(expr.value, bits1)
4077 for i = 1, INT_SIZE do
4078 bits1[i] = 1 - bits1[i]
4079 end
4080 return AstLiteral(unary.token, bitsToInt(bits1))
4081 end
4082
4083 return nil
4084 end,
4085}
4086
4087local binaryFolders = {
4088 ["+"] = function(binary, l, r)
4089 -- numberLike + numberLike -> number
4090 if l.type == "literal" and r.type == "literal" and isValueNumberLike(l.value) and isValueNumberLike(r.value) then
4091 return AstLiteral(binary.token, l.value+r.value)
4092 end
4093
4094 return nil
4095 end,
4096
4097 ["-"] = function(binary, l, r)
4098 -- numberLike - numberLike -> number
4099 if l.type == "literal" and r.type == "literal" and isValueNumberLike(l.value) and isValueNumberLike(r.value) then
4100 return AstLiteral(binary.token, l.value-r.value)
4101 end
4102
4103 return nil
4104 end,
4105
4106 ["*"] = function(binary, l, r)
4107 -- numberLike * numberLike -> number
4108 if l.type == "literal" and r.type == "literal" and isValueNumberLike(l.value) and isValueNumberLike(r.value) then
4109 return AstLiteral(binary.token, l.value*r.value)
4110 end
4111
4112 return nil
4113 end,
4114
4115 ["/"] = function(binary, l, r)
4116 -- numberLike / numberLike -> number
4117 if l.type == "literal" and r.type == "literal" and isValueNumberLike(l.value) and isValueNumberLike(r.value) then
4118 return AstLiteral(binary.token, l.value/r.value)
4119 end
4120
4121 return nil
4122 end,
4123
4124 ["//"] = function(binary, l, r)
4125 -- numberLike // numberLike -> number
4126 if l.type == "literal" and r.type == "literal" and isValueNumberLike(l.value) and isValueNumberLike(r.value) then
4127 return AstLiteral(binary.token, mathFloor(l.value/r.value))
4128 end
4129
4130 return nil
4131 end,
4132
4133 ["^"] = function(binary, l, r)
4134 -- numberLike ^ numberLike -> number
4135 if l.type == "literal" and r.type == "literal" and isValueNumberLike(l.value) and isValueNumberLike(r.value) then
4136 return AstLiteral(binary.token, l.value^r.value)
4137 end
4138
4139 return nil
4140 end,
4141
4142 ["%"] = function(binary, l, r)
4143 -- numberLike % numberLike -> number
4144 if l.type == "literal" and r.type == "literal" and isValueNumberLike(l.value) and isValueNumberLike(r.value) then
4145 return AstLiteral(binary.token, l.value%r.value)
4146 end
4147
4148 return nil
4149 end,
4150
4151 ["&"] = function(binary, l, r)
4152 -- number & number -> number
4153 if l.type == "literal" and r.type == "literal" and isValueFiniteNumber(l.value) and isValueFiniteNumber(r.value) then
4154 intToBits(l.value, bits1)
4155 intToBits(r.value, bits2)
4156 for i = 1, INT_SIZE do
4157 bits1[i] = (bits1[i] == 1 and bits2[i] == 1) and 1 or 0
4158 end
4159 return AstLiteral(binary.token, bitsToInt(bits1))
4160 end
4161
4162 return nil
4163 end,
4164
4165 ["~"] = function(binary, l, r)
4166 -- number ~ number -> number
4167 if l.type == "literal" and r.type == "literal" and isValueFiniteNumber(l.value) and isValueFiniteNumber(r.value) then
4168 intToBits(l.value, bits1)
4169 intToBits(r.value, bits2)
4170 for i = 1, INT_SIZE do
4171 bits1[i] = (bits1[i] == 1) == (bits2[i] ~= 1) and 1 or 0
4172 end
4173 return AstLiteral(binary.token, bitsToInt(bits1))
4174 end
4175
4176 return nil
4177 end,
4178
4179 ["|"] = function(binary, l, r)
4180 -- number | number -> number
4181 if l.type == "literal" and r.type == "literal" and isValueFiniteNumber(l.value) and isValueFiniteNumber(r.value) then
4182 intToBits(l.value, bits1)
4183 intToBits(r.value, bits2)
4184 for i = 1, INT_SIZE do
4185 if bits1[i] == 0 then bits1[i] = bits2[i] end
4186 end
4187 return AstLiteral(binary.token, bitsToInt(bits1))
4188 end
4189
4190 return nil
4191 end,
4192
4193 [">>"] = function(binary, l, r)
4194 -- number >> number -> number
4195 if l.type == "literal" and r.type == "literal" and isValueFiniteNumber(l.value) and type(r.value) == "number" then
4196 intToBits(l.value, bits1)
4197
4198 local shift = mathFloor(r.value)
4199 local i1 = INT_SIZE
4200 local i2 = 1
4201 local step = -1
4202
4203 if shift < 0 then
4204 i1, i2 = i2, i1
4205 step = -step
4206 end
4207
4208 for i = i1, i2, step do
4209 bits1[i] = bits1[i-shift] or 0
4210 end
4211
4212 return AstLiteral(binary.token, bitsToInt(bits1))
4213 end
4214
4215 return nil
4216 end,
4217
4218 ["<<"] = function(binary, l, r)
4219 -- number << number -> number
4220 if l.type == "literal" and r.type == "literal" and isValueFiniteNumber(l.value) and type(r.value) == "number" then
4221 intToBits(l.value, bits1)
4222
4223 local shift = mathFloor(r.value)
4224 local i1 = 1
4225 local i2 = INT_SIZE
4226 local step = 1
4227
4228 if shift < 0 then
4229 i1, i2 = i2, i1
4230 step = -step
4231 end
4232
4233 for i = i1, i2, step do
4234 bits1[i] = bits1[i+shift] or 0
4235 end
4236
4237 return AstLiteral(binary.token, bitsToInt(bits1))
4238 end
4239
4240 return nil
4241 end,
4242
4243 [".."] = function(binary, l, r)
4244 -- numberOrString .. numberOrString -> string
4245 if l.type == "literal" and r.type == "literal" and isValueNumberOrString(l.value) and isValueNumberOrString(r.value) then
4246 return AstLiteral(binary.token, l.value..r.value)
4247 end
4248
4249 return nil
4250 end,
4251
4252 ["<"] = function(binary, l, r)
4253 -- number < number -> boolean
4254 -- string < string -> boolean
4255 if l.type == "literal" and r.type == "literal" and areValuesNumbersOrStringsAndOfSameType(l.value, r.value) then
4256 return AstLiteral(binary.token, (l.value < r.value))
4257 end
4258
4259 return nil
4260 end,
4261
4262 ["<="] = function(binary, l, r)
4263 -- number <= number -> boolean
4264 -- string <= string -> boolean
4265 if l.type == "literal" and r.type == "literal" and areValuesNumbersOrStringsAndOfSameType(l.value, r.value) then
4266 return AstLiteral(binary.token, (l.value <= r.value))
4267 end
4268
4269 return nil
4270 end,
4271
4272 [">"] = function(binary, l, r)
4273 -- number > number -> boolean
4274 -- string > string -> boolean
4275 if l.type == "literal" and r.type == "literal" and areValuesNumbersOrStringsAndOfSameType(l.value, r.value) then
4276 return AstLiteral(binary.token, (l.value > r.value))
4277 end
4278
4279 return nil
4280 end,
4281
4282 [">="] = function(binary, l, r)
4283 -- number >= number -> boolean
4284 -- string >= string -> boolean
4285 if l.type == "literal" and r.type == "literal" and areValuesNumbersOrStringsAndOfSameType(l.value, r.value) then
4286 return AstLiteral(binary.token, (l.value >= r.value))
4287 end
4288
4289 return nil
4290 end,
4291
4292 ["=="] = function(binary, l, r)
4293 -- literal == literal -> boolean
4294 if l.type == "literal" and r.type == "literal" then
4295 return AstLiteral(binary.token, (l.value == r.value))
4296 end
4297
4298 return nil
4299 end,
4300
4301 ["~="] = function(binary, l, r)
4302 -- literal ~= literal -> boolean
4303 if l.type == "literal" and r.type == "literal" then
4304 return AstLiteral(binary.token, (l.value ~= r.value))
4305 end
4306
4307 return nil
4308 end,
4309
4310 ["and"] = function(binary, l, r)
4311 -- truthfulLiteral and x -> x
4312 -- untruthfulLiteral and x -> untruthfulLiteral
4313 if l.type == "literal" then return l.value and r or l end
4314
4315 return nil
4316 end,
4317
4318 ["or"] = function(binary, l, r)
4319 -- truthfulLiteral or x -> untruthfulLiteral
4320 -- untruthfulLiteral or x -> x
4321 if l.type == "literal" then return l.value and l or r end
4322
4323 return nil
4324 end,
4325}
4326
4327local statsForSimplify
4328
4329local function simplifyNode(node, parent, container, key)
4330 if not parent then
4331 -- void
4332
4333 elseif node.type == "unary" then
4334 -- Note: We don't fold e.g. '- - -expr' into '-expr' because metamethods may
4335 -- be called, and folding '- -expr' into 'expr' would remove what could be a
4336 -- runtime error if 'expr' didn't contain a number.
4337 local replacement = unaryFolders[node.operator](node, node.expression)
4338 if replacement then replace(node, replacement, parent, container, key, statsForSimplify) end
4339
4340 elseif node.type == "binary" then
4341 -- @Incomplete: Fold 'expr - -n' into 'expr + n' etc. (Actually, this will probably mess up metamethods.)
4342 local replacement = binaryFolders[node.operator](node, node.left, node.right)
4343 if replacement then replace(node, replacement, parent, container, key, statsForSimplify) end
4344
4345 elseif node.type == "if" then
4346 -- @Incomplete: Fold 'if not not expr' into 'if expr'. (Also for 'while' and 'repeat' etc., i.e. all conditional expressions.)
4347 local ifNode = node
4348
4349 if ifNode.condition.type == "literal" then -- @Incomplete: There are more values that make simplification possible (e.g. functions, but who would put that here anyway). :SimplifyTruthfulValues
4350 local replacement = ifNode.condition.value and ifNode.bodyTrue or ifNode.bodyFalse
4351
4352 if replacement and replacement.statements[1] then
4353 replace(ifNode, replacement, parent, container, key, statsForSimplify)
4354 return simplifyNode(replacement, parent, container, key)
4355 else
4356 tableRemove(container, key)
4357 tableInsert(statsForSimplify.nodeRemovals, Location(ifNode))
4358 statsForSimplify.nodeRemoveCount = statsForSimplify.nodeRemoveCount + 1
4359 end
4360 end
4361
4362 elseif node.type == "while" then
4363 local whileLoop = node
4364
4365 if whileLoop.condition.type == "literal" then -- :SimplifyTruthfulValues
4366 if whileLoop.condition.value then
4367 whileLoop.condition.value = true -- Convert literal's value to boolean.
4368 else
4369 tableRemove(container, key)
4370 tableInsert(statsForSimplify.nodeRemovals, Location(whileLoop))
4371 statsForSimplify.nodeRemoveCount = statsForSimplify.nodeRemoveCount + 1
4372 end
4373 end
4374
4375 elseif node.type == "repeat" then
4376 local repeatLoop = node
4377
4378 if repeatLoop.condition.type == "literal" then -- :SimplifyTruthfulValues
4379 if repeatLoop.condition.value then
4380 replace(repeatLoop, repeatLoop.body, parent, container, key, statsForSimplify)
4381 return simplifyNode(repeatLoop.body, parent, container, key)
4382 else
4383 repeatLoop.condition.value = false -- Convert literal's value to boolean.
4384 end
4385 end
4386
4387 elseif node.type == "for" then
4388 local forLoop = node
4389
4390 if
4391 forLoop.kind == "numeric"
4392 and forLoop.values[2]
4393 and not forLoop.values[4] -- If this value exists then there will be an error when Lua tries to run the file, but it's not our job to handle that here.
4394 and forLoop.values[1].type == "literal" and type(forLoop.values[1].value) == "number"
4395 and forLoop.values[2].type == "literal" and type(forLoop.values[2].value) == "number"
4396 and (not forLoop.values[3] or (forLoop.values[3].type == "literal" and type(forLoop.values[3].value) == "number"))
4397 then
4398 local from = forLoop.values[1].value
4399 local to = forLoop.values[2].value
4400 local step = forLoop.values[3] and forLoop.values[3].value or 1
4401
4402 if (step > 0 and from > to) or (step < 0 and from < to) then
4403 tableRemove(container, key)
4404 tableInsert(statsForSimplify.nodeRemovals, Location(forLoop))
4405 statsForSimplify.nodeRemoveCount = statsForSimplify.nodeRemoveCount + 1
4406 end
4407 end
4408
4409 elseif node.type == "block" then
4410 if parent.type == "block" then
4411 local block = node
4412 local hasDeclarations = false
4413
4414 for _, statement in ipairs(block.statements) do
4415 if statement.type == "declaration" then
4416 hasDeclarations = true
4417 break
4418 end
4419 end
4420
4421 if not hasDeclarations then
4422 -- Blocks without declarations don't need a scope.
4423 tableRemove(parent.statements, key)
4424 tableInsert(statsForSimplify.nodeReplacements, Location(block, "replacement", nil)) -- We replace the block with its contents.
4425
4426 for i, statement in ipairs(block.statements) do
4427 tableInsert(parent.statements, key+i-1, statement)
4428 end
4429 end
4430 end
4431
4432 elseif node.type == "literal" then
4433 local literal = node
4434 if literal.value == 0 then literal.value = 0 end -- Normalize '-0'.
4435 end
4436end
4437
4438local function _simplify(node, stats)
4439 statsForSimplify = stats
4440 traverseTreeReverse(node, true, simplifyNode)
4441end
4442
4443local function simplify(node)
4444 local stats = Stats()
4445 _simplify(node, stats)
4446 return stats
4447end
4448
4449
4450
4451local function isNodeDeclLike(node)
4452 return node.type == "declaration" or node.type == "function" or node.type == "for"
4453end
4454
4455local function getNameArrayOfDeclLike(declLike)
4456 return declLike.names or declLike.parameters
4457end
4458
4459
4460
4461-- (statement, block) | declIdent | repeatLoop | declLike = findParentStatementAndBlockOrNodeOfInterest( node, declIdent )
4462local function findParentStatementAndBlockOrNodeOfInterest(node, declIdent)
4463 while true do
4464 if node == declIdent then return declIdent, nil end
4465
4466 local lastChild = node
4467 node = node.parent
4468
4469 if not node then return nil end
4470
4471 if node.type == "block" then return lastChild, node end
4472 if node.type == "declaration" and lastChild.container ~= node.values then return node, nil end
4473 if node.type == "function" then return node, nil end
4474 if node.type == "for" and lastChild.container ~= node.values then return node, nil end
4475 if node.type == "repeat" and lastChild == node.condition then return node, nil end
4476 end
4477end
4478
4479-- foundCurrentDeclIdent = lookForCurrentDeclIdentAndRegisterCurrentIdentAsWatcherInBlock( declIdentWatchers, currentIdentInfo, block, statementStartIndex )
4480local function lookForCurrentDeclIdentAndRegisterCurrentIdentAsWatcherInBlock(declIdentWatchers, identInfo, block, iStart)
4481 local statements = block.statements
4482 local currentIdentOrVararg = identInfo.ident
4483 local currentDeclIdent = currentIdentOrVararg.declaration
4484
4485 for i = iStart, 1, -1 do
4486 local statement = statements[i]
4487
4488 if statement.type == "declaration" then
4489 local decl = statement
4490
4491 for _, declIdent in ipairsr(decl.names) do
4492 -- Note: Declaration identifiers also watch themselves. (Is this good?) :DeclarationIdentifiersWatchThemselves
4493 declIdentWatchers[declIdent] = declIdentWatchers[declIdent] or {}
4494 tableInsert(declIdentWatchers[declIdent], currentIdentOrVararg)
4495
4496 if declIdent == currentDeclIdent then return true end
4497 end
4498 end
4499 end
4500
4501 return false
4502end
4503
4504local function getInformationAboutIdentifiersAndUpdateReferences(node)
4505 local identInfos = {--[[ [ident1]=identInfo1, identInfo1, ... ]]} -- identInfo = {ident=identOrVararg, type=lrvalueType}
4506 local declIdentWatchers = {--[[ [declIdent1]={identOrVararg1,...}, ... ]]}
4507
4508 traverseTree(node, function(node, parent, container, key)
4509 node.parent = parent
4510 node.container = container
4511 node.key = key
4512
4513 if node.type == "identifier" or node.type == "vararg" then
4514 local currentIdentOrVararg = node
4515 local findDecl = (currentIdentOrVararg.type == "identifier") and findIdentifierDeclaration or findVarargDeclaration
4516 local currentDeclIdent = findDecl(currentIdentOrVararg) -- We can call this because all parents and previous nodes already have their references updated at this point.
4517 currentIdentOrVararg.declaration = currentDeclIdent
4518
4519 local identType = (
4520 (parent and (
4521 (parent.type == "declaration" and (container == parent.names )) or
4522 (parent.type == "assignment" and (container == parent.targets )) or
4523 (parent.type == "function" and (container == parent.parameters)) or
4524 (parent.type == "for" and (container == parent.names ))
4525 ))
4526 and "lvalue"
4527 or "rvalue"
4528 )
4529
4530 -- if currentDeclIdent then print(F("%s:%d: %s '%s'", currentIdentOrVararg.sourcePath, currentIdentOrVararg.line, identType, (currentIdentOrVararg.name or "..."))) end -- DEBUG
4531
4532 local identInfo = {ident=currentIdentOrVararg, type=identType}
4533 tableInsert(identInfos, identInfo)
4534 identInfos[currentIdentOrVararg] = identInfo
4535
4536 -- Determine visible declarations for the identifier.
4537 local block = currentIdentOrVararg -- Start node for while loop.
4538
4539 while true do
4540 local statementOrInterest
4541 statementOrInterest, block = findParentStatementAndBlockOrNodeOfInterest(block, currentDeclIdent)
4542
4543 if not statementOrInterest then
4544 -- We should only get here for globals (i.e. there should be no declaration).
4545 assert(not currentDeclIdent)
4546 return
4547 end
4548
4549 if block then
4550 local statement = statementOrInterest
4551
4552 assert(type(statement.key) == "number")
4553 assert(statement.container == block.statements)
4554
4555 if lookForCurrentDeclIdentAndRegisterCurrentIdentAsWatcherInBlock(declIdentWatchers, identInfo, block, statement.key-1) then
4556 return
4557 end
4558
4559 -- We found the current declIdent - don't look for more other declIdents to watch!
4560 elseif statementOrInterest == currentDeclIdent then
4561 local declIdent = statementOrInterest
4562
4563 -- :DeclarationIdentifiersWatchThemselves
4564 declIdentWatchers[declIdent] = declIdentWatchers[declIdent] or {}
4565 tableInsert(declIdentWatchers[declIdent], currentIdentOrVararg)
4566
4567 return
4568
4569 -- We can jump from repeat loop conditions into the loop's body.
4570 elseif statementOrInterest.type == "repeat" then
4571 local repeatLoop = statementOrInterest
4572 block = repeatLoop.body -- Start node for while loop.
4573
4574 if lookForCurrentDeclIdentAndRegisterCurrentIdentAsWatcherInBlock(declIdentWatchers, identInfo, block, #block.statements) then
4575 return
4576 end
4577
4578 -- Declaration-likes have names we want to watch.
4579 -- Note that findParentStatementAndBlockOrNodeOfInterest() is smart and skipped declaration-likes that we should completely ignore.
4580 else
4581 local declLike = statementOrInterest
4582 block = declLike -- Start node for while loop.
4583
4584 for _, declIdent in ipairsr(getNameArrayOfDeclLike(declLike)) do
4585 -- :DeclarationIdentifiersWatchThemselves
4586 declIdentWatchers[declIdent] = declIdentWatchers[declIdent] or {}
4587 tableInsert(declIdentWatchers[declIdent], currentIdentOrVararg)
4588
4589 if declIdent == currentDeclIdent then return end
4590 end
4591 end
4592 end
4593
4594 elseif node.type == "goto" then
4595 local gotoNode = node
4596 gotoNode.label = findLabel(gotoNode) -- We can call this because all relevant 'parent' references have been updated at this point.
4597 end
4598 end)
4599
4600 return identInfos, declIdentWatchers
4601end
4602
4603
4604
4605local function unregisterWatchersBeforeNodeRemoval(identInfos, declIdentWatchers, theNode, declIdentReadCount, declIdentAssignmentCount, funcInfos, currentFuncInfo, stats, registerRemoval)
4606 -- ioWrite("unregister ") ; printNode(theNode) -- DEBUG
4607
4608 if registerRemoval then
4609 tableInsert(stats.nodeRemovals, Location(theNode)) -- There's no need to register the location every child node.
4610 stats.nodeRemoveCount = stats.nodeRemoveCount - 1 -- To counteract the +1 below.
4611 end
4612
4613 traverseTree(theNode, true, function(node) -- @Speed
4614 --[[ DEBUG
4615 unregistered = unregistered or {}
4616 if unregistered[node] then
4617 printNode(node)
4618 print(debug.traceback("NEW", 2))
4619 print(unregistered[node])
4620 end
4621 unregistered[node] = debug.traceback("OLD", 2)
4622 --]]
4623
4624 stats.nodeRemoveCount = stats.nodeRemoveCount + 1
4625
4626 if node.type == "identifier" or node.type == "vararg" then
4627 local currentIdentOrVararg = node
4628 local currentIdentInfo = identInfos[currentIdentOrVararg]
4629 assert(currentIdentInfo)
4630
4631 -- Remove identifier info.
4632 for i, identInfo in ipairs(identInfos) do
4633 if identInfo == currentIdentInfo then
4634 removeUnordered(identInfos, i)
4635 break
4636 end
4637 end
4638 identInfos[currentIdentOrVararg] = nil
4639
4640 -- Remove as watcher.
4641 for _, watcherIdents in pairs(declIdentWatchers) do -- @Speed
4642 for i, watcherIdent in ipairs(watcherIdents) do
4643 if watcherIdent == currentIdentOrVararg then
4644 removeUnordered(watcherIdents, i)
4645 break
4646 end
4647 end
4648 end
4649
4650 -- Remove watcher list.
4651 if declIdentWatchers[currentIdentOrVararg] then
4652 declIdentWatchers[currentIdentOrVararg] = nil
4653 removeItemUnordered(currentFuncInfo.declIdents, currentIdentOrVararg)
4654 end
4655
4656 -- Update access count.
4657 if not currentIdentOrVararg.declaration then
4658 -- void
4659 elseif currentIdentInfo.type == "rvalue" then
4660 local declIdent = currentIdentOrVararg.declaration
4661 declIdentReadCount[declIdent] = declIdentReadCount[declIdent] - 1 -- :AccessCount
4662 assert(declIdentReadCount[declIdent] >= 0)
4663 elseif --[[currentIdentInfo.type == "lvalue" and]] currentIdentInfo.ident.parent.type == "assignment" then
4664 local declIdent = currentIdentOrVararg.declaration
4665 declIdentAssignmentCount[declIdent] = declIdentAssignmentCount[declIdent] - 1 -- :AccessCount
4666 assert(declIdentAssignmentCount[declIdent] >= 0)
4667 end
4668
4669 -- removeItemUnordered(currentFuncInfo.locals, currentIdentOrVararg)
4670 -- removeItemUnordered(currentFuncInfo.upvalues, currentIdentOrVararg)
4671 -- removeItemUnordered(currentFuncInfo.globals, currentIdentOrVararg)
4672
4673 elseif node.type == "assignment" then
4674 removeItemUnordered(currentFuncInfo.assignments, node)
4675
4676 elseif isNodeDeclLike(node) then
4677 removeItemUnordered(currentFuncInfo.declLikes, node)
4678 end
4679
4680 if funcInfos[node] then
4681 for i, funcInfo in ipairs(funcInfos) do
4682 if funcInfo.node == node then
4683 removeUnordered(funcInfos, i)
4684 break
4685 end
4686 end
4687 funcInfos[node] = nil
4688 end
4689 end)
4690end
4691
4692local function isNodeValueList(node)
4693 return (node.type == "call" or node.type == "vararg") and not node.adjustToOne
4694end
4695
4696local function areAllLvaluesUnwantedAndAllValuesCalls(lvalues, values, wantToRemoveLvalue)
4697 if not values[1] then return false end -- Need at least one call.
4698
4699 for slot = 1, #lvalues do
4700 if not wantToRemoveLvalue[slot] then return false end
4701 end
4702
4703 for _, valueExpr in ipairs(values) do
4704 if valueExpr.type ~= "call" then return false end
4705 end
4706
4707 return true
4708end
4709
4710local function getInformationAboutFunctions(theNode)
4711 -- Gather functions.
4712 local funcInfos = {}
4713
4714 do
4715 -- We assume theNode is a block, but it's fine if it isn't.
4716 local funcInfo = {node=theNode, declLikes={}, declIdents={}, assignments={}--[[, locals={}, upvalues={}, globals={}]]}
4717 tableInsert(funcInfos, funcInfo)
4718 funcInfos[theNode] = funcInfo
4719 end
4720
4721 traverseTree(theNode, function(node)
4722 if node == theNode then return end
4723
4724 if node.type == "function" then
4725 local funcInfo = {node=node, declLikes={node}, declIdents={}, assignments={}--[[, locals={}, upvalues={}, globals={}]]}
4726 tableInsert(funcInfos, funcInfo)
4727 funcInfos[node] = funcInfo
4728 end
4729 end)
4730
4731 -- Gather relevant nodes.
4732 for _, funcInfo in ipairs(funcInfos) do
4733 traverseTree(funcInfo.node, function(node)
4734 if node == funcInfo.node then return end
4735 if node.type == "function" then return "ignorechildren" end
4736
4737 if node.type == "identifier" or node.type == "vararg" then
4738 local identOrVararg = node
4739 local declIdent = identOrVararg.declaration
4740
4741 if identOrVararg == declIdent then
4742 tableInsert(funcInfo.declIdents, identOrVararg)
4743 end
4744
4745 --[[
4746 if declIdent then
4747 local declLike = declIdent.parent
4748 local isInFunc = true
4749 local parent = identOrVararg.parent
4750
4751 while parent do
4752 if parent == declLike then -- declLike may be a function itself.
4753 break
4754 elseif parent.type == "function" then
4755 isInFunc = false
4756 break
4757 end
4758 parent = parent.parent
4759 end
4760
4761 tableInsert((isInFunc and funcInfo.locals or funcInfo.upvalues), identOrVararg)
4762
4763 else
4764 tableInsert(funcInfo.globals, identOrVararg)
4765 end
4766 ]]
4767
4768 elseif node.type == "declaration" or node.type == "for" then
4769 tableInsert(funcInfo.declLikes, node) -- Note: Identifiers will be added to funcInfo.declIdents when we get to them.
4770
4771 elseif node.type == "assignment" then
4772 tableInsert(funcInfo.assignments, node)
4773 end
4774 end)
4775
4776 --[[ DEBUG
4777 print("--------------")
4778 printNode(funcInfo.node)
4779 for i, ident in ipairs(funcInfo.locals) do
4780 ioWrite("local ", i, " ") ; printNode(ident)
4781 end
4782 for i, ident in ipairs(funcInfo.upvalues) do
4783 ioWrite("upvalue ", i, " ") ; printNode(ident)
4784 end
4785 for i, ident in ipairs(funcInfo.globals) do
4786 ioWrite("global ", i, " ") ; printNode(ident)
4787 end
4788 end
4789 print("--------------")
4790 --[==[]]
4791 end
4792 --]==]
4793
4794 return funcInfos
4795end
4796
4797local function getAccessesOfDeclaredNames(funcInfos, identInfos, declIdentWatchers) -- @Cleanup: Don't require the funcInfos argument (unless it maybe speeds up something somwhere).
4798 local declIdentReadCount = {--[[ [declIdent1]=count, ... ]]}
4799 local declIdentAssignmentCount = {--[[ [declIdent1]=count, ... ]]}
4800
4801 for _, funcInfo in ipairs(funcInfos) do
4802 for _, declIdent in ipairs(funcInfo.declIdents) do
4803 local readCount = 0
4804 local assignmentCount = 0
4805
4806 for _, watcherIdent in ipairs(declIdentWatchers[declIdent]) do
4807 if watcherIdent.declaration == declIdent then
4808 local identInfo = identInfos[watcherIdent]
4809
4810 if identInfo.type == "rvalue" then
4811 readCount = readCount + 1 -- :AccessCount
4812 elseif --[[identInfo.type == "lvalue" and]] identInfo.ident.parent.type == "assignment" then
4813 assignmentCount = assignmentCount + 1 -- :AccessCount
4814 end
4815 end
4816 end
4817
4818 declIdentReadCount [declIdent] = readCount
4819 declIdentAssignmentCount[declIdent] = assignmentCount
4820 end
4821 end
4822
4823 return declIdentReadCount, declIdentAssignmentCount
4824end
4825
4826-- Note: References need to be updated after calling this!
4827local function _optimize(theNode, stats)
4828 _simplify(theNode, stats)
4829
4830 local identInfos, declIdentWatchers = getInformationAboutIdentifiersAndUpdateReferences(theNode)
4831 local funcInfos = getInformationAboutFunctions(theNode)
4832 local declIdentReadCount, declIdentAssignmentCount = getAccessesOfDeclaredNames(funcInfos, identInfos, declIdentWatchers)
4833
4834 --
4835 -- Replace variables that are effectively constants with literals.
4836 --
4837 local replacedConstants = false
4838
4839 for _, funcInfo in ipairs(funcInfos) do
4840 for _, declLike in ipairs(funcInfo.declLikes) do
4841 if declLike.type == "declaration" then
4842 local decl = declLike
4843
4844 for slot = 1, #decl.names do
4845 local declIdent = decl.names[slot]
4846 local valueExpr = decl.values[slot]
4847
4848 if
4849 declIdentAssignmentCount[declIdent] == 0
4850 and declIdentReadCount[declIdent] > 0
4851 and (
4852 (not valueExpr and not (decl.values[1] and isNodeValueList(decl.values[#decl.values])))
4853 or (
4854 valueExpr
4855 and valueExpr.type == "literal"
4856 and not (
4857 (type(valueExpr.value) == "string" and #valueExpr.value > parser.constantNameReplacementStringMaxLength)
4858 -- or (valueExpr.value == 0 and not NORMALIZE_MINUS_ZERO and tostring(valueExpr.value) == "-0") -- No, bad rule!
4859 )
4860 )
4861 )
4862 then
4863 -- where(declIdent, "Constant declaration.") -- DEBUG
4864
4865 local valueLiteral = valueExpr
4866 local valueIsZero = (valueLiteral ~= nil and valueLiteral.value == 0)
4867
4868 for _, watcherIdent in ipairsr(declIdentWatchers[declIdent]) do
4869 if watcherIdent.declaration == declIdent then -- Note: declIdent is never a vararg here (because we only process declarations).
4870 local identInfo = identInfos[watcherIdent]
4871
4872 if
4873 identInfo.type == "rvalue"
4874 -- Avoid creating '-0' (or '- -0') here as that may mess up Lua in weird/surprising ways.
4875 and not (valueIsZero and not NORMALIZE_MINUS_ZERO and watcherIdent.parent.type == "unary" and watcherIdent.parent.operator == "-")
4876 then
4877 -- where(watcherIdent, "Constant value replacement.") -- DEBUG
4878
4879 local v = valueLiteral and valueLiteral.value
4880 local replacementLiteral = AstLiteral(watcherIdent.token, v)
4881
4882 unregisterWatchersBeforeNodeRemoval(identInfos, declIdentWatchers, watcherIdent, declIdentReadCount, declIdentAssignmentCount, funcInfos, funcInfo, stats, false)
4883 replace(watcherIdent, replacementLiteral, watcherIdent.parent, watcherIdent.container, watcherIdent.key, stats)
4884
4885 replacedConstants = true
4886 end
4887 end
4888 end--for declIdentWatchers
4889 end
4890 end--for declIdents
4891 end
4892 end--for declLikes
4893 end--for funcInfos
4894
4895 if replacedConstants then
4896 return _optimize(theNode, stats) -- @Speed
4897 end
4898
4899 --
4900 -- Remove useless assignments and declaration-likes (in that order).
4901 --
4902 -- Note that we go in reverse order almost everywhere! We may remove later stuff when we reach earlier stuff.
4903 --
4904 local function optimizeAssignmentOrDeclLike(statement, funcInfo)
4905 local isDecl = (statement.type == "declaration")
4906 local isFunc = (statement.type == "function")
4907 local isForLoop = (statement.type == "for")
4908 local isAssignment = (statement.type == "assignment")
4909
4910 if isDecl or isAssignment then
4911 local lvalues = statement.targets or statement.names
4912 local values = statement.values
4913
4914 -- Save some adjustment information.
4915 local madeToAdjusted = {}
4916
4917 for slot = 1, #values-1 do -- Skip the last value.
4918 local valueExpr = values[slot]
4919
4920 if isNodeValueList(valueExpr) then
4921 valueExpr.adjustToOne = true
4922 madeToAdjusted[valueExpr] = true
4923 end
4924 end
4925
4926 -- Remove useless extra values.
4927 for slot = #values, #lvalues+1, -1 do
4928 local valueExpr = values[slot]
4929
4930 if not mayNodeBeInvolvedInJump(valueExpr) then
4931 unregisterWatchersBeforeNodeRemoval(identInfos, declIdentWatchers, valueExpr, declIdentReadCount, declIdentAssignmentCount, funcInfos, funcInfo, stats, true)
4932 tableRemove(values, slot)
4933 end
4934 end
4935
4936 for slot = #lvalues+1, #values do
4937 values[slot].key = slot
4938 end
4939
4940 -- Remove useless lvalues.
4941 local wantToRemoveLvalue = {}
4942 local wantToRemoveValueIfExists = {}
4943 local mayRemoveValueIfExists = {}
4944
4945 for slot, lvalue in ipairsr(lvalues) do
4946 local declIdent = (lvalue.type == "identifier") and lvalue.declaration or nil
4947
4948 if declIdent and declIdentReadCount[declIdent] == 0 then
4949 -- ioWrite("useless ") ; printNode(lvalue) -- DEBUG
4950
4951 local valueExpr = values[slot]
4952 local valueExprEffective = valueExpr
4953
4954 if not valueExprEffective then
4955 valueExprEffective = values[#values]
4956 if valueExprEffective and not isNodeValueList(valueExprEffective) then valueExprEffective = nil end
4957 end
4958
4959 wantToRemoveLvalue [slot] = isAssignment or declIdentAssignmentCount[declIdent] == 0
4960 wantToRemoveValueIfExists[slot] = not (valueExpr and mayNodeBeInvolvedInJump(valueExpr))
4961 mayRemoveValueIfExists [slot] = wantToRemoveValueIfExists[slot] and not (not valueExpr and lvalues[slot+1] and valueExprEffective)
4962
4963 -- @Incomplete: Update funcInfo.locals and whatever else (if we end up using them at some point).
4964 -- @Incomplete: Replace 'unused,useless=func()' with 'useless=func()'.
4965 local canRemoveSlot = (wantToRemoveLvalue[slot] and mayRemoveValueIfExists[slot])
4966
4967 -- Maybe remove lvalue.
4968 if canRemoveSlot and #lvalues > 1 then
4969 unregisterWatchersBeforeNodeRemoval(identInfos, declIdentWatchers, lvalue, declIdentReadCount, declIdentAssignmentCount, funcInfos, funcInfo, stats, true)
4970 tableRemove(lvalues, slot)
4971 for slot = slot, #lvalues do
4972 lvalues[slot].key = slot
4973 end
4974 wantToRemoveLvalue[slot] = wantToRemoveLvalue[slot+1] -- May become nil. We no longer care about the value of slot+1 and beyond after this point.
4975 end
4976
4977 -- Maybe remove value.
4978 if wantToRemoveValueIfExists[slot] and valueExpr then
4979 if (canRemoveSlot or not values[slot+1]) and not (isAssignment and not values[2]) then
4980 unregisterWatchersBeforeNodeRemoval(identInfos, declIdentWatchers, valueExpr, declIdentReadCount, declIdentAssignmentCount, funcInfos, funcInfo, stats, true)
4981 tableRemove(values, slot)
4982 for slot = slot, #values do
4983 values[slot].key = slot
4984 end
4985 wantToRemoveValueIfExists[slot] = wantToRemoveValueIfExists[slot+1] -- May become nil. We no longer care about the value of slot+1 and beyond after this point.
4986 mayRemoveValueIfExists [slot] = mayRemoveValueIfExists [slot+1] -- May become nil. We no longer care about the value of slot+1 and beyond after this point.
4987
4988 elseif not (valueExpr.type == "literal" and valueExpr.value == nil) then
4989 unregisterWatchersBeforeNodeRemoval(identInfos, declIdentWatchers, valueExpr, declIdentReadCount, declIdentAssignmentCount, funcInfos, funcInfo, stats, false)
4990 replace(valueExpr, AstLiteral(valueExpr.token, nil), valueExpr.parent, valueExpr.container, valueExpr.key, stats)
4991 end
4992 end
4993 end--if lvalue is relevant
4994 end--for lvalues
4995
4996 -- Maybe remove or replace the whole statement.
4997 local statementIsRemoved = false
4998
4999 do
5000 -- Remove the whole statement.
5001 if
5002 wantToRemoveLvalue[1]
5003 and (
5004 mayRemoveValueIfExists[1]
5005 or not (isAssignment or values[1]) -- Declaration-likes may have no value - assignments must have at least one.
5006 )
5007 and not (lvalues[2] or values[2])
5008 then
5009 unregisterWatchersBeforeNodeRemoval(identInfos, declIdentWatchers, statement, declIdentReadCount, declIdentAssignmentCount, funcInfos, funcInfo, stats, true)
5010
5011 local block = statement.parent
5012
5013 for i = statement.key, #block.statements do
5014 local statement = block.statements[i+1] -- This be nil for the last 'i'.
5015 block.statements[i] = statement
5016
5017 if statement then statement.key = i end
5018 end
5019
5020 statementIsRemoved = true
5021
5022 -- Replace 'unused=func()' with just 'func()'. This is a unique case as call expressions can also be statements.
5023 elseif areAllLvaluesUnwantedAndAllValuesCalls(lvalues, values, wantToRemoveLvalue) then
5024 for _, lvalue in ipairs(lvalues) do
5025 unregisterWatchersBeforeNodeRemoval(identInfos, declIdentWatchers, lvalue, declIdentReadCount, declIdentAssignmentCount, funcInfos, funcInfo, stats, true)
5026 end
5027
5028 tableRemove(statement.container, statement.key) -- The parent ought to be a block!
5029
5030 for slot, call in ipairs(values) do
5031 local i = statement.key + slot - 1
5032 tableInsert(statement.container, i, call)
5033
5034 call.parent = statement.parent
5035 call.container = statement.container
5036 call.key = i
5037 end
5038
5039 statementIsRemoved = true
5040 end
5041 end
5042
5043 -- Restore or remove adjusted flags.
5044 -- @Speed: Don't do anything if statementIsRemoved is set (and we don't need to for some other reason).
5045 for i = 1, #values do
5046 local valueExpr = values[i]
5047 if (valueExpr.type == "call" or valueExpr.type == "vararg") then
5048 if statementIsRemoved or values[i+1] or not lvalues[i+1] or not madeToAdjusted[valueExpr] then
5049 valueExpr.adjustToOne = false
5050 end
5051 end
5052 end
5053
5054 elseif isFunc or isForLoop then
5055 local declIdents = getNameArrayOfDeclLike(statement)
5056
5057 for slot = #declIdents, (isForLoop and 2 or 1), -1 do
5058 local declIdent = declIdents[slot]
5059 if declIdentReadCount[declIdent] == 0 and declIdentAssignmentCount[declIdent] == 0 then
5060 unregisterWatchersBeforeNodeRemoval(identInfos, declIdentWatchers, declIdent, declIdentReadCount, declIdentAssignmentCount, funcInfos, funcInfo, stats, true)
5061 tableRemove(declIdents)
5062 else
5063 break
5064 end
5065 end
5066
5067 for slot, declIdent in ipairs(declIdents) do
5068 declIdent.key = slot
5069 end
5070
5071 else
5072 error(statement.type)
5073 end
5074 end
5075
5076 for _, funcInfo in ipairsr(funcInfos) do
5077 for _, assignment in ipairsr(funcInfo.assignments) do
5078 optimizeAssignmentOrDeclLike(assignment, funcInfo)
5079 end
5080 end
5081 for _, funcInfo in ipairsr(funcInfos) do
5082 for _, declLike in ipairsr(funcInfo.declLikes) do
5083 optimizeAssignmentOrDeclLike(declLike, funcInfo)
5084 end
5085 end
5086
5087 -- @Incomplete: Remove useless return statements etc.
5088
5089 _simplify(theNode, stats) -- Not sure if needed. Or maybe we need to iterate?
5090end
5091
5092-- Note: References need to be updated after calling this!
5093local function optimize(theNode)
5094 local stats = Stats()
5095 _optimize(theNode, stats)
5096 return stats
5097end
5098
5099
5100
5101local generateName
5102do
5103 local BANK_LETTERS = "etaoinshrdlcumwfgypbvkxjqzETAOINSHRDLCUMWFGYPBVKXJQZ" -- http://en.wikipedia.org/wiki/Letter_frequencies
5104 local BANK_ALPHANUM = "etaoinshrdlcumwfgypbvkxjqzETAOINSHRDLCUMWFGYPBVKXJQZ0123456789"
5105
5106 local ILLEGAL_NAMES = {}
5107 for name in pairs(KEYWORDS) do ILLEGAL_NAMES[name] = true end
5108 if HANDLE_ENV then ILLEGAL_NAMES._ENV = true end
5109
5110 local cache = {}
5111
5112 --[[local]] function generateName(nameGeneration)
5113 if not cache[nameGeneration] then
5114 -- @Cleanup: Output the most significant byte first. (We need to know the length beforehand then, probably, so we use the correct bank.)
5115 local charBytes = {}
5116
5117 for i = 1, 1/0 do
5118 nameGeneration = nameGeneration - 1
5119 local charBank = (i == 1) and BANK_LETTERS or BANK_ALPHANUM
5120 local charIndex = nameGeneration % #charBank + 1
5121 charBytes[i] = stringByte(charBank, charIndex)
5122 nameGeneration = mathFloor(nameGeneration / #charBank)
5123
5124 if nameGeneration == 0 then break end
5125 end
5126
5127 local name = stringChar(tableUnpack(charBytes))
5128 if ILLEGAL_NAMES[name] then
5129 -- We will probably realistically never get here, partially because of the limited amount of locals and upvalues Lua allows.
5130 name = name.."_"
5131 end
5132 cache[nameGeneration] = name
5133 end
5134
5135 return cache[nameGeneration]
5136 end
5137
5138 -- for nameGeneration = 1, 3500 do print(generateName(nameGeneration)) end ; error("TEST")
5139 -- for pow = 0, 32 do print(generateName(2^pow)) end ; error("TEST")
5140end
5141
5142-- stats = minify( node [, optimize=false ] )
5143local function minify(node, doOptimize)
5144 local stats = Stats()
5145
5146 if doOptimize then
5147 _optimize(node, stats)
5148 end
5149
5150 -- @Cleanup: Use findShadows()?
5151 local identInfos, declIdentWatchers = getInformationAboutIdentifiersAndUpdateReferences(node)
5152 -- local funcInfos = getInformationAboutFunctions(node)
5153 -- local declIdentReadCount, declIdentAssignmentCount = getAccessesOfDeclaredNames(funcInfos, identInfos, declIdentWatchers)
5154
5155 local allDeclIdents = {}
5156
5157 for _, identInfo in ipairs(identInfos) do
5158 local identOrVararg = identInfo.ident
5159 local declIdent = (identOrVararg.type == "identifier") and identOrVararg.declaration or nil
5160
5161 if declIdent and not allDeclIdents[declIdent] then
5162 tableInsert(allDeclIdents, declIdent)
5163 allDeclIdents[declIdent] = true
5164 end
5165 end
5166
5167 --
5168 -- Make sure frequencies affect who gets shorter names first.
5169 -- (This doesn't seem that useful at the moment. 2021-06-16)
5170 --
5171 --[[ :SortBeforeRename
5172 tableSort(allDeclIdents, function(a, b)
5173 if a.type == "vararg" or b.type == "vararg" then
5174 return a.id < b.id -- We don't care about varargs.
5175 end
5176
5177 local aWatchers = declIdentWatchers[a.declaration]
5178 local bWatchers = declIdentWatchers[b.declaration]
5179
5180 if not (aWatchers and bWatchers) then
5181 return a.id < a.id -- We don't care about globals.
5182 end
5183
5184 if #aWatchers ~= #bWatchers then
5185 return #aWatchers < #bWatchers
5186 end
5187
5188 return a.id < b.id
5189 end)
5190 --]]
5191
5192 --
5193 -- Rename locals!
5194 --
5195 local renamed = {--[[ [declIdent1]=true, ... ]]}
5196 local maxNameGeneration = 0
5197
5198 --[[ :SortBeforeRename
5199 local remoteWatchers = {}
5200 --]]
5201
5202 -- Assign generated names to declarations.
5203 for _, declIdent in ipairs(allDeclIdents) do
5204 local newName
5205
5206 if declIdent.name == "_ENV" and HANDLE_ENV then
5207 -- There are probably some cases where we can safely rename _ENV,
5208 -- but it's likely not worth the effort to detect that.
5209 newName = "_ENV"
5210
5211 else
5212 for nameGeneration = 1, 1/0 do
5213 newName = generateName(nameGeneration)
5214 local collision = false
5215
5216 for _, watcherIdent in ipairs(declIdentWatchers[declIdent]) do
5217 local watcherDeclIdent = watcherIdent.declaration
5218
5219 -- Local watcher.
5220 if watcherDeclIdent then
5221 if renamed[watcherDeclIdent] and watcherDeclIdent.name == newName then
5222 collision = true
5223 break
5224 end
5225
5226 -- Global watcher.
5227 elseif watcherIdent.name == newName then
5228 collision = true
5229 break
5230 end
5231 end--for declIdentWatchers
5232
5233 --[[ :SortBeforeRename
5234 if not collision and remoteWatchers[declIdent] then
5235 for _, watcherDeclIdent in ipairs(remoteWatchers[declIdent]) do
5236 if watcherDeclIdent.name == newName then
5237 collision = true
5238 break
5239 end
5240 end
5241 end
5242 --]]
5243
5244 if not collision then
5245 maxNameGeneration = mathMax(maxNameGeneration, nameGeneration)
5246 break
5247 end
5248 end--for nameGeneration
5249 end
5250
5251 --[[ :SortBeforeRename
5252 for _, watcherIdent in ipairs(declIdentWatchers[declIdent]) do
5253 local watcherDeclIdent = watcherIdent.declaration
5254
5255 if watcherDeclIdent and watcherDeclIdent ~= declIdent then
5256 remoteWatchers[watcherDeclIdent] = remoteWatchers[watcherDeclIdent] or {}
5257
5258 if not remoteWatchers[watcherDeclIdent][declIdent] then
5259 tableInsert(remoteWatchers[watcherDeclIdent], declIdent)
5260 remoteWatchers[watcherDeclIdent][declIdent] = true
5261 end
5262 end
5263 end
5264 --]]
5265
5266 if declIdent.name ~= newName then
5267 declIdent.name = newName
5268 stats.renameCount = stats.renameCount + 1
5269 end
5270 renamed[declIdent] = true
5271 end--for allDeclIdents
5272
5273 stats.generatedNameCount = maxNameGeneration
5274 -- print("maxNameGeneration", maxNameGeneration) -- DEBUG
5275
5276 -- Rename all remaining identifiers.
5277 for _, identInfo in ipairs(identInfos) do
5278 local ident = identInfo.ident -- Could be a vararg.
5279 local declIdent = (ident.type == "identifier") and ident.declaration or nil
5280
5281 if declIdent and ident.name ~= declIdent.name then
5282 ident.name = declIdent.name
5283 stats.renameCount = stats.renameCount + 1
5284 end
5285 end
5286
5287 return stats
5288end
5289
5290
5291
5292local function printTokens(tokens)
5293 local printLocs = parser.printLocations
5294
5295 for i, token in ipairs(tokens) do
5296 local v = ensurePrintable(tostring(token.value))
5297 if #v > 200 then v = stringSub(v, 1, 200-3).."..." end
5298
5299 if printLocs then ioWrite(token.sourcePath, ":", token.lineStart, ": ") end
5300 ioWrite(i, ". ", F("%-11s", token.type), " '", v, "'\n")
5301 end
5302end
5303
5304
5305
5306local toLua
5307do
5308 local writeNode
5309 local writeStatements
5310
5311 -- lastOutput = "" | "alphanum" | "number" | "-" | "."
5312
5313 local function isNumberInRange(n, min, max)
5314 return n ~= nil and n >= min and n <= max
5315 end
5316
5317 local function canNodeBeName(node)
5318 return node.type == "literal" and type(node.value) == "string" and stringFind(node.value, "^[%a_][%w_]*$") and not KEYWORDS[node.value]
5319 end
5320
5321 -- ensureSpaceIfNotPretty( buffer, pretty, lastOutput, value [, value2 ] )
5322 local function ensureSpaceIfNotPretty(buffer, pretty, lastOutput, value, value2)
5323 if not pretty and (lastOutput == value or lastOutput == value2) then
5324 tableInsert(buffer, " ")
5325 end
5326 end
5327
5328 local function choosePretty(node, prettyFallback)
5329 if node.pretty ~= nil then return node.pretty end
5330 return prettyFallback
5331 end
5332
5333 local function writeLua(buffer, lua, lastOutput)
5334 tableInsert(buffer, lua)
5335 return lastOutput
5336 end
5337
5338 local function writeAlphanum(buffer, pretty, s, lastOutput)
5339 ensureSpaceIfNotPretty(buffer, pretty, lastOutput, "alphanum","number")
5340 lastOutput = writeLua(buffer, s, "alphanum")
5341 return "alphanum"
5342 end
5343 local function writeNumber(buffer, pretty, n, lastOutput)
5344 local nStr = formatNumber(n)
5345 if (lastOutput == "-" and stringByte(nStr, 1) == 45--[[ "-" ]]) or lastOutput == "." then
5346 lastOutput = writeLua(buffer, " ", "")
5347 else
5348 ensureSpaceIfNotPretty(buffer, pretty, lastOutput, "alphanum","number")
5349 end
5350 lastOutput = writeLua(buffer, nStr, "number")
5351 return "number"
5352 end
5353
5354 -- Returns nil and a message or error.
5355 local function writeCommaSeparatedList(buffer, pretty, indent, lastOutput, expressions, writeAttributes, nodeCb)
5356 for i, expr in ipairs(expressions) do
5357 if i > 1 then
5358 lastOutput = writeLua(buffer, ",", "")
5359 if pretty then lastOutput = writeLua(buffer, " ", "") end
5360 end
5361
5362 local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, expr, true, nodeCb)
5363 if not ok then return nil, lastOutput end
5364
5365 if writeAttributes and expr.type == "identifier" and expr.attribute ~= "" then
5366 lastOutput = writeLua(buffer, "<", "")
5367 lastOutput = writeAlphanum(buffer, pretty, expr.attribute, lastOutput)
5368 lastOutput = writeLua(buffer, ">", "")
5369 end
5370 end
5371
5372 return true, lastOutput
5373 end
5374
5375 local function isStatementFunctionDeclaration(statement, statementNext)
5376 return true
5377 and statement.type == "declaration" and statementNext.type == "assignment"
5378 and #statement.names == 1 and #statement.values == 0
5379 and #statementNext.targets == 1 and #statementNext.values == 1
5380 and statementNext.targets[1].type == "identifier" and statement.names[1].name == statementNext.targets[1].name
5381 and statementNext.values [1].type == "function"
5382 end
5383
5384 local function writeIndentationIfPretty(buffer, pretty, indent, lastOutput)
5385 if pretty and indent > 0 then
5386 lastOutput = writeLua(buffer, stringRep("\t", indent), "")
5387 end
5388 return lastOutput
5389 end
5390
5391 -- Returns nil and a message or error.
5392 local function writeFunctionParametersAndBody(buffer, pretty, indent, lastOutput, func, explicitParams, selfParam, nodeCb)
5393 lastOutput = writeLua(buffer, "(", "")
5394
5395 if selfParam then
5396 if nodeCb then nodeCb(selfParam, buffer) end
5397 tableInsert(buffer, selfParam.prefix)
5398 tableInsert(buffer, selfParam.suffix)
5399 end
5400
5401 local ok;ok, lastOutput = writeCommaSeparatedList(buffer, pretty, indent, lastOutput, explicitParams, false, nodeCb)
5402 if not ok then return nil, lastOutput end
5403
5404 lastOutput = writeLua(buffer, ")", "")
5405 if nodeCb then nodeCb(func.body, buffer) end
5406 pretty = choosePretty(func.body, pretty)
5407 tableInsert(buffer, func.body.prefix)
5408 if pretty then lastOutput = writeLua(buffer, "\n", "") end
5409
5410 local ok;ok, lastOutput = writeStatements(buffer, pretty, indent+1, lastOutput, func.body.statements, nodeCb)
5411 if not ok then return nil, lastOutput end
5412
5413 lastOutput = writeIndentationIfPretty(buffer, pretty, indent, lastOutput)
5414 tableInsert(buffer, func.body.suffix)
5415 lastOutput = writeAlphanum(buffer, pretty, "end", lastOutput)
5416
5417 return true, lastOutput
5418 end
5419
5420 -- Returns nil and a message or error.
5421 --[[local]] function writeStatements(buffer, pretty, indent, lastOutput, statements, nodeCb)
5422 local skipNext = false
5423
5424 for i, statement in ipairs(statements) do
5425 if skipNext then
5426 skipNext = false
5427
5428 else
5429 local statementNext = statements[i+1]
5430 lastOutput = writeIndentationIfPretty(buffer, pretty, indent, lastOutput)
5431
5432 if statementNext and isStatementFunctionDeclaration(statement, statementNext) then
5433 local decl = statement
5434 local assignment = statementNext
5435 local func = assignment.values[1]
5436 local pretty = choosePretty(assignment, pretty)
5437
5438 if nodeCb then
5439 nodeCb(decl, buffer)
5440 nodeCb(assignment, buffer)
5441 nodeCb(func, buffer)
5442 end
5443
5444 tableInsert(buffer, decl.prefix)
5445 tableInsert(buffer, assignment.prefix)
5446 tableInsert(buffer, func.prefix)
5447
5448 lastOutput = writeAlphanum(buffer, pretty, "local function", lastOutput)
5449 lastOutput = writeLua(buffer, " ", "")
5450
5451 if nodeCb then nodeCb(decl.names[1], buffer) end
5452
5453 tableInsert(buffer, decl.names[1].prefix)
5454 local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, assignment.targets[1], true, nodeCb)
5455 if not ok then return nil, lastOutput end
5456 tableInsert(buffer, decl.names[1].suffix)
5457
5458 local ok;ok, lastOutput = writeFunctionParametersAndBody(buffer, choosePretty(func, pretty), indent, lastOutput, func, func.parameters, nil, nodeCb)
5459 if not ok then return nil, lastOutput end
5460
5461 tableInsert(buffer, func.suffix)
5462 tableInsert(buffer, assignment.suffix)
5463 tableInsert(buffer, decl.suffix)
5464
5465 skipNext = true
5466
5467 else
5468 local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, statement, true, nodeCb)
5469 if not ok then return nil, lastOutput end
5470
5471 if statement.type == "call" then
5472 lastOutput = writeLua(buffer, ";", "") -- @Ugly way of handling call statements. (But what way would be better?)
5473 end
5474 end
5475
5476 if pretty then lastOutput = writeLua(buffer, "\n", "") end
5477 end
5478 end
5479
5480 return true, lastOutput
5481 end
5482
5483 local function doesExpressionNeedParenthesisIfOnTheLeftSide(expr)
5484 local nodeType = expr.type
5485 -- Some things, like "binary" or "vararg", are not here because those expressions add their own parentheses.
5486 return nodeType == "literal" or nodeType == "table" or nodeType == "function"
5487 end
5488
5489 -- Returns nil and a message or error.
5490 local function writeLookup(buffer, pretty, indent, lastOutput, lookup, forMethodCall, nodeCb)
5491 local objNeedParens = doesExpressionNeedParenthesisIfOnTheLeftSide(lookup.object)
5492 if objNeedParens then lastOutput = writeLua(buffer, "(", "") end
5493
5494 local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, lookup.object, false, nodeCb)
5495 if not ok then return nil, lastOutput end
5496
5497 if objNeedParens then lastOutput = writeLua(buffer, ")", "") end
5498
5499 if canNodeBeName(lookup.member) then
5500 lastOutput = writeLua(buffer, (forMethodCall and ":" or "."), "")
5501 if nodeCb then nodeCb(lookup.member, buffer) end
5502 tableInsert(buffer, lookup.member.prefix)
5503 lastOutput = writeAlphanum(buffer, pretty, lookup.member.value, lastOutput)
5504 tableInsert(buffer, lookup.member.suffix)
5505
5506 elseif forMethodCall then
5507 return nil, "Error: AST: Callee for method call is not a lookup."
5508
5509 else
5510 lastOutput = writeLua(buffer, "[", "")
5511
5512 local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, lookup.member, true, nodeCb)
5513 if not ok then return nil, lastOutput end
5514
5515 lastOutput = writeLua(buffer, "]", "")
5516 end
5517
5518 return true, lastOutput
5519 end
5520
5521 local function isAssignmentFunctionAssignment(assignment)
5522 if #assignment.targets ~= 1 then return false end
5523 if #assignment.values ~= 1 then return false end
5524 if assignment.values[1].type ~= "function" then return false end
5525
5526 local targetExpr = assignment.targets[1]
5527 while true do
5528 if targetExpr.type == "identifier" then
5529 return true
5530 elseif not (targetExpr.type == "lookup" and canNodeBeName(targetExpr.member)) then
5531 return false
5532 end
5533 targetExpr = targetExpr.object
5534 end
5535 end
5536
5537 -- Returns nil and a message or error.
5538 local function writeBinaryOperatorChain(buffer, pretty, indent, lastOutput, binary, nodeCb)
5539 local l = binary.left
5540 local r = binary.right
5541
5542 if l.type == "binary" and l.operator == binary.operator then
5543 if nodeCb then nodeCb(l, buffer) end
5544 tableInsert(buffer, l.prefix)
5545 local ok;ok, lastOutput = writeBinaryOperatorChain(buffer, choosePretty(l, pretty), indent, lastOutput, l, nodeCb)
5546 if not ok then return nil, lastOutput end
5547 tableInsert(buffer, l.suffix)
5548 else
5549 local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, l, false, nodeCb)
5550 if not ok then return nil, lastOutput end
5551 end
5552
5553 if pretty then lastOutput = writeLua(buffer, " ", "") end
5554
5555 if binary.operator == ".." then ensureSpaceIfNotPretty(buffer, pretty, lastOutput, "number") end
5556
5557 local nextOutput = (
5558 (binary.operator == "-" and "-" ) or
5559 (binary.operator == ".." and "." ) or
5560 (stringFind(binary.operator, "%w") and "alphanum") or
5561 ""
5562 )
5563 if nextOutput ~= "" then ensureSpaceIfNotPretty(buffer, pretty, lastOutput, nextOutput) end
5564 lastOutput = writeLua(buffer, binary.operator, nextOutput)
5565
5566 if pretty then lastOutput = writeLua(buffer, " ", "") end
5567
5568 if r.type == "binary" and r.operator == binary.operator then
5569 if nodeCb then nodeCb(r, buffer) end
5570 tableInsert(buffer, r.prefix)
5571 local ok;ok, lastOutput = writeBinaryOperatorChain(buffer, choosePretty(r, pretty), indent, lastOutput, r, nodeCb)
5572 if not ok then return nil, lastOutput end
5573 tableInsert(buffer, r.suffix)
5574 else
5575 local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, r, false, nodeCb)
5576 if not ok then return nil, lastOutput end
5577 end
5578
5579 return true, lastOutput
5580 end
5581
5582 -- success, lastOutput = writeNode( buffer, pretty, indent, lastOutput, node, maySafelyOmitParens, nodeCallback )
5583 -- Returns nil and a message or error.
5584 --[[local]] function writeNode(buffer, pretty, indent, lastOutput, node, maySafelyOmitParens, nodeCb)
5585 if nodeCb then nodeCb(node, buffer) end
5586 pretty = choosePretty(node, pretty)
5587
5588 tableInsert(buffer, node.prefix)
5589
5590 local nodeType = node.type
5591
5592 -- Expressions:
5593
5594 if nodeType == "identifier" then
5595 local ident = node
5596 lastOutput = writeAlphanum(buffer, pretty, ident.name, lastOutput)
5597
5598 elseif nodeType == "vararg" then
5599 local vararg = node
5600 if vararg.adjustToOne then lastOutput = writeLua(buffer, "(", "") end
5601 if lastOutput == "." then lastOutput = writeLua(buffer, " ", ".") end
5602 lastOutput = writeLua(buffer, "...", ".")
5603 if vararg.adjustToOne then lastOutput = writeLua(buffer, ")", "") end
5604
5605 elseif nodeType == "literal" then
5606 local literal = node
5607
5608 if node.value == 0 and NORMALIZE_MINUS_ZERO then
5609 lastOutput = writeNumber(buffer, pretty, 0, lastOutput) -- Avoid writing '-0' sometimes.
5610
5611 elseif literal.value == 1/0 then
5612 lastOutput = writeAlphanum(buffer, pretty, "(1/0)", lastOutput) -- Note: We parse this as one literal.
5613
5614 elseif literal.value == -1/0 then
5615 lastOutput = writeLua(buffer, "(-1/0)", "") -- Note: We parse this as one literal.
5616
5617 elseif literal.value ~= literal.value then
5618 lastOutput = writeLua(buffer, "(0/0)", "") -- Note: We parse this as one literal.
5619
5620 elseif literal.value == nil or type(literal.value) == "boolean" then
5621 lastOutput = writeAlphanum(buffer, pretty, tostring(literal.value), lastOutput)
5622
5623 elseif type(literal.value) == "number" or (jit and type(literal.value) == "cdata" and tonumber(literal.value)) then
5624 lastOutput = writeNumber(buffer, pretty, literal.value, lastOutput)
5625
5626 elseif type(literal.value) == "string" then
5627 -- @Speed: Cache!
5628 local R = isNumberInRange
5629 local s = literal.value
5630 local doubleCount = countString(s, '"', true)
5631 local singleCount = countString(s, "'", true)
5632 local quote = singleCount < doubleCount and "'" or '"'
5633 local quoteByte = stringByte(quote)
5634 local pos = 1
5635
5636 lastOutput = writeLua(buffer, quote, "")
5637
5638 while pos <= #s do
5639 local b1, b2, b3, b4 = stringByte(s, pos, pos+3)
5640
5641 -- Printable ASCII.
5642 if R(b1,32,126) then
5643 if b1 == quoteByte then tableInsert(buffer, "\\") ; tableInsert(buffer, quote) ; pos = pos + 1
5644 elseif b1 == 92 then tableInsert(buffer, [[\\]]) ; pos = pos + 1
5645 else tableInsert(buffer, stringSub(s, pos, pos)) ; pos = pos + 1
5646 end
5647
5648 -- Multi-byte UTF-8 sequence.
5649 elseif b2 and R(b1,194,223) and R(b2,128,191) then tableInsert(buffer, stringSub(s, pos, pos+1)) ; pos = pos + 2
5650 elseif b3 and b1== 224 and R(b2,160,191) and R(b3,128,191) then tableInsert(buffer, stringSub(s, pos, pos+2)) ; pos = pos + 3
5651 elseif b3 and R(b1,225,236) and R(b2,128,191) and R(b3,128,191) then tableInsert(buffer, stringSub(s, pos, pos+2)) ; pos = pos + 3
5652 elseif b3 and b1== 237 and R(b2,128,159) and R(b3,128,191) then tableInsert(buffer, stringSub(s, pos, pos+2)) ; pos = pos + 3
5653 elseif b3 and R(b1,238,239) and R(b2,128,191) and R(b3,128,191) then tableInsert(buffer, stringSub(s, pos, pos+2)) ; pos = pos + 3
5654 elseif b4 and b1== 240 and R(b2,144,191) and R(b3,128,191) and R(b4,128,191) then tableInsert(buffer, stringSub(s, pos, pos+3)) ; pos = pos + 4
5655 elseif b4 and R(b1,241,243) and R(b2,128,191) and R(b3,128,191) and R(b4,128,191) then tableInsert(buffer, stringSub(s, pos, pos+3)) ; pos = pos + 4
5656 elseif b4 and b1== 244 and R(b2,128,143) and R(b3,128,191) and R(b4,128,191) then tableInsert(buffer, stringSub(s, pos, pos+3)) ; pos = pos + 4
5657
5658 -- Escape sequence.
5659 elseif b1 == 7 then tableInsert(buffer, [[\a]]) ; pos = pos + 1
5660 elseif b1 == 8 then tableInsert(buffer, [[\b]]) ; pos = pos + 1
5661 elseif b1 == 9 then tableInsert(buffer, [[\t]]) ; pos = pos + 1
5662 elseif b1 == 10 then tableInsert(buffer, [[\n]]) ; pos = pos + 1
5663 elseif b1 == 11 then tableInsert(buffer, [[\v]]) ; pos = pos + 1
5664 elseif b1 == 12 then tableInsert(buffer, [[\f]]) ; pos = pos + 1
5665 elseif b1 == 13 then tableInsert(buffer, [[\r]]) ; pos = pos + 1
5666
5667 -- Other control character or anything else.
5668 elseif b2 and R(b2,48,57) then tableInsert(buffer, F([[\%03d]], b1)) ; pos = pos + 1
5669 else tableInsert(buffer, F([[\%d]], b1)) ; pos = pos + 1
5670 end
5671 end
5672
5673 lastOutput = writeLua(buffer, quote, "")
5674
5675 else
5676 return nil, F("Error: Failed outputting '%s' value '%s'.", type(literal.value), tostring(literal.value))
5677 end
5678
5679 elseif nodeType == "table" then
5680 local tableNode = node
5681 lastOutput = writeLua(buffer, "{", "")
5682
5683 for i, tableField in ipairs(tableNode.fields) do
5684 if i > 1 then
5685 lastOutput = writeLua(buffer, ",", "")
5686 if pretty then lastOutput = writeLua(buffer, " ", "") end
5687 end
5688
5689 if tableField.generatedKey then
5690 if tableField.key then
5691 if nodeCb then nodeCb(tableField.key, buffer) end
5692 tableInsert(buffer, tableField.key.prefix)
5693 tableInsert(buffer, tableField.key.suffix)
5694 end
5695
5696 else
5697 if canNodeBeName(tableField.key) then
5698 if nodeCb then nodeCb(tableField.key, buffer) end
5699 tableInsert(buffer, tableField.key.prefix)
5700 lastOutput = writeLua(buffer, tableField.key.value, "alphanum")
5701 tableInsert(buffer, tableField.key.suffix)
5702
5703 else
5704 lastOutput = writeLua(buffer, "[", "")
5705
5706 local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, tableField.key, true, nodeCb)
5707 if not ok then return nil, lastOutput end
5708
5709 lastOutput = writeLua(buffer, "]", "")
5710 end
5711
5712 lastOutput = writeLua(buffer, "=", "")
5713 end
5714
5715 local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, tableField.value, (not pretty), nodeCb)
5716 if not ok then return nil, lastOutput end
5717 end
5718
5719 lastOutput = writeLua(buffer, "}", "")
5720
5721 elseif nodeType == "lookup" then
5722 local lookup = node
5723 local ok;ok, lastOutput = writeLookup(buffer, pretty, indent, lastOutput, lookup, false, nodeCb)
5724 if not ok then return nil, lastOutput end
5725
5726 elseif nodeType == "unary" then
5727 local unary = node
5728 local operatorOutput = (unary.operator == "-" and "-") or (stringFind(unary.operator, "%w") and "alphanum") or ("")
5729 local prettyAndAlphanum = pretty and operatorOutput == "alphanum"
5730
5731 if prettyAndAlphanum and not maySafelyOmitParens then lastOutput = writeLua(buffer, "(", "") end -- @Polish: Only output parentheses around child unaries/binaries if associativity requires it.
5732
5733 if lastOutput == "-" and operatorOutput == "-" then lastOutput = writeLua(buffer, " ", "")
5734 elseif operatorOutput ~= "" then ensureSpaceIfNotPretty(buffer, pretty, lastOutput, operatorOutput)
5735 end
5736 lastOutput = writeLua(buffer, unary.operator, operatorOutput)
5737
5738 if prettyAndAlphanum then lastOutput = writeLua(buffer, " ", "") end
5739
5740 local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, unary.expression, false, nodeCb)
5741 if not ok then return nil, lastOutput end
5742
5743 if prettyAndAlphanum and not maySafelyOmitParens then lastOutput = writeLua(buffer, ")", "") end
5744
5745 elseif nodeType == "binary" then
5746 local binary = node
5747 local op = binary.operator
5748
5749 if not maySafelyOmitParens then lastOutput = writeLua(buffer, "(", "") end -- @Polish: Only output parentheses around child unaries/binaries if associativity requires it.
5750
5751 if op == ".." or op == "and" or op == "or" or op == "+" or op == "*" or op == "&" or op == "|" then
5752 local ok;ok, lastOutput = writeBinaryOperatorChain(buffer, pretty, indent, lastOutput, binary, nodeCb)
5753 if not ok then return nil, lastOutput end
5754
5755 else
5756 local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, binary.left, false, nodeCb)
5757 if not ok then return nil, lastOutput end
5758
5759 local operatorOutput = ((op == "-" and "-") or (stringFind(op, "%w") and "alphanum") or (""))
5760
5761 if pretty then lastOutput = writeLua(buffer, " ", "") end
5762
5763 if operatorOutput ~= "" then ensureSpaceIfNotPretty(buffer, pretty, lastOutput, operatorOutput) end
5764 lastOutput = writeLua(buffer, op, operatorOutput)
5765
5766 if pretty then lastOutput = writeLua(buffer, " ", "") end
5767
5768 local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, binary.right, false, nodeCb)
5769 if not ok then return nil, lastOutput end
5770 end
5771
5772 if not maySafelyOmitParens then lastOutput = writeLua(buffer, ")", "") end
5773
5774 elseif nodeType == "call" then -- Can be statement too.
5775 local call = node
5776
5777 if call.adjustToOne then lastOutput = writeLua(buffer, "(", "") end
5778
5779 if call.method then
5780 local lookup = call.callee
5781
5782 if lookup.type ~= "lookup" then
5783 return nil, "Error: AST: Callee for method call is not a lookup."
5784 end
5785
5786 if nodeCb then nodeCb(lookup, buffer) end
5787
5788 tableInsert(buffer, lookup.prefix)
5789 local ok;ok, lastOutput = writeLookup(buffer, choosePretty(lookup, pretty), indent, lastOutput, lookup, true, nodeCb)
5790 if not ok then return nil, lastOutput end
5791 tableInsert(buffer, lookup.suffix)
5792
5793 else
5794 local needParens = doesExpressionNeedParenthesisIfOnTheLeftSide(call.callee)
5795 if needParens then lastOutput = writeLua(buffer, "(", "") end
5796
5797 local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, call.callee, false, nodeCb)
5798 if not ok then return nil, lastOutput end
5799
5800 if needParens then lastOutput = writeLua(buffer, ")", "") end
5801 end
5802
5803 lastOutput = writeLua(buffer, "(", "")
5804
5805 local ok;ok, lastOutput = writeCommaSeparatedList(buffer, pretty, indent, lastOutput, call.arguments, false, nodeCb)
5806 if not ok then return nil, lastOutput end
5807
5808 lastOutput = writeLua(buffer, ")", "")
5809 if call.adjustToOne then lastOutput = writeLua(buffer, ")", "") end
5810
5811 elseif nodeType == "function" then
5812 local func = node
5813 lastOutput = writeAlphanum(buffer, pretty, "function", lastOutput)
5814
5815 local ok;ok, lastOutput = writeFunctionParametersAndBody(buffer, pretty, indent, lastOutput, func, func.parameters, nil, nodeCb)
5816 if not ok then return nil, lastOutput end
5817
5818 -- Statements:
5819
5820 elseif nodeType == "break" then
5821 lastOutput = writeAlphanum(buffer, pretty, "break", lastOutput)
5822 lastOutput = writeLua(buffer, ";", "")
5823
5824 elseif nodeType == "return" then
5825 local returnNode = node
5826 lastOutput = writeAlphanum(buffer, pretty, "return", lastOutput)
5827
5828 if returnNode.values[1] then
5829 if pretty then lastOutput = writeLua(buffer, " ", "") end
5830
5831 local ok;ok, lastOutput = writeCommaSeparatedList(buffer, pretty, indent, lastOutput, returnNode.values, false, nodeCb)
5832 if not ok then return nil, lastOutput end
5833 end
5834
5835 lastOutput = writeLua(buffer, ";", "")
5836
5837 elseif nodeType == "label" then
5838 local label = node
5839 local name = label.name
5840 if not (stringFind(name, "^[%a_][%w_]*$") and not KEYWORDS[name]) then
5841 return nil, F("Error: AST: Invalid label '%s'.", name)
5842 end
5843 lastOutput = writeLua(buffer, "::", "")
5844 lastOutput = writeAlphanum(buffer, pretty, name, lastOutput)
5845 lastOutput = writeLua(buffer, "::", "")
5846 lastOutput = writeLua(buffer, ";", "")
5847
5848 elseif nodeType == "goto" then
5849 local gotoNode = node
5850 local name = gotoNode.name
5851 if not (stringFind(name, "^[%a_][%w_]*$") and not KEYWORDS[name]) then
5852 return nil, F("Error: AST: Invalid label '%s'.", name)
5853 end
5854 lastOutput = writeAlphanum(buffer, pretty, "goto", lastOutput)
5855 lastOutput = writeLua(buffer, " ", "")
5856 lastOutput = writeAlphanum(buffer, pretty, name, lastOutput)
5857 lastOutput = writeLua(buffer, ";", "")
5858
5859 elseif nodeType == "block" then
5860 local block = node
5861 lastOutput = writeAlphanum(buffer, pretty, "do", lastOutput)
5862 if pretty then lastOutput = writeLua(buffer, "\n", "") end
5863
5864 local ok;ok, lastOutput = writeStatements(buffer, pretty, indent+1, lastOutput, block.statements, nodeCb)
5865 if not ok then return nil, lastOutput end
5866
5867 lastOutput = writeIndentationIfPretty(buffer, pretty, indent, lastOutput)
5868 lastOutput = writeAlphanum(buffer, pretty, "end", lastOutput)
5869
5870 elseif nodeType == "declaration" then
5871 local decl = node
5872 lastOutput = writeAlphanum(buffer, pretty, "local", lastOutput)
5873 lastOutput = writeLua(buffer, " ", "")
5874
5875 if not decl.names[1] then return nil, "Error: AST: Missing name(s) for declaration." end
5876
5877 local ok;ok, lastOutput = writeCommaSeparatedList(buffer, pretty, indent, lastOutput, decl.names, true, nodeCb)
5878 if not ok then return nil, lastOutput end
5879
5880 if decl.values[1] then
5881 if pretty then lastOutput = writeLua(buffer, " ", "") end
5882 lastOutput = writeLua(buffer, "=", "")
5883 if pretty then lastOutput = writeLua(buffer, " ", "") end
5884
5885 local ok;ok, lastOutput = writeCommaSeparatedList(buffer, pretty, indent, lastOutput, decl.values, false, nodeCb)
5886 if not ok then return nil, lastOutput end
5887 end
5888
5889 lastOutput = writeLua(buffer, ";", "")
5890
5891 elseif nodeType == "assignment" then
5892 local assignment = node
5893 if not assignment.targets[1] then return nil, "Error: AST: Missing target expression(s) for assignment." end
5894 if not assignment.values[1] then return nil, "Error: AST: Missing value(s) for assignment." end
5895
5896 if isAssignmentFunctionAssignment(assignment) then
5897 local func = assignment.values[1]
5898 if nodeCb then nodeCb(func, buffer) end
5899
5900 tableInsert(buffer, func.prefix)
5901
5902 lastOutput = writeAlphanum(buffer, pretty, "function", lastOutput)
5903 lastOutput = writeLua(buffer, " ", "")
5904
5905 local implicitSelfParam = (
5906 func.parameters[1] ~= nil
5907 and func.parameters[1].name == "self"
5908 and assignment.targets[1].type == "lookup"
5909 and canNodeBeName(assignment.targets[1].member)
5910 )
5911
5912 if implicitSelfParam then
5913 if nodeCb then nodeCb(assignment.targets[1], buffer) end
5914 tableInsert(buffer, assignment.targets[1].prefix)
5915 local ok;ok, lastOutput = writeLookup(buffer, pretty, indent, lastOutput, assignment.targets[1], true, nodeCb)
5916 if not ok then return nil, lastOutput end
5917 tableInsert(buffer, assignment.targets[1].suffix)
5918 else
5919 local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, assignment.targets[1], false, nodeCb)
5920 if not ok then return nil, lastOutput end
5921 end
5922
5923 local selfParam = nil
5924 local explicitParams = func.parameters
5925
5926 if implicitSelfParam then
5927 selfParam = explicitParams[1]
5928 explicitParams = {tableUnpack(explicitParams, 2)}
5929 end
5930
5931 local ok;ok, lastOutput = writeFunctionParametersAndBody(buffer, pretty, indent, lastOutput, func, explicitParams, selfParam, nodeCb)
5932 if not ok then return nil, lastOutput end
5933
5934 tableInsert(buffer, func.suffix)
5935
5936 else
5937 local ok;ok, lastOutput = writeCommaSeparatedList(buffer, pretty, indent, lastOutput, assignment.targets, false, nodeCb)
5938 if not ok then return nil, lastOutput end
5939
5940 if pretty then lastOutput = writeLua(buffer, " ", "") end
5941 lastOutput = writeLua(buffer, "=", "")
5942 if pretty then lastOutput = writeLua(buffer, " ", "") end
5943
5944 local ok;ok, lastOutput = writeCommaSeparatedList(buffer, pretty, indent, lastOutput, assignment.values, false, nodeCb)
5945 if not ok then return nil, lastOutput end
5946
5947 lastOutput = writeLua(buffer, ";", "")
5948 end
5949
5950 elseif nodeType == "if" then
5951 local ifNode = node
5952 lastOutput = writeAlphanum(buffer, pretty, "if", lastOutput)
5953 if pretty then lastOutput = writeLua(buffer, " ", "") end
5954
5955 local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, ifNode.condition, true, nodeCb)
5956 if not ok then return nil, lastOutput end
5957
5958 if pretty then lastOutput = writeLua(buffer, " ", "") end
5959 lastOutput = writeAlphanum(buffer, pretty, "then", lastOutput)
5960 if nodeCb then nodeCb(ifNode.bodyTrue, buffer) end
5961 local prettyBody = choosePretty(ifNode.bodyTrue, pretty)
5962 tableInsert(buffer, ifNode.bodyTrue.prefix)
5963 if prettyBody then lastOutput = writeLua(buffer, "\n", "") end
5964
5965 local ok;ok, lastOutput = writeStatements(buffer, prettyBody, indent+1, lastOutput, ifNode.bodyTrue.statements, nodeCb)
5966 if not ok then return nil, lastOutput end
5967
5968 local lastTrueBody = ifNode.bodyTrue
5969 local suffixesForTrailingBodies = {}
5970
5971 while ifNode.bodyFalse do
5972 lastOutput = writeIndentationIfPretty(buffer, prettyBody, indent, lastOutput)
5973
5974 -- Automatically detect what looks like 'elseif'.
5975 if #ifNode.bodyFalse.statements == 1 and ifNode.bodyFalse.statements[1].type == "if" then
5976 tableInsert(buffer, lastTrueBody.suffix)
5977 local body = ifNode.bodyFalse
5978
5979 if nodeCb then nodeCb(body, buffer) end
5980 tableInsert(suffixesForTrailingBodies, body.suffix)
5981 ifNode = body.statements[1]
5982 if nodeCb then nodeCb(ifNode, buffer) end
5983 pretty = choosePretty(ifNode, pretty)
5984
5985 tableInsert(buffer, body.prefix)
5986 tableInsert(buffer, ifNode.prefix)
5987 lastOutput = writeAlphanum(buffer, prettyBody, "elseif", lastOutput)
5988 if pretty then lastOutput = writeLua(buffer, " ", "") end
5989
5990 local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, ifNode.condition, true, nodeCb)
5991 if not ok then return nil, lastOutput end
5992
5993 if pretty then lastOutput = writeLua(buffer, " ", "") end
5994 lastOutput = writeAlphanum(buffer, pretty, "then", lastOutput)
5995 if nodeCb then nodeCb(ifNode.bodyTrue, buffer) end
5996 prettyBody = choosePretty(ifNode.bodyTrue, pretty)
5997 if prettyBody then lastOutput = writeLua(buffer, "\n", "") end
5998
5999 tableInsert(buffer, ifNode.bodyTrue.prefix)
6000 local ok;ok, lastOutput = writeStatements(buffer, prettyBody, indent+1, lastOutput, ifNode.bodyTrue.statements, nodeCb)
6001 if not ok then return nil, lastOutput end
6002 tableInsert(buffer, ifNode.bodyTrue.suffix)
6003
6004 lastTrueBody = ifNode
6005
6006 else
6007 lastOutput = writeAlphanum(buffer, prettyBody, "else", lastOutput)
6008 if nodeCb then nodeCb(ifNode.bodyFalse, buffer) end
6009 prettyBody = choosePretty(ifNode.bodyFalse, pretty)
6010 tableInsert(buffer, ifNode.bodyFalse.prefix)
6011 if prettyBody then lastOutput = writeLua(buffer, "\n", "") end
6012
6013 local ok;ok, lastOutput = writeStatements(buffer, prettyBody, indent+1, lastOutput, ifNode.bodyFalse.statements, nodeCb)
6014 if not ok then return nil, lastOutput end
6015
6016 tableInsert(suffixesForTrailingBodies, ifNode.bodyFalse.suffix)
6017 break
6018 end
6019 end
6020
6021 lastOutput = writeIndentationIfPretty(buffer, prettyBody, indent, lastOutput)
6022 tableInsert(buffer, lastTrueBody.suffix)
6023 for i = #suffixesForTrailingBodies, 1, -1 do
6024 tableInsert(buffer, suffixesForTrailingBodies[i])
6025 end
6026 lastOutput = writeAlphanum(buffer, prettyBody, "end", lastOutput)
6027
6028 elseif nodeType == "while" then
6029 local whileLoop = node
6030 lastOutput = writeAlphanum(buffer, pretty, "while", lastOutput)
6031 if pretty then lastOutput = writeLua(buffer, " ", "") end
6032
6033 local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, whileLoop.condition, true, nodeCb)
6034 if not ok then return nil, lastOutput end
6035
6036 if pretty then lastOutput = writeLua(buffer, " ", "") end
6037 lastOutput = writeAlphanum(buffer, pretty, "do", lastOutput)
6038
6039 if nodeCb then nodeCb(whileLoop.body, buffer) end
6040 local prettyBody = choosePretty(whileLoop.body, pretty)
6041 tableInsert(buffer, whileLoop.body.prefix)
6042 if prettyBody then lastOutput = writeLua(buffer, "\n", "") end
6043
6044 local ok;ok, lastOutput = writeStatements(buffer, prettyBody, indent+1, lastOutput, whileLoop.body.statements, nodeCb)
6045 if not ok then return nil, lastOutput end
6046
6047 lastOutput = writeIndentationIfPretty(buffer, prettyBody, indent, lastOutput)
6048 tableInsert(buffer, whileLoop.body.suffix)
6049 lastOutput = writeAlphanum(buffer, prettyBody, "end", lastOutput)
6050
6051 elseif nodeType == "repeat" then
6052 local repeatLoop = node
6053 lastOutput = writeAlphanum(buffer, pretty, "repeat", lastOutput)
6054 if nodeCb then nodeCb(repeatLoop.body, buffer) end
6055 local prettyBody = choosePretty(repeatLoop.body, pretty)
6056 tableInsert(buffer, repeatLoop.body.prefix)
6057 if prettyBody then lastOutput = writeLua(buffer, "\n", "") end
6058
6059 local ok;ok, lastOutput = writeStatements(buffer, prettyBody, indent+1, lastOutput, repeatLoop.body.statements, nodeCb)
6060 if not ok then return nil, lastOutput end
6061
6062 lastOutput = writeIndentationIfPretty(buffer, prettyBody, indent, lastOutput)
6063 tableInsert(buffer, repeatLoop.body.suffix)
6064 lastOutput = writeAlphanum(buffer, prettyBody, "until", lastOutput)
6065 if pretty then lastOutput = writeLua(buffer, " ", "") end
6066
6067 local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, repeatLoop.condition, true, nodeCb)
6068 if not ok then return nil, lastOutput end
6069
6070 elseif nodeType == "for" then
6071 local forLoop = node
6072 if not forLoop.names[1] then return nil, "Error: AST: Missing name(s) for 'for' loop." end
6073 if not forLoop.values[1] then return nil, "Error: AST: Missing value(s) for 'for' loop." end
6074
6075 lastOutput = writeAlphanum(buffer, pretty, "for", lastOutput)
6076 lastOutput = writeLua(buffer, " ", "")
6077
6078 local ok;ok, lastOutput = writeCommaSeparatedList(buffer, pretty, indent, lastOutput, forLoop.names, false, nodeCb)
6079 if not ok then return nil, lastOutput end
6080
6081 if pretty then lastOutput = writeLua(buffer, " ", "") end
6082
6083 if forLoop.kind == "numeric" then
6084 lastOutput = writeLua(buffer, "=", "")
6085 elseif forLoop.kind == "generic" then
6086 lastOutput = writeAlphanum(buffer, pretty, "in", lastOutput)
6087 else
6088 return nil, F("Error: Unknown 'for' loop kind '%s'.", forLoop.kind)
6089 end
6090
6091 if pretty then lastOutput = writeLua(buffer, " ", "") end
6092
6093 local ok;ok, lastOutput = writeCommaSeparatedList(buffer, pretty, indent, lastOutput, forLoop.values, false, nodeCb)
6094 if not ok then return nil, lastOutput end
6095
6096 if pretty then lastOutput = writeLua(buffer, " ", "") end
6097 lastOutput = writeAlphanum(buffer, pretty, "do", lastOutput)
6098 if nodeCb then nodeCb(forLoop.body, buffer) end
6099 local prettyBody = choosePretty(forLoop.body, pretty)
6100 tableInsert(buffer, forLoop.body.prefix)
6101 if prettyBody then lastOutput = writeLua(buffer, "\n", "") end
6102
6103 local ok;ok, lastOutput = writeStatements(buffer, prettyBody, indent+1, lastOutput, forLoop.body.statements, nodeCb)
6104 if not ok then return nil, lastOutput end
6105
6106 lastOutput = writeIndentationIfPretty(buffer, prettyBody, indent, lastOutput)
6107 tableInsert(buffer, forLoop.body.suffix)
6108 lastOutput = writeAlphanum(buffer, prettyBody, "end", lastOutput)
6109
6110 else
6111 return false, F("Error: Unknown node type '%s'.", tostring(nodeType))
6112 end
6113
6114 tableInsert(buffer, node.suffix)
6115
6116 return true, lastOutput
6117 end
6118
6119 -- luaString = toLua( astNode [, prettyOuput=false, nodeCallback ] )
6120 -- nodeCallback = function( node, outputBuffer )
6121 -- Returns nil and a message on error.
6122 --[[local]] function toLua(node, pretty, nodeCb)
6123 assertArg1("toLua", 1, node, "table")
6124
6125 local buffer = {}
6126
6127 local ok, err
6128 if node.type == "block" then -- @Robustness: This exception isn't great. Should there be a file scope node?
6129 if nodeCb then nodeCb(node, buffer) end
6130 tableInsert(buffer, node.prefix)
6131 ok, err = writeStatements(buffer, choosePretty(node, pretty), 0, "", node.statements, nodeCb)
6132 tableInsert(buffer, node.suffix)
6133 else
6134 ok, err = writeNode(buffer, pretty, 0, "", node, true, nodeCb)
6135 end
6136
6137 if ok then
6138 return tableConcat(buffer)
6139 else
6140 return nil, err
6141 end
6142 end
6143end
6144
6145
6146
6147-- node = getChild( node, fieldName )
6148-- node = getChild( node, fieldName, index ) -- If the node field is an array.
6149-- node = getChild( node, fieldName, index, tableFieldKey ) -- If the node field is a table field array.
6150local function getChild(node, fieldName, i, tableFieldKey)
6151 assertArg1("getChild", 1, node, "table")
6152 assertArg1("getChild", 2, fieldName, "string")
6153
6154 local nodeType = node.type
6155 local childFields = CHILD_FIELDS[nodeType] or errorf(2, "Unknown node type '%s'.", tostring(nodeType))
6156 local childFieldType = childFields[fieldName] or errorf(2, "Unknown node field '%s.%s'.", nodeType, tostring(fieldName))
6157
6158 if childFieldType == "node" then
6159 return node[fieldName]
6160
6161 elseif childFieldType == "nodearray" then
6162 assertArg1("getChild", 3, i, "number")
6163
6164 return node[fieldName][i]
6165
6166 elseif childFieldType == "tablefields" then
6167 assertArg1("getChild", 3, i, "number")
6168 assertArg1("getChild", 4, tableFieldKey, "string")
6169
6170 if not (tableFieldKey == "key" or tableFieldKey == "value") then
6171 errorf(2, "Bad argument #4 to 'getChild'. (Expected %q or %q, got %q)", "key", "value", tableFieldKey)
6172 end
6173
6174 local field = node[fieldName][i]
6175 return field and field[tableFieldKey]
6176
6177 else
6178 error(childFieldType)
6179 end
6180end
6181
6182-- setChild( node, fieldName, childNode )
6183-- setChild( node, fieldName, index, childNode ) -- If the node field is an array.
6184-- setChild( node, fieldName, index, tableFieldKey, childNode ) -- If the node field is a table field array.
6185local function setChild(node, fieldName, i, tableFieldKey, childNode)
6186 assertArg1("setChild", 1, node, "table")
6187 assertArg1("setChild", 2, fieldName, "string")
6188
6189 local nodeType = node.type
6190 local childFields = CHILD_FIELDS[nodeType] or errorf(2, "Unknown node type '%s'.", tostring(nodeType))
6191 local childFieldType = childFields[fieldName] or errorf(2, "Unknown node field '%s.%s'.", nodeType, tostring(fieldName))
6192
6193 if childFieldType == "node" then
6194 childNode = i
6195
6196 if childNode ~= nil then assertArg1("setChild", 3, childNode, "table") end
6197
6198 node[fieldName] = childNode
6199
6200 elseif childFieldType == "nodearray" then
6201 childNode = tableFieldKey
6202
6203 assertArg1("setChild", 3, i, "number")
6204 assertArg1("setChild", 4, childNode, "table")
6205
6206 node[fieldName][i] = childNode
6207
6208 elseif childFieldType == "tablefields" then
6209 assertArg1("setChild", 3, i, "number")
6210 assertArg1("setChild", 4, tableFieldKey, "string")
6211 assertArg1("setChild", 5, childNode, "table")
6212
6213 if not (tableFieldKey == "key" or tableFieldKey == "value") then
6214 errorf(2, "Bad argument #4 to 'setChild'. (Expected %q or %q, got %q)", "key", "value", tableFieldKey)
6215 end
6216
6217 local field = node[fieldName][i] or errorf(2, "No table field at index %d in %s.%s.", i, nodeType, fieldName)
6218 field[tableFieldKey] = childNode
6219
6220 else
6221 error(childFieldType)
6222 end
6223end
6224
6225-- addChild( node, fieldName, [ index=atEnd, ] childNode )
6226-- addChild( node, fieldName, [ index=atEnd, ] keyNode, valueNode ) -- If the node field is a table field array.
6227local function addChild(node, fieldName, i, childNode, extraChildNode)
6228 assertArg1("addChild", 1, node, "table")
6229 assertArg1("addChild", 2, fieldName, "string")
6230
6231 if type(i) ~= "number" then
6232 i, childNode, extraChildNode = nil, i, childNode
6233 end
6234 local postIndexArgOffset = i and 0 or -1
6235
6236 local nodeType = node.type
6237 local childFields = CHILD_FIELDS[nodeType] or errorf(2, "Unknown node type '%s'.", tostring(nodeType))
6238 local childFieldType = childFields[fieldName] or errorf(2, "Unknown node field '%s.%s'.", nodeType, tostring(fieldName))
6239
6240 if childFieldType == "nodearray" then
6241 if i ~= nil then assertArg1("addChild", 3, i, "number") end
6242 assertArg1("addChild", 4+postIndexArgOffset, childNode, "table")
6243
6244 i = i or #node[fieldName]+1
6245 tableInsert(node[fieldName], i, childNode)
6246
6247 elseif childFieldType == "tablefields" then
6248 if i ~= nil then assertArg1("addChild", 3, i, "number") end
6249 assertArg1("addChild", 4+postIndexArgOffset, childNode, "table")
6250 assertArg1("addChild", 5+postIndexArgOffset, extraChildNode, "table")
6251
6252 i = i or #node[fieldName]+1
6253 tableInsert(node[fieldName], i, {key=childNode, value=extraChildNode, generatedKey=false})
6254
6255 else
6256 errorf(2, "Node field '%s.%s' is not an array.", nodeType, tostring(fieldName))
6257 end
6258end
6259
6260-- removeChild( node, fieldName [, index=last ] )
6261local function removeChild(node, fieldName, i)
6262 assertArg1("removeChild", 1, node, "table")
6263 assertArg1("removeChild", 2, fieldName, "string")
6264 assertArg2("removeChild", 3, i, "number","nil")
6265
6266 local nodeType = node.type
6267 local childFields = CHILD_FIELDS[nodeType] or errorf(2, "Unknown node type '%s'.", tostring(nodeType))
6268 local childFieldType = childFields[fieldName] or errorf(2, "Unknown node field '%s.%s'.", nodeType, tostring(fieldName))
6269
6270 if childFieldType == "nodearray" or childFieldType == "tablefields" then
6271 tableRemove(node[fieldName], i) -- This also works if i is nil.
6272 else
6273 errorf(2, "Node field '%s.%s' is not an array.", nodeType, tostring(fieldName))
6274 end
6275end
6276
6277
6278
6279local validateTree
6280do
6281 local function addValidationError(path, errors, s, ...)
6282 tableInsert(errors, F("%s: "..s, tableConcat(path, " > "), ...))
6283 end
6284
6285 local function validateNode(node, path, errors, prefix)
6286 local nodeType = node.type
6287
6288 tableInsert(path,
6289 (prefix and prefix.."."..nodeType or nodeType)
6290 .. (parser.printIds and "#"..node.id or "")
6291 )
6292
6293 if nodeType == "identifier" then
6294 local ident = node
6295 if not stringFind(ident.name, "^[%a_][%w_]*$") then
6296 addValidationError(path, errors, "Invalid identifier name: Bad format: %s", ident.name)
6297 elseif KEYWORDS[ident.name] then
6298 addValidationError(path, errors, "Invalid identifier name: Name is a keyword: %s", ident.name)
6299 end
6300 if not (ident.attribute == "" or ident.attribute == "close" or ident.attribute == "const") then
6301 addValidationError(path, errors, "Invalid identifier attribute '%s'.", ident.attribute)
6302 end
6303
6304 elseif nodeType == "vararg" then
6305 -- void
6306
6307 elseif nodeType == "literal" then
6308 local literal = node
6309 local vType = type(literal.value)
6310 if not (vType == "number" or vType == "string" or vType == "boolean" or vType == "nil" or (jit and vType == "cdata" and tonumber(literal.value))) then
6311 addValidationError(path, errors, "Invalid literal value type '%s'.", vType)
6312 end
6313
6314 elseif nodeType == "break" then
6315 -- void
6316
6317 elseif nodeType == "label" then
6318 local label = node
6319 if not stringFind(label.name, "^[%a_][%w_]*$") then
6320 addValidationError(path, errors, "Invalid label name: Bad format: %s", label.name)
6321 elseif KEYWORDS[label.name] then
6322 addValidationError(path, errors, "Invalid label name: Name is a keyword: %s", label.name)
6323 end
6324
6325 elseif nodeType == "goto" then
6326 local gotoNode = node
6327 if not stringFind(gotoNode.name, "^[%a_][%w_]*$") then
6328 addValidationError(path, errors, "Invalid label name: Bad format: %s", gotoNode.name)
6329 elseif KEYWORDS[gotoNode.name] then
6330 addValidationError(path, errors, "Invalid label name: Name is a keyword: %s", gotoNode.name)
6331 end
6332
6333 elseif nodeType == "lookup" then
6334 -- @Incomplete: Should we detect nil literal objects? :DetectRuntimeErrors
6335 local lookup = node
6336 if not lookup.object then
6337 addValidationError(path, errors, "Missing 'object' field.")
6338 elseif not EXPRESSION_NODES[lookup.object.type] then
6339 addValidationError(path, errors, "The object is not an expression. (It is '%s'.)", lookup.object.type)
6340 else
6341 validateNode(lookup.object, path, errors, "object")
6342 end
6343 if not lookup.member then
6344 addValidationError(path, errors, "Missing 'member' field.")
6345 elseif not EXPRESSION_NODES[lookup.member.type] then
6346 addValidationError(path, errors, "The member is not an expression. (It is '%s'.)", lookup.member.type)
6347 else
6348 validateNode(lookup.member, path, errors, "member")
6349 end
6350
6351 elseif nodeType == "unary" then
6352 local unary = node
6353 if not OPERATORS_UNARY[unary.operator] then
6354 addValidationError(path, errors, "Invalid unary operator '%s'.", unary.operator)
6355 end
6356 if not unary.expression then
6357 addValidationError(path, errors, "Missing 'expression' field.")
6358 elseif not EXPRESSION_NODES[unary.expression.type] then
6359 addValidationError(path, errors, "The 'expression' field does not contain an expression. (It is '%s'.)", unary.expression.type)
6360 else
6361 validateNode(unary.expression, path, errors, nil)
6362 end
6363
6364 elseif nodeType == "binary" then
6365 local binary = node
6366 if not OPERATORS_BINARY[binary.operator] then
6367 addValidationError(path, errors, "Invalid binary operator '%s'.", binary.operator)
6368 end
6369 if not binary.left then
6370 addValidationError(path, errors, "Missing 'left' field.")
6371 elseif not EXPRESSION_NODES[binary.left.type] then
6372 addValidationError(path, errors, "The left side is not an expression. (It is '%s'.)", binary.left.type)
6373 else
6374 validateNode(binary.left, path, errors, "left")
6375 end
6376 if not binary.right then
6377 addValidationError(path, errors, "Missing 'right' field.")
6378 elseif not EXPRESSION_NODES[binary.right.type] then
6379 addValidationError(path, errors, "The right side is not an expression. (It is '%s'.)", binary.right.type)
6380 else
6381 validateNode(binary.right, path, errors, "right")
6382 end
6383
6384 elseif nodeType == "call" then
6385 local call = node
6386 if not call.callee then
6387 addValidationError(path, errors, "Missing 'callee' field.")
6388 elseif not EXPRESSION_NODES[call.callee.type] then
6389 addValidationError(path, errors, "Callee is not an expression. (It is '%s'.)", call.callee.type)
6390 -- elseif call.callee.type == "literal" or call.callee.type == "table" then -- @Incomplete: Do this kind of check? Or maybe we should stick to strictly validating the AST even if the resulting Lua code would raise a runtime error. :DetectRuntimeErrors
6391 -- addValidationError(path, errors, "Callee is uncallable.")
6392 elseif call.method and not (
6393 call.callee.type == "lookup"
6394 and call.callee.member
6395 and call.callee.member.type == "literal"
6396 and type(call.callee.member.value) == "string"
6397 and stringFind(call.callee.member.value, "^[%a_][%w_]*$")
6398 and not KEYWORDS[call.callee.member.value]
6399 ) then
6400 addValidationError(path, errors, "Callee is unsuitable for method call.")
6401 else
6402 validateNode(call.callee, path, errors, "callee")
6403 end
6404 for i, expr in ipairs(call.arguments) do
6405 if not EXPRESSION_NODES[expr.type] then
6406 addValidationError(path, errors, "Argument %d is not an expression. (It is '%s')", i, expr.type)
6407 else
6408 validateNode(expr, path, errors, "arg"..i)
6409 end
6410 end
6411
6412 elseif nodeType == "function" then
6413 local func = node
6414 for i, ident in ipairs(func.parameters) do
6415 if not (ident.type == "identifier" or (ident.type == "vararg" and i == #func.parameters)) then
6416 addValidationError(path, errors, "Parameter %d is not an identifier%s. (It is '%s')", i, (i == #func.parameters and " or vararg" or ""), ident.type)
6417 else
6418 validateNode(ident, path, errors, "param"..i)
6419 end
6420 end
6421 if not func.body then
6422 addValidationError(path, errors, "Missing 'body' field.")
6423 elseif func.body.type ~= "block" then
6424 addValidationError(path, errors, "Body is not a block.")
6425 else
6426 validateNode(func.body, path, errors, "body")
6427 end
6428
6429 elseif nodeType == "return" then
6430 local returnNode = node
6431 for i, expr in ipairs(returnNode.values) do
6432 if not EXPRESSION_NODES[expr.type] then
6433 addValidationError(path, errors, "Value %d is not an expression. (It is '%s')", i, expr.type)
6434 else
6435 validateNode(expr, path, errors, i)
6436 end
6437 end
6438
6439 elseif nodeType == "block" then
6440 local block = node
6441 for i, statement in ipairs(block.statements) do
6442 if not STATEMENT_NODES[statement.type] then
6443 addValidationError(path, errors, "Child node %d is not a statement. (It is '%s'.)", i, statement.type)
6444 else
6445 validateNode(statement, path, errors, i)
6446 end
6447 end
6448
6449 elseif nodeType == "declaration" then
6450 local decl = node
6451 if not decl.names[1] then
6452 addValidationError(path, errors, "Missing name(s).")
6453 end
6454 for i, ident in ipairs(decl.names) do
6455 if ident.type ~= "identifier" then
6456 addValidationError(path, errors, "Name %d is not an identifier. (It is '%s')", i, ident.type)
6457 else
6458 validateNode(ident, path, errors, "name"..i)
6459 end
6460 end
6461 for i, expr in ipairs(decl.values) do
6462 if not EXPRESSION_NODES[expr.type] then
6463 addValidationError(path, errors, "Value %d is not an expression. (It is '%s')", i, expr.type)
6464 else
6465 validateNode(expr, path, errors, "value"..i)
6466 end
6467 end
6468
6469 elseif nodeType == "assignment" then
6470 local assignment = node
6471 if not assignment.targets[1] then
6472 addValidationError(path, errors, "Missing target expression(s).")
6473 end
6474 for i, expr in ipairs(assignment.targets) do
6475 if not (expr.type == "identifier" or expr.type == "lookup") then
6476 addValidationError(path, errors, "Target %d is not an identifier or lookup. (It is '%s')", i, expr.type)
6477 else
6478 validateNode(expr, path, errors, "target"..i)
6479 end
6480 end
6481 if not assignment.values[1] then
6482 addValidationError(path, errors, "Missing value(s).")
6483 end
6484 for i, expr in ipairs(assignment.values) do
6485 if not EXPRESSION_NODES[expr.type] then
6486 addValidationError(path, errors, "Value %d is not an expression. (It is '%s')", i, expr.type)
6487 else
6488 validateNode(expr, path, errors, "value"..i)
6489 end
6490 end
6491
6492 elseif nodeType == "if" then
6493 local ifNode = node
6494 if not ifNode.condition then
6495 addValidationError(path, errors, "Missing 'condition' field.")
6496 elseif not EXPRESSION_NODES[ifNode.condition.type] then
6497 addValidationError(path, errors, "The condition is not an expression. (It is '%s'.)", ifNode.condition.type)
6498 else
6499 validateNode(ifNode.condition, path, errors, "condition")
6500 end
6501 if not ifNode.bodyTrue then
6502 addValidationError(path, errors, "Missing 'bodyTrue' field.")
6503 elseif ifNode.bodyTrue.type ~= "block" then
6504 addValidationError(path, errors, "Body for true branch is not a block.")
6505 else
6506 validateNode(ifNode.bodyTrue, path, errors, "true")
6507 end
6508 if not ifNode.bodyFalse then
6509 -- void
6510 elseif ifNode.bodyFalse.type ~= "block" then
6511 addValidationError(path, errors, "Body for false branch is not a block.")
6512 else
6513 validateNode(ifNode.bodyFalse, path, errors, "false")
6514 end
6515
6516 elseif nodeType == "while" then
6517 local whileLoop = node
6518 if not whileLoop.condition then
6519 addValidationError(path, errors, "Missing 'condition' field.")
6520 elseif not EXPRESSION_NODES[whileLoop.condition.type] then
6521 addValidationError(path, errors, "The condition is not an expression. (It is '%s'.)", whileLoop.condition.type)
6522 else
6523 validateNode(whileLoop.condition, path, errors, "condition")
6524 end
6525 if not whileLoop.body then
6526 addValidationError(path, errors, "Missing 'body' field.")
6527 elseif whileLoop.body.type ~= "block" then
6528 addValidationError(path, errors, "Body is not a block.")
6529 else
6530 validateNode(whileLoop.body, path, errors, "true")
6531 end
6532
6533 elseif nodeType == "repeat" then
6534 local repeatLoop = node
6535 if not repeatLoop.body then
6536 addValidationError(path, errors, "Missing 'body' field.")
6537 elseif repeatLoop.body.type ~= "block" then
6538 addValidationError(path, errors, "Body is not a block.")
6539 else
6540 validateNode(repeatLoop.body, path, errors, "true")
6541 end
6542 if not repeatLoop.condition then
6543 addValidationError(path, errors, "Missing 'condition' field.")
6544 elseif not EXPRESSION_NODES[repeatLoop.condition.type] then
6545 addValidationError(path, errors, "The condition is not an expression. (It is '%s'.)", repeatLoop.condition.type)
6546 else
6547 validateNode(repeatLoop.condition, path, errors, "condition")
6548 end
6549
6550 elseif nodeType == "for" then
6551 local forLoop = node
6552 if not (forLoop.kind == "numeric" or forLoop.kind == "generic") then
6553 addValidationError(path, errors, "Invalid for loop kind '%s'.", forLoop.kind)
6554 end
6555 if not forLoop.names[1] then
6556 addValidationError(path, errors, "Missing name(s).")
6557 elseif forLoop.kind == "numeric" and forLoop.names[2] then
6558 addValidationError(path, errors, "Too many names for numeric loop. (Got %d)", #forLoop.names)
6559 end
6560 for i, ident in ipairs(forLoop.names) do
6561 if ident.type ~= "identifier" then
6562 addValidationError(path, errors, "Name %d is not an identifier. (It is '%s')", i, ident.type)
6563 else
6564 validateNode(ident, path, errors, "name"..i)
6565 end
6566 end
6567 if not forLoop.values[1] then
6568 addValidationError(path, errors, "Missing value(s).")
6569 elseif forLoop.kind == "numeric" and not forLoop.values[2] then
6570 addValidationError(path, errors, "Too few values for numeric loop. (Got %d)", #forLoop.values)
6571 elseif forLoop.kind == "numeric" and forLoop.values[4] then
6572 addValidationError(path, errors, "Too many values for numeric loop. (Got %d)", #forLoop.values)
6573 end
6574 for i, expr in ipairs(forLoop.values) do
6575 if not EXPRESSION_NODES[expr.type] then
6576 addValidationError(path, errors, "Value %d is not an expression. (It is '%s')", i, expr.type)
6577 else
6578 validateNode(expr, path, errors, "value"..i)
6579 end
6580 end
6581 if not forLoop.body then
6582 addValidationError(path, errors, "Missing 'body' field.")
6583 elseif forLoop.body.type ~= "block" then
6584 addValidationError(path, errors, "Body is not a block.")
6585 else
6586 validateNode(forLoop.body, path, errors, "body")
6587 end
6588
6589 elseif nodeType == "table" then
6590 local tableNode = node
6591 for i, tableField in ipairs(tableNode.fields) do
6592 -- @Incomplete: Should we detect nil literal keys? :DetectRuntimeErrors
6593 if not tableField.key then
6594 if not tableField.generatedKey then
6595 addValidationError(path, errors, "Missing 'key' field for table field %d.", i)
6596 end
6597 elseif not EXPRESSION_NODES[tableField.key.type] then
6598 addValidationError(path, errors, "The key for table field %d is not an expression. (It is '%s'.)", i, tableField.key.type)
6599 elseif tableField.generatedKey and tableField.key.type ~= "literal" then
6600 addValidationError(path, errors, "The generated key for table field %d is not a numeral. (It is '%s'.)", i, tableField.key.type)
6601 elseif tableField.generatedKey and type(tableField.key.value) ~= "number" then
6602 addValidationError(path, errors, "The generated key for table field %d is not a number. (It's a '%s' literal.)", i, type(tableField.key.value))
6603 else
6604 validateNode(tableField.key, path, errors, "key")
6605 end
6606 if not tableField.value then
6607 addValidationError(path, errors, "Missing 'value' field for table field %d.", i)
6608 elseif not EXPRESSION_NODES[tableField.value.type] then
6609 addValidationError(path, errors, "The value for table field %d is not an expression. (It is '%s'.)", i, tableField.value.type)
6610 else
6611 validateNode(tableField.value, path, errors, "value")
6612 end
6613 end
6614
6615 else
6616 errorf("Invalid node type '%s'.", tostring(nodeType)) -- We don't call addValidationError() for this - it's just an assertion.
6617 end
6618
6619 path[#path] = nil
6620 end
6621
6622 -- isValid, errors = validateTree( astNode )
6623 --[[local]] function validateTree(node)
6624 local path = {}
6625 local errors = {}
6626
6627 validateNode(node, path, errors, nil)
6628
6629 if errors[1] then
6630 return false, tableConcat(errors, "\n")
6631 else
6632 return true
6633 end
6634 end
6635end
6636
6637
6638
6639local EXPRESSION_TYPES = newSet{"binary","call","function","identifier","literal","lookup","table","unary","vararg"}
6640
6641local function isExpression(node)
6642 return EXPRESSION_TYPES[node.type] == true
6643end
6644
6645local function isStatement(node)
6646 return EXPRESSION_TYPES[node.type] == nil or node.type == "call"
6647end
6648
6649
6650
6651local function resetNextId()
6652 nextSerialNumber = 1
6653end
6654
6655
6656
6657-- astNode = valueToAst( value [, sortTableKeys=false ] )
6658local function valueToAst(v, sortTableKeys)
6659 local vType = type(v)
6660
6661 if vType == "number" or vType == "string" or vType == "boolean" or vType == "nil" then
6662 return AstLiteral(nil, v)
6663
6664 elseif vType == "table" then
6665 local t = v
6666 local tableNode = AstTable(nil)
6667 local keys = {}
6668 local indices = {}
6669
6670 for k in pairs(t) do
6671 tableInsert(keys, k)
6672 end
6673
6674 if sortTableKeys then
6675 local keyStringRepresentations = {}
6676
6677 for _, k in ipairs(keys) do
6678 keyStringRepresentations[k] = keyStringRepresentations[k] or tostring(k)
6679 end
6680
6681 tableSort(keys, function(a, b)
6682 return keyStringRepresentations[a] < keyStringRepresentations[b]
6683 end)
6684 end
6685
6686 for i = 1, #t do
6687 indices[i] = true
6688 end
6689
6690 for _, k in ipairs(keys) do
6691 if not indices[k] then
6692 local tableField = {key=valueToAst(k,sortTableKeys), value=valueToAst(t[k],sortTableKeys), generatedKey=false}
6693 tableInsert(tableNode.fields, tableField)
6694 end
6695 end
6696
6697 for i = 1, #t do
6698 local tableField = {key=valueToAst(i,sortTableKeys), value=valueToAst(t[i],sortTableKeys), generatedKey=true}
6699 tableInsert(tableNode.fields, tableField)
6700 end
6701
6702 return tableNode
6703
6704 else
6705 error("Invalid value type '"..vType.."'.", 2)
6706 end
6707end
6708
6709
6710
6711-- identifiers = findGlobalReferences( astNode )
6712-- Note: updateReferences() must have been called first!
6713local function findGlobalReferences(theNode)
6714 local idents = {}
6715
6716 traverseTree(theNode, function(node)
6717 if node.type == "identifier" and not node.declaration then
6718 tableInsert(idents, node)
6719 end
6720 end)
6721
6722 return idents
6723end
6724
6725
6726
6727-- identifiers = findDeclaredNames( astNode )
6728local function findDeclaredNames(theNode)
6729 local declIdents = {}
6730
6731 traverseTree(theNode, function(node)
6732 -- Note: We don't now, but if we would require updateReferences() to be called first
6733 -- we could just check the type and if node.declaration==node. Decisions...
6734
6735 if node.type == "declaration" or node.type == "for" then
6736 for _, declIdent in ipairs(node.names) do
6737 tableInsert(declIdents, declIdent)
6738 end
6739
6740 elseif node.type == "function" then
6741 for _, declIdent in ipairs(node.parameters) do
6742 if declIdent.type == "identifier" then -- There may be a vararg at the end.
6743 tableInsert(declIdents, declIdent)
6744 end
6745 end
6746 end
6747 end)
6748
6749 return declIdents
6750end
6751
6752
6753
6754-- shadows, foundPrevious = maybeRegisterShadow( shadowSequences, shadowSequenceByIdent, shadows|nil, currentDeclIdent, declIdent )
6755local function maybeRegisterShadow(shadowSequences, shadowSequenceByIdent, shadows, currentDeclIdent, declIdent)
6756 if declIdent.name ~= currentDeclIdent.name then
6757 return shadows, false
6758 end
6759
6760 if not shadows then
6761 shadows = {currentDeclIdent}
6762 shadowSequenceByIdent[currentDeclIdent] = shadows
6763 tableInsert(shadowSequences, shadows)
6764 end
6765
6766 if shadowSequenceByIdent[declIdent] then
6767 -- Shortcut! declIdent is shadowing others, so just copy the existing data.
6768 for _, prevDeclIdent in ipairs(shadowSequenceByIdent[declIdent]) do
6769 tableInsert(shadows, prevDeclIdent)
6770 end
6771 return shadows, true
6772
6773 else
6774 tableInsert(shadows, declIdent)
6775 return shadows, false
6776 end
6777end
6778
6779-- shadowSequences = findShadows( astNode )
6780-- shadowSequences = { shadowSequence1, ... }
6781-- shadowSequence = { shadowingIdentifier, shadowedIdentifier1, ... }
6782-- Note: updateReferences() must have been called first!
6783local function findShadows(theNode)
6784 local shadowSequences = {}
6785 local shadowSequenceByIdent = {}
6786
6787 for _, currentDeclIdent in ipairs(findDeclaredNames(theNode)) do
6788 local shadows = nil
6789 local child = currentDeclIdent
6790 local foundPrevious = false
6791
6792 while child.parent do
6793 local parent = child.parent
6794
6795 if isNodeDeclLike(parent) then
6796 local declIdents = getNameArrayOfDeclLike(parent)
6797 local childIndex = indexOf(declIdents, child) or #declIdents+1
6798
6799 for i = childIndex-1, 1, -1 do
6800 shadows, foundPrevious = maybeRegisterShadow(shadowSequences, shadowSequenceByIdent, shadows, currentDeclIdent, declIdents[i])
6801 if foundPrevious then break end
6802 end
6803 if foundPrevious then break end
6804
6805 elseif parent.type == "block" then
6806 local statements = parent.statements
6807
6808 for i = child.key-1, 1, -1 do
6809 if statements[i].type == "declaration" then
6810 for _, declIdent in ipairs(statements[i].names) do
6811 shadows, foundPrevious = maybeRegisterShadow(shadowSequences, shadowSequenceByIdent, shadows, currentDeclIdent, declIdent)
6812 if foundPrevious then break end
6813 end
6814 if foundPrevious then break end
6815 end
6816 end
6817 if foundPrevious then break end
6818 end
6819
6820 child = parent
6821 if child == theNode then break end -- Stay within theNode (in case theNode has a parent).
6822 end
6823 end
6824
6825 return shadowSequences
6826end
6827
6828
6829
6830parser = {
6831 --
6832 -- Constants.
6833 --
6834 VERSION = PARSER_VERSION,
6835
6836 INT_SIZE = INT_SIZE,
6837 MAX_INT = MAX_INT,
6838 MIN_INT = MIN_INT,
6839
6840 --
6841 -- Functions.
6842 --
6843
6844 -- Tokenizing.
6845 tokenize = tokenize,
6846 tokenizeFile = tokenizeFile,
6847
6848 -- Token actions.
6849 newToken = newToken,
6850 updateToken = updateToken,
6851 cloneToken = cloneToken,
6852 concatTokens = concatTokens,
6853
6854 -- AST parsing.
6855 parse = parse,
6856 parseExpression = parseExpression,
6857 parseFile = parseFile,
6858
6859 -- AST manipulation.
6860 newNode = newNode,
6861 newNodeFast = newNodeFast,
6862 valueToAst = valueToAst,
6863 cloneNode = cloneNode,
6864 cloneTree = cloneTree,
6865 getChild = getChild,
6866 setChild = setChild,
6867 addChild = addChild,
6868 removeChild = removeChild,
6869
6870 -- AST checking.
6871 isExpression = isExpression,
6872 isStatement = isStatement,
6873 validateTree = validateTree,
6874
6875 -- AST traversal.
6876 traverseTree = traverseTree,
6877 traverseTreeReverse = traverseTreeReverse,
6878 updateReferences = updateReferences,
6879
6880 -- Big AST operations.
6881 simplify = simplify,
6882 optimize = optimize,
6883 minify = minify,
6884
6885 -- Conversion.
6886 toLua = toLua,
6887
6888 -- Printing.
6889 printTokens = printTokens,
6890 printNode = printNode,
6891 printTree = printTree,
6892 formatMessage = formatMessage,
6893
6894 -- Utilities.
6895 findDeclaredNames = findDeclaredNames,
6896 findGlobalReferences = findGlobalReferences,
6897 findShadows = findShadows,
6898
6899 -- Misc.
6900 resetNextId = resetNextId, -- @Undocumented
6901
6902 --
6903 -- Settings.
6904 --
6905 printIds = false,
6906 printLocations = false,
6907 indentation = " ",
6908
6909 constantNameReplacementStringMaxLength = 200, -- @Cleanup: Maybe use a better name.
6910}
6911
6912return parser
6913
6914
6915
6916--[[!===========================================================
6917
6918Copyright © 2020-2022 Marcus 'ReFreezed' Thunström
6919
6920Permission is hereby granted, free of charge, to any person obtaining a copy
6921of this software and associated documentation files (the "Software"), to deal
6922in the Software without restriction, including without limitation the rights
6923to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
6924copies of the Software, and to permit persons to whom the Software is
6925furnished to do so, subject to the following conditions:
6926
6927The above copyright notice and this permission notice shall be included in all
6928copies or substantial portions of the Software.
6929
6930THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
6931IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
6932FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
6933AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
6934LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
6935OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
6936SOFTWARE.
6937
6938==============================================================]]