From 16f33cae32955148f1a0438eadf55770ee1fb018 Mon Sep 17 00:00:00 2001 From: tri Date: Wed, 8 Oct 2025 18:01:36 +0700 Subject: [PATCH] cd clib; make djot_combined.lua --- clib/djot_combined.lua | 1 + clib/dumbParser.lua | 6938 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 6939 insertions(+) create mode 100644 clib/djot_combined.lua create mode 100644 clib/dumbParser.lua diff --git a/clib/djot_combined.lua b/clib/djot_combined.lua new file mode 100644 index 0000000..b4e3354 --- /dev/null +++ b/clib/djot_combined.lua @@ -0,0 +1 @@ +package.preload["djot.attributes"]=function()local e,t=string.find,string.sub;local a=0;local o=1;local i=2;local n=3;local s=4;local h=5;local r=6;local d=7;local l=8;local c=9;local u=10;local m=11;local w=12;local f=13;local g={};local y={};y[f]=function(t,o)if e(t.subject,"^{",o)then return a;else return m;end end;y[m]=function(e,e)return m;end;y[w]=function(e,e)return w;end;y[a]=function(s,h)local t=t(s.subject,h,h);if(t==" ")or(t=="\t")or(t=="\n")or(t=="\r")then return a;elseif t=="}"then return w;elseif t=="#"then s.begin=h;return o;elseif t=="%"then s.begin=h;return u;elseif t=="."then s.begin=h;return i;elseif e(t,"^[%a%d_:-]")then s.begin=h;return n;else return m;end end;y[u]=function(e,o)local e=t(e.subject,o,o);if e=="%"then return a;elseif e=="}"then return w;else return u;end end;y[o]=function(i,n)local t=t(i.subject,n,n);if e(t,"^[^%s%p]")or(t=="_")or(t=="-")or(t==":")then return o;elseif t=="}"then if i.lastpos>i.begin then i:add_match(i.begin+1,i.lastpos,"id");end i.begin=nil;return w;elseif e(t,"^%s")then if i.lastpos>i.begin then i:add_match(i.begin+1,i.lastpos,"id");end i.begin=nil;return a;else return m;end end;y[i]=function(o,n)local t=t(o.subject,n,n);if e(t,"^[^%s%p]")or(t=="_")or(t=="-")or(t==":")then return i;elseif t=="}"then if o.lastpos>o.begin then o:add_match(o.begin+1,o.lastpos,"class");end o.begin=nil;return w;elseif e(t,"^%s")then if o.lastpos>o.begin then o:add_match(o.begin+1,o.lastpos,"class");end o.begin=nil;return a;else return m;end end;y[n]=function(a,o)local t=t(a.subject,o,o);if t=="="then a:add_match(a.begin,a.lastpos,"key");a.begin=nil;return s;elseif e(t,"^[%a%d_:-]")then return n;else return m;end end;y[s]=function(a,o)local t=t(a.subject,o,o);if t=='"'then a.begin=o;return r;elseif e(t,"^[%a%d_:-]")then a.begin=o;return h;else return m;end end;y[h]=function(o,i)local t=t(o.subject,i,i);if e(t,"^[%a%d_:-]")then return h;elseif t=="}"then o:add_match(o.begin,o.lastpos,"value");o.begin=nil;return w;elseif e(t,"^%s")then o:add_match(o.begin,o.lastpos,"value");o.begin=nil;return a;else return m;end end;y[l]=function(e,e)return r;end;y[c]=function(e,e)return d;end;y[r]=function(e,o)local t=t(e.subject,o,o);if t=='"'then e:add_match(e.begin+1,e.lastpos,"value");e.begin=nil;return a;elseif t=="\n"then e:add_match(e.begin+1,e.lastpos,"value");e.begin=nil;return d;elseif t=="\\"then return l;else return r;end end;y[d]=function(e,o)local t=t(e.subject,o,o);if e.begin==nil then e.begin=o;end if t=='"'then e:add_match(e.begin,e.lastpos,"value");e.begin=nil;return a;elseif t=="\n"then e:add_match(e.begin,e.lastpos,"value");e.begin=nil;return d;elseif t=="\\"then return c;else return d;end end;function g.new(e,t)local t={subject=t,state=f,begin=nil,lastpos=nil,matches={}};setmetatable(t,e);e.__index=e;return t;end function g.add_match(e,t,a,o)e.matches[#e.matches+1]={t,a,o};end function g.get_matches(e)return e.matches;end function g.feed(e,t,a)local t=t;while t<=a do e.state=y[e.state](e,t);if e.state==w then return"done",t;elseif e.state==m then e.lastpos=t;return"fail",t;else e.lastpos=t;t=t+1;end end return"continue",a;end return{AttributeParser=g};end;package.preload["djot.inline"]=function()local e=unpack or table.unpack;local t=require("djot.attributes");local a,o=string.find,string.byte;local function i(e,t,o,i)local e,t,a,o,n=a(e,t,o);if t and(t<=i)then return e,t,a,o,n;end end local n={};function n.new(e,t,a)local t={warn=a or function()end,subject=t,matches={},openers={},verbatim=0,verbatim_type=nil,destination=false,firstpos=0,lastpos=0,allow_attributes=true,attribute_parser=nil,attribute_start=nil,attribute_slices=nil};setmetatable(t,e);e.__index=e;return t;end function n.add_match(e,t,a,o)e.matches[t]={t,a,o};end function n.add_opener(e,t,...)if not e.openers[t]then e.openers[t]={};end table.insert(e.openers[t],{...});end function n.clear_openers(t,a,o)for t,t in pairs(t.openers)do local i=#t;while t[i]do local e,n,s,s,h=e(t[i]);if(e>=a)and(n<=o)then t[i]=nil;elseif s and(s>=a)and h and(h<=o)then t[i][3]=nil;t[i][4]=nil;t[i][5]=nil;else break;end i=i-1;end end end function n.str_matches(t,a,o)for a=a,o do local o=t.matches[a];if o then local e,o,i=e(o);if(i~="str")and(i~="escape")then t.matches[a]={e,o,"str"};end end end end local function s(e,t)if e then return string.find(e[3],t);end end function n.between_matched(t,i,n,h)return function(r,d,l)n=n or"str";local c=r.subject;local u=a(c,"^%S",d+1);local a=a(c,"^%S",d-1);local s=s(r.matches[d-1],"^open%_marker");local o=((d+1)<=l)and(o(c,d+1)==125);local l=d;local c=d;if type(h)=="function"then u=u and h(r,d);end if s then u=true;a=false;c=d-1;end if not s and o then a=true;u=false;l=d+1;end if s and n:match("^right")then n=n:gsub("^right","left");elseif o and n:match("^left")then n=n:gsub("^left","right");end local h;if o then h="{"..t;else h=t;end local o=r.openers[h];if a and o and(#o>0)then local e,t=e(o[#o]);if t~=(d-1)then r:clear_openers(e,d);r:add_match(e,t,"+"..i);r:add_match(d,l,"-"..i);return l+1;end end if u then if s then h="{"..t;else h=t;end r:add_opener(h,c,d);r:add_match(c,d,n);return d+1;else r:add_match(d,l,n);return l+1;end end;end n.matchers={[96]=function(e,t,o)local n=e.subject;local o,o=i(n,"^`*",t,o);if not o then return nil;end if a(n,"^%$%$",t-2)and not a(n,"^\\",t-3)then e.matches[t-2]=nil;e.matches[t-1]=nil;e:add_match(t-2,o,"+display_math");e.verbatim_type="display_math";elseif a(n,"^%$",t-1)then e.matches[t-1]=nil;e:add_match(t-1,o,"+inline_math");e.verbatim_type="inline_math";else e:add_match(t,o,"+verbatim");e.verbatim_type="verbatim";end e.verbatim=(o-t)+1;return o+1;end,[92]=function(t,o,n)local s=t.subject;local h,h=i(s,"^[ \t]*\r?\n",o+1,n);t:add_match(o,o,"escape");if h then if#t.matches>0 then local e,a,o=e(t.matches[#t.matches]);if o=="str"then while(a>=e)and((s:byte(a)==32)or(s:byte(a)==9))do a=a-1;end if a%s]+%>",t,a);if a then local s=i(o,"^%a+:",t+1,n);local t=i(o,"^[^:]+%@",t+1,n);if t then e:add_match(a,a,"+email");e:add_match(a+1,n-1,"str");e:add_match(n,n,"-email");return n+1;elseif s then e:add_match(a,a,"+url");e:add_match(a+1,n-1,"str");e:add_match(n,n,"-url");return n+1;end end end,[126]=n.between_matched("~","subscript"),[94]=n.between_matched("^","superscript"),[91]=function(e,t,a)local a,o=i(e.subject,"^%^([^]]+)%]",t+1,a);if a then e:add_match(t,o,"footnote_reference");return o+1;else e:add_opener("[",t,t);e:add_match(t,t,"str");return t+1;end end,[93]=function(e,t,a)local o=e.openers["["];local n=e.subject;if o and(#o>0)then local o=o[#o];if o[3]=="reference_link"then local a=i(n,"^!",o[1]-1,a)and not i(n,"^[\\]",o[1]-2,a);if a then e:add_match(o[1]-1,o[1]-1,"image_marker");e:add_match(o[1],o[2],"+imagetext");e:add_match(o[4],o[4],"-imagetext");else e:add_match(o[1],o[2],"+linktext");e:add_match(o[4],o[4],"-linktext");end e:add_match(o[5],o[5],"+reference");e:add_match(t,t,"-reference");e:str_matches(o[5]+1,t-1);e:clear_openers(o[1],t);return t+1;elseif i(n,"^%[",t+1,a)then o[3]="reference_link";o[4]=t;o[5]=t+1;e:add_match(t,t+1,"str");e:clear_openers(o[1]+1,t-1);return t+2;elseif i(n,"^%(",t+1,a)then e.openers["("]={};o[3]="explicit_link";o[4]=t;o[5]=t+1;e.destination=true;e:add_match(t,t+1,"str");e:clear_openers(o[1]+1,t-1);return t+2;elseif i(n,"^%{",t+1,a)then e:add_match(o[1],o[2],"+span");e:add_match(t,t,"-span");e:clear_openers(o[1],t);return t+1;end end end,[40]=function(e,t)if not e.destination then return nil;end e:add_opener("(",t,t);e:add_match(t,t,"str");return t+1;end,[41]=function(e,t,a)if not e.destination then return nil;end local o=e.openers["("];if o and(#o>0)and o[#o][1]then o[#o]=nil;e:add_match(t,t,"str");return t+1;else local o=e.subject;local n=e.openers["["];if n and(#n>0)and(n[#n][3]=="explicit_link")then local n=n[#n];local a=i(o,"^!",n[1]-1,a)and not i(o,"^[\\]",n[1]-2,a);if a then e:add_match(n[1]-1,n[1]-1,"image_marker");e:add_match(n[1],n[2],"+imagetext");e:add_match(n[4],n[4],"-imagetext");else e:add_match(n[1],n[2],"+linktext");e:add_match(n[4],n[4],"-linktext");end e:add_match(n[5],n[5],"+destination");e:add_match(t,t,"-destination");e.destination=false;e:str_matches(n[5]+1,t-1);e:clear_openers(n[1],t);return t+1;end end end,[95]=n.between_matched("_","emph"),[42]=n.between_matched("*","strong"),[123]=function(e,a,o)if i(e.subject,"^[_*~^+='\"-]",a+1,o)then e:add_match(a,a,"open_marker");return a+1;elseif e.allow_attributes then e.attribute_parser=t.AttributeParser:new(e.subject);e.attribute_start=a;e.attribute_slices={};return a;else e:add_match(a,a,"str");return a+1;end end,[58]=function(e,t,a)local a,o=i(e.subject,"^%:[%w_+-]+%:",t,a);if a then e:add_match(a,o,"symbol");return o+1;else e:add_match(t,t,"str");return t+1;end end,[43]=n.between_matched("+","insert","str",function(e,t)return a(e.subject,"^%{",t-1)or a(e.subject,"^%}",t+1);end),[61]=n.between_matched("=","mark","str",function(e,t)return a(e.subject,"^%{",t-1)or a(e.subject,"^%}",t+1);end),[39]=n.between_matched("'","single_quoted","right_single_quote",function(e,t)return(t==1)or a(e.subject,"^[%s\"'-([]",t-1);end),[34]=n.between_matched('"',"double_quoted","left_double_quote"),[45]=function(e,t,i)local s=e.subject;local h;if(o(s,t-1)==123)or(o(s,t+1)==125)then h=n.between_matched("-","delete","str",function(e,t)return a(e.subject,"^%{",t-1)or a(e.subject,"^%}",t+1);end)(e,t,i);return h;end local a,a=a(s,"^%-*",t);if i0 do if a then e:add_match(t,t+2,"em_dash");t=t+3;i=i-3;elseif o then e:add_match(t,t+1,"en_dash");t=t+2;i=i-2;elseif(i>=3)and(((i%2)~=0)or(i>4))then e:add_match(t,t+2,"em_dash");t=t+3;i=i-3;elseif i>=2 then e:add_match(t,t+1,"en_dash");t=t+2;i=i-2;else e:add_match(t,t,"str");t=t+1;i=i-1;end end return t;end,[46]=function(e,t,a)if i(e.subject,"^%.%.",t+1,a)then e:add_match(t,t+2,"ellipses");return t+3;end end};function n.single_char(e,t)e:add_match(t,t,"str");return t+1;end function n.reparse_attributes(t)local a=t.attribute_slices;if not a then return;end t.allow_attributes=false;t.attribute_parser=nil;t.attribute_start=nil;if a then for o=1,#a do t:feed(e(a[o]));end end t.allow_attributes=true;t.attribute_slices=nil;end function n.feed(t,a,n)local s="[][\\`{}_*()!<>~^:=+$\r\n'\".-]";local h=t.subject;local r=t.matchers;local d;if(t.firstpos==0)or(at.lastpos)then t.lastpos=n;end d=a;while d<=n do if t.attribute_parser then local a=d;local o=i(h,s,d,n);if not o or(o>n)then o=n;end local o,i=t.attribute_parser:feed(a,o);if o=="done"then local a=t.attribute_start;t:add_match(a,a,"+attributes");t:add_match(i,i,"-attributes");local a=t.attribute_parser:get_matches();for o=1,#a do t:add_match(e(a[o]));end t.attribute_parser=nil;t.attribute_start=nil;t.attribute_slices=nil;d=i+1;elseif o=="fail"then t:reparse_attributes();d=a;elseif o=="continue"then if#t.attribute_slices==0 then t.attribute_slices={};end t.attribute_slices[#t.attribute_slices+1]={a,i};d=i+1;end else local e=i(h,s,d,n)or(n+1);if e>d then t:add_match(d,e-1,"str");d=e;if d>n then break;end end local e=o(h,d);if(e==13)or(e==10)then if(e==13)and i(h,"^[%n]",d+1,n)then t:add_match(d,d+1,"softbreak");d=d+2;else t:add_match(d,d,"softbreak");d=d+1;end elseif t.verbatim>0 then if e==96 then local e,e=i(h,"^`+",d,n);if e and(((e-d)+1)==t.verbatim)then local a,o=i(h,"^%{%=[^%s{}`]+%}",e+1,n);if a and(t.verbatim_type=="verbatim")then t:add_match(d,e,"-"..t.verbatim_type);t:add_match(a,o,"raw_format");d=o+1;else t:add_match(d,e,"-"..t.verbatim_type);d=e+1;end t.verbatim=0;t.verbatim_type=nil;else e=e or n;t:add_match(d,e,"str");d=e+1;end else t:add_match(d,d,"str");d=d+1;end else local e=r[e];d=(e and e(t,d,n))or t:single_char(d);end end end end function n.in_verbatim(e)return e.verbatim>0;end function n.get_matches(t)local a={};local i=t.subject;local n,s,h;if t.attribute_parser then t:reparse_attributes();end for o=t.firstpos,t.lastpos do if t.matches[o]then local e,i,r=e(t.matches[o]);if(r=="str")and(h=="str")and((s+1)==e)then a[#a]={n,i,r};n,s,h=n,i,r;else a[#a+1]=t.matches[o];n,s,h=e,i,r;end end end if#a>0 then local n=a[#a];local s,h,r=e(n);if r=="softbreak"then a[#a]=nil;n=a[#a];if not n then return a;end s,h,r=e(n);end if(r=="str")and(o(i,h)==32)then while(h>s)and(o(i,h)==32)do h=h-1;end a[#a]={s,h,r};end if t.verbatim>0 then t.warn({message="Unclosed verbatim",pos=h});a[#a+1]={h,h,"-"..t.verbatim_type};end end return a;end return{InlineParser=n};end;package.preload["djot.block"]=function()local e=require("djot.inline").InlineParser;local t=require("djot.attributes");local a=unpack or table.unpack;local o,i,n=string.find,string.sub,string.byte;local s={};function s.new(e,t,a)e=t;local t={};setmetatable(t,e);e.__index=e;if a then for e,a in pairs(a)do t[e]=a;end end return t;end local function h(e)if(e=="+")or(e=="-")or(e=="*")or(e==":")then return{e};elseif o(e,"^[+*-] %[[Xx ]%]")then return{"X"};elseif o(e,"^[(]?%d+[).]")then return{(e:gsub("%d+","1"))};elseif o(e,"^[(]?[ivxlcdm][).]")then return{(e:gsub("%a+","i")),(e:gsub("%a+","a"))};elseif o(e,"^[(]?[IVXLCDM][).]")then return{(e:gsub("%a+","I")),(e:gsub("%a+","A"))};elseif o(e,"^[(]?%l[).]")then return{(e:gsub("%l","a"))};elseif o(e,"^[(]?%u[).]")then return{(e:gsub("%u","A"))};elseif o(e,"^[(]?[ivxlcdm]+[).]")then return{(e:gsub("%a+","i"))};elseif o(e,"^[(]?[IVXLCDM]+[).]")then return{(e:gsub("%a+","I"))};else return{};end end local r={};function r.new(e,t,a)if not t:find("[\r\n]$")then t=t.."\n";end local t={warn=a or function()end,subject=t,indent=0,startline=nil,starteol=nil,endeol=nil,matches={},containers={},pos=1,last_matched_container=0,timer={},finished_line=false,returned=0};setmetatable(t,e);e.__index=e;return t;end function r.parse_table_row(t,i,s)local h=#t.matches;local r=t.pos;t:add_match(i,i,"+row");t.pos=o(t.subject,"%S",i+1);local d={};local l=t.pos;local c=false;while not c do local e,a,o,i,n=o(t.subject,"^(%:?)%-%-*(%:?)([ \t]*%|[ \t]*)",l);if a then local s="separator_default";if(#o>0)and(#i>0)then s="separator_center";elseif#i>0 then s="separator_right";elseif#o>0 then s="separator_left";end d[#d+1]={e,a-#n,s};l=a+1;if l==t.starteol then c=true;break;end else break;end end if c then for e=1,#d do t:add_match(a(d[e]));end t:add_match(t.starteol-1,t.starteol-1,"-row");t.pos=t.starteol;t.finished_line=true;return true;end local d=e:new(t.subject,t.warn);t:add_match(i,i,"+cell");local i=false;while t.pos<=s do local h,r;while not h do r,h=t:find("^[^|\r\n]*|");if not h then break;end if string.find(t.subject,"^\\",h-1)then d:feed(t.pos,h);t.pos=h+1;h=nil;else d:feed(t.pos,h-1);if d:in_verbatim()then d:feed(h,h);t.pos=h+1;h=nil;else t.pos=h+1;end end end i=h;if not i then break;end local i=d:get_matches();for e=1,#i do local a,o,s=a(i[e]);if(e==#i)and(s=="str")then while(n(t.subject,o)==32)and(o>=a)do o=o-1;end end t:add_match(a,o,s);end t:add_match(h,h,"-cell");if h%s")then n.pos=n.pos+1;return true;else return false;end end,open=function(e)if n:find("^%>%s")then n:add_container(s:new(e));n:add_match(n.pos,n.pos,"+blockquote");n.pos=n.pos+1;return true;end end,close=function()n:add_match(n.pos,n.pos,"-blockquote");n.containers[#n.containers]=nil;end},{name="footnote",content="block",continue=function(e)if(n.indent>e.indent)or n:find("^[\r\n]")then return true;else return false;end end,open=function(e)local t,a,o=n:find("^%[%^([^]]+)%]:%s");if not t then return nil;end n:add_container(s:new(e,{note_label=o,indent=n.indent}));n:add_match(t,t,"+footnote");n:add_match(t+2,a-3,"note_label");n.pos=a;return true;end,close=function(e)n:add_match(n.pos,n.pos,"-footnote");n.containers[#n.containers]=nil;end},{name="thematic_break",content=nil,continue=function()return false;end,open=function(e)local t,a=n:find("^[-*][ \t]*[-*][ \t]*[-*][-* \t]*[\r\n]");if a then n:add_container(s:new(e));n:add_match(t,a,"thematic_break");n.pos=a;return true;end end,close=function(e)n.containers[#n.containers]=nil;end},{name="list_item",content="block",continue=function(e)if(n.indent>e.indent)or n:find("^[\r\n]")then return true;else return false;end end,open=function(e)local t,a=n:find("^[-*+:]%s");if not t then t,a=n:find("^%d+[.)]%s");end if not t then t,a=n:find("^%(%d+%)%s");end if not t then t,a=n:find("^[ivxlcdmIVXLCDM]+[.)]%s");end if not t then t,a=n:find("^%([ivxlcdmIVXLCDM]+%)%s");end if not t then t,a=n:find("^%a[.)]%s");end if not t then t,a=n:find("^%(%a%)%s");end if not t then return nil;end local o=i(n.subject,t,a-1);local r=nil;if n:find("^[*+-] %[[Xx ]%]%s",t+1)then o=i(n.subject,t,t+4);r=i(n.subject,t+3,t+3);end local o=h(o);if#o==0 then return nil;end local i={styles=o,indent=n.indent};n:add_container(s:new(e,i));local e="+list_item";for t=1,#o do e=e.."|"..o[t];end n:add_match(t,a-1,e);n.pos=a;if r then if r==" "then n:add_match(t+2,t+4,"checkbox_unchecked");else n:add_match(t+2,t+4,"checkbox_checked");end n.pos=t+5;end return true;end,close=function(e)n:add_match(n.pos,n.pos,"-list_item");n.containers[#n.containers]=nil;end},{name="reference_definition",content=nil,continue=function(e)if e.indent>=n.indent then return false;end local e,e,t=n:find("^(%S+)");if e and(n.starteol==(e+1))then n:add_match((e-#t)+1,e,"reference_value");n.pos=e+1;return true;else return false;end end,open=function(e)local t,a,o,i=n:find("^%[([^]\r\n]*)%]:[ \t]*(%S*)");if a and(n.starteol==(a+1))then n:add_container(s:new(e,{key=o,indent=n.indent}));n:add_match(t,t,"+reference_definition");n:add_match(t,t+#o+1,"reference_key");if#i>0 then n:add_match((a-#i)+1,a,"reference_value");end n.pos=a+1;return true;end end,close=function(e)n:add_match(n.pos,n.pos,"-reference_definition");n.containers[#n.containers]=nil;end},{name="heading",content="inline",continue=function(e)local t,a=n:find("^%#+%s");if t and a and(e.level==(a-t))then n.pos=a;return true;else return false;end end,open=function(t)local a,i=n:find("^#+");if i and o(n.subject,"^%s",i+1)then local o=(i-a)+1;n:add_container(s:new(t,{level=o,inline_parser=e:new(n.subject,n.warn)}));n:add_match(a,i,"+heading");n.pos=i+1;return true;end end,close=function(e)n:get_inline_matches();local e=n.matches[#n.matches]or{n.pos,n.pos,""};local e,e,t=a(e);n:add_match(e+1,e+1,"-heading");n.containers[#n.containers]=nil;end},{name="code_block",content="text",continue=function(e)local t=i(e.border,1,1);local t,a,o=n:find("^("..e.border..t.."*)[ \t]*[\r\n]");if a then e.end_fence_sp=t;e.end_fence_ep=(t+#o)-1;n.pos=a;n.finished_line=true;return false;else return true;end end,open=function(e)local t,a,i,h,r=n:find("^(~~~~*)([ \t]*)(%S*)[ \t]*[\r\n]");if not a then t,a,i,h,r=n:find("^(````*)([ \t]*)([^%s`]*)[ \t]*[\r\n]");end if i then local o=(o(r,"^=")and true)or false;n:add_container(s:new(e,{border=i,indent=n.indent}));n:add_match(t,(t+#i)-1,"+code_block");if#r>0 then local e=t+#i+#h;if o then n:add_match(e,(e+#r)-1,"raw_format");else n:add_match(e,(e+#r)-1,"code_language");end end n.pos=a;n.finished_line=true;return true;end end,close=function(e)local t=e.end_fence_sp or n.pos;local e=e.end_fence_ep or n.pos;n:add_match(t,e,"-code_block");if t==e then n.warn({pos=n.pos,message="Unclosed code block"});end n.containers[#n.containers]=nil;end},{name="fenced_div",content="block",continue=function(e)if n.containers[#n.containers].name=="code_block"then return true;end local t,a,o=n:find("^(::::*)[ \t]*[\r\n]");if a and(#o>=e.equals)then e.end_fence_sp=t;e.end_fence_ep=(t+#o)-1;n.pos=a;return false;else return true;end end,open=function(e)local t,a,i=n:find("^(::::*)[ \t]*");if not a then return false;end local a,h=o(n.subject,"^[%w_-]*",a+1);local o,o=o(n.subject,"^[ \t]*[\r\n]",h+1);if o then n:add_container(s:new(e,{equals=#i}));n:add_match(t,h,"+div");if h>=a then n:add_match(a,h,"class");end n.pos=o+1;n.finished_line=true;return true;end end,close=function(e)local t=e.end_fence_sp or n.pos;local e=e.end_fence_ep or n.pos;n:add_match(t,e,"-div");if t==e then n.warn({pos=n.pos,message="Unclosed div"});end n.containers[#n.containers]=nil;end},{name="table",content="cells",continue=function(e)local e,t=n:find("^|[^\r\n]*|");local a=t and o(n.subject,"^[ \t]*[\r\n]",t+1);if a then return n:parse_table_row(e,t);end end,open=function(e)local t,a=n:find("^|[^\r\n]*|");local o=" *[\r\n]";if t and o then n:add_container(s:new(e,{columns=0}));n:add_match(t,t,"+table");if n:parse_table_row(t,a)then return true;else n.containers[#n.containers]=nil;return false;end end end,close=function(e)n:add_match(n.pos,n.pos,"-table");n.containers[#n.containers]=nil;end},{name="attributes",content="attributes",open=function(e)if n:find("^%{")then local t=t.AttributeParser:new(n.subject);local a,o=t:feed(n.pos,n.endeol);if(a=="fail")or((o+1)t.indent then table.insert(t.slices,{n.pos,n.endeol});local e,a=t.attribute_parser:feed(n.pos,n.endeol);t.status=e;if(e~="fail")or((a+1)a)or((#e.containers>0)and(e.containers[#e.containers].content~="block"))do e.containers[#e.containers]:close();end e.containers[#e.containers+1]=t;end function r.skip_space(e)local t,a=o(e.subject,"[^ \t]",e.pos);if t then e.indent=t-e.startline;e.pos=t;end end function r.get_eol(e)local t,a=o(e.subject,"[\r]?[\n]",e.pos);if not a then t,a=#e.subject,#e.subject;end e.starteol=t;e.endeol=a;end function r.events(e)local t=e:specs();local o=t[1];local i=#e.subject;return function()while e.pos<=i do if e.returned<#e.matches then e.returned=e.returned+1;return a(e.matches[e.returned]);end e.indent=0;e.startline=e.pos;e.finished_line=false;e:get_eol();e.last_matched_container=0;local a=0;while a<#e.containers do a=a+1;local t=e.containers[a];e:skip_space();if t:continue()then e.last_matched_container=a;else break;end end if e.finished_line then while#e.containers>e.last_matched_container do e.containers[#e.containers]:close();end end if not e.finished_line then e:skip_space();local a=e.pos==e.starteol;local i=false;local n=e.containers[e.last_matched_container];local n=not a and(not n or(n.content=="block"))and not e:find("^%a+%s");while n do n=false;for a=1,#t do local t=t[a];if not t.is_para then if t:open()then e.last_matched_container=#e.containers;if e.finished_line then n=false;else e:skip_space();i=true;n=t.content=="block";end break;end end end end if not e.finished_line then e:skip_space();a=e.pos==e.starteol;local t=not a and not i and(e.last_matched_container<#e.containers)and(e.containers[#e.containers].content=="inline");local n=e.last_matched_container;if not t then while(#e.containers>0)and(#e.containers>n)do e.containers[#e.containers]:close();end end local t=e.containers[#e.containers];if not t or(t.content=="block")then if a then if not i then e:add_match(e.pos,e.endeol,"blankline");end else o:open();end t=e.containers[#e.containers];end if t then if t.content=="text"then local a=e.pos;if t.indent and(e.indent>t.indent)then a=a-(e.indent-t.indent);end e:add_match(a,e.endeol,"str");elseif t.content=="inline"then if not a then t.inline_parser:feed(e.pos,e.endeol);end end end end end e.pos=e.endeol+1;end while#e.containers>0 do e.containers[#e.containers]:close();end if e.returned<#e.matches then e.returned=e.returned+1;return a(e.matches[e.returned]);end end;end return{Parser=r,Container=s};end;package.preload["djot.ast"]=function()if not utf8 then local e=pairs;function pairs(t)local a=getmetatable(t);if(type(a)=="table")and(type(a.__pairs)=="function")then return a.__pairs(t);else return e(t);end end end local e=unpack or table.unpack;local e,t,a,o,i=string.find,string.lower,string.sub,string.rep,string.format;local function n(e)local t={line={},col={},charpos={}};local a=1;local o=0;local i=0;local n=1;local s=string.byte(e,n);while s do o=o+1;i=i+1;local h;if s<192 then h=n+1;elseif s<224 then h=n+2;elseif s<240 then h=n+3;else h=n+4;end while n0 do local t=t(a(e,n,n));local t=s[t];assert(t~=nil,"Encountered bad character in roman numeral "..e);if t0);end local function u(e)local e=e or{};setmetatable(e,{__pairs=r(function(e,t)return e0)and a[#a].level)or 0;if o>=t then while a[#a].level>=t do local e=table.remove(a).sec;d(a[#a].sec,e);end end local o=l("section");o.attr=u({id=e.attr.id});e.attr.id=nil;d(o,e);a[#a+1]={sec=o,level=t};else d(a[#a].sec,e);end end while#a>1 do local e=table.remove(a).sec;d(a[#a].sec,e);end assert(a[1].sec==t);return t;end local function p(o,i)local m=o.subject;local p=o.warn;if not p then function p()end end local b;if i then b=n(m);end local i={};local n={};local v={};local function k(e)local e=e:gsub("[][~!@#$%^&*(){}`,.<>\\|=+/?]",""):gsub("^%s+",""):gsub("%s+$",""):gsub("%s+","-");local t=0;local a=e;while(a=="")or v[a]do t=t+1;if e==""then e="s";end a=e.."-"..tostring(t);end v[a]=true;return a;end local function x(e)if e then return string.format("%d:%d:%d",b.line[e],b.col[e],b.charpos[e]);end end local function j(e,t)if b then local t=x(t);if e.pos then e.pos[1]=t;else e.pos={t,nil};end end end local function q(e,t)if b and e.pos then local t=x(t);if e.pos then e.pos[2]=t;else e.pos={nil,t};end end end local x={heading=true,div=true,list=true,list_item=true,code_block=true,para=true,blockquote=true,table=true,thematic_break=true,raw_block=true,reference_definition=true,footnote=true};local z=nil;local function E(e)if z and x[e.t:gsub("%|.*","")]then for t=1,#z do w(e,z[t]);end if e.attr and e.attr.id then v[e.attr.id]=true;end z=nil;end end local v={};local x=0;local function T(e,t,a)local o=0;if a then while(v[t]=="blankline")or(v[t]=="-list_item")do t=t-1;end end for e=e,t do local t=v[e];if t=="blankline"then if not(string.find(v[e+1],"%+list_item")or(string.find(v[e+1],"%-list_item")and(a or string.find(v[e+2],"%-list_item"))))then o=o+1;end end end return o==0;end local function A(e,t)if(e[#e].t=="list")and not((t.t=="list_item")or(t.t=="definition_list_item"))then local t=table.remove(e);A(e,t);end if t.t=="list"then if t.pos then t.pos[2]=t.c[#t.c].pos[2];end local e=true;for a=1,#t.c do e=e and T(t.c[a].startidx,t.c[a].endidx,a==#t.c);t.c[a].startidx=nil;t.c[a].endidx=nil;end t.tight=e;g(t);end d(e[#e],t);end local function d(o,d,g,y)x=x+1;local b,T=string.match(y,"^([-+]?)(.+)");v[x]=y;if s[T]then return;end if b=="+"then local t=l(T);j(t,d);E(t);if T=="heading"then t.level=(g-d)+1;elseif e(T,"^list_item")then t.t="list_item";t.startidx=x;local e,e,e=string.find(T,"(%|.*)");local a={};if e then local t=1;for e in string.gmatch(e,"%|([^%|%]]*)")do a[e]=t;t=t+1;end end t.style_marker=e;local e=string.match(m,"^%S+",d);local i=o[#o];if i.t~="list"then local i=l("list");j(i,d);i.styles=a;i.attr=t.attr;i.startmarker=e;t.attr=nil;o[#o+1]=i;else local n={};local s=false;for e,t in pairs(a)do if i.styles[e]then s=true;n[e]=a[e];end end if s then i.styles=n;else local i=table.remove(o);A(o,i);local i=l("list");j(i,d);i.styles=a;i.attr=t.attr;i.startmarker=e;t.attr=nil;o[#o+1]=i;end end end o[#o+1]=t;elseif b=="-"then if o[#o].t=="list"then local e=table.remove(o);A(o,e);end if T==o[#o].t then local s=table.remove(o);q(s,g);if s.t=="block_attributes"then if not z then z={};end z[#z+1]=s.c;return;elseif s.t=="attributes"then local e=o[#o];local e=c(e)and e.c[#e.c];if e then local t=false;if e.t=="str"then local i=string.find(e.s,"[^%s]+$");if not i then t=true;elseif i>1 then local t=l("str");t.s=a(e.s,i,-1);e.s=a(e.s,1,i-1);A(o,t);e=t;end end if c(s)and not t then w(e,s.c);else p({message="Ignoring unattached attribute",pos=d});end else p({message="Ignoring unattached attribute",pos=d});end return;elseif T=="reference_definition"then local e="";local t;for a=1,#s.c do if s.c[a].t=="reference_key"then t=s.c[a].s;end if s.c[a].t=="reference_value"then e=e..s.c[a].s;end end i[t]=l("reference");i[t].destination=e;if s.attr then i[t].attr=s.attr;end return;elseif T=="footnote"then local e;if c(s)and(s.c[1].t=="note_label")then e=s.c[1].s;table.remove(s.c,1);end if e then n[e]=s;end return;elseif T=="table"then local t=1;local a={};while t<=#s.c do local o,i,n;if s.c[t].t=="row"then local h=s.c[t].c;for t=1,#h do o,n,i=e(h[t].t,"^separator_(.*)");if not o then break;end a[t]=i;end if o and(#a>0)then local e=s.c[t-1];if e and(e.t=="row")then e.head=true;for t=1,#e.c do e.c[t].head=true;if a[t]~="default"then e.c[t].align=a[t];end end end table.remove(s.c,t);else if#a>0 then for e=1,#s.c[t].c do if a[e]~="default"then s.c[t].c[e].align=a[e];end end end t=t+1;end end end elseif T=="code_block"then if c(s)then if s.c[1].t=="code_language"then s.lang=s.c[1].s;table.remove(s.c,1);elseif s.c[1].t=="raw_format"then local e=s.c[1].s:sub(2);table.remove(s.c,1);s.t="raw_block";s.format=e;end end s.s=h(s);s.c=nil;elseif e(T,"^list_item")then s.t="list_item";s.endidx=x;if s.style_marker=="|:"then f(s);end if(s.style_marker=="|X")and c(s)then if s.c[1].t=="checkbox_checked"then s.checkbox="checked";table.remove(s.c,1);elseif s.c[1].t=="checkbox_unchecked"then s.checkbox="unchecked";table.remove(s.c,1);end end s.style_marker=nil;elseif T=="inline_math"then s.t="math";s.s=t(s);s.c=nil;s.display=false;s.attr=u({class="math inline"});elseif T=="display_math"then s.t="math";s.s=t(s);s.c=nil;s.display=true;s.attr=u({class="math display"});elseif T=="imagetext"then s.t="image";elseif T=="linktext"then s.t="link";elseif T=="div"then s.c=s.c or{};if s.c[1]and(s.c[1].t=="class")then s.attr=u(s.attr);r(s.attr,"class",h(s.c[1]));table.remove(s.c,1);end elseif T=="verbatim"then s.s=t(s);s.c=nil;elseif T=="url"then s.destination=h(s);elseif T=="email"then s.destination="mailto:"..h(s);elseif T=="caption"then local e=o[#o];local e=c(e)and e.c[#e.c];if e and(e.t=="table")then table.insert(e.c,1,s);else p({message="Ignoring caption without preceding table",pos=d});end return;elseif T=="heading"then local e=h(s):gsub("^%s+",""):gsub("%s+$","");if not s.attr then s.attr=u({});end if not s.attr.id then r(s.attr,"id",k(e));end if not i[e]then i[e]=l("reference");i[e].destination="#"..s.attr.id;end elseif T=="destination"then local e=o[#o];local e=c(e)and e.c[#e.c];assert(e and((e.t=="image")or(e.t=="link")),"destination with no preceding link or image");e.destination=h(s):gsub("\r?\n","");return;elseif T=="reference"then local e=o[#o];local e=c(e)and e.c[#e.c];assert(e and((e.t=="image")or(e.t=="link")),"reference with no preceding link or image");if c(s)then e.reference=h(s):gsub("\r?\n"," ");else e.reference=h(e):gsub("\r?\n"," ");end return;end A(o,s);else assert(false,"unmatched "..y.." encountered at byte "..d);return;end else local e=l(T);E(e);j(e,d);q(e,g);if T=="softbreak"then e.s=nil;elseif T=="reference_key"then e.s=a(m,d+1,g-1);elseif T=="footnote_reference"then e.s=a(m,d+2,g-1);elseif T=="symbol"then e.alias=a(m,d+1,g-1);elseif T=="raw_format"then local t=o[#o];local t=c(t)and t.c[#t.c];if t and(t.t=="verbatim")then local e=h(t);t.t="raw_inline";t.s=e;t.c=nil;t.format=a(m,d+2,g-1);return;else e.s=a(m,d,g);end else e.s=a(m,d,g);end A(o,e);end end local e=l("doc");local t={e};for e,a,o in o:events()do d(t,e,a,o);end while#t>1 do local e=table.remove(t);A(t,e);if b and t[#t].pos then t[#t].pos[2]=e.pos[2];end end e=y(e);e.references=i;e.footnotes=n;return e;end local function e(t,a,n)n=n or 0;a:write(o(" ",n));if n>128 then a:write("(((DEEPLY NESTED CONTENT OMITTED)))\n");return;end if t.t then a:write(t.t);if t.pos then a:write(i(" (%s-%s)",t.pos[1],t.pos[2]));end for e,t in pairs(t)do if(type(e)=="string")and(e~="children")and(e~="tag")and(e~="pos")and(e~="attr")and(e~="references")and(e~="footnotes")then a:write(i(" %s=%q",e,tostring(t)));end end if t.attr then for e,t in pairs(t.attr)do a:write(i(" %s=%q",e,t));end end else io.stderr:write("Encountered node without tag:\n"..require("inspect")(t));os.exit(1);end a:write("\n");if t.c then for t,t in ipairs(t.c)do e(t,a,n+2);end end end local function t(t,a)e(t,a,0);if next(t.references)~=nil then a:write("references\n");for t,o in pairs(t.references)do a:write(i(" [%q] =\n",t));e(o,a,4);end end if next(t.footnotes)~=nil then a:write("footnotes\n");for t,o in pairs(t.footnotes)do a:write(i(" [%q] =\n",t));e(o,a,4);end end end return{to_ast=p,render=t,insert_attribute=r,copy_attributes=m,new_attributes=u,new_node=l,add_child=d,has_children=c};end;package.preload["djot.html"]=function()local e=require("djot.ast");local t=e.new_node;local a=e.new_attributes;local o=e.add_child;local i=unpack or table.unpack;local n,s=e.insert_attribute,e.copy_attributes;local h=string.format;local r,d=string.find,string.gsub;local function l(e)local t={};if e then for e,a in pairs(e)do local o=a;if type(a)=="table"then o=l(a);end t[e]=o;end end return t;end local function c(e)local t={};if e.t=="str"then t[#t+1]=e.s;elseif e.t=="nbsp"then t[#t+1]="\160";elseif e.t=="softbreak"then t[#t+1]=" ";elseif e.c and(#e.c>0)then for a=1,#e.c do t[#t+1]=c(e.c[a]);end end return table.concat(t);end local u={};function u.new(e)local t={out=function(e)io.stdout:write(e);end,tight=false,footnote_index={},next_footnote_index=1,references=nil,footnotes=nil};setmetatable(t,e);e.__index=e;return t;end u.html_escapes={["<"]="<",[">"]=">",["&"]="&",['"']="""};function u.escape_html(e,t)if r(t,"[<>&]")then return(d(t,"[<>&]",e.html_escapes));else return t;end end function u.escape_html_attribute(e,t)if r(t,'[<>&"]')then return(d(t,'[<>&"]',e.html_escapes));else return t;end end function u.render(e,t,a)e.references=t.references;e.footnotes=t.footnotes;if a then function e.out(e)a:write(e);end end e[t.t](e,t);end function u.render_children(e,t)local t,a=pcall(function()if t.c and(#t.c>0)then local a;if t.tight~=nil then a=e.tight;e.tight=t.tight;end for a=1,#t.c do e[t.c[a].t](e,t.c[a]);end if t.tight~=nil then e.tight=a;end end end);if not t and a:find("stack overflow")then e.out("(((DEEPLY NESTED CONTENT OMITTED)))\n");end end function u.render_attrs(e,t)if t.attr then for t,a in pairs(t.attr)do e.out(" "..t.."="..'"'..e:escape_html_attribute(a)..'"');end end if t.pos then local t,a=i(t.pos);e.out(' data-startpos="'..tostring(t)..'" data-endpos="'..tostring(a)..'"');end end function u.render_tag(e,t,a)e.out("<"..t);e:render_attrs(a);e.out(">");end function u.add_backlink(a,a,i)local n=t("link");n.destination="#fnref"..tostring(i);n.attr=e.new_attributes({role="doc-backlink"});local e=t("str");e.s="↩︎︎";o(n,e);if a.c[#a.c].t=="para"then o(a.c[#a.c],n);else local e=t("para");o(e,n);o(a,e);end end function u.doc(e,t)e:render_children(t);if e.next_footnote_index>1 then local t={};for a,o in pairs(e.footnotes)do if e.footnote_index[a]then t[e.footnote_index[a]]=o;end end e.out('
\n
\n
    \n');for a=1,#t do local t=t[a];if t then e.out(h('
  1. \n',a));e:add_backlink(t,a);e:render_children(t);e.out("
    2. \n");end end e.out("
\n
\n");end end function u.raw_block(e,t)if t.format=="html"then e.out(t.s);end end function u.para(e,t)if not e.tight then e:render_tag("p",t);end e:render_children(t);if not e.tight then e.out("

");end e.out("\n");end function u.blockquote(e,t)e:render_tag("blockquote",t);e.out("\n");e:render_children(t);e.out("\n");end function u.div(e,t)e:render_tag("div",t);e.out("\n");e:render_children(t);e.out("\n");end function u.section(e,t)e:render_tag("section",t);e.out("\n");e:render_children(t);e.out("\n");end function u.heading(e,t)e:render_tag("h"..t.level,t);e:render_children(t);e.out("\n");end function u.thematic_break(e,t)e:render_tag("hr",t);e.out("\n");end function u.code_block(e,t)e:render_tag("pre",t);e.out("0)then e.out(' class="language-'..t.lang..'"');end e.out(">");e.out(e:escape_html(t.s));e.out("\n");end function u.table(e,t)e:render_tag("table",t);e.out("\n");e:render_children(t);e.out("\n");end function u.row(e,t)e:render_tag("tr",t);e.out("\n");e:render_children(t);e.out("\n");end function u.cell(e,t)local a;if t.head then a="th";else a="td";end local o=l(t.attr);if t.align then n(o,"style","text-align: "..t.align..";");end e:render_tag(a,{attr=o});e:render_children(t);e.out("\n");end function u.caption(e,t)e:render_tag("caption",t);e:render_children(t);e.out("\n");end function u.list(e,t)local a=t.style;if(a=="*")or(a=="+")or(a=="-")then e:render_tag("ul",t);e.out("\n");e:render_children(t);e.out("\n");elseif a=="X"then local a=l(t.attr);if a.class then a.class="task-list "..a.class;else n(a,"class","task-list");end e:render_tag("ul",{attr=a});e.out("\n");e:render_children(t);e.out("\n");elseif a==":"then e:render_tag("dl",t);e.out("\n");e:render_children(t);e.out("\n");else e.out("1)then e.out(' start="'..t.start..'"');end local a=d(t.style,"%p","");if a~="1"then e.out(' type="'..a..'"');end e:render_attrs(t);e.out(">\n");e:render_children(t);e.out("\n");end end function u.list_item(e,t)if t.checkbox then if t.checkbox=="checked"then e.out('
  • ');elseif t.checkbox=="unchecked"then e.out('
  • ');end else e:render_tag("li",t);end e.out("\n");e:render_children(t);e.out("
    • \n");end function u.term(e,t)e:render_tag("dt",t);e:render_children(t);e.out("\n");end function u.definition(e,t)e:render_tag("dd",t);e.out("\n");e:render_children(t);e.out("\n");end function u.definition_list_item(e,t)e:render_children(t);end function u.reference_definition(e)end function u.footnote_reference(e,t)local t=t.s;local a=e.footnote_index[t];if not a then a=e.next_footnote_index;e.footnote_index[t]=a;e.next_footnote_index=e.next_footnote_index+1;end e.out(h('    %d',a,a,a));end function u.raw_inline(e,t)if t.format=="html"then e.out(t.s);end end function u.str(e,t)if t.attr then e:render_tag("span",t);e.out(e:escape_html(t.s));e.out("");else e.out(e:escape_html(t.s));end end function u.softbreak(e)e.out("\n");end function u.hardbreak(e)e.out("
    \n");end function u.nbsp(e)e.out(" ");end function u.verbatim(e,t)e:render_tag("code",t);e.out(e:escape_html(t.s));e.out("");end function u.link(e,t)local a=a({});if t.reference then local e=e.references[t.reference];if e then if e.attr then s(a,e.attr);end n(a,"href",e.destination);end elseif t.destination then n(a,"href",t.destination);end s(a,t.attr);e:render_tag("a",{attr=a});e:render_children(t);e.out("");end u.url=u.link;u.email=u.link;function u.image(e,t)local a=a({});local o=c(t);if#o>0 then n(a,"alt",c(t));end if t.reference then local e=e.references[t.reference];if e then if e.attr then s(a,e.attr);end n(a,"src",e.destination);end elseif t.destination then n(a,"src",t.destination);end s(a,t.attr);e:render_tag("img",{attr=a});end function u.span(e,t)e:render_tag("span",t);e:render_children(t);e.out("");end function u.mark(e,t)e:render_tag("mark",t);e:render_children(t);e.out("");end function u.insert(e,t)e:render_tag("ins",t);e:render_children(t);e.out("");end function u.delete(e,t)e:render_tag("del",t);e:render_children(t);e.out("");end function u.subscript(e,t)e:render_tag("sub",t);e:render_children(t);e.out("");end function u.superscript(e,t)e:render_tag("sup",t);e:render_children(t);e.out("");end function u.emph(e,t)e:render_tag("em",t);e:render_children(t);e.out("");end function u.strong(e,t)e:render_tag("strong",t);e:render_children(t);e.out("");end function u.double_quoted(e,t)e.out("“");e:render_children(t);e.out("”");end function u.single_quoted(e,t)e.out("‘");e:render_children(t);e.out("’");end function u.left_double_quote(e)e.out("“");end function u.right_double_quote(e)e.out("”");end function u.left_single_quote(e)e.out("‘");end function u.right_single_quote(e)e.out("’");end function u.ellipses(e)e.out("…");end function u.em_dash(e)e.out("—");end function u.en_dash(e)e.out("–");end function u.symbol(e,t)e.out(":"..t.alias..":");end function u.math(e,t)local a="inline";if r(t.attr.class,"display")then a="display";end e:render_tag("span",t);if a=="inline"then e.out("\\(");else e.out("\\[");end e.out(e:escape_html(t.s));if a=="inline"then e.out("\\)");else e.out("\\]");end e.out("");end return{Renderer=u};end;package.preload["djot.filter"]=function()local function e(t,a)local o=a[t.t];local i,n;if type(o)=="table"then i=o.enter;n=o.exit;elseif type(o)=="function"then n=o;end if i then local e=i(t);if e then return;end end if t.c then for t,t in ipairs(t.c)do e(t,a);end end if t.footnotes then for t,t in pairs(t.footnotes)do e(t,a);end end if n then n(t);end end local function t(t,a)e(t,a);return t;end local function e(e,a)for a,a in ipairs(a)do t(e,a);end end local function t(e)local t=package.path;local e,t=pcall(function()package.path="./?.lua;"..package.path;local e=require(e:gsub("%.lua$",""));package.path=t;return e;end);if not e then return nil,t;elseif type(t)~="table"then return nil,"filter must be a table";end if#t==0 then return{t};else return t;end end local function a(e)local t,a;if _VERSION:match("5.1")then t,a=loadstring(e);else t,a=load(e);end if t then local e=t();if type(e)~="table"then return nil,"filter must be a table";end if#e==0 then return{e};else return e;end else return nil,a;end end return{apply_filter=e,require_filter=t,load_filter=a};end;package.preload["djot.json"]=function()local e={_version="0.1.2"};local t;local a={["\\"]="\\",['"']='"',["\b"]="b",["\f"]="f",["\n"]="n",["\r"]="r",["\t"]="t"};local o={["/"]="/"};for e,t in pairs(a)do o[t]=e;end local function o(e)return"\\"..(a[e]or string.format("u%04x",e:byte()));end local function a(e)return"null";end local function i(e,a)local o={};a=a or{};if a[e]then error("circular reference");end a[e]=true;if(rawget(e,1)~=nil)or(next(e)==nil)then local i=0;for e in pairs(e)do if type(e)~="number"then error("invalid table: mixed or invalid key types");end i=i+1;end if i~=#e then error("invalid table: sparse array");end for e,e in ipairs(e)do table.insert(o,t(e,a));end a[e]=nil;return"["..table.concat(o,",").."]";else for e,i in pairs(e)do if type(e)~="string"then error("invalid table: mixed or invalid key types");end table.insert(o,t(e,a)..":"..t(i,a));end a[e]=nil;return"{"..table.concat(o,",").."}";end end local function n(e)return'"'..e:gsub('[%z\1-\31\\"]',o)..'"';end local function o(e)if(e~=e)or(e<=-math.huge)or(e>=math.huge)then error("unexpected number value '"..tostring(e).."'");end return string.format("%.14g",e);end local a={["nil"]=a,table=i,string=n,number=o,boolean=tostring};function t(e,t)local o=type(e);local a=a[o];if a then return a(e,t);end error("unexpected type '"..o.."'");end function e.encode(e)return(t(e));end return e;end;function package.preload.djot()local e=unpack or table.unpack;local e=require("djot.block").Parser;local t=require("djot.ast");local a=require("djot.html");local o=require("djot.json");local i=require("djot.filter");local i={};function i.new(e)local e={};setmetatable(e,i);i.__index=i;return e;end function i.write(e,t)e[#e+1]=t;end function i.flush(e)return table.concat(e);end local function n(a,o,i)local e=e:new(a,i);return t.to_ast(e,o);end local function s(t,a)return e:new(t):events();end local function e(e)local a=i:new();t.render(e,a);return a:flush();end local function t(e)return o.encode(e).."\n";end local function o(e)local t=i:new();local a=a.Renderer:new();a:render(e,t);return t:flush();end local function a(e,t,a)return string.format("[%q,%d,%d]",a,e,t);end local function h(e,t)local o=i:new();local i=0;for e,t,n in s(e,t)do i=i+1;if i==1 then o:write("[");else o:write(",");end o:write(a(e,t,n).."\n");end o:write("]\n");return o:flush();end local i="0.2.1";local e={parse=n,parse_events=s,parse_and_render_events=h,render_html=o,render_ast_pretty=e,render_ast_json=t,render_event=a,version=i};setmetatable(e,{__index=function(e,t)local a=require("djot."..t);rawset(e,t,a);return e[t];end});return e;end local e=require("djot");return e; \ No newline at end of file diff --git a/clib/dumbParser.lua b/clib/dumbParser.lua new file mode 100644 index 0000000..5714f71 --- /dev/null +++ b/clib/dumbParser.lua @@ -0,0 +1,6938 @@ +--[=[=========================================================== +--= +--= Dumb Lua Parser - Lua parsing library +--= by Marcus 'ReFreezed' Thunström +--= +--= Tokenize Lua code or create ASTs (Abstract Syntax Trees) +--= and convert the data back to Lua. +--= +--= Version: 2.3 (2022-06-23) +--= +--= License: MIT (see the bottom of this file) +--= Website: http://refreezed.com/luaparser/ +--= Documentation: http://refreezed.com/luaparser/docs/ +--= +--= Supported Lua versions: 5.1, 5.2, 5.3, 5.4, LuaJIT +--= +--============================================================== + +1 - Usage +2 - API + 2.1 - Functions + 2.2 - Constants + 2.3 - Settings +3 - Tokens +4 - AST +5 - Other Objects + 5.1 - Stats + 5.2 - Locations +6 - Notes + + +1 - Usage +================================================================ + +local parser = require("dumbParser") + +local tokens = parser.tokenizeFile("cool.lua") +local ast = parser.parse(tokens) + +parser.simplify(ast) +parser.printTree(ast) + +local lua = parser.toLua(ast, true) +print(lua) + + +2 - API +================================================================ + + +2.1 - Functions +---------------------------------------------------------------- + +tokenize, tokenizeFile +newToken, updateToken, cloneToken, concatTokens +parse, parseExpression, parseFile +newNode, newNodeFast, valueToAst, cloneNode, cloneTree, getChild, setChild, addChild, removeChild +validateTree +traverseTree, traverseTreeReverse +updateReferences +simplify, optimize, minify +toLua +printTokens, printNode, printTree +formatMessage +findDeclaredNames, findGlobalReferences, findShadows + +tokenize() + tokens = parser.tokenize( luaString [, pathForErrorMessages="?" ] ) + tokens = parser.tokenize( luaString [, keepWhitespaceTokens=false, pathForErrorMessages="?" ] ) + Convert a Lua string into an array of tokens. (See below for more info.) + Returns nil and a message on error. + +tokenizeFile() + tokens = parser.tokenizeFile( path [, keepWhitespaceTokens=false ] ) + Convert the contents of a file into an array of tokens. (See below for more info.) Uses io.open(). + Returns nil and a message on error. + +newToken() + token = parser.newToken( tokenType, tokenValue ) + Create a new token. (See below or search for 'TokenCreation' for more info.) + +updateToken() + parser.updateToken( token, tokenValue ) + Update the value and representation of an existing token. (Search for 'TokenModification' for more info.) + +cloneToken() + tokenClone = parser.cloneToken( token ) + Clone an existing token. + +concatTokens() + luaString = parser.concatTokens( tokens ) + Concatenate tokens. Whitespace is added between tokens when necessary. + +parse() + astNode = parser.parse( tokens ) + astNode = parser.parse( luaString [, pathForErrorMessages="?" ] ) + Convert tokens or Lua code into an AST representing a block of code. (See below for more info.) + Returns nil and a message on error. + +parseExpression() + astNode = parser.parseExpression( tokens ) + astNode = parser.parseExpression( luaString [, pathForErrorMessages="?" ] ) + Convert tokens or Lua code into an AST representing a value expression. (See below for more info.) + Returns nil and a message on error. + +parseFile() + astNode = parser.parseFile( path ) + Convert a Lua file into an AST. (See below for more info.) Uses io.open(). + Returns nil and a message on error. + +newNode() + astNode = parser.newNode( nodeType, arguments... ) + Create a new AST node. (Search for 'NodeCreation' for more info.) + +newNodeFast() + astNode = parser.newNodeFast( nodeType, arguments... ) + Same as newNode() but without any validation. (Search for 'NodeCreation' for more info.) + +valueToAst() + astNode = parser.valueToAst( value [, sortTableKeys=false ] ) + Convert a Lua value (number, string, boolean, nil or table) to an AST. + +cloneNode() + astNode = parser.cloneNode( astNode ) + Clone an existing AST node (but not any children). + +cloneTree() + astNode = parser.cloneTree( astNode ) + Clone an existing AST node and its children. + +getChild() + childNode = parser.getChild( astNode, fieldName ) + childNode = parser.getChild( astNode, fieldName, index ) -- If the node field is an array. + childNode = parser.getChild( astNode, fieldName, index, tableFieldKey ) -- If the node field is a table field array. + tableFieldKey = "key"|"value" + Get a child node. (Search for 'NodeFields' for field names.) + + The result is the same as doing the following, but with more error checking: + childNode = astNode[fieldName] + childNode = astNode[fieldName][index] + childNode = astNode[fieldName][index][tableFieldKey] + +setChild() + parser.setChild( astNode, fieldName, childNode ) + parser.setChild( astNode, fieldName, index, childNode ) -- If the node field is an array. + parser.setChild( astNode, fieldName, index, tableFieldKey, childNode ) -- If the node field is a table field array. + tableFieldKey = "key"|"value" + Set a child node. (Search for 'NodeFields' for field names.) + + The result is the same as doing the following, but with more error checking: + astNode[fieldName] = childNode + astNode[fieldName][index] = childNode + astNode[fieldName][index][tableFieldKey] = childNode + +addChild() + parser.addChild( astNode, fieldName, [ index=atEnd, ] childNode ) + parser.addChild( astNode, fieldName, [ index=atEnd, ] keyNode, valueNode ) -- If the node field is a table field array. + Add a child node to an array field. (Search for 'NodeFields' for field names.) + + The result is the same as doing the following, but with more error checking: + table.insert(astNode[fieldName], index, childNode) + table.insert(astNode[fieldName], index, {key=keyNode, value=valueNode, generatedKey=false}) + +removeChild() + parser.removeChild( astNode, fieldName [, index=last ] ) + Remove a child node from an array field. (Search for 'NodeFields' for field names.) + + The result is the same as doing the following, but with more error checking: + table.remove(astNode[fieldName], index) + +isExpression() + bool = parser.isExpression( astNode ) + Returns true for expression nodes and false for statements. + Note that call nodes count as expressions for this function, i.e. return true. + +isStatement() + bool = parser.isStatement( astNode ) + Returns true for statements and false for expression nodes. + Note that call nodes count as statements for this function, i.e. return true. + +validateTree() + isValid, errorMessages = parser.validateTree( astNode ) + Check for errors in an AST (e.g. missing condition expressions for if statements). + errorMessages is a multi-line string if isValid is false. + +traverseTree() + didStop = parser.traverseTree( astNode, [ leavesFirst=false, ] callback [, topNodeParent=nil, topNodeContainer=nil, topNodeKey=nil ] ) + action = callback( astNode, parent, container, key ) + action = "stop"|"ignorechildren"|nil -- Returning nil (or nothing) means continue traversal. + Call a function on all nodes in an AST, going from astNode out to the leaf nodes (or from leaf nodes and inwards if leavesFirst is set). + container[key] is the position of the current node in the tree and can be used to replace the node. + +traverseTreeReverse() + didStop = parser.traverseTreeReverse( astNode, [ leavesFirst=false, ] callback [, topNodeParent=nil, topNodeContainer=nil, topNodeKey=nil ] ) + action = callback( astNode, parent, container, key ) + action = "stop"|"ignorechildren"|nil -- Returning nil (or nothing) means continue traversal. + Call a function on all nodes in reverse order in an AST, going from astNode out to the leaf nodes (or from leaf nodes and inwards if leavesFirst is set). + container[key] is the position of the current node in the tree and can be used to replace the node. + +updateReferences() + parser.updateReferences( astNode [, updateTopNodePositionInfo=true ] ) + Update references between nodes in the tree. + This function sets 'parent'+'container'+'key' for all nodes, 'declaration' for identifiers and vararg nodes, and 'label' for goto nodes. + If 'updateTopNodePositionInfo' is false then 'parent', 'container' and 'key' will remain as-is for 'astNode' specifically. + +simplify() + stats = parser.simplify( astNode ) + Simplify/fold expressions and statements involving constants ('1+2' becomes '3', 'false and func()' becomes 'false' etc.). + See the INT_SIZE constant for notes. + See below for more info about stats. + +optimize() + stats = parser.optimize( astNode ) + Attempt to remove nodes that aren't useful, like unused variables, or variables that are essentially constants. + Calls simplify() internally. + This function can be quite slow! + See below for more info about stats. + Note: References may be out-of-date after calling this. + +minify() + stats = parser.minify( astNode [, optimize=false ] ) + Replace local variable names with short names. + This function can be used to obfuscate the code to some extent. + If 'optimize' is set then optimize() is also called automatically. + See below for more info about stats. + Note: References may be out-of-date after calling this. + +toLua() + luaString = parser.toLua( astNode [, prettyOuput=false, nodeCallback ] ) + nodeCallback = function( node, outputBuffer ) + Convert an AST to Lua, optionally call a function on each node before they are turned into Lua. + Any node in the tree with a .pretty attribute will override the 'prettyOuput' flag for itself and its children. + Nodes can also have a .prefix and/or .suffix attribute with Lua code to output before/after the node (e.g. declaration.names[1].suffix="--[[foo]]"). + outputBuffer is an array of Lua code that has been output so far. + Returns nil and a message on error. + +printTokens() + parser.printTokens( tokens ) + Print tokens to stdout. + +printNode() + parser.printNode( astNode ) + Print information about an AST node to stdout. + +printTree() + parser.printTree( astNode ) + Print the structure of a whole AST to stdout. + +formatMessage() + message = parser.formatMessage( [ prefix="Info", ] token, formatString, ... ) + message = parser.formatMessage( [ prefix="Info", ] astNode, formatString, ... ) + message = parser.formatMessage( [ prefix="Info", ] location, formatString, ... ) + Format a message to contain a code preview window with an arrow pointing at the target token, node or location. + This is used internally for formatting error messages. + + -- Example: + if identifier.name ~= "good" then + print(parser.formatMessage("Error", identifier, "This identifier is not good!")) + print(parser.formatMessage(currentStatement, "Current statement.")) + end + +findDeclaredNames() + identifiers = parser.findDeclaredNames( astNode ) + Find all declared names in the tree (i.e. identifiers from AstDeclaration, AstFunction and AstFor nodes). + +findGlobalReferences() + identifiers = parser.findGlobalReferences( astNode ) + Find all identifiers not referring to local variables in the tree. + Note: updateReferences() must be called at some point before you call this - otherwise all variables will be seen as globals! + +findShadows() + shadowSequences = parser.findShadows( astNode ) + shadowSequences = { shadowSequence1, ... } + shadowSequence = { shadowingIdentifier, shadowedIdentifier1, ... } + Find local variable shadowing in the tree. Each shadowSequence is an array of declared identifiers where each identifier shadows the next one. + Note: updateReferences() must be called at some point before you call this - otherwise all variables will be seen as globals! + Note: Shadowing of globals cannot be detected by the function as that would require knowledge of all potential globals in your program. (See findGlobalReferences()) + + +2.2 - Constants +---------------------------------------------------------------- + +INT_SIZE, MAX_INT, MIN_INT +VERSION + +INT_SIZE + parser.INT_SIZE = integer + How many bits integers have. In Lua 5.3 and later this is usually 64, and in earlier versions it's 32. + The int size may affect how bitwise operations involving only constants get simplified (see simplify()), + e.g. the expression '-1>>1' becomes 2147483647 in Lua 5.2 but 9223372036854775807 in Lua 5.3. + +MAX_INT + parser.MAX_INT = integer + The highest representable positive signed integer value, according to INT_SIZE. + This is the same value as math.maxinteger in Lua 5.3 and later. + This only affects simplification of some bitwise operations. + +MIN_INT + parser.MIN_INT = integer + The highest representable negative signed integer value, according to INT_SIZE. + This is the same value as math.mininteger in Lua 5.3 and later. + This only affects simplification of some bitwise operations. + +VERSION + parser.VERSION + The parser's version number (e.g. "1.0.2"). + + +2.3 - Settings +---------------------------------------------------------------- + +printIds, printLocations +indentation +constantNameReplacementStringMaxLength + +printIds + parser.printIds = bool + If AST node IDs should be printed. (All nodes gets assigned a unique ID when created.) + Default: false. + +printLocations + parser.printLocations = bool + If the file location (filename and line number) should be printed for each token or AST node. + Default: false. + +indentation + parser.indentation = string + The indentation used when printing ASTs (with printTree()). + Default: 4 spaces. + +constantNameReplacementStringMaxLength + parser.constantNameReplacementStringMaxLength = length + Normally optimize() replaces variable names that are effectively constants with their value. + The exception is if the value is a string that's longer than what this setting specifies. + Default: 200. + + -- Example: + local ast = parser.parse[==[ + local short = "a" + local long = "xy" + func(short, long) + ]==] + parser.constantNameReplacementStringMaxLength = 1 + parser.optimize(ast) + print(parser.toLua(ast)) -- local long="xy";func("a",long); + + +3 - Tokens +================================================================ + +Tokens are represented by tables. + +Token fields: + + type -- Token type. (See below.) + value -- Token value. All token types have a string value, except "number" tokens which have a number value. + representation -- The token's code representation. (Strings have surrounding quotes, comments start with "--" etc.) + + sourceString -- The original source string, or "" if there is none. + sourcePath -- Path to the source file, or "?" if there is none. + + lineStart -- Start line number in sourceString, or 0 by default. + lineEnd -- End line number in sourceString, or 0 by default. + positionStart -- Start byte position in sourceString, or 0 by default. + positionEnd -- End byte position in sourceString, or 0 by default. + +Token types: + + "comment" -- A comment. + "identifier" -- Word that is not a keyword. + "keyword" -- Lua keyword. + "number" -- Number literal. + "punctuation" -- Any punctuation, e.g. ".." or "(". + "string" -- String value. + "whitespace" -- Sequence of whitespace characters. + + +4 - AST +================================================================ + +AST nodes are represented by tables. + +Node types: + + "assignment" -- Assignment of one or more values to one or more variables. + "binary" -- Binary expression (operation with two operands, e.g. "+" or "and"). + "block" -- List of statements. Blocks inside blocks are 'do...end' statements. + "break" -- Loop break statement. + "call" -- Function call. + "declaration" -- Declaration of one or more local variables, possibly with initial values. + "for" -- A 'for' loop. + "function" -- Anonymous function header and body. + "goto" -- A jump to a label. + "identifier" -- An identifier. + "if" -- If statement with a condition, a body if the condition is true, and possibly another body if the condition is false. + "label" -- Label for goto commands. + "literal" -- Number, string, boolean or nil literal. + "lookup" -- Field lookup on an object. + "repeat" -- A 'repeat' loop. + "return" -- Function/chunk return statement, possibly with values. + "table" -- Table constructor. + "unary" -- Unary expression (operation with one operand, e.g. "-" or "not"). + "vararg" -- Vararg expression ("..."). + "while" -- A 'while' loop. + +Node fields: (Search for 'NodeFields'.) + + +5 - Other Objects +================================================================ + + +5.1 - Stats +---------------------------------------------------------------- + +Some functions return a stats table which contains these fields: + + nodeReplacements -- Array of locations. Locations of nodes that were replaced. (See below for location info.) + nodeRemovals -- Array of locations. Locations of nodes or tree branches that were removed. (See below for location info.) + nodeRemoveCount -- Number. How many nodes were removed, including subnodes of nodeRemovals. + + renameCount -- Number. How many identifiers were renamed. + generatedNameCount -- Number. How many unique names were generated. + + +5.2 - Locations +---------------------------------------------------------------- + +Locations are tables with these fields: + + sourceString -- The original source string, or "" if there is none. + sourcePath -- Path to the source file, or "?" if there is none. + + line -- Line number in sourceString, or 0 by default. + position -- Byte position in sourceString, or 0 by default. + + node -- The node the location points to, or nil if there is none. + replacement -- The node that replaced 'node', or nil if there is none. (This is set for stats.nodeReplacements.) + + +6 - Notes +================================================================ + +Special number notation rules. + + The expression '-n' is parsed as a single number literal if 'n' is a + numeral (i.e. the result is a negative number). + + The expression 'n/0' is parsed as a single number literal if 'n' is a + numeral. If 'n' is positive then the result is math.huge, if 'n' is + negative then the result is -math.huge, or if 'n' is 0 then the result is + NaN. + + +-============================================================]=] + +local PARSER_VERSION = "2.3.0" + +local NORMALIZE_MINUS_ZERO, HANDLE_ENV -- Should HANDLE_ENV be a setting? +do + local n = 0 + NORMALIZE_MINUS_ZERO = tostring(-n) == "0" -- Lua 5.3+ normalizes -0 to 0. +end +do + local pcall = pcall + local _ENV = nil + HANDLE_ENV = not pcall(function() return _G end) -- Looking up the global _G will raise an error if _ENV is supported (Lua 5.2+). +end + +local assert = assert +local error = error +local ipairs = ipairs +local loadstring = loadstring or load +local pairs = pairs +local print = print +local select = select +local tonumber = tonumber +local tostring = tostring +local type = type + +local ioOpen = io.open +local ioWrite = io.write + +local jit = jit + +local mathFloor = math.floor +local mathMax = math.max +local mathMin = math.min +local mathType = math.type -- May be nil. + +local F = string.format +local stringByte = string.byte +local stringChar = string.char +local stringFind = string.find +local stringGmatch = string.gmatch +local stringGsub = string.gsub +local stringMatch = string.match +local stringRep = string.rep +local stringSub = string.sub + +local tableConcat = table.concat +local tableInsert = table.insert +local tableRemove = table.remove +local tableSort = table.sort +local tableUnpack = table.unpack or unpack + +local maybeWrapInt = ( + (jit and function(n) + -- 'n' might be cdata (i.e. a 64-bit integer) here. We have to limit the range + -- with mod once before we convert it to a Lua number to not lose precision, + -- but the result might be negative (and still out of range somehow!) so we + -- have to use mod again. Gah! + return tonumber(n % 0x100000000) % 0x100000000 -- 0x100000000 == 2^32 + end) + or (_VERSION == "Lua 5.2" and require"bit32".band) + or function(n) return n end +) + +local parser + + + +local function newSet(values) + local set = {} + for _, v in ipairs(values) do + set[v] = true + end + return set +end +local function newCharSet(chars) + return newSet{ stringByte(chars, 1, #chars) } +end + +local KEYWORDS = newSet{ + "and", "break", "do", "else", "elseif", "end", "false", "for", "function", "goto", "if", + "in", "local", "nil", "not", "or", "repeat", "return", "then", "true", "until", "while", +} +local PUNCTUATION = newSet{ + "+", "-", "*", "/", "%", "^", "#", + "&", "~", "|", "<<", ">>", "//", + "==", "~=", "<=", ">=", "<", ">", "=", + "(", ")", "{", "}", "[", "]", "::", + ";", ":", ",", ".", "..", "...", +} +local OPERATORS_UNARY = newSet{ + "-", "not", "#", "~", +} +local OPERATORS_BINARY = newSet{ + "+", "-", "*", "/", "//", "^", "%", + "&", "~", "|", ">>", "<<", "..", + "<", "<=", ">", ">=", "==", "~=", + "and", "or", +} +local OPERATOR_PRECEDENCE = { + ["or"] = 1, + ["and"] = 2, + ["<"] = 3, [">"] = 3, ["<="] = 3, [">="] = 3, ["~="] = 3, ["=="] = 3, + ["|"] = 4, + ["~"] = 5, + ["&"] = 6, + ["<<"] = 7, [">>"] = 7, + [".."] = 8, + ["+"] = 9, ["-"] = 9, + ["*"] = 10, ["/"] = 10, ["//"] = 10, ["%"] = 10, + unary = 11, -- "-", "not", "#", "~" + ["^"] = 12, +} + +local EXPRESSION_NODES = newSet{ "binary", "call", "function", "identifier", "literal", "lookup", "table", "unary", "vararg" } +local STATEMENT_NODES = newSet{ "assignment", "block", "break", "call", "declaration", "for", "goto", "if", "label", "repeat", "return", "while" } + +local TOKEN_BYTES = { + NAME_START = newCharSet"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_", + DASH = newCharSet"-", + NUM = newCharSet"0123456789", + QUOTE = newCharSet"\"'", + SQUARE = newCharSet"[", + DOT = newCharSet".", + PUNCT_TWO_CHARS = newCharSet".=~<>:/<>", + PUNCT_ONE_CHAR = newCharSet"+-*/%^#<>=(){}[];:,.&~|", +} + +local NUMERAL_PATTERNS = { + HEX_FRAC_EXP = stringGsub("^( 0[Xx] (%x*) %.(%x+) [Pp]([-+]?%x+) )", " +", ""), + HEX_FRAC = stringGsub("^( 0[Xx] (%x*) %.(%x+) )", " +", ""), + HEX_EXP = stringGsub("^( 0[Xx] (%x+) %.? [Pp]([-+]?%x+) )", " +", ""), + HEX = stringGsub("^( 0[Xx] %x+ %.? )", " +", ""), + BIN = stringGsub("^( 0[Bb] [01]+ )", " +", ""), + DEC_FRAC_EXP = stringGsub("^( %d* %.%d+ [Ee][-+]?%d+ )", " +", ""), + DEC_FRAC = stringGsub("^( %d* %.%d+ )", " +", ""), + DEC_EXP = stringGsub("^( %d+ %.? [Ee][-+]?%d+ )", " +", ""), + DEC = stringGsub("^( %d+ %.? )", " +", ""), +} + +local INT_SIZE, MAX_INT, MIN_INT +do + local hex = F("%x", maybeWrapInt(-1)) + INT_SIZE = #hex * 4 -- This should generally be 64 for Lua 5.3+ and 32 for earlier. + MAX_INT = math.maxinteger or tonumber(stringGsub(hex, "f", "7", 1), 16) + MIN_INT = math.mininteger or -MAX_INT-1 +end + +local nextSerialNumber = 1 + + + +-- :NodeFields + +local function populateCommonNodeFields(token, node) + -- All nodes have these fields. + node.id = nextSerialNumber + nextSerialNumber = nextSerialNumber + 1 + + node.sourceString = token and token.sourceString or "" + node.sourcePath = token and token.sourcePath or "?" + + node.token = token + node.line = token and token.lineStart or 0 + node.position = token and token.positionStart or 0 + + -- These fields are set by updateReferences(): + -- node.parent = nil -- Refers to the node's parent in the tree. + -- node.container = nil -- Refers to the specific table that the node is in, which could be the parent itself or a field in the parent. + -- node.key = nil -- Refers to the specific field in the container that the node is in (which is either a string or an integer). + + -- toLua() uses these fields if present: + -- node.pretty = bool + -- node.prefix = luaString + -- node.suffix = luaString + + return node +end + +-- AST expressions. +local function AstIdentifier (token,name)return populateCommonNodeFields(token,{ + type = "identifier", + name = name, -- String. + attribute = "", -- "" | "close" | "const" -- Only used in declarations. + declaration = nil, -- AstIdentifier (whose parent is an AstDeclaration, AstFunction or AstFor). Updated by updateReferences(). This is nil for globals. +})end +local function AstVararg (token)return populateCommonNodeFields(token,{ + type = "vararg", + declaration = nil, -- AstVararg (whose parent is an AstFunction). Updated by updateReferences(). This is nil in the main chunk (or in a non-vararg function, which is probably an error). + adjustToOne = false, -- True if parentheses surround the vararg. +})end +local function AstLiteral (token,v)return populateCommonNodeFields(token,{ + type = "literal", + value = v, -- Number, string, boolean or nil. +})end +local function AstTable (token)return populateCommonNodeFields(token,{ + type = "table", + fields = {}, -- Array of {key=expression, value=expression, generatedKey=bool}. generatedKey is true for implicit keys (i.e. {x,y}) and false for explicit keys (i.e. {a=x,b=y}). Note that the state of generatedKey affects the output of toLua()! 'key' may be nil if generatedKey is true. +})end +local function AstLookup (token)return populateCommonNodeFields(token,{ + type = "lookup", + object = nil, -- Expression. + member = nil, -- Expression. +})end +local function AstUnary (token,op)return populateCommonNodeFields(token,{ + type = "unary", + operator = op, -- "-" | "not" | "#" | "~" + expression = nil, -- Expression. +})end +local function AstBinary (token,op)return populateCommonNodeFields(token,{ + type = "binary", + operator = op, -- "+" | "-" | "*" | "/" | "//" | "^" | "%" | "&" | "~" | "|" | ">>" | "<<" | ".." | "<" | "<=" | ">" | ">=" | "==" | "~=" | "and" | "or" + left = nil, -- Expression. + right = nil, -- Expression. +})end +local function AstCall (token)return populateCommonNodeFields(token,{ -- Calls can be both expressions and statements. + type = "call", + callee = nil, -- Expression. + arguments = {}, -- Array of expressions. + method = false, -- True if the call is a method call. Method calls must have a callee that is a lookup with a member expression that is a string literal that can pass as an identifier. + adjustToOne = false, -- True if parentheses surround the call. +})end +local function AstFunction (token)return populateCommonNodeFields(token,{ + type = "function", + parameters = {}, -- Array of AstIdentifier and maybe an AstVararg at the end. + body = nil, -- AstBlock. +})end + +-- AST statements. +local function AstBreak (token)return populateCommonNodeFields(token,{ + type = "break", +})end +local function AstReturn (token)return populateCommonNodeFields(token,{ + type = "return", + values = {}, -- Array of expressions. +})end +local function AstLabel (token,name)return populateCommonNodeFields(token,{ + type = "label", + name = name, -- String. The value must be able to pass as an identifier. +})end +local function AstGoto (token,name)return populateCommonNodeFields(token,{ + type = "goto", + name = name, -- String. The value must be able to pass as an identifier. + label = nil, -- AstLabel. Updated by updateReferences(). +})end +local function AstBlock (token)return populateCommonNodeFields(token,{ + type = "block", + statements = {}, -- Array of statements. +})end +local function AstDeclaration (token)return populateCommonNodeFields(token,{ + type = "declaration", + names = {}, -- Array of AstIdentifier. + values = {}, -- Array of expressions. +})end +local function AstAssignment (token)return populateCommonNodeFields(token,{ + type = "assignment", + targets = {}, -- Mixed array of AstIdentifier and AstLookup. + values = {}, -- Array of expressions. +})end +local function AstIf (token)return populateCommonNodeFields(token,{ + type = "if", + condition = nil, -- Expression. + bodyTrue = nil, -- AstBlock. + bodyFalse = nil, -- AstBlock or nil. +})end +local function AstWhile (token)return populateCommonNodeFields(token,{ + type = "while", + condition = nil, -- Expression. + body = nil, -- AstBlock. +})end +local function AstRepeat (token)return populateCommonNodeFields(token,{ + type = "repeat", + body = nil, -- AstBlock. + condition = nil, -- Expression. +})end +local function AstFor (token,kind)return populateCommonNodeFields(token,{ + type = "for", + kind = kind, -- "numeric" | "generic" + names = {}, -- Array of AstIdentifier. + values = {}, -- Array of expressions. + body = nil, -- AstBlock. +})end + + + +local CHILD_FIELDS = { + ["identifier"] = {}, + ["vararg"] = {}, + ["literal"] = {}, + ["table"] = {fields="tablefields"}, + ["lookup"] = {object="node", member="node"}, + ["unary"] = {expressions="node"}, + ["binary"] = {left="node", right="node"}, + ["call"] = {callee="node", arguments="nodearray"}, + ["function"] = {parameters="nodearray", body="node"}, + ["break"] = {}, + ["return"] = {values="nodearray"}, + ["label"] = {}, + ["goto"] = {}, + ["block"] = {statements="nodearray"}, + ["declaration"] = {names="nodearray", values="nodearray"}, + ["assignment"] = {targets="nodearray", values="nodearray"}, + ["if"] = {condition="node", bodyTrue="node", bodyFalse="node"}, + ["while"] = {condition="node", body="node"}, + ["repeat"] = {body="node", condition="node"}, + ["for"] = {names="nodearray", values="nodearray", body="node"}, +} + + + +local function Stats() + return { + -- simplify() and optimize(): + nodeReplacements = {--[[ location1, ... ]]}, + nodeRemovals = {--[[ location1, ... ]]}, + nodeRemoveCount = 0, + + -- minify(): + renameCount = 0, + generatedNameCount = 0, + } +end + +-- location = Location( sourceLocation [, extraKey, extraValue ] ) +-- location = Location( sourceNode [, extraKey, extraValue ] ) +local function Location(sourceLocOrNode, extraK, extraV) + local loc = { + sourceString = sourceLocOrNode.sourceString, + sourcePath = sourceLocOrNode.sourcePath, + + line = sourceLocOrNode.line, + position = sourceLocOrNode.position, + + node = sourceLocOrNode.type and sourceLocOrNode or nil, + } + if extraK then + loc[extraK] = extraV + end + return loc +end + + + +-- count = countString( haystack, needle [, plain=false ] ) +local function countString(haystack, needle, plain) + local count = 0 + local pos = 1 + + while true do + local _, i2 = stringFind(haystack, needle, pos, plain) + if not i2 then return count end + + count = count + 1 + pos = i2 + 1 + end +end + +-- count = countSubString( haystack, startPosition, endPosition, needle [, plain=false ] ) +local function countSubString(haystack, pos, posEnd, needle, plain) + local count = 0 + + while true do + local _, i2 = stringFind(haystack, needle, pos, plain) + if not i2 or i2 > posEnd then return count end + + count = count + 1 + pos = i2 + 1 + end +end + + + +-- errorf( [ level=1, ] format, ... ) +local function errorf(level, s, ...) + if type(level) == "number" then + error(F(s, ...), (level == 0 and 0 or (1+level))) + else + error(F(level, s, ...), 2) + end +end + +-- assertArg1( functionName, argumentNumber, value, expectedType [, level=2 ] ) +-- assertArg2( functionName, argumentNumber, value, expectedType1, expectedType2 [, level=2 ] ) +local function assertArg1(funcName, argNum, v, expectedType, level) + if type(v) == expectedType then return end + errorf(1+(level or 2), "Bad argument #%d to '%s'. (Expected %s, got %s)", argNum, funcName, expectedType, type(v)) +end +local function assertArg2(funcName, argNum, v, expectedType1, expectedType2, level) + if type(v) == expectedType1 or type(v) == expectedType2 then return end + errorf(1+(level or 2), "Bad argument #%d to '%s'. (Expected %s or %s, got %s)", argNum, funcName, expectedType1, expectedType2, type(v)) +end + + + +local ensurePrintable +do + local CONTROL_TO_READABLE = { + ["\0"] = "{NUL}", + ["\n"] = "{NL}", + ["\r"] = "{CR}", + } + + --[[local]] function ensurePrintable(s) + return (stringGsub(s, "[%z\1-\31\127-\255]", function(c) + return CONTROL_TO_READABLE[c] or (stringByte(c) <= 31 or stringByte(c) >= 127) and F("{%d}", stringByte(c)) or nil + end)) + end +end + + + +local function removeUnordered(t, i) + local len = #t + if i > len or i < 1 then return end + + -- Note: This does the correct thing if i==len too. + t[i] = t[len] + t[len] = nil +end + +local function removeItemUnordered(t, v) + for i = 1, #t do + if t[i] == v then + removeUnordered(t, i) + return + end + end +end + + + +local function getLineNumber(s, pos) + return 1 + countSubString(s, 1, pos-1, "\n", true) +end + + + +local formatMessageInFile +do + local function findStartOfLine(s, pos, canBeEmpty) + while pos > 1 do + if stringByte(s, pos-1) == 10--[[\n]] and (canBeEmpty or stringByte(s, pos) ~= 10--[[\n]]) then break end + pos = pos - 1 + end + return mathMax(pos, 1) + end + local function findEndOfLine(s, pos) + while pos < #s do + if stringByte(s, pos+1) == 10--[[\n]] then break end + pos = pos + 1 + end + return mathMin(pos, #s) + end + + local function getSubTextLength(s, pos, posEnd) + local len = 0 + + while pos <= posEnd do + if stringByte(s, pos) == 9 then -- '\t' + len = len + 4 + pos = pos + 1 + else + local _, i2 = stringFind(s, "^[%z\1-\127\194-\253][\128-\191]*", pos) + if i2 and i2 <= posEnd then + len = len + 1 + pos = i2 + 1 + else + len = len + 1 + pos = pos + 1 + end + end + end + + return len + end + + --[[local]] function formatMessageInFile(prefix, contents, path, pos, agent, s, ...) + if agent ~= "" then + agent = "["..agent.."] " + end + + s = F(s, ...) + + if contents == "" then + return F("%s @ %s: %s%s", prefix, path, agent, s) + end + + pos = mathMin(mathMax(pos, 1), #contents+1) + local ln = getLineNumber(contents, pos) + + local lineStart = findStartOfLine(contents, pos, true) + local lineEnd = findEndOfLine (contents, pos-1) + local linePre1Start = findStartOfLine(contents, lineStart-1, false) + local linePre1End = findEndOfLine (contents, linePre1Start-1) + local linePre2Start = findStartOfLine(contents, linePre1Start-1, false) + local linePre2End = findEndOfLine (contents, linePre2Start-1) + -- print(F("pos %d | lines %d..%d, %d..%d, %d..%d", pos, linePre2Start,linePre2End+1, linePre1Start,linePre1End+1, lineStart,lineEnd+1)) -- DEBUG + + return F("%s @ %s:%d: %s%s\n>\n%s%s%s>-%s^", + prefix, path, ln, agent, s, + (linePre2Start < linePre1Start and linePre2Start <= linePre2End) and F("> %s\n", (stringGsub(stringSub(contents, linePre2Start, linePre2End), "\t", " "))) or "", + (linePre1Start < lineStart and linePre1Start <= linePre1End) and F("> %s\n", (stringGsub(stringSub(contents, linePre1Start, linePre1End), "\t", " "))) or "", + ( lineStart <= lineEnd ) and F("> %s\n", (stringGsub(stringSub(contents, lineStart, lineEnd ), "\t", " "))) or ">\n", + stringRep("-", getSubTextLength(contents, lineStart, pos-1)) + ) + end +end + +local function formatMessageAtToken(prefix, token, agent, s, ...) + return (formatMessageInFile(prefix, (token and token.sourceString or ""), (token and token.sourcePath or "?"), (token and token.positionStart or 0), agent, s, ...)) +end +local function formatMessageAfterToken(prefix, token, agent, s, ...) + return (formatMessageInFile(prefix, (token and token.sourceString or ""), (token and token.sourcePath or "?"), (token and token.positionEnd+1 or 0), agent, s, ...)) +end + +local function formatMessageAtNode(prefix, node, agent, s, ...) + return (formatMessageInFile(prefix, node.sourceString, node.sourcePath, node.position, agent, s, ...)) +end + +local function formatMessageHelper(argNumOffset, prefix, nodeOrLocOrToken, s, ...) + assertArg1("formatMessage", 1+argNumOffset, prefix, "string", 3) + assertArg1("formatMessage", 2+argNumOffset, nodeOrLocOrToken, "table", 3) + assertArg1("formatMessage", 3+argNumOffset, s, "string", 3) + + local formatter = nodeOrLocOrToken.representation and formatMessageAtToken or formatMessageAtNode + return (formatter(prefix, nodeOrLocOrToken, "", s, ...)) +end + +-- message = formatMessage( [ prefix="Info", ] token, s, ... ) +-- message = formatMessage( [ prefix="Info", ] astNode, s, ... ) +-- message = formatMessage( [ prefix="Info", ] location, s, ... ) +local function formatMessage(prefix, ...) + if type(prefix) == "string" then + return (formatMessageHelper(0, prefix, ...)) + else + return (formatMessageHelper(-1, "Info", prefix, ...)) + end + +end + + + +local function formatErrorInFile (...) return formatMessageInFile ("Error", ...) end +local function formatErrorAtToken (...) return formatMessageAtToken ("Error", ...) end +local function formatErrorAfterToken(...) return formatMessageAfterToken("Error", ...) end +local function formatErrorAtNode (...) return formatMessageAtNode ("Error", ...) end + + + +local function where(node, s, ...) + if not node then + print("[Where] No node here!") + elseif s then + print(formatMessageAtNode("Info", node, "Where", s, ...)) + else + print(formatMessageAtNode("Info", node, "Where", "Here!")) + end +end + + + +local function iprev(t, i) + i = i-1 + local v = t[i] + + if v ~= nil then return i, v end +end + +local function ipairsr(t) + return iprev, t, #t+1 +end + + + +-- index = indexOf( array, value [, startIndex=1, endIndex=#array ] ) +local function indexOf(t, v, i1, i2) + for i = (i1 or 1), (i2 or #t) do + if t[i] == v then return i end + end + return nil +end + +-- item, index = itemWith1 ( array, key, value ) +-- item, index = lastItemWith1( array, key, value ) +local function itemWith1(t, k, v) + for i, item in ipairs(t) do + if item[k] == v then return item, i end + end + return nil +end +local function lastItemWith1(t, k, v) + for i, item in ipairsr(t) do + if item[k] == v then return item, i end + end + return nil +end + + + +-- text = getRelativeLocationText( sourcePathOfInterest, lineNumberOfInterest, otherSourcePath, otherLineNumber ) +-- text = getRelativeLocationText( lineNumberOfInterest, otherLineNumber ) +local function getRelativeLocationText(sourcePath, ln, otherSourcePath, otherLn) + if type(sourcePath) ~= "string" then + sourcePath, ln, otherSourcePath, otherLn = "", sourcePath, "", ln + end + + if not (ln > 0) then + return "at " + end + + if sourcePath ~= otherSourcePath then return F("at %s:%d", sourcePath, ln) end + if ln+1 == otherLn and otherLn > 0 then return F("on the previous line") end + if ln-1 == otherLn and otherLn > 0 then return F("on the next line") end + if ln ~= otherLn then return F("on line %d", ln) end + return "on the same line" +end + +-- text = getRelativeLocationTextForToken( tokens, tokenOfInterest, otherToken ) +local function getRelativeLocationTextForToken(tokens, tokOfInterest, otherTok) + return getRelativeLocationText((tokens[tokOfInterest] and tokens[tokOfInterest].lineStart or 0), (tokens[otherTok] and tokens[otherTok].lineStart or 0)) +end + +--[[ +-- text = getRelativeLocationTextForNode( nodeOfInterest, otherNode ) +-- text = getRelativeLocationTextForNode( nodeOfInterest, otherSourcePath, otherLineNumber ) +local function getRelativeLocationTextForNode(nodeOfInterest, otherSourcePath, otherLn) + if type(otherSourcePath) == "table" then + return getRelativeLocationTextForNode(nodeOfInterest, otherSourcePath.sourcePath, otherSourcePath.line) + end + + if not (nodeOfInterest.sourcePath ~= "?" and nodeOfInterest.line > 0) then + return "at " + end + + return getRelativeLocationText(nodeOfInterest.sourcePath, nodeOfInterest.line, otherSourcePath, otherLn) +end +]] + + + +local function formatNumber(n) + -- @Speed: Cache! + + -- 64-bit int in LuaJIT (is what we assume, anyway). + if jit and type(n) == "cdata" then + local nStr = tostring(n) + + if stringFind(nStr, "i$") then + if stringFind(nStr, "^0[-+]") then + nStr = stringGsub(nStr, "^0%+?", "") + else + -- + -- LuaJIT doesn't seem to be able to parse nStr if we output it as-is. + -- What is even the notation for complex numbers with a non-zero real part? + -- Oh LuaJIT, you're so mysterious... + -- + -- @Robustness: Make sure we don't choke when trying to simplify() complex numbers. + -- + errorf(2, "Cannot output complex number '%s'.", nStr) + end + end + + return nStr + end + + -- Int (including 64-bit ints in Lua 5.3+, and excluding whole floats). + if n == mathFloor(n) and not (mathType and mathType(n) == "float") then + local nStr = F("%.0f", n) + if tonumber(nStr) == n then return nStr end + end + + -- Anything else. + return (tostring(n) + :gsub("(e[-+])0+(%d+)$", "%1%2") -- Remove unnecessary zeroes after 'e'. + :gsub("e%+", "e" ) -- Remove plus after 'e'. + ) +end + + + +local tokenize +do + local ERROR_UNFINISHED_VALUE = {} + + -- success, equalSignCountIfLong|errorCode, ptr = parseStringlikeToken( s, ptr ) + local function parseStringlikeToken(s, ptr) + local longEqualSigns = stringMatch(s, "^%[(=*)%[", ptr) + local equalSignCountIfLong = longEqualSigns and #longEqualSigns + + -- Single line (comment). + if not equalSignCountIfLong then + local i1, i2 = stringFind(s, "\n", ptr) + ptr = i2 and i2 + 1 or #s + 1 + + -- Multiline. + else + ptr = ptr + 1 + #longEqualSigns + 1 + + local i1, i2 = stringFind(s, "%]"..longEqualSigns.."%]", ptr) + if not i1 then + return false, ERROR_UNFINISHED_VALUE, 0 + end + + ptr = i2 + 1 + end + + return true, equalSignCountIfLong, ptr + end + + local function codepointToString(cp, buffer) + if cp < 0 or cp > 0x10ffff then + -- This error is actually incorrect as Lua supports codepoints up to 2^31. + -- This is probably an issue that no one will ever encounter! + return false, F("Codepoint 0x%X (%.0f) is outside the valid range (0..10FFFF).", cp, cp) + end + + if cp < 128 then + tableInsert(buffer, stringChar(cp)) + return true + end + + local suffix = cp % 64 + local c4 = 128 + suffix + cp = (cp - suffix) / 64 + + if cp < 32 then + tableInsert(buffer, stringChar(192+cp)) + tableInsert(buffer, stringChar(c4)) + return true + end + + suffix = cp % 64 + local c3 = 128 + suffix + cp = (cp - suffix) / 64 + + if cp < 16 then + tableInsert(buffer, stringChar(224+cp)) + tableInsert(buffer, stringChar(c3)) + tableInsert(buffer, stringChar(c4)) + return true + end + + suffix = cp % 64 + cp = (cp - suffix) / 64 + + tableInsert(buffer, stringChar(240+cp)) + tableInsert(buffer, stringChar(128+suffix)) + tableInsert(buffer, stringChar(c3)) + tableInsert(buffer, stringChar(c4)) + return true + end + + local function parseStringContents(s, path, ptrStart, ptrEnd) + local ptr = ptrStart + local buffer = {} + + while ptr <= ptrEnd do + local i1 = stringFind(s, "\\", ptr, true) + if not i1 or i1 > ptrEnd then break end + + if i1 > ptr then + tableInsert(buffer, stringSub(s, ptr, i1-1)) + end + ptr = i1 + 1 + + -- local b1, b2, b3 = stringByte(s, ptr, ptr+2) + + if stringFind(s, "^a", ptr) then tableInsert(buffer, "\a") ; ptr = ptr + 1 + elseif stringFind(s, "^b", ptr) then tableInsert(buffer, "\b") ; ptr = ptr + 1 + elseif stringFind(s, "^t", ptr) then tableInsert(buffer, "\t") ; ptr = ptr + 1 + elseif stringFind(s, "^n", ptr) then tableInsert(buffer, "\n") ; ptr = ptr + 1 + elseif stringFind(s, "^v", ptr) then tableInsert(buffer, "\v") ; ptr = ptr + 1 + elseif stringFind(s, "^f", ptr) then tableInsert(buffer, "\f") ; ptr = ptr + 1 + elseif stringFind(s, "^r", ptr) then tableInsert(buffer, "\r") ; ptr = ptr + 1 + elseif stringFind(s, "^\\",ptr) then tableInsert(buffer, "\\") ; ptr = ptr + 1 + elseif stringFind(s, '^"', ptr) then tableInsert(buffer, "\"") ; ptr = ptr + 1 + elseif stringFind(s, "^'", ptr) then tableInsert(buffer, "\'") ; ptr = ptr + 1 + elseif stringFind(s, "^\n",ptr) then tableInsert(buffer, "\n") ; ptr = ptr + 1 + + elseif stringFind(s, "^z", ptr) then + local i1, i2 = stringFind(s, "^%s*", ptr+1) + ptr = i2 + 1 + + elseif stringFind(s, "^%d", ptr) then + local nStr = stringMatch(s, "^%d%d?%d?", ptr) + local byte = tonumber(nStr) + + if byte > 255 then + return nil, formatErrorInFile( + s, path, ptr, "Tokenizer", + "Byte value '%s' is out-of-range in decimal escape sequence. (String starting %s)", + nStr, getRelativeLocationText(getLineNumber(s, ptrStart), getLineNumber(s, ptr)) + ) + end + + tableInsert(buffer, stringChar(byte)) + ptr = ptr + #nStr + + elseif stringFind(s, "^x%x%x", ptr) then + local hexStr = stringSub(s, ptr+1, ptr+2) + local byte = tonumber(hexStr, 16) + + tableInsert(buffer, stringChar(byte)) + ptr = ptr + 3 + + elseif stringFind(s, "^u{%x+}", ptr) then + local hexStr = stringMatch(s, "^%x+", ptr+2) + local cp = tonumber(hexStr, 16) + + local ok, err = codepointToString(cp, buffer) + if not ok then + return nil, formatErrorInFile( + s, path, ptr+2, "Tokenizer", + "%s (String starting %s)", + err, getRelativeLocationText(getLineNumber(s, ptrStart), getLineNumber(s, ptr)) + ) + end + + ptr = ptr + 3 + #hexStr + + else + return nil, formatErrorInFile( + s, path, ptr-1, "Tokenizer", + "Invalid escape sequence. (String starting %s)", + getRelativeLocationText(getLineNumber(s, ptrStart), getLineNumber(s, ptr)) + ) + end + + end + + if ptr <= ptrEnd then + tableInsert(buffer, stringSub(s, ptr, ptrEnd)) + end + + return tableConcat(buffer) + end + + -- tokens, error = tokenize( luaString [, keepWhitespaceTokens=false ] [, pathForErrorMessages="?" ] ) + --[[local]] function tokenize(s, keepWhitespaceTokens, path) + assertArg1("tokenize", 1, s, "string") + + if type(keepWhitespaceTokens) == "string" then + path = keepWhitespaceTokens + keepWhitespaceTokens = false + else + assertArg2("tokenize", 2, keepWhitespaceTokens, "boolean","nil") + assertArg2("tokenize", 3, path, "string","nil") + end + + if stringFind(s, "\r", 1, true) then + s = stringGsub(s, "\r\n?", "\n") + end + path = path or "?" + + local tokens = {} + local count = 0 + local ptr = 1 + local ln = 1 + + local BYTES_NAME_START = TOKEN_BYTES.NAME_START + local BYTES_DASH = TOKEN_BYTES.DASH + local BYTES_NUM = TOKEN_BYTES.NUM + local BYTES_QUOTE = TOKEN_BYTES.QUOTE + local BYTES_SQUARE = TOKEN_BYTES.SQUARE + local BYTES_DOT = TOKEN_BYTES.DOT + local BYTES_PUNCT_TWO_CHARS = TOKEN_BYTES.PUNCT_TWO_CHARS + local BYTES_PUNCT_ONE_CHAR = TOKEN_BYTES.PUNCT_ONE_CHAR + + local NUM_HEX_FRAC_EXP = NUMERAL_PATTERNS.HEX_FRAC_EXP + local NUM_HEX_FRAC = NUMERAL_PATTERNS.HEX_FRAC + local NUM_HEX_EXP = NUMERAL_PATTERNS.HEX_EXP + local NUM_HEX = NUMERAL_PATTERNS.HEX + local NUM_BIN = NUMERAL_PATTERNS.BIN + local NUM_DEC_FRAC_EXP = NUMERAL_PATTERNS.DEC_FRAC_EXP + local NUM_DEC_FRAC = NUMERAL_PATTERNS.DEC_FRAC + local NUM_DEC_EXP = NUMERAL_PATTERNS.DEC_EXP + local NUM_DEC = NUMERAL_PATTERNS.DEC + + while true do + local i1, i2 = stringFind(s, "^[ \t\n]+", ptr) + + if i1 then + if keepWhitespaceTokens then + local lnStart = ln + local tokRepr = stringSub(s, i1, i2) + ln = ln + countString(tokRepr, "\n", true) + count = count + 1 + + tokens[count] = { + type = "whitespace", + value = tokRepr, + representation = tokRepr, + + sourceString = s, + sourcePath = path, + + lineStart = lnStart, + lineEnd = ln, + positionStart = i1, + positionEnd = i2, + } + + else + ln = ln + countSubString(s, i1, i2, "\n", true) + end + + ptr = i2 + 1 + end + + if ptr > #s then break end + + local ptrStart = ptr + local lnStart = ln + local b1, b2 = stringByte(s, ptr, ptr+1) + local tokType, tokRepr, tokValue + + -- Identifier/keyword. + if BYTES_NAME_START[b1] then + local i1, i2, word = stringFind(s, "^(.[%w_]*)", ptr) + ptr = i2+1 + tokType = KEYWORDS[word] and "keyword" or "identifier" + tokRepr = stringSub(s, ptrStart, ptr-1) + tokValue = tokRepr + + -- Comment. + elseif BYTES_DASH[b1] and BYTES_DASH[b2] then + ptr = ptr + 2 + + local ok, equalSignCountIfLong + ok, equalSignCountIfLong, ptr = parseStringlikeToken(s, ptr) + + if not ok then + local errCode = equalSignCountIfLong + if errCode == ERROR_UNFINISHED_VALUE then + return nil, formatErrorInFile(s, path, ptrStart, "Tokenizer", "Unfinished long comment.") + else + return nil, formatErrorInFile(s, path, ptrStart, "Tokenizer", "Invalid comment.") + end + end + + -- Check for nesting of [[...]] which is deprecated in Lua. Sigh... + if equalSignCountIfLong and equalSignCountIfLong == 0 then + local pos = stringFind(s, "[[", ptrStart+4, true) + if pos and pos < ptr then + return nil, formatErrorInFile(s, path, pos, "Tokenizer", "Cannot have nested comments. (Comment starting %s)", getRelativeLocationText(lnStart, getLineNumber(s, pos))) + end + end + + tokType = "comment" + tokRepr = stringSub(s, ptrStart, ptr-1) + tokRepr = equalSignCountIfLong and tokRepr or (stringFind(tokRepr, "\n$") and tokRepr or tokRepr.."\n") -- Make sure there's a newline at the end of single-line comments. (It may be missing if we've reached EOF.) + tokValue = equalSignCountIfLong and stringSub(tokRepr, 5+equalSignCountIfLong, -3-equalSignCountIfLong) or stringSub(tokRepr, 3, -2) + + -- Number. + elseif BYTES_NUM[b1] or (BYTES_DOT[b1] and BYTES_NUM[b2]) then + local pat, maybeInt, kind, i1, i2, numStr = NUM_HEX_FRAC_EXP, false, "lua52hex", stringFind(s, NUM_HEX_FRAC_EXP, ptr) + if not i1 then pat, maybeInt, kind, i1, i2, numStr = NUM_HEX_FRAC, false, "lua52hex", stringFind(s, NUM_HEX_FRAC, ptr) + if not i1 then pat, maybeInt, kind, i1, i2, numStr = NUM_HEX_EXP, false, "lua52hex", stringFind(s, NUM_HEX_EXP, ptr) + if not i1 then pat, maybeInt, kind, i1, i2, numStr = NUM_HEX, true, "", stringFind(s, NUM_HEX, ptr) + if not i1 then pat, maybeInt, kind, i1, i2, numStr = NUM_BIN, true, "binary", stringFind(s, NUM_BIN, ptr) -- LuaJIT supports these, so why not. + if not i1 then pat, maybeInt, kind, i1, i2, numStr = NUM_DEC_FRAC_EXP, false, "", stringFind(s, NUM_DEC_FRAC_EXP, ptr) + if not i1 then pat, maybeInt, kind, i1, i2, numStr = NUM_DEC_FRAC, false, "", stringFind(s, NUM_DEC_FRAC, ptr) + if not i1 then pat, maybeInt, kind, i1, i2, numStr = NUM_DEC_EXP, false, "", stringFind(s, NUM_DEC_EXP, ptr) + if not i1 then pat, maybeInt, kind, i1, i2, numStr = NUM_DEC, true, "", stringFind(s, NUM_DEC, ptr) + if not numStr then return nil, formatErrorInFile(s, path, ptrStart, "Tokenizer", "Malformed number.") + end end end end end end end end end + + local numStrFallback = numStr + + if jit then + if stringFind(s, "^[Ii]", i2+1) then -- Imaginary part of complex number. + numStr = stringSub(s, i1, i2+1) + i2 = i2 + 1 + + elseif not maybeInt or stringFind(numStr, ".", 1, true) then + -- void + elseif stringFind(s, "^[Uu][Ll][Ll]", i2+1) then -- Unsigned 64-bit integer. + numStr = stringSub(s, i1, i2+3) + i2 = i2 + 3 + elseif stringFind(s, "^[Ll][Ll]", i2+1) then -- Signed 64-bit integer. + numStr = stringSub(s, i1, i2+2) + i2 = i2 + 2 + end + end + + local n = tonumber(numStr) + + if not n and jit then + local chunk = loadstring("return "..numStr) + n = chunk and chunk() or n + end + + n = n or tonumber(numStrFallback) + + if not n then + -- Note: We know we're not running LuaJIT here as it supports hexadecimal floats and binary notation, thus we use numStrFallback instead of numStr. + + -- Support hexadecimal floats if we're running Lua 5.1. + if kind == "lua52hex" then + local _, intStr, fracStr, expStr + if pat == NUM_HEX_FRAC_EXP then _, intStr, fracStr, expStr = stringMatch(numStrFallback, NUM_HEX_FRAC_EXP) + elseif pat == NUM_HEX_FRAC then _, intStr, fracStr = stringMatch(numStrFallback, NUM_HEX_FRAC) ; expStr = "0" + elseif pat == NUM_HEX_EXP then _, intStr, expStr = stringMatch(numStrFallback, NUM_HEX_EXP) ; fracStr = "" + else return nil, formatErrorInFile(s, path, ptrStart, "Tokenizer", "Internal error parsing the number '%s'.", numStrFallback) end + + n = tonumber(intStr, 16) or 0 -- intStr may be "". + + local fracValue = 1 + for i = 1, #fracStr do + fracValue = fracValue / 16 + n = n + tonumber(stringSub(fracStr, i, i), 16) * fracValue + end + + n = n * 2 ^ stringGsub(expStr, "^+", "") + + elseif kind == "binary" then + n = tonumber(stringSub(numStrFallback, 3), 2) + end + end + + if not n then + return nil, formatErrorInFile(s, path, ptrStart, "Tokenizer", "Invalid number.") + end + + ptr = i2+1 + tokType = "number" + tokRepr = numStr + tokValue = n + + if stringFind(s, "^[%w_.]", ptr) then + local after = stringMatch(s, "^%.?%d+", ptr) or stringMatch(s, "^[%w_.][%w_.]?", ptr) + return nil, formatErrorInFile(s, path, ptrStart, "Tokenizer", "Malformed number near '%s%s'.", numStr, after) + end + + -- Quoted string. + elseif BYTES_QUOTE[b1] then + local quote = stringSub(s, ptr, ptr) + local quoteByte = stringByte(quote) + ptr = ptr + 1 + + local pat = "["..quote.."\\\n]" + + while true do + local i1 = stringFind(s, pat, ptr) + if not i1 then + return nil, formatErrorInFile(s, path, ptrStart, "Tokenizer", "Unfinished string.") + end + + ptr = i1 + local b1, b2 = stringByte(s, ptr, ptr+1) + + -- '"' + if b1 == quoteByte then + ptr = ptr + 1 + break + + -- '\' + elseif b1 == 92 then + ptr = ptr + 1 + + if b2 == 122 then -- 'z' + ptr = ptr + 1 + local _, i2 = stringFind(s, "^%s*", ptr) + ptr = i2 + 1 + else + -- Note: We don't have to look for multiple characters after the escape, like \nnn - this algorithm works anyway. + if ptr > #s then + return nil, formatErrorInFile( + s, path, ptr, "Tokenizer", + "Unfinished string after escape character. (String starting %s)", + getRelativeLocationText(lnStart, getLineNumber(s, ptr)) + ) + end + ptr = ptr + 1 -- Just skip the next character, whatever it might be. + end + + -- '\n' + elseif b1 == 10 then + -- Lua, this is silly! + return nil, formatErrorInFile(s, path, ptr, "Tokenizer", "Unescaped newline in string (starting %s).", getRelativeLocationText(lnStart, getLineNumber(s, ptr))) + + else + assert(false) + end + end + + tokType = "string" + tokRepr = stringSub(s, ptrStart, ptr-1) + + local chunk = loadstring("return "..tokRepr, "@") -- Try to make Lua parse the string value before we fall back to our own parser which is probably slower. + if chunk then + tokValue = chunk() + assert(type(tokValue) == "string") + else + local stringValue, err = parseStringContents(s, path, ptrStart+1, ptr-2) + if not stringValue then return nil, err end + tokValue = stringValue + end + + -- Long string. + elseif BYTES_SQUARE[b1] and stringFind(s, "^=*%[", ptr+1) then + local ok, equalSignCountIfLong + ok, equalSignCountIfLong, ptr = parseStringlikeToken(s, ptr) + + if not ok then + local errCode = equalSignCountIfLong + if errCode == ERROR_UNFINISHED_VALUE then + return nil, formatErrorInFile(s, path, ptrStart, "Tokenizer", "Unfinished long string.") + else + return nil, formatErrorInFile(s, path, ptrStart, "Tokenizer", "Invalid long string.") + end + end + + tokType = "string" + tokRepr = stringSub(s, ptrStart, ptr-1) + + local chunk, err = loadstring("return "..tokRepr, "@") + if not chunk then + err = stringGsub(err, "^:%d+: ", "") + return nil, formatErrorInFile(s, path, ptrStart, "Tokenizer", "Could not convert long string token to value. (%s)", err) + end + tokValue = assert(chunk)() + assert(type(tokValue) == "string") + + -- Punctuation. + elseif BYTES_DOT[b1] and stringFind(s, "^%.%.", ptr+1) then + ptr = ptr + 3 + tokType = "punctuation" + tokRepr = stringSub(s, ptrStart, ptr-1) + tokValue = tokRepr + elseif BYTES_PUNCT_TWO_CHARS[b1] and stringFind(s, "^%.%.", ptr) or stringFind(s, "^[=~<>]=", ptr) or stringFind(s, "^::", ptr) or stringFind(s, "^//", ptr) or stringFind(s, "^<<", ptr) or stringFind(s, "^>>", ptr) then + ptr = ptr + 2 + tokType = "punctuation" + tokRepr = stringSub(s, ptrStart, ptr-1) + tokValue = tokRepr + elseif BYTES_PUNCT_ONE_CHAR[b1] then + ptr = ptr + 1 + tokType = "punctuation" + tokRepr = stringSub(s, ptrStart, ptr-1) + tokValue = tokRepr + + else + return nil, formatErrorInFile(s, path, ptrStart, "Tokenizer", "Unknown character.") + end + assert(tokType) + + ln = ln + countString(tokRepr, "\n", true) + count = count + 1 + + tokens[count] = { + type = tokType, + value = tokValue, + representation = tokRepr, + + sourceString = s, + sourcePath = path, + + lineStart = lnStart, + lineEnd = ln, + positionStart = ptrStart, + positionEnd = ptr - 1, + } + + -- print(F("%4d %-11s '%s'", count, tokType, (stringGsub(tokRepr, "\n", "\\n")))) + end + + return tokens + end +end + +-- tokens, error = tokenizeFile( path [, keepWhitespaceTokens=false ] ) +local function tokenizeFile(path, keepWhitespaceTokens) + assertArg1("tokenizeFile", 1, path, "string") + assertArg2("tokenizeFile", 2, keepWhitespaceTokens, "boolean","nil") + + local file, err = ioOpen(path, "r") + if not file then return nil, F("Could not open file '%s'. (%s)", ensurePrintable(path), ensurePrintable(err)) end + + local s = file:read("*a") + file:close() + + return tokenize(s, (keepWhitespaceTokens or false), path) +end + +-- +-- :TokenCreation +-- +-- commentToken = newToken( "comment", contents ) +-- identifierToken = newToken( "identifier", name ) +-- keywordToken = newToken( "keyword", name ) +-- numberToken = newToken( "number", number ) +-- punctuationToken = newToken( "punctuation", punctuationString ) +-- stringToken = newToken( "string", stringValue ) +-- whitespaceToken = newToken( "whitespace", contents ) +-- +local function newToken(tokType, tokValue) + local tokRepr + + if tokType == "keyword" then + if type(tokValue) ~= "string" then errorf(2, "Expected string value for 'keyword' token. (Got %s)", type(tokValue)) end + if not KEYWORDS[tokValue] then errorf(2, "Invalid keyword '%s'.", tokValue) end + tokRepr = tokValue + + elseif tokType == "identifier" then + if type(tokValue) ~= "string" then errorf(2, "Expected string value for 'identifier' token. (Got %s)", type(tokValue)) end + if not stringFind(tokValue, "^[%a_][%w_]*$") then errorf(2, "Invalid identifier '%s'.", tokValue) end + if KEYWORDS[tokValue] then errorf(2, "Invalid identifier '%s'.", tokValue) end + tokRepr = tokValue + + elseif tokType == "number" then + if type(tokValue) ~= "number" then + errorf(2, "Expected number value for 'number' token. (Got %s)", type(tokValue)) + end + tokRepr = ( + tokValue == 0 and NORMALIZE_MINUS_ZERO and "0" or -- Avoid '-0' sometimes. + tokValue == 1/0 and "(1/0)" or + tokValue == -1/0 and "(-1/0)" or + tokValue ~= tokValue and "(0/0)" or + formatNumber(tokValue) + ) + + elseif tokType == "string" then + if type(tokValue) ~= "string" then errorf(2, "Expected string value for 'string' token. (Got %s)", type(tokValue)) end + tokRepr = stringGsub(F("%q", tokValue), "\n", "n") + + elseif tokType == "punctuation" then + if type(tokValue) ~= "string" then errorf(2, "Expected string value for 'punctuation' token. (Got %s)", type(tokValue)) end + if not PUNCTUATION[tokValue] then errorf(2, "Invalid punctuation '%s'.", tokValue) end + tokRepr = tokValue + + elseif tokType == "comment" then + if type(tokValue) ~= "string" then errorf(2, "Expected string value for 'comment' token. (Got %s)", type(tokValue)) end + + if stringFind(tokValue, "\n") then + local equalSigns = stringFind(tokValue, "[[", 1, true) and "=" or "" + + while stringFind(tokValue, "]"..equalSigns.."]", 1, true) do + equalSigns = equalSigns.."=" + end + + tokRepr = F("--[%s[%s]%s]", equalSigns, tokValue, equalSigns) + + else + tokRepr = F("--%s\n", tokValue) + end + + elseif tokType == "whitespace" then + if type(tokValue) ~= "string" then errorf(2, "Expected string value for 'whitespace' token. (Got %s)", type(tokValue)) end + if tokValue == "" then errorf(2, "Value is empty.") end -- Having a token that is zero characters long would be weird, so we disallow it. + if stringFind(tokValue, "[^ \t\n]") then errorf(2, "Value has non-whitespace characters.") end + tokRepr = tokValue + + else + errorf(2, "Invalid token type '%s'.", tostring(tokType)) + end + + return { + type = tokType, + value = tokValue, + representation = tokRepr, + + sourceString = "", + sourcePath = "?", + + lineStart = 0, + lineEnd = 0, + positionStart = 0, + positionEnd = 0, + } +end + +-- +-- :TokenModification +-- +-- updateToken( commentToken, contents ) +-- updateToken( identifierToken, name ) +-- updateToken( keywordToken, name ) +-- updateToken( numberToken, number ) +-- updateToken( punctuationToken, punctuationString ) +-- updateToken( stringToken, stringValue ) +-- updateToken( whitespaceToken, contents ) +-- +local function updateToken(tok, tokValue) + -- @Copypaste from newToken(). + + if tok.type == "keyword" then + if type(tokValue) ~= "string" then errorf(2, "Expected string value for 'keyword' token. (Got %s)", type(tokValue)) end + if not KEYWORDS[tokValue] then errorf(2, "Invalid keyword '%s'.", tokValue) end + tok.representation = tokValue + + elseif tok.type == "identifier" then + if type(tokValue) ~= "string" then errorf(2, "Expected string value for 'identifier' token. (Got %s)", type(tokValue)) end + if not stringFind(tokValue, "^[%a_][%w_]*$") then errorf(2, "Invalid identifier '%s'.", tokValue) end + if KEYWORDS[tokValue] then errorf(2, "Invalid identifier '%s'.", tokValue) end + tok.representation = tokValue + + elseif tok.type == "number" then + if type(tokValue) ~= "number" then + errorf(2, "Expected number value for 'number' token. (Got %s)", type(tokValue)) + end + tok.representation = ( + tokValue == 0 and NORMALIZE_MINUS_ZERO and "0" or -- Avoid '-0' sometimes. + tokValue == 1/0 and "(1/0)" or + tokValue == -1/0 and "(-1/0)" or + tokValue ~= tokValue and "(0/0)" or + formatNumber(tokValue) + ) + + elseif tok.type == "string" then + if type(tokValue) ~= "string" then errorf(2, "Expected string value for 'string' token. (Got %s)", type(tokValue)) end + tok.representation = stringGsub(F("%q", tokValue), "\n", "n") + + elseif tok.type == "punctuation" then + if type(tokValue) ~= "string" then errorf(2, "Expected string value for 'punctuation' token. (Got %s)", type(tokValue)) end + if not PUNCTUATION[tokValue] then errorf(2, "Invalid punctuation '%s'.", tokValue) end + tok.representation = tokValue + + elseif tok.type == "comment" then + if type(tokValue) ~= "string" then errorf(2, "Expected string value for 'comment' token. (Got %s)", type(tokValue)) end + + if stringFind(tokValue, "\n") then + local equalSigns = stringFind(tokValue, "[[", 1, true) and "=" or "" + + while stringFind(tokValue, "]"..equalSigns.."]", 1, true) do + equalSigns = equalSigns.."=" + end + + tok.representation = F("--[%s[%s]%s]", equalSigns, tokValue, equalSigns) + + else + tok.representation = F("--%s\n", tokValue) + end + + elseif tok.type == "whitespace" then + if type(tokValue) ~= "string" then errorf(2, "Expected string value for 'whitespace' token. (Got %s)", type(tokValue)) end + if tokValue == "" then errorf(2, "Value is empty.") end -- Having a token that is zero characters long would be weird, so we disallow it. + if stringFind(tokValue, "[^ \t\n]") then errorf(2, "Value has non-whitespace characters.") end + tok.representation = tokValue + + else + errorf(2, "Internal error: Invalid token type '%s'.", tostring(tok.type)) + end + + tok.value = tokValue +end + +local function cloneToken(tok) + return { + type = tok.type, + value = tok.value, + representation = tok.representation, + + sourceString = tok.sourceString, + sourcePath = tok.sourcePath, + + lineStart = tok.lineStart, + lineEnd = tok.lineEnd, + positionStart = tok.positionStart, + positionEnd = tok.positionEnd, + } +end + +local function concatTokens(tokens) + local parts = {} + + for tok = 1, #tokens do + local tokRepr = tokens[tok ].representation + local lastTokRepr = tok > 1 and tokens[tok-1].representation + + if lastTokRepr and ( + (stringFind(tokRepr, "^[%w_]") and stringFind(lastTokRepr, "[%w_]$")) or + (stringFind(tokRepr, "^%." ) and stringFind(lastTokRepr, "%.$" )) or + (stringFind(tokRepr, "^%-" ) and stringFind(lastTokRepr, "%-$" )) or + (stringFind(tokRepr, "^/" ) and stringFind(lastTokRepr, "/$" )) or + + (tok > 1 and tokens[tok-1].type == "number" and stringFind(tokRepr, "^[%w_.]")) or + ( tokens[tok ].type == "number" and stringFind(lastTokRepr, "%.$") and not stringFind(lastTokRepr, "%.%.$")) + ) then + tableInsert(parts, " ") + end + tableInsert(parts, tokRepr) + end + + return tableConcat(parts) +end + + + +local function isToken(token, tokType, tokValue) + return token ~= nil and token.type == tokType and token.value == tokValue +end + +local function isTokenType(token, tokType) + return token ~= nil and token.type == tokType +end + +local function isTokenAnyValue(token, tokValueSet) + return token ~= nil and tokValueSet[token.value] == true +end + + + +local function getLeftmostToken(node) + if node.type == "binary" then + return getLeftmostToken(node.left) + else + return node.token + end +end + + + +local parseExpressionInternal, parseExpressionList, parseFunctionParametersAndBody, parseBlock + +local function parseIdentifier(tokens, tok) --> ident, token, error + if not isTokenType(tokens[tok], "identifier") then + return nil, tok, formatErrorAtToken(tokens[tok], "Parser", "Expected an identifier.") + end + + local ident = AstIdentifier(tokens[tok], tokens[tok].value) + tok = tok + 1 + + return ident, tok +end + +local function parseNameList(tokens, tok, names, allowVararg, allowAttributes) --> success, token, error|nil + while true do + if allowVararg and isToken(tokens[tok], "punctuation", "...") then + local vararg = AstVararg(tokens[tok]) + tok = tok + 1 -- '...' + + tableInsert(names, vararg) + + return true, tok + end + + local ident, tokNext, err = parseIdentifier(tokens, tok) + if not ident then return false, tok, err end + tok = tokNext + + if allowAttributes and isToken(tokens[tok], "punctuation", "<") then + tok = tok + 1 -- '<' + + local attrIdent, tokNext, err = parseIdentifier(tokens, tok) + if not attrIdent then + return false, tok, err + elseif not (attrIdent.name == "close" or attrIdent.name == "const") then + return false, tok, formatErrorAtToken(tokens[tok], "Parser", "Expected 'close' or 'const' for attribute name.") + end + tok = tokNext + + ident.attribute = attrIdent.name + + if not isToken(tokens[tok], "punctuation", ">") then + return nil, tok, formatErrorAfterToken(tokens[tok-1], "Parser", "Expected '>' after attribute name.") + end + tok = tok + 1 -- '>' + end + + tableInsert(names, ident) + + if not isToken(tokens[tok], "punctuation", ",") then + return true, tok + end + tok = tok + 1 -- ',' + end + + return true, tok +end + +local function parseTable(tokens, tokStart) --> tableNode, token, error + local tok = tokStart + local tableNode = AstTable(tokens[tok]) + tok = tok + 1 -- '{' + + local generatedIndex = 0 + + while true do + if isToken(tokens[tok], "punctuation", "}") then + tok = tok + 1 -- '}' + break + + elseif isToken(tokens[tok], "punctuation", "[") then + tok = tok + 1 -- '[' + + local keyExpr, tokNext, err = parseExpressionInternal(tokens, tok, 0) + if not keyExpr then return nil, tok, err end + tok = tokNext + + if not isToken(tokens[tok], "punctuation", "]") then + return nil, tok, formatErrorAfterToken(tokens[tok-1], "Parser", "Expected ']' after key value.") + end + tok = tok + 1 -- ']' + + if not isToken(tokens[tok], "punctuation", "=") then + return nil, tok, formatErrorAfterToken(tokens[tok-1], "Parser", "Expected '=' after key.") + end + tok = tok + 1 -- '=' + + local valueExpr, tokNext, err = parseExpressionInternal(tokens, tok, 0) + if not valueExpr then return nil, tok, err end + tok = tokNext + + local tableField = {key=keyExpr, value=valueExpr, generatedKey=false} + tableInsert(tableNode.fields, tableField) + + elseif isTokenType(tokens[tok], "identifier") and isToken(tokens[tok+1], "punctuation", "=") then + local keyExpr = AstLiteral(tokens[tok], tokens[tok].value) + tok = tok + 1 -- identifier + + if not isToken(tokens[tok], "punctuation", "=") then + return nil, tok, formatErrorAfterToken(tokens[tok-1], "Parser", "Expected '=' after key name.") + end + tok = tok + 1 -- '=' + + local valueExpr, tokNext, err = parseExpressionInternal(tokens, tok, 0) + if not valueExpr then return nil, tok, err end + tok = tokNext + + local tableField = {key=keyExpr, value=valueExpr, generatedKey=false} + tableInsert(tableNode.fields, tableField) + + else + generatedIndex = generatedIndex + 1 + local keyExpr = AstLiteral(tokens[tok], generatedIndex) + + local valueExpr, tokNext, err = parseExpressionInternal(tokens, tok, 0) + if not valueExpr then return nil, tok, err end + tok = tokNext + + local tableField = {key=keyExpr, value=valueExpr, generatedKey=true} + tableInsert(tableNode.fields, tableField) + end + + if isToken(tokens[tok], "punctuation", ",") or isToken(tokens[tok], "punctuation", ";") then + tok = tok + 1 -- ',' or ';' + -- Continue... + + elseif isToken(tokens[tok], "punctuation", "}") then + tok = tok + 1 -- '}' + break + + else + return nil, tok, formatErrorAfterToken( + tokens[tok-1], "Parser", + "Expected ',' or '}' after value in table constructor (starting %s).", + getRelativeLocationTextForToken(tokens, tokStart, tok-1) + ) + end + end + + return tableNode, tok +end + +--[[local]] function parseExpressionInternal(tokens, tokStart, lastPrecedence) --> expression, token, error + local tok = tokStart + local canParseLookupOrCall = false + local currentToken = tokens[tok] + local expr + + -- identifier + if isTokenType(currentToken, "identifier") then + local ident, tokNext, err = parseIdentifier(tokens, tok) + if not ident then return nil, tok, err end + tok = tokNext + + expr = ident + canParseLookupOrCall = true + + -- ... + elseif isToken(currentToken, "punctuation", "...") then + local vararg = AstVararg(currentToken) + tok = tok + 1 -- '...' + expr = vararg + + -- literal + elseif isTokenType(currentToken, "string") or isTokenType(currentToken, "number") then + local literal = AstLiteral(currentToken, currentToken.value) + tok = tok + 1 -- literal + expr = literal + elseif isToken(currentToken, "keyword", "true") then + local literal = AstLiteral(currentToken, true) + tok = tok + 1 -- 'true' + expr = literal + elseif isToken(currentToken, "keyword", "false") then + local literal = AstLiteral(currentToken, false) + tok = tok + 1 -- 'false' + expr = literal + elseif isToken(currentToken, "keyword", "nil") then + local literal = AstLiteral(currentToken, nil) + tok = tok + 1 -- 'nil' + expr = literal + + -- unary + elseif + (isToken(currentToken, "keyword", "not") or (isTokenType(currentToken, "punctuation") and isTokenAnyValue(currentToken, OPERATORS_UNARY))) + and OPERATOR_PRECEDENCE.unary > lastPrecedence + then + local unary = AstUnary(currentToken, currentToken.value) + tok = tok + 1 -- operator + + local subExpr, tokNext, err = parseExpressionInternal(tokens, tok, OPERATOR_PRECEDENCE.unary-1) + if not subExpr then return nil, tok, err end + unary.expression = subExpr + tok = tokNext + + expr = unary + + -- Special rule: Treat '-n' as one literal (but not '-n^n' because of operator precedence). + if + unary.operator == "-" + and subExpr.type == "literal" + and type(subExpr.value) == "number" + and isTokenType(subExpr.token, "number") + and not (subExpr.value == 0 and NORMALIZE_MINUS_ZERO) -- We cannot store -0 in Lua 5.3+, thus we need to keep the unary expression. + then + subExpr.value = -subExpr.value + subExpr.token = unary.token + expr = subExpr + end + + -- {...} + elseif isToken(currentToken, "punctuation", "{") then + local tableNode, tokNext, err = parseTable(tokens, tok) + if not tableNode then return nil, tok, err end + tok = tokNext + + expr = tableNode + + -- function + elseif isToken(currentToken, "keyword", "function") then + local funcTok = tok + tok = tok + 1 -- 'function' + + local func, tokNext, err = parseFunctionParametersAndBody(tokens, tok, funcTok) + if not func then return nil, tok, err end + func.token = tok + tok = tokNext + + expr = func + + -- (...) + elseif isToken(currentToken, "punctuation", "(") then + tok = tok + 1 -- '(' + + local _expr, tokNext, err = parseExpressionInternal(tokens, tok, 0) + if not _expr then return nil, tok, err end + tok = tokNext + + if _expr.type == "call" or _expr.type == "vararg" then + _expr.adjustToOne = true + end + + if not isToken(tokens[tok], "punctuation", ")") then + return nil, tok, formatErrorAfterToken(tokens[tok-1], "Parser", "Expected ')' (to end parenthesis expression starting %s).", getRelativeLocationTextForToken(tokens, tokStart, tok-1)) + end + tok = tok + 1 -- ')' + + expr = _expr + canParseLookupOrCall = true + + else + return nil, tok, formatErrorAtToken(currentToken, "Parser", "Failed parsing expression.") + end + + assert(expr) + + -- Binary expressions, including lookups and calls. + while true do + currentToken = tokens[tok] + + -- a + b + if + ( + (isTokenType(currentToken, "punctuation") and isTokenAnyValue(currentToken, OPERATORS_BINARY)) + or isToken(currentToken, "keyword", "and") + or isToken(currentToken, "keyword", "or") + ) + and OPERATOR_PRECEDENCE[currentToken.value] > lastPrecedence + then + local rightAssociative = isToken(currentToken, "punctuation", "..") or isToken(currentToken, "punctuation", "^") + + local tokOp = tok + local binary = AstBinary(currentToken, currentToken.value) + tok = tok + 1 -- operator + + local lhsExpr = expr + + local rhsExpr, tokNext, err = parseExpressionInternal(tokens, tok, OPERATOR_PRECEDENCE[binary.operator] + (rightAssociative and -1 or 0)) + if not rhsExpr then return nil, tok, err end + tok = tokNext + + binary.left = expr + binary.right = rhsExpr + + expr = binary + + -- Special rule: Treat 'n/0' and '-n/0' as one literal (because that's how toLua() outputs infinity/NaN). + if + binary.operator == "/" + and lhsExpr.type == "literal" and type(lhsExpr.value) == "number" + and rhsExpr.type == "literal" and rhsExpr.value == 0 + and ( + isTokenType(lhsExpr.token, "number") + or ( + isToken(lhsExpr.token, "punctuation", "-") + and isTokenType(tokens[indexOf(tokens, lhsExpr.token, tokStart, tokOp-1) + 1], "number") -- @Speed: Don't use indexOf(). + ) + ) + and isTokenType(rhsExpr.token, "number") + then + lhsExpr.value = lhsExpr.value / 0 + expr = lhsExpr + end + + elseif not canParseLookupOrCall then + break + + -- t.k + elseif isToken(currentToken, "punctuation", ".") then + local lookup = AstLookup(currentToken) + tok = tok + 1 -- '.' + + local ident, tokNext, err = parseIdentifier(tokens, tok) + if not ident then return nil, tok, err end + tok = tokNext + + local literal = AstLiteral(ident.token, ident.name) + + lookup.object = expr + lookup.member = literal + + expr = lookup + + -- t[k] + elseif isToken(currentToken, "punctuation", "[") then + local lookup = AstLookup(currentToken) + tok = tok + 1 -- '[' + + local memberExpr, tokNext, err = parseExpressionInternal(tokens, tok, 0) + if not memberExpr then return nil, tok, err end + tok = tokNext + + if not isToken(tokens[tok], "punctuation", "]") then + return nil, tok, formatErrorAfterToken(tokens[tok-1], "Parser", "Expected ']' after lookup key value.") + end + tok = tok + 1 -- ']' + + lookup.object = expr + lookup.member = memberExpr + + expr = lookup + + -- f"" + elseif isTokenType(currentToken, "string") then + local call = AstCall(currentToken) + + local literal = AstLiteral(currentToken, currentToken.value) + tok = tok + 1 -- string + call.arguments[1] = literal + + call.callee = expr + expr = call + + -- f{} + elseif isToken(currentToken, "punctuation", "{") then + local call = AstCall(currentToken) + + local tableNode, tokNext, err = parseTable(tokens, tok) + if not tableNode then return nil, tok, err end + call.arguments[1] = tableNode + tok = tokNext + + call.callee = expr + expr = call + + -- f() + elseif isToken(currentToken, "punctuation", "(") then + if tok >= 2 and currentToken.lineStart > tokens[tok-1].lineEnd then + return nil, tok, formatErrorAtToken(currentToken, "Parser", "Ambigous syntax. Is this a function call or a new statement?") + end + + local call = AstCall(currentToken) + tok = tok + 1 -- '(' + + if not isToken(tokens[tok], "punctuation", ")") then + local ok, tokNext, err = parseExpressionList(tokens, tok, call.arguments) + if not ok then return nil, tok, err end + tok = tokNext + end + + if not isToken(tokens[tok], "punctuation", ")") then + return nil, tok, formatErrorAfterToken(tokens[tok-1], "Parser", "Expected ')' to end argument list for call.") + end + tok = tok + 1 -- ')' + + call.callee = expr + expr = call + + -- o:m() + elseif isToken(currentToken, "punctuation", ":") then + do + local lookup = AstLookup(currentToken) + tok = tok + 1 -- ':' + + local ident, tokNext, err = parseIdentifier(tokens, tok) + if not ident then return nil, tok, err end + tok = tokNext + + local literal = AstLiteral(ident.token, ident.name) + + lookup.object = expr + lookup.member = literal + + expr = lookup + end + + do + local call = AstCall(tokens[tok]) + call.method = true + + if isTokenType(tokens[tok], "string") then + local literal = AstLiteral(tokens[tok], tokens[tok].value) + tok = tok + 1 -- string + call.arguments[1] = literal + + elseif isToken(tokens[tok], "punctuation", "{") then + local tableNode, tokNext, err = parseTable(tokens, tok) + if not tableNode then return nil, tok, err end + call.arguments[1] = tableNode + tok = tokNext + + elseif isToken(tokens[tok], "punctuation", "(") then + if tok >= 2 and tokens[tok].lineStart > tokens[tok-1].lineEnd then + return nil, tok, formatErrorAtToken(tokens[tok], "Parser", "Ambigous syntax. Is this a function call or a new statement?") + end + + tok = tok + 1 -- '(' + + if not isToken(tokens[tok], "punctuation", ")") then + local ok, tokNext, err = parseExpressionList(tokens, tok, call.arguments) + if not ok then return nil, tok, err end + tok = tokNext + end + + if not isToken(tokens[tok], "punctuation", ")") then + return nil, tok, formatErrorAfterToken(tokens[tok-1], "Parser", "Expected ')' after argument list for method call.") + end + tok = tok + 1 -- ')' + + else + return nil, tok, formatErrorAfterToken(tokens[tok-1], "Parser", "Expected '(' to start argument list for method call.") + end + + call.callee = expr + expr = call + end + + else + break + end + + assert(expr) + end + + return expr, tok +end + +--[[local]] function parseExpressionList(tokens, tok, expressions) --> success, token, error + while true do + local expr, tokNext, err = parseExpressionInternal(tokens, tok, 0) + if not expr then return false, tok, err end + tok = tokNext + + tableInsert(expressions, expr) + + if not isToken(tokens[tok], "punctuation", ",") then + return true, tok + end + tok = tok + 1 -- ',' + end +end + +--[[local]] function parseFunctionParametersAndBody(tokens, tokStart, funcTok) --> func, token, error + local tok = tokStart + local func = AstFunction(tokens[funcTok]) + + if not isToken(tokens[tok], "punctuation", "(") then + return nil, tok, formatErrorAfterToken(tokens[tok-1], "Parser", "Expected '(' to start parameter list for function.") + end + tok = tok + 1 -- '(' + + if not isToken(tokens[tok], "punctuation", ")") then + local ok, tokNext, err = parseNameList(tokens, tok, func.parameters, true, false) + if not ok then return nil, tok, err end + tok = tokNext + -- @Cleanup: Move the vararg parameter parsing here. + end + + if not isToken(tokens[tok], "punctuation", ")") then + return nil, tok, formatErrorAfterToken(tokens[tok-1], "Parser", "Expected ')' to end parameter list for function.") + end + tok = tok + 1 -- ')' + + local block, tokNext, err = parseBlock(tokens, tok, tok-1, true) + if not block then return nil, tok, err end + func.body = block + tok = tokNext + + if not isToken(tokens[tok], "keyword", "end") then + return nil, tok, formatErrorAtToken(tokens[tok], "Parser", "Expected 'end' to end function (starting %s).", getRelativeLocationTextForToken(tokens, tokStart, tok)) + end + tok = tok + 1 -- 'end' + + return func, tok +end + +local BLOCK_END_TOKEN_TYPES = newSet{ "end", "else", "elseif", "until" } + +local function parseOneOrPossiblyMoreStatements(tokens, tokStart, statements) --> success, token, error -- The error message may be empty. + --[[ + stat ::= ';' + varlist '=' explist | + functioncall | + label | + break | + goto Name | + do block end | + while exp do block end | + repeat block until exp | + if exp then block {elseif exp then block} [else block] end | + for Name '=' exp ',' exp [',' exp] do block end | + for namelist in explist do block end | + function funcname funcbody | + local function Name funcbody | + local attnamelist ['=' explist] + + retstat ::= return [explist] [';'] + ]] + local tok = tokStart + local currentToken = tokens[tok] + + -- do + if isToken(currentToken, "keyword", "do") then + tok = tok + 1 -- 'do' + + local block, tokNext, err = parseBlock(tokens, tok, tok-1, true) + if not block then return false, tok, err end + block.token = tok - 1 + tok = tokNext + + if not isToken(tokens[tok], "keyword", "end") then + return false, tok, formatErrorAtToken(tokens[tok], "Parser", "Expected 'end' to end 'do' block (starting %s).", getRelativeLocationTextForToken(tokens, tokStart, tok)) + end + tok = tok + 1 -- 'end' + + tableInsert(statements, block) + return true, tok + + -- while + elseif isToken(currentToken, "keyword", "while") then + local whileLoop = AstWhile(currentToken) + tok = tok + 1 -- 'while' + + local expr, tokNext, err = parseExpressionInternal(tokens, tok, 0) + if not expr then return false, tok, err end + whileLoop.condition = expr + tok = tokNext + + if not isToken(tokens[tok], "keyword", "do") then + return false, tok, formatErrorAfterToken(tokens[tok-1], "Parser", "Expected 'do' to start body for 'while' loop.") + end + tok = tok + 1 -- 'do' + + local block, tokNext, err = parseBlock(tokens, tok, tok-1, true) + if not block then return false, tok, err end + block.token = tok - 1 + whileLoop.body = block + tok = tokNext + + if not isToken(tokens[tok], "keyword", "end") then + return false, tok, formatErrorAtToken(tokens[tok], "Parser", "Expected 'end' to end 'while' loop (starting %s).", getRelativeLocationTextForToken(tokens, tokStart, tok)) + end + tok = tok + 1 -- 'end' + + tableInsert(statements, whileLoop) + return true, tok + + -- repeat + elseif isToken(currentToken, "keyword", "repeat") then + local repeatLoop = AstRepeat(currentToken) + tok = tok + 1 -- 'repeat' + + local block, tokNext, err = parseBlock(tokens, tok, tok-1, true) + if not block then return false, tok, err end + repeatLoop.body = block + tok = tokNext + + if not isToken(tokens[tok], "keyword", "until") then + return false, tok, formatErrorAtToken(tokens[tok], "Parser", "Expected 'until' at the end of 'repeat' loop (starting %s).", getRelativeLocationTextForToken(tokens, tokStart, tok)) + end + tok = tok + 1 -- 'until' + + local expr, tokNext, err = parseExpressionInternal(tokens, tok, 0) + if not expr then return false, tok, err end + repeatLoop.condition = expr + tok = tokNext + + tableInsert(statements, repeatLoop) + return true, tok + + -- if + elseif isToken(currentToken, "keyword", "if") then + local ifNode = AstIf(currentToken) + tok = tok + 1 -- 'if' + + local expr, tokNext, err = parseExpressionInternal(tokens, tok, 0) + if not expr then return false, tok, err end + ifNode.condition = expr + tok = tokNext + + if not isToken(tokens[tok], "keyword", "then") then + return false, tok, formatErrorAtToken(tokens[tok], "Parser", "Expected 'then' after 'if' condition.") + end + tok = tok + 1 -- 'then' + + local block, tokNext, err = parseBlock(tokens, tok, tok-1, true) + if not block then return false, tok, err end + ifNode.bodyTrue = block + tok = tokNext + + local ifNodeLeaf = ifNode + + while isToken(tokens[tok], "keyword", "elseif") do + tok = tok + 1 -- 'elseif' + + ifNodeLeaf.bodyFalse = AstBlock(tokens[tok]) + ifNodeLeaf.bodyFalse.statements[1] = AstIf (tokens[tok]) + ifNodeLeaf = ifNodeLeaf.bodyFalse.statements[1] + + local expr, tokNext, err = parseExpressionInternal(tokens, tok, 0) + if not expr then return false, tok, err end + ifNodeLeaf.condition = expr + tok = tokNext + + if not isToken(tokens[tok], "keyword", "then") then + return false, tok, formatErrorAtToken(tokens[tok], "Parser", "Expected 'then' after 'elseif' condition.") + end + tok = tok + 1 -- 'then' + + local block, tokNext, err = parseBlock(tokens, tok, tok-1, true) + if not block then return false, tok, err end + ifNodeLeaf.bodyTrue = block + tok = tokNext + end + + if isToken(tokens[tok], "keyword", "else") then + tok = tok + 1 -- 'else' + + local block, tokNext, err = parseBlock(tokens, tok, tok-1, true) + if not block then return false, tok, err end + ifNodeLeaf.bodyFalse = block + tok = tokNext + end + + if not isToken(tokens[tok], "keyword", "end") then + return false, tok, formatErrorAtToken(tokens[tok], "Parser", "Expected 'end' to end 'if' statement (starting %s).", getRelativeLocationTextForToken(tokens, tokStart, tok)) + end + tok = tok + 1 -- 'end' + + tableInsert(statements, ifNode) + return true, tok + + -- for + elseif isToken(currentToken, "keyword", "for") then + local forLoop = AstFor(currentToken, "") + tok = tok + 1 -- 'for' + + local ok, tokNext, err = parseNameList(tokens, tok, forLoop.names, false, false) + if not ok then return false, tok, err end + tok = tokNext + + if isToken(tokens[tok], "keyword", "in") then + forLoop.kind = "generic" + tok = tok + 1 -- 'in' + + elseif isToken(tokens[tok], "punctuation", "=") then + if forLoop.names[2] then + return false, tok, formatErrorAtToken(tokens[tok], "Parser", "Expected 'in' for generic loop.") + end + + forLoop.kind = "numeric" + tok = tok + 1 -- '=' + + else + return false, tok, formatErrorAfterToken(tokens[tok-1], "Parser", "Expected '=' or 'in' for 'for' loop.") + end + + local valuesStartTok = tok + + local ok, tokNext, err = parseExpressionList(tokens, tok, forLoop.values, 0) + if not ok then return false, tok, err end + tok = tokNext + + if forLoop.kind ~= "numeric" then + -- void + elseif not forLoop.values[2] then + return false, tok, formatErrorAtToken(tokens[valuesStartTok], "Parser", "Numeric loop: Too few values.") + elseif forLoop.values[4] then + -- @Cleanup: Instead of using getLeftmostToken(), make parseExpressionList() return a list of expression start tokens. + return false, tok, formatErrorAtToken(getLeftmostToken(forLoop.values[4]), "Parser", "Numeric loop: Too many values.") + end + + if not isToken(tokens[tok], "keyword", "do") then + return false, tok, formatErrorAfterToken(tokens[tok-1], "Parser", "Expected 'do' to start body for 'for' loop.") + end + tok = tok + 1 -- 'do' + + local block, tokNext, err = parseBlock(tokens, tok, tok-1, true) + if not block then return false, tok, err end + forLoop.body = block + tok = tokNext + + if not isToken(tokens[tok], "keyword", "end") then + return false, tok, formatErrorAtToken(tokens[tok], "Parser", "Expected 'end' to end 'for' loop (starting %s).", getRelativeLocationTextForToken(tokens, tokStart, tok)) + end + tok = tok + 1 -- 'end' + + tableInsert(statements, forLoop) + return true, tok + + -- function + elseif isToken(currentToken, "keyword", "function") then + local funcTok = tok + local assignment = AstAssignment(currentToken) + tok = tok + 1 -- 'function' + + local targetExpr, tokNext, err = parseIdentifier(tokens, tok) + if not targetExpr then return false, tok, err end + tok = tokNext + + while isToken(tokens[tok], "punctuation", ".") do + local lookup = AstLookup(tokens[tok]) + tok = tok + 1 -- '.' + + local ident, tokNext, err = parseIdentifier(tokens, tok) + if not ident then return false, tok, err end + tok = tokNext + + local literal = AstLiteral(ident.token, ident.name) + lookup.member = literal + + lookup.object = targetExpr + lookup.member = literal + + targetExpr = lookup + end + + local isMethod = isToken(tokens[tok], "punctuation", ":") + + if isMethod then + local lookup = AstLookup(tokens[tok]) + tok = tok + 1 -- ':' + + local ident, tokNext, err = parseIdentifier(tokens, tok) + if not ident then return false, tok, err end + tok = tokNext + + local literal = AstLiteral(ident.token, ident.name) + lookup.member = literal + + lookup.object = targetExpr + lookup.member = literal + + targetExpr = lookup + end + + local func, tokNext, err = parseFunctionParametersAndBody(tokens, tok, funcTok) + if not func then return false, tok, err end + tok = tokNext + + if isMethod then + local ident = AstIdentifier(func.token, "self") + tableInsert(func.parameters, 1, ident) + end + + assignment.targets[1] = targetExpr + assignment.values[1] = func + + tableInsert(statements, assignment) + return true, tok + + -- local function + elseif isToken(currentToken, "keyword", "local") and isToken(tokens[tok+1], "keyword", "function") then + local funcTok = tok + 1 + local decl = AstDeclaration(currentToken) + local assignment = AstAssignment(currentToken) + tok = tok + 2 -- 'local function' + + local ident, tokNext, err = parseIdentifier(tokens, tok) + if not ident then return false, tok, err end + local identCopy = parseIdentifier(tokens, tok) + tok = tokNext + + local func, tokNext, err = parseFunctionParametersAndBody(tokens, tok, funcTok) + if not func then return false, tok, err end + tok = tokNext + + decl.names[1] = ident + assignment.targets[1] = identCopy + assignment.values[1] = func + + tableInsert(statements, decl) + tableInsert(statements, assignment) + return true, tok + + -- local + elseif isToken(currentToken, "keyword", "local") then + local decl = AstDeclaration(currentToken) + tok = tok + 1 -- 'local' + + local ok, tokNext, err = parseNameList(tokens, tok, decl.names, false, true) + if not ok then return false, tok, err end + tok = tokNext + + if isToken(tokens[tok], "punctuation", "=") then + tok = tok + 1 -- '=' + + local ok, tokNext, err = parseExpressionList(tokens, tok, decl.values) + if not ok then return false, tok, err end + tok = tokNext + end + + tableInsert(statements, decl) + return true, tok + + -- ::label:: + elseif isToken(currentToken, "punctuation", "::") then + local label = AstLabel(currentToken, "") + tok = tok + 1 -- '::' + + local labelIdent, tokNext, err = parseIdentifier(tokens, tok) + if not labelIdent then return false, tok, err end + tok = tokNext + + label.name = labelIdent.name + + if not isToken(tokens[tok], "punctuation", "::") then + return false, tok, formatErrorAfterToken(tokens[tok-1], "Parser", "Expected '::' after label name.") + end + tok = tok + 1 -- '::' + + tableInsert(statements, label) + return true, tok + + -- goto + elseif isToken(currentToken, "keyword", "goto") then + local gotoNode = AstGoto(currentToken, "") + tok = tok + 1 -- 'goto' + + local labelIdent, tokNext, err = parseIdentifier(tokens, tok) + if not labelIdent then return false, tok, err end + tok = tokNext + + gotoNode.name = labelIdent.name + + tableInsert(statements, gotoNode) + return true, tok + + -- break + elseif isToken(currentToken, "keyword", "break") then + local breakNode = AstBreak(currentToken) + tok = tok + 1 -- 'break' + + tableInsert(statements, breakNode) + return true, tok + + -- return (last) + elseif isToken(currentToken, "keyword", "return") then + local returnNode = AstReturn(currentToken) + tok = tok + 1 -- 'return' + + if tokens[tok] and not ( + (isTokenType(tokens[tok], "keyword") and isTokenAnyValue(tokens[tok], BLOCK_END_TOKEN_TYPES)) + or isToken(tokens[tok], "punctuation", ";") + or isTokenType(tokens[tok], "end") + ) then + local ok, tokNext, err = parseExpressionList(tokens, tok, returnNode.values) + if not ok then return false, tok, err end + tok = tokNext + end + + tableInsert(statements, returnNode) + return true, tok + + elseif isTokenType(currentToken, "keyword") then + return false, tok, "" + + else + local lookahead, tokNext, err = parseExpressionInternal(tokens, tok, 0) + if not lookahead then return false, tok, err end + + if lookahead.type == "call" then + local call = lookahead + tok = tokNext + + tableInsert(statements, call) + return true, tok + + elseif isToken(tokens[tokNext], "punctuation", "=") or isToken(tokens[tokNext], "punctuation", ",") then + local assignment = AstAssignment(tokens[tokNext]) + + local ok, tokNext, err = parseExpressionList(tokens, tok, assignment.targets) + if not ok then return false, tok, err end + tok = tokNext + + if not isToken(tokens[tok], "punctuation", "=") then + return false, tok, formatErrorAfterToken(tokens[tok-1], "Parser", "Expected '=' for an assignment.") + end + tok = tok + 1 -- '=' + + for _, targetExpr in ipairs(assignment.targets) do + if not (targetExpr.type == "identifier" or targetExpr.type == "lookup") then + return false, tok, formatErrorAtNode(targetExpr, "Parser", "Invalid assignment target.") + end + end + + local ok, tokNext, err = parseExpressionList(tokens, tok, assignment.values) + if not ok then return false, tok, err end + tok = tokNext + + tableInsert(statements, assignment) + return true, tok + + else + return false, tok, "" + end + end + + assert(false) +end + +local statementErrorReported = false + +--[[local]] function parseBlock(tokens, tok, blockTok, stopAtEndKeyword) --> block, token, error + local block = AstBlock(tokens[blockTok]) + local statements = block.statements + + while tok <= #tokens and tokens[tok].type ~= "end" do + while isToken(tokens[tok], "punctuation", ";") do + -- Empty statements are valid in Lua 5.2+. + tok = tok + 1 -- ';' + end + + local statementStartTok = tok + + if stopAtEndKeyword and isTokenType(tokens[tok], "keyword") and isTokenAnyValue(tokens[tok], BLOCK_END_TOKEN_TYPES) then + break + end + + local ok, tokNext, err = parseOneOrPossiblyMoreStatements(tokens, tok, statements) + if not ok then + if not statementErrorReported then + statementErrorReported = true + err = (err ~= "" and err.."\n" or "") .. formatErrorAtToken(tokens[tok], "Parser", "Failed parsing statement.") + end + return nil, tok, err + end + tok = tokNext + + if isToken(tokens[tok], "punctuation", ";") then + tok = tok + 1 -- ';' + end + + local lastAddedStatement = statements[#statements] + + if lastAddedStatement.type == "return" then -- Note: 'break' statements are allowed in the middle of blocks as of Lua 5.2. + break + + elseif lastAddedStatement.type == "call" and lastAddedStatement.adjustToOne then + statementErrorReported = true + return nil, tok, formatErrorAtToken(tokens[statementStartTok], "Parser", "Syntax error.") + end + end + + return block, tok +end + +-- block, error = tokensToAst( tokens, asBlock ) +local function tokensToAst(tokens, asBlock) + local tokensPurged = {} + local count = 0 + + do + -- Dummy start token. + local token = tokens[1] + count = count + 1 + + tokensPurged[count] = { + type = "start", + value = "", + representation = "", + + sourceString = token and token.sourceString or "", + sourcePath = token and token.sourcePath or "?", + + lineStart = 1, + lineEnd = 1, + positionStart = 1, + positionEnd = 1, + } + end + + -- Remove useless tokens. + for tok = 1, #tokens do + local tokType = tokens[tok].type + + if not (tokType == "comment" or tokType == "whitespace") then + count = count + 1 + tokensPurged[count] = tokens[tok] + end + end + + do + -- Dummy end token. + local token = tokens[#tokens] + local ln = token and 1+countString(token.sourceString, "\n", true) or 0 + local pos = token and #token.sourceString+1 or 0 + count = count + 1 + + tokensPurged[count] = { + type = "end", + value = "", + representation = "", + + sourceString = token and token.sourceString or "", + sourcePath = token and token.sourcePath or "?", + + lineStart = ln, + lineEnd = ln, + positionStart = pos, + positionEnd = pos, + } + end + + statementErrorReported = false + + local ast, _, err + if asBlock then + ast, _, err = parseBlock(tokensPurged, 2, 2, false) + else + ast, _, err = parseExpressionInternal(tokensPurged, 2, 0) + end + if not ast then return nil, err end + + return ast +end + +-- ast, error = parse( tokens ) +-- ast, error = parse( luaString [, pathForErrorMessages="?" ] ) +local function parse(luaOrTokens, path) + assertArg2("parse", 1, luaOrTokens, "string","table") + + -- ast, error = parse( tokens ) + if type(luaOrTokens) == "table" then + assertArg1("parse", 2, path, "nil") + + return tokensToAst(luaOrTokens, true) + + -- ast, error = parse( luaString, pathForErrorMessages ) + else + if path == nil then + path = "?" + else + assertArg1("parse", 2, path, "string") + end + + local tokens, err = tokenize(luaOrTokens, path) + if not tokens then return nil, err end + + return tokensToAst(tokens, true) + end +end + +-- ast, error = parseExpression( tokens ) +-- ast, error = parseExpression( luaString [, pathForErrorMessages="?" ] ) +local function parseExpression(luaOrTokens, path) + assertArg2("parseExpression", 1, luaOrTokens, "string","table") + + -- ast, error = parseExpression( tokens ) + if type(luaOrTokens) == "table" then + assertArg1("parseExpression", 2, path, "nil") + + return tokensToAst(luaOrTokens, false) + + -- ast, error = parseExpression( luaString, pathForErrorMessages ) + else + if path == nil then + path = "?" + else + assertArg1("parseExpression", 2, path, "string") + end + + local tokens, err = tokenize(luaOrTokens, path) + if not tokens then return nil, err end + + return tokensToAst(tokens, false) + end +end + +-- ast, error = parseFile( path ) +local function parseFile(path) + assertArg1("parseFile", 1, path, "string") + + local tokens, err = tokenizeFile(path) + if not tokens then return nil, err end + + return tokensToAst(tokens, true) +end + + + +local nodeConstructors = { + ["vararg"] = function() return AstVararg (nil) end, + ["table"] = function() return AstTable (nil) end, + ["lookup"] = function() return AstLookup (nil) end, + ["call"] = function() return AstCall (nil) end, + ["function"] = function() return AstFunction (nil) end, + ["break"] = function() return AstBreak (nil) end, + ["return"] = function() return AstReturn (nil) end, + ["block"] = function() return AstBlock (nil) end, + ["declaration"] = function() return AstDeclaration(nil) end, + ["assignment"] = function() return AstAssignment (nil) end, + ["if"] = function() return AstIf (nil) end, + ["while"] = function() return AstWhile (nil) end, + ["repeat"] = function() return AstRepeat (nil) end, + + ["identifier"] = function(argCount, name, attribute) + if argCount == 0 then + errorf(3, "Missing name argument for identifier.") + elseif type(name) ~= "string" then + errorf(3, "Invalid name argument value type '%s'. (Expected string)", type(name)) + elseif not stringFind(name, "^[%a_][%w_]*$") or KEYWORDS[name] then + errorf(3, "Invalid identifier name '%s'.", name) + end + + if attribute == nil or attribute == "" then + -- void + elseif type(attribute) ~= "string" then + errorf(3, "Invalid attribute argument value type '%s'. (Expected string)", type(attribute)) + elseif not (attribute == "close" or attribute == "const") then + errorf(3, "Invalid attribute name '%s'. (Must be 'close' or 'const'.)", attribute) + end + + local ident = AstIdentifier(nil, name) + ident.attribute = attribute or "" + return ident + end, + + ["label"] = function(argCount, name) + if argCount == 0 then + errorf(3, "Missing name argument for label.") + elseif type(name) ~= "string" then + errorf(3, "Invalid name argument value type '%s'. (Expected string)", type(name)) + elseif not stringFind(name, "^[%a_][%w_]*$") or KEYWORDS[name] then + errorf(3, "Invalid label name '%s'.", name) + end + + return AstLabel(nil, name) + end, + + ["goto"] = function(argCount, name) + if argCount == 0 then + errorf(3, "Missing label name argument for goto.") + elseif type(name) ~= "string" then + errorf(3, "Invalid label name argument value type '%s'. (Expected string)", type(name)) + elseif not stringFind(name, "^[%a_][%w_]*$") or KEYWORDS[name] then + errorf(3, "Invalid label name '%s'.", name) + end + + return AstGoto(nil, name) + end, + + ["literal"] = function(argCount, value) + if argCount == 0 then + errorf(3, "Missing value argument for literal.") + elseif not (type(value) == "number" or type(value) == "string" or type(value) == "boolean" or type(value) == "nil") then + errorf(3, "Invalid literal value type '%s'. (Expected number, string, boolean or nil)", type(value)) + end + + return AstLiteral(nil, value) + end, + + ["unary"] = function(argCount, op) + if argCount == 0 then + errorf(3, "Missing operator argument for unary expression.") + elseif not OPERATORS_UNARY[op] then + errorf(3, "Invalid unary operator '%s'.", tostring(op)) + end + + return AstUnary(nil, op) + end, + + ["binary"] = function(argCount, op) + if argCount == 0 then + errorf(3, "Missing operator argument for binary expression.") + elseif not OPERATORS_BINARY[op] then + errorf(3, "Invalid binary operator '%s'.", tostring(op)) + end + + return AstBinary(nil, op) + end, + + ["for"] = function(argCount, kind) + if argCount == 0 then + errorf(3, "Missing kind argument for 'for' loop.") + elseif not (kind == "numeric" or kind == "generic") then + errorf(3, "Invalid for loop kind '%s'. (Must be 'numeric' or 'generic')", tostring(kind)) + end + + return AstFor(nil, kind) + end, +} + +local nodeConstructorsFast = { + ["vararg"] = function() return AstVararg (nil) end, + ["table"] = function() return AstTable (nil) end, + ["lookup"] = function() return AstLookup (nil) end, + ["call"] = function() return AstCall (nil) end, + ["function"] = function() return AstFunction (nil) end, + ["break"] = function() return AstBreak (nil) end, + ["return"] = function() return AstReturn (nil) end, + ["block"] = function() return AstBlock (nil) end, + ["declaration"] = function() return AstDeclaration(nil) end, + ["assignment"] = function() return AstAssignment (nil) end, + ["if"] = function() return AstIf (nil) end, + ["while"] = function() return AstWhile (nil) end, + ["repeat"] = function() return AstRepeat (nil) end, + + ["label"] = function(name) return AstLabel (nil, name) end, + ["goto"] = function(name) return AstGoto (nil, name) end, + ["literal"] = function(value) return AstLiteral(nil, value) end, + ["unary"] = function(op) return AstUnary (nil, op) end, + ["binary"] = function(op) return AstBinary (nil, op) end, + ["for"] = function(kind) return AstFor (nil, kind) end, + + ["identifier"] = function(name, attribute) + local ident = AstIdentifier(nil, name) + ident.attribute = attribute or "" + return ident + end, +} + + + +-- +-- :NodeCreation +-- +-- identifier = newNode( "identifier", name [, attributeName="" ] ) -- 'attributeName' can be "close", "const" or "". +-- vararg = newNode( "vararg" ) +-- literal = newNode( "literal", value ) -- 'value' must be a number, a string, a boolean or nil. +-- tableNode = newNode( "table" ) +-- lookup = newNode( "lookup" ) +-- unary = newNode( "unary", unaryOperator ) +-- binary = newNode( "binary", binaryOperator ) +-- call = newNode( "call" ) +-- functionNode = newNode( "function" ) +-- breakNode = newNode( "break" ) +-- returnNode = newNode( "return" ) +-- label = newNode( "label", labelName ) +-- gotoNode = newNode( "goto", labelName ) +-- block = newNode( "block" ) +-- declaration = newNode( "declaration" ) +-- assignment = newNode( "assignment" ) +-- ifNode = newNode( "if" ) +-- whileLoop = newNode( "while" ) +-- repeatLoop = newNode( "repeat" ) +-- forLoop = newNode( "for", forLoopKind ) -- 'forLoopKind' can be "numeric" or "generic". +-- +-- Search for 'NodeFields' for each node's fields. +-- +local function newNode(nodeType, ...) + if nodeConstructors[nodeType] then + return (nodeConstructors[nodeType](select("#", ...), ...)) + else + errorf(2, "Invalid node type '%s'.", tostring(nodeType)) + end +end +local function newNodeFast(nodeType, ...) + return nodeConstructorsFast[nodeType](...) +end + +local cloneNodeArrayAndChildren + +local function cloneNodeAndMaybeChildren(node, cloneChildren) + local nodeType = node.type + local clone + + if nodeType == "identifier" then + clone = AstIdentifier(nil, node.name) + clone.attribute = node.attribute + + elseif nodeType == "vararg" then + clone = AstVararg(nil) + clone.adjustToOne = node.adjustToOne + + elseif nodeType == "literal" then + clone = AstLiteral(nil, node.value) + + elseif nodeType == "break" then + clone = AstBreak(nil) + + elseif nodeType == "label" then + clone = AstLabel(nil, node.name) + + elseif nodeType == "goto" then + clone = AstGoto(nil, node.name) + + elseif nodeType == "lookup" then + clone = AstLookup(nil) + + if cloneChildren then + clone.object = node.object and cloneNodeAndMaybeChildren(node.object, true) + clone.member = node.member and cloneNodeAndMaybeChildren(node.member, true) + end + + elseif nodeType == "unary" then + clone = AstUnary(nil, node.operator) + + if cloneChildren then + clone.expression = node.expression and cloneNodeAndMaybeChildren(node.expression, true) + end + + elseif nodeType == "binary" then + clone = AstBinary(nil, node.operator) + + if cloneChildren then + clone.left = node.left and cloneNodeAndMaybeChildren(node.left, true) + clone.right = node.right and cloneNodeAndMaybeChildren(node.right, true) + end + + elseif nodeType == "call" then + clone = AstCall(nil) + clone.method = node.method + clone.adjustToOne = node.adjustToOne + + if cloneChildren then + clone.callee = node.callee and cloneNodeAndMaybeChildren(node.callee, true) + cloneNodeArrayAndChildren(clone.arguments, node.arguments) + end + + elseif nodeType == "function" then + clone = AstFunction(nil) + + if cloneChildren then + cloneNodeArrayAndChildren(clone.parameters, node.parameters) + clone.body = node.body and cloneNodeAndMaybeChildren(node.body, true) + end + + elseif nodeType == "return" then + clone = AstReturn(nil) + + if cloneChildren then + cloneNodeArrayAndChildren(clone.values, node.values) + end + + elseif nodeType == "block" then + clone = AstBlock(nil) + + if cloneChildren then + cloneNodeArrayAndChildren(clone.statements, node.statements) + end + + elseif nodeType == "declaration" then + clone = AstDeclaration(nil) + + if cloneChildren then + cloneNodeArrayAndChildren(clone.names, node.names) + cloneNodeArrayAndChildren(clone.values, node.values) + end + + elseif nodeType == "assignment" then + clone = AstAssignment(nil) + + if cloneChildren then + cloneNodeArrayAndChildren(clone.targets, node.targets) + cloneNodeArrayAndChildren(clone.values, node.values) + end + + elseif nodeType == "if" then + clone = AstIf(nil) + + if cloneChildren then + clone.condition = node.condition and cloneNodeAndMaybeChildren(node.condition, true) + clone.bodyTrue = node.bodyTrue and cloneNodeAndMaybeChildren(node.bodyTrue, true) + clone.bodyFalse = node.bodyFalse and cloneNodeAndMaybeChildren(node.bodyFalse, true) + end + + elseif nodeType == "while" then + clone = AstWhile(nil) + + if cloneChildren then + clone.condition = node.condition and cloneNodeAndMaybeChildren(node.condition, true) + clone.body = node.body and cloneNodeAndMaybeChildren(node.body, true) + end + + elseif nodeType == "repeat" then + clone = AstRepeat(nil) + + if cloneChildren then + clone.body = node.body and cloneNodeAndMaybeChildren(node.body, true) + clone.condition = node.condition and cloneNodeAndMaybeChildren(node.condition, true) + end + + elseif nodeType == "for" then + clone = AstFor(nil, node.kind) + + if cloneChildren then + cloneNodeArrayAndChildren(clone.names, node.names) + cloneNodeArrayAndChildren(clone.values, node.values) + clone.body = node.body and cloneNodeAndMaybeChildren(node.body, true) + end + + elseif nodeType == "table" then + clone = AstTable(nil) + + if cloneChildren then + for i, tableField in ipairs(node.fields) do + clone.fields[i] = { + key = tableField.key and cloneNodeAndMaybeChildren(tableField.key, true), + value = tableField.value and cloneNodeAndMaybeChildren(tableField.value, true), + generatedKey = tableField.generatedKey, + } + end + end + + else + errorf("Invalid node type '%s'.", tostring(nodeType)) + end + + clone.pretty = node.pretty + clone.prefix = node.prefix + clone.suffix = node.suffix + -- Should we set node.token etc. too? @Incomplete + + return clone +end + +--[[local]] function cloneNodeArrayAndChildren(cloneArray, sourceArray) + for i, node in ipairs(sourceArray) do + cloneArray[i] = cloneNodeAndMaybeChildren(node, true) + end +end + +local function cloneNode(node) + return (cloneNodeAndMaybeChildren(node, false)) +end + +local function cloneTree(node) + return (cloneNodeAndMaybeChildren(node, true)) +end + + + +local INVOLVED_NEVER = newSet{ "function", "literal", "vararg" } +local INVOLVED_ALWAYS = newSet{ "break", "call", "goto", "label", "lookup", "return" } + +local mayAnyNodeBeInvolvedInJump + +local function mayNodeBeInvolvedInJump(node) + if INVOLVED_NEVER[node.type] then + return false + + elseif INVOLVED_ALWAYS[node.type] then + return true + + elseif node.type == "identifier" then + return (node.declaration == nil) -- Globals may invoke a metamethod on the environment. + + elseif node.type == "binary" then + return mayNodeBeInvolvedInJump(node.left) or mayNodeBeInvolvedInJump(node.right) + elseif node.type == "unary" then + return mayNodeBeInvolvedInJump(node.expression) + + elseif node.type == "block" then + return mayAnyNodeBeInvolvedInJump(node.statements) + + elseif node.type == "if" then + return mayNodeBeInvolvedInJump(node.condition) or mayNodeBeInvolvedInJump(node.bodyTrue) or (node.bodyFalse ~= nil and mayNodeBeInvolvedInJump(node.bodyFalse)) + + elseif node.type == "for" then + return mayAnyNodeBeInvolvedInJump(node.values) or mayNodeBeInvolvedInJump(node.body) + elseif node.type == "repeat" or node.type == "while" then + return mayNodeBeInvolvedInJump(node.condition) or mayNodeBeInvolvedInJump(node.body) + + elseif node.type == "declaration" then + return mayAnyNodeBeInvolvedInJump(node.values) + elseif node.type == "assignment" then + return mayAnyNodeBeInvolvedInJump(node.targets) or mayAnyNodeBeInvolvedInJump(node.values) -- Targets may be identifiers or lookups. + + elseif node.type == "table" then + for _, tableField in ipairs(node.fields) do + if mayNodeBeInvolvedInJump(tableField.key) then return true end + if mayNodeBeInvolvedInJump(tableField.value) then return true end + end + return false + + else + errorf("Invalid/unhandled node type '%s'.", tostring(node.type)) + end +end + +--[[local]] function mayAnyNodeBeInvolvedInJump(nodes) + for _, node in ipairs(nodes) do + if mayNodeBeInvolvedInJump(node) then return true end + end + return false +end + + + +local printNode, printTree +do + local function _printNode(node) + local nodeType = node.type + + ioWrite(nodeType) + + if parser.printIds then ioWrite("#", node.id) end + + -- if mayNodeBeInvolvedInJump(node) then ioWrite("[MAYJUMP]") end -- DEBUG + + if nodeType == "identifier" then + ioWrite(" (", node.name, ")") + + if node.declaration then + ioWrite(" (decl=", node.declaration.type) + if parser.printIds then ioWrite("#", node.declaration.id) end + ioWrite(")") + end + + elseif nodeType == "vararg" then + if node.adjustToOne then ioWrite(" (adjustToOne)") end + + if node.declaration then + ioWrite(" (decl=", node.declaration.type) + if parser.printIds then ioWrite("#", node.declaration.id) end + ioWrite(")") + end + + elseif nodeType == "literal" then + if node.value == nil or node.value == true or node.value == false then + ioWrite(" (", tostring(node.value), ")") + elseif type(node.value) == "string" then + ioWrite(' (string="', ensurePrintable(node.value), '")') + else + ioWrite(" (", type(node.value), "=", tostring(node.value), ")") + end + + elseif nodeType == "unary" then + ioWrite(" (", node.operator, ")") + + elseif nodeType == "binary" then + ioWrite(" (", node.operator, ")") + + elseif nodeType == "call" then + if node.method then ioWrite(" (method)" ) end + if node.adjustToOne then ioWrite(" (adjustToOne)") end + + elseif nodeType == "function" then + if node.parameters[1] and node.parameters[#node.parameters].type == "vararg" then ioWrite(" (vararg)") end + + elseif nodeType == "for" then + ioWrite(" (", node.kind, ")") + + elseif nodeType == "label" then + ioWrite(" (", node.name, ")") + + elseif nodeType == "goto" then + ioWrite(" (", node.name, ")") + + if node.label then + ioWrite(" (label") + if parser.printIds then ioWrite("#", node.label.id) end + ioWrite(")") + end + end + + if parser.printLocations then ioWrite(" @ ", node.sourcePath, ":", node.line) end + + ioWrite("\n") + end + + local function _printTree(node, indent, key) + for i = 1, indent do ioWrite(parser.indentation) end + indent = indent+1 + + if key ~= nil then + ioWrite(tostring(key), " ") + end + + _printNode(node) + + local nodeType = node.type + + if nodeType == "table" then + for i, tableField in ipairs(node.fields) do + if tableField.key then + _printTree(tableField.key, indent, i..(tableField.generatedKey and "KEYGEN" or "KEY ")) + elseif tableField.generatedKey then + for i = 1, indent do ioWrite(parser.indentation) end + ioWrite(i, "KEYGEN -\n") + end + if tableField.value then _printTree(tableField.value, indent, i.."VALUE ") end + end + + elseif nodeType == "lookup" then + if node.object then _printTree(node.object, indent, "OBJECT") end + if node.member then _printTree(node.member, indent, "MEMBER") end + + elseif nodeType == "unary" then + if node.expression then _printTree(node.expression, indent, nil) end + + elseif nodeType == "binary" then + if node.left then _printTree(node.left, indent, nil) end + for i = 1, indent do ioWrite(parser.indentation) end ; ioWrite(node.operator, "\n") + if node.right then _printTree(node.right, indent, nil) end + + elseif nodeType == "call" then + if node.callee then _printTree(node.callee, indent, "CALLEE") end + for i, expr in ipairs(node.arguments) do _printTree(expr, indent, "ARG"..i) end + + elseif nodeType == "function" then + for i, ident in ipairs(node.parameters) do _printTree(ident, indent, "PARAM"..i) end + if node.body then _printTree(node.body, indent, "BODY") end + + elseif nodeType == "return" then + for i, expr in ipairs(node.values) do _printTree(expr, indent, tostring(i)) end + + elseif nodeType == "block" then + for i, statement in ipairs(node.statements) do _printTree(statement, indent, tostring(i)) end + + elseif nodeType == "declaration" then + for i, ident in ipairs(node.names) do _printTree(ident, indent, "NAME"..i) end + for i, expr in ipairs(node.values) do _printTree(expr, indent, "VALUE"..i) end + + elseif nodeType == "assignment" then + for i, expr in ipairs(node.targets) do _printTree(expr, indent, "TARGET"..i) end + for i, expr in ipairs(node.values) do _printTree(expr, indent, "VALUE"..i) end + + elseif nodeType == "if" then + if node.condition then _printTree(node.condition, indent, "CONDITION") end + if node.bodyTrue then _printTree(node.bodyTrue, indent, "BODY" ) end + + local i = 1 + + while node.bodyFalse do + -- Automatically detect what looks like 'elseif'. + if #node.bodyFalse.statements == 1 and node.bodyFalse.statements[1].type == "if" then + i = i+1 + node = node.bodyFalse.statements[1] + + if node.condition then _printTree(node.condition, indent, "ELSEIF" ) end + if node.bodyTrue then _printTree(node.bodyTrue, indent, "BODY"..i) end + + else + _printTree(node.bodyFalse, indent, "ELSE") + break + end + end + + elseif nodeType == "while" then + if node.condition then _printTree(node.condition, indent, "CONDITION") end + if node.body then _printTree(node.body, indent, "BODY" ) end + + elseif nodeType == "repeat" then + if node.body then _printTree(node.body, indent, "BODY" ) end + if node.condition then _printTree(node.condition, indent, "CONDITION") end + + elseif nodeType == "for" then + for i, ident in ipairs(node.names) do _printTree(ident, indent, "NAME"..i) end + for i, expr in ipairs(node.values) do _printTree(expr, indent, "VALUE"..i) end + if node.body then _printTree(node.body, indent, "BODY") end + end + end + + --[[local]] function printNode(node) + _printNode(node) + end + + --[[local]] function printTree(node) + _printTree(node, 0, nil) + end +end + + + +-- didStop = traverseTree( astNode, [ leavesFirst=false, ] callback [, topNodeParent=nil, topNodeContainer=nil, topNodeKey=nil ] ) +-- action = callback( astNode, parent, container, key ) +-- action = "stop"|"ignorechildren"|nil -- Returning nil (or nothing) means continue traversal. +local function traverseTree(node, leavesFirst, cb, parent, container, k) + assertArg1("traverseTree", 1, node, "table") + + if type(leavesFirst) == "boolean" then + assertArg1("traverseTree", 3, cb, "function") + else + leavesFirst, cb, parent, container, k = false, leavesFirst, cb, parent, container + assertArg1("traverseTree", 2, cb, "function") + end + + if not leavesFirst then + local action = cb(node, parent, container, k) + if action == "stop" then return true end + if action == "ignorechildren" then return false end + if action then errorf("Unknown traversal action '%s' returned from callback.", tostring(action)) end + end + + local nodeType = node.type + + if nodeType == "identifier" or nodeType == "vararg" or nodeType == "literal" or nodeType == "break" or nodeType == "label" or nodeType == "goto" then + -- void No child nodes. + + elseif nodeType == "table" then + for _, tableField in ipairs(node.fields) do + if tableField.key and traverseTree(tableField.key, leavesFirst, cb, node, tableField, "key") then return true end + if tableField.value and traverseTree(tableField.value, leavesFirst, cb, node, tableField, "value") then return true end + end + + elseif nodeType == "lookup" then + if node.object and traverseTree(node.object, leavesFirst, cb, node, node, "object") then return true end + if node.member and traverseTree(node.member, leavesFirst, cb, node, node, "member") then return true end + + elseif nodeType == "unary" then + if node.expression and traverseTree(node.expression, leavesFirst, cb, node, node, "expression") then return true end + + elseif nodeType == "binary" then + if node.left and traverseTree(node.left, leavesFirst, cb, node, node, "left") then return true end + if node.right and traverseTree(node.right, leavesFirst, cb, node, node, "right") then return true end + + elseif nodeType == "call" then + if node.callee and traverseTree(node.callee, leavesFirst, cb, node, node, "callee") then return true end + for i, expr in ipairs(node.arguments) do + if traverseTree(expr, leavesFirst, cb, node, node.arguments, i) then return true end + end + + elseif nodeType == "function" then + for i, name in ipairs(node.parameters) do + if traverseTree(name, leavesFirst, cb, node, node.parameters, i) then return true end + end + if node.body and traverseTree(node.body, leavesFirst, cb, node, node, "body") then return true end + + elseif nodeType == "return" then + for i, expr in ipairs(node.values) do + if traverseTree(expr, leavesFirst, cb, node, node.values, i) then return true end + end + + elseif nodeType == "block" then + for i, statement in ipairs(node.statements) do + if traverseTree(statement, leavesFirst, cb, node, node.statements, i) then return true end + end + + elseif nodeType == "declaration" then + for i, ident in ipairs(node.names) do + if traverseTree(ident, leavesFirst, cb, node, node.names, i) then return true end + end + for i, expr in ipairs(node.values) do + if traverseTree(expr, leavesFirst, cb, node, node.values, i) then return true end + end + + elseif nodeType == "assignment" then + for i, expr in ipairs(node.targets) do + if traverseTree(expr, leavesFirst, cb, node, node.targets, i) then return true end + end + for i, expr in ipairs(node.values) do + if traverseTree(expr, leavesFirst, cb, node, node.values, i) then return true end + end + + elseif nodeType == "if" then + if node.condition and traverseTree(node.condition, leavesFirst, cb, node, node, "condition") then return true end + if node.bodyTrue and traverseTree(node.bodyTrue, leavesFirst, cb, node, node, "bodyTrue") then return true end + if node.bodyFalse and traverseTree(node.bodyFalse, leavesFirst, cb, node, node, "bodyFalse") then return true end + + elseif nodeType == "while" then + if node.condition and traverseTree(node.condition, leavesFirst, cb, node, node, "condition") then return true end + if node.body and traverseTree(node.body, leavesFirst, cb, node, node, "body") then return true end + + elseif nodeType == "repeat" then + if node.body and traverseTree(node.body, leavesFirst, cb, node, node, "body") then return true end + if node.condition and traverseTree(node.condition, leavesFirst, cb, node, node, "condition") then return true end + + elseif nodeType == "for" then + for i, ident in ipairs(node.names) do + if traverseTree(ident, leavesFirst, cb, node, node.names, i) then return true end + end + for i, expr in ipairs(node.values) do + if traverseTree(expr, leavesFirst, cb, node, node.values, i) then return true end + end + if node.body and traverseTree(node.body, leavesFirst, cb, node, node, "body") then return true end + + else + errorf("Invalid node type '%s'.", tostring(nodeType)) + end + + if leavesFirst then + local action = cb(node, parent, container, k) + if action == "stop" then return true end + if action == "ignorechildren" then errorf("Cannot ignore children when leavesFirst is set.") end + if action then errorf("Unknown traversal action '%s' returned from callback.", tostring(action)) end + end + + return false +end + +-- didStop = traverseTreeReverse( astNode, [ leavesFirst=false, ] callback [, topNodeParent=nil, topNodeContainer=nil, topNodeKey=nil ] ) +-- action = callback( astNode, parent, container, key ) +-- action = "stop"|"ignorechildren"|nil -- Returning nil (or nothing) means continue traversal. +local function traverseTreeReverse(node, leavesFirst, cb, parent, container, k) + assertArg1("traverseTreeReverse", 1, node, "table") + + if type(leavesFirst) == "boolean" then + assertArg1("traverseTreeReverse", 3, cb, "function") + else + leavesFirst, cb, parent, container, k = false, leavesFirst, cb, parent, container + assertArg1("traverseTreeReverse", 2, cb, "function") + end + + if not leavesFirst then + local action = cb(node, parent, container, k) + if action == "stop" then return true end + if action == "ignorechildren" then return false end + if action then errorf("Unknown traversal action '%s' returned from callback.", tostring(action)) end + end + + local nodeType = node.type + + if nodeType == "identifier" or nodeType == "vararg" or nodeType == "literal" or nodeType == "break" or nodeType == "label" or nodeType == "goto" then + -- void No child nodes. + + elseif nodeType == "table" then + for _, tableField in ipairsr(node.fields) do + if tableField.value and traverseTreeReverse(tableField.value, leavesFirst, cb, node, tableField, "value") then return true end + if tableField.key and traverseTreeReverse(tableField.key, leavesFirst, cb, node, tableField, "key") then return true end + end + + elseif nodeType == "lookup" then + if node.member and traverseTreeReverse(node.member, leavesFirst, cb, node, node, "member") then return true end + if node.object and traverseTreeReverse(node.object, leavesFirst, cb, node, node, "object") then return true end + + elseif nodeType == "unary" then + if node.expression and traverseTreeReverse(node.expression, leavesFirst, cb, node, node, "expression") then return true end + + elseif nodeType == "binary" then + if node.right and traverseTreeReverse(node.right, leavesFirst, cb, node, node, "right") then return true end + if node.left and traverseTreeReverse(node.left, leavesFirst, cb, node, node, "left") then return true end + + elseif nodeType == "call" then + for i, expr in ipairsr(node.arguments) do + if traverseTreeReverse(expr, leavesFirst, cb, node, node.arguments, i) then return true end + end + if node.callee and traverseTreeReverse(node.callee, leavesFirst, cb, node, node, "callee") then return true end + + elseif nodeType == "function" then + if node.body and traverseTreeReverse(node.body, leavesFirst, cb, node, node, "body") then return true end + for i, name in ipairsr(node.parameters) do + if traverseTreeReverse(name, leavesFirst, cb, node, node.parameters, i) then return true end + end + + elseif nodeType == "return" then + for i, expr in ipairsr(node.values) do + if traverseTreeReverse(expr, leavesFirst, cb, node, node.values, i) then return true end + end + + elseif nodeType == "block" then + for i, statement in ipairsr(node.statements) do + if traverseTreeReverse(statement, leavesFirst, cb, node, node.statements, i) then return true end + end + + elseif nodeType == "declaration" then + for i, expr in ipairsr(node.values) do + if traverseTreeReverse(expr, leavesFirst, cb, node, node.values, i) then return true end + end + for i, ident in ipairsr(node.names) do + if traverseTreeReverse(ident, leavesFirst, cb, node, node.names, i) then return true end + end + + elseif nodeType == "assignment" then + for i, expr in ipairsr(node.values) do + if traverseTreeReverse(expr, leavesFirst, cb, node, node.values, i) then return true end + end + for i, expr in ipairsr(node.targets) do + if traverseTreeReverse(expr, leavesFirst, cb, node, node.targets, i) then return true end + end + + elseif nodeType == "if" then + if node.bodyFalse and traverseTreeReverse(node.bodyFalse, leavesFirst, cb, node, node, "bodyFalse") then return true end + if node.bodyTrue and traverseTreeReverse(node.bodyTrue, leavesFirst, cb, node, node, "bodyTrue") then return true end + if node.condition and traverseTreeReverse(node.condition, leavesFirst, cb, node, node, "condition") then return true end + + elseif nodeType == "while" then + if node.body and traverseTreeReverse(node.body, leavesFirst, cb, node, node, "body") then return true end + if node.condition and traverseTreeReverse(node.condition, leavesFirst, cb, node, node, "condition") then return true end + + elseif nodeType == "repeat" then + if node.condition and traverseTreeReverse(node.condition, leavesFirst, cb, node, node, "condition") then return true end + if node.body and traverseTreeReverse(node.body, leavesFirst, cb, node, node, "body") then return true end + + elseif nodeType == "for" then + if node.body and traverseTreeReverse(node.body, leavesFirst, cb, node, node, "body") then return true end + for i, expr in ipairsr(node.values) do + if traverseTreeReverse(expr, leavesFirst, cb, node, node.values, i) then return true end + end + for i, ident in ipairsr(node.names) do + if traverseTreeReverse(ident, leavesFirst, cb, node, node.names, i) then return true end + end + + else + errorf("Invalid node type '%s'.", tostring(nodeType)) + end + + if leavesFirst then + local action = cb(node, parent, container, k) + if action == "stop" then return true end + if action == "ignorechildren" then errorf("Cannot ignore children when leavesFirst is set.") end + if action then errorf("Unknown traversal action '%s' returned from callback.", tostring(action)) end + end + + return false +end + + + +-- declIdent | nil = findIdentifierDeclaration( ident ) +-- declIdent.parent = decl | func | forLoop +local function findIdentifierDeclaration(ident) + local name = ident.name + local parent = ident + + while true do + local lastChild = parent + parent = parent.parent + + if not parent then return nil end + + if parent.type == "declaration" then + local decl = parent + + if lastChild.container == decl.names then + assert(lastChild == ident) + return ident -- ident is the declaration node. + end + + elseif parent.type == "function" then + local func = parent + + if lastChild.container == func.parameters then + assert(lastChild == ident) + return ident -- ident is the declaration node. + else + local func = parent + local declIdent = lastItemWith1(func.parameters, "name", name) -- Note: This will ignore any vararg parameter. + if declIdent then return declIdent end + end + + elseif parent.type == "for" then + local forLoop = parent + + if lastChild.container == forLoop.names then + assert(lastChild == ident) + return ident -- ident is the declaration node. + elseif lastChild.container ~= forLoop.values then + local declIdent = lastItemWith1(forLoop.names, "name", name) + if declIdent then return declIdent end + end + + elseif parent.type == "block" then + local block = parent + + -- Look through statements above lastChild. + for i = lastChild.key-1, 1, -1 do + local statement = block.statements[i] + + if statement.type == "declaration" then + local decl = statement + local declIdent = lastItemWith1(decl.names, "name", name) + if declIdent then return declIdent end + end + end + + elseif parent.type == "repeat" then + local repeatLoop = parent + + -- Repeat loop conditions can see into the loop block. + if lastChild == repeatLoop.condition then + local block = repeatLoop.body + + for i = #block.statements, 1, -1 do + local statement = block.statements[i] + + if statement.type == "declaration" then + local decl = statement + local declIdent = lastItemWith1(decl.names, "name", name) + if declIdent then return declIdent end + end + end + end + end + end +end + +-- declVararg|nil = findVarargDeclaration( vararg ) +-- declVararg.parent = func +local function findVarargDeclaration(vararg) + local parent = vararg + + while true do + parent = parent.parent + if not parent then return nil end + + if parent.type == "function" then + local lastParam = parent.parameters[#parent.parameters] + + if lastParam and lastParam.type == "vararg" then + return lastParam + else + return nil + end + end + end +end + +local function findLabel(gotoNode) + local name = gotoNode.name + local parent = gotoNode + + while true do + parent = parent.parent + if not parent then return nil end + + if parent.type == "block" then + local block = parent + + for _, statement in ipairs(block.statements) do + if statement.type == "label" and statement.name == name then + return statement + end + end + + elseif parent.type == "function" then + return nil + end + end +end + +local function updateReferences(node, updateTopNodePositionInfo) + local topNodeParent = nil + local topNodeContainer = nil + local topNodeKey = nil + + if updateTopNodePositionInfo == false then + topNodeParent = node.parent + topNodeContainer = node.container + topNodeKey = node.key + end + + traverseTree(node, function(node, parent, container, key) + node.parent = parent + node.container = container + node.key = key + + if node.type == "identifier" then + local ident = node + ident.declaration = findIdentifierDeclaration(ident) -- We can call this because all parents and previous nodes already have their references updated at this point. + + --[[ DEBUG + print(F( + "%-10s %-12s %s", + ident.name, + ( ident.declaration and ident.declaration.type or ""), + tostring(ident.declaration and ident.declaration.id or "") + )) + --]] + + elseif node.type == "vararg" then + local vararg = node + vararg.declaration = findVarargDeclaration(vararg) -- We can call this because all relevant 'parent' references have been updated at this point. + + --[[ DEBUG + print(F( + "%-10s %-12s %s", + "...", + ( vararg.declaration and vararg.declaration.type or ""), + tostring(vararg.declaration and vararg.declaration.id or "") + )) + --]] + + elseif node.type == "goto" then + local gotoNode = node + gotoNode.label = findLabel(gotoNode) -- We can call this because all relevant 'parent' references have been updated at this point. + end + end, topNodeParent, topNodeContainer, topNodeKey) +end + + + +local function replace(node, replacement, parent, container, key, stats) + tableInsert(stats.nodeReplacements, Location(container[key], "replacement", replacement)) + + container[key] = replacement + + replacement.sourceString = node.sourceString + replacement.sourcePath = node.sourcePath + + replacement.token = node.token + replacement.line = node.line + replacement.position = node.position + + replacement.parent = parent + replacement.container = container + replacement.key = key +end + + + +local PATTERN_INT_TO_HEX = F("%%0%dx", INT_SIZE/4) + +local HEX_DIGIT_TO_BITS = { + ["0"]={0,0,0,0}, ["1"]={0,0,0,1}, ["2"]={0,0,1,0}, ["3"]={0,0,1,1}, ["4"]={0,1,0,0}, ["5"]={0,1,0,1}, ["6"]={0,1,1,0}, ["7"]={0,1,1,1}, + ["8"]={1,0,0,0}, ["9"]={1,0,0,1}, ["a"]={1,0,1,0}, ["b"]={1,0,1,1}, ["c"]={1,1,0,0}, ["d"]={1,1,0,1}, ["e"]={1,1,1,0}, ["f"]={1,1,1,1}, +} + +local function intToBits(n, bitsOut) + local hexNumber = stringSub(F(PATTERN_INT_TO_HEX, maybeWrapInt(n)), -INT_SIZE/4) -- The stringSub() call is probably not needed, but just to be safe. + local i = 1 + + for hexDigit in stringGmatch(hexNumber, ".") do + local bits = HEX_DIGIT_TO_BITS[hexDigit] + + for iOffset = 1, 4 do + bitsOut[i] = bits[iOffset] + i = i + 1 + end + end +end + +local function bitsToInt(bits) + local n = 0 + local multi = 1 + + for i = INT_SIZE, 2, -1 do + n = n + multi * bits[i] + multi = multi * 2 + end + + return (bits[1] == 1 and MIN_INT+n or n) +end + +local function isValueNumberOrString(v) + return type(v) == "number" or type(v) == "string" +end +local function isValueFiniteNumber(v) -- Should we actually use the logic of isValueNumberLike() where this function is used? @Incomplete + return type(v) == "number" and v == v and v ~= 1/0 and v ~= -1/0 +end +local function isValueNumberLike(v) + return tonumber(v) ~= nil +end +local function areValuesNumbersOrStringsAndOfSameType(v1, v2) + local type1 = type(v1) + return type1 == type(v2) and (type1 == "number" or type1 == "string") +end + +local bits1 = {} +local bits2 = {} + +local unaryFolders = { + ["-"] = function(unary, expr) + -- -numberLike -> number + if expr.type == "literal" and isValueNumberLike(expr.value) then + return AstLiteral(unary.token, -expr.value) -- This may convert a string to a number. + end + + return nil + end, + + ["not"] = function(unary, expr) + -- not literal -> boolean + if expr.type == "literal" then -- :SimplifyTruthfulValues + return AstLiteral(unary.token, (not expr.value)) + + -- not (x == y) -> x ~= y + -- not (x ~= y) -> x == y + elseif expr.type == "binary" then + if expr.operator == "==" then + local binary = AstBinary(unary.token, "~=") + binary.left = expr.left + binary.right = expr.right + return binary + elseif expr.operator == "~=" then + local binary = AstBinary(unary.token, "==") + binary.left = expr.left + binary.right = expr.right + return binary + end + end + + return nil + end, + + ["#"] = function(unary, expr) + -- #string -> number + if expr.type == "literal" and type(expr.value) == "string" then + return AstLiteral(unary.token, #expr.value) + end + + -- We could get the length of tables containing only constants, but who in their right mind writes #{}? + + return nil + end, + + ["~"] = function(unary, expr) + -- ~number -> number + if expr.type == "literal" and isValueFiniteNumber(expr.value) then + intToBits(expr.value, bits1) + for i = 1, INT_SIZE do + bits1[i] = 1 - bits1[i] + end + return AstLiteral(unary.token, bitsToInt(bits1)) + end + + return nil + end, +} + +local binaryFolders = { + ["+"] = function(binary, l, r) + -- numberLike + numberLike -> number + if l.type == "literal" and r.type == "literal" and isValueNumberLike(l.value) and isValueNumberLike(r.value) then + return AstLiteral(binary.token, l.value+r.value) + end + + return nil + end, + + ["-"] = function(binary, l, r) + -- numberLike - numberLike -> number + if l.type == "literal" and r.type == "literal" and isValueNumberLike(l.value) and isValueNumberLike(r.value) then + return AstLiteral(binary.token, l.value-r.value) + end + + return nil + end, + + ["*"] = function(binary, l, r) + -- numberLike * numberLike -> number + if l.type == "literal" and r.type == "literal" and isValueNumberLike(l.value) and isValueNumberLike(r.value) then + return AstLiteral(binary.token, l.value*r.value) + end + + return nil + end, + + ["/"] = function(binary, l, r) + -- numberLike / numberLike -> number + if l.type == "literal" and r.type == "literal" and isValueNumberLike(l.value) and isValueNumberLike(r.value) then + return AstLiteral(binary.token, l.value/r.value) + end + + return nil + end, + + ["//"] = function(binary, l, r) + -- numberLike // numberLike -> number + if l.type == "literal" and r.type == "literal" and isValueNumberLike(l.value) and isValueNumberLike(r.value) then + return AstLiteral(binary.token, mathFloor(l.value/r.value)) + end + + return nil + end, + + ["^"] = function(binary, l, r) + -- numberLike ^ numberLike -> number + if l.type == "literal" and r.type == "literal" and isValueNumberLike(l.value) and isValueNumberLike(r.value) then + return AstLiteral(binary.token, l.value^r.value) + end + + return nil + end, + + ["%"] = function(binary, l, r) + -- numberLike % numberLike -> number + if l.type == "literal" and r.type == "literal" and isValueNumberLike(l.value) and isValueNumberLike(r.value) then + return AstLiteral(binary.token, l.value%r.value) + end + + return nil + end, + + ["&"] = function(binary, l, r) + -- number & number -> number + if l.type == "literal" and r.type == "literal" and isValueFiniteNumber(l.value) and isValueFiniteNumber(r.value) then + intToBits(l.value, bits1) + intToBits(r.value, bits2) + for i = 1, INT_SIZE do + bits1[i] = (bits1[i] == 1 and bits2[i] == 1) and 1 or 0 + end + return AstLiteral(binary.token, bitsToInt(bits1)) + end + + return nil + end, + + ["~"] = function(binary, l, r) + -- number ~ number -> number + if l.type == "literal" and r.type == "literal" and isValueFiniteNumber(l.value) and isValueFiniteNumber(r.value) then + intToBits(l.value, bits1) + intToBits(r.value, bits2) + for i = 1, INT_SIZE do + bits1[i] = (bits1[i] == 1) == (bits2[i] ~= 1) and 1 or 0 + end + return AstLiteral(binary.token, bitsToInt(bits1)) + end + + return nil + end, + + ["|"] = function(binary, l, r) + -- number | number -> number + if l.type == "literal" and r.type == "literal" and isValueFiniteNumber(l.value) and isValueFiniteNumber(r.value) then + intToBits(l.value, bits1) + intToBits(r.value, bits2) + for i = 1, INT_SIZE do + if bits1[i] == 0 then bits1[i] = bits2[i] end + end + return AstLiteral(binary.token, bitsToInt(bits1)) + end + + return nil + end, + + [">>"] = function(binary, l, r) + -- number >> number -> number + if l.type == "literal" and r.type == "literal" and isValueFiniteNumber(l.value) and type(r.value) == "number" then + intToBits(l.value, bits1) + + local shift = mathFloor(r.value) + local i1 = INT_SIZE + local i2 = 1 + local step = -1 + + if shift < 0 then + i1, i2 = i2, i1 + step = -step + end + + for i = i1, i2, step do + bits1[i] = bits1[i-shift] or 0 + end + + return AstLiteral(binary.token, bitsToInt(bits1)) + end + + return nil + end, + + ["<<"] = function(binary, l, r) + -- number << number -> number + if l.type == "literal" and r.type == "literal" and isValueFiniteNumber(l.value) and type(r.value) == "number" then + intToBits(l.value, bits1) + + local shift = mathFloor(r.value) + local i1 = 1 + local i2 = INT_SIZE + local step = 1 + + if shift < 0 then + i1, i2 = i2, i1 + step = -step + end + + for i = i1, i2, step do + bits1[i] = bits1[i+shift] or 0 + end + + return AstLiteral(binary.token, bitsToInt(bits1)) + end + + return nil + end, + + [".."] = function(binary, l, r) + -- numberOrString .. numberOrString -> string + if l.type == "literal" and r.type == "literal" and isValueNumberOrString(l.value) and isValueNumberOrString(r.value) then + return AstLiteral(binary.token, l.value..r.value) + end + + return nil + end, + + ["<"] = function(binary, l, r) + -- number < number -> boolean + -- string < string -> boolean + if l.type == "literal" and r.type == "literal" and areValuesNumbersOrStringsAndOfSameType(l.value, r.value) then + return AstLiteral(binary.token, (l.value < r.value)) + end + + return nil + end, + + ["<="] = function(binary, l, r) + -- number <= number -> boolean + -- string <= string -> boolean + if l.type == "literal" and r.type == "literal" and areValuesNumbersOrStringsAndOfSameType(l.value, r.value) then + return AstLiteral(binary.token, (l.value <= r.value)) + end + + return nil + end, + + [">"] = function(binary, l, r) + -- number > number -> boolean + -- string > string -> boolean + if l.type == "literal" and r.type == "literal" and areValuesNumbersOrStringsAndOfSameType(l.value, r.value) then + return AstLiteral(binary.token, (l.value > r.value)) + end + + return nil + end, + + [">="] = function(binary, l, r) + -- number >= number -> boolean + -- string >= string -> boolean + if l.type == "literal" and r.type == "literal" and areValuesNumbersOrStringsAndOfSameType(l.value, r.value) then + return AstLiteral(binary.token, (l.value >= r.value)) + end + + return nil + end, + + ["=="] = function(binary, l, r) + -- literal == literal -> boolean + if l.type == "literal" and r.type == "literal" then + return AstLiteral(binary.token, (l.value == r.value)) + end + + return nil + end, + + ["~="] = function(binary, l, r) + -- literal ~= literal -> boolean + if l.type == "literal" and r.type == "literal" then + return AstLiteral(binary.token, (l.value ~= r.value)) + end + + return nil + end, + + ["and"] = function(binary, l, r) + -- truthfulLiteral and x -> x + -- untruthfulLiteral and x -> untruthfulLiteral + if l.type == "literal" then return l.value and r or l end + + return nil + end, + + ["or"] = function(binary, l, r) + -- truthfulLiteral or x -> untruthfulLiteral + -- untruthfulLiteral or x -> x + if l.type == "literal" then return l.value and l or r end + + return nil + end, +} + +local statsForSimplify + +local function simplifyNode(node, parent, container, key) + if not parent then + -- void + + elseif node.type == "unary" then + -- Note: We don't fold e.g. '- - -expr' into '-expr' because metamethods may + -- be called, and folding '- -expr' into 'expr' would remove what could be a + -- runtime error if 'expr' didn't contain a number. + local replacement = unaryFolders[node.operator](node, node.expression) + if replacement then replace(node, replacement, parent, container, key, statsForSimplify) end + + elseif node.type == "binary" then + -- @Incomplete: Fold 'expr - -n' into 'expr + n' etc. (Actually, this will probably mess up metamethods.) + local replacement = binaryFolders[node.operator](node, node.left, node.right) + if replacement then replace(node, replacement, parent, container, key, statsForSimplify) end + + elseif node.type == "if" then + -- @Incomplete: Fold 'if not not expr' into 'if expr'. (Also for 'while' and 'repeat' etc., i.e. all conditional expressions.) + local ifNode = node + + if ifNode.condition.type == "literal" then -- @Incomplete: There are more values that make simplification possible (e.g. functions, but who would put that here anyway). :SimplifyTruthfulValues + local replacement = ifNode.condition.value and ifNode.bodyTrue or ifNode.bodyFalse + + if replacement and replacement.statements[1] then + replace(ifNode, replacement, parent, container, key, statsForSimplify) + return simplifyNode(replacement, parent, container, key) + else + tableRemove(container, key) + tableInsert(statsForSimplify.nodeRemovals, Location(ifNode)) + statsForSimplify.nodeRemoveCount = statsForSimplify.nodeRemoveCount + 1 + end + end + + elseif node.type == "while" then + local whileLoop = node + + if whileLoop.condition.type == "literal" then -- :SimplifyTruthfulValues + if whileLoop.condition.value then + whileLoop.condition.value = true -- Convert literal's value to boolean. + else + tableRemove(container, key) + tableInsert(statsForSimplify.nodeRemovals, Location(whileLoop)) + statsForSimplify.nodeRemoveCount = statsForSimplify.nodeRemoveCount + 1 + end + end + + elseif node.type == "repeat" then + local repeatLoop = node + + if repeatLoop.condition.type == "literal" then -- :SimplifyTruthfulValues + if repeatLoop.condition.value then + replace(repeatLoop, repeatLoop.body, parent, container, key, statsForSimplify) + return simplifyNode(repeatLoop.body, parent, container, key) + else + repeatLoop.condition.value = false -- Convert literal's value to boolean. + end + end + + elseif node.type == "for" then + local forLoop = node + + if + forLoop.kind == "numeric" + and forLoop.values[2] + and not forLoop.values[4] -- If this value exists then there will be an error when Lua tries to run the file, but it's not our job to handle that here. + and forLoop.values[1].type == "literal" and type(forLoop.values[1].value) == "number" + and forLoop.values[2].type == "literal" and type(forLoop.values[2].value) == "number" + and (not forLoop.values[3] or (forLoop.values[3].type == "literal" and type(forLoop.values[3].value) == "number")) + then + local from = forLoop.values[1].value + local to = forLoop.values[2].value + local step = forLoop.values[3] and forLoop.values[3].value or 1 + + if (step > 0 and from > to) or (step < 0 and from < to) then + tableRemove(container, key) + tableInsert(statsForSimplify.nodeRemovals, Location(forLoop)) + statsForSimplify.nodeRemoveCount = statsForSimplify.nodeRemoveCount + 1 + end + end + + elseif node.type == "block" then + if parent.type == "block" then + local block = node + local hasDeclarations = false + + for _, statement in ipairs(block.statements) do + if statement.type == "declaration" then + hasDeclarations = true + break + end + end + + if not hasDeclarations then + -- Blocks without declarations don't need a scope. + tableRemove(parent.statements, key) + tableInsert(statsForSimplify.nodeReplacements, Location(block, "replacement", nil)) -- We replace the block with its contents. + + for i, statement in ipairs(block.statements) do + tableInsert(parent.statements, key+i-1, statement) + end + end + end + + elseif node.type == "literal" then + local literal = node + if literal.value == 0 then literal.value = 0 end -- Normalize '-0'. + end +end + +local function _simplify(node, stats) + statsForSimplify = stats + traverseTreeReverse(node, true, simplifyNode) +end + +local function simplify(node) + local stats = Stats() + _simplify(node, stats) + return stats +end + + + +local function isNodeDeclLike(node) + return node.type == "declaration" or node.type == "function" or node.type == "for" +end + +local function getNameArrayOfDeclLike(declLike) + return declLike.names or declLike.parameters +end + + + +-- (statement, block) | declIdent | repeatLoop | declLike = findParentStatementAndBlockOrNodeOfInterest( node, declIdent ) +local function findParentStatementAndBlockOrNodeOfInterest(node, declIdent) + while true do + if node == declIdent then return declIdent, nil end + + local lastChild = node + node = node.parent + + if not node then return nil end + + if node.type == "block" then return lastChild, node end + if node.type == "declaration" and lastChild.container ~= node.values then return node, nil end + if node.type == "function" then return node, nil end + if node.type == "for" and lastChild.container ~= node.values then return node, nil end + if node.type == "repeat" and lastChild == node.condition then return node, nil end + end +end + +-- foundCurrentDeclIdent = lookForCurrentDeclIdentAndRegisterCurrentIdentAsWatcherInBlock( declIdentWatchers, currentIdentInfo, block, statementStartIndex ) +local function lookForCurrentDeclIdentAndRegisterCurrentIdentAsWatcherInBlock(declIdentWatchers, identInfo, block, iStart) + local statements = block.statements + local currentIdentOrVararg = identInfo.ident + local currentDeclIdent = currentIdentOrVararg.declaration + + for i = iStart, 1, -1 do + local statement = statements[i] + + if statement.type == "declaration" then + local decl = statement + + for _, declIdent in ipairsr(decl.names) do + -- Note: Declaration identifiers also watch themselves. (Is this good?) :DeclarationIdentifiersWatchThemselves + declIdentWatchers[declIdent] = declIdentWatchers[declIdent] or {} + tableInsert(declIdentWatchers[declIdent], currentIdentOrVararg) + + if declIdent == currentDeclIdent then return true end + end + end + end + + return false +end + +local function getInformationAboutIdentifiersAndUpdateReferences(node) + local identInfos = {--[[ [ident1]=identInfo1, identInfo1, ... ]]} -- identInfo = {ident=identOrVararg, type=lrvalueType} + local declIdentWatchers = {--[[ [declIdent1]={identOrVararg1,...}, ... ]]} + + traverseTree(node, function(node, parent, container, key) + node.parent = parent + node.container = container + node.key = key + + if node.type == "identifier" or node.type == "vararg" then + local currentIdentOrVararg = node + local findDecl = (currentIdentOrVararg.type == "identifier") and findIdentifierDeclaration or findVarargDeclaration + local currentDeclIdent = findDecl(currentIdentOrVararg) -- We can call this because all parents and previous nodes already have their references updated at this point. + currentIdentOrVararg.declaration = currentDeclIdent + + local identType = ( + (parent and ( + (parent.type == "declaration" and (container == parent.names )) or + (parent.type == "assignment" and (container == parent.targets )) or + (parent.type == "function" and (container == parent.parameters)) or + (parent.type == "for" and (container == parent.names )) + )) + and "lvalue" + or "rvalue" + ) + + -- if currentDeclIdent then print(F("%s:%d: %s '%s'", currentIdentOrVararg.sourcePath, currentIdentOrVararg.line, identType, (currentIdentOrVararg.name or "..."))) end -- DEBUG + + local identInfo = {ident=currentIdentOrVararg, type=identType} + tableInsert(identInfos, identInfo) + identInfos[currentIdentOrVararg] = identInfo + + -- Determine visible declarations for the identifier. + local block = currentIdentOrVararg -- Start node for while loop. + + while true do + local statementOrInterest + statementOrInterest, block = findParentStatementAndBlockOrNodeOfInterest(block, currentDeclIdent) + + if not statementOrInterest then + -- We should only get here for globals (i.e. there should be no declaration). + assert(not currentDeclIdent) + return + end + + if block then + local statement = statementOrInterest + + assert(type(statement.key) == "number") + assert(statement.container == block.statements) + + if lookForCurrentDeclIdentAndRegisterCurrentIdentAsWatcherInBlock(declIdentWatchers, identInfo, block, statement.key-1) then + return + end + + -- We found the current declIdent - don't look for more other declIdents to watch! + elseif statementOrInterest == currentDeclIdent then + local declIdent = statementOrInterest + + -- :DeclarationIdentifiersWatchThemselves + declIdentWatchers[declIdent] = declIdentWatchers[declIdent] or {} + tableInsert(declIdentWatchers[declIdent], currentIdentOrVararg) + + return + + -- We can jump from repeat loop conditions into the loop's body. + elseif statementOrInterest.type == "repeat" then + local repeatLoop = statementOrInterest + block = repeatLoop.body -- Start node for while loop. + + if lookForCurrentDeclIdentAndRegisterCurrentIdentAsWatcherInBlock(declIdentWatchers, identInfo, block, #block.statements) then + return + end + + -- Declaration-likes have names we want to watch. + -- Note that findParentStatementAndBlockOrNodeOfInterest() is smart and skipped declaration-likes that we should completely ignore. + else + local declLike = statementOrInterest + block = declLike -- Start node for while loop. + + for _, declIdent in ipairsr(getNameArrayOfDeclLike(declLike)) do + -- :DeclarationIdentifiersWatchThemselves + declIdentWatchers[declIdent] = declIdentWatchers[declIdent] or {} + tableInsert(declIdentWatchers[declIdent], currentIdentOrVararg) + + if declIdent == currentDeclIdent then return end + end + end + end + + elseif node.type == "goto" then + local gotoNode = node + gotoNode.label = findLabel(gotoNode) -- We can call this because all relevant 'parent' references have been updated at this point. + end + end) + + return identInfos, declIdentWatchers +end + + + +local function unregisterWatchersBeforeNodeRemoval(identInfos, declIdentWatchers, theNode, declIdentReadCount, declIdentAssignmentCount, funcInfos, currentFuncInfo, stats, registerRemoval) + -- ioWrite("unregister ") ; printNode(theNode) -- DEBUG + + if registerRemoval then + tableInsert(stats.nodeRemovals, Location(theNode)) -- There's no need to register the location every child node. + stats.nodeRemoveCount = stats.nodeRemoveCount - 1 -- To counteract the +1 below. + end + + traverseTree(theNode, true, function(node) -- @Speed + --[[ DEBUG + unregistered = unregistered or {} + if unregistered[node] then + printNode(node) + print(debug.traceback("NEW", 2)) + print(unregistered[node]) + end + unregistered[node] = debug.traceback("OLD", 2) + --]] + + stats.nodeRemoveCount = stats.nodeRemoveCount + 1 + + if node.type == "identifier" or node.type == "vararg" then + local currentIdentOrVararg = node + local currentIdentInfo = identInfos[currentIdentOrVararg] + assert(currentIdentInfo) + + -- Remove identifier info. + for i, identInfo in ipairs(identInfos) do + if identInfo == currentIdentInfo then + removeUnordered(identInfos, i) + break + end + end + identInfos[currentIdentOrVararg] = nil + + -- Remove as watcher. + for _, watcherIdents in pairs(declIdentWatchers) do -- @Speed + for i, watcherIdent in ipairs(watcherIdents) do + if watcherIdent == currentIdentOrVararg then + removeUnordered(watcherIdents, i) + break + end + end + end + + -- Remove watcher list. + if declIdentWatchers[currentIdentOrVararg] then + declIdentWatchers[currentIdentOrVararg] = nil + removeItemUnordered(currentFuncInfo.declIdents, currentIdentOrVararg) + end + + -- Update access count. + if not currentIdentOrVararg.declaration then + -- void + elseif currentIdentInfo.type == "rvalue" then + local declIdent = currentIdentOrVararg.declaration + declIdentReadCount[declIdent] = declIdentReadCount[declIdent] - 1 -- :AccessCount + assert(declIdentReadCount[declIdent] >= 0) + elseif --[[currentIdentInfo.type == "lvalue" and]] currentIdentInfo.ident.parent.type == "assignment" then + local declIdent = currentIdentOrVararg.declaration + declIdentAssignmentCount[declIdent] = declIdentAssignmentCount[declIdent] - 1 -- :AccessCount + assert(declIdentAssignmentCount[declIdent] >= 0) + end + + -- removeItemUnordered(currentFuncInfo.locals, currentIdentOrVararg) + -- removeItemUnordered(currentFuncInfo.upvalues, currentIdentOrVararg) + -- removeItemUnordered(currentFuncInfo.globals, currentIdentOrVararg) + + elseif node.type == "assignment" then + removeItemUnordered(currentFuncInfo.assignments, node) + + elseif isNodeDeclLike(node) then + removeItemUnordered(currentFuncInfo.declLikes, node) + end + + if funcInfos[node] then + for i, funcInfo in ipairs(funcInfos) do + if funcInfo.node == node then + removeUnordered(funcInfos, i) + break + end + end + funcInfos[node] = nil + end + end) +end + +local function isNodeValueList(node) + return (node.type == "call" or node.type == "vararg") and not node.adjustToOne +end + +local function areAllLvaluesUnwantedAndAllValuesCalls(lvalues, values, wantToRemoveLvalue) + if not values[1] then return false end -- Need at least one call. + + for slot = 1, #lvalues do + if not wantToRemoveLvalue[slot] then return false end + end + + for _, valueExpr in ipairs(values) do + if valueExpr.type ~= "call" then return false end + end + + return true +end + +local function getInformationAboutFunctions(theNode) + -- Gather functions. + local funcInfos = {} + + do + -- We assume theNode is a block, but it's fine if it isn't. + local funcInfo = {node=theNode, declLikes={}, declIdents={}, assignments={}--[[, locals={}, upvalues={}, globals={}]]} + tableInsert(funcInfos, funcInfo) + funcInfos[theNode] = funcInfo + end + + traverseTree(theNode, function(node) + if node == theNode then return end + + if node.type == "function" then + local funcInfo = {node=node, declLikes={node}, declIdents={}, assignments={}--[[, locals={}, upvalues={}, globals={}]]} + tableInsert(funcInfos, funcInfo) + funcInfos[node] = funcInfo + end + end) + + -- Gather relevant nodes. + for _, funcInfo in ipairs(funcInfos) do + traverseTree(funcInfo.node, function(node) + if node == funcInfo.node then return end + if node.type == "function" then return "ignorechildren" end + + if node.type == "identifier" or node.type == "vararg" then + local identOrVararg = node + local declIdent = identOrVararg.declaration + + if identOrVararg == declIdent then + tableInsert(funcInfo.declIdents, identOrVararg) + end + + --[[ + if declIdent then + local declLike = declIdent.parent + local isInFunc = true + local parent = identOrVararg.parent + + while parent do + if parent == declLike then -- declLike may be a function itself. + break + elseif parent.type == "function" then + isInFunc = false + break + end + parent = parent.parent + end + + tableInsert((isInFunc and funcInfo.locals or funcInfo.upvalues), identOrVararg) + + else + tableInsert(funcInfo.globals, identOrVararg) + end + ]] + + elseif node.type == "declaration" or node.type == "for" then + tableInsert(funcInfo.declLikes, node) -- Note: Identifiers will be added to funcInfo.declIdents when we get to them. + + elseif node.type == "assignment" then + tableInsert(funcInfo.assignments, node) + end + end) + + --[[ DEBUG + print("--------------") + printNode(funcInfo.node) + for i, ident in ipairs(funcInfo.locals) do + ioWrite("local ", i, " ") ; printNode(ident) + end + for i, ident in ipairs(funcInfo.upvalues) do + ioWrite("upvalue ", i, " ") ; printNode(ident) + end + for i, ident in ipairs(funcInfo.globals) do + ioWrite("global ", i, " ") ; printNode(ident) + end + end + print("--------------") + --[==[]] + end + --]==] + + return funcInfos +end + +local function getAccessesOfDeclaredNames(funcInfos, identInfos, declIdentWatchers) -- @Cleanup: Don't require the funcInfos argument (unless it maybe speeds up something somwhere). + local declIdentReadCount = {--[[ [declIdent1]=count, ... ]]} + local declIdentAssignmentCount = {--[[ [declIdent1]=count, ... ]]} + + for _, funcInfo in ipairs(funcInfos) do + for _, declIdent in ipairs(funcInfo.declIdents) do + local readCount = 0 + local assignmentCount = 0 + + for _, watcherIdent in ipairs(declIdentWatchers[declIdent]) do + if watcherIdent.declaration == declIdent then + local identInfo = identInfos[watcherIdent] + + if identInfo.type == "rvalue" then + readCount = readCount + 1 -- :AccessCount + elseif --[[identInfo.type == "lvalue" and]] identInfo.ident.parent.type == "assignment" then + assignmentCount = assignmentCount + 1 -- :AccessCount + end + end + end + + declIdentReadCount [declIdent] = readCount + declIdentAssignmentCount[declIdent] = assignmentCount + end + end + + return declIdentReadCount, declIdentAssignmentCount +end + +-- Note: References need to be updated after calling this! +local function _optimize(theNode, stats) + _simplify(theNode, stats) + + local identInfos, declIdentWatchers = getInformationAboutIdentifiersAndUpdateReferences(theNode) + local funcInfos = getInformationAboutFunctions(theNode) + local declIdentReadCount, declIdentAssignmentCount = getAccessesOfDeclaredNames(funcInfos, identInfos, declIdentWatchers) + + -- + -- Replace variables that are effectively constants with literals. + -- + local replacedConstants = false + + for _, funcInfo in ipairs(funcInfos) do + for _, declLike in ipairs(funcInfo.declLikes) do + if declLike.type == "declaration" then + local decl = declLike + + for slot = 1, #decl.names do + local declIdent = decl.names[slot] + local valueExpr = decl.values[slot] + + if + declIdentAssignmentCount[declIdent] == 0 + and declIdentReadCount[declIdent] > 0 + and ( + (not valueExpr and not (decl.values[1] and isNodeValueList(decl.values[#decl.values]))) + or ( + valueExpr + and valueExpr.type == "literal" + and not ( + (type(valueExpr.value) == "string" and #valueExpr.value > parser.constantNameReplacementStringMaxLength) + -- or (valueExpr.value == 0 and not NORMALIZE_MINUS_ZERO and tostring(valueExpr.value) == "-0") -- No, bad rule! + ) + ) + ) + then + -- where(declIdent, "Constant declaration.") -- DEBUG + + local valueLiteral = valueExpr + local valueIsZero = (valueLiteral ~= nil and valueLiteral.value == 0) + + for _, watcherIdent in ipairsr(declIdentWatchers[declIdent]) do + if watcherIdent.declaration == declIdent then -- Note: declIdent is never a vararg here (because we only process declarations). + local identInfo = identInfos[watcherIdent] + + if + identInfo.type == "rvalue" + -- Avoid creating '-0' (or '- -0') here as that may mess up Lua in weird/surprising ways. + and not (valueIsZero and not NORMALIZE_MINUS_ZERO and watcherIdent.parent.type == "unary" and watcherIdent.parent.operator == "-") + then + -- where(watcherIdent, "Constant value replacement.") -- DEBUG + + local v = valueLiteral and valueLiteral.value + local replacementLiteral = AstLiteral(watcherIdent.token, v) + + unregisterWatchersBeforeNodeRemoval(identInfos, declIdentWatchers, watcherIdent, declIdentReadCount, declIdentAssignmentCount, funcInfos, funcInfo, stats, false) + replace(watcherIdent, replacementLiteral, watcherIdent.parent, watcherIdent.container, watcherIdent.key, stats) + + replacedConstants = true + end + end + end--for declIdentWatchers + end + end--for declIdents + end + end--for declLikes + end--for funcInfos + + if replacedConstants then + return _optimize(theNode, stats) -- @Speed + end + + -- + -- Remove useless assignments and declaration-likes (in that order). + -- + -- Note that we go in reverse order almost everywhere! We may remove later stuff when we reach earlier stuff. + -- + local function optimizeAssignmentOrDeclLike(statement, funcInfo) + local isDecl = (statement.type == "declaration") + local isFunc = (statement.type == "function") + local isForLoop = (statement.type == "for") + local isAssignment = (statement.type == "assignment") + + if isDecl or isAssignment then + local lvalues = statement.targets or statement.names + local values = statement.values + + -- Save some adjustment information. + local madeToAdjusted = {} + + for slot = 1, #values-1 do -- Skip the last value. + local valueExpr = values[slot] + + if isNodeValueList(valueExpr) then + valueExpr.adjustToOne = true + madeToAdjusted[valueExpr] = true + end + end + + -- Remove useless extra values. + for slot = #values, #lvalues+1, -1 do + local valueExpr = values[slot] + + if not mayNodeBeInvolvedInJump(valueExpr) then + unregisterWatchersBeforeNodeRemoval(identInfos, declIdentWatchers, valueExpr, declIdentReadCount, declIdentAssignmentCount, funcInfos, funcInfo, stats, true) + tableRemove(values, slot) + end + end + + for slot = #lvalues+1, #values do + values[slot].key = slot + end + + -- Remove useless lvalues. + local wantToRemoveLvalue = {} + local wantToRemoveValueIfExists = {} + local mayRemoveValueIfExists = {} + + for slot, lvalue in ipairsr(lvalues) do + local declIdent = (lvalue.type == "identifier") and lvalue.declaration or nil + + if declIdent and declIdentReadCount[declIdent] == 0 then + -- ioWrite("useless ") ; printNode(lvalue) -- DEBUG + + local valueExpr = values[slot] + local valueExprEffective = valueExpr + + if not valueExprEffective then + valueExprEffective = values[#values] + if valueExprEffective and not isNodeValueList(valueExprEffective) then valueExprEffective = nil end + end + + wantToRemoveLvalue [slot] = isAssignment or declIdentAssignmentCount[declIdent] == 0 + wantToRemoveValueIfExists[slot] = not (valueExpr and mayNodeBeInvolvedInJump(valueExpr)) + mayRemoveValueIfExists [slot] = wantToRemoveValueIfExists[slot] and not (not valueExpr and lvalues[slot+1] and valueExprEffective) + + -- @Incomplete: Update funcInfo.locals and whatever else (if we end up using them at some point). + -- @Incomplete: Replace 'unused,useless=func()' with 'useless=func()'. + local canRemoveSlot = (wantToRemoveLvalue[slot] and mayRemoveValueIfExists[slot]) + + -- Maybe remove lvalue. + if canRemoveSlot and #lvalues > 1 then + unregisterWatchersBeforeNodeRemoval(identInfos, declIdentWatchers, lvalue, declIdentReadCount, declIdentAssignmentCount, funcInfos, funcInfo, stats, true) + tableRemove(lvalues, slot) + for slot = slot, #lvalues do + lvalues[slot].key = slot + end + wantToRemoveLvalue[slot] = wantToRemoveLvalue[slot+1] -- May become nil. We no longer care about the value of slot+1 and beyond after this point. + end + + -- Maybe remove value. + if wantToRemoveValueIfExists[slot] and valueExpr then + if (canRemoveSlot or not values[slot+1]) and not (isAssignment and not values[2]) then + unregisterWatchersBeforeNodeRemoval(identInfos, declIdentWatchers, valueExpr, declIdentReadCount, declIdentAssignmentCount, funcInfos, funcInfo, stats, true) + tableRemove(values, slot) + for slot = slot, #values do + values[slot].key = slot + end + wantToRemoveValueIfExists[slot] = wantToRemoveValueIfExists[slot+1] -- May become nil. We no longer care about the value of slot+1 and beyond after this point. + mayRemoveValueIfExists [slot] = mayRemoveValueIfExists [slot+1] -- May become nil. We no longer care about the value of slot+1 and beyond after this point. + + elseif not (valueExpr.type == "literal" and valueExpr.value == nil) then + unregisterWatchersBeforeNodeRemoval(identInfos, declIdentWatchers, valueExpr, declIdentReadCount, declIdentAssignmentCount, funcInfos, funcInfo, stats, false) + replace(valueExpr, AstLiteral(valueExpr.token, nil), valueExpr.parent, valueExpr.container, valueExpr.key, stats) + end + end + end--if lvalue is relevant + end--for lvalues + + -- Maybe remove or replace the whole statement. + local statementIsRemoved = false + + do + -- Remove the whole statement. + if + wantToRemoveLvalue[1] + and ( + mayRemoveValueIfExists[1] + or not (isAssignment or values[1]) -- Declaration-likes may have no value - assignments must have at least one. + ) + and not (lvalues[2] or values[2]) + then + unregisterWatchersBeforeNodeRemoval(identInfos, declIdentWatchers, statement, declIdentReadCount, declIdentAssignmentCount, funcInfos, funcInfo, stats, true) + + local block = statement.parent + + for i = statement.key, #block.statements do + local statement = block.statements[i+1] -- This be nil for the last 'i'. + block.statements[i] = statement + + if statement then statement.key = i end + end + + statementIsRemoved = true + + -- Replace 'unused=func()' with just 'func()'. This is a unique case as call expressions can also be statements. + elseif areAllLvaluesUnwantedAndAllValuesCalls(lvalues, values, wantToRemoveLvalue) then + for _, lvalue in ipairs(lvalues) do + unregisterWatchersBeforeNodeRemoval(identInfos, declIdentWatchers, lvalue, declIdentReadCount, declIdentAssignmentCount, funcInfos, funcInfo, stats, true) + end + + tableRemove(statement.container, statement.key) -- The parent ought to be a block! + + for slot, call in ipairs(values) do + local i = statement.key + slot - 1 + tableInsert(statement.container, i, call) + + call.parent = statement.parent + call.container = statement.container + call.key = i + end + + statementIsRemoved = true + end + end + + -- Restore or remove adjusted flags. + -- @Speed: Don't do anything if statementIsRemoved is set (and we don't need to for some other reason). + for i = 1, #values do + local valueExpr = values[i] + if (valueExpr.type == "call" or valueExpr.type == "vararg") then + if statementIsRemoved or values[i+1] or not lvalues[i+1] or not madeToAdjusted[valueExpr] then + valueExpr.adjustToOne = false + end + end + end + + elseif isFunc or isForLoop then + local declIdents = getNameArrayOfDeclLike(statement) + + for slot = #declIdents, (isForLoop and 2 or 1), -1 do + local declIdent = declIdents[slot] + if declIdentReadCount[declIdent] == 0 and declIdentAssignmentCount[declIdent] == 0 then + unregisterWatchersBeforeNodeRemoval(identInfos, declIdentWatchers, declIdent, declIdentReadCount, declIdentAssignmentCount, funcInfos, funcInfo, stats, true) + tableRemove(declIdents) + else + break + end + end + + for slot, declIdent in ipairs(declIdents) do + declIdent.key = slot + end + + else + error(statement.type) + end + end + + for _, funcInfo in ipairsr(funcInfos) do + for _, assignment in ipairsr(funcInfo.assignments) do + optimizeAssignmentOrDeclLike(assignment, funcInfo) + end + end + for _, funcInfo in ipairsr(funcInfos) do + for _, declLike in ipairsr(funcInfo.declLikes) do + optimizeAssignmentOrDeclLike(declLike, funcInfo) + end + end + + -- @Incomplete: Remove useless return statements etc. + + _simplify(theNode, stats) -- Not sure if needed. Or maybe we need to iterate? +end + +-- Note: References need to be updated after calling this! +local function optimize(theNode) + local stats = Stats() + _optimize(theNode, stats) + return stats +end + + + +local generateName +do + local BANK_LETTERS = "etaoinshrdlcumwfgypbvkxjqzETAOINSHRDLCUMWFGYPBVKXJQZ" -- http://en.wikipedia.org/wiki/Letter_frequencies + local BANK_ALPHANUM = "etaoinshrdlcumwfgypbvkxjqzETAOINSHRDLCUMWFGYPBVKXJQZ0123456789" + + local ILLEGAL_NAMES = {} + for name in pairs(KEYWORDS) do ILLEGAL_NAMES[name] = true end + if HANDLE_ENV then ILLEGAL_NAMES._ENV = true end + + local cache = {} + + --[[local]] function generateName(nameGeneration) + if not cache[nameGeneration] then + -- @Cleanup: Output the most significant byte first. (We need to know the length beforehand then, probably, so we use the correct bank.) + local charBytes = {} + + for i = 1, 1/0 do + nameGeneration = nameGeneration - 1 + local charBank = (i == 1) and BANK_LETTERS or BANK_ALPHANUM + local charIndex = nameGeneration % #charBank + 1 + charBytes[i] = stringByte(charBank, charIndex) + nameGeneration = mathFloor(nameGeneration / #charBank) + + if nameGeneration == 0 then break end + end + + local name = stringChar(tableUnpack(charBytes)) + if ILLEGAL_NAMES[name] then + -- We will probably realistically never get here, partially because of the limited amount of locals and upvalues Lua allows. + name = name.."_" + end + cache[nameGeneration] = name + end + + return cache[nameGeneration] + end + + -- for nameGeneration = 1, 3500 do print(generateName(nameGeneration)) end ; error("TEST") + -- for pow = 0, 32 do print(generateName(2^pow)) end ; error("TEST") +end + +-- stats = minify( node [, optimize=false ] ) +local function minify(node, doOptimize) + local stats = Stats() + + if doOptimize then + _optimize(node, stats) + end + + -- @Cleanup: Use findShadows()? + local identInfos, declIdentWatchers = getInformationAboutIdentifiersAndUpdateReferences(node) + -- local funcInfos = getInformationAboutFunctions(node) + -- local declIdentReadCount, declIdentAssignmentCount = getAccessesOfDeclaredNames(funcInfos, identInfos, declIdentWatchers) + + local allDeclIdents = {} + + for _, identInfo in ipairs(identInfos) do + local identOrVararg = identInfo.ident + local declIdent = (identOrVararg.type == "identifier") and identOrVararg.declaration or nil + + if declIdent and not allDeclIdents[declIdent] then + tableInsert(allDeclIdents, declIdent) + allDeclIdents[declIdent] = true + end + end + + -- + -- Make sure frequencies affect who gets shorter names first. + -- (This doesn't seem that useful at the moment. 2021-06-16) + -- + --[[ :SortBeforeRename + tableSort(allDeclIdents, function(a, b) + if a.type == "vararg" or b.type == "vararg" then + return a.id < b.id -- We don't care about varargs. + end + + local aWatchers = declIdentWatchers[a.declaration] + local bWatchers = declIdentWatchers[b.declaration] + + if not (aWatchers and bWatchers) then + return a.id < a.id -- We don't care about globals. + end + + if #aWatchers ~= #bWatchers then + return #aWatchers < #bWatchers + end + + return a.id < b.id + end) + --]] + + -- + -- Rename locals! + -- + local renamed = {--[[ [declIdent1]=true, ... ]]} + local maxNameGeneration = 0 + + --[[ :SortBeforeRename + local remoteWatchers = {} + --]] + + -- Assign generated names to declarations. + for _, declIdent in ipairs(allDeclIdents) do + local newName + + if declIdent.name == "_ENV" and HANDLE_ENV then + -- There are probably some cases where we can safely rename _ENV, + -- but it's likely not worth the effort to detect that. + newName = "_ENV" + + else + for nameGeneration = 1, 1/0 do + newName = generateName(nameGeneration) + local collision = false + + for _, watcherIdent in ipairs(declIdentWatchers[declIdent]) do + local watcherDeclIdent = watcherIdent.declaration + + -- Local watcher. + if watcherDeclIdent then + if renamed[watcherDeclIdent] and watcherDeclIdent.name == newName then + collision = true + break + end + + -- Global watcher. + elseif watcherIdent.name == newName then + collision = true + break + end + end--for declIdentWatchers + + --[[ :SortBeforeRename + if not collision and remoteWatchers[declIdent] then + for _, watcherDeclIdent in ipairs(remoteWatchers[declIdent]) do + if watcherDeclIdent.name == newName then + collision = true + break + end + end + end + --]] + + if not collision then + maxNameGeneration = mathMax(maxNameGeneration, nameGeneration) + break + end + end--for nameGeneration + end + + --[[ :SortBeforeRename + for _, watcherIdent in ipairs(declIdentWatchers[declIdent]) do + local watcherDeclIdent = watcherIdent.declaration + + if watcherDeclIdent and watcherDeclIdent ~= declIdent then + remoteWatchers[watcherDeclIdent] = remoteWatchers[watcherDeclIdent] or {} + + if not remoteWatchers[watcherDeclIdent][declIdent] then + tableInsert(remoteWatchers[watcherDeclIdent], declIdent) + remoteWatchers[watcherDeclIdent][declIdent] = true + end + end + end + --]] + + if declIdent.name ~= newName then + declIdent.name = newName + stats.renameCount = stats.renameCount + 1 + end + renamed[declIdent] = true + end--for allDeclIdents + + stats.generatedNameCount = maxNameGeneration + -- print("maxNameGeneration", maxNameGeneration) -- DEBUG + + -- Rename all remaining identifiers. + for _, identInfo in ipairs(identInfos) do + local ident = identInfo.ident -- Could be a vararg. + local declIdent = (ident.type == "identifier") and ident.declaration or nil + + if declIdent and ident.name ~= declIdent.name then + ident.name = declIdent.name + stats.renameCount = stats.renameCount + 1 + end + end + + return stats +end + + + +local function printTokens(tokens) + local printLocs = parser.printLocations + + for i, token in ipairs(tokens) do + local v = ensurePrintable(tostring(token.value)) + if #v > 200 then v = stringSub(v, 1, 200-3).."..." end + + if printLocs then ioWrite(token.sourcePath, ":", token.lineStart, ": ") end + ioWrite(i, ". ", F("%-11s", token.type), " '", v, "'\n") + end +end + + + +local toLua +do + local writeNode + local writeStatements + + -- lastOutput = "" | "alphanum" | "number" | "-" | "." + + local function isNumberInRange(n, min, max) + return n ~= nil and n >= min and n <= max + end + + local function canNodeBeName(node) + return node.type == "literal" and type(node.value) == "string" and stringFind(node.value, "^[%a_][%w_]*$") and not KEYWORDS[node.value] + end + + -- ensureSpaceIfNotPretty( buffer, pretty, lastOutput, value [, value2 ] ) + local function ensureSpaceIfNotPretty(buffer, pretty, lastOutput, value, value2) + if not pretty and (lastOutput == value or lastOutput == value2) then + tableInsert(buffer, " ") + end + end + + local function choosePretty(node, prettyFallback) + if node.pretty ~= nil then return node.pretty end + return prettyFallback + end + + local function writeLua(buffer, lua, lastOutput) + tableInsert(buffer, lua) + return lastOutput + end + + local function writeAlphanum(buffer, pretty, s, lastOutput) + ensureSpaceIfNotPretty(buffer, pretty, lastOutput, "alphanum","number") + lastOutput = writeLua(buffer, s, "alphanum") + return "alphanum" + end + local function writeNumber(buffer, pretty, n, lastOutput) + local nStr = formatNumber(n) + if (lastOutput == "-" and stringByte(nStr, 1) == 45--[[ "-" ]]) or lastOutput == "." then + lastOutput = writeLua(buffer, " ", "") + else + ensureSpaceIfNotPretty(buffer, pretty, lastOutput, "alphanum","number") + end + lastOutput = writeLua(buffer, nStr, "number") + return "number" + end + + -- Returns nil and a message or error. + local function writeCommaSeparatedList(buffer, pretty, indent, lastOutput, expressions, writeAttributes, nodeCb) + for i, expr in ipairs(expressions) do + if i > 1 then + lastOutput = writeLua(buffer, ",", "") + if pretty then lastOutput = writeLua(buffer, " ", "") end + end + + local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, expr, true, nodeCb) + if not ok then return nil, lastOutput end + + if writeAttributes and expr.type == "identifier" and expr.attribute ~= "" then + lastOutput = writeLua(buffer, "<", "") + lastOutput = writeAlphanum(buffer, pretty, expr.attribute, lastOutput) + lastOutput = writeLua(buffer, ">", "") + end + end + + return true, lastOutput + end + + local function isStatementFunctionDeclaration(statement, statementNext) + return true + and statement.type == "declaration" and statementNext.type == "assignment" + and #statement.names == 1 and #statement.values == 0 + and #statementNext.targets == 1 and #statementNext.values == 1 + and statementNext.targets[1].type == "identifier" and statement.names[1].name == statementNext.targets[1].name + and statementNext.values [1].type == "function" + end + + local function writeIndentationIfPretty(buffer, pretty, indent, lastOutput) + if pretty and indent > 0 then + lastOutput = writeLua(buffer, stringRep("\t", indent), "") + end + return lastOutput + end + + -- Returns nil and a message or error. + local function writeFunctionParametersAndBody(buffer, pretty, indent, lastOutput, func, explicitParams, selfParam, nodeCb) + lastOutput = writeLua(buffer, "(", "") + + if selfParam then + if nodeCb then nodeCb(selfParam, buffer) end + tableInsert(buffer, selfParam.prefix) + tableInsert(buffer, selfParam.suffix) + end + + local ok;ok, lastOutput = writeCommaSeparatedList(buffer, pretty, indent, lastOutput, explicitParams, false, nodeCb) + if not ok then return nil, lastOutput end + + lastOutput = writeLua(buffer, ")", "") + if nodeCb then nodeCb(func.body, buffer) end + pretty = choosePretty(func.body, pretty) + tableInsert(buffer, func.body.prefix) + if pretty then lastOutput = writeLua(buffer, "\n", "") end + + local ok;ok, lastOutput = writeStatements(buffer, pretty, indent+1, lastOutput, func.body.statements, nodeCb) + if not ok then return nil, lastOutput end + + lastOutput = writeIndentationIfPretty(buffer, pretty, indent, lastOutput) + tableInsert(buffer, func.body.suffix) + lastOutput = writeAlphanum(buffer, pretty, "end", lastOutput) + + return true, lastOutput + end + + -- Returns nil and a message or error. + --[[local]] function writeStatements(buffer, pretty, indent, lastOutput, statements, nodeCb) + local skipNext = false + + for i, statement in ipairs(statements) do + if skipNext then + skipNext = false + + else + local statementNext = statements[i+1] + lastOutput = writeIndentationIfPretty(buffer, pretty, indent, lastOutput) + + if statementNext and isStatementFunctionDeclaration(statement, statementNext) then + local decl = statement + local assignment = statementNext + local func = assignment.values[1] + local pretty = choosePretty(assignment, pretty) + + if nodeCb then + nodeCb(decl, buffer) + nodeCb(assignment, buffer) + nodeCb(func, buffer) + end + + tableInsert(buffer, decl.prefix) + tableInsert(buffer, assignment.prefix) + tableInsert(buffer, func.prefix) + + lastOutput = writeAlphanum(buffer, pretty, "local function", lastOutput) + lastOutput = writeLua(buffer, " ", "") + + if nodeCb then nodeCb(decl.names[1], buffer) end + + tableInsert(buffer, decl.names[1].prefix) + local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, assignment.targets[1], true, nodeCb) + if not ok then return nil, lastOutput end + tableInsert(buffer, decl.names[1].suffix) + + local ok;ok, lastOutput = writeFunctionParametersAndBody(buffer, choosePretty(func, pretty), indent, lastOutput, func, func.parameters, nil, nodeCb) + if not ok then return nil, lastOutput end + + tableInsert(buffer, func.suffix) + tableInsert(buffer, assignment.suffix) + tableInsert(buffer, decl.suffix) + + skipNext = true + + else + local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, statement, true, nodeCb) + if not ok then return nil, lastOutput end + + if statement.type == "call" then + lastOutput = writeLua(buffer, ";", "") -- @Ugly way of handling call statements. (But what way would be better?) + end + end + + if pretty then lastOutput = writeLua(buffer, "\n", "") end + end + end + + return true, lastOutput + end + + local function doesExpressionNeedParenthesisIfOnTheLeftSide(expr) + local nodeType = expr.type + -- Some things, like "binary" or "vararg", are not here because those expressions add their own parentheses. + return nodeType == "literal" or nodeType == "table" or nodeType == "function" + end + + -- Returns nil and a message or error. + local function writeLookup(buffer, pretty, indent, lastOutput, lookup, forMethodCall, nodeCb) + local objNeedParens = doesExpressionNeedParenthesisIfOnTheLeftSide(lookup.object) + if objNeedParens then lastOutput = writeLua(buffer, "(", "") end + + local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, lookup.object, false, nodeCb) + if not ok then return nil, lastOutput end + + if objNeedParens then lastOutput = writeLua(buffer, ")", "") end + + if canNodeBeName(lookup.member) then + lastOutput = writeLua(buffer, (forMethodCall and ":" or "."), "") + if nodeCb then nodeCb(lookup.member, buffer) end + tableInsert(buffer, lookup.member.prefix) + lastOutput = writeAlphanum(buffer, pretty, lookup.member.value, lastOutput) + tableInsert(buffer, lookup.member.suffix) + + elseif forMethodCall then + return nil, "Error: AST: Callee for method call is not a lookup." + + else + lastOutput = writeLua(buffer, "[", "") + + local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, lookup.member, true, nodeCb) + if not ok then return nil, lastOutput end + + lastOutput = writeLua(buffer, "]", "") + end + + return true, lastOutput + end + + local function isAssignmentFunctionAssignment(assignment) + if #assignment.targets ~= 1 then return false end + if #assignment.values ~= 1 then return false end + if assignment.values[1].type ~= "function" then return false end + + local targetExpr = assignment.targets[1] + while true do + if targetExpr.type == "identifier" then + return true + elseif not (targetExpr.type == "lookup" and canNodeBeName(targetExpr.member)) then + return false + end + targetExpr = targetExpr.object + end + end + + -- Returns nil and a message or error. + local function writeBinaryOperatorChain(buffer, pretty, indent, lastOutput, binary, nodeCb) + local l = binary.left + local r = binary.right + + if l.type == "binary" and l.operator == binary.operator then + if nodeCb then nodeCb(l, buffer) end + tableInsert(buffer, l.prefix) + local ok;ok, lastOutput = writeBinaryOperatorChain(buffer, choosePretty(l, pretty), indent, lastOutput, l, nodeCb) + if not ok then return nil, lastOutput end + tableInsert(buffer, l.suffix) + else + local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, l, false, nodeCb) + if not ok then return nil, lastOutput end + end + + if pretty then lastOutput = writeLua(buffer, " ", "") end + + if binary.operator == ".." then ensureSpaceIfNotPretty(buffer, pretty, lastOutput, "number") end + + local nextOutput = ( + (binary.operator == "-" and "-" ) or + (binary.operator == ".." and "." ) or + (stringFind(binary.operator, "%w") and "alphanum") or + "" + ) + if nextOutput ~= "" then ensureSpaceIfNotPretty(buffer, pretty, lastOutput, nextOutput) end + lastOutput = writeLua(buffer, binary.operator, nextOutput) + + if pretty then lastOutput = writeLua(buffer, " ", "") end + + if r.type == "binary" and r.operator == binary.operator then + if nodeCb then nodeCb(r, buffer) end + tableInsert(buffer, r.prefix) + local ok;ok, lastOutput = writeBinaryOperatorChain(buffer, choosePretty(r, pretty), indent, lastOutput, r, nodeCb) + if not ok then return nil, lastOutput end + tableInsert(buffer, r.suffix) + else + local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, r, false, nodeCb) + if not ok then return nil, lastOutput end + end + + return true, lastOutput + end + + -- success, lastOutput = writeNode( buffer, pretty, indent, lastOutput, node, maySafelyOmitParens, nodeCallback ) + -- Returns nil and a message or error. + --[[local]] function writeNode(buffer, pretty, indent, lastOutput, node, maySafelyOmitParens, nodeCb) + if nodeCb then nodeCb(node, buffer) end + pretty = choosePretty(node, pretty) + + tableInsert(buffer, node.prefix) + + local nodeType = node.type + + -- Expressions: + + if nodeType == "identifier" then + local ident = node + lastOutput = writeAlphanum(buffer, pretty, ident.name, lastOutput) + + elseif nodeType == "vararg" then + local vararg = node + if vararg.adjustToOne then lastOutput = writeLua(buffer, "(", "") end + if lastOutput == "." then lastOutput = writeLua(buffer, " ", ".") end + lastOutput = writeLua(buffer, "...", ".") + if vararg.adjustToOne then lastOutput = writeLua(buffer, ")", "") end + + elseif nodeType == "literal" then + local literal = node + + if node.value == 0 and NORMALIZE_MINUS_ZERO then + lastOutput = writeNumber(buffer, pretty, 0, lastOutput) -- Avoid writing '-0' sometimes. + + elseif literal.value == 1/0 then + lastOutput = writeAlphanum(buffer, pretty, "(1/0)", lastOutput) -- Note: We parse this as one literal. + + elseif literal.value == -1/0 then + lastOutput = writeLua(buffer, "(-1/0)", "") -- Note: We parse this as one literal. + + elseif literal.value ~= literal.value then + lastOutput = writeLua(buffer, "(0/0)", "") -- Note: We parse this as one literal. + + elseif literal.value == nil or type(literal.value) == "boolean" then + lastOutput = writeAlphanum(buffer, pretty, tostring(literal.value), lastOutput) + + elseif type(literal.value) == "number" or (jit and type(literal.value) == "cdata" and tonumber(literal.value)) then + lastOutput = writeNumber(buffer, pretty, literal.value, lastOutput) + + elseif type(literal.value) == "string" then + -- @Speed: Cache! + local R = isNumberInRange + local s = literal.value + local doubleCount = countString(s, '"', true) + local singleCount = countString(s, "'", true) + local quote = singleCount < doubleCount and "'" or '"' + local quoteByte = stringByte(quote) + local pos = 1 + + lastOutput = writeLua(buffer, quote, "") + + while pos <= #s do + local b1, b2, b3, b4 = stringByte(s, pos, pos+3) + + -- Printable ASCII. + if R(b1,32,126) then + if b1 == quoteByte then tableInsert(buffer, "\\") ; tableInsert(buffer, quote) ; pos = pos + 1 + elseif b1 == 92 then tableInsert(buffer, [[\\]]) ; pos = pos + 1 + else tableInsert(buffer, stringSub(s, pos, pos)) ; pos = pos + 1 + end + + -- Multi-byte UTF-8 sequence. + elseif b2 and R(b1,194,223) and R(b2,128,191) then tableInsert(buffer, stringSub(s, pos, pos+1)) ; pos = pos + 2 + elseif b3 and b1== 224 and R(b2,160,191) and R(b3,128,191) then tableInsert(buffer, stringSub(s, pos, pos+2)) ; pos = pos + 3 + elseif b3 and R(b1,225,236) and R(b2,128,191) and R(b3,128,191) then tableInsert(buffer, stringSub(s, pos, pos+2)) ; pos = pos + 3 + elseif b3 and b1== 237 and R(b2,128,159) and R(b3,128,191) then tableInsert(buffer, stringSub(s, pos, pos+2)) ; pos = pos + 3 + elseif b3 and R(b1,238,239) and R(b2,128,191) and R(b3,128,191) then tableInsert(buffer, stringSub(s, pos, pos+2)) ; pos = pos + 3 + elseif b4 and b1== 240 and R(b2,144,191) and R(b3,128,191) and R(b4,128,191) then tableInsert(buffer, stringSub(s, pos, pos+3)) ; pos = pos + 4 + elseif b4 and R(b1,241,243) and R(b2,128,191) and R(b3,128,191) and R(b4,128,191) then tableInsert(buffer, stringSub(s, pos, pos+3)) ; pos = pos + 4 + elseif b4 and b1== 244 and R(b2,128,143) and R(b3,128,191) and R(b4,128,191) then tableInsert(buffer, stringSub(s, pos, pos+3)) ; pos = pos + 4 + + -- Escape sequence. + elseif b1 == 7 then tableInsert(buffer, [[\a]]) ; pos = pos + 1 + elseif b1 == 8 then tableInsert(buffer, [[\b]]) ; pos = pos + 1 + elseif b1 == 9 then tableInsert(buffer, [[\t]]) ; pos = pos + 1 + elseif b1 == 10 then tableInsert(buffer, [[\n]]) ; pos = pos + 1 + elseif b1 == 11 then tableInsert(buffer, [[\v]]) ; pos = pos + 1 + elseif b1 == 12 then tableInsert(buffer, [[\f]]) ; pos = pos + 1 + elseif b1 == 13 then tableInsert(buffer, [[\r]]) ; pos = pos + 1 + + -- Other control character or anything else. + elseif b2 and R(b2,48,57) then tableInsert(buffer, F([[\%03d]], b1)) ; pos = pos + 1 + else tableInsert(buffer, F([[\%d]], b1)) ; pos = pos + 1 + end + end + + lastOutput = writeLua(buffer, quote, "") + + else + return nil, F("Error: Failed outputting '%s' value '%s'.", type(literal.value), tostring(literal.value)) + end + + elseif nodeType == "table" then + local tableNode = node + lastOutput = writeLua(buffer, "{", "") + + for i, tableField in ipairs(tableNode.fields) do + if i > 1 then + lastOutput = writeLua(buffer, ",", "") + if pretty then lastOutput = writeLua(buffer, " ", "") end + end + + if tableField.generatedKey then + if tableField.key then + if nodeCb then nodeCb(tableField.key, buffer) end + tableInsert(buffer, tableField.key.prefix) + tableInsert(buffer, tableField.key.suffix) + end + + else + if canNodeBeName(tableField.key) then + if nodeCb then nodeCb(tableField.key, buffer) end + tableInsert(buffer, tableField.key.prefix) + lastOutput = writeLua(buffer, tableField.key.value, "alphanum") + tableInsert(buffer, tableField.key.suffix) + + else + lastOutput = writeLua(buffer, "[", "") + + local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, tableField.key, true, nodeCb) + if not ok then return nil, lastOutput end + + lastOutput = writeLua(buffer, "]", "") + end + + lastOutput = writeLua(buffer, "=", "") + end + + local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, tableField.value, (not pretty), nodeCb) + if not ok then return nil, lastOutput end + end + + lastOutput = writeLua(buffer, "}", "") + + elseif nodeType == "lookup" then + local lookup = node + local ok;ok, lastOutput = writeLookup(buffer, pretty, indent, lastOutput, lookup, false, nodeCb) + if not ok then return nil, lastOutput end + + elseif nodeType == "unary" then + local unary = node + local operatorOutput = (unary.operator == "-" and "-") or (stringFind(unary.operator, "%w") and "alphanum") or ("") + local prettyAndAlphanum = pretty and operatorOutput == "alphanum" + + if prettyAndAlphanum and not maySafelyOmitParens then lastOutput = writeLua(buffer, "(", "") end -- @Polish: Only output parentheses around child unaries/binaries if associativity requires it. + + if lastOutput == "-" and operatorOutput == "-" then lastOutput = writeLua(buffer, " ", "") + elseif operatorOutput ~= "" then ensureSpaceIfNotPretty(buffer, pretty, lastOutput, operatorOutput) + end + lastOutput = writeLua(buffer, unary.operator, operatorOutput) + + if prettyAndAlphanum then lastOutput = writeLua(buffer, " ", "") end + + local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, unary.expression, false, nodeCb) + if not ok then return nil, lastOutput end + + if prettyAndAlphanum and not maySafelyOmitParens then lastOutput = writeLua(buffer, ")", "") end + + elseif nodeType == "binary" then + local binary = node + local op = binary.operator + + if not maySafelyOmitParens then lastOutput = writeLua(buffer, "(", "") end -- @Polish: Only output parentheses around child unaries/binaries if associativity requires it. + + if op == ".." or op == "and" or op == "or" or op == "+" or op == "*" or op == "&" or op == "|" then + local ok;ok, lastOutput = writeBinaryOperatorChain(buffer, pretty, indent, lastOutput, binary, nodeCb) + if not ok then return nil, lastOutput end + + else + local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, binary.left, false, nodeCb) + if not ok then return nil, lastOutput end + + local operatorOutput = ((op == "-" and "-") or (stringFind(op, "%w") and "alphanum") or ("")) + + if pretty then lastOutput = writeLua(buffer, " ", "") end + + if operatorOutput ~= "" then ensureSpaceIfNotPretty(buffer, pretty, lastOutput, operatorOutput) end + lastOutput = writeLua(buffer, op, operatorOutput) + + if pretty then lastOutput = writeLua(buffer, " ", "") end + + local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, binary.right, false, nodeCb) + if not ok then return nil, lastOutput end + end + + if not maySafelyOmitParens then lastOutput = writeLua(buffer, ")", "") end + + elseif nodeType == "call" then -- Can be statement too. + local call = node + + if call.adjustToOne then lastOutput = writeLua(buffer, "(", "") end + + if call.method then + local lookup = call.callee + + if lookup.type ~= "lookup" then + return nil, "Error: AST: Callee for method call is not a lookup." + end + + if nodeCb then nodeCb(lookup, buffer) end + + tableInsert(buffer, lookup.prefix) + local ok;ok, lastOutput = writeLookup(buffer, choosePretty(lookup, pretty), indent, lastOutput, lookup, true, nodeCb) + if not ok then return nil, lastOutput end + tableInsert(buffer, lookup.suffix) + + else + local needParens = doesExpressionNeedParenthesisIfOnTheLeftSide(call.callee) + if needParens then lastOutput = writeLua(buffer, "(", "") end + + local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, call.callee, false, nodeCb) + if not ok then return nil, lastOutput end + + if needParens then lastOutput = writeLua(buffer, ")", "") end + end + + lastOutput = writeLua(buffer, "(", "") + + local ok;ok, lastOutput = writeCommaSeparatedList(buffer, pretty, indent, lastOutput, call.arguments, false, nodeCb) + if not ok then return nil, lastOutput end + + lastOutput = writeLua(buffer, ")", "") + if call.adjustToOne then lastOutput = writeLua(buffer, ")", "") end + + elseif nodeType == "function" then + local func = node + lastOutput = writeAlphanum(buffer, pretty, "function", lastOutput) + + local ok;ok, lastOutput = writeFunctionParametersAndBody(buffer, pretty, indent, lastOutput, func, func.parameters, nil, nodeCb) + if not ok then return nil, lastOutput end + + -- Statements: + + elseif nodeType == "break" then + lastOutput = writeAlphanum(buffer, pretty, "break", lastOutput) + lastOutput = writeLua(buffer, ";", "") + + elseif nodeType == "return" then + local returnNode = node + lastOutput = writeAlphanum(buffer, pretty, "return", lastOutput) + + if returnNode.values[1] then + if pretty then lastOutput = writeLua(buffer, " ", "") end + + local ok;ok, lastOutput = writeCommaSeparatedList(buffer, pretty, indent, lastOutput, returnNode.values, false, nodeCb) + if not ok then return nil, lastOutput end + end + + lastOutput = writeLua(buffer, ";", "") + + elseif nodeType == "label" then + local label = node + local name = label.name + if not (stringFind(name, "^[%a_][%w_]*$") and not KEYWORDS[name]) then + return nil, F("Error: AST: Invalid label '%s'.", name) + end + lastOutput = writeLua(buffer, "::", "") + lastOutput = writeAlphanum(buffer, pretty, name, lastOutput) + lastOutput = writeLua(buffer, "::", "") + lastOutput = writeLua(buffer, ";", "") + + elseif nodeType == "goto" then + local gotoNode = node + local name = gotoNode.name + if not (stringFind(name, "^[%a_][%w_]*$") and not KEYWORDS[name]) then + return nil, F("Error: AST: Invalid label '%s'.", name) + end + lastOutput = writeAlphanum(buffer, pretty, "goto", lastOutput) + lastOutput = writeLua(buffer, " ", "") + lastOutput = writeAlphanum(buffer, pretty, name, lastOutput) + lastOutput = writeLua(buffer, ";", "") + + elseif nodeType == "block" then + local block = node + lastOutput = writeAlphanum(buffer, pretty, "do", lastOutput) + if pretty then lastOutput = writeLua(buffer, "\n", "") end + + local ok;ok, lastOutput = writeStatements(buffer, pretty, indent+1, lastOutput, block.statements, nodeCb) + if not ok then return nil, lastOutput end + + lastOutput = writeIndentationIfPretty(buffer, pretty, indent, lastOutput) + lastOutput = writeAlphanum(buffer, pretty, "end", lastOutput) + + elseif nodeType == "declaration" then + local decl = node + lastOutput = writeAlphanum(buffer, pretty, "local", lastOutput) + lastOutput = writeLua(buffer, " ", "") + + if not decl.names[1] then return nil, "Error: AST: Missing name(s) for declaration." end + + local ok;ok, lastOutput = writeCommaSeparatedList(buffer, pretty, indent, lastOutput, decl.names, true, nodeCb) + if not ok then return nil, lastOutput end + + if decl.values[1] then + if pretty then lastOutput = writeLua(buffer, " ", "") end + lastOutput = writeLua(buffer, "=", "") + if pretty then lastOutput = writeLua(buffer, " ", "") end + + local ok;ok, lastOutput = writeCommaSeparatedList(buffer, pretty, indent, lastOutput, decl.values, false, nodeCb) + if not ok then return nil, lastOutput end + end + + lastOutput = writeLua(buffer, ";", "") + + elseif nodeType == "assignment" then + local assignment = node + if not assignment.targets[1] then return nil, "Error: AST: Missing target expression(s) for assignment." end + if not assignment.values[1] then return nil, "Error: AST: Missing value(s) for assignment." end + + if isAssignmentFunctionAssignment(assignment) then + local func = assignment.values[1] + if nodeCb then nodeCb(func, buffer) end + + tableInsert(buffer, func.prefix) + + lastOutput = writeAlphanum(buffer, pretty, "function", lastOutput) + lastOutput = writeLua(buffer, " ", "") + + local implicitSelfParam = ( + func.parameters[1] ~= nil + and func.parameters[1].name == "self" + and assignment.targets[1].type == "lookup" + and canNodeBeName(assignment.targets[1].member) + ) + + if implicitSelfParam then + if nodeCb then nodeCb(assignment.targets[1], buffer) end + tableInsert(buffer, assignment.targets[1].prefix) + local ok;ok, lastOutput = writeLookup(buffer, pretty, indent, lastOutput, assignment.targets[1], true, nodeCb) + if not ok then return nil, lastOutput end + tableInsert(buffer, assignment.targets[1].suffix) + else + local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, assignment.targets[1], false, nodeCb) + if not ok then return nil, lastOutput end + end + + local selfParam = nil + local explicitParams = func.parameters + + if implicitSelfParam then + selfParam = explicitParams[1] + explicitParams = {tableUnpack(explicitParams, 2)} + end + + local ok;ok, lastOutput = writeFunctionParametersAndBody(buffer, pretty, indent, lastOutput, func, explicitParams, selfParam, nodeCb) + if not ok then return nil, lastOutput end + + tableInsert(buffer, func.suffix) + + else + local ok;ok, lastOutput = writeCommaSeparatedList(buffer, pretty, indent, lastOutput, assignment.targets, false, nodeCb) + if not ok then return nil, lastOutput end + + if pretty then lastOutput = writeLua(buffer, " ", "") end + lastOutput = writeLua(buffer, "=", "") + if pretty then lastOutput = writeLua(buffer, " ", "") end + + local ok;ok, lastOutput = writeCommaSeparatedList(buffer, pretty, indent, lastOutput, assignment.values, false, nodeCb) + if not ok then return nil, lastOutput end + + lastOutput = writeLua(buffer, ";", "") + end + + elseif nodeType == "if" then + local ifNode = node + lastOutput = writeAlphanum(buffer, pretty, "if", lastOutput) + if pretty then lastOutput = writeLua(buffer, " ", "") end + + local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, ifNode.condition, true, nodeCb) + if not ok then return nil, lastOutput end + + if pretty then lastOutput = writeLua(buffer, " ", "") end + lastOutput = writeAlphanum(buffer, pretty, "then", lastOutput) + if nodeCb then nodeCb(ifNode.bodyTrue, buffer) end + local prettyBody = choosePretty(ifNode.bodyTrue, pretty) + tableInsert(buffer, ifNode.bodyTrue.prefix) + if prettyBody then lastOutput = writeLua(buffer, "\n", "") end + + local ok;ok, lastOutput = writeStatements(buffer, prettyBody, indent+1, lastOutput, ifNode.bodyTrue.statements, nodeCb) + if not ok then return nil, lastOutput end + + local lastTrueBody = ifNode.bodyTrue + local suffixesForTrailingBodies = {} + + while ifNode.bodyFalse do + lastOutput = writeIndentationIfPretty(buffer, prettyBody, indent, lastOutput) + + -- Automatically detect what looks like 'elseif'. + if #ifNode.bodyFalse.statements == 1 and ifNode.bodyFalse.statements[1].type == "if" then + tableInsert(buffer, lastTrueBody.suffix) + local body = ifNode.bodyFalse + + if nodeCb then nodeCb(body, buffer) end + tableInsert(suffixesForTrailingBodies, body.suffix) + ifNode = body.statements[1] + if nodeCb then nodeCb(ifNode, buffer) end + pretty = choosePretty(ifNode, pretty) + + tableInsert(buffer, body.prefix) + tableInsert(buffer, ifNode.prefix) + lastOutput = writeAlphanum(buffer, prettyBody, "elseif", lastOutput) + if pretty then lastOutput = writeLua(buffer, " ", "") end + + local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, ifNode.condition, true, nodeCb) + if not ok then return nil, lastOutput end + + if pretty then lastOutput = writeLua(buffer, " ", "") end + lastOutput = writeAlphanum(buffer, pretty, "then", lastOutput) + if nodeCb then nodeCb(ifNode.bodyTrue, buffer) end + prettyBody = choosePretty(ifNode.bodyTrue, pretty) + if prettyBody then lastOutput = writeLua(buffer, "\n", "") end + + tableInsert(buffer, ifNode.bodyTrue.prefix) + local ok;ok, lastOutput = writeStatements(buffer, prettyBody, indent+1, lastOutput, ifNode.bodyTrue.statements, nodeCb) + if not ok then return nil, lastOutput end + tableInsert(buffer, ifNode.bodyTrue.suffix) + + lastTrueBody = ifNode + + else + lastOutput = writeAlphanum(buffer, prettyBody, "else", lastOutput) + if nodeCb then nodeCb(ifNode.bodyFalse, buffer) end + prettyBody = choosePretty(ifNode.bodyFalse, pretty) + tableInsert(buffer, ifNode.bodyFalse.prefix) + if prettyBody then lastOutput = writeLua(buffer, "\n", "") end + + local ok;ok, lastOutput = writeStatements(buffer, prettyBody, indent+1, lastOutput, ifNode.bodyFalse.statements, nodeCb) + if not ok then return nil, lastOutput end + + tableInsert(suffixesForTrailingBodies, ifNode.bodyFalse.suffix) + break + end + end + + lastOutput = writeIndentationIfPretty(buffer, prettyBody, indent, lastOutput) + tableInsert(buffer, lastTrueBody.suffix) + for i = #suffixesForTrailingBodies, 1, -1 do + tableInsert(buffer, suffixesForTrailingBodies[i]) + end + lastOutput = writeAlphanum(buffer, prettyBody, "end", lastOutput) + + elseif nodeType == "while" then + local whileLoop = node + lastOutput = writeAlphanum(buffer, pretty, "while", lastOutput) + if pretty then lastOutput = writeLua(buffer, " ", "") end + + local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, whileLoop.condition, true, nodeCb) + if not ok then return nil, lastOutput end + + if pretty then lastOutput = writeLua(buffer, " ", "") end + lastOutput = writeAlphanum(buffer, pretty, "do", lastOutput) + + if nodeCb then nodeCb(whileLoop.body, buffer) end + local prettyBody = choosePretty(whileLoop.body, pretty) + tableInsert(buffer, whileLoop.body.prefix) + if prettyBody then lastOutput = writeLua(buffer, "\n", "") end + + local ok;ok, lastOutput = writeStatements(buffer, prettyBody, indent+1, lastOutput, whileLoop.body.statements, nodeCb) + if not ok then return nil, lastOutput end + + lastOutput = writeIndentationIfPretty(buffer, prettyBody, indent, lastOutput) + tableInsert(buffer, whileLoop.body.suffix) + lastOutput = writeAlphanum(buffer, prettyBody, "end", lastOutput) + + elseif nodeType == "repeat" then + local repeatLoop = node + lastOutput = writeAlphanum(buffer, pretty, "repeat", lastOutput) + if nodeCb then nodeCb(repeatLoop.body, buffer) end + local prettyBody = choosePretty(repeatLoop.body, pretty) + tableInsert(buffer, repeatLoop.body.prefix) + if prettyBody then lastOutput = writeLua(buffer, "\n", "") end + + local ok;ok, lastOutput = writeStatements(buffer, prettyBody, indent+1, lastOutput, repeatLoop.body.statements, nodeCb) + if not ok then return nil, lastOutput end + + lastOutput = writeIndentationIfPretty(buffer, prettyBody, indent, lastOutput) + tableInsert(buffer, repeatLoop.body.suffix) + lastOutput = writeAlphanum(buffer, prettyBody, "until", lastOutput) + if pretty then lastOutput = writeLua(buffer, " ", "") end + + local ok;ok, lastOutput = writeNode(buffer, pretty, indent, lastOutput, repeatLoop.condition, true, nodeCb) + if not ok then return nil, lastOutput end + + elseif nodeType == "for" then + local forLoop = node + if not forLoop.names[1] then return nil, "Error: AST: Missing name(s) for 'for' loop." end + if not forLoop.values[1] then return nil, "Error: AST: Missing value(s) for 'for' loop." end + + lastOutput = writeAlphanum(buffer, pretty, "for", lastOutput) + lastOutput = writeLua(buffer, " ", "") + + local ok;ok, lastOutput = writeCommaSeparatedList(buffer, pretty, indent, lastOutput, forLoop.names, false, nodeCb) + if not ok then return nil, lastOutput end + + if pretty then lastOutput = writeLua(buffer, " ", "") end + + if forLoop.kind == "numeric" then + lastOutput = writeLua(buffer, "=", "") + elseif forLoop.kind == "generic" then + lastOutput = writeAlphanum(buffer, pretty, "in", lastOutput) + else + return nil, F("Error: Unknown 'for' loop kind '%s'.", forLoop.kind) + end + + if pretty then lastOutput = writeLua(buffer, " ", "") end + + local ok;ok, lastOutput = writeCommaSeparatedList(buffer, pretty, indent, lastOutput, forLoop.values, false, nodeCb) + if not ok then return nil, lastOutput end + + if pretty then lastOutput = writeLua(buffer, " ", "") end + lastOutput = writeAlphanum(buffer, pretty, "do", lastOutput) + if nodeCb then nodeCb(forLoop.body, buffer) end + local prettyBody = choosePretty(forLoop.body, pretty) + tableInsert(buffer, forLoop.body.prefix) + if prettyBody then lastOutput = writeLua(buffer, "\n", "") end + + local ok;ok, lastOutput = writeStatements(buffer, prettyBody, indent+1, lastOutput, forLoop.body.statements, nodeCb) + if not ok then return nil, lastOutput end + + lastOutput = writeIndentationIfPretty(buffer, prettyBody, indent, lastOutput) + tableInsert(buffer, forLoop.body.suffix) + lastOutput = writeAlphanum(buffer, prettyBody, "end", lastOutput) + + else + return false, F("Error: Unknown node type '%s'.", tostring(nodeType)) + end + + tableInsert(buffer, node.suffix) + + return true, lastOutput + end + + -- luaString = toLua( astNode [, prettyOuput=false, nodeCallback ] ) + -- nodeCallback = function( node, outputBuffer ) + -- Returns nil and a message on error. + --[[local]] function toLua(node, pretty, nodeCb) + assertArg1("toLua", 1, node, "table") + + local buffer = {} + + local ok, err + if node.type == "block" then -- @Robustness: This exception isn't great. Should there be a file scope node? + if nodeCb then nodeCb(node, buffer) end + tableInsert(buffer, node.prefix) + ok, err = writeStatements(buffer, choosePretty(node, pretty), 0, "", node.statements, nodeCb) + tableInsert(buffer, node.suffix) + else + ok, err = writeNode(buffer, pretty, 0, "", node, true, nodeCb) + end + + if ok then + return tableConcat(buffer) + else + return nil, err + end + end +end + + + +-- node = getChild( node, fieldName ) +-- node = getChild( node, fieldName, index ) -- If the node field is an array. +-- node = getChild( node, fieldName, index, tableFieldKey ) -- If the node field is a table field array. +local function getChild(node, fieldName, i, tableFieldKey) + assertArg1("getChild", 1, node, "table") + assertArg1("getChild", 2, fieldName, "string") + + local nodeType = node.type + local childFields = CHILD_FIELDS[nodeType] or errorf(2, "Unknown node type '%s'.", tostring(nodeType)) + local childFieldType = childFields[fieldName] or errorf(2, "Unknown node field '%s.%s'.", nodeType, tostring(fieldName)) + + if childFieldType == "node" then + return node[fieldName] + + elseif childFieldType == "nodearray" then + assertArg1("getChild", 3, i, "number") + + return node[fieldName][i] + + elseif childFieldType == "tablefields" then + assertArg1("getChild", 3, i, "number") + assertArg1("getChild", 4, tableFieldKey, "string") + + if not (tableFieldKey == "key" or tableFieldKey == "value") then + errorf(2, "Bad argument #4 to 'getChild'. (Expected %q or %q, got %q)", "key", "value", tableFieldKey) + end + + local field = node[fieldName][i] + return field and field[tableFieldKey] + + else + error(childFieldType) + end +end + +-- setChild( node, fieldName, childNode ) +-- setChild( node, fieldName, index, childNode ) -- If the node field is an array. +-- setChild( node, fieldName, index, tableFieldKey, childNode ) -- If the node field is a table field array. +local function setChild(node, fieldName, i, tableFieldKey, childNode) + assertArg1("setChild", 1, node, "table") + assertArg1("setChild", 2, fieldName, "string") + + local nodeType = node.type + local childFields = CHILD_FIELDS[nodeType] or errorf(2, "Unknown node type '%s'.", tostring(nodeType)) + local childFieldType = childFields[fieldName] or errorf(2, "Unknown node field '%s.%s'.", nodeType, tostring(fieldName)) + + if childFieldType == "node" then + childNode = i + + if childNode ~= nil then assertArg1("setChild", 3, childNode, "table") end + + node[fieldName] = childNode + + elseif childFieldType == "nodearray" then + childNode = tableFieldKey + + assertArg1("setChild", 3, i, "number") + assertArg1("setChild", 4, childNode, "table") + + node[fieldName][i] = childNode + + elseif childFieldType == "tablefields" then + assertArg1("setChild", 3, i, "number") + assertArg1("setChild", 4, tableFieldKey, "string") + assertArg1("setChild", 5, childNode, "table") + + if not (tableFieldKey == "key" or tableFieldKey == "value") then + errorf(2, "Bad argument #4 to 'setChild'. (Expected %q or %q, got %q)", "key", "value", tableFieldKey) + end + + local field = node[fieldName][i] or errorf(2, "No table field at index %d in %s.%s.", i, nodeType, fieldName) + field[tableFieldKey] = childNode + + else + error(childFieldType) + end +end + +-- addChild( node, fieldName, [ index=atEnd, ] childNode ) +-- addChild( node, fieldName, [ index=atEnd, ] keyNode, valueNode ) -- If the node field is a table field array. +local function addChild(node, fieldName, i, childNode, extraChildNode) + assertArg1("addChild", 1, node, "table") + assertArg1("addChild", 2, fieldName, "string") + + if type(i) ~= "number" then + i, childNode, extraChildNode = nil, i, childNode + end + local postIndexArgOffset = i and 0 or -1 + + local nodeType = node.type + local childFields = CHILD_FIELDS[nodeType] or errorf(2, "Unknown node type '%s'.", tostring(nodeType)) + local childFieldType = childFields[fieldName] or errorf(2, "Unknown node field '%s.%s'.", nodeType, tostring(fieldName)) + + if childFieldType == "nodearray" then + if i ~= nil then assertArg1("addChild", 3, i, "number") end + assertArg1("addChild", 4+postIndexArgOffset, childNode, "table") + + i = i or #node[fieldName]+1 + tableInsert(node[fieldName], i, childNode) + + elseif childFieldType == "tablefields" then + if i ~= nil then assertArg1("addChild", 3, i, "number") end + assertArg1("addChild", 4+postIndexArgOffset, childNode, "table") + assertArg1("addChild", 5+postIndexArgOffset, extraChildNode, "table") + + i = i or #node[fieldName]+1 + tableInsert(node[fieldName], i, {key=childNode, value=extraChildNode, generatedKey=false}) + + else + errorf(2, "Node field '%s.%s' is not an array.", nodeType, tostring(fieldName)) + end +end + +-- removeChild( node, fieldName [, index=last ] ) +local function removeChild(node, fieldName, i) + assertArg1("removeChild", 1, node, "table") + assertArg1("removeChild", 2, fieldName, "string") + assertArg2("removeChild", 3, i, "number","nil") + + local nodeType = node.type + local childFields = CHILD_FIELDS[nodeType] or errorf(2, "Unknown node type '%s'.", tostring(nodeType)) + local childFieldType = childFields[fieldName] or errorf(2, "Unknown node field '%s.%s'.", nodeType, tostring(fieldName)) + + if childFieldType == "nodearray" or childFieldType == "tablefields" then + tableRemove(node[fieldName], i) -- This also works if i is nil. + else + errorf(2, "Node field '%s.%s' is not an array.", nodeType, tostring(fieldName)) + end +end + + + +local validateTree +do + local function addValidationError(path, errors, s, ...) + tableInsert(errors, F("%s: "..s, tableConcat(path, " > "), ...)) + end + + local function validateNode(node, path, errors, prefix) + local nodeType = node.type + + tableInsert(path, + (prefix and prefix.."."..nodeType or nodeType) + .. (parser.printIds and "#"..node.id or "") + ) + + if nodeType == "identifier" then + local ident = node + if not stringFind(ident.name, "^[%a_][%w_]*$") then + addValidationError(path, errors, "Invalid identifier name: Bad format: %s", ident.name) + elseif KEYWORDS[ident.name] then + addValidationError(path, errors, "Invalid identifier name: Name is a keyword: %s", ident.name) + end + if not (ident.attribute == "" or ident.attribute == "close" or ident.attribute == "const") then + addValidationError(path, errors, "Invalid identifier attribute '%s'.", ident.attribute) + end + + elseif nodeType == "vararg" then + -- void + + elseif nodeType == "literal" then + local literal = node + local vType = type(literal.value) + if not (vType == "number" or vType == "string" or vType == "boolean" or vType == "nil" or (jit and vType == "cdata" and tonumber(literal.value))) then + addValidationError(path, errors, "Invalid literal value type '%s'.", vType) + end + + elseif nodeType == "break" then + -- void + + elseif nodeType == "label" then + local label = node + if not stringFind(label.name, "^[%a_][%w_]*$") then + addValidationError(path, errors, "Invalid label name: Bad format: %s", label.name) + elseif KEYWORDS[label.name] then + addValidationError(path, errors, "Invalid label name: Name is a keyword: %s", label.name) + end + + elseif nodeType == "goto" then + local gotoNode = node + if not stringFind(gotoNode.name, "^[%a_][%w_]*$") then + addValidationError(path, errors, "Invalid label name: Bad format: %s", gotoNode.name) + elseif KEYWORDS[gotoNode.name] then + addValidationError(path, errors, "Invalid label name: Name is a keyword: %s", gotoNode.name) + end + + elseif nodeType == "lookup" then + -- @Incomplete: Should we detect nil literal objects? :DetectRuntimeErrors + local lookup = node + if not lookup.object then + addValidationError(path, errors, "Missing 'object' field.") + elseif not EXPRESSION_NODES[lookup.object.type] then + addValidationError(path, errors, "The object is not an expression. (It is '%s'.)", lookup.object.type) + else + validateNode(lookup.object, path, errors, "object") + end + if not lookup.member then + addValidationError(path, errors, "Missing 'member' field.") + elseif not EXPRESSION_NODES[lookup.member.type] then + addValidationError(path, errors, "The member is not an expression. (It is '%s'.)", lookup.member.type) + else + validateNode(lookup.member, path, errors, "member") + end + + elseif nodeType == "unary" then + local unary = node + if not OPERATORS_UNARY[unary.operator] then + addValidationError(path, errors, "Invalid unary operator '%s'.", unary.operator) + end + if not unary.expression then + addValidationError(path, errors, "Missing 'expression' field.") + elseif not EXPRESSION_NODES[unary.expression.type] then + addValidationError(path, errors, "The 'expression' field does not contain an expression. (It is '%s'.)", unary.expression.type) + else + validateNode(unary.expression, path, errors, nil) + end + + elseif nodeType == "binary" then + local binary = node + if not OPERATORS_BINARY[binary.operator] then + addValidationError(path, errors, "Invalid binary operator '%s'.", binary.operator) + end + if not binary.left then + addValidationError(path, errors, "Missing 'left' field.") + elseif not EXPRESSION_NODES[binary.left.type] then + addValidationError(path, errors, "The left side is not an expression. (It is '%s'.)", binary.left.type) + else + validateNode(binary.left, path, errors, "left") + end + if not binary.right then + addValidationError(path, errors, "Missing 'right' field.") + elseif not EXPRESSION_NODES[binary.right.type] then + addValidationError(path, errors, "The right side is not an expression. (It is '%s'.)", binary.right.type) + else + validateNode(binary.right, path, errors, "right") + end + + elseif nodeType == "call" then + local call = node + if not call.callee then + addValidationError(path, errors, "Missing 'callee' field.") + elseif not EXPRESSION_NODES[call.callee.type] then + addValidationError(path, errors, "Callee is not an expression. (It is '%s'.)", call.callee.type) + -- elseif call.callee.type == "literal" or call.callee.type == "table" then -- @Incomplete: Do this kind of check? Or maybe we should stick to strictly validating the AST even if the resulting Lua code would raise a runtime error. :DetectRuntimeErrors + -- addValidationError(path, errors, "Callee is uncallable.") + elseif call.method and not ( + call.callee.type == "lookup" + and call.callee.member + and call.callee.member.type == "literal" + and type(call.callee.member.value) == "string" + and stringFind(call.callee.member.value, "^[%a_][%w_]*$") + and not KEYWORDS[call.callee.member.value] + ) then + addValidationError(path, errors, "Callee is unsuitable for method call.") + else + validateNode(call.callee, path, errors, "callee") + end + for i, expr in ipairs(call.arguments) do + if not EXPRESSION_NODES[expr.type] then + addValidationError(path, errors, "Argument %d is not an expression. (It is '%s')", i, expr.type) + else + validateNode(expr, path, errors, "arg"..i) + end + end + + elseif nodeType == "function" then + local func = node + for i, ident in ipairs(func.parameters) do + if not (ident.type == "identifier" or (ident.type == "vararg" and i == #func.parameters)) then + addValidationError(path, errors, "Parameter %d is not an identifier%s. (It is '%s')", i, (i == #func.parameters and " or vararg" or ""), ident.type) + else + validateNode(ident, path, errors, "param"..i) + end + end + if not func.body then + addValidationError(path, errors, "Missing 'body' field.") + elseif func.body.type ~= "block" then + addValidationError(path, errors, "Body is not a block.") + else + validateNode(func.body, path, errors, "body") + end + + elseif nodeType == "return" then + local returnNode = node + for i, expr in ipairs(returnNode.values) do + if not EXPRESSION_NODES[expr.type] then + addValidationError(path, errors, "Value %d is not an expression. (It is '%s')", i, expr.type) + else + validateNode(expr, path, errors, i) + end + end + + elseif nodeType == "block" then + local block = node + for i, statement in ipairs(block.statements) do + if not STATEMENT_NODES[statement.type] then + addValidationError(path, errors, "Child node %d is not a statement. (It is '%s'.)", i, statement.type) + else + validateNode(statement, path, errors, i) + end + end + + elseif nodeType == "declaration" then + local decl = node + if not decl.names[1] then + addValidationError(path, errors, "Missing name(s).") + end + for i, ident in ipairs(decl.names) do + if ident.type ~= "identifier" then + addValidationError(path, errors, "Name %d is not an identifier. (It is '%s')", i, ident.type) + else + validateNode(ident, path, errors, "name"..i) + end + end + for i, expr in ipairs(decl.values) do + if not EXPRESSION_NODES[expr.type] then + addValidationError(path, errors, "Value %d is not an expression. (It is '%s')", i, expr.type) + else + validateNode(expr, path, errors, "value"..i) + end + end + + elseif nodeType == "assignment" then + local assignment = node + if not assignment.targets[1] then + addValidationError(path, errors, "Missing target expression(s).") + end + for i, expr in ipairs(assignment.targets) do + if not (expr.type == "identifier" or expr.type == "lookup") then + addValidationError(path, errors, "Target %d is not an identifier or lookup. (It is '%s')", i, expr.type) + else + validateNode(expr, path, errors, "target"..i) + end + end + if not assignment.values[1] then + addValidationError(path, errors, "Missing value(s).") + end + for i, expr in ipairs(assignment.values) do + if not EXPRESSION_NODES[expr.type] then + addValidationError(path, errors, "Value %d is not an expression. (It is '%s')", i, expr.type) + else + validateNode(expr, path, errors, "value"..i) + end + end + + elseif nodeType == "if" then + local ifNode = node + if not ifNode.condition then + addValidationError(path, errors, "Missing 'condition' field.") + elseif not EXPRESSION_NODES[ifNode.condition.type] then + addValidationError(path, errors, "The condition is not an expression. (It is '%s'.)", ifNode.condition.type) + else + validateNode(ifNode.condition, path, errors, "condition") + end + if not ifNode.bodyTrue then + addValidationError(path, errors, "Missing 'bodyTrue' field.") + elseif ifNode.bodyTrue.type ~= "block" then + addValidationError(path, errors, "Body for true branch is not a block.") + else + validateNode(ifNode.bodyTrue, path, errors, "true") + end + if not ifNode.bodyFalse then + -- void + elseif ifNode.bodyFalse.type ~= "block" then + addValidationError(path, errors, "Body for false branch is not a block.") + else + validateNode(ifNode.bodyFalse, path, errors, "false") + end + + elseif nodeType == "while" then + local whileLoop = node + if not whileLoop.condition then + addValidationError(path, errors, "Missing 'condition' field.") + elseif not EXPRESSION_NODES[whileLoop.condition.type] then + addValidationError(path, errors, "The condition is not an expression. (It is '%s'.)", whileLoop.condition.type) + else + validateNode(whileLoop.condition, path, errors, "condition") + end + if not whileLoop.body then + addValidationError(path, errors, "Missing 'body' field.") + elseif whileLoop.body.type ~= "block" then + addValidationError(path, errors, "Body is not a block.") + else + validateNode(whileLoop.body, path, errors, "true") + end + + elseif nodeType == "repeat" then + local repeatLoop = node + if not repeatLoop.body then + addValidationError(path, errors, "Missing 'body' field.") + elseif repeatLoop.body.type ~= "block" then + addValidationError(path, errors, "Body is not a block.") + else + validateNode(repeatLoop.body, path, errors, "true") + end + if not repeatLoop.condition then + addValidationError(path, errors, "Missing 'condition' field.") + elseif not EXPRESSION_NODES[repeatLoop.condition.type] then + addValidationError(path, errors, "The condition is not an expression. (It is '%s'.)", repeatLoop.condition.type) + else + validateNode(repeatLoop.condition, path, errors, "condition") + end + + elseif nodeType == "for" then + local forLoop = node + if not (forLoop.kind == "numeric" or forLoop.kind == "generic") then + addValidationError(path, errors, "Invalid for loop kind '%s'.", forLoop.kind) + end + if not forLoop.names[1] then + addValidationError(path, errors, "Missing name(s).") + elseif forLoop.kind == "numeric" and forLoop.names[2] then + addValidationError(path, errors, "Too many names for numeric loop. (Got %d)", #forLoop.names) + end + for i, ident in ipairs(forLoop.names) do + if ident.type ~= "identifier" then + addValidationError(path, errors, "Name %d is not an identifier. (It is '%s')", i, ident.type) + else + validateNode(ident, path, errors, "name"..i) + end + end + if not forLoop.values[1] then + addValidationError(path, errors, "Missing value(s).") + elseif forLoop.kind == "numeric" and not forLoop.values[2] then + addValidationError(path, errors, "Too few values for numeric loop. (Got %d)", #forLoop.values) + elseif forLoop.kind == "numeric" and forLoop.values[4] then + addValidationError(path, errors, "Too many values for numeric loop. (Got %d)", #forLoop.values) + end + for i, expr in ipairs(forLoop.values) do + if not EXPRESSION_NODES[expr.type] then + addValidationError(path, errors, "Value %d is not an expression. (It is '%s')", i, expr.type) + else + validateNode(expr, path, errors, "value"..i) + end + end + if not forLoop.body then + addValidationError(path, errors, "Missing 'body' field.") + elseif forLoop.body.type ~= "block" then + addValidationError(path, errors, "Body is not a block.") + else + validateNode(forLoop.body, path, errors, "body") + end + + elseif nodeType == "table" then + local tableNode = node + for i, tableField in ipairs(tableNode.fields) do + -- @Incomplete: Should we detect nil literal keys? :DetectRuntimeErrors + if not tableField.key then + if not tableField.generatedKey then + addValidationError(path, errors, "Missing 'key' field for table field %d.", i) + end + elseif not EXPRESSION_NODES[tableField.key.type] then + addValidationError(path, errors, "The key for table field %d is not an expression. (It is '%s'.)", i, tableField.key.type) + elseif tableField.generatedKey and tableField.key.type ~= "literal" then + addValidationError(path, errors, "The generated key for table field %d is not a numeral. (It is '%s'.)", i, tableField.key.type) + elseif tableField.generatedKey and type(tableField.key.value) ~= "number" then + addValidationError(path, errors, "The generated key for table field %d is not a number. (It's a '%s' literal.)", i, type(tableField.key.value)) + else + validateNode(tableField.key, path, errors, "key") + end + if not tableField.value then + addValidationError(path, errors, "Missing 'value' field for table field %d.", i) + elseif not EXPRESSION_NODES[tableField.value.type] then + addValidationError(path, errors, "The value for table field %d is not an expression. (It is '%s'.)", i, tableField.value.type) + else + validateNode(tableField.value, path, errors, "value") + end + end + + else + errorf("Invalid node type '%s'.", tostring(nodeType)) -- We don't call addValidationError() for this - it's just an assertion. + end + + path[#path] = nil + end + + -- isValid, errors = validateTree( astNode ) + --[[local]] function validateTree(node) + local path = {} + local errors = {} + + validateNode(node, path, errors, nil) + + if errors[1] then + return false, tableConcat(errors, "\n") + else + return true + end + end +end + + + +local EXPRESSION_TYPES = newSet{"binary","call","function","identifier","literal","lookup","table","unary","vararg"} + +local function isExpression(node) + return EXPRESSION_TYPES[node.type] == true +end + +local function isStatement(node) + return EXPRESSION_TYPES[node.type] == nil or node.type == "call" +end + + + +local function resetNextId() + nextSerialNumber = 1 +end + + + +-- astNode = valueToAst( value [, sortTableKeys=false ] ) +local function valueToAst(v, sortTableKeys) + local vType = type(v) + + if vType == "number" or vType == "string" or vType == "boolean" or vType == "nil" then + return AstLiteral(nil, v) + + elseif vType == "table" then + local t = v + local tableNode = AstTable(nil) + local keys = {} + local indices = {} + + for k in pairs(t) do + tableInsert(keys, k) + end + + if sortTableKeys then + local keyStringRepresentations = {} + + for _, k in ipairs(keys) do + keyStringRepresentations[k] = keyStringRepresentations[k] or tostring(k) + end + + tableSort(keys, function(a, b) + return keyStringRepresentations[a] < keyStringRepresentations[b] + end) + end + + for i = 1, #t do + indices[i] = true + end + + for _, k in ipairs(keys) do + if not indices[k] then + local tableField = {key=valueToAst(k,sortTableKeys), value=valueToAst(t[k],sortTableKeys), generatedKey=false} + tableInsert(tableNode.fields, tableField) + end + end + + for i = 1, #t do + local tableField = {key=valueToAst(i,sortTableKeys), value=valueToAst(t[i],sortTableKeys), generatedKey=true} + tableInsert(tableNode.fields, tableField) + end + + return tableNode + + else + error("Invalid value type '"..vType.."'.", 2) + end +end + + + +-- identifiers = findGlobalReferences( astNode ) +-- Note: updateReferences() must have been called first! +local function findGlobalReferences(theNode) + local idents = {} + + traverseTree(theNode, function(node) + if node.type == "identifier" and not node.declaration then + tableInsert(idents, node) + end + end) + + return idents +end + + + +-- identifiers = findDeclaredNames( astNode ) +local function findDeclaredNames(theNode) + local declIdents = {} + + traverseTree(theNode, function(node) + -- Note: We don't now, but if we would require updateReferences() to be called first + -- we could just check the type and if node.declaration==node. Decisions... + + if node.type == "declaration" or node.type == "for" then + for _, declIdent in ipairs(node.names) do + tableInsert(declIdents, declIdent) + end + + elseif node.type == "function" then + for _, declIdent in ipairs(node.parameters) do + if declIdent.type == "identifier" then -- There may be a vararg at the end. + tableInsert(declIdents, declIdent) + end + end + end + end) + + return declIdents +end + + + +-- shadows, foundPrevious = maybeRegisterShadow( shadowSequences, shadowSequenceByIdent, shadows|nil, currentDeclIdent, declIdent ) +local function maybeRegisterShadow(shadowSequences, shadowSequenceByIdent, shadows, currentDeclIdent, declIdent) + if declIdent.name ~= currentDeclIdent.name then + return shadows, false + end + + if not shadows then + shadows = {currentDeclIdent} + shadowSequenceByIdent[currentDeclIdent] = shadows + tableInsert(shadowSequences, shadows) + end + + if shadowSequenceByIdent[declIdent] then + -- Shortcut! declIdent is shadowing others, so just copy the existing data. + for _, prevDeclIdent in ipairs(shadowSequenceByIdent[declIdent]) do + tableInsert(shadows, prevDeclIdent) + end + return shadows, true + + else + tableInsert(shadows, declIdent) + return shadows, false + end +end + +-- shadowSequences = findShadows( astNode ) +-- shadowSequences = { shadowSequence1, ... } +-- shadowSequence = { shadowingIdentifier, shadowedIdentifier1, ... } +-- Note: updateReferences() must have been called first! +local function findShadows(theNode) + local shadowSequences = {} + local shadowSequenceByIdent = {} + + for _, currentDeclIdent in ipairs(findDeclaredNames(theNode)) do + local shadows = nil + local child = currentDeclIdent + local foundPrevious = false + + while child.parent do + local parent = child.parent + + if isNodeDeclLike(parent) then + local declIdents = getNameArrayOfDeclLike(parent) + local childIndex = indexOf(declIdents, child) or #declIdents+1 + + for i = childIndex-1, 1, -1 do + shadows, foundPrevious = maybeRegisterShadow(shadowSequences, shadowSequenceByIdent, shadows, currentDeclIdent, declIdents[i]) + if foundPrevious then break end + end + if foundPrevious then break end + + elseif parent.type == "block" then + local statements = parent.statements + + for i = child.key-1, 1, -1 do + if statements[i].type == "declaration" then + for _, declIdent in ipairs(statements[i].names) do + shadows, foundPrevious = maybeRegisterShadow(shadowSequences, shadowSequenceByIdent, shadows, currentDeclIdent, declIdent) + if foundPrevious then break end + end + if foundPrevious then break end + end + end + if foundPrevious then break end + end + + child = parent + if child == theNode then break end -- Stay within theNode (in case theNode has a parent). + end + end + + return shadowSequences +end + + + +parser = { + -- + -- Constants. + -- + VERSION = PARSER_VERSION, + + INT_SIZE = INT_SIZE, + MAX_INT = MAX_INT, + MIN_INT = MIN_INT, + + -- + -- Functions. + -- + + -- Tokenizing. + tokenize = tokenize, + tokenizeFile = tokenizeFile, + + -- Token actions. + newToken = newToken, + updateToken = updateToken, + cloneToken = cloneToken, + concatTokens = concatTokens, + + -- AST parsing. + parse = parse, + parseExpression = parseExpression, + parseFile = parseFile, + + -- AST manipulation. + newNode = newNode, + newNodeFast = newNodeFast, + valueToAst = valueToAst, + cloneNode = cloneNode, + cloneTree = cloneTree, + getChild = getChild, + setChild = setChild, + addChild = addChild, + removeChild = removeChild, + + -- AST checking. + isExpression = isExpression, + isStatement = isStatement, + validateTree = validateTree, + + -- AST traversal. + traverseTree = traverseTree, + traverseTreeReverse = traverseTreeReverse, + updateReferences = updateReferences, + + -- Big AST operations. + simplify = simplify, + optimize = optimize, + minify = minify, + + -- Conversion. + toLua = toLua, + + -- Printing. + printTokens = printTokens, + printNode = printNode, + printTree = printTree, + formatMessage = formatMessage, + + -- Utilities. + findDeclaredNames = findDeclaredNames, + findGlobalReferences = findGlobalReferences, + findShadows = findShadows, + + -- Misc. + resetNextId = resetNextId, -- @Undocumented + + -- + -- Settings. + -- + printIds = false, + printLocations = false, + indentation = " ", + + constantNameReplacementStringMaxLength = 200, -- @Cleanup: Maybe use a better name. +} + +return parser + + + +--[[!=========================================================== + +Copyright © 2020-2022 Marcus 'ReFreezed' Thunström + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +==============================================================]] -- 2.47.3