From c0109d48634636c3d1792fe1bc85f777c1b2a7da Mon Sep 17 00:00:00 2001 From: John MacFarlane Date: Mon, 2 Jan 2023 16:15:32 -0800 Subject: [PATCH] Initial commit (split from jgm/djot) --- LICENSE | 20 + Makefile | 119 ++++ README.md | 435 +++++++++++++ bin/main.lua | 132 ++++ clib/Makefile | 79 +++ clib/README.md | 24 + clib/combine.lua | 31 + clib/djot.c | 138 ++++ clib/djot.h | 55 ++ clib/main.c | 40 ++ clib/tests.c | 85 +++ config.ld | 7 + djot-reader.lua | 459 ++++++++++++++ djot-writer.lua | 448 +++++++++++++ djot.lua | 158 +++++ djot/ast.lua | 1012 ++++++++++++++++++++++++++++++ djot/attributes.lua | 270 ++++++++ djot/block.lua | 823 ++++++++++++++++++++++++ djot/filter.lua | 174 +++++ djot/html.lua | 549 ++++++++++++++++ djot/inline.lua | 679 ++++++++++++++++++++ djot/json.lua | 137 ++++ doc/api/index.html | 71 +++ doc/api/ldoc.css | 303 +++++++++ doc/api/modules/djot.ast.html | 166 +++++ doc/api/modules/djot.filter.html | 173 +++++ doc/api/modules/djot.html | 173 +++++ doc/api/modules/djot.json.html | 68 ++ doc/code-examples.lua | 15 + doc/djot.1 | 172 +++++ doc/djot.md | 151 +++++ full-coverage.lua | 77 +++ fuzz.lua | 95 +++ ldoc.ltp | 135 ++++ lua51.nix | 14 + luajit.nix | 14 + pathological_tests.lua | 58 ++ rockspec.in | 42 ++ run.sh | 2 + test.lua | 230 +++++++ test/attributes.test | 298 +++++++++ test/blockquote.test | 152 +++++ test/code_blocks.test | 65 ++ test/definition_lists.test | 93 +++ test/emphasis.test | 239 +++++++ test/escapes.test | 53 ++ test/fenced_divs.test | 134 ++++ test/filters.test | 68 ++ test/footnotes.test | 46 ++ test/headings.test | 182 ++++++ test/insert_delete_mark.test | 29 + test/links_and_images.test | 242 +++++++ test/lists.test | 494 +++++++++++++++ test/math.test | 44 ++ test/para.test | 7 + test/raw.test | 38 ++ test/regression.test | 49 ++ test/smart.test | 192 ++++++ test/sourcepos.test | 13 + test/spans.test | 19 + test/super_subscript.test | 23 + test/symbol.test | 18 + test/tables.test | 125 ++++ test/task_lists.test | 31 + test/thematic_breaks.test | 45 ++ test/verbatim.test | 47 ++ 66 files changed, 10579 insertions(+) create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100755 bin/main.lua create mode 100644 clib/Makefile create mode 100644 clib/README.md create mode 100644 clib/combine.lua create mode 100644 clib/djot.c create mode 100644 clib/djot.h create mode 100644 clib/main.c create mode 100644 clib/tests.c create mode 100644 config.ld create mode 100644 djot-reader.lua create mode 100644 djot-writer.lua create mode 100644 djot.lua create mode 100644 djot/ast.lua create mode 100644 djot/attributes.lua create mode 100644 djot/block.lua create mode 100644 djot/filter.lua create mode 100644 djot/html.lua create mode 100644 djot/inline.lua create mode 100644 djot/json.lua create mode 100644 doc/api/index.html create mode 100644 doc/api/ldoc.css create mode 100644 doc/api/modules/djot.ast.html create mode 100644 doc/api/modules/djot.filter.html create mode 100644 doc/api/modules/djot.html create mode 100644 doc/api/modules/djot.json.html create mode 100644 doc/code-examples.lua create mode 100644 doc/djot.1 create mode 100644 doc/djot.md create mode 100644 full-coverage.lua create mode 100644 fuzz.lua create mode 100644 ldoc.ltp create mode 100755 lua51.nix create mode 100755 luajit.nix create mode 100644 pathological_tests.lua create mode 100644 rockspec.in create mode 100755 run.sh create mode 100644 test.lua create mode 100644 test/attributes.test create mode 100644 test/blockquote.test create mode 100644 test/code_blocks.test create mode 100644 test/definition_lists.test create mode 100644 test/emphasis.test create mode 100644 test/escapes.test create mode 100644 test/fenced_divs.test create mode 100644 test/filters.test create mode 100644 test/footnotes.test create mode 100644 test/headings.test create mode 100644 test/insert_delete_mark.test create mode 100644 test/links_and_images.test create mode 100644 test/lists.test create mode 100644 test/math.test create mode 100644 test/para.test create mode 100644 test/raw.test create mode 100644 test/regression.test create mode 100644 test/smart.test create mode 100644 test/sourcepos.test create mode 100644 test/spans.test create mode 100644 test/super_subscript.test create mode 100644 test/symbol.test create mode 100644 test/tables.test create mode 100644 test/task_lists.test create mode 100644 test/thematic_breaks.test create mode 100644 test/verbatim.test diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..a9f5933 --- /dev/null +++ b/LICENSE @@ -0,0 +1,20 @@ +Copyright (C) 2022 John MacFarlane + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..83e804c --- /dev/null +++ b/Makefile @@ -0,0 +1,119 @@ +VERSION=$(shell grep "version = \"" djot.lua | sed -e 's/.*"\([^"]*\).*"/\1/') +REVISION=1 +ROCKSPEC=djot-$(VERSION)-$(REVISION).rockspec +MODULES=djot.lua djot/attributes.lua djot/inline.lua djot/block.lua djot/ast.lua djot/html.lua djot/filter.lua +SOURCES=$(MODULES) bin/main.lua +TESTSOURCES=test.lua pathological_tests.lua +BUNDLE=djot +VIMDIR?=~/.vim +TIMEOUT=perl -e 'alarm shift; exec @ARGV' +TEMPFILE := $(shell mktemp) + +all: test doc/syntax.html doc/djot.1 doc/api/index.html + +test: $(ROCKSPEC) + luarocks test +.PHONY: test + +testall: test pathological fuzz +.PHONY: testall + +ci: testall install + make -C clib + make -C web oldplayground/djot.js + pandoc --print-default-data-file MANUAL.txt > m.txt + pandoc -t djot-writer.lua m.txt -o m.dj + pandoc -f djot-reader.lua m.dj -o m.html + rm m.dj m.html +.PHONY: ci + +fuzz: + LUA_PATH="./?.lua;$$LUA_PATH" $(TIMEOUT) 90 lua fuzz.lua 500000 +.PHONY: fuzz + +pathological: + LUA_PATH="./?.lua;$$LUA_PATH" \ + $(TIMEOUT) 10 lua pathological_tests.lua +.PHONY: pathological + +bench: bench-lua bench-luajit +.PHONY: bench + +bench-lua: m.dj + du -h m.dj + LUA_PATH="./?.lua" hyperfine --warmup 2 "lua bin/main.lua m.dj" + LUA_PATH="./?.lua" hyperfine --warmup 2 "lua bin/main.lua -m m.dj" + LUA_PATH="./?.lua" hyperfine --warmup 2 "lua bin/main.lua -p m.dj" +.PHONY: bench-lua + +bench-luajit: m.dj + du -h m.dj + LUA_PATH="./?.lua" hyperfine --warmup 2 "luajit bin/main.lua m.dj" + LUA_PATH="./?.lua" hyperfine --warmup 2 "luajit bin/main.lua -m m.dj" + LUA_PATH="./?.lua" hyperfine --warmup 2 "luajit bin/main.lua -p m.dj" +.PHONY: bench-luajit + + +m.dj: + pandoc -t djot-writer.lua https://raw.githubusercontent.com/jgm/pandoc/2.18/MANUAL.txt -o m.dj + +djot-reader.amalg.lua: djot-reader.lua $(MODULES) + LUA_PATH="./?.lua;" amalg.lua djot djot.ast djot.block djot.filter djot.inline djot.attributes djot.html djot.json -s $< -o $@ + +djot-writer.amalg.lua: djot-writer.lua $(MODULES) + LUA_PATH="./?.lua;" amalg.lua djot djot.ast djot.block djot.filter djot.inline djot.attributes djot.html djot.json -s $< -o $@ + +linecount: + wc -l $(SOURCES) +.PHONY: linecount + +check: + luacheck $(SOURCES) $(TESTSOURCES) +.PHONY: check + +doc/syntax.html: doc/syntax.md + pandoc --lua-filter doc/code-examples.lua $< -t html -o $@ -s --css doc/syntax.css --self-contained --wrap=preserve --toc --section-divs -Vpagetitle="Djot syntax reference" + +doc/djot.1: doc/djot.md + pandoc \ + --metadata title="DJOT(1)" \ + --metadata author="" \ + --variable footer="djot $(VERSION)" \ + $< -s -o $@ + +# luarocks packaging + +install: $(ROCKSPEC) + luarocks make $(ROCKSPEC) +.PHONY: install + +rock: $(ROCKSPEC) + luarocks --local make $(ROCKSPEC) +.PHONY: rock + +doc/api: + -mkdir $@ + +doc/api/index.html: djot.lua djot/ast.lua djot/filter.lua doc/api + ldoc . + +vim: + cp editors/vim/syntax/djot.vim $(VIMDIR)/syntax/ + cp editors/vim/ftdetect/djot.vim $(VIMDIR)/ftdetect/ +.PHONY: vim + +## start up nix env with lua 5.1 +lua51: + nix-shell --pure lua51.nix + rm ~/.luarocks/default-lua-version.lua +.PHONY: lua51 + +## start up nix env with luajiit +luajit: + nix-shell --pure luajit.nix + rm ~/.luarocks/default-lua-version.lua +.PHONY: luajit + +$(ROCKSPEC): rockspec.in + sed -e "s/_VERSION/$(VERSION)/g; s/_REVISION/$(REVISION)/g" $< > $@ + diff --git a/README.md b/README.md new file mode 100644 index 0000000..9495e6c --- /dev/null +++ b/README.md @@ -0,0 +1,435 @@ +# Djot + +[![GitHub +CI](https://github.com/jgm/djot/workflows/CI%20tests/badge.svg)](https://github.com/jgm/djot/actions) + +Djot is a light markup syntax. It derives most of its features +from [commonmark](https://spec.commonmark.org), but it fixes +a few things that make commonmark's syntax complex and difficult +to parse efficiently. It is also much fuller-featured than +commonmark, with support for definition lists, footnotes, +tables, several new kinds of inline formatting (insert, delete, +highlight, superscript, subscript), math, smart punctuation, +attributes that can be applied to any element, and generic +containers for block-level, inline-level, and raw content. + +The project began as an attempt to implement some of the +ideas I suggested in my essay [Beyond Markdown](https://johnmacfarlane.net/beyond-markdown.html). (See [Rationale](#rationale), below.) + +This repository contains a reference implementation, written +in Lua, and a +[Syntax Description](https://htmlpreview.github.io/?https://github.com/jgm/djot/blob/master/doc/syntax.html). +There is also a [Cheatsheet](doc/cheatsheet.md) and a +[Quick Start for Markdown Users](doc/quickstart-for-markdown-users.md) +that outlines the main differences between djot and Markdown, +as well as a [Playground](https://djot.net/playground/), +originally designed by @dtinth. +(Originally this ran the Lua code compiled to WASM, but +now it uses an experimental [typescript port of +djot](https://github.com/jgm/djot.js).) + +Despite being +written in an interpreted language, the reference implementation +is very fast (converting a 260K test document in 141 ms on an M1 mac +using the standard `lua` interpreter). It can produce +an AST, rendered HTML, or a stream of match tokens that identify +elements by source position, which could be used for syntax +highlighting or a linting tool. + +We also provide a custom pandoc writer for djot (`djot-writer.lua`), +so that documents in other formats can be converted to djot +format, and a custom pandoc reader (`djot-reader.lua`), so that +djot documents can be converted to any format pandoc supports. +To use these, just put them in your working directory and use +`pandoc -f djot-reader.lua` to convert from djot, and `pandoc -t +djot-writer.lua` to convert to djot. You'll need pandoc version +2.18 or higher, and you'll need the djot library to be installed +in your `LUA_PATH`; see [Installing](#installing), below. If +you're using the dev version of djot or don't want to worry +about the djot library being installed, you can create +self-contained versions of the custom reader and writer +using the `amalg` tool: + + luarocks install amalg + make djot-reader.amalg.lua + make djot-writer.amalg.lua + +These can be moved anywhere and do not require any Lua libraries +to be installed. + +## Rationale + +Here are some design goals: + +1. It should be possible to parse djot markup in linear time, + with no backtracking. + +2. Parsing of inline elements should be "local" and not depend + on what references are defined later. This is not the case + in commonmark: `[foo][bar]` might be "[foo]" followed by + a link with text "bar", or "[foo][bar]", or a link with + text "foo", or a link with text "foo" followed by + "[bar]", depending on whether the references `[foo]` and + `[bar]` are defined elsewhere (perhaps later) in the + document. This non-locality makes accurate syntax highlighting + nearly impossible. + +3. Rules for emphasis should be simpler. The fact that doubled + characters are used for strong emphasis in commonmark leads to + many potential ambiguities, which are resolved by a daunting + list of 17 rules. It is hard to form a good mental model + of these rules. Most of the time they interpret things the + way a human would most naturally interpret them---but not always. + +4. Expressive blind spots should be avoided. In commonmark, + you're out of luck if you want to produce the HTML + `a?b`, because the flanking rules classify + the first asterisk in `a*?*b` as right-flanking. There is a + way around this, but it's ugly (using a numerical entity instead + of `a`). In djot there should not be expressive blind spots of + this kind. + +5. Rules for what content belongs to a list item should be simple. + In commonmark, content under a list item must be indented as far + as the first non-space content after the list marker (or five + spaces after the marker, in case the list item begins with indented + code). Many people get confused when their indented content is + not indented far enough and does not get included in the list item. + +6. Parsers should not be forced to recognize unicode character classes, + HTML tags, or entities, or perform unicode case folding. + That adds a lot of complexity. + +7. The syntax should be friendly to hard-wrapping: hard-wrapping + a paragraph should not lead to different interpretations, e.g. + when a number followed by a period ends up at the beginning of + a line. (I anticipate that many will ask, why hard-wrap at + all? Answer: so that your document is readable just as it + is, without conversion to HTML and without special editor + modes that soft-wrap long lines. Remember that source readability + was one of the prime goals of Markdown and Commonmark.) + +8. The syntax should compose uniformly, in the following sense: + if a sequence of lines has a certain meaning outside a list + item or block quote, it should have the same meaning inside it. + This principle is [articulated in the commonmark + spec](https://spec.commonmark.org/0.30/#principle-of-uniformity), + but the spec doesn't completely abide by it (see + commonmark/commonmark-spec#634). + +9. It should be possible to attach arbitrary attributes to any + element. + +10. There should be generic containers for text, inline content, + and block-level content, to which arbitrary attributes can be applied. + This allows for extensibility using AST transformations. + +11. The syntax should be kept as simple as possible, consistent with + these goals. Thus, for example, we don't need two different + styles of headings or code blocks. + +These goals motivated the following decisions: + + +- Block-level elements can't interrupt paragraphs (or headings), + because of goal 7. So in djot the following is a single paragraph, not + (as commonmark sees it) a paragraph followed by an ordered list + followed by a block quote followed by a section heading: + + ``` + My favorite number is probably the number + 1. It's the smallest natural number that is + > 0. With pencils, though, I prefer a + # 2. + ``` + + Commonmark does make some concessions to goal 7, by forbidding + lists beginning with markers other than `1.` to interrupt paragraphs. + But this is a compromise and a sacrifice of regularity and + predictability in the syntax. Better just to have a general rule. + +- An implication of the last decision is that, although "tight" + lists are still possible (without blank lines between items), + a *sublist* must always be preceded by a blank line. Thus, + instead of + + ``` + - Fruits + - apple + - orange + ``` + + you must write + + ``` + - Fruits + + - apple + - orange + ``` + + (This blank line doesn't count against "tightness.") + reStructuredText makes the same design decision. + +- Also to promote goal 7, we allow headings to "lazily" + span multiple lines: + + ``` + ## My excessively long section heading is too + long to fit on one line. + ``` + + While we're at it, we'll simplify by removing setext-style + (underlined) headings. We don't really need two heading + syntaxes (goal 11). + +- To meet goal 5, we have a very simple rule: anything that is + indented beyond the start of the list marker belongs in + the list item. + + ``` + 1. list item + + > block quote inside item 1 + + 2. second item + ``` + + In commonmark, this would be parsed as two separate lists with + a block quote between them, because the block quote is not + indented far enough. What kept us from using this simple rule + in commonmark was indented code blocks. If list items are + going to contain an indented code block, we need to know at + what column to start counting the indentation, so we fixed on + the column that makes the list look best (the first column of + non-space content after the marker): + + ``` + 1. A commonmark list item with an indented code block in it. + + code! + ``` + + In djot, we just get rid of indented code blocks. Most people + prefer fenced code blocks anyway, and we don't need two + different ways of writing code blocks (goal 11). + +- To meet goal 6 and to avoid the complex rules commonmark + adopted for handling raw HTML, we simply do not allow raw HTML, + except in explicitly marked contexts, e.g. + `` ``{=html} `` or + + ```` + ``` =html + + +
foo
+ ``` + ```` + + Unlike Markdown, djot is not HTML-centric. Djot documents + might be rendered to a variety of different formats, so although + we want to provide the flexibility to include raw content in + any output format, there is no reason to privilege HTML. For + similar reasons we do not interpret HTML entities, as + commonmark does. + +- To meet goal 2, we make reference link parsing local. + Anything that looks like `[foo][bar]` or `[foo][]` gets + treated as a reference link, regardless of whether `[foo]` + is defined later in the document. A corollary is that we + must get rid of shortcut link syntax, with just a single + bracket pair, `[like this]`. It must always be clear what is a + link without needing to know the surrounding context. + +- In support of goal 6, reference links are no longer + case-insensitive. Supporting this beyond an ASCII context + would require building in unicode case folding to every + implementation, and it doesn't seem necessary. + +- A space or newline is required after `>` in block quotes, + to avoid the violations of the principle of uniformity + noted in goal 8: + + ``` + >This is not a + >block quote in djot. + ``` + +- To meet goal 3, we avoid using doubled characters for + strong emphasis. Instead, we use `_` for emphasis and `*` for + strong emphasis. Emphasis can begin with one of these + characters, as long as it is not followed by a space, + and will end when a similar character is encountered, + as long as it is not preceded by a space and some + different characters have occurred in between. In the case + of overlap, the first one to be closed takes precedence. + (This simple rule also avoids the need we had in commonmark to + determine unicode character classes---goal 6.) + +- Taken just by itself, this last change would introduce a + number of expressive blind spots. For example, given the + simple rule, + ``` + _(_foo_)_ + ``` + parses as + ``` html + (foo) + ``` + rather than + ``` html + (foo) + ``` + If you want the latter + interpretation, djot allows you to use the syntax + ``` + _({_foo_})_ + ``` + The `{_` is a `_` that can only open emphasis, and the `_}` is + a `_` that can only close emphasis. The same can be done with + `*` or any other inline formatting marker that is ambiguous + between an opener and closer. These curly braces are + *required* for certain inline markup, e.g. `{=highlighting=}`, + `{+insert+}`, and `{-delete-}`, since the characters `=`, `+`, + and `-` are found often in ordinary text. + +- In support of goal 1, code span parsing does not backtrack. + So if you open a code span and don't close it, it extends to + the end of the paragraph. That is similar to the way fenced + code blocks work in commonmark. + + ``` + This is `inline code. + ``` + +- In support of goal 9, a generic attribute syntax is + introduced. Attributes can be attached to any block-level + element by putting them on the line before it, and to any + inline-level element by putting them directly after it. + + ``` + {#introduction} + This is the introductory paragraph, with + an identifier `introduction`. + + {.important color="blue" #heading} + ## heading + + The word *atelier*{weight="600"} is French. + ``` + +- Since we are going to have generic attributes, we no longer + support quoted titles in links. One can add a title + attribute if needed, but this isn't very common, so we don't + need a special syntax for it: + + ``` + [Link text](url){title="Click me!"} + ``` + +- Fenced divs and bracketed spans are introduced in order to + allow attributes to be attached to arbitrary sequences of + block-level or inline-level elements. For example, + + ``` + {#warning .sidebar} + ::: Warning + This is a warning. + Here is a word in [français]{lang=fr}. + ::: + ``` + +## Syntax + +For a full syntax reference, see the +[syntax description](https://htmlpreview.github.io/?https://github.com/jgm/djot/blob/master/doc/syntax.html). + +A vim syntax highlighting definition for djot is provided in +`editors/vim/`. + +## Installing + +To install djot using [luarocks](https://luarocks.org), just + +``` +luarocks install djot +``` + +This will install both the library and the executable `djot`. + +## Using the Lua library + +### Quick start + +If you just want to parse some input and produce HTML: + +``` lua +local djot = require("djot") +local input = "This is *djot*" +local doc = djot.parse(input) +local html = djot.render_html(doc) +``` + +The AST is available as a Lua table, `doc.ast`. + +To render the AST: + +``` lua +local rendered = djot.render_ast_pretty(doc) +``` + +Or as JSON: + +``` lua +local rendered = djot.render_ast_json(doc) +``` + +To alter the AST with a filter: + +``` lua +local src = "return { str = function(e) e.text = e.text:upper() end }" +local filter = djot.filter.load_filter(src) +djot.filter.apply_filter(doc, filter) +``` + +For a streaming parser: + +``` lua +for startpos, endpos, annotation in djot.parse_events("*hello there*") do + print(startpos, endpos, annotation) +end +``` + +(This will print start and end byte offsets into the input +for annotated tokens.) + +## The code + +The code for djot (excluding the test suite) is standard Lua, +compatible with lua 5.1--5.4 and luajit. Djot has no external +dependencies. You can run it without installing it using +`./run.sh`. + +`make install` will build the rockspec and install the +library and executable using luarocks. Once installed, +the library can be used by Lua programs, and the executable can +be run using `djot`. `djot -h` will give help output. + +If you can't assume that lua or luajit will be installed on +the target machine, you can use `make djot` in the `clib` +directory to create a portable binary that bakes in a lua +interpreter and the necessary scripts. + +`make test` will run the tests, and `make testall` will also +run some tests of pathological cases. + +## File extension + +The extension `.dj` may be used to indicate that the contents +of a file are djot-formatted text. + +## License + +The code and documentation are released under the MIT license. + diff --git a/bin/main.lua b/bin/main.lua new file mode 100755 index 0000000..56444e9 --- /dev/null +++ b/bin/main.lua @@ -0,0 +1,132 @@ +local djot = require("djot") + +local help = [[ +djot [opts] [file*] + +Options: +--matches -m Show matches. +--ast -a Show AST. +--json -j Use JSON for -m or -a. +--sourcepos -p Include source positions in AST. +--filter FILE -f FILE Filter AST using filter in FILE. +--verbose -v Verbose (show warnings). +--version Show version information. +--help -h Help. +]] + +local function err(msg, code) + io.stderr:write(msg .. "\n") + os.exit(code) +end + +local opts = {} +local files = {} + +local shortcuts = + { m = "--matches", + a = "--ast", + j = "--json", + p = "--sourcepos", + v = "--verbose", + f = "--filter", + h = "--help" } + +local argi = 1 +while arg[argi] do + local thisarg = arg[argi] + local longopts = {} + if string.find(thisarg, "^%-%-%a") then + longopts[#longopts + 1] = thisarg + elseif string.find(thisarg, "^%-%a") then + string.gsub(thisarg, "(%a)", + function(x) + longopts[#longopts + 1] = shortcuts[x] or ("-"..x) + end) + else + files[#files + 1] = thisarg + end + for _,x in ipairs(longopts) do + if x == "--matches" then + opts.matches = true + elseif x == "--ast" then + opts.ast = true + elseif x == "--json" then + opts.json = true + elseif x == "--sourcepos" then + opts.sourcepos = true + elseif x == "--verbose" then + opts.verbose = true + elseif x == "--filter" then + if arg[argi + 1] then + opts.filters = opts.filters or {} + table.insert(opts.filters, arg[argi + 1]) + argi = argi + 1 + end + elseif x == "--version" then + io.stdout:write("djot " .. djot.version .. "\n") + os.exit(0) + elseif x == "--help" then + io.stdout:write(help) + os.exit(0) + else + err("Unknown option " .. x, 1) + end + end + argi = argi + 1 +end + +local inp +if #files == 0 then + inp = io.read("*all") +else + local buff = {} + for _,f in ipairs(files) do + local ok, msg = pcall(function() io.input(f) end) + if ok then + table.insert(buff, io.read("*all")) + else + err(msg, 7) + end + end + inp = table.concat(buff, "\n") +end + +local warn = function(warning) + if opts.verbose then + io.stderr:write(string.format("%s at byte position %d\n", + warning.message, warning.pos)) + end +end + +if opts.matches then + + io.stdout:write(djot.parse_and_render_events(inp, warn)) + +else + + local ast = djot.parse(inp, opts.sourcepos, warn) + + if opts.filters then + for _,fp in ipairs(opts.filters) do + local filt, err = djot.filter.require_filter(fp) + if filt then + djot.filter.apply_filter(ast, filt) + else + io.stderr:write("Error loading filter " .. fp .. ":\n" .. err .. "\n") + end + end + end + + if opts.ast then + if opts.json then + io.stdout:write(djot.render_ast_json(ast)) + else + io.stdout:write(djot.render_ast_pretty(ast)) + end + else + io.stdout:write(djot.render_html(ast)) + end + +end + +os.exit(0) diff --git a/clib/Makefile b/clib/Makefile new file mode 100644 index 0000000..f9356ef --- /dev/null +++ b/clib/Makefile @@ -0,0 +1,79 @@ +LUAVERSION=5.4.4 +LUAURL=https://www.lua.org/ftp/lua-$(LUAVERSION).tar.gz +LUA=$(shell which lua || echo "lua-src/lua") +LUASRC=lua-src +LIBLUA=$(LUASRC)/liblua.a +LUAHEADERS=$(LUASRC) +# Or, for example, to use homebrew-installed luajit on macos: +# LIBLUA=$(shell brew --prefix)/lib/libluajit.a +# LUAHEADERS=$(shell brew --prefix)/include/luajit-2.1 +MODULES=$(patsubst %, ../%, djot/attributes.lua djot/inline.lua djot/block.lua djot/ast.lua djot/html.lua djot/filter.lua djot/json.lua djot.lua) +LUASRCS=$(patsubst %,$(LUASRC)/%.c,lapi lgc lstate lauxlib linit lstring lbaselib liolib lstrlib lcode llex ltable lcorolib lmathlib ltablib lctype lmem ltm ldblib loadlib luac ldebug lobject lundump ldo lopcodes lutf8lib ldump loslib lvm lfunc lparser lzio) +CFLAGS= -std=c99 -O2 -Wall -DLUA_COMPAT_5_3 $(SYSCFLAGS) $(MYCFLAGS) +UNAME=$(shell uname) +ifeq (MSYS,$(findstring MSYS,$(UNAME))) +PLAT=mingw +CC=gcc +else +PLAT=guess +endif +ifeq ($(UNAME), Linux) +LDFLAGS=-static -lm $(SYSLDFLAGS) $(MYLDFLAGS) +else +LDFLAGS=$(SYSLDFLAGS) $(MYLDFLAGS) +endif + +test: djot tests + ./tests +.PHONY: test + +tests: tests.c djot.o $(LIBLUA) + $(CC) $(CFLAGS) $^ -I . -I $(LUAHEADERS) -o $@ $(LDFLAGS) + +djot: main.c djot.o $(LIBLUA) djot_main.inc + $(CC) $(CFLAGS) $< djot.o $(LIBLUA) -I . -I $(LUAHEADERS) -o $@ $(LDFLAGS) + +libdjot.a: djot.o + ar -rc $@ $< + +$(LUASRC)/liblua.a: $(LUASRC) + make -C $(LUASRC) liblua.a + +$(LUASRC): + mkdir -p $(LUASRC) + curl $(LUAURL) | tar xzv -C $(LUASRC) --strip-components=2 lua-$(LUAVERSION)/src + +djot.o: djot.c djot_combined.inc $(LUASRC) + $(CC) $(CFLAGS) -c -I . -I $(LUAHEADERS) $< + +$(LUASRC)/lua: $(LUASRC) + make -C $(LUASRC) $(PLAT) + +djot_combined.lua: $(LUA) $(MODULES) dumbParser.lua + $(LUA) combine.lua $(MODULES) > $@ + +djot_combined.inc: djot_combined.lua + echo "unsigned char djot_combined_lua[] = {" > $@ + (cat $< && printf "\0") | xxd -i >> $@ + echo "};" >> $@ + +djot_main.inc: ../bin/main.lua + echo "unsigned char djot_main_lua[] = {" > $@ + (cat $< && printf "\0") | xxd -i >> $@ + echo "};" >> $@ + +dumbParser.lua: + curl -L https://github.com/ReFreezed/DumbLuaParser/releases/download/2.3.0/dumbParser.lua > $@ + +wasm: djot.js +.PHONY: wasm + +# This also creates djot.wasm +djot.js: djot.c djot_combined.inc $(LUAOBJS) + emcc -g0 -sALLOW_MEMORY_GROWTH -Oz -sFILESYSTEM=0 -s 'EXPORTED_RUNTIME_METHODS=["cwrap"]' -s 'EXPORTED_FUNCTIONS=["_djot_open", "_djot_report_error", "_djot_close", "_djot_parse", "_djot_render_ast_pretty", "_djot_render_ast_json", "_djot_parse_and_render_events", "_djot_render_html", "_djot_apply_filter", "_djot_get_error"]' -I $(LUASRC) -I . $(LUASRCS) -o $@ $< + +clean: + rm -rf djot_combined.lua djot_combined.inc *.o test +distclean: clean + -rm -rf $(LUASRC) djot.js djot.wasm dumbParser.lua +.PHONY: clean diff --git a/clib/README.md b/clib/README.md new file mode 100644 index 0000000..6fc1bd2 --- /dev/null +++ b/clib/README.md @@ -0,0 +1,24 @@ +# libdjot + +Some experiments in creating a C library that embeds the djot lua code. + +`make` builds a static library `libdjot.a` and an executable +`tests`, then runs the tests. + +Note: you may need to adjust the paths in the Makefile pointing +to your lua library installation. + +For documentation, see the comments in `djot.h`. + +For an example of the use of the library, see `tests.c`. + +If you have emscripten installed (`emcc`), you can compile to +wasm/js and run djot in the browser: + +``` +$ make wasm +$ cd web/dist +$ python3 -m http.server +$ open http://localhost:8000/ +``` + diff --git a/clib/combine.lua b/clib/combine.lua new file mode 100644 index 0000000..3ce26d8 --- /dev/null +++ b/clib/combine.lua @@ -0,0 +1,31 @@ +-- combine modules (specified as cli arguments) into one file +package.path = "./?.lua;" .. package.path +local parser = require("dumbParser") + +local modules = {} +for i=1,#arg do + modules[#modules + 1] = + arg[i]:gsub("^../",""):gsub("%.lua$",""):gsub("%/",".") +end + +local buffer = {} +local function out(s) + buffer[#buffer + 1] = s +end + +for _,module in ipairs(modules) do + out(string.format('package.preload["%s"] = function()', module)) + local path = "../" .. module:gsub("%.","/") .. ".lua" + local f = assert(io.open(path, "r")) + local content = f:read("*all") + out(content) + out('end\n') +end + +out('local djot = require("djot")') +out('return djot') + +local combined = table.concat(buffer, "\n") +local ast = parser.parse(combined) +parser.minify(ast) +io.stdout:write(parser.toLua(ast, false)) diff --git a/clib/djot.c b/clib/djot.c new file mode 100644 index 0000000..15a1b02 --- /dev/null +++ b/clib/djot.c @@ -0,0 +1,138 @@ +#include +#include +#include +#include +#include +#include + +#include "djot_combined.inc" +/* unsigned char djot_combined_lua[] */ + +void djot_report_error(lua_State *L) { + if(!L) { + fprintf(stderr, "lua_State is NULL\n"); + } else { + fprintf(stderr, "error: %s\n", lua_tostring(L, -1)); + } +} + +const char *djot_get_error(lua_State *L) { + if(!L) { + return "lua_State is NULL\n"; + } else { + return lua_tostring(L, -1); + } +} + +lua_State *djot_open() { + lua_State *L = luaL_newstate(); /* create Lua state */ + if (L == NULL) { + return NULL; + } + luaL_openlibs(L); /* opens Lua libraries */ + + if (luaL_dostring(L, (const char*)djot_combined_lua) != LUA_OK) { + djot_report_error(L); + return NULL; + } + + lua_setglobal(L, "djot"); + + return L; +} + +void djot_close(lua_State *L) { + lua_close(L); +} + +/* Parse input (optionally including source positions) and add + * a global 'doc' with the parsed AST. The + * subordinate functions djot_render_html, djot_render_ast, + * djot_render_matches, djot_apply_filter can then be used to manipulate + * or render the content. Returns 1 on success, 0 on error. */ +int djot_parse(lua_State *L, char *input, bool sourcepos) { + lua_getglobal(L, "djot"); + lua_getfield(L, -1, "parse"); + lua_pushstring(L, input); + lua_pushboolean(L, sourcepos); + if (lua_pcall(L, 2, 1, 0) != LUA_OK) { + return 0; + } + lua_setglobal(L, "doc"); + return 1; +} + +/* Render the document in the global 'doc' as HTML, returning a string, + * or NULL on error. */ +char *djot_render_html(lua_State *L) { + lua_getglobal(L, "djot"); + lua_getfield(L, -1, "render_html"); + lua_getglobal(L, "doc"); + if (lua_pcall(L, 1, 1, 0) != LUA_OK) { + return NULL; + } + return (char *)lua_tostring(L, -1); +} + +/* Render the AST of the document in the global 'doc' as JSON. + * NULL is returned on error. */ +char *djot_render_ast_json(lua_State *L) { + lua_getglobal(L, "djot"); + lua_getfield(L, -1, "render_ast_json"); + lua_getglobal(L, "doc"); + if (lua_pcall(L, 1, 1, 0) != LUA_OK) { + return NULL; + } + return (char *)lua_tostring(L, -1); +} + +/* Render the AST of the document in the global 'doc' as JSON. + * NULL is returned on error. */ +char *djot_render_ast_pretty(lua_State *L) { + lua_getglobal(L, "djot"); + lua_getfield(L, -1, "render_ast_pretty"); + lua_getglobal(L, "doc"); + if (lua_pcall(L, 1, 1, 0) != LUA_OK) { + return NULL; + } + return (char *)lua_tostring(L, -1); +} + +/* Load a filter from a string and apply it to the AST in global 'doc'. + * Return 1 on success, 0 on error. */ +int djot_apply_filter(lua_State *L, char *filter) { + lua_getglobal(L, "djot"); + lua_getfield(L, -1, "filter"); + lua_getfield(L, -1, "load_filter"); + lua_pushstring(L, filter); + if (lua_pcall(L, 1, 1, 0) != LUA_OK) { + return 0; + } + // Now we should have the loaded filter on top of stack, or nil and an error + if lua_isnil(L, -2) { + return 0; + } + // If we're here, top of stack should be the compiled filter + lua_getglobal(L, "djot"); + lua_getfield(L, -1, "filter"); + lua_getfield(L, -1, "apply_filter"); + lua_getglobal(L, "doc"); + lua_pushvalue(L, -5); /* push the compiled filter to top of stack */ + if (lua_pcall(L, 2, 1, 0) != LUA_OK) { + return 0; + } + return 1; +} + +/* Parse input and render the events as a JSON array. + * NULL is returned on error. */ +char *djot_parse_and_render_events(lua_State *L, char *input) { + lua_getglobal(L, "djot"); + lua_getfield(L, -1, "parse_and_render_events"); + lua_pushstring(L, input); + if (lua_pcall(L, 1, 1, 0) != LUA_OK) { + return NULL; + } + return (char *)lua_tostring(L, -1); +} + diff --git a/clib/djot.h b/clib/djot.h new file mode 100644 index 0000000..8d899cf --- /dev/null +++ b/clib/djot.h @@ -0,0 +1,55 @@ +#ifndef DJOT_H +#define DJOT_H +#include +#include +#include +#include + +/* Open a Lua virtual machine and load the djot code. + * This should only be done once, before all use of djot functions. + * The state can be closed with djot_close. */ +lua_State *djot_open(); + +/* Close the Lua virtual machine opened by djot_open, freeing its + * memory. */ +void djot_close(lua_State *L); + +/* Report the error on the top of the Lua stack. This should + * be run immediately if a function that is supposed to return + * a pointer returns NULL. */ +void djot_report_error(lua_State *L); + +/* Return string version of error on top of Lua stack. +* This should be run immediately if a function that is supposed to return +* a pointer returns NULL. */ +const char *djot_get_error(lua_State *L); + +/* Parse input (optionally including source positions) and add + * a global 'doc' with the parsed AST. The + * subordinate functions djot_render_html, djot_render_ast, + * djot_render_matches, djot_apply_filter can then be used to manipulate + * or render the content. Returns 1 on success, 0 on error. */ +int djot_parse(lua_State *L, char *input, bool sourcepos); + +/* Render the document in the global 'doc' as HTML, returning a string, + * or NULL on error. */ +char *djot_render_html(lua_State *L); + +/* Render the AST of the document in the global 'doc' as JSON. + * NULL is returned on error. */ +char *djot_render_ast_json(lua_State *L); + +/* Render the AST of the document in the global 'doc' as JSON. + * NULL is returned on error. */ +char *djot_render_ast_pretty(lua_State *L); + +/* Tokenize input and render the matches. + * If 'as_json' is true, use JSON, otherwise, produce a compact + * human-readable tree. NULL is returned on error. */ +char *djot_parse_and_render_events(lua_State *L, char *input); + +/* Load a filter from a string and apply it to the AST in global 'doc'. + * Return 1 on success, 0 on error. */ +int djot_apply_filter(lua_State *L, char *filter); + +#endif diff --git a/clib/main.c b/clib/main.c new file mode 100644 index 0000000..e743c29 --- /dev/null +++ b/clib/main.c @@ -0,0 +1,40 @@ +#include "djot.h" +#include +#include +#include +#include "lauxlib.h" + +#include "djot_main.inc" +/* unsigned char djot_main_lua[], unsigned int djot_main_lua_len */ + +int main(int argc, char* argv[]) { + int status; + /* Do this once, before any use of the djot library */ + lua_State *L = djot_open(); + if (!L) { + fprintf(stderr, "djot_open returned NULL.\n"); + return -1; + } + + // start array structure + lua_newtable( L ); + + for (int i=1; i<=argc; i++) { + + lua_pushnumber( L, i ); + lua_pushstring( L, argv[i] ); + lua_rawset( L, -3 ); + + } + + lua_setglobal( L, "arg" ); + + status = luaL_dostring(L, (char*)djot_main_lua); + if (status != LUA_OK) { + djot_report_error(L); + } + + djot_close(L); + return 0; +} + diff --git a/clib/tests.c b/clib/tests.c new file mode 100644 index 0000000..95d9ec1 --- /dev/null +++ b/clib/tests.c @@ -0,0 +1,85 @@ +#include +#include +#include +#include +#include "djot.h" + +int failed = 0; +int num = 0; +int result; + +static void asserteq(char *actual, char *expected) { + num = num + 1; + if (strcmp(actual, expected) == 0) { + printf("Test %4d PASSED\n", num); + } else { + printf("Test %4d FAILED\nExpected:\n%s\nGot:\n%s\n", num, expected, actual); + failed = failed + 1; + } +} + +static int error(lua_State *L) { + djot_report_error(L); + exit(1); +} + +int main (void) { + char *out; + int ok; + + /* Do this once, before any use of the djot library */ + lua_State *L = djot_open(); + if (!L) { + fprintf(stderr, "djot_open returned NULL.\n"); + exit(1); + } + + out = djot_parse_and_render_events(L, "hi *there*\n"); + if (!out) error(L); + asserteq(out, +"[[\"+para\",1,1]\n\ +,[\"str\",1,3]\n\ +,[\"+strong\",4,4]\n\ +,[\"str\",5,9]\n\ +,[\"-strong\",10,10]\n\ +,[\"-para\",11,11]\n\ +]\n"); + + ok = djot_parse(L, "hi *there*\n", true); + if (!ok) error(L); + out = djot_render_html(L); + if (!out) error(L); + asserteq(out, +"

hi there

\n"); + + out = djot_render_ast_json(L); + if (!out) error(L); + asserteq(out, +"{\"tag\":\"doc\",\"children\":[{\"tag\":\"para\",\"pos\":[\"1:1:1\",\"1:11:11\"],\"children\":[{\"tag\":\"str\",\"text\":\"hi \",\"pos\":[\"1:1:1\",\"1:3:3\"]},{\"tag\":\"strong\",\"pos\":[\"1:4:4\",\"1:10:10\"],\"children\":[{\"tag\":\"str\",\"text\":\"there\",\"pos\":[\"1:5:5\",\"1:9:9\"]}]}]}],\"references\":[],\"footnotes\":[]}\n"); + + char *capsfilter = "return {\n\ +str = function(e)\n\ + e.text = e.text:upper()\n\ +end\n\ +}\n"; + + result = djot_apply_filter(L, capsfilter); + if (!result) { + error(L); + } else { + out = djot_render_html(L); + if (!out) error(L); + asserteq(out, +"

HI THERE

\n"); + } + + /* When you're finished, close the djot library */ + djot_close(L); + + if (failed) { + printf("%d tests failed.\n", failed); + } else { + printf("All good.\n"); + } + return failed; +} diff --git a/config.ld b/config.ld new file mode 100644 index 0000000..2365898 --- /dev/null +++ b/config.ld @@ -0,0 +1,7 @@ +project = 'djot' +description = 'djot markup conversion' +file = {'djot.lua', 'djot'} +dir = 'doc/api' +format = 'markdown' +not_luadoc = true +template = '.' diff --git a/djot-reader.lua b/djot-reader.lua new file mode 100644 index 0000000..6973868 --- /dev/null +++ b/djot-reader.lua @@ -0,0 +1,459 @@ +local djot = require("djot") +local ast = require("djot.ast") +local insert_attribute, copy_attributes = + ast.insert_attribute, ast.copy_attributes +local emoji -- require this later, only if emoji encountered +local format = string.format +local find, gsub = string.find, string.gsub + +-- Produce a copy of a table. +local function copy(tbl) + local result = {} + if tbl then + for k,v in pairs(tbl) do + local newv = v + if type(v) == "table" then + newv = copy(v) + end + result[k] = newv + end + end + return result +end + +local Renderer = {} + +function Renderer:new() + local state = { + tight = false, + footnotes = nil, + references = nil + } + setmetatable(state, self) + self.__index = self + return state +end + +local function words(s) + if s then + local res = {} + string.gsub(s, "(%S+)", function(x) table.insert(res, x) end) + return res + else + return {} + end +end + +local function to_attr(attr) + if not attr then + return nil + end + local result = copy(attr) + result.id = nil + result.class = nil + return pandoc.Attr(attr.id or "", words(attr.class), result) +end + +function Renderer:with_optional_span(node, f) + local base = f(self:render_children(node)) + if node.attr then + return pandoc.Span(base, to_attr(node.attr)) + else + return base + end +end + +function Renderer:with_optional_div(node, f) + local base = f(self:render_children(node)) + if node.attr then + return pandoc.Div(base, to_attr(node.attr)) + else + return base + end +end + +function Renderer:render_node(node) + return self[node.tag](self, node) +end + +function Renderer:render_children(node) + local buff = {} + local inline = false + if node.children and #node.children > 0 then + local oldtight + if node.tight ~= nil then + oldtight = self.tight + self.tight = node.tight + end + local function integrate_elt(elt) + if elt.__name == "Inlines" or elt.__name == "Blocks" then + for i=1,#elt do + integrate_elt(elt[i]) + end + else + buff[#buff + 1] = elt + end + end + for _,child in ipairs(node.children) do + local elt = self:render_node(child) + integrate_elt(elt) + end + if node.tight ~= nil then + self.tight = oldtight + end + end + return buff +end + +function Renderer:doc(node) + self.footnotes = node.footnotes + self.references = node.references + return pandoc.Pandoc(self:render_children(node)) +end + +function Renderer:section(node) + local attrs = to_attr(node.attr) + table.insert(attrs.classes, 1, "section") + return pandoc.Div(self:render_children(node), attrs) +end + +function Renderer:raw_block(node) + return pandoc.RawBlock(node.format, node.text) +end + +function Renderer:para(node) + local constructor = pandoc.Para + if self.tight then + constructor = pandoc.Plain + end + return self:with_optional_div(node, constructor) +end + +function Renderer:blockquote(node) + return self:with_optional_div(node, pandoc.BlockQuote) +end + +function Renderer:div(node) + return pandoc.Div(self:render_children(node), to_attr(node.attr)) +end + +function Renderer:heading(node) + return pandoc.Header(node.level, + self:render_children(node), + to_attr(node.attr)) +end + +function Renderer:thematic_break(node) + if node.attr then + return pandoc.Div(pandoc.HorizontalRule(), to_attr(node.attr)) + else + return pandoc.HorizontalRule() + end +end + +function Renderer:code_block(node) + local attr = copy(to_attr(node.attr)) + if not attr.class then + attr.class = node.lang + else + attr.class = node.lang .. " " .. attr.class + end + return pandoc.CodeBlock(node.text:gsub("\n$",""), attr) +end + +function Renderer:table(node) + local rows = {} + local headers = {} + local caption = {} + local aligns = nil + local widths = nil + for i=2,#node do + local row = node[i] + if row[1] == "caption" then + caption = self:render_children(row) + elseif row[1] == "row" then + if not aligns then + aligns = {} + widths = {} + for j=2,#row do + local align = row[j].align + if not align then + aligns[j - 1] = "AlignDefault" + elseif align == "center" then + aligns[j - 1] = "AlignCenter" + elseif align == "left" then + aligns[j - 1] = "AlignLeft" + elseif align == "right" then + aligns[j - 1] = "AlignRight" + end + widths[j - 1] = 0 + end + end + local cells = self:render_children(row) + if i == 2 and row.head then + headers = cells + else + rows[#rows + 1] = cells + end + end + end + return pandoc.utils.from_simple_table( + pandoc.SimpleTable(caption, aligns, widths, headers, rows)) +end + +function Renderer:cell(node) + return { pandoc.Plain(self:render_children(node)) } +end + +function Renderer:list(node) + local sty = node.style + if sty == "*" or sty == "+" or sty == "-" then + return self:with_optional_div(node, pandoc.BulletList) + elseif sty == "X" then + return self:with_optional_div(node, pandoc.BulletList) + elseif sty == ":" then + return self:with_optional_div(node, pandoc.DefinitionList) + else + local start = 1 + local sty = "DefaultStyle" + local delim = "DefaultDelim" + if node.start and node.start > 1 then + start = node.start + end + local list_type = gsub(node.style, "%p", "") + if list_type == "a" then + sty = "LowerAlpha" + elseif list_type == "A" then + sty = "UpperAlpha" + elseif list_type == "i" then + sty = "LowerRoman" + elseif list_type == "I" then + sty = "UpperRoman" + end + local list_delim = gsub(node.style, "%P", "") + if list_delim == ")" then + delim = "OneParen" + elseif list_delim == "()" then + delim = "TwoParens" + end + return self:with_optional_div(node, function(x) + return pandoc.OrderedList(x, + pandoc.ListAttributes(start, sty, delim)) + end) + end +end + +function Renderer:list_item(node) + local children = self:render_children(node) + if node.checkbox then + local box = (node.checkbox == "checked" and "☒") or "☐" + local tag = children[1].tag + if tag == "Para" or tag == "Plain" then + children[1].content:insert(1, pandoc.Space()) + children[1].content:insert(1, pandoc.Str(box)) + else + children:insert(1, pandoc.Para{pandoc.Str(box), pandoc.Space()}) + end + end + return children +end + +function Renderer:definition_list_item(node) + local term = self:render_node(node.children[1]) + local defn = self:render_node(node.children[2]) + return { term, defn } +end + +function Renderer:term(node) + return self:render_children(node) +end + +function Renderer:definition(node) + return self:render_children(node) +end + +function Renderer:reference_definition() + return "" +end + +function Renderer:footnote_reference(node) + local label = node.text + local note = self.footnotes[label] + if note then + return pandoc.Note(self:render_children(note)) + else + io.stderr:write("Note " .. label .. " not found.") + return pandoc.Str("[^" .. label .. "]") + end +end + +function Renderer:raw_inline(node) + return pandoc.RawInline(node.format, node.text) +end + +function Renderer:str(node) + -- add a span, if needed, to contain attribute on a bare string: + if node.attr then + return pandoc.Span(pandoc.Inlines(node.text), to_attr(node.attr)) + else + return pandoc.Inlines(node.text) + end +end + +function Renderer:softbreak() + return pandoc.SoftBreak() +end + +function Renderer:hardbreak() + return pandoc.LineBreak() +end + +function Renderer:nbsp() + return pandoc.Str(" ") +end + +function Renderer:verbatim(node) + return pandoc.Code(node.text, to_attr(node.attr)) +end + +function Renderer:link(node) + local attrs = {} + local dest = node.destination + if node.reference then + local ref = self.references[node.reference] + if ref then + if ref.attributes then + attrs = copy(ref.attributes) + end + dest = ref.destination + else + dest = "#" -- empty href is illegal + end + end + -- link's attributes override reference's: + copy_attributes(attrs, node.attr) + local title = attrs.title + attrs.title = nil + return pandoc.Link(self:render_children(node), dest, + title, to_attr(attrs)) +end + +function Renderer:image(node) + local attrs = {} + local dest = node.destination + if node.reference then + local ref = self.references[node.reference] + if ref then + if ref.attributes then + attrs = copy(ref.attributes) + end + dest = ref.destination + else + dest = "#" -- empty href is illegal + end + end + -- image's attributes override reference's: + copy_attributes(attrs, node.attr) + return pandoc.Image(self:render_children(node), dest, + title, to_attr(attrs)) +end + +function Renderer:span(node) + return pandoc.Span(self:render_children(node), to_attr(node.attr)) +end + +function Renderer:mark(node) + local attr = copy(node.attr) + if attr.class then + attr.class = "mark " .. attr.class + else + attr = { class = "mark" } + end + return pandoc.Span(self:render_children(node), to_attr(attr)) +end + +function Renderer:insert(node) + local attr = copy(node.attr) + if attr.class then + attr.class = "insert " .. attr.class + else + attr = { class = "insert" } + end + return pandoc.Span(self:render_children(node), to_attr(attr)) +end + +function Renderer:delete(node) + return self:with_optional_span(node, pandoc.Strikeout) +end + +function Renderer:subscript(node) + return self:with_optional_span(node, pandoc.Subscript) +end + +function Renderer:superscript(node) + return self:with_optional_span(node, pandoc.Superscript) +end + +function Renderer:emph(node) + return self:with_optional_span(node, pandoc.Emph) +end + +function Renderer:strong(node) + return self:with_optional_span(node, pandoc.Strong) +end + +function Renderer:double_quoted(node) + return self:with_optional_span(node, + function(x) return pandoc.Quoted("DoubleQuote", x) end) +end + +function Renderer:single_quoted(node) + return self:with_optional_span(node, + function(x) return pandoc.Quoted("SingleQuote", x) end) +end + +function Renderer:left_double_quote() + return "“" +end + +function Renderer:right_double_quote() + return "”" +end + +function Renderer:left_single_quote() + return "‘" +end + +function Renderer:right_single_quote() + return "’" +end + +function Renderer:ellipses() + return "…" +end + +function Renderer:em_dash() + return "—" +end + +function Renderer:en_dash() + return "–" +end + +function Renderer:symbol(node) + return pandoc.Span(":" .. node.alias .. ":", + pandoc.Attr("",{"symbol"},{["alias"] = node.alias})) +end + +function Renderer:math(node) + local math_type = "InlineMath" + if find(node.attr.class, "display") then + math_type = "DisplayMath" + end + return pandoc.Math(math_type, node.text) +end + +function Reader(input) + local doc = djot.parse(tostring(input)) + return Renderer:render_node(doc) +end diff --git a/djot-writer.lua b/djot-writer.lua new file mode 100644 index 0000000..ac33bf4 --- /dev/null +++ b/djot-writer.lua @@ -0,0 +1,448 @@ +-- custom writer for pandoc + +local unpack = unpack or table.unpack +local format = string.format +local layout = pandoc.layout +local literal, empty, cr, concat, blankline, chomp, space, cblock, rblock, + prefixed, nest, hang, nowrap = + layout.literal, layout.empty, layout.cr, layout.concat, layout.blankline, + layout.chomp, layout.space, layout.cblock, layout.rblock, + layout.prefixed, layout.nest, layout.hang, layout.nowrap +local to_roman = pandoc.utils.to_roman_numeral + +local footnotes = {} + +-- Escape special characters +local function escape(s) + return (s:gsub("[][\\`{}_*<>~^'\"]", function(s) return "\\" .. s end)) +end + +local format_number = {} +format_number.Decimal = function(n) + return format("%d", n) +end +format_number.Example = format_number.Decimal +format_number.DefaultStyle = format_number.Decimal +format_number.LowerAlpha = function(n) + return string.char(96 + (n % 26)) +end +format_number.UpperAlpha = function(n) + return string.char(64 + (n % 26)) +end +format_number.UpperRoman = function(n) + return to_roman(n) +end +format_number.LowerRoman = function(n) + return string.lower(to_roman(n)) +end + +local function is_tight_list(el) + if not (el.tag == "BulletList" or el.tag == "OrderedList" or + el.tag == "DefinitionList") then + return false + end + for i=1,#el.content do + if #el.content[i] == 1 and el.content[i][1].tag == "Plain" then + -- no change + elseif #el.content[i] == 2 and el.content[i][1].tag == "Plain" and + el.content[i][2].tag:match("List") then + -- no change + else + return false + end + end + return true +end + +local function has_attributes(el) + return el.attr and + (#el.attr.identifier > 0 or #el.attr.classes > 0 or #el.attr.attributes > 0) +end + +local function render_attributes(el, isblock) + if not has_attributes(el) then + return empty + end + local attr = el.attr + local buff = {"{"} + if #attr.identifier > 0 then + buff[#buff + 1] = "#" .. attr.identifier + end + for i=1,#attr.classes do + if #buff > 1 then + buff[#buff + 1] = space + end + buff[#buff + 1] = "." .. attr.classes[i] + end + for k,v in pairs(attr.attributes) do + if #buff > 1 then + buff[#buff + 1] = space + end + buff[#buff + 1] = k .. '="' .. v:gsub('"', '\\"') .. '"' + end + buff[#buff + 1] = "}" + if isblock then + return rblock(nowrap(concat(buff)), PANDOC_WRITER_OPTIONS.columns) + else + return concat(buff) + end +end + +Blocks = {} +Blocks.mt = {} +Blocks.mt.__index = function(tbl,key) + return function() io.stderr:write("Unimplemented " .. key .. "\n") end +end +setmetatable(Blocks, Blocks.mt) + +Inlines = {} +Inlines.mt = {} +Inlines.mt.__index = function(tbl,key) + return function() io.stderr:write("Unimplemented " .. key .. "\n") end +end +setmetatable(Inlines, Inlines.mt) + +local function inlines(ils) + local buff = {} + for i=1,#ils do + local el = ils[i] + buff[#buff + 1] = Inlines[el.tag](el) + end + return concat(buff) +end + +local function blocks(bs, sep) + local dbuff = {} + for i=1,#bs do + local el = bs[i] + dbuff[#dbuff + 1] = Blocks[el.tag](el) + end + return concat(dbuff, sep) +end + +Blocks.Para = function(el) + return inlines(el.content) +end + +Blocks.Plain = function(el) + return inlines(el.content) +end + +Blocks.BlockQuote = function(el) + return prefixed(nest(blocks(el.content, blankline), 1), ">") +end + +Blocks.Header = function(el) + local attr = render_attributes(el, true) + local result = {attr, cr, (string.rep("#", el.level)), space, inlines(el.content)} + return concat(result) +end + +Blocks.Div = function(el) + if el.classes:includes("section") then + -- sections are implicit in djot + if el.identifier and el.content[1].t == "Header" and + el.content[1].identifier == "" then + el.content[1].identifier = el.identifier + end + return blocks(el.content, blankline) + else + local attr = render_attributes(el, true) + return concat{attr, cr, ":::", cr, blocks(el.content, blankline), cr, ":::"} + end +end + +Blocks.RawBlock = function(el) + if el.format == "djot" then + return concat{el.text, cr} + else + local ticks = 3 + el.text:gsub("(`+)", function(s) if #s >= ticks then ticks = #s + 1 end end) + local fence = string.rep("`", ticks) + return concat{fence, " =" .. el.format, cr, + el.text, cr, fence, cr} + end +end + +Blocks.Null = function(el) + return empty +end + +Blocks.LineBlock = function(el) + local result = {} + for i=1,#el.content do + result[#result + 1] = inlines(el.content[i]) + end + return concat(result, concat{"\\", cr}) +end + +Blocks.Table = function(el) + local attr = render_attributes(el, true) + local tbl = pandoc.utils.to_simple_table(el) + -- sanity check to make sure a pipe table will work: + for i=1,#tbl.rows do + for j=1,#tbl.rows[i] do + local cell = tbl.rows[i][j] + if not (#cell == 0 or + (#cell == 1 and (cell.tag == "Plain" or cell.tag == "Para"))) then + -- can't be pipe table, so return a code block with plain table + local plaintable = pandoc.write(pandoc.Pandoc({el}), "plain") + return Blocks.CodeBlock(pandoc.CodeBlock(plaintable)) + end + end + end + local cellsep = " | " + local rows = {} + local hdrcells = {} + for j=1, #tbl.headers do + local cell = tbl.headers[j] + hdrcells[#hdrcells + 1] = blocks(cell, blankline) + end + if #hdrcells > 0 then + rows[#rows + 1] = + concat{"| ", concat(hdrcells, cellsep), " |", cr} + local bordercells = {} + for j=1, #hdrcells do + local w = layout.offset(hdrcells[j]) + local lm, rm = "-", "-" + local align = tbl.aligns[j] + if align == "AlignLeft" or align == "AlignCenter" then + lm = ":" + end + if align == "AlignRight" or align == "AlignCenter" then + rm = ":" + end + bordercells[#bordercells + 1] = lm .. string.rep("-", w) .. rm + end + rows[#rows + 1] = + nowrap(concat{"|", concat(bordercells, "|"), "|", cr}) + end + for i=1, #tbl.rows do + local cells = {} + local row = tbl.rows[i] + for j=1, #row do + local cell = row[j] + cells[#cells + 1] = blocks(cell, blankline) + end + rows[#rows + 1] = + nowrap(concat{"| ", concat(cells, cellsep), " |", cr}) + end + local caption = empty + if #tbl.caption > 0 then + caption = concat{blankline, "^ ", inlines(tbl.caption), cr} + end + return concat{attr, concat(rows), caption} +end + +Blocks.DefinitionList = function(el) + local result = {} + for i=1,#el.content do + local term , defs = unpack(el.content[i]) + local inner = empty + for j=1,#defs do + inner = concat{inner, blankline, blocks(defs[j], blankline)} + end + result[#result + 1] = + hang(inner, 2, concat{ ":", space, inlines(term), cr }) + end + return concat(result, blankline) +end + +Blocks.BulletList = function(el) + local attr = render_attributes(el, true) + local result = {attr, cr} + for i=1,#el.content do + result[#result + 1] = hang(blocks(el.content[i], blankline), 2, concat{"-",space}) + end + local sep = blankline + if is_tight_list(el) then + sep = cr + end + return concat(result, sep) +end + +Blocks.OrderedList = function(el) + local attr = render_attributes(el, true) + local result = {attr, cr} + local num = el.start + local width = 3 + local maxnum = num + #el.content + if maxnum > 9 then + width = 4 + end + local delimfmt = "%s." + if el.delimiter == "OneParen" then + delimfmt = "%s)" + elseif el.delimiter == "TwoParens" then + delimfmt = "(%s)" + end + local sty = el.style + for i=1,#el.content do + local barenum = format_number[sty](num) + local numstr = format(delimfmt, barenum) + local sps = width - #numstr + local numsp + if sps < 1 then + numsp = space + else + numsp = string.rep(" ", sps) + end + result[#result + 1] = hang(blocks(el.content[i], blankline), width, concat{numstr,numsp}) + num = num + 1 + end + local sep = blankline + if is_tight_list(el) then + sep = cr + end + return concat(result, sep) +end + +Blocks.CodeBlock = function(el) + local ticks = 3 + el.text:gsub("(`+)", function(s) if #s >= ticks then ticks = #s + 1 end end) + local fence = string.rep("`", ticks) + local lang = empty + if #el.classes > 0 then + lang = " " .. el.classes[1] + table.remove(el.classes, 1) + end + local attr = render_attributes(el, true) + local result = { attr, cr, fence, lang, cr, el.text, cr, fence, cr } + return concat(result) +end + +Blocks.HorizontalRule = function(el) + return cblock("* * * * *", PANDOC_WRITER_OPTIONS.columns) +end + +Inlines.Str = function(el) + return escape(el.text) +end + +Inlines.Space = function(el) + return space +end + +Inlines.SoftBreak = function(el) + if PANDOC_WRITER_OPTIONS.wrap_text == "wrap-preserve" then + return cr + else + return space + end +end + +Inlines.LineBreak = function(el) + return concat{ "\\", cr } +end + +Inlines.RawInline = function(el) + if el.format == "djot" then + return el.text + else + return concat{Inlines.Code(el), "{=", el.format, "}"} + end +end + +Inlines.Code = function(el) + local ticks = 0 + el.text:gsub("(`+)", function(s) if #s > ticks then ticks = #s end end) + local use_spaces = el.text:match("^`") or el.text:match("`$") + local start = string.rep("`", ticks + 1) .. (use_spaces and " " or "") + local finish = (use_spaces and " " or "") .. string.rep("`", ticks + 1) + local attr = render_attributes(el) + local result = { start, el.text, finish, attr } + return concat(result) +end + +Inlines.Emph = function(el) + return concat{ "_", inlines(el.content), "_" } +end + +Inlines.Strong = function(el) + return concat{ "*", inlines(el.content), "*" } +end + +Inlines.Strikeout = function(el) + return concat{ "{-", inlines(el.content), "-}"} +end + +Inlines.Subscript = function(el) + return concat{ "{~", inlines(el.content), "~}"} +end + +Inlines.Superscript = function(el) + return concat{ "{^", inlines(el.content), "^}"} +end + +Inlines.SmallCaps = function(el) + return concat{ "[", inlines(el.content), "]{.smallcaps}"} +end + +Inlines.Underline = function(el) + return concat{ "[", inlines(el.content), "]{.underline}"} +end + +Inlines.Cite = function(el) + return inlines(el.content) +end + +Inlines.Math = function(el) + local marker + if el.mathtype == "DisplayMath" then + marker = "$$" + else + marker = "$" + end + return concat{ marker, Inlines.Code(el) } +end + +Inlines.Span = function(el) + local attr = render_attributes(el) + return concat{"[", inlines(el.content), "]", attr} +end + +Inlines.Link = function(el) + if el.title and #el.title > 0 then + el.attributes.title = el.title + el.title = nil + end + local attr = render_attributes(el) + local result = {"[", inlines(el.content), "](", + el.target, ")", attr} + return concat(result) +end + +Inlines.Image = function(el) + if el.title and #el.title > 0 then + el.attributes.title = el.title + el.title = nil + end + local attr = render_attributes(el) + local result = {"![", inlines(el.caption), "](", + el.src, ")", attr} + return concat(result) +end + +Inlines.Quoted = function(el) + if el.quotetype == "DoubleQuote" then + return concat{'"', inlines(el.content), '"'} + else + return concat{"'", inlines(el.content), "'"} + end +end + +Inlines.Note = function(el) + footnotes[#footnotes + 1] = el.content + local num = #footnotes + return literal(format("[^%d]", num)) +end + +function Writer (doc, opts) + local d = blocks(doc.blocks, blankline) + local notes = {} + for i=1,#footnotes do + local note = hang(blocks(footnotes[i], blankline), 4, concat{format("[^%d]:",i),space}) + table.insert(notes, note) + end + return layout.render(concat{d, blankline, concat(notes, blankline)}, opts.columns) +end diff --git a/djot.lua b/djot.lua new file mode 100644 index 0000000..e2d7c5b --- /dev/null +++ b/djot.lua @@ -0,0 +1,158 @@ +--- @module djot +--- Parse and render djot light markup format. See https://djot.net. +--- +--- @usage +--- local djot = require("djot") +--- local input = "This is *djot*" +--- local doc = djot.parse(input) +--- -- render as HTML: +--- print(djot.render_html(doc)) +--- +--- -- render as AST: +--- print(djot.render_ast_pretty(doc)) +--- +--- -- or in JSON: +--- print(djot.render_ast_json(doc)) +--- +--- -- alter the AST with a filter: +--- local src = "return { str = function(e) e.text = e.text:upper() end }" +--- -- subordinate modules like filter can be accessed as fields +--- -- and are lazily loaded. +--- local filter = djot.filter.load_filter(src) +--- djot.filter.apply_filter(doc, filter) +--- +--- -- streaming parser: +--- for startpos, endpos, annotation in djot.parse_events("*hello there*") do +--- print(startpos, endpos, annotation) +--- end + +local unpack = unpack or table.unpack +local Parser = require("djot.block").Parser +local ast = require("djot.ast") +local html = require("djot.html") +local json = require("djot.json") +local filter = require("djot.filter") + +local StringHandle = {} + +function StringHandle:new() + local buffer = {} + setmetatable(buffer, StringHandle) + StringHandle.__index = StringHandle + return buffer +end + +function StringHandle:write(s) + self[#self + 1] = s +end + +function StringHandle:flush() + return table.concat(self) +end + +--- Parse a djot text and construct an abstract syntax tree (AST) +--- representing the document. +--- @param input input string +--- @param sourcepos if true, source positions are included in the AST +--- @param warn function that processes a warning, accepting a warning +--- object with `pos` and `message` fields. +--- @return AST +local function parse(input, sourcepos, warn) + local parser = Parser:new(input, warn) + return ast.to_ast(parser, sourcepos) +end + +--- Parses a djot text and returns an iterator over events, consisting +--- of a start position (bytes), and an position (bytes), and an +--- annotation. +--- @param input input string +--- @param warn function that processes a warning, accepting a warning +--- object with `pos` and `message` fields. +--- @return an iterator over events. +--- +--- for startpos, endpos, annotation in djot.parse_events("hello *world") do +--- ... +--- end +local function parse_events(input, warn) + return Parser:new(input):events() +end + +--- Render a document's AST in human-readable form. +--- @param doc the AST +--- @return rendered AST (string) +local function render_ast_pretty(doc) + local handle = StringHandle:new() + ast.render(doc, handle) + return handle:flush() +end + +--- Render a document's AST in JSON. +--- @param doc the AST +--- @return rendered AST (JSON string) +local function render_ast_json(doc) + return json.encode(doc) .. "\n" +end + +--- Render a document as HTML. +--- @param doc the AST +--- @return rendered document (HTML string) +local function render_html(doc) + local handle = StringHandle:new() + local renderer = html.Renderer:new() + renderer:render(doc, handle) + return handle:flush() +end + +--- Render an event as a JSON array. +--- @param startpos starting byte position +--- @param endpos ending byte position +--- @param annotation annotation of event +--- @return rendered event (JSON string) +local function render_event(startpos, endpos, annotation) + return string.format("[%q,%d,%d]", annotation, startpos, endpos) +end + +--- Parse a document and render as a JSON array of events. +--- @param input the djot document (string) +--- @param warn function that emits warnings, taking as argumnet +--- an object with fields 'message' and 'pos' +--- @return rendered events (JSON string) +local function parse_and_render_events(input, warn) + local handle = StringHandle:new() + local idx = 0 + for startpos, endpos, annotation in parse_events(input, warn) do + idx = idx + 1 + if idx == 1 then + handle:write("[") + else + handle:write(",") + end + handle:write(render_event(startpos, endpos, annotation) .. "\n") + end + handle:write("]\n") + return handle:flush() +end + +--- djot version (string) +local version = "0.2.0" + +--- @export +local G = { + parse = parse, + parse_events = parse_events, + parse_and_render_events = parse_and_render_events, + render_html = render_html, + render_ast_pretty = render_ast_pretty, + render_ast_json = render_ast_json, + render_event = render_event, + version = version +} + +-- Lazily load submodules, e.g. djot.filter +setmetatable(G,{ __index = function(t,name) + local mod = require("djot." .. name) + rawset(t,name,mod) + return t[name] + end }) + +return G diff --git a/djot/ast.lua b/djot/ast.lua new file mode 100644 index 0000000..b6bbcc2 --- /dev/null +++ b/djot/ast.lua @@ -0,0 +1,1012 @@ +--- @module djot.ast +--- Construct an AST for a djot document. + +if not utf8 then -- if not lua 5.3 or higher... + -- this is needed for the __pairs metamethod, used below + -- The following code is derived from the compat53 rock: + -- override pairs + local oldpairs = pairs + pairs = function(t) + local mt = getmetatable(t) + if type(mt) == "table" and type(mt.__pairs) == "function" then + return mt.__pairs(t) + else + return oldpairs(t) + end + end +end +local unpack = unpack or table.unpack + +local find, lower, sub, rep, format = + string.find, string.lower, string.sub, string.rep, string.format + +-- Creates a sparse array whose indices are byte positions. +-- sourcepos_map[bytepos] = "line:column:charpos" +local function make_sourcepos_map(input) + local sourcepos_map = {line = {}, col = {}, charpos = {}} + local line = 1 + local col = 0 + local charpos = 0 + local bytepos = 1 + + local byte = string.byte(input, bytepos) + while byte do + col = col + 1 + charpos = charpos + 1 + -- get next code point: + local newbytepos + if byte < 0xC0 then + newbytepos = bytepos + 1 + elseif byte < 0xE0 then + newbytepos = bytepos + 2 + elseif byte < 0xF0 then + newbytepos = bytepos + 3 + else + newbytepos = bytepos + 4 + end + while bytepos < newbytepos do + sourcepos_map.line[bytepos] = line + sourcepos_map.col[bytepos] = col + sourcepos_map.charpos[bytepos] = charpos + bytepos = bytepos + 1 + end + if byte == 10 then -- newline + line = line + 1 + col = 0 + end + byte = string.byte(input, bytepos) + end + + sourcepos_map.line[bytepos] = line + 1 + sourcepos_map.col[bytepos] = 1 + sourcepos_map.charpos[bytepos] = charpos + 1 + + return sourcepos_map +end + +local function add_string_content(node, buffer) + if node.s then + buffer[#buffer + 1] = node.s + elseif node.t == "softbreak" then + buffer[#buffer + 1] = "\n" + elseif node.c then + for i=1, #node.c do + add_string_content(node.c[i], buffer) + end + end +end + +local function get_string_content(node) + local buffer = {}; + add_string_content(node, buffer) + return table.concat(buffer) +end + +local roman_digits = { + i = 1, + v = 5, + x = 10, + l = 50, + c = 100, + d = 500, + m = 1000 } + +local function roman_to_number(s) + -- go backwards through the digits + local total = 0 + local prevdigit = 0 + local i=#s + while i > 0 do + local c = lower(sub(s,i,i)) + local n = roman_digits[c] + assert(n ~= nil, "Encountered bad character in roman numeral " .. s) + if n < prevdigit then -- e.g. ix + total = total - n + else + total = total + n + end + prevdigit = n + i = i - 1 + end + return total +end + +local function get_list_start(marker, style) + local numtype = string.gsub(style, "%p", "") + local s = string.gsub(marker, "%p", "") + if numtype == "1" then + return tonumber(s) + elseif numtype == "A" then + return (string.byte(s) - string.byte("A") + 1) + elseif numtype == "a" then + return (string.byte(s) - string.byte("a") + 1) + elseif numtype == "I" then + return roman_to_number(s) + elseif numtype == "i" then + return roman_to_number(s) + elseif numtype == "" then + return nil + end +end + +local ignorable = { + image_marker = true, + escape = true, + blankline = true +} + +local function sortedpairs(compare_function, to_displaykey) + return function(tbl) + local keys = {} + local k = nil + k = next(tbl, k) + while k do + keys[#keys + 1] = k + k = next(tbl, k) + end + table.sort(keys, compare_function) + local keyindex = 0 + local function ordered_next(tabl,_) + keyindex = keyindex + 1 + local key = keys[keyindex] + -- use canonical names + local displaykey = to_displaykey(key) + if key then + return displaykey, tabl[key] + else + return nil + end + end + -- Return an iterator function, the table, starting point + return ordered_next, tbl, nil + end +end + +-- provide children, tag, and text as aliases of c, t, s, +-- which we use above for better performance: +local mt = {} +local special = { + children = 'c', + text = 's', + tag = 't' } +local displaykeys = { + c = 'children', + s = 'text', + t = 'tag' } +mt.__index = function(table, key) + local k = special[key] + if k then + return rawget(table, k) + else + return rawget(table, key) + end +end +mt.__newindex = function(table, key, val) + local k = special[key] + if k then + rawset(table, k, val) + else + rawset(table, key, val) + end +end +mt.__pairs = sortedpairs(function(a,b) + if a == "t" then -- t is always first + return true + elseif a == "s" then -- s is always second + return (b ~= "t") + elseif a == "c" then -- c only before references, footnotes + return (b == "references" or b == "footnotes") + elseif a == "references" then + return (b == "footnotes") + elseif a == "footnotes" then + return false + elseif b == "t" or b == "s" then + return false + elseif b == "c" or b == "references" or b == "footnotes" then + return true + else + return (a < b) + end + end, function(k) return displaykeys[k] or k end) + + +--- Create a new AST node. +--- @param tag (string) tag for the node +--- @return node (table) +local function new_node(tag) + local node = { t = tag, c = nil } + setmetatable(node, mt) + return node +end + +--- Add `child` as a child of `node`. +--- @param node parent node +--- @param child child node +local function add_child(node, child) + if (not node.c) then + node.c = {child} + else + node.c[#node.c + 1] = child + end +end + +--- Returns true if `node` has children. +--- @param node node to check +--- @return true if node has children +local function has_children(node) + return (node.c and #node.c > 0) +end + +--- Returns an attributes object. +--- @param tbl table of attributes and values +--- @return attributes object (table including special metatable for +--- deterministic order of iteration) +local function new_attributes(tbl) + local attr = tbl or {} + -- ensure deterministic order of iteration + setmetatable(attr, {__pairs = sortedpairs(function(a,b) return a < b end, + function(k) return k end)}) + return attr +end + +--- Insert an attribute into an attributes object. +--- @param attr attributes object +--- @param key (string) key of new attribute +--- @param val (string) value of new attribute +local function insert_attribute(attr, key, val) + val = val:gsub("%s+", " ") -- normalize spaces + if key == "class" then + if attr.class then + attr.class = attr.class .. " " .. val + else + attr.class = val + end + else + attr[key] = val + end +end + +--- Copy attributes from `source` to `target`. +--- @param target attributes object +--- @param source table associating keys and values +local function copy_attributes(target, source) + if source then + for k,v in pairs(source) do + insert_attribute(target, k, v) + end + end +end + +local function insert_attributes_from_nodes(targetnode, cs) + targetnode.attr = targetnode.attr or new_attributes() + local i=1 + while i <= #cs do + local x, y = cs[i].t, cs[i].s + if x == "id" or x == "class" then + insert_attribute(targetnode.attr, x, y) + elseif x == "key" then + local val = {} + while cs[i + 1] and cs[i + 1].t == "value" do + val[#val + 1] = cs[i + 1].s:gsub("\\(%p)", "%1") + -- resolve backslash escapes + i = i + 1 + end + insert_attribute(targetnode.attr, y, table.concat(val,"\n")) + end + i = i + 1 + end +end + +local function make_definition_list_item(node) + node.t = "definition_list_item" + if not has_children(node) then + node.c = {} + end + if node.c[1] and node.c[1].t == "para" then + node.c[1].t = "term" + else + table.insert(node.c, 1, new_node("term")) + end + if node.c[2] then + local defn = new_node("definition") + defn.c = {} + for i=2,#node.c do + defn.c[#defn.c + 1] = node.c[i] + node.c[i] = nil + end + node.c[2] = defn + end +end + +local function resolve_style(list) + local style = nil + for k,i in pairs(list.styles) do + if not style or i < style.priority then + style = {name = k, priority = i} + end + end + list.style = style.name + list.styles = nil + list.start = get_list_start(list.startmarker, list.style) + list.startmarker = nil +end + +local function get_verbatim_content(node) + local s = get_string_content(node) + -- trim space next to ` at beginning or end + if find(s, "^ +`") then + s = s:sub(2) + end + if find(s, "` +$") then + s = s:sub(1, #s - 1) + end + return s +end + +local function add_sections(ast) + if not has_children(ast) then + return ast + end + local newast = new_node("doc") + local secs = { {sec = newast, level = 0 } } + for _,node in ipairs(ast.c) do + if node.t == "heading" then + local level = node.level + local curlevel = (#secs > 0 and secs[#secs].level) or 0 + if curlevel >= level then + while secs[#secs].level >= level do + local sec = table.remove(secs).sec + add_child(secs[#secs].sec, sec) + end + end + -- now we know: curlevel < level + local newsec = new_node("section") + newsec.attr = new_attributes{id = node.attr.id} + node.attr.id = nil + add_child(newsec, node) + secs[#secs + 1] = {sec = newsec, level = level} + else + add_child(secs[#secs].sec, node) + end + end + while #secs > 1 do + local sec = table.remove(secs).sec + add_child(secs[#secs].sec, sec) + end + assert(secs[1].sec == newast) + return newast +end + + +--- Create an abstract syntax tree based on an event +--- stream and references. +--- @param parser djot streaming parser +--- @param sourcepos if true, include source positions +--- @return table representing the AST +local function to_ast(parser, sourcepos) + local subject = parser.subject + local warn = parser.warn + if not warn then + warn = function() end + end + local sourceposmap + if sourcepos then + sourceposmap = make_sourcepos_map(subject) + end + local references = {} + local footnotes = {} + local identifiers = {} -- identifiers used (to ensure uniqueness) + + -- generate auto identifier for heading + local function get_identifier(s) + local base = s:gsub("[][~!@#$%^&*(){}`,.<>\\|=+/?]","") + :gsub("^%s+",""):gsub("%s+$","") + :gsub("%s+","-") + local i = 0 + local ident = base + -- generate unique id + while ident == "" or identifiers[ident] do + i = i + 1 + if base == "" then + base = "s" + end + ident = base .. "-" .. tostring(i) + end + identifiers[ident] = true + return ident + end + + local function format_sourcepos(bytepos) + if bytepos then + return string.format("%d:%d:%d", sourceposmap.line[bytepos], + sourceposmap.col[bytepos], sourceposmap.charpos[bytepos]) + end + end + + local function set_startpos(node, pos) + if sourceposmap then + local sp = format_sourcepos(pos) + if node.pos then + node.pos[1] = sp + else + node.pos = {sp, nil} + end + end + end + + local function set_endpos(node, pos) + if sourceposmap and node.pos then + local ep = format_sourcepos(pos) + if node.pos then + node.pos[2] = ep + else + node.pos = {nil, ep} + end + end + end + + local blocktag = { + heading = true, + div = true, + list = true, + list_item = true, + code_block = true, + para = true, + blockquote = true, + table = true, + thematic_break = true, + raw_block = true, + reference_definition = true, + footnote = true + } + + local block_attributes = nil + local function add_block_attributes(node) + if block_attributes and blocktag[node.t:gsub("%|.*","")] then + for i=1,#block_attributes do + insert_attributes_from_nodes(node, block_attributes[i]) + end + -- add to identifiers table so we don't get duplicate auto-generated ids + if node.attr and node.attr.id then + identifiers[node.attr.id] = true + end + block_attributes = nil + end + end + + -- two variables used for tight/loose list determination: + local tags = {} -- used to keep track of blank lines + local matchidx = 0 -- keep track of the index of the match + + local function is_tight(startidx, endidx, is_last_item) + -- see if there are any blank lines between blocks in a list item. + local blanklines = 0 + -- we don't care about blank lines at very end of list + if is_last_item then + while tags[endidx] == "blankline" or tags[endidx] == "-list_item" do + endidx = endidx - 1 + end + end + for i=startidx, endidx do + local tag = tags[i] + if tag == "blankline" then + if not ((string.find(tags[i+1], "%+list_item") or + (string.find(tags[i+1], "%-list_item") and + (is_last_item or + string.find(tags[i+2], "%-list_item"))))) then + -- don't count blank lines before list starts + -- don't count blank lines at end of nested lists or end of last item + blanklines = blanklines + 1 + end + end + end + return (blanklines == 0) + end + + local function add_child_to_tip(containers, child) + if containers[#containers].t == "list" and + not (child.t == "list_item" or child.t == "definition_list_item") then + -- close list + local oldlist = table.remove(containers) + add_child_to_tip(containers, oldlist) + end + if child.t == "list" then + if child.pos then + child.pos[2] = child.c[#child.c].pos[2] + end + -- calculate tightness (TODO not quite right) + local tight = true + for i=1,#child.c do + tight = tight and is_tight(child.c[i].startidx, + child.c[i].endidx, i == #child.c) + child.c[i].startidx = nil + child.c[i].endidx = nil + end + child.tight = tight + + -- resolve style if still ambiguous + resolve_style(child) + end + add_child(containers[#containers], child) + end + + + -- process a match: + -- containers is the stack of containers, with #container + -- being the one that would receive a new node + local function handle_match(containers, startpos, endpos, annot) + matchidx = matchidx + 1 + local mod, tag = string.match(annot, "^([-+]?)(.+)") + tags[matchidx] = annot + if ignorable[tag] then + return + end + if mod == "+" then + -- process open match: + -- * open a new node and put it at end of containers stack + -- * depending on the tag name, do other things + local node = new_node(tag) + set_startpos(node, startpos) + + -- add block attributes if any have accumulated: + add_block_attributes(node) + + if tag == "heading" then + node.level = (endpos - startpos) + 1 + + elseif find(tag, "^list_item") then + node.t = "list_item" + node.startidx = matchidx -- for tight/loose determination + local _, _, style_marker = string.find(tag, "(%|.*)") + local styles = {} + if style_marker then + local i=1 + for sty in string.gmatch(style_marker, "%|([^%|%]]*)") do + styles[sty] = i + i = i + 1 + end + end + node.style_marker = style_marker + + local marker = string.match(subject, "^%S+", startpos) + + -- adjust container stack so that the tip can accept this + -- kind of list item, adding a list if needed and possibly + -- closing an existing list + + local tip = containers[#containers] + if tip.t ~= "list" then + -- container is not a list ; add one + local list = new_node("list") + set_startpos(list, startpos) + list.styles = styles + list.attr = node.attr + list.startmarker = marker + node.attr = nil + containers[#containers + 1] = list + else + -- it's a list, but is it the right kind? + local matched_styles = {} + local has_match = false + for k,_ in pairs(styles) do + if tip.styles[k] then + has_match = true + matched_styles[k] = styles[k] + end + end + if has_match then + -- yes, list can accept this item + tip.styles = matched_styles + else + -- no, list can't accept this item ; close it + local oldlist = table.remove(containers) + add_child_to_tip(containers, oldlist) + -- add a new sibling list node with the right style + local list = new_node("list") + set_startpos(list, startpos) + list.styles = styles + list.attr = node.attr + list.startmarker = marker + node.attr = nil + containers[#containers + 1] = list + end + end + + + end + + -- add to container stack + containers[#containers + 1] = node + + elseif mod == "-" then + -- process close match: + -- * check end of containers stack; if tag matches, add + -- end position, pop the item off the stack, and add + -- it as a child of the next container on the stack + -- * if it doesn't match, issue a warning and ignore this tag + + if containers[#containers].t == "list" then + local listnode = table.remove(containers) + add_child_to_tip(containers, listnode) + end + + if tag == containers[#containers].t then + local node = table.remove(containers) + set_endpos(node, endpos) + + if node.t == "block_attributes" then + if not block_attributes then + block_attributes = {} + end + block_attributes[#block_attributes + 1] = node.c + return -- we don't add this to parent; instead we store + -- the block attributes and add them to the next block + + elseif node.t == "attributes" then + -- parse attributes, add to last node + local tip = containers[#containers] + local prevnode = has_children(tip) and tip.c[#tip.c] + if prevnode then + local endswithspace = false + if prevnode.t == "str" then + -- split off last consecutive word of string + -- to which to attach attributes + local lastwordpos = string.find(prevnode.s, "[^%s]+$") + if not lastwordpos then + endswithspace = true + elseif lastwordpos > 1 then + local newnode = new_node("str") + newnode.s = sub(prevnode.s, lastwordpos, -1) + prevnode.s = sub(prevnode.s, 1, lastwordpos - 1) + add_child_to_tip(containers, newnode) + prevnode = newnode + end + end + if has_children(node) and not endswithspace then + insert_attributes_from_nodes(prevnode, node.c) + else + warn({message = "Ignoring unattached attribute", pos = startpos}) + end + else + warn({message = "Ignoring unattached attribute", pos = startpos}) + end + return -- don't add the attribute node to the tree + + elseif tag == "reference_definition" then + local dest = "" + local key + for i=1,#node.c do + if node.c[i].t == "reference_key" then + key = node.c[i].s + end + if node.c[i].t == "reference_value" then + dest = dest .. node.c[i].s + end + end + references[key] = new_node("reference") + references[key].destination = dest + if node.attr then + references[key].attr = node.attr + end + return -- don't include in tree + + elseif tag == "footnote" then + local label + if has_children(node) and node.c[1].t == "note_label" then + label = node.c[1].s + table.remove(node.c, 1) + end + if label then + footnotes[label] = node + end + return -- don't include in tree + + + elseif tag == "table" then + + -- Children are the rows. Look for a separator line: + -- if found, make the preceding rows headings + -- and set attributes for column alignments on the table. + + local i=1 + local aligns = {} + while i <= #node.c do + local found, align, _ + if node.c[i].t == "row" then + local row = node.c[i].c + for j=1,#row do + found, _, align = find(row[j].t, "^separator_(.*)") + if not found then + break + end + aligns[j] = align + end + if found and #aligns > 0 then + -- set previous row to head and adjust aligns + local prevrow = node.c[i - 1] + if prevrow and prevrow.t == "row" then + prevrow.head = true + for k=1,#prevrow.c do + -- set head on cells too + prevrow.c[k].head = true + if aligns[k] ~= "default" then + prevrow.c[k].align = aligns[k] + end + end + end + table.remove(node.c, i) -- remove sep line + -- we don't need to increment i because we removed ith elt + else + if #aligns > 0 then + for l=1,#node.c[i].c do + if aligns[l] ~= "default" then + node.c[i].c[l].align = aligns[l] + end + end + end + i = i + 1 + end + end + end + + elseif tag == "code_block" then + if has_children(node) then + if node.c[1].t == "code_language" then + node.lang = node.c[1].s + table.remove(node.c, 1) + elseif node.c[1].t == "raw_format" then + local fmt = node.c[1].s:sub(2) + table.remove(node.c, 1) + node.t = "raw_block" + node.format = fmt + end + end + node.s = get_string_content(node) + node.c = nil + + elseif find(tag, "^list_item") then + node.t = "list_item" + node.endidx = matchidx -- for tight/loose determination + + if node.style_marker == "|:" then + make_definition_list_item(node) + end + + if node.style_marker == "|X" and has_children(node) then + if node.c[1].t == "checkbox_checked" then + node.checkbox = "checked" + table.remove(node.c, 1) + elseif node.c[1].t == "checkbox_unchecked" then + node.checkbox = "unchecked" + table.remove(node.c, 1) + end + end + + node.style_marker = nil + + elseif tag == "inline_math" then + node.t = "math" + node.s = get_verbatim_content(node) + node.c = nil + node.display = false + node.attr = new_attributes{class = "math inline"} + + elseif tag == "display_math" then + node.t = "math" + node.s = get_verbatim_content(node) + node.c = nil + node.display = true + node.attr = new_attributes{class = "math display"} + + elseif tag == "imagetext" then + node.t = "image" + + elseif tag == "linktext" then + node.t = "link" + + elseif tag == "div" then + node.c = node.c or {} + if node.c[1] and node.c[1].t == "class" then + node.attr = new_attributes(node.attr) + insert_attribute(node.attr, "class", get_string_content(node.c[1])) + table.remove(node.c, 1) + end + + elseif tag == "verbatim" then + node.s = get_verbatim_content(node) + node.c = nil + + elseif tag == "url" then + node.destination = get_string_content(node) + + elseif tag == "email" then + node.destination = "mailto:" .. get_string_content(node) + + elseif tag == "caption" then + local tip = containers[#containers] + local prevnode = has_children(tip) and tip.c[#tip.c] + if prevnode and prevnode.t == "table" then + -- move caption in table node + table.insert(prevnode.c, 1, node) + else + warn({ message = "Ignoring caption without preceding table", + pos = startpos }) + end + return + + elseif tag == "heading" then + local heading_str = + get_string_content(node):gsub("^%s+",""):gsub("%s+$","") + if not node.attr then + node.attr = new_attributes{} + end + if not node.attr.id then -- generate id attribute from heading + insert_attribute(node.attr, "id", get_identifier(heading_str)) + end + -- insert into references unless there's a same-named one already: + if not references[heading_str] then + references[heading_str] = + new_node("reference") + references[heading_str].destination = "#" .. node.attr.id + end + + elseif tag == "destination" then + local tip = containers[#containers] + local prevnode = has_children(tip) and tip.c[#tip.c] + assert(prevnode and (prevnode.t == "image" or prevnode.t == "link"), + "destination with no preceding link or image") + prevnode.destination = get_string_content(node):gsub("\r?\n", "") + return -- do not put on container stack + + elseif tag == "reference" then + local tip = containers[#containers] + local prevnode = has_children(tip) and tip.c[#tip.c] + assert(prevnode and (prevnode.t == "image" or prevnode.t == "link"), + "reference with no preceding link or image") + if has_children(node) then + prevnode.reference = get_string_content(node):gsub("\r?\n", " ") + else + prevnode.reference = get_string_content(prevnode):gsub("\r?\n", " ") + end + return -- do not put on container stack + end + + add_child_to_tip(containers, node) + else + assert(false, "unmatched " .. annot .. " encountered at byte " .. + startpos) + return + end + else + -- process leaf node: + -- * add position info + -- * special handling depending on tag type + -- * add node as child of container at end of containers stack + local node = new_node(tag) + add_block_attributes(node) + set_startpos(node, startpos) + set_endpos(node, endpos) + + -- special handling: + if tag == "softbreak" then + node.s = nil + elseif tag == "reference_key" then + node.s = sub(subject, startpos + 1, endpos - 1) + elseif tag == "footnote_reference" then + node.s = sub(subject, startpos + 2, endpos - 1) + elseif tag == "symbol" then + node.alias = sub(subject, startpos + 1, endpos - 1) + elseif tag == "raw_format" then + local tip = containers[#containers] + local prevnode = has_children(tip) and tip.c[#tip.c] + if prevnode and prevnode.t == "verbatim" then + local s = get_string_content(prevnode) + prevnode.t = "raw_inline" + prevnode.s = s + prevnode.c = nil + prevnode.format = sub(subject, startpos + 2, endpos - 1) + return -- don't add this node to containers + else + node.s = sub(subject, startpos, endpos) + end + else + node.s = sub(subject, startpos, endpos) + end + + add_child_to_tip(containers, node) + + end + end + + local doc = new_node("doc") + local containers = {doc} + for sp, ep, annot in parser:events() do + handle_match(containers, sp, ep, annot) + end + -- close any open containers + while #containers > 1 do + local node = table.remove(containers) + add_child_to_tip(containers, node) + -- note: doc container doesn't have pos, so we check: + if sourceposmap and containers[#containers].pos then + containers[#containers].pos[2] = node.pos[2] + end + end + doc = add_sections(doc) + + doc.references = references + doc.footnotes = footnotes + + return doc +end + +local function render_node(node, handle, indent) + indent = indent or 0 + handle:write(rep(" ", indent)) + if indent > 128 then + handle:write("(((DEEPLY NESTED CONTENT OMITTED)))\n") + return + end + + if node.t then + handle:write(node.t) + if node.pos then + handle:write(format(" (%s-%s)", node.pos[1], node.pos[2])) + end + for k,v in pairs(node) do + if type(k) == "string" and k ~= "children" and + k ~= "tag" and k ~= "pos" and k ~= "attr" and + k ~= "references" and k ~= "footnotes" then + handle:write(format(" %s=%q", k, tostring(v))) + end + end + if node.attr then + for k,v in pairs(node.attr) do + handle:write(format(" %s=%q", k, v)) + end + end + else + io.stderr:write("Encountered node without tag:\n" .. + require'inspect'(node)) + os.exit(1) + end + handle:write("\n") + if node.c then + for _,v in ipairs(node.c) do + render_node(v, handle, indent + 2) + end + end +end + +--- Render an AST in human-readable form, with indentation +--- showing the hierarchy. +--- @param doc (table) djot AST +--- @param handle handle to which to write content +--- @return result of flushing handle +local function render(doc, handle) + render_node(doc, handle, 0) + if next(doc.references) ~= nil then + handle:write("references\n") + for k,v in pairs(doc.references) do + handle:write(format(" [%q] =\n", k)) + render_node(v, handle, 4) + end + end + if next(doc.footnotes) ~= nil then + handle:write("footnotes\n") + for k,v in pairs(doc.footnotes) do + handle:write(format(" [%q] =\n", k)) + render_node(v, handle, 4) + end + end +end + +--- @export +return { to_ast = to_ast, + render = render, + insert_attribute = insert_attribute, + copy_attributes = copy_attributes, + new_attributes = new_attributes, + new_node = new_node, + add_child = add_child, + has_children = has_children } diff --git a/djot/attributes.lua b/djot/attributes.lua new file mode 100644 index 0000000..259cd65 --- /dev/null +++ b/djot/attributes.lua @@ -0,0 +1,270 @@ +local find, sub = string.find, string.sub + +-- Parser for attributes +-- attributes { id = "foo", class = "bar baz", +-- key1 = "val1", key2 = "val2" } +-- syntax: +-- +-- attributes <- '{' whitespace* attribute (whitespace attribute)* whitespace* '}' +-- attribute <- identifier | class | keyval +-- identifier <- '#' name +-- class <- '.' name +-- name <- (nonspace, nonpunctuation other than ':', '_', '-')+ +-- keyval <- key '=' val +-- key <- (ASCII_ALPHANUM | ':' | '_' | '-')+ +-- val <- bareval | quotedval +-- bareval <- (ASCII_ALPHANUM | ':' | '_' | '-')+ +-- quotedval <- '"' ([^"] | '\"') '"' + +-- states: +local SCANNING = 0 +local SCANNING_ID = 1 +local SCANNING_CLASS= 2 +local SCANNING_KEY = 3 +local SCANNING_VALUE = 4 +local SCANNING_BARE_VALUE = 5 +local SCANNING_QUOTED_VALUE = 6 +local SCANNING_QUOTED_VALUE_CONTINUATION = 7 +local SCANNING_ESCAPED = 8 +local SCANNING_ESCAPED_IN_CONTINUATION = 9 +local SCANNING_COMMENT = 10 +local FAIL = 11 +local DONE = 12 +local START = 13 + +local AttributeParser = {} + +local handlers = {} + +handlers[START] = function(self, pos) + if find(self.subject, "^{", pos) then + return SCANNING + else + return FAIL + end +end + +handlers[FAIL] = function(_self, _pos) + return FAIL +end + +handlers[DONE] = function(_self, _pos) + return DONE +end + +handlers[SCANNING] = function(self, pos) + local c = sub(self.subject, pos, pos) + if c == ' ' or c == '\t' or c == '\n' or c == '\r' then + return SCANNING + elseif c == '}' then + return DONE + elseif c == '#' then + self.begin = pos + return SCANNING_ID + elseif c == '%' then + self.begin = pos + return SCANNING_COMMENT + elseif c == '.' then + self.begin = pos + return SCANNING_CLASS + elseif find(c, "^[%a%d_:-]") then + self.begin = pos + return SCANNING_KEY + else -- TODO + return FAIL + end +end + +handlers[SCANNING_COMMENT] = function(self, pos) + if sub(self.subject, pos, pos) == "%" then + return SCANNING + else + return SCANNING_COMMENT + end +end + +handlers[SCANNING_ID] = function(self, pos) + local c = sub(self.subject, pos, pos) + if find(c, "^[^%s%p]") or c == "_" or c == "-" or c == ":" then + return SCANNING_ID + elseif c == '}' then + if self.lastpos > self.begin then + self:add_match(self.begin + 1, self.lastpos, "id") + end + self.begin = nil + return DONE + elseif find(c, "^%s") then + if self.lastpos > self.begin then + self:add_match(self.begin + 1, self.lastpos, "id") + end + self.begin = nil + return SCANNING + else + return FAIL + end +end + +handlers[SCANNING_CLASS] = function(self, pos) + local c = sub(self.subject, pos, pos) + if find(c, "^[^%s%p]") or c == "_" or c == "-" or c == ":" then + return SCANNING_CLASS + elseif c == '}' then + if self.lastpos > self.begin then + self:add_match(self.begin + 1, self.lastpos, "class") + end + self.begin = nil + return DONE + elseif find(c, "^%s") then + if self.lastpos > self.begin then + self:add_match(self.begin + 1, self.lastpos, "class") + end + self.begin = nil + return SCANNING + else + return FAIL + end +end + +handlers[SCANNING_KEY] = function(self, pos) + local c = sub(self.subject, pos, pos) + if c == "=" then + self:add_match(self.begin, self.lastpos, "key") + self.begin = nil + return SCANNING_VALUE + elseif find(c, "^[%a%d_:-]") then + return SCANNING_KEY + else + return FAIL + end +end + +handlers[SCANNING_VALUE] = function(self, pos) + local c = sub(self.subject, pos, pos) + if c == '"' then + self.begin = pos + return SCANNING_QUOTED_VALUE + elseif find(c, "^[%a%d_:-]") then + self.begin = pos + return SCANNING_BARE_VALUE + else + return FAIL + end +end + +handlers[SCANNING_BARE_VALUE] = function(self, pos) + local c = sub(self.subject, pos, pos) + if find(c, "^[%a%d_:-]") then + return SCANNING_BARE_VALUE + elseif c == '}' then + self:add_match(self.begin, self.lastpos, "value") + self.begin = nil + return DONE + elseif find(c, "^%s") then + self:add_match(self.begin, self.lastpos, "value") + self.begin = nil + return SCANNING + else + return FAIL + end +end + +handlers[SCANNING_ESCAPED] = function(_self, _pos) + return SCANNING_QUOTED_VALUE +end + +handlers[SCANNING_ESCAPED_IN_CONTINUATION] = function(_self, _pos) + return SCANNING_QUOTED_VALUE_CONTINUATION +end + +handlers[SCANNING_QUOTED_VALUE] = function(self, pos) + local c = sub(self.subject, pos, pos) + if c == '"' then + self:add_match(self.begin + 1, self.lastpos, "value") + self.begin = nil + return SCANNING + elseif c == "\n" then + self:add_match(self.begin + 1, self.lastpos, "value") + self.begin = nil + return SCANNING_QUOTED_VALUE_CONTINUATION + elseif c == "\\" then + return SCANNING_ESCAPED + else + return SCANNING_QUOTED_VALUE + end +end + +handlers[SCANNING_QUOTED_VALUE_CONTINUATION] = function(self, pos) + local c = sub(self.subject, pos, pos) + if self.begin == nil then + self.begin = pos + end + if c == '"' then + self:add_match(self.begin, self.lastpos, "value") + self.begin = nil + return SCANNING + elseif c == "\n" then + self:add_match(self.begin, self.lastpos, "value") + self.begin = nil + return SCANNING_QUOTED_VALUE_CONTINUATION + elseif c == "\\" then + return SCANNING_ESCAPED_IN_CONTINUATION + else + return SCANNING_QUOTED_VALUE_CONTINUATION + end +end + +function AttributeParser:new(subject) + local state = { + subject = subject, + state = START, + begin = nil, + lastpos = nil, + matches = {} + } + setmetatable(state, self) + self.__index = self + return state +end + +function AttributeParser:add_match(sp, ep, tag) + self.matches[#self.matches + 1] = {sp, ep, tag} +end + +function AttributeParser:get_matches() + return self.matches +end + +-- Feed parser a slice of text from the subject, between +-- startpos and endpos inclusive. Return status, position, +-- where status is either "done" (position should point to +-- final '}'), "fail" (position should point to first character +-- that could not be parsed), or "continue" (position should +-- point to last character parsed). +function AttributeParser:feed(startpos, endpos) + local pos = startpos + while pos <= endpos do + self.state = handlers[self.state](self, pos) + if self.state == DONE then + return "done", pos + elseif self.state == FAIL then + self.lastpos = pos + return "fail", pos + else + self.lastpos = pos + pos = pos + 1 + end + end + return "continue", endpos +end + +--[[ +local test = function() + local parser = AttributeParser:new("{a=b #ident\n.class\nkey=val1\n .class key2=\"val two \\\" ok\" x") + local x,y,z = parser:feed(1,56) + print(require'inspect'(parser:get_matches{})) +end + +test() +--]] + +return { AttributeParser = AttributeParser } diff --git a/djot/block.lua b/djot/block.lua new file mode 100644 index 0000000..eaa7a6a --- /dev/null +++ b/djot/block.lua @@ -0,0 +1,823 @@ +local InlineParser = require("djot.inline").InlineParser +local attributes = require("djot.attributes") +local unpack = unpack or table.unpack +local find, sub, byte = string.find, string.sub, string.byte + +local Container = {} + +function Container:new(spec, data) + self = spec + local contents = {} + setmetatable(contents, self) + self.__index = self + if data then + for k,v in pairs(data) do + contents[k] = v + end + end + return contents +end + +local function get_list_styles(marker) + if marker == "+" or marker == "-" or marker == "*" or marker == ":" then + return {marker} + elseif find(marker, "^[+*-] %[[Xx ]%]") then + return {"X"} -- task list + elseif find(marker, "^[(]?%d+[).]") then + return {(marker:gsub("%d+","1"))} + -- in ambiguous cases we return two values + elseif find(marker, "^[(]?[ivxlcdm][).]") then + return {(marker:gsub("%a+", "a")), (marker:gsub("%a+", "i"))} + elseif find(marker, "^[(]?[IVXLCDM][).]") then + return {(marker:gsub("%a+", "A")), (marker:gsub("%a+", "I"))} + elseif find(marker, "^[(]?%l[).]") then + return {(marker:gsub("%l", "a"))} + elseif find(marker, "^[(]?%u[).]") then + return {(marker:gsub("%u", "A"))} + elseif find(marker, "^[(]?[ivxlcdm]+[).]") then + return {(marker:gsub("%a+", "i"))} + elseif find(marker, "^[(]?[IVXLCDM]+[).]") then + return {(marker:gsub("%a+", "I"))} + else -- doesn't match any list style + return {} + end +end + +local Parser = {} + +function Parser:new(subject, warn) + -- ensure the subject ends with a newline character + if not subject:find("[\r\n]$") then + subject = subject .. "\n" + end + local state = { + warn = warn or function() end, + subject = subject, + indent = 0, + startline = nil, + starteol = nil, + endeol = nil, + matches = {}, + containers = {}, + pos = 1, + last_matched_container = 0, + timer = {}, + finished_line = false, + returned = 0 } + setmetatable(state, self) + self.__index = self + return state +end + +-- parameters are start and end position +function Parser:parse_table_row(sp, ep) + local orig_matches = #self.matches -- so we can rewind + local startpos = self.pos + self:add_match(sp, sp, "+row") + -- skip | and any initial space in the cell: + self.pos = find(self.subject, "%S", sp + 1) + -- check to see if we have a separator line + local seps = {} + local p = self.pos + local sepfound = false + while not sepfound do + local sepsp, sepep, left, right, trailing = + find(self.subject, "^(%:?)%-%-*(%:?)([ \t]*%|[ \t]*)", p) + if sepep then + local st = "separator_default" + if #left > 0 and #right > 0 then + st = "separator_center" + elseif #right > 0 then + st = "separator_right" + elseif #left > 0 then + st = "separator_left" + end + seps[#seps + 1] = {sepsp, sepep - #trailing, st} + p = sepep + 1 + if p == self.starteol then + sepfound = true + break + end + else + break + end + end + if sepfound then + for i=1,#seps do + self:add_match(unpack(seps[i])) + end + self:add_match(self.starteol - 1, self.starteol - 1, "-row") + self.pos = self.starteol + self.finished_line = true + return true + end + local inline_parser = InlineParser:new(self.subject, self.warn) + self:add_match(sp, sp, "+cell") + local complete_cell = false + while self.pos <= ep do + -- parse a chunk as inline content + local nextbar, _ + while not nextbar do + _, nextbar = self:find("^[^|\r\n]*|") + if not nextbar then + break + end + if string.find(self.subject, "^\\", nextbar - 1) then -- \| + inline_parser:feed(self.pos, nextbar) + self.pos = nextbar + 1 + nextbar = nil + else + inline_parser:feed(self.pos, nextbar - 1) + if inline_parser:in_verbatim() then + inline_parser:feed(nextbar, nextbar) + self.pos = nextbar + 1 + nextbar = nil + else + self.pos = nextbar + 1 + end + end + end + complete_cell = nextbar + if not complete_cell then + break + end + -- add a table cell + local cell_matches = inline_parser:get_matches() + for i=1,#cell_matches do + local s,e,ann = unpack(cell_matches[i]) + if i == #cell_matches and ann == "str" then + -- strip trailing space + while byte(self.subject, e) == 32 and e >= s do + e = e - 1 + end + end + self:add_match(s,e,ann) + end + self:add_match(nextbar, nextbar, "-cell") + if nextbar < ep then + -- reset inline parser state + inline_parser = InlineParser:new(self.subject, self.warn) + self:add_match(nextbar, nextbar, "+cell") + self.pos = find(self.subject, "%S", self.pos) + end + end + if not complete_cell then + -- rewind, this is not a valid table row + self.pos = startpos + for i = orig_matches,#self.matches do + self.matches[i] = nil + end + return false + else + self:add_match(self.pos, self.pos, "-row") + self.pos = self.starteol + self.finished_line = true + return true + end +end + +function Parser:specs() + return { + { name = "para", + is_para = true, + content = "inline", + continue = function() + if self:find("^%S") then + return true + else + return false + end + end, + open = function(spec) + self:add_container(Container:new(spec, + { inline_parser = + InlineParser:new(self.subject, self.warn) })) + self:add_match(self.pos, self.pos, "+para") + return true + end, + close = function() + self:get_inline_matches() + local last = self.matches[#self.matches] or {self.pos, self.pos, ""} + local sp, ep, annot = unpack(last) + self:add_match(ep + 1, ep + 1, "-para") + self.containers[#self.containers] = nil + end + }, + + { name = "caption", + is_para = false, + content = "inline", + continue = function() + return self:find("^%S") + end, + open = function(spec) + local _, ep = self:find("^%^[ \t]+") + if ep then + self.pos = ep + 1 + self:add_container(Container:new(spec, + { inline_parser = + InlineParser:new(self.subject, self.warn) })) + self:add_match(self.pos, self.pos, "+caption") + return true + end + end, + close = function() + self:get_inline_matches() + self:add_match(self.pos - 1, self.pos - 1, "-caption") + self.containers[#self.containers] = nil + end + }, + + { name = "blockquote", + content = "block", + continue = function() + if self:find("^%>%s") then + self.pos = self.pos + 1 + return true + else + return false + end + end, + open = function(spec) + if self:find("^%>%s") then + self:add_container(Container:new(spec)) + self:add_match(self.pos, self.pos, "+blockquote") + self.pos = self.pos + 1 + return true + end + end, + close = function() + self:add_match(self.pos, self.pos, "-blockquote") + self.containers[#self.containers] = nil + end + }, + + -- should go before reference definitions + { name = "footnote", + content = "block", + continue = function(container) + if self.indent > container.indent or self:find("^[\r\n]") then + return true + else + return false + end + end, + open = function(spec) + local sp, ep, label = self:find("^%[%^([^]]+)%]:%s") + if not sp then + return nil + end + -- adding container will close others + self:add_container(Container:new(spec, {note_label = label, + indent = self.indent})) + self:add_match(sp, sp, "+footnote") + self:add_match(sp + 2, ep - 3, "note_label") + self.pos = ep + return true + end, + close = function(_container) + self:add_match(self.pos, self.pos, "-footnote") + self.containers[#self.containers] = nil + end + }, + + -- should go before list_item_spec + { name = "thematic_break", + content = nil, + continue = function() + return false + end, + open = function(spec) + local sp, ep = self:find("^[-*][ \t]*[-*][ \t]*[-*][-* \t]*[\r\n]") + if ep then + self:add_container(Container:new(spec)) + self:add_match(sp, ep, "thematic_break") + self.pos = ep + return true + end + end, + close = function(_container) + self.containers[#self.containers] = nil + end + }, + + { name = "list_item", + content = "block", + continue = function(container) + if self.indent > container.indent or self:find("^[\r\n]") then + return true + else + return false + end + end, + open = function(spec) + local sp, ep = self:find("^[-*+:]%s") + if not sp then + sp, ep = self:find("^%d+[.)]%s") + end + if not sp then + sp, ep = self:find("^%(%d+%)%s") + end + if not sp then + sp, ep = self:find("^[ivxlcdmIVXLCDM]+[.)]%s") + end + if not sp then + sp, ep = self:find("^%([ivxlcdmIVXLCDM]+%)%s") + end + if not sp then + sp, ep = self:find("^%a[.)]%s") + end + if not sp then + sp, ep = self:find("^%(%a%)%s") + end + if not sp then + return nil + end + local marker = sub(self.subject, sp, ep - 1) + local checkbox = nil + if self:find("^[*+-] %[[Xx ]%]%s", sp + 1) then -- task list + marker = sub(self.subject, sp, sp + 4) + checkbox = sub(self.subject, sp + 3, sp + 3) + end + -- some items have ambiguous style + local styles = get_list_styles(marker) + if #styles == 0 then + return nil + end + local data = { styles = styles, + indent = self.indent } + -- adding container will close others + self:add_container(Container:new(spec, data)) + local annot = "+list_item" + for i=1,#styles do + annot = annot .. "|" .. styles[i] + end + self:add_match(sp, ep - 1, annot) + self.pos = ep + if checkbox then + if checkbox == " " then + self:add_match(sp + 2, sp + 4, "checkbox_unchecked") + else + self:add_match(sp + 2, sp + 4, "checkbox_checked") + end + self.pos = sp + 5 + end + return true + end, + close = function(_container) + self:add_match(self.pos, self.pos, "-list_item") + self.containers[#self.containers] = nil + end + }, + + { name = "reference_definition", + content = nil, + continue = function(container) + if container.indent >= self.indent then + return false + end + local _, ep, rest = self:find("^(%S+)") + if ep and self.starteol == ep + 1 then + self:add_match(ep - #rest + 1, ep, "reference_value") + self.pos = ep + 1 + return true + else + return false + end + end, + open = function(spec) + local sp, ep, label, rest = self:find("^%[([^]\r\n]*)%]:[ \t]*(%S*)") + if ep and self.starteol == ep + 1 then + self:add_container(Container:new(spec, + { key = label, + indent = self.indent })) + self:add_match(sp, sp, "+reference_definition") + self:add_match(sp, sp + #label + 1, "reference_key") + if #rest > 0 then + self:add_match(ep - #rest + 1, ep, "reference_value") + end + self.pos = ep + 1 + return true + end + end, + close = function(_container) + self:add_match(self.pos, self.pos, "-reference_definition") + self.containers[#self.containers] = nil + end + }, + + { name = "heading", + content = "inline", + continue = function(container) + local sp, ep = self:find("^%#+%s") + if sp and ep and container.level == ep - sp then + self.pos = ep + return true + else + return false + end + end, + open = function(spec) + local sp, ep = self:find("^#+") + if ep and find(self.subject, "^%s", ep + 1) then + local level = ep - sp + 1 + self:add_container(Container:new(spec, {level = level, + inline_parser = InlineParser:new(self.subject, self.warn) })) + self:add_match(sp, ep, "+heading") + self.pos = ep + 1 + return true + end + end, + close = function(_container) + self:get_inline_matches() + local last = self.matches[#self.matches] or {self.pos, self.pos, ""} + local sp, ep, annot = unpack(last) + self:add_match(ep + 1, ep + 1, "-heading") + self.containers[#self.containers] = nil + end + }, + + { name = "code_block", + content = "text", + continue = function(container) + local char = sub(container.border, 1, 1) + local sp, ep, border = self:find("^(" .. container.border .. + char .. "*)[ \t]*[\r\n]") + if ep then + container.end_fence_sp = sp + container.end_fence_ep = sp + #border - 1 + self.pos = ep -- before newline + self.finished_line = true + return false + else + return true + end + end, + open = function(spec) + local sp, ep, border, ws, lang = + self:find("^(~~~~*)([ \t]*)(%S*)[ \t]*[\r\n]") + if not ep then + sp, ep, border, ws, lang = + self:find("^(````*)([ \t]*)([^%s`]*)[ \t]*[\r\n]") + end + if border then + local is_raw = find(lang, "^=") and true or false + self:add_container(Container:new(spec, {border = border, + indent = self.indent })) + self:add_match(sp, sp + #border - 1, "+code_block") + if #lang > 0 then + local langstart = sp + #border + #ws + if is_raw then + self:add_match(langstart, langstart + #lang - 1, "raw_format") + else + self:add_match(langstart, langstart + #lang - 1, "code_language") + end + end + self.pos = ep -- before newline + self.finished_line = true + return true + end + end, + close = function(container) + local sp = container.end_fence_sp or self.pos + local ep = container.end_fence_ep or self.pos + self:add_match(sp, ep, "-code_block") + if sp == ep then + self.warn({ pos = self.pos, message = "Unclosed code block" }) + end + self.containers[#self.containers] = nil + end + }, + + { name = "fenced_div", + content = "block", + continue = function(container) + if self.containers[#self.containers].name == "code_block" then + return true -- see #109 + end + local sp, ep, equals = self:find("^(::::*)[ \t]*[\r\n]") + if ep and #equals >= container.equals then + container.end_fence_sp = sp + container.end_fence_ep = sp + #equals - 1 + self.pos = ep -- before newline + return false + else + return true + end + end, + open = function(spec) + local sp, ep1, equals = self:find("^(::::*)[ \t]*") + if not ep1 then + return false + end + local clsp, ep = find(self.subject, "^[%w_-]*", ep1 + 1) + local _, eol = find(self.subject, "^[ \t]*[\r\n]", ep + 1) + if eol then + self:add_container(Container:new(spec, {equals = #equals})) + self:add_match(sp, ep, "+div") + if ep >= clsp then + self:add_match(clsp, ep, "class") + end + self.pos = eol + 1 + self.finished_line = true + return true + end + end, + close = function(container) + local sp = container.end_fence_sp or self.pos + local ep = container.end_fence_ep or self.pos + -- check to make sure the match is in order + self:add_match(sp, ep, "-div") + if sp == ep then + self.warn({pos = self.pos, message = "Unclosed div"}) + end + self.containers[#self.containers] = nil + end + }, + + { name = "table", + content = "cells", + continue = function(_container) + local sp, ep = self:find("^|[^\r\n]*|") + local eolsp = ep and find(self.subject, "^[ \t]*[\r\n]", ep + 1); + if eolsp then + return self:parse_table_row(sp, ep) + end + end, + open = function(spec) + local sp, ep = self:find("^|[^\r\n]*|") + local eolsp = " *[\r\n]" -- make sure at end of line + if sp and eolsp then + self:add_container(Container:new(spec, { columns = 0 })) + self:add_match(sp, sp, "+table") + if self:parse_table_row(sp, ep) then + return true + else + self.containers[#self.containers] = nil + return false + end + end + end, + close = function(_container) + self:add_match(self.pos, self.pos, "-table") + self.containers[#self.containers] = nil + end + }, + + { name = "attributes", + content = "attributes", + open = function(spec) + if self:find("^%{") then + local attribute_parser = + attributes.AttributeParser:new(self.subject) + local status, ep = + attribute_parser:feed(self.pos, self.endeol) + if status == 'fail' or ep + 1 < self.endeol then + return false + else + self:add_container(Container:new(spec, + { status = status, + indent = self.indent, + startpos = self.pos, + slices = {}, + attribute_parser = attribute_parser })) + local container = self.containers[#self.containers] + container.slices = { {self.pos, self.endeol } } + self.pos = self.starteol + return true + end + + end + end, + continue = function(container) + if self.indent > container.indent then + table.insert(container.slices, { self.pos, self.endeol }) + local status, ep = + container.attribute_parser:feed(self.pos, self.endeol) + container.status = status + if status ~= 'fail' or ep + 1 < self.endeol then + self.pos = self.starteol + return true + end + end + -- if we get to here, we don't continue; either we + -- reached the end of indentation or we failed in + -- parsing attributes + if container.status == 'done' then + return false + else -- attribute parsing failed; convert to para and continue + -- with that + local para_spec = self:specs()[1] + local para = Container:new(para_spec, + { inline_parser = + InlineParser:new(self.subject, self.warn) }) + self:add_match(container.startpos, container.startpos, "+para") + self.containers[#self.containers] = para + -- reparse the text we couldn't parse as a block attribute: + para.inline_parser.attribute_slices = container.slices + para.inline_parser:reparse_attributes() + self.pos = para.inline_parser.lastpos + 1 + return true + end + end, + close = function(container) + local attr_matches = container.attribute_parser:get_matches() + self:add_match(container.startpos, container.startpos, "+block_attributes") + for i=1,#attr_matches do + self:add_match(unpack(attr_matches[i])) + end + self:add_match(self.pos, self.pos, "-block_attributes") + self.containers[#self.containers] = nil + end + } + } +end + +function Parser:get_inline_matches() + local matches = + self.containers[#self.containers].inline_parser:get_matches() + for i=1,#matches do + self.matches[#self.matches + 1] = matches[i] + end +end + +function Parser:find(patt) + return find(self.subject, patt, self.pos) +end + +function Parser:add_match(startpos, endpos, annotation) + self.matches[#self.matches + 1] = {startpos, endpos, annotation} +end + +function Parser:add_container(container) + local last_matched = self.last_matched_container + while #self.containers > last_matched or + (#self.containers > 0 and + self.containers[#self.containers].content ~= "block") do + self.containers[#self.containers]:close() + end + self.containers[#self.containers + 1] = container +end + +function Parser:skip_space() + local newpos, _ = find(self.subject, "[^ \t]", self.pos) + if newpos then + self.indent = newpos - self.startline + self.pos = newpos + end +end + +function Parser:get_eol() + local starteol, endeol = find(self.subject, "[\r]?[\n]", self.pos) + if not endeol then + starteol, endeol = #self.subject, #self.subject + end + self.starteol = starteol + self.endeol = endeol +end + +-- Returns an iterator over events. At each iteration, the iterator +-- returns three values: start byte position, end byte position, +-- and annotation. +function Parser:events() + local specs = self:specs() + local para_spec = specs[1] + local subjectlen = #self.subject + + return function() -- iterator + + while self.pos <= subjectlen do + + -- return any accumulated matches + if self.returned < #self.matches then + self.returned = self.returned + 1 + return unpack(self.matches[self.returned]) + end + + self.indent = 0 + self.startline = self.pos + self.finished_line = false + self:get_eol() + + -- check open containers for continuation + self.last_matched_container = 0 + local idx = 0 + while idx < #self.containers do + idx = idx + 1 + local container = self.containers[idx] + -- skip any indentation + self:skip_space() + if container:continue() then + self.last_matched_container = idx + else + break + end + end + + -- if we hit a close fence, we can move to next line + if self.finished_line then + while #self.containers > self.last_matched_container do + self.containers[#self.containers]:close() + end + end + + if not self.finished_line then + -- check for new containers + self:skip_space() + local is_blank = (self.pos == self.starteol) + + local new_starts = false + local last_match = self.containers[self.last_matched_container] + local check_starts = not is_blank and + (not last_match or last_match.content == "block") and + not self:find("^%a+%s") -- optimization + while check_starts do + check_starts = false + for i=1,#specs do + local spec = specs[i] + if not spec.is_para then + if spec:open() then + self.last_matched_container = #self.containers + if self.finished_line then + check_starts = false + else + self:skip_space() + new_starts = true + check_starts = spec.content == "block" + end + break + end + end + end + end + + if not self.finished_line then + -- handle remaining content + self:skip_space() + + is_blank = (self.pos == self.starteol) + + local is_lazy = not is_blank and + not new_starts and + self.last_matched_container < #self.containers and + self.containers[#self.containers].content == 'inline' + + local last_matched = self.last_matched_container + if not is_lazy then + while #self.containers > 0 and #self.containers > last_matched do + self.containers[#self.containers]:close() + end + end + + local tip = self.containers[#self.containers] + + -- add para by default if there's text + if not tip or tip.content == 'block' then + if is_blank then + if not new_starts then + -- need to track these for tight/loose lists + self:add_match(self.pos, self.endeol, "blankline") + end + else + para_spec:open() + end + tip = self.containers[#self.containers] + end + + if tip then + if tip.content == "text" then + local startpos = self.pos + if tip.indent and self.indent > tip.indent then + -- get back the leading spaces we gobbled + startpos = startpos - (self.indent - tip.indent) + end + self:add_match(startpos, self.endeol, "str") + elseif tip.content == "inline" then + if not is_blank then + tip.inline_parser:feed(self.pos, self.endeol) + end + end + end + end + end + + self.pos = self.endeol + 1 + + end + + -- close unmatched containers + while #self.containers > 0 do + self.containers[#self.containers]:close() + end + -- return any accumulated matches + if self.returned < #self.matches then + self.returned = self.returned + 1 + return unpack(self.matches[self.returned]) + end + + end + +end + +return { Parser = Parser, + Container = Container } diff --git a/djot/filter.lua b/djot/filter.lua new file mode 100644 index 0000000..0026bdf --- /dev/null +++ b/djot/filter.lua @@ -0,0 +1,174 @@ +--- @module djot.filter +--- Support filters that walk the AST and transform a +--- document between parsing and rendering, like pandoc Lua filters. +--- +--- This filter uppercases all str elements. +--- +--- return { +--- str = function(e) +--- e.text = e.text:upper() +--- end +--- } +--- +--- A filter may define functions for as many different tag types +--- as it likes. traverse will walk the AST and apply matching +--- functions to each node. +--- +--- To load a filter: +--- +--- local filter = require_filter(path) +--- +--- or +--- +--- local filter = load_filter(string) +--- +--- By default filters do a bottom-up traversal; that is, the +--- filter for a node is run after its children have been processed. +--- It is possible to do a top-down travel, though, and even +--- to run separate actions on entering a node (before processing the +--- children) and on exiting (after processing the children). To do +--- this, associate the node's tag with a table containing `enter` and/or +--- `exit` functions. The following filter will capitalize text +--- that is nested inside emphasis, but not other text: +--- +--- local capitalize = 0 +--- return { +--- emph = { +--- enter = function(e) +--- capitalize = capitalize + 1 +--- end, +--- exit = function(e) +--- capitalize = capitalize - 1 +--- end, +--- }, +--- str = function(e) +--- if capitalize > 0 then +--- e.text = e.text:upper() +--- end +--- end +--- } +--- +--- For a top-down traversal, you'd just use the `enter` functions. +--- If the tag is associated directly with a function, as in the +--- first example above, it is treated as an `exit` function. +--- +--- It is possible to inhibit traversal into the children of a node, +--- by having the `enter` function return the value true (or any truish +--- value, say `'stop'`). This can be used, for example, to prevent +--- the contents of a footnote from being processed: +--- +--- return { +--- footnote = { +--- enter = function(e) +--- return true +--- end +--- } +--- } +--- +--- A single filter may return a table with multiple tables, which will be +--- applied sequentially. + +local function handle_node(node, filterpart) + local action = filterpart[node.t] + local action_in, action_out + if type(action) == "table" then + action_in = action.enter + action_out = action.exit + elseif type(action) == "function" then + action_out = action + end + if action_in then + local stop_traversal = action_in(node) + if stop_traversal then + return + end + end + if node.c then + for _,child in ipairs(node.c) do + handle_node(child, filterpart) + end + end + if node.footnotes then + for _, note in pairs(node.footnotes) do + handle_node(note, filterpart) + end + end + if action_out then + action_out(node) + end +end + +local function traverse(node, filterpart) + handle_node(node, filterpart) + return node +end + +--- Apply a filter to a document. +--- @param node document (AST) +--- @param filter the filter to apply +local function apply_filter(node, filter) + for _,filterpart in ipairs(filter) do + traverse(node, filterpart) + end +end + +--- Returns a table containing the filter defined in `fp`. +--- `fp` will be sought using `require`, so it may occur anywhere +--- on the `LUA_PATH`, or in the working directory. On error, +--- returns nil and an error message. +--- @param fp path of file containing filter +--- @return the compiled filter, or nil and and error message +local function require_filter(fp) + local oldpackagepath = package.path + -- allow omitting or providing the .lua extension: + local ok, filter = pcall(function() + package.path = "./?.lua;" .. package.path + local f = require(fp:gsub("%.lua$","")) + package.path = oldpackagepath + return f + end) + if not ok then + return nil, filter + elseif type(filter) ~= "table" then + return nil, "filter must be a table" + end + if #filter == 0 then -- just a single filter part given + return {filter} + else + return filter + end +end + +--- Load filter from a string, which should have the +--- form `return { ... }`. On error, return nil and an +--- error message. +--- @param s string containing the filter +--- @return the compiled filter, or nil and and error message +local function load_filter(s) + local fn, err + if _VERSION:match("5.1") then + fn, err = loadstring(s) + else + fn, err = load(s) + end + if fn then + local filter = fn() + if type(filter) ~= "table" then + return nil, "filter must be a table" + end + if #filter == 0 then -- just a single filter given + return {filter} + else + return filter + end + else + return nil, err + end +end + +--- @export +return { + apply_filter = apply_filter, + require_filter = require_filter, + load_filter = load_filter +} diff --git a/djot/html.lua b/djot/html.lua new file mode 100644 index 0000000..48ed50c --- /dev/null +++ b/djot/html.lua @@ -0,0 +1,549 @@ +local ast = require("djot.ast") +local new_node = ast.new_node +local new_attributes = ast.new_attributes +local add_child = ast.add_child +local unpack = unpack or table.unpack +local insert_attribute, copy_attributes = + ast.insert_attribute, ast.copy_attributes +local format = string.format +local find, gsub = string.find, string.gsub + +-- Produce a copy of a table. +local function copy(tbl) + local result = {} + if tbl then + for k,v in pairs(tbl) do + local newv = v + if type(v) == "table" then + newv = copy(v) + end + result[k] = newv + end + end + return result +end + +local function to_text(node) + local buffer = {} + if node.t == "str" then + buffer[#buffer + 1] = node.s + elseif node.t == "nbsp" then + buffer[#buffer + 1] = "\160" + elseif node.t == "softbreak" then + buffer[#buffer + 1] = " " + elseif node.c and #node.c > 0 then + for i=1,#node.c do + buffer[#buffer + 1] = to_text(node.c[i]) + end + end + return table.concat(buffer) +end + +local Renderer = {} + +function Renderer:new() + local state = { + out = function(s) + io.stdout:write(s) + end, + tight = false, + footnote_index = {}, + next_footnote_index = 1, + references = nil, + footnotes = nil } + setmetatable(state, self) + self.__index = self + return state +end + +Renderer.html_escapes = + { ["<"] = "<", + [">"] = ">", + ["&"] = "&", + ['"'] = """ } + +function Renderer:escape_html(s) + if find(s, '[<>&]') then + return (gsub(s, '[<>&]', self.html_escapes)) + else + return s + end +end + +function Renderer:escape_html_attribute(s) + if find(s, '[<>&"]') then + return (gsub(s, '[<>&"]', self.html_escapes)) + else + return s + end +end + +function Renderer:render(doc, handle) + self.references = doc.references + self.footnotes = doc.footnotes + if handle then + self.out = function(s) + handle:write(s) + end + end + self[doc.t](self, doc) +end + + +function Renderer:render_children(node) + -- trap stack overflow + local ok, err = pcall(function () + if node.c and #node.c > 0 then + local oldtight + if node.tight ~= nil then + oldtight = self.tight + self.tight = node.tight + end + for i=1,#node.c do + self[node.c[i].t](self, node.c[i]) + end + if node.tight ~= nil then + self.tight = oldtight + end + end + end) + if not ok and err:find("stack overflow") then + self.out("(((DEEPLY NESTED CONTENT OMITTED)))\n") + end +end + +function Renderer:render_attrs(node) + if node.attr then + for k,v in pairs(node.attr) do + self.out(" " .. k .. "=" .. '"' .. + self:escape_html_attribute(v) .. '"') + end + end + if node.pos then + local sp, ep = unpack(node.pos) + self.out(' data-startpos="' .. tostring(sp) .. + '" data-endpos="' .. tostring(ep) .. '"') + end +end + +function Renderer:render_tag(tag, node) + self.out("<" .. tag) + self:render_attrs(node) + self.out(">") +end + +function Renderer:add_backlink(nodes, i) + local backlink = new_node("link") + backlink.destination = "#fnref" .. tostring(i) + backlink.attr = ast.new_attributes({role = "doc-backlink"}) + local arrow = new_node("str") + arrow.s = "↩︎︎" + add_child(backlink, arrow) + if nodes.c[#nodes.c].t == "para" then + add_child(nodes.c[#nodes.c], backlink) + else + local para = new_node("para") + add_child(para, backlink) + add_child(nodes, para) + end +end + +function Renderer:doc(node) + self:render_children(node) + -- render notes + if self.next_footnote_index > 1 then + local ordered_footnotes = {} + for k,v in pairs(self.footnotes) do + if self.footnote_index[k] then + ordered_footnotes[self.footnote_index[k]] = v + end + end + self.out('
\n
\n
    \n') + for i=1,#ordered_footnotes do + local note = ordered_footnotes[i] + if note then + self.out(format('
  1. \n', i)) + self:add_backlink(note,i) + self:render_children(note) + self.out('
    2. \n') + end + end + self.out('
\n
\n') + end +end + +function Renderer:raw_block(node) + if node.format == "html" then + self.out(node.s) -- no escaping + end +end + +function Renderer:para(node) + if not self.tight then + self:render_tag("p", node) + end + self:render_children(node) + if not self.tight then + self.out("

") + end + self.out("\n") +end + +function Renderer:blockquote(node) + self:render_tag("blockquote", node) + self.out("\n") + self:render_children(node) + self.out("\n") +end + +function Renderer:div(node) + self:render_tag("div", node) + self.out("\n") + self:render_children(node) + self.out("\n") +end + +function Renderer:section(node) + self:render_tag("section", node) + self.out("\n") + self:render_children(node) + self.out("\n") +end + +function Renderer:heading(node) + self:render_tag("h" .. node.level , node) + self:render_children(node) + self.out("\n") +end + +function Renderer:thematic_break(node) + self:render_tag("hr", node) + self.out("\n") +end + +function Renderer:code_block(node) + self:render_tag("pre", node) + self.out(" 0 then + self.out(" class=\"language-" .. node.lang .. "\"") + end + self.out(">") + self.out(self:escape_html(node.s)) + self.out("\n") +end + +function Renderer:table(node) + self:render_tag("table", node) + self.out("\n") + self:render_children(node) + self.out("\n") +end + +function Renderer:row(node) + self:render_tag("tr", node) + self.out("\n") + self:render_children(node) + self.out("\n") +end + +function Renderer:cell(node) + local tag + if node.head then + tag = "th" + else + tag = "td" + end + local attr = copy(node.attr) + if node.align then + insert_attribute(attr, "style", "text-align: " .. node.align .. ";") + end + self:render_tag(tag, {attr = attr}) + self:render_children(node) + self.out("\n") +end + +function Renderer:caption(node) + self:render_tag("caption", node) + self:render_children(node) + self.out("\n") +end + +function Renderer:list(node) + local sty = node.style + if sty == "*" or sty == "+" or sty == "-" then + self:render_tag("ul", node) + self.out("\n") + self:render_children(node) + self.out("\n") + elseif sty == "X" then + local attr = copy(node.attr) + if attr.class then + attr.class = "task-list " .. attr.class + else + insert_attribute(attr, "class", "task-list") + end + self:render_tag("ul", {attr = attr}) + self.out("\n") + self:render_children(node) + self.out("\n") + elseif sty == ":" then + self:render_tag("dl", node) + self.out("\n") + self:render_children(node) + self.out("\n") + else + self.out(" 1 then + self.out(" start=\"" .. node.start .. "\"") + end + local list_type = gsub(node.style, "%p", "") + if list_type ~= "1" then + self.out(" type=\"" .. list_type .. "\"") + end + self:render_attrs(node) + self.out(">\n") + self:render_children(node) + self.out("\n") + end +end + +function Renderer:list_item(node) + if node.checkbox then + if node.checkbox == "checked" then + self.out('
  • ') + elseif node.checkbox == "unchecked" then + self.out('
  • ') + end + else + self:render_tag("li", node) + end + self.out("\n") + self:render_children(node) + self.out("
    • \n") +end + +function Renderer:term(node) + self:render_tag("dt", node) + self:render_children(node) + self.out("\n") +end + +function Renderer:definition(node) + self:render_tag("dd", node) + self.out("\n") + self:render_children(node) + self.out("\n") +end + +function Renderer:definition_list_item(node) + self:render_children(node) +end + +function Renderer:reference_definition() +end + +function Renderer:footnote_reference(node) + local label = node.s + local index = self.footnote_index[label] + if not index then + index = self.next_footnote_index + self.footnote_index[label] = index + self.next_footnote_index = self.next_footnote_index + 1 + end + self.out(format('    %d', index, index, index)) +end + +function Renderer:raw_inline(node) + if node.format == "html" then + self.out(node.s) -- no escaping + end +end + +function Renderer:str(node) + -- add a span, if needed, to contain attribute on a bare string: + if node.attr then + self:render_tag("span", node) + self.out(self:escape_html(node.s)) + self.out("") + else + self.out(self:escape_html(node.s)) + end +end + +function Renderer:softbreak() + self.out("\n") +end + +function Renderer:hardbreak() + self.out("
    \n") +end + +function Renderer:nbsp() + self.out(" ") +end + +function Renderer:verbatim(node) + self:render_tag("code", node) + self.out(self:escape_html(node.s)) + self.out("") +end + +function Renderer:link(node) + local attrs = new_attributes{} + if node.reference then + local ref = self.references[node.reference] + if ref then + if ref.attr then + copy_attributes(attrs, ref.attr) + end + insert_attribute(attrs, "href", ref.destination) + end + elseif node.destination then + insert_attribute(attrs, "href", node.destination) + end + -- link's attributes override reference's: + copy_attributes(attrs, node.attr) + self:render_tag("a", {attr = attrs}) + self:render_children(node) + self.out("") +end + +Renderer.url = Renderer.link + +Renderer.email = Renderer.link + +function Renderer:image(node) + local attrs = new_attributes{} + local alt_text = to_text(node) + if #alt_text > 0 then + insert_attribute(attrs, "alt", to_text(node)) + end + if node.reference then + local ref = self.references[node.reference] + if ref then + if ref.attr then + copy_attributes(attrs, ref.attr) + end + insert_attribute(attrs, "src", ref.destination) + end + elseif node.destination then + insert_attribute(attrs, "src", node.destination) + end + -- image's attributes override reference's: + copy_attributes(attrs, node.attr) + self:render_tag("img", {attr = attrs}) +end + +function Renderer:span(node) + self:render_tag("span", node) + self:render_children(node) + self.out("") +end + +function Renderer:mark(node) + self:render_tag("mark", node) + self:render_children(node) + self.out("") +end + +function Renderer:insert(node) + self:render_tag("ins", node) + self:render_children(node) + self.out("") +end + +function Renderer:delete(node) + self:render_tag("del", node) + self:render_children(node) + self.out("") +end + +function Renderer:subscript(node) + self:render_tag("sub", node) + self:render_children(node) + self.out("") +end + +function Renderer:superscript(node) + self:render_tag("sup", node) + self:render_children(node) + self.out("") +end + +function Renderer:emph(node) + self:render_tag("em", node) + self:render_children(node) + self.out("") +end + +function Renderer:strong(node) + self:render_tag("strong", node) + self:render_children(node) + self.out("") +end + +function Renderer:double_quoted(node) + self.out("“") + self:render_children(node) + self.out("”") +end + +function Renderer:single_quoted(node) + self.out("‘") + self:render_children(node) + self.out("’") +end + +function Renderer:left_double_quote() + self.out("“") +end + +function Renderer:right_double_quote() + self.out("”") +end + +function Renderer:left_single_quote() + self.out("‘") +end + +function Renderer:right_single_quote() + self.out("’") +end + +function Renderer:ellipses() + self.out("…") +end + +function Renderer:em_dash() + self.out("—") +end + +function Renderer:en_dash() + self.out("–") +end + +function Renderer:symbol(node) + self.out(":" .. node.alias .. ":") +end + +function Renderer:math(node) + local math_t = "inline" + if find(node.attr.class, "display") then + math_t = "display" + end + self:render_tag("span", node) + if math_t == "inline" then + self.out("\\(") + else + self.out("\\[") + end + self.out(self:escape_html(node.s)) + if math_t == "inline" then + self.out("\\)") + else + self.out("\\]") + end + self.out("") +end + +return { Renderer = Renderer } diff --git a/djot/inline.lua b/djot/inline.lua new file mode 100644 index 0000000..8a3c2ad --- /dev/null +++ b/djot/inline.lua @@ -0,0 +1,679 @@ +-- this allows the code to work with both lua and luajit: +local unpack = unpack or table.unpack +local attributes = require("djot.attributes") +local find, byte = string.find, string.byte + +-- allow up to 3 captures... +local function bounded_find(subj, patt, startpos, endpos) + local sp,ep,c1,c2,c3 = find(subj, patt, startpos) + if ep and ep <= endpos then + return sp,ep,c1,c2,c3 + end +end + +-- General note on the parsing strategy: our objective is to +-- parse without backtracking. To that end, we keep a stack of +-- potential 'openers' for links, images, emphasis, and other +-- inline containers. When we parse a potential closer for +-- one of these constructions, we can scan the stack of openers +-- for a match, which will tell us the location of the potential +-- opener. We can then change the annotation of the match at +-- that location to '+emphasis' or whatever. + +local InlineParser = {} + +function InlineParser:new(subject, warn) + local state = + { warn = warn or function() end, -- function to issue warnings + subject = subject, -- text to parse + matches = {}, -- table pos : (endpos, annotation) + openers = {}, -- map from closer_type to array of (pos, data) in reverse order + verbatim = 0, -- parsing verbatim span to be ended by n backticks + verbatim_type = nil, -- whether verbatim is math or regular + destination = false, -- parsing link destination in () + firstpos = 0, -- position of first slice + lastpos = 0, -- position of last slice + allow_attributes = true, -- allow parsing of attributes + attribute_parser = nil, -- attribute parser + attribute_start = nil, -- start of potential attribute + attribute_slices = nil, -- slices we've tried to parse as attributes + } + setmetatable(state, self) + self.__index = self + return state +end + +function InlineParser:add_match(startpos, endpos, annotation) + self.matches[startpos] = {startpos, endpos, annotation} +end + +function InlineParser:add_opener(name, ...) + -- 1 = startpos, 2 = endpos, 3 = annotation, 4 = substartpos, 5 = endpos + -- + -- [link text](url) + -- ^ ^^ + -- 1,2 4 5 3 = "explicit_link" + + if not self.openers[name] then + self.openers[name] = {} + end + table.insert(self.openers[name], {...}) +end + +function InlineParser:clear_openers(startpos, endpos) + -- remove other openers in between the matches + for _,v in pairs(self.openers) do + local i = #v + while v[i] do + local sp,ep,_,sp2,ep2 = unpack(v[i]) + if sp >= startpos and ep <= endpos then + v[i] = nil + elseif (sp2 and sp2 >= startpos) and (ep2 and ep2 <= endpos) then + v[i][3] = nil + v[i][4] = nil + v[i][5] = nil + else + break + end + i = i - 1 + end + end +end + +function InlineParser:str_matches(startpos, endpos) + for i = startpos, endpos do + local m = self.matches[i] + if m then + local sp, ep, annot = unpack(m) + if annot ~= "str" and annot ~= "escape" then + self.matches[i] = {sp, ep, "str"} + end + end + end +end + +local function matches_pattern(match, patt) + if match then + return string.find(match[3], patt) + end +end + + +function InlineParser.between_matched(c, annotation, defaultmatch, opentest) + return function(self, pos, endpos) + defaultmatch = defaultmatch or "str" + local subject = self.subject + local can_open = find(subject, "^%S", pos + 1) + local can_close = find(subject, "^%S", pos - 1) + local has_open_marker = matches_pattern(self.matches[pos - 1], "^open%_marker") + local has_close_marker = pos + 1 <= endpos and + byte(subject, pos + 1) == 125 -- } + local endcloser = pos + local startopener = pos + + if type(opentest) == "function" then + can_open = can_open and opentest(self, pos) + end + + -- allow explicit open/close markers to override: + if has_open_marker then + can_open = true + can_close = false + startopener = pos - 1 + end + if not has_open_marker and has_close_marker then + can_close = true + can_open = false + endcloser = pos + 1 + end + + if has_open_marker and defaultmatch:match("^right") then + defaultmatch = defaultmatch:gsub("^right", "left") + elseif has_close_marker and defaultmatch:match("^left") then + defaultmatch = defaultmatch:gsub("^left", "right") + end + + local d + if has_close_marker then + d = "{" .. c + else + d = c + end + local openers = self.openers[d] + if can_close and openers and #openers > 0 then + -- check openers for a match + local openpos, openposend = unpack(openers[#openers]) + if openposend ~= pos - 1 then -- exclude empty emph + self:clear_openers(openpos, pos) + self:add_match(openpos, openposend, "+" .. annotation) + self:add_match(pos, endcloser, "-" .. annotation) + return endcloser + 1 + end + end + + -- if we get here, we didn't match an opener + if can_open then + if has_open_marker then + d = "{" .. c + else + d = c + end + self:add_opener(d, startopener, pos) + self:add_match(startopener, pos, defaultmatch) + return pos + 1 + else + self:add_match(pos, endcloser, defaultmatch) + return endcloser + 1 + end + end +end + +InlineParser.matchers = { + -- 96 = ` + [96] = function(self, pos, endpos) + local subject = self.subject + local _, endchar = bounded_find(subject, "^`*", pos, endpos) + if not endchar then + return nil + end + if find(subject, "^%$%$", pos - 2) and + not find(subject, "^\\", pos - 3) then + self.matches[pos - 2] = nil + self.matches[pos - 1] = nil + self:add_match(pos - 2, endchar, "+display_math") + self.verbatim_type = "display_math" + elseif find(subject, "^%$", pos - 1) then + self.matches[pos - 1] = nil + self:add_match(pos - 1, endchar, "+inline_math") + self.verbatim_type = "inline_math" + else + self:add_match(pos, endchar, "+verbatim") + self.verbatim_type = "verbatim" + end + self.verbatim = endchar - pos + 1 + return endchar + 1 + end, + + -- 92 = \ + [92] = function(self, pos, endpos) + local subject = self.subject + local _, endchar = bounded_find(subject, "^[ \t]*\r?\n", pos + 1, endpos) + self:add_match(pos, pos, "escape") + if endchar then + -- see if there were preceding spaces + if #self.matches > 0 then + local sp, ep, annot = unpack(self.matches[#self.matches]) + if annot == "str" then + while ep >= sp and + (subject:byte(ep) == 32 or subject:byte(ep) == 9) do + ep = ep -1 + end + if ep < sp then + self.matches[#self.matches] = nil + else + self:add_match(sp, ep, "str") + end + end + end + self:add_match(pos + 1, endchar, "hardbreak") + return endchar + 1 + else + local _, ec = bounded_find(subject, "^[%p ]", pos + 1, endpos) + if not ec then + self:add_match(pos, pos, "str") + return pos + 1 + else + self:add_match(pos, pos, "escape") + if find(subject, "^ ", pos + 1) then + self:add_match(pos + 1, ec, "nbsp") + else + self:add_match(pos + 1, ec, "str") + end + return ec + 1 + end + end + end, + + -- 60 = < + [60] = function(self, pos, endpos) + local subject = self.subject + local starturl, endurl = + bounded_find(subject, "^%<[^<>%s]+%>", pos, endpos) + if starturl then + local is_url = bounded_find(subject, "^%a+:", pos + 1, endurl) + local is_email = bounded_find(subject, "^[^:]+%@", pos + 1, endurl) + if is_email then + self:add_match(starturl, starturl, "+email") + self:add_match(starturl + 1, endurl - 1, "str") + self:add_match(endurl, endurl, "-email") + return endurl + 1 + elseif is_url then + self:add_match(starturl, starturl, "+url") + self:add_match(starturl + 1, endurl - 1, "str") + self:add_match(endurl, endurl, "-url") + return endurl + 1 + end + end + end, + + -- 126 = ~ + [126] = InlineParser.between_matched('~', 'subscript'), + + -- 94 = ^ + [94] = InlineParser.between_matched('^', 'superscript'), + + -- 91 = [ + [91] = function(self, pos, endpos) + local sp, ep = bounded_find(self.subject, "^%^([^]]+)%]", pos + 1, endpos) + if sp then -- footnote ref + self:add_match(pos, ep, "footnote_reference") + return ep + 1 + else + self:add_opener("[", pos, pos) + self:add_match(pos, pos, "str") + return pos + 1 + end + end, + + -- 93 = ] + [93] = function(self, pos, endpos) + local openers = self.openers["["] + local subject = self.subject + if openers and #openers > 0 then + local opener = openers[#openers] + if opener[3] == "reference_link" then + -- found a reference link + -- add the matches + local is_image = bounded_find(subject, "^!", opener[1] - 1, endpos) + and not bounded_find(subject, "^[\\]", opener[1] - 2, endpos) + if is_image then + self:add_match(opener[1] - 1, opener[1] - 1, "image_marker") + self:add_match(opener[1], opener[2], "+imagetext") + self:add_match(opener[4], opener[4], "-imagetext") + else + self:add_match(opener[1], opener[2], "+linktext") + self:add_match(opener[4], opener[4], "-linktext") + end + self:add_match(opener[5], opener[5], "+reference") + self:add_match(pos, pos, "-reference") + -- convert all matches to str + self:str_matches(opener[5] + 1, pos - 1) + -- remove from openers + self:clear_openers(opener[1], pos) + return pos + 1 + elseif bounded_find(subject, "^%[", pos + 1, endpos) then + opener[3] = "reference_link" + opener[4] = pos -- intermediate ] + opener[5] = pos + 1 -- intermediate [ + self:add_match(pos, pos + 1, "str") + -- remove any openers between [ and ] + self:clear_openers(opener[1] + 1, pos - 1) + return pos + 2 + elseif bounded_find(subject, "^%(", pos + 1, endpos) then + self.openers["("] = {} -- clear ( openers + opener[3] = "explicit_link" + opener[4] = pos -- intermediate ] + opener[5] = pos + 1 -- intermediate ( + self.destination = true + self:add_match(pos, pos + 1, "str") + -- remove any openers between [ and ] + self:clear_openers(opener[1] + 1, pos - 1) + return pos + 2 + elseif bounded_find(subject, "^%{", pos + 1, endpos) then + -- assume this is attributes, bracketed span + self:add_match(opener[1], opener[2], "+span") + self:add_match(pos, pos, "-span") + -- remove any openers between [ and ] + self:clear_openers(opener[1], pos) + return pos + 1 + end + end + end, + + + -- 40 = ( + [40] = function(self, pos) + if not self.destination then return nil end + self:add_opener("(", pos, pos) + self:add_match(pos, pos, "str") + return pos + 1 + end, + + -- 41 = ) + [41] = function(self, pos, endpos) + if not self.destination then return nil end + local parens = self.openers["("] + if parens and #parens > 0 and parens[#parens][1] then + parens[#parens] = nil -- clear opener + self:add_match(pos, pos, "str") + return pos + 1 + else + local subject = self.subject + local openers = self.openers["["] + if openers and #openers > 0 + and openers[#openers][3] == "explicit_link" then + local opener = openers[#openers] + -- we have inline link + local is_image = bounded_find(subject, "^!", opener[1] - 1, endpos) + and not bounded_find(subject, "^[\\]", opener[1] - 2, endpos) + if is_image then + self:add_match(opener[1] - 1, opener[1] - 1, "image_marker") + self:add_match(opener[1], opener[2], "+imagetext") + self:add_match(opener[4], opener[4], "-imagetext") + else + self:add_match(opener[1], opener[2], "+linktext") + self:add_match(opener[4], opener[4], "-linktext") + end + self:add_match(opener[5], opener[5], "+destination") + self:add_match(pos, pos, "-destination") + self.destination = false + -- convert all matches to str + self:str_matches(opener[5] + 1, pos - 1) + -- remove from openers + self:clear_openers(opener[1], pos) + return pos + 1 + end + end + end, + + -- 95 = _ + [95] = InlineParser.between_matched('_', 'emph'), + + -- 42 = * + [42] = InlineParser.between_matched('*', 'strong'), + + -- 123 = { + [123] = function(self, pos, endpos) + if bounded_find(self.subject, "^[_*~^+='\"-]", pos + 1, endpos) then + self:add_match(pos, pos, "open_marker") + return pos + 1 + elseif self.allow_attributes then + self.attribute_parser = attributes.AttributeParser:new(self.subject) + self.attribute_start = pos + self.attribute_slices = {} + return pos + else + self:add_match(pos, pos, "str") + return pos + 1 + end + end, + + -- 58 = : + [58] = function(self, pos, endpos) + local sp, ep = bounded_find(self.subject, "^%:[%w_+-]+%:", pos, endpos) + if sp then + self:add_match(sp, ep, "symbol") + return ep + 1 + else + self:add_match(pos, pos, "str") + return pos + 1 + end + end, + + -- 43 = + + [43] = InlineParser.between_matched("+", "insert", "str", + function(self, pos) + return find(self.subject, "^%{", pos - 1) or + find(self.subject, "^%}", pos + 1) + end), + + -- 61 = = + [61] = InlineParser.between_matched("=", "mark", "str", + function(self, pos) + return find(self.subject, "^%{", pos - 1) or + find(self.subject, "^%}", pos + 1) + end), + + -- 39 = ' + [39] = InlineParser.between_matched("'", "single_quoted", "right_single_quote", + function(self, pos) -- test to open + return pos == 1 or + find(self.subject, "^[%s\"'-([]", pos - 1) + end), + + -- 34 = " + [34] = InlineParser.between_matched('"', "double_quoted", "left_double_quote"), + + -- 45 = - + [45] = function(self, pos, endpos) + local subject = self.subject + local nextpos + if byte(subject, pos - 1) == 123 or + byte(subject, pos + 1) == 125 then -- (123 = { 125 = }) + nextpos = InlineParser.between_matched("-", "delete", "str", + function(slf, p) + return find(slf.subject, "^%{", p - 1) or + find(slf.subject, "^%}", p + 1) + end)(self, pos, endpos) + return nextpos + end + -- didn't match a del, try for smart hyphens: + local _, ep = find(subject, "^%-*", pos) + if endpos < ep then + ep = endpos + end + local hyphens = 1 + ep - pos + if byte(subject, ep + 1) == 125 then -- 125 = } + hyphens = hyphens - 1 -- last hyphen is close del + end + if hyphens == 0 then -- this means we have '-}' + self:add_match(pos, pos + 1, "str") + return pos + 2 + end + -- Try to construct a homogeneous sequence of dashes + local all_em = hyphens % 3 == 0 + local all_en = hyphens % 2 == 0 + while hyphens > 0 do + if all_em then + self:add_match(pos, pos + 2, "em_dash") + pos = pos + 3 + hyphens = hyphens - 3 + elseif all_en then + self:add_match(pos, pos + 1, "en_dash") + pos = pos + 2 + hyphens = hyphens - 2 + elseif hyphens >= 3 and (hyphens % 2 ~= 0 or hyphens > 4) then + self:add_match(pos, pos + 2, "em_dash") + pos = pos + 3 + hyphens = hyphens - 3 + elseif hyphens >= 2 then + self:add_match(pos, pos + 1, "en_dash") + pos = pos + 2 + hyphens = hyphens - 2 + else + self:add_match(pos, pos, "str") + pos = pos + 1 + hyphens = hyphens - 1 + end + end + return pos + end, + + -- 46 = . + [46] = function(self, pos, endpos) + if bounded_find(self.subject, "^%.%.", pos + 1, endpos) then + self:add_match(pos, pos +2, "ellipses") + return pos + 3 + end + end + } + +function InlineParser:single_char(pos) + self:add_match(pos, pos, "str") + return pos + 1 +end + +-- Reparse attribute_slices that we tried to parse as an attribute +function InlineParser:reparse_attributes() + local slices = self.attribute_slices + if not slices then + return + end + self.allow_attributes = false + self.attribute_parser = nil + self.attribute_start = nil + if slices then + for i=1,#slices do + self:feed(unpack(slices[i])) + end + end + self.allow_attributes = true + self.attribute_slices = nil +end + +-- Feed a slice to the parser, updating state. +function InlineParser:feed(spos, endpos) + local special = "[][\\`{}_*()!<>~^:=+$\r\n'\".-]" + local subject = self.subject + local matchers = self.matchers + local pos + if self.firstpos == 0 or spos < self.firstpos then + self.firstpos = spos + end + if self.lastpos == 0 or endpos > self.lastpos then + self.lastpos = endpos + end + pos = spos + while pos <= endpos do + if self.attribute_parser then + local sp = pos + local ep2 = bounded_find(subject, special, pos, endpos) + if not ep2 or ep2 > endpos then + ep2 = endpos + end + local status, ep = self.attribute_parser:feed(sp, ep2) + if status == "done" then + local attribute_start = self.attribute_start + -- add attribute matches + self:add_match(attribute_start, attribute_start, "+attributes") + self:add_match(ep, ep, "-attributes") + local attr_matches = self.attribute_parser:get_matches() + -- add attribute matches + for i=1,#attr_matches do + self:add_match(unpack(attr_matches[i])) + end + -- restore state to prior to adding attribute parser: + self.attribute_parser = nil + self.attribute_start = nil + self.attribute_slices = nil + pos = ep + 1 + elseif status == "fail" then + self:reparse_attributes() + pos = sp -- we'll want to go over the whole failed portion again, + -- as no slice was added for it + elseif status == "continue" then + if #self.attribute_slices == 0 then + self.attribute_slices = {} + end + self.attribute_slices[#self.attribute_slices + 1] = {sp,ep} + pos = ep + 1 + end + else + -- find next interesting character: + local newpos = bounded_find(subject, special, pos, endpos) or endpos + 1 + if newpos > pos then + self:add_match(pos, newpos - 1, "str") + pos = newpos + if pos > endpos then + break -- otherwise, fall through: + end + end + -- if we get here, then newpos = pos, + -- i.e. we have something interesting at pos + local c = byte(subject, pos) + + if c == 13 or c == 10 then -- cr or lf + if c == 13 and bounded_find(subject, "^[%n]", pos + 1, endpos) then + self:add_match(pos, pos + 1, "softbreak") + pos = pos + 2 + else + self:add_match(pos, pos, "softbreak") + pos = pos + 1 + end + elseif self.verbatim > 0 then + if c == 96 then + local _, endchar = bounded_find(subject, "^`+", pos, endpos) + if endchar and endchar - pos + 1 == self.verbatim then + -- check for raw attribute + local sp, ep = + bounded_find(subject, "^%{%=[^%s{}`]+%}", endchar + 1, endpos) + if sp and self.verbatim_type == "verbatim" then -- raw + self:add_match(pos, endchar, "-" .. self.verbatim_type) + self:add_match(sp, ep, "raw_format") + pos = ep + 1 + else + self:add_match(pos, endchar, "-" .. self.verbatim_type) + pos = endchar + 1 + end + self.verbatim = 0 + self.verbatim_type = nil + else + endchar = endchar or endpos + self:add_match(pos, endchar, "str") + pos = endchar + 1 + end + else + self:add_match(pos, pos, "str") + pos = pos + 1 + end + else + local matcher = matchers[c] + pos = (matcher and matcher(self, pos, endpos)) or self:single_char(pos) + end + end + end +end + + -- Return true if we're parsing verbatim content. +function InlineParser:in_verbatim() + return self.verbatim > 0 +end + +function InlineParser:get_matches() + local sorted = {} + local subject = self.subject + local lastsp, lastep, lastannot + if self.attribute_parser then -- we're still in an attribute parse + self:reparse_attributes() + end + for i=self.firstpos, self.lastpos do + if self.matches[i] then + local sp, ep, annot = unpack(self.matches[i]) + if annot == "str" and lastannot == "str" and lastep + 1 == sp then + -- consolidate adjacent strs + sorted[#sorted] = {lastsp, ep, annot} + lastsp, lastep, lastannot = lastsp, ep, annot + else + sorted[#sorted + 1] = self.matches[i] + lastsp, lastep, lastannot = sp, ep, annot + end + end + end + if #sorted > 0 then + local last = sorted[#sorted] + local startpos, endpos, annot = unpack(last) + -- remove final softbreak + if annot == "softbreak" then + sorted[#sorted] = nil + last = sorted[#sorted] + if not last then + return sorted + end + startpos, endpos, annot = unpack(last) + end + -- remove trailing spaces + if annot == "str" and byte(subject, endpos) == 32 then + while endpos > startpos and byte(subject, endpos) == 32 do + endpos = endpos - 1 + end + sorted[#sorted] = {startpos, endpos, annot} + end + if self.verbatim > 0 then -- unclosed verbatim + self.warn({ message = "Unclosed verbatim", pos = endpos }) + sorted[#sorted + 1] = {endpos, endpos, "-" .. self.verbatim_type} + end + end + return sorted +end + +return { InlineParser = InlineParser } diff --git a/djot/json.lua b/djot/json.lua new file mode 100644 index 0000000..e17996f --- /dev/null +++ b/djot/json.lua @@ -0,0 +1,137 @@ +-- Modified from +-- json.lua +-- Copyright (c) 2020 rxi +-- +-- Permission is hereby granted, free of charge, to any person obtaining a copy of +-- this software and associated documentation files (the "Software"), to deal in +-- the Software without restriction, including without limitation the rights to +-- use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +-- of the Software, and to permit persons to whom the Software is furnished to do +-- so, subject to the following conditions: +-- +-- The above copyright notice and this permission notice shall be included in all +-- copies or substantial portions of the Software. +-- +-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +-- SOFTWARE. +-- +-- Modifications to the original code: +-- +-- * Removed JSON decoding code + +local json = { _version = "0.1.2" } + +-- Encode + +local encode + +local escape_char_map = { + [ "\\" ] = "\\", + [ "\"" ] = "\"", + [ "\b" ] = "b", + [ "\f" ] = "f", + [ "\n" ] = "n", + [ "\r" ] = "r", + [ "\t" ] = "t", +} + +local escape_char_map_inv = { [ "/" ] = "/" } +for k, v in pairs(escape_char_map) do + escape_char_map_inv[v] = k +end + + +local function escape_char(c) + return "\\" .. (escape_char_map[c] or string.format("u%04x", c:byte())) +end + + +local function encode_nil(val) + return "null" +end + +local function encode_table(val, stack) + local res = {} + stack = stack or {} + + -- Circular reference? + if stack[val] then error("circular reference") end + + stack[val] = true + + if rawget(val, 1) ~= nil or next(val) == nil then + -- Treat as array -- check keys are valid and it is not sparse + local n = 0 + for k in pairs(val) do + if type(k) ~= "number" then + error("invalid table: mixed or invalid key types") + end + n = n + 1 + end + if n ~= #val then + error("invalid table: sparse array") + end + -- Encode + for i, v in ipairs(val) do + table.insert(res, encode(v, stack)) + end + stack[val] = nil + return "[" .. table.concat(res, ",") .. "]" + + else + -- Treat as an object + for k, v in pairs(val) do + if type(k) ~= "string" then + error("invalid table: mixed or invalid key types") + end + table.insert(res, encode(k, stack) .. ":" .. encode(v, stack)) + end + stack[val] = nil + return "{" .. table.concat(res, ",") .. "}" + end +end + + +local function encode_string(val) + return '"' .. val:gsub('[%z\1-\31\\"]', escape_char) .. '"' +end + + +local function encode_number(val) + -- Check for NaN, -inf and inf + if val ~= val or val <= -math.huge or val >= math.huge then + error("unexpected number value '" .. tostring(val) .. "'") + end + return string.format("%.14g", val) +end + + +local type_func_map = { + [ "nil" ] = encode_nil, + [ "table" ] = encode_table, + [ "string" ] = encode_string, + [ "number" ] = encode_number, + [ "boolean" ] = tostring, +} + + +encode = function(val, stack) + local t = type(val) + local f = type_func_map[t] + if f then + return f(val, stack) + end + error("unexpected type '" .. t .. "'") +end + + +function json.encode(val) + return ( encode(val) ) +end + +return json diff --git a/doc/api/index.html b/doc/api/index.html new file mode 100644 index 0000000..e1c9f42 --- /dev/null +++ b/doc/api/index.html @@ -0,0 +1,71 @@ + + + + + Reference + + + + +
    + +
    + +
    +
    +
    + +
    + + + + + + + + +
    + + +

    djot markup conversion

    + +

    Modules

    + + + + + +
    djot + +
    + + + + +
    djot.ast + +
    + + + + +
    djot.filter + +
    + +
    +
    +
    +
    +
    + + + diff --git a/doc/api/ldoc.css b/doc/api/ldoc.css new file mode 100644 index 0000000..52c4ad2 --- /dev/null +++ b/doc/api/ldoc.css @@ -0,0 +1,303 @@ +/* BEGIN RESET + +Copyright (c) 2010, Yahoo! Inc. All rights reserved. +Code licensed under the BSD License: +http://developer.yahoo.com/yui/license.html +version: 2.8.2r1 +*/ +html { + color: #000; + background: #FFF; +} +body,div,dl,dt,dd,ul,ol,li,h1,h2,h3,h4,h5,h6,pre,code,form,fieldset,legend,input,button,textarea,p,blockquote,th,td { + margin: 0; + padding: 0; +} +table { + border-collapse: collapse; + border-spacing: 0; +} +fieldset,img { + border: 0; +} +address,caption,cite,code,dfn,em,strong,th,var,optgroup { + font-style: inherit; + font-weight: inherit; +} +del,ins { + text-decoration: none; +} +li { + margin-left: 20px; +} +caption,th { + text-align: left; +} +h1,h2,h3,h4,h5,h6 { + font-size: 100%; + font-weight: bold; +} +q:before,q:after { + content: ''; +} +abbr,acronym { + border: 0; + font-variant: normal; +} +sup { + vertical-align: baseline; +} +sub { + vertical-align: baseline; +} +legend { + color: #000; +} +input,button,textarea,select,optgroup,option { + font-family: inherit; + font-size: inherit; + font-style: inherit; + font-weight: inherit; +} +input,button,textarea,select {*font-size:100%; +} +/* END RESET */ + +body { + margin-left: 1em; + margin-right: 1em; + font-family: arial, helvetica, geneva, sans-serif; + background-color: #ffffff; margin: 0px; +} + +code, tt { font-family: monospace; font-size: 1.1em; } +span.parameter { font-family:monospace; } +span.parameter:after { content:":"; } +span.types:before { content:"("; } +span.types:after { content:")"; } +.type { font-weight: bold; font-style:italic } + +body, p, td, th { font-size: .95em; line-height: 1.2em;} + +p, ul { margin: 10px 0 0 0px;} + +strong { font-weight: bold;} + +em { font-style: italic;} + +h1 { + font-size: 1.5em; + margin: 20px 0 20px 0; +} +h2, h3, h4 { margin: 15px 0 10px 0; } +h2 { font-size: 1.25em; } +h3 { font-size: 1.15em; } +h4 { font-size: 1.06em; } + +a:link { font-weight: bold; color: #004080; text-decoration: none; } +a:visited { font-weight: bold; color: #006699; text-decoration: none; } +a:link:hover { text-decoration: underline; } + +hr { + color:#cccccc; + background: #00007f; + height: 1px; +} + +blockquote { margin-left: 3em; } + +ul { list-style-type: disc; } + +p.name { + font-family: "Andale Mono", monospace; + padding-top: 1em; +} + +pre { + background-color: rgb(245, 245, 245); + border: 1px solid #C0C0C0; /* silver */ + padding: 10px; + margin: 10px 0 10px 0; + overflow: auto; + font-family: "Andale Mono", monospace; +} + +pre.example { + font-size: .85em; +} + +table.index { border: 1px #00007f; } +table.index td { text-align: left; vertical-align: top; } + +#container { + margin-left: 1em; + margin-right: 1em; + background-color: #f0f0f0; +} + +#product { + text-align: center; + border-bottom: 1px solid #cccccc; + background-color: #ffffff; +} + +#product big { + font-size: 2em; +} + +#main { + background-color: #f0f0f0; + border-left: 2px solid #cccccc; +} + +#navigation { + float: left; + width: 14em; + vertical-align: top; + background-color: #f0f0f0; + overflow: visible; +} + +#navigation h2 { + background-color:#e7e7e7; + font-size:1.1em; + color:#000000; + text-align: left; + padding:0.2em; + border-top:1px solid #dddddd; + border-bottom:1px solid #dddddd; +} + +#navigation ul +{ + font-size:1em; + list-style-type: none; + margin: 1px 1px 10px 1px; +} + +#navigation li { + text-indent: -1em; + display: block; + margin: 3px 0px 0px 22px; +} + +#navigation li li a { + margin: 0px 3px 0px -1em; +} + +#content { + margin-left: 14em; + padding: 1em; + width: 700px; + border-left: 2px solid #cccccc; + border-right: 2px solid #cccccc; + background-color: #ffffff; +} + +#about { + clear: both; + padding: 5px; + border-top: 2px solid #cccccc; + background-color: #ffffff; +} + +@media print { + body { + font: 12pt "Times New Roman", "TimeNR", Times, serif; + } + a { font-weight: bold; color: #004080; text-decoration: underline; } + + #main { + background-color: #ffffff; + border-left: 0px; + } + + #container { + margin-left: 2%; + margin-right: 2%; + background-color: #ffffff; + } + + #content { + padding: 1em; + background-color: #ffffff; + } + + #navigation { + display: none; + } + pre.example { + font-family: "Andale Mono", monospace; + font-size: 10pt; + page-break-inside: avoid; + } +} + +table.module_list { + border-width: 1px; + border-style: solid; + border-color: #cccccc; + border-collapse: collapse; +} +table.module_list td { + border-width: 1px; + padding: 3px; + border-style: solid; + border-color: #cccccc; +} +table.module_list td.name { background-color: #f0f0f0; min-width: 200px; } +table.module_list td.summary { width: 100%; } + + +table.function_list { + border-width: 1px; + border-style: solid; + border-color: #cccccc; + border-collapse: collapse; +} +table.function_list td { + border-width: 1px; + padding: 3px; + border-style: solid; + border-color: #cccccc; +} +table.function_list td.name { background-color: #f0f0f0; min-width: 200px; } +table.function_list td.summary { width: 100%; } + +ul.nowrap { + overflow:auto; + white-space:nowrap; +} + +dl.table dt, dl.function dt {border-top: 1px solid #ccc; padding-top: 1em;} +dl.table dd, dl.function dd {padding-bottom: 1em; margin: 10px 0 0 20px;} +dl.table h3, dl.function h3 {font-size: .95em;} + +/* stop sublists from having initial vertical space */ +ul ul { margin-top: 0px; } +ol ul { margin-top: 0px; } +ol ol { margin-top: 0px; } +ul ol { margin-top: 0px; } + +/* make the target distinct; helps when we're navigating to a function */ +a:target + * { + background-color: #FF9; +} + + +/* styles for prettification of source */ +pre .comment { color: #558817; } +pre .constant { color: #a8660d; } +pre .escape { color: #844631; } +pre .keyword { color: #aa5050; font-weight: bold; } +pre .library { color: #0e7c6b; } +pre .marker { color: #512b1e; background: #fedc56; font-weight: bold; } +pre .string { color: #8080ff; } +pre .number { color: #f8660d; } +pre .operator { color: #2239a8; font-weight: bold; } +pre .preprocessor, pre .prepro { color: #a33243; } +pre .global { color: #800080; } +pre .user-keyword { color: #800080; } +pre .prompt { color: #558817; } +pre .url { color: #272fc2; text-decoration: underline; } + diff --git a/doc/api/modules/djot.ast.html b/doc/api/modules/djot.ast.html new file mode 100644 index 0000000..26ab5e5 --- /dev/null +++ b/doc/api/modules/djot.ast.html @@ -0,0 +1,166 @@ + + + + + Reference + + + + +
    + +
    + +
    +
    +
    + +
    + + + + + + + + +
    + + +

    Module djot.ast

    + +

    + +

    +

    Construct an AST for a djot document.

    + +
    +
    + +
    +
    + + new_node (tag) +
    +
    + Create a new AST node. + + + + + +
    +
    + + add_child (node, child) +
    +
    + Add child as a child of node. + + + + + +
    +
    + + has_children (node) +
    +
    + Returns true if node has children. + + + + + +
    +
    + + new_attributes (tbl) +
    +
    + Returns an attributes object. + deterministic order of iteration) + + +
    +
    + + insert_attribute (attr, key, val) +
    +
    + Insert an attribute into an attributes object. + + + + + +
    +
    + + copy_attributes (target, source) +
    +
    + Copy attributes from source to target. + + + + + +
    +
    + + to_ast (parser, sourcepos) +
    +
    + Create an abstract syntax tree based on an event + stream and references. + + + + + +
    +
    + + render (doc, handle) +
    +
    + Render an AST in human-readable form, with indentation + showing the hierarchy. + + + + + +
    +
    + + +
    +
    +
    +
    +
    + + + diff --git a/doc/api/modules/djot.filter.html b/doc/api/modules/djot.filter.html new file mode 100644 index 0000000..a48a6d9 --- /dev/null +++ b/doc/api/modules/djot.filter.html @@ -0,0 +1,173 @@ + + + + + Reference + + + + +
    + +
    + +
    +
    +
    + +
    + + + + + + + + +
    + + +

    Module djot.filter

    + +

    + +

    +

    Support filters that walk the AST and transform a + document between parsing and rendering, like pandoc Lua filters.

    + +

    This filter uppercases all str elements.

    + +
     return {
+   str = function(e)
+     e.text = e.text:upper()
+    end
+ }
+
    + +

    A filter may define functions for as many different tag types + as it likes. traverse will walk the AST and apply matching + functions to each node.

    + +

    To load a filter:

    + +
     local filter = require_filter(path)
+
    + +

    or

    + +
     local filter = load_filter(string)
+
    + +

    By default filters do a bottom-up traversal; that is, the + filter for a node is run after its children have been processed. + It is possible to do a top-down travel, though, and even + to run separate actions on entering a node (before processing the + children) and on exiting (after processing the children). To do + this, associate the node's tag with a table containing enter and/or + exit functions. The following filter will capitalize text + that is nested inside emphasis, but not other text:

    + +
     local capitalize = 0
+ return {
+    emph = {
+      enter = function(e)
+        capitalize = capitalize + 1
+      end,
+      exit = function(e)
+        capitalize = capitalize - 1
+      end,
+    },
+    str = function(e)
+      if capitalize > 0 then
+        e.text = e.text:upper()
+       end
+    end
+ }
+
    + +

    For a top-down traversal, you'd just use the enter functions. + If the tag is associated directly with a function, as in the + first example above, it is treated as an exit function.

    + +

    It is possible to inhibit traversal into the children of a node, + by having the enter function return the value true (or any truish + value, say 'stop'). This can be used, for example, to prevent + the contents of a footnote from being processed:

    + +
     return {
+   footnote = {
+     enter = function(e)
+       return true
+     end
+    }
+ }
+
    + +

    A single filter may return a table with multiple tables, which will be + applied sequentially.

    + +
    +
    + +
    +
    + + apply_filter (node, filter) +
    +
    + Apply a filter to a document. + + + + + +
    +
    + + require_filter (fp) +
    +
    + Returns a table containing the filter defined in fp. + fp will be sought using require, so it may occur anywhere + on the LUA_PATH, or in the working directory. On error, + returns nil and an error message. + + +
    +
    + + load_filter (s) +
    +
    + Load filter from a string, which should have the + form `return { ... + }`. On error, return nil and an + error message. + + +
    +
    + + +
    +
    +
    +
    +
    + + + diff --git a/doc/api/modules/djot.html b/doc/api/modules/djot.html new file mode 100644 index 0000000..e6992b1 --- /dev/null +++ b/doc/api/modules/djot.html @@ -0,0 +1,173 @@ + + + + + Reference + + + + +
    + +
    + +
    +
    +
    + +
    + + + + + + + + +
    + + +

    Module djot

    + +

    + +

    +

    Parse and render djot light markup format. See https://djot.net.

    + +
    +
    + +
    +
    + + parse (input, sourcepos, warn) +
    +
    + Parse a djot text and construct an abstract syntax tree (AST) + representing the document. + object with pos and message fields. + + +
    +
    + + parse_events (input, warn) +
    +
    + Parses a djot text and returns an iterator over events, consisting + of a start position (bytes), and an position (bytes), and an + annotation. + + +

    object with pos and message fields.

    + +
     for startpos, endpos, annotation in djot.parse_events("hello *world") do
+ ...
+ end
+
    + + + +
    +
    + + render_ast_pretty (doc) +
    +
    + Render a document's AST in human-readable form. + + + + + +
    +
    + + render_ast_json (doc) +
    +
    + Render a document's AST in JSON. + + + + + +
    +
    + + render_html (doc) +
    +
    + Render a document as HTML. + + + + + +
    +
    + + render_event (startpos, endpos, annotation) +
    +
    + Render an event as a JSON array. + + + + + +
    +
    + + parse_and_render_events (input, warn) +
    +
    + Parse a document and render as a JSON array of events. + an object with fields 'message' and 'pos' + + +
    +
    +
    +
    + + version +
    +
    + djot version (string) + + + + + +
    +
    + + +
    +
    +
    +
    +
    + + + diff --git a/doc/api/modules/djot.json.html b/doc/api/modules/djot.json.html new file mode 100644 index 0000000..a4236e0 --- /dev/null +++ b/doc/api/modules/djot.json.html @@ -0,0 +1,68 @@ + + + + + Reference + + + + +
    + +
    + +
    +
    +
    + + +
    + + + + + + +
    + +

    Module djot.json

    +

    Encode

    +

    + +

    + + + +
    +
    + + + + +
    +
    +
    +generated by LDoc 1.4.6 +Last updated 2022-12-01 22:10:28 +
    +
    + + diff --git a/doc/code-examples.lua b/doc/code-examples.lua new file mode 100644 index 0000000..73814ff --- /dev/null +++ b/doc/code-examples.lua @@ -0,0 +1,15 @@ +local djot = require("djot") + +function renderdjot(txt) + return djot.render_html(djot.parse(txt)) +end + +function CodeBlock(el) + local rendered = renderdjot(el.text) + return { + pandoc.Div( + { pandoc.Div({pandoc.CodeBlock(el.text)}, {class="djot"}), + pandoc.Div({pandoc.CodeBlock(rendered)}, {class="html"}) + }, {class="example"}) + } +end diff --git a/doc/djot.1 b/doc/djot.1 new file mode 100644 index 0000000..66c7848 --- /dev/null +++ b/doc/djot.1 @@ -0,0 +1,172 @@ +.\" Automatically generated by Pandoc 2.19.2 +.\" +.\" Define V font for inline verbatim, using C font in formats +.\" that render this, and otherwise B font. +.ie "\f[CB]x\f[]"x" \{\ +. ftr V B +. ftr VI BI +. ftr VB B +. ftr VBI BI +.\} +.el \{\ +. ftr V CR +. ftr VI CI +. ftr VB CB +. ftr VBI CBI +.\} +.TH "DJOT" "1" "" "djot 0.2.0" "" +.hy +.SH NAME +.PP +djot \[en] converts djot markup. +.SH SYNOPSIS +.PP +djot options [file..] +.SH DESCRIPTION +.PP +djot is a command-line parser for djot markup (https://djot.net). +It can produce +.IP \[bu] 2 +an HMTL document (default behavior) +.IP \[bu] 2 +a stream of annotated tokens with byte offsets (\f[V]--matches\f[R]) +.IP \[bu] 2 +an AST in either human-readable or JSON form (\f[V]--ast\f[R]). +.SH OPTIONS +.TP +\f[V]--matches, -m\f[R] +Show matches (annotated tokens with source positions). +.TP +\f[V]--ast\f[R], \f[V]-a\f[R] +Produce and render an abstract syntax tree. +.TP +\f[V]--json\f[R], \f[V]-j\f[R] +Use machine-readable JSON format when used with \f[V]--matches\f[R] or +\f[V]--ast\f[R]. +.TP +\f[V]--sourcepos\f[R], \f[V]-p\f[R] +Include source positions in the AST or HTML document. +.TP +\f[V]--filter\f[R] \f[I]FILE\f[R], \f[V]-f\f[R] \f[I]FILE\f[R] +Run the filter defined in \f[I]FILE\f[R] on the AST between parsing and +rendering. +The \f[V]--filter\f[R] option may be used multiple times; filters will +be applied in the order specified on the command line. +See FILTERS below for a description of filters. +.TP +\f[V]--verbose\f[R], \f[V]-v\f[R] +Verbose output, including warnings. +.TP +\f[V]--version\f[R] +Print the djot version. +.TP +\f[V]--help\f[R], \f[V]-h\f[R] +Print usage information. +.SH FILTERS +.PP +Filters are small Lua programs that modify the parsed document prior to +rendering. +Here is an example of a filter that capitalizes all the content text in +a document: +.IP +.nf +\f[C] +return { + str = function(e) + e.text = e.text:upper() + end +} +\f[R] +.fi +.PP +Save this as \f[V]caps.lua\f[R] use tell djot to use it using +.IP +.nf +\f[C] +djot --filter caps input.djot +\f[R] +.fi +.PP +Note that djot will search your LUA_PATH for the filter if it is not +found in the working directory, so you can in principle install filters +using luarocks. +.PP +Here\[cq]s a filter that prints a list of all the URLs you link to in a +document. +This filter doesn\[cq]t alter the document at all; it just prints the +list to stderr. +.IP +.nf +\f[C] +return { + link = function(el) + io.stderr:write(el.destination .. \[dq]\[rs]n\[dq]) + end +} +\f[R] +.fi +.PP +A filter walks the document\[cq]s abstract syntax tree, applying +functions to like-tagged nodes, so you will want to get familiar with +how djot\[cq]s AST is designed. +The easiest way to do this is to use \f[V]djot --ast\f[R]. +.PP +By default filters do a bottom-up traversal; that is, the filter for a +node is run after its children have been processed. +It is possible to do a top-down travel, though, and even to run separate +actions on entering a node (before processing the children) and on +exiting (after processing the children). +To do this, associate the node\[cq]s tag with a table containing +\f[V]enter\f[R] and/or \f[V]exit\f[R] functions. +The following filter will capitalize text that is nested inside +emphasis, but not other text: +.IP +.nf +\f[C] +local capitalize = 0 +return { + emph = { + enter = function(e) + capitalize = capitalize + 1 + end, + exit = function(e) + capitalize = capitalize - 1 + end, + }, + str = function(e) + if capitalize > 0 then + e.text = e.text:upper() + end + end +} +\f[R] +.fi +.PP +For a top-down traversal, you\[cq]d just use the \f[V]enter\f[R] +functions. +If the tag is associated directly with a function, as in the first +example above, it is treated as an \[ga]exit\[cq] function. +.PP +It is possible to inhibit traversal into the children of a node, by +having the \f[V]enter\f[R] function return the value true (or any truish +value, say \f[V]\[dq]stop\[dq]\f[R]). +This can be used, for example, to prevent the contents of a footnote +from being processed: +.IP +.nf +\f[C] +return { + footnote = { + enter = function(e) + return true + end + } +} +\f[R] +.fi +.PP +A single filter may return a table with multiple tables, which will be +applied sequentially. +.SH AUTHORS +.PP +John MacFarlane (). diff --git a/doc/djot.md b/doc/djot.md new file mode 100644 index 0000000..61e3c96 --- /dev/null +++ b/doc/djot.md @@ -0,0 +1,151 @@ +# NAME + +djot -- converts djot markup. + +# SYNOPSIS + +djot [options] [file..] + +# DESCRIPTION + +djot is a command-line parser for [djot markup](https://djot.net). +It can produce + +- an HMTL document (default behavior) +- a stream of annotated tokens with byte offsets (`--matches`) +- an AST in either human-readable or JSON form (`--ast`). + +# OPTIONS + +`--matches, -m` + +: Show matches (annotated tokens with source positions). + +`--ast`, `-a` + +: Produce and render an abstract syntax tree. + +`--json`, `-j` + +: Use machine-readable JSON format when used with `--matches` + or `--ast`. + +`--sourcepos`, `-p` + +: Include source positions in the AST or HTML document. + +`--filter` *FILE*, `-f` *FILE* + +: Run the filter defined in *FILE* on the AST between parsing + and rendering. The `--filter` option may be used multiple + times; filters will be applied in the order specified on the + command line. See [FILTERS][] below for a description of + filters. + +`--verbose`, `-v` + +: Verbose output, including warnings. + +`--version` + +: Print the djot version. + +`--help`, `-h` + +: Print usage information. + +# FILTERS + +Filters are small Lua programs that modify the parsed document +prior to rendering. Here is an example of a filter that +capitalizes all the content text in a document: + +``` +return { + str = function(e) + e.text = e.text:upper() + end +} +``` + +Save this as `caps.lua` use tell djot to use it using + +``` +djot --filter caps input.djot +``` + +Note that djot will search your LUA_PATH for the filter if +it is not found in the working directory, so you can in +principle install filters using luarocks. + +Here's a filter that prints a list of all the URLs you +link to in a document. This filter doesn't alter the +document at all; it just prints the list to stderr. + +``` +return { + link = function(el) + io.stderr:write(el.destination .. "\n") + end +} +``` + +A filter walks the document's abstract syntax tree, applying +functions to like-tagged nodes, so you will want to get familiar +with how djot's AST is designed. The easiest way to do this is +to use `djot --ast`. + +By default filters do a bottom-up traversal; that is, the +filter for a node is run after its children have been processed. +It is possible to do a top-down travel, though, and even +to run separate actions on entering a node (before processing the +children) and on exiting (after processing the children). To do +this, associate the node's tag with a table containing `enter` and/or +`exit` functions. The following filter will capitalize text +that is nested inside emphasis, but not other text: + +``` +local capitalize = 0 +return { + emph = { + enter = function(e) + capitalize = capitalize + 1 + end, + exit = function(e) + capitalize = capitalize - 1 + end, + }, + str = function(e) + if capitalize > 0 then + e.text = e.text:upper() + end + end +} +``` + +For a top-down traversal, you'd just use the `enter` functions. +If the tag is associated directly with a function, as in the +first example above, it is treated as an `exit' function. + +It is possible to inhibit traversal into the children of a node, +by having the `enter` function return the value true (or any truish +value, say `"stop"`). This can be used, for example, to prevent +the contents of a footnote from being processed: + +``` +return { + footnote = { + enter = function(e) + return true + end + } +} +``` + +A single filter may return a table with multiple tables, which will be +applied sequentially. + +# AUTHORS + +John MacFarlane (). + diff --git a/full-coverage.lua b/full-coverage.lua new file mode 100644 index 0000000..904b8ba --- /dev/null +++ b/full-coverage.lua @@ -0,0 +1,77 @@ +package.path = "./?.lua;" .. package.path +-- Full coverage test +-- +-- This test is similar to fuzz.lua, but, rather than generating +-- random strings, we exhaustively enumerate short strings. + +local djot = require("djot") +local to_html = function(s) + local doc = djot.parse(s) + return djot.render_html(doc) +end + +local function combinations(alpha, n) + if n == 0 then + return {""} + end + if alpha == "" then + return {} + end + local res = {} + local first = alpha:sub(1, 1) + local rest = alpha:sub(2) + for _, s in ipairs(combinations(rest, n)) do + res[#res + 1] = s + end + for _, s in ipairs(combinations(rest, n - 1)) do + res[#res + 1] = first .. s + end + return res +end + + +local n = 4 -- We select n interesting characters +local m = 6 -- and generate every string of length m. +local swarm = combinations(" -*|[]{}()_`:ai\n", n) + +local iter = 0 +for _, alphabet in ipairs(swarm) do + iter = iter + 1 + if iter % 10 == 0 then + print(iter, "of", #swarm) + end + + -- Tricky bit: we essentially want to write m nested + -- 'for i=1,m' loops. We can't do that, so instead we + -- track `m` loop variables in `ii` manually. + -- + -- That is, `ii` is "vector of `i`s". + local ii = {} + for i=1,m do + ii[i] = 1 + end + + local done = false + while not done do + local s = "" + for i=1,m do + s = s .. alphabet:sub(ii[i], ii[i]) + end + + to_html(s) + + -- Increment the innermost index, reset others to 1. + done = true + for i=m,1,-1 do + if ii[i] ~= #alphabet then + ii[i] = ii[i] + 1 + for j=i+1,m do + ii[j] = 1 + end + done = false + break + end + end + + end +end diff --git a/fuzz.lua b/fuzz.lua new file mode 100644 index 0000000..b952b47 --- /dev/null +++ b/fuzz.lua @@ -0,0 +1,95 @@ +local djot = require("djot") +local to_html = function(s) + local doc = djot.parse(s) + return djot.render_html(doc) +end +local signal = require("posix.signal") +local resource = require("posix.sys.resource") +local times = require 'posix.sys.times'.times + +-- if you want to be able to interrupt stuck fuzz tests. + +math.randomseed(os.time()) + +local MAXLINES = 5 +local MAXLENGTH = 5 +local NUMTESTS = arg[1] or 200000 + +local activechars = { + '\t', ' ', '[', ']', '1', '2', 'a', 'b', + 'A', 'B', 'I', 'V', 'i', 'v', '.', ')', '(', + '{', '}', '=', '+', '_', '-', '*', '!', '>', + '<', '`', '~' +} + +local function randomstring() + local numlines = math.random(1,MAXLINES) + local buffer = {} + for j=1,numlines do + -- -1 to privilege blank lines + local res = "" + local len = math.random(-1,MAXLENGTH) + if len < 0 then len = 0 end + for i=1,len do + local charclass = math.random(1, 4) + if charclass < 4 then + res = res .. activechars[math.random(1, #activechars)] + elseif utf8 then + res = res .. utf8.char(math.random(1, 200)) + else + res = res .. string.char(math.random(1, 127)) + end + end + buffer[#buffer + 1] = res + end + local res = table.concat(buffer, "\n") + return res +end + +local failures = 0 + +io.stderr:write("Running fuzz tests: ") +for i=1,NUMTESTS do + local s = randomstring() + if i % 1000 == 0 then + io.stderr:write("."); + end + local ok, err = pcall(function () + signal.signal(signal.SIGINT, function(signum) + io.stderr:write(string.format("\nInterrupted processing on input %q\n", s)) + io.stderr:flush() + os.exit(128 + signum) + end) + return to_html(s) + end) + if not ok then + -- try to minimize case + local minimal = false + local trim_from_front = true + while not minimal do + local s2 + if trim_from_front then + s2 = string.sub(s, 2, -1) + else + s2 = string.sub(s, 1, -2) + end + local ok2, _ = pcall(function () return to_html(s2) end) + if ok2 then + if trim_from_front then + trim_from_front = false + else + minimal = true + end + else + s = s2 + end + end + failures = failures + 1 + io.stderr:write(string.format("\nFAILURE on\n%q\n", s)) + io.stderr:write(err .. "\n") + end +end + +io.stderr:write("\n") +os.exit(failures) + diff --git a/ldoc.ltp b/ldoc.ltp new file mode 100644 index 0000000..24d786a --- /dev/null +++ b/ldoc.ltp @@ -0,0 +1,135 @@ + + + + + $(ldoc.title) + + + + +
    + +
    + +
    +
    +
    + +
    + +# local iter = ldoc.modules.iter +# local M = ldoc.markup + + + +# local function no_spaces(s) return s:gsub('%s','_') end +# local function use_li(ls) +# if #ls > 1 then return '
  • ','
    • ' else return '','' end +# end +# local function display_name(item) +# if item.type == 'function' then return item.name..' '..item.args +# else return item.name end +# end + + + + +
    + +# if module then + +

    Module $(module.name)

    + +

    $(M(module.summary))

    +

    $(M(module.description))

    + +
    +
    + +# --- currently works for both Functions and Tables. The params field either contains +# --- function parameters or table fields. +# for kind, items in module.kinds() do +
    +# for item in items() do +
    + + $(display_name(item)) +
    +
    + $(M(item.summary)) + $(M(item.description)) + +# if item.usage then +# local li,il = use_li(item.usage) +

    Usage:

    +
      +# for usage in iter(item.usage) do + $(li)
      $(usage)
      $(il) +# end -- for +
    +# end -- if usage + +# if item.see then +# local li,il = use_li(item.see) +

    see also:

    +
      +# for see in iter(item.see) do + $(li)$(see.label)$(il) +# end -- for +
    +# end -- if see +
    +# end -- for items +
    +# end -- for kinds + +# else -- if module + +# if ldoc.description then +

    $(M(ldoc.description))

    +# end + +# for kind, mods in ldoc.kinds() do +

    $(kind)

    +# kind = kind:lower() +# for m in mods() do + + + + + +# end -- for modules +
    $(m.name)$(M(m.summary))
    +# end -- for kinds +# end -- if module + +
    +
    +
    +
    +
    + + + diff --git a/lua51.nix b/lua51.nix new file mode 100755 index 0000000..d6062e0 --- /dev/null +++ b/lua51.nix @@ -0,0 +1,14 @@ +{ pkgs ? import {} }: +let + myLua = pkgs.lua5_1; + myLuaWithPackages = myLua.withPackages(ps: with ps; [ + luaposix + ]); +in +pkgs.mkShell { + packages = [ pkgs.hyperfine pkgs.perl pkgs.luarocks myLuaWithPackages + ]; + shellHook = '' + luarocks config lua_version 5.1 + ''; +} diff --git a/luajit.nix b/luajit.nix new file mode 100755 index 0000000..18e0e3a --- /dev/null +++ b/luajit.nix @@ -0,0 +1,14 @@ +{ pkgs ? import {} }: +let + myLua = pkgs.luajit; + myLuaWithPackages = myLua.withPackages(ps: with ps; [ + luaposix + ]); +in +pkgs.mkShell { + packages = [ pkgs.hyperfine pkgs.perl pkgs.luarocks myLuaWithPackages + ]; + shellHook = '' + luarocks config lua_version 5.1 + ''; +} diff --git a/pathological_tests.lua b/pathological_tests.lua new file mode 100644 index 0000000..5db87e5 --- /dev/null +++ b/pathological_tests.lua @@ -0,0 +1,58 @@ +local djot = require("djot") + +local n = 500 + +local deeplynested = {} +for i = 1,n do + deeplynested[#deeplynested + 1] = string.rep(" ", i) .. "* a\n" +end + +local backticks = {} +for i = 1, 5 * n do + backticks[#backticks + 1] = "e" .. string.rep("`", i) +end + +local tests = { + ["nested strong emph"] = + string.rep("_a *a ", 65*n) .. "b" .. string.rep(" a* a_", 65*n), + ["many emph closers with no openers"] = + string.rep("a_ ", 65*n), + ["many emph openers with no closers"] = + string.rep("_a ", 65*n), + ["many link closers with no openers"] = + string.rep("a]", 65*n), + ["many link openers with no closers"] = + string.rep("[a", 65*n), + ["mismatched openers and closers"] = + string.rep("*a_ ", 50*n), + ["issue cmark#389"] = + string.rep("*a ", 20*n) .. string.rep("_a*_ ", 20*n), + ["openers and closers multiple of 3"] = + "a**b" .. string.rep("8* ", 50 * n), + ["link openers and emph closers"] = + string.rep("[ a_", 50 * n), + ["pattern [ (]( repeated"] = + string.rep("[ (](", 80 * n), + ["nested brackets"] = + string.rep("[", 50 * n) .. "a" .. string.rep("]", 50*n), + ["nested block quotes"] = + string.rep("> ", 50*n) .. "a", + ["deeply nested lists"] = + table.concat(deeplynested), + ["backticks"] = + table.concat(backticks), + ["unclosed links"] = + string.rep("[a](= 5.1" +} +test_dependencies = { + "lua >= 5.1", +} +build = { + type = "builtin", + modules = { + ["djot"] = "djot.lua", + ["djot.attributes"] = "djot/attributes.lua", + ["djot.inline"] = "djot/inline.lua", + ["djot.block"] = "djot/block.lua", + ["djot.ast"] = "djot/ast.lua", + ["djot.html"] = "djot/html.lua", + ["djot.filter"] = "djot/filter.lua", + ["djot.json"] = "djot/json.lua", + }, + install = { + bin = { + ["djot"] = "bin/main.lua", + } + } +} diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..c21cf92 --- /dev/null +++ b/run.sh @@ -0,0 +1,2 @@ +#!/bin/sh +LUA_PATH="./?.lua;$LUA_PATH" lua bin/main.lua "$@" diff --git a/test.lua b/test.lua new file mode 100644 index 0000000..47c9ef9 --- /dev/null +++ b/test.lua @@ -0,0 +1,230 @@ +-- run tests +package.path = "./?.lua;" .. package.path +local djot = require("djot") + +local testcases = { + "attributes.test", + "blockquote.test", + "code_blocks.test", + "definition_lists.test", + "symbol.test", + "emphasis.test", + "escapes.test", + "fenced_divs.test", + "filters.test", + "footnotes.test", + "headings.test", + "insert_delete_mark.test", + "links_and_images.test", + "lists.test", + "math.test", + "para.test", + "raw.test", + "regression.test", + "smart.test", + "spans.test", + "sourcepos.test", + "super_subscript.test", + "tables.test", + "task_lists.test", + "thematic_breaks.test", + "verbatim.test" +} + +local opts = {} +local i=1 +while i <= #arg do + local thisarg = arg[i] + if string.find(thisarg, "^%-") then + if thisarg == "-v" then + opts.verbose = true + elseif thisarg == "-p" then + opts.pattern = true + elseif thisarg == "--accept" then + opts.accept = true + end + elseif opts.pattern == true then + opts.pattern = thisarg + end + i = i + 1 +end + +local Tests = {} + +function Tests:new() + local contents = { + passed = 0, + failed = 0, + errors = 0, + accept = opts.accept, + verbose = opts.verbose + } + setmetatable(contents, Tests) + Tests.__index = Tests + return contents +end + +function Tests:do_test(test) + if self.verbose then + io.write(string.format("Testing %s at linen %d\n", test.file, test.linenum)) + end + local sourcepos = false + if test.options:match("p") then + sourcepos = true + end + local actual = "" + if test.options:match("m") then + actual = actual .. djot.parse_and_render_events(test.input) + else + local doc = djot.parse(test.input, sourcepos) + for _,filt in ipairs(test.filters) do + local f, err = djot.filter.load_filter(filt) + if not f then + error(err) + end + djot.filter.apply_filter(doc, f) + end + if test.options:match("a") then + actual = actual .. djot.render_ast_pretty(doc) + else -- match 'h' or empty + actual = actual .. djot.render_html(doc) + end + end + if self.accept then + test.output = actual + end + if actual == test.output then + self.passed = self.passed + 1 + return true + else + io.write(string.format("FAILED at %s line %d\n", test.file, test.linenum)) + io.write(string.format("--- INPUT -------------------------------------\n%s--- EXPECTED ----------------------------------\n%s--- GOT ---------------------------------------\n%s-----------------------------------------------\n\n", test.input, test.output, actual)) + self.failed = self.failed + 1 + return false + end +end + +local function read_tests(file) + local f = io.open("test/" .. file,"r") + assert(f ~= nil, "File " .. file .. " cannot be read") + local line + local linenum = 0 + return function() + while true do + local inp = "" + local out = "" + line = f:read() + local pretext = {} + linenum = linenum + 1 + while line and not line:match("^```") do + pretext[#pretext + 1] = line + line = f:read() + linenum = linenum + 1 + end + local testlinenum = linenum + if not line then + break + end + local ticks, options = line:match("^(`+)%s*(.*)") + + -- parse input + line = f:read() + linenum = linenum + 1 + while not line:match("^[%.%!]$") do + inp = inp .. line .. "\n" + line = f:read() + linenum = linenum + 1 + end + + local filters = {} + while line == "!" do -- parse filter + line = f:read() + linenum = linenum + 1 + local filt = "" + while not line:match("^[%.%!]$") do + filt = filt .. line .. "\n" + line = f:read() + linenum = linenum + 1 + end + table.insert(filters, filt) + end + + -- parse output + line = f:read() + linenum = linenum + 1 + while not line:match("^" .. ticks) do + out = out .. line .. "\n" + line = f:read() + linenum = linenum + 1 + end + + return { file = file, + linenum = testlinenum, + pretext = table.concat(pretext, "\n"), + options = options, + filters = filters, + input = inp, + output = out } + end + end +end + +function Tests:do_tests(file) + local tests = {} + for test in read_tests(file) do + tests[#tests + 1] = test + local ok, err = pcall(function() + self:do_test(test) + end) + if not ok then + io.stderr:write(string.format("Error running test %s line %d:\n%s\n", + test.file, test.linenum, err)) + self.errors = self.errors + 1 + end + end + if self.accept then -- rewrite file + local fh = io.open("test/" .. file, "w") + for idx,test in ipairs(tests) do + local numticks = 3 + string.gsub(test.input .. test.output, "(````*)", + function(x) + if #x >= numticks then + numticks = #x + 1 + end + end) + local ticks = string.rep("`", numticks) + local pretext = test.pretext + if #pretext > 0 or idx > 1 then + pretext = pretext .. "\n" + end + + fh:write(string.format("%s%s%s\n%s", + pretext, + ticks, + (test.options == "" and "") or " " .. test.options, + test.input)) + for _,f in ipairs(test.filters) do + fh:write(string.format("!\n%s", f)) + end + fh:write(string.format(".\n%s%s\n", test.output, ticks)) + end + fh:close() + end +end + +local tests = Tests:new() +local starttime = os.clock() +for _,case in ipairs(testcases) do + if not opts.pattern or string.find(case, opts.pattern) then + tests:do_tests(case) + end +end +local endtime = os.clock() + +io.write(string.format("%d tests completed in %0.3f s\n", + tests.passed + tests.failed + tests.errors, endtime - starttime)) +io.write(string.format("PASSED: %4d\n", tests.passed)) +io.write(string.format("FAILED: %4d\n", tests.failed)) +io.write(string.format("ERRORS: %4d\n", tests.errors)) +os.exit(tests.failed + tests.errors) + diff --git a/test/attributes.test b/test/attributes.test new file mode 100644 index 0000000..f84151c --- /dev/null +++ b/test/attributes.test @@ -0,0 +1,298 @@ +An inline attribute allies to the preceding element, which might +be complex (span, emphasis, link) or a simple word (defined as a +sequence of non-ASCII-whitespace characters). +``` +foo привет{.ru} +. +

    foo привет

    +``` + +``` +(some text){.attr} +. +

    (some text)

    +``` + +``` +[some text]{.attr} +. +

    some text

    +``` + +Ensure that emphasis that starts before the attribute can still close, +even if the attribute contains a potential closer. + +``` +a *b{#id key="*"}* +. +

    a b

    +``` + +``` +a *b{#id key="*"}o +. +

    a *bo

    +``` + +Don't mind braces in quotes: + +``` +hi{key="{#hi"} +. +

    hi

    +``` + +Don't allow attributes to start when we're parsing a potential +attribute. + +``` +hi\{key="abc{#hi}" +. +

    hi{key=“abc”

    +``` + +``` +hi{key="\"#hi"} +. +

    hi

    +``` + +``` +hi{key="hi\"#hi"} +. +

    hi

    +``` + +Line break: + +``` +hi{#id .class +key="value"} +. +

    hi

    +``` + +Here there is nothing for the attribute to attach to: + +``` +{#id} at beginning +. +

    at beginning

    +``` + +``` +After {#id} space +{.class} +. +

    After space +

    +``` + +Block attributes come before the block, on a line by themselves. + +``` +{#id .class} +A paragraph +. +

    A paragraph

    +``` + +Use indentation if you need to continue the attributes over a line break. + +``` +{#id .class + style="color:red"} +A paragraph +. +

    A paragraph

    +``` + +If the attribute block can't be parsed as attributes, it will be +parsed as a regular paragraph: + +``` +{#id .cla*ss* +. +

    {#id .class

    +``` + +You can use consecutive attribute blocks. +In case of conflict, later values take precedence over earlier ones, +but classes accumulate: + +``` +{#id} +{key=val} +{.foo .bar} +{key=val2} +{.baz} +{#id2} +Okay +. +

    Okay

    +``` + +Attributes on different kinds of blocks: + +``` +{#id} +> Block quote +. +
    +

    Block quote

    +
    +``` + +``` +{#id} +# Heading +. +
    +

    Heading

    +
    +``` + +``` +{.blue} +- - - - - +. +
    +``` + +```` +{highlight=3} +``` ruby +x = 3 +``` +. +
    x = 3
+
    +```` + +``` +{.special} +1. one +2. two +. +
      +
    1. +one +
      2. +
    2. +two +
      3. +
    +``` + +``` +> {.foo} +> > {.bar} +> > nested +. +
    +
    +

    nested

    +
    +
    +``` + +Comments start at a `%` character +(not in quotes) and end with another `%`. +These can be used to comment up an attribute +list or without any real attributes. + +``` +foo{#ident % this is a comment % .class} +. +

    foo

    +``` + +In block-level comment, subsequent lines must +be indented, as with attributes: + +``` +{% This is a comment before a + block-level item. %} +Paragraph. +. +

    Paragraph.

    +``` + +Inline attributes can be empty: + +``` +hi{} +. +

    hi

    +``` + +Block attributes can be empty: + +``` +{} +hi +. +

    hi

    +``` + +Non-attributes: + +``` +text{a=x + +hello +. +

    text{a=x

    +

    hello

    +``` + +``` +{a=x +hello +. +

    {a=x +hello

    +``` + +``` +text{a=x +# non-heading +. +

    text{a=x +# non-heading

    +``` + +``` +{a=x +# non-heading +. +

    {a=x +# non-heading

    +``` + +``` a +{ + attr="long + value + spanning + multiple + lines" + } +> a +. +doc + blockquote attr="long value spanning multiple lines" + para + str text="a" +``` + +``` a +> {key="bar +> a\$bim"} +> ou +. +doc + blockquote + para key="bar a$bim" + str text="ou" +``` diff --git a/test/blockquote.test b/test/blockquote.test new file mode 100644 index 0000000..988e935 --- /dev/null +++ b/test/blockquote.test @@ -0,0 +1,152 @@ +``` +> Basic +> block _quote_. +. +
    +

    Basic +block quote.

    +
    +``` + +``` +> Lazy +block _quote_. +. +
    +

    Lazy +block quote.

    +
    +``` + +``` +> block +> +> quote +. +
    +

    block

    +

    quote

    +
    +``` + +``` +> block + +> quote +. +
    +

    block

    +
    +
    +

    quote

    +
    +``` + +``` +> > > nested +. +
    +
    +
    +

    nested

    +
    +
    +
    +``` + +``` +> > > nested +lazy +. +
    +
    +
    +

    nested +lazy

    +
    +
    +
    +``` + +``` +> > > nested +> lazy +. +
    +
    +
    +

    nested +lazy

    +
    +
    +
    +``` + +``` +> nested +> +> > more +. +
    +

    nested

    +
    +

    more

    +
    +
    +``` + +``` +>not blockquote +. +

    >not blockquote

    +``` + +``` +>> not blockquote +. +

    >> not blockquote

    +``` + +``` +> +. +
    +
    +``` + +``` +> # Heading +. +
    +

    Heading

    +
    +``` + +``` +> hi +>there +. +
    +

    hi +>there

    +
    +``` + +``` +aaa +> bbb +. +

    aaa +> bbb

    +``` + +``` +aaa + +> bbb +. +

    aaa

    +
    +

    bbb

    +
    +``` diff --git a/test/code_blocks.test b/test/code_blocks.test new file mode 100644 index 0000000..54f71b7 --- /dev/null +++ b/test/code_blocks.test @@ -0,0 +1,65 @@ +``` +~~~ +code + block +~~~ +. +
    code
+  block
+
    +``` + +```` +``` python +x = y + 3 +``` +. +
    x = y + 3
+
    +```` + +```` + ``` python + if true: + x = 3 + ``` +. +
    if true:
+  x = 3
+
    +```` + +```` +``` not a code block ``` +. +

    not a code block

    +```` + +```` +``` not a code block +. +

    not a code block

    +```` + +```` +``` +hi +``` +``` +two +``` +. +
    hi
+
    +
    two
+
    +```` + +Empty code block: + +```` +``` +``` +. +
    +```` diff --git a/test/definition_lists.test b/test/definition_lists.test new file mode 100644 index 0000000..d2d0eeb --- /dev/null +++ b/test/definition_lists.test @@ -0,0 +1,93 @@ +Definition lists are just like ordinary bullet lists, but with +`:` as the marker instead of `-`, `+`, or `*`. The first +paragraph of the list item is interpreted as the term, and +the rest as the definition. + +``` +: apple + + red fruit +: banana + + yellow fruit +. +
    +
    apple
    +
    +

    red fruit

    +
    +
    banana
    +
    +

    yellow fruit

    +
    +
    +``` + +Loose: + +``` +: apple + + red fruit + +: banana + + yellow fruit +. +
    +
    apple
    +
    +

    red fruit

    +
    +
    banana
    +
    +

    yellow fruit

    +
    +
    +``` + +``` +: apple + fruit + + Paragraph one + + Paragraph two + + - sub + - list + +: orange +. +
    +
    apple +fruit
    +
    +

    Paragraph one

    +

    Paragraph two

    +
      +
    • +sub +
      • +
    • +list +
      • +
    +
    +
    orange
    +
    +``` + +```` +: ``` + ok + ``` +. +
    +
    +
    +
    ok
+
    +
    +
    +```` diff --git a/test/emphasis.test b/test/emphasis.test new file mode 100644 index 0000000..f765245 --- /dev/null +++ b/test/emphasis.test @@ -0,0 +1,239 @@ +``` +*foo bar* +. +

    foo bar

    +``` + +``` +a* foo bar* +. +

    a* foo bar*

    +``` + +``` +*foo bar * +. +

    *foo bar *

    +``` + +Unicode spaces don't block emphasis. + +``` +* a * +. +

     a 

    +``` + +Intraword: + +``` +foo*bar*baz +. +

    foobarbaz

    +``` + +``` +_foo bar_ +. +

    foo bar

    +``` + +``` +_ foo bar_ +. +

    _ foo bar_

    +``` + +``` +_foo bar _ +. +

    _foo bar _

    +``` + +Unicode spaces don't block emphasis. + +``` +_ a _ +. +

     a 

    +``` + +Intraword: + +``` +foo_bar_baz +. +

    foobarbaz

    +``` + +``` +aa_"bb"_cc +. +

    aa“bb”cc

    +``` + +``` +*foo_ +. +

    *foo_

    +``` + +``` +_foo* +. +

    _foo*

    +``` + +A line ending counts as whitespace: + +``` +_foo bar +_ +. +

    _foo bar +_

    +``` + +So does a tab: + +``` +_ a_ +. +

    _ a_

    +``` + +This one is different from commonmark: + +``` +_(_foo_)_ +. +

    (foo)

    +``` + +But you can force the second `_` to be an opener +using the marker `{`. + +``` +_({_foo_})_ +. +

    (foo)

    +``` + +Note that an explicitly marked opener can only be closed +by an explicitly marked closer, and a non-marked opener +can only be closed by a non-marked closer: + +``` +{_ x_ _} _x_} +. +

    x_ _x_}

    +``` + + +``` +_(*foo*)_ +. +

    (foo)

    +``` + +Overlapping scopes (first to close wins): + +``` +_foo *bar_ baz* +. +

    foo *bar baz*

    +``` + +Over line break: + +``` +_foo +bar_ +. +

    foo +bar

    +``` + +Inline content allowed: + +``` +*foo [link](url) `*`* +. +

    foo link *

    +``` + +Can't emph an underscore: + +``` +___ +. +

    ___

    +``` + +Unless you escape it: + +``` +_\__ +. +

    _

    +``` + +No empty emph: + +``` +__ +. +

    __

    +``` + +``` +_}b_ +. +

    _}b_

    +``` + +``` +_\}b_ +. +

    }b

    +``` + +``` +_ab\_c_ +. +

    ab_c

    +``` + +``` +*****a***** +. +

    a

    +``` + +``` +_[bar_](url) +. +

    [bar](url)

    +``` + +``` +\_[bar_](url) +. +

    _bar_

    +``` + +Code takes precedence: + +``` +_`a_`b +. +

    _a_b

    +``` + +Autolinks take precedence: + +``` +_ +. +

    _http://example.com/a_b

    +``` diff --git a/test/escapes.test b/test/escapes.test new file mode 100644 index 0000000..9e0f35e --- /dev/null +++ b/test/escapes.test @@ -0,0 +1,53 @@ +ASCII punctuation characters can be escaped: + +``` +\`\*\_\[\# +. +

    `*_[#

    +``` + +Non-ASCII punctuation characters can't be escaped: + +``` +\a\« +. +

    \a\«

    +``` + +An escaped newline is a hard break: + +``` +ab\ +c +. +

    ab
    +c

    +``` + +There can be spaces and tabs between the backslash and the newline: + +``` +ab\ +c +. +

    ab
    +c

    +``` + +There can also be spaces and tabs before the backslash, which are ignored: + +``` +ab \ +c +. +

    ab
    +c

    +``` + +An escaped space is a non-breaking space: + +``` +a\ b +. +

    a b

    +``` diff --git a/test/fenced_divs.test b/test/fenced_divs.test new file mode 100644 index 0000000..caa4fae --- /dev/null +++ b/test/fenced_divs.test @@ -0,0 +1,134 @@ +Fenced divs are containers for sequences of blocks, to +which an attribute can be attached. + +A fenced div begins with an opening fence: a line with +three or more consecutive `:` characters, followed optionally by +a class name and optionally whitespace. + +It ends with a closing fence: a line beginning with three +or more consecutive `:` characters, followed by optional +whitespace and the end of the line. The number of `:` characters +in the closing fence must be at least the number in the opening fence. + +If the end of the input (or enclosing block) is encountered +before a closing fence, the fenced div is implicitly closed. + +``` +:::::::::: foo +Hi + +> A block quote. +::::::::::: +. +
    +

    Hi

    +
    +

    A block quote.

    +
    +
    +``` + +``` +{#bar .foo} +::: +Hi + +> A block quote. +::::::::::::: +. +
    +

    Hi

    +
    +

    A block quote.

    +
    +
    +``` + +Fenced divs may be nested. + +``` +{#bar .foo} +:::: +Hi + +::: baz +> A block quote. +::: +:::: +. +
    +

    Hi

    +
    +
    +

    A block quote.

    +
    +
    +
    +``` + +A fenced div cannot interrupt a paragraph, without +an intervening blank line. + +``` +Paragraph text +:::: +Hi +:::: +. +

    Paragraph text +:::: +Hi +::::

    +``` + +A fenced div need not have attributes or a class name. + +``` +:::: +Hi +:::: +. +
    +

    Hi

    +
    +``` + +The closing fence must be at least as long as the opening fence. + +``` +::::::::: foo +Hi +:::: +. +
    +

    Hi +::::

    +
    +``` + +If the end of the input (or enclosing block) is encountered +before a closing fence, the fenced div is implicitly closed. + +``` +> :::: foo +> Hi +. +
    +
    +

    Hi

    +
    +
    +``` + +```` +::: outer +``` +::: +``` +::: +. +
    +
    :::
+
    +
    +```` diff --git a/test/filters.test b/test/filters.test new file mode 100644 index 0000000..7674aa6 --- /dev/null +++ b/test/filters.test @@ -0,0 +1,68 @@ +Capitalize text: + +``` +*Hello* world `code` +! +return { + str = function(e) + e.text = e.text:upper() + end +} +. +

    HELLO WORLD code

    +``` + +Capitalize text inside emphasis only: + +``` +_Hello *world*_ outside +! +local capitalize = 0 +return { + emph = { + enter = function(e) + capitalize = capitalize + 1 + end, + exit = function(e) + capitalize = capitalize - 1 + end, + }, + str = function(e) + if capitalize > 0 then + e.text = e.text:upper() + end + end +} +. +

    HELLO WORLD outside

    +``` + +Capitalize text except in footnotes: + +``` a +Hello[^1]. + +[^1]: This is a note. +! +return { + str = function(e) + e.text = e.text:upper() + end, + footnote = { + enter = function(e) + return true -- prevent traversing into children + end + } +} +. +doc + para + str text="HELLO" + footnote_reference text="1" + str text="." +footnotes + ["1"] = + footnote + para + str text="This is a note." +``` diff --git a/test/footnotes.test b/test/footnotes.test new file mode 100644 index 0000000..42e98cd --- /dev/null +++ b/test/footnotes.test @@ -0,0 +1,46 @@ +```` +test[^a] and another[^foo_bar]. + +[^a]: This is a note. + + Second paragraph. + +[^foo_bar]: + ``` + code + ``` + +another ref to the first note[^a]. +. +

    test1 and another2.

    +

    another ref to the first note1.

    +
    +
    +
      +
    1. +

      This is a note.

      +

      Second paragraph.↩︎︎

      +
      2. +
    2. +
      code
+
      +

      ↩︎︎

      +
      3. +
    +
    +```` + +``` +test[^nonexistent] + +[^unused]: note + + more +. +

    test1

    +
    +
    +
      +
    +
    +``` diff --git a/test/headings.test b/test/headings.test new file mode 100644 index 0000000..e5923f5 --- /dev/null +++ b/test/headings.test @@ -0,0 +1,182 @@ +``` +## Heading +. +
    +

    Heading

    +
    +``` + +``` +# Heading + +# another +. +
    +

    Heading

    +
    +
    +

    another

    +
    +``` + +``` +# Heading +# continued +. +
    +

    Heading +continued

    +
    +``` + +``` +## +heading + +para +. +
    +

    heading

    +

    para

    +
    +``` + +``` +## +. +
    +

    +
    +``` + +``` +## Heading +### Next level +. +
    +

    Heading

    +
    +

    Next level

    +
    +
    +``` + +``` +# Heading +lazy +. +
    +

    Heading +lazy

    +
    +``` + +``` +# Heading +lazy +# more +lazy + +text +. +
    +

    Heading +lazy +more +lazy

    +

    text

    +
    +``` + +``` +##Notheading +. +

    ##Notheading

    +``` + +``` + ## Heading +. +
    +

    Heading

    +
    +``` + +``` +## heading ## +. +
    +

    heading ##

    +
    +``` + +``` +# # heading +. +
    +

    # heading

    +
    +``` + +Auto-identifiers: + +``` +{#Foo-bar} +Paragraph + +# Foo bar + +## Foo bar + +{#baz} +# Foo bar +. +

    Paragraph

    +
    +

    Foo bar

    +
    +

    Foo bar

    +
    +
    +
    +

    Foo bar

    +
    +``` + +Implicit header references: + +``` +See [Introduction][]. + +# Introduction +. +

    See Introduction.

    +
    +

    Introduction

    +
    +``` + +``` +See [Introduction][]. + +{#foo} +# Introduction +. +

    See Introduction.

    +
    +

    Introduction

    +
    +``` + +``` +See [Introduction][]. + +# Introduction + +[Introduction]: #bar +. +

    See Introduction.

    +
    +

    Introduction

    +
    +``` diff --git a/test/insert_delete_mark.test b/test/insert_delete_mark.test new file mode 100644 index 0000000..e290dc2 --- /dev/null +++ b/test/insert_delete_mark.test @@ -0,0 +1,29 @@ +``` +This is {-deleted +_text_-}. The braces are -required-. +And they must be in the -}right order{-. +. +

    This is deleted +text. The braces are -required-. +And they must be in the -}right order{-.

    +``` + +``` +{+ Inserted text +} +. +

    Inserted text

    +``` + +Interaction with smart: + +``` +{--hello--} +. +

    -hello-

    +``` + +``` +This is {=marked *text*=}. +. +

    This is marked text.

    +``` diff --git a/test/links_and_images.test b/test/links_and_images.test new file mode 100644 index 0000000..f14a58f --- /dev/null +++ b/test/links_and_images.test @@ -0,0 +1,242 @@ +``` +[basic _link_][a_b_] + +[a_b_]: url +. +

    basic link

    +``` + +``` +![basic _image_][a_b_] + +[a_b_]: url +. +

    basic image

    +``` + +``` +[link][] + +[link]: url +. +

    link

    +``` + +``` +[link][] + +[link]: + url +. +

    link

    +``` + +The URL can be split over multiple lines: + +``` +[link][] + +[link]: + url + andurl +. +

    link

    +``` + +``` +[link](url +andurl) +. +

    link

    +``` + +``` +[link][] + +[link]: +[link2]: url +. +

    link

    +``` + +``` +[link][] +[link][link2] + +[link2]: + url2 +[link]: + url +. +

    link +link

    +``` + +``` +[link][a and +b] + +[a and b]: url +. +

    link

    +``` + +If the reference isn't found, we get an empty link. + +``` +[link][a and +b] +. +

    link

    +``` + +Reference definitions can't have line breaks in the key: + +``` +[link][a and +b] + +[a and +b]: url +. +

    link

    +

    [a and +b]: url

    +``` + +No case normalization is done on reference definitions: + +``` +[Link][] + +[link]: /url +. +

    Link

    +``` + +Attributes on reference definitions get transferred to +the link: + +``` +{title=foo} +[ref]: /url + +[ref][] +. +

    ref

    +``` + +Attributes on the link override those on references: + +``` +{title=foo} +[ref]: /url + +[ref][]{title=bar} +. +

    ref

    +``` + +``` +[link _and_ link][] + +[link and link]: url +. +

    link and link

    +``` + +``` +![basic _image_](url) +. +

    basic image

    +``` + +``` +[![image](img.jpg)](url) +. +

    image

    +``` + +``` +[unclosed](hello *a +b* +. +

    [unclosed](hello a +b

    +``` + +Note that soft breaks are ignored, so long URLs +can be split over multiple lines: +``` +[closed](hello *a +b*) +. +

    closed

    +``` + +Here the strong takes precedence over the link because it +starts first: +``` +*[closed](hello*) +. +

    [closed](hello)

    +``` + +Avoid this with a backslash escape: +``` +*[closed](hello\*) +. +

    *closed

    +``` + +Link in link? +``` +[[foo](bar)](baz) +. +

    foo

    +``` + +Link in image? +``` +![[link](url)](img) +. +

    link

    +``` + +Image in link? +``` +[![image](img)](url) +. +

    image

    +``` + +Autolinks: +``` + + +. +

    http://example.com/foo +me@example.com

    +``` + +Openers inside `[..](` or `[..][` or `[..]{` can't match +outside them, even if the construction doesn't turn out to be +a link or span or image. + +``` +[x_y](x_y) +. +

    x_y

    +``` + +``` +[x_y](x_ +. +

    [x_y](x_

    +``` + +``` +[x_y]{.bar_} +. +

    x_y

    +``` diff --git a/test/lists.test b/test/lists.test new file mode 100644 index 0000000..9094e0c --- /dev/null +++ b/test/lists.test @@ -0,0 +1,494 @@ +``` +- one +- two +. +
      +
    • +one +
      • +
    • +two +
      • +
    +``` + +``` +- one + - two + - three +. +
      +
    • +one +- two +- three +
      • +
    +``` + +``` +- one + + - two + + - three +. +
      +
    • +one +
        +
      • +two +
          +
        • +three +
          • +
        +
        • +
      +
      • +
    +``` + +``` +- one + and + + another paragraph + + - a list + +- two +. +
      +
    • +

      one +and

      +

      another paragraph

      +
        +
      • +a list +
        • +
      +
      • +
    • +

      two

      +
      • +
    +``` + +``` +- one +lazy +- two +. +
      +
    • +one +lazy +
      • +
    • +two +
      • +
    +``` + +``` +- a +- b ++ c +. +
      +
    • +a +
      • +
    • +b +
      • +
    +
      +
    • +c +
      • +
    +``` + +``` +- a + +- b +. +
      +
    • +

      a

      +
      • +
    • +

      b

      +
      • +
    +``` + +``` +- a + - b + + - c +- d +. +
      +
    • +a +- b +
        +
      • +c +
        • +
      +
      • +
    • +d +
      • +
    +``` + +``` +- a + - b + + - c + +- d +. +
      +
    • +a +- b +
        +
      • +c +
        • +
      +
      • +
    • +d +
      • +
    +``` + +``` +- a + + b +- c +. +
      +
    • +

      a

      +

      b

      +
      • +
    • +

      c

      +
      • +
    +``` + +``` +- a + + - b + - c +- d +. +
      +
    • +a +
        +
      • +b +
        • +
      • +c +
        • +
      +
      • +
    • +d +
      • +
    +``` + +``` +- a + + - b + - c + +- d +. +
      +
    • +a +
        +
      • +b +
        • +
      • +c +
        • +
      +
      • +
    • +d +
      • +
    +``` + +``` +- a + + * b +cd +. +
      +
    • +a +
        +
      • +b +cd +
        • +
      +
      • +
    +``` + +``` +- - - a +. +
      +
    • +
        +
      • +
          +
        • +a +
          • +
        +
        • +
      +
      • +
    +``` + +``` +1. one +1. two +. +
      +
    1. +one +
      2. +
    2. +two +
      3. +
    +``` + +``` +1. one + + 1. two +. +
      +
    1. +one +
        +
      1. +two +
        2. +
      +
      2. +
    +``` + +``` +4. one +5. two +. +
      +
    1. +one +
      2. +
    2. +two +
      3. +
    +``` + +``` +1) one +2) two +. +
      +
    1. +one +
      2. +
    2. +two +
      3. +
    +``` + +``` +(1) one +(2) two +. +
      +
    1. +one +
      2. +
    2. +two +
      3. +
    +``` + +``` +(a) one +(b) two +. +
      +
    1. +one +
      2. +
    2. +two +
      3. +
    +``` + +``` +(D) one +(E) two +. +
      +
    1. +one +
      2. +
    2. +two +
      3. +
    +``` + +``` +a. one +b. two +. +
      +
    1. +one +
      2. +
    2. +two +
      3. +
    +``` + +``` +i. one +ii. two +. +
      +
    1. +one +
      2. +
    2. +two +
      3. +
    +``` + +``` +xli) one +xlii) two +. +
      +
    1. +one +
      2. +
    2. +two +
      3. +
    +``` + +``` +(IV) one +(V) two +. +
      +
    1. +one +
      2. +
    2. +two +
      3. +
    +``` + +``` +i. a +ii. b +. +
      +
    1. +a +
      2. +
    2. +b +
      3. +
    +``` + +``` +i. a +j. b +. +
      +
    1. +a +
      2. +
    2. +b +
      3. +
    +``` + +``` +I. a +II. b +E. d +. +
      +
    1. +a +
      2. +
    2. +b +
      3. +
    +
      +
    1. +d +
      2. +
    +``` + +``` +The civil war ended in +1865. And this should not start a list. +. +

    The civil war ended in +1865. And this should not start a list.

    +``` diff --git a/test/math.test b/test/math.test new file mode 100644 index 0000000..c814f92 --- /dev/null +++ b/test/math.test @@ -0,0 +1,44 @@ +Math goes in verbatim spans prefixed with either `$` (for +inline math) or `$$` (for display math). + +``` +$`e=mc^2` +. +

    \(e=mc^2\)

    +``` + +``` +My equation: $`e=mc^2` +. +

    My equation: \(e=mc^2\)

    +``` + +``` +$$`e=mc^2` +. +

    \[e=mc^2\]

    +``` + +``` +My equation: $$`e=mc^2` +. +

    My equation: \[e=mc^2\]

    +``` + +Newlines are allowed, just as in verbatim: + +``` +$`e= +mc^2` +. +

    \(e= +mc^2\)

    +``` + +`$` characters are allowed inside: + +``` +$`e=\text{the number $\pi$}` +. +

    \(e=\text{the number $\pi$}\)

    +``` diff --git a/test/para.test b/test/para.test new file mode 100644 index 0000000..d5d45e7 --- /dev/null +++ b/test/para.test @@ -0,0 +1,7 @@ +``` +hi +there +. +

    hi +there

    +``` diff --git a/test/raw.test b/test/raw.test new file mode 100644 index 0000000..95b2699 --- /dev/null +++ b/test/raw.test @@ -0,0 +1,38 @@ +Raw inline content: + +``` +``{=html} +. +

    +``` + +Raw block-level content: + +```` +``` =html + +``` +. +
    +```` + +You can't mix regular attributes and raw syntax: + +```` +`foo`{=html #id} +``` +. +

    <b>foo</b>{=html #id} +

    +```` + +Attributes attached to raw content will just be ignored: + +```` +{.foo} +``` =html +
    +``` +. +
    +```` diff --git a/test/regression.test b/test/regression.test new file mode 100644 index 0000000..a2b2c6d --- /dev/null +++ b/test/regression.test @@ -0,0 +1,49 @@ +Issue #104: + +``` +{1--} + +{1-} +. +

    {1--}

    +

    {1-}

    +``` + +Issue #106: + +``` + +|`| +. +

    ||

    +``` + +``` m + +|`|x +. +[["blankline",1,1] +,["+para",2,2] +,["str",2,2] +,["+verbatim",3,3] +,["str",4,5] +,["-verbatim",5,5] +,["-para",6,6] +] +``` + +Issue #127: + +``` +\$$`a` +. +

    $\(a\)

    +``` + +``` +{ + .` +. +

    { +.

    +``` diff --git a/test/smart.test b/test/smart.test new file mode 100644 index 0000000..8524f03 --- /dev/null +++ b/test/smart.test @@ -0,0 +1,192 @@ +Open quotes are matched with closed quotes. +The same method is used for matching openers and closers +as is used in emphasis parsing: + +``` +"Hello," said the spider. +"'Shelob' is my name." +. +

    “Hello,” said the spider. +“‘Shelob’ is my name.”

    +``` + +``` +'A', 'B', and 'C' are letters. +. +

    ‘A’, ‘B’, and ‘C’ are letters.

    +``` + +``` +'Oak,' 'elm,' and 'beech' are names of trees. +So is 'pine.' +. +

    ‘Oak,’ ‘elm,’ and ‘beech’ are names of trees. +So is ‘pine.’

    +``` + +``` +'He said, "I want to go."' +. +

    ‘He said, “I want to go.”’

    +``` + +A single quote that isn't an open quote matched +with a close quote will be treated as an +apostrophe: + +``` +Were you alive in the '70s? +. +

    Were you alive in the ’70s?

    +``` + +``` +Here is some quoted '`code`' and a "[quoted link](url)". +. +

    Here is some quoted ‘code’ and a “quoted link”.

    +``` + +Here the first `'` is treated as an apostrophe, not +an open quote, because the final single quote is matched +by the single quote before `jolly`: + +``` +'tis the season to be 'jolly' +. +

    ’tis the season to be ‘jolly’

    +``` + +Multiple apostrophes should not be marked as open/closing quotes. + +``` +'We'll use Jane's boat and John's truck,' Jenna said. +. +

    ‘We’ll use Jane’s boat and John’s truck,’ Jenna said.

    +``` + +An unmatched double quote will be interpreted as a +left double quote, to facilitate this style: + +``` +"A paragraph with no closing quote. + +"Second paragraph by same speaker, in fiction." +. +

    “A paragraph with no closing quote.

    +

    “Second paragraph by same speaker, in fiction.”

    +``` + +A quote following a `]` or `)` character cannot +be an open quote: + +``` +[a]'s b' +. +

    [a]’s b’

    +``` + +Quotes that are escaped come out as literal straight +quotes: + +``` +\"This is not smart.\" +This isn\'t either. +5\'8\" +. +

    "This is not smart." +This isn't either. +5'8"

    +``` + +Doubled quotes are treated as nested: + +``` +''hi'' +. +

    ‘‘hi’’

    +``` + +Heuristics for determining openers and closers can +be overridden using `{` and `}`: + +``` +{''}hi{''} +. +

    ‘’hi‘’

    +``` + +Two hyphens form an en-dash, three an em-dash. + +``` +Some dashes: em---em +en--en +em --- em +en -- en +2--3 +. +

    Some dashes: em—em +en–en +em — em +en – en +2–3

    +``` + +A sequence of more than three hyphens is +parsed as a sequence of em and/or en dashes, +with no hyphens. If possible, a homogeneous +sequence of dashes is used (so, 10 hyphens += 5 en dashes, and 9 hyphens = 3 em dashes). +When a heterogeneous sequence must be used, +the em dashes come first, followed by the en +dashes, and as few en dashes as possible are +used (so, 7 hyphens = 2 em dashes an 1 en +dash). + +``` +one- +two-- +three--- +four---- +five----- +six------ +seven------- +eight-------- +nine--------- +thirteen-------------. +. +

    one- +two– +three— +four–– +five—– +six—— +seven—–– +eight–––– +nine——— +thirteen———––.

    +``` + +Hyphens can be escaped: + +``` +Escaped hyphens: \-- \-\-\-. +. +

    Escaped hyphens: -- ---.

    +``` + +Three periods form an ellipsis: + +``` +Ellipses...and...and.... +. +

    Ellipses…and…and….

    +``` + +Periods can be escaped if ellipsis-formation +is not wanted: + +``` +No ellipses\.\.\. +. +

    No ellipses...

    +``` diff --git a/test/sourcepos.test b/test/sourcepos.test new file mode 100644 index 0000000..66f8bcb --- /dev/null +++ b/test/sourcepos.test @@ -0,0 +1,13 @@ +``` ap + - a + - b +. +doc + list (1:2:2-4:1:11) style="-" tight="true" + list_item (1:2:2-2:2:7) + para (1:4:4-1:5:5) + str (1:4:4-1:4:4) text="a" + list_item (2:2:7-4:1:11) + para (2:4:9-2:5:10) + str (2:4:9-2:4:9) text="b" +``` diff --git a/test/spans.test b/test/spans.test new file mode 100644 index 0000000..29b1a36 --- /dev/null +++ b/test/spans.test @@ -0,0 +1,19 @@ +``` +This is a [test of +*color*]{.blue}. +. +

    This is a test of +color.

    +``` + +``` +not a [span] {#id}. +. +

    not a [span] .

    +``` + +``` +[nested [span]{.blue}]{#ident} +. +

    nested span

    +``` diff --git a/test/super_subscript.test b/test/super_subscript.test new file mode 100644 index 0000000..5026a14 --- /dev/null +++ b/test/super_subscript.test @@ -0,0 +1,23 @@ +``` +H~2~O +. +

    H2O

    +``` + +``` +mc^2^ +. +

    mc2

    +``` + +``` +test^of superscript ~with subscript~^ +. +

    testof superscript with subscript

    +``` + +``` +H{~2 ~}O +. +

    H2 O

    +``` diff --git a/test/symbol.test b/test/symbol.test new file mode 100644 index 0000000..a518292 --- /dev/null +++ b/test/symbol.test @@ -0,0 +1,18 @@ +``` a +:+1: :scream: +. +doc + para + symbol alias="+1" + str text=" " + symbol alias="scream" +``` + +``` a +:ice:scream: +. +doc + para + symbol alias="ice" + str text="scream:" +``` diff --git a/test/tables.test b/test/tables.test new file mode 100644 index 0000000..1d0eafc --- /dev/null +++ b/test/tables.test @@ -0,0 +1,125 @@ +Simplest table: + +``` +| a | +. +
     + + + +
    a
    +``` + +``` +|a| *b*| +|*c| d* | +. + + + + + + + + + +
    ab
    *cd*
    +``` + +``` +| `a |` +. +

    | a |

    +``` + +``` +| a | b | + +^ With a _caption_ +and another line. +. + + + + + + +
    With a caption +and another line.
    ab
    +``` + +Table headers: note that we can have multiple headers; each +determines the alignment for following cells, until the next header. + +``` +|a|b| +|:-|---:| +|c|d| +|cc|dd| +|-:|:-:| +|e|f| +|g|h| +. + + + + + + + + + + + + + + + + + + + + + +
    ab
    cd
    ccdd
    ef
    gh
    +``` + +``` +|--|--| +. + +
    +``` + +``` +|---|---| +| a | b | +. + + + + + +
    ab
    +``` + +``` +| | +. + + + + +
    +``` + +``` +| just two \| `|` | cells in this table | +. + + + + + +
    just two | |cells in this table
    +``` diff --git a/test/task_lists.test b/test/task_lists.test new file mode 100644 index 0000000..71a3ab0 --- /dev/null +++ b/test/task_lists.test @@ -0,0 +1,31 @@ +``` +- [ ] an unchecked task list item +- [x] checked item +. +
      +
    • +an unchecked task list item +
      • +
    • +checked item +
      • +
    +``` + +``` +* [ ] an unchecked task list item + + with two paragraphs + +* [x] checked item +. +
      +
    • +

      an unchecked task list item

      +

      with two paragraphs

      +
      • +
    • +

      checked item

      +
      • +
    +``` diff --git a/test/thematic_breaks.test b/test/thematic_breaks.test new file mode 100644 index 0000000..4a614e9 --- /dev/null +++ b/test/thematic_breaks.test @@ -0,0 +1,45 @@ +``` +hello + +- - - + +there +. +

    hello

    +
    +

    there

    +``` + +``` +hello + + ** ** + +there +. +

    hello

    +
    +

    there

    +``` + +``` +hello + + *-*-*-* + +there +. +

    hello

    +
    +

    there

    +``` + +``` +hello + *-*-*-* +there +. +

    hello +--- +there

    +``` diff --git a/test/verbatim.test b/test/verbatim.test new file mode 100644 index 0000000..db1d08b --- /dev/null +++ b/test/verbatim.test @@ -0,0 +1,47 @@ +``` +Some `code` +. +

    Some code

    +``` + +``` +Some `code +with a line break` +. +

    Some code +with a line break

    +``` + +``` +Special characters: `*hi*` +. +

    Special characters: *hi*

    +``` + +``` +*foo`*` +. +

    *foo*

    +``` + +``````` +`````a`a``a```a````a``````a````` +. +

    a`a``a```a````a``````a

    +``````` + +``` +` ``a`` ` +. +

    ``a``

    +``` + +Implicitly closed by end of paragraph: + +``` +` a +c +. +

    a +c

    +``` -- 2.47.3