v0.6.0

.gitignore

@@ -7,3 +7,5 @@ chroma/
 pygments/
 shard.lock
 .vscode/
+.crystal/
+venv/

README.md

@@ -4,17 +4,17 @@ Tartrazine is a library to syntax-highlight code. It is
 a port of [Pygments](https://pygments.org/) to
 [Crystal](https://crystal-lang.org/). Kind of.
 
-It's not currently usable because it's not finished, but:
+The CLI tool can be used to highlight many things in many styles.
-
-* The lexers work for the implemented languages
-* The provided styles work
-* There is a very very simple HTML formatter
 
 # A port of what? Why "kind of"?
 
-Because I did not read the Pygments code. And this is actually
+Pygments is a staple of the Python ecosystem, and it's great.
-based on [Chroma](https://github.com/alecthomas/chroma) ...
+It lets you highlight code in many languages, and it has many
-although I did not read that code either.
+themes. Chroma is "Pygments for Go", it's actually a port of
+Pygments to Go, and it's great too.
+
+I wanted that in Crystal, so I started this project. But I did
+not read much of the Pygments code. Or much of Chroma's.
 
 Chroma has taken most of the Pygments lexers and turned them into
 XML descriptions. What I did was take those XML files from Chroma
@@ -47,7 +47,14 @@ To build from source:
 2. Run `make` to build the `tartrazine` binary
 3. Copy the binary somewhere in your PATH.
 
-## Usage
+## Usage as a CLI tool
+
+```shell
+$ tartrazine whatever.c -l c -t catppuccin-macchiato --line-numbers \
+  --standalone -o whatever.html 
+```
+
+## Usage as a Library
 
 This works:
 
@@ -56,7 +63,9 @@ require "tartrazine"
 
 lexer = Tartrazine.lexer("crystal")
 theme = Tartrazine.theme("catppuccin-macchiato")
-puts Tartrazine::Html.new.format(File.read(ARGV[0]), lexer, theme)
+formatter = Tartrazine::Html.new
+formatter.theme = theme
+puts formatter.format(File.read(ARGV[0]), lexer)
 ```
 
 ## Contributing
@@ -69,4 +78,4 @@ puts Tartrazine::Html.new.format(File.read(ARGV[0]), lexer, theme)
 
 ## Contributors
 
-- [Roberto Alsina](https://github.com/ralsina) - creator and maintainer
+- [Roberto Alsina](https://github.com/ralsina) - creator and maintainer

TODO.md

@@ -8,4 +8,6 @@
 * ✅ Implement lexer loader that respects aliases
 * ✅ Implement lexer loader by file extension
 * ✅ Add --line-numbers to terminal formatter
-* Implement lexer loader by mime type
+* Implement lexer loader by mime type
+* ✅ Implement Delegating lexers
+* Add RstLexer maybe others?

lexers/LiquidLexer.xml

@@ -0,0 +1,130 @@
+
+<lexer>
+  <config>
+    <name>liquid</name>
+    <alias>liquid</alias>
+    <filename>*.liquid</filename>
+  </config>
+  <rules>
+    <state name="root">
+      <rule pattern="[^{]+"><token type="Text"/></rule>
+      <rule pattern="(\{%)(\s*)"><bygroups><token type="Punctuation"/><token type="TextWhitespace"/></bygroups><push state="tag-or-block"/></rule>
+      <rule pattern="(\{\{)(\s*)([^\s}]+)"><bygroups><token type="Punctuation"/><token type="TextWhitespace"/><usingself state="generic"/></bygroups><push state="output"/></rule>
+      <rule pattern="\{"><token type="Text"/></rule>
+    </state>
+    <state name="tag-or-block">
+      <rule pattern="(if|unless|elsif|case)(?=\s+)"><token type="KeywordReserved"/><push state="condition"/></rule>
+      <rule pattern="(when)(\s+)"><bygroups><token type="KeywordReserved"/><token type="TextWhitespace"/></bygroups><combined state="end-of-block" state="whitespace" state="generic"/></rule>
+      <rule pattern="(else)(\s*)(%\})"><bygroups><token type="KeywordReserved"/><token type="TextWhitespace"/><token type="Punctuation"/></bygroups><pop depth="1"/></rule>
+      <rule pattern="(capture)(\s+)([^\s%]+)(\s*)(%\})"><bygroups><token type="NameTag"/><token type="TextWhitespace"/><usingself state="variable"/><token type="TextWhitespace"/><token type="Punctuation"/></bygroups><pop depth="1"/></rule>
+      <rule pattern="(comment)(\s*)(%\})"><bygroups><token type="NameTag"/><token type="TextWhitespace"/><token type="Punctuation"/></bygroups><push state="comment"/></rule>
+      <rule pattern="(raw)(\s*)(%\})"><bygroups><token type="NameTag"/><token type="TextWhitespace"/><token type="Punctuation"/></bygroups><push state="raw"/></rule>
+      <rule pattern="(end(case|unless|if))(\s*)(%\})"><bygroups><token type="KeywordReserved"/>None<token type="TextWhitespace"/><token type="Punctuation"/></bygroups><pop depth="1"/></rule>
+      <rule pattern="(end([^\s%]+))(\s*)(%\})"><bygroups><token type="NameTag"/>None<token type="TextWhitespace"/><token type="Punctuation"/></bygroups><pop depth="1"/></rule>
+      <rule pattern="(cycle)(\s+)(?:([^\s:]*)(:))?(\s*)"><bygroups><token type="NameTag"/><token type="TextWhitespace"/><usingself state="generic"/><token type="Punctuation"/><token type="TextWhitespace"/></bygroups><push state="variable-tag-markup"/></rule>
+      <rule pattern="([^\s%]+)(\s*)"><bygroups><token type="NameTag"/><token type="TextWhitespace"/></bygroups><push state="tag-markup"/></rule>
+    </state>
+    <state name="output">
+      <rule><include state="whitespace"/></rule>
+      <rule pattern="\}\}"><token type="Punctuation"/><pop depth="1"/></rule>
+      <rule pattern="\|"><token type="Punctuation"/><push state="filters"/></rule>
+    </state>
+    <state name="filters">
+      <rule><include state="whitespace"/></rule>
+      <rule pattern="\}\}"><token type="Punctuation"/><push state="#pop" state="#pop"/></rule>
+      <rule pattern="([^\s|:]+)(:?)(\s*)"><bygroups><token type="NameFunction"/><token type="Punctuation"/><token type="TextWhitespace"/></bygroups><push state="filter-markup"/></rule>
+    </state>
+    <state name="filter-markup">
+      <rule pattern="\|"><token type="Punctuation"/><pop depth="1"/></rule>
+      <rule><include state="end-of-tag"/></rule>
+      <rule><include state="default-param-markup"/></rule>
+    </state>
+    <state name="condition">
+      <rule><include state="end-of-block"/></rule>
+      <rule><include state="whitespace"/></rule>
+      <rule pattern="([^\s=!&gt;&lt;]+)(\s*)([=!&gt;&lt;]=?)(\s*)(\S+)(\s*)(%\})"><bygroups><usingself state="generic"/><token type="TextWhitespace"/><token type="Operator"/><token type="TextWhitespace"/><usingself state="generic"/><token type="TextWhitespace"/><token type="Punctuation"/></bygroups></rule>
+      <rule pattern="\b!"><token type="Operator"/></rule>
+      <rule pattern="\bnot\b"><token type="OperatorWord"/></rule>
+      <rule pattern="([\w.\&#x27;&quot;]+)(\s+)(contains)(\s+)([\w.\&#x27;&quot;]+)"><bygroups><usingself state="generic"/><token type="TextWhitespace"/><token type="OperatorWord"/><token type="TextWhitespace"/><usingself state="generic"/></bygroups></rule>
+      <rule><include state="generic"/></rule>
+      <rule><include state="whitespace"/></rule>
+    </state>
+    <state name="generic-value">
+      <rule><include state="generic"/></rule>
+      <rule><include state="end-at-whitespace"/></rule>
+    </state>
+    <state name="operator">
+      <rule pattern="(\s*)((=|!|&gt;|&lt;)=?)(\s*)"><bygroups><token type="TextWhitespace"/><token type="Operator"/>None<token type="TextWhitespace"/></bygroups><pop depth="1"/></rule>
+      <rule pattern="(\s*)(\bcontains\b)(\s*)"><bygroups><token type="TextWhitespace"/><token type="OperatorWord"/><token type="TextWhitespace"/></bygroups><pop depth="1"/></rule>
+    </state>
+    <state name="end-of-tag">
+      <rule pattern="\}\}"><token type="Punctuation"/><pop depth="1"/></rule>
+    </state>
+    <state name="end-of-block">
+      <rule pattern="%\}"><token type="Punctuation"/><push state="#pop" state="#pop"/></rule>
+    </state>
+    <state name="end-at-whitespace">
+      <rule pattern="\s+"><token type="TextWhitespace"/><pop depth="1"/></rule>
+    </state>
+    <state name="param-markup">
+      <rule><include state="whitespace"/></rule>
+      <rule pattern="([^\s=:]+)(\s*)(=|:)"><bygroups><token type="NameAttribute"/><token type="TextWhitespace"/><token type="Operator"/></bygroups></rule>
+      <rule pattern="(\{\{)(\s*)([^\s}])(\s*)(\}\})"><bygroups><token type="Punctuation"/><token type="TextWhitespace"/><usingself state="variable"/><token type="TextWhitespace"/><token type="Punctuation"/></bygroups></rule>
+      <rule><include state="string"/></rule>
+      <rule><include state="number"/></rule>
+      <rule><include state="keyword"/></rule>
+      <rule pattern=","><token type="Punctuation"/></rule>
+    </state>
+    <state name="default-param-markup">
+      <rule><include state="param-markup"/></rule>
+      <rule pattern="."><token type="Text"/></rule>
+    </state>
+    <state name="variable-param-markup">
+      <rule><include state="param-markup"/></rule>
+      <rule><include state="variable"/></rule>
+      <rule pattern="."><token type="Text"/></rule>
+    </state>
+    <state name="tag-markup">
+      <rule pattern="%\}"><token type="Punctuation"/><push state="#pop" state="#pop"/></rule>
+      <rule><include state="default-param-markup"/></rule>
+    </state>
+    <state name="variable-tag-markup">
+      <rule pattern="%\}"><token type="Punctuation"/><push state="#pop" state="#pop"/></rule>
+      <rule><include state="variable-param-markup"/></rule>
+    </state>
+    <state name="keyword">
+      <rule pattern="\b(false|true)\b"><token type="KeywordConstant"/></rule>
+    </state>
+    <state name="variable">
+      <rule pattern="[a-zA-Z_]\w*"><token type="NameVariable"/></rule>
+      <rule pattern="(?&lt;=\w)\.(?=\w)"><token type="Punctuation"/></rule>
+    </state>
+    <state name="string">
+      <rule pattern="&#x27;[^&#x27;]*&#x27;"><token type="LiteralStringSingle"/></rule>
+      <rule pattern="&quot;[^&quot;]*&quot;"><token type="LiteralStringDouble"/></rule>
+    </state>
+    <state name="number">
+      <rule pattern="\d+\.\d+"><token type="LiteralNumberFloat"/></rule>
+      <rule pattern="\d+"><token type="LiteralNumberInteger"/></rule>
+    </state>
+    <state name="generic">
+      <rule><include state="keyword"/></rule>
+      <rule><include state="string"/></rule>
+      <rule><include state="number"/></rule>
+      <rule><include state="variable"/></rule>
+    </state>
+    <state name="whitespace">
+      <rule pattern="[ \t]+"><token type="TextWhitespace"/></rule>
+    </state>
+    <state name="comment">
+      <rule pattern="(\{%)(\s*)(endcomment)(\s*)(%\})"><bygroups><token type="Punctuation"/><token type="TextWhitespace"/><token type="NameTag"/><token type="TextWhitespace"/><token type="Punctuation"/></bygroups><push state="#pop" state="#pop"/></rule>
+      <rule pattern="."><token type="Comment"/></rule>
+    </state>
+    <state name="raw">
+      <rule pattern="[^{]+"><token type="Text"/></rule>
+      <rule pattern="(\{%)(\s*)(endraw)(\s*)(%\})"><bygroups><token type="Punctuation"/><token type="TextWhitespace"/><token type="NameTag"/><token type="TextWhitespace"/><token type="Punctuation"/></bygroups><pop depth="1"/></rule>
+      <rule pattern="\{"><token type="Text"/></rule>
+    </state>
+  </rules>
+</lexer>
+

lexers/TwigLexer.xml

@@ -0,0 +1,47 @@
+
+<lexer>
+  <config>
+    <name>Twig</name>
+    <alias>twig</alias>
+    <mime_type>application/x-twig</mime_type>
+    <dot_all>true</dot_all>
+  </config>
+  <rules>
+    <state name="root">
+      <rule pattern="[^{]+"><token type="Other"/></rule>
+      <rule pattern="\{\{"><token type="CommentPreproc"/><push state="var"/></rule>
+      <rule pattern="\{\#.*?\#\}"><token type="Comment"/></rule>
+      <rule pattern="(\{%)(-?\s*)(raw)(\s*-?)(%\})(.*?)(\{%)(-?\s*)(endraw)(\s*-?)(%\})"><bygroups><token type="CommentPreproc"/><token type="Text"/><token type="Keyword"/><token type="Text"/><token type="CommentPreproc"/><token type="Other"/><token type="CommentPreproc"/><token type="Text"/><token type="Keyword"/><token type="Text"/><token type="CommentPreproc"/></bygroups></rule>
+      <rule pattern="(\{%)(-?\s*)(verbatim)(\s*-?)(%\})(.*?)(\{%)(-?\s*)(endverbatim)(\s*-?)(%\})"><bygroups><token type="CommentPreproc"/><token type="Text"/><token type="Keyword"/><token type="Text"/><token type="CommentPreproc"/><token type="Other"/><token type="CommentPreproc"/><token type="Text"/><token type="Keyword"/><token type="Text"/><token type="CommentPreproc"/></bygroups></rule>
+      <rule pattern="(\{%)(-?\s*)(filter)(\s+)((?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w-]|[^\x00-\x7f])*)"><bygroups><token type="CommentPreproc"/><token type="Text"/><token type="Keyword"/><token type="Text"/><token type="NameFunction"/></bygroups><push state="tag"/></rule>
+      <rule pattern="(\{%)(-?\s*)([a-zA-Z_]\w*)"><bygroups><token type="CommentPreproc"/><token type="Text"/><token type="Keyword"/></bygroups><push state="tag"/></rule>
+      <rule pattern="\{"><token type="Other"/></rule>
+    </state>
+    <state name="varnames">
+      <rule pattern="(\|)(\s*)((?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w-]|[^\x00-\x7f])*)"><bygroups><token type="Operator"/><token type="Text"/><token type="NameFunction"/></bygroups></rule>
+      <rule pattern="(is)(\s+)(not)?(\s*)((?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w-]|[^\x00-\x7f])*)"><bygroups><token type="Keyword"/><token type="Text"/><token type="Keyword"/><token type="Text"/><token type="NameFunction"/></bygroups></rule>
+      <rule pattern="(?i)(true|false|none|null)\b"><token type="KeywordPseudo"/></rule>
+      <rule pattern="(in|not|and|b-and|or|b-or|b-xor|isif|elseif|else|importconstant|defined|divisibleby|empty|even|iterable|odd|sameasmatches|starts\s+with|ends\s+with)\b"><token type="Keyword"/></rule>
+      <rule pattern="(loop|block|parent)\b"><token type="NameBuiltin"/></rule>
+      <rule pattern="(?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w-]|[^\x00-\x7f])*"><token type="NameVariable"/></rule>
+      <rule pattern="\.(?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w-]|[^\x00-\x7f])*"><token type="NameVariable"/></rule>
+      <rule pattern="\.[0-9]+"><token type="LiteralNumber"/></rule>
+      <rule pattern=":?&quot;(\\\\|\\[^\\]|[^&quot;\\])*&quot;"><token type="LiteralStringDouble"/></rule>
+      <rule pattern=":?&#x27;(\\\\|\\[^\\]|[^&#x27;\\])*&#x27;"><token type="LiteralStringSingle"/></rule>
+      <rule pattern="([{}()\[\]+\-*/,:~%]|\.\.|\?|:|\*\*|\/\/|!=|[&gt;&lt;=]=?)"><token type="Operator"/></rule>
+      <rule pattern="[0-9](\.[0-9]*)?(eE[+-][0-9])?[flFLdD]?|0[xX][0-9a-fA-F]+[Ll]?"><token type="LiteralNumber"/></rule>
+    </state>
+    <state name="var">
+      <rule pattern="\s+"><token type="Text"/></rule>
+      <rule pattern="(-?)(\}\})"><bygroups><token type="Text"/><token type="CommentPreproc"/></bygroups><pop depth="1"/></rule>
+      <rule><include state="varnames"/></rule>
+    </state>
+    <state name="tag">
+      <rule pattern="\s+"><token type="Text"/></rule>
+      <rule pattern="(-?)(%\})"><bygroups><token type="Text"/><token type="CommentPreproc"/></bygroups><pop depth="1"/></rule>
+      <rule><include state="varnames"/></rule>
+      <rule pattern="."><token type="Punctuation"/></rule>
+    </state>
+  </rules>
+</lexer>
+

lexers/VelocityLexer.xml

@@ -0,0 +1,55 @@
+
+<lexer>
+  <config>
+    <name>Velocity</name>
+    <alias>velocity</alias>
+    <filename>*.vm</filename>
+    <filename>*.fhtml</filename>
+    <dot_all>true</dot_all>
+  </config>
+  <rules>
+    <state name="root">
+      <rule pattern="[^{#$]+"><token type="Other"/></rule>
+      <rule pattern="(#)(\*.*?\*)(#)"><bygroups><token type="CommentPreproc"/><token type="Comment"/><token type="CommentPreproc"/></bygroups></rule>
+      <rule pattern="(##)(.*?$)"><bygroups><token type="CommentPreproc"/><token type="Comment"/></bygroups></rule>
+      <rule pattern="(#\{?)([a-zA-Z_]\w*)(\}?)(\s?\()"><bygroups><token type="CommentPreproc"/><token type="NameFunction"/><token type="CommentPreproc"/><token type="Punctuation"/></bygroups><push state="directiveparams"/></rule>
+      <rule pattern="(#\{?)([a-zA-Z_]\w*)(\}|\b)"><bygroups><token type="CommentPreproc"/><token type="NameFunction"/><token type="CommentPreproc"/></bygroups></rule>
+      <rule pattern="\$!?\{?"><token type="Punctuation"/><push state="variable"/></rule>
+    </state>
+    <state name="variable">
+      <rule pattern="[a-zA-Z_]\w*"><token type="NameVariable"/></rule>
+      <rule pattern="\("><token type="Punctuation"/><push state="funcparams"/></rule>
+      <rule pattern="(\.)([a-zA-Z_]\w*)"><bygroups><token type="Punctuation"/><token type="NameVariable"/></bygroups><push/></rule>
+      <rule pattern="\}"><token type="Punctuation"/><pop depth="1"/></rule>
+      <rule><pop depth="1"/></rule>
+    </state>
+    <state name="directiveparams">
+      <rule pattern="(&amp;&amp;|\|\||==?|!=?|[-&lt;&gt;+*%&amp;|^/])|\b(eq|ne|gt|lt|ge|le|not|in)\b"><token type="Operator"/></rule>
+      <rule pattern="\["><token type="Operator"/><push state="rangeoperator"/></rule>
+      <rule pattern="\b[a-zA-Z_]\w*\b"><token type="NameFunction"/></rule>
+      <rule><include state="funcparams"/></rule>
+    </state>
+    <state name="rangeoperator">
+      <rule pattern="\.\."><token type="Operator"/></rule>
+      <rule><include state="funcparams"/></rule>
+      <rule pattern="\]"><token type="Operator"/><pop depth="1"/></rule>
+    </state>
+    <state name="funcparams">
+      <rule pattern="\$!?\{?"><token type="Punctuation"/><push state="variable"/></rule>
+      <rule pattern="\s+"><token type="Text"/></rule>
+      <rule pattern="[,:]"><token type="Punctuation"/></rule>
+      <rule pattern="&quot;(\\\\|\\[^\\]|[^&quot;\\])*&quot;"><token type="LiteralStringDouble"/></rule>
+      <rule pattern="&#x27;(\\\\|\\[^\\]|[^&#x27;\\])*&#x27;"><token type="LiteralStringSingle"/></rule>
+      <rule pattern="0[xX][0-9a-fA-F]+[Ll]?"><token type="LiteralNumber"/></rule>
+      <rule pattern="\b[0-9]+\b"><token type="LiteralNumber"/></rule>
+      <rule pattern="(true|false|null)\b"><token type="KeywordConstant"/></rule>
+      <rule pattern="\("><token type="Punctuation"/><push/></rule>
+      <rule pattern="\)"><token type="Punctuation"/><pop depth="1"/></rule>
+      <rule pattern="\{"><token type="Punctuation"/><push/></rule>
+      <rule pattern="\}"><token type="Punctuation"/><pop depth="1"/></rule>
+      <rule pattern="\["><token type="Punctuation"/><push/></rule>
+      <rule pattern="\]"><token type="Punctuation"/><pop depth="1"/></rule>
+    </state>
+  </rules>
+</lexer>
+

lexers/bbcode.xml

@@ -0,0 +1,22 @@
+
+<lexer>
+  <config>
+    <name>BBCode</name>
+    <alias>bbcode</alias>
+    <mime_type>text/x-bbcode</mime_type>
+  </config>
+  <rules>
+    <state name="root">
+      <rule pattern="[^[]+"><token type="Text"/></rule>
+      <rule pattern="\[/?\w+"><token type="Keyword"/><push state="tag"/></rule>
+      <rule pattern="\["><token type="Text"/></rule>
+    </state>
+    <state name="tag">
+      <rule pattern="\s+"><token type="Text"/></rule>
+      <rule pattern="(\w+)(=)(&quot;?[^\s&quot;\]]+&quot;?)"><bygroups><token type="NameAttribute"/><token type="Operator"/><token type="LiteralString"/></bygroups></rule>
+      <rule pattern="(=)(&quot;?[^\s&quot;\]]+&quot;?)"><bygroups><token type="Operator"/><token type="LiteralString"/></bygroups></rule>
+      <rule pattern="\]"><token type="Keyword"/><pop depth="1"/></rule>
+    </state>
+  </rules>
+</lexer>
+

lexers/markdown.xml

@@ -0,0 +1,56 @@
+
+<lexer>
+  <config>
+    <name>Markdown</name>
+    <alias>markdown</alias>
+    <alias>md</alias>
+    <filename>*.md</filename>
+    <filename>*.markdown</filename>
+    <mime_type>text/x-markdown</mime_type>
+  </config>
+  <rules>
+    <state name="root">
+      <rule pattern="(^#[^#].+)(\n)"><bygroups><token type="GenericHeading"/><token type="Text"/></bygroups></rule>
+      <rule pattern="(^#{2,6}[^#].+)(\n)"><bygroups><token type="GenericSubheading"/><token type="Text"/></bygroups></rule>
+      <rule pattern="^(.+)(\n)(=+)(\n)"><bygroups><token type="GenericHeading"/><token type="Text"/><token type="GenericHeading"/><token type="Text"/></bygroups></rule>
+      <rule pattern="^(.+)(\n)(-+)(\n)"><bygroups><token type="GenericSubheading"/><token type="Text"/><token type="GenericSubheading"/><token type="Text"/></bygroups></rule>
+      <rule pattern="^(\s*)([*-] )(\[[ xX]\])( .+\n)"><bygroups><token type="TextWhitespace"/><token type="Keyword"/><token type="Keyword"/><usingself state="inline"/></bygroups></rule>
+      <rule pattern="^(\s*)([*-])(\s)(.+\n)"><bygroups><token type="TextWhitespace"/><token type="Keyword"/><token type="TextWhitespace"/><usingself state="inline"/></bygroups></rule>
+      <rule pattern="^(\s*)([0-9]+\.)( .+\n)"><bygroups><token type="TextWhitespace"/><token type="Keyword"/><usingself state="inline"/></bygroups></rule>
+      <rule pattern="^(\s*&gt;\s)(.+\n)"><bygroups><token type="Keyword"/><token type="GenericEmph"/></bygroups></rule>
+      <rule pattern="^(```\n)([\w\W]*?)(^```$)">
+      <bygroups>
+        <token type="LiteralStringBacktick"/>
+        <token type="Text"/>
+        <token type="LiteralStringBacktick"/>
+      </bygroups>
+      </rule>
+      <rule pattern="^(```)(\w+)(\n)([\w\W]*?)(^```$)">
+        <bygroups>
+          <token type="LiteralStringBacktick"/>
+          <token type="NameLabel"/>  
+          <token type="TextWhitespace"/>
+          <UsingByGroup lexer="2" content="4"/>  
+          <token type="LiteralStringBacktick"/>
+        </bygroups>
+      </rule>
+      <rule><include state="inline"/></rule>
+    </state>
+    <state name="inline">
+      <rule pattern="\\."><token type="Text"/></rule>
+      <rule pattern="([^`]?)(`[^`\n]+`)"><bygroups><token type="Text"/><token type="LiteralStringBacktick"/></bygroups></rule>
+      <rule pattern="([^\*]?)(\*\*[^* \n][^*\n]*\*\*)"><bygroups><token type="Text"/><token type="GenericStrong"/></bygroups></rule>
+      <rule pattern="([^_]?)(__[^_ \n][^_\n]*__)"><bygroups><token type="Text"/><token type="GenericStrong"/></bygroups></rule>
+      <rule pattern="([^\*]?)(\*[^* \n][^*\n]*\*)"><bygroups><token type="Text"/><token type="GenericEmph"/></bygroups></rule>
+      <rule pattern="([^_]?)(_[^_ \n][^_\n]*_)"><bygroups><token type="Text"/><token type="GenericEmph"/></bygroups></rule>
+      <rule pattern="([^~]?)(~~[^~ \n][^~\n]*~~)"><bygroups><token type="Text"/><token type="GenericDeleted"/></bygroups></rule>
+      <rule pattern="[@#][\w/:]+"><token type="NameEntity"/></rule>
+      <rule pattern="(!?\[)([^]]+)(\])(\()([^)]+)(\))"><bygroups><token type="Text"/><token type="NameTag"/><token type="Text"/><token type="Text"/><token type="NameAttribute"/><token type="Text"/></bygroups></rule>
+      <rule pattern="(\[)([^]]+)(\])(\[)([^]]*)(\])"><bygroups><token type="Text"/><token type="NameTag"/><token type="Text"/><token type="Text"/><token type="NameLabel"/><token type="Text"/></bygroups></rule>
+      <rule pattern="^(\s*\[)([^]]*)(\]:\s*)(.+)"><bygroups><token type="Text"/><token type="NameLabel"/><token type="Text"/><token type="NameAttribute"/></bygroups></rule>
+      <rule pattern="[^\\\s]+"><token type="Text"/></rule>
+      <rule pattern="."><token type="Text"/></rule>
+    </state>
+  </rules>
+</lexer>
+

lexers/moinwiki.xml

@@ -0,0 +1,34 @@
+
+<lexer>
+  <config>
+    <name>MoinMoin/Trac Wiki markup</name>
+    <alias>trac-wiki</alias>
+    <alias>moin</alias>
+    <mime_type>text/x-trac-wiki</mime_type>
+    <case_insensitive>true</case_insensitive>
+  </config>
+  <rules>
+    <state name="root">
+      <rule pattern="^#.*$"><token type="Comment"/></rule>
+      <rule pattern="(!)(\S+)"><bygroups><token type="Keyword"/><token type="Text"/></bygroups></rule>
+      <rule pattern="^(=+)([^=]+)(=+)(\s*#.+)?$"><bygroups><token type="GenericHeading"/><usingself state="root"/><token type="GenericHeading"/><token type="LiteralString"/></bygroups></rule>
+      <rule pattern="(\{\{\{)(\n#!.+)?"><bygroups><token type="NameBuiltin"/><token type="NameNamespace"/></bygroups><push state="codeblock"/></rule>
+      <rule pattern="(\&#x27;\&#x27;\&#x27;?|\|\||`|__|~~|\^|,,|::)"><token type="Comment"/></rule>
+      <rule pattern="^( +)([.*-])( )"><bygroups><token type="Text"/><token type="NameBuiltin"/><token type="Text"/></bygroups></rule>
+      <rule pattern="^( +)([a-z]{1,5}\.)( )"><bygroups><token type="Text"/><token type="NameBuiltin"/><token type="Text"/></bygroups></rule>
+      <rule pattern="\[\[\w+.*?\]\]"><token type="Keyword"/></rule>
+      <rule pattern="(\[[^\s\]]+)(\s+[^\]]+?)?(\])"><bygroups><token type="Keyword"/><token type="LiteralString"/><token type="Keyword"/></bygroups></rule>
+      <rule pattern="^----+$"><token type="Keyword"/></rule>
+      <rule pattern="[^\n\&#x27;\[{!_~^,|]+"><token type="Text"/></rule>
+      <rule pattern="\n"><token type="Text"/></rule>
+      <rule pattern="."><token type="Text"/></rule>
+    </state>
+    <state name="codeblock">
+      <rule pattern="\}\}\}"><token type="NameBuiltin"/><pop depth="1"/></rule>
+      <rule pattern="\{\{\{"><token type="Text"/><push/></rule>
+      <rule pattern="[^{}]+"><token type="CommentPreproc"/></rule>
+      <rule pattern="."><token type="CommentPreproc"/></rule>
+    </state>
+  </rules>
+</lexer>
+

scripts/lexer_metadata.py

@@ -40,15 +40,18 @@ for fname in glob.glob("lexers/*.xml"):
 with open("src/constants/lexers.cr", "w") as f:
     f.write("module Tartrazine\n")
     f.write("  LEXERS_BY_NAME = {\n")
-    for k, v in lexer_by_name.items():
+    for k in sorted(lexer_by_name.keys()):
+        v = lexer_by_name[k]
         f.write(f'"{k}" => "{v}", \n')
     f.write("}\n")
     f.write("  LEXERS_BY_MIMETYPE = {\n")
-    for k, v in lexer_by_mimetype.items():
+    for k in sorted(lexer_by_mimetype.keys()):
+        v = lexer_by_mimetype[k]
         f.write(f'"{k}" => "{v}", \n')
     f.write("}\n")
     f.write("  LEXERS_BY_FILENAME = {\n")
-    for k, v in lexer_by_filename.items():
+    for k in sorted(lexer_by_filename.keys()):
+        v = lexer_by_filename[k]
         f.write(f'"{k}" => {str(list(v)).replace("'", "\"")}, \n')
     f.write("}\n")
     f.write("end\n")

shard.yml

@@ -1,5 +1,5 @@
 name: tartrazine
-version: 0.3.0
+version: 0.6.0
 
 authors:
   - Roberto Alsina <roberto.alsina@gmail.com>

spec/tartrazine_spec.cr

@@ -14,15 +14,18 @@ unicode_problems = {
   "#{__DIR__}/tests/java/test_string_literals.txt",
   "#{__DIR__}/tests/json/test_strings.txt",
   "#{__DIR__}/tests/systemd/example1.txt",
+  "#{__DIR__}/tests/c++/test_unicode_identifiers.txt",
 }
 
 # These testcases fail because of differences in the way chroma and tartrazine tokenize
 # but tartrazine is correct
 bad_in_chroma = {
   "#{__DIR__}/tests/bash_session/test_comment_after_prompt.txt",
+  "#{__DIR__}/tests/html/javascript_backtracking.txt",
   "#{__DIR__}/tests/java/test_default.txt",
   "#{__DIR__}/tests/java/test_multiline_string.txt",
   "#{__DIR__}/tests/java/test_numeric_literals.txt",
+  "#{__DIR__}/tests/octave/test_multilinecomment.txt",
   "#{__DIR__}/tests/php/test_string_escaping_run.txt",
   "#{__DIR__}/tests/python_2/test_cls_builtin.txt",
 }
@@ -30,19 +33,14 @@ bad_in_chroma = {
 known_bad = {
   "#{__DIR__}/tests/bash_session/fake_ps2_prompt.txt",
   "#{__DIR__}/tests/bash_session/prompt_in_output.txt",
-  "#{__DIR__}/tests/bash_session/test_newline_in_echo_no_ps2.txt",
-  "#{__DIR__}/tests/bash_session/test_newline_in_ls_ps2.txt",
   "#{__DIR__}/tests/bash_session/ps2_prompt.txt",
-  "#{__DIR__}/tests/bash_session/test_newline_in_ls_no_ps2.txt",
+  "#{__DIR__}/tests/bash_session/test_newline_in_echo_no_ps2.txt",
-  "#{__DIR__}/tests/bash_session/test_virtualenv.txt",
   "#{__DIR__}/tests/bash_session/test_newline_in_echo_ps2.txt",
-  "#{__DIR__}/tests/c/test_string_resembling_decl_end.txt",
+  "#{__DIR__}/tests/bash_session/test_newline_in_ls_no_ps2.txt",
-  "#{__DIR__}/tests/html/css_backtracking.txt",
+  "#{__DIR__}/tests/bash_session/test_newline_in_ls_ps2.txt",
+  "#{__DIR__}/tests/bash_session/test_virtualenv.txt",
   "#{__DIR__}/tests/mcfunction/data.txt",
   "#{__DIR__}/tests/mcfunction/selectors.txt",
-  "#{__DIR__}/tests/php/anonymous_class.txt",
-  "#{__DIR__}/tests/html/javascript_unclosed.txt",
-
 }
 
 # Tests that fail because of a limitation in PCRE2
@@ -74,8 +72,8 @@ end
 
 # Helper that creates lexer and tokenizes
 def tokenize(lexer_name, text)
-  lexer = Tartrazine.lexer(lexer_name)
+  tokenizer = Tartrazine.lexer(lexer_name).tokenizer(text)
-  lexer.tokenize(text)
+  Tartrazine::Lexer.collapse_tokens(tokenizer.to_a)
 end
 
 # Helper that tokenizes using chroma to validate the lexer

src/actions.cr

@@ -8,12 +8,33 @@ require "./tartrazine"
 # perform a list of actions. These actions can emit tokens
 # or change the state machine.
 module Tartrazine
-  class Action
+  enum ActionType
-    property type : String
+    Bygroups
-    property xml : XML::Node
+    Combined
+    Include
+    Pop
+    Push
+    Token
+    Using
+    Usingbygroup
+    Usingself
+  end
+
+  struct Action
     property actions : Array(Action) = [] of Action
 
-    def initialize(@type : String, @xml : XML::Node?)
+    @content_index : Int32 = 0
+    @depth : Int32 = 0
+    @lexer_index : Int32 = 0
+    @lexer_name : String = ""
+    @states : Array(String) = [] of String
+    @states_to_push : Array(String) = [] of String
+    @token_type : String = ""
+    @type : ActionType = ActionType::Token
+
+    def initialize(t : String, xml : XML::Node?)
+      @type = ActionType.parse(t.capitalize)
+
       # Some actions may have actions in them, like this:
       # <bygroups>
       # <token type="GenericPrompt"/>
@@ -23,48 +44,56 @@ module Tartrazine
       #
       # The token actions match with the first 2 groups in the regex
       # the using action matches the 3rd and shunts it to another lexer
-      @xml.children.each do |node|
+      xml.children.each do |node|
         next unless node.element?
         @actions << Action.new(node.name, node)
       end
+
+      # Prefetch the attributes we ned from the XML and keep them
+      case @type
+      when ActionType::Token
+        @token_type = xml["type"]
+      when ActionType::Push
+        @states_to_push = xml.attributes.select { |attrib|
+          attrib.name == "state"
+        }.map &.content
+      when ActionType::Pop
+        @depth = xml["depth"].to_i
+      when ActionType::Using
+        @lexer_name = xml["lexer"].downcase
+      when ActionType::Combined
+        @states = xml.attributes.select { |attrib|
+          attrib.name == "state"
+        }.map &.content
+      when ActionType::Usingbygroup
+        @lexer_index = xml["lexer"].to_i
+        @content_index = xml["content"].to_i
+      end
     end
 
     # ameba:disable Metrics/CyclomaticComplexity
-    def emit(match : Regex::MatchData?, lexer : Lexer, match_group = 0) : Array(Token)
+    def emit(match : MatchData, tokenizer : Tokenizer, match_group = 0) : Array(Token)
-      case type
+      case @type
-      when "token"
+      when ActionType::Token
-        raise Exception.new "Can't have a token without a match" if match.nil?
+        raise Exception.new "Can't have a token without a match" if match.empty?
-        [Token.new(type: xml["type"], value: match[match_group])]
+        [Token.new(type: @token_type, value: String.new(match[match_group].value))]
-      when "push"
+      when ActionType::Push
-        states_to_push = xml.attributes.select { |attrib|
+        to_push = @states_to_push.empty? ? [tokenizer.state_stack.last] : @states_to_push
-          attrib.name == "state"
+        to_push.each do |state|
-        }.map &.content
+          if state == "#pop" && tokenizer.state_stack.size > 1
-        if states_to_push.empty?
-          # Push without a state means push the current state
-          states_to_push = [lexer.state_stack.last]
-        end
-        states_to_push.each do |state|
-          if state == "#pop"
             # Pop the state
-            Log.trace { "Popping state" }
+            tokenizer.state_stack.pop
-            lexer.state_stack.pop
           else
             # Really push
-            lexer.state_stack << state
+            tokenizer.state_stack << state
-            Log.trace { "Pushed #{lexer.state_stack}" }
           end
         end
         [] of Token
-      when "pop"
+      when ActionType::Pop
-        depth = xml["depth"].to_i
+        to_pop = [@depth, tokenizer.state_stack.size - 1].min
-        Log.trace { "Popping #{depth} states" }
+        tokenizer.state_stack.pop(to_pop)
-        if lexer.state_stack.size <= depth
-          Log.trace { "Can't pop #{depth} states, only have #{lexer.state_stack.size}" }
-        else
-          lexer.state_stack.pop(depth)
-        end
         [] of Token
-      when "bygroups"
+      when ActionType::Bygroups
         # FIXME: handle
         # ><bygroups>
         # <token type="Punctuation"/>
@@ -79,38 +108,46 @@ module Tartrazine
         # the action is skipped.
         result = [] of Token
         @actions.each_with_index do |e, i|
-          next if match[i + 1]?.nil?
+          begin
-          result += e.emit(match, lexer, i + 1)
+            next if match[i + 1].size == 0
+          rescue IndexError
+            # FIXME: This should not actually happen
+            # No match for this group
+            next
+          end
+          result += e.emit(match, tokenizer, i + 1)
         end
         result
-      when "using"
+      when ActionType::Using
         # Shunt to another lexer entirely
-        return [] of Token if match.nil?
+        return [] of Token if match.empty?
-        lexer_name = xml["lexer"].downcase
+        Tartrazine.lexer(@lexer_name).tokenizer(
-        Log.trace { "to tokenize: #{match[match_group]}" }
+          String.new(match[match_group].value),
-        Tartrazine.lexer(lexer_name).tokenize(match[match_group], usingself: true)
+          secondary: true).to_a
-      when "usingself"
+      when ActionType::Usingself
         # Shunt to another copy of this lexer
-        return [] of Token if match.nil?
+        return [] of Token if match.empty?
-
+        tokenizer.lexer.tokenizer(
-        new_lexer = Lexer.from_xml(lexer.xml)
+          String.new(match[match_group].value),
-        Log.trace { "to tokenize: #{match[match_group]}" }
+          secondary: true).to_a
-        new_lexer.tokenize(match[match_group], usingself: true)
+      when ActionType::Combined
-      when "combined"
+        # Combine two or more states into one anonymous state
-        # Combine two states into one anonymous state
+        new_state = @states.map { |name|
-        states = xml.attributes.select { |attrib|
+          tokenizer.lexer.states[name]
-          attrib.name == "state"
-        }.map &.content
-        new_state = states.map { |name|
-          lexer.states[name]
         }.reduce { |state1, state2|
           state1 + state2
         }
-        lexer.states[new_state.name] = new_state
+        tokenizer.lexer.states[new_state.name] = new_state
-        lexer.state_stack << new_state.name
+        tokenizer.state_stack << new_state.name
         [] of Token
+      when ActionType::Usingbygroup
+        # Shunt to content-specified lexer
+        return [] of Token if match.empty?
+        Tartrazine.lexer(String.new(match[@lexer_index].value)).tokenizer(
+          String.new(match[@content_index].value),
+          secondary: true).to_a
       else
-        raise Exception.new("Unknown action type: #{type}: #{xml}")
+        raise Exception.new("Unknown action type: #{@type}")
       end
     end
   end

src/bytes_regex.cr

@@ -0,0 +1,73 @@
+module BytesRegex
+  extend self
+
+  class Regex
+    def initialize(pattern : String, multiline = false, dotall = false, ignorecase = false, anchored = false)
+      flags = LibPCRE2::UTF | LibPCRE2::UCP | LibPCRE2::NO_UTF_CHECK
+      flags |= LibPCRE2::MULTILINE if multiline
+      flags |= LibPCRE2::DOTALL if dotall
+      flags |= LibPCRE2::CASELESS if ignorecase
+      flags |= LibPCRE2::ANCHORED if anchored
+      if @re = LibPCRE2.compile(
+           pattern,
+           pattern.bytesize,
+           flags,
+           out errorcode,
+           out erroroffset,
+           nil)
+      else
+        msg = String.new(256) do |buffer|
+          bytesize = LibPCRE2.get_error_message(errorcode, buffer, 256)
+          {bytesize, 0}
+        end
+        raise Exception.new "Error #{msg} compiling regex at offset #{erroroffset}"
+      end
+      @match_data = LibPCRE2.match_data_create_from_pattern(@re, nil)
+    end
+
+    def finalize
+      LibPCRE2.match_data_free(@match_data)
+      LibPCRE2.code_free(@re)
+    end
+
+    def match(str : Bytes, pos = 0) : Array(Match)
+      rc = LibPCRE2.match(
+        @re,
+        str,
+        str.size,
+        pos,
+        LibPCRE2::NO_UTF_CHECK,
+        @match_data,
+        nil)
+      if rc > 0
+        ovector = LibPCRE2.get_ovector_pointer(@match_data)
+        (0...rc).map do |i|
+          m_start = ovector[2 * i]
+          m_end = ovector[2 * i + 1]
+          if m_start == m_end
+            m_value = Bytes.new(0)
+          else
+            m_value = str[m_start...m_end]
+          end
+          Match.new(m_value, m_start, m_end - m_start)
+        end
+      else
+        [] of Match
+      end
+    end
+  end
+
+  struct Match
+    property value : Bytes
+    property start : UInt64
+    property size : UInt64
+
+    def initialize(@value : Bytes, @start : UInt64, @size : UInt64)
+    end
+  end
+end
+
+# pattern = "foo"
+# str = "foo bar"
+# re = BytesRegex::Regex.new(pattern)
+# p! String.new(re.match(str.to_slice)[0].value)

src/formatter.cr

@@ -12,6 +12,10 @@ module Tartrazine
     property theme : Theme = Tartrazine.theme("default-dark")
 
     # Format the text using the given lexer.
+    def format(text : String, lexer : Lexer, io : IO = nil) : Nil
+      raise Exception.new("Not implemented")
+    end
+
     def format(text : String, lexer : Lexer) : String
       raise Exception.new("Not implemented")
     end

src/formatters/ansi.cr

@@ -7,17 +7,27 @@ module Tartrazine
     def initialize(@theme : Theme = Tartrazine.theme("default-dark"), @line_numbers : Bool = false)
     end
 
+    private def line_label(i : Int32) : String
+      "#{i + 1}".rjust(4).ljust(5)
+    end
+
     def format(text : String, lexer : Lexer) : String
-      output = String.build do |outp|
+      outp = String::Builder.new("")
-        lexer.group_tokens_in_lines(lexer.tokenize(text)).each_with_index do |line, i|
+      format(text, lexer, outp)
-          label = line_numbers? ? "#{i + 1}".rjust(4).ljust(5) : ""
+      outp.to_s
-          outp << label
+    end
-          line.each do |token|
+
-            outp << colorize(token[:value], token[:type])
+    def format(text : String, lexer : BaseLexer, outp : IO) : Nil
-          end
+      tokenizer = lexer.tokenizer(text)
+      i = 0
+      outp << line_label(i) if line_numbers?
+      tokenizer.each do |token|
+        outp << colorize(token[:value], token[:type])
+        if token[:value].includes?("\n")
+          i += 1
+          outp << line_label(i) if line_numbers?
         end
       end
-      output
     end
 
     def colorize(text : String, token : String) : String

src/formatters/html.cr

@@ -1,5 +1,6 @@
 require "../constants/token_abbrevs.cr"
 require "../formatter"
+require "html"
 
 module Tartrazine
   class Html < Formatter
@@ -30,50 +31,56 @@ module Tartrazine
                    @standalone : Bool = false,
                    @surrounding_pre : Bool = true,
                    @wrap_long_lines : Bool = false,
-                   @weight_of_bold : Int32 = 600,)
+                   @weight_of_bold : Int32 = 600)
     end
 
     def format(text : String, lexer : Lexer) : String
-      text = format_text(text, lexer)
+      outp = String::Builder.new("")
-      if standalone?
+      format(text, lexer, outp)
-        text = wrap_standalone(text)
+      outp.to_s
-      end
+    end
-      text
+
+    def format(text : String, lexer : BaseLexer, io : IO) : Nil
+      pre, post = wrap_standalone
+      io << pre if standalone?
+      format_text(text, lexer, io)
+      io << post if standalone?
     end
 
     # Wrap text into a full HTML document, including the CSS for the theme
-    def wrap_standalone(text) : String
+    def wrap_standalone
       output = String.build do |outp|
         outp << "<!DOCTYPE html><html><head><style>"
         outp << style_defs
         outp << "</style></head><body>"
-        outp << text
-        outp << "</body></html>"
       end
-      output
+      {output.to_s, "</body></html>"}
     end
 
-    def format_text(text : String, lexer : Lexer) : String
+    private def line_label(i : Int32) : String
-      lines = lexer.group_tokens_in_lines(lexer.tokenize(text))
+      line_label = "#{i + 1}".rjust(4).ljust(5)
-      output = String.build do |outp|
+      line_class = highlighted?(i + 1) ? "class=\"#{get_css_class("LineHighlight")}\"" : ""
-        if surrounding_pre?
+      line_id = linkable_line_numbers? ? "id=\"#{line_number_id_prefix}#{i + 1}\"" : ""
-          pre_style = wrap_long_lines? ? "style=\"white-space: pre-wrap; word-break: break-word;\"" : ""
+      "<span #{line_id} #{line_class} style=\"user-select: none;\">#{line_label} </span>"
-          outp << "<pre class=\"#{get_css_class("Background")}\" #{pre_style}>"
+    end
-        end
+
-        outp << "<code class=\"#{get_css_class("Background")}\">"
+    def format_text(text : String, lexer : BaseLexer, outp : IO)
-        lines.each_with_index(offset: line_number_start - 1) do |line, i|
+      tokenizer = lexer.tokenizer(text)
-          line_label = line_numbers? ? "#{i + 1}".rjust(4).ljust(5) : ""
+      i = 0
-          line_class = highlighted?(i + 1) ? "class=\"#{get_css_class("LineHighlight")}\"" : ""
+      if surrounding_pre?
-          line_id = linkable_line_numbers? ? "id=\"#{line_number_id_prefix}#{i + 1}\"" : ""
+        pre_style = wrap_long_lines? ? "style=\"white-space: pre-wrap; word-break: break-word;\"" : ""
-          outp << "<span #{line_id} #{line_class} style=\"user-select: none;\">#{line_label} </span>"
+        outp << "<pre class=\"#{get_css_class("Background")}\" #{pre_style}>"
-          line.each do |token|
-            fragment = "<span class=\"#{get_css_class(token[:type])}\">#{token[:value]}</span>"
-            outp << fragment
-          end
-        end
-        outp << "</code></pre>"
       end
-      output
+      outp << "<code class=\"#{get_css_class("Background")}\">"
+      outp << line_label(i) if line_numbers?
+      tokenizer.each do |token|
+        outp << "<span class=\"#{get_css_class(token[:type])}\">#{HTML.escape(token[:value])}</span>"
+        if token[:value].ends_with? "\n"
+          i += 1
+          outp << line_label(i) if line_numbers?
+        end
+      end
+      outp << "</code></pre>"
     end
 
     # ameba:disable Metrics/CyclomaticComplexity
@@ -104,15 +111,17 @@ module Tartrazine
 
     # Given a token type, return the CSS class to use.
     def get_css_class(token : String) : String
-      return class_prefix + Abbreviations[token] if theme.styles.has_key?(token)
+      if !theme.styles.has_key? token
-
+        # Themes don't contain information for each specific
-      # Themes don't contain information for each specific
+        # token type. However, they may contain information
-      # token type. However, they may contain information
+        # for a parent style. Worst case, we go to the root
-      # for a parent style. Worst case, we go to the root
+        # (Background) style.
-      # (Background) style.
+        parent = theme.style_parents(token).reverse.find { |dad|
-      class_prefix + Abbreviations[theme.style_parents(token).reverse.find { |parent|
+          theme.styles.has_key?(dad)
-        theme.styles.has_key?(parent)
+        }
-      }]
+        theme.styles[token] = theme.styles[parent]
+      end
+      class_prefix + Abbreviations[token]
     end
 
     # Is this line in the highlighted ranges?

src/formatters/json.cr

@@ -4,8 +4,15 @@ module Tartrazine
   class Json < Formatter
     property name = "json"
 
-    def format(text : String, lexer : Lexer, _theme : Theme) : String
+    def format(text : String, lexer : BaseLexer) : String
-      lexer.tokenize(text).to_json
+      outp = String::Builder.new("")
+      format(text, lexer, outp)
+      outp.to_s
+    end
+
+    def format(text : String, lexer : BaseLexer, io : IO) : Nil
+      tokenizer = lexer.tokenizer(text)
+      io << Tartrazine::Lexer.collapse_tokens(tokenizer.to_a).to_json
     end
   end
 end

src/lexer.cr

@@ -1,107 +1,154 @@
+require "baked_file_system"
 require "./constants/lexers"
 
 module Tartrazine
   class LexerFiles
     extend BakedFileSystem
-
     bake_folder "../lexers", __DIR__
   end
 
   # Get the lexer object for a language name
   # FIXME: support mimetypes
-  def self.lexer(name : String? = nil, filename : String? = nil) : Lexer
+  def self.lexer(name : String? = nil, filename : String? = nil) : BaseLexer
-    if name.nil? && filename.nil?
+    return lexer_by_name(name) if name && name != "autodetect"
-      lexer_file_name = LEXERS_BY_NAME["plaintext"]
+    return lexer_by_filename(filename) if filename
-    elsif name && name != "autodetect"
+
-      lexer_file_name = LEXERS_BY_NAME[name.downcase]
+    Lexer.from_xml(LexerFiles.get("/#{LEXERS_BY_NAME["plaintext"]}.xml").gets_to_end)
-    else
+  end
-      # Guess by filename
+
-      candidates = Set(String).new
+  private def self.lexer_by_name(name : String) : BaseLexer
-      LEXERS_BY_FILENAME.each do |k, v|
+    lexer_file_name = LEXERS_BY_NAME.fetch(name.downcase, nil)
-        candidates += v.to_set if File.match?(k, File.basename(filename.to_s))
+    return create_delegating_lexer(name) if lexer_file_name.nil? && name.includes? "+"
-      end
+    raise Exception.new("Unknown lexer: #{name}") if lexer_file_name.nil?
-      case candidates.size
+
-      when 0
-        lexer_file_name = LEXERS_BY_NAME["plaintext"]
-      when 1
-        lexer_file_name = candidates.first
-      else
-        raise Exception.new("Multiple lexers match the filename: #{candidates.to_a.join(", ")}")
-      end
-    end
     Lexer.from_xml(LexerFiles.get("/#{lexer_file_name}.xml").gets_to_end)
   end
 
+  private def self.lexer_by_filename(filename : String) : BaseLexer
+    candidates = Set(String).new
+    LEXERS_BY_FILENAME.each do |k, v|
+      candidates += v.to_set if File.match?(k, File.basename(filename))
+    end
+
+    case candidates.size
+    when 0
+      lexer_file_name = LEXERS_BY_NAME["plaintext"]
+    when 1
+      lexer_file_name = candidates.first
+    else
+      raise Exception.new("Multiple lexers match the filename: #{candidates.to_a.join(", ")}")
+    end
+
+    Lexer.from_xml(LexerFiles.get("/#{lexer_file_name}.xml").gets_to_end)
+  end
+
+  private def self.create_delegating_lexer(name : String) : BaseLexer
+    language, root = name.split("+", 2)
+    language_lexer = lexer(language)
+    root_lexer = lexer(root)
+    DelegatingLexer.new(language_lexer, root_lexer)
+  end
+
   # Return a list of all lexers
   def self.lexers : Array(String)
     LEXERS_BY_NAME.keys.sort!
   end
 
+  # A token, the output of the tokenizer
+  alias Token = NamedTuple(type: String, value: String)
+
+  abstract class BaseTokenizer
+  end
+
+  class Tokenizer < BaseTokenizer
+    include Iterator(Token)
+    property lexer : BaseLexer
+    property text : Bytes
+    property pos : Int32 = 0
+    @dq = Deque(Token).new
+    property state_stack = ["root"]
+
+    def initialize(@lexer : BaseLexer, text : String, secondary = false)
+      # Respect the `ensure_nl` config option
+      if text.size > 0 && text[-1] != '\n' && @lexer.config[:ensure_nl] && !secondary
+        text += "\n"
+      end
+      @text = text.to_slice
+    end
+
+    def next : Iterator::Stop | Token
+      if @dq.size > 0
+        return @dq.shift
+      end
+      if pos == @text.size
+        return stop
+      end
+
+      matched = false
+      while @pos < @text.size
+        @lexer.states[@state_stack.last].rules.each do |rule|
+          matched, new_pos, new_tokens = rule.match(@text, @pos, self)
+          if matched
+            @pos = new_pos
+            split_tokens(new_tokens).each { |token| @dq << token }
+            break
+          end
+        end
+        if !matched
+          if @text[@pos] == 10u8
+            @dq << {type: "Text", value: "\n"}
+            @state_stack = ["root"]
+          else
+            @dq << {type: "Error", value: String.new(@text[@pos..@pos])}
+          end
+          @pos += 1
+          break
+        end
+      end
+      self.next
+    end
+
+    # If a token contains a newline, split it into two tokens
+    def split_tokens(tokens : Array(Token)) : Array(Token)
+      split_tokens = [] of Token
+      tokens.each do |token|
+        if token[:value].includes?("\n")
+          values = token[:value].split("\n")
+          values.each_with_index do |value, index|
+            value += "\n" if index < values.size - 1
+            split_tokens << {type: token[:type], value: value}
+          end
+        else
+          split_tokens << token
+        end
+      end
+      split_tokens
+    end
+  end
+
+  abstract class BaseLexer
+    property config = {
+      name:             "",
+      priority:         0.0,
+      case_insensitive: false,
+      dot_all:          false,
+      not_multiline:    false,
+      ensure_nl:        false,
+    }
+    property states = {} of String => State
+
+    def tokenizer(text : String, secondary = false) : BaseTokenizer
+      Tokenizer.new(self, text, secondary)
+    end
+  end
+
   # This implements a lexer for Pygments RegexLexers as expressed
   # in Chroma's XML serialization.
   #
   # For explanations on what actions and states do
   # the Pygments documentation is a good place to start.
   # https://pygments.org/docs/lexerdevelopment/
-  class Lexer
+  class Lexer < BaseLexer
-    property config = {
-      name:             "",
-      aliases:          [] of String,
-      filenames:        [] of String,
-      mime_types:       [] of String,
-      priority:         0.0,
-      case_insensitive: false,
-      dot_all:          false,
-      not_multiline:    false,
-      ensure_nl:        false,
-    }
-    property xml : String = ""
-
-    property states = {} of String => State
-
-    property state_stack = ["root"]
-
-    # Turn the text into a list of tokens. The `usingself` parameter
-    # is true when the lexer is being used to tokenize a string
-    # from a larger text that is already being tokenized.
-    # So, when it's true, we don't modify the text.
-    def tokenize(text, usingself = false) : Array(Token)
-      @state_stack = ["root"]
-      tokens = [] of Token
-      pos = 0
-      matched = false
-
-      # Respect the `ensure_nl` config option
-      if text.size > 0 && text[-1] != '\n' && config[:ensure_nl] && !usingself
-        text += "\n"
-      end
-
-      # Loop through the text, applying rules
-      while pos < text.size
-        state = states[@state_stack.last]
-        # Log.trace { "Stack is #{@state_stack} State is #{state.name}, pos is #{pos}, text is #{text[pos..pos + 10]}" }
-        state.rules.each do |rule|
-          matched, new_pos, new_tokens = rule.match(text, pos, self)
-          if matched
-            # Move position forward, save the tokens,
-            # tokenize from the new position
-            # Log.trace { "MATCHED: #{rule.xml}" }
-            pos = new_pos
-            tokens += new_tokens
-            break
-          end
-          # Log.trace { "NOT MATCHED: #{rule.xml}" }
-        end
-        # If no rule matches, emit an error token
-        unless matched
-          # Log.trace { "Error at #{pos}" }
-          tokens << {type: "Error", value: "#{text[pos]}"}
-          pos += 1
-        end
-      end
-      Lexer.collapse_tokens(tokens)
-    end
-
     # Collapse consecutive tokens of the same type for easier comparison
     # and smaller output
     def self.collapse_tokens(tokens : Array(Tartrazine::Token)) : Array(Tartrazine::Token)
@@ -124,34 +171,8 @@ module Tartrazine
       result
     end
 
-    # Group tokens into lines, splitting them when a newline is found
-    def group_tokens_in_lines(tokens : Array(Token)) : Array(Array(Token))
-      split_tokens = [] of Token
-      tokens.each do |token|
-        if token[:value].includes?("\n")
-          values = token[:value].split("\n")
-          values.each_with_index do |value, index|
-            value += "\n" if index < values.size - 1
-            split_tokens << {type: token[:type], value: value}
-          end
-        else
-          split_tokens << token
-        end
-      end
-      lines = [Array(Token).new]
-      split_tokens.each do |token|
-        lines.last << token
-        if token[:value].includes?("\n")
-          lines << Array(Token).new
-        end
-      end
-      lines
-    end
-
-    # ameba:disable Metrics/CyclomaticComplexity
     def self.from_xml(xml : String) : Lexer
       l = Lexer.new
-      l.xml = xml
       lexer = XML.parse(xml).first_element_child
       if lexer
         config = lexer.children.find { |node|
@@ -160,9 +181,6 @@ module Tartrazine
         if config
           l.config = {
             name:             xml_to_s(config, name) || "",
-            aliases:          xml_to_a(config, _alias) || [] of String,
-            filenames:        xml_to_a(config, filename) || [] of String,
-            mime_types:       xml_to_a(config, mime_type) || [] of String,
             priority:         xml_to_f(config, priority) || 0.0,
             not_multiline:    xml_to_s(config, not_multiline) == "true",
             dot_all:          xml_to_s(config, dot_all) == "true",
@@ -212,12 +230,66 @@ module Tartrazine
     end
   end
 
+  # A lexer that takes two lexers as arguments. A root lexer
+  # and a language lexer. Everything is scalled using the
+  # language lexer, afterwards all `Other` tokens are lexed
+  # using the root lexer.
+  #
+  # This is useful for things like template languages, where
+  # you have Jinja + HTML or Jinja + CSS and so on.
+  class DelegatingLexer < BaseLexer
+    property language_lexer : BaseLexer
+    property root_lexer : BaseLexer
+
+    def initialize(@language_lexer : BaseLexer, @root_lexer : BaseLexer)
+    end
+
+    def tokenizer(text : String, secondary = false) : DelegatingTokenizer
+      DelegatingTokenizer.new(self, text, secondary)
+    end
+  end
+
+  # This Tokenizer works with a DelegatingLexer. It first tokenizes
+  # using the language lexer, and "Other" tokens are tokenized using
+  # the root lexer.
+  class DelegatingTokenizer < BaseTokenizer
+    include Iterator(Token)
+    @dq = Deque(Token).new
+    @language_tokenizer : BaseTokenizer
+
+    def initialize(@lexer : DelegatingLexer, text : String, secondary = false)
+      # Respect the `ensure_nl` config option
+      if text.size > 0 && text[-1] != '\n' && @lexer.config[:ensure_nl] && !secondary
+        text += "\n"
+      end
+      @language_tokenizer = @lexer.language_lexer.tokenizer(text, true)
+    end
+
+    def next : Iterator::Stop | Token
+      if @dq.size > 0
+        return @dq.shift
+      end
+      token = @language_tokenizer.next
+      if token.is_a? Iterator::Stop
+        return stop
+      elsif token.as(Token).[:type] == "Other"
+        root_tokenizer = @lexer.root_lexer.tokenizer(token.as(Token).[:value], true)
+        root_tokenizer.each do |root_token|
+          @dq << root_token
+        end
+      else
+        @dq << token.as(Token)
+      end
+      self.next
+    end
+  end
+
   # A Lexer state. A state has a name and a list of rules.
   # The state machine has a state stack containing references
   # to states to decide which rules to apply.
-  class State
+  struct State
     property name : String = ""
-    property rules = [] of Rule
+    property rules = [] of BaseRule
 
     def +(other : State)
       new_state = State.new
@@ -226,7 +298,4 @@ module Tartrazine
       new_state
     end
   end
-
-  # A token, the output of the tokenizer
-  alias Token = NamedTuple(type: String, value: String)
 end

src/main.cr

@@ -77,7 +77,7 @@ if options["-f"]
 
   if formatter.is_a?(Tartrazine::Html) && options["--css"]
     File.open("#{options["-t"].as(String)}.css", "w") do |outf|
-      outf.puts formatter.style_defs
+      outf << formatter.style_defs
     end
     exit 0
   end
@@ -85,13 +85,12 @@ if options["-f"]
   lexer = Tartrazine.lexer(name: options["-l"].as(String), filename: options["FILE"].as(String))
 
   input = File.open(options["FILE"].as(String)).gets_to_end
-  output = formatter.format(input, lexer)
 
   if options["-o"].nil?
-    puts output
+    outf = STDOUT
   else
-    File.open(options["-o"].as(String), "w") do |outf|
+    outf = File.open(options["-o"].as(String), "w")
-      outf.puts output
-    end
   end
+  formatter.format(input, lexer, outf)
+  outf.close
 end

src/rules.cr

@@ -1,8 +1,9 @@
 require "./actions"
+require "./bytes_regex"
 require "./formatter"
+require "./lexer"
 require "./rules"
 require "./styles"
-require "./lexer"
 
 # These are lexer rules. They match with the text being parsed
 # and perform actions, either emitting tokens or changing the
@@ -10,40 +11,15 @@ require "./lexer"
 module Tartrazine
   # This rule matches via a regex pattern
 
-  class Rule
+  alias Regex = BytesRegex::Regex
-    property pattern : Regex = Regex.new ""
+  alias Match = BytesRegex::Match
-    property actions : Array(Action) = [] of Action
+  alias MatchData = Array(Match)
-    property xml : String = "foo"
 
-    def match(text, pos, lexer) : Tuple(Bool, Int32, Array(Token))
+  abstract struct BaseRule
-      match = pattern.match(text, pos)
+    abstract def match(text : Bytes, pos : Int32, tokenizer : Tokenizer) : Tuple(Bool, Int32, Array(Token))
-      # We don't match if the match doesn't move the cursor
+    abstract def initialize(node : XML::Node)
-      # because that causes infinite loops
-      return false, pos, [] of Token if match.nil? || match.end == 0
-      # Log.trace { "#{match}, #{pattern.inspect}, #{text}, #{pos}" }
-      tokens = [] of Token
-      # Emit the tokens
-      actions.each do |action|
-        # Emit the token
-        tokens += action.emit(match, lexer)
-      end
-      Log.trace { "#{xml}, #{match.end}, #{tokens}" }
-      return true, match.end, tokens
-    end
 
-    def initialize(node : XML::Node, multiline, dotall, ignorecase)
+    @actions : Array(Action) = [] of Action
-      @xml = node.to_s
-      pattern = node["pattern"]
-      flags = Regex::Options::ANCHORED
-      # MULTILINE implies DOTALL which we don't want, so we
-      # use in-pattern flag (?m) instead
-      # flags |= Regex::Options::MULTILINE if multiline
-      pattern = "(?m)" + pattern if multiline
-      flags |= Regex::Options::DOTALL if dotall
-      flags |= Regex::Options::IGNORE_CASE if ignorecase
-      @pattern = Regex.new(pattern, flags)
-      add_actions(node)
-    end
 
     def add_actions(node : XML::Node)
       node.children.each do |child|
@@ -53,23 +29,42 @@ module Tartrazine
     end
   end
 
+  struct Rule < BaseRule
+    property pattern : Regex = Regex.new ""
+
+    def match(text : Bytes, pos, tokenizer) : Tuple(Bool, Int32, Array(Token))
+      match = pattern.match(text, pos)
+
+      # No match
+      return false, pos, [] of Token if match.size == 0
+      return true, pos + match[0].size, @actions.flat_map(&.emit(match, tokenizer))
+    end
+
+    def initialize(node : XML::Node)
+    end
+
+    def initialize(node : XML::Node, multiline, dotall, ignorecase)
+      pattern = node["pattern"]
+      pattern = "(?m)" + pattern if multiline
+      @pattern = Regex.new(pattern, multiline, dotall, ignorecase, true)
+      add_actions(node)
+    end
+  end
+
   # This rule includes another state. If any of the rules of the
   # included state matches, this rule matches.
-  class IncludeStateRule < Rule
+  struct IncludeStateRule < BaseRule
-    property state : String = ""
+    @state : String = ""
 
-    def match(text, pos, lexer) : Tuple(Bool, Int32, Array(Token))
+    def match(text : Bytes, pos : Int32, tokenizer : Tokenizer) : Tuple(Bool, Int32, Array(Token))
-      Log.trace { "Including state #{state} from #{lexer.state_stack.last}" }
+      tokenizer.@lexer.states[@state].rules.each do |rule|
-      lexer.states[state].rules.each do |rule|
+        matched, new_pos, new_tokens = rule.match(text, pos, tokenizer)
-        matched, new_pos, new_tokens = rule.match(text, pos, lexer)
-        Log.trace { "#{xml}, #{new_pos}, #{new_tokens}" } if matched
         return true, new_pos, new_tokens if matched
       end
       return false, pos, [] of Token
     end
 
     def initialize(node : XML::Node)
-      @xml = node.to_s
       include_node = node.children.find { |child|
         child.name == "include"
       }
@@ -79,17 +74,14 @@ module Tartrazine
   end
 
   # This rule always matches, unconditionally
-  class UnconditionalRule < Rule
+  struct UnconditionalRule < BaseRule
-    def match(text, pos, lexer) : Tuple(Bool, Int32, Array(Token))
+    NO_MATCH = [] of Match
-      tokens = [] of Token
+
-      actions.each do |action|
+    def match(text, pos, tokenizer) : Tuple(Bool, Int32, Array(Token))
-        tokens += action.emit(nil, lexer)
+      return true, pos, @actions.flat_map(&.emit(NO_MATCH, tokenizer))
-      end
-      return true, pos, tokens
     end
 
     def initialize(node : XML::Node)
-      @xml = node.to_s
       add_actions(node)
     end
   end

src/styles.cr

@@ -9,7 +9,7 @@ require "xml"
 module Tartrazine
   alias Color = Sixteen::Color
 
-  class ThemeFiles
+  struct ThemeFiles
     extend BakedFileSystem
     bake_folder "../styles", __DIR__
   end
@@ -39,7 +39,7 @@ module Tartrazine
     themes.to_a.sort!
   end
 
-  class Style
+  struct Style
     # These properties are tri-state.
     # true means it's set
     # false means it's not set
@@ -79,7 +79,7 @@ module Tartrazine
     end
   end
 
-  class Theme
+  struct Theme
     property name : String = ""
 
     property styles = {} of String => Style