From 38196d6e96d729972b902bc3dcc2ccbf432d1932 Mon Sep 17 00:00:00 2001
From: Roberto Alsina <roberto.alsina@gmail.com>
Date: Sat, 24 Aug 2024 19:49:02 -0300
Subject: [PATCH] Rst lexer

---
 TODO.md                 |  4 ++-
 lexers/TwigLexer.xml    | 47 -------------------------
 lexers/rst.xml          | 76 +++++++++++++++++++++++++++++++++++++++++
 src/actions.cr          | 10 ++++--
 src/constants/lexers.cr | 35 +++++++++++--------
 5 files changed, 107 insertions(+), 65 deletions(-)
 delete mode 100644 lexers/TwigLexer.xml
 create mode 100644 lexers/rst.xml

diff --git a/TODO.md b/TODO.md
index b53a8b1..91d427d 100644
--- a/TODO.md
+++ b/TODO.md
@@ -10,4 +10,6 @@
 * ✅ Add --line-numbers to terminal formatter
 * Implement lexer loader by mime type
 * ✅ Implement Delegating lexers
-* Add RstLexer maybe others?
+* ✅ Add RstLexer
+* Add Mako template lexer
+* Implement heuristic lexer detection
diff --git a/lexers/TwigLexer.xml b/lexers/TwigLexer.xml
deleted file mode 100644
index 3413120..0000000
--- a/lexers/TwigLexer.xml
+++ /dev/null
@@ -1,47 +0,0 @@
-
-<lexer>
-  <config>
-    <name>Twig</name>
-    <alias>twig</alias>
-    <mime_type>application/x-twig</mime_type>
-    <dot_all>true</dot_all>
-  </config>
-  <rules>
-    <state name="root">
-      <rule pattern="[^{]+"><token type="Other"/></rule>
-      <rule pattern="\{\{"><token type="CommentPreproc"/><push state="var"/></rule>
-      <rule pattern="\{\#.*?\#\}"><token type="Comment"/></rule>
-      <rule pattern="(\{%)(-?\s*)(raw)(\s*-?)(%\})(.*?)(\{%)(-?\s*)(endraw)(\s*-?)(%\})"><bygroups><token type="CommentPreproc"/><token type="Text"/><token type="Keyword"/><token type="Text"/><token type="CommentPreproc"/><token type="Other"/><token type="CommentPreproc"/><token type="Text"/><token type="Keyword"/><token type="Text"/><token type="CommentPreproc"/></bygroups></rule>
-      <rule pattern="(\{%)(-?\s*)(verbatim)(\s*-?)(%\})(.*?)(\{%)(-?\s*)(endverbatim)(\s*-?)(%\})"><bygroups><token type="CommentPreproc"/><token type="Text"/><token type="Keyword"/><token type="Text"/><token type="CommentPreproc"/><token type="Other"/><token type="CommentPreproc"/><token type="Text"/><token type="Keyword"/><token type="Text"/><token type="CommentPreproc"/></bygroups></rule>
-      <rule pattern="(\{%)(-?\s*)(filter)(\s+)((?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w-]|[^\x00-\x7f])*)"><bygroups><token type="CommentPreproc"/><token type="Text"/><token type="Keyword"/><token type="Text"/><token type="NameFunction"/></bygroups><push state="tag"/></rule>
-      <rule pattern="(\{%)(-?\s*)([a-zA-Z_]\w*)"><bygroups><token type="CommentPreproc"/><token type="Text"/><token type="Keyword"/></bygroups><push state="tag"/></rule>
-      <rule pattern="\{"><token type="Other"/></rule>
-    </state>
-    <state name="varnames">
-      <rule pattern="(\|)(\s*)((?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w-]|[^\x00-\x7f])*)"><bygroups><token type="Operator"/><token type="Text"/><token type="NameFunction"/></bygroups></rule>
-      <rule pattern="(is)(\s+)(not)?(\s*)((?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w-]|[^\x00-\x7f])*)"><bygroups><token type="Keyword"/><token type="Text"/><token type="Keyword"/><token type="Text"/><token type="NameFunction"/></bygroups></rule>
-      <rule pattern="(?i)(true|false|none|null)\b"><token type="KeywordPseudo"/></rule>
-      <rule pattern="(in|not|and|b-and|or|b-or|b-xor|isif|elseif|else|importconstant|defined|divisibleby|empty|even|iterable|odd|sameasmatches|starts\s+with|ends\s+with)\b"><token type="Keyword"/></rule>
-      <rule pattern="(loop|block|parent)\b"><token type="NameBuiltin"/></rule>
-      <rule pattern="(?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w-]|[^\x00-\x7f])*"><token type="NameVariable"/></rule>
-      <rule pattern="\.(?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w-]|[^\x00-\x7f])*"><token type="NameVariable"/></rule>
-      <rule pattern="\.[0-9]+"><token type="LiteralNumber"/></rule>
-      <rule pattern=":?&quot;(\\\\|\\[^\\]|[^&quot;\\])*&quot;"><token type="LiteralStringDouble"/></rule>
-      <rule pattern=":?&#x27;(\\\\|\\[^\\]|[^&#x27;\\])*&#x27;"><token type="LiteralStringSingle"/></rule>
-      <rule pattern="([{}()\[\]+\-*/,:~%]|\.\.|\?|:|\*\*|\/\/|!=|[&gt;&lt;=]=?)"><token type="Operator"/></rule>
-      <rule pattern="[0-9](\.[0-9]*)?(eE[+-][0-9])?[flFLdD]?|0[xX][0-9a-fA-F]+[Ll]?"><token type="LiteralNumber"/></rule>
-    </state>
-    <state name="var">
-      <rule pattern="\s+"><token type="Text"/></rule>
-      <rule pattern="(-?)(\}\})"><bygroups><token type="Text"/><token type="CommentPreproc"/></bygroups><pop depth="1"/></rule>
-      <rule><include state="varnames"/></rule>
-    </state>
-    <state name="tag">
-      <rule pattern="\s+"><token type="Text"/></rule>
-      <rule pattern="(-?)(%\})"><bygroups><token type="Text"/><token type="CommentPreproc"/></bygroups><pop depth="1"/></rule>
-      <rule><include state="varnames"/></rule>
-      <rule pattern="."><token type="Punctuation"/></rule>
-    </state>
-  </rules>
-</lexer>
-
diff --git a/lexers/rst.xml b/lexers/rst.xml
new file mode 100644
index 0000000..f41b9c6
--- /dev/null
+++ b/lexers/rst.xml
@@ -0,0 +1,76 @@
+
+<lexer>
+  <config>
+    <name>reStructuredText</name>
+    <alias>restructuredtext</alias>
+    <alias>rst</alias>
+    <alias>rest</alias>
+    <filename>*.rst</filename>
+    <filename>*.rest</filename>
+    <mime_type>text/x-rst</mime_type>
+    <mime_type>text/prs.fallenstein.rst</mime_type>
+  </config>
+  <rules>
+    <state name="root">
+      <rule pattern="^(=+|-+|`+|:+|\.+|\&#x27;+|&quot;+|~+|\^+|_+|\*+|\++|#+)([ \t]*\n)(.+)(\n)(\1)(\n)"><bygroups><token type="GenericHeading"/><token type="Text"/><token type="GenericHeading"/><token type="Text"/><token type="GenericHeading"/><token type="Text"/></bygroups></rule>
+      <rule pattern="^(\S.*)(\n)(={3,}|-{3,}|`{3,}|:{3,}|\.{3,}|\&#x27;{3,}|&quot;{3,}|~{3,}|\^{3,}|_{3,}|\*{3,}|\+{3,}|#{3,})(\n)"><bygroups><token type="GenericHeading"/><token type="Text"/><token type="GenericHeading"/><token type="Text"/></bygroups></rule>
+      <rule pattern="^(\s*)([-*+])( .+\n(?:\1  .+\n)*)"><bygroups><token type="Text"/><token type="LiteralNumber"/><usingself state="inline"/></bygroups></rule>
+      <rule pattern="^(\s*)([0-9#ivxlcmIVXLCM]+\.)( .+\n(?:\1  .+\n)*)"><bygroups><token type="Text"/><token type="LiteralNumber"/><usingself state="inline"/></bygroups></rule>
+      <rule pattern="^(\s*)(\(?[0-9#ivxlcmIVXLCM]+\))( .+\n(?:\1  .+\n)*)"><bygroups><token type="Text"/><token type="LiteralNumber"/><usingself state="inline"/></bygroups></rule>
+      <rule pattern="^(\s*)([A-Z]+\.)( .+\n(?:\1  .+\n)+)"><bygroups><token type="Text"/><token type="LiteralNumber"/><usingself state="inline"/></bygroups></rule>
+      <rule pattern="^(\s*)(\(?[A-Za-z]+\))( .+\n(?:\1  .+\n)+)"><bygroups><token type="Text"/><token type="LiteralNumber"/><usingself state="inline"/></bygroups></rule>
+      <rule pattern="^(\s*)(\|)( .+\n(?:\|  .+\n)*)"><bygroups><token type="Text"/><token type="Operator"/><usingself state="inline"/></bygroups></rule>
+      <rule pattern="^( *\.\.)(\s*)((?:source)?code(?:-block)?)(::)([ \t]*)([^\n]+)(\n[ \t]*\n)([ \t]+)(.*)(\n)((?:(?:\8.*)?\n)+)"> 
+        <bygroups>
+          <token type="Punctuation"/>
+          <token type="Text"/>
+          <token type="OperatorWord"/>
+          <token type="Punctuation"/>
+          <token type="Text"/>
+          <token type="Keyword"/>
+          <token type="Text"/>
+          <token type="Text"/>
+          <UsingByGroup lexer="6" content="9,10,11"/>
+        </bygroups>
+      </rule>
+      <rule pattern="^( *\.\.)(\s*)([\w:-]+?)(::)(?:([ \t]*)(.*))">
+        <bygroups>
+          <token type="Punctuation"/>
+          <token type="Text"/>
+          <token type="OperatorWord"/>
+          <token type="Punctuation"/>
+          <token type="Text"/>
+          <usingself state="inline"/>
+        </bygroups>
+      </rule>
+      <rule pattern="^( *\.\.)(\s*)(_(?:[^:\\]|\\.)+:)(.*?)$"><bygroups><token type="Punctuation"/><token type="Text"/><token type="NameTag"/><usingself state="inline"/></bygroups></rule>
+      <rule pattern="^( *\.\.)(\s*)(\[.+\])(.*?)$"><bygroups><token type="Punctuation"/><token type="Text"/><token type="NameTag"/><usingself state="inline"/></bygroups></rule>
+      <rule pattern="^( *\.\.)(\s*)(\|.+\|)(\s*)([\w:-]+?)(::)(?:([ \t]*)(.*))"><bygroups><token type="Punctuation"/><token type="Text"/><token type="NameTag"/><token type="Text"/><token type="OperatorWord"/><token type="Punctuation"/><token type="Text"/><usingself state="inline"/></bygroups></rule>
+      <rule pattern="^ *\.\..*(\n( +.*\n|\n)+)?"><token type="Comment"/></rule>
+      <rule pattern="^( *)(:(?:\\\\|\\:|[^:\n])+:(?=\s))([ \t]*)"><bygroups><token type="Text"/><token type="NameClass"/><token type="Text"/></bygroups></rule>
+      <rule pattern="^(\S.*(?&lt;!::)\n)((?:(?: +.*)\n)+)"><bygroups><usingself state="inline"/><usingself state="inline"/></bygroups></rule>
+      <rule pattern="(::)(\n[ \t]*\n)([ \t]+)(.*)(\n)((?:(?:\3.*)?\n)+)"><bygroups><token type="LiteralStringEscape"/><token type="Text"/><token type="LiteralString"/><token type="LiteralString"/><token type="Text"/><token type="LiteralString"/></bygroups></rule>
+      <rule><include state="inline"/></rule>
+    </state>
+    <state name="inline">
+      <rule pattern="\\."><token type="Text"/></rule>
+      <rule pattern="``"><token type="LiteralString"/><push state="literal"/></rule>
+      <rule pattern="(`.+?)(&lt;.+?&gt;)(`__?)"><bygroups><token type="LiteralString"/><token type="LiteralStringInterpol"/><token type="LiteralString"/></bygroups></rule>
+      <rule pattern="`.+?`__?"><token type="LiteralString"/></rule>
+      <rule pattern="(`.+?`)(:[a-zA-Z0-9:-]+?:)?"><bygroups><token type="NameVariable"/><token type="NameAttribute"/></bygroups></rule>
+      <rule pattern="(:[a-zA-Z0-9:-]+?:)(`.+?`)"><bygroups><token type="NameAttribute"/><token type="NameVariable"/></bygroups></rule>
+      <rule pattern="\*\*.+?\*\*"><token type="GenericStrong"/></rule>
+      <rule pattern="\*.+?\*"><token type="GenericEmph"/></rule>
+      <rule pattern="\[.*?\]_"><token type="LiteralString"/></rule>
+      <rule pattern="&lt;.+?&gt;"><token type="NameTag"/></rule>
+      <rule pattern="[^\\\n\[*`:]+"><token type="Text"/></rule>
+      <rule pattern="."><token type="Text"/></rule>
+    </state>
+    <state name="literal">
+      <rule pattern="[^`]+"><token type="LiteralString"/></rule>
+      <rule pattern="``((?=$)|(?=[-/:.,; \n\x00‐‑‒–— &#x27;&quot;\)\]\}&gt;’”»!\?]))"><token type="LiteralString"/><pop depth="1"/></rule>
+      <rule pattern="`"><token type="LiteralString"/></rule>
+    </state>
+  </rules>
+</lexer>
+
diff --git a/src/actions.cr b/src/actions.cr
index 9ae536d..da514a5 100644
--- a/src/actions.cr
+++ b/src/actions.cr
@@ -23,7 +23,7 @@ module Tartrazine
   struct Action
     property actions : Array(Action) = [] of Action
 
-    @content_index : Int32 = 0
+    @content_index : Array(Int32) = [] of Int32
     @depth : Int32 = 0
     @lexer_index : Int32 = 0
     @lexer_name : String = ""
@@ -67,7 +67,7 @@ module Tartrazine
         }.map &.content
       when ActionType::Usingbygroup
         @lexer_index = xml["lexer"].to_i
-        @content_index = xml["content"].to_i
+        @content_index = xml["content"].split(",").map(&.to_i)
       end
     end
 
@@ -143,8 +143,12 @@ module Tartrazine
       when ActionType::Usingbygroup
         # Shunt to content-specified lexer
         return [] of Token if match.empty?
+        content = ""
+        @content_index.each do |i|
+          content += String.new(match[i].value)
+        end
         Tartrazine.lexer(String.new(match[@lexer_index].value)).tokenizer(
-          String.new(match[@content_index].value),
+          content,
           secondary: true).to_a
       else
         raise Exception.new("Unknown action type: #{@type}")
diff --git a/src/constants/lexers.cr b/src/constants/lexers.cr
index 25a253a..1f8a368 100644
--- a/src/constants/lexers.cr
+++ b/src/constants/lexers.cr
@@ -324,10 +324,13 @@ module Tartrazine
     "reg"                         => "reg",
     "registry"                    => "reg",
     "rego"                        => "rego",
+    "rest"                        => "rst",
+    "restructuredtext"            => "rst",
     "rexx"                        => "rexx",
     "rkt"                         => "racket",
     "rpmspec"                     => "rpm_spec",
     "rs"                          => "rust",
+    "rst"                         => "rst",
     "ruby"                        => "ruby",
     "rust"                        => "rust",
     "s"                           => "r",
@@ -395,7 +398,7 @@ module Tartrazine
     "turing"                      => "turing",
     "turtle"                      => "turtle",
     "tv"                          => "tradingview",
-    "twig"                        => "TwigLexer",
+    "twig"                        => "twig",
     "typescript"                  => "typescript",
     "typoscript"                  => "typoscript",
     "typoscriptcssdata"           => "typoscriptcssdata",
@@ -467,7 +470,7 @@ module Tartrazine
     "application/x-fennel"                        => "fennel",
     "application/x-fish"                          => "fish",
     "application/x-forth"                         => "forth",
-    "application/x-gdscript"                      => "gdscript3",
+    "application/x-gdscript"                      => "gdscript",
     "application/x-hcl"                           => "hcl",
     "application/x-hy"                            => "hy",
     "application/x-javascript"                    => "javascript",
@@ -500,7 +503,7 @@ module Tartrazine
     "application/x-thrift"                        => "thrift",
     "application/x-troff"                         => "groff",
     "application/x-turtle"                        => "turtle",
-    "application/x-twig"                          => "TwigLexer",
+    "application/x-twig"                          => "twig",
     "application/x-vue"                           => "vue",
     "application/x.ucode"                         => "ucode",
     "application/xhtml+xml"                       => "html",
@@ -527,6 +530,7 @@ module Tartrazine
     "text/odin"                                   => "odin",
     "text/org"                                    => "org_mode",
     "text/plain"                                  => "plaintext",
+    "text/prs.fallenstein.rst"                    => "rst",
     "text/rust"                                   => "rust",
     "text/s"                                      => "r",
     "text/s-plus"                                 => "r",
@@ -589,7 +593,7 @@ module Tartrazine
     "text/x-fortran"                              => "fortran",
     "text/x-fsharp"                               => "fsharp",
     "text/x-gas"                                  => "gas",
-    "text/x-gdscript"                             => "gdscript3",
+    "text/x-gdscript"                             => "gdscript",
     "text/x-gherkin"                              => "gherkin",
     "text/x-gleam"                                => "gleam",
     "text/x-glslsrc"                              => "glsl",
@@ -657,6 +661,7 @@ module Tartrazine
     "text/x-reasonml"                             => "reasonml",
     "text/x-rexx"                                 => "rexx",
     "text/x-rpm-spec"                             => "rpm_spec",
+    "text/x-rst"                                  => "rst",
     "text/x-ruby"                                 => "ruby",
     "text/x-rust"                                 => "rust",
     "text/x-sas"                                  => "sas",
@@ -725,8 +730,8 @@ module Tartrazine
     "*.applescript"     => ["applescript"],
     "*.aql"             => ["arangodb_aql"],
     "*.arexx"           => ["rexx"],
-    "*.as"              => ["actionscript", "actionscript_3"],
-    "*.asm"             => ["nasm", "z80_assembly", "tasm"],
+    "*.as"              => ["actionscript_3", "actionscript"],
+    "*.asm"             => ["tasm", "nasm", "z80_assembly"],
     "*.au3"             => ["autoit"],
     "*.automount"       => ["systemd"],
     "*.aux"             => ["tex"],
@@ -745,7 +750,7 @@ module Tartrazine
     "*.bnf"             => ["bnf"],
     "*.bqn"             => ["bqn"],
     "*.bzl"             => ["python"],
-    "*.c"               => ["c", "c++"],
+    "*.c"               => ["c++", "c"],
     "*.c++"             => ["c++"],
     "*.capnp"           => ["cap_n_proto"],
     "*.cc"              => ["c++"],
@@ -844,7 +849,7 @@ module Tartrazine
     "*.graphql"         => ["graphql"],
     "*.graphqls"        => ["graphql"],
     "*.groovy"          => ["groovy"],
-    "*.h"               => ["objective-c", "c", "c++"],
+    "*.h"               => ["c++", "c", "objective-c"],
     "*.h++"             => ["c++"],
     "*.ha"              => ["hare"],
     "*.handlebars"      => ["handlebars"],
@@ -852,7 +857,7 @@ module Tartrazine
     "*.hc"              => ["holyc"],
     "*.hc.z"            => ["holyc"],
     "*.hcl"             => ["hcl"],
-    "*.hh"              => ["holyc", "c++"],
+    "*.hh"              => ["c++", "holyc"],
     "*.hlb"             => ["hlb"],
     "*.hlsl"            => ["hlsl"],
     "*.hlsli"           => ["hlsl"],
@@ -867,7 +872,7 @@ module Tartrazine
     "*.idc"             => ["c"],
     "*.idr"             => ["idris"],
     "*.ijs"             => ["j"],
-    "*.inc"             => ["php", "sourcepawn", "objectpascal", "povray"],
+    "*.inc"             => ["objectpascal", "povray", "php", "sourcepawn"],
     "*.inf"             => ["ini"],
     "*.ini"             => ["ini"],
     "*.ino"             => ["arduino"],
@@ -893,7 +898,7 @@ module Tartrazine
     "*.lpk"             => ["objectpascal"],
     "*.lpr"             => ["objectpascal"],
     "*.lua"             => ["lua"],
-    "*.m"               => ["mason", "mathematica", "matlab", "octave", "objective-c"],
+    "*.m"               => ["mathematica", "octave", "matlab", "objective-c", "mason"],
     "*.ma"              => ["mathematica"],
     "*.mak"             => ["makefile"],
     "*.man"             => ["groff"],
@@ -948,7 +953,7 @@ module Tartrazine
     "*.php"             => ["php"],
     "*.php[345]"        => ["php"],
     "*.pig"             => ["pig"],
-    "*.pl"              => ["prolog", "perl"],
+    "*.pl"              => ["perl", "prolog"],
     "*.plc"             => ["plutus_core"],
     "*.plot"            => ["gnuplot"],
     "*.plt"             => ["gnuplot"],
@@ -956,7 +961,7 @@ module Tartrazine
     "*.pml"             => ["promela"],
     "*.pony"            => ["pony"],
     "*.pov"             => ["povray"],
-    "*.pp"              => ["objectpascal", "puppet"],
+    "*.pp"              => ["puppet", "objectpascal"],
     "*.pq"              => ["powerquery"],
     "*.pr"              => ["promela"],
     "*.prm"             => ["promela"],
@@ -992,6 +997,7 @@ module Tartrazine
     "*.reg"             => ["reg"],
     "*.rego"            => ["rego"],
     "*.rei"             => ["reasonml"],
+    "*.rest"            => ["rst"],
     "*.rex"             => ["rexx"],
     "*.rexx"            => ["rexx"],
     "*.rkt"             => ["racket"],
@@ -1001,9 +1007,10 @@ module Tartrazine
     "*.rs"              => ["rust"],
     "*.rs.in"           => ["rust"],
     "*.rss"             => ["xml"],
+    "*.rst"             => ["rst"],
     "*.rvt"             => ["tcl"],
     "*.rx"              => ["rexx"],
-    "*.s"               => ["r", "armasm", "gas"],
+    "*.s"               => ["armasm", "r", "gas"],
     "*.sage"            => ["python"],
     "*.sas"             => ["sas"],
     "*.sass"            => ["sass"],