bump: Release v0.7.0

feat: use the native crystal highlighter

.github/workflows/ci.yml

@@ -0,0 +1,26 @@
+name: Tests
+on:
+  # This can't yet run automatically, because tests fail because of
+  # different versions of chroma. Need to get the same one in my
+  # local env and in CI
+  workflow_dispatch:
+  push:
+permissions:
+  contents: read
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download source
+        uses: actions/checkout@v4
+      - name: Install Crystal
+        uses: crystal-lang/install-crystal@v1
+      - name: Run tests
+        run: |
+          wget https://github.com/alecthomas/chroma/releases/download/v2.14.0/chroma-2.14.0-linux-amd64.tar.gz
+          tar xzvf chroma-2.14.0*gz
+          mkdir ~/.local/bin -p
+          sudo mv chroma ~/.local/bin
+          shards install
+          crystal tool format --check
+          crystal spec -v

.github/workflows/coverage.yml

@@ -0,0 +1,30 @@
+name: Coverage
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 1 * * *"
+permissions:
+  contents: read
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download source
+        uses: actions/checkout@v4
+      - name: Install Crystal
+        uses: crystal-lang/install-crystal@v1
+      - name: Run tests using kcov
+        run: |
+          sudo apt update && sudo apt install kcov
+          wget https://github.com/alecthomas/chroma/releases/download/v2.14.0/chroma-2.14.0-linux-amd64.tar.gz
+          tar xzvf chroma-2.14.0*gz
+          mkdir ~/.local/bin -p
+          sudo mv chroma ~/.local/bin
+          shards install
+          crystal build src/run_tests.cr
+          kcov --clean --include-path=./src $PWD/coverage ./run_tests
+          curl -Os https://uploader.codecov.io/latest/linux/codecov
+          chmod +x codecov
+          ./codecov -t ${CODECOV_TOKEN} -s coverage
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.gitignore

@@ -9,3 +9,6 @@ shard.lock
 .vscode/
 .crystal/
 venv/
+.croupier
+coverage/
+run_tests

.md.rb

@@ -0,0 +1,3 @@
+exclude_rule 'MD033' # Inline HTML
+exclude_rule 'MD005' # 3-space indent for lists
+exclude_rule 'MD024' # Repeated headings

.mdlrc

@@ -0,0 +1 @@
+style ".md.rb"

.pre-commit-config.yaml

@@ -0,0 +1,35 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+
+repos:
+    - repo: https://github.com/pre-commit/pre-commit-hooks
+      rev: v4.6.0
+      hooks:
+          - id: trailing-whitespace
+          - id: end-of-file-fixer
+          - id: check-yaml
+          - id: check-added-large-files
+          - id: check-merge-conflict
+    - repo: https://github.com/jumanjihouse/pre-commit-hooks
+      rev: 3.0.0
+      hooks:
+          - id: shellcheck
+          - id: markdownlint
+            exclude: '^content'
+    - repo: https://github.com/mrtazz/checkmake
+      rev: 0.2.2
+      hooks:
+          - id: checkmake
+            exclude: lexers/makefile.xml
+    - repo: https://github.com/python-jsonschema/check-jsonschema
+      rev: 0.29.2
+      hooks:
+          - id: check-github-workflows
+    - repo: https://github.com/commitizen-tools/commitizen
+      rev: v3.29.0 # automatically updated by Commitizen
+      hooks:
+        - id: commitizen
+        - id: commitizen-branch
+          stages:
+            - post-commit
+            - push

CHANGELOG.md

@@ -0,0 +1,58 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+## [0.7.0] - 2024-09-10
+
+### 🚀 Features
+
+- Higher level API (`to_html` and `to_ansi`)
+- Use the native crystal highlighter
+
+### 🐛 Bug Fixes
+
+- Ameba
+- Variable bame in Hacefile
+- Make it easier to import the Ansi formatter
+- Renamed BaseLexer to Lexer and Lexer to RegexLexer to make API nicer
+- Make install work
+
+### 📚 Documentation
+
+- Mention AUR package
+
+### 🧪 Testing
+
+- Add CI workflows
+
+### ⚙️ Miscellaneous Tasks
+
+- Pre-commit hooks
+- Git-cliff config
+- Started changelog
+- Force conventional commit messages
+- Force conventional commit messages
+- Updated pre-commit
+- *(ignore)* Fix tests
+- Added badges
+- Added badges
+- *(ignore)* Removed random file
+
+### Build
+
+- Switch from Makefile to Hacefile
+- Added do_release script
+- Fix markdown check
+
+### Bump
+
+- Release v0.6.4
+- Release v0.6.4
+
+## [0.6.1] - 2024-08-25
+
+### 📚 Documentation
+
+- Improve readme and help message
+
+<!-- generated by git-cliff -->

Hacefile.yml

@@ -0,0 +1,115 @@
+variables:
+  FLAGS: "-d --error-trace"
+  NAME: "tartrazine"
+
+tasks:
+  build:
+    default: true
+    dependencies:
+      - src
+      - shard.lock
+      - shard.yml
+      - Hacefile.yml
+      - lexers/*xml
+      - styles/*xml
+    outputs:
+      - bin/{{NAME}}
+    commands: |
+      shards build {{FLAGS}}
+
+  get-deps:
+    dependencies:
+      - shard.yml
+    outputs:
+      - shard.lock
+    commands: |
+      shards install
+
+  build-release:
+    phony: true
+    always_run: true
+    commands: |
+      hace build FLAGS="--release"
+
+  install:
+    phony: true
+    always_run: true
+    dependencies:
+      - bin/{{NAME}}
+    commands: |
+      rm ${HOME}/.local/bin/{{NAME}} -f
+      cp bin/{{NAME}} ${HOME}/.local/bin/{{NAME}}
+
+  static:
+    outputs:
+      - bin/{{NAME}}-static-linux-amd64
+      - bin/{{NAME}}-static-linux-arm64
+    commands: |
+      hace clean
+      ./build_static.sh
+
+  test:
+    dependencies:
+      - src
+      - spec
+      - shard.lock
+      - shard.yml
+    commands: |
+      crystal spec -v --error-trace
+    phony: true
+    always_run: true
+
+  lint:
+    dependencies:
+      - src
+      - spec
+      - shard.lock
+      - shard.yml
+    commands: |
+      crystal tool format src/*.cr spec/*.cr
+      ameba --fix
+    always_run: true
+    phony: true
+
+  docs:
+    dependencies:
+      - src
+      - shard.lock
+      - shard.yml
+      - README.md
+    commands: |
+      crystal docs
+    outputs:
+      - docs/index.html
+
+  pre-commit:
+    default: true
+    outputs:
+      - .git/hooks/commit-msg
+      - .git/hooks/pre-commit
+    dependencies:
+      - .pre-commit-config.yaml
+    commands: |
+      pre-commit install --hook-type commit-msg
+      pre-commit install
+
+  clean:
+    phony: true
+    always_run: true
+    commands: |
+      rm -rf shard.lock bin lib
+
+  coverage:
+    dependencies:
+      - src
+      - spec
+      - shard.lock
+      - shard.yml
+    commands: |
+      shards install
+      crystal build -o bin/run_tests src/run_tests.cr
+      rm -rf coverage/
+      mkdir coverage
+      kcov --clean --include-path=./src ${PWD}/coverage ./bin/run_tests
+    outputs:
+      - coverage/index.html

Makefile

@@ -1,7 +0,0 @@
-build: $(wildcard src/**/*.cr) $(wildcard lexers/*xml) $(wildcard styles/*xml) shard.yml
-	shards build -Dstrict_multi_assign -Dno_number_autocast -d --error-trace
-release: $(wildcard src/**/*.cr) $(wildcard lexers/*xml) $(wildcard styles/*xml) shard.yml
-	shards build --release
-static: $(wildcard src/**/*.cr) $(wildcard lexers/*xml) $(wildcard styles/*xml) shard.yml
-	shards build --release --static
-	strip bin/tartrazine

README.md

@@ -1,12 +1,92 @@
 # TARTRAZINE
 
+[![Tests](https://github.com/ralsina/tartrazine/actions/workflows/ci.yml/badge.svg)](https://github.com/ralsina/tartrazine/actions/workflows/ci.yml)
+[![codecov](https://codecov.io/gh/ralsina/tartrazine/branch/main/graph/badge.svg?token=52XBPNL99F)](https://codecov.io/gh/ralsina/tartrazine)
+
 Tartrazine is a library to syntax-highlight code. It is
 a port of [Pygments](https://pygments.org/) to
-[Crystal](https://crystal-lang.org/). Kind of.
+[Crystal](https://crystal-lang.org/).
 
-The CLI tool can be used to highlight many things in many styles.
+It also provides a CLI tool which can be used to highlight many things in many styles.
 
-# A port of what? Why "kind of"?
+Currently Tartrazine supports 247 languages and has 331 themes (63 from Chroma,
+the rest are base16 themes via [Sixteen](https://github.com/ralsina/sixteen)
+
+## Installation
+
+If you are using Arch: Use yay or your favourite AUR helper, package name is `tartrazine`.
+
+From prebuilt binaries:
+
+Each release provides statically-linked binaries that should
+work on any Linux. Get them from the [releases page](https://github.com/ralsina/tartrazine/releases)
+and put them in your PATH.
+
+To build from source:
+
+1. Clone this repo
+2. Run `make` to build the `tartrazine` binary
+3. Copy the binary somewhere in your PATH.
+
+## Usage as a CLI tool
+
+Show a syntax highlighted version of a C source file in your terminal:
+
+```shell
+tartrazine whatever.c -l c -t catppuccin-macchiato --line-numbers -f terminal
+```
+
+Generate a standalone HTML file from a C source file with the syntax highlighted:
+
+```shell
+$ tartrazine whatever.c -t catppuccin-macchiato --line-numbers \
+  --standalone -f html -o whatever.html
+```
+
+## Usage as a Library
+
+This is the high level API:
+
+```crystal
+require "tartrazine"
+
+html = Tartrazine.to_html(
+  "puts \"Hello, world!\"",
+  language: "crystal",
+  theme: "catppuccin-macchiato",
+  standalone: true,
+  line_numbers: true
+)
+```
+
+This does more or less the same thing, but more manually:
+
+```crystal
+lexer = Tartrazine.lexer("crystal")
+formatter = Tartrazine::Html.new (
+  theme: Tartrazine.theme("catppuccin-macchiato"),
+  line_numbers: true,
+  standalone: true,
+)
+puts formatter.format("puts \"Hello, world!\"", lexer)
+```
+
+The reason you may want to use the manual version is to reuse
+the lexer and formatter objects for performance reasons.
+
+## Contributing
+
+1. Fork it (<https://github.com/ralsina/tartrazine/fork>)
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request
+
+## Contributors
+
+- [Roberto Alsina](https://github.com/ralsina) - creator and maintainer
+
+## A port of what, and why "kind of"
 
 Pygments is a staple of the Python ecosystem, and it's great.
 It lets you highlight code in many languages, and it has many
@@ -20,70 +100,15 @@ Chroma has taken most of the Pygments lexers and turned them into
 XML descriptions. What I did was take those XML files from Chroma
 and a pile of test cases from Pygments, and I slapped them together
 until the tests passed and my code produced the same output as
-Chroma. Think of it as *extreme TDD*.
+Chroma. Think of it as [*extreme TDD*](https://ralsina.me/weblog/posts/tartrazine-reimplementing-pygments.html)
 
 Currently the pass rate for tests in the supported languages
 is `96.8%`, which is *not bad for a couple days hacking*.
 
 This only covers the RegexLexers, which are the most common ones,
 but it means the supported languages are a subset of Chroma's, which
-is a subset of Pygments'.
+is a subset of Pygments' and DelegatingLexers (useful for things like template languages)
 
-Currently Tartrazine supports ... 248 languages.
+Then performance was bad, so I hacked and hacked and made it significantly
-
+[faster than chroma](https://ralsina.me/weblog/posts/a-tale-of-optimization.html)
-It has 331 themes (63 from Chroma, the rest are base16 themes via
+which is fun.
-[Sixteen](https://github.com/ralsina/sixteen)
-
-## Installation
-
-From prebuilt binaries:
-
-Each release provides statically-linked binaries that should
-work on any Linux. Get them from the [releases page](https://github.com/ralsina/tartrazine/releases) and put them in your PATH.
-
-To build from source:
-
-1. Clone this repo
-2. Run `make` to build the `tartrazine` binary
-3. Copy the binary somewhere in your PATH.
-
-## Usage as a CLI tool
-
-Show a syntax highlighted version of a C source file in your terminal:
-
-```shell
-$ tartrazine whatever.c -l c -t catppuccin-macchiato --line-numbers -f terminal
-```
-
-Generate a standalone HTML file from a C source file with the syntax highlighted:
-
-```shell
-$ tartrazine whatever.c -l c -t catppuccin-macchiato --line-numbers \
-  --standalone -f html -o whatever.html 
-```
-
-## Usage as a Library
-
-This works:
-
-```crystal
-require "tartrazine"
-
-lexer = Tartrazine.lexer("crystal")
-theme = Tartrazine.theme("catppuccin-macchiato")
-formatter = Tartrazine::Html.new
-formatter.theme = theme
-puts formatter.format(File.read(ARGV[0]), lexer)
-```
-
-## Contributing
-
-1. Fork it (<https://github.com/ralsina/tartrazine/fork>)
-2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Add some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
-5. Create a new Pull Request
-
-## Contributors
-
-- [Roberto Alsina](https://github.com/ralsina) - creator and maintainer

TODO.md

@@ -8,8 +8,8 @@
 * ✅ Implement lexer loader that respects aliases
 * ✅ Implement lexer loader by file extension
 * ✅ Add --line-numbers to terminal formatter
-* Implement lexer loader by mime type
+* ✅ Implement lexer loader by mime type
 * ✅ Implement Delegating lexers
 * ✅ Add RstLexer
 * Add Mako template lexer
-* Implement heuristic lexer detection
+* ✅ Implement heuristic lexer detection

build_static.sh

@@ -7,10 +7,10 @@ docker run --rm --privileged \
 
 # Build for AMD64
 docker build . -f Dockerfile.static -t tartrazine-builder
-docker run -ti --rm -v "$PWD":/app --user="$UID" tartrazine-builder /bin/sh -c "cd /app && rm -rf lib shard.lock && make static"
+docker run -ti --rm -v "$PWD":/app --user="$UID" tartrazine-builder /bin/sh -c "cd /app && rm -rf lib shard.lock && shards build --static --release"
 mv bin/tartrazine bin/tartrazine-static-linux-amd64
 
 # Build for ARM64
 docker build . -f Dockerfile.static --platform linux/arm64 -t tartrazine-builder
-docker run -ti --rm -v "$PWD":/app --platform linux/arm64 --user="$UID" tartrazine-builder /bin/sh -c "cd /app && rm -rf lib shard.lock && make static"
+docker run -ti --rm -v "$PWD":/app --platform linux/arm64 --user="$UID" tartrazine-builder /bin/sh -c "cd /app && rm -rf lib shard.lock && shards build --static --release"
 mv bin/tartrazine bin/tartrazine-static-linux-arm64

cliff.toml

@@ -0,0 +1,79 @@
+# git-cliff ~ default configuration file
+# https://git-cliff.org/docs/configuration
+#
+# Lines starting with "#" are comments.
+# Configuration options are organized into tables and keys.
+# See documentation for more information on available options.
+
+[changelog]
+# template for the changelog header
+header = """
+# Changelog\n
+All notable changes to this project will be documented in this file.\n
+"""
+# template for the changelog body
+# https://keats.github.io/tera/docs/#introduction
+body = """
+{% if version %}\
+    ## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }}
+{% else %}\
+    ## [unreleased]
+{% endif %}\
+{% for group, commits in commits | group_by(attribute="group") %}
+    ### {{ group | striptags | trim | upper_first }}
+    {% for commit in commits %}
+        - {% if commit.scope %}*({{ commit.scope }})* {% endif %}\
+            {% if commit.breaking %}[**breaking**] {% endif %}\
+            {{ commit.message | upper_first }}\
+    {% endfor %}
+{% endfor %}\n
+"""
+# template for the changelog footer
+footer = """
+<!-- generated by git-cliff -->
+"""
+# remove the leading and trailing s
+trim = true
+# postprocessors
+postprocessors = [
+  # { pattern = '<REPO>', replace = "https://github.com/orhun/git-cliff" }, # replace repository URL
+]
+
+[git]
+# parse the commits based on https://www.conventionalcommits.org
+conventional_commits = true
+# filter out the commits that are not conventional
+filter_unconventional = true
+# process each line of a commit as an individual commit
+split_commits = false
+# regex for preprocessing the commit messages
+commit_preprocessors = [
+  # Replace issue numbers
+  #{ pattern = '\((\w+\s)?#([0-9]+)\)', replace = "([#${2}](<REPO>/issues/${2}))"},
+  # Check spelling of the commit with https://github.com/crate-ci/typos
+  # If the spelling is incorrect, it will be automatically fixed.
+  #{ pattern = '.*', replace_command = 'typos --write-changes -' },
+]
+# regex for parsing and grouping commits
+commit_parsers = [
+  { message = "^feat", group = "<!-- 0 -->🚀 Features" },
+  { message = "^fix", group = "<!-- 1 -->🐛 Bug Fixes" },
+  { message = "^doc", group = "<!-- 3 -->📚 Documentation" },
+  { message = "^perf", group = "<!-- 4 -->⚡ Performance" },
+  { message = "^refactor", group = "<!-- 2 -->🚜 Refactor" },
+  { message = "^style", group = "<!-- 5 -->🎨 Styling" },
+  { message = "^test", group = "<!-- 6 -->🧪 Testing" },
+  { message = "^chore\\(release\\): prepare for", skip = true },
+  { message = "^chore\\(deps.*\\)", skip = true },
+  { message = "^chore\\(pr\\)", skip = true },
+  { message = "^chore\\(pull\\)", skip = true },
+  { message = "^chore|^ci", group = "<!-- 7 -->⚙️ Miscellaneous Tasks" },
+  { body = ".*security", group = "<!-- 8 -->🛡️ Security" },
+  { message = "^revert", group = "<!-- 9 -->◀️ Revert" },
+]
+# filter out the commits that are not matched by commit parsers
+filter_commits = false
+# sort the tags topologically
+topo_order = false
+# sort the commits inside sections by oldest/newest order
+sort_commits = "oldest"

do_release.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+set e
+
+PKGNAME=$(basename "$PWD")
+VERSION=$(git cliff --bumped-version |cut -dv -f2)
+
+sed "s/^version:.*$/version: $VERSION/g" -i shard.yml
+git add shard.yml
+hace lint test
+git cliff --bump -o
+git commit -a -m "bump: Release v$VERSION"
+git tag "v$VERSION"
+git push --tags
+hace static
+gh release create "v$VERSION" "bin/$PKGNAME-static-linux-amd64" "bin/$PKGNAME-static-linux-arm64" --title "Release v$VERSION" --notes "$(git cliff -l -s all)"

lexers/LiquidLexer.xml

@@ -127,4 +127,3 @@
     </state>
   </rules>
 </lexer>
-

lexers/VelocityLexer.xml

@@ -52,4 +52,3 @@
     </state>
   </rules>
 </lexer>
-

lexers/agda.xml

@@ -63,4 +63,3 @@
     </state>
   </rules>
 </lexer>
-

lexers/alloy.xml

@@ -55,4 +55,3 @@
     </state>
   </rules>
 </lexer>
-

lexers/autohotkey.xml

@@ -75,4 +75,3 @@
     </state>
   </rules>
 </lexer>
-

lexers/autoit.xml

@@ -67,4 +67,3 @@
     </state>
   </rules>
 </lexer>
-

lexers/bbcode.xml

@@ -19,4 +19,3 @@
     </state>
   </rules>
 </lexer>
-

lexers/crystal.xml

@@ -1,762 +0,0 @@
-<lexer>
-  <config>
-    <name>Crystal</name>
-    <alias>cr</alias>
-    <alias>crystal</alias>
-    <filename>*.cr</filename>
-    <mime_type>text/x-crystal</mime_type>
-    <dot_all>true</dot_all>
-  </config>
-  <rules>
-    <state name="pa-intp-string">
-      <rule pattern="\\[\(]">
-        <token type="LiteralStringOther"/>
-      </rule>
-      <rule pattern="\(">
-        <token type="LiteralStringOther"/>
-        <push/>
-      </rule>
-      <rule pattern="\)">
-        <token type="LiteralStringOther"/>
-        <pop depth="1"/>
-      </rule>
-      <rule>
-        <include state="string-intp-escaped"/>
-      </rule>
-      <rule pattern="[\\#()]">
-        <token type="LiteralStringOther"/>
-      </rule>
-      <rule pattern="[^\\#()]+">
-        <token type="LiteralStringOther"/>
-      </rule>
-    </state>
-    <state name="ab-regex">
-      <rule pattern="\\[\\&lt;&gt;]">
-        <token type="LiteralStringRegex"/>
-      </rule>
-      <rule pattern="&lt;">
-        <token type="LiteralStringRegex"/>
-        <push/>
-      </rule>
-      <rule pattern="&gt;[imsx]*">
-        <token type="LiteralStringRegex"/>
-        <pop depth="1"/>
-      </rule>
-      <rule>
-        <include state="string-intp"/>
-      </rule>
-      <rule pattern="[\\#&lt;&gt;]">
-        <token type="LiteralStringRegex"/>
-      </rule>
-      <rule pattern="[^\\#&lt;&gt;]+">
-        <token type="LiteralStringRegex"/>
-      </rule>
-    </state>
-    <state name="cb-regex">
-      <rule pattern="\\[\\{}]">
-        <token type="LiteralStringRegex"/>
-      </rule>
-      <rule pattern="\{">
-        <token type="LiteralStringRegex"/>
-        <push/>
-      </rule>
-      <rule pattern="\}[imsx]*">
-        <token type="LiteralStringRegex"/>
-        <pop depth="1"/>
-      </rule>
-      <rule>
-        <include state="string-intp"/>
-      </rule>
-      <rule pattern="[\\#{}]">
-        <token type="LiteralStringRegex"/>
-      </rule>
-      <rule pattern="[^\\#{}]+">
-        <token type="LiteralStringRegex"/>
-      </rule>
-    </state>
-    <state name="simple-backtick">
-      <rule>
-        <include state="string-intp-escaped"/>
-      </rule>
-      <rule pattern="[^\\`#]+">
-        <token type="LiteralStringBacktick"/>
-      </rule>
-      <rule pattern="[\\#]">
-        <token type="LiteralStringBacktick"/>
-      </rule>
-      <rule pattern="`">
-        <token type="LiteralStringBacktick"/>
-        <pop depth="1"/>
-      </rule>
-    </state>
-    <state name="string-intp">
-      <rule pattern="#\{">
-        <token type="LiteralStringInterpol"/>
-        <push state="in-intp"/>
-      </rule>
-    </state>
-    <state name="interpolated-regex">
-      <rule>
-        <include state="string-intp"/>
-      </rule>
-      <rule pattern="[\\#]">
-        <token type="LiteralStringRegex"/>
-      </rule>
-      <rule pattern="[^\\#]+">
-        <token type="LiteralStringRegex"/>
-      </rule>
-    </state>
-    <state name="cb-string">
-      <rule pattern="\\[\\{}]">
-        <token type="LiteralStringOther"/>
-      </rule>
-      <rule pattern="\{">
-        <token type="LiteralStringOther"/>
-        <push/>
-      </rule>
-      <rule pattern="\}">
-        <token type="LiteralStringOther"/>
-        <pop depth="1"/>
-      </rule>
-      <rule pattern="[\\#{}]">
-        <token type="LiteralStringOther"/>
-      </rule>
-      <rule pattern="[^\\#{}]+">
-        <token type="LiteralStringOther"/>
-      </rule>
-    </state>
-    <state name="in-macro-control">
-      <rule pattern="\{%">
-        <token type="LiteralStringInterpol"/>
-        <push/>
-      </rule>
-      <rule pattern="%\}">
-        <token type="LiteralStringInterpol"/>
-        <pop depth="1"/>
-      </rule>
-      <rule pattern="for\b|in\b">
-        <token type="Keyword"/>
-      </rule>
-      <rule>
-        <include state="root"/>
-      </rule>
-    </state>
-    <state name="interpolated-string">
-      <rule>
-        <include state="string-intp"/>
-      </rule>
-      <rule pattern="[\\#]">
-        <token type="LiteralStringOther"/>
-      </rule>
-      <rule pattern="[^\\#]+">
-        <token type="LiteralStringOther"/>
-      </rule>
-    </state>
-    <state name="in-macro-expr">
-      <rule pattern="\{\{">
-        <token type="LiteralStringInterpol"/>
-        <push/>
-      </rule>
-      <rule pattern="\}\}">
-        <token type="LiteralStringInterpol"/>
-        <pop depth="1"/>
-      </rule>
-      <rule>
-        <include state="root"/>
-      </rule>
-    </state>
-    <state name="simple-string">
-      <rule>
-        <include state="string-intp-escaped"/>
-      </rule>
-      <rule pattern="[^\\&#34;#]+">
-        <token type="LiteralStringDouble"/>
-      </rule>
-      <rule pattern="[\\#]">
-        <token type="LiteralStringDouble"/>
-      </rule>
-      <rule pattern="&#34;">
-        <token type="LiteralStringDouble"/>
-        <pop depth="1"/>
-      </rule>
-    </state>
-    <state name="cb-intp-string">
-      <rule pattern="\\[\{]">
-        <token type="LiteralStringOther"/>
-      </rule>
-      <rule pattern="\{">
-        <token type="LiteralStringOther"/>
-        <push/>
-      </rule>
-      <rule pattern="\}">
-        <token type="LiteralStringOther"/>
-        <pop depth="1"/>
-      </rule>
-      <rule>
-        <include state="string-intp-escaped"/>
-      </rule>
-      <rule pattern="[\\#{}]">
-        <token type="LiteralStringOther"/>
-      </rule>
-      <rule pattern="[^\\#{}]+">
-        <token type="LiteralStringOther"/>
-      </rule>
-    </state>
-    <state name="string-intp-escaped">
-      <rule>
-        <include state="string-intp"/>
-      </rule>
-      <rule>
-        <include state="string-escaped"/>
-      </rule>
-    </state>
-    <state name="sb-regex">
-      <rule pattern="\\[\\\[\]]">
-        <token type="LiteralStringRegex"/>
-      </rule>
-      <rule pattern="\[">
-        <token type="LiteralStringRegex"/>
-        <push/>
-      </rule>
-      <rule pattern="\][imsx]*">
-        <token type="LiteralStringRegex"/>
-        <pop depth="1"/>
-      </rule>
-      <rule>
-        <include state="string-intp"/>
-      </rule>
-      <rule pattern="[\\#\[\]]">
-        <token type="LiteralStringRegex"/>
-      </rule>
-      <rule pattern="[^\\#\[\]]+">
-        <token type="LiteralStringRegex"/>
-      </rule>
-    </state>
-    <state name="classname">
-      <rule pattern="[A-Z_]\w*">
-        <token type="NameClass"/>
-      </rule>
-      <rule pattern="(\()(\s*)([A-Z_]\w*)(\s*)(\))">
-        <bygroups>
-          <token type="Punctuation"/>
-          <token type="Text"/>
-          <token type="NameClass"/>
-          <token type="Text"/>
-          <token type="Punctuation"/>
-        </bygroups>
-      </rule>
-      <rule>
-        <pop depth="1"/>
-      </rule>
-    </state>
-    <state name="string-escaped">
-      <rule pattern="\\([\\befnstv#&#34;\&#39;]|x[a-fA-F0-9]{1,2}|[0-7]{1,3})">
-        <token type="LiteralStringEscape"/>
-      </rule>
-    </state>
-    <state name="sb-intp-string">
-      <rule pattern="\\[\[]">
-        <token type="LiteralStringOther"/>
-      </rule>
-      <rule pattern="\[">
-        <token type="LiteralStringOther"/>
-        <push/>
-      </rule>
-      <rule pattern="\]">
-        <token type="LiteralStringOther"/>
-        <pop depth="1"/>
-      </rule>
-      <rule>
-        <include state="string-intp-escaped"/>
-      </rule>
-      <rule pattern="[\\#\[\]]">
-        <token type="LiteralStringOther"/>
-      </rule>
-      <rule pattern="[^\\#\[\]]+">
-        <token type="LiteralStringOther"/>
-      </rule>
-    </state>
-    <state name="pa-regex">
-      <rule pattern="\\[\\()]">
-        <token type="LiteralStringRegex"/>
-      </rule>
-      <rule pattern="\(">
-        <token type="LiteralStringRegex"/>
-        <push/>
-      </rule>
-      <rule pattern="\)[imsx]*">
-        <token type="LiteralStringRegex"/>
-        <pop depth="1"/>
-      </rule>
-      <rule>
-        <include state="string-intp"/>
-      </rule>
-      <rule pattern="[\\#()]">
-        <token type="LiteralStringRegex"/>
-      </rule>
-      <rule pattern="[^\\#()]+">
-        <token type="LiteralStringRegex"/>
-      </rule>
-    </state>
-    <state name="in-attr">
-      <rule pattern="\[">
-        <token type="Operator"/>
-        <push/>
-      </rule>
-      <rule pattern="\]">
-        <token type="Operator"/>
-        <pop depth="1"/>
-      </rule>
-      <rule>
-        <include state="root"/>
-      </rule>
-    </state>
-    <state name="ab-intp-string">
-      <rule pattern="\\[&lt;]">
-        <token type="LiteralStringOther"/>
-      </rule>
-      <rule pattern="&lt;">
-        <token type="LiteralStringOther"/>
-        <push/>
-      </rule>
-      <rule pattern="&gt;">
-        <token type="LiteralStringOther"/>
-        <pop depth="1"/>
-      </rule>
-      <rule>
-        <include state="string-intp-escaped"/>
-      </rule>
-      <rule pattern="[\\#&lt;&gt;]">
-        <token type="LiteralStringOther"/>
-      </rule>
-      <rule pattern="[^\\#&lt;&gt;]+">
-        <token type="LiteralStringOther"/>
-      </rule>
-    </state>
-    <state name="in-intp">
-      <rule pattern="\{">
-        <token type="LiteralStringInterpol"/>
-        <push/>
-      </rule>
-      <rule pattern="\}">
-        <token type="LiteralStringInterpol"/>
-        <pop depth="1"/>
-      </rule>
-      <rule>
-        <include state="root"/>
-      </rule>
-    </state>
-    <state name="end-part">
-      <rule pattern=".+">
-        <token type="CommentPreproc"/>
-        <pop depth="1"/>
-      </rule>
-    </state>
-    <state name="root">
-      <rule pattern="#.*?$">
-        <token type="CommentSingle"/>
-      </rule>
-      <rule pattern="(instance_sizeof|pointerof|protected|abstract|require|private|include|unless|typeof|sizeof|return|extend|ensure|rescue|ifdef|super|break|begin|until|while|elsif|yield|next|when|else|then|case|with|end|asm|if|do|as|of)\b">
-        <token type="Keyword"/>
-      </rule>
-      <rule pattern="(false|true|nil)\b">
-        <token type="KeywordConstant"/>
-      </rule>
-      <rule pattern="(module|lib)(\s+)([a-zA-Z_]\w*(?:::[a-zA-Z_]\w*)*)">
-        <bygroups>
-          <token type="Keyword"/>
-          <token type="Text"/>
-          <token type="NameNamespace"/>
-        </bygroups>
-      </rule>
-      <rule pattern="(def|fun|macro)(\s+)((?:[a-zA-Z_]\w*::)*)">
-        <bygroups>
-          <token type="Keyword"/>
-          <token type="Text"/>
-          <token type="NameNamespace"/>
-        </bygroups>
-        <push state="funcname"/>
-      </rule>
-      <rule pattern="def(?=[*%&amp;^`~+-/\[&lt;&gt;=])">
-        <token type="Keyword"/>
-        <push state="funcname"/>
-      </rule>
-      <rule pattern="(class|struct|union|type|alias|enum)(\s+)((?:[a-zA-Z_]\w*::)*)">
-        <bygroups>
-          <token type="Keyword"/>
-          <token type="Text"/>
-          <token type="NameNamespace"/>
-        </bygroups>
-        <push state="classname"/>
-      </rule>
-      <rule pattern="(self|out|uninitialized)\b|(is_a|responds_to)\?">
-        <token type="KeywordPseudo"/>
-      </rule>
-      <rule pattern="(def_equals_and_hash|assert_responds_to|forward_missing_to|def_equals|property|def_hash|parallel|delegate|debugger|getter|record|setter|spawn|pp)\b">
-        <token type="NameBuiltinPseudo"/>
-      </rule>
-      <rule pattern="getter[!?]|property[!?]|__(DIR|FILE|LINE)__\b">
-        <token type="NameBuiltinPseudo"/>
-      </rule>
-      <rule pattern="(?&lt;!\.)(get_stack_top|StaticArray|Concurrent|with_color|Reference|Scheduler|read_line|Exception|at_exit|Pointer|Channel|Float64|sprintf|Float32|Process|Object|Struct|caller|UInt16|UInt32|UInt64|system|future|Number|printf|String|Symbol|Int32|Range|Slice|Regex|Mutex|sleep|Array|Class|raise|Tuple|Deque|delay|Float|Int16|print|abort|Value|UInt8|Int64|puts|Proc|File|Void|exit|fork|Bool|Char|gets|lazy|loop|main|rand|Enum|Int8|Time|Hash|Set|Box|Nil|Dir|Int|p)\b">
-        <token type="NameBuiltin"/>
-      </rule>
-      <rule pattern="(?&lt;!\w)(&lt;&lt;-?)([&#34;`\&#39;]?)([a-zA-Z_]\w*)(\2)(.*?\n)">
-        <token type="LiteralStringHeredoc"/>
-      </rule>
-      <rule pattern="(&lt;&lt;-?)(&#34;|\&#39;)()(\2)(.*?\n)">
-        <token type="LiteralStringHeredoc"/>
-      </rule>
-      <rule pattern="__END__">
-        <token type="CommentPreproc"/>
-        <push state="end-part"/>
-      </rule>
-      <rule pattern="(?:^|(?&lt;=[=&lt;&gt;~!:])|(?&lt;=(?:\s|;)when\s)|(?&lt;=(?:\s|;)or\s)|(?&lt;=(?:\s|;)and\s)|(?&lt;=\.index\s)|(?&lt;=\.scan\s)|(?&lt;=\.sub\s)|(?&lt;=\.sub!\s)|(?&lt;=\.gsub\s)|(?&lt;=\.gsub!\s)|(?&lt;=\.match\s)|(?&lt;=(?:\s|;)if\s)|(?&lt;=(?:\s|;)elsif\s)|(?&lt;=^when\s)|(?&lt;=^index\s)|(?&lt;=^scan\s)|(?&lt;=^sub\s)|(?&lt;=^gsub\s)|(?&lt;=^sub!\s)|(?&lt;=^gsub!\s)|(?&lt;=^match\s)|(?&lt;=^if\s)|(?&lt;=^elsif\s))(\s*)(/)">
-        <bygroups>
-          <token type="Text"/>
-          <token type="LiteralStringRegex"/>
-        </bygroups>
-        <push state="multiline-regex"/>
-      </rule>
-      <rule pattern="(?&lt;=\(|,|\[)/">
-        <token type="LiteralStringRegex"/>
-        <push state="multiline-regex"/>
-      </rule>
-      <rule pattern="(\s+)(/)(?![\s=])">
-        <bygroups>
-          <token type="Text"/>
-          <token type="LiteralStringRegex"/>
-        </bygroups>
-        <push state="multiline-regex"/>
-      </rule>
-      <rule pattern="(0o[0-7]+(?:_[0-7]+)*(?:_?[iu][0-9]+)?)\b(\s*)([/?])?">
-        <bygroups>
-          <token type="LiteralNumberOct"/>
-          <token type="Text"/>
-          <token type="Operator"/>
-        </bygroups>
-      </rule>
-      <rule pattern="(0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*(?:_?[iu][0-9]+)?)\b(\s*)([/?])?">
-        <bygroups>
-          <token type="LiteralNumberHex"/>
-          <token type="Text"/>
-          <token type="Operator"/>
-        </bygroups>
-      </rule>
-      <rule pattern="(0b[01]+(?:_[01]+)*(?:_?[iu][0-9]+)?)\b(\s*)([/?])?">
-        <bygroups>
-          <token type="LiteralNumberBin"/>
-          <token type="Text"/>
-          <token type="Operator"/>
-        </bygroups>
-      </rule>
-      <rule pattern="((?:0(?![0-9])|[1-9][\d_]*)(?:\.\d[\d_]*)(?:e[+-]?[0-9]+)?(?:_?f[0-9]+)?)(\s*)([/?])?">
-        <bygroups>
-          <token type="LiteralNumberFloat"/>
-          <token type="Text"/>
-          <token type="Operator"/>
-        </bygroups>
-      </rule>
-      <rule pattern="((?:0(?![0-9])|[1-9][\d_]*)(?:\.\d[\d_]*)?(?:e[+-]?[0-9]+)(?:_?f[0-9]+)?)(\s*)([/?])?">
-        <bygroups>
-          <token type="LiteralNumberFloat"/>
-          <token type="Text"/>
-          <token type="Operator"/>
-        </bygroups>
-      </rule>
-      <rule pattern="((?:0(?![0-9])|[1-9][\d_]*)(?:\.\d[\d_]*)?(?:e[+-]?[0-9]+)?(?:_?f[0-9]+))(\s*)([/?])?">
-        <bygroups>
-          <token type="LiteralNumberFloat"/>
-          <token type="Text"/>
-          <token type="Operator"/>
-        </bygroups>
-      </rule>
-      <rule pattern="(0\b|[1-9][\d]*(?:_\d+)*(?:_?[iu][0-9]+)?)\b(\s*)([/?])?">
-        <bygroups>
-          <token type="LiteralNumberInteger"/>
-          <token type="Text"/>
-          <token type="Operator"/>
-        </bygroups>
-      </rule>
-      <rule pattern="@@[a-zA-Z_]\w*">
-        <token type="NameVariableClass"/>
-      </rule>
-      <rule pattern="@[a-zA-Z_]\w*">
-        <token type="NameVariableInstance"/>
-      </rule>
-      <rule pattern="\$\w+">
-        <token type="NameVariableGlobal"/>
-      </rule>
-      <rule pattern="\$[!@&amp;`\&#39;+~=/\\,;.&lt;&gt;_*$?:&#34;^-]">
-        <token type="NameVariableGlobal"/>
-      </rule>
-      <rule pattern="\$-[0adFiIlpvw]">
-        <token type="NameVariableGlobal"/>
-      </rule>
-      <rule pattern="::">
-        <token type="Operator"/>
-      </rule>
-      <rule>
-        <include state="strings"/>
-      </rule>
-      <rule pattern="\?(\\[MC]-)*(\\([\\befnrtv#&#34;\&#39;]|x[a-fA-F0-9]{1,2}|[0-7]{1,3})|\S)(?!\w)">
-        <token type="LiteralStringChar"/>
-      </rule>
-      <rule pattern="[A-Z][A-Z_]+\b">
-        <token type="NameConstant"/>
-      </rule>
-      <rule pattern="\{%">
-        <token type="LiteralStringInterpol"/>
-        <push state="in-macro-control"/>
-      </rule>
-      <rule pattern="\{\{">
-        <token type="LiteralStringInterpol"/>
-        <push state="in-macro-expr"/>
-      </rule>
-      <rule pattern="(@\[)(\s*)([A-Z]\w*)">
-        <bygroups>
-          <token type="Operator"/>
-          <token type="Text"/>
-          <token type="NameDecorator"/>
-        </bygroups>
-        <push state="in-attr"/>
-      </rule>
-      <rule pattern="(\.|::)(\[\]\?|&lt;=&gt;|===|\[\]=|&gt;&gt;|&amp;&amp;|\*\*|\[\]|\|\||&gt;=|=~|!~|&lt;&lt;|&lt;=|!=|==|&lt;|/|=|-|\+|&gt;|\*|&amp;|%|\^|!|\||~)">
-        <bygroups>
-          <token type="Operator"/>
-          <token type="NameOperator"/>
-        </bygroups>
-      </rule>
-      <rule pattern="(\.|::)([a-zA-Z_]\w*[!?]?|[*%&amp;^`~+\-/\[&lt;&gt;=])">
-        <bygroups>
-          <token type="Operator"/>
-          <token type="Name"/>
-        </bygroups>
-      </rule>
-      <rule pattern="[a-zA-Z_]\w*(?:[!?](?!=))?">
-        <token type="Name"/>
-      </rule>
-      <rule pattern="(\[|\]\??|\*\*|&lt;=&gt;?|&gt;=|&lt;&lt;?|&gt;&gt;?|=~|===|!~|&amp;&amp;?|\|\||\.{1,3})">
-        <token type="Operator"/>
-      </rule>
-      <rule pattern="[-+/*%=&lt;&gt;&amp;!^|~]=?">
-        <token type="Operator"/>
-      </rule>
-      <rule pattern="[(){};,/?:\\]">
-        <token type="Punctuation"/>
-      </rule>
-      <rule pattern="\s+">
-        <token type="Text"/>
-      </rule>
-    </state>
-    <state name="multiline-regex">
-      <rule>
-        <include state="string-intp"/>
-      </rule>
-      <rule pattern="\\\\">
-        <token type="LiteralStringRegex"/>
-      </rule>
-      <rule pattern="\\/">
-        <token type="LiteralStringRegex"/>
-      </rule>
-      <rule pattern="[\\#]">
-        <token type="LiteralStringRegex"/>
-      </rule>
-      <rule pattern="[^\\/#]+">
-        <token type="LiteralStringRegex"/>
-      </rule>
-      <rule pattern="/[imsx]*">
-        <token type="LiteralStringRegex"/>
-        <pop depth="1"/>
-      </rule>
-    </state>
-    <state name="ab-string">
-      <rule pattern="\\[\\&lt;&gt;]">
-        <token type="LiteralStringOther"/>
-      </rule>
-      <rule pattern="&lt;">
-        <token type="LiteralStringOther"/>
-        <push/>
-      </rule>
-      <rule pattern="&gt;">
-        <token type="LiteralStringOther"/>
-        <pop depth="1"/>
-      </rule>
-      <rule pattern="[\\#&lt;&gt;]">
-        <token type="LiteralStringOther"/>
-      </rule>
-      <rule pattern="[^\\#&lt;&gt;]+">
-        <token type="LiteralStringOther"/>
-      </rule>
-    </state>
-    <state name="pa-string">
-      <rule pattern="\\[\\()]">
-        <token type="LiteralStringOther"/>
-      </rule>
-      <rule pattern="\(">
-        <token type="LiteralStringOther"/>
-        <push/>
-      </rule>
-      <rule pattern="\)">
-        <token type="LiteralStringOther"/>
-        <pop depth="1"/>
-      </rule>
-      <rule pattern="[\\#()]">
-        <token type="LiteralStringOther"/>
-      </rule>
-      <rule pattern="[^\\#()]+">
-        <token type="LiteralStringOther"/>
-      </rule>
-    </state>
-    <state name="strings">
-      <rule pattern="\:@{0,2}[a-zA-Z_]\w*[!?]?">
-        <token type="LiteralStringSymbol"/>
-      </rule>
-      <rule pattern="\:@{0,2}(\[\]\?|&lt;=&gt;|===|\[\]=|&gt;&gt;|&amp;&amp;|\*\*|\[\]|\|\||&gt;=|=~|!~|&lt;&lt;|&lt;=|!=|==|&lt;|/|=|-|\+|&gt;|\*|&amp;|%|\^|!|\||~)">
-        <token type="LiteralStringSymbol"/>
-      </rule>
-      <rule pattern=":&#39;(\\\\|\\&#39;|[^&#39;])*&#39;">
-        <token type="LiteralStringSymbol"/>
-      </rule>
-      <rule pattern="&#39;(\\\\|\\&#39;|[^&#39;]|\\[^&#39;\\]+)&#39;">
-        <token type="LiteralStringChar"/>
-      </rule>
-      <rule pattern=":&#34;">
-        <token type="LiteralStringSymbol"/>
-        <push state="simple-sym"/>
-      </rule>
-      <rule pattern="([a-zA-Z_]\w*)(:)(?!:)">
-        <bygroups>
-          <token type="LiteralStringSymbol"/>
-          <token type="Punctuation"/>
-        </bygroups>
-      </rule>
-      <rule pattern="&#34;">
-        <token type="LiteralStringDouble"/>
-        <push state="simple-string"/>
-      </rule>
-      <rule pattern="(?&lt;!\.)`">
-        <token type="LiteralStringBacktick"/>
-        <push state="simple-backtick"/>
-      </rule>
-      <rule pattern="%\{">
-        <token type="LiteralStringOther"/>
-        <push state="cb-intp-string"/>
-      </rule>
-      <rule pattern="%[wi]\{">
-        <token type="LiteralStringOther"/>
-        <push state="cb-string"/>
-      </rule>
-      <rule pattern="%r\{">
-        <token type="LiteralStringRegex"/>
-        <push state="cb-regex"/>
-      </rule>
-      <rule pattern="%\[">
-        <token type="LiteralStringOther"/>
-        <push state="sb-intp-string"/>
-      </rule>
-      <rule pattern="%[wi]\[">
-        <token type="LiteralStringOther"/>
-        <push state="sb-string"/>
-      </rule>
-      <rule pattern="%r\[">
-        <token type="LiteralStringRegex"/>
-        <push state="sb-regex"/>
-      </rule>
-      <rule pattern="%\(">
-        <token type="LiteralStringOther"/>
-        <push state="pa-intp-string"/>
-      </rule>
-      <rule pattern="%[wi]\(">
-        <token type="LiteralStringOther"/>
-        <push state="pa-string"/>
-      </rule>
-      <rule pattern="%r\(">
-        <token type="LiteralStringRegex"/>
-        <push state="pa-regex"/>
-      </rule>
-      <rule pattern="%&lt;">
-        <token type="LiteralStringOther"/>
-        <push state="ab-intp-string"/>
-      </rule>
-      <rule pattern="%[wi]&lt;">
-        <token type="LiteralStringOther"/>
-        <push state="ab-string"/>
-      </rule>
-      <rule pattern="%r&lt;">
-        <token type="LiteralStringRegex"/>
-        <push state="ab-regex"/>
-      </rule>
-      <rule pattern="(%r([\W_]))((?:\\\2|(?!\2).)*)(\2[imsx]*)">
-        <token type="LiteralString"/>
-      </rule>
-      <rule pattern="(%[wi]([\W_]))((?:\\\2|(?!\2).)*)(\2)">
-        <token type="LiteralString"/>
-      </rule>
-      <rule pattern="(?&lt;=[-+/*%=&lt;&gt;&amp;!^|~,(])(\s*)(%([\t ])(?:(?:\\\3|(?!\3).)*)\3)">
-        <bygroups>
-          <token type="Text"/>
-          <token type="LiteralStringOther"/>
-          <token type="None"/>
-        </bygroups>
-      </rule>
-      <rule pattern="^(\s*)(%([\t ])(?:(?:\\\3|(?!\3).)*)\3)">
-        <bygroups>
-          <token type="Text"/>
-          <token type="LiteralStringOther"/>
-          <token type="None"/>
-        </bygroups>
-      </rule>
-      <rule pattern="(%([\[{(&lt;]))((?:\\\2|(?!\2).)*)(\2)">
-        <token type="LiteralString"/>
-      </rule>
-    </state>
-    <state name="sb-string">
-      <rule pattern="\\[\\\[\]]">
-        <token type="LiteralStringOther"/>
-      </rule>
-      <rule pattern="\[">
-        <token type="LiteralStringOther"/>
-        <push/>
-      </rule>
-      <rule pattern="\]">
-        <token type="LiteralStringOther"/>
-        <pop depth="1"/>
-      </rule>
-      <rule pattern="[\\#\[\]]">
-        <token type="LiteralStringOther"/>
-      </rule>
-      <rule pattern="[^\\#\[\]]+">
-        <token type="LiteralStringOther"/>
-      </rule>
-    </state>
-    <state name="funcname">
-      <rule pattern="(?:([a-zA-Z_]\w*)(\.))?([a-zA-Z_]\w*[!?]?|\*\*?|[-+]@?|[/%&amp;|^`~]|\[\]=?|&lt;&lt;|&gt;&gt;|&lt;=?&gt;|&gt;=?|===?)">
-        <bygroups>
-          <token type="NameClass"/>
-          <token type="Operator"/>
-          <token type="NameFunction"/>
-        </bygroups>
-        <pop depth="1"/>
-      </rule>
-      <rule>
-        <pop depth="1"/>
-      </rule>
-    </state>
-    <state name="simple-sym">
-      <rule>
-        <include state="string-escaped"/>
-      </rule>
-      <rule pattern="[^\\&#34;#]+">
-        <token type="LiteralStringSymbol"/>
-      </rule>
-      <rule pattern="[\\#]">
-        <token type="LiteralStringSymbol"/>
-      </rule>
-      <rule pattern="&#34;">
-        <token type="LiteralStringSymbol"/>
-        <pop depth="1"/>
-      </rule>
-    </state>
-  </rules>
-</lexer>

lexers/groff.xml

@@ -3,6 +3,7 @@
     <name>Groff</name>
     <alias>groff</alias>
     <alias>nroff</alias>
+    <alias>roff</alias>
     <alias>man</alias>
     <filename>*.[1-9]</filename>
     <filename>*.1p</filename>

lexers/heuristics.yml

@@ -30,12 +30,12 @@
 disambiguations:
 - extensions: ['.1', '.2', '.3', '.4', '.5', '.6', '.7', '.8', '.9']
   rules:
-  - language: Roff Manpage
+  - language: man
     and:
     - named_pattern: mdoc-date
     - named_pattern: mdoc-title
     - named_pattern: mdoc-heading
-  - language: Roff Manpage
+  - language: man
     and:
     - named_pattern: man-title
     - named_pattern: man-heading
@@ -43,12 +43,12 @@ disambiguations:
     pattern: '^\.(?:[A-Za-z]{2}(?:\s|$)|\\")'
 - extensions: ['.1in', '.1m', '.1x', '.3in', '.3m', '.3p', '.3pm', '.3qt', '.3x', '.man', '.mdoc']
   rules:
-  - language: Roff Manpage
+  - language: man
     and:
     - named_pattern: mdoc-date
     - named_pattern: mdoc-title
     - named_pattern: mdoc-heading
-  - language: Roff Manpage
+  - language: man
     and:
     - named_pattern: man-title
     - named_pattern: man-heading

lexers/markdown.xml

@@ -53,4 +53,3 @@
     </state>
   </rules>
 </lexer>
-

lexers/moinwiki.xml

@@ -31,4 +31,3 @@
     </state>
   </rules>
 </lexer>
-

lexers/rpm_spec.xml

@@ -55,4 +55,3 @@
     </state>
   </rules>
 </lexer>
-

lexers/rst.xml

@@ -73,4 +73,3 @@
     </state>
   </rules>
 </lexer>
-

lexers/smali.xml

@@ -70,4 +70,3 @@
     </state>
   </rules>
 </lexer>
-

lexers/tal.xml

@@ -40,4 +40,3 @@
     </state>
   </rules>
 </lexer>
-

scripts/lexer_metadata.py

@@ -52,6 +52,6 @@ with open("src/constants/lexers.cr", "w") as f:
     f.write("  LEXERS_BY_FILENAME = {\n")
     for k in sorted(lexer_by_filename.keys()):
         v = lexer_by_filename[k]
-        f.write(f'"{k}" => {str(list(v)).replace("'", "\"")}, \n')
+        f.write(f'"{k}" => {str(sorted(list(v))).replace("'", "\"")}, \n')
     f.write("}\n")
     f.write("end\n")

shard.yml

@@ -1,5 +1,5 @@
 name: tartrazine
-version: 0.6.0
+version: 0.7.0
 
 authors:
   - Roberto Alsina <roberto.alsina@gmail.com>
@@ -10,7 +10,8 @@ targets:
 
 dependencies:
   baked_file_system:
-    github: schovi/baked_file_system
+    github: ralsina/baked_file_system
+    branch: master
   base58:
     github: crystal-china/base58.cr
   sixteen:

spec/tartrazine_spec.cr

@@ -28,6 +28,7 @@ bad_in_chroma = {
   "#{__DIR__}/tests/octave/test_multilinecomment.txt",
   "#{__DIR__}/tests/php/test_string_escaping_run.txt",
   "#{__DIR__}/tests/python_2/test_cls_builtin.txt",
+  "#{__DIR__}/tests/bqn/test_syntax_roles.txt", # This one only fails in CI
 }
 
 known_bad = {
@@ -68,12 +69,37 @@ describe Tartrazine do
       end
     end
   end
+
+  describe "to_html" do
+    it "should do basic highlighting" do
+      html = Tartrazine.to_html("puts 'Hello, World!'", "ruby", standalone: false)
+      html.should eq(
+        "<pre class=\"b\" ><code class=\"b\"><span class=\"nb\">puts</span><span class=\"t\"> </span><span class=\"lss\">&#39;Hello, World!&#39;</span></code></pre>"
+      )
+    end
+  end
+  describe "to_ansi" do
+    it "should do basic highlighting" do
+      ansi = Tartrazine.to_ansi("puts 'Hello, World!'", "ruby")
+      if ENV.fetch("CI", nil)
+        # In Github Actions there is no terminal so these don't
+        # really work
+        ansi.should eq(
+          "puts 'Hello, World!'"
+        )
+      else
+        ansi.should eq(
+          "\e[38;2;171;70;66mputs\e[0m\e[38;2;216;216;216m \e[0m'Hello, World!'"
+        )
+      end
+    end
+  end
 end
 
 # Helper that creates lexer and tokenizes
 def tokenize(lexer_name, text)
   tokenizer = Tartrazine.lexer(lexer_name).tokenizer(text)
-  Tartrazine::Lexer.collapse_tokens(tokenizer.to_a)
+  Tartrazine::RegexLexer.collapse_tokens(tokenizer.to_a)
 end
 
 # Helper that tokenizes using chroma to validate the lexer
@@ -85,5 +111,5 @@ def chroma_tokenize(lexer_name, text)
     ["-f", "json", "-l", lexer_name],
     input: input, output: output
   )
-  Tartrazine::Lexer.collapse_tokens(Array(Tartrazine::Token).from_json(output.to_s))
+  Tartrazine::RegexLexer.collapse_tokens(Array(Tartrazine::Token).from_json(output.to_s))
 end

spec/tests/crystal/test_annotation.txt

@@ -1,16 +0,0 @@
----input---
-@[FOO::Bar::Baz(opt: "xx")]
-
----tokens---
-'@['          Operator
-'FOO::Bar::Baz' Name.Decorator
-'('           Punctuation
-'opt'         Literal.String.Symbol
-':'           Punctuation
-' '           Text.Whitespace
-'"'           Literal.String.Double
-'xx'          Literal.String.Double
-'"'           Literal.String.Double
-')'           Punctuation
-']'           Operator
-'\n'          Text.Whitespace

spec/tests/crystal/test_array_access.txt

@@ -1,11 +0,0 @@
----input---
-[5][5]?
-
----tokens---
-'['           Operator
-'5'           Literal.Number.Integer
-']'           Operator
-'['           Operator
-'5'           Literal.Number.Integer
-']?'          Operator
-'\n'          Text.Whitespace

spec/tests/crystal/test_chars.txt

@@ -1,25 +0,0 @@
----input---
-'a'
-'я'
-'\u{1234}'
-'
-'
-'abc'
-
----tokens---
-"'a'"         Literal.String.Char
-'\n'          Text.Whitespace
-
-"'я'"         Literal.String.Char
-'\n'          Text.Whitespace
-
-"'\\u{1234}'" Literal.String.Char
-'\n'          Text.Whitespace
-
-"'\n'"        Literal.String.Char
-'\n'          Text.Whitespace
-
-"'"           Error
-'abc'         Name
-"'"           Error
-'\n'          Text.Whitespace

spec/tests/crystal/test_constant_and_module.txt

@@ -1,14 +0,0 @@
----input---
-HTTP
-HTTP::Server.new
-
----tokens---
-'HTTP'        Name.Constant
-'\n'          Text.Whitespace
-
-'HTTP'        Name
-'::'          Operator
-'Server'      Name
-'.'           Operator
-'new'         Name
-'\n'          Text.Whitespace

spec/tests/crystal/test_empty_percent_strings.txt

@@ -1,27 +0,0 @@
----input---
-%()
-%[]
-%{}
-%<>
-%||
-
----tokens---
-'%('          Literal.String.Other
-')'           Literal.String.Other
-'\n'          Text.Whitespace
-
-'%['          Literal.String.Other
-']'           Literal.String.Other
-'\n'          Text.Whitespace
-
-'%{'          Literal.String.Other
-'}'           Literal.String.Other
-'\n'          Text.Whitespace
-
-'%<'          Literal.String.Other
-'>'           Literal.String.Other
-'\n'          Text.Whitespace
-
-'%|'          Literal.String.Other
-'|'           Literal.String.Other
-'\n'          Text.Whitespace

spec/tests/crystal/test_escaped_bracestring.txt

@@ -1,19 +0,0 @@
----input---
-str.gsub(%r{\\\\}, "/")
-
----tokens---
-'str'         Name
-'.'           Operator
-'gsub'        Name
-'('           Punctuation
-'%r{'         Literal.String.Regex
-'\\\\'        Literal.String.Regex
-'\\\\'        Literal.String.Regex
-'}'           Literal.String.Regex
-','           Punctuation
-' '           Text.Whitespace
-'"'           Literal.String.Double
-'/'           Literal.String.Double
-'"'           Literal.String.Double
-')'           Punctuation
-'\n'          Text.Whitespace

spec/tests/crystal/test_escaped_interpolation.txt

@@ -1,9 +0,0 @@
----input---
-"\#{a + b}"
-
----tokens---
-'"'           Literal.String.Double
-'\\#'         Literal.String.Escape
-'{a + b}'     Literal.String.Double
-'"'           Literal.String.Double
-'\n'          Text.Whitespace

spec/tests/crystal/test_interpolation_nested_curly.txt

@@ -1,56 +0,0 @@
----input---
-"A#{ (3..5).group_by { |x| x/2}.map do |k,v| "#{k}" end.join }" + "Z"
-
----tokens---
-'"'           Literal.String.Double
-'A'           Literal.String.Double
-'#{'          Literal.String.Interpol
-' '           Text.Whitespace
-'('           Punctuation
-'3'           Literal.Number.Integer
-'..'          Operator
-'5'           Literal.Number.Integer
-')'           Punctuation
-'.'           Operator
-'group_by'    Name
-' '           Text.Whitespace
-'{'           Literal.String.Interpol
-' '           Text.Whitespace
-'|'           Operator
-'x'           Name
-'|'           Operator
-' '           Text.Whitespace
-'x'           Name
-'/'           Operator
-'2'           Literal.Number.Integer
-'}'           Literal.String.Interpol
-'.'           Operator
-'map'         Name
-' '           Text.Whitespace
-'do'          Keyword
-' '           Text.Whitespace
-'|'           Operator
-'k'           Name
-','           Punctuation
-'v'           Name
-'|'           Operator
-' '           Text.Whitespace
-'"'           Literal.String.Double
-'#{'          Literal.String.Interpol
-'k'           Name
-'}'           Literal.String.Interpol
-'"'           Literal.String.Double
-' '           Text.Whitespace
-'end'         Keyword
-'.'           Operator
-'join'        Name
-' '           Text.Whitespace
-'}'           Literal.String.Interpol
-'"'           Literal.String.Double
-' '           Text.Whitespace
-'+'           Operator
-' '           Text.Whitespace
-'"'           Literal.String.Double
-'Z'           Literal.String.Double
-'"'           Literal.String.Double
-'\n'          Text.Whitespace

spec/tests/crystal/test_lib.txt

@@ -1,58 +0,0 @@
----input---
-@[Link("some")]
-lib LibSome
-@[CallConvention("X86_StdCall")]
-fun foo="some.foo"(thing : Void*) : LibC::Int
-end
-
----tokens---
-'@['          Operator
-'Link'        Name.Decorator
-'('           Punctuation
-'"'           Literal.String.Double
-'some'        Literal.String.Double
-'"'           Literal.String.Double
-')'           Punctuation
-']'           Operator
-'\n'          Text.Whitespace
-
-'lib'         Keyword
-' '           Text.Whitespace
-'LibSome'     Name.Namespace
-'\n'          Text.Whitespace
-
-'@['          Operator
-'CallConvention' Name.Decorator
-'('           Punctuation
-'"'           Literal.String.Double
-'X86_StdCall' Literal.String.Double
-'"'           Literal.String.Double
-')'           Punctuation
-']'           Operator
-'\n'          Text.Whitespace
-
-'fun'         Keyword
-' '           Text.Whitespace
-'foo'         Name.Function
-'='           Operator
-'"'           Literal.String.Double
-'some.foo'    Literal.String.Double
-'"'           Literal.String.Double
-'('           Punctuation
-'thing'       Name
-' '           Text.Whitespace
-':'           Punctuation
-' '           Text.Whitespace
-'Void'        Name
-'*'           Operator
-')'           Punctuation
-' '           Text.Whitespace
-':'           Punctuation
-' '           Text.Whitespace
-'LibC'        Name
-'::'          Operator
-'Int'         Name
-'\n'          Text.Whitespace
-
-'end'         Keyword
-'\n'          Text.Whitespace

spec/tests/crystal/test_macro.txt

@@ -1,76 +0,0 @@
----input---
-def<=>(other : self) : Int
-{%for field in %w(first_name middle_name last_name)%}
-cmp={{field.id}}<=>other.{{field.id}}
-return cmp if cmp!=0
-{%end%}
-0
-end
-
----tokens---
-'def'         Keyword
-'<=>'         Name.Function
-'('           Punctuation
-'other'       Name
-' '           Text.Whitespace
-':'           Punctuation
-' '           Text.Whitespace
-'self'        Keyword
-')'           Punctuation
-' '           Text.Whitespace
-':'           Punctuation
-' '           Text.Whitespace
-'Int'         Name
-'\n'          Text.Whitespace
-
-'{%'          Literal.String.Interpol
-'for'         Keyword
-' '           Text.Whitespace
-'field'       Name
-' '           Text.Whitespace
-'in'          Keyword
-' '           Text.Whitespace
-'%w('         Literal.String.Other
-'first_name middle_name last_name' Literal.String.Other
-')'           Literal.String.Other
-'%}'          Literal.String.Interpol
-'\n'          Text.Whitespace
-
-'cmp'         Name
-'='           Operator
-'{{'          Literal.String.Interpol
-'field'       Name
-'.'           Operator
-'id'          Name
-'}}'          Literal.String.Interpol
-'<=>'         Operator
-'other'       Name
-'.'           Operator
-'{{'          Literal.String.Interpol
-'field'       Name
-'.'           Operator
-'id'          Name
-'}}'          Literal.String.Interpol
-'\n'          Text.Whitespace
-
-'return'      Keyword
-' '           Text.Whitespace
-'cmp'         Name
-' '           Text.Whitespace
-'if'          Keyword
-' '           Text.Whitespace
-'cmp'         Name
-'!='          Operator
-'0'           Literal.Number.Integer
-'\n'          Text.Whitespace
-
-'{%'          Literal.String.Interpol
-'end'         Keyword
-'%}'          Literal.String.Interpol
-'\n'          Text.Whitespace
-
-'0'           Literal.Number.Integer
-'\n'          Text.Whitespace
-
-'end'         Keyword
-'\n'          Text.Whitespace

spec/tests/crystal/test_numbers.txt

@@ -1,84 +0,0 @@
----input---
-# Integers
-0
-1
-1_000_000
-1u8
-11231231231121312i64
-
-# Floats
-0.0
-1.0_f32
-1_f32
-0f64
-1e+4
-1e111
-1_234.567_890
-
-# Error
-01
-0b2
-0x129g2
-0o12358
-
----tokens---
-'# Integers'  Comment.Single
-'\n'          Text.Whitespace
-
-'0'           Literal.Number.Integer
-'\n'          Text.Whitespace
-
-'1'           Literal.Number.Integer
-'\n'          Text.Whitespace
-
-'1_000_000'   Literal.Number.Integer
-'\n'          Text.Whitespace
-
-'1u8'         Literal.Number.Integer
-'\n'          Text.Whitespace
-
-'11231231231121312i64' Literal.Number.Integer
-'\n\n'        Text.Whitespace
-
-'# Floats'    Comment.Single
-'\n'          Text.Whitespace
-
-'0.0'         Literal.Number.Float
-'\n'          Text.Whitespace
-
-'1.0_f32'     Literal.Number.Float
-'\n'          Text.Whitespace
-
-'1_f32'       Literal.Number.Float
-'\n'          Text.Whitespace
-
-'0f64'        Literal.Number.Float
-'\n'          Text.Whitespace
-
-'1e+4'        Literal.Number.Float
-'\n'          Text.Whitespace
-
-'1e111'       Literal.Number.Float
-'\n'          Text.Whitespace
-
-'1_234.567_890' Literal.Number.Float
-'\n\n'        Text.Whitespace
-
-'# Error'     Comment.Single
-'\n'          Text.Whitespace
-
-'0'           Error
-'1'           Literal.Number.Integer
-'\n'          Text.Whitespace
-
-'0'           Error
-'b2'          Name
-'\n'          Text.Whitespace
-
-'0'           Error
-'x129g2'      Name
-'\n'          Text.Whitespace
-
-'0'           Error
-'o12358'      Name
-'\n'          Text.Whitespace

spec/tests/crystal/test_operator_methods.txt

@@ -1,18 +0,0 @@
----input---
-([] of Int32).[]?(5)
-
----tokens---
-'('           Punctuation
-'['           Operator
-']'           Operator
-' '           Text.Whitespace
-'of'          Keyword
-' '           Text.Whitespace
-'Int32'       Name
-')'           Punctuation
-'.'           Operator
-'[]?'         Name.Operator
-'('           Punctuation
-'5'           Literal.Number.Integer
-')'           Punctuation
-'\n'          Text.Whitespace

spec/tests/crystal/test_percent_strings.txt

@@ -1,41 +0,0 @@
----input---
-%(hello ("world"))
-%[hello ["world"]]
-%{hello "world"}
-%<hello <"world">>
-%|hello "world"|
-
----tokens---
-'%('          Literal.String.Other
-'hello '      Literal.String.Other
-'('           Literal.String.Other
-'"world"'     Literal.String.Other
-')'           Literal.String.Other
-')'           Literal.String.Other
-'\n'          Text.Whitespace
-
-'%['          Literal.String.Other
-'hello '      Literal.String.Other
-'['           Literal.String.Other
-'"world"'     Literal.String.Other
-']'           Literal.String.Other
-']'           Literal.String.Other
-'\n'          Text.Whitespace
-
-'%{'          Literal.String.Other
-'hello "world"' Literal.String.Other
-'}'           Literal.String.Other
-'\n'          Text.Whitespace
-
-'%<'          Literal.String.Other
-'hello '      Literal.String.Other
-'<'           Literal.String.Other
-'"world"'     Literal.String.Other
-'>'           Literal.String.Other
-'>'           Literal.String.Other
-'\n'          Text.Whitespace
-
-'%|'          Literal.String.Other
-'hello "world"' Literal.String.Other
-'|'           Literal.String.Other
-'\n'          Text.Whitespace

spec/tests/crystal/test_percent_strings_special.txt

@@ -1,31 +0,0 @@
----input---
-%Q(hello \n #{name})
-%q(hello \n #{name})
-%w(foo\nbar baz)
-
----tokens---
-'%Q('         Literal.String.Other
-'hello '      Literal.String.Other
-'\\n'         Literal.String.Escape
-' '           Literal.String.Other
-'#{'          Literal.String.Interpol
-'name'        Name
-'}'           Literal.String.Interpol
-')'           Literal.String.Other
-'\n'          Text.Whitespace
-
-'%q('         Literal.String.Other
-'hello '      Literal.String.Other
-'\\'          Literal.String.Other
-'n '          Literal.String.Other
-'#'           Literal.String.Other
-'{name}'      Literal.String.Other
-')'           Literal.String.Other
-'\n'          Text.Whitespace
-
-'%w('         Literal.String.Other
-'foo'         Literal.String.Other
-'\\'          Literal.String.Other
-'nbar baz'    Literal.String.Other
-')'           Literal.String.Other
-'\n'          Text.Whitespace

spec/tests/crystal/test_pseudo_builtins.txt

@@ -1,20 +0,0 @@
----input---
-record Cls do
-def_equals s
-end
-
----tokens---
-'record'      Name.Builtin.Pseudo
-' '           Text.Whitespace
-'Cls'         Name
-' '           Text.Whitespace
-'do'          Keyword
-'\n'          Text.Whitespace
-
-'def_equals'  Name.Builtin.Pseudo
-' '           Text.Whitespace
-'s'           Name
-'\n'          Text.Whitespace
-
-'end'         Keyword
-'\n'          Text.Whitespace

spec/tests/crystal/test_pseudo_keywords.txt

@@ -1,50 +0,0 @@
----input---
-def f(x : T, line = __LINE__) forall T
-if x.is_a?(String)
-pp! x
-end
-end
-
----tokens---
-'def'         Keyword
-' '           Text.Whitespace
-'f'           Name.Function
-'('           Punctuation
-'x'           Name
-' '           Text.Whitespace
-':'           Punctuation
-' '           Text.Whitespace
-'T'           Name
-','           Punctuation
-' '           Text.Whitespace
-'line'        Name
-' '           Text.Whitespace
-'='           Operator
-' '           Text.Whitespace
-'__LINE__'    Keyword.Pseudo
-')'           Punctuation
-' '           Text.Whitespace
-'forall'      Keyword.Pseudo
-' '           Text.Whitespace
-'T'           Name
-'\n'          Text.Whitespace
-
-'if'          Keyword
-' '           Text.Whitespace
-'x'           Name
-'.is_a?'      Keyword.Pseudo
-'('           Punctuation
-'String'      Name
-')'           Punctuation
-'\n'          Text.Whitespace
-
-'pp!'         Name.Builtin.Pseudo
-' '           Text.Whitespace
-'x'           Name
-'\n'          Text.Whitespace
-
-'end'         Keyword
-'\n'          Text.Whitespace
-
-'end'         Keyword
-'\n'          Text.Whitespace

spec/tests/crystal/test_range_syntax1.txt

@@ -1,8 +0,0 @@
----input---
-1...3
-
----tokens---
-'1'           Literal.Number.Integer
-'...'         Operator
-'3'           Literal.Number.Integer
-'\n'          Text.Whitespace

spec/tests/crystal/test_range_syntax2.txt

@@ -1,10 +0,0 @@
----input---
-1 .. 3
-
----tokens---
-'1'           Literal.Number.Integer
-' '           Text.Whitespace
-'..'          Operator
-' '           Text.Whitespace
-'3'           Literal.Number.Integer
-'\n'          Text.Whitespace

spec/tests/crystal/test_string_escapes.txt

@@ -1,58 +0,0 @@
----input---
-"a\nz"
-"a\az"
-"a\xffz"
-"a\u1234z"
-"a\000z"
-"a\u{0}z"
-"a\u{10AfF9}z"
-
----tokens---
-'"'           Literal.String.Double
-'a'           Literal.String.Double
-'\\n'         Literal.String.Escape
-'z'           Literal.String.Double
-'"'           Literal.String.Double
-'\n'          Text.Whitespace
-
-'"'           Literal.String.Double
-'a'           Literal.String.Double
-'\\a'         Literal.String.Escape
-'z'           Literal.String.Double
-'"'           Literal.String.Double
-'\n'          Text.Whitespace
-
-'"'           Literal.String.Double
-'a'           Literal.String.Double
-'\\xff'       Literal.String.Escape
-'z'           Literal.String.Double
-'"'           Literal.String.Double
-'\n'          Text.Whitespace
-
-'"'           Literal.String.Double
-'a'           Literal.String.Double
-'\\u1234'     Literal.String.Escape
-'z'           Literal.String.Double
-'"'           Literal.String.Double
-'\n'          Text.Whitespace
-
-'"'           Literal.String.Double
-'a'           Literal.String.Double
-'\\000'       Literal.String.Escape
-'z'           Literal.String.Double
-'"'           Literal.String.Double
-'\n'          Text.Whitespace
-
-'"'           Literal.String.Double
-'a'           Literal.String.Double
-'\\u{0}'      Literal.String.Escape
-'z'           Literal.String.Double
-'"'           Literal.String.Double
-'\n'          Text.Whitespace
-
-'"'           Literal.String.Double
-'a'           Literal.String.Double
-'\\u{10AfF9}' Literal.String.Escape
-'z'           Literal.String.Double
-'"'           Literal.String.Double
-'\n'          Text.Whitespace

spec/tests/crystal/test_symbols.txt

@@ -1,20 +0,0 @@
----input---
-:sym_bol
-:あ
-:question?
-:"symbol"
-
----tokens---
-':sym_bol'    Literal.String.Symbol
-'\n'          Text.Whitespace
-
-':あ'          Literal.String.Symbol
-'\n'          Text.Whitespace
-
-':question?'  Literal.String.Symbol
-'\n'          Text.Whitespace
-
-':"'          Literal.String.Symbol
-'symbol'      Literal.String.Symbol
-'"'           Literal.String.Symbol
-'\n'          Text.Whitespace

src/constants/lexers.cr

@@ -328,6 +328,7 @@ module Tartrazine
     "restructuredtext"            => "rst",
     "rexx"                        => "rexx",
     "rkt"                         => "racket",
+    "roff"                        => "groff",
     "rpmspec"                     => "rpm_spec",
     "rs"                          => "rust",
     "rst"                         => "rst",
@@ -730,8 +731,8 @@ module Tartrazine
     "*.applescript"     => ["applescript"],
     "*.aql"             => ["arangodb_aql"],
     "*.arexx"           => ["rexx"],
-    "*.as"              => ["actionscript_3", "actionscript"],
+    "*.as"              => ["actionscript", "actionscript_3"],
-    "*.asm"             => ["tasm", "nasm", "z80_assembly"],
+    "*.asm"             => ["nasm", "tasm", "z80_assembly"],
     "*.au3"             => ["autoit"],
     "*.automount"       => ["systemd"],
     "*.aux"             => ["tex"],
@@ -739,7 +740,7 @@ module Tartrazine
     "*.awk"             => ["awk"],
     "*.b"               => ["brainfuck"],
     "*.bal"             => ["ballerina"],
-    "*.bas"             => ["vb_net", "qbasic"],
+    "*.bas"             => ["qbasic", "vb_net"],
     "*.bash"            => ["bash"],
     "*.bat"             => ["batchfile"],
     "*.batch"           => ["psl"],
@@ -750,7 +751,7 @@ module Tartrazine
     "*.bnf"             => ["bnf"],
     "*.bqn"             => ["bqn"],
     "*.bzl"             => ["python"],
-    "*.c"               => ["c++", "c"],
+    "*.c"               => ["c", "c++"],
     "*.c++"             => ["c++"],
     "*.capnp"           => ["cap_n_proto"],
     "*.cc"              => ["c++"],
@@ -839,7 +840,7 @@ module Tartrazine
     "*.fx"              => ["hlsl"],
     "*.fxh"             => ["hlsl"],
     "*.fzn"             => ["minizinc"],
-    "*.gd"              => ["gdscript3", "gdscript"],
+    "*.gd"              => ["gdscript", "gdscript3"],
     "*.gemspec"         => ["ruby"],
     "*.geo"             => ["glsl"],
     "*.gleam"           => ["gleam"],
@@ -849,7 +850,7 @@ module Tartrazine
     "*.graphql"         => ["graphql"],
     "*.graphqls"        => ["graphql"],
     "*.groovy"          => ["groovy"],
-    "*.h"               => ["c++", "c", "objective-c"],
+    "*.h"               => ["c", "c++", "objective-c"],
     "*.h++"             => ["c++"],
     "*.ha"              => ["hare"],
     "*.handlebars"      => ["handlebars"],
@@ -872,7 +873,7 @@ module Tartrazine
     "*.idc"             => ["c"],
     "*.idr"             => ["idris"],
     "*.ijs"             => ["j"],
-    "*.inc"             => ["objectpascal", "povray", "php", "sourcepawn"],
+    "*.inc"             => ["objectpascal", "php", "povray", "sourcepawn"],
     "*.inf"             => ["ini"],
     "*.ini"             => ["ini"],
     "*.ino"             => ["arduino"],
@@ -898,13 +899,13 @@ module Tartrazine
     "*.lpk"             => ["objectpascal"],
     "*.lpr"             => ["objectpascal"],
     "*.lua"             => ["lua"],
-    "*.m"               => ["mathematica", "octave", "matlab", "objective-c", "mason"],
+    "*.m"               => ["mason", "mathematica", "matlab", "objective-c", "octave"],
     "*.ma"              => ["mathematica"],
     "*.mak"             => ["makefile"],
     "*.man"             => ["groff"],
     "*.mao"             => ["mako"],
     "*.markdown"        => ["markdown"],
-    "*.mc"              => ["monkeyc", "mason"],
+    "*.mc"              => ["mason", "monkeyc"],
     "*.mcfunction"      => ["mcfunction"],
     "*.md"              => ["markdown"],
     "*.metal"           => ["metal"],
@@ -961,7 +962,7 @@ module Tartrazine
     "*.pml"             => ["promela"],
     "*.pony"            => ["pony"],
     "*.pov"             => ["povray"],
-    "*.pp"              => ["puppet", "objectpascal"],
+    "*.pp"              => ["objectpascal", "puppet"],
     "*.pq"              => ["powerquery"],
     "*.pr"              => ["promela"],
     "*.prm"             => ["promela"],
@@ -1010,7 +1011,7 @@ module Tartrazine
     "*.rst"             => ["rst"],
     "*.rvt"             => ["tcl"],
     "*.rx"              => ["rexx"],
-    "*.s"               => ["armasm", "r", "gas"],
+    "*.s"               => ["armasm", "gas", "r"],
     "*.sage"            => ["python"],
     "*.sas"             => ["sas"],
     "*.sass"            => ["sass"],
@@ -1023,7 +1024,7 @@ module Tartrazine
     "*.scope"           => ["systemd"],
     "*.scss"            => ["scss"],
     "*.sed"             => ["sed"],
-    "*.service"         => ["systemd", "ini"],
+    "*.service"         => ["ini", "systemd"],
     "*.sh"              => ["bash"],
     "*.sh-session"      => ["bash_session"],
     "*.sieve"           => ["sieve"],
@@ -1033,7 +1034,7 @@ module Tartrazine
     "*.smali"           => ["smali"],
     "*.sml"             => ["standard_ml"],
     "*.snobol"          => ["snobol"],
-    "*.socket"          => ["systemd", "ini"],
+    "*.socket"          => ["ini", "systemd"],
     "*.sol"             => ["solidity"],
     "*.sp"              => ["sourcepawn"],
     "*.sparql"          => ["sparql"],
@@ -1068,7 +1069,7 @@ module Tartrazine
     "*.tpl"             => ["smarty"],
     "*.tpp"             => ["c++"],
     "*.trig"            => ["psl"],
-    "*.ts"              => ["typoscript", "typescript"],
+    "*.ts"              => ["typescript", "typoscript"],
     "*.tst"             => ["scilab"],
     "*.tsx"             => ["typescript"],
     "*.ttl"             => ["turtle"],
@@ -1104,7 +1105,7 @@ module Tartrazine
     "*.xml"             => ["xml"],
     "*.xsd"             => ["xml"],
     "*.xsl"             => ["xml"],
-    "*.xslt"            => ["xml", "html"],
+    "*.xslt"            => ["html", "xml"],
     "*.yaml"            => ["yaml"],
     "*.yang"            => ["yang"],
     "*.yml"             => ["yaml"],

src/formatters/ansi.cr

@@ -1,6 +1,15 @@
 require "../formatter"
 
 module Tartrazine
+  def self.to_ansi(text : String, language : String,
+                   theme : String = "default-dark",
+                   line_numbers : Bool = false) : String
+    Tartrazine::Ansi.new(
+      theme: Tartrazine.theme(theme),
+      line_numbers: line_numbers
+    ).format(text, Tartrazine.lexer(name: language))
+  end
+
   class Ansi < Formatter
     property? line_numbers : Bool = false
 
@@ -11,7 +20,7 @@ module Tartrazine
       "#{i + 1}".rjust(4).ljust(5)
     end
 
-    def format(text : String, lexer : Lexer) : String
+    def format(text : String, lexer : BaseLexer) : String
       outp = String::Builder.new("")
       format(text, lexer, outp)
       outp.to_s

src/formatters/html.cr

@@ -3,6 +3,17 @@ require "../formatter"
 require "html"
 
 module Tartrazine
+  def self.to_html(text : String, language : String,
+                   theme : String = "default-dark",
+                   standalone : Bool = true,
+                   line_numbers : Bool = false) : String
+    Tartrazine::Html.new(
+      theme: Tartrazine.theme(theme),
+      standalone: standalone,
+      line_numbers: line_numbers
+    ).format(text, Tartrazine.lexer(name: language))
+  end
+
   class Html < Formatter
     # property line_number_in_table : Bool = false
     # property with_classes : Bool = true

src/formatters/json.cr

@@ -12,7 +12,7 @@ module Tartrazine
 
     def format(text : String, lexer : BaseLexer, io : IO) : Nil
       tokenizer = lexer.tokenizer(text)
-      io << Tartrazine::Lexer.collapse_tokens(tokenizer.to_a).to_json
+      io << Tartrazine::RegexLexer.collapse_tokens(tokenizer.to_a).to_json
     end
   end
 end

src/heuristics.cr

@@ -1,13 +1,12 @@
 require "yaml"
 
-  # Use linguist's heuristics to disambiguate between languages
+# Use linguist's heuristics to disambiguate between languages
-  # This is *shamelessly* stolen from https://github.com/github-linguist/linguist
+# This is *shamelessly* stolen from https://github.com/github-linguist/linguist
-  # and ported to Crystal. Deepest thanks to the authors of Linguist
+# and ported to Crystal. Deepest thanks to the authors of Linguist
-  # for licensing it liberally.
+# for licensing it liberally.
-  # 
+#
-  # Consider this code (c) 2017 GitHub, Inc. even if I wrote it.
+# Consider this code (c) 2017 GitHub, Inc. even if I wrote it.
-  module Linguist
+module Linguist
-
   class Heuristic
     include YAML::Serializable
 
@@ -80,7 +79,3 @@ require "yaml"
     end
   end
 end
-
-h = Linguist::Heuristic.from_yaml(File.read("heuristics/heuristics.yml"))
-fname = "/usr/include/sqlite3.h"
-p! h.run(fname, File.read(fname))

src/lexer.cr

@@ -1,5 +1,7 @@
-require "baked_file_system"
 require "./constants/lexers"
+require "./heuristics"
+require "baked_file_system"
+require "crystal/syntax_highlighter"
 
 module Tartrazine
   class LexerFiles
@@ -9,22 +11,35 @@ module Tartrazine
 
   # Get the lexer object for a language name
   # FIXME: support mimetypes
-  def self.lexer(name : String? = nil, filename : String? = nil) : BaseLexer
+  def self.lexer(name : String? = nil, filename : String? = nil, mimetype : String? = nil) : BaseLexer
     return lexer_by_name(name) if name && name != "autodetect"
     return lexer_by_filename(filename) if filename
+    return lexer_by_mimetype(mimetype) if mimetype
 
-    Lexer.from_xml(LexerFiles.get("/#{LEXERS_BY_NAME["plaintext"]}.xml").gets_to_end)
+    RegexLexer.from_xml(LexerFiles.get("/#{LEXERS_BY_NAME["plaintext"]}.xml").gets_to_end)
+  end
+
+  private def self.lexer_by_mimetype(mimetype : String) : BaseLexer
+    lexer_file_name = LEXERS_BY_MIMETYPE.fetch(mimetype, nil)
+    raise Exception.new("Unknown mimetype: #{mimetype}") if lexer_file_name.nil?
+
+    RegexLexer.from_xml(LexerFiles.get("/#{lexer_file_name}.xml").gets_to_end)
   end
 
   private def self.lexer_by_name(name : String) : BaseLexer
+    return CrystalLexer.new if name == "crystal"
     lexer_file_name = LEXERS_BY_NAME.fetch(name.downcase, nil)
     return create_delegating_lexer(name) if lexer_file_name.nil? && name.includes? "+"
     raise Exception.new("Unknown lexer: #{name}") if lexer_file_name.nil?
 
-    Lexer.from_xml(LexerFiles.get("/#{lexer_file_name}.xml").gets_to_end)
+    RegexLexer.from_xml(LexerFiles.get("/#{lexer_file_name}.xml").gets_to_end)
   end
 
   private def self.lexer_by_filename(filename : String) : BaseLexer
+    if filename.ends_with?(".cr")
+      return CrystalLexer.new
+    end
+
     candidates = Set(String).new
     LEXERS_BY_FILENAME.each do |k, v|
       candidates += v.to_set if File.match?(k, File.basename(filename))
@@ -36,10 +51,28 @@ module Tartrazine
     when 1
       lexer_file_name = candidates.first
     else
-      raise Exception.new("Multiple lexers match the filename: #{candidates.to_a.join(", ")}")
+      lexer_file_name = self.lexer_by_content(filename)
+      begin
+        return self.lexer(lexer_file_name)
+      rescue ex : Exception
+        raise Exception.new("Multiple lexers match the filename: #{candidates.to_a.join(", ")}, heuristics suggest #{lexer_file_name} but there is no matching lexer.")
+      end
     end
 
-    Lexer.from_xml(LexerFiles.get("/#{lexer_file_name}.xml").gets_to_end)
+    RegexLexer.from_xml(LexerFiles.get("/#{lexer_file_name}.xml").gets_to_end)
+  end
+
+  private def self.lexer_by_content(fname : String) : String?
+    h = Linguist::Heuristic.from_yaml(LexerFiles.get("/heuristics.yml").gets_to_end)
+    result = h.run(fname, File.read(fname))
+    case result
+    when Nil
+      raise Exception.new "No lexer found for #{fname}"
+    when String
+      result.as(String)
+    when Array(String)
+      result.first
+    end
   end
 
   private def self.create_delegating_lexer(name : String) : BaseLexer
@@ -126,7 +159,9 @@ module Tartrazine
     end
   end
 
-  abstract class BaseLexer
+  alias BaseLexer = Lexer
+
+  abstract class Lexer
     property config = {
       name:             "",
       priority:         0.0,
@@ -148,7 +183,7 @@ module Tartrazine
   # For explanations on what actions and states do
   # the Pygments documentation is a good place to start.
   # https://pygments.org/docs/lexerdevelopment/
-  class Lexer < BaseLexer
+  class RegexLexer < BaseLexer
     # Collapse consecutive tokens of the same type for easier comparison
     # and smaller output
     def self.collapse_tokens(tokens : Array(Tartrazine::Token)) : Array(Tartrazine::Token)
@@ -172,7 +207,7 @@ module Tartrazine
     end
 
     def self.from_xml(xml : String) : Lexer
-      l = Lexer.new
+      l = RegexLexer.new
       lexer = XML.parse(xml).first_element_child
       if lexer
         config = lexer.children.find { |node|
@@ -237,7 +272,7 @@ module Tartrazine
   #
   # This is useful for things like template languages, where
   # you have Jinja + HTML or Jinja + CSS and so on.
-  class DelegatingLexer < BaseLexer
+  class DelegatingLexer < Lexer
     property language_lexer : BaseLexer
     property root_lexer : BaseLexer
 
@@ -298,4 +333,81 @@ module Tartrazine
       new_state
     end
   end
+
+  class CustomCrystalHighlighter < Crystal::SyntaxHighlighter
+    @tokens = [] of Token
+
+    def render_delimiter(&block)
+      @tokens << {type: "LiteralString", value: block.call.to_s}
+    end
+
+    def render_interpolation(&block)
+      @tokens << {type: "LiteralStringInterpol", value: "\#{"}
+      @tokens << {type: "Text", value: block.call.to_s}
+      @tokens << {type: "LiteralStringInterpol", value: "}"}
+    end
+
+    def render_string_array(&block)
+      @tokens << {type: "LiteralString", value: block.call.to_s}
+    end
+
+    # ameba:disable Metrics/CyclomaticComplexity
+    def render(type : TokenType, value : String)
+      case type
+      when .comment?
+        @tokens << {type: "Comment", value: value}
+      when .number?
+        @tokens << {type: "LiteralNumber", value: value}
+      when .char?
+        @tokens << {type: "LiteralStringChar", value: value}
+      when .symbol?
+        @tokens << {type: "LiteralStringSymbol", value: value}
+      when .const?
+        @tokens << {type: "NameConstant", value: value}
+      when .string?
+        @tokens << {type: "LiteralString", value: value}
+      when .ident?
+        @tokens << {type: "NameVariable", value: value}
+      when .keyword?, .self?
+        @tokens << {type: "NameKeyword", value: value}
+      when .primitive_literal?
+        @tokens << {type: "Literal", value: value}
+      when .operator?
+        @tokens << {type: "Operator", value: value}
+      when Crystal::SyntaxHighlighter::TokenType::DELIMITED_TOKEN, Crystal::SyntaxHighlighter::TokenType::DELIMITER_START, Crystal::SyntaxHighlighter::TokenType::DELIMITER_END
+        @tokens << {type: "LiteralString", value: value}
+      else
+        @tokens << {type: "Text", value: value}
+      end
+    end
+  end
+
+  class CrystalTokenizer < Tartrazine::BaseTokenizer
+    include Iterator(Token)
+    @hl = CustomCrystalHighlighter.new
+    @lexer : BaseLexer
+    @iter : Iterator(Token)
+
+    # delegate next, to: @iter
+
+    def initialize(@lexer : BaseLexer, text : String, secondary = false)
+      # Respect the `ensure_nl` config option
+      if text.size > 0 && text[-1] != '\n' && @lexer.config[:ensure_nl] && !secondary
+        text += "\n"
+      end
+      # Just do the tokenizing
+      @hl.highlight(text)
+      @iter = @hl.@tokens.each
+    end
+
+    def next : Iterator::Stop | Token
+      @iter.next
+    end
+  end
+
+  class CrystalLexer < BaseLexer
+    def tokenizer(text : String, secondary = false) : BaseTokenizer
+      CrystalTokenizer.new(self, text, secondary)
+    end
+  end
 end

src/main.cr

@@ -1,5 +1,5 @@
 require "docopt"
-require "./**"
+require "./tartrazine"
 
 HELP = <<-HELP
 tartrazine: a syntax highlighting tool

src/rules.cr

@@ -17,7 +17,6 @@ module Tartrazine
 
   abstract struct BaseRule
     abstract def match(text : Bytes, pos : Int32, tokenizer : Tokenizer) : Tuple(Bool, Int32, Array(Token))
-    abstract def initialize(node : XML::Node)
 
     @actions : Array(Action) = [] of Action
 
@@ -40,9 +39,6 @@ module Tartrazine
       return true, pos + match[0].size, @actions.flat_map(&.emit(match, tokenizer))
     end
 
-    def initialize(node : XML::Node)
-    end
-
     def initialize(node : XML::Node, multiline, dotall, ignorecase)
       pattern = node["pattern"]
       pattern = "(?m)" + pattern if multiline

src/run_tests.cr

@@ -0,0 +1 @@
+require "../spec/**"

src/styles.cr

@@ -84,27 +84,6 @@ module Tartrazine
 
     property styles = {} of String => Style
 
-    # Get the style for a token.
-    def style(token)
-      styles[token] = Style.new unless styles.has_key?(token)
-      s = styles[token]
-
-      # We already got the data from the style hierarchy
-      return s if s.complete?
-
-      # Form the hierarchy of parent styles
-      parents = style_parents(token)
-
-      s = parents.map do |parent|
-        styles[parent]
-      end.reduce(s) do |acc, style|
-        acc + style
-      end
-      s.complete = true
-      styles[token] = s
-      s
-    end
-
     def style_parents(token)
       parents = ["Background"]
       parts = token.underscore.split("_").map(&.capitalize)

src/tartrazine.cr

@@ -1,5 +1,6 @@
 require "./actions"
 require "./formatter"
+require "./formatters/**"
 require "./rules"
 require "./styles"
 require "./tartrazine"