mirror of
https://github.com/ralsina/tartrazine.git
synced 2025-07-01 20:37:08 -03:00
Compare commits
20 Commits
Author | SHA1 | Date | |
---|---|---|---|
f2d802e391 | |||
11cb5fc48e | |||
bf2f08c1d0 | |||
84980459cf | |||
c011bd8347 | |||
6a38f2f5fb | |||
c4a2d1a752 | |||
358be51e27 | |||
2cff0fea48 | |||
40202eb2d6 | |||
3ed4a7eab8 | |||
6f797f999a | |||
b762307660 | |||
eb0cc089a9 | |||
88f2aace20 | |||
fe943fa399 | |||
08f8138e05 | |||
3c0b3c38e2 | |||
a1318501a5 | |||
daf24189bf |
26
.github/workflows/ci.yml
vendored
Normal file
26
.github/workflows/ci.yml
vendored
Normal file
@ -0,0 +1,26 @@
|
||||
name: Tests
|
||||
on:
|
||||
# This can't yet run automatically, because tests fail because of
|
||||
# different versions of chroma. Need to get the same one in my
|
||||
# local env and in CI
|
||||
workflow_dispatch:
|
||||
push:
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Download source
|
||||
uses: actions/checkout@v4
|
||||
- name: Install Crystal
|
||||
uses: crystal-lang/install-crystal@v1
|
||||
- name: Run tests
|
||||
run: |
|
||||
wget https://github.com/alecthomas/chroma/releases/download/v2.14.0/chroma-2.14.0-linux-amd64.tar.gz
|
||||
tar xzvf chroma-2.14.0*gz
|
||||
mkdir ~/.local/bin -p
|
||||
sudo mv chroma ~/.local/bin
|
||||
shards install
|
||||
crystal tool format --check
|
||||
crystal spec -v
|
30
.github/workflows/coverage.yml
vendored
Normal file
30
.github/workflows/coverage.yml
vendored
Normal file
@ -0,0 +1,30 @@
|
||||
name: Coverage
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "0 1 * * *"
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Download source
|
||||
uses: actions/checkout@v4
|
||||
- name: Install Crystal
|
||||
uses: crystal-lang/install-crystal@v1
|
||||
- name: Run tests using kcov
|
||||
run: |
|
||||
sudo apt update && sudo apt install kcov
|
||||
wget https://github.com/alecthomas/chroma/releases/download/v2.14.0/chroma-2.14.0-linux-amd64.tar.gz
|
||||
tar xzvf chroma-2.14.0*gz
|
||||
mkdir ~/.local/bin -p
|
||||
sudo mv chroma ~/.local/bin
|
||||
shards install
|
||||
crystal build src/run_tests.cr
|
||||
kcov --clean --include-path=./src $PWD/coverage ./run_tests
|
||||
curl -Os https://uploader.codecov.io/latest/linux/codecov
|
||||
chmod +x codecov
|
||||
./codecov -t ${CODECOV_TOKEN} -s coverage
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
3
.gitignore
vendored
3
.gitignore
vendored
@ -9,3 +9,6 @@ shard.lock
|
||||
.vscode/
|
||||
.crystal/
|
||||
venv/
|
||||
.croupier
|
||||
coverage/
|
||||
run_tests
|
||||
|
3
.md.rb
Normal file
3
.md.rb
Normal file
@ -0,0 +1,3 @@
|
||||
exclude_rule 'MD033' # Inline HTML
|
||||
exclude_rule 'MD005' # 3-space indent for lists
|
||||
exclude_rule 'MD024' # Repeated headings
|
42
CHANGELOG.md
42
CHANGELOG.md
@ -2,12 +2,52 @@
|
||||
|
||||
All notable changes to this project will be documented in this file.
|
||||
|
||||
## [unreleased]
|
||||
## [0.7.0] - 2024-09-10
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Higher level API (`to_html` and `to_ansi`)
|
||||
- Use the native crystal highlighter
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Ameba
|
||||
- Variable bame in Hacefile
|
||||
- Make it easier to import the Ansi formatter
|
||||
- Renamed BaseLexer to Lexer and Lexer to RegexLexer to make API nicer
|
||||
- Make install work
|
||||
|
||||
### 📚 Documentation
|
||||
|
||||
- Mention AUR package
|
||||
|
||||
### 🧪 Testing
|
||||
|
||||
- Add CI workflows
|
||||
|
||||
### ⚙️ Miscellaneous Tasks
|
||||
|
||||
- Pre-commit hooks
|
||||
- Git-cliff config
|
||||
- Started changelog
|
||||
- Force conventional commit messages
|
||||
- Force conventional commit messages
|
||||
- Updated pre-commit
|
||||
- *(ignore)* Fix tests
|
||||
- Added badges
|
||||
- Added badges
|
||||
- *(ignore)* Removed random file
|
||||
|
||||
### Build
|
||||
|
||||
- Switch from Makefile to Hacefile
|
||||
- Added do_release script
|
||||
- Fix markdown check
|
||||
|
||||
### Bump
|
||||
|
||||
- Release v0.6.4
|
||||
- Release v0.6.4
|
||||
|
||||
## [0.6.1] - 2024-08-25
|
||||
|
||||
|
115
Hacefile.yml
Normal file
115
Hacefile.yml
Normal file
@ -0,0 +1,115 @@
|
||||
variables:
|
||||
FLAGS: "-d --error-trace"
|
||||
NAME: "tartrazine"
|
||||
|
||||
tasks:
|
||||
build:
|
||||
default: true
|
||||
dependencies:
|
||||
- src
|
||||
- shard.lock
|
||||
- shard.yml
|
||||
- Hacefile.yml
|
||||
- lexers/*xml
|
||||
- styles/*xml
|
||||
outputs:
|
||||
- bin/{{NAME}}
|
||||
commands: |
|
||||
shards build {{FLAGS}}
|
||||
|
||||
get-deps:
|
||||
dependencies:
|
||||
- shard.yml
|
||||
outputs:
|
||||
- shard.lock
|
||||
commands: |
|
||||
shards install
|
||||
|
||||
build-release:
|
||||
phony: true
|
||||
always_run: true
|
||||
commands: |
|
||||
hace build FLAGS="--release"
|
||||
|
||||
install:
|
||||
phony: true
|
||||
always_run: true
|
||||
dependencies:
|
||||
- bin/{{NAME}}
|
||||
commands: |
|
||||
rm ${HOME}/.local/bin/{{NAME}} -f
|
||||
cp bin/{{NAME}} ${HOME}/.local/bin/{{NAME}}
|
||||
|
||||
static:
|
||||
outputs:
|
||||
- bin/{{NAME}}-static-linux-amd64
|
||||
- bin/{{NAME}}-static-linux-arm64
|
||||
commands: |
|
||||
hace clean
|
||||
./build_static.sh
|
||||
|
||||
test:
|
||||
dependencies:
|
||||
- src
|
||||
- spec
|
||||
- shard.lock
|
||||
- shard.yml
|
||||
commands: |
|
||||
crystal spec -v --error-trace
|
||||
phony: true
|
||||
always_run: true
|
||||
|
||||
lint:
|
||||
dependencies:
|
||||
- src
|
||||
- spec
|
||||
- shard.lock
|
||||
- shard.yml
|
||||
commands: |
|
||||
crystal tool format src/*.cr spec/*.cr
|
||||
ameba --fix
|
||||
always_run: true
|
||||
phony: true
|
||||
|
||||
docs:
|
||||
dependencies:
|
||||
- src
|
||||
- shard.lock
|
||||
- shard.yml
|
||||
- README.md
|
||||
commands: |
|
||||
crystal docs
|
||||
outputs:
|
||||
- docs/index.html
|
||||
|
||||
pre-commit:
|
||||
default: true
|
||||
outputs:
|
||||
- .git/hooks/commit-msg
|
||||
- .git/hooks/pre-commit
|
||||
dependencies:
|
||||
- .pre-commit-config.yaml
|
||||
commands: |
|
||||
pre-commit install --hook-type commit-msg
|
||||
pre-commit install
|
||||
|
||||
clean:
|
||||
phony: true
|
||||
always_run: true
|
||||
commands: |
|
||||
rm -rf shard.lock bin lib
|
||||
|
||||
coverage:
|
||||
dependencies:
|
||||
- src
|
||||
- spec
|
||||
- shard.lock
|
||||
- shard.yml
|
||||
commands: |
|
||||
shards install
|
||||
crystal build -o bin/run_tests src/run_tests.cr
|
||||
rm -rf coverage/
|
||||
mkdir coverage
|
||||
kcov --clean --include-path=./src ${PWD}/coverage ./bin/run_tests
|
||||
outputs:
|
||||
- coverage/index.html
|
21
Makefile
21
Makefile
@ -1,21 +0,0 @@
|
||||
all: build
|
||||
|
||||
build: $(wildcard src/**/*.cr) $(wildcard lexers/*xml) $(wildcard styles/*xml) shard.yml
|
||||
shards build -Dstrict_multi_assign -Dno_number_autocast -d --error-trace
|
||||
release: $(wildcard src/**/*.cr) $(wildcard lexers/*xml) $(wildcard styles/*xml) shard.yml
|
||||
shards build --release
|
||||
static: $(wildcard src/**/*.cr) $(wildcard lexers/*xml) $(wildcard styles/*xml) shard.yml
|
||||
shards build --release --static
|
||||
strip bin/tartrazine
|
||||
|
||||
|
||||
clean:
|
||||
rm -rf bin lib shard.lock
|
||||
|
||||
test:
|
||||
crystal spec
|
||||
|
||||
lint:
|
||||
ameba --fix src spec
|
||||
|
||||
.PHONY: clean all test bin lint
|
32
README.md
32
README.md
@ -1,5 +1,8 @@
|
||||
# TARTRAZINE
|
||||
|
||||
[](https://github.com/ralsina/tartrazine/actions/workflows/ci.yml)
|
||||
[](https://codecov.io/gh/ralsina/tartrazine)
|
||||
|
||||
Tartrazine is a library to syntax-highlight code. It is
|
||||
a port of [Pygments](https://pygments.org/) to
|
||||
[Crystal](https://crystal-lang.org/).
|
||||
@ -42,18 +45,35 @@ $ tartrazine whatever.c -t catppuccin-macchiato --line-numbers \
|
||||
|
||||
## Usage as a Library
|
||||
|
||||
This works:
|
||||
This is the high level API:
|
||||
|
||||
```crystal
|
||||
require "tartrazine"
|
||||
|
||||
lexer = Tartrazine.lexer("crystal")
|
||||
theme = Tartrazine.theme("catppuccin-macchiato")
|
||||
formatter = Tartrazine::Html.new
|
||||
formatter.theme = theme
|
||||
puts formatter.format(File.read(ARGV[0]), lexer)
|
||||
html = Tartrazine.to_html(
|
||||
"puts \"Hello, world!\"",
|
||||
language: "crystal",
|
||||
theme: "catppuccin-macchiato",
|
||||
standalone: true,
|
||||
line_numbers: true
|
||||
)
|
||||
```
|
||||
|
||||
This does more or less the same thing, but more manually:
|
||||
|
||||
```crystal
|
||||
lexer = Tartrazine.lexer("crystal")
|
||||
formatter = Tartrazine::Html.new (
|
||||
theme: Tartrazine.theme("catppuccin-macchiato"),
|
||||
line_numbers: true,
|
||||
standalone: true,
|
||||
)
|
||||
puts formatter.format("puts \"Hello, world!\"", lexer)
|
||||
```
|
||||
|
||||
The reason you may want to use the manual version is to reuse
|
||||
the lexer and formatter objects for performance reasons.
|
||||
|
||||
## Contributing
|
||||
|
||||
1. Fork it (<https://github.com/ralsina/tartrazine/fork>)
|
||||
|
@ -7,10 +7,10 @@ docker run --rm --privileged \
|
||||
|
||||
# Build for AMD64
|
||||
docker build . -f Dockerfile.static -t tartrazine-builder
|
||||
docker run -ti --rm -v "$PWD":/app --user="$UID" tartrazine-builder /bin/sh -c "cd /app && rm -rf lib shard.lock && make static"
|
||||
docker run -ti --rm -v "$PWD":/app --user="$UID" tartrazine-builder /bin/sh -c "cd /app && rm -rf lib shard.lock && shards build --static --release"
|
||||
mv bin/tartrazine bin/tartrazine-static-linux-amd64
|
||||
|
||||
# Build for ARM64
|
||||
docker build . -f Dockerfile.static --platform linux/arm64 -t tartrazine-builder
|
||||
docker run -ti --rm -v "$PWD":/app --platform linux/arm64 --user="$UID" tartrazine-builder /bin/sh -c "cd /app && rm -rf lib shard.lock && make static"
|
||||
docker run -ti --rm -v "$PWD":/app --platform linux/arm64 --user="$UID" tartrazine-builder /bin/sh -c "cd /app && rm -rf lib shard.lock && shards build --static --release"
|
||||
mv bin/tartrazine bin/tartrazine-static-linux-arm64
|
||||
|
15
do_release.sh
Executable file
15
do_release.sh
Executable file
@ -0,0 +1,15 @@
|
||||
#!/bin/bash
|
||||
set e
|
||||
|
||||
PKGNAME=$(basename "$PWD")
|
||||
VERSION=$(git cliff --bumped-version |cut -dv -f2)
|
||||
|
||||
sed "s/^version:.*$/version: $VERSION/g" -i shard.yml
|
||||
git add shard.yml
|
||||
hace lint test
|
||||
git cliff --bump -o
|
||||
git commit -a -m "bump: Release v$VERSION"
|
||||
git tag "v$VERSION"
|
||||
git push --tags
|
||||
hace static
|
||||
gh release create "v$VERSION" "bin/$PKGNAME-static-linux-amd64" "bin/$PKGNAME-static-linux-arm64" --title "Release v$VERSION" --notes "$(git cliff -l -s all)"
|
@ -1,762 +0,0 @@
|
||||
<lexer>
|
||||
<config>
|
||||
<name>Crystal</name>
|
||||
<alias>cr</alias>
|
||||
<alias>crystal</alias>
|
||||
<filename>*.cr</filename>
|
||||
<mime_type>text/x-crystal</mime_type>
|
||||
<dot_all>true</dot_all>
|
||||
</config>
|
||||
<rules>
|
||||
<state name="pa-intp-string">
|
||||
<rule pattern="\\[\(]">
|
||||
<token type="LiteralStringOther"/>
|
||||
</rule>
|
||||
<rule pattern="\(">
|
||||
<token type="LiteralStringOther"/>
|
||||
<push/>
|
||||
</rule>
|
||||
<rule pattern="\)">
|
||||
<token type="LiteralStringOther"/>
|
||||
<pop depth="1"/>
|
||||
</rule>
|
||||
<rule>
|
||||
<include state="string-intp-escaped"/>
|
||||
</rule>
|
||||
<rule pattern="[\\#()]">
|
||||
<token type="LiteralStringOther"/>
|
||||
</rule>
|
||||
<rule pattern="[^\\#()]+">
|
||||
<token type="LiteralStringOther"/>
|
||||
</rule>
|
||||
</state>
|
||||
<state name="ab-regex">
|
||||
<rule pattern="\\[\\<>]">
|
||||
<token type="LiteralStringRegex"/>
|
||||
</rule>
|
||||
<rule pattern="<">
|
||||
<token type="LiteralStringRegex"/>
|
||||
<push/>
|
||||
</rule>
|
||||
<rule pattern=">[imsx]*">
|
||||
<token type="LiteralStringRegex"/>
|
||||
<pop depth="1"/>
|
||||
</rule>
|
||||
<rule>
|
||||
<include state="string-intp"/>
|
||||
</rule>
|
||||
<rule pattern="[\\#<>]">
|
||||
<token type="LiteralStringRegex"/>
|
||||
</rule>
|
||||
<rule pattern="[^\\#<>]+">
|
||||
<token type="LiteralStringRegex"/>
|
||||
</rule>
|
||||
</state>
|
||||
<state name="cb-regex">
|
||||
<rule pattern="\\[\\{}]">
|
||||
<token type="LiteralStringRegex"/>
|
||||
</rule>
|
||||
<rule pattern="\{">
|
||||
<token type="LiteralStringRegex"/>
|
||||
<push/>
|
||||
</rule>
|
||||
<rule pattern="\}[imsx]*">
|
||||
<token type="LiteralStringRegex"/>
|
||||
<pop depth="1"/>
|
||||
</rule>
|
||||
<rule>
|
||||
<include state="string-intp"/>
|
||||
</rule>
|
||||
<rule pattern="[\\#{}]">
|
||||
<token type="LiteralStringRegex"/>
|
||||
</rule>
|
||||
<rule pattern="[^\\#{}]+">
|
||||
<token type="LiteralStringRegex"/>
|
||||
</rule>
|
||||
</state>
|
||||
<state name="simple-backtick">
|
||||
<rule>
|
||||
<include state="string-intp-escaped"/>
|
||||
</rule>
|
||||
<rule pattern="[^\\`#]+">
|
||||
<token type="LiteralStringBacktick"/>
|
||||
</rule>
|
||||
<rule pattern="[\\#]">
|
||||
<token type="LiteralStringBacktick"/>
|
||||
</rule>
|
||||
<rule pattern="`">
|
||||
<token type="LiteralStringBacktick"/>
|
||||
<pop depth="1"/>
|
||||
</rule>
|
||||
</state>
|
||||
<state name="string-intp">
|
||||
<rule pattern="#\{">
|
||||
<token type="LiteralStringInterpol"/>
|
||||
<push state="in-intp"/>
|
||||
</rule>
|
||||
</state>
|
||||
<state name="interpolated-regex">
|
||||
<rule>
|
||||
<include state="string-intp"/>
|
||||
</rule>
|
||||
<rule pattern="[\\#]">
|
||||
<token type="LiteralStringRegex"/>
|
||||
</rule>
|
||||
<rule pattern="[^\\#]+">
|
||||
<token type="LiteralStringRegex"/>
|
||||
</rule>
|
||||
</state>
|
||||
<state name="cb-string">
|
||||
<rule pattern="\\[\\{}]">
|
||||
<token type="LiteralStringOther"/>
|
||||
</rule>
|
||||
<rule pattern="\{">
|
||||
<token type="LiteralStringOther"/>
|
||||
<push/>
|
||||
</rule>
|
||||
<rule pattern="\}">
|
||||
<token type="LiteralStringOther"/>
|
||||
<pop depth="1"/>
|
||||
</rule>
|
||||
<rule pattern="[\\#{}]">
|
||||
<token type="LiteralStringOther"/>
|
||||
</rule>
|
||||
<rule pattern="[^\\#{}]+">
|
||||
<token type="LiteralStringOther"/>
|
||||
</rule>
|
||||
</state>
|
||||
<state name="in-macro-control">
|
||||
<rule pattern="\{%">
|
||||
<token type="LiteralStringInterpol"/>
|
||||
<push/>
|
||||
</rule>
|
||||
<rule pattern="%\}">
|
||||
<token type="LiteralStringInterpol"/>
|
||||
<pop depth="1"/>
|
||||
</rule>
|
||||
<rule pattern="for\b|in\b">
|
||||
<token type="Keyword"/>
|
||||
</rule>
|
||||
<rule>
|
||||
<include state="root"/>
|
||||
</rule>
|
||||
</state>
|
||||
<state name="interpolated-string">
|
||||
<rule>
|
||||
<include state="string-intp"/>
|
||||
</rule>
|
||||
<rule pattern="[\\#]">
|
||||
<token type="LiteralStringOther"/>
|
||||
</rule>
|
||||
<rule pattern="[^\\#]+">
|
||||
<token type="LiteralStringOther"/>
|
||||
</rule>
|
||||
</state>
|
||||
<state name="in-macro-expr">
|
||||
<rule pattern="\{\{">
|
||||
<token type="LiteralStringInterpol"/>
|
||||
<push/>
|
||||
</rule>
|
||||
<rule pattern="\}\}">
|
||||
<token type="LiteralStringInterpol"/>
|
||||
<pop depth="1"/>
|
||||
</rule>
|
||||
<rule>
|
||||
<include state="root"/>
|
||||
</rule>
|
||||
</state>
|
||||
<state name="simple-string">
|
||||
<rule>
|
||||
<include state="string-intp-escaped"/>
|
||||
</rule>
|
||||
<rule pattern="[^\\"#]+">
|
||||
<token type="LiteralStringDouble"/>
|
||||
</rule>
|
||||
<rule pattern="[\\#]">
|
||||
<token type="LiteralStringDouble"/>
|
||||
</rule>
|
||||
<rule pattern=""">
|
||||
<token type="LiteralStringDouble"/>
|
||||
<pop depth="1"/>
|
||||
</rule>
|
||||
</state>
|
||||
<state name="cb-intp-string">
|
||||
<rule pattern="\\[\{]">
|
||||
<token type="LiteralStringOther"/>
|
||||
</rule>
|
||||
<rule pattern="\{">
|
||||
<token type="LiteralStringOther"/>
|
||||
<push/>
|
||||
</rule>
|
||||
<rule pattern="\}">
|
||||
<token type="LiteralStringOther"/>
|
||||
<pop depth="1"/>
|
||||
</rule>
|
||||
<rule>
|
||||
<include state="string-intp-escaped"/>
|
||||
</rule>
|
||||
<rule pattern="[\\#{}]">
|
||||
<token type="LiteralStringOther"/>
|
||||
</rule>
|
||||
<rule pattern="[^\\#{}]+">
|
||||
<token type="LiteralStringOther"/>
|
||||
</rule>
|
||||
</state>
|
||||
<state name="string-intp-escaped">
|
||||
<rule>
|
||||
<include state="string-intp"/>
|
||||
</rule>
|
||||
<rule>
|
||||
<include state="string-escaped"/>
|
||||
</rule>
|
||||
</state>
|
||||
<state name="sb-regex">
|
||||
<rule pattern="\\[\\\[\]]">
|
||||
<token type="LiteralStringRegex"/>
|
||||
</rule>
|
||||
<rule pattern="\[">
|
||||
<token type="LiteralStringRegex"/>
|
||||
<push/>
|
||||
</rule>
|
||||
<rule pattern="\][imsx]*">
|
||||
<token type="LiteralStringRegex"/>
|
||||
<pop depth="1"/>
|
||||
</rule>
|
||||
<rule>
|
||||
<include state="string-intp"/>
|
||||
</rule>
|
||||
<rule pattern="[\\#\[\]]">
|
||||
<token type="LiteralStringRegex"/>
|
||||
</rule>
|
||||
<rule pattern="[^\\#\[\]]+">
|
||||
<token type="LiteralStringRegex"/>
|
||||
</rule>
|
||||
</state>
|
||||
<state name="classname">
|
||||
<rule pattern="[A-Z_]\w*">
|
||||
<token type="NameClass"/>
|
||||
</rule>
|
||||
<rule pattern="(\()(\s*)([A-Z_]\w*)(\s*)(\))">
|
||||
<bygroups>
|
||||
<token type="Punctuation"/>
|
||||
<token type="Text"/>
|
||||
<token type="NameClass"/>
|
||||
<token type="Text"/>
|
||||
<token type="Punctuation"/>
|
||||
</bygroups>
|
||||
</rule>
|
||||
<rule>
|
||||
<pop depth="1"/>
|
||||
</rule>
|
||||
</state>
|
||||
<state name="string-escaped">
|
||||
<rule pattern="\\([\\befnstv#"\']|x[a-fA-F0-9]{1,2}|[0-7]{1,3})">
|
||||
<token type="LiteralStringEscape"/>
|
||||
</rule>
|
||||
</state>
|
||||
<state name="sb-intp-string">
|
||||
<rule pattern="\\[\[]">
|
||||
<token type="LiteralStringOther"/>
|
||||
</rule>
|
||||
<rule pattern="\[">
|
||||
<token type="LiteralStringOther"/>
|
||||
<push/>
|
||||
</rule>
|
||||
<rule pattern="\]">
|
||||
<token type="LiteralStringOther"/>
|
||||
<pop depth="1"/>
|
||||
</rule>
|
||||
<rule>
|
||||
<include state="string-intp-escaped"/>
|
||||
</rule>
|
||||
<rule pattern="[\\#\[\]]">
|
||||
<token type="LiteralStringOther"/>
|
||||
</rule>
|
||||
<rule pattern="[^\\#\[\]]+">
|
||||
<token type="LiteralStringOther"/>
|
||||
</rule>
|
||||
</state>
|
||||
<state name="pa-regex">
|
||||
<rule pattern="\\[\\()]">
|
||||
<token type="LiteralStringRegex"/>
|
||||
</rule>
|
||||
<rule pattern="\(">
|
||||
<token type="LiteralStringRegex"/>
|
||||
<push/>
|
||||
</rule>
|
||||
<rule pattern="\)[imsx]*">
|
||||
<token type="LiteralStringRegex"/>
|
||||
<pop depth="1"/>
|
||||
</rule>
|
||||
<rule>
|
||||
<include state="string-intp"/>
|
||||
</rule>
|
||||
<rule pattern="[\\#()]">
|
||||
<token type="LiteralStringRegex"/>
|
||||
</rule>
|
||||
<rule pattern="[^\\#()]+">
|
||||
<token type="LiteralStringRegex"/>
|
||||
</rule>
|
||||
</state>
|
||||
<state name="in-attr">
|
||||
<rule pattern="\[">
|
||||
<token type="Operator"/>
|
||||
<push/>
|
||||
</rule>
|
||||
<rule pattern="\]">
|
||||
<token type="Operator"/>
|
||||
<pop depth="1"/>
|
||||
</rule>
|
||||
<rule>
|
||||
<include state="root"/>
|
||||
</rule>
|
||||
</state>
|
||||
<state name="ab-intp-string">
|
||||
<rule pattern="\\[<]">
|
||||
<token type="LiteralStringOther"/>
|
||||
</rule>
|
||||
<rule pattern="<">
|
||||
<token type="LiteralStringOther"/>
|
||||
<push/>
|
||||
</rule>
|
||||
<rule pattern=">">
|
||||
<token type="LiteralStringOther"/>
|
||||
<pop depth="1"/>
|
||||
</rule>
|
||||
<rule>
|
||||
<include state="string-intp-escaped"/>
|
||||
</rule>
|
||||
<rule pattern="[\\#<>]">
|
||||
<token type="LiteralStringOther"/>
|
||||
</rule>
|
||||
<rule pattern="[^\\#<>]+">
|
||||
<token type="LiteralStringOther"/>
|
||||
</rule>
|
||||
</state>
|
||||
<state name="in-intp">
|
||||
<rule pattern="\{">
|
||||
<token type="LiteralStringInterpol"/>
|
||||
<push/>
|
||||
</rule>
|
||||
<rule pattern="\}">
|
||||
<token type="LiteralStringInterpol"/>
|
||||
<pop depth="1"/>
|
||||
</rule>
|
||||
<rule>
|
||||
<include state="root"/>
|
||||
</rule>
|
||||
</state>
|
||||
<state name="end-part">
|
||||
<rule pattern=".+">
|
||||
<token type="CommentPreproc"/>
|
||||
<pop depth="1"/>
|
||||
</rule>
|
||||
</state>
|
||||
<state name="root">
|
||||
<rule pattern="#.*?$">
|
||||
<token type="CommentSingle"/>
|
||||
</rule>
|
||||
<rule pattern="(instance_sizeof|pointerof|protected|abstract|require|private|include|unless|typeof|sizeof|return|extend|ensure|rescue|ifdef|super|break|begin|until|while|elsif|yield|next|when|else|then|case|with|end|asm|if|do|as|of)\b">
|
||||
<token type="Keyword"/>
|
||||
</rule>
|
||||
<rule pattern="(false|true|nil)\b">
|
||||
<token type="KeywordConstant"/>
|
||||
</rule>
|
||||
<rule pattern="(module|lib)(\s+)([a-zA-Z_]\w*(?:::[a-zA-Z_]\w*)*)">
|
||||
<bygroups>
|
||||
<token type="Keyword"/>
|
||||
<token type="Text"/>
|
||||
<token type="NameNamespace"/>
|
||||
</bygroups>
|
||||
</rule>
|
||||
<rule pattern="(def|fun|macro)(\s+)((?:[a-zA-Z_]\w*::)*)">
|
||||
<bygroups>
|
||||
<token type="Keyword"/>
|
||||
<token type="Text"/>
|
||||
<token type="NameNamespace"/>
|
||||
</bygroups>
|
||||
<push state="funcname"/>
|
||||
</rule>
|
||||
<rule pattern="def(?=[*%&^`~+-/\[<>=])">
|
||||
<token type="Keyword"/>
|
||||
<push state="funcname"/>
|
||||
</rule>
|
||||
<rule pattern="(class|struct|union|type|alias|enum)(\s+)((?:[a-zA-Z_]\w*::)*)">
|
||||
<bygroups>
|
||||
<token type="Keyword"/>
|
||||
<token type="Text"/>
|
||||
<token type="NameNamespace"/>
|
||||
</bygroups>
|
||||
<push state="classname"/>
|
||||
</rule>
|
||||
<rule pattern="(self|out|uninitialized)\b|(is_a|responds_to)\?">
|
||||
<token type="KeywordPseudo"/>
|
||||
</rule>
|
||||
<rule pattern="(def_equals_and_hash|assert_responds_to|forward_missing_to|def_equals|property|def_hash|parallel|delegate|debugger|getter|record|setter|spawn|pp)\b">
|
||||
<token type="NameBuiltinPseudo"/>
|
||||
</rule>
|
||||
<rule pattern="getter[!?]|property[!?]|__(DIR|FILE|LINE)__\b">
|
||||
<token type="NameBuiltinPseudo"/>
|
||||
</rule>
|
||||
<rule pattern="(?<!\.)(get_stack_top|StaticArray|Concurrent|with_color|Reference|Scheduler|read_line|Exception|at_exit|Pointer|Channel|Float64|sprintf|Float32|Process|Object|Struct|caller|UInt16|UInt32|UInt64|system|future|Number|printf|String|Symbol|Int32|Range|Slice|Regex|Mutex|sleep|Array|Class|raise|Tuple|Deque|delay|Float|Int16|print|abort|Value|UInt8|Int64|puts|Proc|File|Void|exit|fork|Bool|Char|gets|lazy|loop|main|rand|Enum|Int8|Time|Hash|Set|Box|Nil|Dir|Int|p)\b">
|
||||
<token type="NameBuiltin"/>
|
||||
</rule>
|
||||
<rule pattern="(?<!\w)(<<-?)(["`\']?)([a-zA-Z_]\w*)(\2)(.*?\n)">
|
||||
<token type="LiteralStringHeredoc"/>
|
||||
</rule>
|
||||
<rule pattern="(<<-?)("|\')()(\2)(.*?\n)">
|
||||
<token type="LiteralStringHeredoc"/>
|
||||
</rule>
|
||||
<rule pattern="__END__">
|
||||
<token type="CommentPreproc"/>
|
||||
<push state="end-part"/>
|
||||
</rule>
|
||||
<rule pattern="(?:^|(?<=[=<>~!:])|(?<=(?:\s|;)when\s)|(?<=(?:\s|;)or\s)|(?<=(?:\s|;)and\s)|(?<=\.index\s)|(?<=\.scan\s)|(?<=\.sub\s)|(?<=\.sub!\s)|(?<=\.gsub\s)|(?<=\.gsub!\s)|(?<=\.match\s)|(?<=(?:\s|;)if\s)|(?<=(?:\s|;)elsif\s)|(?<=^when\s)|(?<=^index\s)|(?<=^scan\s)|(?<=^sub\s)|(?<=^gsub\s)|(?<=^sub!\s)|(?<=^gsub!\s)|(?<=^match\s)|(?<=^if\s)|(?<=^elsif\s))(\s*)(/)">
|
||||
<bygroups>
|
||||
<token type="Text"/>
|
||||
<token type="LiteralStringRegex"/>
|
||||
</bygroups>
|
||||
<push state="multiline-regex"/>
|
||||
</rule>
|
||||
<rule pattern="(?<=\(|,|\[)/">
|
||||
<token type="LiteralStringRegex"/>
|
||||
<push state="multiline-regex"/>
|
||||
</rule>
|
||||
<rule pattern="(\s+)(/)(?![\s=])">
|
||||
<bygroups>
|
||||
<token type="Text"/>
|
||||
<token type="LiteralStringRegex"/>
|
||||
</bygroups>
|
||||
<push state="multiline-regex"/>
|
||||
</rule>
|
||||
<rule pattern="(0o[0-7]+(?:_[0-7]+)*(?:_?[iu][0-9]+)?)\b(\s*)([/?])?">
|
||||
<bygroups>
|
||||
<token type="LiteralNumberOct"/>
|
||||
<token type="Text"/>
|
||||
<token type="Operator"/>
|
||||
</bygroups>
|
||||
</rule>
|
||||
<rule pattern="(0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*(?:_?[iu][0-9]+)?)\b(\s*)([/?])?">
|
||||
<bygroups>
|
||||
<token type="LiteralNumberHex"/>
|
||||
<token type="Text"/>
|
||||
<token type="Operator"/>
|
||||
</bygroups>
|
||||
</rule>
|
||||
<rule pattern="(0b[01]+(?:_[01]+)*(?:_?[iu][0-9]+)?)\b(\s*)([/?])?">
|
||||
<bygroups>
|
||||
<token type="LiteralNumberBin"/>
|
||||
<token type="Text"/>
|
||||
<token type="Operator"/>
|
||||
</bygroups>
|
||||
</rule>
|
||||
<rule pattern="((?:0(?![0-9])|[1-9][\d_]*)(?:\.\d[\d_]*)(?:e[+-]?[0-9]+)?(?:_?f[0-9]+)?)(\s*)([/?])?">
|
||||
<bygroups>
|
||||
<token type="LiteralNumberFloat"/>
|
||||
<token type="Text"/>
|
||||
<token type="Operator"/>
|
||||
</bygroups>
|
||||
</rule>
|
||||
<rule pattern="((?:0(?![0-9])|[1-9][\d_]*)(?:\.\d[\d_]*)?(?:e[+-]?[0-9]+)(?:_?f[0-9]+)?)(\s*)([/?])?">
|
||||
<bygroups>
|
||||
<token type="LiteralNumberFloat"/>
|
||||
<token type="Text"/>
|
||||
<token type="Operator"/>
|
||||
</bygroups>
|
||||
</rule>
|
||||
<rule pattern="((?:0(?![0-9])|[1-9][\d_]*)(?:\.\d[\d_]*)?(?:e[+-]?[0-9]+)?(?:_?f[0-9]+))(\s*)([/?])?">
|
||||
<bygroups>
|
||||
<token type="LiteralNumberFloat"/>
|
||||
<token type="Text"/>
|
||||
<token type="Operator"/>
|
||||
</bygroups>
|
||||
</rule>
|
||||
<rule pattern="(0\b|[1-9][\d]*(?:_\d+)*(?:_?[iu][0-9]+)?)\b(\s*)([/?])?">
|
||||
<bygroups>
|
||||
<token type="LiteralNumberInteger"/>
|
||||
<token type="Text"/>
|
||||
<token type="Operator"/>
|
||||
</bygroups>
|
||||
</rule>
|
||||
<rule pattern="@@[a-zA-Z_]\w*">
|
||||
<token type="NameVariableClass"/>
|
||||
</rule>
|
||||
<rule pattern="@[a-zA-Z_]\w*">
|
||||
<token type="NameVariableInstance"/>
|
||||
</rule>
|
||||
<rule pattern="\$\w+">
|
||||
<token type="NameVariableGlobal"/>
|
||||
</rule>
|
||||
<rule pattern="\$[!@&`\'+~=/\\,;.<>_*$?:"^-]">
|
||||
<token type="NameVariableGlobal"/>
|
||||
</rule>
|
||||
<rule pattern="\$-[0adFiIlpvw]">
|
||||
<token type="NameVariableGlobal"/>
|
||||
</rule>
|
||||
<rule pattern="::">
|
||||
<token type="Operator"/>
|
||||
</rule>
|
||||
<rule>
|
||||
<include state="strings"/>
|
||||
</rule>
|
||||
<rule pattern="\?(\\[MC]-)*(\\([\\befnrtv#"\']|x[a-fA-F0-9]{1,2}|[0-7]{1,3})|\S)(?!\w)">
|
||||
<token type="LiteralStringChar"/>
|
||||
</rule>
|
||||
<rule pattern="[A-Z][A-Z_]+\b">
|
||||
<token type="NameConstant"/>
|
||||
</rule>
|
||||
<rule pattern="\{%">
|
||||
<token type="LiteralStringInterpol"/>
|
||||
<push state="in-macro-control"/>
|
||||
</rule>
|
||||
<rule pattern="\{\{">
|
||||
<token type="LiteralStringInterpol"/>
|
||||
<push state="in-macro-expr"/>
|
||||
</rule>
|
||||
<rule pattern="(@\[)(\s*)([A-Z]\w*)">
|
||||
<bygroups>
|
||||
<token type="Operator"/>
|
||||
<token type="Text"/>
|
||||
<token type="NameDecorator"/>
|
||||
</bygroups>
|
||||
<push state="in-attr"/>
|
||||
</rule>
|
||||
<rule pattern="(\.|::)(\[\]\?|<=>|===|\[\]=|>>|&&|\*\*|\[\]|\|\||>=|=~|!~|<<|<=|!=|==|<|/|=|-|\+|>|\*|&|%|\^|!|\||~)">
|
||||
<bygroups>
|
||||
<token type="Operator"/>
|
||||
<token type="NameOperator"/>
|
||||
</bygroups>
|
||||
</rule>
|
||||
<rule pattern="(\.|::)([a-zA-Z_]\w*[!?]?|[*%&^`~+\-/\[<>=])">
|
||||
<bygroups>
|
||||
<token type="Operator"/>
|
||||
<token type="Name"/>
|
||||
</bygroups>
|
||||
</rule>
|
||||
<rule pattern="[a-zA-Z_]\w*(?:[!?](?!=))?">
|
||||
<token type="Name"/>
|
||||
</rule>
|
||||
<rule pattern="(\[|\]\??|\*\*|<=>?|>=|<<?|>>?|=~|===|!~|&&?|\|\||\.{1,3})">
|
||||
<token type="Operator"/>
|
||||
</rule>
|
||||
<rule pattern="[-+/*%=<>&!^|~]=?">
|
||||
<token type="Operator"/>
|
||||
</rule>
|
||||
<rule pattern="[(){};,/?:\\]">
|
||||
<token type="Punctuation"/>
|
||||
</rule>
|
||||
<rule pattern="\s+">
|
||||
<token type="Text"/>
|
||||
</rule>
|
||||
</state>
|
||||
<state name="multiline-regex">
|
||||
<rule>
|
||||
<include state="string-intp"/>
|
||||
</rule>
|
||||
<rule pattern="\\\\">
|
||||
<token type="LiteralStringRegex"/>
|
||||
</rule>
|
||||
<rule pattern="\\/">
|
||||
<token type="LiteralStringRegex"/>
|
||||
</rule>
|
||||
<rule pattern="[\\#]">
|
||||
<token type="LiteralStringRegex"/>
|
||||
</rule>
|
||||
<rule pattern="[^\\/#]+">
|
||||
<token type="LiteralStringRegex"/>
|
||||
</rule>
|
||||
<rule pattern="/[imsx]*">
|
||||
<token type="LiteralStringRegex"/>
|
||||
<pop depth="1"/>
|
||||
</rule>
|
||||
</state>
|
||||
<state name="ab-string">
|
||||
<rule pattern="\\[\\<>]">
|
||||
<token type="LiteralStringOther"/>
|
||||
</rule>
|
||||
<rule pattern="<">
|
||||
<token type="LiteralStringOther"/>
|
||||
<push/>
|
||||
</rule>
|
||||
<rule pattern=">">
|
||||
<token type="LiteralStringOther"/>
|
||||
<pop depth="1"/>
|
||||
</rule>
|
||||
<rule pattern="[\\#<>]">
|
||||
<token type="LiteralStringOther"/>
|
||||
</rule>
|
||||
<rule pattern="[^\\#<>]+">
|
||||
<token type="LiteralStringOther"/>
|
||||
</rule>
|
||||
</state>
|
||||
<state name="pa-string">
|
||||
<rule pattern="\\[\\()]">
|
||||
<token type="LiteralStringOther"/>
|
||||
</rule>
|
||||
<rule pattern="\(">
|
||||
<token type="LiteralStringOther"/>
|
||||
<push/>
|
||||
</rule>
|
||||
<rule pattern="\)">
|
||||
<token type="LiteralStringOther"/>
|
||||
<pop depth="1"/>
|
||||
</rule>
|
||||
<rule pattern="[\\#()]">
|
||||
<token type="LiteralStringOther"/>
|
||||
</rule>
|
||||
<rule pattern="[^\\#()]+">
|
||||
<token type="LiteralStringOther"/>
|
||||
</rule>
|
||||
</state>
|
||||
<state name="strings">
|
||||
<rule pattern="\:@{0,2}[a-zA-Z_]\w*[!?]?">
|
||||
<token type="LiteralStringSymbol"/>
|
||||
</rule>
|
||||
<rule pattern="\:@{0,2}(\[\]\?|<=>|===|\[\]=|>>|&&|\*\*|\[\]|\|\||>=|=~|!~|<<|<=|!=|==|<|/|=|-|\+|>|\*|&|%|\^|!|\||~)">
|
||||
<token type="LiteralStringSymbol"/>
|
||||
</rule>
|
||||
<rule pattern=":'(\\\\|\\'|[^'])*'">
|
||||
<token type="LiteralStringSymbol"/>
|
||||
</rule>
|
||||
<rule pattern="'(\\\\|\\'|[^']|\\[^'\\]+)'">
|
||||
<token type="LiteralStringChar"/>
|
||||
</rule>
|
||||
<rule pattern=":"">
|
||||
<token type="LiteralStringSymbol"/>
|
||||
<push state="simple-sym"/>
|
||||
</rule>
|
||||
<rule pattern="([a-zA-Z_]\w*)(:)(?!:)">
|
||||
<bygroups>
|
||||
<token type="LiteralStringSymbol"/>
|
||||
<token type="Punctuation"/>
|
||||
</bygroups>
|
||||
</rule>
|
||||
<rule pattern=""">
|
||||
<token type="LiteralStringDouble"/>
|
||||
<push state="simple-string"/>
|
||||
</rule>
|
||||
<rule pattern="(?<!\.)`">
|
||||
<token type="LiteralStringBacktick"/>
|
||||
<push state="simple-backtick"/>
|
||||
</rule>
|
||||
<rule pattern="%\{">
|
||||
<token type="LiteralStringOther"/>
|
||||
<push state="cb-intp-string"/>
|
||||
</rule>
|
||||
<rule pattern="%[wi]\{">
|
||||
<token type="LiteralStringOther"/>
|
||||
<push state="cb-string"/>
|
||||
</rule>
|
||||
<rule pattern="%r\{">
|
||||
<token type="LiteralStringRegex"/>
|
||||
<push state="cb-regex"/>
|
||||
</rule>
|
||||
<rule pattern="%\[">
|
||||
<token type="LiteralStringOther"/>
|
||||
<push state="sb-intp-string"/>
|
||||
</rule>
|
||||
<rule pattern="%[wi]\[">
|
||||
<token type="LiteralStringOther"/>
|
||||
<push state="sb-string"/>
|
||||
</rule>
|
||||
<rule pattern="%r\[">
|
||||
<token type="LiteralStringRegex"/>
|
||||
<push state="sb-regex"/>
|
||||
</rule>
|
||||
<rule pattern="%\(">
|
||||
<token type="LiteralStringOther"/>
|
||||
<push state="pa-intp-string"/>
|
||||
</rule>
|
||||
<rule pattern="%[wi]\(">
|
||||
<token type="LiteralStringOther"/>
|
||||
<push state="pa-string"/>
|
||||
</rule>
|
||||
<rule pattern="%r\(">
|
||||
<token type="LiteralStringRegex"/>
|
||||
<push state="pa-regex"/>
|
||||
</rule>
|
||||
<rule pattern="%<">
|
||||
<token type="LiteralStringOther"/>
|
||||
<push state="ab-intp-string"/>
|
||||
</rule>
|
||||
<rule pattern="%[wi]<">
|
||||
<token type="LiteralStringOther"/>
|
||||
<push state="ab-string"/>
|
||||
</rule>
|
||||
<rule pattern="%r<">
|
||||
<token type="LiteralStringRegex"/>
|
||||
<push state="ab-regex"/>
|
||||
</rule>
|
||||
<rule pattern="(%r([\W_]))((?:\\\2|(?!\2).)*)(\2[imsx]*)">
|
||||
<token type="LiteralString"/>
|
||||
</rule>
|
||||
<rule pattern="(%[wi]([\W_]))((?:\\\2|(?!\2).)*)(\2)">
|
||||
<token type="LiteralString"/>
|
||||
</rule>
|
||||
<rule pattern="(?<=[-+/*%=<>&!^|~,(])(\s*)(%([\t ])(?:(?:\\\3|(?!\3).)*)\3)">
|
||||
<bygroups>
|
||||
<token type="Text"/>
|
||||
<token type="LiteralStringOther"/>
|
||||
<token type="None"/>
|
||||
</bygroups>
|
||||
</rule>
|
||||
<rule pattern="^(\s*)(%([\t ])(?:(?:\\\3|(?!\3).)*)\3)">
|
||||
<bygroups>
|
||||
<token type="Text"/>
|
||||
<token type="LiteralStringOther"/>
|
||||
<token type="None"/>
|
||||
</bygroups>
|
||||
</rule>
|
||||
<rule pattern="(%([\[{(<]))((?:\\\2|(?!\2).)*)(\2)">
|
||||
<token type="LiteralString"/>
|
||||
</rule>
|
||||
</state>
|
||||
<state name="sb-string">
|
||||
<rule pattern="\\[\\\[\]]">
|
||||
<token type="LiteralStringOther"/>
|
||||
</rule>
|
||||
<rule pattern="\[">
|
||||
<token type="LiteralStringOther"/>
|
||||
<push/>
|
||||
</rule>
|
||||
<rule pattern="\]">
|
||||
<token type="LiteralStringOther"/>
|
||||
<pop depth="1"/>
|
||||
</rule>
|
||||
<rule pattern="[\\#\[\]]">
|
||||
<token type="LiteralStringOther"/>
|
||||
</rule>
|
||||
<rule pattern="[^\\#\[\]]+">
|
||||
<token type="LiteralStringOther"/>
|
||||
</rule>
|
||||
</state>
|
||||
<state name="funcname">
|
||||
<rule pattern="(?:([a-zA-Z_]\w*)(\.))?([a-zA-Z_]\w*[!?]?|\*\*?|[-+]@?|[/%&|^`~]|\[\]=?|<<|>>|<=?>|>=?|===?)">
|
||||
<bygroups>
|
||||
<token type="NameClass"/>
|
||||
<token type="Operator"/>
|
||||
<token type="NameFunction"/>
|
||||
</bygroups>
|
||||
<pop depth="1"/>
|
||||
</rule>
|
||||
<rule>
|
||||
<pop depth="1"/>
|
||||
</rule>
|
||||
</state>
|
||||
<state name="simple-sym">
|
||||
<rule>
|
||||
<include state="string-escaped"/>
|
||||
</rule>
|
||||
<rule pattern="[^\\"#]+">
|
||||
<token type="LiteralStringSymbol"/>
|
||||
</rule>
|
||||
<rule pattern="[\\#]">
|
||||
<token type="LiteralStringSymbol"/>
|
||||
</rule>
|
||||
<rule pattern=""">
|
||||
<token type="LiteralStringSymbol"/>
|
||||
<pop depth="1"/>
|
||||
</rule>
|
||||
</state>
|
||||
</rules>
|
||||
</lexer>
|
@ -1,5 +1,5 @@
|
||||
name: tartrazine
|
||||
version: 0.6.3
|
||||
version: 0.7.0
|
||||
|
||||
authors:
|
||||
- Roberto Alsina <roberto.alsina@gmail.com>
|
||||
|
@ -28,6 +28,7 @@ bad_in_chroma = {
|
||||
"#{__DIR__}/tests/octave/test_multilinecomment.txt",
|
||||
"#{__DIR__}/tests/php/test_string_escaping_run.txt",
|
||||
"#{__DIR__}/tests/python_2/test_cls_builtin.txt",
|
||||
"#{__DIR__}/tests/bqn/test_syntax_roles.txt", # This one only fails in CI
|
||||
}
|
||||
|
||||
known_bad = {
|
||||
@ -68,12 +69,37 @@ describe Tartrazine do
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe "to_html" do
|
||||
it "should do basic highlighting" do
|
||||
html = Tartrazine.to_html("puts 'Hello, World!'", "ruby", standalone: false)
|
||||
html.should eq(
|
||||
"<pre class=\"b\" ><code class=\"b\"><span class=\"nb\">puts</span><span class=\"t\"> </span><span class=\"lss\">'Hello, World!'</span></code></pre>"
|
||||
)
|
||||
end
|
||||
end
|
||||
describe "to_ansi" do
|
||||
it "should do basic highlighting" do
|
||||
ansi = Tartrazine.to_ansi("puts 'Hello, World!'", "ruby")
|
||||
if ENV.fetch("CI", nil)
|
||||
# In Github Actions there is no terminal so these don't
|
||||
# really work
|
||||
ansi.should eq(
|
||||
"puts 'Hello, World!'"
|
||||
)
|
||||
else
|
||||
ansi.should eq(
|
||||
"\e[38;2;171;70;66mputs\e[0m\e[38;2;216;216;216m \e[0m'Hello, World!'"
|
||||
)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Helper that creates lexer and tokenizes
|
||||
def tokenize(lexer_name, text)
|
||||
tokenizer = Tartrazine.lexer(lexer_name).tokenizer(text)
|
||||
Tartrazine::Lexer.collapse_tokens(tokenizer.to_a)
|
||||
Tartrazine::RegexLexer.collapse_tokens(tokenizer.to_a)
|
||||
end
|
||||
|
||||
# Helper that tokenizes using chroma to validate the lexer
|
||||
@ -85,5 +111,5 @@ def chroma_tokenize(lexer_name, text)
|
||||
["-f", "json", "-l", lexer_name],
|
||||
input: input, output: output
|
||||
)
|
||||
Tartrazine::Lexer.collapse_tokens(Array(Tartrazine::Token).from_json(output.to_s))
|
||||
Tartrazine::RegexLexer.collapse_tokens(Array(Tartrazine::Token).from_json(output.to_s))
|
||||
end
|
||||
|
@ -1,16 +0,0 @@
|
||||
---input---
|
||||
@[FOO::Bar::Baz(opt: "xx")]
|
||||
|
||||
---tokens---
|
||||
'@[' Operator
|
||||
'FOO::Bar::Baz' Name.Decorator
|
||||
'(' Punctuation
|
||||
'opt' Literal.String.Symbol
|
||||
':' Punctuation
|
||||
' ' Text.Whitespace
|
||||
'"' Literal.String.Double
|
||||
'xx' Literal.String.Double
|
||||
'"' Literal.String.Double
|
||||
')' Punctuation
|
||||
']' Operator
|
||||
'\n' Text.Whitespace
|
@ -1,11 +0,0 @@
|
||||
---input---
|
||||
[5][5]?
|
||||
|
||||
---tokens---
|
||||
'[' Operator
|
||||
'5' Literal.Number.Integer
|
||||
']' Operator
|
||||
'[' Operator
|
||||
'5' Literal.Number.Integer
|
||||
']?' Operator
|
||||
'\n' Text.Whitespace
|
@ -1,25 +0,0 @@
|
||||
---input---
|
||||
'a'
|
||||
'я'
|
||||
'\u{1234}'
|
||||
'
|
||||
'
|
||||
'abc'
|
||||
|
||||
---tokens---
|
||||
"'a'" Literal.String.Char
|
||||
'\n' Text.Whitespace
|
||||
|
||||
"'я'" Literal.String.Char
|
||||
'\n' Text.Whitespace
|
||||
|
||||
"'\\u{1234}'" Literal.String.Char
|
||||
'\n' Text.Whitespace
|
||||
|
||||
"'\n'" Literal.String.Char
|
||||
'\n' Text.Whitespace
|
||||
|
||||
"'" Error
|
||||
'abc' Name
|
||||
"'" Error
|
||||
'\n' Text.Whitespace
|
@ -1,14 +0,0 @@
|
||||
---input---
|
||||
HTTP
|
||||
HTTP::Server.new
|
||||
|
||||
---tokens---
|
||||
'HTTP' Name.Constant
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'HTTP' Name
|
||||
'::' Operator
|
||||
'Server' Name
|
||||
'.' Operator
|
||||
'new' Name
|
||||
'\n' Text.Whitespace
|
@ -1,27 +0,0 @@
|
||||
---input---
|
||||
%()
|
||||
%[]
|
||||
%{}
|
||||
%<>
|
||||
%||
|
||||
|
||||
---tokens---
|
||||
'%(' Literal.String.Other
|
||||
')' Literal.String.Other
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'%[' Literal.String.Other
|
||||
']' Literal.String.Other
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'%{' Literal.String.Other
|
||||
'}' Literal.String.Other
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'%<' Literal.String.Other
|
||||
'>' Literal.String.Other
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'%|' Literal.String.Other
|
||||
'|' Literal.String.Other
|
||||
'\n' Text.Whitespace
|
@ -1,19 +0,0 @@
|
||||
---input---
|
||||
str.gsub(%r{\\\\}, "/")
|
||||
|
||||
---tokens---
|
||||
'str' Name
|
||||
'.' Operator
|
||||
'gsub' Name
|
||||
'(' Punctuation
|
||||
'%r{' Literal.String.Regex
|
||||
'\\\\' Literal.String.Regex
|
||||
'\\\\' Literal.String.Regex
|
||||
'}' Literal.String.Regex
|
||||
',' Punctuation
|
||||
' ' Text.Whitespace
|
||||
'"' Literal.String.Double
|
||||
'/' Literal.String.Double
|
||||
'"' Literal.String.Double
|
||||
')' Punctuation
|
||||
'\n' Text.Whitespace
|
@ -1,9 +0,0 @@
|
||||
---input---
|
||||
"\#{a + b}"
|
||||
|
||||
---tokens---
|
||||
'"' Literal.String.Double
|
||||
'\\#' Literal.String.Escape
|
||||
'{a + b}' Literal.String.Double
|
||||
'"' Literal.String.Double
|
||||
'\n' Text.Whitespace
|
@ -1,56 +0,0 @@
|
||||
---input---
|
||||
"A#{ (3..5).group_by { |x| x/2}.map do |k,v| "#{k}" end.join }" + "Z"
|
||||
|
||||
---tokens---
|
||||
'"' Literal.String.Double
|
||||
'A' Literal.String.Double
|
||||
'#{' Literal.String.Interpol
|
||||
' ' Text.Whitespace
|
||||
'(' Punctuation
|
||||
'3' Literal.Number.Integer
|
||||
'..' Operator
|
||||
'5' Literal.Number.Integer
|
||||
')' Punctuation
|
||||
'.' Operator
|
||||
'group_by' Name
|
||||
' ' Text.Whitespace
|
||||
'{' Literal.String.Interpol
|
||||
' ' Text.Whitespace
|
||||
'|' Operator
|
||||
'x' Name
|
||||
'|' Operator
|
||||
' ' Text.Whitespace
|
||||
'x' Name
|
||||
'/' Operator
|
||||
'2' Literal.Number.Integer
|
||||
'}' Literal.String.Interpol
|
||||
'.' Operator
|
||||
'map' Name
|
||||
' ' Text.Whitespace
|
||||
'do' Keyword
|
||||
' ' Text.Whitespace
|
||||
'|' Operator
|
||||
'k' Name
|
||||
',' Punctuation
|
||||
'v' Name
|
||||
'|' Operator
|
||||
' ' Text.Whitespace
|
||||
'"' Literal.String.Double
|
||||
'#{' Literal.String.Interpol
|
||||
'k' Name
|
||||
'}' Literal.String.Interpol
|
||||
'"' Literal.String.Double
|
||||
' ' Text.Whitespace
|
||||
'end' Keyword
|
||||
'.' Operator
|
||||
'join' Name
|
||||
' ' Text.Whitespace
|
||||
'}' Literal.String.Interpol
|
||||
'"' Literal.String.Double
|
||||
' ' Text.Whitespace
|
||||
'+' Operator
|
||||
' ' Text.Whitespace
|
||||
'"' Literal.String.Double
|
||||
'Z' Literal.String.Double
|
||||
'"' Literal.String.Double
|
||||
'\n' Text.Whitespace
|
@ -1,58 +0,0 @@
|
||||
---input---
|
||||
@[Link("some")]
|
||||
lib LibSome
|
||||
@[CallConvention("X86_StdCall")]
|
||||
fun foo="some.foo"(thing : Void*) : LibC::Int
|
||||
end
|
||||
|
||||
---tokens---
|
||||
'@[' Operator
|
||||
'Link' Name.Decorator
|
||||
'(' Punctuation
|
||||
'"' Literal.String.Double
|
||||
'some' Literal.String.Double
|
||||
'"' Literal.String.Double
|
||||
')' Punctuation
|
||||
']' Operator
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'lib' Keyword
|
||||
' ' Text.Whitespace
|
||||
'LibSome' Name.Namespace
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'@[' Operator
|
||||
'CallConvention' Name.Decorator
|
||||
'(' Punctuation
|
||||
'"' Literal.String.Double
|
||||
'X86_StdCall' Literal.String.Double
|
||||
'"' Literal.String.Double
|
||||
')' Punctuation
|
||||
']' Operator
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'fun' Keyword
|
||||
' ' Text.Whitespace
|
||||
'foo' Name.Function
|
||||
'=' Operator
|
||||
'"' Literal.String.Double
|
||||
'some.foo' Literal.String.Double
|
||||
'"' Literal.String.Double
|
||||
'(' Punctuation
|
||||
'thing' Name
|
||||
' ' Text.Whitespace
|
||||
':' Punctuation
|
||||
' ' Text.Whitespace
|
||||
'Void' Name
|
||||
'*' Operator
|
||||
')' Punctuation
|
||||
' ' Text.Whitespace
|
||||
':' Punctuation
|
||||
' ' Text.Whitespace
|
||||
'LibC' Name
|
||||
'::' Operator
|
||||
'Int' Name
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'end' Keyword
|
||||
'\n' Text.Whitespace
|
@ -1,76 +0,0 @@
|
||||
---input---
|
||||
def<=>(other : self) : Int
|
||||
{%for field in %w(first_name middle_name last_name)%}
|
||||
cmp={{field.id}}<=>other.{{field.id}}
|
||||
return cmp if cmp!=0
|
||||
{%end%}
|
||||
0
|
||||
end
|
||||
|
||||
---tokens---
|
||||
'def' Keyword
|
||||
'<=>' Name.Function
|
||||
'(' Punctuation
|
||||
'other' Name
|
||||
' ' Text.Whitespace
|
||||
':' Punctuation
|
||||
' ' Text.Whitespace
|
||||
'self' Keyword
|
||||
')' Punctuation
|
||||
' ' Text.Whitespace
|
||||
':' Punctuation
|
||||
' ' Text.Whitespace
|
||||
'Int' Name
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'{%' Literal.String.Interpol
|
||||
'for' Keyword
|
||||
' ' Text.Whitespace
|
||||
'field' Name
|
||||
' ' Text.Whitespace
|
||||
'in' Keyword
|
||||
' ' Text.Whitespace
|
||||
'%w(' Literal.String.Other
|
||||
'first_name middle_name last_name' Literal.String.Other
|
||||
')' Literal.String.Other
|
||||
'%}' Literal.String.Interpol
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'cmp' Name
|
||||
'=' Operator
|
||||
'{{' Literal.String.Interpol
|
||||
'field' Name
|
||||
'.' Operator
|
||||
'id' Name
|
||||
'}}' Literal.String.Interpol
|
||||
'<=>' Operator
|
||||
'other' Name
|
||||
'.' Operator
|
||||
'{{' Literal.String.Interpol
|
||||
'field' Name
|
||||
'.' Operator
|
||||
'id' Name
|
||||
'}}' Literal.String.Interpol
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'return' Keyword
|
||||
' ' Text.Whitespace
|
||||
'cmp' Name
|
||||
' ' Text.Whitespace
|
||||
'if' Keyword
|
||||
' ' Text.Whitespace
|
||||
'cmp' Name
|
||||
'!=' Operator
|
||||
'0' Literal.Number.Integer
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'{%' Literal.String.Interpol
|
||||
'end' Keyword
|
||||
'%}' Literal.String.Interpol
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'0' Literal.Number.Integer
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'end' Keyword
|
||||
'\n' Text.Whitespace
|
@ -1,84 +0,0 @@
|
||||
---input---
|
||||
# Integers
|
||||
0
|
||||
1
|
||||
1_000_000
|
||||
1u8
|
||||
11231231231121312i64
|
||||
|
||||
# Floats
|
||||
0.0
|
||||
1.0_f32
|
||||
1_f32
|
||||
0f64
|
||||
1e+4
|
||||
1e111
|
||||
1_234.567_890
|
||||
|
||||
# Error
|
||||
01
|
||||
0b2
|
||||
0x129g2
|
||||
0o12358
|
||||
|
||||
---tokens---
|
||||
'# Integers' Comment.Single
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'0' Literal.Number.Integer
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'1' Literal.Number.Integer
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'1_000_000' Literal.Number.Integer
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'1u8' Literal.Number.Integer
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'11231231231121312i64' Literal.Number.Integer
|
||||
'\n\n' Text.Whitespace
|
||||
|
||||
'# Floats' Comment.Single
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'0.0' Literal.Number.Float
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'1.0_f32' Literal.Number.Float
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'1_f32' Literal.Number.Float
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'0f64' Literal.Number.Float
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'1e+4' Literal.Number.Float
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'1e111' Literal.Number.Float
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'1_234.567_890' Literal.Number.Float
|
||||
'\n\n' Text.Whitespace
|
||||
|
||||
'# Error' Comment.Single
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'0' Error
|
||||
'1' Literal.Number.Integer
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'0' Error
|
||||
'b2' Name
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'0' Error
|
||||
'x129g2' Name
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'0' Error
|
||||
'o12358' Name
|
||||
'\n' Text.Whitespace
|
@ -1,18 +0,0 @@
|
||||
---input---
|
||||
([] of Int32).[]?(5)
|
||||
|
||||
---tokens---
|
||||
'(' Punctuation
|
||||
'[' Operator
|
||||
']' Operator
|
||||
' ' Text.Whitespace
|
||||
'of' Keyword
|
||||
' ' Text.Whitespace
|
||||
'Int32' Name
|
||||
')' Punctuation
|
||||
'.' Operator
|
||||
'[]?' Name.Operator
|
||||
'(' Punctuation
|
||||
'5' Literal.Number.Integer
|
||||
')' Punctuation
|
||||
'\n' Text.Whitespace
|
@ -1,41 +0,0 @@
|
||||
---input---
|
||||
%(hello ("world"))
|
||||
%[hello ["world"]]
|
||||
%{hello "world"}
|
||||
%<hello <"world">>
|
||||
%|hello "world"|
|
||||
|
||||
---tokens---
|
||||
'%(' Literal.String.Other
|
||||
'hello ' Literal.String.Other
|
||||
'(' Literal.String.Other
|
||||
'"world"' Literal.String.Other
|
||||
')' Literal.String.Other
|
||||
')' Literal.String.Other
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'%[' Literal.String.Other
|
||||
'hello ' Literal.String.Other
|
||||
'[' Literal.String.Other
|
||||
'"world"' Literal.String.Other
|
||||
']' Literal.String.Other
|
||||
']' Literal.String.Other
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'%{' Literal.String.Other
|
||||
'hello "world"' Literal.String.Other
|
||||
'}' Literal.String.Other
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'%<' Literal.String.Other
|
||||
'hello ' Literal.String.Other
|
||||
'<' Literal.String.Other
|
||||
'"world"' Literal.String.Other
|
||||
'>' Literal.String.Other
|
||||
'>' Literal.String.Other
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'%|' Literal.String.Other
|
||||
'hello "world"' Literal.String.Other
|
||||
'|' Literal.String.Other
|
||||
'\n' Text.Whitespace
|
@ -1,31 +0,0 @@
|
||||
---input---
|
||||
%Q(hello \n #{name})
|
||||
%q(hello \n #{name})
|
||||
%w(foo\nbar baz)
|
||||
|
||||
---tokens---
|
||||
'%Q(' Literal.String.Other
|
||||
'hello ' Literal.String.Other
|
||||
'\\n' Literal.String.Escape
|
||||
' ' Literal.String.Other
|
||||
'#{' Literal.String.Interpol
|
||||
'name' Name
|
||||
'}' Literal.String.Interpol
|
||||
')' Literal.String.Other
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'%q(' Literal.String.Other
|
||||
'hello ' Literal.String.Other
|
||||
'\\' Literal.String.Other
|
||||
'n ' Literal.String.Other
|
||||
'#' Literal.String.Other
|
||||
'{name}' Literal.String.Other
|
||||
')' Literal.String.Other
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'%w(' Literal.String.Other
|
||||
'foo' Literal.String.Other
|
||||
'\\' Literal.String.Other
|
||||
'nbar baz' Literal.String.Other
|
||||
')' Literal.String.Other
|
||||
'\n' Text.Whitespace
|
@ -1,20 +0,0 @@
|
||||
---input---
|
||||
record Cls do
|
||||
def_equals s
|
||||
end
|
||||
|
||||
---tokens---
|
||||
'record' Name.Builtin.Pseudo
|
||||
' ' Text.Whitespace
|
||||
'Cls' Name
|
||||
' ' Text.Whitespace
|
||||
'do' Keyword
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'def_equals' Name.Builtin.Pseudo
|
||||
' ' Text.Whitespace
|
||||
's' Name
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'end' Keyword
|
||||
'\n' Text.Whitespace
|
@ -1,50 +0,0 @@
|
||||
---input---
|
||||
def f(x : T, line = __LINE__) forall T
|
||||
if x.is_a?(String)
|
||||
pp! x
|
||||
end
|
||||
end
|
||||
|
||||
---tokens---
|
||||
'def' Keyword
|
||||
' ' Text.Whitespace
|
||||
'f' Name.Function
|
||||
'(' Punctuation
|
||||
'x' Name
|
||||
' ' Text.Whitespace
|
||||
':' Punctuation
|
||||
' ' Text.Whitespace
|
||||
'T' Name
|
||||
',' Punctuation
|
||||
' ' Text.Whitespace
|
||||
'line' Name
|
||||
' ' Text.Whitespace
|
||||
'=' Operator
|
||||
' ' Text.Whitespace
|
||||
'__LINE__' Keyword.Pseudo
|
||||
')' Punctuation
|
||||
' ' Text.Whitespace
|
||||
'forall' Keyword.Pseudo
|
||||
' ' Text.Whitespace
|
||||
'T' Name
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'if' Keyword
|
||||
' ' Text.Whitespace
|
||||
'x' Name
|
||||
'.is_a?' Keyword.Pseudo
|
||||
'(' Punctuation
|
||||
'String' Name
|
||||
')' Punctuation
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'pp!' Name.Builtin.Pseudo
|
||||
' ' Text.Whitespace
|
||||
'x' Name
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'end' Keyword
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'end' Keyword
|
||||
'\n' Text.Whitespace
|
@ -1,8 +0,0 @@
|
||||
---input---
|
||||
1...3
|
||||
|
||||
---tokens---
|
||||
'1' Literal.Number.Integer
|
||||
'...' Operator
|
||||
'3' Literal.Number.Integer
|
||||
'\n' Text.Whitespace
|
@ -1,10 +0,0 @@
|
||||
---input---
|
||||
1 .. 3
|
||||
|
||||
---tokens---
|
||||
'1' Literal.Number.Integer
|
||||
' ' Text.Whitespace
|
||||
'..' Operator
|
||||
' ' Text.Whitespace
|
||||
'3' Literal.Number.Integer
|
||||
'\n' Text.Whitespace
|
@ -1,58 +0,0 @@
|
||||
---input---
|
||||
"a\nz"
|
||||
"a\az"
|
||||
"a\xffz"
|
||||
"a\u1234z"
|
||||
"a\000z"
|
||||
"a\u{0}z"
|
||||
"a\u{10AfF9}z"
|
||||
|
||||
---tokens---
|
||||
'"' Literal.String.Double
|
||||
'a' Literal.String.Double
|
||||
'\\n' Literal.String.Escape
|
||||
'z' Literal.String.Double
|
||||
'"' Literal.String.Double
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'"' Literal.String.Double
|
||||
'a' Literal.String.Double
|
||||
'\\a' Literal.String.Escape
|
||||
'z' Literal.String.Double
|
||||
'"' Literal.String.Double
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'"' Literal.String.Double
|
||||
'a' Literal.String.Double
|
||||
'\\xff' Literal.String.Escape
|
||||
'z' Literal.String.Double
|
||||
'"' Literal.String.Double
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'"' Literal.String.Double
|
||||
'a' Literal.String.Double
|
||||
'\\u1234' Literal.String.Escape
|
||||
'z' Literal.String.Double
|
||||
'"' Literal.String.Double
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'"' Literal.String.Double
|
||||
'a' Literal.String.Double
|
||||
'\\000' Literal.String.Escape
|
||||
'z' Literal.String.Double
|
||||
'"' Literal.String.Double
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'"' Literal.String.Double
|
||||
'a' Literal.String.Double
|
||||
'\\u{0}' Literal.String.Escape
|
||||
'z' Literal.String.Double
|
||||
'"' Literal.String.Double
|
||||
'\n' Text.Whitespace
|
||||
|
||||
'"' Literal.String.Double
|
||||
'a' Literal.String.Double
|
||||
'\\u{10AfF9}' Literal.String.Escape
|
||||
'z' Literal.String.Double
|
||||
'"' Literal.String.Double
|
||||
'\n' Text.Whitespace
|
@ -1,20 +0,0 @@
|
||||
---input---
|
||||
:sym_bol
|
||||
:あ
|
||||
:question?
|
||||
:"symbol"
|
||||
|
||||
---tokens---
|
||||
':sym_bol' Literal.String.Symbol
|
||||
'\n' Text.Whitespace
|
||||
|
||||
':あ' Literal.String.Symbol
|
||||
'\n' Text.Whitespace
|
||||
|
||||
':question?' Literal.String.Symbol
|
||||
'\n' Text.Whitespace
|
||||
|
||||
':"' Literal.String.Symbol
|
||||
'symbol' Literal.String.Symbol
|
||||
'"' Literal.String.Symbol
|
||||
'\n' Text.Whitespace
|
@ -1,6 +1,15 @@
|
||||
require "../formatter"
|
||||
|
||||
module Tartrazine
|
||||
def self.to_ansi(text : String, language : String,
|
||||
theme : String = "default-dark",
|
||||
line_numbers : Bool = false) : String
|
||||
Tartrazine::Ansi.new(
|
||||
theme: Tartrazine.theme(theme),
|
||||
line_numbers: line_numbers
|
||||
).format(text, Tartrazine.lexer(name: language))
|
||||
end
|
||||
|
||||
class Ansi < Formatter
|
||||
property? line_numbers : Bool = false
|
||||
|
||||
|
@ -3,6 +3,17 @@ require "../formatter"
|
||||
require "html"
|
||||
|
||||
module Tartrazine
|
||||
def self.to_html(text : String, language : String,
|
||||
theme : String = "default-dark",
|
||||
standalone : Bool = true,
|
||||
line_numbers : Bool = false) : String
|
||||
Tartrazine::Html.new(
|
||||
theme: Tartrazine.theme(theme),
|
||||
standalone: standalone,
|
||||
line_numbers: line_numbers
|
||||
).format(text, Tartrazine.lexer(name: language))
|
||||
end
|
||||
|
||||
class Html < Formatter
|
||||
# property line_number_in_table : Bool = false
|
||||
# property with_classes : Bool = true
|
||||
|
@ -12,7 +12,7 @@ module Tartrazine
|
||||
|
||||
def format(text : String, lexer : BaseLexer, io : IO) : Nil
|
||||
tokenizer = lexer.tokenizer(text)
|
||||
io << Tartrazine::Lexer.collapse_tokens(tokenizer.to_a).to_json
|
||||
io << Tartrazine::RegexLexer.collapse_tokens(tokenizer.to_a).to_json
|
||||
end
|
||||
end
|
||||
end
|
||||
|
104
src/lexer.cr
104
src/lexer.cr
@ -1,5 +1,7 @@
|
||||
require "baked_file_system"
|
||||
require "./constants/lexers"
|
||||
require "./heuristics"
|
||||
require "baked_file_system"
|
||||
require "crystal/syntax_highlighter"
|
||||
|
||||
module Tartrazine
|
||||
class LexerFiles
|
||||
@ -14,25 +16,30 @@ module Tartrazine
|
||||
return lexer_by_filename(filename) if filename
|
||||
return lexer_by_mimetype(mimetype) if mimetype
|
||||
|
||||
Lexer.from_xml(LexerFiles.get("/#{LEXERS_BY_NAME["plaintext"]}.xml").gets_to_end)
|
||||
RegexLexer.from_xml(LexerFiles.get("/#{LEXERS_BY_NAME["plaintext"]}.xml").gets_to_end)
|
||||
end
|
||||
|
||||
private def self.lexer_by_mimetype(mimetype : String) : BaseLexer
|
||||
lexer_file_name = LEXERS_BY_MIMETYPE.fetch(mimetype, nil)
|
||||
raise Exception.new("Unknown mimetype: #{mimetype}") if lexer_file_name.nil?
|
||||
|
||||
Lexer.from_xml(LexerFiles.get("/#{lexer_file_name}.xml").gets_to_end)
|
||||
RegexLexer.from_xml(LexerFiles.get("/#{lexer_file_name}.xml").gets_to_end)
|
||||
end
|
||||
|
||||
private def self.lexer_by_name(name : String) : BaseLexer
|
||||
return CrystalLexer.new if name == "crystal"
|
||||
lexer_file_name = LEXERS_BY_NAME.fetch(name.downcase, nil)
|
||||
return create_delegating_lexer(name) if lexer_file_name.nil? && name.includes? "+"
|
||||
raise Exception.new("Unknown lexer: #{name}") if lexer_file_name.nil?
|
||||
|
||||
Lexer.from_xml(LexerFiles.get("/#{lexer_file_name}.xml").gets_to_end)
|
||||
RegexLexer.from_xml(LexerFiles.get("/#{lexer_file_name}.xml").gets_to_end)
|
||||
end
|
||||
|
||||
private def self.lexer_by_filename(filename : String) : BaseLexer
|
||||
if filename.ends_with?(".cr")
|
||||
return CrystalLexer.new
|
||||
end
|
||||
|
||||
candidates = Set(String).new
|
||||
LEXERS_BY_FILENAME.each do |k, v|
|
||||
candidates += v.to_set if File.match?(k, File.basename(filename))
|
||||
@ -52,7 +59,7 @@ module Tartrazine
|
||||
end
|
||||
end
|
||||
|
||||
Lexer.from_xml(LexerFiles.get("/#{lexer_file_name}.xml").gets_to_end)
|
||||
RegexLexer.from_xml(LexerFiles.get("/#{lexer_file_name}.xml").gets_to_end)
|
||||
end
|
||||
|
||||
private def self.lexer_by_content(fname : String) : String?
|
||||
@ -152,7 +159,9 @@ module Tartrazine
|
||||
end
|
||||
end
|
||||
|
||||
abstract class BaseLexer
|
||||
alias BaseLexer = Lexer
|
||||
|
||||
abstract class Lexer
|
||||
property config = {
|
||||
name: "",
|
||||
priority: 0.0,
|
||||
@ -174,7 +183,7 @@ module Tartrazine
|
||||
# For explanations on what actions and states do
|
||||
# the Pygments documentation is a good place to start.
|
||||
# https://pygments.org/docs/lexerdevelopment/
|
||||
class Lexer < BaseLexer
|
||||
class RegexLexer < BaseLexer
|
||||
# Collapse consecutive tokens of the same type for easier comparison
|
||||
# and smaller output
|
||||
def self.collapse_tokens(tokens : Array(Tartrazine::Token)) : Array(Tartrazine::Token)
|
||||
@ -198,7 +207,7 @@ module Tartrazine
|
||||
end
|
||||
|
||||
def self.from_xml(xml : String) : Lexer
|
||||
l = Lexer.new
|
||||
l = RegexLexer.new
|
||||
lexer = XML.parse(xml).first_element_child
|
||||
if lexer
|
||||
config = lexer.children.find { |node|
|
||||
@ -263,7 +272,7 @@ module Tartrazine
|
||||
#
|
||||
# This is useful for things like template languages, where
|
||||
# you have Jinja + HTML or Jinja + CSS and so on.
|
||||
class DelegatingLexer < BaseLexer
|
||||
class DelegatingLexer < Lexer
|
||||
property language_lexer : BaseLexer
|
||||
property root_lexer : BaseLexer
|
||||
|
||||
@ -324,4 +333,81 @@ module Tartrazine
|
||||
new_state
|
||||
end
|
||||
end
|
||||
|
||||
class CustomCrystalHighlighter < Crystal::SyntaxHighlighter
|
||||
@tokens = [] of Token
|
||||
|
||||
def render_delimiter(&block)
|
||||
@tokens << {type: "LiteralString", value: block.call.to_s}
|
||||
end
|
||||
|
||||
def render_interpolation(&block)
|
||||
@tokens << {type: "LiteralStringInterpol", value: "\#{"}
|
||||
@tokens << {type: "Text", value: block.call.to_s}
|
||||
@tokens << {type: "LiteralStringInterpol", value: "}"}
|
||||
end
|
||||
|
||||
def render_string_array(&block)
|
||||
@tokens << {type: "LiteralString", value: block.call.to_s}
|
||||
end
|
||||
|
||||
# ameba:disable Metrics/CyclomaticComplexity
|
||||
def render(type : TokenType, value : String)
|
||||
case type
|
||||
when .comment?
|
||||
@tokens << {type: "Comment", value: value}
|
||||
when .number?
|
||||
@tokens << {type: "LiteralNumber", value: value}
|
||||
when .char?
|
||||
@tokens << {type: "LiteralStringChar", value: value}
|
||||
when .symbol?
|
||||
@tokens << {type: "LiteralStringSymbol", value: value}
|
||||
when .const?
|
||||
@tokens << {type: "NameConstant", value: value}
|
||||
when .string?
|
||||
@tokens << {type: "LiteralString", value: value}
|
||||
when .ident?
|
||||
@tokens << {type: "NameVariable", value: value}
|
||||
when .keyword?, .self?
|
||||
@tokens << {type: "NameKeyword", value: value}
|
||||
when .primitive_literal?
|
||||
@tokens << {type: "Literal", value: value}
|
||||
when .operator?
|
||||
@tokens << {type: "Operator", value: value}
|
||||
when Crystal::SyntaxHighlighter::TokenType::DELIMITED_TOKEN, Crystal::SyntaxHighlighter::TokenType::DELIMITER_START, Crystal::SyntaxHighlighter::TokenType::DELIMITER_END
|
||||
@tokens << {type: "LiteralString", value: value}
|
||||
else
|
||||
@tokens << {type: "Text", value: value}
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
class CrystalTokenizer < Tartrazine::BaseTokenizer
|
||||
include Iterator(Token)
|
||||
@hl = CustomCrystalHighlighter.new
|
||||
@lexer : BaseLexer
|
||||
@iter : Iterator(Token)
|
||||
|
||||
# delegate next, to: @iter
|
||||
|
||||
def initialize(@lexer : BaseLexer, text : String, secondary = false)
|
||||
# Respect the `ensure_nl` config option
|
||||
if text.size > 0 && text[-1] != '\n' && @lexer.config[:ensure_nl] && !secondary
|
||||
text += "\n"
|
||||
end
|
||||
# Just do the tokenizing
|
||||
@hl.highlight(text)
|
||||
@iter = @hl.@tokens.each
|
||||
end
|
||||
|
||||
def next : Iterator::Stop | Token
|
||||
@iter.next
|
||||
end
|
||||
end
|
||||
|
||||
class CrystalLexer < BaseLexer
|
||||
def tokenizer(text : String, secondary = false) : BaseTokenizer
|
||||
CrystalTokenizer.new(self, text, secondary)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
@ -1,5 +1,5 @@
|
||||
require "docopt"
|
||||
require "./**"
|
||||
require "./tartrazine"
|
||||
|
||||
HELP = <<-HELP
|
||||
tartrazine: a syntax highlighting tool
|
||||
|
@ -17,7 +17,6 @@ module Tartrazine
|
||||
|
||||
abstract struct BaseRule
|
||||
abstract def match(text : Bytes, pos : Int32, tokenizer : Tokenizer) : Tuple(Bool, Int32, Array(Token))
|
||||
abstract def initialize(node : XML::Node)
|
||||
|
||||
@actions : Array(Action) = [] of Action
|
||||
|
||||
@ -40,9 +39,6 @@ module Tartrazine
|
||||
return true, pos + match[0].size, @actions.flat_map(&.emit(match, tokenizer))
|
||||
end
|
||||
|
||||
def initialize(node : XML::Node)
|
||||
end
|
||||
|
||||
def initialize(node : XML::Node, multiline, dotall, ignorecase)
|
||||
pattern = node["pattern"]
|
||||
pattern = "(?m)" + pattern if multiline
|
||||
|
1
src/run_tests.cr
Normal file
1
src/run_tests.cr
Normal file
@ -0,0 +1 @@
|
||||
require "../spec/**"
|
@ -84,27 +84,6 @@ module Tartrazine
|
||||
|
||||
property styles = {} of String => Style
|
||||
|
||||
# Get the style for a token.
|
||||
def style(token)
|
||||
styles[token] = Style.new unless styles.has_key?(token)
|
||||
s = styles[token]
|
||||
|
||||
# We already got the data from the style hierarchy
|
||||
return s if s.complete?
|
||||
|
||||
# Form the hierarchy of parent styles
|
||||
parents = style_parents(token)
|
||||
|
||||
s = parents.map do |parent|
|
||||
styles[parent]
|
||||
end.reduce(s) do |acc, style|
|
||||
acc + style
|
||||
end
|
||||
s.complete = true
|
||||
styles[token] = s
|
||||
s
|
||||
end
|
||||
|
||||
def style_parents(token)
|
||||
parents = ["Background"]
|
||||
parts = token.underscore.split("_").map(&.capitalize)
|
||||
|
@ -1,5 +1,6 @@
|
||||
require "./actions"
|
||||
require "./formatter"
|
||||
require "./formatters/**"
|
||||
require "./rules"
|
||||
require "./styles"
|
||||
require "./tartrazine"
|
||||
|
Reference in New Issue
Block a user