Added helper files

.gitignore

@@ -6,3 +6,4 @@
 chroma/
 pygments/
 shard.lock
+.vscode/

Makefile

@@ -0,0 +1,7 @@
+build: $(wildcard src/**/*.cr) $(wildcard lexers/*xml) $(wildcard styles/*xml) shard.yml
+	shards build -Dstrict_multi_assign -Dno_number_autocast
+release: $(wildcard src/**/*.cr) $(wildcard lexers/*xml) $(wildcard styles/*xml) shard.yml
+	shards build --release
+static: $(wildcard src/**/*.cr) $(wildcard lexers/*xml) $(wildcard styles/*xml) shard.yml
+	shards build --release --static
+	strip bin/tartrazine

build_static.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -e
+
+docker run --rm --privileged \
+  multiarch/qemu-user-static \
+  --reset -p yes
+
+# Build for AMD64
+docker build . -f Dockerfile.static -t tartrazine-builder
+docker run -ti --rm -v "$PWD":/app --user="$UID" tartrazine-builder /bin/sh -c "cd /app && rm -rf lib shard.lock && make static"
+mv bin/tartrazine bin/tartrazine-static-linux-amd64
+
+# Build for ARM64
+docker build . -f Dockerfile.static --platform linux/arm64 -t tartrazine-builder
+docker run -ti --rm -v "$PWD":/app --platform linux/arm64 --user="$UID" tartrazine-builder /bin/sh -c "cd /app && rm -rf lib shard.lock && make static"
+mv bin/tartrazine bin/tartrazine-static-linux-arm64

shard.yml

@@ -9,11 +9,12 @@ targets:
     main: src/main.cr
 
 dependencies:
+  baked_file_system:
+    github: schovi/baked_file_system
   base58:
     github: crystal-china/base58.cr
   sixteen:
     github: ralsina/sixteen
-    branch: main
 
 crystal: ">= 1.13.0"
 

src/actions.cr

@@ -1,4 +1,9 @@
-require "xml"
+require "./actions"
+require "./constants"
+require "./formatter"
+require "./rules"
+require "./styles"
+require "./tartrazine"
 
 # These are Lexer actions. When a rule matches, it will
 # perform a list of actions. These actions can emit tokens
@@ -26,11 +31,11 @@ module Tartrazine
     end
 
     # ameba:disable Metrics/CyclomaticComplexity
-    def emit(matches : Pointer(LibCre2::StringPiece), lexer : Lexer, match_group = 0) : Array(Token)
+    def emit(match : Regex::MatchData?, lexer : Lexer, match_group = 0) : Array(Token)
       case type
       when "token"
-        raise Exception.new "Can't have a token without a match" if matches[0].length == 0
-        [Token.new(type: xml["type"], value: String.new(Slice.new(matches[0].data, matches[0].length)))]
+        raise Exception.new "Can't have a token without a match" if match.nil?
+        [Token.new(type: xml["type"], value: match[match_group])]
       when "push"
         states_to_push = xml.attributes.select { |attrib|
           attrib.name == "state"
@@ -63,37 +68,35 @@ module Tartrazine
       when "bygroups"
         # FIXME: handle
         # ><bygroups>
-        # <token type="Punctuation"/>https://github.com/google/re2/wiki/Syntax
+        # <token type="Punctuation"/>
         # None
         # <token type="LiteralStringRegex"/>
         #
         # where that None means skipping a group
         #
-        raise Exception.new "Can't have a bygroups without a match" if matches[0].length == 0
+        raise Exception.new "Can't have a token without a match" if match.nil?
 
         # Each group matches an action. If the group match is empty,
         # the action is skipped.
         result = [] of Token
         @actions.each_with_index do |e, i|
-          next if matches[i].length == 0
-          result += e.emit(matches, lexer, i)
+          next if match[i + 1]?.nil?
+          result += e.emit(match, lexer, i + 1)
         end
         result
       when "using"
         # Shunt to another lexer entirely
-        return [] of Token if matches[0].length == 0
+        return [] of Token if match.nil?
         lexer_name = xml["lexer"].downcase
-        # Log.trace { "to tokenize: #{match[match_group]}" }
-        to_tokenize = String.new(Slice.new(matches[match_group].data, matches[match_group].length))
-        Tartrazine.lexer(lexer_name).tokenize(to_tokenize, usingself: true)
+        Log.trace { "to tokenize: #{match[match_group]}" }
+        Tartrazine.lexer(lexer_name).tokenize(match[match_group], usingself: true)
       when "usingself"
         # Shunt to another copy of this lexer
-        return [] of Token if matches[0].length == 0
+        return [] of Token if match.nil?
 
         new_lexer = Lexer.from_xml(lexer.xml)
-        # Log.trace { "to tokenize: #{match[match_group]}" }
-        to_tokenize = String.new(Slice.new(matches[match_group].data, matches[match_group].length))
-        new_lexer.tokenize(to_tokenize, usingself: true)
+        Log.trace { "to tokenize: #{match[match_group]}" }
+        new_lexer.tokenize(match[match_group], usingself: true)
       when "combined"
         # Combine two states into one anonymous state
         states = xml.attributes.select { |attrib|

src/cre2/Makefile

@@ -1,5 +0,0 @@
-all: cre2.o
-clean:
-	rm -f cre2.o
-cre2.o: cre2.cpp cre2.h
-	g++ -O3 -c -o cre2.o cre2.cpp

src/cre2/cre2.cpp

@@ -1,122 +0,0 @@
-#include <re2/re2.h>
-#include "cre2.h"
-
-#define TO_OPT(opt) (reinterpret_cast<RE2::Options *>(opt))
-
-cre2_options *cre2_opt_new(void) {
-    return reinterpret_cast<void*>(new RE2::Options());
-}
-
-void cre2_opt_delete(cre2_options *opt) {
-    delete TO_OPT(opt);
-}
-
-
-#define OPT_bool(name)  \
-void cre2_opt_##name(cre2_options *opt, int flag) {  \
-    TO_OPT(opt)->set_##name(bool(flag));             \
-}
-
-OPT_bool(posix_syntax)
-OPT_bool(longest_match)
-OPT_bool(log_errors)
-OPT_bool(literal)
-OPT_bool(never_nl)
-OPT_bool(dot_nl)
-OPT_bool(case_sensitive)
-OPT_bool(perl_classes)
-OPT_bool(word_boundary)
-OPT_bool(one_line)
-
-#undef OPT_BOOL
-
-
-void cre2_opt_encoding(cre2_options *opt, encoding_t enc) {
-    switch (enc) {
-        case CRE2_UTF8:
-            TO_OPT(opt)->set_encoding(RE2::Options::EncodingUTF8);
-            break;
-        case CRE2_Latin1:
-            TO_OPT(opt)->set_encoding(RE2::Options::EncodingLatin1);
-            break;
-    }
-}
-
-void cre2_opt_max_mem(cre2_options *opt, int m) {
-    TO_OPT(opt)->set_max_mem(m);
-}
-
-
-#define TO_RE2(re)       (reinterpret_cast<RE2 *>(re))
-#define TO_CONST_RE2(re) (reinterpret_cast<const RE2 *>(re))
-
-cre2 *cre2_new(const char *pattern, int patternlen, const cre2_options *opt) {
-    re2::StringPiece pattern_re2(pattern, patternlen);
-    return reinterpret_cast<void*>(
-        new RE2(pattern_re2, *reinterpret_cast<const RE2::Options *>(opt)));
-}
-
-void cre2_delete(cre2 *re) {
-    delete TO_RE2(re);
-}
-
-
-int cre2_error_code(const cre2 *re) {
-    return int(TO_CONST_RE2(re)->error_code());
-}
-
-const char *cre2_error_string(const cre2 *re) {
-    return TO_CONST_RE2(re)->error().c_str();
-}
-
-void cre2_error_arg(const cre2 *re, struct string_piece *arg) {
-    const std::string &argstr = TO_CONST_RE2(re)->error_arg();
-    arg->data   = argstr.data();
-    arg->length = argstr.length();
-}
-
-int cre2_num_capturing_groups(const cre2 *re) {
-    return TO_CONST_RE2(re)->NumberOfCapturingGroups();
-}
-
-int cre2_program_size(const cre2 *re) {
-    return TO_CONST_RE2(re)->ProgramSize();
-}
-
-
-int cre2_match(
-    const cre2 *re
-  , const char *text
-  , int textlen
-  , int startpos
-  , int endpos
-  , anchor_t anchor
-  , struct string_piece *match
-  , int nmatch) {
-
-    re2::StringPiece text_re2(text, textlen);
-    // FIXME: exceptions?
-    re2::StringPiece *match_re2 = new re2::StringPiece[nmatch];
-
-    RE2::Anchor anchor_re2 = RE2::UNANCHORED;
-    switch (anchor) {
-        case CRE2_ANCHOR_START:
-            anchor_re2 = RE2::ANCHOR_START; break;
-        case CRE2_ANCHOR_BOTH:
-            anchor_re2 = RE2::ANCHOR_BOTH;  break;
-    }
-
-    bool ret = TO_CONST_RE2(re)
-        ->Match(text_re2, startpos, endpos, anchor_re2, match_re2, nmatch);
-
-    if (ret) {
-        for (int i=0; i<nmatch; i++) {
-            match[i].data   = match_re2[i].data();
-            match[i].length = match_re2[i].length();
-        }
-    }
-
-    delete [] match_re2;
-
-    return int(ret);
-}

src/cre2/cre2.cr

@@ -1,70 +0,0 @@
-@[Link(ldflags: "#{__DIR__}/cre2.o -Wl,--copy-dt-needed-entries `pkg-config --libs re2`")]
-lib LibCre2
-  type Options = Void*
-
-  fun opt_new = cre2_opt_new : Options
-  fun opt_delete = cre2_opt_delete(op : Options) : Nil
-
-  fun opt_posix_syntax = cre2_opt_posix_syntax(op : Options, flag : Bool) : Nil
-  fun opt_longest_match = cre2_opt_longest_match(op : Options, flag : Bool) : Nil
-  fun opt_log_errors = cre2_opt_log_errors(op : Options, flag : Bool) : Nil
-  fun opt_literal = cre2_opt_literal(op : Options, flag : Bool) : Nil
-  fun opt_never_nl = cre2_opt_never_nl(op : Options, flag : Bool) : Nil
-  fun opt_case_sensitive = cre2_opt_case_sensitive(op : Options, flag : Bool) : Nil
-  fun opt_perl_classes = cre2_opt_perl_classes(op : Options, flag : Bool) : Nil
-  fun opt_word_boundary = cre2_opt_word_boundary(op : Options, flag : Bool) : Nil
-  fun opt_one_line = cre2_opt_one_line(op : Options, flag : Bool) : Nil
-  fun opt_dot_nl = cre2_opt_dot_nl(op : Options, flag : Bool) : Nil
-  fun opt_encoding = cre2_opt_encoding(op : Options, encoding : Int32) : Nil
-  fun opt_max_mem = cre2_opt_max_mem(op : Options, flag : Bool) : Nil
-
-  struct StringPiece
-    data : LibC::Char*
-    length : Int32
-  end
-
-  type CRe2 = Void*
-
-  fun new = cre2_new(pattern : LibC::Char*, patternlen : UInt32, opt : Options) : CRe2
-  fun del = cre2_delete(re : CRe2) : Nil
-  fun error_code = cre2_error_core(re : CRe2) : Int32
-  fun num_capturing_groups(re : CRe2) : Int32
-  fun program_size(re : CRe2) : Int32
-
-  # Invalidated by further re use
-  fun error_string = cre2_error_string(re : CRe2) : LibC::Char*
-  fun error_arg = cre2_error_arg(re : CRe2, arg : StringPiece*) : Nil
-
-  CRE2_UNANCHORED   = 1
-  CRE2_ANCHOR_START = 2
-  CRE2_ANCHOR_BOTH  = 3
-
-  fun match = cre2_match(
-    re : CRe2,
-    text : LibC::Char*,
-    textlen : UInt32,
-    startpos : UInt32,
-    endpos : UInt32,
-    anchor : Int32,
-    match : StringPiece*,
-    nmatch : Int32
-  ) : Int32
-end
-
-# match = Pointer(LibCre2::StringPiece).malloc(10)
-# opts = LibCre2.opt_new
-# LibCre2.opt_posix_syntax(opts, true)
-# LibCre2.opt_longest_match(opts, true)
-# LibCre2.opt_perl_classes(opts, true)
-# LibCre2.opt_encoding(opts, 1)
-# # LibCre2.opt_one_line(opts, false)
-# # LibCre2.opt_never_nl(opts, false)
-
-# pattern = "(\\s+)(foo)"
-# text = "  foo"
-# re = LibCre2.new(pattern, pattern.size, opts)
-# p! LibCre2.match(re, text, text.size, 0, text.size,
-#   LibCre2::CRE2_ANCHOR_START, match, 10)
-# (0...10).each do |i|
-#   p! String.new(Slice.new(match[i].data, match[i].length))
-# end

src/cre2/cre2.h

@@ -1,67 +0,0 @@
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-
-typedef void cre2_options;
-
-typedef int encoding_t;
-#define CRE2_UTF8   1
-#define CRE2_Latin1 2
-
-cre2_options *cre2_opt_new(void);
-void cre2_opt_delete(cre2_options *opt);
-
-void cre2_opt_posix_syntax(cre2_options *opt, int flag);
-void cre2_opt_longest_match(cre2_options *opt, int flag);
-void cre2_opt_log_errors(cre2_options *opt, int flag);
-void cre2_opt_literal(cre2_options *opt, int flag);
-void cre2_opt_never_nl(cre2_options *opt, int flag);
-void cre2_opt_case_sensitive(cre2_options *opt, int flag);
-void cre2_opt_perl_classes(cre2_options *opt, int flag);
-void cre2_opt_word_boundary(cre2_options *opt, int flag);
-void cre2_opt_one_line(cre2_options *opt, int flag);
-void cre2_opt_dot_nl(cre2_options *opt, int flag);
-void cre2_opt_encoding(cre2_options *opt, encoding_t enc);
-void cre2_opt_max_mem(cre2_options *opt, int m);
-
-
-struct string_piece {
-    const char *data;
-    int length;
-};
-
-
-typedef void cre2;
-
-cre2 *cre2_new(const char *pattern, int patternlen, const cre2_options *opt);
-void cre2_delete(cre2 *re);
-
-int cre2_error_code(const cre2 *re);
-int cre2_num_capturing_groups(const cre2 *re);
-int cre2_program_size(const cre2 *re);
-
-// invalidated by further re use
-const char *cre2_error_string(const cre2 *re);
-void cre2_error_arg(const cre2 *re, struct string_piece *arg);
-
-
-typedef int anchor_t;
-#define CRE2_UNANCHORED   1
-#define CRE2_ANCHOR_START 2
-#define CRE2_ANCHOR_BOTH  3
-
-int cre2_match(
-      const cre2 *re
-    , const char *text
-    , int textlen
-    , int startpos
-    , int endpos
-    , anchor_t anchor
-    , struct string_piece *match
-    , int nmatch);
-
-
-#ifdef __cplusplus
-} // extern "C"
-#endif

src/formatter.cr

@@ -1,6 +1,10 @@
-require "./constants.cr"
-require "./styles.cr"
-require "./tartrazine.cr"
+require "./actions"
+require "./constants"
+require "./formatter"
+require "./rules"
+require "./styles"
+require "./tartrazine"
+require "colorize"
 
 module Tartrazine
   # This is the base class for all formatters.
@@ -11,30 +15,42 @@ module Tartrazine
       raise Exception.new("Not implemented")
     end
 
-    # ameba:disable Metrics/CyclomaticComplexity
     def get_style_defs(theme : Theme) : String
+      raise Exception.new("Not implemented")
+    end
+  end
+
+  class Ansi < Formatter
+    def format(text : String, lexer : Lexer, theme : Theme) : String
       output = String.build do |outp|
-        theme.styles.each do |token, style|
-          outp << ".#{get_css_class(token, theme)} {"
-          # These are set or nil
-          outp << "color: #{style.color};" if style.color
-          outp << "background-color: #{style.background};" if style.background
-          outp << "border: 1px solid #{style.border};" if style.border
-
-          # These are true/false/nil
-          outp << "border: none;" if style.border == false
-          outp << "font-weight: bold;" if style.bold
-          outp << "font-weight: 400;" if style.bold == false
-          outp << "font-style: italic;" if style.italic
-          outp << "font-style: normal;" if style.italic == false
-          outp << "text-decoration: underline;" if style.underline
-          outp << "text-decoration: none;" if style.underline == false
-
-          outp << "}"
+        lexer.tokenize(text).each do |token|
+          outp << self.colorize(token[:value], token[:type], theme)
         end
       end
       output
     end
+
+    def colorize(text : String, token : String, theme : Theme) : String
+      style = theme.styles.fetch(token, nil)
+      return text if style.nil?
+      if theme.styles.has_key?(token)
+        s = theme.styles[token]
+      else
+        # Themes don't contain information for each specific
+        # token type. However, they may contain information
+        # for a parent style. Worst case, we go to the root
+        # (Background) style.
+        s = theme.styles[theme.style_parents(token).reverse.find { |parent|
+          theme.styles.has_key?(parent)
+        }]
+      end
+      colorized = text.colorize(s.color.try &.colorize)
+      # Intentionally not setting background color
+      colorized.mode(:bold) if s.bold
+      colorized.mode(:italic) if s.italic
+      colorized.mode(:underline) if s.underline
+      colorized.to_s
+    end
   end
 
   class Html < Formatter
@@ -53,6 +69,31 @@ module Tartrazine
       output
     end
 
+    # ameba:disable Metrics/CyclomaticComplexity
+    def get_style_defs(theme : Theme) : String
+      output = String.build do |outp|
+        theme.styles.each do |token, style|
+          outp << ".#{get_css_class(token, theme)} {"
+          # These are set or nil
+          outp << "color: #{style.color.try &.hex};" if style.color
+          outp << "background-color: #{style.background.try &.hex};" if style.background
+          outp << "border: 1px solid #{style.border.try &.hex};" if style.border
+
+          # These are true/false/nil
+          outp << "border: none;" if style.border == false
+          outp << "font-weight: bold;" if style.bold
+          outp << "font-weight: 400;" if style.bold == false
+          outp << "font-style: italic;" if style.italic
+          outp << "font-style: normal;" if style.italic == false
+          outp << "text-decoration: underline;" if style.underline
+          outp << "text-decoration: none;" if style.underline == false
+
+          outp << "}"
+        end
+      end
+      output
+    end
+
     # Given a token type, return the CSS class to use.
     def get_css_class(token, theme)
       return Abbreviations[token] if theme.styles.has_key?(token)

src/rules.cr

@@ -1,5 +1,9 @@
-require "./cre2/cre2"
 require "./actions"
+require "./constants"
+require "./formatter"
+require "./rules"
+require "./styles"
+require "./tartrazine"
 
 # These are lexer rules. They match with the text being parsed
 # and perform actions, either emitting tokens or changing the
@@ -8,30 +12,29 @@ module Tartrazine
   # This rule matches via a regex pattern
 
   class Rule
-    property pattern : Re3 = Re3.new ""
+    property pattern : Regex = Re2.new ""
     property actions : Array(Action) = [] of Action
     property xml : String = "foo"
 
     def match(text, pos, lexer) : Tuple(Bool, Int32, Array(Token))
-      matched, matches = pattern.match(text, pos)
+      match = pattern.match(text, pos)
       # We don't match if the match doesn't move the cursor
       # because that causes infinite loops
-
-      return false, pos, [] of Token unless matched
+      return false, pos, [] of Token if match.nil? || match.end == 0
       # Log.trace { "#{match}, #{pattern.inspect}, #{text}, #{pos}" }
       tokens = [] of Token
       # Emit the tokens
       actions.each do |action|
         # Emit the token
-        tokens += action.emit(matches, lexer)
+        tokens += action.emit(match, lexer)
       end
-      # Log.trace { "#{xml}, #{match.end}, #{tokens}" }
-      return true, matches[0].length, tokens
+      Log.trace { "#{xml}, #{match.end}, #{tokens}" }
+      return true, match.end, tokens
     end
 
     def initialize(node : XML::Node, multiline, dotall, ignorecase)
       @xml = node.to_s
-      @pattern = Re3.new(
+      @pattern = Re2.new(
         node["pattern"],
         multiline,
         dotall,
@@ -78,7 +81,7 @@ module Tartrazine
     def match(text, pos, lexer) : Tuple(Bool, Int32, Array(Token))
       tokens = [] of Token
       actions.each do |action|
-        tokens += action.emit(Pointer(LibCre2::StringPiece).malloc(1), lexer)
+        tokens += action.emit(nil, lexer)
       end
       return true, pos, tokens
     end
@@ -103,49 +106,10 @@ module Tartrazine
       flags |= LibPCRE2::DOTALL if dotall
       flags |= LibPCRE2::CASELESS if ignorecase
       flags |= LibPCRE2::ANCHORED if anchored
+      flags |= LibPCRE2::NO_UTF_CHECK
       @re = Regex::PCRE2.compile(pattern, flags) do |error_message|
         raise Exception.new(error_message)
       end
     end
   end
-
-  class Re3
-    @matches = Pointer(LibCre2::StringPiece).malloc(50)
-    @opts : LibCre2::Options
-
-    @re : LibCre2::CRe2
-
-    def group_count
-      LibCre2.num_capturing_groups(@re)
-    end
-
-    def initialize(pattern : String, multiline = false, dotall = false,
-                   ignorecase = false, anchored = false)
-      @opts = LibCre2.opt_new
-      LibCre2.opt_posix_syntax(@opts, false)
-      LibCre2.opt_longest_match(@opts, false)
-      # These 3 are ignored when posix_syntax is false
-      # LibCre2.opt_one_line(@opts, !multiline)
-      # LibCre2.opt_perl_classes(@opts, true)
-      # LibCre2.opt_word_boundary(@opts, true)
-      LibCre2.opt_encoding(@opts, 1)
-      LibCre2.opt_case_sensitive(@opts, !ignorecase)
-      LibCre2.opt_dot_nl(@opts, dotall)
-      pattern = "(?m)#{pattern}" if multiline
-      @re = LibCre2.new(pattern, pattern.size, @opts)
-    end
-
-    def match(text, pos)
-      matched = LibCre2.match(@re, text, text.size, pos, text.size,
-        LibCre2::CRE2_ANCHOR_START, @matches, 50)
-      return {matched != 0, @matches}
-    end
-  end
 end
-
-
-# re2 doesn't support this (should match "x")
-# re = Tartrazine::Re3.new("x(?!foo)", multiline: true, dotall: false)
-# m = re.match("xfoo", 0)
-# p m[0], m[1][0]
-

src/styles.cr

@@ -1,11 +1,23 @@
+require "./actions"
+require "./constants"
+require "./formatter"
+require "./rules"
+require "./styles"
+require "./tartrazine"
 require "sixteen"
 require "xml"
 
 module Tartrazine
+  alias Color = Sixteen::Color
+
   def self.theme(name : String) : Theme
     return Theme.from_base16(name[7..]) if name.starts_with? "base16_"
-    path = File.join("styles", "#{name}.xml")
-    Theme.from_xml(File.read(path))
+    Theme.from_xml(ThemeFiles.get("/#{name}.xml").gets_to_end)
+  end
+
+  class ThemeFiles
+    extend BakedFileSystem
+    bake_folder "../styles", __DIR__
   end
 
   class Style
@@ -19,9 +31,9 @@ module Tartrazine
 
     # These properties are either set or nil
     # (inherit from parent style)
-    property background : String?
-    property border : String?
-    property color : String?
+    property background : Color?
+    property border : Color?
+    property color : Color?
 
     # Styles are incomplete by default and inherit
     # from parents. If this is true, this style
@@ -91,33 +103,33 @@ module Tartrazine
       # The color assignments are adapted from
       # https://github.com/mohd-akram/base16-pygments/
 
-      theme.styles["Background"] = Style.new(color: t.palette["base05"], background: t.palette["base00"])
-      theme.styles["Text"] = Style.new(color: t.palette["base05"])
-      theme.styles["Error"] = Style.new(color: t.palette["base08"])
-      theme.styles["Comment"] = Style.new(color: t.palette["base03"])
-      theme.styles["CommentPreproc"] = Style.new(color: t.palette["base0F"])
-      theme.styles["CommentPreprocFile"] = Style.new(color: t.palette["base0B"])
-      theme.styles["Keyword"] = Style.new(color: t.palette["base0E"])
-      theme.styles["KeywordType"] = Style.new(color: t.palette["base08"])
-      theme.styles["NameAttribute"] = Style.new(color: t.palette["base0D"])
-      theme.styles["NameBuiltin"] = Style.new(color: t.palette["base08"])
-      theme.styles["NameBuiltinPseudo"] = Style.new(color: t.palette["base08"])
-      theme.styles["NameClass"] = Style.new(color: t.palette["base0D"])
-      theme.styles["NameConstant"] = Style.new(color: t.palette["base09"])
-      theme.styles["NameDecorator"] = Style.new(color: t.palette["base09"])
-      theme.styles["NameFunction"] = Style.new(color: t.palette["base0D"])
-      theme.styles["NameNamespace"] = Style.new(color: t.palette["base0D"])
-      theme.styles["NameTag"] = Style.new(color: t.palette["base0E"])
-      theme.styles["NameVariable"] = Style.new(color: t.palette["base0D"])
-      theme.styles["NameVariableInstance"] = Style.new(color: t.palette["base08"])
-      theme.styles["LiteralNumber"] = Style.new(color: t.palette["base09"])
-      theme.styles["Operator"] = Style.new(color: t.palette["base0C"])
-      theme.styles["OperatorWord"] = Style.new(color: t.palette["base0E"])
-      theme.styles["Literal"] = Style.new(color: t.palette["base0B"])
-      theme.styles["LiteralString"] = Style.new(color: t.palette["base0B"])
-      theme.styles["LiteralStringInterpol"] = Style.new(color: t.palette["base0F"])
-      theme.styles["LiteralStringRegex"] = Style.new(color: t.palette["base0C"])
-      theme.styles["LiteralStringSymbol"] = Style.new(color: t.palette["base09"])
+      theme.styles["Background"] = Style.new(color: t["base05"], background: t["base00"])
+      theme.styles["Text"] = Style.new(color: t["base05"])
+      theme.styles["Error"] = Style.new(color: t["base08"])
+      theme.styles["Comment"] = Style.new(color: t["base03"])
+      theme.styles["CommentPreproc"] = Style.new(color: t["base0F"])
+      theme.styles["CommentPreprocFile"] = Style.new(color: t["base0B"])
+      theme.styles["Keyword"] = Style.new(color: t["base0E"])
+      theme.styles["KeywordType"] = Style.new(color: t["base08"])
+      theme.styles["NameAttribute"] = Style.new(color: t["base0D"])
+      theme.styles["NameBuiltin"] = Style.new(color: t["base08"])
+      theme.styles["NameBuiltinPseudo"] = Style.new(color: t["base08"])
+      theme.styles["NameClass"] = Style.new(color: t["base0D"])
+      theme.styles["NameConstant"] = Style.new(color: t["base09"])
+      theme.styles["NameDecorator"] = Style.new(color: t["base09"])
+      theme.styles["NameFunction"] = Style.new(color: t["base0D"])
+      theme.styles["NameNamespace"] = Style.new(color: t["base0D"])
+      theme.styles["NameTag"] = Style.new(color: t["base0E"])
+      theme.styles["NameVariable"] = Style.new(color: t["base0D"])
+      theme.styles["NameVariableInstance"] = Style.new(color: t["base08"])
+      theme.styles["LiteralNumber"] = Style.new(color: t["base09"])
+      theme.styles["Operator"] = Style.new(color: t["base0C"])
+      theme.styles["OperatorWord"] = Style.new(color: t["base0E"])
+      theme.styles["Literal"] = Style.new(color: t["base0B"])
+      theme.styles["LiteralString"] = Style.new(color: t["base0B"])
+      theme.styles["LiteralStringInterpol"] = Style.new(color: t["base0F"])
+      theme.styles["LiteralStringRegex"] = Style.new(color: t["base0C"])
+      theme.styles["LiteralStringSymbol"] = Style.new(color: t["base09"])
       theme
     end
 
@@ -144,9 +156,9 @@ module Tartrazine
         s.underline = true if style.includes?("underline")
         s.underline = false if style.includes?("nounderline")
 
-        s.color = style.find(&.starts_with?("#")).try &.split("#").last
-        s.background = style.find(&.starts_with?("bg:#")).try &.split("#").last
-        s.border = style.find(&.starts_with?("border:#")).try &.split("#").last
+        s.color = style.find(&.starts_with?("#")).try { |v| Color.new v.split("#").last }
+        s.background = style.find(&.starts_with?("bg:#")).try { |v| Color.new v.split("#").last }
+        s.border = style.find(&.starts_with?("border:#")).try { |v| Color.new v.split("#").last }
 
         theme.styles[node["type"]] = s
       end

src/tartrazine.cr

@@ -1,5 +1,10 @@
 require "./actions"
+require "./constants"
+require "./formatter"
 require "./rules"
+require "./styles"
+require "./tartrazine"
+require "baked_file_system"
 require "base58"
 require "json"
 require "log"
@@ -33,6 +38,12 @@ module Tartrazine
     end
   end
 
+  class LexerFiles
+    extend BakedFileSystem
+
+    bake_folder "../lexers", __DIR__
+  end
+
   # A token, the output of the tokenizer
   alias Token = NamedTuple(type: String, value: String)
 
@@ -72,22 +83,22 @@ module Tartrazine
       # Loop through the text, applying rules
       while pos < text.size
         state = states[@state_stack.last]
-        Log.trace { "Stack is #{@state_stack} State is #{state.name}, pos is #{pos}, text is #{text[pos..pos + 10]}" }
+        # Log.trace { "Stack is #{@state_stack} State is #{state.name}, pos is #{pos}, text is #{text[pos..pos + 10]}" }
         state.rules.each do |rule|
           matched, new_pos, new_tokens = rule.match(text, pos, self)
           if matched
             # Move position forward, save the tokens,
             # tokenize from the new position
-            Log.trace { "MATCHED: #{rule.xml}" }
+            # Log.trace { "MATCHED: #{rule.xml}" }
             pos = new_pos
             tokens += new_tokens
             break
           end
-          Log.trace { "NOT MATCHED: #{rule.xml}" }
+          # Log.trace { "NOT MATCHED: #{rule.xml}" }
         end
         # If no rule matches, emit an error token
         unless matched
-          Log.trace { "Error at #{pos}" }
+          # Log.trace { "Error at #{pos}" }
           tokens << {type: "Error", value: "#{text[pos]}"}
           pos += 1
         end
@@ -182,7 +193,7 @@ module Tartrazine
   end
 
   def self.lexer(name : String) : Lexer
-    Lexer.from_xml(File.read("lexers/#{name}.xml"))
+    Lexer.from_xml(LexerFiles.get("/#{name}.xml").gets_to_end)
   end
 end