0.1.1

.gitignore

@@ -6,3 +6,4 @@
 chroma/
 pygments/
 shard.lock
+.vscode/

Dockerfile.static

@@ -0,0 +1,15 @@
+FROM --platform=${TARGETPLATFORM:-linux/amd64} alpine:3.20 AS build
+RUN apk add --no-cache \
+    crystal \
+    shards \
+    yaml-dev \
+    yaml-static \
+    openssl-dev \
+    openssl-libs-static \
+    libxml2-dev \
+    libxml2-static \
+    zlib-dev \
+    zlib-static \
+    xz-dev \
+    xz-static \
+    make

Makefile

@@ -0,0 +1,7 @@
+build: $(wildcard src/**/*.cr) $(wildcard lexers/*xml) $(wildcard styles/*xml) shard.yml
+	shards build -Dstrict_multi_assign -Dno_number_autocast
+release: $(wildcard src/**/*.cr) $(wildcard lexers/*xml) $(wildcard styles/*xml) shard.yml
+	shards build --release
+static: $(wildcard src/**/*.cr) $(wildcard lexers/*xml) $(wildcard styles/*xml) shard.yml
+	shards build --release --static
+	strip bin/tartrazine

build_static.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -e
+
+docker run --rm --privileged \
+  multiarch/qemu-user-static \
+  --reset -p yes
+
+# Build for AMD64
+docker build . -f Dockerfile.static -t tartrazine-builder
+docker run -ti --rm -v "$PWD":/app --user="$UID" tartrazine-builder /bin/sh -c "cd /app && rm -rf lib shard.lock && make static"
+mv bin/tartrazine bin/tartrazine-static-linux-amd64
+
+# Build for ARM64
+docker build . -f Dockerfile.static --platform linux/arm64 -t tartrazine-builder
+docker run -ti --rm -v "$PWD":/app --platform linux/arm64 --user="$UID" tartrazine-builder /bin/sh -c "cd /app && rm -rf lib shard.lock && make static"
+mv bin/tartrazine bin/tartrazine-static-linux-arm64

shard.yml

@@ -1,5 +1,5 @@
 name: tartrazine
-version: 0.1.0
+version: 0.1.1
 
 authors:
   - Roberto Alsina <roberto.alsina@gmail.com>
@@ -9,11 +9,12 @@ targets:
     main: src/main.cr
 
 dependencies:
+  baked_file_system:
+    github: schovi/baked_file_system
   base58:
     github: crystal-china/base58.cr
   sixteen:
     github: ralsina/sixteen
-    branch: main
 
 crystal: ">= 1.13.0"
 

src/actions.cr

@@ -1,3 +1,10 @@
+require "./actions"
+require "./constants"
+require "./formatter"
+require "./rules"
+require "./styles"
+require "./tartrazine"
+
 # These are Lexer actions. When a rule matches, it will
 # perform a list of actions. These actions can emit tokens
 # or change the state machine.

src/formatter.cr

@@ -1,6 +1,10 @@
-require "./constants.cr"
+require "./actions"
-require "./styles.cr"
+require "./constants"
-require "./tartrazine.cr"
+require "./formatter"
+require "./rules"
+require "./styles"
+require "./tartrazine"
+require "colorize"
 
 module Tartrazine
   # This is the base class for all formatters.
@@ -11,30 +15,43 @@ module Tartrazine
       raise Exception.new("Not implemented")
     end
 
-    # ameba:disable Metrics/CyclomaticComplexity
     def get_style_defs(theme : Theme) : String
+      raise Exception.new("Not implemented")
+    end
+  end
+
+  class Ansi < Formatter
+    def format(text : String, lexer : Lexer, theme : Theme) : String
       output = String.build do |outp|
-        theme.styles.each do |token, style|
+        lexer.tokenize(text).each do |token|
-          outp << ".#{get_css_class(token, theme)} {"
+          outp << self.colorize(token[:value], token[:type], theme)
-          # These are set or nil
-          outp << "color: #{style.color};" if style.color
-          outp << "background-color: #{style.background};" if style.background
-          outp << "border: 1px solid #{style.border};" if style.border
-
-          # These are true/false/nil
-          outp << "border: none;" if style.border == false
-          outp << "font-weight: bold;" if style.bold
-          outp << "font-weight: 400;" if style.bold == false
-          outp << "font-style: italic;" if style.italic
-          outp << "font-style: normal;" if style.italic == false
-          outp << "text-decoration: underline;" if style.underline
-          outp << "text-decoration: none;" if style.underline == false
-
-          outp << "}"
         end
       end
       output
     end
+
+    def colorize(text : String, token : String, theme : Theme) : String
+      style = theme.styles.fetch(token, nil)
+      return text if style.nil?
+      if theme.styles.has_key?(token)
+        s = theme.styles[token]
+      else
+        # Themes don't contain information for each specific
+        # token type. However, they may contain information
+        # for a parent style. Worst case, we go to the root
+        # (Background) style.
+        s = theme.styles[theme.style_parents(token).reverse.find { |parent|
+          theme.styles.has_key?(parent)
+        }]
+      end
+      colorized = text.colorize
+      s.color.try { |c| colorized = colorized.fore(c.colorize) }
+      # Intentionally not setting background color
+      colorized.mode(:bold) if s.bold
+      colorized.mode(:italic) if s.italic
+      colorized.mode(:underline) if s.underline
+      colorized.to_s
+    end
   end
 
   class Html < Formatter
@@ -53,6 +70,31 @@ module Tartrazine
       output
     end
 
+    # ameba:disable Metrics/CyclomaticComplexity
+    def get_style_defs(theme : Theme) : String
+      output = String.build do |outp|
+        theme.styles.each do |token, style|
+          outp << ".#{get_css_class(token, theme)} {"
+          # These are set or nil
+          outp << "color: #{style.color.try &.hex};" if style.color
+          outp << "background-color: #{style.background.try &.hex};" if style.background
+          outp << "border: 1px solid #{style.border.try &.hex};" if style.border
+
+          # These are true/false/nil
+          outp << "border: none;" if style.border == false
+          outp << "font-weight: bold;" if style.bold
+          outp << "font-weight: 400;" if style.bold == false
+          outp << "font-style: italic;" if style.italic
+          outp << "font-style: normal;" if style.italic == false
+          outp << "text-decoration: underline;" if style.underline
+          outp << "text-decoration: none;" if style.underline == false
+
+          outp << "}"
+        end
+      end
+      output
+    end
+
     # Given a token type, return the CSS class to use.
     def get_css_class(token, theme)
       return Abbreviations[token] if theme.styles.has_key?(token)

src/rules.cr

@@ -1,4 +1,9 @@
 require "./actions"
+require "./constants"
+require "./formatter"
+require "./rules"
+require "./styles"
+require "./tartrazine"
 
 # These are lexer rules. They match with the text being parsed
 # and perform actions, either emitting tokens or changing the
@@ -101,6 +106,7 @@ module Tartrazine
       flags |= LibPCRE2::DOTALL if dotall
       flags |= LibPCRE2::CASELESS if ignorecase
       flags |= LibPCRE2::ANCHORED if anchored
+      flags |= LibPCRE2::NO_UTF_CHECK
       @re = Regex::PCRE2.compile(pattern, flags) do |error_message|
         raise Exception.new(error_message)
       end

src/styles.cr

@@ -1,11 +1,23 @@
+require "./actions"
+require "./constants"
+require "./formatter"
+require "./rules"
+require "./styles"
+require "./tartrazine"
 require "sixteen"
 require "xml"
 
 module Tartrazine
+  alias Color = Sixteen::Color
+
   def self.theme(name : String) : Theme
     return Theme.from_base16(name[7..]) if name.starts_with? "base16_"
-    path = File.join("styles", "#{name}.xml")
+    Theme.from_xml(ThemeFiles.get("/#{name}.xml").gets_to_end)
-    Theme.from_xml(File.read(path))
+  end
+
+  class ThemeFiles
+    extend BakedFileSystem
+    bake_folder "../styles", __DIR__
   end
 
   class Style
@@ -19,9 +31,9 @@ module Tartrazine
 
     # These properties are either set or nil
     # (inherit from parent style)
-    property background : String?
+    property background : Color?
-    property border : String?
+    property border : Color?
-    property color : String?
+    property color : Color?
 
     # Styles are incomplete by default and inherit
     # from parents. If this is true, this style
@@ -91,33 +103,33 @@ module Tartrazine
       # The color assignments are adapted from
       # https://github.com/mohd-akram/base16-pygments/
 
-      theme.styles["Background"] = Style.new(color: t.palette["base05"], background: t.palette["base00"])
+      theme.styles["Background"] = Style.new(color: t["base05"], background: t["base00"])
-      theme.styles["Text"] = Style.new(color: t.palette["base05"])
+      theme.styles["Text"] = Style.new(color: t["base05"])
-      theme.styles["Error"] = Style.new(color: t.palette["base08"])
+      theme.styles["Error"] = Style.new(color: t["base08"])
-      theme.styles["Comment"] = Style.new(color: t.palette["base03"])
+      theme.styles["Comment"] = Style.new(color: t["base03"])
-      theme.styles["CommentPreproc"] = Style.new(color: t.palette["base0F"])
+      theme.styles["CommentPreproc"] = Style.new(color: t["base0F"])
-      theme.styles["CommentPreprocFile"] = Style.new(color: t.palette["base0B"])
+      theme.styles["CommentPreprocFile"] = Style.new(color: t["base0B"])
-      theme.styles["Keyword"] = Style.new(color: t.palette["base0E"])
+      theme.styles["Keyword"] = Style.new(color: t["base0E"])
-      theme.styles["KeywordType"] = Style.new(color: t.palette["base08"])
+      theme.styles["KeywordType"] = Style.new(color: t["base08"])
-      theme.styles["NameAttribute"] = Style.new(color: t.palette["base0D"])
+      theme.styles["NameAttribute"] = Style.new(color: t["base0D"])
-      theme.styles["NameBuiltin"] = Style.new(color: t.palette["base08"])
+      theme.styles["NameBuiltin"] = Style.new(color: t["base08"])
-      theme.styles["NameBuiltinPseudo"] = Style.new(color: t.palette["base08"])
+      theme.styles["NameBuiltinPseudo"] = Style.new(color: t["base08"])
-      theme.styles["NameClass"] = Style.new(color: t.palette["base0D"])
+      theme.styles["NameClass"] = Style.new(color: t["base0D"])
-      theme.styles["NameConstant"] = Style.new(color: t.palette["base09"])
+      theme.styles["NameConstant"] = Style.new(color: t["base09"])
-      theme.styles["NameDecorator"] = Style.new(color: t.palette["base09"])
+      theme.styles["NameDecorator"] = Style.new(color: t["base09"])
-      theme.styles["NameFunction"] = Style.new(color: t.palette["base0D"])
+      theme.styles["NameFunction"] = Style.new(color: t["base0D"])
-      theme.styles["NameNamespace"] = Style.new(color: t.palette["base0D"])
+      theme.styles["NameNamespace"] = Style.new(color: t["base0D"])
-      theme.styles["NameTag"] = Style.new(color: t.palette["base0E"])
+      theme.styles["NameTag"] = Style.new(color: t["base0E"])
-      theme.styles["NameVariable"] = Style.new(color: t.palette["base0D"])
+      theme.styles["NameVariable"] = Style.new(color: t["base0D"])
-      theme.styles["NameVariableInstance"] = Style.new(color: t.palette["base08"])
+      theme.styles["NameVariableInstance"] = Style.new(color: t["base08"])
-      theme.styles["LiteralNumber"] = Style.new(color: t.palette["base09"])
+      theme.styles["LiteralNumber"] = Style.new(color: t["base09"])
-      theme.styles["Operator"] = Style.new(color: t.palette["base0C"])
+      theme.styles["Operator"] = Style.new(color: t["base0C"])
-      theme.styles["OperatorWord"] = Style.new(color: t.palette["base0E"])
+      theme.styles["OperatorWord"] = Style.new(color: t["base0E"])
-      theme.styles["Literal"] = Style.new(color: t.palette["base0B"])
+      theme.styles["Literal"] = Style.new(color: t["base0B"])
-      theme.styles["LiteralString"] = Style.new(color: t.palette["base0B"])
+      theme.styles["LiteralString"] = Style.new(color: t["base0B"])
-      theme.styles["LiteralStringInterpol"] = Style.new(color: t.palette["base0F"])
+      theme.styles["LiteralStringInterpol"] = Style.new(color: t["base0F"])
-      theme.styles["LiteralStringRegex"] = Style.new(color: t.palette["base0C"])
+      theme.styles["LiteralStringRegex"] = Style.new(color: t["base0C"])
-      theme.styles["LiteralStringSymbol"] = Style.new(color: t.palette["base09"])
+      theme.styles["LiteralStringSymbol"] = Style.new(color: t["base09"])
       theme
     end
 
@@ -144,9 +156,9 @@ module Tartrazine
         s.underline = true if style.includes?("underline")
         s.underline = false if style.includes?("nounderline")
 
-        s.color = style.find(&.starts_with?("#")).try &.split("#").last
+        s.color = style.find(&.starts_with?("#")).try { |v| Color.new v.split("#").last }
-        s.background = style.find(&.starts_with?("bg:#")).try &.split("#").last
+        s.background = style.find(&.starts_with?("bg:#")).try { |v| Color.new v.split("#").last }
-        s.border = style.find(&.starts_with?("border:#")).try &.split("#").last
+        s.border = style.find(&.starts_with?("border:#")).try { |v| Color.new v.split("#").last }
 
         theme.styles[node["type"]] = s
       end

src/tartrazine.cr

@@ -1,5 +1,10 @@
 require "./actions"
+require "./constants"
+require "./formatter"
 require "./rules"
+require "./styles"
+require "./tartrazine"
+require "baked_file_system"
 require "base58"
 require "json"
 require "log"
@@ -7,7 +12,7 @@ require "xml"
 
 module Tartrazine
   extend self
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 
   Log = ::Log.for("tartrazine")
 
@@ -33,6 +38,12 @@ module Tartrazine
     end
   end
 
+  class LexerFiles
+    extend BakedFileSystem
+
+    bake_folder "../lexers", __DIR__
+  end
+
   # A token, the output of the tokenizer
   alias Token = NamedTuple(type: String, value: String)
 
@@ -63,8 +74,6 @@ module Tartrazine
       tokens = [] of Token
       pos = 0
       matched = false
-      time = 0
-      count = 0
 
       # Respect the `ensure_nl` config option
       if text.size > 0 && text[-1] != '\n' && config[:ensure_nl] && !usingself
@@ -74,22 +83,22 @@ module Tartrazine
       # Loop through the text, applying rules
       while pos < text.size
         state = states[@state_stack.last]
-        Log.trace { "Stack is #{@state_stack} State is #{state.name}, pos is #{pos}, text is #{text[pos..pos + 10]}" }
+        # Log.trace { "Stack is #{@state_stack} State is #{state.name}, pos is #{pos}, text is #{text[pos..pos + 10]}" }
         state.rules.each do |rule|
           matched, new_pos, new_tokens = rule.match(text, pos, self)
           if matched
             # Move position forward, save the tokens,
             # tokenize from the new position
-            Log.trace { "MATCHED: #{rule.xml}" }
+            # Log.trace { "MATCHED: #{rule.xml}" }
             pos = new_pos
             tokens += new_tokens
             break
           end
-          Log.trace { "NOT MATCHED: #{rule.xml}" }
+          # Log.trace { "NOT MATCHED: #{rule.xml}" }
         end
         # If no rule matches, emit an error token
         unless matched
-          Log.trace { "Error at #{pos}" }
+          # Log.trace { "Error at #{pos}" }
           tokens << {type: "Error", value: "#{text[pos]}"}
           pos += 1
         end
@@ -184,7 +193,7 @@ module Tartrazine
   end
 
   def self.lexer(name : String) : Lexer
-    Lexer.from_xml(File.read("lexers/#{name}.xml"))
+    Lexer.from_xml(LexerFiles.get("/#{name}.xml").gets_to_end)
   end
 end