bump: Release v0.11.1

Conditional lexers and themes

CHANGELOG.md

@@ -2,6 +2,52 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.11.1] - 2024-10-14
+
+### 🐛 Bug Fixes
+
+- Support choosing lexers when used as a library
+
+## [0.11.0] - 2024-10-14
+
+### 🚀 Features
+
+- Support selecting only some themes
+
+## [0.10.0] - 2024-09-26
+
+### 🚀 Features
+
+- Optional conditional baking of lexers
+
+### 🐛 Bug Fixes
+
+- Strip binaries for release artifacts
+- Fix metadata to show crystal
+
+## [0.9.1] - 2024-09-22
+
+### 🐛 Bug Fixes
+
+- Terminal formatter was skipping things that it could highlight
+- Bug in high-level API for png formatter
+
+### 🧪 Testing
+
+- Added minimal tests for svg and png formatters
+
+## [0.9.0] - 2024-09-21
+
+### 🚀 Features
+
+- PNG writer based on Stumpy libs
+
+### ⚙️ Miscellaneous Tasks
+
+- Clean
+- Detect version bump in release script
+- Improve changelog handling
+
 ## [0.8.0] - 2024-09-21
 
 ### 🚀 Features

Hacefile.yml

@@ -113,3 +113,11 @@ tasks:
       kcov --clean --include-path=./src ${PWD}/coverage ./bin/run_tests
     outputs:
       - coverage/index.html
+
+  loc:
+    phony: true
+    always_run: true
+    dependencies:
+      - src
+    commands: |
+      tokei src -e src/constants/

README.md

@@ -82,6 +82,51 @@ puts formatter.format("puts \"Hello, world!\"", lexer)
 The reason you may want to use the manual version is to reuse
 the lexer and formatter objects for performance reasons.
 
+## Choosing what Lexers you want
+
+By default Tartrazine will support all its lexers by embedding
+them in the binary. This makes the binary large. If you are
+using it as a library, you may want to just include a selection of lexers. To do that:
+
+* Pass the `-Dnolexers` flag to the compiler
+* Set the `TT_LEXERS` environment variable to a
+  comma-separated list of lexers you want to include.
+
+
+This builds a binary with only the python, markdown, bash and yaml lexers (enough to highlight this `README.md`):
+
+```bash
+> TT_LEXERS=python,markdown,bash,yaml shards build -Dnolexers -d --error-trace
+Dependencies are satisfied
+Building: tartrazine
+```
+
+## Choosing what themes you want
+
+Themes come from two places, tartrazine itself and [Sixteen](https://github.com/ralsina/sixteen).
+
+To only embed selected themes, build your project with the `-Dnothemes` option, and
+you can set two environment variables to control which themes are included:
+
+* `TT_THEMES` is a comma-separated list of themes to include from tartrazine (see the styles directory in the source)
+* `SIXTEEN_THEMES` is a comma-separated list of themes to include from Sixteen (see the base16 directory in the sixteen source)
+
+For example (using the tartrazine CLI as the project):
+
+```bash
+$ TT_THEMES=colorful,autumn SIXTEEN_THEMES=pasque,pico shards build -Dnothemes
+Dependencies are satisfied
+Building: tartrazine
+
+$ ./bin/tartrazine  --list-themes
+autumn
+colorful
+pasque
+pico
+```
+
+Be careful not to build without any themes at all, nothing will work.
+
 ## Contributing
 
 1. Fork it (<https://github.com/ralsina/tartrazine/fork>)

build_static.sh

@@ -7,10 +7,10 @@ docker run --rm --privileged \
 
 # Build for AMD64
 docker build . -f Dockerfile.static -t tartrazine-builder
-docker run -ti --rm -v "$PWD":/app --user="$UID" tartrazine-builder /bin/sh -c "cd /app && rm -rf lib shard.lock && shards build --static --release"
+docker run -ti --rm -v "$PWD":/app --user="$UID" tartrazine-builder /bin/sh -c "cd /app && rm -rf lib shard.lock && shards build --static --release && strip bin/tartrazine"
 mv bin/tartrazine bin/tartrazine-static-linux-amd64
 
 # Build for ARM64
 docker build . -f Dockerfile.static --platform linux/arm64 -t tartrazine-builder
-docker run -ti --rm -v "$PWD":/app --platform linux/arm64 --user="$UID" tartrazine-builder /bin/sh -c "cd /app && rm -rf lib shard.lock && shards build --static --release"
+docker run -ti --rm -v "$PWD":/app --platform linux/arm64 --user="$UID" tartrazine-builder /bin/sh -c "cd /app && rm -rf lib shard.lock && shards build --static --release && strip bin/tartrazine"
 mv bin/tartrazine bin/tartrazine-static-linux-arm64

cliff.toml

@@ -67,7 +67,6 @@ commit_parsers = [
   { message = "^chore\\(deps.*\\)", skip = true },
   { message = "^chore\\(pr\\)", skip = true },
   { message = "^chore\\(pull\\)", skip = true },
-  { message = "^chore\\(ignore\\)", skip = true },
   { message = "^chore|^ci", group = "<!-- 7 -->⚙️ Miscellaneous Tasks" },
   { body = ".*security", group = "<!-- 8 -->🛡️ Security" },
   { message = "^revert", group = "<!-- 9 -->◀️ Revert" },

do_release.sh

@@ -2,14 +2,14 @@
 set e
 
 PKGNAME=$(basename "$PWD")
-VERSION=$(git cliff --bumped-version |cut -dv -f2)
+VERSION=$(git cliff --bumped-version --unreleased |cut -dv -f2)
 
 sed "s/^version:.*$/version: $VERSION/g" -i shard.yml
 git add shard.yml
 hace lint test
-git cliff --bump -o
+git cliff --bump -u -p CHANGELOG.md
 git commit -a -m "bump: Release v$VERSION"
+hace static
 git tag "v$VERSION"
 git push --tags
-hace static
 gh release create "v$VERSION" "bin/$PKGNAME-static-linux-amd64" "bin/$PKGNAME-static-linux-arm64" --title "Release v$VERSION" --notes "$(git cliff -l -s all)"

scripts/lexer_metadata.py

@@ -38,6 +38,12 @@ for fname in glob.glob("lexers/*.xml"):
         lexer_by_filename[filename].add(lexer_name)
 
 with open("src/constants/lexers.cr", "w") as f:
+    # Crystal doesn't come from a xml file
+    lexer_by_name["crystal"] = "crystal"
+    lexer_by_name["cr"] = "crystal"
+    lexer_by_filename["*.cr"] = ["crystal"]
+    lexer_by_mimetype["text/x-crystal"] = "crystal"
+
     f.write("module Tartrazine\n")
     f.write("  LEXERS_BY_NAME = {\n")
     for k in sorted(lexer_by_name.keys()):

shard.yml

@@ -1,5 +1,5 @@
 name: tartrazine
-version: 0.8.0
+version: 0.11.1
 
 authors:
   - Roberto Alsina <roberto.alsina@gmail.com>
@@ -18,6 +18,10 @@ dependencies:
     github: ralsina/sixteen
   docopt:
     github: chenkovsky/docopt.cr
+  stumpy_utils:
+    github: stumpycr/stumpy_utils
+  stumpy_png:
+    github: stumpycr/stumpy_png
 
 crystal: ">= 1.13.0"
 

spec/tartrazine_spec.cr

@@ -1,4 +1,5 @@
 require "./spec_helper"
+require "digest/sha1"
 
 # These are the testcases from Pygments
 testcases = Dir.glob("#{__DIR__}/tests/**/*txt").sort
@@ -103,6 +104,7 @@ describe Tartrazine do
       )
     end
   end
+
   describe "to_ansi" do
     it "should do basic highlighting" do
       ansi = Tartrazine.to_ansi("puts 'Hello, World!'", "ruby")
@@ -114,11 +116,29 @@ describe Tartrazine do
         )
       else
         ansi.should eq(
-          "\e[38;2;171;70;66mputs\e[0m\e[38;2;216;216;216m \e[0m'Hello, World!'"
+          "\e[38;2;171;70;66mputs\e[0m\e[38;2;216;216;216m \e[0m\e[38;2;161;181;108m'Hello, World!'\e[0m"
         )
       end
     end
   end
+
+  describe "to_svg" do
+    it "should do basic highlighting" do
+      svg = Tartrazine.to_svg("puts 'Hello, World!'", "ruby", standalone: false)
+      svg.should eq(
+        "<text x=\"0\" y=\"19\" xml:space=\"preserve\"><tspan fill=\"#ab4642\">puts</tspan><tspan fill=\"#d8d8d8\"> </tspan><tspan fill=\"#a1b56c\">&#39;Hello, World!&#39;</tspan></text>"
+      )
+    end
+  end
+
+  describe "to_png" do
+    it "should do basic highlighting" do
+      png = Digest::SHA1.hexdigest(Tartrazine.to_png("puts 'Hello, World!'", "ruby"))
+      png.should eq(
+        "62d419dcd263fffffc265a0f04c156dc2530c362"
+      )
+    end
+  end
 end
 
 # Helper that creates lexer and tokenizes

src/constants/lexers.cr

@@ -471,7 +471,7 @@ module Tartrazine
     "application/x-fennel"                        => "fennel",
     "application/x-fish"                          => "fish",
     "application/x-forth"                         => "forth",
-    "application/x-gdscript"                      => "gdscript",
+    "application/x-gdscript"                      => "gdscript3",
     "application/x-hcl"                           => "hcl",
     "application/x-hy"                            => "hy",
     "application/x-javascript"                    => "javascript",
@@ -594,7 +594,7 @@ module Tartrazine
     "text/x-fortran"                              => "fortran",
     "text/x-fsharp"                               => "fsharp",
     "text/x-gas"                                  => "gas",
-    "text/x-gdscript"                             => "gdscript",
+    "text/x-gdscript"                             => "gdscript3",
     "text/x-gherkin"                              => "gherkin",
     "text/x-gleam"                                => "gleam",
     "text/x-glslsrc"                              => "glsl",

src/formatters/ansi.cr

@@ -34,8 +34,6 @@ module Tartrazine
     end
 
     def colorize(text : String, token : String) : String
-      style = theme.styles.fetch(token, nil)
-      return text if style.nil?
       if theme.styles.has_key?(token)
         s = theme.styles[token]
       else

src/formatters/png.cr

@@ -0,0 +1,117 @@
+require "../formatter"
+require "compress/gzip"
+require "digest/sha1"
+require "stumpy_png"
+require "stumpy_utils"
+
+module Tartrazine
+  def self.to_png(text : String, language : String,
+                  theme : String = "default-dark",
+                  line_numbers : Bool = false) : String
+    buf = IO::Memory.new
+
+    Tartrazine::Png.new(
+      theme: Tartrazine.theme(theme),
+      line_numbers: line_numbers
+    ).format(text, Tartrazine.lexer(name: language), buf)
+    buf.to_s
+  end
+
+  class FontFiles
+    extend BakedFileSystem
+    bake_folder "../../fonts", __DIR__
+  end
+
+  class Png < Formatter
+    include StumpyPNG
+    property? line_numbers : Bool = false
+    @font_regular : PCFParser::Font
+    @font_bold : PCFParser::Font
+    @font_oblique : PCFParser::Font
+    @font_bold_oblique : PCFParser::Font
+    @font_width = 15
+    @font_height = 24
+
+    def initialize(@theme : Theme = Tartrazine.theme("default-dark"), @line_numbers : Bool = false)
+      @font_regular = load_font("/courier-regular.pcf.gz")
+      @font_bold = load_font("/courier-bold.pcf.gz")
+      @font_oblique = load_font("/courier-oblique.pcf.gz")
+      @font_bold_oblique = load_font("/courier-bold-oblique.pcf.gz")
+    end
+
+    private def load_font(name : String) : PCFParser::Font
+      compressed = FontFiles.get(name)
+      uncompressed = Compress::Gzip::Reader.open(compressed) do |gzip|
+        gzip.gets_to_end
+      end
+      PCFParser::Font.new(IO::Memory.new uncompressed)
+    end
+
+    private def line_label(i : Int32) : String
+      "#{i + 1}".rjust(4).ljust(5)
+    end
+
+    def format(text : String, lexer : BaseLexer, outp : IO) : Nil
+      # Create canvas of correct size
+      lines = text.split("\n")
+      canvas_height = lines.size * @font_height
+      canvas_width = lines.max_of(&.size)
+      canvas_width += 5 if line_numbers?
+      canvas_width *= @font_width
+
+      bg_color = RGBA.from_hex("##{theme.styles["Background"].background.try &.hex}")
+      canvas = Canvas.new(canvas_width, canvas_height, bg_color)
+
+      tokenizer = lexer.tokenizer(text)
+      x = 0
+      y = @font_height
+      i = 0
+      if line_numbers?
+        canvas.text(x, y, line_label(i), @font_regular, RGBA.from_hex("##{theme.styles["Background"].color.try &.hex}"))
+        x += 5 * @font_width
+      end
+
+      tokenizer.each do |token|
+        font, color = token_style(token[:type])
+        # These fonts are very limited
+        t = token[:value].gsub(/[^[:ascii:]]/, "?")
+        canvas.text(x, y, t.rstrip("\n"), font, color)
+        if token[:value].includes?("\n")
+          x = 0
+          y += @font_height
+          i += 1
+          if line_numbers?
+            canvas.text(x, y, line_label(i), @font_regular, RGBA.from_hex("##{theme.styles["Background"].color.try &.hex}"))
+            x += 4 * @font_width
+          end
+        end
+
+        x += token[:value].size * @font_width
+      end
+
+      StumpyPNG.write(canvas, outp)
+    end
+
+    def token_style(token : String) : {PCFParser::Font, RGBA}
+      if theme.styles.has_key?(token)
+        s = theme.styles[token]
+      else
+        # Themes don't contain information for each specific
+        # token type. However, they may contain information
+        # for a parent style. Worst case, we go to the root
+        # (Background) style.
+        s = theme.styles[theme.style_parents(token).reverse.find { |parent|
+          theme.styles.has_key?(parent)
+        }]
+      end
+
+      color = RGBA.from_hex("##{theme.styles["Background"].color.try &.hex}")
+      color = RGBA.from_hex("##{s.color.try &.hex}") if s.color
+
+      return {@font_bold_oblique, color} if s.bold && s.italic
+      return {@font_bold, color} if s.bold
+      return {@font_oblique, color} if s.italic
+      return {@font_regular, color}
+    end
+  end
+end

src/lexer.cr

@@ -6,11 +6,21 @@ require "crystal/syntax_highlighter"
 module Tartrazine
   class LexerFiles
     extend BakedFileSystem
-    bake_folder "../lexers", __DIR__
+
+    macro bake_selected_lexers
+      {% for lexer in env("TT_LEXERS").split "," %}
+      bake_file {{ lexer }}+".xml", {{ read_file "#{__DIR__}/../lexers/" + lexer + ".xml" }}
+      {% end %}
+    end
+
+    {% if flag?(:nolexers) %}
+      bake_selected_lexers
+    {% else %}
+      bake_folder "../lexers", __DIR__
+    {% end %}
   end
 
   # Get the lexer object for a language name
-  # FIXME: support mimetypes
   def self.lexer(name : String? = nil, filename : String? = nil, mimetype : String? = nil) : BaseLexer
     return lexer_by_name(name) if name && name != "autodetect"
     return lexer_by_filename(filename) if filename
@@ -33,6 +43,8 @@ module Tartrazine
     raise Exception.new("Unknown lexer: #{name}") if lexer_file_name.nil?
 
     RegexLexer.from_xml(LexerFiles.get("/#{lexer_file_name}.xml").gets_to_end)
+  rescue ex : BakedFileSystem::NoSuchFileError
+    raise Exception.new("Unknown lexer: #{name}")
   end
 
   private def self.lexer_by_filename(filename : String) : BaseLexer
@@ -84,7 +96,8 @@ module Tartrazine
 
   # Return a list of all lexers
   def self.lexers : Array(String)
-    LEXERS_BY_NAME.keys.sort!
+    file_map = LexerFiles.files.map(&.path)
+    LEXERS_BY_NAME.keys.select { |k| file_map.includes?("/#{k}.xml") }.sort!
   end
 
   # A token, the output of the tokenizer

src/main.cr

@@ -4,6 +4,10 @@ require "./tartrazine"
 HELP = <<-HELP
 tartrazine: a syntax highlighting tool
 
+You can use the CLI to generate HTML, terminal, JSON or SVG output
+from a source file using different themes.
+Keep in mind that not all formatters support all features.
+
 Usage:
   tartrazine (-h, --help)
   tartrazine FILE -f html [-t theme][--standalone][--line-numbers]
@@ -13,6 +17,8 @@ Usage:
                               [-o output]
   tartrazine FILE -f svg  [-t theme][--standalone][--line-numbers]
                           [-l lexer][-o output]
+  tartrazine FILE -f png  [-t theme][--line-numbers]
+                          [-l lexer][-o output]
   tartrazine FILE -f json [-o output]
   tartrazine --list-themes
   tartrazine --list-lexers
@@ -78,6 +84,10 @@ if options["-f"]
     formatter.standalone = options["--standalone"] != nil
     formatter.line_numbers = options["--line-numbers"] != nil
     formatter.theme = theme
+  when "png"
+    formatter = Tartrazine::Png.new
+    formatter.line_numbers = options["--line-numbers"] != nil
+    formatter.theme = theme
   else
     puts "Invalid formatter: #{formatter}"
     exit 1

src/styles.cr

@@ -11,7 +11,20 @@ module Tartrazine
 
   struct ThemeFiles
     extend BakedFileSystem
-    bake_folder "../styles", __DIR__
+
+    macro bake_selected_themes
+      {% if env("TT_THEMES") %}
+      {% for theme in env("TT_THEMES").split "," %}
+      bake_file {{ theme }}+".xml", {{ read_file "#{__DIR__}/../styles/" + theme + ".xml" }}
+      {% end %}
+      {% end %}
+    end
+
+    {% if flag?(:nothemes) %}
+      bake_selected_themes
+    {% else %}
+      bake_folder "../styles", __DIR__
+    {% end %}
   end
 
   def self.theme(name : String) : Theme