test: added minimal tests for svg and png formatters

CHANGELOG.md

@@ -2,6 +2,41 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.9.0] - 2024-09-21
+
+### 🚀 Features
+
+- PNG writer based on Stumpy libs
+
+### ⚙️ Miscellaneous Tasks
+
+- Clean
+- Detect version bump in release script
+- Improve changelog handling
+
+## [0.8.0] - 2024-09-21
+
+### 🚀 Features
+
+- SVG formatter
+
+### 🐛 Bug Fixes
+
+- HTML formatter was setting bold wrong
+
+### 📚 Documentation
+
+- Added instructions to add as a dependency
+
+### 🧪 Testing
+
+- Add basic tests for crystal and delegating lexers
+- Added tests for CSS generation
+
+### ⚙ Miscellaneous Tasks
+
+- Fix example code in README
+
 ## [0.7.0] - 2024-09-10
 
 ### 🚀 Features

Hacefile.yml

@@ -113,3 +113,11 @@ tasks:
       kcov --clean --include-path=./src ${PWD}/coverage ./bin/run_tests
     outputs:
       - coverage/index.html
+
+  loc:
+    phony: true
+    always_run: true
+    dependencies:
+      - src
+    commands: |
+      tokei src -e src/constants/

cliff.toml

@@ -67,7 +67,6 @@ commit_parsers = [
   { message = "^chore\\(deps.*\\)", skip = true },
   { message = "^chore\\(pr\\)", skip = true },
   { message = "^chore\\(pull\\)", skip = true },
-  { message = "^chore\\(ignore\\)", skip = true },
   { message = "^chore|^ci", group = "<!-- 7 -->⚙️ Miscellaneous Tasks" },
   { body = ".*security", group = "<!-- 8 -->🛡️ Security" },
   { message = "^revert", group = "<!-- 9 -->◀️ Revert" },

do_release.sh

@@ -2,12 +2,12 @@
 set e
 
 PKGNAME=$(basename "$PWD")
-VERSION=$(git cliff --bumped-version |cut -dv -f2)
+VERSION=$(git cliff --bumped-version --unreleased |cut -dv -f2)
 
 sed "s/^version:.*$/version: $VERSION/g" -i shard.yml
 git add shard.yml
 hace lint test
-git cliff --bump -o
+git cliff --bump -u -p CHANGELOG.md
 git commit -a -m "bump: Release v$VERSION"
 git tag "v$VERSION"
 git push --tags

shard.yml

@@ -1,5 +1,5 @@
 name: tartrazine
-version: 0.7.0
+version: 0.9.0
 
 authors:
   - Roberto Alsina <roberto.alsina@gmail.com>
@@ -18,6 +18,10 @@ dependencies:
     github: ralsina/sixteen
   docopt:
     github: chenkovsky/docopt.cr
+  stumpy_utils:
+    github: stumpycr/stumpy_utils
+  stumpy_png:
+    github: stumpycr/stumpy_png
 
 crystal: ">= 1.13.0"
 

spec/css/manni.css

@@ -1 +1 @@
-.e {color: #aa0000;background-color: #ffaaaa;}.b {background-color: #f0f3f3;tab-size: 8;}.k {color: #006699;font-weight: bold;}.kp {font-weight: 600;}.kt {color: #007788;}.na {color: #330099;}.nb {color: #336666;}.nc {color: #00aa88;font-weight: bold;}.nc {color: #336600;}.nd {color: #9999ff;}.ne {color: #999999;font-weight: bold;}.ne {color: #cc0000;font-weight: bold;}.nf {color: #cc00ff;}.nl {color: #9999ff;}.nn {color: #00ccff;font-weight: bold;}.nt {color: #330099;font-weight: bold;}.nv {color: #003333;}.ls {color: #cc3300;}.lsd {font-style: italic;}.lse {color: #cc3300;font-weight: bold;}.lsi {color: #aa0000;}.lso {color: #cc3300;}.lsr {color: #33aaaa;}.lss {color: #ffcc33;}.ln {color: #ff6600;}.o {color: #555555;}.ow {color: #000000;font-weight: bold;}.c {color: #0099ff;font-style: italic;}.cs {font-weight: bold;}.cp {color: #009999;font-style: normal;}.gd {background-color: #ffcccc;border: 1px solid #cc0000;}.ge {font-style: italic;}.ge {color: #ff0000;}.gh {color: #003300;font-weight: bold;}.gi {background-color: #ccffcc;border: 1px solid #00cc00;}.go {color: #aaaaaa;}.gp {color: #000099;font-weight: bold;}.gs {font-weight: bold;}.gs {color: #003300;font-weight: bold;}.gt {color: #99cc66;}.gu {text-decoration: underline;}.tw {color: #bbbbbb;}.lh {}
+.e {color: #aa0000;background-color: #ffaaaa;}.b {background-color: #f0f3f3;tab-size: 8;}.k {color: #006699;font-weight: 600;}.kp {}.kt {color: #007788;}.na {color: #330099;}.nb {color: #336666;}.nc {color: #00aa88;font-weight: 600;}.nc {color: #336600;}.nd {color: #9999ff;}.ne {color: #999999;font-weight: 600;}.ne {color: #cc0000;font-weight: 600;}.nf {color: #cc00ff;}.nl {color: #9999ff;}.nn {color: #00ccff;font-weight: 600;}.nt {color: #330099;font-weight: 600;}.nv {color: #003333;}.ls {color: #cc3300;}.lsd {font-style: italic;}.lse {color: #cc3300;font-weight: 600;}.lsi {color: #aa0000;}.lso {color: #cc3300;}.lsr {color: #33aaaa;}.lss {color: #ffcc33;}.ln {color: #ff6600;}.o {color: #555555;}.ow {color: #000000;font-weight: 600;}.c {color: #0099ff;font-style: italic;}.cs {font-weight: 600;}.cp {color: #009999;font-style: normal;}.gd {background-color: #ffcccc;border: 1px solid #cc0000;}.ge {font-style: italic;}.ge {color: #ff0000;}.gh {color: #003300;font-weight: 600;}.gi {background-color: #ccffcc;border: 1px solid #00cc00;}.go {color: #aaaaaa;}.gp {color: #000099;font-weight: 600;}.gs {font-weight: 600;}.gs {color: #003300;font-weight: 600;}.gt {color: #99cc66;}.gu {text-decoration: underline;}.tw {color: #bbbbbb;}.lh {}

spec/css/vesper.css

@@ -1 +1 @@
-.b {color: #b7b7b7;background-color: #101010;font-weight: bold;tab-size: 8;}.lh {color: #8eaaaa;background-color: #232323;}.t {color: #b7b7b7;}.e {color: #de6e6e;}.c {color: #333333;}.cp {color: #876c4f;}.cpf {color: #5f8787;}.k {color: #d69094;}.kt {color: #de6e6e;}.na {color: #8eaaaa;}.nb {color: #de6e6e;}.nbp {color: #de6e6e;}.nc {color: #8eaaaa;}.nc {color: #dab083;}.nd {color: #dab083;}.nf {color: #8eaaaa;}.nn {color: #8eaaaa;}.nt {color: #d69094;}.nv {color: #8eaaaa;}.nvi {color: #de6e6e;}.ln {color: #dab083;}.o {color: #60a592;}.ow {color: #d69094;}.l {color: #5f8787;}.ls {color: #5f8787;}.lsi {color: #876c4f;}.lsr {color: #60a592;}.lss {color: #dab083;}
+.b {color: #b7b7b7;background-color: #101010;font-weight: 600;tab-size: 8;}.lh {color: #8eaaaa;background-color: #232323;}.t {color: #b7b7b7;}.e {color: #de6e6e;}.c {color: #333333;}.cp {color: #876c4f;}.cpf {color: #5f8787;}.k {color: #d69094;}.kt {color: #de6e6e;}.na {color: #8eaaaa;}.nb {color: #de6e6e;}.nbp {color: #de6e6e;}.nc {color: #8eaaaa;}.nc {color: #dab083;}.nd {color: #dab083;}.nf {color: #8eaaaa;}.nn {color: #8eaaaa;}.nt {color: #d69094;}.nv {color: #8eaaaa;}.nvi {color: #de6e6e;}.ln {color: #dab083;}.o {color: #60a592;}.ow {color: #d69094;}.l {color: #5f8787;}.ls {color: #5f8787;}.lsi {color: #876c4f;}.lsr {color: #60a592;}.lss {color: #dab083;}

spec/tartrazine_spec.cr

@@ -1,4 +1,5 @@
 require "./spec_helper"
+require "digest/sha1"
 
 # These are the testcases from Pygments
 testcases = Dir.glob("#{__DIR__}/tests/**/*txt").sort
@@ -103,6 +104,7 @@ describe Tartrazine do
       )
     end
   end
+
   describe "to_ansi" do
     it "should do basic highlighting" do
       ansi = Tartrazine.to_ansi("puts 'Hello, World!'", "ruby")
@@ -114,11 +116,29 @@ describe Tartrazine do
         )
       else
         ansi.should eq(
-          "\e[38;2;171;70;66mputs\e[0m\e[38;2;216;216;216m \e[0m'Hello, World!'"
+          "\e[38;2;171;70;66mputs\e[0m\e[38;2;216;216;216m \e[0m\e[38;2;161;181;108m'Hello, World!'\e[0m"
         )
       end
     end
   end
+
+  describe "to_svg" do
+    it "should do basic highlighting" do
+      svg = Tartrazine.to_svg("puts 'Hello, World!'", "ruby", standalone: false)
+      svg.should eq(
+        "<text x=\"0\" y=\"19\" xml:space=\"preserve\"><tspan fill=\"#ab4642\">puts</tspan><tspan fill=\"#d8d8d8\"> </tspan><tspan fill=\"#a1b56c\">&#39;Hello, World!&#39;</tspan></text>"
+      )
+    end
+  end
+
+  describe "to_png" do
+    it "should do basic highlighting" do
+      png = Digest::SHA1.hexdigest(Tartrazine.to_png("puts 'Hello, World!'", "ruby"))
+      png.should eq(
+        "62d419dcd263fffffc265a0f04c156dc2530c362"
+      )
+    end
+  end
 end
 
 # Helper that creates lexer and tokenizes

src/formatters/ansi.cr

@@ -34,8 +34,6 @@ module Tartrazine
     end
 
     def colorize(text : String, token : String) : String
-      style = theme.styles.fetch(token, nil)
-      return text if style.nil?
       if theme.styles.has_key?(token)
         s = theme.styles[token]
       else

src/formatters/html.cr

@@ -106,8 +106,7 @@ module Tartrazine
 
           # These are true/false/nil
           outp << "border: none;" if style.border == false
-          outp << "font-weight: bold;" if style.bold
-          outp << "font-weight: #{@weight_of_bold};" if style.bold == false
+          outp << "font-weight: #{@weight_of_bold};" if style.bold
           outp << "font-style: italic;" if style.italic
           outp << "font-style: normal;" if style.italic == false
           outp << "text-decoration: underline;" if style.underline

src/formatters/png.cr

@@ -0,0 +1,117 @@
+require "../formatter"
+require "compress/gzip"
+require "digest/sha1"
+require "stumpy_png"
+require "stumpy_utils"
+
+module Tartrazine
+  def self.to_png(text : String, language : String,
+                  theme : String = "default-dark",
+                  line_numbers : Bool = false) : String
+    buf = IO::Memory.new
+
+    Tartrazine::Png.new(
+      theme: Tartrazine.theme(theme),
+      line_numbers: line_numbers
+    ).format(text, Tartrazine.lexer(name: language), buf)
+    buf.to_s
+  end
+
+  class FontFiles
+    extend BakedFileSystem
+    bake_folder "../../fonts", __DIR__
+  end
+
+  class Png < Formatter
+    include StumpyPNG
+    property? line_numbers : Bool = false
+    @font_regular : PCFParser::Font
+    @font_bold : PCFParser::Font
+    @font_oblique : PCFParser::Font
+    @font_bold_oblique : PCFParser::Font
+    @font_width = 15
+    @font_height = 24
+
+    def initialize(@theme : Theme = Tartrazine.theme("default-dark"), @line_numbers : Bool = false)
+      @font_regular = load_font("/courier-regular.pcf.gz")
+      @font_bold = load_font("/courier-bold.pcf.gz")
+      @font_oblique = load_font("/courier-oblique.pcf.gz")
+      @font_bold_oblique = load_font("/courier-bold-oblique.pcf.gz")
+    end
+
+    private def load_font(name : String) : PCFParser::Font
+      compressed = FontFiles.get(name)
+      uncompressed = Compress::Gzip::Reader.open(compressed) do |gzip|
+        gzip.gets_to_end
+      end
+      PCFParser::Font.new(IO::Memory.new uncompressed)
+    end
+
+    private def line_label(i : Int32) : String
+      "#{i + 1}".rjust(4).ljust(5)
+    end
+
+    def format(text : String, lexer : BaseLexer, outp : IO) : Nil
+      # Create canvas of correct size
+      lines = text.split("\n")
+      canvas_height = lines.size * @font_height
+      canvas_width = lines.max_of(&.size)
+      canvas_width += 5 if line_numbers?
+      canvas_width *= @font_width
+
+      bg_color = RGBA.from_hex("##{theme.styles["Background"].background.try &.hex}")
+      canvas = Canvas.new(canvas_width, canvas_height, bg_color)
+
+      tokenizer = lexer.tokenizer(text)
+      x = 0
+      y = @font_height
+      i = 0
+      if line_numbers?
+        canvas.text(x, y, line_label(i), @font_regular, RGBA.from_hex("##{theme.styles["Background"].color.try &.hex}"))
+        x += 5 * @font_width
+      end
+
+      tokenizer.each do |token|
+        font, color = token_style(token[:type])
+        # These fonts are very limited
+        t = token[:value].gsub(/[^[:ascii:]]/, "?")
+        canvas.text(x, y, t.rstrip("\n"), font, color)
+        if token[:value].includes?("\n")
+          x = 0
+          y += @font_height
+          i += 1
+          if line_numbers?
+            canvas.text(x, y, line_label(i), @font_regular, RGBA.from_hex("##{theme.styles["Background"].color.try &.hex}"))
+            x += 4 * @font_width
+          end
+        end
+
+        x += token[:value].size * @font_width
+      end
+
+      StumpyPNG.write(canvas, outp)
+    end
+
+    def token_style(token : String) : {PCFParser::Font, RGBA}
+      if theme.styles.has_key?(token)
+        s = theme.styles[token]
+      else
+        # Themes don't contain information for each specific
+        # token type. However, they may contain information
+        # for a parent style. Worst case, we go to the root
+        # (Background) style.
+        s = theme.styles[theme.style_parents(token).reverse.find { |parent|
+          theme.styles.has_key?(parent)
+        }]
+      end
+
+      color = RGBA.from_hex("##{theme.styles["Background"].color.try &.hex}")
+      color = RGBA.from_hex("##{s.color.try &.hex}") if s.color
+
+      return {@font_bold_oblique, color} if s.bold && s.italic
+      return {@font_bold, color} if s.bold
+      return {@font_oblique, color} if s.italic
+      return {@font_regular, color}
+    end
+  end
+end

src/main.cr

@@ -4,6 +4,10 @@ require "./tartrazine"
 HELP = <<-HELP
 tartrazine: a syntax highlighting tool
 
+You can use the CLI to generate HTML, terminal, JSON or SVG output
+from a source file using different themes.
+Keep in mind that not all formatters support all features.
+
 Usage:
   tartrazine (-h, --help)
   tartrazine FILE -f html [-t theme][--standalone][--line-numbers]
@@ -13,6 +17,8 @@ Usage:
                               [-o output]
   tartrazine FILE -f svg  [-t theme][--standalone][--line-numbers]
                           [-l lexer][-o output]
+  tartrazine FILE -f png  [-t theme][--line-numbers]
+                          [-l lexer][-o output]
   tartrazine FILE -f json [-o output]
   tartrazine --list-themes
   tartrazine --list-lexers
@@ -78,6 +84,10 @@ if options["-f"]
     formatter.standalone = options["--standalone"] != nil
     formatter.line_numbers = options["--line-numbers"] != nil
     formatter.theme = theme
+  when "png"
+    formatter = Tartrazine::Png.new
+    formatter.line_numbers = options["--line-numbers"] != nil
+    formatter.theme = theme
   else
     puts "Invalid formatter: #{formatter}"
     exit 1