bump: Release v0.8.0

.ameba.yml

@@ -1,9 +1,9 @@
 # This configuration file was generated by `ameba --gen-config`
-# on 2024-08-12 22:00:49 UTC using Ameba version 1.6.1.
+# on 2024-09-21 14:59:30 UTC using Ameba version 1.6.1.
 # The point is for the user to remove these configuration records
 # one by one as the reported problems are removed from the code base.
 
-# Problems found: 2
+# Problems found: 3
 # Run `ameba --only Documentation/DocumentationAdmonition` for details
 Documentation/DocumentationAdmonition:
   Description: Reports documentation admonitions
@@ -18,104 +18,17 @@ Documentation/DocumentationAdmonition:
   Enabled: true
   Severity: Warning
 
-# Problems found: 22
-# Run `ameba --only Lint/MissingBlockArgument` for details
-Lint/MissingBlockArgument:
-  Description: Disallows yielding method definitions without block argument
-  Excluded:
-  - pygments/tests/examplefiles/cr/test.cr
-  Enabled: true
-  Severity: Warning
-
 # Problems found: 1
-# Run `ameba --only Lint/NotNil` for details
-Lint/NotNil:
-  Description: Identifies usage of `not_nil!` calls
+# Run `ameba --only Lint/SpecFilename` for details
+Lint/SpecFilename:
+  Description: Enforces spec filenames to have `_spec` suffix
   Excluded:
-  - pygments/tests/examplefiles/cr/test.cr
+  - spec/examples/crystal/hello.cr
+  IgnoredDirs:
+  - spec/support
+  - spec/fixtures
+  - spec/data
+  IgnoredFilenames:
+  - spec_helper
   Enabled: true
   Severity: Warning
-
-# Problems found: 34
-# Run `ameba --only Lint/ShadowingOuterLocalVar` for details
-Lint/ShadowingOuterLocalVar:
-  Description: Disallows the usage of the same name as outer local variables for block
-    or proc arguments
-  Excluded:
-  - pygments/tests/examplefiles/cr/test.cr
-  Enabled: true
-  Severity: Warning
-
-# Problems found: 1
-# Run `ameba --only Lint/UnreachableCode` for details
-Lint/UnreachableCode:
-  Description: Reports unreachable code
-  Excluded:
-  - pygments/tests/examplefiles/cr/test.cr
-  Enabled: true
-  Severity: Warning
-
-# Problems found: 6
-# Run `ameba --only Lint/UselessAssign` for details
-Lint/UselessAssign:
-  Description: Disallows useless variable assignments
-  ExcludeTypeDeclarations: false
-  Excluded:
-  - pygments/tests/examplefiles/cr/test.cr
-  Enabled: true
-  Severity: Warning
-
-# Problems found: 3
-# Run `ameba --only Naming/BlockParameterName` for details
-Naming/BlockParameterName:
-  Description: Disallows non-descriptive block parameter names
-  MinNameLength: 3
-  AllowNamesEndingInNumbers: true
-  Excluded:
-  - pygments/tests/examplefiles/cr/test.cr
-  AllowedNames:
-  - _
-  - e
-  - i
-  - j
-  - k
-  - v
-  - x
-  - y
-  - ex
-  - io
-  - ws
-  - op
-  - tx
-  - id
-  - ip
-  - k1
-  - k2
-  - v1
-  - v2
-  ForbiddenNames: []
-  Enabled: true
-  Severity: Convention
-
-# Problems found: 1
-# Run `ameba --only Naming/RescuedExceptionsVariableName` for details
-Naming/RescuedExceptionsVariableName:
-  Description: Makes sure that rescued exceptions variables are named as expected
-  Excluded:
-  - pygments/tests/examplefiles/cr/test.cr
-  AllowedNames:
-  - e
-  - ex
-  - exception
-  - error
-  Enabled: true
-  Severity: Convention
-
-# Problems found: 6
-# Run `ameba --only Naming/TypeNames` for details
-Naming/TypeNames:
-  Description: Enforces type names in camelcase manner
-  Excluded:
-  - pygments/tests/examplefiles/cr/test.cr
-  Enabled: true
-  Severity: Convention

CHANGELOG.md

@@ -2,17 +2,52 @@
 
 All notable changes to this project will be documented in this file.
 
-## [0.6.4] - 2024-08-28
+## [0.8.0] - 2024-09-21
+
+### 🚀 Features
+
+- SVG formatter
+
+### 🐛 Bug Fixes
+
+- HTML formatter was setting bold wrong
+
+### 📚 Documentation
+
+- Added instructions to add as a dependency
+
+### 🧪 Testing
+
+- Add basic tests for crystal and delegating lexers
+- Added tests for CSS generation
+
+### ⚙ Miscellaneous Tasks
+
+- Fix example code in README
+
+## [0.7.0] - 2024-09-10
+
+### 🚀 Features
+
+- Higher level API (`to_html` and `to_ansi`)
+- Use the native crystal highlighter
 
 ### 🐛 Bug Fixes
 
 - Ameba
 - Variable bame in Hacefile
+- Make it easier to import the Ansi formatter
+- Renamed BaseLexer to Lexer and Lexer to RegexLexer to make API nicer
+- Make install work
 
 ### 📚 Documentation
 
 - Mention AUR package
 
+### 🧪 Testing
+
+- Add CI workflows
+
 ### ⚙️ Miscellaneous Tasks
 
 - Pre-commit hooks
@@ -21,6 +56,10 @@ All notable changes to this project will be documented in this file.
 - Force conventional commit messages
 - Force conventional commit messages
 - Updated pre-commit
+- *(ignore)* Fix tests
+- Added badges
+- Added badges
+- *(ignore)* Removed random file
 
 ### Build
 
@@ -30,6 +69,7 @@ All notable changes to this project will be documented in this file.
 
 ### Bump
 
+- Release v0.6.4
 - Release v0.6.4
 
 ## [0.6.1] - 2024-08-25

README.md

@@ -45,6 +45,14 @@ $ tartrazine whatever.c -t catppuccin-macchiato --line-numbers \
 
 ## Usage as a Library
 
+Add to your `shard.yml`:
+
+```yaml
+dependencies:
+  tartrazine:
+    github: ralsina/tartrazine
+```
+
 This is the high level API:
 
 ```crystal
@@ -63,7 +71,7 @@ This does more or less the same thing, but more manually:
 
 ```crystal
 lexer = Tartrazine.lexer("crystal")
-formatter = Tartrazine::Html.new (
+formatter = Tartrazine::Html.new(
   theme: Tartrazine.theme("catppuccin-macchiato"),
   line_numbers: true,
   standalone: true,

cliff.toml

@@ -67,6 +67,7 @@ commit_parsers = [
   { message = "^chore\\(deps.*\\)", skip = true },
   { message = "^chore\\(pr\\)", skip = true },
   { message = "^chore\\(pull\\)", skip = true },
+  { message = "^chore\\(ignore\\)", skip = true },
   { message = "^chore|^ci", group = "<!-- 7 -->⚙️ Miscellaneous Tasks" },
   { body = ".*security", group = "<!-- 8 -->🛡️ Security" },
   { message = "^revert", group = "<!-- 9 -->◀️ Revert" },

shard.yml

@@ -1,5 +1,5 @@
 name: tartrazine
-version: 0.6.4
+version: 0.8.0
 
 authors:
   - Roberto Alsina <roberto.alsina@gmail.com>

spec/css/manni.css

@@ -0,0 +1 @@
+.e {color: #aa0000;background-color: #ffaaaa;}.b {background-color: #f0f3f3;tab-size: 8;}.k {color: #006699;font-weight: 600;}.kp {}.kt {color: #007788;}.na {color: #330099;}.nb {color: #336666;}.nc {color: #00aa88;font-weight: 600;}.nc {color: #336600;}.nd {color: #9999ff;}.ne {color: #999999;font-weight: 600;}.ne {color: #cc0000;font-weight: 600;}.nf {color: #cc00ff;}.nl {color: #9999ff;}.nn {color: #00ccff;font-weight: 600;}.nt {color: #330099;font-weight: 600;}.nv {color: #003333;}.ls {color: #cc3300;}.lsd {font-style: italic;}.lse {color: #cc3300;font-weight: 600;}.lsi {color: #aa0000;}.lso {color: #cc3300;}.lsr {color: #33aaaa;}.lss {color: #ffcc33;}.ln {color: #ff6600;}.o {color: #555555;}.ow {color: #000000;font-weight: 600;}.c {color: #0099ff;font-style: italic;}.cs {font-weight: 600;}.cp {color: #009999;font-style: normal;}.gd {background-color: #ffcccc;border: 1px solid #cc0000;}.ge {font-style: italic;}.ge {color: #ff0000;}.gh {color: #003300;font-weight: 600;}.gi {background-color: #ccffcc;border: 1px solid #00cc00;}.go {color: #aaaaaa;}.gp {color: #000099;font-weight: 600;}.gs {font-weight: 600;}.gs {color: #003300;font-weight: 600;}.gt {color: #99cc66;}.gu {text-decoration: underline;}.tw {color: #bbbbbb;}.lh {}

spec/css/vesper.css

@@ -0,0 +1 @@
+.b {color: #b7b7b7;background-color: #101010;font-weight: 600;tab-size: 8;}.lh {color: #8eaaaa;background-color: #232323;}.t {color: #b7b7b7;}.e {color: #de6e6e;}.c {color: #333333;}.cp {color: #876c4f;}.cpf {color: #5f8787;}.k {color: #d69094;}.kt {color: #de6e6e;}.na {color: #8eaaaa;}.nb {color: #de6e6e;}.nbp {color: #de6e6e;}.nc {color: #8eaaaa;}.nc {color: #dab083;}.nd {color: #dab083;}.nf {color: #8eaaaa;}.nn {color: #8eaaaa;}.nt {color: #d69094;}.nv {color: #8eaaaa;}.nvi {color: #de6e6e;}.ln {color: #dab083;}.o {color: #60a592;}.ow {color: #d69094;}.l {color: #5f8787;}.ls {color: #5f8787;}.lsi {color: #876c4f;}.lsr {color: #60a592;}.lss {color: #dab083;}

spec/examples/crystal/hello.cr

@@ -0,0 +1 @@
+puts "Hello Crystal!"

spec/examples/crystal/hello.cr.json

@@ -0,0 +1 @@
+[{"type":"Text","value":"puts "},{"type":"LiteralString","value":"\"Hello Crystal!\""},{"type":"Text","value":"\n"}]

spec/examples/jinja+python/funko.py.j2

@@ -0,0 +1,11 @@
+from flask import Flask, request
+
+app = Flask("{{name}}")
+
+@app.route('/')
+def handle():
+    return "Hello World from Flask!"
+
+@app.route('/ping')
+def ping():
+    return "OK"

spec/examples/jinja+python/funko.py.j2.json

@@ -0,0 +1 @@
+[{"type":"KeywordNamespace","value":"from"},{"type":"Text","value":" "},{"type":"NameNamespace","value":"flask"},{"type":"Text","value":" "},{"type":"KeywordNamespace","value":"import"},{"type":"Text","value":" "},{"type":"Name","value":"Flask"},{"type":"Punctuation","value":","},{"type":"Text","value":" "},{"type":"Name","value":"request"},{"type":"Text","value":"\n\n"},{"type":"Name","value":"app"},{"type":"Text","value":" "},{"type":"Operator","value":"="},{"type":"Text","value":" "},{"type":"Name","value":"Flask"},{"type":"Punctuation","value":"("},{"type":"LiteralStringDouble","value":"\""},{"type":"CommentPreproc","value":"{{"},{"type":"NameVariable","value":"name"},{"type":"CommentPreproc","value":"}}"},{"type":"LiteralStringDouble","value":"\")"},{"type":"Text","value":"\n\n"},{"type":"NameDecorator","value":"@app.route"},{"type":"Punctuation","value":"("},{"type":"LiteralStringSingle","value":"'/'"},{"type":"Punctuation","value":")"},{"type":"Text","value":"\n"},{"type":"Keyword","value":"def"},{"type":"Text","value":" "},{"type":"NameFunction","value":"handle"},{"type":"Punctuation","value":"():"},{"type":"Text","value":"\n    "},{"type":"Keyword","value":"return"},{"type":"Text","value":" "},{"type":"LiteralStringDouble","value":"\"Hello World from Flask!\""},{"type":"Text","value":"\n\n"},{"type":"NameDecorator","value":"@app.route"},{"type":"Punctuation","value":"("},{"type":"LiteralStringSingle","value":"'/ping'"},{"type":"Punctuation","value":")"},{"type":"Text","value":"\n"},{"type":"Keyword","value":"def"},{"type":"Text","value":" "},{"type":"NameFunction","value":"ping"},{"type":"Punctuation","value":"():"},{"type":"Text","value":"\n    "},{"type":"Keyword","value":"return"},{"type":"Text","value":" "},{"type":"LiteralStringDouble","value":"\"OK\""},{"type":"Text","value":"\n"}]

spec/tartrazine_spec.cr

@@ -3,6 +3,12 @@ require "./spec_helper"
 # These are the testcases from Pygments
 testcases = Dir.glob("#{__DIR__}/tests/**/*txt").sort
 
+# These are custom testcases
+examples = Dir.glob("#{__DIR__}/examples/**/*.*").reject(&.ends_with? ".json").sort!
+
+# CSS Stylesheets
+css_files = Dir.glob("#{__DIR__}/css/*.css")
+
 # These lexers don't load because of parsing issues
 failing_lexers = {
   "webgpu_shading_language",
@@ -51,6 +57,14 @@ not_my_fault = {
 
 describe Tartrazine do
   describe "Lexer" do
+    examples.each do |example|
+      it "parses #{example}".split("/")[-2...].join("/") do
+        lexer = Tartrazine.lexer(name: File.basename(File.dirname(example)).downcase)
+        text = File.read(example)
+        expected = Array(Tartrazine::Token).from_json(File.read("#{example}.json"))
+        Tartrazine::RegexLexer.collapse_tokens(lexer.tokenizer(text).to_a).should eq expected
+      end
+    end
     testcases.each do |testcase|
       if known_bad.includes?(testcase)
         pending "parses #{testcase}".split("/")[-2...].join("/") do
@@ -70,6 +84,17 @@ describe Tartrazine do
     end
   end
 
+  describe "formatter" do
+    css_files.each do |css_file|
+      it "generates #{css_file}" do
+        css = File.read(css_file)
+        theme = Tartrazine.theme(File.basename(css_file, ".css"))
+        formatter = Tartrazine::Html.new(theme: theme)
+        formatter.style_defs.strip.should eq css.strip
+      end
+    end
+  end
+
   describe "to_html" do
     it "should do basic highlighting" do
       html = Tartrazine.to_html("puts 'Hello, World!'", "ruby", standalone: false)

src/formatter.cr

@@ -17,12 +17,19 @@ module Tartrazine
     end
 
     def format(text : String, lexer : Lexer) : String
-      raise Exception.new("Not implemented")
+      outp = String::Builder.new("")
+      format(text, lexer, outp)
+      outp.to_s
     end
 
     # Return the styles, if the formatter supports it.
     def style_defs : String
       raise Exception.new("Not implemented")
     end
+
+    # Is this line in the highlighted ranges?
+    def highlighted?(line : Int) : Bool
+      highlight_lines.any?(&.includes?(line))
+    end
   end
 end

src/formatters/ansi.cr

@@ -20,12 +20,6 @@ module Tartrazine
       "#{i + 1}".rjust(4).ljust(5)
     end
 
-    def format(text : String, lexer : BaseLexer) : String
-      outp = String::Builder.new("")
-      format(text, lexer, outp)
-      outp.to_s
-    end
-
     def format(text : String, lexer : BaseLexer, outp : IO) : Nil
       tokenizer = lexer.tokenizer(text)
       i = 0

src/formatters/html.cr

@@ -106,8 +106,7 @@ module Tartrazine
 
           # These are true/false/nil
           outp << "border: none;" if style.border == false
-          outp << "font-weight: bold;" if style.bold
-          outp << "font-weight: #{@weight_of_bold};" if style.bold == false
+          outp << "font-weight: #{@weight_of_bold};" if style.bold
           outp << "font-style: italic;" if style.italic
           outp << "font-style: normal;" if style.italic == false
           outp << "text-decoration: underline;" if style.underline
@@ -134,10 +133,5 @@ module Tartrazine
       end
       class_prefix + Abbreviations[token]
     end
-
-    # Is this line in the highlighted ranges?
-    def highlighted?(line : Int) : Bool
-      highlight_lines.any?(&.includes?(line))
-    end
   end
 end

src/formatters/svg.cr

@@ -0,0 +1,129 @@
+require "../constants/token_abbrevs.cr"
+require "../formatter"
+require "html"
+
+module Tartrazine
+  def self.to_svg(text : String, language : String,
+                  theme : String = "default-dark",
+                  standalone : Bool = true,
+                  line_numbers : Bool = false) : String
+    Tartrazine::Svg.new(
+      theme: Tartrazine.theme(theme),
+      standalone: standalone,
+      line_numbers: line_numbers
+    ).format(text, Tartrazine.lexer(name: language))
+  end
+
+  class Svg < Formatter
+    property highlight_lines : Array(Range(Int32, Int32)) = [] of Range(Int32, Int32)
+    property line_number_id_prefix : String = "line-"
+    property line_number_start : Int32 = 1
+    property tab_width = 8
+    property? line_numbers : Bool = false
+    property? linkable_line_numbers : Bool = true
+    property? standalone : Bool = false
+    property weight_of_bold : Int32 = 600
+    property fs : Int32
+    property ystep : Int32
+
+    property theme : Theme
+
+    def initialize(@theme : Theme = Tartrazine.theme("default-dark"), *,
+                   @highlight_lines = [] of Range(Int32, Int32),
+                   @class_prefix : String = "",
+                   @line_number_id_prefix = "line-",
+                   @line_number_start = 1,
+                   @tab_width = 8,
+                   @line_numbers : Bool = false,
+                   @linkable_line_numbers : Bool = true,
+                   @standalone : Bool = false,
+                   @weight_of_bold : Int32 = 600,
+                   @font_family : String = "monospace",
+                   @font_size : String = "14px")
+      if font_size.ends_with? "px"
+        @fs = font_size[0...-2].to_i
+      else
+        @fs = font_size.to_i
+      end
+      @ystep = @fs + 5
+    end
+
+    def format(text : String, lexer : BaseLexer, io : IO) : Nil
+      pre, post = wrap_standalone
+      io << pre if standalone?
+      format_text(text, lexer, io)
+      io << post if standalone?
+    end
+
+    # Wrap text into a full HTML document, including the CSS for the theme
+    def wrap_standalone
+      output = String.build do |outp|
+        outp << %(<?xml version="1.0" encoding="utf-8"?>
+        <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.0//EN" "http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd">
+        <svg xmlns="http://www.w3.org/2000/svg">
+        <g font-family="#{self.@font_family}" font-size="#{self.@font_size}">)
+      end
+      {output.to_s, "</g></svg>"}
+    end
+
+    private def line_label(i : Int32, x : Int32, y : Int32) : String
+      line_label = "#{i + 1}".rjust(4).ljust(5)
+      line_style = highlighted?(i + 1) ? "font-weight=\"#{@weight_of_bold}\"" : ""
+      line_id = linkable_line_numbers? ? "id=\"#{line_number_id_prefix}#{i + 1}\"" : ""
+      %(<text #{line_style} #{line_id}  x="#{4*ystep}" y="#{y}" text-anchor="end">#{line_label}</text>)
+    end
+
+    def format_text(text : String, lexer : BaseLexer, outp : IO)
+      x = 0
+      y = ystep
+      i = 0
+      line_x = x
+      line_x += 5 * ystep if line_numbers?
+      tokenizer = lexer.tokenizer(text)
+      outp << line_label(i, x, y) if line_numbers?
+      outp << %(<text x="#{line_x}" y="#{y}" xml:space="preserve">)
+      tokenizer.each do |token|
+        if token[:value].ends_with? "\n"
+          outp << "<tspan #{get_style(token[:type])}>#{HTML.escape(token[:value][0...-1])}</tspan>"
+          outp << "</text>"
+          x = 0
+          y += ystep
+          i += 1
+          outp << line_label(i, x, y) if line_numbers?
+          outp << %(<text x="#{line_x}" y="#{y}" xml:space="preserve">)
+        else
+          outp << "<tspan#{get_style(token[:type])}>#{HTML.escape(token[:value])}</tspan>"
+          x += token[:value].size * ystep
+        end
+      end
+      outp << "</text>"
+    end
+
+    # Given a token type, return the style.
+    def get_style(token : String) : String
+      if !theme.styles.has_key? token
+        # Themes don't contain information for each specific
+        # token type. However, they may contain information
+        # for a parent style. Worst case, we go to the root
+        # (Background) style.
+        parent = theme.style_parents(token).reverse.find { |dad|
+          theme.styles.has_key?(dad)
+        }
+        theme.styles[token] = theme.styles[parent]
+      end
+      output = String.build do |outp|
+        style = theme.styles[token]
+        outp << " fill=\"##{style.color.try &.hex}\"" if style.color
+        # No support for background color or border in SVG
+
+        outp << " font-weight=\"#{@weight_of_bold}\"" if style.bold
+        outp << " font-weight=\"normal\"" if style.bold == false
+        outp << " font-style=\"italic\"" if style.italic
+        outp << " font-style=\"normal\"" if style.italic == false
+        outp << " text-decoration=\"underline\"" if style.underline
+        outp << " text-decoration=\"none" if style.underline == false
+      end
+      output
+    end
+  end
+end

src/main.cr

@@ -11,6 +11,8 @@ Usage:
   tartrazine -f html -t theme --css
   tartrazine FILE -f terminal [-t theme][-l lexer][--line-numbers]
                               [-o output]
+  tartrazine FILE -f svg  [-t theme][--standalone][--line-numbers]
+                          [-l lexer][-o output]
   tartrazine FILE -f json [-o output]
   tartrazine --list-themes
   tartrazine --list-lexers
@@ -18,7 +20,7 @@ Usage:
   tartrazine --version
 
 Options:
-  -f <formatter>      Format to use (html, terminal, json)
+  -f <formatter>      Format to use (html, terminal, json, svg)
   -t <theme>          Theme to use, see --list-themes [default: default-dark]
   -l <lexer>          Lexer (language) to use, see --list-lexers. Use more than
                       one lexer with "+" (e.g. jinja+yaml) [default: autodetect]
@@ -71,6 +73,11 @@ if options["-f"]
     formatter.theme = theme
   when "json"
     formatter = Tartrazine::Json.new
+  when "svg"
+    formatter = Tartrazine::Svg.new
+    formatter.standalone = options["--standalone"] != nil
+    formatter.line_numbers = options["--line-numbers"] != nil
+    formatter.theme = theme
   else
     puts "Invalid formatter: #{formatter}"
     exit 1

src/run_tests.cr

@@ -1 +1 @@
-require "../spec/**"
+require "../spec/tartrazine_spec.cr"