bump: Release v0.11.1

Conditional lexers and themes

CHANGELOG.md

@@ -2,6 +2,29 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.11.1] - 2024-10-14
+
+### 🐛 Bug Fixes
+
+- Support choosing lexers when used as a library
+
+## [0.11.0] - 2024-10-14
+
+### 🚀 Features
+
+- Support selecting only some themes
+
+## [0.10.0] - 2024-09-26
+
+### 🚀 Features
+
+- Optional conditional baking of lexers
+
+### 🐛 Bug Fixes
+
+- Strip binaries for release artifacts
+- Fix metadata to show crystal
+
 ## [0.9.1] - 2024-09-22
 
 ### 🐛 Bug Fixes

README.md

@@ -82,6 +82,51 @@ puts formatter.format("puts \"Hello, world!\"", lexer)
 The reason you may want to use the manual version is to reuse
 the lexer and formatter objects for performance reasons.
 
+## Choosing what Lexers you want
+
+By default Tartrazine will support all its lexers by embedding
+them in the binary. This makes the binary large. If you are
+using it as a library, you may want to just include a selection of lexers. To do that:
+
+* Pass the `-Dnolexers` flag to the compiler
+* Set the `TT_LEXERS` environment variable to a
+  comma-separated list of lexers you want to include.
+
+
+This builds a binary with only the python, markdown, bash and yaml lexers (enough to highlight this `README.md`):
+
+```bash
+> TT_LEXERS=python,markdown,bash,yaml shards build -Dnolexers -d --error-trace
+Dependencies are satisfied
+Building: tartrazine
+```
+
+## Choosing what themes you want
+
+Themes come from two places, tartrazine itself and [Sixteen](https://github.com/ralsina/sixteen).
+
+To only embed selected themes, build your project with the `-Dnothemes` option, and
+you can set two environment variables to control which themes are included:
+
+* `TT_THEMES` is a comma-separated list of themes to include from tartrazine (see the styles directory in the source)
+* `SIXTEEN_THEMES` is a comma-separated list of themes to include from Sixteen (see the base16 directory in the sixteen source)
+
+For example (using the tartrazine CLI as the project):
+
+```bash
+$ TT_THEMES=colorful,autumn SIXTEEN_THEMES=pasque,pico shards build -Dnothemes
+Dependencies are satisfied
+Building: tartrazine
+
+$ ./bin/tartrazine  --list-themes
+autumn
+colorful
+pasque
+pico
+```
+
+Be careful not to build without any themes at all, nothing will work.
+
 ## Contributing
 
 1. Fork it (<https://github.com/ralsina/tartrazine/fork>)

build_static.sh

@@ -7,10 +7,10 @@ docker run --rm --privileged \
 
 # Build for AMD64
 docker build . -f Dockerfile.static -t tartrazine-builder
-docker run -ti --rm -v "$PWD":/app --user="$UID" tartrazine-builder /bin/sh -c "cd /app && rm -rf lib shard.lock && shards build --static --release"
+docker run -ti --rm -v "$PWD":/app --user="$UID" tartrazine-builder /bin/sh -c "cd /app && rm -rf lib shard.lock && shards build --static --release && strip bin/tartrazine"
 mv bin/tartrazine bin/tartrazine-static-linux-amd64
 
 # Build for ARM64
 docker build . -f Dockerfile.static --platform linux/arm64 -t tartrazine-builder
-docker run -ti --rm -v "$PWD":/app --platform linux/arm64 --user="$UID" tartrazine-builder /bin/sh -c "cd /app && rm -rf lib shard.lock && shards build --static --release"
+docker run -ti --rm -v "$PWD":/app --platform linux/arm64 --user="$UID" tartrazine-builder /bin/sh -c "cd /app && rm -rf lib shard.lock && shards build --static --release && strip bin/tartrazine"
 mv bin/tartrazine bin/tartrazine-static-linux-arm64

do_release.sh

@@ -9,7 +9,7 @@ git add shard.yml
 hace lint test
 git cliff --bump -u -p CHANGELOG.md
 git commit -a -m "bump: Release v$VERSION"
+hace static
 git tag "v$VERSION"
 git push --tags
-hace static
 gh release create "v$VERSION" "bin/$PKGNAME-static-linux-amd64" "bin/$PKGNAME-static-linux-arm64" --title "Release v$VERSION" --notes "$(git cliff -l -s all)"

scripts/lexer_metadata.py

@@ -38,6 +38,12 @@ for fname in glob.glob("lexers/*.xml"):
         lexer_by_filename[filename].add(lexer_name)
 
 with open("src/constants/lexers.cr", "w") as f:
+    # Crystal doesn't come from a xml file
+    lexer_by_name["crystal"] = "crystal"
+    lexer_by_name["cr"] = "crystal"
+    lexer_by_filename["*.cr"] = ["crystal"]
+    lexer_by_mimetype["text/x-crystal"] = "crystal"
+
     f.write("module Tartrazine\n")
     f.write("  LEXERS_BY_NAME = {\n")
     for k in sorted(lexer_by_name.keys()):

shard.yml

@@ -1,5 +1,5 @@
 name: tartrazine
-version: 0.9.1
+version: 0.11.1
 
 authors:
   - Roberto Alsina <roberto.alsina@gmail.com>

src/constants/lexers.cr

@@ -471,7 +471,7 @@ module Tartrazine
     "application/x-fennel"                        => "fennel",
     "application/x-fish"                          => "fish",
     "application/x-forth"                         => "forth",
-    "application/x-gdscript"                      => "gdscript",
+    "application/x-gdscript"                      => "gdscript3",
     "application/x-hcl"                           => "hcl",
     "application/x-hy"                            => "hy",
     "application/x-javascript"                    => "javascript",
@@ -594,7 +594,7 @@ module Tartrazine
     "text/x-fortran"                              => "fortran",
     "text/x-fsharp"                               => "fsharp",
     "text/x-gas"                                  => "gas",
-    "text/x-gdscript"                             => "gdscript",
+    "text/x-gdscript"                             => "gdscript3",
     "text/x-gherkin"                              => "gherkin",
     "text/x-gleam"                                => "gleam",
     "text/x-glslsrc"                              => "glsl",

src/lexer.cr

@@ -6,11 +6,21 @@ require "crystal/syntax_highlighter"
 module Tartrazine
   class LexerFiles
     extend BakedFileSystem
-    bake_folder "../lexers", __DIR__
+
+    macro bake_selected_lexers
+      {% for lexer in env("TT_LEXERS").split "," %}
+      bake_file {{ lexer }}+".xml", {{ read_file "#{__DIR__}/../lexers/" + lexer + ".xml" }}
+      {% end %}
+    end
+
+    {% if flag?(:nolexers) %}
+      bake_selected_lexers
+    {% else %}
+      bake_folder "../lexers", __DIR__
+    {% end %}
   end
 
   # Get the lexer object for a language name
-  # FIXME: support mimetypes
   def self.lexer(name : String? = nil, filename : String? = nil, mimetype : String? = nil) : BaseLexer
     return lexer_by_name(name) if name && name != "autodetect"
     return lexer_by_filename(filename) if filename
@@ -33,6 +43,8 @@ module Tartrazine
     raise Exception.new("Unknown lexer: #{name}") if lexer_file_name.nil?
 
     RegexLexer.from_xml(LexerFiles.get("/#{lexer_file_name}.xml").gets_to_end)
+  rescue ex : BakedFileSystem::NoSuchFileError
+    raise Exception.new("Unknown lexer: #{name}")
   end
 
   private def self.lexer_by_filename(filename : String) : BaseLexer
@@ -84,7 +96,8 @@ module Tartrazine
 
   # Return a list of all lexers
   def self.lexers : Array(String)
-    LEXERS_BY_NAME.keys.sort!
+    file_map = LexerFiles.files.map(&.path)
+    LEXERS_BY_NAME.keys.select { |k| file_map.includes?("/#{k}.xml") }.sort!
   end
 
   # A token, the output of the tokenizer

src/styles.cr

@@ -11,7 +11,20 @@ module Tartrazine
 
   struct ThemeFiles
     extend BakedFileSystem
-    bake_folder "../styles", __DIR__
+
+    macro bake_selected_themes
+      {% if env("TT_THEMES") %}
+      {% for theme in env("TT_THEMES").split "," %}
+      bake_file {{ theme }}+".xml", {{ read_file "#{__DIR__}/../styles/" + theme + ".xml" }}
+      {% end %}
+      {% end %}
+    end
+
+    {% if flag?(:nothemes) %}
+      bake_selected_themes
+    {% else %}
+      bake_folder "../styles", __DIR__
+    {% end %}
   end
 
   def self.theme(name : String) : Theme