diff --git a/TODO.md b/TODO.md
index 51cf8e9..71e22bf 100644
--- a/TODO.md
+++ b/TODO.md
@@ -9,4 +9,4 @@
* ✅ Implement lexer loader by file extension
* ✅ Add --line-numbers to terminal formatter
* Implement lexer loader by mime type
-* Implement Delegating lexers
\ No newline at end of file
+* ✅ Implement Delegating lexers
diff --git a/lexers/markdown.xml b/lexers/markdown.xml
index 058f9b6..e415ebf 100644
--- a/lexers/markdown.xml
+++ b/lexers/markdown.xml
@@ -18,19 +18,21 @@
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/src/actions.cr b/src/actions.cr
index 74ac190..9ae536d 100644
--- a/src/actions.cr
+++ b/src/actions.cr
@@ -16,13 +16,16 @@ module Tartrazine
Push
Token
Using
+ Usingbygroup
Usingself
end
struct Action
property actions : Array(Action) = [] of Action
+ @content_index : Int32 = 0
@depth : Int32 = 0
+ @lexer_index : Int32 = 0
@lexer_name : String = ""
@states : Array(String) = [] of String
@states_to_push : Array(String) = [] of String
@@ -62,6 +65,9 @@ module Tartrazine
@states = xml.attributes.select { |attrib|
attrib.name == "state"
}.map &.content
+ when ActionType::Usingbygroup
+ @lexer_index = xml["lexer"].to_i
+ @content_index = xml["content"].to_i
end
end
@@ -134,6 +140,12 @@ module Tartrazine
tokenizer.lexer.states[new_state.name] = new_state
tokenizer.state_stack << new_state.name
[] of Token
+ when ActionType::Usingbygroup
+ # Shunt to content-specified lexer
+ return [] of Token if match.empty?
+ Tartrazine.lexer(String.new(match[@lexer_index].value)).tokenizer(
+ String.new(match[@content_index].value),
+ secondary: true).to_a
else
raise Exception.new("Unknown action type: #{@type}")
end
diff --git a/src/lexer.cr b/src/lexer.cr
index a515baf..e38a261 100644
--- a/src/lexer.cr
+++ b/src/lexer.cr
@@ -12,24 +12,24 @@ module Tartrazine
def self.lexer(name : String? = nil, filename : String? = nil) : BaseLexer
return lexer_by_name(name) if name && name != "autodetect"
return lexer_by_filename(filename) if filename
-
+
Lexer.from_xml(LexerFiles.get("/#{LEXERS_BY_NAME["plaintext"]}.xml").gets_to_end)
end
-
+
private def self.lexer_by_name(name : String) : BaseLexer
lexer_file_name = LEXERS_BY_NAME.fetch(name.downcase, nil)
return create_delegating_lexer(name) if lexer_file_name.nil? && name.includes? "+"
raise Exception.new("Unknown lexer: #{name}") if lexer_file_name.nil?
-
+
Lexer.from_xml(LexerFiles.get("/#{lexer_file_name}.xml").gets_to_end)
end
-
+
private def self.lexer_by_filename(filename : String) : BaseLexer
candidates = Set(String).new
LEXERS_BY_FILENAME.each do |k, v|
candidates += v.to_set if File.match?(k, File.basename(filename))
end
-
+
case candidates.size
when 0
lexer_file_name = LEXERS_BY_NAME["plaintext"]
@@ -38,16 +38,17 @@ module Tartrazine
else
raise Exception.new("Multiple lexers match the filename: #{candidates.to_a.join(", ")}")
end
-
+
Lexer.from_xml(LexerFiles.get("/#{lexer_file_name}.xml").gets_to_end)
end
-
+
private def self.create_delegating_lexer(name : String) : BaseLexer
language, root = name.split("+", 2)
language_lexer = lexer(language)
root_lexer = lexer(root)
DelegatingLexer.new(language_lexer, root_lexer)
end
+
# Return a list of all lexers
def self.lexers : Array(String)
LEXERS_BY_NAME.keys.sort!