ralsina/tartrazine
1
0
Fork 0
mirror of https://github.com/ralsina/tartrazine.git synced 2025-06-27 06:42:04 -03:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Beginning deserialization of data

Browse Source
This commit is contained in:
Roberto Alsina
2024-08-15 15:39:02 -03:00
parent 38196d6e96
commit 6264bfc754
8 changed files with 1065 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
src/actions.cr
View File

@ -23,17 +23,14 @@ module Tartrazine
struct Action
property actions : Array(Action) = [] of Action
@content_index : Array(Int32) = [] of Int32
@depth : Int32 = 0
@lexer_index : Int32 = 0
@lexer_name : String = ""
@states : Array(String) = [] of String
@states_to_push : Array(String) = [] of String
@token_type : String = ""
@type : ActionType = ActionType::Token
property token_type : String = ""
property states_to_push : Array(String) = [] of String
property depth = 0
property lexer_name : String = ""
property states_to_combine : Array(String) = [] of String
def initialize(t : String, xml : XML::Node?)
@type = ActionType.parse(t.capitalize)
def initialize(@type : String, @xml : XML::Node?)
# Extract information from the XML node we will use later
# Some actions may have actions in them, like this:
# <bygroups>
@ -44,30 +41,31 @@ module Tartrazine
#
# The token actions match with the first 2 groups in the regex
# the using action matches the 3rd and shunts it to another lexer
xml.children.each do |node|
known_types = %w(token push pop bygroups using usingself include combined)
raise Exception.new(
"Unknown action type: #{@type}") unless known_types.includes? @type
@xml.children.each do |node|
next unless node.element?
@actions << Action.new(node.name, node)
end
# Prefetch the attributes we ned from the XML and keep them
case @type
when ActionType::Token
@token_type = xml["type"]
when ActionType::Push
when "token"
@token_type = xml["type"]? || ""
when "push"
@states_to_push = xml.attributes.select { |attrib|
attrib.name == "state"
}.map &.content
when ActionType::Pop
@depth = xml["depth"].to_i
when ActionType::Using
@lexer_name = xml["lexer"].downcase
when ActionType::Combined
@states = xml.attributes.select { |attrib|
}.map &.content || [] of String
when "pop"
@depth = xml["depth"]?.try &.to_i || 0
when "using"
@lexer_name = xml["lexer"]?.try &.downcase || ""
when "combined"
@states_to_combine = xml.attributes.select { |attrib|
attrib.name == "state"
}.map &.content
when ActionType::Usingbygroup
@lexer_index = xml["lexer"].to_i
@content_index = xml["content"].split(",").map(&.to_i)
end
end
@ -77,21 +75,25 @@ module Tartrazine
when ActionType::Token
raise Exception.new "Can't have a token without a match" if match.empty?
[Token.new(type: @token_type, value: String.new(match[match_group].value))]
when ActionType::Push
to_push = @states_to_push.empty? ? [tokenizer.state_stack.last] : @states_to_push
to_push.each do |state|
if state == "#pop" && tokenizer.state_stack.size > 1
when "push"
if @states_to_push.empty?
# Push without a state means push the current state
@states_to_push = [lexer.state_stack.last]
end
@states_to_push.each do |state|
if state == "#pop"
# Pop the state
tokenizer.state_stack.pop
lexer.state_stack.pop
else
# Really push
tokenizer.state_stack << state
lexer.state_stack << state
end
end
[] of Token
when ActionType::Pop
to_pop = [@depth, tokenizer.state_stack.size - 1].min
tokenizer.state_stack.pop(to_pop)
when "pop"
if lexer.state_stack.size > @depth
lexer.state_stack.pop(@depth)
end
[] of Token
when ActionType::Bygroups
# FIXME: handle
@ -102,7 +104,7 @@ module Tartrazine
#
# where that None means skipping a group
#
raise Exception.new "Can't have a token without a match" if match.nil?
raise Exception.new "Can't have a token without a match" if match.empty?
# Each group matches an action. If the group match is empty,
# the action is skipped.
@ -111,8 +113,7 @@ module Tartrazine
begin
next if match[i + 1].size == 0
rescue IndexError
# FIXME: This should not actually happen
# No match for this group
# No match for the last group
next
end
result += e.emit(match, tokenizer, i + 1)
@ -121,19 +122,16 @@ module Tartrazine
when ActionType::Using
# Shunt to another lexer entirely
return [] of Token if match.empty?
Tartrazine.lexer(@lexer_name).tokenizer(
String.new(match[match_group].value),
secondary: true).to_a
when ActionType::Usingself
Tartrazine.lexer(@lexer_name).tokenize(String.new(match[match_group].value), usingself: true)
when "usingself"
# Shunt to another copy of this lexer
return [] of Token if match.empty?
tokenizer.lexer.tokenizer(
String.new(match[match_group].value),
secondary: true).to_a
when ActionType::Combined
# Combine two or more states into one anonymous state
new_state = @states.map { |name|
tokenizer.lexer.states[name]
new_lexer = Lexer.from_xml(lexer.xml)
new_lexer.tokenize(String.new(match[match_group].value), usingself: true)
when "combined"
# Combine two states into one anonymous state
new_state = @states_to_combine.map { |name|
lexer.states[name]
}.reduce { |state1, state2|
state1 + state2
}
@ -151,7 +149,7 @@ module Tartrazine
content,
secondary: true).to_a
else
raise Exception.new("Unknown action type: #{@type}")
raise Exception.new("Unhandled action type: #{type}")
end
end
end

6
src/bytes_regex.cr
View File

@ -31,6 +31,7 @@ module BytesRegex
end
def match(str : Bytes, pos = 0) : Array(Match)
match = [] of Match
rc = LibPCRE2.match(
@re,
str,
@ -39,9 +40,9 @@ module BytesRegex
LibPCRE2::NO_UTF_CHECK,
@match_data,
nil)
if rc > 0
if rc >= 0
ovector = LibPCRE2.get_ovector_pointer(@match_data)
(0...rc).map do |i|
(0...rc).each do |i|
m_start = ovector[2 * i]
m_end = ovector[2 * i + 1]
if m_start == m_end
@ -54,6 +55,7 @@ module BytesRegex
else
[] of Match
end
match
end
end

42
src/heuristics.cr Normal file
View File

@ -0,0 +1,42 @@
require "yaml"
module Tartrazine
# Use linguist's heuristics to disambiguate between languages
class Heuristic
include YAML::Serializable
property disambiguations : Array(Disambiguation)
property named_patterns : Hash(String, String | Array(String))
# Run the heuristics on the given filename and content
def run(filename, content)
ext = File.extname filename
disambiguation = disambiguations.find do |item|
item.extensions.includes? ext
end
p! disambiguation
end
end
class Disambiguation
include YAML::Serializable
property extensions : Array(String)
property rules : Array(LangRule)
end
class Rule
include YAML::Serializable
property pattern : (String | Array(String))?
property named_pattern : String?
property and : Array(Rule)?
end
class LangRule < Rule
include YAML::Serializable
property language : String | Array(String)
end
end
h = Tartrazine::Heuristic.from_yaml(File.read("heuristics/heuristics.yml"))
h.run("../elkjs/src/elk.h", File.read("../elkjs/src/elk.h"))

13
src/main.cr
View File

@ -1,6 +1,18 @@
require "docopt"
require "./**"
# Performance data (in milliseconds):
#
# Docopt parsing: 0.5
# Instantiating a theme: 0.1
# Instantiating a formatter: 1.0
# Instantiating a lexer: 2.0
# Tokenizing crycco.cr: 16.0
# Formatting: 0.5
# I/O: 1.5
# ---------------------------------
# Total: 21.6
HELP = <<-HELP
tartrazine: a syntax highlighting tool
@ -84,7 +96,6 @@ if options["-f"]
end
lexer = Tartrazine.lexer(name: options["-l"].as(String), filename: options["FILE"].as(String))
input = File.open(options["FILE"].as(String)).gets_to_end
if options["-o"].nil?

31
src/rules.cr
View File

@ -15,11 +15,28 @@ module Tartrazine
alias Match = BytesRegex::Match
alias MatchData = Array(Match)
abstract struct BaseRule
abstract def match(text : Bytes, pos : Int32, tokenizer : Tokenizer) : Tuple(Bool, Int32, Array(Token))
abstract def initialize(node : XML::Node)
class Rule
property pattern : Regex = Regex.new ""
property actions : Array(Action) = [] of Action
@actions : Array(Action) = [] of Action
def match(text : Bytes, pos, lexer) : Tuple(Bool, Int32, Array(Token))
match = pattern.match(text, pos)
# We don't match if the match doesn't move the cursor
# because that causes infinite loops
return false, pos, [] of Token if match.empty? || match[0].size == 0
tokens = [] of Token
actions.each do |action|
tokens += action.emit(match, lexer)
end
return true, pos + match[0].size, tokens
end
def initialize(node : XML::Node, multiline, dotall, ignorecase)
pattern = node["pattern"]
pattern = "(?m)" + pattern if multiline
@pattern = Regex.new(pattern, multiline, dotall, ignorecase, true)
add_actions(node)
end
def add_actions(node : XML::Node)
node.children.each do |child|
@ -56,9 +73,9 @@ module Tartrazine
struct IncludeStateRule < BaseRule
@state : String = ""
def match(text : Bytes, pos : Int32, tokenizer : Tokenizer) : Tuple(Bool, Int32, Array(Token))
tokenizer.@lexer.states[@state].rules.each do |rule|
matched, new_pos, new_tokens = rule.match(text, pos, tokenizer)
def match(text, pos, lexer) : Tuple(Bool, Int32, Array(Token))
lexer.states[state].rules.each do |rule|
matched, new_pos, new_tokens = rule.match(text, pos, lexer)
return true, new_pos, new_tokens if matched
end
return false, pos, [] of Token