Nicer tokenize flow

This commit is contained in:
Roberto Alsina 2024-08-05 07:54:14 -03:00
parent da963e8a31
commit 03b81c2e54

View File

@ -67,14 +67,14 @@ module Tartrazine
Log.trace { "Stack is #{@state_stack} State is #{state.name}, pos is #{pos}, text is #{text[pos..pos + 10]}" }
state.rules.each do |rule|
matched, new_pos, new_tokens = rule.match(text, pos, self)
Log.trace { "NOT MATCHED: #{rule.xml}" }
next unless matched
if matched
Log.trace { "MATCHED: #{rule.xml}" }
pos = new_pos
tokens += new_tokens
break # We go back to processing with current state
end
Log.trace { "NOT MATCHED: #{rule.xml}" }
end
# If no rule matches, emit an error token
unless matched
Log.trace { "Error at #{pos}" }