ralsina/tartrazine
1
0
Fork 0
mirror of https://github.com/ralsina/tartrazine.git synced 2024-11-10 05:22:23 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Beginning deserialization of data

Browse Source
This commit is contained in:
Roberto Alsina 2024-08-15 15:39:02 -03:00
parent 38196d6e96
commit 6264bfc754
8 changed files with 1065 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
TODO.md
View File

@ -9,7 +9,4 @@
* ✅ Implement lexer loader by file extension
* ✅ Add --line-numbers to terminal formatter
* Implement lexer loader by mime type
* ✅ Implement Delegating lexers
* ✅ Add RstLexer
* Add Mako template lexer
* Implement heuristic lexer detection
* Implement Pygment's "DelegateLexer"

22
heuristics/LICENSE.txt Normal file
View File

@ -0,0 +1,22 @@
Copyright (c) 2017 GitHub, Inc.
Permission is hereby granted, free of charge, to any person
obtaining a copy of this software and associated documentation
files (the "Software"), to deal in the Software without
restriction, including without limitation the rights to use,
copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following
conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.

913
heuristics/heuristics.yml Normal file
View File

@ -0,0 +1,913 @@
# A collection of simple regexp-based rules that can be applied to content
# to disambiguate languages with the same file extension.
#
# There are two top-level keys: disambiguations and named_patterns.
#
# disambiguations - a list of disambiguation rules, one for each
# extension or group of extensions.
# extensions - an array of file extensions that this block applies to.
# rules - list of rules that are applied in order to the content
# of a file with a matching extension. Rules are evaluated
# until one of them matches. If none matches, no language
# is returned.
# language - Language to be returned if the rule matches.
# pattern - Ruby-compatible regular expression that makes the rule
# match. If no pattern is specified, the rule always matches.
# Pattern can be a string with a single regular expression
# or an array of strings that will be merged in a single
# regular expression (with union).
# and - An and block merges multiple rules and checks that all of
# them must match.
# negative_pattern - Same as pattern, but checks for absence of matches.
# named_pattern - A pattern can be reused by specifying it in the
# named_patterns section and referencing it here by its
# key.
# named_patterns - Key-value map of reusable named patterns.
#
# Please keep this list alphabetized.
#
---
disambiguations:
- extensions: ['.1', '.2', '.3', '.4', '.5', '.6', '.7', '.8', '.9']
rules:
- language: Roff Manpage
and:
- named_pattern: mdoc-date
- named_pattern: mdoc-title
- named_pattern: mdoc-heading
- language: Roff Manpage
and:
- named_pattern: man-title
- named_pattern: man-heading
- language: Roff
pattern: '^\.(?:[A-Za-z]{2}(?:\s|$)|\\")'
- extensions: ['.1in', '.1m', '.1x', '.3in', '.3m', '.3p', '.3pm', '.3qt', '.3x', '.man', '.mdoc']
rules:
- language: Roff Manpage
and:
- named_pattern: mdoc-date
- named_pattern: mdoc-title
- named_pattern: mdoc-heading
- language: Roff Manpage
and:
- named_pattern: man-title
- named_pattern: man-heading
- language: Roff
- extensions: ['.al']
rules:
# AL pattern source from https://github.com/microsoft/AL/blob/master/grammar/alsyntax.tmlanguage - keyword.other.applicationobject.al
- language: AL
and:
- pattern: '\b(?i:(CODEUNIT|PAGE|PAGEEXTENSION|PAGECUSTOMIZATION|DOTNET|ENUM|ENUMEXTENSION|VALUE|QUERY|REPORT|TABLE|TABLEEXTENSION|XMLPORT|PROFILE|CONTROLADDIN|REPORTEXTENSION|INTERFACE|PERMISSIONSET|PERMISSIONSETEXTENSION|ENTITLEMENT))\b'
# Open-ended fallback to Perl AutoLoader
- language: Perl
- extensions: ['.app']
rules:
- language: Erlang
pattern: '^\{\s*(?:application|''application'')\s*,\s*(?:[a-z]+[\w@]*|''[^'']+'')\s*,\s*\[(?:.|[\r\n])*\]\s*\}\.[ \t]*$'
- extensions: ['.as']
rules:
- language: ActionScript
pattern: '^\s*(?:package(?:\s+[\w.]+)?\s+(?:\{|$)|import\s+[\w.*]+\s*;|(?=.*?(?:intrinsic|extends))(intrinsic\s+)?class\s+[\w<>.]+(?:\s+extends\s+[\w<>.]+)?|(?:(?:public|protected|private|static)\s+)*(?:(?:var|const|local)\s+\w+\s*:\s*[\w<>.]+(?:\s*=.*)?\s*;|function\s+\w+\s*\((?:\s*\w+\s*:\s*[\w<>.]+\s*(,\s*\w+\s*:\s*[\w<>.]+\s*)*)?\)))'
- extensions: ['.asc']
rules:
- language: Public Key
pattern: '^(----[- ]BEGIN|ssh-(rsa|dss)) '
- language: AsciiDoc
pattern: '^[=-]+\s|\{\{[A-Za-z]'
- language: AGS Script
pattern: '^(\/\/.+|((import|export)\s+)?(function|int|float|char)\s+((room|repeatedly|on|game)_)?([A-Za-z]+[A-Za-z_0-9]+)\s*[;\(])'
- extensions: ['.asm']
rules:
- language: Motorola 68K Assembly
named_pattern: m68k
- extensions: ['.asy']
rules:
- language: LTspice Symbol
pattern: '^SymbolType[ \t]'
- language: Asymptote
- extensions: ['.bas']
rules:
- language: FreeBasic
pattern: '^[ \t]*#(?i)(?:define|endif|endmacro|ifn?def|include|lang|macro)(?:$|\s)'
- language: BASIC
pattern: '\A\s*\d'
- language: VBA
and:
- named_pattern: vb-module
- named_pattern: vba
- language: Visual Basic 6.0
named_pattern: vb-module
- extensions: ['.bb']
rules:
- language: BlitzBasic
pattern: '(<^\s*; |End Function)'
- language: BitBake
pattern: '^(# |include|require|inherit)\b'
- language: Clojure
pattern: '\((def|defn|defmacro|let)\s'
- extensions: ['.bf']
rules:
- language: Beef
pattern: '(?-m)^\s*using\s+(System|Beefy)(\.(.*))?;\s*$'
- language: HyPhy
pattern:
- '(?-m)^\s*#include\s+".*";\s*$'
- '\sfprintf\s*\('
- language: Brainfuck
pattern: '(>\+>|>\+<)'
- extensions: ['.bi']
rules:
- language: FreeBasic
pattern: '^[ \t]*#(?i)(?:define|endif|endmacro|ifn?def|if|include|lang|macro)(?:$|\s)'
- extensions: ['.bs']
rules:
- language: Bikeshed
pattern: '^(?i:<pre\s+class)\s*=\s*(''|\"|\b)metadata\b\1[^>\r\n]*>'
- language: BrighterScript
pattern:
- (?i:^\s*(?=^sub\s)(?:sub\s*\w+\(.*?\))|(?::\s*sub\(.*?\))$)
- (?i:^\s*(end\ssub)$)
- (?i:^\s*(?=^function\s)(?:function\s*\w+\(.*?\)\s*as\s*\w*)|(?::\s*function\(.*?\)\s*as\s*\w*)$)
- (?i:^\s*(end\sfunction)$)
- language: Bluespec BH
pattern: '^package\s+[A-Za-z_][A-Za-z0-9_'']*(?:\s*\(|\s+where)'
- extensions: ['.builds']
rules:
- language: XML
pattern: '^(\s*)(?i:<Project|<Import|<Property|<?xml|xmlns)'
- extensions: ['.ch']
rules:
- language: xBase
pattern: '^\s*#\s*(?i:if|ifdef|ifndef|define|command|xcommand|translate|xtranslate|include|pragma|undef)\b'
- extensions: ['.cl']
rules:
- language: Common Lisp
pattern: '^\s*\((?i:defun|in-package|defpackage) '
- language: Cool
pattern: '^class'
- language: OpenCL
pattern: '\/\* |\/\/ |^\}'
- extensions: ['.cls']
rules:
- language: Visual Basic 6.0
and:
- named_pattern: vb-class
- pattern: '^\s*BEGIN(?:\r?\n|\r)\s*MultiUse\s*=.*(?:\r?\n|\r)\s*Persistable\s*='
- language: VBA
named_pattern: vb-class
- language: TeX
pattern: '^\s*\\(?:NeedsTeXFormat|ProvidesClass)\{'
- language: ObjectScript
pattern: '^Class\s'
- extensions: ['.cmp']
rules:
- language: Gerber Image
pattern: '^[DGMT][0-9]{2}\*(?:\r?\n|\r)'
- extensions: ['.cs']
rules:
- language: Smalltalk
pattern: '![\w\s]+methodsFor: '
- language: 'C#'
pattern: '^\s*(using\s+[A-Z][\s\w.]+;|namespace\s*[\w\.]+\s*(\{|;)|\/\/)'
- extensions: ['.csc']
rules:
- language: GSC
named_pattern: gsc
- extensions: ['.csl']
rules:
- language: XML
pattern: '(?i:^\s*(<\?xml|xmlns))'
- language: Kusto
pattern: '(^\|\s*(where|extend|project|limit|summarize))|(^\.\w+)'
- extensions: ['.d']
rules:
- language: D
# see http://dlang.org/spec/grammar
# ModuleDeclaration | ImportDeclaration | FuncDeclaration | unittest
pattern: '^module\s+[\w.]*\s*;|import\s+[\w\s,.:]*;|\w+\s+\w+\s*\(.*\)(?:\(.*\))?\s*\{[^}]*\}|unittest\s*(?:\(.*\))?\s*\{[^}]*\}'
- language: DTrace
# see http://dtrace.org/guide/chp-prog.html, http://dtrace.org/guide/chp-profile.html, http://dtrace.org/guide/chp-opt.html
pattern: '^(\w+:\w*:\w*:\w*|BEGIN|END|provider\s+|(tick|profile)-\w+\s+\{[^}]*\}|#pragma\s+D\s+(option|attributes|depends_on)\s|#pragma\s+ident\s)'
- language: Makefile
# path/target : dependency \
# target : \
# : dependency
# path/file.ext1 : some/path/../file.ext2
pattern: '([\/\\].*:\s+.*\s\\$|: \\$|^[ %]:|^[\w\s\/\\.]+\w+\.\w+\s*:\s+[\w\s\/\\.]+\w+\.\w+)'
- extensions: ['.dsp']
rules:
- language: Microsoft Developer Studio Project
pattern: '# Microsoft Developer Studio Generated Build File'
- language: Faust
pattern: '\bprocess\s*[(=]|\b(library|import)\s*\(\s*"|\bdeclare\s+(name|version|author|copyright|license)\s+"'
- extensions: ['.e']
rules:
- language: E
pattern:
- '^\s*(def|var)\s+(.+):='
- '^\s*(def|to)\s+(\w+)(\(.+\))?\s+\{'
- '^\s*(when)\s+(\(.+\))\s+->\s+\{'
- language: Eiffel
pattern:
- '^\s*\w+\s*(?:,\s*\w+)*[:]\s*\w+\s'
- '^\s*\w+\s*(?:\(\s*\w+[:][^)]+\))?(?:[:]\s*\w+)?(?:--.+\s+)*\s+(?:do|local)\s'
- '^\s*(?:across|deferred|elseif|ensure|feature|from|inherit|inspect|invariant|note|once|require|undefine|variant|when)\s*$'
- language: Euphoria
named_pattern: euphoria
- extensions: ['.ecl']
rules:
- language: ECLiPSe
pattern: '^[^#]+:-'
- language: ECL
pattern: ':='
- extensions: ['.es']
rules:
- language: Erlang
pattern: '^\s*(?:%%|main\s*\(.*?\)\s*->)'
- language: JavaScript
pattern: '\/\/|("|'')use strict\1|export\s+default\s|\/\*(?:.|[\r\n])*?\*\/'
- extensions: ['.ex']
rules:
- language: Elixir
pattern:
- '^\s*@moduledoc\s'
- '^\s*(?:cond|import|quote|unless)\s'
- '^\s*def(?:exception|impl|macro|module|protocol)[(\s]'
- language: Euphoria
named_pattern: euphoria
- extensions: ['.f']
rules:
- language: Forth
pattern: '^: '
- language: Filebench WML
pattern: 'flowop'
- language: Fortran
named_pattern: fortran
- extensions: ['.for']
rules:
- language: Forth
pattern: '^: '
- language: Fortran
named_pattern: fortran
- extensions: ['.fr']
rules:
- language: Forth
pattern: '^(: |also |new-device|previous )'
- language: Frege
pattern: '^\s*(import|module|package|data|type) '
- language: Text
- extensions: ['.frm']
rules:
- language: VBA
and:
- named_pattern: vb-form
- pattern: '^\s*Begin\s+\{[0-9A-Z\-]*\}\s?'
- language: Visual Basic 6.0
and:
- named_pattern: vb-form
- pattern: '^\s*Begin\s+VB\.Form\s+'
- extensions: ['.fs']
rules:
- language: Forth
pattern: '^(: |new-device)'
- language: 'F#'
pattern: '^\s*(#light|import|let|module|namespace|open|type)'
- language: GLSL
pattern: '^\s*(#version|precision|uniform|varying|vec[234])'
- language: Filterscript
pattern: '#include|#pragma\s+(rs|version)|__attribute__'
- extensions: ['.ftl']
rules:
- language: FreeMarker
pattern: '^(?:<|[a-zA-Z-][a-zA-Z0-9_-]+[ \t]+\w)|\$\{\w+[^\r\n]*?\}|^[ \t]*(?:<#--.*?-->|<#([a-z]+)(?=\s|>)[^>]*>.*?</#\1>|\[#--.*?--\]|\[#([a-z]+)(?=\s|\])[^\]]*\].*?\[#\2\])'
- language: Fluent
pattern: '^-?[a-zA-Z][a-zA-Z0-9_-]* *=|\{\$-?[a-zA-Z][-\w]*(?:\.[a-zA-Z][-\w]*)?\}'
- extensions: ['.g']
rules:
- language: GAP
pattern: '\s*(Declare|BindGlobal|KeyDependentOperation|Install(Method|GlobalFunction)|SetPackageInfo)'
- language: G-code
pattern: '^[MG][0-9]+(?:\r?\n|\r)'
- extensions: ['.gd']
rules:
- language: GAP
pattern: '\s*(Declare|BindGlobal|KeyDependentOperation)'
- language: GDScript
pattern: '\s*(extends|var|const|enum|func|class|signal|tool|yield|assert|onready)'
- extensions: ['.gml']
rules:
- language: XML
pattern: '(?i:^\s*(<\?xml|xmlns))'
- language: Graph Modeling Language
pattern: '(?i:^\s*(graph|node)\s+\[$)'
- language: Gerber Image
pattern: '^[DGMT][0-9]{2}\*$'
- language: Game Maker Language
- extensions: ['.gs']
rules:
- language: GLSL
pattern: '^#version\s+[0-9]+\b'
- language: Gosu
pattern: '^uses (java|gw)\.'
- language: Genie
pattern: '^\[indent=[0-9]+\]'
- extensions: ['.gsc']
rules:
- language: GSC
named_pattern: gsc
- extensions: ['.gsh']
rules:
- language: GSC
named_pattern: gsc
- extensions: ['.gts']
rules:
- language: Gerber Image
pattern: '^G0.'
- language: Glimmer TS
negative_pattern: '^G0.'
- extensions: ['.h']
rules:
- language: Objective-C
named_pattern: objectivec
- language: C++
named_pattern: cpp
- language: C
- extensions: ['.hh']
rules:
- language: Hack
pattern: '<\?hh'
- extensions: ['.html']
rules:
- language: Ecmarkup
pattern: '<emu-(?:alg|annex|biblio|clause|eqn|example|figure|gann|gmod|gprose|grammar|intro|not-ref|note|nt|prodref|production|rhs|table|t|xref)(?:$|\s|>)'
- language: HTML
- extensions: ['.i']
rules:
- language: Motorola 68K Assembly
named_pattern: m68k
- language: SWIG
pattern: '^[ \t]*%[a-z_]+\b|^%[{}]$'
- extensions: ['.ice']
rules:
- language: JSON
pattern: '\A\s*[{\[]'
- language: Slice
- extensions: ['.inc']
rules:
- language: Motorola 68K Assembly
named_pattern: m68k
- language: PHP
pattern: '^<\?(?:php)?'
- language: SourcePawn
pattern:
- '^public\s+(?:SharedPlugin(?:\s+|:)__pl_\w+\s*=(?:\s*\{)?|(?:void\s+)?__pl_\w+_SetNTVOptional\(\)(?:\s*\{)?)'
- '^methodmap\s+\w+\s+<\s+\w+'
- '^\s*MarkNativeAsOptional\s*\('
- language: NASL
pattern:
- '^\s*include\s*\(\s*(?:"|'')[\\/\w\-\.:\s]+\.(?:nasl|inc)\s*(?:"|'')\s*\)\s*;'
- '^\s*(?:global|local)_var\s+(?:\w+(?:\s*=\s*[\w\-"'']+)?\s*)(?:,\s*\w+(?:\s*=\s*[\w\-"'']+)?\s*)*+\s*;'
- '^\s*namespace\s+\w+\s*\{'
- '^\s*object\s+\w+\s*(?:extends\s+\w+(?:::\w+)?)?\s*\{'
- '^\s*(?:public\s+|private\s+|\s*)function\s+\w+\s*\([\w\s,]*\)\s*\{'
- language: POV-Ray SDL
pattern: '^\s*#(declare|local|macro|while)\s'
- language: Pascal
pattern:
- '(?i:^\s*\{\$(?:mode|ifdef|undef|define)[ ]+[a-z0-9_]+\})'
- '^\s*end[.;]\s*$'
- language: BitBake
pattern: '^inherit(\s+[\w.-]+)+\s*$'
- extensions: ['.json']
rules:
- language: OASv2-json
pattern: '"swagger":\s?"2.[0-9.]+"'
- language: OASv3-json
pattern: '"openapi":\s?"3.[0-9.]+"'
- language: JSON
- extensions: ['.l']
rules:
- language: Common Lisp
pattern: '\(def(un|macro)\s'
- language: Lex
pattern: '^(%[%{}]xs|<.*>)'
- language: Roff
pattern: '^\.[A-Za-z]{2}(\s|$)'
- language: PicoLisp
pattern: '^\((de|class|rel|code|data|must)\s'
- extensions: ['.lean']
rules:
- language: Lean
pattern: '^import [a-z]'
- language: Lean 4
pattern: '^import [A-Z]'
- extensions: ['.ls']
rules:
- language: LoomScript
pattern: '^\s*package\s*[\w\.\/\*\s]*\s*\{'
- language: LiveScript
- extensions: ['.lsp', '.lisp']
rules:
- language: Common Lisp
pattern: '^\s*\((?i:defun|in-package|defpackage) '
- language: NewLisp
pattern: '^\s*\(define '
- extensions: ['.m']
rules:
- language: Objective-C
named_pattern: objectivec
- language: Mercury
pattern: ':- module'
- language: MUF
pattern: '^: '
- language: M
pattern: '^\s*;'
- language: Mathematica
and:
- pattern: '\(\*'
- pattern: '\*\)$'
- language: MATLAB
pattern: '^\s*%'
- language: Limbo
pattern: '^\w+\s*:\s*module\s*\{'
- extensions: ['.m4']
rules:
- language: M4Sugar
pattern:
- 'AC_DEFUN|AC_PREREQ|AC_INIT'
- '^_?m4_'
- language: 'M4'
- extensions: ['.mask']
rules:
- language: Unity3D Asset
pattern: 'tag:unity3d.com'
- extensions: ['.mc']
rules:
- language: Win32 Message File
pattern: '(?i)^[ \t]*(?>\/\*\s*)?MessageId=|^\.$'
- language: M4
pattern: '^dnl|^divert\((?:-?\d+)?\)|^\w+\(`[^\r\n]*?''[),]'
- language: Monkey C
pattern: '\b(?:using|module|function|class|var)\s+\w'
- extensions: ['.md']
rules:
- language: Markdown
pattern:
- '(^[-A-Za-z0-9=#!\*\[|>])|<\/'
- '\A\z'
- language: GCC Machine Description
pattern: '^(;;|\(define_)'
- language: Markdown
- extensions: ['.ml']
rules:
- language: OCaml
pattern: '(^\s*module)|let rec |match\s+(\S+\s)+with'
- language: Standard ML
pattern: '=> |case\s+(\S+\s)+of'
- extensions: ['.mod']
rules:
- language: XML
pattern: '<!ENTITY '
- language: NMODL
pattern: '\b(NEURON|INITIAL|UNITS)\b'
- language: Modula-2
pattern: '^\s*(?i:MODULE|END) [\w\.]+;'
- language: [Linux Kernel Module, AMPL]
- extensions: ['.mojo']
rules:
- language: Mojo
pattern: '^\s*(alias|def|from|fn|import|struct|trait)\s'
- language: XML
pattern: '^\s*<\?xml'
- extensions: ['.ms']
rules:
- language: Roff
pattern: '^[.''][A-Za-z]{2}(\s|$)'
- language: Unix Assembly
and:
- negative_pattern: '/\*'
- pattern: '^\s*\.(?:include\s|globa?l\s|[A-Za-z][_A-Za-z0-9]*:)'
- language: MAXScript
- extensions: ['.n']
rules:
- language: Roff
pattern: '^[.'']'
- language: Nemerle
pattern: '^(module|namespace|using)\s'
- extensions: ['.ncl']
rules:
- language: XML
pattern: '^\s*<\?xml\s+version'
- language: Gerber Image
pattern: '^[DGMT][0-9]{2}\*(?:\r?\n|\r)'
- language: Text
pattern: 'THE_TITLE'
- extensions: ['.nl']
rules:
- language: NL
pattern: '^(b|g)[0-9]+ '
- language: NewLisp
- extensions: ['.nu']
rules:
- language: Nushell
pattern: '^\s*(import|export|module|def|let|let-env) '
- language: Nu
- extensions: ['.odin']
rules:
- language: Object Data Instance Notation
pattern: '(?:^|<)\s*[A-Za-z0-9_]+\s*=\s*<'
- language: Odin
pattern: 'package\s+\w+|\b(?:im|ex)port\s*"[\w:./]+"|\w+\s*::\s*(?:proc|struct)\s*\(|^\s*//\s'
- extensions: ['.p']
rules:
- language: Gnuplot
pattern:
- '^s?plot\b'
- '^set\s+(term|terminal|out|output|[xy]tics|[xy]label|[xy]range|style)\b'
- language: OpenEdge ABL
- extensions: ['.php']
rules:
- language: Hack
pattern: '<\?hh'
- language: PHP
pattern: '<\?[^h]'
- extensions: ['.pkl']
rules:
- language: Pkl
pattern:
- '^\s*(module|import|amends|extends|local|const|fixed|abstract|open|class|typealias|@\w+)\b'
- '^\s*[a-zA-Z0-9_$]+\s*(=|{|:)|^\s*`[^`]+`\s*(=|{|:)|for\s*\(|when\s*\('
- language: Pickle
- extensions: ['.pl']
rules:
- language: Prolog
pattern: '^[^#]*:-'
- language: Perl
and:
- negative_pattern: '^\s*use\s+v6\b'
- named_pattern: perl
- language: Raku
named_pattern: raku
- extensions: ['.plist']
rules:
- language: XML Property List
pattern: '^\s*(?:<\?xml\s|<!DOCTYPE\s+plist|<plist(?:\s+version\s*=\s*(["''])\d+(?:\.\d+)?\1)?\s*>\s*$)'
- language: OpenStep Property List
- extensions: ['.plt']
rules:
- language: Prolog
pattern: '^\s*:-'
- extensions: ['.pm']
rules:
- language: Perl
and:
- negative_pattern: '^\s*use\s+v6\b'
- named_pattern: perl
- language: Raku
named_pattern: raku
- language: X PixMap
pattern: '^\s*\/\* XPM \*\/'
- extensions: ['.pod']
rules:
- language: Pod 6
pattern: '^[\s&&[^\r\n]]*=(comment|begin pod|begin para|item\d+)'
- language: Pod
- extensions: ['.pp']
rules:
- language: Pascal
pattern: '^\s*end[.;]'
- language: Puppet
pattern: '^\s+\w+\s+=>\s'
- extensions: ['.pro']
rules:
- language: Proguard
pattern: '^-(include\b.*\.pro$|keep\b|keepclassmembers\b|keepattributes\b)'
- language: Prolog
pattern: '^[^\[#]+:-'
- language: INI
pattern: 'last_client='
- language: QMake
and:
- pattern: HEADERS
- pattern: SOURCES
- language: IDL
pattern: '^\s*(?i:function|pro|compile_opt) \w[ \w,:]*$'
- extensions: ['.properties']
rules:
- language: INI
and:
- named_pattern: key_equals_value
- pattern: '^[;\[]'
- language: Java Properties
and:
- named_pattern: key_equals_value
- pattern: '^[#!]'
- language: INI
named_pattern: key_equals_value
- language: Java Properties
pattern: '^[^#!][^:]*:'
- extensions: ['.q']
rules:
- language: q
pattern: '((?i:[A-Z.][\w.]*:\{)|^\\(cd?|d|l|p|ts?) )'
- language: HiveQL
pattern: '(?i:SELECT\s+[\w*,]+\s+FROM|(CREATE|ALTER|DROP)\s(DATABASE|SCHEMA|TABLE))'
- extensions: ['.qs']
rules:
- language: Q#
pattern: '^((\/{2,3})?\s*(namespace|operation)\b)'
- language: Qt Script
pattern: '(\w+\.prototype\.\w+|===|\bvar\b)'
- extensions: ['.r']
rules:
- language: Rebol
pattern: '(?i:\bRebol\b)'
- language: Rez
pattern: '(#include\s+["<](Types\.r|Carbon\/Carbon\.r)[">])|((resource|data|type)\s+''[A-Za-z0-9]{4}''\s+((\(.*\)\s+){0,1}){)'
- language: R
pattern: '<-|^\s*#'
- extensions: ['.re']
rules:
- language: Reason
pattern:
- '^\s*module\s+type\s'
- '^\s*(?:include|open)\s+\w+\s*;\s*$'
- '^\s*let\s+(?:module\s\w+\s*=\s*\{|\w+:\s+.*=.*;\s*$)'
- language: C++
pattern:
- '^\s*#(?:(?:if|ifdef|define|pragma)\s+\w|\s*include\s+<[^>]+>)'
- '^\s*template\s*<'
- extensions: ['.res']
rules:
- language: ReScript
pattern:
- '^\s*(let|module|type)\s+\w*\s+=\s+'
- '^\s*(?:include|open)\s+\w+\s*$'
- extensions: ['.rno']
rules:
- language: RUNOFF
pattern: '(?i:^\.!|^\f|\f$|^\.end lit(?:eral)?\b|^\.[a-zA-Z].*?;\.[a-zA-Z](?:[; \t])|\^\*[^\s*][^*]*\\\*(?=$|\s)|^\.c;[ \t]*\w+)'
- language: Roff
pattern: '^\.\\" '
- extensions: ['.rpy']
rules:
- language: Python
pattern: '^(import|from|class|def)\s'
- language: "Ren'Py"
- extensions: ['.rs']
rules:
- language: Rust
pattern: '^(use |fn |mod |pub |macro_rules|impl|#!?\[)'
- language: RenderScript
pattern: '#include|#pragma\s+(rs|version)|__attribute__'
- language: XML
pattern: '^\s*<\?xml'
- extensions: ['.s']
rules:
- language: Motorola 68K Assembly
named_pattern: m68k
- extensions: ['.sc']
rules:
- language: SuperCollider
pattern: '(?i:\^(this|super)\.|^\s*~\w+\s*=\.)'
- language: Scala
pattern: '(^\s*import (scala|java)\.|^\s*class\b)'
- extensions: ['.scd']
rules:
- language: SuperCollider
pattern: '(?i:\^(this|super)\.|^\s*(~\w+\s*=\.|SynthDef\b))'
- language: Markdown
# Markdown syntax for scdoc
pattern: '^#+\s+(NAME|SYNOPSIS|DESCRIPTION)'
- extensions: ['.sol']
rules:
- language: Solidity
pattern: '\bpragma\s+solidity\b|\b(?:abstract\s+)?contract\s+(?!\d)[a-zA-Z0-9$_]+(?:\s+is\s+(?:[a-zA-Z0-9$_][^\{]*?)?)?\s*\{'
- language: Gerber Image
pattern: '^[DGMT][0-9]{2}\*(?:\r?\n|\r)'
- extensions: ['.sql']
rules:
# Postgres
- language: PLpgSQL
pattern: '(?i:^\\i\b|AS\s+\$\$|LANGUAGE\s+''?plpgsql''?|BEGIN(\s+WORK)?\s*;)'
# IBM db2
- language: SQLPL
pattern: '(?i:ALTER\s+MODULE|MODE\s+DB2SQL|\bSYS(CAT|PROC)\.|ASSOCIATE\s+RESULT\s+SET|\bEND!\s*$)'
# Oracle
- language: PLSQL
pattern: '(?i:\$\$PLSQL_|XMLTYPE|systimestamp|\.nextval|CONNECT\s+BY|AUTHID\s+(DEFINER|CURRENT_USER)|constructor\W+function)'
# T-SQL
- language: TSQL
pattern: '(?i:^\s*GO\b|BEGIN(\s+TRY|\s+CATCH)|OUTPUT\s+INSERTED|DECLARE\s+@|\[dbo\])'
- language: SQL
- extensions: ['.srt']
rules:
- language: SubRip Text
pattern: '^(\d{2}:\d{2}:\d{2},\d{3})\s*(-->)\s*(\d{2}:\d{2}:\d{2},\d{3})$'
- extensions: ['.st']
rules:
- language: StringTemplate
pattern: '\$\w+[($]|(.)!\s*.+?\s*!\1|<!\s*.+?\s*!>|\[!\s*.+?\s*!\]|\{!\s*.+?\s*!\}'
- language: Smalltalk
pattern: '\A\s*[\[{(^"''\w#]|[a-zA-Z_]\w*\s*:=\s*[a-zA-Z_]\w*|class\s*>>\s*[a-zA-Z_]\w*|^[a-zA-Z_]\w*\s+[a-zA-Z_]\w*:|^Class\s*\{|if(?:True|False):\s*\['
- extensions: ['.star']
rules:
- language: STAR
pattern: '^loop_\s*$'
- language: Starlark
- extensions: ['.stl']
rules:
- language: STL
pattern: '\A\s*solid(?:$|\s)[\s\S]*^endsolid(?:$|\s)'
- extensions: ['.sw']
rules:
- language: Sway
pattern: '^\s*(?:(?:abi|dep|fn|impl|mod|pub|trait)\s|#\[)'
- language: XML
pattern: '^\s*<\?xml\s+version'
- extensions: ['.t']
rules:
- language: Perl
and:
- negative_pattern: '^\s*use\s+v6\b'
- named_pattern: perl
- language: Raku
pattern: '^\s*(?:use\s+v6\b|\bmodule\b|\bmy\s+class\b)'
- language: Turing
pattern: '^\s*%[ \t]+|^\s*var\s+\w+(\s*:\s*\w+)?\s*:=\s*\w+'
- extensions: ['.tag']
rules:
- language: Java Server Pages
pattern: '<%[@!=\s]?\s*(taglib|tag|include|attribute|variable)\s'
- extensions: ['.tlv']
rules:
- language: TL-Verilog
pattern: '^\\.{0,10}TLV_version'
- extensions: ['.toc']
rules:
- language: World of Warcraft Addon Data
pattern: '^## |@no-lib-strip@'
- language: TeX
pattern: '^\\(contentsline|defcounter|beamer|boolfalse)'
- extensions: ['.ts']
rules:
- language: XML
pattern: '<TS\b'
- language: TypeScript
- extensions: ['.tst']
rules:
- language: GAP
pattern: 'gap> '
# Heads up - we don't usually write heuristics like this (with no regex match)
- language: Scilab
- extensions: ['.tsx']
rules:
- language: TSX
pattern: '^\s*(import.+(from\s+|require\()[''"]react|\/\/\/\s*<reference\s)'
- language: XML
pattern: '(?i:^\s*<\?xml\s+version)'
- extensions: ['.txt']
rules:
# The following RegExp is simply a collapsed and simplified form of the
# VIM_MODELINE pattern in `./lib/linguist/strategy/modeline.rb`.
- language: Vim Help File
pattern: '(?:(?:^|[ \t])(?:vi|Vi(?=m))(?:m[<=>]?[0-9]+|m)?|[ \t]ex)(?=:(?=[ \t]*set?[ \t][^\r\n:]+:)|:(?![ \t]*set?[ \t]))(?:(?:[ \t]*:[ \t]*|[ \t])\w*(?:[ \t]*=(?:[^\\\s]|\\.)*)?)*[ \t:](?:filetype|ft|syntax)[ \t]*=(help)(?=$|\s|:)'
- language: Adblock Filter List
pattern: |-
(?x)\A
\[
(?<version>
(?:
[Aa]d[Bb]lock
(?:[ \t][Pp]lus)?
|
u[Bb]lock
(?:[ \t][Oo]rigin)?
|
[Aa]d[Gg]uard
)
(?:[ \t] \d+(?:\.\d+)*+)?
)
(?:
[ \t]?;[ \t]?
\g<version>
)*+
\]
# HACK: This is a contrived use of heuristics needed to address
# an unusual edge-case. See https://git.io/JULye for discussion.
- language: Text
- extensions: ['.typ']
rules:
- language: Typst
pattern: '^#(import|show|let|set)'
- language: XML
- extensions: ['.url']
rules:
- language: INI
pattern: '^\[InternetShortcut\](?:\r?\n|\r)(?>[^\s\[][^\r\n]*(?:\r?\n|\r))*URL='
- extensions: ['.v']
rules:
- language: Coq
pattern: '(?:^|\s)(?:Proof|Qed)\.(?:$|\s)|(?:^|\s)Require[ \t]+(Import|Export)\s'
- language: Verilog
pattern: '^[ \t]*module\s+[^\s()]+\s+\#?\(|^[ \t]*`(?:define|ifdef|ifndef|include|timescale)|^[ \t]*always[ \t]+@|^[ \t]*initial[ \t]+(begin|@)'
- language: V
pattern: '\$(?:if|else)[ \t]|^[ \t]*fn\s+[^\s()]+\(.*?\).*?\{|^[ \t]*for\s*\{'
- extensions: ['.vba']
rules:
- language: Vim Script
pattern: '^UseVimball'
- language: VBA
- extensions: ['.w']
rules:
- language: OpenEdge ABL
pattern: '&ANALYZE-SUSPEND _UIB-CODE-BLOCK _CUSTOM _DEFINITIONS'
- language: CWeb
pattern: '^@(<|\w+\.)'
- extensions: ['.x']
rules:
- language: DirectX 3D File
pattern: '^xof 030(2|3)(?:txt|bin|tzip|bzip)\b'
- language: RPC
pattern: '\b(program|version)\s+\w+\s*\{|\bunion\s+\w+\s+switch\s*\('
- language: Logos
pattern: '^%(end|ctor|hook|group)\b'
- language: Linker Script
pattern: 'OUTPUT_ARCH\(|OUTPUT_FORMAT\(|SECTIONS'
- extensions: ['.yaml', '.yml']
rules:
- language: MiniYAML
pattern: '^\t+.*?[^\s:].*?:'
negative_pattern: '---'
- language: OASv2-yaml
pattern: 'swagger:\s?''?"?2.[0-9.]+''?"?'
- language: OASv3-yaml
pattern: 'openapi:\s?''?"?3.[0-9.]+''?"?'
- language: YAML
- extensions: ['.yy']
rules:
- language: JSON
pattern: '\"modelName\"\:\s*\"GM'
- language: Yacc
named_patterns:
cpp:
- '^\s*#\s*include <(cstdint|string|vector|map|list|array|bitset|queue|stack|forward_list|unordered_map|unordered_set|(i|o|io)stream)>'
- '^\s*template\s*<'
- '^[ \t]*(try|constexpr)'
- '^[ \t]*catch\s*\('
- '^[ \t]*(class|(using[ \t]+)?namespace)\s+\w+'
- '^[ \t]*(private|public|protected):$'
- '__has_cpp_attribute|__cplusplus >'
- 'std::\w+'
euphoria:
- '^\s*namespace\s'
- '^\s*(?:public\s+)?include\s'
- '^\s*(?:(?:public|export|global)\s+)?(?:atom|constant|enum|function|integer|object|procedure|sequence|type)\s'
fortran: '^(?i:[c*][^abd-z]| (subroutine|program|end|data)\s|\s*!)'
gsc:
- '^\s*#\s*(?:using|insert|include|define|namespace)[ \t]+\w'
- '^\s*(?>(?:autoexec|private)\s+){0,2}function\s+(?>(?:autoexec|private)\s+){0,2}\w+\s*\('
- '\b(?:level|self)[ \t]+thread[ \t]+(?:\[\[[ \t]*(?>\w+\.)*\w+[ \t]*\]\]|\w+)[ \t]*\([^\r\n\)]*\)[ \t]*;'
- '^[ \t]*#[ \t]*(?:precache|using_animtree)[ \t]*\('
key_equals_value: '^[^#!;][^=]*='
m68k:
- '(?im)\bmoveq(?:\.l)?\s+#(?:\$-?[0-9a-f]{1,3}|%[0-1]{1,8}|-?[0-9]{1,3}),\s*d[0-7]\b'
- '(?im)^\s*move(?:\.[bwl])?\s+(?:sr|usp),\s*[^\s]+'
- '(?im)^\s*move\.[bwl]\s+.*\b[ad]\d'
- '(?im)^\s*movem\.[bwl]\b'
- '(?im)^\s*move[mp](?:\.[wl])?\b'
- '(?im)^\s*btst\b'
- '(?im)^\s*dbra\b'
man-heading: '^[.''][ \t]*SH +(?:[^"\s]+|"[^"\s]+)'
man-title: '^[.''][ \t]*TH +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)'
mdoc-date: '^[.''][ \t]*Dd +(?:[^"\s]+|"[^"]+")'
mdoc-heading: '^[.''][ \t]*Sh +(?:[^"\s]|"[^"]+")'
mdoc-title: '^[.''][ \t]*Dt +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)'
objectivec: '^\s*(@(interface|class|protocol|property|end|synchronised|selector|implementation)\b|#import\s+.+\.h[">])'
perl:
- '\buse\s+(?:strict\b|v?5\b)'
- '^\s*use\s+(?:constant|overload)\b'
- '^\s*(?:\*|(?:our\s*)?@)EXPORT\s*='
- '^\s*package\s+[^\W\d]\w*(?:::\w+)*\s*(?:[;{]|\sv?\d)'
- '[\s$][^\W\d]\w*(?::\w+)*->[a-zA-Z_\[({]'
raku: '^\s*(?:use\s+v6\b|\bmodule\b|\b(?:my\s+)?class\b)'
vb-class: '^[ ]*VERSION [0-9]\.[0-9] CLASS'
vb-form: '^[ ]*VERSION [0-9]\.[0-9]{2}'
vb-module: '^[ ]*Attribute VB_Name = '
vba:
- '\b(?:VBA|[vV]ba)(?:\b|[0-9A-Z_])'
# VBA7 new 64-bit features
- '^[ ]*(?:Public|Private)? Declare PtrSafe (?:Sub|Function)\b'
- '^[ ]*#If Win64\b'
- '^[ ]*(?:Dim|Const) [0-9a-zA-Z_]*[ ]*As Long(?:Ptr|Long)\b'
# Top module declarations unique to VBA
- '^[ ]*Option (?:Private Module|Compare (?:Database|Text|Binary))\b'
# General VBA libraries and objects
- '(?: |\()(?:Access|Excel|Outlook|PowerPoint|Visio|Word|VBIDE)\.\w'
- '\b(?:(?:Active)?VBProjects?|VBComponents?|Application\.(?:VBE|ScreenUpdating))\b'
# AutoCAD, Outlook, PowerPoint and Word objects
- '\b(?:ThisDrawing|AcadObject|Active(?:Explorer|Inspector|Window\.Presentation|Presentation|Document)|Selection\.(?:Find|Paragraphs))\b'
# Excel objects
- '\b(?:(?:This|Active)?Workbooks?|Worksheets?|Active(?:Sheet|Chart|Cell)|WorksheetFunction)\b'
- '\b(?:Range\(".*|Cells\([0-9a-zA-Z_]*, (?:[0-9a-zA-Z_]*|"[a-zA-Z]{1,3}"))\)'

96
src/actions.cr
View File

@ -23,17 +23,14 @@ module Tartrazine
struct Action
property actions : Array(Action) = [] of Action
@content_index : Array(Int32) = [] of Int32
@depth : Int32 = 0
@lexer_index : Int32 = 0
@lexer_name : String = ""
@states : Array(String) = [] of String
@states_to_push : Array(String) = [] of String
@token_type : String = ""
@type : ActionType = ActionType::Token
property token_type : String = ""
property states_to_push : Array(String) = [] of String
property depth = 0
property lexer_name : String = ""
property states_to_combine : Array(String) = [] of String
def initialize(t : String, xml : XML::Node?)
@type = ActionType.parse(t.capitalize)
def initialize(@type : String, @xml : XML::Node?)
# Extract information from the XML node we will use later
# Some actions may have actions in them, like this:
# <bygroups>
@ -44,30 +41,31 @@ module Tartrazine
#
# The token actions match with the first 2 groups in the regex
# the using action matches the 3rd and shunts it to another lexer
xml.children.each do |node|
known_types = %w(token push pop bygroups using usingself include combined)
raise Exception.new(
"Unknown action type: #{@type}") unless known_types.includes? @type
@xml.children.each do |node|
next unless node.element?
@actions << Action.new(node.name, node)
end
# Prefetch the attributes we ned from the XML and keep them
case @type
when ActionType::Token
@token_type = xml["type"]
when ActionType::Push
when "token"
@token_type = xml["type"]? || ""
when "push"
@states_to_push = xml.attributes.select { |attrib|
attrib.name == "state"
}.map &.content
when ActionType::Pop
@depth = xml["depth"].to_i
when ActionType::Using
@lexer_name = xml["lexer"].downcase
when ActionType::Combined
@states = xml.attributes.select { |attrib|
}.map &.content || [] of String
when "pop"
@depth = xml["depth"]?.try &.to_i || 0
when "using"
@lexer_name = xml["lexer"]?.try &.downcase || ""
when "combined"
@states_to_combine = xml.attributes.select { |attrib|
attrib.name == "state"
}.map &.content
when ActionType::Usingbygroup
@lexer_index = xml["lexer"].to_i
@content_index = xml["content"].split(",").map(&.to_i)
end
end
@ -77,21 +75,25 @@ module Tartrazine
when ActionType::Token
raise Exception.new "Can't have a token without a match" if match.empty?
[Token.new(type: @token_type, value: String.new(match[match_group].value))]
when ActionType::Push
to_push = @states_to_push.empty? ? [tokenizer.state_stack.last] : @states_to_push
to_push.each do |state|
if state == "#pop" && tokenizer.state_stack.size > 1
when "push"
if @states_to_push.empty?
# Push without a state means push the current state
@states_to_push = [lexer.state_stack.last]
end
@states_to_push.each do |state|
if state == "#pop"
# Pop the state
tokenizer.state_stack.pop
lexer.state_stack.pop
else
# Really push
tokenizer.state_stack << state
lexer.state_stack << state
end
end
[] of Token
when ActionType::Pop
to_pop = [@depth, tokenizer.state_stack.size - 1].min
tokenizer.state_stack.pop(to_pop)
when "pop"
if lexer.state_stack.size > @depth
lexer.state_stack.pop(@depth)
end
[] of Token
when ActionType::Bygroups
# FIXME: handle
@ -102,7 +104,7 @@ module Tartrazine
#
# where that None means skipping a group
#
raise Exception.new "Can't have a token without a match" if match.nil?
raise Exception.new "Can't have a token without a match" if match.empty?
# Each group matches an action. If the group match is empty,
# the action is skipped.
@ -111,8 +113,7 @@ module Tartrazine
begin
next if match[i + 1].size == 0
rescue IndexError
# FIXME: This should not actually happen
# No match for this group
# No match for the last group
next
end
result += e.emit(match, tokenizer, i + 1)
@ -121,19 +122,16 @@ module Tartrazine
when ActionType::Using
# Shunt to another lexer entirely
return [] of Token if match.empty?
Tartrazine.lexer(@lexer_name).tokenizer(
String.new(match[match_group].value),
secondary: true).to_a
when ActionType::Usingself
Tartrazine.lexer(@lexer_name).tokenize(String.new(match[match_group].value), usingself: true)
when "usingself"
# Shunt to another copy of this lexer
return [] of Token if match.empty?
tokenizer.lexer.tokenizer(
String.new(match[match_group].value),
secondary: true).to_a
when ActionType::Combined
# Combine two or more states into one anonymous state
new_state = @states.map { |name|
tokenizer.lexer.states[name]
new_lexer = Lexer.from_xml(lexer.xml)
new_lexer.tokenize(String.new(match[match_group].value), usingself: true)
when "combined"
# Combine two states into one anonymous state
new_state = @states_to_combine.map { |name|
lexer.states[name]
}.reduce { |state1, state2|
state1 + state2
}
@ -151,7 +149,7 @@ module Tartrazine
content,
secondary: true).to_a
else
raise Exception.new("Unknown action type: #{@type}")
raise Exception.new("Unhandled action type: #{type}")
end
end
end

6
src/bytes_regex.cr
View File

@ -31,6 +31,7 @@ module BytesRegex
end
def match(str : Bytes, pos = 0) : Array(Match)
match = [] of Match
rc = LibPCRE2.match(
@re,
str,
@ -39,9 +40,9 @@ module BytesRegex
LibPCRE2::NO_UTF_CHECK,
@match_data,
nil)
if rc > 0
if rc >= 0
ovector = LibPCRE2.get_ovector_pointer(@match_data)
(0...rc).map do |i|
(0...rc).each do |i|
m_start = ovector[2 * i]
m_end = ovector[2 * i + 1]
if m_start == m_end
@ -54,6 +55,7 @@ module BytesRegex
else
[] of Match
end
match
end
end

42
src/heuristics.cr Normal file
View File

@ -0,0 +1,42 @@
require "yaml"
module Tartrazine
# Use linguist's heuristics to disambiguate between languages
class Heuristic
include YAML::Serializable
property disambiguations : Array(Disambiguation)
property named_patterns : Hash(String, String | Array(String))
# Run the heuristics on the given filename and content
def run(filename, content)
ext = File.extname filename
disambiguation = disambiguations.find do |item|
item.extensions.includes? ext
end
p! disambiguation
end
end
class Disambiguation
include YAML::Serializable
property extensions : Array(String)
property rules : Array(LangRule)
end
class Rule
include YAML::Serializable
property pattern : (String | Array(String))?
property named_pattern : String?
property and : Array(Rule)?
end
class LangRule < Rule
include YAML::Serializable
property language : String | Array(String)
end
end
h = Tartrazine::Heuristic.from_yaml(File.read("heuristics/heuristics.yml"))
h.run("../elkjs/src/elk.h", File.read("../elkjs/src/elk.h"))

13
src/main.cr
View File

@ -1,6 +1,18 @@
require "docopt"
require "./**"
# Performance data (in milliseconds):
#
# Docopt parsing: 0.5
# Instantiating a theme: 0.1
# Instantiating a formatter: 1.0
# Instantiating a lexer: 2.0
# Tokenizing crycco.cr: 16.0
# Formatting: 0.5
# I/O: 1.5
# ---------------------------------
# Total: 21.6
HELP = <<-HELP
tartrazine: a syntax highlighting tool
@ -84,7 +96,6 @@ if options["-f"]
end
lexer = Tartrazine.lexer(name: options["-l"].as(String), filename: options["FILE"].as(String))
input = File.open(options["FILE"].as(String)).gets_to_end
if options["-o"].nil?

31
src/rules.cr
View File

@ -15,11 +15,28 @@ module Tartrazine
alias Match = BytesRegex::Match
alias MatchData = Array(Match)
abstract struct BaseRule
abstract def match(text : Bytes, pos : Int32, tokenizer : Tokenizer) : Tuple(Bool, Int32, Array(Token))
abstract def initialize(node : XML::Node)
class Rule
property pattern : Regex = Regex.new ""
property actions : Array(Action) = [] of Action
@actions : Array(Action) = [] of Action
def match(text : Bytes, pos, lexer) : Tuple(Bool, Int32, Array(Token))
match = pattern.match(text, pos)
# We don't match if the match doesn't move the cursor
# because that causes infinite loops
return false, pos, [] of Token if match.empty? || match[0].size == 0
tokens = [] of Token
actions.each do |action|
tokens += action.emit(match, lexer)
end
return true, pos + match[0].size, tokens
end
def initialize(node : XML::Node, multiline, dotall, ignorecase)
pattern = node["pattern"]
pattern = "(?m)" + pattern if multiline
@pattern = Regex.new(pattern, multiline, dotall, ignorecase, true)
add_actions(node)
end
def add_actions(node : XML::Node)
node.children.each do |child|
@ -56,9 +73,9 @@ module Tartrazine
struct IncludeStateRule < BaseRule
@state : String = ""
def match(text : Bytes, pos : Int32, tokenizer : Tokenizer) : Tuple(Bool, Int32, Array(Token))
tokenizer.@lexer.states[@state].rules.each do |rule|
matched, new_pos, new_tokens = rule.match(text, pos, tokenizer)
def match(text, pos, lexer) : Tuple(Bool, Int32, Array(Token))
lexer.states[state].rules.each do |rule|
matched, new_pos, new_tokens = rule.match(text, pos, lexer)
return true, new_pos, new_tokens if matched
end
return false, pos, [] of Token