diff --git a/src/tartrazine.cr b/src/tartrazine.cr index 70805da..d237064 100644 --- a/src/tartrazine.cr +++ b/src/tartrazine.cr @@ -85,9 +85,11 @@ module Tartrazine raise Exception.new "Can't have a token without a match" if match.nil? [Token.new(type: xml["type"], value: match[0])] when "push" + puts "Pushing state #{xml["state"]}" lexer.state_stack << xml["state"] [] of Token when "pop" + puts "Popping #{xml["depth"]} states" lexer.state_stack.pop(xml["depth"].to_i) [] of Token else diff --git a/tests/apacheconf/test_directive_no_args.txt b/tests/apacheconf/test_directive_no_args.txt new file mode 100644 index 0000000..a2f3354 --- /dev/null +++ b/tests/apacheconf/test_directive_no_args.txt @@ -0,0 +1,12 @@ +---input--- +Example +ServerName localhost + +---tokens--- +'Example' Name.Builtin +'\n' Text.Whitespace + +'ServerName' Name.Builtin +' ' Text.Whitespace +'localhost' Text +'\n' Text.Whitespace diff --git a/tests/apacheconf/test_fix_lock_absolute_path.txt b/tests/apacheconf/test_fix_lock_absolute_path.txt new file mode 100644 index 0000000..a274834 --- /dev/null +++ b/tests/apacheconf/test_fix_lock_absolute_path.txt @@ -0,0 +1,8 @@ +---input--- +LockFile /var/lock/apache2/accept.lock + +---tokens--- +'LockFile' Name.Builtin +' ' Text.Whitespace +'/var/lock/apache2/accept.lock' Literal.String.Other +'\n' Text.Whitespace diff --git a/tests/apacheconf/test_include_globs.txt b/tests/apacheconf/test_include_globs.txt new file mode 100644 index 0000000..495242e --- /dev/null +++ b/tests/apacheconf/test_include_globs.txt @@ -0,0 +1,8 @@ +---input--- +Include /etc/httpd/conf.d/*.conf + +---tokens--- +'Include' Name.Builtin +' ' Text.Whitespace +'/etc/httpd/conf.d/*.conf' Literal.String.Other +'\n' Text.Whitespace diff --git a/tests/apacheconf/test_malformed_scoped_directive_closing_tag.txt b/tests/apacheconf/test_malformed_scoped_directive_closing_tag.txt new file mode 100644 index 0000000..c65628b --- /dev/null +++ b/tests/apacheconf/test_malformed_scoped_directive_closing_tag.txt @@ -0,0 +1,19 @@ +---input--- + + + +---tokens--- +'' Name.Tag +'\n' Text.Whitespace + +'<' Error +'/' Error +'VirtualHost' Name.Builtin +'\n' Text.Whitespace + +'>' Error +'\n' Text.Whitespace diff --git a/tests/apacheconf/test_multi_include_globs.txt b/tests/apacheconf/test_multi_include_globs.txt new file mode 100644 index 0000000..abf5bb0 --- /dev/null +++ b/tests/apacheconf/test_multi_include_globs.txt @@ -0,0 +1,8 @@ +---input--- +Include /etc/httpd/conf.d/*/*.conf + +---tokens--- +'Include' Name.Builtin +' ' Text.Whitespace +'/etc/httpd/conf.d/*/*.conf' Literal.String.Other +'\n' Text.Whitespace diff --git a/tests/apacheconf/test_multi_include_globs_root.txt b/tests/apacheconf/test_multi_include_globs_root.txt new file mode 100644 index 0000000..5df63e4 --- /dev/null +++ b/tests/apacheconf/test_multi_include_globs_root.txt @@ -0,0 +1,8 @@ +---input--- +Include /*conf/*.conf + +---tokens--- +'Include' Name.Builtin +' ' Text.Whitespace +'/*conf/*.conf' Literal.String.Other +'\n' Text.Whitespace diff --git a/tests/apacheconf/test_multiline_argument.txt b/tests/apacheconf/test_multiline_argument.txt new file mode 100644 index 0000000..4163970 --- /dev/null +++ b/tests/apacheconf/test_multiline_argument.txt @@ -0,0 +1,20 @@ +---input--- +SecAction \ + "id:'900001', \ + phase:1, \ + t:none, \ + setvar:tx.critical_anomaly_score=5, \ + setvar:tx.error_anomaly_score=4, \ + setvar:tx.warning_anomaly_score=3, \ + setvar:tx.notice_anomaly_score=2, \ + nolog, \ + pass" + +---tokens--- +'SecAction' Name.Builtin +' ' Text.Whitespace +'\\\n' Text + +' ' Text.Whitespace +'"id:\'900001\', \\\n phase:1, \\\n t:none, \\\n setvar:tx.critical_anomaly_score=5, \\\n setvar:tx.error_anomaly_score=4, \\\n setvar:tx.warning_anomaly_score=3, \\\n setvar:tx.notice_anomaly_score=2, \\\n nolog, \\\n pass"' Literal.String.Double +'\n' Text.Whitespace diff --git a/tests/apacheconf/test_multiline_comment.txt b/tests/apacheconf/test_multiline_comment.txt new file mode 100644 index 0000000..cdcfe01 --- /dev/null +++ b/tests/apacheconf/test_multiline_comment.txt @@ -0,0 +1,12 @@ +---input--- +#SecAction \ + "id:'900004', \ + phase:1, \ + t:none, \ + setvar:tx.anomaly_score_blocking=on, \ + nolog, \ + pass" + +---tokens--- +'#SecAction \\\n "id:\'900004\', \\\n phase:1, \\\n t:none, \\\n setvar:tx.anomaly_score_blocking=on, \\\n nolog, \\\n pass"' Comment +'\n' Text.Whitespace diff --git a/tests/apacheconf/test_normal_scoped_directive.txt b/tests/apacheconf/test_normal_scoped_directive.txt new file mode 100644 index 0000000..f00496d --- /dev/null +++ b/tests/apacheconf/test_normal_scoped_directive.txt @@ -0,0 +1,14 @@ +---input--- + + + +---tokens--- +'' Name.Tag +'\n' Text.Whitespace + +'' Name.Tag +'\n' Text.Whitespace diff --git a/tests/apl/test_leading_underscore.txt b/tests/apl/test_leading_underscore.txt new file mode 100644 index 0000000..c11258d --- /dev/null +++ b/tests/apl/test_leading_underscore.txt @@ -0,0 +1,26 @@ +---input--- +_op_←{_←⍺ ⍵⋄(⍺⍺ ⍺)+⍵⍵ ⍵} + +---tokens--- +'_op_' Name.Variable +'←' Keyword.Declaration +'{' Keyword.Type +'_' Name.Variable +'←' Keyword.Declaration +'⍺' Name.Builtin.Pseudo +' ' Text.Whitespace +'⍵' Name.Builtin.Pseudo +'⋄' Punctuation +'(' Punctuation +'⍺' Name.Builtin.Pseudo +'⍺' Name.Builtin.Pseudo +' ' Text.Whitespace +'⍺' Name.Builtin.Pseudo +')' Punctuation +'+' Operator +'⍵' Name.Builtin.Pseudo +'⍵' Name.Builtin.Pseudo +' ' Text.Whitespace +'⍵' Name.Builtin.Pseudo +'}' Keyword.Type +'\n' Text.Whitespace diff --git a/tests/asm/test_cpuid.txt b/tests/asm/test_cpuid.txt new file mode 100644 index 0000000..8256618 --- /dev/null +++ b/tests/asm/test_cpuid.txt @@ -0,0 +1,9 @@ +# CPU is a valid directive, and we don't want to parse this as +# cpu id, but as a single token. See bug #1517 + +---input--- +cpuid + +---tokens--- +'cpuid' Name.Function +'\n' Text.Whitespace diff --git a/tests/asn1/certificate.txt b/tests/asn1/certificate.txt new file mode 100644 index 0000000..734cb3f --- /dev/null +++ b/tests/asn1/certificate.txt @@ -0,0 +1,147 @@ +---input--- +Certificate ::= SEQUENCE { + tbsCertificate TBSCertificate, + signatureAlgorithm AlgorithmIdentifier, + signatureValue BIT STRING } + +TBSCertificate ::= SEQUENCE { + version [0] EXPLICIT Version DEFAULT v1, + serialNumber CertificateSerialNumber, + signature AlgorithmIdentifier, + issuer Name, + validity Validity, + subject Name, + subjectPublicKeyInfo SubjectPublicKeyInfo, + issuerUniqueID [1] IMPLICIT UniqueIdentifier OPTIONAL, + -- If present, version MUST be v2 or v3 + subjectUniqueID [2] IMPLICIT UniqueIdentifier OPTIONAL, + -- If present, version MUST be v2 or v3 + extensions [3] EXPLICIT Extensions OPTIONAL + -- If present, version MUST be v3 + } + +---tokens--- +'Certificate' Name.Type +' ' Text.Whitespace +'::=' Operator +' ' Text.Whitespace +'SEQUENCE' Keyword.Declaration +' ' Text.Whitespace +'{' Punctuation +'\n ' Text.Whitespace +'tbsCertificate' Name.Variable +' ' Text.Whitespace +'TBSCertificate' Name.Type +',' Punctuation +'\n ' Text.Whitespace +'signatureAlgorithm' Name.Variable +' ' Text.Whitespace +'AlgorithmIdentifier' Name.Type +',' Punctuation +'\n ' Text.Whitespace +'signatureValue' Name.Variable +' ' Text.Whitespace +'BIT STRING' Keyword.Type +' ' Text.Whitespace +'}' Punctuation +'\n\n' Text.Whitespace + +'TBSCertificate' Name.Type +' ' Text.Whitespace +'::=' Operator +' ' Text.Whitespace +'SEQUENCE' Keyword.Declaration +' ' Text.Whitespace +'{' Punctuation +'\n ' Text.Whitespace +'version' Name.Variable +' ' Text.Whitespace +'[' Punctuation +'0' Literal.Number.Integer +']' Punctuation +' ' Text.Whitespace +'EXPLICIT' Keyword +' ' Text.Whitespace +'Version' Name.Type +' ' Text.Whitespace +'DEFAULT' Keyword.Declaration +' ' Text.Whitespace +'v1' Name.Variable +',' Punctuation +'\n ' Text.Whitespace +'serialNumber' Name.Variable +' ' Text.Whitespace +'CertificateSerialNumber' Name.Type +',' Punctuation +'\n ' Text.Whitespace +'signature' Name.Variable +' ' Text.Whitespace +'AlgorithmIdentifier' Name.Type +',' Punctuation +'\n ' Text.Whitespace +'issuer' Name.Variable +' ' Text.Whitespace +'Name' Name.Type +',' Punctuation +'\n ' Text.Whitespace +'validity' Name.Variable +' ' Text.Whitespace +'Validity' Name.Type +',' Punctuation +'\n ' Text.Whitespace +'subject' Name.Variable +' ' Text.Whitespace +'Name' Name.Type +',' Punctuation +'\n ' Text.Whitespace +'subjectPublicKeyInfo' Name.Variable +' ' Text.Whitespace +'SubjectPublicKeyInfo' Name.Type +',' Punctuation +'\n ' Text.Whitespace +'issuerUniqueID' Name.Variable +' ' Text.Whitespace +'[' Punctuation +'1' Literal.Number.Integer +']' Punctuation +' ' Text.Whitespace +'IMPLICIT' Keyword +' ' Text.Whitespace +'UniqueIdentifier' Name.Type +' ' Text.Whitespace +'OPTIONAL' Keyword.Declaration +',' Punctuation +'\n ' Text.Whitespace +'-- If present, version MUST be v2 or v3' Comment.Single +'\n ' Text.Whitespace +'subjectUniqueID' Name.Variable +' ' Text.Whitespace +'[' Punctuation +'2' Literal.Number.Integer +']' Punctuation +' ' Text.Whitespace +'IMPLICIT' Keyword +' ' Text.Whitespace +'UniqueIdentifier' Name.Type +' ' Text.Whitespace +'OPTIONAL' Keyword.Declaration +',' Punctuation +'\n ' Text.Whitespace +'-- If present, version MUST be v2 or v3' Comment.Single +'\n ' Text.Whitespace +'extensions' Name.Variable +' ' Text.Whitespace +'[' Punctuation +'3' Literal.Number.Integer +']' Punctuation +' ' Text.Whitespace +'EXPLICIT' Keyword +' ' Text.Whitespace +'Extensions' Name.Type +' ' Text.Whitespace +'OPTIONAL' Keyword.Declaration +'\n ' Text.Whitespace +'-- If present, version MUST be v3' Comment.Single +'\n ' Text.Whitespace +'}' Punctuation +'\n' Text.Whitespace diff --git a/tests/asn1/nested-comment.txt b/tests/asn1/nested-comment.txt new file mode 100644 index 0000000..2d4a0c8 --- /dev/null +++ b/tests/asn1/nested-comment.txt @@ -0,0 +1,49 @@ +---input--- +/* + Outer comment + /* + Inner comment + */ + Outer comment +*/ + +test ::= SEQUENCE { + a INTEGER, + b OCTET STRING, + b BIT STRING } + +---tokens--- +'/*' Comment.Multiline +'\n Outer comment\n ' Comment.Multiline +'/*' Comment.Multiline +'\n Inner comment\n ' Comment.Multiline +'*/' Comment.Multiline +'\n Outer comment\n' Comment.Multiline + +'*/' Comment.Multiline +'\n\n' Text.Whitespace + +'test' Name.Variable +' ' Text.Whitespace +'::=' Operator +' ' Text.Whitespace +'SEQUENCE' Keyword.Declaration +' ' Text.Whitespace +'{' Punctuation +'\n ' Text.Whitespace +'a' Error +' ' Text.Whitespace +'INTEGER' Keyword.Type +',' Punctuation +'\n ' Text.Whitespace +'b' Error +' ' Text.Whitespace +'OCTET STRING' Keyword.Type +',' Punctuation +'\n ' Text.Whitespace +'b' Error +' ' Text.Whitespace +'BIT STRING' Keyword.Type +' ' Text.Whitespace +'}' Punctuation +'\n' Text.Whitespace diff --git a/tests/awk/test_ternary.txt b/tests/awk/test_ternary.txt new file mode 100644 index 0000000..b67049a --- /dev/null +++ b/tests/awk/test_ternary.txt @@ -0,0 +1,34 @@ +---input--- +BEGIN {a = 5;b=6; print (a==b)? "a==b" : "a!=b"} + +---tokens--- +'BEGIN' Name.Builtin +' ' Text +'{' Punctuation +'a' Name.Other +' ' Text +'=' Operator +' ' Text +'5' Literal.Number.Integer +';' Punctuation +'b' Name.Other +'=' Operator +'6' Literal.Number.Integer +';' Punctuation +' ' Text +'print' Keyword.Reserved +' ' Text +'(' Punctuation +'a' Name.Other +'==' Operator +'b' Name.Other +')' Punctuation +'?' Operator +' ' Text +'"a==b"' Literal.String.Double +' ' Text +':' Operator +' ' Text +'"a!=b"' Literal.String.Double +'}' Punctuation +'\n' Text diff --git a/tests/bibtex/test_basic_bst.txt b/tests/bibtex/test_basic_bst.txt new file mode 100644 index 0000000..c519ae0 --- /dev/null +++ b/tests/bibtex/test_basic_bst.txt @@ -0,0 +1,54 @@ +---input--- +% BibTeX standard bibliography style `plain' + +INTEGERS { output.state before.all } + +FUNCTION {sort.format.title} +{ 't := +"A " #2 + "An " #3 + "The " #4 t chop.word + chop.word +chop.word +sortify +#1 global.max$ substring$ +} + +ITERATE {call.type$} + +---tokens--- +"% BibTeX standard bibliography style `plain'" Comment +'\n\n' Text.Whitespace + +'INTEGERS { output.state before.all }' Comment +'\n\n' Text.Whitespace + +'FUNCTION {sort.format.title}' Comment +'\n' Text.Whitespace + +"{ 't :=" Comment +'\n' Text.Whitespace + +'"A " #2' Comment +'\n ' Text.Whitespace +'"An " #3' Comment +'\n ' Text.Whitespace +'"The " #4 t chop.word' Comment +'\n ' Text.Whitespace +'chop.word' Comment +'\n' Text.Whitespace + +'chop.word' Comment +'\n' Text.Whitespace + +'sortify' Comment +'\n' Text.Whitespace + +'#1 global.max$ substring$' Comment +'\n' Text.Whitespace + +'}' Comment +'\n\n' Text.Whitespace + +'ITERATE {call.type$}' Comment +'\n' Text.Whitespace diff --git a/tests/bibtex/test_comment.txt b/tests/bibtex/test_comment.txt new file mode 100644 index 0000000..6e98d64 --- /dev/null +++ b/tests/bibtex/test_comment.txt @@ -0,0 +1,7 @@ +---input--- +@COMMENT{test} + +---tokens--- +'@COMMENT' Comment +'{test}' Comment +'\n' Text.Whitespace diff --git a/tests/bibtex/test_entry.txt b/tests/bibtex/test_entry.txt new file mode 100644 index 0000000..c712a2f --- /dev/null +++ b/tests/bibtex/test_entry.txt @@ -0,0 +1,63 @@ +---input--- +This is a comment. + +@ARTICLE{ruckenstein-diffusion, + author = "Liu, Hongquin" # and # "Ruckenstein, Eli", + year = 1997, + month = JAN, + pages = "888-895" +} + +---tokens--- +'This is a comment.' Comment +'\n\n' Text.Whitespace + +'@ARTICLE' Name.Class +'{' Punctuation +'ruckenstein-diffusion' Name.Label +',' Punctuation +'\n ' Text.Whitespace +'author' Name.Attribute +' ' Text.Whitespace +'=' Punctuation +' ' Text.Whitespace +'"' Literal.String +'Liu, Hongquin' Literal.String +'"' Literal.String +' ' Text.Whitespace +'#' Punctuation +' ' Text.Whitespace +'and' Name.Variable +' ' Text.Whitespace +'#' Punctuation +' ' Text.Whitespace +'"' Literal.String +'Ruckenstein, Eli' Literal.String +'"' Literal.String +',' Punctuation +'\n ' Text.Whitespace +'year' Name.Attribute +' ' Text.Whitespace +'=' Punctuation +' ' Text.Whitespace +'1997' Literal.Number +',' Punctuation +'\n ' Text.Whitespace +'month' Name.Attribute +' ' Text.Whitespace +'=' Punctuation +' ' Text.Whitespace +'JAN' Name.Variable +',' Punctuation +'\n ' Text.Whitespace +'pages' Name.Attribute +' ' Text.Whitespace +'=' Punctuation +' ' Text.Whitespace +'"' Literal.String +'888-895' Literal.String +'"' Literal.String +'\n' Text.Whitespace + +'}' Punctuation +'\n' Text.Whitespace diff --git a/tests/bibtex/test_mismatched_brace.txt b/tests/bibtex/test_mismatched_brace.txt new file mode 100644 index 0000000..6c1deda --- /dev/null +++ b/tests/bibtex/test_mismatched_brace.txt @@ -0,0 +1,10 @@ +---input--- +@PREAMBLE(""} + +---tokens--- +'@PREAMBLE' Name.Class +'(' Punctuation +'"' Literal.String +'"' Literal.String +'}' Error +'\n' Text.Whitespace diff --git a/tests/bibtex/test_missing_body.txt b/tests/bibtex/test_missing_body.txt new file mode 100644 index 0000000..24dad98 --- /dev/null +++ b/tests/bibtex/test_missing_body.txt @@ -0,0 +1,10 @@ +---input--- +@ARTICLE xxx + +---tokens--- +'@ARTICLE' Name.Class +' ' Text.Whitespace +'x' Error +'x' Error +'x' Error +'\n' Text.Whitespace diff --git a/tests/bibtex/test_preamble.txt b/tests/bibtex/test_preamble.txt new file mode 100644 index 0000000..9625f96 --- /dev/null +++ b/tests/bibtex/test_preamble.txt @@ -0,0 +1,11 @@ +---input--- +@PREAMBLE{"% some LaTeX code here"} + +---tokens--- +'@PREAMBLE' Name.Class +'{' Punctuation +'"' Literal.String +'% some LaTeX code here' Literal.String +'"' Literal.String +'}' Punctuation +'\n' Text.Whitespace diff --git a/tests/bibtex/test_string.txt b/tests/bibtex/test_string.txt new file mode 100644 index 0000000..f76a31b --- /dev/null +++ b/tests/bibtex/test_string.txt @@ -0,0 +1,15 @@ +---input--- +@STRING(SCI = "Science") + +---tokens--- +'@STRING' Name.Class +'(' Punctuation +'SCI' Name.Attribute +' ' Text.Whitespace +'=' Punctuation +' ' Text.Whitespace +'"' Literal.String +'Science' Literal.String +'"' Literal.String +')' Punctuation +'\n' Text.Whitespace diff --git a/tests/bqn/test_arguments.txt b/tests/bqn/test_arguments.txt new file mode 100644 index 0000000..a8ccafb --- /dev/null +++ b/tests/bqn/test_arguments.txt @@ -0,0 +1,28 @@ +---input--- +Ambiv ← { ⟨1,𝕩⟩ ; ⟨2,𝕨,𝕩⟩ } + +---tokens--- +'Ambiv' Operator +' ' Text.Whitespace +'←' Keyword.Declaration +' ' Text.Whitespace +'{' Keyword.Type +' ' Text.Whitespace +'⟨' Punctuation +'1' Literal.Number +',' Punctuation +'𝕩' Name.Entity +'⟩' Punctuation +' ' Text.Whitespace +';' Name.Entity +' ' Text.Whitespace +'⟨' Punctuation +'2' Literal.Number +',' Punctuation +'𝕨' Name.Entity +',' Punctuation +'𝕩' Name.Entity +'⟩' Punctuation +' ' Text.Whitespace +'}' Keyword.Type +'\n' Text.Whitespace diff --git a/tests/bqn/test_comment.txt b/tests/bqn/test_comment.txt new file mode 100644 index 0000000..efbee58 --- /dev/null +++ b/tests/bqn/test_comment.txt @@ -0,0 +1,12 @@ +---input--- +'#' - 1 #This is the comment + +---tokens--- +"'#'" Literal.String.Single +' ' Text.Whitespace +'-' Operator +' ' Text.Whitespace +'1' Literal.Number +' ' Text.Whitespace +'#This is the comment' Comment.Single +'\n' Text.Whitespace diff --git a/tests/bqn/test_define.txt b/tests/bqn/test_define.txt new file mode 100644 index 0000000..ac288db --- /dev/null +++ b/tests/bqn/test_define.txt @@ -0,0 +1,42 @@ +---input--- +⟨alias⇐a, b⟩ ← { + b‿c⇐ + a⇐2 + c←÷b↩1+a +} + +---tokens--- +'⟨' Punctuation +'alias' Name.Variable +'⇐' Keyword.Declaration +'a' Name.Variable +',' Punctuation +' ' Text.Whitespace +'b' Name.Variable +'⟩' Punctuation +' ' Text.Whitespace +'←' Keyword.Declaration +' ' Text.Whitespace +'{' Keyword.Type +'\n ' Text.Whitespace +'b' Name.Variable +'‿' Punctuation +'c' Name.Variable +'⇐' Keyword.Declaration +'\n ' Text.Whitespace +'a' Name.Variable +'⇐' Keyword.Declaration +'2' Literal.Number +'\n ' Text.Whitespace +'c' Name.Variable +'←' Keyword.Declaration +'÷' Operator +'b' Name.Variable +'↩' Keyword.Declaration +'1' Literal.Number +'+' Operator +'a' Name.Variable +'\n' Text.Whitespace + +'}' Keyword.Type +'\n' Text.Whitespace diff --git a/tests/bqn/test_syntax_roles.txt b/tests/bqn/test_syntax_roles.txt new file mode 100644 index 0000000..c611e6e --- /dev/null +++ b/tests/bqn/test_syntax_roles.txt @@ -0,0 +1,23 @@ +---input--- +⟨ט,√⟩ {𝕎𝕩}⌜ 1‿4‿9 + +---tokens--- +'⟨' Punctuation +'×' Operator +'˜' Name.Attribute +',' Punctuation +'√' Operator +'⟩' Punctuation +' ' Text.Whitespace +'{' Keyword.Type +'𝕎' Operator +'𝕩' Name.Entity +'}' Keyword.Type +'⌜' Name.Attribute +' ' Text.Whitespace +'1' Literal.Number +'‿' Punctuation +'4' Literal.Number +'‿' Punctuation +'9' Literal.Number +'\n' Text.Whitespace diff --git a/tests/c/builtin_types.txt b/tests/c/builtin_types.txt new file mode 100644 index 0000000..8677859 --- /dev/null +++ b/tests/c/builtin_types.txt @@ -0,0 +1,13 @@ +---input--- +__int128 +_BitInt(2) + +---tokens--- +'__int128' Keyword.Type +'\n' Text.Whitespace + +'_BitInt' Keyword.Type +'(' Punctuation +'2' Literal.Number.Integer +')' Punctuation +'\n' Text.Whitespace diff --git a/tests/c/test_comment_end.txt b/tests/c/test_comment_end.txt new file mode 100644 index 0000000..2a21d65 --- /dev/null +++ b/tests/c/test_comment_end.txt @@ -0,0 +1,31 @@ +In the past the "*/" was marked as an error to "helpfully" +indicate a wrong comment end. + +---input--- +int m21=((result_0*0+result_1*/*0<-----*/1)%mod+mod)%mod; + +---tokens--- +'int' Keyword.Type +' ' Text.Whitespace +'m21' Name +'=' Operator +'(' Punctuation +'(' Punctuation +'result_0' Name +'*' Operator +'0' Literal.Number.Integer +'+' Operator +'result_1' Name +'*' Operator +'/*0<-----*/' Comment.Multiline +'1' Literal.Number.Integer +')' Punctuation +'%' Operator +'mod' Name +'+' Operator +'mod' Name +')' Punctuation +'%' Operator +'mod' Name +';' Punctuation +'\n' Text.Whitespace diff --git a/tests/c/test_function_comments.txt b/tests/c/test_function_comments.txt new file mode 100644 index 0000000..e8d8a69 --- /dev/null +++ b/tests/c/test_function_comments.txt @@ -0,0 +1,409 @@ +---input--- +int func1(int x, int y) + /*@requires y >= 0*/ +{ + return x / y; +} + + +int func2(int x, int y) //@requires y >= 0; +{ + return x / y; +} + + +void func3() +//#test{}; +{ + fun(2,3)//test1; + ; +} + + +int func4(int x, int y) + /*@requires y >= 0;*/ +{ + return x / y; +} + + +int func5(int x, int y) + /*@requires y >= 0 + { + return x / y; + } + */ + { + return 2; + } + + +//@requires y >= 0; +//@requires y >= 0 +/* +calling(2,5) +*/ +/* +calling(2,5); +*/ +int func6(int x, int y) + //@requires y >= 0 + //@requires y >= 0; + /* + hello(2,3); + */ + /* + hello(2,3) + */ + { + // haha(2,3); + return x / y; + /* + callblabla(x, y); + */ + } +//@requires y >= 0; +//@requires y >= 0 +/* +calling(2,5) +*/ +/* +calling(2,5); +*/ + + +int * //@# a pointer to int +func7 /* @# why a comment here? */ ( + int /* the index has to be an int */ a, // index into the array + int *b //the array @! +) +/* + The end of the func params @ (@ will result error if parsed incorrectly) +*/ +{ + // yet another comment + return b[a]; +} + +---tokens--- +'int' Keyword.Type +' ' Text.Whitespace +'func1' Name.Function +'(' Punctuation +'int' Keyword.Type +' ' Text.Whitespace +'x' Name +',' Punctuation +' ' Text.Whitespace +'int' Keyword.Type +' ' Text.Whitespace +'y' Name +')' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'/*@requires y >= 0*/' Comment.Multiline +'\n' Text.Whitespace + +'{' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'return' Keyword +' ' Text.Whitespace +'x' Name +' ' Text.Whitespace +'/' Operator +' ' Text.Whitespace +'y' Name +';' Punctuation +'\n' Text.Whitespace + +'}' Punctuation +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'int' Keyword.Type +' ' Text.Whitespace +'func2' Name.Function +'(' Punctuation +'int' Keyword.Type +' ' Text.Whitespace +'x' Name +',' Punctuation +' ' Text.Whitespace +'int' Keyword.Type +' ' Text.Whitespace +'y' Name +')' Punctuation +' ' Text.Whitespace +'//@requires y >= 0;\n' Comment.Single + +'{' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'return' Keyword +' ' Text.Whitespace +'x' Name +' ' Text.Whitespace +'/' Operator +' ' Text.Whitespace +'y' Name +';' Punctuation +'\n' Text.Whitespace + +'}' Punctuation +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'void' Keyword.Type +' ' Text.Whitespace +'func3' Name.Function +'(' Punctuation +')' Punctuation +'\n' Text.Whitespace + +'//#test{};\n' Comment.Single + +'{' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'fun' Name +'(' Punctuation +'2' Literal.Number.Integer +',' Punctuation +'3' Literal.Number.Integer +')' Punctuation +'//test1;\n' Comment.Single + +' ' Text.Whitespace +';' Punctuation +'\n' Text.Whitespace + +'}' Punctuation +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'int' Keyword.Type +' ' Text.Whitespace +'func4' Name.Function +'(' Punctuation +'int' Keyword.Type +' ' Text.Whitespace +'x' Name +',' Punctuation +' ' Text.Whitespace +'int' Keyword.Type +' ' Text.Whitespace +'y' Name +')' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'/*@requires y >= 0;*/' Comment.Multiline +'\n' Text.Whitespace + +'{' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'return' Keyword +' ' Text.Whitespace +'x' Name +' ' Text.Whitespace +'/' Operator +' ' Text.Whitespace +'y' Name +';' Punctuation +'\n' Text.Whitespace + +'}' Punctuation +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'int' Keyword.Type +' ' Text.Whitespace +'func5' Name.Function +'(' Punctuation +'int' Keyword.Type +' ' Text.Whitespace +'x' Name +',' Punctuation +' ' Text.Whitespace +'int' Keyword.Type +' ' Text.Whitespace +'y' Name +')' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'/*@requires y >= 0\n {\n return x / y;\n }\n */' Comment.Multiline +'\n' Text.Whitespace + +' ' Text.Whitespace +'{' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'return' Keyword +' ' Text.Whitespace +'2' Literal.Number.Integer +';' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'}' Punctuation +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'//@requires y >= 0;\n' Comment.Single + +'//@requires y >= 0\n' Comment.Single + +'/*\ncalling(2,5)\n*/' Comment.Multiline +'\n' Text.Whitespace + +'/*\ncalling(2,5);\n*/' Comment.Multiline +'\n' Text.Whitespace + +'int' Keyword.Type +' ' Text.Whitespace +'func6' Name.Function +'(' Punctuation +'int' Keyword.Type +' ' Text.Whitespace +'x' Name +',' Punctuation +' ' Text.Whitespace +'int' Keyword.Type +' ' Text.Whitespace +'y' Name +')' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'//@requires y >= 0\n' Comment.Single + +' ' Text.Whitespace +'//@requires y >= 0;\n' Comment.Single + +' ' Text.Whitespace +'/*\n hello(2,3);\n */' Comment.Multiline +'\n' Text.Whitespace + +' ' Text.Whitespace +'/*\n hello(2,3)\n */' Comment.Multiline +'\n' Text.Whitespace + +' ' Text.Whitespace +'{' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'// haha(2,3);\n' Comment.Single + +' ' Text.Whitespace +'return' Keyword +' ' Text.Whitespace +'x' Name +' ' Text.Whitespace +'/' Operator +' ' Text.Whitespace +'y' Name +';' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'/*\n callblabla(x, y);\n */' Comment.Multiline +'\n' Text.Whitespace + +' ' Text.Whitespace +'}' Punctuation +'\n' Text.Whitespace + +'//@requires y >= 0;\n' Comment.Single + +'//@requires y >= 0\n' Comment.Single + +'/*\ncalling(2,5)\n*/' Comment.Multiline +'\n' Text.Whitespace + +'/*\ncalling(2,5);\n*/' Comment.Multiline +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'int' Keyword.Type +' ' Text.Whitespace +'*' Operator +' ' Text.Whitespace +'//@# a pointer to int\n' Comment.Single + +'func7' Name.Function +' ' Text.Whitespace +'/* @# why a comment here? */' Comment.Multiline +' ' Text.Whitespace +'(' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'int' Keyword.Type +' ' Text.Whitespace +'/* the index has to be an int */' Comment.Multiline +' ' Text.Whitespace +'a' Name +',' Punctuation +' ' Text.Whitespace +'// index into the array\n' Comment.Single + +' ' Text.Whitespace +'int' Keyword.Type +' ' Text.Whitespace +'*' Operator +'b' Name +' ' Text.Whitespace +'//the array @!\n' Comment.Single + +')' Punctuation +'\n' Text.Whitespace + +'/*\n The end of the func params @ (@ will result error if parsed incorrectly)\n*/' Comment.Multiline +'\n' Text.Whitespace + +'{' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'// yet another comment\n' Comment.Single + +' ' Text.Whitespace +'return' Keyword +' ' Text.Whitespace +'b' Name +'[' Punctuation +'a' Name +']' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'}' Punctuation +'\n' Text.Whitespace diff --git a/tests/c/test_label.txt b/tests/c/test_label.txt new file mode 100644 index 0000000..6f4ee80 --- /dev/null +++ b/tests/c/test_label.txt @@ -0,0 +1,31 @@ +---input--- +int main() +{ +foo: + goto foo; +} + +---tokens--- +'int' Keyword.Type +' ' Text.Whitespace +'main' Name.Function +'(' Punctuation +')' Punctuation +'\n' Text.Whitespace + +'{' Punctuation +'\n' Text.Whitespace + +'foo' Name.Label +':' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'goto' Keyword +' ' Text.Whitespace +'foo' Name +';' Punctuation +'\n' Text.Whitespace + +'}' Punctuation +'\n' Text.Whitespace diff --git a/tests/c/test_label_followed_by_statement.txt b/tests/c/test_label_followed_by_statement.txt new file mode 100644 index 0000000..6dc8468 --- /dev/null +++ b/tests/c/test_label_followed_by_statement.txt @@ -0,0 +1,35 @@ +---input--- +int main() +{ +foo:return 0; + goto foo; +} + +---tokens--- +'int' Keyword.Type +' ' Text.Whitespace +'main' Name.Function +'(' Punctuation +')' Punctuation +'\n' Text.Whitespace + +'{' Punctuation +'\n' Text.Whitespace + +'foo' Name.Label +':' Punctuation +'return' Keyword +' ' Text.Whitespace +'0' Literal.Number.Integer +';' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'goto' Keyword +' ' Text.Whitespace +'foo' Name +';' Punctuation +'\n' Text.Whitespace + +'}' Punctuation +'\n' Text.Whitespace diff --git a/tests/c/test_label_space_before_colon.txt b/tests/c/test_label_space_before_colon.txt new file mode 100644 index 0000000..2c3d065 --- /dev/null +++ b/tests/c/test_label_space_before_colon.txt @@ -0,0 +1,32 @@ +---input--- +int main() +{ +foo : + goto foo; +} + +---tokens--- +'int' Keyword.Type +' ' Text.Whitespace +'main' Name.Function +'(' Punctuation +')' Punctuation +'\n' Text.Whitespace + +'{' Punctuation +'\n' Text.Whitespace + +'foo' Name.Label +' ' Text.Whitespace +':' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'goto' Keyword +' ' Text.Whitespace +'foo' Name +';' Punctuation +'\n' Text.Whitespace + +'}' Punctuation +'\n' Text.Whitespace diff --git a/tests/c/test_numbers.txt b/tests/c/test_numbers.txt new file mode 100644 index 0000000..75af6fd --- /dev/null +++ b/tests/c/test_numbers.txt @@ -0,0 +1,20 @@ +---input--- +42 23.42 23. .42 023 0xdeadbeef 23e+42 42e-23 + +---tokens--- +'42' Literal.Number.Integer +' ' Text.Whitespace +'23.42' Literal.Number.Float +' ' Text.Whitespace +'23.' Literal.Number.Float +' ' Text.Whitespace +'.42' Literal.Number.Float +' ' Text.Whitespace +'023' Literal.Number.Oct +' ' Text.Whitespace +'0xdeadbeef' Literal.Number.Hex +' ' Text.Whitespace +'23e+42' Literal.Number.Float +' ' Text.Whitespace +'42e-23' Literal.Number.Float +'\n' Text.Whitespace diff --git a/tests/c/test_preproc_file.txt b/tests/c/test_preproc_file.txt new file mode 100644 index 0000000..2b5449e --- /dev/null +++ b/tests/c/test_preproc_file.txt @@ -0,0 +1,17 @@ +---input--- +#include +# include + +---tokens--- +'#' Comment.Preproc +'include' Comment.Preproc +' ' Text.Whitespace +'' Comment.PreprocFile +'\n' Comment.Preproc + +'#' Comment.Preproc +' ' Text.Whitespace +'include' Comment.Preproc +' ' Text.Whitespace +'' Comment.PreprocFile +'\n' Comment.Preproc diff --git a/tests/c/test_preproc_file2.txt b/tests/c/test_preproc_file2.txt new file mode 100644 index 0000000..5d6ef3b --- /dev/null +++ b/tests/c/test_preproc_file2.txt @@ -0,0 +1,17 @@ +---input--- +#include "foo.h" +# include "foo.h" + +---tokens--- +'#' Comment.Preproc +'include' Comment.Preproc +' ' Text.Whitespace +'"foo.h"' Comment.PreprocFile +'\n' Comment.Preproc + +'#' Comment.Preproc +' ' Text.Whitespace +'include' Comment.Preproc +' ' Text.Whitespace +'"foo.h"' Comment.PreprocFile +'\n' Comment.Preproc diff --git a/tests/c/test_preproc_file3.txt b/tests/c/test_preproc_file3.txt new file mode 100644 index 0000000..b36db4c --- /dev/null +++ b/tests/c/test_preproc_file3.txt @@ -0,0 +1,18 @@ +Space around line break before macro is valid C, but failed to parse previously. + +---input--- +foo(); + #define FOO 0 + +---tokens--- +'foo' Name +'(' Punctuation +')' Punctuation +';' Punctuation +' ' Text.Whitespace +'\n' Text.Whitespace + +' ' Text.Whitespace +'#' Comment.Preproc +'define FOO 0' Comment.Preproc +'\n' Comment.Preproc diff --git a/tests/c/test_preproc_file4.txt b/tests/c/test_preproc_file4.txt new file mode 100644 index 0000000..db1592d --- /dev/null +++ b/tests/c/test_preproc_file4.txt @@ -0,0 +1,13 @@ +Comments can precede preprocessor macros + +---input--- +/* Comment */ /* Another */ #define FOO 0 + +---tokens--- +'/* Comment */' Comment.Multiline +' ' Text.Whitespace +'/* Another */' Comment.Multiline +' ' Text.Whitespace +'#' Comment.Preproc +'define FOO 0' Comment.Preproc +'\n' Comment.Preproc diff --git a/tests/c/test_preproc_file5.txt b/tests/c/test_preproc_file5.txt new file mode 100644 index 0000000..f4a727b --- /dev/null +++ b/tests/c/test_preproc_file5.txt @@ -0,0 +1,19 @@ +Preprocessor macros should appear only at the beginning of the line. +Otherwise we should produce an error token. + +---input--- +foo(); #define FOO 0 + +---tokens--- +'foo' Name +'(' Punctuation +')' Punctuation +';' Punctuation +' ' Text.Whitespace +'#' Error +'define' Name +' ' Text.Whitespace +'FOO' Name +' ' Text.Whitespace +'0' Literal.Number.Integer +'\n' Text.Whitespace diff --git a/tests/c/test_string_resembling_decl_end.txt b/tests/c/test_string_resembling_decl_end.txt new file mode 100644 index 0000000..17ce223 --- /dev/null +++ b/tests/c/test_string_resembling_decl_end.txt @@ -0,0 +1,41 @@ +---input--- +// This should not be recognized as a function declaration followed by +// garbage. +string xyz(");"); + +// This should not be recognized as a function definition. + +string xyz("){ }"); + +---tokens--- +'// This should not be recognized as a function declaration followed by\n' Comment.Single + +'// garbage.\n' Comment.Single + +'string' Name +' ' Text.Whitespace +'xyz' Name +'(' Punctuation +'"' Literal.String +');' Literal.String +'"' Literal.String +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'// This should not be recognized as a function definition.\n' Comment.Single + +'\n' Text.Whitespace + +'string' Name +' ' Text.Whitespace +'xyz' Name +'(' Punctuation +'"' Literal.String +'){ }' Literal.String +'"' Literal.String +')' Punctuation +';' Punctuation +'\n' Text.Whitespace diff --git a/tests/c/test_switch.txt b/tests/c/test_switch.txt new file mode 100644 index 0000000..d1c1148 --- /dev/null +++ b/tests/c/test_switch.txt @@ -0,0 +1,56 @@ +---input--- +int main() +{ + switch (0) + { + case 0: + default: + ; + } +} + +---tokens--- +'int' Keyword.Type +' ' Text.Whitespace +'main' Name.Function +'(' Punctuation +')' Punctuation +'\n' Text.Whitespace + +'{' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'switch' Keyword +' ' Text.Whitespace +'(' Punctuation +'0' Literal.Number.Integer +')' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'{' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'case' Keyword +' ' Text.Whitespace +'0' Literal.Number.Integer +':' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'default' Keyword +':' Operator +'\n' Text.Whitespace + +' ' Text.Whitespace +';' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'}' Punctuation +'\n' Text.Whitespace + +'}' Punctuation +'\n' Text.Whitespace diff --git a/tests/c/test_switch_space_before_colon.txt b/tests/c/test_switch_space_before_colon.txt new file mode 100644 index 0000000..583f77c --- /dev/null +++ b/tests/c/test_switch_space_before_colon.txt @@ -0,0 +1,58 @@ +---input--- +int main() +{ + switch (0) + { + case 0 : + default : + ; + } +} + +---tokens--- +'int' Keyword.Type +' ' Text.Whitespace +'main' Name.Function +'(' Punctuation +')' Punctuation +'\n' Text.Whitespace + +'{' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'switch' Keyword +' ' Text.Whitespace +'(' Punctuation +'0' Literal.Number.Integer +')' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'{' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'case' Keyword +' ' Text.Whitespace +'0' Literal.Number.Integer +' ' Text.Whitespace +':' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'default' Keyword +' ' Text.Whitespace +':' Operator +'\n' Text.Whitespace + +' ' Text.Whitespace +';' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'}' Punctuation +'\n' Text.Whitespace + +'}' Punctuation +'\n' Text.Whitespace diff --git a/tests/carbon/unterminated_comment.txt b/tests/carbon/unterminated_comment.txt new file mode 100644 index 0000000..f0e9c5e --- /dev/null +++ b/tests/carbon/unterminated_comment.txt @@ -0,0 +1,80 @@ +---input--- +/* +This +is +a +long +unterminated +comment +with +many +newlines. + +It +should +take +a +reasonable +time +to +lex. + +---tokens--- +'/' Operator +'*' Operator +'\n' Text.Whitespace + +'This' Name.Other +'\n' Text.Whitespace + +'is' Keyword +'\n' Text.Whitespace + +'a' Name.Other +'\n' Text.Whitespace + +'long' Name.Other +'\n' Text.Whitespace + +'unterminated' Name.Other +'\n' Text.Whitespace + +'comment' Name.Other +'\n' Text.Whitespace + +'with' Name.Other +'\n' Text.Whitespace + +'many' Name.Other +'\n' Text.Whitespace + +'newlines' Name.Other +'.' Punctuation +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'It' Name.Other +'\n' Text.Whitespace + +'should' Name.Other +'\n' Text.Whitespace + +'take' Name.Other +'\n' Text.Whitespace + +'a' Name.Other +'\n' Text.Whitespace + +'reasonable' Name.Other +'\n' Text.Whitespace + +'time' Name.Other +'\n' Text.Whitespace + +'to' Name.Other +'\n' Text.Whitespace + +'lex' Name.Other +'.' Punctuation +'\n' Text.Whitespace diff --git a/tests/cfm/test_basic_comment.txt b/tests/cfm/test_basic_comment.txt new file mode 100644 index 0000000..c07a72a --- /dev/null +++ b/tests/cfm/test_basic_comment.txt @@ -0,0 +1,8 @@ +---input--- + + +---tokens--- +'' Comment.Multiline +'\n' Text diff --git a/tests/cfm/test_nested_comment.txt b/tests/cfm/test_nested_comment.txt new file mode 100644 index 0000000..f8aaf4c --- /dev/null +++ b/tests/cfm/test_nested_comment.txt @@ -0,0 +1,12 @@ +---input--- + ---> + +---tokens--- +'' Comment.Multiline +' ' Comment.Multiline +'--->' Comment.Multiline +'\n' Text diff --git a/tests/coffeescript/test_beware_infinite_loop.txt b/tests/coffeescript/test_beware_infinite_loop.txt new file mode 100644 index 0000000..886b706 --- /dev/null +++ b/tests/coffeescript/test_beware_infinite_loop.txt @@ -0,0 +1,14 @@ +# This demonstrates the case that "This isn't really guarding" comment +# refers to. + +---input--- +/a/x; + +---tokens--- +'' Text +'/' Operator +'a' Name.Other +'/' Operator +'x' Name.Other +';' Punctuation +'\n' Text.Whitespace diff --git a/tests/coffeescript/test_mixed_slashes.txt b/tests/coffeescript/test_mixed_slashes.txt new file mode 100644 index 0000000..8701fad --- /dev/null +++ b/tests/coffeescript/test_mixed_slashes.txt @@ -0,0 +1,13 @@ +---input--- +a?/foo/:1/2; + +---tokens--- +'a' Name.Other +'?' Operator +'/foo/' Literal.String.Regex +':' Operator +'1' Literal.Number.Integer +'/' Operator +'2' Literal.Number.Integer +';' Punctuation +'\n' Text.Whitespace diff --git a/tests/console/fake_ps2_prompt.txt b/tests/console/fake_ps2_prompt.txt new file mode 100644 index 0000000..683b36d --- /dev/null +++ b/tests/console/fake_ps2_prompt.txt @@ -0,0 +1,14 @@ +# Test that missing backslash means it's no prompt. + +---input--- +$ echo "> not a prompt" +> not a prompt + +---tokens--- +'$ ' Generic.Prompt +'echo' Name.Builtin +' ' Text.Whitespace +'"> not a prompt"' Literal.String.Double +'\n' Text.Whitespace + +'> not a prompt\n' Generic.Output diff --git a/tests/console/prompt_in_output.txt b/tests/console/prompt_in_output.txt new file mode 100644 index 0000000..993bc75 --- /dev/null +++ b/tests/console/prompt_in_output.txt @@ -0,0 +1,21 @@ +# Test that output that looks like a prompt is not detected as such. + +---input--- +$ cat \ +> test.txt +line1 +> file content, not prompt! + +---tokens--- +'$ ' Generic.Prompt +'cat' Text +' ' Text.Whitespace +'\\\n' Literal.String.Escape + +'> ' Generic.Prompt +'test.txt' Text +'\n' Text.Whitespace + +'line1\n' Generic.Output + +'> file content, not prompt!\n' Generic.Output diff --git a/tests/console/ps2_prompt.txt b/tests/console/ps2_prompt.txt new file mode 100644 index 0000000..175a473 --- /dev/null +++ b/tests/console/ps2_prompt.txt @@ -0,0 +1,15 @@ +---input--- +$ ls\ +> /does/not/exist +ls: cannot access ... + +---tokens--- +'$ ' Generic.Prompt +'ls' Text +'\\\n' Literal.String.Escape + +'> ' Generic.Prompt +'/does/not/exist' Text +'\n' Text.Whitespace + +'ls: cannot access ...\n' Generic.Output diff --git a/tests/console/test_comment_after_prompt.txt b/tests/console/test_comment_after_prompt.txt new file mode 100644 index 0000000..f115715 --- /dev/null +++ b/tests/console/test_comment_after_prompt.txt @@ -0,0 +1,6 @@ +---input--- +$# comment + +---tokens--- +'$' Generic.Prompt +'# comment\n' Comment.Single diff --git a/tests/console/test_newline_in_echo_no_ps2.txt b/tests/console/test_newline_in_echo_no_ps2.txt new file mode 100644 index 0000000..57a1190 --- /dev/null +++ b/tests/console/test_newline_in_echo_no_ps2.txt @@ -0,0 +1,16 @@ +---input--- +$ echo \ + hi +hi + +---tokens--- +'$ ' Generic.Prompt +'echo' Name.Builtin +' ' Text.Whitespace +'\\\n' Literal.String.Escape + +' ' Text.Whitespace +'hi' Text +'\n' Text.Whitespace + +'hi\n' Generic.Output diff --git a/tests/console/test_newline_in_echo_ps2.txt b/tests/console/test_newline_in_echo_ps2.txt new file mode 100644 index 0000000..b90eead --- /dev/null +++ b/tests/console/test_newline_in_echo_ps2.txt @@ -0,0 +1,16 @@ +---input--- +$ echo \ +> hi +hi + +---tokens--- +'$ ' Generic.Prompt +'echo' Name.Builtin +' ' Text.Whitespace +'\\\n' Literal.String.Escape + +'> ' Generic.Prompt +'hi' Text +'\n' Text.Whitespace + +'hi\n' Generic.Output diff --git a/tests/console/test_newline_in_ls_no_ps2.txt b/tests/console/test_newline_in_ls_no_ps2.txt new file mode 100644 index 0000000..3366bc0 --- /dev/null +++ b/tests/console/test_newline_in_ls_no_ps2.txt @@ -0,0 +1,16 @@ +---input--- +$ ls \ + hi +hi + +---tokens--- +'$ ' Generic.Prompt +'ls' Text +' ' Text.Whitespace +'\\\n' Literal.String.Escape + +' ' Text.Whitespace +'hi' Text +'\n' Text.Whitespace + +'hi\n' Generic.Output diff --git a/tests/console/test_newline_in_ls_ps2.txt b/tests/console/test_newline_in_ls_ps2.txt new file mode 100644 index 0000000..bf1bae5 --- /dev/null +++ b/tests/console/test_newline_in_ls_ps2.txt @@ -0,0 +1,16 @@ +---input--- +$ ls \ +> hi +hi + +---tokens--- +'$ ' Generic.Prompt +'ls' Text +' ' Text.Whitespace +'\\\n' Literal.String.Escape + +'> ' Generic.Prompt +'hi' Text +'\n' Text.Whitespace + +'hi\n' Generic.Output diff --git a/tests/console/test_virtualenv.txt b/tests/console/test_virtualenv.txt new file mode 100644 index 0000000..420b07a --- /dev/null +++ b/tests/console/test_virtualenv.txt @@ -0,0 +1,11 @@ +---input--- +(env) [~/project]$ foo -h + +---tokens--- +'(env)' Generic.Prompt.VirtualEnv +' ' Text +'[~/project]$ ' Generic.Prompt +'foo' Text +' ' Text.Whitespace +'-h' Text +'\n' Text.Whitespace diff --git a/tests/coq/test_unicode.txt b/tests/coq/test_unicode.txt new file mode 100644 index 0000000..2007f7d --- /dev/null +++ b/tests/coq/test_unicode.txt @@ -0,0 +1,15 @@ +---input--- +Check (α ≻ β). + +---tokens--- +'Check' Keyword.Namespace +' ' Text +'(' Operator +'α' Name +' ' Text +'≻' Name.Builtin.Pseudo +' ' Text +'β' Name +')' Operator +'.' Operator +'\n' Text diff --git a/tests/cpp/alternative_tokens.txt b/tests/cpp/alternative_tokens.txt new file mode 100644 index 0000000..69919cc --- /dev/null +++ b/tests/cpp/alternative_tokens.txt @@ -0,0 +1,46 @@ +---input--- +and +and_eq +bitand +bitor +compl +not +not_eq +or +or_eq +xor +xor_eq + +---tokens--- +'and' Keyword +'\n' Text.Whitespace + +'and_eq' Keyword +'\n' Text.Whitespace + +'bitand' Keyword +'\n' Text.Whitespace + +'bitor' Keyword +'\n' Text.Whitespace + +'compl' Keyword +'\n' Text.Whitespace + +'not' Keyword +'\n' Text.Whitespace + +'not_eq' Keyword +'\n' Text.Whitespace + +'or' Keyword +'\n' Text.Whitespace + +'or_eq' Keyword +'\n' Text.Whitespace + +'xor' Keyword +'\n' Text.Whitespace + +'xor_eq' Keyword +'\n' Text.Whitespace diff --git a/tests/cpp/extension_keywords.txt b/tests/cpp/extension_keywords.txt new file mode 100644 index 0000000..e47eb33 --- /dev/null +++ b/tests/cpp/extension_keywords.txt @@ -0,0 +1,17 @@ +---input--- +__restrict +__int128 +_BitInt(2) + +---tokens--- +'__restrict' Keyword +'\n' Text.Whitespace + +'__int128' Keyword.Type +'\n' Text.Whitespace + +'_BitInt' Keyword.Type +'(' Punctuation +'2' Literal.Number.Integer +')' Punctuation +'\n' Text.Whitespace diff --git a/tests/cpp/test_good_comment.txt b/tests/cpp/test_good_comment.txt new file mode 100644 index 0000000..115516d --- /dev/null +++ b/tests/cpp/test_good_comment.txt @@ -0,0 +1,6 @@ +---input--- +/* foo */ + +---tokens--- +'/* foo */' Comment.Multiline +'\n' Text.Whitespace diff --git a/tests/cpp/test_open_comment.txt b/tests/cpp/test_open_comment.txt new file mode 100644 index 0000000..3799214 --- /dev/null +++ b/tests/cpp/test_open_comment.txt @@ -0,0 +1,5 @@ +---input--- +/* foo + +---tokens--- +'/* foo\n' Comment.Multiline diff --git a/tests/cpp/test_unicode_identifiers.txt b/tests/cpp/test_unicode_identifiers.txt new file mode 100644 index 0000000..c2f3e03 --- /dev/null +++ b/tests/cpp/test_unicode_identifiers.txt @@ -0,0 +1,146 @@ +---input--- +namespace 𝐨𝐩𝐭𝐢𝐨𝐧 { + int _hello(); +} + +int cześć = 2; + +class 𝐨𝐩𝐭𝐢𝐨𝐧𝐬 final { + 𝐨𝐩𝐭𝐢𝐨𝐧𝐬() { + 爴: + int a = 𝐨𝐩𝐭𝐢𝐨𝐧::hello(); + }; + + static 𝐨𝐩𝐭𝐢𝐨𝐧𝐬 𝔡𝔢𝔣𝔞𝔲𝔩𝔱; + static 𝐨𝐩𝐭𝐢𝐨𝐧𝐬 𝔢𝔵𝔠𝔢𝔭𝔱𝔦𝔬𝔫𝔰; +}; + +enum class ⅭⅤ { red, green = 15, blue }; + +---tokens--- +'namespace' Keyword +' ' Text.Whitespace +'𝐨𝐩𝐭𝐢𝐨𝐧' Name.Namespace +' ' Text.Whitespace +'{' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'int' Keyword.Type +' ' Text.Whitespace +'_hello' Name.Function +'(' Punctuation +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'}' Punctuation +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'int' Keyword.Type +' ' Text.Whitespace +'cześć' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'2' Literal.Number.Integer +';' Punctuation +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'class' Keyword +' ' Text.Whitespace +'𝐨𝐩𝐭𝐢𝐨𝐧𝐬' Name.Class +' ' Text.Whitespace +'final' Keyword +' ' Text.Whitespace +'{' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'𝐨𝐩𝐭𝐢𝐨𝐧𝐬' Name +'(' Punctuation +')' Punctuation +' ' Text.Whitespace +'{' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'爴' Name.Label +':' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'int' Keyword.Type +' ' Text.Whitespace +'a' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'𝐨𝐩𝐭𝐢𝐨𝐧' Name +':' Operator +':' Operator +'hello' Name +'(' Punctuation +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'}' Punctuation +';' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'\n' Text.Whitespace + +' ' Text.Whitespace +'static' Keyword +' ' Text.Whitespace +'𝐨𝐩𝐭𝐢𝐨𝐧𝐬' Name +' ' Text.Whitespace +'𝔡𝔢𝔣𝔞𝔲𝔩𝔱' Name +';' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'static' Keyword +' ' Text.Whitespace +'𝐨𝐩𝐭𝐢𝐨𝐧𝐬' Name +' ' Text.Whitespace +'𝔢𝔵𝔠𝔢𝔭𝔱𝔦𝔬𝔫𝔰' Name +';' Punctuation +'\n' Text.Whitespace + +'}' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'enum' Keyword +' ' Text.Whitespace +'class' Keyword +' ' Text.Whitespace +'ⅭⅤ' Name.Class +' ' Text.Whitespace +'{' Punctuation +' ' Text.Whitespace +'red' Name +',' Punctuation +' ' Text.Whitespace +'green' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'15' Literal.Number.Integer +',' Punctuation +' ' Text.Whitespace +'blue' Name +' ' Text.Whitespace +'}' Punctuation +';' Punctuation +'\n' Text.Whitespace diff --git a/tests/crystal/test_annotation.txt b/tests/crystal/test_annotation.txt new file mode 100644 index 0000000..b585fa1 --- /dev/null +++ b/tests/crystal/test_annotation.txt @@ -0,0 +1,16 @@ +---input--- +@[FOO::Bar::Baz(opt: "xx")] + +---tokens--- +'@[' Operator +'FOO::Bar::Baz' Name.Decorator +'(' Punctuation +'opt' Literal.String.Symbol +':' Punctuation +' ' Text.Whitespace +'"' Literal.String.Double +'xx' Literal.String.Double +'"' Literal.String.Double +')' Punctuation +']' Operator +'\n' Text.Whitespace diff --git a/tests/crystal/test_array_access.txt b/tests/crystal/test_array_access.txt new file mode 100644 index 0000000..6b36c51 --- /dev/null +++ b/tests/crystal/test_array_access.txt @@ -0,0 +1,11 @@ +---input--- +[5][5]? + +---tokens--- +'[' Operator +'5' Literal.Number.Integer +']' Operator +'[' Operator +'5' Literal.Number.Integer +']?' Operator +'\n' Text.Whitespace diff --git a/tests/crystal/test_chars.txt b/tests/crystal/test_chars.txt new file mode 100644 index 0000000..c254bfb --- /dev/null +++ b/tests/crystal/test_chars.txt @@ -0,0 +1,25 @@ +---input--- +'a' +'я' +'\u{1234}' +' +' +'abc' + +---tokens--- +"'a'" Literal.String.Char +'\n' Text.Whitespace + +"'я'" Literal.String.Char +'\n' Text.Whitespace + +"'\\u{1234}'" Literal.String.Char +'\n' Text.Whitespace + +"'\n'" Literal.String.Char +'\n' Text.Whitespace + +"'" Error +'abc' Name +"'" Error +'\n' Text.Whitespace diff --git a/tests/crystal/test_constant_and_module.txt b/tests/crystal/test_constant_and_module.txt new file mode 100644 index 0000000..f8d33ff --- /dev/null +++ b/tests/crystal/test_constant_and_module.txt @@ -0,0 +1,14 @@ +---input--- +HTTP +HTTP::Server.new + +---tokens--- +'HTTP' Name.Constant +'\n' Text.Whitespace + +'HTTP' Name +'::' Operator +'Server' Name +'.' Operator +'new' Name +'\n' Text.Whitespace diff --git a/tests/crystal/test_empty_percent_strings.txt b/tests/crystal/test_empty_percent_strings.txt new file mode 100644 index 0000000..3cc0844 --- /dev/null +++ b/tests/crystal/test_empty_percent_strings.txt @@ -0,0 +1,27 @@ +---input--- +%() +%[] +%{} +%<> +%|| + +---tokens--- +'%(' Literal.String.Other +')' Literal.String.Other +'\n' Text.Whitespace + +'%[' Literal.String.Other +']' Literal.String.Other +'\n' Text.Whitespace + +'%{' Literal.String.Other +'}' Literal.String.Other +'\n' Text.Whitespace + +'%<' Literal.String.Other +'>' Literal.String.Other +'\n' Text.Whitespace + +'%|' Literal.String.Other +'|' Literal.String.Other +'\n' Text.Whitespace diff --git a/tests/crystal/test_escaped_bracestring.txt b/tests/crystal/test_escaped_bracestring.txt new file mode 100644 index 0000000..14718b9 --- /dev/null +++ b/tests/crystal/test_escaped_bracestring.txt @@ -0,0 +1,19 @@ +---input--- +str.gsub(%r{\\\\}, "/") + +---tokens--- +'str' Name +'.' Operator +'gsub' Name +'(' Punctuation +'%r{' Literal.String.Regex +'\\\\' Literal.String.Regex +'\\\\' Literal.String.Regex +'}' Literal.String.Regex +',' Punctuation +' ' Text.Whitespace +'"' Literal.String.Double +'/' Literal.String.Double +'"' Literal.String.Double +')' Punctuation +'\n' Text.Whitespace diff --git a/tests/crystal/test_escaped_interpolation.txt b/tests/crystal/test_escaped_interpolation.txt new file mode 100644 index 0000000..c623464 --- /dev/null +++ b/tests/crystal/test_escaped_interpolation.txt @@ -0,0 +1,9 @@ +---input--- +"\#{a + b}" + +---tokens--- +'"' Literal.String.Double +'\\#' Literal.String.Escape +'{a + b}' Literal.String.Double +'"' Literal.String.Double +'\n' Text.Whitespace diff --git a/tests/crystal/test_interpolation_nested_curly.txt b/tests/crystal/test_interpolation_nested_curly.txt new file mode 100644 index 0000000..f4a69f7 --- /dev/null +++ b/tests/crystal/test_interpolation_nested_curly.txt @@ -0,0 +1,56 @@ +---input--- +"A#{ (3..5).group_by { |x| x/2}.map do |k,v| "#{k}" end.join }" + "Z" + +---tokens--- +'"' Literal.String.Double +'A' Literal.String.Double +'#{' Literal.String.Interpol +' ' Text.Whitespace +'(' Punctuation +'3' Literal.Number.Integer +'..' Operator +'5' Literal.Number.Integer +')' Punctuation +'.' Operator +'group_by' Name +' ' Text.Whitespace +'{' Literal.String.Interpol +' ' Text.Whitespace +'|' Operator +'x' Name +'|' Operator +' ' Text.Whitespace +'x' Name +'/' Operator +'2' Literal.Number.Integer +'}' Literal.String.Interpol +'.' Operator +'map' Name +' ' Text.Whitespace +'do' Keyword +' ' Text.Whitespace +'|' Operator +'k' Name +',' Punctuation +'v' Name +'|' Operator +' ' Text.Whitespace +'"' Literal.String.Double +'#{' Literal.String.Interpol +'k' Name +'}' Literal.String.Interpol +'"' Literal.String.Double +' ' Text.Whitespace +'end' Keyword +'.' Operator +'join' Name +' ' Text.Whitespace +'}' Literal.String.Interpol +'"' Literal.String.Double +' ' Text.Whitespace +'+' Operator +' ' Text.Whitespace +'"' Literal.String.Double +'Z' Literal.String.Double +'"' Literal.String.Double +'\n' Text.Whitespace diff --git a/tests/crystal/test_lib.txt b/tests/crystal/test_lib.txt new file mode 100644 index 0000000..6f6f107 --- /dev/null +++ b/tests/crystal/test_lib.txt @@ -0,0 +1,58 @@ +---input--- +@[Link("some")] +lib LibSome +@[CallConvention("X86_StdCall")] +fun foo="some.foo"(thing : Void*) : LibC::Int +end + +---tokens--- +'@[' Operator +'Link' Name.Decorator +'(' Punctuation +'"' Literal.String.Double +'some' Literal.String.Double +'"' Literal.String.Double +')' Punctuation +']' Operator +'\n' Text.Whitespace + +'lib' Keyword +' ' Text.Whitespace +'LibSome' Name.Namespace +'\n' Text.Whitespace + +'@[' Operator +'CallConvention' Name.Decorator +'(' Punctuation +'"' Literal.String.Double +'X86_StdCall' Literal.String.Double +'"' Literal.String.Double +')' Punctuation +']' Operator +'\n' Text.Whitespace + +'fun' Keyword +' ' Text.Whitespace +'foo' Name.Function +'=' Operator +'"' Literal.String.Double +'some.foo' Literal.String.Double +'"' Literal.String.Double +'(' Punctuation +'thing' Name +' ' Text.Whitespace +':' Punctuation +' ' Text.Whitespace +'Void' Name +'*' Operator +')' Punctuation +' ' Text.Whitespace +':' Punctuation +' ' Text.Whitespace +'LibC' Name +'::' Operator +'Int' Name +'\n' Text.Whitespace + +'end' Keyword +'\n' Text.Whitespace diff --git a/tests/crystal/test_macro.txt b/tests/crystal/test_macro.txt new file mode 100644 index 0000000..765caa2 --- /dev/null +++ b/tests/crystal/test_macro.txt @@ -0,0 +1,76 @@ +---input--- +def<=>(other : self) : Int +{%for field in %w(first_name middle_name last_name)%} +cmp={{field.id}}<=>other.{{field.id}} +return cmp if cmp!=0 +{%end%} +0 +end + +---tokens--- +'def' Keyword +'<=>' Name.Function +'(' Punctuation +'other' Name +' ' Text.Whitespace +':' Punctuation +' ' Text.Whitespace +'self' Keyword +')' Punctuation +' ' Text.Whitespace +':' Punctuation +' ' Text.Whitespace +'Int' Name +'\n' Text.Whitespace + +'{%' Literal.String.Interpol +'for' Keyword +' ' Text.Whitespace +'field' Name +' ' Text.Whitespace +'in' Keyword +' ' Text.Whitespace +'%w(' Literal.String.Other +'first_name middle_name last_name' Literal.String.Other +')' Literal.String.Other +'%}' Literal.String.Interpol +'\n' Text.Whitespace + +'cmp' Name +'=' Operator +'{{' Literal.String.Interpol +'field' Name +'.' Operator +'id' Name +'}}' Literal.String.Interpol +'<=>' Operator +'other' Name +'.' Operator +'{{' Literal.String.Interpol +'field' Name +'.' Operator +'id' Name +'}}' Literal.String.Interpol +'\n' Text.Whitespace + +'return' Keyword +' ' Text.Whitespace +'cmp' Name +' ' Text.Whitespace +'if' Keyword +' ' Text.Whitespace +'cmp' Name +'!=' Operator +'0' Literal.Number.Integer +'\n' Text.Whitespace + +'{%' Literal.String.Interpol +'end' Keyword +'%}' Literal.String.Interpol +'\n' Text.Whitespace + +'0' Literal.Number.Integer +'\n' Text.Whitespace + +'end' Keyword +'\n' Text.Whitespace diff --git a/tests/crystal/test_numbers.txt b/tests/crystal/test_numbers.txt new file mode 100644 index 0000000..0ddfec2 --- /dev/null +++ b/tests/crystal/test_numbers.txt @@ -0,0 +1,84 @@ +---input--- +# Integers +0 +1 +1_000_000 +1u8 +11231231231121312i64 + +# Floats +0.0 +1.0_f32 +1_f32 +0f64 +1e+4 +1e111 +1_234.567_890 + +# Error +01 +0b2 +0x129g2 +0o12358 + +---tokens--- +'# Integers' Comment.Single +'\n' Text.Whitespace + +'0' Literal.Number.Integer +'\n' Text.Whitespace + +'1' Literal.Number.Integer +'\n' Text.Whitespace + +'1_000_000' Literal.Number.Integer +'\n' Text.Whitespace + +'1u8' Literal.Number.Integer +'\n' Text.Whitespace + +'11231231231121312i64' Literal.Number.Integer +'\n\n' Text.Whitespace + +'# Floats' Comment.Single +'\n' Text.Whitespace + +'0.0' Literal.Number.Float +'\n' Text.Whitespace + +'1.0_f32' Literal.Number.Float +'\n' Text.Whitespace + +'1_f32' Literal.Number.Float +'\n' Text.Whitespace + +'0f64' Literal.Number.Float +'\n' Text.Whitespace + +'1e+4' Literal.Number.Float +'\n' Text.Whitespace + +'1e111' Literal.Number.Float +'\n' Text.Whitespace + +'1_234.567_890' Literal.Number.Float +'\n\n' Text.Whitespace + +'# Error' Comment.Single +'\n' Text.Whitespace + +'0' Error +'1' Literal.Number.Integer +'\n' Text.Whitespace + +'0' Error +'b2' Name +'\n' Text.Whitespace + +'0' Error +'x129g2' Name +'\n' Text.Whitespace + +'0' Error +'o12358' Name +'\n' Text.Whitespace diff --git a/tests/crystal/test_operator_methods.txt b/tests/crystal/test_operator_methods.txt new file mode 100644 index 0000000..084771f --- /dev/null +++ b/tests/crystal/test_operator_methods.txt @@ -0,0 +1,18 @@ +---input--- +([] of Int32).[]?(5) + +---tokens--- +'(' Punctuation +'[' Operator +']' Operator +' ' Text.Whitespace +'of' Keyword +' ' Text.Whitespace +'Int32' Name +')' Punctuation +'.' Operator +'[]?' Name.Operator +'(' Punctuation +'5' Literal.Number.Integer +')' Punctuation +'\n' Text.Whitespace diff --git a/tests/crystal/test_percent_strings.txt b/tests/crystal/test_percent_strings.txt new file mode 100644 index 0000000..80f247c --- /dev/null +++ b/tests/crystal/test_percent_strings.txt @@ -0,0 +1,41 @@ +---input--- +%(hello ("world")) +%[hello ["world"]] +%{hello "world"} +%> +%|hello "world"| + +---tokens--- +'%(' Literal.String.Other +'hello ' Literal.String.Other +'(' Literal.String.Other +'"world"' Literal.String.Other +')' Literal.String.Other +')' Literal.String.Other +'\n' Text.Whitespace + +'%[' Literal.String.Other +'hello ' Literal.String.Other +'[' Literal.String.Other +'"world"' Literal.String.Other +']' Literal.String.Other +']' Literal.String.Other +'\n' Text.Whitespace + +'%{' Literal.String.Other +'hello "world"' Literal.String.Other +'}' Literal.String.Other +'\n' Text.Whitespace + +'%<' Literal.String.Other +'hello ' Literal.String.Other +'<' Literal.String.Other +'"world"' Literal.String.Other +'>' Literal.String.Other +'>' Literal.String.Other +'\n' Text.Whitespace + +'%|' Literal.String.Other +'hello "world"' Literal.String.Other +'|' Literal.String.Other +'\n' Text.Whitespace diff --git a/tests/crystal/test_percent_strings_special.txt b/tests/crystal/test_percent_strings_special.txt new file mode 100644 index 0000000..4ca1c0b --- /dev/null +++ b/tests/crystal/test_percent_strings_special.txt @@ -0,0 +1,31 @@ +---input--- +%Q(hello \n #{name}) +%q(hello \n #{name}) +%w(foo\nbar baz) + +---tokens--- +'%Q(' Literal.String.Other +'hello ' Literal.String.Other +'\\n' Literal.String.Escape +' ' Literal.String.Other +'#{' Literal.String.Interpol +'name' Name +'}' Literal.String.Interpol +')' Literal.String.Other +'\n' Text.Whitespace + +'%q(' Literal.String.Other +'hello ' Literal.String.Other +'\\' Literal.String.Other +'n ' Literal.String.Other +'#' Literal.String.Other +'{name}' Literal.String.Other +')' Literal.String.Other +'\n' Text.Whitespace + +'%w(' Literal.String.Other +'foo' Literal.String.Other +'\\' Literal.String.Other +'nbar baz' Literal.String.Other +')' Literal.String.Other +'\n' Text.Whitespace diff --git a/tests/crystal/test_pseudo_builtins.txt b/tests/crystal/test_pseudo_builtins.txt new file mode 100644 index 0000000..9c8c917 --- /dev/null +++ b/tests/crystal/test_pseudo_builtins.txt @@ -0,0 +1,20 @@ +---input--- +record Cls do +def_equals s +end + +---tokens--- +'record' Name.Builtin.Pseudo +' ' Text.Whitespace +'Cls' Name +' ' Text.Whitespace +'do' Keyword +'\n' Text.Whitespace + +'def_equals' Name.Builtin.Pseudo +' ' Text.Whitespace +'s' Name +'\n' Text.Whitespace + +'end' Keyword +'\n' Text.Whitespace diff --git a/tests/crystal/test_pseudo_keywords.txt b/tests/crystal/test_pseudo_keywords.txt new file mode 100644 index 0000000..18827ad --- /dev/null +++ b/tests/crystal/test_pseudo_keywords.txt @@ -0,0 +1,50 @@ +---input--- +def f(x : T, line = __LINE__) forall T +if x.is_a?(String) +pp! x +end +end + +---tokens--- +'def' Keyword +' ' Text.Whitespace +'f' Name.Function +'(' Punctuation +'x' Name +' ' Text.Whitespace +':' Punctuation +' ' Text.Whitespace +'T' Name +',' Punctuation +' ' Text.Whitespace +'line' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'__LINE__' Keyword.Pseudo +')' Punctuation +' ' Text.Whitespace +'forall' Keyword.Pseudo +' ' Text.Whitespace +'T' Name +'\n' Text.Whitespace + +'if' Keyword +' ' Text.Whitespace +'x' Name +'.is_a?' Keyword.Pseudo +'(' Punctuation +'String' Name +')' Punctuation +'\n' Text.Whitespace + +'pp!' Name.Builtin.Pseudo +' ' Text.Whitespace +'x' Name +'\n' Text.Whitespace + +'end' Keyword +'\n' Text.Whitespace + +'end' Keyword +'\n' Text.Whitespace diff --git a/tests/crystal/test_range_syntax1.txt b/tests/crystal/test_range_syntax1.txt new file mode 100644 index 0000000..a3ba24a --- /dev/null +++ b/tests/crystal/test_range_syntax1.txt @@ -0,0 +1,8 @@ +---input--- +1...3 + +---tokens--- +'1' Literal.Number.Integer +'...' Operator +'3' Literal.Number.Integer +'\n' Text.Whitespace diff --git a/tests/crystal/test_range_syntax2.txt b/tests/crystal/test_range_syntax2.txt new file mode 100644 index 0000000..08bf4b1 --- /dev/null +++ b/tests/crystal/test_range_syntax2.txt @@ -0,0 +1,10 @@ +---input--- +1 .. 3 + +---tokens--- +'1' Literal.Number.Integer +' ' Text.Whitespace +'..' Operator +' ' Text.Whitespace +'3' Literal.Number.Integer +'\n' Text.Whitespace diff --git a/tests/crystal/test_string_escapes.txt b/tests/crystal/test_string_escapes.txt new file mode 100644 index 0000000..3ea12f4 --- /dev/null +++ b/tests/crystal/test_string_escapes.txt @@ -0,0 +1,58 @@ +---input--- +"a\nz" +"a\az" +"a\xffz" +"a\u1234z" +"a\000z" +"a\u{0}z" +"a\u{10AfF9}z" + +---tokens--- +'"' Literal.String.Double +'a' Literal.String.Double +'\\n' Literal.String.Escape +'z' Literal.String.Double +'"' Literal.String.Double +'\n' Text.Whitespace + +'"' Literal.String.Double +'a' Literal.String.Double +'\\a' Literal.String.Escape +'z' Literal.String.Double +'"' Literal.String.Double +'\n' Text.Whitespace + +'"' Literal.String.Double +'a' Literal.String.Double +'\\xff' Literal.String.Escape +'z' Literal.String.Double +'"' Literal.String.Double +'\n' Text.Whitespace + +'"' Literal.String.Double +'a' Literal.String.Double +'\\u1234' Literal.String.Escape +'z' Literal.String.Double +'"' Literal.String.Double +'\n' Text.Whitespace + +'"' Literal.String.Double +'a' Literal.String.Double +'\\000' Literal.String.Escape +'z' Literal.String.Double +'"' Literal.String.Double +'\n' Text.Whitespace + +'"' Literal.String.Double +'a' Literal.String.Double +'\\u{0}' Literal.String.Escape +'z' Literal.String.Double +'"' Literal.String.Double +'\n' Text.Whitespace + +'"' Literal.String.Double +'a' Literal.String.Double +'\\u{10AfF9}' Literal.String.Escape +'z' Literal.String.Double +'"' Literal.String.Double +'\n' Text.Whitespace diff --git a/tests/crystal/test_symbols.txt b/tests/crystal/test_symbols.txt new file mode 100644 index 0000000..3d2cb0a --- /dev/null +++ b/tests/crystal/test_symbols.txt @@ -0,0 +1,20 @@ +---input--- +:sym_bol +:あ +:question? +:"symbol" + +---tokens--- +':sym_bol' Literal.String.Symbol +'\n' Text.Whitespace + +':あ' Literal.String.Symbol +'\n' Text.Whitespace + +':question?' Literal.String.Symbol +'\n' Text.Whitespace + +':"' Literal.String.Symbol +'symbol' Literal.String.Symbol +'"' Literal.String.Symbol +'\n' Text.Whitespace diff --git a/tests/csound/test_braced_strings.txt b/tests/csound/test_braced_strings.txt new file mode 100644 index 0000000..6921af9 --- /dev/null +++ b/tests/csound/test_braced_strings.txt @@ -0,0 +1,11 @@ +---input--- +{{ +characters$MACRO. +}} + +---tokens--- +'{{' Literal.String +'\ncharacters$MACRO.\n' Literal.String + +'}}' Literal.String +'\n' Text.Whitespace diff --git a/tests/csound/test_comments.txt b/tests/csound/test_comments.txt new file mode 100644 index 0000000..e660c2b --- /dev/null +++ b/tests/csound/test_comments.txt @@ -0,0 +1,16 @@ +---input--- +/* + * comment + */ +; comment +// comment + +---tokens--- +'/*\n * comment\n */' Comment.Multiline +'\n' Text.Whitespace + +'; comment' Comment.Single +'\n' Text.Whitespace + +'// comment' Comment.Single +'\n' Text.Whitespace diff --git a/tests/csound/test_escape_sequences.txt b/tests/csound/test_escape_sequences.txt new file mode 100644 index 0000000..8e7ba10 --- /dev/null +++ b/tests/csound/test_escape_sequences.txt @@ -0,0 +1,122 @@ +---input--- +"\\" +{{\\}} +"\a" +{{\a}} +"\b" +{{\b}} +"\n" +{{\n}} +"\r" +{{\r}} +"\t" +{{\t}} +"\"" +{{\"}} +"\012" +{{\012}} +"\345" +{{\345}} +"\67" +{{\67}} + +---tokens--- +'"' Literal.String +'\\\\' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'{{' Literal.String +'\\\\' Literal.String.Escape +'}}' Literal.String +'\n' Text.Whitespace + +'"' Literal.String +'\\a' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'{{' Literal.String +'\\a' Literal.String.Escape +'}}' Literal.String +'\n' Text.Whitespace + +'"' Literal.String +'\\b' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'{{' Literal.String +'\\b' Literal.String.Escape +'}}' Literal.String +'\n' Text.Whitespace + +'"' Literal.String +'\\n' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'{{' Literal.String +'\\n' Literal.String.Escape +'}}' Literal.String +'\n' Text.Whitespace + +'"' Literal.String +'\\r' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'{{' Literal.String +'\\r' Literal.String.Escape +'}}' Literal.String +'\n' Text.Whitespace + +'"' Literal.String +'\\t' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'{{' Literal.String +'\\t' Literal.String.Escape +'}}' Literal.String +'\n' Text.Whitespace + +'"' Literal.String +'\\"' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'{{' Literal.String +'\\"' Literal.String.Escape +'}}' Literal.String +'\n' Text.Whitespace + +'"' Literal.String +'\\012' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'{{' Literal.String +'\\012' Literal.String.Escape +'}}' Literal.String +'\n' Text.Whitespace + +'"' Literal.String +'\\345' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'{{' Literal.String +'\\345' Literal.String.Escape +'}}' Literal.String +'\n' Text.Whitespace + +'"' Literal.String +'\\67' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'{{' Literal.String +'\\67' Literal.String.Escape +'}}' Literal.String +'\n' Text.Whitespace diff --git a/tests/csound/test_function_like_macro_definitions.txt b/tests/csound/test_function_like_macro_definitions.txt new file mode 100644 index 0000000..555f4ad --- /dev/null +++ b/tests/csound/test_function_like_macro_definitions.txt @@ -0,0 +1,44 @@ +---input--- +#define MACRO(ARG1#ARG2) #macro_body# +#define/**/ +MACRO(ARG1'ARG2' ARG3)/**/ +#\#macro +body\## + +---tokens--- +'#define' Comment.Preproc +' ' Text.Whitespace +'MACRO' Comment.Preproc +'(' Punctuation +'ARG1' Comment.Preproc +'#' Punctuation +'ARG2' Comment.Preproc +')' Punctuation +' ' Text.Whitespace +'#' Punctuation +'macro_body' Comment.Preproc +'#' Punctuation +'\n' Text.Whitespace + +'#define' Comment.Preproc +'/**/' Comment.Multiline +'\n' Text.Whitespace + +'MACRO' Comment.Preproc +'(' Punctuation +'ARG1' Comment.Preproc +"'" Punctuation +'ARG2' Comment.Preproc +"'" Punctuation +' ' Text.Whitespace +'ARG3' Comment.Preproc +')' Punctuation +'/**/' Comment.Multiline +'\n' Text.Whitespace + +'#' Punctuation +'\\#' Comment.Preproc +'macro\nbody' Comment.Preproc +'\\#' Comment.Preproc +'#' Punctuation +'\n' Text.Whitespace diff --git a/tests/csound/test_function_like_macros.txt b/tests/csound/test_function_like_macros.txt new file mode 100644 index 0000000..955cc18 --- /dev/null +++ b/tests/csound/test_function_like_macros.txt @@ -0,0 +1,40 @@ +---input--- +$MACRO.(((x#y\)))' "(#'x)\)x\))"# {{x\))x)\)(#'}}); + +---tokens--- +'$MACRO.' Comment.Preproc +'(' Punctuation +'(' Comment.Preproc +'(' Comment.Preproc +'x#y\\)' Comment.Preproc +')' Comment.Preproc +')' Comment.Preproc +"'" Punctuation +' ' Comment.Preproc +'"' Literal.String +'(' Error +'#' Error +"'" Error +'x' Literal.String +')' Error +'\\)' Comment.Preproc +'x' Literal.String +'\\)' Comment.Preproc +')' Error +'"' Literal.String +'#' Punctuation +' ' Comment.Preproc +'{{' Literal.String +'x' Literal.String +'\\)' Comment.Preproc +')' Error +'x' Literal.String +')' Error +'\\)' Comment.Preproc +'(' Error +'#' Error +"'" Error +'}}' Literal.String +')' Punctuation +';' Comment.Single +'\n' Text.Whitespace diff --git a/tests/csound/test_global_value_identifiers.txt b/tests/csound/test_global_value_identifiers.txt new file mode 100644 index 0000000..4604200 --- /dev/null +++ b/tests/csound/test_global_value_identifiers.txt @@ -0,0 +1,30 @@ +---input--- +0dbfs +A4 +kr +ksmps +nchnls +nchnls_i +sr + +---tokens--- +'0dbfs' Name.Variable.Global +'\n' Text.Whitespace + +'A4' Name.Variable.Global +'\n' Text.Whitespace + +'kr' Name.Variable.Global +'\n' Text.Whitespace + +'ksmps' Name.Variable.Global +'\n' Text.Whitespace + +'nchnls' Name.Variable.Global +'\n' Text.Whitespace + +'nchnls_i' Name.Variable.Global +'\n' Text.Whitespace + +'sr' Name.Variable.Global +'\n' Text.Whitespace diff --git a/tests/csound/test_goto_statements.txt b/tests/csound/test_goto_statements.txt new file mode 100644 index 0000000..95a76ab --- /dev/null +++ b/tests/csound/test_goto_statements.txt @@ -0,0 +1,176 @@ +---input--- +goto aLabel +igoto aLabel +kgoto aLabel +reinit aLabel +rigoto aLabel +tigoto aLabel +cggoto 1==0, aLabel +cigoto 1==0, aLabel +cingoto 1==0, aLabel +ckgoto 1==0, aLabel +cngoto 1==0, aLabel +cnkgoto 1==0, aLabel +timout 0, 0, aLabel +loop_ge 0, 0, 0, aLabel +loop_gt 0, 0, 0, aLabel +loop_le 0, 0, 0, aLabel +loop_lt 0, 0, 0, aLabel + +---tokens--- +'goto' Keyword +' ' Text.Whitespace +'aLabel' Name.Label +'\n' Text.Whitespace + +'igoto' Keyword +' ' Text.Whitespace +'aLabel' Name.Label +'\n' Text.Whitespace + +'kgoto' Keyword +' ' Text.Whitespace +'aLabel' Name.Label +'\n' Text.Whitespace + +'reinit' Keyword.Pseudo +' ' Text.Whitespace +'aLabel' Name.Label +'\n' Text.Whitespace + +'rigoto' Keyword.Pseudo +' ' Text.Whitespace +'aLabel' Name.Label +'\n' Text.Whitespace + +'tigoto' Keyword.Pseudo +' ' Text.Whitespace +'aLabel' Name.Label +'\n' Text.Whitespace + +'cggoto' Keyword.Pseudo +' ' Text.Whitespace +'1' Literal.Number.Integer +'==' Operator +'0' Literal.Number.Integer +',' Punctuation +' ' Text.Whitespace +'aLabel' Name.Label +'\n' Text.Whitespace + +'cigoto' Keyword.Pseudo +' ' Text.Whitespace +'1' Literal.Number.Integer +'==' Operator +'0' Literal.Number.Integer +',' Punctuation +' ' Text.Whitespace +'aLabel' Name.Label +'\n' Text.Whitespace + +'cingoto' Keyword.Pseudo +' ' Text.Whitespace +'1' Literal.Number.Integer +'==' Operator +'0' Literal.Number.Integer +',' Punctuation +' ' Text.Whitespace +'aLabel' Name.Label +'\n' Text.Whitespace + +'ckgoto' Keyword.Pseudo +' ' Text.Whitespace +'1' Literal.Number.Integer +'==' Operator +'0' Literal.Number.Integer +',' Punctuation +' ' Text.Whitespace +'aLabel' Name.Label +'\n' Text.Whitespace + +'cngoto' Keyword.Pseudo +' ' Text.Whitespace +'1' Literal.Number.Integer +'==' Operator +'0' Literal.Number.Integer +',' Punctuation +' ' Text.Whitespace +'aLabel' Name.Label +'\n' Text.Whitespace + +'cnkgoto' Keyword.Pseudo +' ' Text.Whitespace +'1' Literal.Number.Integer +'==' Operator +'0' Literal.Number.Integer +',' Punctuation +' ' Text.Whitespace +'aLabel' Name.Label +'\n' Text.Whitespace + +'timout' Keyword.Pseudo +' ' Text.Whitespace +'0' Literal.Number.Integer +',' Punctuation +' ' Text.Whitespace +'0' Literal.Number.Integer +',' Punctuation +' ' Text.Whitespace +'aLabel' Name.Label +'\n' Text.Whitespace + +'loop_ge' Keyword.Pseudo +' ' Text.Whitespace +'0' Literal.Number.Integer +',' Punctuation +' ' Text.Whitespace +'0' Literal.Number.Integer +',' Punctuation +' ' Text.Whitespace +'0' Literal.Number.Integer +',' Punctuation +' ' Text.Whitespace +'aLabel' Name.Label +'\n' Text.Whitespace + +'loop_gt' Keyword.Pseudo +' ' Text.Whitespace +'0' Literal.Number.Integer +',' Punctuation +' ' Text.Whitespace +'0' Literal.Number.Integer +',' Punctuation +' ' Text.Whitespace +'0' Literal.Number.Integer +',' Punctuation +' ' Text.Whitespace +'aLabel' Name.Label +'\n' Text.Whitespace + +'loop_le' Keyword.Pseudo +' ' Text.Whitespace +'0' Literal.Number.Integer +',' Punctuation +' ' Text.Whitespace +'0' Literal.Number.Integer +',' Punctuation +' ' Text.Whitespace +'0' Literal.Number.Integer +',' Punctuation +' ' Text.Whitespace +'aLabel' Name.Label +'\n' Text.Whitespace + +'loop_lt' Keyword.Pseudo +' ' Text.Whitespace +'0' Literal.Number.Integer +',' Punctuation +' ' Text.Whitespace +'0' Literal.Number.Integer +',' Punctuation +' ' Text.Whitespace +'0' Literal.Number.Integer +',' Punctuation +' ' Text.Whitespace +'aLabel' Name.Label +'\n' Text.Whitespace diff --git a/tests/csound/test_include_directives.txt b/tests/csound/test_include_directives.txt new file mode 100644 index 0000000..b90f300 --- /dev/null +++ b/tests/csound/test_include_directives.txt @@ -0,0 +1,14 @@ +---input--- +#include/**/"file.udo" +#include/**/|file.udo| + +---tokens--- +'#include' Comment.Preproc +'/**/' Comment.Multiline +'"file.udo"' Literal.String +'\n' Text.Whitespace + +'#include' Comment.Preproc +'/**/' Comment.Multiline +'|file.udo|' Literal.String +'\n' Text.Whitespace diff --git a/tests/csound/test_includestr_directives.txt b/tests/csound/test_includestr_directives.txt new file mode 100644 index 0000000..0511c5d --- /dev/null +++ b/tests/csound/test_includestr_directives.txt @@ -0,0 +1,11 @@ +---input--- +#includestr/**/"$MACRO..udo" + +---tokens--- +'#includestr' Comment.Preproc +'/**/' Comment.Multiline +'"' Literal.String +'$MACRO.' Comment.Preproc +'.udo' Literal.String +'"' Literal.String +'\n' Text.Whitespace diff --git a/tests/csound/test_instrument_blocks.txt b/tests/csound/test_instrument_blocks.txt new file mode 100644 index 0000000..b6b0ba9 --- /dev/null +++ b/tests/csound/test_instrument_blocks.txt @@ -0,0 +1,42 @@ +---input--- +instr/**/1,/**/N_a_M_e_,/**/+Name/**/// + iDuration = p3 + outc:a(aSignal) +endin + +---tokens--- +'instr' Keyword.Declaration +'/**/' Comment.Multiline +'1' Name.Function +',' Punctuation +'/**/' Comment.Multiline +'N_a_M_e_' Name.Function +',' Punctuation +'/**/' Comment.Multiline +'+' Punctuation +'Name' Name.Function +'/**/' Comment.Multiline +'//' Comment.Single +'\n' Text.Whitespace + +' ' Text.Whitespace +'i' Keyword.Type +'Duration' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'p3' Name.Variable.Instance +'\n' Text.Whitespace + +' ' Text.Whitespace +'outc' Name.Builtin +':' Punctuation +'a' Keyword.Type +'(' Punctuation +'a' Keyword.Type +'Signal' Name +')' Punctuation +'\n' Text.Whitespace + +'endin' Keyword.Declaration +'\n' Text.Whitespace diff --git a/tests/csound/test_keywords.txt b/tests/csound/test_keywords.txt new file mode 100644 index 0000000..eb67eca --- /dev/null +++ b/tests/csound/test_keywords.txt @@ -0,0 +1,62 @@ +---input--- +do +else +elseif +endif +enduntil +fi +if +ithen +kthen +od +then +until +while +return +rireturn + +---tokens--- +'do' Keyword +'\n' Text.Whitespace + +'else' Keyword +'\n' Text.Whitespace + +'elseif' Keyword +'\n' Text.Whitespace + +'endif' Keyword +'\n' Text.Whitespace + +'enduntil' Keyword +'\n' Text.Whitespace + +'fi' Keyword +'\n' Text.Whitespace + +'if' Keyword +'\n' Text.Whitespace + +'ithen' Keyword +'\n' Text.Whitespace + +'kthen' Keyword +'\n' Text.Whitespace + +'od' Keyword +'\n' Text.Whitespace + +'then' Keyword +'\n' Text.Whitespace + +'until' Keyword +'\n' Text.Whitespace + +'while' Keyword +'\n' Text.Whitespace + +'return' Keyword.Pseudo +'\n' Text.Whitespace + +'rireturn' Keyword.Pseudo +'\n' Text.Whitespace diff --git a/tests/csound/test_labels.txt b/tests/csound/test_labels.txt new file mode 100644 index 0000000..afa6be9 --- /dev/null +++ b/tests/csound/test_labels.txt @@ -0,0 +1,13 @@ +---input--- +aLabel: + label2: + +---tokens--- +'aLabel' Name.Label +':' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'label2' Name.Label +':' Punctuation +'\n' Text.Whitespace diff --git a/tests/csound/test_macro_preprocessor_directives.txt b/tests/csound/test_macro_preprocessor_directives.txt new file mode 100644 index 0000000..0a576f6 --- /dev/null +++ b/tests/csound/test_macro_preprocessor_directives.txt @@ -0,0 +1,20 @@ +---input--- +#ifdef MACRO +#ifndef MACRO +#undef MACRO + +---tokens--- +'#ifdef' Comment.Preproc +' ' Text.Whitespace +'MACRO' Comment.Preproc +'\n' Text.Whitespace + +'#ifndef' Comment.Preproc +' ' Text.Whitespace +'MACRO' Comment.Preproc +'\n' Text.Whitespace + +'#undef' Comment.Preproc +' ' Text.Whitespace +'MACRO' Comment.Preproc +'\n' Text.Whitespace diff --git a/tests/csound/test_name.txt b/tests/csound/test_name.txt new file mode 100644 index 0000000..7ae5d58 --- /dev/null +++ b/tests/csound/test_name.txt @@ -0,0 +1,9 @@ +---input--- +kG:V + +---tokens--- +'k' Keyword.Type +'G' Name +':' Punctuation +'V' Name +'\n' Text.Whitespace diff --git a/tests/csound/test_numbers.txt b/tests/csound/test_numbers.txt new file mode 100644 index 0000000..63a8106 --- /dev/null +++ b/tests/csound/test_numbers.txt @@ -0,0 +1,52 @@ +---input--- +123 0123456789 +0xabcdef0123456789 0XABCDEF +1e2 +3e+4 +5e-6 +7E8 +9E+0 +1E-2 +3. +4.56 +.789 + +---tokens--- +'123' Literal.Number.Integer +' ' Text.Whitespace +'0123456789' Literal.Number.Integer +'\n' Text.Whitespace + +'0x' Keyword.Type +'abcdef0123456789' Literal.Number.Hex +' ' Text.Whitespace +'0X' Keyword.Type +'ABCDEF' Literal.Number.Hex +'\n' Text.Whitespace + +'1e2' Literal.Number.Float +'\n' Text.Whitespace + +'3e+4' Literal.Number.Float +'\n' Text.Whitespace + +'5e-6' Literal.Number.Float +'\n' Text.Whitespace + +'7E8' Literal.Number.Float +'\n' Text.Whitespace + +'9E+0' Literal.Number.Float +'\n' Text.Whitespace + +'1E-2' Literal.Number.Float +'\n' Text.Whitespace + +'3.' Literal.Number.Float +'\n' Text.Whitespace + +'4.56' Literal.Number.Float +'\n' Text.Whitespace + +'.789' Literal.Number.Float +'\n' Text.Whitespace diff --git a/tests/csound/test_object_like_macro_definitions.txt b/tests/csound/test_object_like_macro_definitions.txt new file mode 100644 index 0000000..dd165f3 --- /dev/null +++ b/tests/csound/test_object_like_macro_definitions.txt @@ -0,0 +1,30 @@ +---input--- +# define MACRO#macro_body# +#define/**/ +MACRO/**/ +#\#macro +body\## + +---tokens--- +'# \tdefine' Comment.Preproc +' ' Text.Whitespace +'MACRO' Comment.Preproc +'#' Punctuation +'macro_body' Comment.Preproc +'#' Punctuation +'\n' Text.Whitespace + +'#define' Comment.Preproc +'/**/' Comment.Multiline +'\n' Text.Whitespace + +'MACRO' Comment.Preproc +'/**/' Comment.Multiline +'\n' Text.Whitespace + +'#' Punctuation +'\\#' Comment.Preproc +'macro\nbody' Comment.Preproc +'\\#' Comment.Preproc +'#' Punctuation +'\n' Text.Whitespace diff --git a/tests/csound/test_operators.txt b/tests/csound/test_operators.txt new file mode 100644 index 0000000..1154a44 --- /dev/null +++ b/tests/csound/test_operators.txt @@ -0,0 +1,114 @@ +---input--- ++ +- +~ +¬ +! +* +/ +^ +% +<< +>> +< +> +<= +>= +== +!= +& +# +| +&& +|| +? +: ++= +-= +*= +/= + +---tokens--- +'+' Operator +'\n' Text.Whitespace + +'-' Operator +'\n' Text.Whitespace + +'~' Operator +'\n' Text.Whitespace + +'¬' Operator +'\n' Text.Whitespace + +'!' Operator +'\n' Text.Whitespace + +'*' Operator +'\n' Text.Whitespace + +'/' Operator +'\n' Text.Whitespace + +'^' Operator +'\n' Text.Whitespace + +'%' Operator +'\n' Text.Whitespace + +'<<' Operator +'\n' Text.Whitespace + +'>>' Operator +'\n' Text.Whitespace + +'<' Operator +'\n' Text.Whitespace + +'>' Operator +'\n' Text.Whitespace + +'<=' Operator +'\n' Text.Whitespace + +'>=' Operator +'\n' Text.Whitespace + +'==' Operator +'\n' Text.Whitespace + +'!=' Operator +'\n' Text.Whitespace + +'&' Operator +'\n' Text.Whitespace + +'#' Operator +'\n' Text.Whitespace + +'|' Operator +'\n' Text.Whitespace + +'&&' Operator +'\n' Text.Whitespace + +'||' Operator +'\n' Text.Whitespace + +'?' Operator +'\n' Text.Whitespace + +':' Operator +'\n' Text.Whitespace + +'+=' Operator +'\n' Text.Whitespace + +'-=' Operator +'\n' Text.Whitespace + +'*=' Operator +'\n' Text.Whitespace + +'/=' Operator +'\n' Text.Whitespace diff --git a/tests/csound/test_other_preprocessor_directives.txt b/tests/csound/test_other_preprocessor_directives.txt new file mode 100644 index 0000000..95026a0 --- /dev/null +++ b/tests/csound/test_other_preprocessor_directives.txt @@ -0,0 +1,26 @@ +---input--- +#else +#end +#endif +### +@ 12345 +@@ 67890 + +---tokens--- +'#else' Comment.Preproc +'\n' Text.Whitespace + +'#end' Comment.Preproc +'\n' Text.Whitespace + +'#endif' Comment.Preproc +'\n' Text.Whitespace + +'###' Comment.Preproc +'\n' Text.Whitespace + +'@ \t12345' Comment.Preproc +'\n' Text.Whitespace + +'@@ \t67890' Comment.Preproc +'\n' Text.Whitespace diff --git a/tests/csound/test_printks_and_prints_escape_sequences.txt b/tests/csound/test_printks_and_prints_escape_sequences.txt new file mode 100644 index 0000000..3663dbb --- /dev/null +++ b/tests/csound/test_printks_and_prints_escape_sequences.txt @@ -0,0 +1,290 @@ +---input--- +printks "%!" +printks "%%" +printks "%n" +printks "%N" +printks "%r" +printks "%R" +printks "%t" +printks "%T" +printks "\\a" +printks "\\A" +printks "\\b" +printks "\\B" +printks "\\n" +printks "\\N" +printks "\\r" +printks "\\R" +printks "\\t" +printks "\\T" +prints "%!" +prints "%%" +prints "%n" +prints "%N" +prints "%r" +prints "%R" +prints "%t" +prints "%T" +prints "\\a" +prints "\\A" +prints "\\b" +prints "\\B" +prints "\\n" +prints "\\N" +prints "\\r" +prints "\\R" +prints "\\t" +prints "\\T" + +---tokens--- +'printks' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'%!' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'printks' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'%%' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'printks' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'%n' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'printks' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'%N' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'printks' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'%r' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'printks' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'%R' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'printks' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'%t' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'printks' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'%T' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'printks' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'\\\\a' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'printks' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'\\\\A' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'printks' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'\\\\b' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'printks' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'\\\\B' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'printks' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'\\\\n' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'printks' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'\\\\N' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'printks' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'\\\\r' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'printks' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'\\\\R' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'printks' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'\\\\t' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'printks' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'\\\\T' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'prints' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'%!' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'prints' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'%%' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'prints' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'%n' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'prints' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'%N' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'prints' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'%r' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'prints' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'%R' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'prints' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'%t' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'prints' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'%T' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'prints' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'\\\\a' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'prints' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'\\\\A' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'prints' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'\\\\b' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'prints' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'\\\\B' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'prints' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'\\\\n' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'prints' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'\\\\N' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'prints' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'\\\\r' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'prints' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'\\\\R' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'prints' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'\\\\t' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'prints' Name.Builtin +' ' Text.Whitespace +'"' Literal.String +'\\\\T' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace diff --git a/tests/csound/test_quoted_strings.txt b/tests/csound/test_quoted_strings.txt new file mode 100644 index 0000000..d7e828a --- /dev/null +++ b/tests/csound/test_quoted_strings.txt @@ -0,0 +1,9 @@ +---input--- +"characters$MACRO." + +---tokens--- +'"' Literal.String +'characters' Literal.String +'$MACRO.' Comment.Preproc +'"' Literal.String +'\n' Text.Whitespace diff --git a/tests/csound/test_user_defined_opcodes.txt b/tests/csound/test_user_defined_opcodes.txt new file mode 100644 index 0000000..6320566 --- /dev/null +++ b/tests/csound/test_user_defined_opcodes.txt @@ -0,0 +1,24 @@ +---input--- +opcode/**/aUDO,/**/i[],/**/aik// + aUDO +endop + +---tokens--- +'opcode' Keyword.Declaration +'/**/' Comment.Multiline +'aUDO' Name.Function +',' Punctuation +'/**/' Comment.Multiline +'i[]' Keyword.Type +',' Punctuation +'/**/' Comment.Multiline +'aik' Keyword.Type +'//' Comment.Single +'\n' Text.Whitespace + +' ' Text.Whitespace +'aUDO' Name.Function +'\n' Text.Whitespace + +'endop' Keyword.Declaration +'\n' Text.Whitespace diff --git a/tests/css/percent_in_func.txt b/tests/css/percent_in_func.txt new file mode 100644 index 0000000..9fce9bd --- /dev/null +++ b/tests/css/percent_in_func.txt @@ -0,0 +1,51 @@ +---input--- +.target-img { + position: absolute; + top: 25%; + left: 50%; + transform: translate(-50%, -50%); +} + +---tokens--- +'.' Punctuation +'target-img' Name.Class +' ' Text.Whitespace +'{' Punctuation +'\n ' Text.Whitespace +'position' Keyword +':' Punctuation +' ' Text.Whitespace +'absolute' Keyword.Constant +';' Punctuation +' \n ' Text.Whitespace +'top' Keyword +':' Punctuation +' ' Text.Whitespace +'25' Literal.Number.Integer +'%' Keyword.Type +';' Punctuation +' \n ' Text.Whitespace +'left' Keyword +':' Punctuation +' ' Text.Whitespace +'50' Literal.Number.Integer +'%' Keyword.Type +';' Punctuation +' \n ' Text.Whitespace +'transform' Keyword +':' Punctuation +' ' Text.Whitespace +'translate' Name.Builtin +'(' Punctuation +'-50' Literal.Number.Integer +'%' Keyword.Type +',' Punctuation +' ' Text.Whitespace +'-50' Literal.Number.Integer +'%' Keyword.Type +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'}' Punctuation +' \n' Text.Whitespace diff --git a/tests/desktop/example.txt b/tests/desktop/example.txt new file mode 100644 index 0000000..1ea630a --- /dev/null +++ b/tests/desktop/example.txt @@ -0,0 +1,107 @@ +---input--- +[Desktop Entry] +Version=1.0 +Type=Application +Name=Foo Viewer +Comment=The best viewer for Foo objects available! +Comment[fr]=Le meilleur visionneur d'objets Foo disponible ! +TryExec=fooview +Exec=fooview %F +Icon=fooview +MimeType=image/x-foo; +Actions=Gallery;Create; + +[Desktop Action Gallery] +Exec=fooview --gallery +Name=Browse Gallery + +[Desktop Action Create] +Exec=fooview --create-new +Name=Create a new Foo! +Icon=fooview-new + +---tokens--- +'[Desktop Entry]' Keyword +'\n' Text.Whitespace + +'Version' Name.Attribute +'=' Operator +'1.0' Literal.String +'\n' Text.Whitespace + +'Type' Name.Attribute +'=' Operator +'Application' Literal.String +'\n' Text.Whitespace + +'Name' Name.Attribute +'=' Operator +'Foo Viewer' Literal.String +'\n' Text.Whitespace + +'Comment' Name.Attribute +'=' Operator +'The best viewer for Foo objects available!' Literal.String +'\n' Text.Whitespace + +'Comment' Name.Attribute +'[fr]' Name.Namespace +'=' Operator +"Le meilleur visionneur d'objets Foo disponible !" Literal.String +'\n' Text.Whitespace + +'TryExec' Name.Attribute +'=' Operator +'fooview' Literal.String +'\n' Text.Whitespace + +'Exec' Name.Attribute +'=' Operator +'fooview %F' Literal.String +'\n' Text.Whitespace + +'Icon' Name.Attribute +'=' Operator +'fooview' Literal.String +'\n' Text.Whitespace + +'MimeType' Name.Attribute +'=' Operator +'image/x-foo;' Literal.String +'\n' Text.Whitespace + +'Actions' Name.Attribute +'=' Operator +'Gallery;Create;' Literal.String +'\n\n' Text.Whitespace + +'[Desktop Action Gallery]' Keyword +'\n' Text.Whitespace + +'Exec' Name.Attribute +'=' Operator +'fooview --gallery' Literal.String +'\n' Text.Whitespace + +'Name' Name.Attribute +'=' Operator +'Browse Gallery' Literal.String +'\n\n' Text.Whitespace + +'[Desktop Action Create]' Keyword +'\n' Text.Whitespace + +'Exec' Name.Attribute +'=' Operator +'fooview --create-new' Literal.String +'\n' Text.Whitespace + +'Name' Name.Attribute +'=' Operator +'Create a new Foo!' Literal.String +'\n' Text.Whitespace + +'Icon' Name.Attribute +'=' Operator +'fooview-new' Literal.String +'\n' Text.Whitespace diff --git a/tests/devicetree/test_fragment_out_of_root_node.txt b/tests/devicetree/test_fragment_out_of_root_node.txt new file mode 100644 index 0000000..eef534c --- /dev/null +++ b/tests/devicetree/test_fragment_out_of_root_node.txt @@ -0,0 +1,72 @@ +---input--- +nodelabel: node@0 { foo = "bar"; }; +nodelabel: node { foo = "bar"; }; +nodelabel0: nodelabel1: node@0 { foo = "bar"; }; + +---tokens--- +'nodelabel' Name.Label +':' Punctuation +' ' Text.Whitespace +'node' Name.Function +'@' Operator +'0' Literal.Number.Integer +' ' Comment.Multiline +'{' Punctuation +' ' Text.Whitespace +'foo' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'"' Literal.String +'bar' Literal.String +'"' Literal.String +';' Punctuation +' ' Text.Whitespace +'}' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'nodelabel' Name.Label +':' Punctuation +' ' Text.Whitespace +'node' Name.Function +' ' Comment.Multiline +'{' Punctuation +' ' Text.Whitespace +'foo' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'"' Literal.String +'bar' Literal.String +'"' Literal.String +';' Punctuation +' ' Text.Whitespace +'}' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'nodelabel0' Name.Label +':' Punctuation +' ' Text.Whitespace +'nodelabel1' Name.Label +':' Punctuation +' ' Text.Whitespace +'node' Name.Function +'@' Operator +'0' Literal.Number.Integer +' ' Comment.Multiline +'{' Punctuation +' ' Text.Whitespace +'foo' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'"' Literal.String +'bar' Literal.String +'"' Literal.String +';' Punctuation +' ' Text.Whitespace +'}' Punctuation +';' Punctuation +'\n' Text.Whitespace diff --git a/tests/diff/normal.txt b/tests/diff/normal.txt new file mode 100644 index 0000000..ae3fd5b --- /dev/null +++ b/tests/diff/normal.txt @@ -0,0 +1,38 @@ +---input--- +1,2d0 +< A +< A +4c2 +< C +--- +> F +5a4 +> E + +---tokens--- +'1,2d0' Generic.Subheading +'\n' Text.Whitespace + +'< A' Generic.Deleted +'\n' Text.Whitespace + +'< A' Generic.Deleted +'\n' Text.Whitespace + +'4c2' Generic.Subheading +'\n' Text.Whitespace + +'< C' Generic.Deleted +'\n' Text.Whitespace + +'---' Generic.Strong +'\n' Text.Whitespace + +'> F' Generic.Inserted +'\n' Text.Whitespace + +'5a4' Generic.Subheading +'\n' Text.Whitespace + +'> E' Generic.Inserted +'\n' Text.Whitespace diff --git a/tests/diff/unified.txt b/tests/diff/unified.txt new file mode 100644 index 0000000..360086b --- /dev/null +++ b/tests/diff/unified.txt @@ -0,0 +1,44 @@ +---input--- +--- old.txt 2023-01-17 21:02:15.449417575 -0700 ++++ new.txt 2023-01-17 21:02:12.489441682 -0700 +@@ -1,5 +1,4 @@ +-A +-A + B +-C ++F + D ++E + +---tokens--- +'--- old.txt\t2023-01-17 21:02:15.449417575 -0700' Generic.Deleted +'\n' Text.Whitespace + +'+++ new.txt\t2023-01-17 21:02:12.489441682 -0700' Generic.Inserted +'\n' Text.Whitespace + +'@@ -1,5 +1,4 @@' Generic.Subheading +'\n' Text.Whitespace + +'-A' Generic.Deleted +'\n' Text.Whitespace + +'-A' Generic.Deleted +'\n' Text.Whitespace + +' ' Text.Whitespace +'B' Text +'\n' Text.Whitespace + +'-C' Generic.Deleted +'\n' Text.Whitespace + +'+F' Generic.Inserted +'\n' Text.Whitespace + +' ' Text.Whitespace +'D' Text +'\n' Text.Whitespace + +'+E' Generic.Inserted +'\n' Text.Whitespace diff --git a/tests/doscon/test_gt_only.txt b/tests/doscon/test_gt_only.txt new file mode 100644 index 0000000..b37b8fa --- /dev/null +++ b/tests/doscon/test_gt_only.txt @@ -0,0 +1,11 @@ +---input--- +> py +hi + +---tokens--- +'>' Generic.Prompt +' ' Text +'py' Text +'\n' Text + +'hi\n' Generic.Output diff --git a/tests/elpi/test_catastrophic_backtracking.txt b/tests/elpi/test_catastrophic_backtracking.txt new file mode 100644 index 0000000..a14a054 --- /dev/null +++ b/tests/elpi/test_catastrophic_backtracking.txt @@ -0,0 +1,6 @@ +---input--- +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + +---tokens--- +'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' Text +'\n' Text.Whitespace diff --git a/tests/elpi/test_chr.txt b/tests/elpi/test_chr.txt new file mode 100644 index 0000000..75291a3 --- /dev/null +++ b/tests/elpi/test_chr.txt @@ -0,0 +1,54 @@ +---input--- +constraint foo, bar { + + :name "myrule" + rule (odd X) \ (even X) | true <=> fail. + +} +rule. % not a kwd + +---tokens--- +'constraint' Keyword.Declaration +' ' Text.Whitespace +'foo, bar ' Name.Function +'{' Text +'\n\n ' Text.Whitespace +':name' Keyword.Mode +' ' Text.Whitespace +'"' Literal.String.Double +'myrule' Literal.String.Double +'"' Literal.String.Double +'\n ' Text.Whitespace +'rule' Keyword.Declaration +' ' Text.Whitespace +'(' Text +'odd' Text +' ' Text.Whitespace +'X' Name.Variable +')' Operator +' ' Text.Whitespace +'\\' Keyword.Declaration +' ' Text.Whitespace +'(' Text +'even' Text +' ' Text.Whitespace +'X' Name.Variable +')' Operator +' ' Text.Whitespace +'|' Keyword.Declaration +' ' Text.Whitespace +'true' Text +' ' Text.Whitespace +'<=>' Keyword.Declaration +' ' Text.Whitespace +'fail' Text +'.' Operator +'\n\n' Text.Whitespace + +'}' Text +'\n' Text.Whitespace + +'rule' Text +'.' Operator +' ' Text.Whitespace +'% not a kwd\n' Comment diff --git a/tests/elpi/test_clause.txt b/tests/elpi/test_clause.txt new file mode 100644 index 0000000..e485753 --- /dev/null +++ b/tests/elpi/test_clause.txt @@ -0,0 +1,67 @@ +---input--- +true. +stop :- !. +of (fun F) :- pi x\ of x => of (F x). +match (uvar as Y) :- print Y. + +---tokens--- +'true' Text +'.' Operator +'\n' Text.Whitespace + +'stop' Text +' ' Text.Whitespace +':-' Keyword.Declaration +' ' Text.Whitespace +'!' Keyword.Declaration +'.' Operator +'\n' Text.Whitespace + +'of' Text +' ' Text.Whitespace +'(' Text +'fun' Text +' ' Text.Whitespace +'F' Name.Variable +')' Operator +' ' Text.Whitespace +':-' Keyword.Declaration +' ' Text.Whitespace +'pi' Keyword.Declaration +' ' Text.Whitespace +'x' Name.Variable +'\\' Text +' ' Text.Whitespace +'of' Text +' ' Text.Whitespace +'x' Text +' ' Text.Whitespace +'=>' Keyword.Declaration +' ' Text.Whitespace +'of' Text +' ' Text.Whitespace +'(' Text +'F' Name.Variable +' ' Text.Whitespace +'x' Text +')' Operator +'.' Operator +'\n' Text.Whitespace + +'match' Text +' ' Text.Whitespace +'(' Text +'uvar' Keyword.Declaration +' ' Text.Whitespace +'as' Keyword.Declaration +' ' Text.Whitespace +'Y' Name.Variable +')' Operator +' ' Text.Whitespace +':-' Keyword.Declaration +' ' Text.Whitespace +'print' Text +' ' Text.Whitespace +'Y' Name.Variable +'.' Operator +'\n' Text.Whitespace diff --git a/tests/elpi/test_namespace.txt b/tests/elpi/test_namespace.txt new file mode 100644 index 0000000..98a35b0 --- /dev/null +++ b/tests/elpi/test_namespace.txt @@ -0,0 +1,35 @@ +---input--- +namespace foo.bar { + baz :- std.do! []. +} +shorten foo. { bar }. + +---tokens--- +'namespace' Keyword.Declaration +' ' Text.Whitespace +'foo.bar' Text +' ' Text.Whitespace +'{\n' Text + +' ' Text.Whitespace +'baz' Text +' ' Text.Whitespace +':-' Keyword.Declaration +' ' Text.Whitespace +'std.do!' Text +' ' Text.Whitespace +'[]' Keyword.Declaration +'.' Operator +'\n' Text.Whitespace + +'}\n' Text + +'shorten' Keyword.Declaration +' ' Text.Whitespace +'foo.' Text +' ' Text.Whitespace +'{ ' Text +'bar' Text +' ' Text.Whitespace +'}.' Text +'\n' Text.Whitespace diff --git a/tests/elpi/test_pred.txt b/tests/elpi/test_pred.txt new file mode 100644 index 0000000..657c8fd --- /dev/null +++ b/tests/elpi/test_pred.txt @@ -0,0 +1,60 @@ +---input--- +pred p1. +pred p2 i:int, o:list A. +pred p3 i:(bool -> prop). +:index(_ 2) pred p4 i:int, i:A. + +---tokens--- +'pred' Keyword.Declaration +' ' Text.Whitespace +'p1' Name.Function +'.' Text +'\n' Text.Whitespace + +'pred' Keyword.Declaration +' ' Text.Whitespace +'p2' Name.Function +' ' Text.Whitespace +'i:' Keyword.Mode +'int' Keyword.Type +',' Text +' ' Text.Whitespace +'o:' Keyword.Mode +'list' Keyword.Type +' ' Text.Whitespace +'A' Keyword.Type +'.' Text +'\n' Text.Whitespace + +'pred' Keyword.Declaration +' ' Text.Whitespace +'p3' Name.Function +' ' Text.Whitespace +'i:' Keyword.Mode +'(' Keyword.Type +'bool' Keyword.Type +' ' Text.Whitespace +'->' Keyword.Type +' ' Text.Whitespace +'prop' Keyword.Type +')' Keyword.Type +'.' Text +'\n' Text.Whitespace + +':index' Keyword.Mode +'(' Text.Whitespace +'_ 2' Literal.Number.Integer +')' Text +' ' Text.Whitespace +'pred' Keyword.Declaration +' ' Text.Whitespace +'p4' Name.Function +' ' Text.Whitespace +'i:' Keyword.Mode +'int' Keyword.Type +',' Text +' ' Text.Whitespace +'i:' Keyword.Mode +'A' Keyword.Type +'.' Text +'\n' Text.Whitespace diff --git a/tests/elpi/test_quotations.txt b/tests/elpi/test_quotations.txt new file mode 100644 index 0000000..05352c7 --- /dev/null +++ b/tests/elpi/test_quotations.txt @@ -0,0 +1,33 @@ +---input--- +tc {{ Eq lp:A }} {{ f lp:B l } }} :- foo A B. + +---tokens--- +'tc' Text +' ' Text.Whitespace +'{{' Punctuation +' Eq ' Text +'lp:' Keyword +'A' Name.Variable +' ' Text +'}}' Punctuation +' ' Text.Whitespace +'{{' Punctuation +' f ' Text +'lp:' Keyword +'B' Name.Variable +' ' Text +'l' Text +' ' Text +'}' Text +' ' Text +'}}' Punctuation +' ' Text.Whitespace +':-' Keyword.Declaration +' ' Text.Whitespace +'foo' Text +' ' Text.Whitespace +'A' Name.Variable +' ' Text.Whitespace +'B' Name.Variable +'.' Operator +'\n' Text.Whitespace diff --git a/tests/elpi/test_type.txt b/tests/elpi/test_type.txt new file mode 100644 index 0000000..8a506ce --- /dev/null +++ b/tests/elpi/test_type.txt @@ -0,0 +1,112 @@ +---input--- +kind list type -> type. +type nil list A. +type cons A -> list A -> list A. +kind tm type. +type fun (tm -> tm) -> tm. +type app tm -> tm -> tm. +pred foo i:(tm -> tm), o:tm. + +---tokens--- +'kind' Keyword.Declaration +' ' Text.Whitespace +'list' Name.Function +' ' Text.Whitespace +'type' Keyword.Type +' ' Text.Whitespace +'->' Keyword.Type +' ' Text.Whitespace +'type' Keyword.Type +'.' Text +'\n' Text.Whitespace + +'type' Keyword.Declaration +' ' Text.Whitespace +'nil' Name.Function +' ' Text.Whitespace +'list' Keyword.Type +' ' Text.Whitespace +'A' Keyword.Type +'.' Text +'\n' Text.Whitespace + +'type' Keyword.Declaration +' ' Text.Whitespace +'cons' Name.Function +' ' Text.Whitespace +'A' Keyword.Type +' ' Text.Whitespace +'->' Keyword.Type +' ' Text.Whitespace +'list' Keyword.Type +' ' Text.Whitespace +'A' Keyword.Type +' ' Text.Whitespace +'->' Keyword.Type +' ' Text.Whitespace +'list' Keyword.Type +' ' Text.Whitespace +'A' Keyword.Type +'.' Text +'\n' Text.Whitespace + +'kind' Keyword.Declaration +' ' Text.Whitespace +'tm' Name.Function +' ' Text.Whitespace +'type' Keyword.Type +'.' Text +'\n' Text.Whitespace + +'type' Keyword.Declaration +' ' Text.Whitespace +'fun' Name.Function +' ' Text.Whitespace +'(' Keyword.Type +'tm' Keyword.Type +' ' Text.Whitespace +'->' Keyword.Type +' ' Text.Whitespace +'tm' Keyword.Type +')' Keyword.Type +' ' Text.Whitespace +'->' Keyword.Type +' ' Text.Whitespace +'tm' Keyword.Type +'.' Text +'\n' Text.Whitespace + +'type' Keyword.Declaration +' ' Text.Whitespace +'app' Name.Function +' ' Text.Whitespace +'tm' Keyword.Type +' ' Text.Whitespace +'->' Keyword.Type +' ' Text.Whitespace +'tm' Keyword.Type +' ' Text.Whitespace +'->' Keyword.Type +' ' Text.Whitespace +'tm' Keyword.Type +'.' Text +'\n' Text.Whitespace + +'pred' Keyword.Declaration +' ' Text.Whitespace +'foo' Name.Function +' ' Text.Whitespace +'i:' Keyword.Mode +'(' Keyword.Type +'tm' Keyword.Type +' ' Text.Whitespace +'->' Keyword.Type +' ' Text.Whitespace +'tm' Keyword.Type +')' Keyword.Type +',' Text +' ' Text.Whitespace +'o:' Keyword.Mode +'tm' Keyword.Type +'.' Text +'\n' Text.Whitespace diff --git a/tests/ezhil/test_function.txt b/tests/ezhil/test_function.txt new file mode 100644 index 0000000..f89a357 --- /dev/null +++ b/tests/ezhil/test_function.txt @@ -0,0 +1,100 @@ +---input--- +# (C) முத்தையா அண்ணாமலை 2013, 2015 +நிரல்பாகம் gcd ( x, y ) +மு = max(x,y) + q = min(x,y) + +@( q == 0 ) ஆனால் + பின்கொடு மு +முடி +பின்கொடு gcd( மு - q , q ) +முடி + +---tokens--- +'# (C) முத்தையா அண்ணாமலை 2013, 2015' Comment.Single +'\n' Text.Whitespace + +'நிரல்பாகம்' Keyword +' ' Text.Whitespace +'gcd' Name +' ' Text.Whitespace +'(' Punctuation +' ' Text.Whitespace +'x' Name +',' Operator +' ' Text.Whitespace +'y' Name +' ' Text.Whitespace +')' Punctuation +'\n' Text.Whitespace + +'மு' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'max' Name.Builtin +'(' Punctuation +'x' Name +',' Operator +'y' Name +')' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'q' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'min' Name.Builtin +'(' Punctuation +'x' Name +',' Operator +'y' Name +')' Punctuation +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'@' Operator +'(' Punctuation +' ' Text.Whitespace +'q' Name +' ' Text.Whitespace +'==' Operator +' ' Text.Whitespace +'0' Literal.Number.Integer +' ' Text.Whitespace +')' Punctuation +' ' Text.Whitespace +'ஆனால்' Keyword +'\n' Text.Whitespace + +' ' Text.Whitespace +'பின்கொடு' Keyword +' ' Text.Whitespace +'மு' Name +'\n' Text.Whitespace + +'முடி' Keyword +'\n' Text.Whitespace + +'பின்கொடு' Keyword +' ' Text.Whitespace +'gcd' Name +'(' Punctuation +' ' Text.Whitespace +'மு' Name +' ' Text.Whitespace +'-' Operator +' ' Text.Whitespace +'q' Name +' ' Text.Whitespace +',' Operator +' ' Text.Whitespace +'q' Name +' ' Text.Whitespace +')' Punctuation +'\n' Text.Whitespace + +'முடி' Keyword +'\n' Text.Whitespace diff --git a/tests/ezhil/test_gcd_expr.txt b/tests/ezhil/test_gcd_expr.txt new file mode 100644 index 0000000..f4aaed9 --- /dev/null +++ b/tests/ezhil/test_gcd_expr.txt @@ -0,0 +1,21 @@ +---input--- +1^3+(5-5)*gcd(a,b) + +---tokens--- +'1' Literal.Number.Integer +'^' Operator +'3' Literal.Number.Integer +'+' Operator +'(' Punctuation +'5' Literal.Number.Integer +'-' Operator +'5' Literal.Number.Integer +')' Punctuation +'*' Operator +'gcd' Name +'(' Punctuation +'a' Name +',' Operator +'b' Name +')' Punctuation +'\n' Text.Whitespace diff --git a/tests/ezhil/test_if_statement.txt b/tests/ezhil/test_if_statement.txt new file mode 100644 index 0000000..2fe35b7 --- /dev/null +++ b/tests/ezhil/test_if_statement.txt @@ -0,0 +1,28 @@ +---input--- +@( 0 > 3 ) ஆனால் + பதிப்பி "wont print" +முடி + +---tokens--- +'@' Operator +'(' Punctuation +' ' Text.Whitespace +'0' Literal.Number.Integer +' ' Text.Whitespace +'>' Operator +' ' Text.Whitespace +'3' Literal.Number.Integer +' ' Text.Whitespace +')' Punctuation +' ' Text.Whitespace +'ஆனால்' Keyword +'\n' Text.Whitespace + +' ' Text.Whitespace +'பதிப்பி' Keyword +' ' Text.Whitespace +'"wont print"' Literal.String +'\n' Text.Whitespace + +'முடி' Keyword +'\n' Text.Whitespace diff --git a/tests/ezhil/test_sum.txt b/tests/ezhil/test_sum.txt new file mode 100644 index 0000000..fa2063d --- /dev/null +++ b/tests/ezhil/test_sum.txt @@ -0,0 +1,8 @@ +---input--- +1+3 + +---tokens--- +'1' Literal.Number.Integer +'+' Operator +'3' Literal.Number.Integer +'\n' Text.Whitespace diff --git a/tests/fortran/test_string_cataback.txt b/tests/fortran/test_string_cataback.txt new file mode 100644 index 0000000..9f4b9f1 --- /dev/null +++ b/tests/fortran/test_string_cataback.txt @@ -0,0 +1,112 @@ +---input--- +! Bad string, there isn't an even number of backslashes. +! This should not cause catastrophic backtracking. +'\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\' + +---tokens--- +"! Bad string, there isn't an even number of backslashes.\n" Comment + +'! This should not cause catastrophic backtracking.\n' Comment + +"'" Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +'\\' Error +"'" Error +'\n' Text.Whitespace diff --git a/tests/gas/test_comments.txt b/tests/gas/test_comments.txt new file mode 100644 index 0000000..b385d27 --- /dev/null +++ b/tests/gas/test_comments.txt @@ -0,0 +1,29 @@ +---input--- +lock addq $0, /* comments */ (%rsp) /* +// comments +*/ xorq %rax, %rax // comments + +---tokens--- +'lock' Name.Attribute +' ' Text.Whitespace +'addq' Name.Function +' ' Text.Whitespace +'$0' Name.Constant +',' Punctuation +' ' Text.Whitespace +'/* comments */' Comment.Multiline +' ' Text.Whitespace +'(' Punctuation +'%rsp' Name.Variable +')' Punctuation +' ' Text.Whitespace +'/*\n// comments\n*/' Comment.Multiline +' ' Text.Whitespace +'xorq' Name.Function +' ' Text.Whitespace +'%rax' Name.Variable +',' Punctuation +' ' Text.Whitespace +'%rax' Name.Variable +' ' Text.Whitespace +'// comments\n' Comment.Single diff --git a/tests/gdscript/test_comment.txt b/tests/gdscript/test_comment.txt new file mode 100644 index 0000000..ee78bc5 --- /dev/null +++ b/tests/gdscript/test_comment.txt @@ -0,0 +1,6 @@ +---input--- +# Comment + +---tokens--- +'# Comment' Comment.Single +'\n' Text.Whitespace diff --git a/tests/gdscript/test_export_array.txt b/tests/gdscript/test_export_array.txt new file mode 100644 index 0000000..24a313a --- /dev/null +++ b/tests/gdscript/test_export_array.txt @@ -0,0 +1,17 @@ +---input--- +export (Array, AudioStream) var streams + +---tokens--- +'export' Keyword +' ' Text.Whitespace +'(' Punctuation +'Array' Name.Builtin.Type +',' Punctuation +' ' Text.Whitespace +'AudioStream' Name +')' Punctuation +' ' Text.Whitespace +'var' Keyword +' ' Text.Whitespace +'streams' Name +'\n' Text.Whitespace diff --git a/tests/gdscript/test_function_with_types.txt b/tests/gdscript/test_function_with_types.txt new file mode 100644 index 0000000..eedf371 --- /dev/null +++ b/tests/gdscript/test_function_with_types.txt @@ -0,0 +1,33 @@ +---input--- +func abc(arg: String) -> void: + print("Hello", arg) + +---tokens--- +'func' Keyword +' ' Text.Whitespace +'abc' Name +'(' Punctuation +'arg' Name +':' Punctuation +' ' Text.Whitespace +'String' Name.Builtin.Type +')' Punctuation +' ' Text.Whitespace +'-' Operator +'>' Operator +' ' Text.Whitespace +'void' Name.Builtin.Type +':' Punctuation +'\n' Text.Whitespace + +'\t' Text.Whitespace +'print' Name.Builtin +'(' Punctuation +'"' Literal.String.Double +'Hello' Literal.String.Double +'"' Literal.String.Double +',' Punctuation +' ' Text.Whitespace +'arg' Name +')' Punctuation +'\n' Text.Whitespace diff --git a/tests/gdscript/test_inner_class.txt b/tests/gdscript/test_inner_class.txt new file mode 100644 index 0000000..734242b --- /dev/null +++ b/tests/gdscript/test_inner_class.txt @@ -0,0 +1,20 @@ +---input--- +class InnerClass: + var a = 5 + +---tokens--- +'class' Keyword +' ' Text.Whitespace +'InnerClass' Name +':' Punctuation +'\n' Text.Whitespace + +'\t' Text.Whitespace +'var' Keyword +' ' Text.Whitespace +'a' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'5' Literal.Number.Integer +'\n' Text.Whitespace diff --git a/tests/gdscript/test_multiline_string.txt b/tests/gdscript/test_multiline_string.txt new file mode 100644 index 0000000..b098207 --- /dev/null +++ b/tests/gdscript/test_multiline_string.txt @@ -0,0 +1,8 @@ +---input--- +""" +Multiline +""" + +---tokens--- +'"""\nMultiline\n"""' Literal.String.Doc +'\n' Text.Whitespace diff --git a/tests/gdscript/test_signal.txt b/tests/gdscript/test_signal.txt new file mode 100644 index 0000000..43aa8ec --- /dev/null +++ b/tests/gdscript/test_signal.txt @@ -0,0 +1,15 @@ +---input--- +signal sig (arg1, arg2) + +---tokens--- +'signal' Keyword +' ' Text.Whitespace +'sig' Name +' ' Text.Whitespace +'(' Punctuation +'arg1' Name +',' Punctuation +' ' Text.Whitespace +'arg2' Name +')' Punctuation +'\n' Text.Whitespace diff --git a/tests/gdscript/test_simple_function.txt b/tests/gdscript/test_simple_function.txt new file mode 100644 index 0000000..2f444ef --- /dev/null +++ b/tests/gdscript/test_simple_function.txt @@ -0,0 +1,22 @@ +---input--- +func abc(arg): + print("Hello, World!") + +---tokens--- +'func' Keyword +' ' Text.Whitespace +'abc' Name +'(' Punctuation +'arg' Name +')' Punctuation +':' Punctuation +'\n' Text.Whitespace + +'\t' Text.Whitespace +'print' Name.Builtin +'(' Punctuation +'"' Literal.String.Double +'Hello, World!' Literal.String.Double +'"' Literal.String.Double +')' Punctuation +'\n' Text.Whitespace diff --git a/tests/gdscript/test_variable_declaration_and_assigment.txt b/tests/gdscript/test_variable_declaration_and_assigment.txt new file mode 100644 index 0000000..b2ee890 --- /dev/null +++ b/tests/gdscript/test_variable_declaration_and_assigment.txt @@ -0,0 +1,12 @@ +---input--- +var abc = 5.4 + +---tokens--- +'var' Keyword +' ' Text.Whitespace +'abc' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'5.4' Literal.Number.Float +'\n' Text.Whitespace diff --git a/tests/haskell/test_promoted_names.txt b/tests/haskell/test_promoted_names.txt new file mode 100644 index 0000000..42d7755 --- /dev/null +++ b/tests/haskell/test_promoted_names.txt @@ -0,0 +1,10 @@ +---input--- +'x ': '[] + +---tokens--- +"'x" Name +' ' Text.Whitespace +"':" Keyword.Type +' ' Text.Whitespace +"'[]" Keyword.Type +'\n' Text.Whitespace diff --git a/tests/html/css.txt b/tests/html/css.txt new file mode 100644 index 0000000..08b7e72 --- /dev/null +++ b/tests/html/css.txt @@ -0,0 +1,29 @@ +---input--- + + +---tokens--- +'<' Punctuation +'style' Name.Tag +'>' Punctuation +'\n' Text.Whitespace + +'.' Punctuation +'ui-helper-hidden' Name.Class +' ' Text.Whitespace +'{' Punctuation +' ' Text.Whitespace +'display' Keyword +':' Punctuation +' ' Text.Whitespace +'none' Keyword.Constant +' ' Text.Whitespace +'}' Punctuation +'\n' Text.Whitespace + +'<' Punctuation +'/' Punctuation +'style' Name.Tag +'>' Punctuation +'\n' Text diff --git a/tests/html/css_backtracking.txt b/tests/html/css_backtracking.txt new file mode 100644 index 0000000..7efb4c1 --- /dev/null +++ b/tests/html/css_backtracking.txt @@ -0,0 +1,93 @@ +---input--- +