ralsina/tartrazine
1
0
Fork 0
mirror of https://github.com/ralsina/tartrazine.git synced 2024-11-10 05:22:23 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

tokenizer: fix a bug and regenerate the code \w latest Go

Browse Source 
See https://github.com/bzz/enry/pull/4 for details.

Test Plan:
 - go test ./...
This commit is contained in:
Alexander Bezzubov 2020-03-19 19:08:21 +01:00
parent e08125d7ee
commit e32a70a784
No known key found for this signature in database
GPG Key ID: 8039F5787EFCD05D
5 changed files with 192457 additions and 194383 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
.travis.yml
View File

@ -1,8 +1,9 @@
dist: trusty
language: go
go:
- '1.12.x'
- '1.11.x'
- '1.14.x'
- '1.13.x'
env:
global:
- GO_VERSION_FOR_JVM='1.11.x'
@ -12,8 +13,6 @@ env:
matrix:
- ONIGURUMA=0
- ONIGURUMA=1
matrix:
fast_finish: true
stages:
- name: test
@ -22,7 +21,6 @@ stages:
- name: publish
if: tag IS present
stage: test
install:
- >
if [[ "${ONIGURUMA}" -gt 0 ]]; then
@ -36,7 +34,7 @@ install:
sudo dpkg -i "libonig-dev_${ONIGURUMA_VERSION}-1_amd64.deb"
fi;
script:
- make test-coverage
- go test ./...
after_success:
- bash <(curl -s https://codecov.io/bash)

193399
data/frequencies.go
View File

File diff suppressed because it is too large Load Diff

193399
internal/code-generator/generator/test_files/frequencies.gold
View File

File diff suppressed because it is too large Load Diff

6
internal/tokenizer/tokenize.go
View File

@ -18,10 +18,6 @@ func Tokenize(content []byte) []string {
content = content[:ByteLimit]
}
// Copy the input so that changes wrought by the tokenization steps do not
// modify the caller's copy of the input. See #196.
content = append([]byte(nil), content...)
tokens := make([][]byte, 0, 50)
for _, extract := range extractTokens {
var extractedTokens [][]byte
@ -162,7 +158,7 @@ func extractAndReplaceSGML(content []byte) ([]byte, [][]byte) {
continue
}
token := append(match[1], '>')
token := append(append([]byte(nil), match[1]...), '>')
SGMLTokens = append(SGMLTokens, token)
attributes := getSGMLAttributes(match[0])
SGMLTokens = append(SGMLTokens, attributes...)

26
internal/tokenizer/tokenize_test.go
View File

@ -78,18 +78,19 @@ varBool = 3<=2>
)
var (
tokensFromTestContent = []string{"SHEBANG#!ruby", "SHEBANG#!node", "SHEBANG#!awk", "<!DOCTYPE>", "PUBLIC", "W3C", "DTD", "XHTML", "1", "0",
"Strict", "EN", "http", "www", "w3", "org", "TR", "xhtml1", "DTD", "xhtml1", "strict", "dtd", "<html>", "<head>", "<title>", "class=",
"</title>", "<style>", "<![CDATA[>", "example", "background", "color", "yellow", "</style>", "</head>", "<body>", "<div>", "<strong>",
"</strong>", "</div>", "</body>", "</html>", "(", "[", "]", ")", "[", "]", "{", "(", ")", "(", ")", "{", "}", "(", ")", ";", "{", ";",
"}", "]", "]", "#", "/usr/bin/ruby", "#", "/usr/bin/env", "node", "aaa", "#", "/usr/bin/env", "A", "B", "foo", "bar", "awk", "f", "#",
"python", "func", "Tokenize", "content", "byte", "string", "splitted", "bytes.Fields", "content", "tokens", "othercode", "ppp", "no",
"comment", "abb", "tokenByte", "notcatchasanumber", "number", "*", "anotherNumber", "if", "isTrue", "isToo", "b", "return", "tokens",
"oneBool", "varBool", "#ifndef", "#i", "m", "not", "a", "comment", "if", "the", "single", "line", "comment", "symbol", "is", "not",
"followed", "by", "a", "white", "PyErr_SetString", "PyExc_RuntimeError", "html", "PUBLIC", "xmlns", "id", "class", "This", "is", "a",
"XHTML", "sample", "file", "type", "#example", "background", "color", "yellow", "id", "Just", "a", "simple", "XHTML", "test", "page.",
"-", "|", "+", "&&", "<", "<", "-", "!", "!", "!", "=", "=", "!", ":", "=", ":", "=", ",", ",", "=", ">", ">", "=", "=", "=", "=", ">",
"'", ",", ">", "=", ">", "=", "=", ">", "=", ">", ":", ">", "=", ">"}
tokensFromTestContent = []string{"SHEBANG#!ruby", "SHEBANG#!node", "SHEBANG#!awk", "<!DOCTYPE>", "html", "PUBLIC",
"W3C", "DTD", "XHTML", "1", "0", "Strict", "EN", "http", "www", "w3", "org", "TR", "xhtml1", "DTD", "xhtml1",
"strict", "dtd", "<html>", "xmlns=", "<head>", "<title>", "id=", "class=", "</title>", "<style>", "type=",
"<![CDATA[>", "example", "background", "color", "yellow", "</style>", "</head>", "<body>", "<div>", "id=",
"<strong>", "</strong>", "</div>", "</body>", "</html>", "(", "[", "]", ")", "[", "]", "{", "(", ")", "(", ")",
"{", "}", "(", ")", ";", "#", "/usr/bin/ruby", "#", "/usr/bin/env", "node", "aaa", "#", "/usr/bin/env", "A",
"B", "foo", "bar", "awk", "f", "#", "python", "func", "Tokenize", "content", "byte", "string", "splitted",
"bytes.Fields", "content", "tokens", "othercode", "ppp", "no", "comment", "abb", "tokenByte",
"notcatchasanumber", "number", "*", "anotherNumber", "if", "isTrue", "isToo", "b", "return", "tokens",
"oneBool", "varBool", "#ifndef", "#i", "m", "not", "a", "comment", "if", "the", "single", "line", "comment",
"symbol", "is", "not", "followed", "by", "a", "white", "PyErr_SetString", "PyExc_RuntimeError", "This", "is",
"a", "XHTML", "sample", "file", "Just", "a", "simple", "XHTML", "test", "page.", "-", "|", "+", "&&", "<", "<",
"!", "!", "!", "=", "=", "!", ":", "=", ":", "=", ",", ",", "=", ">", ">", "=", "=", "=", "=", ">", "'", ","}
tests = []struct {
name string
@ -108,6 +109,7 @@ func TestTokenize(t *testing.T) {
after := string(test.content)
require.Equal(t, before, after, "the input slice was modified")
require.Equal(t, len(test.expected), len(tokens), fmt.Sprintf("token' slice length = %v, want %v", len(test.expected), len(tokens)))
for i, expectedToken := range test.expected {
assert.Equal(t, expectedToken, tokens[i], fmt.Sprintf("token = %v, want %v", tokens[i], expectedToken))
}