ralsina/tartrazine
1
0
Fork 0
mirror of https://github.com/ralsina/tartrazine.git synced 2025-06-19 06:33:06 -03:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Rearranged code

Browse Source
This commit is contained in:
Manuel Carmona
2017-05-29 10:05:16 +02:00
parent f8b8f7f5c4
commit 5b304524d1
52 changed files with 74004 additions and 80610 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
internal/tokenizer/tokenize.go
View File

@ -5,7 +5,13 @@ import (
"regexp"
)
const byteLimit = 100000
func Tokenize(content []byte) []string {
if len(content) > byteLimit {
content = content[:byteLimit]
}
tokens := make([][]byte, 0, 50)
for _, extract := range extractTokens {
var extractedTokens [][]byte