ralsina/tartrazine
1
0
Fork 0
mirror of https://github.com/ralsina/tartrazine.git synced 2025-06-19 06:33:06 -03:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

refactor to build tags

Browse Source 
Signed-off-by: Alexander Bezzubov <bzz@apache.org>
This commit is contained in:
Alexander Bezzubov
2019-03-24 18:55:05 +01:00
parent 553399ed76
commit 8756fbdcb4
10 changed files with 141 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
internal/tokenizer/tokenize.go
View File

@ -1,6 +1,5 @@
// Package tokenizer implements file tokenization used by the enry content
// classifier. This package is an implementation detail of enry and should not
// be imported by other packages.
// +build !flex
package tokenizer
import (
@ -9,8 +8,6 @@ import (
"gopkg.in/src-d/enry.v1/regex"
)
const byteLimit = 100000
// Tokenize returns language-agnostic lexical tokens from content. The tokens
// returned should match what the Linguist library returns. At most the first
// 100KB of content are tokenized.