ralsina/tartrazine
1
0
Fork 0
mirror of https://github.com/ralsina/tartrazine.git synced 2024-11-10 13:32:24 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Apply suggestions from review.

Browse Source 
Signed-off-by: M. J. Fromberger <michael.j.fromberger@gmail.com>
This commit is contained in:
M. J. Fromberger 2019-01-29 11:25:53 -08:00
parent 4027b494b3
commit dabb41527f
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
internal/tokenizer/tokenize.go
View File

@ -11,9 +11,9 @@ import (
const byteLimit = 100000
// Tokenize returns classification tokens from content. The tokens returned
// should match what the Linguist library returns. At most the first 100KB of
// content are tokenized.
// Tokenize returns language-agnostic lexical tokens from content. The tokens
// returned should match what the Linguist library returns. At most the first
// 100KB of content are tokenized.
func Tokenize(content []byte) []string {
if len(content) > byteLimit {
content = content[:byteLimit]