mirror of
https://github.com/ralsina/tartrazine.git
synced 2024-09-20 15:31:21 +00:00
Apply suggestions from review.
Signed-off-by: M. J. Fromberger <michael.j.fromberger@gmail.com>
This commit is contained in:
parent
4027b494b3
commit
dabb41527f
@ -11,9 +11,9 @@ import (
|
|||||||
|
|
||||||
const byteLimit = 100000
|
const byteLimit = 100000
|
||||||
|
|
||||||
// Tokenize returns classification tokens from content. The tokens returned
|
// Tokenize returns language-agnostic lexical tokens from content. The tokens
|
||||||
// should match what the Linguist library returns. At most the first 100KB of
|
// returned should match what the Linguist library returns. At most the first
|
||||||
// content are tokenized.
|
// 100KB of content are tokenized.
|
||||||
func Tokenize(content []byte) []string {
|
func Tokenize(content []byte) []string {
|
||||||
if len(content) > byteLimit {
|
if len(content) > byteLimit {
|
||||||
content = content[:byteLimit]
|
content = content[:byteLimit]
|
||||||
|
Loading…
Reference in New Issue
Block a user