From dabb41527f55e6ef88f3a8302d6d3f243ce1f972 Mon Sep 17 00:00:00 2001 From: "M. J. Fromberger" Date: Tue, 29 Jan 2019 11:25:53 -0800 Subject: [PATCH] Apply suggestions from review. Signed-off-by: M. J. Fromberger --- internal/tokenizer/tokenize.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/tokenizer/tokenize.go b/internal/tokenizer/tokenize.go index d7f1c43..5c5094b 100644 --- a/internal/tokenizer/tokenize.go +++ b/internal/tokenizer/tokenize.go @@ -11,9 +11,9 @@ import ( const byteLimit = 100000 -// Tokenize returns classification tokens from content. The tokens returned -// should match what the Linguist library returns. At most the first 100KB of -// content are tokenized. +// Tokenize returns language-agnostic lexical tokens from content. The tokens +// returned should match what the Linguist library returns. At most the first +// 100KB of content are tokenized. func Tokenize(content []byte) []string { if len(content) > byteLimit { content = content[:byteLimit]