mirror of
https://github.com/ralsina/tartrazine.git
synced 2024-11-13 23:12:24 +00:00
7 lines
232 B
Go
7 lines
232 B
Go
// Package tokenizer implements file tokenization used by the enry content
|
|
// classifier. This package is an implementation detail of enry and should not
|
|
// be imported by other packages.
|
|
package tokenizer
|
|
|
|
const byteLimit = 100000
|