mirror of
https://github.com/ralsina/tartrazine.git
synced 2025-05-15 16:49:31 -03:00
Roberto Alsina
f955c625ad
git-subtree-dir: go-enry git-subtree-split: 7e3a9a7241b14559854ad041e73d5203a6d3272c
8 lines
313 B
Go
8 lines
313 B
Go
// Package tokenizer implements file tokenization used by the enry content
|
|
// classifier. This package is an implementation detail of enry and should not
|
|
// be imported by other packages.
|
|
package tokenizer
|
|
|
|
// ByteLimit defines the maximum prefix of an input text that will be tokenized.
|
|
const ByteLimit = 100000
|