Merge commit 'f955c625aded244864e83a872b396868a490dbc5' as 'go-enry'

go-enry/internal/tokenizer/common.go

@@ -0,0 +1,7 @@
+// Package tokenizer implements file tokenization used by the enry content
+// classifier. This package is an implementation detail of enry and should not
+// be imported by other packages.
+package tokenizer
+
+// ByteLimit defines the maximum prefix of an input text that will be tokenized.
+const ByteLimit = 100000