Merge pull request #15 from mcarmonaa/vendor

utils.go generation from vendor.yml

content.go

@@ -2,7 +2,7 @@ package slinguist
 
 // CODE GENERATED AUTOMATICALLY WITH github.com/src-d/simple-linguist/cli/slinguist-generate
 // THIS FILE SHOULD NOT BE EDITED BY HAND
-// Extracted from github/linguist commit: 3a2a62baad5363a604828706f8df730902a542b0
+// Extracted from github/linguist commit: dae33dc2b20cddc85d1300435c3be7118a7115a9
 
 import (
 	"path/filepath"

internal/code-generator/assets/vendor.go.tmpl

@@ -0,0 +1,13 @@
+package slinguist
+
+// CODE GENERATED AUTOMATICALLY WITH github.com/src-d/simple-linguist/cli/slinguist-generate
+// THIS FILE SHOULD NOT BE EDITED BY HAND
+// Extracted from github/linguist commit: {{ getCommit }}
+
+import "gopkg.in/toqueteos/substring.v1"
+
+var vendorMatchers = substring.Or(
+	{{range $regexp := . -}}
+	substring.Regexp(`{{ $regexp }}`),
+	{{end -}}
+)

internal/code-generator/generator/generator_test.go

@@ -13,6 +13,7 @@ const (
 	// FromFile test
 	formatedLangGold    = "test_files/formated_languages.gold"
 	formatedContentGold = "test_files/formated_content.gold"
+	formatedVendorGold  = "test_files/formated_vendor.gold"
 
 	// Languages test
 	ymlTestFile           = "test_files/languages.test.yml"
@@ -27,6 +28,13 @@ const (
 	contentTestTmplPath  = "test_files/content.test.go.tmpl"
 	contentTestTmplName  = "content.test.go.tmpl"
 	commitHeuristicsTest = "fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7"
+
+	// Vendor test
+	vendorTestFile     = "test_files/vendor.test.yml"
+	vendorGold         = "test_files/vendor.gold"
+	vendorTestTmplPath = "test_files/vendor.test.go.tmpl"
+	vendorTestTmplName = "vendor.test.go.tmpl"
+	commitVendorTest   = "fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7"
 )
 
 func TestFromFile(t *testing.T) {
@@ -36,6 +44,9 @@ func TestFromFile(t *testing.T) {
 	goldContent, err := ioutil.ReadFile(formatedContentGold)
 	assert.NoError(t, err)
 
+	goldVendor, err := ioutil.ReadFile(formatedVendorGold)
+	assert.NoError(t, err)
+
 	outPathLang, err := ioutil.TempFile("/tmp", "generator-test-")
 	assert.NoError(t, err)
 	defer os.Remove(outPathLang.Name())
@@ -44,6 +55,10 @@ func TestFromFile(t *testing.T) {
 	assert.NoError(t, err)
 	defer os.Remove(outPathContent.Name())
 
+	outPathVendor, err := ioutil.TempFile("/tmp", "generator-test-")
+	assert.NoError(t, err)
+	defer os.Remove(outPathContent.Name())
+
 	tests := []struct {
 		name        string
 		fileToParse string
@@ -74,6 +89,16 @@ func TestFromFile(t *testing.T) {
 			generate:    Heuristics,
 			wantOut:     goldContent,
 		},
+		{
+			name:        "TestFromFile_Vendor",
+			fileToParse: vendorTestFile,
+			outPath:     outPathVendor.Name(),
+			tmplPath:    vendorTestTmplPath,
+			tmplName:    vendorTestTmplName,
+			commit:      commitVendorTest,
+			generate:    Vendor,
+			wantOut:     goldVendor,
+		},
 	}
 
 	for _, tt := range tests {
@@ -154,3 +179,37 @@ func TestHeuristics(t *testing.T) {
 		})
 	}
 }
+
+func TestVendor(t *testing.T) {
+	gold, err := ioutil.ReadFile(vendorGold)
+	assert.NoError(t, err)
+
+	input, err := ioutil.ReadFile(vendorTestFile)
+	assert.NoError(t, err)
+
+	tests := []struct {
+		name     string
+		input    []byte
+		tmplPath string
+		tmplName string
+		commit   string
+		wantOut  []byte
+	}{
+		{
+			name:     "TestVendor",
+			input:    input,
+			tmplPath: vendorTestTmplPath,
+			tmplName: vendorTestTmplName,
+			commit:   commitVendorTest,
+			wantOut:  gold,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			out, err := Vendor(tt.input, tt.tmplPath, tt.tmplName, tt.commit)
+			assert.NoError(t, err)
+			assert.EqualValues(t, tt.wantOut, out, fmt.Sprintf("Vendor() = %v, want %v", string(out), string(tt.wantOut)))
+		})
+	}
+}

internal/code-generator/generator/test_files/formated_vendor.gold

@@ -0,0 +1,24 @@
+package slinguist
+
+// CODE GENERATED AUTOMATICALLY WITH github.com/src-d/simple-linguist/cli/slinguist-generate
+// THIS FILE SHOULD NOT BE EDITED BY HAND
+// Extracted from github/linguist commit: fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7
+
+import "gopkg.in/toqueteos/substring.v1"
+
+var vendorMatchers = substring.Or(
+	substring.Regexp(`(^|/)cache/`),
+	substring.Regexp(`^[Dd]ependencies/`),
+	substring.Regexp(`(^|/)dist/`),
+	substring.Regexp(`^deps/`),
+	substring.Regexp(`^tools/`),
+	substring.Regexp(`(^|/)configure$`),
+	substring.Regexp(`(^|/)config.guess$`),
+	substring.Regexp(`(^|/)config.sub$`),
+	substring.Regexp(`(^|/)aclocal.m4`),
+	substring.Regexp(`(^|/)libtool.m4`),
+	substring.Regexp(`(^|/)ltoptions.m4`),
+	substring.Regexp(`(^|/)ltsugar.m4`),
+	substring.Regexp(`(^|/)ltversion.m4`),
+	substring.Regexp(`(^|/)lt~obsolete.m4`),
+)

internal/code-generator/generator/test_files/vendor.gold

@@ -0,0 +1,24 @@
+package slinguist
+
+// CODE GENERATED AUTOMATICALLY WITH github.com/src-d/simple-linguist/cli/slinguist-generate
+// THIS FILE SHOULD NOT BE EDITED BY HAND
+// Extracted from github/linguist commit: fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7
+
+import "gopkg.in/toqueteos/substring.v1"
+
+var vendorMatchers = substring.Or(
+	substring.Regexp(`(^|/)cache/`),
+	substring.Regexp(`^[Dd]ependencies/`),
+	substring.Regexp(`(^|/)dist/`),
+	substring.Regexp(`^deps/`),
+	substring.Regexp(`^tools/`),
+	substring.Regexp(`(^|/)configure$`),
+	substring.Regexp(`(^|/)config.guess$`),
+	substring.Regexp(`(^|/)config.sub$`),
+	substring.Regexp(`(^|/)aclocal.m4`),
+	substring.Regexp(`(^|/)libtool.m4`),
+	substring.Regexp(`(^|/)ltoptions.m4`),
+	substring.Regexp(`(^|/)ltsugar.m4`),
+	substring.Regexp(`(^|/)ltversion.m4`),
+	substring.Regexp(`(^|/)lt~obsolete.m4`),
+	)

internal/code-generator/generator/test_files/vendor.test.go.tmpl

@@ -0,0 +1,13 @@
+package slinguist
+
+// CODE GENERATED AUTOMATICALLY WITH github.com/src-d/simple-linguist/cli/slinguist-generate
+// THIS FILE SHOULD NOT BE EDITED BY HAND
+// Extracted from github/linguist commit: {{ getCommit }}
+
+import "gopkg.in/toqueteos/substring.v1"
+
+var vendorMatchers = substring.Or(
+	{{range $regexp := . -}}
+	substring.Regexp(`{{ $regexp }}`),
+	{{end -}}
+)

internal/code-generator/generator/test_files/vendor.test.yml

@@ -0,0 +1,24 @@
+# Caches
+- (^|/)cache/
+
+# Dependencies
+- ^[Dd]ependencies/
+
+# Distributions
+- (^|/)dist/
+
+# C deps
+#  https://github.com/joyent/node
+- ^deps/
+- ^tools/
+- (^|/)configure$
+- (^|/)config.guess$
+- (^|/)config.sub$
+
+# stuff autogenerated by autoconf - still C deps
+- (^|/)aclocal.m4
+- (^|/)libtool.m4
+- (^|/)ltoptions.m4
+- (^|/)ltsugar.m4
+- (^|/)ltversion.m4
+- (^|/)lt~obsolete.m4

internal/code-generator/generator/vendor.go

@@ -0,0 +1,37 @@
+package generator
+
+import (
+	"bytes"
+	"html/template"
+	"io"
+
+	yaml "gopkg.in/yaml.v2"
+)
+
+// Vendor reads from buf and builds vendor_matchers.go file from vendorTmplPath.
+func Vendor(data []byte, vendorTmplPath, vendorTmplName, commit string) ([]byte, error) {
+	var regexpList []string
+	if err := yaml.Unmarshal(data, &regexpList); err != nil {
+		return nil, err
+	}
+
+	buf := &bytes.Buffer{}
+	if err := executeVendorTemplate(buf, regexpList, vendorTmplPath, vendorTmplName, commit); err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}
+
+func executeVendorTemplate(out io.Writer, regexpList []string, vendorTmplPath, vendorTmpl, commit string) error {
+	fmap := template.FuncMap{
+		"getCommit": func() string { return commit },
+	}
+
+	t := template.Must(template.New(vendorTmpl).Funcs(fmap).ParseFiles(vendorTmplPath))
+	if err := t.Execute(out, regexpList); err != nil {
+		return err
+	}
+
+	return nil
+}

internal/code-generator/main.go

@@ -18,6 +18,11 @@ const (
 	contentTmplPath = "internal/code-generator/assets/content.go.tmpl"
 	contentTmpl     = "content.go.tmpl"
 
+	vendorYAML     = ".linguist/lib/linguist/vendor.yml"
+	vendorFile     = "vendor_matchers.go"
+	vendorTmplPath = "internal/code-generator/assets/vendor.go.tmpl"
+	vendorTmpl     = "vendor.go.tmpl"
+
 	commitPath = ".git/refs/heads/master"
 )
 
@@ -34,6 +39,10 @@ func main() {
 	if err := generator.FromFile(heuristicsRuby, contentFile, contentTmplPath, contentTmpl, commit, generator.Heuristics); err != nil {
 		log.Println(err)
 	}
+
+	if err := generator.FromFile(vendorYAML, vendorFile, vendorTmplPath, vendorTmpl, commit, generator.Vendor); err != nil {
+		log.Println(err)
+	}
 }
 
 func getCommit(path string) (string, error) {

languages.go

@@ -2,7 +2,7 @@ package slinguist
 
 // CODE GENERATED AUTOMATICALLY WITH github.com/src-d/simple-linguist/cli/slinguist-generate
 // THIS FILE SHOULD NOT BE EDITED BY HAND
-// Extracted from github/linguist commit: 3a2a62baad5363a604828706f8df730902a542b0
+// Extracted from github/linguist commit: dae33dc2b20cddc85d1300435c3be7118a7115a9
 
 var languagesByExtension = map[string][]string{
 	".1":                   {"Roff"},

utils.go

@@ -52,222 +52,6 @@ func IsBinary(data []byte) bool {
 	return true
 }
 
-// From github/linguist.
-// curl https://raw.githubusercontent.com/github/linguist/master/lib/linguist/vendor.yml | python -c 'import sys, yaml; l = yaml.load(sys.stdin.read()); print "var skipped = []*regexp.Regexp{\n" + "\n".join(["\tregexp.MustCompile(`" + i + "`)," for i in l]) + "\n}"'
-
-var vendorMatchers = substring.Or(
-	substring.Suffix(`.framer`),
-	substring.SuffixGroup(`.js`,
-		substring.After(`cordova-`, substring.Regexp(`\d\.\d(\.\d)?`)),
-		substring.After(`d3`, substring.Regexp(`(\.v\d+)?([^.]*)`)),
-		substring.After(`jquery-`, substring.Regexp(`\d\.\d+(\.\d+)?`)),
-		substring.After(`jquery-ui`, substring.Regexp(`(\-\d\.\d+(\.\d+)?)?(\.\w+)?`)),
-		substring.After(`knockout-`, substring.Regexp(`(\d+\.){3}(debug\.)?$`)),
-		substring.After(`microsoft`, substring.Regexp(`([Mm]vc)?([Aa]jax|[Vv]alidation)(\.debug)?`)),
-		substring.After(`Microsoft`, substring.Regexp(`([Mm]vc)?([Aa]jax|[Vv]alidation)(\.debug)?`)),
-		substring.After(`modernizr-`, substring.Regexp(`\d\.\d+(\.\d+)?`)),
-		substring.After(`mootools`, substring.Regexp(`([^.]*)\d+\.\d+.\d+([^.]*)`)),
-		substring.Has(`angular`),
-		substring.Has(`bootstrap`),
-		substring.Has(`cordova`),
-		substring.Has(`less`),
-		substring.Has(`custom.bootstrap`),
-		substring.Has(`extjs/`),
-		substring.Has(`foundation`),
-		substring.Has(`jquery.effects.`),
-		substring.Has(`jquery.ui.`),
-		substring.Has(`jquery`),
-		substring.Has(`modernizr.custom.`),
-		substring.Has(`prototype`),
-		substring.Has(`backbone`),
-		substring.Has(`three`),
-		substring.Has(`ember`),
-		substring.Has(`babylon`),
-		substring.Has(`react`),
-		substring.Has(`shBrush`),
-		substring.Has(`tiny_mce`),
-		substring.Has(`yahoo-`),
-		substring.Has(`html5-`),
-		substring.Has(`yui`),
-		substring.Has(`underscore`),
-		substring.Has(`lodash`),
-		substring.Has(`lodash.core`),
-		substring.Has(`coffee-script`),
-		substring.Suffixes(
-			`-vsdoc.js`,
-			`.intellisense.js`,
-			`Chart.js`,
-			`ckeditor.js`,
-			`controls.js`,
-			`dojo.js`,
-			`dragdrop.js`,
-			`effects.js`,
-			`html5shiv.js`,
-			`MochiKit.js`,
-			`shCore.js`,
-			`shLegacy.js`,
-			`.min.js`,
-			`-min.js`,
-		),
-	),
-	substring.SuffixGroup(`.css`,
-		substring.Has(`bootstrap`),
-		substring.Has(`custom.bootstrap`),
-		substring.Has(`jquery.effects.`),
-		substring.Has(`jquery.ui.`),
-		substring.Has(`octicons.css`),
-		substring.After(`jquery-ui`, substring.Regexp(`(\-\d\.\d+(\.\d+)?)?(\.\w+)?`)),
-		substring.Suffixes(
-			`animate.css`,
-			`bourbon.css`,
-			`Bourbon.css`,
-			`font-awesome.css`,
-			`foundation.css`,
-			`import.css`,
-			`normalize.css`,
-			`.min.css`,
-			`-min.css`,
-		),
-	),
-	substring.SuffixGroup(`.scss`,
-		substring.Has(`bootstrap`),
-		substring.Has(`custom.bootstrap`),
-		substring.Has(`sprockets-octicons.scss`),
-		substring.Suffixes(
-			`animate.scss`,
-			`bourbon.scss`,
-			`Bourbon.scss`,
-			`font-awesome.scss`,
-			`foundation.scss`,
-			`import.scss`,
-			`normalize.scss`,
-		),
-	),
-	substring.SuffixGroup(`.less`,
-		substring.Has(`bootstrap`),
-		substring.Has(`custom.bootstrap`),
-		substring.Suffixes(
-			`animate.less`,
-			`bourbon.less`,
-			`Bourbon.less`,
-			`font-awesome.less`,
-			`foundation.less`,
-			`import.less`,
-			`normalize.less`,
-		),
-	),
-	substring.SuffixGroup(`.styl`,
-		substring.Has(`bootstrap`),
-		substring.Has(`custom.bootstrap`),
-		substring.Suffixes(
-			`animate.styl`,
-			`bourbon.styl`,
-			`Bourbon.styl`,
-			`font-awesome.styl`,
-			`foundation.styl`,
-			`import.styl`,
-			`normalize.styl`,
-		),
-	),
-	substring.After(`codemirror/`, substring.Or(
-		substring.Exact(`lib`),
-		substring.Exact(`mode`),
-		substring.Exact(`theme`),
-		substring.Exact(`addon`),
-		substring.Exact(`keymap`),
-	)),
-	substring.After(`extjs/`, substring.Or(
-		substring.Suffixes(`.html`, `.properties`, `.txt`, `.xml`),
-		substring.Exact(`.sencha/`),
-		substring.Exact(`builds/`),
-		substring.Exact(`cmd/`),
-		substring.Exact(`docs/`),
-		substring.Exact(`examples/`),
-		substring.Exact(`locale/`),
-		substring.Exact(`packages/`),
-		substring.Exact(`plugins/`),
-		substring.Exact(`resources/`),
-		substring.Exact(`src/`),
-		substring.Exact(`welcome/`),
-	)),
-	substring.After(`tiny_mce/`, substring.Or(
-		substring.Exact(`langs`),
-		substring.Exact(`plugins`),
-		substring.Exact(`themes`),
-		substring.Exact(`utils`),
-	)),
-	substring.Has(`3rd-party/`),
-	substring.Has(`3rd_party/`),
-	substring.Has(`3rdparty/`),
-	substring.Has(`admin_media/`),
-	substring.Has(`bower_components/`),
-	substring.Has(`cache/`),
-	substring.Has(`Dependencies/`),
-	substring.Has(`dependencies/`),
-	substring.Has(`erlang.mk`),
-	substring.Has(`extern/`),
-	substring.Has(`external/`),
-	substring.Has(`Godeps/_workspace/`),
-	substring.Has(`gradle/wrapper/`),
-	substring.Has(`MathJax/`),
-	substring.Has(`node_modules/`),
-	substring.Has(`proguard-rules.pro`),
-	substring.Has(`proguard.pro`),
-	substring.Has(`Sparkle/`),
-	substring.Has(`third-party/`),
-	substring.Has(`third_party/`),
-	substring.Has(`thirdparty/`),
-	substring.Has(`vendor/`),
-	substring.Has(`vendors/`),
-	substring.Exact(`.osx`),
-	substring.Exact(`rebar`),
-	substring.Exact(`Vagrantfile`),
-	substring.Exact(`waf`),
-	substring.Prefixes(
-		`.google_apis/`,
-		`debian/`,
-		`deps/`,
-		`inst/extdata/`,
-		`packages/`,
-		`Packages/`,
-		`Pods/`,
-		`Samples/`,
-		`samples/`,
-		`Test/fixture/`,
-		`test/fixture/`,
-		`Test/fixtures/`,
-		`test/fixtures/`,
-		`tools/`,
-		`vignettes/`,
-	),
-	substring.Suffixes(
-		`.d.ts`,
-		`.travis.yml`,
-		`activator.bat`,
-		`activator`,
-		`circle.yml`,
-		`config.guess`,
-		`config.sub`,
-		`configure.ac`,
-		`configure`,
-		`fabfile.py`,
-		`gitattributes`,
-		`gitignore`,
-		`gitmodules`,
-		`gradlew.bat`,
-		`gradlew`,
-		`run.n`,
-		`.DS_Store`,
-		`.DS_store`,
-		`.dS_Store`,
-		`.dS_store`,
-		`.Ds_Store`,
-		`.Ds_store`,
-		`.ds_Store`,
-		`.ds_store`,
-	),
-)
-
 var documentationMatchers = substring.Or(
 	substring.Regexp(`^docs?/`),
 	substring.Regexp(`(^|/)[Dd]ocumentation/`),

utils_test.go

@@ -2,7 +2,6 @@ package slinguist
 
 import (
 	"bytes"
-	"regexp"
 
 	. "gopkg.in/check.v1"
 )
@@ -10,25 +9,13 @@ import (
 func (s *TSuite) TestIsVendor(c *C) {
 	c.Assert(IsVendor("foo/bar"), Equals, false)
 	c.Assert(IsVendor("foo/vendor/foo"), Equals, true)
-	c.Assert(IsVendor(".travis.yml"), Equals, true)
-	c.Assert(IsVendor("foo.framer"), Equals, true)
-	c.Assert(IsVendor("foo/bar/coffee-script.js"), Equals, true)
-	c.Assert(IsVendor("foo/bar/less.js"), Equals, true)
-	c.Assert(IsVendor("foo/bar/underscore.js"), Equals, true)
-	c.Assert(IsVendor("foo/bar/lodash.js"), Equals, true)
-	c.Assert(IsVendor("foo/bar/lodash.js"), Equals, true)
-	c.Assert(IsVendor("foo/bar/lodash.core.js"), Equals, true)
-	c.Assert(IsVendor("foo/bar/backbone.js"), Equals, true)
-	c.Assert(IsVendor("foo/bar/ember.js"), Equals, true)
-	c.Assert(IsVendor("foo/bar/ember.debug.js"), Equals, true)
-	c.Assert(IsVendor("foo/bar/ember.prod.js"), Equals, true)
-	c.Assert(IsVendor("foo/bar/three.js"), Equals, true)
-	c.Assert(IsVendor("foo/bar/babylon.js"), Equals, true)
-	c.Assert(IsVendor("foo/bar/babylon.2.3.js"), Equals, true)
-	c.Assert(IsVendor("foo/bar/babylon.2.3.core.js"), Equals, true)
-	c.Assert(IsVendor("foo/bar/babylon.2.3.max.js"), Equals, true)
-	c.Assert(IsVendor("foo/bar/babylon.2.3.noworker.js"), Equals, true)
-	c.Assert(IsVendor("foo/bar/html5-3.6-respond-1.4.2.js"), Equals, true)
+	c.Assert(IsVendor(".sublime-project"), Equals, true)
+	c.Assert(IsVendor("leaflet.draw-src.js"), Equals, true)
+	c.Assert(IsVendor("foo/bar/MochiKit.js"), Equals, true)
+	c.Assert(IsVendor("foo/bar/dojo.js"), Equals, true)
+	c.Assert(IsVendor("foo/env/whatever"), Equals, true)
+	c.Assert(IsVendor("foo/.imageset/bar"), Equals, true)
+	c.Assert(IsVendor("Vagrantfile"), Equals, true)
 }
 
 func (s *TSuite) TestIsDocumentation(c *C) {
@@ -69,137 +56,3 @@ func (s *TSuite) BenchmarkVendorJS(c *C) {
 		_ = IsVendor(jsPath)
 	}
 }
-
-var vendorRegexp = []*regexp.Regexp{
-	regexp.MustCompile(`(^|/)cache/`),
-	regexp.MustCompile(`^[Dd]ependencies/`),
-	regexp.MustCompile(`^deps/`),
-	regexp.MustCompile(`^tools/`),
-	regexp.MustCompile(`(^|/)configure$`),
-	regexp.MustCompile(`(^|/)configure.ac$`),
-	regexp.MustCompile(`(^|/)config.guess$`),
-	regexp.MustCompile(`(^|/)config.sub$`),
-	regexp.MustCompile(`node_modules/`),
-	regexp.MustCompile(`bower_components/`),
-	regexp.MustCompile(`^rebar$`),
-	regexp.MustCompile(`erlang.mk`),
-	regexp.MustCompile(`Godeps/_workspace/`),
-	regexp.MustCompile(`(\.|-)min\.(js|css)$`),
-	regexp.MustCompile(`([^\s]*)import\.(css|less|scss|styl)$`),
-	regexp.MustCompile(`(^|/)bootstrap([^.]*)\.(js|css|less|scss|styl)$`),
-	regexp.MustCompile(`(^|/)custom\.bootstrap([^\s]*)(js|css|less|scss|styl)$`),
-	regexp.MustCompile(`(^|/)font-awesome\.(css|less|scss|styl)$`),
-	regexp.MustCompile(`(^|/)foundation\.(css|less|scss|styl)$`),
-	regexp.MustCompile(`(^|/)normalize\.(css|less|scss|styl)$`),
-	regexp.MustCompile(`(^|/)[Bb]ourbon/.*\.(css|less|scss|styl)$`),
-	regexp.MustCompile(`(^|/)animate\.(css|less|scss|styl)$`),
-	regexp.MustCompile(`third[-_]?party/`),
-	regexp.MustCompile(`3rd[-_]?party/`),
-	regexp.MustCompile(`vendors?/`),
-	regexp.MustCompile(`extern(al)?/`),
-	regexp.MustCompile(`^debian/`),
-	regexp.MustCompile(`run.n$`),
-	regexp.MustCompile(`(^|/)jquery([^.]*)\.js$`),
-	regexp.MustCompile(`(^|/)jquery\-\d\.\d+(\.\d+)?\.js$`),
-	regexp.MustCompile(`(^|/)jquery\-ui(\-\d\.\d+(\.\d+)?)?(\.\w+)?\.(js|css)$`),
-	regexp.MustCompile(`(^|/)jquery\.(ui|effects)\.([^.]*)\.(js|css)$`),
-	regexp.MustCompile(`(^|/)prototype(.*)\.js$`),
-	regexp.MustCompile(`(^|/)effects\.js$`),
-	regexp.MustCompile(`(^|/)controls\.js$`),
-	regexp.MustCompile(`(^|/)dragdrop\.js$`),
-	regexp.MustCompile(`(.*?)\.d\.ts$`),
-	regexp.MustCompile(`(^|/)mootools([^.]*)\d+\.\d+.\d+([^.]*)\.js$`),
-	regexp.MustCompile(`(^|/)dojo\.js$`),
-	regexp.MustCompile(`(^|/)MochiKit\.js$`),
-	regexp.MustCompile(`(^|/)yahoo-([^.]*)\.js$`),
-	regexp.MustCompile(`(^|/)yui([^.]*)\.js$`),
-	regexp.MustCompile(`(^|/)ckeditor\.js$`),
-	regexp.MustCompile(`(^|/)tiny_mce([^.]*)\.js$`),
-	regexp.MustCompile(`(^|/)tiny_mce/(langs|plugins|themes|utils)`),
-	regexp.MustCompile(`(^|/)MathJax/`),
-	regexp.MustCompile(`(^|/)Chart\.js$`),
-	regexp.MustCompile(`(^|/)[Cc]ode[Mm]irror/(lib|mode|theme|addon|keymap)`),
-	regexp.MustCompile(`(^|/)shBrush([^.]*)\.js$`),
-	regexp.MustCompile(`(^|/)shCore\.js$`),
-	regexp.MustCompile(`(^|/)shLegacy\.js$`),
-	regexp.MustCompile(`(^|/)angular([^.]*)\.js$`),
-	regexp.MustCompile(`(^|\/)d3(\.v\d+)?([^.]*)\.js$`),
-	regexp.MustCompile(`(^|/)react(-[^.]*)?\.js$`),
-	regexp.MustCompile(`(^|/)modernizr\-\d\.\d+(\.\d+)?\.js$`),
-	regexp.MustCompile(`(^|/)modernizr\.custom\.\d+\.js$`),
-	regexp.MustCompile(`(^|/)knockout-(\d+\.){3}(debug\.)?js$`),
-	regexp.MustCompile(`(^|/)admin_media/`),
-	regexp.MustCompile(`^fabfile\.py$`),
-	regexp.MustCompile(`^waf$`),
-	regexp.MustCompile(`^.osx$`),
-	regexp.MustCompile(`^Pods/`),
-	regexp.MustCompile(`(^|/)Sparkle/`),
-	regexp.MustCompile(`(^|/)gradlew$`),
-	regexp.MustCompile(`(^|/)gradlew\.bat$`),
-	regexp.MustCompile(`(^|/)gradle/wrapper/`),
-	regexp.MustCompile(`-vsdoc\.js$`),
-	regexp.MustCompile(`\.intellisense\.js$`),
-	regexp.MustCompile(`(^|/)jquery([^.]*)\.validate(\.unobtrusive)?\.js$`),
-	regexp.MustCompile(`(^|/)jquery([^.]*)\.unobtrusive\-ajax\.js$`),
-	regexp.MustCompile(`(^|/)[Mm]icrosoft([Mm]vc)?([Aa]jax|[Vv]alidation)(\.debug)?\.js$`),
-	regexp.MustCompile(`^[Pp]ackages\/.+\.\d+\/`),
-	regexp.MustCompile(`(^|/)extjs/.*?\.js$`),
-	regexp.MustCompile(`(^|/)extjs/.*?\.xml$`),
-	regexp.MustCompile(`(^|/)extjs/.*?\.txt$`),
-	regexp.MustCompile(`(^|/)extjs/.*?\.html$`),
-	regexp.MustCompile(`(^|/)extjs/.*?\.properties$`),
-	regexp.MustCompile(`(^|/)extjs/.sencha/`),
-	regexp.MustCompile(`(^|/)extjs/docs/`),
-	regexp.MustCompile(`(^|/)extjs/builds/`),
-	regexp.MustCompile(`(^|/)extjs/cmd/`),
-	regexp.MustCompile(`(^|/)extjs/examples/`),
-	regexp.MustCompile(`(^|/)extjs/locale/`),
-	regexp.MustCompile(`(^|/)extjs/packages/`),
-	regexp.MustCompile(`(^|/)extjs/plugins/`),
-	regexp.MustCompile(`(^|/)extjs/resources/`),
-	regexp.MustCompile(`(^|/)extjs/src/`),
-	regexp.MustCompile(`(^|/)extjs/welcome/`),
-	regexp.MustCompile(`(^|/)html5shiv\.js$`),
-	regexp.MustCompile(`^[Ss]amples/`),
-	regexp.MustCompile(`^[Tt]est/fixtures/`),
-	regexp.MustCompile(`(^|/)cordova([^.]*)\.js$`),
-	regexp.MustCompile(`(^|/)cordova\-\d\.\d(\.\d)?\.js$`),
-	regexp.MustCompile(`foundation(\..*)?\.js$`),
-	regexp.MustCompile(`^Vagrantfile$`),
-	regexp.MustCompile(`.[Dd][Ss]_[Ss]tore$`),
-	regexp.MustCompile(`^vignettes/`),
-	regexp.MustCompile(`^inst/extdata/`),
-	regexp.MustCompile(`octicons.css`),
-	regexp.MustCompile(`sprockets-octicons.scss`),
-	regexp.MustCompile(`(^|/)activator$`),
-	regexp.MustCompile(`(^|/)activator\.bat$`),
-	regexp.MustCompile(`proguard.pro`),
-	regexp.MustCompile(`proguard-rules.pro`),
-	regexp.MustCompile(`gitattributes$`),
-	regexp.MustCompile(`gitignore$`),
-	regexp.MustCompile(`gitmodules$`),
-	regexp.MustCompile(`.travis.yml$`),
-	regexp.MustCompile(`circle.yml$`),
-}
-
-func isVendorRegexp(s string) bool {
-	for _, re := range vendorRegexp {
-		found := re.FindStringIndex(s)
-		if found != nil {
-			return found[1] >= 0
-		}
-	}
-	return false
-}
-
-func (s *TSuite) BenchmarkVendorRegexp(c *C) {
-	for i := 0; i < c.N; i++ {
-		_ = isVendorRegexp(htmlPath)
-	}
-}
-
-func (s *TSuite) BenchmarkVendorRegexpJS(c *C) {
-	for i := 0; i < c.N; i++ {
-		_ = isVendorRegexp(htmlPath)
-	}
-}

vendor_matchers.go

@@ -0,0 +1,156 @@
+package slinguist
+
+// CODE GENERATED AUTOMATICALLY WITH github.com/src-d/simple-linguist/cli/slinguist-generate
+// THIS FILE SHOULD NOT BE EDITED BY HAND
+// Extracted from github/linguist commit: dae33dc2b20cddc85d1300435c3be7118a7115a9
+
+import "gopkg.in/toqueteos/substring.v1"
+
+var vendorMatchers = substring.Or(
+	substring.Regexp(`(^|/)cache/`),
+	substring.Regexp(`^[Dd]ependencies/`),
+	substring.Regexp(`(^|/)dist/`),
+	substring.Regexp(`^deps/`),
+	substring.Regexp(`^tools/`),
+	substring.Regexp(`(^|/)configure$`),
+	substring.Regexp(`(^|/)config.guess$`),
+	substring.Regexp(`(^|/)config.sub$`),
+	substring.Regexp(`(^|/)aclocal.m4`),
+	substring.Regexp(`(^|/)libtool.m4`),
+	substring.Regexp(`(^|/)ltoptions.m4`),
+	substring.Regexp(`(^|/)ltsugar.m4`),
+	substring.Regexp(`(^|/)ltversion.m4`),
+	substring.Regexp(`(^|/)lt~obsolete.m4`),
+	substring.Regexp(`cpplint.py`),
+	substring.Regexp(`node_modules/`),
+	substring.Regexp(`bower_components/`),
+	substring.Regexp(`^rebar$`),
+	substring.Regexp(`erlang.mk`),
+	substring.Regexp(`Godeps/_workspace/`),
+	substring.Regexp(`.indent.pro`),
+	substring.Regexp(`(\.|-)min\.(js|css)$`),
+	substring.Regexp(`([^\s]*)import\.(css|less|scss|styl)$`),
+	substring.Regexp(`(^|/)bootstrap([^.]*)\.(js|css|less|scss|styl)$`),
+	substring.Regexp(`(^|/)custom\.bootstrap([^\s]*)(js|css|less|scss|styl)$`),
+	substring.Regexp(`(^|/)font-awesome\.(css|less|scss|styl)$`),
+	substring.Regexp(`(^|/)foundation\.(css|less|scss|styl)$`),
+	substring.Regexp(`(^|/)normalize\.(css|less|scss|styl)$`),
+	substring.Regexp(`(^|/)[Bb]ourbon/.*\.(css|less|scss|styl)$`),
+	substring.Regexp(`(^|/)animate\.(css|less|scss|styl)$`),
+	substring.Regexp(`third[-_]?party/`),
+	substring.Regexp(`3rd[-_]?party/`),
+	substring.Regexp(`vendors?/`),
+	substring.Regexp(`extern(al)?/`),
+	substring.Regexp(`(^|/)[Vv]+endor/`),
+	substring.Regexp(`^debian/`),
+	substring.Regexp(`run.n$`),
+	substring.Regexp(`bootstrap-datepicker/`),
+	substring.Regexp(`(^|/)jquery([^.]*)\.js$`),
+	substring.Regexp(`(^|/)jquery\-\d\.\d+(\.\d+)?\.js$`),
+	substring.Regexp(`(^|/)jquery\-ui(\-\d\.\d+(\.\d+)?)?(\.\w+)?\.(js|css)$`),
+	substring.Regexp(`(^|/)jquery\.(ui|effects)\.([^.]*)\.(js|css)$`),
+	substring.Regexp(`jquery.fn.gantt.js`),
+	substring.Regexp(`jquery.fancybox.(js|css)`),
+	substring.Regexp(`fuelux.js`),
+	substring.Regexp(`(^|/)jquery\.fileupload(-\w+)?\.js$`),
+	substring.Regexp(`(^|/)slick\.\w+.js$`),
+	substring.Regexp(`(^|/)Leaflet\.Coordinates-\d+\.\d+\.\d+\.src\.js$`),
+	substring.Regexp(`leaflet.draw-src.js`),
+	substring.Regexp(`leaflet.draw.css`),
+	substring.Regexp(`Control.FullScreen.css`),
+	substring.Regexp(`Control.FullScreen.js`),
+	substring.Regexp(`leaflet.spin.js`),
+	substring.Regexp(`wicket-leaflet.js`),
+	substring.Regexp(`.sublime-project`),
+	substring.Regexp(`.sublime-workspace`),
+	substring.Regexp(`(^|/)prototype(.*)\.js$`),
+	substring.Regexp(`(^|/)effects\.js$`),
+	substring.Regexp(`(^|/)controls\.js$`),
+	substring.Regexp(`(^|/)dragdrop\.js$`),
+	substring.Regexp(`(.*?)\.d\.ts$`),
+	substring.Regexp(`(^|/)mootools([^.]*)\d+\.\d+.\d+([^.]*)\.js$`),
+	substring.Regexp(`(^|/)dojo\.js$`),
+	substring.Regexp(`(^|/)MochiKit\.js$`),
+	substring.Regexp(`(^|/)yahoo-([^.]*)\.js$`),
+	substring.Regexp(`(^|/)yui([^.]*)\.js$`),
+	substring.Regexp(`(^|/)ckeditor\.js$`),
+	substring.Regexp(`(^|/)tiny_mce([^.]*)\.js$`),
+	substring.Regexp(`(^|/)tiny_mce/(langs|plugins|themes|utils)`),
+	substring.Regexp(`(^|/)ace-builds/`),
+	substring.Regexp(`(^|/)fontello(.*?)\.css$`),
+	substring.Regexp(`(^|/)MathJax/`),
+	substring.Regexp(`(^|/)Chart\.js$`),
+	substring.Regexp(`(^|/)[Cc]ode[Mm]irror/(\d+\.\d+/)?(lib|mode|theme|addon|keymap|demo)`),
+	substring.Regexp(`(^|/)shBrush([^.]*)\.js$`),
+	substring.Regexp(`(^|/)shCore\.js$`),
+	substring.Regexp(`(^|/)shLegacy\.js$`),
+	substring.Regexp(`(^|/)angular([^.]*)\.js$`),
+	substring.Regexp(`(^|\/)d3(\.v\d+)?([^.]*)\.js$`),
+	substring.Regexp(`(^|/)react(-[^.]*)?\.js$`),
+	substring.Regexp(`(^|/)modernizr\-\d\.\d+(\.\d+)?\.js$`),
+	substring.Regexp(`(^|/)modernizr\.custom\.\d+\.js$`),
+	substring.Regexp(`(^|/)knockout-(\d+\.){3}(debug\.)?js$`),
+	substring.Regexp(`(^|/)docs?/_?(build|themes?|templates?|static)/`),
+	substring.Regexp(`(^|/)admin_media/`),
+	substring.Regexp(`(^|/)env/`),
+	substring.Regexp(`^fabfile\.py$`),
+	substring.Regexp(`^waf$`),
+	substring.Regexp(`^.osx$`),
+	substring.Regexp(`\.xctemplate/`),
+	substring.Regexp(`\.imageset/`),
+	substring.Regexp(`^Carthage/`),
+	substring.Regexp(`^Pods/`),
+	substring.Regexp(`(^|/)Sparkle/`),
+	substring.Regexp(`Crashlytics.framework/`),
+	substring.Regexp(`Fabric.framework/`),
+	substring.Regexp(`BuddyBuildSDK.framework/`),
+	substring.Regexp(`Realm.framework`),
+	substring.Regexp(`RealmSwift.framework`),
+	substring.Regexp(`gitattributes$`),
+	substring.Regexp(`gitignore$`),
+	substring.Regexp(`gitmodules$`),
+	substring.Regexp(`(^|/)gradlew$`),
+	substring.Regexp(`(^|/)gradlew\.bat$`),
+	substring.Regexp(`(^|/)gradle/wrapper/`),
+	substring.Regexp(`-vsdoc\.js$`),
+	substring.Regexp(`\.intellisense\.js$`),
+	substring.Regexp(`(^|/)jquery([^.]*)\.validate(\.unobtrusive)?\.js$`),
+	substring.Regexp(`(^|/)jquery([^.]*)\.unobtrusive\-ajax\.js$`),
+	substring.Regexp(`(^|/)[Mm]icrosoft([Mm]vc)?([Aa]jax|[Vv]alidation)(\.debug)?\.js$`),
+	substring.Regexp(`^[Pp]ackages\/.+\.\d+\/`),
+	substring.Regexp(`(^|/)extjs/.*?\.js$`),
+	substring.Regexp(`(^|/)extjs/.*?\.xml$`),
+	substring.Regexp(`(^|/)extjs/.*?\.txt$`),
+	substring.Regexp(`(^|/)extjs/.*?\.html$`),
+	substring.Regexp(`(^|/)extjs/.*?\.properties$`),
+	substring.Regexp(`(^|/)extjs/.sencha/`),
+	substring.Regexp(`(^|/)extjs/docs/`),
+	substring.Regexp(`(^|/)extjs/builds/`),
+	substring.Regexp(`(^|/)extjs/cmd/`),
+	substring.Regexp(`(^|/)extjs/examples/`),
+	substring.Regexp(`(^|/)extjs/locale/`),
+	substring.Regexp(`(^|/)extjs/packages/`),
+	substring.Regexp(`(^|/)extjs/plugins/`),
+	substring.Regexp(`(^|/)extjs/resources/`),
+	substring.Regexp(`(^|/)extjs/src/`),
+	substring.Regexp(`(^|/)extjs/welcome/`),
+	substring.Regexp(`(^|/)html5shiv\.js$`),
+	substring.Regexp(`^[Tt]ests?/fixtures/`),
+	substring.Regexp(`^[Ss]pecs?/fixtures/`),
+	substring.Regexp(`(^|/)cordova([^.]*)\.js$`),
+	substring.Regexp(`(^|/)cordova\-\d\.\d(\.\d)?\.js$`),
+	substring.Regexp(`foundation(\..*)?\.js$`),
+	substring.Regexp(`^Vagrantfile$`),
+	substring.Regexp(`.[Dd][Ss]_[Ss]tore$`),
+	substring.Regexp(`^vignettes/`),
+	substring.Regexp(`^inst/extdata/`),
+	substring.Regexp(`octicons.css`),
+	substring.Regexp(`sprockets-octicons.scss`),
+	substring.Regexp(`(^|/)activator$`),
+	substring.Regexp(`(^|/)activator\.bat$`),
+	substring.Regexp(`proguard.pro`),
+	substring.Regexp(`proguard-rules.pro`),
+	substring.Regexp(`^puphpet/`),
+	substring.Regexp(`(^|/)\.google_apis/`),
+	substring.Regexp(`^Jenkinsfile$`),
+)