Added utils.go generation

internal/code-generator/assets/utils.go.tmpl

@@ -0,0 +1,84 @@
+package slinguist
+
+// CODE GENERATED AUTOMATICALLY WITH github.com/src-d/simple-linguist/cli/slinguist-generate
+// THIS FILE SHOULD NOT BE EDITED BY HAND
+// Extracted from github/linguist commit: {{ getCommit }}
+
+import (
+	"bytes"
+	"path/filepath"
+	"strings"
+
+	"gopkg.in/toqueteos/substring.v1"
+)
+
+func IsAuxiliaryLanguage(lang string) bool {
+	_, ok := auxiliaryLanguages[lang]
+	return ok
+}
+
+func IsConfiguration(path string) bool {
+	lang, _ := GetLanguageByExtension(path)
+	_, is := configurationLanguages[lang]
+
+	return is
+}
+
+func IsDotFile(path string) bool {
+	return strings.HasPrefix(filepath.Base(path), ".")
+}
+
+func IsVendor(path string) bool {
+	return findIndex(path, vendorMatchers) >= 0
+}
+
+func IsDocumentation(path string) bool {
+	return findIndex(path, documentationMatchers) >= 0
+}
+
+func findIndex(path string, matchers substring.StringsMatcher) int {
+	return matchers.MatchIndex(path)
+}
+
+const sniffLen = 8000
+
+//IsBinary detects if data is a binary value based on:
+//http://git.kernel.org/cgit/git/git.git/tree/xdiff-interface.c?id=HEAD#n198
+func IsBinary(data []byte) bool {
+	if len(data) > sniffLen {
+		data = data[:sniffLen]
+	}
+
+	if bytes.IndexByte(data, byte(0)) == -1 {
+		return false
+	}
+
+	return true
+}
+
+var vendorMatchers = substring.Or(
+	{{range $regexp := . -}}
+	substring.Regexp(`{{ $regexp }}`),
+	{{end -}}
+)
+
+var documentationMatchers = substring.Or(
+	substring.Regexp(`^docs?/`),
+	substring.Regexp(`(^|/)[Dd]ocumentation/`),
+	substring.Regexp(`(^|/)javadoc/`),
+	substring.Regexp(`^man/`),
+	substring.Regexp(`^[Ee]xamples/`),
+	substring.Regexp(`(^|/)CHANGE(S|LOG)?(\.|$)`),
+	substring.Regexp(`(^|/)CONTRIBUTING(\.|$)`),
+	substring.Regexp(`(^|/)COPYING(\.|$)`),
+	substring.Regexp(`(^|/)INSTALL(\.|$)`),
+	substring.Regexp(`(^|/)LICEN[CS]E(\.|$)`),
+	substring.Regexp(`(^|/)[Ll]icen[cs]e(\.|$)`),
+	substring.Regexp(`(^|/)README(\.|$)`),
+	substring.Regexp(`(^|/)[Rr]eadme(\.|$)`),
+	substring.Regexp(`^[Ss]amples/`),
+)
+
+var configurationLanguages = map[string]bool{
+	"XML": true, "JSON": true, "TOML": true, "YAML": true, "INI": true, "SQL": true,
+}

internal/code-generator/generator/generator_test.go

@@ -13,6 +13,7 @@ const (
 	// FromFile test
 	formatedLangGold    = "test_files/formated_languages.gold"
 	formatedContentGold = "test_files/formated_content.gold"
+	formatedUtilsGold   = "test_files/formated_utils.gold"
 
 	// Languages test
 	ymlTestFile           = "test_files/languages.test.yml"
@@ -27,6 +28,13 @@ const (
 	contentTestTmplPath  = "test_files/content.test.go.tmpl"
 	contentTestTmplName  = "content.test.go.tmpl"
 	commitHeuristicsTest = "fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7"
+
+	// Vendor test
+	vendorTestFile    = "test_files/vendor.test.yml"
+	utilsGold         = "test_files/utils.gold"
+	utilsTestTmplPath = "test_files/utils.test.go.tmpl"
+	utilsTestTmplName = "utils.test.go.tmpl"
+	commitVendorTest  = "fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7"
 )
 
 func TestFromFile(t *testing.T) {
@@ -36,6 +44,9 @@ func TestFromFile(t *testing.T) {
 	goldContent, err := ioutil.ReadFile(formatedContentGold)
 	assert.NoError(t, err)
 
+	goldUtils, err := ioutil.ReadFile(formatedUtilsGold)
+	assert.NoError(t, err)
+
 	outPathLang, err := ioutil.TempFile("/tmp", "generator-test-")
 	assert.NoError(t, err)
 	defer os.Remove(outPathLang.Name())
@@ -44,6 +55,10 @@ func TestFromFile(t *testing.T) {
 	assert.NoError(t, err)
 	defer os.Remove(outPathContent.Name())
 
+	outPathUtils, err := ioutil.TempFile("/tmp", "generator-test-")
+	assert.NoError(t, err)
+	defer os.Remove(outPathContent.Name())
+
 	tests := []struct {
 		name        string
 		fileToParse string
@@ -74,6 +89,16 @@ func TestFromFile(t *testing.T) {
 			generate:    Heuristics,
 			wantOut:     goldContent,
 		},
+		{
+			name:        "TestFromFile_Vendor",
+			fileToParse: vendorTestFile,
+			outPath:     outPathUtils.Name(),
+			tmplPath:    utilsTestTmplPath,
+			tmplName:    utilsTestTmplName,
+			commit:      commitVendorTest,
+			generate:    Vendor,
+			wantOut:     goldUtils,
+		},
 	}
 
 	for _, tt := range tests {
@@ -154,3 +179,37 @@ func TestHeuristics(t *testing.T) {
 		})
 	}
 }
+
+func TestVendor(t *testing.T) {
+	gold, err := ioutil.ReadFile(utilsGold)
+	assert.NoError(t, err)
+
+	input, err := ioutil.ReadFile(vendorTestFile)
+	assert.NoError(t, err)
+
+	tests := []struct {
+		name     string
+		input    []byte
+		tmplPath string
+		tmplName string
+		commit   string
+		wantOut  []byte
+	}{
+		{
+			name:     "TestVendor",
+			input:    input,
+			tmplPath: utilsTestTmplPath,
+			tmplName: utilsTestTmplName,
+			commit:   commitVendorTest,
+			wantOut:  gold,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			out, err := Vendor(tt.input, tt.tmplPath, tt.tmplName, tt.commit)
+			assert.NoError(t, err)
+			assert.EqualValues(t, tt.wantOut, out, fmt.Sprintf("Vendor() = %v, want %v", string(out), string(tt.wantOut)))
+		})
+	}
+}

internal/code-generator/generator/test_files/formated_utils.gold

@@ -0,0 +1,95 @@
+package slinguist
+
+// CODE GENERATED AUTOMATICALLY WITH github.com/src-d/simple-linguist/cli/slinguist-generate
+// THIS FILE SHOULD NOT BE EDITED BY HAND
+// Extracted from github/linguist commit: fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7
+
+import (
+	"bytes"
+	"path/filepath"
+	"strings"
+
+	"gopkg.in/toqueteos/substring.v1"
+)
+
+func IsAuxiliaryLanguage(lang string) bool {
+	_, ok := auxiliaryLanguages[lang]
+	return ok
+}
+
+func IsConfiguration(path string) bool {
+	lang, _ := GetLanguageByExtension(path)
+	_, is := configurationLanguages[lang]
+
+	return is
+}
+
+func IsDotFile(path string) bool {
+	return strings.HasPrefix(filepath.Base(path), ".")
+}
+
+func IsVendor(path string) bool {
+	return findIndex(path, vendorMatchers) >= 0
+}
+
+func IsDocumentation(path string) bool {
+	return findIndex(path, documentationMatchers) >= 0
+}
+
+func findIndex(path string, matchers substring.StringsMatcher) int {
+	return matchers.MatchIndex(path)
+}
+
+const sniffLen = 8000
+
+//IsBinary detects if data is a binary value based on:
+//http://git.kernel.org/cgit/git/git.git/tree/xdiff-interface.c?id=HEAD#n198
+func IsBinary(data []byte) bool {
+	if len(data) > sniffLen {
+		data = data[:sniffLen]
+	}
+
+	if bytes.IndexByte(data, byte(0)) == -1 {
+		return false
+	}
+
+	return true
+}
+
+var vendorMatchers = substring.Or(
+	substring.Regexp(`(^|/)cache/`),
+	substring.Regexp(`^[Dd]ependencies/`),
+	substring.Regexp(`(^|/)dist/`),
+	substring.Regexp(`^deps/`),
+	substring.Regexp(`^tools/`),
+	substring.Regexp(`(^|/)configure$`),
+	substring.Regexp(`(^|/)config.guess$`),
+	substring.Regexp(`(^|/)config.sub$`),
+	substring.Regexp(`(^|/)aclocal.m4`),
+	substring.Regexp(`(^|/)libtool.m4`),
+	substring.Regexp(`(^|/)ltoptions.m4`),
+	substring.Regexp(`(^|/)ltsugar.m4`),
+	substring.Regexp(`(^|/)ltversion.m4`),
+	substring.Regexp(`(^|/)lt~obsolete.m4`),
+)
+
+var documentationMatchers = substring.Or(
+	substring.Regexp(`^docs?/`),
+	substring.Regexp(`(^|/)[Dd]ocumentation/`),
+	substring.Regexp(`(^|/)javadoc/`),
+	substring.Regexp(`^man/`),
+	substring.Regexp(`^[Ee]xamples/`),
+	substring.Regexp(`(^|/)CHANGE(S|LOG)?(\.|$)`),
+	substring.Regexp(`(^|/)CONTRIBUTING(\.|$)`),
+	substring.Regexp(`(^|/)COPYING(\.|$)`),
+	substring.Regexp(`(^|/)INSTALL(\.|$)`),
+	substring.Regexp(`(^|/)LICEN[CS]E(\.|$)`),
+	substring.Regexp(`(^|/)[Ll]icen[cs]e(\.|$)`),
+	substring.Regexp(`(^|/)README(\.|$)`),
+	substring.Regexp(`(^|/)[Rr]eadme(\.|$)`),
+	substring.Regexp(`^[Ss]amples/`),
+)
+
+var configurationLanguages = map[string]bool{
+	"XML": true, "JSON": true, "TOML": true, "YAML": true, "INI": true, "SQL": true,
+}

internal/code-generator/generator/test_files/utils.gold

@@ -0,0 +1,95 @@
+package slinguist
+
+// CODE GENERATED AUTOMATICALLY WITH github.com/src-d/simple-linguist/cli/slinguist-generate
+// THIS FILE SHOULD NOT BE EDITED BY HAND
+// Extracted from github/linguist commit: fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7
+
+import (
+	"bytes"
+	"path/filepath"
+	"strings"
+
+	"gopkg.in/toqueteos/substring.v1"
+)
+
+func IsAuxiliaryLanguage(lang string) bool {
+	_, ok := auxiliaryLanguages[lang]
+	return ok
+}
+
+func IsConfiguration(path string) bool {
+	lang, _ := GetLanguageByExtension(path)
+	_, is := configurationLanguages[lang]
+
+	return is
+}
+
+func IsDotFile(path string) bool {
+	return strings.HasPrefix(filepath.Base(path), ".")
+}
+
+func IsVendor(path string) bool {
+	return findIndex(path, vendorMatchers) >= 0
+}
+
+func IsDocumentation(path string) bool {
+	return findIndex(path, documentationMatchers) >= 0
+}
+
+func findIndex(path string, matchers substring.StringsMatcher) int {
+	return matchers.MatchIndex(path)
+}
+
+const sniffLen = 8000
+
+//IsBinary detects if data is a binary value based on:
+//http://git.kernel.org/cgit/git/git.git/tree/xdiff-interface.c?id=HEAD#n198
+func IsBinary(data []byte) bool {
+	if len(data) > sniffLen {
+		data = data[:sniffLen]
+	}
+
+	if bytes.IndexByte(data, byte(0)) == -1 {
+		return false
+	}
+
+	return true
+}
+
+var vendorMatchers = substring.Or(
+	substring.Regexp(`(^|/)cache/`),
+	substring.Regexp(`^[Dd]ependencies/`),
+	substring.Regexp(`(^|/)dist/`),
+	substring.Regexp(`^deps/`),
+	substring.Regexp(`^tools/`),
+	substring.Regexp(`(^|/)configure$`),
+	substring.Regexp(`(^|/)config.guess$`),
+	substring.Regexp(`(^|/)config.sub$`),
+	substring.Regexp(`(^|/)aclocal.m4`),
+	substring.Regexp(`(^|/)libtool.m4`),
+	substring.Regexp(`(^|/)ltoptions.m4`),
+	substring.Regexp(`(^|/)ltsugar.m4`),
+	substring.Regexp(`(^|/)ltversion.m4`),
+	substring.Regexp(`(^|/)lt~obsolete.m4`),
+	)
+
+var documentationMatchers = substring.Or(
+	substring.Regexp(`^docs?/`),
+	substring.Regexp(`(^|/)[Dd]ocumentation/`),
+	substring.Regexp(`(^|/)javadoc/`),
+	substring.Regexp(`^man/`),
+	substring.Regexp(`^[Ee]xamples/`),
+	substring.Regexp(`(^|/)CHANGE(S|LOG)?(\.|$)`),
+	substring.Regexp(`(^|/)CONTRIBUTING(\.|$)`),
+	substring.Regexp(`(^|/)COPYING(\.|$)`),
+	substring.Regexp(`(^|/)INSTALL(\.|$)`),
+	substring.Regexp(`(^|/)LICEN[CS]E(\.|$)`),
+	substring.Regexp(`(^|/)[Ll]icen[cs]e(\.|$)`),
+	substring.Regexp(`(^|/)README(\.|$)`),
+	substring.Regexp(`(^|/)[Rr]eadme(\.|$)`),
+	substring.Regexp(`^[Ss]amples/`),
+)
+
+var configurationLanguages = map[string]bool{
+	"XML": true, "JSON": true, "TOML": true, "YAML": true, "INI": true, "SQL": true,
+}

internal/code-generator/generator/test_files/utils.test.go.tmpl

@@ -0,0 +1,84 @@
+package slinguist
+
+// CODE GENERATED AUTOMATICALLY WITH github.com/src-d/simple-linguist/cli/slinguist-generate
+// THIS FILE SHOULD NOT BE EDITED BY HAND
+// Extracted from github/linguist commit: {{ getCommit }}
+
+import (
+	"bytes"
+	"path/filepath"
+	"strings"
+
+	"gopkg.in/toqueteos/substring.v1"
+)
+
+func IsAuxiliaryLanguage(lang string) bool {
+	_, ok := auxiliaryLanguages[lang]
+	return ok
+}
+
+func IsConfiguration(path string) bool {
+	lang, _ := GetLanguageByExtension(path)
+	_, is := configurationLanguages[lang]
+
+	return is
+}
+
+func IsDotFile(path string) bool {
+	return strings.HasPrefix(filepath.Base(path), ".")
+}
+
+func IsVendor(path string) bool {
+	return findIndex(path, vendorMatchers) >= 0
+}
+
+func IsDocumentation(path string) bool {
+	return findIndex(path, documentationMatchers) >= 0
+}
+
+func findIndex(path string, matchers substring.StringsMatcher) int {
+	return matchers.MatchIndex(path)
+}
+
+const sniffLen = 8000
+
+//IsBinary detects if data is a binary value based on:
+//http://git.kernel.org/cgit/git/git.git/tree/xdiff-interface.c?id=HEAD#n198
+func IsBinary(data []byte) bool {
+	if len(data) > sniffLen {
+		data = data[:sniffLen]
+	}
+
+	if bytes.IndexByte(data, byte(0)) == -1 {
+		return false
+	}
+
+	return true
+}
+
+var vendorMatchers = substring.Or(
+	{{range $regexp := . -}}
+	substring.Regexp(`{{ $regexp }}`),
+	{{end -}}
+)
+
+var documentationMatchers = substring.Or(
+	substring.Regexp(`^docs?/`),
+	substring.Regexp(`(^|/)[Dd]ocumentation/`),
+	substring.Regexp(`(^|/)javadoc/`),
+	substring.Regexp(`^man/`),
+	substring.Regexp(`^[Ee]xamples/`),
+	substring.Regexp(`(^|/)CHANGE(S|LOG)?(\.|$)`),
+	substring.Regexp(`(^|/)CONTRIBUTING(\.|$)`),
+	substring.Regexp(`(^|/)COPYING(\.|$)`),
+	substring.Regexp(`(^|/)INSTALL(\.|$)`),
+	substring.Regexp(`(^|/)LICEN[CS]E(\.|$)`),
+	substring.Regexp(`(^|/)[Ll]icen[cs]e(\.|$)`),
+	substring.Regexp(`(^|/)README(\.|$)`),
+	substring.Regexp(`(^|/)[Rr]eadme(\.|$)`),
+	substring.Regexp(`^[Ss]amples/`),
+)
+
+var configurationLanguages = map[string]bool{
+	"XML": true, "JSON": true, "TOML": true, "YAML": true, "INI": true, "SQL": true,
+}

internal/code-generator/generator/test_files/vendor.test.yml

@@ -0,0 +1,24 @@
+# Caches
+- (^|/)cache/
+
+# Dependencies
+- ^[Dd]ependencies/
+
+# Distributions
+- (^|/)dist/
+
+# C deps
+#  https://github.com/joyent/node
+- ^deps/
+- ^tools/
+- (^|/)configure$
+- (^|/)config.guess$
+- (^|/)config.sub$
+
+# stuff autogenerated by autoconf - still C deps
+- (^|/)aclocal.m4
+- (^|/)libtool.m4
+- (^|/)ltoptions.m4
+- (^|/)ltsugar.m4
+- (^|/)ltversion.m4
+- (^|/)lt~obsolete.m4

internal/code-generator/generator/vendor.go

@@ -0,0 +1,37 @@
+package generator
+
+import (
+	"bytes"
+	"html/template"
+	"io"
+
+	yaml "gopkg.in/yaml.v2"
+)
+
+// Vendor reads from buf and builds utils.go file from utilsTmplPath.
+func Vendor(data []byte, uitlsTmplPath, utilsTmplName, commit string) ([]byte, error) {
+	var regexpList []string
+	if err := yaml.Unmarshal(data, &regexpList); err != nil {
+		return nil, err
+	}
+
+	buf := &bytes.Buffer{}
+	if err := executeUtilsTemplate(buf, regexpList, uitlsTmplPath, utilsTmplName, commit); err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}
+
+func executeUtilsTemplate(out io.Writer, regexpList []string, languagesTmplPath, languagesTmpl, commit string) error {
+	fmap := template.FuncMap{
+		"getCommit": func() string { return commit },
+	}
+
+	t := template.Must(template.New(languagesTmpl).Funcs(fmap).ParseFiles(languagesTmplPath))
+	if err := t.Execute(out, regexpList); err != nil {
+		return err
+	}
+
+	return nil
+}

internal/code-generator/main.go

@@ -18,6 +18,11 @@ const (
 	contentTmplPath = "internal/code-generator/assets/content.go.tmpl"
 	contentTmpl     = "content.go.tmpl"
 
+	vendorYAML    = ".linguist/lib/linguist/vendor.yml"
+	utilsFile     = "utils.go"
+	utilsTmplPath = "internal/code-generator/assets/utils.go.tmpl"
+	utilsTmpl     = "utils.go.tmpl"
+
 	commitPath = ".git/refs/heads/master"
 )
 
@@ -34,6 +39,14 @@ func main() {
 	if err := generator.FromFile(heuristicsRuby, contentFile, contentTmplPath, contentTmpl, commit, generator.Heuristics); err != nil {
 		log.Println(err)
 	}
+
+	if err := generator.FromFile(vendorYAML, utilsFile, utilsTmplPath, utilsTmpl, commit, generator.Vendor); err != nil {
+		log.Println(err)
+	}
+
+	if err := generator.FromFile(vendorYAML, utilsFile, utilsTmplPath, utilsTmpl, commit, generator.Vendor); err != nil {
+		log.Println(err)
+	}
 }
 
 func getCommit(path string) (string, error) {