Merge pull request #19 from mcarmonaa/documentation

Added documentation_matchers.go generation

documentation_matchers.go

@@ -0,0 +1,25 @@
+package slinguist
+
+// CODE GENERATED AUTOMATICALLY WITH github.com/src-d/simple-linguist/cli/slinguist-generate
+// THIS FILE SHOULD NOT BE EDITED BY HAND
+// Extracted from github/linguist commit: dae33dc2b20cddc85d1300435c3be7118a7115a9
+
+import "gopkg.in/toqueteos/substring.v1"
+
+var documentationMatchers = substring.Or(
+	substring.Regexp(`^[Dd]ocs?/`),
+	substring.Regexp(`(^|/)[Dd]ocumentation/`),
+	substring.Regexp(`(^|/)[Jj]avadoc/`),
+	substring.Regexp(`^[Mm]an/`),
+	substring.Regexp(`^[Ee]xamples/`),
+	substring.Regexp(`^[Dd]emos?/`),
+	substring.Regexp(`(^|/)CHANGE(S|LOG)?(\.|$)`),
+	substring.Regexp(`(^|/)CONTRIBUTING(\.|$)`),
+	substring.Regexp(`(^|/)COPYING(\.|$)`),
+	substring.Regexp(`(^|/)INSTALL(\.|$)`),
+	substring.Regexp(`(^|/)LICEN[CS]E(\.|$)`),
+	substring.Regexp(`(^|/)[Ll]icen[cs]e(\.|$)`),
+	substring.Regexp(`(^|/)README(\.|$)`),
+	substring.Regexp(`(^|/)[Rr]eadme(\.|$)`),
+	substring.Regexp(`^[Ss]amples?/`),
+)

internal/code-generator/assets/documentation.go.tmpl

@@ -0,0 +1,13 @@
+package slinguist
+
+// CODE GENERATED AUTOMATICALLY WITH github.com/src-d/simple-linguist/cli/slinguist-generate
+// THIS FILE SHOULD NOT BE EDITED BY HAND
+// Extracted from github/linguist commit: {{ getCommit }}
+
+import "gopkg.in/toqueteos/substring.v1"
+
+var documentationMatchers = substring.Or(
+	{{range $regexp := . -}}
+	substring.Regexp(`{{ $regexp }}`),
+	{{end -}}
+)

internal/code-generator/generator/documentation.go

@@ -0,0 +1,37 @@
+package generator
+
+import (
+	"bytes"
+	"html/template"
+	"io"
+
+	yaml "gopkg.in/yaml.v2"
+)
+
+// Documentation reads from buf and builds documentation_matchers.go file from documentationTmplPath.
+func Documentation(data []byte, documentationTmplPath, documentationTmplName, commit string) ([]byte, error) {
+	var regexpList []string
+	if err := yaml.Unmarshal(data, &regexpList); err != nil {
+		return nil, err
+	}
+
+	buf := &bytes.Buffer{}
+	if err := executeVendorTemplate(buf, regexpList, documentationTmplPath, documentationTmplName, commit); err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}
+
+func executeDocumentationTemplate(out io.Writer, regexpList []string, documentationTmplPath, documentationTmpl, commit string) error {
+	fmap := template.FuncMap{
+		"getCommit": func() string { return commit },
+	}
+
+	t := template.Must(template.New(documentationTmpl).Funcs(fmap).ParseFiles(documentationTmplPath))
+	if err := t.Execute(out, regexpList); err != nil {
+		return err
+	}
+
+	return nil
+}

internal/code-generator/generator/generator_test.go

@@ -10,31 +10,37 @@ import (
 )
 
 const (
+	commitTest = "fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7"
+
 	// FromFile test
 	formatedLangGold          = "test_files/formated_languages.gold"
 	formatedContentGold       = "test_files/formated_content.gold"
 	formatedVendorGold        = "test_files/formated_vendor.gold"
+	formatedDocumentationGold = "test_files/formated_documentation.gold"
 
 	// Languages test
 	ymlTestFile           = "test_files/languages.test.yml"
 	langGold              = "test_files/languages.gold"
 	languagesTestTmplPath = "test_files/languages.test.tmpl"
 	languagesTestTmplName = "languages.test.tmpl"
-	commitLangTest        = "fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7"
 
 	// Heuristics test
 	heuristicsTestFile  = "test_files/heuristics.test.rb"
 	contentGold         = "test_files/content.gold"
 	contentTestTmplPath = "test_files/content.test.go.tmpl"
 	contentTestTmplName = "content.test.go.tmpl"
-	commitHeuristicsTest = "fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7"
 
 	// Vendor test
 	vendorTestFile     = "test_files/vendor.test.yml"
 	vendorGold         = "test_files/vendor.gold"
 	vendorTestTmplPath = "test_files/vendor.test.go.tmpl"
 	vendorTestTmplName = "vendor.test.go.tmpl"
-	commitVendorTest   = "fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7"
+
+	// Documentation test
+	documentationTestFile     = "test_files/documentation.test.yml"
+	documentationGold         = "test_files/documentation.gold"
+	documentationTestTmplPath = "test_files/documentation.test.go.tmpl"
+	documentationTestTmplName = "documentation.test.go.tmpl"
 )
 
 func TestFromFile(t *testing.T) {
@@ -47,6 +53,9 @@ func TestFromFile(t *testing.T) {
 	goldVendor, err := ioutil.ReadFile(formatedVendorGold)
 	assert.NoError(t, err)
 
+	goldDocumentation, err := ioutil.ReadFile(formatedDocumentationGold)
+	assert.NoError(t, err)
+
 	outPathLang, err := ioutil.TempFile("/tmp", "generator-test-")
 	assert.NoError(t, err)
 	defer os.Remove(outPathLang.Name())
@@ -57,7 +66,11 @@ func TestFromFile(t *testing.T) {
 
 	outPathVendor, err := ioutil.TempFile("/tmp", "generator-test-")
 	assert.NoError(t, err)
-	defer os.Remove(outPathContent.Name())
+	defer os.Remove(outPathVendor.Name())
+
+	outPathDocumentation, err := ioutil.TempFile("/tmp", "generator-test-")
+	assert.NoError(t, err)
+	defer os.Remove(outPathDocumentation.Name())
 
 	tests := []struct {
 		name        string
@@ -75,7 +88,7 @@ func TestFromFile(t *testing.T) {
 			outPath:     outPathLang.Name(),
 			tmplPath:    languagesTestTmplPath,
 			tmplName:    languagesTestTmplName,
-			commit:      commitLangTest,
+			commit:      commitTest,
 			generate:    Languages,
 			wantOut:     goldLang,
 		},
@@ -85,7 +98,7 @@ func TestFromFile(t *testing.T) {
 			outPath:     outPathContent.Name(),
 			tmplPath:    contentTestTmplPath,
 			tmplName:    contentTestTmplName,
-			commit:      commitHeuristicsTest,
+			commit:      commitTest,
 			generate:    Heuristics,
 			wantOut:     goldContent,
 		},
@@ -95,10 +108,20 @@ func TestFromFile(t *testing.T) {
 			outPath:     outPathVendor.Name(),
 			tmplPath:    vendorTestTmplPath,
 			tmplName:    vendorTestTmplName,
-			commit:      commitVendorTest,
+			commit:      commitTest,
 			generate:    Vendor,
 			wantOut:     goldVendor,
 		},
+		{
+			name:        "TestFromFile_Documentation",
+			fileToParse: documentationTestFile,
+			outPath:     outPathDocumentation.Name(),
+			tmplPath:    documentationTestTmplPath,
+			tmplName:    documentationTestTmplName,
+			commit:      commitTest,
+			generate:    Documentation,
+			wantOut:     goldDocumentation,
+		},
 	}
 
 	for _, tt := range tests {
@@ -132,7 +155,7 @@ func TestLanguages(t *testing.T) {
 			input:    input,
 			tmplPath: languagesTestTmplPath,
 			tmplName: languagesTestTmplName,
-			commit:   commitLangTest,
+			commit:   commitTest,
 			wantOut:  gold,
 		},
 	}
@@ -166,7 +189,7 @@ func TestHeuristics(t *testing.T) {
 			input:    input,
 			tmplPath: contentTestTmplPath,
 			tmplName: contentTestTmplName,
-			commit:   commitHeuristicsTest,
+			commit:   commitTest,
 			wantOut:  gold,
 		},
 	}
@@ -200,7 +223,7 @@ func TestVendor(t *testing.T) {
 			input:    input,
 			tmplPath: vendorTestTmplPath,
 			tmplName: vendorTestTmplName,
-			commit:   commitVendorTest,
+			commit:   commitTest,
 			wantOut:  gold,
 		},
 	}
@@ -213,3 +236,37 @@ func TestVendor(t *testing.T) {
 		})
 	}
 }
+
+func TestDocumentation(t *testing.T) {
+	gold, err := ioutil.ReadFile(documentationGold)
+	assert.NoError(t, err)
+
+	input, err := ioutil.ReadFile(documentationTestFile)
+	assert.NoError(t, err)
+
+	tests := []struct {
+		name     string
+		input    []byte
+		tmplPath string
+		tmplName string
+		commit   string
+		wantOut  []byte
+	}{
+		{
+			name:     "TestDocumentation",
+			input:    input,
+			tmplPath: documentationTestTmplPath,
+			tmplName: documentationTestTmplName,
+			commit:   commitTest,
+			wantOut:  gold,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			out, err := Documentation(tt.input, tt.tmplPath, tt.tmplName, tt.commit)
+			assert.NoError(t, err)
+			assert.EqualValues(t, tt.wantOut, out, fmt.Sprintf("Documentation() = %v, want %v", string(out), string(tt.wantOut)))
+		})
+	}
+}

internal/code-generator/generator/test_files/documentation.gold

@@ -0,0 +1,25 @@
+package slinguist
+
+// CODE GENERATED AUTOMATICALLY WITH github.com/src-d/simple-linguist/cli/slinguist-generate
+// THIS FILE SHOULD NOT BE EDITED BY HAND
+// Extracted from github/linguist commit: fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7
+
+import "gopkg.in/toqueteos/substring.v1"
+
+var documentationMatchers = substring.Or(
+	substring.Regexp(`^[Dd]ocs?/`),
+	substring.Regexp(`(^|/)[Dd]ocumentation/`),
+	substring.Regexp(`(^|/)[Jj]avadoc/`),
+	substring.Regexp(`^[Mm]an/`),
+	substring.Regexp(`^[Ee]xamples/`),
+	substring.Regexp(`^[Dd]emos?/`),
+	substring.Regexp(`(^|/)CHANGE(S|LOG)?(\.|$)`),
+	substring.Regexp(`(^|/)CONTRIBUTING(\.|$)`),
+	substring.Regexp(`(^|/)COPYING(\.|$)`),
+	substring.Regexp(`(^|/)INSTALL(\.|$)`),
+	substring.Regexp(`(^|/)LICEN[CS]E(\.|$)`),
+	substring.Regexp(`(^|/)[Ll]icen[cs]e(\.|$)`),
+	substring.Regexp(`(^|/)README(\.|$)`),
+	substring.Regexp(`(^|/)[Rr]eadme(\.|$)`),
+	substring.Regexp(`^[Ss]amples?/`),
+	)

internal/code-generator/generator/test_files/documentation.test.go.tmpl

@@ -0,0 +1,13 @@
+package slinguist
+
+// CODE GENERATED AUTOMATICALLY WITH github.com/src-d/simple-linguist/cli/slinguist-generate
+// THIS FILE SHOULD NOT BE EDITED BY HAND
+// Extracted from github/linguist commit: {{ getCommit }}
+
+import "gopkg.in/toqueteos/substring.v1"
+
+var documentationMatchers = substring.Or(
+	{{range $regexp := . -}}
+	substring.Regexp(`{{ $regexp }}`),
+	{{end -}}
+)

internal/code-generator/generator/test_files/documentation.test.yml

@@ -0,0 +1,22 @@
+## Documentation directories ##
+
+- ^[Dd]ocs?/
+- (^|/)[Dd]ocumentation/
+- (^|/)[Jj]avadoc/
+- ^[Mm]an/
+- ^[Ee]xamples/
+- ^[Dd]emos?/
+
+## Documentation files ##
+
+- (^|/)CHANGE(S|LOG)?(\.|$)
+- (^|/)CONTRIBUTING(\.|$)
+- (^|/)COPYING(\.|$)
+- (^|/)INSTALL(\.|$)
+- (^|/)LICEN[CS]E(\.|$)
+- (^|/)[Ll]icen[cs]e(\.|$)
+- (^|/)README(\.|$)
+- (^|/)[Rr]eadme(\.|$)
+
+# Samples folders
+- ^[Ss]amples?/

internal/code-generator/generator/test_files/formated_documentation.gold

@@ -0,0 +1,25 @@
+package slinguist
+
+// CODE GENERATED AUTOMATICALLY WITH github.com/src-d/simple-linguist/cli/slinguist-generate
+// THIS FILE SHOULD NOT BE EDITED BY HAND
+// Extracted from github/linguist commit: fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7
+
+import "gopkg.in/toqueteos/substring.v1"
+
+var documentationMatchers = substring.Or(
+	substring.Regexp(`^[Dd]ocs?/`),
+	substring.Regexp(`(^|/)[Dd]ocumentation/`),
+	substring.Regexp(`(^|/)[Jj]avadoc/`),
+	substring.Regexp(`^[Mm]an/`),
+	substring.Regexp(`^[Ee]xamples/`),
+	substring.Regexp(`^[Dd]emos?/`),
+	substring.Regexp(`(^|/)CHANGE(S|LOG)?(\.|$)`),
+	substring.Regexp(`(^|/)CONTRIBUTING(\.|$)`),
+	substring.Regexp(`(^|/)COPYING(\.|$)`),
+	substring.Regexp(`(^|/)INSTALL(\.|$)`),
+	substring.Regexp(`(^|/)LICEN[CS]E(\.|$)`),
+	substring.Regexp(`(^|/)[Ll]icen[cs]e(\.|$)`),
+	substring.Regexp(`(^|/)README(\.|$)`),
+	substring.Regexp(`(^|/)[Rr]eadme(\.|$)`),
+	substring.Regexp(`^[Ss]amples?/`),
+)

internal/code-generator/main.go

@@ -23,6 +23,11 @@ const (
 	vendorTmplPath = "internal/code-generator/assets/vendor.go.tmpl"
 	vendorTmpl     = "vendor.go.tmpl"
 
+	documentationYAML     = ".linguist/lib/linguist/documentation.yml"
+	documentationFile     = "documentation_matchers.go"
+	documentationTmplPath = "internal/code-generator/assets/documentation.go.tmpl"
+	documentationTmpl     = "documentation.go.tmpl"
+
 	commitPath = ".git/refs/heads/master"
 )
 
@@ -43,6 +48,10 @@ func main() {
 	if err := generator.FromFile(vendorYAML, vendorFile, vendorTmplPath, vendorTmpl, commit, generator.Vendor); err != nil {
 		log.Println(err)
 	}
+
+	if err := generator.FromFile(documentationYAML, documentationFile, documentationTmplPath, documentationTmpl, commit, generator.Documentation); err != nil {
+		log.Println(err)
+	}
 }
 
 func getCommit(path string) (string, error) {

utils.go

@@ -52,23 +52,6 @@ func IsBinary(data []byte) bool {
 	return true
 }
 
-var documentationMatchers = substring.Or(
-	substring.Regexp(`^docs?/`),
-	substring.Regexp(`(^|/)[Dd]ocumentation/`),
-	substring.Regexp(`(^|/)javadoc/`),
-	substring.Regexp(`^man/`),
-	substring.Regexp(`^[Ee]xamples/`),
-	substring.Regexp(`(^|/)CHANGE(S|LOG)?(\.|$)`),
-	substring.Regexp(`(^|/)CONTRIBUTING(\.|$)`),
-	substring.Regexp(`(^|/)COPYING(\.|$)`),
-	substring.Regexp(`(^|/)INSTALL(\.|$)`),
-	substring.Regexp(`(^|/)LICEN[CS]E(\.|$)`),
-	substring.Regexp(`(^|/)[Ll]icen[cs]e(\.|$)`),
-	substring.Regexp(`(^|/)README(\.|$)`),
-	substring.Regexp(`(^|/)[Rr]eadme(\.|$)`),
-	substring.Regexp(`^[Ss]amples/`),
-)
-
 var configurationLanguages = map[string]bool{
 	"XML": true, "JSON": true, "TOML": true, "YAML": true, "INI": true, "SQL": true,
 }

utils_test.go

@@ -21,6 +21,13 @@ func (s *TSuite) TestIsVendor(c *C) {
 func (s *TSuite) TestIsDocumentation(c *C) {
 	c.Assert(IsDocumentation("foo"), Equals, false)
 	c.Assert(IsDocumentation("README"), Equals, true)
+	c.Assert(IsDocumentation("samples/something"), Equals, true)
+	c.Assert(IsDocumentation("Docs/whatever"), Equals, true)
+	c.Assert(IsDocumentation("/javadoc/*"), Equals, true)
+	c.Assert(IsDocumentation("License"), Equals, true)
+	c.Assert(IsDocumentation("CONTRIBUTING.bar"), Equals, true)
+	c.Assert(IsDocumentation("/CHANGES"), Equals, true)
+	c.Assert(IsDocumentation("INSTALL"), Equals, true)
 }
 
 func (s *TSuite) TestIsConfiguration(c *C) {