diff --git a/documentation_matchers.go b/documentation_matchers.go new file mode 100644 index 0000000..edd8ed9 --- /dev/null +++ b/documentation_matchers.go @@ -0,0 +1,25 @@ +package slinguist + +// CODE GENERATED AUTOMATICALLY WITH github.com/src-d/simple-linguist/cli/slinguist-generate +// THIS FILE SHOULD NOT BE EDITED BY HAND +// Extracted from github/linguist commit: dae33dc2b20cddc85d1300435c3be7118a7115a9 + +import "gopkg.in/toqueteos/substring.v1" + +var documentationMatchers = substring.Or( + substring.Regexp(`^[Dd]ocs?/`), + substring.Regexp(`(^|/)[Dd]ocumentation/`), + substring.Regexp(`(^|/)[Jj]avadoc/`), + substring.Regexp(`^[Mm]an/`), + substring.Regexp(`^[Ee]xamples/`), + substring.Regexp(`^[Dd]emos?/`), + substring.Regexp(`(^|/)CHANGE(S|LOG)?(\.|$)`), + substring.Regexp(`(^|/)CONTRIBUTING(\.|$)`), + substring.Regexp(`(^|/)COPYING(\.|$)`), + substring.Regexp(`(^|/)INSTALL(\.|$)`), + substring.Regexp(`(^|/)LICEN[CS]E(\.|$)`), + substring.Regexp(`(^|/)[Ll]icen[cs]e(\.|$)`), + substring.Regexp(`(^|/)README(\.|$)`), + substring.Regexp(`(^|/)[Rr]eadme(\.|$)`), + substring.Regexp(`^[Ss]amples?/`), +) diff --git a/internal/code-generator/assets/documentation.go.tmpl b/internal/code-generator/assets/documentation.go.tmpl new file mode 100644 index 0000000..ff25707 --- /dev/null +++ b/internal/code-generator/assets/documentation.go.tmpl @@ -0,0 +1,13 @@ +package slinguist + +// CODE GENERATED AUTOMATICALLY WITH github.com/src-d/simple-linguist/cli/slinguist-generate +// THIS FILE SHOULD NOT BE EDITED BY HAND +// Extracted from github/linguist commit: {{ getCommit }} + +import "gopkg.in/toqueteos/substring.v1" + +var documentationMatchers = substring.Or( + {{range $regexp := . -}} + substring.Regexp(`{{ $regexp }}`), + {{end -}} +) diff --git a/internal/code-generator/generator/documentation.go b/internal/code-generator/generator/documentation.go new file mode 100644 index 0000000..b5a4bd9 --- /dev/null +++ b/internal/code-generator/generator/documentation.go @@ -0,0 +1,37 @@ +package generator + +import ( + "bytes" + "html/template" + "io" + + yaml "gopkg.in/yaml.v2" +) + +// Documentation reads from buf and builds documentation_matchers.go file from documentationTmplPath. +func Documentation(data []byte, documentationTmplPath, documentationTmplName, commit string) ([]byte, error) { + var regexpList []string + if err := yaml.Unmarshal(data, ®expList); err != nil { + return nil, err + } + + buf := &bytes.Buffer{} + if err := executeVendorTemplate(buf, regexpList, documentationTmplPath, documentationTmplName, commit); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +func executeDocumentationTemplate(out io.Writer, regexpList []string, documentationTmplPath, documentationTmpl, commit string) error { + fmap := template.FuncMap{ + "getCommit": func() string { return commit }, + } + + t := template.Must(template.New(documentationTmpl).Funcs(fmap).ParseFiles(documentationTmplPath)) + if err := t.Execute(out, regexpList); err != nil { + return err + } + + return nil +} diff --git a/internal/code-generator/generator/generator_test.go b/internal/code-generator/generator/generator_test.go index 81f7560..28634eb 100644 --- a/internal/code-generator/generator/generator_test.go +++ b/internal/code-generator/generator/generator_test.go @@ -10,31 +10,37 @@ import ( ) const ( + commitTest = "fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7" + // FromFile test - formatedLangGold = "test_files/formated_languages.gold" - formatedContentGold = "test_files/formated_content.gold" - formatedVendorGold = "test_files/formated_vendor.gold" + formatedLangGold = "test_files/formated_languages.gold" + formatedContentGold = "test_files/formated_content.gold" + formatedVendorGold = "test_files/formated_vendor.gold" + formatedDocumentationGold = "test_files/formated_documentation.gold" // Languages test ymlTestFile = "test_files/languages.test.yml" langGold = "test_files/languages.gold" languagesTestTmplPath = "test_files/languages.test.tmpl" languagesTestTmplName = "languages.test.tmpl" - commitLangTest = "fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7" // Heuristics test - heuristicsTestFile = "test_files/heuristics.test.rb" - contentGold = "test_files/content.gold" - contentTestTmplPath = "test_files/content.test.go.tmpl" - contentTestTmplName = "content.test.go.tmpl" - commitHeuristicsTest = "fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7" + heuristicsTestFile = "test_files/heuristics.test.rb" + contentGold = "test_files/content.gold" + contentTestTmplPath = "test_files/content.test.go.tmpl" + contentTestTmplName = "content.test.go.tmpl" // Vendor test vendorTestFile = "test_files/vendor.test.yml" vendorGold = "test_files/vendor.gold" vendorTestTmplPath = "test_files/vendor.test.go.tmpl" vendorTestTmplName = "vendor.test.go.tmpl" - commitVendorTest = "fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7" + + // Documentation test + documentationTestFile = "test_files/documentation.test.yml" + documentationGold = "test_files/documentation.gold" + documentationTestTmplPath = "test_files/documentation.test.go.tmpl" + documentationTestTmplName = "documentation.test.go.tmpl" ) func TestFromFile(t *testing.T) { @@ -47,6 +53,9 @@ func TestFromFile(t *testing.T) { goldVendor, err := ioutil.ReadFile(formatedVendorGold) assert.NoError(t, err) + goldDocumentation, err := ioutil.ReadFile(formatedDocumentationGold) + assert.NoError(t, err) + outPathLang, err := ioutil.TempFile("/tmp", "generator-test-") assert.NoError(t, err) defer os.Remove(outPathLang.Name()) @@ -57,7 +66,11 @@ func TestFromFile(t *testing.T) { outPathVendor, err := ioutil.TempFile("/tmp", "generator-test-") assert.NoError(t, err) - defer os.Remove(outPathContent.Name()) + defer os.Remove(outPathVendor.Name()) + + outPathDocumentation, err := ioutil.TempFile("/tmp", "generator-test-") + assert.NoError(t, err) + defer os.Remove(outPathDocumentation.Name()) tests := []struct { name string @@ -75,7 +88,7 @@ func TestFromFile(t *testing.T) { outPath: outPathLang.Name(), tmplPath: languagesTestTmplPath, tmplName: languagesTestTmplName, - commit: commitLangTest, + commit: commitTest, generate: Languages, wantOut: goldLang, }, @@ -85,7 +98,7 @@ func TestFromFile(t *testing.T) { outPath: outPathContent.Name(), tmplPath: contentTestTmplPath, tmplName: contentTestTmplName, - commit: commitHeuristicsTest, + commit: commitTest, generate: Heuristics, wantOut: goldContent, }, @@ -95,10 +108,20 @@ func TestFromFile(t *testing.T) { outPath: outPathVendor.Name(), tmplPath: vendorTestTmplPath, tmplName: vendorTestTmplName, - commit: commitVendorTest, + commit: commitTest, generate: Vendor, wantOut: goldVendor, }, + { + name: "TestFromFile_Documentation", + fileToParse: documentationTestFile, + outPath: outPathDocumentation.Name(), + tmplPath: documentationTestTmplPath, + tmplName: documentationTestTmplName, + commit: commitTest, + generate: Documentation, + wantOut: goldDocumentation, + }, } for _, tt := range tests { @@ -132,7 +155,7 @@ func TestLanguages(t *testing.T) { input: input, tmplPath: languagesTestTmplPath, tmplName: languagesTestTmplName, - commit: commitLangTest, + commit: commitTest, wantOut: gold, }, } @@ -166,7 +189,7 @@ func TestHeuristics(t *testing.T) { input: input, tmplPath: contentTestTmplPath, tmplName: contentTestTmplName, - commit: commitHeuristicsTest, + commit: commitTest, wantOut: gold, }, } @@ -200,7 +223,7 @@ func TestVendor(t *testing.T) { input: input, tmplPath: vendorTestTmplPath, tmplName: vendorTestTmplName, - commit: commitVendorTest, + commit: commitTest, wantOut: gold, }, } @@ -213,3 +236,37 @@ func TestVendor(t *testing.T) { }) } } + +func TestDocumentation(t *testing.T) { + gold, err := ioutil.ReadFile(documentationGold) + assert.NoError(t, err) + + input, err := ioutil.ReadFile(documentationTestFile) + assert.NoError(t, err) + + tests := []struct { + name string + input []byte + tmplPath string + tmplName string + commit string + wantOut []byte + }{ + { + name: "TestDocumentation", + input: input, + tmplPath: documentationTestTmplPath, + tmplName: documentationTestTmplName, + commit: commitTest, + wantOut: gold, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + out, err := Documentation(tt.input, tt.tmplPath, tt.tmplName, tt.commit) + assert.NoError(t, err) + assert.EqualValues(t, tt.wantOut, out, fmt.Sprintf("Documentation() = %v, want %v", string(out), string(tt.wantOut))) + }) + } +} diff --git a/internal/code-generator/generator/test_files/documentation.gold b/internal/code-generator/generator/test_files/documentation.gold new file mode 100644 index 0000000..8a45f29 --- /dev/null +++ b/internal/code-generator/generator/test_files/documentation.gold @@ -0,0 +1,25 @@ +package slinguist + +// CODE GENERATED AUTOMATICALLY WITH github.com/src-d/simple-linguist/cli/slinguist-generate +// THIS FILE SHOULD NOT BE EDITED BY HAND +// Extracted from github/linguist commit: fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7 + +import "gopkg.in/toqueteos/substring.v1" + +var documentationMatchers = substring.Or( + substring.Regexp(`^[Dd]ocs?/`), + substring.Regexp(`(^|/)[Dd]ocumentation/`), + substring.Regexp(`(^|/)[Jj]avadoc/`), + substring.Regexp(`^[Mm]an/`), + substring.Regexp(`^[Ee]xamples/`), + substring.Regexp(`^[Dd]emos?/`), + substring.Regexp(`(^|/)CHANGE(S|LOG)?(\.|$)`), + substring.Regexp(`(^|/)CONTRIBUTING(\.|$)`), + substring.Regexp(`(^|/)COPYING(\.|$)`), + substring.Regexp(`(^|/)INSTALL(\.|$)`), + substring.Regexp(`(^|/)LICEN[CS]E(\.|$)`), + substring.Regexp(`(^|/)[Ll]icen[cs]e(\.|$)`), + substring.Regexp(`(^|/)README(\.|$)`), + substring.Regexp(`(^|/)[Rr]eadme(\.|$)`), + substring.Regexp(`^[Ss]amples?/`), + ) diff --git a/internal/code-generator/generator/test_files/documentation.test.go.tmpl b/internal/code-generator/generator/test_files/documentation.test.go.tmpl new file mode 100644 index 0000000..ff25707 --- /dev/null +++ b/internal/code-generator/generator/test_files/documentation.test.go.tmpl @@ -0,0 +1,13 @@ +package slinguist + +// CODE GENERATED AUTOMATICALLY WITH github.com/src-d/simple-linguist/cli/slinguist-generate +// THIS FILE SHOULD NOT BE EDITED BY HAND +// Extracted from github/linguist commit: {{ getCommit }} + +import "gopkg.in/toqueteos/substring.v1" + +var documentationMatchers = substring.Or( + {{range $regexp := . -}} + substring.Regexp(`{{ $regexp }}`), + {{end -}} +) diff --git a/internal/code-generator/generator/test_files/documentation.test.yml b/internal/code-generator/generator/test_files/documentation.test.yml new file mode 100644 index 0000000..4bd8372 --- /dev/null +++ b/internal/code-generator/generator/test_files/documentation.test.yml @@ -0,0 +1,22 @@ +## Documentation directories ## + +- ^[Dd]ocs?/ +- (^|/)[Dd]ocumentation/ +- (^|/)[Jj]avadoc/ +- ^[Mm]an/ +- ^[Ee]xamples/ +- ^[Dd]emos?/ + +## Documentation files ## + +- (^|/)CHANGE(S|LOG)?(\.|$) +- (^|/)CONTRIBUTING(\.|$) +- (^|/)COPYING(\.|$) +- (^|/)INSTALL(\.|$) +- (^|/)LICEN[CS]E(\.|$) +- (^|/)[Ll]icen[cs]e(\.|$) +- (^|/)README(\.|$) +- (^|/)[Rr]eadme(\.|$) + +# Samples folders +- ^[Ss]amples?/ diff --git a/internal/code-generator/generator/test_files/formated_documentation.gold b/internal/code-generator/generator/test_files/formated_documentation.gold new file mode 100644 index 0000000..26b33e7 --- /dev/null +++ b/internal/code-generator/generator/test_files/formated_documentation.gold @@ -0,0 +1,25 @@ +package slinguist + +// CODE GENERATED AUTOMATICALLY WITH github.com/src-d/simple-linguist/cli/slinguist-generate +// THIS FILE SHOULD NOT BE EDITED BY HAND +// Extracted from github/linguist commit: fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7 + +import "gopkg.in/toqueteos/substring.v1" + +var documentationMatchers = substring.Or( + substring.Regexp(`^[Dd]ocs?/`), + substring.Regexp(`(^|/)[Dd]ocumentation/`), + substring.Regexp(`(^|/)[Jj]avadoc/`), + substring.Regexp(`^[Mm]an/`), + substring.Regexp(`^[Ee]xamples/`), + substring.Regexp(`^[Dd]emos?/`), + substring.Regexp(`(^|/)CHANGE(S|LOG)?(\.|$)`), + substring.Regexp(`(^|/)CONTRIBUTING(\.|$)`), + substring.Regexp(`(^|/)COPYING(\.|$)`), + substring.Regexp(`(^|/)INSTALL(\.|$)`), + substring.Regexp(`(^|/)LICEN[CS]E(\.|$)`), + substring.Regexp(`(^|/)[Ll]icen[cs]e(\.|$)`), + substring.Regexp(`(^|/)README(\.|$)`), + substring.Regexp(`(^|/)[Rr]eadme(\.|$)`), + substring.Regexp(`^[Ss]amples?/`), +) diff --git a/internal/code-generator/main.go b/internal/code-generator/main.go index 41b0052..731a002 100644 --- a/internal/code-generator/main.go +++ b/internal/code-generator/main.go @@ -23,6 +23,11 @@ const ( vendorTmplPath = "internal/code-generator/assets/vendor.go.tmpl" vendorTmpl = "vendor.go.tmpl" + documentationYAML = ".linguist/lib/linguist/documentation.yml" + documentationFile = "documentation_matchers.go" + documentationTmplPath = "internal/code-generator/assets/documentation.go.tmpl" + documentationTmpl = "documentation.go.tmpl" + commitPath = ".git/refs/heads/master" ) @@ -43,6 +48,10 @@ func main() { if err := generator.FromFile(vendorYAML, vendorFile, vendorTmplPath, vendorTmpl, commit, generator.Vendor); err != nil { log.Println(err) } + + if err := generator.FromFile(documentationYAML, documentationFile, documentationTmplPath, documentationTmpl, commit, generator.Documentation); err != nil { + log.Println(err) + } } func getCommit(path string) (string, error) { diff --git a/utils.go b/utils.go index 57e378b..fe57685 100644 --- a/utils.go +++ b/utils.go @@ -52,23 +52,6 @@ func IsBinary(data []byte) bool { return true } -var documentationMatchers = substring.Or( - substring.Regexp(`^docs?/`), - substring.Regexp(`(^|/)[Dd]ocumentation/`), - substring.Regexp(`(^|/)javadoc/`), - substring.Regexp(`^man/`), - substring.Regexp(`^[Ee]xamples/`), - substring.Regexp(`(^|/)CHANGE(S|LOG)?(\.|$)`), - substring.Regexp(`(^|/)CONTRIBUTING(\.|$)`), - substring.Regexp(`(^|/)COPYING(\.|$)`), - substring.Regexp(`(^|/)INSTALL(\.|$)`), - substring.Regexp(`(^|/)LICEN[CS]E(\.|$)`), - substring.Regexp(`(^|/)[Ll]icen[cs]e(\.|$)`), - substring.Regexp(`(^|/)README(\.|$)`), - substring.Regexp(`(^|/)[Rr]eadme(\.|$)`), - substring.Regexp(`^[Ss]amples/`), -) - var configurationLanguages = map[string]bool{ "XML": true, "JSON": true, "TOML": true, "YAML": true, "INI": true, "SQL": true, } diff --git a/utils_test.go b/utils_test.go index 9bc1a45..28ba604 100644 --- a/utils_test.go +++ b/utils_test.go @@ -21,6 +21,13 @@ func (s *TSuite) TestIsVendor(c *C) { func (s *TSuite) TestIsDocumentation(c *C) { c.Assert(IsDocumentation("foo"), Equals, false) c.Assert(IsDocumentation("README"), Equals, true) + c.Assert(IsDocumentation("samples/something"), Equals, true) + c.Assert(IsDocumentation("Docs/whatever"), Equals, true) + c.Assert(IsDocumentation("/javadoc/*"), Equals, true) + c.Assert(IsDocumentation("License"), Equals, true) + c.Assert(IsDocumentation("CONTRIBUTING.bar"), Equals, true) + c.Assert(IsDocumentation("/CHANGES"), Equals, true) + c.Assert(IsDocumentation("INSTALL"), Equals, true) } func (s *TSuite) TestIsConfiguration(c *C) {