Merge pull request #23 from mcarmonaa/filenames

Added language detection by filename strategy

cli/slinguist/main.go

@@ -48,7 +48,7 @@ func main() {
 			return nil
 		}
 
-		l := slinguist.GetLanguage(path, content)
+		l := slinguist.GetLanguage(filepath.Base(path), content)
 
 		r, err := filepath.Rel(root, path)
 		if err != nil {

common.go

@@ -36,6 +36,10 @@ func GetLanguageExtensions(language string) []string {
 
 // GetLanguage return the Language for a given filename and file content.
 func GetLanguage(filename string, content []byte) string {
+	if lang, safe := GetLanguageByFilename(filename); safe {
+		return lang
+	}
+
 	if lang, safe := GetLanguageByShebang(content); safe {
 		return lang
 	}

filename.go

@@ -0,0 +1,10 @@
+package slinguist
+
+func GetLanguageByFilename(filename string) (lang string, safe bool) {
+	lang, safe = languagesByFilename[filename]
+	if lang == "" {
+		lang = OtherLanguage
+	}
+
+	return
+}

filename_test.go

@@ -0,0 +1,37 @@
+package slinguist
+
+import . "gopkg.in/check.v1"
+
+func (s *TSuite) TestGetLanguageByFilename(c *C) {
+	lang, safe := GetLanguageByFilename(`unknown.interpreter`)
+	c.Assert(lang, Equals, OtherLanguage)
+	c.Assert(safe, Equals, false)
+
+	lang, safe = GetLanguageByFilename(`.bashrc`)
+	c.Assert(lang, Equals, "Shell")
+	c.Assert(safe, Equals, true)
+
+	lang, safe = GetLanguageByFilename(`Dockerfile`)
+	c.Assert(lang, Equals, "Dockerfile")
+	c.Assert(safe, Equals, true)
+
+	lang, safe = GetLanguageByFilename(`Makefile.frag`)
+	c.Assert(lang, Equals, "Makefile")
+	c.Assert(safe, Equals, true)
+
+	lang, safe = GetLanguageByFilename(`makefile`)
+	c.Assert(lang, Equals, "Makefile")
+	c.Assert(safe, Equals, true)
+
+	lang, safe = GetLanguageByFilename(`Vagrantfile`)
+	c.Assert(lang, Equals, "Ruby")
+	c.Assert(safe, Equals, true)
+
+	lang, safe = GetLanguageByFilename(`_vimrc`)
+	c.Assert(lang, Equals, "Vim script")
+	c.Assert(safe, Equals, true)
+
+	lang, safe = GetLanguageByFilename(`pom.xml`)
+	c.Assert(lang, Equals, "Maven POM")
+	c.Assert(safe, Equals, true)
+}

filenames_map.go

@@ -0,0 +1,140 @@
+package slinguist
+
+// CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator
+// THIS FILE SHOULD NOT BE EDITED BY HAND
+// Extracted from github/linguist commit: dae33dc2b20cddc85d1300435c3be7118a7115a9
+
+var languagesByFilename = map[string]string{
+	".Rprofile":          "R",
+	".XCompose":          "XCompose",
+	".abbrev_defs":       "Emacs Lisp",
+	".arcconfig":         "JSON",
+	".babelrc":           "JSON5",
+	".bash_history":      "Shell",
+	".bash_logout":       "Shell",
+	".bash_profile":      "Shell",
+	".bashrc":            "Shell",
+	".clang-format":      "YAML",
+	".classpath":         "XML",
+	".emacs":             "Emacs Lisp",
+	".emacs.desktop":     "Emacs Lisp",
+	".factor-boot-rc":    "Factor",
+	".factor-rc":         "Factor",
+	".gclient":           "Python",
+	".gnus":              "Emacs Lisp",
+	".jshintrc":          "JSON",
+	".nvimrc":            "Vim script",
+	".php_cs":            "PHP",
+	".php_cs.dist":       "PHP",
+	".project":           "XML",
+	".pryrc":             "Ruby",
+	".spacemacs":         "Emacs Lisp",
+	".vimrc":             "Vim script",
+	".viper":             "Emacs Lisp",
+	"APKBUILD":           "Alpine Abuild",
+	"App.config":         "XML",
+	"Appraisals":         "Ruby",
+	"BSDmakefile":        "Makefile",
+	"BUCK":               "Python",
+	"BUILD":              "Python",
+	"Berksfile":          "Ruby",
+	"Brewfile":           "Ruby",
+	"Buildfile":          "Ruby",
+	"CMakeLists.txt":     "CMake",
+	"COPYING":            "Text",
+	"COPYRIGHT.regex":    "Text",
+	"Cakefile":           "CoffeeScript",
+	"Cask":               "Emacs Lisp",
+	"Dangerfile":         "Ruby",
+	"Deliverfile":        "Ruby",
+	"Dockerfile":         "Dockerfile",
+	"Emakefile":          "Erlang",
+	"FONTLOG":            "Text",
+	"Fakefile":           "Fancy",
+	"Fastfile":           "Ruby",
+	"GNUmakefile":        "Makefile",
+	"Gemfile":            "Ruby",
+	"Gemfile.lock":       "Ruby",
+	"Guardfile":          "Ruby",
+	"INSTALL":            "Text",
+	"INSTALL.mysql":      "Text",
+	"Jakefile":           "JavaScript",
+	"Jarfile":            "Ruby",
+	"Jenkinsfile":        "Groovy",
+	"Kbuild":             "Makefile",
+	"LICENSE":            "Text",
+	"LICENSE.mysql":      "Text",
+	"Makefile":           "Makefile",
+	"Makefile.am":        "Makefile",
+	"Makefile.boot":      "Makefile",
+	"Makefile.frag":      "Makefile",
+	"Makefile.in":        "Makefile",
+	"Makefile.inc":       "Makefile",
+	"Mavenfile":          "Ruby",
+	"Modulefile":         "Puppet",
+	"NEWS":               "Text",
+	"Notebook":           "Jupyter Notebook",
+	"NuGet.config":       "XML",
+	"Nukefile":           "Nu",
+	"PKGBUILD":           "Shell",
+	"Phakefile":          "PHP",
+	"Podfile":            "Ruby",
+	"Project.ede":        "Emacs Lisp",
+	"Puppetfile":         "Ruby",
+	"README.1ST":         "Text",
+	"README.me":          "Text",
+	"README.mysql":       "Text",
+	"ROOT":               "Isabelle ROOT",
+	"Rexfile":            "Perl6",
+	"SConscript":         "Python",
+	"SConstruct":         "Python",
+	"Settings.StyleCop":  "XML",
+	"Slakefile":          "LiveScript",
+	"Snakefile":          "Python",
+	"Snapfile":           "Ruby",
+	"Thorfile":           "Ruby",
+	"Vagrantfile":        "Ruby",
+	"WORKSPACE":          "Python",
+	"Web.Debug.config":   "XML",
+	"Web.Release.config": "XML",
+	"Web.config":         "XML",
+	"XCompose":           "XCompose",
+	"_emacs":             "Emacs Lisp",
+	"_vimrc":             "Vim script",
+	"abbrev_defs":        "Emacs Lisp",
+	"ant.xml":            "Ant Build System",
+	"build.xml":          "Ant Build System",
+	"buildfile":          "Ruby",
+	"click.me":           "Text",
+	"composer.lock":      "JSON",
+	"configure.ac":       "M4Sugar",
+	"delete.me":          "Text",
+	"descrip.mmk":        "Module Management System",
+	"descrip.mms":        "Module Management System",
+	"gradlew":            "Shell",
+	"gvimrc":             "Vim script",
+	"keep.me":            "Text",
+	"ld.script":          "Linker Script",
+	"makefile":           "Makefile",
+	"makefile.sco":       "Makefile",
+	"mcmod.info":         "JSON",
+	"meson.build":        "Meson",
+	"meson_options.txt":  "Meson",
+	"mix.lock":           "Elixir",
+	"mkfile":             "Makefile",
+	"mmn":                "Roff",
+	"mmt":                "Roff",
+	"nginx.conf":         "Nginx",
+	"nvimrc":             "Vim script",
+	"packages.config":    "XML",
+	"pom.xml":            "Maven POM",
+	"read.me":            "Text",
+	"rebar.config":       "Erlang",
+	"rebar.config.lock":  "Erlang",
+	"rebar.lock":         "Erlang",
+	"riemann.config":     "Clojure",
+	"test.me":            "Text",
+	"vimrc":              "Vim script",
+	"wscript":            "Python",
+	"xcompose":           "XCompose",
+}

internal/code-generator/assets/filenames.go.tmpl

@@ -4,10 +4,8 @@ package slinguist
 // THIS FILE SHOULD NOT BE EDITED BY HAND
 // Extracted from github/linguist commit: {{ getCommit }}
 
-import "gopkg.in/toqueteos/substring.v1"
+var languagesByFilename = map[string]string{
-
+	{{range $filename, $language := . -}}
-var vendorMatchers = substring.Or(
+	"{{ $filename }}":	{{- printf "%q" $language -}},
-	{{range $regexp := . -}}
-	substring.Regexp(`{{ $regexp }}`),
 	{{end -}}
-)
+}

internal/code-generator/generator/documentation.go

@@ -16,7 +16,7 @@ func Documentation(data []byte, documentationTmplPath, documentationTmplName, co
 	}
 
 	buf := &bytes.Buffer{}
-	if err := executeVendorTemplate(buf, regexpList, documentationTmplPath, documentationTmplName, commit); err != nil {
+	if err := executeDocumentationTemplate(buf, regexpList, documentationTmplPath, documentationTmplName, commit); err != nil {
 		return nil, err
 	}
 

internal/code-generator/generator/extensions.go

@@ -3,21 +3,14 @@ package generator
 import (
 	"bytes"
 	"io"
-	"sort"
 	"strings"
 	"text/template"
 
-	"gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v2"
 )
 
-type languageInfo struct {
+// Extensions reads from buf and builds extensions_map.go file from extensionsTmplPath.
-	Type         string   `yaml:"type,omitempty"`
+func Extensions(data []byte, extensionsTmplPath, extensionsTmplName, commit string) ([]byte, error) {
-	Extensions   []string `yaml:"extensions,omitempty,flow"`
-	Interpreters []string `yaml:"interpreters,omitempty,flow"`
-}
-
-// Languages reads from buf and builds languages.go file from languagesTmplPath.
-func Languages(data []byte, languagesTmplPath, languagesTmplName, commit string) ([]byte, error) {
 	languages := make(map[string]*languageInfo)
 	if err := yaml.Unmarshal(data, &languages); err != nil {
 		return nil, err
@@ -27,23 +20,13 @@ func Languages(data []byte, languagesTmplPath, languagesTmplName, commit string)
 	languagesByExtension := buildExtensionLanguageMap(languages, orderedKeyList)
 
 	buf := &bytes.Buffer{}
-	if err := executeLanguagesTemplate(buf, languagesByExtension, languagesTmplPath, languagesTmplName, commit); err != nil {
+	if err := executeExtensionsTemplate(buf, languagesByExtension, extensionsTmplPath, extensionsTmplName, commit); err != nil {
 		return nil, err
 	}
 
 	return buf.Bytes(), nil
 }
 
-func getAlphabeticalOrderedKeys(languages map[string]*languageInfo) []string {
-	keyList := make([]string, 0)
-	for lang := range languages {
-		keyList = append(keyList, lang)
-	}
-
-	sort.Strings(keyList)
-	return keyList
-}
-
 func buildExtensionLanguageMap(languages map[string]*languageInfo, orderedKeyList []string) map[string][]string {
 	extensionLangsMap := make(map[string][]string)
 	for _, lang := range orderedKeyList {
@@ -56,13 +39,13 @@ func buildExtensionLanguageMap(languages map[string]*languageInfo, orderedKeyLis
 	return extensionLangsMap
 }
 
-func executeLanguagesTemplate(out io.Writer, languagesByExtension map[string][]string, languagesTmplPath, languagesTmpl, commit string) error {
+func executeExtensionsTemplate(out io.Writer, languagesByExtension map[string][]string, extensionsTmplPath, extensionsTmpl, commit string) error {
 	fmap := template.FuncMap{
 		"getCommit":         func() string { return commit },
 		"formatStringSlice": func(slice []string) string { return `"` + strings.Join(slice, `","`) + `"` },
 	}
 
-	t := template.Must(template.New(languagesTmpl).Funcs(fmap).ParseFiles(languagesTmplPath))
+	t := template.Must(template.New(extensionsTmpl).Funcs(fmap).ParseFiles(extensionsTmplPath))
 	if err := t.Execute(out, languagesByExtension); err != nil {
 		return err
 	}

internal/code-generator/generator/filenames.go

@@ -0,0 +1,50 @@
+package generator
+
+import (
+	"bytes"
+	"io"
+	"text/template"
+
+	yaml "gopkg.in/yaml.v2"
+)
+
+// Filenames reads from buf and builds filenames_map.go file from filenamesTmplPath.
+func Filenames(data []byte, filenamesTmplPath, filenamesTmplName, commit string) ([]byte, error) {
+	languages := make(map[string]*languageInfo)
+	if err := yaml.Unmarshal(data, &languages); err != nil {
+		return nil, err
+	}
+
+	languagesByFilename := buildFilenameLanguageMap(languages)
+
+	buf := &bytes.Buffer{}
+	if err := executeFilenamesTemplate(buf, languagesByFilename, filenamesTmplPath, filenamesTmplName, commit); err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}
+
+func buildFilenameLanguageMap(languages map[string]*languageInfo) map[string]string {
+	filenameLangMap := make(map[string]string)
+	for lang, langInfo := range languages {
+		for _, filename := range langInfo.Filenames {
+			filenameLangMap[filename] = lang
+		}
+	}
+
+	return filenameLangMap
+}
+
+func executeFilenamesTemplate(out io.Writer, languagesByFilename map[string]string, filenamesTmplPath, filenamesTmpl, commit string) error {
+	fmap := template.FuncMap{
+		"getCommit": func() string { return commit },
+	}
+
+	t := template.Must(template.New(filenamesTmpl).Funcs(fmap).ParseFiles(filenamesTmplPath))
+	if err := t.Execute(out, languagesByFilename); err != nil {
+		return err
+	}
+
+	return nil
+}

internal/code-generator/generator/generator_test.go

@@ -13,164 +13,137 @@ const (
 	commitTest = "fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7"
 
 	// Languages test
-	ymlTestFile           = "test_files/languages.test.yml"
+	extensionsTestFile     = "test_files/extensions.test.yml"
-	langGold              = "test_files/languages.gold"
+	extensionsGold         = "test_files/extensions.gold"
-	languagesTestTmplPath = "test_files/languages.test.go.tmpl"
+	extensionsTestTmplPath = "../assets/extensions.go.tmpl"
-	languagesTestTmplName = "languages.test.go.tmpl"
+	extensionsTestTmplName = "extensions.go.tmpl"
 
 	// Heuristics test
 	heuristicsTestFile  = "test_files/heuristics.test.rb"
 	contentGold         = "test_files/content.gold"
-	contentTestTmplPath = "test_files/content.test.go.tmpl"
+	contentTestTmplPath = "../assets/content.go.tmpl"
-	contentTestTmplName = "content.test.go.tmpl"
+	contentTestTmplName = "content.go.tmpl"
 
 	// Vendor test
 	vendorTestFile     = "test_files/vendor.test.yml"
 	vendorGold         = "test_files/vendor.gold"
-	vendorTestTmplPath = "test_files/vendor.test.go.tmpl"
+	vendorTestTmplPath = "../assets/vendor.go.tmpl"
-	vendorTestTmplName = "vendor.test.go.tmpl"
+	vendorTestTmplName = "vendor.go.tmpl"
 
 	// Documentation test
 	documentationTestFile     = "test_files/documentation.test.yml"
 	documentationGold         = "test_files/documentation.gold"
-	documentationTestTmplPath = "test_files/documentation.test.go.tmpl"
+	documentationTestTmplPath = "../assets/documentation.go.tmpl"
-	documentationTestTmplName = "documentation.test.go.tmpl"
+	documentationTestTmplName = "documentation.go.tmpl"
 
 	// Types test
 	typesTestFile     = "test_files/type.test.yml"
 	typesGold         = "test_files/type.gold"
-	typesTestTmplPath = "test_files/type.test.go.tmpl"
+	typesTestTmplPath = "../assets/type.go.tmpl"
-	typesTestTmplName = "type.test.go.tmpl"
+	typesTestTmplName = "type.go.tmpl"
 
 	// Interpreters test
 	interpretersTestFile     = "test_files/interpreters.test.yml"
 	interpretersGold         = "test_files/interpreters.gold"
-	interpretersTestTmplPath = "test_files/interpreters.test.go.tmpl"
+	interpretersTestTmplPath = "../assets/interpreters.go.tmpl"
-	interpretersTestTmplName = "interpreters.test.go.tmpl"
+	interpretersTestTmplName = "interpreters.go.tmpl"
+
+	// Filenames test
+	filenamesTestFile     = "test_files/filenames.test.yml"
+	filenamesGold         = "test_files/filenames.gold"
+	filenamesTestTmplPath = "../assets/filenames.go.tmpl"
+	filenamesTestTmplName = "filenames.go.tmpl"
 )
 
 func TestFromFile(t *testing.T) {
-	goldLang, err := ioutil.ReadFile(langGold)
-	assert.NoError(t, err)
-
-	goldContent, err := ioutil.ReadFile(contentGold)
-	assert.NoError(t, err)
-
-	goldVendor, err := ioutil.ReadFile(vendorGold)
-	assert.NoError(t, err)
-
-	goldDocumentation, err := ioutil.ReadFile(documentationGold)
-	assert.NoError(t, err)
-
-	goldTypes, err := ioutil.ReadFile(typesGold)
-	assert.NoError(t, err)
-
-	goldInterpreters, err := ioutil.ReadFile(interpretersGold)
-	assert.NoError(t, err)
-
-	outPathLang, err := ioutil.TempFile("/tmp", "generator-test-")
-	assert.NoError(t, err)
-	defer os.Remove(outPathLang.Name())
-
-	outPathContent, err := ioutil.TempFile("/tmp", "generator-test-")
-	assert.NoError(t, err)
-	defer os.Remove(outPathContent.Name())
-
-	outPathVendor, err := ioutil.TempFile("/tmp", "generator-test-")
-	assert.NoError(t, err)
-	defer os.Remove(outPathVendor.Name())
-
-	outPathDocumentation, err := ioutil.TempFile("/tmp", "generator-test-")
-	assert.NoError(t, err)
-	defer os.Remove(outPathDocumentation.Name())
-
-	outPathTypes, err := ioutil.TempFile("/tmp", "generator-test-")
-	assert.NoError(t, err)
-	defer os.Remove(outPathTypes.Name())
-
-	outPathInterpreters, err := ioutil.TempFile("/tmp", "generator-test-")
-	assert.NoError(t, err)
-	defer os.Remove(outPathInterpreters.Name())
-
 	tests := []struct {
 		name        string
 		fileToParse string
-		outPath     string
 		tmplPath    string
 		tmplName    string
 		commit      string
 		generate    Func
-		wantOut     []byte
+		wantOut     string
 	}{
 		{
 			name:        "TestFromFile_Language",
-			fileToParse: ymlTestFile,
+			fileToParse: extensionsTestFile,
-			outPath:     outPathLang.Name(),
+			tmplPath:    extensionsTestTmplPath,
-			tmplPath:    languagesTestTmplPath,
+			tmplName:    extensionsTestTmplName,
-			tmplName:    languagesTestTmplName,
 			commit:      commitTest,
-			generate:    Languages,
+			generate:    Extensions,
-			wantOut:     goldLang,
+			wantOut:     extensionsGold,
 		},
 		{
 			name:        "TestFromFile_Heuristics",
 			fileToParse: heuristicsTestFile,
-			outPath:     outPathContent.Name(),
 			tmplPath:    contentTestTmplPath,
 			tmplName:    contentTestTmplName,
 			commit:      commitTest,
 			generate:    Heuristics,
-			wantOut:     goldContent,
+			wantOut:     contentGold,
 		},
 		{
 			name:        "TestFromFile_Vendor",
 			fileToParse: vendorTestFile,
-			outPath:     outPathVendor.Name(),
 			tmplPath:    vendorTestTmplPath,
 			tmplName:    vendorTestTmplName,
 			commit:      commitTest,
 			generate:    Vendor,
-			wantOut:     goldVendor,
+			wantOut:     vendorGold,
 		},
 		{
 			name:        "TestFromFile_Documentation",
 			fileToParse: documentationTestFile,
-			outPath:     outPathDocumentation.Name(),
 			tmplPath:    documentationTestTmplPath,
 			tmplName:    documentationTestTmplName,
 			commit:      commitTest,
 			generate:    Documentation,
-			wantOut:     goldDocumentation,
+			wantOut:     documentationGold,
 		},
 		{
 			name:        "TestFromFile_Types",
 			fileToParse: typesTestFile,
-			outPath:     outPathTypes.Name(),
 			tmplPath:    typesTestTmplPath,
 			tmplName:    typesTestTmplName,
 			commit:      commitTest,
 			generate:    Types,
-			wantOut:     goldTypes,
+			wantOut:     typesGold,
 		},
 		{
 			name:        "TestFromFile_Interpreters",
 			fileToParse: interpretersTestFile,
-			outPath:     outPathInterpreters.Name(),
 			tmplPath:    interpretersTestTmplPath,
 			tmplName:    interpretersTestTmplName,
 			commit:      commitTest,
 			generate:    Interpreters,
-			wantOut:     goldInterpreters,
+			wantOut:     interpretersGold,
+		},
+		{
+			name:        "TestFromFile_Filenames",
+			fileToParse: filenamesTestFile,
+			tmplPath:    filenamesTestTmplPath,
+			tmplName:    filenamesTestTmplName,
+			commit:      commitTest,
+			generate:    Filenames,
+			wantOut:     filenamesGold,
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			err := FromFile(tt.fileToParse, tt.outPath, tt.tmplPath, tt.tmplName, tt.commit, tt.generate)
+			gold, err := ioutil.ReadFile(tt.wantOut)
 			assert.NoError(t, err)
-			out, err := ioutil.ReadFile(tt.outPath)
+
+			outPath, err := ioutil.TempFile("/tmp", "generator-test-")
 			assert.NoError(t, err)
-			assert.EqualValues(t, tt.wantOut, out, fmt.Sprintf("FromFile() = %v, want %v", string(out), string(tt.wantOut)))
+			defer os.Remove(outPath.Name())
+
+			err = FromFile(tt.fileToParse, outPath.Name(), tt.tmplPath, tt.tmplName, tt.commit, tt.generate)
+			assert.NoError(t, err)
+			out, err := ioutil.ReadFile(outPath.Name())
+			assert.NoError(t, err)
+			assert.EqualValues(t, gold, out, fmt.Sprintf("FromFile() = %v, want %v", string(out), string(tt.wantOut)))
 		})
 	}
 }

internal/code-generator/generator/heuristics.go

@@ -299,7 +299,7 @@ func getHeuristics(line string) []*heuristic {
 		}
 
 		if reg != "" {
-			reg = convToValidRegexp(reg)
+			reg = convertToValidRegexp(reg)
 			heuristics = append(heuristics, &heuristic{Regexp: reg})
 		}
 	}
@@ -327,7 +327,7 @@ func replaceRegexpVariables(reg string) string {
 	return repl
 }
 
-func convToValidRegexp(reg string) string {
+func convertToValidRegexp(reg string) string {
 	// example: `/^(\s*)(<Project|<Import|<Property|<?xml|xmlns)/i``
 	// Ruby modifier "m" matches multiple lines, recognizing newlines as normal characters, Go use flag "s" for that.
 	const (

internal/code-generator/generator/langinfo.go

@@ -0,0 +1,20 @@
+package generator
+
+import "sort"
+
+type languageInfo struct {
+	Type         string   `yaml:"type,omitempty"`
+	Extensions   []string `yaml:"extensions,omitempty,flow"`
+	Interpreters []string `yaml:"interpreters,omitempty,flow"`
+	Filenames    []string `yaml:"filenames,omitempty,flow"`
+}
+
+func getAlphabeticalOrderedKeys(languages map[string]*languageInfo) []string {
+	keyList := make([]string, 0)
+	for lang := range languages {
+		keyList = append(keyList, lang)
+	}
+
+	sort.Strings(keyList)
+	return keyList
+}

internal/code-generator/generator/test_files/content.gold

@@ -34,6 +34,53 @@ var matchers = map[string]languageMatcher{
 
 		return OtherLanguage, false
 	},
+	".f": func(i []byte) (string, bool) {
+		if f_Forth_Matcher_0.Match(i) {
+			return "Forth", true
+		} else if f_FilebenchWML_Matcher_0.Match(i) {
+			return "Filebench WML", true
+		} else if f_FORTRAN_Matcher_0.Match(i) {
+			return "FORTRAN", true
+		}
+
+		return OtherLanguage, false
+	},
+	".h": func(i []byte) (string, bool) {
+		if h_ObjectiveDashC_Matcher_0.Match(i) {
+			return "Objective-C", true
+		} else if h_CPlusPlus_Matcher_0.Match(i) || h_CPlusPlus_Matcher_1.Match(i) || h_CPlusPlus_Matcher_2.Match(i) || h_CPlusPlus_Matcher_3.Match(i) || h_CPlusPlus_Matcher_4.Match(i) || h_CPlusPlus_Matcher_5.Match(i) || h_CPlusPlus_Matcher_6.Match(i) {
+			return "C++", true
+		}
+
+		return OtherLanguage, false
+	},
+	".lsp": func(i []byte) (string, bool) {
+		if lsp_CommonLisp_Matcher_0.Match(i) {
+			return "Common Lisp", true
+		} else if lsp_NewLisp_Matcher_0.Match(i) {
+			return "NewLisp", true
+		}
+
+		return OtherLanguage, false
+	},
+	".lisp": func(i []byte) (string, bool) {
+		if lisp_CommonLisp_Matcher_0.Match(i) {
+			return "Common Lisp", true
+		} else if lisp_NewLisp_Matcher_0.Match(i) {
+			return "NewLisp", true
+		}
+
+		return OtherLanguage, false
+	},
+	".md": func(i []byte) (string, bool) {
+		if md_Markdown_Matcher_0.Match(i) || md_Markdown_Matcher_1.Match(i) {
+			return "Markdown", true
+		} else if md_GCCmachinedescription_Matcher_0.Match(i) {
+			return "GCC machine description", true
+		}
+
+		return "Markdown", true
+	},
 	".ms": func(i []byte) (string, bool) {
 		if ms_Groff_Matcher_0.Match(i) {
 			return "Groff", true
@@ -63,12 +110,37 @@ var matchers = map[string]languageMatcher{
 
 		return OtherLanguage, false
 	},
+	".rpy": func(i []byte) (string, bool) {
+		if rpy_Python_Matcher_0.Match(i) {
+			return "Python", true
+		}
+
+		return "Ren'Py", true
+	},
 }
 
 var (
 	asc_PublicKey_Matcher_0            = regexp.MustCompile(`(?m)^(----[- ]BEGIN|ssh-(rsa|dss)) `)
 	asc_AsciiDoc_Matcher_0             = regexp.MustCompile(`(?m)^[=-]+(\s|\n)|{{[A-Za-z]`)
 	asc_AGSScript_Matcher_0            = regexp.MustCompile(`(?m)^(\/\/.+|((import|export)\s+)?(function|int|float|char)\s+((room|repeatedly|on|game)_)?([A-Za-z]+[A-Za-z_0-9]+)\s*[;\(])`)
+	f_Forth_Matcher_0                  = regexp.MustCompile(`(?m)^: `)
+	f_FilebenchWML_Matcher_0           = regexp.MustCompile(`(?m)flowop`)
+	f_FORTRAN_Matcher_0                = regexp.MustCompile(`(?mi)^([c*][^abd-z]|      (subroutine|program|end|data)\s|\s*!)`)
+	h_ObjectiveDashC_Matcher_0         = regexp.MustCompile(`(?m)^\s*(@(interface|class|protocol|property|end|synchronised|selector|implementation)\b|#import\s+.+\.h[">])`)
+	h_CPlusPlus_Matcher_0              = regexp.MustCompile(`(?m)^\s*#\s*include <(cstdint|string|vector|map|list|array|bitset|queue|stack|forward_list|unordered_map|unordered_set|(i|o|io)stream)>`)
+	h_CPlusPlus_Matcher_1              = regexp.MustCompile(`(?m)^\s*template\s*<`)
+	h_CPlusPlus_Matcher_2              = regexp.MustCompile(`(?m)^[ \t]*try`)
+	h_CPlusPlus_Matcher_3              = regexp.MustCompile(`(?m)^[ \t]*catch\s*\(`)
+	h_CPlusPlus_Matcher_4              = regexp.MustCompile(`(?m)^[ \t]*(class|(using[ \t]+)?namespace)\s+\w+`)
+	h_CPlusPlus_Matcher_5              = regexp.MustCompile(`(?m)^[ \t]*(private|public|protected):$`)
+	h_CPlusPlus_Matcher_6              = regexp.MustCompile(`(?m)std::\w+`)
+	lsp_CommonLisp_Matcher_0           = regexp.MustCompile(`(?mi)^\s*\((defun|in-package|defpackage) `)
+	lsp_NewLisp_Matcher_0              = regexp.MustCompile(`(?m)^\s*\(define `)
+	lisp_CommonLisp_Matcher_0          = regexp.MustCompile(`(?mi)^\s*\((defun|in-package|defpackage) `)
+	lisp_NewLisp_Matcher_0             = regexp.MustCompile(`(?m)^\s*\(define `)
+	md_Markdown_Matcher_0              = regexp.MustCompile(`(?mi)(^[-a-z0-9=#!\*\[|>])|<\/`)
+	md_Markdown_Matcher_1              = regexp.MustCompile(`(?m)^$`)
+	md_GCCmachinedescription_Matcher_0 = regexp.MustCompile(`(?m)^(;;|\(define_)`)
 	ms_Groff_Matcher_0                 = regexp.MustCompile(`(?mi)^[.'][a-z][a-z](\s|$)`)
 	mod_XML_Matcher_0                  = regexp.MustCompile(`(?m)<!ENTITY `)
 	mod_ModulaDash2_Matcher_0          = regexp.MustCompile(`(?mi)^\s*MODULE [\w\.]+;`)
@@ -78,4 +150,5 @@ var (
 	pro_QMake_Matcher_0                = regexp.MustCompile(`(?m)HEADERS`)
 	pro_QMake_Matcher_1                = regexp.MustCompile(`(?m)SOURCES`)
 	pro_IDL_Matcher_0                  = regexp.MustCompile(`(?m)^\s*function[ \w,]+$`)
+	rpy_Python_Matcher_0               = regexp.MustCompile(`(?ms)(^(import|from|class|def)\s)`)
 )

internal/code-generator/generator/test_files/content.test.go.tmpl

@@ -1,51 +0,0 @@
-package slinguist
-
-// CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator
-// THIS FILE SHOULD NOT BE EDITED BY HAND
-// Extracted from github/linguist commit: {{ getCommit }}
-
-import (
-	"path/filepath"
-	"regexp"
-	"strings"
-)
-
-func GetLanguageByContent(filename string, content []byte) (lang string, safe bool) {
-	ext := strings.ToLower(filepath.Ext(filename))
-	if fnMatcher, ok := matchers[ext]; ok {
-		lang, safe = fnMatcher(content)
-		return
-	}
-
-	return GetLanguageByExtension(filename)
-}
-
-type languageMatcher func ([]byte) (string, bool)
-
-var matchers = map[string]languageMatcher{
-	{{ range $index, $disambiguator := . -}}
-	{{ printf "%q" $disambiguator.Extension }}: func(i []byte) (string, bool) {
-		{{ range $i, $language := $disambiguator.Languages -}}
-
-		{{- if not (avoidLanguage $language) }}
-		{{- if gt (len $language.Heuristics) 0 }}
-		{{- if gt $i 0 }} else {{ end -}} 
-		if {{- range $j, $heuristic := $language.Heuristics }} {{ $heuristic.Name }}.Match(i) 
-		{{- if lt $j (len $language.LogicRelations) }} {{index $language.LogicRelations $j}} {{- end -}} {{ end }} {
-			return {{ printf "%q" $language.Language }}, true
-		}
-		
-		{{- end -}}
-		{{- end -}}
-		{{- end}}
-	
-		return {{ returnLanguage $disambiguator.Languages }}, {{ safeLanguage $disambiguator.Languages }}
-	},
-	{{ end -}}
-}
-
-var (
-    {{ range $index, $heuristic := getAllHeuristics . -}}
-    {{ $heuristic.Name }} = regexp.MustCompile(`{{ $heuristic.Regexp }}`)
-    {{ end -}}
-)

internal/code-generator/generator/test_files/documentation.test.go.tmpl

@@ -1,13 +0,0 @@
-package slinguist
-
-// CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator
-// THIS FILE SHOULD NOT BE EDITED BY HAND
-// Extracted from github/linguist commit: {{ getCommit }}
-
-import "gopkg.in/toqueteos/substring.v1"
-
-var documentationMatchers = substring.Or(
-	{{range $regexp := . -}}
-	substring.Regexp(`{{ $regexp }}`),
-	{{end -}}
-)

internal/code-generator/generator/test_files/filenames.gold

@@ -0,0 +1,12 @@
+package slinguist
+
+// CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator
+// THIS FILE SHOULD NOT BE EDITED BY HAND
+// Extracted from github/linguist commit: fe8b44ab8a225b1ffa75b983b916ea22fee5b6f7
+
+var languagesByFilename = map[string]string{
+	"APKBUILD":       "Alpine Abuild",
+	"CMakeLists.txt": "CMake",
+	"Cakefile":       "CoffeeScript",
+	"mix.lock":       "Elixir",
+}

internal/code-generator/generator/test_files/filenames.test.yml

@@ -0,0 +1,13 @@
+---
+Alpine Abuild:
+  filenames:
+  - APKBUILD
+CMake:
+  filenames:
+  - CMakeLists.txt
+CoffeeScript:
+  filenames:
+  - Cakefile
+Elixir:
+  filenames:
+  - mix.lock

internal/code-generator/generator/test_files/heuristics.test.rb

@@ -11,6 +11,45 @@
       end
     end
 
+    fortran_rx = /^([c*][^abd-z]|      (subroutine|program|end|data)\s|\s*!)/i
+
+    disambiguate ".f" do |data|
+      if /^: /.match(data)
+        Language["Forth"]
+      elsif data.include?("flowop")
+        Language["Filebench WML"]
+      elsif fortran_rx.match(data)
+        Language["FORTRAN"]
+      end
+    end
+
+    disambiguate ".h" do |data|
+      if ObjectiveCRegex.match(data)
+        Language["Objective-C"]
+      elsif (/^\s*#\s*include <(cstdint|string|vector|map|list|array|bitset|queue|stack|forward_list|unordered_map|unordered_set|(i|o|io)stream)>/.match(data) ||
+        /^\s*template\s*</.match(data) || /^[ \t]*try/.match(data) || /^[ \t]*catch\s*\(/.match(data) || /^[ \t]*(class|(using[ \t]+)?namespace)\s+\w+/.match(data) || /^[ \t]*(private|public|protected):$/.match(data) || /std::\w+/.match(data))
+        Language["C++"]
+      end
+    end
+
+    disambiguate ".lsp", ".lisp" do |data|
+      if /^\s*\((defun|in-package|defpackage) /i.match(data)
+        Language["Common Lisp"]
+      elsif /^\s*\(define /.match(data)
+        Language["NewLisp"]
+      end
+    end
+
+    disambiguate ".md" do |data|
+      if /(^[-a-z0-9=#!\*\[|>])|<\//i.match(data) || data.empty?
+        Language["Markdown"]
+      elsif /^(;;|\(define_)/.match(data)
+        Language["GCC machine description"]
+      else
+        Language["Markdown"]
+      end
+    end
+
     disambiguate ".ms" do |data|
       if /^[.'][a-z][a-z](\s|$)/i.match(data)
         Language["Groff"]
@@ -42,3 +81,11 @@
         Language["IDL"]
       end
     end
+
+    disambiguate ".rpy" do |data|
+      if /(^(import|from|class|def)\s)/m.match(data)
+        Language["Python"]
+      else
+        Language["Ren'Py"]
+      end
+    end

internal/code-generator/generator/test_files/interpreters.test.go.tmpl

@@ -1,11 +0,0 @@
-package slinguist
-
-// CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator
-// THIS FILE SHOULD NOT BE EDITED BY HAND
-// Extracted from github/linguist commit: {{ getCommit }}
-
-var languagesByInterpreter = map[string][]string{
-	{{range $interpreter, $languages := . -}}
-	"{{ $interpreter }}":	{ {{- $languages | formatStringSlice -}} },
-	{{end -}}
-}

internal/code-generator/generator/test_files/languages.test.go.tmpl

@@ -1,11 +0,0 @@
-package slinguist
-
-// CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/simple-linguist.v1/internal/code-generator
-// THIS FILE SHOULD NOT BE EDITED BY HAND
-// Extracted from github/linguist commit: {{ getCommit }}
-
-var languagesByExtension = map[string][]string{
-	{{range $extension, $languages := . -}}
-	"{{ $extension }}":	{ {{- $languages | formatStringSlice -}} },
-	{{end -}}
-}

internal/code-generator/main.go

@@ -8,11 +8,13 @@ import (
 )
 
 const (
-	// languages.go generation
+	// languages info file
 	languagesYAML = ".linguist/lib/linguist/languages.yml"
-	langFile          = "languages.go"
+
-	languagesTmplPath = "internal/code-generator/assets/languages.go.tmpl"
+	// extensions_map.go generation
-	languagesTmpl     = "languages.go.tmpl"
+	extensionsFile     = "extensions_map.go"
+	extensionsTmplPath = "internal/code-generator/assets/extensions.go.tmpl"
+	extensionsTmpl     = "extensions.go.tmpl"
 
 	// content.go generation
 	heuristicsRuby  = ".linguist/lib/linguist/heuristics.rb"
@@ -42,6 +44,11 @@ const (
 	interpretersTmplPath = "internal/code-generator/assets/interpreters.go.tmpl"
 	interpretersTmpl     = "interpreters.go.tmpl"
 
+	// filenames_map.go generation
+	filenamesFile     = "filenames_map.go"
+	filenamesTmplPath = "internal/code-generator/assets/filenames.go.tmpl"
+	filenamesTmpl     = "filenames.go.tmpl"
+
 	commitPath = ".git/refs/heads/master"
 )
 
@@ -61,12 +68,13 @@ func main() {
 	}
 
 	argsList := []*generatorArgs{
-		&generatorArgs{languagesYAML, langFile, languagesTmplPath, languagesTmpl, commit, generator.Languages},
+		&generatorArgs{languagesYAML, extensionsFile, extensionsTmplPath, extensionsTmpl, commit, generator.Extensions},
 		&generatorArgs{heuristicsRuby, contentFile, contentTmplPath, contentTmpl, commit, generator.Heuristics},
 		&generatorArgs{vendorYAML, vendorFile, vendorTmplPath, vendorTmpl, commit, generator.Vendor},
 		&generatorArgs{documentationYAML, documentationFile, documentationTmplPath, documentationTmpl, commit, generator.Documentation},
 		&generatorArgs{languagesYAML, typeFile, typeTmplPath, typeTmpl, commit, generator.Types},
 		&generatorArgs{languagesYAML, interpretersFile, interpretersTmplPath, interpretersTmpl, commit, generator.Interpreters},
+		&generatorArgs{languagesYAML, filenamesFile, filenamesTmplPath, filenamesTmpl, commit, generator.Filenames},
 	}
 
 	for _, args := range argsList {

shebang.go

@@ -7,6 +7,8 @@ import (
 	"strings"
 )
 
+const shebang = `#!`
+
 var (
 	shebangExecHack = regexp.MustCompile(`exec (\w+).+\$0.+\$@`)
 	pythonVersion   = regexp.MustCompile(`python\d\.\d+`)
@@ -68,7 +70,7 @@ func getFirstLine(data []byte) []byte {
 }
 
 func hasShebang(line []byte) bool {
-	shebang := []byte{'#', '!'}
+	shebang := []byte(shebang)
 	return bytes.HasPrefix(line, shebang)
 }